Why custom RSA engine is not calling default RSA method?

[Shariful Alam]

Hello,
I have the following simple RSA engine code from *e_dasync.c. * Following
code compiles and works. Until now, I was under the impression that if I do
not supply an alternative method in function *bind_dasync(), the *engine
will use the default method. However, it doesn't seem to be the case. If I
comment out line 37 and try to perform an encryption operation with the
following command,

*"openssl rsautl -encrypt -inkey public.pem -pubin -in msg.txt -out msg.enc
-engine rsa-engine-new" *

I get segmentation errors.

Can anyone please tell me why this is happening?

==============================================================================

   1. /* Engine Id and Name */
   2. static const char *engine_rsa_id = "rsa-engine-new";
   3. static const char *engine_rsa_name = "RSA engine for testing";
   4.
   5. // data encryption function
   6. static int eng_rsa_pub_enc(int flen, const unsigned char *from,
   7.                           unsigned char *to, RSA *rsa, int padding) {
   8.     printf("Encryption\n");
   9.     return 0;
   10. }
   11.
   12. // signature verify
   13. static int eng_rsa_pub_dec(int flen, const unsigned char *from,
   unsigned char *to, RSA *rsa, int padding){
   14.     printf("Signature verify:\n");
   15.     return 0;
   16. }
   17.
   18. // signature
   19. static int eng_rsa_priv_enc(int flen, const unsigned char *from,
   unsigned char *to, RSA *rsa, int padding){
   20.     printf("Signature:\n");
   21.     return 0;
   22. }
   23.
   24. // data decryption
   25. static int eng_rsa_priv_dec(int flen, const unsigned char *from,
   unsigned char *to, RSA *rsa, int padding){
   26.     printf("Decryption\n");
   27.     return 0;
   28. }
   29.
   30.
   31. static RSA_METHOD *test_rsa_method = NULL;
   32.
   33.
   34. static int bind_dasync(ENGINE *e){
   35.     /* Setup RSA_METHOD */
   36.     if ((test_rsa_method = RSA_meth_new("Test RSA method", 0)) ==
   NULL
   37.        // || RSA_meth_set_pub_enc(test_rsa_method, eng_rsa_pub_enc)
   == 0
   38.         || RSA_meth_set_pub_dec(test_rsa_method, eng_rsa_pub_dec) ==
   0
   39.         || RSA_meth_set_priv_enc(test_rsa_method, eng_rsa_priv_enc)
   == 0
   40.         || RSA_meth_set_priv_dec(test_rsa_method, eng_rsa_priv_dec)
   == 0
   41.             ) {
   42.
   43.         return 0;
   44.     }
   45.
   46.     /* Ensure the dasync error handling is set up */
   47.
   48.     if (!ENGINE_set_id(e, engine_rsa_id)
   49.         || !ENGINE_set_name(e, engine_rsa_name)
   50.         || !ENGINE_set_RSA(e, test_rsa_method)
   51.             ) {
   52.         return 0;
   53.     }
   54.     return 1;
   55. }
   56.
   57. static int bind_helper(ENGINE *e, const char *id){
   58.     if (!bind_dasync(e)){
   59.         printf("2_Error: Inside Bind helper\n");
   60.         return 0;
   61.     }
   62.     return 1;
   63. }
   64.
   65. IMPLEMENT_DYNAMIC_BIND_FN(bind_helper)
   66. IMPLEMENT_DYNAMIC_CHECK_FN()

==============================================================================

Regards,
Shariful Alam
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://mta.openssl.org/pipermail/openssl-users/attachments/20210927/1a683e76/attachment.html>