Why custom RSA engine is not calling default RSA method?

[Dmitry Belyavsky]

Dear Shariful,

Could you please also provide a stack trace of your segfault?

On Tue, Sep 28, 2021 at 1:06 AM Shariful Alam <dipto181 at gmail.com> wrote:

> Hello,
> I have the following simple RSA engine code from *e_dasync.c. * Following
> code compiles and works. Until now, I was under the impression that if I do
> not supply an alternative method in function *bind_dasync(), the *engine
> will use the default method. However, it doesn't seem to be the case. If I
> comment out line 37 and try to perform an encryption operation with the
> following command,
>
> *"openssl rsautl -encrypt -inkey public.pem -pubin -in msg.txt -out
> msg.enc -engine rsa-engine-new" *
>
> I get segmentation errors.
>
> Can anyone please tell me why this is happening?
>
>
> ==============================================================================
>
>    1. /* Engine Id and Name */
>    2. static const char *engine_rsa_id = "rsa-engine-new";
>    3. static const char *engine_rsa_name = "RSA engine for testing";
>    4.
>    5. // data encryption function
>    6. static int eng_rsa_pub_enc(int flen, const unsigned char *from,
>    7.                           unsigned char *to, RSA *rsa, int padding)
>    {
>    8.     printf("Encryption\n");
>    9.     return 0;
>    10. }
>    11.
>    12. // signature verify
>    13. static int eng_rsa_pub_dec(int flen, const unsigned char *from,
>    unsigned char *to, RSA *rsa, int padding){
>    14.     printf("Signature verify:\n");
>    15.     return 0;
>    16. }
>    17.
>    18. // signature
>    19. static int eng_rsa_priv_enc(int flen, const unsigned char *from,
>    unsigned char *to, RSA *rsa, int padding){
>    20.     printf("Signature:\n");
>    21.     return 0;
>    22. }
>    23.
>    24. // data decryption
>    25. static int eng_rsa_priv_dec(int flen, const unsigned char *from,
>    unsigned char *to, RSA *rsa, int padding){
>    26.     printf("Decryption\n");
>    27.     return 0;
>    28. }
>    29.
>    30.
>    31. static RSA_METHOD *test_rsa_method = NULL;
>    32.
>    33.
>    34. static int bind_dasync(ENGINE *e){
>    35.     /* Setup RSA_METHOD */
>    36.     if ((test_rsa_method = RSA_meth_new("Test RSA method", 0)) ==
>    NULL
>    37.        // || RSA_meth_set_pub_enc(test_rsa_method,
>    eng_rsa_pub_enc) == 0
>    38.         || RSA_meth_set_pub_dec(test_rsa_method, eng_rsa_pub_dec)
>    == 0
>    39.         || RSA_meth_set_priv_enc(test_rsa_method,
>    eng_rsa_priv_enc) == 0
>    40.         || RSA_meth_set_priv_dec(test_rsa_method,
>    eng_rsa_priv_dec) == 0
>    41.             ) {
>    42.
>    43.         return 0;
>    44.     }
>    45.
>    46.     /* Ensure the dasync error handling is set up */
>    47.
>    48.     if (!ENGINE_set_id(e, engine_rsa_id)
>    49.         || !ENGINE_set_name(e, engine_rsa_name)
>    50.         || !ENGINE_set_RSA(e, test_rsa_method)
>    51.             ) {
>    52.         return 0;
>    53.     }
>    54.     return 1;
>    55. }
>    56.
>    57. static int bind_helper(ENGINE *e, const char *id){
>    58.     if (!bind_dasync(e)){
>    59.         printf("2_Error: Inside Bind helper\n");
>    60.         return 0;
>    61.     }
>    62.     return 1;
>    63. }
>    64.
>    65. IMPLEMENT_DYNAMIC_BIND_FN(bind_helper)
>    66. IMPLEMENT_DYNAMIC_CHECK_FN()
>
>
> ==============================================================================
>
> Regards,
> Shariful Alam
>
>

-- 
SY, Dmitry Belyavsky
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://mta.openssl.org/pipermail/openssl-users/attachments/20210928/26433e19/attachment-0001.html>