Why custom RSA engine is not calling default RSA method?

[Dmitry Belyavsky]

Dear Shariful,

Could you please try the patch from
https://github.com/openssl/openssl/pull/16734?

On Wed, Sep 29, 2021 at 6:59 PM Dmitry Belyavsky <beldmit at gmail.com> wrote:

> Dear Shariful,
>
> Yes. You have to provide all the RSA_METHO functions your app is going to
> use.
>
> On Tue, Sep 28, 2021 at 5:46 PM Shariful Alam <dipto181 at gmail.com> wrote:
>
>> Dear Dmitry,
>> Thank you for your response.
>>
>> Here is the stack trace
>>
>>
>> I was trying using gdb to debug the error. I get to until
>> RSA_public_encrypt(), then if I step in, I get the segmentation fault,
>>
>> ==================================================================================================
>> bt
>> #0  0x0000000000000000 in ?? ()
>> #1  0x00007ffff77dbfc0 in RSA_public_encrypt (flen=72, from=0x6d8860 "Hi.
>> This is a test message. Hope to see some performace gain with this.\nme",
>> to=0x6d8a70 "x\353;\367\377\177", rsa=0x6d8540, padding=1) at
>> crypto/rsa/rsa_crpt.c:30
>> #2  0x00000000004479a7 in rsautl_main (argc=0, argv=0x7fffffffde10) at
>> apps/rsautl.c:248
>> #3  0x00000000004379fa in do_cmd (prog=0x6d5930, argc=11,
>> argv=0x7fffffffde10) at apps/openssl.c:564
>> #4  0x0000000000436e4d in main (argc=11, argv=0x7fffffffde10) at
>> apps/openssl.c:183
>>
>> ==================================================================================================
>>
>> To my understanding, instead of pointing to the default encryption
>> function, it is pointing to  0x0000000000000000, thus causing the
>> segmentation fault.
>>
>
>
>
>>
>> Thanks,
>> Shariful
>>
>>
>>
>> On Tue, Sep 28, 2021 at 1:40 AM Dmitry Belyavsky <beldmit at gmail.com>
>> wrote:
>>
>>> Dear Shariful,
>>>
>>> Could you please also provide a stack trace of your segfault?
>>>
>>> On Tue, Sep 28, 2021 at 1:06 AM Shariful Alam <dipto181 at gmail.com>
>>> wrote:
>>>
>>>> Hello,
>>>> I have the following simple RSA engine code from *e_dasync.c. * Following
>>>> code compiles and works. Until now, I was under the impression that if I do
>>>> not supply an alternative method in function *bind_dasync(), the *engine
>>>> will use the default method. However, it doesn't seem to be the case. If I
>>>> comment out line 37 and try to perform an encryption operation with the
>>>> following command,
>>>>
>>>> *"openssl rsautl -encrypt -inkey public.pem -pubin -in msg.txt -out
>>>> msg.enc -engine rsa-engine-new" *
>>>>
>>>> I get segmentation errors.
>>>>
>>>> Can anyone please tell me why this is happening?
>>>>
>>>>
>>>> ==============================================================================
>>>>
>>>>    1. /* Engine Id and Name */
>>>>    2. static const char *engine_rsa_id = "rsa-engine-new";
>>>>    3. static const char *engine_rsa_name = "RSA engine for testing";
>>>>    4.
>>>>    5. // data encryption function
>>>>    6. static int eng_rsa_pub_enc(int flen, const unsigned char *from,
>>>>    7.                           unsigned char *to, RSA *rsa, int
>>>>    padding) {
>>>>    8.     printf("Encryption\n");
>>>>    9.     return 0;
>>>>    10. }
>>>>    11.
>>>>    12. // signature verify
>>>>    13. static int eng_rsa_pub_dec(int flen, const unsigned char *from,
>>>>    unsigned char *to, RSA *rsa, int padding){
>>>>    14.     printf("Signature verify:\n");
>>>>    15.     return 0;
>>>>    16. }
>>>>    17.
>>>>    18. // signature
>>>>    19. static int eng_rsa_priv_enc(int flen, const unsigned char
>>>>    *from, unsigned char *to, RSA *rsa, int padding){
>>>>    20.     printf("Signature:\n");
>>>>    21.     return 0;
>>>>    22. }
>>>>    23.
>>>>    24. // data decryption
>>>>    25. static int eng_rsa_priv_dec(int flen, const unsigned char
>>>>    *from, unsigned char *to, RSA *rsa, int padding){
>>>>    26.     printf("Decryption\n");
>>>>    27.     return 0;
>>>>    28. }
>>>>    29.
>>>>    30.
>>>>    31. static RSA_METHOD *test_rsa_method = NULL;
>>>>    32.
>>>>    33.
>>>>    34. static int bind_dasync(ENGINE *e){
>>>>    35.     /* Setup RSA_METHOD */
>>>>    36.     if ((test_rsa_method = RSA_meth_new("Test RSA method", 0))
>>>>    == NULL
>>>>    37.        // || RSA_meth_set_pub_enc(test_rsa_method,
>>>>    eng_rsa_pub_enc) == 0
>>>>    38.         || RSA_meth_set_pub_dec(test_rsa_method,
>>>>    eng_rsa_pub_dec) == 0
>>>>    39.         || RSA_meth_set_priv_enc(test_rsa_method,
>>>>    eng_rsa_priv_enc) == 0
>>>>    40.         || RSA_meth_set_priv_dec(test_rsa_method,
>>>>    eng_rsa_priv_dec) == 0
>>>>    41.             ) {
>>>>    42.
>>>>    43.         return 0;
>>>>    44.     }
>>>>    45.
>>>>    46.     /* Ensure the dasync error handling is set up */
>>>>    47.
>>>>    48.     if (!ENGINE_set_id(e, engine_rsa_id)
>>>>    49.         || !ENGINE_set_name(e, engine_rsa_name)
>>>>    50.         || !ENGINE_set_RSA(e, test_rsa_method)
>>>>    51.             ) {
>>>>    52.         return 0;
>>>>    53.     }
>>>>    54.     return 1;
>>>>    55. }
>>>>    56.
>>>>    57. static int bind_helper(ENGINE *e, const char *id){
>>>>    58.     if (!bind_dasync(e)){
>>>>    59.         printf("2_Error: Inside Bind helper\n");
>>>>    60.         return 0;
>>>>    61.     }
>>>>    62.     return 1;
>>>>    63. }
>>>>    64.
>>>>    65. IMPLEMENT_DYNAMIC_BIND_FN(bind_helper)
>>>>    66. IMPLEMENT_DYNAMIC_CHECK_FN()
>>>>
>>>>
>>>> ==============================================================================
>>>>
>>>> Regards,
>>>> Shariful Alam
>>>>
>>>>
>>>
>>> --
>>> SY, Dmitry Belyavsky
>>>
>>
>
> --
> SY, Dmitry Belyavsky
>


-- 
SY, Dmitry Belyavsky
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://mta.openssl.org/pipermail/openssl-users/attachments/20211004/5f229adc/attachment-0001.html>