Why custom RSA engine is not calling default RSA method?

[Dmitry Belyavsky]

Dear Shariful,

Yes. You have to provide all the RSA_METHO functions your app is going to
use.

On Tue, Sep 28, 2021 at 5:46 PM Shariful Alam <dipto181 at gmail.com> wrote:

> Dear Dmitry,
> Thank you for your response.
>
> Here is the stack trace
>
>
> I was trying using gdb to debug the error. I get to until
> RSA_public_encrypt(), then if I step in, I get the segmentation fault,
>
> ==================================================================================================
> bt
> #0  0x0000000000000000 in ?? ()
> #1  0x00007ffff77dbfc0 in RSA_public_encrypt (flen=72, from=0x6d8860 "Hi.
> This is a test message. Hope to see some performace gain with this.\nme",
> to=0x6d8a70 "x\353;\367\377\177", rsa=0x6d8540, padding=1) at
> crypto/rsa/rsa_crpt.c:30
> #2  0x00000000004479a7 in rsautl_main (argc=0, argv=0x7fffffffde10) at
> apps/rsautl.c:248
> #3  0x00000000004379fa in do_cmd (prog=0x6d5930, argc=11,
> argv=0x7fffffffde10) at apps/openssl.c:564
> #4  0x0000000000436e4d in main (argc=11, argv=0x7fffffffde10) at
> apps/openssl.c:183
>
> ==================================================================================================
>
> To my understanding, instead of pointing to the default encryption
> function, it is pointing to  0x0000000000000000, thus causing the
> segmentation fault.
>



>
> Thanks,
> Shariful
>
>
>
> On Tue, Sep 28, 2021 at 1:40 AM Dmitry Belyavsky <beldmit at gmail.com>
> wrote:
>
>> Dear Shariful,
>>
>> Could you please also provide a stack trace of your segfault?
>>
>> On Tue, Sep 28, 2021 at 1:06 AM Shariful Alam <dipto181 at gmail.com> wrote:
>>
>>> Hello,
>>> I have the following simple RSA engine code from *e_dasync.c. * Following
>>> code compiles and works. Until now, I was under the impression that if I do
>>> not supply an alternative method in function *bind_dasync(), the *engine
>>> will use the default method. However, it doesn't seem to be the case. If I
>>> comment out line 37 and try to perform an encryption operation with the
>>> following command,
>>>
>>> *"openssl rsautl -encrypt -inkey public.pem -pubin -in msg.txt -out
>>> msg.enc -engine rsa-engine-new" *
>>>
>>> I get segmentation errors.
>>>
>>> Can anyone please tell me why this is happening?
>>>
>>>
>>> ==============================================================================
>>>
>>>    1. /* Engine Id and Name */
>>>    2. static const char *engine_rsa_id = "rsa-engine-new";
>>>    3. static const char *engine_rsa_name = "RSA engine for testing";
>>>    4.
>>>    5. // data encryption function
>>>    6. static int eng_rsa_pub_enc(int flen, const unsigned char *from,
>>>    7.                           unsigned char *to, RSA *rsa, int
>>>    padding) {
>>>    8.     printf("Encryption\n");
>>>    9.     return 0;
>>>    10. }
>>>    11.
>>>    12. // signature verify
>>>    13. static int eng_rsa_pub_dec(int flen, const unsigned char *from,
>>>    unsigned char *to, RSA *rsa, int padding){
>>>    14.     printf("Signature verify:\n");
>>>    15.     return 0;
>>>    16. }
>>>    17.
>>>    18. // signature
>>>    19. static int eng_rsa_priv_enc(int flen, const unsigned char *from,
>>>    unsigned char *to, RSA *rsa, int padding){
>>>    20.     printf("Signature:\n");
>>>    21.     return 0;
>>>    22. }
>>>    23.
>>>    24. // data decryption
>>>    25. static int eng_rsa_priv_dec(int flen, const unsigned char *from,
>>>    unsigned char *to, RSA *rsa, int padding){
>>>    26.     printf("Decryption\n");
>>>    27.     return 0;
>>>    28. }
>>>    29.
>>>    30.
>>>    31. static RSA_METHOD *test_rsa_method = NULL;
>>>    32.
>>>    33.
>>>    34. static int bind_dasync(ENGINE *e){
>>>    35.     /* Setup RSA_METHOD */
>>>    36.     if ((test_rsa_method = RSA_meth_new("Test RSA method", 0))
>>>    == NULL
>>>    37.        // || RSA_meth_set_pub_enc(test_rsa_method,
>>>    eng_rsa_pub_enc) == 0
>>>    38.         || RSA_meth_set_pub_dec(test_rsa_method,
>>>    eng_rsa_pub_dec) == 0
>>>    39.         || RSA_meth_set_priv_enc(test_rsa_method,
>>>    eng_rsa_priv_enc) == 0
>>>    40.         || RSA_meth_set_priv_dec(test_rsa_method,
>>>    eng_rsa_priv_dec) == 0
>>>    41.             ) {
>>>    42.
>>>    43.         return 0;
>>>    44.     }
>>>    45.
>>>    46.     /* Ensure the dasync error handling is set up */
>>>    47.
>>>    48.     if (!ENGINE_set_id(e, engine_rsa_id)
>>>    49.         || !ENGINE_set_name(e, engine_rsa_name)
>>>    50.         || !ENGINE_set_RSA(e, test_rsa_method)
>>>    51.             ) {
>>>    52.         return 0;
>>>    53.     }
>>>    54.     return 1;
>>>    55. }
>>>    56.
>>>    57. static int bind_helper(ENGINE *e, const char *id){
>>>    58.     if (!bind_dasync(e)){
>>>    59.         printf("2_Error: Inside Bind helper\n");
>>>    60.         return 0;
>>>    61.     }
>>>    62.     return 1;
>>>    63. }
>>>    64.
>>>    65. IMPLEMENT_DYNAMIC_BIND_FN(bind_helper)
>>>    66. IMPLEMENT_DYNAMIC_CHECK_FN()
>>>
>>>
>>> ==============================================================================
>>>
>>> Regards,
>>> Shariful Alam
>>>
>>>
>>
>> --
>> SY, Dmitry Belyavsky
>>
>

-- 
SY, Dmitry Belyavsky
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://mta.openssl.org/pipermail/openssl-users/attachments/20210929/e3b1f6f6/attachment.html>