Why custom RSA engine is not calling default RSA method?
Shariful Alam
dipto181 at gmail.com
Tue Sep 28 15:45:52 UTC 2021
Dear Dmitry,
Thank you for your response.
Here is the stack trace
=============================================================================================================================
$ strace openssl rsautl -encrypt -inkey public.pem -pubin -in msg.txt -out
msg.enc -engine rsa-engine-new
execve("/usr/bin/openssl", ["openssl", "rsautl", "-encrypt", "-inkey",
"public.pem", "-pubin", "-in", "msg.txt", "-out", "msg.enc", "-engine",
"rsa-engine-new"], [/* 65 vars */]) = 0
brk(NULL) = 0x1634000
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or
directory)
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or
directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=103650, ...}) = 0
mmap(NULL, 103650, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f3997ce3000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or
directory)
open("/opt/openssl_test/lib/libssl.so.1.1", O_RDONLY|O_CLOEXEC) = 3
read(3,
"\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P\327\1\0\0\0\0\0"..., 832)
= 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1713040, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
0x7f3997ce2000
mmap(NULL, 2836448, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) =
0x7f3997823000
mprotect(0x7f39978cb000, 2097152, PROT_NONE) = 0
mmap(0x7f3997acb000, 53248, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xa8000) = 0x7f3997acb000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or
directory)
open("/opt/openssl_test/lib/libcrypto.so.1.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0p\7\0\0\0\0\0"...,
832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=6286208, ...}) = 0
mmap(NULL, 5508104, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) =
0x7f39972e2000
mprotect(0x7f39975f3000, 2093056, PROT_NONE) = 0
mmap(0x7f39977f2000, 188416, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x310000) = 0x7f39977f2000
mmap(0x7f3997820000, 11272, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f3997820000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or
directory)
open("/lib/x86_64-linux-gnu/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3,
"\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260`\0\0\0\0\0\0"..., 832)
= 832
fstat(3, {st_mode=S_IFREG|0755, st_size=138696, ...}) = 0
mmap(NULL, 2212904, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) =
0x7f39970c5000
mprotect(0x7f39970dd000, 2093056, PROT_NONE) = 0
mmap(0x7f39972dc000, 8192, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x17000) = 0x7f39972dc000
mmap(0x7f39972de000, 13352, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f39972de000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or
directory)
open("/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`\t\2\0\0\0\0\0"...,
832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1868984, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
0x7f3997ce1000
mmap(NULL, 3971488, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) =
0x7f3996cfb000
mprotect(0x7f3996ebb000, 2097152, PROT_NONE) = 0
mmap(0x7f39970bb000, 24576, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1c0000) = 0x7f39970bb000
mmap(0x7f39970c1000, 14752, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f39970c1000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or
directory)
open("/lib/x86_64-linux-gnu/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3,
"\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240\r\0\0\0\0\0\0"..., 832)
= 832
fstat(3, {st_mode=S_IFREG|0644, st_size=14608, ...}) = 0
mmap(NULL, 2109680, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) =
0x7f3996af7000
mprotect(0x7f3996afa000, 2093056, PROT_NONE) = 0
mmap(0x7f3996cf9000, 8192, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f3996cf9000
close(3) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
0x7f3997ce0000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
0x7f3997cdf000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
0x7f3997cde000
arch_prctl(ARCH_SET_FS, 0x7f3997cdf700) = 0
mprotect(0x7f39970bb000, 16384, PROT_READ) = 0
mprotect(0x7f3996cf9000, 4096, PROT_READ) = 0
mprotect(0x7f39972dc000, 4096, PROT_READ) = 0
mprotect(0x7f39977f2000, 176128, PROT_READ) = 0
mprotect(0x7f3997acb000, 32768, PROT_READ) = 0
mprotect(0x6ac000, 4096, PROT_READ) = 0
mprotect(0x7f3997cfd000, 4096, PROT_READ) = 0
munmap(0x7f3997ce3000, 103650) = 0
set_tid_address(0x7f3997cdf9d0) = 6006
set_robust_list(0x7f3997cdf9e0, 24) = 0
rt_sigaction(SIGRTMIN, {0x7f39970cab50, [], SA_RESTORER|SA_SIGINFO,
0x7f39970d6390}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {0x7f39970cabe0, [],
SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x7f39970d6390}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
brk(NULL) = 0x1634000
brk(0x1655000) = 0x1655000
futex(0x7f39978213d8, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f39978213e4, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f39978213ec, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f3997821368, FUTEX_WAKE_PRIVATE, 2147483647) = 0
rt_sigaction(SIGPIPE, {SIG_IGN, [PIPE], SA_RESTORER|SA_RESTART,
0x7f3996d304c0}, {SIG_DFL, [], 0}, 8) = 0
futex(0x7f3997821530, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f3997821400, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f3997821408, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f399781f880, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f399781f940, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f399781f94c, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f39978213f4, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f3997821438, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f3997821440, FUTEX_WAKE_PRIVATE, 2147483647) = 0
open("/opt/openssl_test/ssl/openssl.cnf", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=11162, ...}) = 0
read(3, "#\n# OpenSSL example configuratio"..., 4096) = 4096
read(3, "\n# MASK:XXXX a literal mask valu"..., 4096) = 4096
read(3, "ts=CA:FALSE\n\n# Here are some exa"..., 4096) = 2970
brk(0x1676000) = 0x1676000
read(3, "", 4096) = 0
close(3) = 0
futex(0x7f3997821410, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f3997ad7698, FUTEX_WAKE_PRIVATE, 2147483647) = 0
sysinfo({uptime=2961, loads=[30016, 30688, 27392], totalram=16689545216,
freeram=12341272576, sharedram=140345344, bufferram=240164864,
totalswap=1023406080, freeswap=1023406080, procs=922, totalhigh=0,
freehigh=0, mem_unit=1}) = 0
futex(0x7f3997ad7778, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f3996cfa0a8, FUTEX_WAKE_PRIVATE, 2147483647) = 0
open("/opt/openssl_test/lib/engines-1.1/rsa-engine-new.so",
O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0p0\0\0\0\0\0\0"...,
832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=143576, ...}) = 0
mmap(NULL, 2241392, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) =
0x7f39968d3000
mprotect(0x7f39968ea000, 2093056, PROT_NONE) = 0
mmap(0x7f3996ae9000, 8192, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x16000) = 0x7f3996ae9000
mmap(0x7f3996aeb000, 45936, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f3996aeb000
close(3) = 0
mprotect(0x7f3996ae9000, 4096, PROT_READ) = 0
write(2, "engine \"rsa-engine-new\" set.\n", 29engine "rsa-engine-new" set.
) = 29
open("public.pem", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0664, st_size=451, ...}) = 0
read(3, "-----BEGIN PUBLIC KEY-----\nMIIBI"..., 4096) = 451
brk(0x1674000) = 0x1674000
close(3) = 0
open("msg.txt", O_RDONLY) = 3
open("msg.enc", O_WRONLY|O_CREAT|O_TRUNC, 0666) = 4
fstat(3, {st_mode=S_IFREG|0664, st_size=72, ...}) = 0
read(3, "Hi. This is a test message. Hope"..., 4096) = 72
read(3, "", 4096) = 0
--- SIGSEGV {si_signo=SIGSEGV, si_code=SEGV_MAPERR, si_addr=0} ---
+++ killed by SIGSEGV (core dumped) +++
Segmentation fault (core dumped)
=============================================================================================================================
I was trying using gdb to debug the error. I get to until
RSA_public_encrypt(), then if I step in, I get the segmentation fault,
==================================================================================================
bt
#0 0x0000000000000000 in ?? ()
#1 0x00007ffff77dbfc0 in RSA_public_encrypt (flen=72, from=0x6d8860 "Hi.
This is a test message. Hope to see some performace gain with this.\nme",
to=0x6d8a70 "x\353;\367\377\177", rsa=0x6d8540, padding=1) at
crypto/rsa/rsa_crpt.c:30
#2 0x00000000004479a7 in rsautl_main (argc=0, argv=0x7fffffffde10) at
apps/rsautl.c:248
#3 0x00000000004379fa in do_cmd (prog=0x6d5930, argc=11,
argv=0x7fffffffde10) at apps/openssl.c:564
#4 0x0000000000436e4d in main (argc=11, argv=0x7fffffffde10) at
apps/openssl.c:183
==================================================================================================
To my understanding, instead of pointing to the default encryption
function, it is pointing to 0x0000000000000000, thus causing the
segmentation fault.
Thanks,
Shariful
On Tue, Sep 28, 2021 at 1:40 AM Dmitry Belyavsky <beldmit at gmail.com> wrote:
> Dear Shariful,
>
> Could you please also provide a stack trace of your segfault?
>
> On Tue, Sep 28, 2021 at 1:06 AM Shariful Alam <dipto181 at gmail.com> wrote:
>
>> Hello,
>> I have the following simple RSA engine code from *e_dasync.c. * Following
>> code compiles and works. Until now, I was under the impression that if I do
>> not supply an alternative method in function *bind_dasync(), the *engine
>> will use the default method. However, it doesn't seem to be the case. If I
>> comment out line 37 and try to perform an encryption operation with the
>> following command,
>>
>> *"openssl rsautl -encrypt -inkey public.pem -pubin -in msg.txt -out
>> msg.enc -engine rsa-engine-new" *
>>
>> I get segmentation errors.
>>
>> Can anyone please tell me why this is happening?
>>
>>
>> ==============================================================================
>>
>> 1. /* Engine Id and Name */
>> 2. static const char *engine_rsa_id = "rsa-engine-new";
>> 3. static const char *engine_rsa_name = "RSA engine for testing";
>> 4.
>> 5. // data encryption function
>> 6. static int eng_rsa_pub_enc(int flen, const unsigned char *from,
>> 7. unsigned char *to, RSA *rsa, int
>> padding) {
>> 8. printf("Encryption\n");
>> 9. return 0;
>> 10. }
>> 11.
>> 12. // signature verify
>> 13. static int eng_rsa_pub_dec(int flen, const unsigned char *from,
>> unsigned char *to, RSA *rsa, int padding){
>> 14. printf("Signature verify:\n");
>> 15. return 0;
>> 16. }
>> 17.
>> 18. // signature
>> 19. static int eng_rsa_priv_enc(int flen, const unsigned char *from,
>> unsigned char *to, RSA *rsa, int padding){
>> 20. printf("Signature:\n");
>> 21. return 0;
>> 22. }
>> 23.
>> 24. // data decryption
>> 25. static int eng_rsa_priv_dec(int flen, const unsigned char *from,
>> unsigned char *to, RSA *rsa, int padding){
>> 26. printf("Decryption\n");
>> 27. return 0;
>> 28. }
>> 29.
>> 30.
>> 31. static RSA_METHOD *test_rsa_method = NULL;
>> 32.
>> 33.
>> 34. static int bind_dasync(ENGINE *e){
>> 35. /* Setup RSA_METHOD */
>> 36. if ((test_rsa_method = RSA_meth_new("Test RSA method", 0)) ==
>> NULL
>> 37. // || RSA_meth_set_pub_enc(test_rsa_method,
>> eng_rsa_pub_enc) == 0
>> 38. || RSA_meth_set_pub_dec(test_rsa_method, eng_rsa_pub_dec)
>> == 0
>> 39. || RSA_meth_set_priv_enc(test_rsa_method,
>> eng_rsa_priv_enc) == 0
>> 40. || RSA_meth_set_priv_dec(test_rsa_method,
>> eng_rsa_priv_dec) == 0
>> 41. ) {
>> 42.
>> 43. return 0;
>> 44. }
>> 45.
>> 46. /* Ensure the dasync error handling is set up */
>> 47.
>> 48. if (!ENGINE_set_id(e, engine_rsa_id)
>> 49. || !ENGINE_set_name(e, engine_rsa_name)
>> 50. || !ENGINE_set_RSA(e, test_rsa_method)
>> 51. ) {
>> 52. return 0;
>> 53. }
>> 54. return 1;
>> 55. }
>> 56.
>> 57. static int bind_helper(ENGINE *e, const char *id){
>> 58. if (!bind_dasync(e)){
>> 59. printf("2_Error: Inside Bind helper\n");
>> 60. return 0;
>> 61. }
>> 62. return 1;
>> 63. }
>> 64.
>> 65. IMPLEMENT_DYNAMIC_BIND_FN(bind_helper)
>> 66. IMPLEMENT_DYNAMIC_CHECK_FN()
>>
>>
>> ==============================================================================
>>
>> Regards,
>> Shariful Alam
>>
>>
>
> --
> SY, Dmitry Belyavsky
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://mta.openssl.org/pipermail/openssl-users/attachments/20210928/8cd521bc/attachment-0001.html>
More information about the openssl-users
mailing list