Crash in x86_64-mont5.s when running speed for RSA on openssl-1.1.1a

dengwenbin_0301 dengwenbin_0301 at 126.com
Mon Jun 10 07:20:17 UTC 2019


Hello,

Starting from the rsa choice "rsa2048", there is crash due to segment fault when running speed.  RSA512 and rsa1024 doesn't have this issue.

Following is the call stacks, registers info when crash happened. It shows that the register %rbp is zero at that point. The value of %rbp is set by instruction "movq    %xmm2,$nptr " in label .L8x_no_tail. I don't understand how the algorithm in the x86_64-mont5.s works. So i have no idea why a zero value of %xmm2 was moved to %rbp at that point. Please help have a look.

(wrdbg) bt
#0  0x0000000000242d23 at __bn_sqrx8x_reduction+0x6c
#1  0x000000000023f96b at bn_sqr8x_mont+0xcb
#2  0x000000000023b014 at bn_mul_mont_fixed_top+0x64
#3  0x00000000002b61f5 at BN_mod_exp_mont+0x445
#4  0x0000000000281b1f at rsa_ossl_public_decrypt+0x28f
#5  0x0000000000285c0a at int_rsa_verify+0x8a
#6  0x0000000000285f92 at RSA_verify+0x32
#7  0x0000000000217963 at RSA_verify_loop+0x53
#8  0x00000000002176d8 at run_benchmark+0x108
#9  0x0000000000212e80 at speed_main+0x1400
#10 0x0000000000201411 in main (argc=1, argv=0x68ff20) at rtp.c:33
#11 0x0000000000201073 at _start+0x72
(wrdbg) info registers
  rax             0x0000000000000000  0                                
  rdx             0x68f8f88932a43dc0  7564068842312580544                       
  rcx             0xfffffffffffffff8  18446744073709551608                       
  rbx             0x123959978a8b3640  1313179273754523200                       
  rsi             0x0000000000000000  0                                
  rdi             0x0000000000688430  6849584                          
  rbp             0x0000000000000000  0                                
  rsp             0x00000000006883b8  6849464                          
  r8              0x123959978a8b3640  1313179273754523200                       
  r9              0xf7a5b45905344009  17844867392957857801                       
  r10             0x73dbc32a132b324b  8348480919738987083                       
  r11             0x37cda4961a791379  4021051006907913081                       
  r12             0x701103ae9d18bf8a  8075239655346847626                       
  r13             0xbfbd1a5d2ab85477  13816228219293553783                       
  r14             0x3f3610a53ecd4756  4554846375041124182                       
  r15             0xea5e26bc1a343a39  16887978242021276217                       
  rip             0x0000000000242d23  2370851                           PC
  eflags          0x00000246          582        [ pf zf if ]          
  fpu                                 None                             
  xmm                                 None     

                  __bn_sqrx8x_reduction:
0000000000242cb7:   xor     eax, eax
0000000000242cb9:   mov     rbx, qword ptr [rsp+0x28]
0000000000242cbe:   mov     rdx, qword ptr [rsp+0x38]
0000000000242cc3:   lea     rcx, ptr [rbp+r9*1-0x40]
0000000000242cc8:   mov     qword ptr [rsp+0x8], rcx
0000000000242ccd:   mov     qword ptr [rsp+0x10], rdi
0000000000242cd2:   lea     rdi, ptr [rsp+0x38]
0000000000242cd7:   jmp     0x242ce0 <__bn_sqrx8x_reduction+41>
0000000000242cd9:   nop     dword ptr [rax], eax
0000000000242ce0:   mov     r9, qword ptr [rdi+0x8]
0000000000242ce4:   mov     r10, qword ptr [rdi+0x10]
0000000000242ce8:   mov     r11, qword ptr [rdi+0x18]
0000000000242cec:   mov     r12, qword ptr [rdi+0x20]
0000000000242cf0:   mov     r8, rdx
0000000000242cf3:   imul    rdx, rbx
0000000000242cf7:   mov     r13, qword ptr [rdi+0x28]
0000000000242cfb:   mov     r14, qword ptr [rdi+0x30]
0000000000242cff:   mov     r15, qword ptr [rdi+0x38]
0000000000242d03:   mov     qword ptr [rsp+0x20], rax
0000000000242d08:   lea     rdi, ptr [rdi+0x40]
0000000000242d0c:   xor     rsi, rsi
0000000000242d0f:   mov     rcx, -8
0000000000242d16:   jmp     0x242d20 <__bn_sqrx8x_reduction+105>
0000000000242d18:   nop     dword ptr [rax+rax*1], eax
0000000000242d20:   mov     rbx, r8
0000000000242d23:   mulx    r8, rax, qword ptr [rbp]
0000000000242d29:   adcx    rax, rbx
0000000000242d2f:   adox    r8, r9

L8x_no_tail:
    adc 8*0($tptr),%r8
    adc 8*1($tptr),%r9
    adc 8*2($tptr),%r10
    adc 8*3($tptr),%r11
    adc 8*4($tptr),%r12
    adc 8*5($tptr),%r13
    adc 8*6($tptr),%r14
    adc 8*7($tptr),%r15
    adc \$0,%rax        # top-most carry
     mov    -8($nptr),%rcx      # np[num-1]
     xor    $carry,$carry
    movq    %xmm2,$nptr     # restore $nptr
    mov %r8,8*0($tptr)      # store top 512 bits
    mov %r9,8*1($tptr)
     movq   %xmm3,$num      # $num is %r9, can't be moved upwards


configdata.pm --dump:
Command line (with current working directory = .):

    /usr/bin/perl ../Configure vxworks-x86-64 no-weak-ssl-ciphers no-aria no-blake2 no-camellia no-chacha no-cmac no-md2 no-md4 no-mdc2 no-ocb no-poly1305 no-rc2 no-rc5 no-scrypt no-seed no-siphash no-sm2 no-sm3 no-sm4 no-whirlpool no-autoerrinit no-comp no-sctp no-srtp --with-rand-seed=none -DOPENSSL_SYS_VXWORKS_SUPPORT -DVXWORKS_USER_SPACE -fPIC

Perl information:

    /usr/bin/perl
    5.22.1 for x86_64-linux-gnu-thread-multi

Enabled features:

    asm
    async
    autoalginit
    autoload-config
    bf
    capieng
    cast
    cms
    ct
    deprecated
    des
    dgram
    dh
    dsa
    dso
    dtls
    dynamic-engine
    ec
    ec2m
    ecdh
    ecdsa
    engine
    err
    filenames
    gost
    hw(-.+)?
    idea
    multiblock
    nextprotoneg
    ocsp
    pic
    posix-io
    psk
    rc4
    rdrand
    rfc3779
    rmd160
    shared
    sock
    srp
    sse2
    ssl
    static-engine
    stdio
    tests
    tls
    ts
    ui-console
    tls1
    tls1-method
    tls1_1
    tls1_1-method
    tls1_2
    tls1_2-method
    tls1_3
    dtls1
    dtls1-method
    dtls1_2
    dtls1_2-method

Disabled features:

    afalgeng                [not-linux]  
    aria                    [option]      OPENSSL_NO_ARIA (skip crypto/aria)
    asan                    [default]     OPENSSL_NO_ASAN
    autoerrinit             [option]      OPENSSL_NO_AUTOERRINIT
    blake2                  [option]      OPENSSL_NO_BLAKE2 (skip crypto/blake2)
    camellia                [option]      OPENSSL_NO_CAMELLIA (skip crypto/camellia)
    chacha                  [option]      OPENSSL_NO_CHACHA (skip crypto/chacha)
    cmac                    [option]      OPENSSL_NO_CMAC (skip crypto/cmac)
    comp                    [option]      OPENSSL_NO_COMP (skip crypto/comp)
    crypto-mdebug           [default]     OPENSSL_NO_CRYPTO_MDEBUG
    crypto-mdebug-backtrace [default]     OPENSSL_NO_CRYPTO_MDEBUG_BACKTRACE
    devcryptoeng            [default]     OPENSSL_NO_DEVCRYPTOENG
    ec_nistp_64_gcc_128     [default]     OPENSSL_NO_EC_NISTP_64_GCC_128
    egd                     [default]     OPENSSL_NO_EGD
    external-tests          [default]     OPENSSL_NO_EXTERNAL_TESTS
    fuzz-libfuzzer          [default]     OPENSSL_NO_FUZZ_LIBFUZZER
    fuzz-afl                [default]     OPENSSL_NO_FUZZ_AFL
    heartbeats              [default]     OPENSSL_NO_HEARTBEATS
    makedepend              [unavailable]
    md2                     [option]      OPENSSL_NO_MD2 (skip crypto/md2)
    md4                     [option]      OPENSSL_NO_MD4 (skip crypto/md4)
    mdc2                    [option]      OPENSSL_NO_MDC2 (skip crypto/mdc2)
    msan                    [default]     OPENSSL_NO_MSAN
    ocb                     [option]      OPENSSL_NO_OCB
    poly1305                [option]      OPENSSL_NO_POLY1305 (skip crypto/poly1305)
    rc2                     [option]      OPENSSL_NO_RC2 (skip crypto/rc2)
    rc5                     [option]      OPENSSL_NO_RC5 (skip crypto/rc5)
    scrypt                  [option]      OPENSSL_NO_SCRYPT
    sctp                    [option]      OPENSSL_NO_SCTP
    seed                    [option]      OPENSSL_NO_SEED (skip crypto/seed)
    siphash                 [option]      OPENSSL_NO_SIPHASH (skip crypto/siphash)
    sm2                     [option]      OPENSSL_NO_SM2 (skip crypto/sm2)
    sm3                     [option]      OPENSSL_NO_SM3 (skip crypto/sm3)
    sm4                     [option]      OPENSSL_NO_SM4 (skip crypto/sm4)
    srtp                    [option]      OPENSSL_NO_SRTP
    ssl-trace               [default]     OPENSSL_NO_SSL_TRACE
    threads                 [unavailable]
    ubsan                   [default]     OPENSSL_NO_UBSAN
    unit-test               [default]     OPENSSL_NO_UNIT_TEST
    whirlpool               [option]     
    weak-ssl-ciphers        [option]      OPENSSL_NO_WEAK_SSL_CIPHERS
    zlib                    [default]    
    zlib-dynamic            [default]    
    ssl3                    [default]     OPENSSL_NO_SSL3
    ssl3-method             [default]     OPENSSL_NO_SSL3_METHOD

Config target attributes:

    AR => "ar",
    ARFLAGS => "r",
    CC => "dummy",
    CFLAGS => "",
    HASHBANGPERL => "/usr/bin/env perl",
    RANLIB => "ranlib",
    RC => "windres",
    aes_asm_src => "aes-x86_64.s vpaes-x86_64.s bsaes-x86_64.s aesni-x86_64.s aesni-sha1-x86_64.s aesni-sha256-x86_64.s aesni-mb-x86_64.s",
    aes_obj => "aes-x86_64.o vpaes-x86_64.o bsaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o aesni-sha256-x86_64.o aesni-mb-x86_64.o",
    apps_aux_src => "",
    apps_init_src => "",
    apps_obj => "",
    bf_asm_src => "bf_enc.c",
    bf_obj => "bf_enc.o",
    bn_asm_src => "asm/x86_64-gcc.c x86_64-mont.s x86_64-mont5.s x86_64-gf2m.s rsaz_exp.c rsaz-x86_64.s rsaz-avx2.s",
    bn_obj => "asm/x86_64-gcc.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o rsaz_exp.o rsaz-x86_64.o rsaz-avx2.o",
    bn_ops => "SIXTY_FOUR_BIT_LONG",
    build_file => "Makefile",
    build_scheme => [ "unified", "unix" ],
    cast_asm_src => "c_enc.c",
    cast_obj => "c_enc.o",
    cflags => "",
    chacha_asm_src => "chacha-x86_64.s",
    chacha_obj => "chacha-x86_64.o",
    cmll_asm_src => "cmll-x86_64.s cmll_misc.c",
    cmll_obj => "cmll-x86_64.o cmll_misc.o",
    cppflags => "\$(VSB_CONFIG_FILE_MACRO)",
    cpuid_asm_src => "x86_64cpuid.s",
    cpuid_obj => "x86_64cpuid.o",
    defines => [  ],
    des_asm_src => "des_enc.c fcrypt_b.c",
    des_obj => "des_enc.o fcrypt_b.o",
    disable => [  ],
    dso_extension => ".so",
    ec_asm_src => "ecp_nistz256.c ecp_nistz256-x86_64.s x25519-x86_64.s",
    ec_obj => "ecp_nistz256.o ecp_nistz256-x86_64.o x25519-x86_64.o",
    enable => [  ],
    exe_extension => "",
    includes => [  ],
    keccak1600_asm_src => "keccak1600-x86_64.s",
    keccak1600_obj => "keccak1600-x86_64.o",
    lflags => "",
    lib_cflags => "",
    lib_cppflags => "",
    lib_defines => [  ],
    md5_asm_src => "md5-x86_64.s",
    md5_obj => "md5-x86_64.o",
    modes_asm_src => "ghash-x86_64.s aesni-gcm-x86_64.s",
    modes_obj => "ghash-x86_64.o aesni-gcm-x86_64.o",
    module_cflags => "",
    module_cxxflags => "",
    module_ldflags => "-shared",
    padlock_asm_src => "e_padlock-x86_64.s",
    padlock_obj => "e_padlock-x86_64.o",
    poly1305_asm_src => "poly1305-x86_64.s",
    poly1305_obj => "poly1305-x86_64.o",
    rc4_asm_src => "rc4-x86_64.s rc4-md5-x86_64.s",
    rc4_obj => "rc4-x86_64.o rc4-md5-x86_64.o",
    rc5_asm_src => "rc5_enc.c",
    rc5_obj => "rc5_enc.o",
    rmd160_asm_src => "",
    rmd160_obj => "",
    sha1_asm_src => "sha1-x86_64.s sha256-x86_64.s sha512-x86_64.s sha1-mb-x86_64.s sha256-mb-x86_64.s",
    sha1_obj => "sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o sha1-mb-x86_64.o sha256-mb-x86_64.o",
    shared_cflag => "",
    shared_defines => [  ],
    shared_extension => ".so.\$(SHLIB_VERSION_NUMBER)",
    shared_extension_simple => ".so",
    shared_ldflag => "-shared",
    shared_rcflag => "",
    shared_target => "dummy",
    sys_id => "VXWORKS",
    thread_defines => [  ],
    thread_scheme => "(unknown)",
    unistd => "<unistd.h>",
    uplink_aux_src => "",
    uplink_obj => "",
    wp_asm_src => "wp-x86_64.s",
    wp_obj => "wp-x86_64.o",

Recorded environment:

    AR =
    BUILDFILE =
    CC =
    CROSS_COMPILE =
    CXX =
    HASHBANGPERL =
    OPENSSL_LOCAL_CONFIG_DIR =
    PERL =
    RANLIB =
    RC =
    WINDRES =
    __CNF_CFLAGS =
    __CNF_CPPDEFINES =
    __CNF_CPPFLAGS =
    __CNF_CPPINCLUDES =
    __CNF_CXXFLAGS =
    __CNF_LDFLAGS =
    __CNF_LDLIBS =

Makevars:

    AR              = ar
    ARFLAGS         = r
    CC              = dummy
    CFLAGS          = -fPIC
    CPPDEFINES      = OPENSSL_SYS_VXWORKS_SUPPORT VXWORKS_USER_SPACE
    CPPFLAGS        =
    CPPINCLUDES     =
    CXXFLAGS        = -fPIC
    HASHBANGPERL    = /usr/bin/env perl
    LDFLAGS         =
    LDLIBS          =
    PERL            = /usr/bin/perl
    RANLIB          = ranlib
    RC              = windres

NOTE: These variables only represent the configuration view.  The build file
template may have processed these variables further, please have a look at the
build file for more exact data:
    Makefile

build file:

    Makefile

build file templates:

    ../Configurations/common0.tmpl
    ../Configurations/unix-Makefile.tmpl
    ../Configurations/common.tmpl


Thanks,
Wenbin

-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mta.openssl.org/pipermail/openssl-users/attachments/20190610/b1aec7b6/attachment-0001.html>


More information about the openssl-users mailing list