RSA public and private key encrypt with fips issue

[Yi tiger]

Hi guys,

I have download openssl 1.0.2s and fips 2.0.16 and build successfully on windows and then I try to write a simple app encrypted with public key and decrypt with private key. But It failed decrypted with private key when fips mode is turned on, decrypt will be success if fips is turned off.
I got below error

[cid:image001.png at 01D52176.09181B50]

The first error occurs if I pass RSA_sizeof(rsa) as  RSA_private_decrypt ‘s first parameter, the second error if I pass the encrypted buffer’s length as RSA_private_decrypt’s first parameter.
I also noticed return value of RSA_size is different when fips mode is turned on.
Its size is 256 if fips is turned off, but become 128 if fips turned on.
RSA_public_encrypt return value is equal to RSA_size when fips is turned off, but its return value will great than RSA_size when truned on, maybe this cause decrypt or something its my code problem?

Below is my source code,


#include "openssl\crypto.h"
#include "openssl\rsa.h"
#include "openssl\err.h"
#include "openssl/pem.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

char* privatekey;
int privlen;
char* publickey;
int publen;

void generate()
{
       int mode = FIPS_mode();
       if (mode == 0)
       {
              int ret = FIPS_mode_set(1);
              if (ret != 1)
              {
                     printf("error %s", ERR_get_error());
              }
       }

       RSA *rsa = RSA_generate_key(2048, 3, NULL, NULL);

       BIO *bio = BIO_new(BIO_s_mem());
       PEM_write_bio_RSAPrivateKey(bio, rsa, NULL, NULL, 0, NULL, NULL);
       int rszsize = RSA_size(rsa);
       privlen = BIO_pending(bio);
       privatekey = (char*)malloc(privlen + 1);
       BIO_read(bio, privatekey, privlen);

       PEM_write_bio_RSAPublicKey(bio, rsa);
       publen = BIO_pending(bio);
       publickey = (char*)malloc(publen + 1);
       BIO_read(bio, publickey, publen);

       BIO_free_all(bio);
       RSA_free(rsa);
}

void Encrypt(const char* password)
{
       BIO *bio = BIO_new(BIO_s_mem());
       RSA *rsa = NULL;
       BIO_write(bio, publickey, publen);
       RSA* temp = PEM_read_bio_RSAPublicKey(bio, &rsa, NULL, NULL);
       char encrypted[500];
       int len = 0;
       if (temp != NULL)
       {
              int rsasize = RSA_size(rsa);
              len = RSA_public_encrypt((int)strlen(password), (unsigned char*)password, (unsigned char*)encrypted, rsa, RSA_PKCS1_OAEP_PADDING);
              if (len == -1)
              {
                     int e = ERR_get_error();
                     char * err = ERR_error_string(ERR_get_error(), encrypted);
                     printf("RSA_public_encrypt failed.\n");
              }
              else
              {
                     printf("%s", encrypted);
              }
       }

       BIO *newbio = BIO_new_mem_buf(privatekey, privlen);
       RSA *privRsa = NULL;
       PEM_read_bio_RSAPrivateKey(newbio, &privRsa, NULL, NULL);

       char p[4096] = { 0 };
       int rsasize = RSA_size(privRsa);
       if (privRsa != NULL)
       {
              int ret = RSA_private_decrypt(rsasize, (unsigned char*)encrypted, (unsigned char*)p, privRsa, RSA_PKCS1_OAEP_PADDING);
              if (ret == -1)
              {
                     int e = ERR_get_error();
                     char * err = ERR_error_string(ERR_get_error(), p);
                     printf("error");
              }
       }
}

Thanks,
Tiger

-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mta.openssl.org/pipermail/openssl-users/attachments/20190612/c309c5c2/attachment-0001.html>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: 152DDD620E3544FC80BD3B3AE89C45CB.png
Type: image/png
Size: 4386 bytes
Desc: 152DDD620E3544FC80BD3B3AE89C45CB.png
URL: <http://mta.openssl.org/pipermail/openssl-users/attachments/20190612/c309c5c2/attachment-0001.png>