<html>
<head>
<style><!--
.hmmessage P
{
margin:0px;
padding:0px
}
body.hmmessage
{
font-size: 12pt;
font-family:Calibri
}
--></style></head>
<body class='hmmessage'><div dir='ltr'><div>I'm creating a simple utility to encrypt and decrypt files using a key pair. I'm on Windows and coding against the 1.1.0 version of Openssl. I can load the key pair and encrypt the file fine, but when I try to decrypt EVP_PKEY_decrypt always returns -1. I traced this to the rsa padding check functions and they are returning -1 but I can't figure out why. I've tried changing the padding from RSA_PKCS1_OAEP_PADDING to RSA_PKCS1_PADDING and still have the same problem. Any insight would be appreciated, here are my encrypt and decrypt functions:</div><div><br></div><div>#define <span style="font-size: 12pt;">FILE_BUFFER_LENGTH 1</span></div><div><span style="font-size: 12pt;">#define </span><span style="font-size: 12pt;">ENC_BUFFER_LENGTH 2048</span></div><div><span style="font-size: 12pt;"><br></span></div><div><div>int encryptfile(EVP_PKEY *key, FILE *srcfp, FILE *tgtfp) {</div><div><span class="Apple-tab-span" style="white-space:pre"> </span></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>EVP_PKEY_CTX *ctx = EVP_PKEY_CTX_new(key, NULL);</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>char *inbuf;</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>unsigned char *outbuf;</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>size_t in_len = 0;</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>size_t out_len = ENC_BUFFER_LENGTH;</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>int x;</div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>inbuf = (char*)malloc(sizeof(char)*FILE_BUFFER_LENGTH+1);</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>outbuf = (char*)malloc(sizeof(char)*ENC_BUFFER_LENGTH+1);</div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>if (ctx == NULL) {</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>fprintf(stderr, "Error while creating encryption context.\n");</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>return 0;</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>}</div><div><span class="Apple-tab-span" style="white-space:pre"> </span></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>if (EVP_PKEY_encrypt_init(ctx) <= 0) {</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>fprintf(stderr, "Error while initializing encryption context.\n");</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>return 0;</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>}</div><div><span class="Apple-tab-span" style="white-space:pre"> </span></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>if (EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_PADDING) <= 0) {</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>fprintf(stderr, "Error while setting encryption padding.\n");</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>return 0;</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>}</div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span></div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>while (1) {</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>in_len = fread(inbuf, 1, FILE_BUFFER_LENGTH, srcfp);</div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>if (in_len == 0) {break;}</div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>if (EVP_PKEY_encrypt(ctx, outbuf, &out_len, inbuf, in_len) <= 0) {</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>fprintf(stderr, "Error while encrypting data.\n");</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>return 0;</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>}</div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>x = fwrite(outbuf, sizeof(char), in_len, tgtfp);</div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>if (x != in_len) {</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>fprintf(stderr, "Error while writing to target file.\n");</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>return 0;</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>}</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>}</div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>return 1;</div><div>}</div></div><div><br></div><div><br></div><div><div>int decryptfile(EVP_PKEY *key, FILE *srcfp, FILE *tgtfp) {</div><div><span class="Apple-tab-span" style="white-space:pre"> </span></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>ENGINE *e = ENGINE_new();</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>EVP_PKEY_CTX *ctx = EVP_PKEY_CTX_new(key, NULL);</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>unsigned char *inbuf;</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>unsigned char *outbuf;</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>size_t in_len = 0;</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>size_t out_len = ENC_BUFFER_LENGTH;</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>int x;</div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>inbuf = (char*)malloc(sizeof(char)*FILE_BUFFER_LENGTH + 1);</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>outbuf = (char*)malloc(sizeof(char)*ENC_BUFFER_LENGTH + 1);</div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>if (ctx == NULL) {</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>fprintf(stderr, "Error while creating decryption context.\n");</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>return 0;</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>}</div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>if (EVP_PKEY_decrypt_init(ctx) <= 0) {</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>fprintf(stderr, "Error while initializing decryption context.\n");</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>return 0;</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>}</div><div><span class="Apple-tab-span" style="white-space:pre"> </span></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>if (EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_PADDING) <= 0) {</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>fprintf(stderr, "Error while setting decryption padding.\n");</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>return 0;</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>}</div><div><br></div><div><br></div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>while (1) {</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>in_len = fread(inbuf, 1, FILE_BUFFER_LENGTH, srcfp);</div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>if (in_len == 0) { break; }</div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>if (EVP_PKEY_decrypt(ctx, outbuf, &out_len, inbuf, in_len) <= 0) {</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>fprintf(stderr, "Error while decrypting data.\n");</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>return 0;</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>}</div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>x = fwrite(outbuf, sizeof(char), in_len, tgtfp);</div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>if (x != in_len) {</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>fprintf(stderr, "Error while writing decrypted data to target file.\n");</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>return 0;</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>}</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>}</div><div><span class="Apple-tab-span" style="white-space:pre"> </span></div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>return 1;</div><div>}</div></div><div><br></div><div>-Mike M.</div><div><br></div> </div></body>
</html>