[openssl-users] Unable to decrypt: RSA_R_PADDING_CHECK_FAILED 1.1.0

[M. Moore]

I'm creating a simple utility to encrypt and decrypt files using a key pair. I'm on Windows and coding against the 1.1.0 version of Openssl. I can load the key pair and encrypt the file fine, but when I try to decrypt EVP_PKEY_decrypt always returns -1. I traced this to the rsa padding check functions and they are returning -1 but I can't figure out why. I've tried changing the padding from RSA_PKCS1_OAEP_PADDING  to RSA_PKCS1_PADDING and still have the same problem. Any insight would be appreciated, here are my encrypt and decrypt functions:
#define FILE_BUFFER_LENGTH 1#define ENC_BUFFER_LENGTH 2048
int encryptfile(EVP_PKEY *key, FILE *srcfp, FILE *tgtfp) {		EVP_PKEY_CTX *ctx = EVP_PKEY_CTX_new(key, NULL);	char *inbuf;	unsigned char *outbuf;	size_t in_len = 0;	size_t out_len = ENC_BUFFER_LENGTH;	int x;
	inbuf = (char*)malloc(sizeof(char)*FILE_BUFFER_LENGTH+1);	outbuf = (char*)malloc(sizeof(char)*ENC_BUFFER_LENGTH+1);
	if (ctx == NULL) {		fprintf(stderr, "Error while creating encryption context.\n");		return 0;	}		if (EVP_PKEY_encrypt_init(ctx) <= 0) {		fprintf(stderr, "Error while initializing encryption context.\n");		return 0;	}		if (EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_PADDING) <= 0) {		fprintf(stderr, "Error while setting encryption padding.\n");		return 0;	}
	
	while (1) {		in_len = fread(inbuf, 1, FILE_BUFFER_LENGTH, srcfp);
		if (in_len == 0) {break;}
		if (EVP_PKEY_encrypt(ctx, outbuf, &out_len, inbuf, in_len) <= 0) {			fprintf(stderr, "Error while encrypting data.\n");			return 0;		}
		x = fwrite(outbuf, sizeof(char), in_len, tgtfp);
		if (x != in_len) {			fprintf(stderr, "Error while writing to target file.\n");			return 0;		}	}
	return 1;}

int decryptfile(EVP_PKEY *key, FILE *srcfp, FILE *tgtfp) {		ENGINE *e = ENGINE_new();	EVP_PKEY_CTX *ctx = EVP_PKEY_CTX_new(key, NULL);	unsigned char *inbuf;	unsigned char *outbuf;	size_t in_len = 0;	size_t out_len = ENC_BUFFER_LENGTH;	int x;
	inbuf = (char*)malloc(sizeof(char)*FILE_BUFFER_LENGTH + 1);	outbuf = (char*)malloc(sizeof(char)*ENC_BUFFER_LENGTH + 1);
	if (ctx == NULL) {		fprintf(stderr, "Error while creating decryption context.\n");		return 0;	}
	if (EVP_PKEY_decrypt_init(ctx) <= 0) {		fprintf(stderr, "Error while initializing decryption context.\n");		return 0;	}		if (EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_PADDING) <= 0) {		fprintf(stderr, "Error while setting decryption padding.\n");		return 0;	}


	while (1) {		in_len = fread(inbuf, 1, FILE_BUFFER_LENGTH, srcfp);
		if (in_len == 0) { break; }
		if (EVP_PKEY_decrypt(ctx, outbuf, &out_len, inbuf, in_len) <= 0) {			fprintf(stderr, "Error while decrypting data.\n");			return 0;		}
		x = fwrite(outbuf, sizeof(char), in_len, tgtfp);
		if (x != in_len) {			fprintf(stderr, "Error while writing decrypted data to target file.\n");			return 0;		}	}		
	return 1;}
-Mike M.
 		 	   		  
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mta.openssl.org/pipermail/openssl-users/attachments/20160913/3f46afc4/attachment.html>