[openssl-users] RSA engine is not using the functions mentioned in struct

[khurram ashraf]

I am new to making engines in openssl. Basically I want to implement an OpenSSL RSA engine that uses the functions I mentioned while encrypting and decrypting. My engine compiles and loads but it seems that it is not using the functions I want it to use for encryption and decryption.

#include <openssl/opensslconf.h>
#include <stdio.h>
#include <string.h>
#include <openssl/crypto.h>
#include <openssl/buffer.h>
#include <openssl/engine.h>
#include <openssl/rsa.h>
#include <openssl/bn.h>
#include <openssl/err.h>

static int
eng_rsa_pub_enc (int flen, const unsigned char *from,
     unsigned char *to, RSA * rsa, int padding)
{

    printf ("Engine is encrypting using pub key \n");
    RSA_public_encrypt (flen, from, to, rsa, RSA_PKCS1_PADDING);
}

static int
eng_rsa_pub_dec (int flen, const unsigned char *from,
     unsigned char *to, RSA * rsa, int padding)
{

   printf ("Engine is decrypting using pub key \n");
   RSA_public_decrypt (flen, from, to, rsa, RSA_PKCS1_PADDING);
}

static int
eng_rsa_priv_enc (int flen, const unsigned char *from, unsigned char *to,
      RSA * rsa, int padding __attribute__ ((unused)))
{
   printf ("Engine is encrypting using priv key \n");
   RSA_private_encrypt (flen, from, to, rsa, RSA_PKCS1_PADDING);
}


static int
eng_rsa_priv_dec (int flen, const unsigned char *from, unsigned char *to,
      RSA * rsa, int padding __attribute__ ((unused)))
{
   printf ("Engine is decrypting using priv key \n");
   RSA_private_decrypt (flen, from, to, rsa, RSA_PKCS1_PADDING);
}

/* Constants used when creating the ENGINE */
static const char *engine_rsa_id = "rsa-engine 1";
static const char *engine_rsa_name = "engine for testing 1";



static RSA_METHOD struct_rsa = {
"RSA engine for demo",
eng_rsa_pub_enc,
eng_rsa_pub_dec,
eng_rsa_priv_enc,
eng_rsa_priv_dec,
NULL,
NULL,
NULL,
NULL,
RSA_FLAG_CACHE_PUBLIC | RSA_FLAG_CACHE_PRIVATE,
NULL,
NULL,
NULL
};

static int bind (ENGINE * e, const char *id)
{
  printf ("%s\n", id);

  if (!ENGINE_set_id (e, engine_rsa_id) ||
  !ENGINE_set_name (e, engine_rsa_name) ||
  !ENGINE_set_RSA (e, &struct_rsa))
  return 0;

  return 1;
}

IMPLEMENT_DYNAMIC_BIND_FN (bind)
IMPLEMENT_DYNAMIC_CHECK_FN ()


I am compiling the code using following command.

gcc -fPIC -c rsa-engine.c
gcc -shared -o librsa_engine.so -lcrypto rsa-engine.o
openssl engine -t -c rsa_engine


Here the engine loads but when i try to encrypt a text file by using following command

openssl pkeyutl -encrypt -in message.txt -pubin -inkey pubkey-B.pem -engine rsa_engine -out cipher.bin
openssl pkeyutl -decrypt -in cipher.bin -inkey privkey-B.pem -engine rsa_engine -out rec.txt


It seems that it is not using the functions which I defined in the struct_rsa. It is also not giving the output from the printf in the function.

-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mta.openssl.org/pipermail/openssl-users/attachments/20170213/6a6fa571/attachment-0001.html>