Is there a simple implementation of hooking external private key with openssl-3.0 API?

[Alon Bar-Lev]

Hello,

I am trying to migrate to openssl-3.0 API, it seems to be very
complicated to hook primitive private key usage to a custom function.
This is required, for example, to use private keys that reside on
hardware devices or when an application wishes to externalize private
key operations to other programs using IPC.

I hope I am wrong but it seems like an entirely new provider must be
implemented with complete reimplementation of the default providers,
including serialization, padding etc... While in openssl-0/1 it was
quite easy.

I wrote a testcase program using openssl-1 APIs[1] which also works
using openssl-3, in this testcase I prepare a new RSA method based on
the default method, hook the private operations and then hook the RSA
object to use the custom method.

I am looking for a way to implement the __hook_evp_pkey function in
openssl-3 api, so that when a private key operation is executed on the
EVP_PKEY or EVP_PKEY_CTX a custom callback will be executed while
public key operation continue to be executed normally.

While looking into the existing RSA providers I can see that the
providers continue to use the deprecated RSA_* functions with the
following comment:

    /*
     * RSA low level APIs are deprecated for public use, but still ok for
     * internal use.
     */

This is exactly what I need... :) To have the RSA low level API be
redirected back to the application so that I can enjoy the default
implementation of signature/rsa_sig.c padding etc while being able to
override the private encrypt. But these low level functions are hidden
from the user.

Can anyone help us to create a testcase of openssl-3? This will help
many applications such as opensc/libp11 opensc/pkcs11-helper openvpn
and probably more.

For your convenience, you may find the program here[1].

Regards,
Alon Bar-Lev

[1] https://github.com/alonbl/openssl-external/blob/master/example.c

---

#include <openssl/err.h>
#include <openssl/evp.h>
#include <openssl/pem.h>
#include <openssl/rsa.h>
#include <string.h>
#include <stdio.h>

static RSA_METHOD *__example_rsa_method;
static int __example_rsa_index;

static int __example_rsa_priv_enc(int flen, const unsigned char *from,
unsigned char *to, RSA *rsa, int padding) {
    const RSA_METHOD *rsa_method = NULL;
    int ret = -1;

    if ((rsa_method = RSA_get_method(rsa)) == NULL) {
                goto cleanup;
        }

    /*
     * Do it.
     */
    printf("ENCRYPT\n");
    memset(to, 0, flen);
    ret = 1;

cleanup:

    return ret;
}

static int __example_rsa_priv_dec(int flen, const unsigned char *from,
unsigned char *to, RSA *rsa, int padding) {
    const RSA_METHOD *rsa_method = NULL;
    int ret = -1;

    if ((rsa_method = RSA_get_method(rsa)) == NULL) {
                goto cleanup;
        }

    /*
     * Do it.
     */
    printf("DECRYPT\n");
    memset(to, 0, flen);
    ret = 1;

cleanup:

    return ret;
}


static int __prepare_method(void) {
    int ret = 0;

    if ((__example_rsa_method =
RSA_meth_dup(RSA_get_default_method())) == NULL) {
        goto cleanup;
    }

    if (!RSA_meth_set1_name(__example_rsa_method, "example")) {
        goto cleanup;
    }

    if (!RSA_meth_set_priv_dec(__example_rsa_method, __example_rsa_priv_dec)) {
        goto cleanup;
    }

    if (!RSA_meth_set_priv_enc(__example_rsa_method, __example_rsa_priv_enc)) {
        goto cleanup;
    }

    if ((__example_rsa_index = RSA_get_ex_new_index(0, "example",
NULL, NULL, NULL)) == -1) {
        goto cleanup;
    }

    ret = 1;

cleanup:

    return ret;
}

static int __free_method(void) {
    RSA_meth_free(__example_rsa_method);
}

static int __hook_evp_pkey(EVP_PKEY *evp_pkey) {

    RSA *rsa = NULL;
    int ret = 0;

    /*
     * Hook private key methods
     */

    if (EVP_PKEY_id(evp_pkey) != EVP_PKEY_RSA) {
        goto cleanup;
    }

    if ((rsa = EVP_PKEY_get1_RSA(evp_pkey)) == NULL) {
        goto cleanup;
    }

    if (!RSA_set_method(rsa, __example_rsa_method)) {
        goto cleanup;
    }

    if (!RSA_set_ex_data(rsa, __example_rsa_index, "mystate")) {
        goto cleanup;
    }

    if (EVP_PKEY_set1_RSA(evp_pkey, rsa) != 1) {
        goto cleanup;
    }

    ret = 1;

cleanup:

    RSA_free(rsa);

    return ret;
}

const static char *pem = (
    "-----BEGIN CERTIFICATE-----\n"
    "MIIFMDCCBBigAwIBAgISA6sbShb1HQ3TpSVvhSPOS4JJMA0GCSqGSIb3DQEBCwUA\n"
    "MDIxCzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1MZXQncyBFbmNyeXB0MQswCQYDVQQD\n"
    "EwJSMzAeFw0yMjAzMTAxNzQ4MDdaFw0yMjA2MDgxNzQ4MDZaMBoxGDAWBgNVBAMT\n"
    "D210YS5vcGVuc3NsLm9yZzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB\n"
    "AMZvA0BbvdyVc+06j5e5k6dUr8gqL0KZw0w4xJ0QD6jD/o+czNEMz13YDxuZ5utL\n"
    "YGq8uohlK8l2DWqvDfGfm1T4VYQhD2z0Ky0JDTsxDIb5i6kKA+o2j2VPAivfMkBp\n"
    "f47rLITa4vqZ8/aro3E0ZVWfbpOOGASteM/g9mLEpRLJQA2/o4uu9xLCsyJkLG8F\n"
    "8eTCHUJ8388ZO/3fv8LnN1+/WwciSYcZcZNN44OsrgLNoLh6dzSY+oNZyVGdqxUy\n"
    "ZSO2dURx4/28w26RLzXFnGOZinupE6KoVhCHHM0Wqx7YkfudymzwBCPP3+X4Hkab\n"
    "1gkZZO9wTpRKrhuW3XtaBMkCAwEAAaOCAlYwggJSMA4GA1UdDwEB/wQEAwIFoDAd\n"
    "BgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNV\n"
    "HQ4EFgQUW/ht3YVQnVmfAWGArMLkgIyUFNYwHwYDVR0jBBgwFoAUFC6zF7dYVsuu\n"
    "UAlA5h+vnYsUwsYwVQYIKwYBBQUHAQEESTBHMCEGCCsGAQUFBzABhhVodHRwOi8v\n"
    "cjMuby5sZW5jci5vcmcwIgYIKwYBBQUHMAKGFmh0dHA6Ly9yMy5pLmxlbmNyLm9y\n"
    "Zy8wJwYDVR0RBCAwHoIPbXRhLm9wZW5zc2wub3JnggtvcGVuc3NsLm9yZzBMBgNV\n"
    "HSAERTBDMAgGBmeBDAECATA3BgsrBgEEAYLfEwEBATAoMCYGCCsGAQUFBwIBFhpo\n"
    "dHRwOi8vY3BzLmxldHNlbmNyeXB0Lm9yZzCCAQMGCisGAQQB1nkCBAIEgfQEgfEA\n"
    "7wB2AEHIyrHfIkZKEMahOglCh15OMYsbA+vrS8do8JBilgb2AAABf3Uo37wAAAQD\n"
    "AEcwRQIhAMDDz1KXMWXblh9maYNLF6vlZOcSXNlp3RgxJhRBYhACAiBB8mU+mqDa\n"
    "8RNog7zLQq3426vcfH4r1wufDnQ0su3GyQB1ACl5vvCeOTkh8FZzn2Old+W+V32c\n"
    "YAr4+U1dJlwlXceEAAABf3Uo4ZoAAAQDAEYwRAIgVD5+n6KMePTQF2GN4ZKIE8Oz\n"
    "lzZPeY90EPY5APu3ZrECIE4HWJ/ZQ/qZ3/7x4Vo+1a1gPoPBM4rsh3d3ormsrkiW\n"
    "MA0GCSqGSIb3DQEBCwUAA4IBAQA+TYBjasfMBLlXbwNdYGaVtfbBKyPPhHFHOqi2\n"
    "iJfdRnx2Z/KS0gmBisD6SS62dKAjHrUy4wSfRTSpAHAOvo3n7BuYSE+3HIYwyFpB\n"
    "P54tJTiEYiAHJvWsPRl8rEqxzYnaR+u0zdKL7Wauk9gJMwGX6fdwhhAgS5WmBe05\n"
    "O4mf8jdWgtLQYxS/kvQYrNDTTBA6J+UoNM/JIxXENMh2/6zcFgy0D2ewr0NjAYWU\n"
    "Ylf5jVgHjxleRSGnbt19v8dwZcHyBhq+vdndQt0sDQl7aoNEKiCXU2/y0KAtDjGF\n"
    "tsFic9a3WMzENWlAUcfACBaGx8Qm9161M9BO396tgHavQLQ8\n"
    "-----END CERTIFICATE-----\n"
);

int main(void) {
    BIO *bio = NULL;
    X509 *x509 = NULL;
    EVP_PKEY *evp_pkey = NULL;
    EVP_PKEY_CTX *evp_pkey_ctx = NULL;
    int ret = 1;

    if (__prepare_method() < 1) {
        goto cleanup;
    }

    if ((bio = BIO_new_mem_buf(pem, strlen(pem))) == NULL) {
        goto cleanup;
    }

    if ((x509 = PEM_read_bio_X509(bio, NULL, NULL, NULL)) == NULL) {
        goto cleanup;
    }

    if ((evp_pkey = X509_get_pubkey(x509)) == NULL) {
        goto cleanup;
    }

    if (__hook_evp_pkey(evp_pkey) < 1) {
        goto cleanup;
    }

#if OPENSSL_VERSION_NUMBER < 0x30000000
    if ((evp_pkey_ctx = EVP_PKEY_CTX_new(evp_pkey, NULL)) == NULL) {
        goto cleanup;
    }
#else
    if ((evp_pkey_ctx = EVP_PKEY_CTX_new_from_pkey(NULL, evp_pkey,
NULL)) == NULL) {
        goto cleanup;
    }
#endif

    if (EVP_PKEY_sign_init(evp_pkey_ctx) < 1) {
        goto cleanup;
    }

    {
        char buf[1024];
        size_t len = sizeof(buf);
        if (EVP_PKEY_sign(evp_pkey_ctx, buf, &len, "Test", 4) < 1) {
            goto cleanup;
        }
    }

    ret = 0;

cleanup:

    ERR_print_errors_fp(stdout);

    EVP_PKEY_CTX_free(evp_pkey_ctx);
    EVP_PKEY_free(evp_pkey);
    X509_free(x509);
    BIO_free(bio);

    __free_method();

    return ret;
}