[openssl-dev] [openssl.org #4429] Cannot decrypt RC4-encrypted CMS object

Viktor Dukhovni openssl-users at dukhovni.org
Tue Mar 15 06:33:32 UTC 2016


On Mon, Mar 14, 2016 at 10:34:17PM +0000, Dr. Stephen Henson wrote:

> > Is there any reason why stream ciphers are not supported with CMS?
> 
> Well one reason is that I'm not aware of any standard which defines how to use
> stream ciphers with CMS.
> 
> OpenSSL should really reject these with an appropriate error. 

Mind you, it seems that e.g. BouncyCastle supports CMS EnvelopedData
with RC4 (1.2.840.113549.3.4) as the AlgorithmIdentifier, and that
OpenSSL likely produces a compatible encoding (RC4 OID and no
parameters).

In which case it may suffice to handle absent parameters for ciphers
that don't need any, and RC4 might then "just work".

In crypto/cms/cms_enc.c we have:

    unsigned char iv[EVP_MAX_IV_LENGTH], *piv = NULL;
    ...
    if (enc) {
        int ivlen;
        calg->algorithm = OBJ_nid2obj(EVP_CIPHER_CTX_type(ctx));
        /* Generate a random IV if we need one */
        ivlen = EVP_CIPHER_CTX_iv_length(ctx);
        if (ivlen > 0) {
            if (RAND_bytes(iv, ivlen) <= 0)
                goto err;
            piv = iv;
        }
    } else if (EVP_CIPHER_asn1_to_param(ctx, calg->parameter) <= 0) {
        CMSerr(CMS_F_CMS_ENCRYPTEDCONTENT_INIT_BIO,
               CMS_R_CIPHER_PARAMETER_INITIALISATION_ERROR);
        goto err;
    }
    ...

    if (piv) {
        calg->parameter = ASN1_TYPE_new();
        if (calg->parameter == NULL) {
            CMSerr(CMS_F_CMS_ENCRYPTEDCONTENT_INIT_BIO, ERR_R_MALLOC_FAILURE);
            goto err;
        }
        if (EVP_CIPHER_param_to_asn1(ctx, calg->parameter) <= 0) {
            CMSerr(CMS_F_CMS_ENCRYPTEDCONTENT_INIT_BIO,
                   CMS_R_CIPHER_PARAMETER_INITIALISATION_ERROR);
            goto err;
        }
    }

which omits encoding parameters for ciphers with ivlen <= 0 when
encrypting (e.g. with RC4), but the first "else" clause insists on
valid parameters when decrypting.  So stream cipher support basically
boils down to what makes for valid parameters in EVP_CIPHER_asn1_param().

To that end, the below patch might make RC4 "work" (in master).
The semantic diff is quite small just return 1 when type == NULL
and we have a stream cipher with no get_asn1_parameters method.
The patch is larger because I took the opportunity to reorganize
the code a bit:

    int EVP_CIPHER_asn1_to_param(EVP_CIPHER_CTX *c, ASN1_TYPE *type)
    {
	if (c->cipher->get_asn1_parameters != NULL)
	    return c->cipher->get_asn1_parameters(c, type);

	if (!(c->cipher->flags & EVP_CIPH_FLAG_DEFAULT_ASN1)) {
	    if (type == NULL &&
		EVP_CIPHER_CTX_mode(c) == EVP_CIPH_STREAM_CIPHER)
		return 1;
	    return -1;
	}

	switch (EVP_CIPHER_CTX_mode(c)) {
	default:
	    return EVP_CIPHER_get_asn1_iv(c, type);

	case EVP_CIPH_WRAP_MODE:
	    return 1;

	case EVP_CIPH_GCM_MODE:
	case EVP_CIPH_CCM_MODE:
	case EVP_CIPH_XTS_MODE:
	case EVP_CIPH_OCB_MODE:
	    return -1;
	}
    }

This is completely untested, may not even compile!  Enjoy.

-- 
	Viktor.

diff --git a/crypto/evp/evp_lib.c b/crypto/evp/evp_lib.c
index bc24d5a..8957de2 100644
--- a/crypto/evp/evp_lib.c
+++ b/crypto/evp/evp_lib.c
@@ -93,31 +93,29 @@ int EVP_CIPHER_param_to_asn1(EVP_CIPHER_CTX *c, ASN1_TYPE *type)
 
 int EVP_CIPHER_asn1_to_param(EVP_CIPHER_CTX *c, ASN1_TYPE *type)
 {
-    int ret;
-
     if (c->cipher->get_asn1_parameters != NULL)
-        ret = c->cipher->get_asn1_parameters(c, type);
-    else if (c->cipher->flags & EVP_CIPH_FLAG_DEFAULT_ASN1) {
-        switch (EVP_CIPHER_CTX_mode(c)) {
+        return c->cipher->get_asn1_parameters(c, type);
 
-        case EVP_CIPH_WRAP_MODE:
-            ret = 1;
-            break;
+    if (!(c->cipher->flags & EVP_CIPH_FLAG_DEFAULT_ASN1)) {
+        if (type == NULL &&
+            EVP_CIPHER_CTX_mode(c) == EVP_CIPH_STREAM_CIPHER)
+            return 1;
+        return -1;
+    }
 
-        case EVP_CIPH_GCM_MODE:
-        case EVP_CIPH_CCM_MODE:
-        case EVP_CIPH_XTS_MODE:
-        case EVP_CIPH_OCB_MODE:
-            ret = -1;
-            break;
+    switch (EVP_CIPHER_CTX_mode(c)) {
+    default:
+        return EVP_CIPHER_get_asn1_iv(c, type);
 
-        default:
-            ret = EVP_CIPHER_get_asn1_iv(c, type);
-            break;
-        }
-    } else
-        ret = -1;
-    return (ret);
+    case EVP_CIPH_WRAP_MODE:
+        return 1;
+
+    case EVP_CIPH_GCM_MODE:
+    case EVP_CIPH_CCM_MODE:
+    case EVP_CIPH_XTS_MODE:
+    case EVP_CIPH_OCB_MODE:
+        return -1;
+    }
 }
 
 int EVP_CIPHER_get_asn1_iv(EVP_CIPHER_CTX *c, ASN1_TYPE *type)


More information about the openssl-dev mailing list