[openssl-dev] Calculating DTLS payload MTU

[David Woodhouse]

On Wed, 2016-10-05 at 14:40 +0100, David Woodhouse wrote:
> How's this for a start...

Now I think I have it right for CCM too, although having to use
strstr() for that makes me *very* sad. Next up, Chacha20-Poly1305...
and then maybe I can stop worrying about new modes and ciphersuites
because those won't be added in OpenSSL 1.1 and we can get OpenSSL do
to this for itself before 1.2? :)

/* sets the DTLS MTU and returns the actual tunnel MTU */
unsigned dtls_set_mtu(struct openconnect_info *vpninfo, unsigned mtu)
{
	int tun_mtu;
	int ivlen, maclen, blocksize = 1, pad = 0;

#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
	/* OpenSSL <= 1.0.2 only supports CBC ciphers with PSK */
	ivlen = EVP_CIPHER_iv_length(EVP_CIPHER_CTX_cipher(vpninfo->dtls_ssl->enc_write_ctx));
	maclen = EVP_MD_CTX_size(vpninfo->dtls_ssl->write_hash);
	blocksize = ivlen;
	pad = 1;
#else
	/* Now it gets more fun... */
	const SSL_CIPHER *s_ciph = SSL_get_current_cipher(vpninfo->dtls_ssl);
	const EVP_CIPHER *e_ciph;
	const EVP_MD *e_md;
	char wtf[128];

	e_ciph = EVP_get_cipherbynid(SSL_CIPHER_get_cipher_nid(s_ciph));

	switch (EVP_CIPHER_mode(e_ciph)) {
	case EVP_CIPH_GCM_MODE:
		ivlen = EVP_GCM_TLS_EXPLICIT_IV_LEN;
		maclen = EVP_GCM_TLS_TAG_LEN;
		blocksize = 1;
		pad = 0;
		break;

	case EVP_CIPH_CCM_MODE:
		ivlen = EVP_CCM_TLS_EXPLICIT_IV_LEN;
		SSL_CIPHER_description(s_ciph, wtf, sizeof(wtf));
		if (strstr(wtf, "CCM8"))
			maclen = 8;
		else
			maclen = 16;
		blocksize = 1;
		pad = 0;
		break;

	case EVP_CIPH_CBC_MODE:
		e_md = EVP_get_digestbynid(SSL_CIPHER_get_digest_nid(s_ciph));
		blocksize = EVP_CIPHER_block_size(e_ciph);
		ivlen = EVP_CIPHER_iv_length(e_ciph);
		pad = 1;
		maclen = EVP_MD_size(e_md);
		break;

	case EVP_CIPH_STREAM_CIPHER:
		/* Seen with PSK-CHACHA20-POLY1305 */
	default:
		printf("wtf mode %d\n", EVP_CIPHER_mode(e_ciph));
		// XXX
		;
	}
#endif


	/* Take off the explicit IV and the MAC (XXX: overflow!) */
	printf("iv %d mac %d blk %d\n", ivlen, maclen, blocksize);
	tun_mtu = mtu - DTLS1_RT_HEADER_LENGTH - ivlen - maclen;
	/* For block cipher modes round down to blocksize */
	printf("tun %d & 0x%x == %d\n", tun_mtu, ~(blocksize-1), tun_mtu & (~(blocksize-1)));
	tun_mtu -= tun_mtu % blocksize;
	/* ... and CBC modes require at least one byte to indicate padding length */
	tun_mtu -= pad;

	DTLS_set_link_mtu(vpninfo->dtls_ssl, mtu);

	/* We already set the link MTU, but hopefully by the time we
	 * finish it, this function will be better at working out the
	 * actual tunnel MTU than OpenSSL is. So do that too... */
	SSL_set_mtu(vpninfo->dtls_ssl, tun_mtu);

	return tun_mtu;
}

-- 
dwmw2
-------------- next part --------------
A non-text attachment was scrubbed...
Name: smime.p7s
Type: application/x-pkcs7-signature
Size: 5760 bytes
Desc: not available
URL: <http://mta.openssl.org/pipermail/openssl-dev/attachments/20161005/e9079fd8/attachment.bin>