[openssl-dev] Calculating DTLS payload MTU

[Matt Caswell]

On 05/10/16 15:37, David Woodhouse wrote:
> On Wed, 2016-10-05 at 14:40 +0100, David Woodhouse wrote:
>> How's this for a start...
> 
> Now I think I have it right for CCM too, although having to use
> strstr() for that makes me *very* sad. Next up, Chacha20-Poly1305...
> and then maybe I can stop worrying about new modes and ciphersuites
> because those won't be added in OpenSSL 1.1 and we can get OpenSSL do
> to this for itself before 1.2? :)

Or even 1.1.1! Why don't you pull this together into a github PR?

Matt


> 
> /* sets the DTLS MTU and returns the actual tunnel MTU */
> unsigned dtls_set_mtu(struct openconnect_info *vpninfo, unsigned mtu)
> {
> 	int tun_mtu;
> 	int ivlen, maclen, blocksize = 1, pad = 0;
> 
> #if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
> 	/* OpenSSL <= 1.0.2 only supports CBC ciphers with PSK */
> 	ivlen = EVP_CIPHER_iv_length(EVP_CIPHER_CTX_cipher(vpninfo->dtls_ssl->enc_write_ctx));
> 	maclen = EVP_MD_CTX_size(vpninfo->dtls_ssl->write_hash);
> 	blocksize = ivlen;
> 	pad = 1;
> #else
> 	/* Now it gets more fun... */
> 	const SSL_CIPHER *s_ciph = SSL_get_current_cipher(vpninfo->dtls_ssl);
> 	const EVP_CIPHER *e_ciph;
> 	const EVP_MD *e_md;
> 	char wtf[128];
> 
> 	e_ciph = EVP_get_cipherbynid(SSL_CIPHER_get_cipher_nid(s_ciph));
> 
> 	switch (EVP_CIPHER_mode(e_ciph)) {
> 	case EVP_CIPH_GCM_MODE:
> 		ivlen = EVP_GCM_TLS_EXPLICIT_IV_LEN;
> 		maclen = EVP_GCM_TLS_TAG_LEN;
> 		blocksize = 1;
> 		pad = 0;
> 		break;
> 
> 	case EVP_CIPH_CCM_MODE:
> 		ivlen = EVP_CCM_TLS_EXPLICIT_IV_LEN;
> 		SSL_CIPHER_description(s_ciph, wtf, sizeof(wtf));
> 		if (strstr(wtf, "CCM8"))
> 			maclen = 8;
> 		else
> 			maclen = 16;
> 		blocksize = 1;
> 		pad = 0;
> 		break;
> 
> 	case EVP_CIPH_CBC_MODE:
> 		e_md = EVP_get_digestbynid(SSL_CIPHER_get_digest_nid(s_ciph));
> 		blocksize = EVP_CIPHER_block_size(e_ciph);
> 		ivlen = EVP_CIPHER_iv_length(e_ciph);
> 		pad = 1;
> 		maclen = EVP_MD_size(e_md);
> 		break;
> 
> 	case EVP_CIPH_STREAM_CIPHER:
> 		/* Seen with PSK-CHACHA20-POLY1305 */
> 	default:
> 		printf("wtf mode %d\n", EVP_CIPHER_mode(e_ciph));
> 		// XXX
> 		;
> 	}
> #endif
> 
> 
> 	/* Take off the explicit IV and the MAC (XXX: overflow!) */
> 	printf("iv %d mac %d blk %d\n", ivlen, maclen, blocksize);
> 	tun_mtu = mtu - DTLS1_RT_HEADER_LENGTH - ivlen - maclen;
> 	/* For block cipher modes round down to blocksize */
> 	printf("tun %d & 0x%x == %d\n", tun_mtu, ~(blocksize-1), tun_mtu & (~(blocksize-1)));
> 	tun_mtu -= tun_mtu % blocksize;
> 	/* ... and CBC modes require at least one byte to indicate padding length */
> 	tun_mtu -= pad;
> 
> 	DTLS_set_link_mtu(vpninfo->dtls_ssl, mtu);
> 
> 	/* We already set the link MTU, but hopefully by the time we
> 	 * finish it, this function will be better at working out the
> 	 * actual tunnel MTU than OpenSSL is. So do that too... */
> 	SSL_set_mtu(vpninfo->dtls_ssl, tun_mtu);
> 
> 	return tun_mtu;
> }
>