[openssl-users] reasons for negative return value from BIO_do_connect

[Richard Welty]

i'm trying to figure out why BIO_do_connect is failing with
a negative value. so far as i know i've done all preliminaries
correctly, but apparently i've missed something. i am dumping
the SSL errors from ERR_get_error but am not getting any in
this instance.

OpenSSL version is 1.1.0h

Ubuntu 18.04 running in a VM on a Mac Mini. the network is setup
in bridged adapter mode so that the VM has a distinct IP address
from the host. i have used nmap to verify that i can reach the
target server and that the port the server is listening on is
open. the owner of the server assures me that his app is up and
listening.

i have verified with both wireshark and by looking at the logs
on the server i'm trying to reach that no actual traffic went out.

the following is output from the test run. note that i print out
the values of IP and port using BIO_get_conn_hostname and
BIO_get_conn_port to insure that i did all the processing
correctly.

TLS method obtained
TLS context established
NTS: options and ALPN protos set
NTS: remote peer bio created
141.41.241.68
NTS: remote peer hostname set to '141.41.241.68'
NTS: server port set to '443'
NTS: ssl bio obtained
NTS: ciphers set
16 Oct 13:15:39 ntpd[4439]: failed to connect to server

here is the code which is failing:

struct tls_connection* nts_tls_connect( struct peer *peer){

  long result;
  struct tls_connection* connection = malloc( sizeof (struct
tls_connection));
  connection->sock = -1;
  connection->ctx = NULL;
  connection->remote_peer = NULL;
  connection->ssl = NULL;
  connection->ke_buffer = malloc( MAX_KE_RESPONSE);

  const SSL_METHOD* method = TLSv1_2_client_method();
  if( method == NULL){
    msyslog( LOG_ERR, "Failed to obtain TLS 1.2 method from OpenSSL");
    free_tls_connection( connection);
    return NULL;
  }

  DPRINTF( 1, ("TLS method obtained\n"));

  connection->ctx = SSL_CTX_new( method);
  if( connection->ctx == NULL){
    msyslog( LOG_ERR, "Failed to establish TLS context");
    dprintf_ssl_error();
    free_tls_connection( connection);
    return NULL;
  }

  DPRINTF( 1, ("TLS context established\n"));

  // verification turned off for proof of concept
  // SSL_CTX_set_verify( connection->ctx, SSL_VERIFY_PEER, NULL);
  // SSL_CTX_set_verify_depth( connection->ctx, 4);

  const long flags
    = SSL_OP_NO_SSLv2
    | SSL_OP_NO_SSLv3
    | SSL_OP_NO_TLSv1
    | SSL_OP_NO_TLSv1_1
    | SSL_OP_NO_COMPRESSION;
  SSL_CTX_set_options( connection->ctx, flags);

  result = SSL_CTX_set_alpn_protos( connection->ctx,
				 alpn_ntske,
				 sizeof( alpn_ntske));
  if( result != 0){
    msyslog( LOG_ERR, "failed to set ALPN protos");
    dprintf_ssl_error();
    free_tls_connection( connection);
    return NULL;
  }

  DPRINTF( 1, ("NTS: options and ALPN protos set\n"));

  // verification turned off for hackathon
  // SSL_CTX_load_verify_locations...

  connection->remote_peer = BIO_new_ssl_connect( connection->ctx);
  if( connection->remote_peer == NULL){
    msyslog( LOG_ERR, "failed to create remote_peer BIO chain");
    free_tls_connection( connection);
    return NULL;
  }

  DPRINTF( 1, ("NTS: remote peer bio created\n"));

  if( peer->hostname != NULL){
    result = BIO_set_conn_hostname( connection->remote_peer,
peer->hostname);
  } else {
    char *addr = address_string( &peer->srcadr);
    result = BIO_set_conn_hostname( connection->remote_peer,addr);
    free( addr);
  }
  if( result != 1){
    msyslog( LOG_ERR, "failed to set remote peer hostname");
    free_tls_connection( connection);
    return NULL;
  }

  DPRINTF( 1, ("NTS: remote peer hostname set to \'%s\'\n",
	       BIO_get_conn_hostname( connection->remote_peer)));

  result = BIO_set_conn_port( connection->remote_peer, NTS_KE_PORT);
  if( result != 1){
    msyslog( LOG_ERR, "failed to set server port");
    free_tls_connection( connection);
    return NULL;
  }

  DPRINTF( 1, ("NTS: server port set to \'%s\'\n",
	       BIO_get_conn_port( connection->remote_peer)));

  BIO_get_ssl( connection->remote_peer, &connection->ssl);
  if( connection->ssl == NULL){
    msyslog( LOG_ERR, "failed to get SSL from BIO");
    free_tls_connection( connection);
    return NULL;
  }

  DPRINTF( 1, ("NTS: ssl bio obtained\n"));

  const char* PREFERRED_CIPHERS = "HIGH:AES256:AES128";
  result = SSL_set_cipher_list( connection->ssl, PREFERRED_CIPHERS);
  if( result != 1){
    msyslog( LOG_ERR, "failed to set cipher list");
    dprintf_ssl_error();
    free_tls_connection( connection);
    return NULL;
  }

  DPRINTF( 1, ("NTS: ciphers set\n"));

    /*
   connection->evbase = event_base_new();
  struct buffer_event *bev
    = bufferevent_openssl_socket_new( connection->evbase,
				      -1, connection->ssl,
				      BUFFEREVENT_SSL_CONNECTING,
				      BEV_OPT_CLOSE_ON_FREE);
  */

  result = BIO_do_connect( connection->remote_peer);
  if( result <= 0){
    msyslog( LOG_ERR, "failed to connect to server");
    dprintf_ssl_error();
    free_tls_connection( connection);
    return NULL;
  }