[openssl-users] Query regarding DTLS handshake

[mahesh gs]

Hi,

We are running SCTP connections with DTLS enabled in our application. We
have adapted openssl version (openssl-1.1.0e) to achieve the same.

We have generated the self signed root and node certificates for testing.
We have a strange problem with the incomplete DTLS handshake if we run the
DTLS client and DTLS server is different systems.If we run the DTLS client
and server in same system handshake is successful, handshake is not
successful if run client and server in different VM's.

This strange problem happens only for SCTP/DTLS connection. With the same
set of certificates TCP/TLS connection is successful and we are able to
exchange the application data.

I am attaching the code bits for SSL_accept and SSL_connect and also the
wireshark trace of unsuccessful handshake. Please assist me to debug this
problem.

SSL_accept returns  SSL_ERROR_WANT_READ(2) infinite times but SSL_connect
is called 4 or 5 times and select system call timeout.

Thanks,
Mahesh G S
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mta.openssl.org/pipermail/openssl-users/attachments/20170413/11d60744/attachment-0001.html>
-------------- next part --------------

    bool pollSocketForEvents(long aTlsError)
    {
      /* This function is to implement the SSL Socket call behaviour
              http://jmarshall.com/stuff/handling-nbio-errors-in-openssl.html */
              
      fd_set readFds, writeFds;
      struct timeval timeout;
      int retValue;
      
      int nfds = getSocketId();
      
      FD_SET(nfds, &readFds);
      FD_SET(nfds, &writeFds);

      /* Wait for 5 Seconds */
      timeout.tv_usec = 0;
      timeout.tv_sec = 5;

      if (SSL_ERROR_WANT_READ == aTlsError)
      {
        retValue = select(nfds + 1, &readFds, NULL, NULL, &timeout);
        if (retValue <= 0)
        {
          // Timeout or error just return failure
          return false;
        }
      }

      if (SSL_ERROR_WANT_WRITE == aTlsError)
      {
        retValue = select(nfds + 1, NULL, &writeFds, NULL, &timeout);
        if (retValue <= 0)
        { 
          // Timeout or error just return failure
          return false;
        }
      }

      return true;
    }   


    void acceptTlsConnection()
    {   
        TRACE_CALL_BEGIN_MGR (ivManager);

        long aTlsError;
        bool retry = true;
        const char *aFile;
        int aLine;
        long aRetValue;
                  
        /* Create an SSL Session for server */
        if (createSSLSession(false))
        {
          /* Throw failure exception */
          TRACE_CALL_END_MGR (ivManager);    
          throw (TLSException(TLSException::eSSLContextCreationFailure, " Error while creating context for accepted connection"));            
        }

        do
        {
          /* Clear SSL error queue */
          ERR_clear_error();

          /* Initiate SSL Handshake */
          aRetValue = SSL_accept(ivSSL);

          if (aRetValue <= 0)
          {
            aTlsError = SSL_get_error(ivSSL, aRetValue);          

            switch(aTlsError)
            {
              case SSL_ERROR_WANT_READ:                
              case SSL_ERROR_WANT_WRITE:
              {
                  /* Select on the socket for read/write events */
                  retry = pollSocketForEvents(aTlsError);

                  /* Nothing to do retry for accepting new connection */
                  INSD_LOG(ivManager, INSD_LOG_INFO," SSL_accept() fails to accept new connection "
                  "need to retry, returned error code %d ", aTlsError);
              }
              break;

              case SSL_ERROR_SYSCALL:

              if (EWOULDBLOCK == errno || EAGAIN == errno)
              {
                /* Nothing to do retry for accepting new connection */
                INSD_LOG(ivManager, INSD_LOG_INFO," SSL_accept() fails to accept new connection "
                "need to retry, returned error code %d ", aTlsError);
              }
              else
              {
                int aRet = ERR_get_error_line(&aFile, &aLine);
                
                INSD_LOG(ivManager, INSD_LOG_ERROR," SSL File : %s , Line number : %d , "
                  "Socket Id %d, Linux Error Code %d", aFile, aLine, getSocketId(), errno);
                
                INSD_LOG(ivManager, INSD_LOG_ERROR,"SSL_accept () :: Result Code : %d ", aTlsError);

                retry = false;
              }

              break;

              default:
              {
                int aRet = ERR_get_error_line(&aFile, &aLine);
                
                INSD_LOG(ivManager, INSD_LOG_ERROR,"(SSL_accept) Failed to accept new connection, "
                  " Socket Id %d, Return Value %d ", getSocketId(), aTlsError);
                
                INSD_LOG(ivManager, INSD_LOG_ERROR," SSL File : %s , Line number : %d , Linux Error Code %d", aFile, aLine, errno);
                

                retry = false;
              }
              
              break;
              }        
            }
          }while (aRetValue != 1 && retry != false);

          if (aRetValue <= 0)
          {
            /* Throw failure exception */
            TRACE_CALL_END_MGR (ivManager);    
            throw (TLSException(TLSException::eSSLAcceptFailure, " Error while accepting connection from client "));            
          }
          
          ivTlsState = connected;

          INSD_LOG(ivManager, INSD_LOG_DEBUG, "New connection accepted, Socket ID %d", getSocketId());

          TRACE_CALL_END_MGR (ivManager);    
          return;
    }
		
		
    void initTlsConnection()
    {

        TRACE_CALL_BEGIN_MGR (ivManager);

        long aTlsError = -1;
        bool retry = true;
        const char *aFile;
        int aLine;
        long aRetValue;

        /* DTLS handshake should be established only after the SCTP connectx() is successful */
        if (ivSocketState == DiameterSocket::eConnected)
        {        
          /* Create an SSL Session for client*/
          if (createSSLSession())
          {
            TRACE_CALL_END_MGR (ivManager);    

            /* Throw failure exception */
            throw (TLSException(TLSException::eSSLContextCreationFailure, " Error while creating context for client connection"));            
          }
          
          do
          {
            /* Clear openssl error queue */
            ERR_clear_error();

            /* Initiate SSL Handshake */
            aRetValue = SSL_connect(ivSSL);

            if (aRetValue <= 0)
            {
              aTlsError = SSL_get_error(ivSSL, aRetValue);          
              
              switch(aTlsError)
              {
                case SSL_ERROR_WANT_READ:                
                case SSL_ERROR_WANT_WRITE:
                {
                  /* Select on the socket for read/write events */
                  retry = pollSocketForEvents(aTlsError);

                  /* Nothing to do, retry to connect again*/
                  INSD_LOG(ivManager, INSD_LOG_INFO," SSL_connect() fails to connect "
                  "need to retry, returned error code %d , retry ? %s", aTlsError, retry?"true":"false");
                }
                break;

                case SSL_ERROR_SYSCALL:

                if (EWOULDBLOCK == errno || EAGAIN == errno)
                {
                  /* Nothing to do, retry to connect again */
                }
                else
                {
                  int aRet = ERR_get_error_line(&aFile, &aLine);
                  
                  INSD_LOG(ivManager, INSD_LOG_ERROR," SSL File : %s , Line number : %d , "
                    "Socket Id %d, Linux Error Code %d", aFile, aLine, getSocketId(), errno);
                  
                  INSD_LOG(ivManager, INSD_LOG_ERROR,"SSL_connect () :: Result Code : %d ", aTlsError);

                  retry = false;
                }

                break;

                default:
                {
                  int aRet = ERR_get_error_line(&aFile, &aLine);
                  
                  INSD_LOG(ivManager, INSD_LOG_ERROR,"(SSL_connect) Failed to connect to server, "
                    " Socket Id %d, Return Value %d ", getSocketId(), aTlsError);
                  
                  INSD_LOG(ivManager, INSD_LOG_ERROR," SSL File : %s , Line number : %d , Linux Error Code %d", aFile, aLine, errno);

                  retry = false;
                }
                
                break;
              }        
            }
          }while (aRetValue != 1 && retry != false);

          if (aRetValue <= 0)
          {
            TRACE_CALL_END_MGR (ivManager);    

            /* Throw failure exception */
            throw (TLSException(TLSException::eSSLConnectFailure, " Error while connecting to server "));            
          }
          
          ivTlsState = connected;

          INSD_LOG(ivManager, INSD_LOG_DEBUG, "Connection established");

        }

        TRACE_CALL_END_MGR (ivManager);    
        return;
    }
		
-------------- next part --------------
A non-text attachment was scrubbed...
Name: proxy.cap
Type: application/octet-stream
Size: 13806 bytes
Desc: not available
URL: <http://mta.openssl.org/pipermail/openssl-users/attachments/20170413/11d60744/attachment-0001.obj>