[openssl-commits] [openssl] OpenSSL_1_0_2-stable update

[Richard Levitte]

The branch OpenSSL_1_0_2-stable has been updated
       via  78a3e80a31b9c2b66bd8e1d33903d386915aefbb (commit)
      from  fecd4c265527459e3db5ac37bbf77ab47aa10dc7 (commit)


- Log -----------------------------------------------------------------
commit 78a3e80a31b9c2b66bd8e1d33903d386915aefbb
Author: Richard Levitte <levitte at openssl.org>
Date:   Fri Dec 9 23:41:01 2016 +0100

    VMS UI_OpenSSL: if the TT device isn't a tty, flag instead of error
    
    On all platforms, if the controlling tty isn't an actual tty, this is
    flagged by setting is_a_tty to zero...  except on VMS, where this was
    treated as an error.  Change this to behave like the other platforms.
    
    Reviewed-by: Rich Salz <rsalz at openssl.org>
    (Merged from https://github.com/openssl/openssl/pull/2064)

-----------------------------------------------------------------------

Summary of changes:
 crypto/ui/ui_openssl.c | 48 ++++++++++++++++++++++++++++--------------------
 1 file changed, 28 insertions(+), 20 deletions(-)

diff --git a/crypto/ui/ui_openssl.c b/crypto/ui/ui_openssl.c
index 295fbc5..377384b 100644
--- a/crypto/ui/ui_openssl.c
+++ b/crypto/ui/ui_openssl.c
@@ -514,13 +514,17 @@ static int open_console(UI *ui)
 #endif
 #ifdef OPENSSL_SYS_VMS
     status = sys$assign(&terminal, &channel, 0, 0);
+
+    /* if there isn't a TT device, something is very wrong */
     if (status != SS$_NORMAL)
         return 0;
-    status =
-        sys$qiow(0, channel, IO$_SENSEMODE, &iosb, 0, 0, tty_orig, 12, 0, 0,
-                 0, 0);
+
+    status = sys$qiow(0, channel, IO$_SENSEMODE, &iosb, 0, 0, tty_orig, 12,
+                      0, 0, 0, 0);
+
+    /* If IO$_SENSEMODE doesn't work, this is not a terminal device */
     if ((status != SS$_NORMAL) || (iosb.iosb$w_value != SS$_NORMAL))
-        return 0;
+        is_a_tty = 0;
 #endif
     return 1;
 }
@@ -537,14 +541,15 @@ static int noecho_console(UI *ui)
         return 0;
 #endif
 #ifdef OPENSSL_SYS_VMS
-    tty_new[0] = tty_orig[0];
-    tty_new[1] = tty_orig[1] | TT$M_NOECHO;
-    tty_new[2] = tty_orig[2];
-    status =
-        sys$qiow(0, channel, IO$_SETMODE, &iosb, 0, 0, tty_new, 12, 0, 0, 0,
-                 0);
-    if ((status != SS$_NORMAL) || (iosb.iosb$w_value != SS$_NORMAL))
-        return 0;
+    if (is_a_tty) {
+        tty_new[0] = tty_orig[0];
+        tty_new[1] = tty_orig[1] | TT$M_NOECHO;
+        tty_new[2] = tty_orig[2];
+        status = sys$qiow(0, channel, IO$_SETMODE, &iosb, 0, 0, tty_new, 12,
+                          0, 0, 0, 0);
+        if ((status != SS$_NORMAL) || (iosb.iosb$w_value != SS$_NORMAL))
+            return 0;
+    }
 #endif
     return 1;
 }
@@ -561,14 +566,15 @@ static int echo_console(UI *ui)
         return 0;
 #endif
 #ifdef OPENSSL_SYS_VMS
-    tty_new[0] = tty_orig[0];
-    tty_new[1] = tty_orig[1] & ~TT$M_NOECHO;
-    tty_new[2] = tty_orig[2];
-    status =
-        sys$qiow(0, channel, IO$_SETMODE, &iosb, 0, 0, tty_new, 12, 0, 0, 0,
-                 0);
-    if ((status != SS$_NORMAL) || (iosb.iosb$w_value != SS$_NORMAL))
-        return 0;
+    if (is_a_tty) {
+        tty_new[0] = tty_orig[0];
+        tty_new[1] = tty_orig[1] & ~TT$M_NOECHO;
+        tty_new[2] = tty_orig[2];
+        status = sys$qiow(0, channel, IO$_SETMODE, &iosb, 0, 0, tty_new, 12,
+                          0, 0, 0, 0);
+        if ((status != SS$_NORMAL) || (iosb.iosb$w_value != SS$_NORMAL))
+            return 0;
+    }
 #endif
     return 1;
 }
@@ -581,6 +587,8 @@ static int close_console(UI *ui)
         fclose(tty_out);
 #ifdef OPENSSL_SYS_VMS
     status = sys$dassgn(channel);
+    if (status != SS$_NORMAL)
+        return 0;
 #endif
     CRYPTO_w_unlock(CRYPTO_LOCK_UI);