[openssl-commits] [openssl] master update
Richard Levitte
levitte at openssl.org
Mon May 15 12:48:33 UTC 2017
The branch master has been updated
via 4861933a135658a85eaaab80ce2133af4a9d413c (commit)
via 5584fd7f3d306ea0f610be9d69e64050421b2a0d (commit)
from e0011aa831d6d6765c3e65a1bec57f5b8be6f1e5 (commit)
- Log -----------------------------------------------------------------
commit 4861933a135658a85eaaab80ce2133af4a9d413c
Author: Pauli <paul.dale at oracle.com>
Date: Tue May 9 08:13:35 2017 +1000
Update one CI test to use randomised ordering.
[extended tests]
Reviewed-by: Rich Salz <rsalz at openssl.org>
Reviewed-by: Richard Levitte <levitte at openssl.org>
(Merged from https://github.com/openssl/openssl/pull/3393)
commit 5584fd7f3d306ea0f610be9d69e64050421b2a0d
Author: Pauli <paul.dale at oracle.com>
Date: Fri May 5 13:29:40 2017 +1000
Randomise the ordering of the C unit tests.
Reviewed-by: Rich Salz <rsalz at openssl.org>
Reviewed-by: Richard Levitte <levitte at openssl.org>
(Merged from https://github.com/openssl/openssl/pull/3393)
-----------------------------------------------------------------------
Summary of changes:
.travis.yml | 2 +-
test/bioprinttest.c | 75 +++++++++++++++++++++++++++++++-------------------
test/testutil/driver.c | 62 ++++++++++++++++++++++++++++++++++-------
3 files changed, 99 insertions(+), 40 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index 61a9acc..03028f3 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -121,7 +121,7 @@ matrix:
sources:
- ubuntu-toolchain-r-test
compiler: gcc-5
- env: EXTENDED_TEST="yes" CONFIG_OPTS="no-asm enable-ubsan enable-rc5 enable-md2 enable-aria -DPEDANTIC"
+ env: EXTENDED_TEST="yes" CONFIG_OPTS="no-asm enable-ubsan enable-rc5 enable-md2 enable-aria -DPEDANTIC" OPENSSL_TEST_RAND_ORDER=0
- os: linux
addons:
apt:
diff --git a/test/bioprinttest.c b/test/bioprinttest.c
index 3034cda..2c51b7a 100644
--- a/test/bioprinttest.c
+++ b/test/bioprinttest.c
@@ -1,5 +1,5 @@
/*
- * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 2016-2017 The OpenSSL Project Authors. All Rights Reserved.
*
* Licensed under the OpenSSL license (the "License"). You may not use
* this file except in compliance with the License. You can obtain a copy
@@ -20,17 +20,20 @@
static int justprint = 0;
-static char *fpexpected[][5] = {
- /* 0 */ { "0.0000e+00", "0.0000", "0", "0.0000E+00", "0" },
- /* 1 */ { "6.7000e-01", "0.6700", "0.67", "6.7000E-01", "0.67" },
- /* 2 */ { "6.6667e-01", "0.6667", "0.6667", "6.6667E-01", "0.6667" },
- /* 3 */ { "6.6667e-04", "0.0007", "0.0006667", "6.6667E-04", "0.0006667" },
- /* 4 */ { "6.6667e-05", "0.0001", "6.667e-05", "6.6667E-05", "6.667E-05" },
- /* 5 */ { "6.6667e+00", "6.6667", "6.667", "6.6667E+00", "6.667" },
- /* 6 */ { "6.6667e+01", "66.6667", "66.67", "6.6667E+01", "66.67" },
- /* 7 */ { "6.6667e+02", "666.6667", "666.7", "6.6667E+02", "666.7" },
- /* 8 */ { "6.6667e+03", "6666.6667", "6667", "6.6667E+03", "6667" },
- /* 9 */ { "6.6667e+04", "66666.6667", "6.667e+04", "6.6667E+04", "6.667E+04" },
+static char *fpexpected[][10][5] = {
+ {
+ /* 00 */ { "0.0000e+00", "0.0000", "0", "0.0000E+00", "0" },
+ /* 01 */ { "6.7000e-01", "0.6700", "0.67", "6.7000E-01", "0.67" },
+ /* 02 */ { "6.6667e-01", "0.6667", "0.6667", "6.6667E-01", "0.6667" },
+ /* 03 */ { "6.6667e-04", "0.0007", "0.0006667", "6.6667E-04", "0.0006667" },
+ /* 04 */ { "6.6667e-05", "0.0001", "6.667e-05", "6.6667E-05", "6.667E-05" },
+ /* 05 */ { "6.6667e+00", "6.6667", "6.667", "6.6667E+00", "6.667" },
+ /* 06 */ { "6.6667e+01", "66.6667", "66.67", "6.6667E+01", "66.67" },
+ /* 07 */ { "6.6667e+02", "666.6667", "666.7", "6.6667E+02", "666.7" },
+ /* 08 */ { "6.6667e+03", "6666.6667", "6667", "6.6667E+03", "6667" },
+ /* 09 */ { "6.6667e+04", "66666.6667", "6.667e+04", "6.6667E+04", "6.667E+04" },
+ },
+ {
/* 10 */ { "0.00000e+00", "0.00000", "0", "0.00000E+00", "0" },
/* 11 */ { "6.70000e-01", "0.67000", "0.67", "6.70000E-01", "0.67" },
/* 12 */ { "6.66667e-01", "0.66667", "0.66667", "6.66667E-01", "0.66667" },
@@ -41,6 +44,8 @@ static char *fpexpected[][5] = {
/* 17 */ { "6.66667e+02", "666.66667", "666.67", "6.66667E+02", "666.67" },
/* 18 */ { "6.66667e+03", "6666.66667", "6666.7", "6.66667E+03", "6666.7" },
/* 19 */ { "6.66667e+04", "66666.66667", "66667", "6.66667E+04", "66667" },
+ },
+ {
/* 20 */ { " 0.0000e+00", " 0.0000", " 0", " 0.0000E+00", " 0" },
/* 21 */ { " 6.7000e-01", " 0.6700", " 0.67", " 6.7000E-01", " 0.67" },
/* 22 */ { " 6.6667e-01", " 0.6667", " 0.6667", " 6.6667E-01", " 0.6667" },
@@ -51,6 +56,8 @@ static char *fpexpected[][5] = {
/* 27 */ { " 6.6667e+02", " 666.6667", " 666.7", " 6.6667E+02", " 666.7" },
/* 28 */ { " 6.6667e+03", " 6666.6667", " 6667", " 6.6667E+03", " 6667" },
/* 29 */ { " 6.6667e+04", " 66666.6667", " 6.667e+04", " 6.6667E+04", " 6.667E+04" },
+ },
+ {
/* 30 */ { " 0.00000e+00", " 0.00000", " 0", " 0.00000E+00", " 0" },
/* 31 */ { " 6.70000e-01", " 0.67000", " 0.67", " 6.70000E-01", " 0.67" },
/* 32 */ { " 6.66667e-01", " 0.66667", " 0.66667", " 6.66667E-01", " 0.66667" },
@@ -61,6 +68,8 @@ static char *fpexpected[][5] = {
/* 37 */ { " 6.66667e+02", " 666.66667", " 666.67", " 6.66667E+02", " 666.67" },
/* 38 */ { " 6.66667e+03", " 6666.66667", " 6666.7", " 6.66667E+03", " 6666.7" },
/* 39 */ { " 6.66667e+04", " 66666.66667", " 66667", " 6.66667E+04", " 66667" },
+ },
+ {
/* 40 */ { "0e+00", "0", "0", "0E+00", "0" },
/* 41 */ { "7e-01", "1", "0.7", "7E-01", "0.7" },
/* 42 */ { "7e-01", "1", "0.7", "7E-01", "0.7" },
@@ -71,6 +80,8 @@ static char *fpexpected[][5] = {
/* 47 */ { "7e+02", "667", "7e+02", "7E+02", "7E+02" },
/* 48 */ { "7e+03", "6667", "7e+03", "7E+03", "7E+03" },
/* 49 */ { "7e+04", "66667", "7e+04", "7E+04", "7E+04" },
+ },
+ {
/* 50 */ { "0.000000e+00", "0.000000", "0", "0.000000E+00", "0" },
/* 51 */ { "6.700000e-01", "0.670000", "0.67", "6.700000E-01", "0.67" },
/* 52 */ { "6.666667e-01", "0.666667", "0.666667", "6.666667E-01", "0.666667" },
@@ -81,6 +92,8 @@ static char *fpexpected[][5] = {
/* 57 */ { "6.666667e+02", "666.666667", "666.667", "6.666667E+02", "666.667" },
/* 58 */ { "6.666667e+03", "6666.666667", "6666.67", "6.666667E+03", "6666.67" },
/* 59 */ { "6.666667e+04", "66666.666667", "66666.7", "6.666667E+04", "66666.7" },
+ },
+ {
/* 60 */ { "0.0000e+00", "000.0000", "00000000", "0.0000E+00", "00000000" },
/* 61 */ { "6.7000e-01", "000.6700", "00000.67", "6.7000E-01", "00000.67" },
/* 62 */ { "6.6667e-01", "000.6667", "000.6667", "6.6667E-01", "000.6667" },
@@ -91,6 +104,7 @@ static char *fpexpected[][5] = {
/* 67 */ { "6.6667e+02", "666.6667", "000666.7", "6.6667E+02", "000666.7" },
/* 68 */ { "6.6667e+03", "6666.6667", "00006667", "6.6667E+03", "00006667" },
/* 69 */ { "6.6667e+04", "66666.6667", "6.667e+04", "6.6667E+04", "6.667E+04" },
+ },
};
typedef struct z_data_st {
@@ -170,7 +184,7 @@ static pw pw_params[] = {
{ 4, "08" }
};
-static int dofptest(int test, double val, const char *width, int prec)
+static int dofptest(int test, int sub, double val, const char *width, int prec)
{
static const char *fspecs[] = {
"e", "f", "g", "E", "G"
@@ -190,12 +204,12 @@ static int dofptest(int test, double val, const char *width, int prec)
if (justprint) {
if (i == 0)
- printf(" /* %d */ { \"%s\"", test, result);
+ printf(" /* %d%d */ { \"%s\"", test, sub, result);
else
printf(", \"%s\"", result);
- } else if (!TEST_str_eq(fpexpected[test][i], result)) {
+ } else if (!TEST_str_eq(fpexpected[test][sub][i], result)) {
TEST_info("test %d format=|%s| exp=|%s|, ret=|%s|",
- test, format, fpexpected[test][i], result);
+ test, format, fpexpected[test][sub][i], result);
ret = 0;
}
}
@@ -206,22 +220,25 @@ static int dofptest(int test, double val, const char *width, int prec)
static int test_fp(int i)
{
- static int t = 0;
+ int t = 0, r;
const double frac = 2.0 / 3.0;
const pw *pwp = &pw_params[i];
- if (!TEST_true(dofptest(t++, 0.0, pwp->w, pwp->p))
- || !TEST_true(dofptest(t++, 0.67, pwp->w, pwp->p))
- || !TEST_true(dofptest(t++, frac, pwp->w, pwp->p))
- || !TEST_true(dofptest(t++, frac / 1000, pwp->w, pwp->p))
- || !TEST_true(dofptest(t++, frac / 10000, pwp->w, pwp->p))
- || !TEST_true(dofptest(t++, 6.0 + frac, pwp->w, pwp->p))
- || !TEST_true(dofptest(t++, 66.0 + frac, pwp->w, pwp->p))
- || !TEST_true(dofptest(t++, 666.0 + frac, pwp->w, pwp->p))
- || !TEST_true(dofptest(t++, 6666.0 + frac, pwp->w, pwp->p))
- || !TEST_true(dofptest(t++, 66666.0 + frac, pwp->w, pwp->p)))
- return 0;
- return 1;
+ if (justprint)
+ printf(" {\n");
+ r = TEST_true(dofptest(i, t++, 0.0, pwp->w, pwp->p))
+ && TEST_true(dofptest(i, t++, 0.67, pwp->w, pwp->p))
+ && TEST_true(dofptest(i, t++, frac, pwp->w, pwp->p))
+ && TEST_true(dofptest(i, t++, frac / 1000, pwp->w, pwp->p))
+ && TEST_true(dofptest(i, t++, frac / 10000, pwp->w, pwp->p))
+ && TEST_true(dofptest(i, t++, 6.0 + frac, pwp->w, pwp->p))
+ && TEST_true(dofptest(i, t++, 66.0 + frac, pwp->w, pwp->p))
+ && TEST_true(dofptest(i, t++, 666.0 + frac, pwp->w, pwp->p))
+ && TEST_true(dofptest(i, t++, 6666.0 + frac, pwp->w, pwp->p))
+ && TEST_true(dofptest(i, t++, 66666.0 + frac, pwp->w, pwp->p));
+ if (justprint)
+ printf(" },\n");
+ return r;
}
static int test_big(void)
diff --git a/test/testutil/driver.c b/test/testutil/driver.c
index 8587304..3b62e59 100644
--- a/test/testutil/driver.c
+++ b/test/testutil/driver.c
@@ -32,6 +32,7 @@ typedef struct test_info {
static TEST_INFO all_tests[1024];
static int num_tests = 0;
+static int seed = 0;
/*
* A parameterised tests runs a loop of test cases.
* |num_test_cases| counts the total number of test cases
@@ -84,14 +85,34 @@ static int should_report_leaks()
}
#endif
+static int gcd(int a, int b)
+{
+ while (b != 0) {
+ int t = b;
+ b = a % b;
+ a = t;
+ }
+ return a;
+}
+
void setup_test()
{
char *TAP_levels = getenv("HARNESS_OSSL_LEVEL");
+ char *test_seed = getenv("OPENSSL_TEST_RAND_ORDER");
test_open_streams();
level = TAP_levels != NULL ? 4 * atoi(TAP_levels) : 0;
+ if (test_seed != NULL) {
+ seed = atoi(test_seed);
+ if (seed <= 0)
+ seed = time(NULL);
+ test_printf_stdout("%*s# RAND SEED %d\n", subtest_level(), "", seed);
+ test_flush_stdout();
+ srand(seed);
+ }
+
#ifndef OPENSSL_NO_CRYPTO_MDEBUG
if (should_report_leaks()) {
CRYPTO_set_mem_debug(1);
@@ -133,7 +154,8 @@ int run_tests(const char *test_prog_name)
{
int num_failed = 0;
char *verdict = NULL;
- int i, j;
+ int ii, i, jj, j, jstep;
+ int permute[OSSL_NELEM(all_tests)];
if (num_tests < 1) {
test_printf_stdout("%*s1..0 # Skipped: %s\n", level, "",
@@ -145,7 +167,18 @@ int run_tests(const char *test_prog_name)
}
test_flush_stdout();
- for (i = 0; i != num_tests; ++i) {
+ for (i = 0; i < num_tests; i++)
+ permute[i] = i;
+ if (seed != 0)
+ for (i = num_tests - 1; i >= 1; i--) {
+ j = rand() % (1 + i);
+ ii = permute[j];
+ permute[j] = permute[i];
+ permute[i] = ii;
+ }
+
+ for (ii = 0; ii != num_tests; ++ii) {
+ i = permute[ii];
if (all_tests[i].num == -1) {
int ret = 0;
@@ -160,7 +193,7 @@ int run_tests(const char *test_prog_name)
verdict = "not ok";
++num_failed;
}
- test_printf_stdout("%*s%s %d - %s\n", level, "", verdict, i + 1,
+ test_printf_stdout("%*s%s %d - %s\n", level, "", verdict, ii + 1,
test_title);
test_flush_stdout();
finalize(ret);
@@ -176,9 +209,18 @@ int run_tests(const char *test_prog_name)
test_flush_stdout();
}
- for (j = 0; j < all_tests[i].num; j++) {
- int ret = 0;
+ j = -1;
+ if (seed == 0 || all_tests[i].num < 3)
+ jstep = 1;
+ else
+ do
+ jstep = rand() % all_tests[i].num;
+ while (jstep == 0 || gcd(all_tests[i].num, jstep) != 1);
+
+ for (jj = 0; jj < all_tests[i].num; jj++) {
+ int ret;
+ j = (j + jstep) % all_tests[i].num;
set_test_title(NULL);
ret = all_tests[i].param_test_fn(j);
@@ -197,11 +239,11 @@ int run_tests(const char *test_prog_name)
++num_failed_inner;
}
if (test_title != NULL)
- test_printf_stdout("%*s%s %d - %s\n", level, "", verdict,
- j + 1, test_title);
+ test_printf_stdout("%*s%s %d - %s\n", level, "",
+ verdict, jj + 1, test_title);
else
- test_printf_stdout("%*s%s %d\n", level, "", verdict,
- j + 1);
+ test_printf_stdout("%*s%s %d - iteration %d\n", level,
+ "", verdict, jj + 1, j + 1);
test_flush_stdout();
}
}
@@ -212,7 +254,7 @@ int run_tests(const char *test_prog_name)
verdict = "not ok";
++num_failed;
}
- test_printf_stdout("%*s%s %d - %s\n", level, "", verdict, i + 1,
+ test_printf_stdout("%*s%s %d - %s\n", level, "", verdict, ii + 1,
all_tests[i].test_case_name);
test_flush_stdout();
}
More information about the openssl-commits
mailing list