[openssl] master update
dev at ddvo.net
dev at ddvo.net
Thu Jul 16 19:46:39 UTC 2020
The branch master has been updated
via cb9bb7350d4192553683e61e64894e8ed197b44c (commit)
via 1e76cb002a8d89b66b67214921b921c4cb9f6506 (commit)
from 0b670a2101c6cdcc3f2a4ed168f75243fe082a2b (commit)
- Log -----------------------------------------------------------------
commit cb9bb7350d4192553683e61e64894e8ed197b44c
Author: Dr. David von Oheimb <David.von.Oheimb at siemens.com>
Date: Fri Jul 3 14:19:43 2020 +0200
99-test_fuzz.t: Clean up and re-organize such that sub-tests could be split easily
Reviewed-by: Richard Levitte <levitte at openssl.org>
(Merged from https://github.com/openssl/openssl/pull/12359)
commit 1e76cb002a8d89b66b67214921b921c4cb9f6506
Author: Dr. David von Oheimb <David.von.Oheimb at siemens.com>
Date: Thu Jul 2 17:59:55 2020 +0200
test/run_tests.pl: In parallel runs, start those tests first that run longest
Also untabify the Perl source file.
Reviewed-by: Richard Levitte <levitte at openssl.org>
(Merged from https://github.com/openssl/openssl/pull/12359)
-----------------------------------------------------------------------
Summary of changes:
fuzz/README.md | 16 +++++++++-
test/README.md | 4 +++
test/recipes/99-test_fuzz.t | 41 +++++++++++-------------
test/recipes/fuzz.pl | 31 ++++++++++++++++++
test/run_tests.pl | 76 +++++++++++++++++++++++++--------------------
5 files changed, 111 insertions(+), 57 deletions(-)
create mode 100644 test/recipes/fuzz.pl
diff --git a/fuzz/README.md b/fuzz/README.md
index a713f85325..deb7a43168 100644
--- a/fuzz/README.md
+++ b/fuzz/README.md
@@ -99,7 +99,7 @@ Reproducing issues
If a fuzzer generates a reproducible error, you can reproduce the problem using
the fuzz/*-test binaries and the file generated by the fuzzer. They binaries
-don't need to be build for fuzzing, there is no need to set CC or the call
+don't need to be built for fuzzing, there is no need to set CC or the call
config with enable-fuzz-* or -fsanitize-coverage, but some of the other options
above might be needed. For instance the enable-asan or enable-ubsan option might
be useful to show you when the problem happens. For the client and server fuzzer
@@ -110,6 +110,20 @@ To reproduce the crash you can run:
fuzz/$FUZZER-test $file
+To do all the tests of a specific fuzzer such as asn1 you can run
+
+ fuzz/asn1-test fuzz/corpora/asn1
+or
+ make test TESTS=fuzz_test FUZZ_TESTS=asn1
+
+To run several fuzz tests you can use for instance:
+
+ make test TESTS=test_fuzz FUZZ_TESTS="cmp cms"
+
+To run all fuzz tests you can use:
+
+ make test TESTS=test_fuzz
+
Random numbers
--------------
diff --git a/test/README.md b/test/README.md
index f9058a0026..f4f0574aef 100644
--- a/test/README.md
+++ b/test/README.md
@@ -121,6 +121,10 @@ Run all tests in test groups 80 to 99 except for tests in group 90:
$ make TESTS='[89]? -90' test
+To run specific fuzz tests you can use for instance:
+
+ $ make test TESTS=test_fuzz FUZZ_TESTS="cmp cms"
+
To stochastically verify that the algorithm that produces uniformly distributed
random numbers is operating correctly (with a false positive rate of 0.01%):
diff --git a/test/recipes/99-test_fuzz.t b/test/recipes/99-test_fuzz.t
index c9e2c961e4..8bacad47de 100644
--- a/test/recipes/99-test_fuzz.t
+++ b/test/recipes/99-test_fuzz.t
@@ -9,35 +9,30 @@
use strict;
use warnings;
-use OpenSSL::Glob;
use OpenSSL::Test qw/:DEFAULT srctop_file/;
use OpenSSL::Test::Utils;
setup("test_fuzz");
-my @fuzzers = ('asn1', 'asn1parse', 'bignum', 'bndiv', 'client', 'conf', 'crl', 'server', 'x509');
-if (!disabled("cmp")) {
- push @fuzzers, 'cmp';
+my @fuzzers = ();
+ at fuzzers = split /\s+/, $ENV{FUZZ_TESTS} if $ENV{FUZZ_TESTS};
+
+if (!@fuzzers) {
+ @fuzzers = (
+ # those commented here as very slow could be moved to separate runs
+ 'asn1', # very slow
+ 'asn1parse', 'bignum', 'bndiv', 'conf','crl',
+ 'client', # very slow
+ 'server', # very slow
+ 'x509'
+ );
+ push @fuzzers, 'cmp' if !disabled("cmp");
+ push @fuzzers, 'cms' if !disabled("cms");
+ push @fuzzers, 'ct' if !disabled("ct");
}
-if (!disabled("cms")) {
- push @fuzzers, 'cms';
-}
-if (!disabled("ct")) {
- push @fuzzers, 'ct';
-}
-plan tests => scalar @fuzzers;
-foreach my $f (@fuzzers) {
- subtest "Fuzzing $f" => sub {
- my @dirs = glob(srctop_file('fuzz', 'corpora', $f));
- push @dirs, glob(srctop_file('fuzz', 'corpora', "$f-*"));
+plan tests => scalar @fuzzers + 1; # one more due to below require_ok(...)
- plan skip_all => "No corpora for $f-test" unless @dirs;
+require_ok(srctop_file('test','recipes','fuzz.pl'));
- plan tests => scalar @dirs;
-
- foreach (@dirs) {
- ok(run(fuzz(["$f-test", $_])));
- }
- }
-}
+&fuzz_tests(@fuzzers);
diff --git a/test/recipes/fuzz.pl b/test/recipes/fuzz.pl
new file mode 100644
index 0000000000..795d85c1df
--- /dev/null
+++ b/test/recipes/fuzz.pl
@@ -0,0 +1,31 @@
+# Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the Apache License 2.0 (the "License"). You may not use
+# this file except in compliance with the License. You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+use strict;
+use warnings;
+
+use OpenSSL::Glob;
+use OpenSSL::Test qw/:DEFAULT srctop_file/;
+
+sub fuzz_tests {
+ my @fuzzers = @_;
+
+ foreach my $f (@fuzzers) {
+ subtest "Fuzzing $f" => sub {
+ my @dir = glob(srctop_file('fuzz', 'corpora', "$f"));
+
+ plan skip_all => "No directory fuzz/corpora/$f" unless @dir;
+ plan tests => scalar @dir; # likely 1
+
+ foreach (@dir) {
+ ok(run(fuzz(["$f-test", $_])));
+ }
+ }
+ }
+}
+
+1;
diff --git a/test/run_tests.pl b/test/run_tests.pl
index d47f7cf1e6..73d4d91931 100644
--- a/test/run_tests.pl
+++ b/test/run_tests.pl
@@ -70,44 +70,54 @@ open $openssl_args{'tap_copy'}, ">$outfilename"
my @alltests = find_matching_tests("*");
my %tests = ();
+sub reorder {
+ my $key = pop;
+
+ # for parallel test runs, do slow tests first
+ if (defined $jobs && $jobs > 1 && $key =~ m/test_ssl_new|test_fuzz/) {
+ $key =~ s/(\d+)-/00-/;
+ }
+ return $key;
+}
+
my $initial_arg = 1;
foreach my $arg (@ARGV ? @ARGV : ('alltests')) {
if ($arg eq 'list') {
- foreach (@alltests) {
- (my $x = basename($_)) =~ s|^[0-9][0-9]-(.*)\.t$|$1|;
- print $x,"\n";
- }
- exit 0;
+ foreach (@alltests) {
+ (my $x = basename($_)) =~ s|^[0-9][0-9]-(.*)\.t$|$1|;
+ print $x,"\n";
+ }
+ exit 0;
}
if ($arg eq 'alltests') {
- warn "'alltests' encountered, ignoring everything before that...\n"
- unless $initial_arg;
- %tests = map { $_ => basename($_) } @alltests;
+ warn "'alltests' encountered, ignoring everything before that...\n"
+ unless $initial_arg;
+ %tests = map { $_ => 1 } @alltests;
} elsif ($arg =~ m/^(-?)(.*)/) {
- my $sign = $1;
- my $test = $2;
- my @matches = find_matching_tests($test);
-
- # If '-foo' is the first arg, it's short for 'alltests -foo'
- if ($sign eq '-' && $initial_arg) {
- %tests = map { $_ => basename($_) } @alltests;
- }
-
- if (scalar @matches == 0) {
- warn "Test $test found no match, skipping ",
- ($sign eq '-' ? "removal" : "addition"),
- "...\n";
- } else {
- foreach $test (@matches) {
- if ($sign eq '-') {
- delete $tests{$test};
- } else {
- $tests{$test} = basename($test);
- }
- }
- }
+ my $sign = $1;
+ my $test = $2;
+ my @matches = find_matching_tests($test);
+
+ # If '-foo' is the first arg, it's short for 'alltests -foo'
+ if ($sign eq '-' && $initial_arg) {
+ %tests = map { $_ => 1 } @alltests;
+ }
+
+ if (scalar @matches == 0) {
+ warn "Test $test found no match, skipping ",
+ ($sign eq '-' ? "removal" : "addition"),
+ "...\n";
+ } else {
+ foreach $test (@matches) {
+ if ($sign eq '-') {
+ delete $tests{$test};
+ } else {
+ $tests{$test} = 1;
+ }
+ }
+ }
} else {
- warn "I don't know what '$arg' is about, ignoring...\n";
+ warn "I don't know what '$arg' is about, ignoring...\n";
}
$initial_arg = 0;
@@ -280,8 +290,8 @@ unless (defined $eres) {
my $harness = $package->new(\%tapargs);
my $ret =
- $harness->runtests(map { [ abs2rel($_, rel2abs(curdir())), $tests{$_} ] }
- sort keys %tests);
+ $harness->runtests(map { [ abs2rel($_, rel2abs(curdir())), basename($_) ] }
+ sort { reorder($a) cmp reorder($b) } keys %tests);
# $ret->has_errors may be any number, not just 0 or 1. On VMS, numbers
# from 2 and on are used as is as VMS statuses, which has severity encoded
More information about the openssl-commits
mailing list