[web] master update

Tue Jul 27 08:31:25 UTC 2021

The branch master has been updated
       via  61f488185e0736cf5196efc9d5f4f4b3370b3f8e (commit)
      from  539bea014de78db5ff5b0785a46bfd7647b0b589 (commit)


- Log -----------------------------------------------------------------
commit 61f488185e0736cf5196efc9d5f4f4b3370b3f8e
Author: Richard Levitte <levitte at openssl.org>
Date:   Tue Jul 27 09:55:07 2021 +0200

    Simplify the CDN purge
    
    Our CDN (Akamai) purge script was run as a standalone automation, in
    parallell with the automatic 'make' run.  The consequence was that the
    CDN could catch a copy of our original web pages in a semi built state,
    as demonstrated by openssl/openssl#16152.
    
    The solution is the ensure that the purge is run in sequence after
    everything is built.  We simplify this further by moving the actual
    script into the web source.
    
    Reviewed-by: Paul Dale <pauli at openssl.org>
    (Merged from https://github.com/openssl/web/pull/248)

-----------------------------------------------------------------------

Summary of changes:
 Makefile           |  7 +++--
 bin/purge-one-hour | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++--
 2 files changed, 93 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 32b9244..fb15dad 100644
--- a/Makefile
+++ b/Makefile
@@ -56,15 +56,18 @@ SRCLISTS = $(foreach S,$(FUTURESERIES) $(SERIES) $(OLDSERIES2) fips,source/old/$
 	@rm -f $@
 	./bin/md-to-html5 $<
 
-all: suball manmaster mancross
+all: suball manmaster mancross akamai-purge
 
 suball: $(SIMPLE) $(SRCLISTS)
 
-relupd: suball manpages mancross
+relupd: suball manpages mancross akamai-purge
 
 clean:
 	rm -f $(SIMPLE) $(SRCLISTS)
 
+akamai-purge:
+	./bin/purge-one-hour
+
 # Legacy targets
 hack-source_htaccess: all
 simple: all
diff --git a/bin/purge-one-hour b/bin/purge-one-hour
index 5e10e49..895967f 100755
--- a/bin/purge-one-hour
+++ b/bin/purge-one-hour
@@ -1,3 +1,89 @@
-#! /bin/sh
+#! /usr/bin/perl
+#
+# script to purge the Akamai cache.
+#
+# Notes:
+#
+# - we limit the purging to files newer than an hour
+# - there must be a file ~openssl/.edgerc with our Akamai credentials
+# - the Akamai supplied program 'akamai-purge' must be installed in
+#   /usr/local/bin
 
-/opt/openssl/maker/triggered-makers/akamai-purge
+use strict;
+use warnings;
+
+# Find all .html files that include a .inc file, and create a map
+my %inc2html = ();
+
+my $debug = $ENV{DEBUG};
+my $dryrun = $ENV{DRYRUN};
+
+my $base = '/var/www/openssl';	# MUST NOT end with a slash
+
+foreach ( `find $base -type f -name '*.html'` ) {
+    chomp;
+    my $file = $_;
+    my ($dn, $fn) = $_ =~ m/^(?:(.*)\/)?([^\/]*)$/;
+    my @incs = ();
+
+    open HTML, $_;
+    foreach ( <HTML> ) {
+	if (/<!--\s*#include\s+virtual="([^"]*)"\s*-->/) {
+	    my $vf = $1;
+	    $vf = ($vf =~ m|^/|) ? "$base$vf" : "$dn/$vf";
+	    push @incs, "$vf";
+	}
+    }
+    close HTML;
+
+    foreach ( @incs ) {
+	push @{$inc2html{$_}}, $file;
+    }
+}
+
+if ($debug) {
+    for ( sort keys %inc2html ) {
+	print STDERR "DEBUG: $_ => ", join(", ", @{$inc2html{$_}}), "\n";
+    }
+}
+
+# Find all files younger than an hour
+# Discard those in .git/ and bin/
+# Discard any .ht*
+# For any virtually included file, use the corresponding .html file instead
+# For all remaining files, turn it into a valid URL
+# For any valid index file, duplicate into two URLs without the file,
+#    one with an ending slash and one without.
+my %files = ();
+
+foreach ( `find $base -type f -mtime -2` ) {
+    chomp;
+    next if /^\Q$base\E\/(\.git|bin)/;
+    next if /\/\.ht\w+$/;
+    my $x = $_;
+    my @files = defined $inc2html{$x} ? @{$inc2html{$x}} : ( $x );
+    foreach ( @files ) {
+	s/^\Q$base\E\//https:\/\/www.openssl.org\//;
+	$files{$_} = 1;
+	if ( /^(.*)\/index.(html|cgi|pl|php|xhtml|htm)$/ ) {
+	    $files{"$1/"} = $files{"$1"} = 1;
+	}
+    }
+}
+
+# Finally, output the result to the akamai-purge program
+my @files = sort keys %files;
+while ( @files ) {
+    my $count = 500;		# Try not to overwhelm Akamai
+    if ( $dryrun || open PURGE, '| /usr/local/bin/akamai-purge invalidate' ) {
+	printf STDERR
+	    "DEBUG: Invoking '/usr/local/bin/akamai-purge invalidate' with:\n"
+	    if $debug;
+	while ( @files && $count-- > 0 ) {
+	    my $file = pop @files;
+	    print STDERR "  ",$file,"\n" if $debug;
+	    print PURGE $file,"\n" unless $dryrun;
+	}
+	close PURGE unless $dryrun;
+    }
+}
