[web] master update
Richard Levitte
levitte at openssl.org
Tue Jul 27 08:31:25 UTC 2021
The branch master has been updated
via 61f488185e0736cf5196efc9d5f4f4b3370b3f8e (commit)
from 539bea014de78db5ff5b0785a46bfd7647b0b589 (commit)
- Log -----------------------------------------------------------------
commit 61f488185e0736cf5196efc9d5f4f4b3370b3f8e
Author: Richard Levitte <levitte at openssl.org>
Date: Tue Jul 27 09:55:07 2021 +0200
Simplify the CDN purge
Our CDN (Akamai) purge script was run as a standalone automation, in
parallell with the automatic 'make' run. The consequence was that the
CDN could catch a copy of our original web pages in a semi built state,
as demonstrated by openssl/openssl#16152.
The solution is the ensure that the purge is run in sequence after
everything is built. We simplify this further by moving the actual
script into the web source.
Reviewed-by: Paul Dale <pauli at openssl.org>
(Merged from https://github.com/openssl/web/pull/248)
-----------------------------------------------------------------------
Summary of changes:
Makefile | 7 +++--
bin/purge-one-hour | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++--
2 files changed, 93 insertions(+), 4 deletions(-)
diff --git a/Makefile b/Makefile
index 32b9244..fb15dad 100644
--- a/Makefile
+++ b/Makefile
@@ -56,15 +56,18 @@ SRCLISTS = $(foreach S,$(FUTURESERIES) $(SERIES) $(OLDSERIES2) fips,source/old/$
@rm -f $@
./bin/md-to-html5 $<
-all: suball manmaster mancross
+all: suball manmaster mancross akamai-purge
suball: $(SIMPLE) $(SRCLISTS)
-relupd: suball manpages mancross
+relupd: suball manpages mancross akamai-purge
clean:
rm -f $(SIMPLE) $(SRCLISTS)
+akamai-purge:
+ ./bin/purge-one-hour
+
# Legacy targets
hack-source_htaccess: all
simple: all
diff --git a/bin/purge-one-hour b/bin/purge-one-hour
index 5e10e49..895967f 100755
--- a/bin/purge-one-hour
+++ b/bin/purge-one-hour
@@ -1,3 +1,89 @@
-#! /bin/sh
+#! /usr/bin/perl
+#
+# script to purge the Akamai cache.
+#
+# Notes:
+#
+# - we limit the purging to files newer than an hour
+# - there must be a file ~openssl/.edgerc with our Akamai credentials
+# - the Akamai supplied program 'akamai-purge' must be installed in
+# /usr/local/bin
-/opt/openssl/maker/triggered-makers/akamai-purge
+use strict;
+use warnings;
+
+# Find all .html files that include a .inc file, and create a map
+my %inc2html = ();
+
+my $debug = $ENV{DEBUG};
+my $dryrun = $ENV{DRYRUN};
+
+my $base = '/var/www/openssl'; # MUST NOT end with a slash
+
+foreach ( `find $base -type f -name '*.html'` ) {
+ chomp;
+ my $file = $_;
+ my ($dn, $fn) = $_ =~ m/^(?:(.*)\/)?([^\/]*)$/;
+ my @incs = ();
+
+ open HTML, $_;
+ foreach ( <HTML> ) {
+ if (/<!--\s*#include\s+virtual="([^"]*)"\s*-->/) {
+ my $vf = $1;
+ $vf = ($vf =~ m|^/|) ? "$base$vf" : "$dn/$vf";
+ push @incs, "$vf";
+ }
+ }
+ close HTML;
+
+ foreach ( @incs ) {
+ push @{$inc2html{$_}}, $file;
+ }
+}
+
+if ($debug) {
+ for ( sort keys %inc2html ) {
+ print STDERR "DEBUG: $_ => ", join(", ", @{$inc2html{$_}}), "\n";
+ }
+}
+
+# Find all files younger than an hour
+# Discard those in .git/ and bin/
+# Discard any .ht*
+# For any virtually included file, use the corresponding .html file instead
+# For all remaining files, turn it into a valid URL
+# For any valid index file, duplicate into two URLs without the file,
+# one with an ending slash and one without.
+my %files = ();
+
+foreach ( `find $base -type f -mtime -2` ) {
+ chomp;
+ next if /^\Q$base\E\/(\.git|bin)/;
+ next if /\/\.ht\w+$/;
+ my $x = $_;
+ my @files = defined $inc2html{$x} ? @{$inc2html{$x}} : ( $x );
+ foreach ( @files ) {
+ s/^\Q$base\E\//https:\/\/www.openssl.org\//;
+ $files{$_} = 1;
+ if ( /^(.*)\/index.(html|cgi|pl|php|xhtml|htm)$/ ) {
+ $files{"$1/"} = $files{"$1"} = 1;
+ }
+ }
+}
+
+# Finally, output the result to the akamai-purge program
+my @files = sort keys %files;
+while ( @files ) {
+ my $count = 500; # Try not to overwhelm Akamai
+ if ( $dryrun || open PURGE, '| /usr/local/bin/akamai-purge invalidate' ) {
+ printf STDERR
+ "DEBUG: Invoking '/usr/local/bin/akamai-purge invalidate' with:\n"
+ if $debug;
+ while ( @files && $count-- > 0 ) {
+ my $file = pop @files;
+ print STDERR " ",$file,"\n" if $debug;
+ print PURGE $file,"\n" unless $dryrun;
+ }
+ close PURGE unless $dryrun;
+ }
+}
More information about the openssl-commits
mailing list