[openssl-commits] [openssl] master update
Andy Polyakov
appro at openssl.org
Mon Jul 24 19:23:56 UTC 2017
The branch master has been updated
via e3c79f0f1901c765a1a7e2bc68e1f6d4200f4a3b (commit)
from e0de4dd5a2b0c0dc27e6a6ab01fabe374d657d23 (commit)
- Log -----------------------------------------------------------------
commit e3c79f0f1901c765a1a7e2bc68e1f6d4200f4a3b
Author: Andy Polyakov <appro at openssl.org>
Date: Sun Jul 23 16:06:26 2017 +0200
sha/asm/keccak1600-avx512.pl: improve performance by 17%.
Improvement is result of combination of data layout ideas from
Keccak Code Package and initial version of this module.
Hardware used for benchmarking courtesy of Atos, experiments run by
Romain Dolbeau <romain.dolbeau at atos.net>. Kudos!
Reviewed-by: Bernd Edlinger <bernd.edlinger at hotmail.de>
Reviewed-by: Rich Salz <rsalz at openssl.org>
-----------------------------------------------------------------------
Summary of changes:
crypto/sha/asm/keccak1600-avx512.pl | 454 ++++++++++++++++++++++--------------
1 file changed, 278 insertions(+), 176 deletions(-)
diff --git a/crypto/sha/asm/keccak1600-avx512.pl b/crypto/sha/asm/keccak1600-avx512.pl
index 70dec4e..2f32151 100755
--- a/crypto/sha/asm/keccak1600-avx512.pl
+++ b/crypto/sha/asm/keccak1600-avx512.pl
@@ -20,28 +20,60 @@
# Below code is KECCAK_1X_ALT implementation (see sha/keccak1600.c).
# Pretty straightforward, the only "magic" is data layout in registers.
# It's impossible to have one that is optimal for every step, hence
-# it's changing as algorithm progresses. Data is saved in order that
-# benefits Chi, but at the same time is easily convertible to order
-# that benefits Theta. Conversion from Chi layout to Theta is
-# explicit and reverse one is kind of fused with Pi...
+# it's changing as algorithm progresses. Data is saved in linear order,
+# but in-register order morphs between rounds. Even rounds take in
+# linear layout, and odd rounds - transposed, or "verticaly-shaped"...
#
########################################################################
# Numbers are cycles per processed byte out of large message.
#
# r=1088(*)
#
-# Knights Landing 8.9
-# Skylake-X 6.7
+# Knights Landing 7.6
+# Skylake-X 5.7
#
# (*) Corresponds to SHA3-256.
########################################################################
-# Coordinates below correspond to those in sha/keccak1600.c. Layout
-# suitable for Chi is one with y coordinates aligned column-wise. Trick
-# is to add regular shift to x coordinate, so that Chi can still be
-# performed with as little as 7 instructions, yet be converted to layout
-# suitable for Theta with intra-register permutations alone. Here is
-# "magic" layout for Chi (with pre-Theta shuffle):
+# Below code is combination of two ideas. One is taken from Keccak Code
+# Package, hereafter KCP, and another one from initial version of this
+# module. What is common is observation that Pi's input and output are
+# "mostly transposed", i.e. if input is aligned by x coordinate, then
+# output is [mostly] aligned by y. Both versions, KCP and predecessor,
+# were trying to use one of them from round to round, which resulted in
+# some kind of transposition in each round. This version still does
+# transpose data, but only every second round. Another essential factor
+# is that KCP transposition has to be performed with instructions that
+# turned to be rather expensive on Knights Landing, both latency- and
+# throughput-wise. Not to mention that some of them have to depend on
+# each other. On the other hand initial version of this module was
+# relying heavily on blend instructions. There were lots of them,
+# resulting in higher instruction count, yet it performed better on
+# Knights Landing, because processor can execute pair of them each
+# cycle and they have minimal latency. This module is an attempt to
+# bring best parts together:-)
+#
+# Coordinates below correspond to those in sha/keccak1600.c. Input
+# layout is straight linear:
+#
+# [0][4] [0][3] [0][2] [0][1] [0][0]
+# [1][4] [1][3] [1][2] [1][1] [1][0]
+# [2][4] [2][3] [2][2] [2][1] [2][0]
+# [3][4] [3][3] [3][2] [3][1] [3][0]
+# [4][4] [4][3] [4][2] [4][1] [4][0]
+#
+# It's perfect for Theta, while Pi is reduced to intra-register
+# permutations which yield layout perfect for Chi:
+#
+# [4][0] [3][0] [2][0] [1][0] [0][0]
+# [4][1] [3][1] [2][1] [1][1] [0][1]
+# [4][2] [3][2] [2][2] [1][2] [0][2]
+# [4][3] [3][3] [2][3] [1][3] [0][3]
+# [4][4] [3][4] [2][4] [1][4] [0][4]
+#
+# Now instead of performing full transposition and feeding it to next
+# identical round, we perform kind of diagonal transposition to layout
+# from initial version of this module, and make it suitable for Theta:
#
# [4][4] [3][3] [2][2] [1][1] [0][0]>4.3.2.1.0>[4][4] [3][3] [2][2] [1][1] [0][0]
# [4][0] [3][4] [2][3] [1][2] [0][1]>3.2.1.0.4>[3][4] [2][3] [1][2] [0][1] [4][0]
@@ -49,53 +81,52 @@
# [4][2] [3][1] [2][0] [1][4] [0][3]>1.0.4.3.2>[1][4] [0][3] [4][2] [3][1] [2][0]
# [4][3] [3][2] [2][1] [1][0] [0][4]>0.4.3.2.1>[0][4] [4][3] [3][2] [2][1] [1][0]
#
-# Layout suitable to Theta has x coordinates aligned column-wise
-# [it's interleaved with Pi indices transformation for reference]:
+# Now intra-register permutations yield initial [almost] straight
+# linear layout:
#
-# [4][4] [3][3] [2][2] [1][1] [0][0] $A00
+# [4][4] [3][3] [2][2] [1][1] [0][0]
##[0][4] [0][3] [0][2] [0][1] [0][0]
-# [3][4] [2][3] [1][2] [0][1] [4][0] $A01
+# [3][4] [2][3] [1][2] [0][1] [4][0]
##[2][3] [2][2] [2][1] [2][0] [2][4]
-# [2][4] [1][3] [0][2] [4][1] [3][0] $A02
+# [2][4] [1][3] [0][2] [4][1] [3][0]
##[4][2] [4][1] [4][0] [4][4] [4][3]
-# [1][4] [0][3] [4][2] [3][1] [2][0] $A03
+# [1][4] [0][3] [4][2] [3][1] [2][0]
##[1][1] [1][0] [1][4] [1][3] [1][2]
-# [0][4] [4][3] [3][2] [2][1] [1][0] $A04
+# [0][4] [4][3] [3][2] [2][1] [1][0]
##[3][0] [3][4] [3][3] [3][2] [3][1]
#
-# Pi itself is performed by blending above data and finally shuffling it
-# to original Chi layout:
-#
-# [1][1] [2][2] [3][3] [4][4] [0][0]>1.2.3.4.0>[4][4] [3][3] [2][2] [1][1] [0][0]
-# [2][3] [3][4] [4][0] [0][1] [1][2]>2.3.4.0.1>[4][0] [3][4] [2][3] [1][2] [0][1]
-# [3][0] [4][1] [0][2] [1][3] [2][4]>3.4.0.1.2>[4][1] [3][0] [2][4] [1][3] [0][2]
-# [4][2] [0][3] [1][4] [2][0] [3][1]>4.0.1.2.3>[4][2] [3][1] [2][0] [1][4] [0][3]
-# [0][4] [1][0] [2][1] [3][2] [4][3]>0.1.2.3.4>[4][3] [3][2] [2][1] [1][0] [0][4]
+# This means that odd round Chi is performed in less suitable layout,
+# with a number of additional permutations. But overall it turned to be
+# a win. Permutations are fastest possible on Knights Landing and they
+# are laid down to be independent of each other. In the essence I traded
+# 20 blend instructions for 3 permutations. The result is 13% faster
+# than KCP on Skylake-X, and >40% on Knights Landing.
#
-# As implied, data is loaded in Chi layout. Digits in variables' names
-# represent right most coordinates of loaded data chunk:
-
-my ($A00, # [4][4] [3][3] [2][2] [1][1] [0][0]
- $A01, # [4][0] [3][4] [2][3] [1][2] [0][1]
- $A02, # [4][1] [3][0] [2][4] [1][3] [0][2]
- $A03, # [4][2] [3][1] [2][0] [1][4] [0][3]
- $A04) = # [4][3] [3][2] [2][1] [1][0] [0][4]
+# As implied, data is loaded in straight linear order. Digits in
+# variables' names represent coordinates of right-most element of
+# loaded data chunk:
+
+my ($A00, # [0][4] [0][3] [0][2] [0][1] [0][0]
+ $A10, # [1][4] [1][3] [1][2] [1][1] [1][0]
+ $A20, # [2][4] [2][3] [2][2] [2][1] [2][0]
+ $A30, # [3][4] [3][3] [3][2] [3][1] [3][0]
+ $A40) = # [4][4] [4][3] [4][2] [4][1] [4][0]
map("%zmm$_",(0..4));
# We also need to map the magic order into offsets within structure:
-my @A_jagged = ([0,0], [1,0], [2,0], [3,0], [4,0],
- [4,1], [0,1], [1,1], [2,1], [3,1],
- [3,2], [4,2], [0,2], [1,2], [2,2],
- [2,3], [3,3], [4,3], [0,3], [1,3],
- [1,4], [2,4], [3,4], [4,4], [0,4]);
- @A_jagged_in = map(8*($$_[0]*8+$$_[1]), @A_jagged); # ... and now linear
- @A_jagged_out = map(8*($$_[0]*5+$$_[1]), @A_jagged); # ... and now linear
+my @A_jagged = ([0,0], [0,1], [0,2], [0,3], [0,4],
+ [1,0], [1,1], [1,2], [1,3], [1,4],
+ [2,0], [2,1], [2,2], [2,3], [2,4],
+ [3,0], [3,1], [3,2], [3,3], [3,4],
+ [4,0], [4,1], [4,2], [4,3], [4,4]);
+ @A_jagged = map(8*($$_[0]*8+$$_[1]), @A_jagged); # ... and now linear
-my @T = map("%zmm$_",(5..7,16..17));
-my @Chi = map("%zmm$_",(18..22));
-my @Theta = map("%zmm$_",(33,23..26)); # invalid @Theta[0] is not typo
-my @Rhotate = map("%zmm$_",(27..31));
+my @T = map("%zmm$_",(5..12));
+my @Theta = map("%zmm$_",(33,13..16)); # invalid @Theta[0] is not typo
+my @Pi0 = map("%zmm$_",(17..21));
+my @Rhotate0 = map("%zmm$_",(22..26));
+my @Rhotate1 = map("%zmm$_",(27..31));
my ($C00,$D00) = @T[0..1];
my ($k00001,$k00010,$k00100,$k01000,$k10000,$k11111) = map("%k$_",(1..6));
@@ -107,82 +138,136 @@ $code.=<<___;
.align 32
__KeccakF1600:
lea iotas(%rip),%r10
- mov \$24,%eax
+ mov \$12,%eax
jmp .Loop_avx512
.align 32
.Loop_avx512:
- ######################################### Theta
- #vpermq $A00, at Theta[0],$A00 # doesn't actually change order
- vpermq $A01, at Theta[1],$A01
- vpermq $A02, at Theta[2],$A02
- vpermq $A03, at Theta[3],$A03
- vpermq $A04, at Theta[4],$A04
-
+ ######################################### Theta, even round
vmovdqa64 $A00, at T[0] # put aside original A00
- vpternlogq \$0x96,$A02,$A01,$A00 # and use it as "C00"
- vpternlogq \$0x96,$A04,$A03,$A00
+ vpternlogq \$0x96,$A20,$A10,$A00 # and use it as "C00"
+ vpternlogq \$0x96,$A40,$A30,$A00
vprolq \$1,$A00,$D00
vpermq $A00, at Theta[1],$A00
vpermq $D00, at Theta[4],$D00
vpternlogq \$0x96,$A00,$D00, at T[0] # T[0] is original A00
- vpternlogq \$0x96,$A00,$D00,$A01
- vpternlogq \$0x96,$A00,$D00,$A02
- vpternlogq \$0x96,$A00,$D00,$A03
- vpternlogq \$0x96,$A00,$D00,$A04
+ vpternlogq \$0x96,$A00,$D00,$A10
+ vpternlogq \$0x96,$A00,$D00,$A20
+ vpternlogq \$0x96,$A00,$D00,$A30
+ vpternlogq \$0x96,$A00,$D00,$A40
######################################### Rho
- vprolvq @Rhotate[0], at T[0],$A00 # T[0] is original A00
- vprolvq @Rhotate[1],$A01,$A01
- vprolvq @Rhotate[2],$A02,$A02
- vprolvq @Rhotate[3],$A03,$A03
- vprolvq @Rhotate[4],$A04,$A04
+ vprolvq @Rhotate0[0], at T[0],$A00 # T[0] is original A00
+ vprolvq @Rhotate0[1],$A10,$A10
+ vprolvq @Rhotate0[2],$A20,$A20
+ vprolvq @Rhotate0[3],$A30,$A30
+ vprolvq @Rhotate0[4],$A40,$A40
######################################### Pi
- vpblendmq $A02,$A00,@{T[0]}{$k00010}
- vpblendmq $A00,$A03,@{T[1]}{$k00010}
- vpblendmq $A03,$A01,@{T[2]}{$k00010}
- vpblendmq $A01,$A04,@{T[3]}{$k00010}
- vpblendmq $A04,$A02,@{T[4]}{$k00010}
-
- vpblendmq $A04, at T[0],@{T[0]}{$k00100}
- vpblendmq $A02, at T[1],@{T[1]}{$k00100}
- vpblendmq $A00, at T[2],@{T[2]}{$k00100}
- vpblendmq $A03, at T[3],@{T[3]}{$k00100}
- vpblendmq $A01, at T[4],@{T[4]}{$k00100}
-
- vpblendmq $A01, at T[0],@{T[0]}{$k01000}
- vpblendmq $A04, at T[1],@{T[1]}{$k01000}
- vpblendmq $A02, at T[2],@{T[2]}{$k01000}
- vpblendmq $A00, at T[3],@{T[3]}{$k01000}
- vpblendmq $A03, at T[4],@{T[4]}{$k01000}
-
- vpblendmq $A03, at T[0],@{T[0]}{$k10000}
- vpblendmq $A01, at T[1],@{T[1]}{$k10000}
- vpblendmq $A04, at T[2],@{T[2]}{$k10000}
- vpblendmq $A02, at T[3],@{T[3]}{$k10000}
- vpblendmq $A00, at T[4],@{T[4]}{$k10000}
-
- vpermq @T[0], at Chi[0],$A00
- vpermq @T[1], at Chi[1],$A01
- vpermq @T[2], at Chi[2],$A02
- vpermq @T[3], at Chi[3],$A03
- vpermq @T[4], at Chi[4],$A04
+ vpermq $A00, at Pi0[0],$A00
+ vpermq $A10, at Pi0[1],$A10
+ vpermq $A20, at Pi0[2],$A20
+ vpermq $A30, at Pi0[3],$A30
+ vpermq $A40, at Pi0[4],$A40
######################################### Chi
vmovdqa64 $A00, at T[0]
- vpternlogq \$0xD2,$A02,$A01,$A00
- vmovdqa64 $A01, at T[1]
- vpternlogq \$0xD2,$A03,$A02,$A01
- vpternlogq \$0xD2,$A04,$A03,$A02
- vpternlogq \$0xD2, at T[0],$A04,$A03
- vpternlogq \$0xD2, at T[1], at T[0],$A04
+ vmovdqa64 $A10, at T[1]
+ vpternlogq \$0xD2,$A20,$A10,$A00
+ vpternlogq \$0xD2,$A30,$A20,$A10
+ vpternlogq \$0xD2,$A40,$A30,$A20
+ vpternlogq \$0xD2, at T[0],$A40,$A30
+ vpternlogq \$0xD2, at T[1], at T[0],$A40
######################################### Iota
vpxorq (%r10),$A00,${A00}{$k00001}
- lea 8(%r10),%r10
+ lea 16(%r10),%r10
+
+ ######################################### Harmonize rounds
+ vpblendmq $A20,$A10,@{T[1]}{$k00010}
+ vpblendmq $A30,$A20,@{T[2]}{$k00010}
+ vpblendmq $A40,$A30,@{T[3]}{$k00010}
+ vpblendmq $A10,$A00,@{T[0]}{$k00010}
+ vpblendmq $A00,$A40,@{T[4]}{$k00010}
+
+ vpblendmq $A30, at T[1],@{T[1]}{$k00100}
+ vpblendmq $A40, at T[2],@{T[2]}{$k00100}
+ vpblendmq $A20, at T[0],@{T[0]}{$k00100}
+ vpblendmq $A00, at T[3],@{T[3]}{$k00100}
+ vpblendmq $A10, at T[4],@{T[4]}{$k00100}
+
+ vpblendmq $A40, at T[1],@{T[1]}{$k01000}
+ vpblendmq $A30, at T[0],@{T[0]}{$k01000}
+ vpblendmq $A00, at T[2],@{T[2]}{$k01000}
+ vpblendmq $A10, at T[3],@{T[3]}{$k01000}
+ vpblendmq $A20, at T[4],@{T[4]}{$k01000}
+
+ vpblendmq $A40, at T[0],@{T[0]}{$k10000}
+ vpblendmq $A00, at T[1],@{T[1]}{$k10000}
+ vpblendmq $A10, at T[2],@{T[2]}{$k10000}
+ vpblendmq $A20, at T[3],@{T[3]}{$k10000}
+ vpblendmq $A30, at T[4],@{T[4]}{$k10000}
+
+ #vpermq @T[0], at Theta[0],$A00 # doesn't actually change order
+ vpermq @T[1], at Theta[1],$A10
+ vpermq @T[2], at Theta[2],$A20
+ vpermq @T[3], at Theta[3],$A30
+ vpermq @T[4], at Theta[4],$A40
+
+ ######################################### Theta, odd round
+ vmovdqa64 $T[0],$A00 # real A00
+ vpternlogq \$0x96,$A20,$A10,$C00 # C00 is @T[0]'s alias
+ vpternlogq \$0x96,$A40,$A30,$C00
+
+ vprolq \$1,$C00,$D00
+ vpermq $C00, at Theta[1],$C00
+ vpermq $D00, at Theta[4],$D00
+
+ vpternlogq \$0x96,$C00,$D00,$A00
+ vpternlogq \$0x96,$C00,$D00,$A30
+ vpternlogq \$0x96,$C00,$D00,$A10
+ vpternlogq \$0x96,$C00,$D00,$A40
+ vpternlogq \$0x96,$C00,$D00,$A20
+
+ ######################################### Rho
+ vprolvq @Rhotate1[0],$A00,$A00
+ vprolvq @Rhotate1[3],$A30, at T[1]
+ vprolvq @Rhotate1[1],$A10, at T[2]
+ vprolvq @Rhotate1[4],$A40, at T[3]
+ vprolvq @Rhotate1[2],$A20, at T[4]
+
+ vpermq $A00, at Theta[4], at T[5]
+ vpermq $A00, at Theta[3], at T[6]
+
+ ######################################### Iota
+ vpxorq -8(%r10),$A00,${A00}{$k00001}
+
+ ######################################### Pi
+ vpermq @T[1], at Theta[2],$A10
+ vpermq @T[2], at Theta[4],$A20
+ vpermq @T[3], at Theta[1],$A30
+ vpermq @T[4], at Theta[3],$A40
+
+ ######################################### Chi
+ vpternlogq \$0xD2, at T[6], at T[5],$A00
+
+ vpermq @T[1], at Theta[1], at T[7]
+ #vpermq @T[1], at Theta[0], at T[1]
+ vpternlogq \$0xD2, at T[1], at T[7],$A10
+
+ vpermq @T[2], at Theta[3], at T[0]
+ vpermq @T[2], at Theta[2], at T[2]
+ vpternlogq \$0xD2, at T[2], at T[0],$A20
+
+ #vpermq @T[3], at Theta[0], at T[3]
+ vpermq @T[3], at Theta[4], at T[1]
+ vpternlogq \$0xD2, at T[1], at T[3],$A30
+
+ vpermq @T[4], at Theta[2], at T[0]
+ vpermq @T[4], at Theta[1], at T[4]
+ vpternlogq \$0xD2, at T[4], at T[0],$A40
dec %eax
jnz .Loop_avx512
@@ -208,8 +293,6 @@ SHA3_absorb:
lea 96($inp),$inp
lea 128(%rsp),%r9
- vzeroupper
-
lea theta_perm(%rip),%r8
kxnorw $k11111,$k11111,$k11111
@@ -226,24 +309,30 @@ SHA3_absorb:
vmovdqa64 64*3(%r8), at Theta[3]
vmovdqa64 64*4(%r8), at Theta[4]
- vmovdqa64 64*5(%r8), at Rhotate[0]
- vmovdqa64 64*6(%r8), at Rhotate[1]
- vmovdqa64 64*7(%r8), at Rhotate[2]
- vmovdqa64 64*8(%r8), at Rhotate[3]
- vmovdqa64 64*9(%r8), at Rhotate[4]
+ vmovdqa64 64*5(%r8), at Rhotate1[0]
+ vmovdqa64 64*6(%r8), at Rhotate1[1]
+ vmovdqa64 64*7(%r8), at Rhotate1[2]
+ vmovdqa64 64*8(%r8), at Rhotate1[3]
+ vmovdqa64 64*9(%r8), at Rhotate1[4]
+
+ vmovdqa64 64*10(%r8), at Rhotate0[0]
+ vmovdqa64 64*11(%r8), at Rhotate0[1]
+ vmovdqa64 64*12(%r8), at Rhotate0[2]
+ vmovdqa64 64*13(%r8), at Rhotate0[3]
+ vmovdqa64 64*14(%r8), at Rhotate0[4]
- vmovdqa64 64*10(%r8), at Chi[0]
- vmovdqa64 64*11(%r8), at Chi[1]
- vmovdqa64 64*12(%r8), at Chi[2]
- vmovdqa64 64*13(%r8), at Chi[3]
- vmovdqa64 64*14(%r8), at Chi[4]
+ vmovdqa64 64*15(%r8), at Pi0[0]
+ vmovdqa64 64*16(%r8), at Pi0[1]
+ vmovdqa64 64*17(%r8), at Pi0[2]
+ vmovdqa64 64*18(%r8), at Pi0[3]
+ vmovdqa64 64*19(%r8), at Pi0[4]
vmovdqu64 40*0-96($A_flat),${A00}{$k11111}{z}
vpxorq @T[0], at T[0], at T[0]
- vmovdqu64 40*1-96($A_flat),${A01}{$k11111}{z}
- vmovdqu64 40*2-96($A_flat),${A02}{$k11111}{z}
- vmovdqu64 40*3-96($A_flat),${A03}{$k11111}{z}
- vmovdqu64 40*4-96($A_flat),${A04}{$k11111}{z}
+ vmovdqu64 40*1-96($A_flat),${A10}{$k11111}{z}
+ vmovdqu64 40*2-96($A_flat),${A20}{$k11111}{z}
+ vmovdqu64 40*3-96($A_flat),${A30}{$k11111}{z}
+ vmovdqu64 40*4-96($A_flat),${A40}{$k11111}{z}
vmovdqa64 @T[0],0*64-128(%r9) # zero transfer area on stack
vmovdqa64 @T[0],1*64-128(%r9)
@@ -263,7 +352,7 @@ ___
for(my $i=0; $i<25; $i++) {
$code.=<<___
mov 8*$i-96($inp),%r8
- mov %r8,$A_jagged_in[$i]-128(%r9)
+ mov %r8,$A_jagged[$i]-128(%r9)
dec %eax
jz .Labsorved_avx512
___
@@ -273,10 +362,10 @@ $code.=<<___;
lea ($inp,$bsz),$inp
vpxorq 64*0-128(%r9),$A00,$A00
- vpxorq 64*1-128(%r9),$A01,$A01
- vpxorq 64*2-128(%r9),$A02,$A02
- vpxorq 64*3-128(%r9),$A03,$A03
- vpxorq 64*4-128(%r9),$A04,$A04
+ vpxorq 64*1-128(%r9),$A10,$A10
+ vpxorq 64*2-128(%r9),$A20,$A20
+ vpxorq 64*3-128(%r9),$A30,$A30
+ vpxorq 64*4-128(%r9),$A40,$A40
call __KeccakF1600
@@ -285,10 +374,10 @@ $code.=<<___;
.align 32
.Ldone_absorb_avx512:
vmovdqu64 $A00,40*0-96($A_flat){$k11111}
- vmovdqu64 $A01,40*1-96($A_flat){$k11111}
- vmovdqu64 $A02,40*2-96($A_flat){$k11111}
- vmovdqu64 $A03,40*3-96($A_flat){$k11111}
- vmovdqu64 $A04,40*4-96($A_flat){$k11111}
+ vmovdqu64 $A10,40*1-96($A_flat){$k11111}
+ vmovdqu64 $A20,40*2-96($A_flat){$k11111}
+ vmovdqu64 $A30,40*3-96($A_flat){$k11111}
+ vmovdqu64 $A40,40*4-96($A_flat){$k11111}
vzeroupper
@@ -307,8 +396,6 @@ SHA3_squeeze:
cmp $bsz,$len
jbe .Lno_output_extension_avx512
- vzeroupper
-
lea theta_perm(%rip),%r8
kxnorw $k11111,$k11111,$k11111
@@ -325,65 +412,72 @@ SHA3_squeeze:
vmovdqa64 64*3(%r8), at Theta[3]
vmovdqa64 64*4(%r8), at Theta[4]
- vmovdqa64 64*5(%r8), at Rhotate[0]
- vmovdqa64 64*6(%r8), at Rhotate[1]
- vmovdqa64 64*7(%r8), at Rhotate[2]
- vmovdqa64 64*8(%r8), at Rhotate[3]
- vmovdqa64 64*9(%r8), at Rhotate[4]
+ vmovdqa64 64*5(%r8), at Rhotate1[0]
+ vmovdqa64 64*6(%r8), at Rhotate1[1]
+ vmovdqa64 64*7(%r8), at Rhotate1[2]
+ vmovdqa64 64*8(%r8), at Rhotate1[3]
+ vmovdqa64 64*9(%r8), at Rhotate1[4]
+
+ vmovdqa64 64*10(%r8), at Rhotate0[0]
+ vmovdqa64 64*11(%r8), at Rhotate0[1]
+ vmovdqa64 64*12(%r8), at Rhotate0[2]
+ vmovdqa64 64*13(%r8), at Rhotate0[3]
+ vmovdqa64 64*14(%r8), at Rhotate0[4]
- vmovdqa64 64*10(%r8), at Chi[0]
- vmovdqa64 64*11(%r8), at Chi[1]
- vmovdqa64 64*12(%r8), at Chi[2]
- vmovdqa64 64*13(%r8), at Chi[3]
- vmovdqa64 64*14(%r8), at Chi[4]
+ vmovdqa64 64*15(%r8), at Pi0[0]
+ vmovdqa64 64*16(%r8), at Pi0[1]
+ vmovdqa64 64*17(%r8), at Pi0[2]
+ vmovdqa64 64*18(%r8), at Pi0[3]
+ vmovdqa64 64*19(%r8), at Pi0[4]
vmovdqu64 40*0-96($A_flat),${A00}{$k11111}{z}
- vmovdqu64 40*1-96($A_flat),${A01}{$k11111}{z}
- vmovdqu64 40*2-96($A_flat),${A02}{$k11111}{z}
- vmovdqu64 40*3-96($A_flat),${A03}{$k11111}{z}
- vmovdqu64 40*4-96($A_flat),${A04}{$k11111}{z}
+ vmovdqu64 40*1-96($A_flat),${A10}{$k11111}{z}
+ vmovdqu64 40*2-96($A_flat),${A20}{$k11111}{z}
+ vmovdqu64 40*3-96($A_flat),${A30}{$k11111}{z}
+ vmovdqu64 40*4-96($A_flat),${A40}{$k11111}{z}
.Lno_output_extension_avx512:
shr \$3,$bsz
+ lea -96($A_flat),%r9
mov $bsz,%rax
+ jmp .Loop_squeeze_avx512
+.align 32
.Loop_squeeze_avx512:
- mov @A_jagged_out[$i]-96($A_flat),%r8
-___
-for (my $i=0; $i<25; $i++) {
-$code.=<<___;
- sub \$8,$len
- jc .Ltail_squeeze_avx512
+ cmp \$8,$len
+ jb .Ltail_squeeze_avx512
+
+ mov (%r9),%r8
+ lea 8(%r9),%r9
mov %r8,($out)
lea 8($out),$out
- je .Ldone_squeeze_avx512
- dec %eax
- je .Lextend_output_avx512
- mov @A_jagged_out[$i+1]-96($A_flat),%r8
-___
-}
-$code.=<<___;
-.Lextend_output_avx512:
- call __KeccakF1600
+ sub \$8,$len # len -= 8
+ jz .Ldone_squeeze_avx512
+
+ sub \$1,%rax # bsz--
+ jnz .Loop_squeeze_avx512
+
+ #vpermq @Theta[4], at Theta[4], at Theta[3]
+ #vpermq @Theta[3], at Theta[4], at Theta[2]
+ #vpermq @Theta[3], at Theta[3], at Theta[1]
+
+ call __KeccakF1600
vmovdqu64 $A00,40*0-96($A_flat){$k11111}
- vmovdqu64 $A01,40*1-96($A_flat){$k11111}
- vmovdqu64 $A02,40*2-96($A_flat){$k11111}
- vmovdqu64 $A03,40*3-96($A_flat){$k11111}
- vmovdqu64 $A04,40*4-96($A_flat){$k11111}
+ vmovdqu64 $A10,40*1-96($A_flat){$k11111}
+ vmovdqu64 $A20,40*2-96($A_flat){$k11111}
+ vmovdqu64 $A30,40*3-96($A_flat){$k11111}
+ vmovdqu64 $A40,40*4-96($A_flat){$k11111}
+ lea -96($A_flat),%r9
mov $bsz,%rax
jmp .Loop_squeeze_avx512
-
.Ltail_squeeze_avx512:
- add \$8,$len
-.Loop_tail_avx512:
- mov %r8b,($out)
- lea 1($out),$out
- shr \$8,%r8
- dec $len
- jnz .Loop_tail_avx512
+ mov %r9,%rsi
+ mov $out,%rdi
+ mov $len,%rcx
+ .byte 0xf3,0xa4 # rep movsb
.Ldone_squeeze_avx512:
vzeroupper
@@ -400,19 +494,27 @@ theta_perm:
.quad 2, 3, 4, 0, 1, 5, 6, 7
.quad 1, 2, 3, 4, 0, 5, 6, 7
-rhotates:
+rhotates1:
.quad 0, 44, 43, 21, 14, 0, 0, 0 # [0][0] [1][1] [2][2] [3][3] [4][4]
.quad 18, 1, 6, 25, 8, 0, 0, 0 # [4][0] [0][1] [1][2] [2][3] [3][4]
.quad 41, 2, 62, 55, 39, 0, 0, 0 # [3][0] [4][1] [0][2] [1][3] [2][4]
.quad 3, 45, 61, 28, 20, 0, 0, 0 # [2][0] [3][1] [4][2] [0][3] [1][4]
.quad 36, 10, 15, 56, 27, 0, 0, 0 # [1][0] [2][1] [3][2] [4][3] [0][4]
-chi_perm:
- .quad 0, 4, 3, 2, 1, 5, 6, 7
- .quad 1, 0, 4, 3, 2, 5, 6, 7
- .quad 2, 1, 0, 4, 3, 5, 6, 7
- .quad 3, 2, 1, 0, 4, 5, 6, 7
- .quad 4, 3, 2, 1, 0, 5, 6, 7
+rhotates0:
+ .quad 0, 1, 62, 28, 27, 0, 0, 0
+ .quad 36, 44, 6, 55, 20, 0, 0, 0
+ .quad 3, 10, 43, 25, 39, 0, 0, 0
+ .quad 41, 45, 15, 21, 8, 0, 0, 0
+ .quad 18, 2, 61, 56, 14, 0, 0, 0
+
+pi0_perm:
+ .quad 0, 3, 1, 4, 2, 5, 6, 7
+ .quad 1, 4, 2, 0, 3, 5, 6, 7
+ .quad 2, 0, 3, 1, 4, 5, 6, 7
+ .quad 3, 1, 4, 2, 0, 5, 6, 7
+ .quad 4, 2, 0, 3, 1, 5, 6, 7
+
iotas:
.quad 0x0000000000000001
More information about the openssl-commits
mailing list