Hakase/openssl-custom
1
0 Fork 0
Code Issues Pull Requests Releases Wiki Activity

OpenSSL 1.1.1-pre2

Browse Source
This commit is contained in:
Hakase
2018-04-07 17:29:40 +09:00
parent 82a44d2483
commit bbac8ca55d
17755 changed files with 221242 additions and 98415 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crypto/sha/asm/keccak1600-armv4.pl
Executable
+1586
View File
@@ -0,0 +1,1586 @@
#!/usr/bin/env perl
# Copyright 2017 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License"). You may not use
# this file except in compliance with the License. You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html
#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# Keccak-1600 for ARMv4.
#
# June 2017.
#
# Non-NEON code is KECCAK_1X variant (see sha/keccak1600.c) with bit
# interleaving. How does it compare to Keccak Code Package? It's as
# fast, but several times smaller, and is endian- and ISA-neutral. ISA
# neutrality means that minimum ISA requirement is ARMv4, yet it can
# be assembled even as Thumb-2. NEON code path is KECCAK_1X_ALT with
# register layout taken from Keccak Code Package. It's also as fast,
# in fact faster by 10-15% on some processors, and endian-neutral.
#
# August 2017.
#
# Switch to KECCAK_2X variant for non-NEON code and merge almost 1/2
# of rotate instructions with logical ones. This resulted in ~10%
# improvement on most processors. Switch to KECCAK_2X effectively
# minimizes re-loads from temporary storage, and merged rotates just
# eliminate corresponding instructions. As for latter. When examining
# code you'll notice commented ror instructions. These are eliminated
# ones, and you should trace destination register below to see what's
# going on. Just in case, why not all rotates are eliminated. Trouble
# is that you have operations that require both inputs to be rotated,
# e.g. 'eor a,b>>>x,c>>>y'. This conundrum is resolved by using
# 'eor a,b,c>>>(x-y)' and then merge-rotating 'a' in next operation
# that takes 'a' as input. And thing is that this next operation can
# be in next round. It's totally possible to "carry" rotate "factors"
# to the next round, but it makes code more complex. And the last word
# is the keyword, i.e. "almost 1/2" is kind of complexity cap [for the
# time being]...
#
# Reduce per-round instruction count in Thumb-2 case by 16%. This is
# achieved by folding ldr/str pairs to their double-word counterparts.
# Theoretically this should have improved performance on single-issue
# cores, such as Cortex-A5/A7, by 19%. Reality is a bit different, as
# usual...
#
########################################################################
# Numbers are cycles per processed byte. Non-NEON results account even
# for input bit interleaving.
#
# r=1088(*) Thumb-2(**) NEON
#
# ARM11xx 82/+150%
# Cortex-A5 88/+160%, 86, 36
# Cortex-A7 78/+160%, 68, 34
# Cortex-A8 51/+230%, 57, 30
# Cortex-A9 53/+210%, 51, 26
# Cortex-A15 42/+160%, 38, 18
# Snapdragon S4 43/+210%, 38, 24
#
# (*) Corresponds to SHA3-256. Percentage after slash is improvement
# over compiler-generated KECCAK_2X reference code.
# (**) Thumb-2 results for Cortex-A5/A7 are likely to apply even to
# Cortex-Mx, x>=3. Otherwise, non-NEON results for NEON-capable
# processors are presented mostly for reference purposes.
my @C = map("r$_",(0..9));
my @E = map("r$_",(10..12,14));
########################################################################
# Stack layout
# ----->+-----------------------+
# | uint64_t A[5][5] |
# | ... |
# +200->+-----------------------+
# | uint64_t D[5] |
# | ... |
# +240->+-----------------------+
# | uint64_t T[5][5] |
# | ... |
# +440->+-----------------------+
# | saved lr |
# +444->+-----------------------+
# | loop counter |
# +448->+-----------------------+
# | ...
my @A = map([ 8*$_, 8*($_+1), 8*($_+2), 8*($_+3), 8*($_+4) ], (0,5,10,15,20));
my @D = map(8*$_, (25..29));
my @T = map([ 8*$_, 8*($_+1), 8*($_+2), 8*($_+3), 8*($_+4) ], (30,35,40,45,50));
$code.=<<___;
.text
#if defined(__thumb2__)
.syntax unified
.thumb
#else
.code 32
#endif
.type iotas32, %object
.align 5
iotas32:
.long 0x00000001, 0x00000000
.long 0x00000000, 0x00000089
.long 0x00000000, 0x8000008b
.long 0x00000000, 0x80008080
.long 0x00000001, 0x0000008b
.long 0x00000001, 0x00008000
.long 0x00000001, 0x80008088
.long 0x00000001, 0x80000082
.long 0x00000000, 0x0000000b
.long 0x00000000, 0x0000000a
.long 0x00000001, 0x00008082
.long 0x00000000, 0x00008003
.long 0x00000001, 0x0000808b
.long 0x00000001, 0x8000000b
.long 0x00000001, 0x8000008a
.long 0x00000001, 0x80000081
.long 0x00000000, 0x80000081
.long 0x00000000, 0x80000008
.long 0x00000000, 0x00000083
.long 0x00000000, 0x80008003
.long 0x00000001, 0x80008088
.long 0x00000000, 0x80000088
.long 0x00000001, 0x00008000
.long 0x00000000, 0x80008082
.size iotas32,.-iotas32
.type KeccakF1600_int, %function
.align 5
KeccakF1600_int:
add @C[9],sp,#$A[4][2]
add @E[2],sp,#$A[0][0]
add @E[0],sp,#$A[1][0]
ldmia @C[9],{@C[4]-@C[9]} @ A[4][2..4]
KeccakF1600_enter:
str lr,[sp,#440]
eor @E[1],@E[1],@E[1]
str @E[1],[sp,#444]
b .Lround2x
.align 4
.Lround2x:
___
sub Round {
my (@A,@R); (@A[0..4],@R) = @_;
$code.=<<___;
ldmia @E[2],{@C[0]-@C[3]} @ A[0][0..1]
ldmia @E[0],{@E[0]-@E[2],@E[3]} @ A[1][0..1]
#ifdef __thumb2__
eor @C[0],@C[0],@E[0]
eor @C[1],@C[1],@E[1]
eor @C[2],@C[2],@E[2]
ldrd @E[0],@E[1],[sp,#$A[1][2]]
eor @C[3],@C[3],@E[3]
ldrd @E[2],@E[3],[sp,#$A[1][3]]
eor @C[4],@C[4],@E[0]
eor @C[5],@C[5],@E[1]
eor @C[6],@C[6],@E[2]
ldrd @E[0],@E[1],[sp,#$A[1][4]]
eor @C[7],@C[7],@E[3]
ldrd @E[2],@E[3],[sp,#$A[2][0]]
eor @C[8],@C[8],@E[0]
eor @C[9],@C[9],@E[1]
eor @C[0],@C[0],@E[2]
ldrd @E[0],@E[1],[sp,#$A[2][1]]
eor @C[1],@C[1],@E[3]
ldrd @E[2],@E[3],[sp,#$A[2][2]]
eor @C[2],@C[2],@E[0]
eor @C[3],@C[3],@E[1]
eor @C[4],@C[4],@E[2]
ldrd @E[0],@E[1],[sp,#$A[2][3]]
eor @C[5],@C[5],@E[3]
ldrd @E[2],@E[3],[sp,#$A[2][4]]
eor @C[6],@C[6],@E[0]
eor @C[7],@C[7],@E[1]
eor @C[8],@C[8],@E[2]
ldrd @E[0],@E[1],[sp,#$A[3][0]]
eor @C[9],@C[9],@E[3]
ldrd @E[2],@E[3],[sp,#$A[3][1]]
eor @C[0],@C[0],@E[0]
eor @C[1],@C[1],@E[1]
eor @C[2],@C[2],@E[2]
ldrd @E[0],@E[1],[sp,#$A[3][2]]
eor @C[3],@C[3],@E[3]
ldrd @E[2],@E[3],[sp,#$A[3][3]]
eor @C[4],@C[4],@E[0]
eor @C[5],@C[5],@E[1]
eor @C[6],@C[6],@E[2]
ldrd @E[0],@E[1],[sp,#$A[3][4]]
eor @C[7],@C[7],@E[3]
ldrd @E[2],@E[3],[sp,#$A[4][0]]
eor @C[8],@C[8],@E[0]
eor @C[9],@C[9],@E[1]
eor @C[0],@C[0],@E[2]
ldrd @E[0],@E[1],[sp,#$A[4][1]]
eor @C[1],@C[1],@E[3]
ldrd @E[2],@E[3],[sp,#$A[0][2]]
eor @C[2],@C[2],@E[0]
eor @C[3],@C[3],@E[1]
eor @C[4],@C[4],@E[2]
ldrd @E[0],@E[1],[sp,#$A[0][3]]
eor @C[5],@C[5],@E[3]
ldrd @E[2],@E[3],[sp,#$A[0][4]]
#else
eor @C[0],@C[0],@E[0]
add @E[0],sp,#$A[1][2]
eor @C[1],@C[1],@E[1]
eor @C[2],@C[2],@E[2]
eor @C[3],@C[3],@E[3]
ldmia @E[0],{@E[0]-@E[2],@E[3]} @ A[1][2..3]
eor @C[4],@C[4],@E[0]
add @E[0],sp,#$A[1][4]
eor @C[5],@C[5],@E[1]
eor @C[6],@C[6],@E[2]
eor @C[7],@C[7],@E[3]
ldmia @E[0],{@E[0]-@E[2],@E[3]} @ A[1][4]..A[2][0]
eor @C[8],@C[8],@E[0]
add @E[0],sp,#$A[2][1]
eor @C[9],@C[9],@E[1]
eor @C[0],@C[0],@E[2]
eor @C[1],@C[1],@E[3]
ldmia @E[0],{@E[0]-@E[2],@E[3]} @ A[2][1..2]
eor @C[2],@C[2],@E[0]
add @E[0],sp,#$A[2][3]
eor @C[3],@C[3],@E[1]
eor @C[4],@C[4],@E[2]
eor @C[5],@C[5],@E[3]
ldmia @E[0],{@E[0]-@E[2],@E[3]} @ A[2][3..4]
eor @C[6],@C[6],@E[0]
add @E[0],sp,#$A[3][0]
eor @C[7],@C[7],@E[1]
eor @C[8],@C[8],@E[2]
eor @C[9],@C[9],@E[3]
ldmia @E[0],{@E[0]-@E[2],@E[3]} @ A[3][0..1]
eor @C[0],@C[0],@E[0]
add @E[0],sp,#$A[3][2]
eor @C[1],@C[1],@E[1]
eor @C[2],@C[2],@E[2]
eor @C[3],@C[3],@E[3]
ldmia @E[0],{@E[0]-@E[2],@E[3]} @ A[3][2..3]
eor @C[4],@C[4],@E[0]
add @E[0],sp,#$A[3][4]
eor @C[5],@C[5],@E[1]
eor @C[6],@C[6],@E[2]
eor @C[7],@C[7],@E[3]
ldmia @E[0],{@E[0]-@E[2],@E[3]} @ A[3][4]..A[4][0]
eor @C[8],@C[8],@E[0]
ldr @E[0],[sp,#$A[4][1]] @ A[4][1]
eor @C[9],@C[9],@E[1]
ldr @E[1],[sp,#$A[4][1]+4]
eor @C[0],@C[0],@E[2]
ldr @E[2],[sp,#$A[0][2]] @ A[0][2]
eor @C[1],@C[1],@E[3]
ldr @E[3],[sp,#$A[0][2]+4]
eor @C[2],@C[2],@E[0]
add @E[0],sp,#$A[0][3]
eor @C[3],@C[3],@E[1]
eor @C[4],@C[4],@E[2]
eor @C[5],@C[5],@E[3]
ldmia @E[0],{@E[0]-@E[2],@E[3]} @ A[0][3..4]
#endif
eor @C[6],@C[6],@E[0]
eor @C[7],@C[7],@E[1]
eor @C[8],@C[8],@E[2]
eor @C[9],@C[9],@E[3]
eor @E[0],@C[0],@C[5],ror#32-1 @ E[0] = ROL64(C[2], 1) ^ C[0];
str.l @E[0],[sp,#$D[1]] @ D[1] = E[0]
eor @E[1],@C[1],@C[4]
str.h @E[1],[sp,#$D[1]+4]
eor @E[2],@C[6],@C[1],ror#32-1 @ E[1] = ROL64(C[0], 1) ^ C[3];
eor @E[3],@C[7],@C[0]
str.l @E[2],[sp,#$D[4]] @ D[4] = E[1]
eor @C[0],@C[8],@C[3],ror#32-1 @ C[0] = ROL64(C[1], 1) ^ C[4];
str.h @E[3],[sp,#$D[4]+4]
eor @C[1],@C[9],@C[2]
str.l @C[0],[sp,#$D[0]] @ D[0] = C[0]
eor @C[2],@C[2],@C[7],ror#32-1 @ C[1] = ROL64(C[3], 1) ^ C[1];
ldr.l @C[7],[sp,#$A[3][3]]
eor @C[3],@C[3],@C[6]
str.h @C[1],[sp,#$D[0]+4]
ldr.h @C[6],[sp,#$A[3][3]+4]
str.l @C[2],[sp,#$D[2]] @ D[2] = C[1]
eor @C[4],@C[4],@C[9],ror#32-1 @ C[2] = ROL64(C[4], 1) ^ C[2];
str.h @C[3],[sp,#$D[2]+4]
eor @C[5],@C[5],@C[8]
ldr.l @C[8],[sp,#$A[4][4]]
ldr.h @C[9],[sp,#$A[4][4]+4]
str.l @C[4],[sp,#$D[3]] @ D[3] = C[2]
eor @C[7],@C[7],@C[4]
str.h @C[5],[sp,#$D[3]+4]
eor @C[6],@C[6],@C[5]
ldr.l @C[4],[sp,#$A[0][0]]
@ ror @C[7],@C[7],#32-10 @ C[3] = ROL64(A[3][3] ^ C[2], rhotates[3][3]); /* D[3] */
@ ror @C[6],@C[6],#32-11
ldr.h @C[5],[sp,#$A[0][0]+4]
eor @C[8],@C[8],@E[2]
eor @C[9],@C[9],@E[3]
ldr.l @E[2],[sp,#$A[2][2]]
eor @C[0],@C[0],@C[4]
ldr.h @E[3],[sp,#$A[2][2]+4]
@ ror @C[8],@C[8],#32-7 @ C[4] = ROL64(A[4][4] ^ E[1], rhotates[4][4]); /* D[4] */
@ ror @C[9],@C[9],#32-7
eor @C[1],@C[1],@C[5] @ C[0] = A[0][0] ^ C[0]; /* rotate by 0 */ /* D[0] */
eor @E[2],@E[2],@C[2]
ldr.l @C[2],[sp,#$A[1][1]]
eor @E[3],@E[3],@C[3]
ldr.h @C[3],[sp,#$A[1][1]+4]
ror @C[5],@E[2],#32-21 @ C[2] = ROL64(A[2][2] ^ C[1], rhotates[2][2]); /* D[2] */
ldr @E[2],[sp,#444] @ load counter
eor @C[2],@C[2],@E[0]
adr @E[0],iotas32
ror @C[4],@E[3],#32-22
add @E[3],@E[0],@E[2]
eor @C[3],@C[3],@E[1]
___
$code.=<<___ if ($A[0][0] != $T[0][0]);
ldmia @E[3],{@E[0],@E[1]} @ iotas[i]
___
$code.=<<___ if ($A[0][0] == $T[0][0]);
ldr.l @E[0],[@E[3],#8] @ iotas[i].lo
add @E[2],@E[2],#16
ldr.h @E[1],[@E[3],#12] @ iotas[i].hi
cmp @E[2],#192
str @E[2],[sp,#444] @ store counter
___
$code.=<<___;
bic @E[2],@C[4],@C[2],ror#32-22
bic @E[3],@C[5],@C[3],ror#32-22
ror @C[2],@C[2],#32-22 @ C[1] = ROL64(A[1][1] ^ E[0], rhotates[1][1]); /* D[1] */
ror @C[3],@C[3],#32-22
eor @E[2],@E[2],@C[0]
eor @E[3],@E[3],@C[1]
eor @E[0],@E[0],@E[2]
eor @E[1],@E[1],@E[3]
str.l @E[0],[sp,#$R[0][0]] @ R[0][0] = C[0] ^ (~C[1] & C[2]) ^ iotas[i];
bic @E[2],@C[6],@C[4],ror#11
str.h @E[1],[sp,#$R[0][0]+4]
bic @E[3],@C[7],@C[5],ror#10
bic @E[0],@C[8],@C[6],ror#32-(11-7)
bic @E[1],@C[9],@C[7],ror#32-(10-7)
eor @E[2],@C[2],@E[2],ror#32-11
str.l @E[2],[sp,#$R[0][1]] @ R[0][1] = C[1] ^ (~C[2] & C[3]);
eor @E[3],@C[3],@E[3],ror#32-10
str.h @E[3],[sp,#$R[0][1]+4]
eor @E[0],@C[4],@E[0],ror#32-7
eor @E[1],@C[5],@E[1],ror#32-7
str.l @E[0],[sp,#$R[0][2]] @ R[0][2] = C[2] ^ (~C[3] & C[4]);
bic @E[2],@C[0],@C[8],ror#32-7
str.h @E[1],[sp,#$R[0][2]+4]
bic @E[3],@C[1],@C[9],ror#32-7
eor @E[2],@E[2],@C[6],ror#32-11
str.l @E[2],[sp,#$R[0][3]] @ R[0][3] = C[3] ^ (~C[4] & C[0]);
eor @E[3],@E[3],@C[7],ror#32-10
str.h @E[3],[sp,#$R[0][3]+4]
bic @E[0],@C[2],@C[0]
add @E[3],sp,#$D[3]
ldr.l @C[0],[sp,#$A[0][3]] @ A[0][3]
bic @E[1],@C[3],@C[1]
ldr.h @C[1],[sp,#$A[0][3]+4]
eor @E[0],@E[0],@C[8],ror#32-7
eor @E[1],@E[1],@C[9],ror#32-7
str.l @E[0],[sp,#$R[0][4]] @ R[0][4] = C[4] ^ (~C[0] & C[1]);
add @C[9],sp,#$D[0]
str.h @E[1],[sp,#$R[0][4]+4]
ldmia @E[3],{@E[0]-@E[2],@E[3]} @ D[3..4]
ldmia @C[9],{@C[6]-@C[9]} @ D[0..1]
ldr.l @C[2],[sp,#$A[1][4]] @ A[1][4]
eor @C[0],@C[0],@E[0]
ldr.h @C[3],[sp,#$A[1][4]+4]
eor @C[1],@C[1],@E[1]
@ ror @C[0],@C[0],#32-14 @ C[0] = ROL64(A[0][3] ^ D[3], rhotates[0][3]);
ldr.l @E[0],[sp,#$A[3][1]] @ A[3][1]
@ ror @C[1],@C[1],#32-14
ldr.h @E[1],[sp,#$A[3][1]+4]
eor @C[2],@C[2],@E[2]
ldr.l @C[4],[sp,#$A[2][0]] @ A[2][0]
eor @C[3],@C[3],@E[3]
ldr.h @C[5],[sp,#$A[2][0]+4]
@ ror @C[2],@C[2],#32-10 @ C[1] = ROL64(A[1][4] ^ D[4], rhotates[1][4]);
@ ror @C[3],@C[3],#32-10
eor @C[6],@C[6],@C[4]
ldr.l @E[2],[sp,#$D[2]] @ D[2]
eor @C[7],@C[7],@C[5]
ldr.h @E[3],[sp,#$D[2]+4]
ror @C[5],@C[6],#32-1 @ C[2] = ROL64(A[2][0] ^ D[0], rhotates[2][0]);
ror @C[4],@C[7],#32-2
eor @E[0],@E[0],@C[8]
ldr.l @C[8],[sp,#$A[4][2]] @ A[4][2]
eor @E[1],@E[1],@C[9]
ldr.h @C[9],[sp,#$A[4][2]+4]
ror @C[7],@E[0],#32-22 @ C[3] = ROL64(A[3][1] ^ D[1], rhotates[3][1]);
ror @C[6],@E[1],#32-23
bic @E[0],@C[4],@C[2],ror#32-10
bic @E[1],@C[5],@C[3],ror#32-10
eor @E[2],@E[2],@C[8]
eor @E[3],@E[3],@C[9]
ror @C[9],@E[2],#32-30 @ C[4] = ROL64(A[4][2] ^ D[2], rhotates[4][2]);
ror @C[8],@E[3],#32-31
eor @E[0],@E[0],@C[0],ror#32-14
eor @E[1],@E[1],@C[1],ror#32-14
str.l @E[0],[sp,#$R[1][0]] @ R[1][0] = C[0] ^ (~C[1] & C[2])
bic @E[2],@C[6],@C[4]
str.h @E[1],[sp,#$R[1][0]+4]
bic @E[3],@C[7],@C[5]
eor @E[2],@E[2],@C[2],ror#32-10
str.l @E[2],[sp,#$R[1][1]] @ R[1][1] = C[1] ^ (~C[2] & C[3]);
eor @E[3],@E[3],@C[3],ror#32-10
str.h @E[3],[sp,#$R[1][1]+4]
bic @E[0],@C[8],@C[6]
bic @E[1],@C[9],@C[7]
bic @E[2],@C[0],@C[8],ror#14
bic @E[3],@C[1],@C[9],ror#14
eor @E[0],@E[0],@C[4]
eor @E[1],@E[1],@C[5]
str.l @E[0],[sp,#$R[1][2]] @ R[1][2] = C[2] ^ (~C[3] & C[4]);
bic @C[2],@C[2],@C[0],ror#32-(14-10)
str.h @E[1],[sp,#$R[1][2]+4]
eor @E[2],@C[6],@E[2],ror#32-14
bic @E[1],@C[3],@C[1],ror#32-(14-10)
str.l @E[2],[sp,#$R[1][3]] @ R[1][3] = C[3] ^ (~C[4] & C[0]);
eor @E[3],@C[7],@E[3],ror#32-14
str.h @E[3],[sp,#$R[1][3]+4]
add @E[2],sp,#$D[1]
ldr.l @C[1],[sp,#$A[0][1]] @ A[0][1]
eor @E[0],@C[8],@C[2],ror#32-10
ldr.h @C[0],[sp,#$A[0][1]+4]
eor @E[1],@C[9],@E[1],ror#32-10
str.l @E[0],[sp,#$R[1][4]] @ R[1][4] = C[4] ^ (~C[0] & C[1]);
str.h @E[1],[sp,#$R[1][4]+4]
add @C[9],sp,#$D[3]
ldmia @E[2],{@E[0]-@E[2],@E[3]} @ D[1..2]
ldr.l @C[2],[sp,#$A[1][2]] @ A[1][2]
ldr.h @C[3],[sp,#$A[1][2]+4]
ldmia @C[9],{@C[6]-@C[9]} @ D[3..4]
eor @C[1],@C[1],@E[0]
ldr.l @C[4],[sp,#$A[2][3]] @ A[2][3]
eor @C[0],@C[0],@E[1]
ldr.h @C[5],[sp,#$A[2][3]+4]
ror @C[0],@C[0],#32-1 @ C[0] = ROL64(A[0][1] ^ D[1], rhotates[0][1]);
eor @C[2],@C[2],@E[2]
ldr.l @E[0],[sp,#$A[3][4]] @ A[3][4]
eor @C[3],@C[3],@E[3]
ldr.h @E[1],[sp,#$A[3][4]+4]
@ ror @C[2],@C[2],#32-3 @ C[1] = ROL64(A[1][2] ^ D[2], rhotates[1][2]);
ldr.l @E[2],[sp,#$D[0]] @ D[0]
@ ror @C[3],@C[3],#32-3
ldr.h @E[3],[sp,#$D[0]+4]
eor @C[4],@C[4],@C[6]
eor @C[5],@C[5],@C[7]
@ ror @C[5],@C[6],#32-12 @ C[2] = ROL64(A[2][3] ^ D[3], rhotates[2][3]);
@ ror @C[4],@C[7],#32-13 @ [track reverse order below]
eor @E[0],@E[0],@C[8]
ldr.l @C[8],[sp,#$A[4][0]] @ A[4][0]
eor @E[1],@E[1],@C[9]
ldr.h @C[9],[sp,#$A[4][0]+4]
ror @C[6],@E[0],#32-4 @ C[3] = ROL64(A[3][4] ^ D[4], rhotates[3][4]);
ror @C[7],@E[1],#32-4
eor @E[2],@E[2],@C[8]
eor @E[3],@E[3],@C[9]
ror @C[8],@E[2],#32-9 @ C[4] = ROL64(A[4][0] ^ D[0], rhotates[4][0]);
ror @C[9],@E[3],#32-9
bic @E[0],@C[5],@C[2],ror#13-3
bic @E[1],@C[4],@C[3],ror#12-3
bic @E[2],@C[6],@C[5],ror#32-13
bic @E[3],@C[7],@C[4],ror#32-12
eor @E[0],@C[0],@E[0],ror#32-13
eor @E[1],@C[1],@E[1],ror#32-12
str.l @E[0],[sp,#$R[2][0]] @ R[2][0] = C[0] ^ (~C[1] & C[2])
eor @E[2],@E[2],@C[2],ror#32-3
str.h @E[1],[sp,#$R[2][0]+4]
eor @E[3],@E[3],@C[3],ror#32-3
str.l @E[2],[sp,#$R[2][1]] @ R[2][1] = C[1] ^ (~C[2] & C[3]);
bic @E[0],@C[8],@C[6]
bic @E[1],@C[9],@C[7]
str.h @E[3],[sp,#$R[2][1]+4]
eor @E[0],@E[0],@C[5],ror#32-13
eor @E[1],@E[1],@C[4],ror#32-12
str.l @E[0],[sp,#$R[2][2]] @ R[2][2] = C[2] ^ (~C[3] & C[4]);
bic @E[2],@C[0],@C[8]
str.h @E[1],[sp,#$R[2][2]+4]
bic @E[3],@C[1],@C[9]
eor @E[2],@E[2],@C[6]
eor @E[3],@E[3],@C[7]
str.l @E[2],[sp,#$R[2][3]] @ R[2][3] = C[3] ^ (~C[4] & C[0]);
bic @E[0],@C[2],@C[0],ror#3
str.h @E[3],[sp,#$R[2][3]+4]
bic @E[1],@C[3],@C[1],ror#3
ldr.l @C[1],[sp,#$A[0][4]] @ A[0][4] [in reverse order]
eor @E[0],@C[8],@E[0],ror#32-3
ldr.h @C[0],[sp,#$A[0][4]+4]
eor @E[1],@C[9],@E[1],ror#32-3
str.l @E[0],[sp,#$R[2][4]] @ R[2][4] = C[4] ^ (~C[0] & C[1]);
add @C[9],sp,#$D[1]
str.h @E[1],[sp,#$R[2][4]+4]
ldr.l @E[0],[sp,#$D[4]] @ D[4]
ldr.h @E[1],[sp,#$D[4]+4]
ldr.l @E[2],[sp,#$D[0]] @ D[0]
ldr.h @E[3],[sp,#$D[0]+4]
ldmia @C[9],{@C[6]-@C[9]} @ D[1..2]
eor @C[1],@C[1],@E[0]
ldr.l @C[2],[sp,#$A[1][0]] @ A[1][0]
eor @C[0],@C[0],@E[1]
ldr.h @C[3],[sp,#$A[1][0]+4]
@ ror @C[1],@E[0],#32-13 @ C[0] = ROL64(A[0][4] ^ D[4], rhotates[0][4]);
ldr.l @C[4],[sp,#$A[2][1]] @ A[2][1]
@ ror @C[0],@E[1],#32-14 @ [was loaded in reverse order]
ldr.h @C[5],[sp,#$A[2][1]+4]
eor @C[2],@C[2],@E[2]
ldr.l @E[0],[sp,#$A[3][2]] @ A[3][2]
eor @C[3],@C[3],@E[3]
ldr.h @E[1],[sp,#$A[3][2]+4]
@ ror @C[2],@C[2],#32-18 @ C[1] = ROL64(A[1][0] ^ D[0], rhotates[1][0]);
ldr.l @E[2],[sp,#$D[3]] @ D[3]
@ ror @C[3],@C[3],#32-18
ldr.h @E[3],[sp,#$D[3]+4]
eor @C[6],@C[6],@C[4]
eor @C[7],@C[7],@C[5]
ror @C[4],@C[6],#32-5 @ C[2] = ROL64(A[2][1] ^ D[1], rhotates[2][1]);
ror @C[5],@C[7],#32-5
eor @E[0],@E[0],@C[8]
ldr.l @C[8],[sp,#$A[4][3]] @ A[4][3]
eor @E[1],@E[1],@C[9]
ldr.h @C[9],[sp,#$A[4][3]+4]
ror @C[7],@E[0],#32-7 @ C[3] = ROL64(A[3][2] ^ D[2], rhotates[3][2]);
ror @C[6],@E[1],#32-8
eor @E[2],@E[2],@C[8]
eor @E[3],@E[3],@C[9]
ror @C[8],@E[2],#32-28 @ C[4] = ROL64(A[4][3] ^ D[3], rhotates[4][3]);
ror @C[9],@E[3],#32-28
bic @E[0],@C[4],@C[2],ror#32-18
bic @E[1],@C[5],@C[3],ror#32-18
eor @E[0],@E[0],@C[0],ror#32-14
eor @E[1],@E[1],@C[1],ror#32-13
str.l @E[0],[sp,#$R[3][0]] @ R[3][0] = C[0] ^ (~C[1] & C[2])
bic @E[2],@C[6],@C[4]
str.h @E[1],[sp,#$R[3][0]+4]
bic @E[3],@C[7],@C[5]
eor @E[2],@E[2],@C[2],ror#32-18
str.l @E[2],[sp,#$R[3][1]] @ R[3][1] = C[1] ^ (~C[2] & C[3]);
eor @E[3],@E[3],@C[3],ror#32-18
str.h @E[3],[sp,#$R[3][1]+4]
bic @E[0],@C[8],@C[6]
bic @E[1],@C[9],@C[7]
bic @E[2],@C[0],@C[8],ror#14
bic @E[3],@C[1],@C[9],ror#13
eor @E[0],@E[0],@C[4]
eor @E[1],@E[1],@C[5]
str.l @E[0],[sp,#$R[3][2]] @ R[3][2] = C[2] ^ (~C[3] & C[4]);
bic @C[2],@C[2],@C[0],ror#18-14
str.h @E[1],[sp,#$R[3][2]+4]
eor @E[2],@C[6],@E[2],ror#32-14
bic @E[1],@C[3],@C[1],ror#18-13
eor @E[3],@C[7],@E[3],ror#32-13
str.l @E[2],[sp,#$R[3][3]] @ R[3][3] = C[3] ^ (~C[4] & C[0]);
str.h @E[3],[sp,#$R[3][3]+4]
add @E[3],sp,#$D[2]
ldr.l @C[0],[sp,#$A[0][2]] @ A[0][2]
eor @E[0],@C[8],@C[2],ror#32-18
ldr.h @C[1],[sp,#$A[0][2]+4]
eor @E[1],@C[9],@E[1],ror#32-18
str.l @E[0],[sp,#$R[3][4]] @ R[3][4] = C[4] ^ (~C[0] & C[1]);
str.h @E[1],[sp,#$R[3][4]+4]
ldmia @E[3],{@E[0]-@E[2],@E[3]} @ D[2..3]
ldr.l @C[2],[sp,#$A[1][3]] @ A[1][3]
ldr.h @C[3],[sp,#$A[1][3]+4]
ldr.l @C[6],[sp,#$D[4]] @ D[4]
ldr.h @C[7],[sp,#$D[4]+4]
eor @C[0],@C[0],@E[0]
ldr.l @C[4],[sp,#$A[2][4]] @ A[2][4]
eor @C[1],@C[1],@E[1]
ldr.h @C[5],[sp,#$A[2][4]+4]
@ ror @C[0],@C[0],#32-31 @ C[0] = ROL64(A[0][2] ^ D[2], rhotates[0][2]);
ldr.l @C[8],[sp,#$D[0]] @ D[0]
@ ror @C[1],@C[1],#32-31
ldr.h @C[9],[sp,#$D[0]+4]
eor @E[2],@E[2],@C[2]
ldr.l @E[0],[sp,#$A[3][0]] @ A[3][0]
eor @E[3],@E[3],@C[3]
ldr.h @E[1],[sp,#$A[3][0]+4]
ror @C[3],@E[2],#32-27 @ C[1] = ROL64(A[1][3] ^ D[3], rhotates[1][3]);
ldr.l @E[2],[sp,#$D[1]] @ D[1]
ror @C[2],@E[3],#32-28
ldr.h @E[3],[sp,#$D[1]+4]
eor @C[6],@C[6],@C[4]
eor @C[7],@C[7],@C[5]
ror @C[5],@C[6],#32-19 @ C[2] = ROL64(A[2][4] ^ D[4], rhotates[2][4]);
ror @C[4],@C[7],#32-20
eor @E[0],@E[0],@C[8]
ldr.l @C[8],[sp,#$A[4][1]] @ A[4][1]
eor @E[1],@E[1],@C[9]
ldr.h @C[9],[sp,#$A[4][1]+4]
ror @C[7],@E[0],#32-20 @ C[3] = ROL64(A[3][0] ^ D[0], rhotates[3][0]);
ror @C[6],@E[1],#32-21
eor @C[8],@C[8],@E[2]
eor @C[9],@C[9],@E[3]
@ ror @C[8],@C[2],#32-1 @ C[4] = ROL64(A[4][1] ^ D[1], rhotates[4][1]);
@ ror @C[9],@C[3],#32-1
bic @E[0],@C[4],@C[2]
bic @E[1],@C[5],@C[3]
eor @E[0],@E[0],@C[0],ror#32-31
str.l @E[0],[sp,#$R[4][0]] @ R[4][0] = C[0] ^ (~C[1] & C[2])
eor @E[1],@E[1],@C[1],ror#32-31
str.h @E[1],[sp,#$R[4][0]+4]
bic @E[2],@C[6],@C[4]
bic @E[3],@C[7],@C[5]
eor @E[2],@E[2],@C[2]
eor @E[3],@E[3],@C[3]
str.l @E[2],[sp,#$R[4][1]] @ R[4][1] = C[1] ^ (~C[2] & C[3]);
bic @E[0],@C[8],@C[6],ror#1
str.h @E[3],[sp,#$R[4][1]+4]
bic @E[1],@C[9],@C[7],ror#1
bic @E[2],@C[0],@C[8],ror#31-1
bic @E[3],@C[1],@C[9],ror#31-1
eor @C[4],@C[4],@E[0],ror#32-1
str.l @C[4],[sp,#$R[4][2]] @ R[4][2] = C[2] ^= (~C[3] & C[4]);
eor @C[5],@C[5],@E[1],ror#32-1
str.h @C[5],[sp,#$R[4][2]+4]
eor @C[6],@C[6],@E[2],ror#32-31
eor @C[7],@C[7],@E[3],ror#32-31
str.l @C[6],[sp,#$R[4][3]] @ R[4][3] = C[3] ^= (~C[4] & C[0]);
bic @E[0],@C[2],@C[0],ror#32-31
str.h @C[7],[sp,#$R[4][3]+4]
bic @E[1],@C[3],@C[1],ror#32-31
add @E[2],sp,#$R[0][0]
eor @C[8],@E[0],@C[8],ror#32-1
add @E[0],sp,#$R[1][0]
eor @C[9],@E[1],@C[9],ror#32-1
str.l @C[8],[sp,#$R[4][4]] @ R[4][4] = C[4] ^= (~C[0] & C[1]);
str.h @C[9],[sp,#$R[4][4]+4]
___
}
Round(@A,@T);
Round(@T,@A);
$code.=<<___;
blo .Lround2x
ldr pc,[sp,#440]
.size KeccakF1600_int,.-KeccakF1600_int
.type KeccakF1600, %function
.align 5
KeccakF1600:
stmdb sp!,{r0,r4-r11,lr}
sub sp,sp,#440+16 @ space for A[5][5],D[5],T[5][5],...
add @E[0],r0,#$A[1][0]
add @E[1],sp,#$A[1][0]
ldmia r0, {@C[0]-@C[9]} @ copy A[5][5] to stack
stmia sp, {@C[0]-@C[9]}
ldmia @E[0]!,{@C[0]-@C[9]}
stmia @E[1]!,{@C[0]-@C[9]}
ldmia @E[0]!,{@C[0]-@C[9]}
stmia @E[1]!,{@C[0]-@C[9]}
ldmia @E[0]!,{@C[0]-@C[9]}
stmia @E[1]!,{@C[0]-@C[9]}
ldmia @E[0], {@C[0]-@C[9]}
add @E[2],sp,#$A[0][0]
add @E[0],sp,#$A[1][0]
stmia @E[1], {@C[0]-@C[9]}
bl KeccakF1600_enter
ldr @E[1], [sp,#440+16] @ restore pointer to A
ldmia sp, {@C[0]-@C[9]}
stmia @E[1]!,{@C[0]-@C[9]} @ return A[5][5]
ldmia @E[0]!,{@C[0]-@C[9]}
stmia @E[1]!,{@C[0]-@C[9]}
ldmia @E[0]!,{@C[0]-@C[9]}
stmia @E[1]!,{@C[0]-@C[9]}
ldmia @E[0]!,{@C[0]-@C[9]}
stmia @E[1]!,{@C[0]-@C[9]}
ldmia @E[0], {@C[0]-@C[9]}
stmia @E[1], {@C[0]-@C[9]}
add sp,sp,#440+20
ldmia sp!,{r4-r11,pc}
.size KeccakF1600,.-KeccakF1600
___
{ my ($A_flat,$inp,$len,$bsz) = map("r$_",(10..12,14));
########################################################################
# Stack layout
# ----->+-----------------------+
# | uint64_t A[5][5] |
# | ... |
# | ... |
# +456->+-----------------------+
# | 0x55555555 |
# +460->+-----------------------+
# | 0x33333333 |
# +464->+-----------------------+
# | 0x0f0f0f0f |
# +468->+-----------------------+
# | 0x00ff00ff |
# +472->+-----------------------+
# | uint64_t *A |
# +476->+-----------------------+
# | const void *inp |
# +480->+-----------------------+
# | size_t len |
# +484->+-----------------------+
# | size_t bs |
# +488->+-----------------------+
# | ....
$code.=<<___;
.global SHA3_absorb
.type SHA3_absorb,%function
.align 5
SHA3_absorb:
stmdb sp!,{r0-r12,lr}
sub sp,sp,#456+16
add $A_flat,r0,#$A[1][0]
@ mov $inp,r1
mov $len,r2
mov $bsz,r3
cmp r2,r3
blo .Labsorb_abort
add $inp,sp,#0
ldmia r0, {@C[0]-@C[9]} @ copy A[5][5] to stack
stmia $inp!, {@C[0]-@C[9]}
ldmia $A_flat!,{@C[0]-@C[9]}
stmia $inp!, {@C[0]-@C[9]}
ldmia $A_flat!,{@C[0]-@C[9]}
stmia $inp!, {@C[0]-@C[9]}
ldmia $A_flat!,{@C[0]-@C[9]}
stmia $inp!, {@C[0]-@C[9]}
ldmia $A_flat!,{@C[0]-@C[9]}
stmia $inp, {@C[0]-@C[9]}
ldr $inp,[sp,#476] @ restore $inp
#ifdef __thumb2__
mov r9,#0x00ff00ff
mov r8,#0x0f0f0f0f
mov r7,#0x33333333
mov r6,#0x55555555
#else
mov r6,#0x11 @ compose constants
mov r8,#0x0f
mov r9,#0xff
orr r6,r6,r6,lsl#8
orr r8,r8,r8,lsl#8
orr r6,r6,r6,lsl#16 @ 0x11111111
orr r9,r9,r9,lsl#16 @ 0x00ff00ff
orr r8,r8,r8,lsl#16 @ 0x0f0f0f0f
orr r7,r6,r6,lsl#1 @ 0x33333333
orr r6,r6,r6,lsl#2 @ 0x55555555
#endif
str r9,[sp,#468]
str r8,[sp,#464]
str r7,[sp,#460]
str r6,[sp,#456]
b .Loop_absorb
.align 4
.Loop_absorb:
subs r0,$len,$bsz
blo .Labsorbed
add $A_flat,sp,#0
str r0,[sp,#480] @ save len - bsz
.align 4
.Loop_block:
ldrb r0,[$inp],#1
ldrb r1,[$inp],#1
ldrb r2,[$inp],#1
ldrb r3,[$inp],#1
ldrb r4,[$inp],#1
orr r0,r0,r1,lsl#8
ldrb r1,[$inp],#1
orr r0,r0,r2,lsl#16
ldrb r2,[$inp],#1
orr r0,r0,r3,lsl#24 @ lo
ldrb r3,[$inp],#1
orr r1,r4,r1,lsl#8
orr r1,r1,r2,lsl#16
orr r1,r1,r3,lsl#24 @ hi
and r2,r0,r6 @ &=0x55555555
and r0,r0,r6,lsl#1 @ &=0xaaaaaaaa
and r3,r1,r6 @ &=0x55555555
and r1,r1,r6,lsl#1 @ &=0xaaaaaaaa
orr r2,r2,r2,lsr#1
orr r0,r0,r0,lsl#1
orr r3,r3,r3,lsr#1
orr r1,r1,r1,lsl#1
and r2,r2,r7 @ &=0x33333333
and r0,r0,r7,lsl#2 @ &=0xcccccccc
and r3,r3,r7 @ &=0x33333333
and r1,r1,r7,lsl#2 @ &=0xcccccccc
orr r2,r2,r2,lsr#2
orr r0,r0,r0,lsl#2
orr r3,r3,r3,lsr#2
orr r1,r1,r1,lsl#2
and r2,r2,r8 @ &=0x0f0f0f0f
and r0,r0,r8,lsl#4 @ &=0xf0f0f0f0
and r3,r3,r8 @ &=0x0f0f0f0f
and r1,r1,r8,lsl#4 @ &=0xf0f0f0f0
ldmia $A_flat,{r4-r5} @ A_flat[i]
orr r2,r2,r2,lsr#4
orr r0,r0,r0,lsl#4
orr r3,r3,r3,lsr#4
orr r1,r1,r1,lsl#4
and r2,r2,r9 @ &=0x00ff00ff
and r0,r0,r9,lsl#8 @ &=0xff00ff00
and r3,r3,r9 @ &=0x00ff00ff
and r1,r1,r9,lsl#8 @ &=0xff00ff00
orr r2,r2,r2,lsr#8
orr r0,r0,r0,lsl#8
orr r3,r3,r3,lsr#8
orr r1,r1,r1,lsl#8
lsl r2,r2,#16
lsr r1,r1,#16
eor r4,r4,r3,lsl#16
eor r5,r5,r0,lsr#16
eor r4,r4,r2,lsr#16
eor r5,r5,r1,lsl#16
stmia $A_flat!,{r4-r5} @ A_flat[i++] ^= BitInterleave(inp[0..7])
subs $bsz,$bsz,#8
bhi .Loop_block
str $inp,[sp,#476]
bl KeccakF1600_int
add r14,sp,#456
ldmia r14,{r6-r12,r14} @ restore constants and variables
b .Loop_absorb
.align 4
.Labsorbed:
add $inp,sp,#$A[1][0]
ldmia sp, {@C[0]-@C[9]}
stmia $A_flat!,{@C[0]-@C[9]} @ return A[5][5]
ldmia $inp!, {@C[0]-@C[9]}
stmia $A_flat!,{@C[0]-@C[9]}
ldmia $inp!, {@C[0]-@C[9]}
stmia $A_flat!,{@C[0]-@C[9]}
ldmia $inp!, {@C[0]-@C[9]}
stmia $A_flat!,{@C[0]-@C[9]}
ldmia $inp, {@C[0]-@C[9]}
stmia $A_flat, {@C[0]-@C[9]}
.Labsorb_abort:
add sp,sp,#456+32
mov r0,$len @ return value
ldmia sp!,{r4-r12,pc}
.size SHA3_absorb,.-SHA3_absorb
___
}
{ my ($out,$len,$A_flat,$bsz) = map("r$_", (4,5,10,12));
$code.=<<___;
.global SHA3_squeeze
.type SHA3_squeeze,%function
.align 5
SHA3_squeeze:
stmdb sp!,{r0,r3-r10,lr}
mov $A_flat,r0
mov $out,r1
mov $len,r2
mov $bsz,r3
#ifdef __thumb2__
mov r9,#0x00ff00ff
mov r8,#0x0f0f0f0f
mov r7,#0x33333333
mov r6,#0x55555555
#else
mov r6,#0x11 @ compose constants
mov r8,#0x0f
mov r9,#0xff
orr r6,r6,r6,lsl#8
orr r8,r8,r8,lsl#8
orr r6,r6,r6,lsl#16 @ 0x11111111
orr r9,r9,r9,lsl#16 @ 0x00ff00ff
orr r8,r8,r8,lsl#16 @ 0x0f0f0f0f
orr r7,r6,r6,lsl#1 @ 0x33333333
orr r6,r6,r6,lsl#2 @ 0x55555555
#endif
stmdb sp!,{r6-r9}
mov r14,$A_flat
b .Loop_squeeze
.align 4
.Loop_squeeze:
ldmia $A_flat!,{r0,r1} @ A_flat[i++]
lsl r2,r0,#16
lsl r3,r1,#16 @ r3 = r1 << 16
lsr r2,r2,#16 @ r2 = r0 & 0x0000ffff
lsr r1,r1,#16
lsr r0,r0,#16 @ r0 = r0 >> 16
lsl r1,r1,#16 @ r1 = r1 & 0xffff0000
orr r2,r2,r2,lsl#8
orr r3,r3,r3,lsr#8
orr r0,r0,r0,lsl#8
orr r1,r1,r1,lsr#8
and r2,r2,r9 @ &=0x00ff00ff
and r3,r3,r9,lsl#8 @ &=0xff00ff00
and r0,r0,r9 @ &=0x00ff00ff
and r1,r1,r9,lsl#8 @ &=0xff00ff00
orr r2,r2,r2,lsl#4
orr r3,r3,r3,lsr#4
orr r0,r0,r0,lsl#4
orr r1,r1,r1,lsr#4
and r2,r2,r8 @ &=0x0f0f0f0f
and r3,r3,r8,lsl#4 @ &=0xf0f0f0f0
and r0,r0,r8 @ &=0x0f0f0f0f
and r1,r1,r8,lsl#4 @ &=0xf0f0f0f0
orr r2,r2,r2,lsl#2
orr r3,r3,r3,lsr#2
orr r0,r0,r0,lsl#2
orr r1,r1,r1,lsr#2
and r2,r2,r7 @ &=0x33333333
and r3,r3,r7,lsl#2 @ &=0xcccccccc
and r0,r0,r7 @ &=0x33333333
and r1,r1,r7,lsl#2 @ &=0xcccccccc
orr r2,r2,r2,lsl#1
orr r3,r3,r3,lsr#1
orr r0,r0,r0,lsl#1
orr r1,r1,r1,lsr#1
and r2,r2,r6 @ &=0x55555555
and r3,r3,r6,lsl#1 @ &=0xaaaaaaaa
and r0,r0,r6 @ &=0x55555555
and r1,r1,r6,lsl#1 @ &=0xaaaaaaaa
orr r2,r2,r3
orr r0,r0,r1
cmp $len,#8
blo .Lsqueeze_tail
lsr r1,r2,#8
strb r2,[$out],#1
lsr r3,r2,#16
strb r1,[$out],#1
lsr r2,r2,#24
strb r3,[$out],#1
strb r2,[$out],#1
lsr r1,r0,#8
strb r0,[$out],#1
lsr r3,r0,#16
strb r1,[$out],#1
lsr r0,r0,#24
strb r3,[$out],#1
strb r0,[$out],#1
subs $len,$len,#8
beq .Lsqueeze_done
subs $bsz,$bsz,#8 @ bsz -= 8
bhi .Loop_squeeze
mov r0,r14 @ original $A_flat
bl KeccakF1600
ldmia sp,{r6-r10,r12} @ restore constants and variables
mov r14,$A_flat
b .Loop_squeeze
.align 4
.Lsqueeze_tail:
strb r2,[$out],#1
lsr r2,r2,#8
subs $len,$len,#1
beq .Lsqueeze_done
strb r2,[$out],#1
lsr r2,r2,#8
subs $len,$len,#1
beq .Lsqueeze_done
strb r2,[$out],#1
lsr r2,r2,#8
subs $len,$len,#1
beq .Lsqueeze_done
strb r2,[$out],#1
subs $len,$len,#1
beq .Lsqueeze_done
strb r0,[$out],#1
lsr r0,r0,#8
subs $len,$len,#1
beq .Lsqueeze_done
strb r0,[$out],#1
lsr r0,r0,#8
subs $len,$len,#1
beq .Lsqueeze_done
strb r0,[$out]
b .Lsqueeze_done
.align 4
.Lsqueeze_done:
add sp,sp,#24
ldmia sp!,{r4-r10,pc}
.size SHA3_squeeze,.-SHA3_squeeze
___
}
$code.=<<___;
.fpu neon
.type iotas64, %object
.align 5
iotas64:
.quad 0x0000000000000001
.quad 0x0000000000008082
.quad 0x800000000000808a
.quad 0x8000000080008000
.quad 0x000000000000808b
.quad 0x0000000080000001
.quad 0x8000000080008081
.quad 0x8000000000008009
.quad 0x000000000000008a
.quad 0x0000000000000088
.quad 0x0000000080008009
.quad 0x000000008000000a
.quad 0x000000008000808b
.quad 0x800000000000008b
.quad 0x8000000000008089
.quad 0x8000000000008003
.quad 0x8000000000008002
.quad 0x8000000000000080
.quad 0x000000000000800a
.quad 0x800000008000000a
.quad 0x8000000080008081
.quad 0x8000000000008080
.quad 0x0000000080000001
.quad 0x8000000080008008
.size iotas64,.-iotas64
.type KeccakF1600_neon, %function
.align 5
KeccakF1600_neon:
add r1, r0, #16
adr r2, iotas64
mov r3, #24 @ loop counter
b .Loop_neon
.align 4
.Loop_neon:
@ Theta
vst1.64 {q4}, [r0:64] @ offload A[0..1][4]
veor q13, q0, q5 @ A[0..1][0]^A[2..3][0]
vst1.64 {d18}, [r1:64] @ offload A[2][4]
veor q14, q1, q6 @ A[0..1][1]^A[2..3][1]
veor q15, q2, q7 @ A[0..1][2]^A[2..3][2]
veor d26, d26, d27 @ C[0]=A[0][0]^A[1][0]^A[2][0]^A[3][0]
veor d27, d28, d29 @ C[1]=A[0][1]^A[1][1]^A[2][1]^A[3][1]
veor q14, q3, q8 @ A[0..1][3]^A[2..3][3]
veor q4, q4, q9 @ A[0..1][4]^A[2..3][4]
veor d30, d30, d31 @ C[2]=A[0][2]^A[1][2]^A[2][2]^A[3][2]
veor d31, d28, d29 @ C[3]=A[0][3]^A[1][3]^A[2][3]^A[3][3]
veor d25, d8, d9 @ C[4]=A[0][4]^A[1][4]^A[2][4]^A[3][4]
veor q13, q13, q10 @ C[0..1]^=A[4][0..1]
veor q14, q15, q11 @ C[2..3]^=A[4][2..3]
veor d25, d25, d24 @ C[4]^=A[4][4]
vadd.u64 q4, q13, q13 @ C[0..1]<<1
vadd.u64 q15, q14, q14 @ C[2..3]<<1
vadd.u64 d18, d25, d25 @ C[4]<<1
vsri.u64 q4, q13, #63 @ ROL64(C[0..1],1)
vsri.u64 q15, q14, #63 @ ROL64(C[2..3],1)
vsri.u64 d18, d25, #63 @ ROL64(C[4],1)
veor d25, d25, d9 @ D[0] = C[4] ^= ROL64(C[1],1)
veor q13, q13, q15 @ D[1..2] = C[0..1] ^ ROL64(C[2..3],1)
veor d28, d28, d18 @ D[3] = C[2] ^= ROL64(C[4],1)
veor d29, d29, d8 @ D[4] = C[3] ^= ROL64(C[0],1)
veor d0, d0, d25 @ A[0][0] ^= C[4]
veor d1, d1, d25 @ A[1][0] ^= C[4]
veor d10, d10, d25 @ A[2][0] ^= C[4]
veor d11, d11, d25 @ A[3][0] ^= C[4]
veor d20, d20, d25 @ A[4][0] ^= C[4]
veor d2, d2, d26 @ A[0][1] ^= D[1]
veor d3, d3, d26 @ A[1][1] ^= D[1]
veor d12, d12, d26 @ A[2][1] ^= D[1]
veor d13, d13, d26 @ A[3][1] ^= D[1]
veor d21, d21, d26 @ A[4][1] ^= D[1]
vmov d26, d27
veor d6, d6, d28 @ A[0][3] ^= C[2]
veor d7, d7, d28 @ A[1][3] ^= C[2]
veor d16, d16, d28 @ A[2][3] ^= C[2]
veor d17, d17, d28 @ A[3][3] ^= C[2]
veor d23, d23, d28 @ A[4][3] ^= C[2]
vld1.64 {q4}, [r0:64] @ restore A[0..1][4]
vmov d28, d29
vld1.64 {d18}, [r1:64] @ restore A[2][4]
veor q2, q2, q13 @ A[0..1][2] ^= D[2]
veor q7, q7, q13 @ A[2..3][2] ^= D[2]
veor d22, d22, d27 @ A[4][2] ^= D[2]
veor q4, q4, q14 @ A[0..1][4] ^= C[3]
veor q9, q9, q14 @ A[2..3][4] ^= C[3]
veor d24, d24, d29 @ A[4][4] ^= C[3]
@ Rho + Pi
vmov d26, d2 @ C[1] = A[0][1]
vshl.u64 d2, d3, #44
vmov d27, d4 @ C[2] = A[0][2]
vshl.u64 d4, d14, #43
vmov d28, d6 @ C[3] = A[0][3]
vshl.u64 d6, d17, #21
vmov d29, d8 @ C[4] = A[0][4]
vshl.u64 d8, d24, #14
vsri.u64 d2, d3, #64-44 @ A[0][1] = ROL64(A[1][1], rhotates[1][1])
vsri.u64 d4, d14, #64-43 @ A[0][2] = ROL64(A[2][2], rhotates[2][2])
vsri.u64 d6, d17, #64-21 @ A[0][3] = ROL64(A[3][3], rhotates[3][3])
vsri.u64 d8, d24, #64-14 @ A[0][4] = ROL64(A[4][4], rhotates[4][4])
vshl.u64 d3, d9, #20
vshl.u64 d14, d16, #25
vshl.u64 d17, d15, #15
vshl.u64 d24, d21, #2
vsri.u64 d3, d9, #64-20 @ A[1][1] = ROL64(A[1][4], rhotates[1][4])
vsri.u64 d14, d16, #64-25 @ A[2][2] = ROL64(A[2][3], rhotates[2][3])
vsri.u64 d17, d15, #64-15 @ A[3][3] = ROL64(A[3][2], rhotates[3][2])
vsri.u64 d24, d21, #64-2 @ A[4][4] = ROL64(A[4][1], rhotates[4][1])
vshl.u64 d9, d22, #61
@ vshl.u64 d16, d19, #8
vshl.u64 d15, d12, #10
vshl.u64 d21, d7, #55
vsri.u64 d9, d22, #64-61 @ A[1][4] = ROL64(A[4][2], rhotates[4][2])
vext.8 d16, d19, d19, #8-1 @ A[2][3] = ROL64(A[3][4], rhotates[3][4])
vsri.u64 d15, d12, #64-10 @ A[3][2] = ROL64(A[2][1], rhotates[2][1])
vsri.u64 d21, d7, #64-55 @ A[4][1] = ROL64(A[1][3], rhotates[1][3])
vshl.u64 d22, d18, #39
@ vshl.u64 d19, d23, #56
vshl.u64 d12, d5, #6
vshl.u64 d7, d13, #45
vsri.u64 d22, d18, #64-39 @ A[4][2] = ROL64(A[2][4], rhotates[2][4])
vext.8 d19, d23, d23, #8-7 @ A[3][4] = ROL64(A[4][3], rhotates[4][3])
vsri.u64 d12, d5, #64-6 @ A[2][1] = ROL64(A[1][2], rhotates[1][2])
vsri.u64 d7, d13, #64-45 @ A[1][3] = ROL64(A[3][1], rhotates[3][1])
vshl.u64 d18, d20, #18
vshl.u64 d23, d11, #41
vshl.u64 d5, d10, #3
vshl.u64 d13, d1, #36
vsri.u64 d18, d20, #64-18 @ A[2][4] = ROL64(A[4][0], rhotates[4][0])
vsri.u64 d23, d11, #64-41 @ A[4][3] = ROL64(A[3][0], rhotates[3][0])
vsri.u64 d5, d10, #64-3 @ A[1][2] = ROL64(A[2][0], rhotates[2][0])
vsri.u64 d13, d1, #64-36 @ A[3][1] = ROL64(A[1][0], rhotates[1][0])
vshl.u64 d1, d28, #28
vshl.u64 d10, d26, #1
vshl.u64 d11, d29, #27
vshl.u64 d20, d27, #62
vsri.u64 d1, d28, #64-28 @ A[1][0] = ROL64(C[3], rhotates[0][3])
vsri.u64 d10, d26, #64-1 @ A[2][0] = ROL64(C[1], rhotates[0][1])
vsri.u64 d11, d29, #64-27 @ A[3][0] = ROL64(C[4], rhotates[0][4])
vsri.u64 d20, d27, #64-62 @ A[4][0] = ROL64(C[2], rhotates[0][2])
@ Chi + Iota
vbic q13, q2, q1
vbic q14, q3, q2
vbic q15, q4, q3
veor q13, q13, q0 @ A[0..1][0] ^ (~A[0..1][1] & A[0..1][2])
veor q14, q14, q1 @ A[0..1][1] ^ (~A[0..1][2] & A[0..1][3])
veor q2, q2, q15 @ A[0..1][2] ^= (~A[0..1][3] & A[0..1][4])
vst1.64 {q13}, [r0:64] @ offload A[0..1][0]
vbic q13, q0, q4
vbic q15, q1, q0
vmov q1, q14 @ A[0..1][1]
veor q3, q3, q13 @ A[0..1][3] ^= (~A[0..1][4] & A[0..1][0])
veor q4, q4, q15 @ A[0..1][4] ^= (~A[0..1][0] & A[0..1][1])
vbic q13, q7, q6
vmov q0, q5 @ A[2..3][0]
vbic q14, q8, q7
vmov q15, q6 @ A[2..3][1]
veor q5, q5, q13 @ A[2..3][0] ^= (~A[2..3][1] & A[2..3][2])
vbic q13, q9, q8
veor q6, q6, q14 @ A[2..3][1] ^= (~A[2..3][2] & A[2..3][3])
vbic q14, q0, q9
veor q7, q7, q13 @ A[2..3][2] ^= (~A[2..3][3] & A[2..3][4])
vbic q13, q15, q0
veor q8, q8, q14 @ A[2..3][3] ^= (~A[2..3][4] & A[2..3][0])
vmov q14, q10 @ A[4][0..1]
veor q9, q9, q13 @ A[2..3][4] ^= (~A[2..3][0] & A[2..3][1])
vld1.64 d25, [r2:64]! @ Iota[i++]
vbic d26, d22, d21
vbic d27, d23, d22
vld1.64 {q0}, [r0:64] @ restore A[0..1][0]
veor d20, d20, d26 @ A[4][0] ^= (~A[4][1] & A[4][2])
vbic d26, d24, d23
veor d21, d21, d27 @ A[4][1] ^= (~A[4][2] & A[4][3])
vbic d27, d28, d24
veor d22, d22, d26 @ A[4][2] ^= (~A[4][3] & A[4][4])
vbic d26, d29, d28
veor d23, d23, d27 @ A[4][3] ^= (~A[4][4] & A[4][0])
veor d0, d0, d25 @ A[0][0] ^= Iota[i]
veor d24, d24, d26 @ A[4][4] ^= (~A[4][0] & A[4][1])
subs r3, r3, #1
bne .Loop_neon
bx lr
.size KeccakF1600_neon,.-KeccakF1600_neon
.global SHA3_absorb_neon
.type SHA3_absorb_neon, %function
.align 5
SHA3_absorb_neon:
stmdb sp!, {r4-r6,lr}
vstmdb sp!, {d8-d15}
mov r4, r1 @ inp
mov r5, r2 @ len
mov r6, r3 @ bsz
vld1.32 {d0}, [r0:64]! @ A[0][0]
vld1.32 {d2}, [r0:64]! @ A[0][1]
vld1.32 {d4}, [r0:64]! @ A[0][2]
vld1.32 {d6}, [r0:64]! @ A[0][3]
vld1.32 {d8}, [r0:64]! @ A[0][4]
vld1.32 {d1}, [r0:64]! @ A[1][0]
vld1.32 {d3}, [r0:64]! @ A[1][1]
vld1.32 {d5}, [r0:64]! @ A[1][2]
vld1.32 {d7}, [r0:64]! @ A[1][3]
vld1.32 {d9}, [r0:64]! @ A[1][4]
vld1.32 {d10}, [r0:64]! @ A[2][0]
vld1.32 {d12}, [r0:64]! @ A[2][1]
vld1.32 {d14}, [r0:64]! @ A[2][2]
vld1.32 {d16}, [r0:64]! @ A[2][3]
vld1.32 {d18}, [r0:64]! @ A[2][4]
vld1.32 {d11}, [r0:64]! @ A[3][0]
vld1.32 {d13}, [r0:64]! @ A[3][1]
vld1.32 {d15}, [r0:64]! @ A[3][2]
vld1.32 {d17}, [r0:64]! @ A[3][3]
vld1.32 {d19}, [r0:64]! @ A[3][4]
vld1.32 {d20-d23}, [r0:64]! @ A[4][0..3]
vld1.32 {d24}, [r0:64] @ A[4][4]
sub r0, r0, #24*8 @ rewind
b .Loop_absorb_neon
.align 4
.Loop_absorb_neon:
subs r12, r5, r6 @ len - bsz
blo .Labsorbed_neon
mov r5, r12
vld1.8 {d31}, [r4]! @ endian-neutral loads...
cmp r6, #8*2
veor d0, d0, d31 @ A[0][0] ^= *inp++
blo .Lprocess_neon
vld1.8 {d31}, [r4]!
veor d2, d2, d31 @ A[0][1] ^= *inp++
beq .Lprocess_neon
vld1.8 {d31}, [r4]!
cmp r6, #8*4
veor d4, d4, d31 @ A[0][2] ^= *inp++
blo .Lprocess_neon
vld1.8 {d31}, [r4]!
veor d6, d6, d31 @ A[0][3] ^= *inp++
beq .Lprocess_neon
vld1.8 {d31},[r4]!
cmp r6, #8*6
veor d8, d8, d31 @ A[0][4] ^= *inp++
blo .Lprocess_neon
vld1.8 {d31}, [r4]!
veor d1, d1, d31 @ A[1][0] ^= *inp++
beq .Lprocess_neon
vld1.8 {d31}, [r4]!
cmp r6, #8*8
veor d3, d3, d31 @ A[1][1] ^= *inp++
blo .Lprocess_neon
vld1.8 {d31}, [r4]!
veor d5, d5, d31 @ A[1][2] ^= *inp++
beq .Lprocess_neon
vld1.8 {d31}, [r4]!
cmp r6, #8*10
veor d7, d7, d31 @ A[1][3] ^= *inp++
blo .Lprocess_neon
vld1.8 {d31}, [r4]!
veor d9, d9, d31 @ A[1][4] ^= *inp++
beq .Lprocess_neon
vld1.8 {d31}, [r4]!
cmp r6, #8*12
veor d10, d10, d31 @ A[2][0] ^= *inp++
blo .Lprocess_neon
vld1.8 {d31}, [r4]!
veor d12, d12, d31 @ A[2][1] ^= *inp++
beq .Lprocess_neon
vld1.8 {d31}, [r4]!
cmp r6, #8*14
veor d14, d14, d31 @ A[2][2] ^= *inp++
blo .Lprocess_neon
vld1.8 {d31}, [r4]!
veor d16, d16, d31 @ A[2][3] ^= *inp++
beq .Lprocess_neon
vld1.8 {d31}, [r4]!
cmp r6, #8*16
veor d18, d18, d31 @ A[2][4] ^= *inp++
blo .Lprocess_neon
vld1.8 {d31}, [r4]!
veor d11, d11, d31 @ A[3][0] ^= *inp++
beq .Lprocess_neon
vld1.8 {d31}, [r4]!
cmp r6, #8*18
veor d13, d13, d31 @ A[3][1] ^= *inp++
blo .Lprocess_neon
vld1.8 {d31}, [r4]!
veor d15, d15, d31 @ A[3][2] ^= *inp++
beq .Lprocess_neon
vld1.8 {d31}, [r4]!
cmp r6, #8*20
veor d17, d17, d31 @ A[3][3] ^= *inp++
blo .Lprocess_neon
vld1.8 {d31}, [r4]!
veor d19, d19, d31 @ A[3][4] ^= *inp++
beq .Lprocess_neon
vld1.8 {d31}, [r4]!
cmp r6, #8*22
veor d20, d20, d31 @ A[4][0] ^= *inp++
blo .Lprocess_neon
vld1.8 {d31}, [r4]!
veor d21, d21, d31 @ A[4][1] ^= *inp++
beq .Lprocess_neon
vld1.8 {d31}, [r4]!
cmp r6, #8*24
veor d22, d22, d31 @ A[4][2] ^= *inp++
blo .Lprocess_neon
vld1.8 {d31}, [r4]!
veor d23, d23, d31 @ A[4][3] ^= *inp++
beq .Lprocess_neon
vld1.8 {d31}, [r4]!
veor d24, d24, d31 @ A[4][4] ^= *inp++
.Lprocess_neon:
bl KeccakF1600_neon
b .Loop_absorb_neon
.align 4
.Labsorbed_neon:
vst1.32 {d0}, [r0:64]! @ A[0][0..4]
vst1.32 {d2}, [r0:64]!
vst1.32 {d4}, [r0:64]!
vst1.32 {d6}, [r0:64]!
vst1.32 {d8}, [r0:64]!
vst1.32 {d1}, [r0:64]! @ A[1][0..4]
vst1.32 {d3}, [r0:64]!
vst1.32 {d5}, [r0:64]!
vst1.32 {d7}, [r0:64]!
vst1.32 {d9}, [r0:64]!
vst1.32 {d10}, [r0:64]! @ A[2][0..4]
vst1.32 {d12}, [r0:64]!
vst1.32 {d14}, [r0:64]!
vst1.32 {d16}, [r0:64]!
vst1.32 {d18}, [r0:64]!
vst1.32 {d11}, [r0:64]! @ A[3][0..4]
vst1.32 {d13}, [r0:64]!
vst1.32 {d15}, [r0:64]!
vst1.32 {d17}, [r0:64]!
vst1.32 {d19}, [r0:64]!
vst1.32 {d20-d23}, [r0:64]! @ A[4][0..4]
vst1.32 {d24}, [r0:64]
mov r0, r5 @ return value
vldmia sp!, {d8-d15}
ldmia sp!, {r4-r6,pc}
.size SHA3_absorb_neon,.-SHA3_absorb_neon
.global SHA3_squeeze_neon
.type SHA3_squeeze_neon, %function
.align 5
SHA3_squeeze_neon:
stmdb sp!, {r4-r6,lr}
mov r4, r1 @ out
mov r5, r2 @ len
mov r6, r3 @ bsz
mov r12, r0 @ A_flat
mov r14, r3 @ bsz
b .Loop_squeeze_neon
.align 4
.Loop_squeeze_neon:
cmp r5, #8
blo .Lsqueeze_neon_tail
vld1.32 {d0}, [r12]!
vst1.8 {d0}, [r4]! @ endian-neutral store
subs r5, r5, #8 @ len -= 8
beq .Lsqueeze_neon_done
subs r14, r14, #8 @ bsz -= 8
bhi .Loop_squeeze_neon
vstmdb sp!, {d8-d15}
vld1.32 {d0}, [r0:64]! @ A[0][0..4]
vld1.32 {d2}, [r0:64]!
vld1.32 {d4}, [r0:64]!
vld1.32 {d6}, [r0:64]!
vld1.32 {d8}, [r0:64]!
vld1.32 {d1}, [r0:64]! @ A[1][0..4]
vld1.32 {d3}, [r0:64]!
vld1.32 {d5}, [r0:64]!
vld1.32 {d7}, [r0:64]!
vld1.32 {d9}, [r0:64]!
vld1.32 {d10}, [r0:64]! @ A[2][0..4]
vld1.32 {d12}, [r0:64]!
vld1.32 {d14}, [r0:64]!
vld1.32 {d16}, [r0:64]!
vld1.32 {d18}, [r0:64]!
vld1.32 {d11}, [r0:64]! @ A[3][0..4]
vld1.32 {d13}, [r0:64]!
vld1.32 {d15}, [r0:64]!
vld1.32 {d17}, [r0:64]!
vld1.32 {d19}, [r0:64]!
vld1.32 {d20-d23}, [r0:64]! @ A[4][0..4]
vld1.32 {d24}, [r0:64]
sub r0, r0, #24*8 @ rewind
bl KeccakF1600_neon
mov r12, r0 @ A_flat
vst1.32 {d0}, [r0:64]! @ A[0][0..4]
vst1.32 {d2}, [r0:64]!
vst1.32 {d4}, [r0:64]!
vst1.32 {d6}, [r0:64]!
vst1.32 {d8}, [r0:64]!
vst1.32 {d1}, [r0:64]! @ A[1][0..4]
vst1.32 {d3}, [r0:64]!
vst1.32 {d5}, [r0:64]!
vst1.32 {d7}, [r0:64]!
vst1.32 {d9}, [r0:64]!
vst1.32 {d10}, [r0:64]! @ A[2][0..4]
vst1.32 {d12}, [r0:64]!
vst1.32 {d14}, [r0:64]!
vst1.32 {d16}, [r0:64]!
vst1.32 {d18}, [r0:64]!
vst1.32 {d11}, [r0:64]! @ A[3][0..4]
vst1.32 {d13}, [r0:64]!
vst1.32 {d15}, [r0:64]!
vst1.32 {d17}, [r0:64]!
vst1.32 {d19}, [r0:64]!
vst1.32 {d20-d23}, [r0:64]! @ A[4][0..4]
mov r14, r6 @ bsz
vst1.32 {d24}, [r0:64]
mov r0, r12 @ rewind
vldmia sp!, {d8-d15}
b .Loop_squeeze_neon
.align 4
.Lsqueeze_neon_tail:
ldmia r12, {r2,r3}
cmp r5, #2
strb r2, [r4],#1 @ endian-neutral store
lsr r2, r2, #8
blo .Lsqueeze_neon_done
strb r2, [r4], #1
lsr r2, r2, #8
beq .Lsqueeze_neon_done
strb r2, [r4], #1
lsr r2, r2, #8
cmp r5, #4
blo .Lsqueeze_neon_done
strb r2, [r4], #1
beq .Lsqueeze_neon_done
strb r3, [r4], #1
lsr r3, r3, #8
cmp r5, #6
blo .Lsqueeze_neon_done
strb r3, [r4], #1
lsr r3, r3, #8
beq .Lsqueeze_neon_done
strb r3, [r4], #1
.Lsqueeze_neon_done:
ldmia sp!, {r4-r6,pc}
.size SHA3_squeeze_neon,.-SHA3_squeeze_neon
.asciz "Keccak-1600 absorb and squeeze for ARMv4/NEON, CRYPTOGAMS by <appro\@openssl.org>"
.align 2
___
{
my %ldr, %str;
sub ldrd {
my ($mnemonic,$half,$reg,$ea) = @_;
my $op = $mnemonic eq "ldr" ? \%ldr : \%str;
if ($half eq "l") {
$$op{reg} = $reg;
$$op{ea} = $ea;
sprintf "#ifndef __thumb2__\n" .
" %s\t%s,%s\n" .
"#endif", $mnemonic,$reg,$ea;
} else {
sprintf "#ifndef __thumb2__\n" .
" %s\t%s,%s\n" .
"#else\n" .
" %sd\t%s,%s,%s\n" .
"#endif", $mnemonic,$reg,$ea,
$mnemonic,$$op{reg},$reg,$$op{ea};
}
}
}
foreach (split($/,$code)) {
s/\`([^\`]*)\`/eval $1/ge;
s/^\s+(ldr|str)\.([lh])\s+(r[0-9]+),\s*(\[.*)/ldrd($1,$2,$3,$4)/ge or
s/\bret\b/bx lr/g or
s/\bbx\s+lr\b/.word\t0xe12fff1e/g; # make it possible to compile with -march=armv4
print $_,"\n";
}
close STDOUT; # enforce flush
crypto/sha/asm/keccak1600-armv8.pl
Executable
+866
View File
@@ -0,0 +1,866 @@
#!/usr/bin/env perl
# Copyright 2017-2018 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License"). You may not use
# this file except in compliance with the License. You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html
#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# Keccak-1600 for ARMv8.
#
# June 2017.
#
# This is straightforward KECCAK_1X_ALT implementation. It makes no
# sense to attempt SIMD/NEON implementation for following reason.
# 64-bit lanes of vector registers can't be addressed as easily as in
# 32-bit mode. This means that 64-bit NEON is bound to be slower than
# 32-bit NEON, and this implementation is faster than 32-bit NEON on
# same processor. Even though it takes more scalar xor's and andn's,
# it gets compensated by availability of rotate. Not to forget that
# most processors achieve higher issue rate with scalar instructions.
#
# February 2018.
#
# Add hardware-assisted ARMv8.2 implementation. It's KECCAK_1X_ALT
# variant with register permutation/rotation twist that allows to
# eliminate copies to temporary registers. If you look closely you'll
# notice that it uses only one lane of vector registers. The new
# instructions effectively facilitate parallel hashing, which we don't
# support [yet?]. But lowest-level core procedure is prepared for it.
# The inner round is 67 [vector] instructions, so it's not actually
# obvious that it will provide performance improvement [in serial
# hash] as long as vector instructions issue rate is limited to 1 per
# cycle...
#
######################################################################
# Numbers are cycles per processed byte.
#
# r=1088(*)
#
# Cortex-A53 13
# Cortex-A57 12
# X-Gene 14
# Mongoose 10
# Kryo 12
# Denver 7.8
# Apple A7 7.2
#
# (*) Corresponds to SHA3-256. No improvement coefficients are listed
# because they vary too much from compiler to compiler. Newer
# compiler does much better and improvement varies from 5% on
# Cortex-A57 to 25% on Cortex-A53. While in comparison to older
# compiler this code is at least 2x faster...
$flavour = shift;
$output = shift;
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
die "can't locate arm-xlate.pl";
open OUT,"| \"$^X\" $xlate $flavour $output";
*STDOUT=*OUT;
my @rhotates = ([ 0, 1, 62, 28, 27 ],
[ 36, 44, 6, 55, 20 ],
[ 3, 10, 43, 25, 39 ],
[ 41, 45, 15, 21, 8 ],
[ 18, 2, 61, 56, 14 ]);
$code.=<<___;
.text
.align 8 // strategic alignment and padding that allows to use
// address value as loop termination condition...
.quad 0,0,0,0,0,0,0,0
.type iotas,%object
iotas:
.quad 0x0000000000000001
.quad 0x0000000000008082
.quad 0x800000000000808a
.quad 0x8000000080008000
.quad 0x000000000000808b
.quad 0x0000000080000001
.quad 0x8000000080008081
.quad 0x8000000000008009
.quad 0x000000000000008a
.quad 0x0000000000000088
.quad 0x0000000080008009
.quad 0x000000008000000a
.quad 0x000000008000808b
.quad 0x800000000000008b
.quad 0x8000000000008089
.quad 0x8000000000008003
.quad 0x8000000000008002
.quad 0x8000000000000080
.quad 0x000000000000800a
.quad 0x800000008000000a
.quad 0x8000000080008081
.quad 0x8000000000008080
.quad 0x0000000080000001
.quad 0x8000000080008008
.size iotas,.-iotas
___
{{{
my @A = map([ "x$_", "x".($_+1), "x".($_+2), "x".($_+3), "x".($_+4) ],
(0, 5, 10, 15, 20));
$A[3][3] = "x25"; # x18 is reserved
my @C = map("x$_", (26,27,28,30));
$code.=<<___;
.type KeccakF1600_int,%function
.align 5
KeccakF1600_int:
adr $C[2],iotas
stp $C[2],x30,[sp,#16] // 32 bytes on top are mine
b .Loop
.align 4
.Loop:
////////////////////////////////////////// Theta
eor $C[0],$A[0][0],$A[1][0]
stp $A[0][4],$A[1][4],[sp,#0] // offload pair...
eor $C[1],$A[0][1],$A[1][1]
eor $C[2],$A[0][2],$A[1][2]
eor $C[3],$A[0][3],$A[1][3]
___
$C[4]=$A[0][4];
$C[5]=$A[1][4];
$code.=<<___;
eor $C[4],$A[0][4],$A[1][4]
eor $C[0],$C[0],$A[2][0]
eor $C[1],$C[1],$A[2][1]
eor $C[2],$C[2],$A[2][2]
eor $C[3],$C[3],$A[2][3]
eor $C[4],$C[4],$A[2][4]
eor $C[0],$C[0],$A[3][0]
eor $C[1],$C[1],$A[3][1]
eor $C[2],$C[2],$A[3][2]
eor $C[3],$C[3],$A[3][3]
eor $C[4],$C[4],$A[3][4]
eor $C[0],$C[0],$A[4][0]
eor $C[2],$C[2],$A[4][2]
eor $C[1],$C[1],$A[4][1]
eor $C[3],$C[3],$A[4][3]
eor $C[4],$C[4],$A[4][4]
eor $C[5],$C[0],$C[2],ror#63
eor $A[0][1],$A[0][1],$C[5]
eor $A[1][1],$A[1][1],$C[5]
eor $A[2][1],$A[2][1],$C[5]
eor $A[3][1],$A[3][1],$C[5]
eor $A[4][1],$A[4][1],$C[5]
eor $C[5],$C[1],$C[3],ror#63
eor $C[2],$C[2],$C[4],ror#63
eor $C[3],$C[3],$C[0],ror#63
eor $C[4],$C[4],$C[1],ror#63
eor $C[1], $A[0][2],$C[5] // mov $C[1],$A[0][2]
eor $A[1][2],$A[1][2],$C[5]
eor $A[2][2],$A[2][2],$C[5]
eor $A[3][2],$A[3][2],$C[5]
eor $A[4][2],$A[4][2],$C[5]
eor $A[0][0],$A[0][0],$C[4]
eor $A[1][0],$A[1][0],$C[4]
eor $A[2][0],$A[2][0],$C[4]
eor $A[3][0],$A[3][0],$C[4]
eor $A[4][0],$A[4][0],$C[4]
___
$C[4]=undef;
$C[5]=undef;
$code.=<<___;
ldp $A[0][4],$A[1][4],[sp,#0] // re-load offloaded data
eor $C[0], $A[0][3],$C[2] // mov $C[0],$A[0][3]
eor $A[1][3],$A[1][3],$C[2]
eor $A[2][3],$A[2][3],$C[2]
eor $A[3][3],$A[3][3],$C[2]
eor $A[4][3],$A[4][3],$C[2]
eor $C[2], $A[0][4],$C[3] // mov $C[2],$A[0][4]
eor $A[1][4],$A[1][4],$C[3]
eor $A[2][4],$A[2][4],$C[3]
eor $A[3][4],$A[3][4],$C[3]
eor $A[4][4],$A[4][4],$C[3]
////////////////////////////////////////// Rho+Pi
mov $C[3],$A[0][1]
ror $A[0][1],$A[1][1],#64-$rhotates[1][1]
//mov $C[1],$A[0][2]
ror $A[0][2],$A[2][2],#64-$rhotates[2][2]
//mov $C[0],$A[0][3]
ror $A[0][3],$A[3][3],#64-$rhotates[3][3]
//mov $C[2],$A[0][4]
ror $A[0][4],$A[4][4],#64-$rhotates[4][4]
ror $A[1][1],$A[1][4],#64-$rhotates[1][4]
ror $A[2][2],$A[2][3],#64-$rhotates[2][3]
ror $A[3][3],$A[3][2],#64-$rhotates[3][2]
ror $A[4][4],$A[4][1],#64-$rhotates[4][1]
ror $A[1][4],$A[4][2],#64-$rhotates[4][2]
ror $A[2][3],$A[3][4],#64-$rhotates[3][4]
ror $A[3][2],$A[2][1],#64-$rhotates[2][1]
ror $A[4][1],$A[1][3],#64-$rhotates[1][3]
ror $A[4][2],$A[2][4],#64-$rhotates[2][4]
ror $A[3][4],$A[4][3],#64-$rhotates[4][3]
ror $A[2][1],$A[1][2],#64-$rhotates[1][2]
ror $A[1][3],$A[3][1],#64-$rhotates[3][1]
ror $A[2][4],$A[4][0],#64-$rhotates[4][0]
ror $A[4][3],$A[3][0],#64-$rhotates[3][0]
ror $A[1][2],$A[2][0],#64-$rhotates[2][0]
ror $A[3][1],$A[1][0],#64-$rhotates[1][0]
ror $A[1][0],$C[0],#64-$rhotates[0][3]
ror $A[2][0],$C[3],#64-$rhotates[0][1]
ror $A[3][0],$C[2],#64-$rhotates[0][4]
ror $A[4][0],$C[1],#64-$rhotates[0][2]
////////////////////////////////////////// Chi+Iota
bic $C[0],$A[0][2],$A[0][1]
bic $C[1],$A[0][3],$A[0][2]
bic $C[2],$A[0][0],$A[0][4]
bic $C[3],$A[0][1],$A[0][0]
eor $A[0][0],$A[0][0],$C[0]
bic $C[0],$A[0][4],$A[0][3]
eor $A[0][1],$A[0][1],$C[1]
ldr $C[1],[sp,#16]
eor $A[0][3],$A[0][3],$C[2]
eor $A[0][4],$A[0][4],$C[3]
eor $A[0][2],$A[0][2],$C[0]
ldr $C[3],[$C[1]],#8 // Iota[i++]
bic $C[0],$A[1][2],$A[1][1]
tst $C[1],#255 // are we done?
str $C[1],[sp,#16]
bic $C[1],$A[1][3],$A[1][2]
bic $C[2],$A[1][0],$A[1][4]
eor $A[0][0],$A[0][0],$C[3] // A[0][0] ^= Iota
bic $C[3],$A[1][1],$A[1][0]
eor $A[1][0],$A[1][0],$C[0]
bic $C[0],$A[1][4],$A[1][3]
eor $A[1][1],$A[1][1],$C[1]
eor $A[1][3],$A[1][3],$C[2]
eor $A[1][4],$A[1][4],$C[3]
eor $A[1][2],$A[1][2],$C[0]
bic $C[0],$A[2][2],$A[2][1]
bic $C[1],$A[2][3],$A[2][2]
bic $C[2],$A[2][0],$A[2][4]
bic $C[3],$A[2][1],$A[2][0]
eor $A[2][0],$A[2][0],$C[0]
bic $C[0],$A[2][4],$A[2][3]
eor $A[2][1],$A[2][1],$C[1]
eor $A[2][3],$A[2][3],$C[2]
eor $A[2][4],$A[2][4],$C[3]
eor $A[2][2],$A[2][2],$C[0]
bic $C[0],$A[3][2],$A[3][1]
bic $C[1],$A[3][3],$A[3][2]
bic $C[2],$A[3][0],$A[3][4]
bic $C[3],$A[3][1],$A[3][0]
eor $A[3][0],$A[3][0],$C[0]
bic $C[0],$A[3][4],$A[3][3]
eor $A[3][1],$A[3][1],$C[1]
eor $A[3][3],$A[3][3],$C[2]
eor $A[3][4],$A[3][4],$C[3]
eor $A[3][2],$A[3][2],$C[0]
bic $C[0],$A[4][2],$A[4][1]
bic $C[1],$A[4][3],$A[4][2]
bic $C[2],$A[4][0],$A[4][4]
bic $C[3],$A[4][1],$A[4][0]
eor $A[4][0],$A[4][0],$C[0]
bic $C[0],$A[4][4],$A[4][3]
eor $A[4][1],$A[4][1],$C[1]
eor $A[4][3],$A[4][3],$C[2]
eor $A[4][4],$A[4][4],$C[3]
eor $A[4][2],$A[4][2],$C[0]
bne .Loop
ldr x30,[sp,#24]
ret
.size KeccakF1600_int,.-KeccakF1600_int
.type KeccakF1600,%function
.align 5
KeccakF1600:
stp x29,x30,[sp,#-128]!
add x29,sp,#0
stp x19,x20,[sp,#16]
stp x21,x22,[sp,#32]
stp x23,x24,[sp,#48]
stp x25,x26,[sp,#64]
stp x27,x28,[sp,#80]
sub sp,sp,#48
str x0,[sp,#32] // offload argument
mov $C[0],x0
ldp $A[0][0],$A[0][1],[x0,#16*0]
ldp $A[0][2],$A[0][3],[$C[0],#16*1]
ldp $A[0][4],$A[1][0],[$C[0],#16*2]
ldp $A[1][1],$A[1][2],[$C[0],#16*3]
ldp $A[1][3],$A[1][4],[$C[0],#16*4]
ldp $A[2][0],$A[2][1],[$C[0],#16*5]
ldp $A[2][2],$A[2][3],[$C[0],#16*6]
ldp $A[2][4],$A[3][0],[$C[0],#16*7]
ldp $A[3][1],$A[3][2],[$C[0],#16*8]
ldp $A[3][3],$A[3][4],[$C[0],#16*9]
ldp $A[4][0],$A[4][1],[$C[0],#16*10]
ldp $A[4][2],$A[4][3],[$C[0],#16*11]
ldr $A[4][4],[$C[0],#16*12]
bl KeccakF1600_int
ldr $C[0],[sp,#32]
stp $A[0][0],$A[0][1],[$C[0],#16*0]
stp $A[0][2],$A[0][3],[$C[0],#16*1]
stp $A[0][4],$A[1][0],[$C[0],#16*2]
stp $A[1][1],$A[1][2],[$C[0],#16*3]
stp $A[1][3],$A[1][4],[$C[0],#16*4]
stp $A[2][0],$A[2][1],[$C[0],#16*5]
stp $A[2][2],$A[2][3],[$C[0],#16*6]
stp $A[2][4],$A[3][0],[$C[0],#16*7]
stp $A[3][1],$A[3][2],[$C[0],#16*8]
stp $A[3][3],$A[3][4],[$C[0],#16*9]
stp $A[4][0],$A[4][1],[$C[0],#16*10]
stp $A[4][2],$A[4][3],[$C[0],#16*11]
str $A[4][4],[$C[0],#16*12]
ldp x19,x20,[x29,#16]
add sp,sp,#48
ldp x21,x22,[x29,#32]
ldp x23,x24,[x29,#48]
ldp x25,x26,[x29,#64]
ldp x27,x28,[x29,#80]
ldp x29,x30,[sp],#128
ret
.size KeccakF1600,.-KeccakF1600
.globl SHA3_absorb
.type SHA3_absorb,%function
.align 5
SHA3_absorb:
stp x29,x30,[sp,#-128]!
add x29,sp,#0
stp x19,x20,[sp,#16]
stp x21,x22,[sp,#32]
stp x23,x24,[sp,#48]
stp x25,x26,[sp,#64]
stp x27,x28,[sp,#80]
sub sp,sp,#64
stp x0,x1,[sp,#32] // offload arguments
stp x2,x3,[sp,#48]
mov $C[0],x0 // uint64_t A[5][5]
mov $C[1],x1 // const void *inp
mov $C[2],x2 // size_t len
mov $C[3],x3 // size_t bsz
ldp $A[0][0],$A[0][1],[$C[0],#16*0]
ldp $A[0][2],$A[0][3],[$C[0],#16*1]
ldp $A[0][4],$A[1][0],[$C[0],#16*2]
ldp $A[1][1],$A[1][2],[$C[0],#16*3]
ldp $A[1][3],$A[1][4],[$C[0],#16*4]
ldp $A[2][0],$A[2][1],[$C[0],#16*5]
ldp $A[2][2],$A[2][3],[$C[0],#16*6]
ldp $A[2][4],$A[3][0],[$C[0],#16*7]
ldp $A[3][1],$A[3][2],[$C[0],#16*8]
ldp $A[3][3],$A[3][4],[$C[0],#16*9]
ldp $A[4][0],$A[4][1],[$C[0],#16*10]
ldp $A[4][2],$A[4][3],[$C[0],#16*11]
ldr $A[4][4],[$C[0],#16*12]
b .Loop_absorb
.align 4
.Loop_absorb:
subs $C[0],$C[2],$C[3] // len - bsz
blo .Labsorbed
str $C[0],[sp,#48] // save len - bsz
___
for (my $i=0; $i<24; $i+=2) {
my $j = $i+1;
$code.=<<___;
ldr $C[0],[$C[1]],#8 // *inp++
#ifdef __AARCH64EB__
rev $C[0],$C[0]
#endif
eor $A[$i/5][$i%5],$A[$i/5][$i%5],$C[0]
cmp $C[3],#8*($i+2)
blo .Lprocess_block
ldr $C[0],[$C[1]],#8 // *inp++
#ifdef __AARCH64EB__
rev $C[0],$C[0]
#endif
eor $A[$j/5][$j%5],$A[$j/5][$j%5],$C[0]
beq .Lprocess_block
___
}
$code.=<<___;
ldr $C[0],[$C[1]],#8 // *inp++
#ifdef __AARCH64EB__
rev $C[0],$C[0]
#endif
eor $A[4][4],$A[4][4],$C[0]
.Lprocess_block:
str $C[1],[sp,#40] // save inp
bl KeccakF1600_int
ldr $C[1],[sp,#40] // restore arguments
ldp $C[2],$C[3],[sp,#48]
b .Loop_absorb
.align 4
.Labsorbed:
ldr $C[1],[sp,#32]
stp $A[0][0],$A[0][1],[$C[1],#16*0]
stp $A[0][2],$A[0][3],[$C[1],#16*1]
stp $A[0][4],$A[1][0],[$C[1],#16*2]
stp $A[1][1],$A[1][2],[$C[1],#16*3]
stp $A[1][3],$A[1][4],[$C[1],#16*4]
stp $A[2][0],$A[2][1],[$C[1],#16*5]
stp $A[2][2],$A[2][3],[$C[1],#16*6]
stp $A[2][4],$A[3][0],[$C[1],#16*7]
stp $A[3][1],$A[3][2],[$C[1],#16*8]
stp $A[3][3],$A[3][4],[$C[1],#16*9]
stp $A[4][0],$A[4][1],[$C[1],#16*10]
stp $A[4][2],$A[4][3],[$C[1],#16*11]
str $A[4][4],[$C[1],#16*12]
mov x0,$C[2] // return value
ldp x19,x20,[x29,#16]
add sp,sp,#64
ldp x21,x22,[x29,#32]
ldp x23,x24,[x29,#48]
ldp x25,x26,[x29,#64]
ldp x27,x28,[x29,#80]
ldp x29,x30,[sp],#128
ret
.size SHA3_absorb,.-SHA3_absorb
___
{
my ($A_flat,$out,$len,$bsz) = map("x$_",(19..22));
$code.=<<___;
.globl SHA3_squeeze
.type SHA3_squeeze,%function
.align 5
SHA3_squeeze:
stp x29,x30,[sp,#-48]!
add x29,sp,#0
stp x19,x20,[sp,#16]
stp x21,x22,[sp,#32]
mov $A_flat,x0 // put aside arguments
mov $out,x1
mov $len,x2
mov $bsz,x3
.Loop_squeeze:
ldr x4,[x0],#8
cmp $len,#8
blo .Lsqueeze_tail
#ifdef __AARCH64EB__
rev x4,x4
#endif
str x4,[$out],#8
subs $len,$len,#8
beq .Lsqueeze_done
subs x3,x3,#8
bhi .Loop_squeeze
mov x0,$A_flat
bl KeccakF1600
mov x0,$A_flat
mov x3,$bsz
b .Loop_squeeze
.align 4
.Lsqueeze_tail:
strb w4,[$out],#1
lsr x4,x4,#8
subs $len,$len,#1
beq .Lsqueeze_done
strb w4,[$out],#1
lsr x4,x4,#8
subs $len,$len,#1
beq .Lsqueeze_done
strb w4,[$out],#1
lsr x4,x4,#8
subs $len,$len,#1
beq .Lsqueeze_done
strb w4,[$out],#1
lsr x4,x4,#8
subs $len,$len,#1
beq .Lsqueeze_done
strb w4,[$out],#1
lsr x4,x4,#8
subs $len,$len,#1
beq .Lsqueeze_done
strb w4,[$out],#1
lsr x4,x4,#8
subs $len,$len,#1
beq .Lsqueeze_done
strb w4,[$out],#1
.Lsqueeze_done:
ldp x19,x20,[sp,#16]
ldp x21,x22,[sp,#32]
ldp x29,x30,[sp],#48
ret
.size SHA3_squeeze,.-SHA3_squeeze
___
} }}}
{{{
my @A = map([ "v".$_.".16b", "v".($_+1).".16b", "v".($_+2).".16b",
"v".($_+3).".16b", "v".($_+4).".16b" ],
(0, 5, 10, 15, 20));
my @C = map("v$_.16b", (25..31));
$code.=<<___;
.type KeccakF1600_ce,%function
.align 5
KeccakF1600_ce:
mov x9,#6
adr x10,iotas
b .Loop_ce
.align 4
.Loop_ce:
___
for($i=0; $i<4; $i++) {
$code.=<<___;
////////////////////////////////////////////////// Theta
eor3 $C[0],$A[0][0],$A[1][0],$A[2][0]
eor3 $C[1],$A[0][1],$A[1][1],$A[2][1]
eor3 $C[2],$A[0][2],$A[1][2],$A[2][2]
eor3 $C[3],$A[0][3],$A[1][3],$A[2][3]
eor3 $C[4],$A[0][4],$A[1][4],$A[2][4]
eor3 $C[0],$C[0], $A[3][0],$A[4][0]
eor3 $C[1],$C[1], $A[3][1],$A[4][1]
eor3 $C[2],$C[2], $A[3][2],$A[4][2]
eor3 $C[3],$C[3], $A[3][3],$A[4][3]
eor3 $C[4],$C[4], $A[3][4],$A[4][4]
rax1 $C[5],$C[0],$C[2] // D[1]
rax1 $C[6],$C[1],$C[3] // D[2]
rax1 $C[2],$C[2],$C[4] // D[3]
rax1 $C[3],$C[3],$C[0] // D[4]
rax1 $C[4],$C[4],$C[1] // D[0]
////////////////////////////////////////////////// Theta+Rho+Pi
xar $C[0], $A[1][1],$C[5],#64-$rhotates[1][1] // C[0]=A[0][1]
xar $A[1][1],$A[1][4],$C[3],#64-$rhotates[1][4]
xar $A[1][4],$A[4][2],$C[6],#64-$rhotates[4][2]
xar $A[4][2],$A[2][4],$C[3],#64-$rhotates[2][4]
xar $A[2][4],$A[4][0],$C[4],#64-$rhotates[4][0]
xar $A[4][0],$A[0][2],$C[6],#64-$rhotates[0][2]
xar $A[0][2],$A[2][2],$C[6],#64-$rhotates[2][2]
xar $A[2][2],$A[2][3],$C[2],#64-$rhotates[2][3]
xar $A[2][3],$A[3][4],$C[3],#64-$rhotates[3][4]
xar $A[3][4],$A[4][3],$C[2],#64-$rhotates[4][3]
xar $A[4][3],$A[3][0],$C[4],#64-$rhotates[3][0]
xar $A[3][0],$A[0][4],$C[3],#64-$rhotates[0][4]
eor $A[0][0],$A[0][0],$C[4]
ldr x11,[x10],#8
xar $C[1] ,$A[3][3],$C[2],#64-$rhotates[3][3] // C[1]=A[0][3]
xar $A[3][3],$A[3][2],$C[6],#64-$rhotates[3][2]
xar $A[3][2],$A[2][1],$C[5],#64-$rhotates[2][1]
xar $A[2][1],$A[1][2],$C[6],#64-$rhotates[1][2]
xar $A[1][2],$A[2][0],$C[4],#64-$rhotates[2][0]
xar $A[2][0],$A[0][1],$C[5],#64-$rhotates[0][1] // *
xar $A[0][4],$A[4][4],$C[3],#64-$rhotates[4][4]
xar $A[4][4],$A[4][1],$C[5],#64-$rhotates[4][1]
xar $A[4][1],$A[1][3],$C[2],#64-$rhotates[1][3]
xar $A[1][3],$A[3][1],$C[5],#64-$rhotates[3][1]
xar $A[3][1],$A[1][0],$C[4],#64-$rhotates[1][0]
xar $A[1][0],$A[0][3],$C[2],#64-$rhotates[0][3] // *
////////////////////////////////////////////////// Chi+Iota
dup $C[6],x11 // borrow C[6]
bcax $C[3], $A[0][0],$A[0][2],$C[0] // *
bcax $A[0][1],$C[0], $C[1], $A[0][2] // *
bcax $A[0][2],$A[0][2],$A[0][4],$C[1]
bcax $A[0][3],$C[1], $A[0][0],$A[0][4]
bcax $A[0][4],$A[0][4],$C[0], $A[0][0]
bcax $C[0], $A[1][0],$A[1][2],$A[1][1] // *
bcax $C[1], $A[1][1],$A[1][3],$A[1][2] // *
bcax $A[1][2],$A[1][2],$A[1][4],$A[1][3]
bcax $A[1][3],$A[1][3],$A[1][0],$A[1][4]
bcax $A[1][4],$A[1][4],$A[1][1],$A[1][0]
eor $A[0][0],$C[3],$C[6] // Iota
bcax $C[2], $A[2][0],$A[2][2],$A[2][1] // *
bcax $C[3], $A[2][1],$A[2][3],$A[2][2] // *
bcax $A[2][2],$A[2][2],$A[2][4],$A[2][3]
bcax $A[2][3],$A[2][3],$A[2][0],$A[2][4]
bcax $A[2][4],$A[2][4],$A[2][1],$A[2][0]
bcax $A[2][0],$A[3][0],$A[3][2],$A[3][1] // *
bcax $A[2][1],$A[3][1],$A[3][3],$A[3][2] // *
bcax $A[3][2],$A[3][2],$A[3][4],$A[3][3]
bcax $A[3][3],$A[3][3],$A[3][0],$A[3][4]
bcax $A[3][4],$A[3][4],$A[3][1],$A[3][0]
bcax $A[3][0],$A[4][0],$A[4][2],$A[4][1] // *
bcax $A[3][1],$A[4][1],$A[4][3],$A[4][2] // *
bcax $A[4][2],$A[4][2],$A[4][4],$A[4][3]
bcax $A[4][3],$A[4][3],$A[4][0],$A[4][4]
bcax $A[4][4],$A[4][4],$A[4][1],$A[4][0]
___
($A[1][0],$A[1][1], $C[0],$C[1])
= ($C[0],$C[1], $A[1][0],$A[1][1]);
($A[2][0],$A[2][1], $A[3][0],$A[3][1], $A[4][0],$A[4][1], $C[2],$C[3])
= ($C[2],$C[3], $A[2][0],$A[2][1], $A[3][0],$A[3][1], $A[4][0],$A[4][1]);
}
$code.=<<___;
subs x9,x9,#1
bne .Loop_ce
ret
.size KeccakF1600_ce,.-KeccakF1600_ce
.type KeccakF1600_cext,%function
.align 5
KeccakF1600_cext:
stp x29,x30,[sp,#-80]!
add x29,sp,#0
stp d8,d9,[sp,#16] // per ABI requirement
stp d10,d11,[sp,#32]
stp d12,d13,[sp,#48]
stp d14,d15,[sp,#64]
___
for($i=0; $i<24; $i+=2) { # load A[5][5]
my $j=$i+1;
$code.=<<___;
ldp d$i,d$j,[x0,#8*$i]
___
}
$code.=<<___;
ldr d24,[x0,#8*$i]
bl KeccakF1600_ce
ldr x30,[sp,#8]
___
for($i=0; $i<24; $i+=2) { # store A[5][5]
my $j=$i+1;
$code.=<<___;
stp d$i,d$j,[x0,#8*$i]
___
}
$code.=<<___;
str d24,[x0,#8*$i]
ldp d8,d9,[sp,#16]
ldp d10,d11,[sp,#32]
ldp d12,d13,[sp,#48]
ldp d14,d15,[sp,#64]
ldr x29,[sp],#80
ret
.size KeccakF1600_cext,.-KeccakF1600_cext
___
{
my ($ctx,$inp,$len,$bsz) = map("x$_",(0..3));
$code.=<<___;
.globl SHA3_absorb_cext
.type SHA3_absorb_cext,%function
.align 5
SHA3_absorb_cext:
stp x29,x30,[sp,#-80]!
add x29,sp,#0
stp d8,d9,[sp,#16] // per ABI requirement
stp d10,d11,[sp,#32]
stp d12,d13,[sp,#48]
stp d14,d15,[sp,#64]
___
for($i=0; $i<24; $i+=2) { # load A[5][5]
my $j=$i+1;
$code.=<<___;
ldp d$i,d$j,[x0,#8*$i]
___
}
$code.=<<___;
ldr d24,[x0,#8*$i]
b .Loop_absorb_ce
.align 4
.Loop_absorb_ce:
subs $len,$len,$bsz // len - bsz
blo .Labsorbed_ce
___
for (my $i=0; $i<24; $i+=2) {
my $j = $i+1;
$code.=<<___;
ldr d31,[$inp],#8 // *inp++
#ifdef __AARCH64EB__
rev64 v31.16b,v31.16b
#endif
eor $A[$i/5][$i%5],$A[$i/5][$i%5],v31.16b
cmp $bsz,#8*($i+2)
blo .Lprocess_block_ce
ldr d31,[$inp],#8 // *inp++
#ifdef __AARCH64EB__
rev v31.16b,v31.16b
#endif
eor $A[$j/5][$j%5],$A[$j/5][$j%5],v31.16b
beq .Lprocess_block_ce
___
}
$code.=<<___;
ldr d31,[$inp],#8 // *inp++
#ifdef __AARCH64EB__
rev v31.16b,v31.16b
#endif
eor $A[4][4],$A[4][4],v31.16b
.Lprocess_block_ce:
bl KeccakF1600_ce
b .Loop_absorb_ce
.align 4
.Labsorbed_ce:
___
for($i=0; $i<24; $i+=2) { # store A[5][5]
my $j=$i+1;
$code.=<<___;
stp d$i,d$j,[x0,#8*$i]
___
}
$code.=<<___;
str d24,[x0,#8*$i]
add x0,$len,$bsz // return value
ldp d8,d9,[sp,#16]
ldp d10,d11,[sp,#32]
ldp d12,d13,[sp,#48]
ldp d14,d15,[sp,#64]
ldp x29,x30,[sp],#80
ret
.size SHA3_absorb_cext,.-SHA3_absorb_cext
___
}
{
my ($ctx,$out,$len,$bsz) = map("x$_",(0..3));
$code.=<<___;
.globl SHA3_squeeze_cext
.type SHA3_squeeze_cext,%function
.align 5
SHA3_squeeze_cext:
stp x29,x30,[sp,#-16]!
add x29,sp,#0
mov x9,$ctx
mov x10,$bsz
.Loop_squeeze_ce:
ldr x4,[x9],#8
cmp $len,#8
blo .Lsqueeze_tail_ce
#ifdef __AARCH64EB__
rev x4,x4
#endif
str x4,[$out],#8
beq .Lsqueeze_done_ce
sub $len,$len,#8
subs x10,x10,#8
bhi .Loop_squeeze_ce
bl KeccakF1600_cext
ldr x30,[sp,#8]
mov x9,$ctx
mov x10,$bsz
b .Loop_squeeze_ce
.align 4
.Lsqueeze_tail_ce:
strb w4,[$out],#1
lsr x4,x4,#8
subs $len,$len,#1
beq .Lsqueeze_done_ce
strb w4,[$out],#1
lsr x4,x4,#8
subs $len,$len,#1
beq .Lsqueeze_done_ce
strb w4,[$out],#1
lsr x4,x4,#8
subs $len,$len,#1
beq .Lsqueeze_done_ce
strb w4,[$out],#1
lsr x4,x4,#8
subs $len,$len,#1
beq .Lsqueeze_done_ce
strb w4,[$out],#1
lsr x4,x4,#8
subs $len,$len,#1
beq .Lsqueeze_done_ce
strb w4,[$out],#1
lsr x4,x4,#8
subs $len,$len,#1
beq .Lsqueeze_done_ce
strb w4,[$out],#1
.Lsqueeze_done_ce:
ldr x29,[sp],#16
ret
.size SHA3_squeeze_cext,.-SHA3_squeeze_cext
___
} }}}
$code.=<<___;
.asciz "Keccak-1600 absorb and squeeze for ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
___
{ my %opcode = (
"rax1" => 0xce608c00, "eor3" => 0xce000000,
"bcax" => 0xce200000, "xar" => 0xce800000 );
sub unsha3 {
my ($mnemonic,$arg)=@_;
$arg =~ m/[qv]([0-9]+)[^,]*,\s*[qv]([0-9]+)[^,]*(?:,\s*[qv]([0-9]+)[^,]*(?:,\s*[qv#]([0-9\-]+))?)?/
&&
sprintf ".inst\t0x%08x\t//%s %s",
$opcode{$mnemonic}|$1|($2<<5)|($3<<16)|(eval($4)<<10),
$mnemonic,$arg;
}
}
foreach(split("\n",$code)) {
s/\`([^\`]*)\`/eval($1)/ge;
m/\bdup\b/ and s/\.16b/.2d/g or
s/\b(eor3|rax1|xar|bcax)\s+(v.*)/unsha3($1,$2)/ge;
print $_,"\n";
}
close STDOUT;
crypto/sha/asm/keccak1600-avx2.pl
Executable
+480
View File
@@ -0,0 +1,480 @@
#!/usr/bin/env perl
# Copyright 2017 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License"). You may not use
# this file except in compliance with the License. You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html
#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# Keccak-1600 for AVX2.
#
# July 2017.
#
# To paraphrase Gilles Van Assche, if you contemplate Fig. 2.3 on page
# 20 of The Keccak reference [or Fig. 5 of FIPS PUB 202], and load data
# other than A[0][0] in magic order into 6 [256-bit] registers, *each
# dedicated to one axis*, Pi permutation is reduced to intra-register
# shuffles...
#
# It makes other steps more intricate, but overall, is it a win? To be
# more specific index permutations organized by quadruples are:
#
# [4][4] [3][3] [2][2] [1][1]<-+
# [0][4] [0][3] [0][2] [0][1]<-+
# [3][0] [1][0] [4][0] [2][0] |
# [4][3] [3][1] [2][4] [1][2] |
# [3][4] [1][3] [4][2] [2][1] |
# [2][3] [4][1] [1][4] [3][2] |
# [2][2] [4][4] [1][1] [3][3] -+
#
# This however is highly impractical for Theta and Chi. What would help
# Theta is if x indices were aligned column-wise, or in other words:
#
# [0][4] [0][3] [0][2] [0][1]
# [3][0] [1][0] [4][0] [2][0]
#vpermq([4][3] [3][1] [2][4] [1][2], 0b01110010)
# [2][4] [4][3] [1][2] [3][1]
#vpermq([4][2] [3][4] [2][1] [1][3], 0b10001101)
# [3][4] [1][3] [4][2] [2][1]
#vpermq([2][3] [4][1] [1][4] [3][2], 0b01110010)
# [1][4] [2][3] [3][2] [4][1]
#vpermq([1][1] [2][2] [3][3] [4][4], 0b00011011)
# [4][4] [3][3] [2][2] [1][1]
#
# So here we have it, lines not marked with vpermq() represent the magic
# order in which data is to be loaded and maintained. [And lines marked
# with vpermq() represent Pi circular permutation in chosen layout. Note
# that first step is permutation-free.] A[0][0] is loaded to register of
# its own, to all lanes. [A[0][0] is not part of Pi permutation or Rho.]
# Digits in variables' names denote right-most coordinates:
my ($A00, # [0][0] [0][0] [0][0] [0][0] # %ymm0
$A01, # [0][4] [0][3] [0][2] [0][1] # %ymm1
$A20, # [3][0] [1][0] [4][0] [2][0] # %ymm2
$A31, # [2][4] [4][3] [1][2] [3][1] # %ymm3
$A21, # [3][4] [1][3] [4][2] [2][1] # %ymm4
$A41, # [1][4] [2][3] [3][2] [4][1] # %ymm5
$A11) = # [4][4] [3][3] [2][2] [1][1] # %ymm6
map("%ymm$_",(0..6));
# We also need to map the magic order into offsets within structure:
my @A_jagged = ([0,0], [1,0], [1,1], [1,2], [1,3], # [0][0..4]
[2,2], [6,0], [3,1], [4,2], [5,3], # [1][0..4]
[2,0], [4,0], [6,1], [5,2], [3,3], # [2][0..4]
[2,3], [3,0], [5,1], [6,2], [4,3], # [3][0..4]
[2,1], [5,0], [4,1], [3,2], [6,3]); # [4][0..4]
@A_jagged = map(8*($$_[0]*4+$$_[1]), @A_jagged); # ... and now linear
# But on the other hand Chi is much better off if y indices were aligned
# column-wise, not x. For this reason we have to shuffle data prior
# Chi and revert it afterwards. Prior shuffle is naturally merged with
# Pi itself:
#
# [0][4] [0][3] [0][2] [0][1]
# [3][0] [1][0] [4][0] [2][0]
#vpermq([4][3] [3][1] [2][4] [1][2], 0b01110010)
#vpermq([2][4] [4][3] [1][2] [3][1], 0b00011011) = 0b10001101
# [3][1] [1][2] [4][3] [2][4]
#vpermq([4][2] [3][4] [2][1] [1][3], 0b10001101)
#vpermq([3][4] [1][3] [4][2] [2][1], 0b11100100) = 0b10001101
# [3][4] [1][3] [4][2] [2][1]
#vpermq([2][3] [4][1] [1][4] [3][2], 0b01110010)
#vpermq([1][4] [2][3] [3][2] [4][1], 0b01110010) = 0b00011011
# [3][2] [1][4] [4][1] [2][3]
#vpermq([1][1] [2][2] [3][3] [4][4], 0b00011011)
#vpermq([4][4] [3][3] [2][2] [1][1], 0b10001101) = 0b01110010
# [3][3] [1][1] [4][4] [2][2]
#
# And reverse post-Chi permutation:
#
# [0][4] [0][3] [0][2] [0][1]
# [3][0] [1][0] [4][0] [2][0]
#vpermq([3][1] [1][2] [4][3] [2][4], 0b00011011)
# [2][4] [4][3] [1][2] [3][1]
#vpermq([3][4] [1][3] [4][2] [2][1], 0b11100100) = nop :-)
# [3][4] [1][3] [4][2] [2][1]
#vpermq([3][2] [1][4] [4][1] [2][3], 0b10001101)
# [1][4] [2][3] [3][2] [4][1]
#vpermq([3][3] [1][1] [4][4] [2][2], 0b01110010)
# [4][4] [3][3] [2][2] [1][1]
#
########################################################################
# Numbers are cycles per processed byte out of large message.
#
# r=1088(*)
#
# Haswell 8.7/+10%
# Skylake 7.8/+20%
# Ryzen 17(**)
#
# (*) Corresponds to SHA3-256. Percentage after slash is improvement
# coefficient in comparison to scalar keccak1600-x86_64.pl.
# (**) It's expected that Ryzen performs poorly, because instruction
# issue rate is limited to two AVX2 instructions per cycle and
# in addition vpblendd is reportedly bound to specific port.
# Obviously this code path should not be executed on Ryzen.
my @T = map("%ymm$_",(7..15));
my ($C14,$C00,$D00,$D14) = @T[5..8];
$code.=<<___;
.text
.type __KeccakF1600,\@function
.align 32
__KeccakF1600:
lea rhotates_left+96(%rip),%r8
lea rhotates_right+96(%rip),%r9
lea iotas(%rip),%r10
mov \$24,%eax
jmp .Loop_avx2
.align 32
.Loop_avx2:
######################################### Theta
vpshufd \$0b01001110,$A20,$C00
vpxor $A31,$A41,$C14
vpxor $A11,$A21,@T[2]
vpxor $A01,$C14,$C14
vpxor @T[2],$C14,$C14 # C[1..4]
vpermq \$0b10010011,$C14,@T[4]
vpxor $A20,$C00,$C00
vpermq \$0b01001110,$C00,@T[0]
vpsrlq \$63,$C14,@T[1]
vpaddq $C14,$C14,@T[2]
vpor @T[2],@T[1],@T[1] # ROL64(C[1..4],1)
vpermq \$0b00111001,@T[1],$D14
vpxor @T[4],@T[1],$D00
vpermq \$0b00000000,$D00,$D00 # D[0..0] = ROL64(C[1],1) ^ C[4]
vpxor $A00,$C00,$C00
vpxor @T[0],$C00,$C00 # C[0..0]
vpsrlq \$63,$C00,@T[0]
vpaddq $C00,$C00,@T[1]
vpor @T[0],@T[1],@T[1] # ROL64(C[0..0],1)
vpxor $D00,$A20,$A20 # ^= D[0..0]
vpxor $D00,$A00,$A00 # ^= D[0..0]
vpblendd \$0b11000000,@T[1],$D14,$D14
vpblendd \$0b00000011,$C00,@T[4],@T[4]
vpxor @T[4],$D14,$D14 # D[1..4] = ROL64(C[2..4,0),1) ^ C[0..3]
######################################### Rho + Pi + pre-Chi shuffle
vpsllvq 0*32-96(%r8),$A20,@T[3]
vpsrlvq 0*32-96(%r9),$A20,$A20
vpor @T[3],$A20,$A20
vpxor $D14,$A31,$A31 # ^= D[1..4] from Theta
vpsllvq 2*32-96(%r8),$A31,@T[4]
vpsrlvq 2*32-96(%r9),$A31,$A31
vpor @T[4],$A31,$A31
vpxor $D14,$A21,$A21 # ^= D[1..4] from Theta
vpsllvq 3*32-96(%r8),$A21,@T[5]
vpsrlvq 3*32-96(%r9),$A21,$A21
vpor @T[5],$A21,$A21
vpxor $D14,$A41,$A41 # ^= D[1..4] from Theta
vpsllvq 4*32-96(%r8),$A41,@T[6]
vpsrlvq 4*32-96(%r9),$A41,$A41
vpor @T[6],$A41,$A41
vpxor $D14,$A11,$A11 # ^= D[1..4] from Theta
vpermq \$0b10001101,$A20,@T[3] # $A20 -> future $A31
vpermq \$0b10001101,$A31,@T[4] # $A31 -> future $A21
vpsllvq 5*32-96(%r8),$A11,@T[7]
vpsrlvq 5*32-96(%r9),$A11,@T[1]
vpor @T[7],@T[1],@T[1] # $A11 -> future $A01
vpxor $D14,$A01,$A01 # ^= D[1..4] from Theta
vpermq \$0b00011011,$A21,@T[5] # $A21 -> future $A41
vpermq \$0b01110010,$A41,@T[6] # $A41 -> future $A11
vpsllvq 1*32-96(%r8),$A01,@T[8]
vpsrlvq 1*32-96(%r9),$A01,@T[2]
vpor @T[8],@T[2],@T[2] # $A01 -> future $A20
######################################### Chi
vpsrldq \$8,@T[1],@T[7]
vpandn @T[7],@T[1],@T[0] # tgting [0][0] [0][0] [0][0] [0][0]
vpblendd \$0b00001100,@T[6],@T[2],$A31 # [4][4] [2][0]
vpblendd \$0b00001100,@T[2],@T[4],@T[8] # [4][0] [2][1]
vpblendd \$0b00001100,@T[4],@T[3],$A41 # [4][2] [2][4]
vpblendd \$0b00001100,@T[3],@T[2],@T[7] # [4][3] [2][0]
vpblendd \$0b00110000,@T[4],$A31,$A31 # [1][3] [4][4] [2][0]
vpblendd \$0b00110000,@T[5],@T[8],@T[8] # [1][4] [4][0] [2][1]
vpblendd \$0b00110000,@T[2],$A41,$A41 # [1][0] [4][2] [2][4]
vpblendd \$0b00110000,@T[6],@T[7],@T[7] # [1][1] [4][3] [2][0]
vpblendd \$0b11000000,@T[5],$A31,$A31 # [3][2] [1][3] [4][4] [2][0]
vpblendd \$0b11000000,@T[6],@T[8],@T[8] # [3][3] [1][4] [4][0] [2][1]
vpblendd \$0b11000000,@T[6],$A41,$A41 # [3][3] [1][0] [4][2] [2][4]
vpblendd \$0b11000000,@T[4],@T[7],@T[7] # [3][4] [1][1] [4][3] [2][0]
vpandn @T[8],$A31,$A31 # tgting [3][1] [1][2] [4][3] [2][4]
vpandn @T[7],$A41,$A41 # tgting [3][2] [1][4] [4][1] [2][3]
vpblendd \$0b00001100,@T[2],@T[5],$A11 # [4][0] [2][3]
vpblendd \$0b00001100,@T[5],@T[3],@T[8] # [4][1] [2][4]
vpxor @T[3],$A31,$A31
vpblendd \$0b00110000,@T[3],$A11,$A11 # [1][2] [4][0] [2][3]
vpblendd \$0b00110000,@T[4],@T[8],@T[8] # [1][3] [4][1] [2][4]
vpxor @T[5],$A41,$A41
vpblendd \$0b11000000,@T[4],$A11,$A11 # [3][4] [1][2] [4][0] [2][3]
vpblendd \$0b11000000,@T[2],@T[8],@T[8] # [3][0] [1][3] [4][1] [2][4]
vpandn @T[8],$A11,$A11 # tgting [3][3] [1][1] [4][4] [2][2]
vpxor @T[6],$A11,$A11
vpermq \$0b00011110,@T[1],$A21 # [0][1] [0][2] [0][4] [0][3]
vpblendd \$0b00110000,$A00,$A21,@T[8] # [0][1] [0][0] [0][4] [0][3]
vpermq \$0b00111001,@T[1],$A01 # [0][1] [0][4] [0][3] [0][2]
vpblendd \$0b11000000,$A00,$A01,$A01 # [0][0] [0][4] [0][3] [0][2]
vpandn @T[8],$A01,$A01 # tgting [0][4] [0][3] [0][2] [0][1]
vpblendd \$0b00001100,@T[5],@T[4],$A20 # [4][1] [2][1]
vpblendd \$0b00001100,@T[4],@T[6],@T[7] # [4][2] [2][2]
vpblendd \$0b00110000,@T[6],$A20,$A20 # [1][1] [4][1] [2][1]
vpblendd \$0b00110000,@T[3],@T[7],@T[7] # [1][2] [4][2] [2][2]
vpblendd \$0b11000000,@T[3],$A20,$A20 # [3][1] [1][1] [4][1] [2][1]
vpblendd \$0b11000000,@T[5],@T[7],@T[7] # [3][2] [1][2] [4][2] [2][2]
vpandn @T[7],$A20,$A20 # tgting [3][0] [1][0] [4][0] [2][0]
vpxor @T[2],$A20,$A20
vpermq \$0b00000000,@T[0],@T[0] # [0][0] [0][0] [0][0] [0][0]
vpermq \$0b00011011,$A31,$A31 # post-Chi shuffle
vpermq \$0b10001101,$A41,$A41
vpermq \$0b01110010,$A11,$A11
vpblendd \$0b00001100,@T[3],@T[6],$A21 # [4][3] [2][2]
vpblendd \$0b00001100,@T[6],@T[5],@T[7] # [4][4] [2][3]
vpblendd \$0b00110000,@T[5],$A21,$A21 # [1][4] [4][3] [2][2]
vpblendd \$0b00110000,@T[2],@T[7],@T[7] # [1][0] [4][4] [2][3]
vpblendd \$0b11000000,@T[2],$A21,$A21 # [3][0] [1][4] [4][3] [2][2]
vpblendd \$0b11000000,@T[3],@T[7],@T[7] # [3][1] [1][0] [4][4] [2][3]
vpandn @T[7],$A21,$A21 # tgting [3][4] [1][3] [4][2] [2][1]
vpxor @T[0],$A00,$A00
vpxor @T[1],$A01,$A01
vpxor @T[4],$A21,$A21
######################################### Iota
vpxor (%r10),$A00,$A00
lea 32(%r10),%r10
dec %eax
jnz .Loop_avx2
ret
.size __KeccakF1600,.-__KeccakF1600
___
my ($A_flat,$inp,$len,$bsz) = ("%rdi","%rsi","%rdx","%rcx");
my $out = $inp; # in squeeze
$code.=<<___;
.globl SHA3_absorb
.type SHA3_absorb,\@function
.align 32
SHA3_absorb:
mov %rsp,%r11
lea -240(%rsp),%rsp
and \$-32,%rsp
lea 96($A_flat),$A_flat
lea 96($inp),$inp
lea 96(%rsp),%r10
vzeroupper
vpbroadcastq -96($A_flat),$A00 # load A[5][5]
vmovdqu 8+32*0-96($A_flat),$A01
vmovdqu 8+32*1-96($A_flat),$A20
vmovdqu 8+32*2-96($A_flat),$A31
vmovdqu 8+32*3-96($A_flat),$A21
vmovdqu 8+32*4-96($A_flat),$A41
vmovdqu 8+32*5-96($A_flat),$A11
vpxor @T[0],@T[0],@T[0]
vmovdqa @T[0],32*2-96(%r10) # zero transfer area on stack
vmovdqa @T[0],32*3-96(%r10)
vmovdqa @T[0],32*4-96(%r10)
vmovdqa @T[0],32*5-96(%r10)
vmovdqa @T[0],32*6-96(%r10)
.Loop_absorb_avx2:
mov $bsz,%rax
sub $bsz,$len
jc .Ldone_absorb_avx2
shr \$3,%eax
vpbroadcastq 0-96($inp),@T[0]
vmovdqu 8-96($inp),@T[1]
sub \$4,%eax
___
for(my $i=5; $i<25; $i++) {
$code.=<<___
dec %eax
jz .Labsorved_avx2
mov 8*$i-96($inp),%r8
mov %r8,$A_jagged[$i]-96(%r10)
___
}
$code.=<<___;
.Labsorved_avx2:
lea ($inp,$bsz),$inp
vpxor @T[0],$A00,$A00
vpxor @T[1],$A01,$A01
vpxor 32*2-96(%r10),$A20,$A20
vpxor 32*3-96(%r10),$A31,$A31
vpxor 32*4-96(%r10),$A21,$A21
vpxor 32*5-96(%r10),$A41,$A41
vpxor 32*6-96(%r10),$A11,$A11
call __KeccakF1600
lea 96(%rsp),%r10
jmp .Loop_absorb_avx2
.Ldone_absorb_avx2:
vmovq %xmm0,-96($A_flat)
vmovdqu $A01,8+32*0-96($A_flat)
vmovdqu $A20,8+32*1-96($A_flat)
vmovdqu $A31,8+32*2-96($A_flat)
vmovdqu $A21,8+32*3-96($A_flat)
vmovdqu $A41,8+32*4-96($A_flat)
vmovdqu $A11,8+32*5-96($A_flat)
vzeroupper
lea (%r11),%rsp
lea ($len,$bsz),%rax # return value
ret
.size SHA3_absorb,.-SHA3_absorb
.globl SHA3_squeeze
.type SHA3_squeeze,\@function
.align 32
SHA3_squeeze:
mov %rsp,%r11
lea 96($A_flat),$A_flat
shr \$3,$bsz
vzeroupper
vpbroadcastq -96($A_flat),$A00
vpxor @T[0],@T[0],@T[0]
vmovdqu 8+32*0-96($A_flat),$A01
vmovdqu 8+32*1-96($A_flat),$A20
vmovdqu 8+32*2-96($A_flat),$A31
vmovdqu 8+32*3-96($A_flat),$A21
vmovdqu 8+32*4-96($A_flat),$A41
vmovdqu 8+32*5-96($A_flat),$A11
mov $bsz,%rax
.Loop_squeeze_avx2:
mov @A_jagged[$i]-96($A_flat),%r8
___
for (my $i=0; $i<25; $i++) {
$code.=<<___;
sub \$8,$len
jc .Ltail_squeeze_avx2
mov %r8,($out)
lea 8($out),$out
je .Ldone_squeeze_avx2
dec %eax
je .Lextend_output_avx2
mov @A_jagged[$i+1]-120($A_flat),%r8
___
}
$code.=<<___;
.Lextend_output_avx2:
call __KeccakF1600
vmovq %xmm0,-96($A_flat)
vmovdqu $A01,8+32*0-96($A_flat)
vmovdqu $A20,8+32*1-96($A_flat)
vmovdqu $A31,8+32*2-96($A_flat)
vmovdqu $A21,8+32*3-96($A_flat)
vmovdqu $A41,8+32*4-96($A_flat)
vmovdqu $A11,8+32*5-96($A_flat)
mov $bsz,%rax
jmp .Loop_squeeze_avx2
.Ltail_squeeze_avx2:
add \$8,$len
.Loop_tail_avx2:
mov %r8b,($out)
lea 1($out),$out
shr \$8,%r8
dec $len
jnz .Loop_tail_avx2
.Ldone_squeeze_avx2:
vzeroupper
lea (%r11),%rsp
ret
.size SHA3_squeeze,.-SHA3_squeeze
.align 64
rhotates_left:
.quad 3, 18, 36, 41 # [2][0] [4][0] [1][0] [3][0]
.quad 1, 62, 28, 27 # [0][1] [0][2] [0][3] [0][4]
.quad 45, 6, 56, 39 # [3][1] [1][2] [4][3] [2][4]
.quad 10, 61, 55, 8 # [2][1] [4][2] [1][3] [3][4]
.quad 2, 15, 25, 20 # [4][1] [3][2] [2][3] [1][4]
.quad 44, 43, 21, 14 # [1][1] [2][2] [3][3] [4][4]
rhotates_right:
.quad 64-3, 64-18, 64-36, 64-41
.quad 64-1, 64-62, 64-28, 64-27
.quad 64-45, 64-6, 64-56, 64-39
.quad 64-10, 64-61, 64-55, 64-8
.quad 64-2, 64-15, 64-25, 64-20
.quad 64-44, 64-43, 64-21, 64-14
iotas:
.quad 0x0000000000000001, 0x0000000000000001, 0x0000000000000001, 0x0000000000000001
.quad 0x0000000000008082, 0x0000000000008082, 0x0000000000008082, 0x0000000000008082
.quad 0x800000000000808a, 0x800000000000808a, 0x800000000000808a, 0x800000000000808a
.quad 0x8000000080008000, 0x8000000080008000, 0x8000000080008000, 0x8000000080008000
.quad 0x000000000000808b, 0x000000000000808b, 0x000000000000808b, 0x000000000000808b
.quad 0x0000000080000001, 0x0000000080000001, 0x0000000080000001, 0x0000000080000001
.quad 0x8000000080008081, 0x8000000080008081, 0x8000000080008081, 0x8000000080008081
.quad 0x8000000000008009, 0x8000000000008009, 0x8000000000008009, 0x8000000000008009
.quad 0x000000000000008a, 0x000000000000008a, 0x000000000000008a, 0x000000000000008a
.quad 0x0000000000000088, 0x0000000000000088, 0x0000000000000088, 0x0000000000000088
.quad 0x0000000080008009, 0x0000000080008009, 0x0000000080008009, 0x0000000080008009
.quad 0x000000008000000a, 0x000000008000000a, 0x000000008000000a, 0x000000008000000a
.quad 0x000000008000808b, 0x000000008000808b, 0x000000008000808b, 0x000000008000808b
.quad 0x800000000000008b, 0x800000000000008b, 0x800000000000008b, 0x800000000000008b
.quad 0x8000000000008089, 0x8000000000008089, 0x8000000000008089, 0x8000000000008089
.quad 0x8000000000008003, 0x8000000000008003, 0x8000000000008003, 0x8000000000008003
.quad 0x8000000000008002, 0x8000000000008002, 0x8000000000008002, 0x8000000000008002
.quad 0x8000000000000080, 0x8000000000000080, 0x8000000000000080, 0x8000000000000080
.quad 0x000000000000800a, 0x000000000000800a, 0x000000000000800a, 0x000000000000800a
.quad 0x800000008000000a, 0x800000008000000a, 0x800000008000000a, 0x800000008000000a
.quad 0x8000000080008081, 0x8000000080008081, 0x8000000080008081, 0x8000000080008081
.quad 0x8000000000008080, 0x8000000000008080, 0x8000000000008080, 0x8000000000008080
.quad 0x0000000080000001, 0x0000000080000001, 0x0000000080000001, 0x0000000080000001
.quad 0x8000000080008008, 0x8000000080008008, 0x8000000080008008, 0x8000000080008008
.asciz "Keccak-1600 absorb and squeeze for AVX2, CRYPTOGAMS by <appro\@openssl.org>"
___
print $code;
close STDOUT;
crypto/sha/asm/keccak1600-avx512.pl
Executable
+549
View File
@@ -0,0 +1,549 @@
#!/usr/bin/env perl
# Copyright 2017 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License"). You may not use
# this file except in compliance with the License. You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html
#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# Keccak-1600 for AVX-512F.
#
# July 2017.
#
# Below code is KECCAK_1X_ALT implementation (see sha/keccak1600.c).
# Pretty straightforward, the only "magic" is data layout in registers.
# It's impossible to have one that is optimal for every step, hence
# it's changing as algorithm progresses. Data is saved in linear order,
# but in-register order morphs between rounds. Even rounds take in
# linear layout, and odd rounds - transposed, or "verticaly-shaped"...
#
########################################################################
# Numbers are cycles per processed byte out of large message.
#
# r=1088(*)
#
# Knights Landing 7.6
# Skylake-X 5.7
#
# (*) Corresponds to SHA3-256.
########################################################################
# Below code is combination of two ideas. One is taken from Keccak Code
# Package, hereafter KCP, and another one from initial version of this
# module. What is common is observation that Pi's input and output are
# "mostly transposed", i.e. if input is aligned by x coordinate, then
# output is [mostly] aligned by y. Both versions, KCP and predecessor,
# were trying to use one of them from round to round, which resulted in
# some kind of transposition in each round. This version still does
# transpose data, but only every second round. Another essential factor
# is that KCP transposition has to be performed with instructions that
# turned to be rather expensive on Knights Landing, both latency- and
# throughput-wise. Not to mention that some of them have to depend on
# each other. On the other hand initial version of this module was
# relying heavily on blend instructions. There were lots of them,
# resulting in higher instruction count, yet it performed better on
# Knights Landing, because processor can execute pair of them each
# cycle and they have minimal latency. This module is an attempt to
# bring best parts together:-)
#
# Coordinates below correspond to those in sha/keccak1600.c. Input
# layout is straight linear:
#
# [0][4] [0][3] [0][2] [0][1] [0][0]
# [1][4] [1][3] [1][2] [1][1] [1][0]
# [2][4] [2][3] [2][2] [2][1] [2][0]
# [3][4] [3][3] [3][2] [3][1] [3][0]
# [4][4] [4][3] [4][2] [4][1] [4][0]
#
# It's perfect for Theta, while Pi is reduced to intra-register
# permutations which yield layout perfect for Chi:
#
# [4][0] [3][0] [2][0] [1][0] [0][0]
# [4][1] [3][1] [2][1] [1][1] [0][1]
# [4][2] [3][2] [2][2] [1][2] [0][2]
# [4][3] [3][3] [2][3] [1][3] [0][3]
# [4][4] [3][4] [2][4] [1][4] [0][4]
#
# Now instead of performing full transposition and feeding it to next
# identical round, we perform kind of diagonal transposition to layout
# from initial version of this module, and make it suitable for Theta:
#
# [4][4] [3][3] [2][2] [1][1] [0][0]>4.3.2.1.0>[4][4] [3][3] [2][2] [1][1] [0][0]
# [4][0] [3][4] [2][3] [1][2] [0][1]>3.2.1.0.4>[3][4] [2][3] [1][2] [0][1] [4][0]
# [4][1] [3][0] [2][4] [1][3] [0][2]>2.1.0.4.3>[2][4] [1][3] [0][2] [4][1] [3][0]
# [4][2] [3][1] [2][0] [1][4] [0][3]>1.0.4.3.2>[1][4] [0][3] [4][2] [3][1] [2][0]
# [4][3] [3][2] [2][1] [1][0] [0][4]>0.4.3.2.1>[0][4] [4][3] [3][2] [2][1] [1][0]
#
# Now intra-register permutations yield initial [almost] straight
# linear layout:
#
# [4][4] [3][3] [2][2] [1][1] [0][0]
##[0][4] [0][3] [0][2] [0][1] [0][0]
# [3][4] [2][3] [1][2] [0][1] [4][0]
##[2][3] [2][2] [2][1] [2][0] [2][4]
# [2][4] [1][3] [0][2] [4][1] [3][0]
##[4][2] [4][1] [4][0] [4][4] [4][3]
# [1][4] [0][3] [4][2] [3][1] [2][0]
##[1][1] [1][0] [1][4] [1][3] [1][2]
# [0][4] [4][3] [3][2] [2][1] [1][0]
##[3][0] [3][4] [3][3] [3][2] [3][1]
#
# This means that odd round Chi is performed in less suitable layout,
# with a number of additional permutations. But overall it turned to be
# a win. Permutations are fastest possible on Knights Landing and they
# are laid down to be independent of each other. In the essence I traded
# 20 blend instructions for 3 permutations. The result is 13% faster
# than KCP on Skylake-X, and >40% on Knights Landing.
#
# As implied, data is loaded in straight linear order. Digits in
# variables' names represent coordinates of right-most element of
# loaded data chunk:
my ($A00, # [0][4] [0][3] [0][2] [0][1] [0][0]
$A10, # [1][4] [1][3] [1][2] [1][1] [1][0]
$A20, # [2][4] [2][3] [2][2] [2][1] [2][0]
$A30, # [3][4] [3][3] [3][2] [3][1] [3][0]
$A40) = # [4][4] [4][3] [4][2] [4][1] [4][0]
map("%zmm$_",(0..4));
# We also need to map the magic order into offsets within structure:
my @A_jagged = ([0,0], [0,1], [0,2], [0,3], [0,4],
[1,0], [1,1], [1,2], [1,3], [1,4],
[2,0], [2,1], [2,2], [2,3], [2,4],
[3,0], [3,1], [3,2], [3,3], [3,4],
[4,0], [4,1], [4,2], [4,3], [4,4]);
@A_jagged = map(8*($$_[0]*8+$$_[1]), @A_jagged); # ... and now linear
my @T = map("%zmm$_",(5..12));
my @Theta = map("%zmm$_",(33,13..16)); # invalid @Theta[0] is not typo
my @Pi0 = map("%zmm$_",(17..21));
my @Rhotate0 = map("%zmm$_",(22..26));
my @Rhotate1 = map("%zmm$_",(27..31));
my ($C00,$D00) = @T[0..1];
my ($k00001,$k00010,$k00100,$k01000,$k10000,$k11111) = map("%k$_",(1..6));
$code.=<<___;
.text
.type __KeccakF1600,\@function
.align 32
__KeccakF1600:
lea iotas(%rip),%r10
mov \$12,%eax
jmp .Loop_avx512
.align 32
.Loop_avx512:
######################################### Theta, even round
vmovdqa64 $A00,@T[0] # put aside original A00
vpternlogq \$0x96,$A20,$A10,$A00 # and use it as "C00"
vpternlogq \$0x96,$A40,$A30,$A00
vprolq \$1,$A00,$D00
vpermq $A00,@Theta[1],$A00
vpermq $D00,@Theta[4],$D00
vpternlogq \$0x96,$A00,$D00,@T[0] # T[0] is original A00
vpternlogq \$0x96,$A00,$D00,$A10
vpternlogq \$0x96,$A00,$D00,$A20
vpternlogq \$0x96,$A00,$D00,$A30
vpternlogq \$0x96,$A00,$D00,$A40
######################################### Rho
vprolvq @Rhotate0[0],@T[0],$A00 # T[0] is original A00
vprolvq @Rhotate0[1],$A10,$A10
vprolvq @Rhotate0[2],$A20,$A20
vprolvq @Rhotate0[3],$A30,$A30
vprolvq @Rhotate0[4],$A40,$A40
######################################### Pi
vpermq $A00,@Pi0[0],$A00
vpermq $A10,@Pi0[1],$A10
vpermq $A20,@Pi0[2],$A20
vpermq $A30,@Pi0[3],$A30
vpermq $A40,@Pi0[4],$A40
######################################### Chi
vmovdqa64 $A00,@T[0]
vmovdqa64 $A10,@T[1]
vpternlogq \$0xD2,$A20,$A10,$A00
vpternlogq \$0xD2,$A30,$A20,$A10
vpternlogq \$0xD2,$A40,$A30,$A20
vpternlogq \$0xD2,@T[0],$A40,$A30
vpternlogq \$0xD2,@T[1],@T[0],$A40
######################################### Iota
vpxorq (%r10),$A00,${A00}{$k00001}
lea 16(%r10),%r10
######################################### Harmonize rounds
vpblendmq $A20,$A10,@{T[1]}{$k00010}
vpblendmq $A30,$A20,@{T[2]}{$k00010}
vpblendmq $A40,$A30,@{T[3]}{$k00010}
vpblendmq $A10,$A00,@{T[0]}{$k00010}
vpblendmq $A00,$A40,@{T[4]}{$k00010}
vpblendmq $A30,@T[1],@{T[1]}{$k00100}
vpblendmq $A40,@T[2],@{T[2]}{$k00100}
vpblendmq $A20,@T[0],@{T[0]}{$k00100}
vpblendmq $A00,@T[3],@{T[3]}{$k00100}
vpblendmq $A10,@T[4],@{T[4]}{$k00100}
vpblendmq $A40,@T[1],@{T[1]}{$k01000}
vpblendmq $A30,@T[0],@{T[0]}{$k01000}
vpblendmq $A00,@T[2],@{T[2]}{$k01000}
vpblendmq $A10,@T[3],@{T[3]}{$k01000}
vpblendmq $A20,@T[4],@{T[4]}{$k01000}
vpblendmq $A40,@T[0],@{T[0]}{$k10000}
vpblendmq $A00,@T[1],@{T[1]}{$k10000}
vpblendmq $A10,@T[2],@{T[2]}{$k10000}
vpblendmq $A20,@T[3],@{T[3]}{$k10000}
vpblendmq $A30,@T[4],@{T[4]}{$k10000}
#vpermq @T[0],@Theta[0],$A00 # doesn't actually change order
vpermq @T[1],@Theta[1],$A10
vpermq @T[2],@Theta[2],$A20
vpermq @T[3],@Theta[3],$A30
vpermq @T[4],@Theta[4],$A40
######################################### Theta, odd round
vmovdqa64 $T[0],$A00 # real A00
vpternlogq \$0x96,$A20,$A10,$C00 # C00 is @T[0]'s alias
vpternlogq \$0x96,$A40,$A30,$C00
vprolq \$1,$C00,$D00
vpermq $C00,@Theta[1],$C00
vpermq $D00,@Theta[4],$D00
vpternlogq \$0x96,$C00,$D00,$A00
vpternlogq \$0x96,$C00,$D00,$A30
vpternlogq \$0x96,$C00,$D00,$A10
vpternlogq \$0x96,$C00,$D00,$A40
vpternlogq \$0x96,$C00,$D00,$A20
######################################### Rho
vprolvq @Rhotate1[0],$A00,$A00
vprolvq @Rhotate1[3],$A30,@T[1]
vprolvq @Rhotate1[1],$A10,@T[2]
vprolvq @Rhotate1[4],$A40,@T[3]
vprolvq @Rhotate1[2],$A20,@T[4]
vpermq $A00,@Theta[4],@T[5]
vpermq $A00,@Theta[3],@T[6]
######################################### Iota
vpxorq -8(%r10),$A00,${A00}{$k00001}
######################################### Pi
vpermq @T[1],@Theta[2],$A10
vpermq @T[2],@Theta[4],$A20
vpermq @T[3],@Theta[1],$A30
vpermq @T[4],@Theta[3],$A40
######################################### Chi
vpternlogq \$0xD2,@T[6],@T[5],$A00
vpermq @T[1],@Theta[1],@T[7]
#vpermq @T[1],@Theta[0],@T[1]
vpternlogq \$0xD2,@T[1],@T[7],$A10
vpermq @T[2],@Theta[3],@T[0]
vpermq @T[2],@Theta[2],@T[2]
vpternlogq \$0xD2,@T[2],@T[0],$A20
#vpermq @T[3],@Theta[0],@T[3]
vpermq @T[3],@Theta[4],@T[1]
vpternlogq \$0xD2,@T[1],@T[3],$A30
vpermq @T[4],@Theta[2],@T[0]
vpermq @T[4],@Theta[1],@T[4]
vpternlogq \$0xD2,@T[4],@T[0],$A40
dec %eax
jnz .Loop_avx512
ret
.size __KeccakF1600,.-__KeccakF1600
___
my ($A_flat,$inp,$len,$bsz) = ("%rdi","%rsi","%rdx","%rcx");
my $out = $inp; # in squeeze
$code.=<<___;
.globl SHA3_absorb
.type SHA3_absorb,\@function
.align 32
SHA3_absorb:
mov %rsp,%r11
lea -320(%rsp),%rsp
and \$-64,%rsp
lea 96($A_flat),$A_flat
lea 96($inp),$inp
lea 128(%rsp),%r9
lea theta_perm(%rip),%r8
kxnorw $k11111,$k11111,$k11111
kshiftrw \$15,$k11111,$k00001
kshiftrw \$11,$k11111,$k11111
kshiftlw \$1,$k00001,$k00010
kshiftlw \$2,$k00001,$k00100
kshiftlw \$3,$k00001,$k01000
kshiftlw \$4,$k00001,$k10000
#vmovdqa64 64*0(%r8),@Theta[0]
vmovdqa64 64*1(%r8),@Theta[1]
vmovdqa64 64*2(%r8),@Theta[2]
vmovdqa64 64*3(%r8),@Theta[3]
vmovdqa64 64*4(%r8),@Theta[4]
vmovdqa64 64*5(%r8),@Rhotate1[0]
vmovdqa64 64*6(%r8),@Rhotate1[1]
vmovdqa64 64*7(%r8),@Rhotate1[2]
vmovdqa64 64*8(%r8),@Rhotate1[3]
vmovdqa64 64*9(%r8),@Rhotate1[4]
vmovdqa64 64*10(%r8),@Rhotate0[0]
vmovdqa64 64*11(%r8),@Rhotate0[1]
vmovdqa64 64*12(%r8),@Rhotate0[2]
vmovdqa64 64*13(%r8),@Rhotate0[3]
vmovdqa64 64*14(%r8),@Rhotate0[4]
vmovdqa64 64*15(%r8),@Pi0[0]
vmovdqa64 64*16(%r8),@Pi0[1]
vmovdqa64 64*17(%r8),@Pi0[2]
vmovdqa64 64*18(%r8),@Pi0[3]
vmovdqa64 64*19(%r8),@Pi0[4]
vmovdqu64 40*0-96($A_flat),${A00}{$k11111}{z}
vpxorq @T[0],@T[0],@T[0]
vmovdqu64 40*1-96($A_flat),${A10}{$k11111}{z}
vmovdqu64 40*2-96($A_flat),${A20}{$k11111}{z}
vmovdqu64 40*3-96($A_flat),${A30}{$k11111}{z}
vmovdqu64 40*4-96($A_flat),${A40}{$k11111}{z}
vmovdqa64 @T[0],0*64-128(%r9) # zero transfer area on stack
vmovdqa64 @T[0],1*64-128(%r9)
vmovdqa64 @T[0],2*64-128(%r9)
vmovdqa64 @T[0],3*64-128(%r9)
vmovdqa64 @T[0],4*64-128(%r9)
jmp .Loop_absorb_avx512
.align 32
.Loop_absorb_avx512:
mov $bsz,%rax
sub $bsz,$len
jc .Ldone_absorb_avx512
shr \$3,%eax
___
for(my $i=0; $i<25; $i++) {
$code.=<<___
mov 8*$i-96($inp),%r8
mov %r8,$A_jagged[$i]-128(%r9)
dec %eax
jz .Labsorved_avx512
___
}
$code.=<<___;
.Labsorved_avx512:
lea ($inp,$bsz),$inp
vpxorq 64*0-128(%r9),$A00,$A00
vpxorq 64*1-128(%r9),$A10,$A10
vpxorq 64*2-128(%r9),$A20,$A20
vpxorq 64*3-128(%r9),$A30,$A30
vpxorq 64*4-128(%r9),$A40,$A40
call __KeccakF1600
jmp .Loop_absorb_avx512
.align 32
.Ldone_absorb_avx512:
vmovdqu64 $A00,40*0-96($A_flat){$k11111}
vmovdqu64 $A10,40*1-96($A_flat){$k11111}
vmovdqu64 $A20,40*2-96($A_flat){$k11111}
vmovdqu64 $A30,40*3-96($A_flat){$k11111}
vmovdqu64 $A40,40*4-96($A_flat){$k11111}
vzeroupper
lea (%r11),%rsp
lea ($len,$bsz),%rax # return value
ret
.size SHA3_absorb,.-SHA3_absorb
.globl SHA3_squeeze
.type SHA3_squeeze,\@function
.align 32
SHA3_squeeze:
mov %rsp,%r11
lea 96($A_flat),$A_flat
cmp $bsz,$len
jbe .Lno_output_extension_avx512
lea theta_perm(%rip),%r8
kxnorw $k11111,$k11111,$k11111
kshiftrw \$15,$k11111,$k00001
kshiftrw \$11,$k11111,$k11111
kshiftlw \$1,$k00001,$k00010
kshiftlw \$2,$k00001,$k00100
kshiftlw \$3,$k00001,$k01000
kshiftlw \$4,$k00001,$k10000
#vmovdqa64 64*0(%r8),@Theta[0]
vmovdqa64 64*1(%r8),@Theta[1]
vmovdqa64 64*2(%r8),@Theta[2]
vmovdqa64 64*3(%r8),@Theta[3]
vmovdqa64 64*4(%r8),@Theta[4]
vmovdqa64 64*5(%r8),@Rhotate1[0]
vmovdqa64 64*6(%r8),@Rhotate1[1]
vmovdqa64 64*7(%r8),@Rhotate1[2]
vmovdqa64 64*8(%r8),@Rhotate1[3]
vmovdqa64 64*9(%r8),@Rhotate1[4]
vmovdqa64 64*10(%r8),@Rhotate0[0]
vmovdqa64 64*11(%r8),@Rhotate0[1]
vmovdqa64 64*12(%r8),@Rhotate0[2]
vmovdqa64 64*13(%r8),@Rhotate0[3]
vmovdqa64 64*14(%r8),@Rhotate0[4]
vmovdqa64 64*15(%r8),@Pi0[0]
vmovdqa64 64*16(%r8),@Pi0[1]
vmovdqa64 64*17(%r8),@Pi0[2]
vmovdqa64 64*18(%r8),@Pi0[3]
vmovdqa64 64*19(%r8),@Pi0[4]
vmovdqu64 40*0-96($A_flat),${A00}{$k11111}{z}
vmovdqu64 40*1-96($A_flat),${A10}{$k11111}{z}
vmovdqu64 40*2-96($A_flat),${A20}{$k11111}{z}
vmovdqu64 40*3-96($A_flat),${A30}{$k11111}{z}
vmovdqu64 40*4-96($A_flat),${A40}{$k11111}{z}
.Lno_output_extension_avx512:
shr \$3,$bsz
lea -96($A_flat),%r9
mov $bsz,%rax
jmp .Loop_squeeze_avx512
.align 32
.Loop_squeeze_avx512:
cmp \$8,$len
jb .Ltail_squeeze_avx512
mov (%r9),%r8
lea 8(%r9),%r9
mov %r8,($out)
lea 8($out),$out
sub \$8,$len # len -= 8
jz .Ldone_squeeze_avx512
sub \$1,%rax # bsz--
jnz .Loop_squeeze_avx512
#vpermq @Theta[4],@Theta[4],@Theta[3]
#vpermq @Theta[3],@Theta[4],@Theta[2]
#vpermq @Theta[3],@Theta[3],@Theta[1]
call __KeccakF1600
vmovdqu64 $A00,40*0-96($A_flat){$k11111}
vmovdqu64 $A10,40*1-96($A_flat){$k11111}
vmovdqu64 $A20,40*2-96($A_flat){$k11111}
vmovdqu64 $A30,40*3-96($A_flat){$k11111}
vmovdqu64 $A40,40*4-96($A_flat){$k11111}
lea -96($A_flat),%r9
mov $bsz,%rax
jmp .Loop_squeeze_avx512
.Ltail_squeeze_avx512:
mov $out,%rdi
mov %r9,%rsi
mov $len,%rcx
.byte 0xf3,0xa4 # rep movsb
.Ldone_squeeze_avx512:
vzeroupper
lea (%r11),%rsp
ret
.size SHA3_squeeze,.-SHA3_squeeze
.align 64
theta_perm:
.quad 0, 1, 2, 3, 4, 5, 6, 7 # [not used]
.quad 4, 0, 1, 2, 3, 5, 6, 7
.quad 3, 4, 0, 1, 2, 5, 6, 7
.quad 2, 3, 4, 0, 1, 5, 6, 7
.quad 1, 2, 3, 4, 0, 5, 6, 7
rhotates1:
.quad 0, 44, 43, 21, 14, 0, 0, 0 # [0][0] [1][1] [2][2] [3][3] [4][4]
.quad 18, 1, 6, 25, 8, 0, 0, 0 # [4][0] [0][1] [1][2] [2][3] [3][4]
.quad 41, 2, 62, 55, 39, 0, 0, 0 # [3][0] [4][1] [0][2] [1][3] [2][4]
.quad 3, 45, 61, 28, 20, 0, 0, 0 # [2][0] [3][1] [4][2] [0][3] [1][4]
.quad 36, 10, 15, 56, 27, 0, 0, 0 # [1][0] [2][1] [3][2] [4][3] [0][4]
rhotates0:
.quad 0, 1, 62, 28, 27, 0, 0, 0
.quad 36, 44, 6, 55, 20, 0, 0, 0
.quad 3, 10, 43, 25, 39, 0, 0, 0
.quad 41, 45, 15, 21, 8, 0, 0, 0
.quad 18, 2, 61, 56, 14, 0, 0, 0
pi0_perm:
.quad 0, 3, 1, 4, 2, 5, 6, 7
.quad 1, 4, 2, 0, 3, 5, 6, 7
.quad 2, 0, 3, 1, 4, 5, 6, 7
.quad 3, 1, 4, 2, 0, 5, 6, 7
.quad 4, 2, 0, 3, 1, 5, 6, 7
iotas:
.quad 0x0000000000000001
.quad 0x0000000000008082
.quad 0x800000000000808a
.quad 0x8000000080008000
.quad 0x000000000000808b
.quad 0x0000000080000001
.quad 0x8000000080008081
.quad 0x8000000000008009
.quad 0x000000000000008a
.quad 0x0000000000000088
.quad 0x0000000080008009
.quad 0x000000008000000a
.quad 0x000000008000808b
.quad 0x800000000000008b
.quad 0x8000000000008089
.quad 0x8000000000008003
.quad 0x8000000000008002
.quad 0x8000000000000080
.quad 0x000000000000800a
.quad 0x800000008000000a
.quad 0x8000000080008081
.quad 0x8000000000008080
.quad 0x0000000080000001
.quad 0x8000000080008008
.asciz "Keccak-1600 absorb and squeeze for AVX-512F, CRYPTOGAMS by <appro\@openssl.org>"
___
print $code;
close STDOUT;
crypto/sha/asm/keccak1600-avx512vl.pl
Executable
+390
View File
@@ -0,0 +1,390 @@
#!/usr/bin/env perl
# Copyright 2017 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License"). You may not use
# this file except in compliance with the License. You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html
#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# Keccak-1600 for AVX512VL.
#
# December 2017.
#
# This is an adaptation of AVX2 module that reuses register data
# layout, but utilizes new 256-bit AVX512VL instructions. See AVX2
# module for further information on layout.
#
########################################################################
# Numbers are cycles per processed byte out of large message.
#
# r=1088(*)
#
# Skylake-X 6.4/+47%
#
# (*) Corresponds to SHA3-256. Percentage after slash is improvement
# coefficient in comparison to scalar keccak1600-x86_64.pl.
# Digits in variables' names denote right-most coordinates:
my ($A00, # [0][0] [0][0] [0][0] [0][0] # %ymm0
$A01, # [0][4] [0][3] [0][2] [0][1] # %ymm1
$A20, # [3][0] [1][0] [4][0] [2][0] # %ymm2
$A31, # [2][4] [4][3] [1][2] [3][1] # %ymm3
$A21, # [3][4] [1][3] [4][2] [2][1] # %ymm4
$A41, # [1][4] [2][3] [3][2] [4][1] # %ymm5
$A11) = # [4][4] [3][3] [2][2] [1][1] # %ymm6
map("%ymm$_",(0..6));
# We also need to map the magic order into offsets within structure:
my @A_jagged = ([0,0], [1,0], [1,1], [1,2], [1,3], # [0][0..4]
[2,2], [6,0], [3,1], [4,2], [5,3], # [1][0..4]
[2,0], [4,0], [6,1], [5,2], [3,3], # [2][0..4]
[2,3], [3,0], [5,1], [6,2], [4,3], # [3][0..4]
[2,1], [5,0], [4,1], [3,2], [6,3]); # [4][0..4]
@A_jagged = map(8*($$_[0]*4+$$_[1]), @A_jagged); # ... and now linear
my @T = map("%ymm$_",(7..15));
my ($C14,$C00,$D00,$D14) = @T[5..8];
my ($R20,$R01,$R31,$R21,$R41,$R11) = map("%ymm$_",(16..21));
$code.=<<___;
.text
.type __KeccakF1600,\@function
.align 32
__KeccakF1600:
lea iotas(%rip),%r10
mov \$24,%eax
jmp .Loop_avx512vl
.align 32
.Loop_avx512vl:
######################################### Theta
vpshufd \$0b01001110,$A20,$C00
vpxor $A31,$A41,$C14
vpxor $A11,$A21,@T[2]
vpternlogq \$0x96,$A01,$T[2],$C14 # C[1..4]
vpxor $A20,$C00,$C00
vpermq \$0b01001110,$C00,@T[0]
vpermq \$0b10010011,$C14,@T[4]
vprolq \$1,$C14,@T[1] # ROL64(C[1..4],1)
vpermq \$0b00111001,@T[1],$D14
vpxor @T[4],@T[1],$D00
vpermq \$0b00000000,$D00,$D00 # D[0..0] = ROL64(C[1],1) ^ C[4]
vpternlogq \$0x96,@T[0],$A00,$C00 # C[0..0]
vprolq \$1,$C00,@T[1] # ROL64(C[0..0],1)
vpxor $D00,$A00,$A00 # ^= D[0..0]
vpblendd \$0b11000000,@T[1],$D14,$D14
vpblendd \$0b00000011,$C00,@T[4],@T[0]
######################################### Rho + Pi + pre-Chi shuffle
vpxor $D00,$A20,$A20 # ^= D[0..0] from Theta
vprolvq $R20,$A20,$A20
vpternlogq \$0x96,@T[0],$D14,$A31 # ^= D[1..4] from Theta
vprolvq $R31,$A31,$A31
vpternlogq \$0x96,@T[0],$D14,$A21 # ^= D[1..4] from Theta
vprolvq $R21,$A21,$A21
vpternlogq \$0x96,@T[0],$D14,$A41 # ^= D[1..4] from Theta
vprolvq $R41,$A41,$A41
vpermq \$0b10001101,$A20,@T[3] # $A20 -> future $A31
vpermq \$0b10001101,$A31,@T[4] # $A31 -> future $A21
vpternlogq \$0x96,@T[0],$D14,$A11 # ^= D[1..4] from Theta
vprolvq $R11,$A11,@T[1] # $A11 -> future $A01
vpermq \$0b00011011,$A21,@T[5] # $A21 -> future $A41
vpermq \$0b01110010,$A41,@T[6] # $A41 -> future $A11
vpternlogq \$0x96,@T[0],$D14,$A01 # ^= D[1..4] from Theta
vprolvq $R01,$A01,@T[2] # $A01 -> future $A20
######################################### Chi
vpblendd \$0b00001100,@T[6],@T[2],$A31 # [4][4] [2][0]
vpblendd \$0b00001100,@T[2],@T[4],@T[8] # [4][0] [2][1]
vpblendd \$0b00001100,@T[4],@T[3],$A41 # [4][2] [2][4]
vpblendd \$0b00001100,@T[3],@T[2],@T[7] # [4][3] [2][0]
vpblendd \$0b00110000,@T[4],$A31,$A31 # [1][3] [4][4] [2][0]
vpblendd \$0b00110000,@T[5],@T[8],@T[8] # [1][4] [4][0] [2][1]
vpblendd \$0b00110000,@T[2],$A41,$A41 # [1][0] [4][2] [2][4]
vpblendd \$0b00110000,@T[6],@T[7],@T[7] # [1][1] [4][3] [2][0]
vpblendd \$0b11000000,@T[5],$A31,$A31 # [3][2] [1][3] [4][4] [2][0]
vpblendd \$0b11000000,@T[6],@T[8],@T[8] # [3][3] [1][4] [4][0] [2][1]
vpblendd \$0b11000000,@T[6],$A41,$A41 # [3][3] [1][0] [4][2] [2][4]
vpblendd \$0b11000000,@T[4],@T[7],@T[7] # [3][4] [1][1] [4][3] [2][0]
vpternlogq \$0xC6,@T[8],@T[3],$A31 # [3][1] [1][2] [4][3] [2][4]
vpternlogq \$0xC6,@T[7],@T[5],$A41 # [3][2] [1][4] [4][1] [2][3]
vpsrldq \$8,@T[1],@T[0]
vpandn @T[0],@T[1],@T[0] # tgting [0][0] [0][0] [0][0] [0][0]
vpblendd \$0b00001100,@T[2],@T[5],$A11 # [4][0] [2][3]
vpblendd \$0b00001100,@T[5],@T[3],@T[8] # [4][1] [2][4]
vpblendd \$0b00110000,@T[3],$A11,$A11 # [1][2] [4][0] [2][3]
vpblendd \$0b00110000,@T[4],@T[8],@T[8] # [1][3] [4][1] [2][4]
vpblendd \$0b11000000,@T[4],$A11,$A11 # [3][4] [1][2] [4][0] [2][3]
vpblendd \$0b11000000,@T[2],@T[8],@T[8] # [3][0] [1][3] [4][1] [2][4]
vpternlogq \$0xC6,@T[8],@T[6],$A11 # [3][3] [1][1] [4][4] [2][2]
vpermq \$0b00011110,@T[1],$A21 # [0][1] [0][2] [0][4] [0][3]
vpblendd \$0b00110000,$A00,$A21,@T[8] # [0][1] [0][0] [0][4] [0][3]
vpermq \$0b00111001,@T[1],$A01 # [0][1] [0][4] [0][3] [0][2]
vpblendd \$0b11000000,$A00,$A01,$A01 # [0][0] [0][4] [0][3] [0][2]
vpblendd \$0b00001100,@T[5],@T[4],$A20 # [4][1] [2][1]
vpblendd \$0b00001100,@T[4],@T[6],@T[7] # [4][2] [2][2]
vpblendd \$0b00110000,@T[6],$A20,$A20 # [1][1] [4][1] [2][1]
vpblendd \$0b00110000,@T[3],@T[7],@T[7] # [1][2] [4][2] [2][2]
vpblendd \$0b11000000,@T[3],$A20,$A20 # [3][1] [1][1] [4][1] [2][1]
vpblendd \$0b11000000,@T[5],@T[7],@T[7] # [3][2] [1][2] [4][2] [2][2]
vpternlogq \$0xC6,@T[7],@T[2],$A20 # [3][0] [1][0] [4][0] [2][0]
vpermq \$0b00000000,@T[0],@T[0] # [0][0] [0][0] [0][0] [0][0]
vpermq \$0b00011011,$A31,$A31 # post-Chi shuffle
vpermq \$0b10001101,$A41,$A41
vpermq \$0b01110010,$A11,$A11
vpblendd \$0b00001100,@T[3],@T[6],$A21 # [4][3] [2][2]
vpblendd \$0b00001100,@T[6],@T[5],@T[7] # [4][4] [2][3]
vpblendd \$0b00110000,@T[5],$A21,$A21 # [1][4] [4][3] [2][2]
vpblendd \$0b00110000,@T[2],@T[7],@T[7] # [1][0] [4][4] [2][3]
vpblendd \$0b11000000,@T[2],$A21,$A21 # [3][0] [1][4] [4][3] [2][2]
vpblendd \$0b11000000,@T[3],@T[7],@T[7] # [3][1] [1][0] [4][4] [2][3]
vpternlogq \$0xC6,@T[8],@T[1],$A01 # [0][4] [0][3] [0][2] [0][1]
vpternlogq \$0xC6,@T[7],@T[4],$A21 # [3][4] [1][3] [4][2] [2][1]
######################################### Iota
vpternlogq \$0x96,(%r10),@T[0],$A00
lea 32(%r10),%r10
dec %eax
jnz .Loop_avx512vl
ret
.size __KeccakF1600,.-__KeccakF1600
___
my ($A_flat,$inp,$len,$bsz) = ("%rdi","%rsi","%rdx","%rcx");
my $out = $inp; # in squeeze
$code.=<<___;
.globl SHA3_absorb
.type SHA3_absorb,\@function
.align 32
SHA3_absorb:
mov %rsp,%r11
lea -240(%rsp),%rsp
and \$-32,%rsp
lea 96($A_flat),$A_flat
lea 96($inp),$inp
lea 96(%rsp),%r10
lea rhotates_left(%rip),%r8
vzeroupper
vpbroadcastq -96($A_flat),$A00 # load A[5][5]
vmovdqu 8+32*0-96($A_flat),$A01
vmovdqu 8+32*1-96($A_flat),$A20
vmovdqu 8+32*2-96($A_flat),$A31
vmovdqu 8+32*3-96($A_flat),$A21
vmovdqu 8+32*4-96($A_flat),$A41
vmovdqu 8+32*5-96($A_flat),$A11
vmovdqa64 0*32(%r8),$R20 # load "rhotate" indices
vmovdqa64 1*32(%r8),$R01
vmovdqa64 2*32(%r8),$R31
vmovdqa64 3*32(%r8),$R21
vmovdqa64 4*32(%r8),$R41
vmovdqa64 5*32(%r8),$R11
vpxor @T[0],@T[0],@T[0]
vmovdqa @T[0],32*2-96(%r10) # zero transfer area on stack
vmovdqa @T[0],32*3-96(%r10)
vmovdqa @T[0],32*4-96(%r10)
vmovdqa @T[0],32*5-96(%r10)
vmovdqa @T[0],32*6-96(%r10)
.Loop_absorb_avx512vl:
mov $bsz,%rax
sub $bsz,$len
jc .Ldone_absorb_avx512vl
shr \$3,%eax
vpbroadcastq 0-96($inp),@T[0]
vmovdqu 8-96($inp),@T[1]
sub \$4,%eax
___
for(my $i=5; $i<25; $i++) {
$code.=<<___
dec %eax
jz .Labsorved_avx512vl
mov 8*$i-96($inp),%r8
mov %r8,$A_jagged[$i]-96(%r10)
___
}
$code.=<<___;
.Labsorved_avx512vl:
lea ($inp,$bsz),$inp
vpxor @T[0],$A00,$A00
vpxor @T[1],$A01,$A01
vpxor 32*2-96(%r10),$A20,$A20
vpxor 32*3-96(%r10),$A31,$A31
vpxor 32*4-96(%r10),$A21,$A21
vpxor 32*5-96(%r10),$A41,$A41
vpxor 32*6-96(%r10),$A11,$A11
call __KeccakF1600
lea 96(%rsp),%r10
jmp .Loop_absorb_avx512vl
.Ldone_absorb_avx512vl:
vmovq %xmm0,-96($A_flat)
vmovdqu $A01,8+32*0-96($A_flat)
vmovdqu $A20,8+32*1-96($A_flat)
vmovdqu $A31,8+32*2-96($A_flat)
vmovdqu $A21,8+32*3-96($A_flat)
vmovdqu $A41,8+32*4-96($A_flat)
vmovdqu $A11,8+32*5-96($A_flat)
vzeroupper
lea (%r11),%rsp
lea ($len,$bsz),%rax # return value
ret
.size SHA3_absorb,.-SHA3_absorb
.globl SHA3_squeeze
.type SHA3_squeeze,\@function
.align 32
SHA3_squeeze:
mov %rsp,%r11
lea 96($A_flat),$A_flat
lea rhotates_left(%rip),%r8
shr \$3,$bsz
vzeroupper
vpbroadcastq -96($A_flat),$A00
vpxor @T[0],@T[0],@T[0]
vmovdqu 8+32*0-96($A_flat),$A01
vmovdqu 8+32*1-96($A_flat),$A20
vmovdqu 8+32*2-96($A_flat),$A31
vmovdqu 8+32*3-96($A_flat),$A21
vmovdqu 8+32*4-96($A_flat),$A41
vmovdqu 8+32*5-96($A_flat),$A11
vmovdqa64 0*32(%r8),$R20 # load "rhotate" indices
vmovdqa64 1*32(%r8),$R01
vmovdqa64 2*32(%r8),$R31
vmovdqa64 3*32(%r8),$R21
vmovdqa64 4*32(%r8),$R41
vmovdqa64 5*32(%r8),$R11
mov $bsz,%rax
.Loop_squeeze_avx512vl:
mov @A_jagged[$i]-96($A_flat),%r8
___
for (my $i=0; $i<25; $i++) {
$code.=<<___;
sub \$8,$len
jc .Ltail_squeeze_avx512vl
mov %r8,($out)
lea 8($out),$out
je .Ldone_squeeze_avx512vl
dec %eax
je .Lextend_output_avx512vl
mov @A_jagged[$i+1]-120($A_flat),%r8
___
}
$code.=<<___;
.Lextend_output_avx512vl:
call __KeccakF1600
vmovq %xmm0,-96($A_flat)
vmovdqu $A01,8+32*0-96($A_flat)
vmovdqu $A20,8+32*1-96($A_flat)
vmovdqu $A31,8+32*2-96($A_flat)
vmovdqu $A21,8+32*3-96($A_flat)
vmovdqu $A41,8+32*4-96($A_flat)
vmovdqu $A11,8+32*5-96($A_flat)
mov $bsz,%rax
jmp .Loop_squeeze_avx512vl
.Ltail_squeeze_avx512vl:
add \$8,$len
.Loop_tail_avx512vl:
mov %r8b,($out)
lea 1($out),$out
shr \$8,%r8
dec $len
jnz .Loop_tail_avx512vl
.Ldone_squeeze_avx512vl:
vzeroupper
lea (%r11),%rsp
ret
.size SHA3_squeeze,.-SHA3_squeeze
.align 64
rhotates_left:
.quad 3, 18, 36, 41 # [2][0] [4][0] [1][0] [3][0]
.quad 1, 62, 28, 27 # [0][1] [0][2] [0][3] [0][4]
.quad 45, 6, 56, 39 # [3][1] [1][2] [4][3] [2][4]
.quad 10, 61, 55, 8 # [2][1] [4][2] [1][3] [3][4]
.quad 2, 15, 25, 20 # [4][1] [3][2] [2][3] [1][4]
.quad 44, 43, 21, 14 # [1][1] [2][2] [3][3] [4][4]
iotas:
.quad 0x0000000000000001, 0x0000000000000001, 0x0000000000000001, 0x0000000000000001
.quad 0x0000000000008082, 0x0000000000008082, 0x0000000000008082, 0x0000000000008082
.quad 0x800000000000808a, 0x800000000000808a, 0x800000000000808a, 0x800000000000808a
.quad 0x8000000080008000, 0x8000000080008000, 0x8000000080008000, 0x8000000080008000
.quad 0x000000000000808b, 0x000000000000808b, 0x000000000000808b, 0x000000000000808b
.quad 0x0000000080000001, 0x0000000080000001, 0x0000000080000001, 0x0000000080000001
.quad 0x8000000080008081, 0x8000000080008081, 0x8000000080008081, 0x8000000080008081
.quad 0x8000000000008009, 0x8000000000008009, 0x8000000000008009, 0x8000000000008009
.quad 0x000000000000008a, 0x000000000000008a, 0x000000000000008a, 0x000000000000008a
.quad 0x0000000000000088, 0x0000000000000088, 0x0000000000000088, 0x0000000000000088
.quad 0x0000000080008009, 0x0000000080008009, 0x0000000080008009, 0x0000000080008009
.quad 0x000000008000000a, 0x000000008000000a, 0x000000008000000a, 0x000000008000000a
.quad 0x000000008000808b, 0x000000008000808b, 0x000000008000808b, 0x000000008000808b
.quad 0x800000000000008b, 0x800000000000008b, 0x800000000000008b, 0x800000000000008b
.quad 0x8000000000008089, 0x8000000000008089, 0x8000000000008089, 0x8000000000008089
.quad 0x8000000000008003, 0x8000000000008003, 0x8000000000008003, 0x8000000000008003
.quad 0x8000000000008002, 0x8000000000008002, 0x8000000000008002, 0x8000000000008002
.quad 0x8000000000000080, 0x8000000000000080, 0x8000000000000080, 0x8000000000000080
.quad 0x000000000000800a, 0x000000000000800a, 0x000000000000800a, 0x000000000000800a
.quad 0x800000008000000a, 0x800000008000000a, 0x800000008000000a, 0x800000008000000a
.quad 0x8000000080008081, 0x8000000080008081, 0x8000000080008081, 0x8000000080008081
.quad 0x8000000000008080, 0x8000000000008080, 0x8000000000008080, 0x8000000000008080
.quad 0x0000000080000001, 0x0000000080000001, 0x0000000080000001, 0x0000000080000001
.quad 0x8000000080008008, 0x8000000080008008, 0x8000000080008008, 0x8000000080008008
.asciz "Keccak-1600 absorb and squeeze for AVX512VL, CRYPTOGAMS by <appro\@openssl.org>"
___
print $code;
close STDOUT;
crypto/sha/asm/keccak1600-c64x.pl
Executable
+882
View File
@@ -0,0 +1,882 @@
#!/usr/bin/env perl
# Copyright 2017 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License"). You may not use
# this file except in compliance with the License. You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html
#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# [ABI- and endian-neutral] Keccak-1600 for C64x.
#
# June 2017.
#
# This is straightforward KECCAK_1X_ALT variant (see sha/keccak1600.c)
# with bit interleaving. 64-bit values are simply split between A- and
# B-files, with A-file holding least significant halves. This works
# out perfectly, because all operations including cross-communications
# [in rotate operations] are always complementary. Performance is
# [incredible for a 32-bit processor] 10.9 cycles per processed byte
# for r=1088, which corresponds to SHA3-256. This is >15x faster than
# compiler-generated KECCAK_1X_ALT code, and >10x than other variants.
# On average processor ends up issuing ~4.5 instructions per cycle...
my @A = map([ $_, ($_+1), ($_+2), ($_+3), ($_+4) ], (5,10,16,21,26));
$A[1][4] = 31; # B14 is reserved, A14 is used as iota[]
($A[3][0],$A[4][1]) = ($A[4][1],$A[3][0]);
my @C = (0..4,$A[3][0],$A[4][0]);
my $iotas = "A14";
my @rhotates = ([ 0, 1, 62, 28, 27 ],
[ 36, 44, 6, 55, 20 ],
[ 3, 10, 43, 25, 39 ],
[ 41, 45, 15, 21, 8 ],
[ 18, 2, 61, 56, 14 ]);
sub ROL64 {
my ($src,$rot,$dst,$p) = @_;
if ($rot&1) {
$code.=<<___;
$p ROTL B$src,$rot/2+1,A$dst
|| ROTL A$src,$rot/2, B$dst
___
} else {
$code.=<<___;
$p ROTL A$src,$rot/2,A$dst
|| ROTL B$src,$rot/2,B$dst
___
}
}
########################################################################
# Stack frame layout
#
# SP--->+------+------+
# | | |
# +1--->+------+------+<- -9 below 4 slots are used by KeccakF1600_int
# | | |
# +2--->+------+------+<- -8
# | | |
# +3--->+------+------+<- -7
# | A2 | A3 | A3:A2 are preserved by KeccakF1600_int
# +4--->+------+------+<- -6
# | B2 | B3 | B3:B2 are preserved by KeccakF1600_int
# +5--->+------+------+<- -5 below is ABI-compliant layout
# | A10 | A11 |
# +6--->+------+------+<- -4
# | A12 | A13 |
# +7--->+------+------+<- -3
# | A14 | B3 |
# +8--->+------+------+<- -2
# | B10 | B11 |
# +9--->+------+------+<- -1
# | B12 | B13 |
# +------+------+<---FP
# | A15 |
# +------+--
$code.=<<___;
.text
.if .ASSEMBLER_VERSION<7000000
.asg 0,__TI_EABI__
.endif
.if __TI_EABI__
.nocmp
.asg KeccakF1600,_KeccakF1600
.asg SHA3_absorb,_SHA3_absorb
.asg SHA3_squeeze,_SHA3_squeeze
.endif
.asg B3,RA
.asg A15,FP
.asg B15,SP
.align 32
_KeccakF1600_int:
.asmfunc
STDW A3:A2,*FP[-7]
|| STDW B3:B2,*SP[4]
_KeccakF1600_cheat:
.if __TI_EABI__
ADDKPC _KeccakF1600_int,B0
|| MVKL \$PCR_OFFSET(iotas,_KeccakF1600_int),$iotas
MVKH \$PCR_OFFSET(iotas,_KeccakF1600_int),$iotas
.else
ADDKPC _KeccakF1600_int,B0
|| MVKL (iotas-_KeccakF1600_int),$iotas
MVKH (iotas-_KeccakF1600_int),$iotas
.endif
ADD B0,$iotas,$iotas
loop?:
XOR A$A[0][2],A$A[1][2],A$C[2] ; Theta
|| XOR B$A[0][2],B$A[1][2],B$C[2]
|| XOR A$A[0][3],A$A[1][3],A$C[3]
|| XOR B$A[0][3],B$A[1][3],B$C[3]
|| XOR A$A[0][0],A$A[1][0],A$C[0]
|| XOR B$A[0][0],B$A[1][0],B$C[0]
XOR A$A[2][2],A$C[2],A$C[2]
|| XOR B$A[2][2],B$C[2],B$C[2]
|| XOR A$A[2][3],A$C[3],A$C[3]
|| XOR B$A[2][3],B$C[3],B$C[3]
|| XOR A$A[2][0],A$C[0],A$C[0]
|| XOR B$A[2][0],B$C[0],B$C[0]
XOR A$A[3][2],A$C[2],A$C[2]
|| XOR B$A[3][2],B$C[2],B$C[2]
|| XOR A$A[3][3],A$C[3],A$C[3]
|| XOR B$A[3][3],B$C[3],B$C[3]
|| XOR A$A[3][0],A$C[0],A$C[0]
|| XOR B$A[3][0],B$C[0],B$C[0]
XOR A$A[4][2],A$C[2],A$C[2]
|| XOR B$A[4][2],B$C[2],B$C[2]
|| XOR A$A[4][3],A$C[3],A$C[3]
|| XOR B$A[4][3],B$C[3],B$C[3]
|| XOR A$A[4][0],A$C[0],A$C[0]
|| XOR B$A[4][0],B$C[0],B$C[0]
XOR A$A[0][4],A$A[1][4],A$C[4]
|| XOR B$A[0][4],B$A[1][4],B$C[4]
|| XOR A$A[0][1],A$A[1][1],A$C[1]
|| XOR B$A[0][1],B$A[1][1],B$C[1]
|| STDW A$A[3][0]:A$A[4][0],*SP[1] ; offload some data
STDW B$A[3][0]:B$A[4][0],*SP[2]
|| XOR A$A[2][4],A$C[4],A$C[4]
|| XOR B$A[2][4],B$C[4],B$C[4]
|| XOR A$A[2][1],A$C[1],A$C[1]
|| XOR B$A[2][1],B$C[1],B$C[1]
|| ROTL B$C[2],1,A$C[5] ; ROL64(C[2],1)
|| ROTL A$C[2],0,B$C[5]
XOR A$A[3][4],A$C[4],A$C[4]
|| XOR B$A[3][4],B$C[4],B$C[4]
|| XOR A$A[3][1],A$C[1],A$C[1]
|| XOR B$A[3][1],B$C[1],B$C[1]
|| ROTL B$C[3],1,A$C[6] ; ROL64(C[3],1)
|| ROTL A$C[3],0,B$C[6]
XOR A$A[4][4],A$C[4],A$C[4]
|| XOR B$A[4][4],B$C[4],B$C[4]
|| XOR A$A[4][1],A$C[1],A$C[1]
|| XOR B$A[4][1],B$C[1],B$C[1]
|| XOR A$C[0],A$C[5],A$C[5] ; C[0] ^ ROL64(C[2],1)
|| XOR B$C[0],B$C[5],B$C[5]
XOR A$C[5],A$A[0][1],A$A[0][1]
|| XOR B$C[5],B$A[0][1],B$A[0][1]
|| XOR A$C[5],A$A[1][1],A$A[1][1]
|| XOR B$C[5],B$A[1][1],B$A[1][1]
|| XOR A$C[5],A$A[2][1],A$A[2][1]
|| XOR B$C[5],B$A[2][1],B$A[2][1]
XOR A$C[5],A$A[3][1],A$A[3][1]
|| XOR B$C[5],B$A[3][1],B$A[3][1]
|| XOR A$C[5],A$A[4][1],A$A[4][1]
|| XOR B$C[5],B$A[4][1],B$A[4][1]
|| ROTL B$C[4],1,A$C[5] ; ROL64(C[4],1)
|| ROTL A$C[4],0,B$C[5]
|| XOR A$C[1],A$C[6],A$C[6] ; C[1] ^ ROL64(C[3],1)
|| XOR B$C[1],B$C[6],B$C[6]
XOR A$C[6],A$A[0][2],A$A[0][2]
|| XOR B$C[6],B$A[0][2],B$A[0][2]
|| XOR A$C[6],A$A[1][2],A$A[1][2]
|| XOR B$C[6],B$A[1][2],B$A[1][2]
|| XOR A$C[6],A$A[2][2],A$A[2][2]
|| XOR B$C[6],B$A[2][2],B$A[2][2]
|| ROTL B$C[1],1,A$C[1] ; ROL64(C[1],1)
|| ROTL A$C[1],0,B$C[1]
XOR A$C[6],A$A[3][2],A$A[3][2]
|| XOR B$C[6],B$A[3][2],B$A[3][2]
|| XOR A$C[6],A$A[4][2],A$A[4][2]
|| XOR B$C[6],B$A[4][2],B$A[4][2]
|| ROTL B$C[0],1,A$C[6] ; ROL64(C[0],1)
|| ROTL A$C[0],0,B$C[6]
|| XOR A$C[5],A$C[2],A$C[2] ; C[2] ^= ROL64(C[4],1)
|| XOR B$C[5],B$C[2],B$C[2]
XOR A$C[2],A$A[0][3],A$A[0][3]
|| XOR B$C[2],B$A[0][3],B$A[0][3]
|| XOR A$C[2],A$A[1][3],A$A[1][3]
|| XOR B$C[2],B$A[1][3],B$A[1][3]
|| XOR A$C[2],A$A[2][3],A$A[2][3]
|| XOR B$C[2],B$A[2][3],B$A[2][3]
XOR A$C[6],A$C[3],A$C[3] ; C[3] ^= ROL64(C[0],1)
|| XOR B$C[6],B$C[3],B$C[3]
|| LDDW *FP[-9],A$A[3][0]:A$A[4][0] ; restore offloaded data
|| LDDW *SP[2],B$A[3][0]:B$A[4][0]
|| XOR A$C[2],A$A[3][3],A$A[3][3]
|| XOR B$C[2],B$A[3][3],B$A[3][3]
XOR A$C[2],A$A[4][3],A$A[4][3]
|| XOR B$C[2],B$A[4][3],B$A[4][3]
|| XOR A$C[3],A$A[0][4],A$A[0][4]
|| XOR B$C[3],B$A[0][4],B$A[0][4]
|| XOR A$C[3],A$A[1][4],A$A[1][4]
|| XOR B$C[3],B$A[1][4],B$A[1][4]
XOR A$C[3],A$A[2][4],A$A[2][4]
|| XOR B$C[3],B$A[2][4],B$A[2][4]
|| XOR A$C[3],A$A[3][4],A$A[3][4]
|| XOR B$C[3],B$A[3][4],B$A[3][4]
|| XOR A$C[3],A$A[4][4],A$A[4][4]
|| XOR B$C[3],B$A[4][4],B$A[4][4]
XOR A$C[1],A$C[4],A$C[4] ; C[4] ^= ROL64(C[1],1)
|| XOR B$C[1],B$C[4],B$C[4]
|| MV A$A[0][1],A$C[1] ; Rho+Pi, "early start"
|| MV B$A[0][1],B$C[1]
___
&ROL64 ($A[1][1],$rhotates[1][1],$A[0][1],"||");
$code.=<<___;
XOR A$C[4],A$A[0][0],A$A[0][0]
|| XOR B$C[4],B$A[0][0],B$A[0][0]
|| XOR A$C[4],A$A[1][0],A$A[1][0]
|| XOR B$C[4],B$A[1][0],B$A[1][0]
|| MV A$A[0][3],A$C[3]
|| MV B$A[0][3],B$C[3]
___
&ROL64 ($A[3][3],$rhotates[3][3],$A[0][3],"||");
$code.=<<___;
XOR A$C[4],A$A[2][0],A$A[2][0]
|| XOR B$C[4],B$A[2][0],B$A[2][0]
|| XOR A$C[4],A$A[3][0],A$A[3][0]
|| XOR B$C[4],B$A[3][0],B$A[3][0]
|| MV A$A[0][2],A$C[2]
|| MV B$A[0][2],B$C[2]
___
&ROL64 ($A[2][2],$rhotates[2][2],$A[0][2],"||");
$code.=<<___;
XOR A$C[4],A$A[4][0],A$A[4][0]
|| XOR B$C[4],B$A[4][0],B$A[4][0]
|| MV A$A[0][4],A$C[4]
|| MV B$A[0][4],B$C[4]
___
&ROL64 ($A[4][4],$rhotates[4][4],$A[0][4],"||");
&ROL64 ($A[1][4],$rhotates[1][4],$A[1][1]);
$code.=<<___;
|| LDW *${iotas}++[2],A$C[0]
___
&ROL64 ($A[2][3],$rhotates[2][3],$A[2][2]);
$code.=<<___;
|| LDW *${iotas}[-1],B$C[0]
___
&ROL64 ($A[3][2],$rhotates[3][2],$A[3][3]);
&ROL64 ($A[4][1],$rhotates[4][1],$A[4][4]);
&ROL64 ($A[4][2],$rhotates[4][2],$A[1][4]);
&ROL64 ($A[3][4],$rhotates[3][4],$A[2][3]);
&ROL64 ($A[2][1],$rhotates[2][1],$A[3][2]);
&ROL64 ($A[1][3],$rhotates[1][3],$A[4][1]);
&ROL64 ($A[2][4],$rhotates[2][4],$A[4][2]);
&ROL64 ($A[4][3],$rhotates[4][3],$A[3][4]);
&ROL64 ($A[1][2],$rhotates[1][2],$A[2][1]);
&ROL64 ($A[3][1],$rhotates[3][1],$A[1][3]);
&ROL64 ($A[4][0],$rhotates[4][0],$A[2][4]);
&ROL64 ($A[3][0],$rhotates[3][0],$A[4][3]);
&ROL64 ($A[2][0],$rhotates[2][0],$A[1][2]);
&ROL64 ($A[1][0],$rhotates[1][0],$A[3][1]);
#&ROL64 ($C[3], $rhotates[0][3],$A[1][0]); # moved below
&ROL64 ($C[1], $rhotates[0][1],$A[2][0]);
&ROL64 ($C[4], $rhotates[0][4],$A[3][0]);
&ROL64 ($C[2], $rhotates[0][2],$A[4][0]);
$code.=<<___;
|| ANDN A$A[0][2],A$A[0][1],A$C[4] ; Chi+Iota
|| ANDN B$A[0][2],B$A[0][1],B$C[4]
|| ANDN A$A[0][3],A$A[0][2],A$C[1]
|| ANDN B$A[0][3],B$A[0][2],B$C[1]
|| ANDN A$A[0][4],A$A[0][3],A$C[2]
|| ANDN B$A[0][4],B$A[0][3],B$C[2]
___
&ROL64 ($C[3], $rhotates[0][3],$A[1][0]);
$code.=<<___;
|| ANDN A$A[0][0],A$A[0][4],A$C[3]
|| ANDN B$A[0][0],B$A[0][4],B$C[3]
|| XOR A$C[4],A$A[0][0],A$A[0][0]
|| XOR B$C[4],B$A[0][0],B$A[0][0]
|| ANDN A$A[0][1],A$A[0][0],A$C[4]
|| ANDN B$A[0][1],B$A[0][0],B$C[4]
XOR A$C[1],A$A[0][1],A$A[0][1]
|| XOR B$C[1],B$A[0][1],B$A[0][1]
|| XOR A$C[2],A$A[0][2],A$A[0][2]
|| XOR B$C[2],B$A[0][2],B$A[0][2]
|| XOR A$C[3],A$A[0][3],A$A[0][3]
|| XOR B$C[3],B$A[0][3],B$A[0][3]
XOR A$C[4],A$A[0][4],A$A[0][4]
|| XOR B$C[4],B$A[0][4],B$A[0][4]
|| XOR A$C[0],A$A[0][0],A$A[0][0] ; A[0][0] ^= iotas[i++];
|| XOR B$C[0],B$A[0][0],B$A[0][0]
|| EXTU $iotas,24,24,A0 ; A0 is A$C[0], as we done?
ANDN A$A[1][2],A$A[1][1],A$C[4]
|| ANDN B$A[1][2],B$A[1][1],B$C[4]
|| ANDN A$A[1][3],A$A[1][2],A$C[1]
|| ANDN B$A[1][3],B$A[1][2],B$C[1]
|| ANDN A$A[1][4],A$A[1][3],A$C[2]
|| ANDN B$A[1][4],B$A[1][3],B$C[2]
ANDN A$A[1][0],A$A[1][4],A$C[3]
|| ANDN B$A[1][0],B$A[1][4],B$C[3]
|| XOR A$C[4],A$A[1][0],A$A[1][0]
|| XOR B$C[4],B$A[1][0],B$A[1][0]
|| ANDN A$A[1][1],A$A[1][0],A$C[4]
|| ANDN B$A[1][1],B$A[1][0],B$C[4]
XOR A$C[1],A$A[1][1],A$A[1][1]
|| XOR B$C[1],B$A[1][1],B$A[1][1]
|| XOR A$C[2],A$A[1][2],A$A[1][2]
|| XOR B$C[2],B$A[1][2],B$A[1][2]
|| XOR A$C[3],A$A[1][3],A$A[1][3]
|| XOR B$C[3],B$A[1][3],B$A[1][3]
XOR A$C[4],A$A[1][4],A$A[1][4]
|| XOR B$C[4],B$A[1][4],B$A[1][4]
|| ANDN A$A[2][2],A$A[2][1],A$C[4]
|| ANDN B$A[2][2],B$A[2][1],B$C[4]
|| ANDN A$A[2][3],A$A[2][2],A$C[1]
|| ANDN B$A[2][3],B$A[2][2],B$C[1]
ANDN A$A[2][4],A$A[2][3],A$C[2]
|| ANDN B$A[2][4],B$A[2][3],B$C[2]
|| ANDN A$A[2][0],A$A[2][4],A$C[3]
|| ANDN B$A[2][0],B$A[2][4],B$C[3]
|| XOR A$C[4],A$A[2][0],A$A[2][0]
|| XOR B$C[4],B$A[2][0],B$A[2][0]
ANDN A$A[2][1],A$A[2][0],A$C[4]
|| ANDN B$A[2][1],B$A[2][0],B$C[4]
|| XOR A$C[1],A$A[2][1],A$A[2][1]
|| XOR B$C[1],B$A[2][1],B$A[2][1]
|| XOR A$C[2],A$A[2][2],A$A[2][2]
|| XOR B$C[2],B$A[2][2],B$A[2][2]
XOR A$C[3],A$A[2][3],A$A[2][3]
|| XOR B$C[3],B$A[2][3],B$A[2][3]
|| XOR A$C[4],A$A[2][4],A$A[2][4]
|| XOR B$C[4],B$A[2][4],B$A[2][4]
ANDN A$A[3][2],A$A[3][1],A$C[4]
|| ANDN B$A[3][2],B$A[3][1],B$C[4]
|| ANDN A$A[3][3],A$A[3][2],A$C[1]
|| ANDN B$A[3][3],B$A[3][2],B$C[1]
|| ANDN A$A[3][4],A$A[3][3],A$C[2]
|| ANDN B$A[3][4],B$A[3][3],B$C[2]
ANDN A$A[3][0],A$A[3][4],A$C[3]
|| ANDN B$A[3][0],B$A[3][4],B$C[3]
|| XOR A$C[4],A$A[3][0],A$A[3][0]
|| XOR B$C[4],B$A[3][0],B$A[3][0]
|| ANDN A$A[3][1],A$A[3][0],A$C[4]
|| ANDN B$A[3][1],B$A[3][0],B$C[4]
XOR A$C[1],A$A[3][1],A$A[3][1]
|| XOR B$C[1],B$A[3][1],B$A[3][1]
|| XOR A$C[2],A$A[3][2],A$A[3][2]
|| XOR B$C[2],B$A[3][2],B$A[3][2]
|| XOR A$C[3],A$A[3][3],A$A[3][3]
||[A0] BNOP loop?
XOR B$C[3],B$A[3][3],B$A[3][3]
|| XOR A$C[4],A$A[3][4],A$A[3][4]
|| XOR B$C[4],B$A[3][4],B$A[3][4]
||[!A0] LDDW *FP[-7],A3:A2
||[!A0] LDDW *SP[4], RA:B2
ANDN A$A[4][2],A$A[4][1],A$C[4]
|| ANDN B$A[4][2],B$A[4][1],B$C[4]
|| ANDN A$A[4][3],A$A[4][2],A$C[1]
|| ANDN B$A[4][3],B$A[4][2],B$C[1]
|| ANDN A$A[4][4],A$A[4][3],A$C[2]
|| ANDN B$A[4][4],B$A[4][3],B$C[2]
ANDN A$A[4][0],A$A[4][4],A$C[3]
|| ANDN B$A[4][0],B$A[4][4],B$C[3]
|| XOR A$C[4],A$A[4][0],A$A[4][0]
|| XOR B$C[4],B$A[4][0],B$A[4][0]
|| ANDN A$A[4][1],A$A[4][0],A$C[4]
|| ANDN B$A[4][1],B$A[4][0],B$C[4]
XOR A$C[1],A$A[4][1],A$A[4][1]
|| XOR B$C[1],B$A[4][1],B$A[4][1]
|| XOR A$C[2],A$A[4][2],A$A[4][2]
|| XOR B$C[2],B$A[4][2],B$A[4][2]
|| XOR A$C[3],A$A[4][3],A$A[4][3]
|| XOR B$C[3],B$A[4][3],B$A[4][3]
XOR A$C[4],A$A[4][4],A$A[4][4]
|| XOR B$C[4],B$A[4][4],B$A[4][4]
;;===== branch to loop? is taken here
BNOP RA,5
.endasmfunc
.newblock
.global _KeccakF1600
.align 32
_KeccakF1600:
.asmfunc stack_usage(80)
STW FP,*SP--(80) ; save frame pointer
|| MV SP,FP
STDW B13:B12,*SP[9]
|| STDW A13:A12,*FP[-4]
STDW B11:B10,*SP[8]
|| STDW A11:A10,*FP[-5]
STW RA, *SP[15]
|| STW A14,*FP[-6]
|| MV A4,A2
|| ADD 4,A4,B2
LDW *A2++[2],A$A[0][0] ; load A[5][5]
|| LDW *B2++[2],B$A[0][0]
LDW *A2++[2],A$A[0][1]
|| LDW *B2++[2],B$A[0][1]
LDW *A2++[2],A$A[0][2]
|| LDW *B2++[2],B$A[0][2]
LDW *A2++[2],A$A[0][3]
|| LDW *B2++[2],B$A[0][3]
LDW *A2++[2],A$A[0][4]
|| LDW *B2++[2],B$A[0][4]
LDW *A2++[2],A$A[1][0]
|| LDW *B2++[2],B$A[1][0]
LDW *A2++[2],A$A[1][1]
|| LDW *B2++[2],B$A[1][1]
LDW *A2++[2],A$A[1][2]
|| LDW *B2++[2],B$A[1][2]
LDW *A2++[2],A$A[1][3]
|| LDW *B2++[2],B$A[1][3]
LDW *A2++[2],A$A[1][4]
|| LDW *B2++[2],B$A[1][4]
LDW *A2++[2],A$A[2][0]
|| LDW *B2++[2],B$A[2][0]
LDW *A2++[2],A$A[2][1]
|| LDW *B2++[2],B$A[2][1]
LDW *A2++[2],A$A[2][2]
|| LDW *B2++[2],B$A[2][2]
LDW *A2++[2],A$A[2][3]
|| LDW *B2++[2],B$A[2][3]
LDW *A2++[2],A$A[2][4]
|| LDW *B2++[2],B$A[2][4]
LDW *A2++[2],A$A[3][0]
|| LDW *B2++[2],B$A[3][0]
LDW *A2++[2],A$A[3][1]
|| LDW *B2++[2],B$A[3][1]
LDW *A2++[2],A$A[3][2]
|| LDW *B2++[2],B$A[3][2]
LDW *A2++[2],A$A[3][3]
|| LDW *B2++[2],B$A[3][3]
LDW *A2++[2],A$A[3][4]
|| LDW *B2++[2],B$A[3][4]
|| BNOP _KeccakF1600_int
ADDKPC ret?,RA
|| LDW *A2++[2],A$A[4][0]
|| LDW *B2++[2],B$A[4][0]
LDW *A2++[2],A$A[4][1]
|| LDW *B2++[2],B$A[4][1]
LDW *A2++[2],A$A[4][2]
|| LDW *B2++[2],B$A[4][2]
LDW *A2++[2],A$A[4][3]
|| LDW *B2++[2],B$A[4][3]
LDW *A2,A$A[4][4]
|| LDW *B2,B$A[4][4]
|| ADDK -192,A2 ; rewind
|| ADDK -192,B2
.align 16
ret?:
STW A$A[0][0],*A2++[2] ; store A[5][5]
|| STW B$A[0][0],*B2++[2]
STW A$A[0][1],*A2++[2]
|| STW B$A[0][1],*B2++[2]
STW A$A[0][2],*A2++[2]
|| STW B$A[0][2],*B2++[2]
STW A$A[0][3],*A2++[2]
|| STW B$A[0][3],*B2++[2]
STW A$A[0][4],*A2++[2]
|| STW B$A[0][4],*B2++[2]
STW A$A[1][0],*A2++[2]
|| STW B$A[1][0],*B2++[2]
STW A$A[1][1],*A2++[2]
|| STW B$A[1][1],*B2++[2]
STW A$A[1][2],*A2++[2]
|| STW B$A[1][2],*B2++[2]
STW A$A[1][3],*A2++[2]
|| STW B$A[1][3],*B2++[2]
STW A$A[1][4],*A2++[2]
|| STW B$A[1][4],*B2++[2]
STW A$A[2][0],*A2++[2]
|| STW B$A[2][0],*B2++[2]
STW A$A[2][1],*A2++[2]
|| STW B$A[2][1],*B2++[2]
STW A$A[2][2],*A2++[2]
|| STW B$A[2][2],*B2++[2]
STW A$A[2][3],*A2++[2]
|| STW B$A[2][3],*B2++[2]
STW A$A[2][4],*A2++[2]
|| STW B$A[2][4],*B2++[2]
STW A$A[3][0],*A2++[2]
|| STW B$A[3][0],*B2++[2]
STW A$A[3][1],*A2++[2]
|| STW B$A[3][1],*B2++[2]
STW A$A[3][2],*A2++[2]
|| STW B$A[3][2],*B2++[2]
STW A$A[3][3],*A2++[2]
|| STW B$A[3][3],*B2++[2]
STW A$A[3][4],*A2++[2]
|| STW B$A[3][4],*B2++[2]
LDW *SP[15],RA
|| LDW *FP[-6],A14
STW A$A[4][0],*A2++[2]
|| STW B$A[4][0],*B2++[2]
STW A$A[4][1],*A2++[2]
|| STW B$A[4][1],*B2++[2]
STW A$A[4][2],*A2++[2]
|| STW B$A[4][2],*B2++[2]
STW A$A[4][3],*A2++[2]
|| STW B$A[4][3],*B2++[2]
STW A$A[4][4],*A2
|| STW B$A[4][4],*B2
|| ADDK -192,A2 ; rewind
MV A2,A4 ; return original A4
|| LDDW *SP[8], B11:B10
|| LDDW *FP[-5],A11:A10
LDDW *SP[9], B13:B12
|| LDDW *FP[-4],A13:A12
|| BNOP RA
LDW *++SP(80),FP ; restore frame pointer
NOP 4 ; wait till FP is committed
.endasmfunc
.newblock
.asg B2,BSZ
.asg A2,INP
.asg A3,LEN
.global _SHA3_absorb
.align 32
_SHA3_absorb:
.asmfunc stack_usage(80)
STW FP,*SP--(80) ; save frame pointer
|| MV SP,FP
STDW B13:B12,*SP[9]
|| STDW A13:A12,*FP[-4]
STDW B11:B10,*SP[8]
|| STDW A11:A10,*FP[-5]
STW RA, *SP[15]
|| STW A14,*FP[-6]
STW A4,*SP[1] ; save A[][]
|| MV B4,INP ; reassign arguments
|| MV A6,LEN
|| MV B6,BSZ
|| ADD 4,A4,B4
LDW *A4++[2],A$A[0][0] ; load A[5][5]
|| LDW *B4++[2],B$A[0][0]
LDW *A4++[2],A$A[0][1]
|| LDW *B4++[2],B$A[0][1]
LDW *A4++[2],A$A[0][2]
|| LDW *B4++[2],B$A[0][2]
LDW *A4++[2],A$A[0][3]
|| LDW *B4++[2],B$A[0][3]
LDW *A4++[2],A$A[0][4]
|| LDW *B4++[2],B$A[0][4]
LDW *A4++[2],A$A[1][0]
|| LDW *B4++[2],B$A[1][0]
LDW *A4++[2],A$A[1][1]
|| LDW *B4++[2],B$A[1][1]
LDW *A4++[2],A$A[1][2]
|| LDW *B4++[2],B$A[1][2]
LDW *A4++[2],A$A[1][3]
|| LDW *B4++[2],B$A[1][3]
LDW *A4++[2],A$A[1][4]
|| LDW *B4++[2],B$A[1][4]
LDW *A4++[2],A$A[2][0]
|| LDW *B4++[2],B$A[2][0]
LDW *A4++[2],A$A[2][1]
|| LDW *B4++[2],B$A[2][1]
LDW *A4++[2],A$A[2][2]
|| LDW *B4++[2],B$A[2][2]
LDW *A4++[2],A$A[2][3]
|| LDW *B4++[2],B$A[2][3]
LDW *A4++[2],A$A[2][4]
|| LDW *B4++[2],B$A[2][4]
LDW *A4++[2],A$A[3][0]
|| LDW *B4++[2],B$A[3][0]
LDW *A4++[2],A$A[3][1]
|| LDW *B4++[2],B$A[3][1]
LDW *A4++[2],A$A[3][2]
|| LDW *B4++[2],B$A[3][2]
LDW *A4++[2],A$A[3][3]
|| LDW *B4++[2],B$A[3][3]
LDW *A4++[2],A$A[3][4]
|| LDW *B4++[2],B$A[3][4]
LDW *A4++[2],A$A[4][0]
|| LDW *B4++[2],B$A[4][0]
LDW *A4++[2],A$A[4][1]
|| LDW *B4++[2],B$A[4][1]
LDW *A4++[2],A$A[4][2]
|| LDW *B4++[2],B$A[4][2]
LDW *A4++[2],A$A[4][3]
|| LDW *B4++[2],B$A[4][3]
LDW *A4,A$A[4][4]
|| LDW *B4,B$A[4][4]
|| ADDKPC loop?,RA
STDW RA:BSZ,*SP[4]
loop?:
CMPLTU LEN,BSZ,A0 ; len < bsz?
|| SHRU BSZ,3,BSZ
[A0] BNOP ret?
||[A0] ZERO BSZ
||[A0] LDW *SP[1],A2 ; pull A[][]
[BSZ] LDNDW *INP++,A1:A0
||[BSZ] SUB LEN,8,LEN
||[BSZ] SUB BSZ,1,BSZ
NOP 4
___
for ($y = 0; $y < 5; $y++) {
for ($x = 0; $x < ($y<4 ? 5 : 4); $x++) {
$code.=<<___;
.if .BIG_ENDIAN
SWAP2 A0,A1
|| SWAP2 A1,A0
SWAP4 A0,A0
SWAP4 A1,A1
||[!BSZ]BNOP _KeccakF1600_cheat
||[!BSZ]STDW LEN:INP,*SP[3]
|| DEAL A0,A0
.else
[!BSZ]BNOP _KeccakF1600_cheat
||[!BSZ]STDW LEN:INP,*SP[3]
|| DEAL A0,A0
.endif
[BSZ] LDNDW *INP++,A1:A0
|| DEAL A1,A1
[BSZ] SUB LEN,8,LEN
||[BSZ] SUB BSZ,1,BSZ
PACK2 A1,A0,A0
|| PACKH2 A1,A0,A1
XOR A0,A$A[$y][$x],A$A[$y][$x]
XOR A1,B$A[$y][$x],B$A[$y][$x]
___
}
}
$code.=<<___;
.if .BIG_ENDIAN
SWAP2 A0,A1
|| SWAP2 A1,A0
SWAP4 A0,A0
SWAP4 A1,A1
.endif
BNOP _KeccakF1600_cheat
|| STDW LEN:INP,*SP[3]
|| DEAL A0,A0
DEAL A1,A1
NOP
PACK2 A1,A0,A0
|| PACKH2 A1,A0,A1
XOR A0,A$A[4][4],A$A[4][4]
XOR A1,B$A[4][4],B$A[4][4]
.align 16
ret?:
MV LEN,A4 ; return value
|| ADD 4,A2,B2
STW A$A[0][0],*A2++[2] ; store A[5][5]
|| STW B$A[0][0],*B2++[2]
STW A$A[0][1],*A2++[2]
|| STW B$A[0][1],*B2++[2]
STW A$A[0][2],*A2++[2]
|| STW B$A[0][2],*B2++[2]
STW A$A[0][3],*A2++[2]
|| STW B$A[0][3],*B2++[2]
STW A$A[0][4],*A2++[2]
|| STW B$A[0][4],*B2++[2]
STW A$A[1][0],*A2++[2]
|| STW B$A[1][0],*B2++[2]
STW A$A[1][1],*A2++[2]
|| STW B$A[1][1],*B2++[2]
STW A$A[1][2],*A2++[2]
|| STW B$A[1][2],*B2++[2]
STW A$A[1][3],*A2++[2]
|| STW B$A[1][3],*B2++[2]
STW A$A[1][4],*A2++[2]
|| STW B$A[1][4],*B2++[2]
STW A$A[2][0],*A2++[2]
|| STW B$A[2][0],*B2++[2]
STW A$A[2][1],*A2++[2]
|| STW B$A[2][1],*B2++[2]
STW A$A[2][2],*A2++[2]
|| STW B$A[2][2],*B2++[2]
STW A$A[2][3],*A2++[2]
|| STW B$A[2][3],*B2++[2]
STW A$A[2][4],*A2++[2]
|| STW B$A[2][4],*B2++[2]
LDW *SP[15],RA
|| LDW *FP[-6],A14
STW A$A[3][0],*A2++[2]
|| STW B$A[3][0],*B2++[2]
STW A$A[3][1],*A2++[2]
|| STW B$A[3][1],*B2++[2]
STW A$A[3][2],*A2++[2]
|| STW B$A[3][2],*B2++[2]
STW A$A[3][3],*A2++[2]
|| STW B$A[3][3],*B2++[2]
STW A$A[3][4],*A2++[2]
|| STW B$A[3][4],*B2++[2]
LDDW *SP[8], B11:B10
|| LDDW *FP[-5],A11:A10
LDDW *SP[9], B13:B12
|| LDDW *FP[-4],A13:A12
BNOP RA
|| LDW *++SP(80),FP ; restore frame pointer
STW A$A[4][0],*A2++[2]
|| STW B$A[4][0],*B2++[2]
STW A$A[4][1],*A2++[2]
|| STW B$A[4][1],*B2++[2]
STW A$A[4][2],*A2++[2]
|| STW B$A[4][2],*B2++[2]
STW A$A[4][3],*A2++[2]
|| STW B$A[4][3],*B2++[2]
STW A$A[4][4],*A2++[2]
|| STW B$A[4][4],*B2++[2]
.endasmfunc
.newblock
.global _SHA3_squeeze
.asg A12,OUT
.asg A13,LEN
.asg A14,BSZ
.align 32
_SHA3_squeeze:
.asmfunc stack_usage(24)
STW FP,*SP--(24) ; save frame pointer
|| MV SP,FP
STW RA, *SP[5]
|| STW A14,*FP[-2]
STDW A13:A12,*FP[-2]
|| MV B4,OUT ; reassign arguments
MV A6,LEN
|| MV B6,BSZ
loop?:
LDW *SP[5],RA ; reload RA
|| SHRU BSZ,3,A1
|| MV A4,A8
|| ADD 4,A4,B8
block?:
CMPLTU LEN,8,A0 ; len < 8?
[A0] BNOP tail?
LDW *A8++[2],A9
|| LDW *B8++[2],B9
|| SUB LEN,8,LEN ; len -= 8
MV LEN,A0
|| SUB A1,1,A1 ; bsz--
|| NOP 4
.if .BIG_ENDIAN
SWAP4 A9,A9
|| SWAP4 B9,B9
SWAP2 A9,A9
|| SWAP2 B9,B9
.endif
[!A0] BNOP ret?
||[!A0] ZERO A1
PACK2 B9,A9,B7
||[A1] BNOP block?
PACKH2 B9,A9,B9
|| SHFL B7,B7
SHFL B9,B9
STNW B7,*OUT++
STNW B9,*OUT++
NOP
BNOP _KeccakF1600,4
ADDKPC loop?,RA
.align 16
tail?:
.if .BIG_ENDIAN
SWAP4 A9,A9
|| SWAP4 B9,B9
SWAP2 A9,A9
|| SWAP2 B9,B9
.endif
PACK2 B9,A9,B7
PACKH2 B9,A9,B9
|| SHFL B7,B7
SHFL B9,B9
STB B7,*OUT++
|| SHRU B7,8,B7
|| ADD LEN,7,A0
[A0] STB B7,*OUT++
||[A0] SHRU B7,8,B7
||[A0] SUB A0,1,A0
[A0] STB B7,*OUT++
||[A0] SHRU B7,8,B7
||[A0] SUB A0,1,A0
[A0] STB B7,*OUT++
||[A0] SUB A0,1,A0
[A0] STB B9,*OUT++
||[A0] SHRU B9,8,B9
||[A0] SUB A0,1,A0
[A0] STB B9,*OUT++
||[A0] SHRU B9,8,B9
||[A0] SUB A0,1,A0
[A0] STB B9,*OUT++
ret?:
LDDW *FP[-2],A13:A12
BNOP RA
|| LDW *FP[-2],A14
LDW *++SP(24),FP ; restore frame pointer
NOP 4 ; wait till FP is committed
.endasmfunc
.if __TI_EABI__
.sect ".text:sha_asm.const"
.else
.sect ".const:sha_asm"
.endif
.align 256
.uword 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
iotas:
.uword 0x00000001, 0x00000000
.uword 0x00000000, 0x00000089
.uword 0x00000000, 0x8000008b
.uword 0x00000000, 0x80008080
.uword 0x00000001, 0x0000008b
.uword 0x00000001, 0x00008000
.uword 0x00000001, 0x80008088
.uword 0x00000001, 0x80000082
.uword 0x00000000, 0x0000000b
.uword 0x00000000, 0x0000000a
.uword 0x00000001, 0x00008082
.uword 0x00000000, 0x00008003
.uword 0x00000001, 0x0000808b
.uword 0x00000001, 0x8000000b
.uword 0x00000001, 0x8000008a
.uword 0x00000001, 0x80000081
.uword 0x00000000, 0x80000081
.uword 0x00000000, 0x80000008
.uword 0x00000000, 0x00000083
.uword 0x00000000, 0x80008003
.uword 0x00000001, 0x80008088
.uword 0x00000000, 0x80000088
.uword 0x00000001, 0x00008000
.uword 0x00000000, 0x80008082
.cstring "Keccak-1600 absorb and squeeze for C64x, CRYPTOGAMS by <appro\@openssl.org>"
.align 4
___
print $code;
crypto/sha/asm/keccak1600-mmx.pl
Executable
+440
View File
@@ -0,0 +1,440 @@
#!/usr/bin/env perl
# Copyright 2017 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License"). You may not use
# this file except in compliance with the License. You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html
#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# Keccak-1600 for x86 MMX.
#
# June 2017.
#
# Below code is KECCAK_2X implementation (see sha/keccak1600.c) with
# C[5] held in register bank and D[5] offloaded to memory. Though
# instead of actually unrolling the loop pair-wise I simply flip
# pointers to T[][] and A[][] and the end of round. Since number of
# rounds is even, last round writes to A[][] and everything works out.
# It's argued that MMX is the only code path meaningful to implement
# for x86. This is because non-MMX-capable processors is an extinct
# breed, and they as well can lurk executing compiler-generated code.
# For reference gcc-5.x-generated KECCAK_2X code takes 89 cycles per
# processed byte on Pentium. Which is fair result. But older compilers
# produce worse code. On the other hand one can wonder why not 128-bit
# SSE2? Well, SSE2 won't provide double improvement, rather far from
# that, if any at all on some processors, because it will take extra
# permutations and inter-bank data trasfers. Besides, contemporary
# CPUs are better off executing 64-bit code, and it makes lesser sense
# to invest into fancy 32-bit code. And the decision doesn't seem to
# be inadequate, if one compares below results to "64-bit platforms in
# 32-bit mode" SIMD data points available at
# http://keccak.noekeon.org/sw_performance.html.
#
########################################################################
# Numbers are cycles per processed byte out of large message.
#
# r=1088(i)
#
# PIII 30/+150%
# Pentium M 27/+150%
# P4 40/+85%
# Core 2 19/+170%
# Sandy Bridge(ii) 18/+140%
# Atom 33/+180%
# Silvermont(ii) 30/+180%
# VIA Nano(ii) 43/+60%
# Sledgehammer(ii)(iii) 24/+130%
#
# (i) Corresponds to SHA3-256. Numbers after slash are improvement
# coefficients over KECCAK_2X [with bit interleave and lane
# complementing] position-independent *scalar* code generated
# by gcc-5.x. It's not exactly fair comparison, but it's a
# datapoint...
# (ii) 64-bit processor executing 32-bit code.
# (iii) Result is considered to be representative even for older AMD
# processors.
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
push(@INC,"${dir}","${dir}../../perlasm");
require "x86asm.pl";
$output=pop;
open STDOUT,">$output";
&asm_init($ARGV[0],$ARGV[$#ARGV] eq "386");
my @C = map("mm$_",(0..4));
my @T = map("mm$_",(5..7));
my @A = map([ 8*$_-100, 8*($_+1)-100, 8*($_+2)-100,
8*($_+3)-100, 8*($_+4)-100 ], (0,5,10,15,20));
my @D = map(8*$_+4, (0..4));
my @rhotates = ([ 0, 1, 62, 28, 27 ],
[ 36, 44, 6, 55, 20 ],
[ 3, 10, 43, 25, 39 ],
[ 41, 45, 15, 21, 8 ],
[ 18, 2, 61, 56, 14 ]);
&static_label("iotas");
&function_begin_B("_KeccakF1600");
&movq (@C[0],&QWP($A[4][0],"esi"));
&movq (@C[1],&QWP($A[4][1],"esi"));
&movq (@C[2],&QWP($A[4][2],"esi"));
&movq (@C[3],&QWP($A[4][3],"esi"));
&movq (@C[4],&QWP($A[4][4],"esi"));
&mov ("ecx",24); # loop counter
&jmp (&label("loop"));
&set_label("loop",16);
######################################### Theta
&pxor (@C[0],&QWP($A[0][0],"esi"));
&pxor (@C[1],&QWP($A[0][1],"esi"));
&pxor (@C[2],&QWP($A[0][2],"esi"));
&pxor (@C[3],&QWP($A[0][3],"esi"));
&pxor (@C[4],&QWP($A[0][4],"esi"));
&pxor (@C[0],&QWP($A[1][0],"esi"));
&pxor (@C[1],&QWP($A[1][1],"esi"));
&pxor (@C[2],&QWP($A[1][2],"esi"));
&pxor (@C[3],&QWP($A[1][3],"esi"));
&pxor (@C[4],&QWP($A[1][4],"esi"));
&pxor (@C[0],&QWP($A[2][0],"esi"));
&pxor (@C[1],&QWP($A[2][1],"esi"));
&pxor (@C[2],&QWP($A[2][2],"esi"));
&pxor (@C[3],&QWP($A[2][3],"esi"));
&pxor (@C[4],&QWP($A[2][4],"esi"));
&pxor (@C[2],&QWP($A[3][2],"esi"));
&pxor (@C[0],&QWP($A[3][0],"esi"));
&pxor (@C[1],&QWP($A[3][1],"esi"));
&pxor (@C[3],&QWP($A[3][3],"esi"));
&movq (@T[0],@C[2]);
&pxor (@C[4],&QWP($A[3][4],"esi"));
&movq (@T[2],@C[2]);
&psrlq (@T[0],63);
&movq (@T[1],@C[0]);
&psllq (@T[2],1);
&pxor (@T[0],@C[0]);
&psrlq (@C[0],63);
&pxor (@T[0],@T[2]);
&psllq (@T[1],1);
&movq (@T[2],@C[1]);
&movq (&QWP(@D[1],"esp"),@T[0]); # D[1] = E[0] = ROL64(C[2], 1) ^ C[0];
&pxor (@T[1],@C[0]);
&psrlq (@T[2],63);
&pxor (@T[1],@C[3]);
&movq (@C[0],@C[1]);
&movq (&QWP(@D[4],"esp"),@T[1]); # D[4] = E[1] = ROL64(C[0], 1) ^ C[3];
&psllq (@C[0],1);
&pxor (@T[2],@C[4]);
&pxor (@C[0],@T[2]);
&movq (@T[2],@C[3]);
&psrlq (@C[3],63);
&movq (&QWP(@D[0],"esp"),@C[0]); # D[0] = C[0] = ROL64(C[1], 1) ^ C[4];
&psllq (@T[2],1);
&movq (@T[0],@C[4]);
&psrlq (@C[4],63);
&pxor (@C[1],@C[3]);
&psllq (@T[0],1);
&pxor (@C[1],@T[2]);
&pxor (@C[2],@C[4]);
&movq (&QWP(@D[2],"esp"),@C[1]); # D[2] = C[1] = ROL64(C[3], 1) ^ C[1];
&pxor (@C[2],@T[0]);
######################################### first Rho(0) is special
&movq (@C[3],&QWP($A[3][3],"esi"));
&movq (&QWP(@D[3],"esp"),@C[2]); # D[3] = C[2] = ROL64(C[4], 1) ^ C[2];
&pxor (@C[3],@C[2]);
&movq (@C[4],&QWP($A[4][4],"esi"));
&movq (@T[2],@C[3]);
&psrlq (@C[3],64-$rhotates[3][3]);
&pxor (@C[4],@T[1]);
&psllq (@T[2],$rhotates[3][3]);
&movq (@T[1],@C[4]);
&psrlq (@C[4],64-$rhotates[4][4]);
&por (@C[3],@T[2]); # C[3] = ROL64(A[3][3] ^ C[2], rhotates[3][3]); /* D[3] */
&psllq (@T[1],$rhotates[4][4]);
&movq (@C[2],&QWP($A[2][2],"esi"));
&por (@C[4],@T[1]); # C[4] = ROL64(A[4][4] ^ E[1], rhotates[4][4]); /* D[4] */
&pxor (@C[2],@C[1]);
&movq (@C[1],&QWP($A[1][1],"esi"));
&movq (@T[1],@C[2]);
&psrlq (@C[2],64-$rhotates[2][2]);
&pxor (@C[1],&QWP(@D[1],"esp"));
&psllq (@T[1],$rhotates[2][2]);
&movq (@T[2],@C[1]);
&psrlq (@C[1],64-$rhotates[1][1]);
&por (@C[2],@T[1]); # C[2] = ROL64(A[2][2] ^ C[1], rhotates[2][2]); /* D[2] */
&psllq (@T[2],$rhotates[1][1]);
&pxor (@C[0],&QWP($A[0][0],"esi")); # /* rotate by 0 */ /* D[0] */
&por (@C[1],@T[2]); # C[1] = ROL64(A[1][1] ^ D[1], rhotates[1][1]);
sub Chi() { ######### regular Chi step
my ($y,$xrho) = @_;
&movq (@T[0],@C[1]);
&movq (@T[1],@C[2]);
&pandn (@T[0],@C[2]);
&pandn (@C[2],@C[3]);
&pxor (@T[0],@C[0]);
&pxor (@C[2],@C[1]);
&pxor (@T[0],&QWP(0,"ebx")) if ($y == 0);
&lea ("ebx",&DWP(8,"ebx")) if ($y == 0);
&movq (@T[2],@C[3]);
&movq (&QWP($A[$y][0],"edi"),@T[0]); # R[0][0] = C[0] ^ (~C[1] & C[2]) ^ iotas[i];
&movq (@T[0],@C[4]);
&pandn (@C[3],@C[4]);
&pandn (@C[4],@C[0]);
&pxor (@C[3],@T[1]);
&movq (&QWP($A[$y][1],"edi"),@C[2]); # R[0][1] = C[1] ^ (~C[2] & C[3]);
&pxor (@C[4],@T[2]);
&movq (@T[2],&QWP($A[0][$xrho],"esi")) if (defined($xrho));
&movq (&QWP($A[$y][2],"edi"),@C[3]); # R[0][2] = C[2] ^ (~C[3] & C[4]);
&pandn (@C[0],@C[1]);
&movq (&QWP($A[$y][3],"edi"),@C[4]); # R[0][3] = C[3] ^ (~C[4] & C[0]);
&pxor (@C[0],@T[0]);
&pxor (@T[2],&QWP(@D[$xrho],"esp")) if (defined($xrho));
&movq (&QWP($A[$y][4],"edi"),@C[0]); # R[0][4] = C[4] ^ (~C[0] & C[1]);
}
&Chi (0, 3);
sub Rho() { ######### regular Rho step
my $x = shift;
#&movq (@T[2],&QWP($A[0][$x],"esi")); # moved to Chi
#&pxor (@T[2],&QWP(@D[$x],"esp")); # moved to Chi
&movq (@C[0],@T[2]);
&psrlq (@T[2],64-$rhotates[0][$x]);
&movq (@C[1],&QWP($A[1][($x+1)%5],"esi"));
&psllq (@C[0],$rhotates[0][$x]);
&pxor (@C[1],&QWP(@D[($x+1)%5],"esp"));
&por (@C[0],@T[2]); # C[0] = ROL64(A[0][3] ^ D[3], rhotates[0][3]);
&movq (@T[1],@C[1]);
&psrlq (@C[1],64-$rhotates[1][($x+1)%5]);
&movq (@C[2],&QWP($A[2][($x+2)%5],"esi"));
&psllq (@T[1],$rhotates[1][($x+1)%5]);
&pxor (@C[2],&QWP(@D[($x+2)%5],"esp"));
&por (@C[1],@T[1]); # C[1] = ROL64(A[1][4] ^ D[4], rhotates[1][4]);
&movq (@T[2],@C[2]);
&psrlq (@C[2],64-$rhotates[2][($x+2)%5]);
&movq (@C[3],&QWP($A[3][($x+3)%5],"esi"));
&psllq (@T[2],$rhotates[2][($x+2)%5]);
&pxor (@C[3],&QWP(@D[($x+3)%5],"esp"));
&por (@C[2],@T[2]); # C[2] = ROL64(A[2][0] ^ D[0], rhotates[2][0]);
&movq (@T[0],@C[3]);
&psrlq (@C[3],64-$rhotates[3][($x+3)%5]);
&movq (@C[4],&QWP($A[4][($x+4)%5],"esi"));
&psllq (@T[0],$rhotates[3][($x+3)%5]);
&pxor (@C[4],&QWP(@D[($x+4)%5],"esp"));
&por (@C[3],@T[0]); # C[3] = ROL64(A[3][1] ^ D[1], rhotates[3][1]);
&movq (@T[1],@C[4]);
&psrlq (@C[4],64-$rhotates[4][($x+4)%5]);
&psllq (@T[1],$rhotates[4][($x+4)%5]);
&por (@C[4],@T[1]); # C[4] = ROL64(A[4][2] ^ D[2], rhotates[4][2]);
}
&Rho (3); &Chi (1, 1);
&Rho (1); &Chi (2, 4);
&Rho (4); &Chi (3, 2);
&Rho (2); ###&Chi (4);
&movq (@T[0],@C[0]); ######### last Chi(4) is special
&xor ("edi","esi"); # &xchg ("esi","edi");
&movq (&QWP(@D[1],"esp"),@C[1]);
&xor ("esi","edi");
&xor ("edi","esi");
&movq (@T[1],@C[1]);
&movq (@T[2],@C[2]);
&pandn (@T[1],@C[2]);
&pandn (@T[2],@C[3]);
&pxor (@C[0],@T[1]);
&pxor (@C[1],@T[2]);
&movq (@T[1],@C[3]);
&movq (&QWP($A[4][0],"esi"),@C[0]); # R[4][0] = C[0] ^= (~C[1] & C[2]);
&pandn (@T[1],@C[4]);
&movq (&QWP($A[4][1],"esi"),@C[1]); # R[4][1] = C[1] ^= (~C[2] & C[3]);
&pxor (@C[2],@T[1]);
&movq (@T[2],@C[4]);
&movq (&QWP($A[4][2],"esi"),@C[2]); # R[4][2] = C[2] ^= (~C[3] & C[4]);
&pandn (@T[2],@T[0]);
&pandn (@T[0],&QWP(@D[1],"esp"));
&pxor (@C[3],@T[2]);
&pxor (@C[4],@T[0]);
&movq (&QWP($A[4][3],"esi"),@C[3]); # R[4][3] = C[3] ^= (~C[4] & D[0]);
&sub ("ecx",1);
&movq (&QWP($A[4][4],"esi"),@C[4]); # R[4][4] = C[4] ^= (~D[0] & D[1]);
&jnz (&label("loop"));
&lea ("ebx",&DWP(-192,"ebx")); # rewind iotas
&ret ();
&function_end_B("_KeccakF1600");
&function_begin("KeccakF1600");
&mov ("esi",&wparam(0));
&mov ("ebp","esp");
&sub ("esp",240);
&call (&label("pic_point"));
&set_label("pic_point");
&blindpop("ebx");
&lea ("ebx",&DWP(&label("iotas")."-".&label("pic_point"),"ebx"));
&and ("esp",-8);
&lea ("esi",&DWP(100,"esi")); # size optimization
&lea ("edi",&DWP(8*5+100,"esp")); # size optimization
&call ("_KeccakF1600");
&mov ("esp","ebp");
&emms ();
&function_end("KeccakF1600");
&function_begin("SHA3_absorb");
&mov ("esi",&wparam(0)); # A[][]
&mov ("eax",&wparam(1)); # inp
&mov ("ecx",&wparam(2)); # len
&mov ("edx",&wparam(3)); # bsz
&mov ("ebp","esp");
&sub ("esp",240+8);
&call (&label("pic_point"));
&set_label("pic_point");
&blindpop("ebx");
&lea ("ebx",&DWP(&label("iotas")."-".&label("pic_point"),"ebx"));
&and ("esp",-8);
&mov ("edi","esi");
&lea ("esi",&DWP(100,"esi")); # size optimization
&mov (&DWP(-4,"ebp"),"edx"); # save bsz
&jmp (&label("loop"));
&set_label("loop",16);
&cmp ("ecx","edx"); # len < bsz?
&jc (&label("absorbed"));
&shr ("edx",3); # bsz /= 8
&set_label("block");
&movq ("mm0",&QWP(0,"eax"));
&lea ("eax",&DWP(8,"eax"));
&pxor ("mm0",&QWP(0,"edi"));
&lea ("edi",&DWP(8,"edi"));
&sub ("ecx",8); # len -= 8
&movq (&QWP(-8,"edi"),"mm0");
&dec ("edx"); # bsz--
&jnz (&label("block"));
&lea ("edi",&DWP(8*5+100,"esp")); # size optimization
&mov (&DWP(-8,"ebp"),"ecx"); # save len
&call ("_KeccakF1600");
&mov ("ecx",&DWP(-8,"ebp")); # pull len
&mov ("edx",&DWP(-4,"ebp")); # pull bsz
&lea ("edi",&DWP(-100,"esi"));
&jmp (&label("loop"));
&set_label("absorbed",16);
&mov ("eax","ecx"); # return value
&mov ("esp","ebp");
&emms ();
&function_end("SHA3_absorb");
&function_begin("SHA3_squeeze");
&mov ("esi",&wparam(0)); # A[][]
&mov ("eax",&wparam(1)); # out
&mov ("ecx",&wparam(2)); # len
&mov ("edx",&wparam(3)); # bsz
&mov ("ebp","esp");
&sub ("esp",240+8);
&call (&label("pic_point"));
&set_label("pic_point");
&blindpop("ebx");
&lea ("ebx",&DWP(&label("iotas")."-".&label("pic_point"),"ebx"));
&and ("esp",-8);
&shr ("edx",3); # bsz /= 8
&mov ("edi","esi");
&lea ("esi",&DWP(100,"esi")); # size optimization
&mov (&DWP(-4,"ebp"),"edx"); # save bsz
&jmp (&label("loop"));
&set_label("loop",16);
&cmp ("ecx",8); # len < 8?
&jc (&label("tail"));
&movq ("mm0",&QWP(0,"edi"));
&lea ("edi",&DWP(8,"edi"));
&movq (&QWP(0,"eax"),"mm0");
&lea ("eax",&DWP(8,"eax"));
&sub ("ecx",8); # len -= 8
&jz (&label("done"));
&dec ("edx"); # bsz--
&jnz (&label("loop"));
&lea ("edi",&DWP(8*5+100,"esp")); # size optimization
&mov (&DWP(-8,"ebp"),"ecx"); # save len
&call ("_KeccakF1600");
&mov ("ecx",&DWP(-8,"ebp")); # pull len
&mov ("edx",&DWP(-4,"ebp")); # pull bsz
&lea ("edi",&DWP(-100,"esi"));
&jmp (&label("loop"));
&set_label("tail",16);
&mov ("esi","edi");
&mov ("edi","eax");
&data_word("0xA4F39066"); # rep movsb
&set_label("done");
&mov ("esp","ebp");
&emms ();
&function_end("SHA3_squeeze");
&set_label("iotas",32);
&data_word(0x00000001,0x00000000);
&data_word(0x00008082,0x00000000);
&data_word(0x0000808a,0x80000000);
&data_word(0x80008000,0x80000000);
&data_word(0x0000808b,0x00000000);
&data_word(0x80000001,0x00000000);
&data_word(0x80008081,0x80000000);
&data_word(0x00008009,0x80000000);
&data_word(0x0000008a,0x00000000);
&data_word(0x00000088,0x00000000);
&data_word(0x80008009,0x00000000);
&data_word(0x8000000a,0x00000000);
&data_word(0x8000808b,0x00000000);
&data_word(0x0000008b,0x80000000);
&data_word(0x00008089,0x80000000);
&data_word(0x00008003,0x80000000);
&data_word(0x00008002,0x80000000);
&data_word(0x00000080,0x80000000);
&data_word(0x0000800a,0x00000000);
&data_word(0x8000000a,0x80000000);
&data_word(0x80008081,0x80000000);
&data_word(0x00008080,0x80000000);
&data_word(0x80000001,0x00000000);
&data_word(0x80008008,0x80000000);
&asciz("Keccak-1600 absorb and squeeze for MMX, CRYPTOGAMS by <appro\@openssl.org>");
&asm_finish();
close STDOUT;
crypto/sha/asm/keccak1600-ppc64.pl
Executable
+757
View File
@@ -0,0 +1,757 @@
#!/usr/bin/env perl
# Copyright 2017 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License"). You may not use
# this file except in compliance with the License. You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html
#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# Keccak-1600 for PPC64.
#
# June 2017.
#
# This is straightforward KECCAK_1X_ALT implementation that works on
# *any* PPC64. Then PowerISA 2.07 adds 2x64-bit vector rotate, and
# it's possible to achieve performance better than below, but that is
# naturally option only for POWER8 and successors...
#
######################################################################
# Numbers are cycles per processed byte.
#
# r=1088(*)
#
# PPC970/G5 14.6/+120%
# POWER7 10.3/+100%
# POWER8 11.5/+85%
#
# (*) Corresponds to SHA3-256. Percentage after slash is improvement
# over gcc-4.x-generated KECCAK_1X_ALT code. Newer compilers do
# much better (but watch out for them generating code specific
# to processor they execute on).
$flavour = shift;
if ($flavour =~ /64/) {
$SIZE_T =8;
$LRSAVE =2*$SIZE_T;
$UCMP ="cmpld";
$STU ="stdu";
$POP ="ld";
$PUSH ="std";
} else { die "nonsense $flavour"; }
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
die "can't locate ppc-xlate.pl";
open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
$FRAME=24*$SIZE_T+6*$SIZE_T+32;
$LOCALS=6*$SIZE_T;
$TEMP=$LOCALS+6*$SIZE_T;
my $sp ="r1";
my @A = map([ "r$_", "r".($_+1), "r".($_+2), "r".($_+3), "r".($_+4) ],
(7, 12, 17, 22, 27));
$A[1][1] = "r6"; # r13 is reserved
my @C = map("r$_", (0,3,4,5));
my @rhotates = ([ 0, 1, 62, 28, 27 ],
[ 36, 44, 6, 55, 20 ],
[ 3, 10, 43, 25, 39 ],
[ 41, 45, 15, 21, 8 ],
[ 18, 2, 61, 56, 14 ]);
$code.=<<___;
.text
.type KeccakF1600_int,\@function
.align 5
KeccakF1600_int:
li r0,24
mtctr r0
b .Loop
.align 4
.Loop:
xor $C[0],$A[0][0],$A[1][0] ; Theta
std $A[0][4],`$TEMP+0`($sp)
xor $C[1],$A[0][1],$A[1][1]
std $A[1][4],`$TEMP+8`($sp)
xor $C[2],$A[0][2],$A[1][2]
std $A[2][4],`$TEMP+16`($sp)
xor $C[3],$A[0][3],$A[1][3]
std $A[3][4],`$TEMP+24`($sp)
___
$C[4]=$A[0][4];
$C[5]=$A[1][4];
$C[6]=$A[2][4];
$C[7]=$A[3][4];
$code.=<<___;
xor $C[4],$A[0][4],$A[1][4]
xor $C[0],$C[0],$A[2][0]
xor $C[1],$C[1],$A[2][1]
xor $C[2],$C[2],$A[2][2]
xor $C[3],$C[3],$A[2][3]
xor $C[4],$C[4],$A[2][4]
xor $C[0],$C[0],$A[3][0]
xor $C[1],$C[1],$A[3][1]
xor $C[2],$C[2],$A[3][2]
xor $C[3],$C[3],$A[3][3]
xor $C[4],$C[4],$A[3][4]
xor $C[0],$C[0],$A[4][0]
xor $C[2],$C[2],$A[4][2]
xor $C[1],$C[1],$A[4][1]
xor $C[3],$C[3],$A[4][3]
rotldi $C[5],$C[2],1
xor $C[4],$C[4],$A[4][4]
rotldi $C[6],$C[3],1
xor $C[5],$C[5],$C[0]
rotldi $C[7],$C[4],1
xor $A[0][1],$A[0][1],$C[5]
xor $A[1][1],$A[1][1],$C[5]
xor $A[2][1],$A[2][1],$C[5]
xor $A[3][1],$A[3][1],$C[5]
xor $A[4][1],$A[4][1],$C[5]
rotldi $C[5],$C[0],1
xor $C[6],$C[6],$C[1]
xor $C[2],$C[2],$C[7]
rotldi $C[7],$C[1],1
xor $C[3],$C[3],$C[5]
xor $C[4],$C[4],$C[7]
xor $C[1], $A[0][2],$C[6] ;mr $C[1],$A[0][2]
xor $A[1][2],$A[1][2],$C[6]
xor $A[2][2],$A[2][2],$C[6]
xor $A[3][2],$A[3][2],$C[6]
xor $A[4][2],$A[4][2],$C[6]
xor $A[0][0],$A[0][0],$C[4]
xor $A[1][0],$A[1][0],$C[4]
xor $A[2][0],$A[2][0],$C[4]
xor $A[3][0],$A[3][0],$C[4]
xor $A[4][0],$A[4][0],$C[4]
___
$C[4]=undef;
$C[5]=undef;
$C[6]=undef;
$C[7]=undef;
$code.=<<___;
ld $A[0][4],`$TEMP+0`($sp)
xor $C[0], $A[0][3],$C[2] ;mr $C[0],$A[0][3]
ld $A[1][4],`$TEMP+8`($sp)
xor $A[1][3],$A[1][3],$C[2]
ld $A[2][4],`$TEMP+16`($sp)
xor $A[2][3],$A[2][3],$C[2]
ld $A[3][4],`$TEMP+24`($sp)
xor $A[3][3],$A[3][3],$C[2]
xor $A[4][3],$A[4][3],$C[2]
xor $C[2], $A[0][4],$C[3] ;mr $C[2],$A[0][4]
xor $A[1][4],$A[1][4],$C[3]
xor $A[2][4],$A[2][4],$C[3]
xor $A[3][4],$A[3][4],$C[3]
xor $A[4][4],$A[4][4],$C[3]
mr $C[3],$A[0][1] ; Rho+Pi
rotldi $A[0][1],$A[1][1],$rhotates[1][1]
;mr $C[1],$A[0][2]
rotldi $A[0][2],$A[2][2],$rhotates[2][2]
;mr $C[0],$A[0][3]
rotldi $A[0][3],$A[3][3],$rhotates[3][3]
;mr $C[2],$A[0][4]
rotldi $A[0][4],$A[4][4],$rhotates[4][4]
rotldi $A[1][1],$A[1][4],$rhotates[1][4]
rotldi $A[2][2],$A[2][3],$rhotates[2][3]
rotldi $A[3][3],$A[3][2],$rhotates[3][2]
rotldi $A[4][4],$A[4][1],$rhotates[4][1]
rotldi $A[1][4],$A[4][2],$rhotates[4][2]
rotldi $A[2][3],$A[3][4],$rhotates[3][4]
rotldi $A[3][2],$A[2][1],$rhotates[2][1]
rotldi $A[4][1],$A[1][3],$rhotates[1][3]
rotldi $A[4][2],$A[2][4],$rhotates[2][4]
rotldi $A[3][4],$A[4][3],$rhotates[4][3]
rotldi $A[2][1],$A[1][2],$rhotates[1][2]
rotldi $A[1][3],$A[3][1],$rhotates[3][1]
rotldi $A[2][4],$A[4][0],$rhotates[4][0]
rotldi $A[4][3],$A[3][0],$rhotates[3][0]
rotldi $A[1][2],$A[2][0],$rhotates[2][0]
rotldi $A[3][1],$A[1][0],$rhotates[1][0]
rotldi $A[1][0],$C[0],$rhotates[0][3]
rotldi $A[2][0],$C[3],$rhotates[0][1]
rotldi $A[3][0],$C[2],$rhotates[0][4]
rotldi $A[4][0],$C[1],$rhotates[0][2]
andc $C[0],$A[0][2],$A[0][1] ; Chi+Iota
andc $C[1],$A[0][3],$A[0][2]
andc $C[2],$A[0][0],$A[0][4]
andc $C[3],$A[0][1],$A[0][0]
xor $A[0][0],$A[0][0],$C[0]
andc $C[0],$A[0][4],$A[0][3]
xor $A[0][1],$A[0][1],$C[1]
ld $C[1],`$LOCALS+4*$SIZE_T`($sp)
xor $A[0][3],$A[0][3],$C[2]
xor $A[0][4],$A[0][4],$C[3]
xor $A[0][2],$A[0][2],$C[0]
ldu $C[3],8($C[1]) ; Iota[i++]
andc $C[0],$A[1][2],$A[1][1]
std $C[1],`$LOCALS+4*$SIZE_T`($sp)
andc $C[1],$A[1][3],$A[1][2]
andc $C[2],$A[1][0],$A[1][4]
xor $A[0][0],$A[0][0],$C[3] ; A[0][0] ^= Iota
andc $C[3],$A[1][1],$A[1][0]
xor $A[1][0],$A[1][0],$C[0]
andc $C[0],$A[1][4],$A[1][3]
xor $A[1][1],$A[1][1],$C[1]
xor $A[1][3],$A[1][3],$C[2]
xor $A[1][4],$A[1][4],$C[3]
xor $A[1][2],$A[1][2],$C[0]
andc $C[0],$A[2][2],$A[2][1]
andc $C[1],$A[2][3],$A[2][2]
andc $C[2],$A[2][0],$A[2][4]
andc $C[3],$A[2][1],$A[2][0]
xor $A[2][0],$A[2][0],$C[0]
andc $C[0],$A[2][4],$A[2][3]
xor $A[2][1],$A[2][1],$C[1]
xor $A[2][3],$A[2][3],$C[2]
xor $A[2][4],$A[2][4],$C[3]
xor $A[2][2],$A[2][2],$C[0]
andc $C[0],$A[3][2],$A[3][1]
andc $C[1],$A[3][3],$A[3][2]
andc $C[2],$A[3][0],$A[3][4]
andc $C[3],$A[3][1],$A[3][0]
xor $A[3][0],$A[3][0],$C[0]
andc $C[0],$A[3][4],$A[3][3]
xor $A[3][1],$A[3][1],$C[1]
xor $A[3][3],$A[3][3],$C[2]
xor $A[3][4],$A[3][4],$C[3]
xor $A[3][2],$A[3][2],$C[0]
andc $C[0],$A[4][2],$A[4][1]
andc $C[1],$A[4][3],$A[4][2]
andc $C[2],$A[4][0],$A[4][4]
andc $C[3],$A[4][1],$A[4][0]
xor $A[4][0],$A[4][0],$C[0]
andc $C[0],$A[4][4],$A[4][3]
xor $A[4][1],$A[4][1],$C[1]
xor $A[4][3],$A[4][3],$C[2]
xor $A[4][4],$A[4][4],$C[3]
xor $A[4][2],$A[4][2],$C[0]
bdnz .Loop
blr
.long 0
.byte 0,12,0x14,0,0,0,0,0
.size KeccakF1600_int,.-KeccakF1600_int
.type KeccakF1600,\@function
.align 5
KeccakF1600:
$STU $sp,-$FRAME($sp)
mflr r0
$PUSH r14,`$FRAME-$SIZE_T*18`($sp)
$PUSH r15,`$FRAME-$SIZE_T*17`($sp)
$PUSH r16,`$FRAME-$SIZE_T*16`($sp)
$PUSH r17,`$FRAME-$SIZE_T*15`($sp)
$PUSH r18,`$FRAME-$SIZE_T*14`($sp)
$PUSH r19,`$FRAME-$SIZE_T*13`($sp)
$PUSH r20,`$FRAME-$SIZE_T*12`($sp)
$PUSH r21,`$FRAME-$SIZE_T*11`($sp)
$PUSH r22,`$FRAME-$SIZE_T*10`($sp)
$PUSH r23,`$FRAME-$SIZE_T*9`($sp)
$PUSH r24,`$FRAME-$SIZE_T*8`($sp)
$PUSH r25,`$FRAME-$SIZE_T*7`($sp)
$PUSH r26,`$FRAME-$SIZE_T*6`($sp)
$PUSH r27,`$FRAME-$SIZE_T*5`($sp)
$PUSH r28,`$FRAME-$SIZE_T*4`($sp)
$PUSH r29,`$FRAME-$SIZE_T*3`($sp)
$PUSH r30,`$FRAME-$SIZE_T*2`($sp)
$PUSH r31,`$FRAME-$SIZE_T*1`($sp)
$PUSH r0,`$FRAME+$LRSAVE`($sp)
bl PICmeup
subi r12,r12,8 ; prepare for ldu
$PUSH r3,`$LOCALS+0*$SIZE_T`($sp)
;$PUSH r4,`$LOCALS+1*$SIZE_T`($sp)
;$PUSH r5,`$LOCALS+2*$SIZE_T`($sp)
;$PUSH r6,`$LOCALS+3*$SIZE_T`($sp)
$PUSH r12,`$LOCALS+4*$SIZE_T`($sp)
ld $A[0][0],`8*0`(r3) ; load A[5][5]
ld $A[0][1],`8*1`(r3)
ld $A[0][2],`8*2`(r3)
ld $A[0][3],`8*3`(r3)
ld $A[0][4],`8*4`(r3)
ld $A[1][0],`8*5`(r3)
ld $A[1][1],`8*6`(r3)
ld $A[1][2],`8*7`(r3)
ld $A[1][3],`8*8`(r3)
ld $A[1][4],`8*9`(r3)
ld $A[2][0],`8*10`(r3)
ld $A[2][1],`8*11`(r3)
ld $A[2][2],`8*12`(r3)
ld $A[2][3],`8*13`(r3)
ld $A[2][4],`8*14`(r3)
ld $A[3][0],`8*15`(r3)
ld $A[3][1],`8*16`(r3)
ld $A[3][2],`8*17`(r3)
ld $A[3][3],`8*18`(r3)
ld $A[3][4],`8*19`(r3)
ld $A[4][0],`8*20`(r3)
ld $A[4][1],`8*21`(r3)
ld $A[4][2],`8*22`(r3)
ld $A[4][3],`8*23`(r3)
ld $A[4][4],`8*24`(r3)
bl KeccakF1600_int
$POP r3,`$LOCALS+0*$SIZE_T`($sp)
std $A[0][0],`8*0`(r3) ; return A[5][5]
std $A[0][1],`8*1`(r3)
std $A[0][2],`8*2`(r3)
std $A[0][3],`8*3`(r3)
std $A[0][4],`8*4`(r3)
std $A[1][0],`8*5`(r3)
std $A[1][1],`8*6`(r3)
std $A[1][2],`8*7`(r3)
std $A[1][3],`8*8`(r3)
std $A[1][4],`8*9`(r3)
std $A[2][0],`8*10`(r3)
std $A[2][1],`8*11`(r3)
std $A[2][2],`8*12`(r3)
std $A[2][3],`8*13`(r3)
std $A[2][4],`8*14`(r3)
std $A[3][0],`8*15`(r3)
std $A[3][1],`8*16`(r3)
std $A[3][2],`8*17`(r3)
std $A[3][3],`8*18`(r3)
std $A[3][4],`8*19`(r3)
std $A[4][0],`8*20`(r3)
std $A[4][1],`8*21`(r3)
std $A[4][2],`8*22`(r3)
std $A[4][3],`8*23`(r3)
std $A[4][4],`8*24`(r3)
$POP r0,`$FRAME+$LRSAVE`($sp)
$POP r14,`$FRAME-$SIZE_T*18`($sp)
$POP r15,`$FRAME-$SIZE_T*17`($sp)
$POP r16,`$FRAME-$SIZE_T*16`($sp)
$POP r17,`$FRAME-$SIZE_T*15`($sp)
$POP r18,`$FRAME-$SIZE_T*14`($sp)
$POP r19,`$FRAME-$SIZE_T*13`($sp)
$POP r20,`$FRAME-$SIZE_T*12`($sp)
$POP r21,`$FRAME-$SIZE_T*11`($sp)
$POP r22,`$FRAME-$SIZE_T*10`($sp)
$POP r23,`$FRAME-$SIZE_T*9`($sp)
$POP r24,`$FRAME-$SIZE_T*8`($sp)
$POP r25,`$FRAME-$SIZE_T*7`($sp)
$POP r26,`$FRAME-$SIZE_T*6`($sp)
$POP r27,`$FRAME-$SIZE_T*5`($sp)
$POP r28,`$FRAME-$SIZE_T*4`($sp)
$POP r29,`$FRAME-$SIZE_T*3`($sp)
$POP r30,`$FRAME-$SIZE_T*2`($sp)
$POP r31,`$FRAME-$SIZE_T*1`($sp)
mtlr r0
addi $sp,$sp,$FRAME
blr
.long 0
.byte 0,12,4,1,0x80,18,1,0
.long 0
.size KeccakF1600,.-KeccakF1600
.type dword_le_load,\@function
.align 5
dword_le_load:
lbzu r0,1(r3)
lbzu r4,1(r3)
lbzu r5,1(r3)
insrdi r0,r4,8,48
lbzu r4,1(r3)
insrdi r0,r5,8,40
lbzu r5,1(r3)
insrdi r0,r4,8,32
lbzu r4,1(r3)
insrdi r0,r5,8,24
lbzu r5,1(r3)
insrdi r0,r4,8,16
lbzu r4,1(r3)
insrdi r0,r5,8,8
insrdi r0,r4,8,0
blr
.long 0
.byte 0,12,0x14,0,0,0,1,0
.long 0
.size dword_le_load,.-dword_le_load
.globl SHA3_absorb
.type SHA3_absorb,\@function
.align 5
SHA3_absorb:
$STU $sp,-$FRAME($sp)
mflr r0
$PUSH r14,`$FRAME-$SIZE_T*18`($sp)
$PUSH r15,`$FRAME-$SIZE_T*17`($sp)
$PUSH r16,`$FRAME-$SIZE_T*16`($sp)
$PUSH r17,`$FRAME-$SIZE_T*15`($sp)
$PUSH r18,`$FRAME-$SIZE_T*14`($sp)
$PUSH r19,`$FRAME-$SIZE_T*13`($sp)
$PUSH r20,`$FRAME-$SIZE_T*12`($sp)
$PUSH r21,`$FRAME-$SIZE_T*11`($sp)
$PUSH r22,`$FRAME-$SIZE_T*10`($sp)
$PUSH r23,`$FRAME-$SIZE_T*9`($sp)
$PUSH r24,`$FRAME-$SIZE_T*8`($sp)
$PUSH r25,`$FRAME-$SIZE_T*7`($sp)
$PUSH r26,`$FRAME-$SIZE_T*6`($sp)
$PUSH r27,`$FRAME-$SIZE_T*5`($sp)
$PUSH r28,`$FRAME-$SIZE_T*4`($sp)
$PUSH r29,`$FRAME-$SIZE_T*3`($sp)
$PUSH r30,`$FRAME-$SIZE_T*2`($sp)
$PUSH r31,`$FRAME-$SIZE_T*1`($sp)
$PUSH r0,`$FRAME+$LRSAVE`($sp)
bl PICmeup
subi r4,r4,1 ; prepare for lbzu
subi r12,r12,8 ; prepare for ldu
$PUSH r3,`$LOCALS+0*$SIZE_T`($sp) ; save A[][]
$PUSH r4,`$LOCALS+1*$SIZE_T`($sp) ; save inp
$PUSH r5,`$LOCALS+2*$SIZE_T`($sp) ; save len
$PUSH r6,`$LOCALS+3*$SIZE_T`($sp) ; save bsz
mr r0,r6
$PUSH r12,`$LOCALS+4*$SIZE_T`($sp)
ld $A[0][0],`8*0`(r3) ; load A[5][5]
ld $A[0][1],`8*1`(r3)
ld $A[0][2],`8*2`(r3)
ld $A[0][3],`8*3`(r3)
ld $A[0][4],`8*4`(r3)
ld $A[1][0],`8*5`(r3)
ld $A[1][1],`8*6`(r3)
ld $A[1][2],`8*7`(r3)
ld $A[1][3],`8*8`(r3)
ld $A[1][4],`8*9`(r3)
ld $A[2][0],`8*10`(r3)
ld $A[2][1],`8*11`(r3)
ld $A[2][2],`8*12`(r3)
ld $A[2][3],`8*13`(r3)
ld $A[2][4],`8*14`(r3)
ld $A[3][0],`8*15`(r3)
ld $A[3][1],`8*16`(r3)
ld $A[3][2],`8*17`(r3)
ld $A[3][3],`8*18`(r3)
ld $A[3][4],`8*19`(r3)
ld $A[4][0],`8*20`(r3)
ld $A[4][1],`8*21`(r3)
ld $A[4][2],`8*22`(r3)
ld $A[4][3],`8*23`(r3)
ld $A[4][4],`8*24`(r3)
mr r3,r4
mr r4,r5
mr r5,r0
b .Loop_absorb
.align 4
.Loop_absorb:
$UCMP r4,r5 ; len < bsz?
blt .Labsorbed
sub r4,r4,r5 ; len -= bsz
srwi r5,r5,3
$PUSH r4,`$LOCALS+2*$SIZE_T`($sp) ; save len
mtctr r5
bl dword_le_load ; *inp++
xor $A[0][0],$A[0][0],r0
bdz .Lprocess_block
bl dword_le_load ; *inp++
xor $A[0][1],$A[0][1],r0
bdz .Lprocess_block
bl dword_le_load ; *inp++
xor $A[0][2],$A[0][2],r0
bdz .Lprocess_block
bl dword_le_load ; *inp++
xor $A[0][3],$A[0][3],r0
bdz .Lprocess_block
bl dword_le_load ; *inp++
xor $A[0][4],$A[0][4],r0
bdz .Lprocess_block
bl dword_le_load ; *inp++
xor $A[1][0],$A[1][0],r0
bdz .Lprocess_block
bl dword_le_load ; *inp++
xor $A[1][1],$A[1][1],r0
bdz .Lprocess_block
bl dword_le_load ; *inp++
xor $A[1][2],$A[1][2],r0
bdz .Lprocess_block
bl dword_le_load ; *inp++
xor $A[1][3],$A[1][3],r0
bdz .Lprocess_block
bl dword_le_load ; *inp++
xor $A[1][4],$A[1][4],r0
bdz .Lprocess_block
bl dword_le_load ; *inp++
xor $A[2][0],$A[2][0],r0
bdz .Lprocess_block
bl dword_le_load ; *inp++
xor $A[2][1],$A[2][1],r0
bdz .Lprocess_block
bl dword_le_load ; *inp++
xor $A[2][2],$A[2][2],r0
bdz .Lprocess_block
bl dword_le_load ; *inp++
xor $A[2][3],$A[2][3],r0
bdz .Lprocess_block
bl dword_le_load ; *inp++
xor $A[2][4],$A[2][4],r0
bdz .Lprocess_block
bl dword_le_load ; *inp++
xor $A[3][0],$A[3][0],r0
bdz .Lprocess_block
bl dword_le_load ; *inp++
xor $A[3][1],$A[3][1],r0
bdz .Lprocess_block
bl dword_le_load ; *inp++
xor $A[3][2],$A[3][2],r0
bdz .Lprocess_block
bl dword_le_load ; *inp++
xor $A[3][3],$A[3][3],r0
bdz .Lprocess_block
bl dword_le_load ; *inp++
xor $A[3][4],$A[3][4],r0
bdz .Lprocess_block
bl dword_le_load ; *inp++
xor $A[4][0],$A[4][0],r0
bdz .Lprocess_block
bl dword_le_load ; *inp++
xor $A[4][1],$A[4][1],r0
bdz .Lprocess_block
bl dword_le_load ; *inp++
xor $A[4][2],$A[4][2],r0
bdz .Lprocess_block
bl dword_le_load ; *inp++
xor $A[4][3],$A[4][3],r0
bdz .Lprocess_block
bl dword_le_load ; *inp++
xor $A[4][4],$A[4][4],r0
.Lprocess_block:
$PUSH r3,`$LOCALS+1*$SIZE_T`($sp) ; save inp
bl KeccakF1600_int
$POP r0,`$LOCALS+4*$SIZE_T`($sp) ; pull iotas[24]
$POP r5,`$LOCALS+3*$SIZE_T`($sp) ; restore bsz
$POP r4,`$LOCALS+2*$SIZE_T`($sp) ; restore len
$POP r3,`$LOCALS+1*$SIZE_T`($sp) ; restore inp
addic r0,r0,`-8*24` ; rewind iotas
$PUSH r0,`$LOCALS+4*$SIZE_T`($sp)
b .Loop_absorb
.align 4
.Labsorbed:
$POP r3,`$LOCALS+0*$SIZE_T`($sp)
std $A[0][0],`8*0`(r3) ; return A[5][5]
std $A[0][1],`8*1`(r3)
std $A[0][2],`8*2`(r3)
std $A[0][3],`8*3`(r3)
std $A[0][4],`8*4`(r3)
std $A[1][0],`8*5`(r3)
std $A[1][1],`8*6`(r3)
std $A[1][2],`8*7`(r3)
std $A[1][3],`8*8`(r3)
std $A[1][4],`8*9`(r3)
std $A[2][0],`8*10`(r3)
std $A[2][1],`8*11`(r3)
std $A[2][2],`8*12`(r3)
std $A[2][3],`8*13`(r3)
std $A[2][4],`8*14`(r3)
std $A[3][0],`8*15`(r3)
std $A[3][1],`8*16`(r3)
std $A[3][2],`8*17`(r3)
std $A[3][3],`8*18`(r3)
std $A[3][4],`8*19`(r3)
std $A[4][0],`8*20`(r3)
std $A[4][1],`8*21`(r3)
std $A[4][2],`8*22`(r3)
std $A[4][3],`8*23`(r3)
std $A[4][4],`8*24`(r3)
mr r3,r4 ; return value
$POP r0,`$FRAME+$LRSAVE`($sp)
$POP r14,`$FRAME-$SIZE_T*18`($sp)
$POP r15,`$FRAME-$SIZE_T*17`($sp)
$POP r16,`$FRAME-$SIZE_T*16`($sp)
$POP r17,`$FRAME-$SIZE_T*15`($sp)
$POP r18,`$FRAME-$SIZE_T*14`($sp)
$POP r19,`$FRAME-$SIZE_T*13`($sp)
$POP r20,`$FRAME-$SIZE_T*12`($sp)
$POP r21,`$FRAME-$SIZE_T*11`($sp)
$POP r22,`$FRAME-$SIZE_T*10`($sp)
$POP r23,`$FRAME-$SIZE_T*9`($sp)
$POP r24,`$FRAME-$SIZE_T*8`($sp)
$POP r25,`$FRAME-$SIZE_T*7`($sp)
$POP r26,`$FRAME-$SIZE_T*6`($sp)
$POP r27,`$FRAME-$SIZE_T*5`($sp)
$POP r28,`$FRAME-$SIZE_T*4`($sp)
$POP r29,`$FRAME-$SIZE_T*3`($sp)
$POP r30,`$FRAME-$SIZE_T*2`($sp)
$POP r31,`$FRAME-$SIZE_T*1`($sp)
mtlr r0
addi $sp,$sp,$FRAME
blr
.long 0
.byte 0,12,4,1,0x80,18,4,0
.long 0
.size SHA3_absorb,.-SHA3_absorb
___
{
my ($A_flat,$out,$len,$bsz) = map("r$_",(28..31));
$code.=<<___;
.globl SHA3_squeeze
.type SHA3_squeeze,\@function
.align 5
SHA3_squeeze:
$STU $sp,`-10*$SIZE_T`($sp)
mflr r0
$PUSH r28,`6*$SIZE_T`($sp)
$PUSH r29,`7*$SIZE_T`($sp)
$PUSH r30,`8*$SIZE_T`($sp)
$PUSH r31,`9*$SIZE_T`($sp)
$PUSH r0,`10*$SIZE_T+$LRSAVE`($sp)
mr $A_flat,r3
subi r3,r3,8 ; prepare for ldu
subi $out,r4,1 ; prepare for stbu
mr $len,r5
mr $bsz,r6
b .Loop_squeeze
.align 4
.Loop_squeeze:
ldu r0,8(r3)
${UCMP}i $len,8
blt .Lsqueeze_tail
stbu r0,1($out)
srdi r0,r0,8
stbu r0,1($out)
srdi r0,r0,8
stbu r0,1($out)
srdi r0,r0,8
stbu r0,1($out)
srdi r0,r0,8
stbu r0,1($out)
srdi r0,r0,8
stbu r0,1($out)
srdi r0,r0,8
stbu r0,1($out)
srdi r0,r0,8
stbu r0,1($out)
subic. $len,$len,8
beq .Lsqueeze_done
subic. r6,r6,8
bgt .Loop_squeeze
mr r3,$A_flat
bl KeccakF1600
subi r3,$A_flat,8 ; prepare for ldu
mr r6,$bsz
b .Loop_squeeze
.align 4
.Lsqueeze_tail:
mtctr $len
.Loop_tail:
stbu r0,1($out)
srdi r0,r0,8
bdnz .Loop_tail
.Lsqueeze_done:
$POP r0,`10*$SIZE_T+$LRSAVE`($sp)
$POP r28,`6*$SIZE_T`($sp)
$POP r29,`7*$SIZE_T`($sp)
$POP r30,`8*$SIZE_T`($sp)
$POP r31,`9*$SIZE_T`($sp)
mtlr r0
addi $sp,$sp,`10*$SIZE_T`
blr
.long 0
.byte 0,12,4,1,0x80,4,4,0
.long 0
.size SHA3_squeeze,.-SHA3_squeeze
___
}
# Ugly hack here, because PPC assembler syntax seem to vary too
# much from platforms to platform...
$code.=<<___;
.align 6
PICmeup:
mflr r0
bcl 20,31,\$+4
mflr r12 ; vvvvvv "distance" between . and 1st data entry
addi r12,r12,`64-8`
mtlr r0
blr
.long 0
.byte 0,12,0x14,0,0,0,0,0
.space `64-9*4`
.type iotas,\@object
iotas:
.quad 0x0000000000000001
.quad 0x0000000000008082
.quad 0x800000000000808a
.quad 0x8000000080008000
.quad 0x000000000000808b
.quad 0x0000000080000001
.quad 0x8000000080008081
.quad 0x8000000000008009
.quad 0x000000000000008a
.quad 0x0000000000000088
.quad 0x0000000080008009
.quad 0x000000008000000a
.quad 0x000000008000808b
.quad 0x800000000000008b
.quad 0x8000000000008089
.quad 0x8000000000008003
.quad 0x8000000000008002
.quad 0x8000000000000080
.quad 0x000000000000800a
.quad 0x800000008000000a
.quad 0x8000000080008081
.quad 0x8000000000008080
.quad 0x0000000080000001
.quad 0x8000000080008008
.size iotas,.-iotas
.asciz "Keccak-1600 absorb and squeeze for PPC64, CRYPTOGAMS by <appro\@openssl.org>"
___
$code =~ s/\`([^\`]*)\`/eval $1/gem;
print $code;
close STDOUT;
crypto/sha/asm/keccak1600-s390x.pl
Executable
+561
View File
@@ -0,0 +1,561 @@
#!/usr/bin/env perl
# Copyright 2017 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License"). You may not use
# this file except in compliance with the License. You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html
#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# Keccak-1600 for s390x.
#
# June 2017.
#
# Below code is [lane complementing] KECCAK_2X implementation (see
# sha/keccak1600.c) with C[5] and D[5] held in register bank. Though
# instead of actually unrolling the loop pair-wise I simply flip
# pointers to T[][] and A[][] at the end of round. Since number of
# rounds is even, last round writes to A[][] and everything works out.
# In the nutshell it's transliteration of x86_64 module, because both
# architectures have similar capabilities/limitations. Performance
# measurement is problematic as I don't have access to an idle system.
# It looks like z13 processes one byte [out of long message] in ~14
# cycles. At least the result is consistent with estimate based on
# amount of instruction and assumed instruction issue rate. It's ~2.5x
# faster than compiler-generated code.
$flavour = shift;
if ($flavour =~ /3[12]/) {
$SIZE_T=4;
$g="";
} else {
$SIZE_T=8;
$g="g";
}
while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {}
open STDOUT,">$output";
my @A = map([ 8*$_, 8*($_+1), 8*($_+2), 8*($_+3), 8*($_+4) ], (0,5,10,15,20));
my @C = map("%r$_",(0,1,5..7));
my @D = map("%r$_",(8..12));
my @T = map("%r$_",(13..14));
my ($src,$dst,$iotas) = map("%r$_",(2..4));
my $sp = "%r15";
$stdframe=16*$SIZE_T+4*8;
$frame=$stdframe+25*8;
my @rhotates = ([ 0, 1, 62, 28, 27 ],
[ 36, 44, 6, 55, 20 ],
[ 3, 10, 43, 25, 39 ],
[ 41, 45, 15, 21, 8 ],
[ 18, 2, 61, 56, 14 ]);
{ my @C = @C; # copy, because we mess them up...
my @D = @D;
$code.=<<___;
.text
.type __KeccakF1600,\@function
.align 32
__KeccakF1600:
st${g} %r14,$SIZE_T*14($sp)
lg @C[0],$A[4][0]($src)
lg @C[1],$A[4][1]($src)
lg @C[2],$A[4][2]($src)
lg @C[3],$A[4][3]($src)
lg @C[4],$A[4][4]($src)
larl $iotas,iotas
j .Loop
.align 16
.Loop:
lg @D[0],$A[0][0]($src)
lg @D[1],$A[1][1]($src)
lg @D[2],$A[2][2]($src)
lg @D[3],$A[3][3]($src)
xgr @C[0],@D[0]
xg @C[1],$A[0][1]($src)
xg @C[2],$A[0][2]($src)
xg @C[3],$A[0][3]($src)
lgr @D[4],@C[4]
xg @C[4],$A[0][4]($src)
xg @C[0],$A[1][0]($src)
xgr @C[1],@D[1]
xg @C[2],$A[1][2]($src)
xg @C[3],$A[1][3]($src)
xg @C[4],$A[1][4]($src)
xg @C[0],$A[2][0]($src)
xg @C[1],$A[2][1]($src)
xgr @C[2],@D[2]
xg @C[3],$A[2][3]($src)
xg @C[4],$A[2][4]($src)
xg @C[0],$A[3][0]($src)
xg @C[1],$A[3][1]($src)
xg @C[2],$A[3][2]($src)
xgr @C[3],@D[3]
xg @C[4],$A[3][4]($src)
lgr @T[0],@C[2]
rllg @C[2],@C[2],1
xgr @C[2],@C[0] # D[1] = ROL64(C[2], 1) ^ C[0]
rllg @C[0],@C[0],1
xgr @C[0],@C[3] # D[4] = ROL64(C[0], 1) ^ C[3]
rllg @C[3],@C[3],1
xgr @C[3],@C[1] # D[2] = ROL64(C[3], 1) ^ C[1]
rllg @C[1],@C[1],1
xgr @C[1],@C[4] # D[0] = ROL64(C[1], 1) ^ C[4]
rllg @C[4],@C[4],1
xgr @C[4],@T[0] # D[3] = ROL64(C[4], 1) ^ C[2]
___
(@D[0..4], @C) = (@C[1..4,0], @D);
$code.=<<___;
xgr @C[1],@D[1]
xgr @C[2],@D[2]
xgr @C[3],@D[3]
rllg @C[1],@C[1],$rhotates[1][1]
xgr @C[4],@D[4]
rllg @C[2],@C[2],$rhotates[2][2]
xgr @C[0],@D[0]
lgr @T[0],@C[1]
ogr @C[1],@C[2]
rllg @C[3],@C[3],$rhotates[3][3]
xgr @C[1],@C[0] # C[0] ^ ( C[1] | C[2])
rllg @C[4],@C[4],$rhotates[4][4]
xg @C[1],0($iotas)
la $iotas,8($iotas)
stg @C[1],$A[0][0]($dst) # R[0][0] = C[0] ^ ( C[1] | C[2]) ^ iotas[i]
lgr @T[1],@C[4]
ngr @C[4],@C[3]
lghi @C[1],-1 # no 'not' instruction :-(
xgr @C[4],@C[2] # C[2] ^ ( C[4] & C[3])
xgr @C[2],@C[1] # not @C[2]
stg @C[4],$A[0][2]($dst) # R[0][2] = C[2] ^ ( C[4] & C[3])
ogr @C[2],@C[3]
xgr @C[2],@T[0] # C[1] ^ (~C[2] | C[3])
ngr @T[0],@C[0]
stg @C[2],$A[0][1]($dst) # R[0][1] = C[1] ^ (~C[2] | C[3])
xgr @T[0],@T[1] # C[4] ^ ( C[1] & C[0])
ogr @T[1],@C[0]
stg @T[0],$A[0][4]($dst) # R[0][4] = C[4] ^ ( C[1] & C[0])
xgr @T[1],@C[3] # C[3] ^ ( C[4] | C[0])
stg @T[1],$A[0][3]($dst) # R[0][3] = C[3] ^ ( C[4] | C[0])
lg @C[0],$A[0][3]($src)
lg @C[4],$A[4][2]($src)
lg @C[3],$A[3][1]($src)
lg @C[1],$A[1][4]($src)
lg @C[2],$A[2][0]($src)
xgr @C[0],@D[3]
xgr @C[4],@D[2]
rllg @C[0],@C[0],$rhotates[0][3]
xgr @C[3],@D[1]
rllg @C[4],@C[4],$rhotates[4][2]
xgr @C[1],@D[4]
rllg @C[3],@C[3],$rhotates[3][1]
xgr @C[2],@D[0]
lgr @T[0],@C[0]
ogr @C[0],@C[4]
rllg @C[1],@C[1],$rhotates[1][4]
xgr @C[0],@C[3] # C[3] ^ (C[0] | C[4])
rllg @C[2],@C[2],$rhotates[2][0]
stg @C[0],$A[1][3]($dst) # R[1][3] = C[3] ^ (C[0] | C[4])
lgr @T[1],@C[1]
ngr @C[1],@T[0]
lghi @C[0],-1 # no 'not' instruction :-(
xgr @C[1],@C[4] # C[4] ^ (C[1] & C[0])
xgr @C[4],@C[0] # not @C[4]
stg @C[1],$A[1][4]($dst) # R[1][4] = C[4] ^ (C[1] & C[0])
ogr @C[4],@C[3]
xgr @C[4],@C[2] # C[2] ^ (~C[4] | C[3])
ngr @C[3],@C[2]
stg @C[4],$A[1][2]($dst) # R[1][2] = C[2] ^ (~C[4] | C[3])
xgr @C[3],@T[1] # C[1] ^ (C[3] & C[2])
ogr @T[1],@C[2]
stg @C[3],$A[1][1]($dst) # R[1][1] = C[1] ^ (C[3] & C[2])
xgr @T[1],@T[0] # C[0] ^ (C[1] | C[2])
stg @T[1],$A[1][0]($dst) # R[1][0] = C[0] ^ (C[1] | C[2])
lg @C[2],$A[2][3]($src)
lg @C[3],$A[3][4]($src)
lg @C[1],$A[1][2]($src)
lg @C[4],$A[4][0]($src)
lg @C[0],$A[0][1]($src)
xgr @C[2],@D[3]
xgr @C[3],@D[4]
rllg @C[2],@C[2],$rhotates[2][3]
xgr @C[1],@D[2]
rllg @C[3],@C[3],$rhotates[3][4]
xgr @C[4],@D[0]
rllg @C[1],@C[1],$rhotates[1][2]
xgr @C[0],@D[1]
lgr @T[0],@C[2]
ngr @C[2],@C[3]
rllg @C[4],@C[4],$rhotates[4][0]
xgr @C[2],@C[1] # C[1] ^ ( C[2] & C[3])
lghi @T[1],-1 # no 'not' instruction :-(
stg @C[2],$A[2][1]($dst) # R[2][1] = C[1] ^ ( C[2] & C[3])
xgr @C[3],@T[1] # not @C[3]
lgr @T[1],@C[4]
ngr @C[4],@C[3]
rllg @C[0],@C[0],$rhotates[0][1]
xgr @C[4],@T[0] # C[2] ^ ( C[4] & ~C[3])
ogr @T[0],@C[1]
stg @C[4],$A[2][2]($dst) # R[2][2] = C[2] ^ ( C[4] & ~C[3])
xgr @T[0],@C[0] # C[0] ^ ( C[2] | C[1])
ngr @C[1],@C[0]
stg @T[0],$A[2][0]($dst) # R[2][0] = C[0] ^ ( C[2] | C[1])
xgr @C[1],@T[1] # C[4] ^ ( C[1] & C[0])
ogr @C[0],@T[1]
stg @C[1],$A[2][4]($dst) # R[2][4] = C[4] ^ ( C[1] & C[0])
xgr @C[0],@C[3] # ~C[3] ^ ( C[0] | C[4])
stg @C[0],$A[2][3]($dst) # R[2][3] = ~C[3] ^ ( C[0] | C[4])
lg @C[2],$A[2][1]($src)
lg @C[3],$A[3][2]($src)
lg @C[1],$A[1][0]($src)
lg @C[4],$A[4][3]($src)
lg @C[0],$A[0][4]($src)
xgr @C[2],@D[1]
xgr @C[3],@D[2]
rllg @C[2],@C[2],$rhotates[2][1]
xgr @C[1],@D[0]
rllg @C[3],@C[3],$rhotates[3][2]
xgr @C[4],@D[3]
rllg @C[1],@C[1],$rhotates[1][0]
xgr @C[0],@D[4]
rllg @C[4],@C[4],$rhotates[4][3]
lgr @T[0],@C[2]
ogr @C[2],@C[3]
lghi @T[1],-1 # no 'not' instruction :-(
xgr @C[2],@C[1] # C[1] ^ ( C[2] | C[3])
xgr @C[3],@T[1] # not @C[3]
stg @C[2],$A[3][1]($dst) # R[3][1] = C[1] ^ ( C[2] | C[3])
lgr @T[1],@C[4]
ogr @C[4],@C[3]
rllg @C[0],@C[0],$rhotates[0][4]
xgr @C[4],@T[0] # C[2] ^ ( C[4] | ~C[3])
ngr @T[0],@C[1]
stg @C[4],$A[3][2]($dst) # R[3][2] = C[2] ^ ( C[4] | ~C[3])
xgr @T[0],@C[0] # C[0] ^ ( C[2] & C[1])
ogr @C[1],@C[0]
stg @T[0],$A[3][0]($dst) # R[3][0] = C[0] ^ ( C[2] & C[1])
xgr @C[1],@T[1] # C[4] ^ ( C[1] | C[0])
ngr @C[0],@T[1]
stg @C[1],$A[3][4]($dst) # R[3][4] = C[4] ^ ( C[1] | C[0])
xgr @C[0],@C[3] # ~C[3] ^ ( C[0] & C[4])
stg @C[0],$A[3][3]($dst) # R[3][3] = ~C[3] ^ ( C[0] & C[4])
xg @D[2],$A[0][2]($src)
xg @D[3],$A[1][3]($src)
xg @D[1],$A[4][1]($src)
xg @D[4],$A[2][4]($src)
xgr $dst,$src # xchg $dst,$src
rllg @D[2],@D[2],$rhotates[0][2]
xg @D[0],$A[3][0]($src)
rllg @D[3],@D[3],$rhotates[1][3]
xgr $src,$dst
rllg @D[1],@D[1],$rhotates[4][1]
xgr $dst,$src
rllg @D[4],@D[4],$rhotates[2][4]
___
@C = @D[2..4,0,1];
$code.=<<___;
lgr @T[0],@C[0]
ngr @C[0],@C[1]
lghi @T[1],-1 # no 'not' instruction :-(
xgr @C[0],@C[4] # C[4] ^ ( C[0] & C[1])
xgr @C[1],@T[1] # not @C[1]
stg @C[0],$A[4][4]($src) # R[4][4] = C[4] ^ ( C[0] & C[1])
lgr @T[1],@C[2]
ngr @C[2],@C[1]
rllg @D[0],@D[0],$rhotates[3][0]
xgr @C[2],@T[0] # C[0] ^ ( C[2] & ~C[1])
ogr @T[0],@C[4]
stg @C[2],$A[4][0]($src) # R[4][0] = C[0] ^ ( C[2] & ~C[1])
xgr @T[0],@C[3] # C[3] ^ ( C[0] | C[4])
ngr @C[4],@C[3]
stg @T[0],$A[4][3]($src) # R[4][3] = C[3] ^ ( C[0] | C[4])
xgr @C[4],@T[1] # C[2] ^ ( C[4] & C[3])
ogr @C[3],@T[1]
stg @C[4],$A[4][2]($src) # R[4][2] = C[2] ^ ( C[4] & C[3])
xgr @C[3],@C[1] # ~C[1] ^ ( C[2] | C[3])
lgr @C[1],@C[0] # harmonize with the loop top
lgr @C[0],@T[0]
stg @C[3],$A[4][1]($src) # R[4][1] = ~C[1] ^ ( C[2] | C[3])
tmll $iotas,255
jnz .Loop
l${g} %r14,$SIZE_T*14($sp)
br %r14
.size __KeccakF1600,.-__KeccakF1600
___
}
{
$code.=<<___;
.globl KeccakF1600
.type KeccakF1600,\@function
.align 32
KeccakF1600:
.LKeccakF1600:
lghi %r1,-$frame
stm${g} %r6,%r15,$SIZE_T*6($sp)
lgr %r0,$sp
la $sp,0(%r1,$sp)
st${g} %r0,0($sp)
lghi @D[0],-1 # no 'not' instruction :-(
lghi @D[1],-1
lghi @D[2],-1
lghi @D[3],-1
lghi @D[4],-1
lghi @T[0],-1
xg @D[0],$A[0][1]($src)
xg @D[1],$A[0][2]($src)
xg @D[2],$A[1][3]($src)
xg @D[3],$A[2][2]($src)
xg @D[4],$A[3][2]($src)
xg @T[0],$A[4][0]($src)
stmg @D[0],@D[1],$A[0][1]($src)
stg @D[2],$A[1][3]($src)
stg @D[3],$A[2][2]($src)
stg @D[4],$A[3][2]($src)
stg @T[0],$A[4][0]($src)
la $dst,$stdframe($sp)
bras %r14,__KeccakF1600
lghi @D[0],-1 # no 'not' instruction :-(
lghi @D[1],-1
lghi @D[2],-1
lghi @D[3],-1
lghi @D[4],-1
lghi @T[0],-1
xg @D[0],$A[0][1]($src)
xg @D[1],$A[0][2]($src)
xg @D[2],$A[1][3]($src)
xg @D[3],$A[2][2]($src)
xg @D[4],$A[3][2]($src)
xg @T[0],$A[4][0]($src)
stmg @D[0],@D[1],$A[0][1]($src)
stg @D[2],$A[1][3]($src)
stg @D[3],$A[2][2]($src)
stg @D[4],$A[3][2]($src)
stg @T[0],$A[4][0]($src)
lm${g} %r6,%r15,$frame+6*$SIZE_T($sp)
br %r14
.size KeccakF1600,.-KeccakF1600
___
}
{ my ($A_flat,$inp,$len,$bsz) = map("%r$_",(2..5));
$code.=<<___;
.globl SHA3_absorb
.type SHA3_absorb,\@function
.align 32
SHA3_absorb:
lghi %r1,-$frame
stm${g} %r5,%r15,$SIZE_T*5($sp)
lgr %r0,$sp
la $sp,0(%r1,$sp)
st${g} %r0,0($sp)
lghi @D[0],-1 # no 'not' instruction :-(
lghi @D[1],-1
lghi @D[2],-1
lghi @D[3],-1
lghi @D[4],-1
lghi @T[0],-1
xg @D[0],$A[0][1]($src)
xg @D[1],$A[0][2]($src)
xg @D[2],$A[1][3]($src)
xg @D[3],$A[2][2]($src)
xg @D[4],$A[3][2]($src)
xg @T[0],$A[4][0]($src)
stmg @D[0],@D[1],$A[0][1]($src)
stg @D[2],$A[1][3]($src)
stg @D[3],$A[2][2]($src)
stg @D[4],$A[3][2]($src)
stg @T[0],$A[4][0]($src)
.Loop_absorb:
cl${g}r $len,$bsz
jl .Ldone_absorb
srl${g} $bsz,3
la %r1,0($A_flat)
.Lblock_absorb:
lrvg %r0,0($inp)
la $inp,8($inp)
xg %r0,0(%r1)
la %r1,8(%r1)
a${g}hi $len,-8
stg %r0,-8(%r1)
brct $bsz,.Lblock_absorb
stm${g} $inp,$len,$frame+3*$SIZE_T($sp)
la $dst,$stdframe($sp)
bras %r14,__KeccakF1600
lm${g} $inp,$bsz,$frame+3*$SIZE_T($sp)
j .Loop_absorb
.align 16
.Ldone_absorb:
lghi @D[0],-1 # no 'not' instruction :-(
lghi @D[1],-1
lghi @D[2],-1
lghi @D[3],-1
lghi @D[4],-1
lghi @T[0],-1
xg @D[0],$A[0][1]($src)
xg @D[1],$A[0][2]($src)
xg @D[2],$A[1][3]($src)
xg @D[3],$A[2][2]($src)
xg @D[4],$A[3][2]($src)
xg @T[0],$A[4][0]($src)
stmg @D[0],@D[1],$A[0][1]($src)
stg @D[2],$A[1][3]($src)
stg @D[3],$A[2][2]($src)
stg @D[4],$A[3][2]($src)
stg @T[0],$A[4][0]($src)
lgr %r2,$len # return value
lm${g} %r6,%r15,$frame+6*$SIZE_T($sp)
br %r14
.size SHA3_absorb,.-SHA3_absorb
___
}
{ my ($A_flat,$out,$len,$bsz) = map("%r$_",(2..5));
$code.=<<___;
.globl SHA3_squeeze
.type SHA3_squeeze,\@function
.align 32
SHA3_squeeze:
srl${g} $bsz,3
st${g} %r14,2*$SIZE_T($sp)
lghi %r14,8
st${g} $bsz,5*$SIZE_T($sp)
la %r1,0($A_flat)
j .Loop_squeeze
.align 16
.Loop_squeeze:
cl${g}r $len,%r14
jl .Ltail_squeeze
lrvg %r0,0(%r1)
la %r1,8(%r1)
stg %r0,0($out)
la $out,8($out)
a${g}hi $len,-8 # len -= 8
jz .Ldone_squeeze
brct $bsz,.Loop_squeeze # bsz--
stm${g} $out,$len,3*$SIZE_T($sp)
bras %r14,.LKeccakF1600
lm${g} $out,$bsz,3*$SIZE_T($sp)
lghi %r14,8
la %r1,0($A_flat)
j .Loop_squeeze
.Ltail_squeeze:
lg %r0,0(%r1)
.Loop_tail_squeeze:
stc %r0,0($out)
la $out,1($out)
srlg %r0,8
brct $len,.Loop_tail_squeeze
.Ldone_squeeze:
l${g} %r14,2*$SIZE_T($sp)
br %r14
.size SHA3_squeeze,.-SHA3_squeeze
___
}
$code.=<<___;
.align 256
.quad 0,0,0,0,0,0,0,0
.type iotas,\@object
iotas:
.quad 0x0000000000000001
.quad 0x0000000000008082
.quad 0x800000000000808a
.quad 0x8000000080008000
.quad 0x000000000000808b
.quad 0x0000000080000001
.quad 0x8000000080008081
.quad 0x8000000000008009
.quad 0x000000000000008a
.quad 0x0000000000000088
.quad 0x0000000080008009
.quad 0x000000008000000a
.quad 0x000000008000808b
.quad 0x800000000000008b
.quad 0x8000000000008089
.quad 0x8000000000008003
.quad 0x8000000000008002
.quad 0x8000000000000080
.quad 0x000000000000800a
.quad 0x800000008000000a
.quad 0x8000000080008081
.quad 0x8000000000008080
.quad 0x0000000080000001
.quad 0x8000000080008008
.size iotas,.-iotas
.asciz "Keccak-1600 absorb and squeeze for s390x, CRYPTOGAMS by <appro\@openssl.org>"
___
# unlike 32-bit shift 64-bit one takes three arguments
$code =~ s/(srlg\s+)(%r[0-9]+),/$1$2,$2,/gm;
print $code;
close STDOUT;
crypto/sha/asm/keccak1600-x86_64.pl
Executable
+608
View File
@@ -0,0 +1,608 @@
#!/usr/bin/env perl
# Copyright 2017 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License"). You may not use
# this file except in compliance with the License. You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html
#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# Keccak-1600 for x86_64.
#
# June 2017.
#
# Below code is [lane complementing] KECCAK_2X implementation (see
# sha/keccak1600.c) with C[5] and D[5] held in register bank. Though
# instead of actually unrolling the loop pair-wise I simply flip
# pointers to T[][] and A[][] at the end of round. Since number of
# rounds is even, last round writes to A[][] and everything works out.
# How does it compare to x86_64 assembly module in Keccak Code Package?
# Depending on processor it's either as fast or faster by up to 15%...
#
########################################################################
# Numbers are cycles per processed byte out of large message.
#
# r=1088(*)
#
# P4 25.8
# Core 2 12.9
# Westmere 13.7
# Sandy Bridge 12.9(**)
# Haswell 9.6
# Skylake 9.4
# Silvermont 22.8
# Goldmont 15.8
# VIA Nano 17.3
# Sledgehammer 13.3
# Bulldozer 16.5
# Ryzen 8.8
#
# (*) Corresponds to SHA3-256. Improvement over compiler-generate
# varies a lot, most commont coefficient is 15% in comparison to
# gcc-5.x, 50% for gcc-4.x, 90% for gcc-3.x.
# (**) Sandy Bridge has broken rotate instruction. Performance can be
# improved by 14% by replacing rotates with double-precision
# shift with same register as source and destination.
$flavour = shift;
$output = shift;
if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
die "can't locate x86_64-xlate.pl";
open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";
*STDOUT=*OUT;
my @A = map([ 8*$_-100, 8*($_+1)-100, 8*($_+2)-100,
8*($_+3)-100, 8*($_+4)-100 ], (0,5,10,15,20));
my @C = ("%rax","%rbx","%rcx","%rdx","%rbp");
my @D = map("%r$_",(8..12));
my @T = map("%r$_",(13..14));
my $iotas = "%r15";
my @rhotates = ([ 0, 1, 62, 28, 27 ],
[ 36, 44, 6, 55, 20 ],
[ 3, 10, 43, 25, 39 ],
[ 41, 45, 15, 21, 8 ],
[ 18, 2, 61, 56, 14 ]);
$code.=<<___;
.text
.type __KeccakF1600,\@function
.align 32
__KeccakF1600:
mov $A[4][0](%rdi),@C[0]
mov $A[4][1](%rdi),@C[1]
mov $A[4][2](%rdi),@C[2]
mov $A[4][3](%rdi),@C[3]
mov $A[4][4](%rdi),@C[4]
jmp .Loop
.align 32
.Loop:
mov $A[0][0](%rdi),@D[0]
mov $A[1][1](%rdi),@D[1]
mov $A[2][2](%rdi),@D[2]
mov $A[3][3](%rdi),@D[3]
xor $A[0][2](%rdi),@C[2]
xor $A[0][3](%rdi),@C[3]
xor @D[0], @C[0]
xor $A[0][1](%rdi),@C[1]
xor $A[1][2](%rdi),@C[2]
xor $A[1][0](%rdi),@C[0]
mov @C[4],@D[4]
xor $A[0][4](%rdi),@C[4]
xor @D[2], @C[2]
xor $A[2][0](%rdi),@C[0]
xor $A[1][3](%rdi),@C[3]
xor @D[1], @C[1]
xor $A[1][4](%rdi),@C[4]
xor $A[3][2](%rdi),@C[2]
xor $A[3][0](%rdi),@C[0]
xor $A[2][3](%rdi),@C[3]
xor $A[2][1](%rdi),@C[1]
xor $A[2][4](%rdi),@C[4]
mov @C[2],@T[0]
rol \$1,@C[2]
xor @C[0],@C[2] # D[1] = ROL64(C[2], 1) ^ C[0]
xor @D[3], @C[3]
rol \$1,@C[0]
xor @C[3],@C[0] # D[4] = ROL64(C[0], 1) ^ C[3]
xor $A[3][1](%rdi),@C[1]
rol \$1,@C[3]
xor @C[1],@C[3] # D[2] = ROL64(C[3], 1) ^ C[1]
xor $A[3][4](%rdi),@C[4]
rol \$1,@C[1]
xor @C[4],@C[1] # D[0] = ROL64(C[1], 1) ^ C[4]
rol \$1,@C[4]
xor @T[0],@C[4] # D[3] = ROL64(C[4], 1) ^ C[2]
___
(@D[0..4], @C) = (@C[1..4,0], @D);
$code.=<<___;
xor @D[1],@C[1]
xor @D[2],@C[2]
rol \$$rhotates[1][1],@C[1]
xor @D[3],@C[3]
xor @D[4],@C[4]
rol \$$rhotates[2][2],@C[2]
xor @D[0],@C[0]
mov @C[1],@T[0]
rol \$$rhotates[3][3],@C[3]
or @C[2],@C[1]
xor @C[0],@C[1] # C[0] ^ ( C[1] | C[2])
rol \$$rhotates[4][4],@C[4]
xor ($iotas),@C[1]
lea 8($iotas),$iotas
mov @C[4],@T[1]
and @C[3],@C[4]
mov @C[1],$A[0][0](%rsi) # R[0][0] = C[0] ^ ( C[1] | C[2]) ^ iotas[i]
xor @C[2],@C[4] # C[2] ^ ( C[4] & C[3])
not @C[2]
mov @C[4],$A[0][2](%rsi) # R[0][2] = C[2] ^ ( C[4] & C[3])
or @C[3],@C[2]
mov $A[4][2](%rdi),@C[4]
xor @T[0],@C[2] # C[1] ^ (~C[2] | C[3])
mov @C[2],$A[0][1](%rsi) # R[0][1] = C[1] ^ (~C[2] | C[3])
and @C[0],@T[0]
mov $A[1][4](%rdi),@C[1]
xor @T[1],@T[0] # C[4] ^ ( C[1] & C[0])
mov $A[2][0](%rdi),@C[2]
mov @T[0],$A[0][4](%rsi) # R[0][4] = C[4] ^ ( C[1] & C[0])
or @C[0],@T[1]
mov $A[0][3](%rdi),@C[0]
xor @C[3],@T[1] # C[3] ^ ( C[4] | C[0])
mov $A[3][1](%rdi),@C[3]
mov @T[1],$A[0][3](%rsi) # R[0][3] = C[3] ^ ( C[4] | C[0])
xor @D[3],@C[0]
xor @D[2],@C[4]
rol \$$rhotates[0][3],@C[0]
xor @D[1],@C[3]
xor @D[4],@C[1]
rol \$$rhotates[4][2],@C[4]
rol \$$rhotates[3][1],@C[3]
xor @D[0],@C[2]
rol \$$rhotates[1][4],@C[1]
mov @C[0],@T[0]
or @C[4],@C[0]
rol \$$rhotates[2][0],@C[2]
xor @C[3],@C[0] # C[3] ^ (C[0] | C[4])
mov @C[0],$A[1][3](%rsi) # R[1][3] = C[3] ^ (C[0] | C[4])
mov @C[1],@T[1]
and @T[0],@C[1]
mov $A[0][1](%rdi),@C[0]
xor @C[4],@C[1] # C[4] ^ (C[1] & C[0])
not @C[4]
mov @C[1],$A[1][4](%rsi) # R[1][4] = C[4] ^ (C[1] & C[0])
or @C[3],@C[4]
mov $A[1][2](%rdi),@C[1]
xor @C[2],@C[4] # C[2] ^ (~C[4] | C[3])
mov @C[4],$A[1][2](%rsi) # R[1][2] = C[2] ^ (~C[4] | C[3])
and @C[2],@C[3]
mov $A[4][0](%rdi),@C[4]
xor @T[1],@C[3] # C[1] ^ (C[3] & C[2])
mov @C[3],$A[1][1](%rsi) # R[1][1] = C[1] ^ (C[3] & C[2])
or @C[2],@T[1]
mov $A[2][3](%rdi),@C[2]
xor @T[0],@T[1] # C[0] ^ (C[1] | C[2])
mov $A[3][4](%rdi),@C[3]
mov @T[1],$A[1][0](%rsi) # R[1][0] = C[0] ^ (C[1] | C[2])
xor @D[3],@C[2]
xor @D[4],@C[3]
rol \$$rhotates[2][3],@C[2]
xor @D[2],@C[1]
rol \$$rhotates[3][4],@C[3]
xor @D[0],@C[4]
rol \$$rhotates[1][2],@C[1]
xor @D[1],@C[0]
rol \$$rhotates[4][0],@C[4]
mov @C[2],@T[0]
and @C[3],@C[2]
rol \$$rhotates[0][1],@C[0]
not @C[3]
xor @C[1],@C[2] # C[1] ^ ( C[2] & C[3])
mov @C[2],$A[2][1](%rsi) # R[2][1] = C[1] ^ ( C[2] & C[3])
mov @C[4],@T[1]
and @C[3],@C[4]
mov $A[2][1](%rdi),@C[2]
xor @T[0],@C[4] # C[2] ^ ( C[4] & ~C[3])
mov @C[4],$A[2][2](%rsi) # R[2][2] = C[2] ^ ( C[4] & ~C[3])
or @C[1],@T[0]
mov $A[4][3](%rdi),@C[4]
xor @C[0],@T[0] # C[0] ^ ( C[2] | C[1])
mov @T[0],$A[2][0](%rsi) # R[2][0] = C[0] ^ ( C[2] | C[1])
and @C[0],@C[1]
xor @T[1],@C[1] # C[4] ^ ( C[1] & C[0])
mov @C[1],$A[2][4](%rsi) # R[2][4] = C[4] ^ ( C[1] & C[0])
or @C[0],@T[1]
mov $A[1][0](%rdi),@C[1]
xor @C[3],@T[1] # ~C[3] ^ ( C[0] | C[4])
mov $A[3][2](%rdi),@C[3]
mov @T[1],$A[2][3](%rsi) # R[2][3] = ~C[3] ^ ( C[0] | C[4])
mov $A[0][4](%rdi),@C[0]
xor @D[1],@C[2]
xor @D[2],@C[3]
rol \$$rhotates[2][1],@C[2]
xor @D[0],@C[1]
rol \$$rhotates[3][2],@C[3]
xor @D[3],@C[4]
rol \$$rhotates[1][0],@C[1]
xor @D[4],@C[0]
rol \$$rhotates[4][3],@C[4]
mov @C[2],@T[0]
or @C[3],@C[2]
rol \$$rhotates[0][4],@C[0]
not @C[3]
xor @C[1],@C[2] # C[1] ^ ( C[2] | C[3])
mov @C[2],$A[3][1](%rsi) # R[3][1] = C[1] ^ ( C[2] | C[3])
mov @C[4],@T[1]
or @C[3],@C[4]
xor @T[0],@C[4] # C[2] ^ ( C[4] | ~C[3])
mov @C[4],$A[3][2](%rsi) # R[3][2] = C[2] ^ ( C[4] | ~C[3])
and @C[1],@T[0]
xor @C[0],@T[0] # C[0] ^ ( C[2] & C[1])
mov @T[0],$A[3][0](%rsi) # R[3][0] = C[0] ^ ( C[2] & C[1])
or @C[0],@C[1]
xor @T[1],@C[1] # C[4] ^ ( C[1] | C[0])
mov @C[1],$A[3][4](%rsi) # R[3][4] = C[4] ^ ( C[1] | C[0])
and @T[1],@C[0]
xor @C[3],@C[0] # ~C[3] ^ ( C[0] & C[4])
mov @C[0],$A[3][3](%rsi) # R[3][3] = ~C[3] ^ ( C[0] & C[4])
xor $A[0][2](%rdi),@D[2]
xor $A[1][3](%rdi),@D[3]
rol \$$rhotates[0][2],@D[2]
xor $A[4][1](%rdi),@D[1]
rol \$$rhotates[1][3],@D[3]
xor $A[2][4](%rdi),@D[4]
rol \$$rhotates[4][1],@D[1]
xor $A[3][0](%rdi),@D[0]
xchg %rsi,%rdi
rol \$$rhotates[2][4],@D[4]
rol \$$rhotates[3][0],@D[0]
___
@C = @D[2..4,0,1];
$code.=<<___;
mov @C[0],@T[0]
and @C[1],@C[0]
not @C[1]
xor @C[4],@C[0] # C[4] ^ ( C[0] & C[1])
mov @C[0],$A[4][4](%rdi) # R[4][4] = C[4] ^ ( C[0] & C[1])
mov @C[2],@T[1]
and @C[1],@C[2]
xor @T[0],@C[2] # C[0] ^ ( C[2] & ~C[1])
mov @C[2],$A[4][0](%rdi) # R[4][0] = C[0] ^ ( C[2] & ~C[1])
or @C[4],@T[0]
xor @C[3],@T[0] # C[3] ^ ( C[0] | C[4])
mov @T[0],$A[4][3](%rdi) # R[4][3] = C[3] ^ ( C[0] | C[4])
and @C[3],@C[4]
xor @T[1],@C[4] # C[2] ^ ( C[4] & C[3])
mov @C[4],$A[4][2](%rdi) # R[4][2] = C[2] ^ ( C[4] & C[3])
or @T[1],@C[3]
xor @C[1],@C[3] # ~C[1] ^ ( C[2] | C[3])
mov @C[3],$A[4][1](%rdi) # R[4][1] = ~C[1] ^ ( C[2] | C[3])
mov @C[0],@C[1] # harmonize with the loop top
mov @T[0],@C[0]
test \$255,$iotas
jnz .Loop
lea -192($iotas),$iotas # rewind iotas
ret
.size __KeccakF1600,.-__KeccakF1600
.globl KeccakF1600
.type KeccakF1600,\@function
.align 32
KeccakF1600:
.cfi_startproc
push %rbx
.cfi_push %rbx
push %rbp
.cfi_push %rbp
push %r12
.cfi_push %r12
push %r13
.cfi_push %r13
push %r14
.cfi_push %r14
push %r15
.cfi_push %r15
lea 100(%rdi),%rdi # size optimization
sub \$200,%rsp
.cfi_adjust_cfa_offset 200
notq $A[0][1](%rdi)
notq $A[0][2](%rdi)
notq $A[1][3](%rdi)
notq $A[2][2](%rdi)
notq $A[3][2](%rdi)
notq $A[4][0](%rdi)
lea iotas(%rip),$iotas
lea 100(%rsp),%rsi # size optimization
call __KeccakF1600
notq $A[0][1](%rdi)
notq $A[0][2](%rdi)
notq $A[1][3](%rdi)
notq $A[2][2](%rdi)
notq $A[3][2](%rdi)
notq $A[4][0](%rdi)
lea -100(%rdi),%rdi # preserve A[][]
add \$200,%rsp
.cfi_adjust_cfa_offset -200
pop %r15
.cfi_pop %r15
pop %r14
.cfi_pop %r14
pop %r13
.cfi_pop %r13
pop %r12
.cfi_pop %r12
pop %rbp
.cfi_pop %rbp
pop %rbx
.cfi_pop %rbx
ret
.cfi_endproc
.size KeccakF1600,.-KeccakF1600
___
{ my ($A_flat,$inp,$len,$bsz) = ("%rdi","%rsi","%rdx","%rcx");
($A_flat,$inp) = ("%r8","%r9");
$code.=<<___;
.globl SHA3_absorb
.type SHA3_absorb,\@function
.align 32
SHA3_absorb:
.cfi_startproc
push %rbx
.cfi_push %rbx
push %rbp
.cfi_push %rbp
push %r12
.cfi_push %r12
push %r13
.cfi_push %r13
push %r14
.cfi_push %r14
push %r15
.cfi_push %r15
lea 100(%rdi),%rdi # size optimization
sub \$232,%rsp
.cfi_adjust_cfa_offset 232
mov %rsi,$inp
lea 100(%rsp),%rsi # size optimization
notq $A[0][1](%rdi)
notq $A[0][2](%rdi)
notq $A[1][3](%rdi)
notq $A[2][2](%rdi)
notq $A[3][2](%rdi)
notq $A[4][0](%rdi)
lea iotas(%rip),$iotas
mov $bsz,216-100(%rsi) # save bsz
.Loop_absorb:
cmp $bsz,$len
jc .Ldone_absorb
shr \$3,$bsz
lea -100(%rdi),$A_flat
.Lblock_absorb:
mov ($inp),%rax
lea 8($inp),$inp
xor ($A_flat),%rax
lea 8($A_flat),$A_flat
sub \$8,$len
mov %rax,-8($A_flat)
sub \$1,$bsz
jnz .Lblock_absorb
mov $inp,200-100(%rsi) # save inp
mov $len,208-100(%rsi) # save len
call __KeccakF1600
mov 200-100(%rsi),$inp # pull inp
mov 208-100(%rsi),$len # pull len
mov 216-100(%rsi),$bsz # pull bsz
jmp .Loop_absorb
.align 32
.Ldone_absorb:
mov $len,%rax # return value
notq $A[0][1](%rdi)
notq $A[0][2](%rdi)
notq $A[1][3](%rdi)
notq $A[2][2](%rdi)
notq $A[3][2](%rdi)
notq $A[4][0](%rdi)
add \$232,%rsp
.cfi_adjust_cfa_offset -232
pop %r15
.cfi_pop %r15
pop %r14
.cfi_pop %r14
pop %r13
.cfi_pop %r13
pop %r12
.cfi_pop %r12
pop %rbp
.cfi_pop %rbp
pop %rbx
.cfi_pop %rbx
ret
.cfi_endproc
.size SHA3_absorb,.-SHA3_absorb
___
}
{ my ($A_flat,$out,$len,$bsz) = ("%rdi","%rsi","%rdx","%rcx");
($out,$len,$bsz) = ("%r12","%r13","%r14");
$code.=<<___;
.globl SHA3_squeeze
.type SHA3_squeeze,\@function
.align 32
SHA3_squeeze:
.cfi_startproc
push %r12
.cfi_push %r12
push %r13
.cfi_push %r13
push %r14
.cfi_push %r14
shr \$3,%rcx
mov $A_flat,%r8
mov %rsi,$out
mov %rdx,$len
mov %rcx,$bsz
jmp .Loop_squeeze
.align 32
.Loop_squeeze:
cmp \$8,$len
jb .Ltail_squeeze
mov (%r8),%rax
lea 8(%r8),%r8
mov %rax,($out)
lea 8($out),$out
sub \$8,$len # len -= 8
jz .Ldone_squeeze
sub \$1,%rcx # bsz--
jnz .Loop_squeeze
call KeccakF1600
mov $A_flat,%r8
mov $bsz,%rcx
jmp .Loop_squeeze
.Ltail_squeeze:
mov %r8, %rsi
mov $out,%rdi
mov $len,%rcx
.byte 0xf3,0xa4 # rep movsb
.Ldone_squeeze:
pop %r14
.cfi_pop %r14
pop %r13
.cfi_pop %r13
pop %r12
.cfi_pop %r13
ret
.cfi_endproc
.size SHA3_squeeze,.-SHA3_squeeze
___
}
$code.=<<___;
.align 256
.quad 0,0,0,0,0,0,0,0
.type iotas,\@object
iotas:
.quad 0x0000000000000001
.quad 0x0000000000008082
.quad 0x800000000000808a
.quad 0x8000000080008000
.quad 0x000000000000808b
.quad 0x0000000080000001
.quad 0x8000000080008081
.quad 0x8000000000008009
.quad 0x000000000000008a
.quad 0x0000000000000088
.quad 0x0000000080008009
.quad 0x000000008000000a
.quad 0x000000008000808b
.quad 0x800000000000008b
.quad 0x8000000000008089
.quad 0x8000000000008003
.quad 0x8000000000008002
.quad 0x8000000000000080
.quad 0x000000000000800a
.quad 0x800000008000000a
.quad 0x8000000080008081
.quad 0x8000000000008080
.quad 0x0000000080000001
.quad 0x8000000080008008
.size iotas,.-iotas
.asciz "Keccak-1600 absorb and squeeze for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
___
foreach (split("\n",$code)) {
# Below replacement results in 11.2 on Sandy Bridge, 9.4 on
# Haswell, but it hurts other processors by up to 2-3-4x...
#s/rol\s+(\$[0-9]+),(%[a-z][a-z0-9]+)/shld\t$1,$2,$2/;
# Below replacement results in 9.3 on Haswell [as well as
# on Ryzen, i.e. it *hurts* Ryzen]...
#s/rol\s+\$([0-9]+),(%[a-z][a-z0-9]+)/rorx\t\$64-$1,$2,$2/;
print $_, "\n";
}
close STDOUT;
crypto/sha/asm/keccak1600p8-ppc.pl
Executable
+850
View File
@@ -0,0 +1,850 @@
#!/usr/bin/env perl
# Copyright 2017 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License"). You may not use
# this file except in compliance with the License. You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html
#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
# ====================================================================
#
# Keccak-1600 for PowerISA 2.07.
#
# June 2017.
#
# This is straightforward KECCAK_1X_ALT SIMD implementation, but with
# disjoint Rho and Pi. The module is ABI-bitness- and endian-neutral.
# POWER8 processor spends 9.8 cycles to process byte out of large
# buffer for r=1088, which matches SHA3-256. This is 17% better than
# scalar PPC64 code. It probably should be noted that if POWER8's
# successor can achieve higher scalar instruction issue rate, then
# this module will loose...
$flavour = shift;
if ($flavour =~ /64/) {
$SIZE_T =8;
$LRSAVE =2*$SIZE_T;
$UCMP ="cmpld";
$STU ="stdu";
$POP ="ld";
$PUSH ="std";
} elsif ($flavour =~ /32/) {
$SIZE_T =4;
$LRSAVE =$SIZE_T;
$STU ="stwu";
$POP ="lwz";
$PUSH ="stw";
$UCMP ="cmplw";
} else { die "nonsense $flavour"; }
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
die "can't locate ppc-xlate.pl";
open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
$FRAME=6*$SIZE_T+13*16; # 13*16 is for v20-v31 offload
my $sp ="r1";
my $iotas = "r12";
########################################################################
# Register layout:
#
# v0 A[0][0] A[1][0]
# v1 A[0][1] A[1][1]
# v2 A[0][2] A[1][2]
# v3 A[0][3] A[1][3]
# v4 A[0][4] A[1][4]
#
# v5 A[2][0] A[3][0]
# v6 A[2][1] A[3][1]
# v7 A[2][2] A[3][2]
# v8 A[2][3] A[3][3]
# v9 A[2][4] A[3][4]
#
# v10 A[4][0] A[4][1]
# v11 A[4][2] A[4][3]
# v12 A[4][4] A[4][4]
#
# v13..25 rhotates[][]
# v26..31 volatile
#
$code.=<<___;
.machine "any"
.text
.type KeccakF1600_int,\@function
.align 5
KeccakF1600_int:
li r0,24
mtctr r0
li r0,0
b .Loop
.align 4
.Loop:
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Theta
vxor v26,v0, v5 ; A[0..1][0]^A[2..3][0]
vxor v27,v1, v6 ; A[0..1][1]^A[2..3][1]
vxor v28,v2, v7 ; A[0..1][2]^A[2..3][2]
vxor v29,v3, v8 ; A[0..1][3]^A[2..3][3]
vxor v30,v4, v9 ; A[0..1][4]^A[2..3][4]
vpermdi v31,v26,v27,0b00 ; A[0][0..1]^A[2][0..1]
vpermdi v26,v26,v27,0b11 ; A[1][0..1]^A[3][0..1]
vpermdi v27,v28,v29,0b00 ; A[0][2..3]^A[2][2..3]
vpermdi v28,v28,v29,0b11 ; A[1][2..3]^A[3][2..3]
vpermdi v29,v30,v30,0b10 ; A[1..0][4]^A[3..2][4]
vxor v26,v26,v31 ; C[0..1]
vxor v27,v27,v28 ; C[2..3]
vxor v28,v29,v30 ; C[4..4]
vspltisb v31,1
vxor v26,v26,v10 ; C[0..1] ^= A[4][0..1]
vxor v27,v27,v11 ; C[2..3] ^= A[4][2..3]
vxor v28,v28,v12 ; C[4..4] ^= A[4][4..4], low!
vrld v29,v26,v31 ; ROL64(C[0..1],1)
vrld v30,v27,v31 ; ROL64(C[2..3],1)
vrld v31,v28,v31 ; ROL64(C[4..4],1)
vpermdi v31,v31,v29,0b10
vxor v26,v26,v30 ; C[0..1] ^= ROL64(C[2..3],1)
vxor v27,v27,v31 ; C[2..3] ^= ROL64(C[4..0],1)
vxor v28,v28,v29 ; C[4..4] ^= ROL64(C[0..1],1), low!
vpermdi v29,v26,v26,0b00 ; C[0..0]
vpermdi v30,v28,v26,0b10 ; C[4..0]
vpermdi v31,v28,v28,0b11 ; C[4..4]
vxor v1, v1, v29 ; A[0..1][1] ^= C[0..0]
vxor v6, v6, v29 ; A[2..3][1] ^= C[0..0]
vxor v10,v10,v30 ; A[4][0..1] ^= C[4..0]
vxor v0, v0, v31 ; A[0..1][0] ^= C[4..4]
vxor v5, v5, v31 ; A[2..3][0] ^= C[4..4]
vpermdi v29,v27,v27,0b00 ; C[2..2]
vpermdi v30,v26,v26,0b11 ; C[1..1]
vpermdi v31,v26,v27,0b10 ; C[1..2]
vxor v3, v3, v29 ; A[0..1][3] ^= C[2..2]
vxor v8, v8, v29 ; A[2..3][3] ^= C[2..2]
vxor v2, v2, v30 ; A[0..1][2] ^= C[1..1]
vxor v7, v7, v30 ; A[2..3][2] ^= C[1..1]
vxor v11,v11,v31 ; A[4][2..3] ^= C[1..2]
vpermdi v29,v27,v27,0b11 ; C[3..3]
vxor v4, v4, v29 ; A[0..1][4] ^= C[3..3]
vxor v9, v9, v29 ; A[2..3][4] ^= C[3..3]
vxor v12,v12,v29 ; A[4..4][4] ^= C[3..3]
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Rho
vrld v26,v0, v13 ; v0
vrld v1, v1, v14
vrld v27,v2, v15 ; v2
vrld v28,v3, v16 ; v3
vrld v4, v4, v17
vrld v5, v5, v18
vrld v6, v6, v19
vrld v29,v7, v20 ; v7
vrld v8, v8, v21
vrld v9, v9, v22
vrld v10,v10,v23
vrld v30,v11,v24 ; v11
vrld v12,v12,v25
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Pi
vpermdi v0, v26,v28,0b00 ; [0][0] [1][0] < [0][0] [0][3]
vpermdi v2, v29,v5, 0b00 ; [0][2] [1][2] < [2][2] [2][0]
vpermdi v11,v9, v5, 0b01 ; [4][2] [4][3] < [2][4] [3][0]
vpermdi v5, v1, v4, 0b00 ; [2][0] [3][0] < [0][1] [0][4]
vpermdi v1, v1, v4, 0b11 ; [0][1] [1][1] < [1][1] [1][4]
vpermdi v3, v8, v6, 0b11 ; [0][3] [1][3] < [3][3] [3][1]
vpermdi v4, v12,v30,0b10 ; [0][4] [1][4] < [4][4] [4][2]
vpermdi v7, v8, v6, 0b00 ; [2][2] [3][2] < [2][3] [2][1]
vpermdi v6, v27,v26,0b11 ; [2][1] [3][1] < [1][2] [1][0]
vpermdi v8, v9, v29,0b11 ; [2][3] [3][3] < [3][4] [3][2]
vpermdi v12,v10,v10,0b11 ; [4][4] [4][4] < [4][1] [4][1]
vpermdi v9, v10,v30,0b01 ; [2][4] [3][4] < [4][0] [4][3]
vpermdi v10,v27,v28,0b01 ; [4][0] [4][1] < [0][2] [1][3]
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Chi + Iota
lvx_u v31,$iotas,r0 ; iotas[index]
addic r0,r0,16 ; index++
vandc v26,v2, v1 ; (~A[0..1][1] & A[0..1][2])
vandc v27,v3, v2 ; (~A[0..1][2] & A[0..1][3])
vandc v28,v4, v3 ; (~A[0..1][3] & A[0..1][4])
vandc v29,v0, v4 ; (~A[0..1][4] & A[0..1][0])
vandc v30,v1, v0 ; (~A[0..1][0] & A[0..1][1])
vxor v0, v0, v26 ; A[0..1][0] ^= (~A[0..1][1] & A[0..1][2])
vxor v1, v1, v27 ; A[0..1][1] ^= (~A[0..1][2] & A[0..1][3])
vxor v2, v2, v28 ; A[0..1][2] ^= (~A[0..1][3] & A[0..1][4])
vxor v3, v3, v29 ; A[0..1][3] ^= (~A[0..1][4] & A[0..1][0])
vxor v4, v4, v30 ; A[0..1][4] ^= (~A[0..1][0] & A[0..1][1])
vandc v26,v7, v6 ; (~A[2..3][1] & A[2..3][2])
vandc v27,v8, v7 ; (~A[2..3][2] & A[2..3][3])
vandc v28,v9, v8 ; (~A[2..3][3] & A[2..3][4])
vandc v29,v5, v9 ; (~A[2..3][4] & A[2..3][0])
vandc v30,v6, v5 ; (~A[2..3][0] & A[2..3][1])
vxor v5, v5, v26 ; A[2..3][0] ^= (~A[2..3][1] & A[2..3][2])
vxor v6, v6, v27 ; A[2..3][1] ^= (~A[2..3][2] & A[2..3][3])
vxor v7, v7, v28 ; A[2..3][2] ^= (~A[2..3][3] & A[2..3][4])
vxor v8, v8, v29 ; A[2..3][3] ^= (~A[2..3][4] & A[2..3][0])
vxor v9, v9, v30 ; A[2..3][4] ^= (~A[2..3][0] & A[2..3][1])
vxor v0, v0, v31 ; A[0][0] ^= iotas[index++]
vpermdi v26,v10,v11,0b10 ; A[4][1..2]
vpermdi v27,v12,v10,0b00 ; A[4][4..0]
vpermdi v28,v11,v12,0b10 ; A[4][3..4]
vpermdi v29,v10,v10,0b10 ; A[4][1..0]
vandc v26,v11,v26 ; (~A[4][1..2] & A[4][2..3])
vandc v27,v27,v28 ; (~A[4][3..4] & A[4][4..0])
vandc v28,v10,v29 ; (~A[4][1..0] & A[4][0..1])
vxor v10,v10,v26 ; A[4][0..1] ^= (~A[4][1..2] & A[4][2..3])
vxor v11,v11,v27 ; A[4][2..3] ^= (~A[4][3..4] & A[4][4..0])
vxor v12,v12,v28 ; A[4][4..4] ^= (~A[4][0..1] & A[4][1..0])
bdnz .Loop
vpermdi v12,v12,v12,0b11 ; broadcast A[4][4]
blr
.long 0
.byte 0,12,0x14,0,0,0,0,0
.size KeccakF1600_int,.-KeccakF1600_int
.type KeccakF1600,\@function
.align 5
KeccakF1600:
$STU $sp,-$FRAME($sp)
li r10,`15+6*$SIZE_T`
li r11,`31+6*$SIZE_T`
mflr r8
mfspr r7, 256 ; save vrsave
stvx v20,r10,$sp
addi r10,r10,32
stvx v21,r11,$sp
addi r11,r11,32
stvx v22,r10,$sp
addi r10,r10,32
stvx v23,r11,$sp
addi r11,r11,32
stvx v24,r10,$sp
addi r10,r10,32
stvx v25,r11,$sp
addi r11,r11,32
stvx v26,r10,$sp
addi r10,r10,32
stvx v27,r11,$sp
addi r11,r11,32
stvx v28,r10,$sp
addi r10,r10,32
stvx v29,r11,$sp
addi r11,r11,32
stvx v30,r10,$sp
stvx v31,r11,$sp
stw r7,`$FRAME-4`($sp) ; save vrsave
li r0, -1
$PUSH r8,`$FRAME+$LRSAVE`($sp)
mtspr 256, r0 ; preserve all AltiVec registers
li r11,16
lvx_4w v0,0,r3 ; load A[5][5]
li r10,32
lvx_4w v1,r11,r3
addi r11,r11,32
lvx_4w v2,r10,r3
addi r10,r10,32
lvx_4w v3,r11,r3
addi r11,r11,32
lvx_4w v4,r10,r3
addi r10,r10,32
lvx_4w v5,r11,r3
addi r11,r11,32
lvx_4w v6,r10,r3
addi r10,r10,32
lvx_4w v7,r11,r3
addi r11,r11,32
lvx_4w v8,r10,r3
addi r10,r10,32
lvx_4w v9,r11,r3
addi r11,r11,32
lvx_4w v10,r10,r3
addi r10,r10,32
lvx_4w v11,r11,r3
lvx_splt v12,r10,r3
bl PICmeup
li r11,16
lvx_u v13,0,r12 ; load rhotates
li r10,32
lvx_u v14,r11,r12
addi r11,r11,32
lvx_u v15,r10,r12
addi r10,r10,32
lvx_u v16,r11,r12
addi r11,r11,32
lvx_u v17,r10,r12
addi r10,r10,32
lvx_u v18,r11,r12
addi r11,r11,32
lvx_u v19,r10,r12
addi r10,r10,32
lvx_u v20,r11,r12
addi r11,r11,32
lvx_u v21,r10,r12
addi r10,r10,32
lvx_u v22,r11,r12
addi r11,r11,32
lvx_u v23,r10,r12
addi r10,r10,32
lvx_u v24,r11,r12
lvx_u v25,r10,r12
addi r12,r12,`16*16` ; points at iotas
bl KeccakF1600_int
li r11,16
stvx_4w v0,0,r3 ; return A[5][5]
li r10,32
stvx_4w v1,r11,r3
addi r11,r11,32
stvx_4w v2,r10,r3
addi r10,r10,32
stvx_4w v3,r11,r3
addi r11,r11,32
stvx_4w v4,r10,r3
addi r10,r10,32
stvx_4w v5,r11,r3
addi r11,r11,32
stvx_4w v6,r10,r3
addi r10,r10,32
stvx_4w v7,r11,r3
addi r11,r11,32
stvx_4w v8,r10,r3
addi r10,r10,32
stvx_4w v9,r11,r3
addi r11,r11,32
stvx_4w v10,r10,r3
addi r10,r10,32
stvx_4w v11,r11,r3
stvdx_u v12,r10,r3
li r10,`15+6*$SIZE_T`
li r11,`31+6*$SIZE_T`
mtlr r8
mtspr 256, r7 ; restore vrsave
lvx v20,r10,$sp
addi r10,r10,32
lvx v21,r11,$sp
addi r11,r11,32
lvx v22,r10,$sp
addi r10,r10,32
lvx v23,r11,$sp
addi r11,r11,32
lvx v24,r10,$sp
addi r10,r10,32
lvx v25,r11,$sp
addi r11,r11,32
lvx v26,r10,$sp
addi r10,r10,32
lvx v27,r11,$sp
addi r11,r11,32
lvx v28,r10,$sp
addi r10,r10,32
lvx v29,r11,$sp
addi r11,r11,32
lvx v30,r10,$sp
lvx v31,r11,$sp
addi $sp,$sp,$FRAME
blr
.long 0
.byte 0,12,0x04,1,0x80,0,1,0
.long 0
.size KeccakF1600,.-KeccakF1600
___
{
my ($A_jagged,$inp,$len,$bsz) = map("r$_",(3..6));
$code.=<<___;
.globl SHA3_absorb
.type SHA3_absorb,\@function
.align 5
SHA3_absorb:
$STU $sp,-$FRAME($sp)
li r10,`15+6*$SIZE_T`
li r11,`31+6*$SIZE_T`
mflr r8
mfspr r7, 256 ; save vrsave
stvx v20,r10,$sp
addi r10,r10,32
stvx v21,r11,$sp
addi r11,r11,32
stvx v22,r10,$sp
addi r10,r10,32
stvx v23,r11,$sp
addi r11,r11,32
stvx v24,r10,$sp
addi r10,r10,32
stvx v25,r11,$sp
addi r11,r11,32
stvx v26,r10,$sp
addi r10,r10,32
stvx v27,r11,$sp
addi r11,r11,32
stvx v28,r10,$sp
addi r10,r10,32
stvx v29,r11,$sp
addi r11,r11,32
stvx v30,r10,$sp
stvx v31,r11,$sp
stw r7,`$FRAME-4`($sp) ; save vrsave
li r0, -1
$PUSH r8,`$FRAME+$LRSAVE`($sp)
mtspr 256, r0 ; preserve all AltiVec registers
li r11,16
lvx_4w v0,0,$A_jagged ; load A[5][5]
li r10,32
lvx_4w v1,r11,$A_jagged
addi r11,r11,32
lvx_4w v2,r10,$A_jagged
addi r10,r10,32
lvx_4w v3,r11,$A_jagged
addi r11,r11,32
lvx_4w v4,r10,$A_jagged
addi r10,r10,32
lvx_4w v5,r11,$A_jagged
addi r11,r11,32
lvx_4w v6,r10,$A_jagged
addi r10,r10,32
lvx_4w v7,r11,$A_jagged
addi r11,r11,32
lvx_4w v8,r10,$A_jagged
addi r10,r10,32
lvx_4w v9,r11,$A_jagged
addi r11,r11,32
lvx_4w v10,r10,$A_jagged
addi r10,r10,32
lvx_4w v11,r11,$A_jagged
lvx_splt v12,r10,$A_jagged
bl PICmeup
li r11,16
lvx_u v13,0,r12 ; load rhotates
li r10,32
lvx_u v14,r11,r12
addi r11,r11,32
lvx_u v15,r10,r12
addi r10,r10,32
lvx_u v16,r11,r12
addi r11,r11,32
lvx_u v17,r10,r12
addi r10,r10,32
lvx_u v18,r11,r12
addi r11,r11,32
lvx_u v19,r10,r12
addi r10,r10,32
lvx_u v20,r11,r12
addi r11,r11,32
lvx_u v21,r10,r12
addi r10,r10,32
lvx_u v22,r11,r12
addi r11,r11,32
lvx_u v23,r10,r12
addi r10,r10,32
lvx_u v24,r11,r12
lvx_u v25,r10,r12
li r10,-32
li r11,-16
addi r12,r12,`16*16` ; points at iotas
b .Loop_absorb
.align 4
.Loop_absorb:
$UCMP $len,$bsz ; len < bsz?
blt .Labsorbed
sub $len,$len,$bsz ; len -= bsz
srwi r0,$bsz,3
mtctr r0
lvx_u v30,r10,r12 ; permutation masks
lvx_u v31,r11,r12
?vspltisb v27,7 ; prepare masks for byte swap
?vxor v30,v30,v27 ; on big-endian
?vxor v31,v31,v27
vxor v27,v27,v27 ; zero
lvdx_u v26,0,$inp
addi $inp,$inp,8
vperm v26,v26,v27,v30
vxor v0, v0, v26
bdz .Lprocess_block
lvdx_u v26,0,$inp
addi $inp,$inp,8
vperm v26,v26,v27,v30
vxor v1, v1, v26
bdz .Lprocess_block
lvdx_u v26,0,$inp
addi $inp,$inp,8
vperm v26,v26,v27,v30
vxor v2, v2, v26
bdz .Lprocess_block
lvdx_u v26,0,$inp
addi $inp,$inp,8
vperm v26,v26,v27,v30
vxor v3, v3, v26
bdz .Lprocess_block
lvdx_u v26,0,$inp
addi $inp,$inp,8
vperm v26,v26,v27,v30
vxor v4, v4, v26
bdz .Lprocess_block
lvdx_u v26,0,$inp
addi $inp,$inp,8
vperm v26,v26,v27,v31
vxor v0, v0, v26
bdz .Lprocess_block
lvdx_u v26,0,$inp
addi $inp,$inp,8
vperm v26,v26,v27,v31
vxor v1, v1, v26
bdz .Lprocess_block
lvdx_u v26,0,$inp
addi $inp,$inp,8
vperm v26,v26,v27,v31
vxor v2, v2, v26
bdz .Lprocess_block
lvdx_u v26,0,$inp
addi $inp,$inp,8
vperm v26,v26,v27,v31
vxor v3, v3, v26
bdz .Lprocess_block
lvdx_u v26,0,$inp
addi $inp,$inp,8
vperm v26,v26,v27,v31
vxor v4, v4, v26
bdz .Lprocess_block
lvdx_u v26,0,$inp
addi $inp,$inp,8
vperm v26,v26,v27,v30
vxor v5, v5, v26
bdz .Lprocess_block
lvdx_u v26,0,$inp
addi $inp,$inp,8
vperm v26,v26,v27,v30
vxor v6, v6, v26
bdz .Lprocess_block
lvdx_u v26,0,$inp
addi $inp,$inp,8
vperm v26,v26,v27,v30
vxor v7, v7, v26
bdz .Lprocess_block
lvdx_u v26,0,$inp
addi $inp,$inp,8
vperm v26,v26,v27,v30
vxor v8, v8, v26
bdz .Lprocess_block
lvdx_u v26,0,$inp
addi $inp,$inp,8
vperm v26,v26,v27,v30
vxor v9, v9, v26
bdz .Lprocess_block
lvdx_u v26,0,$inp
addi $inp,$inp,8
vperm v26,v26,v27,v31
vxor v5, v5, v26
bdz .Lprocess_block
lvdx_u v26,0,$inp
addi $inp,$inp,8
vperm v26,v26,v27,v31
vxor v6, v6, v26
bdz .Lprocess_block
lvdx_u v26,0,$inp
addi $inp,$inp,8
vperm v26,v26,v27,v31
vxor v7, v7, v26
bdz .Lprocess_block
lvdx_u v26,0,$inp
addi $inp,$inp,8
vperm v26,v26,v27,v31
vxor v8, v8, v26
bdz .Lprocess_block
lvdx_u v26,0,$inp
addi $inp,$inp,8
vperm v26,v26,v27,v31
vxor v9, v9, v26
bdz .Lprocess_block
lvdx_u v26,0,$inp
addi $inp,$inp,8
vperm v26,v26,v27,v30
vxor v10, v10, v26
bdz .Lprocess_block
lvdx_u v26,0,$inp
addi $inp,$inp,8
vperm v26,v26,v27,v31
vxor v10, v10, v26
bdz .Lprocess_block
lvdx_u v26,0,$inp
addi $inp,$inp,8
vperm v26,v26,v27,v30
vxor v11, v11, v26
bdz .Lprocess_block
lvdx_u v26,0,$inp
addi $inp,$inp,8
vperm v26,v26,v27,v31
vxor v11, v11, v26
bdz .Lprocess_block
lvdx_u v26,0,$inp
addi $inp,$inp,8
vperm v26,v26,v27,v31
vxor v12, v12, v26
.Lprocess_block:
bl KeccakF1600_int
b .Loop_absorb
.align 4
.Labsorbed:
li r11,16
stvx_4w v0,0,$A_jagged ; return A[5][5]
li r10,32
stvx_4w v1,r11,$A_jagged
addi r11,r11,32
stvx_4w v2,r10,$A_jagged
addi r10,r10,32
stvx_4w v3,r11,$A_jagged
addi r11,r11,32
stvx_4w v4,r10,$A_jagged
addi r10,r10,32
stvx_4w v5,r11,$A_jagged
addi r11,r11,32
stvx_4w v6,r10,$A_jagged
addi r10,r10,32
stvx_4w v7,r11,$A_jagged
addi r11,r11,32
stvx_4w v8,r10,$A_jagged
addi r10,r10,32
stvx_4w v9,r11,$A_jagged
addi r11,r11,32
stvx_4w v10,r10,$A_jagged
addi r10,r10,32
stvx_4w v11,r11,$A_jagged
stvdx_u v12,r10,$A_jagged
mr r3,$len ; return value
li r10,`15+6*$SIZE_T`
li r11,`31+6*$SIZE_T`
mtlr r8
mtspr 256, r7 ; restore vrsave
lvx v20,r10,$sp
addi r10,r10,32
lvx v21,r11,$sp
addi r11,r11,32
lvx v22,r10,$sp
addi r10,r10,32
lvx v23,r11,$sp
addi r11,r11,32
lvx v24,r10,$sp
addi r10,r10,32
lvx v25,r11,$sp
addi r11,r11,32
lvx v26,r10,$sp
addi r10,r10,32
lvx v27,r11,$sp
addi r11,r11,32
lvx v28,r10,$sp
addi r10,r10,32
lvx v29,r11,$sp
addi r11,r11,32
lvx v30,r10,$sp
lvx v31,r11,$sp
addi $sp,$sp,$FRAME
blr
.long 0
.byte 0,12,0x04,1,0x80,0,4,0
.long 0
.size SHA3_absorb,.-SHA3_absorb
___
}
{
my ($A_jagged,$out,$len,$bsz) = map("r$_",(3..6));
$code.=<<___;
.globl SHA3_squeeze
.type SHA3_squeeze,\@function
.align 5
SHA3_squeeze:
mflr r9 ; r9 is not touched by KeccakF1600
subi $out,$out,1 ; prepare for stbu
addi r8,$A_jagged,4 ; prepare volatiles
mr r10,$bsz
li r11,0
b .Loop_squeeze
.align 4
.Loop_squeeze:
lwzx r7,r11,r8 ; lo
lwzx r0,r11,$A_jagged ; hi
${UCMP}i $len,8
blt .Lsqueeze_tail
stbu r7,1($out) ; write lo
srwi r7,r7,8
stbu r7,1($out)
srwi r7,r7,8
stbu r7,1($out)
srwi r7,r7,8
stbu r7,1($out)
stbu r0,1($out) ; write hi
srwi r0,r0,8
stbu r0,1($out)
srwi r0,r0,8
stbu r0,1($out)
srwi r0,r0,8
stbu r0,1($out)
subic. $len,$len,8
beqlr ; return if done
subic. r10,r10,8
ble .Loutput_expand
addi r11,r11,16 ; calculate jagged index
cmplwi r11,`16*5`
blt .Loop_squeeze
subi r11,r11,72
beq .Loop_squeeze
addi r11,r11,72
cmplwi r11,`16*5+8`
subi r11,r11,8
beq .Loop_squeeze
addi r11,r11,8
cmplwi r11,`16*10`
subi r11,r11,72
beq .Loop_squeeze
addi r11,r11,72
blt .Loop_squeeze
subi r11,r11,8
b .Loop_squeeze
.align 4
.Loutput_expand:
bl KeccakF1600
mtlr r9
addi r8,$A_jagged,4 ; restore volatiles
mr r10,$bsz
li r11,0
b .Loop_squeeze
.align 4
.Lsqueeze_tail:
mtctr $len
subic. $len,$len,4
ble .Loop_tail_lo
li r8,4
mtctr r8
.Loop_tail_lo:
stbu r7,1($out)
srdi r7,r7,8
bdnz .Loop_tail_lo
ble .Lsqueeze_done
mtctr $len
.Loop_tail_hi:
stbu r0,1($out)
srdi r0,r0,8
bdnz .Loop_tail_hi
.Lsqueeze_done:
blr
.long 0
.byte 0,12,0x14,0,0,0,4,0
.long 0
.size SHA3_squeeze,.-SHA3_squeeze
___
}
$code.=<<___;
.align 6
PICmeup:
mflr r0
bcl 20,31,\$+4
mflr r12 ; vvvvvv "distance" between . and 1st data entry
addi r12,r12,`64-8`
mtlr r0
blr
.long 0
.byte 0,12,0x14,0,0,0,0,0
.space `64-9*4`
.type rhotates,\@object
.align 6
rhotates:
.quad 0, 36
.quad 1, 44
.quad 62, 6
.quad 28, 55
.quad 27, 20
.quad 3, 41
.quad 10, 45
.quad 43, 15
.quad 25, 21
.quad 39, 8
.quad 18, 2
.quad 61, 56
.quad 14, 14
.size rhotates,.-rhotates
.quad 0,0
.quad 0x0001020304050607,0x1011121314151617
.quad 0x1011121314151617,0x0001020304050607
.type iotas,\@object
iotas:
.quad 0x0000000000000001,0
.quad 0x0000000000008082,0
.quad 0x800000000000808a,0
.quad 0x8000000080008000,0
.quad 0x000000000000808b,0
.quad 0x0000000080000001,0
.quad 0x8000000080008081,0
.quad 0x8000000000008009,0
.quad 0x000000000000008a,0
.quad 0x0000000000000088,0
.quad 0x0000000080008009,0
.quad 0x000000008000000a,0
.quad 0x000000008000808b,0
.quad 0x800000000000008b,0
.quad 0x8000000000008089,0
.quad 0x8000000000008003,0
.quad 0x8000000000008002,0
.quad 0x8000000000000080,0
.quad 0x000000000000800a,0
.quad 0x800000008000000a,0
.quad 0x8000000080008081,0
.quad 0x8000000000008080,0
.quad 0x0000000080000001,0
.quad 0x8000000080008008,0
.size iotas,.-iotas
.asciz "Keccak-1600 absorb and squeeze for PowerISA 2.07, CRYPTOGAMS by <appro\@openssl.org>"
___
foreach (split("\n",$code)) {
s/\`([^\`]*)\`/eval $1/ge;
if ($flavour =~ /le$/) { # little-endian
s/\?([a-z]+)/;$1/;
} else { # big-endian
s/\?([a-z]+)/$1/;
}
print $_,"\n";
}
close STDOUT;
crypto/sha/asm/sha1-586.pl
+8 -5
View File
@@ -35,10 +35,9 @@
# P4 +85%(!) +45%
#
# As you can see Pentium came out as looser:-( Yet I reckoned that
# improvement on P4 outweights the loss and incorporate this
# improvement on P4 outweighs the loss and incorporate this
# re-tuned code to 0.9.7 and later.
# ----------------------------------------------------------------
# <appro@fy.chalmers.se>
# August 2009.
#
@@ -104,10 +103,12 @@
# Sandy Bridge 8.8 6.2/+40% 5.1(**)/+73%
# Ivy Bridge 7.2 4.8/+51% 4.7(**)/+53%
# Haswell 6.5 4.3/+51% 4.1(**)/+58%
# Skylake 6.4 4.1/+55% 4.1(**)/+55%
# Bulldozer 11.6 6.0/+92%
# VIA Nano 10.6 7.5/+41%
# Atom 12.5 9.3(*)/+35%
# Silvermont 14.5 9.9(*)/+46%
# Goldmont 8.8 6.7/+30% 1.7(***)/+415%
#
# (*) Loop is 1056 instructions long and expected result is ~8.25.
# The discrepancy is because of front-end limitations, so
@@ -115,6 +116,8 @@
# limited parallelism.
#
# (**) As per above comment, the result is for AVX *plus* sh[rl]d.
#
# (***) SHAEXT result
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
push(@INC,"${dir}","${dir}../../perlasm");
@@ -123,7 +126,7 @@ require "x86asm.pl";
$output=pop;
open STDOUT,">$output";
&asm_init($ARGV[0],"sha1-586.pl",$ARGV[$#ARGV] eq "386");
&asm_init($ARGV[0],$ARGV[$#ARGV] eq "386");
$xmm=$ymm=0;
for (@ARGV) { $xmm=1 if (/-DOPENSSL_IA32_SSE2/); }
@@ -133,7 +136,7 @@ $ymm=1 if ($xmm &&
=~ /GNU assembler version ([2-9]\.[0-9]+)/ &&
$1>=2.19); # first version supporting AVX
$ymm=1 if ($xmm && !$ymm && $ARGV[0] eq "win32n" &&
$ymm=1 if ($xmm && !$ymm && $ARGV[0] eq "win32n" &&
`nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/ &&
$1>=2.03); # first version supporting AVX
@@ -546,7 +549,7 @@ for($i=0;$i<20-4;$i+=2) {
# being implemented in SSSE3). Once 8 quadruples or 32 elements are
# collected, it switches to routine proposed by Max Locktyukhin.
#
# Calculations inevitably require temporary reqisters, and there are
# Calculations inevitably require temporary registers, and there are
# no %xmm registers left to spare. For this reason part of the ring
# buffer, X[2..4] to be specific, is offloaded to 3 quadriples ring
# buffer on the stack. Keep in mind that X[2] is alias X[-6], X[3] -
crypto/sha/asm/sha1-alpha.pl
+1 -1
View File
@@ -8,7 +8,7 @@
# ====================================================================
# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
crypto/sha/asm/sha1-armv8.pl
+1
View File
@@ -26,6 +26,7 @@
# Denver 2.13 3.97 (+0%)(**)
# X-Gene 8.80 (+200%)
# Mongoose 2.05 6.50 (+160%)
# Kryo 1.88 8.00 (+90%)
#
# (*) Software results are presented mostly for reference purposes.
# (**) Keep in mind that Denver relies on binary translation, which
crypto/sha/asm/sha1-ia64.pl
+1 -1
View File
@@ -8,7 +8,7 @@
#
# ====================================================================
# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
crypto/sha/asm/sha1-mb-x86_64.pl
+53 -7
View File
@@ -95,7 +95,7 @@ $K="%xmm15";
if (1) {
# Atom-specific optimization aiming to eliminate pshufb with high
# registers [and thus get rid of 48 cycles accumulated penalty]
# registers [and thus get rid of 48 cycles accumulated penalty]
@Xi=map("%xmm$_",(0..4));
($tx,$t0,$t1,$t2,$t3)=map("%xmm$_",(5..9));
@V=($A,$B,$C,$D,$E)=map("%xmm$_",(10..14));
@@ -126,7 +126,7 @@ my $k=$i+2;
# ...
# $i==13: 14,15,15,15,
# $i==14: 15
#
#
# Then at $i==15 Xupdate is applied one iteration in advance...
$code.=<<___ if ($i==0);
movd (@ptr[0]),@Xi[0]
@@ -363,6 +363,7 @@ $code.=<<___;
.type sha1_multi_block,\@function,3
.align 32
sha1_multi_block:
.cfi_startproc
mov OPENSSL_ia32cap_P+4(%rip),%rcx
bt \$61,%rcx # check SHA bit
jc _shaext_shortcut
@@ -373,8 +374,11 @@ $code.=<<___ if ($avx);
___
$code.=<<___;
mov %rsp,%rax
.cfi_def_cfa_register %rax
push %rbx
.cfi_push %rbx
push %rbp
.cfi_push %rbx
___
$code.=<<___ if ($win64);
lea -0xa8(%rsp),%rsp
@@ -393,6 +397,7 @@ $code.=<<___;
sub \$`$REG_SZ*18`,%rsp
and \$-256,%rsp
mov %rax,`$REG_SZ*17`(%rsp) # original %rsp
.cfi_cfa_expression %rsp+`$REG_SZ*17`,deref,+8
.Lbody:
lea K_XX_XX(%rip),$Tbl
lea `$REG_SZ*16`(%rsp),%rbx
@@ -439,7 +444,7 @@ for(;$i<80;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); }
$code.=<<___;
movdqa (%rbx),@Xi[0] # pull counters
mov \$1,%ecx
cmp 4*0(%rbx),%ecx # examinte counters
cmp 4*0(%rbx),%ecx # examine counters
pxor $t2,$t2
cmovge $Tbl,@ptr[0] # cancel input
cmp 4*1(%rbx),%ecx
@@ -487,6 +492,7 @@ $code.=<<___;
.Ldone:
mov `$REG_SZ*17`(%rsp),%rax # original %rsp
.cfi_def_cfa %rax,8
___
$code.=<<___ if ($win64);
movaps -0xb8(%rax),%xmm6
@@ -502,10 +508,14 @@ $code.=<<___ if ($win64);
___
$code.=<<___;
mov -16(%rax),%rbp
.cfi_restore %rbp
mov -8(%rax),%rbx
.cfi_restore %rbx
lea (%rax),%rsp
.cfi_def_cfa_register %rsp
.Lepilogue:
ret
.cfi_endproc
.size sha1_multi_block,.-sha1_multi_block
___
{{{
@@ -517,10 +527,14 @@ $code.=<<___;
.type sha1_multi_block_shaext,\@function,3
.align 32
sha1_multi_block_shaext:
.cfi_startproc
_shaext_shortcut:
mov %rsp,%rax
.cfi_def_cfa_register %rax
push %rbx
.cfi_push %rbx
push %rbp
.cfi_push %rbp
___
$code.=<<___ if ($win64);
lea -0xa8(%rsp),%rsp
@@ -756,10 +770,14 @@ $code.=<<___ if ($win64);
___
$code.=<<___;
mov -16(%rax),%rbp
.cfi_restore %rbp
mov -8(%rax),%rbx
.cfi_restore %rbx
lea (%rax),%rsp
.cfi_def_cfa_register %rsp
.Lepilogue_shaext:
ret
.cfi_endproc
.size sha1_multi_block_shaext,.-sha1_multi_block_shaext
___
}}}
@@ -1002,6 +1020,7 @@ $code.=<<___;
.type sha1_multi_block_avx,\@function,3
.align 32
sha1_multi_block_avx:
.cfi_startproc
_avx_shortcut:
___
$code.=<<___ if ($avx>1);
@@ -1016,8 +1035,11 @@ $code.=<<___ if ($avx>1);
___
$code.=<<___;
mov %rsp,%rax
.cfi_def_cfa_register %rax
push %rbx
.cfi_push %rbx
push %rbp
.cfi_push %rbp
___
$code.=<<___ if ($win64);
lea -0xa8(%rsp),%rsp
@@ -1036,6 +1058,7 @@ $code.=<<___;
sub \$`$REG_SZ*18`, %rsp
and \$-256,%rsp
mov %rax,`$REG_SZ*17`(%rsp) # original %rsp
.cfi_cfa_expression %rsp+`$REG_SZ*17`,deref,+8
.Lbody_avx:
lea K_XX_XX(%rip),$Tbl
lea `$REG_SZ*16`(%rsp),%rbx
@@ -1125,6 +1148,7 @@ $code.=<<___;
.Ldone_avx:
mov `$REG_SZ*17`(%rsp),%rax # original %rsp
.cfi_def_cfa %rax,8
vzeroupper
___
$code.=<<___ if ($win64);
@@ -1141,10 +1165,14 @@ $code.=<<___ if ($win64);
___
$code.=<<___;
mov -16(%rax),%rbp
.cfi_restore %rbp
mov -8(%rax),%rbx
.cfi_restore %rbx
lea (%rax),%rsp
.cfi_def_cfa_register %rsp
.Lepilogue_avx:
ret
.cfi_endproc
.size sha1_multi_block_avx,.-sha1_multi_block_avx
___
@@ -1164,14 +1192,22 @@ $code.=<<___;
.type sha1_multi_block_avx2,\@function,3
.align 32
sha1_multi_block_avx2:
.cfi_startproc
_avx2_shortcut:
mov %rsp,%rax
.cfi_def_cfa_register %rax
push %rbx
.cfi_push %rbx
push %rbp
.cfi_push %rbp
push %r12
.cfi_push %r12
push %r13
.cfi_push %r13
push %r14
.cfi_push %r14
push %r15
.cfi_push %r15
___
$code.=<<___ if ($win64);
lea -0xa8(%rsp),%rsp
@@ -1190,6 +1226,7 @@ $code.=<<___;
sub \$`$REG_SZ*18`, %rsp
and \$-256,%rsp
mov %rax,`$REG_SZ*17`(%rsp) # original %rsp
.cfi_cfa_expression %rsp+`$REG_SZ*17`,deref,+8
.Lbody_avx2:
lea K_XX_XX(%rip),$Tbl
shr \$1,$num
@@ -1280,6 +1317,7 @@ $code.=<<___;
.Ldone_avx2:
mov `$REG_SZ*17`(%rsp),%rax # original %rsp
.cfi_def_cfa %rax,8
vzeroupper
___
$code.=<<___ if ($win64);
@@ -1296,14 +1334,22 @@ $code.=<<___ if ($win64);
___
$code.=<<___;
mov -48(%rax),%r15
.cfi_restore %r15
mov -40(%rax),%r14
.cfi_restore %r14
mov -32(%rax),%r13
.cfi_restore %r13
mov -24(%rax),%r12
.cfi_restore %r12
mov -16(%rax),%rbp
.cfi_restore %rbp
mov -8(%rax),%rbx
.cfi_restore %rbx
lea (%rax),%rsp
.cfi_def_cfa_register %rsp
.Lepilogue_avx2:
ret
.cfi_endproc
.size sha1_multi_block_avx2,.-sha1_multi_block_avx2
___
} }}}
@@ -1462,10 +1508,10 @@ avx2_handler:
mov -48(%rax),%r15
mov %rbx,144($context) # restore context->Rbx
mov %rbp,160($context) # restore context->Rbp
mov %r12,216($context) # restore cotnext->R12
mov %r13,224($context) # restore cotnext->R13
mov %r14,232($context) # restore cotnext->R14
mov %r15,240($context) # restore cotnext->R15
mov %r12,216($context) # restore context->R12
mov %r13,224($context) # restore context->R13
mov %r14,232($context) # restore context->R14
mov %r15,240($context) # restore context->R15
lea -56-10*16(%rax),%rsi
lea 512($context),%rdi # &context.Xmm6
crypto/sha/asm/sha1-mips.pl
+17 -13
View File
@@ -8,7 +8,7 @@
# ====================================================================
# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
@@ -56,15 +56,15 @@
$flavour = shift || "o32"; # supported flavours are o32,n32,64,nubi32,nubi64
if ($flavour =~ /64|n32/i) {
$PTR_ADD="dadd"; # incidentally works even on n32
$PTR_SUB="dsub"; # incidentally works even on n32
$PTR_ADD="daddu"; # incidentally works even on n32
$PTR_SUB="dsubu"; # incidentally works even on n32
$REG_S="sd";
$REG_L="ld";
$PTR_SLL="dsll"; # incidentally works even on n32
$SZREG=8;
} else {
$PTR_ADD="add";
$PTR_SUB="sub";
$PTR_ADD="addu";
$PTR_SUB="subu";
$REG_S="sw";
$REG_L="lw";
$PTR_SLL="sll";
@@ -126,10 +126,14 @@ $code.=<<___;
addu $e,$K # $i
xor $t0,$c,$d
rotr $t1,$a,27
lwl @X[$j],$j*4+$MSB($inp)
and $t0,$b
addu $e,$t1
#if defined(_MIPS_ARCH_MIPS32R6) || defined(_MIPS_ARCH_MIPS64R6)
lw @X[$j],$j*4($inp)
#else
lwl @X[$j],$j*4+$MSB($inp)
lwr @X[$j],$j*4+$LSB($inp)
#endif
xor $t0,$d
addu $e,@X[$i]
rotr $b,$b,2
@@ -336,13 +340,7 @@ $FRAMESIZE=16; # large enough to accommodate NUBI saved registers
$SAVED_REGS_MASK = ($flavour =~ /nubi/i) ? "0xc0fff008" : "0xc0ff0000";
$code=<<___;
#ifdef OPENSSL_FIPSCANISTER
# include <openssl/fipssyms.h>
#endif
#if defined(__mips_smartmips) && !defined(_MIPS_ARCH_MIPS32R2)
#define _MIPS_ARCH_MIPS32R2
#endif
#include "mips_arch.h"
.text
@@ -387,10 +385,16 @@ $code.=<<___;
.align 4
.Loop:
.set reorder
#if defined(_MIPS_ARCH_MIPS32R6) || defined(_MIPS_ARCH_MIPS64R6)
lui $K,0x5a82
lw @X[0],($inp)
ori $K,0x7999 # K_00_19
#else
lwl @X[0],$MSB($inp)
lui $K,0x5a82
lwr @X[0],$LSB($inp)
ori $K,0x7999 # K_00_19
#endif
___
for ($i=0;$i<15;$i++) { &BODY_00_14($i,@V); unshift(@V,pop(@V)); }
for (;$i<20;$i++) { &BODY_15_19($i,@V); unshift(@V,pop(@V)); }
crypto/sha/asm/sha1-parisc.pl
+1 -1
View File
@@ -8,7 +8,7 @@
# ====================================================================
# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
crypto/sha/asm/sha1-ppc.pl
+1 -1
View File
@@ -8,7 +8,7 @@
# ====================================================================
# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
crypto/sha/asm/sha1-s390x.pl
+4 -2
View File
@@ -8,7 +8,7 @@
# ====================================================================
# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
@@ -160,6 +160,8 @@ ___
}
$code.=<<___;
#include "s390x_arch.h"
.text
.align 64
.type Ktable,\@object
@@ -172,7 +174,7 @@ sha1_block_data_order:
___
$code.=<<___ if ($kimdfunc);
larl %r1,OPENSSL_s390xcap_P
lg %r0,16(%r1) # check kimd capabilities
lg %r0,S390X_KIMD(%r1) # check kimd capabilities
tmhh %r0,`0x8000>>$kimdfunc`
jz .Lsoftware
lghi %r0,$kimdfunc
crypto/sha/asm/sha1-sparcv9.pl
+3 -3
View File
@@ -8,12 +8,12 @@
# ====================================================================
# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
#
# Hardware SPARC T4 support by David S. Miller <davem@davemloft.net>.
# Hardware SPARC T4 support by David S. Miller
# ====================================================================
# Performance improvement is not really impressive on pre-T1 CPU: +8%
@@ -227,7 +227,7 @@ sha1_block_data_order:
ldd [%o1 + 0x20], %f16
ldd [%o1 + 0x28], %f18
ldd [%o1 + 0x30], %f20
subcc %o2, 1, %o2 ! done yet?
subcc %o2, 1, %o2 ! done yet?
ldd [%o1 + 0x38], %f22
add %o1, 0x40, %o1
prefetch [%o1 + 63], 20
crypto/sha/asm/sha1-sparcv9a.pl
+2 -2
View File
@@ -8,7 +8,7 @@
# ====================================================================
# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
@@ -519,7 +519,7 @@ $code.=<<___;
mov $Cctx,$C
mov $Dctx,$D
mov $Ectx,$E
alignaddr %g0,$tmp0,%g0
alignaddr %g0,$tmp0,%g0
dec 1,$len
ba .Loop
mov $nXfer,$Xfer
crypto/sha/asm/sha1-thumb.pl
+2 -2
View File
@@ -8,7 +8,7 @@
# ====================================================================
# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
@@ -21,7 +21,7 @@
# The code does not present direct interest to OpenSSL, because of low
# performance. Its purpose is to establish _size_ benchmark. Pretty
# useless one I must say, because 30% or 88 bytes larger ARMv4 code
# [avialable on demand] is almost _twice_ as fast. It should also be
# [available on demand] is almost _twice_ as fast. It should also be
# noted that in-lining of .Lcommon and .Lrotate improves performance
# by over 40%, while code increases by only 10% or 32 bytes. But once
# again, the goal was to establish _size_ benchmark, not performance.
crypto/sha/asm/sha1-x86_64.pl
+126 -71
View File
@@ -82,9 +82,11 @@
# Haswell 5.45 4.15/+31% 3.57/+53%
# Skylake 5.18 4.06/+28% 3.54/+46%
# Bulldozer 9.11 5.95/+53%
# Ryzen 4.75 3.80/+24% 1.93/+150%(**)
# VIA Nano 9.32 7.15/+30%
# Atom 10.3 9.17/+12%
# Silvermont 13.1(*) 9.37/+40%
# Knights L 13.2(*) 9.68/+36% 8.30/+59%
# Goldmont 8.13 6.42/+27% 1.70/+380%(**)
#
# (*) obviously suboptimal result, nothing was done about it,
@@ -257,6 +259,7 @@ $code.=<<___;
.type sha1_block_data_order,\@function,3
.align 16
sha1_block_data_order:
.cfi_startproc
mov OPENSSL_ia32cap_P+0(%rip),%r9d
mov OPENSSL_ia32cap_P+4(%rip),%r8d
mov OPENSSL_ia32cap_P+8(%rip),%r10d
@@ -264,7 +267,7 @@ sha1_block_data_order:
jz .Lialu
___
$code.=<<___ if ($shaext);
test \$`1<<29`,%r10d # check SHA bit
test \$`1<<29`,%r10d # check SHA bit
jnz _shaext_shortcut
___
$code.=<<___ if ($avx>1);
@@ -285,17 +288,24 @@ $code.=<<___;
.align 16
.Lialu:
mov %rsp,%rax
.cfi_def_cfa_register %rax
push %rbx
.cfi_push %rbx
push %rbp
.cfi_push %rbp
push %r12
.cfi_push %r12
push %r13
.cfi_push %r13
push %r14
.cfi_push %r14
mov %rdi,$ctx # reassigned argument
sub \$`8+16*4`,%rsp
mov %rsi,$inp # reassigned argument
and \$-64,%rsp
mov %rdx,$num # reassigned argument
mov %rax,`16*4`(%rsp)
.cfi_cfa_expression %rsp+64,deref,+8
.Lprologue:
mov 0($ctx),$A
@@ -329,14 +339,22 @@ $code.=<<___;
jnz .Lloop
mov `16*4`(%rsp),%rsi
.cfi_def_cfa %rsi,8
mov -40(%rsi),%r14
.cfi_restore %r14
mov -32(%rsi),%r13
.cfi_restore %r13
mov -24(%rsi),%r12
.cfi_restore %r12
mov -16(%rsi),%rbp
.cfi_restore %rbp
mov -8(%rsi),%rbx
.cfi_restore %rbx
lea (%rsi),%rsp
.cfi_def_cfa_register %rsp
.Lepilogue:
ret
.cfi_endproc
.size sha1_block_data_order,.-sha1_block_data_order
___
if ($shaext) {{{
@@ -352,6 +370,7 @@ $code.=<<___;
.align 32
sha1_block_data_order_shaext:
_shaext_shortcut:
.cfi_startproc
___
$code.=<<___ if ($win64);
lea `-8-4*16`(%rsp),%rsp
@@ -449,6 +468,7 @@ $code.=<<___ if ($win64);
.Lepilogue_shaext:
___
$code.=<<___;
.cfi_endproc
ret
.size sha1_block_data_order_shaext,.-sha1_block_data_order_shaext
___
@@ -462,7 +482,8 @@ my @V=($A,$B,$C,$D,$E)=("%eax","%ebx","%ecx","%edx","%ebp"); # size optimization
my @T=("%esi","%edi");
my $j=0;
my $rx=0;
my $K_XX_XX="%r11";
my $K_XX_XX="%r14";
my $fp="%r11";
my $_rol=sub { &rol(@_) };
my $_ror=sub { &ror(@_) };
@@ -483,25 +504,31 @@ $code.=<<___;
.align 16
sha1_block_data_order_ssse3:
_ssse3_shortcut:
mov %rsp,%rax
.cfi_startproc
mov %rsp,$fp # frame pointer
.cfi_def_cfa_register $fp
push %rbx
.cfi_push %rbx
push %rbp
.cfi_push %rbp
push %r12
.cfi_push %r12
push %r13 # redundant, done to share Win64 SE handler
.cfi_push %r13
push %r14
.cfi_push %r14
lea `-64-($win64?6*16:0)`(%rsp),%rsp
___
$code.=<<___ if ($win64);
movaps %xmm6,-40-6*16(%rax)
movaps %xmm7,-40-5*16(%rax)
movaps %xmm8,-40-4*16(%rax)
movaps %xmm9,-40-3*16(%rax)
movaps %xmm10,-40-2*16(%rax)
movaps %xmm11,-40-1*16(%rax)
movaps %xmm6,-40-6*16($fp)
movaps %xmm7,-40-5*16($fp)
movaps %xmm8,-40-4*16($fp)
movaps %xmm9,-40-3*16($fp)
movaps %xmm10,-40-2*16($fp)
movaps %xmm11,-40-1*16($fp)
.Lprologue_ssse3:
___
$code.=<<___;
mov %rax,%r14 # original %rsp
and \$-64,%rsp
mov %rdi,$ctx # reassigned argument
mov %rsi,$inp # reassigned argument
@@ -908,23 +935,29 @@ $code.=<<___;
mov $E,16($ctx)
___
$code.=<<___ if ($win64);
movaps -40-6*16(%r14),%xmm6
movaps -40-5*16(%r14),%xmm7
movaps -40-4*16(%r14),%xmm8
movaps -40-3*16(%r14),%xmm9
movaps -40-2*16(%r14),%xmm10
movaps -40-1*16(%r14),%xmm11
movaps -40-6*16($fp),%xmm6
movaps -40-5*16($fp),%xmm7
movaps -40-4*16($fp),%xmm8
movaps -40-3*16($fp),%xmm9
movaps -40-2*16($fp),%xmm10
movaps -40-1*16($fp),%xmm11
___
$code.=<<___;
lea (%r14),%rsi
mov -40(%rsi),%r14
mov -32(%rsi),%r13
mov -24(%rsi),%r12
mov -16(%rsi),%rbp
mov -8(%rsi),%rbx
lea (%rsi),%rsp
mov -40($fp),%r14
.cfi_restore %r14
mov -32($fp),%r13
.cfi_restore %r13
mov -24($fp),%r12
.cfi_restore %r12
mov -16($fp),%rbp
.cfi_restore %rbp
mov -8($fp),%rbx
.cfi_restore %rbx
lea ($fp),%rsp
.cfi_def_cfa_register %rsp
.Lepilogue_ssse3:
ret
.cfi_endproc
.size sha1_block_data_order_ssse3,.-sha1_block_data_order_ssse3
___
@@ -945,26 +978,32 @@ $code.=<<___;
.align 16
sha1_block_data_order_avx:
_avx_shortcut:
mov %rsp,%rax
.cfi_startproc
mov %rsp,$fp
.cfi_def_cfa_register $fp
push %rbx
.cfi_push %rbx
push %rbp
.cfi_push %rbp
push %r12
.cfi_push %r12
push %r13 # redundant, done to share Win64 SE handler
.cfi_push %r13
push %r14
.cfi_push %r14
lea `-64-($win64?6*16:0)`(%rsp),%rsp
vzeroupper
___
$code.=<<___ if ($win64);
vmovaps %xmm6,-40-6*16(%rax)
vmovaps %xmm7,-40-5*16(%rax)
vmovaps %xmm8,-40-4*16(%rax)
vmovaps %xmm9,-40-3*16(%rax)
vmovaps %xmm10,-40-2*16(%rax)
vmovaps %xmm11,-40-1*16(%rax)
vmovaps %xmm6,-40-6*16($fp)
vmovaps %xmm7,-40-5*16($fp)
vmovaps %xmm8,-40-4*16($fp)
vmovaps %xmm9,-40-3*16($fp)
vmovaps %xmm10,-40-2*16($fp)
vmovaps %xmm11,-40-1*16($fp)
.Lprologue_avx:
___
$code.=<<___;
mov %rax,%r14 # original %rsp
and \$-64,%rsp
mov %rdi,$ctx # reassigned argument
mov %rsi,$inp # reassigned argument
@@ -1272,23 +1311,29 @@ $code.=<<___;
mov $E,16($ctx)
___
$code.=<<___ if ($win64);
movaps -40-6*16(%r14),%xmm6
movaps -40-5*16(%r14),%xmm7
movaps -40-4*16(%r14),%xmm8
movaps -40-3*16(%r14),%xmm9
movaps -40-2*16(%r14),%xmm10
movaps -40-1*16(%r14),%xmm11
movaps -40-6*16($fp),%xmm6
movaps -40-5*16($fp),%xmm7
movaps -40-4*16($fp),%xmm8
movaps -40-3*16($fp),%xmm9
movaps -40-2*16($fp),%xmm10
movaps -40-1*16($fp),%xmm11
___
$code.=<<___;
lea (%r14),%rsi
mov -40(%rsi),%r14
mov -32(%rsi),%r13
mov -24(%rsi),%r12
mov -16(%rsi),%rbp
mov -8(%rsi),%rbx
lea (%rsi),%rsp
mov -40($fp),%r14
.cfi_restore %r14
mov -32($fp),%r13
.cfi_restore %r13
mov -24($fp),%r12
.cfi_restore %r12
mov -16($fp),%rbp
.cfi_restore %rbp
mov -8($fp),%rbx
.cfi_restore %rbx
lea ($fp),%rsp
.cfi_def_cfa_register %rsp
.Lepilogue_avx:
ret
.cfi_endproc
.size sha1_block_data_order_avx,.-sha1_block_data_order_avx
___
@@ -1312,26 +1357,32 @@ $code.=<<___;
.align 16
sha1_block_data_order_avx2:
_avx2_shortcut:
mov %rsp,%rax
.cfi_startproc
mov %rsp,$fp
.cfi_def_cfa_register $fp
push %rbx
.cfi_push %rbx
push %rbp
.cfi_push %rbp
push %r12
.cfi_push %r12
push %r13
.cfi_push %r13
push %r14
.cfi_push %r14
vzeroupper
___
$code.=<<___ if ($win64);
lea -6*16(%rsp),%rsp
vmovaps %xmm6,-40-6*16(%rax)
vmovaps %xmm7,-40-5*16(%rax)
vmovaps %xmm8,-40-4*16(%rax)
vmovaps %xmm9,-40-3*16(%rax)
vmovaps %xmm10,-40-2*16(%rax)
vmovaps %xmm11,-40-1*16(%rax)
vmovaps %xmm6,-40-6*16($fp)
vmovaps %xmm7,-40-5*16($fp)
vmovaps %xmm8,-40-4*16($fp)
vmovaps %xmm9,-40-3*16($fp)
vmovaps %xmm10,-40-2*16($fp)
vmovaps %xmm11,-40-1*16($fp)
.Lprologue_avx2:
___
$code.=<<___;
mov %rax,%r14 # original %rsp
mov %rdi,$ctx # reassigned argument
mov %rsi,$inp # reassigned argument
mov %rdx,$num # reassigned argument
@@ -1751,23 +1802,29 @@ $code.=<<___;
vzeroupper
___
$code.=<<___ if ($win64);
movaps -40-6*16(%r14),%xmm6
movaps -40-5*16(%r14),%xmm7
movaps -40-4*16(%r14),%xmm8
movaps -40-3*16(%r14),%xmm9
movaps -40-2*16(%r14),%xmm10
movaps -40-1*16(%r14),%xmm11
movaps -40-6*16($fp),%xmm6
movaps -40-5*16($fp),%xmm7
movaps -40-4*16($fp),%xmm8
movaps -40-3*16($fp),%xmm9
movaps -40-2*16($fp),%xmm10
movaps -40-1*16($fp),%xmm11
___
$code.=<<___;
lea (%r14),%rsi
mov -40(%rsi),%r14
mov -32(%rsi),%r13
mov -24(%rsi),%r12
mov -16(%rsi),%rbp
mov -8(%rsi),%rbx
lea (%rsi),%rsp
mov -40($fp),%r14
.cfi_restore %r14
mov -32($fp),%r13
.cfi_restore %r13
mov -24($fp),%r12
.cfi_restore %r12
mov -16($fp),%rbp
.cfi_restore %rbp
mov -8($fp),%rbx
.cfi_restore %rbx
lea ($fp),%rsp
.cfi_def_cfa_register %rsp
.Lepilogue_avx2:
ret
.cfi_endproc
.size sha1_block_data_order_avx2,.-sha1_block_data_order_avx2
___
}
@@ -1908,15 +1965,13 @@ ssse3_handler:
cmp %r10,%rbx # context->Rip<prologue label
jb .Lcommon_seh_tail
mov 152($context),%rax # pull context->Rsp
mov 208($context),%rax # pull context->R11
mov 4(%r11),%r10d # HandlerData[1]
lea (%rsi,%r10),%r10 # epilogue label
cmp %r10,%rbx # context->Rip>=epilogue label
jae .Lcommon_seh_tail
mov 232($context),%rax # pull context->R14
lea -40-6*16(%rax),%rsi
lea 512($context),%rdi # &context.Xmm6
mov \$12,%ecx
@@ -1929,9 +1984,9 @@ ssse3_handler:
mov -40(%rax),%r14
mov %rbx,144($context) # restore context->Rbx
mov %rbp,160($context) # restore context->Rbp
mov %r12,216($context) # restore cotnext->R12
mov %r13,224($context) # restore cotnext->R13
mov %r14,232($context) # restore cotnext->R14
mov %r12,216($context) # restore context->R12
mov %r13,224($context) # restore context->R13
mov %r14,232($context) # restore context->R14
.Lcommon_seh_tail:
mov 8(%rax),%rdi
crypto/sha/asm/sha256-586.pl
+8 -5
View File
@@ -18,7 +18,7 @@
#
# Performance improvement over compiler generated code varies from
# 10% to 40% [see below]. Not very impressive on some µ-archs, but
# it's 5 times smaller and optimizies amount of writes.
# it's 5 times smaller and optimizes amount of writes.
#
# May 2012.
#
@@ -47,7 +47,7 @@
#
# Performance in clock cycles per processed byte (less is better):
#
# gcc icc x86 asm(*) SIMD x86_64 asm(**)
# gcc icc x86 asm(*) SIMD x86_64 asm(**)
# Pentium 46 57 40/38 - -
# PIII 36 33 27/24 - -
# P4 41 38 28 - 17.3
@@ -57,14 +57,17 @@
# Sandy Bridge 25 - 15.9 12.4 11.6
# Ivy Bridge 24 - 15.0 11.4 10.3
# Haswell 22 - 13.9 9.46 7.80
# Skylake 20 - 14.9 9.50 7.70
# Bulldozer 36 - 27/22 17.0 13.6
# VIA Nano 36 - 25/22 16.8 16.5
# Atom 50 - 30/25 21.9 18.9
# Silvermont 40 - 34/31 22.9 20.6
# Goldmont 29 - 20 16.3(***)
#
# (*) numbers after slash are for unrolled loop, where applicable;
# (**) x86_64 assembly performance is presented for reference
# purposes, results are best-available;
# (***) SHAEXT result is 4.1, strangely enough better than 64-bit one;
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
push(@INC,"${dir}","${dir}../../perlasm");
@@ -73,7 +76,7 @@ require "x86asm.pl";
$output=pop;
open STDOUT,">$output";
&asm_init($ARGV[0],"sha512-586.pl",$ARGV[$#ARGV] eq "386");
&asm_init($ARGV[0],$ARGV[$#ARGV] eq "386");
$xmm=$avx=0;
for (@ARGV) { $xmm=1 if (/-DOPENSSL_IA32_SSE2/); }
@@ -276,7 +279,7 @@ my $suffix=shift;
&mov ($Coff,"ecx");
&mov ($Doff,"edi");
&mov (&DWP(0,"esp"),"ebx"); # magic
&mov ($E,&DWP(16,"esi"));
&mov ($E,&DWP(16,"esi"));
&mov ("ebx",&DWP(20,"esi"));
&mov ("ecx",&DWP(24,"esi"));
&mov ("edi",&DWP(28,"esi"));
@@ -385,7 +388,7 @@ my @AH=($A,$K256);
&xor ($AH[1],"ecx"); # magic
&mov (&DWP(8,"esp"),"ecx");
&mov (&DWP(12,"esp"),"ebx");
&mov ($E,&DWP(16,"esi"));
&mov ($E,&DWP(16,"esi"));
&mov ("ebx",&DWP(20,"esi"));
&mov ("ecx",&DWP(24,"esi"));
&mov ("esi",&DWP(28,"esi"));
crypto/sha/asm/sha256-mb-x86_64.pl
+51 -5
View File
@@ -36,7 +36,7 @@
# (iii) "this" is for n=8, when we gather twice as much data, result
# for n=4 is 20.3+4.44=24.7;
# (iv) presented improvement coefficients are asymptotic limits and
# in real-life application are somewhat lower, e.g. for 2KB
# in real-life application are somewhat lower, e.g. for 2KB
# fragments they range from 75% to 130% (on Haswell);
$flavour = shift;
@@ -244,6 +244,7 @@ $code.=<<___;
.type sha256_multi_block,\@function,3
.align 32
sha256_multi_block:
.cfi_startproc
mov OPENSSL_ia32cap_P+4(%rip),%rcx
bt \$61,%rcx # check SHA bit
jc _shaext_shortcut
@@ -254,8 +255,11 @@ $code.=<<___ if ($avx);
___
$code.=<<___;
mov %rsp,%rax
.cfi_def_cfa_register %rax
push %rbx
.cfi_push %rbx
push %rbp
.cfi_push %rbp
___
$code.=<<___ if ($win64);
lea -0xa8(%rsp),%rsp
@@ -274,6 +278,7 @@ $code.=<<___;
sub \$`$REG_SZ*18`, %rsp
and \$-256,%rsp
mov %rax,`$REG_SZ*17`(%rsp) # original %rsp
.cfi_cfa_expression %rsp+`$REG_SZ*17`,deref,+8
.Lbody:
lea K256+128(%rip),$Tbl
lea `$REG_SZ*16`(%rsp),%rbx
@@ -391,6 +396,7 @@ $code.=<<___;
.Ldone:
mov `$REG_SZ*17`(%rsp),%rax # original %rsp
.cfi_def_cfa %rax,8
___
$code.=<<___ if ($win64);
movaps -0xb8(%rax),%xmm6
@@ -406,10 +412,14 @@ $code.=<<___ if ($win64);
___
$code.=<<___;
mov -16(%rax),%rbp
.cfi_restore %rbp
mov -8(%rax),%rbx
.cfi_restore %rbx
lea (%rax),%rsp
.cfi_def_cfa_register %rsp
.Lepilogue:
ret
.cfi_endproc
.size sha256_multi_block,.-sha256_multi_block
___
{{{
@@ -421,10 +431,14 @@ $code.=<<___;
.type sha256_multi_block_shaext,\@function,3
.align 32
sha256_multi_block_shaext:
.cfi_startproc
_shaext_shortcut:
mov %rsp,%rax
.cfi_def_cfa_register %rax
push %rbx
.cfi_push %rbx
push %rbp
.cfi_push %rbp
___
$code.=<<___ if ($win64);
lea -0xa8(%rsp),%rsp
@@ -758,10 +772,14 @@ $code.=<<___ if ($win64);
___
$code.=<<___;
mov -16(%rax),%rbp
.cfi_restore %rbp
mov -8(%rax),%rbx
.cfi_restore %rbx
lea (%rax),%rsp
.cfi_def_cfa_register %rsp
.Lepilogue_shaext:
ret
.cfi_endproc
.size sha256_multi_block_shaext,.-sha256_multi_block_shaext
___
}}}
@@ -921,6 +939,7 @@ $code.=<<___;
.type sha256_multi_block_avx,\@function,3
.align 32
sha256_multi_block_avx:
.cfi_startproc
_avx_shortcut:
___
$code.=<<___ if ($avx>1);
@@ -935,8 +954,11 @@ $code.=<<___ if ($avx>1);
___
$code.=<<___;
mov %rsp,%rax
.cfi_def_cfa_register %rax
push %rbx
.cfi_push %rbx
push %rbp
.cfi_push %rbp
___
$code.=<<___ if ($win64);
lea -0xa8(%rsp),%rsp
@@ -955,6 +977,7 @@ $code.=<<___;
sub \$`$REG_SZ*18`, %rsp
and \$-256,%rsp
mov %rax,`$REG_SZ*17`(%rsp) # original %rsp
.cfi_cfa_expression %rsp+`$REG_SZ*17`,deref,+8
.Lbody_avx:
lea K256+128(%rip),$Tbl
lea `$REG_SZ*16`(%rsp),%rbx
@@ -1070,6 +1093,7 @@ $code.=<<___;
.Ldone_avx:
mov `$REG_SZ*17`(%rsp),%rax # original %rsp
.cfi_def_cfa %rax,8
vzeroupper
___
$code.=<<___ if ($win64);
@@ -1086,10 +1110,14 @@ $code.=<<___ if ($win64);
___
$code.=<<___;
mov -16(%rax),%rbp
.cfi_restore %rbp
mov -8(%rax),%rbx
.cfi_restore %rbx
lea (%rax),%rsp
.cfi_def_cfa_register %rsp
.Lepilogue_avx:
ret
.cfi_endproc
.size sha256_multi_block_avx,.-sha256_multi_block_avx
___
if ($avx>1) {
@@ -1105,14 +1133,22 @@ $code.=<<___;
.type sha256_multi_block_avx2,\@function,3
.align 32
sha256_multi_block_avx2:
.cfi_startproc
_avx2_shortcut:
mov %rsp,%rax
.cfi_def_cfa_register %rax
push %rbx
.cfi_push %rbx
push %rbp
.cfi_push %rbp
push %r12
.cfi_push %r12
push %r13
.cfi_push %r13
push %r14
.cfi_push %r14
push %r15
.cfi_push %r15
___
$code.=<<___ if ($win64);
lea -0xa8(%rsp),%rsp
@@ -1131,6 +1167,7 @@ $code.=<<___;
sub \$`$REG_SZ*18`, %rsp
and \$-256,%rsp
mov %rax,`$REG_SZ*17`(%rsp) # original %rsp
.cfi_cfa_expression %rsp+`$REG_SZ*17`,deref,+8
.Lbody_avx2:
lea K256+128(%rip),$Tbl
lea 0x80($ctx),$ctx # size optimization
@@ -1246,6 +1283,7 @@ $code.=<<___;
.Ldone_avx2:
mov `$REG_SZ*17`(%rsp),%rax # original %rsp
.cfi_def_cfa %rax,8
vzeroupper
___
$code.=<<___ if ($win64);
@@ -1262,14 +1300,22 @@ $code.=<<___ if ($win64);
___
$code.=<<___;
mov -48(%rax),%r15
.cfi_restore %r15
mov -40(%rax),%r14
.cfi_restore %r14
mov -32(%rax),%r13
.cfi_restore %r13
mov -24(%rax),%r12
.cfi_restore %r12
mov -16(%rax),%rbp
.cfi_restore %rbp
mov -8(%rax),%rbx
.cfi_restore %rbx
lea (%rax),%rsp
.cfi_def_cfa_register %rsp
.Lepilogue_avx2:
ret
.cfi_endproc
.size sha256_multi_block_avx2,.-sha256_multi_block_avx2
___
} }}}
@@ -1462,10 +1508,10 @@ avx2_handler:
mov -48(%rax),%r15
mov %rbx,144($context) # restore context->Rbx
mov %rbp,160($context) # restore context->Rbp
mov %r12,216($context) # restore cotnext->R12
mov %r13,224($context) # restore cotnext->R13
mov %r14,232($context) # restore cotnext->R14
mov %r15,240($context) # restore cotnext->R15
mov %r12,216($context) # restore context->R12
mov %r13,224($context) # restore context->R13
mov %r14,232($context) # restore context->R14
mov %r15,240($context) # restore context->R15
lea -56-10*16(%rax),%rsi
lea 512($context),%rdi # &context.Xmm6
crypto/sha/asm/sha512-586.pl
+4 -3
View File
@@ -32,6 +32,7 @@
# Sandy Bridge 58 - 35 11.9 11.2
# Ivy Bridge 50 - 33 11.5 8.17
# Haswell 46 - 29 11.3 7.66
# Skylake 40 - 26 13.3 7.25
# Bulldozer 121 - 50 14.0 13.5
# VIA Nano 91 - 52 33 14.7
# Atom 126 - 68 48(***) 14.7
@@ -41,7 +42,7 @@
# (*) whichever best applicable.
# (**) x86_64 assembler performance is presented for reference
# purposes, the results are for integer-only code.
# (***) paddq is increadibly slow on Atom.
# (***) paddq is incredibly slow on Atom.
#
# IALU code-path is optimized for elder Pentiums. On vanilla Pentium
# performance improvement over compiler generated code reaches ~60%,
@@ -61,7 +62,7 @@ require "x86asm.pl";
$output=pop;
open STDOUT,">$output";
&asm_init($ARGV[0],"sha512-586.pl",$ARGV[$#ARGV] eq "386");
&asm_init($ARGV[0],$ARGV[$#ARGV] eq "386");
$sse2=0;
for (@ARGV) { $sse2=1 if (/-DOPENSSL_IA32_SSE2/); }
@@ -384,7 +385,7 @@ if ($sse2) {
&set_label("16_79_sse2",16);
for ($j=0;$j<2;$j++) { # 2x unroll
#&movq ("mm7",&QWP(8*(9+16-1),"esp")); # prefetched in BODY_00_15
#&movq ("mm7",&QWP(8*(9+16-1),"esp")); # prefetched in BODY_00_15
&movq ("mm5",&QWP(8*(9+16-14),"esp"));
&movq ("mm1","mm7");
&psrlq ("mm7",1);
crypto/sha/asm/sha512-armv8.pl
+482 -25
View File
@@ -1,17 +1,18 @@
#! /usr/bin/env perl
# Copyright 2014-2016 The OpenSSL Project Authors. All Rights Reserved.
# Copyright 2014-2018 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License"). You may not use
# this file except in compliance with the License. You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html
#
# ====================================================================
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
#
# Permission to use under GPLv2 terms is granted.
# ====================================================================
#
# SHA256/512 for ARMv8.
@@ -26,7 +27,8 @@
# Denver 2.01 10.5 (+26%) 6.70 (+8%)
# X-Gene 20.0 (+100%) 12.8 (+300%(***))
# Mongoose 2.36 13.0 (+50%) 8.36 (+33%)
#
# Kryo 1.92 17.4 (+30%) 11.2 (+8%)
#
# (*) Software SHA256 results are of lesser relevance, presented
# mostly for informational purposes.
# (**) The result is a trade-off: it's possible to improve it by
@@ -34,19 +36,37 @@
# on Cortex-A53 (or by 4 cycles per round).
# (***) Super-impressive coefficients over gcc-generated code are
# indication of some compiler "pathology", most notably code
# generated with -mgeneral-regs-only is significanty faster
# generated with -mgeneral-regs-only is significantly faster
# and the gap is only 40-90%.
#
# October 2016.
#
# Originally it was reckoned that it makes no sense to implement NEON
# version of SHA256 for 64-bit processors. This is because performance
# improvement on most wide-spread Cortex-A5x processors was observed
# to be marginal, same on Cortex-A53 and ~10% on A57. But then it was
# observed that 32-bit NEON SHA256 performs significantly better than
# 64-bit scalar version on *some* of the more recent processors. As
# result 64-bit NEON version of SHA256 was added to provide best
# all-round performance. For example it executes ~30% faster on X-Gene
# and Mongoose. [For reference, NEON version of SHA512 is bound to
# deliver much less improvement, likely *negative* on Cortex-A5x.
# Which is why NEON support is limited to SHA256.]
$flavour=shift;
$output=shift;
$output=pop;
$flavour=pop;
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
die "can't locate arm-xlate.pl";
if ($flavour && $flavour ne "void") {
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
die "can't locate arm-xlate.pl";
open OUT,"| \"$^X\" $xlate $flavour $output";
*STDOUT=*OUT;
open OUT,"| \"$^X\" $xlate $flavour $output";
*STDOUT=*OUT;
} else {
open STDOUT,">$output";
}
if ($output =~ /512/) {
$BITS=512;
@@ -83,7 +103,7 @@ my ($T0,$T1,$T2)=(@X[($i-8)&15],@X[($i-9)&15],@X[($i-10)&15]);
$T0=@X[$i+3] if ($i<11);
$code.=<<___ if ($i<16);
#ifndef __ARMEB__
#ifndef __AARCH64EB__
rev @X[$i],@X[$i] // $i
#endif
___
@@ -166,7 +186,9 @@ ___
}
$code.=<<___;
#include "arm_arch.h"
#ifndef __KERNEL__
# include "arm_arch.h"
#endif
.text
@@ -175,20 +197,28 @@ $code.=<<___;
.type $func,%function
.align 6
$func:
___
$code.=<<___ if ($SZ==4);
#ifdef __ILP32__
#ifndef __KERNEL__
# ifdef __ILP32__
ldrsw x16,.LOPENSSL_armcap_P
#else
# else
ldr x16,.LOPENSSL_armcap_P
#endif
# endif
adr x17,.LOPENSSL_armcap_P
add x16,x16,x17
ldr w16,[x16]
___
$code.=<<___ if ($SZ==4);
tst w16,#ARMV8_SHA256
b.ne .Lv8_entry
tst w16,#ARMV7_NEON
b.ne .Lneon_entry
___
$code.=<<___ if ($SZ==8);
tst w16,#ARMV8_SHA512
b.ne .Lv8_entry
___
$code.=<<___;
#endif
stp x29,x30,[sp,#-128]!
add x29,sp,#0
@@ -321,12 +351,14 @@ $code.=<<___ if ($SZ==4);
___
$code.=<<___;
.size .LK$BITS,.-.LK$BITS
#ifndef __KERNEL__
.align 3
.LOPENSSL_armcap_P:
#ifdef __ILP32__
# ifdef __ILP32__
.long OPENSSL_armcap_P-.
#else
# else
.quad OPENSSL_armcap_P-.
# endif
#endif
.asciz "SHA$BITS block transform for ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
.align 2
@@ -341,6 +373,7 @@ my ($W0,$W1)=("v16.4s","v17.4s");
my ($ABCD_SAVE,$EFGH_SAVE)=("v18.16b","v19.16b");
$code.=<<___;
#ifndef __KERNEL__
.type sha256_block_armv8,%function
.align 6
sha256_block_armv8:
@@ -409,11 +442,406 @@ $code.=<<___;
ldr x29,[sp],#16
ret
.size sha256_block_armv8,.-sha256_block_armv8
#endif
___
}
if ($SZ==4) { ######################################### NEON stuff #
# You'll surely note a lot of similarities with sha256-armv4 module,
# and of course it's not a coincidence. sha256-armv4 was used as
# initial template, but was adapted for ARMv8 instruction set and
# extensively re-tuned for all-round performance.
my @V = ($A,$B,$C,$D,$E,$F,$G,$H) = map("w$_",(3..10));
my ($t0,$t1,$t2,$t3,$t4) = map("w$_",(11..15));
my $Ktbl="x16";
my $Xfer="x17";
my @X = map("q$_",(0..3));
my ($T0,$T1,$T2,$T3,$T4,$T5,$T6,$T7) = map("q$_",(4..7,16..19));
my $j=0;
sub AUTOLOAD() # thunk [simplified] x86-style perlasm
{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./;
my $arg = pop;
$arg = "#$arg" if ($arg*1 eq $arg);
$code .= "\t$opcode\t".join(',',@_,$arg)."\n";
}
sub Dscalar { shift =~ m|[qv]([0-9]+)|?"d$1":""; }
sub Dlo { shift =~ m|[qv]([0-9]+)|?"v$1.d[0]":""; }
sub Dhi { shift =~ m|[qv]([0-9]+)|?"v$1.d[1]":""; }
sub Xupdate()
{ use integer;
my $body = shift;
my @insns = (&$body,&$body,&$body,&$body);
my ($a,$b,$c,$d,$e,$f,$g,$h);
&ext_8 ($T0,@X[0],@X[1],4); # X[1..4]
eval(shift(@insns));
eval(shift(@insns));
eval(shift(@insns));
&ext_8 ($T3,@X[2],@X[3],4); # X[9..12]
eval(shift(@insns));
eval(shift(@insns));
&mov (&Dscalar($T7),&Dhi(@X[3])); # X[14..15]
eval(shift(@insns));
eval(shift(@insns));
&ushr_32 ($T2,$T0,$sigma0[0]);
eval(shift(@insns));
&ushr_32 ($T1,$T0,$sigma0[2]);
eval(shift(@insns));
&add_32 (@X[0],@X[0],$T3); # X[0..3] += X[9..12]
eval(shift(@insns));
&sli_32 ($T2,$T0,32-$sigma0[0]);
eval(shift(@insns));
eval(shift(@insns));
&ushr_32 ($T3,$T0,$sigma0[1]);
eval(shift(@insns));
eval(shift(@insns));
&eor_8 ($T1,$T1,$T2);
eval(shift(@insns));
eval(shift(@insns));
&sli_32 ($T3,$T0,32-$sigma0[1]);
eval(shift(@insns));
eval(shift(@insns));
&ushr_32 ($T4,$T7,$sigma1[0]);
eval(shift(@insns));
eval(shift(@insns));
&eor_8 ($T1,$T1,$T3); # sigma0(X[1..4])
eval(shift(@insns));
eval(shift(@insns));
&sli_32 ($T4,$T7,32-$sigma1[0]);
eval(shift(@insns));
eval(shift(@insns));
&ushr_32 ($T5,$T7,$sigma1[2]);
eval(shift(@insns));
eval(shift(@insns));
&ushr_32 ($T3,$T7,$sigma1[1]);
eval(shift(@insns));
eval(shift(@insns));
&add_32 (@X[0],@X[0],$T1); # X[0..3] += sigma0(X[1..4])
eval(shift(@insns));
eval(shift(@insns));
&sli_u32 ($T3,$T7,32-$sigma1[1]);
eval(shift(@insns));
eval(shift(@insns));
&eor_8 ($T5,$T5,$T4);
eval(shift(@insns));
eval(shift(@insns));
eval(shift(@insns));
&eor_8 ($T5,$T5,$T3); # sigma1(X[14..15])
eval(shift(@insns));
eval(shift(@insns));
eval(shift(@insns));
&add_32 (@X[0],@X[0],$T5); # X[0..1] += sigma1(X[14..15])
eval(shift(@insns));
eval(shift(@insns));
eval(shift(@insns));
&ushr_32 ($T6,@X[0],$sigma1[0]);
eval(shift(@insns));
&ushr_32 ($T7,@X[0],$sigma1[2]);
eval(shift(@insns));
eval(shift(@insns));
&sli_32 ($T6,@X[0],32-$sigma1[0]);
eval(shift(@insns));
&ushr_32 ($T5,@X[0],$sigma1[1]);
eval(shift(@insns));
eval(shift(@insns));
&eor_8 ($T7,$T7,$T6);
eval(shift(@insns));
eval(shift(@insns));
&sli_32 ($T5,@X[0],32-$sigma1[1]);
eval(shift(@insns));
eval(shift(@insns));
&ld1_32 ("{$T0}","[$Ktbl], #16");
eval(shift(@insns));
&eor_8 ($T7,$T7,$T5); # sigma1(X[16..17])
eval(shift(@insns));
eval(shift(@insns));
&eor_8 ($T5,$T5,$T5);
eval(shift(@insns));
eval(shift(@insns));
&mov (&Dhi($T5), &Dlo($T7));
eval(shift(@insns));
eval(shift(@insns));
eval(shift(@insns));
&add_32 (@X[0],@X[0],$T5); # X[2..3] += sigma1(X[16..17])
eval(shift(@insns));
eval(shift(@insns));
eval(shift(@insns));
&add_32 ($T0,$T0,@X[0]);
while($#insns>=1) { eval(shift(@insns)); }
&st1_32 ("{$T0}","[$Xfer], #16");
eval(shift(@insns));
push(@X,shift(@X)); # "rotate" X[]
}
sub Xpreload()
{ use integer;
my $body = shift;
my @insns = (&$body,&$body,&$body,&$body);
my ($a,$b,$c,$d,$e,$f,$g,$h);
eval(shift(@insns));
eval(shift(@insns));
&ld1_8 ("{@X[0]}","[$inp],#16");
eval(shift(@insns));
eval(shift(@insns));
&ld1_32 ("{$T0}","[$Ktbl],#16");
eval(shift(@insns));
eval(shift(@insns));
eval(shift(@insns));
eval(shift(@insns));
&rev32 (@X[0],@X[0]);
eval(shift(@insns));
eval(shift(@insns));
eval(shift(@insns));
eval(shift(@insns));
&add_32 ($T0,$T0,@X[0]);
foreach (@insns) { eval; } # remaining instructions
&st1_32 ("{$T0}","[$Xfer], #16");
push(@X,shift(@X)); # "rotate" X[]
}
sub body_00_15 () {
(
'($a,$b,$c,$d,$e,$f,$g,$h)=@V;'.
'&add ($h,$h,$t1)', # h+=X[i]+K[i]
'&add ($a,$a,$t4);'. # h+=Sigma0(a) from the past
'&and ($t1,$f,$e)',
'&bic ($t4,$g,$e)',
'&eor ($t0,$e,$e,"ror#".($Sigma1[1]-$Sigma1[0]))',
'&add ($a,$a,$t2)', # h+=Maj(a,b,c) from the past
'&orr ($t1,$t1,$t4)', # Ch(e,f,g)
'&eor ($t0,$t0,$e,"ror#".($Sigma1[2]-$Sigma1[0]))', # Sigma1(e)
'&eor ($t4,$a,$a,"ror#".($Sigma0[1]-$Sigma0[0]))',
'&add ($h,$h,$t1)', # h+=Ch(e,f,g)
'&ror ($t0,$t0,"#$Sigma1[0]")',
'&eor ($t2,$a,$b)', # a^b, b^c in next round
'&eor ($t4,$t4,$a,"ror#".($Sigma0[2]-$Sigma0[0]))', # Sigma0(a)
'&add ($h,$h,$t0)', # h+=Sigma1(e)
'&ldr ($t1,sprintf "[sp,#%d]",4*(($j+1)&15)) if (($j&15)!=15);'.
'&ldr ($t1,"[$Ktbl]") if ($j==15);'.
'&and ($t3,$t3,$t2)', # (b^c)&=(a^b)
'&ror ($t4,$t4,"#$Sigma0[0]")',
'&add ($d,$d,$h)', # d+=h
'&eor ($t3,$t3,$b)', # Maj(a,b,c)
'$j++; unshift(@V,pop(@V)); ($t2,$t3)=($t3,$t2);'
)
}
$code.=<<___;
#ifdef __KERNEL__
.globl sha256_block_neon
#endif
.type sha256_block_neon,%function
.align 4
sha256_block_neon:
.Lneon_entry:
stp x29, x30, [sp, #-16]!
mov x29, sp
sub sp,sp,#16*4
adr $Ktbl,.LK256
add $num,$inp,$num,lsl#6 // len to point at the end of inp
ld1.8 {@X[0]},[$inp], #16
ld1.8 {@X[1]},[$inp], #16
ld1.8 {@X[2]},[$inp], #16
ld1.8 {@X[3]},[$inp], #16
ld1.32 {$T0},[$Ktbl], #16
ld1.32 {$T1},[$Ktbl], #16
ld1.32 {$T2},[$Ktbl], #16
ld1.32 {$T3},[$Ktbl], #16
rev32 @X[0],@X[0] // yes, even on
rev32 @X[1],@X[1] // big-endian
rev32 @X[2],@X[2]
rev32 @X[3],@X[3]
mov $Xfer,sp
add.32 $T0,$T0,@X[0]
add.32 $T1,$T1,@X[1]
add.32 $T2,$T2,@X[2]
st1.32 {$T0-$T1},[$Xfer], #32
add.32 $T3,$T3,@X[3]
st1.32 {$T2-$T3},[$Xfer]
sub $Xfer,$Xfer,#32
ldp $A,$B,[$ctx]
ldp $C,$D,[$ctx,#8]
ldp $E,$F,[$ctx,#16]
ldp $G,$H,[$ctx,#24]
ldr $t1,[sp,#0]
mov $t2,wzr
eor $t3,$B,$C
mov $t4,wzr
b .L_00_48
.align 4
.L_00_48:
___
&Xupdate(\&body_00_15);
&Xupdate(\&body_00_15);
&Xupdate(\&body_00_15);
&Xupdate(\&body_00_15);
$code.=<<___;
cmp $t1,#0 // check for K256 terminator
ldr $t1,[sp,#0]
sub $Xfer,$Xfer,#64
bne .L_00_48
sub $Ktbl,$Ktbl,#256 // rewind $Ktbl
cmp $inp,$num
mov $Xfer, #64
csel $Xfer, $Xfer, xzr, eq
sub $inp,$inp,$Xfer // avoid SEGV
mov $Xfer,sp
___
&Xpreload(\&body_00_15);
&Xpreload(\&body_00_15);
&Xpreload(\&body_00_15);
&Xpreload(\&body_00_15);
$code.=<<___;
add $A,$A,$t4 // h+=Sigma0(a) from the past
ldp $t0,$t1,[$ctx,#0]
add $A,$A,$t2 // h+=Maj(a,b,c) from the past
ldp $t2,$t3,[$ctx,#8]
add $A,$A,$t0 // accumulate
add $B,$B,$t1
ldp $t0,$t1,[$ctx,#16]
add $C,$C,$t2
add $D,$D,$t3
ldp $t2,$t3,[$ctx,#24]
add $E,$E,$t0
add $F,$F,$t1
ldr $t1,[sp,#0]
stp $A,$B,[$ctx,#0]
add $G,$G,$t2
mov $t2,wzr
stp $C,$D,[$ctx,#8]
add $H,$H,$t3
stp $E,$F,[$ctx,#16]
eor $t3,$B,$C
stp $G,$H,[$ctx,#24]
mov $t4,wzr
mov $Xfer,sp
b.ne .L_00_48
ldr x29,[x29]
add sp,sp,#16*4+16
ret
.size sha256_block_neon,.-sha256_block_neon
___
}
if ($SZ==8) {
my $Ktbl="x3";
my @H = map("v$_.16b",(0..4));
my ($fg,$de,$m9_10)=map("v$_.16b",(5..7));
my @MSG=map("v$_.16b",(16..23));
my ($W0,$W1)=("v24.2d","v25.2d");
my ($AB,$CD,$EF,$GH)=map("v$_.16b",(26..29));
$code.=<<___;
#ifndef __KERNEL__
.type sha512_block_armv8,%function
.align 6
sha512_block_armv8:
.Lv8_entry:
stp x29,x30,[sp,#-16]!
add x29,sp,#0
ld1 {@MSG[0]-@MSG[3]},[$inp],#64 // load input
ld1 {@MSG[4]-@MSG[7]},[$inp],#64
ld1.64 {@H[0]-@H[3]},[$ctx] // load context
adr $Ktbl,.LK512
rev64 @MSG[0],@MSG[0]
rev64 @MSG[1],@MSG[1]
rev64 @MSG[2],@MSG[2]
rev64 @MSG[3],@MSG[3]
rev64 @MSG[4],@MSG[4]
rev64 @MSG[5],@MSG[5]
rev64 @MSG[6],@MSG[6]
rev64 @MSG[7],@MSG[7]
b .Loop_hw
.align 4
.Loop_hw:
ld1.64 {$W0},[$Ktbl],#16
subs $num,$num,#1
sub x4,$inp,#128
orr $AB,@H[0],@H[0] // offload
orr $CD,@H[1],@H[1]
orr $EF,@H[2],@H[2]
orr $GH,@H[3],@H[3]
csel $inp,$inp,x4,ne // conditional rewind
___
for($i=0;$i<32;$i++) {
$code.=<<___;
add.i64 $W0,$W0,@MSG[0]
ld1.64 {$W1},[$Ktbl],#16
ext $W0,$W0,$W0,#8
ext $fg,@H[2],@H[3],#8
ext $de,@H[1],@H[2],#8
add.i64 @H[3],@H[3],$W0 // "T1 + H + K512[i]"
sha512su0 @MSG[0],@MSG[1]
ext $m9_10,@MSG[4],@MSG[5],#8
sha512h @H[3],$fg,$de
sha512su1 @MSG[0],@MSG[7],$m9_10
add.i64 @H[4],@H[1],@H[3] // "D + T1"
sha512h2 @H[3],$H[1],@H[0]
___
($W0,$W1)=($W1,$W0); push(@MSG,shift(@MSG));
@H = (@H[3],@H[0],@H[4],@H[2],@H[1]);
}
for(;$i<40;$i++) {
$code.=<<___ if ($i<39);
ld1.64 {$W1},[$Ktbl],#16
___
$code.=<<___ if ($i==39);
sub $Ktbl,$Ktbl,#$rounds*$SZ // rewind
___
$code.=<<___;
add.i64 $W0,$W0,@MSG[0]
ld1 {@MSG[0]},[$inp],#16 // load next input
ext $W0,$W0,$W0,#8
ext $fg,@H[2],@H[3],#8
ext $de,@H[1],@H[2],#8
add.i64 @H[3],@H[3],$W0 // "T1 + H + K512[i]"
sha512h @H[3],$fg,$de
rev64 @MSG[0],@MSG[0]
add.i64 @H[4],@H[1],@H[3] // "D + T1"
sha512h2 @H[3],$H[1],@H[0]
___
($W0,$W1)=($W1,$W0); push(@MSG,shift(@MSG));
@H = (@H[3],@H[0],@H[4],@H[2],@H[1]);
}
$code.=<<___;
add.i64 @H[0],@H[0],$AB // accumulate
add.i64 @H[1],@H[1],$CD
add.i64 @H[2],@H[2],$EF
add.i64 @H[3],@H[3],$GH
cbnz $num,.Loop_hw
st1.64 {@H[0]-@H[3]},[$ctx] // store context
ldr x29,[sp],#16
ret
.size sha512_block_armv8,.-sha512_block_armv8
#endif
___
}
$code.=<<___;
#ifndef __KERNEL__
.comm OPENSSL_armcap_P,4,4
#endif
___
{ my %opcode = (
@@ -431,14 +859,43 @@ ___
}
}
{ my %opcode = (
"sha512h" => 0xce608000, "sha512h2" => 0xce608400,
"sha512su0" => 0xcec08000, "sha512su1" => 0xce608800 );
sub unsha512 {
my ($mnemonic,$arg)=@_;
$arg =~ m/[qv]([0-9]+)[^,]*,\s*[qv]([0-9]+)[^,]*(?:,\s*[qv]([0-9]+))?/o
&&
sprintf ".inst\t0x%08x\t//%s %s",
$opcode{$mnemonic}|$1|($2<<5)|($3<<16),
$mnemonic,$arg;
}
}
open SELF,$0;
while(<SELF>) {
next if (/^#!/);
last if (!s/^#/\/\// and !/^$/);
print;
}
close SELF;
foreach(split("\n",$code)) {
s/\`([^\`]*)\`/eval($1)/geo;
s/\`([^\`]*)\`/eval($1)/ge;
s/\b(sha256\w+)\s+([qv].*)/unsha256($1,$2)/geo;
s/\b(sha512\w+)\s+([qv].*)/unsha512($1,$2)/ge or
s/\b(sha256\w+)\s+([qv].*)/unsha256($1,$2)/ge;
s/\.\w?32\b//o and s/\.16b/\.4s/go;
m/(ld|st)1[^\[]+\[0\]/o and s/\.4s/\.s/go;
s/\bq([0-9]+)\b/v$1.16b/g; # old->new registers
s/\.[ui]?8(\s)/$1/;
s/\.\w?64\b// and s/\.16b/\.2d/g or
s/\.\w?32\b// and s/\.16b/\.4s/g;
m/\bext\b/ and s/\.2d/\.16b/g or
m/(ld|st)1[^\[]+\[0\]/ and s/\.4s/\.s/g;
print $_,"\n";
}
crypto/sha/asm/sha512-mips.pl
+13 -11
View File
@@ -60,16 +60,16 @@ $flavour = shift || "o32"; # supported flavours are o32,n32,64,nubi32,nubi64
if ($flavour =~ /64|n32/i) {
$PTR_LA="dla";
$PTR_ADD="dadd"; # incidentally works even on n32
$PTR_SUB="dsub"; # incidentally works even on n32
$PTR_ADD="daddu"; # incidentally works even on n32
$PTR_SUB="dsubu"; # incidentally works even on n32
$REG_S="sd";
$REG_L="ld";
$PTR_SLL="dsll"; # incidentally works even on n32
$SZREG=8;
} else {
$PTR_LA="la";
$PTR_ADD="add";
$PTR_SUB="sub";
$PTR_ADD="addu";
$PTR_SUB="subu";
$REG_S="sw";
$REG_L="lw";
$PTR_SLL="sll";
@@ -135,8 +135,12 @@ my ($i,$a,$b,$c,$d,$e,$f,$g,$h)=@_;
my ($T1,$tmp0,$tmp1,$tmp2)=(@X[4],@X[5],@X[6],@X[7]);
$code.=<<___ if ($i<15);
#if defined(_MIPS_ARCH_MIPS32R6) || defined(_MIPS_ARCH_MIPS64R6)
${LD} @X[1],`($i+1)*$SZ`($inp)
#else
${LD}l @X[1],`($i+1)*$SZ+$MSB`($inp)
${LD}r @X[1],`($i+1)*$SZ+$LSB`($inp)
#endif
___
$code.=<<___ if (!$big_endian && $i<16 && $SZ==4);
#if defined(_MIPS_ARCH_MIPS32R2) || defined(_MIPS_ARCH_MIPS64R2)
@@ -298,13 +302,7 @@ $FRAMESIZE=16*$SZ+16*$SZREG;
$SAVED_REGS_MASK = ($flavour =~ /nubi/i) ? "0xc0fff008" : "0xc0ff0000";
$code.=<<___;
#ifdef OPENSSL_FIPSCANISTER
# include <openssl/fipssyms.h>
#endif
#if defined(__mips_smartmips) && !defined(_MIPS_ARCH_MIPS32R2)
#define _MIPS_ARCH_MIPS32R2
#endif
#include "mips_arch.h"
.text
.set noat
@@ -369,8 +367,12 @@ $code.=<<___;
.align 5
.Loop:
#if defined(_MIPS_ARCH_MIPS32R6) || defined(_MIPS_ARCH_MIPS64R6)
${LD} @X[0],($inp)
#else
${LD}l @X[0],$MSB($inp)
${LD}r @X[0],$LSB($inp)
#endif
___
for ($i=0;$i<16;$i++)
{ &BODY_00_15($i,@V); unshift(@V,pop(@V)); push(@X,shift(@X)); }
crypto/sha/asm/sha512-parisc.pl
+4 -4
View File
@@ -8,7 +8,7 @@
# ====================================================================
# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
@@ -368,7 +368,7 @@ L\$parisc1
___
@V=( $Ahi, $Alo, $Bhi, $Blo, $Chi, $Clo, $Dhi, $Dlo,
$Ehi, $Elo, $Fhi, $Flo, $Ghi, $Glo, $Hhi, $Hlo) =
$Ehi, $Elo, $Fhi, $Flo, $Ghi, $Glo, $Hhi, $Hlo) =
( "%r1", "%r2", "%r3", "%r4", "%r5", "%r6", "%r7", "%r8",
"%r9","%r10","%r11","%r12","%r13","%r14","%r15","%r16");
$a0 ="%r17";
@@ -419,7 +419,7 @@ $code.=<<___;
add $t0,$hlo,$hlo
shd $ahi,$alo,$Sigma0[0],$t0
addc $t1,$hhi,$hhi ; h += Sigma1(e)
shd $alo,$ahi,$Sigma0[0],$t1
shd $alo,$ahi,$Sigma0[0],$t1
add $a0,$hlo,$hlo
shd $ahi,$alo,$Sigma0[1],$t2
addc $a1,$hhi,$hhi ; h += Ch(e,f,g)
@@ -773,7 +773,7 @@ foreach (split("\n",$code)) {
s/shd\s+(%r[0-9]+),(%r[0-9]+),([0-9]+)/
$3>31 ? sprintf("shd\t%$2,%$1,%d",$3-32) # rotation for >=32
: sprintf("shd\t%$1,%$2,%d",$3)/e or
# translate made up instructons: _ror, _shr, _align, _shl
# translate made up instructions: _ror, _shr, _align, _shl
s/_ror(\s+)(%r[0-9]+),/
($SZ==4 ? "shd" : "shrpd")."$1$2,$2,"/e or
crypto/sha/asm/sha512-ppc.pl
+1 -1
View File
@@ -26,7 +26,7 @@
#
# (*) 64-bit code in 32-bit application context, which actually is
# on TODO list. It should be noted that for safe deployment in
# 32-bit *mutli-threaded* context asyncronous signals should be
# 32-bit *multi-threaded* context asynchronous signals should be
# blocked upon entry to SHA512 block routine. This is because
# 32-bit signaling procedure invalidates upper halves of GPRs.
# Context switch procedure preserves them, but not signaling:-(
crypto/sha/asm/sha512-s390x.pl
+5 -3
View File
@@ -8,7 +8,7 @@
# ====================================================================
# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
@@ -170,6 +170,8 @@ ___
}
$code.=<<___;
#include "s390x_arch.h"
.text
.align 64
.type $Table,\@object
@@ -244,7 +246,7 @@ $Func:
___
$code.=<<___ if ($kimdfunc);
larl %r1,OPENSSL_s390xcap_P
lg %r0,16(%r1) # check kimd capabilities
lg %r0,S390X_KIMD(%r1) # check kimd capabilities
tmhh %r0,`0x8000>>$kimdfunc`
jz .Lsoftware
lghi %r0,$kimdfunc
@@ -308,7 +310,7 @@ $code.=<<___;
cl${g} $inp,`$frame+4*$SIZE_T`($sp)
jne .Lloop
lm${g} %r6,%r15,`$frame+6*$SIZE_T`($sp)
lm${g} %r6,%r15,`$frame+6*$SIZE_T`($sp)
br %r14
.size $Func,.-$Func
.string "SHA${label} block transform for s390x, CRYPTOGAMS by <appro\@openssl.org>"
crypto/sha/asm/sha512-sparcv9.pl
+4 -4
View File
@@ -8,12 +8,12 @@
# ====================================================================
# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
#
# Hardware SPARC T4 support by David S. Miller <davem@davemloft.net>.
# Hardware SPARC T4 support by David S. Miller
# ====================================================================
# SHA256 performance improvement over compiler generated code varies
@@ -102,7 +102,7 @@ if ($output =~ /512/) {
$locals=0; # X[16] is register resident
@X=("%o0","%o1","%o2","%o3","%o4","%o5","%g1","%o7");
$A="%l0";
$B="%l1";
$C="%l2";
@@ -254,7 +254,7 @@ $code.=<<___;
$SLL $a,`$SZ*8-@Sigma0[1]`,$tmp1
xor $tmp0,$h,$h
$SRL $a,@Sigma0[2],$tmp0
xor $tmp1,$h,$h
xor $tmp1,$h,$h
$SLL $a,`$SZ*8-@Sigma0[0]`,$tmp1
xor $tmp0,$h,$h
xor $tmp1,$h,$h ! Sigma0(a)
crypto/sha/asm/sha512-x86_64.pl
+139 -48
View File
@@ -95,9 +95,11 @@
# Haswell 12.2 9.28(+31%) 7.80(+56%) 7.66 5.40(+42%)
# Skylake 11.4 9.03(+26%) 7.70(+48%) 7.25 5.20(+40%)
# Bulldozer 21.1 13.6(+54%) 13.6(+54%(***)) 13.5 8.58(+57%)
# Ryzen 11.0 9.02(+22%) 2.05(+440%) 7.05 5.67(+20%)
# VIA Nano 23.0 16.5(+39%) - 14.7 -
# Atom 23.0 18.9(+22%) - 14.7 -
# Silvermont 27.4 20.6(+33%) - 17.5 -
# Knights L 27.4 21.0(+30%) 19.6(+40%) 17.5 12.8(+37%)
# Goldmont 18.9 14.3(+32%) 4.16(+350%) 12.0 -
#
# (*) whichever best applicable, including SHAEXT;
@@ -176,7 +178,7 @@ $Tbl="%rbp";
$_ctx="16*$SZ+0*8(%rsp)";
$_inp="16*$SZ+1*8(%rsp)";
$_end="16*$SZ+2*8(%rsp)";
$_rsp="16*$SZ+3*8(%rsp)";
$_rsp="`16*$SZ+3*8`(%rsp)";
$framesz="16*$SZ+4*8";
@@ -269,6 +271,7 @@ $code=<<___;
.type $func,\@function,3
.align 16
$func:
.cfi_startproc
___
$code.=<<___ if ($SZ==4 || $avx);
lea OPENSSL_ia32cap_P(%rip),%r11
@@ -301,13 +304,20 @@ $code.=<<___ if ($SZ==4);
jnz .Lssse3_shortcut
___
$code.=<<___;
mov %rsp,%rax # copy %rsp
.cfi_def_cfa_register %rax
push %rbx
.cfi_push %rbx
push %rbp
.cfi_push %rbp
push %r12
.cfi_push %r12
push %r13
.cfi_push %r13
push %r14
.cfi_push %r14
push %r15
mov %rsp,%r11 # copy %rsp
.cfi_push %r15
shl \$4,%rdx # num*16
sub \$$framesz,%rsp
lea ($inp,%rdx,$SZ),%rdx # inp+num*16*$SZ
@@ -315,7 +325,8 @@ $code.=<<___;
mov $ctx,$_ctx # save ctx, 1st arg
mov $inp,$_inp # save inp, 2nd arh
mov %rdx,$_end # save end pointer, "3rd" arg
mov %r11,$_rsp # save copy of %rsp
mov %rax,$_rsp # save copy of %rsp
.cfi_cfa_expression $_rsp,deref,+8
.Lprologue:
mov $SZ*0($ctx),$A
@@ -382,15 +393,24 @@ $code.=<<___;
jb .Lloop
mov $_rsp,%rsi
mov (%rsi),%r15
mov 8(%rsi),%r14
mov 16(%rsi),%r13
mov 24(%rsi),%r12
mov 32(%rsi),%rbp
mov 40(%rsi),%rbx
lea 48(%rsi),%rsp
.cfi_def_cfa %rsi,8
mov -48(%rsi),%r15
.cfi_restore %r15
mov -40(%rsi),%r14
.cfi_restore %r14
mov -32(%rsi),%r13
.cfi_restore %r13
mov -24(%rsi),%r12
.cfi_restore %r12
mov -16(%rsi),%rbp
.cfi_restore %rbp
mov -8(%rsi),%rbx
.cfi_restore %rbx
lea (%rsi),%rsp
.cfi_def_cfa_register %rsp
.Lepilogue:
ret
.cfi_endproc
.size $func,.-$func
___
@@ -760,14 +780,22 @@ $code.=<<___;
.type ${func}_ssse3,\@function,3
.align 64
${func}_ssse3:
.cfi_startproc
.Lssse3_shortcut:
mov %rsp,%rax # copy %rsp
.cfi_def_cfa_register %rax
push %rbx
.cfi_push %rbx
push %rbp
.cfi_push %rbp
push %r12
.cfi_push %r12
push %r13
.cfi_push %r13
push %r14
.cfi_push %r14
push %r15
mov %rsp,%r11 # copy %rsp
.cfi_push %r15
shl \$4,%rdx # num*16
sub \$`$framesz+$win64*16*4`,%rsp
lea ($inp,%rdx,$SZ),%rdx # inp+num*16*$SZ
@@ -775,7 +803,8 @@ ${func}_ssse3:
mov $ctx,$_ctx # save ctx, 1st arg
mov $inp,$_inp # save inp, 2nd arh
mov %rdx,$_end # save end pointer, "3rd" arg
mov %r11,$_rsp # save copy of %rsp
mov %rax,$_rsp # save copy of %rsp
.cfi_cfa_expression $_rsp,deref,+8
___
$code.=<<___ if ($win64);
movaps %xmm6,16*$SZ+32(%rsp)
@@ -1074,6 +1103,7 @@ $code.=<<___;
jb .Lloop_ssse3
mov $_rsp,%rsi
.cfi_def_cfa %rsi,8
___
$code.=<<___ if ($win64);
movaps 16*$SZ+32(%rsp),%xmm6
@@ -1082,15 +1112,23 @@ $code.=<<___ if ($win64);
movaps 16*$SZ+80(%rsp),%xmm9
___
$code.=<<___;
mov (%rsi),%r15
mov 8(%rsi),%r14
mov 16(%rsi),%r13
mov 24(%rsi),%r12
mov 32(%rsi),%rbp
mov 40(%rsi),%rbx
lea 48(%rsi),%rsp
mov -48(%rsi),%r15
.cfi_restore %r15
mov -40(%rsi),%r14
.cfi_restore %r14
mov -32(%rsi),%r13
.cfi_restore %r13
mov -24(%rsi),%r12
.cfi_restore %r12
mov -16(%rsi),%rbp
.cfi_restore %rbp
mov -8(%rsi),%rbx
.cfi_restore %rbx
lea (%rsi),%rsp
.cfi_def_cfa_register %rsp
.Lepilogue_ssse3:
ret
.cfi_endproc
.size ${func}_ssse3,.-${func}_ssse3
___
}
@@ -1104,14 +1142,22 @@ $code.=<<___;
.type ${func}_xop,\@function,3
.align 64
${func}_xop:
.cfi_startproc
.Lxop_shortcut:
mov %rsp,%rax # copy %rsp
.cfi_def_cfa_register %rax
push %rbx
.cfi_push %rbx
push %rbp
.cfi_push %rbp
push %r12
.cfi_push %r12
push %r13
.cfi_push %r13
push %r14
.cfi_push %r14
push %r15
mov %rsp,%r11 # copy %rsp
.cfi_push %r15
shl \$4,%rdx # num*16
sub \$`$framesz+$win64*16*($SZ==4?4:6)`,%rsp
lea ($inp,%rdx,$SZ),%rdx # inp+num*16*$SZ
@@ -1119,7 +1165,8 @@ ${func}_xop:
mov $ctx,$_ctx # save ctx, 1st arg
mov $inp,$_inp # save inp, 2nd arh
mov %rdx,$_end # save end pointer, "3rd" arg
mov %r11,$_rsp # save copy of %rsp
mov %rax,$_rsp # save copy of %rsp
.cfi_cfa_expression $_rsp,deref,+8
___
$code.=<<___ if ($win64);
movaps %xmm6,16*$SZ+32(%rsp)
@@ -1446,6 +1493,7 @@ $code.=<<___;
jb .Lloop_xop
mov $_rsp,%rsi
.cfi_def_cfa %rsi,8
vzeroupper
___
$code.=<<___ if ($win64);
@@ -1459,15 +1507,23 @@ $code.=<<___ if ($win64 && $SZ>4);
movaps 16*$SZ+112(%rsp),%xmm11
___
$code.=<<___;
mov (%rsi),%r15
mov 8(%rsi),%r14
mov 16(%rsi),%r13
mov 24(%rsi),%r12
mov 32(%rsi),%rbp
mov 40(%rsi),%rbx
lea 48(%rsi),%rsp
mov -48(%rsi),%r15
.cfi_restore %r15
mov -40(%rsi),%r14
.cfi_restore %r14
mov -32(%rsi),%r13
.cfi_restore %r13
mov -24(%rsi),%r12
.cfi_restore %r12
mov -16(%rsi),%rbp
.cfi_restore %rbp
mov -8(%rsi),%rbx
.cfi_restore %rbx
lea (%rsi),%rsp
.cfi_def_cfa_register %rsp
.Lepilogue_xop:
ret
.cfi_endproc
.size ${func}_xop,.-${func}_xop
___
}
@@ -1480,14 +1536,22 @@ $code.=<<___;
.type ${func}_avx,\@function,3
.align 64
${func}_avx:
.cfi_startproc
.Lavx_shortcut:
mov %rsp,%rax # copy %rsp
.cfi_def_cfa_register %rax
push %rbx
.cfi_push %rbx
push %rbp
.cfi_push %rbp
push %r12
.cfi_push %r12
push %r13
.cfi_push %r13
push %r14
.cfi_push %r14
push %r15
mov %rsp,%r11 # copy %rsp
.cfi_push %r15
shl \$4,%rdx # num*16
sub \$`$framesz+$win64*16*($SZ==4?4:6)`,%rsp
lea ($inp,%rdx,$SZ),%rdx # inp+num*16*$SZ
@@ -1495,7 +1559,8 @@ ${func}_avx:
mov $ctx,$_ctx # save ctx, 1st arg
mov $inp,$_inp # save inp, 2nd arh
mov %rdx,$_end # save end pointer, "3rd" arg
mov %r11,$_rsp # save copy of %rsp
mov %rax,$_rsp # save copy of %rsp
.cfi_cfa_expression $_rsp,deref,+8
___
$code.=<<___ if ($win64);
movaps %xmm6,16*$SZ+32(%rsp)
@@ -1754,6 +1819,7 @@ $code.=<<___;
jb .Lloop_avx
mov $_rsp,%rsi
.cfi_def_cfa %rsi,8
vzeroupper
___
$code.=<<___ if ($win64);
@@ -1767,15 +1833,23 @@ $code.=<<___ if ($win64 && $SZ>4);
movaps 16*$SZ+112(%rsp),%xmm11
___
$code.=<<___;
mov (%rsi),%r15
mov 8(%rsi),%r14
mov 16(%rsi),%r13
mov 24(%rsi),%r12
mov 32(%rsi),%rbp
mov 40(%rsi),%rbx
lea 48(%rsi),%rsp
mov -48(%rsi),%r15
.cfi_restore %r15
mov -40(%rsi),%r14
.cfi_restore %r14
mov -32(%rsi),%r13
.cfi_restore %r13
mov -24(%rsi),%r12
.cfi_restore %r12
mov -16(%rsi),%rbp
.cfi_restore %rbp
mov -8(%rsi),%rbx
.cfi_restore %rbx
lea (%rsi),%rsp
.cfi_def_cfa_register %rsp
.Lepilogue_avx:
ret
.cfi_endproc
.size ${func}_avx,.-${func}_avx
___
@@ -1783,7 +1857,7 @@ if ($avx>1) {{
######################################################################
# AVX2+BMI code path
#
my $a5=$SZ==4?"%esi":"%rsi"; # zap $inp
my $a5=$SZ==4?"%esi":"%rsi"; # zap $inp
my $PUSH8=8*2*$SZ;
use integer;
@@ -1831,14 +1905,22 @@ $code.=<<___;
.type ${func}_avx2,\@function,3
.align 64
${func}_avx2:
.cfi_startproc
.Lavx2_shortcut:
mov %rsp,%rax # copy %rsp
.cfi_def_cfa_register %rax
push %rbx
.cfi_push %rbx
push %rbp
.cfi_push %rbp
push %r12
.cfi_push %r12
push %r13
.cfi_push %r13
push %r14
.cfi_push %r14
push %r15
mov %rsp,%r11 # copy %rsp
.cfi_push %r15
sub \$`2*$SZ*$rounds+4*8+$win64*16*($SZ==4?4:6)`,%rsp
shl \$4,%rdx # num*16
and \$-256*$SZ,%rsp # align stack frame
@@ -1847,7 +1929,8 @@ ${func}_avx2:
mov $ctx,$_ctx # save ctx, 1st arg
mov $inp,$_inp # save inp, 2nd arh
mov %rdx,$_end # save end pointer, "3rd" arg
mov %r11,$_rsp # save copy of %rsp
mov %rax,$_rsp # save copy of %rsp
.cfi_cfa_expression $_rsp,deref,+8
___
$code.=<<___ if ($win64);
movaps %xmm6,16*$SZ+32(%rsp)
@@ -2128,6 +2211,7 @@ $code.=<<___;
.Ldone_avx2:
lea ($Tbl),%rsp
mov $_rsp,%rsi
.cfi_def_cfa %rsi,8
vzeroupper
___
$code.=<<___ if ($win64);
@@ -2141,15 +2225,23 @@ $code.=<<___ if ($win64 && $SZ>4);
movaps 16*$SZ+112(%rsp),%xmm11
___
$code.=<<___;
mov (%rsi),%r15
mov 8(%rsi),%r14
mov 16(%rsi),%r13
mov 24(%rsi),%r12
mov 32(%rsi),%rbp
mov 40(%rsi),%rbx
lea 48(%rsi),%rsp
mov -48(%rsi),%r15
.cfi_restore %r15
mov -40(%rsi),%r14
.cfi_restore %r14
mov -32(%rsi),%r13
.cfi_restore %r13
mov -24(%rsi),%r12
.cfi_restore %r12
mov -16(%rsi),%rbp
.cfi_restore %rbp
mov -8(%rsi),%rbx
.cfi_restore %rbx
lea (%rsi),%rsp
.cfi_def_cfa_register %rsp
.Lepilogue_avx2:
ret
.cfi_endproc
.size ${func}_avx2,.-${func}_avx2
___
}}
@@ -2209,7 +2301,6 @@ ___
$code.=<<___;
mov %rax,%rsi # put aside Rsp
mov 16*$SZ+3*8(%rax),%rax # pull $_rsp
lea 48(%rax),%rax
mov -8(%rax),%rbx
mov -16(%rax),%rbp
crypto/sha/asm/sha512p8-ppc.pl
+10 -1
View File
@@ -25,11 +25,20 @@
# sha1-ppc.pl and 1.6x slower than aes-128-cbc. Another interesting
# result is degree of computational resources' utilization. POWER8 is
# "massively multi-threaded chip" and difference between single- and
# maximum multi-process benchmark results tells that utlization is
# maximum multi-process benchmark results tells that utilization is
# whooping 94%. For sha512-ppc.pl we get [not unimpressive] 84% and
# for sha1-ppc.pl - 73%. 100% means that multi-process result equals
# to single-process one, given that all threads end up on the same
# physical core.
#
######################################################################
# Believed-to-be-accurate results in cycles per processed byte [on
# little-endian system]. Numbers in square brackets are for 64-bit
# build of sha512-ppc.pl, presented for reference.
#
# POWER8
# SHA256 9.9 [15.8]
# SHA512 6.3 [10.3]
$flavour=shift;
$output =shift;