Latest update and remove TLSv1.3 draft
This commit is contained in:
+1440
-108
@@ -8,10 +8,10 @@
|
||||
|
||||
#
|
||||
# ====================================================================
|
||||
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
|
||||
# project. The module is, however, dual licensed under OpenSSL and
|
||||
# CRYPTOGAMS licenses depending on where you obtain it. For further
|
||||
# details see http://www.openssl.org/~appro/cryptogams/.
|
||||
# Written by Andy Polyakov, @dot-asm, initially for use in the OpenSSL
|
||||
# project. The module is dual licensed under OpenSSL and CRYPTOGAMS
|
||||
# licenses depending on where you obtain it. For further details see
|
||||
# https://github.com/dot-asm/cryptogams/.
|
||||
# ====================================================================
|
||||
#
|
||||
# This module implements Poly1305 hash for PowerPC.
|
||||
@@ -44,6 +44,13 @@
|
||||
#
|
||||
# On side note, Power ISA 2.07 enables vector base 2^26 implementation,
|
||||
# and POWER8 might have capacity to break 1.0 cycle per byte barrier...
|
||||
#
|
||||
# January 2019
|
||||
#
|
||||
# ... Unfortunately not:-( Estimate was a projection of ARM result,
|
||||
# but ARM has vector multiply-n-add instruction, while PowerISA does
|
||||
# not, not one usable in the context. Improvement is ~40% over -m64
|
||||
# result above and is ~1.43 on little-endian systems.
|
||||
|
||||
$flavour = shift;
|
||||
|
||||
@@ -99,6 +106,7 @@ $code.=<<___;
|
||||
std r0,0($ctx) # zero hash value
|
||||
std r0,8($ctx)
|
||||
std r0,16($ctx)
|
||||
stw r0,24($ctx) # clear is_base2_26
|
||||
|
||||
$UCMP $inp,r0
|
||||
beq- Lno_key
|
||||
@@ -140,6 +148,7 @@ Lno_key:
|
||||
.globl .poly1305_blocks
|
||||
.align 4
|
||||
.poly1305_blocks:
|
||||
Lpoly1305_blocks:
|
||||
srdi. $len,$len,4
|
||||
beq- Labort
|
||||
|
||||
@@ -238,60 +247,120 @@ Labort:
|
||||
.long 0
|
||||
.byte 0,12,4,1,0x80,5,4,0
|
||||
.size .poly1305_blocks,.-.poly1305_blocks
|
||||
___
|
||||
{
|
||||
my ($h0,$h1,$h2,$h3,$h4,$t0) = map("r$_",(7..12));
|
||||
|
||||
$code.=<<___;
|
||||
.globl .poly1305_emit
|
||||
.align 4
|
||||
.align 5
|
||||
.poly1305_emit:
|
||||
ld $h0,0($ctx) # load hash
|
||||
ld $h1,8($ctx)
|
||||
ld $h2,16($ctx)
|
||||
ld $padbit,0($nonce) # load nonce
|
||||
ld $nonce,8($nonce)
|
||||
lwz $h0,0($ctx) # load hash value base 2^26
|
||||
lwz $h1,4($ctx)
|
||||
lwz $h2,8($ctx)
|
||||
lwz $h3,12($ctx)
|
||||
lwz $h4,16($ctx)
|
||||
lwz r0,24($ctx) # is_base2_26
|
||||
|
||||
addic $d0,$h0,5 # compare to modulus
|
||||
addze $d1,$h1
|
||||
addze $d2,$h2
|
||||
sldi $h1,$h1,26 # base 2^26 -> base 2^64
|
||||
sldi $t0,$h2,52
|
||||
srdi $h2,$h2,12
|
||||
sldi $h3,$h3,14
|
||||
add $h0,$h0,$h1
|
||||
addc $h0,$h0,$t0
|
||||
sldi $t0,$h4,40
|
||||
srdi $h4,$h4,24
|
||||
adde $h1,$h2,$h3
|
||||
addc $h1,$h1,$t0
|
||||
addze $h2,$h4
|
||||
|
||||
srdi $mask,$d2,2 # did it carry/borrow?
|
||||
neg $mask,$mask
|
||||
ld $h3,0($ctx) # load hash value base 2^64
|
||||
ld $h4,8($ctx)
|
||||
ld $t0,16($ctx)
|
||||
|
||||
neg r0,r0
|
||||
xor $h0,$h0,$h3 # choose between radixes
|
||||
xor $h1,$h1,$h4
|
||||
xor $h2,$h2,$t0
|
||||
and $h0,$h0,r0
|
||||
and $h1,$h1,r0
|
||||
and $h2,$h2,r0
|
||||
xor $h0,$h0,$h3
|
||||
xor $h1,$h1,$h4
|
||||
xor $h2,$h2,$t0
|
||||
|
||||
addic $h3,$h0,5 # compare to modulus
|
||||
addze $h4,$h1
|
||||
addze $t0,$h2
|
||||
|
||||
srdi $t0,$t0,2 # see if it carried/borrowed
|
||||
neg $t0,$t0
|
||||
|
||||
andc $h0,$h0,$t0
|
||||
and $h3,$h3,$t0
|
||||
andc $h1,$h1,$t0
|
||||
and $h4,$h4,$t0
|
||||
or $h0,$h0,$h3
|
||||
or $h1,$h1,$h4
|
||||
|
||||
lwz $t0,4($nonce)
|
||||
lwz $h2,12($nonce)
|
||||
lwz $h3,0($nonce)
|
||||
lwz $h4,8($nonce)
|
||||
|
||||
insrdi $h3,$t0,32,0
|
||||
insrdi $h4,$h2,32,0
|
||||
|
||||
addc $h0,$h0,$h3 # accumulate nonce
|
||||
adde $h1,$h1,$h4
|
||||
|
||||
addi $ctx,$mac,-1
|
||||
addi $mac,$mac,7
|
||||
|
||||
stbu $h0,1($ctx) # write [little-endian] result
|
||||
srdi $h0,$h0,8
|
||||
stbu $h1,1($mac)
|
||||
srdi $h1,$h1,8
|
||||
|
||||
stbu $h0,1($ctx)
|
||||
srdi $h0,$h0,8
|
||||
stbu $h1,1($mac)
|
||||
srdi $h1,$h1,8
|
||||
|
||||
stbu $h0,1($ctx)
|
||||
srdi $h0,$h0,8
|
||||
stbu $h1,1($mac)
|
||||
srdi $h1,$h1,8
|
||||
|
||||
stbu $h0,1($ctx)
|
||||
srdi $h0,$h0,8
|
||||
stbu $h1,1($mac)
|
||||
srdi $h1,$h1,8
|
||||
|
||||
stbu $h0,1($ctx)
|
||||
srdi $h0,$h0,8
|
||||
stbu $h1,1($mac)
|
||||
srdi $h1,$h1,8
|
||||
|
||||
stbu $h0,1($ctx)
|
||||
srdi $h0,$h0,8
|
||||
stbu $h1,1($mac)
|
||||
srdi $h1,$h1,8
|
||||
|
||||
stbu $h0,1($ctx)
|
||||
srdi $h0,$h0,8
|
||||
stbu $h1,1($mac)
|
||||
srdi $h1,$h1,8
|
||||
|
||||
stbu $h0,1($ctx)
|
||||
stbu $h1,1($mac)
|
||||
|
||||
andc $h0,$h0,$mask
|
||||
and $d0,$d0,$mask
|
||||
andc $h1,$h1,$mask
|
||||
and $d1,$d1,$mask
|
||||
or $h0,$h0,$d0
|
||||
or $h1,$h1,$d1
|
||||
___
|
||||
$code.=<<___ if (!$LITTLE_ENDIAN);
|
||||
rotldi $padbit,$padbit,32 # flip nonce words
|
||||
rotldi $nonce,$nonce,32
|
||||
___
|
||||
$code.=<<___;
|
||||
addc $h0,$h0,$padbit # accumulate nonce
|
||||
adde $h1,$h1,$nonce
|
||||
___
|
||||
$code.=<<___ if ($LITTLE_ENDIAN);
|
||||
std $h0,0($mac) # write result
|
||||
std $h1,8($mac)
|
||||
___
|
||||
$code.=<<___ if (!$LITTLE_ENDIAN);
|
||||
extrdi r0,$h0,32,0
|
||||
li $d0,4
|
||||
stwbrx $h0,0,$mac # write result
|
||||
extrdi $h0,$h1,32,0
|
||||
li $d1,8
|
||||
stwbrx r0,$d0,$mac
|
||||
li $d2,12
|
||||
stwbrx $h1,$d1,$mac
|
||||
stwbrx $h0,$d2,$mac
|
||||
___
|
||||
$code.=<<___;
|
||||
blr
|
||||
.long 0
|
||||
.byte 0,12,0x14,0,0,0,3,0
|
||||
.size .poly1305_emit,.-.poly1305_emit
|
||||
___
|
||||
} else {
|
||||
} } else {
|
||||
###############################################################################
|
||||
# base 2^32 implementation
|
||||
|
||||
@@ -309,6 +378,7 @@ $code.=<<___;
|
||||
stw r0,8($ctx)
|
||||
stw r0,12($ctx)
|
||||
stw r0,16($ctx)
|
||||
stw r0,24($ctx) # clear is_base2_26
|
||||
|
||||
$UCMP $inp,r0
|
||||
beq- Lno_key
|
||||
@@ -353,6 +423,7 @@ Lno_key:
|
||||
.globl .poly1305_blocks
|
||||
.align 4
|
||||
.poly1305_blocks:
|
||||
Lpoly1305_blocks:
|
||||
srwi. $len,$len,4
|
||||
beq- Labort
|
||||
|
||||
@@ -560,17 +631,389 @@ Labort:
|
||||
.long 0
|
||||
.byte 0,12,4,1,0x80,18,4,0
|
||||
.size .poly1305_blocks,.-.poly1305_blocks
|
||||
___
|
||||
{
|
||||
my ($h0,$h1,$h2,$h3,$h4,$t0,$t1) = map("r$_",(6..12));
|
||||
|
||||
$code.=<<___;
|
||||
.globl .poly1305_emit
|
||||
.align 4
|
||||
.align 5
|
||||
.poly1305_emit:
|
||||
$STU $sp,-$FRAME($sp)
|
||||
lwz r0,24($ctx) # is_base2_26
|
||||
lwz $h0,0($ctx) # load hash value
|
||||
lwz $h1,4($ctx)
|
||||
lwz $h2,8($ctx)
|
||||
lwz $h3,12($ctx)
|
||||
lwz $h4,16($ctx)
|
||||
cmplwi r0,0
|
||||
beq Lemit_base2_32
|
||||
|
||||
slwi $t0,$h1,26 # base 2^26 -> base 2^32
|
||||
srwi $h1,$h1,6
|
||||
slwi $t1,$h2,20
|
||||
srwi $h2,$h2,12
|
||||
addc $h0,$h0,$t0
|
||||
slwi $t0,$h3,14
|
||||
srwi $h3,$h3,18
|
||||
adde $h1,$h1,$t1
|
||||
slwi $t1,$h4,8
|
||||
srwi $h4,$h4,24
|
||||
adde $h2,$h2,$t0
|
||||
adde $h3,$h3,$t1
|
||||
addze $h4,$h4
|
||||
|
||||
Lemit_base2_32:
|
||||
addic r0,$h0,5 # compare to modulus
|
||||
addze r0,$h1
|
||||
addze r0,$h2
|
||||
addze r0,$h3
|
||||
addze r0,$h4
|
||||
|
||||
srwi r0,r0,2 # see if it carried/borrowed
|
||||
neg r0,r0
|
||||
andi. r0,r0,5
|
||||
|
||||
addc $h0,$h0,r0
|
||||
lwz r0,0($nonce)
|
||||
addze $h1,$h1
|
||||
lwz $t0,4($nonce)
|
||||
addze $h2,$h2
|
||||
lwz $t1,8($nonce)
|
||||
addze $h3,$h3
|
||||
lwz $h4,12($nonce)
|
||||
|
||||
addc $h0,$h0,r0 # accumulate nonce
|
||||
adde $h1,$h1,$t0
|
||||
adde $h2,$h2,$t1
|
||||
adde $h3,$h3,$h4
|
||||
|
||||
addi $ctx,$mac,-1
|
||||
addi $mac,$mac,7
|
||||
|
||||
stbu $h0,1($ctx) # write [little-endian] result
|
||||
srwi $h0,$h0,8
|
||||
stbu $h2,1($mac)
|
||||
srwi $h2,$h2,8
|
||||
|
||||
stbu $h0,1($ctx)
|
||||
srwi $h0,$h0,8
|
||||
stbu $h2,1($mac)
|
||||
srwi $h2,$h2,8
|
||||
|
||||
stbu $h0,1($ctx)
|
||||
srwi $h0,$h0,8
|
||||
stbu $h2,1($mac)
|
||||
srwi $h2,$h2,8
|
||||
|
||||
stbu $h0,1($ctx)
|
||||
stbu $h2,1($mac)
|
||||
|
||||
stbu $h1,1($ctx)
|
||||
srwi $h1,$h1,8
|
||||
stbu $h3,1($mac)
|
||||
srwi $h3,$h3,8
|
||||
|
||||
stbu $h1,1($ctx)
|
||||
srwi $h1,$h1,8
|
||||
stbu $h3,1($mac)
|
||||
srwi $h3,$h3,8
|
||||
|
||||
stbu $h1,1($ctx)
|
||||
srwi $h1,$h1,8
|
||||
stbu $h3,1($mac)
|
||||
srwi $h3,$h3,8
|
||||
|
||||
stbu $h1,1($ctx)
|
||||
stbu $h3,1($mac)
|
||||
|
||||
blr
|
||||
.long 0
|
||||
.byte 0,12,0x14,0,0,0,3,0
|
||||
.size .poly1305_emit,.-.poly1305_emit
|
||||
___
|
||||
} }
|
||||
{{{
|
||||
########################################################################
|
||||
# PowerISA 2.07/VSX section #
|
||||
########################################################################
|
||||
|
||||
my $LOCALS= 6*$SIZE_T;
|
||||
my $VSXFRAME = $LOCALS + 6*$SIZE_T;
|
||||
$VSXFRAME += 128; # local variables
|
||||
$VSXFRAME += 13*16; # v20-v31 offload
|
||||
|
||||
my $BIG_ENDIAN = ($flavour !~ /le/) ? 4 : 0;
|
||||
|
||||
########################################################################
|
||||
# Layout of opaque area is following:
|
||||
#
|
||||
# unsigned __int32 h[5]; # current hash value base 2^26
|
||||
# unsigned __int32 pad;
|
||||
# unsigned __int32 is_base2_26, pad;
|
||||
# unsigned __int64 r[2]; # key value base 2^64
|
||||
# struct { unsigned __int32 r^2, r^4, r^1, r^3; } r[9];
|
||||
#
|
||||
# where r^n are base 2^26 digits of powers of multiplier key. There are
|
||||
# 5 digits, but last four are interleaved with multiples of 5, totalling
|
||||
# in 9 elements: r0, r1, 5*r1, r2, 5*r2, r3, 5*r3, r4, 5*r4. Order of
|
||||
# powers is as they appear in register, not memory.
|
||||
|
||||
my ($H0, $H1, $H2, $H3, $H4) = map("v$_",(0..4));
|
||||
my ($I0, $I1, $I2, $I3, $I4) = map("v$_",(5..9));
|
||||
my ($R0, $R1, $S1, $R2, $S2) = map("v$_",(10..14));
|
||||
my ($R3, $S3, $R4, $S4) = ($R1, $S1, $R2, $S2);
|
||||
my ($ACC0, $ACC1, $ACC2, $ACC3, $ACC4) = map("v$_",(15..19));
|
||||
my ($T0, $T1, $T2, $T3, $T4) = map("v$_",(20..24));
|
||||
my ($_26,$_4,$_40,$_14,$mask26,$padbits,$I2perm) = map("v$_",(25..31));
|
||||
my ($x00,$x60,$x70,$x10,$x20,$x30,$x40,$x50) = (0, map("r$_",(7,8,27..31)));
|
||||
my ($ctx_,$_ctx,$const) = map("r$_",(10..12));
|
||||
|
||||
if ($flavour =~ /64/) {
|
||||
###############################################################################
|
||||
# setup phase of poly1305_blocks_vsx is different on 32- and 64-bit platforms,
|
||||
# but the base 2^26 computational part is same...
|
||||
|
||||
my ($h0,$h1,$h2,$d0,$d1,$d2, $r0,$r1,$s1, $t0,$t1) = map("r$_",(6..11,27..31));
|
||||
my $mask = "r0";
|
||||
|
||||
$code.=<<___;
|
||||
.globl .poly1305_blocks_vsx
|
||||
.align 5
|
||||
.poly1305_blocks_vsx:
|
||||
lwz r7,24($ctx) # is_base2_26
|
||||
cmpldi $len,128
|
||||
bge __poly1305_blocks_vsx
|
||||
|
||||
neg r0,r7 # is_base2_26 as mask
|
||||
lwz r7,0($ctx) # load hash base 2^26
|
||||
lwz r8,4($ctx)
|
||||
lwz r9,8($ctx)
|
||||
lwz r10,12($ctx)
|
||||
lwz r11,16($ctx)
|
||||
|
||||
sldi r8,r8,26 # base 2^26 -> base 2^64
|
||||
sldi r12,r9,52
|
||||
add r7,r7,r8
|
||||
srdi r9,r9,12
|
||||
sldi r10,r10,14
|
||||
addc r7,r7,r12
|
||||
sldi r8,r11,40
|
||||
adde r9,r9,r10
|
||||
srdi r11,r11,24
|
||||
addc r9,r9,r8
|
||||
addze r11,r11
|
||||
|
||||
ld r8,0($ctx) # load hash base 2^64
|
||||
ld r10,8($ctx)
|
||||
ld r12,16($ctx)
|
||||
|
||||
xor r7,r7,r8 # select between radixes
|
||||
xor r9,r9,r10
|
||||
xor r11,r11,r12
|
||||
and r7,r7,r0
|
||||
and r9,r9,r0
|
||||
and r11,r11,r0
|
||||
xor r7,r7,r8
|
||||
xor r9,r9,r10
|
||||
xor r11,r11,r12
|
||||
|
||||
li r0,0
|
||||
std r7,0($ctx) # store hash base 2^64
|
||||
std r9,8($ctx)
|
||||
std r11,16($ctx)
|
||||
stw r0,24($ctx) # clear is_base2_26
|
||||
|
||||
b Lpoly1305_blocks
|
||||
.long 0
|
||||
.byte 0,12,0x14,0,0,0,4,0
|
||||
.size .poly1305_blocks_vsx,.-.poly1305_blocks_vsx
|
||||
|
||||
.align 5
|
||||
__poly1305_mul:
|
||||
mulld $d0,$h0,$r0 # h0*r0
|
||||
mulhdu $d1,$h0,$r0
|
||||
|
||||
mulld $t0,$h1,$s1 # h1*5*r1
|
||||
mulhdu $t1,$h1,$s1
|
||||
addc $d0,$d0,$t0
|
||||
adde $d1,$d1,$t1
|
||||
|
||||
mulld $t0,$h0,$r1 # h0*r1
|
||||
mulhdu $d2,$h0,$r1
|
||||
addc $d1,$d1,$t0
|
||||
addze $d2,$d2
|
||||
|
||||
mulld $t0,$h1,$r0 # h1*r0
|
||||
mulhdu $t1,$h1,$r0
|
||||
addc $d1,$d1,$t0
|
||||
adde $d2,$d2,$t1
|
||||
|
||||
mulld $t0,$h2,$s1 # h2*5*r1
|
||||
mulld $t1,$h2,$r0 # h2*r0
|
||||
addc $d1,$d1,$t0
|
||||
adde $d2,$d2,$t1
|
||||
|
||||
andc $t0,$d2,$mask # final reduction step
|
||||
and $h2,$d2,$mask
|
||||
srdi $t1,$t0,2
|
||||
add $t0,$t0,$t1
|
||||
addc $h0,$d0,$t0
|
||||
addze $h1,$d1
|
||||
addze $h2,$h2
|
||||
|
||||
blr
|
||||
.long 0
|
||||
.byte 0,12,0x14,0,0,0,0,0
|
||||
.size __poly1305_mul,.-__poly1305_mul
|
||||
|
||||
.align 5
|
||||
__poly1305_splat:
|
||||
extrdi $d0,$h0,26,38
|
||||
extrdi $d1,$h0,26,12
|
||||
stw $d0,0x00($t1)
|
||||
|
||||
extrdi $d2,$h0,12,0
|
||||
slwi $d0,$d1,2
|
||||
stw $d1,0x10($t1)
|
||||
add $d0,$d0,$d1 # * 5
|
||||
stw $d0,0x20($t1)
|
||||
|
||||
insrdi $d2,$h1,14,38
|
||||
slwi $d0,$d2,2
|
||||
stw $d2,0x30($t1)
|
||||
add $d0,$d0,$d2 # * 5
|
||||
stw $d0,0x40($t1)
|
||||
|
||||
extrdi $d1,$h1,26,24
|
||||
extrdi $d2,$h1,24,0
|
||||
slwi $d0,$d1,2
|
||||
stw $d1,0x50($t1)
|
||||
add $d0,$d0,$d1 # * 5
|
||||
stw $d0,0x60($t1)
|
||||
|
||||
insrdi $d2,$h2,3,37
|
||||
slwi $d0,$d2,2
|
||||
stw $d2,0x70($t1)
|
||||
add $d0,$d0,$d2 # * 5
|
||||
stw $d0,0x80($t1)
|
||||
|
||||
blr
|
||||
.long 0
|
||||
.byte 0,12,0x14,0,0,0,0,0
|
||||
.size __poly1305_splat,.-__poly1305_splat
|
||||
|
||||
.align 5
|
||||
__poly1305_blocks_vsx:
|
||||
$STU $sp,-$VSXFRAME($sp)
|
||||
mflr r0
|
||||
$PUSH r28,`$FRAME-$SIZE_T*4`($sp)
|
||||
$PUSH r29,`$FRAME-$SIZE_T*3`($sp)
|
||||
$PUSH r30,`$FRAME-$SIZE_T*2`($sp)
|
||||
$PUSH r31,`$FRAME-$SIZE_T*1`($sp)
|
||||
$PUSH r0,`$FRAME+$LRSAVE`($sp)
|
||||
li r10,`15+$LOCALS+128`
|
||||
li r11,`31+$LOCALS+128`
|
||||
mfspr r12,256
|
||||
stvx v20,r10,$sp
|
||||
addi r10,r10,32
|
||||
stvx v21,r11,$sp
|
||||
addi r11,r11,32
|
||||
stvx v22,r10,$sp
|
||||
addi r10,r10,32
|
||||
stvx v23,r10,$sp
|
||||
addi r10,r10,32
|
||||
stvx v24,r11,$sp
|
||||
addi r11,r11,32
|
||||
stvx v25,r10,$sp
|
||||
addi r10,r10,32
|
||||
stvx v26,r10,$sp
|
||||
addi r10,r10,32
|
||||
stvx v27,r11,$sp
|
||||
addi r11,r11,32
|
||||
stvx v28,r10,$sp
|
||||
addi r10,r10,32
|
||||
stvx v29,r11,$sp
|
||||
addi r11,r11,32
|
||||
stvx v30,r10,$sp
|
||||
stvx v31,r11,$sp
|
||||
stw r12,`$VSXFRAME-$SIZE_T*5-4`($sp)# save vrsave
|
||||
li r12,-1
|
||||
mtspr 256,r12 # preserve all AltiVec registers
|
||||
$PUSH r27,`$VSXFRAME-$SIZE_T*5`($sp)
|
||||
$PUSH r28,`$VSXFRAME-$SIZE_T*4`($sp)
|
||||
$PUSH r29,`$VSXFRAME-$SIZE_T*3`($sp)
|
||||
$PUSH r30,`$VSXFRAME-$SIZE_T*2`($sp)
|
||||
$PUSH r31,`$VSXFRAME-$SIZE_T*1`($sp)
|
||||
$PUSH r0,`$VSXFRAME+$LRSAVE`($sp)
|
||||
|
||||
bl LPICmeup
|
||||
|
||||
li $x10,0x10
|
||||
li $x20,0x20
|
||||
li $x30,0x30
|
||||
li $x40,0x40
|
||||
li $x50,0x50
|
||||
lvx_u $mask26,$x00,$const
|
||||
lvx_u $_26,$x10,$const
|
||||
lvx_u $_40,$x20,$const
|
||||
lvx_u $I2perm,$x30,$const
|
||||
lvx_u $padbits,$x40,$const
|
||||
|
||||
cmplwi r7,0 # is_base2_26?
|
||||
bne Lskip_init_vsx
|
||||
|
||||
ld $r0,32($ctx) # load key base 2^64
|
||||
ld $r1,40($ctx)
|
||||
srdi $s1,$r1,2
|
||||
li $mask,3
|
||||
add $s1,$s1,$r1 # s1 = r1 + r1>>2
|
||||
|
||||
mr $h0,$r0 # "calculate" r^1
|
||||
mr $h1,$r1
|
||||
li $h2,0
|
||||
addi $t1,$ctx,`48+(12^$BIG_ENDIAN)`
|
||||
bl __poly1305_splat
|
||||
|
||||
bl __poly1305_mul # caclulate r^2
|
||||
addi $t1,$ctx,`48+(4^$BIG_ENDIAN)`
|
||||
bl __poly1305_splat
|
||||
|
||||
bl __poly1305_mul # caclulate r^3
|
||||
addi $t1,$ctx,`48+(8^$BIG_ENDIAN)`
|
||||
bl __poly1305_splat
|
||||
|
||||
bl __poly1305_mul # caclulate r^4
|
||||
addi $t1,$ctx,`48+(0^$BIG_ENDIAN)`
|
||||
bl __poly1305_splat
|
||||
|
||||
ld $h0,0($ctx) # load hash
|
||||
ld $h1,8($ctx)
|
||||
ld $h2,16($ctx)
|
||||
|
||||
extrdi $d0,$h0,26,38 # base 2^64 -> base 2^26
|
||||
extrdi $d1,$h0,26,12
|
||||
extrdi $d2,$h0,12,0
|
||||
mtvrwz $H0,$d0
|
||||
insrdi $d2,$h1,14,38
|
||||
mtvrwz $H1,$d1
|
||||
extrdi $d1,$h1,26,24
|
||||
mtvrwz $H2,$d2
|
||||
extrdi $d2,$h1,24,0
|
||||
mtvrwz $H3,$d1
|
||||
insrdi $d2,$h2,3,37
|
||||
mtvrwz $H4,$d2
|
||||
___
|
||||
} else {
|
||||
###############################################################################
|
||||
# 32-bit initialization
|
||||
|
||||
my ($h0,$h1,$h2,$h3,$h4,$t0,$t1) = map("r$_",(7..11,0,12));
|
||||
my ($R3,$S3,$R4,$S4)=($I1,$I2,$I3,$I4);
|
||||
|
||||
$code.=<<___;
|
||||
.globl .poly1305_blocks_vsx
|
||||
.align 5
|
||||
.poly1305_blocks_vsx:
|
||||
lwz r7,24($ctx) # is_base2_26
|
||||
cmplwi $len,128
|
||||
bge __poly1305_blocks_vsx
|
||||
cmplwi r7,0
|
||||
beq Lpoly1305_blocks
|
||||
|
||||
lwz $h0,0($ctx) # load hash
|
||||
lwz $h1,4($ctx)
|
||||
@@ -578,68 +1021,957 @@ Labort:
|
||||
lwz $h3,12($ctx)
|
||||
lwz $h4,16($ctx)
|
||||
|
||||
addic $d0,$h0,5 # compare to modulus
|
||||
addze $d1,$h1
|
||||
addze $d2,$h2
|
||||
addze $d3,$h3
|
||||
addze $mask,$h4
|
||||
slwi $t0,$h1,26 # base 2^26 -> base 2^32
|
||||
srwi $h1,$h1,6
|
||||
slwi $t1,$h2,20
|
||||
srwi $h2,$h2,12
|
||||
addc $h0,$h0,$t0
|
||||
slwi $t0,$h3,14
|
||||
srwi $h3,$h3,18
|
||||
adde $h1,$h1,$t1
|
||||
slwi $t1,$h4,8
|
||||
srwi $h4,$h4,24
|
||||
adde $h2,$h2,$t0
|
||||
li $t0,0
|
||||
adde $h3,$h3,$t1
|
||||
addze $h4,$h4
|
||||
|
||||
srwi $mask,$mask,2 # did it carry/borrow?
|
||||
neg $mask,$mask
|
||||
stw $h0,0($ctx) # store hash base 2^32
|
||||
stw $h1,4($ctx)
|
||||
stw $h2,8($ctx)
|
||||
stw $h3,12($ctx)
|
||||
stw $h4,16($ctx)
|
||||
stw $t0,24($ctx) # clear is_base2_26
|
||||
|
||||
andc $h0,$h0,$mask
|
||||
and $d0,$d0,$mask
|
||||
andc $h1,$h1,$mask
|
||||
and $d1,$d1,$mask
|
||||
or $h0,$h0,$d0
|
||||
lwz $d0,0($nonce) # load nonce
|
||||
andc $h2,$h2,$mask
|
||||
and $d2,$d2,$mask
|
||||
or $h1,$h1,$d1
|
||||
lwz $d1,4($nonce)
|
||||
andc $h3,$h3,$mask
|
||||
and $d3,$d3,$mask
|
||||
or $h2,$h2,$d2
|
||||
lwz $d2,8($nonce)
|
||||
or $h3,$h3,$d3
|
||||
lwz $d3,12($nonce)
|
||||
b Lpoly1305_blocks
|
||||
.long 0
|
||||
.byte 0,12,0x14,0,0,0,4,0
|
||||
.size .poly1305_blocks_vsx,.-.poly1305_blocks_vsx
|
||||
|
||||
.align 5
|
||||
__poly1305_mul:
|
||||
vmulouw $ACC0,$H0,$R0
|
||||
vmulouw $ACC1,$H1,$R0
|
||||
vmulouw $ACC2,$H2,$R0
|
||||
vmulouw $ACC3,$H3,$R0
|
||||
vmulouw $ACC4,$H4,$R0
|
||||
|
||||
vmulouw $T0,$H4,$S1
|
||||
vaddudm $ACC0,$ACC0,$T0
|
||||
vmulouw $T0,$H0,$R1
|
||||
vaddudm $ACC1,$ACC1,$T0
|
||||
vmulouw $T0,$H1,$R1
|
||||
vaddudm $ACC2,$ACC2,$T0
|
||||
vmulouw $T0,$H2,$R1
|
||||
vaddudm $ACC3,$ACC3,$T0
|
||||
vmulouw $T0,$H3,$R1
|
||||
vaddudm $ACC4,$ACC4,$T0
|
||||
|
||||
vmulouw $T0,$H3,$S2
|
||||
vaddudm $ACC0,$ACC0,$T0
|
||||
vmulouw $T0,$H4,$S2
|
||||
vaddudm $ACC1,$ACC1,$T0
|
||||
vmulouw $T0,$H0,$R2
|
||||
vaddudm $ACC2,$ACC2,$T0
|
||||
vmulouw $T0,$H1,$R2
|
||||
vaddudm $ACC3,$ACC3,$T0
|
||||
vmulouw $T0,$H2,$R2
|
||||
vaddudm $ACC4,$ACC4,$T0
|
||||
|
||||
vmulouw $T0,$H2,$S3
|
||||
vaddudm $ACC0,$ACC0,$T0
|
||||
vmulouw $T0,$H3,$S3
|
||||
vaddudm $ACC1,$ACC1,$T0
|
||||
vmulouw $T0,$H4,$S3
|
||||
vaddudm $ACC2,$ACC2,$T0
|
||||
vmulouw $T0,$H0,$R3
|
||||
vaddudm $ACC3,$ACC3,$T0
|
||||
vmulouw $T0,$H1,$R3
|
||||
vaddudm $ACC4,$ACC4,$T0
|
||||
|
||||
vmulouw $T0,$H1,$S4
|
||||
vaddudm $ACC0,$ACC0,$T0
|
||||
vmulouw $T0,$H2,$S4
|
||||
vaddudm $ACC1,$ACC1,$T0
|
||||
vmulouw $T0,$H3,$S4
|
||||
vaddudm $ACC2,$ACC2,$T0
|
||||
vmulouw $T0,$H4,$S4
|
||||
vaddudm $ACC3,$ACC3,$T0
|
||||
vmulouw $T0,$H0,$R4
|
||||
vaddudm $ACC4,$ACC4,$T0
|
||||
|
||||
################################################################
|
||||
# lazy reduction
|
||||
|
||||
vspltisb $T0,2
|
||||
vsrd $H4,$ACC3,$_26
|
||||
vsrd $H1,$ACC0,$_26
|
||||
vand $H3,$ACC3,$mask26
|
||||
vand $H0,$ACC0,$mask26
|
||||
vaddudm $H4,$H4,$ACC4 # h3 -> h4
|
||||
vaddudm $H1,$H1,$ACC1 # h0 -> h1
|
||||
|
||||
vsrd $ACC4,$H4,$_26
|
||||
vsrd $ACC1,$H1,$_26
|
||||
vand $H4,$H4,$mask26
|
||||
vand $H1,$H1,$mask26
|
||||
vaddudm $H0,$H0,$ACC4
|
||||
vaddudm $H2,$ACC2,$ACC1 # h1 -> h2
|
||||
|
||||
vsld $ACC4,$ACC4,$T0 # <<2
|
||||
vsrd $ACC2,$H2,$_26
|
||||
vand $H2,$H2,$mask26
|
||||
vaddudm $H0,$H0,$ACC4 # h4 -> h0
|
||||
vaddudm $H3,$H3,$ACC2 # h2 -> h3
|
||||
|
||||
vsrd $ACC0,$H0,$_26
|
||||
vsrd $ACC3,$H3,$_26
|
||||
vand $H0,$H0,$mask26
|
||||
vand $H3,$H3,$mask26
|
||||
vaddudm $H1,$H1,$ACC0 # h0 -> h1
|
||||
vaddudm $H4,$H4,$ACC3 # h3 -> h4
|
||||
|
||||
addc $h0,$h0,$d0 # accumulate nonce
|
||||
adde $h1,$h1,$d1
|
||||
adde $h2,$h2,$d2
|
||||
adde $h3,$h3,$d3
|
||||
___
|
||||
$code.=<<___ if ($LITTLE_ENDIAN);
|
||||
stw $h0,0($mac) # write result
|
||||
stw $h1,4($mac)
|
||||
stw $h2,8($mac)
|
||||
stw $h3,12($mac)
|
||||
___
|
||||
$code.=<<___ if (!$LITTLE_ENDIAN);
|
||||
li $d1,4
|
||||
stwbrx $h0,0,$mac # write result
|
||||
li $d2,8
|
||||
stwbrx $h1,$d1,$mac
|
||||
li $d3,12
|
||||
stwbrx $h2,$d2,$mac
|
||||
stwbrx $h3,$d3,$mac
|
||||
___
|
||||
$code.=<<___;
|
||||
$POP r28,`$FRAME-$SIZE_T*4`($sp)
|
||||
$POP r29,`$FRAME-$SIZE_T*3`($sp)
|
||||
$POP r30,`$FRAME-$SIZE_T*2`($sp)
|
||||
$POP r31,`$FRAME-$SIZE_T*1`($sp)
|
||||
addi $sp,$sp,$FRAME
|
||||
blr
|
||||
.long 0
|
||||
.byte 0,12,4,1,0x80,4,3,0
|
||||
.size .poly1305_emit,.-.poly1305_emit
|
||||
.byte 0,12,0x14,0,0,0,0,0
|
||||
.size __poly1305_mul,.-__poly1305_mul
|
||||
|
||||
.align 5
|
||||
__poly1305_blocks_vsx:
|
||||
$STU $sp,-$VSXFRAME($sp)
|
||||
mflr r0
|
||||
li r10,`15+$LOCALS+128`
|
||||
li r11,`31+$LOCALS+128`
|
||||
mfspr r12,256
|
||||
stvx v20,r10,$sp
|
||||
addi r10,r10,32
|
||||
stvx v21,r11,$sp
|
||||
addi r11,r11,32
|
||||
stvx v22,r10,$sp
|
||||
addi r10,r10,32
|
||||
stvx v23,r10,$sp
|
||||
addi r10,r10,32
|
||||
stvx v24,r11,$sp
|
||||
addi r11,r11,32
|
||||
stvx v25,r10,$sp
|
||||
addi r10,r10,32
|
||||
stvx v26,r10,$sp
|
||||
addi r10,r10,32
|
||||
stvx v27,r11,$sp
|
||||
addi r11,r11,32
|
||||
stvx v28,r10,$sp
|
||||
addi r10,r10,32
|
||||
stvx v29,r11,$sp
|
||||
addi r11,r11,32
|
||||
stvx v30,r10,$sp
|
||||
stvx v31,r11,$sp
|
||||
stw r12,`$VSXFRAME-$SIZE_T*5-4`($sp)# save vrsave
|
||||
li r12,-1
|
||||
mtspr 256,r12 # preserve all AltiVec registers
|
||||
$PUSH r27,`$VSXFRAME-$SIZE_T*5`($sp)
|
||||
$PUSH r28,`$VSXFRAME-$SIZE_T*4`($sp)
|
||||
$PUSH r29,`$VSXFRAME-$SIZE_T*3`($sp)
|
||||
$PUSH r30,`$VSXFRAME-$SIZE_T*2`($sp)
|
||||
$PUSH r31,`$VSXFRAME-$SIZE_T*1`($sp)
|
||||
$PUSH r0,`$VSXFRAME+$LRSAVE`($sp)
|
||||
|
||||
bl LPICmeup
|
||||
|
||||
li $x10,0x10
|
||||
li $x20,0x20
|
||||
li $x30,0x30
|
||||
li $x40,0x40
|
||||
li $x50,0x50
|
||||
lvx_u $mask26,$x00,$const
|
||||
lvx_u $_26,$x10,$const
|
||||
lvx_u $_40,$x20,$const
|
||||
lvx_u $I2perm,$x30,$const
|
||||
lvx_u $padbits,$x40,$const
|
||||
|
||||
cmplwi r7,0 # is_base2_26?
|
||||
bne Lskip_init_vsx
|
||||
|
||||
lwz $h1,32($ctx) # load key base 2^32
|
||||
lwz $h2,36($ctx)
|
||||
lwz $h3,40($ctx)
|
||||
lwz $h4,44($ctx)
|
||||
|
||||
extrwi $h0,$h1,26,6 # base 2^32 -> base 2^26
|
||||
extrwi $h1,$h1,6,0
|
||||
insrwi $h1,$h2,20,6
|
||||
extrwi $h2,$h2,12,0
|
||||
insrwi $h2,$h3,14,6
|
||||
extrwi $h3,$h3,18,0
|
||||
insrwi $h3,$h4,8,6
|
||||
extrwi $h4,$h4,24,0
|
||||
|
||||
mtvrwz $R0,$h0
|
||||
slwi $h0,$h1,2
|
||||
mtvrwz $R1,$h1
|
||||
add $h1,$h1,$h0
|
||||
mtvrwz $S1,$h1
|
||||
slwi $h1,$h2,2
|
||||
mtvrwz $R2,$h2
|
||||
add $h2,$h2,$h1
|
||||
mtvrwz $S2,$h2
|
||||
slwi $h2,$h3,2
|
||||
mtvrwz $R3,$h3
|
||||
add $h3,$h3,$h2
|
||||
mtvrwz $S3,$h3
|
||||
slwi $h3,$h4,2
|
||||
mtvrwz $R4,$h4
|
||||
add $h4,$h4,$h3
|
||||
mtvrwz $S4,$h4
|
||||
|
||||
vmr $H0,$R0
|
||||
vmr $H1,$R1
|
||||
vmr $H2,$R2
|
||||
vmr $H3,$R3
|
||||
vmr $H4,$R4
|
||||
|
||||
bl __poly1305_mul # r^1:- * r^1:-
|
||||
|
||||
vpermdi $R0,$H0,$R0,0b00
|
||||
vpermdi $R1,$H1,$R1,0b00
|
||||
vpermdi $R2,$H2,$R2,0b00
|
||||
vpermdi $R3,$H3,$R3,0b00
|
||||
vpermdi $R4,$H4,$R4,0b00
|
||||
vpermdi $H0,$H0,$H0,0b00
|
||||
vpermdi $H1,$H1,$H1,0b00
|
||||
vpermdi $H2,$H2,$H2,0b00
|
||||
vpermdi $H3,$H3,$H3,0b00
|
||||
vpermdi $H4,$H4,$H4,0b00
|
||||
vsld $S1,$R1,$T0 # <<2
|
||||
vsld $S2,$R2,$T0
|
||||
vsld $S3,$R3,$T0
|
||||
vsld $S4,$R4,$T0
|
||||
vaddudm $S1,$S1,$R1
|
||||
vaddudm $S2,$S2,$R2
|
||||
vaddudm $S3,$S3,$R3
|
||||
vaddudm $S4,$S4,$R4
|
||||
|
||||
bl __poly1305_mul # r^2:r^2 * r^2:r^1
|
||||
|
||||
addi $h0,$ctx,0x60
|
||||
lwz $h1,0($ctx) # load hash
|
||||
lwz $h2,4($ctx)
|
||||
lwz $h3,8($ctx)
|
||||
lwz $h4,12($ctx)
|
||||
lwz $t0,16($ctx)
|
||||
|
||||
vmrgow $R0,$R0,$H0 # r^2:r^4:r^1:r^3
|
||||
vmrgow $R1,$R1,$H1
|
||||
vmrgow $R2,$R2,$H2
|
||||
vmrgow $R3,$R3,$H3
|
||||
vmrgow $R4,$R4,$H4
|
||||
vslw $S1,$R1,$T0 # <<2
|
||||
vslw $S2,$R2,$T0
|
||||
vslw $S3,$R3,$T0
|
||||
vslw $S4,$R4,$T0
|
||||
vadduwm $S1,$S1,$R1
|
||||
vadduwm $S2,$S2,$R2
|
||||
vadduwm $S3,$S3,$R3
|
||||
vadduwm $S4,$S4,$R4
|
||||
|
||||
stvx_u $R0,$x30,$ctx
|
||||
stvx_u $R1,$x40,$ctx
|
||||
stvx_u $S1,$x50,$ctx
|
||||
stvx_u $R2,$x00,$h0
|
||||
stvx_u $S2,$x10,$h0
|
||||
stvx_u $R3,$x20,$h0
|
||||
stvx_u $S3,$x30,$h0
|
||||
stvx_u $R4,$x40,$h0
|
||||
stvx_u $S4,$x50,$h0
|
||||
|
||||
extrwi $h0,$h1,26,6 # base 2^32 -> base 2^26
|
||||
extrwi $h1,$h1,6,0
|
||||
mtvrwz $H0,$h0
|
||||
insrwi $h1,$h2,20,6
|
||||
extrwi $h2,$h2,12,0
|
||||
mtvrwz $H1,$h1
|
||||
insrwi $h2,$h3,14,6
|
||||
extrwi $h3,$h3,18,0
|
||||
mtvrwz $H2,$h2
|
||||
insrwi $h3,$h4,8,6
|
||||
extrwi $h4,$h4,24,0
|
||||
mtvrwz $H3,$h3
|
||||
insrwi $h4,$t0,3,5
|
||||
mtvrwz $H4,$h4
|
||||
___
|
||||
}
|
||||
$code.=<<___;
|
||||
.asciz "Poly1305 for PPC, CRYPTOGAMS by <appro\@openssl.org>"
|
||||
li r0,1
|
||||
stw r0,24($ctx) # set is_base2_26
|
||||
b Loaded_vsx
|
||||
|
||||
.align 4
|
||||
Lskip_init_vsx:
|
||||
li $x10,4
|
||||
li $x20,8
|
||||
li $x30,12
|
||||
li $x40,16
|
||||
lvwzx_u $H0,$x00,$ctx
|
||||
lvwzx_u $H1,$x10,$ctx
|
||||
lvwzx_u $H2,$x20,$ctx
|
||||
lvwzx_u $H3,$x30,$ctx
|
||||
lvwzx_u $H4,$x40,$ctx
|
||||
|
||||
Loaded_vsx:
|
||||
li $x10,0x10
|
||||
li $x20,0x20
|
||||
li $x30,0x30
|
||||
li $x40,0x40
|
||||
li $x50,0x50
|
||||
li $x60,0x60
|
||||
li $x70,0x70
|
||||
addi $ctx_,$ctx,64 # &ctx->r[1]
|
||||
addi $_ctx,$sp,`$LOCALS+15` # &ctx->r[1], r^2:r^4 shadow
|
||||
|
||||
vxor $T0,$T0,$T0 # ensure second half is zero
|
||||
vpermdi $H0,$H0,$T0,0b00
|
||||
vpermdi $H1,$H1,$T0,0b00
|
||||
vpermdi $H2,$H2,$T0,0b00
|
||||
vpermdi $H3,$H3,$T0,0b00
|
||||
vpermdi $H4,$H4,$T0,0b00
|
||||
|
||||
be?lvx_u $_4,$x50,$const # byte swap mask
|
||||
lvx_u $T1,$x00,$inp # load first input block
|
||||
lvx_u $T2,$x10,$inp
|
||||
lvx_u $T3,$x20,$inp
|
||||
lvx_u $T4,$x30,$inp
|
||||
be?vperm $T1,$T1,$T1,$_4
|
||||
be?vperm $T2,$T2,$T2,$_4
|
||||
be?vperm $T3,$T3,$T3,$_4
|
||||
be?vperm $T4,$T4,$T4,$_4
|
||||
|
||||
vpermdi $I0,$T1,$T2,0b00 # smash input to base 2^26
|
||||
vspltisb $_4,4
|
||||
vperm $I2,$T1,$T2,$I2perm # 0x...0e0f0001...1e1f1011
|
||||
vspltisb $_14,14
|
||||
vpermdi $I3,$T1,$T2,0b11
|
||||
|
||||
vsrd $I1,$I0,$_26
|
||||
vsrd $I2,$I2,$_4
|
||||
vsrd $I4,$I3,$_40
|
||||
vsrd $I3,$I3,$_14
|
||||
vand $I0,$I0,$mask26
|
||||
vand $I1,$I1,$mask26
|
||||
vand $I2,$I2,$mask26
|
||||
vand $I3,$I3,$mask26
|
||||
|
||||
vpermdi $T1,$T3,$T4,0b00
|
||||
vperm $T2,$T3,$T4,$I2perm # 0x...0e0f0001...1e1f1011
|
||||
vpermdi $T3,$T3,$T4,0b11
|
||||
|
||||
vsrd $T0,$T1,$_26
|
||||
vsrd $T2,$T2,$_4
|
||||
vsrd $T4,$T3,$_40
|
||||
vsrd $T3,$T3,$_14
|
||||
vand $T1,$T1,$mask26
|
||||
vand $T0,$T0,$mask26
|
||||
vand $T2,$T2,$mask26
|
||||
vand $T3,$T3,$mask26
|
||||
|
||||
# inp[2]:inp[0]:inp[3]:inp[1]
|
||||
vmrgow $I4,$T4,$I4
|
||||
vmrgow $I0,$T1,$I0
|
||||
vmrgow $I1,$T0,$I1
|
||||
vmrgow $I2,$T2,$I2
|
||||
vmrgow $I3,$T3,$I3
|
||||
vor $I4,$I4,$padbits
|
||||
|
||||
lvx_splt $R0,$x30,$ctx # taking lvx_vsplt out of loop
|
||||
lvx_splt $R1,$x00,$ctx_ # gives ~8% improvement
|
||||
lvx_splt $S1,$x10,$ctx_
|
||||
lvx_splt $R2,$x20,$ctx_
|
||||
lvx_splt $S2,$x30,$ctx_
|
||||
lvx_splt $T1,$x40,$ctx_
|
||||
lvx_splt $T2,$x50,$ctx_
|
||||
lvx_splt $T3,$x60,$ctx_
|
||||
lvx_splt $T4,$x70,$ctx_
|
||||
stvx $R1,$x00,$_ctx
|
||||
stvx $S1,$x10,$_ctx
|
||||
stvx $R2,$x20,$_ctx
|
||||
stvx $S2,$x30,$_ctx
|
||||
stvx $T1,$x40,$_ctx
|
||||
stvx $T2,$x50,$_ctx
|
||||
stvx $T3,$x60,$_ctx
|
||||
stvx $T4,$x70,$_ctx
|
||||
|
||||
addi $inp,$inp,0x40
|
||||
addi $const,$const,0x50
|
||||
addi r0,$len,-64
|
||||
srdi r0,r0,6
|
||||
mtctr r0
|
||||
b Loop_vsx
|
||||
|
||||
.align 4
|
||||
Loop_vsx:
|
||||
################################################################
|
||||
## ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
|
||||
## ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
|
||||
## \___________________/
|
||||
##
|
||||
## Note that we start with inp[2:3]*r^2. This is because it
|
||||
## doesn't depend on reduction in previous iteration.
|
||||
################################################################
|
||||
## d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
|
||||
## d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
|
||||
## d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
|
||||
## d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
|
||||
## d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
|
||||
|
||||
vmuleuw $ACC0,$I0,$R0
|
||||
vmuleuw $ACC1,$I0,$R1
|
||||
vmuleuw $ACC2,$I0,$R2
|
||||
vmuleuw $ACC3,$I1,$R2
|
||||
|
||||
vmuleuw $T0,$I1,$R0
|
||||
vaddudm $ACC1,$ACC1,$T0
|
||||
vmuleuw $T0,$I1,$R1
|
||||
vaddudm $ACC2,$ACC2,$T0
|
||||
vmuleuw $ACC4,$I2,$R2
|
||||
vmuleuw $T0,$I4,$S1
|
||||
vaddudm $ACC0,$ACC0,$T0
|
||||
vmuleuw $T0,$I2,$R1
|
||||
vaddudm $ACC3,$ACC3,$T0
|
||||
lvx $S3,$x50,$_ctx
|
||||
vmuleuw $T0,$I3,$R1
|
||||
vaddudm $ACC4,$ACC4,$T0
|
||||
lvx $R3,$x40,$_ctx
|
||||
|
||||
vaddudm $H2,$H2,$I2
|
||||
vaddudm $H0,$H0,$I0
|
||||
vaddudm $H3,$H3,$I3
|
||||
vaddudm $H1,$H1,$I1
|
||||
vaddudm $H4,$H4,$I4
|
||||
|
||||
vmuleuw $T0,$I3,$S2
|
||||
vaddudm $ACC0,$ACC0,$T0
|
||||
vmuleuw $T0,$I4,$S2
|
||||
vaddudm $ACC1,$ACC1,$T0
|
||||
vmuleuw $T0,$I2,$R0
|
||||
vaddudm $ACC2,$ACC2,$T0
|
||||
vmuleuw $T0,$I3,$R0
|
||||
vaddudm $ACC3,$ACC3,$T0
|
||||
lvx $S4,$x70,$_ctx
|
||||
vmuleuw $T0,$I4,$R0
|
||||
vaddudm $ACC4,$ACC4,$T0
|
||||
lvx $R4,$x60,$_ctx
|
||||
|
||||
vmuleuw $T0,$I2,$S3
|
||||
vaddudm $ACC0,$ACC0,$T0
|
||||
vmuleuw $T0,$I3,$S3
|
||||
vaddudm $ACC1,$ACC1,$T0
|
||||
vmuleuw $T0,$I4,$S3
|
||||
vaddudm $ACC2,$ACC2,$T0
|
||||
vmuleuw $T0,$I0,$R3
|
||||
vaddudm $ACC3,$ACC3,$T0
|
||||
vmuleuw $T0,$I1,$R3
|
||||
vaddudm $ACC4,$ACC4,$T0
|
||||
|
||||
be?lvx_u $_4,$x00,$const # byte swap mask
|
||||
lvx_u $T1,$x00,$inp # load next input block
|
||||
lvx_u $T2,$x10,$inp
|
||||
lvx_u $T3,$x20,$inp
|
||||
lvx_u $T4,$x30,$inp
|
||||
be?vperm $T1,$T1,$T1,$_4
|
||||
be?vperm $T2,$T2,$T2,$_4
|
||||
be?vperm $T3,$T3,$T3,$_4
|
||||
be?vperm $T4,$T4,$T4,$_4
|
||||
|
||||
vmuleuw $T0,$I1,$S4
|
||||
vaddudm $ACC0,$ACC0,$T0
|
||||
vmuleuw $T0,$I2,$S4
|
||||
vaddudm $ACC1,$ACC1,$T0
|
||||
vmuleuw $T0,$I3,$S4
|
||||
vaddudm $ACC2,$ACC2,$T0
|
||||
vmuleuw $T0,$I4,$S4
|
||||
vaddudm $ACC3,$ACC3,$T0
|
||||
vmuleuw $T0,$I0,$R4
|
||||
vaddudm $ACC4,$ACC4,$T0
|
||||
|
||||
vpermdi $I0,$T1,$T2,0b00 # smash input to base 2^26
|
||||
vspltisb $_4,4
|
||||
vperm $I2,$T1,$T2,$I2perm # 0x...0e0f0001...1e1f1011
|
||||
vpermdi $I3,$T1,$T2,0b11
|
||||
|
||||
# (hash + inp[0:1]) * r^4
|
||||
vmulouw $T0,$H0,$R0
|
||||
vaddudm $ACC0,$ACC0,$T0
|
||||
vmulouw $T0,$H1,$R0
|
||||
vaddudm $ACC1,$ACC1,$T0
|
||||
vmulouw $T0,$H2,$R0
|
||||
vaddudm $ACC2,$ACC2,$T0
|
||||
vmulouw $T0,$H3,$R0
|
||||
vaddudm $ACC3,$ACC3,$T0
|
||||
vmulouw $T0,$H4,$R0
|
||||
vaddudm $ACC4,$ACC4,$T0
|
||||
|
||||
vpermdi $T1,$T3,$T4,0b00
|
||||
vperm $T2,$T3,$T4,$I2perm # 0x...0e0f0001...1e1f1011
|
||||
vpermdi $T3,$T3,$T4,0b11
|
||||
|
||||
vmulouw $T0,$H2,$S3
|
||||
vaddudm $ACC0,$ACC0,$T0
|
||||
vmulouw $T0,$H3,$S3
|
||||
vaddudm $ACC1,$ACC1,$T0
|
||||
vmulouw $T0,$H4,$S3
|
||||
vaddudm $ACC2,$ACC2,$T0
|
||||
vmulouw $T0,$H0,$R3
|
||||
vaddudm $ACC3,$ACC3,$T0
|
||||
lvx $S1,$x10,$_ctx
|
||||
vmulouw $T0,$H1,$R3
|
||||
vaddudm $ACC4,$ACC4,$T0
|
||||
lvx $R1,$x00,$_ctx
|
||||
|
||||
vsrd $I1,$I0,$_26
|
||||
vsrd $I2,$I2,$_4
|
||||
vsrd $I4,$I3,$_40
|
||||
vsrd $I3,$I3,$_14
|
||||
|
||||
vmulouw $T0,$H1,$S4
|
||||
vaddudm $ACC0,$ACC0,$T0
|
||||
vmulouw $T0,$H2,$S4
|
||||
vaddudm $ACC1,$ACC1,$T0
|
||||
vmulouw $T0,$H3,$S4
|
||||
vaddudm $ACC2,$ACC2,$T0
|
||||
vmulouw $T0,$H4,$S4
|
||||
vaddudm $ACC3,$ACC3,$T0
|
||||
lvx $S2,$x30,$_ctx
|
||||
vmulouw $T0,$H0,$R4
|
||||
vaddudm $ACC4,$ACC4,$T0
|
||||
lvx $R2,$x20,$_ctx
|
||||
|
||||
vand $I0,$I0,$mask26
|
||||
vand $I1,$I1,$mask26
|
||||
vand $I2,$I2,$mask26
|
||||
vand $I3,$I3,$mask26
|
||||
|
||||
vmulouw $T0,$H4,$S1
|
||||
vaddudm $ACC0,$ACC0,$T0
|
||||
vmulouw $T0,$H0,$R1
|
||||
vaddudm $ACC1,$ACC1,$T0
|
||||
vmulouw $T0,$H1,$R1
|
||||
vaddudm $ACC2,$ACC2,$T0
|
||||
vmulouw $T0,$H2,$R1
|
||||
vaddudm $ACC3,$ACC3,$T0
|
||||
vmulouw $T0,$H3,$R1
|
||||
vaddudm $ACC4,$ACC4,$T0
|
||||
|
||||
vsrd $T2,$T2,$_4
|
||||
vsrd $_4,$T1,$_26
|
||||
vsrd $T4,$T3,$_40
|
||||
vsrd $T3,$T3,$_14
|
||||
|
||||
vmulouw $T0,$H3,$S2
|
||||
vaddudm $ACC0,$ACC0,$T0
|
||||
vmulouw $T0,$H4,$S2
|
||||
vaddudm $ACC1,$ACC1,$T0
|
||||
vmulouw $T0,$H0,$R2
|
||||
vaddudm $ACC2,$ACC2,$T0
|
||||
vmulouw $T0,$H1,$R2
|
||||
vaddudm $ACC3,$ACC3,$T0
|
||||
vmulouw $T0,$H2,$R2
|
||||
vaddudm $ACC4,$ACC4,$T0
|
||||
|
||||
vand $T1,$T1,$mask26
|
||||
vand $_4,$_4,$mask26
|
||||
vand $T2,$T2,$mask26
|
||||
vand $T3,$T3,$mask26
|
||||
|
||||
################################################################
|
||||
# lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
|
||||
# and P. Schwabe
|
||||
|
||||
vspltisb $T0,2
|
||||
vsrd $H4,$ACC3,$_26
|
||||
vsrd $H1,$ACC0,$_26
|
||||
vand $H3,$ACC3,$mask26
|
||||
vand $H0,$ACC0,$mask26
|
||||
vaddudm $H4,$H4,$ACC4 # h3 -> h4
|
||||
vaddudm $H1,$H1,$ACC1 # h0 -> h1
|
||||
|
||||
vmrgow $I4,$T4,$I4
|
||||
vmrgow $I0,$T1,$I0
|
||||
vmrgow $I1,$_4,$I1
|
||||
vmrgow $I2,$T2,$I2
|
||||
vmrgow $I3,$T3,$I3
|
||||
vor $I4,$I4,$padbits
|
||||
|
||||
vsrd $ACC4,$H4,$_26
|
||||
vsrd $ACC1,$H1,$_26
|
||||
vand $H4,$H4,$mask26
|
||||
vand $H1,$H1,$mask26
|
||||
vaddudm $H0,$H0,$ACC4
|
||||
vaddudm $H2,$ACC2,$ACC1 # h1 -> h2
|
||||
|
||||
vsld $ACC4,$ACC4,$T0 # <<2
|
||||
vsrd $ACC2,$H2,$_26
|
||||
vand $H2,$H2,$mask26
|
||||
vaddudm $H0,$H0,$ACC4 # h4 -> h0
|
||||
vaddudm $H3,$H3,$ACC2 # h2 -> h3
|
||||
|
||||
vsrd $ACC0,$H0,$_26
|
||||
vsrd $ACC3,$H3,$_26
|
||||
vand $H0,$H0,$mask26
|
||||
vand $H3,$H3,$mask26
|
||||
vaddudm $H1,$H1,$ACC0 # h0 -> h1
|
||||
vaddudm $H4,$H4,$ACC3 # h3 -> h4
|
||||
|
||||
addi $inp,$inp,0x40
|
||||
bdnz Loop_vsx
|
||||
|
||||
neg $len,$len
|
||||
andi. $len,$len,0x30
|
||||
sub $inp,$inp,$len
|
||||
|
||||
lvx_u $R0,$x30,$ctx # load all powers
|
||||
lvx_u $R1,$x00,$ctx_
|
||||
lvx_u $S1,$x10,$ctx_
|
||||
lvx_u $R2,$x20,$ctx_
|
||||
lvx_u $S2,$x30,$ctx_
|
||||
|
||||
Last_vsx:
|
||||
vmuleuw $ACC0,$I0,$R0
|
||||
vmuleuw $ACC1,$I1,$R0
|
||||
vmuleuw $ACC2,$I2,$R0
|
||||
vmuleuw $ACC3,$I3,$R0
|
||||
vmuleuw $ACC4,$I4,$R0
|
||||
|
||||
vmuleuw $T0,$I4,$S1
|
||||
vaddudm $ACC0,$ACC0,$T0
|
||||
vmuleuw $T0,$I0,$R1
|
||||
vaddudm $ACC1,$ACC1,$T0
|
||||
vmuleuw $T0,$I1,$R1
|
||||
vaddudm $ACC2,$ACC2,$T0
|
||||
vmuleuw $T0,$I2,$R1
|
||||
vaddudm $ACC3,$ACC3,$T0
|
||||
lvx_u $S3,$x50,$ctx_
|
||||
vmuleuw $T0,$I3,$R1
|
||||
vaddudm $ACC4,$ACC4,$T0
|
||||
lvx_u $R3,$x40,$ctx_
|
||||
|
||||
vaddudm $H2,$H2,$I2
|
||||
vaddudm $H0,$H0,$I0
|
||||
vaddudm $H3,$H3,$I3
|
||||
vaddudm $H1,$H1,$I1
|
||||
vaddudm $H4,$H4,$I4
|
||||
|
||||
vmuleuw $T0,$I3,$S2
|
||||
vaddudm $ACC0,$ACC0,$T0
|
||||
vmuleuw $T0,$I4,$S2
|
||||
vaddudm $ACC1,$ACC1,$T0
|
||||
vmuleuw $T0,$I0,$R2
|
||||
vaddudm $ACC2,$ACC2,$T0
|
||||
vmuleuw $T0,$I1,$R2
|
||||
vaddudm $ACC3,$ACC3,$T0
|
||||
lvx_u $S4,$x70,$ctx_
|
||||
vmuleuw $T0,$I2,$R2
|
||||
vaddudm $ACC4,$ACC4,$T0
|
||||
lvx_u $R4,$x60,$ctx_
|
||||
|
||||
vmuleuw $T0,$I2,$S3
|
||||
vaddudm $ACC0,$ACC0,$T0
|
||||
vmuleuw $T0,$I3,$S3
|
||||
vaddudm $ACC1,$ACC1,$T0
|
||||
vmuleuw $T0,$I4,$S3
|
||||
vaddudm $ACC2,$ACC2,$T0
|
||||
vmuleuw $T0,$I0,$R3
|
||||
vaddudm $ACC3,$ACC3,$T0
|
||||
vmuleuw $T0,$I1,$R3
|
||||
vaddudm $ACC4,$ACC4,$T0
|
||||
|
||||
vmuleuw $T0,$I1,$S4
|
||||
vaddudm $ACC0,$ACC0,$T0
|
||||
vmuleuw $T0,$I2,$S4
|
||||
vaddudm $ACC1,$ACC1,$T0
|
||||
vmuleuw $T0,$I3,$S4
|
||||
vaddudm $ACC2,$ACC2,$T0
|
||||
vmuleuw $T0,$I4,$S4
|
||||
vaddudm $ACC3,$ACC3,$T0
|
||||
vmuleuw $T0,$I0,$R4
|
||||
vaddudm $ACC4,$ACC4,$T0
|
||||
|
||||
# (hash + inp[0:1]) * r^4
|
||||
vmulouw $T0,$H0,$R0
|
||||
vaddudm $ACC0,$ACC0,$T0
|
||||
vmulouw $T0,$H1,$R0
|
||||
vaddudm $ACC1,$ACC1,$T0
|
||||
vmulouw $T0,$H2,$R0
|
||||
vaddudm $ACC2,$ACC2,$T0
|
||||
vmulouw $T0,$H3,$R0
|
||||
vaddudm $ACC3,$ACC3,$T0
|
||||
vmulouw $T0,$H4,$R0
|
||||
vaddudm $ACC4,$ACC4,$T0
|
||||
|
||||
vmulouw $T0,$H2,$S3
|
||||
vaddudm $ACC0,$ACC0,$T0
|
||||
vmulouw $T0,$H3,$S3
|
||||
vaddudm $ACC1,$ACC1,$T0
|
||||
vmulouw $T0,$H4,$S3
|
||||
vaddudm $ACC2,$ACC2,$T0
|
||||
vmulouw $T0,$H0,$R3
|
||||
vaddudm $ACC3,$ACC3,$T0
|
||||
lvx_u $S1,$x10,$ctx_
|
||||
vmulouw $T0,$H1,$R3
|
||||
vaddudm $ACC4,$ACC4,$T0
|
||||
lvx_u $R1,$x00,$ctx_
|
||||
|
||||
vmulouw $T0,$H1,$S4
|
||||
vaddudm $ACC0,$ACC0,$T0
|
||||
vmulouw $T0,$H2,$S4
|
||||
vaddudm $ACC1,$ACC1,$T0
|
||||
vmulouw $T0,$H3,$S4
|
||||
vaddudm $ACC2,$ACC2,$T0
|
||||
vmulouw $T0,$H4,$S4
|
||||
vaddudm $ACC3,$ACC3,$T0
|
||||
lvx_u $S2,$x30,$ctx_
|
||||
vmulouw $T0,$H0,$R4
|
||||
vaddudm $ACC4,$ACC4,$T0
|
||||
lvx_u $R2,$x20,$ctx_
|
||||
|
||||
vmulouw $T0,$H4,$S1
|
||||
vaddudm $ACC0,$ACC0,$T0
|
||||
vmulouw $T0,$H0,$R1
|
||||
vaddudm $ACC1,$ACC1,$T0
|
||||
vmulouw $T0,$H1,$R1
|
||||
vaddudm $ACC2,$ACC2,$T0
|
||||
vmulouw $T0,$H2,$R1
|
||||
vaddudm $ACC3,$ACC3,$T0
|
||||
vmulouw $T0,$H3,$R1
|
||||
vaddudm $ACC4,$ACC4,$T0
|
||||
|
||||
vmulouw $T0,$H3,$S2
|
||||
vaddudm $ACC0,$ACC0,$T0
|
||||
vmulouw $T0,$H4,$S2
|
||||
vaddudm $ACC1,$ACC1,$T0
|
||||
vmulouw $T0,$H0,$R2
|
||||
vaddudm $ACC2,$ACC2,$T0
|
||||
vmulouw $T0,$H1,$R2
|
||||
vaddudm $ACC3,$ACC3,$T0
|
||||
vmulouw $T0,$H2,$R2
|
||||
vaddudm $ACC4,$ACC4,$T0
|
||||
|
||||
################################################################
|
||||
# horizontal addition
|
||||
|
||||
vpermdi $H0,$ACC0,$ACC0,0b10
|
||||
vpermdi $H1,$ACC1,$ACC1,0b10
|
||||
vpermdi $H2,$ACC2,$ACC2,0b10
|
||||
vpermdi $H3,$ACC3,$ACC3,0b10
|
||||
vpermdi $H4,$ACC4,$ACC4,0b10
|
||||
vaddudm $ACC0,$ACC0,$H0
|
||||
vaddudm $ACC1,$ACC1,$H1
|
||||
vaddudm $ACC2,$ACC2,$H2
|
||||
vaddudm $ACC3,$ACC3,$H3
|
||||
vaddudm $ACC4,$ACC4,$H4
|
||||
|
||||
################################################################
|
||||
# lazy reduction
|
||||
|
||||
vspltisb $T0,2
|
||||
vsrd $H4,$ACC3,$_26
|
||||
vsrd $H1,$ACC0,$_26
|
||||
vand $H3,$ACC3,$mask26
|
||||
vand $H0,$ACC0,$mask26
|
||||
vaddudm $H4,$H4,$ACC4 # h3 -> h4
|
||||
vaddudm $H1,$H1,$ACC1 # h0 -> h1
|
||||
|
||||
vsrd $ACC4,$H4,$_26
|
||||
vsrd $ACC1,$H1,$_26
|
||||
vand $H4,$H4,$mask26
|
||||
vand $H1,$H1,$mask26
|
||||
vaddudm $H0,$H0,$ACC4
|
||||
vaddudm $H2,$ACC2,$ACC1 # h1 -> h2
|
||||
|
||||
vsld $ACC4,$ACC4,$T0 # <<2
|
||||
vsrd $ACC2,$H2,$_26
|
||||
vand $H2,$H2,$mask26
|
||||
vaddudm $H0,$H0,$ACC4 # h4 -> h0
|
||||
vaddudm $H3,$H3,$ACC2 # h2 -> h3
|
||||
|
||||
vsrd $ACC0,$H0,$_26
|
||||
vsrd $ACC3,$H3,$_26
|
||||
vand $H0,$H0,$mask26
|
||||
vand $H3,$H3,$mask26
|
||||
vaddudm $H1,$H1,$ACC0 # h0 -> h1
|
||||
vaddudm $H4,$H4,$ACC3 # h3 -> h4
|
||||
|
||||
beq Ldone_vsx
|
||||
|
||||
add r6,$const,$len
|
||||
|
||||
be?lvx_u $_4,$x00,$const # byte swap mask
|
||||
lvx_u $T1,$x00,$inp # load last partial input block
|
||||
lvx_u $T2,$x10,$inp
|
||||
lvx_u $T3,$x20,$inp
|
||||
lvx_u $T4,$x30,$inp
|
||||
be?vperm $T1,$T1,$T1,$_4
|
||||
be?vperm $T2,$T2,$T2,$_4
|
||||
be?vperm $T3,$T3,$T3,$_4
|
||||
be?vperm $T4,$T4,$T4,$_4
|
||||
|
||||
vpermdi $I0,$T1,$T2,0b00 # smash input to base 2^26
|
||||
vspltisb $_4,4
|
||||
vperm $I2,$T1,$T2,$I2perm # 0x...0e0f0001...1e1f1011
|
||||
vpermdi $I3,$T1,$T2,0b11
|
||||
|
||||
vsrd $I1,$I0,$_26
|
||||
vsrd $I2,$I2,$_4
|
||||
vsrd $I4,$I3,$_40
|
||||
vsrd $I3,$I3,$_14
|
||||
vand $I0,$I0,$mask26
|
||||
vand $I1,$I1,$mask26
|
||||
vand $I2,$I2,$mask26
|
||||
vand $I3,$I3,$mask26
|
||||
|
||||
vpermdi $T0,$T3,$T4,0b00
|
||||
vperm $T1,$T3,$T4,$I2perm # 0x...0e0f0001...1e1f1011
|
||||
vpermdi $T2,$T3,$T4,0b11
|
||||
|
||||
lvx_u $ACC0,$x00,r6
|
||||
lvx_u $ACC1,$x30,r6
|
||||
|
||||
vsrd $T3,$T0,$_26
|
||||
vsrd $T1,$T1,$_4
|
||||
vsrd $T4,$T2,$_40
|
||||
vsrd $T2,$T2,$_14
|
||||
vand $T0,$T0,$mask26
|
||||
vand $T3,$T3,$mask26
|
||||
vand $T1,$T1,$mask26
|
||||
vand $T2,$T2,$mask26
|
||||
|
||||
# inp[2]:inp[0]:inp[3]:inp[1]
|
||||
vmrgow $I4,$T4,$I4
|
||||
vmrgow $I0,$T0,$I0
|
||||
vmrgow $I1,$T3,$I1
|
||||
vmrgow $I2,$T1,$I2
|
||||
vmrgow $I3,$T2,$I3
|
||||
vor $I4,$I4,$padbits
|
||||
|
||||
vperm $H0,$H0,$H0,$ACC0 # move hash to right lane
|
||||
vand $I0,$I0, $ACC1 # mask redundant input lane[s]
|
||||
vperm $H1,$H1,$H1,$ACC0
|
||||
vand $I1,$I1, $ACC1
|
||||
vperm $H2,$H2,$H2,$ACC0
|
||||
vand $I2,$I2, $ACC1
|
||||
vperm $H3,$H3,$H3,$ACC0
|
||||
vand $I3,$I3, $ACC1
|
||||
vperm $H4,$H4,$H4,$ACC0
|
||||
vand $I4,$I4, $ACC1
|
||||
|
||||
vaddudm $I0,$I0,$H0 # accumulate hash
|
||||
vxor $H0,$H0,$H0 # wipe hash value
|
||||
vaddudm $I1,$I1,$H1
|
||||
vxor $H1,$H1,$H1
|
||||
vaddudm $I2,$I2,$H2
|
||||
vxor $H2,$H2,$H2
|
||||
vaddudm $I3,$I3,$H3
|
||||
vxor $H3,$H3,$H3
|
||||
vaddudm $I4,$I4,$H4
|
||||
vxor $H4,$H4,$H4
|
||||
|
||||
xor. $len,$len,$len
|
||||
b Last_vsx
|
||||
|
||||
.align 4
|
||||
Ldone_vsx:
|
||||
$POP r0,`$VSXFRAME+$LRSAVE`($sp)
|
||||
li $x10,4
|
||||
li $x20,8
|
||||
li $x30,12
|
||||
li $x40,16
|
||||
stvwx_u $H0,$x00,$ctx # store hash
|
||||
stvwx_u $H1,$x10,$ctx
|
||||
stvwx_u $H2,$x20,$ctx
|
||||
stvwx_u $H3,$x30,$ctx
|
||||
stvwx_u $H4,$x40,$ctx
|
||||
|
||||
lwz r12,`$VSXFRAME-$SIZE_T*5-4`($sp)# pull vrsave
|
||||
mtlr r0
|
||||
li r10,`15+$LOCALS+128`
|
||||
li r11,`31+$LOCALS+128`
|
||||
mtspr 256,r12 # restore vrsave
|
||||
lvx v20,r10,$sp
|
||||
addi r10,r10,32
|
||||
lvx v21,r10,$sp
|
||||
addi r10,r10,32
|
||||
lvx v22,r11,$sp
|
||||
addi r11,r11,32
|
||||
lvx v23,r10,$sp
|
||||
addi r10,r10,32
|
||||
lvx v24,r11,$sp
|
||||
addi r11,r11,32
|
||||
lvx v25,r10,$sp
|
||||
addi r10,r10,32
|
||||
lvx v26,r11,$sp
|
||||
addi r11,r11,32
|
||||
lvx v27,r10,$sp
|
||||
addi r10,r10,32
|
||||
lvx v28,r11,$sp
|
||||
addi r11,r11,32
|
||||
lvx v29,r10,$sp
|
||||
addi r10,r10,32
|
||||
lvx v30,r11,$sp
|
||||
lvx v31,r10,$sp
|
||||
$POP r27,`$VSXFRAME-$SIZE_T*5`($sp)
|
||||
$POP r28,`$VSXFRAME-$SIZE_T*4`($sp)
|
||||
$POP r29,`$VSXFRAME-$SIZE_T*3`($sp)
|
||||
$POP r30,`$VSXFRAME-$SIZE_T*2`($sp)
|
||||
$POP r31,`$VSXFRAME-$SIZE_T*1`($sp)
|
||||
addi $sp,$sp,$VSXFRAME
|
||||
blr
|
||||
.long 0
|
||||
.byte 0,12,0x04,1,0x80,5,4,0
|
||||
.long 0
|
||||
.size __poly1305_blocks_vsx,.-__poly1305_blocks_vsx
|
||||
|
||||
.align 6
|
||||
LPICmeup:
|
||||
mflr r0
|
||||
bcl 20,31,\$+4
|
||||
mflr $const # vvvvvv "distance" between . and 1st data entry
|
||||
addi $const,$const,`64-8`
|
||||
mtlr r0
|
||||
blr
|
||||
.long 0
|
||||
.byte 0,12,0x14,0,0,0,0,0
|
||||
.space `64-9*4`
|
||||
|
||||
.quad 0x0000000003ffffff,0x0000000003ffffff # mask26
|
||||
.quad 0x000000000000001a,0x000000000000001a # _26
|
||||
.quad 0x0000000000000028,0x0000000000000028 # _40
|
||||
.quad 0x000000000e0f0001,0x000000001e1f1011 # I2perm
|
||||
.quad 0x0100000001000000,0x0100000001000000 # padbits
|
||||
.quad 0x0706050403020100,0x0f0e0d0c0b0a0908 # byte swap for big-endian
|
||||
|
||||
.quad 0x0000000000000000,0x0000000004050607 # magic tail masks
|
||||
.quad 0x0405060700000000,0x0000000000000000
|
||||
.quad 0x0000000000000000,0x0405060700000000
|
||||
|
||||
.quad 0xffffffff00000000,0xffffffffffffffff
|
||||
.quad 0xffffffff00000000,0xffffffff00000000
|
||||
.quad 0x0000000000000000,0xffffffff00000000
|
||||
___
|
||||
}}}
|
||||
$code.=<<___;
|
||||
.asciz "Poly1305 for PPC, CRYPTOGAMS by \@dot-asm"
|
||||
___
|
||||
|
||||
$code =~ s/\`([^\`]*)\`/eval $1/gem;
|
||||
print $code;
|
||||
foreach (split("\n",$code)) {
|
||||
s/\`([^\`]*)\`/eval($1)/ge;
|
||||
|
||||
# instructions prefixed with '?' are endian-specific and need
|
||||
# to be adjusted accordingly...
|
||||
if ($flavour !~ /le$/) { # big-endian
|
||||
s/be\?// or
|
||||
s/le\?/#le#/
|
||||
} else { # little-endian
|
||||
s/le\?// or
|
||||
s/be\?/#be#/
|
||||
}
|
||||
|
||||
print $_,"\n";
|
||||
}
|
||||
close STDOUT;
|
||||
@@ -24,204 +24,820 @@
|
||||
#
|
||||
# On side note, z13 enables vector base 2^26 implementation...
|
||||
|
||||
$flavour = shift;
|
||||
#
|
||||
# January 2019
|
||||
#
|
||||
# Add vx code path (base 2^26).
|
||||
#
|
||||
# Copyright IBM Corp. 2019
|
||||
# Author: Patrick Steuer <patrick.steuer@de.ibm.com>
|
||||
|
||||
use strict;
|
||||
use FindBin qw($Bin);
|
||||
use lib "$Bin/../..";
|
||||
use perlasm::s390x qw(:DEFAULT :VX AUTOLOAD LABEL);
|
||||
|
||||
my $flavour = shift;
|
||||
|
||||
my ($z,$SIZE_T);
|
||||
if ($flavour =~ /3[12]/) {
|
||||
$z=0; # S/390 ABI
|
||||
$SIZE_T=4;
|
||||
$g="";
|
||||
} else {
|
||||
$z=1; # zSeries ABI
|
||||
$SIZE_T=8;
|
||||
$g="g";
|
||||
}
|
||||
|
||||
my $output;
|
||||
while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {}
|
||||
open STDOUT,">$output";
|
||||
|
||||
$sp="%r15";
|
||||
my $sp="%r15";
|
||||
|
||||
# novx code path ctx layout
|
||||
# ---------------------------------
|
||||
# var value base off
|
||||
# ---------------------------------
|
||||
# u64 h[3] hash 2^64 0
|
||||
# u32 pad[2]
|
||||
# u64 r[2] key 2^64 32
|
||||
|
||||
# vx code path ctx layout
|
||||
# ---------------------------------
|
||||
# var value base off
|
||||
# ---------------------------------
|
||||
# u32 acc1[5] r^2-acc 2^26 0
|
||||
# u32 pad
|
||||
# u32 acc2[5] r-acc 2^26 24
|
||||
# u32 pad
|
||||
# u32 r1[5] r 2^26 48
|
||||
# u32 r15[5] 5*r 2^26 68
|
||||
# u32 r2[5] r^2 2^26 88
|
||||
# u32 r25[5] 5*r^2 2^26 108
|
||||
# u32 r4[5] r^4 2^26 128
|
||||
# u32 r45[5] 5*r^4 2^26 148
|
||||
|
||||
PERLASM_BEGIN($output);
|
||||
|
||||
TEXT ();
|
||||
|
||||
################
|
||||
# static void poly1305_init(void *ctx, const unsigned char key[16])
|
||||
{
|
||||
my ($ctx,$key)=map("%r$_",(2..3));
|
||||
my ($r0,$r1,$r2)=map("%r$_",(9,11,13));
|
||||
|
||||
sub MUL_RKEY { # r*=key
|
||||
my ($d0hi,$d0lo,$d1hi,$d1lo)=map("%r$_",(4..7));
|
||||
my ($t0,$t1,$s1)=map("%r$_",(8,10,12));
|
||||
|
||||
lg ("%r0","32($ctx)");
|
||||
lg ("%r1","40($ctx)");
|
||||
|
||||
srlg ($s1,"%r1",2);
|
||||
algr ($s1,"%r1");
|
||||
|
||||
lgr ($d0lo,$r0);
|
||||
lgr ($d1lo,$r1);
|
||||
|
||||
mlgr ($d0hi,"%r0");
|
||||
lgr ($r1,$d1lo);
|
||||
mlgr ($d1hi,$s1);
|
||||
|
||||
mlgr ($t0,"%r1");
|
||||
mlgr ($t1,"%r0");
|
||||
|
||||
algr ($d0lo,$d1lo);
|
||||
lgr ($d1lo,$r2);
|
||||
alcgr ($d0hi,$d1hi);
|
||||
lghi ($d1hi,0);
|
||||
|
||||
algr ($r1,$r0);
|
||||
alcgr ($t1,$t0);
|
||||
|
||||
msgr ($d1lo,$s1);
|
||||
msgr ($r2,"%r0");
|
||||
|
||||
algr ($r1,$d1lo);
|
||||
alcgr ($t1,$d1hi);
|
||||
|
||||
algr ($r1,$d0hi);
|
||||
alcgr ($r2,$t1);
|
||||
|
||||
lghi ($r0,-4);
|
||||
ngr ($r0,$r2);
|
||||
srlg ($t0,$r2,2);
|
||||
algr ($r0,$t0);
|
||||
lghi ($t1,3);
|
||||
ngr ($r2,$t1);
|
||||
|
||||
algr ($r0,$d0lo);
|
||||
alcgr ($r1,$d1hi);
|
||||
alcgr ($r2,$d1hi);
|
||||
}
|
||||
|
||||
sub ST_R5R { # store r,5*r -> base 2^26
|
||||
my @d=map("%r$_",(4..8));
|
||||
my @off=@_;
|
||||
|
||||
lgr (@d[2],$r0);
|
||||
lr ("%r1",@d[2]);
|
||||
nilh ("%r1",1023);
|
||||
lgr (@d[3],$r1);
|
||||
lr (@d[0],"%r1");
|
||||
srlg ("%r1",@d[2],52);
|
||||
lgr (@d[4],$r2);
|
||||
srlg ("%r0",@d[2],26);
|
||||
sll (@d[4],24);
|
||||
lr (@d[2],@d[3]);
|
||||
nilh ("%r0",1023);
|
||||
sll (@d[2],12);
|
||||
lr (@d[1],"%r0");
|
||||
&or (@d[2],"%r1");
|
||||
srlg ("%r1",@d[3],40);
|
||||
nilh (@d[2],1023);
|
||||
&or (@d[4],"%r1");
|
||||
srlg (@d[3],@d[3],14);
|
||||
nilh (@d[4],1023);
|
||||
nilh (@d[3],1023);
|
||||
|
||||
stm (@d[0],@d[4],"@off[0]($ctx)");
|
||||
mhi (@d[$_],5) for (0..4);
|
||||
stm (@d[0],@d[4],"@off[1]($ctx)");
|
||||
}
|
||||
|
||||
GLOBL ("poly1305_init");
|
||||
TYPE ("poly1305_init","\@function");
|
||||
ALIGN (16);
|
||||
LABEL ("poly1305_init");
|
||||
lghi ("%r0",0);
|
||||
lghi ("%r1",-1);
|
||||
stg ("%r0","0($ctx)"); # zero hash value / acc1
|
||||
stg ("%r0","8($ctx)");
|
||||
stg ("%r0","16($ctx)");
|
||||
|
||||
&{$z? \&clgr:\&clr} ($key,"%r0");
|
||||
je (".Ldone");
|
||||
|
||||
lrvg ("%r4","0($key)"); # load little-endian key
|
||||
lrvg ("%r5","8($key)");
|
||||
|
||||
nihl ("%r1",0xffc0); # 0xffffffc0ffffffff
|
||||
srlg ("%r0","%r1",4); # 0x0ffffffc0fffffff
|
||||
srlg ("%r1","%r1",4);
|
||||
nill ("%r1",0xfffc); # 0x0ffffffc0ffffffc
|
||||
|
||||
ngr ("%r4","%r0");
|
||||
ngr ("%r5","%r1");
|
||||
|
||||
stg ("%r4","32($ctx)");
|
||||
stg ("%r5","40($ctx)");
|
||||
|
||||
larl ("%r1","OPENSSL_s390xcap_P");
|
||||
lg ("%r0","16(%r1)");
|
||||
tmhh ("%r0",0x4000); # check for vector facility
|
||||
jz (".Ldone");
|
||||
|
||||
larl ("%r4","poly1305_blocks_vx");
|
||||
larl ("%r5","poly1305_emit_vx");
|
||||
|
||||
&{$z? \&stmg:\&stm} ("%r6","%r13","6*$SIZE_T($sp)");
|
||||
&{$z? \&stmg:\&stm} ("%r4","%r5","4*$z+228($ctx)");
|
||||
|
||||
lg ($r0,"32($ctx)");
|
||||
lg ($r1,"40($ctx)");
|
||||
lghi ($r2,0);
|
||||
|
||||
ST_R5R (48,68); # store r,5*r
|
||||
|
||||
MUL_RKEY();
|
||||
ST_R5R (88,108); # store r^2,5*r^2
|
||||
|
||||
MUL_RKEY();
|
||||
MUL_RKEY();
|
||||
ST_R5R (128,148); # store r^4,5*r^4
|
||||
|
||||
lghi ("%r0",0);
|
||||
stg ("%r0","24($ctx)"); # zero acc2
|
||||
stg ("%r0","32($ctx)");
|
||||
stg ("%r0","40($ctx)");
|
||||
|
||||
&{$z? \&lmg:\&lm} ("%r6","%r13","6*$SIZE_T($sp)");
|
||||
lghi ("%r2",1);
|
||||
br ("%r14");
|
||||
|
||||
LABEL (".Ldone");
|
||||
lghi ("%r2",0);
|
||||
br ("%r14");
|
||||
SIZE ("poly1305_init",".-poly1305_init");
|
||||
}
|
||||
|
||||
# VX CODE PATH
|
||||
{
|
||||
my $frame=8*16;
|
||||
my @m01=map("%v$_",(0..4));
|
||||
my @m23=map("%v$_",(5..9));
|
||||
my @tmp=@m23;
|
||||
my @acc=map("%v$_",(10..14));
|
||||
my @r=map("%v$_",(15..19));
|
||||
my @r5=map("%v$_",(20..24));
|
||||
my $padvec="%v26";
|
||||
my $mask4="%v27";
|
||||
my @vperm=map("%v$_",(28..30));
|
||||
my $mask="%v31";
|
||||
|
||||
sub REDUCE {
|
||||
vesrlg (@tmp[0],@acc[0],26);
|
||||
vesrlg (@tmp[3],@acc[3],26);
|
||||
vn (@acc[0],@acc[0],$mask);
|
||||
vn (@acc[3],@acc[3],$mask);
|
||||
vag (@acc[1],@acc[1],@tmp[0]); # carry 0->1
|
||||
vag (@acc[4],@acc[4],@tmp[3]); # carry 3->4
|
||||
|
||||
vesrlg (@tmp[1],@acc[1],26);
|
||||
vesrlg (@tmp[4],@acc[4],26);
|
||||
vn (@acc[1],@acc[1],$mask);
|
||||
vn (@acc[4],@acc[4],$mask);
|
||||
veslg (@tmp[0],@tmp[4],2);
|
||||
vag (@tmp[4],@tmp[4],@tmp[0]); # h[4]*=5
|
||||
vag (@acc[2],@acc[2],@tmp[1]); # carry 1->2
|
||||
vag (@acc[0],@acc[0],@tmp[4]); # carry 4->0
|
||||
|
||||
vesrlg (@tmp[2],@acc[2],26);
|
||||
vesrlg (@tmp[0],@acc[0],26);
|
||||
vn (@acc[2],@acc[2],$mask);
|
||||
vn (@acc[0],@acc[0],$mask);
|
||||
vag (@acc[3],@acc[3],@tmp[2]); # carry 2->3
|
||||
vag (@acc[1],@acc[1],@tmp[0]); # carry 0->1
|
||||
|
||||
vesrlg (@tmp[3],@acc[3],26);
|
||||
vn (@acc[3],@acc[3],$mask);
|
||||
vag (@acc[4],@acc[4],@tmp[3]); # carry 3->4
|
||||
}
|
||||
|
||||
################
|
||||
# static void poly1305_blocks_vx(void *ctx, const unsigned char *inp,
|
||||
# size_t len, u32 padbit)
|
||||
{
|
||||
my ($ctx,$inp,$len) = map("%r$_",(2..4));
|
||||
my $padbit="%r0";
|
||||
|
||||
GLOBL ("poly1305_blocks_vx");
|
||||
TYPE ("poly1305_blocks_vx","\@function");
|
||||
ALIGN (16);
|
||||
LABEL ("poly1305_blocks_vx");
|
||||
if ($z) {
|
||||
aghi ($sp,-$frame);
|
||||
vstm ("%v8","%v15","0($sp)");
|
||||
} else {
|
||||
std ("%f4","16*$SIZE_T+2*8($sp)");
|
||||
std ("%f6","16*$SIZE_T+3*8($sp)");
|
||||
llgfr ($len,$len);
|
||||
}
|
||||
llgfr ($padbit,"%r5");
|
||||
vlef (@acc[$_],"4*$_($ctx)",1) for (0..4); # load acc1
|
||||
larl ("%r5",".Lconst");
|
||||
vlef (@acc[$_],"24+4*$_($ctx)",3) for (0..4); # load acc2
|
||||
sllg ($padbit,$padbit,24);
|
||||
vlm (@vperm[0],$mask,"0(%r5)"); # load vperm ops, mask
|
||||
vgbm ($mask4,0x0707);
|
||||
vlvgp ($padvec,$padbit,$padbit);
|
||||
|
||||
srlg ("%r1",$len,6);
|
||||
ltgr ("%r1","%r1");
|
||||
jz (".Lvx_4x_done");
|
||||
|
||||
ALIGN (16);
|
||||
LABEL (".Lvx_4x");
|
||||
vlm ("%v20","%v23","0($inp)"); # load m0,m1,m2,m3
|
||||
|
||||
# m01,m23 -> base 2^26
|
||||
|
||||
vperm (@m01[0],"%v20","%v21",@vperm[0]);
|
||||
vperm (@m23[0],"%v22","%v23",@vperm[0]);
|
||||
vperm (@m01[2],"%v20","%v21",@vperm[1]);
|
||||
vperm (@m23[2],"%v22","%v23",@vperm[1]);
|
||||
vperm (@m01[4],"%v20","%v21",@vperm[2]);
|
||||
vperm (@m23[4],"%v22","%v23",@vperm[2]);
|
||||
|
||||
vesrlg (@m01[1],@m01[0],26);
|
||||
vesrlg (@m23[1],@m23[0],26);
|
||||
vesrlg (@m01[3],@m01[2],30);
|
||||
vesrlg (@m23[3],@m23[2],30);
|
||||
vesrlg (@m01[2],@m01[2],4);
|
||||
vesrlg (@m23[2],@m23[2],4);
|
||||
|
||||
vn (@m01[4],@m01[4],$mask4);
|
||||
vn (@m23[4],@m23[4],$mask4);
|
||||
for (0..3) {
|
||||
vn (@m01[$_],@m01[$_],$mask);
|
||||
vn (@m23[$_],@m23[$_],$mask);
|
||||
}
|
||||
vaf (@m01[4],@m01[4],$padvec); # pad m01
|
||||
vaf (@m23[4],@m23[4],$padvec); # pad m23
|
||||
|
||||
# acc = acc * r^4 + m01 * r^2 + m23
|
||||
|
||||
vlrepf (@r5[$_],"4*$_+108($ctx)") for (0..4); # load 5*r^2
|
||||
vlrepf (@r[$_],"4*$_+88($ctx)") for (0..4); # load r^2
|
||||
|
||||
vmalof (@tmp[0],@m01[4],@r5[1],@m23[0]);
|
||||
vmalof (@tmp[1],@m01[4],@r5[2],@m23[1]);
|
||||
vmalof (@tmp[2],@m01[4],@r5[3],@m23[2]);
|
||||
vmalof (@tmp[3],@m01[4],@r5[4],@m23[3]);
|
||||
vmalof (@tmp[4],@m01[4],@r[0],@m23[4]);
|
||||
|
||||
vmalof (@tmp[0],@m01[3],@r5[2],@tmp[0]);
|
||||
vmalof (@tmp[1],@m01[3],@r5[3],@tmp[1]);
|
||||
vmalof (@tmp[2],@m01[3],@r5[4],@tmp[2]);
|
||||
vmalof (@tmp[3],@m01[3],@r[0],@tmp[3]);
|
||||
vmalof (@tmp[4],@m01[3],@r[1],@tmp[4]);
|
||||
|
||||
vmalof (@tmp[0],@m01[2],@r5[3],@tmp[0]);
|
||||
vmalof (@tmp[1],@m01[2],@r5[4],@tmp[1]);
|
||||
vmalof (@tmp[2],@m01[2],@r[0],@tmp[2]);
|
||||
vmalof (@tmp[3],@m01[2],@r[1],@tmp[3]);
|
||||
vmalof (@tmp[4],@m01[2],@r[2],@tmp[4]);
|
||||
|
||||
vmalof (@tmp[0],@m01[1],@r5[4],@tmp[0]);
|
||||
vmalof (@tmp[1],@m01[1],@r[0],@tmp[1]);
|
||||
vmalof (@tmp[2],@m01[1],@r[1],@tmp[2]);
|
||||
vmalof (@tmp[3],@m01[1],@r[2],@tmp[3]);
|
||||
vmalof (@tmp[4],@m01[1],@r[3],@tmp[4]);
|
||||
|
||||
vmalof (@tmp[0],@m01[0],@r[0],@tmp[0]);
|
||||
vmalof (@tmp[1],@m01[0],@r[1],@tmp[1]);
|
||||
vmalof (@tmp[2],@m01[0],@r[2],@tmp[2]);
|
||||
vmalof (@tmp[3],@m01[0],@r[3],@tmp[3]);
|
||||
vmalof (@tmp[4],@m01[0],@r[4],@tmp[4]);
|
||||
|
||||
vlrepf (@r5[$_],"4*$_+148($ctx)") for (0..4); # load 5*r^4
|
||||
vlrepf (@r[$_],"4*$_+128($ctx)") for (0..4); # load r^4
|
||||
|
||||
vmalof (@tmp[0],@acc[4],@r5[1],@tmp[0]);
|
||||
vmalof (@tmp[1],@acc[4],@r5[2],@tmp[1]);
|
||||
vmalof (@tmp[2],@acc[4],@r5[3],@tmp[2]);
|
||||
vmalof (@tmp[3],@acc[4],@r5[4],@tmp[3]);
|
||||
vmalof (@tmp[4],@acc[4],@r[0],@tmp[4]);
|
||||
|
||||
vmalof (@tmp[0],@acc[3],@r5[2],@tmp[0]);
|
||||
vmalof (@tmp[1],@acc[3],@r5[3],@tmp[1]);
|
||||
vmalof (@tmp[2],@acc[3],@r5[4],@tmp[2]);
|
||||
vmalof (@tmp[3],@acc[3],@r[0],@tmp[3]);
|
||||
vmalof (@tmp[4],@acc[3],@r[1],@tmp[4]);
|
||||
|
||||
vmalof (@tmp[0],@acc[2],@r5[3],@tmp[0]);
|
||||
vmalof (@tmp[1],@acc[2],@r5[4],@tmp[1]);
|
||||
vmalof (@tmp[2],@acc[2],@r[0],@tmp[2]);
|
||||
vmalof (@tmp[3],@acc[2],@r[1],@tmp[3]);
|
||||
vmalof (@tmp[4],@acc[2],@r[2],@tmp[4]);
|
||||
|
||||
vmalof (@tmp[0],@acc[1],@r5[4],@tmp[0]);
|
||||
vmalof (@tmp[1],@acc[1],@r[0],@tmp[1]);
|
||||
vmalof (@tmp[2],@acc[1],@r[1],@tmp[2]);
|
||||
vmalof (@tmp[3],@acc[1],@r[2],@tmp[3]);
|
||||
vmalof (@tmp[4],@acc[1],@r[3],@tmp[4]);
|
||||
|
||||
vmalof (@acc[1],@acc[0],@r[1],@tmp[1]);
|
||||
vmalof (@acc[2],@acc[0],@r[2],@tmp[2]);
|
||||
vmalof (@acc[3],@acc[0],@r[3],@tmp[3]);
|
||||
vmalof (@acc[4],@acc[0],@r[4],@tmp[4]);
|
||||
vmalof (@acc[0],@acc[0],@r[0],@tmp[0]);
|
||||
|
||||
REDUCE ();
|
||||
|
||||
la ($inp,"64($inp)");
|
||||
brctg ("%r1",".Lvx_4x");
|
||||
|
||||
ALIGN (16);
|
||||
LABEL (".Lvx_4x_done");
|
||||
tml ($len,32);
|
||||
jz (".Lvx_2x_done");
|
||||
|
||||
vlm ("%v20","%v21","0($inp)"); # load m0,m1
|
||||
|
||||
# m01 -> base 2^26
|
||||
|
||||
vperm (@m01[0],"%v20","%v21",@vperm[0]);
|
||||
vperm (@m01[2],"%v20","%v21",@vperm[1]);
|
||||
vperm (@m01[4],"%v20","%v21",@vperm[2]);
|
||||
|
||||
vesrlg (@m01[1],@m01[0],26);
|
||||
vesrlg (@m01[3],@m01[2],30);
|
||||
vesrlg (@m01[2],@m01[2],4);
|
||||
|
||||
vn (@m01[4],@m01[4],$mask4);
|
||||
vn (@m01[$_],@m01[$_],$mask) for (0..3);
|
||||
|
||||
vaf (@m01[4],@m01[4],$padvec); # pad m01
|
||||
|
||||
# acc = acc * r^2+ m01
|
||||
|
||||
vlrepf (@r5[$_],"4*$_+108($ctx)") for (0..4); # load 5*r^2
|
||||
vlrepf (@r[$_],"4*$_+88($ctx)") for (0..4); # load r^2
|
||||
|
||||
vmalof (@tmp[0],@acc[4],@r5[1],@m01[0]);
|
||||
vmalof (@tmp[1],@acc[4],@r5[2],@m01[1]);
|
||||
vmalof (@tmp[2],@acc[4],@r5[3],@m01[2]);
|
||||
vmalof (@tmp[3],@acc[4],@r5[4],@m01[3]);
|
||||
vmalof (@tmp[4],@acc[4],@r[0],@m01[4]);
|
||||
|
||||
vmalof (@tmp[0],@acc[3],@r5[2],@tmp[0]);
|
||||
vmalof (@tmp[1],@acc[3],@r5[3],@tmp[1]);
|
||||
vmalof (@tmp[2],@acc[3],@r5[4],@tmp[2]);
|
||||
vmalof (@tmp[3],@acc[3],@r[0],@tmp[3]);
|
||||
vmalof (@tmp[4],@acc[3],@r[1],@tmp[4]);
|
||||
|
||||
vmalof (@tmp[0],@acc[2],@r5[3],@tmp[0]);
|
||||
vmalof (@tmp[1],@acc[2],@r5[4],@tmp[1]);
|
||||
vmalof (@tmp[2],@acc[2],@r[0],@tmp[2]);
|
||||
vmalof (@tmp[3],@acc[2],@r[1],@tmp[3]);
|
||||
vmalof (@tmp[4],@acc[2],@r[2],@tmp[4]);
|
||||
|
||||
vmalof (@tmp[0],@acc[1],@r5[4],@tmp[0]);
|
||||
vmalof (@tmp[1],@acc[1],@r[0],@tmp[1]);
|
||||
vmalof (@tmp[2],@acc[1],@r[1],@tmp[2]);
|
||||
vmalof (@tmp[3],@acc[1],@r[2],@tmp[3]);
|
||||
vmalof (@tmp[4],@acc[1],@r[3],@tmp[4]);
|
||||
|
||||
vmalof (@acc[1],@acc[0],@r[1],@tmp[1]);
|
||||
vmalof (@acc[2],@acc[0],@r[2],@tmp[2]);
|
||||
vmalof (@acc[3],@acc[0],@r[3],@tmp[3]);
|
||||
vmalof (@acc[4],@acc[0],@r[4],@tmp[4]);
|
||||
vmalof (@acc[0],@acc[0],@r[0],@tmp[0]);
|
||||
|
||||
REDUCE ();
|
||||
|
||||
la ($inp,"32($inp)");
|
||||
|
||||
ALIGN (16);
|
||||
LABEL (".Lvx_2x_done");
|
||||
tml ($len,16);
|
||||
jz (".Lvx_done");
|
||||
|
||||
vleig ($padvec,0,0);
|
||||
|
||||
vzero ("%v20");
|
||||
vl ("%v21","0($inp)"); # load m0
|
||||
|
||||
# m0 -> base 2^26
|
||||
|
||||
vperm (@m01[0],"%v20","%v21",@vperm[0]);
|
||||
vperm (@m01[2],"%v20","%v21",@vperm[1]);
|
||||
vperm (@m01[4],"%v20","%v21",@vperm[2]);
|
||||
|
||||
vesrlg (@m01[1],@m01[0],26);
|
||||
vesrlg (@m01[3],@m01[2],30);
|
||||
vesrlg (@m01[2],@m01[2],4);
|
||||
|
||||
vn (@m01[4],@m01[4],$mask4);
|
||||
vn (@m01[$_],@m01[$_],$mask) for (0..3);
|
||||
|
||||
vaf (@m01[4],@m01[4],$padvec); # pad m0
|
||||
|
||||
# acc = acc * r + m01
|
||||
|
||||
vlrepf (@r5[$_],"4*$_+68($ctx)") for (0..4); # load 5*r
|
||||
vlrepf (@r[$_],"4*$_+48($ctx)") for (0..4); # load r
|
||||
|
||||
vmalof (@tmp[0],@acc[4],@r5[1],@m01[0]);
|
||||
vmalof (@tmp[1],@acc[4],@r5[2],@m01[1]);
|
||||
vmalof (@tmp[2],@acc[4],@r5[3],@m01[2]);
|
||||
vmalof (@tmp[3],@acc[4],@r5[4],@m01[3]);
|
||||
vmalof (@tmp[4],@acc[4],@r[0],@m01[4]);
|
||||
|
||||
vmalof (@tmp[0],@acc[3],@r5[2],@tmp[0]);
|
||||
vmalof (@tmp[1],@acc[3],@r5[3],@tmp[1]);
|
||||
vmalof (@tmp[2],@acc[3],@r5[4],@tmp[2]);
|
||||
vmalof (@tmp[3],@acc[3],@r[0],@tmp[3]);
|
||||
vmalof (@tmp[4],@acc[3],@r[1],@tmp[4]);
|
||||
|
||||
vmalof (@tmp[0],@acc[2],@r5[3],@tmp[0]);
|
||||
vmalof (@tmp[1],@acc[2],@r5[4],@tmp[1]);
|
||||
vmalof (@tmp[2],@acc[2],@r[0],@tmp[2]);
|
||||
vmalof (@tmp[3],@acc[2],@r[1],@tmp[3]);
|
||||
vmalof (@tmp[4],@acc[2],@r[2],@tmp[4]);
|
||||
|
||||
vmalof (@tmp[0],@acc[1],@r5[4],@tmp[0]);
|
||||
vmalof (@tmp[1],@acc[1],@r[0],@tmp[1]);
|
||||
vmalof (@tmp[2],@acc[1],@r[1],@tmp[2]);
|
||||
vmalof (@tmp[3],@acc[1],@r[2],@tmp[3]);
|
||||
vmalof (@tmp[4],@acc[1],@r[3],@tmp[4]);
|
||||
|
||||
vmalof (@acc[1],@acc[0],@r[1],@tmp[1]);
|
||||
vmalof (@acc[2],@acc[0],@r[2],@tmp[2]);
|
||||
vmalof (@acc[3],@acc[0],@r[3],@tmp[3]);
|
||||
vmalof (@acc[4],@acc[0],@r[4],@tmp[4]);
|
||||
vmalof (@acc[0],@acc[0],@r[0],@tmp[0]);
|
||||
|
||||
REDUCE ();
|
||||
|
||||
ALIGN (16);
|
||||
LABEL (".Lvx_done");
|
||||
vstef (@acc[$_],"4*$_($ctx)",1) for (0..4); # store acc
|
||||
vstef (@acc[$_],"24+4*$_($ctx)",3) for (0..4);
|
||||
|
||||
if ($z) {
|
||||
vlm ("%v8","%v15","0($sp)");
|
||||
la ($sp,"$frame($sp)");
|
||||
} else {
|
||||
ld ("%f4","16*$SIZE_T+2*8($sp)");
|
||||
ld ("%f6","16*$SIZE_T+3*8($sp)");
|
||||
}
|
||||
br ("%r14");
|
||||
SIZE ("poly1305_blocks_vx",".-poly1305_blocks_vx");
|
||||
}
|
||||
|
||||
################
|
||||
# static void poly1305_emit_vx(void *ctx, unsigned char mac[16],
|
||||
# const u32 nonce[4])
|
||||
{
|
||||
my ($ctx,$mac,$nonce) = map("%r$_",(2..4));
|
||||
|
||||
GLOBL ("poly1305_emit_vx");
|
||||
TYPE ("poly1305_emit_vx","\@function");
|
||||
ALIGN (16);
|
||||
LABEL ("poly1305_emit_vx");
|
||||
if ($z) {
|
||||
aghi ($sp,-$frame);
|
||||
vstm ("%v8","%v15","0($sp)");
|
||||
} else {
|
||||
std ("%f4","16*$SIZE_T+2*8($sp)");
|
||||
std ("%f6","16*$SIZE_T+3*8($sp)");
|
||||
}
|
||||
larl ("%r5",".Lconst");
|
||||
|
||||
vlef (@acc[$_],"4*$_($ctx)",1) for (0..4); # load acc1
|
||||
vlef (@acc[$_],"24+4*$_($ctx)",3) for (0..4); # load acc2
|
||||
vlef (@r5[$_],"108+4*$_($ctx)",1) for (0..4); # load 5*r^2
|
||||
vlef (@r[$_],"88+4*$_($ctx)",1) for (0..4); # load r^2
|
||||
vlef (@r5[$_],"68+4*$_($ctx)",3) for (0..4); # load 5*r
|
||||
vlef (@r[$_],"48+4*$_($ctx)",3) for (0..4); # load r
|
||||
vl ($mask,"48(%r5)"); # load mask
|
||||
|
||||
# acc = acc1 * r^2 + acc2 * r
|
||||
|
||||
vmlof (@tmp[0],@acc[4],@r5[1]);
|
||||
vmlof (@tmp[1],@acc[4],@r5[2]);
|
||||
vmlof (@tmp[2],@acc[4],@r5[3]);
|
||||
vmlof (@tmp[3],@acc[4],@r5[4]);
|
||||
vmlof (@tmp[4],@acc[4],@r[0]);
|
||||
|
||||
vmalof (@tmp[0],@acc[3],@r5[2],@tmp[0]);
|
||||
vmalof (@tmp[1],@acc[3],@r5[3],@tmp[1]);
|
||||
vmalof (@tmp[2],@acc[3],@r5[4],@tmp[2]);
|
||||
vmalof (@tmp[3],@acc[3],@r[0],@tmp[3]);
|
||||
vmalof (@tmp[4],@acc[3],@r[1],@tmp[4]);
|
||||
|
||||
vmalof (@tmp[0],@acc[2],@r5[3],@tmp[0]);
|
||||
vmalof (@tmp[1],@acc[2],@r5[4],@tmp[1]);
|
||||
vmalof (@tmp[2],@acc[2],@r[0],@tmp[2]);
|
||||
vmalof (@tmp[3],@acc[2],@r[1],@tmp[3]);
|
||||
vmalof (@tmp[4],@acc[2],@r[2],@tmp[4]);
|
||||
|
||||
vmalof (@tmp[0],@acc[1],@r5[4],@tmp[0]);
|
||||
vmalof (@tmp[1],@acc[1],@r[0],@tmp[1]);
|
||||
vmalof (@tmp[2],@acc[1],@r[1],@tmp[2]);
|
||||
vmalof (@tmp[3],@acc[1],@r[2],@tmp[3]);
|
||||
vmalof (@tmp[4],@acc[1],@r[3],@tmp[4]);
|
||||
|
||||
vmalof (@acc[1],@acc[0],@r[1],@tmp[1]);
|
||||
vmalof (@acc[2],@acc[0],@r[2],@tmp[2]);
|
||||
vmalof (@acc[3],@acc[0],@r[3],@tmp[3]);
|
||||
vmalof (@acc[4],@acc[0],@r[4],@tmp[4]);
|
||||
vmalof (@acc[0],@acc[0],@r[0],@tmp[0]);
|
||||
|
||||
vzero ("%v27");
|
||||
vsumqg (@acc[$_],@acc[$_],"%v27") for (0..4);
|
||||
|
||||
REDUCE ();
|
||||
|
||||
vesrlg (@tmp[1],@acc[1],26);
|
||||
vn (@acc[1],@acc[1],$mask);
|
||||
vag (@acc[2],@acc[2],@tmp[1]); # carry 1->2
|
||||
|
||||
vesrlg (@tmp[2],@acc[2],26);
|
||||
vn (@acc[2],@acc[2],$mask);
|
||||
vag (@acc[3],@acc[3],@tmp[2]); # carry 2->3
|
||||
|
||||
vesrlg (@tmp[3],@acc[3],26);
|
||||
vn (@acc[3],@acc[3],$mask);
|
||||
vag (@acc[4],@acc[4],@tmp[3]); # carry 3->4
|
||||
|
||||
# acc -> base 2^64
|
||||
vleib ("%v30",6*8,7);
|
||||
vleib ("%v29",13*8,7);
|
||||
vleib ("%v28",3*8,7);
|
||||
|
||||
veslg (@acc[1],@acc[1],26);
|
||||
veslg (@acc[3],@acc[3],26);
|
||||
vo (@acc[0],@acc[0],@acc[1]);
|
||||
vo (@acc[2],@acc[2],@acc[3]);
|
||||
|
||||
veslg (@acc[2],@acc[2],4);
|
||||
vslb (@acc[2],@acc[2],"%v30"); # <<52
|
||||
vo (@acc[0],@acc[0],@acc[2]);
|
||||
|
||||
vslb (@tmp[4],@acc[4],"%v29"); # <<104
|
||||
vo (@acc[0],@acc[0],@tmp[4]);
|
||||
|
||||
vsrlb (@acc[1],@acc[4],"%v28"); # >>24
|
||||
|
||||
# acc %= 2^130-5
|
||||
vone ("%v26");
|
||||
vleig ("%v27",5,1);
|
||||
vone ("%v29");
|
||||
vleig ("%v26",-4,1);
|
||||
|
||||
vaq (@tmp[0],@acc[0],"%v27");
|
||||
vaccq (@tmp[1],@acc[0],"%v27");
|
||||
|
||||
vaq (@tmp[1],@tmp[1],"%v26");
|
||||
vaccq (@tmp[1],@tmp[1],@acc[1]);
|
||||
|
||||
vaq (@tmp[1],@tmp[1],"%v29");
|
||||
|
||||
vn (@tmp[2],@tmp[1],@acc[0]);
|
||||
vnc (@tmp[3],@tmp[0],@tmp[1]);
|
||||
vo (@acc[0],@tmp[2],@tmp[3]);
|
||||
|
||||
# acc += nonce
|
||||
vl (@vperm[0],"64(%r5)");
|
||||
vlef (@tmp[0],"4*$_($nonce)",3-$_) for (0..3);
|
||||
|
||||
vaq (@acc[0],@acc[0],@tmp[0]);
|
||||
|
||||
vperm (@acc[0],@acc[0],@acc[0],@vperm[0]);
|
||||
vst (@acc[0],"0($mac)"); # store mac
|
||||
|
||||
if ($z) {
|
||||
vlm ("%v8","%v15","0($sp)");
|
||||
la ($sp,"$frame($sp)");
|
||||
} else {
|
||||
ld ("%f4","16*$SIZE_T+2*8($sp)");
|
||||
ld ("%f6","16*$SIZE_T+3*8($sp)");
|
||||
}
|
||||
br ("%r14");
|
||||
SIZE ("poly1305_emit_vx",".-poly1305_emit_vx");
|
||||
}
|
||||
}
|
||||
|
||||
# NOVX CODE PATH
|
||||
{
|
||||
################
|
||||
# static void poly1305_blocks(void *ctx, const unsigned char *inp, size_t len,
|
||||
# u32 padbit)
|
||||
{
|
||||
my ($ctx,$inp,$len,$padbit) = map("%r$_",(2..5));
|
||||
|
||||
$code.=<<___;
|
||||
.text
|
||||
|
||||
.globl poly1305_init
|
||||
.type poly1305_init,\@function
|
||||
.align 16
|
||||
poly1305_init:
|
||||
lghi %r0,0
|
||||
lghi %r1,-1
|
||||
stg %r0,0($ctx) # zero hash value
|
||||
stg %r0,8($ctx)
|
||||
stg %r0,16($ctx)
|
||||
|
||||
cl${g}r $inp,%r0
|
||||
je .Lno_key
|
||||
|
||||
lrvg %r4,0($inp) # load little-endian key
|
||||
lrvg %r5,8($inp)
|
||||
|
||||
nihl %r1,0xffc0 # 0xffffffc0ffffffff
|
||||
srlg %r0,%r1,4 # 0x0ffffffc0fffffff
|
||||
srlg %r1,%r1,4
|
||||
nill %r1,0xfffc # 0x0ffffffc0ffffffc
|
||||
|
||||
ngr %r4,%r0
|
||||
ngr %r5,%r1
|
||||
|
||||
stg %r4,32($ctx)
|
||||
stg %r5,40($ctx)
|
||||
|
||||
.Lno_key:
|
||||
lghi %r2,0
|
||||
br %r14
|
||||
.size poly1305_init,.-poly1305_init
|
||||
___
|
||||
{
|
||||
my ($d0hi,$d0lo,$d1hi,$d1lo,$t0,$h0,$t1,$h1,$h2) = map("%r$_",(6..14));
|
||||
my ($r0,$r1,$s1) = map("%r$_",(0..2));
|
||||
GLOBL ("poly1305_blocks");
|
||||
TYPE ("poly1305_blocks","\@function");
|
||||
ALIGN (16);
|
||||
LABEL ("poly1305_blocks");
|
||||
$z? srlg ($len,$len,4) :srl ($len,4);
|
||||
lghi ("%r0",0);
|
||||
&{$z? \&clgr:\&clr} ($len,"%r0");
|
||||
je (".Lno_data");
|
||||
|
||||
$code.=<<___;
|
||||
.globl poly1305_blocks
|
||||
.type poly1305_blocks,\@function
|
||||
.align 16
|
||||
poly1305_blocks:
|
||||
srl${g} $len,4 # fixed-up in 64-bit build
|
||||
lghi %r0,0
|
||||
cl${g}r $len,%r0
|
||||
je .Lno_data
|
||||
&{$z? \&stmg:\&stm} ("%r6","%r14","6*$SIZE_T($sp)");
|
||||
|
||||
stm${g} %r6,%r14,`6*$SIZE_T`($sp)
|
||||
|
||||
llgfr $padbit,$padbit # clear upper half, much needed with
|
||||
llgfr ($padbit,$padbit); # clear upper half, much needed with
|
||||
# non-64-bit ABI
|
||||
lg $r0,32($ctx) # load key
|
||||
lg $r1,40($ctx)
|
||||
lg ($r0,"32($ctx)"); # load key
|
||||
lg ($r1,"40($ctx)");
|
||||
|
||||
lg $h0,0($ctx) # load hash value
|
||||
lg $h1,8($ctx)
|
||||
lg $h2,16($ctx)
|
||||
lg ($h0,"0($ctx)"); # load hash value
|
||||
lg ($h1,"8($ctx)");
|
||||
lg ($h2,"16($ctx)");
|
||||
|
||||
st$g $ctx,`2*$SIZE_T`($sp) # off-load $ctx
|
||||
srlg $s1,$r1,2
|
||||
algr $s1,$r1 # s1 = r1 + r1>>2
|
||||
j .Loop
|
||||
&{$z? \&stg:\&st} ($ctx,"2*$SIZE_T($sp)"); # off-load $ctx
|
||||
srlg ($s1,$r1,2);
|
||||
algr ($s1,$r1); # s1 = r1 + r1>>2
|
||||
j (".Loop");
|
||||
|
||||
.align 16
|
||||
.Loop:
|
||||
lrvg $d0lo,0($inp) # load little-endian input
|
||||
lrvg $d1lo,8($inp)
|
||||
la $inp,16($inp)
|
||||
ALIGN (16);
|
||||
LABEL (".Loop");
|
||||
lrvg ($d0lo,"0($inp)"); # load little-endian input
|
||||
lrvg ($d1lo,"8($inp)");
|
||||
la ($inp,"16($inp)");
|
||||
|
||||
algr $d0lo,$h0 # accumulate input
|
||||
alcgr $d1lo,$h1
|
||||
algr ($d0lo,$h0); # accumulate input
|
||||
alcgr ($d1lo,$h1);
|
||||
|
||||
lgr $h0,$d0lo
|
||||
mlgr $d0hi,$r0 # h0*r0 -> $d0hi:$d0lo
|
||||
lgr $h1,$d1lo
|
||||
mlgr $d1hi,$s1 # h1*5*r1 -> $d1hi:$d1lo
|
||||
lgr ($h0,$d0lo);
|
||||
mlgr ($d0hi,$r0); # h0*r0 -> $d0hi:$d0lo
|
||||
lgr ($h1,$d1lo);
|
||||
mlgr ($d1hi,$s1); # h1*5*r1 -> $d1hi:$d1lo
|
||||
|
||||
mlgr $t0,$r1 # h0*r1 -> $t0:$h0
|
||||
mlgr $t1,$r0 # h1*r0 -> $t1:$h1
|
||||
alcgr $h2,$padbit
|
||||
mlgr ($t0,$r1); # h0*r1 -> $t0:$h0
|
||||
mlgr ($t1,$r0); # h1*r0 -> $t1:$h1
|
||||
alcgr ($h2,$padbit);
|
||||
|
||||
algr $d0lo,$d1lo
|
||||
lgr $d1lo,$h2
|
||||
alcgr $d0hi,$d1hi
|
||||
lghi $d1hi,0
|
||||
algr ($d0lo,$d1lo);
|
||||
lgr ($d1lo,$h2);
|
||||
alcgr ($d0hi,$d1hi);
|
||||
lghi ($d1hi,0);
|
||||
|
||||
algr $h1,$h0
|
||||
alcgr $t1,$t0
|
||||
algr ($h1,$h0);
|
||||
alcgr ($t1,$t0);
|
||||
|
||||
msgr $d1lo,$s1 # h2*s1
|
||||
msgr $h2,$r0 # h2*r0
|
||||
msgr ($d1lo,$s1); # h2*s1
|
||||
msgr ($h2,$r0); # h2*r0
|
||||
|
||||
algr $h1,$d1lo
|
||||
alcgr $t1,$d1hi # $d1hi is zero
|
||||
algr ($h1,$d1lo);
|
||||
alcgr ($t1,$d1hi); # $d1hi is zero
|
||||
|
||||
algr $h1,$d0hi
|
||||
alcgr $h2,$t1
|
||||
algr ($h1,$d0hi);
|
||||
alcgr ($h2,$t1);
|
||||
|
||||
lghi $h0,-4 # final reduction step
|
||||
ngr $h0,$h2
|
||||
srlg $t0,$h2,2
|
||||
algr $h0,$t0
|
||||
lghi $t1,3
|
||||
ngr $h2,$t1
|
||||
lghi ($h0,-4); # final reduction step
|
||||
ngr ($h0,$h2);
|
||||
srlg ($t0,$h2,2);
|
||||
algr ($h0,$t0);
|
||||
lghi ($t1,3);
|
||||
ngr ($h2,$t1);
|
||||
|
||||
algr $h0,$d0lo
|
||||
alcgr $h1,$d1hi # $d1hi is still zero
|
||||
alcgr $h2,$d1hi # $d1hi is still zero
|
||||
algr ($h0,$d0lo);
|
||||
alcgr ($h1,$d1hi); # $d1hi is still zero
|
||||
alcgr ($h2,$d1hi); # $d1hi is still zero
|
||||
|
||||
brct$g $len,.Loop
|
||||
&{$z? \&brctg:\&brct} ($len,".Loop");
|
||||
|
||||
l$g $ctx,`2*$SIZE_T`($sp) # restore $ctx
|
||||
&{$z? \&lg:\&l} ($ctx,"2*$SIZE_T($sp)");# restore $ctx
|
||||
|
||||
stg $h0,0($ctx) # store hash value
|
||||
stg $h1,8($ctx)
|
||||
stg $h2,16($ctx)
|
||||
stg ($h0,"0($ctx)"); # store hash value
|
||||
stg ($h1,"8($ctx)");
|
||||
stg ($h2,"16($ctx)");
|
||||
|
||||
lm${g} %r6,%r14,`6*$SIZE_T`($sp)
|
||||
.Lno_data:
|
||||
br %r14
|
||||
.size poly1305_blocks,.-poly1305_blocks
|
||||
___
|
||||
&{$z? \&lmg:\&lm} ("%r6","%r14","6*$SIZE_T($sp)");
|
||||
LABEL (".Lno_data");
|
||||
br ("%r14");
|
||||
SIZE ("poly1305_blocks",".-poly1305_blocks");
|
||||
}
|
||||
|
||||
################
|
||||
# static void poly1305_emit(void *ctx, unsigned char mac[16],
|
||||
# const u32 nonce[4])
|
||||
{
|
||||
my ($mac,$nonce)=($inp,$len);
|
||||
my ($ctx,$mac,$nonce) = map("%r$_",(2..4));
|
||||
my ($h0,$h1,$h2,$d0,$d1)=map("%r$_",(5..9));
|
||||
|
||||
$code.=<<___;
|
||||
.globl poly1305_emit
|
||||
.type poly1305_emit,\@function
|
||||
.align 16
|
||||
poly1305_emit:
|
||||
stm${g} %r6,%r9,`6*$SIZE_T`($sp)
|
||||
GLOBL ("poly1305_emit");
|
||||
TYPE ("poly1305_emit","\@function");
|
||||
ALIGN (16);
|
||||
LABEL ("poly1305_emit");
|
||||
&{$z? \&stmg:\&stm} ("%r6","%r9","6*$SIZE_T($sp)");
|
||||
|
||||
lg $h0,0($ctx)
|
||||
lg $h1,8($ctx)
|
||||
lg $h2,16($ctx)
|
||||
lg ($h0,"0($ctx)");
|
||||
lg ($h1,"8($ctx)");
|
||||
lg ($h2,"16($ctx)");
|
||||
|
||||
lghi %r0,5
|
||||
lghi %r1,0
|
||||
lgr $d0,$h0
|
||||
lgr $d1,$h1
|
||||
lghi ("%r0",5);
|
||||
lghi ("%r1",0);
|
||||
lgr ($d0,$h0);
|
||||
lgr ($d1,$h1);
|
||||
|
||||
algr $h0,%r0 # compare to modulus
|
||||
alcgr $h1,%r1
|
||||
alcgr $h2,%r1
|
||||
algr ($h0,"%r0"); # compare to modulus
|
||||
alcgr ($h1,"%r1");
|
||||
alcgr ($h2,"%r1");
|
||||
|
||||
srlg $h2,$h2,2 # did it borrow/carry?
|
||||
slgr %r1,$h2 # 0-$h2>>2
|
||||
lg $h2,0($nonce) # load nonce
|
||||
lghi %r0,-1
|
||||
lg $ctx,8($nonce)
|
||||
xgr %r0,%r1 # ~%r1
|
||||
srlg ($h2,$h2,2); # did it borrow/carry?
|
||||
slgr ("%r1",$h2); # 0-$h2>>2
|
||||
lg ($h2,"0($nonce)"); # load nonce
|
||||
lghi ("%r0",-1);
|
||||
lg ($ctx,"8($nonce)");
|
||||
xgr ("%r0","%r1"); # ~%r1
|
||||
|
||||
ngr $h0,%r1
|
||||
ngr $d0,%r0
|
||||
ngr $h1,%r1
|
||||
ngr $d1,%r0
|
||||
ogr $h0,$d0
|
||||
rllg $d0,$h2,32 # flip nonce words
|
||||
ogr $h1,$d1
|
||||
rllg $d1,$ctx,32
|
||||
ngr ($h0,"%r1");
|
||||
ngr ($d0,"%r0");
|
||||
ngr ($h1,"%r1");
|
||||
ngr ($d1,"%r0");
|
||||
ogr ($h0,$d0);
|
||||
rllg ($d0,$h2,32); # flip nonce words
|
||||
ogr ($h1,$d1);
|
||||
rllg ($d1,$ctx,32);
|
||||
|
||||
algr $h0,$d0 # accumulate nonce
|
||||
alcgr $h1,$d1
|
||||
algr ($h0,$d0); # accumulate nonce
|
||||
alcgr ($h1,$d1);
|
||||
|
||||
strvg $h0,0($mac) # write little-endian result
|
||||
strvg $h1,8($mac)
|
||||
strvg ($h0,"0($mac)"); # write little-endian result
|
||||
strvg ($h1,"8($mac)");
|
||||
|
||||
lm${g} %r6,%r9,`6*$SIZE_T`($sp)
|
||||
br %r14
|
||||
.size poly1305_emit,.-poly1305_emit
|
||||
|
||||
.string "Poly1305 for s390x, CRYPTOGAMS by <appro\@openssl.org>"
|
||||
___
|
||||
&{$z? \&lmg:\&lm} ("%r6","%r9","6*$SIZE_T($sp)");
|
||||
br ("%r14");
|
||||
SIZE ("poly1305_emit",".-poly1305_emit");
|
||||
}
|
||||
}
|
||||
################
|
||||
|
||||
$code =~ s/\`([^\`]*)\`/eval $1/gem;
|
||||
$code =~ s/\b(srlg\s+)(%r[0-9]+\s*,)\s*([0-9]+)/$1$2$2$3/gm;
|
||||
ALIGN (128);
|
||||
LABEL (".Lconst");
|
||||
LONG (0x00060504,0x03020100,0x00161514,0x13121110); # vperm op[m[1],m[0]]
|
||||
LONG (0x000c0b0a,0x09080706,0x001c1b1a,0x19181716); # vperm op[m[3],m[2]]
|
||||
LONG (0x00000000,0x000f0e0d,0x00000000,0x001f1e1d); # vperm op[ - ,m[4]]
|
||||
LONG (0x00000000,0x03ffffff,0x00000000,0x03ffffff); # [0,2^26-1,0,2^26-1]
|
||||
LONG (0x0f0e0d0c,0x0b0a0908,0x07060504,0x03020100); # vperm op endian
|
||||
STRING ("\"Poly1305 for s390x, CRYPTOGAMS by <appro\@openssl.org>\"");
|
||||
|
||||
print $code;
|
||||
close STDOUT;
|
||||
PERLASM_END();
|
||||
@@ -17,8 +17,5 @@ GENERATE[poly1305-armv8.S]=asm/poly1305-armv8.pl $(PERLASM_SCHEME)
|
||||
INCLUDE[poly1305-armv8.o]=..
|
||||
GENERATE[poly1305-mips.S]=asm/poly1305-mips.pl $(PERLASM_SCHEME)
|
||||
INCLUDE[poly1305-mips.o]=..
|
||||
|
||||
BEGINRAW[Makefile(unix)]
|
||||
{- $builddir -}/poly1305-%.S: {- $sourcedir -}/asm/poly1305-%.pl
|
||||
CC="$(CC)" $(PERL) $< $(PERLASM_SCHEME) $@
|
||||
ENDRAW[Makefile(unix)]
|
||||
GENERATE[poly1305-c64xplus.S]=asm/poly1305-c64xplus.pl $(PERLASM_SCHEME)
|
||||
GENERATE[poly1305-s390x.S]=asm/poly1305-s390x.pl $(PERLASM_SCHEME)
|
||||
Reference in New Issue
Block a user