Hakase/openssl-custom
1
0 Fork 0
Code Issues Pull Requests Releases Wiki Activity

Latest update.

Browse Source
This commit is contained in:
Hakase
2020-01-17 19:45:12 +09:00
parent 016df9f433
commit 0f7abb4eb6
1100 changed files with 47785 additions and 16292 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crypto/aes/asm/aes-x86_64.pl
+4
View File
@@ -328,6 +328,7 @@ $code.=<<___;
.type _x86_64_AES_encrypt,\@abi-omnipotent
.align 16
_x86_64_AES_encrypt:
.cfi_startproc
xor 0($key),$s0 # xor with key
xor 4($key),$s1
xor 8($key),$s2
@@ -363,6 +364,7 @@ ___
}
$code.=<<___;
.byte 0xf3,0xc3 # rep ret
.cfi_endproc
.size _x86_64_AES_encrypt,.-_x86_64_AES_encrypt
___
@@ -912,6 +914,7 @@ $code.=<<___;
.type _x86_64_AES_decrypt,\@abi-omnipotent
.align 16
_x86_64_AES_decrypt:
.cfi_startproc
xor 0($key),$s0 # xor with key
xor 4($key),$s1
xor 8($key),$s2
@@ -954,6 +957,7 @@ ___
}
$code.=<<___;
.byte 0xf3,0xc3 # rep ret
.cfi_endproc
.size _x86_64_AES_decrypt,.-_x86_64_AES_decrypt
___
crypto/aes/asm/aesni-mb-x86_64.pl
+1 -1
View File
@@ -71,7 +71,7 @@ if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
$avx = ($1>=10) + ($1>=11);
}
if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([3-9]\.[0-9]+)/) {
if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([0-9]+\.[0-9]+)/) {
$avx = ($2>=3.0) + ($2>3.0);
}
crypto/aes/asm/aesni-sha1-x86_64.pl
+7 -1
View File
@@ -109,7 +109,7 @@ $avx=1 if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
$avx=1 if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
`ml64 2>&1` =~ /Version ([0-9]+)\./ &&
$1>=10);
$avx=1 if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([3-9]\.[0-9]+)/ && $2>=3.0);
$avx=1 if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([0-9]+\.[0-9]+)/ && $2>=3.0);
$shaext=1; ### set to zero if compiling for 1.0.1
@@ -135,6 +135,7 @@ $code.=<<___;
.type aesni_cbc_sha1_enc,\@abi-omnipotent
.align 32
aesni_cbc_sha1_enc:
.cfi_startproc
# caller should check for SSSE3 and AES-NI bits
mov OPENSSL_ia32cap_P+0(%rip),%r10d
mov OPENSSL_ia32cap_P+4(%rip),%r11
@@ -153,6 +154,7 @@ ___
$code.=<<___;
jmp aesni_cbc_sha1_enc_ssse3
ret
.cfi_endproc
.size aesni_cbc_sha1_enc,.-aesni_cbc_sha1_enc
___
@@ -842,6 +844,7 @@ $code.=<<___;
.type aesni256_cbc_sha1_dec,\@abi-omnipotent
.align 32
aesni256_cbc_sha1_dec:
.cfi_startproc
# caller should check for SSSE3 and AES-NI bits
mov OPENSSL_ia32cap_P+0(%rip),%r10d
mov OPENSSL_ia32cap_P+4(%rip),%r11d
@@ -856,6 +859,7 @@ ___
$code.=<<___;
jmp aesni256_cbc_sha1_dec_ssse3
ret
.cfi_endproc
.size aesni256_cbc_sha1_dec,.-aesni256_cbc_sha1_dec
.type aesni256_cbc_sha1_dec_ssse3,\@function,6
@@ -1762,6 +1766,7 @@ $code.=<<___;
.type aesni_cbc_sha1_enc_shaext,\@function,6
.align 32
aesni_cbc_sha1_enc_shaext:
.cfi_startproc
mov `($win64?56:8)`(%rsp),$inp # load 7th argument
___
$code.=<<___ if ($win64);
@@ -1913,6 +1918,7 @@ $code.=<<___ if ($win64);
___
$code.=<<___;
ret
.cfi_endproc
.size aesni_cbc_sha1_enc_shaext,.-aesni_cbc_sha1_enc_shaext
___
}}}
crypto/aes/asm/aesni-sha256-x86_64.pl
+47 -15
View File
@@ -71,7 +71,7 @@ if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
$avx = ($1>=10) + ($1>=12);
}
if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([3-9]\.[0-9]+)/) {
if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([0-9]+\.[0-9]+)/) {
$avx = ($2>=3.0) + ($2>3.0);
}
@@ -125,6 +125,7 @@ $code=<<___;
.type $func,\@abi-omnipotent
.align 16
$func:
.cfi_startproc
___
if ($avx) {
$code.=<<___;
@@ -164,6 +165,7 @@ $code.=<<___;
ud2
.Lprobe:
ret
.cfi_endproc
.size $func,.-$func
.align 64
@@ -1086,7 +1088,23 @@ $code.=<<___;
vmovdqa $t0,0x00(%rsp)
xor $a1,$a1
vmovdqa $t1,0x20(%rsp)
___
$code.=<<___ if (!$win64);
# temporarily use %rsi as frame pointer
mov $_rsp,%rsi
.cfi_def_cfa %rsi,8
___
$code.=<<___;
lea -$PUSH8(%rsp),%rsp
___
$code.=<<___ if (!$win64);
# the frame info is at $_rsp, but the stack is moving...
# so a second frame pointer is saved at -8(%rsp)
# that is in the red zone
mov %rsi,-8(%rsp)
.cfi_cfa_expression %rsp-8,deref,+8
___
$code.=<<___;
mov $B,$a3
vmovdqa $t2,0x00(%rsp)
xor $C,$a3 # magic
@@ -1108,7 +1126,17 @@ my @X = @_;
my @insns = (&$body,&$body,&$body,&$body); # 96 instructions
my $base = "+2*$PUSH8(%rsp)";
&lea ("%rsp","-$PUSH8(%rsp)") if (($j%2)==0);
if (($j%2)==0) {
&lea ("%rsp","-$PUSH8(%rsp)");
$code.=<<___ if (!$win64);
.cfi_cfa_expression %rsp+`$PUSH8-8`,deref,+8
# copy secondary frame pointer to new location again at -8(%rsp)
pushq $PUSH8-8(%rsp)
.cfi_cfa_expression %rsp,deref,+8
lea 8(%rsp),%rsp
.cfi_cfa_expression %rsp-8,deref,+8
___
}
foreach (Xupdate_256_AVX()) { # 29 instructions
eval;
eval(shift(@insns));
@@ -1234,26 +1262,28 @@ $code.=<<___;
jbe .Loop_avx2
lea (%rsp),$Tbl
# temporarily use $Tbl as index to $_rsp
# this avoids the need to save a secondary frame pointer at -8(%rsp)
.cfi_cfa_expression $Tbl+`16*$SZ+7*8`,deref,+8
.Ldone_avx2:
lea ($Tbl),%rsp
mov $_ivp,$ivp
mov $_rsp,%rsi
mov 16*$SZ+4*8($Tbl),$ivp
mov 16*$SZ+7*8($Tbl),%rsi
.cfi_def_cfa %rsi,8
vmovdqu $iv,($ivp) # output IV
vzeroall
___
$code.=<<___ if ($win64);
movaps `$framesz+16*0`(%rsp),%xmm6
movaps `$framesz+16*1`(%rsp),%xmm7
movaps `$framesz+16*2`(%rsp),%xmm8
movaps `$framesz+16*3`(%rsp),%xmm9
movaps `$framesz+16*4`(%rsp),%xmm10
movaps `$framesz+16*5`(%rsp),%xmm11
movaps `$framesz+16*6`(%rsp),%xmm12
movaps `$framesz+16*7`(%rsp),%xmm13
movaps `$framesz+16*8`(%rsp),%xmm14
movaps `$framesz+16*9`(%rsp),%xmm15
movaps `$framesz+16*0`($Tbl),%xmm6
movaps `$framesz+16*1`($Tbl),%xmm7
movaps `$framesz+16*2`($Tbl),%xmm8
movaps `$framesz+16*3`($Tbl),%xmm9
movaps `$framesz+16*4`($Tbl),%xmm10
movaps `$framesz+16*5`($Tbl),%xmm11
movaps `$framesz+16*6`($Tbl),%xmm12
movaps `$framesz+16*7`($Tbl),%xmm13
movaps `$framesz+16*8`($Tbl),%xmm14
movaps `$framesz+16*9`($Tbl),%xmm15
___
$code.=<<___;
mov -48(%rsi),%r15
@@ -1341,6 +1371,7 @@ $code.=<<___;
.type ${func}_shaext,\@function,6
.align 32
${func}_shaext:
.cfi_startproc
mov `($win64?56:8)`(%rsp),$inp # load 7th argument
___
$code.=<<___ if ($win64);
@@ -1557,6 +1588,7 @@ $code.=<<___ if ($win64);
___
$code.=<<___;
ret
.cfi_endproc
.size ${func}_shaext,.-${func}_shaext
___
}
crypto/aes/asm/aesni-x86_64.pl
+18 -2
View File
@@ -59,7 +59,7 @@
# nothing one can do and the result appears optimal. CCM result is
# identical to CBC, because CBC-MAC is essentially CBC encrypt without
# saving output. CCM CTR "stays invisible," because it's neatly
# interleaved wih CBC-MAC. This provides ~30% improvement over
# interleaved with CBC-MAC. This provides ~30% improvement over
# "straightforward" CCM implementation with CTR and CBC-MAC performed
# disjointly. Parallelizable modes practically achieve the theoretical
# limit.
@@ -986,6 +986,7 @@ $code.=<<___;
.type aesni_ccm64_encrypt_blocks,\@function,6
.align 16
aesni_ccm64_encrypt_blocks:
.cfi_startproc
___
$code.=<<___ if ($win64);
lea -0x58(%rsp),%rsp
@@ -1068,6 +1069,7 @@ $code.=<<___ if ($win64);
___
$code.=<<___;
ret
.cfi_endproc
.size aesni_ccm64_encrypt_blocks,.-aesni_ccm64_encrypt_blocks
___
######################################################################
@@ -1076,6 +1078,7 @@ $code.=<<___;
.type aesni_ccm64_decrypt_blocks,\@function,6
.align 16
aesni_ccm64_decrypt_blocks:
.cfi_startproc
___
$code.=<<___ if ($win64);
lea -0x58(%rsp),%rsp
@@ -1175,6 +1178,7 @@ $code.=<<___ if ($win64);
___
$code.=<<___;
ret
.cfi_endproc
.size aesni_ccm64_decrypt_blocks,.-aesni_ccm64_decrypt_blocks
___
}
@@ -3033,6 +3037,7 @@ $code.=<<___;
.type __ocb_encrypt6,\@abi-omnipotent
.align 32
__ocb_encrypt6:
.cfi_startproc
pxor $rndkey0l,@offset[5] # offset_i ^ round[0]
movdqu ($L_p,$i1),@offset[1]
movdqa @offset[0],@offset[2]
@@ -3130,11 +3135,13 @@ __ocb_encrypt6:
aesenclast @offset[4],$inout4
aesenclast @offset[5],$inout5
ret
.cfi_endproc
.size __ocb_encrypt6,.-__ocb_encrypt6
.type __ocb_encrypt4,\@abi-omnipotent
.align 32
__ocb_encrypt4:
.cfi_startproc
pxor $rndkey0l,@offset[5] # offset_i ^ round[0]
movdqu ($L_p,$i1),@offset[1]
movdqa @offset[0],@offset[2]
@@ -3199,11 +3206,13 @@ __ocb_encrypt4:
aesenclast @offset[2],$inout2
aesenclast @offset[3],$inout3
ret
.cfi_endproc
.size __ocb_encrypt4,.-__ocb_encrypt4
.type __ocb_encrypt1,\@abi-omnipotent
.align 32
__ocb_encrypt1:
.cfi_startproc
pxor @offset[5],$inout5 # offset_i
pxor $rndkey0l,$inout5 # offset_i ^ round[0]
pxor $inout0,$checksum # accumulate checksum
@@ -3234,6 +3243,7 @@ __ocb_encrypt1:
aesenclast $inout5,$inout0
ret
.cfi_endproc
.size __ocb_encrypt1,.-__ocb_encrypt1
.globl aesni_ocb_decrypt
@@ -3515,6 +3525,7 @@ $code.=<<___;
.type __ocb_decrypt6,\@abi-omnipotent
.align 32
__ocb_decrypt6:
.cfi_startproc
pxor $rndkey0l,@offset[5] # offset_i ^ round[0]
movdqu ($L_p,$i1),@offset[1]
movdqa @offset[0],@offset[2]
@@ -3606,11 +3617,13 @@ __ocb_decrypt6:
aesdeclast @offset[4],$inout4
aesdeclast @offset[5],$inout5
ret
.cfi_endproc
.size __ocb_decrypt6,.-__ocb_decrypt6
.type __ocb_decrypt4,\@abi-omnipotent
.align 32
__ocb_decrypt4:
.cfi_startproc
pxor $rndkey0l,@offset[5] # offset_i ^ round[0]
movdqu ($L_p,$i1),@offset[1]
movdqa @offset[0],@offset[2]
@@ -3671,11 +3684,13 @@ __ocb_decrypt4:
aesdeclast @offset[2],$inout2
aesdeclast @offset[3],$inout3
ret
.cfi_endproc
.size __ocb_decrypt4,.-__ocb_decrypt4
.type __ocb_decrypt1,\@abi-omnipotent
.align 32
__ocb_decrypt1:
.cfi_startproc
pxor @offset[5],$inout5 # offset_i
pxor $rndkey0l,$inout5 # offset_i ^ round[0]
pxor $inout5,$inout0 # input ^ round[0] ^ offset_i
@@ -3705,6 +3720,7 @@ __ocb_decrypt1:
aesdeclast $inout5,$inout0
ret
.cfi_endproc
.size __ocb_decrypt1,.-__ocb_decrypt1
___
} }}
@@ -4639,7 +4655,6 @@ __aesni_set_encrypt_key:
add \$8,%rsp
.cfi_adjust_cfa_offset -8
ret
.cfi_endproc
.LSEH_end_set_encrypt_key:
.align 16
@@ -4710,6 +4725,7 @@ __aesni_set_encrypt_key:
shufps \$0b10101010,%xmm1,%xmm1 # critical path
xorps %xmm1,%xmm2
ret
.cfi_endproc
.size ${PREFIX}_set_encrypt_key,.-${PREFIX}_set_encrypt_key
.size __aesni_set_encrypt_key,.-__aesni_set_encrypt_key
___
crypto/aes/asm/aesv8-armx.pl
+830
View File
@@ -384,6 +384,836 @@ ___
&gen_block("en");
&gen_block("de");
}}}
# Performance in cycles per byte.
# Processed with AES-ECB different key size.
# It shows the value before and after optimization as below:
# (before/after):
#
# AES-128-ECB AES-192-ECB AES-256-ECB
# Cortex-A57 1.85/0.82 2.16/0.96 2.47/1.10
# Cortex-A72 1.64/0.85 1.82/0.99 2.13/1.14
# Optimization is implemented by loop unrolling and interleaving.
# Commonly, we choose the unrolling factor as 5, if the input
# data size smaller than 5 blocks, but not smaller than 3 blocks,
# choose 3 as the unrolling factor.
# If the input data size dsize >= 5*16 bytes, then take 5 blocks
# as one iteration, every loop the left size lsize -= 5*16.
# If 5*16 > lsize >= 3*16 bytes, take 3 blocks as one iteration,
# every loop lsize -=3*16.
# If lsize < 3*16 bytes, treat them as the tail, interleave the
# two blocks AES instructions.
# There is one special case, if the original input data size dsize
# = 16 bytes, we will treat it seperately to improve the
# performance: one independent code block without LR, FP load and
# store, just looks like what the original ECB implementation does.
{{{
my ($inp,$out,$len,$key)=map("x$_",(0..3));
my ($enc,$rounds,$cnt,$key_,$step)=("w4","w5","w6","x7","x8");
my ($dat0,$dat1,$in0,$in1,$tmp0,$tmp1,$tmp2,$rndlast)=map("q$_",(0..7));
my ($dat,$tmp,$rndzero_n_last)=($dat0,$tmp0,$tmp1);
### q7 last round key
### q10-q15 q7 Last 7 round keys
### q8-q9 preloaded round keys except last 7 keys for big size
### q5, q6, q8-q9 preloaded round keys except last 7 keys for only 16 byte
{
my ($dat2,$in2,$tmp2)=map("q$_",(10,11,9));
my ($dat3,$in3,$tmp3); # used only in 64-bit mode
my ($dat4,$in4,$tmp4);
if ($flavour =~ /64/) {
($dat2,$dat3,$dat4,$in2,$in3,$in4,$tmp3,$tmp4)=map("q$_",(16..23));
}
$code.=<<___;
.globl ${prefix}_ecb_encrypt
.type ${prefix}_ecb_encrypt,%function
.align 5
${prefix}_ecb_encrypt:
___
$code.=<<___ if ($flavour =~ /64/);
subs $len,$len,#16
// Original input data size bigger than 16, jump to big size processing.
b.ne .Lecb_big_size
vld1.8 {$dat0},[$inp]
cmp $enc,#0 // en- or decrypting?
ldr $rounds,[$key,#240]
vld1.32 {q5-q6},[$key],#32 // load key schedule...
b.eq .Lecb_small_dec
aese $dat0,q5
aesmc $dat0,$dat0
vld1.32 {q8-q9},[$key],#32 // load key schedule...
aese $dat0,q6
aesmc $dat0,$dat0
subs $rounds,$rounds,#10 // if rounds==10, jump to aes-128-ecb processing
b.eq .Lecb_128_enc
.Lecb_round_loop:
aese $dat0,q8
aesmc $dat0,$dat0
vld1.32 {q8},[$key],#16 // load key schedule...
aese $dat0,q9
aesmc $dat0,$dat0
vld1.32 {q9},[$key],#16 // load key schedule...
subs $rounds,$rounds,#2 // bias
b.gt .Lecb_round_loop
.Lecb_128_enc:
vld1.32 {q10-q11},[$key],#32 // load key schedule...
aese $dat0,q8
aesmc $dat0,$dat0
aese $dat0,q9
aesmc $dat0,$dat0
vld1.32 {q12-q13},[$key],#32 // load key schedule...
aese $dat0,q10
aesmc $dat0,$dat0
aese $dat0,q11
aesmc $dat0,$dat0
vld1.32 {q14-q15},[$key],#32 // load key schedule...
aese $dat0,q12
aesmc $dat0,$dat0
aese $dat0,q13
aesmc $dat0,$dat0
vld1.32 {$rndlast},[$key]
aese $dat0,q14
aesmc $dat0,$dat0
aese $dat0,q15
veor $dat0,$dat0,$rndlast
vst1.8 {$dat0},[$out]
b .Lecb_Final_abort
.Lecb_small_dec:
aesd $dat0,q5
aesimc $dat0,$dat0
vld1.32 {q8-q9},[$key],#32 // load key schedule...
aesd $dat0,q6
aesimc $dat0,$dat0
subs $rounds,$rounds,#10 // bias
b.eq .Lecb_128_dec
.Lecb_dec_round_loop:
aesd $dat0,q8
aesimc $dat0,$dat0
vld1.32 {q8},[$key],#16 // load key schedule...
aesd $dat0,q9
aesimc $dat0,$dat0
vld1.32 {q9},[$key],#16 // load key schedule...
subs $rounds,$rounds,#2 // bias
b.gt .Lecb_dec_round_loop
.Lecb_128_dec:
vld1.32 {q10-q11},[$key],#32 // load key schedule...
aesd $dat0,q8
aesimc $dat0,$dat0
aesd $dat0,q9
aesimc $dat0,$dat0
vld1.32 {q12-q13},[$key],#32 // load key schedule...
aesd $dat0,q10
aesimc $dat0,$dat0
aesd $dat0,q11
aesimc $dat0,$dat0
vld1.32 {q14-q15},[$key],#32 // load key schedule...
aesd $dat0,q12
aesimc $dat0,$dat0
aesd $dat0,q13
aesimc $dat0,$dat0
vld1.32 {$rndlast},[$key]
aesd $dat0,q14
aesimc $dat0,$dat0
aesd $dat0,q15
veor $dat0,$dat0,$rndlast
vst1.8 {$dat0},[$out]
b .Lecb_Final_abort
.Lecb_big_size:
___
$code.=<<___ if ($flavour =~ /64/);
stp x29,x30,[sp,#-16]!
add x29,sp,#0
___
$code.=<<___ if ($flavour !~ /64/);
mov ip,sp
stmdb sp!,{r4-r8,lr}
vstmdb sp!,{d8-d15} @ ABI specification says so
ldmia ip,{r4-r5} @ load remaining args
subs $len,$len,#16
___
$code.=<<___;
mov $step,#16
b.lo .Lecb_done
cclr $step,eq
cmp $enc,#0 // en- or decrypting?
ldr $rounds,[$key,#240]
and $len,$len,#-16
vld1.8 {$dat},[$inp],$step
vld1.32 {q8-q9},[$key] // load key schedule...
sub $rounds,$rounds,#6
add $key_,$key,x5,lsl#4 // pointer to last 7 round keys
sub $rounds,$rounds,#2
vld1.32 {q10-q11},[$key_],#32
vld1.32 {q12-q13},[$key_],#32
vld1.32 {q14-q15},[$key_],#32
vld1.32 {$rndlast},[$key_]
add $key_,$key,#32
mov $cnt,$rounds
b.eq .Lecb_dec
vld1.8 {$dat1},[$inp],#16
subs $len,$len,#32 // bias
add $cnt,$rounds,#2
vorr $in1,$dat1,$dat1
vorr $dat2,$dat1,$dat1
vorr $dat1,$dat,$dat
b.lo .Lecb_enc_tail
vorr $dat1,$in1,$in1
vld1.8 {$dat2},[$inp],#16
___
$code.=<<___ if ($flavour =~ /64/);
cmp $len,#32
b.lo .Loop3x_ecb_enc
vld1.8 {$dat3},[$inp],#16
vld1.8 {$dat4},[$inp],#16
sub $len,$len,#32 // bias
mov $cnt,$rounds
.Loop5x_ecb_enc:
aese $dat0,q8
aesmc $dat0,$dat0
aese $dat1,q8
aesmc $dat1,$dat1
aese $dat2,q8
aesmc $dat2,$dat2
aese $dat3,q8
aesmc $dat3,$dat3
aese $dat4,q8
aesmc $dat4,$dat4
vld1.32 {q8},[$key_],#16
subs $cnt,$cnt,#2
aese $dat0,q9
aesmc $dat0,$dat0
aese $dat1,q9
aesmc $dat1,$dat1
aese $dat2,q9
aesmc $dat2,$dat2
aese $dat3,q9
aesmc $dat3,$dat3
aese $dat4,q9
aesmc $dat4,$dat4
vld1.32 {q9},[$key_],#16
b.gt .Loop5x_ecb_enc
aese $dat0,q8
aesmc $dat0,$dat0
aese $dat1,q8
aesmc $dat1,$dat1
aese $dat2,q8
aesmc $dat2,$dat2
aese $dat3,q8
aesmc $dat3,$dat3
aese $dat4,q8
aesmc $dat4,$dat4
cmp $len,#0x40 // because .Lecb_enc_tail4x
sub $len,$len,#0x50
aese $dat0,q9
aesmc $dat0,$dat0
aese $dat1,q9
aesmc $dat1,$dat1
aese $dat2,q9
aesmc $dat2,$dat2
aese $dat3,q9
aesmc $dat3,$dat3
aese $dat4,q9
aesmc $dat4,$dat4
csel x6,xzr,$len,gt // borrow x6, $cnt, "gt" is not typo
mov $key_,$key
aese $dat0,q10
aesmc $dat0,$dat0
aese $dat1,q10
aesmc $dat1,$dat1
aese $dat2,q10
aesmc $dat2,$dat2
aese $dat3,q10
aesmc $dat3,$dat3
aese $dat4,q10
aesmc $dat4,$dat4
add $inp,$inp,x6 // $inp is adjusted in such way that
// at exit from the loop $dat1-$dat4
// are loaded with last "words"
add x6,$len,#0x60 // because .Lecb_enc_tail4x
aese $dat0,q11
aesmc $dat0,$dat0
aese $dat1,q11
aesmc $dat1,$dat1
aese $dat2,q11
aesmc $dat2,$dat2
aese $dat3,q11
aesmc $dat3,$dat3
aese $dat4,q11
aesmc $dat4,$dat4
aese $dat0,q12
aesmc $dat0,$dat0
aese $dat1,q12
aesmc $dat1,$dat1
aese $dat2,q12
aesmc $dat2,$dat2
aese $dat3,q12
aesmc $dat3,$dat3
aese $dat4,q12
aesmc $dat4,$dat4
aese $dat0,q13
aesmc $dat0,$dat0
aese $dat1,q13
aesmc $dat1,$dat1
aese $dat2,q13
aesmc $dat2,$dat2
aese $dat3,q13
aesmc $dat3,$dat3
aese $dat4,q13
aesmc $dat4,$dat4
aese $dat0,q14
aesmc $dat0,$dat0
aese $dat1,q14
aesmc $dat1,$dat1
aese $dat2,q14
aesmc $dat2,$dat2
aese $dat3,q14
aesmc $dat3,$dat3
aese $dat4,q14
aesmc $dat4,$dat4
aese $dat0,q15
vld1.8 {$in0},[$inp],#16
aese $dat1,q15
vld1.8 {$in1},[$inp],#16
aese $dat2,q15
vld1.8 {$in2},[$inp],#16
aese $dat3,q15
vld1.8 {$in3},[$inp],#16
aese $dat4,q15
vld1.8 {$in4},[$inp],#16
cbz x6,.Lecb_enc_tail4x
vld1.32 {q8},[$key_],#16 // re-pre-load rndkey[0]
veor $tmp0,$rndlast,$dat0
vorr $dat0,$in0,$in0
veor $tmp1,$rndlast,$dat1
vorr $dat1,$in1,$in1
veor $tmp2,$rndlast,$dat2
vorr $dat2,$in2,$in2
veor $tmp3,$rndlast,$dat3
vorr $dat3,$in3,$in3
veor $tmp4,$rndlast,$dat4
vst1.8 {$tmp0},[$out],#16
vorr $dat4,$in4,$in4
vst1.8 {$tmp1},[$out],#16
mov $cnt,$rounds
vst1.8 {$tmp2},[$out],#16
vld1.32 {q9},[$key_],#16 // re-pre-load rndkey[1]
vst1.8 {$tmp3},[$out],#16
vst1.8 {$tmp4},[$out],#16
b.hs .Loop5x_ecb_enc
add $len,$len,#0x50
cbz $len,.Lecb_done
add $cnt,$rounds,#2
subs $len,$len,#0x30
vorr $dat0,$in2,$in2
vorr $dat1,$in3,$in3
vorr $dat2,$in4,$in4
b.lo .Lecb_enc_tail
b .Loop3x_ecb_enc
.align 4
.Lecb_enc_tail4x:
veor $tmp1,$rndlast,$dat1
veor $tmp2,$rndlast,$dat2
veor $tmp3,$rndlast,$dat3
veor $tmp4,$rndlast,$dat4
vst1.8 {$tmp1},[$out],#16
vst1.8 {$tmp2},[$out],#16
vst1.8 {$tmp3},[$out],#16
vst1.8 {$tmp4},[$out],#16
b .Lecb_done
.align 4
___
$code.=<<___;
.Loop3x_ecb_enc:
aese $dat0,q8
aesmc $dat0,$dat0
aese $dat1,q8
aesmc $dat1,$dat1
aese $dat2,q8
aesmc $dat2,$dat2
vld1.32 {q8},[$key_],#16
subs $cnt,$cnt,#2
aese $dat0,q9
aesmc $dat0,$dat0
aese $dat1,q9
aesmc $dat1,$dat1
aese $dat2,q9
aesmc $dat2,$dat2
vld1.32 {q9},[$key_],#16
b.gt .Loop3x_ecb_enc
aese $dat0,q8
aesmc $dat0,$dat0
aese $dat1,q8
aesmc $dat1,$dat1
aese $dat2,q8
aesmc $dat2,$dat2
subs $len,$len,#0x30
mov.lo x6,$len // x6, $cnt, is zero at this point
aese $dat0,q9
aesmc $dat0,$dat0
aese $dat1,q9
aesmc $dat1,$dat1
aese $dat2,q9
aesmc $dat2,$dat2
add $inp,$inp,x6 // $inp is adjusted in such way that
// at exit from the loop $dat1-$dat2
// are loaded with last "words"
mov $key_,$key
aese $dat0,q12
aesmc $dat0,$dat0
aese $dat1,q12
aesmc $dat1,$dat1
aese $dat2,q12
aesmc $dat2,$dat2
vld1.8 {$in0},[$inp],#16
aese $dat0,q13
aesmc $dat0,$dat0
aese $dat1,q13
aesmc $dat1,$dat1
aese $dat2,q13
aesmc $dat2,$dat2
vld1.8 {$in1},[$inp],#16
aese $dat0,q14
aesmc $dat0,$dat0
aese $dat1,q14
aesmc $dat1,$dat1
aese $dat2,q14
aesmc $dat2,$dat2
vld1.8 {$in2},[$inp],#16
aese $dat0,q15
aese $dat1,q15
aese $dat2,q15
vld1.32 {q8},[$key_],#16 // re-pre-load rndkey[0]
add $cnt,$rounds,#2
veor $tmp0,$rndlast,$dat0
veor $tmp1,$rndlast,$dat1
veor $dat2,$dat2,$rndlast
vld1.32 {q9},[$key_],#16 // re-pre-load rndkey[1]
vst1.8 {$tmp0},[$out],#16
vorr $dat0,$in0,$in0
vst1.8 {$tmp1},[$out],#16
vorr $dat1,$in1,$in1
vst1.8 {$dat2},[$out],#16
vorr $dat2,$in2,$in2
b.hs .Loop3x_ecb_enc
cmn $len,#0x30
b.eq .Lecb_done
nop
.Lecb_enc_tail:
aese $dat1,q8
aesmc $dat1,$dat1
aese $dat2,q8
aesmc $dat2,$dat2
vld1.32 {q8},[$key_],#16
subs $cnt,$cnt,#2
aese $dat1,q9
aesmc $dat1,$dat1
aese $dat2,q9
aesmc $dat2,$dat2
vld1.32 {q9},[$key_],#16
b.gt .Lecb_enc_tail
aese $dat1,q8
aesmc $dat1,$dat1
aese $dat2,q8
aesmc $dat2,$dat2
aese $dat1,q9
aesmc $dat1,$dat1
aese $dat2,q9
aesmc $dat2,$dat2
aese $dat1,q12
aesmc $dat1,$dat1
aese $dat2,q12
aesmc $dat2,$dat2
cmn $len,#0x20
aese $dat1,q13
aesmc $dat1,$dat1
aese $dat2,q13
aesmc $dat2,$dat2
aese $dat1,q14
aesmc $dat1,$dat1
aese $dat2,q14
aesmc $dat2,$dat2
aese $dat1,q15
aese $dat2,q15
b.eq .Lecb_enc_one
veor $tmp1,$rndlast,$dat1
veor $tmp2,$rndlast,$dat2
vst1.8 {$tmp1},[$out],#16
vst1.8 {$tmp2},[$out],#16
b .Lecb_done
.Lecb_enc_one:
veor $tmp1,$rndlast,$dat2
vst1.8 {$tmp1},[$out],#16
b .Lecb_done
___
$code.=<<___;
.align 5
.Lecb_dec:
vld1.8 {$dat1},[$inp],#16
subs $len,$len,#32 // bias
add $cnt,$rounds,#2
vorr $in1,$dat1,$dat1
vorr $dat2,$dat1,$dat1
vorr $dat1,$dat,$dat
b.lo .Lecb_dec_tail
vorr $dat1,$in1,$in1
vld1.8 {$dat2},[$inp],#16
___
$code.=<<___ if ($flavour =~ /64/);
cmp $len,#32
b.lo .Loop3x_ecb_dec
vld1.8 {$dat3},[$inp],#16
vld1.8 {$dat4},[$inp],#16
sub $len,$len,#32 // bias
mov $cnt,$rounds
.Loop5x_ecb_dec:
aesd $dat0,q8
aesimc $dat0,$dat0
aesd $dat1,q8
aesimc $dat1,$dat1
aesd $dat2,q8
aesimc $dat2,$dat2
aesd $dat3,q8
aesimc $dat3,$dat3
aesd $dat4,q8
aesimc $dat4,$dat4
vld1.32 {q8},[$key_],#16
subs $cnt,$cnt,#2
aesd $dat0,q9
aesimc $dat0,$dat0
aesd $dat1,q9
aesimc $dat1,$dat1
aesd $dat2,q9
aesimc $dat2,$dat2
aesd $dat3,q9
aesimc $dat3,$dat3
aesd $dat4,q9
aesimc $dat4,$dat4
vld1.32 {q9},[$key_],#16
b.gt .Loop5x_ecb_dec
aesd $dat0,q8
aesimc $dat0,$dat0
aesd $dat1,q8
aesimc $dat1,$dat1
aesd $dat2,q8
aesimc $dat2,$dat2
aesd $dat3,q8
aesimc $dat3,$dat3
aesd $dat4,q8
aesimc $dat4,$dat4
cmp $len,#0x40 // because .Lecb_tail4x
sub $len,$len,#0x50
aesd $dat0,q9
aesimc $dat0,$dat0
aesd $dat1,q9
aesimc $dat1,$dat1
aesd $dat2,q9
aesimc $dat2,$dat2
aesd $dat3,q9
aesimc $dat3,$dat3
aesd $dat4,q9
aesimc $dat4,$dat4
csel x6,xzr,$len,gt // borrow x6, $cnt, "gt" is not typo
mov $key_,$key
aesd $dat0,q10
aesimc $dat0,$dat0
aesd $dat1,q10
aesimc $dat1,$dat1
aesd $dat2,q10
aesimc $dat2,$dat2
aesd $dat3,q10
aesimc $dat3,$dat3
aesd $dat4,q10
aesimc $dat4,$dat4
add $inp,$inp,x6 // $inp is adjusted in such way that
// at exit from the loop $dat1-$dat4
// are loaded with last "words"
add x6,$len,#0x60 // because .Lecb_tail4x
aesd $dat0,q11
aesimc $dat0,$dat0
aesd $dat1,q11
aesimc $dat1,$dat1
aesd $dat2,q11
aesimc $dat2,$dat2
aesd $dat3,q11
aesimc $dat3,$dat3
aesd $dat4,q11
aesimc $dat4,$dat4
aesd $dat0,q12
aesimc $dat0,$dat0
aesd $dat1,q12
aesimc $dat1,$dat1
aesd $dat2,q12
aesimc $dat2,$dat2
aesd $dat3,q12
aesimc $dat3,$dat3
aesd $dat4,q12
aesimc $dat4,$dat4
aesd $dat0,q13
aesimc $dat0,$dat0
aesd $dat1,q13
aesimc $dat1,$dat1
aesd $dat2,q13
aesimc $dat2,$dat2
aesd $dat3,q13
aesimc $dat3,$dat3
aesd $dat4,q13
aesimc $dat4,$dat4
aesd $dat0,q14
aesimc $dat0,$dat0
aesd $dat1,q14
aesimc $dat1,$dat1
aesd $dat2,q14
aesimc $dat2,$dat2
aesd $dat3,q14
aesimc $dat3,$dat3
aesd $dat4,q14
aesimc $dat4,$dat4
aesd $dat0,q15
vld1.8 {$in0},[$inp],#16
aesd $dat1,q15
vld1.8 {$in1},[$inp],#16
aesd $dat2,q15
vld1.8 {$in2},[$inp],#16
aesd $dat3,q15
vld1.8 {$in3},[$inp],#16
aesd $dat4,q15
vld1.8 {$in4},[$inp],#16
cbz x6,.Lecb_tail4x
vld1.32 {q8},[$key_],#16 // re-pre-load rndkey[0]
veor $tmp0,$rndlast,$dat0
vorr $dat0,$in0,$in0
veor $tmp1,$rndlast,$dat1
vorr $dat1,$in1,$in1
veor $tmp2,$rndlast,$dat2
vorr $dat2,$in2,$in2
veor $tmp3,$rndlast,$dat3
vorr $dat3,$in3,$in3
veor $tmp4,$rndlast,$dat4
vst1.8 {$tmp0},[$out],#16
vorr $dat4,$in4,$in4
vst1.8 {$tmp1},[$out],#16
mov $cnt,$rounds
vst1.8 {$tmp2},[$out],#16
vld1.32 {q9},[$key_],#16 // re-pre-load rndkey[1]
vst1.8 {$tmp3},[$out],#16
vst1.8 {$tmp4},[$out],#16
b.hs .Loop5x_ecb_dec
add $len,$len,#0x50
cbz $len,.Lecb_done
add $cnt,$rounds,#2
subs $len,$len,#0x30
vorr $dat0,$in2,$in2
vorr $dat1,$in3,$in3
vorr $dat2,$in4,$in4
b.lo .Lecb_dec_tail
b .Loop3x_ecb_dec
.align 4
.Lecb_tail4x:
veor $tmp1,$rndlast,$dat1
veor $tmp2,$rndlast,$dat2
veor $tmp3,$rndlast,$dat3
veor $tmp4,$rndlast,$dat4
vst1.8 {$tmp1},[$out],#16
vst1.8 {$tmp2},[$out],#16
vst1.8 {$tmp3},[$out],#16
vst1.8 {$tmp4},[$out],#16
b .Lecb_done
.align 4
___
$code.=<<___;
.Loop3x_ecb_dec:
aesd $dat0,q8
aesimc $dat0,$dat0
aesd $dat1,q8
aesimc $dat1,$dat1
aesd $dat2,q8
aesimc $dat2,$dat2
vld1.32 {q8},[$key_],#16
subs $cnt,$cnt,#2
aesd $dat0,q9
aesimc $dat0,$dat0
aesd $dat1,q9
aesimc $dat1,$dat1
aesd $dat2,q9
aesimc $dat2,$dat2
vld1.32 {q9},[$key_],#16
b.gt .Loop3x_ecb_dec
aesd $dat0,q8
aesimc $dat0,$dat0
aesd $dat1,q8
aesimc $dat1,$dat1
aesd $dat2,q8
aesimc $dat2,$dat2
subs $len,$len,#0x30
mov.lo x6,$len // x6, $cnt, is zero at this point
aesd $dat0,q9
aesimc $dat0,$dat0
aesd $dat1,q9
aesimc $dat1,$dat1
aesd $dat2,q9
aesimc $dat2,$dat2
add $inp,$inp,x6 // $inp is adjusted in such way that
// at exit from the loop $dat1-$dat2
// are loaded with last "words"
mov $key_,$key
aesd $dat0,q12
aesimc $dat0,$dat0
aesd $dat1,q12
aesimc $dat1,$dat1
aesd $dat2,q12
aesimc $dat2,$dat2
vld1.8 {$in0},[$inp],#16
aesd $dat0,q13
aesimc $dat0,$dat0
aesd $dat1,q13
aesimc $dat1,$dat1
aesd $dat2,q13
aesimc $dat2,$dat2
vld1.8 {$in1},[$inp],#16
aesd $dat0,q14
aesimc $dat0,$dat0
aesd $dat1,q14
aesimc $dat1,$dat1
aesd $dat2,q14
aesimc $dat2,$dat2
vld1.8 {$in2},[$inp],#16
aesd $dat0,q15
aesd $dat1,q15
aesd $dat2,q15
vld1.32 {q8},[$key_],#16 // re-pre-load rndkey[0]
add $cnt,$rounds,#2
veor $tmp0,$rndlast,$dat0
veor $tmp1,$rndlast,$dat1
veor $dat2,$dat2,$rndlast
vld1.32 {q9},[$key_],#16 // re-pre-load rndkey[1]
vst1.8 {$tmp0},[$out],#16
vorr $dat0,$in0,$in0
vst1.8 {$tmp1},[$out],#16
vorr $dat1,$in1,$in1
vst1.8 {$dat2},[$out],#16
vorr $dat2,$in2,$in2
b.hs .Loop3x_ecb_dec
cmn $len,#0x30
b.eq .Lecb_done
nop
.Lecb_dec_tail:
aesd $dat1,q8
aesimc $dat1,$dat1
aesd $dat2,q8
aesimc $dat2,$dat2
vld1.32 {q8},[$key_],#16
subs $cnt,$cnt,#2
aesd $dat1,q9
aesimc $dat1,$dat1
aesd $dat2,q9
aesimc $dat2,$dat2
vld1.32 {q9},[$key_],#16
b.gt .Lecb_dec_tail
aesd $dat1,q8
aesimc $dat1,$dat1
aesd $dat2,q8
aesimc $dat2,$dat2
aesd $dat1,q9
aesimc $dat1,$dat1
aesd $dat2,q9
aesimc $dat2,$dat2
aesd $dat1,q12
aesimc $dat1,$dat1
aesd $dat2,q12
aesimc $dat2,$dat2
cmn $len,#0x20
aesd $dat1,q13
aesimc $dat1,$dat1
aesd $dat2,q13
aesimc $dat2,$dat2
aesd $dat1,q14
aesimc $dat1,$dat1
aesd $dat2,q14
aesimc $dat2,$dat2
aesd $dat1,q15
aesd $dat2,q15
b.eq .Lecb_dec_one
veor $tmp1,$rndlast,$dat1
veor $tmp2,$rndlast,$dat2
vst1.8 {$tmp1},[$out],#16
vst1.8 {$tmp2},[$out],#16
b .Lecb_done
.Lecb_dec_one:
veor $tmp1,$rndlast,$dat2
vst1.8 {$tmp1},[$out],#16
.Lecb_done:
___
}
$code.=<<___ if ($flavour !~ /64/);
vldmia sp!,{d8-d15}
ldmia sp!,{r4-r8,pc}
___
$code.=<<___ if ($flavour =~ /64/);
ldr x29,[sp],#16
___
$code.=<<___ if ($flavour =~ /64/);
.Lecb_Final_abort:
ret
___
$code.=<<___;
.size ${prefix}_ecb_encrypt,.-${prefix}_ecb_encrypt
___
}}}
{{{
my ($inp,$out,$len,$key,$ivp)=map("x$_",(0..4)); my $enc="w5";
my ($rounds,$cnt,$key_,$step,$step1)=($enc,"w6","x7","x8","x12");
crypto/aes/asm/bsaes-x86_64.pl
+1 -1
View File
@@ -27,7 +27,7 @@
# - code was made position-independent;
# - rounds were folded into a loop resulting in >5x size reduction
# from 12.5KB to 2.2KB;
# - above was possibile thanks to mixcolumns() modification that
# - above was possible thanks to mixcolumns() modification that
# allowed to feed its output back to aesenc[last], this was
# achieved at cost of two additional inter-registers moves;
# - some instruction reordering and interleaving;