Hakase/openssl-custom
1
0 Fork 0
Code Issues Pull Requests Releases Wiki Activity

Latest update

Browse Source
This commit is contained in:
Hakase
2019-02-22 12:53:31 +09:00
parent 6523a76e2e
commit 25fbda8e8b
186 changed files with 9758 additions and 1640 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crypto/aes/asm/aes-armv4.pl
+2 -1
View File
@@ -76,7 +76,6 @@ $code=<<___;
# define __ARM_ARCH__ __LINUX_ARM_ARCH__
#endif
.text
#if defined(__thumb2__) && !defined(__APPLE__)
.syntax unified
.thumb
@@ -85,6 +84,8 @@ $code=<<___;
#undef __thumb2__
#endif
.text
.type AES_Te,%object
.align 5
AES_Te:
crypto/aes/asm/aes-mips.pl
+85 -49
View File
@@ -1,5 +1,5 @@
#! /usr/bin/env perl
# Copyright 2010-2018 The OpenSSL Project Authors. All Rights Reserved.
# Copyright 2010-2019 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the Apache License 2.0 (the "License"). You may not use
# this file except in compliance with the License. You can obtain a copy
@@ -34,6 +34,11 @@
# instead, code path is chosen upon pre-process time, pass -mips32r2
# or/and -msmartmips.
# February 2019
#
# Normalize MIPS32R2 AES table address calculation by always using EXT
# instruction. This reduces the standard codebase by another 10%.
######################################################################
# There is a number of MIPS ABI in use, O32 and N32/64 are most
# widely used. Then there is a new contender: NUBI. It appears that if
@@ -223,6 +228,33 @@ _mips_AES_encrypt:
ext $i0,$s1,16,8
_xtr $i0,$s1,16-2
#else
#if defined(_MIPS_ARCH_MIPS32R2) || defined(_MIPS_ARCH_MIPS64R2)
move $i0,$Tbl
move $i1,$Tbl
move $i2,$Tbl
move $i3,$Tbl
ext $t0,$s1,16,8
.Loop_enc:
ext $t1,$s2,16,8
ext $t2,$s3,16,8
ext $t3,$s0,16,8
$PTR_INS $i0,$t0,2,8
$PTR_INS $i1,$t1,2,8
$PTR_INS $i2,$t2,2,8
$PTR_INS $i3,$t3,2,8
lw $t0,0($i0) # Te1[s1>>16]
ext $t4,$s2,8,8
lw $t1,0($i1) # Te1[s2>>16]
ext $t5,$s3,8,8
lw $t2,0($i2) # Te1[s3>>16]
ext $t6,$s0,8,8
lw $t3,0($i3) # Te1[s0>>16]
ext $t7,$s1,8,8
$PTR_INS $i0,$t4,2,8
$PTR_INS $i1,$t5,2,8
$PTR_INS $i2,$t6,2,8
$PTR_INS $i3,$t7,2,8
#else
_xtr $i0,$s1,16-2
.Loop_enc:
@@ -237,16 +269,6 @@ _mips_AES_encrypt:
$PTR_ADD $i1,$Tbl
$PTR_ADD $i2,$Tbl
$PTR_ADD $i3,$Tbl
#if defined(_MIPS_ARCH_MIPS32R2) || defined(_MIPS_ARCH_MIPS64R2)
lw $t0,0($i0) # Te1[s1>>16]
_xtr $i0,$s2,8-2
lw $t1,0($i1) # Te1[s2>>16]
_xtr $i1,$s3,8-2
lw $t2,0($i2) # Te1[s3>>16]
_xtr $i2,$s0,8-2
lw $t3,0($i3) # Te1[s0>>16]
_xtr $i3,$s1,8-2
#else
lwl $t0,3($i0) # Te1[s1>>16]
lwl $t1,3($i1) # Te1[s2>>16]
lwl $t2,3($i2) # Te1[s3>>16]
@@ -259,7 +281,6 @@ _mips_AES_encrypt:
_xtr $i2,$s0,8-2
lwr $t3,2($i3) # Te1[s0>>16]
_xtr $i3,$s1,8-2
#endif
and $i0,0x3fc
and $i1,0x3fc
and $i2,0x3fc
@@ -268,6 +289,7 @@ _mips_AES_encrypt:
$PTR_ADD $i1,$Tbl
$PTR_ADD $i2,$Tbl
$PTR_ADD $i3,$Tbl
#endif
#if defined(_MIPS_ARCH_MIPS32R2) || defined(_MIPS_ARCH_MIPS64R2)
rotr $t0,$t0,8
rotr $t1,$t1,8
@@ -275,22 +297,18 @@ _mips_AES_encrypt:
rotr $t3,$t3,8
# if defined(_MIPSEL)
lw $t4,0($i0) # Te2[s2>>8]
_xtr $i0,$s3,0-2
ext $t8,$s3,0,8
lw $t5,0($i1) # Te2[s3>>8]
_xtr $i1,$s0,0-2
ext $t9,$s0,0,8
lw $t6,0($i2) # Te2[s0>>8]
_xtr $i2,$s1,0-2
ext $t10,$s1,0,8
lw $t7,0($i3) # Te2[s1>>8]
_xtr $i3,$s2,0-2
ext $t11,$s2,0,8
$PTR_INS $i0,$t8,2,8
$PTR_INS $i1,$t9,2,8
$PTR_INS $i2,$t10,2,8
$PTR_INS $i3,$t11,2,8
and $i0,0x3fc
and $i1,0x3fc
and $i2,0x3fc
and $i3,0x3fc
$PTR_ADD $i0,$Tbl
$PTR_ADD $i1,$Tbl
$PTR_ADD $i2,$Tbl
$PTR_ADD $i3,$Tbl
lw $t8,0($i0) # Te3[s3]
$PTR_INS $i0,$s0,2,8
lw $t9,0($i1) # Te3[s0]
@@ -411,6 +429,9 @@ _mips_AES_encrypt:
xor $s3,$t3
.set noreorder
bnez $cnt,.Loop_enc
#if defined(_MIPS_ARCH_MIPS32R2) || defined(_MIPS_ARCH_MIPS64R2)
ext $t0,$s1,16,8
#endif
_xtr $i0,$s1,16-2
#endif
@@ -811,6 +832,33 @@ _mips_AES_decrypt:
ext $i0,$s3,16,8
_xtr $i0,$s3,16-2
#else
#if defined(_MIPS_ARCH_MIPS32R2) || defined(_MIPS_ARCH_MIPS64R2)
move $i0,$Tbl
move $i1,$Tbl
move $i2,$Tbl
move $i3,$Tbl
ext $t0,$s3,16,8
.Loop_dec:
ext $t1,$s0,16,8
ext $t2,$s1,16,8
ext $t3,$s2,16,8
$PTR_INS $i0,$t0,2,8
$PTR_INS $i1,$t1,2,8
$PTR_INS $i2,$t2,2,8
$PTR_INS $i3,$t3,2,8
lw $t0,0($i0) # Td1[s3>>16]
ext $t4,$s2,8,8
lw $t1,0($i1) # Td1[s0>>16]
ext $t5,$s3,8,8
lw $t2,0($i2) # Td1[s1>>16]
ext $t6,$s0,8,8
lw $t3,0($i3) # Td1[s2>>16]
ext $t7,$s1,8,8
$PTR_INS $i0,$t4,2,8
$PTR_INS $i1,$t5,2,8
$PTR_INS $i2,$t6,2,8
$PTR_INS $i3,$t7,2,8
#else
_xtr $i0,$s3,16-2
.Loop_dec:
@@ -825,16 +873,6 @@ _mips_AES_decrypt:
$PTR_ADD $i1,$Tbl
$PTR_ADD $i2,$Tbl
$PTR_ADD $i3,$Tbl
#if defined(_MIPS_ARCH_MIPS32R2) || defined(_MIPS_ARCH_MIPS64R2)
lw $t0,0($i0) # Td1[s3>>16]
_xtr $i0,$s2,8-2
lw $t1,0($i1) # Td1[s0>>16]
_xtr $i1,$s3,8-2
lw $t2,0($i2) # Td1[s1>>16]
_xtr $i2,$s0,8-2
lw $t3,0($i3) # Td1[s2>>16]
_xtr $i3,$s1,8-2
#else
lwl $t0,3($i0) # Td1[s3>>16]
lwl $t1,3($i1) # Td1[s0>>16]
lwl $t2,3($i2) # Td1[s1>>16]
@@ -847,8 +885,6 @@ _mips_AES_decrypt:
_xtr $i2,$s0,8-2
lwr $t3,2($i3) # Td1[s2>>16]
_xtr $i3,$s1,8-2
#endif
and $i0,0x3fc
and $i1,0x3fc
and $i2,0x3fc
@@ -857,6 +893,7 @@ _mips_AES_decrypt:
$PTR_ADD $i1,$Tbl
$PTR_ADD $i2,$Tbl
$PTR_ADD $i3,$Tbl
#endif
#if defined(_MIPS_ARCH_MIPS32R2) || defined(_MIPS_ARCH_MIPS64R2)
rotr $t0,$t0,8
rotr $t1,$t1,8
@@ -864,22 +901,17 @@ _mips_AES_decrypt:
rotr $t3,$t3,8
# if defined(_MIPSEL)
lw $t4,0($i0) # Td2[s2>>8]
_xtr $i0,$s1,0-2
ext $t8,$s1,0,8
lw $t5,0($i1) # Td2[s3>>8]
_xtr $i1,$s2,0-2
ext $t9,$s2,0,8
lw $t6,0($i2) # Td2[s0>>8]
_xtr $i2,$s3,0-2
ext $t10,$s3,0,8
lw $t7,0($i3) # Td2[s1>>8]
_xtr $i3,$s0,0-2
and $i0,0x3fc
and $i1,0x3fc
and $i2,0x3fc
and $i3,0x3fc
$PTR_ADD $i0,$Tbl
$PTR_ADD $i1,$Tbl
$PTR_ADD $i2,$Tbl
$PTR_ADD $i3,$Tbl
ext $t11,$s0,0,8
$PTR_INS $i0,$t8,2,8
$PTR_INS $i1,$t9,2,8
$PTR_INS $i2,$t10,2,8
$PTR_INS $i3,$t11,2,8
lw $t8,0($i0) # Td3[s1]
$PTR_INS $i0,$s0,2,8
lw $t9,0($i1) # Td3[s2]
@@ -1001,6 +1033,10 @@ _mips_AES_decrypt:
xor $s3,$t3
.set noreorder
bnez $cnt,.Loop_dec
#if defined(_MIPS_ARCH_MIPS32R2) || defined(_MIPS_ARCH_MIPS64R2)
ext $t0,$s3,16,8
#endif
_xtr $i0,$s3,16-2
#endif
crypto/aes/asm/aes-x86_64.pl
+16 -2
View File
@@ -554,6 +554,7 @@ $code.=<<___;
.type _x86_64_AES_encrypt_compact,\@abi-omnipotent
.align 16
_x86_64_AES_encrypt_compact:
.cfi_startproc
lea 128($sbox),$inp # size optimization
mov 0-128($inp),$acc1 # prefetch Te4
mov 32-128($inp),$acc2
@@ -587,6 +588,7 @@ $code.=<<___;
xor 8($key),$s2
xor 12($key),$s3
.byte 0xf3,0xc3 # rep ret
.cfi_endproc
.size _x86_64_AES_encrypt_compact,.-_x86_64_AES_encrypt_compact
___
@@ -1161,6 +1163,7 @@ $code.=<<___;
.type _x86_64_AES_decrypt_compact,\@abi-omnipotent
.align 16
_x86_64_AES_decrypt_compact:
.cfi_startproc
lea 128($sbox),$inp # size optimization
mov 0-128($inp),$acc1 # prefetch Td4
mov 32-128($inp),$acc2
@@ -1203,6 +1206,7 @@ $code.=<<___;
xor 8($key),$s2
xor 12($key),$s3
.byte 0xf3,0xc3 # rep ret
.cfi_endproc
.size _x86_64_AES_decrypt_compact,.-_x86_64_AES_decrypt_compact
___
@@ -1365,6 +1369,7 @@ AES_set_encrypt_key:
.type _x86_64_AES_set_encrypt_key,\@abi-omnipotent
.align 16
_x86_64_AES_set_encrypt_key:
.cfi_startproc
mov %esi,%ecx # %ecx=bits
mov %rdi,%rsi # %rsi=userKey
mov %rdx,%rdi # %rdi=key
@@ -1546,6 +1551,7 @@ $code.=<<___;
mov \$-1,%rax
.Lexit:
.byte 0xf3,0xc3 # rep ret
.cfi_endproc
.size _x86_64_AES_set_encrypt_key,.-_x86_64_AES_set_encrypt_key
___
@@ -1728,7 +1734,9 @@ AES_cbc_encrypt:
cmp \$0,%rdx # check length
je .Lcbc_epilogue
pushfq
.cfi_push 49 # %rflags
# This could be .cfi_push 49, but libunwind fails on registers it does not
# recognize. See https://bugzilla.redhat.com/show_bug.cgi?id=217087.
.cfi_adjust_cfa_offset 8
push %rbx
.cfi_push %rbx
push %rbp
@@ -1751,6 +1759,7 @@ AES_cbc_encrypt:
cmp \$0,%r9
cmoveq %r10,$sbox
.cfi_remember_state
mov OPENSSL_ia32cap_P(%rip),%r10d
cmp \$$speed_limit,%rdx
jb .Lcbc_slow_prologue
@@ -1986,6 +1995,7 @@ AES_cbc_encrypt:
#--------------------------- SLOW ROUTINE ---------------------------#
.align 16
.Lcbc_slow_prologue:
.cfi_restore_state
# allocate aligned stack frame...
lea -88(%rsp),%rbp
and \$-64,%rbp
@@ -1997,8 +2007,10 @@ AES_cbc_encrypt:
sub %r10,%rbp
xchg %rsp,%rbp
.cfi_def_cfa_register %rbp
#add \$8,%rsp # reserve for return address!
mov %rbp,$_rsp # save %rsp
.cfi_cfa_expression $_rsp,deref,+64
.Lcbc_slow_body:
#mov %rdi,$_inp # save copy of inp
#mov %rsi,$_out # save copy of out
@@ -2187,7 +2199,9 @@ AES_cbc_encrypt:
.cfi_def_cfa %rsp,16
.Lcbc_popfq:
popfq
.cfi_pop 49 # %rflags
# This could be .cfi_pop 49, but libunwind fails on registers it does not
# recognize. See https://bugzilla.redhat.com/show_bug.cgi?id=217087.
.cfi_adjust_cfa_offset -8
.Lcbc_epilogue:
ret
.cfi_endproc
crypto/aes/asm/aesni-x86_64.pl
+16
View File
@@ -274,6 +274,7 @@ $code.=<<___;
.type ${PREFIX}_encrypt,\@abi-omnipotent
.align 16
${PREFIX}_encrypt:
.cfi_startproc
movups ($inp),$inout0 # load input
mov 240($key),$rounds # key->rounds
___
@@ -284,12 +285,14 @@ $code.=<<___;
movups $inout0,($out) # output
pxor $inout0,$inout0
ret
.cfi_endproc
.size ${PREFIX}_encrypt,.-${PREFIX}_encrypt
.globl ${PREFIX}_decrypt
.type ${PREFIX}_decrypt,\@abi-omnipotent
.align 16
${PREFIX}_decrypt:
.cfi_startproc
movups ($inp),$inout0 # load input
mov 240($key),$rounds # key->rounds
___
@@ -300,6 +303,7 @@ $code.=<<___;
movups $inout0,($out) # output
pxor $inout0,$inout0
ret
.cfi_endproc
.size ${PREFIX}_decrypt, .-${PREFIX}_decrypt
___
}
@@ -325,6 +329,7 @@ $code.=<<___;
.type _aesni_${dir}rypt2,\@abi-omnipotent
.align 16
_aesni_${dir}rypt2:
.cfi_startproc
$movkey ($key),$rndkey0
shl \$4,$rounds
$movkey 16($key),$rndkey1
@@ -350,6 +355,7 @@ _aesni_${dir}rypt2:
aes${dir}last $rndkey0,$inout0
aes${dir}last $rndkey0,$inout1
ret
.cfi_endproc
.size _aesni_${dir}rypt2,.-_aesni_${dir}rypt2
___
}
@@ -361,6 +367,7 @@ $code.=<<___;
.type _aesni_${dir}rypt3,\@abi-omnipotent
.align 16
_aesni_${dir}rypt3:
.cfi_startproc
$movkey ($key),$rndkey0
shl \$4,$rounds
$movkey 16($key),$rndkey1
@@ -391,6 +398,7 @@ _aesni_${dir}rypt3:
aes${dir}last $rndkey0,$inout1
aes${dir}last $rndkey0,$inout2
ret
.cfi_endproc
.size _aesni_${dir}rypt3,.-_aesni_${dir}rypt3
___
}
@@ -406,6 +414,7 @@ $code.=<<___;
.type _aesni_${dir}rypt4,\@abi-omnipotent
.align 16
_aesni_${dir}rypt4:
.cfi_startproc
$movkey ($key),$rndkey0
shl \$4,$rounds
$movkey 16($key),$rndkey1
@@ -442,6 +451,7 @@ _aesni_${dir}rypt4:
aes${dir}last $rndkey0,$inout2
aes${dir}last $rndkey0,$inout3
ret
.cfi_endproc
.size _aesni_${dir}rypt4,.-_aesni_${dir}rypt4
___
}
@@ -453,6 +463,7 @@ $code.=<<___;
.type _aesni_${dir}rypt6,\@abi-omnipotent
.align 16
_aesni_${dir}rypt6:
.cfi_startproc
$movkey ($key),$rndkey0
shl \$4,$rounds
$movkey 16($key),$rndkey1
@@ -503,6 +514,7 @@ _aesni_${dir}rypt6:
aes${dir}last $rndkey0,$inout4
aes${dir}last $rndkey0,$inout5
ret
.cfi_endproc
.size _aesni_${dir}rypt6,.-_aesni_${dir}rypt6
___
}
@@ -514,6 +526,7 @@ $code.=<<___;
.type _aesni_${dir}rypt8,\@abi-omnipotent
.align 16
_aesni_${dir}rypt8:
.cfi_startproc
$movkey ($key),$rndkey0
shl \$4,$rounds
$movkey 16($key),$rndkey1
@@ -574,6 +587,7 @@ _aesni_${dir}rypt8:
aes${dir}last $rndkey0,$inout6
aes${dir}last $rndkey0,$inout7
ret
.cfi_endproc
.size _aesni_${dir}rypt8,.-_aesni_${dir}rypt8
___
}
@@ -598,6 +612,7 @@ $code.=<<___;
.type aesni_ecb_encrypt,\@function,5
.align 16
aesni_ecb_encrypt:
.cfi_startproc
___
$code.=<<___ if ($win64);
lea -0x58(%rsp),%rsp
@@ -943,6 +958,7 @@ $code.=<<___ if ($win64);
___
$code.=<<___;
ret
.cfi_endproc
.size aesni_ecb_encrypt,.-aesni_ecb_encrypt
___
crypto/aes/asm/aesv8-armx.pl
+20 -6
View File
@@ -53,18 +53,27 @@ open OUT,"| \"$^X\" $xlate $flavour $output";
$prefix="aes_v8";
$_byte = ($flavour =~ /win/ ? "DCB" : ".byte");
$code=<<___;
#include "arm_arch.h"
#if __ARM_MAX_ARCH__>=7
.text
___
$code.=".arch armv8-a+crypto\n" if ($flavour =~ /64/);
$code.=".arch armv8-a+crypto\n.text\n" if ($flavour =~ /64/);
$code.=<<___ if ($flavour !~ /64/);
.arch armv7-a // don't confuse not-so-latest binutils with argv8 :-)
.fpu neon
#ifdef __thumb2__
.syntax unified
.thumb
# define INST(a,b,c,d) $_byte c,d|0xc,a,b
#else
.code 32
#undef __thumb2__
# define INST(a,b,c,d) $_byte a,b,c,d
#endif
.text
___
# Assembler mnemonics are an eclectic mix of 32- and 64-bit syntax,
@@ -262,6 +271,7 @@ $code.=<<___;
${prefix}_set_decrypt_key:
___
$code.=<<___ if ($flavour =~ /64/);
.inst 0xd503233f // paciasp
stp x29,x30,[sp,#-16]!
add x29,sp,#0
___
@@ -305,6 +315,7 @@ $code.=<<___ if ($flavour !~ /64/);
___
$code.=<<___ if ($flavour =~ /64/);
ldp x29,x30,[sp],#16
.inst 0xd50323bf // autiasp
ret
___
$code.=<<___;
@@ -953,7 +964,7 @@ if ($flavour =~ /64/) { ######## 64-bit code
# since ARMv7 instructions are always encoded little-endian.
# correct solution is to use .inst directive, but older
# assemblers don't implement it:-(
sprintf ".byte\t0x%02x,0x%02x,0x%02x,0x%02x\t@ %s %s",
sprintf "INST(0x%02x,0x%02x,0x%02x,0x%02x)\t@ %s %s",
$word&0xff,($word>>8)&0xff,
($word>>16)&0xff,($word>>24)&0xff,
$mnemonic,$arg;
@@ -994,14 +1005,17 @@ if ($flavour =~ /64/) { ######## 64-bit code
s/\],#[0-9]+/]!/o;
s/[v]?(aes\w+)\s+([qv].*)/unaes($1,$2)/geo or
s/cclr\s+([^,]+),\s*([a-z]+)/mov$2 $1,#0/o or
s/cclr\s+([^,]+),\s*([a-z]+)/mov.$2 $1,#0/o or
s/vtbl\.8\s+(.*)/unvtbl($1)/geo or
s/vdup\.32\s+(.*)/unvdup32($1)/geo or
s/vmov\.32\s+(.*)/unvmov32($1)/geo or
s/^(\s+)b\./$1b/o or
s/^(\s+)mov\./$1mov/o or
s/^(\s+)ret/$1bx\tlr/o;
if (s/^(\s+)mov\.([a-z]+)/$1mov$2/) {
print " it $2\n";
}
print $_,"\n";
}
}
crypto/aes/asm/bsaes-armv7.pl
+4 -3
View File
@@ -728,7 +728,6 @@ $code.=<<___;
.arch armv7-a
.fpu neon
.text
.syntax unified @ ARMv7-capable assembler is expected to handle this
#if defined(__thumb2__) && !defined(__APPLE__)
.thumb
@@ -737,6 +736,8 @@ $code.=<<___;
# undef __thumb2__
#endif
.text
.type _bsaes_decrypt8,%function
.align 4
_bsaes_decrypt8:
@@ -1125,9 +1126,9 @@ bsaes_cbc_encrypt:
#ifndef __thumb__
blo AES_cbc_encrypt
#else
bhs 1f
bhs .Lcbc_do_bsaes
b AES_cbc_encrypt
1:
.Lcbc_do_bsaes:
#endif
#endif
crypto/aes/asm/bsaes-x86_64.pl
+6
View File
@@ -816,6 +816,7 @@ $code.=<<___;
.type _bsaes_encrypt8,\@abi-omnipotent
.align 64
_bsaes_encrypt8:
.cfi_startproc
lea .LBS0(%rip), $const # constants table
movdqa ($key), @XMM[9] # round 0 key
@@ -875,11 +876,13 @@ $code.=<<___;
pxor @XMM[8], @XMM[0]
pxor @XMM[8], @XMM[1]
ret
.cfi_endproc
.size _bsaes_encrypt8,.-_bsaes_encrypt8
.type _bsaes_decrypt8,\@abi-omnipotent
.align 64
_bsaes_decrypt8:
.cfi_startproc
lea .LBS0(%rip), $const # constants table
movdqa ($key), @XMM[9] # round 0 key
@@ -937,6 +940,7 @@ $code.=<<___;
pxor @XMM[8], @XMM[0]
pxor @XMM[8], @XMM[1]
ret
.cfi_endproc
.size _bsaes_decrypt8,.-_bsaes_decrypt8
___
}
@@ -971,6 +975,7 @@ $code.=<<___;
.type _bsaes_key_convert,\@abi-omnipotent
.align 16
_bsaes_key_convert:
.cfi_startproc
lea .Lmasks(%rip), $const
movdqu ($inp), %xmm7 # load round 0 key
lea 0x10($inp), $inp
@@ -1049,6 +1054,7 @@ _bsaes_key_convert:
movdqa 0x50($const), %xmm7 # .L63
#movdqa %xmm6, ($out) # don't save last round key
ret
.cfi_endproc
.size _bsaes_key_convert,.-_bsaes_key_convert
___
}
crypto/aes/asm/vpaes-armv8.pl
+156 -138
View File
@@ -150,12 +150,12 @@ my ($sb1u,$sb1t,$sb2u,$sb2t) = map("v$_.16b",(24..27));
my ($sb9u,$sb9t,$sbdu,$sbdt,$sbbu,$sbbt,$sbeu,$sbet)=map("v$_.16b",(24..31));
$code.=<<___;
##
## _aes_preheat
##
## Fills register %r10 -> .aes_consts (so you can -fPIC)
## and %xmm9-%xmm15 as specified below.
##
//
// _aes_preheat
//
// Fills register %r10 -> .aes_consts (so you can -fPIC)
// and %xmm9-%xmm15 as specified below.
//
.type _vpaes_encrypt_preheat,%function
.align 4
_vpaes_encrypt_preheat:
@@ -167,21 +167,21 @@ _vpaes_encrypt_preheat:
ret
.size _vpaes_encrypt_preheat,.-_vpaes_encrypt_preheat
##
## _aes_encrypt_core
##
## AES-encrypt %xmm0.
##
## Inputs:
## %xmm0 = input
## %xmm9-%xmm15 as in _vpaes_preheat
## (%rdx) = scheduled keys
##
## Output in %xmm0
## Clobbers %xmm1-%xmm5, %r9, %r10, %r11, %rax
## Preserves %xmm6 - %xmm8 so you get some local vectors
##
##
//
// _aes_encrypt_core
//
// AES-encrypt %xmm0.
//
// Inputs:
// %xmm0 = input
// %xmm9-%xmm15 as in _vpaes_preheat
// (%rdx) = scheduled keys
//
// Output in %xmm0
// Clobbers %xmm1-%xmm5, %r9, %r10, %r11, %rax
// Preserves %xmm6 - %xmm8 so you get some local vectors
//
//
.type _vpaes_encrypt_core,%function
.align 4
_vpaes_encrypt_core:
@@ -255,6 +255,7 @@ _vpaes_encrypt_core:
.type vpaes_encrypt,%function
.align 4
vpaes_encrypt:
.inst 0xd503233f // paciasp
stp x29,x30,[sp,#-16]!
add x29,sp,#0
@@ -264,6 +265,7 @@ vpaes_encrypt:
st1 {v0.16b}, [$out]
ldp x29,x30,[sp],#16
.inst 0xd50323bf // autiasp
ret
.size vpaes_encrypt,.-vpaes_encrypt
@@ -385,11 +387,11 @@ _vpaes_decrypt_preheat:
ret
.size _vpaes_decrypt_preheat,.-_vpaes_decrypt_preheat
##
## Decryption core
##
## Same API as encryption core.
##
//
// Decryption core
//
// Same API as encryption core.
//
.type _vpaes_decrypt_core,%function
.align 4
_vpaes_decrypt_core:
@@ -486,6 +488,7 @@ _vpaes_decrypt_core:
.type vpaes_decrypt,%function
.align 4
vpaes_decrypt:
.inst 0xd503233f // paciasp
stp x29,x30,[sp,#-16]!
add x29,sp,#0
@@ -495,6 +498,7 @@ vpaes_decrypt:
st1 {v0.16b}, [$out]
ldp x29,x30,[sp],#16
.inst 0xd50323bf // autiasp
ret
.size vpaes_decrypt,.-vpaes_decrypt
@@ -639,11 +643,11 @@ my ($inp,$bits,$out,$dir)=("x0","w1","x2","w3");
my ($invlo,$invhi,$iptlo,$ipthi,$rcon) = map("v$_.16b",(18..21,8));
$code.=<<___;
########################################################
## ##
## AES key schedule ##
## ##
########################################################
////////////////////////////////////////////////////////
// //
// AES key schedule //
// //
////////////////////////////////////////////////////////
.type _vpaes_key_preheat,%function
.align 4
_vpaes_key_preheat:
@@ -665,6 +669,7 @@ _vpaes_key_preheat:
.type _vpaes_schedule_core,%function
.align 4
_vpaes_schedule_core:
.inst 0xd503233f // paciasp
stp x29, x30, [sp,#-16]!
add x29,sp,#0
@@ -698,14 +703,14 @@ _vpaes_schedule_core:
b.eq .Lschedule_192
// 128: fall though
##
## .schedule_128
##
## 128-bit specific part of key schedule.
##
## This schedule is really simple, because all its parts
## are accomplished by the subroutines.
##
//
// .schedule_128
//
// 128-bit specific part of key schedule.
//
// This schedule is really simple, because all its parts
// are accomplished by the subroutines.
//
.Lschedule_128:
mov $inp, #10 // mov \$10, %esi
@@ -716,21 +721,21 @@ _vpaes_schedule_core:
bl _vpaes_schedule_mangle // write output
b .Loop_schedule_128
##
## .aes_schedule_192
##
## 192-bit specific part of key schedule.
##
## The main body of this schedule is the same as the 128-bit
## schedule, but with more smearing. The long, high side is
## stored in %xmm7 as before, and the short, low side is in
## the high bits of %xmm6.
##
## This schedule is somewhat nastier, however, because each
## round produces 192 bits of key material, or 1.5 round keys.
## Therefore, on each cycle we do 2 rounds and produce 3 round
## keys.
##
//
// .aes_schedule_192
//
// 192-bit specific part of key schedule.
//
// The main body of this schedule is the same as the 128-bit
// schedule, but with more smearing. The long, high side is
// stored in %xmm7 as before, and the short, low side is in
// the high bits of %xmm6.
//
// This schedule is somewhat nastier, however, because each
// round produces 192 bits of key material, or 1.5 round keys.
// Therefore, on each cycle we do 2 rounds and produce 3 round
// keys.
//
.align 4
.Lschedule_192:
sub $inp, $inp, #8
@@ -754,16 +759,16 @@ _vpaes_schedule_core:
bl _vpaes_schedule_192_smear
b .Loop_schedule_192
##
## .aes_schedule_256
##
## 256-bit specific part of key schedule.
##
## The structure here is very similar to the 128-bit
## schedule, but with an additional "low side" in
## %xmm6. The low side's rounds are the same as the
## high side's, except no rcon and no rotation.
##
//
// .aes_schedule_256
//
// 256-bit specific part of key schedule.
//
// The structure here is very similar to the 128-bit
// schedule, but with an additional "low side" in
// %xmm6. The low side's rounds are the same as the
// high side's, except no rcon and no rotation.
//
.align 4
.Lschedule_256:
ld1 {v0.16b}, [$inp] // vmovdqu 16(%rdi),%xmm0 # load key part 2 (unaligned)
@@ -790,16 +795,16 @@ _vpaes_schedule_core:
b .Loop_schedule_256
##
## .aes_schedule_mangle_last
##
## Mangler for last round of key schedule
## Mangles %xmm0
## when encrypting, outputs out(%xmm0) ^ 63
## when decrypting, outputs unskew(%xmm0)
##
## Always called right before return... jumps to cleanup and exits
##
//
// .aes_schedule_mangle_last
//
// Mangler for last round of key schedule
// Mangles %xmm0
// when encrypting, outputs out(%xmm0) ^ 63
// when decrypting, outputs unskew(%xmm0)
//
// Always called right before return... jumps to cleanup and exits
//
.align 4
.Lschedule_mangle_last:
// schedule last round key from xmm0
@@ -829,23 +834,24 @@ _vpaes_schedule_core:
eor v6.16b, v6.16b, v6.16b // vpxor %xmm6, %xmm6, %xmm6
eor v7.16b, v7.16b, v7.16b // vpxor %xmm7, %xmm7, %xmm7
ldp x29, x30, [sp],#16
.inst 0xd50323bf // autiasp
ret
.size _vpaes_schedule_core,.-_vpaes_schedule_core
##
## .aes_schedule_192_smear
##
## Smear the short, low side in the 192-bit key schedule.
##
## Inputs:
## %xmm7: high side, b a x y
## %xmm6: low side, d c 0 0
## %xmm13: 0
##
## Outputs:
## %xmm6: b+c+d b+c 0 0
## %xmm0: b+c+d b+c b a
##
//
// .aes_schedule_192_smear
//
// Smear the short, low side in the 192-bit key schedule.
//
// Inputs:
// %xmm7: high side, b a x y
// %xmm6: low side, d c 0 0
// %xmm13: 0
//
// Outputs:
// %xmm6: b+c+d b+c 0 0
// %xmm0: b+c+d b+c b a
//
.type _vpaes_schedule_192_smear,%function
.align 4
_vpaes_schedule_192_smear:
@@ -861,24 +867,24 @@ _vpaes_schedule_192_smear:
ret
.size _vpaes_schedule_192_smear,.-_vpaes_schedule_192_smear
##
## .aes_schedule_round
##
## Runs one main round of the key schedule on %xmm0, %xmm7
##
## Specifically, runs subbytes on the high dword of %xmm0
## then rotates it by one byte and xors into the low dword of
## %xmm7.
##
## Adds rcon from low byte of %xmm8, then rotates %xmm8 for
## next rcon.
##
## Smears the dwords of %xmm7 by xoring the low into the
## second low, result into third, result into highest.
##
## Returns results in %xmm7 = %xmm0.
## Clobbers %xmm1-%xmm4, %r11.
##
//
// .aes_schedule_round
//
// Runs one main round of the key schedule on %xmm0, %xmm7
//
// Specifically, runs subbytes on the high dword of %xmm0
// then rotates it by one byte and xors into the low dword of
// %xmm7.
//
// Adds rcon from low byte of %xmm8, then rotates %xmm8 for
// next rcon.
//
// Smears the dwords of %xmm7 by xoring the low into the
// second low, result into third, result into highest.
//
// Returns results in %xmm7 = %xmm0.
// Clobbers %xmm1-%xmm4, %r11.
//
.type _vpaes_schedule_round,%function
.align 4
_vpaes_schedule_round:
@@ -926,15 +932,15 @@ _vpaes_schedule_low_round:
ret
.size _vpaes_schedule_round,.-_vpaes_schedule_round
##
## .aes_schedule_transform
##
## Linear-transform %xmm0 according to tables at (%r11)
##
## Requires that %xmm9 = 0x0F0F... as in preheat
## Output in %xmm0
## Clobbers %xmm1, %xmm2
##
//
// .aes_schedule_transform
//
// Linear-transform %xmm0 according to tables at (%r11)
//
// Requires that %xmm9 = 0x0F0F... as in preheat
// Output in %xmm0
// Clobbers %xmm1, %xmm2
//
.type _vpaes_schedule_transform,%function
.align 4
_vpaes_schedule_transform:
@@ -948,29 +954,29 @@ _vpaes_schedule_transform:
ret
.size _vpaes_schedule_transform,.-_vpaes_schedule_transform
##
## .aes_schedule_mangle
##
## Mangle xmm0 from (basis-transformed) standard version
## to our version.
##
## On encrypt,
## xor with 0x63
## multiply by circulant 0,1,1,1
## apply shiftrows transform
##
## On decrypt,
## xor with 0x63
## multiply by "inverse mixcolumns" circulant E,B,D,9
## deskew
## apply shiftrows transform
##
##
## Writes out to (%rdx), and increments or decrements it
## Keeps track of round number mod 4 in %r8
## Preserves xmm0
## Clobbers xmm1-xmm5
##
//
// .aes_schedule_mangle
//
// Mangle xmm0 from (basis-transformed) standard version
// to our version.
//
// On encrypt,
// xor with 0x63
// multiply by circulant 0,1,1,1
// apply shiftrows transform
//
// On decrypt,
// xor with 0x63
// multiply by "inverse mixcolumns" circulant E,B,D,9
// deskew
// apply shiftrows transform
//
//
// Writes out to (%rdx), and increments or decrements it
// Keeps track of round number mod 4 in %r8
// Preserves xmm0
// Clobbers xmm1-xmm5
//
.type _vpaes_schedule_mangle,%function
.align 4
_vpaes_schedule_mangle:
@@ -1041,6 +1047,7 @@ _vpaes_schedule_mangle:
.type vpaes_set_encrypt_key,%function
.align 4
vpaes_set_encrypt_key:
.inst 0xd503233f // paciasp
stp x29,x30,[sp,#-16]!
add x29,sp,#0
stp d8,d9,[sp,#-16]! // ABI spec says so
@@ -1056,6 +1063,7 @@ vpaes_set_encrypt_key:
ldp d8,d9,[sp],#16
ldp x29,x30,[sp],#16
.inst 0xd50323bf // autiasp
ret
.size vpaes_set_encrypt_key,.-vpaes_set_encrypt_key
@@ -1063,6 +1071,7 @@ vpaes_set_encrypt_key:
.type vpaes_set_decrypt_key,%function
.align 4
vpaes_set_decrypt_key:
.inst 0xd503233f // paciasp
stp x29,x30,[sp,#-16]!
add x29,sp,#0
stp d8,d9,[sp,#-16]! // ABI spec says so
@@ -1082,6 +1091,7 @@ vpaes_set_decrypt_key:
ldp d8,d9,[sp],#16
ldp x29,x30,[sp],#16
.inst 0xd50323bf // autiasp
ret
.size vpaes_set_decrypt_key,.-vpaes_set_decrypt_key
___
@@ -1098,6 +1108,7 @@ vpaes_cbc_encrypt:
cmp w5, #0 // check direction
b.eq vpaes_cbc_decrypt
.inst 0xd503233f // paciasp
stp x29,x30,[sp,#-16]!
add x29,sp,#0
@@ -1120,6 +1131,7 @@ vpaes_cbc_encrypt:
st1 {v0.16b}, [$ivec] // write ivec
ldp x29,x30,[sp],#16
.inst 0xd50323bf // autiasp
.Lcbc_abort:
ret
.size vpaes_cbc_encrypt,.-vpaes_cbc_encrypt
@@ -1127,6 +1139,7 @@ vpaes_cbc_encrypt:
.type vpaes_cbc_decrypt,%function
.align 4
vpaes_cbc_decrypt:
.inst 0xd503233f // paciasp
stp x29,x30,[sp,#-16]!
add x29,sp,#0
stp d8,d9,[sp,#-16]! // ABI spec says so
@@ -1168,6 +1181,7 @@ vpaes_cbc_decrypt:
ldp d10,d11,[sp],#16
ldp d8,d9,[sp],#16
ldp x29,x30,[sp],#16
.inst 0xd50323bf // autiasp
ret
.size vpaes_cbc_decrypt,.-vpaes_cbc_decrypt
___
@@ -1177,6 +1191,7 @@ $code.=<<___;
.type vpaes_ecb_encrypt,%function
.align 4
vpaes_ecb_encrypt:
.inst 0xd503233f // paciasp
stp x29,x30,[sp,#-16]!
add x29,sp,#0
stp d8,d9,[sp,#-16]! // ABI spec says so
@@ -1210,6 +1225,7 @@ vpaes_ecb_encrypt:
ldp d10,d11,[sp],#16
ldp d8,d9,[sp],#16
ldp x29,x30,[sp],#16
.inst 0xd50323bf // autiasp
ret
.size vpaes_ecb_encrypt,.-vpaes_ecb_encrypt
@@ -1217,6 +1233,7 @@ vpaes_ecb_encrypt:
.type vpaes_ecb_decrypt,%function
.align 4
vpaes_ecb_decrypt:
.inst 0xd503233f // paciasp
stp x29,x30,[sp,#-16]!
add x29,sp,#0
stp d8,d9,[sp,#-16]! // ABI spec says so
@@ -1250,6 +1267,7 @@ vpaes_ecb_decrypt:
ldp d10,d11,[sp],#16
ldp d8,d9,[sp],#16
ldp x29,x30,[sp],#16
.inst 0xd50323bf // autiasp
ret
.size vpaes_ecb_decrypt,.-vpaes_ecb_decrypt
___
crypto/aes/asm/vpaes-x86_64.pl
+26
View File
@@ -91,6 +91,7 @@ $code.=<<___;
.type _vpaes_encrypt_core,\@abi-omnipotent
.align 16
_vpaes_encrypt_core:
.cfi_startproc
mov %rdx, %r9
mov \$16, %r11
mov 240(%rdx),%eax
@@ -171,6 +172,7 @@ _vpaes_encrypt_core:
pxor %xmm4, %xmm0 # 0 = A
pshufb %xmm1, %xmm0
ret
.cfi_endproc
.size _vpaes_encrypt_core,.-_vpaes_encrypt_core
##
@@ -181,6 +183,7 @@ _vpaes_encrypt_core:
.type _vpaes_decrypt_core,\@abi-omnipotent
.align 16
_vpaes_decrypt_core:
.cfi_startproc
mov %rdx, %r9 # load key
mov 240(%rdx),%eax
movdqa %xmm9, %xmm1
@@ -277,6 +280,7 @@ _vpaes_decrypt_core:
pxor %xmm4, %xmm0 # 0 = A
pshufb %xmm2, %xmm0
ret
.cfi_endproc
.size _vpaes_decrypt_core,.-_vpaes_decrypt_core
########################################################
@@ -287,6 +291,7 @@ _vpaes_decrypt_core:
.type _vpaes_schedule_core,\@abi-omnipotent
.align 16
_vpaes_schedule_core:
.cfi_startproc
# rdi = key
# rsi = size in bits
# rdx = buffer
@@ -453,6 +458,7 @@ _vpaes_schedule_core:
pxor %xmm6, %xmm6
pxor %xmm7, %xmm7
ret
.cfi_endproc
.size _vpaes_schedule_core,.-_vpaes_schedule_core
##
@@ -472,6 +478,7 @@ _vpaes_schedule_core:
.type _vpaes_schedule_192_smear,\@abi-omnipotent
.align 16
_vpaes_schedule_192_smear:
.cfi_startproc
pshufd \$0x80, %xmm6, %xmm1 # d c 0 0 -> c 0 0 0
pshufd \$0xFE, %xmm7, %xmm0 # b a _ _ -> b b b a
pxor %xmm1, %xmm6 # -> c+d c 0 0
@@ -480,6 +487,7 @@ _vpaes_schedule_192_smear:
movdqa %xmm6, %xmm0
movhlps %xmm1, %xmm6 # clobber low side with zeros
ret
.cfi_endproc
.size _vpaes_schedule_192_smear,.-_vpaes_schedule_192_smear
##
@@ -503,6 +511,7 @@ _vpaes_schedule_192_smear:
.type _vpaes_schedule_round,\@abi-omnipotent
.align 16
_vpaes_schedule_round:
.cfi_startproc
# extract rcon from xmm8
pxor %xmm1, %xmm1
palignr \$15, %xmm8, %xmm1
@@ -556,6 +565,7 @@ _vpaes_schedule_low_round:
pxor %xmm7, %xmm0
movdqa %xmm0, %xmm7
ret
.cfi_endproc
.size _vpaes_schedule_round,.-_vpaes_schedule_round
##
@@ -570,6 +580,7 @@ _vpaes_schedule_low_round:
.type _vpaes_schedule_transform,\@abi-omnipotent
.align 16
_vpaes_schedule_transform:
.cfi_startproc
movdqa %xmm9, %xmm1
pandn %xmm0, %xmm1
psrld \$4, %xmm1
@@ -580,6 +591,7 @@ _vpaes_schedule_transform:
pshufb %xmm1, %xmm0
pxor %xmm2, %xmm0
ret
.cfi_endproc
.size _vpaes_schedule_transform,.-_vpaes_schedule_transform
##
@@ -608,6 +620,7 @@ _vpaes_schedule_transform:
.type _vpaes_schedule_mangle,\@abi-omnipotent
.align 16
_vpaes_schedule_mangle:
.cfi_startproc
movdqa %xmm0, %xmm4 # save xmm0 for later
movdqa .Lk_mc_forward(%rip),%xmm5
test %rcx, %rcx
@@ -672,6 +685,7 @@ _vpaes_schedule_mangle:
and \$0x30, %r8
movdqu %xmm3, (%rdx)
ret
.cfi_endproc
.size _vpaes_schedule_mangle,.-_vpaes_schedule_mangle
#
@@ -681,6 +695,7 @@ _vpaes_schedule_mangle:
.type ${PREFIX}_set_encrypt_key,\@function,3
.align 16
${PREFIX}_set_encrypt_key:
.cfi_startproc
___
$code.=<<___ if ($win64);
lea -0xb8(%rsp),%rsp
@@ -723,12 +738,14 @@ ___
$code.=<<___;
xor %eax,%eax
ret
.cfi_endproc
.size ${PREFIX}_set_encrypt_key,.-${PREFIX}_set_encrypt_key
.globl ${PREFIX}_set_decrypt_key
.type ${PREFIX}_set_decrypt_key,\@function,3
.align 16
${PREFIX}_set_decrypt_key:
.cfi_startproc
___
$code.=<<___ if ($win64);
lea -0xb8(%rsp),%rsp
@@ -776,12 +793,14 @@ ___
$code.=<<___;
xor %eax,%eax
ret
.cfi_endproc
.size ${PREFIX}_set_decrypt_key,.-${PREFIX}_set_decrypt_key
.globl ${PREFIX}_encrypt
.type ${PREFIX}_encrypt,\@function,3
.align 16
${PREFIX}_encrypt:
.cfi_startproc
___
$code.=<<___ if ($win64);
lea -0xb8(%rsp),%rsp
@@ -819,12 +838,14 @@ $code.=<<___ if ($win64);
___
$code.=<<___;
ret
.cfi_endproc
.size ${PREFIX}_encrypt,.-${PREFIX}_encrypt
.globl ${PREFIX}_decrypt
.type ${PREFIX}_decrypt,\@function,3
.align 16
${PREFIX}_decrypt:
.cfi_startproc
___
$code.=<<___ if ($win64);
lea -0xb8(%rsp),%rsp
@@ -862,6 +883,7 @@ $code.=<<___ if ($win64);
___
$code.=<<___;
ret
.cfi_endproc
.size ${PREFIX}_decrypt,.-${PREFIX}_decrypt
___
{
@@ -874,6 +896,7 @@ $code.=<<___;
.type ${PREFIX}_cbc_encrypt,\@function,6
.align 16
${PREFIX}_cbc_encrypt:
.cfi_startproc
xchg $key,$len
___
($len,$key)=($key,$len);
@@ -944,6 +967,7 @@ ___
$code.=<<___;
.Lcbc_abort:
ret
.cfi_endproc
.size ${PREFIX}_cbc_encrypt,.-${PREFIX}_cbc_encrypt
___
}
@@ -957,6 +981,7 @@ $code.=<<___;
.type _vpaes_preheat,\@abi-omnipotent
.align 16
_vpaes_preheat:
.cfi_startproc
lea .Lk_s0F(%rip), %r10
movdqa -0x20(%r10), %xmm10 # .Lk_inv
movdqa -0x10(%r10), %xmm11 # .Lk_inv+16
@@ -966,6 +991,7 @@ _vpaes_preheat:
movdqa 0x50(%r10), %xmm15 # .Lk_sb2
movdqa 0x60(%r10), %xmm14 # .Lk_sb2+16
ret
.cfi_endproc
.size _vpaes_preheat,.-_vpaes_preheat
########################################################
## ##