Hakase/openssl-custom
1
0 Fork 0
Code Issues Pull Requests Releases Wiki Activity

OpenSSL 1.1.1-pre2

Browse Source
This commit is contained in:
Hakase
2018-04-07 17:29:40 +09:00
parent 82a44d2483
commit bbac8ca55d
17755 changed files with 221242 additions and 98415 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crypto/aes/aes_core.c
+3 -3
View File
@@ -14,9 +14,9 @@
*
* Optimised ANSI C code for the Rijndael cipher (now AES)
*
* @author Vincent Rijmen <vincent.rijmen@esat.kuleuven.ac.be>
* @author Antoon Bosselaers <antoon.bosselaers@esat.kuleuven.ac.be>
* @author Paulo Barreto <paulo.barreto@terra.com.br>
* @author Vincent Rijmen
* @author Antoon Bosselaers
* @author Paulo Barreto
*
* This code is hereby placed in the public domain.
*
crypto/aes/aes_x86core.c
+11 -12
View File
@@ -7,6 +7,14 @@
* https://www.openssl.org/source/license.html
*/
/*
* This is experimental x86[_64] derivative. It assumes little-endian
* byte order and expects CPU to sustain unaligned memory references.
* It is used as playground for cache-time attack mitigations and
* serves as reference C implementation for x86[_64] as well as some
* other assembly modules.
*/
/**
* rijndael-alg-fst.c
*
@@ -14,9 +22,9 @@
*
* Optimised ANSI C code for the Rijndael cipher (now AES)
*
* @author Vincent Rijmen <vincent.rijmen@esat.kuleuven.ac.be>
* @author Antoon Bosselaers <antoon.bosselaers@esat.kuleuven.ac.be>
* @author Paulo Barreto <paulo.barreto@terra.com.br>
* @author Vincent Rijmen
* @author Antoon Bosselaers
* @author Paulo Barreto
*
* This code is hereby placed in the public domain.
*
@@ -33,15 +41,6 @@
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/*
* This is experimental x86[_64] derivative. It assumes little-endian
* byte order and expects CPU to sustain unaligned memory references.
* It is used as playground for cache-time attack mitigations and
* serves as reference C implementation for x86[_64] assembler.
*
* <appro@fy.chalmers.se>
*/
#include <assert.h>
crypto/aes/asm/aes-586.pl
+17 -17
View File
@@ -8,7 +8,7 @@
#
# ====================================================================
# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
@@ -39,7 +39,7 @@
# for scaling too, I [try to] avoid the latter by favoring off-by-2
# shifts and masking the result with 0xFF<<2 instead of "boring" 0xFF.
#
# As was shown by Dean Gaudet <dean@arctic.org>, the above note turned
# As was shown by Dean Gaudet, the above note turned out to be
# void. Performance improvement with off-by-2 shifts was observed on
# intermediate implementation, which was spilling yet another register
# to stack... Final offset*4 code below runs just a tad faster on P4,
@@ -55,8 +55,8 @@
# better performance on most recent µ-archs...
#
# Third version adds AES_cbc_encrypt implementation, which resulted in
# up to 40% performance imrovement of CBC benchmark results. 40% was
# observed on P4 core, where "overall" imrovement coefficient, i.e. if
# up to 40% performance improvement of CBC benchmark results. 40% was
# observed on P4 core, where "overall" improvement coefficient, i.e. if
# compared to PIC generated by GCC and in CBC mode, was observed to be
# as large as 4x:-) CBC performance is virtually identical to ECB now
# and on some platforms even better, e.g. 17.6 "small" cycles/byte on
@@ -123,7 +123,7 @@
# words every cache-line is *guaranteed* to be accessed within ~50
# cycles window. Why just SSE? Because it's needed on hyper-threading
# CPU! Which is also why it's prefetched with 64 byte stride. Best
# part is that it has no negative effect on performance:-)
# part is that it has no negative effect on performance:-)
#
# Version 4.3 implements switch between compact and non-compact block
# functions in AES_cbc_encrypt depending on how much data was asked
@@ -159,7 +159,7 @@
# combinations then attack becomes infeasible. This is why revised
# AES_cbc_encrypt "dares" to switch to larger S-box when larger chunk
# of data is to be processed in one stroke. The current size limit of
# 512 bytes is chosen to provide same [diminishigly low] probability
# 512 bytes is chosen to provide same [diminishingly low] probability
# for cache-line to remain untouched in large chunk operation with
# large S-box as for single block operation with compact S-box and
# surely needs more careful consideration...
@@ -171,12 +171,12 @@
# yield execution to process performing AES just before timer fires
# off the scheduler, immediately regain control of CPU and analyze the
# cache state. For this attack to be efficient attacker would have to
# effectively slow down the operation by several *orders* of magnitute,
# effectively slow down the operation by several *orders* of magnitude,
# by ratio of time slice to duration of handful of AES rounds, which
# unlikely to remain unnoticed. Not to mention that this also means
# that he would spend correspondigly more time to collect enough
# that he would spend correspondingly more time to collect enough
# statistical data to mount the attack. It's probably appropriate to
# say that if adeversary reckons that this attack is beneficial and
# say that if adversary reckons that this attack is beneficial and
# risks to be noticed, you probably have larger problems having him
# mere opportunity. In other words suggested code design expects you
# to preclude/mitigate this attack by overall system security design.
@@ -202,7 +202,7 @@ $output = pop;
open OUT,">$output";
*STDOUT=*OUT;
&asm_init($ARGV[0],"aes-586.pl",$x86only = $ARGV[$#ARGV] eq "386");
&asm_init($ARGV[0],$x86only = $ARGV[$#ARGV] eq "386");
&static_label("AES_Te");
&static_label("AES_Td");
@@ -240,7 +240,7 @@ $small_footprint=1; # $small_footprint=1 code is ~5% slower [on
# contention and in hope to "collect" 5% back
# in real-life applications...
$vertical_spin=0; # shift "verticaly" defaults to 0, because of
$vertical_spin=0; # shift "vertically" defaults to 0, because of
# its proof-of-concept status...
# Note that there is no decvert(), as well as last encryption round is
# performed with "horizontal" shifts. This is because this "vertical"
@@ -585,7 +585,7 @@ sub enctransform()
# +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
# | mm4 | mm0 |
# +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
# | s3 | s2 | s1 | s0 |
# | s3 | s2 | s1 | s0 |
# +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
# |15|14|13|12|11|10| 9| 8| 7| 6| 5| 4| 3| 2| 1| 0|
# +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+
@@ -805,7 +805,7 @@ sub encstep()
if ($i==3) { $tmp=$s[3]; &mov ($s[2],$__s1); }##%ecx
elsif($i==2){ &movz ($tmp,&HB($s[3])); }#%ebx[2]
else { &mov ($tmp,$s[3]);
else { &mov ($tmp,$s[3]);
&shr ($tmp,24) }
&xor ($out,&DWP(1,$te,$tmp,8));
if ($i<2) { &mov (&DWP(4+4*$i,"esp"),$out); }
@@ -1558,7 +1558,7 @@ sub sse_deccompact()
&pxor ("mm1","mm3"); &pxor ("mm5","mm7"); # tp4
&pshufw ("mm3","mm1",0xb1); &pshufw ("mm7","mm5",0xb1);
&pxor ("mm0","mm1"); &pxor ("mm4","mm5"); # ^= tp4
&pxor ("mm0","mm3"); &pxor ("mm4","mm7"); # ^= ROTATE(tp4,16)
&pxor ("mm0","mm3"); &pxor ("mm4","mm7"); # ^= ROTATE(tp4,16)
&pxor ("mm3","mm3"); &pxor ("mm7","mm7");
&pcmpgtb("mm3","mm1"); &pcmpgtb("mm7","mm5");
@@ -1606,7 +1606,7 @@ sub decstep()
# no instructions are reordered, as performance appears
# optimal... or rather that all attempts to reorder didn't
# result in better performance [which by the way is not a
# bit lower than ecryption].
# bit lower than encryption].
if($i==3) { &mov ($key,$__key); }
else { &mov ($out,$s[0]); }
&and ($out,0xFF);
@@ -2028,7 +2028,7 @@ sub declast()
{
# stack frame layout
# -4(%esp) # return address 0(%esp)
# 0(%esp) # s0 backing store 4(%esp)
# 0(%esp) # s0 backing store 4(%esp)
# 4(%esp) # s1 backing store 8(%esp)
# 8(%esp) # s2 backing store 12(%esp)
# 12(%esp) # s3 backing store 16(%esp)
@@ -2738,7 +2738,7 @@ sub enckey()
&mov (&DWP(80,"edi"),10); # setup number of rounds
&xor ("eax","eax");
&jmp (&label("exit"));
&set_label("12rounds");
&mov ("eax",&DWP(0,"esi")); # copy first 6 dwords
&mov ("ebx",&DWP(4,"esi"));
crypto/aes/asm/aes-armv4.pl
+4 -4
View File
@@ -1,5 +1,5 @@
#! /usr/bin/env perl
# Copyright 2007-2018 The OpenSSL Project Authors. All Rights Reserved.
# Copyright 2007-2016 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License"). You may not use
# this file except in compliance with the License. You can obtain a copy
@@ -203,7 +203,7 @@ AES_encrypt:
adr r3,.
#endif
stmdb sp!,{r1,r4-r12,lr}
#if defined(__thumb2__) || defined(__APPLE__)
#ifdef __APPLE__
adr $tbl,AES_Te
#else
sub $tbl,r3,#AES_encrypt-AES_Te @ Te
@@ -481,7 +481,7 @@ _armv4_AES_set_encrypt_key:
mov lr,r1 @ bits
mov $key,r2 @ key
#if defined(__thumb2__) || defined(__APPLE__)
#ifdef __APPLE__
adr $tbl,AES_Te+1024 @ Te4
#else
sub $tbl,r3,#_armv4_AES_set_encrypt_key-AES_Te-1024 @ Te4
@@ -979,7 +979,7 @@ AES_decrypt:
adr r3,.
#endif
stmdb sp!,{r1,r4-r12,lr}
#if defined(__thumb2__) || defined(__APPLE__)
#ifdef __APPLE__
adr $tbl,AES_Td
#else
sub $tbl,r3,#AES_decrypt-AES_Td @ Td
crypto/aes/asm/aes-ia64.S
+2 -2
View File
@@ -6,7 +6,7 @@
// https://www.openssl.org/source/license.html
//
// ====================================================================
// Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
// Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
// project. Rights for redistribution and usage in source and binary
// forms are granted according to the OpenSSL license.
// ====================================================================
@@ -33,7 +33,7 @@
// 64 bytes line size and L2 - 128 bytes...
.ident "aes-ia64.S, version 1.2"
.ident "IA-64 ISA artwork by Andy Polyakov <appro@fy.chalmers.se>"
.ident "IA-64 ISA artwork by Andy Polyakov <appro@openssl.org>"
.explicit
.text
crypto/aes/asm/aes-mips.pl
+66 -27
View File
@@ -65,8 +65,8 @@ $flavour = shift || "o32"; # supported flavours are o32,n32,64,nubi32,nubi64
if ($flavour =~ /64|n32/i) {
$PTR_LA="dla";
$PTR_ADD="dadd"; # incidentally works even on n32
$PTR_SUB="dsub"; # incidentally works even on n32
$PTR_ADD="daddu"; # incidentally works even on n32
$PTR_SUB="dsubu"; # incidentally works even on n32
$PTR_INS="dins";
$REG_S="sd";
$REG_L="ld";
@@ -74,8 +74,8 @@ if ($flavour =~ /64|n32/i) {
$SZREG=8;
} else {
$PTR_LA="la";
$PTR_ADD="add";
$PTR_SUB="sub";
$PTR_ADD="addu";
$PTR_SUB="subu";
$PTR_INS="ins";
$REG_S="sw";
$REG_L="lw";
@@ -102,15 +102,9 @@ open STDOUT,">$output";
my ($MSB,$LSB)=(0,3); # automatically converted to little-endian
$code.=<<___;
#include "mips_arch.h"
.text
#ifdef OPENSSL_FIPSCANISTER
# include <openssl/fipssyms.h>
#endif
#if defined(__mips_smartmips) && !defined(_MIPS_ARCH_MIPS32R2)
#define _MIPS_ARCH_MIPS32R2
#endif
#if !defined(__mips_eabi) && (!defined(__vxworks) || defined(__pic__))
.option pic2
#endif
@@ -126,7 +120,7 @@ my ($i0,$i1,$i2,$i3)=($at,$t0,$t1,$t2);
my ($t0,$t1,$t2,$t3,$t4,$t5,$t6,$t7,$t8,$t9,$t10,$t11) = map("\$$_",(12..23));
my ($key0,$cnt)=($gp,$fp);
# instuction ordering is "stolen" from output from MIPSpro assembler
# instruction ordering is "stolen" from output from MIPSpro assembler
# invoked with -mips3 -O3 arguments...
$code.=<<___;
.align 5
@@ -146,7 +140,7 @@ _mips_AES_encrypt:
xor $s2,$t2
xor $s3,$t3
sub $cnt,1
subu $cnt,1
#if defined(__mips_smartmips)
ext $i0,$s1,16,8
.Loop_enc:
@@ -218,7 +212,7 @@ _mips_AES_encrypt:
xor $t2,$t6
xor $t3,$t7
sub $cnt,1
subu $cnt,1
$PTR_ADD $key0,16
xor $s0,$t0
xor $s1,$t1
@@ -409,7 +403,7 @@ _mips_AES_encrypt:
xor $t2,$t6
xor $t3,$t7
sub $cnt,1
subu $cnt,1
$PTR_ADD $key0,16
xor $s0,$t0
xor $s1,$t1
@@ -657,6 +651,12 @@ $code.=<<___;
.set reorder
$PTR_LA $Tbl,AES_Te # PIC-ified 'load address'
#if defined(_MIPS_ARCH_MIPS32R6) || defined(_MIPS_ARCH_MIPS64R6)
lw $s0,0($inp)
lw $s1,4($inp)
lw $s2,8($inp)
lw $s3,12($inp)
#else
lwl $s0,0+$MSB($inp)
lwl $s1,4+$MSB($inp)
lwl $s2,8+$MSB($inp)
@@ -665,9 +665,16 @@ $code.=<<___;
lwr $s1,4+$LSB($inp)
lwr $s2,8+$LSB($inp)
lwr $s3,12+$LSB($inp)
#endif
bal _mips_AES_encrypt
#if defined(_MIPS_ARCH_MIPS32R6) || defined(_MIPS_ARCH_MIPS64R6)
sw $s0,0($out)
sw $s1,4($out)
sw $s2,8($out)
sw $s3,12($out)
#else
swr $s0,0+$LSB($out)
swr $s1,4+$LSB($out)
swr $s2,8+$LSB($out)
@@ -676,6 +683,7 @@ $code.=<<___;
swl $s1,4+$MSB($out)
swl $s2,8+$MSB($out)
swl $s3,12+$MSB($out)
#endif
.set noreorder
$REG_L $ra,$FRAMESIZE-1*$SZREG($sp)
@@ -720,7 +728,7 @@ _mips_AES_decrypt:
xor $s2,$t2
xor $s3,$t3
sub $cnt,1
subu $cnt,1
#if defined(__mips_smartmips)
ext $i0,$s3,16,8
.Loop_dec:
@@ -792,7 +800,7 @@ _mips_AES_decrypt:
xor $t2,$t6
xor $t3,$t7
sub $cnt,1
subu $cnt,1
$PTR_ADD $key0,16
xor $s0,$t0
xor $s1,$t1
@@ -985,7 +993,7 @@ _mips_AES_decrypt:
xor $t2,$t6
xor $t3,$t7
sub $cnt,1
subu $cnt,1
$PTR_ADD $key0,16
xor $s0,$t0
xor $s1,$t1
@@ -1228,6 +1236,12 @@ $code.=<<___;
.set reorder
$PTR_LA $Tbl,AES_Td # PIC-ified 'load address'
#if defined(_MIPS_ARCH_MIPS32R6) || defined(_MIPS_ARCH_MIPS64R6)
lw $s0,0($inp)
lw $s1,4($inp)
lw $s2,8($inp)
lw $s3,12($inp)
#else
lwl $s0,0+$MSB($inp)
lwl $s1,4+$MSB($inp)
lwl $s2,8+$MSB($inp)
@@ -1236,9 +1250,16 @@ $code.=<<___;
lwr $s1,4+$LSB($inp)
lwr $s2,8+$LSB($inp)
lwr $s3,12+$LSB($inp)
#endif
bal _mips_AES_decrypt
#if defined(_MIPS_ARCH_MIPS32R6) || defined(_MIPS_ARCH_MIPS64R6)
sw $s0,0($out)
sw $s1,4($out)
sw $s2,8($out)
sw $s3,12($out)
#else
swr $s0,0+$LSB($out)
swr $s1,4+$LSB($out)
swr $s2,8+$LSB($out)
@@ -1247,6 +1268,7 @@ $code.=<<___;
swl $s1,4+$MSB($out)
swl $s2,8+$MSB($out)
swl $s3,12+$MSB($out)
#endif
.set noreorder
$REG_L $ra,$FRAMESIZE-1*$SZREG($sp)
@@ -1295,35 +1317,52 @@ _mips_AES_set_encrypt_key:
$PTR_ADD $rcon,$Tbl,256
.set reorder
#if defined(_MIPS_ARCH_MIPS32R6) || defined(_MIPS_ARCH_MIPS64R6)
lw $rk0,0($inp) # load 128 bits
lw $rk1,4($inp)
lw $rk2,8($inp)
lw $rk3,12($inp)
#else
lwl $rk0,0+$MSB($inp) # load 128 bits
lwl $rk1,4+$MSB($inp)
lwl $rk2,8+$MSB($inp)
lwl $rk3,12+$MSB($inp)
li $at,128
lwr $rk0,0+$LSB($inp)
lwr $rk1,4+$LSB($inp)
lwr $rk2,8+$LSB($inp)
lwr $rk3,12+$LSB($inp)
#endif
li $at,128
.set noreorder
beq $bits,$at,.L128bits
li $cnt,10
.set reorder
#if defined(_MIPS_ARCH_MIPS32R6) || defined(_MIPS_ARCH_MIPS64R6)
lw $rk4,16($inp) # load 192 bits
lw $rk5,20($inp)
#else
lwl $rk4,16+$MSB($inp) # load 192 bits
lwl $rk5,20+$MSB($inp)
li $at,192
lwr $rk4,16+$LSB($inp)
lwr $rk5,20+$LSB($inp)
#endif
li $at,192
.set noreorder
beq $bits,$at,.L192bits
li $cnt,8
.set reorder
#if defined(_MIPS_ARCH_MIPS32R6) || defined(_MIPS_ARCH_MIPS64R6)
lw $rk6,24($inp) # load 256 bits
lw $rk7,28($inp)
#else
lwl $rk6,24+$MSB($inp) # load 256 bits
lwl $rk7,28+$MSB($inp)
li $at,256
lwr $rk6,24+$LSB($inp)
lwr $rk7,28+$LSB($inp)
#endif
li $at,256
.set noreorder
beq $bits,$at,.L256bits
li $cnt,7
@@ -1353,7 +1392,7 @@ _mips_AES_set_encrypt_key:
sw $rk1,4($key)
sw $rk2,8($key)
sw $rk3,12($key)
sub $cnt,1
subu $cnt,1
$PTR_ADD $key,16
_bias $i0,24
@@ -1410,7 +1449,7 @@ _mips_AES_set_encrypt_key:
sw $rk3,12($key)
sw $rk4,16($key)
sw $rk5,20($key)
sub $cnt,1
subu $cnt,1
$PTR_ADD $key,24
_bias $i0,24
@@ -1471,7 +1510,7 @@ _mips_AES_set_encrypt_key:
sw $rk5,20($key)
sw $rk6,24($key)
sw $rk7,28($key)
sub $cnt,1
subu $cnt,1
_bias $i0,24
_bias $i1,16
@@ -1653,7 +1692,7 @@ $code.=<<___;
lw $tp1,16($key) # modulo-scheduled
lui $x80808080,0x8080
sub $cnt,1
subu $cnt,1
or $x80808080,0x8080
sll $cnt,2
$PTR_ADD $key,16
@@ -1716,7 +1755,7 @@ $code.=<<___;
lw $tp1,4($key) # modulo-scheduled
xor $tpe,$tp2
#endif
sub $cnt,1
subu $cnt,1
sw $tpe,0($key)
$PTR_ADD $key,4
bnez $cnt,.Lmix
crypto/aes/asm/aes-parisc.pl
+2 -2
View File
@@ -8,7 +8,7 @@
# ====================================================================
# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
@@ -1015,7 +1015,7 @@ ___
foreach (split("\n",$code)) {
s/\`([^\`]*)\`/eval $1/ge;
# translate made up instructons: _ror, _srm
# translate made up instructions: _ror, _srm
s/_ror(\s+)(%r[0-9]+),/shd$1$2,$2,/ or
s/_srm(\s+%r[0-9]+),([0-9]+),/
crypto/aes/asm/aes-ppc.pl
+5 -5
View File
@@ -8,7 +8,7 @@
# ====================================================================
# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
@@ -1433,10 +1433,10 @@ $code.=<<___;
xor $s1,$s1,$acc05
xor $s2,$s2,$acc06
xor $s3,$s3,$acc07
xor $s0,$s0,$acc08 # ^= ROTATE(r8,8)
xor $s1,$s1,$acc09
xor $s2,$s2,$acc10
xor $s3,$s3,$acc11
xor $s0,$s0,$acc08 # ^= ROTATE(r8,8)
xor $s1,$s1,$acc09
xor $s2,$s2,$acc10
xor $s3,$s3,$acc11
b Ldec_compact_loop
.align 4
crypto/aes/asm/aes-s390x.pl
+78 -22
View File
@@ -1,5 +1,5 @@
#! /usr/bin/env perl
# Copyright 2007-2016 The OpenSSL Project Authors. All Rights Reserved.
# Copyright 2007-2018 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License"). You may not use
# this file except in compliance with the License. You can obtain a copy
@@ -8,7 +8,7 @@
# ====================================================================
# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
@@ -44,7 +44,7 @@
# minimize/avoid Address Generation Interlock hazard and to favour
# dual-issue z10 pipeline. This gave ~25% improvement on z10 and
# almost 50% on z9. The gain is smaller on z10, because being dual-
# issue z10 makes it improssible to eliminate the interlock condition:
# issue z10 makes it impossible to eliminate the interlock condition:
# critial path is not long enough. Yet it spends ~24 cycles per byte
# processed with 128-bit key.
#
@@ -129,6 +129,8 @@ sub _data_word()
}
$code=<<___;
#include "s390x_arch.h"
.text
.type AES_Te,\@object
@@ -404,7 +406,7 @@ _s390x_AES_encrypt:
or $s1,$t1
or $t2,$i2
or $t3,$i3
srlg $i1,$s2,`8-3` # i0
srlg $i2,$s2,`16-3` # i1
nr $i1,$mask
@@ -457,7 +459,7 @@ _s390x_AES_encrypt:
x $s2,24($key)
x $s3,28($key)
br $ra
br $ra
.size _s390x_AES_encrypt,.-_s390x_AES_encrypt
___
@@ -779,7 +781,7 @@ _s390x_AES_decrypt:
x $s2,24($key)
x $s3,28($key)
br $ra
br $ra
.size _s390x_AES_decrypt,.-_s390x_AES_decrypt
___
@@ -823,8 +825,8 @@ $code.=<<___ if (!$softonly);
larl %r1,OPENSSL_s390xcap_P
llihh %r0,0x8000
srlg %r0,%r0,0(%r5)
ng %r0,32(%r1) # check availability of both km...
ng %r0,48(%r1) # ...and kmc support for given key length
ng %r0,S390X_KM(%r1) # check availability of both km...
ng %r0,S390X_KMC(%r1) # ...and kmc support for given key length
jz .Lekey_internal
lmg %r0,%r1,0($inp) # just copy 128 bits...
@@ -1084,7 +1086,7 @@ $code.=<<___ if (!$softonly);
lhi $t1,16
cr $t0,$t1
jl .Lgo
oill $t0,0x80 # set "decrypt" bit
oill $t0,S390X_DECRYPT # set "decrypt" bit
st $t0,240($key)
br $ra
___
@@ -1223,7 +1225,7 @@ $code.=<<___ if (!$softonly);
.align 16
.Lkmc_truncated:
ahi $key,-1 # it's the way it's encoded in mvc
tmll %r0,0x80
tmll %r0,S390X_DECRYPT
jnz .Lkmc_truncated_dec
lghi %r1,0
stg %r1,16*$SIZE_T($sp)
@@ -1294,7 +1296,7 @@ $code.=<<___;
.Lcbc_enc_done:
l${g} $ivp,6*$SIZE_T($sp)
st $s0,0($ivp)
st $s1,4($ivp)
st $s1,4($ivp)
st $s2,8($ivp)
st $s3,12($ivp)
@@ -1403,7 +1405,61 @@ $code.=<<___ if (!$softonly);
clr %r0,%r1
jl .Lctr32_software
stm${g} %r6,$s3,6*$SIZE_T($sp)
st${g} $s2,10*$SIZE_T($sp)
st${g} $s3,11*$SIZE_T($sp)
clr $len,%r1 # does work even in 64-bit mode
jle .Lctr32_nokma # kma is slower for <= 16 blocks
larl %r1,OPENSSL_s390xcap_P
lr $s2,%r0
llihh $s3,0x8000
srlg $s3,$s3,0($s2)
ng $s3,S390X_KMA(%r1) # check kma capability vector
jz .Lctr32_nokma
l${g}hi %r1,-$stdframe-112
l${g}r $s3,$sp
la $sp,0(%r1,$sp) # prepare parameter block
lhi %r1,0x0600
sllg $len,$len,4
or %r0,%r1 # set HS and LAAD flags
st${g} $s3,0($sp) # backchain
la %r1,$stdframe($sp)
lmg $s2,$s3,0($key) # copy key
stg $s2,$stdframe+80($sp)
stg $s3,$stdframe+88($sp)
lmg $s2,$s3,16($key)
stg $s2,$stdframe+96($sp)
stg $s3,$stdframe+104($sp)
lmg $s2,$s3,0($ivp) # copy iv
stg $s2,$stdframe+64($sp)
ahi $s3,-1 # kma requires counter-1
stg $s3,$stdframe+72($sp)
st $s3,$stdframe+12($sp) # copy counter
lghi $s2,0 # no AAD
lghi $s3,0
.long 0xb929a042 # kma $out,$s2,$inp
brc 1,.-4 # pay attention to "partial completion"
stg %r0,$stdframe+80($sp) # wipe key
stg %r0,$stdframe+88($sp)
stg %r0,$stdframe+96($sp)
stg %r0,$stdframe+104($sp)
la $sp,$stdframe+112($sp)
lm${g} $s2,$s3,10*$SIZE_T($sp)
br $ra
.align 16
.Lctr32_nokma:
stm${g} %r6,$s1,6*$SIZE_T($sp)
slgr $out,$inp
la %r1,0($key) # %r1 is permanent copy of $key
@@ -1442,7 +1498,7 @@ $code.=<<___ if (!$softonly && 0);# kmctr code was measured to be ~12% slower
larl %r1,OPENSSL_s390xcap_P
llihh %r0,0x8000 # check if kmctr supports the function code
srlg %r0,%r0,0($s0)
ng %r0,64(%r1) # check kmctr capability vector
ng %r0,S390X_KMCTR(%r1) # check kmctr capability vector
lgr %r0,$s0
lgr %r1,$s1
jz .Lctr32_km_loop
@@ -1567,8 +1623,8 @@ ___
}
########################################################################
# void AES_xts_encrypt(const char *inp,char *out,size_t len,
# const AES_KEY *key1, const AES_KEY *key2,
# void AES_xts_encrypt(const unsigned char *inp, unsigned char *out,
# size_t len, const AES_KEY *key1, const AES_KEY *key2,
# const unsigned char iv[16]);
#
{
@@ -1592,7 +1648,7 @@ $code.=<<___ if(1);
larl %r1,OPENSSL_s390xcap_P
llihh %r0,0x8000
srlg %r0,%r0,32($s1) # check for 32+function code
ng %r0,32(%r1) # check km capability vector
ng %r0,S390X_KM(%r1) # check km capability vector
lgr %r0,$s0 # restore the function code
la %r1,0($key1) # restore $key1
jz .Lxts_km_vanilla
@@ -1627,7 +1683,7 @@ $code.=<<___ if(1);
llgc $len,2*$SIZE_T-1($sp)
nill $len,0x0f # $len%=16
br $ra
.align 16
.Lxts_km_vanilla:
___
@@ -1854,7 +1910,7 @@ $code.=<<___;
xgr $s1,%r1
lrvgr $s1,$s1 # flip byte order
lrvgr $s3,$s3
srlg $s0,$s1,32 # smash the tweak to 4x32-bits
srlg $s0,$s1,32 # smash the tweak to 4x32-bits
stg $s1,$tweak+0($sp) # save the tweak
llgfr $s1,$s1
srlg $s2,$s3,32
@@ -1905,7 +1961,7 @@ $code.=<<___;
xgr $s1,%r1
lrvgr $s1,$s1 # flip byte order
lrvgr $s3,$s3
srlg $s0,$s1,32 # smash the tweak to 4x32-bits
srlg $s0,$s1,32 # smash the tweak to 4x32-bits
stg $s1,$tweak+0($sp) # save the tweak
llgfr $s1,$s1
srlg $s2,$s3,32
@@ -1936,8 +1992,8 @@ $code.=<<___;
br $ra
.size AES_xts_encrypt,.-AES_xts_encrypt
___
# void AES_xts_decrypt(const char *inp,char *out,size_t len,
# const AES_KEY *key1, const AES_KEY *key2,
# void AES_xts_decrypt(const unsigned char *inp, unsigned char *out,
# size_t len, const AES_KEY *key1, const AES_KEY *key2,
# const unsigned char iv[16]);
#
$code.=<<___;
@@ -2097,7 +2153,7 @@ $code.=<<___;
xgr $s1,%r1
lrvgr $s1,$s1 # flip byte order
lrvgr $s3,$s3
srlg $s0,$s1,32 # smash the tweak to 4x32-bits
srlg $s0,$s1,32 # smash the tweak to 4x32-bits
stg $s1,$tweak+0($sp) # save the tweak
llgfr $s1,$s1
srlg $s2,$s3,32
crypto/aes/asm/aes-sparcv9.pl
+1 -1
View File
@@ -8,7 +8,7 @@
#
# ====================================================================
# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. Rights for redistribution and usage in source and binary
# forms are granted according to the OpenSSL license.
# ====================================================================
crypto/aes/asm/aes-x86_64.pl
+108 -26
View File
@@ -8,7 +8,7 @@
#
# ====================================================================
# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
@@ -599,15 +599,23 @@ $code.=<<___;
.hidden asm_AES_encrypt
asm_AES_encrypt:
AES_encrypt:
.cfi_startproc
mov %rsp,%rax
.cfi_def_cfa_register %rax
push %rbx
.cfi_push %rbx
push %rbp
.cfi_push %rbp
push %r12
.cfi_push %r12
push %r13
.cfi_push %r13
push %r14
.cfi_push %r14
push %r15
.cfi_push %r15
# allocate frame "above" key schedule
mov %rsp,%r10
lea -63(%rdx),%rcx # %rdx is key argument
and \$-64,%rsp
sub %rsp,%rcx
@@ -617,7 +625,8 @@ AES_encrypt:
sub \$32,%rsp
mov %rsi,16(%rsp) # save out
mov %r10,24(%rsp) # save real stack pointer
mov %rax,24(%rsp) # save original stack pointer
.cfi_cfa_expression %rsp+24,deref,+8
.Lenc_prologue:
mov %rdx,$key
@@ -644,20 +653,29 @@ AES_encrypt:
mov 16(%rsp),$out # restore out
mov 24(%rsp),%rsi # restore saved stack pointer
.cfi_def_cfa %rsi,8
mov $s0,0($out) # write output vector
mov $s1,4($out)
mov $s2,8($out)
mov $s3,12($out)
mov (%rsi),%r15
mov 8(%rsi),%r14
mov 16(%rsi),%r13
mov 24(%rsi),%r12
mov 32(%rsi),%rbp
mov 40(%rsi),%rbx
lea 48(%rsi),%rsp
mov -48(%rsi),%r15
.cfi_restore %r15
mov -40(%rsi),%r14
.cfi_restore %r14
mov -32(%rsi),%r13
.cfi_restore %r13
mov -24(%rsi),%r12
.cfi_restore %r12
mov -16(%rsi),%rbp
.cfi_restore %rbp
mov -8(%rsi),%rbx
.cfi_restore %rbx
lea (%rsi),%rsp
.cfi_def_cfa_register %rsp
.Lenc_epilogue:
ret
.cfi_endproc
.size AES_encrypt,.-AES_encrypt
___
@@ -1197,15 +1215,23 @@ $code.=<<___;
.hidden asm_AES_decrypt
asm_AES_decrypt:
AES_decrypt:
.cfi_startproc
mov %rsp,%rax
.cfi_def_cfa_register %rax
push %rbx
.cfi_push %rbx
push %rbp
.cfi_push %rbp
push %r12
.cfi_push %r12
push %r13
.cfi_push %r13
push %r14
.cfi_push %r14
push %r15
.cfi_push %r15
# allocate frame "above" key schedule
mov %rsp,%r10
lea -63(%rdx),%rcx # %rdx is key argument
and \$-64,%rsp
sub %rsp,%rcx
@@ -1215,7 +1241,8 @@ AES_decrypt:
sub \$32,%rsp
mov %rsi,16(%rsp) # save out
mov %r10,24(%rsp) # save real stack pointer
mov %rax,24(%rsp) # save original stack pointer
.cfi_cfa_expression %rsp+24,deref,+8
.Ldec_prologue:
mov %rdx,$key
@@ -1244,20 +1271,29 @@ AES_decrypt:
mov 16(%rsp),$out # restore out
mov 24(%rsp),%rsi # restore saved stack pointer
.cfi_def_cfa %rsi,8
mov $s0,0($out) # write output vector
mov $s1,4($out)
mov $s2,8($out)
mov $s3,12($out)
mov (%rsi),%r15
mov 8(%rsi),%r14
mov 16(%rsi),%r13
mov 24(%rsi),%r12
mov 32(%rsi),%rbp
mov 40(%rsi),%rbx
lea 48(%rsi),%rsp
mov -48(%rsi),%r15
.cfi_restore %r15
mov -40(%rsi),%r14
.cfi_restore %r14
mov -32(%rsi),%r13
.cfi_restore %r13
mov -24(%rsi),%r12
.cfi_restore %r12
mov -16(%rsi),%rbp
.cfi_restore %rbp
mov -8(%rsi),%rbx
.cfi_restore %rbx
lea (%rsi),%rsp
.cfi_def_cfa_register %rsp
.Ldec_epilogue:
ret
.cfi_endproc
.size AES_decrypt,.-AES_decrypt
___
#------------------------------------------------------------------#
@@ -1296,22 +1332,34 @@ $code.=<<___;
.type AES_set_encrypt_key,\@function,3
.align 16
AES_set_encrypt_key:
.cfi_startproc
push %rbx
.cfi_push %rbx
push %rbp
push %r12 # redundant, but allows to share
.cfi_push %rbp
push %r12 # redundant, but allows to share
.cfi_push %r12
push %r13 # exception handler...
.cfi_push %r13
push %r14
.cfi_push %r14
push %r15
.cfi_push %r15
sub \$8,%rsp
.cfi_adjust_cfa_offset 8
.Lenc_key_prologue:
call _x86_64_AES_set_encrypt_key
mov 40(%rsp),%rbp
.cfi_restore %rbp
mov 48(%rsp),%rbx
.cfi_restore %rbx
add \$56,%rsp
.cfi_adjust_cfa_offset -56
.Lenc_key_epilogue:
ret
.cfi_endproc
.size AES_set_encrypt_key,.-AES_set_encrypt_key
.type _x86_64_AES_set_encrypt_key,\@abi-omnipotent
@@ -1424,7 +1472,7 @@ $code.=<<___;
xor %rax,%rax
jmp .Lexit
.L14rounds:
.L14rounds:
mov 0(%rsi),%rax # copy first 8 dwords
mov 8(%rsi),%rbx
mov 16(%rsi),%rcx
@@ -1562,13 +1610,21 @@ $code.=<<___;
.type AES_set_decrypt_key,\@function,3
.align 16
AES_set_decrypt_key:
.cfi_startproc
push %rbx
.cfi_push %rbx
push %rbp
.cfi_push %rbp
push %r12
.cfi_push %r12
push %r13
.cfi_push %r13
push %r14
.cfi_push %r14
push %r15
.cfi_push %r15
push %rdx # save key schedule
.cfi_adjust_cfa_offset 8
.Ldec_key_prologue:
call _x86_64_AES_set_encrypt_key
@@ -1622,14 +1678,22 @@ $code.=<<___;
xor %rax,%rax
.Labort:
mov 8(%rsp),%r15
.cfi_restore %r15
mov 16(%rsp),%r14
.cfi_restore %r14
mov 24(%rsp),%r13
.cfi_restore %r13
mov 32(%rsp),%r12
.cfi_restore %r12
mov 40(%rsp),%rbp
.cfi_restore %rbp
mov 48(%rsp),%rbx
.cfi_restore %rbx
add \$56,%rsp
.cfi_adjust_cfa_offset -56
.Ldec_key_epilogue:
ret
.cfi_endproc
.size AES_set_decrypt_key,.-AES_set_decrypt_key
___
@@ -1660,25 +1724,32 @@ $code.=<<___;
.hidden asm_AES_cbc_encrypt
asm_AES_cbc_encrypt:
AES_cbc_encrypt:
.cfi_startproc
cmp \$0,%rdx # check length
je .Lcbc_epilogue
pushfq
.cfi_push 49 # %rflags
push %rbx
.cfi_push %rbx
push %rbp
.cfi_push %rbp
push %r12
.cfi_push %r12
push %r13
.cfi_push %r13
push %r14
.cfi_push %r14
push %r15
.cfi_push %r15
.Lcbc_prologue:
cld
mov %r9d,%r9d # clear upper half of enc
lea .LAES_Te(%rip),$sbox
lea .LAES_Td(%rip),%r10
cmp \$0,%r9
jne .Lcbc_picked_te
lea .LAES_Td(%rip),$sbox
.Lcbc_picked_te:
cmoveq %r10,$sbox
mov OPENSSL_ia32cap_P(%rip),%r10d
cmp \$$speed_limit,%rdx
@@ -1714,8 +1785,10 @@ AES_cbc_encrypt:
.Lcbc_te_ok:
xchg %rsp,$key
.cfi_def_cfa_register $key
#add \$8,%rsp # reserve for return address!
mov $key,$_rsp # save %rsp
.cfi_cfa_expression $_rsp,deref,+64
.Lcbc_fast_body:
mov %rdi,$_inp # save copy of inp
mov %rsi,$_out # save copy of out
@@ -1945,7 +2018,7 @@ AES_cbc_encrypt:
lea ($key,%rax),%rax
mov %rax,$keyend
# pick Te4 copy which can't "overlap" with stack frame or key scdedule
# pick Te4 copy which can't "overlap" with stack frame or key schedule
lea 2048($sbox),$sbox
lea 768-8(%rsp),%rax
sub $sbox,%rax
@@ -2097,17 +2170,27 @@ AES_cbc_encrypt:
.align 16
.Lcbc_exit:
mov $_rsp,%rsi
.cfi_def_cfa %rsi,64
mov (%rsi),%r15
.cfi_restore %r15
mov 8(%rsi),%r14
.cfi_restore %r14
mov 16(%rsi),%r13
.cfi_restore %r13
mov 24(%rsi),%r12
.cfi_restore %r12
mov 32(%rsi),%rbp
.cfi_restore %rbp
mov 40(%rsi),%rbx
.cfi_restore %rbx
lea 48(%rsi),%rsp
.cfi_def_cfa %rsp,16
.Lcbc_popfq:
popfq
.cfi_pop 49 # %rflags
.Lcbc_epilogue:
ret
.cfi_endproc
.size AES_cbc_encrypt,.-AES_cbc_encrypt
___
}
@@ -2580,7 +2663,6 @@ block_se_handler:
jae .Lin_block_prologue
mov 24(%rax),%rax # pull saved real stack pointer
lea 48(%rax),%rax # adjust...
mov -8(%rax),%rbx
mov -16(%rax),%rbp
crypto/aes/asm/aesfx-sparcv9.pl
+1 -1
View File
@@ -22,7 +22,7 @@
# April 2016
#
# Add "teaser" CBC and CTR mode-specific subroutines. "Teaser" means
# that parallelizeable nature of CBC decrypt and CTR is not utilized
# that parallelizable nature of CBC decrypt and CTR is not utilized
# yet. CBC encrypt on the other hand is as good as it can possibly
# get processing one byte in 4.1 cycles with 128-bit key on SPARC64 X.
# This is ~6x faster than pure software implementation...
crypto/aes/asm/aesni-mb-x86_64.pl
+90 -18
View File
@@ -105,6 +105,7 @@ $code.=<<___;
.type aesni_multi_cbc_encrypt,\@function,3
.align 32
aesni_multi_cbc_encrypt:
.cfi_startproc
___
$code.=<<___ if ($avx);
cmp \$2,$num
@@ -118,12 +119,19 @@ $code.=<<___ if ($avx);
___
$code.=<<___;
mov %rsp,%rax
.cfi_def_cfa_register %rax
push %rbx
.cfi_push %rbx
push %rbp
.cfi_push %rbp
push %r12
.cfi_push %r12
push %r13
.cfi_push %r13
push %r14
.cfi_push %r14
push %r15
.cfi_push %r15
___
$code.=<<___ if ($win64);
lea -0xa8(%rsp),%rsp
@@ -134,7 +142,7 @@ $code.=<<___ if ($win64);
movaps %xmm10,0x40(%rsp)
movaps %xmm11,0x50(%rsp)
movaps %xmm12,0x60(%rsp)
movaps %xmm13,-0x68(%rax) # not used, saved to share se_handler
movaps %xmm13,-0x68(%rax) # not used, saved to share se_handler
movaps %xmm14,-0x58(%rax)
movaps %xmm15,-0x48(%rax)
___
@@ -148,6 +156,7 @@ $code.=<<___;
sub \$48,%rsp
and \$-64,%rsp
mov %rax,16(%rsp) # original %rsp
.cfi_cfa_expression %rsp+16,deref,+8
.Lenc4x_body:
movdqu ($key),$zero # 0-round key
@@ -308,9 +317,9 @@ $code.=<<___;
movups @out[0],-16(@outptr[0],$offset)
pxor @inp[0],@out[0]
movups @out[1],-16(@outptr[1],$offset)
movups @out[1],-16(@outptr[1],$offset)
pxor @inp[1],@out[1]
movups @out[2],-16(@outptr[2],$offset)
movups @out[2],-16(@outptr[2],$offset)
pxor @inp[2],@out[2]
movups @out[3],-16(@outptr[3],$offset)
pxor @inp[3],@out[3]
@@ -319,6 +328,7 @@ $code.=<<___;
jnz .Loop_enc4x
mov 16(%rsp),%rax # original %rsp
.cfi_def_cfa %rax,8
mov 24(%rsp),$num
#pxor @inp[0],@out[0]
@@ -350,20 +360,29 @@ $code.=<<___ if ($win64);
___
$code.=<<___;
mov -48(%rax),%r15
.cfi_restore %r15
mov -40(%rax),%r14
.cfi_restore %r14
mov -32(%rax),%r13
.cfi_restore %r13
mov -24(%rax),%r12
.cfi_restore %r12
mov -16(%rax),%rbp
.cfi_restore %rbp
mov -8(%rax),%rbx
.cfi_restore %rbx
lea (%rax),%rsp
.cfi_def_cfa_register %rsp
.Lenc4x_epilogue:
ret
.cfi_endproc
.size aesni_multi_cbc_encrypt,.-aesni_multi_cbc_encrypt
.globl aesni_multi_cbc_decrypt
.type aesni_multi_cbc_decrypt,\@function,3
.align 32
aesni_multi_cbc_decrypt:
.cfi_startproc
___
$code.=<<___ if ($avx);
cmp \$2,$num
@@ -377,12 +396,19 @@ $code.=<<___ if ($avx);
___
$code.=<<___;
mov %rsp,%rax
.cfi_def_cfa_register %rax
push %rbx
.cfi_push %rbx
push %rbp
.cfi_push %rbp
push %r12
.cfi_push %r12
push %r13
.cfi_push %r13
push %r14
.cfi_push %r14
push %r15
.cfi_push %r15
___
$code.=<<___ if ($win64);
lea -0xa8(%rsp),%rsp
@@ -393,7 +419,7 @@ $code.=<<___ if ($win64);
movaps %xmm10,0x40(%rsp)
movaps %xmm11,0x50(%rsp)
movaps %xmm12,0x60(%rsp)
movaps %xmm13,-0x68(%rax) # not used, saved to share se_handler
movaps %xmm13,-0x68(%rax) # not used, saved to share se_handler
movaps %xmm14,-0x58(%rax)
movaps %xmm15,-0x48(%rax)
___
@@ -407,6 +433,7 @@ $code.=<<___;
sub \$48,%rsp
and \$-64,%rsp
mov %rax,16(%rsp) # original %rsp
.cfi_cfa_expression %rsp+16,deref,+8
.Ldec4x_body:
movdqu ($key),$zero # 0-round key
@@ -563,10 +590,10 @@ $code.=<<___;
movups @out[0],-16(@outptr[0],$offset)
movdqu (@inptr[0],$offset),@out[0]
movups @out[1],-16(@outptr[1],$offset)
movups @out[1],-16(@outptr[1],$offset)
movdqu (@inptr[1],$offset),@out[1]
pxor $zero,@out[0]
movups @out[2],-16(@outptr[2],$offset)
movups @out[2],-16(@outptr[2],$offset)
movdqu (@inptr[2],$offset),@out[2]
pxor $zero,@out[1]
movups @out[3],-16(@outptr[3],$offset)
@@ -578,6 +605,7 @@ $code.=<<___;
jnz .Loop_dec4x
mov 16(%rsp),%rax # original %rsp
.cfi_def_cfa %rax,8
mov 24(%rsp),$num
lea `40*4`($inp),$inp
@@ -600,14 +628,22 @@ $code.=<<___ if ($win64);
___
$code.=<<___;
mov -48(%rax),%r15
.cfi_restore %r15
mov -40(%rax),%r14
.cfi_restore %r14
mov -32(%rax),%r13
.cfi_restore %r13
mov -24(%rax),%r12
.cfi_restore %r12
mov -16(%rax),%rbp
.cfi_restore %rbp
mov -8(%rax),%rbx
.cfi_restore %rbx
lea (%rax),%rsp
.cfi_def_cfa_register %rsp
.Ldec4x_epilogue:
ret
.cfi_endproc
.size aesni_multi_cbc_decrypt,.-aesni_multi_cbc_decrypt
___
@@ -623,14 +659,22 @@ $code.=<<___;
.type aesni_multi_cbc_encrypt_avx,\@function,3
.align 32
aesni_multi_cbc_encrypt_avx:
.cfi_startproc
_avx_cbc_enc_shortcut:
mov %rsp,%rax
.cfi_def_cfa_register %rax
push %rbx
.cfi_push %rbx
push %rbp
.cfi_push %rbp
push %r12
.cfi_push %r12
push %r13
.cfi_push %r13
push %r14
.cfi_push %r14
push %r15
.cfi_push %r15
___
$code.=<<___ if ($win64);
lea -0xa8(%rsp),%rsp
@@ -657,6 +701,7 @@ $code.=<<___;
sub \$192,%rsp
and \$-128,%rsp
mov %rax,16(%rsp) # original %rsp
.cfi_cfa_expression %rsp+16,deref,+8
.Lenc8x_body:
vzeroupper
@@ -835,10 +880,10 @@ $code.=<<___;
vmovups @out[0],-16(@ptr[0]) # write output
sub $offset,@ptr[0] # switch to input
vpxor 0x00($offload),@out[0],@out[0]
vmovups @out[1],-16(@ptr[1])
vmovups @out[1],-16(@ptr[1])
sub `64+1*8`(%rsp),@ptr[1]
vpxor 0x10($offload),@out[1],@out[1]
vmovups @out[2],-16(@ptr[2])
vmovups @out[2],-16(@ptr[2])
sub `64+2*8`(%rsp),@ptr[2]
vpxor 0x20($offload),@out[2],@out[2]
vmovups @out[3],-16(@ptr[3])
@@ -847,10 +892,10 @@ $code.=<<___;
vmovups @out[4],-16(@ptr[4])
sub `64+4*8`(%rsp),@ptr[4]
vpxor @inp[0],@out[4],@out[4]
vmovups @out[5],-16(@ptr[5])
vmovups @out[5],-16(@ptr[5])
sub `64+5*8`(%rsp),@ptr[5]
vpxor @inp[1],@out[5],@out[5]
vmovups @out[6],-16(@ptr[6])
vmovups @out[6],-16(@ptr[6])
sub `64+6*8`(%rsp),@ptr[6]
vpxor @inp[2],@out[6],@out[6]
vmovups @out[7],-16(@ptr[7])
@@ -861,6 +906,7 @@ $code.=<<___;
jnz .Loop_enc8x
mov 16(%rsp),%rax # original %rsp
.cfi_def_cfa %rax,8
#mov 24(%rsp),$num
#lea `40*8`($inp),$inp
#dec $num
@@ -883,27 +929,43 @@ $code.=<<___ if ($win64);
___
$code.=<<___;
mov -48(%rax),%r15
.cfi_restore %r15
mov -40(%rax),%r14
.cfi_restore %r14
mov -32(%rax),%r13
.cfi_restore %r13
mov -24(%rax),%r12
.cfi_restore %r12
mov -16(%rax),%rbp
.cfi_restore %rbp
mov -8(%rax),%rbx
.cfi_restore %rbx
lea (%rax),%rsp
.cfi_def_cfa_register %rsp
.Lenc8x_epilogue:
ret
.cfi_endproc
.size aesni_multi_cbc_encrypt_avx,.-aesni_multi_cbc_encrypt_avx
.type aesni_multi_cbc_decrypt_avx,\@function,3
.align 32
aesni_multi_cbc_decrypt_avx:
.cfi_startproc
_avx_cbc_dec_shortcut:
mov %rsp,%rax
.cfi_def_cfa_register %rax
push %rbx
.cfi_push %rbx
push %rbp
.cfi_push %rbp
push %r12
.cfi_push %r12
push %r13
.cfi_push %r13
push %r14
.cfi_push %r14
push %r15
.cfi_push %r15
___
$code.=<<___ if ($win64);
lea -0xa8(%rsp),%rsp
@@ -932,6 +994,7 @@ $code.=<<___;
and \$-256,%rsp
sub \$192,%rsp
mov %rax,16(%rsp) # original %rsp
.cfi_cfa_expression %rsp+16,deref,+8
.Ldec8x_body:
vzeroupper
@@ -1128,12 +1191,12 @@ $code.=<<___;
sub $offset,@ptr[0] # switch to input
vmovdqu 128+0(%rsp),@out[0]
vpxor 0x70($offload),@out[7],@out[7]
vmovups @out[1],-16(@ptr[1])
vmovups @out[1],-16(@ptr[1])
sub `64+1*8`(%rsp),@ptr[1]
vmovdqu @out[0],0x00($offload)
vpxor $zero,@out[0],@out[0]
vmovdqu 128+16(%rsp),@out[1]
vmovups @out[2],-16(@ptr[2])
vmovups @out[2],-16(@ptr[2])
sub `64+2*8`(%rsp),@ptr[2]
vmovdqu @out[1],0x10($offload)
vpxor $zero,@out[1],@out[1]
@@ -1149,11 +1212,11 @@ $code.=<<___;
vpxor $zero,@out[3],@out[3]
vmovdqu @inp[0],0x40($offload)
vpxor @inp[0],$zero,@out[4]
vmovups @out[5],-16(@ptr[5])
vmovups @out[5],-16(@ptr[5])
sub `64+5*8`(%rsp),@ptr[5]
vmovdqu @inp[1],0x50($offload)
vpxor @inp[1],$zero,@out[5]
vmovups @out[6],-16(@ptr[6])
vmovups @out[6],-16(@ptr[6])
sub `64+6*8`(%rsp),@ptr[6]
vmovdqu @inp[2],0x60($offload)
vpxor @inp[2],$zero,@out[6]
@@ -1167,6 +1230,7 @@ $code.=<<___;
jnz .Loop_dec8x
mov 16(%rsp),%rax # original %rsp
.cfi_def_cfa %rax,8
#mov 24(%rsp),$num
#lea `40*8`($inp),$inp
#dec $num
@@ -1189,14 +1253,22 @@ $code.=<<___ if ($win64);
___
$code.=<<___;
mov -48(%rax),%r15
.cfi_restore %r15
mov -40(%rax),%r14
.cfi_restore %r14
mov -32(%rax),%r13
.cfi_restore %r13
mov -24(%rax),%r12
.cfi_restore %r12
mov -16(%rax),%rbp
.cfi_restore %rbp
mov -8(%rax),%rbx
.cfi_restore %rbx
lea (%rax),%rsp
.cfi_def_cfa_register %rsp
.Ldec8x_epilogue:
ret
.cfi_endproc
.size aesni_multi_cbc_decrypt_avx,.-aesni_multi_cbc_decrypt_avx
___
}}}
@@ -1253,10 +1325,10 @@ se_handler:
mov -48(%rax),%r15
mov %rbx,144($context) # restore context->Rbx
mov %rbp,160($context) # restore context->Rbp
mov %r12,216($context) # restore cotnext->R12
mov %r13,224($context) # restore cotnext->R13
mov %r14,232($context) # restore cotnext->R14
mov %r15,240($context) # restore cotnext->R15
mov %r12,216($context) # restore context->R12
mov %r13,224($context) # restore context->R13
mov %r14,232($context) # restore context->R14
mov %r15,240($context) # restore context->R15
lea -56-10*16(%rax),%rsi
lea 512($context),%rdi # &context.Xmm6
crypto/aes/asm/aesni-sha1-x86_64.pl
+69 -1
View File
@@ -191,16 +191,24 @@ $code.=<<___;
.type aesni_cbc_sha1_enc_ssse3,\@function,6
.align 32
aesni_cbc_sha1_enc_ssse3:
.cfi_startproc
mov `($win64?56:8)`(%rsp),$inp # load 7th argument
#shr \$6,$len # debugging artefact
#jz .Lepilogue_ssse3 # debugging artefact
push %rbx
.cfi_push %rbx
push %rbp
.cfi_push %rbp
push %r12
.cfi_push %r12
push %r13
.cfi_push %r13
push %r14
.cfi_push %r14
push %r15
.cfi_push %r15
lea `-104-($win64?10*16:0)`(%rsp),%rsp
.cfi_adjust_cfa_offset `104+($win64?10*16:0)`
#mov $in0,$inp # debugging artefact
#lea 64(%rsp),$ctx # debugging artefact
___
@@ -726,15 +734,24 @@ $code.=<<___ if ($win64);
___
$code.=<<___;
lea `104+($win64?10*16:0)`(%rsp),%rsi
.cfi_def_cfa %rsi,56
mov 0(%rsi),%r15
.cfi_restore %r15
mov 8(%rsi),%r14
.cfi_restore %r14
mov 16(%rsi),%r13
.cfi_restore %r13
mov 24(%rsi),%r12
.cfi_restore %r12
mov 32(%rsi),%rbp
.cfi_restore %rbp
mov 40(%rsi),%rbx
.cfi_restore %rbx
lea 48(%rsi),%rsp
.cfi_def_cfa %rsp,8
.Lepilogue_ssse3:
ret
.cfi_endproc
.size aesni_cbc_sha1_enc_ssse3,.-aesni_cbc_sha1_enc_ssse3
___
@@ -798,7 +815,7 @@ sub body_00_19_dec () { # ((c^d)&b)^d
sub body_20_39_dec () { # b^d^c
# on entry @T[0]=b^d
return &body_40_59_dec() if ($rx==39);
my @r=@body_20_39;
unshift (@r,@aes256_dec[$rx]) if (@aes256_dec[$rx]);
@@ -842,14 +859,22 @@ $code.=<<___;
.type aesni256_cbc_sha1_dec_ssse3,\@function,6
.align 32
aesni256_cbc_sha1_dec_ssse3:
.cfi_startproc
mov `($win64?56:8)`(%rsp),$inp # load 7th argument
push %rbx
.cfi_push %rbx
push %rbp
.cfi_push %rbp
push %r12
.cfi_push %r12
push %r13
.cfi_push %r13
push %r14
.cfi_push %r14
push %r15
.cfi_push %r15
lea `-104-($win64?10*16:0)`(%rsp),%rsp
.cfi_adjust_cfa_offset `104+($win64?10*16:0)`
___
$code.=<<___ if ($win64);
movaps %xmm6,96+0(%rsp)
@@ -997,15 +1022,24 @@ $code.=<<___ if ($win64);
___
$code.=<<___;
lea `104+($win64?10*16:0)`(%rsp),%rsi
.cfi_cfa_def %rsi,56
mov 0(%rsi),%r15
.cfi_restore %r15
mov 8(%rsi),%r14
.cfi_restore %r14
mov 16(%rsi),%r13
.cfi_restore %r13
mov 24(%rsi),%r12
.cfi_restore %r12
mov 32(%rsi),%rbp
.cfi_restore %rbp
mov 40(%rsi),%rbx
.cfi_restore %rbx
lea 48(%rsi),%rsp
.cfi_cfa_def %rsp,8
.Lepilogue_dec_ssse3:
ret
.cfi_endproc
.size aesni256_cbc_sha1_dec_ssse3,.-aesni256_cbc_sha1_dec_ssse3
___
}}}
@@ -1031,16 +1065,24 @@ $code.=<<___;
.type aesni_cbc_sha1_enc_avx,\@function,6
.align 32
aesni_cbc_sha1_enc_avx:
.cfi_startproc
mov `($win64?56:8)`(%rsp),$inp # load 7th argument
#shr \$6,$len # debugging artefact
#jz .Lepilogue_avx # debugging artefact
push %rbx
.cfi_push %rbx
push %rbp
.cfi_push %rbp
push %r12
.cfi_push %r12
push %r13
.cfi_push %r13
push %r14
.cfi_push %r14
push %r15
.cfi_push %r15
lea `-104-($win64?10*16:0)`(%rsp),%rsp
.cfi_adjust_cfa_offset `104+($win64?10*16:0)`
#mov $in0,$inp # debugging artefact
#lea 64(%rsp),$ctx # debugging artefact
___
@@ -1439,15 +1481,24 @@ $code.=<<___ if ($win64);
___
$code.=<<___;
lea `104+($win64?10*16:0)`(%rsp),%rsi
.cfi_def_cfa %rsi,56
mov 0(%rsi),%r15
.cfi_restore %r15
mov 8(%rsi),%r14
.cfi_restore %r14
mov 16(%rsi),%r13
.cfi_restore %r13
mov 24(%rsi),%r12
.cfi_restore %r12
mov 32(%rsi),%rbp
.cfi_restore %rbp
mov 40(%rsi),%rbx
.cfi_restore %rbx
lea 48(%rsi),%rsp
.cfi_def_cfa %rsp,8
.Lepilogue_avx:
ret
.cfi_endproc
.size aesni_cbc_sha1_enc_avx,.-aesni_cbc_sha1_enc_avx
___
@@ -1496,14 +1547,22 @@ $code.=<<___;
.type aesni256_cbc_sha1_dec_avx,\@function,6
.align 32
aesni256_cbc_sha1_dec_avx:
.cfi_startproc
mov `($win64?56:8)`(%rsp),$inp # load 7th argument
push %rbx
.cfi_push %rbx
push %rbp
.cfi_push %rbp
push %r12
.cfi_push %r12
push %r13
.cfi_push %r13
push %r14
.cfi_push %r14
push %r15
.cfi_push %r15
lea `-104-($win64?10*16:0)`(%rsp),%rsp
.cfi_adjust_cfa_offset `104+($win64?10*16:0)`
___
$code.=<<___ if ($win64);
movaps %xmm6,96+0(%rsp)
@@ -1650,15 +1709,24 @@ $code.=<<___ if ($win64);
___
$code.=<<___;
lea `104+($win64?10*16:0)`(%rsp),%rsi
.cfi_def_cfa %rsi,56
mov 0(%rsi),%r15
.cfi_restore %r15
mov 8(%rsi),%r14
.cfi_restore %r14
mov 16(%rsi),%r13
.cfi_restore %r13
mov 24(%rsi),%r12
.cfi_restore %r12
mov 32(%rsi),%rbp
.cfi_restore %rbp
mov 40(%rsi),%rbx
.cfi_restore %rbx
lea 48(%rsi),%rsp
.cfi_def_cfa %rsp,8
.Lepilogue_dec_avx:
ret
.cfi_endproc
.size aesni256_cbc_sha1_dec_avx,.-aesni256_cbc_sha1_dec_avx
___
}}}
crypto/aes/asm/aesni-sha256-x86_64.pl
+83 -30
View File
@@ -112,7 +112,7 @@ $_key="16*$SZ+3*8(%rsp)";
$_ivp="16*$SZ+4*8(%rsp)";
$_ctx="16*$SZ+5*8(%rsp)";
$_in0="16*$SZ+6*8(%rsp)";
$_rsp="16*$SZ+7*8(%rsp)";
$_rsp="`16*$SZ+7*8`(%rsp)";
$framesz=16*$SZ+8*8;
$code=<<___;
@@ -342,15 +342,23 @@ $code.=<<___;
.type ${func}_xop,\@function,6
.align 64
${func}_xop:
.cfi_startproc
.Lxop_shortcut:
mov `($win64?56:8)`(%rsp),$in0 # load 7th parameter
mov %rsp,%rax # copy %rsp
.cfi_def_cfa_register %rax
push %rbx
.cfi_push %rbx
push %rbp
.cfi_push %rbp
push %r12
.cfi_push %r12
push %r13
.cfi_push %r13
push %r14
.cfi_push %r14
push %r15
mov %rsp,%r11 # copy %rsp
.cfi_push %r15
sub \$`$framesz+$win64*16*10`,%rsp
and \$-64,%rsp # align stack frame
@@ -366,7 +374,8 @@ ${func}_xop:
mov $ivp,$_ivp
mov $ctx,$_ctx
mov $in0,$_in0
mov %r11,$_rsp
mov %rax,$_rsp
.cfi_cfa_expression $_rsp,deref,+8
___
$code.=<<___ if ($win64);
movaps %xmm6,`$framesz+16*0`(%rsp)
@@ -604,6 +613,7 @@ $code.=<<___;
mov $_ivp,$ivp
mov $_rsp,%rsi
.cfi_def_cfa %rsi,8
vmovdqu $iv,($ivp) # output IV
vzeroall
___
@@ -620,15 +630,23 @@ $code.=<<___ if ($win64);
movaps `$framesz+16*9`(%rsp),%xmm15
___
$code.=<<___;
mov (%rsi),%r15
mov 8(%rsi),%r14
mov 16(%rsi),%r13
mov 24(%rsi),%r12
mov 32(%rsi),%rbp
mov 40(%rsi),%rbx
lea 48(%rsi),%rsp
mov -48(%rsi),%r15
.cfi_restore %r15
mov -40(%rsi),%r14
.cfi_restore %r14
mov -32(%rsi),%r13
.cfi_restore %r13
mov -24(%rsi),%r12
.cfi_restore %r12
mov -16(%rsi),%rbp
.cfi_restore %rbp
mov -8(%rsi),%rbx
.cfi_restore %rbx
lea (%rsi),%rsp
.cfi_def_cfa_register %rsp
.Lepilogue_xop:
ret
.cfi_endproc
.size ${func}_xop,.-${func}_xop
___
######################################################################
@@ -640,15 +658,23 @@ $code.=<<___;
.type ${func}_avx,\@function,6
.align 64
${func}_avx:
.cfi_startproc
.Lavx_shortcut:
mov `($win64?56:8)`(%rsp),$in0 # load 7th parameter
mov %rsp,%rax # copy %rsp
.cfi_def_cfa_register %rax
push %rbx
.cfi_push %rbx
push %rbp
.cfi_push %rbp
push %r12
.cfi_push %r12
push %r13
.cfi_push %r13
push %r14
.cfi_push %r14
push %r15
mov %rsp,%r11 # copy %rsp
.cfi_push %r15
sub \$`$framesz+$win64*16*10`,%rsp
and \$-64,%rsp # align stack frame
@@ -664,7 +690,8 @@ ${func}_avx:
mov $ivp,$_ivp
mov $ctx,$_ctx
mov $in0,$_in0
mov %r11,$_rsp
mov %rax,$_rsp
.cfi_cfa_expression $_rsp,deref,+8
___
$code.=<<___ if ($win64);
movaps %xmm6,`$framesz+16*0`(%rsp)
@@ -855,6 +882,7 @@ $code.=<<___;
mov $_ivp,$ivp
mov $_rsp,%rsi
.cfi_def_cfa %rsi,8
vmovdqu $iv,($ivp) # output IV
vzeroall
___
@@ -871,15 +899,23 @@ $code.=<<___ if ($win64);
movaps `$framesz+16*9`(%rsp),%xmm15
___
$code.=<<___;
mov (%rsi),%r15
mov 8(%rsi),%r14
mov 16(%rsi),%r13
mov 24(%rsi),%r12
mov 32(%rsi),%rbp
mov 40(%rsi),%rbx
lea 48(%rsi),%rsp
mov -48(%rsi),%r15
.cfi_restore %r15
mov -40(%rsi),%r14
.cfi_restore %r14
mov -32(%rsi),%r13
.cfi_restore %r13
mov -24(%rsi),%r12
.cfi_restore %r12
mov -16(%rsi),%rbp
.cfi_restore %rbp
mov -8(%rsi),%rbx
.cfi_restore %rbx
lea (%rsi),%rsp
.cfi_def_cfa_register %rsp
.Lepilogue_avx:
ret
.cfi_endproc
.size ${func}_avx,.-${func}_avx
___
@@ -887,7 +923,7 @@ if ($avx>1) {{
######################################################################
# AVX2+BMI code path
#
my $a5=$SZ==4?"%esi":"%rsi"; # zap $inp
my $a5=$SZ==4?"%esi":"%rsi"; # zap $inp
my $PUSH8=8*2*$SZ;
use integer;
@@ -936,15 +972,23 @@ $code.=<<___;
.type ${func}_avx2,\@function,6
.align 64
${func}_avx2:
.cfi_startproc
.Lavx2_shortcut:
mov `($win64?56:8)`(%rsp),$in0 # load 7th parameter
mov %rsp,%rax # copy %rsp
.cfi_def_cfa_register %rax
push %rbx
.cfi_push %rbx
push %rbp
.cfi_push %rbp
push %r12
.cfi_push %r12
push %r13
.cfi_push %r13
push %r14
.cfi_push %r14
push %r15
mov %rsp,%r11 # copy %rsp
.cfi_push %r15
sub \$`2*$SZ*$rounds+8*8+$win64*16*10`,%rsp
and \$-256*$SZ,%rsp # align stack frame
add \$`2*$SZ*($rounds-8)`,%rsp
@@ -961,7 +1005,8 @@ ${func}_avx2:
mov $ivp,$_ivp
mov $ctx,$_ctx
mov $in0,$_in0
mov %r11,$_rsp
mov %rax,$_rsp
.cfi_cfa_expression $_rsp,deref,+8
___
$code.=<<___ if ($win64);
movaps %xmm6,`$framesz+16*0`(%rsp)
@@ -1192,6 +1237,7 @@ $code.=<<___;
lea ($Tbl),%rsp
mov $_ivp,$ivp
mov $_rsp,%rsi
.cfi_def_cfa %rsi,8
vmovdqu $iv,($ivp) # output IV
vzeroall
___
@@ -1208,15 +1254,23 @@ $code.=<<___ if ($win64);
movaps `$framesz+16*9`(%rsp),%xmm15
___
$code.=<<___;
mov (%rsi),%r15
mov 8(%rsi),%r14
mov 16(%rsi),%r13
mov 24(%rsi),%r12
mov 32(%rsi),%rbp
mov 40(%rsi),%rbx
lea 48(%rsi),%rsp
mov -48(%rsi),%r15
.cfi_restore %r15
mov -40(%rsi),%r14
.cfi_restore %r14
mov -32(%rsi),%r13
.cfi_restore %r13
mov -24(%rsi),%r12
.cfi_restore %r12
mov -16(%rsi),%rbp
.cfi_restore %rbp
mov -8(%rsi),%rbx
.cfi_restore %rbx
lea (%rsi),%rsp
.cfi_def_cfa_register %rsp
.Lepilogue_avx2:
ret
.cfi_endproc
.size ${func}_avx2,.-${func}_avx2
___
}}
@@ -1573,7 +1627,6 @@ ___
$code.=<<___;
mov %rax,%rsi # put aside Rsp
mov 16*$SZ+7*8(%rax),%rax # pull $_rsp
lea 48(%rax),%rax
mov -8(%rax),%rbx
mov -16(%rax),%rbp
crypto/aes/asm/aesni-x86.pl
+5 -3
View File
@@ -62,7 +62,9 @@
# Westmere 3.77/1.37 1.37 1.52 1.27
# * Bridge 5.07/0.98 0.99 1.09 0.91 1.10
# Haswell 4.44/0.80 0.97 1.03 0.72 0.76
# Skylake 2.68/0.65 0.65 0.66 0.64 0.66
# Silvermont 5.77/3.56 3.67 4.03 3.46 4.03
# Goldmont 3.84/1.39 1.39 1.63 1.31 1.70
# Bulldozer 5.80/0.98 1.05 1.24 0.93 1.23
$PREFIX="aesni"; # if $PREFIX is set to "AES", the script
@@ -78,7 +80,7 @@ $output = pop;
open OUT,">$output";
*STDOUT=*OUT;
&asm_init($ARGV[0],$0);
&asm_init($ARGV[0]);
&external_label("OPENSSL_ia32cap_P");
&static_label("key_const");
@@ -237,7 +239,7 @@ sub aesni_generate1 # fully unrolled loop
# can schedule aes[enc|dec] every cycle optimal interleave factor
# equals to corresponding instructions latency. 8x is optimal for
# * Bridge, but it's unfeasible to accommodate such implementation
# in XMM registers addreassable in 32-bit mode and therefore maximum
# in XMM registers addressable in 32-bit mode and therefore maximum
# of 6x is used instead...
sub aesni_generate2
@@ -1051,7 +1053,7 @@ if ($PREFIX eq "aesni") {
&set_label("ctr32_one_shortcut",16);
&movups ($inout0,&QWP(0,$rounds_)); # load ivec
&mov ($rounds,&DWP(240,$key));
&set_label("ctr32_one");
if ($inline)
{ &aesni_inline_generate1("enc"); }
crypto/aes/asm/aesni-x86_64.pl
+217 -152
View File
@@ -34,7 +34,7 @@
# ECB 4.25/4.25 1.38/1.38 1.28/1.28 1.26/1.26 1.26/1.26
# CTR 5.42/5.42 1.92/1.92 1.44/1.44 1.28/1.28 1.26/1.26
# CBC 4.38/4.43 4.15/1.43 4.07/1.32 4.07/1.29 4.06/1.28
# CCM 5.66/9.42 4.42/5.41 4.16/4.40 4.09/4.15 4.06/4.07
# CCM 5.66/9.42 4.42/5.41 4.16/4.40 4.09/4.15 4.06/4.07
# OFB 5.42/5.42 4.64/4.64 4.44/4.44 4.39/4.39 4.38/4.38
# CFB 5.73/5.85 5.56/5.62 5.48/5.56 5.47/5.55 5.47/5.55
#
@@ -60,7 +60,7 @@
# identical to CBC, because CBC-MAC is essentially CBC encrypt without
# saving output. CCM CTR "stays invisible," because it's neatly
# interleaved wih CBC-MAC. This provides ~30% improvement over
# "straghtforward" CCM implementation with CTR and CBC-MAC performed
# "straightforward" CCM implementation with CTR and CBC-MAC performed
# disjointly. Parallelizable modes practically achieve the theoretical
# limit.
#
@@ -118,7 +118,7 @@
# performance is achieved by interleaving instructions working on
# independent blocks. In which case asymptotic limit for such modes
# can be obtained by dividing above mentioned numbers by AES
# instructions' interleave factor. Westmere can execute at most 3
# instructions' interleave factor. Westmere can execute at most 3
# instructions at a time, meaning that optimal interleave factor is 3,
# and that's where the "magic" number of 1.25 come from. "Optimal
# interleave factor" means that increase of interleave factor does
@@ -143,14 +143,14 @@
# asymptotic, if it can be surpassed, isn't it? What happens there?
# Rewind to CBC paragraph for the answer. Yes, out-of-order execution
# magic is responsible for this. Processor overlaps not only the
# additional instructions with AES ones, but even AES instuctions
# additional instructions with AES ones, but even AES instructions
# processing adjacent triplets of independent blocks. In the 6x case
# additional instructions still claim disproportionally small amount
# of additional cycles, but in 8x case number of instructions must be
# a tad too high for out-of-order logic to cope with, and AES unit
# remains underutilized... As you can see 8x interleave is hardly
# justifiable, so there no need to feel bad that 32-bit aesni-x86.pl
# utilizies 6x interleave because of limited register bank capacity.
# utilizes 6x interleave because of limited register bank capacity.
#
# Higher interleave factors do have negative impact on Westmere
# performance. While for ECB mode it's negligible ~1.5%, other
@@ -179,8 +179,10 @@
# Haswell 4.44/0.63 0.63 0.73 0.63 0.70
# Skylake 2.62/0.63 0.63 0.63 0.63
# Silvermont 5.75/3.54 3.56 4.12 3.87(*) 4.11
# Knights L 2.54/0.77 0.78 0.85 - 1.50
# Goldmont 3.82/1.26 1.26 1.29 1.29 1.50
# Bulldozer 5.77/0.70 0.72 0.90 0.70 0.95
# Ryzen 2.71/0.35 0.35 0.44 0.38 0.49
#
# (*) Atom Silvermont ECB result is suboptimal because of penalties
# incurred by operations on %xmm8-15. As ECB is not considered
@@ -313,7 +315,7 @@ ___
# on 2x subroutine on Atom Silvermont account. For processors that
# can schedule aes[enc|dec] every cycle optimal interleave factor
# equals to corresponding instructions latency. 8x is optimal for
# * Bridge and "super-optimal" for other Intel CPUs...
# * Bridge and "super-optimal" for other Intel CPUs...
sub aesni_generate2 {
my $dir=shift;
@@ -1172,7 +1174,7 @@ ___
# with zero-round key xor.
{
my ($in0,$in1,$in2,$in3,$in4,$in5)=map("%xmm$_",(10..15));
my ($key0,$ctr)=("${key_}d","${ivp}d");
my ($key0,$ctr)=("%ebp","${ivp}d");
my $frame_size = 0x80 + ($win64?160:0);
$code.=<<___;
@@ -1180,6 +1182,7 @@ $code.=<<___;
.type aesni_ctr32_encrypt_blocks,\@function,5
.align 16
aesni_ctr32_encrypt_blocks:
.cfi_startproc
cmp \$1,$len
jne .Lctr32_bulk
@@ -1201,26 +1204,27 @@ $code.=<<___;
.align 16
.Lctr32_bulk:
lea (%rsp),%rax
lea (%rsp),$key_ # use $key_ as frame pointer
.cfi_def_cfa_register $key_
push %rbp
.cfi_push %rbp
sub \$$frame_size,%rsp
and \$-16,%rsp # Linux kernel stack can be incorrectly seeded
___
$code.=<<___ if ($win64);
movaps %xmm6,-0xa8(%rax) # offload everything
movaps %xmm7,-0x98(%rax)
movaps %xmm8,-0x88(%rax)
movaps %xmm9,-0x78(%rax)
movaps %xmm10,-0x68(%rax)
movaps %xmm11,-0x58(%rax)
movaps %xmm12,-0x48(%rax)
movaps %xmm13,-0x38(%rax)
movaps %xmm14,-0x28(%rax)
movaps %xmm15,-0x18(%rax)
movaps %xmm6,-0xa8($key_) # offload everything
movaps %xmm7,-0x98($key_)
movaps %xmm8,-0x88($key_)
movaps %xmm9,-0x78($key_)
movaps %xmm10,-0x68($key_)
movaps %xmm11,-0x58($key_)
movaps %xmm12,-0x48($key_)
movaps %xmm13,-0x38($key_)
movaps %xmm14,-0x28($key_)
movaps %xmm15,-0x18($key_)
.Lctr32_body:
___
$code.=<<___;
lea -8(%rax),%rbp
# 8 16-byte words on top of stack are counter values
# xor-ed with zero-round key
@@ -1272,7 +1276,7 @@ $code.=<<___;
lea 7($ctr),%r9
mov %r10d,0x60+12(%rsp)
bswap %r9d
mov OPENSSL_ia32cap_P+4(%rip),%r10d
mov OPENSSL_ia32cap_P+4(%rip),%r10d
xor $key0,%r9d
and \$`1<<26|1<<22`,%r10d # isolate XSAVE+MOVBE
mov %r9d,0x70+12(%rsp)
@@ -1546,13 +1550,13 @@ $code.=<<___;
sub \$8,$len
jnc .Lctr32_loop8 # loop if $len-=8 didn't borrow
add \$8,$len # restore real remainig $len
add \$8,$len # restore real remaining $len
jz .Lctr32_done # done if ($len==0)
lea -0x80($key),$key
.Lctr32_tail:
# note that at this point $inout0..5 are populated with
# counter values xor-ed with 0-round key
# counter values xor-ed with 0-round key
lea 16($key),$key
cmp \$4,$len
jb .Lctr32_loop3
@@ -1663,7 +1667,7 @@ $code.=<<___;
movups $inout2,0x20($out) # $len was 3, stop store
.Lctr32_done:
xorps %xmm0,%xmm0 # clear regiser bank
xorps %xmm0,%xmm0 # clear register bank
xor $key0,$key0
pxor %xmm1,%xmm1
pxor %xmm2,%xmm2
@@ -1692,26 +1696,26 @@ $code.=<<___ if (!$win64);
pxor %xmm15,%xmm15
___
$code.=<<___ if ($win64);
movaps -0xa0(%rbp),%xmm6
movaps %xmm0,-0xa0(%rbp) # clear stack
movaps -0x90(%rbp),%xmm7
movaps %xmm0,-0x90(%rbp)
movaps -0x80(%rbp),%xmm8
movaps %xmm0,-0x80(%rbp)
movaps -0x70(%rbp),%xmm9
movaps %xmm0,-0x70(%rbp)
movaps -0x60(%rbp),%xmm10
movaps %xmm0,-0x60(%rbp)
movaps -0x50(%rbp),%xmm11
movaps %xmm0,-0x50(%rbp)
movaps -0x40(%rbp),%xmm12
movaps %xmm0,-0x40(%rbp)
movaps -0x30(%rbp),%xmm13
movaps %xmm0,-0x30(%rbp)
movaps -0x20(%rbp),%xmm14
movaps %xmm0,-0x20(%rbp)
movaps -0x10(%rbp),%xmm15
movaps %xmm0,-0x10(%rbp)
movaps -0xa8($key_),%xmm6
movaps %xmm0,-0xa8($key_) # clear stack
movaps -0x98($key_),%xmm7
movaps %xmm0,-0x98($key_)
movaps -0x88($key_),%xmm8
movaps %xmm0,-0x88($key_)
movaps -0x78($key_),%xmm9
movaps %xmm0,-0x78($key_)
movaps -0x68($key_),%xmm10
movaps %xmm0,-0x68($key_)
movaps -0x58($key_),%xmm11
movaps %xmm0,-0x58($key_)
movaps -0x48($key_),%xmm12
movaps %xmm0,-0x48($key_)
movaps -0x38($key_),%xmm13
movaps %xmm0,-0x38($key_)
movaps -0x28($key_),%xmm14
movaps %xmm0,-0x28($key_)
movaps -0x18($key_),%xmm15
movaps %xmm0,-0x18($key_)
movaps %xmm0,0x00(%rsp)
movaps %xmm0,0x10(%rsp)
movaps %xmm0,0x20(%rsp)
@@ -1722,10 +1726,13 @@ $code.=<<___ if ($win64);
movaps %xmm0,0x70(%rsp)
___
$code.=<<___;
lea (%rbp),%rsp
pop %rbp
mov -8($key_),%rbp
.cfi_restore %rbp
lea ($key_),%rsp
.cfi_def_cfa_register %rsp
.Lctr32_epilogue:
ret
.cfi_endproc
.size aesni_ctr32_encrypt_blocks,.-aesni_ctr32_encrypt_blocks
___
}
@@ -1740,32 +1747,35 @@ my @tweak=map("%xmm$_",(10..15));
my ($twmask,$twres,$twtmp)=("%xmm8","%xmm9",@tweak[4]);
my ($key2,$ivp,$len_)=("%r8","%r9","%r9");
my $frame_size = 0x70 + ($win64?160:0);
my $key_ = "%rbp"; # override so that we can use %r11 as FP
$code.=<<___;
.globl aesni_xts_encrypt
.type aesni_xts_encrypt,\@function,6
.align 16
aesni_xts_encrypt:
lea (%rsp),%rax
.cfi_startproc
lea (%rsp),%r11 # frame pointer
.cfi_def_cfa_register %r11
push %rbp
.cfi_push %rbp
sub \$$frame_size,%rsp
and \$-16,%rsp # Linux kernel stack can be incorrectly seeded
___
$code.=<<___ if ($win64);
movaps %xmm6,-0xa8(%rax) # offload everything
movaps %xmm7,-0x98(%rax)
movaps %xmm8,-0x88(%rax)
movaps %xmm9,-0x78(%rax)
movaps %xmm10,-0x68(%rax)
movaps %xmm11,-0x58(%rax)
movaps %xmm12,-0x48(%rax)
movaps %xmm13,-0x38(%rax)
movaps %xmm14,-0x28(%rax)
movaps %xmm15,-0x18(%rax)
movaps %xmm6,-0xa8(%r11) # offload everything
movaps %xmm7,-0x98(%r11)
movaps %xmm8,-0x88(%r11)
movaps %xmm9,-0x78(%r11)
movaps %xmm10,-0x68(%r11)
movaps %xmm11,-0x58(%r11)
movaps %xmm12,-0x48(%r11)
movaps %xmm13,-0x38(%r11)
movaps %xmm14,-0x28(%r11)
movaps %xmm15,-0x18(%r11)
.Lxts_enc_body:
___
$code.=<<___;
lea -8(%rax),%rbp
movups ($ivp),$inout0 # load clear-text tweak
mov 240(%r8),$rounds # key2->rounds
mov 240($key),$rnds_ # key1->rounds
@@ -1846,7 +1856,7 @@ $code.=<<___;
lea `16*6`($inp),$inp
pxor $twmask,$inout5
pxor $twres,@tweak[0] # calclulate tweaks^round[last]
pxor $twres,@tweak[0] # calculate tweaks^round[last]
aesenc $rndkey1,$inout4
pxor $twres,@tweak[1]
movdqa @tweak[0],`16*0`(%rsp) # put aside tweaks^round[last]
@@ -2183,26 +2193,26 @@ $code.=<<___ if (!$win64);
pxor %xmm15,%xmm15
___
$code.=<<___ if ($win64);
movaps -0xa0(%rbp),%xmm6
movaps %xmm0,-0xa0(%rbp) # clear stack
movaps -0x90(%rbp),%xmm7
movaps %xmm0,-0x90(%rbp)
movaps -0x80(%rbp),%xmm8
movaps %xmm0,-0x80(%rbp)
movaps -0x70(%rbp),%xmm9
movaps %xmm0,-0x70(%rbp)
movaps -0x60(%rbp),%xmm10
movaps %xmm0,-0x60(%rbp)
movaps -0x50(%rbp),%xmm11
movaps %xmm0,-0x50(%rbp)
movaps -0x40(%rbp),%xmm12
movaps %xmm0,-0x40(%rbp)
movaps -0x30(%rbp),%xmm13
movaps %xmm0,-0x30(%rbp)
movaps -0x20(%rbp),%xmm14
movaps %xmm0,-0x20(%rbp)
movaps -0x10(%rbp),%xmm15
movaps %xmm0,-0x10(%rbp)
movaps -0xa8(%r11),%xmm6
movaps %xmm0,-0xa8(%r11) # clear stack
movaps -0x98(%r11),%xmm7
movaps %xmm0,-0x98(%r11)
movaps -0x88(%r11),%xmm8
movaps %xmm0,-0x88(%r11)
movaps -0x78(%r11),%xmm9
movaps %xmm0,-0x78(%r11)
movaps -0x68(%r11),%xmm10
movaps %xmm0,-0x68(%r11)
movaps -0x58(%r11),%xmm11
movaps %xmm0,-0x58(%r11)
movaps -0x48(%r11),%xmm12
movaps %xmm0,-0x48(%r11)
movaps -0x38(%r11),%xmm13
movaps %xmm0,-0x38(%r11)
movaps -0x28(%r11),%xmm14
movaps %xmm0,-0x28(%r11)
movaps -0x18(%r11),%xmm15
movaps %xmm0,-0x18(%r11)
movaps %xmm0,0x00(%rsp)
movaps %xmm0,0x10(%rsp)
movaps %xmm0,0x20(%rsp)
@@ -2212,10 +2222,13 @@ $code.=<<___ if ($win64);
movaps %xmm0,0x60(%rsp)
___
$code.=<<___;
lea (%rbp),%rsp
pop %rbp
mov -8(%r11),%rbp
.cfi_restore %rbp
lea (%r11),%rsp
.cfi_def_cfa_register %rsp
.Lxts_enc_epilogue:
ret
.cfi_endproc
.size aesni_xts_encrypt,.-aesni_xts_encrypt
___
@@ -2224,26 +2237,28 @@ $code.=<<___;
.type aesni_xts_decrypt,\@function,6
.align 16
aesni_xts_decrypt:
lea (%rsp),%rax
.cfi_startproc
lea (%rsp),%r11 # frame pointer
.cfi_def_cfa_register %r11
push %rbp
.cfi_push %rbp
sub \$$frame_size,%rsp
and \$-16,%rsp # Linux kernel stack can be incorrectly seeded
___
$code.=<<___ if ($win64);
movaps %xmm6,-0xa8(%rax) # offload everything
movaps %xmm7,-0x98(%rax)
movaps %xmm8,-0x88(%rax)
movaps %xmm9,-0x78(%rax)
movaps %xmm10,-0x68(%rax)
movaps %xmm11,-0x58(%rax)
movaps %xmm12,-0x48(%rax)
movaps %xmm13,-0x38(%rax)
movaps %xmm14,-0x28(%rax)
movaps %xmm15,-0x18(%rax)
movaps %xmm6,-0xa8(%r11) # offload everything
movaps %xmm7,-0x98(%r11)
movaps %xmm8,-0x88(%r11)
movaps %xmm9,-0x78(%r11)
movaps %xmm10,-0x68(%r11)
movaps %xmm11,-0x58(%r11)
movaps %xmm12,-0x48(%r11)
movaps %xmm13,-0x38(%r11)
movaps %xmm14,-0x28(%r11)
movaps %xmm15,-0x18(%r11)
.Lxts_dec_body:
___
$code.=<<___;
lea -8(%rax),%rbp
movups ($ivp),$inout0 # load clear-text tweak
mov 240($key2),$rounds # key2->rounds
mov 240($key),$rnds_ # key1->rounds
@@ -2327,7 +2342,7 @@ $code.=<<___;
lea `16*6`($inp),$inp
pxor $twmask,$inout5
pxor $twres,@tweak[0] # calclulate tweaks^round[last]
pxor $twres,@tweak[0] # calculate tweaks^round[last]
aesdec $rndkey1,$inout4
pxor $twres,@tweak[1]
movdqa @tweak[0],`16*0`(%rsp) # put aside tweaks^last round key
@@ -2687,26 +2702,26 @@ $code.=<<___ if (!$win64);
pxor %xmm15,%xmm15
___
$code.=<<___ if ($win64);
movaps -0xa0(%rbp),%xmm6
movaps %xmm0,-0xa0(%rbp) # clear stack
movaps -0x90(%rbp),%xmm7
movaps %xmm0,-0x90(%rbp)
movaps -0x80(%rbp),%xmm8
movaps %xmm0,-0x80(%rbp)
movaps -0x70(%rbp),%xmm9
movaps %xmm0,-0x70(%rbp)
movaps -0x60(%rbp),%xmm10
movaps %xmm0,-0x60(%rbp)
movaps -0x50(%rbp),%xmm11
movaps %xmm0,-0x50(%rbp)
movaps -0x40(%rbp),%xmm12
movaps %xmm0,-0x40(%rbp)
movaps -0x30(%rbp),%xmm13
movaps %xmm0,-0x30(%rbp)
movaps -0x20(%rbp),%xmm14
movaps %xmm0,-0x20(%rbp)
movaps -0x10(%rbp),%xmm15
movaps %xmm0,-0x10(%rbp)
movaps -0xa8(%r11),%xmm6
movaps %xmm0,-0xa8(%r11) # clear stack
movaps -0x98(%r11),%xmm7
movaps %xmm0,-0x98(%r11)
movaps -0x88(%r11),%xmm8
movaps %xmm0,-0x88(%r11)
movaps -0x78(%r11),%xmm9
movaps %xmm0,-0x78(%r11)
movaps -0x68(%r11),%xmm10
movaps %xmm0,-0x68(%r11)
movaps -0x58(%r11),%xmm11
movaps %xmm0,-0x58(%r11)
movaps -0x48(%r11),%xmm12
movaps %xmm0,-0x48(%r11)
movaps -0x38(%r11),%xmm13
movaps %xmm0,-0x38(%r11)
movaps -0x28(%r11),%xmm14
movaps %xmm0,-0x28(%r11)
movaps -0x18(%r11),%xmm15
movaps %xmm0,-0x18(%r11)
movaps %xmm0,0x00(%rsp)
movaps %xmm0,0x10(%rsp)
movaps %xmm0,0x20(%rsp)
@@ -2716,10 +2731,13 @@ $code.=<<___ if ($win64);
movaps %xmm0,0x60(%rsp)
___
$code.=<<___;
lea (%rbp),%rsp
pop %rbp
mov -8(%r11),%rbp
.cfi_restore %rbp
lea (%r11),%rsp
.cfi_def_cfa_register %rsp
.Lxts_dec_epilogue:
ret
.cfi_endproc
.size aesni_xts_decrypt,.-aesni_xts_decrypt
___
}
@@ -2744,12 +2762,18 @@ $code.=<<___;
.type aesni_ocb_encrypt,\@function,6
.align 32
aesni_ocb_encrypt:
.cfi_startproc
lea (%rsp),%rax
push %rbx
.cfi_push %rbx
push %rbp
.cfi_push %rbp
push %r12
.cfi_push %r12
push %r13
.cfi_push %r13
push %r14
.cfi_push %r14
___
$code.=<<___ if ($win64);
lea -0xa0(%rsp),%rsp
@@ -2943,6 +2967,8 @@ $code.=<<___ if (!$win64);
pxor %xmm13,%xmm13
pxor %xmm14,%xmm14
pxor %xmm15,%xmm15
lea 0x28(%rsp),%rax
.cfi_def_cfa %rax,8
___
$code.=<<___ if ($win64);
movaps 0x00(%rsp),%xmm6
@@ -2967,16 +2993,23 @@ $code.=<<___ if ($win64);
movaps %xmm0,0x90(%rsp)
lea 0xa0+0x28(%rsp),%rax
.Locb_enc_pop:
lea 0xa0(%rsp),%rsp
___
$code.=<<___;
pop %r14
pop %r13
pop %r12
pop %rbp
pop %rbx
mov -40(%rax),%r14
.cfi_restore %r14
mov -32(%rax),%r13
.cfi_restore %r13
mov -24(%rax),%r12
.cfi_restore %r12
mov -16(%rax),%rbp
.cfi_restore %rbp
mov -8(%rax),%rbx
.cfi_restore %rbx
lea (%rax),%rsp
.cfi_def_cfa_register %rsp
.Locb_enc_epilogue:
ret
.cfi_endproc
.size aesni_ocb_encrypt,.-aesni_ocb_encrypt
.type __ocb_encrypt6,\@abi-omnipotent
@@ -3189,12 +3222,18 @@ __ocb_encrypt1:
.type aesni_ocb_decrypt,\@function,6
.align 32
aesni_ocb_decrypt:
.cfi_startproc
lea (%rsp),%rax
push %rbx
.cfi_push %rbx
push %rbp
.cfi_push %rbp
push %r12
.cfi_push %r12
push %r13
.cfi_push %r13
push %r14
.cfi_push %r14
___
$code.=<<___ if ($win64);
lea -0xa0(%rsp),%rsp
@@ -3410,6 +3449,8 @@ $code.=<<___ if (!$win64);
pxor %xmm13,%xmm13
pxor %xmm14,%xmm14
pxor %xmm15,%xmm15
lea 0x28(%rsp),%rax
.cfi_def_cfa %rax,8
___
$code.=<<___ if ($win64);
movaps 0x00(%rsp),%xmm6
@@ -3434,16 +3475,23 @@ $code.=<<___ if ($win64);
movaps %xmm0,0x90(%rsp)
lea 0xa0+0x28(%rsp),%rax
.Locb_dec_pop:
lea 0xa0(%rsp),%rsp
___
$code.=<<___;
pop %r14
pop %r13
pop %r12
pop %rbp
pop %rbx
mov -40(%rax),%r14
.cfi_restore %r14
mov -32(%rax),%r13
.cfi_restore %r13
mov -24(%rax),%r12
.cfi_restore %r12
mov -16(%rax),%rbp
.cfi_restore %rbp
mov -8(%rax),%rbx
.cfi_restore %rbx
lea (%rax),%rsp
.cfi_def_cfa_register %rsp
.Locb_dec_epilogue:
ret
.cfi_endproc
.size aesni_ocb_decrypt,.-aesni_ocb_decrypt
.type __ocb_decrypt6,\@abi-omnipotent
@@ -3650,13 +3698,13 @@ ___
{
my $frame_size = 0x10 + ($win64?0xa0:0); # used in decrypt
my ($iv,$in0,$in1,$in2,$in3,$in4)=map("%xmm$_",(10..15));
my $inp_=$key_;
$code.=<<___;
.globl ${PREFIX}_cbc_encrypt
.type ${PREFIX}_cbc_encrypt,\@function,6
.align 16
${PREFIX}_cbc_encrypt:
.cfi_startproc
test $len,$len # check length
jz .Lcbc_ret
@@ -3732,8 +3780,10 @@ $code.=<<___;
jmp .Lcbc_ret
.align 16
.Lcbc_decrypt_bulk:
lea (%rsp),%rax
lea (%rsp),%r11 # frame pointer
.cfi_def_cfa_register %r11
push %rbp
.cfi_push %rbp
sub \$$frame_size,%rsp
and \$-16,%rsp # Linux kernel stack can be incorrectly seeded
___
@@ -3750,8 +3800,11 @@ $code.=<<___ if ($win64);
movaps %xmm15,0xa0(%rsp)
.Lcbc_decrypt_body:
___
my $inp_=$key_="%rbp"; # reassign $key_
$code.=<<___;
lea -8(%rax),%rbp
mov $key,$key_ # [re-]backup $key [after reassignment]
movups ($ivp),$iv
mov $rnds_,$rounds
cmp \$0x50,$len
@@ -3791,7 +3844,7 @@ $code.=<<___;
pxor $rndkey0,$inout1
$movkey 0x10-0x70($key),$rndkey1
pxor $rndkey0,$inout2
xor $inp_,$inp_
mov \$-1,$inp_
cmp \$0x70,$len # is there at least 0x60 bytes ahead?
pxor $rndkey0,$inout3
pxor $rndkey0,$inout4
@@ -3807,8 +3860,8 @@ $code.=<<___;
aesdec $rndkey1,$inout4
aesdec $rndkey1,$inout5
aesdec $rndkey1,$inout6
setnc ${inp_}b
shl \$7,$inp_
adc \$0,$inp_
and \$128,$inp_
aesdec $rndkey1,$inout7
add $inp,$inp_
$movkey 0x30-0x70($key),$rndkey1
@@ -4172,10 +4225,13 @@ $code.=<<___ if ($win64);
movaps %xmm0,0xa0(%rsp)
___
$code.=<<___;
lea (%rbp),%rsp
pop %rbp
mov -8(%r11),%rbp
.cfi_restore %rbp
lea (%r11),%rsp
.cfi_def_cfa_register %rsp
.Lcbc_ret:
ret
.cfi_endproc
.size ${PREFIX}_cbc_encrypt,.-${PREFIX}_cbc_encrypt
___
}
@@ -4196,7 +4252,9 @@ $code.=<<___;
.type ${PREFIX}_set_decrypt_key,\@abi-omnipotent
.align 16
${PREFIX}_set_decrypt_key:
.cfi_startproc
.byte 0x48,0x83,0xEC,0x08 # sub rsp,8
.cfi_adjust_cfa_offset 8
call __aesni_set_encrypt_key
shl \$4,$bits # rounds-1 after _aesni_set_encrypt_key
test %eax,%eax
@@ -4229,15 +4287,16 @@ ${PREFIX}_set_decrypt_key:
pxor %xmm0,%xmm0
.Ldec_key_ret:
add \$8,%rsp
.cfi_adjust_cfa_offset -8
ret
.cfi_endproc
.LSEH_end_set_decrypt_key:
.size ${PREFIX}_set_decrypt_key,.-${PREFIX}_set_decrypt_key
___
# This is based on submission by
#
# Huang Ying <ying.huang@intel.com>
# Vinodh Gopal <vinodh.gopal@intel.com>
# This is based on submission from Intel by
# Huang Ying
# Vinodh Gopal
# Kahraman Akdemir
#
# Aggressively optimized in respect to aeskeygenassist's critical path
@@ -4265,7 +4324,9 @@ $code.=<<___;
.align 16
${PREFIX}_set_encrypt_key:
__aesni_set_encrypt_key:
.cfi_startproc
.byte 0x48,0x83,0xEC,0x08 # sub rsp,8
.cfi_adjust_cfa_offset 8
mov \$-1,%rax
test $inp,$inp
jz .Lenc_key_ret
@@ -4454,7 +4515,7 @@ __aesni_set_encrypt_key:
.align 16
.L14rounds:
movups 16($inp),%xmm2 # remaning half of *userKey
movups 16($inp),%xmm2 # remaining half of *userKey
mov \$13,$bits # 14 rounds for 256
lea 16(%rax),%rax
cmp \$`1<<28`,%r10d # AVX, but no XOP
@@ -4558,7 +4619,9 @@ __aesni_set_encrypt_key:
pxor %xmm4,%xmm4
pxor %xmm5,%xmm5
add \$8,%rsp
.cfi_adjust_cfa_offset -8
ret
.cfi_endproc
.LSEH_end_set_encrypt_key:
.align 16
@@ -4744,13 +4807,16 @@ ctr_xts_se_handler:
cmp %r10,%rbx # context->Rip>=epilogue label
jae .Lcommon_seh_tail
mov 160($context),%rax # pull context->Rbp
lea -0xa0(%rax),%rsi # %xmm save area
mov 208($context),%rax # pull context->R11
lea -0xa8(%rax),%rsi # %xmm save area
lea 512($context),%rdi # & context.Xmm6
mov \$20,%ecx # 10*sizeof(%xmm0)/sizeof(%rax)
.long 0xa548f3fc # cld; rep movsq
jmp .Lcommon_rbp_tail
mov -8(%rax),%rbp # restore saved %rbp
mov %rbp,160($context) # restore context->Rbp
jmp .Lcommon_seh_tail
.size ctr_xts_se_handler,.-ctr_xts_se_handler
.type ocb_se_handler,\@abi-omnipotent
@@ -4834,9 +4900,13 @@ cbc_se_handler:
cmp %r10,%rbx # context->Rip<"prologue" label
jb .Lcommon_seh_tail
mov 120($context),%rax # pull context->Rax
lea .Lcbc_decrypt_body(%rip),%r10
cmp %r10,%rbx # context->Rip<cbc_decrypt_body
jb .Lrestore_cbc_rax
jb .Lcommon_seh_tail
mov 152($context),%rax # pull context->Rsp
lea .Lcbc_ret(%rip),%r10
cmp %r10,%rbx # context->Rip>="epilogue" label
@@ -4847,15 +4917,10 @@ cbc_se_handler:
mov \$20,%ecx # 10*sizeof(%xmm0)/sizeof(%rax)
.long 0xa548f3fc # cld; rep movsq
.Lcommon_rbp_tail:
mov 160($context),%rax # pull context->Rbp
mov (%rax),%rbp # restore saved %rbp
lea 8(%rax),%rax # adjust stack pointer
mov %rbp,160($context) # restore context->Rbp
jmp .Lcommon_seh_tail
mov 208($context),%rax # pull context->R11
.Lrestore_cbc_rax:
mov 120($context),%rax
mov -8(%rax),%rbp # restore saved %rbp
mov %rbp,160($context) # restore context->Rbp
.Lcommon_seh_tail:
mov 8(%rax),%rdi
crypto/aes/asm/aesp8-ppc.pl
+1 -1
View File
@@ -3773,7 +3773,7 @@ foreach(split("\n",$code)) {
if ($flavour =~ /le$/o) {
SWITCH: for($conv) {
/\?inv/ && do { @bytes=map($_^0xf,@bytes); last; };
/\?rev/ && do { @bytes=reverse(@bytes); last; };
/\?rev/ && do { @bytes=reverse(@bytes); last; };
}
}
crypto/aes/asm/aest4-sparcv9.pl
+4 -4
View File
@@ -8,9 +8,9 @@
# ====================================================================
# Written by David S. Miller <davem@devemloft.net> and Andy Polyakov
# <appro@openssl.org>. The module is licensed under 2-clause BSD
# license. October 2012. All rights reserved.
# Written by David S. Miller and Andy Polyakov.
# The module is licensed under 2-clause BSD license. October 2012.
# All rights reserved.
# ====================================================================
######################################################################
@@ -44,7 +44,7 @@
# instructions with those on critical path. Amazing!
#
# As with Intel AES-NI, question is if it's possible to improve
# performance of parallelizeable modes by interleaving round
# performance of parallelizable modes by interleaving round
# instructions. Provided round instruction latency and throughput
# optimal interleave factor is 2. But can we expect 2x performance
# improvement? Well, as round instructions can be issued one per
crypto/aes/asm/aesv8-armx.pl
+6 -5
View File
@@ -35,6 +35,7 @@
# Cortex-A57(*) 1.95 0.85 0.93
# Denver 1.96 0.86 0.80
# Mongoose 1.33 1.20 1.20
# Kryo 1.26 0.94 1.00
#
# (*) original 3.64/1.34/1.32 results were for r0p0 revision
# and are still same even for updated module;
@@ -929,7 +930,7 @@ if ($flavour =~ /64/) { ######## 64-bit code
s/^(\s+)v/$1/o or # strip off v prefix
s/\bbx\s+lr\b/ret/o;
# fix up remainig legacy suffixes
# fix up remaining legacy suffixes
s/\.[ui]?8//o;
m/\],#8/o and s/\.16b/\.8b/go;
s/\.[ui]?32//o and s/\.16b/\.4s/go;
@@ -964,21 +965,21 @@ if ($flavour =~ /64/) { ######## 64-bit code
$arg =~ m/q([0-9]+),\s*\{q([0-9]+)\},\s*q([0-9]+)/o &&
sprintf "vtbl.8 d%d,{q%d},d%d\n\t".
"vtbl.8 d%d,{q%d},d%d", 2*$1,$2,2*$3, 2*$1+1,$2,2*$3+1;
"vtbl.8 d%d,{q%d},d%d", 2*$1,$2,2*$3, 2*$1+1,$2,2*$3+1;
}
sub unvdup32 {
my $arg=shift;
$arg =~ m/q([0-9]+),\s*q([0-9]+)\[([0-3])\]/o &&
sprintf "vdup.32 q%d,d%d[%d]",$1,2*$2+($3>>1),$3&1;
sprintf "vdup.32 q%d,d%d[%d]",$1,2*$2+($3>>1),$3&1;
}
sub unvmov32 {
my $arg=shift;
$arg =~ m/q([0-9]+)\[([0-3])\],(.*)/o &&
sprintf "vmov.32 d%d[%d],%s",2*$1+($2>>1),$2&1,$3;
sprintf "vmov.32 d%d[%d],%s",2*$1+($2>>1),$2&1,$3;
}
foreach(split("\n",$code)) {
@@ -988,7 +989,7 @@ if ($flavour =~ /64/) { ######## 64-bit code
s/\bv([0-9])\.[12468]+[bsd]\b/q$1/go; # new->old registers
s/\/\/\s?/@ /o; # new->old style commentary
# fix up remainig new-style suffixes
# fix up remaining new-style suffixes
s/\{q([0-9]+)\},\s*\[(.+)\],#8/sprintf "{d%d},[$2]!",2*$1/eo or
s/\],#[0-9]+/]!/o;
crypto/aes/asm/bsaes-armv7.pl
+7 -11
View File
@@ -1,5 +1,5 @@
#! /usr/bin/env perl
# Copyright 2012-2018 The OpenSSL Project Authors. All Rights Reserved.
# Copyright 2012-2016 The OpenSSL Project Authors. All Rights Reserved.
#
# Licensed under the OpenSSL license (the "License"). You may not use
# this file except in compliance with the License. You can obtain a copy
@@ -14,8 +14,7 @@
# details see http://www.openssl.org/~appro/cryptogams/.
#
# Specific modes and adaptation for Linux kernel by Ard Biesheuvel
# <ard.biesheuvel@linaro.org>. Permission to use under GPL terms is
# granted.
# of Linaro. Permission to use under GPL terms is granted.
# ====================================================================
# Bit-sliced AES for ARM NEON
@@ -49,10 +48,7 @@
# <appro@openssl.org>
# April-August 2013
#
# Add CBC, CTR and XTS subroutines, adapt for kernel use.
#
# <ard.biesheuvel@linaro.org>
# Add CBC, CTR and XTS subroutines and adapt for kernel use; courtesy of Ard.
$flavour = shift;
if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; }
@@ -91,7 +87,7 @@ my @s=@_[12..15];
sub InBasisChange {
# input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb
# output in lsb > [b6, b5, b0, b3, b7, b1, b4, b2] < msb
# output in lsb > [b6, b5, b0, b3, b7, b1, b4, b2] < msb
my @b=@_[0..7];
$code.=<<___;
veor @b[2], @b[2], @b[1]
@@ -746,7 +742,7 @@ $code.=<<___;
_bsaes_decrypt8:
adr $const,.
vldmia $key!, {@XMM[9]} @ round 0 key
#if defined(__thumb2__) || defined(__APPLE__)
#ifdef __APPLE__
adr $const,.LM0ISR
#else
add $const,$const,#.LM0ISR-_bsaes_decrypt8
@@ -845,7 +841,7 @@ _bsaes_const:
_bsaes_encrypt8:
adr $const,.
vldmia $key!, {@XMM[9]} @ round 0 key
#if defined(__thumb2__) || defined(__APPLE__)
#ifdef __APPLE__
adr $const,.LM0SR
#else
sub $const,$const,#_bsaes_encrypt8-.LM0SR
@@ -953,7 +949,7 @@ $code.=<<___;
_bsaes_key_convert:
adr $const,.
vld1.8 {@XMM[7]}, [$inp]! @ load round 0 key
#if defined(__thumb2__) || defined(__APPLE__)
#ifdef __APPLE__
adr $const,.LM0
#else
sub $const,$const,#_bsaes_key_convert-.LM0
crypto/aes/asm/bsaes-x86_64.pl
+196 -74
View File
@@ -131,7 +131,7 @@ my @s=@_[12..15];
sub InBasisChange {
# input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb
# output in lsb > [b6, b5, b0, b3, b7, b1, b4, b2] < msb
# output in lsb > [b6, b5, b0, b3, b7, b1, b4, b2] < msb
my @b=@_[0..7];
$code.=<<___;
pxor @b[6], @b[5]
@@ -381,7 +381,7 @@ $code.=<<___;
pxor @s[0], @t[3]
pxor @s[1], @t[2]
pxor @s[2], @t[1]
pxor @s[3], @t[0]
pxor @s[3], @t[0]
#Inv_GF16 \t0, \t1, \t2, \t3, \s0, \s1, \s2, \s3
@@ -1165,15 +1165,23 @@ $code.=<<___;
.type bsaes_ecb_encrypt_blocks,\@abi-omnipotent
.align 16
bsaes_ecb_encrypt_blocks:
.cfi_startproc
mov %rsp, %rax
.Lecb_enc_prologue:
push %rbp
.cfi_push %rbp
push %rbx
.cfi_push %rbx
push %r12
.cfi_push %r12
push %r13
.cfi_push %r13
push %r14
.cfi_push %r14
push %r15
.cfi_push %r15
lea -0x48(%rsp),%rsp
.cfi_adjust_cfa_offset 0x48
___
$code.=<<___ if ($win64);
lea -0xa0(%rsp), %rsp
@@ -1191,6 +1199,7 @@ $code.=<<___ if ($win64);
___
$code.=<<___;
mov %rsp,%rbp # backup %rsp
.cfi_def_cfa_register %rbp
mov 240($arg4),%eax # rounds
mov $arg1,$inp # backup arguments
mov $arg2,$out
@@ -1334,7 +1343,8 @@ $code.=<<___;
cmp %rax, %rbp
jb .Lecb_enc_bzero
lea (%rbp),%rsp # restore %rsp
lea 0x78(%rbp),%rax
.cfi_def_cfa %rax,8
___
$code.=<<___ if ($win64);
movaps 0x40(%rbp), %xmm6
@@ -1347,34 +1357,50 @@ $code.=<<___ if ($win64);
movaps 0xb0(%rbp), %xmm13
movaps 0xc0(%rbp), %xmm14
movaps 0xd0(%rbp), %xmm15
lea 0xa0(%rbp), %rsp
lea 0xa0(%rax), %rax
.Lecb_enc_tail:
___
$code.=<<___;
mov 0x48(%rsp), %r15
mov 0x50(%rsp), %r14
mov 0x58(%rsp), %r13
mov 0x60(%rsp), %r12
mov 0x68(%rsp), %rbx
mov 0x70(%rsp), %rax
lea 0x78(%rsp), %rsp
mov %rax, %rbp
mov -48(%rax), %r15
.cfi_restore %r15
mov -40(%rax), %r14
.cfi_restore %r14
mov -32(%rax), %r13
.cfi_restore %r13
mov -24(%rax), %r12
.cfi_restore %r12
mov -16(%rax), %rbx
.cfi_restore %rbx
mov -8(%rax), %rbp
.cfi_restore %rbp
lea (%rax), %rsp # restore %rsp
.cfi_def_cfa_register %rsp
.Lecb_enc_epilogue:
ret
.cfi_endproc
.size bsaes_ecb_encrypt_blocks,.-bsaes_ecb_encrypt_blocks
.globl bsaes_ecb_decrypt_blocks
.type bsaes_ecb_decrypt_blocks,\@abi-omnipotent
.align 16
bsaes_ecb_decrypt_blocks:
.cfi_startproc
mov %rsp, %rax
.Lecb_dec_prologue:
push %rbp
.cfi_push %rbp
push %rbx
.cfi_push %rbx
push %r12
.cfi_push %r12
push %r13
.cfi_push %r13
push %r14
.cfi_push %r14
push %r15
.cfi_push %r15
lea -0x48(%rsp),%rsp
.cfi_adjust_cfa_offset 0x48
___
$code.=<<___ if ($win64);
lea -0xa0(%rsp), %rsp
@@ -1392,6 +1418,7 @@ $code.=<<___ if ($win64);
___
$code.=<<___;
mov %rsp,%rbp # backup %rsp
.cfi_def_cfa_register %rbp
mov 240($arg4),%eax # rounds
mov $arg1,$inp # backup arguments
mov $arg2,$out
@@ -1536,7 +1563,8 @@ $code.=<<___;
cmp %rax, %rbp
jb .Lecb_dec_bzero
lea (%rbp),%rsp # restore %rsp
lea 0x78(%rbp),%rax
.cfi_def_cfa %rax,8
___
$code.=<<___ if ($win64);
movaps 0x40(%rbp), %xmm6
@@ -1549,19 +1577,27 @@ $code.=<<___ if ($win64);
movaps 0xb0(%rbp), %xmm13
movaps 0xc0(%rbp), %xmm14
movaps 0xd0(%rbp), %xmm15
lea 0xa0(%rbp), %rsp
lea 0xa0(%rax), %rax
.Lecb_dec_tail:
___
$code.=<<___;
mov 0x48(%rsp), %r15
mov 0x50(%rsp), %r14
mov 0x58(%rsp), %r13
mov 0x60(%rsp), %r12
mov 0x68(%rsp), %rbx
mov 0x70(%rsp), %rax
lea 0x78(%rsp), %rsp
mov %rax, %rbp
mov -48(%rax), %r15
.cfi_restore %r15
mov -40(%rax), %r14
.cfi_restore %r14
mov -32(%rax), %r13
.cfi_restore %r13
mov -24(%rax), %r12
.cfi_restore %r12
mov -16(%rax), %rbx
.cfi_restore %rbx
mov -8(%rax), %rbp
.cfi_restore %rbp
lea (%rax), %rsp # restore %rsp
.cfi_def_cfa_register %rsp
.Lecb_dec_epilogue:
ret
.cfi_endproc
.size bsaes_ecb_decrypt_blocks,.-bsaes_ecb_decrypt_blocks
___
}
@@ -1571,6 +1607,7 @@ $code.=<<___;
.type bsaes_cbc_encrypt,\@abi-omnipotent
.align 16
bsaes_cbc_encrypt:
.cfi_startproc
___
$code.=<<___ if ($win64);
mov 48(%rsp),$arg6 # pull direction flag
@@ -1584,12 +1621,19 @@ $code.=<<___;
mov %rsp, %rax
.Lcbc_dec_prologue:
push %rbp
.cfi_push %rbp
push %rbx
.cfi_push %rbx
push %r12
.cfi_push %r12
push %r13
.cfi_push %r13
push %r14
.cfi_push %r14
push %r15
.cfi_push %r15
lea -0x48(%rsp), %rsp
.cfi_adjust_cfa_offset 0x48
___
$code.=<<___ if ($win64);
mov 0xa0(%rsp),$arg5 # pull ivp
@@ -1608,6 +1652,7 @@ $code.=<<___ if ($win64);
___
$code.=<<___;
mov %rsp, %rbp # backup %rsp
.cfi_def_cfa_register %rbp
mov 240($arg4), %eax # rounds
mov $arg1, $inp # backup arguments
mov $arg2, $out
@@ -1826,7 +1871,8 @@ $code.=<<___;
cmp %rax, %rbp
ja .Lcbc_dec_bzero
lea (%rbp),%rsp # restore %rsp
lea 0x78(%rbp),%rax
.cfi_def_cfa %rax,8
___
$code.=<<___ if ($win64);
movaps 0x40(%rbp), %xmm6
@@ -1839,34 +1885,50 @@ $code.=<<___ if ($win64);
movaps 0xb0(%rbp), %xmm13
movaps 0xc0(%rbp), %xmm14
movaps 0xd0(%rbp), %xmm15
lea 0xa0(%rbp), %rsp
lea 0xa0(%rax), %rax
.Lcbc_dec_tail:
___
$code.=<<___;
mov 0x48(%rsp), %r15
mov 0x50(%rsp), %r14
mov 0x58(%rsp), %r13
mov 0x60(%rsp), %r12
mov 0x68(%rsp), %rbx
mov 0x70(%rsp), %rax
lea 0x78(%rsp), %rsp
mov %rax, %rbp
mov -48(%rax), %r15
.cfi_restore %r15
mov -40(%rax), %r14
.cfi_restore %r14
mov -32(%rax), %r13
.cfi_restore %r13
mov -24(%rax), %r12
.cfi_restore %r12
mov -16(%rax), %rbx
.cfi_restore %rbx
mov -8(%rax), %rbp
.cfi_restore %rbp
lea (%rax), %rsp # restore %rsp
.cfi_def_cfa_register %rsp
.Lcbc_dec_epilogue:
ret
.cfi_endproc
.size bsaes_cbc_encrypt,.-bsaes_cbc_encrypt
.globl bsaes_ctr32_encrypt_blocks
.type bsaes_ctr32_encrypt_blocks,\@abi-omnipotent
.align 16
bsaes_ctr32_encrypt_blocks:
.cfi_startproc
mov %rsp, %rax
.Lctr_enc_prologue:
push %rbp
.cfi_push %rbp
push %rbx
.cfi_push %rbx
push %r12
.cfi_push %r12
push %r13
.cfi_push %r13
push %r14
.cfi_push %r14
push %r15
.cfi_push %r15
lea -0x48(%rsp), %rsp
.cfi_adjust_cfa_offset 0x48
___
$code.=<<___ if ($win64);
mov 0xa0(%rsp),$arg5 # pull ivp
@@ -1885,6 +1947,7 @@ $code.=<<___ if ($win64);
___
$code.=<<___;
mov %rsp, %rbp # backup %rsp
.cfi_def_cfa_register %rbp
movdqu ($arg5), %xmm0 # load counter
mov 240($arg4), %eax # rounds
mov $arg1, $inp # backup arguments
@@ -2058,7 +2121,8 @@ $code.=<<___;
cmp %rax, %rbp
ja .Lctr_enc_bzero
lea (%rbp),%rsp # restore %rsp
lea 0x78(%rbp),%rax
.cfi_def_cfa %rax,8
___
$code.=<<___ if ($win64);
movaps 0x40(%rbp), %xmm6
@@ -2071,19 +2135,27 @@ $code.=<<___ if ($win64);
movaps 0xb0(%rbp), %xmm13
movaps 0xc0(%rbp), %xmm14
movaps 0xd0(%rbp), %xmm15
lea 0xa0(%rbp), %rsp
lea 0xa0(%rax), %rax
.Lctr_enc_tail:
___
$code.=<<___;
mov 0x48(%rsp), %r15
mov 0x50(%rsp), %r14
mov 0x58(%rsp), %r13
mov 0x60(%rsp), %r12
mov 0x68(%rsp), %rbx
mov 0x70(%rsp), %rax
lea 0x78(%rsp), %rsp
mov %rax, %rbp
mov -48(%rax), %r15
.cfi_restore %r15
mov -40(%rax), %r14
.cfi_restore %r14
mov -32(%rax), %r13
.cfi_restore %r13
mov -24(%rax), %r12
.cfi_restore %r12
mov -16(%rax), %rbx
.cfi_restore %rbx
mov -8(%rax), %rbp
.cfi_restore %rbp
lea (%rax), %rsp # restore %rsp
.cfi_def_cfa_register %rsp
.Lctr_enc_epilogue:
ret
.cfi_endproc
.size bsaes_ctr32_encrypt_blocks,.-bsaes_ctr32_encrypt_blocks
___
######################################################################
@@ -2099,15 +2171,23 @@ $code.=<<___;
.type bsaes_xts_encrypt,\@abi-omnipotent
.align 16
bsaes_xts_encrypt:
.cfi_startproc
mov %rsp, %rax
.Lxts_enc_prologue:
push %rbp
.cfi_push %rbp
push %rbx
.cfi_push %rbx
push %r12
.cfi_push %r12
push %r13
.cfi_push %r13
push %r14
.cfi_push %r14
push %r15
.cfi_push %r15
lea -0x48(%rsp), %rsp
.cfi_adjust_cfa_offset 0x48
___
$code.=<<___ if ($win64);
mov 0xa0(%rsp),$arg5 # pull key2
@@ -2127,6 +2207,7 @@ $code.=<<___ if ($win64);
___
$code.=<<___;
mov %rsp, %rbp # backup %rsp
.cfi_def_cfa_register %rbp
mov $arg1, $inp # backup arguments
mov $arg2, $out
mov $arg3, $len
@@ -2448,7 +2529,8 @@ $code.=<<___;
cmp %rax, %rbp
ja .Lxts_enc_bzero
lea (%rbp),%rsp # restore %rsp
lea 0x78(%rbp),%rax
.cfi_def_cfa %rax,8
___
$code.=<<___ if ($win64);
movaps 0x40(%rbp), %xmm6
@@ -2461,34 +2543,50 @@ $code.=<<___ if ($win64);
movaps 0xb0(%rbp), %xmm13
movaps 0xc0(%rbp), %xmm14
movaps 0xd0(%rbp), %xmm15
lea 0xa0(%rbp), %rsp
lea 0xa0(%rax), %rax
.Lxts_enc_tail:
___
$code.=<<___;
mov 0x48(%rsp), %r15
mov 0x50(%rsp), %r14
mov 0x58(%rsp), %r13
mov 0x60(%rsp), %r12
mov 0x68(%rsp), %rbx
mov 0x70(%rsp), %rax
lea 0x78(%rsp), %rsp
mov %rax, %rbp
mov -48(%rax), %r15
.cfi_restore %r15
mov -40(%rax), %r14
.cfi_restore %r14
mov -32(%rax), %r13
.cfi_restore %r13
mov -24(%rax), %r12
.cfi_restore %r12
mov -16(%rax), %rbx
.cfi_restore %rbx
mov -8(%rax), %rbp
.cfi_restore %rbp
lea (%rax), %rsp # restore %rsp
.cfi_def_cfa_register %rsp
.Lxts_enc_epilogue:
ret
.cfi_endproc
.size bsaes_xts_encrypt,.-bsaes_xts_encrypt
.globl bsaes_xts_decrypt
.type bsaes_xts_decrypt,\@abi-omnipotent
.align 16
bsaes_xts_decrypt:
.cfi_startproc
mov %rsp, %rax
.Lxts_dec_prologue:
push %rbp
.cfi_push %rbp
push %rbx
.cfi_push %rbx
push %r12
.cfi_push %r12
push %r13
.cfi_push %r13
push %r14
.cfi_push %r14
push %r15
.cfi_push %r15
lea -0x48(%rsp), %rsp
.cfi_adjust_cfa_offset 0x48
___
$code.=<<___ if ($win64);
mov 0xa0(%rsp),$arg5 # pull key2
@@ -2855,7 +2953,8 @@ $code.=<<___;
cmp %rax, %rbp
ja .Lxts_dec_bzero
lea (%rbp),%rsp # restore %rsp
lea 0x78(%rbp),%rax
.cfi_def_cfa %rax,8
___
$code.=<<___ if ($win64);
movaps 0x40(%rbp), %xmm6
@@ -2868,19 +2967,27 @@ $code.=<<___ if ($win64);
movaps 0xb0(%rbp), %xmm13
movaps 0xc0(%rbp), %xmm14
movaps 0xd0(%rbp), %xmm15
lea 0xa0(%rbp), %rsp
lea 0xa0(%rax), %rax
.Lxts_dec_tail:
___
$code.=<<___;
mov 0x48(%rsp), %r15
mov 0x50(%rsp), %r14
mov 0x58(%rsp), %r13
mov 0x60(%rsp), %r12
mov 0x68(%rsp), %rbx
mov 0x70(%rsp), %rax
lea 0x78(%rsp), %rsp
mov %rax, %rbp
mov -48(%rax), %r15
.cfi_restore %r15
mov -40(%rax), %r14
.cfi_restore %r14
mov -32(%rax), %r13
.cfi_restore %r13
mov -24(%rax), %r12
.cfi_restore %r12
mov -16(%rax), %rbx
.cfi_restore %rbx
mov -8(%rax), %rbp
.cfi_restore %rbp
lea (%rax), %rsp # restore %rsp
.cfi_def_cfa_register %rsp
.Lxts_dec_epilogue:
ret
.cfi_endproc
.size bsaes_xts_decrypt,.-bsaes_xts_decrypt
___
}
@@ -2974,31 +3081,34 @@ se_handler:
mov 0(%r11),%r10d # HandlerData[0]
lea (%rsi,%r10),%r10 # prologue label
cmp %r10,%rbx # context->Rip<prologue label
jb .Lin_prologue
mov 152($context),%rax # pull context->Rsp
cmp %r10,%rbx # context->Rip<=prologue label
jbe .Lin_prologue
mov 4(%r11),%r10d # HandlerData[1]
lea (%rsi,%r10),%r10 # epilogue label
cmp %r10,%rbx # context->Rip>=epilogue label
jae .Lin_prologue
mov 8(%r11),%r10d # HandlerData[2]
lea (%rsi,%r10),%r10 # epilogue label
cmp %r10,%rbx # context->Rip>=tail label
jae .Lin_tail
mov 160($context),%rax # pull context->Rbp
lea 0x40(%rax),%rsi # %xmm save area
lea 512($context),%rdi # &context.Xmm6
mov \$20,%ecx # 10*sizeof(%xmm0)/sizeof(%rax)
.long 0xa548f3fc # cld; rep movsq
lea 0xa0(%rax),%rax # adjust stack pointer
lea 0xa0+0x78(%rax),%rax # adjust stack pointer
mov 0x70(%rax),%rbp
mov 0x68(%rax),%rbx
mov 0x60(%rax),%r12
mov 0x58(%rax),%r13
mov 0x50(%rax),%r14
mov 0x48(%rax),%r15
lea 0x78(%rax),%rax # adjust stack pointer
.Lin_tail:
mov -48(%rax),%rbp
mov -40(%rax),%rbx
mov -32(%rax),%r12
mov -24(%rax),%r13
mov -16(%rax),%r14
mov -8(%rax),%r15
mov %rbx,144($context) # restore context->Rbx
mov %rbp,160($context) # restore context->Rbp
mov %r12,216($context) # restore context->R12
@@ -3079,28 +3189,40 @@ $code.=<<___ if ($ecb);
.byte 9,0,0,0
.rva se_handler
.rva .Lecb_enc_body,.Lecb_enc_epilogue # HandlerData[]
.rva .Lecb_enc_tail
.long 0
.Lecb_dec_info:
.byte 9,0,0,0
.rva se_handler
.rva .Lecb_dec_body,.Lecb_dec_epilogue # HandlerData[]
.rva .Lecb_dec_tail
.long 0
___
$code.=<<___;
.Lcbc_dec_info:
.byte 9,0,0,0
.rva se_handler
.rva .Lcbc_dec_body,.Lcbc_dec_epilogue # HandlerData[]
.rva .Lcbc_dec_tail
.long 0
.Lctr_enc_info:
.byte 9,0,0,0
.rva se_handler
.rva .Lctr_enc_body,.Lctr_enc_epilogue # HandlerData[]
.rva .Lctr_enc_tail
.long 0
.Lxts_enc_info:
.byte 9,0,0,0
.rva se_handler
.rva .Lxts_enc_body,.Lxts_enc_epilogue # HandlerData[]
.rva .Lxts_enc_tail
.long 0
.Lxts_dec_info:
.byte 9,0,0,0
.rva se_handler
.rva .Lxts_dec_body,.Lxts_dec_epilogue # HandlerData[]
.rva .Lxts_dec_tail
.long 0
___
}
crypto/aes/asm/vpaes-armv8.pl
+6 -6
View File
@@ -31,7 +31,7 @@
# Apple A7(***) 22.7(**) 10.9/14.3 [8.45/10.0 ]
# Mongoose(***) 26.3(**) 21.0/25.0(**) [13.3/16.8 ]
#
# (*) ECB denotes approximate result for parallelizeable modes
# (*) ECB denotes approximate result for parallelizable modes
# such as CBC decrypt, CTR, etc.;
# (**) these results are worse than scalar compiler-generated
# code, but it's constant-time and therefore preferred;
@@ -137,7 +137,7 @@ _vpaes_consts:
.quad 0x07E4A34047A4E300, 0x1DFEB95A5DBEF91A
.quad 0x5F36B5DC83EA6900, 0x2841C2ABF49D1E77
.asciz "Vector Permutaion AES for ARMv8, Mike Hamburg (Stanford University)"
.asciz "Vector Permutation AES for ARMv8, Mike Hamburg (Stanford University)"
.size _vpaes_consts,.-_vpaes_consts
.align 6
___
@@ -769,7 +769,7 @@ _vpaes_schedule_core:
ld1 {v0.16b}, [$inp] // vmovdqu 16(%rdi),%xmm0 # load key part 2 (unaligned)
bl _vpaes_schedule_transform // input transform
mov $inp, #7 // mov \$7, %esi
.Loop_schedule_256:
sub $inp, $inp, #1 // dec %esi
bl _vpaes_schedule_mangle // output low result
@@ -778,7 +778,7 @@ _vpaes_schedule_core:
// high round
bl _vpaes_schedule_round
cbz $inp, .Lschedule_mangle_last
bl _vpaes_schedule_mangle
bl _vpaes_schedule_mangle
// low round. swap xmm7 and xmm6
dup v0.4s, v0.s[3] // vpshufd \$0xFF, %xmm0, %xmm0
@@ -787,7 +787,7 @@ _vpaes_schedule_core:
mov v7.16b, v6.16b // vmovdqa %xmm6, %xmm7
bl _vpaes_schedule_low_round
mov v7.16b, v5.16b // vmovdqa %xmm5, %xmm7
b .Loop_schedule_256
##
@@ -814,7 +814,7 @@ _vpaes_schedule_core:
.Lschedule_mangle_last_dec:
ld1 {v20.2d-v21.2d}, [x11] // reload constants
sub $out, $out, #16 // add \$-16, %rdx
sub $out, $out, #16 // add \$-16, %rdx
eor v0.16b, v0.16b, v16.16b // vpxor .Lk_s63(%rip), %xmm0, %xmm0
bl _vpaes_schedule_transform // output transform
st1 {v0.2d}, [$out] // vmovdqu %xmm0, (%rdx) # save last key
crypto/aes/asm/vpaes-ppc.pl
+4 -4
View File
@@ -1075,7 +1075,7 @@ Loop_schedule_256:
# high round
bl _vpaes_schedule_round
bdz Lschedule_mangle_last # dec %esi
bl _vpaes_schedule_mangle
bl _vpaes_schedule_mangle
# low round. swap xmm7 and xmm6
?vspltw v0, v0, 3 # vpshufd \$0xFF, %xmm0, %xmm0
@@ -1083,7 +1083,7 @@ Loop_schedule_256:
vmr v7, v6 # vmovdqa %xmm6, %xmm7
bl _vpaes_schedule_low_round
vmr v7, v5 # vmovdqa %xmm5, %xmm7
b Loop_schedule_256
##
## .aes_schedule_mangle_last
@@ -1131,7 +1131,7 @@ Lschedule_mangle_last:
Lschedule_mangle_last_dec:
lvx $iptlo, r11, r12 # reload $ipt
lvx $ipthi, r9, r12
addi $out, $out, -16 # add \$-16, %rdx
addi $out, $out, -16 # add \$-16, %rdx
vxor v0, v0, v26 # vpxor .Lk_s63(%rip), %xmm0, %xmm0
bl _vpaes_schedule_transform # output transform
@@ -1566,7 +1566,7 @@ foreach (split("\n",$code)) {
if ($flavour =~ /le$/o) {
SWITCH: for($conv) {
/\?inv/ && do { @bytes=map($_^0xf,@bytes); last; };
/\?rev/ && do { @bytes=reverse(@bytes); last; };
/\?rev/ && do { @bytes=reverse(@bytes); last; };
}
}
crypto/aes/asm/vpaes-x86.pl
+5 -5
View File
@@ -62,7 +62,7 @@ $output = pop;
open OUT,">$output";
*STDOUT=*OUT;
&asm_init($ARGV[0],"vpaes-x86.pl",$x86only = $ARGV[$#ARGV] eq "386");
&asm_init($ARGV[0],$x86only = $ARGV[$#ARGV] eq "386");
$PREFIX="vpaes";
@@ -445,7 +445,7 @@ $k_dsbo=0x2c0; # decryption sbox final output
##
&set_label("schedule_192",16);
&movdqu ("xmm0",&QWP(8,$inp)); # load key part 2 (very unaligned)
&call ("_vpaes_schedule_transform"); # input transform
&call ("_vpaes_schedule_transform"); # input transform
&movdqa ("xmm6","xmm0"); # save short part
&pxor ("xmm4","xmm4"); # clear 4
&movhlps("xmm6","xmm4"); # clobber low side with zeros
@@ -476,7 +476,7 @@ $k_dsbo=0x2c0; # decryption sbox final output
##
&set_label("schedule_256",16);
&movdqu ("xmm0",&QWP(16,$inp)); # load key part 2 (unaligned)
&call ("_vpaes_schedule_transform"); # input transform
&call ("_vpaes_schedule_transform"); # input transform
&mov ($round,7);
&set_label("loop_schedule_256");
@@ -487,7 +487,7 @@ $k_dsbo=0x2c0; # decryption sbox final output
&call ("_vpaes_schedule_round");
&dec ($round);
&jz (&label("schedule_mangle_last"));
&call ("_vpaes_schedule_mangle");
&call ("_vpaes_schedule_mangle");
# low round. swap xmm7 and xmm6
&pshufd ("xmm0","xmm0",0xFF);
@@ -610,7 +610,7 @@ $k_dsbo=0x2c0; # decryption sbox final output
# subbyte
&movdqa ("xmm4",&QWP($k_s0F,$const));
&movdqa ("xmm5",&QWP($k_inv,$const)); # 4 : 1/j
&movdqa ("xmm1","xmm4");
&movdqa ("xmm1","xmm4");
&pandn ("xmm1","xmm0");
&psrld ("xmm1",4); # 1 = i
&pand ("xmm0","xmm4"); # 0 = k
crypto/aes/asm/vpaes-x86_64.pl
+10 -10
View File
@@ -172,7 +172,7 @@ _vpaes_encrypt_core:
pshufb %xmm1, %xmm0
ret
.size _vpaes_encrypt_core,.-_vpaes_encrypt_core
##
## Decryption core
##
@@ -333,7 +333,7 @@ _vpaes_schedule_core:
##
.Lschedule_128:
mov \$10, %esi
.Loop_schedule_128:
call _vpaes_schedule_round
dec %rsi
@@ -367,7 +367,7 @@ _vpaes_schedule_core:
.Loop_schedule_192:
call _vpaes_schedule_round
palignr \$8,%xmm6,%xmm0
palignr \$8,%xmm6,%xmm0
call _vpaes_schedule_mangle # save key n
call _vpaes_schedule_192_smear
call _vpaes_schedule_mangle # save key n+1
@@ -393,7 +393,7 @@ _vpaes_schedule_core:
movdqu 16(%rdi),%xmm0 # load key part 2 (unaligned)
call _vpaes_schedule_transform # input transform
mov \$7, %esi
.Loop_schedule_256:
call _vpaes_schedule_mangle # output low result
movdqa %xmm0, %xmm6 # save cur_lo in xmm6
@@ -402,7 +402,7 @@ _vpaes_schedule_core:
call _vpaes_schedule_round
dec %rsi
jz .Lschedule_mangle_last
call _vpaes_schedule_mangle
call _vpaes_schedule_mangle
# low round. swap xmm7 and xmm6
pshufd \$0xFF, %xmm0, %xmm0
@@ -410,10 +410,10 @@ _vpaes_schedule_core:
movdqa %xmm6, %xmm7
call _vpaes_schedule_low_round
movdqa %xmm5, %xmm7
jmp .Loop_schedule_256
##
## .aes_schedule_mangle_last
##
@@ -512,9 +512,9 @@ _vpaes_schedule_round:
# rotate
pshufd \$0xFF, %xmm0, %xmm0
palignr \$1, %xmm0, %xmm0
# fall through...
# low round: same as high round, but no rotation and no rcon.
_vpaes_schedule_low_round:
# smear xmm7
@@ -553,7 +553,7 @@ _vpaes_schedule_low_round:
pxor %xmm4, %xmm0 # 0 = sbox output
# add in smeared stuff
pxor %xmm7, %xmm0
pxor %xmm7, %xmm0
movdqa %xmm0, %xmm7
ret
.size _vpaes_schedule_round,.-_vpaes_schedule_round
crypto/aes/build.info
+10 -3
View File
@@ -5,11 +5,14 @@ SOURCE[../../libcrypto]=\
GENERATE[aes-ia64.s]=asm/aes-ia64.S
GENERATE[aes-586.s]=asm/aes-586.pl $(PERLASM_SCHEME) $(CFLAGS) $(LIB_CFLAGS) $(PROCESSOR)
GENERATE[aes-586.s]=asm/aes-586.pl \
$(PERLASM_SCHEME) $(LIB_CFLAGS) $(LIB_CPPFLAGS) $(PROCESSOR)
DEPEND[aes-586.s]=../perlasm/x86asm.pl
GENERATE[vpaes-x86.s]=asm/vpaes-x86.pl $(PERLASM_SCHEME) $(CFLAGS) $(LIB_CFLAGS) $(PROCESSOR)
GENERATE[vpaes-x86.s]=asm/vpaes-x86.pl \
$(PERLASM_SCHEME) $(LIB_CFLAGS) $(LIB_CPPFLAGS) $(PROCESSOR)
DEPEND[vpaes-586.s]=../perlasm/x86asm.pl
GENERATE[aesni-x86.s]=asm/aesni-x86.pl $(PERLASM_SCHEME) $(CFLAGS) $(LIB_CFLAGS) $(PROCESSOR)
GENERATE[aesni-x86.s]=asm/aesni-x86.pl \
$(PERLASM_SCHEME) $(LIB_CFLAGS) $(LIB_CPPFLAGS) $(PROCESSOR)
DEPEND[aesni-586.s]=../perlasm/x86asm.pl
GENERATE[aes-x86_64.s]=asm/aes-x86_64.pl $(PERLASM_SCHEME)
@@ -35,6 +38,7 @@ GENERATE[aesp8-ppc.s]=asm/aesp8-ppc.pl $(PERLASM_SCHEME)
GENERATE[aes-parisc.s]=asm/aes-parisc.pl $(PERLASM_SCHEME)
GENERATE[aes-mips.S]=asm/aes-mips.pl $(PERLASM_SCHEME)
INCLUDE[aes-mips.o]=..
GENERATE[aesv8-armx.S]=asm/aesv8-armx.pl $(PERLASM_SCHEME)
INCLUDE[aesv8-armx.o]=..
@@ -45,6 +49,9 @@ INCLUDE[aes-armv4.o]=..
GENERATE[bsaes-armv7.S]=asm/bsaes-armv7.pl $(PERLASM_SCHEME)
INCLUDE[bsaes-armv7.o]=..
GENERATE[aes-s390x.S]=asm/aes-s390x.pl $(PERLASM_SCHEME)
INCLUDE[aes-s390x.o]=..
BEGINRAW[Makefile]
##### AES assembler implementations