Hakase/openssl-custom
1
0 Fork 0
Code Issues Pull Requests Releases Wiki Activity

Latest update

Browse Source
This commit is contained in:
Hakase
2019-04-26 08:41:22 +09:00
parent 704c3822e0
commit a7bf77581f
91 changed files with 4677 additions and 389 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crypto/aes/asm/aesv8-armx.pl
+389 -5
View File
@@ -27,18 +27,34 @@
# CBC encrypt case. On Cortex-A57 parallelizable mode performance
# seems to be limited by sheer amount of NEON instructions...
#
# April 2019
#
# Key to performance of parallelize-able modes is round instruction
# interleaving. But which factor to use? There is optimal one for
# each combination of instruction latency and issue rate, beyond
# which increasing interleave factor doesn't pay off. While on cons
# side we have code size increase and resource waste on platforms for
# which interleave factor is too high. In other words you want it to
# be just right. So far interleave factor of 3x was serving well all
# platforms. But for ThunderX2 optimal interleave factor was measured
# to be 5x...
#
# Performance in cycles per byte processed with 128-bit key:
#
# CBC enc CBC dec CTR
# Apple A7 2.39 1.20 1.20
# Cortex-A53 1.32 1.29 1.46
# Cortex-A57(*) 1.95 0.85 0.93
# Denver 1.96 0.86 0.80
# Mongoose 1.33 1.20 1.20
# Kryo 1.26 0.94 1.00
# Cortex-A53 1.32 1.17/1.29(**) 1.36/1.46
# Cortex-A57(*) 1.95 0.82/0.85 0.89/0.93
# Cortex-A72 1.33 0.85/0.88 0.92/0.96
# Denver 1.96 0.65/0.86 0.76/0.80
# Mongoose 1.33 1.23/1.20 1.30/1.20
# Kryo 1.26 0.87/0.94 1.00/1.00
# ThunderX2 5.95 1.25 1.30
#
# (*) original 3.64/1.34/1.32 results were for r0p0 revision
# and are still same even for updated module;
# (**) numbers after slash are for 32-bit code, which is 3x-
# interleaved;
$flavour = shift;
$output = shift;
@@ -523,6 +539,13 @@ $code.=<<___;
___
{
my ($dat2,$in2,$tmp2)=map("q$_",(10,11,9));
my ($dat3,$in3,$tmp3); # used only in 64-bit mode
my ($dat4,$in4,$tmp4);
if ($flavour =~ /64/) {
($dat2,$dat3,$dat4,$in2,$in3,$in4,$tmp3,$tmp4)=map("q$_",(16..23));
}
$code.=<<___;
.align 5
.Lcbc_dec:
@@ -539,7 +562,196 @@ $code.=<<___;
vorr $in0,$dat,$dat
vorr $in1,$dat1,$dat1
vorr $in2,$dat2,$dat2
___
$code.=<<___ if ($flavour =~ /64/);
cmp $len,#32
b.lo .Loop3x_cbc_dec
vld1.8 {$dat3},[$inp],#16
vld1.8 {$dat4},[$inp],#16
sub $len,$len,#32 // bias
mov $cnt,$rounds
vorr $in3,$dat3,$dat3
vorr $in4,$dat4,$dat4
.Loop5x_cbc_dec:
aesd $dat0,q8
aesimc $dat0,$dat0
aesd $dat1,q8
aesimc $dat1,$dat1
aesd $dat2,q8
aesimc $dat2,$dat2
aesd $dat3,q8
aesimc $dat3,$dat3
aesd $dat4,q8
aesimc $dat4,$dat4
vld1.32 {q8},[$key_],#16
subs $cnt,$cnt,#2
aesd $dat0,q9
aesimc $dat0,$dat0
aesd $dat1,q9
aesimc $dat1,$dat1
aesd $dat2,q9
aesimc $dat2,$dat2
aesd $dat3,q9
aesimc $dat3,$dat3
aesd $dat4,q9
aesimc $dat4,$dat4
vld1.32 {q9},[$key_],#16
b.gt .Loop5x_cbc_dec
aesd $dat0,q8
aesimc $dat0,$dat0
aesd $dat1,q8
aesimc $dat1,$dat1
aesd $dat2,q8
aesimc $dat2,$dat2
aesd $dat3,q8
aesimc $dat3,$dat3
aesd $dat4,q8
aesimc $dat4,$dat4
cmp $len,#0x40 // because .Lcbc_tail4x
sub $len,$len,#0x50
aesd $dat0,q9
aesimc $dat0,$dat0
aesd $dat1,q9
aesimc $dat1,$dat1
aesd $dat2,q9
aesimc $dat2,$dat2
aesd $dat3,q9
aesimc $dat3,$dat3
aesd $dat4,q9
aesimc $dat4,$dat4
csel x6,xzr,$len,gt // borrow x6, $cnt, "gt" is not typo
mov $key_,$key
aesd $dat0,q10
aesimc $dat0,$dat0
aesd $dat1,q10
aesimc $dat1,$dat1
aesd $dat2,q10
aesimc $dat2,$dat2
aesd $dat3,q10
aesimc $dat3,$dat3
aesd $dat4,q10
aesimc $dat4,$dat4
add $inp,$inp,x6 // $inp is adjusted in such way that
// at exit from the loop $dat1-$dat4
// are loaded with last "words"
add x6,$len,#0x60 // because .Lcbc_tail4x
aesd $dat0,q11
aesimc $dat0,$dat0
aesd $dat1,q11
aesimc $dat1,$dat1
aesd $dat2,q11
aesimc $dat2,$dat2
aesd $dat3,q11
aesimc $dat3,$dat3
aesd $dat4,q11
aesimc $dat4,$dat4
aesd $dat0,q12
aesimc $dat0,$dat0
aesd $dat1,q12
aesimc $dat1,$dat1
aesd $dat2,q12
aesimc $dat2,$dat2
aesd $dat3,q12
aesimc $dat3,$dat3
aesd $dat4,q12
aesimc $dat4,$dat4
aesd $dat0,q13
aesimc $dat0,$dat0
aesd $dat1,q13
aesimc $dat1,$dat1
aesd $dat2,q13
aesimc $dat2,$dat2
aesd $dat3,q13
aesimc $dat3,$dat3
aesd $dat4,q13
aesimc $dat4,$dat4
aesd $dat0,q14
aesimc $dat0,$dat0
aesd $dat1,q14
aesimc $dat1,$dat1
aesd $dat2,q14
aesimc $dat2,$dat2
aesd $dat3,q14
aesimc $dat3,$dat3
aesd $dat4,q14
aesimc $dat4,$dat4
veor $tmp0,$ivec,$rndlast
aesd $dat0,q15
veor $tmp1,$in0,$rndlast
vld1.8 {$in0},[$inp],#16
aesd $dat1,q15
veor $tmp2,$in1,$rndlast
vld1.8 {$in1},[$inp],#16
aesd $dat2,q15
veor $tmp3,$in2,$rndlast
vld1.8 {$in2},[$inp],#16
aesd $dat3,q15
veor $tmp4,$in3,$rndlast
vld1.8 {$in3},[$inp],#16
aesd $dat4,q15
vorr $ivec,$in4,$in4
vld1.8 {$in4},[$inp],#16
cbz x6,.Lcbc_tail4x
vld1.32 {q8},[$key_],#16 // re-pre-load rndkey[0]
veor $tmp0,$tmp0,$dat0
vorr $dat0,$in0,$in0
veor $tmp1,$tmp1,$dat1
vorr $dat1,$in1,$in1
veor $tmp2,$tmp2,$dat2
vorr $dat2,$in2,$in2
veor $tmp3,$tmp3,$dat3
vorr $dat3,$in3,$in3
veor $tmp4,$tmp4,$dat4
vst1.8 {$tmp0},[$out],#16
vorr $dat4,$in4,$in4
vst1.8 {$tmp1},[$out],#16
mov $cnt,$rounds
vst1.8 {$tmp2},[$out],#16
vld1.32 {q9},[$key_],#16 // re-pre-load rndkey[1]
vst1.8 {$tmp3},[$out],#16
vst1.8 {$tmp4},[$out],#16
b.hs .Loop5x_cbc_dec
add $len,$len,#0x50
cbz $len,.Lcbc_done
add $cnt,$rounds,#2
subs $len,$len,#0x30
vorr $dat0,$in2,$in2
vorr $in0,$in2,$in2
vorr $dat1,$in3,$in3
vorr $in1,$in3,$in3
vorr $dat2,$in4,$in4
vorr $in2,$in4,$in4
b.lo .Lcbc_dec_tail
b .Loop3x_cbc_dec
.align 4
.Lcbc_tail4x:
veor $tmp1,$tmp0,$dat1
veor $tmp2,$tmp2,$dat2
veor $tmp3,$tmp3,$dat3
veor $tmp4,$tmp4,$dat4
vst1.8 {$tmp1},[$out],#16
vst1.8 {$tmp2},[$out],#16
vst1.8 {$tmp3},[$out],#16
vst1.8 {$tmp4},[$out],#16
b .Lcbc_done
.align 4
___
$code.=<<___;
.Loop3x_cbc_dec:
aesd $dat0,q8
aesimc $dat0,$dat0
@@ -700,6 +912,9 @@ my $step="x12"; # aliases with $tctr2
my ($dat0,$dat1,$in0,$in1,$tmp0,$tmp1,$ivec,$rndlast)=map("q$_",(0..7));
my ($dat2,$in2,$tmp2)=map("q$_",(10,11,9));
# used only in 64-bit mode...
my ($dat3,$dat4,$in3,$in4)=map("q$_",(16..23));
my ($dat,$tmp)=($dat0,$tmp0);
### q8-q15 preloaded key schedule
@@ -752,6 +967,175 @@ $code.=<<___;
rev $tctr2, $ctr
sub $len,$len,#3 // bias
vmov.32 ${dat2}[3],$tctr2
___
$code.=<<___ if ($flavour =~ /64/);
cmp $len,#2
b.lo .Loop3x_ctr32
add w13,$ctr,#1
add w14,$ctr,#2
vorr $dat3,$dat0,$dat0
rev w13,w13
vorr $dat4,$dat0,$dat0
rev w14,w14
vmov.32 ${dat3}[3],w13
sub $len,$len,#2 // bias
vmov.32 ${dat4}[3],w14
add $ctr,$ctr,#2
b .Loop5x_ctr32
.align 4
.Loop5x_ctr32:
aese $dat0,q8
aesmc $dat0,$dat0
aese $dat1,q8
aesmc $dat1,$dat1
aese $dat2,q8
aesmc $dat2,$dat2
aese $dat3,q8
aesmc $dat3,$dat3
aese $dat4,q8
aesmc $dat4,$dat4
vld1.32 {q8},[$key_],#16
subs $cnt,$cnt,#2
aese $dat0,q9
aesmc $dat0,$dat0
aese $dat1,q9
aesmc $dat1,$dat1
aese $dat2,q9
aesmc $dat2,$dat2
aese $dat3,q9
aesmc $dat3,$dat3
aese $dat4,q9
aesmc $dat4,$dat4
vld1.32 {q9},[$key_],#16
b.gt .Loop5x_ctr32
mov $key_,$key
aese $dat0,q8
aesmc $dat0,$dat0
aese $dat1,q8
aesmc $dat1,$dat1
aese $dat2,q8
aesmc $dat2,$dat2
aese $dat3,q8
aesmc $dat3,$dat3
aese $dat4,q8
aesmc $dat4,$dat4
vld1.32 {q8},[$key_],#16 // re-pre-load rndkey[0]
aese $dat0,q9
aesmc $dat0,$dat0
aese $dat1,q9
aesmc $dat1,$dat1
aese $dat2,q9
aesmc $dat2,$dat2
aese $dat3,q9
aesmc $dat3,$dat3
aese $dat4,q9
aesmc $dat4,$dat4
vld1.32 {q9},[$key_],#16 // re-pre-load rndkey[1]
aese $dat0,q12
aesmc $dat0,$dat0
add $tctr0,$ctr,#1
add $tctr1,$ctr,#2
aese $dat1,q12
aesmc $dat1,$dat1
add $tctr2,$ctr,#3
add w13,$ctr,#4
aese $dat2,q12
aesmc $dat2,$dat2
add w14,$ctr,#5
rev $tctr0,$tctr0
aese $dat3,q12
aesmc $dat3,$dat3
rev $tctr1,$tctr1
rev $tctr2,$tctr2
aese $dat4,q12
aesmc $dat4,$dat4
rev w13,w13
rev w14,w14
aese $dat0,q13
aesmc $dat0,$dat0
aese $dat1,q13
aesmc $dat1,$dat1
aese $dat2,q13
aesmc $dat2,$dat2
aese $dat3,q13
aesmc $dat3,$dat3
aese $dat4,q13
aesmc $dat4,$dat4
aese $dat0,q14
aesmc $dat0,$dat0
vld1.8 {$in0},[$inp],#16
aese $dat1,q14
aesmc $dat1,$dat1
vld1.8 {$in1},[$inp],#16
aese $dat2,q14
aesmc $dat2,$dat2
vld1.8 {$in2},[$inp],#16
aese $dat3,q14
aesmc $dat3,$dat3
vld1.8 {$in3},[$inp],#16
aese $dat4,q14
aesmc $dat4,$dat4
vld1.8 {$in4},[$inp],#16
aese $dat0,q15
veor $in0,$in0,$rndlast
aese $dat1,q15
veor $in1,$in1,$rndlast
aese $dat2,q15
veor $in2,$in2,$rndlast
aese $dat3,q15
veor $in3,$in3,$rndlast
aese $dat4,q15
veor $in4,$in4,$rndlast
veor $in0,$in0,$dat0
vorr $dat0,$ivec,$ivec
veor $in1,$in1,$dat1
vorr $dat1,$ivec,$ivec
veor $in2,$in2,$dat2
vorr $dat2,$ivec,$ivec
veor $in3,$in3,$dat3
vorr $dat3,$ivec,$ivec
veor $in4,$in4,$dat4
vorr $dat4,$ivec,$ivec
vst1.8 {$in0},[$out],#16
vmov.32 ${dat0}[3],$tctr0
vst1.8 {$in1},[$out],#16
vmov.32 ${dat1}[3],$tctr1
vst1.8 {$in2},[$out],#16
vmov.32 ${dat2}[3],$tctr2
vst1.8 {$in3},[$out],#16
vmov.32 ${dat3}[3],w13
vst1.8 {$in4},[$out],#16
vmov.32 ${dat4}[3],w14
mov $cnt,$rounds
cbz $len,.Lctr32_done
add $ctr,$ctr,#5
subs $len,$len,#5
b.hs .Loop5x_ctr32
add $len,$len,#5
sub $ctr,$ctr,#5
cmp $len,#2
mov $step,#16
cclr $step,lo
b.ls .Lctr32_tail
sub $len,$len,#3 // bias
add $ctr,$ctr,#3
___
$code.=<<___;
b .Loop3x_ctr32
.align 4
crypto/aes/asm/vpaes-armv8.pl
+1
View File
@@ -30,6 +30,7 @@
# Denver(***) 16.6(**) 15.1/17.8(**) [8.80/9.93 ]
# Apple A7(***) 22.7(**) 10.9/14.3 [8.45/10.0 ]
# Mongoose(***) 26.3(**) 21.0/25.0(**) [13.3/16.8 ]
# ThunderX2(***) 39.4(**) 33.8/48.6(**)
#
# (*) ECB denotes approximate result for parallelizable modes
# such as CBC decrypt, CTR, etc.;
crypto/bio/bss_mem.c
+29 -12
View File
@@ -57,7 +57,12 @@ static const BIO_METHOD secmem_method = {
NULL, /* mem_callback_ctrl */
};
/* BIO memory stores buffer and read pointer */
/*
* BIO memory stores buffer and read pointer
* however the roles are different for read only BIOs.
* In that case the readp just stores the original state
* to be used for reset.
*/
typedef struct bio_buf_mem_st {
struct buf_mem_st *buf; /* allocated buffer */
struct buf_mem_st *readp; /* read pointer */
@@ -192,11 +197,14 @@ static int mem_read(BIO *b, char *out, int outl)
BIO_BUF_MEM *bbm = (BIO_BUF_MEM *)b->ptr;
BUF_MEM *bm = bbm->readp;
if (b->flags & BIO_FLAGS_MEM_RDONLY)
bm = bbm->buf;
BIO_clear_retry_flags(b);
ret = (outl >= 0 && (size_t)outl > bm->length) ? (int)bm->length : outl;
if ((out != NULL) && (ret > 0)) {
memcpy(out, bm->data, ret);
bm->length -= ret;
bm->max -= ret;
bm->data += ret;
} else if (bm->length == 0) {
ret = b->num;
@@ -241,29 +249,36 @@ static long mem_ctrl(BIO *b, int cmd, long num, void *ptr)
BIO_BUF_MEM *bbm = (BIO_BUF_MEM *)b->ptr;
BUF_MEM *bm;
if (b->flags & BIO_FLAGS_MEM_RDONLY)
bm = bbm->buf;
else
bm = bbm->readp;
switch (cmd) {
case BIO_CTRL_RESET:
bm = bbm->buf;
if (bm->data != NULL) {
/* For read only case reset to the start again */
if ((b->flags & BIO_FLAGS_MEM_RDONLY) || (b->flags & BIO_FLAGS_NONCLEAR_RST)) {
bm->length = bm->max;
if (!(b->flags & BIO_FLAGS_MEM_RDONLY)) {
if (b->flags & BIO_FLAGS_NONCLEAR_RST) {
bm->length = bm->max;
} else {
memset(bm->data, 0, bm->max);
bm->length = 0;
}
*bbm->readp = *bbm->buf;
} else {
memset(bm->data, 0, bm->max);
bm->length = 0;
/* For read only case just reset to the start again */
*bbm->buf = *bbm->readp;
}
*bbm->readp = *bbm->buf;
}
break;
case BIO_CTRL_EOF:
bm = bbm->readp;
ret = (long)(bm->length == 0);
break;
case BIO_C_SET_BUF_MEM_EOF_RETURN:
b->num = (int)num;
break;
case BIO_CTRL_INFO:
bm = bbm->readp;
ret = (long)bm->length;
if (ptr != NULL) {
pptr = (char **)ptr;
@@ -278,8 +293,9 @@ static long mem_ctrl(BIO *b, int cmd, long num, void *ptr)
break;
case BIO_C_GET_BUF_MEM_PTR:
if (ptr != NULL) {
mem_buf_sync(b);
bm = bbm->readp;
if (!(b->flags & BIO_FLAGS_MEM_RDONLY))
mem_buf_sync(b);
bm = bbm->buf;
pptr = (char **)ptr;
*pptr = (char *)bm;
}
@@ -294,7 +310,6 @@ static long mem_ctrl(BIO *b, int cmd, long num, void *ptr)
ret = 0L;
break;
case BIO_CTRL_PENDING:
bm = bbm->readp;
ret = (long)bm->length;
break;
case BIO_CTRL_DUP:
@@ -318,6 +333,8 @@ static int mem_gets(BIO *bp, char *buf, int size)
BIO_BUF_MEM *bbm = (BIO_BUF_MEM *)bp->ptr;
BUF_MEM *bm = bbm->readp;
if (bp->flags & BIO_FLAGS_MEM_RDONLY)
bm = bbm->buf;
BIO_clear_retry_flags(bp);
j = bm->length;
if ((size - 1) < j)
crypto/bn/bn_prime.c
+2 -2
View File
@@ -329,8 +329,6 @@ int bn_miller_rabin_is_prime(const BIGNUM *w, int iterations, BN_CTX *ctx,
if (BN_is_one(z))
goto composite;
}
if (!BN_GENCB_call(cb, 1, i))
goto err;
/* At this point z = b^((w-1)/2) mod w */
/* (Steps 4.8 - 4.9) x = z, z = x^2 mod w */
if (!BN_copy(x, z) || !BN_mod_mul(z, x, x, w, ctx))
@@ -358,6 +356,8 @@ composite:
goto err;
outer_loop: ;
/* (Step 4.1.5) */
if (!BN_GENCB_call(cb, 1, i))
goto err;
}
/* (Step 5) */
*status = BN_PRIMETEST_PROBABLY_PRIME;
crypto/build.info
+1 -1
View File
@@ -14,7 +14,7 @@ SOURCE[../libcrypto]=provider_core.c provider_predefined.c provider_conf.c \
# Central utilities
SOURCE[../libcrypto]=\
cryptlib.c mem.c mem_dbg.c cversion.c ex_data.c cpt_err.c \
cryptlib.c mem.c mem_dbg.c cversion.c info.c ex_data.c cpt_err.c \
ebcdic.c uid.c o_time.c o_str.c o_dir.c o_fopen.c ctype.c \
threads_pthread.c threads_win.c threads_none.c getenv.c \
o_init.c o_fips.c mem_sec.c init.c context.c sparse_array.c \
crypto/chacha/asm/chacha-armv8.pl
+347 -194
View File
@@ -18,22 +18,31 @@
#
# ChaCha20 for ARMv8.
#
# April 2019
#
# Replace 3xNEON+1xIALU code path with 4+1. 4+1 is actually fastest
# option on most(*), but not all, processors, yet 6+2 is retained.
# This is because penalties are considered tolerable in comparison to
# improvement on processors where 6+2 helps. Most notably +37% on
# ThunderX2. It's server-oriented processor which will have to serve
# as many requests as possible. While others are mostly clients, when
# performance doesn't have to be absolute top-notch, just fast enough,
# as majority of time is spent "entertaining" relatively slow human.
#
# Performance in cycles per byte out of large buffer.
#
# IALU/gcc-4.9 3xNEON+1xIALU 6xNEON+2xIALU
# IALU/gcc-4.9 4xNEON+1xIALU 6xNEON+2xIALU
#
# Apple A7 5.50/+49% 3.33 1.70
# Cortex-A53 8.40/+80% 4.72 4.72(*)
# Cortex-A57 8.06/+43% 4.90 4.43(**)
# Denver 4.50/+82% 2.63 2.67(*)
# X-Gene 9.50/+46% 8.82 8.89(*)
# Mongoose 8.00/+44% 3.64 3.25
# Kryo 8.17/+50% 4.83 4.65
# Apple A7 5.50/+49% 2.72 1.60
# Cortex-A53 8.40/+80% 4.06 4.45(*)
# Cortex-A57 8.06/+43% 4.15 4.40(*)
# Denver 4.50/+82% 2.30 2.70(*)
# X-Gene 9.50/+46% 8.20 8.90(*)
# Mongoose 8.00/+44% 2.74 3.12(*)
# Kryo 8.17/+50% 4.47 4.65(*)
# ThunderX2 7.22/+48% 5.64 4.10
#
# (*) it's expected that doubling interleave factor doesn't help
# all processors, only those with higher NEON latency and
# higher instruction issue rate;
# (**) expected improvement was actually higher;
# (*) slower than 4+1:-(
$flavour=shift;
$output=shift;
@@ -120,18 +129,21 @@ my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
}
$code.=<<___;
#include "arm_arch.h"
#ifndef __KERNEL__
# include "arm_arch.h"
.extern OPENSSL_armcap_P
#endif
.text
.extern OPENSSL_armcap_P
.align 5
.Lsigma:
.quad 0x3320646e61707865,0x6b20657479622d32 // endian-neutral
.Lone:
.long 1,0,0,0
.asciz "ChaCha20 for ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
.long 1,2,3,4
.Lrot24:
.long 0x02010003,0x06050407,0x0a09080b,0x0e0d0c0f
.asciz "ChaCha20 for ARMv8, CRYPTOGAMS by \@dot-asm"
.globl ChaCha20_ctr32
.type ChaCha20_ctr32,%function
@@ -141,10 +153,12 @@ ChaCha20_ctr32:
cmp $len,#192
b.lo .Lshort
#ifndef __KERNEL__
adrp x17,OPENSSL_armcap_P
ldr w17,[x17,#:lo12:OPENSSL_armcap_P]
tst w17,#ARMV7_NEON
b.ne .LChaCha20_neon
#endif
.Lshort:
.inst 0xd503233f // paciasp
@@ -163,7 +177,7 @@ ChaCha20_ctr32:
ldp @d[2],@d[3],[$key] // load key
ldp @d[4],@d[5],[$key,#16]
ldp @d[6],@d[7],[$ctr] // load counter
#ifdef __ARMEB__
#ifdef __AARCH64EB__
ror @d[2],@d[2],#32
ror @d[3],@d[3],#32
ror @d[4],@d[4],#32
@@ -232,7 +246,7 @@ $code.=<<___;
add @x[14],@x[14],@x[15],lsl#32
ldp @x[13],@x[15],[$inp,#48]
add $inp,$inp,#64
#ifdef __ARMEB__
#ifdef __AARCH64EB__
rev @x[0],@x[0]
rev @x[2],@x[2]
rev @x[4],@x[4]
@@ -289,7 +303,7 @@ $code.=<<___;
add @x[10],@x[10],@x[11],lsl#32
add @x[12],@x[12],@x[13],lsl#32
add @x[14],@x[14],@x[15],lsl#32
#ifdef __ARMEB__
#ifdef __AARCH64EB__
rev @x[0],@x[0]
rev @x[2],@x[2]
rev @x[4],@x[4]
@@ -330,43 +344,87 @@ $code.=<<___;
___
{{{
my ($A0,$B0,$C0,$D0,$A1,$B1,$C1,$D1,$A2,$B2,$C2,$D2,$T0,$T1,$T2,$T3) =
map("v$_.4s",(0..7,16..23));
my (@K)=map("v$_.4s",(24..30));
my $ONE="v31.4s";
my @K = map("v$_.4s",(0..3));
my ($xt0,$xt1,$xt2,$xt3, $CTR,$ROT24) = map("v$_.4s",(4..9));
my @X = map("v$_.4s",(16,20,24,28, 17,21,25,29, 18,22,26,30, 19,23,27,31));
my ($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3,
$xc0,$xc1,$xc2,$xc3, $xd0,$xd1,$xd2,$xd3) = @X;
sub NEONROUND {
my $odd = pop;
my ($a,$b,$c,$d,$t)=@_;
sub NEON_lane_ROUND {
my ($a0,$b0,$c0,$d0)=@_;
my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
my @x=map("'$_'",@X);
(
"&add ('$a','$a','$b')",
"&eor ('$d','$d','$a')",
"&rev32_16 ('$d','$d')", # vrot ($d,16)
"&add (@x[$a0],@x[$a0],@x[$b0])", # Q1
"&add (@x[$a1],@x[$a1],@x[$b1])", # Q2
"&add (@x[$a2],@x[$a2],@x[$b2])", # Q3
"&add (@x[$a3],@x[$a3],@x[$b3])", # Q4
"&eor (@x[$d0],@x[$d0],@x[$a0])",
"&eor (@x[$d1],@x[$d1],@x[$a1])",
"&eor (@x[$d2],@x[$d2],@x[$a2])",
"&eor (@x[$d3],@x[$d3],@x[$a3])",
"&rev32_16 (@x[$d0],@x[$d0])",
"&rev32_16 (@x[$d1],@x[$d1])",
"&rev32_16 (@x[$d2],@x[$d2])",
"&rev32_16 (@x[$d3],@x[$d3])",
"&add ('$c','$c','$d')",
"&eor ('$t','$b','$c')",
"&ushr ('$b','$t',20)",
"&sli ('$b','$t',12)",
"&add (@x[$c0],@x[$c0],@x[$d0])",
"&add (@x[$c1],@x[$c1],@x[$d1])",
"&add (@x[$c2],@x[$c2],@x[$d2])",
"&add (@x[$c3],@x[$c3],@x[$d3])",
"&eor ('$xt0',@x[$b0],@x[$c0])",
"&eor ('$xt1',@x[$b1],@x[$c1])",
"&eor ('$xt2',@x[$b2],@x[$c2])",
"&eor ('$xt3',@x[$b3],@x[$c3])",
"&ushr (@x[$b0],'$xt0',20)",
"&ushr (@x[$b1],'$xt1',20)",
"&ushr (@x[$b2],'$xt2',20)",
"&ushr (@x[$b3],'$xt3',20)",
"&sli (@x[$b0],'$xt0',12)",
"&sli (@x[$b1],'$xt1',12)",
"&sli (@x[$b2],'$xt2',12)",
"&sli (@x[$b3],'$xt3',12)",
"&add ('$a','$a','$b')",
"&eor ('$t','$d','$a')",
"&ushr ('$d','$t',24)",
"&sli ('$d','$t',8)",
"&add (@x[$a0],@x[$a0],@x[$b0])",
"&add (@x[$a1],@x[$a1],@x[$b1])",
"&add (@x[$a2],@x[$a2],@x[$b2])",
"&add (@x[$a3],@x[$a3],@x[$b3])",
"&eor ('$xt0',@x[$d0],@x[$a0])",
"&eor ('$xt1',@x[$d1],@x[$a1])",
"&eor ('$xt2',@x[$d2],@x[$a2])",
"&eor ('$xt3',@x[$d3],@x[$a3])",
"&tbl (@x[$d0],'{$xt0}','$ROT24')",
"&tbl (@x[$d1],'{$xt1}','$ROT24')",
"&tbl (@x[$d2],'{$xt2}','$ROT24')",
"&tbl (@x[$d3],'{$xt3}','$ROT24')",
"&add ('$c','$c','$d')",
"&eor ('$t','$b','$c')",
"&ushr ('$b','$t',25)",
"&sli ('$b','$t',7)",
"&ext ('$c','$c','$c',8)",
"&ext ('$d','$d','$d',$odd?4:12)",
"&ext ('$b','$b','$b',$odd?12:4)"
"&add (@x[$c0],@x[$c0],@x[$d0])",
"&add (@x[$c1],@x[$c1],@x[$d1])",
"&add (@x[$c2],@x[$c2],@x[$d2])",
"&add (@x[$c3],@x[$c3],@x[$d3])",
"&eor ('$xt0',@x[$b0],@x[$c0])",
"&eor ('$xt1',@x[$b1],@x[$c1])",
"&eor ('$xt2',@x[$b2],@x[$c2])",
"&eor ('$xt3',@x[$b3],@x[$c3])",
"&ushr (@x[$b0],'$xt0',25)",
"&ushr (@x[$b1],'$xt1',25)",
"&ushr (@x[$b2],'$xt2',25)",
"&ushr (@x[$b3],'$xt3',25)",
"&sli (@x[$b0],'$xt0',7)",
"&sli (@x[$b1],'$xt1',7)",
"&sli (@x[$b2],'$xt2',7)",
"&sli (@x[$b3],'$xt3',7)"
);
}
$code.=<<___;
#ifdef __KERNEL__
.globl ChaCha20_neon
#endif
.type ChaCha20_neon,%function
.align 5
ChaCha20_neon:
@@ -393,8 +451,9 @@ ChaCha20_neon:
ld1 {@K[1],@K[2]},[$key]
ldp @d[6],@d[7],[$ctr] // load counter
ld1 {@K[3]},[$ctr]
ld1 {$ONE},[@x[0]]
#ifdef __ARMEB__
stp d8,d9,[sp] // meet ABI requirements
ld1 {$CTR,$ROT24},[@x[0]]
#ifdef __AARCH64EB__
rev64 @K[0],@K[0]
ror @d[2],@d[2],#32
ror @d[3],@d[3],#32
@@ -403,115 +462,129 @@ ChaCha20_neon:
ror @d[6],@d[6],#32
ror @d[7],@d[7],#32
#endif
add @K[3],@K[3],$ONE // += 1
add @K[4],@K[3],$ONE
add @K[5],@K[4],$ONE
shl $ONE,$ONE,#2 // 1 -> 4
.Loop_outer_neon:
mov.32 @x[0],@d[0] // unpack key block
lsr @x[1],@d[0],#32
mov $A0,@K[0]
mov.32 @x[2],@d[1]
lsr @x[3],@d[1],#32
mov $A1,@K[0]
mov.32 @x[4],@d[2]
lsr @x[5],@d[2],#32
mov $A2,@K[0]
mov.32 @x[6],@d[3]
mov $B0,@K[1]
lsr @x[7],@d[3],#32
mov $B1,@K[1]
mov.32 @x[8],@d[4]
mov $B2,@K[1]
lsr @x[9],@d[4],#32
mov $D0,@K[3]
mov.32 @x[10],@d[5]
mov $D1,@K[4]
lsr @x[11],@d[5],#32
mov $D2,@K[5]
mov.32 @x[12],@d[6]
mov $C0,@K[2]
lsr @x[13],@d[6],#32
mov $C1,@K[2]
mov.32 @x[14],@d[7]
mov $C2,@K[2]
lsr @x[15],@d[7],#32
dup $xa0,@{K[0]}[0] // unpack key block
mov.32 @x[0],@d[0]
dup $xa1,@{K[0]}[1]
lsr @x[1],@d[0],#32
dup $xa2,@{K[0]}[2]
mov.32 @x[2],@d[1]
dup $xa3,@{K[0]}[3]
lsr @x[3],@d[1],#32
dup $xb0,@{K[1]}[0]
mov.32 @x[4],@d[2]
dup $xb1,@{K[1]}[1]
lsr @x[5],@d[2],#32
dup $xb2,@{K[1]}[2]
mov.32 @x[6],@d[3]
dup $xb3,@{K[1]}[3]
lsr @x[7],@d[3],#32
dup $xd0,@{K[3]}[0]
mov.32 @x[8],@d[4]
dup $xd1,@{K[3]}[1]
lsr @x[9],@d[4],#32
dup $xd2,@{K[3]}[2]
mov.32 @x[10],@d[5]
dup $xd3,@{K[3]}[3]
lsr @x[11],@d[5],#32
add $xd0,$xd0,$CTR
mov.32 @x[12],@d[6]
dup $xc0,@{K[2]}[0]
lsr @x[13],@d[6],#32
dup $xc1,@{K[2]}[1]
mov.32 @x[14],@d[7]
dup $xc2,@{K[2]}[2]
lsr @x[15],@d[7],#32
dup $xc3,@{K[2]}[3]
mov $ctr,#10
subs $len,$len,#256
subs $len,$len,#320
.Loop_neon:
sub $ctr,$ctr,#1
___
my @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,0);
my @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,0);
my @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,0);
my @thread3=&ROUND(0,4,8,12);
my @plus_one=&ROUND(0,4,8,12);
foreach (&NEON_lane_ROUND(0,4,8,12)) { eval; eval(shift(@plus_one)); }
foreach (@thread0) {
eval; eval(shift(@thread3));
eval(shift(@thread1)); eval(shift(@thread3));
eval(shift(@thread2)); eval(shift(@thread3));
}
@thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,1);
@thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,1);
@thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,1);
@thread3=&ROUND(0,5,10,15);
foreach (@thread0) {
eval; eval(shift(@thread3));
eval(shift(@thread1)); eval(shift(@thread3));
eval(shift(@thread2)); eval(shift(@thread3));
}
@plus_one=&ROUND(0,5,10,15);
foreach (&NEON_lane_ROUND(0,5,10,15)) { eval; eval(shift(@plus_one)); }
$code.=<<___;
cbnz $ctr,.Loop_neon
add.32 @x[0],@x[0],@d[0] // accumulate key block
add $A0,$A0,@K[0]
add @x[1],@x[1],@d[0],lsr#32
add $A1,$A1,@K[0]
add.32 @x[2],@x[2],@d[1]
add $A2,$A2,@K[0]
add @x[3],@x[3],@d[1],lsr#32
add $C0,$C0,@K[2]
add.32 @x[4],@x[4],@d[2]
add $C1,$C1,@K[2]
add @x[5],@x[5],@d[2],lsr#32
add $C2,$C2,@K[2]
add.32 @x[6],@x[6],@d[3]
add $D0,$D0,@K[3]
add @x[7],@x[7],@d[3],lsr#32
add.32 @x[8],@x[8],@d[4]
add $D1,$D1,@K[4]
add @x[9],@x[9],@d[4],lsr#32
add.32 @x[10],@x[10],@d[5]
add $D2,$D2,@K[5]
add @x[11],@x[11],@d[5],lsr#32
add.32 @x[12],@x[12],@d[6]
add $B0,$B0,@K[1]
add @x[13],@x[13],@d[6],lsr#32
add.32 @x[14],@x[14],@d[7]
add $B1,$B1,@K[1]
add @x[15],@x[15],@d[7],lsr#32
add $B2,$B2,@K[1]
add $xd0,$xd0,$CTR
zip1 $xt0,$xa0,$xa1 // transpose data
zip1 $xt1,$xa2,$xa3
zip2 $xt2,$xa0,$xa1
zip2 $xt3,$xa2,$xa3
zip1.64 $xa0,$xt0,$xt1
zip2.64 $xa1,$xt0,$xt1
zip1.64 $xa2,$xt2,$xt3
zip2.64 $xa3,$xt2,$xt3
zip1 $xt0,$xb0,$xb1
zip1 $xt1,$xb2,$xb3
zip2 $xt2,$xb0,$xb1
zip2 $xt3,$xb2,$xb3
zip1.64 $xb0,$xt0,$xt1
zip2.64 $xb1,$xt0,$xt1
zip1.64 $xb2,$xt2,$xt3
zip2.64 $xb3,$xt2,$xt3
zip1 $xt0,$xc0,$xc1
add.32 @x[0],@x[0],@d[0] // accumulate key block
zip1 $xt1,$xc2,$xc3
add @x[1],@x[1],@d[0],lsr#32
zip2 $xt2,$xc0,$xc1
add.32 @x[2],@x[2],@d[1]
zip2 $xt3,$xc2,$xc3
add @x[3],@x[3],@d[1],lsr#32
zip1.64 $xc0,$xt0,$xt1
add.32 @x[4],@x[4],@d[2]
zip2.64 $xc1,$xt0,$xt1
add @x[5],@x[5],@d[2],lsr#32
zip1.64 $xc2,$xt2,$xt3
add.32 @x[6],@x[6],@d[3]
zip2.64 $xc3,$xt2,$xt3
add @x[7],@x[7],@d[3],lsr#32
zip1 $xt0,$xd0,$xd1
add.32 @x[8],@x[8],@d[4]
zip1 $xt1,$xd2,$xd3
add @x[9],@x[9],@d[4],lsr#32
zip2 $xt2,$xd0,$xd1
add.32 @x[10],@x[10],@d[5]
zip2 $xt3,$xd2,$xd3
add @x[11],@x[11],@d[5],lsr#32
zip1.64 $xd0,$xt0,$xt1
add.32 @x[12],@x[12],@d[6]
zip2.64 $xd1,$xt0,$xt1
add @x[13],@x[13],@d[6],lsr#32
zip1.64 $xd2,$xt2,$xt3
add.32 @x[14],@x[14],@d[7]
zip2.64 $xd3,$xt2,$xt3
add @x[15],@x[15],@d[7],lsr#32
b.lo .Ltail_neon
add @x[0],@x[0],@x[1],lsl#32 // pack
add @x[2],@x[2],@x[3],lsl#32
ldp @x[1],@x[3],[$inp,#0] // load input
add $xa0,$xa0,@K[0] // accumulate key block
add @x[4],@x[4],@x[5],lsl#32
add @x[6],@x[6],@x[7],lsl#32
ldp @x[5],@x[7],[$inp,#16]
add $xb0,$xb0,@K[1]
add @x[8],@x[8],@x[9],lsl#32
add @x[10],@x[10],@x[11],lsl#32
ldp @x[9],@x[11],[$inp,#32]
add $xc0,$xc0,@K[2]
add @x[12],@x[12],@x[13],lsl#32
add @x[14],@x[14],@x[15],lsl#32
ldp @x[13],@x[15],[$inp,#48]
add $xd0,$xd0,@K[3]
add $inp,$inp,#64
#ifdef __ARMEB__
#ifdef __AARCH64EB__
rev @x[0],@x[0]
rev @x[2],@x[2]
rev @x[4],@x[4]
@@ -521,48 +594,68 @@ $code.=<<___;
rev @x[12],@x[12]
rev @x[14],@x[14]
#endif
ld1.8 {$T0-$T3},[$inp],#64
ld1.8 {$xt0-$xt3},[$inp],#64
eor @x[0],@x[0],@x[1]
add $xa1,$xa1,@K[0]
eor @x[2],@x[2],@x[3]
add $xb1,$xb1,@K[1]
eor @x[4],@x[4],@x[5]
add $xc1,$xc1,@K[2]
eor @x[6],@x[6],@x[7]
add $xd1,$xd1,@K[3]
eor @x[8],@x[8],@x[9]
eor $A0,$A0,$T0
eor $xa0,$xa0,$xt0
movi $xt0,#5
eor @x[10],@x[10],@x[11]
eor $B0,$B0,$T1
eor $xb0,$xb0,$xt1
eor @x[12],@x[12],@x[13]
eor $C0,$C0,$T2
eor $xc0,$xc0,$xt2
eor @x[14],@x[14],@x[15]
eor $D0,$D0,$T3
ld1.8 {$T0-$T3},[$inp],#64
eor $xd0,$xd0,$xt3
add $CTR,$CTR,$xt0 // += 5
ld1.8 {$xt0-$xt3},[$inp],#64
stp @x[0],@x[2],[$out,#0] // store output
add @d[6],@d[6],#4 // increment counter
add @d[6],@d[6],#5 // increment counter
stp @x[4],@x[6],[$out,#16]
add @K[3],@K[3],$ONE // += 4
stp @x[8],@x[10],[$out,#32]
add @K[4],@K[4],$ONE
stp @x[12],@x[14],[$out,#48]
add @K[5],@K[5],$ONE
add $out,$out,#64
st1.8 {$A0-$D0},[$out],#64
ld1.8 {$A0-$D0},[$inp],#64
st1.8 {$xa0-$xd0},[$out],#64
add $xa2,$xa2,@K[0]
add $xb2,$xb2,@K[1]
add $xc2,$xc2,@K[2]
add $xd2,$xd2,@K[3]
ld1.8 {$xa0-$xd0},[$inp],#64
eor $A1,$A1,$T0
eor $B1,$B1,$T1
eor $C1,$C1,$T2
eor $D1,$D1,$T3
st1.8 {$A1-$D1},[$out],#64
eor $xa1,$xa1,$xt0
eor $xb1,$xb1,$xt1
eor $xc1,$xc1,$xt2
eor $xd1,$xd1,$xt3
st1.8 {$xa1-$xd1},[$out],#64
add $xa3,$xa3,@K[0]
add $xb3,$xb3,@K[1]
add $xc3,$xc3,@K[2]
add $xd3,$xd3,@K[3]
ld1.8 {$xa1-$xd1},[$inp],#64
eor $A2,$A2,$A0
eor $B2,$B2,$B0
eor $C2,$C2,$C0
eor $D2,$D2,$D0
st1.8 {$A2-$D2},[$out],#64
eor $xa2,$xa2,$xa0
eor $xb2,$xb2,$xb0
eor $xc2,$xc2,$xc0
eor $xd2,$xd2,$xd0
st1.8 {$xa2-$xd2},[$out],#64
eor $xa3,$xa3,$xa1
eor $xb3,$xb3,$xb1
eor $xc3,$xc3,$xc1
eor $xd3,$xd3,$xd1
st1.8 {$xa3-$xd3},[$out],#64
b.hi .Loop_outer_neon
ldp d8,d9,[sp] // meet ABI requirements
ldp x19,x20,[x29,#16]
add sp,sp,#64
ldp x21,x22,[x29,#32]
@@ -573,8 +666,10 @@ $code.=<<___;
.inst 0xd50323bf // autiasp
ret
.align 4
.Ltail_neon:
add $len,$len,#256
add $len,$len,#320
ldp d8,d9,[sp] // meet ABI requirements
cmp $len,#64
b.lo .Less_than_64
@@ -591,7 +686,7 @@ $code.=<<___;
add @x[14],@x[14],@x[15],lsl#32
ldp @x[13],@x[15],[$inp,#48]
add $inp,$inp,#64
#ifdef __ARMEB__
#ifdef __AARCH64EB__
rev @x[0],@x[0]
rev @x[2],@x[2]
rev @x[4],@x[4]
@@ -611,48 +706,68 @@ $code.=<<___;
eor @x[14],@x[14],@x[15]
stp @x[0],@x[2],[$out,#0] // store output
add @d[6],@d[6],#4 // increment counter
add $xa0,$xa0,@K[0] // accumulate key block
stp @x[4],@x[6],[$out,#16]
add $xb0,$xb0,@K[1]
stp @x[8],@x[10],[$out,#32]
add $xc0,$xc0,@K[2]
stp @x[12],@x[14],[$out,#48]
add $xd0,$xd0,@K[3]
add $out,$out,#64
b.eq .Ldone_neon
sub $len,$len,#64
cmp $len,#64
b.lo .Less_than_128
b.lo .Last_neon
ld1.8 {$T0-$T3},[$inp],#64
eor $A0,$A0,$T0
eor $B0,$B0,$T1
eor $C0,$C0,$T2
eor $D0,$D0,$T3
st1.8 {$A0-$D0},[$out],#64
ld1.8 {$xt0-$xt3},[$inp],#64
eor $xa0,$xa0,$xt0
eor $xb0,$xb0,$xt1
eor $xc0,$xc0,$xt2
eor $xd0,$xd0,$xt3
st1.8 {$xa0-$xd0},[$out],#64
b.eq .Ldone_neon
add $xa0,$xa1,@K[0]
add $xb0,$xb1,@K[1]
sub $len,$len,#64
add $xc0,$xc1,@K[2]
cmp $len,#64
b.lo .Less_than_192
add $xd0,$xd1,@K[3]
b.lo .Last_neon
ld1.8 {$T0-$T3},[$inp],#64
eor $A1,$A1,$T0
eor $B1,$B1,$T1
eor $C1,$C1,$T2
eor $D1,$D1,$T3
st1.8 {$A1-$D1},[$out],#64
ld1.8 {$xt0-$xt3},[$inp],#64
eor $xa1,$xa0,$xt0
eor $xb1,$xb0,$xt1
eor $xc1,$xc0,$xt2
eor $xd1,$xd0,$xt3
st1.8 {$xa1-$xd1},[$out],#64
b.eq .Ldone_neon
add $xa0,$xa2,@K[0]
add $xb0,$xb2,@K[1]
sub $len,$len,#64
add $xc0,$xc2,@K[2]
cmp $len,#64
add $xd0,$xd2,@K[3]
b.lo .Last_neon
ld1.8 {$xt0-$xt3},[$inp],#64
eor $xa2,$xa0,$xt0
eor $xb2,$xb0,$xt1
eor $xc2,$xc0,$xt2
eor $xd2,$xd0,$xt3
st1.8 {$xa2-$xd2},[$out],#64
b.eq .Ldone_neon
add $xa0,$xa3,@K[0]
add $xb0,$xb3,@K[1]
add $xc0,$xc3,@K[2]
add $xd0,$xd3,@K[3]
sub $len,$len,#64
st1.8 {$A2-$D2},[sp]
b .Last_neon
.Less_than_128:
st1.8 {$A0-$D0},[sp]
b .Last_neon
.Less_than_192:
st1.8 {$A1-$D1},[sp]
b .Last_neon
.align 4
.Last_neon:
st1.8 {$xa0-$xd0},[sp]
sub $out,$out,#1
add $inp,$inp,$len
add $out,$out,$len
@@ -685,9 +800,41 @@ $code.=<<___;
.size ChaCha20_neon,.-ChaCha20_neon
___
{
my @K = map("v$_.4s",(0..6));
my ($T0,$T1,$T2,$T3,$T4,$T5)=@K;
my ($A0,$B0,$C0,$D0,$A1,$B1,$C1,$D1,$A2,$B2,$C2,$D2,
$A3,$B3,$C3,$D3,$A4,$B4,$C4,$D4,$A5,$B5,$C5,$D5) = map("v$_.4s",(0..23));
$A3,$B3,$C3,$D3,$A4,$B4,$C4,$D4,$A5,$B5,$C5,$D5) = map("v$_.4s",(8..31));
my $rot24 = @K[6];
my $ONE = "v7.4s";
sub NEONROUND {
my $odd = pop;
my ($a,$b,$c,$d,$t)=@_;
(
"&add ('$a','$a','$b')",
"&eor ('$d','$d','$a')",
"&rev32_16 ('$d','$d')", # vrot ($d,16)
"&add ('$c','$c','$d')",
"&eor ('$t','$b','$c')",
"&ushr ('$b','$t',20)",
"&sli ('$b','$t',12)",
"&add ('$a','$a','$b')",
"&eor ('$d','$d','$a')",
"&tbl ('$d','{$d}','$rot24')",
"&add ('$c','$c','$d')",
"&eor ('$t','$b','$c')",
"&ushr ('$b','$t',25)",
"&sli ('$b','$t',7)",
"&ext ('$c','$c','$c',8)",
"&ext ('$d','$d','$d',$odd?4:12)",
"&ext ('$b','$b','$b',$odd?12:4)"
);
}
$code.=<<___;
.type ChaCha20_512_neon,%function
@@ -707,6 +854,7 @@ ChaCha20_512_neon:
.L512_or_more_neon:
sub sp,sp,#128+64
eor $ONE,$ONE,$ONE
ldp @d[0],@d[1],[@x[0]] // load sigma
ld1 {@K[0]},[@x[0]],#16
ldp @d[2],@d[3],[$key] // load key
@@ -714,8 +862,9 @@ ChaCha20_512_neon:
ld1 {@K[1],@K[2]},[$key]
ldp @d[6],@d[7],[$ctr] // load counter
ld1 {@K[3]},[$ctr]
ld1 {$ONE},[@x[0]]
#ifdef __ARMEB__
ld1 {$ONE}[0],[@x[0]]
add $key,@x[0],#16 // .Lrot24
#ifdef __AARCH64EB__
rev64 @K[0],@K[0]
ror @d[2],@d[2],#32
ror @d[3],@d[3],#32
@@ -782,9 +931,10 @@ ChaCha20_512_neon:
mov $C4,@K[2]
stp @K[3],@K[4],[sp,#48] // off-load key block, variable part
mov $C5,@K[2]
str @K[5],[sp,#80]
stp @K[5],@K[6],[sp,#80]
mov $ctr,#5
ld1 {$rot24},[$key]
subs $len,$len,#512
.Loop_upper_neon:
sub $ctr,$ctr,#1
@@ -857,7 +1007,7 @@ $code.=<<___;
add @x[14],@x[14],@x[15],lsl#32
ldp @x[13],@x[15],[$inp,#48]
add $inp,$inp,#64
#ifdef __ARMEB__
#ifdef __AARCH64EB__
rev @x[0],@x[0]
rev @x[2],@x[2]
rev @x[4],@x[4]
@@ -946,6 +1096,7 @@ $code.=<<___;
add.32 @x[2],@x[2],@d[1]
ldp @K[4],@K[5],[sp,#64]
add @x[3],@x[3],@d[1],lsr#32
ldr @K[6],[sp,#96]
add $A0,$A0,@K[0]
add.32 @x[4],@x[4],@d[2]
add $A1,$A1,@K[0]
@@ -998,7 +1149,7 @@ $code.=<<___;
add $inp,$inp,#64
add $B5,$B5,@K[1]
#ifdef __ARMEB__
#ifdef __AARCH64EB__
rev @x[0],@x[0]
rev @x[2],@x[2]
rev @x[4],@x[4]
@@ -1076,24 +1227,24 @@ $code.=<<___;
b.hs .Loop_outer_512_neon
adds $len,$len,#512
ushr $A0,$ONE,#2 // 4 -> 1
ushr $ONE,$ONE,#1 // 4 -> 2
ldp d8,d9,[sp,#128+0] // meet ABI requirements
ldp d10,d11,[sp,#128+16]
ldp d12,d13,[sp,#128+32]
ldp d14,d15,[sp,#128+48]
stp @K[0],$ONE,[sp,#0] // wipe off-load area
stp @K[0],$ONE,[sp,#32]
stp @K[0],$ONE,[sp,#64]
stp @K[0],@K[0],[sp,#0] // wipe off-load area
stp @K[0],@K[0],[sp,#32]
stp @K[0],@K[0],[sp,#64]
b.eq .Ldone_512_neon
sub $key,$key,#16 // .Lone
cmp $len,#192
sub @K[3],@K[3],$A0 // -= 1
sub @K[4],@K[4],$A0
sub @K[5],@K[5],$A0
add sp,sp,#128
sub @K[3],@K[3],$ONE // -= 2
ld1 {$CTR,$ROT24},[$key]
b.hs .Loop_outer_neon
eor @K[1],@K[1],@K[1]
@@ -1123,9 +1274,11 @@ foreach (split("\n",$code)) {
s/\`([^\`]*)\`/eval $1/geo;
(s/\b([a-z]+)\.32\b/$1/ and (s/x([0-9]+)/w$1/g or 1)) or
(m/\b(eor|ext|mov)\b/ and (s/\.4s/\.16b/g or 1)) or
(m/\b(eor|ext|mov|tbl)\b/ and (s/\.4s/\.16b/g or 1)) or
(s/\b((?:ld|st)1)\.8\b/$1/ and (s/\.4s/\.16b/g or 1)) or
(m/\b(ld|st)[rp]\b/ and (s/v([0-9]+)\.4s/q$1/g or 1)) or
(m/\b(dup|ld1)\b/ and (s/\.4(s}?\[[0-3]\])/.$1/g or 1)) or
(s/\b(zip[12])\.64\b/$1/ and (s/\.4s/\.2d/g or 1)) or
(s/\brev32\.16\b/rev32/ and (s/\.4s/\.8h/g or 1));
#s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo;
crypto/chacha/asm/chacha-s390x.pl
+1 -1
View File
@@ -40,7 +40,7 @@
use strict;
use FindBin qw($Bin);
use lib "$Bin/../..";
use perlasm::s390x qw(:DEFAULT :VX AUTOLOAD LABEL INCLUDE);
use perlasm::s390x qw(:DEFAULT :VX :LD AUTOLOAD LABEL INCLUDE);
my $flavour = shift;
crypto/cversion.c
+6
View File
@@ -69,6 +69,12 @@ const char *OpenSSL_version(int t)
return "ENGINESDIR: \"" ENGINESDIR "\"";
#else
return "ENGINESDIR: N/A";
#endif
case OPENSSL_MODULES_DIR:
#ifdef MODULESDIR
return "MODULESDIR: \"" MODULESDIR "\"";
#else
return "MODULESDIR: N/A";
#endif
}
return "not available";
crypto/ec/asm/ecp_nistz256-armv8.pl
+1 -1
View File
@@ -1488,7 +1488,7 @@ $code.=<<___;
////////////////////////////////////////////////////////////////////////
// void ecp_nistz256_ord_sqr_mont(uint64_t res[4], uint64_t a[4],
// int rep);
// uint64_t rep);
.globl ecp_nistz256_ord_sqr_mont
.type ecp_nistz256_ord_sqr_mont,%function
.align 4
crypto/ec/asm/ecp_nistz256-ppc64.pl
+1 -1
View File
@@ -1919,7 +1919,7 @@ $code.=<<___;
################################################################################
# void ecp_nistz256_ord_sqr_mont(uint64_t res[4], uint64_t a[4],
# int rep);
# uint64_t rep);
.globl ecp_nistz256_ord_sqr_mont
.align 5
ecp_nistz256_ord_sqr_mont:
crypto/ec/asm/ecp_nistz256-x86_64.pl
+1 -1
View File
@@ -826,7 +826,7 @@ $code.=<<___;
# void ecp_nistz256_ord_sqr_mont(
# uint64_t res[4],
# uint64_t a[4],
# int rep);
# uint64_t rep);
.globl ecp_nistz256_ord_sqr_mont
.type ecp_nistz256_ord_sqr_mont,\@function,3
crypto/ec/ec_mult.c
+2 -2
View File
@@ -441,7 +441,7 @@ int ec_wNAF_mul(const EC_GROUP *group, EC_POINT *r, const BIGNUM *scalar,
* scalar multiplication implementation based on a Montgomery ladder,
* with various timing attack defenses.
*/
if ((scalar != NULL) && (num == 0)) {
if ((scalar != group->order) && (scalar != NULL) && (num == 0)) {
/*-
* In this case we want to compute scalar * GeneratorPoint: this
* codepath is reached most prominently by (ephemeral) key
@@ -452,7 +452,7 @@ int ec_wNAF_mul(const EC_GROUP *group, EC_POINT *r, const BIGNUM *scalar,
*/
return ec_scalar_mul_ladder(group, r, scalar, NULL, ctx);
}
if ((scalar == NULL) && (num == 1)) {
if ((scalar == NULL) && (num == 1) && (scalars[0] != group->order)) {
/*-
* In this case we want to compute scalar * VariablePoint: this
* codepath is reached most prominently by the second half of ECDH,
crypto/ec/ecp_nistz256.c
+1 -1
View File
@@ -1467,7 +1467,7 @@ void ecp_nistz256_ord_mul_mont(BN_ULONG res[P256_LIMBS],
const BN_ULONG b[P256_LIMBS]);
void ecp_nistz256_ord_sqr_mont(BN_ULONG res[P256_LIMBS],
const BN_ULONG a[P256_LIMBS],
int rep);
BN_ULONG rep);
static int ecp_nistz256_inv_mod_ord(const EC_GROUP *group, BIGNUM *r,
const BIGNUM *x, BN_CTX *ctx)
crypto/err/err.c
+1
View File
@@ -66,6 +66,7 @@ static ERR_STRING_DATA ERR_str_libraries[] = {
{ERR_PACK(ERR_LIB_OSSL_STORE, 0, 0), "STORE routines"},
{ERR_PACK(ERR_LIB_SM2, 0, 0), "SM2 routines"},
{ERR_PACK(ERR_LIB_ESS, 0, 0), "ESS routines"},
{ERR_PACK(ERR_LIB_PROV, 0, 0), "Provider routines"},
{0, NULL},
};
crypto/err/err_all.c
+3 -1
View File
@@ -41,6 +41,7 @@
#include <openssl/storeerr.h>
#include <openssl/esserr.h>
#include "internal/propertyerr.h"
#include "internal/providercommonerr.h"
int err_load_crypto_strings_int(void)
{
@@ -102,7 +103,8 @@ int err_load_crypto_strings_int(void)
#endif
ERR_load_KDF_strings() == 0 ||
ERR_load_OSSL_STORE_strings() == 0 ||
ERR_load_PROP_strings() == 0)
ERR_load_PROP_strings() == 0 ||
ERR_load_PROV_strings() == 0)
return 0;
return 1;
crypto/err/openssl.ec
+1
View File
@@ -37,6 +37,7 @@ L SM2 crypto/include/internal/sm2.h crypto/sm2/sm2_err.c
L OSSL_STORE include/openssl/store.h crypto/store/store_err.c
L ESS include/openssl/ess.h crypto/ess/ess_err.c
L PROP include/internal/property.h crypto/property/property_err.c
L PROV providers/common/include/internal/providercommon.h providers/common/provider_err.c
# additional header files to be scanned for function names
L NONE include/openssl/x509_vfy.h NONE
crypto/err/openssl.txt
+27
View File
@@ -785,6 +785,9 @@ EVP_F_EVP_CIPHER_ASN1_TO_PARAM:204:EVP_CIPHER_asn1_to_param
EVP_F_EVP_CIPHER_CTX_COPY:163:EVP_CIPHER_CTX_copy
EVP_F_EVP_CIPHER_CTX_CTRL:124:EVP_CIPHER_CTX_ctrl
EVP_F_EVP_CIPHER_CTX_SET_KEY_LENGTH:122:EVP_CIPHER_CTX_set_key_length
EVP_F_EVP_CIPHER_CTX_SET_PADDING:237:EVP_CIPHER_CTX_set_padding
EVP_F_EVP_CIPHER_FROM_DISPATCH:238:evp_cipher_from_dispatch
EVP_F_EVP_CIPHER_MODE:239:EVP_CIPHER_mode
EVP_F_EVP_CIPHER_PARAM_TO_ASN1:205:EVP_CIPHER_param_to_asn1
EVP_F_EVP_DECRYPTFINAL_EX:101:EVP_DecryptFinal_ex
EVP_F_EVP_DECRYPTUPDATE:166:EVP_DecryptUpdate
@@ -1102,6 +1105,21 @@ PROP_F_PARSE_NUMBER:104:parse_number
PROP_F_PARSE_OCT:105:parse_oct
PROP_F_PARSE_STRING:106:parse_string
PROP_F_PARSE_UNQUOTED:107:parse_unquoted
PROV_F_AESNI_INIT_KEY:101:aesni_init_key
PROV_F_AES_BLOCK_FINAL:102:aes_block_final
PROV_F_AES_BLOCK_UPDATE:103:aes_block_update
PROV_F_AES_CIPHER:104:aes_cipher
PROV_F_AES_CTX_GET_PARAMS:105:aes_ctx_get_params
PROV_F_AES_CTX_SET_PARAMS:106:aes_ctx_set_params
PROV_F_AES_DINIT:107:aes_dinit
PROV_F_AES_DUPCTX:108:aes_dupctx
PROV_F_AES_EINIT:109:aes_einit
PROV_F_AES_INIT_KEY:110:aes_init_key
PROV_F_AES_STREAM_UPDATE:111:aes_stream_update
PROV_F_AES_T4_INIT_KEY:112:aes_t4_init_key
PROV_F_PROV_AES_KEY_GENERIC_INIT:113:PROV_AES_KEY_generic_init
PROV_F_TRAILINGDATA:114:trailingdata
PROV_F_UNPADBLOCK:100:unpadblock
RAND_F_DRBG_BYTES:101:drbg_bytes
RAND_F_DRBG_CTR_INIT:125:drbg_ctr_init
RAND_F_DRBG_GET_ENTROPY:105:drbg_get_entropy
@@ -2381,6 +2399,7 @@ EVP_R_INVALID_FIPS_MODE:168:invalid fips mode
EVP_R_INVALID_KEY:163:invalid key
EVP_R_INVALID_KEY_LENGTH:130:invalid key length
EVP_R_INVALID_OPERATION:148:invalid operation
EVP_R_INVALID_PROVIDER_FUNCTIONS:193:invalid provider functions
EVP_R_INVALID_SALT_LENGTH:186:invalid salt length
EVP_R_KEYGEN_FAILURE:120:keygen failure
EVP_R_KEY_SETUP_FAILED:180:key setup failed
@@ -2604,6 +2623,14 @@ PROP_R_NO_VALUE:107:no value
PROP_R_PARSE_FAILED:108:parse failed
PROP_R_STRING_TOO_LONG:109:string too long
PROP_R_TRAILING_CHARACTERS:110:trailing characters
PROV_R_AES_KEY_SETUP_FAILED:101:aes key setup failed
PROV_R_BAD_DECRYPT:100:bad decrypt
PROV_R_CIPHER_OPERATION_FAILED:102:cipher operation failed
PROV_R_FAILED_TO_GET_PARAMETER:103:failed to get parameter
PROV_R_FAILED_TO_SET_PARAMETER:104:failed to set parameter
PROV_R_INVALID_KEYLEN:105:invalid keylen
PROV_R_OUTPUT_BUFFER_TOO_SMALL:106:output buffer too small
PROV_R_WRONG_FINAL_BLOCK_LENGTH:107:wrong final block length
RAND_R_ADDITIONAL_INPUT_TOO_LONG:102:additional input too long
RAND_R_ALREADY_INSTANTIATED:103:already instantiated
RAND_R_ARGUMENT_OUT_OF_RANGE:105:argument out of range
crypto/evp/cmeth_lib.c
+30 -2
View File
@@ -11,6 +11,7 @@
#include <openssl/evp.h>
#include "internal/evp_int.h"
#include "internal/provider.h"
#include "evp_locl.h"
EVP_CIPHER *EVP_CIPHER_meth_new(int cipher_type, int block_size, int key_len)
@@ -21,6 +22,12 @@ EVP_CIPHER *EVP_CIPHER_meth_new(int cipher_type, int block_size, int key_len)
cipher->nid = cipher_type;
cipher->block_size = block_size;
cipher->key_len = key_len;
cipher->lock = CRYPTO_THREAD_lock_new();
if (cipher->lock == NULL) {
OPENSSL_free(cipher);
return NULL;
}
cipher->refcnt = 1;
}
return cipher;
}
@@ -30,14 +37,35 @@ EVP_CIPHER *EVP_CIPHER_meth_dup(const EVP_CIPHER *cipher)
EVP_CIPHER *to = EVP_CIPHER_meth_new(cipher->nid, cipher->block_size,
cipher->key_len);
if (to != NULL)
if (to != NULL) {
CRYPTO_RWLOCK *lock = to->lock;
memcpy(to, cipher, sizeof(*to));
to->lock = lock;
}
return to;
}
void EVP_CIPHER_meth_free(EVP_CIPHER *cipher)
{
OPENSSL_free(cipher);
if (cipher != NULL) {
int i;
CRYPTO_DOWN_REF(&cipher->refcnt, &i, cipher->lock);
if (i > 0)
return;
ossl_provider_free(cipher->prov);
CRYPTO_THREAD_lock_free(cipher->lock);
OPENSSL_free(cipher);
}
}
int EVP_CIPHER_upref(EVP_CIPHER *cipher)
{
int ref = 0;
CRYPTO_UP_REF(&cipher->refcnt, &ref, cipher->lock);
return 1;
}
int EVP_CIPHER_meth_set_iv_length(EVP_CIPHER *cipher, int iv_len)
crypto/evp/digest.c
+5 -4
View File
@@ -295,6 +295,7 @@ int EVP_DigestFinal_ex(EVP_MD_CTX *ctx, unsigned char *md, unsigned int *isize)
{
int ret;
size_t size = 0;
size_t mdsize = EVP_MD_size(ctx->digest);
if (ctx->digest == NULL || ctx->digest->prov == NULL)
goto legacy;
@@ -304,7 +305,7 @@ int EVP_DigestFinal_ex(EVP_MD_CTX *ctx, unsigned char *md, unsigned int *isize)
return 0;
}
ret = ctx->digest->dfinal(ctx->provctx, md, &size);
ret = ctx->digest->dfinal(ctx->provctx, md, &size, mdsize);
if (isize != NULL) {
if (size <= UINT_MAX) {
@@ -321,10 +322,10 @@ int EVP_DigestFinal_ex(EVP_MD_CTX *ctx, unsigned char *md, unsigned int *isize)
/* TODO(3.0): Remove legacy code below */
legacy:
OPENSSL_assert(ctx->digest->md_size <= EVP_MAX_MD_SIZE);
OPENSSL_assert(mdsize <= EVP_MAX_MD_SIZE);
ret = ctx->digest->final(ctx, md);
if (isize != NULL)
*isize = ctx->digest->md_size;
*isize = mdsize;
if (ctx->digest->cleanup) {
ctx->digest->cleanup(ctx);
EVP_MD_CTX_set_flags(ctx, EVP_MD_CTX_FLAG_CLEANED);
@@ -516,7 +517,7 @@ static void *evp_md_from_dispatch(int mdtype, const OSSL_DISPATCH *fns,
md->dinit = OSSL_get_OP_digest_init(fns);
fncnt++;
break;
case OSSL_FUNC_DIGEST_UPDDATE:
case OSSL_FUNC_DIGEST_UPDATE:
if (md->dupdate != NULL)
break;
md->dupdate = OSSL_get_OP_digest_update(fns);
crypto/evp/evp_enc.c
+510 -27
View File
@@ -15,25 +15,46 @@
#include <openssl/rand.h>
#include <openssl/rand_drbg.h>
#include <openssl/engine.h>
#include <openssl/params.h>
#include <openssl/core_names.h>
#include "internal/evp_int.h"
#include "internal/provider.h"
#include "evp_locl.h"
int EVP_CIPHER_CTX_reset(EVP_CIPHER_CTX *c)
int EVP_CIPHER_CTX_reset(EVP_CIPHER_CTX *ctx)
{
if (c == NULL)
if (ctx == NULL)
return 1;
if (c->cipher != NULL) {
if (c->cipher->cleanup && !c->cipher->cleanup(c))
if (ctx->cipher == NULL || ctx->cipher->prov == NULL)
goto legacy;
if (ctx->provctx != NULL) {
if (ctx->cipher->freectx != NULL)
ctx->cipher->freectx(ctx->provctx);
ctx->provctx = NULL;
}
if (ctx->fetched_cipher != NULL)
EVP_CIPHER_meth_free(ctx->fetched_cipher);
memset(ctx, 0, sizeof(*ctx));
return 1;
/* TODO(3.0): Remove legacy code below */
legacy:
if (ctx->cipher != NULL) {
if (ctx->cipher->cleanup && !ctx->cipher->cleanup(ctx))
return 0;
/* Cleanse cipher context data */
if (c->cipher_data && c->cipher->ctx_size)
OPENSSL_cleanse(c->cipher_data, c->cipher->ctx_size);
if (ctx->cipher_data && ctx->cipher->ctx_size)
OPENSSL_cleanse(ctx->cipher_data, ctx->cipher->ctx_size);
}
OPENSSL_free(c->cipher_data);
OPENSSL_free(ctx->cipher_data);
#ifndef OPENSSL_NO_ENGINE
ENGINE_finish(c->engine);
ENGINE_finish(ctx->engine);
#endif
memset(c, 0, sizeof(*c));
memset(ctx, 0, sizeof(*ctx));
return 1;
}
@@ -60,13 +81,30 @@ int EVP_CipherInit_ex(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher,
ENGINE *impl, const unsigned char *key,
const unsigned char *iv, int enc)
{
if (enc == -1)
EVP_CIPHER *provciph = NULL;
ENGINE *tmpimpl = NULL;
const EVP_CIPHER *tmpcipher;
/*
* enc == 1 means we are encrypting.
* enc == 0 means we are decrypting.
* enc == -1 means, use the previously initialised value for encrypt/decrypt
*/
if (enc == -1) {
enc = ctx->encrypt;
else {
} else {
if (enc)
enc = 1;
ctx->encrypt = enc;
}
if (cipher == NULL && ctx->cipher == NULL) {
EVPerr(EVP_F_EVP_CIPHERINIT_EX, EVP_R_NO_CIPHER_SET);
return 0;
}
/* TODO(3.0): Legacy work around code below. Remove this */
#ifndef OPENSSL_NO_ENGINE
/*
* Whether it's nice or not, "Inits" can be used on "Final"'d contexts so
@@ -77,11 +115,161 @@ int EVP_CipherInit_ex(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher,
if (ctx->engine && ctx->cipher
&& (cipher == NULL || cipher->nid == ctx->cipher->nid))
goto skip_to_init;
if (cipher != NULL && impl == NULL) {
/* Ask if an ENGINE is reserved for this job */
tmpimpl = ENGINE_get_cipher_engine(cipher->nid);
}
#endif
if (cipher) {
/*
* If there are engines involved then we should use legacy handling for now.
*/
if (ctx->engine != NULL
|| impl != NULL
|| tmpimpl != NULL) {
if (ctx->cipher == ctx->fetched_cipher)
ctx->cipher = NULL;
EVP_CIPHER_meth_free(ctx->fetched_cipher);
ctx->fetched_cipher = NULL;
goto legacy;
}
tmpcipher = (cipher == NULL) ? ctx->cipher : cipher;
if (tmpcipher->prov == NULL) {
switch(tmpcipher->nid) {
case NID_aes_256_ecb:
case NID_aes_192_ecb:
case NID_aes_128_ecb:
case NID_aes_256_cbc:
case NID_aes_192_cbc:
case NID_aes_128_cbc:
case NID_aes_256_ofb128:
case NID_aes_192_ofb128:
case NID_aes_128_ofb128:
case NID_aes_256_cfb128:
case NID_aes_192_cfb128:
case NID_aes_128_cfb128:
case NID_aes_256_cfb1:
case NID_aes_192_cfb1:
case NID_aes_128_cfb1:
case NID_aes_256_cfb8:
case NID_aes_192_cfb8:
case NID_aes_128_cfb8:
case NID_aes_256_ctr:
case NID_aes_192_ctr:
case NID_aes_128_ctr:
break;
default:
goto legacy;
}
}
/*
* Ensure a context left lying around from last time is cleared
* (legacy code)
*/
if (cipher != NULL && ctx->cipher != NULL) {
OPENSSL_clear_free(ctx->cipher_data, ctx->cipher->ctx_size);
ctx->cipher_data = NULL;
}
/* TODO(3.0): Start of non-legacy code below */
/* Ensure a context left lying around from last time is cleared */
if (cipher != NULL && ctx->cipher != NULL) {
unsigned long flags = ctx->flags;
EVP_CIPHER_CTX_reset(ctx);
/* Restore encrypt and flags */
ctx->encrypt = enc;
ctx->flags = flags;
}
if (cipher != NULL)
ctx->cipher = cipher;
else
cipher = ctx->cipher;
if (cipher->prov == NULL) {
provciph = EVP_CIPHER_fetch(NULL, OBJ_nid2sn(cipher->nid), "");
if (provciph == NULL) {
EVPerr(EVP_F_EVP_CIPHERINIT_EX, EVP_R_INITIALIZATION_ERROR);
return 0;
}
cipher = provciph;
EVP_CIPHER_meth_free(ctx->fetched_cipher);
ctx->fetched_cipher = provciph;
}
ctx->cipher = cipher;
if (ctx->provctx == NULL) {
ctx->provctx = ctx->cipher->newctx();
if (ctx->provctx == NULL) {
EVPerr(EVP_F_EVP_CIPHERINIT_EX, EVP_R_INITIALIZATION_ERROR);
return 0;
}
}
if ((ctx->flags & EVP_CIPH_NO_PADDING) != 0) {
/*
* Ensure a context left lying around from last time is cleared (the
* previous check attempted to avoid this if the same ENGINE and
* If this ctx was already set up for no padding then we need to tell
* the new cipher about it.
*/
if (!EVP_CIPHER_CTX_set_padding(ctx, 0))
return 0;
}
switch (EVP_CIPHER_mode(ctx->cipher)) {
case EVP_CIPH_CFB_MODE:
case EVP_CIPH_OFB_MODE:
case EVP_CIPH_CBC_MODE:
/* For these modes we remember the original IV for later use */
if (!ossl_assert(EVP_CIPHER_CTX_iv_length(ctx) <= (int)sizeof(ctx->oiv))) {
EVPerr(EVP_F_EVP_CIPHERINIT_EX, EVP_R_INITIALIZATION_ERROR);
return 0;
}
if (iv != NULL)
memcpy(ctx->oiv, iv, EVP_CIPHER_CTX_iv_length(ctx));
}
if (enc) {
if (ctx->cipher->einit == NULL) {
EVPerr(EVP_F_EVP_CIPHERINIT_EX, EVP_R_INITIALIZATION_ERROR);
return 0;
}
return ctx->cipher->einit(ctx->provctx,
key,
key == NULL ? 0
: EVP_CIPHER_CTX_key_length(ctx),
iv,
iv == NULL ? 0
: EVP_CIPHER_CTX_iv_length(ctx));
}
if (ctx->cipher->dinit == NULL) {
EVPerr(EVP_F_EVP_CIPHERINIT_EX, EVP_R_INITIALIZATION_ERROR);
return 0;
}
return ctx->cipher->dinit(ctx->provctx,
key,
key == NULL ? 0
: EVP_CIPHER_CTX_key_length(ctx),
iv,
iv == NULL ? 0
: EVP_CIPHER_CTX_iv_length(ctx));
/* TODO(3.0): Remove legacy code below */
legacy:
if (cipher != NULL) {
/*
* Ensure a context left lying around from last time is cleared (we
* previously attempted to avoid this if the same ENGINE and
* EVP_CIPHER could be used).
*/
if (ctx->cipher) {
@@ -92,18 +280,19 @@ int EVP_CipherInit_ex(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher,
ctx->flags = flags;
}
#ifndef OPENSSL_NO_ENGINE
if (impl) {
if (impl != NULL) {
if (!ENGINE_init(impl)) {
EVPerr(EVP_F_EVP_CIPHERINIT_EX, EVP_R_INITIALIZATION_ERROR);
return 0;
}
} else
/* Ask if an ENGINE is reserved for this job */
impl = ENGINE_get_cipher_engine(cipher->nid);
if (impl) {
} else {
impl = tmpimpl;
}
if (impl != NULL) {
/* There's an ENGINE for this job ... (apparently) */
const EVP_CIPHER *c = ENGINE_get_cipher(impl, cipher->nid);
if (!c) {
if (c == NULL) {
/*
* One positive side-effect of US's export control history,
* is that we should at least be able to avoid using US
@@ -119,8 +308,9 @@ int EVP_CipherInit_ex(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher,
* from an ENGINE and we need to release it when done.
*/
ctx->engine = impl;
} else
} else {
ctx->engine = NULL;
}
#endif
ctx->cipher = cipher;
@@ -144,9 +334,6 @@ int EVP_CipherInit_ex(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher,
return 0;
}
}
} else if (!ctx->cipher) {
EVPerr(EVP_F_EVP_CIPHERINIT_EX, EVP_R_NO_CIPHER_SET);
return 0;
}
#ifndef OPENSSL_NO_ENGINE
skip_to_init:
@@ -377,12 +564,39 @@ static int evp_EncryptDecryptUpdate(EVP_CIPHER_CTX *ctx,
int EVP_EncryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
const unsigned char *in, int inl)
{
int ret;
size_t soutl;
int blocksize;
/* Prevent accidental use of decryption context when encrypting */
if (!ctx->encrypt) {
EVPerr(EVP_F_EVP_ENCRYPTUPDATE, EVP_R_INVALID_OPERATION);
return 0;
}
if (ctx->cipher == NULL || ctx->cipher->prov == NULL)
goto legacy;
blocksize = EVP_CIPHER_CTX_block_size(ctx);
if (ctx->cipher->cupdate == NULL || blocksize < 1) {
EVPerr(EVP_F_EVP_ENCRYPTUPDATE, EVP_R_UPDATE_ERROR);
return 0;
}
ret = ctx->cipher->cupdate(ctx->provctx, out, &soutl,
inl + (blocksize == 1 ? 0 : blocksize), in,
(size_t)inl);
if (soutl > INT_MAX) {
EVPerr(EVP_F_EVP_ENCRYPTUPDATE, EVP_R_UPDATE_ERROR);
return 0;
}
*outl = soutl;
return ret;
/* TODO(3.0): Remove legacy code below */
legacy:
return evp_EncryptDecryptUpdate(ctx, out, outl, in, inl);
}
@@ -397,6 +611,8 @@ int EVP_EncryptFinal_ex(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl)
{
int n, ret;
unsigned int i, b, bl;
size_t soutl;
int blocksize;
/* Prevent accidental use of decryption context when encrypting */
if (!ctx->encrypt) {
@@ -404,6 +620,30 @@ int EVP_EncryptFinal_ex(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl)
return 0;
}
if (ctx->cipher == NULL || ctx->cipher->prov == NULL)
goto legacy;
blocksize = EVP_CIPHER_CTX_block_size(ctx);
if (blocksize < 1 || ctx->cipher->cfinal == NULL) {
EVPerr(EVP_F_EVP_ENCRYPTFINAL_EX, EVP_R_FINAL_ERROR);
return 0;
}
ret = ctx->cipher->cfinal(ctx->provctx, out, &soutl,
blocksize == 1 ? 0 : blocksize);
if (soutl > INT_MAX) {
EVPerr(EVP_F_EVP_ENCRYPTFINAL_EX, EVP_R_FINAL_ERROR);
return 0;
}
*outl = soutl;
return ret;
/* TODO(3.0): Remove legacy code below */
legacy:
if (ctx->cipher->flags & EVP_CIPH_FLAG_CUSTOM_CIPHER) {
ret = ctx->cipher->do_cipher(ctx, out, NULL, 0);
if (ret < 0)
@@ -444,8 +684,10 @@ int EVP_EncryptFinal_ex(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl)
int EVP_DecryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
const unsigned char *in, int inl)
{
int fix_len, cmpl = inl;
int fix_len, cmpl = inl, ret;
unsigned int b;
size_t soutl;
int blocksize;
/* Prevent accidental use of encryption context when decrypting */
if (ctx->encrypt) {
@@ -453,6 +695,32 @@ int EVP_DecryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
return 0;
}
if (ctx->cipher == NULL || ctx->cipher->prov == NULL)
goto legacy;
blocksize = EVP_CIPHER_CTX_block_size(ctx);
if (ctx->cipher->cupdate == NULL || blocksize < 1) {
EVPerr(EVP_F_EVP_DECRYPTUPDATE, EVP_R_UPDATE_ERROR);
return 0;
}
ret = ctx->cipher->cupdate(ctx->provctx, out, &soutl,
inl + (blocksize == 1 ? 0 : blocksize), in,
(size_t)inl);
if (ret) {
if (soutl > INT_MAX) {
EVPerr(EVP_F_EVP_DECRYPTUPDATE, EVP_R_UPDATE_ERROR);
return 0;
}
*outl = soutl;
}
return ret;
/* TODO(3.0): Remove legacy code below */
legacy:
b = ctx->cipher->block_size;
if (EVP_CIPHER_CTX_test_flags(ctx, EVP_CIPH_FLAG_LENGTH_BITS))
@@ -527,6 +795,9 @@ int EVP_DecryptFinal_ex(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl)
{
int i, n;
unsigned int b;
size_t soutl;
int ret;
int blocksize;
/* Prevent accidental use of encryption context when decrypting */
if (ctx->encrypt) {
@@ -534,6 +805,32 @@ int EVP_DecryptFinal_ex(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl)
return 0;
}
if (ctx->cipher == NULL || ctx->cipher->prov == NULL)
goto legacy;
blocksize = EVP_CIPHER_CTX_block_size(ctx);
if (blocksize < 1 || ctx->cipher->cfinal == NULL) {
EVPerr(EVP_F_EVP_DECRYPTFINAL_EX, EVP_R_FINAL_ERROR);
return 0;
}
ret = ctx->cipher->cfinal(ctx->provctx, out, &soutl,
blocksize == 1 ? 0 : blocksize);
if (ret) {
if (soutl > INT_MAX) {
EVPerr(EVP_F_EVP_DECRYPTFINAL_EX, EVP_R_FINAL_ERROR);
return 0;
}
*outl = soutl;
}
return ret;
/* TODO(3.0): Remove legacy code below */
legacy:
*outl = 0;
if (ctx->cipher->flags & EVP_CIPH_FLAG_CUSTOM_CIPHER) {
@@ -590,7 +887,7 @@ int EVP_CIPHER_CTX_set_key_length(EVP_CIPHER_CTX *c, int keylen)
{
if (c->cipher->flags & EVP_CIPH_CUSTOM_KEY_LENGTH)
return EVP_CIPHER_CTX_ctrl(c, EVP_CTRL_SET_KEY_LENGTH, keylen, NULL);
if (c->key_len == keylen)
if (EVP_CIPHER_CTX_key_length(c) == keylen)
return 1;
if ((keylen > 0) && (c->cipher->flags & EVP_CIPH_VARIABLE_LENGTH)) {
c->key_len = keylen;
@@ -606,6 +903,24 @@ int EVP_CIPHER_CTX_set_padding(EVP_CIPHER_CTX *ctx, int pad)
ctx->flags &= ~EVP_CIPH_NO_PADDING;
else
ctx->flags |= EVP_CIPH_NO_PADDING;
if (ctx->cipher != NULL && ctx->cipher->prov != NULL) {
OSSL_PARAM params[] = {
OSSL_PARAM_int(OSSL_CIPHER_PARAM_PADDING, NULL),
OSSL_PARAM_END
};
params[0].data = &pad;
if (ctx->cipher->ctx_set_params == NULL) {
EVPerr(EVP_F_EVP_CIPHER_CTX_SET_PADDING, EVP_R_CTRL_NOT_IMPLEMENTED);
return 0;
}
if (!ctx->cipher->ctx_set_params(ctx->provctx, params))
return 0;
}
return 1;
}
@@ -636,7 +951,7 @@ int EVP_CIPHER_CTX_rand_key(EVP_CIPHER_CTX *ctx, unsigned char *key)
{
if (ctx->cipher->flags & EVP_CIPH_RAND_KEY)
return EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_RAND_KEY, 0, key);
if (RAND_priv_bytes(key, ctx->key_len) <= 0)
if (RAND_priv_bytes(key, EVP_CIPHER_CTX_key_length(ctx)) <= 0)
return 0;
return 1;
}
@@ -647,6 +962,36 @@ int EVP_CIPHER_CTX_copy(EVP_CIPHER_CTX *out, const EVP_CIPHER_CTX *in)
EVPerr(EVP_F_EVP_CIPHER_CTX_COPY, EVP_R_INPUT_NOT_INITIALIZED);
return 0;
}
if (in->cipher->prov == NULL)
goto legacy;
if (in->cipher->dupctx == NULL) {
EVPerr(EVP_F_EVP_CIPHER_CTX_COPY, EVP_R_NOT_ABLE_TO_COPY_CTX);
return 0;
}
EVP_CIPHER_CTX_reset(out);
*out = *in;
out->provctx = NULL;
if (in->fetched_cipher != NULL && !EVP_CIPHER_upref(in->fetched_cipher)) {
out->fetched_cipher = NULL;
return 0;
}
out->provctx = in->cipher->dupctx(in->provctx);
if (out->provctx == NULL) {
EVPerr(EVP_F_EVP_CIPHER_CTX_COPY, EVP_R_NOT_ABLE_TO_COPY_CTX);
return 0;
}
return 1;
/* TODO(3.0): Remove legacy code below */
legacy:
#ifndef OPENSSL_NO_ENGINE
/* Make sure it's safe to copy a cipher context using an ENGINE */
if (in->engine && !ENGINE_init(in->engine)) {
@@ -676,3 +1021,141 @@ int EVP_CIPHER_CTX_copy(EVP_CIPHER_CTX *out, const EVP_CIPHER_CTX *in)
}
return 1;
}
static void *evp_cipher_from_dispatch(int nid, const OSSL_DISPATCH *fns,
OSSL_PROVIDER *prov)
{
EVP_CIPHER *cipher = NULL;
int fnciphcnt = 0, fnctxcnt = 0;
if ((cipher = EVP_CIPHER_meth_new(nid, 0, 0)) == NULL)
return NULL;
for (; fns->function_id != 0; fns++) {
switch (fns->function_id) {
case OSSL_FUNC_CIPHER_NEWCTX:
if (cipher->newctx != NULL)
break;
cipher->newctx = OSSL_get_OP_cipher_newctx(fns);
fnctxcnt++;
break;
case OSSL_FUNC_CIPHER_ENCRYPT_INIT:
if (cipher->einit != NULL)
break;
cipher->einit = OSSL_get_OP_cipher_encrypt_init(fns);
fnciphcnt++;
break;
case OSSL_FUNC_CIPHER_DECRYPT_INIT:
if (cipher->dinit != NULL)
break;
cipher->dinit = OSSL_get_OP_cipher_decrypt_init(fns);
fnciphcnt++;
break;
case OSSL_FUNC_CIPHER_UPDATE:
if (cipher->cupdate != NULL)
break;
cipher->cupdate = OSSL_get_OP_cipher_update(fns);
fnciphcnt++;
break;
case OSSL_FUNC_CIPHER_FINAL:
if (cipher->cfinal != NULL)
break;
cipher->cfinal = OSSL_get_OP_cipher_final(fns);
fnciphcnt++;
break;
case OSSL_FUNC_CIPHER_CIPHER:
if (cipher->ccipher != NULL)
break;
cipher->ccipher = OSSL_get_OP_cipher_cipher(fns);
break;
case OSSL_FUNC_CIPHER_FREECTX:
if (cipher->freectx != NULL)
break;
cipher->freectx = OSSL_get_OP_cipher_freectx(fns);
fnctxcnt++;
break;
case OSSL_FUNC_CIPHER_DUPCTX:
if (cipher->dupctx != NULL)
break;
cipher->dupctx = OSSL_get_OP_cipher_dupctx(fns);
break;
case OSSL_FUNC_CIPHER_KEY_LENGTH:
if (cipher->key_length != NULL)
break;
cipher->key_length = OSSL_get_OP_cipher_key_length(fns);
break;
case OSSL_FUNC_CIPHER_IV_LENGTH:
if (cipher->iv_length != NULL)
break;
cipher->iv_length = OSSL_get_OP_cipher_iv_length(fns);
break;
case OSSL_FUNC_CIPHER_BLOCK_SIZE:
if (cipher->blocksize != NULL)
break;
cipher->blocksize = OSSL_get_OP_cipher_block_size(fns);
break;
case OSSL_FUNC_CIPHER_GET_PARAMS:
if (cipher->get_params != NULL)
break;
cipher->get_params = OSSL_get_OP_cipher_get_params(fns);
break;
case OSSL_FUNC_CIPHER_CTX_GET_PARAMS:
if (cipher->ctx_get_params != NULL)
break;
cipher->ctx_get_params = OSSL_get_OP_cipher_ctx_get_params(fns);
break;
case OSSL_FUNC_CIPHER_CTX_SET_PARAMS:
if (cipher->ctx_set_params != NULL)
break;
cipher->ctx_set_params = OSSL_get_OP_cipher_ctx_set_params(fns);
break;
}
}
if ((fnciphcnt != 0 && fnciphcnt != 3 && fnciphcnt != 4)
|| (fnciphcnt == 0 && cipher->ccipher == NULL)
|| fnctxcnt != 2
|| cipher->blocksize == NULL
|| cipher->iv_length == NULL
|| cipher->key_length == NULL) {
/*
* In order to be a consistent set of functions we must have at least
* a complete set of "encrypt" functions, or a complete set of "decrypt"
* functions, or a single "cipher" function. In all cases we need a
* complete set of context management functions, as well as the
* blocksize, iv_length and key_length functions.
*/
EVP_CIPHER_meth_free(cipher);
EVPerr(EVP_F_EVP_CIPHER_FROM_DISPATCH, EVP_R_INVALID_PROVIDER_FUNCTIONS);
return NULL;
}
cipher->prov = prov;
if (prov != NULL)
ossl_provider_upref(prov);
return cipher;
}
static int evp_cipher_upref(void *cipher)
{
return EVP_CIPHER_upref(cipher);
}
static void evp_cipher_free(void *cipher)
{
EVP_CIPHER_meth_free(cipher);
}
static int evp_cipher_nid(void *vcipher)
{
EVP_CIPHER *cipher = vcipher;
return cipher->nid;
}
EVP_CIPHER *EVP_CIPHER_fetch(OPENSSL_CTX *ctx, const char *algorithm,
const char *properties)
{
return evp_generic_fetch(ctx, OSSL_OP_CIPHER, algorithm, properties,
evp_cipher_from_dispatch, evp_cipher_upref,
evp_cipher_free, evp_cipher_nid);
}
crypto/evp/evp_err.c
+7
View File
@@ -53,6 +53,11 @@ static const ERR_STRING_DATA EVP_str_functs[] = {
"EVP_CIPHER_CTX_ctrl"},
{ERR_PACK(ERR_LIB_EVP, EVP_F_EVP_CIPHER_CTX_SET_KEY_LENGTH, 0),
"EVP_CIPHER_CTX_set_key_length"},
{ERR_PACK(ERR_LIB_EVP, EVP_F_EVP_CIPHER_CTX_SET_PADDING, 0),
"EVP_CIPHER_CTX_set_padding"},
{ERR_PACK(ERR_LIB_EVP, EVP_F_EVP_CIPHER_FROM_DISPATCH, 0),
"evp_cipher_from_dispatch"},
{ERR_PACK(ERR_LIB_EVP, EVP_F_EVP_CIPHER_MODE, 0), "EVP_CIPHER_mode"},
{ERR_PACK(ERR_LIB_EVP, EVP_F_EVP_CIPHER_PARAM_TO_ASN1, 0),
"EVP_CIPHER_param_to_asn1"},
{ERR_PACK(ERR_LIB_EVP, EVP_F_EVP_DECRYPTFINAL_EX, 0),
@@ -246,6 +251,8 @@ static const ERR_STRING_DATA EVP_str_reasons[] = {
{ERR_PACK(ERR_LIB_EVP, 0, EVP_R_INVALID_KEY), "invalid key"},
{ERR_PACK(ERR_LIB_EVP, 0, EVP_R_INVALID_KEY_LENGTH), "invalid key length"},
{ERR_PACK(ERR_LIB_EVP, 0, EVP_R_INVALID_OPERATION), "invalid operation"},
{ERR_PACK(ERR_LIB_EVP, 0, EVP_R_INVALID_PROVIDER_FUNCTIONS),
"invalid provider functions"},
{ERR_PACK(ERR_LIB_EVP, 0, EVP_R_INVALID_SALT_LENGTH),
"invalid salt length"},
{ERR_PACK(ERR_LIB_EVP, 0, EVP_R_KEYGEN_FAILURE), "keygen failure"},
crypto/evp/evp_fetch.c
+8 -2
View File
@@ -173,11 +173,15 @@ void *evp_generic_fetch(OPENSSL_CTX *libctx, int operation_id,
void (*free_method)(void *),
int (*nid_method)(void *))
{
OSSL_METHOD_STORE *store = get_default_method_store(libctx);
int nid = OBJ_sn2nid(algorithm);
void *method = NULL;
if (store == NULL)
return NULL;
if (nid == NID_undef
|| !ossl_method_store_cache_get(NULL, nid, properties, &method)) {
|| !ossl_method_store_cache_get(store, nid, properties, &method)) {
OSSL_METHOD_CONSTRUCT_METHOD mcm = {
alloc_tmp_method_store,
dealloc_tmp_method_store,
@@ -198,7 +202,9 @@ void *evp_generic_fetch(OPENSSL_CTX *libctx, int operation_id,
method = ossl_method_construct(libctx, operation_id, algorithm,
properties, 0 /* !force_cache */,
&mcm, &mcmdata);
ossl_method_store_cache_set(NULL, nid, properties, method);
ossl_method_store_cache_set(store, nid, properties, method);
} else {
upref_method(method);
}
return method;
crypto/evp/evp_lib.c
+113 -20
View File
@@ -11,6 +11,8 @@
#include "internal/cryptlib.h"
#include <openssl/evp.h>
#include <openssl/objects.h>
#include <openssl/params.h>
#include <openssl/core_names.h>
#include "internal/evp_int.h"
#include "internal/provider.h"
#include "evp_locl.h"
@@ -18,13 +20,28 @@
int EVP_CIPHER_param_to_asn1(EVP_CIPHER_CTX *c, ASN1_TYPE *type)
{
int ret;
const EVP_CIPHER *cipher = c->cipher;
if (c->cipher->set_asn1_parameters != NULL)
ret = c->cipher->set_asn1_parameters(c, type);
else if (c->cipher->flags & EVP_CIPH_FLAG_DEFAULT_ASN1) {
switch (EVP_CIPHER_CTX_mode(c)) {
if (cipher->prov != NULL) {
/*
* The cipher has come from a provider and won't have the default flags.
* Find the implicit form so we can check the flags.
* TODO(3.0): This won't work for 3rd party ciphers we know nothing about
* We'll need to think of something else for those.
*/
cipher = EVP_get_cipherbynid(cipher->nid);
if (cipher == NULL) {
EVPerr(EVP_F_EVP_CIPHER_PARAM_TO_ASN1, ASN1_R_UNSUPPORTED_CIPHER);
return -1;
}
}
if (cipher->set_asn1_parameters != NULL)
ret = cipher->set_asn1_parameters(c, type);
else if (cipher->flags & EVP_CIPH_FLAG_DEFAULT_ASN1) {
switch (EVP_CIPHER_mode(cipher)) {
case EVP_CIPH_WRAP_MODE:
if (EVP_CIPHER_CTX_nid(c) == NID_id_smime_alg_CMS3DESwrap)
if (EVP_CIPHER_nid(cipher) == NID_id_smime_alg_CMS3DESwrap)
ASN1_TYPE_set(type, V_ASN1_NULL, NULL);
ret = 1;
break;
@@ -53,11 +70,22 @@ int EVP_CIPHER_param_to_asn1(EVP_CIPHER_CTX *c, ASN1_TYPE *type)
int EVP_CIPHER_asn1_to_param(EVP_CIPHER_CTX *c, ASN1_TYPE *type)
{
int ret;
const EVP_CIPHER *cipher = c->cipher;
if (c->cipher->get_asn1_parameters != NULL)
ret = c->cipher->get_asn1_parameters(c, type);
else if (c->cipher->flags & EVP_CIPH_FLAG_DEFAULT_ASN1) {
switch (EVP_CIPHER_CTX_mode(c)) {
if (cipher->prov != NULL) {
/*
* The cipher has come from a provider and won't have the default flags.
* Find the implicit form so we can check the flags.
*/
cipher = EVP_get_cipherbynid(cipher->nid);
if (cipher == NULL)
return -1;
}
if (cipher->get_asn1_parameters != NULL)
ret = cipher->get_asn1_parameters(c, type);
else if (cipher->flags & EVP_CIPH_FLAG_DEFAULT_ASN1) {
switch (EVP_CIPHER_mode(cipher)) {
case EVP_CIPH_WRAP_MODE:
ret = 1;
@@ -85,19 +113,23 @@ int EVP_CIPHER_asn1_to_param(EVP_CIPHER_CTX *c, ASN1_TYPE *type)
return ret;
}
int EVP_CIPHER_get_asn1_iv(EVP_CIPHER_CTX *c, ASN1_TYPE *type)
int EVP_CIPHER_get_asn1_iv(EVP_CIPHER_CTX *ctx, ASN1_TYPE *type)
{
int i = 0;
unsigned int l;
if (type != NULL) {
l = EVP_CIPHER_CTX_iv_length(c);
OPENSSL_assert(l <= sizeof(c->iv));
i = ASN1_TYPE_get_octetstring(type, c->oiv, l);
unsigned char iv[EVP_MAX_IV_LENGTH];
l = EVP_CIPHER_CTX_iv_length(ctx);
if (!ossl_assert(l <= sizeof(iv)))
return -1;
i = ASN1_TYPE_get_octetstring(type, iv, l);
if (i != (int)l)
return -1;
else if (i > 0)
memcpy(c->iv, c->oiv, l);
if (!EVP_CipherInit_ex(ctx, NULL, NULL, NULL, iv, -1))
return -1;
}
return i;
}
@@ -175,14 +207,20 @@ int EVP_CIPHER_type(const EVP_CIPHER *ctx)
}
}
int EVP_CIPHER_block_size(const EVP_CIPHER *e)
int EVP_CIPHER_block_size(const EVP_CIPHER *cipher)
{
return e->block_size;
if (cipher->prov != NULL) {
if (cipher->blocksize != NULL)
return cipher->blocksize();
/* We default to a block size of 1 */
return 1;
}
return cipher->block_size;
}
int EVP_CIPHER_CTX_block_size(const EVP_CIPHER_CTX *ctx)
{
return ctx->cipher->block_size;
return EVP_CIPHER_block_size(ctx->cipher);
}
int EVP_CIPHER_impl_ctx_size(const EVP_CIPHER *e)
@@ -193,6 +231,12 @@ int EVP_CIPHER_impl_ctx_size(const EVP_CIPHER *e)
int EVP_Cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
const unsigned char *in, unsigned int inl)
{
if (ctx->cipher->prov != NULL) {
if (ctx->cipher->ccipher != NULL)
return ctx->cipher->ccipher(ctx->provctx, out, in, (size_t)inl);
return 0;
}
return ctx->cipher->do_cipher(ctx, out, in, inl);
}
@@ -238,12 +282,18 @@ void *EVP_CIPHER_CTX_set_cipher_data(EVP_CIPHER_CTX *ctx, void *cipher_data)
int EVP_CIPHER_iv_length(const EVP_CIPHER *cipher)
{
if (cipher->prov != NULL) {
if (cipher->iv_length != NULL)
return (int)cipher->iv_length();
return 0;
}
return cipher->iv_len;
}
int EVP_CIPHER_CTX_iv_length(const EVP_CIPHER_CTX *ctx)
{
return ctx->cipher->iv_len;
return EVP_CIPHER_iv_length(ctx->cipher);
}
const unsigned char *EVP_CIPHER_CTX_original_iv(const EVP_CIPHER_CTX *ctx)
@@ -278,11 +328,23 @@ void EVP_CIPHER_CTX_set_num(EVP_CIPHER_CTX *ctx, int num)
int EVP_CIPHER_key_length(const EVP_CIPHER *cipher)
{
if (cipher->prov != NULL) {
if (cipher->key_length != NULL)
return (int)cipher->key_length();
return -1;
}
return cipher->key_len;
}
int EVP_CIPHER_CTX_key_length(const EVP_CIPHER_CTX *ctx)
{
/*
* TODO(3.0): This may need to change if/when we introduce variable length
* key ciphers into the providers.
*/
if (ctx->cipher != NULL && ctx->cipher->prov != NULL)
return EVP_CIPHER_key_length(ctx->cipher);
return ctx->key_len;
}
@@ -296,6 +358,33 @@ int EVP_CIPHER_CTX_nid(const EVP_CIPHER_CTX *ctx)
return ctx->cipher->nid;
}
int EVP_CIPHER_mode(const EVP_CIPHER *cipher)
{
if (cipher->prov != NULL) {
int mode;
/* Cipher comes from a provider - so ask the provider for the mode */
OSSL_PARAM params[] = {
OSSL_PARAM_int(OSSL_CIPHER_PARAM_MODE, NULL),
OSSL_PARAM_END
};
params[0].data = &mode;
if (cipher->get_params == NULL) {
EVPerr(EVP_F_EVP_CIPHER_MODE, EVP_R_CTRL_NOT_IMPLEMENTED);
return 0;
}
if (!cipher->get_params(params))
return 0;
return mode;
}
return EVP_CIPHER_flags(cipher) & EVP_CIPH_MODE;
}
int EVP_MD_block_size(const EVP_MD *md)
{
if (md == NULL) {
@@ -353,12 +442,16 @@ EVP_MD *EVP_MD_meth_new(int md_type, int pkey_type)
}
return md;
}
EVP_MD *EVP_MD_meth_dup(const EVP_MD *md)
{
EVP_MD *to = EVP_MD_meth_new(md->type, md->pkey_type);
if (to != NULL)
if (to != NULL) {
CRYPTO_RWLOCK *lock = to->lock;
memcpy(to, md, sizeof(*to));
to->lock = lock;
}
return to;
}
crypto/evp/evp_locl.h
+4
View File
@@ -44,6 +44,10 @@ struct evp_cipher_ctx_st {
int final_used;
int block_mask;
unsigned char final[EVP_MAX_BLOCK_LENGTH]; /* possible final block */
/* Provider ctx */
void *provctx;
EVP_CIPHER *fetched_cipher;
} /* EVP_CIPHER_CTX */ ;
struct evp_mac_ctx_st {
crypto/include/internal/evp_int.h
+24
View File
@@ -210,10 +210,14 @@ struct evp_md_st {
struct evp_cipher_st {
int nid;
int block_size;
/* Default value for variable length ciphers */
int key_len;
int iv_len;
/* Legacy structure members */
/* TODO(3.0): Remove these */
/* Various flags */
unsigned long flags;
/* init key */
@@ -234,6 +238,26 @@ struct evp_cipher_st {
int (*ctrl) (EVP_CIPHER_CTX *, int type, int arg, void *ptr);
/* Application data */
void *app_data;
/* New structure members */
/* TODO(3.0): Remove above comment when legacy has gone */
OSSL_PROVIDER *prov;
CRYPTO_REF_COUNT refcnt;
CRYPTO_RWLOCK *lock;
OSSL_OP_cipher_newctx_fn *newctx;
OSSL_OP_cipher_encrypt_init_fn *einit;
OSSL_OP_cipher_decrypt_init_fn *dinit;
OSSL_OP_cipher_update_fn *cupdate;
OSSL_OP_cipher_final_fn *cfinal;
OSSL_OP_cipher_cipher_fn *ccipher;
OSSL_OP_cipher_freectx_fn *freectx;
OSSL_OP_cipher_dupctx_fn *dupctx;
OSSL_OP_cipher_key_length_fn *key_length;
OSSL_OP_cipher_iv_length_fn *iv_length;
OSSL_OP_cipher_block_size_fn *blocksize;
OSSL_OP_cipher_get_params_fn *get_params;
OSSL_OP_cipher_ctx_get_params_fn *ctx_get_params;
OSSL_OP_cipher_ctx_set_params_fn *ctx_set_params;
} /* EVP_CIPHER */ ;
/* Macros to code block cipher wrappers */
crypto/info.c
+44
View File
@@ -0,0 +1,44 @@
/*
* Copyright 2019 The OpenSSL Project Authors. All Rights Reserved.
*
* Licensed under the Apache License 2.0 (the "License"). You may not use
* this file except in compliance with the License. You can obtain a copy
* in the file LICENSE in the source distribution or at
* https://www.openssl.org/source/license.html
*/
#include <stddef.h>
#include <openssl/crypto.h>
#include "internal/dso_conf.h"
#include "e_os.h"
const char *OPENSSL_info(int t)
{
switch (t) {
case OPENSSL_INFO_CONFIG_DIR:
return OPENSSLDIR;
case OPENSSL_INFO_ENGINES_DIR:
return ENGINESDIR;
case OPENSSL_INFO_MODULES_DIR:
return MODULESDIR;
case OPENSSL_INFO_DSO_EXTENSION:
return DSO_EXTENSION;
case OPENSSL_INFO_DIR_FILENAME_SEPARATOR:
#if defined(_WIN32)
return "\\";
#elif defined(__VMS)
return "";
#else /* Assume POSIX */
return "/";
#endif
case OPENSSL_INFO_LIST_SEPARATOR:
{
static const char list_sep[] = { LIST_SEPARATOR_CHAR, '\0' };
return list_sep;
}
default:
break;
}
/* Not an error */
return NULL;
}
crypto/lhash/lhash.c
+1
View File
@@ -98,6 +98,7 @@ void OPENSSL_LH_flush(OPENSSL_LHASH *lh)
OPENSSL_free(n);
n = nn;
}
lh->b[i] = NULL;
}
}
crypto/modes/asm/ghashv8-armx.pl
+1
View File
@@ -42,6 +42,7 @@
# Denver 0.51 0.65 6.02
# Mongoose 0.65 1.10 8.06
# Kryo 0.76 1.16 8.00
# ThunderX2 1.05
#
# (*) presented for reference/comparison purposes;
crypto/modes/ccm128.c
+1 -1
View File
@@ -425,7 +425,7 @@ size_t CRYPTO_ccm128_tag(CCM128_CONTEXT *ctx, unsigned char *tag, size_t len)
M *= 2;
M += 2;
if (len < M)
if (len != M)
return 0;
memcpy(tag, ctx->cmac.c, M);
return M;
crypto/perlasm/s390x.pm
+88 -6
View File
@@ -6,23 +6,37 @@
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html
# Copyright IBM Corp. 2018
# Copyright IBM Corp. 2018-2019
# Author: Patrick Steuer <patrick.steuer@de.ibm.com>
package perlasm::s390x;
use strict;
use warnings;
use bigint;
use Carp qw(confess);
use Exporter qw(import);
our @EXPORT=qw(PERLASM_BEGIN PERLASM_END);
our @EXPORT_OK=qw(AUTOLOAD LABEL INCLUDE stfle);
our %EXPORT_TAGS=(
# long-displacement facility
LD => [qw(clgfi)],
# general-instruction-extension facility
GE => [qw(risbg)],
# extended-immediate facility
EI => [qw(lt)],
# miscellaneous-instruction-extensions facility 1
MI1 => [qw(risbgn)],
# message-security assist
MSA => [qw(kmac km kmc kimd klmd)],
# message-security-assist extension 4
MSA4 => [qw(kmf kmo pcc kmctr)],
# message-security-assist extension 5
MSA5 => [qw(ppno prno)],
# message-security-assist extension 8
MSA8 => [qw(kma)],
# vector facility
VX => [qw(vgef vgeg vgbm vzero vone vgm vgmb vgmh vgmf vgmg
vl vlr vlrep vlrepb vlreph vlrepf vlrepg vleb vleh vlef vleg vleib
vleih vleif vleig vlgv vlgvb vlgvh vlgvf vlgvg vllez vllezb vllezh
@@ -71,6 +85,7 @@ our %EXPORT_TAGS=(
wfmadb vfms vfmsdb wfmsdb vfpso vfpsodb wfpsodb vflcdb wflcdb
vflndb wflndb vflpdb wflpdb vfsq vfsqdb wfsqdb vfs vfsdb wfsdb
vftci vftcidb wftcidb)],
# vector-enhancements facility 1
VXE => [qw(vbperm vllezlf vmsl vmslg vnx vnn voc vpopctb vpopcth
vpopctf vpopctg vfasb wfasb wfaxb wfcsb wfcxb wfksb wfkxb vfcesb
vfcesbs wfcesb wfcesbs wfcexb wfcexbs vfchsb vfchsbs wfchsb wfchsbs
@@ -83,10 +98,11 @@ our %EXPORT_TAGS=(
wfnmsxb vfpsosb wfpsosb vflcsb wflcsb vflnsb wflnsb vflpsb wflpsb
vfpsoxb wfpsoxb vflcxb wflcxb vflnxb wflnxb vflpxb wflpxb vfsqsb
wfsqsb wfsqxb vfssb wfssb wfsxb vftcisb wftcisb wftcixb)],
# vector-packed-decimal facility
VXD => [qw(vlrlr vlrl vstrlr vstrl vap vcp vcvb vcvbg vcvd vcvdg vdp
vlip vmp vmsp vpkz vpsop vrp vsdp vsrp vsp vtp vupkz)],
);
Exporter::export_ok_tags(qw(MSA MSA4 MSA5 MSA8 VX VXE VXD));
Exporter::export_ok_tags(qw(LD GE EI MI1 MSA MSA4 MSA5 MSA8 VX VXE VXD));
our $AUTOLOAD;
@@ -143,6 +159,28 @@ sub stfle {
S(0xb2b0,@_);
}
# MISC
sub clgfi {
confess(err("ARGNUM")) if ($#_!=1);
RILa(0xc2e,@_);
}
sub lt {
confess(err("ARGNUM")) if ($#_!=1);
RXYa(0xe312,@_);
}
sub risbg {
confess(err("ARGNUM")) if ($#_<3||$#_>4);
RIEf(0xec55,@_);
}
sub risbgn {
confess(err("ARGNUM")) if ($#_<3||$#_>4);
RIEf(0xec59,@_);
}
# MSA
sub kmac {
@@ -250,7 +288,7 @@ sub vgmg {
}
sub vl {
confess(err("ARGNUM")) if ($#_!=1);
confess(err("ARGNUM")) if ($#_<1||$#_>2);
VRX(0xe706,@_);
}
@@ -345,7 +383,7 @@ sub vllezg {
}
sub vlm {
confess(err("ARGNUM")) if ($#_!=2);
confess(err("ARGNUM")) if ($#_<2||$#_>3);
VRSa(0xe736,@_);
}
@@ -548,7 +586,7 @@ sub vsegf {
}
sub vst {
confess(err("ARGNUM")) if ($#_!=1);
confess(err("ARGNUM")) if ($#_<1||$#_>2);
VRX(0xe70e,@_);
}
@@ -570,7 +608,7 @@ sub vsteg {
}
sub vstm {
confess(err("ARGNUM")) if ($#_!=2);
confess(err("ARGNUM")) if ($#_<2||$#_>3);
VRSa(0xe73e,@_);
}
@@ -2486,6 +2524,36 @@ sub vupkz {
# Instruction Formats
#
sub RIEf {
confess(err("ARGNUM")) if ($#_<4||5<$#_);
my $ops=join(',',@_[1..$#_]);
my $memn=(caller(1))[3];
$memn=~s/^.*:://;
my ($opcode,$r1,$r2,$i3,$i4,$i5)=(shift,get_R(shift),get_R(shift),
get_I(shift,8),get_I(shift,8),
get_I(shift,8));
$out.="\t.word\t";
$out.=sprintf("%#06x",(($opcode>>8)<<8|$r1<<4|$r2)).",";
$out.=sprintf("%#06x",($i3<<8)|$i4).",";
$out.=sprintf("%#06x",($i5<<8)|($opcode&0xff));
$out.="\t# $memn\t$ops\n"
}
sub RILa {
confess(err("ARGNUM")) if ($#_!=2);
my $ops=join(',',@_[1..$#_]);
my $memn=(caller(1))[3];
$memn=~s/^.*:://;
my ($opcode,$r1,$i2)=(shift,get_R(shift),get_I(shift,32));
$out.="\t.word\t";
$out.=sprintf("%#06x",(($opcode>>4)<<8|$r1<<4|($opcode&0xf))).",";
$out.=sprintf("%#06x",($i2>>16)).",";
$out.=sprintf("%#06x",($i2&0xffff));
$out.="\t# $memn\t$ops\n"
}
sub RRE {
confess(err("ARGNUM")) if ($#_<0||2<$#_);
my $ops=join(',',@_[1..$#_]);
@@ -2510,6 +2578,20 @@ sub RRFb {
$out.="\t# $memn\t$ops\n"
}
sub RXYa {
confess(err("ARGNUM")) if ($#_!=2);
my $ops=join(',',@_[1..$#_]);
my $memn=(caller(1))[3];
$memn=~s/^.*:://;
my ($opcode,$r1,$d2,$x2,$b2)=(shift,get_R(shift),get_DXB(shift));
$out.="\t.word\t";
$out.=sprintf("%#06x",(($opcode>>8)<<8|$r1<<4|$x2)).",";
$out.=sprintf("%#06x",($b2<<12|($d2&0xfff))).",";
$out.=sprintf("%#06x",(($d2>>12)<<8|$opcode&0xff));
$out.="\t# $memn\t$ops\n"
}
sub S {
confess(err("ARGNUM")) if ($#_<0||1<$#_);
my $ops=join(',',@_[1..$#_]);
crypto/poly1305/asm/poly1305-armv8.pl
+1
View File
@@ -29,6 +29,7 @@
# X-Gene 2.13/+68% 2.27
# Mongoose 1.77/+75% 1.12
# Kryo 2.70/+55% 1.13
# ThunderX2 1.17/+95% 1.36
#
# (*) estimate based on resources availability is less than 1.0,
# i.e. measured result is worse than expected, presumably binary
crypto/poly1305/asm/poly1305-s390x.pl
+1 -1
View File
@@ -45,7 +45,7 @@
use strict;
use FindBin qw($Bin);
use lib "$Bin/../..";
use perlasm::s390x qw(:DEFAULT :VX AUTOLOAD LABEL INCLUDE);
use perlasm::s390x qw(:DEFAULT :LD :GE :EI :MI1 :VX AUTOLOAD LABEL INCLUDE);
my $flavour = shift;
crypto/property/property.c
+1 -1
View File
@@ -450,7 +450,7 @@ int ossl_method_store_cache_get(OSSL_METHOD_STORE *store, int nid,
return 0;
}
elem.query = prop_query;
elem.query = prop_query != NULL ? prop_query : "";
r = lh_QUERY_retrieve(alg->cache, &elem);
if (r == NULL) {
ossl_property_unlock(store);
crypto/rand/rand_crng_test.c
+1 -1
View File
@@ -30,7 +30,7 @@ int rand_crngt_get_entropy_cb(unsigned char *buf)
while ((n = rand_pool_acquire_entropy(crngt_pool)) != 0)
if (n >= CRNGT_BUFSIZ) {
p = rand_pool_detach(crngt_pool);
memcpy(crngt_prev, p, CRNGT_BUFSIZ);
memcpy(buf, p, CRNGT_BUFSIZ);
rand_pool_reattach(crngt_pool, p);
return 1;
}
crypto/rsa/rsa_oaep.c
+27
View File
@@ -38,6 +38,13 @@ int RSA_padding_add_PKCS1_OAEP(unsigned char *to, int tlen,
param, plen, NULL, NULL);
}
/*
* Perform ihe padding as per NIST 800-56B 7.2.2.3
* from (K) is the key material.
* param (A) is the additional input.
* Step numbers are included here but not in the constant time inverse below
* to avoid complicating an already difficult enough function.
*/
int RSA_padding_add_PKCS1_OAEP_mgf1(unsigned char *to, int tlen,
const unsigned char *from, int flen,
const unsigned char *param, int plen,
@@ -57,6 +64,7 @@ int RSA_padding_add_PKCS1_OAEP_mgf1(unsigned char *to, int tlen,
mdlen = EVP_MD_size(md);
/* step 2b: check KLen > nLen - 2 HLen - 2 */
if (flen > emlen - 2 * mdlen - 1) {
RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP_MGF1,
RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE);
@@ -69,15 +77,20 @@ int RSA_padding_add_PKCS1_OAEP_mgf1(unsigned char *to, int tlen,
return 0;
}
/* step 3i: EM = 00000000 || maskedMGF || maskedDB */
to[0] = 0;
seed = to + 1;
db = to + mdlen + 1;
/* step 3a: hash the additional input */
if (!EVP_Digest((void *)param, plen, db, NULL, md, NULL))
goto err;
/* step 3b: zero bytes array of length nLen - KLen - 2 HLen -2 */
memset(db + mdlen, 0, emlen - flen - 2 * mdlen - 1);
/* step 3c: DB = HA || PS || 00000001 || K */
db[emlen - flen - mdlen - 1] = 0x01;
memcpy(db + emlen - flen - mdlen, from, (unsigned int)flen);
/* step 3d: generate random byte string */
if (RAND_bytes(seed, mdlen) <= 0)
goto err;
@@ -88,13 +101,17 @@ int RSA_padding_add_PKCS1_OAEP_mgf1(unsigned char *to, int tlen,
goto err;
}
/* step 3e: dbMask = MGF(mgfSeed, nLen - HLen - 1) */
if (PKCS1_MGF1(dbmask, dbmask_len, seed, mdlen, mgf1md) < 0)
goto err;
/* step 3f: maskedDB = DB XOR dbMask */
for (i = 0; i < dbmask_len; i++)
db[i] ^= dbmask[i];
/* step 3g: mgfSeed = MGF(maskedDB, HLen) */
if (PKCS1_MGF1(seedmask, mdlen, db, dbmask_len, mgf1md) < 0)
goto err;
/* stepo 3h: maskedMGFSeed = mgfSeed XOR mgfSeedMask */
for (i = 0; i < mdlen; i++)
seed[i] ^= seedmask[i];
rv = 1;
@@ -270,6 +287,13 @@ int RSA_padding_check_PKCS1_OAEP_mgf1(unsigned char *to, int tlen,
return constant_time_select_int(good, mlen, -1);
}
/*
* Mask Generation Function corresponding to section 7.2.2.2 of NIST SP 800-56B.
* The variables are named differently to NIST:
* mask (T) and len (maskLen)are the returned mask.
* seed (mgfSeed).
* The range checking steps inm the process are performed outside.
*/
int PKCS1_MGF1(unsigned char *mask, long len,
const unsigned char *seed, long seedlen, const EVP_MD *dgst)
{
@@ -285,11 +309,14 @@ int PKCS1_MGF1(unsigned char *mask, long len,
mdlen = EVP_MD_size(dgst);
if (mdlen < 0)
goto err;
/* step 4 */
for (i = 0; outlen < len; i++) {
/* step 4a: D = I2BS(counter, 4) */
cnt[0] = (unsigned char)((i >> 24) & 255);
cnt[1] = (unsigned char)((i >> 16) & 255);
cnt[2] = (unsigned char)((i >> 8)) & 255;
cnt[3] = (unsigned char)(i & 255);
/* step 4b: T =T || hash(mgfSeed || D) */
if (!EVP_DigestInit_ex(c, dgst, NULL)
|| !EVP_DigestUpdate(c, seed, seedlen)
|| !EVP_DigestUpdate(c, cnt, 4))
crypto/sha/asm/keccak1600-armv8.pl
+1
View File
@@ -51,6 +51,7 @@
# Kryo 12
# Denver 7.8
# Apple A7 7.2
# ThunderX2 9.7
#
# (*) Corresponds to SHA3-256. No improvement coefficients are listed
# because they vary too much from compiler to compiler. Newer
crypto/sha/asm/sha1-armv8.pl
+1
View File
@@ -27,6 +27,7 @@
# X-Gene 8.80 (+200%)
# Mongoose 2.05 6.50 (+160%)
# Kryo 1.88 8.00 (+90%)
# ThunderX2 2.64 6.36 (+150%)
#
# (*) Software results are presented mostly for reference purposes.
# (**) Keep in mind that Denver relies on binary translation, which
crypto/sha/asm/sha512-armv8.pl
+1
View File
@@ -28,6 +28,7 @@
# X-Gene 20.0 (+100%) 12.8 (+300%(***))
# Mongoose 2.36 13.0 (+50%) 8.36 (+33%)
# Kryo 1.92 17.4 (+30%) 11.2 (+8%)
# ThunderX2 2.54 13.2 (+40%) 8.40 (+18%)
#
# (*) Software SHA256 results are of lesser relevance, presented
# mostly for informational purposes.