Hakase/openssl-custom
1
0 Fork 0
Code Issues Pull Requests Releases Wiki Activity

Latest update (add quic)

Browse Source
This commit is contained in:
Hakase
2020-07-12 19:52:18 +09:00
parent 3a6d64bb68
commit e85ee33eee
501 changed files with 19166 additions and 10232 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crypto/ec/asm/ecp_nistz256-avx2.pl
-2082
View File
@@ -1,2082 +0,0 @@
#! /usr/bin/env perl
# Copyright 2014-2020 The OpenSSL Project Authors. All Rights Reserved.
# Copyright (c) 2014, Intel Corporation. All Rights Reserved.
#
# Licensed under the Apache License 2.0 (the "License"). You may not use
# this file except in compliance with the License. You can obtain a copy
# in the file LICENSE in the source distribution or at
# https://www.openssl.org/source/license.html
#
# Originally written by Shay Gueron (1, 2), and Vlad Krasnov (1)
# (1) Intel Corporation, Israel Development Center, Haifa, Israel
# (2) University of Haifa, Israel
#
# Reference:
# S.Gueron and V.Krasnov, "Fast Prime Field Elliptic Curve Cryptography with
# 256 Bit Primes"
# $output is the last argument if it looks like a file (it has an extension)
# $flavour is the first argument if it doesn't look like a file
$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
die "can't locate x86_64-xlate.pl";
open OUT,"| \"$^X\" $xlate $flavour \"$output\""
or die "can't call $xlate: $!";
*STDOUT=*OUT;
if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
=~ /GNU assembler version ([2-9]\.[0-9]+)/) {
$avx = ($1>=2.19) + ($1>=2.22);
$addx = ($1>=2.23);
}
if (!$addx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
`nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
$avx = ($1>=2.09) + ($1>=2.10);
$addx = ($1>=2.10);
}
if (!$addx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
`ml64 2>&1` =~ /Version ([0-9]+)\./) {
$avx = ($1>=10) + ($1>=11);
$addx = ($1>=12);
}
if (!$addx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|based on LLVM) ([0-9]+)\.([0-9]+)/) {
my $ver = $2 + $3/100.0; # 3.1->3.01, 3.10->3.10
$avx = ($ver>=3.0) + ($ver>=3.01);
$addx = ($ver>=3.03);
}
if ($avx>=2) {{
$digit_size = "\$29";
$n_digits = "\$9";
$code.=<<___;
.text
.align 64
.LAVX2_AND_MASK:
.LAVX2_POLY:
.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff
.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff
.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff
.quad 0x000001ff, 0x000001ff, 0x000001ff, 0x000001ff
.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000
.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000
.quad 0x00040000, 0x00040000, 0x00040000, 0x00040000
.quad 0x1fe00000, 0x1fe00000, 0x1fe00000, 0x1fe00000
.quad 0x00ffffff, 0x00ffffff, 0x00ffffff, 0x00ffffff
.LAVX2_POLY_x2:
.quad 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC
.quad 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC
.quad 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC
.quad 0x400007FC, 0x400007FC, 0x400007FC, 0x400007FC
.quad 0x3FFFFFFE, 0x3FFFFFFE, 0x3FFFFFFE, 0x3FFFFFFE
.quad 0x3FFFFFFE, 0x3FFFFFFE, 0x3FFFFFFE, 0x3FFFFFFE
.quad 0x400FFFFE, 0x400FFFFE, 0x400FFFFE, 0x400FFFFE
.quad 0x7F7FFFFE, 0x7F7FFFFE, 0x7F7FFFFE, 0x7F7FFFFE
.quad 0x03FFFFFC, 0x03FFFFFC, 0x03FFFFFC, 0x03FFFFFC
.LAVX2_POLY_x8:
.quad 0xFFFFFFF8, 0xFFFFFFF8, 0xFFFFFFF8, 0xFFFFFFF8
.quad 0xFFFFFFF8, 0xFFFFFFF8, 0xFFFFFFF8, 0xFFFFFFF8
.quad 0xFFFFFFF8, 0xFFFFFFF8, 0xFFFFFFF8, 0xFFFFFFF8
.quad 0x80000FF8, 0x80000FF8, 0x80000FF8, 0x80000FF8
.quad 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC
.quad 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC, 0x7FFFFFFC
.quad 0x801FFFFC, 0x801FFFFC, 0x801FFFFC, 0x801FFFFC
.quad 0xFEFFFFFC, 0xFEFFFFFC, 0xFEFFFFFC, 0xFEFFFFFC
.quad 0x07FFFFF8, 0x07FFFFF8, 0x07FFFFF8, 0x07FFFFF8
.LONE:
.quad 0x00000020, 0x00000020, 0x00000020, 0x00000020
.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000
.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000
.quad 0x1fffc000, 0x1fffc000, 0x1fffc000, 0x1fffc000
.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff
.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff
.quad 0x1f7fffff, 0x1f7fffff, 0x1f7fffff, 0x1f7fffff
.quad 0x03ffffff, 0x03ffffff, 0x03ffffff, 0x03ffffff
.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000
# RR = 2^266 mod p in AVX2 format, to transform from the native OpenSSL
# Montgomery form (*2^256) to our format (*2^261)
.LTO_MONT_AVX2:
.quad 0x00000400, 0x00000400, 0x00000400, 0x00000400
.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000
.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000
.quad 0x1ff80000, 0x1ff80000, 0x1ff80000, 0x1ff80000
.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff
.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff
.quad 0x0fffffff, 0x0fffffff, 0x0fffffff, 0x0fffffff
.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff
.quad 0x00000003, 0x00000003, 0x00000003, 0x00000003
.LFROM_MONT_AVX2:
.quad 0x00000001, 0x00000001, 0x00000001, 0x00000001
.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000
.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000
.quad 0x1ffffe00, 0x1ffffe00, 0x1ffffe00, 0x1ffffe00
.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff
.quad 0x1fffffff, 0x1fffffff, 0x1fffffff, 0x1fffffff
.quad 0x1ffbffff, 0x1ffbffff, 0x1ffbffff, 0x1ffbffff
.quad 0x001fffff, 0x001fffff, 0x001fffff, 0x001fffff
.quad 0x00000000, 0x00000000, 0x00000000, 0x00000000
.LIntOne:
.long 1,1,1,1,1,1,1,1
___
{
# This function receives a pointer to an array of four affine points
# (X, Y, <1>) and rearranges the data for AVX2 execution, while
# converting it to 2^29 radix redundant form
my ($X0,$X1,$X2,$X3, $Y0,$Y1,$Y2,$Y3,
$T0,$T1,$T2,$T3, $T4,$T5,$T6,$T7)=map("%ymm$_",(0..15));
$code.=<<___;
.globl ecp_nistz256_avx2_transpose_convert
.type ecp_nistz256_avx2_transpose_convert,\@function,2
.align 64
ecp_nistz256_avx2_transpose_convert:
vzeroupper
___
$code.=<<___ if ($win64);
lea -8-16*10(%rsp), %rsp
vmovaps %xmm6, -8-16*10(%rax)
vmovaps %xmm7, -8-16*9(%rax)
vmovaps %xmm8, -8-16*8(%rax)
vmovaps %xmm9, -8-16*7(%rax)
vmovaps %xmm10, -8-16*6(%rax)
vmovaps %xmm11, -8-16*5(%rax)
vmovaps %xmm12, -8-16*4(%rax)
vmovaps %xmm13, -8-16*3(%rax)
vmovaps %xmm14, -8-16*2(%rax)
vmovaps %xmm15, -8-16*1(%rax)
___
$code.=<<___;
# Load the data
vmovdqa 32*0(%rsi), $X0
lea 112(%rsi), %rax # size optimization
vmovdqa 32*1(%rsi), $Y0
lea .LAVX2_AND_MASK(%rip), %rdx
vmovdqa 32*2(%rsi), $X1
vmovdqa 32*3(%rsi), $Y1
vmovdqa 32*4-112(%rax), $X2
vmovdqa 32*5-112(%rax), $Y2
vmovdqa 32*6-112(%rax), $X3
vmovdqa 32*7-112(%rax), $Y3
# Transpose X and Y independently
vpunpcklqdq $X1, $X0, $T0 # T0 = [B2 A2 B0 A0]
vpunpcklqdq $X3, $X2, $T1 # T1 = [D2 C2 D0 C0]
vpunpckhqdq $X1, $X0, $T2 # T2 = [B3 A3 B1 A1]
vpunpckhqdq $X3, $X2, $T3 # T3 = [D3 C3 D1 C1]
vpunpcklqdq $Y1, $Y0, $T4
vpunpcklqdq $Y3, $Y2, $T5
vpunpckhqdq $Y1, $Y0, $T6
vpunpckhqdq $Y3, $Y2, $T7
vperm2i128 \$0x20, $T1, $T0, $X0 # X0 = [D0 C0 B0 A0]
vperm2i128 \$0x20, $T3, $T2, $X1 # X1 = [D1 C1 B1 A1]
vperm2i128 \$0x31, $T1, $T0, $X2 # X2 = [D2 C2 B2 A2]
vperm2i128 \$0x31, $T3, $T2, $X3 # X3 = [D3 C3 B3 A3]
vperm2i128 \$0x20, $T5, $T4, $Y0
vperm2i128 \$0x20, $T7, $T6, $Y1
vperm2i128 \$0x31, $T5, $T4, $Y2
vperm2i128 \$0x31, $T7, $T6, $Y3
vmovdqa (%rdx), $T7
vpand (%rdx), $X0, $T0 # out[0] = in[0] & mask;
vpsrlq \$29, $X0, $X0
vpand $T7, $X0, $T1 # out[1] = (in[0] >> shift) & mask;
vpsrlq \$29, $X0, $X0
vpsllq \$6, $X1, $T2
vpxor $X0, $T2, $T2
vpand $T7, $T2, $T2 # out[2] = ((in[0] >> (shift*2)) ^ (in[1] << (64-shift*2))) & mask;
vpsrlq \$23, $X1, $X1
vpand $T7, $X1, $T3 # out[3] = (in[1] >> ((shift*3)%64)) & mask;
vpsrlq \$29, $X1, $X1
vpsllq \$12, $X2, $T4
vpxor $X1, $T4, $T4
vpand $T7, $T4, $T4 # out[4] = ((in[1] >> ((shift*4)%64)) ^ (in[2] << (64*2-shift*4))) & mask;
vpsrlq \$17, $X2, $X2
vpand $T7, $X2, $T5 # out[5] = (in[2] >> ((shift*5)%64)) & mask;
vpsrlq \$29, $X2, $X2
vpsllq \$18, $X3, $T6
vpxor $X2, $T6, $T6
vpand $T7, $T6, $T6 # out[6] = ((in[2] >> ((shift*6)%64)) ^ (in[3] << (64*3-shift*6))) & mask;
vpsrlq \$11, $X3, $X3
vmovdqa $T0, 32*0(%rdi)
lea 112(%rdi), %rax # size optimization
vpand $T7, $X3, $T0 # out[7] = (in[3] >> ((shift*7)%64)) & mask;
vpsrlq \$29, $X3, $X3 # out[8] = (in[3] >> ((shift*8)%64)) & mask;
vmovdqa $T1, 32*1(%rdi)
vmovdqa $T2, 32*2(%rdi)
vmovdqa $T3, 32*3(%rdi)
vmovdqa $T4, 32*4-112(%rax)
vmovdqa $T5, 32*5-112(%rax)
vmovdqa $T6, 32*6-112(%rax)
vmovdqa $T0, 32*7-112(%rax)
vmovdqa $X3, 32*8-112(%rax)
lea 448(%rdi), %rax # size optimization
vpand $T7, $Y0, $T0 # out[0] = in[0] & mask;
vpsrlq \$29, $Y0, $Y0
vpand $T7, $Y0, $T1 # out[1] = (in[0] >> shift) & mask;
vpsrlq \$29, $Y0, $Y0
vpsllq \$6, $Y1, $T2
vpxor $Y0, $T2, $T2
vpand $T7, $T2, $T2 # out[2] = ((in[0] >> (shift*2)) ^ (in[1] << (64-shift*2))) & mask;
vpsrlq \$23, $Y1, $Y1
vpand $T7, $Y1, $T3 # out[3] = (in[1] >> ((shift*3)%64)) & mask;
vpsrlq \$29, $Y1, $Y1
vpsllq \$12, $Y2, $T4
vpxor $Y1, $T4, $T4
vpand $T7, $T4, $T4 # out[4] = ((in[1] >> ((shift*4)%64)) ^ (in[2] << (64*2-shift*4))) & mask;
vpsrlq \$17, $Y2, $Y2
vpand $T7, $Y2, $T5 # out[5] = (in[2] >> ((shift*5)%64)) & mask;
vpsrlq \$29, $Y2, $Y2
vpsllq \$18, $Y3, $T6
vpxor $Y2, $T6, $T6
vpand $T7, $T6, $T6 # out[6] = ((in[2] >> ((shift*6)%64)) ^ (in[3] << (64*3-shift*6))) & mask;
vpsrlq \$11, $Y3, $Y3
vmovdqa $T0, 32*9-448(%rax)
vpand $T7, $Y3, $T0 # out[7] = (in[3] >> ((shift*7)%64)) & mask;
vpsrlq \$29, $Y3, $Y3 # out[8] = (in[3] >> ((shift*8)%64)) & mask;
vmovdqa $T1, 32*10-448(%rax)
vmovdqa $T2, 32*11-448(%rax)
vmovdqa $T3, 32*12-448(%rax)
vmovdqa $T4, 32*13-448(%rax)
vmovdqa $T5, 32*14-448(%rax)
vmovdqa $T6, 32*15-448(%rax)
vmovdqa $T0, 32*16-448(%rax)
vmovdqa $Y3, 32*17-448(%rax)
vzeroupper
___
$code.=<<___ if ($win64);
movaps 16*0(%rsp), %xmm6
movaps 16*1(%rsp), %xmm7
movaps 16*2(%rsp), %xmm8
movaps 16*3(%rsp), %xmm9
movaps 16*4(%rsp), %xmm10
movaps 16*5(%rsp), %xmm11
movaps 16*6(%rsp), %xmm12
movaps 16*7(%rsp), %xmm13
movaps 16*8(%rsp), %xmm14
movaps 16*9(%rsp), %xmm15
lea 8+16*10(%rsp), %rsp
___
$code.=<<___;
ret
.size ecp_nistz256_avx2_transpose_convert,.-ecp_nistz256_avx2_transpose_convert
___
}
{
################################################################################
# This function receives a pointer to an array of four AVX2 formatted points
# (X, Y, Z) convert the data to normal representation, and rearranges the data
my ($D0,$D1,$D2,$D3, $D4,$D5,$D6,$D7, $D8)=map("%ymm$_",(0..8));
my ($T0,$T1,$T2,$T3, $T4,$T5,$T6)=map("%ymm$_",(9..15));
$code.=<<___;
.globl ecp_nistz256_avx2_convert_transpose_back
.type ecp_nistz256_avx2_convert_transpose_back,\@function,2
.align 32
ecp_nistz256_avx2_convert_transpose_back:
vzeroupper
___
$code.=<<___ if ($win64);
lea -8-16*10(%rsp), %rsp
vmovaps %xmm6, -8-16*10(%rax)
vmovaps %xmm7, -8-16*9(%rax)
vmovaps %xmm8, -8-16*8(%rax)
vmovaps %xmm9, -8-16*7(%rax)
vmovaps %xmm10, -8-16*6(%rax)
vmovaps %xmm11, -8-16*5(%rax)
vmovaps %xmm12, -8-16*4(%rax)
vmovaps %xmm13, -8-16*3(%rax)
vmovaps %xmm14, -8-16*2(%rax)
vmovaps %xmm15, -8-16*1(%rax)
___
$code.=<<___;
mov \$3, %ecx
.Lconv_loop:
vmovdqa 32*0(%rsi), $D0
lea 160(%rsi), %rax # size optimization
vmovdqa 32*1(%rsi), $D1
vmovdqa 32*2(%rsi), $D2
vmovdqa 32*3(%rsi), $D3
vmovdqa 32*4-160(%rax), $D4
vmovdqa 32*5-160(%rax), $D5
vmovdqa 32*6-160(%rax), $D6
vmovdqa 32*7-160(%rax), $D7
vmovdqa 32*8-160(%rax), $D8
vpsllq \$29, $D1, $D1
vpsllq \$58, $D2, $T0
vpaddq $D1, $D0, $D0
vpaddq $T0, $D0, $D0 # out[0] = (in[0]) ^ (in[1] << shift*1) ^ (in[2] << shift*2);
vpsrlq \$6, $D2, $D2
vpsllq \$23, $D3, $D3
vpsllq \$52, $D4, $T1
vpaddq $D2, $D3, $D3
vpaddq $D3, $T1, $D1 # out[1] = (in[2] >> (64*1-shift*2)) ^ (in[3] << shift*3%64) ^ (in[4] << shift*4%64);
vpsrlq \$12, $D4, $D4
vpsllq \$17, $D5, $D5
vpsllq \$46, $D6, $T2
vpaddq $D4, $D5, $D5
vpaddq $D5, $T2, $D2 # out[2] = (in[4] >> (64*2-shift*4)) ^ (in[5] << shift*5%64) ^ (in[6] << shift*6%64);
vpsrlq \$18, $D6, $D6
vpsllq \$11, $D7, $D7
vpsllq \$40, $D8, $T3
vpaddq $D6, $D7, $D7
vpaddq $D7, $T3, $D3 # out[3] = (in[6] >> (64*3-shift*6)) ^ (in[7] << shift*7%64) ^ (in[8] << shift*8%64);
vpunpcklqdq $D1, $D0, $T0 # T0 = [B2 A2 B0 A0]
vpunpcklqdq $D3, $D2, $T1 # T1 = [D2 C2 D0 C0]
vpunpckhqdq $D1, $D0, $T2 # T2 = [B3 A3 B1 A1]
vpunpckhqdq $D3, $D2, $T3 # T3 = [D3 C3 D1 C1]
vperm2i128 \$0x20, $T1, $T0, $D0 # X0 = [D0 C0 B0 A0]
vperm2i128 \$0x20, $T3, $T2, $D1 # X1 = [D1 C1 B1 A1]
vperm2i128 \$0x31, $T1, $T0, $D2 # X2 = [D2 C2 B2 A2]
vperm2i128 \$0x31, $T3, $T2, $D3 # X3 = [D3 C3 B3 A3]
vmovdqa $D0, 32*0(%rdi)
vmovdqa $D1, 32*3(%rdi)
vmovdqa $D2, 32*6(%rdi)
vmovdqa $D3, 32*9(%rdi)
lea 32*9(%rsi), %rsi
lea 32*1(%rdi), %rdi
dec %ecx
jnz .Lconv_loop
vzeroupper
___
$code.=<<___ if ($win64);
movaps 16*0(%rsp), %xmm6
movaps 16*1(%rsp), %xmm7
movaps 16*2(%rsp), %xmm8
movaps 16*3(%rsp), %xmm9
movaps 16*4(%rsp), %xmm10
movaps 16*5(%rsp), %xmm11
movaps 16*6(%rsp), %xmm12
movaps 16*7(%rsp), %xmm13
movaps 16*8(%rsp), %xmm14
movaps 16*9(%rsp), %xmm15
lea 8+16*10(%rsp), %rsp
___
$code.=<<___;
ret
.size ecp_nistz256_avx2_convert_transpose_back,.-ecp_nistz256_avx2_convert_transpose_back
___
}
{
my ($r_ptr,$a_ptr,$b_ptr,$itr)=("%rdi","%rsi","%rdx","%ecx");
my ($ACC0,$ACC1,$ACC2,$ACC3,$ACC4,$ACC5,$ACC6,$ACC7,$ACC8)=map("%ymm$_",(0..8));
my ($B,$Y,$T0,$AND_MASK,$OVERFLOW)=map("%ymm$_",(9..13));
sub NORMALIZE {
my $ret=<<___;
vpsrlq $digit_size, $ACC0, $T0
vpand $AND_MASK, $ACC0, $ACC0
vpaddq $T0, $ACC1, $ACC1
vpsrlq $digit_size, $ACC1, $T0
vpand $AND_MASK, $ACC1, $ACC1
vpaddq $T0, $ACC2, $ACC2
vpsrlq $digit_size, $ACC2, $T0
vpand $AND_MASK, $ACC2, $ACC2
vpaddq $T0, $ACC3, $ACC3
vpsrlq $digit_size, $ACC3, $T0
vpand $AND_MASK, $ACC3, $ACC3
vpaddq $T0, $ACC4, $ACC4
vpsrlq $digit_size, $ACC4, $T0
vpand $AND_MASK, $ACC4, $ACC4
vpaddq $T0, $ACC5, $ACC5
vpsrlq $digit_size, $ACC5, $T0
vpand $AND_MASK, $ACC5, $ACC5
vpaddq $T0, $ACC6, $ACC6
vpsrlq $digit_size, $ACC6, $T0
vpand $AND_MASK, $ACC6, $ACC6
vpaddq $T0, $ACC7, $ACC7
vpsrlq $digit_size, $ACC7, $T0
vpand $AND_MASK, $ACC7, $ACC7
vpaddq $T0, $ACC8, $ACC8
#vpand $AND_MASK, $ACC8, $ACC8
___
$ret;
}
sub STORE {
my $ret=<<___;
vmovdqa $ACC0, 32*0(%rdi)
lea 160(%rdi), %rax # size optimization
vmovdqa $ACC1, 32*1(%rdi)
vmovdqa $ACC2, 32*2(%rdi)
vmovdqa $ACC3, 32*3(%rdi)
vmovdqa $ACC4, 32*4-160(%rax)
vmovdqa $ACC5, 32*5-160(%rax)
vmovdqa $ACC6, 32*6-160(%rax)
vmovdqa $ACC7, 32*7-160(%rax)
vmovdqa $ACC8, 32*8-160(%rax)
___
$ret;
}
$code.=<<___;
.type avx2_normalize,\@abi-omnipotent
.align 32
avx2_normalize:
vpsrlq $digit_size, $ACC0, $T0
vpand $AND_MASK, $ACC0, $ACC0
vpaddq $T0, $ACC1, $ACC1
vpsrlq $digit_size, $ACC1, $T0
vpand $AND_MASK, $ACC1, $ACC1
vpaddq $T0, $ACC2, $ACC2
vpsrlq $digit_size, $ACC2, $T0
vpand $AND_MASK, $ACC2, $ACC2
vpaddq $T0, $ACC3, $ACC3
vpsrlq $digit_size, $ACC3, $T0
vpand $AND_MASK, $ACC3, $ACC3
vpaddq $T0, $ACC4, $ACC4
vpsrlq $digit_size, $ACC4, $T0
vpand $AND_MASK, $ACC4, $ACC4
vpaddq $T0, $ACC5, $ACC5
vpsrlq $digit_size, $ACC5, $T0
vpand $AND_MASK, $ACC5, $ACC5
vpaddq $T0, $ACC6, $ACC6
vpsrlq $digit_size, $ACC6, $T0
vpand $AND_MASK, $ACC6, $ACC6
vpaddq $T0, $ACC7, $ACC7
vpsrlq $digit_size, $ACC7, $T0
vpand $AND_MASK, $ACC7, $ACC7
vpaddq $T0, $ACC8, $ACC8
#vpand $AND_MASK, $ACC8, $ACC8
ret
.size avx2_normalize,.-avx2_normalize
.type avx2_normalize_n_store,\@abi-omnipotent
.align 32
avx2_normalize_n_store:
vpsrlq $digit_size, $ACC0, $T0
vpand $AND_MASK, $ACC0, $ACC0
vpaddq $T0, $ACC1, $ACC1
vpsrlq $digit_size, $ACC1, $T0
vpand $AND_MASK, $ACC1, $ACC1
vmovdqa $ACC0, 32*0(%rdi)
lea 160(%rdi), %rax # size optimization
vpaddq $T0, $ACC2, $ACC2
vpsrlq $digit_size, $ACC2, $T0
vpand $AND_MASK, $ACC2, $ACC2
vmovdqa $ACC1, 32*1(%rdi)
vpaddq $T0, $ACC3, $ACC3
vpsrlq $digit_size, $ACC3, $T0
vpand $AND_MASK, $ACC3, $ACC3
vmovdqa $ACC2, 32*2(%rdi)
vpaddq $T0, $ACC4, $ACC4
vpsrlq $digit_size, $ACC4, $T0
vpand $AND_MASK, $ACC4, $ACC4
vmovdqa $ACC3, 32*3(%rdi)
vpaddq $T0, $ACC5, $ACC5
vpsrlq $digit_size, $ACC5, $T0
vpand $AND_MASK, $ACC5, $ACC5
vmovdqa $ACC4, 32*4-160(%rax)
vpaddq $T0, $ACC6, $ACC6
vpsrlq $digit_size, $ACC6, $T0
vpand $AND_MASK, $ACC6, $ACC6
vmovdqa $ACC5, 32*5-160(%rax)
vpaddq $T0, $ACC7, $ACC7
vpsrlq $digit_size, $ACC7, $T0
vpand $AND_MASK, $ACC7, $ACC7
vmovdqa $ACC6, 32*6-160(%rax)
vpaddq $T0, $ACC8, $ACC8
#vpand $AND_MASK, $ACC8, $ACC8
vmovdqa $ACC7, 32*7-160(%rax)
vmovdqa $ACC8, 32*8-160(%rax)
ret
.size avx2_normalize_n_store,.-avx2_normalize_n_store
################################################################################
# void avx2_mul_x4(void* RESULTx4, void *Ax4, void *Bx4);
.type avx2_mul_x4,\@abi-omnipotent
.align 32
avx2_mul_x4:
lea .LAVX2_POLY(%rip), %rax
vpxor $ACC0, $ACC0, $ACC0
vpxor $ACC1, $ACC1, $ACC1
vpxor $ACC2, $ACC2, $ACC2
vpxor $ACC3, $ACC3, $ACC3
vpxor $ACC4, $ACC4, $ACC4
vpxor $ACC5, $ACC5, $ACC5
vpxor $ACC6, $ACC6, $ACC6
vpxor $ACC7, $ACC7, $ACC7
vmovdqa 32*7(%rax), %ymm14
vmovdqa 32*8(%rax), %ymm15
mov $n_digits, $itr
lea -512($a_ptr), $a_ptr # strategic bias to control u-op density
jmp .Lavx2_mul_x4_loop
.align 32
.Lavx2_mul_x4_loop:
vmovdqa 32*0($b_ptr), $B
lea 32*1($b_ptr), $b_ptr
vpmuludq 32*0+512($a_ptr), $B, $T0
vpmuludq 32*1+512($a_ptr), $B, $OVERFLOW # borrow $OVERFLOW
vpaddq $T0, $ACC0, $ACC0
vpmuludq 32*2+512($a_ptr), $B, $T0
vpaddq $OVERFLOW, $ACC1, $ACC1
vpand $AND_MASK, $ACC0, $Y
vpmuludq 32*3+512($a_ptr), $B, $OVERFLOW
vpaddq $T0, $ACC2, $ACC2
vpmuludq 32*4+512($a_ptr), $B, $T0
vpaddq $OVERFLOW, $ACC3, $ACC3
vpmuludq 32*5+512($a_ptr), $B, $OVERFLOW
vpaddq $T0, $ACC4, $ACC4
vpmuludq 32*6+512($a_ptr), $B, $T0
vpaddq $OVERFLOW, $ACC5, $ACC5
vpmuludq 32*7+512($a_ptr), $B, $OVERFLOW
vpaddq $T0, $ACC6, $ACC6
# Skip some multiplications, optimizing for the constant poly
vpmuludq $AND_MASK, $Y, $T0
vpaddq $OVERFLOW, $ACC7, $ACC7
vpmuludq 32*8+512($a_ptr), $B, $ACC8
vpaddq $T0, $ACC0, $OVERFLOW
vpaddq $T0, $ACC1, $ACC0
vpsrlq $digit_size, $OVERFLOW, $OVERFLOW
vpaddq $T0, $ACC2, $ACC1
vpmuludq 32*3(%rax), $Y, $T0
vpaddq $OVERFLOW, $ACC0, $ACC0
vpaddq $T0, $ACC3, $ACC2
.byte 0x67
vmovdqa $ACC4, $ACC3
vpsllq \$18, $Y, $OVERFLOW
.byte 0x67
vmovdqa $ACC5, $ACC4
vpmuludq %ymm14, $Y, $T0
vpaddq $OVERFLOW, $ACC6, $ACC5
vpmuludq %ymm15, $Y, $OVERFLOW
vpaddq $T0, $ACC7, $ACC6
vpaddq $OVERFLOW, $ACC8, $ACC7
dec $itr
jnz .Lavx2_mul_x4_loop
vpxor $ACC8, $ACC8, $ACC8
ret
.size avx2_mul_x4,.-avx2_mul_x4
# Function optimized for the constant 1
################################################################################
# void avx2_mul_by1_x4(void* RESULTx4, void *Ax4);
.type avx2_mul_by1_x4,\@abi-omnipotent
.align 32
avx2_mul_by1_x4:
lea .LAVX2_POLY(%rip), %rax
vpxor $ACC0, $ACC0, $ACC0
vpxor $ACC1, $ACC1, $ACC1
vpxor $ACC2, $ACC2, $ACC2
vpxor $ACC3, $ACC3, $ACC3
vpxor $ACC4, $ACC4, $ACC4
vpxor $ACC5, $ACC5, $ACC5
vpxor $ACC6, $ACC6, $ACC6
vpxor $ACC7, $ACC7, $ACC7
vpxor $ACC8, $ACC8, $ACC8
vmovdqa 32*3+.LONE(%rip), %ymm14
vmovdqa 32*7+.LONE(%rip), %ymm15
mov $n_digits, $itr
jmp .Lavx2_mul_by1_x4_loop
.align 32
.Lavx2_mul_by1_x4_loop:
vmovdqa 32*0($a_ptr), $B
.byte 0x48,0x8d,0xb6,0x20,0,0,0 # lea 32*1($a_ptr), $a_ptr
vpsllq \$5, $B, $OVERFLOW
vpmuludq %ymm14, $B, $T0
vpaddq $OVERFLOW, $ACC0, $ACC0
vpaddq $T0, $ACC3, $ACC3
.byte 0x67
vpmuludq $AND_MASK, $B, $T0
vpand $AND_MASK, $ACC0, $Y
vpaddq $T0, $ACC4, $ACC4
vpaddq $T0, $ACC5, $ACC5
vpaddq $T0, $ACC6, $ACC6
vpsllq \$23, $B, $T0
.byte 0x67,0x67
vpmuludq %ymm15, $B, $OVERFLOW
vpsubq $T0, $ACC6, $ACC6
vpmuludq $AND_MASK, $Y, $T0
vpaddq $OVERFLOW, $ACC7, $ACC7
vpaddq $T0, $ACC0, $OVERFLOW
vpaddq $T0, $ACC1, $ACC0
.byte 0x67,0x67
vpsrlq $digit_size, $OVERFLOW, $OVERFLOW
vpaddq $T0, $ACC2, $ACC1
vpmuludq 32*3(%rax), $Y, $T0
vpaddq $OVERFLOW, $ACC0, $ACC0
vpaddq $T0, $ACC3, $ACC2
vmovdqa $ACC4, $ACC3
vpsllq \$18, $Y, $OVERFLOW
vmovdqa $ACC5, $ACC4
vpmuludq 32*7(%rax), $Y, $T0
vpaddq $OVERFLOW, $ACC6, $ACC5
vpaddq $T0, $ACC7, $ACC6
vpmuludq 32*8(%rax), $Y, $ACC7
dec $itr
jnz .Lavx2_mul_by1_x4_loop
ret
.size avx2_mul_by1_x4,.-avx2_mul_by1_x4
################################################################################
# void avx2_sqr_x4(void* RESULTx4, void *Ax4, void *Bx4);
.type avx2_sqr_x4,\@abi-omnipotent
.align 32
avx2_sqr_x4:
lea .LAVX2_POLY(%rip), %rax
vmovdqa 32*7(%rax), %ymm14
vmovdqa 32*8(%rax), %ymm15
vmovdqa 32*0($a_ptr), $B
vmovdqa 32*1($a_ptr), $ACC1
vmovdqa 32*2($a_ptr), $ACC2
vmovdqa 32*3($a_ptr), $ACC3
vmovdqa 32*4($a_ptr), $ACC4
vmovdqa 32*5($a_ptr), $ACC5
vmovdqa 32*6($a_ptr), $ACC6
vmovdqa 32*7($a_ptr), $ACC7
vpaddq $ACC1, $ACC1, $ACC1 # 2*$ACC0..7
vmovdqa 32*8($a_ptr), $ACC8
vpaddq $ACC2, $ACC2, $ACC2
vmovdqa $ACC1, 32*0(%rcx)
vpaddq $ACC3, $ACC3, $ACC3
vmovdqa $ACC2, 32*1(%rcx)
vpaddq $ACC4, $ACC4, $ACC4
vmovdqa $ACC3, 32*2(%rcx)
vpaddq $ACC5, $ACC5, $ACC5
vmovdqa $ACC4, 32*3(%rcx)
vpaddq $ACC6, $ACC6, $ACC6
vmovdqa $ACC5, 32*4(%rcx)
vpaddq $ACC7, $ACC7, $ACC7
vmovdqa $ACC6, 32*5(%rcx)
vpaddq $ACC8, $ACC8, $ACC8
vmovdqa $ACC7, 32*6(%rcx)
vmovdqa $ACC8, 32*7(%rcx)
#itr 1
vpmuludq $B, $B, $ACC0
vpmuludq $B, $ACC1, $ACC1
vpand $AND_MASK, $ACC0, $Y
vpmuludq $B, $ACC2, $ACC2
vpmuludq $B, $ACC3, $ACC3
vpmuludq $B, $ACC4, $ACC4
vpmuludq $B, $ACC5, $ACC5
vpmuludq $B, $ACC6, $ACC6
vpmuludq $AND_MASK, $Y, $T0
vpmuludq $B, $ACC7, $ACC7
vpmuludq $B, $ACC8, $ACC8
vmovdqa 32*1($a_ptr), $B
vpaddq $T0, $ACC0, $OVERFLOW
vpaddq $T0, $ACC1, $ACC0
vpsrlq $digit_size, $OVERFLOW, $OVERFLOW
vpaddq $T0, $ACC2, $ACC1
vpmuludq 32*3(%rax), $Y, $T0
vpaddq $OVERFLOW, $ACC0, $ACC0
vpaddq $T0, $ACC3, $ACC2
vmovdqa $ACC4, $ACC3
vpsllq \$18, $Y, $T0
vmovdqa $ACC5, $ACC4
vpmuludq %ymm14, $Y, $OVERFLOW
vpaddq $T0, $ACC6, $ACC5
vpmuludq %ymm15, $Y, $T0
vpaddq $OVERFLOW, $ACC7, $ACC6
vpaddq $T0, $ACC8, $ACC7
#itr 2
vpmuludq $B, $B, $OVERFLOW
vpand $AND_MASK, $ACC0, $Y
vpmuludq 32*1(%rcx), $B, $T0
vpaddq $OVERFLOW, $ACC1, $ACC1
vpmuludq 32*2(%rcx), $B, $OVERFLOW
vpaddq $T0, $ACC2, $ACC2
vpmuludq 32*3(%rcx), $B, $T0
vpaddq $OVERFLOW, $ACC3, $ACC3
vpmuludq 32*4(%rcx), $B, $OVERFLOW
vpaddq $T0, $ACC4, $ACC4
vpmuludq 32*5(%rcx), $B, $T0
vpaddq $OVERFLOW, $ACC5, $ACC5
vpmuludq 32*6(%rcx), $B, $OVERFLOW
vpaddq $T0, $ACC6, $ACC6
vpmuludq $AND_MASK, $Y, $T0
vpaddq $OVERFLOW, $ACC7, $ACC7
vpmuludq 32*7(%rcx), $B, $ACC8
vmovdqa 32*2($a_ptr), $B
vpaddq $T0, $ACC0, $OVERFLOW
vpaddq $T0, $ACC1, $ACC0
vpsrlq $digit_size, $OVERFLOW, $OVERFLOW
vpaddq $T0, $ACC2, $ACC1
vpmuludq 32*3(%rax), $Y, $T0
vpaddq $OVERFLOW, $ACC0, $ACC0
vpaddq $T0, $ACC3, $ACC2
vmovdqa $ACC4, $ACC3
vpsllq \$18, $Y, $T0
vmovdqa $ACC5, $ACC4
vpmuludq %ymm14, $Y, $OVERFLOW
vpaddq $T0, $ACC6, $ACC5
vpmuludq %ymm15, $Y, $T0
vpaddq $OVERFLOW, $ACC7, $ACC6
vpaddq $T0, $ACC8, $ACC7
#itr 3
vpmuludq $B, $B, $T0
vpand $AND_MASK, $ACC0, $Y
vpmuludq 32*2(%rcx), $B, $OVERFLOW
vpaddq $T0, $ACC2, $ACC2
vpmuludq 32*3(%rcx), $B, $T0
vpaddq $OVERFLOW, $ACC3, $ACC3
vpmuludq 32*4(%rcx), $B, $OVERFLOW
vpaddq $T0, $ACC4, $ACC4
vpmuludq 32*5(%rcx), $B, $T0
vpaddq $OVERFLOW, $ACC5, $ACC5
vpmuludq 32*6(%rcx), $B, $OVERFLOW
vpaddq $T0, $ACC6, $ACC6
vpmuludq $AND_MASK, $Y, $T0
vpaddq $OVERFLOW, $ACC7, $ACC7
vpmuludq 32*7(%rcx), $B, $ACC8
vmovdqa 32*3($a_ptr), $B
vpaddq $T0, $ACC0, $OVERFLOW
vpaddq $T0, $ACC1, $ACC0
vpsrlq $digit_size, $OVERFLOW, $OVERFLOW
vpaddq $T0, $ACC2, $ACC1
vpmuludq 32*3(%rax), $Y, $T0
vpaddq $OVERFLOW, $ACC0, $ACC0
vpaddq $T0, $ACC3, $ACC2
vmovdqa $ACC4, $ACC3
vpsllq \$18, $Y, $T0
vmovdqa $ACC5, $ACC4
vpmuludq %ymm14, $Y, $OVERFLOW
vpaddq $T0, $ACC6, $ACC5
vpmuludq %ymm15, $Y, $T0
vpand $AND_MASK, $ACC0, $Y
vpaddq $OVERFLOW, $ACC7, $ACC6
vpaddq $T0, $ACC8, $ACC7
#itr 4
vpmuludq $B, $B, $OVERFLOW
vpmuludq 32*3(%rcx), $B, $T0
vpaddq $OVERFLOW, $ACC3, $ACC3
vpmuludq 32*4(%rcx), $B, $OVERFLOW
vpaddq $T0, $ACC4, $ACC4
vpmuludq 32*5(%rcx), $B, $T0
vpaddq $OVERFLOW, $ACC5, $ACC5
vpmuludq 32*6(%rcx), $B, $OVERFLOW
vpaddq $T0, $ACC6, $ACC6
vpmuludq $AND_MASK, $Y, $T0
vpaddq $OVERFLOW, $ACC7, $ACC7
vpmuludq 32*7(%rcx), $B, $ACC8
vmovdqa 32*4($a_ptr), $B
vpaddq $T0, $ACC0, $OVERFLOW
vpaddq $T0, $ACC1, $ACC0
vpsrlq $digit_size, $OVERFLOW, $OVERFLOW
vpaddq $T0, $ACC2, $ACC1
vpmuludq 32*3(%rax), $Y, $T0
vpaddq $OVERFLOW, $ACC0, $ACC0
vpaddq $T0, $ACC3, $ACC2
vmovdqa $ACC4, $ACC3
vpsllq \$18, $Y, $T0
vmovdqa $ACC5, $ACC4
vpmuludq %ymm14, $Y, $OVERFLOW
vpaddq $T0, $ACC6, $ACC5
vpmuludq %ymm15, $Y, $T0
vpand $AND_MASK, $ACC0, $Y
vpaddq $OVERFLOW, $ACC7, $ACC6
vpaddq $T0, $ACC8, $ACC7
#itr 5
vpmuludq $B, $B, $T0
vpmuludq 32*4(%rcx), $B, $OVERFLOW
vpaddq $T0, $ACC4, $ACC4
vpmuludq 32*5(%rcx), $B, $T0
vpaddq $OVERFLOW, $ACC5, $ACC5
vpmuludq 32*6(%rcx), $B, $OVERFLOW
vpaddq $T0, $ACC6, $ACC6
vpmuludq $AND_MASK, $Y, $T0
vpaddq $OVERFLOW, $ACC7, $ACC7
vpmuludq 32*7(%rcx), $B, $ACC8
vmovdqa 32*5($a_ptr), $B
vpaddq $T0, $ACC0, $OVERFLOW
vpsrlq $digit_size, $OVERFLOW, $OVERFLOW
vpaddq $T0, $ACC1, $ACC0
vpaddq $T0, $ACC2, $ACC1
vpmuludq 32*3+.LAVX2_POLY(%rip), $Y, $T0
vpaddq $OVERFLOW, $ACC0, $ACC0
vpaddq $T0, $ACC3, $ACC2
vmovdqa $ACC4, $ACC3
vpsllq \$18, $Y, $T0
vmovdqa $ACC5, $ACC4
vpmuludq %ymm14, $Y, $OVERFLOW
vpaddq $T0, $ACC6, $ACC5
vpmuludq %ymm15, $Y, $T0
vpand $AND_MASK, $ACC0, $Y
vpaddq $OVERFLOW, $ACC7, $ACC6
vpaddq $T0, $ACC8, $ACC7
#itr 6
vpmuludq $B, $B, $OVERFLOW
vpmuludq 32*5(%rcx), $B, $T0
vpaddq $OVERFLOW, $ACC5, $ACC5
vpmuludq 32*6(%rcx), $B, $OVERFLOW
vpaddq $T0, $ACC6, $ACC6
vpmuludq $AND_MASK, $Y, $T0
vpaddq $OVERFLOW, $ACC7, $ACC7
vpmuludq 32*7(%rcx), $B, $ACC8
vmovdqa 32*6($a_ptr), $B
vpaddq $T0, $ACC0, $OVERFLOW
vpaddq $T0, $ACC1, $ACC0
vpsrlq $digit_size, $OVERFLOW, $OVERFLOW
vpaddq $T0, $ACC2, $ACC1
vpmuludq 32*3(%rax), $Y, $T0
vpaddq $OVERFLOW, $ACC0, $ACC0
vpaddq $T0, $ACC3, $ACC2
vmovdqa $ACC4, $ACC3
vpsllq \$18, $Y, $T0
vmovdqa $ACC5, $ACC4
vpmuludq %ymm14, $Y, $OVERFLOW
vpaddq $T0, $ACC6, $ACC5
vpmuludq %ymm15, $Y, $T0
vpand $AND_MASK, $ACC0, $Y
vpaddq $OVERFLOW, $ACC7, $ACC6
vpaddq $T0, $ACC8, $ACC7
#itr 7
vpmuludq $B, $B, $T0
vpmuludq 32*6(%rcx), $B, $OVERFLOW
vpaddq $T0, $ACC6, $ACC6
vpmuludq $AND_MASK, $Y, $T0
vpaddq $OVERFLOW, $ACC7, $ACC7
vpmuludq 32*7(%rcx), $B, $ACC8
vmovdqa 32*7($a_ptr), $B
vpaddq $T0, $ACC0, $OVERFLOW
vpsrlq $digit_size, $OVERFLOW, $OVERFLOW
vpaddq $T0, $ACC1, $ACC0
vpaddq $T0, $ACC2, $ACC1
vpmuludq 32*3(%rax), $Y, $T0
vpaddq $OVERFLOW, $ACC0, $ACC0
vpaddq $T0, $ACC3, $ACC2
vmovdqa $ACC4, $ACC3
vpsllq \$18, $Y, $T0
vmovdqa $ACC5, $ACC4
vpmuludq %ymm14, $Y, $OVERFLOW
vpaddq $T0, $ACC6, $ACC5
vpmuludq %ymm15, $Y, $T0
vpand $AND_MASK, $ACC0, $Y
vpaddq $OVERFLOW, $ACC7, $ACC6
vpaddq $T0, $ACC8, $ACC7
#itr 8
vpmuludq $B, $B, $OVERFLOW
vpmuludq $AND_MASK, $Y, $T0
vpaddq $OVERFLOW, $ACC7, $ACC7
vpmuludq 32*7(%rcx), $B, $ACC8
vmovdqa 32*8($a_ptr), $B
vpaddq $T0, $ACC0, $OVERFLOW
vpsrlq $digit_size, $OVERFLOW, $OVERFLOW
vpaddq $T0, $ACC1, $ACC0
vpaddq $T0, $ACC2, $ACC1
vpmuludq 32*3(%rax), $Y, $T0
vpaddq $OVERFLOW, $ACC0, $ACC0
vpaddq $T0, $ACC3, $ACC2
vmovdqa $ACC4, $ACC3
vpsllq \$18, $Y, $T0
vmovdqa $ACC5, $ACC4
vpmuludq %ymm14, $Y, $OVERFLOW
vpaddq $T0, $ACC6, $ACC5
vpmuludq %ymm15, $Y, $T0
vpand $AND_MASK, $ACC0, $Y
vpaddq $OVERFLOW, $ACC7, $ACC6
vpaddq $T0, $ACC8, $ACC7
#itr 9
vpmuludq $B, $B, $ACC8
vpmuludq $AND_MASK, $Y, $T0
vpaddq $T0, $ACC0, $OVERFLOW
vpsrlq $digit_size, $OVERFLOW, $OVERFLOW
vpaddq $T0, $ACC1, $ACC0
vpaddq $T0, $ACC2, $ACC1
vpmuludq 32*3(%rax), $Y, $T0
vpaddq $OVERFLOW, $ACC0, $ACC0
vpaddq $T0, $ACC3, $ACC2
vmovdqa $ACC4, $ACC3
vpsllq \$18, $Y, $T0
vmovdqa $ACC5, $ACC4
vpmuludq %ymm14, $Y, $OVERFLOW
vpaddq $T0, $ACC6, $ACC5
vpmuludq %ymm15, $Y, $T0
vpaddq $OVERFLOW, $ACC7, $ACC6
vpaddq $T0, $ACC8, $ACC7
vpxor $ACC8, $ACC8, $ACC8
ret
.size avx2_sqr_x4,.-avx2_sqr_x4
################################################################################
# void avx2_sub_x4(void* RESULTx4, void *Ax4, void *Bx4);
.type avx2_sub_x4,\@abi-omnipotent
.align 32
avx2_sub_x4:
vmovdqa 32*0($a_ptr), $ACC0
lea 160($a_ptr), $a_ptr
lea .LAVX2_POLY_x8+128(%rip), %rax
lea 128($b_ptr), $b_ptr
vmovdqa 32*1-160($a_ptr), $ACC1
vmovdqa 32*2-160($a_ptr), $ACC2
vmovdqa 32*3-160($a_ptr), $ACC3
vmovdqa 32*4-160($a_ptr), $ACC4
vmovdqa 32*5-160($a_ptr), $ACC5
vmovdqa 32*6-160($a_ptr), $ACC6
vmovdqa 32*7-160($a_ptr), $ACC7
vmovdqa 32*8-160($a_ptr), $ACC8
vpaddq 32*0-128(%rax), $ACC0, $ACC0
vpaddq 32*1-128(%rax), $ACC1, $ACC1
vpaddq 32*2-128(%rax), $ACC2, $ACC2
vpaddq 32*3-128(%rax), $ACC3, $ACC3
vpaddq 32*4-128(%rax), $ACC4, $ACC4
vpaddq 32*5-128(%rax), $ACC5, $ACC5
vpaddq 32*6-128(%rax), $ACC6, $ACC6
vpaddq 32*7-128(%rax), $ACC7, $ACC7
vpaddq 32*8-128(%rax), $ACC8, $ACC8
vpsubq 32*0-128($b_ptr), $ACC0, $ACC0
vpsubq 32*1-128($b_ptr), $ACC1, $ACC1
vpsubq 32*2-128($b_ptr), $ACC2, $ACC2
vpsubq 32*3-128($b_ptr), $ACC3, $ACC3
vpsubq 32*4-128($b_ptr), $ACC4, $ACC4
vpsubq 32*5-128($b_ptr), $ACC5, $ACC5
vpsubq 32*6-128($b_ptr), $ACC6, $ACC6
vpsubq 32*7-128($b_ptr), $ACC7, $ACC7
vpsubq 32*8-128($b_ptr), $ACC8, $ACC8
ret
.size avx2_sub_x4,.-avx2_sub_x4
.type avx2_select_n_store,\@abi-omnipotent
.align 32
avx2_select_n_store:
vmovdqa `8+32*9*8`(%rsp), $Y
vpor `8+32*9*8+32`(%rsp), $Y, $Y
vpandn $ACC0, $Y, $ACC0
vpandn $ACC1, $Y, $ACC1
vpandn $ACC2, $Y, $ACC2
vpandn $ACC3, $Y, $ACC3
vpandn $ACC4, $Y, $ACC4
vpandn $ACC5, $Y, $ACC5
vpandn $ACC6, $Y, $ACC6
vmovdqa `8+32*9*8+32`(%rsp), $B
vpandn $ACC7, $Y, $ACC7
vpandn `8+32*9*8`(%rsp), $B, $B
vpandn $ACC8, $Y, $ACC8
vpand 32*0(%rsi), $B, $T0
lea 160(%rsi), %rax
vpand 32*1(%rsi), $B, $Y
vpxor $T0, $ACC0, $ACC0
vpand 32*2(%rsi), $B, $T0
vpxor $Y, $ACC1, $ACC1
vpand 32*3(%rsi), $B, $Y
vpxor $T0, $ACC2, $ACC2
vpand 32*4-160(%rax), $B, $T0
vpxor $Y, $ACC3, $ACC3
vpand 32*5-160(%rax), $B, $Y
vpxor $T0, $ACC4, $ACC4
vpand 32*6-160(%rax), $B, $T0
vpxor $Y, $ACC5, $ACC5
vpand 32*7-160(%rax), $B, $Y
vpxor $T0, $ACC6, $ACC6
vpand 32*8-160(%rax), $B, $T0
vmovdqa `8+32*9*8+32`(%rsp), $B
vpxor $Y, $ACC7, $ACC7
vpand 32*0(%rdx), $B, $Y
lea 160(%rdx), %rax
vpxor $T0, $ACC8, $ACC8
vpand 32*1(%rdx), $B, $T0
vpxor $Y, $ACC0, $ACC0
vpand 32*2(%rdx), $B, $Y
vpxor $T0, $ACC1, $ACC1
vpand 32*3(%rdx), $B, $T0
vpxor $Y, $ACC2, $ACC2
vpand 32*4-160(%rax), $B, $Y
vpxor $T0, $ACC3, $ACC3
vpand 32*5-160(%rax), $B, $T0
vpxor $Y, $ACC4, $ACC4
vpand 32*6-160(%rax), $B, $Y
vpxor $T0, $ACC5, $ACC5
vpand 32*7-160(%rax), $B, $T0
vpxor $Y, $ACC6, $ACC6
vpand 32*8-160(%rax), $B, $Y
vpxor $T0, $ACC7, $ACC7
vpxor $Y, $ACC8, $ACC8
`&STORE`
ret
.size avx2_select_n_store,.-avx2_select_n_store
___
$code.=<<___ if (0); # inlined
################################################################################
# void avx2_mul_by2_x4(void* RESULTx4, void *Ax4);
.type avx2_mul_by2_x4,\@abi-omnipotent
.align 32
avx2_mul_by2_x4:
vmovdqa 32*0($a_ptr), $ACC0
lea 160($a_ptr), %rax
vmovdqa 32*1($a_ptr), $ACC1
vmovdqa 32*2($a_ptr), $ACC2
vmovdqa 32*3($a_ptr), $ACC3
vmovdqa 32*4-160(%rax), $ACC4
vmovdqa 32*5-160(%rax), $ACC5
vmovdqa 32*6-160(%rax), $ACC6
vmovdqa 32*7-160(%rax), $ACC7
vmovdqa 32*8-160(%rax), $ACC8
vpaddq $ACC0, $ACC0, $ACC0
vpaddq $ACC1, $ACC1, $ACC1
vpaddq $ACC2, $ACC2, $ACC2
vpaddq $ACC3, $ACC3, $ACC3
vpaddq $ACC4, $ACC4, $ACC4
vpaddq $ACC5, $ACC5, $ACC5
vpaddq $ACC6, $ACC6, $ACC6
vpaddq $ACC7, $ACC7, $ACC7
vpaddq $ACC8, $ACC8, $ACC8
ret
.size avx2_mul_by2_x4,.-avx2_mul_by2_x4
___
my ($r_ptr_in,$a_ptr_in,$b_ptr_in)=("%rdi","%rsi","%rdx");
my ($r_ptr,$a_ptr,$b_ptr)=("%r8","%r9","%r10");
$code.=<<___;
################################################################################
# void ecp_nistz256_avx2_point_add_affine_x4(void* RESULTx4, void *Ax4, void *Bx4);
.globl ecp_nistz256_avx2_point_add_affine_x4
.type ecp_nistz256_avx2_point_add_affine_x4,\@function,3
.align 32
ecp_nistz256_avx2_point_add_affine_x4:
mov %rsp, %rax
push %rbp
vzeroupper
___
$code.=<<___ if ($win64);
lea -16*10(%rsp), %rsp
vmovaps %xmm6, -8-16*10(%rax)
vmovaps %xmm7, -8-16*9(%rax)
vmovaps %xmm8, -8-16*8(%rax)
vmovaps %xmm9, -8-16*7(%rax)
vmovaps %xmm10, -8-16*6(%rax)
vmovaps %xmm11, -8-16*5(%rax)
vmovaps %xmm12, -8-16*4(%rax)
vmovaps %xmm13, -8-16*3(%rax)
vmovaps %xmm14, -8-16*2(%rax)
vmovaps %xmm15, -8-16*1(%rax)
___
$code.=<<___;
lea -8(%rax), %rbp
# Result + 32*0 = Result.X
# Result + 32*9 = Result.Y
# Result + 32*18 = Result.Z
# A + 32*0 = A.X
# A + 32*9 = A.Y
# A + 32*18 = A.Z
# B + 32*0 = B.X
# B + 32*9 = B.Y
sub \$`32*9*8+32*2+32*8`, %rsp
and \$-64, %rsp
mov $r_ptr_in, $r_ptr
mov $a_ptr_in, $a_ptr
mov $b_ptr_in, $b_ptr
vmovdqa 32*0($a_ptr_in), %ymm0
vmovdqa .LAVX2_AND_MASK(%rip), $AND_MASK
vpxor %ymm1, %ymm1, %ymm1
lea 256($a_ptr_in), %rax # size optimization
vpor 32*1($a_ptr_in), %ymm0, %ymm0
vpor 32*2($a_ptr_in), %ymm0, %ymm0
vpor 32*3($a_ptr_in), %ymm0, %ymm0
vpor 32*4-256(%rax), %ymm0, %ymm0
lea 256(%rax), %rcx # size optimization
vpor 32*5-256(%rax), %ymm0, %ymm0
vpor 32*6-256(%rax), %ymm0, %ymm0
vpor 32*7-256(%rax), %ymm0, %ymm0
vpor 32*8-256(%rax), %ymm0, %ymm0
vpor 32*9-256(%rax), %ymm0, %ymm0
vpor 32*10-256(%rax), %ymm0, %ymm0
vpor 32*11-256(%rax), %ymm0, %ymm0
vpor 32*12-512(%rcx), %ymm0, %ymm0
vpor 32*13-512(%rcx), %ymm0, %ymm0
vpor 32*14-512(%rcx), %ymm0, %ymm0
vpor 32*15-512(%rcx), %ymm0, %ymm0
vpor 32*16-512(%rcx), %ymm0, %ymm0
vpor 32*17-512(%rcx), %ymm0, %ymm0
vpcmpeqq %ymm1, %ymm0, %ymm0
vmovdqa %ymm0, `32*9*8`(%rsp)
vpxor %ymm1, %ymm1, %ymm1
vmovdqa 32*0($b_ptr), %ymm0
lea 256($b_ptr), %rax # size optimization
vpor 32*1($b_ptr), %ymm0, %ymm0
vpor 32*2($b_ptr), %ymm0, %ymm0
vpor 32*3($b_ptr), %ymm0, %ymm0
vpor 32*4-256(%rax), %ymm0, %ymm0
lea 256(%rax), %rcx # size optimization
vpor 32*5-256(%rax), %ymm0, %ymm0
vpor 32*6-256(%rax), %ymm0, %ymm0
vpor 32*7-256(%rax), %ymm0, %ymm0
vpor 32*8-256(%rax), %ymm0, %ymm0
vpor 32*9-256(%rax), %ymm0, %ymm0
vpor 32*10-256(%rax), %ymm0, %ymm0
vpor 32*11-256(%rax), %ymm0, %ymm0
vpor 32*12-512(%rcx), %ymm0, %ymm0
vpor 32*13-512(%rcx), %ymm0, %ymm0
vpor 32*14-512(%rcx), %ymm0, %ymm0
vpor 32*15-512(%rcx), %ymm0, %ymm0
vpor 32*16-512(%rcx), %ymm0, %ymm0
vpor 32*17-512(%rcx), %ymm0, %ymm0
vpcmpeqq %ymm1, %ymm0, %ymm0
vmovdqa %ymm0, `32*9*8+32`(%rsp)
# Z1^2 = Z1*Z1
lea `32*9*2`($a_ptr), %rsi
lea `32*9*2`(%rsp), %rdi
lea `32*9*8+32*2`(%rsp), %rcx # temporary vector
call avx2_sqr_x4
call avx2_normalize_n_store
# U2 = X2*Z1^2
lea `32*9*0`($b_ptr), %rsi
lea `32*9*2`(%rsp), %rdx
lea `32*9*0`(%rsp), %rdi
call avx2_mul_x4
#call avx2_normalize
`&STORE`
# S2 = Z1*Z1^2 = Z1^3
lea `32*9*2`($a_ptr), %rsi
lea `32*9*2`(%rsp), %rdx
lea `32*9*1`(%rsp), %rdi
call avx2_mul_x4
call avx2_normalize_n_store
# S2 = S2*Y2 = Y2*Z1^3
lea `32*9*1`($b_ptr), %rsi
lea `32*9*1`(%rsp), %rdx
lea `32*9*1`(%rsp), %rdi
call avx2_mul_x4
call avx2_normalize_n_store
# H = U2 - U1 = U2 - X1
lea `32*9*0`(%rsp), %rsi
lea `32*9*0`($a_ptr), %rdx
lea `32*9*3`(%rsp), %rdi
call avx2_sub_x4
call avx2_normalize_n_store
# R = S2 - S1 = S2 - Y1
lea `32*9*1`(%rsp), %rsi
lea `32*9*1`($a_ptr), %rdx
lea `32*9*4`(%rsp), %rdi
call avx2_sub_x4
call avx2_normalize_n_store
# Z3 = H*Z1*Z2
lea `32*9*3`(%rsp), %rsi
lea `32*9*2`($a_ptr), %rdx
lea `32*9*2`($r_ptr), %rdi
call avx2_mul_x4
call avx2_normalize
lea .LONE(%rip), %rsi
lea `32*9*2`($a_ptr), %rdx
call avx2_select_n_store
# R^2 = R^2
lea `32*9*4`(%rsp), %rsi
lea `32*9*6`(%rsp), %rdi
lea `32*9*8+32*2`(%rsp), %rcx # temporary vector
call avx2_sqr_x4
call avx2_normalize_n_store
# H^2 = H^2
lea `32*9*3`(%rsp), %rsi
lea `32*9*5`(%rsp), %rdi
call avx2_sqr_x4
call avx2_normalize_n_store
# H^3 = H^2*H
lea `32*9*3`(%rsp), %rsi
lea `32*9*5`(%rsp), %rdx
lea `32*9*7`(%rsp), %rdi
call avx2_mul_x4
call avx2_normalize_n_store
# U2 = U1*H^2
lea `32*9*0`($a_ptr), %rsi
lea `32*9*5`(%rsp), %rdx
lea `32*9*0`(%rsp), %rdi
call avx2_mul_x4
#call avx2_normalize
`&STORE`
# Hsqr = U2*2
#lea 32*9*0(%rsp), %rsi
#lea 32*9*5(%rsp), %rdi
#call avx2_mul_by2_x4
vpaddq $ACC0, $ACC0, $ACC0 # inlined avx2_mul_by2_x4
lea `32*9*5`(%rsp), %rdi
vpaddq $ACC1, $ACC1, $ACC1
vpaddq $ACC2, $ACC2, $ACC2
vpaddq $ACC3, $ACC3, $ACC3
vpaddq $ACC4, $ACC4, $ACC4
vpaddq $ACC5, $ACC5, $ACC5
vpaddq $ACC6, $ACC6, $ACC6
vpaddq $ACC7, $ACC7, $ACC7
vpaddq $ACC8, $ACC8, $ACC8
call avx2_normalize_n_store
# X3 = R^2 - H^3
#lea 32*9*6(%rsp), %rsi
#lea 32*9*7(%rsp), %rdx
#lea 32*9*5(%rsp), %rcx
#lea 32*9*0($r_ptr), %rdi
#call avx2_sub_x4
#NORMALIZE
#STORE
# X3 = X3 - U2*2
#lea 32*9*0($r_ptr), %rsi
#lea 32*9*0($r_ptr), %rdi
#call avx2_sub_x4
#NORMALIZE
#STORE
lea `32*9*6+128`(%rsp), %rsi
lea .LAVX2_POLY_x2+128(%rip), %rax
lea `32*9*7+128`(%rsp), %rdx
lea `32*9*5+128`(%rsp), %rcx
lea `32*9*0`($r_ptr), %rdi
vmovdqa 32*0-128(%rsi), $ACC0
vmovdqa 32*1-128(%rsi), $ACC1
vmovdqa 32*2-128(%rsi), $ACC2
vmovdqa 32*3-128(%rsi), $ACC3
vmovdqa 32*4-128(%rsi), $ACC4
vmovdqa 32*5-128(%rsi), $ACC5
vmovdqa 32*6-128(%rsi), $ACC6
vmovdqa 32*7-128(%rsi), $ACC7
vmovdqa 32*8-128(%rsi), $ACC8
vpaddq 32*0-128(%rax), $ACC0, $ACC0
vpaddq 32*1-128(%rax), $ACC1, $ACC1
vpaddq 32*2-128(%rax), $ACC2, $ACC2
vpaddq 32*3-128(%rax), $ACC3, $ACC3
vpaddq 32*4-128(%rax), $ACC4, $ACC4
vpaddq 32*5-128(%rax), $ACC5, $ACC5
vpaddq 32*6-128(%rax), $ACC6, $ACC6
vpaddq 32*7-128(%rax), $ACC7, $ACC7
vpaddq 32*8-128(%rax), $ACC8, $ACC8
vpsubq 32*0-128(%rdx), $ACC0, $ACC0
vpsubq 32*1-128(%rdx), $ACC1, $ACC1
vpsubq 32*2-128(%rdx), $ACC2, $ACC2
vpsubq 32*3-128(%rdx), $ACC3, $ACC3
vpsubq 32*4-128(%rdx), $ACC4, $ACC4
vpsubq 32*5-128(%rdx), $ACC5, $ACC5
vpsubq 32*6-128(%rdx), $ACC6, $ACC6
vpsubq 32*7-128(%rdx), $ACC7, $ACC7
vpsubq 32*8-128(%rdx), $ACC8, $ACC8
vpsubq 32*0-128(%rcx), $ACC0, $ACC0
vpsubq 32*1-128(%rcx), $ACC1, $ACC1
vpsubq 32*2-128(%rcx), $ACC2, $ACC2
vpsubq 32*3-128(%rcx), $ACC3, $ACC3
vpsubq 32*4-128(%rcx), $ACC4, $ACC4
vpsubq 32*5-128(%rcx), $ACC5, $ACC5
vpsubq 32*6-128(%rcx), $ACC6, $ACC6
vpsubq 32*7-128(%rcx), $ACC7, $ACC7
vpsubq 32*8-128(%rcx), $ACC8, $ACC8
call avx2_normalize
lea 32*0($b_ptr), %rsi
lea 32*0($a_ptr), %rdx
call avx2_select_n_store
# H = U2 - X3
lea `32*9*0`(%rsp), %rsi
lea `32*9*0`($r_ptr), %rdx
lea `32*9*3`(%rsp), %rdi
call avx2_sub_x4
call avx2_normalize_n_store
#
lea `32*9*3`(%rsp), %rsi
lea `32*9*4`(%rsp), %rdx
lea `32*9*3`(%rsp), %rdi
call avx2_mul_x4
call avx2_normalize_n_store
#
lea `32*9*7`(%rsp), %rsi
lea `32*9*1`($a_ptr), %rdx
lea `32*9*1`(%rsp), %rdi
call avx2_mul_x4
call avx2_normalize_n_store
#
lea `32*9*3`(%rsp), %rsi
lea `32*9*1`(%rsp), %rdx
lea `32*9*1`($r_ptr), %rdi
call avx2_sub_x4
call avx2_normalize
lea 32*9($b_ptr), %rsi
lea 32*9($a_ptr), %rdx
call avx2_select_n_store
#lea 32*9*0($r_ptr), %rsi
#lea 32*9*0($r_ptr), %rdi
#call avx2_mul_by1_x4
#NORMALIZE
#STORE
lea `32*9*1`($r_ptr), %rsi
lea `32*9*1`($r_ptr), %rdi
call avx2_mul_by1_x4
call avx2_normalize_n_store
vzeroupper
___
$code.=<<___ if ($win64);
movaps %xmm6, -16*10(%rbp)
movaps %xmm7, -16*9(%rbp)
movaps %xmm8, -16*8(%rbp)
movaps %xmm9, -16*7(%rbp)
movaps %xmm10, -16*6(%rbp)
movaps %xmm11, -16*5(%rbp)
movaps %xmm12, -16*4(%rbp)
movaps %xmm13, -16*3(%rbp)
movaps %xmm14, -16*2(%rbp)
movaps %xmm15, -16*1(%rbp)
___
$code.=<<___;
mov %rbp, %rsp
pop %rbp
ret
.size ecp_nistz256_avx2_point_add_affine_x4,.-ecp_nistz256_avx2_point_add_affine_x4
################################################################################
# void ecp_nistz256_avx2_point_add_affines_x4(void* RESULTx4, void *Ax4, void *Bx4);
.globl ecp_nistz256_avx2_point_add_affines_x4
.type ecp_nistz256_avx2_point_add_affines_x4,\@function,3
.align 32
ecp_nistz256_avx2_point_add_affines_x4:
mov %rsp, %rax
push %rbp
vzeroupper
___
$code.=<<___ if ($win64);
lea -16*10(%rsp), %rsp
vmovaps %xmm6, -8-16*10(%rax)
vmovaps %xmm7, -8-16*9(%rax)
vmovaps %xmm8, -8-16*8(%rax)
vmovaps %xmm9, -8-16*7(%rax)
vmovaps %xmm10, -8-16*6(%rax)
vmovaps %xmm11, -8-16*5(%rax)
vmovaps %xmm12, -8-16*4(%rax)
vmovaps %xmm13, -8-16*3(%rax)
vmovaps %xmm14, -8-16*2(%rax)
vmovaps %xmm15, -8-16*1(%rax)
___
$code.=<<___;
lea -8(%rax), %rbp
# Result + 32*0 = Result.X
# Result + 32*9 = Result.Y
# Result + 32*18 = Result.Z
# A + 32*0 = A.X
# A + 32*9 = A.Y
# B + 32*0 = B.X
# B + 32*9 = B.Y
sub \$`32*9*8+32*2+32*8`, %rsp
and \$-64, %rsp
mov $r_ptr_in, $r_ptr
mov $a_ptr_in, $a_ptr
mov $b_ptr_in, $b_ptr
vmovdqa 32*0($a_ptr_in), %ymm0
vmovdqa .LAVX2_AND_MASK(%rip), $AND_MASK
vpxor %ymm1, %ymm1, %ymm1
lea 256($a_ptr_in), %rax # size optimization
vpor 32*1($a_ptr_in), %ymm0, %ymm0
vpor 32*2($a_ptr_in), %ymm0, %ymm0
vpor 32*3($a_ptr_in), %ymm0, %ymm0
vpor 32*4-256(%rax), %ymm0, %ymm0
lea 256(%rax), %rcx # size optimization
vpor 32*5-256(%rax), %ymm0, %ymm0
vpor 32*6-256(%rax), %ymm0, %ymm0
vpor 32*7-256(%rax), %ymm0, %ymm0
vpor 32*8-256(%rax), %ymm0, %ymm0
vpor 32*9-256(%rax), %ymm0, %ymm0
vpor 32*10-256(%rax), %ymm0, %ymm0
vpor 32*11-256(%rax), %ymm0, %ymm0
vpor 32*12-512(%rcx), %ymm0, %ymm0
vpor 32*13-512(%rcx), %ymm0, %ymm0
vpor 32*14-512(%rcx), %ymm0, %ymm0
vpor 32*15-512(%rcx), %ymm0, %ymm0
vpor 32*16-512(%rcx), %ymm0, %ymm0
vpor 32*17-512(%rcx), %ymm0, %ymm0
vpcmpeqq %ymm1, %ymm0, %ymm0
vmovdqa %ymm0, `32*9*8`(%rsp)
vpxor %ymm1, %ymm1, %ymm1
vmovdqa 32*0($b_ptr), %ymm0
lea 256($b_ptr), %rax # size optimization
vpor 32*1($b_ptr), %ymm0, %ymm0
vpor 32*2($b_ptr), %ymm0, %ymm0
vpor 32*3($b_ptr), %ymm0, %ymm0
vpor 32*4-256(%rax), %ymm0, %ymm0
lea 256(%rax), %rcx # size optimization
vpor 32*5-256(%rax), %ymm0, %ymm0
vpor 32*6-256(%rax), %ymm0, %ymm0
vpor 32*7-256(%rax), %ymm0, %ymm0
vpor 32*8-256(%rax), %ymm0, %ymm0
vpor 32*9-256(%rax), %ymm0, %ymm0
vpor 32*10-256(%rax), %ymm0, %ymm0
vpor 32*11-256(%rax), %ymm0, %ymm0
vpor 32*12-512(%rcx), %ymm0, %ymm0
vpor 32*13-512(%rcx), %ymm0, %ymm0
vpor 32*14-512(%rcx), %ymm0, %ymm0
vpor 32*15-512(%rcx), %ymm0, %ymm0
vpor 32*16-512(%rcx), %ymm0, %ymm0
vpor 32*17-512(%rcx), %ymm0, %ymm0
vpcmpeqq %ymm1, %ymm0, %ymm0
vmovdqa %ymm0, `32*9*8+32`(%rsp)
# H = U2 - U1 = X2 - X1
lea `32*9*0`($b_ptr), %rsi
lea `32*9*0`($a_ptr), %rdx
lea `32*9*3`(%rsp), %rdi
call avx2_sub_x4
call avx2_normalize_n_store
# R = S2 - S1 = Y2 - Y1
lea `32*9*1`($b_ptr), %rsi
lea `32*9*1`($a_ptr), %rdx
lea `32*9*4`(%rsp), %rdi
call avx2_sub_x4
call avx2_normalize_n_store
# Z3 = H*Z1*Z2 = H
lea `32*9*3`(%rsp), %rsi
lea `32*9*2`($r_ptr), %rdi
call avx2_mul_by1_x4
call avx2_normalize
vmovdqa `32*9*8`(%rsp), $B
vpor `32*9*8+32`(%rsp), $B, $B
vpandn $ACC0, $B, $ACC0
lea .LONE+128(%rip), %rax
vpandn $ACC1, $B, $ACC1
vpandn $ACC2, $B, $ACC2
vpandn $ACC3, $B, $ACC3
vpandn $ACC4, $B, $ACC4
vpandn $ACC5, $B, $ACC5
vpandn $ACC6, $B, $ACC6
vpandn $ACC7, $B, $ACC7
vpand 32*0-128(%rax), $B, $T0
vpandn $ACC8, $B, $ACC8
vpand 32*1-128(%rax), $B, $Y
vpxor $T0, $ACC0, $ACC0
vpand 32*2-128(%rax), $B, $T0
vpxor $Y, $ACC1, $ACC1
vpand 32*3-128(%rax), $B, $Y
vpxor $T0, $ACC2, $ACC2
vpand 32*4-128(%rax), $B, $T0
vpxor $Y, $ACC3, $ACC3
vpand 32*5-128(%rax), $B, $Y
vpxor $T0, $ACC4, $ACC4
vpand 32*6-128(%rax), $B, $T0
vpxor $Y, $ACC5, $ACC5
vpand 32*7-128(%rax), $B, $Y
vpxor $T0, $ACC6, $ACC6
vpand 32*8-128(%rax), $B, $T0
vpxor $Y, $ACC7, $ACC7
vpxor $T0, $ACC8, $ACC8
`&STORE`
# R^2 = R^2
lea `32*9*4`(%rsp), %rsi
lea `32*9*6`(%rsp), %rdi
lea `32*9*8+32*2`(%rsp), %rcx # temporary vector
call avx2_sqr_x4
call avx2_normalize_n_store
# H^2 = H^2
lea `32*9*3`(%rsp), %rsi
lea `32*9*5`(%rsp), %rdi
call avx2_sqr_x4
call avx2_normalize_n_store
# H^3 = H^2*H
lea `32*9*3`(%rsp), %rsi
lea `32*9*5`(%rsp), %rdx
lea `32*9*7`(%rsp), %rdi
call avx2_mul_x4
call avx2_normalize_n_store
# U2 = U1*H^2
lea `32*9*0`($a_ptr), %rsi
lea `32*9*5`(%rsp), %rdx
lea `32*9*0`(%rsp), %rdi
call avx2_mul_x4
#call avx2_normalize
`&STORE`
# Hsqr = U2*2
#lea 32*9*0(%rsp), %rsi
#lea 32*9*5(%rsp), %rdi
#call avx2_mul_by2_x4
vpaddq $ACC0, $ACC0, $ACC0 # inlined avx2_mul_by2_x4
lea `32*9*5`(%rsp), %rdi
vpaddq $ACC1, $ACC1, $ACC1
vpaddq $ACC2, $ACC2, $ACC2
vpaddq $ACC3, $ACC3, $ACC3
vpaddq $ACC4, $ACC4, $ACC4
vpaddq $ACC5, $ACC5, $ACC5
vpaddq $ACC6, $ACC6, $ACC6
vpaddq $ACC7, $ACC7, $ACC7
vpaddq $ACC8, $ACC8, $ACC8
call avx2_normalize_n_store
# X3 = R^2 - H^3
#lea 32*9*6(%rsp), %rsi
#lea 32*9*7(%rsp), %rdx
#lea 32*9*5(%rsp), %rcx
#lea 32*9*0($r_ptr), %rdi
#call avx2_sub_x4
#NORMALIZE
#STORE
# X3 = X3 - U2*2
#lea 32*9*0($r_ptr), %rsi
#lea 32*9*0($r_ptr), %rdi
#call avx2_sub_x4
#NORMALIZE
#STORE
lea `32*9*6+128`(%rsp), %rsi
lea .LAVX2_POLY_x2+128(%rip), %rax
lea `32*9*7+128`(%rsp), %rdx
lea `32*9*5+128`(%rsp), %rcx
lea `32*9*0`($r_ptr), %rdi
vmovdqa 32*0-128(%rsi), $ACC0
vmovdqa 32*1-128(%rsi), $ACC1
vmovdqa 32*2-128(%rsi), $ACC2
vmovdqa 32*3-128(%rsi), $ACC3
vmovdqa 32*4-128(%rsi), $ACC4
vmovdqa 32*5-128(%rsi), $ACC5
vmovdqa 32*6-128(%rsi), $ACC6
vmovdqa 32*7-128(%rsi), $ACC7
vmovdqa 32*8-128(%rsi), $ACC8
vpaddq 32*0-128(%rax), $ACC0, $ACC0
vpaddq 32*1-128(%rax), $ACC1, $ACC1
vpaddq 32*2-128(%rax), $ACC2, $ACC2
vpaddq 32*3-128(%rax), $ACC3, $ACC3
vpaddq 32*4-128(%rax), $ACC4, $ACC4
vpaddq 32*5-128(%rax), $ACC5, $ACC5
vpaddq 32*6-128(%rax), $ACC6, $ACC6
vpaddq 32*7-128(%rax), $ACC7, $ACC7
vpaddq 32*8-128(%rax), $ACC8, $ACC8
vpsubq 32*0-128(%rdx), $ACC0, $ACC0
vpsubq 32*1-128(%rdx), $ACC1, $ACC1
vpsubq 32*2-128(%rdx), $ACC2, $ACC2
vpsubq 32*3-128(%rdx), $ACC3, $ACC3
vpsubq 32*4-128(%rdx), $ACC4, $ACC4
vpsubq 32*5-128(%rdx), $ACC5, $ACC5
vpsubq 32*6-128(%rdx), $ACC6, $ACC6
vpsubq 32*7-128(%rdx), $ACC7, $ACC7
vpsubq 32*8-128(%rdx), $ACC8, $ACC8
vpsubq 32*0-128(%rcx), $ACC0, $ACC0
vpsubq 32*1-128(%rcx), $ACC1, $ACC1
vpsubq 32*2-128(%rcx), $ACC2, $ACC2
vpsubq 32*3-128(%rcx), $ACC3, $ACC3
vpsubq 32*4-128(%rcx), $ACC4, $ACC4
vpsubq 32*5-128(%rcx), $ACC5, $ACC5
vpsubq 32*6-128(%rcx), $ACC6, $ACC6
vpsubq 32*7-128(%rcx), $ACC7, $ACC7
vpsubq 32*8-128(%rcx), $ACC8, $ACC8
call avx2_normalize
lea 32*0($b_ptr), %rsi
lea 32*0($a_ptr), %rdx
call avx2_select_n_store
# H = U2 - X3
lea `32*9*0`(%rsp), %rsi
lea `32*9*0`($r_ptr), %rdx
lea `32*9*3`(%rsp), %rdi
call avx2_sub_x4
call avx2_normalize_n_store
# H = H*R
lea `32*9*3`(%rsp), %rsi
lea `32*9*4`(%rsp), %rdx
lea `32*9*3`(%rsp), %rdi
call avx2_mul_x4
call avx2_normalize_n_store
# S2 = S1 * H^3
lea `32*9*7`(%rsp), %rsi
lea `32*9*1`($a_ptr), %rdx
lea `32*9*1`(%rsp), %rdi
call avx2_mul_x4
call avx2_normalize_n_store
#
lea `32*9*3`(%rsp), %rsi
lea `32*9*1`(%rsp), %rdx
lea `32*9*1`($r_ptr), %rdi
call avx2_sub_x4
call avx2_normalize
lea 32*9($b_ptr), %rsi
lea 32*9($a_ptr), %rdx
call avx2_select_n_store
#lea 32*9*0($r_ptr), %rsi
#lea 32*9*0($r_ptr), %rdi
#call avx2_mul_by1_x4
#NORMALIZE
#STORE
lea `32*9*1`($r_ptr), %rsi
lea `32*9*1`($r_ptr), %rdi
call avx2_mul_by1_x4
call avx2_normalize_n_store
vzeroupper
___
$code.=<<___ if ($win64);
movaps %xmm6, -16*10(%rbp)
movaps %xmm7, -16*9(%rbp)
movaps %xmm8, -16*8(%rbp)
movaps %xmm9, -16*7(%rbp)
movaps %xmm10, -16*6(%rbp)
movaps %xmm11, -16*5(%rbp)
movaps %xmm12, -16*4(%rbp)
movaps %xmm13, -16*3(%rbp)
movaps %xmm14, -16*2(%rbp)
movaps %xmm15, -16*1(%rbp)
___
$code.=<<___;
mov %rbp, %rsp
pop %rbp
ret
.size ecp_nistz256_avx2_point_add_affines_x4,.-ecp_nistz256_avx2_point_add_affines_x4
################################################################################
# void ecp_nistz256_avx2_to_mont(void* RESULTx4, void *Ax4);
.globl ecp_nistz256_avx2_to_mont
.type ecp_nistz256_avx2_to_mont,\@function,2
.align 32
ecp_nistz256_avx2_to_mont:
vzeroupper
___
$code.=<<___ if ($win64);
lea -8-16*10(%rsp), %rsp
vmovaps %xmm6, -8-16*10(%rax)
vmovaps %xmm7, -8-16*9(%rax)
vmovaps %xmm8, -8-16*8(%rax)
vmovaps %xmm9, -8-16*7(%rax)
vmovaps %xmm10, -8-16*6(%rax)
vmovaps %xmm11, -8-16*5(%rax)
vmovaps %xmm12, -8-16*4(%rax)
vmovaps %xmm13, -8-16*3(%rax)
vmovaps %xmm14, -8-16*2(%rax)
vmovaps %xmm15, -8-16*1(%rax)
___
$code.=<<___;
vmovdqa .LAVX2_AND_MASK(%rip), $AND_MASK
lea .LTO_MONT_AVX2(%rip), %rdx
call avx2_mul_x4
call avx2_normalize_n_store
vzeroupper
___
$code.=<<___ if ($win64);
movaps 16*0(%rsp), %xmm6
movaps 16*1(%rsp), %xmm7
movaps 16*2(%rsp), %xmm8
movaps 16*3(%rsp), %xmm9
movaps 16*4(%rsp), %xmm10
movaps 16*5(%rsp), %xmm11
movaps 16*6(%rsp), %xmm12
movaps 16*7(%rsp), %xmm13
movaps 16*8(%rsp), %xmm14
movaps 16*9(%rsp), %xmm15
lea 8+16*10(%rsp), %rsp
___
$code.=<<___;
ret
.size ecp_nistz256_avx2_to_mont,.-ecp_nistz256_avx2_to_mont
################################################################################
# void ecp_nistz256_avx2_from_mont(void* RESULTx4, void *Ax4);
.globl ecp_nistz256_avx2_from_mont
.type ecp_nistz256_avx2_from_mont,\@function,2
.align 32
ecp_nistz256_avx2_from_mont:
vzeroupper
___
$code.=<<___ if ($win64);
lea -8-16*10(%rsp), %rsp
vmovaps %xmm6, -8-16*10(%rax)
vmovaps %xmm7, -8-16*9(%rax)
vmovaps %xmm8, -8-16*8(%rax)
vmovaps %xmm9, -8-16*7(%rax)
vmovaps %xmm10, -8-16*6(%rax)
vmovaps %xmm11, -8-16*5(%rax)
vmovaps %xmm12, -8-16*4(%rax)
vmovaps %xmm13, -8-16*3(%rax)
vmovaps %xmm14, -8-16*2(%rax)
vmovaps %xmm15, -8-16*1(%rax)
___
$code.=<<___;
vmovdqa .LAVX2_AND_MASK(%rip), $AND_MASK
lea .LFROM_MONT_AVX2(%rip), %rdx
call avx2_mul_x4
call avx2_normalize_n_store
vzeroupper
___
$code.=<<___ if ($win64);
movaps 16*0(%rsp), %xmm6
movaps 16*1(%rsp), %xmm7
movaps 16*2(%rsp), %xmm8
movaps 16*3(%rsp), %xmm9
movaps 16*4(%rsp), %xmm10
movaps 16*5(%rsp), %xmm11
movaps 16*6(%rsp), %xmm12
movaps 16*7(%rsp), %xmm13
movaps 16*8(%rsp), %xmm14
movaps 16*9(%rsp), %xmm15
lea 8+16*10(%rsp), %rsp
___
$code.=<<___;
ret
.size ecp_nistz256_avx2_from_mont,.-ecp_nistz256_avx2_from_mont
################################################################################
# void ecp_nistz256_avx2_set1(void* RESULTx4);
.globl ecp_nistz256_avx2_set1
.type ecp_nistz256_avx2_set1,\@function,1
.align 32
ecp_nistz256_avx2_set1:
lea .LONE+128(%rip), %rax
lea 128(%rdi), %rdi
vzeroupper
vmovdqa 32*0-128(%rax), %ymm0
vmovdqa 32*1-128(%rax), %ymm1
vmovdqa 32*2-128(%rax), %ymm2
vmovdqa 32*3-128(%rax), %ymm3
vmovdqa 32*4-128(%rax), %ymm4
vmovdqa 32*5-128(%rax), %ymm5
vmovdqa %ymm0, 32*0-128(%rdi)
vmovdqa 32*6-128(%rax), %ymm0
vmovdqa %ymm1, 32*1-128(%rdi)
vmovdqa 32*7-128(%rax), %ymm1
vmovdqa %ymm2, 32*2-128(%rdi)
vmovdqa 32*8-128(%rax), %ymm2
vmovdqa %ymm3, 32*3-128(%rdi)
vmovdqa %ymm4, 32*4-128(%rdi)
vmovdqa %ymm5, 32*5-128(%rdi)
vmovdqa %ymm0, 32*6-128(%rdi)
vmovdqa %ymm1, 32*7-128(%rdi)
vmovdqa %ymm2, 32*8-128(%rdi)
vzeroupper
ret
.size ecp_nistz256_avx2_set1,.-ecp_nistz256_avx2_set1
___
}
{
################################################################################
# void ecp_nistz256_avx2_multi_gather_w7(void* RESULT, void *in,
# int index0, int index1, int index2, int index3);
################################################################################
my ($val,$in_t,$index0,$index1,$index2,$index3)=("%rdi","%rsi","%edx","%ecx","%r8d","%r9d");
my ($INDEX0,$INDEX1,$INDEX2,$INDEX3)=map("%ymm$_",(0..3));
my ($R0a,$R0b,$R1a,$R1b,$R2a,$R2b,$R3a,$R3b)=map("%ymm$_",(4..11));
my ($M0,$T0,$T1,$TMP0)=map("%ymm$_",(12..15));
$code.=<<___;
.globl ecp_nistz256_avx2_multi_gather_w7
.type ecp_nistz256_avx2_multi_gather_w7,\@function,6
.align 32
ecp_nistz256_avx2_multi_gather_w7:
vzeroupper
___
$code.=<<___ if ($win64);
lea -8-16*10(%rsp), %rsp
vmovaps %xmm6, -8-16*10(%rax)
vmovaps %xmm7, -8-16*9(%rax)
vmovaps %xmm8, -8-16*8(%rax)
vmovaps %xmm9, -8-16*7(%rax)
vmovaps %xmm10, -8-16*6(%rax)
vmovaps %xmm11, -8-16*5(%rax)
vmovaps %xmm12, -8-16*4(%rax)
vmovaps %xmm13, -8-16*3(%rax)
vmovaps %xmm14, -8-16*2(%rax)
vmovaps %xmm15, -8-16*1(%rax)
___
$code.=<<___;
lea .LIntOne(%rip), %rax
vmovd $index0, %xmm0
vmovd $index1, %xmm1
vmovd $index2, %xmm2
vmovd $index3, %xmm3
vpxor $R0a, $R0a, $R0a
vpxor $R0b, $R0b, $R0b
vpxor $R1a, $R1a, $R1a
vpxor $R1b, $R1b, $R1b
vpxor $R2a, $R2a, $R2a
vpxor $R2b, $R2b, $R2b
vpxor $R3a, $R3a, $R3a
vpxor $R3b, $R3b, $R3b
vmovdqa (%rax), $M0
vpermd $INDEX0, $R0a, $INDEX0
vpermd $INDEX1, $R0a, $INDEX1
vpermd $INDEX2, $R0a, $INDEX2
vpermd $INDEX3, $R0a, $INDEX3
mov \$64, %ecx
lea 112($val), $val # size optimization
jmp .Lmulti_select_loop_avx2
# INDEX=0, corresponds to the point at infty (0,0)
.align 32
.Lmulti_select_loop_avx2:
vpcmpeqd $INDEX0, $M0, $TMP0
vmovdqa `32*0+32*64*2*0`($in_t), $T0
vmovdqa `32*1+32*64*2*0`($in_t), $T1
vpand $TMP0, $T0, $T0
vpand $TMP0, $T1, $T1
vpxor $T0, $R0a, $R0a
vpxor $T1, $R0b, $R0b
vpcmpeqd $INDEX1, $M0, $TMP0
vmovdqa `32*0+32*64*2*1`($in_t), $T0
vmovdqa `32*1+32*64*2*1`($in_t), $T1
vpand $TMP0, $T0, $T0
vpand $TMP0, $T1, $T1
vpxor $T0, $R1a, $R1a
vpxor $T1, $R1b, $R1b
vpcmpeqd $INDEX2, $M0, $TMP0
vmovdqa `32*0+32*64*2*2`($in_t), $T0
vmovdqa `32*1+32*64*2*2`($in_t), $T1
vpand $TMP0, $T0, $T0
vpand $TMP0, $T1, $T1
vpxor $T0, $R2a, $R2a
vpxor $T1, $R2b, $R2b
vpcmpeqd $INDEX3, $M0, $TMP0
vmovdqa `32*0+32*64*2*3`($in_t), $T0
vmovdqa `32*1+32*64*2*3`($in_t), $T1
vpand $TMP0, $T0, $T0
vpand $TMP0, $T1, $T1
vpxor $T0, $R3a, $R3a
vpxor $T1, $R3b, $R3b
vpaddd (%rax), $M0, $M0 # increment
lea 32*2($in_t), $in_t
dec %ecx
jnz .Lmulti_select_loop_avx2
vmovdqu $R0a, 32*0-112($val)
vmovdqu $R0b, 32*1-112($val)
vmovdqu $R1a, 32*2-112($val)
vmovdqu $R1b, 32*3-112($val)
vmovdqu $R2a, 32*4-112($val)
vmovdqu $R2b, 32*5-112($val)
vmovdqu $R3a, 32*6-112($val)
vmovdqu $R3b, 32*7-112($val)
vzeroupper
___
$code.=<<___ if ($win64);
movaps 16*0(%rsp), %xmm6
movaps 16*1(%rsp), %xmm7
movaps 16*2(%rsp), %xmm8
movaps 16*3(%rsp), %xmm9
movaps 16*4(%rsp), %xmm10
movaps 16*5(%rsp), %xmm11
movaps 16*6(%rsp), %xmm12
movaps 16*7(%rsp), %xmm13
movaps 16*8(%rsp), %xmm14
movaps 16*9(%rsp), %xmm15
lea 8+16*10(%rsp), %rsp
___
$code.=<<___;
ret
.size ecp_nistz256_avx2_multi_gather_w7,.-ecp_nistz256_avx2_multi_gather_w7
.extern OPENSSL_ia32cap_P
.globl ecp_nistz_avx2_eligible
.type ecp_nistz_avx2_eligible,\@abi-omnipotent
.align 32
ecp_nistz_avx2_eligible:
mov OPENSSL_ia32cap_P+8(%rip),%eax
shr \$5,%eax
and \$1,%eax
ret
.size ecp_nistz_avx2_eligible,.-ecp_nistz_avx2_eligible
___
}
}} else {{ # assembler is too old
$code.=<<___;
.text
.globl ecp_nistz256_avx2_transpose_convert
.globl ecp_nistz256_avx2_convert_transpose_back
.globl ecp_nistz256_avx2_point_add_affine_x4
.globl ecp_nistz256_avx2_point_add_affines_x4
.globl ecp_nistz256_avx2_to_mont
.globl ecp_nistz256_avx2_from_mont
.globl ecp_nistz256_avx2_set1
.globl ecp_nistz256_avx2_multi_gather_w7
.type ecp_nistz256_avx2_multi_gather_w7,\@abi-omnipotent
ecp_nistz256_avx2_transpose_convert:
ecp_nistz256_avx2_convert_transpose_back:
ecp_nistz256_avx2_point_add_affine_x4:
ecp_nistz256_avx2_point_add_affines_x4:
ecp_nistz256_avx2_to_mont:
ecp_nistz256_avx2_from_mont:
ecp_nistz256_avx2_set1:
ecp_nistz256_avx2_multi_gather_w7:
.byte 0x0f,0x0b # ud2
ret
.size ecp_nistz256_avx2_multi_gather_w7,.-ecp_nistz256_avx2_multi_gather_w7
.globl ecp_nistz_avx2_eligible
.type ecp_nistz_avx2_eligible,\@abi-omnipotent
ecp_nistz_avx2_eligible:
xor %eax,%eax
ret
.size ecp_nistz_avx2_eligible,.-ecp_nistz_avx2_eligible
___
}}
foreach (split("\n",$code)) {
s/\`([^\`]*)\`/eval($1)/geo;
print $_,"\n";
}
close STDOUT or die "error closing STDOUT: $!";
crypto/ec/ec2_smpl.c
+2 -1
View File
@@ -489,7 +489,8 @@ int ec_GF2m_simple_invert(const EC_GROUP *group, EC_POINT *point, BN_CTX *ctx)
/* point is its own inverse */
return 1;
if (!EC_POINT_make_affine(group, point, ctx))
if (group->meth->make_affine == NULL
|| !group->meth->make_affine(group, point, ctx))
return 0;
return BN_GF2m_add(point->Y, point->X, point->Y);
}
crypto/ec/ec_ameth.c
+1 -1
View File
@@ -141,7 +141,7 @@ static EC_KEY *eckey_type2param(int ptype, const void *pval)
return NULL;
}
static int eckey_pub_decode(EVP_PKEY *pkey, X509_PUBKEY *pubkey)
static int eckey_pub_decode(EVP_PKEY *pkey, const X509_PUBKEY *pubkey)
{
const unsigned char *p = NULL;
const void *pval;
crypto/ec/ec_asn1.c
+4 -7
View File
@@ -27,8 +27,7 @@ int EC_GROUP_get_basis_type(const EC_GROUP *group)
{
int i;
if (EC_METHOD_get_field_type(EC_GROUP_method_of(group)) !=
NID_X9_62_characteristic_two_field)
if (EC_GROUP_get_field_type(group) != NID_X9_62_characteristic_two_field)
/* everything else is currently not supported */
return 0;
@@ -53,8 +52,7 @@ int EC_GROUP_get_trinomial_basis(const EC_GROUP *group, unsigned int *k)
if (group == NULL)
return 0;
if (EC_METHOD_get_field_type(EC_GROUP_method_of(group)) !=
NID_X9_62_characteristic_two_field
if (EC_GROUP_get_field_type(group) != NID_X9_62_characteristic_two_field
|| !((group->poly[0] != 0) && (group->poly[1] != 0)
&& (group->poly[2] == 0))) {
ECerr(EC_F_EC_GROUP_GET_TRINOMIAL_BASIS,
@@ -74,8 +72,7 @@ int EC_GROUP_get_pentanomial_basis(const EC_GROUP *group, unsigned int *k1,
if (group == NULL)
return 0;
if (EC_METHOD_get_field_type(EC_GROUP_method_of(group)) !=
NID_X9_62_characteristic_two_field
if (EC_GROUP_get_field_type(group) != NID_X9_62_characteristic_two_field
|| !((group->poly[0] != 0) && (group->poly[1] != 0)
&& (group->poly[2] != 0) && (group->poly[3] != 0)
&& (group->poly[4] == 0))) {
@@ -262,7 +259,7 @@ static int ec_asn1_group2fieldid(const EC_GROUP *group, X9_62_FIELDID *field)
ASN1_OBJECT_free(field->fieldType);
ASN1_TYPE_free(field->p.other);
nid = EC_METHOD_get_field_type(EC_GROUP_method_of(group));
nid = EC_GROUP_get_field_type(group);
/* set OID for the field */
if ((field->fieldType = OBJ_nid2obj(nid)) == NULL) {
ECerr(EC_F_EC_ASN1_GROUP2FIELDID, ERR_R_OBJ_LIB);
crypto/ec/ec_backend.c
+7 -8
View File
@@ -19,15 +19,10 @@
* implementations alike.
*/
int ec_set_param_ecdh_cofactor_mode(EC_KEY *ec, const OSSL_PARAM *p)
int ec_set_ecdh_cofactor_mode(EC_KEY *ec, int mode)
{
const EC_GROUP *ecg = EC_KEY_get0_group(ec);
const BIGNUM *cofactor;
int mode;
if (!OSSL_PARAM_get_int(p, &mode))
return 0;
/*
* mode can be only 0 for disable, or 1 for enable here.
*
@@ -224,8 +219,12 @@ int ec_key_otherparams_fromdata(EC_KEY *ec, const OSSL_PARAM params[])
return 0;
p = OSSL_PARAM_locate_const(params, OSSL_PKEY_PARAM_USE_COFACTOR_ECDH);
if (p != NULL && !ec_set_param_ecdh_cofactor_mode(ec, p))
return 0;
if (p != NULL) {
int mode;
if (!OSSL_PARAM_get_int(p, &mode)
|| !ec_set_ecdh_cofactor_mode(ec, mode))
return 0;
}
return 1;
}
crypto/ec/ec_curve.c
+3 -7
View File
@@ -3195,7 +3195,7 @@ static EC_GROUP *ec_group_new_from_data(OPENSSL_CTX *libctx,
/* If no curve data curve method must handle everything */
if (curve.data == NULL)
return EC_GROUP_new_ex(libctx,
return ec_group_new_ex(libctx,
curve.meth != NULL ? curve.meth() : NULL);
if ((ctx = BN_CTX_new_ex(libctx)) == NULL) {
@@ -3218,7 +3218,7 @@ static EC_GROUP *ec_group_new_from_data(OPENSSL_CTX *libctx,
if (curve.meth != 0) {
meth = curve.meth();
if (((group = EC_GROUP_new_ex(libctx, meth)) == NULL) ||
if (((group = ec_group_new_ex(libctx, meth)) == NULL) ||
(!(group->meth->group_set_curve(group, p, a, b, ctx)))) {
ECerr(EC_F_EC_GROUP_NEW_FROM_DATA, ERR_R_EC_LIB);
goto err;
@@ -3388,17 +3388,13 @@ int ec_curve_nid_from_params(const EC_GROUP *group, BN_CTX *ctx)
unsigned char *param_bytes = NULL;
const EC_CURVE_DATA *data;
const EC_POINT *generator = NULL;
const EC_METHOD *meth;
const BIGNUM *cofactor = NULL;
/* An array of BIGNUMs for (p, a, b, x, y, order) */
BIGNUM *bn[NUM_BN_FIELDS] = {NULL, NULL, NULL, NULL, NULL, NULL};
meth = EC_GROUP_method_of(group);
if (meth == NULL)
return -1;
/* Use the optional named curve nid as a search field */
nid = EC_GROUP_get_curve_name(group);
field_type = EC_METHOD_get_field_type(meth);
field_type = EC_GROUP_get_field_type(group);
seed_len = EC_GROUP_get_seed_len(group);
seed = EC_GROUP_get0_seed(group);
cofactor = EC_GROUP_get0_cofactor(group);
crypto/ec/ec_cvt.c
+2 -2
View File
@@ -54,7 +54,7 @@ EC_GROUP *EC_GROUP_new_curve_GFp(const BIGNUM *p, const BIGNUM *a,
meth = EC_GFp_mont_method();
#endif
ret = EC_GROUP_new_ex(bn_get_lib_ctx(ctx), meth);
ret = ec_group_new_ex(bn_get_lib_ctx(ctx), meth);
if (ret == NULL)
return NULL;
@@ -75,7 +75,7 @@ EC_GROUP *EC_GROUP_new_curve_GF2m(const BIGNUM *p, const BIGNUM *a,
meth = EC_GF2m_simple_method();
ret = EC_GROUP_new_ex(bn_get_lib_ctx(ctx), meth);
ret = ec_group_new_ex(bn_get_lib_ctx(ctx), meth);
if (ret == NULL)
return NULL;
crypto/ec/ec_key.c
+4 -3
View File
@@ -117,10 +117,9 @@ EC_KEY *EC_KEY_copy(EC_KEY *dest, const EC_KEY *src)
dest->libctx = src->libctx;
/* copy the parameters */
if (src->group != NULL) {
const EC_METHOD *meth = EC_GROUP_method_of(src->group);
/* clear the old group */
EC_GROUP_free(dest->group);
dest->group = EC_GROUP_new_ex(src->libctx, meth);
dest->group = ec_group_new_ex(src->libctx, src->group->meth);
if (dest->group == NULL)
return NULL;
if (!EC_GROUP_copy(dest->group, src->group))
@@ -398,7 +397,7 @@ static int ec_key_public_range_check(BN_CTX *ctx, const EC_KEY *key)
if (!EC_POINT_get_affine_coordinates(key->group, key->pub_key, x, y, ctx))
goto err;
if (EC_METHOD_get_field_type(key->group->meth) == NID_X9_62_prime_field) {
if (EC_GROUP_get_field_type(key->group) == NID_X9_62_prime_field) {
if (BN_is_negative(x)
|| BN_cmp(x, key->group->field) >= 0
|| BN_is_negative(y)
@@ -781,12 +780,14 @@ void EC_KEY_set_asn1_flag(EC_KEY *key, int flag)
EC_GROUP_set_asn1_flag(key->group, flag);
}
#ifndef OPENSSL_NO_DEPRECATED_3_0
int EC_KEY_precompute_mult(EC_KEY *key, BN_CTX *ctx)
{
if (key->group == NULL)
return 0;
return EC_GROUP_precompute_mult(key->group, ctx);
}
#endif
int EC_KEY_get_flags(const EC_KEY *key)
{
crypto/ec/ec_lib.c
+24 -10
View File
@@ -23,7 +23,7 @@
/* functions for EC_GROUP objects */
EC_GROUP *EC_GROUP_new_ex(OPENSSL_CTX *libctx, const EC_METHOD *meth)
EC_GROUP *ec_group_new_ex(OPENSSL_CTX *libctx, const EC_METHOD *meth)
{
EC_GROUP *ret;
@@ -65,11 +65,13 @@ EC_GROUP *EC_GROUP_new_ex(OPENSSL_CTX *libctx, const EC_METHOD *meth)
return NULL;
}
#ifndef FIPS_MODULE
#ifndef OPENSSL_NO_DEPRECATED_3_0
# ifndef FIPS_MODULE
EC_GROUP *EC_GROUP_new(const EC_METHOD *meth)
{
return EC_GROUP_new_ex(NULL, meth);
return ec_group_new_ex(NULL, meth);
}
# endif
#endif
void EC_pre_comp_free(EC_GROUP *group)
@@ -255,7 +257,7 @@ EC_GROUP *EC_GROUP_dup(const EC_GROUP *a)
if (a == NULL)
return NULL;
if ((t = EC_GROUP_new_ex(a->libctx, a->meth)) == NULL)
if ((t = ec_group_new_ex(a->libctx, a->meth)) == NULL)
return NULL;
if (!EC_GROUP_copy(t, a))
goto err;
@@ -270,6 +272,7 @@ EC_GROUP *EC_GROUP_dup(const EC_GROUP *a)
return t;
}
#ifndef OPENSSL_NO_DEPRECATED_3_0
const EC_METHOD *EC_GROUP_method_of(const EC_GROUP *group)
{
return group->meth;
@@ -279,6 +282,7 @@ int EC_METHOD_get_field_type(const EC_METHOD *meth)
{
return meth->field_type;
}
#endif
static int ec_precompute_mont_data(EC_GROUP *);
@@ -475,6 +479,11 @@ const BIGNUM *EC_GROUP_get0_field(const EC_GROUP *group)
return group->field;
}
int EC_GROUP_get_field_type(const EC_GROUP *group)
{
return group->meth->field_type;
}
void EC_GROUP_set_asn1_flag(EC_GROUP *group, int flag)
{
group->asn1_flag = flag;
@@ -602,8 +611,7 @@ int EC_GROUP_cmp(const EC_GROUP *a, const EC_GROUP *b, BN_CTX *ctx)
#endif
/* compare the field types */
if (EC_METHOD_get_field_type(EC_GROUP_method_of(a)) !=
EC_METHOD_get_field_type(EC_GROUP_method_of(b)))
if (EC_GROUP_get_field_type(a) != EC_GROUP_get_field_type(b))
return 1;
/* compare the curve name (if present in both) */
if (EC_GROUP_get_curve_name(a) && EC_GROUP_get_curve_name(b) &&
@@ -777,10 +785,12 @@ EC_POINT *EC_POINT_dup(const EC_POINT *a, const EC_GROUP *group)
return t;
}
#ifndef OPENSSL_NO_DEPRECATED_3_0
const EC_METHOD *EC_POINT_method_of(const EC_POINT *point)
{
return point->meth;
}
#endif
int EC_POINT_set_to_infinity(const EC_GROUP *group, EC_POINT *point)
{
@@ -1004,6 +1014,7 @@ int EC_POINT_cmp(const EC_GROUP *group, const EC_POINT *a, const EC_POINT *b,
return group->meth->point_cmp(group, a, b, ctx);
}
#ifndef OPENSSL_NO_DEPRECATED_3_0
int EC_POINT_make_affine(const EC_GROUP *group, EC_POINT *point, BN_CTX *ctx)
{
if (group->meth->make_affine == 0) {
@@ -1034,6 +1045,7 @@ int EC_POINTs_make_affine(const EC_GROUP *group, size_t num,
}
return group->meth->points_make_affine(group, num, points, ctx);
}
#endif
/*
* Functions for point multiplication. If group->meth->mul is 0, we use the
@@ -1093,6 +1105,7 @@ int EC_POINT_mul(const EC_GROUP *group, EC_POINT *r, const BIGNUM *g_scalar,
const EC_POINT *point, const BIGNUM *p_scalar, BN_CTX *ctx)
{
int ret = 0;
size_t num;
#ifndef FIPS_MODULE
BN_CTX *new_ctx = NULL;
#endif
@@ -1115,13 +1128,12 @@ int EC_POINT_mul(const EC_GROUP *group, EC_POINT *r, const BIGNUM *g_scalar,
return 0;
}
num = (point != NULL && p_scalar != NULL) ? 1 : 0;
if (group->meth->mul != NULL)
ret = group->meth->mul(group, r, g_scalar, point != NULL
&& p_scalar != NULL, &point, &p_scalar, ctx);
ret = group->meth->mul(group, r, g_scalar, num, &point, &p_scalar, ctx);
else
/* use default */
ret = ec_wNAF_mul(group, r, g_scalar, point != NULL
&& p_scalar != NULL, &point, &p_scalar, ctx);
ret = ec_wNAF_mul(group, r, g_scalar, num, &point, &p_scalar, ctx);
#ifndef FIPS_MODULE
BN_CTX_free(new_ctx);
@@ -1129,6 +1141,7 @@ int EC_POINT_mul(const EC_GROUP *group, EC_POINT *r, const BIGNUM *g_scalar,
return ret;
}
#ifndef OPENSSL_NO_DEPRECATED_3_0
int EC_GROUP_precompute_mult(EC_GROUP *group, BN_CTX *ctx)
{
if (group->meth->mul == 0)
@@ -1153,6 +1166,7 @@ int EC_GROUP_have_precompute_mult(const EC_GROUP *group)
return 0; /* cannot tell whether precomputation has
* been performed */
}
#endif
/*
* ec_precompute_mont_data sets |group->mont_data| from |group->order| and
crypto/ec/ec_local.h
+13
View File
@@ -31,6 +31,10 @@
/* Curve does not support signing operations */
#define EC_FLAGS_NO_SIGN 0x4
#ifdef OPENSSL_NO_DEPRECATED_3_0
typedef struct ec_method_st EC_METHOD;
#endif
/*
* Structure details are not part of the exported interface, so all this may
* change in future versions.
@@ -585,6 +589,15 @@ void ec_GFp_nistp_recode_scalar_bits(unsigned char *sign,
#endif
int ec_group_simple_order_bits(const EC_GROUP *group);
/**
* Creates a new EC_GROUP object
* \param libctx The associated library context or NULL for the default
* library context
* \param meth EC_METHOD to use
* \return newly created EC_GROUP object or NULL in case of an error.
*/
EC_GROUP *ec_group_new_ex(OPENSSL_CTX *libctx, const EC_METHOD *meth);
#ifdef ECP_NISTZ256_ASM
/** Returns GFp methods using montgomery multiplication, with x86-64 optimized
* P256. See http://eprint.iacr.org/2013/816.
crypto/ec/ec_mult.c
+6 -3
View File
@@ -267,7 +267,8 @@ int ec_scalar_mul_ladder(const EC_GROUP *group, EC_POINT *r,
}
/* ensure input point is in affine coords for ladder step efficiency */
if (!p->Z_is_one && !EC_POINT_make_affine(group, p, ctx)) {
if (!p->Z_is_one && (group->meth->make_affine == NULL
|| !group->meth->make_affine(group, p, ctx))) {
ECerr(EC_F_EC_SCALAR_MUL_LADDER, ERR_R_EC_LIB);
goto err;
}
@@ -711,7 +712,8 @@ int ec_wNAF_mul(const EC_GROUP *group, EC_POINT *r, const BIGNUM *scalar,
}
}
if (!EC_POINTs_make_affine(group, num_val, val, ctx))
if (group->meth->points_make_affine == NULL
|| !group->meth->points_make_affine(group, num_val, val, ctx))
goto err;
r_is_at_infinity = 1;
@@ -949,7 +951,8 @@ int ec_wNAF_precompute_mult(EC_GROUP *group, BN_CTX *ctx)
}
}
if (!EC_POINTs_make_affine(group, num, points, ctx))
if (group->meth->points_make_affine == NULL
|| !group->meth->points_make_affine(group, num, points, ctx))
goto err;
pre_comp->group = group;
crypto/ec/ecdh_kdf.c
+3 -3
View File
@@ -32,7 +32,7 @@ int ecdh_KDF_X9_63(unsigned char *out, size_t outlen,
const char *mdname = EVP_MD_name(md);
EVP_KDF *kdf = EVP_KDF_fetch(NULL, OSSL_KDF_NAME_X963KDF, NULL);
if ((kctx = EVP_KDF_CTX_new(kdf)) != NULL) {
if ((kctx = EVP_KDF_new_ctx(kdf)) != NULL) {
*p++ = OSSL_PARAM_construct_utf8_string(OSSL_KDF_PARAM_DIGEST,
(char *)mdname, 0);
*p++ = OSSL_PARAM_construct_octet_string(OSSL_KDF_PARAM_KEY,
@@ -41,9 +41,9 @@ int ecdh_KDF_X9_63(unsigned char *out, size_t outlen,
(void *)sinfo, sinfolen);
*p = OSSL_PARAM_construct_end();
ret = EVP_KDF_CTX_set_params(kctx, params) > 0
ret = EVP_KDF_set_ctx_params(kctx, params) > 0
&& EVP_KDF_derive(kctx, out, outlen) > 0;
EVP_KDF_CTX_free(kctx);
EVP_KDF_free_ctx(kctx);
}
EVP_KDF_free(kdf);
return ret;
crypto/ec/eck_prn.c
+2 -2
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2006-2018 The OpenSSL Project Authors. All Rights Reserved.
* Copyright 2006-2020 The OpenSSL Project Authors. All Rights Reserved.
* Copyright (c) 2002, Oracle and/or its affiliates. All rights reserved
*
* Licensed under the Apache License 2.0 (the "License"). You may not use
@@ -115,7 +115,7 @@ int ECPKParameters_print(BIO *bp, const EC_GROUP *x, int off)
/* explicit parameters */
int is_char_two = 0;
point_conversion_form_t form;
int tmp_nid = EC_METHOD_get_field_type(EC_GROUP_method_of(x));
int tmp_nid = EC_GROUP_get_field_type(x);
if (tmp_nid == NID_X9_62_characteristic_two_field)
is_char_two = 1;
crypto/ec/ecp_nistp224.c
+5 -4
View File
@@ -75,6 +75,7 @@ typedef uint64_t u64;
*/
typedef uint64_t limb;
typedef uint64_t limb_aX __attribute((__aligned__(1)));
typedef uint128_t widelimb;
typedef limb felem[4];
@@ -311,10 +312,10 @@ const EC_METHOD *EC_GFp_nistp224_method(void)
*/
static void bin28_to_felem(felem out, const u8 in[28])
{
out[0] = *((const uint64_t *)(in)) & 0x00ffffffffffffff;
out[1] = (*((const uint64_t *)(in + 7))) & 0x00ffffffffffffff;
out[2] = (*((const uint64_t *)(in + 14))) & 0x00ffffffffffffff;
out[3] = (*((const uint64_t *)(in+20))) >> 8;
out[0] = *((const limb *)(in)) & 0x00ffffffffffffff;
out[1] = (*((const limb_aX *)(in + 7))) & 0x00ffffffffffffff;
out[2] = (*((const limb_aX *)(in + 14))) & 0x00ffffffffffffff;
out[3] = (*((const limb_aX *)(in + 20))) >> 8;
}
static void felem_to_bin28(u8 out[28], const felem in)
crypto/ec/ecp_nistp521.c
+17 -16
View File
@@ -131,6 +131,7 @@ static const felem_bytearray nistp521_curve_params[5] = {
#define NLIMBS 9
typedef uint64_t limb;
typedef limb limb_aX __attribute((__aligned__(1)));
typedef limb felem[NLIMBS];
typedef uint128_t largefelem[NLIMBS];
@@ -144,14 +145,14 @@ static const limb bottom58bits = 0x3ffffffffffffff;
static void bin66_to_felem(felem out, const u8 in[66])
{
out[0] = (*((limb *) & in[0])) & bottom58bits;
out[1] = (*((limb *) & in[7]) >> 2) & bottom58bits;
out[2] = (*((limb *) & in[14]) >> 4) & bottom58bits;
out[3] = (*((limb *) & in[21]) >> 6) & bottom58bits;
out[4] = (*((limb *) & in[29])) & bottom58bits;
out[5] = (*((limb *) & in[36]) >> 2) & bottom58bits;
out[6] = (*((limb *) & in[43]) >> 4) & bottom58bits;
out[7] = (*((limb *) & in[50]) >> 6) & bottom58bits;
out[8] = (*((limb *) & in[58])) & bottom57bits;
out[1] = (*((limb_aX *) & in[7]) >> 2) & bottom58bits;
out[2] = (*((limb_aX *) & in[14]) >> 4) & bottom58bits;
out[3] = (*((limb_aX *) & in[21]) >> 6) & bottom58bits;
out[4] = (*((limb_aX *) & in[29])) & bottom58bits;
out[5] = (*((limb_aX *) & in[36]) >> 2) & bottom58bits;
out[6] = (*((limb_aX *) & in[43]) >> 4) & bottom58bits;
out[7] = (*((limb_aX *) & in[50]) >> 6) & bottom58bits;
out[8] = (*((limb_aX *) & in[58])) & bottom57bits;
}
/*
@@ -162,14 +163,14 @@ static void felem_to_bin66(u8 out[66], const felem in)
{
memset(out, 0, 66);
(*((limb *) & out[0])) = in[0];
(*((limb *) & out[7])) |= in[1] << 2;
(*((limb *) & out[14])) |= in[2] << 4;
(*((limb *) & out[21])) |= in[3] << 6;
(*((limb *) & out[29])) = in[4];
(*((limb *) & out[36])) |= in[5] << 2;
(*((limb *) & out[43])) |= in[6] << 4;
(*((limb *) & out[50])) |= in[7] << 6;
(*((limb *) & out[58])) = in[8];
(*((limb_aX *) & out[7])) |= in[1] << 2;
(*((limb_aX *) & out[14])) |= in[2] << 4;
(*((limb_aX *) & out[21])) |= in[3] << 6;
(*((limb_aX *) & out[29])) = in[4];
(*((limb_aX *) & out[36])) |= in[5] << 2;
(*((limb_aX *) & out[43])) |= in[6] << 4;
(*((limb_aX *) & out[50])) |= in[7] << 6;
(*((limb_aX *) & out[58])) = in[8];
}
/* BN_to_felem converts an OpenSSL BIGNUM into an felem */
crypto/ec/ecp_nistz256.c
+49 -256
View File
@@ -897,7 +897,8 @@ __owur static int ecp_nistz256_mult_precompute(EC_GROUP *group, BN_CTX *ctx)
* It would be faster to use EC_POINTs_make_affine and
* make multiple points affine at the same time.
*/
if (!EC_POINT_make_affine(group, P, ctx))
if (group->meth->make_affine == NULL
|| !group->meth->make_affine(group, P, ctx))
goto err;
if (!ecp_nistz256_bignum_to_field_elem(temp.X, P->X) ||
!ecp_nistz256_bignum_to_field_elem(temp.Y, P->Y)) {
@@ -935,207 +936,6 @@ __owur static int ecp_nistz256_mult_precompute(EC_GROUP *group, BN_CTX *ctx)
return ret;
}
/*
* Note that by default ECP_NISTZ256_AVX2 is undefined. While it's great
* code processing 4 points in parallel, corresponding serial operation
* is several times slower, because it uses 29x29=58-bit multiplication
* as opposite to 64x64=128-bit in integer-only scalar case. As result
* it doesn't provide *significant* performance improvement. Note that
* just defining ECP_NISTZ256_AVX2 is not sufficient to make it work,
* you'd need to compile even asm/ecp_nistz256-avx.pl module.
*/
#if defined(ECP_NISTZ256_AVX2)
# if !(defined(__x86_64) || defined(__x86_64__) || \
defined(_M_AMD64) || defined(_M_X64)) || \
!(defined(__GNUC__) || defined(_MSC_VER)) /* this is for ALIGN32 */
# undef ECP_NISTZ256_AVX2
# else
/* Constant time access, loading four values, from four consecutive tables */
void ecp_nistz256_avx2_multi_gather_w7(void *result, const void *in,
int index0, int index1, int index2,
int index3);
void ecp_nistz256_avx2_transpose_convert(void *RESULTx4, const void *in);
void ecp_nistz256_avx2_convert_transpose_back(void *result, const void *Ax4);
void ecp_nistz256_avx2_point_add_affine_x4(void *RESULTx4, const void *Ax4,
const void *Bx4);
void ecp_nistz256_avx2_point_add_affines_x4(void *RESULTx4, const void *Ax4,
const void *Bx4);
void ecp_nistz256_avx2_to_mont(void *RESULTx4, const void *Ax4);
void ecp_nistz256_avx2_from_mont(void *RESULTx4, const void *Ax4);
void ecp_nistz256_avx2_set1(void *RESULTx4);
int ecp_nistz_avx2_eligible(void);
static void booth_recode_w7(unsigned char *sign,
unsigned char *digit, unsigned char in)
{
unsigned char s, d;
s = ~((in >> 7) - 1);
d = (1 << 8) - in - 1;
d = (d & s) | (in & ~s);
d = (d >> 1) + (d & 1);
*sign = s & 1;
*digit = d;
}
/*
* ecp_nistz256_avx2_mul_g performs multiplication by G, using only the
* precomputed table. It does 4 affine point additions in parallel,
* significantly speeding up point multiplication for a fixed value.
*/
static void ecp_nistz256_avx2_mul_g(P256_POINT *r,
unsigned char p_str[33],
const P256_POINT_AFFINE(*preComputedTable)[64])
{
const unsigned int window_size = 7;
const unsigned int mask = (1 << (window_size + 1)) - 1;
unsigned int wvalue;
/* Using 4 windows at a time */
unsigned char sign0, digit0;
unsigned char sign1, digit1;
unsigned char sign2, digit2;
unsigned char sign3, digit3;
unsigned int idx = 0;
BN_ULONG tmp[P256_LIMBS];
int i;
ALIGN32 BN_ULONG aX4[4 * 9 * 3] = { 0 };
ALIGN32 BN_ULONG bX4[4 * 9 * 2] = { 0 };
ALIGN32 P256_POINT_AFFINE point_arr[4];
ALIGN32 P256_POINT res_point_arr[4];
/* Initial four windows */
wvalue = *((u16 *) & p_str[0]);
wvalue = (wvalue << 1) & mask;
idx += window_size;
booth_recode_w7(&sign0, &digit0, wvalue);
wvalue = *((u16 *) & p_str[(idx - 1) / 8]);
wvalue = (wvalue >> ((idx - 1) % 8)) & mask;
idx += window_size;
booth_recode_w7(&sign1, &digit1, wvalue);
wvalue = *((u16 *) & p_str[(idx - 1) / 8]);
wvalue = (wvalue >> ((idx - 1) % 8)) & mask;
idx += window_size;
booth_recode_w7(&sign2, &digit2, wvalue);
wvalue = *((u16 *) & p_str[(idx - 1) / 8]);
wvalue = (wvalue >> ((idx - 1) % 8)) & mask;
idx += window_size;
booth_recode_w7(&sign3, &digit3, wvalue);
ecp_nistz256_avx2_multi_gather_w7(point_arr, preComputedTable[0],
digit0, digit1, digit2, digit3);
ecp_nistz256_neg(tmp, point_arr[0].Y);
copy_conditional(point_arr[0].Y, tmp, sign0);
ecp_nistz256_neg(tmp, point_arr[1].Y);
copy_conditional(point_arr[1].Y, tmp, sign1);
ecp_nistz256_neg(tmp, point_arr[2].Y);
copy_conditional(point_arr[2].Y, tmp, sign2);
ecp_nistz256_neg(tmp, point_arr[3].Y);
copy_conditional(point_arr[3].Y, tmp, sign3);
ecp_nistz256_avx2_transpose_convert(aX4, point_arr);
ecp_nistz256_avx2_to_mont(aX4, aX4);
ecp_nistz256_avx2_to_mont(&aX4[4 * 9], &aX4[4 * 9]);
ecp_nistz256_avx2_set1(&aX4[4 * 9 * 2]);
wvalue = *((u16 *) & p_str[(idx - 1) / 8]);
wvalue = (wvalue >> ((idx - 1) % 8)) & mask;
idx += window_size;
booth_recode_w7(&sign0, &digit0, wvalue);
wvalue = *((u16 *) & p_str[(idx - 1) / 8]);
wvalue = (wvalue >> ((idx - 1) % 8)) & mask;
idx += window_size;
booth_recode_w7(&sign1, &digit1, wvalue);
wvalue = *((u16 *) & p_str[(idx - 1) / 8]);
wvalue = (wvalue >> ((idx - 1) % 8)) & mask;
idx += window_size;
booth_recode_w7(&sign2, &digit2, wvalue);
wvalue = *((u16 *) & p_str[(idx - 1) / 8]);
wvalue = (wvalue >> ((idx - 1) % 8)) & mask;
idx += window_size;
booth_recode_w7(&sign3, &digit3, wvalue);
ecp_nistz256_avx2_multi_gather_w7(point_arr, preComputedTable[4 * 1],
digit0, digit1, digit2, digit3);
ecp_nistz256_neg(tmp, point_arr[0].Y);
copy_conditional(point_arr[0].Y, tmp, sign0);
ecp_nistz256_neg(tmp, point_arr[1].Y);
copy_conditional(point_arr[1].Y, tmp, sign1);
ecp_nistz256_neg(tmp, point_arr[2].Y);
copy_conditional(point_arr[2].Y, tmp, sign2);
ecp_nistz256_neg(tmp, point_arr[3].Y);
copy_conditional(point_arr[3].Y, tmp, sign3);
ecp_nistz256_avx2_transpose_convert(bX4, point_arr);
ecp_nistz256_avx2_to_mont(bX4, bX4);
ecp_nistz256_avx2_to_mont(&bX4[4 * 9], &bX4[4 * 9]);
/* Optimized when both inputs are affine */
ecp_nistz256_avx2_point_add_affines_x4(aX4, aX4, bX4);
for (i = 2; i < 9; i++) {
wvalue = *((u16 *) & p_str[(idx - 1) / 8]);
wvalue = (wvalue >> ((idx - 1) % 8)) & mask;
idx += window_size;
booth_recode_w7(&sign0, &digit0, wvalue);
wvalue = *((u16 *) & p_str[(idx - 1) / 8]);
wvalue = (wvalue >> ((idx - 1) % 8)) & mask;
idx += window_size;
booth_recode_w7(&sign1, &digit1, wvalue);
wvalue = *((u16 *) & p_str[(idx - 1) / 8]);
wvalue = (wvalue >> ((idx - 1) % 8)) & mask;
idx += window_size;
booth_recode_w7(&sign2, &digit2, wvalue);
wvalue = *((u16 *) & p_str[(idx - 1) / 8]);
wvalue = (wvalue >> ((idx - 1) % 8)) & mask;
idx += window_size;
booth_recode_w7(&sign3, &digit3, wvalue);
ecp_nistz256_avx2_multi_gather_w7(point_arr,
preComputedTable[4 * i],
digit0, digit1, digit2, digit3);
ecp_nistz256_neg(tmp, point_arr[0].Y);
copy_conditional(point_arr[0].Y, tmp, sign0);
ecp_nistz256_neg(tmp, point_arr[1].Y);
copy_conditional(point_arr[1].Y, tmp, sign1);
ecp_nistz256_neg(tmp, point_arr[2].Y);
copy_conditional(point_arr[2].Y, tmp, sign2);
ecp_nistz256_neg(tmp, point_arr[3].Y);
copy_conditional(point_arr[3].Y, tmp, sign3);
ecp_nistz256_avx2_transpose_convert(bX4, point_arr);
ecp_nistz256_avx2_to_mont(bX4, bX4);
ecp_nistz256_avx2_to_mont(&bX4[4 * 9], &bX4[4 * 9]);
ecp_nistz256_avx2_point_add_affine_x4(aX4, aX4, bX4);
}
ecp_nistz256_avx2_from_mont(&aX4[4 * 9 * 0], &aX4[4 * 9 * 0]);
ecp_nistz256_avx2_from_mont(&aX4[4 * 9 * 1], &aX4[4 * 9 * 1]);
ecp_nistz256_avx2_from_mont(&aX4[4 * 9 * 2], &aX4[4 * 9 * 2]);
ecp_nistz256_avx2_convert_transpose_back(res_point_arr, aX4);
/* Last window is performed serially */
wvalue = *((u16 *) & p_str[(idx - 1) / 8]);
wvalue = (wvalue >> ((idx - 1) % 8)) & mask;
booth_recode_w7(&sign0, &digit0, wvalue);
ecp_nistz256_gather_w7((P256_POINT_AFFINE *)r,
preComputedTable[36], digit0);
ecp_nistz256_neg(tmp, r->Y);
copy_conditional(r->Y, tmp, sign0);
memcpy(r->Z, ONE, sizeof(ONE));
/* Sum the four windows */
ecp_nistz256_point_add(r, r, &res_point_arr[0]);
ecp_nistz256_point_add(r, r, &res_point_arr[1]);
ecp_nistz256_point_add(r, r, &res_point_arr[2]);
ecp_nistz256_point_add(r, r, &res_point_arr[3]);
}
# endif
#endif
__owur static int ecp_nistz256_set_from_affine(EC_POINT *out, const EC_GROUP *group,
const P256_POINT_AFFINE *in,
BN_CTX *ctx)
@@ -1225,6 +1025,8 @@ __owur static int ecp_nistz256_points_mul(const EC_GROUP *group,
}
if (preComputedTable) {
BN_ULONG infty;
if ((BN_num_bits(scalar) > 256)
|| BN_is_negative(scalar)) {
if ((tmp_scalar = BN_CTX_get(ctx)) == NULL)
@@ -1256,67 +1058,58 @@ __owur static int ecp_nistz256_points_mul(const EC_GROUP *group,
for (; i < 33; i++)
p_str[i] = 0;
#if defined(ECP_NISTZ256_AVX2)
if (ecp_nistz_avx2_eligible()) {
ecp_nistz256_avx2_mul_g(&p.p, p_str, preComputedTable);
} else
#endif
{
BN_ULONG infty;
/* First window */
wvalue = (p_str[0] << 1) & mask;
idx += window_size;
/* First window */
wvalue = (p_str[0] << 1) & mask;
wvalue = _booth_recode_w7(wvalue);
ecp_nistz256_gather_w7(&p.a, preComputedTable[0],
wvalue >> 1);
ecp_nistz256_neg(p.p.Z, p.p.Y);
copy_conditional(p.p.Y, p.p.Z, wvalue & 1);
/*
* Since affine infinity is encoded as (0,0) and
* Jacobian is (,,0), we need to harmonize them
* by assigning "one" or zero to Z.
*/
infty = (p.p.X[0] | p.p.X[1] | p.p.X[2] | p.p.X[3] |
p.p.Y[0] | p.p.Y[1] | p.p.Y[2] | p.p.Y[3]);
if (P256_LIMBS == 8)
infty |= (p.p.X[4] | p.p.X[5] | p.p.X[6] | p.p.X[7] |
p.p.Y[4] | p.p.Y[5] | p.p.Y[6] | p.p.Y[7]);
infty = 0 - is_zero(infty);
infty = ~infty;
p.p.Z[0] = ONE[0] & infty;
p.p.Z[1] = ONE[1] & infty;
p.p.Z[2] = ONE[2] & infty;
p.p.Z[3] = ONE[3] & infty;
if (P256_LIMBS == 8) {
p.p.Z[4] = ONE[4] & infty;
p.p.Z[5] = ONE[5] & infty;
p.p.Z[6] = ONE[6] & infty;
p.p.Z[7] = ONE[7] & infty;
}
for (i = 1; i < 37; i++) {
unsigned int off = (idx - 1) / 8;
wvalue = p_str[off] | p_str[off + 1] << 8;
wvalue = (wvalue >> ((idx - 1) % 8)) & mask;
idx += window_size;
wvalue = _booth_recode_w7(wvalue);
ecp_nistz256_gather_w7(&p.a, preComputedTable[0],
wvalue >> 1);
ecp_nistz256_gather_w7(&t.a,
preComputedTable[i], wvalue >> 1);
ecp_nistz256_neg(p.p.Z, p.p.Y);
copy_conditional(p.p.Y, p.p.Z, wvalue & 1);
ecp_nistz256_neg(t.p.Z, t.a.Y);
copy_conditional(t.a.Y, t.p.Z, wvalue & 1);
/*
* Since affine infinity is encoded as (0,0) and
* Jacobian ias (,,0), we need to harmonize them
* by assigning "one" or zero to Z.
*/
infty = (p.p.X[0] | p.p.X[1] | p.p.X[2] | p.p.X[3] |
p.p.Y[0] | p.p.Y[1] | p.p.Y[2] | p.p.Y[3]);
if (P256_LIMBS == 8)
infty |= (p.p.X[4] | p.p.X[5] | p.p.X[6] | p.p.X[7] |
p.p.Y[4] | p.p.Y[5] | p.p.Y[6] | p.p.Y[7]);
infty = 0 - is_zero(infty);
infty = ~infty;
p.p.Z[0] = ONE[0] & infty;
p.p.Z[1] = ONE[1] & infty;
p.p.Z[2] = ONE[2] & infty;
p.p.Z[3] = ONE[3] & infty;
if (P256_LIMBS == 8) {
p.p.Z[4] = ONE[4] & infty;
p.p.Z[5] = ONE[5] & infty;
p.p.Z[6] = ONE[6] & infty;
p.p.Z[7] = ONE[7] & infty;
}
for (i = 1; i < 37; i++) {
unsigned int off = (idx - 1) / 8;
wvalue = p_str[off] | p_str[off + 1] << 8;
wvalue = (wvalue >> ((idx - 1) % 8)) & mask;
idx += window_size;
wvalue = _booth_recode_w7(wvalue);
ecp_nistz256_gather_w7(&t.a,
preComputedTable[i], wvalue >> 1);
ecp_nistz256_neg(t.p.Z, t.a.Y);
copy_conditional(t.a.Y, t.p.Z, wvalue & 1);
ecp_nistz256_point_add_affine(&p.p, &p.p, &t.a);
}
ecp_nistz256_point_add_affine(&p.p, &p.p, &t.a);
}
} else {
p_is_infinity = 1;
crypto/ec/ecp_s390x_nistp.c
+6
View File
@@ -7,6 +7,12 @@
* https://www.openssl.org/source/license.html
*/
/*
* EC_METHOD low level APIs are deprecated for public use, but still ok for
* internal use.
*/
#include "internal/deprecated.h"
#include <stdlib.h>
#include <string.h>
#include <openssl/err.h>
crypto/ec/ecx_meth.c
+1 -1
View File
@@ -126,7 +126,7 @@ static int ecx_pub_encode(X509_PUBKEY *pk, const EVP_PKEY *pkey)
return 1;
}
static int ecx_pub_decode(EVP_PKEY *pkey, X509_PUBKEY *pubkey)
static int ecx_pub_decode(EVP_PKEY *pkey, const X509_PUBKEY *pubkey)
{
const unsigned char *p;
int pklen;