964 lines
23 KiB
Perl
964 lines
23 KiB
Perl
|
#!/usr/bin/env perl
|
|||
|
|
#
|
|||
|
|
# ====================================================================
|
|||
|
|
# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
|
|||
|
|
# project. The module is, however, dual licensed under OpenSSL and
|
|||
|
|
# CRYPTOGAMS licenses depending on where you obtain it. For further
|
|||
|
|
# details see http://www.openssl.org/~appro/cryptogams/.
|
|||
|
|
# ====================================================================
|
|||
|
|
#
|
|||
|
|
# This module implements support for Intel AES-NI extension. In
|
|||
|
|
# OpenSSL context it's used with Intel engine, but can also be used as
|
|||
|
|
# drop-in replacement for crypto/aes/asm/aes-x86_64.pl [see below for
|
|||
|
|
# details].
|
|||
|
|
#
|
|||
|
|
# TODO:
|
|||
|
|
# - Win64 SEH handlers;
|
|||
|
|
|
|||
|
|
$PREFIX="aesni"; # if $PREFIX is set to "AES", the script
|
|||
|
|
# generates drop-in replacement for
|
|||
|
|
# crypto/aes/asm/aes-x86_64.pl:-)
|
|||
|
|
|
|||
|
|
$flavour = shift;
|
|||
|
|
$output = shift;
|
|||
|
|
if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
|
|||
|
|
|
|||
|
|
$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
|
|||
|
|
|
|||
|
|
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
|
|||
|
|
( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
|
|||
|
|
( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
|
|||
|
|
die "can't locate x86_64-xlate.pl";
|
|||
|
|
|
|||
|
|
open STDOUT,"| $^X $xlate $flavour $output";
|
|||
|
|
|
|||
|
|
$movkey = $PREFIX eq "aesni" ? "movaps" : "movups";
|
|||
|
|
|
|||
|
|
$code=".text\n";
|
|||
|
|
|
|||
|
|
$rounds="%eax"; # input to and changed by aesni_[en|de]cryptN !!!
|
|||
|
|
|
|||
|
|
# this is natural argument order for public $PREFIX_*crypt...
|
|||
|
|
$inp="%rdi";
|
|||
|
|
$out="%rsi";
|
|||
|
|
# ... and for $PREFIX_[ebc|cbc]_encrypt in particular.
|
|||
|
|
$len="%rdx";
|
|||
|
|
$key="%rcx"; # input to and changed by aesni_[en|de]cryptN !!!
|
|||
|
|
$ivp="%r8"; # cbc
|
|||
|
|
|
|||
|
|
$rnds_="%r10d"; # backup copy for $rounds
|
|||
|
|
$key_="%r11"; # backup copy for $key
|
|||
|
|
|
|||
|
|
# %xmm register layout
|
|||
|
|
$inout0="%xmm0"; $inout1="%xmm1";
|
|||
|
|
$inout2="%xmm2"; $inout3="%xmm3";
|
|||
|
|
$inout4="%xmm4"; $inout5="%xmm5";
|
|||
|
|
$rndkey0="%xmm6"; $rndkey1="%xmm7";
|
|||
|
|
|
|||
|
|
$iv="%xmm8";
|
|||
|
|
$in0="%xmm9"; $in1="%xmm10";
|
|||
|
|
$in2="%xmm11"; $in3="%xmm12";
|
|||
|
|
$in4="%xmm13"; $in5="%xmm14";
|
|||
|
|
|
|||
|
|
# Inline version of internal aesni_[en|de]crypt1.
|
|||
|
|
#
|
|||
|
|
# Why folded loop? Because aes[enc|dec] is slow enough to accommodate
|
|||
|
|
# cycles which take care of loop variables...
|
|||
|
|
{ my $sn;
|
|||
|
|
sub aesni_encrypt1 {
|
|||
|
|
my ($data,$rndkey0,$rndkey1,$key,$rounds)=@_;
|
|||
|
|
++$sn;
|
|||
|
|
$code.=<<___;
|
|||
|
|
$movkey ($key),$rndkey0
|
|||
|
|
$movkey 16($key),$rndkey1
|
|||
|
|
lea 16($key),$key
|
|||
|
|
pxor $rndkey0,$data
|
|||
|
|
dec $rounds
|
|||
|
|
jmp .Loop_enc1_$sn
|
|||
|
|
.align 16
|
|||
|
|
.Loop_enc1_$sn:
|
|||
|
|
aesenc $rndkey1,$data
|
|||
|
|
dec $rounds
|
|||
|
|
lea 16($key),$key
|
|||
|
|
$movkey ($key),$rndkey1
|
|||
|
|
jnz .Loop_enc1_$sn # loop body is 16 bytes
|
|||
|
|
|
|||
|
|
aesenclast $rndkey1,$data
|
|||
|
|
___
|
|||
|
|
}}
|
|||
|
|
{ my $sn;
|
|||
|
|
sub aesni_decrypt1 {
|
|||
|
|
my ($data,$rndkey0,$rndkey1,$key,$rounds)=@_;
|
|||
|
|
++$sn;
|
|||
|
|
$code.=<<___;
|
|||
|
|
$movkey ($key),$rndkey0
|
|||
|
|
$movkey 16($key),$rndkey1
|
|||
|
|
lea 16($key),$key
|
|||
|
|
pxor $rndkey0,$data
|
|||
|
|
dec $rounds
|
|||
|
|
jmp .Loop_dec1_$sn
|
|||
|
|
.align 16
|
|||
|
|
.Loop_dec1_$sn:
|
|||
|
|
aesdec $rndkey1,$data
|
|||
|
|
dec $rounds
|
|||
|
|
lea 16($key),$key
|
|||
|
|
$movkey ($key),$rndkey1
|
|||
|
|
jnz .Loop_dec1_$sn # loop body is 16 bytes
|
|||
|
|
|
|||
|
|
aesdeclast $rndkey1,$data
|
|||
|
|
___
|
|||
|
|
}}
|
|||
|
|
|
|||
|
|
# void $PREFIX_encrypt (const void *inp,void *out,const AES_KEY *key);
|
|||
|
|
#
|
|||
|
|
$code.=<<___;
|
|||
|
|
.globl ${PREFIX}_encrypt
|
|||
|
|
.type ${PREFIX}_encrypt,\@function,3
|
|||
|
|
.align 16
|
|||
|
|
${PREFIX}_encrypt:
|
|||
|
|
movups ($inp),%xmm0 # load input
|
|||
|
|
mov 240(%rdx),$rounds # pull $rounds
|
|||
|
|
___
|
|||
|
|
&aesni_encrypt1("%xmm0","%xmm1","%xmm2","%rdx",$rounds);
|
|||
|
|
$code.=<<___;
|
|||
|
|
movups %xmm0,(%rsi) # output
|
|||
|
|
ret
|
|||
|
|
.size ${PREFIX}_encrypt,.-${PREFIX}_encrypt
|
|||
|
|
___
|
|||
|
|
|
|||
|
|
# void $PREFIX_decrypt (const void *inp,void *out,const AES_KEY *key);
|
|||
|
|
#
|
|||
|
|
$code.=<<___;
|
|||
|
|
.globl ${PREFIX}_decrypt
|
|||
|
|
.type ${PREFIX}_decrypt,\@function,3
|
|||
|
|
.align 16
|
|||
|
|
${PREFIX}_decrypt:
|
|||
|
|
movups ($inp),%xmm0 # load input
|
|||
|
|
mov 240(%rdx),$rounds # pull $rounds
|
|||
|
|
___
|
|||
|
|
&aesni_decrypt1("%xmm0","%xmm1","%xmm2","%rdx",$rounds);
|
|||
|
|
$code.=<<___;
|
|||
|
|
movups %xmm0,($out) # output
|
|||
|
|
ret
|
|||
|
|
.size ${PREFIX}_decrypt, .-${PREFIX}_decrypt
|
|||
|
|
___
|
|||
|
|
|
|||
|
|
# _aesni_[en|de]crypt6 are private interfaces, 6 denotes interleave
|
|||
|
|
# factor. Why 6x? Because aes[enc|dec] latency is 6 and 6x interleave
|
|||
|
|
# provides optimal utilization, so that subroutine's throughput is
|
|||
|
|
# virtually same for *any* number [naturally up to 6] of input blocks
|
|||
|
|
# as for non-interleaved subroutine. This is why it handles even
|
|||
|
|
# double-, tripple-, quad- and penta-block inputs. Larger interleave
|
|||
|
|
# factor, e.g. 8x, would perform suboptimally on these shorter inputs...
|
|||
|
|
sub aesni_generate6 {
|
|||
|
|
my $dir=shift;
|
|||
|
|
# As already mentioned it takes in $key and $rounds, which are *not*
|
|||
|
|
# preserved. $inout[0-5] is cipher/clear text...
|
|||
|
|
$code.=<<___;
|
|||
|
|
.type _aesni_${dir}rypt6,\@abi-omnipotent
|
|||
|
|
.align 16
|
|||
|
|
_aesni_${dir}rypt6:
|
|||
|
|
$movkey ($key),$rndkey0
|
|||
|
|
$movkey 16($key),$rndkey1
|
|||
|
|
shr \$1,$rounds
|
|||
|
|
lea 32($key),$key
|
|||
|
|
dec $rounds
|
|||
|
|
pxor $rndkey0,$inout0
|
|||
|
|
pxor $rndkey0,$inout1
|
|||
|
|
pxor $rndkey0,$inout2
|
|||
|
|
pxor $rndkey0,$inout3
|
|||
|
|
pxor $rndkey0,$inout4
|
|||
|
|
pxor $rndkey0,$inout5
|
|||
|
|
jmp .L${dir}_loop6
|
|||
|
|
.align 16
|
|||
|
|
.L${dir}_loop6:
|
|||
|
|
aes${dir} $rndkey1,$inout0
|
|||
|
|
$movkey ($key),$rndkey0
|
|||
|
|
aes${dir} $rndkey1,$inout1
|
|||
|
|
dec $rounds
|
|||
|
|
aes${dir} $rndkey1,$inout2
|
|||
|
|
aes${dir} $rndkey1,$inout3
|
|||
|
|
aes${dir} $rndkey1,$inout4
|
|||
|
|
aes${dir} $rndkey1,$inout5
|
|||
|
|
aes${dir} $rndkey0,$inout0
|
|||
|
|
$movkey 16($key),$rndkey1
|
|||
|
|
aes${dir} $rndkey0,$inout1
|
|||
|
|
lea 32($key),$key
|
|||
|
|
aes${dir} $rndkey0,$inout2
|
|||
|
|
aes${dir} $rndkey0,$inout3
|
|||
|
|
aes${dir} $rndkey0,$inout4
|
|||
|
|
aes${dir} $rndkey0,$inout5
|
|||
|
|
jnz .L${dir}_loop6
|
|||
|
|
aes${dir} $rndkey1,$inout0
|
|||
|
|
$movkey ($key),$rndkey0
|
|||
|
|
aes${dir} $rndkey1,$inout1
|
|||
|
|
aes${dir} $rndkey1,$inout2
|
|||
|
|
aes${dir} $rndkey1,$inout3
|
|||
|
|
aes${dir} $rndkey1,$inout4
|
|||
|
|
aes${dir} $rndkey1,$inout5
|
|||
|
|
aes${dir}last $rndkey0,$inout0
|
|||
|
|
aes${dir}last $rndkey0,$inout1
|
|||
|
|
aes${dir}last $rndkey0,$inout2
|
|||
|
|
aes${dir}last $rndkey0,$inout3
|
|||
|
|
aes${dir}last $rndkey0,$inout4
|
|||
|
|
aes${dir}last $rndkey0,$inout5
|
|||
|
|
ret
|
|||
|
|
.size _aesni_${dir}rypt6,.-_aesni_${dir}rypt6
|
|||
|
|
___
|
|||
|
|
}
|
|||
|
|
&aesni_generate6("enc");
|
|||
|
|
&aesni_generate6("dec");
|
|||
|
|
|
|||
|
|
if ($PREFIX eq "aesni") {
|
|||
|
|
# void aesni_ecb_encrypt (const void *in, void *out,
|
|||
|
|
# size_t length, const AES_KEY *key,
|
|||
|
|
# int enc);
|
|||
|
|
$code.=<<___;
|
|||
|
|
.globl aesni_ecb_encrypt
|
|||
|
|
.type aesni_ecb_encrypt,\@function,5
|
|||
|
|
.align 16
|
|||
|
|
aesni_ecb_encrypt:
|
|||
|
|
cmp \$16,$len # check length
|
|||
|
|
jb .Lecb_abort
|
|||
|
|
___
|
|||
|
|
$code.=<<___ if ($win64);
|
|||
|
|
lea -0x28(%rsp),%rsp
|
|||
|
|
movaps %xmm6,(%rsp)
|
|||
|
|
movaps %xmm7,16(%rsp)
|
|||
|
|
___
|
|||
|
|
$code.=<<___;
|
|||
|
|
mov 240($key),$rounds # pull $rounds
|
|||
|
|
and \$-16,$len
|
|||
|
|
mov $key,$key_ # backup $key
|
|||
|
|
test %r8d,%r8d
|
|||
|
|
mov $rounds,$rnds_ # backup $rounds
|
|||
|
|
jz .Lecb_decrypt
|
|||
|
|
#--------------------------- ECB ENCRYPT ------------------------------#
|
|||
|
|
sub \$0x60,$len
|
|||
|
|
jc .Lecb_enc_tail
|
|||
|
|
jmp .Lecb_enc_loop6
|
|||
|
|
.align 16
|
|||
|
|
.Lecb_enc_loop6:
|
|||
|
|
movups ($inp),$inout0
|
|||
|
|
movups 0x10($inp),$inout1
|
|||
|
|
movups 0x20($inp),$inout2
|
|||
|
|
movups 0x30($inp),$inout3
|
|||
|
|
movups 0x40($inp),$inout4
|
|||
|
|
movups 0x50($inp),$inout5
|
|||
|
|
call _aesni_encrypt6
|
|||
|
|
movups $inout0,($out)
|
|||
|
|
sub \$0x60,$len
|
|||
|
|
movups $inout1,0x10($out)
|
|||
|
|
lea 0x60($inp),$inp
|
|||
|
|
movups $inout2,0x20($out)
|
|||
|
|
mov $rnds_,$rounds # restore $rounds
|
|||
|
|
movups $inout3,0x30($out)
|
|||
|
|
mov $key_,$key # restore $key
|
|||
|
|
movups $inout4,0x40($out)
|
|||
|
|
movups $inout5,0x50($out)
|
|||
|
|
lea 0x60($out),$out
|
|||
|
|
jnc .Lecb_enc_loop6
|
|||
|
|
|
|||
|
|
.Lecb_enc_tail:
|
|||
|
|
add \$0x60,$len
|
|||
|
|
jz .Lecb_ret
|
|||
|
|
|
|||
|
|
cmp \$0x10,$len
|
|||
|
|
movups ($inp),$inout0
|
|||
|
|
je .Lecb_enc_one
|
|||
|
|
cmp \$0x20,$len
|
|||
|
|
movups 0x10($inp),$inout1
|
|||
|
|
je .Lecb_enc_two
|
|||
|
|
cmp \$0x30,$len
|
|||
|
|
movups 0x20($inp),$inout2
|
|||
|
|
je .Lecb_enc_three
|
|||
|
|
cmp \$0x40,$len
|
|||
|
|
movups 0x30($inp),$inout3
|
|||
|
|
je .Lecb_enc_four
|
|||
|
|
movups 0x40($inp),$inout4
|
|||
|
|
call _aesni_encrypt6
|
|||
|
|
movups $inout0,($out)
|
|||
|
|
movups $inout1,0x10($out)
|
|||
|
|
movups $inout2,0x20($out)
|
|||
|
|
movups $inout3,0x30($out)
|
|||
|
|
movups $inout4,0x40($out)
|
|||
|
|
jmp .Lecb_ret
|
|||
|
|
.align 16
|
|||
|
|
.Lecb_enc_one:
|
|||
|
|
___
|
|||
|
|
&aesni_encrypt1($inout0,$rndkey0,$rndkey1,$key,$rounds);
|
|||
|
|
$code.=<<___;
|
|||
|
|
movups $inout0,($out)
|
|||
|
|
jmp .Lecb_ret
|
|||
|
|
.align 16
|
|||
|
|
.Lecb_enc_two:
|
|||
|
|
call _aesni_encrypt6
|
|||
|
|
movups $inout0,($out)
|
|||
|
|
movups $inout1,0x10($out)
|
|||
|
|
jmp .Lecb_ret
|
|||
|
|
.align 16
|
|||
|
|
.Lecb_enc_three:
|
|||
|
|
call _aesni_encrypt6
|
|||
|
|
movups $inout0,($out)
|
|||
|
|
movups $inout1,0x10($out)
|
|||
|
|
movups $inout2,0x20($out)
|
|||
|
|
jmp .Lecb_ret
|
|||
|
|
.align 16
|
|||
|
|
.Lecb_enc_four:
|
|||
|
|
call _aesni_encrypt6
|
|||
|
|
movups $inout0,($out)
|
|||
|
|
movups $inout1,0x10($out)
|
|||
|
|
movups $inout2,0x20($out)
|
|||
|
|
movups $inout3,0x30($out)
|
|||
|
|
jmp .Lecb_ret
|
|||
|
|
#--------------------------- ECB DECRYPT ------------------------------#
|
|||
|
|
.align 16
|
|||
|
|
.Lecb_decrypt:
|
|||
|
|
sub \$0x60,$len
|
|||
|
|
jc .Lecb_dec_tail
|
|||
|
|
jmp .Lecb_dec_loop6
|
|||
|
|
.align 16
|
|||
|
|
.Lecb_dec_loop6:
|
|||
|
|
movups ($inp),$inout0
|
|||
|
|
movups 0x10($inp),$inout1
|
|||
|
|
movups 0x20($inp),$inout2
|
|||
|
|
movups 0x30($inp),$inout3
|
|||
|
|
movups 0x40($inp),$inout4
|
|||
|
|
movups 0x50($inp),$inout5
|
|||
|
|
call _aesni_decrypt6
|
|||
|
|
movups $inout0,($out)
|
|||
|
|
sub \$0x60,$len
|
|||
|
|
movups $inout1,0x10($out)
|
|||
|
|
lea 0x60($inp),$inp
|
|||
|
|
movups $inout2,0x20($out)
|
|||
|
|
mov $rnds_,$rounds # restore $rounds
|
|||
|
|
movups $inout3,0x30($out)
|
|||
|
|
mov $key_,$key # restore $key
|
|||
|
|
movups $inout4,0x40($out)
|
|||
|
|
movups $inout5,0x50($out)
|
|||
|
|
lea 0x60($out),$out
|
|||
|
|
jnc .Lecb_dec_loop6
|
|||
|
|
|
|||
|
|
.Lecb_dec_tail:
|
|||
|
|
add \$0x60,$len
|
|||
|
|
jz .Lecb_ret
|
|||
|
|
|
|||
|
|
cmp \$0x10,$len
|
|||
|
|
movups ($inp),$inout0
|
|||
|
|
je .Lecb_dec_one
|
|||
|
|
cmp \$0x20,$len
|
|||
|
|
movups 0x10($inp),$inout1
|
|||
|
|
je .Lecb_dec_two
|
|||
|
|
cmp \$0x30,$len
|
|||
|
|
movups 0x20($inp),$inout2
|
|||
|
|
je .Lecb_dec_three
|
|||
|
|
cmp \$0x40,$len
|
|||
|
|
movups 0x30($inp),$inout3
|
|||
|
|
je .Lecb_dec_four
|
|||
|
|
movups 0x40($inp),$inout4
|
|||
|
|
call _aesni_decrypt6
|
|||
|
|
movups $inout0,($out)
|
|||
|
|
movups $inout1,0x10($out)
|
|||
|
|
movups $inout2,0x20($out)
|
|||
|
|
movups $inout3,0x30($out)
|
|||
|
|
movups $inout4,0x40($out)
|
|||
|
|
jmp .Lecb_ret
|
|||
|
|
.align 16
|
|||
|
|
.Lecb_dec_one:
|
|||
|
|
___
|
|||
|
|
&aesni_decrypt1($inout0,$rndkey0,$rndkey1,$key,$rounds);
|
|||
|
|
$code.=<<___;
|
|||
|
|
movups $inout0,($out)
|
|||
|
|
jmp .Lecb_ret
|
|||
|
|
.align 16
|
|||
|
|
.Lecb_dec_two:
|
|||
|
|
call _aesni_decrypt6
|
|||
|
|
movups $inout0,($out)
|
|||
|
|
movups $inout1,0x10($out)
|
|||
|
|
jmp .Lecb_ret
|
|||
|
|
.align 16
|
|||
|
|
.Lecb_dec_three:
|
|||
|
|
call _aesni_decrypt6
|
|||
|
|
movups $inout0,($out)
|
|||
|
|
movups $inout1,0x10($out)
|
|||
|
|
movups $inout2,0x20($out)
|
|||
|
|
jmp .Lecb_ret
|
|||
|
|
.align 16
|
|||
|
|
.Lecb_dec_four:
|
|||
|
|
call _aesni_decrypt6
|
|||
|
|
movups $inout0,($out)
|
|||
|
|
movups $inout1,0x10($out)
|
|||
|
|
movups $inout2,0x20($out)
|
|||
|
|
movups $inout3,0x30($out)
|
|||
|
|
|
|||
|
|
.Lecb_ret:
|
|||
|
|
___
|
|||
|
|
$code.=<<___ if ($win64);
|
|||
|
|
movaps (%rsp),%xmm6
|
|||
|
|
movaps 0x10(%rsp),%xmm7
|
|||
|
|
lea 0x28(%rsp),%rsp
|
|||
|
|
___
|
|||
|
|
$code.=<<___;
|
|||
|
|
.Lecb_abort:
|
|||
|
|
ret
|
|||
|
|
.size aesni_ecb_encrypt,.-aesni_ecb_encrypt
|
|||
|
|
___
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
# void $PREFIX_cbc_encrypt (const void *inp, void *out,
|
|||
|
|
# size_t length, const AES_KEY *key,
|
|||
|
|
# unsigned char *ivp,const int enc);
|
|||
|
|
$reserved = $win64?0x90:-0x18; # used in decrypt
|
|||
|
|
$code.=<<___;
|
|||
|
|
.globl ${PREFIX}_cbc_encrypt
|
|||
|
|
.type ${PREFIX}_cbc_encrypt,\@function,6
|
|||
|
|
.align 16
|
|||
|
|
${PREFIX}_cbc_encrypt:
|
|||
|
|
test $len,$len # check length
|
|||
|
|
jz .Lcbc_ret
|
|||
|
|
mov 240($key),$rounds # pull $rounds
|
|||
|
|
mov $key,$key_ # backup $key
|
|||
|
|
test %r9d,%r9d
|
|||
|
|
mov $rounds,$rnds_ # backup $rounds
|
|||
|
|
jz .Lcbc_decrypt
|
|||
|
|
#--------------------------- CBC ENCRYPT ------------------------------#
|
|||
|
|
movups ($ivp),%xmm0 # load iv as initial state
|
|||
|
|
cmp \$16,$len
|
|||
|
|
jb .Lcbc_enc_tail
|
|||
|
|
sub \$16,$len
|
|||
|
|
jmp .Lcbc_enc_loop
|
|||
|
|
.align 16
|
|||
|
|
.Lcbc_enc_loop:
|
|||
|
|
movups ($inp),%xmm2 # load input
|
|||
|
|
lea 16($inp),$inp
|
|||
|
|
pxor %xmm2,%xmm0
|
|||
|
|
___
|
|||
|
|
&aesni_encrypt1("%xmm0","%xmm1","%xmm2",$key,$rounds);
|
|||
|
|
$code.=<<___;
|
|||
|
|
movups %xmm0,($out) # store output
|
|||
|
|
sub \$16,$len
|
|||
|
|
lea 16($out),$out
|
|||
|
|
mov $rnds_,$rounds # restore $rounds
|
|||
|
|
mov $key_,$key # restore $key
|
|||
|
|
jnc .Lcbc_enc_loop
|
|||
|
|
add \$16,$len
|
|||
|
|
jnz .Lcbc_enc_tail
|
|||
|
|
movups %xmm0,($ivp)
|
|||
|
|
jmp .Lcbc_ret
|
|||
|
|
|
|||
|
|
.Lcbc_enc_tail:
|
|||
|
|
mov $len,%rcx # zaps $key
|
|||
|
|
xchg $inp,$out # $inp is %rsi and $out is %rdi now
|
|||
|
|
.long 0x9066A4F3 # rep movsb
|
|||
|
|
mov \$16,%ecx # zero tail
|
|||
|
|
sub $len,%rcx
|
|||
|
|
xor %eax,%eax
|
|||
|
|
.long 0x9066AAF3 # rep stosb
|
|||
|
|
lea -16(%rdi),%rdi # rewind $out by 1 block
|
|||
|
|
mov $rnds_,$rounds # restore $rounds
|
|||
|
|
mov %rdi,%rsi # $inp and $out are the same
|
|||
|
|
mov $key_,$key # restore $key
|
|||
|
|
xor $len,$len # len=16
|
|||
|
|
jmp .Lcbc_enc_loop # one more spin
|
|||
|
|
#--------------------------- CBC DECRYPT ------------------------------#
|
|||
|
|
.align 16
|
|||
|
|
.Lcbc_decrypt:
|
|||
|
|
___
|
|||
|
|
$code.=<<___ if ($win64);
|
|||
|
|
lea -0xa8(%rsp),%rsp
|
|||
|
|
movaps %xmm6,(%rsp)
|
|||
|
|
movaps %xmm7,0x10(%rsp)
|
|||
|
|
movaps %xmm8,0x20(%rsp)
|
|||
|
|
movaps %xmm9,0x30(%rsp)
|
|||
|
|
movaps %xmm10,0x40(%rsp)
|
|||
|
|
movaps %xmm11,0x50(%rsp)
|
|||
|
|
movaps %xmm12,0x60(%rsp)
|
|||
|
|
movaps %xmm13,0x70(%rsp)
|
|||
|
|
movaps %xmm14,0x80(%rsp)
|
|||
|
|
___
|
|||
|
|
$code.=<<___;
|
|||
|
|
movups ($ivp),$iv
|
|||
|
|
sub \$0x60,$len
|
|||
|
|
jc .Lcbc_dec_tail
|
|||
|
|
jmp .Lcbc_dec_loop6
|
|||
|
|
.align 16
|
|||
|
|
.Lcbc_dec_loop6:
|
|||
|
|
movups ($inp),$inout0
|
|||
|
|
movups 0x10($inp),$inout1
|
|||
|
|
movups 0x20($inp),$inout2
|
|||
|
|
movups 0x30($inp),$inout3
|
|||
|
|
movaps $inout0,$in0
|
|||
|
|
movups 0x40($inp),$inout4
|
|||
|
|
movaps $inout1,$in1
|
|||
|
|
movups 0x50($inp),$inout5
|
|||
|
|
movaps $inout2,$in2
|
|||
|
|
movaps $inout3,$in3
|
|||
|
|
movaps $inout4,$in4
|
|||
|
|
movaps $inout5,$in5
|
|||
|
|
call _aesni_decrypt6
|
|||
|
|
pxor $iv,$inout0
|
|||
|
|
pxor $in0,$inout1
|
|||
|
|
movups $inout0,($out)
|
|||
|
|
sub \$0x60,$len
|
|||
|
|
pxor $in1,$inout2
|
|||
|
|
movups $inout1,0x10($out)
|
|||
|
|
lea 0x60($inp),$inp
|
|||
|
|
pxor $in2,$inout3
|
|||
|
|
movups $inout2,0x20($out)
|
|||
|
|
mov $rnds_,$rounds # restore $rounds
|
|||
|
|
pxor $in3,$inout4
|
|||
|
|
movups $inout3,0x30($out)
|
|||
|
|
mov $key_,$key # restore $key
|
|||
|
|
pxor $in4,$inout5
|
|||
|
|
movups $inout4,0x40($out)
|
|||
|
|
movaps $in5,$iv
|
|||
|
|
movups $inout5,0x50($out)
|
|||
|
|
lea 0x60($out),$out
|
|||
|
|
jnc .Lcbc_dec_loop6
|
|||
|
|
|
|||
|
|
.Lcbc_dec_tail:
|
|||
|
|
add \$0x60,$len
|
|||
|
|
movups $iv,($ivp)
|
|||
|
|
jz .Lcbc_dec_ret
|
|||
|
|
|
|||
|
|
movups ($inp),$inout0
|
|||
|
|
cmp \$0x10,$len
|
|||
|
|
movaps $inout0,$in0
|
|||
|
|
jbe .Lcbc_dec_one
|
|||
|
|
movups 0x10($inp),$inout1
|
|||
|
|
cmp \$0x20,$len
|
|||
|
|
movaps $inout1,$in1
|
|||
|
|
jbe .Lcbc_dec_two
|
|||
|
|
movups 0x20($inp),$inout2
|
|||
|
|
cmp \$0x30,$len
|
|||
|
|
movaps $inout2,$in2
|
|||
|
|
jbe .Lcbc_dec_three
|
|||
|
|
movups 0x30($inp),$inout3
|
|||
|
|
cmp \$0x40,$len
|
|||
|
|
movaps $inout3,$in3
|
|||
|
|
jbe .Lcbc_dec_four
|
|||
|
|
movups 0x40($inp),$inout4
|
|||
|
|
cmp \$0x50,$len
|
|||
|
|
movaps $inout4,$in4
|
|||
|
|
jbe .Lcbc_dec_five
|
|||
|
|
movups 0x50($inp),$inout5
|
|||
|
|
movaps $inout5,$in5
|
|||
|
|
call _aesni_decrypt6
|
|||
|
|
pxor $iv,$inout0
|
|||
|
|
pxor $in0,$inout1
|
|||
|
|
movups $inout0,($out)
|
|||
|
|
pxor $in1,$inout2
|
|||
|
|
movups $inout1,0x10($out)
|
|||
|
|
pxor $in2,$inout3
|
|||
|
|
movups $inout2,0x20($out)
|
|||
|
|
pxor $in3,$inout4
|
|||
|
|
movups $inout3,0x30($out)
|
|||
|
|
pxor $in4,$inout5
|
|||
|
|
movups $inout4,0x40($out)
|
|||
|
|
movaps $in5,$iv
|
|||
|
|
movaps $inout5,$inout0
|
|||
|
|
lea 0x50($out),$out
|
|||
|
|
jmp .Lcbc_dec_tail_collected
|
|||
|
|
.align 16
|
|||
|
|
.Lcbc_dec_one:
|
|||
|
|
___
|
|||
|
|
&aesni_decrypt1($inout0,$rndkey0,$rndkey1,$key,$rounds);
|
|||
|
|
$code.=<<___;
|
|||
|
|
pxor $iv,$inout0
|
|||
|
|
movaps $in0,$iv
|
|||
|
|
jmp .Lcbc_dec_tail_collected
|
|||
|
|
.align 16
|
|||
|
|
.Lcbc_dec_two:
|
|||
|
|
call _aesni_decrypt6
|
|||
|
|
pxor $iv,$inout0
|
|||
|
|
pxor $in0,$inout1
|
|||
|
|
movups $inout0,($out)
|
|||
|
|
movaps $in1,$iv
|
|||
|
|
movaps $inout1,$inout0
|
|||
|
|
lea 0x10($out),$out
|
|||
|
|
jmp .Lcbc_dec_tail_collected
|
|||
|
|
.align 16
|
|||
|
|
.Lcbc_dec_three:
|
|||
|
|
call _aesni_decrypt6
|
|||
|
|
pxor $iv,$inout0
|
|||
|
|
pxor $in0,$inout1
|
|||
|
|
movups $inout0,($out)
|
|||
|
|
pxor $in1,$inout2
|
|||
|
|
movups $inout1,0x10($out)
|
|||
|
|
movaps $in2,$iv
|
|||
|
|
movaps $inout2,$inout0
|
|||
|
|
lea 0x20($out),$out
|
|||
|
|
jmp .Lcbc_dec_tail_collected
|
|||
|
|
.align 16
|
|||
|
|
.Lcbc_dec_four:
|
|||
|
|
call _aesni_decrypt6
|
|||
|
|
pxor $iv,$inout0
|
|||
|
|
pxor $in0,$inout1
|
|||
|
|
movups $inout0,($out)
|
|||
|
|
pxor $in1,$inout2
|
|||
|
|
movups $inout1,0x10($out)
|
|||
|
|
pxor $in2,$inout3
|
|||
|
|
movups $inout2,0x20($out)
|
|||
|
|
movaps $in3,$iv
|
|||
|
|
movaps $inout3,$inout0
|
|||
|
|
lea 0x30($out),$out
|
|||
|
|
jmp .Lcbc_dec_tail_collected
|
|||
|
|
.align 16
|
|||
|
|
.Lcbc_dec_five:
|
|||
|
|
call _aesni_decrypt6
|
|||
|
|
pxor $iv,$inout0
|
|||
|
|
pxor $in0,$inout1
|
|||
|
|
movups $inout0,($out)
|
|||
|
|
pxor $in1,$inout2
|
|||
|
|
movups $inout1,0x10($out)
|
|||
|
|
pxor $in2,$inout3
|
|||
|
|
movups $inout2,0x20($out)
|
|||
|
|
pxor $in3,$inout4
|
|||
|
|
movups $inout3,0x30($out)
|
|||
|
|
movaps $in4,$iv
|
|||
|
|
movaps $inout4,$inout0
|
|||
|
|
lea 0x40($out),$out
|
|||
|
|
jmp .Lcbc_dec_tail_collected
|
|||
|
|
.align 16
|
|||
|
|
.Lcbc_dec_tail_collected:
|
|||
|
|
and \$15,$len
|
|||
|
|
movups $iv,($ivp)
|
|||
|
|
jnz .Lcbc_dec_tail_partial
|
|||
|
|
movups $inout0,($out)
|
|||
|
|
jmp .Lcbc_dec_ret
|
|||
|
|
.Lcbc_dec_tail_partial:
|
|||
|
|
movaps $inout0,$reserved(%rsp)
|
|||
|
|
mov $out,%rdi
|
|||
|
|
mov $len,%rcx
|
|||
|
|
lea $reserved(%rsp),%rsi
|
|||
|
|
.long 0x9066A4F3 # rep movsb
|
|||
|
|
|
|||
|
|
.Lcbc_dec_ret:
|
|||
|
|
___
|
|||
|
|
$code.=<<___ if ($win64);
|
|||
|
|
movaps (%rsp),%xmm6
|
|||
|
|
movaps 0x10(%rsp),%xmm7
|
|||
|
|
movaps 0x20(%rsp),%xmm8
|
|||
|
|
movaps 0x30(%rsp),%xmm9
|
|||
|
|
movaps 0x40(%rsp),%xmm10
|
|||
|
|
movaps 0x50(%rsp),%xmm11
|
|||
|
|
movaps 0x60(%rsp),%xmm12
|
|||
|
|
movaps 0x70(%rsp),%xmm13
|
|||
|
|
movaps 0x80(%rsp),%xmm14
|
|||
|
|
lea 0xa8(%rsp),%rsp
|
|||
|
|
___
|
|||
|
|
$code.=<<___;
|
|||
|
|
.Lcbc_ret:
|
|||
|
|
ret
|
|||
|
|
.size ${PREFIX}_cbc_encrypt,.-${PREFIX}_cbc_encrypt
|
|||
|
|
___
|
|||
|
|
|
|||
|
|
{
|
|||
|
|
# this is natural argument order for $PREFIX_set_[en|de]crypt_key
|
|||
|
|
my $inp="%rdi";
|
|||
|
|
my $bits="%esi";
|
|||
|
|
my $key="%rdx";
|
|||
|
|
|
|||
|
|
# int $PREFIX_set_encrypt_key (const unsigned char *userKey, int bits,
|
|||
|
|
# AES_KEY *key)
|
|||
|
|
$code.=<<___;
|
|||
|
|
.globl ${PREFIX}_set_encrypt_key
|
|||
|
|
.type ${PREFIX}_set_encrypt_key,\@function,3
|
|||
|
|
.align 16
|
|||
|
|
${PREFIX}_set_encrypt_key:
|
|||
|
|
call _aesni_set_encrypt_key
|
|||
|
|
ret
|
|||
|
|
.size ${PREFIX}_set_encrypt_key,.-${PREFIX}_set_encrypt_key
|
|||
|
|
___
|
|||
|
|
# int $PREFIX_set_decrypt_key(const unsigned char *userKey, const int bits,
|
|||
|
|
# AES_KEY *key)
|
|||
|
|
$code.=<<___;
|
|||
|
|
.globl ${PREFIX}_set_decrypt_key
|
|||
|
|
.type ${PREFIX}_set_decrypt_key,\@function,3
|
|||
|
|
.align 16
|
|||
|
|
${PREFIX}_set_decrypt_key:
|
|||
|
|
call _aesni_set_encrypt_key
|
|||
|
|
shl \$4,%esi # actually rounds after _aesni_set_encrypt_key
|
|||
|
|
test %eax,%eax
|
|||
|
|
jnz .Ldec_key_ret
|
|||
|
|
lea (%rdx,%rsi),%rsi# points at the end of key schedule
|
|||
|
|
|
|||
|
|
$movkey (%rdx),%xmm0 # just swap
|
|||
|
|
$movkey (%rsi),%xmm1
|
|||
|
|
$movkey %xmm0,(%rsi)
|
|||
|
|
$movkey %xmm1,(%rdx)
|
|||
|
|
lea 16(%rdx),%rdx
|
|||
|
|
lea -16(%rsi),%rsi
|
|||
|
|
jmp .Ldec_key_inverse
|
|||
|
|
.align 16
|
|||
|
|
.Ldec_key_inverse:
|
|||
|
|
$movkey (%rdx),%xmm0 # swap and inverse
|
|||
|
|
$movkey (%rsi),%xmm1
|
|||
|
|
aesimc %xmm0,%xmm0
|
|||
|
|
aesimc %xmm1,%xmm1
|
|||
|
|
lea 16(%rdx),%rdx
|
|||
|
|
lea -16(%rsi),%rsi
|
|||
|
|
cmp %rdx,%rsi
|
|||
|
|
$movkey %xmm0,16(%rsi)
|
|||
|
|
$movkey %xmm1,-16(%rdx)
|
|||
|
|
ja .Ldec_key_inverse
|
|||
|
|
|
|||
|
|
$movkey (%rdx),%xmm0 # inverse middle
|
|||
|
|
aesimc %xmm0,%xmm0
|
|||
|
|
$movkey %xmm0,(%rsi)
|
|||
|
|
.Ldec_key_ret:
|
|||
|
|
ret
|
|||
|
|
.size ${PREFIX}_set_decrypt_key,.-${PREFIX}_set_decrypt_key
|
|||
|
|
___
|
|||
|
|
|
|||
|
|
# This is based on submission by
|
|||
|
|
#
|
|||
|
|
# Huang Ying <ying.huang@intel.com>
|
|||
|
|
# Vinodh Gopal <vinodh.gopal@intel.com>
|
|||
|
|
# Kahraman Akdemir
|
|||
|
|
#
|
|||
|
|
# Agressively optimized in respect to aeskeygenassist's critical path
|
|||
|
|
# and is contained in %xmm0-5 to meet Win64 ABI requirement.
|
|||
|
|
#
|
|||
|
|
$code.=<<___;
|
|||
|
|
.type _aesni_set_encrypt_key,\@abi-omnipotent
|
|||
|
|
.align 16
|
|||
|
|
_aesni_set_encrypt_key:
|
|||
|
|
test %rdi,%rdi
|
|||
|
|
jz .Lbad_pointer
|
|||
|
|
test %rdx,%rdx
|
|||
|
|
jz .Lbad_pointer
|
|||
|
|
|
|||
|
|
movups (%rdi),%xmm0 # pull first 128 bits of *userKey
|
|||
|
|
pxor %xmm4,%xmm4 # low dword of xmm4 is assumed 0
|
|||
|
|
lea 16(%rdx),%rcx
|
|||
|
|
cmp \$256,%esi
|
|||
|
|
je .L14rounds
|
|||
|
|
cmp \$192,%esi
|
|||
|
|
je .L12rounds
|
|||
|
|
cmp \$128,%esi
|
|||
|
|
jne .Lbad_keybits
|
|||
|
|
|
|||
|
|
.L10rounds:
|
|||
|
|
mov \$10,%esi # 10 rounds for 128-bit key
|
|||
|
|
$movkey %xmm0,(%rdx) # round 0
|
|||
|
|
aeskeygenassist \$0x1,%xmm0,%xmm1 # round 1
|
|||
|
|
call .Lkey_expansion_128_cold
|
|||
|
|
aeskeygenassist \$0x2,%xmm0,%xmm1 # round 2
|
|||
|
|
call .Lkey_expansion_128
|
|||
|
|
aeskeygenassist \$0x4,%xmm0,%xmm1 # round 3
|
|||
|
|
call .Lkey_expansion_128
|
|||
|
|
aeskeygenassist \$0x8,%xmm0,%xmm1 # round 4
|
|||
|
|
call .Lkey_expansion_128
|
|||
|
|
aeskeygenassist \$0x10,%xmm0,%xmm1 # round 5
|
|||
|
|
call .Lkey_expansion_128
|
|||
|
|
aeskeygenassist \$0x20,%xmm0,%xmm1 # round 6
|
|||
|
|
call .Lkey_expansion_128
|
|||
|
|
aeskeygenassist \$0x40,%xmm0,%xmm1 # round 7
|
|||
|
|
call .Lkey_expansion_128
|
|||
|
|
aeskeygenassist \$0x80,%xmm0,%xmm1 # round 8
|
|||
|
|
call .Lkey_expansion_128
|
|||
|
|
aeskeygenassist \$0x1b,%xmm0,%xmm1 # round 9
|
|||
|
|
call .Lkey_expansion_128
|
|||
|
|
aeskeygenassist \$0x36,%xmm0,%xmm1 # round 10
|
|||
|
|
call .Lkey_expansion_128
|
|||
|
|
$movkey %xmm0,(%rcx)
|
|||
|
|
mov %esi,80(%rcx) # 240(%rdx)
|
|||
|
|
xor %eax,%eax
|
|||
|
|
ret
|
|||
|
|
|
|||
|
|
.align 16
|
|||
|
|
.Lkey_expansion_128:
|
|||
|
|
$movkey %xmm0,(%rcx)
|
|||
|
|
lea 16(%rcx),%rcx
|
|||
|
|
.Lkey_expansion_128_cold:
|
|||
|
|
shufps \$0b00010000,%xmm0,%xmm4
|
|||
|
|
pxor %xmm4, %xmm0
|
|||
|
|
shufps \$0b10001100,%xmm0,%xmm4
|
|||
|
|
pxor %xmm4, %xmm0
|
|||
|
|
pshufd \$0b11111111,%xmm1,%xmm1 # critical path
|
|||
|
|
pxor %xmm1,%xmm0
|
|||
|
|
ret
|
|||
|
|
|
|||
|
|
.align 16
|
|||
|
|
.L12rounds:
|
|||
|
|
movq 16(%rdi),%xmm2 # remaining 1/3 of *userKey
|
|||
|
|
mov \$12,%esi # 12 rounds for 192
|
|||
|
|
$movkey %xmm0,(%rdx) # round 0
|
|||
|
|
aeskeygenassist \$0x1,%xmm2,%xmm1 # round 1,2
|
|||
|
|
call .Lkey_expansion_192a_cold
|
|||
|
|
aeskeygenassist \$0x2,%xmm2,%xmm1 # round 2,3
|
|||
|
|
call .Lkey_expansion_192b
|
|||
|
|
aeskeygenassist \$0x4,%xmm2,%xmm1 # round 4,5
|
|||
|
|
call .Lkey_expansion_192a
|
|||
|
|
aeskeygenassist \$0x8,%xmm2,%xmm1 # round 5,6
|
|||
|
|
call .Lkey_expansion_192b
|
|||
|
|
aeskeygenassist \$0x10,%xmm2,%xmm1 # round 7,8
|
|||
|
|
call .Lkey_expansion_192a
|
|||
|
|
aeskeygenassist \$0x20,%xmm2,%xmm1 # round 8,9
|
|||
|
|
call .Lkey_expansion_192b
|
|||
|
|
aeskeygenassist \$0x40,%xmm2,%xmm1 # round 10,11
|
|||
|
|
call .Lkey_expansion_192a
|
|||
|
|
aeskeygenassist \$0x80,%xmm2,%xmm1 # round 11,12
|
|||
|
|
call .Lkey_expansion_192b
|
|||
|
|
$movkey %xmm0,(%rcx)
|
|||
|
|
mov %esi,48(%rcx) # 240(%rdx)
|
|||
|
|
xor %rax, %rax
|
|||
|
|
ret
|
|||
|
|
|
|||
|
|
.align 16
|
|||
|
|
.Lkey_expansion_192a:
|
|||
|
|
$movkey %xmm0,(%rcx)
|
|||
|
|
lea 16(%rcx),%rcx
|
|||
|
|
.Lkey_expansion_192a_cold:
|
|||
|
|
movaps %xmm2, %xmm5
|
|||
|
|
.Lkey_expansion_192b_warm:
|
|||
|
|
shufps \$0b00010000,%xmm0,%xmm4
|
|||
|
|
movaps %xmm2,%xmm3
|
|||
|
|
pxor %xmm4,%xmm0
|
|||
|
|
shufps \$0b10001100,%xmm0,%xmm4
|
|||
|
|
pslldq \$4,%xmm3
|
|||
|
|
pxor %xmm4,%xmm0
|
|||
|
|
pshufd \$0b01010101,%xmm1,%xmm1 # critical path
|
|||
|
|
pxor %xmm3,%xmm2
|
|||
|
|
pxor %xmm1,%xmm0
|
|||
|
|
pshufd \$0b11111111,%xmm0,%xmm3
|
|||
|
|
pxor %xmm3,%xmm2
|
|||
|
|
ret
|
|||
|
|
|
|||
|
|
.align 16
|
|||
|
|
.Lkey_expansion_192b:
|
|||
|
|
movaps %xmm0,%xmm3
|
|||
|
|
shufps \$0b01000100,%xmm0,%xmm5
|
|||
|
|
$movkey %xmm5,(%rcx)
|
|||
|
|
shufps \$0b01001110,%xmm2,%xmm3
|
|||
|
|
$movkey %xmm3,16(%rcx)
|
|||
|
|
lea 32(%rcx),%rcx
|
|||
|
|
jmp .Lkey_expansion_192b_warm
|
|||
|
|
|
|||
|
|
.align 16
|
|||
|
|
.L14rounds:
|
|||
|
|
movups 16(%rdi),%xmm2 # remaning half of *userKey
|
|||
|
|
mov \$14,%esi # 14 rounds for 256
|
|||
|
|
lea 16(%rcx),%rcx
|
|||
|
|
$movkey %xmm0,(%rdx) # round 0
|
|||
|
|
$movkey %xmm2,16(%rdx) # round 1
|
|||
|
|
aeskeygenassist \$0x1,%xmm2,%xmm1 # round 2
|
|||
|
|
call .Lkey_expansion_256a_cold
|
|||
|
|
aeskeygenassist \$0x1,%xmm0,%xmm1 # round 3
|
|||
|
|
call .Lkey_expansion_256b
|
|||
|
|
aeskeygenassist \$0x2,%xmm2,%xmm1 # round 4
|
|||
|
|
call .Lkey_expansion_256a
|
|||
|
|
aeskeygenassist \$0x2,%xmm0,%xmm1 # round 5
|
|||
|
|
call .Lkey_expansion_256b
|
|||
|
|
aeskeygenassist \$0x4,%xmm2,%xmm1 # round 6
|
|||
|
|
call .Lkey_expansion_256a
|
|||
|
|
aeskeygenassist \$0x4,%xmm0,%xmm1 # round 7
|
|||
|
|
call .Lkey_expansion_256b
|
|||
|
|
aeskeygenassist \$0x8,%xmm2,%xmm1 # round 8
|
|||
|
|
call .Lkey_expansion_256a
|
|||
|
|
aeskeygenassist \$0x8,%xmm0,%xmm1 # round 9
|
|||
|
|
call .Lkey_expansion_256b
|
|||
|
|
aeskeygenassist \$0x10,%xmm2,%xmm1 # round 10
|
|||
|
|
call .Lkey_expansion_256a
|
|||
|
|
aeskeygenassist \$0x10,%xmm0,%xmm1 # round 11
|
|||
|
|
call .Lkey_expansion_256b
|
|||
|
|
aeskeygenassist \$0x20,%xmm2,%xmm1 # round 12
|
|||
|
|
call .Lkey_expansion_256a
|
|||
|
|
aeskeygenassist \$0x20,%xmm0,%xmm1 # round 13
|
|||
|
|
call .Lkey_expansion_256b
|
|||
|
|
aeskeygenassist \$0x40,%xmm2,%xmm1 # round 14
|
|||
|
|
call .Lkey_expansion_256a
|
|||
|
|
$movkey %xmm0,(%rcx)
|
|||
|
|
mov %esi,16(%rcx) # 240(%rdx)
|
|||
|
|
xor %rax,%rax
|
|||
|
|
ret
|
|||
|
|
|
|||
|
|
.align 16
|
|||
|
|
.Lkey_expansion_256a:
|
|||
|
|
$movkey %xmm2,(%rcx)
|
|||
|
|
lea 16(%rcx),%rcx
|
|||
|
|
.Lkey_expansion_256a_cold:
|
|||
|
|
shufps \$0b00010000,%xmm0,%xmm4
|
|||
|
|
pxor %xmm4,%xmm0
|
|||
|
|
shufps \$0b10001100,%xmm0,%xmm4
|
|||
|
|
pxor %xmm4,%xmm0
|
|||
|
|
pshufd \$0b11111111,%xmm1,%xmm1 # critical path
|
|||
|
|
pxor %xmm1,%xmm0
|
|||
|
|
ret
|
|||
|
|
|
|||
|
|
.align 16
|
|||
|
|
.Lkey_expansion_256b:
|
|||
|
|
$movkey %xmm0,(%rcx)
|
|||
|
|
lea 16(%rcx),%rcx
|
|||
|
|
|
|||
|
|
shufps \$0b00010000,%xmm2,%xmm4
|
|||
|
|
pxor %xmm4,%xmm2
|
|||
|
|
shufps \$0b10001100,%xmm2,%xmm4
|
|||
|
|
pxor %xmm4,%xmm2
|
|||
|
|
pshufd \$0b10101010,%xmm1,%xmm1 # critical path
|
|||
|
|
pxor %xmm1,%xmm2
|
|||
|
|
ret
|
|||
|
|
|
|||
|
|
.align 16
|
|||
|
|
.Lbad_pointer:
|
|||
|
|
mov \$-1, %rax
|
|||
|
|
ret
|
|||
|
|
.Lbad_keybits:
|
|||
|
|
mov \$-2, %rax
|
|||
|
|
ret
|
|||
|
|
.size _aesni_set_encrypt_key,.-_aesni_set_encrypt_key
|
|||
|
|
___
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
$code.=<<___;
|
|||
|
|
.asciz "AES for Intel AES-NI, CRYPTOGAMS by <appro\@openssl.org>"
|
|||
|
|
.align 64
|
|||
|
|
___
|
|||
|
|
|
|||
|
|
sub rex {
|
|||
|
|
local *opcode=shift;
|
|||
|
|
my ($dst,$src)=@_;
|
|||
|
|
|
|||
|
|
if ($dst>=8 || $src>=8) {
|
|||
|
|
$rex=0x40;
|
|||
|
|
$rex|=0x04 if($dst>=8);
|
|||
|
|
$rex|=0x01 if($src>=8);
|
|||
|
|
push @opcode,$rex;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
sub aesni {
|
|||
|
|
my $line=shift;
|
|||
|
|
my @opcode=(0x66);
|
|||
|
|
|
|||
|
|
if ($line=~/(aeskeygenassist)\s+\$([x0-9a-f]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) {
|
|||
|
|
rex(\@opcode,$4,$3);
|
|||
|
|
push @opcode,0x0f,0x3a,0xdf;
|
|||
|
|
push @opcode,0xc0|($3&7)|(($4&7)<<3); # ModR/M
|
|||
|
|
my $c=$2;
|
|||
|
|
push @opcode,$c=~/^0/?oct($c):$c;
|
|||
|
|
return ".byte\t".join(',',@opcode);
|
|||
|
|
}
|
|||
|
|
elsif ($line=~/(aes[a-z]+)\s+%xmm([0-9]+),\s*%xmm([0-9]+)/) {
|
|||
|
|
my %opcodelet = (
|
|||
|
|
"aesimc" => 0xdb,
|
|||
|
|
"aesenc" => 0xdc, "aesenclast" => 0xdd,
|
|||
|
|
"aesdec" => 0xde, "aesdeclast" => 0xdf
|
|||
|
|
);
|
|||
|
|
return undef if (!defined($opcodelet{$1}));
|
|||
|
|
rex(\@opcode,$3,$2);
|
|||
|
|
push @opcode,0x0f,0x38,$opcodelet{$1};
|
|||
|
|
push @opcode,0xc0|($2&7)|(($3&7)<<3); # ModR/M
|
|||
|
|
return ".byte\t".join(',',@opcode);
|
|||
|
|
}
|
|||
|
|
return $line;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
$code =~ s/\`([^\`]*)\`/eval($1)/gem;
|
|||
|
|
$code =~ s/\b(aes.*%xmm[0-9]+).*$/aesni($1)/gem;
|
|||
|
|
|
|||
|
|
print $code;
|
|||
|
|
|
|||
|
|
close STDOUT;
|