ghash-sparcv9.pl: update from master.
This commit is contained in:
Andy Polyakov
parent
d1896e8ccd
commit
8eb2da5fbb
@ -36,6 +36,15 @@
|
|||||||
# references to input data and Z.hi updates to achieve 12 cycles
|
# references to input data and Z.hi updates to achieve 12 cycles
|
||||||
# timing. To anchor to something else, sha1-sparcv9.pl spends 11.6
|
# timing. To anchor to something else, sha1-sparcv9.pl spends 11.6
|
||||||
# cycles to process one byte on UltraSPARC pre-Tx CPU and ~24 on T1.
|
# cycles to process one byte on UltraSPARC pre-Tx CPU and ~24 on T1.
|
||||||
|
#
|
||||||
|
# October 2012
|
||||||
|
#
|
||||||
|
# Add VIS3 lookup-table-free implementation using polynomial
|
||||||
|
# multiplication xmulx[hi] and extended addition addxc[cc]
|
||||||
|
# instructions. 4.52/7.63x improvement on T3/T4 or in absolute
|
||||||
|
# terms 7.90/2.14 cycles per byte. On T4 multi-process benchmark
|
||||||
|
# saturates at ~15.5x single-process result on 8-core processor,
|
||||||
|
# or ~20.5GBps per 2.85GHz socket.
|
||||||
|
|
||||||
$bits=32;
|
$bits=32;
|
||||||
for (@ARGV) { $bits=64 if (/\-m64/ || /\-xarch\=v9/); }
|
for (@ARGV) { $bits=64 if (/\-m64/ || /\-xarch\=v9/); }
|
||||||
@ -66,6 +75,10 @@ $Htbl="%i1";
|
|||||||
$inp="%i2";
|
$inp="%i2";
|
||||||
$len="%i3";
|
$len="%i3";
|
||||||
|
|
||||||
|
$code.=<<___ if ($bits==64);
|
||||||
|
.register %g2,#scratch
|
||||||
|
.register %g3,#scratch
|
||||||
|
___
|
||||||
$code.=<<___;
|
$code.=<<___;
|
||||||
.section ".text",#alloc,#execinstr
|
.section ".text",#alloc,#execinstr
|
||||||
|
|
||||||
@ -321,10 +334,238 @@ gcm_gmult_4bit:
|
|||||||
restore
|
restore
|
||||||
.type gcm_gmult_4bit,#function
|
.type gcm_gmult_4bit,#function
|
||||||
.size gcm_gmult_4bit,(.-gcm_gmult_4bit)
|
.size gcm_gmult_4bit,(.-gcm_gmult_4bit)
|
||||||
.asciz "GHASH for SPARCv9, CRYPTOGAMS by <appro\@openssl.org>"
|
___
|
||||||
|
|
||||||
|
{{{
|
||||||
|
# Straightforward 128x128-bit multiplication using Karatsuba algorithm
|
||||||
|
# followed by pair of 64-bit reductions [with a shortcut in first one,
|
||||||
|
# which allowed to break dependency between reductions and remove one
|
||||||
|
# multiplication from critical path]. While it might be suboptimal
|
||||||
|
# with regard to sheer number of multiplications, other methods [such
|
||||||
|
# as aggregate reduction] would require more 64-bit registers, which
|
||||||
|
# we don't have in 32-bit application context.
|
||||||
|
|
||||||
|
($Xip,$Htable,$inp,$len)=map("%i$_",(0..3));
|
||||||
|
|
||||||
|
($Hhl,$Hlo,$Hhi,$Xlo,$Xhi,$xE1,$sqr, $C0,$C1,$C2,$C3,$V)=
|
||||||
|
(map("%o$_",(0..5,7)),map("%g$_",(1..5)));
|
||||||
|
|
||||||
|
($shl,$shr)=map("%l$_",(0..7));
|
||||||
|
|
||||||
|
# For details regarding "twisted H" see ghash-x86.pl.
|
||||||
|
$code.=<<___;
|
||||||
|
.globl gcm_init_vis3
|
||||||
|
.align 32
|
||||||
|
gcm_init_vis3:
|
||||||
|
save %sp,-$frame,%sp
|
||||||
|
|
||||||
|
ldx [%i1+0],$Hhi
|
||||||
|
ldx [%i1+8],$Hlo
|
||||||
|
mov 0xE1,$Xhi
|
||||||
|
mov 1,$Xlo
|
||||||
|
sllx $Xhi,57,$Xhi
|
||||||
|
srax $Hhi,63,$C0 ! broadcast carry
|
||||||
|
addcc $Hlo,$Hlo,$Hlo ! H<<=1
|
||||||
|
addxc $Hhi,$Hhi,$Hhi
|
||||||
|
and $C0,$Xlo,$Xlo
|
||||||
|
and $C0,$Xhi,$Xhi
|
||||||
|
xor $Xlo,$Hlo,$Hlo
|
||||||
|
xor $Xhi,$Hhi,$Hhi
|
||||||
|
stx $Hlo,[%i0+8] ! save twisted H
|
||||||
|
stx $Hhi,[%i0+0]
|
||||||
|
|
||||||
|
sethi %hi(0xA0406080),$V
|
||||||
|
sethi %hi(0x20C0E000),%l0
|
||||||
|
or $V,%lo(0xA0406080),$V
|
||||||
|
or %l0,%lo(0x20C0E000),%l0
|
||||||
|
sllx $V,32,$V
|
||||||
|
or %l0,$V,$V ! (0xE0·i)&0xff=0xA040608020C0E000
|
||||||
|
stx $V,[%i0+16]
|
||||||
|
|
||||||
|
ret
|
||||||
|
restore
|
||||||
|
.type gcm_init_vis3,#function
|
||||||
|
.size gcm_init_vis3,.-gcm_init_vis3
|
||||||
|
|
||||||
|
.globl gcm_gmult_vis3
|
||||||
|
.align 32
|
||||||
|
gcm_gmult_vis3:
|
||||||
|
save %sp,-$frame,%sp
|
||||||
|
|
||||||
|
ldx [$Xip+8],$Xlo ! load Xi
|
||||||
|
ldx [$Xip+0],$Xhi
|
||||||
|
ldx [$Htable+8],$Hlo ! load twisted H
|
||||||
|
ldx [$Htable+0],$Hhi
|
||||||
|
|
||||||
|
mov 0xE1,%l7
|
||||||
|
sllx %l7,57,$xE1 ! 57 is not a typo
|
||||||
|
ldx [$Htable+16],$V ! (0xE0·i)&0xff=0xA040608020C0E000
|
||||||
|
|
||||||
|
xor $Hhi,$Hlo,$Hhl ! Karatsuba pre-processing
|
||||||
|
xmulx $Xlo,$Hlo,$C0
|
||||||
|
xor $Xlo,$Xhi,$C2 ! Karatsuba pre-processing
|
||||||
|
xmulx $C2,$Hhl,$C1
|
||||||
|
xmulxhi $Xlo,$Hlo,$Xlo
|
||||||
|
xmulxhi $C2,$Hhl,$C2
|
||||||
|
xmulxhi $Xhi,$Hhi,$C3
|
||||||
|
xmulx $Xhi,$Hhi,$Xhi
|
||||||
|
|
||||||
|
sll $C0,3,$sqr
|
||||||
|
srlx $V,$sqr,$sqr ! ·0xE0 [implicit &(7<<3)]
|
||||||
|
xor $C0,$sqr,$sqr
|
||||||
|
sllx $sqr,57,$sqr ! ($C0·0xE1)<<1<<56 [implicit &0x7f]
|
||||||
|
|
||||||
|
xor $C0,$C1,$C1 ! Karatsuba post-processing
|
||||||
|
xor $Xlo,$C2,$C2
|
||||||
|
xor $sqr,$Xlo,$Xlo ! real destination is $C1
|
||||||
|
xor $C3,$C2,$C2
|
||||||
|
xor $Xlo,$C1,$C1
|
||||||
|
xor $Xhi,$C2,$C2
|
||||||
|
xor $Xhi,$C1,$C1
|
||||||
|
|
||||||
|
xmulxhi $C0,$xE1,$Xlo ! ·0xE1<<1<<56
|
||||||
|
xor $C0,$C2,$C2
|
||||||
|
xmulx $C1,$xE1,$C0
|
||||||
|
xor $C1,$C3,$C3
|
||||||
|
xmulxhi $C1,$xE1,$C1
|
||||||
|
|
||||||
|
xor $Xlo,$C2,$C2
|
||||||
|
xor $C0,$C2,$C2
|
||||||
|
xor $C1,$C3,$C3
|
||||||
|
|
||||||
|
stx $C2,[$Xip+8] ! save Xi
|
||||||
|
stx $C3,[$Xip+0]
|
||||||
|
|
||||||
|
ret
|
||||||
|
restore
|
||||||
|
.type gcm_gmult_vis3,#function
|
||||||
|
.size gcm_gmult_vis3,.-gcm_gmult_vis3
|
||||||
|
|
||||||
|
.globl gcm_ghash_vis3
|
||||||
|
.align 32
|
||||||
|
gcm_ghash_vis3:
|
||||||
|
save %sp,-$frame,%sp
|
||||||
|
|
||||||
|
ldx [$Xip+8],$C2 ! load Xi
|
||||||
|
ldx [$Xip+0],$C3
|
||||||
|
ldx [$Htable+8],$Hlo ! load twisted H
|
||||||
|
ldx [$Htable+0],$Hhi
|
||||||
|
|
||||||
|
mov 0xE1,%l7
|
||||||
|
sllx %l7,57,$xE1 ! 57 is not a typo
|
||||||
|
ldx [$Htable+16],$V ! (0xE0·i)&0xff=0xA040608020C0E000
|
||||||
|
|
||||||
|
and $inp,7,$shl
|
||||||
|
andn $inp,7,$inp
|
||||||
|
sll $shl,3,$shl
|
||||||
|
prefetch [$inp+63], 20
|
||||||
|
sub %g0,$shl,$shr
|
||||||
|
|
||||||
|
xor $Hhi,$Hlo,$Hhl ! Karatsuba pre-processing
|
||||||
|
.Loop:
|
||||||
|
ldx [$inp+8],$Xlo
|
||||||
|
brz,pt $shl,1f
|
||||||
|
ldx [$inp+0],$Xhi
|
||||||
|
|
||||||
|
ldx [$inp+16],$C1 ! align data
|
||||||
|
srlx $Xlo,$shr,$C0
|
||||||
|
sllx $Xlo,$shl,$Xlo
|
||||||
|
sllx $Xhi,$shl,$Xhi
|
||||||
|
srlx $C1,$shr,$C1
|
||||||
|
or $C0,$Xhi,$Xhi
|
||||||
|
or $C1,$Xlo,$Xlo
|
||||||
|
1:
|
||||||
|
add $inp,16,$inp
|
||||||
|
sub $len,16,$len
|
||||||
|
xor $C2,$Xlo,$Xlo
|
||||||
|
xor $C3,$Xhi,$Xhi
|
||||||
|
prefetch [$inp+63], 20
|
||||||
|
|
||||||
|
xmulx $Xlo,$Hlo,$C0
|
||||||
|
xor $Xlo,$Xhi,$C2 ! Karatsuba pre-processing
|
||||||
|
xmulx $C2,$Hhl,$C1
|
||||||
|
xmulxhi $Xlo,$Hlo,$Xlo
|
||||||
|
xmulxhi $C2,$Hhl,$C2
|
||||||
|
xmulxhi $Xhi,$Hhi,$C3
|
||||||
|
xmulx $Xhi,$Hhi,$Xhi
|
||||||
|
|
||||||
|
sll $C0,3,$sqr
|
||||||
|
srlx $V,$sqr,$sqr ! ·0xE0 [implicit &(7<<3)]
|
||||||
|
xor $C0,$sqr,$sqr
|
||||||
|
sllx $sqr,57,$sqr ! ($C0·0xE1)<<1<<56 [implicit &0x7f]
|
||||||
|
|
||||||
|
xor $C0,$C1,$C1 ! Karatsuba post-processing
|
||||||
|
xor $Xlo,$C2,$C2
|
||||||
|
xor $sqr,$Xlo,$Xlo ! real destination is $C1
|
||||||
|
xor $C3,$C2,$C2
|
||||||
|
xor $Xlo,$C1,$C1
|
||||||
|
xor $Xhi,$C2,$C2
|
||||||
|
xor $Xhi,$C1,$C1
|
||||||
|
|
||||||
|
xmulxhi $C0,$xE1,$Xlo ! ·0xE1<<1<<56
|
||||||
|
xor $C0,$C2,$C2
|
||||||
|
xmulx $C1,$xE1,$C0
|
||||||
|
xor $C1,$C3,$C3
|
||||||
|
xmulxhi $C1,$xE1,$C1
|
||||||
|
|
||||||
|
xor $Xlo,$C2,$C2
|
||||||
|
xor $C0,$C2,$C2
|
||||||
|
brnz,pt $len,.Loop
|
||||||
|
xor $C1,$C3,$C3
|
||||||
|
|
||||||
|
stx $C2,[$Xip+8] ! save Xi
|
||||||
|
stx $C3,[$Xip+0]
|
||||||
|
|
||||||
|
ret
|
||||||
|
restore
|
||||||
|
.type gcm_ghash_vis3,#function
|
||||||
|
.size gcm_ghash_vis3,.-gcm_ghash_vis3
|
||||||
|
___
|
||||||
|
}}}
|
||||||
|
$code.=<<___;
|
||||||
|
.asciz "GHASH for SPARCv9/VIS3, CRYPTOGAMS by <appro\@openssl.org>"
|
||||||
.align 4
|
.align 4
|
||||||
___
|
___
|
||||||
|
|
||||||
$code =~ s/\`([^\`]*)\`/eval $1/gem;
|
|
||||||
print $code;
|
# Purpose of these subroutines is to explicitly encode VIS instructions,
|
||||||
|
# so that one can compile the module without having to specify VIS
|
||||||
|
# extentions on compiler command line, e.g. -xarch=v9 vs. -xarch=v9a.
|
||||||
|
# Idea is to reserve for option to produce "universal" binary and let
|
||||||
|
# programmer detect if current CPU is VIS capable at run-time.
|
||||||
|
sub unvis3 {
|
||||||
|
my ($mnemonic,$rs1,$rs2,$rd)=@_;
|
||||||
|
my %bias = ( "g" => 0, "o" => 8, "l" => 16, "i" => 24 );
|
||||||
|
my ($ref,$opf);
|
||||||
|
my %visopf = ( "addxc" => 0x011,
|
||||||
|
"addxccc" => 0x013,
|
||||||
|
"xmulx" => 0x115,
|
||||||
|
"xmulxhi" => 0x116 );
|
||||||
|
|
||||||
|
$ref = "$mnemonic\t$rs1,$rs2,$rd";
|
||||||
|
|
||||||
|
if ($opf=$visopf{$mnemonic}) {
|
||||||
|
foreach ($rs1,$rs2,$rd) {
|
||||||
|
return $ref if (!/%([goli])([0-9])/);
|
||||||
|
$_=$bias{$1}+$2;
|
||||||
|
}
|
||||||
|
|
||||||
|
return sprintf ".word\t0x%08x !%s",
|
||||||
|
0x81b00000|$rd<<25|$rs1<<14|$opf<<5|$rs2,
|
||||||
|
$ref;
|
||||||
|
} else {
|
||||||
|
return $ref;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (split("\n",$code)) {
|
||||||
|
s/\`([^\`]*)\`/eval $1/ge;
|
||||||
|
|
||||||
|
s/\b(xmulx[hi]*|addxc[c]{0,2})\s+(%[goli][0-7]),\s*(%[goli][0-7]),\s*(%[goli][0-7])/
|
||||||
|
&unvis3($1,$2,$3,$4)
|
||||||
|
/ge;
|
||||||
|
|
||||||
|
print $_,"\n";
|
||||||
|
}
|
||||||
|
|
||||||
close STDOUT;
|
close STDOUT;
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user