generic-library/openssl
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

ghash-sparcv9.pl: 22% improvement on T4.

Browse Source
This commit is contained in:
Andy Polyakov 2012-11-05 08:47:26 +00:00
parent 5b0e3daf50
commit 24798c5e59
2 changed files with 143 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

248
crypto/modes/asm/ghash-sparcv9.pl
View File

@ -41,8 +41,10 @@
# #
# Add VIS3 lookup-table-free implementation using polynomial # Add VIS3 lookup-table-free implementation using polynomial
# multiplication xmulx[hi] and extended addition addxc[cc] # multiplication xmulx[hi] and extended addition addxc[cc]
# instructions. 3.96/6.26x improvement on T3/T4 or in absolute # instructions. 4.22/7.63x improvement on T3/T4 or in absolute
# terms 9.02/2.61 cycles per byte. # terms 8.45/2.14 cycles per byte. On T4 multi-process benchmark
# saturates at ~15x single-process result on 8-core processor, or
# ~19.7GBps per 2.85GHz socket.
$bits=32; $bits=32;
for (@ARGV) { $bits=64 if (/\-m64/ || /\-xarch\=v9/); } for (@ARGV) { $bits=64 if (/\-m64/ || /\-xarch\=v9/); }
@ -335,74 +337,103 @@ gcm_gmult_4bit:
___ ___
{{{ {{{
# Straightforward 64-bits-at-a-time approach with pair of 128x64-bit # Straightforward 128x128-bit multiplication using Karatsuba algorithm
# multiplications followed by 64-bit reductions. While it might be # followed by pair of 64-bit reductions [with a shortcut in first one,
# suboptimal with regard to sheer amount of multiplications, other # which allowed to break dependency between reductions and remove one
# methods would require larger amount of 64-bit registers, which we # mulitplication from critical path]. While it might be suboptimal
# don't have in 32-bit application. Also, they [alternative methods # with regard to sheer number of multiplications, other methods [such
# such as aggregated reduction] kind of thrive on fast 128-bit SIMD # as aggregate reduction] would require more 64-bit registers, which
# instructions and these are not option on SPARC... # we don't have in 32-bit application context.
($Xip,$Htable,$inp,$len)=map("%i$_",(0..3)); ($Xip,$Htable,$inp,$len)=map("%i$_",(0..3));
($xE1,$Hhi,$Hlo,$Rhi,$Rlo,$M0hi,$M0lo,$M1hi,$M1lo,$Zhi,$Zlo,$X)= ($Hhl,$Hlo,$Hhi,$Xlo,$Xhi,$xE1,$x384, $C0,$C1,$C2,$C3,$V)=
(map("%g$_",(1..5)),map("%o$_",(0..5,7))); (map("%o$_",(0..5,7)),map("%g$_",(1..5)));
($shl,$shr)=map("%l$_",(0..7));
($shl,$shr,$sqr)=map("%l$_",(0..7));
# For details regarding "twisted H" see ghash-x86.pl.
$code.=<<___; $code.=<<___;
.globl gcm_init_vis3
.align 32
gcm_init_vis3:
save %sp,-$frame,%sp
ldx [%i1+0],$Hhi
ldx [%i1+8],$Hlo
mov 0xE1,$Xhi
mov 1,$Xlo
sllx $Xhi,57,$Xhi
srax $Hhi,63,$C0 ! carry
addcc $Hlo,$Hlo,$Hlo ! H<<=1
addxc $Hhi,$Hhi,$Hhi
and $Xlo,$C0,$Xlo
and $Xhi,$C0,$Xhi
xor $Xlo,$Hlo,$Hlo
xor $Xhi,$Hhi,$Hhi
stx $Hlo,[%i0+8] ! save twisted H
stx $Hhi,[%i0+0]
ret
restore
.type gcm_init_vis3,#function
.size gcm_init_vis3,.-gcm_init_vis3
.globl gcm_gmult_vis3 .globl gcm_gmult_vis3
.align 32 .align 32
gcm_gmult_vis3: gcm_gmult_vis3:
save %sp,-$frame,%sp save %sp,-$frame,%sp
ldx [$Xip+8],$X ! load X.lo ldx [$Xip+8],$Xlo ! load Xi
ldx [$Htable-8], $Hlo ! load H ldx [$Xip+0],$Xhi
ldx [$Htable-16],$Hhi ldx [$Htable+8],$Hlo ! load twisted H
mov 0xE1,$xE1 ldx [$Htable+0],$Hhi
sllx $xE1,57,$xE1
xmulx $X,$Hlo,$M0lo ! H·X.lo sethi %hi(0xA0406080),$V
xmulxhi $X,$Hlo,$M0hi sethi %hi(0x20C0E000),%l0
xmulx $X,$Hhi,$M1lo or $V,%lo(0xA0406080),$V
xmulxhi $X,$Hhi,$M1hi or %l0,%lo(0x20C0E000),%l0
ldx [$Xip+0],$X ! load X.hi sllx $V,32,$V
mov 0xE1,%l1
or %l0,$V,$V ! (0xE0·i)&0xff=0xA040608020C0E000
sllx %l1,57,$xE1 ! 57 is not a typo
sllx %l1,50,$x384
xor $Hhi,$Hlo,$Hhl ! Karatsuba pre-processing
addcc $M0lo,$M0lo,$M0lo ! (H·X.lo)<<1 xmulx $Xlo,$Hlo,$C0
xor $M0hi,$M1lo,$M1lo xor $Xlo,$Xhi,$C2 ! Karatsuba pre-processing
xmulx $C2,$Hhl,$C1
xmulxhi $Xlo,$Hlo,$Xlo
xmulxhi $C2,$Hhl,$C2
xmulxhi $Xhi,$Hhi,$C3
xmulx $Xhi,$Hhi,$Xhi
xmulx $xE1,$M0lo,$Rlo ! res=Z.lo·(0xE1<<57) sll $C0,3,$sqr
xmulxhi $xE1,$M0lo,$Rhi srlx $V,$sqr,$sqr ! ·0xE0 [implicit &(7<<3)]
xor $C0,$sqr,$sqr
and $sqr,0x7f,$sqr
addxccc $M1lo,$M1lo,$Zlo ! Z=((H·X.lo)<<1)>>64 xor $C0,$C1,$C1 ! Karatsuba post-processing
addxc $M1hi,$M1hi,$Zhi xor $Xlo,$C2,$C2
xor $M0lo,$Zhi,$Zhi ! overflow bit from 0xE1<<57 xor $Xhi,$C1,$C1
xor $C3,$C2,$C2
xor $Xlo,$C1,$C1
xmulx $X,$Hlo,$M0lo ! H·X.hi xmulxhi $C0,$xE1,$Xlo ! ·0xE1<<1<<56
xmulxhi $X,$Hlo,$M0hi xor $Xhi,$C2,$C2
xmulx $X,$Hhi,$M1lo xmulx $sqr,$x384,$Xhi ! ·0xE1<<2<<48
xmulxhi $X,$Hhi,$M1hi xor $C0,$C2,$C2
xmulx $C1,$xE1,$C0
xor $C1,$C3,$C3
xmulxhi $C1,$xE1,$C1
xor $Rlo,$Zlo,$Zlo ! Z^=res xor $Xlo,$C2,$C2
xor $Rhi,$Zhi,$Zhi xor $Xhi,$C3,$C3
xor $C0,$C2,$C2
xor $C1,$C3,$C3
addcc $M0lo,$M0lo,$M0lo ! (H·X.lo)<<1 stx $C2,[$Xip+8] ! save Xi
xor $Zlo, $M0lo,$M0lo stx $C3,[$Xip+0]
xor $M0hi,$M1lo,$M1lo
xmulx $xE1,$M0lo,$Rlo ! res=Z.lo·(0xE1<<57)
xmulxhi $xE1,$M0lo,$Rhi
addxccc $M1lo,$M1lo,$M1lo
addxc $M1hi,$M1hi,$M1hi
xor $M1lo,$Zhi,$Zlo ! Z=(Z^(H·X.hi)<<1)>>64
xor $M0lo,$M1hi,$Zhi ! overflow bit from 0xE1<<57
xor $Rlo,$Zlo,$Zlo ! Z^=res
xor $Rhi,$Zhi,$Zhi
stx $Zlo,[$Xip+8] ! save Xi
stx $Zhi,[$Xip+0]
ret ret
restore restore
@ -414,80 +445,83 @@ gcm_gmult_vis3:
gcm_ghash_vis3: gcm_ghash_vis3:
save %sp,-$frame,%sp save %sp,-$frame,%sp
ldx [$Xip+0],$Zhi ! load X.hi ldx [$Xip+8],$C2 ! load Xi
ldx [$Xip+8],$Zlo ! load X.lo ldx [$Xip+0],$C3
ldx [$Htable+8],$Hlo ! load twisted H
ldx [$Htable+0],$Hhi
sethi %hi(0xA0406080),$V
sethi %hi(0x20C0E000),%l6
or $V,%lo(0xA0406080),$V
or %l6,%lo(0x20C0E000),%l6
sllx $V,32,$V
mov 0xE1,%l7
or %l6,$V,$V ! (0xE0·i)&0xff=0xA040608020C0E000
sllx %l7,57,$xE1 ! 57 is not a typo
sllx %l7,50,$x384
and $inp,7,$shl and $inp,7,$shl
andn $inp,7,$inp andn $inp,7,$inp
ldx [$Htable-8], $Hlo ! load H
ldx [$Htable-16],$Hhi
sll $shl,3,$shl sll $shl,3,$shl
prefetch [$inp+63], 20 prefetch [$inp+63], 20
mov 0xE1,$xE1
sub %g0,$shl,$shr sub %g0,$shl,$shr
sllx $xE1,57,$xE1
xor $Hhi,$Hlo,$Hhl ! Karatsuba pre-processing
.Loop: .Loop:
ldx [$inp+8],$Rlo ! load *inp ldx [$inp+8],$Xlo
brz,pt $shl,1f brz,pt $shl,1f
ldx [$inp+0],$Rhi ldx [$inp+0],$Xhi
ldx [$inp+16],$X ! align data
srlx $Rlo,$shr,$M0lo
sllx $Rlo,$shl,$Rlo
sllx $Rhi,$shl,$Rhi
srlx $X,$shr,$X
or $M0lo,$Rhi,$Rhi
or $X,$Rlo,$Rlo
ldx [$inp+16],$C1 ! align data
srlx $Xlo,$shr,$C0
sllx $Xlo,$shl,$Xlo
sllx $Xhi,$shl,$Xhi
srlx $C1,$shr,$C1
or $C0,$Xhi,$Xhi
or $C1,$Xlo,$Xlo
1: 1:
add $inp,16,$inp add $inp,16,$inp
sub $len,16,$len sub $len,16,$len
xor $Rlo,$Zlo,$X xor $C2,$Xlo,$Xlo
xor $C3,$Xhi,$Xhi
prefetch [$inp+63], 20 prefetch [$inp+63], 20
xmulx $X,$Hlo,$M0lo ! H·X.lo xmulx $Xlo,$Hlo,$C0
xmulxhi $X,$Hlo,$M0hi xor $Xlo,$Xhi,$C2 ! Karatsuba pre-processing
xmulx $X,$Hhi,$M1lo xmulx $C2,$Hhl,$C1
xmulxhi $X,$Hhi,$M1hi xmulxhi $Xlo,$Hlo,$Xlo
xor $Rhi,$Zhi,$X xmulxhi $C2,$Hhl,$C2
xmulxhi $Xhi,$Hhi,$C3
xmulx $Xhi,$Hhi,$Xhi
addcc $M0lo,$M0lo,$M0lo ! (H·X.lo)<<1 sll $C0,3,$sqr
xor $M0hi,$M1lo,$M1lo srlx $V,$sqr,$sqr ! ·0xE0 [implicit &(7<<3)]
xor $C0,$sqr,$sqr
and $sqr,0x7f,$sqr
xmulx $xE1,$M0lo,$Rlo ! res=Z.lo·(0xE1<<57) xor $C0,$C1,$C1 ! Karatsuba post-processing
xmulxhi $xE1,$M0lo,$Rhi xor $Xlo,$C2,$C2
xor $Xhi,$C1,$C1
xor $C3,$C2,$C2
xor $Xlo,$C1,$C1
addxccc $M1lo,$M1lo,$Zlo ! Z=((H·X.lo)<<1)>>64 xmulxhi $C0,$xE1,$Xlo ! ·0xE1<<1<<56
addxc $M1hi,$M1hi,$Zhi xor $Xhi,$C2,$C2
xor $M0lo,$Zhi,$Zhi ! overflow bit from 0xE1<<57 xmulx $sqr,$x384,$Xhi ! ·0xE1<<2<<48
xor $C0,$C2,$C2
xmulx $C1,$xE1,$C0
xor $C1,$C3,$C3
xmulxhi $C1,$xE1,$C1
xmulx $X,$Hlo,$M0lo ! H·X.hi xor $Xlo,$C2,$C2
xmulxhi $X,$Hlo,$M0hi xor $Xhi,$C3,$C3
xmulx $X,$Hhi,$M1lo xor $C0,$C2,$C2
xmulxhi $X,$Hhi,$M1hi
xor $Rlo,$Zlo,$Zlo ! Z^=res
xor $Rhi,$Zhi,$Zhi
addcc $M0lo,$M0lo,$M0lo ! (H·X.lo)<<1
xor $Zlo, $M0lo,$M0lo
xor $M0hi,$M1lo,$M1lo
xmulx $xE1,$M0lo,$Rlo ! res=Z.lo·(0xE1<<57)
xmulxhi $xE1,$M0lo,$Rhi
addxccc $M1lo,$M1lo,$M1lo
addxc $M1hi,$M1hi,$M1hi
xor $M1lo,$Zhi,$Zlo ! Z=(Z^(H·X.hi)<<1)>>64
xor $M0lo,$M1hi,$Zhi ! overflow bit from 0xE1<<57
xor $Rlo,$Zlo,$Zlo ! Z^=res
brnz,pt $len,.Loop brnz,pt $len,.Loop
xor $Rhi,$Zhi,$Zhi xor $C1,$C3,$C3
stx $Zlo,[$Xip+8] ! save Xi stx $C2,[$Xip+8] ! save Xi
stx $Zhi,[$Xip+0] stx $C3,[$Xip+0]
ret ret
restore restore

2
crypto/modes/gcm128.c
View File

@ -679,6 +679,7 @@ void gcm_ghash_neon(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len);
# define GHASH_ASM_SPARC # define GHASH_ASM_SPARC
# define GCM_FUNCREF_4BIT # define GCM_FUNCREF_4BIT
extern unsigned int OPENSSL_sparcv9cap_P[]; extern unsigned int OPENSSL_sparcv9cap_P[];
void gcm_init_vis3(u128 Htable[16],const u64 Xi[2]);
void gcm_gmult_vis3(u64 Xi[2],const u128 Htable[16]); void gcm_gmult_vis3(u64 Xi[2],const u128 Htable[16]);
void gcm_ghash_vis3(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len); void gcm_ghash_vis3(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len);
# endif # endif
@ -759,6 +760,7 @@ void CRYPTO_gcm128_init(GCM128_CONTEXT *ctx,void *key,block128_f block)
} }
# elif defined(GHASH_ASM_SPARC) # elif defined(GHASH_ASM_SPARC)
if (OPENSSL_sparcv9cap_P[0] & SPARCV9_VIS3) { if (OPENSSL_sparcv9cap_P[0] & SPARCV9_VIS3) {
gcm_init_vis3(ctx->Htable,ctx->H.u);
ctx->gmult = gcm_gmult_vis3; ctx->gmult = gcm_gmult_vis3;
ctx->ghash = gcm_ghash_vis3; ctx->ghash = gcm_ghash_vis3;
} else { } else {