s390x assembler pack update from HEAD.
This commit is contained in:
Andy Polyakov
parent
4195343c0d
commit
9833757b5d
File diff suppressed because it is too large
Load Diff
221
crypto/bn/asm/s390x-gf2m.pl
Normal file
221
crypto/bn/asm/s390x-gf2m.pl
Normal file
@ -0,0 +1,221 @@
|
||||
#!/usr/bin/env perl
|
||||
#
|
||||
# ====================================================================
|
||||
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
|
||||
# project. The module is, however, dual licensed under OpenSSL and
|
||||
# CRYPTOGAMS licenses depending on where you obtain it. For further
|
||||
# details see http://www.openssl.org/~appro/cryptogams/.
|
||||
# ====================================================================
|
||||
#
|
||||
# May 2011
|
||||
#
|
||||
# The module implements bn_GF2m_mul_2x2 polynomial multiplication used
|
||||
# in bn_gf2m.c. It's kind of low-hanging mechanical port from C for
|
||||
# the time being... gcc 4.3 appeared to generate poor code, therefore
|
||||
# the effort. And indeed, the module delivers 55%-90%(*) improvement
|
||||
# on haviest ECDSA verify and ECDH benchmarks for 163- and 571-bit
|
||||
# key lengths on z990, 30%-55%(*) - on z10, and 70%-110%(*) - on z196.
|
||||
# This is for 64-bit build. In 32-bit "highgprs" case improvement is
|
||||
# even higher, for example on z990 it was measured 80%-150%. ECDSA
|
||||
# sign is modest 9%-12% faster. Keep in mind that these coefficients
|
||||
# are not ones for bn_GF2m_mul_2x2 itself, as not all CPU time is
|
||||
# burnt in it...
|
||||
#
|
||||
# (*) gcc 4.1 was observed to deliver better results than gcc 4.3,
|
||||
# so that improvement coefficients can vary from one specific
|
||||
# setup to another.
|
||||
|
||||
$flavour = shift;
|
||||
|
||||
if ($flavour =~ /3[12]/) {
|
||||
$SIZE_T=4;
|
||||
$g="";
|
||||
} else {
|
||||
$SIZE_T=8;
|
||||
$g="g";
|
||||
}
|
||||
|
||||
while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {}
|
||||
open STDOUT,">$output";
|
||||
|
||||
$stdframe=16*$SIZE_T+4*8;
|
||||
|
||||
$rp="%r2";
|
||||
$a1="%r3";
|
||||
$a0="%r4";
|
||||
$b1="%r5";
|
||||
$b0="%r6";
|
||||
|
||||
$ra="%r14";
|
||||
$sp="%r15";
|
||||
|
||||
@T=("%r0","%r1");
|
||||
@i=("%r12","%r13");
|
||||
|
||||
($a1,$a2,$a4,$a8,$a12,$a48)=map("%r$_",(6..11));
|
||||
($lo,$hi,$b)=map("%r$_",(3..5)); $a=$lo; $mask=$a8;
|
||||
|
||||
$code.=<<___;
|
||||
.text
|
||||
|
||||
.type _mul_1x1,\@function
|
||||
.align 16
|
||||
_mul_1x1:
|
||||
lgr $a1,$a
|
||||
sllg $a2,$a,1
|
||||
sllg $a4,$a,2
|
||||
sllg $a8,$a,3
|
||||
|
||||
srag $lo,$a1,63 # broadcast 63rd bit
|
||||
nihh $a1,0x1fff
|
||||
srag @i[0],$a2,63 # broadcast 62nd bit
|
||||
nihh $a2,0x3fff
|
||||
srag @i[1],$a4,63 # broadcast 61st bit
|
||||
nihh $a4,0x7fff
|
||||
ngr $lo,$b
|
||||
ngr @i[0],$b
|
||||
ngr @i[1],$b
|
||||
|
||||
lghi @T[0],0
|
||||
lgr $a12,$a1
|
||||
stg @T[0],`$stdframe+0*8`($sp) # tab[0]=0
|
||||
xgr $a12,$a2
|
||||
stg $a1,`$stdframe+1*8`($sp) # tab[1]=a1
|
||||
lgr $a48,$a4
|
||||
stg $a2,`$stdframe+2*8`($sp) # tab[2]=a2
|
||||
xgr $a48,$a8
|
||||
stg $a12,`$stdframe+3*8`($sp) # tab[3]=a1^a2
|
||||
xgr $a1,$a4
|
||||
|
||||
stg $a4,`$stdframe+4*8`($sp) # tab[4]=a4
|
||||
xgr $a2,$a4
|
||||
stg $a1,`$stdframe+5*8`($sp) # tab[5]=a1^a4
|
||||
xgr $a12,$a4
|
||||
stg $a2,`$stdframe+6*8`($sp) # tab[6]=a2^a4
|
||||
xgr $a1,$a48
|
||||
stg $a12,`$stdframe+7*8`($sp) # tab[7]=a1^a2^a4
|
||||
xgr $a2,$a48
|
||||
|
||||
stg $a8,`$stdframe+8*8`($sp) # tab[8]=a8
|
||||
xgr $a12,$a48
|
||||
stg $a1,`$stdframe+9*8`($sp) # tab[9]=a1^a8
|
||||
xgr $a1,$a4
|
||||
stg $a2,`$stdframe+10*8`($sp) # tab[10]=a2^a8
|
||||
xgr $a2,$a4
|
||||
stg $a12,`$stdframe+11*8`($sp) # tab[11]=a1^a2^a8
|
||||
|
||||
xgr $a12,$a4
|
||||
stg $a48,`$stdframe+12*8`($sp) # tab[12]=a4^a8
|
||||
srlg $hi,$lo,1
|
||||
stg $a1,`$stdframe+13*8`($sp) # tab[13]=a1^a4^a8
|
||||
sllg $lo,$lo,63
|
||||
stg $a2,`$stdframe+14*8`($sp) # tab[14]=a2^a4^a8
|
||||
srlg @T[0],@i[0],2
|
||||
stg $a12,`$stdframe+15*8`($sp) # tab[15]=a1^a2^a4^a8
|
||||
|
||||
lghi $mask,`0xf<<3`
|
||||
sllg $a1,@i[0],62
|
||||
sllg @i[0],$b,3
|
||||
srlg @T[1],@i[1],3
|
||||
ngr @i[0],$mask
|
||||
sllg $a2,@i[1],61
|
||||
srlg @i[1],$b,4-3
|
||||
xgr $hi,@T[0]
|
||||
ngr @i[1],$mask
|
||||
xgr $lo,$a1
|
||||
xgr $hi,@T[1]
|
||||
xgr $lo,$a2
|
||||
|
||||
xg $lo,$stdframe(@i[0],$sp)
|
||||
srlg @i[0],$b,8-3
|
||||
ngr @i[0],$mask
|
||||
___
|
||||
for($n=1;$n<14;$n++) {
|
||||
$code.=<<___;
|
||||
lg @T[1],$stdframe(@i[1],$sp)
|
||||
srlg @i[1],$b,`($n+2)*4`-3
|
||||
sllg @T[0],@T[1],`$n*4`
|
||||
ngr @i[1],$mask
|
||||
srlg @T[1],@T[1],`64-$n*4`
|
||||
xgr $lo,@T[0]
|
||||
xgr $hi,@T[1]
|
||||
___
|
||||
push(@i,shift(@i)); push(@T,shift(@T));
|
||||
}
|
||||
$code.=<<___;
|
||||
lg @T[1],$stdframe(@i[1],$sp)
|
||||
sllg @T[0],@T[1],`$n*4`
|
||||
srlg @T[1],@T[1],`64-$n*4`
|
||||
xgr $lo,@T[0]
|
||||
xgr $hi,@T[1]
|
||||
|
||||
lg @T[0],$stdframe(@i[0],$sp)
|
||||
sllg @T[1],@T[0],`($n+1)*4`
|
||||
srlg @T[0],@T[0],`64-($n+1)*4`
|
||||
xgr $lo,@T[1]
|
||||
xgr $hi,@T[0]
|
||||
|
||||
br $ra
|
||||
.size _mul_1x1,.-_mul_1x1
|
||||
|
||||
.globl bn_GF2m_mul_2x2
|
||||
.type bn_GF2m_mul_2x2,\@function
|
||||
.align 16
|
||||
bn_GF2m_mul_2x2:
|
||||
stm${g} %r3,%r15,3*$SIZE_T($sp)
|
||||
|
||||
lghi %r1,-$stdframe-128
|
||||
la %r0,0($sp)
|
||||
la $sp,0(%r1,$sp) # alloca
|
||||
st${g} %r0,0($sp) # back chain
|
||||
___
|
||||
if ($SIZE_T==8) {
|
||||
my @r=map("%r$_",(6..9));
|
||||
$code.=<<___;
|
||||
bras $ra,_mul_1x1 # a1·b1
|
||||
stmg $lo,$hi,16($rp)
|
||||
|
||||
lg $a,`$stdframe+128+4*$SIZE_T`($sp)
|
||||
lg $b,`$stdframe+128+6*$SIZE_T`($sp)
|
||||
bras $ra,_mul_1x1 # a0·b0
|
||||
stmg $lo,$hi,0($rp)
|
||||
|
||||
lg $a,`$stdframe+128+3*$SIZE_T`($sp)
|
||||
lg $b,`$stdframe+128+5*$SIZE_T`($sp)
|
||||
xg $a,`$stdframe+128+4*$SIZE_T`($sp)
|
||||
xg $b,`$stdframe+128+6*$SIZE_T`($sp)
|
||||
bras $ra,_mul_1x1 # (a0+a1)·(b0+b1)
|
||||
lmg @r[0],@r[3],0($rp)
|
||||
|
||||
xgr $lo,$hi
|
||||
xgr $hi,@r[1]
|
||||
xgr $lo,@r[0]
|
||||
xgr $hi,@r[2]
|
||||
xgr $lo,@r[3]
|
||||
xgr $hi,@r[3]
|
||||
xgr $lo,$hi
|
||||
stg $hi,16($rp)
|
||||
stg $lo,8($rp)
|
||||
___
|
||||
} else {
|
||||
$code.=<<___;
|
||||
sllg %r3,%r3,32
|
||||
sllg %r5,%r5,32
|
||||
or %r3,%r4
|
||||
or %r5,%r6
|
||||
bras $ra,_mul_1x1
|
||||
rllg $lo,$lo,32
|
||||
rllg $hi,$hi,32
|
||||
stmg $lo,$hi,0($rp)
|
||||
___
|
||||
}
|
||||
$code.=<<___;
|
||||
lm${g} %r6,%r15,`$stdframe+128+6*$SIZE_T`($sp)
|
||||
br $ra
|
||||
.size bn_GF2m_mul_2x2,.-bn_GF2m_mul_2x2
|
||||
.string "GF(2^m) Multiplication for s390x, CRYPTOGAMS by <appro\@openssl.org>"
|
||||
___
|
||||
|
||||
$code =~ s/\`([^\`]*)\`/eval($1)/gem;
|
||||
print $code;
|
||||
close STDOUT;
|
||||
@ -32,6 +32,33 @@
|
||||
# Reschedule to minimize/avoid Address Generation Interlock hazard,
|
||||
# make inner loops counter-based.
|
||||
|
||||
# November 2010.
|
||||
#
|
||||
# Adapt for -m31 build. If kernel supports what's called "highgprs"
|
||||
# feature on Linux [see /proc/cpuinfo], it's possible to use 64-bit
|
||||
# instructions and achieve "64-bit" performance even in 31-bit legacy
|
||||
# application context. The feature is not specific to any particular
|
||||
# processor, as long as it's "z-CPU". Latter implies that the code
|
||||
# remains z/Architecture specific. Compatibility with 32-bit BN_ULONG
|
||||
# is achieved by swapping words after 64-bit loads, follow _dswap-s.
|
||||
# On z990 it was measured to perform 2.6-2.2 times better than
|
||||
# compiler-generated code, less for longer keys...
|
||||
|
||||
$flavour = shift;
|
||||
|
||||
if ($flavour =~ /3[12]/) {
|
||||
$SIZE_T=4;
|
||||
$g="";
|
||||
} else {
|
||||
$SIZE_T=8;
|
||||
$g="g";
|
||||
}
|
||||
|
||||
while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {}
|
||||
open STDOUT,">$output";
|
||||
|
||||
$stdframe=16*$SIZE_T+4*8;
|
||||
|
||||
$mn0="%r0";
|
||||
$num="%r1";
|
||||
|
||||
@ -60,34 +87,44 @@ $code.=<<___;
|
||||
.globl bn_mul_mont
|
||||
.type bn_mul_mont,\@function
|
||||
bn_mul_mont:
|
||||
lgf $num,164($sp) # pull $num
|
||||
sla $num,3 # $num to enumerate bytes
|
||||
lgf $num,`$stdframe+$SIZE_T-4`($sp) # pull $num
|
||||
sla $num,`log($SIZE_T)/log(2)` # $num to enumerate bytes
|
||||
la $bp,0($num,$bp)
|
||||
|
||||
stg %r2,16($sp)
|
||||
st${g} %r2,2*$SIZE_T($sp)
|
||||
|
||||
cghi $num,16 #
|
||||
lghi %r2,0 #
|
||||
blr %r14 # if($num<16) return 0;
|
||||
___
|
||||
$code.=<<___ if ($flavour =~ /3[12]/);
|
||||
tmll $num,4
|
||||
bnzr %r14 # if ($num&1) return 0;
|
||||
___
|
||||
$code.=<<___ if ($flavour !~ /3[12]/);
|
||||
cghi $num,96 #
|
||||
bhr %r14 # if($num>96) return 0;
|
||||
___
|
||||
$code.=<<___;
|
||||
stm${g} %r3,%r15,3*$SIZE_T($sp)
|
||||
|
||||
stmg %r3,%r15,24($sp)
|
||||
|
||||
lghi $rp,-160-8 # leave room for carry bit
|
||||
lghi $rp,-$stdframe-8 # leave room for carry bit
|
||||
lcgr $j,$num # -$num
|
||||
lgr %r0,$sp
|
||||
la $rp,0($rp,$sp)
|
||||
la $sp,0($j,$rp) # alloca
|
||||
stg %r0,0($sp) # back chain
|
||||
st${g} %r0,0($sp) # back chain
|
||||
|
||||
sra $num,3 # restore $num
|
||||
la $bp,0($j,$bp) # restore $bp
|
||||
ahi $num,-1 # adjust $num for inner loop
|
||||
lg $n0,0($n0) # pull n0
|
||||
_dswap $n0
|
||||
|
||||
lg $bi,0($bp)
|
||||
_dswap $bi
|
||||
lg $alo,0($ap)
|
||||
_dswap $alo
|
||||
mlgr $ahi,$bi # ap[0]*bp[0]
|
||||
lgr $AHI,$ahi
|
||||
|
||||
@ -95,6 +132,7 @@ bn_mul_mont:
|
||||
msgr $mn0,$n0
|
||||
|
||||
lg $nlo,0($np) #
|
||||
_dswap $nlo
|
||||
mlgr $nhi,$mn0 # np[0]*m1
|
||||
algr $nlo,$alo # +="tp[0]"
|
||||
lghi $NHI,0
|
||||
@ -106,12 +144,14 @@ bn_mul_mont:
|
||||
.align 16
|
||||
.L1st:
|
||||
lg $alo,0($j,$ap)
|
||||
_dswap $alo
|
||||
mlgr $ahi,$bi # ap[j]*bp[0]
|
||||
algr $alo,$AHI
|
||||
lghi $AHI,0
|
||||
alcgr $AHI,$ahi
|
||||
|
||||
lg $nlo,0($j,$np)
|
||||
_dswap $nlo
|
||||
mlgr $nhi,$mn0 # np[j]*m1
|
||||
algr $nlo,$NHI
|
||||
lghi $NHI,0
|
||||
@ -119,22 +159,24 @@ bn_mul_mont:
|
||||
algr $nlo,$alo
|
||||
alcgr $NHI,$nhi
|
||||
|
||||
stg $nlo,160-8($j,$sp) # tp[j-1]=
|
||||
stg $nlo,$stdframe-8($j,$sp) # tp[j-1]=
|
||||
la $j,8($j) # j++
|
||||
brct $count,.L1st
|
||||
|
||||
algr $NHI,$AHI
|
||||
lghi $AHI,0
|
||||
alcgr $AHI,$AHI # upmost overflow bit
|
||||
stg $NHI,160-8($j,$sp)
|
||||
stg $AHI,160($j,$sp)
|
||||
stg $NHI,$stdframe-8($j,$sp)
|
||||
stg $AHI,$stdframe($j,$sp)
|
||||
la $bp,8($bp) # bp++
|
||||
|
||||
.Louter:
|
||||
lg $bi,0($bp) # bp[i]
|
||||
_dswap $bi
|
||||
lg $alo,0($ap)
|
||||
_dswap $alo
|
||||
mlgr $ahi,$bi # ap[0]*bp[i]
|
||||
alg $alo,160($sp) # +=tp[0]
|
||||
alg $alo,$stdframe($sp) # +=tp[0]
|
||||
lghi $AHI,0
|
||||
alcgr $AHI,$ahi
|
||||
|
||||
@ -142,6 +184,7 @@ bn_mul_mont:
|
||||
msgr $mn0,$n0 # tp[0]*n0
|
||||
|
||||
lg $nlo,0($np) # np[0]
|
||||
_dswap $nlo
|
||||
mlgr $nhi,$mn0 # np[0]*m1
|
||||
algr $nlo,$alo # +="tp[0]"
|
||||
lghi $NHI,0
|
||||
@ -153,14 +196,16 @@ bn_mul_mont:
|
||||
.align 16
|
||||
.Linner:
|
||||
lg $alo,0($j,$ap)
|
||||
_dswap $alo
|
||||
mlgr $ahi,$bi # ap[j]*bp[i]
|
||||
algr $alo,$AHI
|
||||
lghi $AHI,0
|
||||
alcgr $ahi,$AHI
|
||||
alg $alo,160($j,$sp)# +=tp[j]
|
||||
alg $alo,$stdframe($j,$sp)# +=tp[j]
|
||||
alcgr $AHI,$ahi
|
||||
|
||||
lg $nlo,0($j,$np)
|
||||
_dswap $nlo
|
||||
mlgr $nhi,$mn0 # np[j]*m1
|
||||
algr $nlo,$NHI
|
||||
lghi $NHI,0
|
||||
@ -168,31 +213,33 @@ bn_mul_mont:
|
||||
algr $nlo,$alo # +="tp[j]"
|
||||
alcgr $NHI,$nhi
|
||||
|
||||
stg $nlo,160-8($j,$sp) # tp[j-1]=
|
||||
stg $nlo,$stdframe-8($j,$sp) # tp[j-1]=
|
||||
la $j,8($j) # j++
|
||||
brct $count,.Linner
|
||||
|
||||
algr $NHI,$AHI
|
||||
lghi $AHI,0
|
||||
alcgr $AHI,$AHI
|
||||
alg $NHI,160($j,$sp)# accumulate previous upmost overflow bit
|
||||
alg $NHI,$stdframe($j,$sp)# accumulate previous upmost overflow bit
|
||||
lghi $ahi,0
|
||||
alcgr $AHI,$ahi # new upmost overflow bit
|
||||
stg $NHI,160-8($j,$sp)
|
||||
stg $AHI,160($j,$sp)
|
||||
stg $NHI,$stdframe-8($j,$sp)
|
||||
stg $AHI,$stdframe($j,$sp)
|
||||
|
||||
la $bp,8($bp) # bp++
|
||||
clg $bp,160+8+32($j,$sp) # compare to &bp[num]
|
||||
cl${g} $bp,`$stdframe+8+4*$SIZE_T`($j,$sp) # compare to &bp[num]
|
||||
jne .Louter
|
||||
|
||||
lg $rp,160+8+16($j,$sp) # reincarnate rp
|
||||
la $ap,160($sp)
|
||||
l${g} $rp,`$stdframe+8+2*$SIZE_T`($j,$sp) # reincarnate rp
|
||||
la $ap,$stdframe($sp)
|
||||
ahi $num,1 # restore $num, incidentally clears "borrow"
|
||||
|
||||
la $j,0(%r0)
|
||||
lr $count,$num
|
||||
.Lsub: lg $alo,0($j,$ap)
|
||||
slbg $alo,0($j,$np)
|
||||
lg $nlo,0($j,$np)
|
||||
_dswap $nlo
|
||||
slbgr $alo,$nlo
|
||||
stg $alo,0($j,$rp)
|
||||
la $j,8($j)
|
||||
brct $count,.Lsub
|
||||
@ -207,19 +254,24 @@ bn_mul_mont:
|
||||
|
||||
la $j,0(%r0)
|
||||
lgr $count,$num
|
||||
.Lcopy: lg $alo,0($j,$ap) # copy or in-place refresh
|
||||
stg $j,160($j,$sp) # zap tp
|
||||
.Lcopy: lg $alo,0($j,$ap) # copy or in-place refresh
|
||||
_dswap $alo
|
||||
stg $j,$stdframe($j,$sp) # zap tp
|
||||
stg $alo,0($j,$rp)
|
||||
la $j,8($j)
|
||||
brct $count,.Lcopy
|
||||
|
||||
la %r1,160+8+48($j,$sp)
|
||||
lmg %r6,%r15,0(%r1)
|
||||
la %r1,`$stdframe+8+6*$SIZE_T`($j,$sp)
|
||||
lm${g} %r6,%r15,0(%r1)
|
||||
lghi %r2,1 # signal "processed"
|
||||
br %r14
|
||||
.size bn_mul_mont,.-bn_mul_mont
|
||||
.string "Montgomery Multiplication for s390x, CRYPTOGAMS by <appro\@openssl.org>"
|
||||
___
|
||||
|
||||
print $code;
|
||||
foreach (split("\n",$code)) {
|
||||
s/\`([^\`]*)\`/eval $1/ge;
|
||||
s/_dswap\s+(%r[0-9]+)/sprintf("rllg\t%s,%s,32",$1,$1) if($SIZE_T==4)/e;
|
||||
print $_,"\n";
|
||||
}
|
||||
close STDOUT;
|
||||
|
||||
@ -13,6 +13,29 @@
|
||||
# "cluster" Address Generation Interlocks, so that one pipeline stall
|
||||
# resolves several dependencies.
|
||||
|
||||
# November 2010.
|
||||
#
|
||||
# Adapt for -m31 build. If kernel supports what's called "highgprs"
|
||||
# feature on Linux [see /proc/cpuinfo], it's possible to use 64-bit
|
||||
# instructions and achieve "64-bit" performance even in 31-bit legacy
|
||||
# application context. The feature is not specific to any particular
|
||||
# processor, as long as it's "z-CPU". Latter implies that the code
|
||||
# remains z/Architecture specific. On z990 it was measured to perform
|
||||
# 50% better than code generated by gcc 4.3.
|
||||
|
||||
$flavour = shift;
|
||||
|
||||
if ($flavour =~ /3[12]/) {
|
||||
$SIZE_T=4;
|
||||
$g="";
|
||||
} else {
|
||||
$SIZE_T=8;
|
||||
$g="g";
|
||||
}
|
||||
|
||||
while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {}
|
||||
open STDOUT,">$output";
|
||||
|
||||
$rp="%r14";
|
||||
$sp="%r15";
|
||||
$code=<<___;
|
||||
@ -39,7 +62,12 @@ $code.=<<___;
|
||||
.type RC4,\@function
|
||||
.align 64
|
||||
RC4:
|
||||
stmg %r6,%r11,48($sp)
|
||||
stm${g} %r6,%r11,6*$SIZE_T($sp)
|
||||
___
|
||||
$code.=<<___ if ($flavour =~ /3[12]/);
|
||||
llgfr $len,$len
|
||||
___
|
||||
$code.=<<___;
|
||||
llgc $XX[0],0($key)
|
||||
llgc $YY,1($key)
|
||||
la $XX[0],1($XX[0])
|
||||
@ -90,7 +118,7 @@ $code.=<<___;
|
||||
xgr $acc,$TX[1]
|
||||
stg $acc,0($out)
|
||||
la $out,8($out)
|
||||
brct $cnt,.Loop8
|
||||
brctg $cnt,.Loop8
|
||||
|
||||
.Lshort:
|
||||
lghi $acc,7
|
||||
@ -122,7 +150,7 @@ $code.=<<___;
|
||||
ahi $XX[0],-1
|
||||
stc $XX[0],0($key)
|
||||
stc $YY,1($key)
|
||||
lmg %r6,%r11,48($sp)
|
||||
lm${g} %r6,%r11,6*$SIZE_T($sp)
|
||||
br $rp
|
||||
.size RC4,.-RC4
|
||||
.string "RC4 for s390x, CRYPTOGAMS by <appro\@openssl.org>"
|
||||
@ -130,7 +158,7 @@ $code.=<<___;
|
||||
___
|
||||
}
|
||||
|
||||
# void private_RC4_set_key(RC4_KEY *key,unsigned int len,const void *inp)
|
||||
# void RC4_set_key(RC4_KEY *key,unsigned int len,const void *inp)
|
||||
{
|
||||
$cnt="%r0";
|
||||
$idx="%r1";
|
||||
@ -143,11 +171,11 @@ $ikey="%r7";
|
||||
$iinp="%r8";
|
||||
|
||||
$code.=<<___;
|
||||
.globl private_RC4_set_key
|
||||
.type private_RC4_set_key,\@function
|
||||
.globl RC4_set_key
|
||||
.type RC4_set_key,\@function
|
||||
.align 64
|
||||
private_RC4_set_key:
|
||||
stmg %r6,%r8,48($sp)
|
||||
RC4_set_key:
|
||||
stm${g} %r6,%r8,6*$SIZE_T($sp)
|
||||
lhi $cnt,256
|
||||
la $idx,0(%r0)
|
||||
sth $idx,0($key)
|
||||
@ -180,9 +208,9 @@ private_RC4_set_key:
|
||||
la $iinp,0(%r0)
|
||||
j .L2ndloop
|
||||
.Ldone:
|
||||
lmg %r6,%r8,48($sp)
|
||||
lm${g} %r6,%r8,6*$SIZE_T($sp)
|
||||
br $rp
|
||||
.size private_RC4_set_key,.-private_RC4_set_key
|
||||
.size RC4_set_key,.-RC4_set_key
|
||||
|
||||
___
|
||||
}
|
||||
@ -203,3 +231,4 @@ RC4_options:
|
||||
___
|
||||
|
||||
print $code;
|
||||
close STDOUT; # force flush
|
||||
|
||||
@ -4,7 +4,7 @@
|
||||
#include <setjmp.h>
|
||||
#include <signal.h>
|
||||
|
||||
extern unsigned long OPENSSL_s390xcap_P;
|
||||
extern unsigned long OPENSSL_s390xcap_P[];
|
||||
|
||||
static sigjmp_buf ill_jmp;
|
||||
static void ill_handler (int sig) { siglongjmp(ill_jmp,sig); }
|
||||
@ -16,7 +16,9 @@ void OPENSSL_cpuid_setup(void)
|
||||
sigset_t oset;
|
||||
struct sigaction ill_act,oact;
|
||||
|
||||
if (OPENSSL_s390xcap_P) return;
|
||||
if (OPENSSL_s390xcap_P[0]) return;
|
||||
|
||||
OPENSSL_s390xcap_P[0] = 1UL<<(8*sizeof(unsigned long)-1);
|
||||
|
||||
memset(&ill_act,0,sizeof(ill_act));
|
||||
ill_act.sa_handler = ill_handler;
|
||||
@ -27,10 +29,8 @@ void OPENSSL_cpuid_setup(void)
|
||||
sigaction (SIGILL,&ill_act,&oact);
|
||||
|
||||
/* protection against missing store-facility-list-extended */
|
||||
if (sigsetjmp(ill_jmp,0) == 0)
|
||||
OPENSSL_s390xcap_P = OPENSSL_s390x_facilities();
|
||||
else
|
||||
OPENSSL_s390xcap_P = 1UL<<63;
|
||||
if (sigsetjmp(ill_jmp,1) == 0)
|
||||
OPENSSL_s390x_facilities();
|
||||
|
||||
sigaction (SIGILL,&oact,NULL);
|
||||
sigprocmask(SIG_SETMASK,&oset,NULL);
|
||||
|
||||
@ -5,10 +5,14 @@
|
||||
.align 16
|
||||
OPENSSL_s390x_facilities:
|
||||
lghi %r0,0
|
||||
.long 0xb2b0f010 # stfle 16(%r15)
|
||||
lg %r2,16(%r15)
|
||||
larl %r1,OPENSSL_s390xcap_P
|
||||
stg %r2,0(%r1)
|
||||
larl %r2,OPENSSL_s390xcap_P
|
||||
stg %r0,8(%r2)
|
||||
.long 0xb2b02000 # stfle 0(%r2)
|
||||
brc 8,.Ldone
|
||||
lghi %r0,1
|
||||
.long 0xb2b02000 # stfle 0(%r2)
|
||||
.Ldone:
|
||||
lg %r2,0(%r2)
|
||||
br %r14
|
||||
.size OPENSSL_s390x_facilities,.-OPENSSL_s390x_facilities
|
||||
|
||||
@ -58,6 +62,9 @@ OPENSSL_wipe_cpu:
|
||||
.type OPENSSL_cleanse,@function
|
||||
.align 16
|
||||
OPENSSL_cleanse:
|
||||
#if !defined(__s390x__) && !defined(__s390x)
|
||||
llgfr %r3,%r3
|
||||
#endif
|
||||
lghi %r4,15
|
||||
lghi %r0,0
|
||||
clgr %r3,%r4
|
||||
@ -89,4 +96,4 @@ OPENSSL_cleanse:
|
||||
.section .init
|
||||
brasl %r14,OPENSSL_cpuid_setup
|
||||
|
||||
.comm OPENSSL_s390xcap_P,8,8
|
||||
.comm OPENSSL_s390xcap_P,16,8
|
||||
|
||||
@ -21,9 +21,28 @@
|
||||
# instructions to favour dual-issue z10 pipeline. On z10 hardware is
|
||||
# "only" ~2.3x faster than software.
|
||||
|
||||
# November 2010.
|
||||
#
|
||||
# Adapt for -m31 build. If kernel supports what's called "highgprs"
|
||||
# feature on Linux [see /proc/cpuinfo], it's possible to use 64-bit
|
||||
# instructions and achieve "64-bit" performance even in 31-bit legacy
|
||||
# application context. The feature is not specific to any particular
|
||||
# processor, as long as it's "z-CPU". Latter implies that the code
|
||||
# remains z/Architecture specific.
|
||||
|
||||
$kimdfunc=1; # magic function code for kimd instruction
|
||||
|
||||
$output=shift;
|
||||
$flavour = shift;
|
||||
|
||||
if ($flavour =~ /3[12]/) {
|
||||
$SIZE_T=4;
|
||||
$g="";
|
||||
} else {
|
||||
$SIZE_T=8;
|
||||
$g="g";
|
||||
}
|
||||
|
||||
while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {}
|
||||
open STDOUT,">$output";
|
||||
|
||||
$K_00_39="%r0"; $K=$K_00_39;
|
||||
@ -42,13 +61,14 @@ $t1="%r11";
|
||||
@X=("%r12","%r13","%r14");
|
||||
$sp="%r15";
|
||||
|
||||
$frame=160+16*4;
|
||||
$stdframe=16*$SIZE_T+4*8;
|
||||
$frame=$stdframe+16*4;
|
||||
|
||||
sub Xupdate {
|
||||
my $i=shift;
|
||||
|
||||
$code.=<<___ if ($i==15);
|
||||
lg $prefetch,160($sp) ### Xupdate(16) warm-up
|
||||
lg $prefetch,$stdframe($sp) ### Xupdate(16) warm-up
|
||||
lr $X[0],$X[2]
|
||||
___
|
||||
return if ($i&1); # Xupdate is vectorized and executed every 2nd cycle
|
||||
@ -58,8 +78,8 @@ $code.=<<___ if ($i<16);
|
||||
___
|
||||
$code.=<<___ if ($i>=16);
|
||||
xgr $X[0],$prefetch ### Xupdate($i)
|
||||
lg $prefetch,`160+4*(($i+2)%16)`($sp)
|
||||
xg $X[0],`160+4*(($i+8)%16)`($sp)
|
||||
lg $prefetch,`$stdframe+4*(($i+2)%16)`($sp)
|
||||
xg $X[0],`$stdframe+4*(($i+8)%16)`($sp)
|
||||
xgr $X[0],$prefetch
|
||||
rll $X[0],$X[0],1
|
||||
rllg $X[1],$X[0],32
|
||||
@ -68,7 +88,7 @@ $code.=<<___ if ($i>=16);
|
||||
lr $X[2],$X[1] # feedback
|
||||
___
|
||||
$code.=<<___ if ($i<=70);
|
||||
stg $X[0],`160+4*($i%16)`($sp)
|
||||
stg $X[0],`$stdframe+4*($i%16)`($sp)
|
||||
___
|
||||
unshift(@X,pop(@X));
|
||||
}
|
||||
@ -148,9 +168,9 @@ $code.=<<___ if ($kimdfunc);
|
||||
tmhl %r0,0x4000 # check for message-security assist
|
||||
jz .Lsoftware
|
||||
lghi %r0,0
|
||||
la %r1,16($sp)
|
||||
la %r1,`2*$SIZE_T`($sp)
|
||||
.long 0xb93e0002 # kimd %r0,%r2
|
||||
lg %r0,16($sp)
|
||||
lg %r0,`2*$SIZE_T`($sp)
|
||||
tmhh %r0,`0x8000>>$kimdfunc`
|
||||
jz .Lsoftware
|
||||
lghi %r0,$kimdfunc
|
||||
@ -165,11 +185,11 @@ $code.=<<___ if ($kimdfunc);
|
||||
___
|
||||
$code.=<<___;
|
||||
lghi %r1,-$frame
|
||||
stg $ctx,16($sp)
|
||||
stmg %r6,%r15,48($sp)
|
||||
st${g} $ctx,`2*$SIZE_T`($sp)
|
||||
stm${g} %r6,%r15,`6*$SIZE_T`($sp)
|
||||
lgr %r0,$sp
|
||||
la $sp,0(%r1,$sp)
|
||||
stg %r0,0($sp)
|
||||
st${g} %r0,0($sp)
|
||||
|
||||
larl $t0,Ktable
|
||||
llgf $A,0($ctx)
|
||||
@ -199,7 +219,7 @@ ___
|
||||
for (;$i<80;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); }
|
||||
$code.=<<___;
|
||||
|
||||
lg $ctx,`$frame+16`($sp)
|
||||
l${g} $ctx,`$frame+2*$SIZE_T`($sp)
|
||||
la $inp,64($inp)
|
||||
al $A,0($ctx)
|
||||
al $B,4($ctx)
|
||||
@ -211,13 +231,13 @@ $code.=<<___;
|
||||
st $C,8($ctx)
|
||||
st $D,12($ctx)
|
||||
st $E,16($ctx)
|
||||
brct $len,.Lloop
|
||||
brct${g} $len,.Lloop
|
||||
|
||||
lmg %r6,%r15,`$frame+48`($sp)
|
||||
lm${g} %r6,%r15,`$frame+6*$SIZE_T`($sp)
|
||||
br %r14
|
||||
.size sha1_block_data_order,.-sha1_block_data_order
|
||||
.string "SHA1 block transform for s390x, CRYPTOGAMS by <appro\@openssl.org>"
|
||||
.comm OPENSSL_s390xcap_P,8,8
|
||||
.comm OPENSSL_s390xcap_P,16,8
|
||||
___
|
||||
|
||||
$code =~ s/\`([^\`]*)\`/eval $1/gem;
|
||||
|
||||
@ -26,6 +26,26 @@
|
||||
# favour dual-issue z10 pipeline. Hardware SHA256/512 is ~4.7x faster
|
||||
# than software.
|
||||
|
||||
# November 2010.
|
||||
#
|
||||
# Adapt for -m31 build. If kernel supports what's called "highgprs"
|
||||
# feature on Linux [see /proc/cpuinfo], it's possible to use 64-bit
|
||||
# instructions and achieve "64-bit" performance even in 31-bit legacy
|
||||
# application context. The feature is not specific to any particular
|
||||
# processor, as long as it's "z-CPU". Latter implies that the code
|
||||
# remains z/Architecture specific. On z900 SHA256 was measured to
|
||||
# perform 2.4x and SHA512 - 13x better than code generated by gcc 4.3.
|
||||
|
||||
$flavour = shift;
|
||||
|
||||
if ($flavour =~ /3[12]/) {
|
||||
$SIZE_T=4;
|
||||
$g="";
|
||||
} else {
|
||||
$SIZE_T=8;
|
||||
$g="g";
|
||||
}
|
||||
|
||||
$t0="%r0";
|
||||
$t1="%r1";
|
||||
$ctx="%r2"; $t2="%r2";
|
||||
@ -44,7 +64,7 @@ $tbl="%r13";
|
||||
$T1="%r14";
|
||||
$sp="%r15";
|
||||
|
||||
$output=shift;
|
||||
while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {}
|
||||
open STDOUT,">$output";
|
||||
|
||||
if ($output =~ /512/) {
|
||||
@ -78,7 +98,8 @@ if ($output =~ /512/) {
|
||||
}
|
||||
$Func="sha${label}_block_data_order";
|
||||
$Table="K${label}";
|
||||
$frame=160+16*$SZ;
|
||||
$stdframe=16*$SIZE_T+4*8;
|
||||
$frame=$stdframe+16*$SZ;
|
||||
|
||||
sub BODY_00_15 {
|
||||
my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_;
|
||||
@ -93,9 +114,9 @@ $code.=<<___;
|
||||
xgr $t0,$t1
|
||||
$ROT $t1,$t1,`$Sigma1[2]-$Sigma1[1]`
|
||||
xgr $t2,$g
|
||||
$ST $T1,`160+$SZ*($i%16)`($sp)
|
||||
$ST $T1,`$stdframe+$SZ*($i%16)`($sp)
|
||||
xgr $t0,$t1 # Sigma1(e)
|
||||
la $T1,0($T1,$h) # T1+=h
|
||||
algr $T1,$h # T1+=h
|
||||
ngr $t2,$e
|
||||
lgr $t1,$a
|
||||
algr $T1,$t0 # T1+=Sigma1(e)
|
||||
@ -113,7 +134,7 @@ $code.=<<___;
|
||||
ngr $t2,$b
|
||||
algr $h,$T1 # h+=T1
|
||||
ogr $t2,$t1 # Maj(a,b,c)
|
||||
la $d,0($d,$T1) # d+=T1
|
||||
algr $d,$T1 # d+=T1
|
||||
algr $h,$t2 # h+=Maj(a,b,c)
|
||||
___
|
||||
}
|
||||
@ -122,19 +143,19 @@ sub BODY_16_XX {
|
||||
my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_;
|
||||
|
||||
$code.=<<___;
|
||||
$LD $T1,`160+$SZ*(($i+1)%16)`($sp) ### $i
|
||||
$LD $t1,`160+$SZ*(($i+14)%16)`($sp)
|
||||
$LD $T1,`$stdframe+$SZ*(($i+1)%16)`($sp) ### $i
|
||||
$LD $t1,`$stdframe+$SZ*(($i+14)%16)`($sp)
|
||||
$ROT $t0,$T1,$sigma0[0]
|
||||
$SHR $T1,$sigma0[2]
|
||||
$ROT $t2,$t0,`$sigma0[1]-$sigma0[0]`
|
||||
xgr $T1,$t0
|
||||
$ROT $t0,$t1,$sigma1[0]
|
||||
xgr $T1,$t2 # sigma0(X[i+1])
|
||||
xgr $T1,$t2 # sigma0(X[i+1])
|
||||
$SHR $t1,$sigma1[2]
|
||||
$ADD $T1,`160+$SZ*($i%16)`($sp) # +=X[i]
|
||||
$ADD $T1,`$stdframe+$SZ*($i%16)`($sp) # +=X[i]
|
||||
xgr $t1,$t0
|
||||
$ROT $t0,$t0,`$sigma1[1]-$sigma1[0]`
|
||||
$ADD $T1,`160+$SZ*(($i+9)%16)`($sp) # +=X[i+9]
|
||||
$ADD $T1,`$stdframe+$SZ*(($i+9)%16)`($sp) # +=X[i+9]
|
||||
xgr $t1,$t0 # sigma1(X[i+14])
|
||||
algr $T1,$t1 # +=sigma1(X[i+14])
|
||||
___
|
||||
@ -212,6 +233,7 @@ $code.=<<___;
|
||||
.globl $Func
|
||||
.type $Func,\@function
|
||||
$Func:
|
||||
sllg $len,$len,`log(16*$SZ)/log(2)`
|
||||
___
|
||||
$code.=<<___ if ($kimdfunc);
|
||||
larl %r1,OPENSSL_s390xcap_P
|
||||
@ -219,15 +241,15 @@ $code.=<<___ if ($kimdfunc);
|
||||
tmhl %r0,0x4000 # check for message-security assist
|
||||
jz .Lsoftware
|
||||
lghi %r0,0
|
||||
la %r1,16($sp)
|
||||
la %r1,`2*$SIZE_T`($sp)
|
||||
.long 0xb93e0002 # kimd %r0,%r2
|
||||
lg %r0,16($sp)
|
||||
lg %r0,`2*$SIZE_T`($sp)
|
||||
tmhh %r0,`0x8000>>$kimdfunc`
|
||||
jz .Lsoftware
|
||||
lghi %r0,$kimdfunc
|
||||
lgr %r1,$ctx
|
||||
lgr %r2,$inp
|
||||
sllg %r3,$len,`log(16*$SZ)/log(2)`
|
||||
lgr %r3,$len
|
||||
.long 0xb93e0002 # kimd %r0,%r2
|
||||
brc 1,.-4 # pay attention to "partial completion"
|
||||
br %r14
|
||||
@ -235,13 +257,12 @@ $code.=<<___ if ($kimdfunc);
|
||||
.Lsoftware:
|
||||
___
|
||||
$code.=<<___;
|
||||
sllg $len,$len,`log(16*$SZ)/log(2)`
|
||||
lghi %r1,-$frame
|
||||
agr $len,$inp
|
||||
stmg $ctx,%r15,16($sp)
|
||||
la $len,0($len,$inp)
|
||||
stm${g} $ctx,%r15,`2*$SIZE_T`($sp)
|
||||
lgr %r0,$sp
|
||||
la $sp,0(%r1,$sp)
|
||||
stg %r0,0($sp)
|
||||
st${g} %r0,0($sp)
|
||||
|
||||
larl $tbl,$Table
|
||||
$LD $A,`0*$SZ`($ctx)
|
||||
@ -265,7 +286,7 @@ $code.=<<___;
|
||||
clgr $len,$t0
|
||||
jne .Lrounds_16_xx
|
||||
|
||||
lg $ctx,`$frame+16`($sp)
|
||||
l${g} $ctx,`$frame+2*$SIZE_T`($sp)
|
||||
la $inp,`16*$SZ`($inp)
|
||||
$ADD $A,`0*$SZ`($ctx)
|
||||
$ADD $B,`1*$SZ`($ctx)
|
||||
@ -283,14 +304,14 @@ $code.=<<___;
|
||||
$ST $F,`5*$SZ`($ctx)
|
||||
$ST $G,`6*$SZ`($ctx)
|
||||
$ST $H,`7*$SZ`($ctx)
|
||||
clg $inp,`$frame+32`($sp)
|
||||
cl${g} $inp,`$frame+4*$SIZE_T`($sp)
|
||||
jne .Lloop
|
||||
|
||||
lmg %r6,%r15,`$frame+48`($sp)
|
||||
lm${g} %r6,%r15,`$frame+6*$SIZE_T`($sp)
|
||||
br %r14
|
||||
.size $Func,.-$Func
|
||||
.string "SHA${label} block transform for s390x, CRYPTOGAMS by <appro\@openssl.org>"
|
||||
.comm OPENSSL_s390xcap_P,8,8
|
||||
.comm OPENSSL_s390xcap_P,16,8
|
||||
___
|
||||
|
||||
$code =~ s/\`([^\`]*)\`/eval $1/gem;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user