generic-library/openssl
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

ppc64-mont.pl: eliminate dependency on GPRs' upper halves.

Browse Source
This commit is contained in:
Andy Polyakov 2013-11-27 22:50:00 +01:00
parent 07f3e4f3f9
commit b9e87d07cb
2 changed files with 600 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

661
crypto/bn/asm/ppc64-mont.pl
View File

@ -1,7 +1,7 @@
#!/usr/bin/env perl
# ====================================================================
# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
# project. The module is, however, dual licensed under OpenSSL and
# CRYPTOGAMS licenses depending on where you obtain it. For further
# details see http://www.openssl.org/~appro/cryptogams/.
@ -65,6 +65,14 @@
# others alternative would be to break dependence on upper halves of
# GPRs by sticking to 32-bit integer operations...
# December 2012
# Remove above mentioned dependence on GPRs' upper halves in 32-bit
# build. No signal masking overhead, but integer instructions are
# *more* numerous... It's still "universally" faster than 32-bit
# ppc-mont.pl, but improvement coefficient is not as impressive
# for longer keys...
$flavour = shift;
if ($flavour =~ /32/) {
@ -110,6 +118,9 @@ $tp="r10";
$j="r11";
$i="r12";
# non-volatile registers
$c1="r19";
$n1="r20";
$a1="r21";
$nap_d="r22"; # interleaved ap and np in double format
$a0="r23"; # ap[0]
$t0="r24"; # temporary registers
@ -180,8 +191,8 @@ $T3a="f30"; $T3b="f31";
# . .
# +-------------------------------+
# . .
# -12*size_t +-------------------------------+
# | 10 saved gpr, r22-r31 |
# -13*size_t +-------------------------------+
# | 13 saved gpr, r19-r31 |
# . .
# . .
# -12*8 +-------------------------------+
@ -215,6 +226,9 @@ $code=<<___;
mr $i,$sp
$STUX $sp,$sp,$tp ; alloca
$PUSH r19,`-12*8-13*$SIZE_T`($i)
$PUSH r20,`-12*8-12*$SIZE_T`($i)
$PUSH r21,`-12*8-11*$SIZE_T`($i)
$PUSH r22,`-12*8-10*$SIZE_T`($i)
$PUSH r23,`-12*8-9*$SIZE_T`($i)
$PUSH r24,`-12*8-8*$SIZE_T`($i)
@ -237,40 +251,26 @@ $code=<<___;
stfd f29,`-3*8`($i)
stfd f30,`-2*8`($i)
stfd f31,`-1*8`($i)
___
$code.=<<___ if ($SIZE_T==8);
ld $a0,0($ap) ; pull ap[0] value
ld $n0,0($n0) ; pull n0[0] value
ld $t3,0($bp) ; bp[0]
___
$code.=<<___ if ($SIZE_T==4);
mr $t1,$n0
lwz $a0,0($ap) ; pull ap[0,1] value
lwz $t0,4($ap)
lwz $n0,0($t1) ; pull n0[0,1] value
lwz $t1,4($t1)
lwz $t3,0($bp) ; bp[0,1]
lwz $t2,4($bp)
insrdi $a0,$t0,32,0
insrdi $n0,$t1,32,0
insrdi $t3,$t2,32,0
___
$code.=<<___;
addi $tp,$sp,`$FRAME+$TRANSFER+8+64`
li $i,-64
add $nap_d,$tp,$num
and $nap_d,$nap_d,$i ; align to 64 bytes
mulld $t7,$a0,$t3 ; ap[0]*bp[0]
; nap_d is off by 1, because it's used with stfdu/lfdu
addi $nap_d,$nap_d,-8
srwi $j,$num,`3+1` ; counter register, num/2
mulld $t7,$t7,$n0 ; tp[0]*n0
addi $j,$j,-1
addi $tp,$sp,`$FRAME+$TRANSFER-8`
li $carry,0
mtctr $j
___
$code.=<<___ if ($SIZE_T==8);
ld $a0,0($ap) ; pull ap[0] value
ld $t3,0($bp) ; bp[0]
ld $n0,0($n0) ; pull n0[0] value
mulld $t7,$a0,$t3 ; ap[0]*bp[0]
; transfer bp[0] to FPU as 4x16-bit values
extrdi $t0,$t3,16,48
extrdi $t1,$t3,16,32
@ -280,6 +280,8 @@ $code.=<<___;
std $t1,`$FRAME+8`($sp)
std $t2,`$FRAME+16`($sp)
std $t3,`$FRAME+24`($sp)
mulld $t7,$t7,$n0 ; tp[0]*n0
; transfer (ap[0]*bp[0])*n0 to FPU as 4x16-bit values
extrdi $t4,$t7,16,48
extrdi $t5,$t7,16,32
@ -289,21 +291,61 @@ $code.=<<___;
std $t5,`$FRAME+40`($sp)
std $t6,`$FRAME+48`($sp)
std $t7,`$FRAME+56`($sp)
___
$code.=<<___ if ($SIZE_T==8);
lwz $t0,4($ap) ; load a[j] as 32-bit word pair
lwz $t1,0($ap)
lwz $t2,12($ap) ; load a[j+1] as 32-bit word pair
extrdi $t0,$a0,32,32 ; lwz $t0,4($ap)
extrdi $t1,$a0,32,0 ; lwz $t1,0($ap)
lwz $t2,12($ap) ; load a[1] as 32-bit word pair
lwz $t3,8($ap)
lwz $t4,4($np) ; load n[j] as 32-bit word pair
lwz $t4,4($np) ; load n[0] as 32-bit word pair
lwz $t5,0($np)
lwz $t6,12($np) ; load n[j+1] as 32-bit word pair
lwz $t6,12($np) ; load n[1] as 32-bit word pair
lwz $t7,8($np)
___
$code.=<<___ if ($SIZE_T==4);
lwz $t0,0($ap) ; load a[j..j+3] as 32-bit word pairs
lwz $t1,4($ap)
lwz $t2,8($ap)
lwz $a0,0($ap) ; pull ap[0,1] value
mr $n1,$n0
lwz $a1,4($ap)
li $c1,0
lwz $t1,0($bp) ; bp[0,1]
lwz $t3,4($bp)
lwz $n0,0($n1) ; pull n0[0,1] value
lwz $n1,4($n1)
mullw $t4,$a0,$t1 ; mulld ap[0]*bp[0]
mulhwu $t5,$a0,$t1
mullw $t6,$a1,$t1
mullw $t7,$a0,$t3
add $t5,$t5,$t6
add $t5,$t5,$t7
; transfer bp[0] to FPU as 4x16-bit values
extrwi $t0,$t1,16,16
extrwi $t1,$t1,16,0
extrwi $t2,$t3,16,16
extrwi $t3,$t3,16,0
std $t0,`$FRAME+0`($sp) ; yes, std in 32-bit build
std $t1,`$FRAME+8`($sp)
std $t2,`$FRAME+16`($sp)
std $t3,`$FRAME+24`($sp)
mullw $t0,$t4,$n0 ; mulld tp[0]*n0
mulhwu $t1,$t4,$n0
mullw $t2,$t5,$n0
mullw $t3,$t4,$n1
add $t1,$t1,$t2
add $t1,$t1,$t3
; transfer (ap[0]*bp[0])*n0 to FPU as 4x16-bit values
extrwi $t4,$t0,16,16
extrwi $t5,$t0,16,0
extrwi $t6,$t1,16,16
extrwi $t7,$t1,16,0
std $t4,`$FRAME+32`($sp) ; yes, std in 32-bit build
std $t5,`$FRAME+40`($sp)
std $t6,`$FRAME+48`($sp)
std $t7,`$FRAME+56`($sp)
mr $t0,$a0 ; lwz $t0,0($ap)
mr $t1,$a1 ; lwz $t1,4($ap)
lwz $t2,8($ap) ; load a[j..j+3] as 32-bit word pairs
lwz $t3,12($ap)
lwz $t4,0($np) ; load n[j..j+3] as 32-bit word pairs
lwz $t5,4($np)
@ -319,7 +361,7 @@ $code.=<<___;
lfd $nb,`$FRAME+40`($sp)
lfd $nc,`$FRAME+48`($sp)
lfd $nd,`$FRAME+56`($sp)
std $t0,`$FRAME+64`($sp)
std $t0,`$FRAME+64`($sp) ; yes, std even in 32-bit build
std $t1,`$FRAME+72`($sp)
std $t2,`$FRAME+80`($sp)
std $t3,`$FRAME+88`($sp)
@ -441,7 +483,7 @@ $code.=<<___ if ($SIZE_T==4);
lwz $t7,12($np)
___
$code.=<<___;
std $t0,`$FRAME+64`($sp)
std $t0,`$FRAME+64`($sp) ; yes, std even in 32-bit build
std $t1,`$FRAME+72`($sp)
std $t2,`$FRAME+80`($sp)
std $t3,`$FRAME+88`($sp)
@ -449,6 +491,9 @@ $code.=<<___;
std $t5,`$FRAME+104`($sp)
std $t6,`$FRAME+112`($sp)
std $t7,`$FRAME+120`($sp)
___
if ($SIZE_T==8 or $flavour =~ /osx/) {
$code.=<<___;
ld $t0,`$FRAME+0`($sp)
ld $t1,`$FRAME+8`($sp)
ld $t2,`$FRAME+16`($sp)
@ -457,6 +502,20 @@ $code.=<<___;
ld $t5,`$FRAME+40`($sp)
ld $t6,`$FRAME+48`($sp)
ld $t7,`$FRAME+56`($sp)
___
} else {
$code.=<<___;
lwz $t1,`$FRAME+0`($sp)
lwz $t0,`$FRAME+4`($sp)
lwz $t3,`$FRAME+8`($sp)
lwz $t2,`$FRAME+12`($sp)
lwz $t5,`$FRAME+16`($sp)
lwz $t4,`$FRAME+20`($sp)
lwz $t7,`$FRAME+24`($sp)
lwz $t6,`$FRAME+28`($sp)
___
}
$code.=<<___;
lfd $A0,`$FRAME+64`($sp)
lfd $A1,`$FRAME+72`($sp)
lfd $A2,`$FRAME+80`($sp)
@ -488,7 +547,9 @@ $code.=<<___;
fmadd $T0b,$A0,$bb,$dotb
stfd $A2,24($nap_d) ; save a[j+1] in double format
stfd $A3,32($nap_d)
___
if ($SIZE_T==8 or $flavour =~ /osx/) {
$code.=<<___;
fmadd $T1a,$A0,$bc,$T1a
fmadd $T1b,$A0,$bd,$T1b
fmadd $T2a,$A1,$bc,$T2a
@ -561,11 +622,123 @@ $code.=<<___;
stfd $T3b,`$FRAME+56`($sp)
std $t0,8($tp) ; tp[j-1]
stdu $t4,16($tp) ; tp[j]
___
} else {
$code.=<<___;
fmadd $T1a,$A0,$bc,$T1a
fmadd $T1b,$A0,$bd,$T1b
addc $t0,$t0,$carry
adde $t1,$t1,$c1
srwi $carry,$t0,16
fmadd $T2a,$A1,$bc,$T2a
fmadd $T2b,$A1,$bd,$T2b
stfd $N0,40($nap_d) ; save n[j] in double format
stfd $N1,48($nap_d)
srwi $c1,$t1,16
insrwi $carry,$t1,16,0
fmadd $T3a,$A2,$bc,$T3a
fmadd $T3b,$A2,$bd,$T3b
addc $t2,$t2,$carry
adde $t3,$t3,$c1
srwi $carry,$t2,16
fmul $dota,$A3,$bc
fmul $dotb,$A3,$bd
stfd $N2,56($nap_d) ; save n[j+1] in double format
stfdu $N3,64($nap_d)
insrwi $t0,$t2,16,0 ; 0..31 bits
srwi $c1,$t3,16
insrwi $carry,$t3,16,0
fmadd $T1a,$N1,$na,$T1a
fmadd $T1b,$N1,$nb,$T1b
lwz $t3,`$FRAME+32`($sp) ; permuted $t1
lwz $t2,`$FRAME+36`($sp) ; permuted $t0
addc $t4,$t4,$carry
adde $t5,$t5,$c1
srwi $carry,$t4,16
fmadd $T2a,$N2,$na,$T2a
fmadd $T2b,$N2,$nb,$T2b
srwi $c1,$t5,16
insrwi $carry,$t5,16,0
fmadd $T3a,$N3,$na,$T3a
fmadd $T3b,$N3,$nb,$T3b
addc $t6,$t6,$carry
adde $t7,$t7,$c1
srwi $carry,$t6,16
fmadd $T0a,$N0,$na,$T0a
fmadd $T0b,$N0,$nb,$T0b
insrwi $t4,$t6,16,0 ; 32..63 bits
srwi $c1,$t7,16
insrwi $carry,$t7,16,0
fmadd $T1a,$N0,$nc,$T1a
fmadd $T1b,$N0,$nd,$T1b
lwz $t7,`$FRAME+40`($sp) ; permuted $t3
lwz $t6,`$FRAME+44`($sp) ; permuted $t2
addc $t2,$t2,$carry
adde $t3,$t3,$c1
srwi $carry,$t2,16
fmadd $T2a,$N1,$nc,$T2a
fmadd $T2b,$N1,$nd,$T2b
stw $t0,12($tp) ; tp[j-1]
stw $t4,8($tp)
srwi $c1,$t3,16
insrwi $carry,$t3,16,0
fmadd $T3a,$N2,$nc,$T3a
fmadd $T3b,$N2,$nd,$T3b
lwz $t1,`$FRAME+48`($sp) ; permuted $t5
lwz $t0,`$FRAME+52`($sp) ; permuted $t4
addc $t6,$t6,$carry
adde $t7,$t7,$c1
srwi $carry,$t6,16
fmadd $dota,$N3,$nc,$dota
fmadd $dotb,$N3,$nd,$dotb
insrwi $t2,$t6,16,0 ; 64..95 bits
srwi $c1,$t7,16
insrwi $carry,$t7,16,0
fctid $T0a,$T0a
fctid $T0b,$T0b
lwz $t5,`$FRAME+56`($sp) ; permuted $t7
lwz $t4,`$FRAME+60`($sp) ; permuted $t6
addc $t0,$t0,$carry
adde $t1,$t1,$c1
srwi $carry,$t0,16
fctid $T1a,$T1a
fctid $T1b,$T1b
srwi $c1,$t1,16
insrwi $carry,$t1,16,0
fctid $T2a,$T2a
fctid $T2b,$T2b
addc $t4,$t4,$carry
adde $t5,$t5,$c1
srwi $carry,$t4,16
fctid $T3a,$T3a
fctid $T3b,$T3b
insrwi $t0,$t4,16,0 ; 96..127 bits
srwi $c1,$t5,16
insrwi $carry,$t5,16,0
stfd $T0a,`$FRAME+0`($sp)
stfd $T0b,`$FRAME+8`($sp)
stfd $T1a,`$FRAME+16`($sp)
stfd $T1b,`$FRAME+24`($sp)
stfd $T2a,`$FRAME+32`($sp)
stfd $T2b,`$FRAME+40`($sp)
stfd $T3a,`$FRAME+48`($sp)
stfd $T3b,`$FRAME+56`($sp)
stw $t2,20($tp) ; tp[j]
stwu $t0,16($tp)
___
}
$code.=<<___;
bdnz- L1st
fctid $dota,$dota
fctid $dotb,$dotb
___
if ($SIZE_T==8 or $flavour =~ /osx/) {
$code.=<<___;
ld $t0,`$FRAME+0`($sp)
ld $t1,`$FRAME+8`($sp)
ld $t2,`$FRAME+16`($sp)
@ -611,33 +784,117 @@ $code.=<<___;
insrdi $t6,$t7,48,0
srdi $ovf,$t7,48
std $t6,8($tp) ; tp[num-1]
___
} else {
$code.=<<___;
lwz $t1,`$FRAME+0`($sp)
lwz $t0,`$FRAME+4`($sp)
lwz $t3,`$FRAME+8`($sp)
lwz $t2,`$FRAME+12`($sp)
lwz $t5,`$FRAME+16`($sp)
lwz $t4,`$FRAME+20`($sp)
lwz $t7,`$FRAME+24`($sp)
lwz $t6,`$FRAME+28`($sp)
stfd $dota,`$FRAME+64`($sp)
stfd $dotb,`$FRAME+72`($sp)
addc $t0,$t0,$carry
adde $t1,$t1,$c1
srwi $carry,$t0,16
insrwi $carry,$t1,16,0
srwi $c1,$t1,16
addc $t2,$t2,$carry
adde $t3,$t3,$c1
srwi $carry,$t2,16
insrwi $t0,$t2,16,0 ; 0..31 bits
insrwi $carry,$t3,16,0
srwi $c1,$t3,16
addc $t4,$t4,$carry
adde $t5,$t5,$c1
srwi $carry,$t4,16
insrwi $carry,$t5,16,0
srwi $c1,$t5,16
addc $t6,$t6,$carry
adde $t7,$t7,$c1
srwi $carry,$t6,16
insrwi $t4,$t6,16,0 ; 32..63 bits
insrwi $carry,$t7,16,0
srwi $c1,$t7,16
stw $t0,12($tp) ; tp[j-1]
stw $t4,8($tp)
lwz $t3,`$FRAME+32`($sp) ; permuted $t1
lwz $t2,`$FRAME+36`($sp) ; permuted $t0
lwz $t7,`$FRAME+40`($sp) ; permuted $t3
lwz $t6,`$FRAME+44`($sp) ; permuted $t2
lwz $t1,`$FRAME+48`($sp) ; permuted $t5
lwz $t0,`$FRAME+52`($sp) ; permuted $t4
lwz $t5,`$FRAME+56`($sp) ; permuted $t7
lwz $t4,`$FRAME+60`($sp) ; permuted $t6
addc $t2,$t2,$carry
adde $t3,$t3,$c1
srwi $carry,$t2,16
insrwi $carry,$t3,16,0
srwi $c1,$t3,16
addc $t6,$t6,$carry
adde $t7,$t7,$c1
srwi $carry,$t6,16
insrwi $t2,$t6,16,0 ; 64..95 bits
insrwi $carry,$t7,16,0
srwi $c1,$t7,16
addc $t0,$t0,$carry
adde $t1,$t1,$c1
srwi $carry,$t0,16
insrwi $carry,$t1,16,0
srwi $c1,$t1,16
addc $t4,$t4,$carry
adde $t5,$t5,$c1
srwi $carry,$t4,16
insrwi $t0,$t4,16,0 ; 96..127 bits
insrwi $carry,$t5,16,0
srwi $c1,$t5,16
stw $t2,20($tp) ; tp[j]
stwu $t0,16($tp)
lwz $t7,`$FRAME+64`($sp)
lwz $t6,`$FRAME+68`($sp)
lwz $t5,`$FRAME+72`($sp)
lwz $t4,`$FRAME+76`($sp)
addc $t6,$t6,$carry
adde $t7,$t7,$c1
srwi $carry,$t6,16
insrwi $carry,$t7,16,0
srwi $c1,$t7,16
addc $t4,$t4,$carry
adde $t5,$t5,$c1
insrwi $t6,$t4,16,0
srwi $t4,$t4,16
insrwi $t4,$t5,16,0
srwi $ovf,$t5,16
stw $t6,12($tp) ; tp[num-1]
stw $t4,8($tp)
___
}
$code.=<<___;
slwi $t7,$num,2
subf $nap_d,$t7,$nap_d ; rewind pointer
li $i,8 ; i=1
.align 5
Louter:
addi $tp,$sp,`$FRAME+$TRANSFER`
li $carry,0
mtctr $j
___
$code.=<<___ if ($SIZE_T==8);
ldx $t3,$bp,$i ; bp[i]
___
$code.=<<___ if ($SIZE_T==4);
add $t0,$bp,$i
lwz $t3,0($t0) ; bp[i,i+1]
lwz $t0,4($t0)
insrdi $t3,$t0,32,0
___
$code.=<<___;
ldx $t3,$bp,$i ; bp[i]
ld $t6,`$FRAME+$TRANSFER+8`($sp) ; tp[0]
mulld $t7,$a0,$t3 ; ap[0]*bp[i]
addi $tp,$sp,`$FRAME+$TRANSFER`
add $t7,$t7,$t6 ; ap[0]*bp[i]+tp[0]
li $carry,0
mulld $t7,$t7,$n0 ; tp[0]*n0
mtctr $j
mulld $t7,$a0,$t3 ; ap[0]*bp[i]
add $t7,$t7,$t6 ; ap[0]*bp[i]+tp[0]
; transfer bp[i] to FPU as 4x16-bit values
extrdi $t0,$t3,16,48
extrdi $t1,$t3,16,32
@ -647,6 +904,8 @@ $code.=<<___;
std $t1,`$FRAME+8`($sp)
std $t2,`$FRAME+16`($sp)
std $t3,`$FRAME+24`($sp)
mulld $t7,$t7,$n0 ; tp[0]*n0
; transfer (ap[0]*bp[i]+tp[0])*n0 to FPU as 4x16-bit values
extrdi $t4,$t7,16,48
extrdi $t5,$t7,16,32
@ -656,7 +915,50 @@ $code.=<<___;
std $t5,`$FRAME+40`($sp)
std $t6,`$FRAME+48`($sp)
std $t7,`$FRAME+56`($sp)
___
$code.=<<___ if ($SIZE_T==4);
add $t0,$bp,$i
li $c1,0
lwz $t1,0($t0) ; bp[i,i+1]
lwz $t3,4($t0)
mullw $t4,$a0,$t1 ; ap[0]*bp[i]
lwz $t0,`$FRAME+$TRANSFER+8+4`($sp) ; tp[0]
mulhwu $t5,$a0,$t1
lwz $t2,`$FRAME+$TRANSFER+8`($sp) ; tp[0]
mullw $t6,$a1,$t1
mullw $t7,$a0,$t3
add $t5,$t5,$t6
add $t5,$t5,$t7
addc $t4,$t4,$t0 ; ap[0]*bp[i]+tp[0]
adde $t5,$t5,$t2
; transfer bp[i] to FPU as 4x16-bit values
extrwi $t0,$t1,16,16
extrwi $t1,$t1,16,0
extrwi $t2,$t3,16,16
extrwi $t3,$t3,16,0
std $t0,`$FRAME+0`($sp) ; yes, std in 32-bit build
std $t1,`$FRAME+8`($sp)
std $t2,`$FRAME+16`($sp)
std $t3,`$FRAME+24`($sp)
mullw $t0,$t4,$n0 ; mulld tp[0]*n0
mulhwu $t1,$t4,$n0
mullw $t2,$t5,$n0
mullw $t3,$t4,$n1
add $t1,$t1,$t2
add $t1,$t1,$t3
; transfer (ap[0]*bp[i]+tp[0])*n0 to FPU as 4x16-bit values
extrwi $t4,$t0,16,16
extrwi $t5,$t0,16,0
extrwi $t6,$t1,16,16
extrwi $t7,$t1,16,0
std $t4,`$FRAME+32`($sp) ; yes, std in 32-bit build
std $t5,`$FRAME+40`($sp)
std $t6,`$FRAME+48`($sp)
std $t7,`$FRAME+56`($sp)
___
$code.=<<___;
lfd $A0,8($nap_d) ; load a[j] in double format
lfd $A1,16($nap_d)
lfd $A2,24($nap_d) ; load a[j+1] in double format
@ -769,7 +1071,9 @@ Linner:
fmul $dotb,$A3,$bd
lfd $A2,24($nap_d) ; load a[j+1] in double format
lfd $A3,32($nap_d)
___
if ($SIZE_T==8 or $flavour =~ /osx/) {
$code.=<<___;
fmadd $T1a,$N1,$na,$T1a
fmadd $T1b,$N1,$nb,$T1b
ld $t0,`$FRAME+0`($sp)
@ -856,10 +1160,131 @@ $code.=<<___;
addze $carry,$carry
std $t3,-16($tp) ; tp[j-1]
std $t5,-8($tp) ; tp[j]
___
} else {
$code.=<<___;
fmadd $T1a,$N1,$na,$T1a
fmadd $T1b,$N1,$nb,$T1b
lwz $t1,`$FRAME+0`($sp)
lwz $t0,`$FRAME+4`($sp)
fmadd $T2a,$N2,$na,$T2a
fmadd $T2b,$N2,$nb,$T2b
lwz $t3,`$FRAME+8`($sp)
lwz $t2,`$FRAME+12`($sp)
fmadd $T3a,$N3,$na,$T3a
fmadd $T3b,$N3,$nb,$T3b
lwz $t5,`$FRAME+16`($sp)
lwz $t4,`$FRAME+20`($sp)
addc $t0,$t0,$carry
adde $t1,$t1,$c1
srwi $carry,$t0,16
fmadd $T0a,$N0,$na,$T0a
fmadd $T0b,$N0,$nb,$T0b
lwz $t7,`$FRAME+24`($sp)
lwz $t6,`$FRAME+28`($sp)
srwi $c1,$t1,16
insrwi $carry,$t1,16,0
fmadd $T1a,$N0,$nc,$T1a
fmadd $T1b,$N0,$nd,$T1b
addc $t2,$t2,$carry
adde $t3,$t3,$c1
srwi $carry,$t2,16
fmadd $T2a,$N1,$nc,$T2a
fmadd $T2b,$N1,$nd,$T2b
insrwi $t0,$t2,16,0 ; 0..31 bits
srwi $c1,$t3,16
insrwi $carry,$t3,16,0
fmadd $T3a,$N2,$nc,$T3a
fmadd $T3b,$N2,$nd,$T3b
lwz $t2,12($tp) ; tp[j]
lwz $t3,8($tp)
addc $t4,$t4,$carry
adde $t5,$t5,$c1
srwi $carry,$t4,16
fmadd $dota,$N3,$nc,$dota
fmadd $dotb,$N3,$nd,$dotb
srwi $c1,$t5,16
insrwi $carry,$t5,16,0
fctid $T0a,$T0a
addc $t6,$t6,$carry
adde $t7,$t7,$c1
srwi $carry,$t6,16
fctid $T0b,$T0b
insrwi $t4,$t6,16,0 ; 32..63 bits
srwi $c1,$t7,16
insrwi $carry,$t7,16,0
fctid $T1a,$T1a
addc $t0,$t0,$t2
adde $t4,$t4,$t3
lwz $t3,`$FRAME+32`($sp) ; permuted $t1
lwz $t2,`$FRAME+36`($sp) ; permuted $t0
fctid $T1b,$T1b
addze $carry,$carry
addze $c1,$c1
stw $t0,4($tp) ; tp[j-1]
stw $t4,0($tp)
fctid $T2a,$T2a
addc $t2,$t2,$carry
adde $t3,$t3,$c1
srwi $carry,$t2,16
lwz $t7,`$FRAME+40`($sp) ; permuted $t3
lwz $t6,`$FRAME+44`($sp) ; permuted $t2
fctid $T2b,$T2b
srwi $c1,$t3,16
insrwi $carry,$t3,16,0
lwz $t1,`$FRAME+48`($sp) ; permuted $t5
lwz $t0,`$FRAME+52`($sp) ; permuted $t4
fctid $T3a,$T3a
addc $t6,$t6,$carry
adde $t7,$t7,$c1
srwi $carry,$t6,16
lwz $t5,`$FRAME+56`($sp) ; permuted $t7
lwz $t4,`$FRAME+60`($sp) ; permuted $t6
fctid $T3b,$T3b
insrwi $t2,$t6,16,0 ; 64..95 bits
insrwi $carry,$t7,16,0
srwi $c1,$t7,16
lwz $t6,20($tp)
lwzu $t7,16($tp)
addc $t0,$t0,$carry
stfd $T0a,`$FRAME+0`($sp)
adde $t1,$t1,$c1
srwi $carry,$t0,16
stfd $T0b,`$FRAME+8`($sp)
insrwi $carry,$t1,16,0
srwi $c1,$t1,16
addc $t4,$t4,$carry
stfd $T1a,`$FRAME+16`($sp)
adde $t5,$t5,$c1
srwi $carry,$t4,16
insrwi $t0,$t4,16,0 ; 96..127 bits
stfd $T1b,`$FRAME+24`($sp)
insrwi $carry,$t5,16,0
srwi $c1,$t5,16
addc $t2,$t2,$t6
stfd $T2a,`$FRAME+32`($sp)
adde $t0,$t0,$t7
stfd $T2b,`$FRAME+40`($sp)
addze $carry,$carry
stfd $T3a,`$FRAME+48`($sp)
addze $c1,$c1
stfd $T3b,`$FRAME+56`($sp)
stw $t2,-4($tp) ; tp[j]
stw $t0,-8($tp)
___
}
$code.=<<___;
bdnz- Linner
fctid $dota,$dota
fctid $dotb,$dotb
___
if ($SIZE_T==8 or $flavour =~ /osx/) {
$code.=<<___;
ld $t0,`$FRAME+0`($sp)
ld $t1,`$FRAME+8`($sp)
ld $t2,`$FRAME+16`($sp)
@ -926,7 +1351,116 @@ $code.=<<___;
insrdi $t6,$t7,48,0
srdi $ovf,$t7,48
std $t6,0($tp) ; tp[num-1]
___
} else {
$code.=<<___;
lwz $t1,`$FRAME+0`($sp)
lwz $t0,`$FRAME+4`($sp)
lwz $t3,`$FRAME+8`($sp)
lwz $t2,`$FRAME+12`($sp)
lwz $t5,`$FRAME+16`($sp)
lwz $t4,`$FRAME+20`($sp)
lwz $t7,`$FRAME+24`($sp)
lwz $t6,`$FRAME+28`($sp)
stfd $dota,`$FRAME+64`($sp)
stfd $dotb,`$FRAME+72`($sp)
addc $t0,$t0,$carry
adde $t1,$t1,$c1
srwi $carry,$t0,16
insrwi $carry,$t1,16,0
srwi $c1,$t1,16
addc $t2,$t2,$carry
adde $t3,$t3,$c1
srwi $carry,$t2,16
insrwi $t0,$t2,16,0 ; 0..31 bits
lwz $t2,12($tp) ; tp[j]
insrwi $carry,$t3,16,0
srwi $c1,$t3,16
lwz $t3,8($tp)
addc $t4,$t4,$carry
adde $t5,$t5,$c1
srwi $carry,$t4,16
insrwi $carry,$t5,16,0
srwi $c1,$t5,16
addc $t6,$t6,$carry
adde $t7,$t7,$c1
srwi $carry,$t6,16
insrwi $t4,$t6,16,0 ; 32..63 bits
insrwi $carry,$t7,16,0
srwi $c1,$t7,16
addc $t0,$t0,$t2
adde $t4,$t4,$t3
addze $carry,$carry
addze $c1,$c1
stw $t0,4($tp) ; tp[j-1]
stw $t4,0($tp)
lwz $t3,`$FRAME+32`($sp) ; permuted $t1
lwz $t2,`$FRAME+36`($sp) ; permuted $t0
lwz $t7,`$FRAME+40`($sp) ; permuted $t3
lwz $t6,`$FRAME+44`($sp) ; permuted $t2
lwz $t1,`$FRAME+48`($sp) ; permuted $t5
lwz $t0,`$FRAME+52`($sp) ; permuted $t4
lwz $t5,`$FRAME+56`($sp) ; permuted $t7
lwz $t4,`$FRAME+60`($sp) ; permuted $t6
addc $t2,$t2,$carry
adde $t3,$t3,$c1
srwi $carry,$t2,16
insrwi $carry,$t3,16,0
srwi $c1,$t3,16
addc $t6,$t6,$carry
adde $t7,$t7,$c1
srwi $carry,$t6,16
insrwi $t2,$t6,16,0 ; 64..95 bits
lwz $t6,20($tp)
insrwi $carry,$t7,16,0
srwi $c1,$t7,16
lwzu $t7,16($tp)
addc $t0,$t0,$carry
adde $t1,$t1,$c1
srwi $carry,$t0,16
insrwi $carry,$t1,16,0
srwi $c1,$t1,16
addc $t4,$t4,$carry
adde $t5,$t5,$c1
srwi $carry,$t4,16
insrwi $t0,$t4,16,0 ; 96..127 bits
insrwi $carry,$t5,16,0
srwi $c1,$t5,16
addc $t2,$t2,$t6
adde $t0,$t0,$t7
lwz $t7,`$FRAME+64`($sp)
lwz $t6,`$FRAME+68`($sp)
addze $carry,$carry
addze $c1,$c1
lwz $t5,`$FRAME+72`($sp)
lwz $t4,`$FRAME+76`($sp)
addc $t6,$t6,$carry
adde $t7,$t7,$c1
stw $t2,-4($tp) ; tp[j]
stw $t0,-8($tp)
addc $t6,$t6,$ovf
addze $t7,$t7
srwi $carry,$t6,16
insrwi $carry,$t7,16,0
srwi $c1,$t7,16
addc $t4,$t4,$carry
adde $t5,$t5,$c1
insrwi $t6,$t4,16,0
srwi $t4,$t4,16
insrwi $t4,$t5,16,0
srwi $ovf,$t5,16
stw $t6,4($tp) ; tp[num-1]
stw $t4,0($tp)
___
}
$code.=<<___;
slwi $t7,$num,2
addi $i,$i,8
subf $nap_d,$t7,$nap_d ; rewind pointer
@ -994,14 +1528,14 @@ $code.=<<___ if ($SIZE_T==4);
mtctr $j
.align 4
Lsub: ld $t0,8($tp) ; load tp[j..j+3] in 64-bit word order
ldu $t2,16($tp)
Lsub: lwz $t0,12($tp) ; load tp[j..j+3] in 64-bit word order
lwz $t1,8($tp)
lwz $t2,20($tp)
lwzu $t3,16($tp)
lwz $t4,4($np) ; load np[j..j+3] in 32-bit word order
lwz $t5,8($np)
lwz $t6,12($np)
lwzu $t7,16($np)
extrdi $t1,$t0,32,0
extrdi $t3,$t2,32,0
subfe $t4,$t4,$t0 ; tp[j]-np[j]
stw $t0,4($ap) ; save tp[j..j+3] in 32-bit word order
subfe $t5,$t5,$t1 ; tp[j+1]-np[j+1]
@ -1052,6 +1586,9 @@ ___
$code.=<<___;
$POP $i,0($sp)
li r3,1 ; signal "handled"
$POP r19,`-12*8-13*$SIZE_T`($i)
$POP r20,`-12*8-12*$SIZE_T`($i)
$POP r21,`-12*8-11*$SIZE_T`($i)
$POP r22,`-12*8-10*$SIZE_T`($i)
$POP r23,`-12*8-9*$SIZE_T`($i)
$POP r24,`-12*8-8*$SIZE_T`($i)
@ -1077,7 +1614,7 @@ $code.=<<___;
mr $sp,$i
blr
.long 0
.byte 0,12,4,0,0x8c,10,6,0
.byte 0,12,4,0,0x8c,13,6,0
.long 0
.size .$fname,.-.$fname

2
crypto/ppccap.c
View File

@ -25,7 +25,7 @@ int bn_mul_mont(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp, const BN_U
if (sizeof(size_t)==4)
{
#if (defined(__APPLE__) && defined(__MACH__))
#if 1 || (defined(__APPLE__) && defined(__MACH__))
if (num>=8 && (num&3)==0 && (OPENSSL_ppccap_P&PPC_FPU64))
return bn_mul_mont_fpu64(rp,ap,bp,np,n0,num);
#else