From bc34d874279d782211b634f49c4f657a3f46d587 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Wed, 29 Nov 2023 16:19:34 +0000 Subject: [PATCH] erasure_code: update GF_MUL_XOR macro to support VEX encoding Signed-off-by: Marcel Cornu --- .../gf_2vect_dot_prod_avx512_gfni.asm | 2 +- erasure_code/gf_2vect_mad_avx512_gfni.asm | 2 +- .../gf_3vect_dot_prod_avx512_gfni.asm | 2 +- erasure_code/gf_3vect_mad_avx512_gfni.asm | 2 +- .../gf_4vect_dot_prod_avx512_gfni.asm | 2 +- erasure_code/gf_4vect_mad_avx512_gfni.asm | 2 +- .../gf_5vect_dot_prod_avx512_gfni.asm | 2 +- erasure_code/gf_5vect_mad_avx512_gfni.asm | 2 +- .../gf_6vect_dot_prod_avx512_gfni.asm | 2 +- erasure_code/gf_6vect_mad_avx512_gfni.asm | 2 +- erasure_code/gf_vect_dot_prod_avx512_gfni.asm | 2 +- erasure_code/gf_vect_gfni.inc | 47 ++++++++++--------- erasure_code/gf_vect_mad_avx512_gfni.asm | 2 +- 13 files changed, 38 insertions(+), 33 deletions(-) diff --git a/erasure_code/gf_2vect_dot_prod_avx512_gfni.asm b/erasure_code/gf_2vect_dot_prod_avx512_gfni.asm index 12ae8cb..e405ee3 100644 --- a/erasure_code/gf_2vect_dot_prod_avx512_gfni.asm +++ b/erasure_code/gf_2vect_dot_prod_avx512_gfni.asm @@ -152,7 +152,7 @@ section .text vbroadcastf32x2 xgft2, [tmp + vec] add tmp, 8 - GF_MUL_XOR x0, xgft1, xgft1, xp1, xgft2, xgft2, xp2 + GF_MUL_XOR EVEX, x0, xgft1, xgft1, xp1, xgft2, xgft2, xp2 cmp vec_i, vec jl %%next_vect diff --git a/erasure_code/gf_2vect_mad_avx512_gfni.asm b/erasure_code/gf_2vect_mad_avx512_gfni.asm index a656f4e..a50385f 100644 --- a/erasure_code/gf_2vect_mad_avx512_gfni.asm +++ b/erasure_code/gf_2vect_mad_avx512_gfni.asm @@ -133,7 +133,7 @@ section .text XLDR xd2, [dest2 + pos] ;Get next dest vector %endif - GF_MUL_XOR x0, xgft1, xret1, xd1, xgft2, xret2, xd2 + GF_MUL_XOR EVEX, x0, xgft1, xret1, xd1, xgft2, xret2, xd2 %if %0 == 1 vmovdqu8 [dest1 + pos]{%%KMASK}, xd1 diff --git a/erasure_code/gf_3vect_dot_prod_avx512_gfni.asm b/erasure_code/gf_3vect_dot_prod_avx512_gfni.asm index b507119..ec9afa8 100644 --- a/erasure_code/gf_3vect_dot_prod_avx512_gfni.asm +++ b/erasure_code/gf_3vect_dot_prod_avx512_gfni.asm @@ -165,7 +165,7 @@ section .text vbroadcastf32x2 xgft3, [tmp + vec*2] add tmp, 8 - GF_MUL_XOR x0, xgft1, xgft1, xp1, xgft2, xgft2, xp2, xgft3, xgft3, xp3 + GF_MUL_XOR EVEX, x0, xgft1, xgft1, xp1, xgft2, xgft2, xp2, xgft3, xgft3, xp3 cmp vec_i, vec jl %%next_vect diff --git a/erasure_code/gf_3vect_mad_avx512_gfni.asm b/erasure_code/gf_3vect_mad_avx512_gfni.asm index b609ed6..3794c0b 100644 --- a/erasure_code/gf_3vect_mad_avx512_gfni.asm +++ b/erasure_code/gf_3vect_mad_avx512_gfni.asm @@ -144,7 +144,7 @@ section .text XLDR xd3, [dest3 + pos] ;Get next dest vector %endif - GF_MUL_XOR x0, xgft1, xret1, xd1, xgft2, xret2, xd2, xgft3, xret3, xd3 + GF_MUL_XOR EVEX, x0, xgft1, xret1, xd1, xgft2, xret2, xd2, xgft3, xret3, xd3 %if %0 == 1 vmovdqu8 [dest1 + pos]{%%KMASK}, xd1 diff --git a/erasure_code/gf_4vect_dot_prod_avx512_gfni.asm b/erasure_code/gf_4vect_dot_prod_avx512_gfni.asm index a698209..9472f97 100644 --- a/erasure_code/gf_4vect_dot_prod_avx512_gfni.asm +++ b/erasure_code/gf_4vect_dot_prod_avx512_gfni.asm @@ -187,7 +187,7 @@ section .text vbroadcastf32x2 xgft4, [tmp + vskip3] add tmp, 8 - GF_MUL_XOR x0, xgft1, xgft1, xp1, xgft2, xgft2, xp2, xgft3, xgft3, xp3, \ + GF_MUL_XOR EVEX, x0, xgft1, xgft1, xp1, xgft2, xgft2, xp2, xgft3, xgft3, xp3, \ xgft4, xgft4, xp4 cmp vec_i, vec diff --git a/erasure_code/gf_4vect_mad_avx512_gfni.asm b/erasure_code/gf_4vect_mad_avx512_gfni.asm index 011b39c..547ee90 100644 --- a/erasure_code/gf_4vect_mad_avx512_gfni.asm +++ b/erasure_code/gf_4vect_mad_avx512_gfni.asm @@ -157,7 +157,7 @@ section .text XLDR xd4, [dest4 + pos] ;Get next dest vector %endif - GF_MUL_XOR x0, xgft1, xret1, xd1, xgft2, xret2, xd2, xgft3, xret3, xd3, \ + GF_MUL_XOR EVEX, x0, xgft1, xret1, xd1, xgft2, xret2, xd2, xgft3, xret3, xd3, \ xgft4, xret4, xd4 %if %0 == 1 diff --git a/erasure_code/gf_5vect_dot_prod_avx512_gfni.asm b/erasure_code/gf_5vect_dot_prod_avx512_gfni.asm index b7e4925..1e9b308 100644 --- a/erasure_code/gf_5vect_dot_prod_avx512_gfni.asm +++ b/erasure_code/gf_5vect_dot_prod_avx512_gfni.asm @@ -202,7 +202,7 @@ section .text vbroadcastf32x2 xgft5, [tmp + vec*4] add tmp, 8 - GF_MUL_XOR x0, xgft1, xgft1, xp1, xgft2, xgft2, xp2, xgft3, xgft3, xp3, \ + GF_MUL_XOR EVEX, x0, xgft1, xgft1, xp1, xgft2, xgft2, xp2, xgft3, xgft3, xp3, \ xgft4, xgft4, xp4, xgft5, xgft5, xp5 cmp vec_i, vec diff --git a/erasure_code/gf_5vect_mad_avx512_gfni.asm b/erasure_code/gf_5vect_mad_avx512_gfni.asm index bcea47a..018e8ac 100644 --- a/erasure_code/gf_5vect_mad_avx512_gfni.asm +++ b/erasure_code/gf_5vect_mad_avx512_gfni.asm @@ -171,7 +171,7 @@ section .text XLDR xd5, [dest5 + pos] ;Get next dest vector %endif - GF_MUL_XOR x0, xgft1, xret1, xd1, xgft2, xret2, xd2, xgft3, xret3, xd3, \ + GF_MUL_XOR EVEX, x0, xgft1, xret1, xd1, xgft2, xret2, xd2, xgft3, xret3, xd3, \ xgft4, xret4, xd4, xgft5, xret5, xd5 %if %0 == 1 diff --git a/erasure_code/gf_6vect_dot_prod_avx512_gfni.asm b/erasure_code/gf_6vect_dot_prod_avx512_gfni.asm index c361548..c4856bf 100644 --- a/erasure_code/gf_6vect_dot_prod_avx512_gfni.asm +++ b/erasure_code/gf_6vect_dot_prod_avx512_gfni.asm @@ -217,7 +217,7 @@ section .text vbroadcastf32x2 xgft6, [tmp + vskip5] add tmp, 8 - GF_MUL_XOR x0, xgft1, xgft1, xp1, xgft2, xgft2, xp2, xgft3, xgft3, xp3, \ + GF_MUL_XOR EVEX, x0, xgft1, xgft1, xp1, xgft2, xgft2, xp2, xgft3, xgft3, xp3, \ xgft4, xgft4, xp4, xgft5, xgft5, xp5, xgft6, xgft6, xp6 cmp vec_i, vec diff --git a/erasure_code/gf_6vect_mad_avx512_gfni.asm b/erasure_code/gf_6vect_mad_avx512_gfni.asm index 3641183..5539e23 100644 --- a/erasure_code/gf_6vect_mad_avx512_gfni.asm +++ b/erasure_code/gf_6vect_mad_avx512_gfni.asm @@ -185,7 +185,7 @@ section .text XLDR xd6, [dest6 + pos] ;Get next dest vector %endif - GF_MUL_XOR x0, xgft1, xret1, xd1, xgft2, xret2, xd2, xgft3, xret3, xd3, \ + GF_MUL_XOR EVEX, x0, xgft1, xret1, xd1, xgft2, xret2, xd2, xgft3, xret3, xd3, \ xgft4, xret4, xd4, xgft5, xret5, xd5, xgft6, xret6, xd6 %if %0 == 1 diff --git a/erasure_code/gf_vect_dot_prod_avx512_gfni.asm b/erasure_code/gf_vect_dot_prod_avx512_gfni.asm index 6259fd2..9b0e31d 100644 --- a/erasure_code/gf_vect_dot_prod_avx512_gfni.asm +++ b/erasure_code/gf_vect_dot_prod_avx512_gfni.asm @@ -138,7 +138,7 @@ section .text vbroadcastf32x2 xgft1, [tmp] add tmp, 8 - GF_MUL_XOR x0, xgft1, xgft1, xp1 + GF_MUL_XOR EVEX, x0, xgft1, xgft1, xp1 cmp vec_i, vec jl %%next_vect diff --git a/erasure_code/gf_vect_gfni.inc b/erasure_code/gf_vect_gfni.inc index 96bf627..83d362b 100644 --- a/erasure_code/gf_vect_gfni.inc +++ b/erasure_code/gf_vect_gfni.inc @@ -31,28 +31,29 @@ ; Multiply 1 source register to up to 6 different GF table registers ; and XOR the results to partial registers ; -%macro GF_MUL_XOR 4-19 -%define %%SRC %1 -%define %%GFTABLE1 %2 -%define %%TMP1 %3 -%define %%PARTIAL1 %4 -%define %%GFTABLE2 %5 -%define %%TMP2 %6 -%define %%PARTIAL2 %7 -%define %%GFTABLE3 %8 -%define %%TMP3 %9 -%define %%PARTIAL3 %10 -%define %%GFTABLE4 %11 -%define %%TMP4 %12 -%define %%PARTIAL4 %13 -%define %%GFTABLE5 %14 -%define %%TMP5 %15 -%define %%PARTIAL5 %16 -%define %%GFTABLE6 %17 -%define %%TMP6 %18 -%define %%PARTIAL6 %19 +%macro GF_MUL_XOR 5-20 +%define %%ENCODING %1 +%define %%SRC %2 +%define %%GFTABLE1 %3 +%define %%TMP1 %4 +%define %%PARTIAL1 %5 +%define %%GFTABLE2 %6 +%define %%TMP2 %7 +%define %%PARTIAL2 %8 +%define %%GFTABLE3 %9 +%define %%TMP3 %10 +%define %%PARTIAL3 %11 +%define %%GFTABLE4 %12 +%define %%TMP4 %13 +%define %%PARTIAL4 %14 +%define %%GFTABLE5 %15 +%define %%TMP5 %16 +%define %%PARTIAL5 %17 +%define %%GFTABLE6 %18 +%define %%TMP6 %19 +%define %%PARTIAL6 %20 -%define %%N_BLOCKS ((%0 - 1) / 3) +%define %%N_BLOCKS ((%0 - 2) / 3) %assign %%I 1 %rep %%N_BLOCKS @@ -61,7 +62,11 @@ %endrep %assign %%I 1 %rep %%N_BLOCKS +%ifidn %%ENCODING, VEX + vpxor %%PARTIAL %+ %%I, %%TMP %+ %%I +%else vpxorq %%PARTIAL %+ %%I, %%TMP %+ %%I +%endif %assign %%I (%%I + 1) %endrep %endmacro diff --git a/erasure_code/gf_vect_mad_avx512_gfni.asm b/erasure_code/gf_vect_mad_avx512_gfni.asm index 1c96fcd..12a104a 100644 --- a/erasure_code/gf_vect_mad_avx512_gfni.asm +++ b/erasure_code/gf_vect_mad_avx512_gfni.asm @@ -126,7 +126,7 @@ section .text XLDR xd, [dest + pos] ;Get next dest vector %endif - GF_MUL_XOR x0, xgft1, xret1, xd + GF_MUL_XOR EVEX, x0, xgft1, xret1, xd %if %0 == 1 vmovdqu8 [dest + pos]{%%KMASK}, xd