From a7766a91b6d6b86d5f00b532d06ff81d33fa9abb Mon Sep 17 00:00:00 2001 From: sunyuechi Date: Tue, 6 May 2025 12:28:59 +0800 Subject: [PATCH] mem: R-V V mem_zero_detect banana_f3: rvv: mem_zero_detect_perf_warm: runtime = 3062584 usecs, bandwidth 33784 MB in 3.0626 sec = 11031.32 MB/s c: mem_zero_detect_perf_warm: runtime = 3000354 usecs, bandwidth 1594 MB in 3.0004 sec = 531.34 MB/s Signed-off-by: sunyuechi --- mem/Makefile.am | 3 +- mem/riscv64/Makefile.am | 33 +++++++++++++ mem/riscv64/mem_multibinary_riscv64.S | 36 ++++++++++++++ .../mem_multibinary_riscv64_dispatcher.c | 46 +++++++++++++++++ mem/riscv64/mem_zero_detect_rvv.S | 49 +++++++++++++++++++ 5 files changed, 166 insertions(+), 1 deletion(-) create mode 100644 mem/riscv64/Makefile.am create mode 100644 mem/riscv64/mem_multibinary_riscv64.S create mode 100644 mem/riscv64/mem_multibinary_riscv64_dispatcher.c create mode 100644 mem/riscv64/mem_zero_detect_rvv.S diff --git a/mem/Makefile.am b/mem/Makefile.am index 6c537de..e529a09 100644 --- a/mem/Makefile.am +++ b/mem/Makefile.am @@ -29,11 +29,12 @@ include mem/aarch64/Makefile.am +include mem/riscv64/Makefile.am + lsrc += mem/mem_zero_detect_base.c lsrc_base_aliases += mem/mem_zero_detect_base_aliases.c lsrc_ppc64le += mem/mem_zero_detect_base_aliases.c -lsrc_riscv64 += mem/mem_zero_detect_base_aliases.c lsrc_x86_64 += mem/mem_zero_detect_avx512.asm \ mem/mem_zero_detect_avx2.asm \ diff --git a/mem/riscv64/Makefile.am b/mem/riscv64/Makefile.am new file mode 100644 index 0000000..5f801a1 --- /dev/null +++ b/mem/riscv64/Makefile.am @@ -0,0 +1,33 @@ +######################################################################## +# Copyright (c) 2025 Institute of Software Chinese Academy of Sciences (ISCAS). +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of ISCAS nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +######################################################################## + +lsrc_riscv64 += \ + mem/riscv64/mem_multibinary_riscv64_dispatcher.c \ + mem/riscv64/mem_multibinary_riscv64.S \ + mem/riscv64/mem_zero_detect_rvv.S diff --git a/mem/riscv64/mem_multibinary_riscv64.S b/mem/riscv64/mem_multibinary_riscv64.S new file mode 100644 index 0000000..359a304 --- /dev/null +++ b/mem/riscv64/mem_multibinary_riscv64.S @@ -0,0 +1,36 @@ +/********************************************************************** + Copyright (c) 2025 Institute of Software Chinese Academy of Sciences (ISCAS). + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of ISCAS nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ + +#include "riscv64_multibinary.h" + +#if HAVE_RVV + mbin_interface isal_zero_detect +#else + mbin_interface_base isal_zero_detect mem_zero_detect_base +#endif diff --git a/mem/riscv64/mem_multibinary_riscv64_dispatcher.c b/mem/riscv64/mem_multibinary_riscv64_dispatcher.c new file mode 100644 index 0000000..7ba628c --- /dev/null +++ b/mem/riscv64/mem_multibinary_riscv64_dispatcher.c @@ -0,0 +1,46 @@ +/********************************************************************** + Copyright (c) 2025 Institute of Software Chinese Academy of Sciences (ISCAS). + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of ISCAS nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ +#include "riscv64_multibinary.h" +#include + +extern int +mem_zero_detect_rvv(void *buf, size_t n); +extern int +mem_zero_detect_base(void *buf, size_t n); + +DEFINE_INTERFACE_DISPATCHER(isal_zero_detect) +{ +#if HAVE_RVV + const unsigned long hwcap = getauxval(AT_HWCAP); + if (hwcap & HWCAP_RV('V')) + return mem_zero_detect_rvv; + else +#endif + return mem_zero_detect_base; +} diff --git a/mem/riscv64/mem_zero_detect_rvv.S b/mem/riscv64/mem_zero_detect_rvv.S new file mode 100644 index 0000000..772b94e --- /dev/null +++ b/mem/riscv64/mem_zero_detect_rvv.S @@ -0,0 +1,49 @@ +/********************************************************************** + Copyright (c) 2025 Institute of Software Chinese Academy of Sciences (ISCAS). + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of ISCAS nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ +#if HAVE_RVV +.option arch, +v +.global mem_zero_detect_rvv +.type mem_zero_detect_rvv, %function +mem_zero_detect_rvv: +1: + vsetvli t0, a1, e8, m8, ta, ma + vle8.v v8, (a0) + vmsne.vi v0, v8, 0 + vfirst.m t1, v0 + bgez t1, 2f + sub a1, a1, t0 + add a0, a0, t0 + bnez a1, 1b + li a0, 0 + ret +2: + li a0, -1 + ret + +#endif