2019-05-13 12:41:06 +02:00
|
|
|
/**********************************************************************
|
2020-03-02 06:09:08 +01:00
|
|
|
Copyright(c) 2020 Arm Corporation All rights reserved.
|
2019-05-13 12:41:06 +02:00
|
|
|
|
|
|
|
Redistribution and use in source and binary forms, with or without
|
|
|
|
modification, are permitted provided that the following conditions
|
|
|
|
are met:
|
|
|
|
* Redistributions of source code must retain the above copyright
|
|
|
|
notice, this list of conditions and the following disclaimer.
|
|
|
|
* Redistributions in binary form must reproduce the above copyright
|
|
|
|
notice, this list of conditions and the following disclaimer in
|
|
|
|
the documentation and/or other materials provided with the
|
|
|
|
distribution.
|
|
|
|
* Neither the name of Arm Corporation nor the names of its
|
|
|
|
contributors may be used to endorse or promote products derived
|
|
|
|
from this software without specific prior written permission.
|
|
|
|
|
|
|
|
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
|
|
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
|
|
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
|
|
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
|
|
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
|
|
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
|
|
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
|
|
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
|
|
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
|
|
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
|
|
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
**********************************************************************/
|
|
|
|
#ifndef __AARCH64_MULTIBINARY_H__
|
|
|
|
#define __AARCH64_MULTIBINARY_H__
|
|
|
|
#ifndef __aarch64__
|
|
|
|
#error "This file is for aarch64 only"
|
|
|
|
#endif
|
2020-11-21 17:51:37 +01:00
|
|
|
#include "aarch64_label.h"
|
2019-05-13 12:41:06 +02:00
|
|
|
#ifdef __ASSEMBLY__
|
|
|
|
/**
|
|
|
|
* # mbin_interface : the wrapper layer for isal-l api
|
|
|
|
*
|
|
|
|
* ## references:
|
|
|
|
* * https://sourceware.org/git/gitweb.cgi?p=glibc.git;a=blob;f=sysdeps/aarch64/dl-trampoline.S
|
|
|
|
* * http://infocenter.arm.com/help/topic/com.arm.doc.ihi0055b/IHI0055B_aapcs64.pdf
|
|
|
|
* * https://static.docs.arm.com/ihi0057/b/IHI0057B_aadwarf64.pdf?_ga=2.80574487.1870739014.1564969896-1634778941.1548729310
|
|
|
|
*
|
|
|
|
* ## Usage:
|
|
|
|
* 1. Define dispather function
|
|
|
|
* 2. name must be \name\()_dispatcher
|
|
|
|
* 3. Prototype should be *"void * \name\()_dispatcher"*
|
|
|
|
* 4. The dispather should return the right function pointer , revision and a string information .
|
|
|
|
**/
|
|
|
|
.macro mbin_interface name:req
|
2020-11-21 17:51:37 +01:00
|
|
|
.extern cdecl(\name\()_dispatcher)
|
|
|
|
.data
|
2019-05-13 12:41:06 +02:00
|
|
|
.balign 8
|
2020-11-21 17:51:37 +01:00
|
|
|
.global cdecl(\name\()_dispatcher_info)
|
|
|
|
#ifndef __APPLE__
|
2019-05-13 12:41:06 +02:00
|
|
|
.type \name\()_dispatcher_info,%object
|
2020-11-21 17:51:37 +01:00
|
|
|
#endif
|
|
|
|
cdecl(\name\()_dispatcher_info):
|
2019-05-13 12:41:06 +02:00
|
|
|
.quad \name\()_mbinit //func_entry
|
2020-11-21 17:51:37 +01:00
|
|
|
#ifndef __APPLE__
|
2019-05-13 12:41:06 +02:00
|
|
|
.size \name\()_dispatcher_info,. - \name\()_dispatcher_info
|
2020-11-21 17:51:37 +01:00
|
|
|
#endif
|
2019-05-13 12:41:06 +02:00
|
|
|
.balign 8
|
|
|
|
.text
|
|
|
|
\name\()_mbinit:
|
|
|
|
//save lp fp, sub sp
|
|
|
|
.cfi_startproc
|
|
|
|
stp x29, x30, [sp, -224]!
|
|
|
|
|
|
|
|
//add cfi directive to avoid GDB bt cmds error
|
|
|
|
//set cfi(Call Frame Information)
|
|
|
|
.cfi_def_cfa_offset 224
|
|
|
|
.cfi_offset 29, -224
|
|
|
|
.cfi_offset 30, -216
|
|
|
|
|
|
|
|
//save parameter/result/indirect result registers
|
|
|
|
stp x8, x9, [sp, 16]
|
|
|
|
.cfi_offset 8, -208
|
|
|
|
.cfi_offset 9, -200
|
|
|
|
stp x0, x1, [sp, 32]
|
|
|
|
.cfi_offset 0, -192
|
|
|
|
.cfi_offset 1, -184
|
|
|
|
stp x2, x3, [sp, 48]
|
|
|
|
.cfi_offset 2, -176
|
|
|
|
.cfi_offset 3, -168
|
|
|
|
stp x4, x5, [sp, 64]
|
|
|
|
.cfi_offset 4, -160
|
|
|
|
.cfi_offset 5, -152
|
|
|
|
stp x6, x7, [sp, 80]
|
|
|
|
.cfi_offset 6, -144
|
|
|
|
.cfi_offset 7, -136
|
|
|
|
stp q0, q1, [sp, 96]
|
|
|
|
.cfi_offset 64, -128
|
|
|
|
.cfi_offset 65, -112
|
|
|
|
stp q2, q3, [sp, 128]
|
|
|
|
.cfi_offset 66, -96
|
|
|
|
.cfi_offset 67, -80
|
|
|
|
stp q4, q5, [sp, 160]
|
|
|
|
.cfi_offset 68, -64
|
|
|
|
.cfi_offset 69, -48
|
|
|
|
stp q6, q7, [sp, 192]
|
|
|
|
.cfi_offset 70, -32
|
|
|
|
.cfi_offset 71, -16
|
|
|
|
|
|
|
|
/**
|
|
|
|
* The dispatcher functions have the following prototype:
|
|
|
|
* void * function_dispatcher(void)
|
|
|
|
* As the dispatcher is returning a struct, by the AAPCS,
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
2020-11-21 17:51:37 +01:00
|
|
|
bl cdecl(\name\()_dispatcher)
|
2019-05-13 12:41:06 +02:00
|
|
|
//restore temp/indirect result registers
|
|
|
|
ldp x8, x9, [sp, 16]
|
|
|
|
.cfi_restore 8
|
|
|
|
.cfi_restore 9
|
|
|
|
|
|
|
|
// save function entry
|
|
|
|
str x0, [x9]
|
|
|
|
|
|
|
|
//restore parameter/result registers
|
|
|
|
ldp x0, x1, [sp, 32]
|
|
|
|
.cfi_restore 0
|
|
|
|
.cfi_restore 1
|
|
|
|
ldp x2, x3, [sp, 48]
|
|
|
|
.cfi_restore 2
|
|
|
|
.cfi_restore 3
|
|
|
|
ldp x4, x5, [sp, 64]
|
|
|
|
.cfi_restore 4
|
|
|
|
.cfi_restore 5
|
|
|
|
ldp x6, x7, [sp, 80]
|
|
|
|
.cfi_restore 6
|
|
|
|
.cfi_restore 7
|
|
|
|
ldp q0, q1, [sp, 96]
|
|
|
|
.cfi_restore 64
|
|
|
|
.cfi_restore 65
|
|
|
|
ldp q2, q3, [sp, 128]
|
|
|
|
.cfi_restore 66
|
|
|
|
.cfi_restore 67
|
|
|
|
ldp q4, q5, [sp, 160]
|
|
|
|
.cfi_restore 68
|
|
|
|
.cfi_restore 69
|
|
|
|
ldp q6, q7, [sp, 192]
|
|
|
|
.cfi_restore 70
|
|
|
|
.cfi_restore 71
|
|
|
|
//save lp fp and sp
|
|
|
|
ldp x29, x30, [sp], 224
|
|
|
|
//restore cfi setting
|
|
|
|
.cfi_restore 30
|
|
|
|
.cfi_restore 29
|
|
|
|
.cfi_def_cfa_offset 0
|
|
|
|
.cfi_endproc
|
|
|
|
|
2020-11-21 17:51:37 +01:00
|
|
|
.global cdecl(\name)
|
|
|
|
#ifndef __APPLE__
|
2019-05-13 12:41:06 +02:00
|
|
|
.type \name,%function
|
2020-11-21 17:51:37 +01:00
|
|
|
#endif
|
2019-05-13 12:41:06 +02:00
|
|
|
.align 2
|
2020-11-21 17:51:37 +01:00
|
|
|
cdecl(\name\()):
|
|
|
|
#ifndef __APPLE__
|
2019-05-13 12:41:06 +02:00
|
|
|
adrp x9, :got:\name\()_dispatcher_info
|
|
|
|
ldr x9, [x9, #:got_lo12:\name\()_dispatcher_info]
|
2020-11-21 17:51:37 +01:00
|
|
|
#else
|
|
|
|
adrp x9, cdecl(\name\()_dispatcher_info)@GOTPAGE
|
|
|
|
ldr x9, [x9, #cdecl(\name\()_dispatcher_info)@GOTPAGEOFF]
|
|
|
|
#endif
|
2019-05-13 12:41:06 +02:00
|
|
|
ldr x10,[x9]
|
|
|
|
br x10
|
2020-11-21 17:51:37 +01:00
|
|
|
#ifndef __APPLE__
|
2019-05-13 12:41:06 +02:00
|
|
|
.size \name,. - \name
|
2020-11-21 17:51:37 +01:00
|
|
|
#endif
|
2019-05-13 12:41:06 +02:00
|
|
|
.endm
|
|
|
|
|
2019-05-13 12:41:06 +02:00
|
|
|
/**
|
|
|
|
* mbin_interface_base is used for the interfaces which have only
|
|
|
|
* noarch implementation
|
|
|
|
*/
|
|
|
|
.macro mbin_interface_base name:req, base:req
|
|
|
|
.extern \base
|
2020-11-21 17:51:37 +01:00
|
|
|
.data
|
2019-05-13 12:41:06 +02:00
|
|
|
.balign 8
|
2020-11-21 17:51:37 +01:00
|
|
|
.global cdecl(\name\()_dispatcher_info)
|
|
|
|
#ifndef __APPLE__
|
2019-05-13 12:41:06 +02:00
|
|
|
.type \name\()_dispatcher_info,%object
|
2020-11-21 17:51:37 +01:00
|
|
|
#endif
|
|
|
|
cdecl(\name\()_dispatcher_info):
|
2019-05-13 12:41:06 +02:00
|
|
|
.quad \base //func_entry
|
2020-11-21 17:51:37 +01:00
|
|
|
#ifndef __APPLE__
|
2019-05-13 12:41:06 +02:00
|
|
|
.size \name\()_dispatcher_info,. - \name\()_dispatcher_info
|
2020-11-21 17:51:37 +01:00
|
|
|
#endif
|
2019-05-13 12:41:06 +02:00
|
|
|
.balign 8
|
|
|
|
.text
|
2020-11-21 17:51:37 +01:00
|
|
|
.global cdecl(\name)
|
|
|
|
#ifndef __APPLE__
|
2019-05-13 12:41:06 +02:00
|
|
|
.type \name,%function
|
2020-11-21 17:51:37 +01:00
|
|
|
#endif
|
2019-05-13 12:41:06 +02:00
|
|
|
.align 2
|
2020-11-21 17:51:37 +01:00
|
|
|
cdecl(\name\()):
|
|
|
|
#ifndef __APPLE__
|
|
|
|
adrp x9, :got:cdecl(_\name\()_dispatcher_info)
|
|
|
|
ldr x9, [x9, #:got_lo12:cdecl(_\name\()_dispatcher_info)]
|
|
|
|
#else
|
|
|
|
adrp x9, cdecl(_\name\()_dispatcher_info)@GOTPAGE
|
|
|
|
ldr x9, [x9, #cdecl(_\name\()_dispatcher_info)@GOTPAGEOFF]
|
|
|
|
#endif
|
2019-05-13 12:41:06 +02:00
|
|
|
ldr x10,[x9]
|
|
|
|
br x10
|
2020-11-21 17:51:37 +01:00
|
|
|
#ifndef __APPLE__
|
2019-05-13 12:41:06 +02:00
|
|
|
.size \name,. - \name
|
2020-11-21 17:51:37 +01:00
|
|
|
#endif
|
2019-05-13 12:41:06 +02:00
|
|
|
.endm
|
2019-05-13 12:41:06 +02:00
|
|
|
|
|
|
|
#else /* __ASSEMBLY__ */
|
2020-11-21 17:51:37 +01:00
|
|
|
#include <stdint.h>
|
|
|
|
#if defined(__linux__)
|
2019-05-13 12:41:06 +02:00
|
|
|
#include <sys/auxv.h>
|
2022-10-31 07:30:27 +01:00
|
|
|
#include <asm/hwcap.h>
|
2020-11-21 17:51:37 +01:00
|
|
|
#elif defined(__APPLE__)
|
2022-10-31 07:30:27 +01:00
|
|
|
#define SYSCTL_PMULL_KEY "hw.optional.arm.FEAT_PMULL" // from macOS 12 FEAT_* sysctl infos are available
|
|
|
|
#define SYSCTL_CRC32_KEY "hw.optional.armv8_crc32"
|
|
|
|
#define SYSCTL_SVE_KEY "hw.optional.arm.FEAT_SVE" // this one is just a guess and need to check macOS update
|
2020-11-21 17:51:37 +01:00
|
|
|
#include <sys/sysctl.h>
|
|
|
|
#include <stddef.h>
|
|
|
|
static inline int sysctlEnabled(const char* name){
|
|
|
|
int enabled;
|
|
|
|
size_t size = sizeof(enabled);
|
|
|
|
int status = sysctlbyname(name, &enabled, &size, NULL, 0);
|
|
|
|
return status ? 0 : enabled;
|
|
|
|
}
|
|
|
|
#endif
|
2019-05-13 12:41:06 +02:00
|
|
|
|
|
|
|
|
|
|
|
#define DEFINE_INTERFACE_DISPATCHER(name) \
|
|
|
|
void * name##_dispatcher(void)
|
|
|
|
|
|
|
|
#define PROVIDER_BASIC(name) \
|
|
|
|
PROVIDER_INFO(name##_base)
|
|
|
|
|
|
|
|
#define DO_DIGNOSTIC(x) _Pragma GCC diagnostic ignored "-W"#x
|
|
|
|
#define DO_PRAGMA(x) _Pragma (#x)
|
|
|
|
#define DIGNOSTIC_IGNORE(x) DO_PRAGMA(GCC diagnostic ignored #x)
|
|
|
|
#define DIGNOSTIC_PUSH() DO_PRAGMA(GCC diagnostic push)
|
|
|
|
#define DIGNOSTIC_POP() DO_PRAGMA(GCC diagnostic pop)
|
|
|
|
|
|
|
|
|
|
|
|
#define PROVIDER_INFO(_func_entry) \
|
|
|
|
({ DIGNOSTIC_PUSH() \
|
|
|
|
DIGNOSTIC_IGNORE(-Wnested-externs) \
|
2019-10-14 10:46:26 +02:00
|
|
|
extern void _func_entry(void); \
|
2019-05-13 12:41:06 +02:00
|
|
|
DIGNOSTIC_POP() \
|
|
|
|
_func_entry; \
|
|
|
|
})
|
|
|
|
|
2020-03-02 06:09:08 +01:00
|
|
|
/**
|
|
|
|
* Micro-Architector definitions
|
|
|
|
* Reference: https://developer.arm.com/docs/ddi0595/f/aarch64-system-registers/midr_el1
|
|
|
|
*/
|
|
|
|
|
|
|
|
#define CPU_IMPLEMENTER_RESERVE 0x00
|
|
|
|
#define CPU_IMPLEMENTER_ARM 0x41
|
|
|
|
|
|
|
|
|
|
|
|
#define CPU_PART_CORTEX_A57 0xD07
|
|
|
|
#define CPU_PART_CORTEX_A72 0xD08
|
|
|
|
#define CPU_PART_NEOVERSE_N1 0xD0C
|
|
|
|
|
|
|
|
#define MICRO_ARCH_ID(imp,part) \
|
|
|
|
(((CPU_IMPLEMENTER_##imp&0xff)<<24)|((CPU_PART_##part&0xfff)<<4))
|
|
|
|
|
|
|
|
#ifndef HWCAP_CPUID
|
|
|
|
#define HWCAP_CPUID (1<<11)
|
|
|
|
#endif
|
|
|
|
|
|
|
|
/**
|
|
|
|
* @brief get_micro_arch_id
|
|
|
|
*
|
|
|
|
* read micro-architector register instruction if possible.This function
|
|
|
|
* provides microarchitecture information and make microarchitecture optimization
|
|
|
|
* possible.
|
|
|
|
*
|
|
|
|
* Read system registers(MRS) is forbidden in userspace. If executed, it
|
|
|
|
* will raise illegal instruction error. Kernel provides a solution for
|
|
|
|
* this issue. The solution depends on HWCAP_CPUID flags. Reference(1)
|
|
|
|
* describes how to use it. It provides a "illegal insstruction" handler
|
|
|
|
* in kernel space, the handler will execute MRS and return the correct
|
|
|
|
* value to userspace.
|
|
|
|
*
|
|
|
|
* To avoid too many kernel trap, this function MUST be only called in
|
|
|
|
* dispatcher. And HWCAP must be match,That will make sure there are no
|
|
|
|
* illegal instruction errors. HWCAP_CPUID should be available to get the
|
|
|
|
* best performance.
|
|
|
|
*
|
|
|
|
* NOTICE:
|
|
|
|
* - HWCAP_CPUID should be available. Otherwise it returns reserve value
|
|
|
|
* - It MUST be called inside dispather.
|
|
|
|
* - It MUST meet the HWCAP requirements
|
|
|
|
*
|
|
|
|
* Example:
|
|
|
|
* DEFINE_INTERFACE_DISPATCHER(crc32_iscsi)
|
|
|
|
* {
|
|
|
|
* unsigned long auxval = getauxval(AT_HWCAP);
|
|
|
|
* // MUST do the judgement is MUST.
|
|
|
|
* if ((HWCAP_CRC32 | HWCAP_PMULL) == (auxval & (HWCAP_CRC32 | HWCAP_PMULL))) {
|
|
|
|
* switch (get_micro_arch_id()) {
|
|
|
|
* case MICRO_ARCH_ID(ARM, CORTEX_A57):
|
|
|
|
* return PROVIDER_INFO(crc32_pmull_crc_for_a57);
|
|
|
|
* case MICRO_ARCH_ID(ARM, CORTEX_A72):
|
|
|
|
* return PROVIDER_INFO(crc32_pmull_crc_for_a72);
|
|
|
|
* case MICRO_ARCH_ID(ARM, NEOVERSE_N1):
|
|
|
|
* return PROVIDER_INFO(crc32_pmull_crc_for_n1);
|
|
|
|
* case default:
|
|
|
|
* return PROVIDER_INFO(crc32_pmull_crc_for_others);
|
|
|
|
* }
|
|
|
|
* }
|
|
|
|
* return PROVIDER_BASIC(crc32_iscsi);
|
|
|
|
* }
|
|
|
|
* KNOWN ISSUE:
|
|
|
|
* On a heterogeneous system (big.LITTLE), it will work but the performance
|
|
|
|
* might not be the best one as expected.
|
|
|
|
*
|
|
|
|
* If this function is called on the big core, it will return the function
|
|
|
|
* optimized for the big core.
|
|
|
|
*
|
|
|
|
* If execution is then scheduled to the little core. It will still work (1),
|
|
|
|
* but the function won't be optimized for the little core, thus the performance
|
|
|
|
* won't be as expected.
|
|
|
|
*
|
|
|
|
* References:
|
|
|
|
* - [CPU Feature detection](https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/arm64/cpu-feature-registers.rst?h=v5.5)
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
static inline uint32_t get_micro_arch_id(void)
|
|
|
|
{
|
2024-04-25 12:10:51 +02:00
|
|
|
uint64_t id=CPU_IMPLEMENTER_RESERVE;
|
2020-11-21 17:51:37 +01:00
|
|
|
#ifndef __APPLE__
|
2020-03-02 06:09:08 +01:00
|
|
|
if ((getauxval(AT_HWCAP) & HWCAP_CPUID)) {
|
|
|
|
/** Here will trap into kernel space */
|
|
|
|
asm("mrs %0, MIDR_EL1 " : "=r" (id));
|
|
|
|
}
|
2020-11-21 17:51:37 +01:00
|
|
|
#endif
|
2020-03-02 06:09:08 +01:00
|
|
|
return id&0xff00fff0;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
2019-05-13 12:41:06 +02:00
|
|
|
#endif /* __ASSEMBLY__ */
|
|
|
|
#endif
|