diff --git a/crc/aarch64/crc_aarch64_dispatcher.c b/crc/aarch64/crc_aarch64_dispatcher.c index 2e07760d..7ff2e2a7 100644 --- a/crc/aarch64/crc_aarch64_dispatcher.c +++ b/crc/aarch64/crc_aarch64_dispatcher.c @@ -27,18 +27,61 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. **********************************************************************/ #include +#include "crc.h" +#include "crc64.h" + +extern uint16_t +crc16_t10dif_pmull(uint16_t, uint8_t *, uint64_t); + +extern uint16_t +crc16_t10dif_copy_pmull(uint16_t, uint8_t *, uint8_t *, uint64_t); + +extern uint32_t +crc32_ieee_norm_pmull(uint32_t, uint8_t *, uint64_t); + +extern unsigned int +crc32_iscsi_crc_ext(unsigned char *, int, unsigned int); +extern unsigned int +crc32_iscsi_3crc_fold(unsigned char *, int, unsigned int); +extern unsigned int +crc32_iscsi_refl_pmull(unsigned char *, int, unsigned int); + +extern uint32_t +crc32_gzip_refl_crc_ext(uint32_t, uint8_t *, uint64_t); +extern uint32_t +crc32_gzip_refl_3crc_fold(uint32_t, uint8_t *, uint64_t); +extern uint32_t +crc32_gzip_refl_pmull(uint32_t, uint8_t *, uint64_t); + +extern uint64_t +crc64_ecma_refl_pmull(uint64_t, const unsigned char *, uint64_t); + +extern uint64_t +crc64_ecma_norm_pmull(uint64_t, const unsigned char *, uint64_t); + +extern uint64_t +crc64_iso_refl_pmull(uint64_t, const unsigned char *, uint64_t); + +extern uint64_t +crc64_iso_norm_pmull(uint64_t, const unsigned char *, uint64_t); + +extern uint64_t +crc64_jones_refl_pmull(uint64_t, const unsigned char *, uint64_t); + +extern uint64_t +crc64_jones_norm_pmull(uint64_t, const unsigned char *, uint64_t); DEFINE_INTERFACE_DISPATCHER(crc16_t10dif) { #if defined(__linux__) unsigned long auxval = getauxval(AT_HWCAP); if (auxval & HWCAP_PMULL) - return PROVIDER_INFO(crc16_t10dif_pmull); + return crc16_t10dif_pmull; #elif defined(__APPLE__) if (sysctlEnabled(SYSCTL_PMULL_KEY)) - return PROVIDER_INFO(crc16_t10dif_pmull); + return crc16_t10dif_pmull; #endif - return PROVIDER_BASIC(crc16_t10dif); + return crc16_t10dif_base; } DEFINE_INTERFACE_DISPATCHER(crc16_t10dif_copy) @@ -46,12 +89,12 @@ DEFINE_INTERFACE_DISPATCHER(crc16_t10dif_copy) #if defined(__linux__) unsigned long auxval = getauxval(AT_HWCAP); if (auxval & HWCAP_PMULL) - return PROVIDER_INFO(crc16_t10dif_copy_pmull); + return crc16_t10dif_copy_pmull; #elif defined(__APPLE__) if (sysctlEnabled(SYSCTL_PMULL_KEY)) - return PROVIDER_INFO(crc16_t10dif_copy_pmull); + return crc16_t10dif_copy_pmull; #endif - return PROVIDER_BASIC(crc16_t10dif_copy); + return crc16_t10dif_copy_base; } DEFINE_INTERFACE_DISPATCHER(crc32_ieee) @@ -59,13 +102,13 @@ DEFINE_INTERFACE_DISPATCHER(crc32_ieee) #if defined(__linux__) unsigned long auxval = getauxval(AT_HWCAP); if (auxval & HWCAP_PMULL) { - return PROVIDER_INFO(crc32_ieee_norm_pmull); + return crc32_ieee_norm_pmull; } #elif defined(__APPLE__) if (sysctlEnabled(SYSCTL_PMULL_KEY)) - return PROVIDER_INFO(crc32_ieee_norm_pmull); + return crc32_ieee_norm_pmull; #endif - return PROVIDER_BASIC(crc32_ieee); + return crc32_ieee_base; } DEFINE_INTERFACE_DISPATCHER(crc32_iscsi) @@ -77,23 +120,23 @@ DEFINE_INTERFACE_DISPATCHER(crc32_iscsi) case MICRO_ARCH_ID(ARM, NEOVERSE_N1): case MICRO_ARCH_ID(ARM, CORTEX_A57): case MICRO_ARCH_ID(ARM, CORTEX_A72): - return PROVIDER_INFO(crc32_iscsi_crc_ext); + return crc32_iscsi_crc_ext; } } if ((HWCAP_CRC32 | HWCAP_PMULL) == (auxval & (HWCAP_CRC32 | HWCAP_PMULL))) { - return PROVIDER_INFO(crc32_iscsi_3crc_fold); + return crc32_iscsi_3crc_fold; } if (auxval & HWCAP_PMULL) { - return PROVIDER_INFO(crc32_iscsi_refl_pmull); + return crc32_iscsi_refl_pmull; } #elif defined(__APPLE__) if (sysctlEnabled(SYSCTL_CRC32_KEY)) - return PROVIDER_INFO(crc32_iscsi_3crc_fold); + return crc32_iscsi_3crc_fold; if (sysctlEnabled(SYSCTL_PMULL_KEY)) - return PROVIDER_INFO(crc32_iscsi_refl_pmull); + return crc32_iscsi_refl_pmull; #endif - return PROVIDER_BASIC(crc32_iscsi); + return crc32_iscsi_base; } DEFINE_INTERFACE_DISPATCHER(crc32_gzip_refl) @@ -106,22 +149,22 @@ DEFINE_INTERFACE_DISPATCHER(crc32_gzip_refl) case MICRO_ARCH_ID(ARM, NEOVERSE_N1): case MICRO_ARCH_ID(ARM, CORTEX_A57): case MICRO_ARCH_ID(ARM, CORTEX_A72): - return PROVIDER_INFO(crc32_gzip_refl_crc_ext); + return crc32_gzip_refl_crc_ext; } } if ((HWCAP_CRC32 | HWCAP_PMULL) == (auxval & (HWCAP_CRC32 | HWCAP_PMULL))) { - return PROVIDER_INFO(crc32_gzip_refl_3crc_fold); + return crc32_gzip_refl_3crc_fold; } if (auxval & HWCAP_PMULL) - return PROVIDER_INFO(crc32_gzip_refl_pmull); + return crc32_gzip_refl_pmull; #elif defined(__APPLE__) if (sysctlEnabled(SYSCTL_CRC32_KEY)) - return PROVIDER_INFO(crc32_gzip_refl_3crc_fold); + return crc32_gzip_refl_3crc_fold; if (sysctlEnabled(SYSCTL_PMULL_KEY)) - return PROVIDER_INFO(crc32_gzip_refl_pmull); + return crc32_gzip_refl_pmull; #endif - return PROVIDER_BASIC(crc32_gzip_refl); + return crc32_gzip_refl_base; } DEFINE_INTERFACE_DISPATCHER(crc64_ecma_refl) @@ -130,12 +173,12 @@ DEFINE_INTERFACE_DISPATCHER(crc64_ecma_refl) unsigned long auxval = getauxval(AT_HWCAP); if (auxval & HWCAP_PMULL) - return PROVIDER_INFO(crc64_ecma_refl_pmull); + return crc64_ecma_refl_pmull; #elif defined(__APPLE__) if (sysctlEnabled(SYSCTL_PMULL_KEY)) - return PROVIDER_INFO(crc64_ecma_refl_pmull); + return crc64_ecma_refl_pmull; #endif - return PROVIDER_BASIC(crc64_ecma_refl); + return crc64_ecma_refl_base; } DEFINE_INTERFACE_DISPATCHER(crc64_ecma_norm) @@ -143,12 +186,12 @@ DEFINE_INTERFACE_DISPATCHER(crc64_ecma_norm) #if defined(__linux__) unsigned long auxval = getauxval(AT_HWCAP); if (auxval & HWCAP_PMULL) - return PROVIDER_INFO(crc64_ecma_norm_pmull); + return crc64_ecma_norm_pmull; #elif defined(__APPLE__) if (sysctlEnabled(SYSCTL_PMULL_KEY)) - return PROVIDER_INFO(crc64_ecma_norm_pmull); + return crc64_ecma_norm_pmull; #endif - return PROVIDER_BASIC(crc64_ecma_norm); + return crc64_ecma_norm_base; } DEFINE_INTERFACE_DISPATCHER(crc64_iso_refl) @@ -156,12 +199,12 @@ DEFINE_INTERFACE_DISPATCHER(crc64_iso_refl) #if defined(__linux__) unsigned long auxval = getauxval(AT_HWCAP); if (auxval & HWCAP_PMULL) - return PROVIDER_INFO(crc64_iso_refl_pmull); + return crc64_iso_refl_pmull; #elif defined(__APPLE__) if (sysctlEnabled(SYSCTL_PMULL_KEY)) - return PROVIDER_INFO(crc64_iso_refl_pmull); + return crc64_iso_refl_pmull; #endif - return PROVIDER_BASIC(crc64_iso_refl); + return crc64_iso_refl_base; } DEFINE_INTERFACE_DISPATCHER(crc64_iso_norm) @@ -169,12 +212,12 @@ DEFINE_INTERFACE_DISPATCHER(crc64_iso_norm) #if defined(__linux__) unsigned long auxval = getauxval(AT_HWCAP); if (auxval & HWCAP_PMULL) - return PROVIDER_INFO(crc64_iso_norm_pmull); + return crc64_iso_norm_pmull; #elif defined(__APPLE__) if (sysctlEnabled(SYSCTL_PMULL_KEY)) - return PROVIDER_INFO(crc64_iso_norm_pmull); + return crc64_iso_norm_pmull; #endif - return PROVIDER_BASIC(crc64_iso_norm); + return crc64_iso_norm_base; } DEFINE_INTERFACE_DISPATCHER(crc64_jones_refl) @@ -182,12 +225,12 @@ DEFINE_INTERFACE_DISPATCHER(crc64_jones_refl) #if defined(__linux__) unsigned long auxval = getauxval(AT_HWCAP); if (auxval & HWCAP_PMULL) - return PROVIDER_INFO(crc64_jones_refl_pmull); + return crc64_jones_refl_pmull; #elif defined(__APPLE__) if (sysctlEnabled(SYSCTL_PMULL_KEY)) - return PROVIDER_INFO(crc64_jones_refl_pmull); + return crc64_jones_refl_pmull; #endif - return PROVIDER_BASIC(crc64_jones_refl); + return crc64_jones_refl_base; } DEFINE_INTERFACE_DISPATCHER(crc64_jones_norm) @@ -195,10 +238,10 @@ DEFINE_INTERFACE_DISPATCHER(crc64_jones_norm) #if defined(__linux__) unsigned long auxval = getauxval(AT_HWCAP); if (auxval & HWCAP_PMULL) - return PROVIDER_INFO(crc64_jones_norm_pmull); + return crc64_jones_norm_pmull; #elif defined(__APPLE__) if (sysctlEnabled(SYSCTL_PMULL_KEY)) - return PROVIDER_INFO(crc64_jones_norm_pmull); + return crc64_jones_norm_pmull; #endif - return PROVIDER_BASIC(crc64_jones_norm); + return crc64_jones_norm_base; } diff --git a/erasure_code/aarch64/ec_aarch64_dispatcher.c b/erasure_code/aarch64/ec_aarch64_dispatcher.c index 7f90dcb3..f8e819bf 100644 --- a/erasure_code/aarch64/ec_aarch64_dispatcher.c +++ b/erasure_code/aarch64/ec_aarch64_dispatcher.c @@ -27,6 +27,41 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. **********************************************************************/ #include +#include "erasure_code.h" +#include "gf_vect_mul.h" + +extern void +gf_vect_dot_prod_sve(int, int, unsigned char *, unsigned char **, + unsigned char *); +extern void +gf_vect_dot_prod_neon(int, int, unsigned char *, unsigned char **, + unsigned char *); + +extern void +gf_vect_mad_sve(int, int, int, unsigned char *, unsigned char *, + unsigned char *); +extern void +gf_vect_mad_neon(int, int, int, unsigned char *, unsigned char *, + unsigned char *); + +extern void +ec_encode_data_sve(int, int, int, unsigned char *, unsigned char **, + unsigned char **coding); +extern void +ec_encode_data_neon(int, int, int, unsigned char *, unsigned char **, + unsigned char **); + +extern void +ec_encode_data_update_sve(int, int, int, int, unsigned char *, + unsigned char *, unsigned char **); +extern void +ec_encode_data_update_neon(int, int, int, int, unsigned char *, + unsigned char *, unsigned char **); + +extern int +gf_vect_mul_sve(int, unsigned char *, unsigned char *, unsigned char *); +extern int +gf_vect_mul_neon(int, unsigned char *, unsigned char *, unsigned char *); DEFINE_INTERFACE_DISPATCHER(gf_vect_dot_prod) { @@ -34,15 +69,15 @@ DEFINE_INTERFACE_DISPATCHER(gf_vect_dot_prod) unsigned long auxval = getauxval(AT_HWCAP); if (auxval & HWCAP_SVE) - return PROVIDER_INFO(gf_vect_dot_prod_sve); + return gf_vect_dot_prod_sve; if (auxval & HWCAP_ASIMD) - return PROVIDER_INFO(gf_vect_dot_prod_neon); + return gf_vect_dot_prod_neon; #elif defined(__APPLE__) if (sysctlEnabled(SYSCTL_SVE_KEY)) - return PROVIDER_INFO(gf_vect_dot_prod_sve); - return PROVIDER_INFO(gf_vect_dot_prod_neon); + return gf_vect_dot_prod_sve; + return gf_vect_dot_prod_neon; #endif - return PROVIDER_BASIC(gf_vect_dot_prod); + return gf_vect_dot_prod_base; } DEFINE_INTERFACE_DISPATCHER(gf_vect_mad) @@ -51,15 +86,15 @@ DEFINE_INTERFACE_DISPATCHER(gf_vect_mad) unsigned long auxval = getauxval(AT_HWCAP); if (auxval & HWCAP_SVE) - return PROVIDER_INFO(gf_vect_mad_sve); + return gf_vect_mad_sve; if (auxval & HWCAP_ASIMD) - return PROVIDER_INFO(gf_vect_mad_neon); + return gf_vect_mad_neon; #elif defined(__APPLE__) if (sysctlEnabled(SYSCTL_SVE_KEY)) - return PROVIDER_INFO(gf_vect_mad_sve); - return PROVIDER_INFO(gf_vect_mad_neon); + return gf_vect_mad_sve; + return gf_vect_mad_neon; #endif - return PROVIDER_BASIC(gf_vect_mad); + return gf_vect_mad_base; } DEFINE_INTERFACE_DISPATCHER(ec_encode_data) @@ -68,15 +103,15 @@ DEFINE_INTERFACE_DISPATCHER(ec_encode_data) unsigned long auxval = getauxval(AT_HWCAP); if (auxval & HWCAP_SVE) - return PROVIDER_INFO(ec_encode_data_sve); + return ec_encode_data_sve; if (auxval & HWCAP_ASIMD) - return PROVIDER_INFO(ec_encode_data_neon); + return ec_encode_data_neon; #elif defined(__APPLE__) if (sysctlEnabled(SYSCTL_SVE_KEY)) - return PROVIDER_INFO(ec_encode_data_sve); - return PROVIDER_INFO(ec_encode_data_neon); + return ec_encode_data_sve; + return ec_encode_data_neon; #endif - return PROVIDER_BASIC(ec_encode_data); + return ec_encode_data_base; } DEFINE_INTERFACE_DISPATCHER(ec_encode_data_update) @@ -85,15 +120,15 @@ DEFINE_INTERFACE_DISPATCHER(ec_encode_data_update) unsigned long auxval = getauxval(AT_HWCAP); if (auxval & HWCAP_SVE) - return PROVIDER_INFO(ec_encode_data_update_sve); + return ec_encode_data_update_sve; if (auxval & HWCAP_ASIMD) - return PROVIDER_INFO(ec_encode_data_update_neon); + return ec_encode_data_update_neon; #elif defined(__APPLE__) if (sysctlEnabled(SYSCTL_SVE_KEY)) - return PROVIDER_INFO(ec_encode_data_update_sve); - return PROVIDER_INFO(ec_encode_data_update_neon); + return ec_encode_data_update_sve; + return ec_encode_data_update_neon; #endif - return PROVIDER_BASIC(ec_encode_data_update); + return ec_encode_data_update_base; } DEFINE_INTERFACE_DISPATCHER(gf_vect_mul) @@ -102,15 +137,15 @@ DEFINE_INTERFACE_DISPATCHER(gf_vect_mul) unsigned long auxval = getauxval(AT_HWCAP); if (auxval & HWCAP_SVE) - return PROVIDER_INFO(gf_vect_mul_sve); + return gf_vect_mul_sve; if (auxval & HWCAP_ASIMD) - return PROVIDER_INFO(gf_vect_mul_neon); + return gf_vect_mul_neon; #elif defined(__APPLE__) if (sysctlEnabled(SYSCTL_SVE_KEY)) - return PROVIDER_INFO(gf_vect_mul_sve); - return PROVIDER_INFO(gf_vect_mul_neon); + return gf_vect_mul_sve; + return gf_vect_mul_neon; #endif - return PROVIDER_BASIC(gf_vect_mul); + return gf_vect_mul_base; } -DEFINE_INTERFACE_DISPATCHER(ec_init_tables) { return PROVIDER_BASIC(ec_init_tables); } +DEFINE_INTERFACE_DISPATCHER(ec_init_tables) { return ec_init_tables_base; } diff --git a/igzip/aarch64/igzip_multibinary_aarch64_dispatcher.c b/igzip/aarch64/igzip_multibinary_aarch64_dispatcher.c index cadec0c8..1fd4d0e5 100644 --- a/igzip/aarch64/igzip_multibinary_aarch64_dispatcher.c +++ b/igzip/aarch64/igzip_multibinary_aarch64_dispatcher.c @@ -27,17 +27,81 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. **********************************************************************/ #include +#include "igzip_lib.h" +#include "igzip/encode_df.h" + +extern uint32_t +adler32_neon(uint32_t, uint8_t *, uint64_t); +extern uint32_t +adler32_base(uint32_t, uint8_t *, uint64_t); + +extern void +isal_deflate_body_aarch64(struct isal_zstream *); +extern void +isal_deflate_body_base(struct isal_zstream *); + +extern void +isal_deflate_finish_aarch64(struct isal_zstream *); +extern void +isal_deflate_finish_base(struct isal_zstream *); + +extern void +isal_deflate_icf_body_hash_hist_aarch64(struct isal_zstream *); +extern void +isal_deflate_icf_body_hash_hist_base(struct isal_zstream *); + +extern void +isal_deflate_icf_finish_hash_hist_aarch64(struct isal_zstream *); +extern void +isal_deflate_icf_finish_hash_hist_base(struct isal_zstream *); + +extern void +icf_body_hash1_fillgreedy_lazy(struct isal_zstream *); + +extern void +isal_deflate_icf_finish_hash_map_base(struct isal_zstream *); + +extern void +set_long_icf_fg_aarch64(uint8_t *, uint64_t, uint64_t, struct deflate_icf *); + +extern struct deflate_icf * +encode_deflate_icf_aarch64(struct deflate_icf *, struct deflate_icf *, struct BitBuf2 *, + struct hufftables_icf *); + +extern void +isal_update_histogram_aarch64(uint8_t *, int, struct isal_huff_histogram *); +extern void +isal_update_histogram_base(uint8_t *, int, struct isal_huff_histogram *); + +extern uint64_t +gen_icf_map_h1_aarch64(struct isal_zstream *, struct deflate_icf *, + uint64_t input_size); +extern uint64_t +gen_icf_map_h1_base(struct isal_zstream *, struct deflate_icf *, + uint64_t input_size); + +extern void +isal_deflate_hash_aarch64(uint16_t *, uint32_t, uint32_t, + uint8_t *, uint32_t); +extern void +isal_deflate_hash_base(uint16_t *, uint32_t, uint32_t, + uint8_t *, uint32_t); + +extern int +decode_huffman_code_block_stateless_aarch64(struct inflate_state *, uint8_t *); +extern int +decode_huffman_code_block_stateless_base(struct inflate_state *, uint8_t *); DEFINE_INTERFACE_DISPATCHER(isal_adler32) { #if defined(__linux__) unsigned long auxval = getauxval(AT_HWCAP); if (auxval & HWCAP_ASIMD) - return PROVIDER_INFO(adler32_neon); + return adler32_neon; #elif defined(__APPLE__) - return PROVIDER_INFO(adler32_neon); + return adler32_neon; #endif - return PROVIDER_BASIC(adler32); + return adler32_base; } DEFINE_INTERFACE_DISPATCHER(isal_deflate_body) @@ -46,12 +110,12 @@ DEFINE_INTERFACE_DISPATCHER(isal_deflate_body) unsigned long auxval = getauxval(AT_HWCAP); if (auxval & HWCAP_CRC32) - return PROVIDER_INFO(isal_deflate_body_aarch64); + return isal_deflate_body_aarch64; #elif defined(__APPLE__) if (sysctlEnabled(SYSCTL_CRC32_KEY)) - return PROVIDER_INFO(isal_deflate_body_aarch64); + return isal_deflate_body_aarch64; #endif - return PROVIDER_BASIC(isal_deflate_body); + return isal_deflate_body_base; } DEFINE_INTERFACE_DISPATCHER(isal_deflate_finish) @@ -59,12 +123,12 @@ DEFINE_INTERFACE_DISPATCHER(isal_deflate_finish) #if defined(__linux__) unsigned long auxval = getauxval(AT_HWCAP); if (auxval & HWCAP_CRC32) - return PROVIDER_INFO(isal_deflate_finish_aarch64); + return isal_deflate_finish_aarch64; #elif defined(__APPLE__) if (sysctlEnabled(SYSCTL_CRC32_KEY)) - return PROVIDER_INFO(isal_deflate_finish_aarch64); + return isal_deflate_finish_aarch64; #endif - return PROVIDER_BASIC(isal_deflate_finish); + return isal_deflate_finish_base; } DEFINE_INTERFACE_DISPATCHER(isal_deflate_icf_body_lvl1) @@ -72,12 +136,12 @@ DEFINE_INTERFACE_DISPATCHER(isal_deflate_icf_body_lvl1) #if defined(__linux__) unsigned long auxval = getauxval(AT_HWCAP); if (auxval & HWCAP_CRC32) - return PROVIDER_INFO(isal_deflate_icf_body_hash_hist_aarch64); + return isal_deflate_icf_body_hash_hist_aarch64; #elif defined(__APPLE__) if (sysctlEnabled(SYSCTL_CRC32_KEY)) - return PROVIDER_INFO(isal_deflate_icf_body_hash_hist_aarch64); + return isal_deflate_icf_body_hash_hist_aarch64; #endif - return PROVIDER_BASIC(isal_deflate_icf_body_hash_hist); + return isal_deflate_icf_body_hash_hist_base; } DEFINE_INTERFACE_DISPATCHER(isal_deflate_icf_finish_lvl1) @@ -85,12 +149,12 @@ DEFINE_INTERFACE_DISPATCHER(isal_deflate_icf_finish_lvl1) #if defined(__linux__) unsigned long auxval = getauxval(AT_HWCAP); if (auxval & HWCAP_CRC32) - return PROVIDER_INFO(isal_deflate_icf_finish_hash_hist_aarch64); + return isal_deflate_icf_finish_hash_hist_aarch64; #elif defined(__APPLE__) if (sysctlEnabled(SYSCTL_CRC32_KEY)) - return PROVIDER_INFO(isal_deflate_icf_finish_hash_hist_aarch64); + return isal_deflate_icf_finish_hash_hist_aarch64; #endif - return PROVIDER_BASIC(isal_deflate_icf_finish_hash_hist); + return isal_deflate_icf_finish_hash_hist_base; } DEFINE_INTERFACE_DISPATCHER(isal_deflate_icf_body_lvl2) @@ -98,12 +162,12 @@ DEFINE_INTERFACE_DISPATCHER(isal_deflate_icf_body_lvl2) #if defined(__linux__) unsigned long auxval = getauxval(AT_HWCAP); if (auxval & HWCAP_CRC32) - return PROVIDER_INFO(isal_deflate_icf_body_hash_hist_aarch64); + return isal_deflate_icf_body_hash_hist_aarch64; #elif defined(__APPLE__) if (sysctlEnabled(SYSCTL_CRC32_KEY)) - return PROVIDER_INFO(isal_deflate_icf_body_hash_hist_aarch64); + return isal_deflate_icf_body_hash_hist_aarch64; #endif - return PROVIDER_BASIC(isal_deflate_icf_body_hash_hist); + return isal_deflate_icf_body_hash_hist_base; } DEFINE_INTERFACE_DISPATCHER(isal_deflate_icf_finish_lvl2) @@ -111,12 +175,12 @@ DEFINE_INTERFACE_DISPATCHER(isal_deflate_icf_finish_lvl2) #if defined(__linux__) unsigned long auxval = getauxval(AT_HWCAP); if (auxval & HWCAP_CRC32) - return PROVIDER_INFO(isal_deflate_icf_finish_hash_hist_aarch64); + return isal_deflate_icf_finish_hash_hist_aarch64; #elif defined(__APPLE__) if (sysctlEnabled(SYSCTL_CRC32_KEY)) - return PROVIDER_INFO(isal_deflate_icf_finish_hash_hist_aarch64); + return isal_deflate_icf_finish_hash_hist_aarch64; #endif - return PROVIDER_BASIC(isal_deflate_icf_finish_hash_hist); + return isal_deflate_icf_finish_hash_hist_base; } DEFINE_INTERFACE_DISPATCHER(isal_deflate_icf_body_lvl3) @@ -124,12 +188,12 @@ DEFINE_INTERFACE_DISPATCHER(isal_deflate_icf_body_lvl3) #if defined(__linux__) unsigned long auxval = getauxval(AT_HWCAP); if (auxval & HWCAP_CRC32) - return PROVIDER_INFO(icf_body_hash1_fillgreedy_lazy); + return icf_body_hash1_fillgreedy_lazy; #elif defined(__APPLE__) if (sysctlEnabled(SYSCTL_CRC32_KEY)) - return PROVIDER_INFO(icf_body_hash1_fillgreedy_lazy); + return icf_body_hash1_fillgreedy_lazy; #endif - return PROVIDER_INFO(icf_body_hash1_fillgreedy_lazy); + return icf_body_hash1_fillgreedy_lazy; } DEFINE_INTERFACE_DISPATCHER(isal_deflate_icf_finish_lvl3) @@ -137,19 +201,19 @@ DEFINE_INTERFACE_DISPATCHER(isal_deflate_icf_finish_lvl3) #if defined(__linux__) unsigned long auxval = getauxval(AT_HWCAP); if (auxval & HWCAP_CRC32) - return PROVIDER_INFO(isal_deflate_icf_finish_hash_map_base); + return isal_deflate_icf_finish_hash_map_base; #elif defined(__APPLE__) if (sysctlEnabled(SYSCTL_CRC32_KEY)) - return PROVIDER_INFO(isal_deflate_icf_finish_hash_map_base); + return isal_deflate_icf_finish_hash_map_base; #endif - return PROVIDER_BASIC(isal_deflate_icf_finish_hash_map); + return isal_deflate_icf_finish_hash_map_base; } -DEFINE_INTERFACE_DISPATCHER(set_long_icf_fg) { return PROVIDER_INFO(set_long_icf_fg_aarch64); } +DEFINE_INTERFACE_DISPATCHER(set_long_icf_fg) { return set_long_icf_fg_aarch64; } DEFINE_INTERFACE_DISPATCHER(encode_deflate_icf) { - return PROVIDER_INFO(encode_deflate_icf_aarch64); + return encode_deflate_icf_aarch64; } DEFINE_INTERFACE_DISPATCHER(isal_update_histogram) @@ -157,12 +221,12 @@ DEFINE_INTERFACE_DISPATCHER(isal_update_histogram) #if defined(__linux__) unsigned long auxval = getauxval(AT_HWCAP); if (auxval & HWCAP_CRC32) - return PROVIDER_INFO(isal_update_histogram_aarch64); + return isal_update_histogram_aarch64; #elif defined(__APPLE__) if (sysctlEnabled(SYSCTL_CRC32_KEY)) - return PROVIDER_INFO(isal_update_histogram_aarch64); + return isal_update_histogram_aarch64; #endif - return PROVIDER_BASIC(isal_update_histogram); + return isal_update_histogram_base; } DEFINE_INTERFACE_DISPATCHER(gen_icf_map_lh1) @@ -170,13 +234,13 @@ DEFINE_INTERFACE_DISPATCHER(gen_icf_map_lh1) #if defined(__linux__) unsigned long auxval = getauxval(AT_HWCAP); if (auxval & HWCAP_CRC32) { - return PROVIDER_INFO(gen_icf_map_h1_aarch64); + return gen_icf_map_h1_aarch64; } #elif defined(__APPLE__) if (sysctlEnabled(SYSCTL_CRC32_KEY)) - return PROVIDER_INFO(gen_icf_map_h1_aarch64); + return gen_icf_map_h1_aarch64; #endif - return PROVIDER_BASIC(gen_icf_map_h1); + return gen_icf_map_h1_base; } DEFINE_INTERFACE_DISPATCHER(isal_deflate_hash_lvl0) @@ -184,12 +248,12 @@ DEFINE_INTERFACE_DISPATCHER(isal_deflate_hash_lvl0) #if defined(__linux__) unsigned long auxval = getauxval(AT_HWCAP); if (auxval & HWCAP_CRC32) - return PROVIDER_INFO(isal_deflate_hash_aarch64); + return isal_deflate_hash_aarch64; #elif defined(__APPLE__) if (sysctlEnabled(SYSCTL_CRC32_KEY)) - return PROVIDER_INFO(isal_deflate_hash_aarch64); + return isal_deflate_hash_aarch64; #endif - return PROVIDER_BASIC(isal_deflate_hash); + return isal_deflate_hash_base; } DEFINE_INTERFACE_DISPATCHER(isal_deflate_hash_lvl1) @@ -197,12 +261,12 @@ DEFINE_INTERFACE_DISPATCHER(isal_deflate_hash_lvl1) #if defined(__linux__) unsigned long auxval = getauxval(AT_HWCAP); if (auxval & HWCAP_CRC32) - return PROVIDER_INFO(isal_deflate_hash_aarch64); + return isal_deflate_hash_aarch64; #elif defined(__APPLE__) if (sysctlEnabled(SYSCTL_CRC32_KEY)) - return PROVIDER_INFO(isal_deflate_hash_aarch64); + return isal_deflate_hash_aarch64; #endif - return PROVIDER_BASIC(isal_deflate_hash); + return isal_deflate_hash_base; } DEFINE_INTERFACE_DISPATCHER(isal_deflate_hash_lvl2) @@ -210,12 +274,12 @@ DEFINE_INTERFACE_DISPATCHER(isal_deflate_hash_lvl2) #if defined(__linux__) unsigned long auxval = getauxval(AT_HWCAP); if (auxval & HWCAP_CRC32) - return PROVIDER_INFO(isal_deflate_hash_aarch64); + return isal_deflate_hash_aarch64; #elif defined(__APPLE__) if (sysctlEnabled(SYSCTL_CRC32_KEY)) - return PROVIDER_INFO(isal_deflate_hash_aarch64); + return isal_deflate_hash_aarch64; #endif - return PROVIDER_BASIC(isal_deflate_hash); + return isal_deflate_hash_base; } DEFINE_INTERFACE_DISPATCHER(isal_deflate_hash_lvl3) @@ -223,12 +287,12 @@ DEFINE_INTERFACE_DISPATCHER(isal_deflate_hash_lvl3) #if defined(__linux__) unsigned long auxval = getauxval(AT_HWCAP); if (auxval & HWCAP_CRC32) - return PROVIDER_INFO(isal_deflate_hash_aarch64); + return isal_deflate_hash_aarch64; #elif defined(__APPLE__) if (sysctlEnabled(SYSCTL_CRC32_KEY)) - return PROVIDER_INFO(isal_deflate_hash_aarch64); + return isal_deflate_hash_aarch64; #endif - return PROVIDER_BASIC(isal_deflate_hash); + return isal_deflate_hash_base; } DEFINE_INTERFACE_DISPATCHER(decode_huffman_code_block_stateless) @@ -236,10 +300,10 @@ DEFINE_INTERFACE_DISPATCHER(decode_huffman_code_block_stateless) #if defined(__linux__) unsigned long auxval = getauxval(AT_HWCAP); if (auxval & HWCAP_CRC32) - return PROVIDER_INFO(decode_huffman_code_block_stateless_aarch64); + return decode_huffman_code_block_stateless_aarch64; #elif defined(__APPLE__) if (sysctlEnabled(SYSCTL_CRC32_KEY)) - return PROVIDER_INFO(decode_huffman_code_block_stateless_aarch64); + return decode_huffman_code_block_stateless_aarch64; #endif - return PROVIDER_BASIC(decode_huffman_code_block_stateless); + return decode_huffman_code_block_stateless_base; } diff --git a/include/aarch64_multibinary.h b/include/aarch64_multibinary.h index cd5d9996..1b6e5c0c 100644 --- a/include/aarch64_multibinary.h +++ b/include/aarch64_multibinary.h @@ -233,24 +233,12 @@ static inline int sysctlEnabled(const char* name){ #define DEFINE_INTERFACE_DISPATCHER(name) \ void * name##_dispatcher(void) -#define PROVIDER_BASIC(name) \ - PROVIDER_INFO(name##_base) - #define DO_DIGNOSTIC(x) _Pragma GCC diagnostic ignored "-W"#x #define DO_PRAGMA(x) _Pragma (#x) #define DIGNOSTIC_IGNORE(x) DO_PRAGMA(GCC diagnostic ignored #x) #define DIGNOSTIC_PUSH() DO_PRAGMA(GCC diagnostic push) #define DIGNOSTIC_POP() DO_PRAGMA(GCC diagnostic pop) - -#define PROVIDER_INFO(_func_entry) \ - ({ DIGNOSTIC_PUSH() \ - DIGNOSTIC_IGNORE(-Wnested-externs) \ - extern void _func_entry(void); \ - DIGNOSTIC_POP() \ - _func_entry; \ - }) - /** * Micro-Architector definitions * Reference: https://developer.arm.com/docs/ddi0595/f/aarch64-system-registers/midr_el1 @@ -303,16 +291,16 @@ static inline int sysctlEnabled(const char* name){ * if ((HWCAP_CRC32 | HWCAP_PMULL) == (auxval & (HWCAP_CRC32 | HWCAP_PMULL))) { * switch (get_micro_arch_id()) { * case MICRO_ARCH_ID(ARM, CORTEX_A57): - * return PROVIDER_INFO(crc32_pmull_crc_for_a57); + * return crc32_pmull_crc_for_a57; * case MICRO_ARCH_ID(ARM, CORTEX_A72): - * return PROVIDER_INFO(crc32_pmull_crc_for_a72); + * return crc32_pmull_crc_for_a72; * case MICRO_ARCH_ID(ARM, NEOVERSE_N1): - * return PROVIDER_INFO(crc32_pmull_crc_for_n1); + * return crc32_pmull_crc_for_n1; * case default: - * return PROVIDER_INFO(crc32_pmull_crc_for_others); + * return crc32_pmull_crc_for_others; * } * } - * return PROVIDER_BASIC(crc32_iscsi); + * return crc32_iscsi_base; * } * KNOWN ISSUE: * On a heterogeneous system (big.LITTLE), it will work but the performance diff --git a/mem/aarch64/mem_aarch64_dispatcher.c b/mem/aarch64/mem_aarch64_dispatcher.c index 6cdcdc86..c559d7b1 100644 --- a/mem/aarch64/mem_aarch64_dispatcher.c +++ b/mem/aarch64/mem_aarch64_dispatcher.c @@ -27,15 +27,21 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. **********************************************************************/ #include +#include + +extern int +mem_zero_detect_neon(void *, size_t); +extern int +mem_zero_detect_base(void *, size_t); DEFINE_INTERFACE_DISPATCHER(isal_zero_detect) { #if defined(__linux__) unsigned long auxval = getauxval(AT_HWCAP); if (auxval & HWCAP_ASIMD) - return PROVIDER_INFO(mem_zero_detect_neon); + return mem_zero_detect_neon; #elif defined(__APPLE__) - return PROVIDER_INFO(mem_zero_detect_neon); + return mem_zero_detect_neon; #endif - return PROVIDER_BASIC(mem_zero_detect); + return mem_zero_detect_base; } diff --git a/raid/aarch64/raid_aarch64_dispatcher.c b/raid/aarch64/raid_aarch64_dispatcher.c index 3251a814..6f71e1aa 100644 --- a/raid/aarch64/raid_aarch64_dispatcher.c +++ b/raid/aarch64/raid_aarch64_dispatcher.c @@ -27,47 +27,60 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. **********************************************************************/ #include +#include "raid.h" + +extern int +xor_gen_neon(int, int, void **); + +extern int +xor_check_neon(int, int, void **); + +extern int +pq_gen_neon(int, int, void **); + +extern int +pq_check_neon(int, int, void **); DEFINE_INTERFACE_DISPATCHER(xor_gen) { #if defined(__linux__) if (getauxval(AT_HWCAP) & HWCAP_ASIMD) - return PROVIDER_INFO(xor_gen_neon); + return xor_gen_neon; #elif defined(__APPLE__) - return PROVIDER_INFO(xor_gen_neon); + return xor_gen_neon; #endif - return PROVIDER_BASIC(xor_gen); + return xor_gen_base; } DEFINE_INTERFACE_DISPATCHER(xor_check) { #if defined(__linux__) if (getauxval(AT_HWCAP) & HWCAP_ASIMD) - return PROVIDER_INFO(xor_check_neon); + return xor_check_neon; #elif defined(__APPLE__) - return PROVIDER_INFO(xor_check_neon); + return xor_check_neon; #endif - return PROVIDER_BASIC(xor_check); + return xor_check_base; } DEFINE_INTERFACE_DISPATCHER(pq_gen) { #if defined(__linux__) if (getauxval(AT_HWCAP) & HWCAP_ASIMD) - return PROVIDER_INFO(pq_gen_neon); + return pq_gen_neon; #elif defined(__APPLE__) - return PROVIDER_INFO(pq_gen_neon); + return pq_gen_neon; #endif - return PROVIDER_BASIC(pq_gen); + return pq_gen_base; } DEFINE_INTERFACE_DISPATCHER(pq_check) { #if defined(__linux__) if (getauxval(AT_HWCAP) & HWCAP_ASIMD) - return PROVIDER_INFO(pq_check_neon); + return pq_check_neon; #elif defined(__APPLE__) - return PROVIDER_INFO(pq_check_neon); + return pq_check_neon; #endif - return PROVIDER_BASIC(pq_check); + return pq_check_base; }