This patch uses RISC-V vector instructions and zicbop prefetching to optimize memory copies for 129~1600 byte ranges. Signed-off-by: Qiguo Chen <chen.qiguo@zte.com.cn> --- .mailmap | 1 + config/riscv/meson.build | 14 ++ lib/eal/riscv/include/rte_memcpy.h | 303 ++++++++++++++++++++++++++++- 3 files changed, 316 insertions(+), 2 deletions(-) diff --git a/.mailmap b/.mailmap index e4d0590451..8fcdc518f9 100644 --- a/.mailmap +++ b/.mailmap @@ -1291,6 +1291,7 @@ Qi Zhang <qi.z.zhang@intel.com> Qian Hao <qi_an_hao@126.com> Qian Xu <qian.q.xu@intel.com> Qiao Liu <qiao.liu@intel.com> +Qiguo Chen <chen.qiguo@zte.com.cn> Qimai Xiao <qimaix.xiao@intel.com> Qiming Chen <chenqiming_huawei@163.com> Qiming Yang <qiming.yang@intel.com> diff --git a/config/riscv/meson.build b/config/riscv/meson.build index f3daea0c0e..abba474b5e 100644 --- a/config/riscv/meson.build +++ b/config/riscv/meson.build @@ -146,6 +146,20 @@ if (riscv_extension_macros and endif endif +# detect extensions +# Requires intrinsics available in GCC 14.1.0+ and Clang 18.1.0+ +if (riscv_extension_macros and + (cc.get_define('__riscv_zicbop', args: machine_args) != '')) + if ((cc.get_id() == 'gcc' and cc.version().version_compare('>=14.1.0')) + or (cc.get_id() == 'clang' and cc.version().version_compare('>=18.1.0'))) + message('Compiling with the zicbop extension') + machine_args += ['-DRTE_RISCV_FEATURE_PREFETCH'] + else + warning('Detected zicbop extension but cannot use because intrinsics are not available (present in GCC 14.1.0+ and Clang 18.1.0+)') + endif +endif + + # apply flags foreach flag: dpdk_flags if flag.length() > 0 diff --git a/lib/eal/riscv/include/rte_memcpy.h b/lib/eal/riscv/include/rte_memcpy.h index d8a942c5d2..1be3ad748a 100644 --- a/lib/eal/riscv/include/rte_memcpy.h +++ b/lib/eal/riscv/include/rte_memcpy.h @@ -11,13 +11,291 @@ #include <string.h> #include "rte_common.h" +#include <rte_branch_prediction.h> #include "generic/rte_memcpy.h" +#if defined(RTE_RISCV_FEATURE_V) && !(defined(RTE_RISCV_FEATURE_PREFETCH)) +#undef RTE_RISCV_FEATURE_V +#endif + +#if defined(RTE_RISCV_FEATURE_V) +#include "rte_cpuflags.h" +#endif + #ifdef __cplusplus extern "C" { #endif +#if defined(RTE_RISCV_FEATURE_V) + +#define RISCV_VLENB 16 +#define MEMCPY_GLIBC (1U << 0) +#define MEMCPY_RISCV (1U << 1) +#define ALIGNMENT_MASK_128 0x7F +#define ALIGNMENT_MASK_64 0x3F + +static uint8_t memcpy_alg = MEMCPY_GLIBC; + + +static __rte_always_inline void +memcpy_prefetch64_1(const uint8_t *src, uint8_t *dst) +{ + __asm__ ( + "prefetch.r 64(%0)\n" + "prefetch.w 64(%1)" + :: "r"(src), "r"(dst) + ); +} + +static __rte_always_inline void +memcpy_prefetch128_1(const uint8_t *src, uint8_t *dst) +{ + __asm__ ( + "prefetch.r 128(%0)\n" + "prefetch.w 128(%1)" + :: "r"(src), "r"(dst) + ); +} + +static __rte_always_inline void +memcpy_prefetch128_2(const uint8_t *src, uint8_t *dst) +{ + __asm__ ( + "prefetch.r 128(%0);" + "prefetch.w 128(%1);" + "prefetch.r 192(%0);" + "prefetch.w 192(%1)" + :: "r"(src), "r"(dst) + ); +} + + +static __rte_always_inline void +_rte_mov32(uint8_t *dst, const uint8_t *src) +{ + uint32_t n = 32; + asm volatile ( + "vsetvli t1, %2, e8, m2, ta, ma\n" + "vle8.v v2, (%1)\n" + "vse8.v v2, (%0)" + :: "r"(dst), "r"(src), "r"(n) + : "v2", "v3", "t1", "memory" + ); +} + +static __rte_always_inline void +_rte_mov64(uint8_t *dst, const uint8_t *src) +{ + uint32_t n = 64; + asm volatile ( + "vsetvli t3, %2, e8, m4, ta, ma\n" + "vle8.v v8, (%1)\n" + "vse8.v v8, (%0)" + :: "r"(dst), "r"(src), "r"(n) + : "v8", "v9", "v10", "v11", "t3", "memory" + ); +} + +static __rte_always_inline void +_rte_mov128(uint8_t *dst, const uint8_t *src) +{ + uint32_t n = 128; + asm volatile ( + "vsetvli t4, %2, e8, m8, ta, ma\n" + "vle8.v v16, (%1)\n" + "vse8.v v16, (%0)" + :: "r"(dst), "r"(src), "r"(n) + : "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "t4", "memory" + ); +} + +static __rte_always_inline void +_rte_mov256(uint8_t *dst, const uint8_t *src) +{ + memcpy_prefetch128_2(src, dst); + _rte_mov128(dst, src); + _rte_mov128(dst + 128, src + 128); +} + +static __rte_always_inline void +_rte_mov128blocks(uint8_t *dst, const uint8_t *src, size_t n) +{ + asm volatile ( + "prefetch.r 64(%1)\n" + "prefetch.w 64(%0)\n" + "prefetch.r 128(%1)\n" + "prefetch.w 128(%0)\n" + "prefetch.r 192(%1)\n" + "prefetch.w 192(%0)\n" + "prefetch.r 256(%1)\n" + "prefetch.w 256(%0)\n" + "prefetch.r 320(%1)\n" + "prefetch.w 320(%0)\n" + "prefetch.r 384(%1)\n" + "prefetch.w 384(%0)\n" + "prefetch.r 448(%1)\n" + "prefetch.w 448(%0)\n" + "prefetch.r 512(%1)\n" + "li t6, 512\n" + "3:\n" + "li t5, 128;" + "vsetvli zero, t5, e8, m8, ta, ma\n" + "1:;" + "bgt %2, t6, 4f\n" + "j 2f\n" + "4:\n" + "prefetch.r 576(%1)\n" + "prefetch.r 640(%1)\n" + "2:\n" + "vle8.v v16, (%1)\n" + "add %1, %1, t5\n" + "vse8.v v16, (%0)\n" + "add %0, %0, t5\n" + "sub %2, %2, t5\n" + "bnez %2, 1b" + : "+r"(dst), "+r"(src), "+r"(n) + : + : "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "t5", "t6", "memory" + ); +} + +static __rte_always_inline void +_rte_mov(uint8_t *dst, const uint8_t *src, uint32_t n) +{ + asm volatile ( + "1:\n" + "vsetvli t4, %2, e8, m8, ta, ma\n" + "vle8.v v16, (%1)\n" + "add %1, %1, t4\n" + "vse8.v v16, (%0)\n" + "add %0, %0, t4\n" + "sub %2, %2, t4\n" + "bnez %2, 1b" + : "+r"(dst), "+r"(src), "+r"(n) + : + : "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "t4", "memory" + ); +} + +static __rte_always_inline void +_rte_mov_aligned(uint8_t *dst, const uint8_t *src, uint32_t n) +{ + asm volatile ( + "prefetch.r 128(%1)\n" + "prefetch.r 192(%1)\n" + "prefetch.r 256(%1)\n" + "prefetch.r 320(%1)\n" + "prefetch.r 384(%1)\n" + "prefetch.r 448(%1)\n" + "prefetch.r 512(%1)\n" + "prefetch.r 576(%1)\n" + "li t6, 640\n" + "1:\n" + "vsetvli t4, %2, e8, m8, ta, ma\n" + "vle8.v v16, (%1)\n" + "add %1, %1, t4\n" + "vse8.v v16, (%0)\n" + "add %0, %0, t4\n" + "sub %2, %2, t4\n" + "blt %2, t6, 3f\n" + "prefetch.r 512(%1)\n" + "prefetch.r 576(%1)\n" + "3:\n" + "bnez %2, 1b" + : "+r"(dst), "+r"(src), "+r"(n) + : + : "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "t4", "t6", "memory" + ); +} + +static __rte_always_inline void * +_rte_memcpy(uint8_t *dst, const uint8_t *src, size_t n) +{ + void *ret = dst; + size_t dstofss; + uint32_t bn; + + if (n <= 384) { + if (n >= 256) { + memcpy_prefetch128_2(src, dst); + n -= 256; + _rte_mov128(dst, src); + _rte_mov128((uint8_t *)dst + 128, (const uint8_t *)src + 128); + src = (const uint8_t *)src + 256; + dst = (uint8_t *)dst + 256; + } + if (n >= 128) { + memcpy_prefetch128_1(src, dst); + n -= 128; + _rte_mov128(dst, src); + src = (const uint8_t *)src + 128; + dst = (uint8_t *)dst + 128; + } + + if (n >= 64) { + memcpy_prefetch64_1(src, dst); + n -= 64; + _rte_mov64(dst, src); + src = (const uint8_t *)src + 64; + dst = (uint8_t *)dst + 64; + } + + if (n > 32) { + _rte_mov32(dst, src); + _rte_mov32((uint8_t *)dst - 32 + n, + (const uint8_t *)src - 32 + n); + return ret; + } + + if (n > 0) { + _rte_mov32((uint8_t *)dst - 32 + n, + (const uint8_t *)src - 32 + n); + } + return ret; + } + + /** + * Make store aligned when copy size exceeds 256 bytes. + */ + dstofss = (uintptr_t)dst & ALIGNMENT_MASK_64; + if (dstofss > 0) { + dstofss = 64 - dstofss; + n -= dstofss; + _rte_mov64(dst, src); + src = (const uint8_t *)src + dstofss; + dst = (uint8_t *)dst + dstofss; + } + + /** + * Copy 128-byte blocks + */ + if ((uintptr_t)src & ALIGNMENT_MASK_64) { + bn = n - (n & ALIGNMENT_MASK_128); + _rte_mov128blocks(dst, src, bn); + n = n & ALIGNMENT_MASK_128; + src = (const uint8_t *)src + bn; + dst = (uint8_t *)dst + bn; + _rte_mov(dst, src, n); + } else + _rte_mov_aligned(dst, src, n); + + return ret; +} + +#endif + +/*----------------------api---------------------------------------------------*/ +static __rte_always_inline void * +rte_memcpy(void *dst, const void *src, size_t n) +{ +#if defined(RTE_RISCV_FEATURE_V) + if (likely((memcpy_alg == MEMCPY_RISCV) && (n >= 128) && (n < 2048))) + return _rte_memcpy((uint8_t *)dst, (const uint8_t *)src, n); +#endif + return memcpy(dst, src, n); +} + static inline void rte_mov16(uint8_t *dst, const uint8_t *src) { @@ -51,10 +329,31 @@ rte_mov128(uint8_t *dst, const uint8_t *src) static inline void rte_mov256(uint8_t *dst, const uint8_t *src) { - memcpy(dst, src, 256); +#if defined(RTE_RISCV_FEATURE_V) + if (likely(memcpy_alg == MEMCPY_RISCV)) + _rte_mov256(dst, src); + else +#endif + memcpy(dst, src, 256); +} +/*----------------------------------------------------------------------------*/ +#if defined(RTE_RISCV_FEATURE_V) +static inline long +riscv_vlenb(void) +{ + long vlenb; + asm ("csrr %0, 0xc22" : "=r"(vlenb)); + return vlenb; } -#define rte_memcpy(d, s, n) memcpy((d), (s), (n)) +RTE_INIT(rte_vect_memcpy_init) +{ + long vlenb = riscv_vlenb(); + if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_RISCV_ISA_V) && (vlenb >= RISCV_VLENB)) + memcpy_alg = MEMCPY_RISCV; +} +#endif + #ifdef __cplusplus } -- 2.21.0.windows.1