[PATCH] net/crc: improved barrett reduction in x86 SSE42
Shreesh Adiga
16567adigashreesh at gmail.com
Tue Nov 10 13:10:48 CET 2026
Use 64-bit polynomial structure and perform barrett reduction
from 128-bit to 64-bit which involves lesser instructions than
performing the 128 to 64 followed by 64 to 32 sequence.
This also requires one less fold register as we can reuse the x^127 from
single fold loop required for initial 128-bit reduction sequence.
Move CRC16-CCITT and CRC32-ETH folding constants into static const context
structures in their respective handlers, removing the runtime
initialization previously performed by rte_net_crc_sse42_init and avoids
the extra data segment static variables which were being populated.
Signed-off-by: Shreesh Adiga <16567adigashreesh at gmail.com>
---
lib/net/net_crc.h | 3 -
lib/net/net_crc_sse.c | 134 +++++++++++++++++-------------------------
lib/net/rte_net_crc.c | 10 ----
3 files changed, 53 insertions(+), 94 deletions(-)
diff --git a/lib/net/net_crc.h b/lib/net/net_crc.h
index 320b0edca8..0d61ccafd6 100644
--- a/lib/net/net_crc.h
+++ b/lib/net/net_crc.h
@@ -13,9 +13,6 @@
/* SSE4.2 */
-void
-rte_net_crc_sse42_init(void);
-
uint32_t
rte_crc16_ccitt_sse42_handler(const uint8_t *data, uint32_t data_len);
diff --git a/lib/net/net_crc_sse.c b/lib/net/net_crc_sse.c
index e30f8544fc..d5d5fdba3d 100644
--- a/lib/net/net_crc_sse.c
+++ b/lib/net/net_crc_sse.c
@@ -12,15 +12,23 @@
#include "net_crc.h"
/** PCLMULQDQ CRC computation context structure */
-struct crc_pclmulqdq_ctx {
- __m128i rk1_rk2;
- __m128i rk3_rk4;
- __m128i rk5_rk6;
- __m128i rk7_rk8;
+struct __rte_aligned(XMM_SIZE) crc_pclmulqdq_ctx {
+ /**
+ * The constants are derived as below:
+ * P is assumed to be degree 64 polynomial. If the degree is less,
+ * then P = (poly << (64 - deg)) to have a 64-bit polynomial.
+ * rk1 = bitreverse64(x^575 MOD P)
+ * rk2 = bitreverse64(x^511 MOD P)
+ * rk3 = bitreverse64(x^191 MOD P)
+ * rk4 = bitreverse64(x^127 MOD P)
+ * rk5 = (bitreverse64(floor(x^128 / P) - x^64) << 1) | 1
+ * rk6 = (bitreverse64(P - x^64) << 1)
+ */
+ rte_xmm_t rk1_rk2;
+ rte_xmm_t rk3_rk4;
+ rte_xmm_t rk5_rk6;
};
-static alignas(16) struct crc_pclmulqdq_ctx crc32_eth_pclmulqdq;
-static alignas(16) struct crc_pclmulqdq_ctx crc16_ccitt_pclmulqdq;
/**
* @brief Performs one folding round
*
@@ -47,65 +55,56 @@ crcr32_folding_round(__m128i data_block,
__m128i precomp,
__m128i fold)
{
- __m128i tmp0 = _mm_clmulepi64_si128(fold, precomp, 0x01);
- __m128i tmp1 = _mm_clmulepi64_si128(fold, precomp, 0x10);
+ __m128i tmp0 = _mm_clmulepi64_si128(fold, precomp, 0x00);
+ __m128i tmp1 = _mm_clmulepi64_si128(fold, precomp, 0x11);
return _mm_xor_si128(tmp1, _mm_xor_si128(data_block, tmp0));
}
/**
- * Performs reduction from 128 bits to 64 bits
+ * Fold the low qword (upper 64 coefficients) with rk4 = x^127 mod P
+ * and XOR into the high qword.
*
* @param data128
* 128 bits data to be reduced
* @param precomp
- * precomputed constants rk5, rk6
+ * precomputed constants rk3, rk4
*
* @return
- * 64 bits reduced data
+ * 128 bits reduced data
*/
static __rte_always_inline __m128i
-crcr32_reduce_128_to_64(__m128i data128, __m128i precomp)
+crcr32_reduce_to_128(__m128i data128, __m128i precomp)
{
- __m128i tmp0, tmp1, tmp2;
+ __m128i tmp0, tmp1;
- /* 64b fold */
- tmp0 = _mm_clmulepi64_si128(data128, precomp, 0x00);
+ tmp0 = _mm_clmulepi64_si128(data128, precomp, 0x10);
tmp1 = _mm_srli_si128(data128, 8);
- tmp0 = _mm_xor_si128(tmp0, tmp1);
-
- /* 32b fold */
- tmp2 = _mm_slli_si128(tmp0, 4);
- tmp1 = _mm_clmulepi64_si128(tmp2, precomp, 0x10);
-
return _mm_xor_si128(tmp1, tmp0);
}
/**
- * Performs Barret's reduction from 64 bits to 32 bits
+ * Performs Barret's reduction from 128 bits to 32 bits.
*
- * @param data64
- * 64 bits data to be reduced
+ * @param data128
+ * 128 bits data to be reduced
* @param precomp
- * rk7 precomputed constant
+ * rk5, rk6 precomputed constant
*
* @return
* reduced 32 bits data
*/
static __rte_always_inline uint32_t
-crcr32_reduce_64_to_32(__m128i data64, __m128i precomp)
+crcr32_reduce_128_to_32(__m128i data128, __m128i precomp)
{
__m128i tmp0, tmp1, tmp2;
- tmp0 = _mm_blend_epi16(data64, _mm_setzero_si128(), 0x3);
-
- tmp1 = _mm_clmulepi64_si128(tmp0, precomp, 0x00);
- tmp1 = _mm_xor_si128(tmp1, tmp0);
+ tmp0 = _mm_clmulepi64_si128(data128, precomp, 0x00);
+ tmp1 = _mm_clmulepi64_si128(tmp0, precomp, 0x10);
- tmp2 = _mm_clmulepi64_si128(tmp1, precomp, 0x10);
- tmp2 = _mm_xor_si128(tmp2, tmp0);
+ tmp2 = _mm_xor_si128(tmp1, data128);
return _mm_extract_epi32(tmp2, 2);
}
@@ -162,7 +161,7 @@ crc32_eth_calc_pclmulqdq(
fold3 = _mm_loadu_si128((const __m128i *)(data + 32));
fold4 = _mm_loadu_si128((const __m128i *)(data + 48));
fold1 = _mm_xor_si128(fold1, temp);
- k = params->rk1_rk2;
+ k = params->rk1_rk2.x;
for (n = 64; (n + 64) <= data_len; n += 64) {
temp1 = _mm_loadu_si128((const __m128i *)&data[n]);
@@ -175,7 +174,7 @@ crc32_eth_calc_pclmulqdq(
fold4 = crcr32_folding_round(temp4, k, fold4);
}
- k = params->rk3_rk4;
+ k = params->rk3_rk4.x;
fold1 = crcr32_folding_round(fold2, k, fold1);
fold1 = crcr32_folding_round(fold3, k, fold1);
fold = crcr32_folding_round(fold4, k, fold1);
@@ -196,7 +195,8 @@ crc32_eth_calc_pclmulqdq(
goto barret_reduction;
}
fold = xmm_shift_left(fold, 16 - data_len);
- goto reduction_128_64;
+ k = params->rk3_rk4.x;
+ goto reduction_128;
}
/** At least 16 bytes in the buffer */
@@ -205,7 +205,7 @@ crc32_eth_calc_pclmulqdq(
fold = _mm_xor_si128(fold, temp);
/** Single folding loop - the last 16 bytes is processed separately */
- k = params->rk3_rk4;
+ k = params->rk3_rk4.x;
n = 16;
single_fold_loop:
@@ -234,59 +234,25 @@ crc32_eth_calc_pclmulqdq(
}
/** Reduction 128 -> 32 Assumes: fold holds 128bit folded data */
-reduction_128_64:
- k = params->rk5_rk6;
- fold = crcr32_reduce_128_to_64(fold, k);
+reduction_128:
+ fold = crcr32_reduce_to_128(fold, k);
barret_reduction:
- k = params->rk7_rk8;
- n = crcr32_reduce_64_to_32(fold, k);
+ k = params->rk5_rk6.x;
+ n = crcr32_reduce_128_to_32(fold, k);
return n;
}
-void
-rte_net_crc_sse42_init(void)
-{
- uint64_t k1, k2, k3, k4, k5, k6;
- uint64_t p = 0, q = 0;
-
- /** Initialize CRC16 data */
- k1 = 0x14ff2LLU;
- k2 = 0x19a3cLLU;
- k3 = 0x189aeLLU;
- k4 = 0x8e10LLU;
- k5 = 0x189aeLLU;
- k6 = 0x114aaLLU;
- q = 0x11c581910LLU;
- p = 0x10811LLU;
-
- /** Save the params in context structure */
- crc16_ccitt_pclmulqdq.rk1_rk2 = _mm_set_epi64x(k2, k1);
- crc16_ccitt_pclmulqdq.rk3_rk4 = _mm_set_epi64x(k4, k3);
- crc16_ccitt_pclmulqdq.rk5_rk6 = _mm_set_epi64x(k6, k5);
- crc16_ccitt_pclmulqdq.rk7_rk8 = _mm_set_epi64x(p, q);
-
- /** Initialize CRC32 data */
- k1 = 0x1c6e41596LLU;
- k2 = 0x154442bd4LLU;
- k3 = 0xccaa009eLLU;
- k4 = 0x1751997d0LLU;
- k5 = 0xccaa009eLLU;
- k6 = 0x163cd6124LLU;
- q = 0x1f7011640LLU;
- p = 0x1db710641LLU;
-
- /** Save the params in context structure */
- crc32_eth_pclmulqdq.rk1_rk2 = _mm_set_epi64x(k2, k1);
- crc32_eth_pclmulqdq.rk3_rk4 = _mm_set_epi64x(k4, k3);
- crc32_eth_pclmulqdq.rk5_rk6 = _mm_set_epi64x(k6, k5);
- crc32_eth_pclmulqdq.rk7_rk8 = _mm_set_epi64x(p, q);
-}
-
uint32_t
rte_crc16_ccitt_sse42_handler(const uint8_t *data, uint32_t data_len)
{
+ static const struct crc_pclmulqdq_ctx crc16_ccitt_pclmulqdq = {
+ .rk1_rk2 = { .u64 = { 0x922dULL, 0x47e3ULL } },
+ .rk3_rk4 = { .u64 = { 0x8e10ULL, 0x81bfULL } },
+ .rk5_rk6 = { .u64 = { 0x859b040b1c581911ULL, 0x10810ULL } },
+ };
+
/** return 16-bit CRC value */
return (uint16_t)~crc32_eth_calc_pclmulqdq(data,
data_len,
@@ -297,6 +263,12 @@ rte_crc16_ccitt_sse42_handler(const uint8_t *data, uint32_t data_len)
uint32_t
rte_crc32_eth_sse42_handler(const uint8_t *data, uint32_t data_len)
{
+ static const struct crc_pclmulqdq_ctx crc32_eth_pclmulqdq = {
+ .rk1_rk2 = { .u64 = { 0x8f352d95ULL, 0x1d9513d7ULL } },
+ .rk3_rk4 = { .u64 = { 0xae689191ULL, 0xccaa009eULL } },
+ .rk5_rk6 = { .u64 = { 0xb4e5b025f7011641ULL, 0x1db710640ULL } },
+ };
+
return ~crc32_eth_calc_pclmulqdq(data,
data_len,
0xffffffffUL,
diff --git a/lib/net/rte_net_crc.c b/lib/net/rte_net_crc.c
index 3a589bdd6d..6becd3e336 100644
--- a/lib/net/rte_net_crc.c
+++ b/lib/net/rte_net_crc.c
@@ -151,15 +151,6 @@ avx512_vpclmulqdq_init(void)
#define SSE42_PCLMULQDQ_CPU_SUPPORTED \
rte_cpu_get_flag_enabled(RTE_CPUFLAG_PCLMULQDQ)
-static void
-sse42_pclmulqdq_init(void)
-{
-#ifdef RTE_ARCH_X86_64
- if (SSE42_PCLMULQDQ_CPU_SUPPORTED)
- rte_net_crc_sse42_init();
-#endif
-}
-
/* NEON/PMULL handling */
#define NEON_PMULL_CPU_SUPPORTED \
@@ -272,7 +263,6 @@ uint32_t rte_net_crc_calc(const struct rte_net_crc *ctx, const void *data, const
RTE_INIT(rte_net_crc_init)
{
rte_net_crc_scalar_init();
- sse42_pclmulqdq_init();
avx512_vpclmulqdq_init();
neon_pmull_init();
handlers_init(RTE_NET_CRC_SCALAR);
--
2.56.0
More information about the dev
mailing list