[PATCH] net/crc: make NEON CRC folding logic similar to x86 SSE

[Shreesh Adiga]

Hi Wathsala,

I tested this on QEMU for correctness. I don't have access to any ARM
environment for precise benchmarking.
The only device I have is an Android phone where I can't compile DPDK
either.
I can isolate this code and use libbenchmark via Termux if that is
needed, but I don't expect to see large measurable benefits.

This patch is based on my previous work on FFmpeg's av_crc optimization for
aarch64 which you can
refer here: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21651

Thanks,
Shreesh

On Fri, Oct 9, 2026 at 8:54 AM Wathsala Vithanage <
wathsala.vithanage at arm.com> wrote:

> Hi Shreesh,
>
> Looks correct. Can you share before and after performance data in the
> commit message covering short inputs below 16 bytes, lengths exercising
> the different nonzero remainders modulo 16, and larger buffers to show
> the effect on the folding loop?
>
> Thanks
>
> --wathsala
>
> On 10/6/26 10:18, Shreesh Adiga wrote:
> > This patch includes the following minor changes:
> > 1) Use TBL based shift left for variable len shift instead of using
> >     vshift_bytes_left which adds a switch case for 16 len cases.
> > 2) Align the fold constants to use PMULL and PMULL2 instructions
> >     instead of using cross lane operands which requires additional shift.
> > 3) Remove unnecessary static arrays used in crc32_reduce_64_to_32 by
> >     adapting x86 SSE logic.
> > 4) Use x86 SSE logic for processing partial bytes sequence to avoid
> >     using multiple variable len shifts which inserts switch case.
> >
> > Signed-off-by: Shreesh Adiga <16567adigashreesh at gmail.com>
> > ---
> >   lib/net/net_crc_neon.c | 71 +++++++++++++++++++++++++++---------------
> >   1 file changed, 46 insertions(+), 25 deletions(-)
> >
> > diff --git a/lib/net/net_crc_neon.c b/lib/net/net_crc_neon.c
> > index d924aed25d..3833fcba6e 100644
> > --- a/lib/net/net_crc_neon.c
> > +++ b/lib/net/net_crc_neon.c
> > @@ -24,6 +24,31 @@ struct crc_pmull_ctx {
> >   alignas(16) struct crc_pmull_ctx crc32_eth_pmull;
> >   alignas(16) struct crc_pmull_ctx crc16_ccitt_pmull;
> >
> > +static const alignas(16) uint8_t crc_neon_shift_tab[32] = {
> > +     0xff, 0xfe, 0xfd, 0xfc, 0xfb, 0xfa, 0xf9, 0xf8,
> > +     0xf7, 0xf6, 0xf5, 0xf4, 0xf3, 0xf2, 0xf1, 0xf0,
> > +     0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
> > +     0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f
> > +};
> > +
> > +/**
> > + * Shifts left 128 bit register by specified number of bytes
> > + *
> > + * @param reg
> > + *   128 bit value
> > + * @param num
> > + *   number of bytes to shift left reg by (0-16)
> > + *
> > + * @return
> > + *   reg << (num * 8)
> > + */
> > +static inline uint64x2_t
> > +neon_shift_left(uint64x2_t reg, const unsigned int num)
> > +{
> > +     uint8x16_t tbl = vld1q_u8(crc_neon_shift_tab + 16 - num);
> > +     return vreinterpretq_u64_u8(vqtbl1q_u8(vreinterpretq_u8_u64(reg),
> tbl));
> > +}
> > +
> >   /**
> >    * @brief Performs one folding round
> >    *
> > @@ -46,12 +71,12 @@ crcr32_folding_round(uint64x2_t data_block,
> uint64x2_t precomp,
> >       uint64x2_t fold)
> >   {
> >       uint64x2_t tmp0 = vreinterpretq_u64_p128(vmull_p64(
> > -                     vgetq_lane_p64(vreinterpretq_p64_u64(fold), 1),
> > +                     vgetq_lane_p64(vreinterpretq_p64_u64(fold), 0),
> >                       vgetq_lane_p64(vreinterpretq_p64_u64(precomp),
> 0)));
> >
> > -     uint64x2_t tmp1 = vreinterpretq_u64_p128(vmull_p64(
> > -                     vgetq_lane_p64(vreinterpretq_p64_u64(fold), 0),
> > -                     vgetq_lane_p64(vreinterpretq_p64_u64(precomp),
> 1)));
> > +     uint64x2_t tmp1 = vreinterpretq_u64_p128(vmull_high_p64(
> > +                     vreinterpretq_p64_u64(fold),
> > +                     vreinterpretq_p64_u64(precomp)));
> >
> >       return veorq_u64(tmp1, veorq_u64(data_block, tmp0));
> >   }
> > @@ -98,26 +123,19 @@ static inline uint32_t
> >   crcr32_reduce_64_to_32(uint64x2_t data64,
> >       uint64x2_t precomp)
> >   {
> > -     static alignas(16) uint32_t mask1[4] = {
> > -             0xffffffff, 0xffffffff, 0x00000000, 0x00000000
> > -     };
> > -     static alignas(16) uint32_t mask2[4] = {
> > -             0x00000000, 0xffffffff, 0xffffffff, 0xffffffff
> > -     };
> >       uint64x2_t tmp0, tmp1, tmp2;
> >
> > -     tmp0 = vandq_u64(data64, vld1q_u64((uint64_t *)mask2));
> > +     tmp0 = vreinterpretq_u64_u32(
> > +             vsetq_lane_u32(0, vreinterpretq_u32_u64(data64), 0));
> >
> >       tmp1 = vreinterpretq_u64_p128(vmull_p64(
> >               vgetq_lane_p64(vreinterpretq_p64_u64(tmp0), 0),
> >               vgetq_lane_p64(vreinterpretq_p64_u64(precomp), 0)));
> >       tmp1 = veorq_u64(tmp1, tmp0);
> > -     tmp1 = vandq_u64(tmp1, vld1q_u64((uint64_t *)mask1));
> >
> >       tmp2 = vreinterpretq_u64_p128(vmull_p64(
> >               vgetq_lane_p64(vreinterpretq_p64_u64(tmp1), 0),
> >               vgetq_lane_p64(vreinterpretq_p64_u64(precomp), 1)));
> > -     tmp2 = veorq_u64(tmp2, tmp1);
> >       tmp2 = veorq_u64(tmp2, tmp0);
> >
> >       return vgetq_lane_u32(vreinterpretq_u32_u64(tmp2), 2);
> > @@ -177,10 +195,10 @@ crc32_eth_calc_pmull(
> >               fold = vld1q_u64((uint64_t *)buffer);
> >               fold = veorq_u64(fold, temp);
> >               if (unlikely(data_len < 4)) {
> > -                     fold = vshift_bytes_left(fold, 8 - data_len);
> > +                     fold = neon_shift_left(fold, 8 - data_len);
> >                       goto barret_reduction;
> >               }
> > -             fold = vshift_bytes_left(fold, 16 - data_len);
> > +             fold = neon_shift_left(fold, 16 - data_len);
> >               goto reduction_128_64;
> >       }
> >
> > @@ -201,14 +219,17 @@ crc32_eth_calc_pmull(
> >
> >       /** Partial bytes - process last <16 bytes */
> >       if (likely(n < data_len)) {
> > -             uint64x2_t last16, a, b, mask;
> > +             uint8x16_t last16, t1, t2;
> > +             uint64x2_t a, b;
> >               uint32_t rem = data_len & 15;
> >
> > -             last16 = vld1q_u64((const uint64_t *)&data[data_len - 16]);
> > -             a = vshift_bytes_left(fold, 16 - rem);
> > -             b = vshift_bytes_right(fold, rem);
> > -             mask = vshift_bytes_left(vdupq_n_u64(-1), 16 - rem);
> > -             b = vorrq_u64(b, vandq_u64(mask, last16));
> > +             last16 = vld1q_u8((const uint8_t *)&data[data_len - 16]);
> > +             t1 = vld1q_u8(crc_neon_shift_tab + rem);
> > +             a  =
> vreinterpretq_u64_u8(vqtbl1q_u8(vreinterpretq_u8_u64(fold), t1));
> > +             t2 = vmvnq_u8(t1);
> > +             t2 = vqtbl1q_u8(vreinterpretq_u8_u64(fold), t2);
> > +             t1 = vcgezq_s8(vreinterpretq_s8_u8(t1));
> > +             b  = vreinterpretq_u64_u8(vbslq_u8(t1, last16, t2));
> >
> >               /* k = rk3 & rk4 */
> >               fold = crcr32_folding_round(b, k, a);
> > @@ -230,14 +251,14 @@ void
> >   rte_net_crc_neon_init(void)
> >   {
> >       /* Initialize CRC16 data */
> > -     uint64_t ccitt_k1_k2[2] = {0x14ff2LLU, 0x19a3cLLU};
> > -     uint64_t ccitt_k3_k4[2] = {0x189aeLLU, 0x8e10LLU};
> > +     uint64_t ccitt_k1_k2[2] = {0x19a3cLLU, 0x14ff2LLU};
> > +     uint64_t ccitt_k3_k4[2] = {0x8e10LLU, 0x189aeLLU};
> >       uint64_t ccitt_k5_k6[2] = {0x189aeLLU, 0x114aaLLU};
> >       uint64_t ccitt_k7_k8[2] = {0x11c581910LLU, 0x10811LLU};
> >
> >       /* Initialize CRC32 data */
> > -     uint64_t eth_k1_k2[2] = {0x1c6e41596LLU, 0x154442bd4LLU};
> > -     uint64_t eth_k3_k4[2] = {0xccaa009eLLU, 0x1751997d0LLU};
> > +     uint64_t eth_k1_k2[2] = {0x154442bd4LLU, 0x1c6e41596LLU};
> > +     uint64_t eth_k3_k4[2] = {0x1751997d0LLU, 0xccaa009eLLU};
> >       uint64_t eth_k5_k6[2] = {0xccaa009eLLU, 0x163cd6124LLU};
> >       uint64_t eth_k7_k8[2] = {0x1f7011640LLU, 0x1db710641LLU};
> >
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mails.dpdk.org/archives/dev/attachments/20261009/5e1adf1d/attachment-0001.htm>