[PATCH v10 10/20] net/sxe2: add NEON vec Rx/Tx burst functions
liujie5 at linkdatatechnology.com
liujie5 at linkdatatechnology.com
Sat Jun 6 03:07:16 CEST 2026
From: Jie Liu <liujie5 at linkdatatechnology.com>
- Implement sxe2_recv_pkts_vec_neon for bulk packet receiving.
- Implement sxe2_xmit_pkts_vec_neon for bulk packet transmission.
- Added logic to select the vectorized path based on runtime config
and CPU flags (RTE_ARCH_ARM64).
Vectorized path improves throughput for small packets by processing
multiple descriptors simultaneously using SIMD instructions.
Signed-off-by: Jie Liu <liujie5 at linkdatatechnology.com>
---
drivers/net/sxe2/meson.build | 2 +
drivers/net/sxe2/sxe2_txrx.c | 39 +-
drivers/net/sxe2/sxe2_txrx_vec.h | 16 +-
drivers/net/sxe2/sxe2_txrx_vec_common.h | 1 +
drivers/net/sxe2/sxe2_txrx_vec_neon.c | 721 ++++++++++++++++++++++++
5 files changed, 774 insertions(+), 5 deletions(-)
create mode 100644 drivers/net/sxe2/sxe2_txrx_vec_neon.c
diff --git a/drivers/net/sxe2/meson.build b/drivers/net/sxe2/meson.build
index fc4466556b..4565046eae 100644
--- a/drivers/net/sxe2/meson.build
+++ b/drivers/net/sxe2/meson.build
@@ -48,6 +48,8 @@ if arch_subdir == 'x86'
include_directories: includes,
c_args: [cflags, '-mavx2'])
objs += sxe2_avx2_lib.extract_objects('sxe2_txrx_vec_avx2.c')
+elif arch_subdir == 'arm'
+ sources += files('sxe2_txrx_vec_neon.c')
endif
sources += files(
diff --git a/drivers/net/sxe2/sxe2_txrx.c b/drivers/net/sxe2/sxe2_txrx.c
index eaf95259a5..a919a84450 100644
--- a/drivers/net/sxe2/sxe2_txrx.c
+++ b/drivers/net/sxe2/sxe2_txrx.c
@@ -175,6 +175,9 @@ void sxe2_tx_mode_func_set(struct rte_eth_dev *dev)
if ((0 == (tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK)))
tx_mode_flags |= SXE2_TX_MODE_VEC_SSE;
+#elif defined(RTE_ARCH_ARM64)
+ if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON) == 1)
+ tx_mode_flags |= (vec_flags | SXE2_TX_MODE_VEC_NEON);
#endif
if (tx_mode_flags & SXE2_TX_MODE_VEC_SET_MASK) {
ret = sxe2_tx_queues_vec_prepare(dev);
@@ -218,8 +221,15 @@ void sxe2_tx_mode_func_set(struct rte_eth_dev *dev)
dev->tx_pkt_burst = sxe2_tx_pkts_vec_sse_simple;
}
}
- } else {
+#elif defined(RTE_ARCH_ARM64)
+ if (tx_mode_flags & SXE2_TX_MODE_VEC_NEON) {
+ dev->tx_pkt_prepare = sxe2_tx_pkts_prepare;
+ dev->tx_pkt_burst = sxe2_tx_pkts_vec_neon;
+ } else {
+ dev->tx_pkt_burst = sxe2_tx_pkts_vec_neon_simple;
+ }
#endif
+ } else {
if (tx_mode_flags & SXE2_TX_MODE_SIMPLE_BATCH) {
dev->tx_pkt_prepare = NULL;
dev->tx_pkt_burst = sxe2_tx_pkts_simple;
@@ -227,9 +237,7 @@ void sxe2_tx_mode_func_set(struct rte_eth_dev *dev)
dev->tx_pkt_prepare = sxe2_tx_pkts_prepare;
dev->tx_pkt_burst = sxe2_tx_pkts;
}
-#ifdef RTE_ARCH_X86
}
-#endif
}
static const struct {
@@ -253,6 +261,12 @@ static const struct {
{ sxe2_tx_pkts_vec_sse_simple,
"Vector SSE Simple" },
#endif
+#ifdef RTE_ARCH_ARM64
+ { sxe2_tx_pkts_vec_neon,
+ "Vector NEON" },
+ { sxe2_tx_pkts_vec_neon_simple,
+ "Vector NEON Simple" },
+#endif
};
int32_t sxe2_tx_burst_mode_get(struct rte_eth_dev *dev,
@@ -356,6 +370,11 @@ void sxe2_rx_mode_func_set(struct rte_eth_dev *dev)
if (((rx_mode_flags & SXE2_RX_MODE_VEC_SET_MASK) == 0) &&
rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128)
rx_mode_flags |= SXE2_RX_MODE_VEC_SSE;
+
+#elif defined(RTE_ARCH_ARM64)
+ if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON) == 1) {
+ rx_mode_flags |= (vec_flags | SXE2_RX_MODE_VEC_NEON);
+ }
#endif
if ((rx_mode_flags & SXE2_RX_MODE_VEC_SET_MASK) != 0) {
ret = sxe2_rx_queues_vec_prepare(dev);
@@ -387,6 +406,14 @@ void sxe2_rx_mode_func_set(struct rte_eth_dev *dev)
}
return;
}
+#elif defined(RTE_ARCH_ARM64)
+ if (rx_mode_flags & SXE2_RX_MODE_VEC_SET_MASK) {
+ if (rx_mode_flags & SXE2_RX_MODE_VEC_OFFLOAD)
+ dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_neon_offload;
+ else
+ dev->rx_pkt_burst = sxe2_rx_pkts_scattered_vec_neon;
+ return;
+ }
#endif
if (sxe2_rx_offload_en_check(dev, RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT))
dev->rx_pkt_burst = sxe2_rx_pkts_scattered_split;
@@ -416,6 +443,12 @@ static const struct {
{ sxe2_rx_pkts_scattered_vec_sse_offload,
"Vector SSE Scattered" },
#endif
+#ifdef RTE_ARCH_ARM64
+ { sxe2_rx_pkts_scattered_vec_neon,
+ "Vector NEON Scattered" },
+ { sxe2_rx_pkts_scattered_vec_neon_offload,
+ "Offload Vector NEON Scattered" },
+#endif
};
int32_t sxe2_rx_burst_mode_get(struct rte_eth_dev *dev,
diff --git a/drivers/net/sxe2/sxe2_txrx_vec.h b/drivers/net/sxe2/sxe2_txrx_vec.h
index 369777606f..c139aed776 100644
--- a/drivers/net/sxe2/sxe2_txrx_vec.h
+++ b/drivers/net/sxe2/sxe2_txrx_vec.h
@@ -13,19 +13,23 @@
#define SXE2_RX_MODE_VEC_SSE RTE_BIT32(2)
#define SXE2_RX_MODE_VEC_AVX2 RTE_BIT32(3)
#define SXE2_RX_MODE_VEC_AVX512 RTE_BIT32(4)
+#define SXE2_RX_MODE_VEC_NEON RTE_BIT32(5)
#define SXE2_RX_MODE_BATCH_ALLOC RTE_BIT32(10)
#define SXE2_RX_MODE_VEC_SET_MASK (SXE2_RX_MODE_VEC_SIMPLE | \
SXE2_RX_MODE_VEC_OFFLOAD | SXE2_RX_MODE_VEC_SSE | \
- SXE2_RX_MODE_VEC_AVX2 | SXE2_RX_MODE_VEC_AVX512)
+ SXE2_RX_MODE_VEC_AVX2 | SXE2_RX_MODE_VEC_AVX512 | \
+ SXE2_RX_MODE_VEC_NEON)
#define SXE2_TX_MODE_VEC_SIMPLE RTE_BIT32(0)
#define SXE2_TX_MODE_VEC_OFFLOAD RTE_BIT32(1)
#define SXE2_TX_MODE_VEC_SSE RTE_BIT32(2)
#define SXE2_TX_MODE_VEC_AVX2 RTE_BIT32(3)
#define SXE2_TX_MODE_VEC_AVX512 RTE_BIT32(4)
+#define SXE2_TX_MODE_VEC_NEON RTE_BIT32(5)
#define SXE2_TX_MODE_SIMPLE_BATCH RTE_BIT32(10)
#define SXE2_TX_MODE_VEC_SET_MASK (SXE2_TX_MODE_VEC_SIMPLE | \
SXE2_TX_MODE_VEC_OFFLOAD | SXE2_TX_MODE_VEC_SSE | \
- SXE2_TX_MODE_VEC_AVX2 | SXE2_TX_MODE_VEC_AVX512)
+ SXE2_TX_MODE_VEC_AVX2 | SXE2_TX_MODE_VEC_AVX512 | \
+ SXE2_TX_MODE_VEC_NEON)
#define SXE2_TX_VEC_NO_SUPPORT_OFFLOAD ( \
RTE_ETH_TX_OFFLOAD_MULTI_SEGS | \
RTE_ETH_TX_OFFLOAD_QINQ_INSERT | \
@@ -76,6 +80,14 @@ uint16_t sxe2_rx_pkts_scattered_vec_avx2(void *rx_queue,
struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
uint16_t sxe2_rx_pkts_scattered_vec_avx2_offload(void *rx_queue,
struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
+
+#elif defined(RTE_ARCH_ARM64)
+uint16_t sxe2_rx_pkts_scattered_vec_neon(void *rx_queue, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts);
+uint16_t sxe2_rx_pkts_scattered_vec_neon_offload(void *rx_queue, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts);
+uint16_t sxe2_tx_pkts_vec_neon_simple(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
+uint16_t sxe2_tx_pkts_vec_neon(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
#endif
int32_t __rte_cold sxe2_tx_vec_support_check(struct rte_eth_dev *dev, uint32_t *vec_flags);
int32_t __rte_cold sxe2_tx_queues_vec_prepare(struct rte_eth_dev *dev);
diff --git a/drivers/net/sxe2/sxe2_txrx_vec_common.h b/drivers/net/sxe2/sxe2_txrx_vec_common.h
index 6b1649c390..c093c3c80c 100644
--- a/drivers/net/sxe2/sxe2_txrx_vec_common.h
+++ b/drivers/net/sxe2/sxe2_txrx_vec_common.h
@@ -2,6 +2,7 @@
* Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
*/
+
#ifndef SXE2_TXRX_VEC_COMMON_H
#define SXE2_TXRX_VEC_COMMON_H
diff --git a/drivers/net/sxe2/sxe2_txrx_vec_neon.c b/drivers/net/sxe2/sxe2_txrx_vec_neon.c
new file mode 100644
index 0000000000..26d3bef21a
--- /dev/null
+++ b/drivers/net/sxe2/sxe2_txrx_vec_neon.c
@@ -0,0 +1,721 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
+ */
+
+#ifdef RTE_ARCH_ARM64
+#include <arm_neon.h>
+#include <rte_vect.h>
+
+#include "sxe2_txrx_vec_common.h"
+#include "sxe2_txrx_vec.h"
+#include "sxe2_common_log.h"
+
+#define PKTLEN_SHIFT 10
+#define SXE2_UINT16_BIT (CHAR_BIT * sizeof(uint16_t))
+
+static __rte_always_inline void
+sxe2_tx_desc_fill_one_neon(volatile union sxe2_tx_data_desc *desc,
+ struct rte_mbuf *pkt, uint64_t desc_cmd, bool with_offloads)
+{
+ uint64_t desc_qw1;
+ uint32_t desc_offset;
+
+ desc_qw1 = (SXE2_TX_DESC_DTYPE_DATA |
+ ((uint64_t)desc_cmd) << SXE2_TX_DATA_DESC_CMD_SHIFT |
+ ((uint64_t)pkt->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
+
+ desc_offset = SXE2_TX_DATA_DESC_MACLEN_VAL(pkt->l2_len);
+ desc_qw1 |= ((uint64_t)desc_offset) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
+ if (with_offloads)
+ sxe2_tx_desc_fill_offloads(pkt, &desc_qw1);
+
+ uint64x2_t data_desc = { rte_pktmbuf_iova(pkt), desc_qw1 };
+
+ vst1q_u64(RTE_CAST_PTR(uint64_t *, desc), data_desc);
+}
+
+static __rte_always_inline uint16_t
+sxe2_tx_pkts_vec_neon_batch(struct sxe2_tx_queue *txq, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts, bool with_offloads)
+{
+ volatile union sxe2_tx_data_desc *desc;
+ struct sxe2_tx_buffer *buffer;
+ uint16_t next_use;
+ uint16_t res_num;
+ uint16_t tx_num;
+ uint16_t i;
+
+ if (txq->desc_free_num < txq->free_thresh)
+ (void)sxe2_tx_bufs_free_vec(txq);
+
+ nb_pkts = RTE_MIN(txq->desc_free_num, nb_pkts);
+ if (unlikely(nb_pkts == 0)) {
+ PMD_LOG_DEBUG(TX, "Tx pkts neon batch: may not enough free desc, "
+ "free_desc=%u, need_tx_pkts=%u",
+ txq->desc_free_num, nb_pkts);
+ goto l_end;
+ }
+ tx_num = nb_pkts;
+
+ next_use = txq->next_use;
+ desc = &txq->desc_ring[next_use];
+ buffer = &txq->buffer_ring[next_use];
+
+ txq->desc_free_num -= nb_pkts;
+
+ res_num = txq->ring_depth - txq->next_use;
+
+ if (tx_num >= res_num) {
+ sxe2_tx_pkts_mbuf_fill(buffer, tx_pkts, res_num);
+
+ for (i = 0; i < res_num - 1; ++i, ++tx_pkts, ++desc) {
+ sxe2_tx_desc_fill_one_neon(desc, *tx_pkts,
+ SXE2_TX_DATA_DESC_CMD_EOP, with_offloads);
+ }
+
+ sxe2_tx_desc_fill_one_neon(desc, *tx_pkts++,
+ (SXE2_TX_DATA_DESC_CMD_EOP | SXE2_TX_DATA_DESC_CMD_RS),
+ with_offloads);
+
+ tx_num -= res_num;
+
+ next_use = 0;
+ txq->next_rs = txq->rs_thresh - 1;
+ desc = &txq->desc_ring[next_use];
+ buffer = &txq->buffer_ring[next_use];
+ }
+
+ sxe2_tx_pkts_mbuf_fill(buffer, tx_pkts, tx_num);
+
+ for (i = 0; i < tx_num; ++i, ++tx_pkts, ++desc) {
+ sxe2_tx_desc_fill_one_neon(desc, *tx_pkts,
+ SXE2_TX_DATA_DESC_CMD_EOP, with_offloads);
+ }
+
+ next_use += tx_num;
+ if (next_use > txq->next_rs) {
+ txq->desc_ring[txq->next_rs].read.type_cmd_off_bsz_l2t |=
+ rte_cpu_to_le_64(SXE2_TX_DATA_DESC_CMD_RS_MASK);
+
+ txq->next_rs += txq->rs_thresh;
+ }
+ txq->next_use = next_use;
+
+ SXE2_PCI_REG_WRITE_WC(txq->tdt_reg_addr, txq->next_use);
+
+l_end:
+ return nb_pkts;
+}
+
+static __rte_always_inline uint16_t
+sxe2_tx_pkts_vec_neon_common(struct sxe2_tx_queue *txq, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts, bool with_offloads)
+{
+ uint16_t tx_done_num = 0;
+ uint16_t tx_once_num;
+ uint16_t tx_need_num;
+
+ while (nb_pkts) {
+ tx_need_num = RTE_MIN(nb_pkts, txq->rs_thresh);
+ tx_once_num = sxe2_tx_pkts_vec_neon_batch(txq,
+ tx_pkts + tx_done_num,
+ tx_need_num, with_offloads);
+
+ nb_pkts -= tx_once_num;
+ tx_done_num += tx_once_num;
+
+ if (tx_once_num < tx_need_num)
+ break;
+ }
+
+ PMD_LOG_DEBUG(TX, "Tx pkts neon: port_id=%u, queue_id=%u, "
+ "nb_pkts=%u, tx_done_num=%u with_offloads=%u",
+ txq->port_id, txq->idx_in_func, nb_pkts, tx_done_num, with_offloads);
+
+ return tx_done_num;
+}
+
+uint16_t sxe2_tx_pkts_vec_neon_simple(void *tx_queue,
+ struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_tx_pkts_vec_neon_common((struct sxe2_tx_queue *)tx_queue,
+ tx_pkts, nb_pkts, false);
+}
+
+uint16_t sxe2_tx_pkts_vec_neon(void *tx_queue,
+ struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+ return sxe2_tx_pkts_vec_neon_common((struct sxe2_tx_queue *)tx_queue,
+ tx_pkts, nb_pkts, true);
+}
+
+static __rte_always_inline void
+sxe2_rx_desc_ptype_fill_neon(uint16x8_t staterr, struct rte_mbuf **__rte_restrict rx_pkts,
+ const uint32_t *__rte_restrict ptype_tbl)
+{
+ uint16x8_t ptype_mask = {
+ 0, 0x3FFULL,
+ 0, 0x3FFULL,
+ 0, 0x3FFULL,
+ 0, 0x3FFULL,
+ };
+ uint16x8_t ptype_all;
+
+ ptype_all = vandq_u16(staterr, ptype_mask);
+
+ rx_pkts[3]->packet_type = ptype_tbl[vgetq_lane_u16(ptype_all, 3)];
+ rx_pkts[2]->packet_type = ptype_tbl[vgetq_lane_u16(ptype_all, 7)];
+ rx_pkts[1]->packet_type = ptype_tbl[vgetq_lane_u16(ptype_all, 1)];
+ rx_pkts[0]->packet_type = ptype_tbl[vgetq_lane_u16(ptype_all, 5)];
+}
+
+static __rte_always_inline uint32x4_t
+sxe2_rx_desc_fnav_flags_neon(uint64x2_t descs_arr[4])
+{
+ uint32x4_t descs_tmp1, descs_tmp2;
+ uint32x4_t descs_fnav_vld;
+ uint32x4_t v_zeros, v_ffff, v_u32_one;
+ uint32x4_t m_flags;
+
+ const uint32x4_t fdir_flags = vdupq_n_u32(RTE_MBUF_F_RX_FDIR |
+ RTE_MBUF_F_RX_FDIR_ID);
+
+ uint32x4_t d0 = vreinterpretq_u32_u64(descs_arr[0]);
+ uint32x4_t d1 = vreinterpretq_u32_u64(descs_arr[1]);
+ uint32x4_t d2 = vreinterpretq_u32_u64(descs_arr[2]);
+ uint32x4_t d3 = vreinterpretq_u32_u64(descs_arr[3]);
+
+ descs_tmp1 = vzip1q_u32(d1, d0);
+ descs_tmp2 = vzip1q_u32(d3, d2);
+
+ uint64x2_t a = vreinterpretq_u64_u32(descs_tmp1);
+ uint64x2_t b = vreinterpretq_u64_u32(descs_tmp2);
+
+ descs_fnav_vld = vreinterpretq_u32_u64(vcombine_u64(vget_low_u64(a), vget_low_u64(b)));
+
+ descs_fnav_vld = vshlq_n_u32(descs_fnav_vld, 26);
+ descs_fnav_vld = vshrq_n_u32(descs_fnav_vld, 31);
+
+ v_zeros = vdupq_n_u32(0);
+ v_ffff = vceqq_u32(v_zeros, v_zeros);
+ v_u32_one = vshrq_n_u32(v_ffff, 31);
+
+ m_flags = vceqq_u32(descs_fnav_vld, v_u32_one);
+
+ m_flags = vandq_u32(m_flags, fdir_flags);
+ return m_flags;
+}
+
+static __rte_always_inline void
+sxe2_rx_desc_offloads_para_fill_neon(struct sxe2_rx_queue *rxq,
+ volatile union sxe2_rx_desc *desc,
+ uint64x2_t descs[4], struct rte_mbuf **rx_pkts)
+{
+ uint32x4_t desc_lo, desc_hi, flags, tmp_flags;
+ const uint64x2_t mbuf_init = {rxq->mbuf_init_value, 0};
+ uint64x2_t rearm0, rearm1, rearm2, rearm3;
+
+ const uint32x4_t desc_msk = {
+ 0x00001C04, 0x00001C04, 0x00001C04, 0x00001C04};
+
+ const uint32x4_t rss_msk = {
+ 0x20000000, 0x20000000, 0x20000000, 0x20000000};
+
+ const uint32x4_t vlan_msk = {
+ RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED,
+ RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED,
+ RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED,
+ RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED
+ };
+ const uint8x16_t vlan_flags = {
+ 0, 0, 0, 0,
+ RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, 0
+ };
+
+ const uint8x16_t rss_flags = {
+ 0, 0, 0, 0,
+ RTE_MBUF_F_RX_RSS_HASH, 0, 0, 0,
+ 0, 0, 0, 0,
+ 0, 0, 0, 0
+ };
+
+ const uint32x4_t cksum_mask = {
+ RTE_MBUF_F_RX_IP_CKSUM_MASK | RTE_MBUF_F_RX_L4_CKSUM_MASK |
+ RTE_MBUF_F_RX_OUTER_L4_CKSUM_MASK | RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD,
+ RTE_MBUF_F_RX_IP_CKSUM_MASK | RTE_MBUF_F_RX_L4_CKSUM_MASK |
+ RTE_MBUF_F_RX_OUTER_L4_CKSUM_MASK | RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD,
+ RTE_MBUF_F_RX_IP_CKSUM_MASK | RTE_MBUF_F_RX_L4_CKSUM_MASK |
+ RTE_MBUF_F_RX_OUTER_L4_CKSUM_MASK | RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD,
+ RTE_MBUF_F_RX_IP_CKSUM_MASK | RTE_MBUF_F_RX_L4_CKSUM_MASK |
+ RTE_MBUF_F_RX_OUTER_L4_CKSUM_MASK | RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD,
+ };
+
+ const uint8x16_t cksum_flags = {
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD | RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_GOOD | RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD | RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_L4_CKSUM_BAD | RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_GOOD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
+ ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD |
+ RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
+ 0, 0, 0, 0, 0, 0, 0, 0
+ };
+
+ {
+ uint32x4_t d0 = vreinterpretq_u32_u64(descs[0]);
+ uint32x4_t d1 = vreinterpretq_u32_u64(descs[1]);
+ uint32x4_t d2 = vreinterpretq_u32_u64(descs[2]);
+ uint32x4_t d3 = vreinterpretq_u32_u64(descs[3]);
+ uint64x2_t f64, t64;
+
+ flags = vzip2q_u32(d1, d0);
+ tmp_flags = vzip2q_u32(d3, d2);
+ f64 = vreinterpretq_u64_u32(flags);
+ t64 = vreinterpretq_u64_u32(tmp_flags);
+ desc_lo = vreinterpretq_u32_u64(vcombine_u64(vget_low_u64(f64),
+ vget_low_u64(t64)));
+ desc_hi = vreinterpretq_u32_u64(vcombine_u64(vget_high_u64(f64),
+ vget_high_u64(t64)));
+ }
+
+ desc_lo = vandq_u32(desc_lo, desc_msk);
+ desc_hi = vandq_u32(desc_hi, rss_msk);
+
+ tmp_flags = vreinterpretq_u32_u8(vqtbl1q_u8(vlan_flags,
+ vreinterpretq_u8_u32(desc_lo)));
+ flags = vandq_u32(tmp_flags, vlan_msk);
+
+ desc_lo = vshrq_n_u32(desc_lo, 10);
+ tmp_flags = vreinterpretq_u32_u8(vqtbl1q_u8(cksum_flags,
+ vreinterpretq_u8_u32(desc_lo)));
+ tmp_flags = vshlq_n_u32(tmp_flags, 1);
+ tmp_flags = vandq_u32(tmp_flags, cksum_mask);
+ flags = vorrq_u32(flags, tmp_flags);
+
+ desc_hi = vshrq_n_u32(desc_hi, 27);
+ tmp_flags = vreinterpretq_u32_u8(vqtbl1q_u8(rss_flags,
+ vreinterpretq_u8_u32(desc_hi)));
+ flags = vorrq_u32(flags, tmp_flags);
+
+#ifndef RTE_LIBRTE_SXE2_16BYTE_RX_DESC
+ if (rxq->fnav_enable) {
+ uint32x4_t tmp_fnav_flags = sxe2_rx_desc_fnav_flags_neon(descs);
+ flags = vorrq_u32(flags, tmp_fnav_flags);
+
+ rx_pkts[0]->hash.fdir.hi = desc[0].wb.fd_filter_id;
+ rx_pkts[1]->hash.fdir.hi = desc[1].wb.fd_filter_id;
+ rx_pkts[2]->hash.fdir.hi = desc[2].wb.fd_filter_id;
+ rx_pkts[3]->hash.fdir.hi = desc[3].wb.fd_filter_id;
+ }
+#endif
+
+ rearm0 = vsetq_lane_u64(vgetq_lane_u32(flags, 0), mbuf_init, 1);
+ rearm1 = vsetq_lane_u64(vgetq_lane_u32(flags, 1), mbuf_init, 1);
+ rearm2 = vsetq_lane_u64(vgetq_lane_u32(flags, 2), mbuf_init, 1);
+ rearm3 = vsetq_lane_u64(vgetq_lane_u32(flags, 3), mbuf_init, 1);
+
+ vst1q_u64((uint64_t *)&rx_pkts[0]->rearm_data, rearm0);
+ vst1q_u64((uint64_t *)&rx_pkts[1]->rearm_data, rearm1);
+ vst1q_u64((uint64_t *)&rx_pkts[2]->rearm_data, rearm2);
+ vst1q_u64((uint64_t *)&rx_pkts[3]->rearm_data, rearm3);
+}
+
+static inline void sxe2_rx_queue_rearm_neon(struct sxe2_rx_queue *rxq)
+{
+ volatile union sxe2_rx_desc *desc;
+ struct rte_mbuf **buffer;
+ struct rte_mbuf *mbuf0, *mbuf1;
+ uint64x2_t dma_addr0, dma_addr1;
+ uint64x2_t zero = vdupq_n_u64(0);
+ uint64x2_t virt_addr0, virt_addr1;
+ uint64x2_t hdr_room = vdupq_n_u64(RTE_PKTMBUF_HEADROOM);
+ int32_t ret;
+ uint16_t i;
+ uint16_t new_tail;
+
+ buffer = &rxq->buffer_ring[rxq->realloc_start];
+ desc = &rxq->desc_ring[rxq->realloc_start];
+
+ ret = rte_mempool_get_bulk(rxq->mb_pool, (void *)buffer,
+ SXE2_RX_REARM_THRESH_VEC);
+ if (ret != 0) {
+ PMD_LOG_INFO(RX, "Rx mbuf vec alloc failed port_id=%u "
+ "queue_id=%u", rxq->port_id, rxq->idx_in_func);
+
+ if ((rxq->realloc_num + SXE2_RX_REARM_THRESH_VEC) >= rxq->ring_depth) {
+ for (i = 0; i < SXE2_RX_NUM_PER_LOOP_NEON; ++i) {
+ buffer[i] = &rxq->fake_mbuf;
+ vst1q_u64(RTE_CAST_PTR(uint64_t *, &desc[i].read), zero);
+ }
+ }
+
+ rxq->vsi->adapter->dev_info.dev_data->rx_mbuf_alloc_failed +=
+ SXE2_RX_REARM_THRESH_VEC;
+ goto l_end;
+ }
+
+ for (i = 0; i < SXE2_RX_REARM_THRESH_VEC; i += 2, buffer += 2) {
+ mbuf0 = buffer[0];
+ mbuf1 = buffer[1];
+#if RTE_IOVA_IN_MBUF
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) !=
+ offsetof(struct rte_mbuf, buf_addr) + 8);
+#endif
+ virt_addr0 = vld1q_u64((uint64_t *)&mbuf0->buf_addr);
+ virt_addr1 = vld1q_u64((uint64_t *)&mbuf1->buf_addr);
+
+#if RTE_IOVA_IN_MBUF
+ dma_addr0 = vdupq_n_u64((uint64_t)vget_high_u64(virt_addr0));
+ dma_addr1 = vdupq_n_u64((uint64_t)vget_high_u64(virt_addr1));
+#else
+ dma_addr0 = vdupq_n_u64((uint64_t)vget_low_u64(virt_addr0));
+ dma_addr1 = vdupq_n_u64((uint64_t)vget_low_u64(virt_addr1));
+#endif
+ dma_addr0 = vaddq_u64(dma_addr0, hdr_room);
+ dma_addr1 = vaddq_u64(dma_addr1, hdr_room);
+
+ vst1q_u64(RTE_CAST_PTR(uint64_t *, &desc++->read), dma_addr0);
+ vst1q_u64(RTE_CAST_PTR(uint64_t *, &desc++->read), dma_addr1);
+ }
+
+ rxq->realloc_start += SXE2_RX_REARM_THRESH_VEC;
+ if (rxq->realloc_start >= rxq->ring_depth)
+ rxq->realloc_start = 0;
+ rxq->realloc_num -= SXE2_RX_REARM_THRESH_VEC;
+
+ new_tail = (rxq->realloc_start == 0) ?
+ (rxq->ring_depth - 1) : (rxq->realloc_start - 1);
+
+ SXE2_PCI_REG_WRITE_WC(rxq->rdt_reg_addr, new_tail);
+
+l_end:
+ return;
+}
+
+static __rte_always_inline uint16_t
+sxe2_rx_pkts_common_vec_neon(struct sxe2_rx_queue *rxq, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts, uint8_t *split_rxe_flags, uint8_t *umbcast_flags,
+ bool do_offload)
+{
+ volatile union sxe2_rx_desc *desc;
+ struct rte_mbuf **buffer;
+ uint32_t i;
+ uint16_t done_num = 0;
+ const uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl;
+
+ uint8x16_t rvp_shuf_mask = {
+ 0xFF, 0xFF, 0xFF, 0xFF,
+ 12, 13, 0xFF, 0xFF,
+ 12, 13,
+ 2, 3,
+ 4, 5, 6, 7
+ };
+
+ uint16x8_t crc_adjust = {
+ 0, 0,
+ rxq->crc_len,
+ 0, rxq->crc_len,
+ 0, 0, 0
+ };
+
+ desc = &rxq->desc_ring[rxq->processing_idx];
+ rte_prefetch0(desc);
+
+ nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, SXE2_RX_NUM_PER_LOOP_NEON);
+
+ if (rxq->realloc_num > SXE2_RX_REARM_THRESH_VEC)
+ sxe2_rx_queue_rearm_neon(rxq);
+
+ if ((rte_le_to_cpu_64(desc->wb.status_err_ptype_len) &
+ SXE2_RX_DESC_STATUS_DD_MASK) == 0) {
+ goto l_end;
+ }
+
+ buffer = &rxq->buffer_ring[rxq->processing_idx];
+ for (i = 0; i < nb_pkts; i += SXE2_RX_NUM_PER_LOOP_NEON,
+ desc += SXE2_RX_NUM_PER_LOOP_NEON) {
+ uint64x2_t descs[SXE2_RX_NUM_PER_LOOP_NEON];
+ uint8x16_t pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4;
+ uint64x2_t mbp1, mbp2;
+ uint16x8_t staterr;
+ uint16x8_t tmp;
+ uint16_t bit_num;
+
+ descs[3] = vld1q_u64(RTE_CAST_PTR(uint64_t *, desc + 3));
+ rte_atomic_thread_fence(rte_memory_order_acquire);
+ descs[2] = vld1q_u64(RTE_CAST_PTR(uint64_t *, desc + 2));
+ rte_atomic_thread_fence(rte_memory_order_acquire);
+ descs[1] = vld1q_u64(RTE_CAST_PTR(uint64_t *, desc + 1));
+ rte_atomic_thread_fence(rte_memory_order_acquire);
+ descs[0] = vld1q_u64(RTE_CAST_PTR(uint64_t *, desc));
+
+ rte_atomic_thread_fence(rte_memory_order_acquire);
+
+ descs[3] = vld1q_lane_u64(RTE_CAST_PTR(uint64_t *, desc + 3), descs[3], 0);
+ descs[2] = vld1q_lane_u64(RTE_CAST_PTR(uint64_t *, desc + 2), descs[2], 0);
+ descs[1] = vld1q_lane_u64(RTE_CAST_PTR(uint64_t *, desc + 1), descs[1], 0);
+ descs[0] = vld1q_lane_u64(RTE_CAST_PTR(uint64_t *, desc), descs[0], 0);
+
+ mbp1 = vld1q_u64((uint64_t *)&buffer[i]);
+ mbp2 = vld1q_u64((uint64_t *)&buffer[i + 2]);
+
+ vst1q_u64((uint64_t *)&rx_pkts[i], mbp1);
+ vst1q_u64((uint64_t *)&rx_pkts[i + 2], mbp2);
+
+ if (split_rxe_flags) {
+ rte_mbuf_prefetch_part2(rx_pkts[i]);
+ rte_mbuf_prefetch_part2(rx_pkts[i + 1]);
+ rte_mbuf_prefetch_part2(rx_pkts[i + 2]);
+ rte_mbuf_prefetch_part2(rx_pkts[i + 3]);
+ }
+
+ pkt_mb4 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[3]), rvp_shuf_mask);
+ pkt_mb3 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[2]), rvp_shuf_mask);
+ pkt_mb2 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[1]), rvp_shuf_mask);
+ pkt_mb1 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[0]), rvp_shuf_mask);
+
+ if (do_offload) {
+ sxe2_rx_desc_offloads_para_fill_neon(rxq, desc, descs, &rx_pkts[i]);
+ } else {
+ const uint64x2_t mbuf_init = {
+ rxq->mbuf_init_value,
+ 0,
+ };
+
+ vst1q_u64((uint64_t *)&rx_pkts[i]->rearm_data, mbuf_init);
+ vst1q_u64((uint64_t *)&rx_pkts[i + 1]->rearm_data, mbuf_init);
+ vst1q_u64((uint64_t *)&rx_pkts[i + 2]->rearm_data, mbuf_init);
+ vst1q_u64((uint64_t *)&rx_pkts[i + 3]->rearm_data, mbuf_init);
+ }
+
+ tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb4), crc_adjust);
+ pkt_mb4 = vreinterpretq_u8_u16(tmp);
+ tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb3), crc_adjust);
+ pkt_mb3 = vreinterpretq_u8_u16(tmp);
+ tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb2), crc_adjust);
+ pkt_mb2 = vreinterpretq_u8_u16(tmp);
+ tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb1), crc_adjust);
+ pkt_mb1 = vreinterpretq_u8_u16(tmp);
+
+ vst1q_u8((void *)&rx_pkts[i + 3]->rx_descriptor_fields1,
+ pkt_mb4);
+ vst1q_u8((void *)&rx_pkts[i + 2]->rx_descriptor_fields1,
+ pkt_mb3);
+ vst1q_u8((void *)&rx_pkts[i + 1]->rx_descriptor_fields1,
+ pkt_mb2);
+ vst1q_u8((void *)&rx_pkts[i]->rx_descriptor_fields1,
+ pkt_mb1);
+
+ if (likely(i + SXE2_RX_NUM_PER_LOOP_NEON < nb_pkts))
+ rte_prefetch_non_temporal(desc + SXE2_RX_NUM_PER_LOOP_NEON);
+
+ {
+ uint32x4_t d0 = vreinterpretq_u32_u64(descs[0]);
+ uint32x4_t d1 = vreinterpretq_u32_u64(descs[1]);
+ uint32x4_t d2 = vreinterpretq_u32_u64(descs[2]);
+ uint32x4_t d3 = vreinterpretq_u32_u64(descs[3]);
+ uint32x4_t sterr_tmp1 = vzip2q_u32(d1, d0);
+ uint32x4_t sterr_tmp2 = vzip2q_u32(d3, d2);
+ uint32x4_t sterr_u32 = vzip1q_u32(sterr_tmp1, sterr_tmp2);
+
+ staterr = vreinterpretq_u16_u32(sterr_u32);
+ }
+
+ sxe2_rx_desc_ptype_fill_neon(staterr, &rx_pkts[i], ptype_tbl);
+
+ if (umbcast_flags != NULL) {
+ uint32x4_t umbcast_mask = {
+ SXE2_RX_DESC_STATUS_UMBCAST_MASK, SXE2_RX_DESC_STATUS_UMBCAST_MASK,
+ SXE2_RX_DESC_STATUS_UMBCAST_MASK, SXE2_RX_DESC_STATUS_UMBCAST_MASK,
+ };
+
+ uint8x16_t umbcast_shuf_mask = {
+ 0x0B, 0x03, 0x0F, 0x07,
+ 0xFF, 0xFF, 0xFF, 0xFF,
+ 0xFF, 0xFF, 0xFF, 0xFF,
+ 0xFF, 0xFF, 0xFF, 0xFF,
+ };
+ uint8x16_t umbcast_bits =
+ vreinterpretq_u8_u32(vandq_u32(vreinterpretq_u32_u16(staterr),
+ umbcast_mask));
+
+ umbcast_bits = vqtbl1q_u8(umbcast_bits, umbcast_shuf_mask);
+ vst1q_lane_u32((uint32_t *)umbcast_flags,
+ vreinterpretq_u32_u8(umbcast_bits), 0);
+ umbcast_flags += SXE2_RX_NUM_PER_LOOP_NEON;
+ }
+
+ if (split_rxe_flags) {
+ uint8x16_t eop_shuf_mask = {
+ 0x08, 0x00, 0x0C, 0x04,
+ 0xFF, 0xFF, 0xFF, 0xFF,
+ 0xFF, 0xFF, 0xFF, 0xFF,
+ 0xFF, 0xFF, 0xFF, 0xFF};
+ uint8x16_t eop_bits;
+ uint32x4_t rxe_mask = {
+ 0x2080, 0x2080, 0x2080, 0x2080
+ };
+ uint32x4_t rxe_bits;
+ uint32x4_t eop_mask;
+
+ eop_mask = vshlq_n_u32(vdupq_n_u32(1), SXE2_RX_DESC_STATUS_EOP_SHIFT);
+ eop_bits = vandq_u8(vmvnq_u8(vreinterpretq_u8_u16(staterr)),
+ vreinterpretq_u8_u32(eop_mask));
+
+ rxe_bits = vandq_u32(vreinterpretq_u32_u16(staterr), rxe_mask);
+ rxe_bits = vshrq_n_u32(rxe_bits, 7);
+
+ eop_bits = vorrq_u8(eop_bits, vreinterpretq_u8_u32(rxe_bits));
+
+ eop_bits = vqtbl1q_u8(eop_bits, eop_shuf_mask);
+
+ vst1q_lane_u32((uint32_t *)split_rxe_flags,
+ vreinterpretq_u32_u8(eop_bits), 0);
+ split_rxe_flags += SXE2_RX_NUM_PER_LOOP_NEON;
+
+#ifdef RTE_IOVA_IN_MBUF
+ rx_pkts[i]->next = NULL;
+ rx_pkts[i + 1]->next = NULL;
+ rx_pkts[i + 2]->next = NULL;
+ rx_pkts[i + 3]->next = NULL;
+#endif
+ }
+
+ {
+ uint32x4_t dd_mask = vdupq_n_u32(1);
+ uint32x4_t sterr_dd = vandq_u32(vreinterpretq_u32_u16(staterr), dd_mask);
+ uint16x4_t packed_lo = vmovn_u32(sterr_dd);
+ uint64_t dd64 = vget_lane_u64(vreinterpret_u64_u16(packed_lo), 0);
+
+ bit_num = (uint16_t)rte_popcount64(dd64);
+ }
+ done_num += bit_num;
+ if (likely(bit_num != SXE2_RX_NUM_PER_LOOP_NEON))
+ break;
+ }
+
+ rxq->processing_idx += done_num;
+ rxq->processing_idx &= (rxq->ring_depth - 1);
+ rxq->realloc_num += done_num;
+
+l_end:
+ return done_num;
+}
+
+static __rte_always_inline uint16_t
+sxe2_rx_pkts_scattered_batch_vec_neon(struct sxe2_rx_queue *rxq,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts, bool do_offload)
+{
+ const uint64_t *split_flags64;
+ uint8_t split_rxe_flags[SXE2_RX_PKTS_BURST_BATCH_NUM_VEC] = {0};
+ uint8_t umbcast_flags[SXE2_RX_PKTS_BURST_BATCH_NUM_VEC] = {0};
+ uint16_t rx_done_num;
+ uint16_t rx_pkt_done_num;
+
+ rx_pkt_done_num = 0;
+
+ if (rxq->vsi->adapter->devargs.sw_stats_en) {
+ rx_done_num = sxe2_rx_pkts_common_vec_neon((struct sxe2_rx_queue *)rxq,
+ rx_pkts, nb_pkts, split_rxe_flags, umbcast_flags,
+ do_offload);
+ } else {
+ rx_done_num = sxe2_rx_pkts_common_vec_neon((struct sxe2_rx_queue *)rxq,
+ rx_pkts, nb_pkts, split_rxe_flags, NULL,
+ do_offload);
+ }
+
+ if (rx_done_num == 0)
+ goto l_end;
+
+ if (!rxq->vsi->adapter->devargs.sw_stats_en) {
+ split_flags64 = (uint64_t *)split_rxe_flags;
+
+ if (rxq->pkt_first_seg == NULL &&
+ split_flags64[0] == 0 && split_flags64[1] == 0 &&
+ split_flags64[2] == 0 && split_flags64[3] == 0) {
+ rx_pkt_done_num = rx_done_num;
+ goto l_end;
+ }
+
+ if (rxq->pkt_first_seg == NULL) {
+ while (rx_pkt_done_num < rx_done_num &&
+ split_rxe_flags[rx_pkt_done_num] == 0) {
+ rx_pkt_done_num++;
+ }
+
+ if (rx_pkt_done_num == rx_done_num)
+ goto l_end;
+
+ rxq->pkt_first_seg = rx_pkts[rx_pkt_done_num];
+ }
+ }
+
+ rx_pkt_done_num += sxe2_rx_pkts_refactor(rxq, &rx_pkts[rx_pkt_done_num],
+ rx_done_num - rx_pkt_done_num, &split_rxe_flags[rx_pkt_done_num],
+ &umbcast_flags[rx_pkt_done_num]);
+
+l_end:
+ return rx_pkt_done_num;
+}
+
+uint16_t sxe2_rx_pkts_scattered_vec_neon_offload(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+ uint16_t done_num = 0;
+ uint16_t once_num;
+
+ while (nb_pkts > SXE2_RX_PKTS_BURST_BATCH_NUM_VEC) {
+ once_num = sxe2_rx_pkts_scattered_batch_vec_neon((struct sxe2_rx_queue *)rx_queue,
+ rx_pkts + done_num,
+ SXE2_RX_PKTS_BURST_BATCH_NUM_VEC,
+ true);
+
+ done_num += once_num;
+ nb_pkts -= once_num;
+
+ if (once_num < SXE2_RX_PKTS_BURST_BATCH_NUM_VEC)
+ goto l_end;
+ }
+
+ done_num += sxe2_rx_pkts_scattered_batch_vec_neon((struct sxe2_rx_queue *)rx_queue,
+ rx_pkts + done_num,
+ nb_pkts,
+ true);
+l_end:
+ return done_num;
+}
+
+uint16_t sxe2_rx_pkts_scattered_vec_neon(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+ uint16_t done_num = 0;
+ uint16_t once_num;
+
+ while (nb_pkts > SXE2_RX_PKTS_BURST_BATCH_NUM_VEC) {
+ once_num = sxe2_rx_pkts_scattered_batch_vec_neon((struct sxe2_rx_queue *)rx_queue,
+ rx_pkts + done_num,
+ SXE2_RX_PKTS_BURST_BATCH_NUM_VEC,
+ false);
+
+ done_num += once_num;
+ nb_pkts -= once_num;
+
+ if (once_num < SXE2_RX_PKTS_BURST_BATCH_NUM_VEC)
+ goto l_end;
+ }
+
+ done_num += sxe2_rx_pkts_scattered_batch_vec_neon((struct sxe2_rx_queue *)rx_queue,
+ rx_pkts + done_num,
+ nb_pkts,
+ false);
+l_end:
+ return done_num;
+}
+#endif
--
2.52.0
More information about the dev
mailing list