[PATCH 7/7] net/bnxt: fix cache thrashing for vector Rx and Tx paths

[Mohammad Shuab Siddique]

From: Keegan Freyhof <keegan.freyhof at broadcom.com>

On Thor2 with 256 byte packets, Rx throughput dropped by about 60
percent. The AVX2 v3 Rx burst function rearmed the queue at the start
of a burst once rx_free_thresh buffers were pending, so a large bulk
of mbufs was fetched at once and pushed the Rx working set out of the
L1 cache.

Rearm once at the end of every Rx call instead, so that mbufs are
fetched in smaller batches as packets are received. Remove the
unlikely() hint on the early return in bnxt_rxq_rearm(), since it is
now taken on most calls.

Also bound the Tx completion loop in the vector Tx burst function by
the number of packets requested, so that a single call does not
reclaim an unbounded number of completions.

Fixes: ae347ed1b27d ("net/bnxt: add AVX2 Rx for Thor2")
Cc: stable at dpdk.org

Signed-off-by: Keegan Freyhof <keegan.freyhof at broadcom.com>
Signed-off-by: Mohammad Shuab Siddique <mohammad-shuab.siddique at broadcom.com>
---
 drivers/net/bnxt/bnxt_rxtx_vec_avx2.c   | 16 ++++++++++------
 drivers/net/bnxt/bnxt_rxtx_vec_common.h |  2 +-
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/drivers/net/bnxt/bnxt_rxtx_vec_avx2.c b/drivers/net/bnxt/bnxt_rxtx_vec_avx2.c
index ee14fe2c5d..aee0d24d2b 100644
--- a/drivers/net/bnxt/bnxt_rxtx_vec_avx2.c
+++ b/drivers/net/bnxt/bnxt_rxtx_vec_avx2.c
@@ -831,7 +831,7 @@ bnxt_crx_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,
 }
 
 static void
-bnxt_handle_tx_cp_vec(struct bnxt_tx_queue *txq)
+bnxt_handle_tx_cp_vec(struct bnxt_tx_queue *txq, uint16_t nb_pkts)
 {
 	struct bnxt_cp_ring_info *cpr = txq->cp_ring;
 	uint32_t raw_cons = cpr->cp_raw_cons;
@@ -869,7 +869,7 @@ bnxt_handle_tx_cp_vec(struct bnxt_tx_queue *txq)
 			nb_tx_pkts += txcmp->opaque;
 		}
 		raw_cons = NEXT_RAW_CMP(raw_cons);
-	} while (nb_tx_pkts < ring_mask);
+	} while (nb_tx_pkts < nb_pkts && nb_tx_pkts < ring_mask);
 
 	if (nb_tx_pkts) {
 		if (txq->offloads & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE)
@@ -1025,7 +1025,7 @@ bnxt_xmit_pkts_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts,
 
 	/* Handle TX completions */
 	if (bnxt_tx_bds_in_hw(txq) >= txq->tx_free_thresh)
-		bnxt_handle_tx_cp_vec(txq);
+		bnxt_handle_tx_cp_vec(txq, nb_pkts);
 
 	while (nb_pkts) {
 		uint16_t ret, num;
@@ -1369,9 +1369,6 @@ bnxt_recv_pkts_vec_avx2_v3(void *rx_queue, struct rte_mbuf **rx_pkts,
 	while (nb_pkts) {
 		uint32_t cons, mbcons, valid_bit, num, num_vec, received = 0;
 
-		if (rxq->rxrearm_nb >= rxq->rx_free_thresh)
-			bnxt_rxq_rearm(rxq, rxr);
-
 		valid_bit = !!(cpr->cp_raw_cons & cp_ring_size);
 		cons = cpr->cp_raw_cons & (cp_ring_size - 1);
 		mbcons = (cpr->cp_raw_cons / 2) & (rx_ring_size - 1);
@@ -1416,5 +1413,12 @@ bnxt_recv_pkts_vec_avx2_v3(void *rx_queue, struct rte_mbuf **rx_pkts,
 			break;
 	}
 
+	/*
+	 * Rearm after every burst so that mbufs are fetched in small
+	 * batches rather than in one large bulk at rx_free_thresh, which
+	 * evicts the Rx working set from the L1 cache.
+	 */
+	bnxt_rxq_rearm(rxq, rxr);
+
 	return cnt;
 }
diff --git a/drivers/net/bnxt/bnxt_rxtx_vec_common.h b/drivers/net/bnxt/bnxt_rxtx_vec_common.h
index 37aa9fb854..bc387d333f 100644
--- a/drivers/net/bnxt/bnxt_rxtx_vec_common.h
+++ b/drivers/net/bnxt/bnxt_rxtx_vec_common.h
@@ -67,7 +67,7 @@ bnxt_rxq_rearm(struct bnxt_rx_queue *rxq, struct bnxt_rx_ring_info *rxr)
 	 */
 	nb = RTE_MIN(rxq->rxrearm_nb & ~0x3,
 		     rxq->nb_rx_desc - rxq->rxrearm_start);
-	if (unlikely(nb == 0))
+	if (nb == 0)
 		return;
 
 	/* Allocate new mbufs into the software ring. */
-- 
2.47.3