[PATCH v4 01/29] bpf: replace deprecated SMP barriers with C11 fences

[Stephen Hemminger]

Convert the rx/tx callback use counter from rte_smp_mb() and
rte_smp_rmb() to C11 atomics.

The counter has a single writer per queue (MT_LOCKFREE tx is not
handled, before or after), so use a relaxed load and store instead
of a locked increment. Keep the seq_cst fence in inuse and wait for
the counter/cb store-load pair. Use a release store in unuse paired
with acquire loads in wait, so rte_bpf_destroy() is ordered after
the last datapath use. Drop the extra fence in unload.

With enable_stdatomic, x86 now does one locked instruction per burst
instead of three.

Signed-off-by: Stephen Hemminger <stephen at networkplumber.org>
---
 lib/bpf/bpf_pkt.c | 26 +++++++++++++++-----------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/lib/bpf/bpf_pkt.c b/lib/bpf/bpf_pkt.c
index f072fdaaed..f6acc69d25 100644
--- a/lib/bpf/bpf_pkt.c
+++ b/lib/bpf/bpf_pkt.c
@@ -80,9 +80,13 @@ static struct bpf_eth_cbh tx_cbh = {
 static __rte_always_inline void
 bpf_eth_cbi_inuse(struct bpf_eth_cbi *cbi)
 {
-	cbi->use++;
-	/* make sure no store/load reordering could happen */
-	rte_smp_mb();
+	uint32_t puse = rte_atomic_load_explicit(&cbi->use, rte_memory_order_relaxed);
+
+	/* increment use count: does not need to be locked */
+	rte_atomic_store_explicit(&cbi->use, puse + 1, rte_memory_order_relaxed);
+
+	/* full barrier: count must be visible before cb is read */
+	rte_atomic_thread_fence(rte_memory_order_seq_cst);
 }
 
 /*
@@ -91,9 +95,10 @@ bpf_eth_cbi_inuse(struct bpf_eth_cbi *cbi)
 static __rte_always_inline void
 bpf_eth_cbi_unuse(struct bpf_eth_cbi *cbi)
 {
-	/* make sure all previous loads are completed */
-	rte_smp_rmb();
-	cbi->use++;
+	uint32_t puse =	rte_atomic_load_explicit(&cbi->use, rte_memory_order_relaxed);
+
+	/* release: pairs with the acquire bpf_eth_cbi_wait() */
+	rte_atomic_store_explicit(&cbi->use, puse + 1, rte_memory_order_release);
 }
 
 /*
@@ -104,15 +109,15 @@ bpf_eth_cbi_wait(const struct bpf_eth_cbi *cbi)
 {
 	uint32_t puse;
 
-	/* make sure all previous loads and stores are completed */
-	rte_smp_mb();
+	/* full barrier: cleared cb must be visible before counter is read */
+	rte_atomic_thread_fence(rte_memory_order_seq_cst);
 
-	puse = cbi->use;
+	puse = rte_atomic_load_explicit(&cbi->use, rte_memory_order_acquire);
 
 	/* in use, busy wait till current RX/TX iteration is finished */
 	if ((puse & BPF_ETH_CBI_INUSE) != 0) {
 		RTE_WAIT_UNTIL_MASKED((__rte_atomic uint32_t *)(uintptr_t)&cbi->use,
-			UINT32_MAX, !=, puse, rte_memory_order_relaxed);
+			UINT32_MAX, !=, puse, rte_memory_order_acquire);
 	}
 }
 
@@ -439,7 +444,6 @@ bpf_eth_cbi_unload(struct bpf_eth_cbi *bc)
 {
 	/* mark this cbi as empty */
 	bc->cb = NULL;
-	rte_smp_mb();
 
 	/* make sure datapath doesn't use bpf anymore, then destroy bpf */
 	bpf_eth_cbi_wait(bc);
-- 
2.53.0