[PATCH v2 11/33] stack: always use C11 memory model implementation

[Stephen Hemminger]

The generic and C11 lock-free stack implementations differ only in
memory ordering. The generic version uses a full barrier where its
own comments state an acquire fence is sufficient, and seq_cst for
all length counter operations.

Only x86 and ThunderX still used the generic version. On x86 the
switch removes a locked add per CAS attempt in push and pop; TSO
provides the acquire semantics. On ThunderX the pop fence weakens
from dmb ish to dmb ishld and the push fence goes away.

Measured on a 32-core x86 machine, stack_perf_autotest, cycles per
operation, main versus the unified C11 version (n=9 each):

  Test              main          unified C11   delta
  single push/pop   46.62 +-0.30  33.41 +-0.10  -28%
  empty pop          1.47 +-0.01   0.98 +-0.01  -33%
  1 lcore, bulk 8    9.06 +-0.05   8.20 +-0.08  -10%
  1 lcore, bulk 32   6.09 +-0.02   6.15 +-0.03   +1%
  2 HT, bulk 8      42.05 +-0.31  39.24 +-0.52   -7%
  2 HT, bulk 32     11.92 +-0.13  11.89 +-0.10    0
  2 cores, bulk 8   78.90 +-0.60  72.96 +-1.11   -7%
  2 cores, bulk 32  20.74 +-1.56   7.70 +-0.13  -63%
  32 cores, bulk 8  6126 +-72     6121 +-89       0
  32 cores, bulk 32 1953.9 +-2.9  1984.6 +-13.3 +1.6%

The C11 version is faster because it emits no lock prefixed
instructions.

Remove the generic version and use the C11 implementation everywhere.

Signed-off-by: Stephen Hemminger <stephen at networkplumber.org>
Acked-by: Morten Brørup <mb at smartsharesystems.com>
---
 lib/stack/meson.build            |   1 -
 lib/stack/rte_stack_lf.h         |   4 -
 lib/stack/rte_stack_lf_generic.h | 153 -------------------------------
 3 files changed, 158 deletions(-)
 delete mode 100644 lib/stack/rte_stack_lf_generic.h

diff --git a/lib/stack/meson.build b/lib/stack/meson.build
index 18177a742f..1fab46208f 100644
--- a/lib/stack/meson.build
+++ b/lib/stack/meson.build
@@ -7,7 +7,6 @@ headers = files('rte_stack.h')
 indirect_headers += files(
         'rte_stack_std.h',
         'rte_stack_lf.h',
-        'rte_stack_lf_generic.h',
         'rte_stack_lf_c11.h',
         'rte_stack_lf_stubs.h',
 )
diff --git a/lib/stack/rte_stack_lf.h b/lib/stack/rte_stack_lf.h
index f2b012cd0e..1bc6ee8f40 100644
--- a/lib/stack/rte_stack_lf.h
+++ b/lib/stack/rte_stack_lf.h
@@ -8,11 +8,7 @@
 #if !(defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_ARM64))
 #include "rte_stack_lf_stubs.h"
 #else
-#ifdef RTE_USE_C11_MEM_MODEL
 #include "rte_stack_lf_c11.h"
-#else
-#include "rte_stack_lf_generic.h"
-#endif
 
 /**
  * Indicates that RTE_STACK_F_LF is supported.
diff --git a/lib/stack/rte_stack_lf_generic.h b/lib/stack/rte_stack_lf_generic.h
deleted file mode 100644
index cc69e4d168..0000000000
--- a/lib/stack/rte_stack_lf_generic.h
+++ /dev/null
@@ -1,153 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2019 Intel Corporation
- */
-
-#ifndef _RTE_STACK_LF_GENERIC_H_
-#define _RTE_STACK_LF_GENERIC_H_
-
-#include <rte_branch_prediction.h>
-#include <rte_prefetch.h>
-
-static __rte_always_inline unsigned int
-__rte_stack_lf_count(struct rte_stack *s)
-{
-	/* stack_lf_push() and stack_lf_pop() do not update the list's contents
-	 * and stack_lf->len atomically, which can cause the list to appear
-	 * shorter than it actually is if this function is called while other
-	 * threads are modifying the list.
-	 *
-	 * However, given the inherently approximate nature of the get_count
-	 * callback -- even if the list and its size were updated atomically,
-	 * the size could change between when get_count executes and when the
-	 * value is returned to the caller -- this is acceptable.
-	 *
-	 * The stack_lf->len updates are placed such that the list may appear to
-	 * have fewer elements than it does, but will never appear to have more
-	 * elements. If the mempool is near-empty to the point that this is a
-	 * concern, the user should consider increasing the mempool size.
-	 */
-	/* NOTE: review for potential ordering optimization */
-	return rte_atomic_load_explicit(&s->stack_lf.used.len, rte_memory_order_seq_cst);
-}
-
-static __rte_always_inline void
-__rte_stack_lf_push_elems(struct rte_stack_lf_list *list,
-			  struct rte_stack_lf_elem *first,
-			  struct rte_stack_lf_elem *last,
-			  unsigned int num)
-{
-	struct rte_stack_lf_head old_head;
-	int success;
-
-	old_head = list->head;
-
-	do {
-		struct rte_stack_lf_head new_head;
-
-		/* An acquire fence (or stronger) is needed for weak memory
-		 * models to establish a synchronized-with relationship between
-		 * the list->head load and store-release operations (as part of
-		 * the rte_atomic128_cmp_exchange()).
-		 */
-		rte_smp_mb();
-
-		/* Swing the top pointer to the first element in the list and
-		 * make the last element point to the old top.
-		 */
-		new_head.top = first;
-		new_head.cnt = old_head.cnt + 1;
-
-		last->next = old_head.top;
-
-		/* old_head is updated on failure */
-		success = rte_atomic128_cmp_exchange(
-				(rte_int128_t *)&list->head,
-				(rte_int128_t *)&old_head,
-				(rte_int128_t *)&new_head,
-				1, rte_memory_order_release,
-				rte_memory_order_relaxed);
-	} while (success == 0);
-	/* NOTE: review for potential ordering optimization */
-	rte_atomic_fetch_add_explicit(&list->len, num, rte_memory_order_seq_cst);
-}
-
-static __rte_always_inline struct rte_stack_lf_elem *
-__rte_stack_lf_pop_elems(struct rte_stack_lf_list *list,
-			 unsigned int num,
-			 void **obj_table,
-			 struct rte_stack_lf_elem **last)
-{
-	struct rte_stack_lf_head old_head;
-	int success = 0;
-
-	/* Reserve num elements, if available */
-	while (1) {
-		/* NOTE: review for potential ordering optimization */
-		uint64_t len = rte_atomic_load_explicit(&list->len, rte_memory_order_seq_cst);
-
-		/* Does the list contain enough elements? */
-		if (unlikely(len < num))
-			return NULL;
-
-		/* NOTE: review for potential ordering optimization */
-		if (rte_atomic_compare_exchange_strong_explicit(&list->len, &len, len - num,
-				rte_memory_order_seq_cst, rte_memory_order_seq_cst))
-			break;
-	}
-
-	old_head = list->head;
-
-	/* Pop num elements */
-	do {
-		struct rte_stack_lf_head new_head;
-		struct rte_stack_lf_elem *tmp;
-		unsigned int i;
-
-		/* An acquire fence (or stronger) is needed for weak memory
-		 * models to ensure the LF LIFO element reads are properly
-		 * ordered with respect to the head pointer read.
-		 */
-		rte_smp_mb();
-
-		rte_prefetch0(old_head.top);
-
-		tmp = old_head.top;
-
-		/* Traverse the list to find the new head. A next pointer will
-		 * either point to another element or NULL; if a thread
-		 * encounters a pointer that has already been popped, the CAS
-		 * will fail.
-		 */
-		for (i = 0; i < num && tmp != NULL; i++) {
-			rte_prefetch0(tmp->next);
-			if (obj_table)
-				obj_table[i] = tmp->data;
-			if (last)
-				*last = tmp;
-			tmp = tmp->next;
-		}
-
-		/* If NULL was encountered, the list was modified while
-		 * traversing it. Retry.
-		 */
-		if (i != num) {
-			old_head = list->head;
-			continue;
-		}
-
-		new_head.top = tmp;
-		new_head.cnt = old_head.cnt + 1;
-
-		/* old_head is updated on failure */
-		success = rte_atomic128_cmp_exchange(
-				(rte_int128_t *)&list->head,
-				(rte_int128_t *)&old_head,
-				(rte_int128_t *)&new_head,
-				1, rte_memory_order_release,
-				rte_memory_order_relaxed);
-	} while (success == 0);
-
-	return old_head.top;
-}
-
-#endif /* _RTE_STACK_LF_GENERIC_H_ */
-- 
2.53.0