[dpdk-dev] [PATCH RFC v2 1/7] eal: introduce new class of barriers for DMA use cases

[Gavin Hu]

In DPDK we use rte_*mb barriers to ensure that memory accesses to DMA
regions are observed before MMIO accesses to hardware registers.

On AArch64, the rte_*mb barriers are implemented by "DSB" (Data
Synchronisation Barrier) style instructions which are the strongest
barriers possible.

Recently, however, it has been realised [1], that for devices where the
MMIO regions are shared between all CPUs, that it is possible to relax
this memory barrier.

There are cases where we wish to retain the strength of the rte_*mb
memory barriers; thus rather than relax rte_*mb we opt instead to
introduce a new class of barrier rte_dma_*mb.

For AArch64, rte_dma_*mb will be implemented by a relaxed "DMB OSH"
style of barrier.

For other architectures, we implement rte_dma_*mb as rte_*mb so this
should not result in any functional changes.

[1] https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/
commit/?id=22ec71615d824f4f11d38d0e55a88d8956b7e45f

Signed-off-by: Gavin Hu <gavin.hu at arm.com>
Reviewed-by: Steve Capper <steve.capper at arm.com>
---
 lib/librte_eal/arm/include/rte_atomic_32.h  |  6 ++++
 lib/librte_eal/arm/include/rte_atomic_64.h  |  6 ++++
 lib/librte_eal/include/generic/rte_atomic.h | 31 +++++++++++++++++++++
 lib/librte_eal/ppc/include/rte_atomic.h     |  6 ++++
 lib/librte_eal/x86/include/rte_atomic.h     |  6 ++++
 5 files changed, 55 insertions(+)

diff --git a/lib/librte_eal/arm/include/rte_atomic_32.h b/lib/librte_eal/arm/include/rte_atomic_32.h
index 7dc0d06d1..80208467e 100644
--- a/lib/librte_eal/arm/include/rte_atomic_32.h
+++ b/lib/librte_eal/arm/include/rte_atomic_32.h
@@ -33,6 +33,12 @@ extern "C" {
 
 #define rte_io_rmb() rte_rmb()
 
+#define rte_dma_mb() rte_mb()
+
+#define rte_dma_wmb() rte_wmb()
+
+#define rte_dma_rmb() rte_rmb()
+
 #define rte_cio_wmb() rte_wmb()
 
 #define rte_cio_rmb() rte_rmb()
diff --git a/lib/librte_eal/arm/include/rte_atomic_64.h b/lib/librte_eal/arm/include/rte_atomic_64.h
index 7b7099cdc..608726c29 100644
--- a/lib/librte_eal/arm/include/rte_atomic_64.h
+++ b/lib/librte_eal/arm/include/rte_atomic_64.h
@@ -37,6 +37,12 @@ extern "C" {
 
 #define rte_io_rmb() rte_rmb()
 
+#define rte_dma_mb() asm volatile("dmb osh" : : : "memory")
+
+#define rte_dma_wmb() asm volatile("dmb oshst" : : : "memory")
+
+#define rte_dma_rmb() asm volatile("dmb oshld" : : : "memory")
+
 #define rte_cio_wmb() asm volatile("dmb oshst" : : : "memory")
 
 #define rte_cio_rmb() asm volatile("dmb oshld" : : : "memory")
diff --git a/lib/librte_eal/include/generic/rte_atomic.h b/lib/librte_eal/include/generic/rte_atomic.h
index e6ab15a97..042264c7e 100644
--- a/lib/librte_eal/include/generic/rte_atomic.h
+++ b/lib/librte_eal/include/generic/rte_atomic.h
@@ -107,6 +107,37 @@ static inline void rte_io_wmb(void);
 static inline void rte_io_rmb(void);
 ///@}
 
+/** @name DMA Memory Barrier
+ */
+///@{
+/**
+ * memory barrier for DMA use cases
+ *
+ * Guarantees that the LOAD and STORE operations that precede the rte_dma_mb()
+ * call are visible to CPU and I/O device that is shared between all CPUs
+ * before the LOAD and STORE operations that follow it.
+ */
+static inline void rte_dma_mb(void);
+
+/**
+ * Write memory barrier for DMA use cases
+ *
+ * Guarantees that the STORE operations that precede the rte_dma_wmb() call are
+ * visible to CPU and I/O device that is shared between all CPUs before the
+ * STORE operations that follow it.
+ */
+static inline void rte_dma_wmb(void);
+
+/**
+ * Read memory barrier for DMA use cases
+ *
+ * Guarantees that the LOAD operations that precede the rte_dma_rmb() call are
+ * visible to CPU and IO device that is shared between all CPUs before the LOAD
+ * operations that follow it.
+ */
+static inline void rte_dma_rmb(void);
+///@}
+
 /** @name Coherent I/O Memory Barrier
  *
  * Coherent I/O memory barrier is a lightweight version of I/O memory
diff --git a/lib/librte_eal/ppc/include/rte_atomic.h b/lib/librte_eal/ppc/include/rte_atomic.h
index 7e3e13118..faa36bb76 100644
--- a/lib/librte_eal/ppc/include/rte_atomic.h
+++ b/lib/librte_eal/ppc/include/rte_atomic.h
@@ -36,6 +36,12 @@ extern "C" {
 
 #define rte_io_rmb() rte_rmb()
 
+#define rte_dma_mb() rte_mb()
+
+#define rte_dma_wmb() rte_wmb()
+
+#define rte_dma_rmb() rte_rmb()
+
 #define rte_cio_wmb() rte_wmb()
 
 #define rte_cio_rmb() rte_rmb()
diff --git a/lib/librte_eal/x86/include/rte_atomic.h b/lib/librte_eal/x86/include/rte_atomic.h
index 148398f50..0b1d452f3 100644
--- a/lib/librte_eal/x86/include/rte_atomic.h
+++ b/lib/librte_eal/x86/include/rte_atomic.h
@@ -79,6 +79,12 @@ rte_smp_mb(void)
 
 #define rte_io_rmb() rte_compiler_barrier()
 
+#define rte_dma_mb() rte_mb()
+
+#define rte_dma_wmb() rte_wmb()
+
+#define rte_dma_rmb() rte_rmb()
+
 #define rte_cio_wmb() rte_compiler_barrier()
 
 #define rte_cio_rmb() rte_compiler_barrier()
-- 
2.17.1