[PATCH v2] eal/riscv: add support for zawrs extension
Sun Yuechi
sunyuechi at iscas.ac.cn
Sun Nov 8 19:39:19 CET 2026
The Zawrs extension adds a pair of instructions that stall a core until
a memory location is written to. This patch uses one of them to
implement RISC-V specific versions of the rte_wait_until_equal_*
functions. This is potentially more energy efficient than the default
implementation that uses rte_pause/Zihintpause.
The technique works as follows:
* Check the value with a plain load first and return immediately if it
already matches, as the generic implementation does
* Otherwise create a reservation set containing the address we want to
wait on using an atomic load (lr.dw)
* Call wrs.nto - this blocks until the reservation set is invalidated by
someone else writing to that address
* Execution can also resume arbitrarily, so we still need to check
whether a change occurred and loop if not
Due to RISC-V atomics only supporting naturally aligned word (32 bit)
and double word (64 bit) loads, I've used pointer rounding and bit
shifting to implement waiting on 16-bit values.
The code is enabled at build time when the target ISA string contains
the Zawrs extension, detected through the compiler's __riscv_zawrs
architecture test macro, e.g. -Dcpu_instruction_set=rv64gcv_zawrs.
This is based on an earlier RFC by Daniel Gregory.
Signed-off-by: Daniel Gregory <daniel.gregory at bytedance.com>
Signed-off-by: Sun Yuechi <sunyuechi at iscas.ac.cn>
---
v2:
* rebased on main
* use build-time detection via __riscv_zawrs instead of a manual meson flag
* change memorder parameter type from int to rte_memory_order
* fix inverted alignment assert in RTE_WAIT_UNTIL_MASKED
* make the size in RTE_WAIT_UNTIL_MASKED a compile-time constant
* add a plain load before taking the reservation
* add release notes and cross-build documentation
config/riscv/meson.build | 9 +
.../linux_gsg/cross_build_dpdk_for_riscv.rst | 26 +++
doc/guides/rel_notes/release_26_11.rst | 8 +
lib/eal/riscv/include/rte_pause.h | 159 ++++++++++++++++++
4 files changed, 202 insertions(+)
diff --git a/config/riscv/meson.build b/config/riscv/meson.build
index 7f3e3301bb..5370aeac9e 100644
--- a/config/riscv/meson.build
+++ b/config/riscv/meson.build
@@ -153,6 +153,15 @@ if (riscv_extension_macros and
endif
endif
+# detect Zawrs (wait-on-reservation-set) extension, used by rte_wait_until_equal_*
+# to stall the hart until the polled memory location is written instead of spinning
+if (riscv_extension_macros and
+ (cc.get_define('__riscv_zawrs', args: machine_args) != ''))
+ message('Compiling with the Zawrs extension')
+ machine_args += ['-DRTE_RISCV_FEATURE_ZAWRS']
+ dpdk_flags += [['RTE_RISCV_FEATURE_ZAWRS', 1],]
+endif
+
# apply flags
foreach flag: dpdk_flags
if flag.length() > 0
diff --git a/doc/guides/linux_gsg/cross_build_dpdk_for_riscv.rst b/doc/guides/linux_gsg/cross_build_dpdk_for_riscv.rst
index bcba12a604..c7be173b9e 100644
--- a/doc/guides/linux_gsg/cross_build_dpdk_for_riscv.rst
+++ b/doc/guides/linux_gsg/cross_build_dpdk_for_riscv.rst
@@ -115,3 +115,29 @@ Currently the following targets are supported:
To add a new target support, ``config/riscv/meson.build`` has to be modified by
adding a new vendor/architecture id and a corresponding cross-file has to be
added to ``config/riscv`` directory.
+
+
+Optional ISA extensions
+-----------------------
+
+Some optimisations are enabled at build time when the target ISA string
+includes the extension.
+The build looks at the per-extension macros the compiler defines when
+``__riscv_arch_test`` is available, such as ``__riscv_zawrs``:
+
+``v``
+ Vector implementations (``RTE_RISCV_FEATURE_V``).
+
+``zawrs``
+ ``rte_wait_until_equal_*`` stall the hart with ``wrs.nto``
+ instead of spinning (``RTE_RISCV_FEATURE_ZAWRS``).
+
+Pass the full ISA string to enable them.
+For native builds use ``-Dcpu_instruction_set``::
+
+ meson setup build -Dcpu_instruction_set=rv64gcv_zawrs
+
+For cross builds, set ``cpu`` in the cross-file instead::
+
+ [host_machine]
+ cpu = 'rv64gcv_zawrs'
diff --git a/doc/guides/rel_notes/release_26_11.rst b/doc/guides/rel_notes/release_26_11.rst
index 8ca8379cc6..4cce0b478a 100644
--- a/doc/guides/rel_notes/release_26_11.rst
+++ b/doc/guides/rel_notes/release_26_11.rst
@@ -68,6 +68,14 @@ New Features
and use pointer arithmetic instead of integer casts to enable compiler optimizations.
See API changes below.
+* **Added RISC-V Zawrs support for wait-until-equal helpers.**
+
+ On RISC-V targets built with the Zawrs extension in the ISA string
+ (for example ``-Dcpu_instruction_set=rv64gcv_zawrs``),
+ ``rte_wait_until_equal_*`` and ``RTE_WAIT_UNTIL_MASKED`` now stall the hart
+ with ``wrs.nto`` until the polled address is written instead of spinning
+ on ``pause``.
+
* **Added API to get CPU socket ID.**
Added the experimental ``rte_cpu_socket_id()`` function
diff --git a/lib/eal/riscv/include/rte_pause.h b/lib/eal/riscv/include/rte_pause.h
index 3f473cd8db..99d7f46711 100644
--- a/lib/eal/riscv/include/rte_pause.h
+++ b/lib/eal/riscv/include/rte_pause.h
@@ -7,6 +7,14 @@
#ifndef RTE_PAUSE_RISCV_H
#define RTE_PAUSE_RISCV_H
+#include <rte_config.h>
+
+#ifdef RTE_RISCV_FEATURE_ZAWRS
+#define RTE_WAIT_UNTIL_EQUAL_ARCH_DEFINED
+#endif
+
+#include <rte_debug.h>
+
#include "rte_atomic.h"
#include "generic/rte_pause.h"
@@ -24,6 +32,157 @@ static inline void rte_pause(void)
asm volatile(".int 0x0100000F" : : : "memory");
}
+#ifdef RTE_RISCV_FEATURE_ZAWRS
+
+/*
+ * Atomic load from an address, it returns either a sign-extended word or
+ * doubleword and creates a 'reservation set' containing the read memory
+ * location. When someone else writes to the reservation set, it is invalidated,
+ * causing any stalled WRS instructions to resume.
+ *
+ * Address needs to be naturally aligned.
+ */
+#define __RTE_RISCV_LR_32(src, dst, memorder) do { \
+ if ((memorder) == rte_memory_order_relaxed) { \
+ asm volatile("lr.w %0, (%1)" \
+ : "=r" (dst) \
+ : "r" (src) \
+ : "memory"); \
+ } else { \
+ asm volatile("lr.w.aq %0, (%1)" \
+ : "=r" (dst) \
+ : "r" (src) \
+ : "memory"); \
+ } } while (0)
+#define __RTE_RISCV_LR_64(src, dst, memorder) do { \
+ if ((memorder) == rte_memory_order_relaxed) { \
+ asm volatile("lr.d %0, (%1)" \
+ : "=r" (dst) \
+ : "r" (src) \
+ : "memory"); \
+ } else { \
+ asm volatile("lr.d.aq %0, (%1)" \
+ : "=r" (dst) \
+ : "r" (src) \
+ : "memory"); \
+ } } while (0)
+
+/*
+ * There's not a RISC-V atomic load primitive for halfwords, so cast up to a
+ * _naturally aligned_ word and extract the halfword we want
+ */
+#define __RTE_RISCV_LR_16(src, dst, memorder) do { \
+ uint32_t word; \
+ __RTE_RISCV_LR_32(((uintptr_t)(src) & (~3)), word, (memorder)); \
+ if ((size_t)(src) & 3) \
+ (dst) = (typeof(dst))(word >> 16); \
+ else \
+ (dst) = (typeof(dst))(word & 0xFFFF); \
+} while (0)
+
+#define __RTE_RISCV_LR(src, dst, memorder, size) { \
+ RTE_BUILD_BUG_ON(size != 16 && size != 32 && size != 64); \
+ if (size == 16) \
+ __RTE_RISCV_LR_16(src, dst, memorder); \
+ else if (size == 32) \
+ __RTE_RISCV_LR_32(src, dst, memorder); \
+ else if (size == 64) \
+ __RTE_RISCV_LR_64(src, dst, memorder); \
+}
+
+/*
+ * Wait-on-Reservation-Set extension instruction, it stalls execution until the
+ * reservation set is invalidated or an interrupt is observed.
+ * A loop is likely still needed as it may stop stalling arbitrarily.
+ */
+#define __RTE_RISCV_WRS_NTO() { asm volatile("wrs.nto" : : : "memory"); }
+
+static inline void
+rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected,
+ rte_memory_order memorder)
+{
+ uint16_t value;
+
+ RTE_ASSERT(memorder == rte_memory_order_acquire ||
+ memorder == rte_memory_order_relaxed);
+ RTE_ASSERT(((size_t)addr & 1) == 0);
+
+ /* fast path: a plain load is cheaper than taking a reservation */
+ if (rte_atomic_load_explicit((volatile __rte_atomic uint16_t *)addr,
+ memorder) == expected)
+ return;
+
+ __RTE_RISCV_LR_16(addr, value, memorder);
+ while (value != expected) {
+ __RTE_RISCV_WRS_NTO();
+ __RTE_RISCV_LR_16(addr, value, memorder);
+ }
+}
+
+static inline void
+rte_wait_until_equal_32(volatile uint32_t *addr, uint32_t expected,
+ rte_memory_order memorder)
+{
+ uint32_t value;
+
+ RTE_ASSERT(memorder == rte_memory_order_acquire ||
+ memorder == rte_memory_order_relaxed);
+ RTE_ASSERT(((size_t)addr & 3) == 0);
+
+ /* fast path: a plain load is cheaper than taking a reservation */
+ if (rte_atomic_load_explicit((volatile __rte_atomic uint32_t *)addr,
+ memorder) == expected)
+ return;
+
+ __RTE_RISCV_LR_32(addr, value, memorder);
+ while (value != expected) {
+ __RTE_RISCV_WRS_NTO();
+ __RTE_RISCV_LR_32(addr, value, memorder);
+ }
+}
+
+static inline void
+rte_wait_until_equal_64(volatile uint64_t *addr, uint64_t expected,
+ rte_memory_order memorder)
+{
+ uint64_t value;
+
+ RTE_ASSERT(memorder == rte_memory_order_acquire ||
+ memorder == rte_memory_order_relaxed);
+ RTE_ASSERT(((size_t)addr & 7) == 0);
+
+ /* fast path: a plain load is cheaper than taking a reservation */
+ if (rte_atomic_load_explicit((volatile __rte_atomic uint64_t *)addr,
+ memorder) == expected)
+ return;
+
+ __RTE_RISCV_LR_64(addr, value, memorder);
+ while (value != expected) {
+ __RTE_RISCV_WRS_NTO();
+ __RTE_RISCV_LR_64(addr, value, memorder);
+ }
+}
+
+#define RTE_WAIT_UNTIL_MASKED(addr, mask, cond, expected, memorder) do { \
+ RTE_BUILD_BUG_ON(!__builtin_constant_p(memorder)); \
+ RTE_BUILD_BUG_ON(memorder != rte_memory_order_acquire && \
+ memorder != rte_memory_order_relaxed); \
+ RTE_ASSERT(((size_t)(addr) & (sizeof(*(addr)) - 1)) == 0); \
+ enum { size = sizeof(*(addr)) << 3 }; \
+ typeof(*(addr)) expected_value = (expected); \
+ typeof(*(addr)) value; \
+ if ((rte_atomic_load_explicit((addr), (memorder)) & (mask)) \
+ cond expected_value) \
+ break; \
+ __RTE_RISCV_LR((addr), value, memorder, size); \
+ while (!((value & (mask)) cond expected_value)) { \
+ __RTE_RISCV_WRS_NTO(); \
+ __RTE_RISCV_LR((addr), value, memorder, size); \
+ } \
+} while (0)
+
+#endif /* RTE_RISCV_FEATURE_ZAWRS */
+
#ifdef __cplusplus
}
#endif
--
2.56.0
More information about the dev
mailing list