[PATCH 2/2] eal: add Arm WFET in power management intrinsics

[Wathsala Vithanage]

Wait for event with timeout (WFET) puts the CPU in a low power
mode and stays there until an event is signalled (SEV), loss of
an exclusive monitor or a timeout.
WFET is enabled selectively by checking FEAT_WFxT in Linux
auxiliary vector. If FEAT_WFxT is not available power management
will fallback to WFE.
RTE_ARM_USE_WFE macro is not required to enable WFE feature for
PMD power monitoring.

Signed-off-by: Wathsala Vithanage <wathsala.vithanage at arm.com>
Reviewed-by: Dhruv Tripathi <dhruv.tripathi at arm.com>
Reviewed-by: Honnappa Nagarahalli <honnappa.nagarahalli at arm.com>
Reviewed-by: Jack Bond-Preston <jack.bond-preston at foss.arm.com>
Reviewed-by: Nick Connolly <nick.connolly at arm.com>
Reviewed-by: Vinod Krishna <vinod.krishna at arm.com>

---
 .mailmap                                   |  4 +-
 app/test/test_cpuflags.c                   |  3 ++
 lib/eal/arm/include/rte_cpuflags_64.h      |  1 +
 lib/eal/arm/include/rte_pause_64.h         | 21 ++++++--
 lib/eal/arm/include/rte_power_intrinsics.h |  6 +++
 lib/eal/arm/rte_cpuflags.c                 |  3 +-
 lib/eal/arm/rte_power_intrinsics.c         | 56 ++++++++++++----------
 7 files changed, 62 insertions(+), 32 deletions(-)

diff --git a/.mailmap b/.mailmap
index 3843868716..eefcce4935 100644
--- a/.mailmap
+++ b/.mailmap
@@ -332,6 +332,7 @@ Dexia Li <dexia.li at jaguarmicro.com>
 Dexuan Cui <decui at microsoft.com>
 Dharmik Thakkar <dharmikjayesh.thakkar at arm.com> <dharmik.thakkar at arm.com>
 Dheemanth Mallikarjun <dheemanthm at vmware.com>
+Dhruv Tripathi <dhruv.tripathi at arm.com>
 Diana Wang <na.wang at corigine.com>
 Didier Pallard <didier.pallard at 6wind.com>
 Dilshod Urazov <dilshod.urazov at oktetlabs.ru>
@@ -1027,7 +1028,7 @@ Netanel Belgazal <netanel at amazon.com>
 Netanel Gonen <netanelg at mellanox.com>
 Niall Power <niall.power at intel.com>
 Nicholas Pratte <npratte at iol.unh.edu>
-Nick Connolly <nick.connolly at mayadata.io>
+Nick Connolly <nick.connolly at arm.com> <Nick.x.Connolly at gmail.com>
 Nick Nunley <nicholas.d.nunley at intel.com>
 Niclas Storm <niclas.storm at ericsson.com>
 Nicolas Chautru <nicolas.chautru at intel.com>
@@ -1516,6 +1517,7 @@ Vincent Jardin <vincent.jardin at 6wind.com>
 Vincent Li <vincent.mc.li at gmail.com>
 Vincent S. Cojot <vcojot at redhat.com>
 Vinh Tran <vinh.t.tran10 at gmail.com>
+Vinod Krishna <vinod.krishna at arm.com>
 Vipin Varghese <vipin.varghese at amd.com> <vipin.varghese at intel.com>
 Vipul Ashri <vipul.ashri at oracle.com>
 Visa Hankala <visa at hankala.org>
diff --git a/app/test/test_cpuflags.c b/app/test/test_cpuflags.c
index a0ff74720c..22ab4dff0a 100644
--- a/app/test/test_cpuflags.c
+++ b/app/test/test_cpuflags.c
@@ -156,6 +156,9 @@ test_cpuflags(void)
 
 	printf("Check for SVEBF16:\t");
 	CHECK_FOR_FLAG(RTE_CPUFLAG_SVEBF16);
+
+	printf("Check for WFXT:\t");
+	CHECK_FOR_FLAG(RTE_CPUFLAG_WFXT);
 #endif
 
 #if defined(RTE_ARCH_X86_64) || defined(RTE_ARCH_I686)
diff --git a/lib/eal/arm/include/rte_cpuflags_64.h b/lib/eal/arm/include/rte_cpuflags_64.h
index afe70209c3..db61539db2 100644
--- a/lib/eal/arm/include/rte_cpuflags_64.h
+++ b/lib/eal/arm/include/rte_cpuflags_64.h
@@ -35,6 +35,7 @@ enum rte_cpu_flag_t {
 	RTE_CPUFLAG_SVEF32MM,
 	RTE_CPUFLAG_SVEF64MM,
 	RTE_CPUFLAG_SVEBF16,
+	RTE_CPUFLAG_WFXT,
 	RTE_CPUFLAG_AARCH64,
 };
 
diff --git a/lib/eal/arm/include/rte_pause_64.h b/lib/eal/arm/include/rte_pause_64.h
index 5cb8b59056..f732407425 100644
--- a/lib/eal/arm/include/rte_pause_64.h
+++ b/lib/eal/arm/include/rte_pause_64.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2017 Cavium, Inc
- * Copyright(c) 2019 Arm Limited
+ * Copyright(c) 2024 Arm Limited
  */
 
 #ifndef _RTE_PAUSE_ARM64_H_
@@ -23,17 +23,28 @@ static inline void rte_pause(void)
 	asm volatile("yield" ::: "memory");
 }
 
-#ifdef RTE_WAIT_UNTIL_EQUAL_ARCH_DEFINED
 
-/* Send a local event to quit WFE. */
+/* Send a local event to quit WFE/WFxT. */
 #define __RTE_ARM_SEVL() { asm volatile("sevl" : : : "memory"); }
 
-/* Send a global event to quit WFE for all cores. */
+/* Send a global event to quit WFE/WFxT for all cores. */
 #define __RTE_ARM_SEV() { asm volatile("sev" : : : "memory"); }
 
 /* Put processor into low power WFE(Wait For Event) state. */
 #define __RTE_ARM_WFE() { asm volatile("wfe" : : : "memory"); }
 
+/* Put processor into low power WFET (WFE with Timeout) state. */
+#ifdef RTE_ARM_FEATURE_WFXT
+#define __RTE_ARM_WFET(t) {                              \
+	asm volatile("wfet %x[to]"                        \
+			:                                 \
+			: [to] "r" (t)                    \
+			: "memory");                      \
+	}
+#else
+#define __RTE_ARM_WFET(t) { RTE_SET_USED(t); }
+#endif
+
 /*
  * Atomic exclusive load from addr, it returns the 8-bit content of
  * *addr while making it 'monitored', when it is written by someone
@@ -147,6 +158,8 @@ static inline void rte_pause(void)
 		__RTE_ARM_LOAD_EXC_128(src, dst, memorder) \
 }
 
+#ifdef RTE_WAIT_UNTIL_EQUAL_ARCH_DEFINED
+
 static __rte_always_inline void
 rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected,
 		int memorder)
diff --git a/lib/eal/arm/include/rte_power_intrinsics.h b/lib/eal/arm/include/rte_power_intrinsics.h
index 9e498e9ebf..dc80fca60b 100644
--- a/lib/eal/arm/include/rte_power_intrinsics.h
+++ b/lib/eal/arm/include/rte_power_intrinsics.h
@@ -13,6 +13,12 @@ extern "C" {
 
 #include "generic/rte_power_intrinsics.h"
 
+struct rte_power_monitor_info {
+	uint16_t init_done:1,   /* Initialization status bit */
+		wfet_en:1,	/* FEAT_WFET enabled bit */
+		reserved:14;	/* Reserved */
+};
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/eal/arm/rte_cpuflags.c b/lib/eal/arm/rte_cpuflags.c
index 7ba4f8ba97..ad76f2448b 100644
--- a/lib/eal/arm/rte_cpuflags.c
+++ b/lib/eal/arm/rte_cpuflags.c
@@ -115,6 +115,7 @@ const struct feature_entry rte_cpu_feature_table[] = {
 	FEAT_DEF(SVEF32MM,	REG_HWCAP2,   10)
 	FEAT_DEF(SVEF64MM,	REG_HWCAP2,   11)
 	FEAT_DEF(SVEBF16,	REG_HWCAP2,   12)
+	FEAT_DEF(WFXT,		REG_HWCAP2,   31)
 	FEAT_DEF(AARCH64,	REG_PLATFORM,  0)
 };
 #endif /* RTE_ARCH */
@@ -163,7 +164,5 @@ void
 rte_cpu_get_intrinsics_support(struct rte_cpu_intrinsics *intrinsics)
 {
 	memset(intrinsics, 0, sizeof(*intrinsics));
-#ifdef RTE_ARM_USE_WFE
 	intrinsics->power_monitor = 1;
-#endif
 }
diff --git a/lib/eal/arm/rte_power_intrinsics.c b/lib/eal/arm/rte_power_intrinsics.c
index f54cf59e80..4c858192f2 100644
--- a/lib/eal/arm/rte_power_intrinsics.c
+++ b/lib/eal/arm/rte_power_intrinsics.c
@@ -4,20 +4,28 @@
 
 #include <errno.h>
 
+#include <rte_per_lcore.h>
+
+#include "rte_cpuflags.h"
 #include "rte_power_intrinsics.h"
 
 /**
- * This function uses WFE instruction to make lcore suspend
+ * Per core rte_power_monitor_info struct.
+ */
+RTE_DEFINE_PER_LCORE(struct rte_power_monitor_info, pm_info) = {
+	.init_done = 0,
+	.wfet_en = 0,
+};
+
+/**
+ * This function uses WFE/WFET instruction to make lcore suspend
  * execution on ARM.
- * Note that timestamp based timeout is not supported yet.
  */
 int
 rte_power_monitor(const struct rte_power_monitor_cond *pmc,
 		const uint64_t tsc_timestamp)
 {
-	RTE_SET_USED(tsc_timestamp);
-
-#ifdef RTE_ARM_USE_WFE
+	struct rte_power_monitor_info *pminfo;
 	const unsigned int lcore_id = rte_lcore_id();
 	uint64_t cur_value;
 
@@ -31,33 +39,37 @@ rte_power_monitor(const struct rte_power_monitor_cond *pmc,
 	if (pmc->fn == NULL)
 		return -EINVAL;
 
+	pminfo = &RTE_PER_LCORE(pm_info);
+
+	if (unlikely(!(pminfo->init_done))) {
+		if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_WFXT))
+			pminfo->wfet_en = 1;
+		pminfo->init_done = 1;
+	}
+
 	switch (pmc->size) {
 	case sizeof(uint8_t):
-		__RTE_ARM_LOAD_EXC_8(pmc->addr, cur_value, rte_memory_order_relaxed)
-		__RTE_ARM_WFE()
+		__RTE_ARM_LOAD_EXC_8(pmc->addr, cur_value, rte_memory_order_relaxed);
 		break;
 	case sizeof(uint16_t):
-		__RTE_ARM_LOAD_EXC_16(pmc->addr, cur_value, rte_memory_order_relaxed)
-		__RTE_ARM_WFE()
+		__RTE_ARM_LOAD_EXC_16(pmc->addr, cur_value, rte_memory_order_relaxed);
 		break;
 	case sizeof(uint32_t):
-		__RTE_ARM_LOAD_EXC_32(pmc->addr, cur_value, rte_memory_order_relaxed)
-		__RTE_ARM_WFE()
+		__RTE_ARM_LOAD_EXC_32(pmc->addr, cur_value, rte_memory_order_relaxed);
 		break;
 	case sizeof(uint64_t):
-		__RTE_ARM_LOAD_EXC_64(pmc->addr, cur_value, rte_memory_order_relaxed)
-		__RTE_ARM_WFE()
+		__RTE_ARM_LOAD_EXC_64(pmc->addr, cur_value, rte_memory_order_relaxed);
 		break;
 	default:
 		return -EINVAL; /* unexpected size */
 	}
 
-	return 0;
-#else
-	RTE_SET_USED(pmc);
+	if (pminfo->wfet_en)
+		__RTE_ARM_WFET(tsc_timestamp)
+	else
+		__RTE_ARM_WFE()
 
-	return -ENOTSUP;
-#endif
+	return 0;
 }
 
 /**
@@ -80,14 +92,8 @@ int
 rte_power_monitor_wakeup(const unsigned int lcore_id)
 {
 	RTE_SET_USED(lcore_id);
-
-#ifdef RTE_ARM_USE_WFE
-	__RTE_ARM_SEV()
-
+	__RTE_ARM_SEV();
 	return 0;
-#else
-	return -ENOTSUP;
-#endif
 }
 
 int
-- 
2.34.1