[PATCH v2] add high-performance clock

[Morten Brørup]

Reading the common clock, gr_clock_ns(), is implemented as
clock_gettime(CLOCK_MONOTONIC_RAW), which is quite slow, even though the
kernel exposes it as a vDSO.

For improved performance, especially in the dataplane, introduce a high-
performance clock, based on snapshotting the output from gr_clock_ns(),
for use where a snapshot is sufficiently accurate.

Immediately before walking the graph in the dataplane worker thread's main
loop, update the clock, so it is safe to use in the "process" functions
of the graph nodes.

Upgrade some graph node "process" functions to use this clock.

Signed-off-by: Morten Brørup <mb at smartsharesystems.com>
---
Note:
Functions shared by dataplane threads and control plane threads were not
upgraded with this patch.
Future patches will address this.

v2:
* Eliminate the concept of a client API message required to read Grout's
  clock. (Robin Jarry)
* Keep gr_clock_ns() as is.
* For clock source, use gr_clock_ns() instead of rte_rdtsc().
* Add instrumentation for debug builds.
* Use __thread instead of RTE_PER_LCORE.
  This aligns with Grout's existing code style.
---
 api/gr_clock.h                          |  2 +
 main/clock.c                            | 51 +++++++++++++++
 main/clock.h                            | 82 +++++++++++++++++++++++++
 main/meson.build                        |  1 +
 modules/infra/datapath/main_loop.c      |  3 +
 modules/ip/datapath/icmp_input.c        |  5 +-
 modules/ip/datapath/icmp_local_send.c   |  5 +-
 modules/ip6/datapath/icmp6_input.c      |  5 +-
 modules/ip6/datapath/icmp6_local_send.c |  5 +-
 9 files changed, 147 insertions(+), 12 deletions(-)
 create mode 100644 main/clock.c
 create mode 100644 main/clock.h

diff --git a/api/gr_clock.h b/api/gr_clock.h
index d2d98fba..4cc68749 100644
--- a/api/gr_clock.h
+++ b/api/gr_clock.h
@@ -23,6 +23,8 @@ static inline struct timespec gr_clock_raw(void) {
 }
 
 #define GR_NS_PER_S (gr_clock_ns_t)1000000000LL
+#define GR_NS_PER_MS (gr_clock_ns_t)1000000LL
+#define GR_NS_PER_US (gr_clock_ns_t)1000LL
 
 // Get powered-on (non-suspended, non-hibernated) time since last boot [nanoseconds],
 // using a common clock across all processes.
diff --git a/main/clock.c b/main/clock.c
new file mode 100644
index 00000000..8ae57bd1
--- /dev/null
+++ b/main/clock.c
@@ -0,0 +1,51 @@
+// SPDX-License-Identifier: BSD-3-Clause
+// Copyright (c) 2026 SmartShare Systems
+
+#include "clock.h"
+#include "log.h"
+#include "module.h"
+
+LOG_TYPE("clock");
+
+__thread gr_clock_ns_t clock_snapshot_ns = INT64_C(-1);
+__thread int32_t clock_snapshot_s = INT32_C(-1);
+
+void _clock_assert_updated(const char *file, int line, const char *func) {
+	gr_clock_ns_t delta = gr_clock_ns() - clock_snapshot_ns;
+#define CLOCK_UPDATED_THRESHOLD (10 * GR_NS_PER_MS)
+	if (unlikely(delta > CLOCK_UPDATED_THRESHOLD))
+		LOG(ERR, "%s:%d %s: clock not updated, %ld ns behind",
+				file, line, func, delta);
+}
+
+void _clock_log_update(const char *file, int line, const char *func) {
+	LOG(DEBUG, "%s:%d %s: clock update",
+			file, line, func);
+}
+
+void _clock_update(void) {
+	struct timespec tp = gr_clock_raw();
+	clock_snapshot_ns = tp.tv_sec * GR_NS_PER_S + tp.tv_nsec;
+	clock_snapshot_s = tp.tv_sec;
+}
+
+static __rte_cold void clock_init(struct event_base *) {
+	// Update the clock for the main thread.
+	clock_update();
+}
+
+static struct module clock_module = {
+	.name = "clock",
+	.init = clock_init,
+};
+
+RTE_INIT(clock_constructor) {
+	module_register(&clock_module);
+}
+
+// For review of compiled inline functions
+
+gr_clock_ns_t clock_ns_review(void);
+gr_clock_ns_t clock_ns_review(void) {
+	return clock_ns();
+}
diff --git a/main/clock.h b/main/clock.h
new file mode 100644
index 00000000..9572ee58
--- /dev/null
+++ b/main/clock.h
@@ -0,0 +1,82 @@
+// SPDX-License-Identifier: BSD-3-Clause
+// Copyright (c) 2026 SmartShare Systems
+
+#pragma once
+
+#include "log.h"
+
+#include <gr_clock.h>
+
+#include <rte_common.h>
+#include <rte_debug.h>
+#include <rte_eal.h>
+#include <rte_lcore.h>
+
+// (Internal)
+// The clock.
+// Implemented as per-thread snapshot of the common clock.
+// Note: Per thread to avoid certain race conditions.
+extern __thread gr_clock_ns_t clock_snapshot_ns;
+extern __thread int32_t clock_snapshot_s;
+
+// (Internal)
+// Assert that clock has been updated recently for the current thread.
+// No need to call from dataplane worker thread; the main loop keeps the clock updated.
+void _clock_assert_updated(const char *file, int line, const char *func);
+
+// (Internal)
+// Log that the clock was updated.
+// Do not call from dataplane worker thread.
+void _clock_log_update(const char *file, int line, const char *func);
+
+// (Internal)
+static __rte_always_inline gr_clock_ns_t _clock_ns(void) {
+	RTE_ASSERT(clock_snapshot_ns != INT64_C(-1));
+	__rte_assume(clock_snapshot_ns >= 0);
+	return clock_snapshot_ns;
+}
+
+// Get the clock for the current thread [nanoseconds].
+// Does not return negative values.
+#ifndef NDEBUG
+#define clock_ns() ({ \
+	if (unlikely(!rte_lcore_has_role(rte_lcore_id(), ROLE_NON_EAL))) \
+		_clock_assert_updated(__FILE__, __LINE__, __func__); \
+	_clock_ns(); /* statement expression value */ \
+})
+#else
+#define clock_ns() _clock_ns()
+#endif
+
+// (Internal)
+static __rte_always_inline int32_t _clock_s(void) {
+	RTE_ASSERT(clock_snapshot_s != INT32_C(-1));
+	__rte_assume(clock_snapshot_s >= 0);
+	return clock_snapshot_s;
+}
+
+// Get the clock for the current thread [seconds].
+// Does not return negative values.
+#ifndef NDEBUG
+#define clock_s() ({ \
+	if (unlikely(!rte_lcore_has_role(rte_lcore_id(), ROLE_NON_EAL))) \
+		_clock_assert_updated(__FILE__, __LINE__, __func__); \
+	_clock_s(); /* statement expression value */ \
+})
+#else
+#define clock_s() _clock_s()
+#endif
+
+// (Internal)
+void _clock_update(void);
+
+// Update the clock for the current thread.
+#ifndef NDEBUG
+#define clock_update() do { \
+	if (unlikely(!rte_lcore_has_role(rte_lcore_id(), ROLE_NON_EAL))) \
+		_clock_log_update(__FILE__, __LINE__, __func__); \
+	_clock_update(); \
+} while (0)
+#else
+#define clock_update() _clock_update()
+#endif
diff --git a/main/meson.build b/main/meson.build
index 1abc8389..f089a9d2 100644
--- a/main/meson.build
+++ b/main/meson.build
@@ -3,6 +3,7 @@
 
 src += files(
   'api.c',
+  'clock.c',
   'config.c',
   'control_queue.c',
   'dpdk.c',
diff --git a/modules/infra/datapath/main_loop.c b/modules/infra/datapath/main_loop.c
index 2aae4030..51dd167c 100644
--- a/modules/infra/datapath/main_loop.c
+++ b/modules/infra/datapath/main_loop.c
@@ -1,6 +1,8 @@
 // SPDX-License-Identifier: BSD-3-Clause
 // Copyright (c) 2023 Robin Jarry
+// Copyright (c) 2026 SmartShare Systems
 
+#include "clock.h"
 #include "config.h"
 #include "control_input.h"
 #include "datapath.h"
@@ -456,6 +458,7 @@ reconfig:
 	worker_active_inc();
 
 	for (;;) {
+		clock_update();
 		rte_graph_walk(graph);
 
 		if (++loop == HOUSEKEEPING_INTERVAL) {
diff --git a/modules/ip/datapath/icmp_input.c b/modules/ip/datapath/icmp_input.c
index b0426246..537fa477 100644
--- a/modules/ip/datapath/icmp_input.c
+++ b/modules/ip/datapath/icmp_input.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: BSD-3-Clause
 // Copyright (c) 2024 Robin Jarry
 
+#include "clock.h"
 #include "control_output.h"
 #include "graph.h"
 #include "ip4_datapath.h"
@@ -8,8 +9,6 @@
 #include "mbuf.h"
 #include "trace.h"
 
-#include <gr_clock.h>
-
 #include <rte_icmp.h>
 
 enum {
@@ -55,7 +54,7 @@ icmp_input_process(struct rte_graph *graph, struct rte_node *node, void **objs,
 			ip_data->src = ip;
 			edge = OUTPUT;
 		} else if (icmp_cb[icmp->icmp_type]) {
-			control_output_set_cb(mbuf, icmp_cb[icmp->icmp_type], gr_clock_ns());
+			control_output_set_cb(mbuf, icmp_cb[icmp->icmp_type], clock_ns());
 			edge = CONTROL;
 		} else {
 			edge = UNSUPPORTED;
diff --git a/modules/ip/datapath/icmp_local_send.c b/modules/ip/datapath/icmp_local_send.c
index fbbc6fc9..23bc3fe1 100644
--- a/modules/ip/datapath/icmp_local_send.c
+++ b/modules/ip/datapath/icmp_local_send.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: BSD-3-Clause
 // Copyright (c) 2024 Christophe Fontaine
 
+#include "clock.h"
 #include "control_input.h"
 #include "graph.h"
 #include "iface.h"
@@ -8,8 +9,6 @@
 #include "ip4_datapath.h"
 #include "mbuf.h"
 
-#include <gr_clock.h>
-
 #include <rte_icmp.h>
 
 #include <netinet/in.h>
@@ -101,7 +100,7 @@ static uint16_t icmp_local_send_process(
 		);
 
 		payload = rte_pktmbuf_mtod_offset(mbuf, gr_clock_ns_t *, sizeof(*icmp));
-		*payload = gr_clock_ns();
+		*payload = clock_ns();
 
 		// Build ICMP packet
 		icmp->icmp_type = RTE_ICMP_TYPE_ECHO_REQUEST;
diff --git a/modules/ip6/datapath/icmp6_input.c b/modules/ip6/datapath/icmp6_input.c
index 9a001aa3..14bf13cc 100644
--- a/modules/ip6/datapath/icmp6_input.c
+++ b/modules/ip6/datapath/icmp6_input.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: BSD-3-Clause
 // Copyright (c) 2024 Robin Jarry
 
+#include "clock.h"
 #include "control_output.h"
 #include "graph.h"
 #include "icmp6.h"
@@ -10,8 +11,6 @@
 #include "mbuf.h"
 #include "trace.h"
 
-#include <gr_clock.h>
-
 enum {
 	ICMP6_OUTPUT = 0,
 	NEIGH_SOLICIT,
@@ -92,7 +91,7 @@ icmp6_input_process(struct rte_graph *graph, struct rte_node *node, void **objs,
 			break;
 		default:
 			if (icmp6_cb[icmp6->type] != NULL) {
-				control_output_set_cb(mbuf, icmp6_cb[icmp6->type], gr_clock_ns());
+				control_output_set_cb(mbuf, icmp6_cb[icmp6->type], clock_ns());
 				next = CONTROL;
 			} else {
 				next = UNSUPPORTED;
diff --git a/modules/ip6/datapath/icmp6_local_send.c b/modules/ip6/datapath/icmp6_local_send.c
index 66202ea5..da42568b 100644
--- a/modules/ip6/datapath/icmp6_local_send.c
+++ b/modules/ip6/datapath/icmp6_local_send.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: BSD-3-Clause
 // Copyright (c) 2025 Olivier Gournet
 
+#include "clock.h"
 #include "control_input.h"
 #include "graph.h"
 #include "icmp6.h"
@@ -9,8 +10,6 @@
 #include "ip6_datapath.h"
 #include "mbuf.h"
 
-#include <gr_clock.h>
-
 #include <rte_ip6.h>
 
 #include <netinet/in.h>
@@ -111,7 +110,7 @@ static uint16_t icmp6_local_send_process(
 		mbuf->ol_flags |= RTE_MBUF_F_RX_RSS_HASH;
 
 		payload = PAYLOAD(icmp6_echo);
-		*payload = gr_clock_ns();
+		*payload = clock_ns();
 
 		data = ip6_local_mbuf_data(mbuf);
 		data->iface = msg.iface;
-- 
2.43.0