[PATCH v4] add high-performance clock

[Morten Brørup]

Reading the common clock, gr_clock_ns(), is implemented as
clock_gettime(CLOCK_MONOTONIC_RAW), which is quite slow, even though the
kernel exposes it as a vDSO.

For improved performance, especially in the dataplane, introduce a high-
performance clock, based on snapshotting the common clock
(clock_gettime(CLOCK_MONOTONIC_RAW)), for use where a snapshot is
sufficiently accurate.

Immediately before walking the graph in the dataplane worker thread's main
loop, mark the clock as trusted and update it, so it is safe to use in the
"process" functions of the graph nodes.

Signed-off-by: Morten Brørup <mb at smartsharesystems.com>
---
v4:
* Clean up header inclusion order.
  clock.h should only be included by gr_clock.h, and thus
  clock.c should include gr_clock.h instead of clock.h.
v3:
* Change gr_clock_ns(), adding a Grout specific implementation using
  the high-performance clock. This significantly reduces the amount of
  required changes inside Grout. (Robin Jarry)
* Revert upgrade some graph node "process" functions to use
  this clock; they can simply continue using gr_clock_ns().
* Add an explicit per-thread state variable to mark the clock as trusted,
  effectively enabling/disabling use of the high-performance clock.
* Remove unused gr_clock_raw() API.
* Add definition of Grout clock source.
v2:
* Eliminate the concept of a client API message required to read Grout's
  clock. (Robin Jarry)
* Keep gr_clock_ns() as is.
* For clock source, use gr_clock_ns() instead of rte_rdtsc().
* Add instrumentation for debug builds.
* Use __thread instead of RTE_PER_LCORE.
  This aligns with Grout's existing code style.
---
 api/gr_clock.h                     | 26 ++++++-----
 main/clock.c                       | 52 +++++++++++++++++++++
 main/clock.h                       | 73 ++++++++++++++++++++++++++++++
 main/meson.build                   |  1 +
 modules/infra/datapath/main_loop.c |  5 ++
 5 files changed, 146 insertions(+), 11 deletions(-)
 create mode 100644 main/clock.c
 create mode 100644 main/clock.h

diff --git a/api/gr_clock.h b/api/gr_clock.h
index d2d98fba..85ab4838 100644
--- a/api/gr_clock.h
+++ b/api/gr_clock.h
@@ -7,27 +7,31 @@
 #include <stdint.h>
 #include <time.h>
 
+// Clock source.
+// Must be a high-resolution clock, i.e. not a _COARSE variant.
+#define GR_CLOCK_SOURCE CLOCK_MONOTONIC_RAW
+
 // High-resolution clock [nanoseconds].
-// Used with CLOCK_MONOTONIC_RAW, unless otherwise specified.
+// Used with GR_CLOCK_SOURCE, unless otherwise specified.
 // Note: Does not have Y2038 problems. Not even with CLOCK_REALTIME.
 // Note: Using signed, to avoid need for casting to signed
 // in calculations where race conditions may cause negative differences.
 typedef int64_t gr_clock_ns_t;
 
-// Get powered-on (non-suspended, non-hibernated) time since last boot,
-// using a common clock across all processes.
-static inline struct timespec gr_clock_raw(void) {
-	struct timespec tp = {0};
-	clock_gettime(CLOCK_MONOTONIC_RAW, &tp);
-	return tp;
-}
-
-#define GR_NS_PER_S (gr_clock_ns_t)1000000000LL
+#define GR_NS_PER_S  (gr_clock_ns_t)1000000000LL
+#define GR_NS_PER_MS (gr_clock_ns_t)1000000LL
+#define GR_NS_PER_US (gr_clock_ns_t)1000LL
 
 // Get powered-on (non-suspended, non-hibernated) time since last boot [nanoseconds],
 // using a common clock across all processes.
 // Does not return negative values.
+#ifndef __GROUT_MAIN__
 static inline gr_clock_ns_t gr_clock_ns(void) {
-	struct timespec tp = gr_clock_raw();
+	struct timespec tp = {0};
+	clock_gettime(GR_CLOCK_SOURCE, &tp);
 	return tp.tv_sec * GR_NS_PER_S + tp.tv_nsec;
 }
+#else
+// Grout implementation is in main/clock.h
+#include "clock.h"
+#endif
diff --git a/main/clock.c b/main/clock.c
new file mode 100644
index 00000000..8e69ff63
--- /dev/null
+++ b/main/clock.c
@@ -0,0 +1,52 @@
+// SPDX-License-Identifier: BSD-3-Clause
+// Copyright (c) 2026 SmartShare Systems
+
+#include "log.h"
+#include "module.h"
+
+#include <gr_clock.h>
+
+LOG_TYPE("clock");
+
+__thread bool clock_trusted = false;
+
+__thread gr_clock_ns_t clock_snapshot_ns = INT64_C(-1);
+__thread int32_t clock_snapshot_s = INT32_C(-1);
+
+void _clock_log_update(const char *file, int line, const char *func) {
+	LOG(DEBUG, "%s:%d %s: clock update",
+			file, line, func);
+}
+
+void _clock_update(void) {
+	struct timespec tp;
+	clock_gettime(GR_CLOCK_SOURCE, &tp);
+	clock_snapshot_ns = tp.tv_sec * GR_NS_PER_S + tp.tv_nsec;
+	clock_snapshot_s = tp.tv_sec;
+}
+
+static __rte_cold void clock_init(struct event_base *) {
+	// Update the clock for the main thread.
+	clock_update();
+}
+
+static struct module clock_module = {
+	.name = "clock",
+	.init = clock_init,
+};
+
+RTE_INIT(clock_constructor) {
+	module_register(&clock_module);
+}
+
+// For review of compiled inline functions
+
+gr_clock_ns_t clock_ns_review(void);
+gr_clock_ns_t clock_ns_review(void) {
+	return clock_ns();
+}
+
+gr_clock_ns_t gr_clock_ns_review(void);
+gr_clock_ns_t gr_clock_ns_review(void) {
+	return gr_clock_ns();
+}
diff --git a/main/clock.h b/main/clock.h
new file mode 100644
index 00000000..2a9211ee
--- /dev/null
+++ b/main/clock.h
@@ -0,0 +1,73 @@
+// SPDX-License-Identifier: BSD-3-Clause
+// Copyright (c) 2026 SmartShare Systems
+
+#pragma once
+
+#include "log.h"
+
+#include <gr_clock.h>
+
+#include <rte_common.h>
+#include <rte_debug.h>
+
+// Assume that the clock is updated for the current thread,
+// i.e. the clock can be trusted.
+extern __thread bool clock_trusted;
+
+// (Internal)
+// The clock.
+// Implemented as per-thread snapshot of the common clock.
+// Note: Per thread to avoid certain race conditions.
+extern __thread gr_clock_ns_t clock_snapshot_ns;
+extern __thread int32_t clock_snapshot_s;
+
+// Get the clock for the current thread [nanoseconds].
+// Does not return negative values.
+static __rte_always_inline gr_clock_ns_t clock_ns(void) {
+	RTE_ASSERT(clock_snapshot_ns != INT64_C(-1));
+	__rte_assume(clock_snapshot_ns >= 0);
+	return clock_snapshot_ns;
+}
+
+// Get the clock for the current thread [seconds].
+// Does not return negative values.
+static __rte_always_inline int32_t clock_s(void) {
+	RTE_ASSERT(clock_snapshot_s != INT32_C(-1));
+	__rte_assume(clock_snapshot_s >= 0);
+	return clock_snapshot_s;
+}
+
+// (Internal)
+// Log that the clock was updated.
+// Do not call from dataplane worker thread.
+void _clock_log_update(const char *file, int line, const char *func);
+
+// (Internal)
+void _clock_update(void);
+
+// Update the clock for the current thread.
+#ifndef NDEBUG
+#define clock_update() do { \
+	if (unlikely(!clock_trusted)) \
+		_clock_log_update(__FILE__, __LINE__, __func__); \
+	_clock_update(); \
+} while (0)
+#else
+#define clock_update() _clock_update()
+#endif
+
+// Get the common clock [nanoseconds].
+// Like the gr_clock_ns() Grout Client API.
+static inline gr_clock_ns_t clock_common_ns(void) {
+	struct timespec tp = {0};
+	clock_gettime(GR_CLOCK_SOURCE, &tp);
+	gr_clock_ns_t ret = tp.tv_sec * GR_NS_PER_S + tp.tv_nsec;
+	__rte_assume(ret >= 0);
+	return ret;
+}
+
+static __rte_always_inline gr_clock_ns_t gr_clock_ns(void) {
+	gr_clock_ns_t ret = likely(clock_trusted) ? clock_ns() : clock_common_ns();
+	__rte_assume(ret >= 0);
+	return ret;
+}
diff --git a/main/meson.build b/main/meson.build
index 1abc8389..f089a9d2 100644
--- a/main/meson.build
+++ b/main/meson.build
@@ -3,6 +3,7 @@
 
 src += files(
   'api.c',
+  'clock.c',
   'config.c',
   'control_queue.c',
   'dpdk.c',
diff --git a/modules/infra/datapath/main_loop.c b/modules/infra/datapath/main_loop.c
index 2aae4030..562f9f2b 100644
--- a/modules/infra/datapath/main_loop.c
+++ b/modules/infra/datapath/main_loop.c
@@ -1,6 +1,8 @@
 // SPDX-License-Identifier: BSD-3-Clause
 // Copyright (c) 2023 Robin Jarry
+// Copyright (c) 2026 SmartShare Systems
 
+#include "clock.h"
 #include "config.h"
 #include "control_input.h"
 #include "datapath.h"
@@ -456,7 +458,10 @@ reconfig:
 	worker_active_inc();
 
 	for (;;) {
+		clock_trusted = true;
+		clock_update();
 		rte_graph_walk(graph);
+		clock_trusted = false;
 
 		if (++loop == HOUSEKEEPING_INTERVAL) {
 			// When RCU reclamation will be done in datapath workers,
-- 
2.43.0