[PATCH v6] add high-performance clock

[Morten Brørup]

Reading the common clock, gr_clock_ns(), is implemented as
clock_gettime(), which is quite slow, even though the kernel exposes it
as a vDSO.

For improved performance, especially in the dataplane, introduce a high-
performance clock, based on snapshotting the common clock, for use where
a snapshot is sufficiently accurate.

Immediately before walking the graph in the dataplane worker thread's main
loop, mark the clock as trusted and update it, so it is safe to use in the
"process" functions of the graph nodes.

Define GR_CLOCK_SOURCE (as CLOCK_MONOTONIC_RAW), to highlight which clock
source is used by Grout.

Add definitions like GR_NS_PER_S to help convert between nanoseconds and
microseconds (GR_NS_PER_US) respectively milliseconds (GR_NS_PER_MS).

Signed-off-by: Morten Brørup <mb at smartsharesystems.com>
---
v6:
* Remove the functions for review of compiled inline functions;
  they are not used. (Robin)
* Remove the clock with one second resolution; it is unused. (Robin)
* Replace RTE_ASSERT() with standard assert(). (Robin)
* Remove __rte_assume() hints to reduce use of DPDK code. (Robin)
* Add clock_set_trusted() function to set clock trust. (Robin)
v5:
* More header inclusion cleaning.
* Rebased.
v4:
* Clean up header inclusion order.
  clock.h should only be included by gr_clock.h, and thus
  clock.c should include gr_clock.h instead of clock.h.
v3:
* Change gr_clock_ns(), adding a Grout specific implementation using
  the high-performance clock. This significantly reduces the amount of
  required changes inside Grout. (Robin)
* Revert upgrade some graph node "process" functions to use
  this clock; they can simply continue using gr_clock_ns().
* Add an explicit per-thread state variable to mark the clock as trusted,
  effectively enabling/disabling use of the high-performance clock.
* Remove unused gr_clock_raw() API.
* Add definition of Grout clock source.
v2:
* Eliminate the concept of a client API message required to read Grout's
  clock. (Robin)
* Keep gr_clock_ns() as is.
* For clock source, use gr_clock_ns() instead of rte_rdtsc().
* Add instrumentation for debug builds.
* Use __thread instead of RTE_PER_LCORE.
  This aligns with Grout's existing code style.
---
 api/gr_clock.h                     | 24 ++++++++------
 main/clock.c                       | 27 ++++++++++++++++
 main/clock.h                       | 52 ++++++++++++++++++++++++++++++
 main/meson.build                   |  1 +
 modules/infra/datapath/main_loop.c |  6 ++++
 5 files changed, 100 insertions(+), 10 deletions(-)
 create mode 100644 main/clock.c
 create mode 100644 main/clock.h

diff --git a/api/gr_clock.h b/api/gr_clock.h
index d2d98fba..d10f2290 100644
--- a/api/gr_clock.h
+++ b/api/gr_clock.h
@@ -7,27 +7,31 @@
 #include <stdint.h>
 #include <time.h>
 
+// Clock source.
+// Must be a high-resolution clock, i.e. not a _COARSE variant.
+#define GR_CLOCK_SOURCE CLOCK_MONOTONIC_RAW
+
 // High-resolution clock [nanoseconds].
-// Used with CLOCK_MONOTONIC_RAW, unless otherwise specified.
+// Used with GR_CLOCK_SOURCE, unless otherwise specified.
 // Note: Does not have Y2038 problems. Not even with CLOCK_REALTIME.
 // Note: Using signed, to avoid need for casting to signed
 // in calculations where race conditions may cause negative differences.
 typedef int64_t gr_clock_ns_t;
 
-// Get powered-on (non-suspended, non-hibernated) time since last boot,
-// using a common clock across all processes.
-static inline struct timespec gr_clock_raw(void) {
-	struct timespec tp = {0};
-	clock_gettime(CLOCK_MONOTONIC_RAW, &tp);
-	return tp;
-}
-
 #define GR_NS_PER_S (gr_clock_ns_t)1000000000LL
+#define GR_NS_PER_MS (gr_clock_ns_t)1000000LL
+#define GR_NS_PER_US (gr_clock_ns_t)1000LL
 
 // Get powered-on (non-suspended, non-hibernated) time since last boot [nanoseconds],
 // using a common clock across all processes.
 // Does not return negative values.
+#ifndef __GROUT_MAIN__
 static inline gr_clock_ns_t gr_clock_ns(void) {
-	struct timespec tp = gr_clock_raw();
+	struct timespec tp = {0};
+	clock_gettime(GR_CLOCK_SOURCE, &tp);
 	return tp.tv_sec * GR_NS_PER_S + tp.tv_nsec;
 }
+#else
+// Grout implementation is in main/clock.h
+#include "clock.h"
+#endif
diff --git a/main/clock.c b/main/clock.c
new file mode 100644
index 00000000..249a63e8
--- /dev/null
+++ b/main/clock.c
@@ -0,0 +1,27 @@
+// SPDX-License-Identifier: BSD-3-Clause
+// Copyright (c) 2026 SmartShare Systems
+
+#include "log.h"
+#include "module.h"
+
+#include <gr_clock.h>
+
+LOG_TYPE("clock");
+
+__thread bool clock_trusted = false;
+
+__thread gr_clock_ns_t clock_snapshot_ns = INT64_C(-1);
+
+static __rte_cold void clock_init(struct event_base *) {
+	// Update the clock for the main thread.
+	clock_update();
+}
+
+static struct module clock_module = {
+	.name = "clock",
+	.init = clock_init,
+};
+
+RTE_INIT(clock_constructor) {
+	module_register(&clock_module);
+}
diff --git a/main/clock.h b/main/clock.h
new file mode 100644
index 00000000..cfa0412f
--- /dev/null
+++ b/main/clock.h
@@ -0,0 +1,52 @@
+// SPDX-License-Identifier: BSD-3-Clause
+// Copyright (c) 2026 SmartShare Systems
+
+#pragma once
+
+#ifndef GR_CLOCK_SOURCE
+#error "clock.h must not be included directly, include <gr_clock.h> instead"
+#endif
+
+#include <rte_branch_prediction.h>
+
+#include <assert.h>
+
+// (Internal)
+// The clock can be trusted.
+// When true, assume that the clock is updated for the current thread.
+extern __thread bool clock_trusted;
+
+// (Internal)
+// The clock.
+// Implemented as per-thread snapshot of the common clock.
+// Note: Per thread to avoid certain race conditions.
+extern __thread gr_clock_ns_t clock_snapshot_ns;
+
+// Get the common clock [nanoseconds].
+// Like the gr_clock_ns() Grout Client API.
+// Does not return negative values.
+static inline gr_clock_ns_t clock_common_ns(void) {
+	struct timespec tp = {0};
+	clock_gettime(GR_CLOCK_SOURCE, &tp);
+	return tp.tv_sec * GR_NS_PER_S + tp.tv_nsec;
+}
+
+// Get the clock for the current thread [nanoseconds].
+// Does not return negative values.
+static inline gr_clock_ns_t clock_ns(void) {
+	assert(clock_snapshot_ns >= 0);
+	return clock_snapshot_ns;
+}
+
+// Update the clock for the current thread.
+static inline void clock_update(void) {
+	clock_snapshot_ns = clock_common_ns();
+}
+
+static inline void clock_set_trusted(bool trusted) {
+	clock_trusted = trusted;
+}
+
+static inline gr_clock_ns_t gr_clock_ns(void) {
+	return likely(clock_trusted) ? clock_ns() : clock_common_ns();
+}
diff --git a/main/meson.build b/main/meson.build
index 1abc8389..f089a9d2 100644
--- a/main/meson.build
+++ b/main/meson.build
@@ -3,6 +3,7 @@
 
 src += files(
   'api.c',
+  'clock.c',
   'config.c',
   'control_queue.c',
   'dpdk.c',
diff --git a/modules/infra/datapath/main_loop.c b/modules/infra/datapath/main_loop.c
index 2aae4030..c9b6ec16 100644
--- a/modules/infra/datapath/main_loop.c
+++ b/modules/infra/datapath/main_loop.c
@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: BSD-3-Clause
 // Copyright (c) 2023 Robin Jarry
+// Copyright (c) 2026 SmartShare Systems
 
 #include "config.h"
 #include "control_input.h"
@@ -11,6 +12,8 @@
 #include "vec.h"
 #include "worker.h"
 
+#include <gr_clock.h>
+
 #include <rte_common.h>
 #include <rte_eal.h>
 #include <rte_errno.h>
@@ -456,7 +459,10 @@ reconfig:
 	worker_active_inc();
 
 	for (;;) {
+		clock_set_trusted(true);
+		clock_update();
 		rte_graph_walk(graph);
+		clock_set_trusted(false);
 
 		if (++loop == HOUSEKEEPING_INTERVAL) {
 			// When RCU reclamation will be done in datapath workers,
-- 
2.43.0