[PATCH grout] add high-performance clock

[Morten Brørup]

The current clock is based on clock_gettime(CLOCK_MONOTONIC_RAW), which is
significantly slower than rte_rdtsc(), even though the kernel exposes it
as a vDSO.

CLOCK_MONOTONIC_RAW is typically based on and in sync with the TSC, so use
the faster rte_rdtsc() to read the clock when this is the case.

Also, introduce a per-thread snapshot of the clock for use in the
dataplane, where reading the snapshot is sufficiently accurate, and much
faster than reading the clock.

Signed-off-by: Morten Brørup <mb at smartsharesystems.com>
---
Note: Optimizations relying on the clock snapshot will be submitted later.
---
 api/gr_clock.h                     |  38 ++++++++++-
 main/clock.c                       | 103 +++++++++++++++++++++++++++++
 main/clock.h                       |  58 ++++++++++++++++
 main/meson.build                   |   1 +
 modules/infra/datapath/main_loop.c |   3 +
 5 files changed, 201 insertions(+), 2 deletions(-)
 create mode 100644 main/clock.c
 create mode 100644 main/clock.h

diff --git a/api/gr_clock.h b/api/gr_clock.h
index d2d98fba..c70a03a1 100644
--- a/api/gr_clock.h
+++ b/api/gr_clock.h
@@ -14,20 +14,54 @@
 // in calculations where race conditions may cause negative differences.
 typedef int64_t gr_clock_ns_t;
 
+#define GR_NS_PER_S (gr_clock_ns_t)INT64_C(1000000000)
+
+#ifdef __GROUT_MAIN__
+#include <rte_cycles.h>
+
+// Ref: main/clock.h
+extern uint64_t clock_tsc_hz;
+#endif
+
 // Get powered-on (non-suspended, non-hibernated) time since last boot,
 // using a common clock across all processes.
 static inline struct timespec gr_clock_raw(void) {
 	struct timespec tp = {0};
+#ifdef __GROUT_MAIN__
+	if (clock_tsc_hz != 0) {
+		const uint64_t tsc = rte_rdtsc();
+		tp.tv_sec = (tsc / clock_tsc_hz);
+		tp.tv_nsec = (tsc % clock_tsc_hz) * GR_NS_PER_S / clock_tsc_hz;
+	} else {
+		clock_gettime(CLOCK_MONOTONIC_RAW, &tp);
+	}
+	__rte_assume(tp.tv_sec >= 0);
+	__rte_assume(tp.tv_nsec >= 0);
+	return tp;
+#else
 	clock_gettime(CLOCK_MONOTONIC_RAW, &tp);
 	return tp;
+#endif
 }
 
-#define GR_NS_PER_S (gr_clock_ns_t)1000000000LL
-
 // Get powered-on (non-suspended, non-hibernated) time since last boot [nanoseconds],
 // using a common clock across all processes.
 // Does not return negative values.
 static inline gr_clock_ns_t gr_clock_ns(void) {
+#ifdef __GROUT_MAIN__
+	gr_clock_ns_t ret;
+	if (clock_tsc_hz != 0) {
+		const uint64_t tsc = rte_rdtsc();
+		ret = (gr_clock_ns_t)((tsc / clock_tsc_hz) * GR_NS_PER_S
+		      + (tsc % clock_tsc_hz) * GR_NS_PER_S / clock_tsc_hz);
+	} else {
+		struct timespec tp = gr_clock_raw();
+		ret = (gr_clock_ns_t)(tp.tv_sec * GR_NS_PER_S + tp.tv_nsec);
+	}
+	__rte_assume(ret >= 0);
+	return ret;
+#else
 	struct timespec tp = gr_clock_raw();
 	return tp.tv_sec * GR_NS_PER_S + tp.tv_nsec;
+#endif
 }
diff --git a/main/clock.c b/main/clock.c
new file mode 100644
index 00000000..19c2aa6c
--- /dev/null
+++ b/main/clock.c
@@ -0,0 +1,103 @@
+// SPDX-License-Identifier: BSD-3-Clause
+// Copyright (c) 2026 SmartShare Systems
+
+#include "clock.h"
+#include "log.h"
+#include "module.h"
+
+#include <fcntl.h>
+#include <time.h>
+#include <unistd.h>
+
+LOG_TYPE("clock");
+
+#define CLOCKSOURCE_PATH "/sys/devices/system/clocksource/clocksource0/current_clocksource"
+#if defined(RTE_ARCH_X86)
+#define CLOCKSOURCE_TSC "tsc"
+#elif defined(RTE_ARCH_ARM)
+#define CLOCKSOURCE_TSC "arch_timer"
+#endif
+
+RTE_DEFINE_PER_LCORE(uint64_t, clock_ns);
+RTE_DEFINE_PER_LCORE(uint32_t, clock_s);
+
+uint64_t clock_tsc_hz = 0;
+
+static __rte_always_inline void handler_tsc(void) {
+	// Considerations regarding types and boundaries, assuming a 10 GHz CPU:
+	// clock_tsc_hz = 10*10^9 > UINT32_MAX (~4.29*10^9): Requires uint64_t.
+	// max(tsc % clock_tsc_hz) * GR_NS_PER_S = ~9.99*10^18 < UINT64_MAX (~18.4*10^18): OK.
+	const uint64_t tsc = rte_rdtsc();
+	RTE_PER_LCORE(clock_ns) = (tsc / clock_tsc_hz) * GR_NS_PER_S
+	                          + (tsc % clock_tsc_hz) * GR_NS_PER_S / clock_tsc_hz;
+	RTE_PER_LCORE(clock_s) = (tsc / clock_tsc_hz);
+}
+
+static inline void handler_generic(void) {
+	struct timespec tp;
+	clock_gettime(CLOCK_MONOTONIC_RAW, &tp);
+	RTE_PER_LCORE(clock_ns) = (uint64_t)tp.tv_sec * GR_NS_PER_S + (uint64_t)tp.tv_nsec;
+	RTE_PER_LCORE(clock_s) = (uint32_t)tp.tv_sec;
+}
+
+void clock_update(void) {
+	if (likely(clock_tsc_hz != 0))
+		handler_tsc();
+	else
+		handler_generic();
+}
+
+static __rte_cold void clock_init(struct event_base *) {
+	int fd;
+	char clocksource[LINE_MAX + 1];
+	int len;
+
+	strcpy(clocksource, "unknown");
+
+	fd = open(CLOCKSOURCE_PATH, O_RDONLY);
+	if (fd == -1) {
+		LOG(ERR, "open(" CLOCKSOURCE_PATH "): %s", strerror(errno));
+		goto done;
+	}
+	len = read(fd, clocksource, LINE_MAX);
+	if (len == -1) {
+		LOG(ERR, "read(" CLOCKSOURCE_PATH "): %s", strerror(errno));
+		goto done;
+	}
+	if (len > 0 && clocksource[len - 1] == '\n')
+		len--;
+	clocksource[len] = 0;
+
+	if (strcmp(clocksource, CLOCKSOURCE_TSC) == 0)
+		clock_tsc_hz = rte_get_tsc_hz();
+
+done:
+	if (fd != -1)
+		close(fd);
+	LOG(DEBUG, "clock source: %s, handler: %s",
+		clocksource,
+		clock_tsc_hz != 0 ? "TSC" : "CLOCK_MONOTONIC_RAW");
+}
+
+static struct module clock_module = {
+	.name = "clock",
+	.init = clock_init,
+};
+
+RTE_INIT(clock_constructor) {
+	module_register(&clock_module);
+}
+
+// For review of compiled inline functions
+
+#include <gr_clock.h>
+
+struct timespec gr_clock_raw_review(void);
+struct timespec gr_clock_raw_review(void) {
+	return gr_clock_raw();
+}
+
+gr_clock_ns_t gr_clock_ns_review(void);
+gr_clock_ns_t gr_clock_ns_review(void) {
+	return gr_clock_ns();
+}
diff --git a/main/clock.h b/main/clock.h
new file mode 100644
index 00000000..c268f5e6
--- /dev/null
+++ b/main/clock.h
@@ -0,0 +1,58 @@
+// SPDX-License-Identifier: BSD-3-Clause
+// Copyright (c) 2026 SmartShare Systems
+
+#pragma once
+
+#include <gr_clock.h>
+
+#include <rte_common.h>
+#include <rte_cycles.h>
+#include <rte_per_lcore.h>
+
+// TSC frequency in Hz.
+//
+// If non-zero, the TSC is in sync with the common clock.
+// If zero, the TSC is out of sync with the common clock.
+extern uint64_t clock_tsc_hz;
+
+// Get common (monotonically increasing) clock from snapshot [nanoseconds].
+//
+// Resembles CLOCK_MONOTONIC_RAW:
+// - Pauses (does not increase) while the system is suspended or hibernated.
+// - Accurate for short intervals, where NTP adjustments would distort the measurement.
+// - Not accurate for long intervals. It drifts with hardware.
+// - - Drifts up to 4.3 seconds/day = 26 minutes/year. (Typical PC XTAL with 50 PPM accuracy.)
+//
+// Wraps around after hundreds of years.
+// Does not return negative values.
+//
+// Call clock_update() to update the clock snapshots for the current thread.
+static __rte_always_inline gr_clock_ns_t clock_ns(void) {
+	RTE_DECLARE_PER_LCORE(uint64_t, clock_ns);
+
+	const gr_clock_ns_t ret = (gr_clock_ns_t)RTE_PER_LCORE(clock_ns);
+	__rte_assume(ret >= 0);
+	return ret;
+}
+
+// Get common (monotonically increasing) clock from snapshot [seconds].
+//
+// Resembles CLOCK_MONOTONIC_RAW:
+// - Pauses (does not increase) while the system is suspended or hibernated.
+// - Not accurate for long intervals. It drifts with hardware.
+// - - Drifts up to 4.3 seconds/day = 26 minutes/year. (Typical PC XTAL with 50 PPM accuracy.)
+//
+// Wraps around after hundreds of years.
+// Does not return negative values.
+//
+// Call clock_update() to update the clock snapshots for the current thread.
+static __rte_always_inline int32_t clock_s(void) {
+	RTE_DECLARE_PER_LCORE(uint32_t, clock_s);
+
+	const int32_t ret = (int32_t)RTE_PER_LCORE(clock_s);
+	__rte_assume(ret >= 0);
+	return ret;
+}
+
+// Update the clock snapshots for the current thread.
+void clock_update(void);
diff --git a/main/meson.build b/main/meson.build
index a57d8600..f0823ff3 100644
--- a/main/meson.build
+++ b/main/meson.build
@@ -3,6 +3,7 @@
 
 src += files(
   'api.c',
+  'clock.c',
   'control_queue.c',
   'dpdk.c',
   'event.c',
diff --git a/modules/infra/datapath/main_loop.c b/modules/infra/datapath/main_loop.c
index f462cfbd..4127631d 100644
--- a/modules/infra/datapath/main_loop.c
+++ b/modules/infra/datapath/main_loop.c
@@ -1,6 +1,8 @@
 // SPDX-License-Identifier: BSD-3-Clause
 // Copyright (c) 2023 Robin Jarry
+// Copyright (c) 2026 SmartShare Systems
 
+#include "clock.h"
 #include "config.h"
 #include "datapath.h"
 #include "log.h"
@@ -258,6 +260,7 @@ reconfig:
 	sleep = 0;
 	timestamp = rte_rdtsc();
 	for (;;) {
+		clock_update();
 		rte_graph_walk(graph);
 
 		if (++loop == HOUSEKEEPING_INTERVAL) {
-- 
2.43.0