[RFC PATCH v2] api: reduce syscall overhead
Morten Brørup
mb at smartsharesystems.com
Sat Jul 18 17:36:22 CEST 2026
All API messages exhanged between Grout and clients comprise of a header
and optionally a payload.
Introduce buffers, so API messages are sent/received using fewer syscalls,
thereby reducing the transaction overhead and improving the performance.
Introduce ping API call, intended for testing Grout API response time.
Introduce ping2 API call and accompanying ping2_perf smoke test for
performance testing Grout's API transaction rate.
Signed-off-by: Morten Brørup <mb at smartsharesystems.com>
---
v2:
* Improve code readability. (Robin)
* API Client implementation: Add receive buffer.
* API Client implementation: Add send2_all() to send both message
header and payload.
* Add GR_PING and GR_PING2 API messages and functions.
* Add bufferevent_write2() to write both message header and payload.
* Add ping2_perf executable.
---
api/gr_api.h | 30 +++++++++++-
api/gr_api_client_impl.h | 101 +++++++++++++++++++++++++++++++++++----
main/api.c | 88 ++++++++++++++++++++++++++++------
meson.build | 6 +++
smoke/ping2_perf.c | 79 ++++++++++++++++++++++++++++++
5 files changed, 278 insertions(+), 26 deletions(-)
create mode 100644 smoke/ping2_perf.c
diff --git a/api/gr_api.h b/api/gr_api.h
index 0713d7bf..71d207d8 100644
--- a/api/gr_api.h
+++ b/api/gr_api.h
@@ -171,6 +171,8 @@ enum gr_main_requests : uint32_t {
GR_LOG_LEVEL_SET,
GR_EVENT_SUBSCRIBE,
GR_EVENT_UNSUBSCRIBE,
+ GR_PING,
+ GR_PING2,
};
// Client handshake with API version negotiation.
@@ -235,4 +237,30 @@ struct gr_api_event {
// Receive an event notification.
// Caller must free(*event) after use.
// Returns 0 on success, negative errno on failure.
-int gr_api_client_event_recv(const struct gr_api_client *, struct gr_api_event **);
+int gr_api_client_event_recv(struct gr_api_client *, struct gr_api_event **);
+
+// API Ping (without payload, i.e. message header only).
+GR_REQ(GR_PING, struct gr_empty, struct gr_empty);
+
+static inline int gr_api_client_ping(struct gr_api_client *c)
+{
+ return gr_api_client_send_recv(c, GR_PING, 0, NULL, NULL);
+}
+
+// API Ping with payload.
+struct gr_ping2 {
+ char payload[64];
+};
+
+GR_REQ(GR_PING2, struct gr_ping2, struct gr_ping2);
+
+// Send an API Ping with payload request and receive the response.
+// Returns 0 on success, negative errno on failure.
+static inline int gr_api_client_ping2(struct gr_api_client *c)
+{
+ void *resp = NULL;
+ alignas(64) struct gr_ping2 req = { 0 };
+ int ret = gr_api_client_send_recv(c, GR_PING2, sizeof(req), &req, &resp);
+ free(resp);
+ return ret;
+}
diff --git a/api/gr_api_client_impl.h b/api/gr_api_client_impl.h
index 242b286d..ef39d17a 100644
--- a/api/gr_api_client_impl.h
+++ b/api/gr_api_client_impl.h
@@ -104,8 +104,24 @@ struct response {
STAILQ_ENTRY(response) next;
};
+// Receive buffer size.
+// Optimization:
+// Messages received from Grout (e.g. responses and events) comprise of a header
+// (containing information about the payload length) and an optional payload.
+// Read-ahead into a buffer, so receiving small messages only uses one recv() syscall,
+// instead of reading the header and payload separately (using two recv() syscalls).
+// Furthermore, when multiple small messages are ready for receiving,
+// read-ahead may reduce the number of recv() syscalls even more.
+// 0 disables this optimization.
+#define GR_API_RECV_BUFFER 4096
+
struct gr_api_client {
int sock_fd;
+#if GR_API_RECV_BUFFER
+ size_t recv_buf_off;
+ size_t recv_buf_len;
+ char recv_buf[GR_API_RECV_BUFFER];
+#endif
STAILQ_HEAD(, response) responses;
};
@@ -177,22 +193,90 @@ static ssize_t send_all(const struct gr_api_client *c, const void *buf, size_t l
return len;
}
-static ssize_t recv_all(const struct gr_api_client *c, void *buf, size_t len) {
+// Send buffer size.
+// Optimization:
+// Messages sent to Grout (e.g. requets) comprise of a header
+// (containing information about the payload length) and an optional payload.
+// For small payloads, copy the request header and the payload into a buffer,
+// and send the buffer using one send() syscall,
+// instead of sending the header a payload separately (using two send() syscalls).
+// 0 disables this optimization.
+#define GR_API_SEND2_BUFFER 4096
+
+static ssize_t send2_all(
+ const struct gr_api_client *c,
+ const void *hdr,
+ size_t hdr_size,
+ const void *payload,
+ size_t payload_len
+) {
+ if (payload_len == 0)
+ return send_all(c, hdr, hdr_size);
+
+#if GR_API_SEND2_BUFFER
+ if (hdr_size + payload_len <= GR_API_SEND2_BUFFER) {
+ char buf[GR_API_SEND2_BUFFER];
+ memcpy(buf, hdr, hdr_size);
+ memcpy(buf + hdr_size, payload, payload_len);
+ return send_all(c, buf, hdr_size + payload_len);
+ }
+#endif
+
+ ssize_t ret;
+ if ((ret = send_all(c, hdr, hdr_size)) < 0)
+ return ret;
+ if ((ret = send_all(c, payload, payload_len)) < 0)
+ return ret;
+ return hdr_size + payload_len;
+}
+
+static ssize_t recv_all(struct gr_api_client *c, void *buf, size_t len) {
size_t remaining = len;
char *ptr = buf;
ssize_t n;
while (remaining > 0) {
- n = recv(c->sock_fd, ptr, remaining, 0);
+#if GR_API_RECV_BUFFER
+ // Receive from the buffer, if anything in it
+ if (c->recv_buf_len > 0) {
+ if (remaining <= c->recv_buf_len) {
+ memcpy(ptr, c->recv_buf + c->recv_buf_off, remaining);
+ c->recv_buf_off += remaining;
+ c->recv_buf_len -= remaining;
+ return len;
+ }
+
+ memcpy(ptr, c->recv_buf + c->recv_buf_off, c->recv_buf_len);
+ ptr += c->recv_buf_len;
+ remaining -= c->recv_buf_len;
+ c->recv_buf_off = 0;
+ c->recv_buf_len = 0;
+ }
+
+ // Receive from the socket; either into the buffer, or directly to the client
+ if (remaining < GR_API_RECV_BUFFER)
+ n = recv(c->sock_fd, c->recv_buf, GR_API_RECV_BUFFER, 0);
+ else
+#endif
+ n = recv(c->sock_fd, ptr, remaining, 0);
if (n == 0) {
errno = ECONNRESET;
return len - remaining;
} else if (n < 0) {
return n;
}
-
- ptr += n;
- remaining -= n;
+#if GR_API_RECV_BUFFER
+ if (remaining < GR_API_RECV_BUFFER) {
+ // Received into the buffer
+ c->recv_buf_len = n;
+ } else {
+#endif
+ // Received directly to the client
+ ptr += n;
+ remaining -= n;
+#if GR_API_RECV_BUFFER
+ }
+#endif
}
return len;
@@ -215,10 +299,7 @@ long int gr_api_client_send(
.type = req_type,
};
- if (send_all(client, &req, sizeof(req)) < 0)
- return -errno;
-
- if (tx_len > 0 && send_all(client, tx_data, tx_len) < 0)
+ if (send2_all(client, &req, sizeof(req), tx_data, tx_len) < 0)
return -errno;
return req.id;
@@ -306,7 +387,7 @@ err:
return -errno;
}
-int gr_api_client_event_recv(const struct gr_api_client *c, struct gr_api_event **event) {
+int gr_api_client_event_recv(struct gr_api_client *c, struct gr_api_event **event) {
const struct api_message *m;
struct gr_api_event header;
diff --git a/main/api.c b/main/api.c
index 023c9002..1a1809af 100644
--- a/main/api.c
+++ b/main/api.c
@@ -37,6 +37,57 @@ static pid_t socket_pid(int fd) {
return cred.pid;
}
+// Merge the header and payload into one event buffer when writing to the event library.
+// 0 disables this optimization.
+#define BUFFEREVENT_WRITE2_MERGE 1
+
+static int bufferevent_write2(
+ struct bufferevent *bev,
+ const void *hdr,
+ size_t hdr_size,
+ const void *payload,
+ size_t payload_len
+) {
+ assert(hdr != NULL);
+ assert(hdr_size != 0);
+ assert(payload_len == 0 || payload != NULL);
+
+#if BUFFEREVENT_WRITE2_MERGE
+ if (payload_len == 0)
+ return bufferevent_write(bev, hdr, hdr_size);
+
+ // Optimization:
+ // Pass the header and the payload as one event buffer to the event library,
+ // so the write event callback - which uses the write() syscall - is only invoked once,
+ // instead of passing them separately (invoking the write event callback twice).
+ // Note:
+ // Creating the event buffer, filling it, and pasing it to the event library roughly mimics
+ // what the event library would do internally if calling bufferevent_write() twice.
+ // The key difference is that the write event callback is only invoked once.
+ int ret = -1;
+ struct evbuffer *buf;
+ if ((buf = evbuffer_new()) == NULL)
+ goto done;
+ if (evbuffer_add(buf, hdr, hdr_size) < 0)
+ goto done;
+ if (evbuffer_add(buf, payload, payload_len) < 0)
+ goto done;
+ if (bufferevent_write_buffer(bev, buf) < 0)
+ goto done;
+ ret = 0;
+done:
+ if (buf != NULL)
+ evbuffer_free(buf); // Note: Reduces the refcnt, frees the buffer when zero.
+ return ret;
+#else
+ if (bufferevent_write(bev, hdr, hdr_size) < 0)
+ return -1;
+ if (payload_len > 0 && bufferevent_write(bev, payload, payload_len) < 0)
+ return -1;
+ return 0;
+#endif
+};
+
struct subscription {
bool suppress_self_events;
struct bufferevent *bev;
@@ -118,14 +169,12 @@ void api_send_notifications(uint32_t ev_type, const void *obj) {
vec_foreach_ref (s, all_events_subs) {
if (s->suppress_self_events && s->pid == cur_req_pid)
continue;
- bufferevent_write(s->bev, &e, sizeof(e));
- bufferevent_write(s->bev, data, len);
+ bufferevent_write2(s->bev, &e, sizeof(e), data, len);
}
vec_foreach_ref (s, ev_subs) {
if (s->suppress_self_events && s->pid == cur_req_pid)
continue;
- bufferevent_write(s->bev, &e, sizeof(e));
- bufferevent_write(s->bev, data, len);
+ bufferevent_write2(s->bev, &e, sizeof(e), data, len);
}
free(data);
@@ -218,6 +267,18 @@ static struct api_out hello(const void *request, struct api_ctx *) {
return api_out(0, 0, NULL);
}
+static struct api_out ping(const void * /*request*/, struct api_ctx *) {
+ return api_out(0, 0, NULL);
+}
+
+static struct api_out ping2(const void *request, struct api_ctx *) {
+ void * resp = malloc(sizeof(struct gr_ping2));
+ if (resp == NULL)
+ return api_out(ENOMEM, 0, NULL);
+ memcpy(resp, request, sizeof(struct gr_ping2));
+ return api_out(0, sizeof(struct gr_ping2), resp);
+}
+
static void disconnect_client(struct api_ctx *ctx) {
assert(ctx != NULL);
assert(ctx->bev != NULL);
@@ -243,10 +304,8 @@ void api_send(struct api_ctx *ctx, uint32_t len, const void *payload) {
.payload_len = len,
.status = 0,
};
- if (bufferevent_write(ctx->bev, &resp, sizeof(resp)) < 0)
- LOG(ERR, "pid=%d cannot write header", ctx->pid);
- if (bufferevent_write(ctx->bev, payload, len) < 0)
- LOG(ERR, "pid=%d cannot write payload", ctx->pid);
+ if (bufferevent_write2(ctx->bev, &resp, sizeof(resp), payload, len) < 0)
+ LOG(ERR, "pid=%d cannot write message", ctx->pid);
}
static void read_cb(struct bufferevent *bev, void *priv) {
@@ -327,19 +386,16 @@ send:
strerror(out.status),
out.len);
+ assert(out.len == 0 || out.payload != NULL);
+
struct gr_api_response resp = {
.for_id = ctx->header.id,
.status = out.status,
.payload_len = out.len,
};
- if (bufferevent_write(bev, &resp, sizeof(resp)) < 0)
- LOG(ERR, "failed to write header");
- if (out.len > 0) {
- assert(out.payload != NULL);
- if (bufferevent_write(bev, out.payload, out.len) < 0)
- LOG(ERR, "failed to write payload");
- }
+ if (bufferevent_write2(bev, &resp, sizeof(resp), out.payload, out.len) < 0)
+ LOG(ERR, "pid=%d cannot write response", ctx->pid);
bufferevent_flush(bev, EV_WRITE, BEV_FLUSH);
@@ -466,4 +522,6 @@ RTE_INIT(init) {
api_handler(GR_EVENT_SUBSCRIBE, subscribe);
api_handler(GR_EVENT_UNSUBSCRIBE, unsubscribe);
api_handler(GR_HELLO, hello);
+ api_handler(GR_PING, ping);
+ api_handler(GR_PING2, ping2);
}
diff --git a/meson.build b/meson.build
index 073e3243..1e5f51a5 100644
--- a/meson.build
+++ b/meson.build
@@ -201,6 +201,12 @@ executable(
install: false,
)
+executable(
+ 'ping2_perf', files('smoke/ping2_perf.c') + grout_header,
+ include_directories: api_inc,
+ install: false,
+)
+
# docs/ must come after grcli_exe since man pages are generated using grcli --man
subdir('docs')
diff --git a/smoke/ping2_perf.c b/smoke/ping2_perf.c
new file mode 100644
index 00000000..060d4a6f
--- /dev/null
+++ b/smoke/ping2_perf.c
@@ -0,0 +1,79 @@
+// SPDX-License-Identifier: BSD-3-Clause
+// Copyright (c) 2026 SmartShare Systems
+
+// clang-format off
+#include <gr_api_client_impl.h>
+// clang-format on
+
+#include <getopt.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+static int ping2(struct gr_api_client *c, uint32_t count) {
+ void *resp;
+ alignas(64) struct gr_ping2 req = { 0 };
+
+ for (uint32_t i = 0; i < count; i++) {
+ resp = NULL;
+ if (gr_api_client_send_recv(c, GR_PING2, sizeof(req), &req, &resp) < 0) {
+ perror("GR_PING2");
+ return -1;
+ }
+ free(resp);
+ }
+
+ return 0;
+}
+
+static void usage(const char *prog) {
+ fprintf(stderr, "Usage: %s [-s SOCK] [-n COUNT]\n", prog);
+ fprintf(stderr, " -s SOCK API socket path (default: $GROUT_SOCK_PATH)\n");
+ fprintf(stderr, " -n COUNT Number of ping2 calls (default: 10000)\n");
+}
+
+int main(int argc, char **argv) {
+ const char *sock_path = getenv("GROUT_SOCK_PATH");
+ struct gr_api_client *c;
+ unsigned int count = 10000;
+ gr_clock_ns_t time;
+ float duration;
+ int ret;
+ int o;
+
+ while ((o = getopt(argc, argv, "s:n:h")) != -1) {
+ switch (o) {
+ case 's':
+ sock_path = optarg;
+ break;
+ case 'n':
+ count = strtoul(optarg, NULL, 10);
+ break;
+ case 'h':
+ default:
+ usage(argv[0]);
+ return o == 'h' ? EXIT_SUCCESS : EXIT_FAILURE;
+ }
+ }
+ if (sock_path == NULL)
+ sock_path = GR_DEFAULT_SOCK_PATH;
+
+ c = gr_api_client_connect(sock_path);
+ if (c == NULL) {
+ perror("gr_api_client_connect");
+ return EXIT_FAILURE;
+ }
+
+ printf("performing %u ping2 calls\n",
+ count);
+
+ time = gr_clock_ns();
+ ret = ping2(c, count);
+ duration = (float)(gr_clock_ns() - time) / (float)GR_NS_PER_S;
+
+ printf("total time: %.1f s (%.1f calls/s)\n", duration, (float)count / duration);
+
+ gr_api_client_disconnect(c);
+
+ return ret < 0 ? EXIT_FAILURE : EXIT_SUCCESS;
+}
--
2.43.0
More information about the grout
mailing list