[PATCH v3] api: reduce syscall overhead
Morten Brørup
mb at smartsharesystems.com
Tue Jul 21 00:49:24 CEST 2026
All API messages exhanged between Grout and clients comprise of a header
and optionally a payload.
Introduce buffers, so API messages are sent/received using fewer syscalls,
thereby reducing the transaction overhead and improving the performance.
This increases the API transaction rate by ca. 10 %,
when tested on a 4-core virtual machine using the ping2_perf smoke test.
Introduce ping API call, intended for testing Grout API response time.
Introduce ping2 API call and accompanying ping2_perf smoke test for
performance testing Grout's API transaction rate.
Signed-off-by: Morten Brørup <mb at smartsharesystems.com>
---
v3:
* Bugfix: API Client implementation:
When receiving to the receive buffer, its offset must be reset.
* Remove alignas(64) for improved code readbility, they are not required.
* Reduce ping2 payload size.
* Reduce buffer sizes from 4096 to 1024.
* Replace new event buffer with simple buffer and memcpy().
v2:
* Improve code readability. (Robin)
* API Client implementation: Add receive buffer.
* API Client implementation: Add send2_all() to send both message
header and payload.
* Add GR_PING and GR_PING2 API messages and functions.
* Add bufferevent_write2() to write both message header and payload.
* Add ping2_perf executable.
---
api/gr_api.h | 30 +++++++++++-
api/gr_api_client_impl.h | 98 ++++++++++++++++++++++++++++++++++++----
main/api.c | 75 ++++++++++++++++++++++++------
meson.build | 6 +++
smoke/ping2_perf.c | 79 ++++++++++++++++++++++++++++++++
5 files changed, 262 insertions(+), 26 deletions(-)
create mode 100644 smoke/ping2_perf.c
diff --git a/api/gr_api.h b/api/gr_api.h
index eaa8c80e..10d3f746 100644
--- a/api/gr_api.h
+++ b/api/gr_api.h
@@ -171,6 +171,8 @@ enum gr_main_requests : uint32_t {
GR_LOG_LEVEL_SET,
GR_EVENT_SUBSCRIBE,
GR_EVENT_UNSUBSCRIBE,
+ GR_PING,
+ GR_PING2,
};
// Client handshake with API version negotiation.
@@ -235,4 +237,30 @@ struct gr_api_event {
// Receive an event notification.
// Caller must free(*event) after use.
// Returns 0 on success, negative errno on failure.
-int gr_api_client_event_recv(const struct gr_api_client *, struct gr_api_event **);
+int gr_api_client_event_recv(struct gr_api_client *, struct gr_api_event **);
+
+// API Ping (without payload, i.e. message header only).
+GR_REQ(GR_PING, struct gr_empty, struct gr_empty);
+
+static inline int gr_api_client_ping(struct gr_api_client *c)
+{
+ return gr_api_client_send_recv(c, GR_PING, 0, NULL, NULL);
+}
+
+// API Ping with payload.
+struct gr_ping2 {
+ char payload[32];
+};
+
+GR_REQ(GR_PING2, struct gr_ping2, struct gr_ping2);
+
+// Send an API Ping with payload request and receive the response.
+// Returns 0 on success, negative errno on failure.
+static inline int gr_api_client_ping2(struct gr_api_client *c)
+{
+ void *resp = NULL;
+ const struct gr_ping2 req = { 0 };
+ int ret = gr_api_client_send_recv(c, GR_PING2, sizeof(req), &req, &resp);
+ free(resp);
+ return ret;
+}
diff --git a/api/gr_api_client_impl.h b/api/gr_api_client_impl.h
index 242b286d..d1e7347f 100644
--- a/api/gr_api_client_impl.h
+++ b/api/gr_api_client_impl.h
@@ -104,8 +104,24 @@ struct response {
STAILQ_ENTRY(response) next;
};
+// Receive buffer size.
+// Optimization:
+// Messages received from Grout (e.g. responses and events) comprise of a header
+// (containing information about the payload length) and an optional payload.
+// Read-ahead into a buffer, so receiving small messages only uses one recv() syscall,
+// instead of reading the header and payload separately (using two recv() syscalls).
+// Furthermore, when multiple small messages are ready for receiving,
+// read-ahead may reduce the number of recv() syscalls even more.
+// 0 disables this optimization.
+#define GR_API_RECV_BUFFER 1024
+
struct gr_api_client {
int sock_fd;
+#if GR_API_RECV_BUFFER
+ size_t recv_buf_off;
+ size_t recv_buf_len;
+ char recv_buf[GR_API_RECV_BUFFER];
+#endif
STAILQ_HEAD(, response) responses;
};
@@ -177,22 +193,87 @@ static ssize_t send_all(const struct gr_api_client *c, const void *buf, size_t l
return len;
}
-static ssize_t recv_all(const struct gr_api_client *c, void *buf, size_t len) {
+// Send buffer size.
+// Optimization:
+// Messages sent to Grout (e.g. requets) comprise of a header
+// (containing information about the payload length) and an optional payload.
+// For small payloads, copy the request header and the payload into a buffer,
+// and send the buffer using one send() syscall,
+// instead of sending the header a payload separately (using two send() syscalls).
+// 0 disables this optimization.
+#define GR_API_SEND2_BUFFER 1024
+
+static ssize_t send2_all(
+ const struct gr_api_client *c,
+ const void *hdr,
+ size_t hdr_size,
+ const void *payload,
+ size_t payload_len
+) {
+#if GR_API_SEND2_BUFFER
+ if (payload_len > 0 && hdr_size + payload_len <= GR_API_SEND2_BUFFER) {
+ char buf[GR_API_SEND2_BUFFER];
+ memcpy(buf, hdr, hdr_size);
+ memcpy(buf + hdr_size, payload, payload_len);
+ return send_all(c, buf, hdr_size + payload_len);
+ }
+#endif
+
+ ssize_t ret;
+ if ((ret = send_all(c, hdr, hdr_size)) < 0)
+ return ret;
+ if (payload_len > 0 && (ret = send_all(c, payload, payload_len)) < 0)
+ return ret;
+ return hdr_size + payload_len;
+}
+
+static ssize_t recv_all(struct gr_api_client *c, void *buf, size_t len) {
size_t remaining = len;
char *ptr = buf;
ssize_t n;
while (remaining > 0) {
- n = recv(c->sock_fd, ptr, remaining, 0);
+#if GR_API_RECV_BUFFER
+ // Receive from the buffer, if anything in it
+ if (c->recv_buf_len > 0) {
+ if (remaining <= c->recv_buf_len) {
+ memcpy(ptr, c->recv_buf + c->recv_buf_off, remaining);
+ c->recv_buf_off += remaining;
+ c->recv_buf_len -= remaining;
+ return len;
+ }
+
+ memcpy(ptr, c->recv_buf + c->recv_buf_off, c->recv_buf_len);
+ ptr += c->recv_buf_len;
+ remaining -= c->recv_buf_len;
+ c->recv_buf_len = 0;
+ }
+
+ // Receive from the socket; either into the buffer, or directly to the client
+ if (remaining < GR_API_RECV_BUFFER) {
+ c->recv_buf_off = 0;
+ n = recv(c->sock_fd, c->recv_buf, GR_API_RECV_BUFFER, 0);
+ } else
+#endif
+ n = recv(c->sock_fd, ptr, remaining, 0);
if (n == 0) {
errno = ECONNRESET;
return len - remaining;
} else if (n < 0) {
return n;
}
-
- ptr += n;
- remaining -= n;
+#if GR_API_RECV_BUFFER
+ if (remaining < GR_API_RECV_BUFFER) {
+ // Received into the buffer
+ c->recv_buf_len = n;
+ } else {
+#endif
+ // Received directly to the client
+ ptr += n;
+ remaining -= n;
+#if GR_API_RECV_BUFFER
+ }
+#endif
}
return len;
@@ -215,10 +296,7 @@ long int gr_api_client_send(
.type = req_type,
};
- if (send_all(client, &req, sizeof(req)) < 0)
- return -errno;
-
- if (tx_len > 0 && send_all(client, tx_data, tx_len) < 0)
+ if (send2_all(client, &req, sizeof(req), tx_data, tx_len) < 0)
return -errno;
return req.id;
@@ -306,7 +384,7 @@ err:
return -errno;
}
-int gr_api_client_event_recv(const struct gr_api_client *c, struct gr_api_event **event) {
+int gr_api_client_event_recv(struct gr_api_client *c, struct gr_api_event **event) {
const struct api_message *m;
struct gr_api_event header;
diff --git a/main/api.c b/main/api.c
index 023c9002..4b79cb19 100644
--- a/main/api.c
+++ b/main/api.c
@@ -37,6 +37,44 @@ static pid_t socket_pid(int fd) {
return cred.pid;
}
+// Send buffer size.
+// Optimization:
+// Messages sent to the Client (e.g. responses and events) comprise of a header
+// (containing information about the payload length) and an optional payload.
+// For small payloads, copy the request header and the payload into a buffer,
+// and write the buffer using one bufferevent_write() call,
+// so the write event callback - which uses the write() syscall - is only invoked once,
+// instead of writing them separately (invoking the write event callback twice).
+// 0 disables this optimization.
+#define BUFFEREVENT_WRITE2_BUFFER 1024
+
+static inline int bufferevent_write2(
+ struct bufferevent *bev,
+ const void *hdr,
+ size_t hdr_size,
+ const void *payload,
+ size_t payload_len
+) {
+ assert(hdr != NULL);
+ assert(hdr_size != 0);
+ assert(payload_len == 0 || payload != NULL);
+
+#if BUFFEREVENT_WRITE2_BUFFER
+ if (payload_len > 0 && hdr_size + payload_len <= BUFFEREVENT_WRITE2_BUFFER) {
+ char buf[BUFFEREVENT_WRITE2_BUFFER];
+ memcpy(buf, hdr, hdr_size);
+ memcpy(buf + hdr_size, payload, payload_len);
+ return bufferevent_write(bev, buf, hdr_size + payload_len);
+ }
+#endif
+
+ if (bufferevent_write(bev, hdr, hdr_size) < 0)
+ return -1;
+ if (payload_len > 0 && bufferevent_write(bev, payload, payload_len) < 0)
+ return -1;
+ return 0;
+};
+
struct subscription {
bool suppress_self_events;
struct bufferevent *bev;
@@ -118,14 +156,12 @@ void api_send_notifications(uint32_t ev_type, const void *obj) {
vec_foreach_ref (s, all_events_subs) {
if (s->suppress_self_events && s->pid == cur_req_pid)
continue;
- bufferevent_write(s->bev, &e, sizeof(e));
- bufferevent_write(s->bev, data, len);
+ bufferevent_write2(s->bev, &e, sizeof(e), data, len);
}
vec_foreach_ref (s, ev_subs) {
if (s->suppress_self_events && s->pid == cur_req_pid)
continue;
- bufferevent_write(s->bev, &e, sizeof(e));
- bufferevent_write(s->bev, data, len);
+ bufferevent_write2(s->bev, &e, sizeof(e), data, len);
}
free(data);
@@ -218,6 +254,18 @@ static struct api_out hello(const void *request, struct api_ctx *) {
return api_out(0, 0, NULL);
}
+static struct api_out ping(const void * /*request*/, struct api_ctx *) {
+ return api_out(0, 0, NULL);
+}
+
+static struct api_out ping2(const void *request, struct api_ctx *) {
+ struct gr_ping2 *resp = malloc(sizeof(*resp));
+ if (resp == NULL)
+ return api_out(ENOMEM, 0, NULL);
+ *resp = *(const struct gr_ping2 *)request;
+ return api_out(0, sizeof(*resp), resp);
+}
+
static void disconnect_client(struct api_ctx *ctx) {
assert(ctx != NULL);
assert(ctx->bev != NULL);
@@ -243,10 +291,8 @@ void api_send(struct api_ctx *ctx, uint32_t len, const void *payload) {
.payload_len = len,
.status = 0,
};
- if (bufferevent_write(ctx->bev, &resp, sizeof(resp)) < 0)
- LOG(ERR, "pid=%d cannot write header", ctx->pid);
- if (bufferevent_write(ctx->bev, payload, len) < 0)
- LOG(ERR, "pid=%d cannot write payload", ctx->pid);
+ if (bufferevent_write2(ctx->bev, &resp, sizeof(resp), payload, len) < 0)
+ LOG(ERR, "pid=%d cannot write message", ctx->pid);
}
static void read_cb(struct bufferevent *bev, void *priv) {
@@ -327,19 +373,16 @@ send:
strerror(out.status),
out.len);
+ assert(out.len == 0 || out.payload != NULL);
+
struct gr_api_response resp = {
.for_id = ctx->header.id,
.status = out.status,
.payload_len = out.len,
};
- if (bufferevent_write(bev, &resp, sizeof(resp)) < 0)
- LOG(ERR, "failed to write header");
- if (out.len > 0) {
- assert(out.payload != NULL);
- if (bufferevent_write(bev, out.payload, out.len) < 0)
- LOG(ERR, "failed to write payload");
- }
+ if (bufferevent_write2(bev, &resp, sizeof(resp), out.payload, out.len) < 0)
+ LOG(ERR, "pid=%d cannot write response", ctx->pid);
bufferevent_flush(bev, EV_WRITE, BEV_FLUSH);
@@ -466,4 +509,6 @@ RTE_INIT(init) {
api_handler(GR_EVENT_SUBSCRIBE, subscribe);
api_handler(GR_EVENT_UNSUBSCRIBE, unsubscribe);
api_handler(GR_HELLO, hello);
+ api_handler(GR_PING, ping);
+ api_handler(GR_PING2, ping2);
}
diff --git a/meson.build b/meson.build
index 073e3243..1e5f51a5 100644
--- a/meson.build
+++ b/meson.build
@@ -201,6 +201,12 @@ executable(
install: false,
)
+executable(
+ 'ping2_perf', files('smoke/ping2_perf.c') + grout_header,
+ include_directories: api_inc,
+ install: false,
+)
+
# docs/ must come after grcli_exe since man pages are generated using grcli --man
subdir('docs')
diff --git a/smoke/ping2_perf.c b/smoke/ping2_perf.c
new file mode 100644
index 00000000..a1b649db
--- /dev/null
+++ b/smoke/ping2_perf.c
@@ -0,0 +1,79 @@
+// SPDX-License-Identifier: BSD-3-Clause
+// Copyright (c) 2026 SmartShare Systems
+
+// clang-format off
+#include <gr_api_client_impl.h>
+// clang-format on
+
+#include <getopt.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+static int ping2(struct gr_api_client *c, uint32_t count) {
+ void *resp;
+ const struct gr_ping2 req = { 0 };
+
+ for (uint32_t i = 0; i < count; i++) {
+ resp = NULL;
+ if (gr_api_client_send_recv(c, GR_PING2, sizeof(req), &req, &resp) < 0) {
+ perror("GR_PING2");
+ return -1;
+ }
+ free(resp);
+ }
+
+ return 0;
+}
+
+static void usage(const char *prog) {
+ fprintf(stderr, "Usage: %s [-s SOCK] [-n COUNT]\n", prog);
+ fprintf(stderr, " -s SOCK API socket path (default: $GROUT_SOCK_PATH)\n");
+ fprintf(stderr, " -n COUNT Number of ping2 calls (default: 10000)\n");
+}
+
+int main(int argc, char **argv) {
+ const char *sock_path = getenv("GROUT_SOCK_PATH");
+ struct gr_api_client *c;
+ unsigned int count = 10000;
+ gr_clock_ns_t time;
+ float duration;
+ int ret;
+ int o;
+
+ while ((o = getopt(argc, argv, "s:n:h")) != -1) {
+ switch (o) {
+ case 's':
+ sock_path = optarg;
+ break;
+ case 'n':
+ count = strtoul(optarg, NULL, 10);
+ break;
+ case 'h':
+ default:
+ usage(argv[0]);
+ return o == 'h' ? EXIT_SUCCESS : EXIT_FAILURE;
+ }
+ }
+ if (sock_path == NULL)
+ sock_path = GR_DEFAULT_SOCK_PATH;
+
+ c = gr_api_client_connect(sock_path);
+ if (c == NULL) {
+ perror("gr_api_client_connect");
+ return EXIT_FAILURE;
+ }
+
+ printf("performing %u ping2 calls\n",
+ count);
+
+ time = gr_clock_ns();
+ ret = ping2(c, count);
+ duration = (float)(gr_clock_ns() - time) / (float)GR_NS_PER_S;
+
+ printf("total time: %.3f s (%.1f calls/s)\n", duration, (float)count / duration);
+
+ gr_api_client_disconnect(c);
+
+ return ret < 0 ? EXIT_FAILURE : EXIT_SUCCESS;
+}
--
2.43.0
More information about the grout
mailing list