[PATCH grout] api: optimize send small payloads

[Morten Brørup]

Copy small payloads into buffer holding both header and payload,
and send as one, instead of sending header and payload separately.

Signed-off-by: Morten Brørup <mb at smartsharesystems.com>
---
 api/gr_api_client_impl.h | 29 ++++++++++++++++-----
 main/api.c               | 55 ++++++++++++++++++++++++++++++----------
 2 files changed, 63 insertions(+), 21 deletions(-)

diff --git a/api/gr_api_client_impl.h b/api/gr_api_client_impl.h
index 242b286d..3af31e9b 100644
--- a/api/gr_api_client_impl.h
+++ b/api/gr_api_client_impl.h
@@ -198,6 +198,10 @@ static ssize_t recv_all(const struct gr_api_client *c, void *buf, size_t len) {
 	return len;
 }
 
+// Optimization: Threshold for appending small payload to request buffer,
+// and send in one go, instead of sending request header and payload separately.
+#define GR_API_REQUEST_MERGE_PAYLOAD (4096 - sizeof(struct gr_api_request))
+
 long int gr_api_client_send(
 	struct gr_api_client *client,
 	uint32_t req_type,
@@ -209,19 +213,30 @@ long int gr_api_client_send(
 	if (client == NULL || (tx_len == 0 && tx_data != NULL) || (tx_len > 0 && tx_data == NULL))
 		return errno_set(EINVAL);
 
-	struct gr_api_request req = {
+	struct merged_req {
+		struct gr_api_request req;
+		char payload[GR_API_REQUEST_MERGE_PAYLOAD];
+	} mreq;
+
+	mreq.req = (struct gr_api_request){
 		.id = ++message_id,
 		.payload_len = tx_len,
 		.type = req_type,
 	};
 
-	if (send_all(client, &req, sizeof(req)) < 0)
-		return -errno;
-
-	if (tx_len > 0 && send_all(client, tx_data, tx_len) < 0)
-		return -errno;
+	if (tx_len <= GR_API_REQUEST_MERGE_PAYLOAD) {
+		if (tx_len > 0)
+			memcpy(mreq.payload, tx_data, tx_len);
+		if (send_all(client, &mreq, sizeof(mreq.req) + tx_len) < 0)
+			return -errno;
+	} else {
+		if (send_all(client, &mreq, sizeof(mreq.req)) < 0)
+			return -errno;
+		if (send_all(client, tx_data, tx_len) < 0)
+			return -errno;
+	}
 
-	return req.id;
+	return mreq.req.id;
 }
 
 int gr_api_client_recv(
diff --git a/main/api.c b/main/api.c
index 023c9002..8afcf4b6 100644
--- a/main/api.c
+++ b/main/api.c
@@ -231,22 +231,38 @@ static void disconnect_client(struct api_ctx *ctx) {
 	free(ctx);
 }
 
+// Optimization: Threshold for appending small payload to response buffer,
+// and send in one go, instead of sending response header and payload separately.
+#define API_RESPONSE_MERGE_PAYLOAD (256 - sizeof(struct gr_api_response))
+
 void api_send(struct api_ctx *ctx, uint32_t len, const void *payload) {
 	assert(ctx != NULL);
-	assert(len != 0);
-	assert(payload != NULL);
+	assert(payload != NULL || len == 0);
 
 	LOG(DEBUG, "pid=%d for_id=%u len=%u", ctx->pid, ctx->header.id, len);
 
-	struct gr_api_response resp = {
+	struct merged_resp {
+		struct gr_api_response resp;
+		char payload[API_RESPONSE_MERGE_PAYLOAD];
+	} mresp;
+
+	mresp.resp = (struct gr_api_response){
 		.for_id = ctx->header.id,
 		.payload_len = len,
 		.status = 0,
 	};
-	if (bufferevent_write(ctx->bev, &resp, sizeof(resp)) < 0)
-		LOG(ERR, "pid=%d cannot write header", ctx->pid);
-	if (bufferevent_write(ctx->bev, payload, len) < 0)
-		LOG(ERR, "pid=%d cannot write payload", ctx->pid);
+
+	if (len <= API_RESPONSE_MERGE_PAYLOAD) {
+		if (len > 0)
+			memcpy(mresp.payload, payload, len);
+		if (bufferevent_write(ctx->bev, &mresp, sizeof(mresp.resp) + len) < 0)
+			LOG(ERR, "pid=%d cannot write header and payload", ctx->pid);
+	} else {
+		if (bufferevent_write(ctx->bev, &mresp, sizeof(mresp.resp)) < 0)
+			LOG(ERR, "pid=%d cannot write header", ctx->pid);
+		else if (bufferevent_write(ctx->bev, payload, len) < 0)
+			LOG(ERR, "pid=%d cannot write payload", ctx->pid);
+	}
 }
 
 static void read_cb(struct bufferevent *bev, void *priv) {
@@ -327,18 +343,29 @@ send:
 	    strerror(out.status),
 	    out.len);
 
-	struct gr_api_response resp = {
+	assert(out.payload != NULL || out.len == 0);
+
+	struct merged_resp {
+		struct gr_api_response resp;
+		char payload[API_RESPONSE_MERGE_PAYLOAD];
+	} mresp;
+
+	mresp.resp = (struct gr_api_response){
 		.for_id = ctx->header.id,
 		.status = out.status,
 		.payload_len = out.len,
 	};
 
-	if (bufferevent_write(bev, &resp, sizeof(resp)) < 0)
-		LOG(ERR, "failed to write header");
-	if (out.len > 0) {
-		assert(out.payload != NULL);
-		if (bufferevent_write(bev, out.payload, out.len) < 0)
-			LOG(ERR, "failed to write payload");
+	if (out.len <= API_RESPONSE_MERGE_PAYLOAD) {
+		if (out.len > 0)
+			memcpy(mresp.payload, out.payload, out.len);
+		if (bufferevent_write(bev, &mresp, sizeof(mresp.resp) + out.len) < 0)
+			LOG(ERR, "pid=%d cannot write header and payload", ctx->pid);
+	} else {
+		if (bufferevent_write(bev, &mresp, sizeof(mresp.resp)) < 0)
+			LOG(ERR, "pid=%d cannot write header", ctx->pid);
+		else if (bufferevent_write(bev, out.payload, out.len) < 0)
+			LOG(ERR, "pid=%d cannot write payload", ctx->pid);
 	}
 
 	bufferevent_flush(bev, EV_WRITE, BEV_FLUSH);
-- 
2.43.0