[PATCH v2 1/1] lib/dma: add control-plane APIs for inter-domain DMA transfers

[Vamsi Krishna]

From: Vamsi Attunuru <vattunuru at marvell.com>

Secure and controlled inter-domain DMA transfers require dedicated
control-plane APIs to create and manage access groups.

DMA devices used for inter-process data transfer can be categorized as
follows:

Class A: Both endpoints require a DMA device for data transfer (e.g.,
Marvell DMA devices).
Class B: Only one endpoint requires a DMA device; the other does not.
Class C: Other device types not currently classified.

Providing a unified API for all these categories is complex, as Linux
and other operating systems do not offer native control-plane APIs for
this purpose. Therefore, DPDK can implement its own control-plane
mechanisms to support Class A, B, and C devices.

This commit introduces the necessary APIs for Class A DMA devices.
Devices can create or join access groups using token-based
authentication, ensuring that only authorized devices within the same
group can perform DMA transfers across processes or OS domains.

API Usage Flow:

Process 1 (Group Creator):
Invokes rte_dma_access_pair_group_create() to establish a new access
pair group, then shares the group_id and token with Process 2 via IPC.

Process 2 (Group Joiner):
Receives the group_id and token from Process 1 and calls
rte_dma_access_pair_group_join() to join the group.

Both Processes:
Use rte_dma_access_pair_group_handle_get() to obtain handler information
for domains in the group.
Perform DMA transfers as required.

Process 2 (when finished):
Calls rte_dma_access_pair_group_leave() to exit the group.

Process 1:
Monitors for group updates and confirms group membership as needed.

Depends-on: patch-36335 (lib/dma: introduce inter-process and inter-OS DMA)

Signed-off-by: Vamsi Attunuru <vattunuru at marvell.com>
---
 lib/dmadev/rte_dmadev.c       | 137 ++++++++++++++++++++++++++++++++++
 lib/dmadev/rte_dmadev.h       | 100 +++++++++++++++++++++++++
 lib/dmadev/rte_dmadev_pmd.h   |  28 +++++++
 lib/dmadev/rte_dmadev_trace.h |  42 +++++++++++
 4 files changed, 307 insertions(+)

diff --git a/lib/dmadev/rte_dmadev.c b/lib/dmadev/rte_dmadev.c
index d6792449bf..521b372b86 100644
--- a/lib/dmadev/rte_dmadev.c
+++ b/lib/dmadev/rte_dmadev.c
@@ -804,6 +804,143 @@ rte_dma_vchan_status(int16_t dev_id, uint16_t vchan, enum rte_dma_vchan_status *
 	return dev->dev_ops->vchan_status(dev, vchan, status);
 }
 
+int
+rte_dma_access_pair_group_create(int16_t dev_id, rte_uuid_t domain_id, rte_uuid_t token,
+				 int16_t *group_id)
+{
+	struct rte_dma_info dev_info;
+	struct rte_dma_dev *dev;
+
+	if (!rte_dma_is_valid(dev_id) || group_id == NULL)
+		return -EINVAL;
+	dev = &rte_dma_devices[dev_id];
+
+	if (rte_dma_info_get(dev_id, &dev_info)) {
+		RTE_DMA_LOG(ERR, "Device %d get device info fail", dev_id);
+		return -EINVAL;
+	}
+
+	if (!((dev_info.dev_capa & RTE_DMA_CAPA_INTER_PROCESS_DOMAIN) ||
+	    (dev_info.dev_capa & RTE_DMA_CAPA_INTER_OS_DOMAIN))) {
+		RTE_DMA_LOG(ERR, "Device %d don't support inter-process or inter-os domain",
+			    dev_id);
+		return -EINVAL;
+	}
+	if (*dev->dev_ops->access_pair_group_create == NULL)
+		return -ENOTSUP;
+	return (*dev->dev_ops->access_pair_group_create)(dev, domain_id, token, group_id);
+}
+
+int
+rte_dma_access_pair_group_destroy(int16_t dev_id, int16_t group_id)
+{
+	struct rte_dma_info dev_info;
+	struct rte_dma_dev *dev;
+
+	if (!rte_dma_is_valid(dev_id))
+		return -EINVAL;
+	dev = &rte_dma_devices[dev_id];
+
+	if (rte_dma_info_get(dev_id, &dev_info)) {
+		RTE_DMA_LOG(ERR, "Device %d get device info fail", dev_id);
+		return -EINVAL;
+	}
+
+	if (!((dev_info.dev_capa & RTE_DMA_CAPA_INTER_PROCESS_DOMAIN) ||
+	    (dev_info.dev_capa & RTE_DMA_CAPA_INTER_OS_DOMAIN))) {
+		RTE_DMA_LOG(ERR, "Device %d don't support inter-process or inter-os domain",
+			    dev_id);
+		return -EINVAL;
+	}
+
+	if (*dev->dev_ops->access_pair_group_destroy == NULL)
+		return -ENOTSUP;
+	return (*dev->dev_ops->access_pair_group_destroy)(dev, group_id);
+}
+
+int
+rte_dma_access_pair_group_join(int16_t dev_id, int16_t group_id, rte_uuid_t token,
+			       rte_dma_access_pair_leave_cb_t leave_cb)
+{
+	struct rte_dma_info dev_info;
+	struct rte_dma_dev *dev;
+
+	if (!rte_dma_is_valid(dev_id))
+		return -EINVAL;
+	dev = &rte_dma_devices[dev_id];
+
+	if (rte_dma_info_get(dev_id, &dev_info)) {
+		RTE_DMA_LOG(ERR, "Device %d get device info fail", dev_id);
+		return -EINVAL;
+	}
+
+	if (!((dev_info.dev_capa & RTE_DMA_CAPA_INTER_PROCESS_DOMAIN) ||
+	    (dev_info.dev_capa & RTE_DMA_CAPA_INTER_OS_DOMAIN))) {
+		RTE_DMA_LOG(ERR, "Device %d don't support inter-process or inter-os domain",
+			    dev_id);
+		return -EINVAL;
+	}
+
+	if (*dev->dev_ops->access_pair_group_join == NULL)
+		return -ENOTSUP;
+	return (*dev->dev_ops->access_pair_group_join)(dev, group_id, token, leave_cb);
+}
+
+int
+rte_dma_access_pair_group_leave(int16_t dev_id, int16_t group_id)
+{
+	struct rte_dma_info dev_info;
+	struct rte_dma_dev *dev;
+
+	if (!rte_dma_is_valid(dev_id))
+		return -EINVAL;
+	dev = &rte_dma_devices[dev_id];
+
+	if (rte_dma_info_get(dev_id, &dev_info)) {
+		RTE_DMA_LOG(ERR, "Device %d get device info fail", dev_id);
+		return -EINVAL;
+	}
+
+	if (!((dev_info.dev_capa & RTE_DMA_CAPA_INTER_PROCESS_DOMAIN) ||
+	    (dev_info.dev_capa & RTE_DMA_CAPA_INTER_OS_DOMAIN))) {
+		RTE_DMA_LOG(ERR, "Device %d don't support inter-process or inter-os domain",
+			    dev_id);
+		return -EINVAL;
+	}
+
+	if (*dev->dev_ops->access_pair_group_leave == NULL)
+		return -ENOTSUP;
+	return (*dev->dev_ops->access_pair_group_leave)(dev, group_id);
+}
+
+int
+rte_dma_access_pair_group_handle_get(int16_t dev_id, int16_t group_id, rte_uuid_t domain_id,
+				     uint16_t *handle)
+{
+	struct rte_dma_info dev_info;
+	struct rte_dma_dev *dev;
+
+	if (!rte_dma_is_valid(dev_id) || handle == NULL)
+		return -EINVAL;
+	dev = &rte_dma_devices[dev_id];
+
+	if (rte_dma_info_get(dev_id, &dev_info)) {
+		RTE_DMA_LOG(ERR, "Device %d get device info fail", dev_id);
+		return -EINVAL;
+	}
+
+	if (!((dev_info.dev_capa & RTE_DMA_CAPA_INTER_PROCESS_DOMAIN) ||
+	    (dev_info.dev_capa & RTE_DMA_CAPA_INTER_OS_DOMAIN))) {
+		RTE_DMA_LOG(ERR, "Device %d don't support inter-process or inter-os domain",
+			    dev_id);
+		return -EINVAL;
+	}
+
+	if (*dev->dev_ops->access_pair_group_handle_get == NULL)
+		return -ENOTSUP;
+	return (*dev->dev_ops->access_pair_group_handle_get)(dev, group_id, domain_id, handle);
+}
+
 static const char *
 dma_capability_name(uint64_t capability)
 {
diff --git a/lib/dmadev/rte_dmadev.h b/lib/dmadev/rte_dmadev.h
index 4f735bb0c9..a6e73337eb 100644
--- a/lib/dmadev/rte_dmadev.h
+++ b/lib/dmadev/rte_dmadev.h
@@ -148,6 +148,7 @@
 
 #include <rte_bitops.h>
 #include <rte_common.h>
+#include <rte_uuid.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -779,6 +780,105 @@ rte_dma_vchan_status(int16_t dev_id, uint16_t vchan, enum rte_dma_vchan_status *
  */
 int rte_dma_dump(int16_t dev_id, FILE *f);
 
+/**
+ * Callback function invoked when a DMA device leaves an access group or when the group is
+ * destroyed due to some exception or failure.
+ *
+ * @param dma_id
+ *   Identifier of the DMA device that is leaving the group.
+ * @param group_id
+ *   ID of the access group being exited.
+ * @param user_data
+ *   User-defined data passed to the callback for context.
+ */
+typedef void (*rte_dma_access_pair_leave_cb_t)(int16_t dma_id, int16_t group_id, void *user_data);
+
+/**
+ * Create an access pair group to enable secure DMA transfers between devices across different
+ * processes or operating system domains.
+ *
+ * @param dev_id
+ *   Identifier of the DMA device initiating the group.
+ * @param domain_id
+ *   Unique identifier representing the process or OS domain.
+ * @param token
+ *   Authentication token used to establish the access group.
+ * @param[out] group_id
+ *   Pointer to store the ID of the newly created access group.
+ *
+ * @return
+ *   0 on success,
+ *   negative error code on failure.
+ */
+int rte_dma_access_pair_group_create(int16_t dev_id, rte_uuid_t domain_id, rte_uuid_t token,
+				     int16_t *group_id);
+/**
+ * Destroy an access pair group if all participating devices have exited. This operation is only
+ * permitted by the device that originally created the group; attempts by other devices will result
+ * in failure.
+ *
+ * @param dev_id
+ *   Identifier of the device requesting group destruction.
+ * @param group_id
+ *   ID of the access group to be destroyed.
+ * @return
+ *   0 on success,
+ *   negative value on failure indicating the error code.
+ */
+int rte_dma_access_pair_group_destroy(int16_t dev_id, int16_t group_id);
+/**
+ * Join an existing access group to enable secure DMA transfers between devices across different
+ * processes or OS domains.
+ *
+ * @param dev_id
+ *   Identifier of the DMA device attempting to join the group.
+ * @param group_id
+ *   ID of the access group to join.
+ * @param token
+ *   Authentication token used to validate group membership.
+ * @param leave_cb
+ *   Callback function to be invoked when the device leaves the group or when the group
+ *   is destroyed due to some exception or failure.
+ *
+ * @return
+ *   0 on success,
+ *   negative value on failure indicating the error code.
+ */
+int rte_dma_access_pair_group_join(int16_t dev_id, int16_t group_id, rte_uuid_t token,
+				   rte_dma_access_pair_leave_cb_t leave_cb);
+/**
+ * Leave an access group, removing the device's entry from the group table and disabling
+ * inter-domain DMA transfers to and from this device. This operation is not permitted for the
+ * device that originally created the group.
+ *
+ * @param dev_id
+ *   Identifier of the device requesting to leave the group.
+ * @param group_id
+ *   ID of the access group to leave.
+ * @return
+ *   0 on success,
+ *   negative value on failure indicating the error code.
+ */
+int rte_dma_access_pair_group_leave(int16_t dev_id, int16_t group_id);
+/**
+ * Retrieve the handler associated with a specific domain ID, which is used by the application to
+ * query source or destinationin handler to initiate inter-process or inter-OS DMA transfers.
+ *
+ * @param dev_id
+ *   Identifier of the DMA device requesting the handler.
+ * @param group_id
+ *   ID of the access group to query.
+ * @param group_tbl
+ *   Unique identifier of the target process or OS domain.
+ * @param[out] handle
+ *   Pointer to store the retrieved handler value.
+ * @return
+ *   0 on success,
+ *   negative value on failure indicating the error code.
+ */
+int rte_dma_access_pair_group_handle_get(int16_t dev_id, int16_t group_id, rte_uuid_t domain_id,
+					 uint16_t *handle);
+
 /**
  * DMA transfer result status code defines.
  *
diff --git a/lib/dmadev/rte_dmadev_pmd.h b/lib/dmadev/rte_dmadev_pmd.h
index 58729088ff..8be4485593 100644
--- a/lib/dmadev/rte_dmadev_pmd.h
+++ b/lib/dmadev/rte_dmadev_pmd.h
@@ -64,6 +64,28 @@ typedef int (*rte_dma_vchan_status_t)(const struct rte_dma_dev *dev, uint16_t vc
 /** @internal Used to dump internal information. */
 typedef int (*rte_dma_dump_t)(const struct rte_dma_dev *dev, FILE *f);
 
+/** @internal Used to create an access pair group for inter-process or inter-OS DMA transfers. */
+typedef int (*rte_dma_access_pair_group_create_t)(const struct rte_dma_dev *dev,
+						  rte_uuid_t domain_id, rte_uuid_t token,
+						  int16_t *group_id);
+
+/** @internal Used to destroy an access pair group when all other devices have exited. */
+typedef int (*rte_dma_access_pair_group_destroy_t)(const struct rte_dma_dev *dev,
+						   int16_t group_id);
+
+/** @internal Used to join an access pair group for inter-process or inter-OS DMA transfers. */
+typedef int (*rte_dma_access_pair_group_join_t)(const struct rte_dma_dev *dev, int16_t group_id,
+						rte_uuid_t token,
+						rte_dma_access_pair_leave_cb_t leave_cb);
+
+/** @internal Used to leave an access pair group, removing the device from the group. */
+typedef int (*rte_dma_access_pair_group_leave_t)(const struct rte_dma_dev *dev, int16_t group_id);
+
+/** @internal Used to retrieve handler information of the domain_id present in the group. */
+typedef int (*rte_dma_access_pair_group_handle_get_t)(const struct rte_dma_dev *dev,
+						      int16_t group_id, rte_uuid_t domain_id,
+						      uint16_t *handle);
+
 /**
  * DMA device operations function pointer table.
  *
@@ -83,6 +105,12 @@ struct rte_dma_dev_ops {
 
 	rte_dma_vchan_status_t     vchan_status;
 	rte_dma_dump_t             dev_dump;
+
+	rte_dma_access_pair_group_create_t	access_pair_group_create;
+	rte_dma_access_pair_group_destroy_t	access_pair_group_destroy;
+	rte_dma_access_pair_group_join_t	access_pair_group_join;
+	rte_dma_access_pair_group_leave_t	access_pair_group_leave;
+	rte_dma_access_pair_group_handle_get_t	access_pair_group_handle_get;
 };
 
 /**
diff --git a/lib/dmadev/rte_dmadev_trace.h b/lib/dmadev/rte_dmadev_trace.h
index fa619c4090..c25cf8d60a 100644
--- a/lib/dmadev/rte_dmadev_trace.h
+++ b/lib/dmadev/rte_dmadev_trace.h
@@ -101,6 +101,48 @@ RTE_TRACE_POINT(
 	rte_trace_point_emit_int(ret);
 )
 
+RTE_TRACE_POINT(
+	rte_dma_trace_access_pair_group_create,
+	RTE_TRACE_POINT_ARGS(int16_t dev_id, rte_uuid_t domain_id, rte_uuid_t token,
+			     int16_t *group_id),
+	rte_trace_point_emit_i16(dev_id);
+	rte_trace_point_emit_u8_ptr(&domain_id[0]);
+	rte_trace_point_emit_u8_ptr(&token[0]);
+	rte_trace_point_emit_ptr(group_id);
+)
+
+RTE_TRACE_POINT(
+	rte_dma_trace_access_pair_group_destroy,
+	RTE_TRACE_POINT_ARGS(int16_t dev_id, int16_t group_id),
+	rte_trace_point_emit_i16(dev_id);
+	rte_trace_point_emit_i16(group_id);
+)
+
+RTE_TRACE_POINT(
+	rte_dma_trace_access_pair_group_join,
+	RTE_TRACE_POINT_ARGS(int16_t dev_id, int16_t group_id, rte_uuid_t token),
+	rte_trace_point_emit_i16(dev_id);
+	rte_trace_point_emit_i16(group_id);
+	rte_trace_point_emit_u8_ptr(&token[0]);
+)
+
+RTE_TRACE_POINT(
+	rte_dma_trace_access_pair_group_leave,
+	RTE_TRACE_POINT_ARGS(int16_t dev_id, int16_t group_id),
+	rte_trace_point_emit_i16(dev_id);
+	rte_trace_point_emit_i16(group_id);
+)
+
+RTE_TRACE_POINT(
+	rte_dma_trace_access_pair_group_handle_get,
+	RTE_TRACE_POINT_ARGS(int16_t dev_id, int16_t group_id, rte_uuid_t domain_id,
+			     uint16_t *handle),
+	rte_trace_point_emit_i16(dev_id);
+	rte_trace_point_emit_i16(group_id);
+	rte_trace_point_emit_u8_ptr(&domain_id[0]);
+	rte_trace_point_emit_ptr(handle);
+)
+
 #ifdef __cplusplus
 }
 #endif
-- 
2.34.1