[PATCH v4 3/4] dma/ae4dma: add data path operations

[Raghavendra Ningoji]

Implement the dmadev fast path (copy, submit, completed,
completed_status and burst_capacity) for the AMD AE4DMA PMD and wire
the entry points through fp_obj in ae4dma_dmadev_create().

The framework-visible ring index is a free-running 16-bit value; the
HW ring slot is derived as (hw_base + idx) & (nb_desc - 1), which is
why the vchan ring depth is kept a power of two. completed() and
completed_status() always report the last completed ring_idx via
last_idx, even when no new op is reaped, as required by the dmadev
API. Only memory-to-memory copy is advertised, so fp_obj->fill is
left zero-initialised.

AE4DMA-specific completion handling: the engine reports completion by
advancing the per-queue read_idx register rather than reliably writing
a status word back, so the driver uses read_idx as the source of truth
and only reads the descriptor err_code byte to flag failures.

Signed-off-by: Raghavendra Ningoji <raghavendra.ningoji at amd.com>
---
 doc/guides/dmadevs/ae4dma.rst      |  24 +++
 drivers/dma/ae4dma/ae4dma_dmadev.c | 245 +++++++++++++++++++++++++++++
 2 files changed, 269 insertions(+)

diff --git a/doc/guides/dmadevs/ae4dma.rst b/doc/guides/dmadevs/ae4dma.rst
index eeed55c..10446ca 100644
--- a/doc/guides/dmadevs/ae4dma.rst
+++ b/doc/guides/dmadevs/ae4dma.rst
@@ -51,3 +51,27 @@ On probe the PMD performs the following steps for each PCI function:
   IOVA-contiguous memory, programs the queue base address and ring
   depth into the per-queue registers, and enables the queue.
 * Interrupts are masked; completion is polled by the application.
+
+Usage
+-----
+
+Once a dmadev has been started, copies are submitted with
+``rte_dma_copy()`` and completions are reaped with ``rte_dma_completed()``
+or ``rte_dma_completed_status()``. See the
+:ref:`Enqueue / Dequeue API <dmadev_enqueue_dequeue>` section of the
+dmadev library documentation for details.
+
+Limitations
+-----------
+
+* Only memory-to-memory copies are supported. Fill, scatter-gather and
+  any other operation types are not advertised in
+  ``rte_dma_info::dev_capa``.
+* The maximum number of descriptors per virtual channel is fixed by
+  hardware at 32, and the engine reserves one slot to distinguish a full
+  ring from an empty one, so at most 31 operations can be outstanding
+  (``rte_dma_burst_capacity()`` returns at most 31). The PMD rounds the
+  requested ring size up to a power of two and clamps it to 32.
+* Only a single virtual channel per dmadev is supported; use the 16
+  per-PCI-function dmadevs to obtain channel-level parallelism.
+* Interrupt-driven completion is not supported.
diff --git a/drivers/dma/ae4dma/ae4dma_dmadev.c b/drivers/dma/ae4dma/ae4dma_dmadev.c
index 9910e11..8fcb0d5 100644
--- a/drivers/dma/ae4dma/ae4dma_dmadev.c
+++ b/drivers/dma/ae4dma/ae4dma_dmadev.c
@@ -162,6 +162,76 @@ ae4dma_dev_close(struct rte_dma_dev *dev)
 	return 0;
 }
 
+/* Ring the doorbell: publish all enqueued descriptors to the hardware. */
+static inline void
+__submit(struct ae4dma_dmadev *ae4dma)
+{
+	struct ae4dma_cmd_queue *cmd_q = &ae4dma->cmd_q;
+	uint16_t nb = cmd_q->qcfg.nb_desc;
+
+	if (nb == 0)
+		return;
+
+	/* The HW producer index is the ring slot of the next free descriptor. */
+	AE4DMA_WRITE_REG(&cmd_q->hwq_regs->write_idx,
+			(cmd_q->hw_base + cmd_q->next_write) & (nb - 1));
+	cmd_q->stats.submitted += (uint16_t)(cmd_q->next_write - cmd_q->last_write);
+	cmd_q->last_write = cmd_q->next_write;
+}
+
+static int
+ae4dma_submit(void *dev_private, uint16_t vchan __rte_unused)
+{
+	struct ae4dma_dmadev *ae4dma = dev_private;
+
+	__submit(ae4dma);
+	return 0;
+}
+
+/* Write a copy descriptor; returns the free-running ring_idx assigned to it. */
+static inline int
+__write_desc_copy(void *dev_private, rte_iova_t src, rte_iova_t dst,
+		uint32_t len, uint64_t flags)
+{
+	struct ae4dma_dmadev *ae4dma = dev_private;
+	struct ae4dma_cmd_queue *cmd_q = &ae4dma->cmd_q;
+	uint16_t ring_idx = cmd_q->next_write;
+	uint16_t nb = cmd_q->qcfg.nb_desc;
+	struct ae4dma_desc *dma_desc;
+	uint16_t in_flight;
+
+	if (nb == 0)
+		return -EINVAL;
+
+	/* Reserve one slot to distinguish full from empty on the ring. */
+	in_flight = (uint16_t)(cmd_q->next_write - cmd_q->next_read);
+	if (in_flight >= nb - 1)
+		return -ENOSPC;
+
+	dma_desc = &cmd_q->qbase_desc[(cmd_q->hw_base + ring_idx) & (nb - 1)];
+	memset(dma_desc, 0, sizeof(*dma_desc));
+	dma_desc->length = len;
+	dma_desc->src_hi = upper_32_bits(src);
+	dma_desc->src_lo = lower_32_bits(src);
+	dma_desc->dst_hi = upper_32_bits(dst);
+	dma_desc->dst_lo = lower_32_bits(dst);
+
+	cmd_q->next_write++;
+	if (flags & RTE_DMA_OP_FLAG_SUBMIT)
+		__submit(ae4dma);
+
+	/* dmadev ring_idx is a free-running 16-bit value, not a ring slot. */
+	return ring_idx;
+}
+
+/* Enqueue a copy operation onto the ae4dma device. */
+static int
+ae4dma_enqueue_copy(void *dev_private, uint16_t vchan __rte_unused,
+		rte_iova_t src, rte_iova_t dst, uint32_t length, uint64_t flags)
+{
+	return __write_desc_copy(dev_private, src, dst, length, flags);
+}
+
 static int
 ae4dma_dev_dump(const struct rte_dma_dev *dev, FILE *f)
 {
@@ -192,6 +262,174 @@ ae4dma_dev_dump(const struct rte_dma_dev *dev, FILE *f)
 	return 0;
 }
 
+/* Translate an AE4DMA descriptor error code to the dmadev status code. */
+static inline enum rte_dma_status_code
+__translate_status_ae4dma_to_dma(enum ae4dma_dma_err status)
+{
+	switch (status) {
+	case AE4DMA_DMA_ERR_NO_ERR:
+		return RTE_DMA_STATUS_SUCCESSFUL;
+	case AE4DMA_DMA_ERR_INV_LEN:
+		return RTE_DMA_STATUS_INVALID_LENGTH;
+	case AE4DMA_DMA_ERR_INV_SRC:
+		return RTE_DMA_STATUS_INVALID_SRC_ADDR;
+	case AE4DMA_DMA_ERR_INV_DST:
+		return RTE_DMA_STATUS_INVALID_DST_ADDR;
+	default:
+		return RTE_DMA_STATUS_ERROR_UNKNOWN;
+	}
+}
+
+/*
+ * Scan the HW queue for completed descriptors (non-blocking).
+ *
+ * The AE4DMA engine signals completion by advancing the per-queue
+ * read_idx register; it does not (reliably) write a status value back
+ * into the descriptor. We therefore use the HW read_idx register as the
+ * source of truth and only inspect the descriptor dw1.err_code byte to
+ * classify each completion as success or failure.
+ *
+ * next_read/next_write are free-running 16-bit indices; the HW ring slot
+ * is ((hw_base + idx) & mask) and the number of in-flight ops is
+ * (next_write - next_read), both relying on nb_desc being a power of two.
+ */
+static inline uint16_t
+ae4dma_scan_hwq(struct ae4dma_cmd_queue *cmd_q, uint16_t max_ops,
+		uint16_t *failed_count)
+{
+	volatile struct ae4dma_desc *hw_desc;
+	uint16_t events_count = 0, fails = 0;
+	uint16_t nb = cmd_q->qcfg.nb_desc;
+	uint16_t hw_read_pos, tail_pos;
+	uint16_t newly_done, in_flight;
+	uint16_t scan_cap, mask;
+
+	*failed_count = 0;
+	if (nb == 0)
+		return 0;
+	mask = nb - 1;
+
+	in_flight = (uint16_t)(cmd_q->next_write - cmd_q->next_read);
+	if (in_flight == 0)
+		return 0;
+
+	/* Descriptors HW has consumed since our last visit: [tail, read). */
+	hw_read_pos = (uint16_t)(AE4DMA_READ_REG(&cmd_q->hwq_regs->read_idx) & mask);
+	tail_pos = (uint16_t)((cmd_q->hw_base + cmd_q->next_read) & mask);
+	newly_done = (uint16_t)((hw_read_pos - tail_pos) & mask);
+	if (newly_done == 0)
+		return 0;
+
+	scan_cap = newly_done;
+	if (scan_cap > in_flight)
+		scan_cap = in_flight;
+	if (scan_cap > max_ops)
+		scan_cap = max_ops;
+
+	while (events_count < scan_cap) {
+		uint16_t slot = (cmd_q->hw_base + cmd_q->next_read + events_count) & mask;
+		uint8_t hw_status, hw_err;
+
+		hw_desc = &cmd_q->qbase_desc[slot];
+		hw_status = hw_desc->dw1.status;
+		hw_err = hw_desc->dw1.err_code;
+
+		/*
+		 * read_idx advancing is the definitive completion signal.
+		 * The per-descriptor status byte is informational and may
+		 * not yet be written when we observe it, so a slot is only
+		 * a failure if the HW flagged DESC_ERROR or set err_code.
+		 */
+		if (hw_status == AE4DMA_DMA_DESC_ERROR ||
+				hw_err != AE4DMA_DMA_ERR_NO_ERR) {
+			fails++;
+			AE4DMA_PMD_WARN("Desc failed: status=%u err=%u",
+					hw_status, hw_err);
+		}
+		cmd_q->status[events_count] = (enum ae4dma_dma_err)hw_err;
+		events_count++;
+	}
+
+	cmd_q->next_read += events_count;
+	cmd_q->stats.completed += events_count;
+	cmd_q->stats.errors += fails;
+	*failed_count = fails;
+	return events_count;
+}
+
+/* Returns successful operations count and sets error flag if any errors. */
+static uint16_t
+ae4dma_completed(void *dev_private, uint16_t vchan __rte_unused,
+		const uint16_t max_ops, uint16_t *last_idx, bool *has_error)
+{
+	struct ae4dma_dmadev *ae4dma = dev_private;
+	struct ae4dma_cmd_queue *cmd_q = &ae4dma->cmd_q;
+	uint16_t err_count = 0;
+	uint16_t cpl_count;
+
+	*has_error = false;
+	cpl_count = ae4dma_scan_hwq(cmd_q, max_ops, &err_count);
+
+	/*
+	 * last_idx always reports the ring_idx of the most recently completed
+	 * op, even when this call reaps nothing (matches the dmadev contract
+	 * and the ioat/idxd drivers).
+	 */
+	if (last_idx != NULL)
+		*last_idx = (uint16_t)(cmd_q->next_read - 1);
+
+	if (err_count != 0)
+		*has_error = true;
+
+	return cpl_count - err_count;
+}
+
+static uint16_t
+ae4dma_completed_status(void *dev_private, uint16_t vchan __rte_unused,
+		uint16_t max_ops, uint16_t *last_idx,
+		enum rte_dma_status_code *status)
+{
+	struct ae4dma_dmadev *ae4dma = dev_private;
+	struct ae4dma_cmd_queue *cmd_q = &ae4dma->cmd_q;
+	uint16_t err_count = 0;
+	uint16_t cpl_count;
+	uint16_t i;
+
+	cpl_count = ae4dma_scan_hwq(cmd_q, max_ops, &err_count);
+
+	if (last_idx != NULL)
+		*last_idx = (uint16_t)(cmd_q->next_read - 1);
+
+	if (likely(err_count == 0)) {
+		for (i = 0; i < cpl_count; i++)
+			status[i] = RTE_DMA_STATUS_SUCCESSFUL;
+	} else {
+		for (i = 0; i < cpl_count; i++)
+			status[i] = __translate_status_ae4dma_to_dma(cmd_q->status[i]);
+	}
+
+	return cpl_count;
+}
+
+/* Get the remaining capacity of the ring. */
+static uint16_t
+ae4dma_burst_capacity(const void *dev_private, uint16_t vchan __rte_unused)
+{
+	const struct ae4dma_dmadev *ae4dma = dev_private;
+	const struct ae4dma_cmd_queue *cmd_q = &ae4dma->cmd_q;
+	uint16_t nb = cmd_q->qcfg.nb_desc;
+	uint16_t in_flight;
+
+	if (nb == 0)
+		return 0;
+
+	/* One slot reserved to distinguish full from empty. */
+	in_flight = (uint16_t)(cmd_q->next_write - cmd_q->next_read);
+	if (in_flight >= nb - 1)
+		return 0;
+	return (uint16_t)(nb - 1 - in_flight);
+}
+
 static int
 ae4dma_stats_get(const struct rte_dma_dev *dev, uint16_t vchan __rte_unused,
 		struct rte_dma_stats *rte_stats, uint32_t size __rte_unused)
@@ -339,6 +577,13 @@ ae4dma_dmadev_create(const char *name, struct rte_pci_device *dev, uint8_t qn)
 	dmadev->fp_obj->dev_private = dmadev->data->dev_private;
 	dmadev->dev_ops = &ae4dma_dmadev_ops;
 
+	dmadev->fp_obj->burst_capacity = ae4dma_burst_capacity;
+	dmadev->fp_obj->completed = ae4dma_completed;
+	dmadev->fp_obj->completed_status = ae4dma_completed_status;
+	dmadev->fp_obj->copy = ae4dma_enqueue_copy;
+	dmadev->fp_obj->submit = ae4dma_submit;
+	/* fill capability not advertised: leave fp_obj->fill as zero-initialised. */
+
 	ae4dma = dmadev->data->dev_private;
 
 	if (ae4dma_add_queue(ae4dma, dev, qn, name) != 0)
-- 
2.34.1