[PATCH v5 3/4] dma/ae4dma: add data path operations

[Raghavendra Ningoji]

Implement the dmadev fast path (copy, submit, completed,
completed_status and burst_capacity) for the AMD AE4DMA PMD and wire
the entry points through fp_obj in ae4dma_dmadev_create().

The framework-visible ring index is a free-running 16-bit value; the
HW ring slot is derived as (hw_base + idx) & (nb_desc - 1), which is
why the vchan ring depth is kept a power of two. completed() and
completed_status() always report the last completed ring_idx via
last_idx, even when no new op is reaped, as required by the dmadev
API. Only memory-to-memory copy is advertised, so fp_obj->fill is
left zero-initialised.

AE4DMA-specific completion handling: the engine reports completion by
advancing the per-queue read_idx register rather than reliably writing
a status word back, so the driver uses read_idx as the source of truth
and only reads the descriptor err_code byte to flag failures.

Signed-off-by: Raghavendra Ningoji <raghavendra.ningoji at amd.com>
---
 doc/guides/dmadevs/ae4dma.rst      |  24 +++
 drivers/dma/ae4dma/ae4dma_dmadev.c | 249 +++++++++++++++++++++++++++++
 2 files changed, 273 insertions(+)

diff --git a/doc/guides/dmadevs/ae4dma.rst b/doc/guides/dmadevs/ae4dma.rst
index eeed55c283..10446ca2e8 100644
--- a/doc/guides/dmadevs/ae4dma.rst
+++ b/doc/guides/dmadevs/ae4dma.rst
@@ -51,3 +51,27 @@ On probe the PMD performs the following steps for each PCI function:
   IOVA-contiguous memory, programs the queue base address and ring
   depth into the per-queue registers, and enables the queue.
 * Interrupts are masked; completion is polled by the application.
+
+Usage
+-----
+
+Once a dmadev has been started, copies are submitted with
+``rte_dma_copy()`` and completions are reaped with ``rte_dma_completed()``
+or ``rte_dma_completed_status()``. See the
+:ref:`Enqueue / Dequeue API <dmadev_enqueue_dequeue>` section of the
+dmadev library documentation for details.
+
+Limitations
+-----------
+
+* Only memory-to-memory copies are supported. Fill, scatter-gather and
+  any other operation types are not advertised in
+  ``rte_dma_info::dev_capa``.
+* The maximum number of descriptors per virtual channel is fixed by
+  hardware at 32, and the engine reserves one slot to distinguish a full
+  ring from an empty one, so at most 31 operations can be outstanding
+  (``rte_dma_burst_capacity()`` returns at most 31). The PMD rounds the
+  requested ring size up to a power of two and clamps it to 32.
+* Only a single virtual channel per dmadev is supported; use the 16
+  per-PCI-function dmadevs to obtain channel-level parallelism.
+* Interrupt-driven completion is not supported.
diff --git a/drivers/dma/ae4dma/ae4dma_dmadev.c b/drivers/dma/ae4dma/ae4dma_dmadev.c
index 9910e115b2..70c7c21ddf 100644
--- a/drivers/dma/ae4dma/ae4dma_dmadev.c
+++ b/drivers/dma/ae4dma/ae4dma_dmadev.c
@@ -162,6 +162,76 @@ ae4dma_dev_close(struct rte_dma_dev *dev)
 	return 0;
 }
 
+/* Ring the doorbell: publish all enqueued descriptors to the hardware. */
+static inline void
+__submit(struct ae4dma_dmadev *ae4dma)
+{
+	struct ae4dma_cmd_queue *cmd_q = &ae4dma->cmd_q;
+	uint16_t nb = cmd_q->qcfg.nb_desc;
+
+	if (nb == 0)
+		return;
+
+	/* The HW producer index is the ring slot of the next free descriptor. */
+	AE4DMA_WRITE_REG(&cmd_q->hwq_regs->write_idx,
+			(cmd_q->hw_base + cmd_q->next_write) & (nb - 1));
+	cmd_q->stats.submitted += (uint16_t)(cmd_q->next_write - cmd_q->last_write);
+	cmd_q->last_write = cmd_q->next_write;
+}
+
+static int
+ae4dma_submit(void *dev_private, uint16_t vchan __rte_unused)
+{
+	struct ae4dma_dmadev *ae4dma = dev_private;
+
+	__submit(ae4dma);
+	return 0;
+}
+
+/* Write a copy descriptor; returns the free-running ring_idx assigned to it. */
+static inline int
+__write_desc_copy(void *dev_private, rte_iova_t src, rte_iova_t dst,
+		uint32_t len, uint64_t flags)
+{
+	struct ae4dma_dmadev *ae4dma = dev_private;
+	struct ae4dma_cmd_queue *cmd_q = &ae4dma->cmd_q;
+	uint16_t ring_idx = cmd_q->next_write;
+	uint16_t nb = cmd_q->qcfg.nb_desc;
+	struct ae4dma_desc *dma_desc;
+	uint16_t in_flight;
+
+	if (nb == 0)
+		return -EINVAL;
+
+	/* Reserve one slot to distinguish full from empty on the ring. */
+	in_flight = (uint16_t)(cmd_q->next_write - cmd_q->next_read);
+	if (in_flight >= nb - 1)
+		return -ENOSPC;
+
+	dma_desc = &cmd_q->qbase_desc[(cmd_q->hw_base + ring_idx) & (nb - 1)];
+	memset(dma_desc, 0, sizeof(*dma_desc));
+	dma_desc->length = len;
+	dma_desc->src_hi = upper_32_bits(src);
+	dma_desc->src_lo = lower_32_bits(src);
+	dma_desc->dst_hi = upper_32_bits(dst);
+	dma_desc->dst_lo = lower_32_bits(dst);
+
+	cmd_q->next_write++;
+	if (flags & RTE_DMA_OP_FLAG_SUBMIT)
+		__submit(ae4dma);
+
+	/* dmadev ring_idx is a free-running 16-bit value, not a ring slot. */
+	return ring_idx;
+}
+
+/* Enqueue a copy operation onto the ae4dma device. */
+static int
+ae4dma_enqueue_copy(void *dev_private, uint16_t vchan __rte_unused,
+		rte_iova_t src, rte_iova_t dst, uint32_t length, uint64_t flags)
+{
+	return __write_desc_copy(dev_private, src, dst, length, flags);
+}
+
 static int
 ae4dma_dev_dump(const struct rte_dma_dev *dev, FILE *f)
 {
@@ -192,6 +262,178 @@ ae4dma_dev_dump(const struct rte_dma_dev *dev, FILE *f)
 	return 0;
 }
 
+/* Translate an AE4DMA descriptor error code to the dmadev status code. */
+static inline enum rte_dma_status_code
+__translate_status_ae4dma_to_dma(enum ae4dma_dma_err status)
+{
+	switch (status) {
+	case AE4DMA_DMA_ERR_NO_ERR:
+		return RTE_DMA_STATUS_SUCCESSFUL;
+	case AE4DMA_DMA_ERR_INV_LEN:
+		return RTE_DMA_STATUS_INVALID_LENGTH;
+	case AE4DMA_DMA_ERR_INV_SRC:
+		return RTE_DMA_STATUS_INVALID_SRC_ADDR;
+	case AE4DMA_DMA_ERR_INV_DST:
+		return RTE_DMA_STATUS_INVALID_DST_ADDR;
+	default:
+		return RTE_DMA_STATUS_ERROR_UNKNOWN;
+	}
+}
+
+/* True if the descriptor at ring slot has been flagged as failed by HW. */
+static inline bool
+ae4dma_desc_failed(volatile struct ae4dma_desc *hw_desc)
+{
+	/*
+	 * read_idx advancing is the definitive completion signal. The
+	 * per-descriptor status byte is informational and may not yet be
+	 * written when we observe it, so a slot is only a failure if the
+	 * HW flagged DESC_ERROR or set a non-zero err_code.
+	 */
+	return hw_desc->dw1.status == AE4DMA_DMA_DESC_ERROR ||
+			hw_desc->dw1.err_code != AE4DMA_DMA_ERR_NO_ERR;
+}
+
+/*
+ * Number of descriptors the HW has completed since our last reap, without
+ * consuming them (non-blocking, no state change).
+ *
+ * The AE4DMA engine signals completion by advancing the per-queue read_idx
+ * register; it does not (reliably) write a status word back into the
+ * descriptor. We therefore use the HW read_idx register as the source of
+ * truth for how many ops are done.
+ *
+ * next_read/next_write are free-running 16-bit indices; the HW ring slot is
+ * ((hw_base + idx) & mask) and the number of in-flight ops is
+ * (next_write - next_read), both relying on nb_desc being a power of two.
+ */
+static inline uint16_t
+ae4dma_pending(struct ae4dma_cmd_queue *cmd_q)
+{
+	uint16_t nb = cmd_q->qcfg.nb_desc;
+	uint16_t hw_read_pos, tail_pos;
+	uint16_t newly_done, in_flight;
+	uint16_t mask;
+
+	if (nb == 0)
+		return 0;
+	mask = nb - 1;
+
+	in_flight = (uint16_t)(cmd_q->next_write - cmd_q->next_read);
+	if (in_flight == 0)
+		return 0;
+
+	/* Descriptors HW has consumed since our last visit: [tail, read). */
+	hw_read_pos = (uint16_t)(AE4DMA_READ_REG(&cmd_q->hwq_regs->read_idx) & mask);
+	tail_pos = (uint16_t)((cmd_q->hw_base + cmd_q->next_read) & mask);
+	newly_done = (uint16_t)((hw_read_pos - tail_pos) & mask);
+
+	return RTE_MIN(newly_done, in_flight);
+}
+
+/*
+ * Report ops that completed successfully, stopping at the first failure.
+ *
+ * Per the dmadev contract, this reaps only the run of successful ops at the
+ * head of the completed region and stops as soon as it meets a failed one.
+ * The failed op is left in place (not consumed) and *has_error is set so the
+ * application switches to rte_dma_completed_status() to retrieve its status.
+ * last_idx is the ring_idx of the last successfully completed op.
+ */
+static uint16_t
+ae4dma_completed(void *dev_private, uint16_t vchan __rte_unused,
+		const uint16_t max_ops, uint16_t *last_idx, bool *has_error)
+{
+	struct ae4dma_dmadev *ae4dma = dev_private;
+	struct ae4dma_cmd_queue *cmd_q = &ae4dma->cmd_q;
+	uint16_t mask = cmd_q->qcfg.nb_desc - 1;
+	uint16_t ready, count;
+
+	*has_error = false;
+
+	ready = ae4dma_pending(cmd_q);
+	if (ready > max_ops)
+		ready = max_ops;
+
+	for (count = 0; count < ready; count++) {
+		uint16_t slot = (cmd_q->hw_base + cmd_q->next_read + count) & mask;
+
+		if (ae4dma_desc_failed(&cmd_q->qbase_desc[slot])) {
+			*has_error = true;
+			break;
+		}
+	}
+
+	cmd_q->next_read += count;
+	cmd_q->stats.completed += count;
+	*last_idx = (uint16_t)(cmd_q->next_read - 1);
+
+	return count;
+}
+
+/*
+ * Report per-op status for all completed ops, including failures.
+ *
+ * Unlike ae4dma_completed(), this consumes failed ops too so the application
+ * can drain past an error. last_idx is the ring_idx of the last op reported.
+ */
+static uint16_t
+ae4dma_completed_status(void *dev_private, uint16_t vchan __rte_unused,
+		uint16_t max_ops, uint16_t *last_idx,
+		enum rte_dma_status_code *status)
+{
+	struct ae4dma_dmadev *ae4dma = dev_private;
+	struct ae4dma_cmd_queue *cmd_q = &ae4dma->cmd_q;
+	uint16_t mask = cmd_q->qcfg.nb_desc - 1;
+	uint16_t ready, fails = 0;
+	uint16_t i;
+
+	ready = ae4dma_pending(cmd_q);
+	if (ready > max_ops)
+		ready = max_ops;
+
+	for (i = 0; i < ready; i++) {
+		uint16_t slot = (cmd_q->hw_base + cmd_q->next_read + i) & mask;
+		volatile struct ae4dma_desc *hw_desc = &cmd_q->qbase_desc[slot];
+
+		if (ae4dma_desc_failed(hw_desc)) {
+			status[i] = __translate_status_ae4dma_to_dma(
+					(enum ae4dma_dma_err)hw_desc->dw1.err_code);
+			AE4DMA_PMD_WARN("Desc failed: status=%u err=%u",
+					hw_desc->dw1.status, hw_desc->dw1.err_code);
+			fails++;
+		} else {
+			status[i] = RTE_DMA_STATUS_SUCCESSFUL;
+		}
+	}
+
+	cmd_q->next_read += ready;
+	cmd_q->stats.completed += ready;
+	cmd_q->stats.errors += fails;
+	*last_idx = (uint16_t)(cmd_q->next_read - 1);
+
+	return ready;
+}
+
+/* Get the remaining capacity of the ring. */
+static uint16_t
+ae4dma_burst_capacity(const void *dev_private, uint16_t vchan __rte_unused)
+{
+	const struct ae4dma_dmadev *ae4dma = dev_private;
+	const struct ae4dma_cmd_queue *cmd_q = &ae4dma->cmd_q;
+	uint16_t nb = cmd_q->qcfg.nb_desc;
+	uint16_t in_flight;
+
+	if (nb == 0)
+		return 0;
+
+	/* One slot reserved to distinguish full from empty. */
+	in_flight = (uint16_t)(cmd_q->next_write - cmd_q->next_read);
+	if (in_flight >= nb - 1)
+		return 0;
+	return (uint16_t)(nb - 1 - in_flight);
+}
+
 static int
 ae4dma_stats_get(const struct rte_dma_dev *dev, uint16_t vchan __rte_unused,
 		struct rte_dma_stats *rte_stats, uint32_t size __rte_unused)
@@ -339,6 +581,13 @@ ae4dma_dmadev_create(const char *name, struct rte_pci_device *dev, uint8_t qn)
 	dmadev->fp_obj->dev_private = dmadev->data->dev_private;
 	dmadev->dev_ops = &ae4dma_dmadev_ops;
 
+	dmadev->fp_obj->burst_capacity = ae4dma_burst_capacity;
+	dmadev->fp_obj->completed = ae4dma_completed;
+	dmadev->fp_obj->completed_status = ae4dma_completed_status;
+	dmadev->fp_obj->copy = ae4dma_enqueue_copy;
+	dmadev->fp_obj->submit = ae4dma_submit;
+	/* fill capability not advertised: leave fp_obj->fill as zero-initialised. */
+
 	ae4dma = dmadev->data->dev_private;
 
 	if (ae4dma_add_queue(ae4dma, dev, qn, name) != 0)
-- 
2.34.1