<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=us-ascii">
<style type="text/css" style="display:none;"> P {margin-top:0;margin-bottom:0;} </style>
</head>
<body dir="ltr">
<div class="elementToProof" style="direction: ltr; font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
Thanks for the your help, patch queued to 23.11.4 LTS release staging.</div>
<div class="elementToProof" style="direction: ltr; font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
<br>
</div>
<div class="elementToProof" style="direction: ltr; font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
Regards,</div>
<div class="elementToProof" style="direction: ltr; font-family: Aptos, Aptos_EmbeddedFont, Aptos_MSFontService, Calibri, Helvetica, sans-serif; font-size: 12pt; color: rgb(0, 0, 0);">
Xueming</div>
<div id="appendonsend"></div>
<hr style="display:inline-block;width:98%" tabindex="-1">
<div id="divRplyFwdMsg" dir="ltr"><font face="Calibri, sans-serif" style="font-size:11pt" color="#000000"><b>From:</b> Joshua Washington <joshwash@google.com><br>
<b>Sent:</b> Tuesday, March 4, 2025 7:06 AM<br>
<b>To:</b> stable@dpdk.org <stable@dpdk.org>; Junfeng Guo <junfeng.guo@intel.com>; Jeroen de Borst <jeroendb@google.com>; Rushil Gupta <rushilg@google.com>; Joshua Washington <joshwash@google.com>; Xiaoyun Li <xiaoyun.li@intel.com><br>
<b>Cc:</b> Praveen Kaligineedi <pkaligineedi@google.com><br>
<b>Subject:</b> [PATCH 23.11] net/gve: allocate Rx QPL pages using malloc</font>
<div> </div>
</div>
<div class="BodyFragment"><font size="2"><span style="font-size:11pt;">
<div class="PlainText">From: Praveen Kaligineedi <pkaligineedi@google.com><br>
<br>
Allocating QPL for an RX queue might fail if enough contiguous IOVA<br>
memory cannot be allocated. This can commonly occur when using 2MB huge<br>
pages because the 1024 4K buffers are allocated for each RX ring by<br>
default, resulting in 4MB for each ring. However, the only requirement<br>
for RX QPLs is that each 4K buffer be IOVA contiguous, not the entire<br>
QPL. Therefore, malloc will be used to allocate RX QPLs instead.<br>
<br>
Note that TX queues require the entire QPL to be IOVA contiguous, so it<br>
will continue to use the memzone-based allocation.<br>
<br>
Fixes: a46583cf43c8 ("net/gve: support Rx/Tx")<br>
Cc: stable@dpdk.org<br>
<br>
Signed-off-by: Praveen Kaligineedi <pkaligineedi@google.com><br>
Signed-off-by: Joshua Washington <joshwash@google.com><br>
---<br>
 drivers/net/gve/gve_ethdev.c | 139 +++++++++++++++++++++++++++++------<br>
 drivers/net/gve/gve_ethdev.h |   5 +-<br>
 drivers/net/gve/gve_rx.c     |   2 +-<br>
 3 files changed, 122 insertions(+), 24 deletions(-)<br>
<br>
diff --git a/drivers/net/gve/gve_ethdev.c b/drivers/net/gve/gve_ethdev.c<br>
index ecd37ff37f..d020e0be66 100644<br>
--- a/drivers/net/gve/gve_ethdev.c<br>
+++ b/drivers/net/gve/gve_ethdev.c<br>
@@ -20,13 +20,45 @@ gve_write_version(uint8_t *driver_version_register)<br>
         writeb('\n', driver_version_register);<br>
 }<br>
 <br>
+static const struct rte_memzone *<br>
+gve_alloc_using_mz(const char *name, uint32_t num_pages)<br>
+{<br>
+       const struct rte_memzone *mz;<br>
+       mz = rte_memzone_reserve_aligned(name, num_pages * PAGE_SIZE,<br>
+                                        rte_socket_id(),<br>
+                                        RTE_MEMZONE_IOVA_CONTIG, PAGE_SIZE);<br>
+       if (mz == NULL)<br>
+               PMD_DRV_LOG(ERR, "Failed to alloc memzone %s.", name);<br>
+       return mz;<br>
+}<br>
+<br>
 static int<br>
-gve_alloc_queue_page_list(struct gve_priv *priv, uint32_t id, uint32_t pages)<br>
+gve_alloc_using_malloc(void **bufs, uint32_t num_entries)<br>
+{<br>
+       uint32_t i;<br>
+<br>
+       for (i = 0; i < num_entries; i++) {<br>
+               bufs[i] = rte_malloc_socket(NULL, PAGE_SIZE, PAGE_SIZE, rte_socket_id());<br>
+               if (bufs[i] == NULL) {<br>
+                       PMD_DRV_LOG(ERR, "Failed to malloc");<br>
+                       goto free_bufs;<br>
+               }<br>
+       }<br>
+       return 0;<br>
+<br>
+free_bufs:<br>
+       while (i > 0)<br>
+               rte_free(bufs[--i]);<br>
+<br>
+       return -ENOMEM;<br>
+}<br>
+<br>
+static int<br>
+gve_alloc_queue_page_list(struct gve_priv *priv, uint32_t id, uint32_t pages,<br>
+                         bool is_rx)<br>
 {<br>
-       char z_name[RTE_MEMZONE_NAMESIZE];<br>
         struct gve_queue_page_list *qpl;<br>
-       const struct rte_memzone *mz;<br>
-       dma_addr_t page_bus;<br>
+       int err = 0;<br>
         uint32_t i;<br>
 <br>
         if (priv->num_registered_pages + pages ><br>
@@ -37,31 +69,79 @@ gve_alloc_queue_page_list(struct gve_priv *priv, uint32_t id, uint32_t pages)<br>
                 return -EINVAL;<br>
         }<br>
         qpl = &priv->qpl[id];<br>
-       snprintf(z_name, sizeof(z_name), "gve_%s_qpl%d", priv->pci_dev->device.name, id);<br>
-       mz = rte_memzone_reserve_aligned(z_name, pages * PAGE_SIZE,<br>
-                                        rte_socket_id(),<br>
-                                        RTE_MEMZONE_IOVA_CONTIG, PAGE_SIZE);<br>
-       if (mz == NULL) {<br>
-               PMD_DRV_LOG(ERR, "Failed to alloc %s.", z_name);<br>
-               return -ENOMEM;<br>
-       }<br>
+<br>
         qpl->page_buses = rte_zmalloc("qpl page buses", pages * sizeof(dma_addr_t), 0);<br>
         if (qpl->page_buses == NULL) {<br>
                 PMD_DRV_LOG(ERR, "Failed to alloc qpl %u page buses", id);<br>
                 return -ENOMEM;<br>
         }<br>
-       page_bus = mz->iova;<br>
-       for (i = 0; i < pages; i++) {<br>
-               qpl->page_buses[i] = page_bus;<br>
-               page_bus += PAGE_SIZE;<br>
+<br>
+       if (is_rx) {<br>
+               /* RX QPL need not be IOVA contiguous.<br>
+                * Allocate 4K size buffers using malloc<br>
+                */<br>
+               qpl->qpl_bufs = rte_zmalloc("qpl bufs",<br>
+                       pages * sizeof(void *), 0);<br>
+               if (qpl->qpl_bufs == NULL) {<br>
+                       PMD_DRV_LOG(ERR, "Failed to alloc qpl bufs");<br>
+                       err = -ENOMEM;<br>
+                       goto free_qpl_page_buses;<br>
+               }<br>
+<br>
+               err = gve_alloc_using_malloc(qpl->qpl_bufs, pages);<br>
+               if (err)<br>
+                       goto free_qpl_page_bufs;<br>
+<br>
+               /* Populate the IOVA addresses */<br>
+               for (i = 0; i < pages; i++)<br>
+                       qpl->page_buses[i] =<br>
+                               rte_malloc_virt2iova(qpl->qpl_bufs[i]);<br>
+       } else {<br>
+               char z_name[RTE_MEMZONE_NAMESIZE];<br>
+<br>
+               snprintf(z_name, sizeof(z_name), "gve_%s_qpl%d", priv->pci_dev->device.name, id);<br>
+<br>
+               /* TX QPL needs to be IOVA contiguous<br>
+                * Allocate QPL using memzone<br>
+                */<br>
+               qpl->mz = gve_alloc_using_mz(z_name, pages);<br>
+               if (!qpl->mz) {<br>
+                       err = -ENOMEM;<br>
+                       goto free_qpl_page_buses;<br>
+               }<br>
+<br>
+               /* Populate the IOVA addresses */<br>
+               for (i = 0; i < pages; i++)<br>
+                       qpl->page_buses[i] = qpl->mz->iova + i * PAGE_SIZE;<br>
         }<br>
+<br>
         qpl->id = id;<br>
-       qpl->mz = mz;<br>
         qpl->num_entries = pages;<br>
 <br>
         priv->num_registered_pages += pages;<br>
 <br>
         return 0;<br>
+<br>
+free_qpl_page_bufs:<br>
+       rte_free(qpl->qpl_bufs);<br>
+free_qpl_page_buses:<br>
+       rte_free(qpl->page_buses);<br>
+       return err;<br>
+}<br>
+<br>
+/*<br>
+ * Free QPL bufs in RX QPLs. Should not be used on TX QPLs.<br>
+ **/<br>
+static void<br>
+gve_free_qpl_bufs(struct gve_queue_page_list *qpl)<br>
+{<br>
+       uint32_t i;<br>
+<br>
+       for (i = 0; i < qpl->num_entries; i++)<br>
+               rte_free(qpl->qpl_bufs[i]);<br>
+<br>
+       rte_free(qpl->qpl_bufs);<br>
+       qpl->qpl_bufs = NULL;<br>
 }<br>
 <br>
 static void<br>
@@ -74,9 +154,19 @@ gve_free_qpls(struct gve_priv *priv)<br>
         if (priv->queue_format != GVE_GQI_QPL_FORMAT)<br>
                 return;<br>
 <br>
-       for (i = 0; i < nb_txqs + nb_rxqs; i++) {<br>
-               if (priv->qpl[i].mz != NULL)<br>
+       /* Free TX QPLs. */<br>
+       for (i = 0; i < nb_txqs; i++) {<br>
+               if (priv->qpl[i].mz) {<br>
                         rte_memzone_free(priv->qpl[i].mz);<br>
+                       priv->qpl[i].mz = NULL;<br>
+               }<br>
+               rte_free(priv->qpl[i].page_buses);<br>
+       }<br>
+<br>
+       /* Free RX QPLs. */<br>
+       for (; i < nb_rxqs; i++) {<br>
+               if (priv->qpl[i].qpl_bufs)<br>
+                       gve_free_qpl_bufs(&priv->qpl[i]);<br>
                 rte_free(priv->qpl[i].page_buses);<br>
         }<br>
 <br>
@@ -755,11 +845,16 @@ gve_init_priv(struct gve_priv *priv, bool skip_describe_device)<br>
                 }<br>
 <br>
                 for (i = 0; i < priv->max_nb_txq + priv->max_nb_rxq; i++) {<br>
-                       if (i < priv->max_nb_txq)<br>
+                       bool is_rx;<br>
+<br>
+                       if (i < priv->max_nb_txq) {<br>
                                 pages = priv->tx_pages_per_qpl;<br>
-                       else<br>
+                               is_rx = false;<br>
+                       } else {<br>
                                 pages = priv->rx_data_slot_cnt;<br>
-                       err = gve_alloc_queue_page_list(priv, i, pages);<br>
+                               is_rx = true;<br>
+                       }<br>
+                       err = gve_alloc_queue_page_list(priv, i, pages, is_rx);<br>
                         if (err != 0) {<br>
                                 PMD_DRV_LOG(ERR, "Failed to alloc qpl %u.", i);<br>
                                 goto err_qpl;<br>
diff --git a/drivers/net/gve/gve_ethdev.h b/drivers/net/gve/gve_ethdev.h<br>
index 58d8943e71..59febc153e 100644<br>
--- a/drivers/net/gve/gve_ethdev.h<br>
+++ b/drivers/net/gve/gve_ethdev.h<br>
@@ -40,7 +40,10 @@ struct gve_queue_page_list {<br>
         uint32_t id; /* unique id */<br>
         uint32_t num_entries;<br>
         dma_addr_t *page_buses; /* the dma addrs of the pages */<br>
-       const struct rte_memzone *mz;<br>
+       union {<br>
+               const struct rte_memzone *mz; /* memzone allocated for TX queue */<br>
+               void **qpl_bufs; /* RX qpl-buffer list allocated using malloc*/<br>
+       };<br>
 };<br>
 <br>
 /* A TX desc ring entry */<br>
diff --git a/drivers/net/gve/gve_rx.c b/drivers/net/gve/gve_rx.c<br>
index 36a1b73c65..b8ef625b5c 100644<br>
--- a/drivers/net/gve/gve_rx.c<br>
+++ b/drivers/net/gve/gve_rx.c<br>
@@ -117,7 +117,7 @@ gve_rx_mbuf(struct gve_rx_queue *rxq, struct rte_mbuf *rxe, uint16_t len,<br>
                 rxq->ctx.mbuf_tail = rxe;<br>
         }<br>
         if (rxq->is_gqi_qpl) {<br>
-               addr = (uint64_t)(rxq->qpl->mz->addr) + rx_id * PAGE_SIZE + padding;<br>
+               addr = (uint64_t)rxq->qpl->qpl_bufs[rx_id] + padding;<br>
                 rte_memcpy((void *)((size_t)rxe->buf_addr + rxe->data_off),<br>
                                     (void *)(size_t)addr, len);<br>
         }<br>
-- <br>
2.48.1.601.g30ceb7b040-goog<br>
<br>
</div>
</span></font></div>
</body>
</html>