[dpdk-dev] [RFC 1/6] net/virtio: optimize the rx path

Tiwei Bie tiwei.bie at intel.com
Tue Jul 18 17:40:16 CEST 2017


Signed-off-by: Tiwei Bie <tiwei.bie at intel.com>
---
 drivers/net/virtio/virtio_ethdev.c |   5 +-
 drivers/net/virtio/virtio_ethdev.h |   2 +
 drivers/net/virtio/virtio_rxtx.c   | 255 ++++++++++++++++++++-----------------
 3 files changed, 142 insertions(+), 120 deletions(-)

diff --git a/drivers/net/virtio/virtio_ethdev.c b/drivers/net/virtio/virtio_ethdev.c
index 334c4b8..566eede 100644
--- a/drivers/net/virtio/virtio_ethdev.c
+++ b/drivers/net/virtio/virtio_ethdev.c
@@ -1241,7 +1241,10 @@ static void
 rx_func_get(struct rte_eth_dev *eth_dev)
 {
 	struct virtio_hw *hw = eth_dev->data->dev_private;
-	if (0 && vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF))
+
+	if (vtpci_version_1_1(hw))
+		eth_dev->rx_pkt_burst = &virtio_recv_pkts_1_1;
+	else if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF))
 		eth_dev->rx_pkt_burst = &virtio_recv_mergeable_pkts;
 	else
 		eth_dev->rx_pkt_burst = &virtio_recv_pkts;
diff --git a/drivers/net/virtio/virtio_ethdev.h b/drivers/net/virtio/virtio_ethdev.h
index deed34e..8bd699d 100644
--- a/drivers/net/virtio/virtio_ethdev.h
+++ b/drivers/net/virtio/virtio_ethdev.h
@@ -99,6 +99,8 @@ int  virtio_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id,
 
 uint16_t virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 		uint16_t nb_pkts);
+uint16_t virtio_recv_pkts_1_1(void *rx_queue, struct rte_mbuf **rx_pkts,
+		uint16_t nb_pkts);
 
 uint16_t virtio_recv_mergeable_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 		uint16_t nb_pkts);
diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
index 3dc5eaf..3983626 100644
--- a/drivers/net/virtio/virtio_rxtx.c
+++ b/drivers/net/virtio/virtio_rxtx.c
@@ -116,7 +116,7 @@ vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
 }
 
 static inline uint16_t
-virtqueue_dequeue_burst_rx_1_0(struct virtqueue *vq, struct rte_mbuf **rx_pkts,
+virtqueue_dequeue_burst_rx(struct virtqueue *vq, struct rte_mbuf **rx_pkts,
 			   uint32_t *len, uint16_t num)
 {
 	struct vring_used_elem *uep;
@@ -151,53 +151,6 @@ virtqueue_dequeue_burst_rx_1_0(struct virtqueue *vq, struct rte_mbuf **rx_pkts,
 	return i;
 }
 
-static inline uint16_t
-virtqueue_dequeue_burst_rx_1_1(struct virtqueue *vq, struct rte_mbuf **rx_pkts,
-			   uint32_t *len, uint16_t num)
-{
-	struct vring_desc_1_1 *desc = vq->vq_ring.desc_1_1;
-	struct rte_mbuf *cookie;
-	uint16_t used_idx;
-	uint16_t i;
-
-	for (i = 0; i < num ; i++) {
-		used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1));
-		if ((desc[used_idx].flags & DESC_HW))
-			break;
-
-		len[i] = desc[used_idx].len;
-		cookie = vq->vq_descx[used_idx].cookie;
-
-		if (unlikely(cookie == NULL)) {
-			PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u\n",
-				vq->vq_used_cons_idx);
-			break;
-		}
-		vq->vq_descx[used_idx].cookie = NULL;
-
-		rte_prefetch0(cookie);
-#if 0
-		rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *));
-#endif
-		rx_pkts[i]  = cookie;
-
-		vq->vq_used_cons_idx++;
-		vq->vq_free_cnt++;
-	}
-
-	return i;
-}
-
-static inline uint16_t
-virtqueue_dequeue_burst_rx(struct virtqueue *vq, struct rte_mbuf **rx_pkts,
-			   uint32_t *len, uint16_t num)
-{
-	if (vtpci_version_1_1(vq->hw))
-		return virtqueue_dequeue_burst_rx_1_1(vq, rx_pkts, len, num);
-	else
-		return virtqueue_dequeue_burst_rx_1_0(vq, rx_pkts, len, num);
-}
-
 #ifndef DEFAULT_TX_FREE_THRESH
 #define DEFAULT_TX_FREE_THRESH 32
 #endif
@@ -226,9 +179,8 @@ virtio_xmit_cleanup(struct virtqueue *vq, uint16_t num)
 	}
 }
 
-
 static inline int
-virtqueue_enqueue_recv_refill_1_0(struct virtqueue *vq, struct rte_mbuf *cookie)
+virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf *cookie)
 {
 	struct vq_desc_extra *dxp;
 	struct virtio_hw *hw = vq->hw;
@@ -267,50 +219,6 @@ virtqueue_enqueue_recv_refill_1_0(struct virtqueue *vq, struct rte_mbuf *cookie)
 	return 0;
 }
 
-static inline int
-virtqueue_enqueue_recv_refill_1_1(struct virtqueue *vq, struct rte_mbuf *cookie)
-{
-	struct vq_desc_extra *dxp;
-	struct virtio_hw *hw = vq->hw;
-	uint16_t needed = 1;
-	uint16_t idx;
-	struct vring_desc_1_1 *desc = vq->vq_ring.desc_1_1;
-
-	if (unlikely(vq->vq_free_cnt == 0))
-		return -ENOSPC;
-	if (unlikely(vq->vq_free_cnt < needed))
-		return -EMSGSIZE;
-
-	idx = vq->vq_desc_head_idx & (vq->vq_nentries - 1);
-	if (unlikely(desc[idx].flags & DESC_HW))
-		return -EFAULT;
-
-	dxp = &vq->vq_descx[idx];
-	dxp->cookie = cookie;
-	dxp->ndescs = needed;
-
-	desc[idx].addr =
-		VIRTIO_MBUF_ADDR(cookie, vq) +
-		RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size;
-	desc[idx].len =
-		cookie->buf_len - RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size;
-	desc[idx].flags =  VRING_DESC_F_WRITE;
-	vq->vq_desc_head_idx++;
-
-	vq->vq_free_cnt -= needed;
-
-	return 0;
-}
-
-static inline int
-virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf *cookie)
-{
-	if (vtpci_version_1_1(vq->hw))
-		return virtqueue_enqueue_recv_refill_1_1(vq, cookie);
-	else
-		return virtqueue_enqueue_recv_refill_1_0(vq, cookie);
-}
-
 static inline void
 virtqueue_enqueue_xmit(struct virtnet_tx *txvq, struct rte_mbuf *cookie,
 		       uint16_t needed)
@@ -395,6 +303,30 @@ virtio_dev_rx_queue_setup(struct rte_eth_dev *dev,
 	}
 	dev->data->rx_queues[queue_idx] = rxvq;
 
+	if (vtpci_version_1_1(hw)) {
+		struct vring_desc_1_1 *desc;
+		struct vq_desc_extra *dxp;
+
+		for (desc_idx = 0; desc_idx < vq->vq_nentries;
+				desc_idx++) {
+			m = rte_mbuf_raw_alloc(rxvq->mpool);
+			if (unlikely(m == NULL))
+				return -ENOMEM;
+
+			dxp = &vq->vq_descx[desc_idx];
+			dxp->cookie = m;
+			dxp->ndescs = 1;
+
+			desc = &vq->vq_ring.desc_1_1[desc_idx];
+			desc->addr = VIRTIO_MBUF_ADDR(m, vq) +
+				RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size;
+			desc->len = m->buf_len - RTE_PKTMBUF_HEADROOM +
+				hw->vtnet_hdr_size;
+			desc->flags = VRING_DESC_F_WRITE | DESC_HW;
+		}
+
+		return 0;
+	}
 
 	/* Allocate blank mbufs for the each rx descriptor */
 	nbufs = 0;
@@ -692,6 +624,114 @@ rx_offload_enabled(struct virtio_hw *hw)
 		vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_TSO6);
 }
 
+uint16_t
+virtio_recv_pkts_1_1(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+	struct virtnet_rx *rxvq = rx_queue;
+	struct virtqueue *vq = rxvq->vq;
+	struct virtio_hw *hw = vq->hw;
+	struct rte_mbuf *rxm, *nmb;
+	uint16_t nb_rx;
+	uint32_t len;
+	uint32_t i;
+	uint32_t hdr_size;
+	int offload;
+	struct virtio_net_hdr *hdr;
+	struct vring_desc_1_1 *descs = vq->vq_ring.desc_1_1;
+	struct vring_desc_1_1 *desc;
+	uint16_t used_idx = vq->vq_used_cons_idx;
+	struct vq_desc_extra *dxp;
+
+	nb_rx = 0;
+	if (unlikely(hw->started == 0))
+		return nb_rx;
+
+	hdr_size = hw->vtnet_hdr_size;
+	offload = rx_offload_enabled(hw);
+
+	for (i = 0; i < nb_pkts; i++) {
+		desc = &descs[used_idx & (vq->vq_nentries - 1)];
+		if (desc->flags & DESC_HW)
+			break;
+
+		nmb = rte_mbuf_raw_alloc(rxvq->mpool);
+		if (unlikely(nmb == NULL)) {
+			struct rte_eth_dev *dev
+				= &rte_eth_devices[rxvq->port_id];
+			dev->data->rx_mbuf_alloc_failed++;
+			break;
+		}
+
+		dxp = &vq->vq_descx[used_idx & (vq->vq_nentries - 1)];
+
+		len = desc->len;
+		rxm = dxp->cookie;
+		dxp->cookie = nmb;
+		dxp->ndescs = 1;
+
+		desc->addr = VIRTIO_MBUF_ADDR(nmb, vq) +
+			RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size;
+		desc->len = nmb->buf_len - RTE_PKTMBUF_HEADROOM +
+			hw->vtnet_hdr_size;
+		desc->flags = VRING_DESC_F_WRITE;
+
+		PMD_RX_LOG(DEBUG, "packet len:%d", len);
+
+		if (unlikely(len < hdr_size + ETHER_HDR_LEN)) {
+			PMD_RX_LOG(ERR, "Packet drop");
+			rte_pktmbuf_free(rxm);
+			rxvq->stats.errors++;
+			continue;
+		}
+
+		rxm->port = rxvq->port_id;
+		rxm->data_off = RTE_PKTMBUF_HEADROOM;
+		rxm->ol_flags = 0;
+		rxm->vlan_tci = 0;
+
+		rxm->pkt_len = (uint32_t)(len - hdr_size);
+		rxm->data_len = (uint16_t)(len - hdr_size);
+
+		hdr = (struct virtio_net_hdr *)((char *)rxm->buf_addr +
+			RTE_PKTMBUF_HEADROOM - hdr_size);
+
+		if (hw->vlan_strip)
+			rte_vlan_strip(rxm);
+
+		if (offload && virtio_rx_offload(rxm, hdr) < 0) {
+			rte_pktmbuf_free(rxm);
+			rxvq->stats.errors++;
+			continue;
+		}
+
+		VIRTIO_DUMP_PACKET(rxm, rxm->data_len);
+
+		rxvq->stats.bytes += rxm->pkt_len;
+		virtio_update_packet_stats(&rxvq->stats, rxm);
+
+		used_idx++;
+
+		rx_pkts[nb_rx++] = rxm;
+	}
+
+	if (nb_rx > 0) {
+		uint16_t idx = (vq->vq_used_cons_idx + 1) & (vq->vq_nentries - 1);
+		for (i = 1; i < nb_rx; i++) {
+			descs[idx].flags |= DESC_HW;
+			idx = (idx + 1) & (vq->vq_nentries - 1);
+		}
+
+		rte_smp_wmb();
+		descs[vq->vq_used_cons_idx & (vq->vq_nentries - 1)].flags |= DESC_HW;
+	}
+
+	rxvq->stats.packets += nb_rx;
+
+	vq->vq_used_cons_idx = used_idx;
+
+	return nb_rx;
+}
+
 #define VIRTIO_MBUF_BURST_SZ 64
 #define DESC_PER_CACHELINE (RTE_CACHE_LINE_SIZE / sizeof(struct vring_desc))
 uint16_t
@@ -709,22 +749,12 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 	uint32_t hdr_size;
 	int offload;
 	struct virtio_net_hdr *hdr;
-	uint16_t head_idx, idx;
 
 	nb_rx = 0;
 	if (unlikely(hw->started == 0))
 		return nb_rx;
 
-	/*
-	 * we have no idea to know how many used entries without scanning
-	 * the desc for virtio 1.1. Thus, let's simply set nb_used to nb_pkts
-	 * and let virtqueue_dequeue_burst_rx() to figure out the real
-	 * number.
-	 */
-	if (vtpci_version_1_1(hw))
-		nb_used = nb_pkts;
-	else
-		nb_used = VIRTQUEUE_NUSED(vq);
+	nb_used = VIRTQUEUE_NUSED(vq);
 
 	virtio_rmb();
 
@@ -783,8 +813,6 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 
 	rxvq->stats.packets += nb_rx;
 
-	head_idx = vq->vq_desc_head_idx;
-
 	/* Allocate new mbuf for the used descriptor */
 	error = ENOSPC;
 	int count = 0;
@@ -805,18 +833,7 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 		count++;
 	}
 
-	if (vtpci_version_1_1(hw)) {
-		struct vring_desc_1_1 *desc = vq->vq_ring.desc_1_1;
-		if (count > 0) {
-			rte_smp_wmb();
-			idx = head_idx + 1;
-			while (--count) {
-				desc[idx & (vq->vq_nentries - 1)].flags |= DESC_HW;
-				idx++;
-			}
-			desc[head_idx & (vq->vq_nentries - 1)].flags |= DESC_HW;
-		}
-	} else if (likely(nb_enqueued)) {
+	if (likely(nb_enqueued)) {
 		vq_update_avail_idx(vq);
 
 		if (unlikely(virtqueue_kick_prepare(vq))) {
-- 
2.7.4



More information about the dev mailing list