[dpdk-dev] [PATCH v3 19/21] vhost: support mergeable rx buffers with packed queues

Jens Freimann jfreimann at redhat.com
Fri Apr 6 16:07:17 CEST 2018


On Fri, Apr 06, 2018 at 03:04:47PM +0200, Maxime Coquelin wrote:
>
>
>On 04/05/2018 12:10 PM, Jens Freimann wrote:
>>This implements support for mergeable receive buffers in vhost when using
>>packed virtqueues. The difference to split virtqueues is not big, it differs
>>mostly where descriptor flags are touched and virtio features are checked.
>>
>>Signed-off-by: Jens Freimann <jfreimann at redhat.com>
>>---
>>  lib/librte_vhost/vhost.c      |   2 +
>>  lib/librte_vhost/virtio_net.c | 160 +++++++++++++++++++++++++++++++++---------
>>  2 files changed, 127 insertions(+), 35 deletions(-)
>>
>>diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c
>>index eb5a98875..3c633e71e 100644
>>--- a/lib/librte_vhost/vhost.c
>>+++ b/lib/librte_vhost/vhost.c
>>@@ -580,6 +580,8 @@ rte_vhost_enable_guest_notification(int vid, uint16_t queue_id, int enable)
>>  	if (dev == NULL)
>>  		return -1;
>>+	if (dev->features & (1ULL << VIRTIO_F_RING_PACKED))
>>+		return -1;
>
>Is this change really related to mergeable buffers?

no, will fix

>
>>  	if (enable) {
>>  		RTE_LOG(ERR, VHOST_CONFIG,
>>diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c
>>index 18e67fdc1..b82c24081 100644
>>--- a/lib/librte_vhost/virtio_net.c
>>+++ b/lib/librte_vhost/virtio_net.c
>>@@ -401,17 +401,53 @@ virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id,
>>  }
>>  static __rte_always_inline int
>>-fill_vec_buf(struct virtio_net *dev, struct vhost_virtqueue *vq,
>>-			 uint32_t avail_idx, uint32_t *vec_idx,
>>-			 struct buf_vector *buf_vec, uint16_t *desc_chain_head,
>>-			 uint16_t *desc_chain_len)
>>+__fill_vec_buf_packed(struct virtio_net *dev, struct vhost_virtqueue *vq,
>>+			 struct buf_vector *buf_vec,
>>+			 uint32_t *len, uint32_t *vec_id)
>>+{
>>+	uint16_t idx = vq->last_avail_idx & (vq->size - 1);
>>+	struct vring_desc_packed *descs= vq->desc_packed;
>>+	uint32_t _vec_id = *vec_id;
>>+
>>+	if (vq->desc_packed[idx].flags & VRING_DESC_F_INDIRECT) {
>>+		descs = (struct vring_desc_packed *)(uintptr_t)
>>+			vhost_iova_to_vva(dev, vq, vq->desc_packed[idx].addr,
>>+						vq->desc_packed[idx].len,
>>+						VHOST_ACCESS_RO);
>>+		if (unlikely(!descs))
>>+			return -1;
>>+
>>+		idx = 0;
>>+	}
>>+
>>+	while (1) {
>>+		if (unlikely(_vec_id >= BUF_VECTOR_MAX || idx >= vq->size))
>>+			return -1;
>>+
>>+		*len += descs[idx & (vq->size - 1)].len;
>>+		buf_vec[_vec_id].buf_addr = descs[idx].addr;
>>+		buf_vec[_vec_id].buf_len  = descs[idx].len;
>>+		buf_vec[_vec_id].desc_idx = idx;
>>+		_vec_id++;
>>+
>>+		if ((descs[idx & (vq->size - 1)].flags & VRING_DESC_F_NEXT) == 0)
>>+			break;
>>+
>>+		idx++;
>>+	}
>>+	*vec_id = _vec_id;
>>+
>>+	return 0;
>>+}
>>+
>>+static __rte_always_inline int
>>+__fill_vec_buf_split(struct virtio_net *dev, struct vhost_virtqueue *vq,
>>+			 struct buf_vector *buf_vec,
>>+			 uint32_t *len, uint32_t *vec_id, uint32_t avail_idx)
>>  {
>>  	uint16_t idx = vq->avail->ring[avail_idx & (vq->size - 1)];
>>-	uint32_t vec_id = *vec_idx;
>>-	uint32_t len    = 0;
>>  	struct vring_desc *descs = vq->desc;
>>-
>>-	*desc_chain_head = idx;
>>+	uint32_t _vec_id = *vec_id;
>>  	if (vq->desc[idx].flags & VRING_DESC_F_INDIRECT) {
>>  		descs = (struct vring_desc *)(uintptr_t)
>>@@ -425,20 +461,53 @@ fill_vec_buf(struct virtio_net *dev, struct vhost_virtqueue *vq,
>>  	}
>>  	while (1) {
>>-		if (unlikely(vec_id >= BUF_VECTOR_MAX || idx >= vq->size))
>>+		if (unlikely(_vec_id >= BUF_VECTOR_MAX || idx >= vq->size))
>>  			return -1;
>>-		len += descs[idx].len;
>>-		buf_vec[vec_id].buf_addr = descs[idx].addr;
>>-		buf_vec[vec_id].buf_len  = descs[idx].len;
>>-		buf_vec[vec_id].desc_idx = idx;
>>-		vec_id++;
>>+		*len += descs[idx].len;
>>+		buf_vec[_vec_id].buf_addr = descs[idx].addr;
>>+		buf_vec[_vec_id].buf_len  = descs[idx].len;
>>+		buf_vec[_vec_id].desc_idx = idx;
>>+		_vec_id++;
>>  		if ((descs[idx].flags & VRING_DESC_F_NEXT) == 0)
>>  			break;
>>  		idx = descs[idx].next;
>>  	}
>>+	*vec_id = _vec_id;
>>+
>>+	return 0;
>>+}
>>+
>>+static __rte_always_inline int
>>+fill_vec_buf(struct virtio_net *dev, struct vhost_virtqueue *vq,
>>+			 uint32_t avail_idx, uint32_t *vec_idx,
>>+			 struct buf_vector *buf_vec, uint16_t *desc_chain_head,
>>+			 uint16_t *desc_chain_len)
>>+{
>>+	uint16_t idx;
>>+	uint32_t vec_id = *vec_idx;
>>+	uint32_t len    = 0;
>>+
>>+	if (dev->features & (1ULL << VIRTIO_F_RING_PACKED)) {
>>+		idx = vq->last_avail_idx & (vq->size -1);
>>+	} else {
>>+		idx = vq->avail->ring[avail_idx & (vq->size - 1)];
>>+	}
>>+
>>+
>>+	*desc_chain_head = idx;
>>+
>>+	if (dev->features & (1ULL << VIRTIO_F_RING_PACKED)) {
>>+		if (__fill_vec_buf_packed(dev, vq,
>>+				buf_vec, &len, &vec_id))
>>+			return -1;
>>+	} else {
>>+		if (__fill_vec_buf_split(dev, vq,
>>+				buf_vec, &len, &vec_id, avail_idx))
>>+			return -1;
>>+	}
>>  	*desc_chain_len = len;
>>  	*vec_idx = vec_id;
>>@@ -465,14 +534,16 @@ reserve_avail_buf_mergeable(struct virtio_net *dev, struct vhost_virtqueue *vq,
>>  	cur_idx  = vq->last_avail_idx;
>>  	while (size > 0) {
>>-		if (unlikely(cur_idx == avail_head))
>>+		if (unlikely(cur_idx == avail_head) &&
>>+			!(dev->features & (1ull < VIRTIO_F_RING_PACKED)))
>>  			return -1;
>>  		if (unlikely(fill_vec_buf(dev, vq, cur_idx, &vec_idx, buf_vec,
>>  						&head_idx, &len) < 0))
>>  			return -1;
>>  		len = RTE_MIN(len, size);
>>-		update_shadow_used_ring(vq, head_idx, len);
>>+		if (!(dev->features & (1ULL << VIRTIO_F_RING_PACKED)))
>
>This is cosmetics, but I wonder if we could have a helper like:
>
>static inline bool
>vring_is_packed(struct virtio_net *dev) {
>	return dev->features & (1ULL << VIRTIO_F_RING_PACKED);
>}

Yes, I will add one

>
>>+			update_shadow_used_ring(vq, head_idx, len);
>>  		size -= len;
>>  		cur_idx++;
>>@@ -620,6 +691,8 @@ virtio_dev_merge_rx(struct virtio_net *dev, uint16_t queue_id,
>>  	uint16_t num_buffers;
>>  	struct buf_vector buf_vec[BUF_VECTOR_MAX];
>>  	uint16_t avail_head;
>>+	uint16_t i;
>>+	struct vring_desc_packed *descs = NULL;
>>  	VHOST_LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__);
>>  	if (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->nr_vring))) {
>>@@ -634,7 +707,6 @@ virtio_dev_merge_rx(struct virtio_net *dev, uint16_t queue_id,
>>  	if (unlikely(vq->enabled == 0))
>>  		goto out_access_unlock;
>>-
>>  	if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
>>  		vhost_user_iotlb_rd_lock(vq);
>>@@ -648,10 +720,14 @@ virtio_dev_merge_rx(struct virtio_net *dev, uint16_t queue_id,
>>  	vq->batch_copy_nb_elems = 0;
>>-	rte_prefetch0(&vq->avail->ring[vq->last_avail_idx & (vq->size - 1)]);
>>-
>>-	vq->shadow_used_idx = 0;
>>-	avail_head = *((volatile uint16_t *)&vq->avail->idx);
>>+	if (dev->features & (1ULL << VIRTIO_F_RING_PACKED)) {
>>+		avail_head = vq->last_avail_idx;
>>+		descs = vq->desc_packed;
>>+	} else {
>>+		rte_prefetch0(&vq->avail->ring[vq->last_avail_idx & (vq->size - 1)]);
>>+		avail_head = *((volatile uint16_t *)&vq->avail->idx);
>>+		vq->shadow_used_idx = 0;
>>+	}
>>  	for (pkt_idx = 0; pkt_idx < count; pkt_idx++) {
>>  		uint32_t pkt_len = pkts[pkt_idx]->pkt_len + dev->vhost_hlen;
>>@@ -661,7 +737,9 @@ virtio_dev_merge_rx(struct virtio_net *dev, uint16_t queue_id,
>>  			VHOST_LOG_DEBUG(VHOST_DATA,
>>  				"(%d) failed to get enough desc from vring\n",
>>  				dev->vid);
>>-			vq->shadow_used_idx -= num_buffers;
>>+
>>+			if (!dev->features & (1ULL & VIRTIO_F_RING_PACKED))
>>+				vq->shadow_used_idx -= num_buffers;
>>  			break;
>>  		}
>>@@ -671,7 +749,8 @@ virtio_dev_merge_rx(struct virtio_net *dev, uint16_t queue_id,
>>  		if (copy_mbuf_to_desc_mergeable(dev, vq, pkts[pkt_idx],
>>  						buf_vec, num_buffers) < 0) {
>>-			vq->shadow_used_idx -= num_buffers;
>>+			if (!dev->features & (1ULL & VIRTIO_F_RING_PACKED))
>>+				vq->shadow_used_idx -= num_buffers;
>>  			break;
>>  		}
>>@@ -680,9 +759,18 @@ virtio_dev_merge_rx(struct virtio_net *dev, uint16_t queue_id,
>>  	do_data_copy_enqueue(dev, vq);
>>-	if (likely(vq->shadow_used_idx)) {
>>-		flush_shadow_used_ring(dev, vq);
>>-		vhost_vring_call(dev, vq);
>>+	if (!(dev->features & (1ULL << VIRTIO_F_RING_PACKED))) {
>>+		if (likely(vq->shadow_used_idx)) {
>>+			flush_shadow_used_ring(dev, vq);
>>+			vhost_vring_call(dev, vq);
>>+		}
>>+	} else {
>>+		rte_smp_wmb();
>>+		for (i = avail_head; i < vq->last_avail_idx; i++) {
>>+			if ((i & (vq->size - 1)) == 0)
>>+				toggle_wrap_counter(vq);
>>+			set_desc_used(vq, &descs[i & (vq->size - 1)]);
>>+		}
>>  	}
>>  out:
>>@@ -774,7 +862,7 @@ vhost_enqueue_burst_packed(struct virtio_net *dev, uint16_t queue_id,
>>  					goto out;
>>  				}
>>-				idx = (idx+1) & (vq->size - 1);
>>+				idx = (idx + 1) & mask;
>Are you sure this change is in the right patch?

Pretty sure it's not :)

>
>>  				desc = &descs[idx];
>>  				if (unlikely(!desc_is_avail(vq, desc)))
>>  					goto out ;
>>@@ -840,10 +928,11 @@ rte_vhost_enqueue_burst(int vid, uint16_t queue_id,
>>  		return 0;
>>  	}
>>-	if (dev->features & (1ULL << VIRTIO_F_RING_PACKED))
>>-		return vhost_enqueue_burst_packed(dev, queue_id, pkts, count);
>>-	else if (dev->features & (1 << VIRTIO_NET_F_MRG_RXBUF))
>>+	if ((dev->features & (1 << VIRTIO_NET_F_MRG_RXBUF)) &&
>>+	    (dev->features & (1ULL << VIRTIO_F_RING_PACKED)))
>>  		return virtio_dev_merge_rx(dev, queue_id, pkts, count);
>>+	else if (dev->features & (1ULL << VIRTIO_F_RING_PACKED))
>>+		return vhost_enqueue_burst_packed(dev, queue_id, pkts, count);
>>  	else
>>  		return virtio_dev_rx(dev, queue_id, pkts, count);
>>  }
>>@@ -1266,8 +1355,6 @@ dequeue_desc_packed(struct virtio_net *dev, struct vhost_virtqueue *vq,
>>  	int wrap_counter = vq->used_wrap_counter;
>>  	int rc = 0;
>>-	rte_spinlock_lock(&vq->access_lock);
>>-
>hmm
>
>>  	if (unlikely(vq->enabled == 0))
>>  		goto out;
>>@@ -1451,6 +1538,9 @@ vhost_dequeue_burst_packed(struct virtio_net *dev, struct vhost_virtqueue *vq,
>>  	struct vring_desc_packed *desc = vq->desc_packed;
>>  	int err;
>>+	if (unlikely(rte_spinlock_trylock(&vq->access_lock) == 0))
>>+		return 0;
>>+
>>  	count = RTE_MIN(MAX_PKT_BURST, count);
>>  	for (i = 0; i < count; i++) {
>>  		idx = vq->last_used_idx & (vq->size - 1);
>>@@ -1509,15 +1599,15 @@ rte_vhost_dequeue_burst(int vid, uint16_t queue_id,
>>  	vq = dev->virtqueue[queue_id];
>>+	if (dev->features & (1ULL << VIRTIO_F_RING_PACKED))
>>+		return vhost_dequeue_burst_packed(dev, vq, mbuf_pool, pkts, count);
>>+
>>  	if (unlikely(rte_spinlock_trylock(&vq->access_lock) == 0))
>>  		return 0;
>>  	if (unlikely(vq->enabled == 0))
>>  		goto out_access_unlock;
>>-	if (dev->features & (1ULL << VIRTIO_F_RING_PACKED))
>>-		return vhost_dequeue_burst_packed(dev, vq, mbuf_pool, pkts, count);
>>-
>>  	vq->batch_copy_nb_elems = 0;
>>  	if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))
>>
>
>all above changes, if valid, are for dequeue, so 'tx', so certainly in 
>the wrong patch.

yes, I'll move them. Thanks!

regards,
Jens 


More information about the dev mailing list