[dpdk-dev] [PATCH 06/10] vdpa/sfc: add support for dev conf and dev close ops

[Maxime Coquelin]

On 7/6/21 6:44 PM, Vijay Srivastava wrote:
> From: Vijay Kumar Srivastava <vsrivast at xilinx.com>
> 
> Implement vDPA ops dev_conf and dev_close for DMA mapping,
> interrupt and virtqueue configurations.
> 
> Signed-off-by: Vijay Kumar Srivastava <vsrivast at xilinx.com>
> ---
>  drivers/vdpa/sfc/sfc_vdpa.c     |   6 +
>  drivers/vdpa/sfc/sfc_vdpa.h     |  43 ++++
>  drivers/vdpa/sfc/sfc_vdpa_hw.c  |  70 ++++++
>  drivers/vdpa/sfc/sfc_vdpa_ops.c | 527 ++++++++++++++++++++++++++++++++++++++--
>  drivers/vdpa/sfc/sfc_vdpa_ops.h |  28 +++
>  5 files changed, 654 insertions(+), 20 deletions(-)
> 

...

> diff --git a/drivers/vdpa/sfc/sfc_vdpa_hw.c b/drivers/vdpa/sfc/sfc_vdpa_hw.c
> index 84e680f..047bcc4 100644
> --- a/drivers/vdpa/sfc/sfc_vdpa_hw.c
> +++ b/drivers/vdpa/sfc/sfc_vdpa_hw.c
> @@ -8,6 +8,7 @@
>  #include <rte_common.h>
>  #include <rte_errno.h>
>  #include <rte_vfio.h>
> +#include <rte_vhost.h>
>  
>  #include "efx.h"
>  #include "sfc_vdpa.h"
> @@ -104,6 +105,75 @@
>  	memset(esmp, 0, sizeof(*esmp));
>  }
>  
> +int
> +sfc_vdpa_dma_map(struct sfc_vdpa_ops_data *ops_data, bool do_map)
> +{
> +	uint32_t i, j;
> +	int rc;
> +	struct rte_vhost_memory *vhost_mem = NULL;
> +	struct rte_vhost_mem_region *mem_reg = NULL;
> +	int vfio_container_fd;
> +	void *dev;
> +
> +	dev = ops_data->dev_handle;
> +	vfio_container_fd =
> +		sfc_vdpa_adapter_by_dev_handle(dev)->vfio_container_fd;
> +
> +	rc = rte_vhost_get_mem_table(ops_data->vid, &vhost_mem);
> +	if (rc < 0) {
> +		sfc_vdpa_err(dev,
> +			     "failed to get VM memory layout");
> +		goto error;
> +	}
> +
> +	for (i = 0; i < vhost_mem->nregions; i++) {
> +		mem_reg = &vhost_mem->regions[i];
> +
> +		if (do_map) {
> +			rc = rte_vfio_container_dma_map(vfio_container_fd,
> +						mem_reg->host_user_addr,
> +						mem_reg->guest_phys_addr,
> +						mem_reg->size);
> +			if (rc < 0) {
> +				sfc_vdpa_err(dev,
> +					     "DMA map failed : %s",
> +					     rte_strerror(rte_errno));
> +				goto failed_vfio_dma_map;
> +			}
> +		} else {
> +			rc = rte_vfio_container_dma_unmap(vfio_container_fd,
> +						mem_reg->host_user_addr,
> +						mem_reg->guest_phys_addr,
> +						mem_reg->size);
> +			if (rc < 0) {
> +				sfc_vdpa_err(dev,
> +					     "DMA unmap failed : %s",
> +					     rte_strerror(rte_errno));
> +				goto error;
> +			}
> +		}
> +	}
> +
> +	free(vhost_mem);
> +
> +	return 0;
> +
> +failed_vfio_dma_map:
> +	for (j = 0; j < i; j++) {
> +		mem_reg = &vhost_mem->regions[j];
> +		rc = rte_vfio_container_dma_unmap(vfio_container_fd,
> +						  mem_reg->host_user_addr,
> +						  mem_reg->guest_phys_addr,
> +						  mem_reg->size);
> +	}
> +
> +error:
> +	if (vhost_mem)

The NULL check is not necessary, free() takes care of doing it.

> +		free(vhost_mem);
> +
> +	return rc;
> +}
> +
>  static int
>  sfc_vdpa_mem_bar_init(struct sfc_vdpa_adapter *sva,
>  		      const efx_bar_region_t *mem_ebrp)
> diff --git a/drivers/vdpa/sfc/sfc_vdpa_ops.c b/drivers/vdpa/sfc/sfc_vdpa_ops.c

...

>  
> +static uint64_t
> +hva_to_gpa(int vid, uint64_t hva)
> +{
> +	struct rte_vhost_memory *vhost_mem = NULL;
> +	struct rte_vhost_mem_region *mem_reg = NULL;
> +	uint32_t i;
> +	uint64_t gpa = 0;
> +
> +	if (rte_vhost_get_mem_table(vid, &vhost_mem) < 0)
> +		goto error;
> +
> +	for (i = 0; i < vhost_mem->nregions; i++) {
> +		mem_reg = &vhost_mem->regions[i];
> +
> +		if (hva >= mem_reg->host_user_addr &&
> +				hva < mem_reg->host_user_addr + mem_reg->size) {
> +			gpa = (hva - mem_reg->host_user_addr) +
> +				mem_reg->guest_phys_addr;
> +			break;
> +		}
> +	}
> +
> +error:
> +	if (vhost_mem)

Ditto.

> +		free(vhost_mem);
> +	return gpa;
> +}
> +
> +static int
> +sfc_vdpa_enable_vfio_intr(struct sfc_vdpa_ops_data *ops_data)
> +{
> +	int rc;
> +	int *irq_fd_ptr;
> +	int vfio_dev_fd;
> +	uint32_t i, num_vring;
> +	struct rte_vhost_vring vring;
> +	struct vfio_irq_set *irq_set;
> +	struct rte_pci_device *pci_dev;
> +	char irq_set_buf[SFC_VDPA_MSIX_IRQ_SET_BUF_LEN];
> +	void *dev;
> +
> +	num_vring = rte_vhost_get_vring_num(ops_data->vid);
> +	dev = ops_data->dev_handle;
> +	vfio_dev_fd = sfc_vdpa_adapter_by_dev_handle(dev)->vfio_dev_fd;
> +	pci_dev = sfc_vdpa_adapter_by_dev_handle(dev)->pdev;
> +
> +	irq_set = (struct vfio_irq_set *)irq_set_buf;
> +	irq_set->argsz = sizeof(irq_set_buf);
> +	irq_set->count = num_vring + 1;
> +	irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
> +			 VFIO_IRQ_SET_ACTION_TRIGGER;
> +	irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
> +	irq_set->start = 0;
> +	irq_fd_ptr = (int *)&irq_set->data;
> +	irq_fd_ptr[RTE_INTR_VEC_ZERO_OFFSET] = pci_dev->intr_handle.fd;
> +
> +	for (i = 0; i < num_vring; i++) {
> +		rte_vhost_get_vhost_vring(ops_data->vid, i, &vring);

This function may fail (even if really unlikely that it happens), maybe
better to avoid using non-initialized callfd value if it happened.

> +		irq_fd_ptr[RTE_INTR_VEC_RXTX_OFFSET + i] = vring.callfd;
> +	}
> +
> +	rc = ioctl(vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
> +	if (rc) {
> +		sfc_vdpa_err(ops_data->dev_handle,
> +			     "error enabling MSI-X interrupts: %s",
> +			     strerror(errno));
> +		return -1;
> +	}
> +
> +	return 0;
> +}
> +