[dpdk-dev] [PATCH v2 3/3] net/ifcvf: add ifcvf driver

[Maxime Coquelin]

On 03/21/2018 02:21 PM, Xiao Wang wrote:
> ifcvf driver uses vdev as a control domain to manage ifc VFs that belong
> to it. It registers vDPA device ops to vhost lib to enable these VFs to be
> used as vhost data path accelerator.
> 
> Live migration feature is supported by ifc VF and this driver enables
> it based on vhost lib.
> 
> Because vDPA driver needs to set up MSI-X vector to interrupt the guest,
> only vfio-pci is supported currently.
> 
> Signed-off-by: Xiao Wang <xiao.w.wang at intel.com>
> Signed-off-by: Rosen Xu <rosen.xu at intel.com>
> ---
> v2:
> - Rebase on Zhihong's vDPA v3 patch set.
> ---
>   config/common_base                      |    6 +
>   config/common_linuxapp                  |    1 +
>   drivers/net/Makefile                    |    1 +
>   drivers/net/ifcvf/Makefile              |   40 +
>   drivers/net/ifcvf/base/ifcvf.c          |  329 ++++++++
>   drivers/net/ifcvf/base/ifcvf.h          |  156 ++++
>   drivers/net/ifcvf/base/ifcvf_osdep.h    |   52 ++
>   drivers/net/ifcvf/ifcvf_ethdev.c        | 1240 +++++++++++++++++++++++++++++++
>   drivers/net/ifcvf/rte_ifcvf_version.map |    4 +
>   mk/rte.app.mk                           |    1 +
>   10 files changed, 1830 insertions(+)
>   create mode 100644 drivers/net/ifcvf/Makefile
>   create mode 100644 drivers/net/ifcvf/base/ifcvf.c
>   create mode 100644 drivers/net/ifcvf/base/ifcvf.h
>   create mode 100644 drivers/net/ifcvf/base/ifcvf_osdep.h
>   create mode 100644 drivers/net/ifcvf/ifcvf_ethdev.c
>   create mode 100644 drivers/net/ifcvf/rte_ifcvf_version.map
> 

...

> +static int
> +eth_dev_ifcvf_create(struct rte_vdev_device *dev,
> +		struct rte_pci_addr *pci_addr, int devices)
> +{
> +	const char *name = rte_vdev_device_name(dev);
> +	struct rte_eth_dev *eth_dev = NULL;
> +	struct ether_addr *eth_addr = NULL;
> +	struct ifcvf_internal *internal = NULL;
> +	struct internal_list *list = NULL;
> +	struct rte_eth_dev_data *data = NULL;
> +	struct rte_pci_addr pf_addr = *pci_addr;
> +	int i;
> +
> +	list = rte_zmalloc_socket(name, sizeof(*list), 0,
> +			dev->device.numa_node);
> +	if (list == NULL)
> +		goto error;
> +
> +	/* reserve an ethdev entry */
> +	eth_dev = rte_eth_vdev_allocate(dev, sizeof(*internal));
> +	if (eth_dev == NULL)
> +		goto error;
> +
> +	eth_addr = rte_zmalloc_socket(name, sizeof(*eth_addr), 0,
> +			dev->device.numa_node);
> +	if (eth_addr == NULL)
> +		goto error;
> +
> +	*eth_addr = base_eth_addr;
> +	eth_addr->addr_bytes[5] = eth_dev->data->port_id;
> +
> +	internal = eth_dev->data->dev_private;
> +	internal->dev_name = strdup(name);
> +	if (internal->dev_name == NULL)
> +		goto error;
> +
> +	internal->eng_addr.pci_addr = *pci_addr;
> +	for (i = 0; i < devices; i++) {
> +		pf_addr.domain = pci_addr->domain;
> +		pf_addr.bus = pci_addr->bus;
> +		pf_addr.devid = pci_addr->devid + (i + 1) / 8;
> +		pf_addr.function = pci_addr->function + (i + 1) % 8;
> +		internal->vf_info[i].pdev.addr = pf_addr;
> +		rte_spinlock_init(&internal->vf_info[i].lock);
> +	}
> +	internal->max_devices = devices;
> +
> +	list->eth_dev = eth_dev;
> +	pthread_mutex_lock(&internal_list_lock);
> +	TAILQ_INSERT_TAIL(&internal_list, list, next);
> +	pthread_mutex_unlock(&internal_list_lock);
> +
> +	data = eth_dev->data;
> +	data->nb_rx_queues = IFCVF_MAX_QUEUES;
> +	data->nb_tx_queues = IFCVF_MAX_QUEUES;
> +	data->dev_link = vdpa_link;
> +	data->mac_addrs = eth_addr;

We might want one ethernet device per VF, as for example you set
dev_link.link_status to UP as soon as a VF is configured, and DOWN
as when a single VF is removed.

> +	data->dev_flags = RTE_ETH_DEV_INTR_LSC;
> +	eth_dev->dev_ops = &ops;
> +
> +	/* assign rx and tx ops, could be used as vDPA fallback */
> +	eth_dev->rx_pkt_burst = eth_ifcvf_rx;
> +	eth_dev->tx_pkt_burst = eth_ifcvf_tx;
> +
> +	if (rte_vdpa_register_engine(vdpa_ifcvf_driver.name,
> +				&internal->eng_addr) < 0)
> +		goto error;
> +
> +	return 0;
> +
> +error:
> +	rte_free(list);
> +	rte_free(eth_addr);
> +	if (internal && internal->dev_name)
> +		free(internal->dev_name);
> +	rte_free(internal);
> +	if (eth_dev)
> +		rte_eth_dev_release_port(eth_dev);
> +
> +	return -1;
> +}
> +
> +static int
> +get_pci_addr(const char *key __rte_unused, const char *value, void *extra_args)
> +{
> +	if (value == NULL || extra_args == NULL)
> +		return -1;
> +
> +	return rte_pci_addr_parse(value, extra_args);
> +}
> +
> +static inline int
> +open_int(const char *key __rte_unused, const char *value, void *extra_args)
> +{
> +	uint16_t *n = extra_args;
> +
> +	if (value == NULL || extra_args == NULL)
> +		return -EINVAL;
> +
> +	*n = (uint16_t)strtoul(value, NULL, 0);
> +	if (*n == USHRT_MAX && errno == ERANGE)
> +		return -1;
> +
> +	return 0;
> +}
> +
> +/*
> + * If this vdev is created by user, then ifcvf will be taken by
> + * this vdev.
> + */
> +static int
> +ifcvf_take_over(struct rte_pci_addr *pci_addr, int num)
> +{
> +	uint16_t port_id;
> +	int i, ret;
> +	char devname[RTE_DEV_NAME_MAX_LEN];
> +	struct rte_pci_addr vf_addr = *pci_addr;
> +
> +	for (i = 0; i < num; i++) {
> +		vf_addr.function += i % 8;
> +		vf_addr.devid += i / 8;
> +		rte_pci_device_name(&vf_addr, devname, RTE_DEV_NAME_MAX_LEN);
> +		ret = rte_eth_dev_get_port_by_name(devname, &port_id);
> +		if (ret == 0) {
> +			rte_eth_dev_close(port_id);
> +			if (rte_eth_dev_detach(port_id, devname) < 0)
> +				return -1;
> +		}
That seems a bit hard.
Shouldn't we at least check the port is not started?

> +	}
> +
> +	return 0;
> +}
> +
> +static int
> +rte_ifcvf_probe(struct rte_vdev_device *dev)
> +{
> +	struct rte_kvargs *kvlist = NULL;
> +	int ret = 0;
> +	struct rte_pci_addr pci_addr;
> +	int devices;
> +
> +	RTE_LOG(INFO, PMD, "Initializing ifcvf for %s\n",
> +			rte_vdev_device_name(dev));
> +
> +	kvlist = rte_kvargs_parse(rte_vdev_device_args(dev), valid_arguments);
> +	if (kvlist == NULL)
> +		return -1;
> +
> +	if (rte_kvargs_count(kvlist, ETH_IFCVF_BDF_ARG) == 1) {
> +		ret = rte_kvargs_process(kvlist, ETH_IFCVF_BDF_ARG,
> +				&get_pci_addr, &pci_addr);
> +		if (ret < 0)
> +			goto out_free;
> +
> +	} else {
> +		ret = -1;
> +		goto out_free;
> +	}
> +
> +	if (rte_kvargs_count(kvlist, ETH_IFCVF_DEVICES_ARG) == 1) {
> +		ret = rte_kvargs_process(kvlist, ETH_IFCVF_DEVICES_ARG,
> +				&open_int, &devices);
> +		if (ret < 0 || devices > IFCVF_MAX_DEVICES)
> +			goto out_free;
> +	} else {
> +		devices = 1;
> +	}
> +
> +	ret = ifcvf_take_over(&pci_addr, devices);
> +	if (ret < 0)
> +		goto out_free;
> +
> +	eth_dev_ifcvf_create(dev, &pci_addr, devices);
> +
> +out_free:
> +	rte_kvargs_free(kvlist);
> +	return ret;
> +}
> +
> +static int
> +rte_ifcvf_remove(struct rte_vdev_device *dev)
> +{
> +	const char *name;
> +	struct rte_eth_dev *eth_dev = NULL;
> +
> +	name = rte_vdev_device_name(dev);
> +	RTE_LOG(INFO, PMD, "Un-Initializing ifcvf for %s\n", name);
> +
> +	/* find an ethdev entry */
> +	eth_dev = rte_eth_dev_allocated(name);
> +	if (eth_dev == NULL)
> +		return -ENODEV;
> +
> +	eth_dev_close(eth_dev);
> +	rte_free(eth_dev->data);
> +	rte_eth_dev_release_port(eth_dev);
> +
> +	return 0;
> +}
> +
> +static struct rte_vdev_driver ifcvf_drv = {
> +	.probe = rte_ifcvf_probe,
> +	.remove = rte_ifcvf_remove,
> +};
> +
> +RTE_PMD_REGISTER_VDEV(net_ifcvf, ifcvf_drv);
> +RTE_PMD_REGISTER_ALIAS(net_ifcvf, eth_ifcvf);
> +RTE_PMD_REGISTER_PARAM_STRING(net_ifcvf,
> +	"bdf=<bdf> "
> +	"devices=<int>");
> diff --git a/drivers/net/ifcvf/rte_ifcvf_version.map b/drivers/net/ifcvf/rte_ifcvf_version.map
> new file mode 100644
> index 000000000..33d237913
> --- /dev/null
> +++ b/drivers/net/ifcvf/rte_ifcvf_version.map
> @@ -0,0 +1,4 @@
> +EXPERIMENTAL {
> +
> +	local: *;
> +};
> diff --git a/mk/rte.app.mk b/mk/rte.app.mk
> index 3eb41d176..be5f765e4 100644
> --- a/mk/rte.app.mk
> +++ b/mk/rte.app.mk
> @@ -171,6 +171,7 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_VDEV_NETVSC_PMD) += -lrte_pmd_vdev_netvsc
>   _LDLIBS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD)     += -lrte_pmd_virtio
>   ifeq ($(CONFIG_RTE_LIBRTE_VHOST),y)
>   _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_VHOST)      += -lrte_pmd_vhost
> +_LDLIBS-$(CONFIG_RTE_LIBRTE_IFCVF)          += -lrte_ifcvf
>   endif # $(CONFIG_RTE_LIBRTE_VHOST)
>   _LDLIBS-$(CONFIG_RTE_LIBRTE_VMXNET3_PMD)    += -lrte_pmd_vmxnet3_uio
>   
>