[dpdk-dev] [PATCH 5/5] uio: integrate MSI-X support

[Liang, Cunming]

On 5/19/2015 1:40 AM, Stephen Hemminger wrote:
> +/* enable MSI-X interrupts */
> +static int
> +uio_msix_enable(struct rte_intr_handle *intr_handle)
> +{
> +	int i, max_intr;
> +
> +	if (!intr_handle->max_intr ||
> +	    intr_handle->max_intr > RTE_MAX_RXTX_INTR_VEC_ID)
> +		max_intr = RTE_MAX_RXTX_INTR_VEC_ID + 1;
> +	else
> +		max_intr = intr_handle->max_intr;
> +
> +	/* Actual number of MSI-X interrupts might be less than requested */
> +	for (i = 0; i < max_intr; i++) {
> +		struct uio_msi_irq_set irqs = {
> +			.vec = i,
> +			.fd = intr_handle->efds[i],
> +		};
> +
> +		if (i == max_intr - 1)
> +			irqs.fd = intr_handle->fd;
> +
> +		if (ioctl(intr_handle->vfio_dev_fd, UIO_MSI_IRQ_SET, &irqs) < 0) {
It would be strange if using vfio_dev_fd in 'uio_msix_' related function.
> +			RTE_LOG(ERR, EAL,
> +				"Error enabling MSI-X event %u fd %d (%s)\n",
> +				irqs.vec, irqs.fd, strerror(errno));
> +			return -1;
> +		}
> +	}
> +
> +	return 0;
> +}
> +
> +
>   /* map the PCI resource of a PCI device in virtual memory */
>   int
>   pci_uio_map_resource(struct rte_pci_device *dev)
>   {
> -	int i, map_idx;
> +	int i, fd, map_idx;
>   	char dirname[PATH_MAX];
> -	char cfgname[PATH_MAX];
>   	char devname[PATH_MAX]; /* contains the /dev/uioX */
>   	void *mapaddr;
>   	int uio_num;
> @@ -274,11 +304,15 @@ pci_uio_map_resource(struct rte_pci_device *dev)
>   	struct mapped_pci_resource *uio_res;
>   	struct mapped_pci_res_list *uio_res_list = RTE_TAILQ_CAST(rte_uio_tailq.head, mapped_pci_res_list);
>   	struct pci_map *maps;
> +	char cfgname[PATH_MAX];
>   
>   	dev->intr_handle.fd = -1;
> -	dev->intr_handle.uio_cfg_fd = -1;
> +	dev->intr_handle.vfio_dev_fd = -1;
>   	dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN;
>   
> +	for (i = 0; i < RTE_MAX_RXTX_INTR_VEC_ID; i++)
> +		dev->intr_handle.efds[i] = -1;
> +
>   	/* secondary processes - use already recorded details */
>   	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
>   		return pci_uio_map_secondary(dev);
> @@ -293,15 +327,15 @@ pci_uio_map_resource(struct rte_pci_device *dev)
>   	snprintf(devname, sizeof(devname), "/dev/uio%u", uio_num);
>   
>   	/* save fd if in primary process */
> -	dev->intr_handle.fd = open(devname, O_RDWR);
> -	if (dev->intr_handle.fd < 0) {
> +	fd = open(devname, O_RDWR);
> +	if (fd < 0) {
>   		RTE_LOG(ERR, EAL, "Cannot open %s: %s\n",
>   			devname, strerror(errno));
>   		return -1;
>   	}
>   
>   	snprintf(cfgname, sizeof(cfgname),
> -			"/sys/class/uio/uio%u/device/config", uio_num);
> +		 "/sys/class/uio/uio%u/device/config", uio_num);
>   	dev->intr_handle.uio_cfg_fd = open(cfgname, O_RDWR);
>   	if (dev->intr_handle.uio_cfg_fd < 0) {
>   		RTE_LOG(ERR, EAL, "Cannot open %s: %s\n",
> @@ -309,9 +343,17 @@ pci_uio_map_resource(struct rte_pci_device *dev)
>   		return -1;
>   	}
>   
> -	if (dev->kdrv == RTE_KDRV_IGB_UIO)
> +	if (dev->kdrv == RTE_KDRV_UIO_MSIX) {
> +		dev->intr_handle.vfio_dev_fd = fd;
I understand now uio_cfg_fd is used for uio configure and intx, so one 
additional event fd required by uio_msi for msi/msix other cause intr.
What about store it in epfd[max_intr - 1] or define a new 'uio_msi_efd' 
so as to avoid using vfio_dev_fd.
As uio_msi defines only to support msi/msix mode, while igb_uio support 
either intx or msix(one intr vector).
It makes confusing for user to choose which to insmod.
The ideal way probably be one uio kernel module support all mode, by new 
added ioctl it can configure which intr mode {intx, msi, msix}(by new 
eal option --uio-intr or merge with --vfio-intr to an unify --intr-mode) 
user app expect to use.

> +		dev->intr_handle.type = RTE_INTR_HANDLE_UIO_MSIX;
> +		if (pci_uio_msix_init(dev) < 0)
> +			return -1;
> +	} else if (dev->kdrv == RTE_KDRV_IGB_UIO) {
> +		dev->intr_handle.fd = fd;
>   		dev->intr_handle.type = RTE_INTR_HANDLE_UIO;
> -	else {
> +	} else {
> +
> +		dev->intr_handle.fd = fd;
>   		dev->intr_handle.type = RTE_INTR_HANDLE_UIO_INTX;
>   
>   		/* set bus master that is not done by uio_pci_generic */
> @@ -460,6 +502,7 @@ pci_uio_unmap_resource(struct rte_pci_device *dev)
>   
>   	/* close fd if in primary process */
>   	close(dev->intr_handle.fd);
> +	close(dev->intr_handle.uio_cfg_fd);
>   
>   	dev->intr_handle.fd = -1;
>   	dev->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN;
>