[PATCH v5 4/5] bus/cdx: add support for MSI

[David Marchand]

On Thu, May 25, 2023 at 12:09 PM Nipun Gupta <nipun.gupta at amd.com> wrote:
>
> MSI's are exposed to the devices using VFIO (vfio-cdx). This
> patch uses the same to add support for MSI for the devices on
> the cdx bus.
>
> A couple of API's have been introduced in the EAL interrupt
> framework:
> - rte_intr_irq_count_set: This API is used to set the total
>     interrupts on the interrupt handle. This would be provided
>     by VFIO (irq.count) for VFIO enabled devices.
> - rte_intr_irq_count_get: This API returns the total number
>     interrupts which were set.
>
> Signed-off-by: Nipun Gupta <nipun.gupta at amd.com>
> Acked-by: Ferruh Yigit <ferruh.yigit at amd.com>
> ---
>  drivers/bus/cdx/bus_cdx_driver.h |  25 +++++
>  drivers/bus/cdx/cdx.c            |  11 ++
>  drivers/bus/cdx/cdx_vfio.c       | 182 ++++++++++++++++++++++++++++++-
>  drivers/bus/cdx/version.map      |   2 +
>  4 files changed, 218 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/bus/cdx/bus_cdx_driver.h b/drivers/bus/cdx/bus_cdx_driver.h
> index f1dce06a16..a8c54d728e 100644
> --- a/drivers/bus/cdx/bus_cdx_driver.h
> +++ b/drivers/bus/cdx/bus_cdx_driver.h
> @@ -67,6 +67,7 @@ struct rte_cdx_device {
>         struct rte_cdx_id id;                   /**< CDX ID. */
>         struct rte_mem_resource mem_resource[CDX_MAX_RESOURCE];
>                                                 /**< CDX Memory Resource */
> +       struct rte_intr_handle *intr_handle;    /**< Interrupt handle */
>  };
>
>  /**
> @@ -168,6 +169,30 @@ void rte_cdx_unmap_device(struct rte_cdx_device *dev);
>  __rte_internal
>  void rte_cdx_register(struct rte_cdx_driver *driver);
>
> +/**
> + * Enables VFIO Interrupts for CDX bus devices.
> + *
> + * @param intr_handle
> + *   Pointer to the interrupt handle.
> + *
> + *  @return
> + *  0 on success, -1 on error.
> + */
> +__rte_internal
> +int rte_cdx_vfio_intr_enable(const struct rte_intr_handle *intr_handle);
> +
> +/**
> + * Disable VFIO Interrupts for CDX bus devices.
> + *
> + * @param intr_handle
> + *   Pointer to the interrupt handle.
> + *
> + *  @return
> + *  0 on success, -1 on error.
> + */
> +__rte_internal
> +int rte_cdx_vfio_intr_disable(const struct rte_intr_handle *intr_handle);
> +
>  /**
>   * Helper for CDX device registration from driver (eth, crypto, raw) instance
>   */
> diff --git a/drivers/bus/cdx/cdx.c b/drivers/bus/cdx/cdx.c
> index 64ea879f3b..c691c38e04 100644
> --- a/drivers/bus/cdx/cdx.c
> +++ b/drivers/bus/cdx/cdx.c
> @@ -201,6 +201,15 @@ cdx_scan_one(const char *dirname, const char *dev_name)
>                 goto err;
>         }
>
> +       /* Allocate interrupt instance for cdx device */
> +       dev->intr_handle =
> +               rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_PRIVATE);
> +       if (dev->intr_handle == NULL) {
> +               CDX_BUS_ERR("Failed to create interrupt instance for %s\n",

CDX_BUS_ERR already appends a \n.

> +                       dev->device.name);
> +               return -ENOMEM;
> +       }
> +
>         /*
>          * Check if device is bound to 'vfio-cdx' driver, so that user-space
>          * can gracefully access the device.
> @@ -391,6 +400,8 @@ cdx_probe_one_driver(struct rte_cdx_driver *dr,
>         return ret;
>
>  error_probe:
> +       rte_intr_instance_free(dev->intr_handle);
> +       dev->intr_handle = NULL;
>         cdx_vfio_unmap_resource(dev);
>  error_map_device:
>         return ret;
> diff --git a/drivers/bus/cdx/cdx_vfio.c b/drivers/bus/cdx/cdx_vfio.c
> index e54432de5b..ec6512e158 100644
> --- a/drivers/bus/cdx/cdx_vfio.c
> +++ b/drivers/bus/cdx/cdx_vfio.c
> @@ -50,6 +50,10 @@ struct mapped_cdx_resource {
>  /** mapped cdx device list */
>  TAILQ_HEAD(mapped_cdx_res_list, mapped_cdx_resource);
>
> +/* IRQ set buffer length for MSI interrupts */
> +#define MSI_IRQ_SET_BUF_LEN (sizeof(struct vfio_irq_set) + \
> +                             sizeof(int) * (RTE_MAX_RXTX_INTR_VEC_ID + 1))
> +
>  static struct rte_tailq_elem cdx_vfio_tailq = {
>         .name = "VFIO_CDX_RESOURCE_LIST",
>  };
> @@ -94,6 +98,27 @@ cdx_vfio_unmap_resource_primary(struct rte_cdx_device *dev)
>         char cdx_addr[PATH_MAX] = {0};
>         struct mapped_cdx_resource *vfio_res = NULL;
>         struct mapped_cdx_res_list *vfio_res_list;
> +       int ret, vfio_dev_fd;
> +
> +       if (rte_intr_fd_get(dev->intr_handle) < 0)
> +               return -1;
> +
> +       if (close(rte_intr_fd_get(dev->intr_handle)) < 0) {
> +               CDX_BUS_ERR("Error when closing eventfd file descriptor for %s",
> +                       dev->device.name);
> +               return -1;
> +       }
> +
> +       vfio_dev_fd = rte_intr_dev_fd_get(dev->intr_handle);
> +       if (vfio_dev_fd < 0)
> +               return -1;
> +
> +       ret = rte_vfio_release_device(rte_cdx_get_sysfs_path(), dev->device.name,
> +                                     vfio_dev_fd);
> +       if (ret < 0) {
> +               CDX_BUS_ERR("Cannot release VFIO device");
> +               return ret;
> +       }
>
>         vfio_res_list =
>                 RTE_TAILQ_CAST(cdx_vfio_tailq.head, mapped_cdx_res_list);
> @@ -116,6 +141,18 @@ cdx_vfio_unmap_resource_secondary(struct rte_cdx_device *dev)
>  {
>         struct mapped_cdx_resource *vfio_res = NULL;
>         struct mapped_cdx_res_list *vfio_res_list;
> +       int ret, vfio_dev_fd;
> +
> +       vfio_dev_fd = rte_intr_dev_fd_get(dev->intr_handle);
> +       if (vfio_dev_fd < 0)
> +               return -1;
> +
> +       ret = rte_vfio_release_device(rte_cdx_get_sysfs_path(), dev->device.name,
> +                                     vfio_dev_fd);
> +       if (ret < 0) {
> +               CDX_BUS_ERR("Cannot release VFIO device");
> +               return ret;
> +       }
>
>         vfio_res_list =
>                 RTE_TAILQ_CAST(cdx_vfio_tailq.head, mapped_cdx_res_list);
> @@ -140,9 +177,80 @@ cdx_vfio_unmap_resource(struct rte_cdx_device *dev)
>                 return cdx_vfio_unmap_resource_secondary(dev);
>  }
>
> +/* set up interrupt support (but not enable interrupts) */
>  static int
> -cdx_rte_vfio_setup_device(int vfio_dev_fd)
> +cdx_vfio_setup_interrupts(struct rte_cdx_device *dev, int vfio_dev_fd,

Why rename this function?


> +               int num_irqs)
>  {
> +       int i, ret;
> +
> +       if (num_irqs == 0)
> +               return 0;
> +
> +       /* start from MSI interrupt type */
> +       for (i = 0; i < num_irqs; i++) {


[snip]


-- 
David Marchand