[dpdk-dev] [PATCH v3 3/3] bus/fslmc: support bus close API

[Kinsella, Ray]

On 26/08/2020 06:52, rohit.raj at nxp.com wrote:
> From: Rohit Raj <rohit.raj at nxp.com>
> 
> This patch add support for closing the bus objects which
> were acquired In the bus probe.
> 
> Some devices need to be cleaned while in both primary and
> secondary process and while some need to be cleaned only in
> case of primary process.
> 
> The devices are closed as per the white list used while
> creating the objects in a particular process.
> 
> Signed-off-by: Rohit Raj <rohit.raj at nxp.com>
> Acked-by: Sachin Saxena <sachin.saxena at oss.nxp.com>
> ---
>  drivers/bus/fslmc/fslmc_bus.c               | 15 +++-
>  drivers/bus/fslmc/fslmc_vfio.c              | 89 ++++++++++++++++++++-
>  drivers/bus/fslmc/fslmc_vfio.h              |  3 +-
>  drivers/bus/fslmc/mc/fsl_dpcon.h            |  1 +
>  drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c    | 31 ++++++-
>  drivers/bus/fslmc/portal/dpaa2_hw_dpci.c    | 32 +++++++-
>  drivers/bus/fslmc/portal/dpaa2_hw_dpio.c    | 34 ++++++++
>  drivers/bus/fslmc/rte_bus_fslmc_version.map |  1 +
>  drivers/bus/fslmc/rte_fslmc.h               |  5 +-
>  drivers/event/dpaa2/dpaa2_hw_dpcon.c        | 32 +++++++-
>  drivers/net/dpaa2/dpaa2_mux.c               | 18 ++++-
>  11 files changed, 253 insertions(+), 8 deletions(-)
> 
> diff --git a/drivers/bus/fslmc/fslmc_bus.c b/drivers/bus/fslmc/fslmc_bus.c
> index beb3dd008..9d3da6be3 100644
> --- a/drivers/bus/fslmc/fslmc_bus.c
> +++ b/drivers/bus/fslmc/fslmc_bus.c
> @@ -1,6 +1,6 @@
>  /* SPDX-License-Identifier: BSD-3-Clause
>   *
> - *   Copyright 2016,2018-2019 NXP
> + *   Copyright 2016,2018-2020 NXP
>   *
>   */
>  
> @@ -365,6 +365,18 @@ rte_fslmc_match(struct rte_dpaa2_driver *dpaa2_drv,
>  	return 1;
>  }
>  
> +static int
> +rte_fslmc_close(void)
> +{
> +	int ret = 0;
> +
> +	ret = fslmc_vfio_close_group();
> +	if (ret)
> +		DPAA2_BUS_ERR("Unable to close devices %d", ret);
> +
> +	return 0;
> +}
> +
>  static int
>  rte_fslmc_probe(void)
>  {
> @@ -637,6 +649,7 @@ struct rte_fslmc_bus rte_fslmc_bus = {
>  	.bus = {
>  		.scan = rte_fslmc_scan,
>  		.probe = rte_fslmc_probe,
> +		.close = rte_fslmc_close,
>  		.parse = rte_fslmc_parse,
>  		.find_device = rte_fslmc_find_device,
>  		.get_iommu_class = rte_dpaa2_get_iommu_class,
> diff --git a/drivers/bus/fslmc/fslmc_vfio.c b/drivers/bus/fslmc/fslmc_vfio.c
> index 9134ec552..34ccc9db3 100644
> --- a/drivers/bus/fslmc/fslmc_vfio.c
> +++ b/drivers/bus/fslmc/fslmc_vfio.c
> @@ -1,7 +1,7 @@
>  /* SPDX-License-Identifier: BSD-3-Clause
>   *
>   *   Copyright (c) 2015-2016 Freescale Semiconductor, Inc. All rights reserved.
> - *   Copyright 2016-2019 NXP
> + *   Copyright 2016-2020 NXP
>   *
>   */
>  
> @@ -692,6 +692,54 @@ rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle,
>  	return -1;
>  }
>  
> +static void
> +fslmc_close_iodevices(struct rte_dpaa2_device *dev)
> +{
> +	struct rte_dpaa2_object *object = NULL;
> +	struct rte_dpaa2_driver *drv;
> +	int ret, probe_all;
> +
> +	switch (dev->dev_type) {
> +	case DPAA2_IO:
> +	case DPAA2_CON:
> +	case DPAA2_CI:
> +	case DPAA2_BPOOL:
> +	case DPAA2_MUX:
> +		TAILQ_FOREACH(object, &dpaa2_obj_list, next) {
> +			if (dev->dev_type == object->dev_type)
> +				object->close(dev->object_id);
> +			else
> +				continue;
> +		}
> +		break;
> +	case DPAA2_ETH:
> +	case DPAA2_CRYPTO:
> +	case DPAA2_QDMA:
> +		probe_all = rte_fslmc_bus.bus.conf.scan_mode !=
> +			    RTE_BUS_SCAN_WHITELIST;
> +		TAILQ_FOREACH(drv, &rte_fslmc_bus.driver_list, next) {
> +			if (drv->drv_type != dev->dev_type)
> +				continue;
> +			if (rte_dev_is_probed(&dev->device))
> +				continue;
> +			if (probe_all ||
> +			    (dev->device.devargs &&
> +			     dev->device.devargs->policy ==
> +			     RTE_DEV_WHITELISTED)) {
> +				ret = drv->remove(dev);
> +				if (ret)
> +					DPAA2_BUS_ERR("Unable to remove");
> +			}
> +		}
> +		break;
> +	default:
> +		break;
> +	}
> +
> +	DPAA2_BUS_LOG(DEBUG, "Device (%s) Closed",
> +		      dev->device.name);
> +}
> +
>  /*
>   * fslmc_process_iodevices for processing only IO (ETH, CRYPTO, and possibly
>   * EVENT) devices.
> @@ -796,6 +844,45 @@ fslmc_process_mcp(struct rte_dpaa2_device *dev)
>  	return ret;
>  }
>  
> +int
> +fslmc_vfio_close_group(void)
> +{
> +	struct rte_dpaa2_device *dev, *dev_temp;
> +
> +	TAILQ_FOREACH_SAFE(dev, &rte_fslmc_bus.device_list, next, dev_temp) {
> +		if (dev->device.devargs &&
> +		    dev->device.devargs->policy == RTE_DEV_BLACKLISTED) {
> +			DPAA2_BUS_LOG(DEBUG, "%s Blacklisted, skipping",
> +				      dev->device.name);
> +			TAILQ_REMOVE(&rte_fslmc_bus.device_list, dev, next);
> +				continue;
> +		}
> +		switch (dev->dev_type) {
> +		case DPAA2_ETH:
> +		case DPAA2_CRYPTO:
> +		case DPAA2_QDMA:
> +		case DPAA2_IO:
> +			fslmc_close_iodevices(dev);
> +			break;
> +		case DPAA2_CON:
> +		case DPAA2_CI:
> +		case DPAA2_BPOOL:
> +		case DPAA2_MUX:
> +			if (rte_eal_process_type() == RTE_PROC_SECONDARY)
> +				continue;
> +
> +			fslmc_close_iodevices(dev);
> +			break;
> +		case DPAA2_DPRTC:
> +		default:
> +			DPAA2_BUS_DEBUG("Device cannot be closed: Not supported (%s)",
> +					dev->device.name);
> +		}
> +	}
> +
> +	return 0;
> +}
> +
>  int
>  fslmc_vfio_process_group(void)
>  {
> diff --git a/drivers/bus/fslmc/fslmc_vfio.h b/drivers/bus/fslmc/fslmc_vfio.h
> index bc7c6f62d..0a29fe739 100644
> --- a/drivers/bus/fslmc/fslmc_vfio.h
> +++ b/drivers/bus/fslmc/fslmc_vfio.h
> @@ -1,7 +1,7 @@
>  /* SPDX-License-Identifier: BSD-3-Clause
>   *
>   *   Copyright (c) 2015-2016 Freescale Semiconductor, Inc. All rights reserved.
> - *   Copyright 2016,2019 NXP
> + *   Copyright 2016,2019-2020 NXP
>   *
>   */
>  
> @@ -55,6 +55,7 @@ int rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle,
>  
>  int fslmc_vfio_setup_group(void);
>  int fslmc_vfio_process_group(void);
> +int fslmc_vfio_close_group(void);
>  char *fslmc_get_container(void);
>  int fslmc_get_container_group(int *gropuid);
>  int rte_fslmc_vfio_dmamap(void);
> diff --git a/drivers/bus/fslmc/mc/fsl_dpcon.h b/drivers/bus/fslmc/mc/fsl_dpcon.h
> index 7caa6c68a..8053a6db1 100644
> --- a/drivers/bus/fslmc/mc/fsl_dpcon.h
> +++ b/drivers/bus/fslmc/mc/fsl_dpcon.h
> @@ -26,6 +26,7 @@ int dpcon_open(struct fsl_mc_io *mc_io,
>  	       int dpcon_id,
>  	       uint16_t *token);
>  
> +__rte_internal
>  int dpcon_close(struct fsl_mc_io *mc_io,
>  		uint32_t cmd_flags,
>  		uint16_t token);
> diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
> index d9619848d..9a4cdfd7a 100644
> --- a/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
> +++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
> @@ -1,7 +1,7 @@
>  /* SPDX-License-Identifier: BSD-3-Clause
>   *
>   *   Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
> - *   Copyright 2016 NXP
> + *   Copyright 2016,2020 NXP
>   *
>   */
>  
> @@ -33,6 +33,19 @@ TAILQ_HEAD(dpbp_dev_list, dpaa2_dpbp_dev);
>  static struct dpbp_dev_list dpbp_dev_list
>  	= TAILQ_HEAD_INITIALIZER(dpbp_dev_list); /*!< DPBP device list */
>  
> +static struct dpaa2_dpbp_dev *get_dpbp_from_id(uint32_t dpbp_id)
> +{
> +	struct dpaa2_dpbp_dev *dpbp_dev = NULL;
> +
> +	/* Get DPBP dev handle from list using index */
> +	TAILQ_FOREACH(dpbp_dev, &dpbp_dev_list, next) {
> +		if (dpbp_dev->dpbp_id == dpbp_id)
> +			break;
> +	}
> +
> +	return dpbp_dev;
> +}
> +
>  static int
>  dpaa2_create_dpbp_device(int vdev_fd __rte_unused,
>  			 struct vfio_device_info *obj_info __rte_unused,
> @@ -116,9 +129,25 @@ int dpaa2_dpbp_supported(void)
>  	return 0;
>  }
>  
> +static void
> +dpaa2_close_dpbp_device(int object_id)
> +{
> +	struct dpaa2_dpbp_dev *dpbp_dev = NULL;
> +
> +	dpbp_dev = get_dpbp_from_id((uint32_t)object_id);
> +
> +	if (dpbp_dev) {
> +		dpaa2_free_dpbp_dev(dpbp_dev);
> +		dpbp_close(&dpbp_dev->dpbp, CMD_PRI_LOW, dpbp_dev->token);
> +		TAILQ_REMOVE(&dpbp_dev_list, dpbp_dev, next);
> +		rte_free(dpbp_dev);
> +	}
> +}
> +
>  static struct rte_dpaa2_object rte_dpaa2_dpbp_obj = {
>  	.dev_type = DPAA2_BPOOL,
>  	.create = dpaa2_create_dpbp_device,
> +	.close = dpaa2_close_dpbp_device,
>  };
>  
>  RTE_PMD_REGISTER_DPAA2_OBJECT(dpbp, rte_dpaa2_dpbp_obj);
> diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c
> index d393ce618..9902f38ce 100644
> --- a/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c
> +++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c
> @@ -1,6 +1,6 @@
>  /* SPDX-License-Identifier: BSD-3-Clause
>   *
> - *   Copyright 2017 NXP
> + *   Copyright 2017,2020 NXP
>   *
>   */
>  
> @@ -30,6 +30,19 @@ TAILQ_HEAD(dpci_dev_list, dpaa2_dpci_dev);
>  static struct dpci_dev_list dpci_dev_list
>  	= TAILQ_HEAD_INITIALIZER(dpci_dev_list); /*!< DPCI device list */
>  
> +static struct dpaa2_dpci_dev *get_dpci_from_id(uint32_t dpci_id)
> +{
> +	struct dpaa2_dpci_dev *dpci_dev = NULL;
> +
> +	/* Get DPCI dev handle from list using index */
> +	TAILQ_FOREACH(dpci_dev, &dpci_dev_list, next) {
> +		if (dpci_dev->dpci_id == dpci_id)
> +			break;
> +	}
> +
> +	return dpci_dev;
> +}
> +
>  static int
>  rte_dpaa2_create_dpci_device(int vdev_fd __rte_unused,
>  			     struct vfio_device_info *obj_info __rte_unused,
> @@ -179,9 +192,26 @@ void rte_dpaa2_free_dpci_dev(struct dpaa2_dpci_dev *dpci)
>  	}
>  }
>  
> +
> +static void
> +rte_dpaa2_close_dpci_device(int object_id)
> +{
> +	struct dpaa2_dpci_dev *dpci_dev = NULL;
> +
> +	dpci_dev = get_dpci_from_id((uint32_t)object_id);
> +
> +	if (dpci_dev) {
> +		rte_dpaa2_free_dpci_dev(dpci_dev);
> +		dpci_close(&dpci_dev->dpci, CMD_PRI_LOW, dpci_dev->token);
> +		TAILQ_REMOVE(&dpci_dev_list, dpci_dev, next);
> +		rte_free(dpci_dev);
> +	}
> +}
> +
>  static struct rte_dpaa2_object rte_dpaa2_dpci_obj = {
>  	.dev_type = DPAA2_CI,
>  	.create = rte_dpaa2_create_dpci_device,
> +	.close = rte_dpaa2_close_dpci_device,
>  };
>  
>  RTE_PMD_REGISTER_DPAA2_OBJECT(dpci, rte_dpaa2_dpci_obj);
> diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
> index 97be76116..9563cd620 100644
> --- a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
> +++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
> @@ -86,6 +86,19 @@ static int dpaa2_cluster_sz = 2;
>   * Cluster 4 (ID = x07) : CPU14, CPU15;
>   */
>  
> +static struct dpaa2_dpio_dev *get_dpio_dev_from_id(int32_t dpio_id)
> +{
> +	struct dpaa2_dpio_dev *dpio_dev = NULL;
> +
> +	/* Get DPIO dev handle from list using index */
> +	TAILQ_FOREACH(dpio_dev, &dpio_dev_list, next) {
> +		if (dpio_dev->hw_id == dpio_id)
> +			break;
> +	}
> +
> +	return dpio_dev;
> +}
> +
>  static int
>  dpaa2_get_core_id(void)
>  {
> @@ -360,6 +373,26 @@ static void dpaa2_portal_finish(void *arg)
>  	pthread_setspecific(dpaa2_portal_key, NULL);
>  }
>  
> +static void
> +dpaa2_close_dpio_device(int object_id)
> +{
> +	struct dpaa2_dpio_dev *dpio_dev = NULL;
> +
> +	dpio_dev = get_dpio_dev_from_id((int32_t)object_id);
> +
> +	if (dpio_dev) {
> +		if (dpio_dev->dpio) {
> +			dpio_disable(dpio_dev->dpio, CMD_PRI_LOW,
> +				     dpio_dev->token);
> +			dpio_close(dpio_dev->dpio, CMD_PRI_LOW,
> +				   dpio_dev->token);
> +			rte_free(dpio_dev->dpio);
> +		}
> +		TAILQ_REMOVE(&dpio_dev_list, dpio_dev, next);
> +		rte_free(dpio_dev);
> +	}
> +}
> +
>  static int
>  dpaa2_create_dpio_device(int vdev_fd,
>  			 struct vfio_device_info *obj_info,
> @@ -623,6 +656,7 @@ dpaa2_free_eq_descriptors(void)
>  static struct rte_dpaa2_object rte_dpaa2_dpio_obj = {
>  	.dev_type = DPAA2_IO,
>  	.create = dpaa2_create_dpio_device,
> +	.close = dpaa2_close_dpio_device,
>  };
>  
>  RTE_PMD_REGISTER_DPAA2_OBJECT(dpio, rte_dpaa2_dpio_obj);
> diff --git a/drivers/bus/fslmc/rte_bus_fslmc_version.map b/drivers/bus/fslmc/rte_bus_fslmc_version.map
> index b169f5228..ac4ee2bf8 100644
> --- a/drivers/bus/fslmc/rte_bus_fslmc_version.map
> +++ b/drivers/bus/fslmc/rte_bus_fslmc_version.map
> @@ -30,6 +30,7 @@ INTERNAL {
>  	dpci_get_opr;
>  	dpci_set_opr;
>  	dpci_set_rx_queue;
> +	dpcon_close;
>  	dpcon_get_attributes;
>  	dpcon_open;
>  	dpdmai_close;
> diff --git a/drivers/bus/fslmc/rte_fslmc.h b/drivers/bus/fslmc/rte_fslmc.h
> index 80873fffc..a938ab8f9 100644
> --- a/drivers/bus/fslmc/rte_fslmc.h
> +++ b/drivers/bus/fslmc/rte_fslmc.h
> @@ -1,6 +1,6 @@
>  /* SPDX-License-Identifier: BSD-3-Clause
>   *
> - *   Copyright 2016,2019 NXP
> + *   Copyright 2016,2019-2020 NXP
>   *
>   */
>  
> @@ -79,6 +79,8 @@ typedef int (*rte_dpaa2_obj_create_t)(int vdev_fd,
>  				      struct vfio_device_info *obj_info,
>  				      int object_id);
>  
> +typedef void (*rte_dpaa2_obj_close_t)(int object_id);
> +
>  /**
>   * A structure describing a DPAA2 object.
>   */
> @@ -87,6 +89,7 @@ struct rte_dpaa2_object {
>  	const char *name;                   /**< Name of Object. */
>  	enum rte_dpaa2_dev_type dev_type;   /**< Type of device */
>  	rte_dpaa2_obj_create_t create;
> +	rte_dpaa2_obj_close_t close;
>  };
>  
>  /**
> diff --git a/drivers/event/dpaa2/dpaa2_hw_dpcon.c b/drivers/event/dpaa2/dpaa2_hw_dpcon.c
> index 200b71640..86844f49f 100644
> --- a/drivers/event/dpaa2/dpaa2_hw_dpcon.c
> +++ b/drivers/event/dpaa2/dpaa2_hw_dpcon.c
> @@ -1,6 +1,6 @@
>  /* SPDX-License-Identifier: BSD-3-Clause
>   *
> - *   Copyright 2017 NXP
> + *   Copyright 2017,2020 NXP
>   *
>   */
>  
> @@ -30,6 +30,19 @@ TAILQ_HEAD(dpcon_dev_list, dpaa2_dpcon_dev);
>  static struct dpcon_dev_list dpcon_dev_list
>  	= TAILQ_HEAD_INITIALIZER(dpcon_dev_list); /*!< DPCON device list */
>  
> +static struct dpaa2_dpcon_dev *get_dpcon_from_id(uint32_t dpcon_id)
> +{
> +	struct dpaa2_dpcon_dev *dpcon_dev = NULL;
> +
> +	/* Get DPCONC dev handle from list using index */
> +	TAILQ_FOREACH(dpcon_dev, &dpcon_dev_list, next) {
> +		if (dpcon_dev->dpcon_id == dpcon_id)
> +			break;
> +	}
> +
> +	return dpcon_dev;
> +}
> +
>  static int
>  rte_dpaa2_create_dpcon_device(int dev_fd __rte_unused,
>  			      struct vfio_device_info *obj_info __rte_unused,
> @@ -105,9 +118,26 @@ void rte_dpaa2_free_dpcon_dev(struct dpaa2_dpcon_dev *dpcon)
>  	}
>  }
>  
> +
> +static void
> +rte_dpaa2_close_dpcon_device(int object_id)
> +{
> +	struct dpaa2_dpcon_dev *dpcon_dev = NULL;
> +
> +	dpcon_dev = get_dpcon_from_id((uint32_t)object_id);
> +
> +	if (dpcon_dev) {
> +		rte_dpaa2_free_dpcon_dev(dpcon_dev);
> +		dpcon_close(&dpcon_dev->dpcon, CMD_PRI_LOW, dpcon_dev->token);
> +		TAILQ_REMOVE(&dpcon_dev_list, dpcon_dev, next);
> +		rte_free(dpcon_dev);
> +	}
> +}
> +
>  static struct rte_dpaa2_object rte_dpaa2_dpcon_obj = {
>  	.dev_type = DPAA2_CON,
>  	.create = rte_dpaa2_create_dpcon_device,
> +	.close = rte_dpaa2_close_dpcon_device,
>  };
>  
>  RTE_PMD_REGISTER_DPAA2_OBJECT(dpcon, rte_dpaa2_dpcon_obj);
> diff --git a/drivers/net/dpaa2/dpaa2_mux.c b/drivers/net/dpaa2/dpaa2_mux.c
> index f8366e839..15114dac9 100644
> --- a/drivers/net/dpaa2/dpaa2_mux.c
> +++ b/drivers/net/dpaa2/dpaa2_mux.c
> @@ -44,7 +44,7 @@ static struct dpaa2_dpdmux_dev *get_dpdmux_from_id(uint32_t dpdmux_id)
>  {
>  	struct dpaa2_dpdmux_dev *dpdmux_dev = NULL;
>  
> -	/* Get DPBP dev handle from list using index */
> +	/* Get DPDMUX dev handle from list using index */
>  	TAILQ_FOREACH(dpdmux_dev, &dpdmux_dev_list, next) {
>  		if (dpdmux_dev->dpdmux_id == dpdmux_id)
>  			break;
> @@ -261,9 +261,25 @@ dpaa2_create_dpdmux_device(int vdev_fd __rte_unused,
>  	return -1;
>  }
>  
> +static void
> +dpaa2_close_dpdmux_device(int object_id)
> +{
> +	struct dpaa2_dpdmux_dev *dpdmux_dev;
> +
> +	dpdmux_dev = get_dpdmux_from_id((uint32_t)object_id);
> +
> +	if (dpdmux_dev) {
> +		dpdmux_close(&dpdmux_dev->dpdmux, CMD_PRI_LOW,
> +			     dpdmux_dev->token);
> +		TAILQ_REMOVE(&dpdmux_dev_list, dpdmux_dev, next);
> +		rte_free(dpdmux_dev);
> +	}
> +}
> +
>  static struct rte_dpaa2_object rte_dpaa2_dpdmux_obj = {
>  	.dev_type = DPAA2_MUX,
>  	.create = dpaa2_create_dpdmux_device,
> +	.close = dpaa2_close_dpdmux_device,
>  };
>  
>  RTE_PMD_REGISTER_DPAA2_OBJECT(dpdmux, rte_dpaa2_dpdmux_obj);
> 

Acked-by: Ray Kinsella <mdr at ashroe.eu>