[PATCH 11/13] net/mlx5: add sibling device iteration macro
Dariusz Sosnowski
dsosnowski at nvidia.com
Mon Oct 26 17:27:07 CET 2026
In some embedded switch topologies, not all DPDK representor ports
will be based on the same rte_device.
One such example is ConnectX socket direct configurations
or nested switches based on VFs/SFs.
mlx5 networking driver in many cases requires to iterate
over all DPDK ethdev ports belonging to the same switch.
In the past it relied on "one rte_device" assumption.
This patch introduces a new macro - MLX5_ETH_FOREACH_SIBLING -
which iterates over all ethdev ports managed by mlx5 driver,
which share the same mlx5 common device.
All relevant usages of previous iteration macros are replaced with
MLX5_ETH_FOREACH_SIBLING, in preparation for allowing usage of multiple
rte_device's in mlx5 networking driver.
Signed-off-by: Dariusz Sosnowski <dsosnowski at nvidia.com>
Acked-by: Viacheslav Ovsiienko <viacheslavo at nvidia.com>
---
drivers/net/mlx5/linux/mlx5_ethdev_os.c | 13 ++---
drivers/net/mlx5/linux/mlx5_os.c | 22 +++-----
drivers/net/mlx5/mlx5.c | 70 ++++++++++++++++++++-----
drivers/net/mlx5/mlx5.h | 6 +++
drivers/net/mlx5/mlx5_ethdev.c | 27 +++-------
drivers/net/mlx5/mlx5_flow.c | 24 ++++-----
drivers/net/mlx5/mlx5_mac.c | 2 +-
drivers/net/mlx5/mlx5_trigger.c | 10 ++--
drivers/net/mlx5/windows/mlx5_os.c | 24 ++++-----
9 files changed, 105 insertions(+), 93 deletions(-)
diff --git a/drivers/net/mlx5/linux/mlx5_ethdev_os.c b/drivers/net/mlx5/linux/mlx5_ethdev_os.c
index f6394ebd19..55cae148f0 100644
--- a/drivers/net/mlx5/linux/mlx5_ethdev_os.c
+++ b/drivers/net/mlx5/linux/mlx5_ethdev_os.c
@@ -300,18 +300,13 @@ mlx5_find_master_dev(struct rte_eth_dev *dev)
{
struct mlx5_priv *priv;
uint16_t port_id;
- uint16_t domain_id;
priv = dev->data->dev_private;
- domain_id = priv->domain_id;
MLX5_ASSERT(priv->representor);
- MLX5_ETH_FOREACH_DEV(port_id, dev->device) {
- struct mlx5_priv *opriv =
- rte_eth_devices[port_id].data->dev_private;
- if (opriv &&
- opriv->master &&
- opriv->domain_id == domain_id &&
- opriv->sh == priv->sh)
+ MLX5_ETH_FOREACH_SIBLING(port_id, priv->sh->cdev) {
+ struct mlx5_priv *opriv = rte_eth_devices[port_id].data->dev_private;
+
+ if (opriv->master)
return &rte_eth_devices[port_id];
}
return NULL;
diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 42b23e7f58..1b1b1bb013 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -1585,14 +1585,10 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
* Look for sibling devices in order to reuse their switch domain
* if any, otherwise allocate one.
*/
- MLX5_ETH_FOREACH_DEV(port_id, dpdk_dev) {
- const struct mlx5_priv *opriv =
- rte_eth_devices[port_id].data->dev_private;
-
- if (!opriv ||
- opriv->sh != priv->sh ||
- opriv->domain_id ==
- RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID)
+ MLX5_ETH_FOREACH_SIBLING(port_id, priv->sh->cdev) {
+ const struct mlx5_priv *opriv = rte_eth_devices[port_id].data->dev_private;
+
+ if (opriv->domain_id == RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID)
continue;
priv->domain_id = opriv->domain_id;
DRV_LOG(DEBUG, "dev_port-%u inherit domain_id=%u\n",
@@ -1711,13 +1707,9 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
if (priv->representor) {
eth_dev->data->dev_flags |= RTE_ETH_DEV_REPRESENTOR;
eth_dev->data->representor_id = eth_dev->data->port_id;
- MLX5_ETH_FOREACH_DEV(port_id, dpdk_dev) {
- struct mlx5_priv *opriv =
- rte_eth_devices[port_id].data->dev_private;
- if (opriv &&
- opriv->master &&
- opriv->domain_id == priv->domain_id &&
- opriv->sh == priv->sh) {
+ MLX5_ETH_FOREACH_SIBLING(port_id, priv->sh->cdev) {
+ struct mlx5_priv *opriv = rte_eth_devices[port_id].data->dev_private;
+ if (opriv->master) {
eth_dev->data->backer_port_id = port_id;
break;
}
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 7093eefa15..615b2ff943 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -2565,23 +2565,14 @@ mlx5_dev_close(struct rte_eth_dev *dev)
* since 'mlx5_dev_interrupt_nl_cb' uses priv.
*/
rte_io_wmb();
- /*
- * Free the shared context in last turn, because the cleanup
- * routines above may use some shared fields, like
- * mlx5_os_mac_addr_flush() uses ibdev_path for retrieving
- * ifindex if Netlink fails.
- */
- mlx5_free_shared_dev_ctx(sh);
if (priv->domain_id != RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID) {
unsigned int c = 0;
uint16_t port_id;
- MLX5_ETH_FOREACH_DEV(port_id, dev->device) {
- struct mlx5_priv *opriv =
- rte_eth_devices[port_id].data->dev_private;
+ MLX5_ETH_FOREACH_SIBLING(port_id, priv->sh->cdev) {
+ struct mlx5_priv *opriv = rte_eth_devices[port_id].data->dev_private;
- if (!opriv ||
- opriv->domain_id != priv->domain_id ||
+ if (opriv->domain_id != priv->domain_id ||
&rte_eth_devices[port_id] == dev)
continue;
++c;
@@ -2590,6 +2581,13 @@ mlx5_dev_close(struct rte_eth_dev *dev)
if (!c)
claim_zero(rte_eth_switch_domain_free(priv->domain_id));
}
+ /*
+ * Free the shared context in last turn, because the cleanup
+ * routines above may use some shared fields, like
+ * mlx5_os_mac_addr_flush() uses ibdev_path for retrieving
+ * ifindex if Netlink fails.
+ */
+ mlx5_free_shared_dev_ctx(sh);
mlx5_flow_pools_destroy(priv);
memset(priv, 0, sizeof(*priv));
priv->domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID;
@@ -3508,6 +3506,54 @@ mlx5_eth_find_next(uint16_t port_id, struct rte_device *odev)
return port_id;
}
+/**
+ * Look for the ethernet device belonging to mlx5 driver, which share the device context.
+ *
+ * If port index is found, then caller can assume that:
+ *
+ * - Port is a valid ethdev port, managed by mlx5 driver.
+ * - Port has private device data set (mlx5_priv).
+ * - Port has shared mlx5 device context set (priv->sh).
+ * - Port is based on the @p cdev common device.
+ *
+ * @param[in] start_port_id
+ * Port index to start looking for device.
+ * @param[in] cdev
+ * Pointer to common device.
+ *
+ * @return
+ * Port index of found device, RTE_MAX_ETHPORTS if not found.
+ */
+uint16_t
+mlx5_eth_find_next_sibling(uint16_t start_port_id, struct mlx5_common_device *cdev)
+{
+ struct mlx5_priv *priv;
+ uint16_t port_id;
+
+ if (cdev == NULL)
+ return RTE_MAX_ETHPORTS;
+
+ for (port_id = start_port_id; port_id < RTE_MAX_ETHPORTS; port_id++) {
+ if (rte_eth_dev_is_valid_port(port_id) == 0)
+ continue;
+
+ if (!mlx5_dev_is_managed(rte_eth_devices[port_id].device))
+ continue;
+
+ priv = rte_eth_devices[port_id].data->dev_private;
+ if (priv == NULL || priv->sh == NULL)
+ continue;
+
+ if (priv->sh->cdev == cdev)
+ break;
+ }
+
+ if (port_id >= RTE_MAX_ETHPORTS)
+ return RTE_MAX_ETHPORTS;
+
+ return port_id;
+}
+
static int
mlx5_hairpin_queue_counter_supported(struct mlx5_priv *priv)
{
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index b1493745ee..e4ebdd4c2f 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -2295,6 +2295,7 @@ void mlx5_proc_priv_uninit(struct rte_eth_dev *dev);
int mlx5_udp_tunnel_port_add(struct rte_eth_dev *dev,
struct rte_eth_udp_tunnel *udp_tunnel);
uint16_t mlx5_eth_find_next(uint16_t port_id, struct rte_device *odev);
+uint16_t mlx5_eth_find_next_sibling(uint16_t port_id, struct mlx5_common_device *cdev);
int mlx5_dev_close(struct rte_eth_dev *dev);
int mlx5_net_remove(struct mlx5_common_device *cdev);
bool mlx5_is_hpf(struct rte_eth_dev *dev);
@@ -2312,6 +2313,11 @@ int mlx5_disable_per_queue_hairpin_counter(struct rte_eth_dev *dev, uint64_t id)
for (port_id = mlx5_eth_find_next(0, dev); \
port_id < RTE_MAX_ETHPORTS; \
port_id = mlx5_eth_find_next(port_id + 1, dev))
+/* Macro to iterate over all valid ports for mlx5 driver spawned on given common device. */
+#define MLX5_ETH_FOREACH_SIBLING(port_id, cdev) \
+ for (port_id = mlx5_eth_find_next_sibling(0, cdev); \
+ port_id < RTE_MAX_ETHPORTS; \
+ port_id = mlx5_eth_find_next_sibling(port_id + 1, cdev))
void mlx5_rt_timestamp_config(struct mlx5_dev_ctx_shared *sh,
struct mlx5_hca_attr *hca_attr);
struct mlx5_dev_ctx_shared *
diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c
index 306c1cd734..f41724725c 100644
--- a/drivers/net/mlx5/mlx5_ethdev.c
+++ b/drivers/net/mlx5/mlx5_ethdev.c
@@ -424,15 +424,12 @@ mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info)
if (priv->representor) {
uint16_t port_id;
- MLX5_ETH_FOREACH_DEV(port_id, dev->device) {
- struct mlx5_priv *opriv =
- rte_eth_devices[port_id].data->dev_private;
-
- if (!opriv ||
- opriv->representor ||
- opriv->sh != priv->sh ||
- opriv->domain_id != priv->domain_id)
+ MLX5_ETH_FOREACH_SIBLING(port_id, priv->sh->cdev) {
+ struct mlx5_priv *opriv = rte_eth_devices[port_id].data->dev_private;
+
+ if (opriv->representor)
continue;
+
/*
* Override switch name with that of the master
* device.
@@ -523,14 +520,9 @@ mlx5_representor_info_count(struct rte_eth_dev *dev)
uint16_t port_id;
unsigned int count = 0;
- MLX5_ETH_FOREACH_DEV(port_id, dev->device) {
+ MLX5_ETH_FOREACH_SIBLING(port_id, priv->sh->cdev) {
struct mlx5_priv *opriv = rte_eth_devices[port_id].data->dev_private;
- if (!opriv ||
- opriv->sh != priv->sh ||
- opriv->domain_id != priv->domain_id)
- continue;
-
count += mlx5_representor_info_count_one(opriv);
}
@@ -629,14 +621,9 @@ mlx5_representor_info_fill(struct rte_eth_dev *dev,
info->controller = priv->port_info.ctrl_num;
info->pf = RTE_BUS_DEVICE(dev->device, struct rte_pci_device)->addr.function;
- MLX5_ETH_FOREACH_DEV(port_id, dev->device) {
+ MLX5_ETH_FOREACH_SIBLING(port_id, priv->sh->cdev) {
struct mlx5_priv *opriv = rte_eth_devices[port_id].data->dev_private;
- if (!opriv ||
- opriv->sh != priv->sh ||
- opriv->domain_id != priv->domain_id)
- continue;
-
mlx5_representor_info_fill_one(opriv, info);
}
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index b0eac185b5..cceb6668f9 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -1665,15 +1665,12 @@ mlx5_flow_rxq_mark_flag_set(struct rte_eth_dev *dev)
if (priv->sh->shared_mark_enabled)
return;
if (priv->master || priv->representor) {
- MLX5_ETH_FOREACH_DEV(port_id, dev->device) {
- struct mlx5_priv *opriv =
- rte_eth_devices[port_id].data->dev_private;
-
- if (!opriv ||
- opriv->sh != priv->sh ||
- opriv->domain_id != priv->domain_id ||
- opriv->mark_enabled)
+ MLX5_ETH_FOREACH_SIBLING(port_id, priv->sh->cdev) {
+ struct mlx5_priv *opriv = rte_eth_devices[port_id].data->dev_private;
+
+ if (opriv->mark_enabled)
continue;
+
LIST_FOREACH(rxq_ctrl, &opriv->rxqsctrl, next) {
rxq_ctrl->rxq.mark = 1;
}
@@ -12290,12 +12287,9 @@ mlx5_flow_pick_transfer_proxy(struct rte_eth_dev *dev,
*proxy_port_id = dev->data->port_id;
return 0;
}
- MLX5_ETH_FOREACH_DEV(port_id, dev->device) {
- const struct rte_eth_dev *port_dev = &rte_eth_devices[port_id];
- const struct mlx5_priv *port_priv = port_dev->data->dev_private;
-
- if (port_priv->master &&
- port_priv->domain_id == priv->domain_id) {
+ MLX5_ETH_FOREACH_SIBLING(port_id, priv->sh->cdev) {
+ const struct mlx5_priv *port_priv = rte_eth_devices[port_id].data->dev_private;
+ if (port_priv->master) {
*proxy_port_id = port_id;
return 0;
}
@@ -12511,7 +12505,7 @@ flow_disable_steering_run_on_related(struct rte_eth_dev *dev,
}
/* Run callback on representors. */
- MLX5_ETH_FOREACH_DEV(other_port_id, dev->device) {
+ MLX5_ETH_FOREACH_SIBLING(other_port_id, priv->sh->cdev) {
struct rte_eth_dev *other_dev = &rte_eth_devices[other_port_id];
if (other_port_id != proxy_port_id)
diff --git a/drivers/net/mlx5/mlx5_mac.c b/drivers/net/mlx5/mlx5_mac.c
index 9e7bf7f2ec..8cbaaf67dd 100644
--- a/drivers/net/mlx5/mlx5_mac.c
+++ b/drivers/net/mlx5/mlx5_mac.c
@@ -191,7 +191,7 @@ mlx5_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr)
mac_addr,
MLX5_REPRESENTOR_REPR(priv->representor_id));
}
- RTE_ETH_FOREACH_DEV_SIBLING(port_id, dev->data->port_id) {
+ MLX5_ETH_FOREACH_SIBLING(port_id, priv->sh->cdev) {
pf_priv = rte_eth_devices[port_id].data->dev_private;
if (pf_priv->master == 1)
return mlx5_os_vf_mac_addr_modify
diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c
index c5f493117b..ae0584f1b4 100644
--- a/drivers/net/mlx5/mlx5_trigger.c
+++ b/drivers/net/mlx5/mlx5_trigger.c
@@ -1578,13 +1578,11 @@ mlx5_hw_proxy_port_allowed_stop(struct rte_eth_dev *dev)
/* If transfer proxy port was not configured for HWS, then stopping it is allowed. */
if (!priv->dr_ctx)
return 0;
- MLX5_ETH_FOREACH_DEV(port_id, dev->device) {
- const struct rte_eth_dev *port_dev = &rte_eth_devices[port_id];
- const struct mlx5_priv *port_priv = port_dev->data->dev_private;
+ MLX5_ETH_FOREACH_SIBLING(port_id, priv->sh->cdev) {
+ struct rte_eth_dev *odev = &rte_eth_devices[port_id];
+ struct mlx5_priv *opriv = odev->data->dev_private;
- if (port_id != dev->data->port_id &&
- port_priv->domain_id == priv->domain_id &&
- port_dev->data->dev_started)
+ if (opriv->representor && odev->data->dev_started)
representor_started = true;
}
if (representor_started) {
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index 8e3baa95b4..2dbcbb1acb 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -437,16 +437,14 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
* Look for sibling devices in order to reuse their switch domain
* if any, otherwise allocate one.
*/
- MLX5_ETH_FOREACH_DEV(port_id, dpdk_dev) {
- const struct mlx5_priv *opriv =
- rte_eth_devices[port_id].data->dev_private;
-
- if (!opriv ||
- opriv->sh != priv->sh ||
- opriv->domain_id ==
- RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID)
+ MLX5_ETH_FOREACH_SIBLING(port_id, priv->sh->cdev) {
+ const struct mlx5_priv *opriv = rte_eth_devices[port_id].data->dev_private;
+
+ if (opriv->domain_id == RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID)
continue;
priv->domain_id = opriv->domain_id;
+ DRV_LOG(DEBUG, "dev_port-%u inherit domain_id=%u",
+ priv->dev_port, priv->domain_id);
break;
}
if (priv->domain_id == RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID) {
@@ -476,13 +474,9 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
if (priv->representor) {
eth_dev->data->dev_flags |= RTE_ETH_DEV_REPRESENTOR;
eth_dev->data->representor_id = priv->representor_id;
- MLX5_ETH_FOREACH_DEV(port_id, dpdk_dev) {
- struct mlx5_priv *opriv =
- rte_eth_devices[port_id].data->dev_private;
- if (opriv &&
- opriv->master &&
- opriv->domain_id == priv->domain_id &&
- opriv->sh == priv->sh) {
+ MLX5_ETH_FOREACH_SIBLING(port_id, priv->sh->cdev) {
+ struct mlx5_priv *opriv = rte_eth_devices[port_id].data->dev_private;
+ if (opriv->master) {
eth_dev->data->backer_port_id = port_id;
break;
}
--
2.43.0
More information about the dev
mailing list