[PATCH 7/9] net/gve: implement proactive device reset
Jasper Tran O'Leary
jtranoleary at google.com
Tue Oct 27 19:54:16 CET 2026
Report RTE_ETH_ERROR_HANDLE_MODE_PROACTIVE in device info and monitor
for device reset requests during periodic status polling. When a reset
is detected, spawn a dedicated control recovery thread to re-establish
the control plane, re-create queues from saved configurations, redirect
the fast-path operations table to prevent concurrent crashes, and emit
proactive recovery DPDK lifecycle events (ERR_RECOVERING,
RECOVERY_SUCCESS, RECOVERY_FAILED).
Proactive device reset supercedes the previously implemented passive
device reset convention for the gve driver.
Signed-off-by: Jasper Tran O'Leary <jtranoleary at google.com>
Reviewed-by: Joshua Washington <joshwash at google.com>
---
doc/guides/nics/gve.rst | 67 ++--
doc/guides/rel_notes/release_26_11.rst | 5 +
drivers/net/gve/gve_ethdev.c | 384 ++++++++++++++------
3 files changed, 321 insertions(+), 135 deletions(-)
diff --git a/doc/guides/nics/gve.rst b/doc/guides/nics/gve.rst
index c5e9ab70d..1b7a72cf1 100644
--- a/doc/guides/nics/gve.rst
+++ b/doc/guides/nics/gve.rst
@@ -153,37 +153,60 @@ require the DQO queue format.
Device Reset
^^^^^^^^^^^^
-The driver allows an application to reset the gVNIC device.
-This function will tear down and reinitialize the device's resources,
-including queues and administrative queues.
-
-It is the application's responsibility to reinitialize
-and restart the device after resetting it.
-
-In addition, the driver supports device-requested resets, which are triggered
-when the device encounters an unrecoverable error. The driver detects this via
-a device status register and raises an ``RTE_ETH_EVENT_INTR_RESET`` event.
-
-The application must register a callback to handle this event. The callback
-should handle the reset process (reset, reconfigure, restart).
+The driver supports proactive error recovery
+(``RTE_ETH_ERROR_HANDLE_MODE_PROACTIVE``)
+for both device-requested resets and application-initiated resets
+via ``rte_eth_dev_reset()``.
+
+A background control thread periodically polls the device status register
+to detect when the device requests a reset.
+When a reset is triggered, the driver automatically recovers the port
+without requiring the application to reconfigure or restart the device:
+
+- Emits ``RTE_ETH_EVENT_ERR_RECOVERING`` and redirects Rx/Tx burst functions
+ to dummy handlers so concurrent datapath threads can continue polling safely.
+- Tears down and re-initializes the control plane,
+ re-registers device resources,
+ and recreates Rx and Tx queues from their saved configurations.
+- Restarts the device (if it was previously started),
+ restores the active Rx/Tx burst functions,
+ and emits ``RTE_ETH_EVENT_RECOVERY_SUCCESS``
+ (or ``RTE_ETH_EVENT_RECOVERY_FAILED`` if recovery fails).
+
+Applications do not need to call ``rte_eth_dev_reset()`` in response to a
+device-requested reset,
+but may optionally register callbacks for the recovery lifecycle events
+to monitor reset progress or handle unrecoverable failures:
.. code-block:: c
static int
- dev_rst_req_callback(uint16_t port_id, enum rte_eth_event_type type,
- void *param, void *ret_param)
+ dev_recovery_callback(uint16_t port_id, enum rte_eth_event_type type,
+ void *param, void *ret_param)
{
RTE_SET_USED(param);
RTE_SET_USED(ret_param);
- if (type == RTE_ETH_EVENT_INTR_RESET) {
- printf("Device requested reset on port %u\n", port_id);
- rte_eth_dev_reset(port_id);
- /* Reconfigure and restart port ... */
+ switch (type) {
+ case RTE_ETH_EVENT_ERR_RECOVERING:
+ printf("Port %u: entering recovery\n", port_id);
+ break;
+ case RTE_ETH_EVENT_RECOVERY_SUCCESS:
+ printf("Port %u: recovery succeeded\n", port_id);
+ break;
+ case RTE_ETH_EVENT_RECOVERY_FAILED:
+ printf("Port %u: recovery failed\n", port_id);
+ break;
+ default:
+ break;
}
return 0;
}
- /* Register this callback in the main execution flow */
- rte_eth_dev_callback_register(port_id, RTE_ETH_EVENT_INTR_RESET,
- dev_rst_req_callback, NULL);
+ /* Optionally register callbacks in the main execution flow */
+ rte_eth_dev_callback_register(port_id, RTE_ETH_EVENT_ERR_RECOVERING,
+ dev_recovery_callback, NULL);
+ rte_eth_dev_callback_register(port_id, RTE_ETH_EVENT_RECOVERY_SUCCESS,
+ dev_recovery_callback, NULL);
+ rte_eth_dev_callback_register(port_id, RTE_ETH_EVENT_RECOVERY_FAILED,
+ dev_recovery_callback, NULL);
diff --git a/doc/guides/rel_notes/release_26_11.rst b/doc/guides/rel_notes/release_26_11.rst
index c62d9e897..abaad4677 100644
--- a/doc/guides/rel_notes/release_26_11.rst
+++ b/doc/guides/rel_notes/release_26_11.rst
@@ -146,6 +146,11 @@ New Features
* Raised the maximum Tx and Rx ring descriptor counts
from 4096/8192 to 16384 each.
+* **Updated Google Virtual Ethernet (gve) ethernet driver.**
+
+ * Added support for proactive device reset recovery
+ (``RTE_ETH_ERROR_HANDLE_MODE_PROACTIVE``).
+
* **Updated Intel iavf driver.**
* Runtime Rx/Tx queue setup is now automatically disabled
diff --git a/drivers/net/gve/gve_ethdev.c b/drivers/net/gve/gve_ethdev.c
index cfd043dbf..66a77c123 100644
--- a/drivers/net/gve/gve_ethdev.c
+++ b/drivers/net/gve/gve_ethdev.c
@@ -13,7 +13,6 @@
#include <ethdev_driver.h>
#include <rte_cycles.h>
-static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device);
static void gve_start_dev_status_polling(struct rte_eth_dev *dev);
static void gve_stop_dev_status_polling(struct rte_eth_dev *dev);
@@ -907,50 +906,6 @@ gve_dev_close(struct rte_eth_dev *dev)
return err;
}
-static int
-gve_dev_reset(struct rte_eth_dev *dev)
-{
- struct gve_process_private *process_priv = dev->process_private;
- struct gve_priv *priv = dev->data->dev_private;
- int err;
-
- if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
- PMD_DRV_LOG(ERR,
- "Device reset on port %u not supported in secondary processes.",
- dev->data->port_id);
- return -EPERM;
- }
-
- pthread_mutex_lock(&priv->reset_lock);
-
- /* Tear down all device resources before re-initializing. */
- if (gve_get_flow_subsystem_ok(priv))
- gve_teardown_flow_subsystem(priv);
-
- /*
- * Note that gve_teardown_flow_subsystem does not destroy the
- * flow_rule_lock. The lock is preserved across device resets and only
- * destroyed on dev_close.
- */
- gve_free_queues(dev);
- gve_deconfigure_device_resources(priv);
- process_priv->ctrl_ops->free_ctrl_plane(priv);
- gve_free_device_resources(priv);
-
- err = gve_init_priv(priv, true);
- if (err != 0) {
- PMD_DRV_LOG(ERR,
- "Failed to re-init device on port %u after reset.",
- dev->data->port_id);
- pthread_mutex_unlock(&priv->reset_lock);
- return err;
- }
-
- pthread_mutex_unlock(&priv->reset_lock);
-
- return 0;
-}
-
static int
gve_verify_driver_compatibility(struct gve_priv *priv)
{
@@ -1071,6 +1026,7 @@ gve_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
dev_info->flow_type_rss_offloads = GVE_RTE_RSS_OFFLOAD_ALL;
dev_info->hash_key_size = GVE_RSS_HASH_KEY_SIZE;
dev_info->reta_size = GVE_RSS_INDIR_SIZE;
+ dev_info->err_handle_mode = RTE_ETH_ERROR_HANDLE_MODE_PROACTIVE;
return 0;
}
@@ -1529,63 +1485,6 @@ gve_read_clock(struct rte_eth_dev *dev, uint64_t *clock)
return 0;
}
-static const struct eth_dev_ops gve_eth_dev_ops = {
- .dev_configure = gve_dev_configure,
- .dev_start = gve_dev_start,
- .dev_stop = gve_dev_stop,
- .dev_close = gve_dev_close,
- .dev_reset = gve_dev_reset,
- .dev_infos_get = gve_dev_info_get,
- .rx_queue_setup = gve_rx_queue_setup,
- .tx_queue_setup = gve_tx_queue_setup,
- .rx_queue_release = gve_rx_queue_release,
- .tx_queue_release = gve_tx_queue_release,
- .rx_queue_start = gve_rx_queue_start,
- .tx_queue_start = gve_tx_queue_start,
- .rx_queue_stop = gve_rx_queue_stop,
- .tx_queue_stop = gve_tx_queue_stop,
- .flow_ops_get = gve_flow_ops_get,
- .link_update = gve_link_update,
- .stats_get = gve_dev_stats_get,
- .stats_reset = gve_dev_stats_reset,
- .mtu_set = gve_dev_mtu_set,
- .xstats_get = gve_xstats_get,
- .xstats_get_names = gve_xstats_get_names,
- .rss_hash_update = gve_rss_hash_update,
- .rss_hash_conf_get = gve_rss_hash_conf_get,
- .reta_update = gve_rss_reta_update,
- .reta_query = gve_rss_reta_query,
-};
-
-static const struct eth_dev_ops gve_eth_dev_ops_dqo = {
- .dev_configure = gve_dev_configure,
- .dev_start = gve_dev_start,
- .dev_stop = gve_dev_stop,
- .dev_close = gve_dev_close,
- .dev_reset = gve_dev_reset,
- .dev_infos_get = gve_dev_info_get,
- .rx_queue_setup = gve_rx_queue_setup_dqo,
- .tx_queue_setup = gve_tx_queue_setup_dqo,
- .rx_queue_release = gve_rx_queue_release_dqo,
- .tx_queue_release = gve_tx_queue_release_dqo,
- .rx_queue_start = gve_rx_queue_start_dqo,
- .tx_queue_start = gve_tx_queue_start_dqo,
- .rx_queue_stop = gve_rx_queue_stop_dqo,
- .tx_queue_stop = gve_tx_queue_stop_dqo,
- .flow_ops_get = gve_flow_ops_get,
- .link_update = gve_link_update,
- .stats_get = gve_dev_stats_get,
- .stats_reset = gve_dev_stats_reset,
- .mtu_set = gve_dev_mtu_set,
- .xstats_get = gve_xstats_get,
- .xstats_get_names = gve_xstats_get_names,
- .rss_hash_update = gve_rss_hash_update,
- .rss_hash_conf_get = gve_rss_hash_conf_get,
- .reta_update = gve_rss_reta_update,
- .reta_query = gve_rss_reta_query,
- .read_clock = gve_read_clock,
-};
-
static int
pci_dev_msix_vec_count(struct rte_pci_device *pdev)
{
@@ -1708,6 +1607,171 @@ gve_set_default_ring_size_bounds(struct gve_priv *priv)
priv->min_rx_desc_cnt = GVE_DEFAULT_MIN_RX_RING_SIZE;
}
+static int
+gve_recreate_queues(struct rte_eth_dev *dev)
+{
+ struct gve_priv *priv = dev->data->dev_private;
+ uint16_t port_id = dev->data->port_id;
+ uint16_t i;
+ int ret;
+
+ if (priv->rxq_configs == NULL || priv->txq_configs == NULL) {
+ PMD_DRV_LOG(ERR, "Port %u: Queue configurations not allocated",
+ port_id);
+ return -EINVAL;
+ }
+
+ for (i = 0; i < dev->data->nb_rx_queues; i++) {
+ struct gve_rxq_config *rx_cfg = &priv->rxq_configs[i];
+
+ if (!rx_cfg->allocated)
+ continue;
+
+ ret = dev->dev_ops->rx_queue_setup(dev, i, rx_cfg->nb_descriptors,
+ rx_cfg->socket_id,
+ &rx_cfg->conf,
+ rx_cfg->mb_pool);
+ if (ret != 0) {
+ PMD_DRV_LOG(ERR,
+ "Port %u: Failed to re-create RX queue %u",
+ port_id, i);
+ return ret;
+ }
+ }
+
+ for (i = 0; i < dev->data->nb_tx_queues; i++) {
+ struct gve_txq_config *tx_cfg = &priv->txq_configs[i];
+
+ if (!tx_cfg->allocated)
+ continue;
+
+ ret = dev->dev_ops->tx_queue_setup(dev, i, tx_cfg->nb_descriptors,
+ tx_cfg->socket_id,
+ &tx_cfg->conf);
+ if (ret != 0) {
+ PMD_DRV_LOG(ERR,
+ "Port %u: Failed to re-create TX queue %u",
+ port_id, i);
+ return ret;
+ }
+ }
+
+ return 0;
+}
+
+static int
+gve_internal_recover_device(struct rte_eth_dev *dev)
+{
+ struct gve_process_private *process_priv = dev->process_private;
+ struct gve_priv *priv = dev->data->dev_private;
+ uint16_t port_id = dev->data->port_id;
+ uint16_t i;
+ int ret;
+
+ rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_ERR_RECOVERING, NULL);
+
+ rte_eth_fp_ops[port_id].rx_pkt_burst = rte_eth_pkt_burst_dummy;
+ rte_eth_fp_ops[port_id].tx_pkt_burst = rte_eth_pkt_burst_dummy;
+ dev->rx_pkt_burst = rte_eth_pkt_burst_dummy;
+ dev->tx_pkt_burst = rte_eth_pkt_burst_dummy;
+
+ /* Ensure dummy burst fns are visible before clearing queue pointers. */
+ rte_atomic_thread_fence(rte_memory_order_release);
+
+ for (i = 0; i < dev->data->nb_rx_queues; i++) {
+ if (dev->data->rx_queues[i] != NULL) {
+ gve_add_zombie_rx_queue(priv, dev->data->rx_queues[i]);
+ dev->data->rx_queues[i] = NULL;
+ }
+ }
+ for (i = 0; i < dev->data->nb_tx_queues; i++) {
+ if (dev->data->tx_queues[i] != NULL) {
+ gve_add_zombie_tx_queue(priv, dev->data->tx_queues[i]);
+ dev->data->tx_queues[i] = NULL;
+ }
+ }
+
+ gve_stop_dev_status_polling(dev);
+
+ if (gve_is_gqi(priv))
+ gve_free_stats_report(dev);
+
+ gve_deconfigure_device_resources(priv);
+
+ process_priv->ctrl_ops->free_ctrl_plane(priv);
+
+ /*
+ * Keep device resources allocated, except for the RSS config which must
+ * be freed as the driver cache will be stale after reset.
+ */
+ gve_free_rss_config(&priv->rss_config);
+ if (process_priv->ctrl_ops->query_rss != NULL)
+ priv->rss_cache_dirty = true;
+
+ priv->reset_generation++;
+ priv->num_registered_pages = 0;
+ ret = process_priv->ctrl_ops->init_ctrl_plane(priv);
+ if (ret != 0) {
+ PMD_DRV_LOG(ERR,
+ "Port %u: Failed to re-init control plane after reset",
+ port_id);
+ goto recover_fail;
+ }
+
+ ret = gve_configure_device_resources(priv);
+ if (ret != 0)
+ goto recover_fail;
+
+ ret = gve_recreate_queues(dev);
+ if (ret != 0)
+ goto recover_fail;
+
+ ret = gve_dev_start(dev);
+ if (ret != 0) {
+ PMD_DRV_LOG(ERR,
+ "Port %u: Failed to restart device after reset",
+ port_id);
+ goto recover_fail;
+ }
+
+ if (gve_is_gqi(priv)) {
+ gve_set_rx_function(dev);
+ gve_set_tx_function(dev);
+ } else {
+ gve_set_rx_function_dqo(dev);
+ gve_set_tx_function_dqo(dev);
+ }
+
+ rte_eth_fp_ops[port_id].rxq.data = dev->data->rx_queues;
+ rte_eth_fp_ops[port_id].txq.data = dev->data->tx_queues;
+
+ /* Ensure queue data pointers are visible before restoring burst fns. */
+ rte_atomic_thread_fence(rte_memory_order_release);
+
+ rte_eth_fp_ops[port_id].rx_pkt_burst = dev->rx_pkt_burst;
+ rte_eth_fp_ops[port_id].tx_pkt_burst = dev->tx_pkt_burst;
+
+ PMD_DRV_LOG(INFO, "Port %u: Proactive reset recovery successful",
+ port_id);
+ rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_RECOVERY_SUCCESS, NULL);
+
+ return 0;
+
+recover_fail:
+ gve_free_queues(dev);
+ gve_deconfigure_device_resources(priv);
+
+ process_priv->ctrl_ops->free_ctrl_plane(priv);
+
+ rte_eth_fp_ops[port_id].rx_pkt_burst = rte_eth_pkt_burst_dummy;
+ rte_eth_fp_ops[port_id].tx_pkt_burst = rte_eth_pkt_burst_dummy;
+ PMD_DRV_LOG(ERR, "Port %u: Proactive reset recovery failed",
+ port_id);
+ rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_RECOVERY_FAILED, NULL);
+
+ return ret;
+}
+
static void
gve_check_device_status(void *arg)
{
@@ -1716,22 +1780,39 @@ gve_check_device_status(void *arg)
struct gve_priv *priv = dev->data->dev_private;
int ret;
- process_priv = rte_eth_devices[priv->port_id].process_private;
+ process_priv = dev->process_private;
+
+ gve_clean_zombie_queues(priv, false);
if (process_priv->ctrl_ops->check_device_needs_reset(priv)) {
- PMD_DRV_LOG(INFO,
- "Device on port %u requests a reset. Stopping device status polling.",
- dev->data->port_id);
- rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET,
- NULL);
- } else {
+ pthread_mutex_lock(&priv->reset_lock);
+
+ /*
+ * Defend against TOCTOU by checking if a reset is still needed
+ * after acquiring the lock.
+ */
+ if (process_priv->ctrl_ops->check_device_needs_reset(priv)) {
+ PMD_DRV_LOG(NOTICE,
+ "Port %u: Device requested reset. Executing proactive recovery.",
+ dev->data->port_id);
+
+ ret = gve_internal_recover_device(dev);
+ if (ret == 0) {
+ pthread_mutex_unlock(&priv->reset_lock);
+ return;
+ }
+ }
+
+ pthread_mutex_unlock(&priv->reset_lock);
+ }
+
+ if (dev->data->dev_started) {
ret = rte_eal_alarm_set(GVE_DEV_POLL_INTERVAL_US,
gve_check_device_status, dev);
- if (ret != 0) {
+ if (ret != 0)
PMD_DRV_LOG(ERR,
- "Port %u: Failed to re-arm alarm poller!",
- dev->data->port_id);
- }
+ "Port %u: Failed to re-arm alarm poller!",
+ dev->data->port_id);
}
}
@@ -1765,6 +1846,83 @@ gve_stop_dev_status_polling(struct rte_eth_dev *dev)
rte_eal_alarm_cancel(gve_check_device_status, dev);
}
+static int
+gve_dev_reset(struct rte_eth_dev *dev)
+{
+ struct gve_priv *priv = dev->data->dev_private;
+ int ret;
+
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+ PMD_DRV_LOG(ERR,
+ "Device reset on port %u not supported in secondary processes.",
+ dev->data->port_id);
+ return -EPERM;
+ }
+
+ pthread_mutex_lock(&priv->reset_lock);
+ ret = gve_internal_recover_device(dev);
+ pthread_mutex_unlock(&priv->reset_lock);
+
+ return ret;
+}
+
+static const struct eth_dev_ops gve_eth_dev_ops = {
+ .dev_configure = gve_dev_configure,
+ .dev_start = gve_dev_start,
+ .dev_stop = gve_dev_stop,
+ .dev_close = gve_dev_close,
+ .dev_reset = gve_dev_reset,
+ .dev_infos_get = gve_dev_info_get,
+ .rx_queue_setup = gve_rx_queue_setup,
+ .tx_queue_setup = gve_tx_queue_setup,
+ .rx_queue_release = gve_rx_queue_release,
+ .tx_queue_release = gve_tx_queue_release,
+ .rx_queue_start = gve_rx_queue_start,
+ .tx_queue_start = gve_tx_queue_start,
+ .rx_queue_stop = gve_rx_queue_stop,
+ .tx_queue_stop = gve_tx_queue_stop,
+ .flow_ops_get = gve_flow_ops_get,
+ .link_update = gve_link_update,
+ .stats_get = gve_dev_stats_get,
+ .stats_reset = gve_dev_stats_reset,
+ .mtu_set = gve_dev_mtu_set,
+ .xstats_get = gve_xstats_get,
+ .xstats_get_names = gve_xstats_get_names,
+ .rss_hash_update = gve_rss_hash_update,
+ .rss_hash_conf_get = gve_rss_hash_conf_get,
+ .reta_update = gve_rss_reta_update,
+ .reta_query = gve_rss_reta_query,
+};
+
+static const struct eth_dev_ops gve_eth_dev_ops_dqo = {
+ .dev_configure = gve_dev_configure,
+ .dev_start = gve_dev_start,
+ .dev_stop = gve_dev_stop,
+ .dev_close = gve_dev_close,
+ .dev_reset = gve_dev_reset,
+ .dev_infos_get = gve_dev_info_get,
+ .rx_queue_setup = gve_rx_queue_setup_dqo,
+ .tx_queue_setup = gve_tx_queue_setup_dqo,
+ .rx_queue_release = gve_rx_queue_release_dqo,
+ .tx_queue_release = gve_tx_queue_release_dqo,
+ .rx_queue_start = gve_rx_queue_start_dqo,
+ .tx_queue_start = gve_tx_queue_start_dqo,
+ .rx_queue_stop = gve_rx_queue_stop_dqo,
+ .tx_queue_stop = gve_tx_queue_stop_dqo,
+ .flow_ops_get = gve_flow_ops_get,
+ .link_update = gve_link_update,
+ .stats_get = gve_dev_stats_get,
+ .stats_reset = gve_dev_stats_reset,
+ .mtu_set = gve_dev_mtu_set,
+ .xstats_get = gve_xstats_get,
+ .xstats_get_names = gve_xstats_get_names,
+ .rss_hash_update = gve_rss_hash_update,
+ .rss_hash_conf_get = gve_rss_hash_conf_get,
+ .reta_update = gve_rss_reta_update,
+ .reta_query = gve_rss_reta_query,
+ .read_clock = gve_read_clock,
+};
+
static int
gve_adminq_get_device_properties(struct gve_priv *priv)
{
--
2.56.0.307.g5fe7bd680c-goog
More information about the dev
mailing list