[dpdk-users] Intel 10G X550T NIC takes a long time(25s+) after DPDK initialization to passing traffic correctly.

[Jun Gan]

Hi,

We have an intel 10G X550T NIC, after bind it to igb_uio and launched our dpdk application.

Although I don’t see any error reported while calling “rte_eth_tx_burst”, I cannot see any packets on peer side, until at least 25 seconds later. Everything finally became normal.

I don’t see this issue with my Mellanox ConnectX-5 NIC, using same code.

My dpdk init code, nothing special I think:

Status InitDpdk() {
  Status status = SpdkHelper::Init();
  if (status != kStatusOk) {
    return GeneralError::EC_FAILURE;
  }

  rc = rte_openlog_stream(stderr);
  assert(rc == 0);

  int dev_socket_id = rte_eth_dev_socket_id(eth_port_);

  rte_eth_dev_info_get(eth_port_, &dev_info_);
  if ((dev_info_.tx_offload_capa & DEV_TX_OFFLOAD_MULTI_SEGS) == 0) {
    return GeneralError::EC_FAILURE;
  }

  int size_pktpool =
      2048 //dev_info_.rx_desc_lim.nb_max + dev_info_.tx_desc_lim.nb_max
          + 512 /* PKTPOOL_EXTRA_SIZE */;

  // Packet mem pool for tx
  tx_pkt_pool_ = rte_pktmbuf_pool_create("pkt_pool_tx",
                                         size_pktpool,
                                         32 /* per lcore cache size*/,
                                         0,
                                         RTE_MBUF_DEFAULT_BUF_SIZE,
                                         dev_socket_id);
  assert(tx_pkt_pool_ != NULL);

  // Packet mem pool for rx
  rx_pkt_pool_ = rte_pktmbuf_pool_create("pkt_pool_rx",
                                         size_pktpool,
                                         32 /* per lcore cache size*/,
                                         0,
                                         RTE_MBUF_DEFAULT_BUF_SIZE,
                                         dev_socket_id);
  assert(rx_pkt_pool_ != NULL);

  struct rte_eth_conf cfg_port;
  memset(&cfg_port, 0, sizeof(cfg_port));
  cfg_port.txmode.mq_mode = ETH_MQ_TX_NONE;
  cfg_port.txmode.offloads |= DEV_TX_OFFLOAD_MULTI_SEGS;

  // Reference for how to offload:
  // https://doc.dpdk.org/guides/prog_guide/mbuf_lib.html
  // Section "Meta Information".
  // Also rte_mbuf.h and examples.

  // Offload Tx checksums if possible
  std::string offloads;
  if (dev_info_.tx_offload_capa & DEV_TX_OFFLOAD_IPV4_CKSUM) {
    offloads += " IPv4 Tx cksum";
    cfg_port.txmode.offloads |= DEV_TX_OFFLOAD_IPV4_CKSUM;
  }
  if (dev_info_.tx_offload_capa & DEV_TX_OFFLOAD_UDP_CKSUM) {
    offloads += " UDP Tx cksum";
    cfg_port.txmode.offloads |= DEV_TX_OFFLOAD_UDP_CKSUM;
  }
  // Offload Rx checksums if possible
  if (dev_info_.rx_offload_capa & DEV_RX_OFFLOAD_IPV4_CKSUM) {
    offloads += " IPv4 Rx cksum";
    cfg_port.rxmode.offloads |= DEV_RX_OFFLOAD_IPV4_CKSUM;
  }
  if (dev_info_.rx_offload_capa & DEV_RX_OFFLOAD_UDP_CKSUM) {
    offloads += " UDP Rx cksum";
    cfg_port.rxmode.offloads |= DEV_RX_OFFLOAD_UDP_CKSUM;
  }

  rc = rte_eth_dev_configure(eth_port_, 1, 1, &cfg_port);
  assert(rc == 0);
  if (rc) {
    return GeneralError::EC_FAILURE;
  }

  uint16_t n_rx_desc = 1024;
  uint16_t n_tx_desc = 1024;
  rc = rte_eth_dev_adjust_nb_rx_tx_desc(eth_port_, &n_rx_desc, &n_tx_desc);
  assert(rc == 0);
  if (rc) {
    return GeneralError::EC_FAILURE;
  }

  struct rte_eth_rxconf rxconf;
  rxconf = dev_info_.default_rxconf;
  rc = rte_eth_rx_queue_setup(eth_port_, 0, n_rx_desc,
                              dev_socket_id, &rxconf, rx_pkt_pool_);
  assert(rc == 0);
  if (rc) {
    return GeneralError::EC_FAILURE;
  }

  struct rte_eth_txconf txconf;
  txconf = dev_info_.default_txconf;
  rc = rte_eth_tx_queue_setup(eth_port_, 0, n_tx_desc,
                              dev_socket_id, &txconf);
  assert(rc == 0);
  if (rc) {
    return GeneralError::EC_FAILURE;
  }

  rc = rte_eth_dev_start(eth_port_);
  assert(rc == 0);
  if (rc) {
    return GeneralError::EC_FAILURE;
  }

  // RTE Ring for our TX packets, with multiple producers but single consumer
  tx_ring_ = rte_ring_create("TX Ring", tx_ring_size_,
                             dev_socket_id, RING_F_SC_DEQ);
  assert(tx_ring_ != nullptr);
  if (tx_ring_ == nullptr) {
    return GeneralError::EC_FAILURE;
  }
  return kStatusOk;
}

Really appreciate it if someone can help!