Add simple RX recv functions (zxdh_recv_single_pkts) for single-segment packet recv. And optimize Rx recv pkts packed ops. Signed-off-by: Junlong Wang <wang.junlong1@zte.com.cn> --- drivers/net/zxdh/zxdh_ethdev.c | 40 +++++-- drivers/net/zxdh/zxdh_ethdev_ops.c | 24 ++-- drivers/net/zxdh/zxdh_ethdev_ops.h | 4 + drivers/net/zxdh/zxdh_rxtx.c | 179 +++++++++++++++++++++++------ drivers/net/zxdh/zxdh_rxtx.h | 16 +-- 5 files changed, 200 insertions(+), 63 deletions(-) diff --git a/drivers/net/zxdh/zxdh_ethdev.c b/drivers/net/zxdh/zxdh_ethdev.c index 625ce9d74c..7363d4c3d3 100644 --- a/drivers/net/zxdh/zxdh_ethdev.c +++ b/drivers/net/zxdh/zxdh_ethdev.c @@ -1269,18 +1269,44 @@ zxdh_dev_close(struct rte_eth_dev *dev) return ret; } -static int32_t -zxdh_set_rxtx_funcs(struct rte_eth_dev *eth_dev) +/* + * Determine whether the current configuration requires support for scattered + * receive; return 1 if scattered receive is required and 0 if not. + */ +static int zxdh_scattered_rx(struct rte_eth_dev *eth_dev) { - struct zxdh_hw *hw = eth_dev->data->dev_private; + uint16_t buf_size; - if (!zxdh_pci_with_feature(hw, ZXDH_NET_F_MRG_RXBUF)) { - PMD_DRV_LOG(ERR, "port %u not support rx mergeable", eth_dev->data->port_id); - return -1; + if (eth_dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO) { + eth_dev->data->lro = 1; + return 1; } + + if (eth_dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_SCATTER) + return 1; + + + PMD_DRV_LOG(DEBUG, "port %d min_rx_buf_size %d", + eth_dev->data->port_id, eth_dev->data->min_rx_buf_size); + buf_size = eth_dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM; + if (eth_dev->data->mtu + ZXDH_ETH_OVERHEAD > buf_size) + return 1; + + return 0; +} + +static int32_t +zxdh_set_rxtx_funcs(struct rte_eth_dev *eth_dev) +{ eth_dev->tx_pkt_prepare = zxdh_xmit_pkts_prepare; + eth_dev->data->scattered_rx = zxdh_scattered_rx(eth_dev); + eth_dev->tx_pkt_burst = &zxdh_xmit_pkts_packed; - eth_dev->rx_pkt_burst = &zxdh_recv_pkts_packed; + + if (eth_dev->data->scattered_rx) + eth_dev->rx_pkt_burst = &zxdh_recv_pkts_packed; + else + eth_dev->rx_pkt_burst = &zxdh_recv_single_pkts; return 0; } diff --git a/drivers/net/zxdh/zxdh_ethdev_ops.c b/drivers/net/zxdh/zxdh_ethdev_ops.c index 50247116d9..e2c2885add 100644 --- a/drivers/net/zxdh/zxdh_ethdev_ops.c +++ b/drivers/net/zxdh/zxdh_ethdev_ops.c @@ -95,10 +95,6 @@ static const struct rte_zxdh_xstats_name_off zxdh_rxq_stat_strings[] = { {"good_bytes", offsetof(struct zxdh_virtnet_rx, stats.bytes)}, {"errors", offsetof(struct zxdh_virtnet_rx, stats.errors)}, {"idle", offsetof(struct zxdh_virtnet_rx, stats.idle)}, - {"full", offsetof(struct zxdh_virtnet_rx, stats.full)}, - {"norefill", offsetof(struct zxdh_virtnet_rx, stats.norefill)}, - {"multicast_packets", offsetof(struct zxdh_virtnet_rx, stats.multicast)}, - {"broadcast_packets", offsetof(struct zxdh_virtnet_rx, stats.broadcast)}, {"truncated_err", offsetof(struct zxdh_virtnet_rx, stats.truncated_err)}, {"offload_cfg_err", offsetof(struct zxdh_virtnet_rx, stats.offload_cfg_err)}, {"invalid_hdr_len_err", offsetof(struct zxdh_virtnet_rx, stats.invalid_hdr_len_err)}, @@ -117,14 +113,12 @@ static const struct rte_zxdh_xstats_name_off zxdh_txq_stat_strings[] = { {"good_packets", offsetof(struct zxdh_virtnet_tx, stats.packets)}, {"good_bytes", offsetof(struct zxdh_virtnet_tx, stats.bytes)}, {"errors", offsetof(struct zxdh_virtnet_tx, stats.errors)}, - {"idle", offsetof(struct zxdh_virtnet_tx, stats.idle)}, - {"norefill", offsetof(struct zxdh_virtnet_tx, stats.norefill)}, - {"multicast_packets", offsetof(struct zxdh_virtnet_tx, stats.multicast)}, - {"broadcast_packets", offsetof(struct zxdh_virtnet_tx, stats.broadcast)}, + {"idle", offsetof(struct zxdh_virtnet_tx, stats.idle)}, {"truncated_err", offsetof(struct zxdh_virtnet_tx, stats.truncated_err)}, {"offload_cfg_err", offsetof(struct zxdh_virtnet_tx, stats.offload_cfg_err)}, {"invalid_hdr_len_err", offsetof(struct zxdh_virtnet_tx, stats.invalid_hdr_len_err)}, {"no_segs_err", offsetof(struct zxdh_virtnet_tx, stats.no_segs_err)}, + {"no_free_tx_desc_err", offsetof(struct zxdh_virtnet_tx, stats.no_free_tx_desc_err)}, {"undersize_packets", offsetof(struct zxdh_virtnet_tx, stats.size_bins[0])}, {"size_64_packets", offsetof(struct zxdh_virtnet_tx, stats.size_bins[1])}, {"size_65_127_packets", offsetof(struct zxdh_virtnet_tx, stats.size_bins[2])}, @@ -2026,6 +2020,20 @@ int zxdh_dev_mtu_set(struct rte_eth_dev *dev, uint16_t new_mtu) uint16_t vfid = zxdh_vport_to_vfid(hw->vport); int ret; + /* If device is started, refuse mtu that requires the support of + * scattered packets when this feature has not been enabled before. + */ + if (dev->data->dev_started && + ((!dev->data->scattered_rx && + ((uint32_t)ZXDH_MTU_TO_PKTLEN(new_mtu) > + (dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM))) || + (dev->data->scattered_rx && + ((uint32_t)ZXDH_MTU_TO_PKTLEN(new_mtu) <= + (dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM))))) { + PMD_DRV_LOG(ERR, "Stop port first."); + return -EINVAL; + } + if (hw->is_pf) { ret = zxdh_get_panel_attr(dev, &panel); if (ret != 0) { diff --git a/drivers/net/zxdh/zxdh_ethdev_ops.h b/drivers/net/zxdh/zxdh_ethdev_ops.h index 6dfe4be473..c49d79c232 100644 --- a/drivers/net/zxdh/zxdh_ethdev_ops.h +++ b/drivers/net/zxdh/zxdh_ethdev_ops.h @@ -40,6 +40,10 @@ #define ZXDH_SPM_SPEED_4X_100G RTE_BIT32(10) #define ZXDH_SPM_SPEED_4X_200G RTE_BIT32(11) +#define ZXDH_VLAN_TAG_LEN 4 +#define ZXDH_ETH_OVERHEAD (RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN + ZXDH_VLAN_TAG_LEN * 2) +#define ZXDH_MTU_TO_PKTLEN(mtu) ((mtu) + ZXDH_ETH_OVERHEAD) + struct zxdh_np_stats_data { uint64_t n_pkts_dropped; uint64_t n_bytes_dropped; diff --git a/drivers/net/zxdh/zxdh_rxtx.c b/drivers/net/zxdh/zxdh_rxtx.c index 93506a4b49..cc266e8659 100644 --- a/drivers/net/zxdh/zxdh_rxtx.c +++ b/drivers/net/zxdh/zxdh_rxtx.c @@ -114,6 +114,8 @@ RTE_MBUF_F_TX_SEC_OFFLOAD | \ RTE_MBUF_F_TX_UDP_SEG) +#define rxq_get_vq(q) ((q)->vq) + uint32_t zxdh_outer_l2_type[16] = { 0, RTE_PTYPE_L2_ETHER, @@ -613,10 +615,12 @@ zxdh_dequeue_burst_rx_packed(struct zxdh_virtqueue *vq, uint16_t i, used_idx; uint16_t id; + used_idx = vq->vq_used_cons_idx; + rte_prefetch0(&desc[used_idx]); + for (i = 0; i < num; i++) { used_idx = vq->vq_used_cons_idx; - /** - * desc_is_used has a load-acquire or rte_io_rmb inside + /* desc_is_used has a load-acquire or rte_io_rmb inside * and wait for used desc in virtqueue. */ if (!desc_is_used(&desc[used_idx], vq)) @@ -823,17 +827,52 @@ zxdh_rx_update_mbuf(struct zxdh_hw *hw, struct rte_mbuf *m, struct zxdh_net_hdr_ } } -static void zxdh_discard_rxbuf(struct zxdh_virtqueue *vq, struct rte_mbuf *m) +static void refill_desc_unwrap(struct zxdh_virtqueue *vq, + struct rte_mbuf **cookie, uint16_t nb_pkts) { - int32_t error = 0; - /* - * Requeue the discarded mbuf. This should always be - * successful since it was just dequeued. - */ - error = zxdh_enqueue_recv_refill_packed(vq, &m, 1); - if (unlikely(error)) { - PMD_RX_LOG(ERR, "cannot enqueue discarded mbuf"); - rte_pktmbuf_free(m); + struct zxdh_vring_packed_desc *start_dp = vq->vq_packed.ring.desc; + struct zxdh_vq_desc_extra *dxp; + uint16_t flags = vq->cached_flags; + int32_t i; + uint16_t idx; + + idx = vq->vq_avail_idx; + for (i = 0; i < nb_pkts; i++) { + dxp = &vq->vq_descx[idx]; + dxp->cookie = (void *)cookie[i]; + start_dp[idx].addr = rte_mbuf_iova_get(cookie[i]) + RTE_PKTMBUF_HEADROOM; + start_dp[idx].len = cookie[i]->buf_len - RTE_PKTMBUF_HEADROOM; + zxdh_queue_store_flags_packed(&start_dp[idx], flags); + idx++; + } + vq->vq_avail_idx += nb_pkts; + vq->vq_free_cnt = vq->vq_free_cnt - nb_pkts; +} + +static void refill_que_descs(struct zxdh_virtqueue *vq, struct rte_eth_dev *dev) +{ + /* free_cnt may include mrg descs */ + struct rte_mbuf *new_pkts[ZXDH_MBUF_BURST_SZ]; + uint16_t free_cnt = RTE_MIN(ZXDH_MBUF_BURST_SZ, vq->vq_free_cnt); + struct zxdh_virtnet_rx *rxvq = &vq->rxq; + uint16_t unwrap_cnt, left_cnt; + + if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) { + left_cnt = free_cnt; + unwrap_cnt = 0; + if ((vq->vq_avail_idx + free_cnt) >= vq->vq_nentries) { + unwrap_cnt = vq->vq_nentries - vq->vq_avail_idx; + left_cnt = free_cnt - unwrap_cnt; + refill_desc_unwrap(vq, new_pkts, unwrap_cnt); + vq->vq_avail_idx = 0; + vq->cached_flags ^= ZXDH_VRING_PACKED_DESC_F_AVAIL_USED; + } + if (left_cnt) + refill_desc_unwrap(vq, new_pkts + unwrap_cnt, left_cnt); + + rte_io_wmb(); + } else { + dev->data->rx_mbuf_alloc_failed += free_cnt; } } @@ -842,7 +881,7 @@ zxdh_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) { struct zxdh_virtnet_rx *rxvq = rx_queue; - struct zxdh_virtqueue *vq = rxvq->vq; + struct zxdh_virtqueue *vq = rxq_get_vq(rxvq); struct zxdh_hw *hw = vq->hw; struct rte_mbuf *rxm = NULL; struct rte_mbuf *prev = NULL; @@ -852,7 +891,6 @@ zxdh_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t len = 0; uint32_t seg_num = 0; uint32_t seg_res = 0; - uint32_t error = 0; uint16_t hdr_size = 0; uint16_t nb_rx = 0; uint16_t i; @@ -873,7 +911,8 @@ zxdh_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts, rx_pkts[nb_rx] = rxm; prev = rxm; len = lens[i]; - header = rte_pktmbuf_mtod(rxm, struct zxdh_net_hdr_ul *); + header = (struct zxdh_net_hdr_ul *)((char *) + rxm->buf_addr + RTE_PKTMBUF_HEADROOM); seg_num = header->type_hdr.num_buffers; @@ -886,7 +925,7 @@ zxdh_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts, rxvq->stats.invalid_hdr_len_err++; continue; } - rxm->data_off += hdr_size; + rxm->data_off = RTE_PKTMBUF_HEADROOM + hdr_size; rxm->nb_segs = seg_num; rxm->ol_flags = 0; rcvd_pkt_len = len - hdr_size; @@ -902,18 +941,19 @@ zxdh_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts, len = lens[i]; rxm = rcv_pkts[i]; rxm->data_len = len; + rxm->data_off = RTE_PKTMBUF_HEADROOM; rcvd_pkt_len += len; prev->next = rxm; prev = rxm; rxm->next = NULL; - seg_res -= 1; + seg_res--; } if (!seg_res) { if (rcvd_pkt_len != rx_pkts[nb_rx]->pkt_len) { PMD_RX_LOG(ERR, "dropped rcvd_pkt_len %d pktlen %d", rcvd_pkt_len, rx_pkts[nb_rx]->pkt_len); - zxdh_discard_rxbuf(vq, rx_pkts[nb_rx]); + rte_pktmbuf_free(rx_pkts[nb_rx]); rxvq->stats.errors++; rxvq->stats.truncated_err++; continue; @@ -942,14 +982,14 @@ zxdh_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts, prev->next = rxm; prev = rxm; rxm->next = NULL; - extra_idx += 1; + extra_idx++; } seg_res -= rcv_cnt; if (!seg_res) { if (unlikely(rcvd_pkt_len != rx_pkts[nb_rx]->pkt_len)) { PMD_RX_LOG(ERR, "dropped rcvd_pkt_len %d pktlen %d", rcvd_pkt_len, rx_pkts[nb_rx]->pkt_len); - zxdh_discard_rxbuf(vq, rx_pkts[nb_rx]); + rte_pktmbuf_free(rx_pkts[nb_rx]); rxvq->stats.errors++; rxvq->stats.truncated_err++; continue; @@ -961,26 +1001,91 @@ zxdh_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts, rxvq->stats.packets += nb_rx; refill: - /* Allocate new mbuf for the used descriptor */ - if (likely(!zxdh_queue_full(vq))) { - struct rte_mbuf *new_pkts[ZXDH_MBUF_BURST_SZ]; - /* free_cnt may include mrg descs */ - uint16_t free_cnt = RTE_MIN(vq->vq_free_cnt, ZXDH_MBUF_BURST_SZ); - - if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) { - error = zxdh_enqueue_recv_refill_packed(vq, new_pkts, free_cnt); - if (unlikely(error)) { - for (i = 0; i < free_cnt; i++) - rte_pktmbuf_free(new_pkts[i]); - } + if (vq->vq_free_cnt > 0) { + struct rte_eth_dev *dev = hw->eth_dev; + refill_que_descs(vq, dev); + zxdh_queue_notify(vq); + } - if (unlikely(zxdh_queue_kick_prepare_packed(vq))) - zxdh_queue_notify(vq); - } else { - struct rte_eth_dev *dev = hw->eth_dev; + return nb_rx; +} + +static inline int zxdh_init_mbuf(struct rte_mbuf *rxm, uint16_t len, + struct zxdh_hw *hw, struct zxdh_virtnet_rx *rxvq) +{ + uint16_t hdr_size = 0; + struct zxdh_net_hdr_ul *header; + + header = (struct zxdh_net_hdr_ul *)((char *) + rxm->buf_addr + RTE_PKTMBUF_HEADROOM); + rxm->ol_flags = 0; + rxm->vlan_tci = 0; + rxm->vlan_tci_outer = 0; + + hdr_size = header->type_hdr.pd_len << 1; + if (unlikely(header->type_hdr.num_buffers != 1)) { + PMD_RX_LOG(DEBUG, "hdr_size:%u nb_segs %d is invalid", + hdr_size, header->type_hdr.num_buffers); + rte_pktmbuf_free(rxm); + rxvq->stats.invalid_hdr_len_err++; + return -1; + } + zxdh_rx_update_mbuf(hw, rxm, header); + + rxm->nb_segs = 1; + rxm->data_off = RTE_PKTMBUF_HEADROOM + hdr_size; + rxm->data_len = len - hdr_size; + rxm->port = hw->port_id; + + if (rxm->data_len != rxm->pkt_len) { + PMD_RX_LOG(ERR, "dropped rcvd_pkt_len %d pktlen %d bufaddr %p.", + rxm->data_len, rxm->pkt_len, rxm->buf_addr); + rte_pktmbuf_free(rxm); + rxvq->stats.truncated_err++; + rxvq->stats.errors++; + return -1; + } + return 0; +} + +uint16_t zxdh_recv_single_pkts(void *rx_queue, struct rte_mbuf **rcv_pkts, uint16_t nb_pkts) +{ + struct zxdh_virtnet_rx *rxvq = rx_queue; + struct zxdh_virtqueue *vq = rxq_get_vq(rxvq); + struct zxdh_hw *hw = vq->hw; + struct rte_mbuf *rxm; + uint32_t lens[ZXDH_MBUF_BURST_SZ]; + uint16_t len = 0; + uint16_t nb_rx = 0; + uint16_t num; + uint16_t i = 0; - dev->data->rx_mbuf_alloc_failed += free_cnt; + num = nb_pkts; + if (unlikely(num > ZXDH_MBUF_BURST_SZ)) + num = ZXDH_MBUF_BURST_SZ; + num = zxdh_dequeue_burst_rx_packed(vq, rcv_pkts, lens, num); + if (num == 0) { + rxvq->stats.idle++; + goto refill; + } + + for (i = 0; i < num; i++) { + rxm = rcv_pkts[i]; + len = lens[i]; + if (unlikely(zxdh_init_mbuf(rxm, len, hw, &vq->rxq) < 0)) { + rte_pktmbuf_free(rxm); + continue; } + zxdh_update_packet_stats(&rxvq->stats, rxm); + nb_rx++; + } + rxvq->stats.packets += nb_rx; + +refill: + if (vq->vq_free_cnt > 0) { + struct rte_eth_dev *dev = hw->eth_dev; + refill_que_descs(vq, dev); + zxdh_queue_notify(vq); } return nb_rx; } diff --git a/drivers/net/zxdh/zxdh_rxtx.h b/drivers/net/zxdh/zxdh_rxtx.h index 424048607e..dba9567414 100644 --- a/drivers/net/zxdh/zxdh_rxtx.h +++ b/drivers/net/zxdh/zxdh_rxtx.h @@ -36,29 +36,22 @@ struct zxdh_virtnet_stats { uint64_t bytes; uint64_t errors; uint64_t idle; - uint64_t full; - uint64_t norefill; - uint64_t multicast; - uint64_t broadcast; uint64_t truncated_err; uint64_t offload_cfg_err; uint64_t invalid_hdr_len_err; uint64_t no_segs_err; + uint64_t no_free_tx_desc_err; uint64_t size_bins[8]; }; struct __rte_cache_aligned zxdh_virtnet_rx { struct zxdh_virtqueue *vq; - - uint64_t mbuf_initializer; /* value to init mbufs. */ struct rte_mempool *mpool; /* mempool for mbuf allocation */ - uint16_t queue_id; /* DPDK queue index. */ - uint16_t port_id; /* Device port identifier. */ struct zxdh_virtnet_stats stats; const struct rte_memzone *mz; /* mem zone to populate RX ring. */ - - /* dummy mbuf, for wraparound when processing RX ring. */ - struct rte_mbuf fake_mbuf; + uint64_t offloads; + uint16_t queue_id; /* DPDK queue index. */ + uint16_t port_id; /* Device port identifier. */ }; struct __rte_cache_aligned zxdh_virtnet_tx { @@ -75,5 +68,6 @@ struct __rte_cache_aligned zxdh_virtnet_tx { uint16_t zxdh_xmit_pkts_packed(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts); uint16_t zxdh_xmit_pkts_prepare(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts); uint16_t zxdh_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts); +uint16_t zxdh_recv_single_pkts(void *rx_queue, struct rte_mbuf **rcv_pkts, uint16_t nb_pkts); #endif /* ZXDH_RXTX_H */ -- 2.27.0