update rx/tx process to latest version. Signed-off-by: Junlong Wang <wang.junlong1@zte.com.cn> --- drivers/net/zxdh/zxdh_common.c | 19 + drivers/net/zxdh/zxdh_common.h | 3 + drivers/net/zxdh/zxdh_ethdev.c | 45 ++- drivers/net/zxdh/zxdh_ethdev.h | 15 +- drivers/net/zxdh/zxdh_queue.c | 9 +- drivers/net/zxdh/zxdh_queue.h | 118 +++--- drivers/net/zxdh/zxdh_rxtx.c | 696 +++++++++++++++++++-------------- drivers/net/zxdh/zxdh_rxtx.h | 27 ++ 8 files changed, 567 insertions(+), 365 deletions(-) diff --git a/drivers/net/zxdh/zxdh_common.c b/drivers/net/zxdh/zxdh_common.c index 0c9696e7ce..3d67fcc7dd 100644 --- a/drivers/net/zxdh/zxdh_common.c +++ b/drivers/net/zxdh/zxdh_common.c @@ -13,6 +13,7 @@ #include "zxdh_logs.h" #include "zxdh_msg.h" #include "zxdh_common.h" +#include "zxdh_pci.h" #define ZXDH_MSG_RSP_SIZE_MAX 512 @@ -427,3 +428,21 @@ zxdh_datach_set(struct rte_eth_dev *dev) return ret; } + +bool +zxdh_rx_offload_enabled(struct zxdh_hw *hw) +{ + return zxdh_pci_with_feature(hw, ZXDH_NET_F_GUEST_CSUM) || + zxdh_pci_with_feature(hw, ZXDH_NET_F_GUEST_TSO4) || + zxdh_pci_with_feature(hw, ZXDH_NET_F_GUEST_TSO6) || + (hw->vlan_offload_cfg.vlan_strip == 1); +} + +bool +zxdh_tx_offload_enabled(struct zxdh_hw *hw) +{ + return zxdh_pci_with_feature(hw, ZXDH_NET_F_CSUM) || + zxdh_pci_with_feature(hw, ZXDH_NET_F_HOST_TSO4) || + zxdh_pci_with_feature(hw, ZXDH_NET_F_HOST_TSO6) || + zxdh_pci_with_feature(hw, ZXDH_NET_F_HOST_UFO); +} diff --git a/drivers/net/zxdh/zxdh_common.h b/drivers/net/zxdh/zxdh_common.h index 826f1fb95d..d78a822ebf 100644 --- a/drivers/net/zxdh/zxdh_common.h +++ b/drivers/net/zxdh/zxdh_common.h @@ -31,4 +31,7 @@ uint32_t zxdh_read_comm_reg(uint64_t pci_comm_cfg_baseaddr, uint32_t reg); void zxdh_write_comm_reg(uint64_t pci_comm_cfg_baseaddr, uint32_t reg, uint32_t val); int32_t zxdh_datach_set(struct rte_eth_dev *dev); +bool zxdh_rx_offload_enabled(struct zxdh_hw *hw); +bool zxdh_tx_offload_enabled(struct zxdh_hw *hw); + #endif /* ZXDH_COMMON_H */ diff --git a/drivers/net/zxdh/zxdh_ethdev.c b/drivers/net/zxdh/zxdh_ethdev.c index c76818d015..255d4b5b79 100644 --- a/drivers/net/zxdh/zxdh_ethdev.c +++ b/drivers/net/zxdh/zxdh_ethdev.c @@ -23,6 +23,7 @@ struct zxdh_shared_data *zxdh_shared_data; const char *ZXDH_PMD_SHARED_DATA_MZ = "zxdh_pmd_shared_data"; rte_spinlock_t zxdh_shared_data_lock = RTE_SPINLOCK_INITIALIZER; struct zxdh_dev_shared_data g_dev_sd[ZXDH_SLOT_MAX]; +struct zxdh_net_hdr_dl g_net_hdr_dl[RTE_MAX_ETHPORTS]; #define ZXDH_INVALID_DTBQUE 0xFFFF #define ZXDH_INVALID_SLOT_IDX 0xFFFF @@ -405,6 +406,28 @@ zxdh_configure_intr(struct rte_eth_dev *dev) return ret; } +static void +zxdh_update_net_hdr_dl(struct zxdh_hw *hw) +{ + struct zxdh_net_hdr_dl *net_hdr_dl = &g_net_hdr_dl[hw->port_id]; + memset(net_hdr_dl, 0, ZXDH_DL_NET_HDR_SIZE); + + if (zxdh_tx_offload_enabled(hw)) { + net_hdr_dl->type_hdr.port = ZXDH_PORT_DTP; + net_hdr_dl->type_hdr.pd_len = ZXDH_DL_NET_HDR_SIZE >> 1; + + net_hdr_dl->pipd_hdr_dl.pi_hdr.pi_len = (ZXDH_PI_HDR_SIZE >> 4) - 1; + net_hdr_dl->pipd_hdr_dl.pi_hdr.pkt_flag_hi8 = ZXDH_PI_FLAG | ZXDH_PI_TYPE_PI; + net_hdr_dl->pipd_hdr_dl.pi_hdr.pkt_type = ZXDH_PKT_FORM_CPU; + hw->dl_net_hdr_len = ZXDH_DL_NET_HDR_SIZE; + + } else { + net_hdr_dl->type_hdr.port = ZXDH_PORT_NP; + net_hdr_dl->type_hdr.pd_len = ZXDH_DL_NET_HDR_NOPI_SIZE >> 1; + hw->dl_net_hdr_len = ZXDH_DL_NET_HDR_NOPI_SIZE; + } +} + static int32_t zxdh_features_update(struct zxdh_hw *hw, const struct rte_eth_rxmode *rxmode, @@ -451,23 +474,6 @@ zxdh_features_update(struct zxdh_hw *hw, return 0; } -static bool -zxdh_rx_offload_enabled(struct zxdh_hw *hw) -{ - return zxdh_pci_with_feature(hw, ZXDH_NET_F_GUEST_CSUM) || - zxdh_pci_with_feature(hw, ZXDH_NET_F_GUEST_TSO4) || - zxdh_pci_with_feature(hw, ZXDH_NET_F_GUEST_TSO6); -} - -static bool -zxdh_tx_offload_enabled(struct zxdh_hw *hw) -{ - return zxdh_pci_with_feature(hw, ZXDH_NET_F_CSUM) || - zxdh_pci_with_feature(hw, ZXDH_NET_F_HOST_TSO4) || - zxdh_pci_with_feature(hw, ZXDH_NET_F_HOST_TSO6) || - zxdh_pci_with_feature(hw, ZXDH_NET_F_HOST_UFO); -} - static void zxdh_dev_free_mbufs(struct rte_eth_dev *dev) { @@ -892,6 +898,7 @@ zxdh_dev_configure(struct rte_eth_dev *dev) const struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode; const struct rte_eth_txmode *txmode = &dev->data->dev_conf.txmode; struct zxdh_hw *hw = dev->data->dev_private; + uint64_t rx_offloads = rxmode->offloads; int32_t ret = 0; if (dev->data->nb_rx_queues > hw->max_queue_pairs || @@ -932,6 +939,9 @@ zxdh_dev_configure(struct rte_eth_dev *dev) } } + if (rx_offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP) + hw->vlan_offload_cfg.vlan_strip = 1; + hw->has_tx_offload = zxdh_tx_offload_enabled(hw); hw->has_rx_offload = zxdh_rx_offload_enabled(hw); @@ -982,6 +992,7 @@ zxdh_dev_configure(struct rte_eth_dev *dev) end: zxdh_dev_conf_offload(dev); + zxdh_update_net_hdr_dl(hw); return ret; } diff --git a/drivers/net/zxdh/zxdh_ethdev.h b/drivers/net/zxdh/zxdh_ethdev.h index c42f638c8d..7fe561ae24 100644 --- a/drivers/net/zxdh/zxdh_ethdev.h +++ b/drivers/net/zxdh/zxdh_ethdev.h @@ -58,6 +58,14 @@ struct zxdh_chnl_context { uint16_t ph_chno; }; +struct zxdh_vlan_offload_cfg { + uint8_t vlan_strip:1; + uint8_t vlan_filter:1; + uint8_t vlan_extend:1; + uint8_t qinq_strip:1; + uint8_t resv:4; +}; + struct zxdh_hw { struct rte_eth_dev *eth_dev; struct zxdh_pci_common_cfg *common_cfg; @@ -89,11 +97,10 @@ struct zxdh_hw { uint16_t *notify_base; uint8_t *isr; - uint8_t weak_barriers; uint8_t intr_enabled; uint8_t mac_addr[RTE_ETHER_ADDR_LEN]; - uint8_t use_msix; + uint8_t duplex; uint8_t is_pf; uint8_t msg_chan_init; @@ -112,7 +119,9 @@ struct zxdh_hw { uint8_t que_set_flag; uint16_t queue_pool_count; uint16_t queue_pool_start; - uint8_t rsv[3]; + struct zxdh_vlan_offload_cfg vlan_offload_cfg; + uint8_t dl_net_hdr_len; + uint8_t rsv[2]; }; struct zxdh_dtb_shared_data { diff --git a/drivers/net/zxdh/zxdh_queue.c b/drivers/net/zxdh/zxdh_queue.c index 3e3bac2efc..d92d3bcab9 100644 --- a/drivers/net/zxdh/zxdh_queue.c +++ b/drivers/net/zxdh/zxdh_queue.c @@ -291,12 +291,10 @@ zxdh_dev_tx_queue_setup(struct rte_eth_dev *dev, int32_t zxdh_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) { - struct zxdh_hw *hw = dev->data->dev_private; struct zxdh_virtnet_rx *rxvq = dev->data->rx_queues[queue_id]; struct zxdh_virtqueue *vq = rxvq->vq; zxdh_queue_enable_intr(vq); - zxdh_mb(hw->weak_barriers); return 0; } @@ -314,7 +312,6 @@ int32_t zxdh_enqueue_recv_refill_packed(struct zxdh_virtqueue *vq, struct rte_mbuf **cookie, uint16_t num) { struct zxdh_vring_packed_desc *start_dp = vq->vq_packed.ring.desc; - struct zxdh_hw *hw = vq->hw; struct zxdh_vq_desc_extra *dxp; uint16_t flags = vq->vq_packed.cached_flags; int32_t i; @@ -328,10 +325,8 @@ int32_t zxdh_enqueue_recv_refill_packed(struct zxdh_virtqueue *vq, /* rx pkt fill in data_off */ start_dp[idx].addr = rte_mbuf_iova_get(cookie[i]) + RTE_PKTMBUF_HEADROOM; start_dp[idx].len = cookie[i]->buf_len - RTE_PKTMBUF_HEADROOM; - vq->vq_desc_head_idx = dxp->next; - if (vq->vq_desc_head_idx == ZXDH_VQ_RING_DESC_CHAIN_END) - vq->vq_desc_tail_idx = vq->vq_desc_head_idx; - zxdh_queue_store_flags_packed(&start_dp[idx], flags, hw->weak_barriers); + + zxdh_queue_store_flags_packed(&start_dp[idx], flags); if (++vq->vq_avail_idx >= vq->vq_nentries) { vq->vq_avail_idx -= vq->vq_nentries; vq->vq_packed.cached_flags ^= ZXDH_VRING_PACKED_DESC_F_AVAIL_USED; diff --git a/drivers/net/zxdh/zxdh_queue.h b/drivers/net/zxdh/zxdh_queue.h index ba946dee29..05f391f080 100644 --- a/drivers/net/zxdh/zxdh_queue.h +++ b/drivers/net/zxdh/zxdh_queue.h @@ -49,13 +49,23 @@ enum { ZXDH_VTNET_RQ = 0, ZXDH_VTNET_TQ = 1 }; #define ZXDH_RQ_QUEUE_IDX 0 #define ZXDH_TQ_QUEUE_IDX 1 +#define ZXDH_UL_1588_HDR_SIZE 8 #define ZXDH_TYPE_HDR_SIZE sizeof(struct zxdh_type_hdr) #define ZXDH_PI_HDR_SIZE sizeof(struct zxdh_pi_hdr) #define ZXDH_DL_NET_HDR_SIZE sizeof(struct zxdh_net_hdr_dl) #define ZXDH_UL_NET_HDR_SIZE sizeof(struct zxdh_net_hdr_ul) +#define ZXDH_DL_PD_HDR_SIZE sizeof(struct zxdh_pd_hdr_dl) +#define ZXDH_UL_PD_HDR_SIZE sizeof(struct zxdh_pd_hdr_ul) +#define ZXDH_DL_NET_HDR_NOPI_SIZE (ZXDH_TYPE_HDR_SIZE + \ + ZXDH_DL_PD_HDR_SIZE) +#define ZXDH_UL_NOPI_HDR_SIZE_MAX (ZXDH_TYPE_HDR_SIZE + \ + ZXDH_UL_PD_HDR_SIZE + \ + ZXDH_UL_1588_HDR_SIZE) #define ZXDH_PD_HDR_SIZE_MAX 256 #define ZXDH_PD_HDR_SIZE_MIN ZXDH_TYPE_HDR_SIZE +#define rte_packet_prefetch(p) do {} while (0) + /* * ring descriptors: 16 bytes. * These can chain together via "next". @@ -192,18 +202,29 @@ struct __rte_packed_begin zxdh_pi_hdr { } __rte_packed_end; /* 32B */ struct __rte_packed_begin zxdh_pd_hdr_dl { - uint32_t ol_flag; + uint16_t ol_flag; + uint8_t rsv; + uint8_t panel_id; + + uint16_t svlan_insert; + uint16_t cvlan_insert; + uint8_t tag_idx; uint8_t tag_data; uint16_t dst_vfid; - uint32_t svlan_insert; - uint32_t cvlan_insert; } __rte_packed_end; /* 16B */ -struct __rte_packed_begin zxdh_net_hdr_dl { - struct zxdh_type_hdr type_hdr; /* 4B */ +struct __rte_packed_begin zxdh_pipd_hdr_dl { struct zxdh_pi_hdr pi_hdr; /* 32B */ - struct zxdh_pd_hdr_dl pd_hdr; /* 16B */ + struct zxdh_pd_hdr_dl pd_hdr; /* 12B */ +} __rte_packed_end; /* 44B */ + +struct __rte_packed_begin zxdh_net_hdr_dl { + struct zxdh_type_hdr type_hdr; /* 4B */ + union { + struct zxdh_pd_hdr_dl pd_hdr; /* 12B */ + struct zxdh_pipd_hdr_dl pipd_hdr_dl; /* 44B */ + }; } __rte_packed_end; struct __rte_packed_begin zxdh_pd_hdr_ul { @@ -211,17 +232,27 @@ struct __rte_packed_begin zxdh_pd_hdr_ul { uint32_t rss_hash; uint32_t fd; uint32_t striped_vlan_tci; + + uint16_t pkt_type_out; + uint16_t pkt_type_in; + uint16_t pkt_len; + uint8_t tag_idx; uint8_t tag_data; uint16_t src_vfid; - uint16_t pkt_type_out; - uint16_t pkt_type_in; } __rte_packed_end; /* 24B */ -struct __rte_packed_begin zxdh_net_hdr_ul { - struct zxdh_type_hdr type_hdr; /* 4B */ +struct __rte_packed_begin zxdh_pipd_hdr_ul { struct zxdh_pi_hdr pi_hdr; /* 32B */ - struct zxdh_pd_hdr_ul pd_hdr; /* 24B */ + struct zxdh_pd_hdr_ul pd_hdr; /* 26B */ +} __rte_packed_end; + +struct __rte_packed_begin zxdh_net_hdr_ul { + struct zxdh_type_hdr type_hdr; /* 4B */ + union { + struct zxdh_pd_hdr_ul pd_hdr; /* 26 */ + struct zxdh_pipd_hdr_ul pipd_hdr_ul; /* 58B */ + }; } __rte_packed_end; /* 60B */ @@ -316,6 +347,19 @@ zxdh_mb(uint8_t weak_barriers) rte_mb(); } +static inline +int32_t desc_is_used(struct zxdh_vring_packed_desc *desc, struct zxdh_virtqueue *vq) +{ + uint16_t flags; + uint16_t used, avail; + + flags = desc->flags; + rte_io_rmb(); + used = !!(flags & ZXDH_VRING_PACKED_DESC_F_USED); + avail = !!(flags & ZXDH_VRING_PACKED_DESC_F_AVAIL); + return avail == used && used == vq->vq_packed.used_wrap_counter; +} + static inline int32_t zxdh_queue_full(const struct zxdh_virtqueue *vq) { @@ -323,48 +367,22 @@ zxdh_queue_full(const struct zxdh_virtqueue *vq) } static inline void -zxdh_queue_store_flags_packed(struct zxdh_vring_packed_desc *dp, - uint16_t flags, uint8_t weak_barriers) - { - if (weak_barriers) { - #ifdef RTE_ARCH_X86_64 - rte_io_wmb(); - dp->flags = flags; - #else - rte_atomic_store_explicit(&dp->flags, flags, rte_memory_order_release); - #endif - } else { - rte_io_wmb(); - dp->flags = flags; - } -} - -static inline uint16_t -zxdh_queue_fetch_flags_packed(struct zxdh_vring_packed_desc *dp, - uint8_t weak_barriers) - { - uint16_t flags; - if (weak_barriers) { - #ifdef RTE_ARCH_X86_64 - flags = dp->flags; - rte_io_rmb(); - #else - flags = rte_atomic_load_explicit(&dp->flags, rte_memory_order_acquire); - #endif - } else { - flags = dp->flags; - rte_io_rmb(); - } - - return flags; +zxdh_queue_store_flags_packed(struct zxdh_vring_packed_desc *dp, uint16_t flags) +{ + rte_io_wmb(); + dp->flags = flags; } static inline int32_t zxdh_desc_used(struct zxdh_vring_packed_desc *desc, struct zxdh_virtqueue *vq) { - uint16_t flags = zxdh_queue_fetch_flags_packed(desc, vq->hw->weak_barriers); - uint16_t used = !!(flags & ZXDH_VRING_PACKED_DESC_F_USED); - uint16_t avail = !!(flags & ZXDH_VRING_PACKED_DESC_F_AVAIL); + uint16_t flags; + uint16_t used, avail; + + flags = desc->flags; + rte_io_rmb(); + used = !!(flags & ZXDH_VRING_PACKED_DESC_F_USED); + avail = !!(flags & ZXDH_VRING_PACKED_DESC_F_AVAIL); return avail == used && used == vq->vq_packed.used_wrap_counter; } @@ -378,12 +396,14 @@ zxdh_queue_kick_prepare_packed(struct zxdh_virtqueue *vq) { uint16_t flags = 0; - zxdh_mb(vq->hw->weak_barriers); + zxdh_mb(1); flags = vq->vq_packed.ring.device->desc_event_flags; return (flags != ZXDH_RING_EVENT_FLAGS_DISABLE); } +extern struct zxdh_net_hdr_dl g_net_hdr_dl[RTE_MAX_ETHPORTS]; + struct rte_mbuf *zxdh_queue_detach_unused(struct zxdh_virtqueue *vq); int32_t zxdh_free_queues(struct rte_eth_dev *dev); int32_t zxdh_get_queue_type(uint16_t vtpci_queue_idx); diff --git a/drivers/net/zxdh/zxdh_rxtx.c b/drivers/net/zxdh/zxdh_rxtx.c index 6353d496f2..5c1795ce1d 100644 --- a/drivers/net/zxdh/zxdh_rxtx.c +++ b/drivers/net/zxdh/zxdh_rxtx.c @@ -9,6 +9,8 @@ #include "zxdh_logs.h" #include "zxdh_pci.h" +#include "zxdh_common.h" +#include "zxdh_rxtx.h" #include "zxdh_queue.h" #define ZXDH_SVLAN_TPID 0x88a8 @@ -24,8 +26,12 @@ #define ZXDH_PI_L3TYPE_RSV 0xC0 #define ZXDH_PI_L3TYPE_MASK 0xC0 +#define ZXDH_PD_OFFLOAD_SPEC_PHYPORT (1 << 15) #define ZXDH_PD_OFFLOAD_SVLAN_INSERT (1 << 14) #define ZXDH_PD_OFFLOAD_CVLAN_INSERT (1 << 13) +#define ZXDH_PD_OFFLOAD_OUTER_IPCSUM (1 << 12) +#define ZXDH_PD_OFFLOAD_PRIO_MASK (0x7 << 8) +#define ZXDH_PD_OFFLOAD_DELAY_STAT (1 << 7) #define ZXDH_PCODE_MASK 0x1F #define ZXDH_PCODE_IP_PKT_TYPE 0x01 @@ -34,9 +40,78 @@ #define ZXDH_PCODE_NO_IP_PKT_TYPE 0x09 #define ZXDH_PCODE_NO_REASSMBLE_TCP_PKT_TYPE 0x0C +/* Uplink pd header byte0~1 */ +#define ZXDH_MBUF_F_RX_OUTER_L4_CKSUM_GOOD 0x00080000 +#define ZXDH_MBUF_F_RX_QINQ 0x00100000 +#define ZXDH_MBUF_F_RX_SEC_OFFLOAD 0x00200000 +#define ZXDH_MBUF_F_RX_QINQ_STRIPPED 0x00400000 +#define FELX_4BYTE 0x00800000 +#define FELX_8BYTE 0x01000000 +#define ZXDH_MBUF_F_RX_FDIR_FLX_MASK 0x01800000 +#define ZXDH_MBUF_F_RX_FDIR_ID 0x02000000 +#define ZXDH_MBUF_F_RX_1588_TMST 0x04000000 +#define ZXDH_MBUF_F_RX_1588_PTP 0x08000000 +#define ZXDH_MBUF_F_RX_VLAN_STRIPPED 0x10000000 +#define ZXDH_MBUF_F_RX_OUTER_IP_CKSUM_BAD 0x20000000 +#define ZXDH_MBUF_F_RX_FDIR 0x40000000 +#define ZXDH_MBUF_F_RX_RSS_HASH 0x80000000 + +/* Outer/Inner L2 type */ +#define ZXDH_PD_L2TYPE_MASK 0xf000 +#define ZXDH_PTYPE_L2_ETHER 0x1000 +#define ZXDH_PTYPE_L2_ETHER_TIMESYNC 0x2000 +#define ZXDH_PTYPE_L2_ETHER_ARP 0x3000 +#define ZXDH_PTYPE_L2_ETHER_LLDP 0x4000 +#define ZXDH_PTYPE_L2_ETHER_NSH 0x5000 +#define ZXDH_PTYPE_L2_ETHER_VLAN 0x6000 +#define ZXDH_PTYPE_L2_ETHER_QINQ 0x7000 +#define ZXDH_PTYPE_L2_ETHER_PPPOE 0x8000 +#define ZXDH_PTYPE_L2_ETHER_FCOE 0x9000 +#define ZXDH_PTYPE_L2_ETHER_MPLS 0xa000 + +/* Outer/Inner L3 type */ +#define ZXDH_PD_L3TYPE_MASK 0x0f00 +#define ZXDH_PTYPE_L3_IPV4 0x0100 +#define ZXDH_PTYPE_L3_IPV4_EXT 0x0200 +#define ZXDH_PTYPE_L3_IPV6 0x0300 +#define ZXDH_PTYPE_L3_IPV4_EXT_UNKNOWN 0x0400 +#define ZXDH_PTYPE_L3_IPV6_EXT 0x0500 +#define ZXDH_PTYPE_L3_IPV6_EXT_UNKNOWN 0x0600 + +/* Outer/Inner L4 type */ +#define ZXDH_PD_L4TYPE_MASK 0x00f0 +#define ZXDH_PTYPE_L4_TCP 0x0010 +#define ZXDH_PTYPE_L4_UDP 0x0020 +#define ZXDH_PTYPE_L4_FRAG 0x0030 +#define ZXDH_PTYPE_L4_SCTP 0x0040 +#define ZXDH_PTYPE_L4_ICMP 0x0050 +#define ZXDH_PTYPE_L4_NONFRAG 0x0060 +#define ZXDH_PTYPE_L4_IGMP 0x0070 + #define ZXDH_TX_MAX_SEGS 31 #define ZXDH_RX_MAX_SEGS 31 +#define ZXDH_PI_LRO_FALG 0x00000001 + +#define ZXDH_MIN_MSS 64 +#define ZXDH_VLAN_ID_MASK 0xfff + +#define ZXDH_MTU_MSS_UNIT_SHIFTBIT 2 +#define ZXDH_MTU_MSS_MASK 0xFFF +#define ZXDH_PD_HDR_SIZE_MAX 256 + +/* error code */ +#define ZXDH_UDP_CSUM_ERR 0x0020 +#define ZXDH_TCP_CSUM_ERR 0x0040 +#define ZXDH_IPV4_CSUM_ERR 0x0100 + +#define ZXDH_DTPOFFLOAD_MASK ( \ + RTE_MBUF_F_TX_IP_CKSUM | \ + RTE_MBUF_F_TX_L4_MASK | \ + RTE_MBUF_F_TX_TCP_SEG | \ + RTE_MBUF_F_TX_SEC_OFFLOAD | \ + RTE_MBUF_F_TX_UDP_SEG) + uint32_t zxdh_outer_l2_type[16] = { 0, RTE_PTYPE_L2_ETHER, @@ -161,259 +236,196 @@ zxdh_xmit_cleanup_inorder_packed(struct zxdh_virtqueue *vq, int32_t num) vq->vq_free_cnt += free_cnt; } -static void -zxdh_ring_free_id_packed(struct zxdh_virtqueue *vq, uint16_t id) +static inline uint16_t +zxdh_get_mtu(struct zxdh_virtqueue *vq) { - struct zxdh_vq_desc_extra *dxp = NULL; - - dxp = &vq->vq_descx[id]; - vq->vq_free_cnt += dxp->ndescs; - - if (vq->vq_desc_tail_idx == ZXDH_VQ_RING_DESC_CHAIN_END) - vq->vq_desc_head_idx = id; - else - vq->vq_descx[vq->vq_desc_tail_idx].next = id; + struct rte_eth_dev *eth_dev = vq->hw->eth_dev; - vq->vq_desc_tail_idx = id; - dxp->next = ZXDH_VQ_RING_DESC_CHAIN_END; -} - -static void -zxdh_xmit_cleanup_normal_packed(struct zxdh_virtqueue *vq, int32_t num) -{ - uint16_t used_idx = 0; - uint16_t id = 0; - uint16_t size = vq->vq_nentries; - struct zxdh_vring_packed_desc *desc = vq->vq_packed.ring.desc; - struct zxdh_vq_desc_extra *dxp = NULL; - - used_idx = vq->vq_used_cons_idx; - /* desc_is_used has a load-acquire or rte_io_rmb inside - * and wait for used desc in virtqueue. - */ - while (num-- && zxdh_desc_used(&desc[used_idx], vq)) { - id = desc[used_idx].id; - dxp = &vq->vq_descx[id]; - vq->vq_used_cons_idx += dxp->ndescs; - if (vq->vq_used_cons_idx >= size) { - vq->vq_used_cons_idx -= size; - vq->vq_packed.used_wrap_counter ^= 1; - } - zxdh_ring_free_id_packed(vq, id); - if (dxp->cookie != NULL) { - rte_pktmbuf_free(dxp->cookie); - dxp->cookie = NULL; - } - used_idx = vq->vq_used_cons_idx; - } + return eth_dev->data->mtu; } static void -zxdh_xmit_cleanup_packed(struct zxdh_virtqueue *vq, int32_t num, int32_t in_order) -{ - if (in_order) - zxdh_xmit_cleanup_inorder_packed(vq, num); - else - zxdh_xmit_cleanup_normal_packed(vq, num); -} - -static uint8_t -zxdh_xmit_get_ptype(struct rte_mbuf *m) -{ - uint8_t pcode = ZXDH_PCODE_NO_IP_PKT_TYPE; - uint8_t l3_ptype = ZXDH_PI_L3TYPE_NOIP; - - if ((m->packet_type & RTE_PTYPE_INNER_L3_MASK) == RTE_PTYPE_INNER_L3_IPV4 || - ((!(m->packet_type & RTE_PTYPE_TUNNEL_MASK)) && - (m->packet_type & RTE_PTYPE_L3_MASK) == RTE_PTYPE_L3_IPV4)) { - l3_ptype = ZXDH_PI_L3TYPE_IP; - pcode = ZXDH_PCODE_IP_PKT_TYPE; - } else if ((m->packet_type & RTE_PTYPE_INNER_L3_MASK) == RTE_PTYPE_INNER_L3_IPV6 || - ((!(m->packet_type & RTE_PTYPE_TUNNEL_MASK)) && - (m->packet_type & RTE_PTYPE_L3_MASK) == RTE_PTYPE_L3_IPV6)) { - l3_ptype = ZXDH_PI_L3TYPE_IPV6; - pcode = ZXDH_PCODE_IP_PKT_TYPE; - } else { - goto end; - } - - if ((m->packet_type & RTE_PTYPE_INNER_L4_MASK) == RTE_PTYPE_INNER_L4_TCP || - ((!(m->packet_type & RTE_PTYPE_TUNNEL_MASK)) && - (m->packet_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP)) - pcode = ZXDH_PCODE_TCP_PKT_TYPE; - else if ((m->packet_type & RTE_PTYPE_INNER_L4_MASK) == RTE_PTYPE_INNER_L4_UDP || - ((!(m->packet_type & RTE_PTYPE_TUNNEL_MASK)) && - (m->packet_type & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_UDP)) - pcode = ZXDH_PCODE_UDP_PKT_TYPE; - -end: - return l3_ptype | ZXDH_PKT_FORM_CPU | pcode; -} - -static void zxdh_xmit_fill_net_hdr(struct rte_mbuf *cookie, +zxdh_xmit_fill_net_hdr(struct zxdh_virtqueue *vq, struct rte_mbuf *cookie, struct zxdh_net_hdr_dl *hdr) { + uint16_t mtu_or_mss = 0; uint16_t pkt_flag_lw16 = ZXDH_NO_IPID_UPDATE; uint16_t l3_offset; + uint8_t pcode = ZXDH_PCODE_NO_IP_PKT_TYPE; + uint8_t l3_ptype = ZXDH_PI_L3TYPE_NOIP; + struct zxdh_pi_hdr *pi_hdr = NULL; + struct zxdh_pd_hdr_dl *pd_hdr = NULL; + struct zxdh_hw *hw = vq->hw; + struct zxdh_net_hdr_dl *net_hdr_dl = &g_net_hdr_dl[hw->port_id]; + uint8_t hdr_len = hw->dl_net_hdr_len; uint32_t ol_flag = 0; - hdr->pi_hdr.pkt_flag_lw16 = rte_be_to_cpu_16(pkt_flag_lw16); + rte_memcpy(hdr, net_hdr_dl, hdr_len); + if (hw->has_tx_offload) { + pi_hdr = &hdr->pipd_hdr_dl.pi_hdr; + pd_hdr = &hdr->pipd_hdr_dl.pd_hdr; - hdr->pi_hdr.pkt_type = zxdh_xmit_get_ptype(cookie); - l3_offset = ZXDH_DL_NET_HDR_SIZE + cookie->outer_l2_len + - cookie->outer_l3_len + cookie->l2_len; - hdr->pi_hdr.l3_offset = rte_be_to_cpu_16(l3_offset); - hdr->pi_hdr.l4_offset = rte_be_to_cpu_16(l3_offset + cookie->l3_len); + pcode = ZXDH_PCODE_IP_PKT_TYPE; + if (cookie->ol_flags & RTE_MBUF_F_TX_IPV6) + l3_ptype = ZXDH_PI_L3TYPE_IPV6; + else if (cookie->ol_flags & RTE_MBUF_F_TX_IPV4) + l3_ptype = ZXDH_PI_L3TYPE_IP; + else + pcode = ZXDH_PCODE_NO_IP_PKT_TYPE; + + if (cookie->ol_flags & RTE_MBUF_F_TX_TCP_SEG) { + mtu_or_mss = (cookie->tso_segsz >= ZXDH_MIN_MSS) + ? cookie->tso_segsz + : ZXDH_MIN_MSS; + pi_hdr->pkt_flag_hi8 |= ZXDH_TX_TCPUDP_CKSUM_CAL; + pkt_flag_lw16 |= ZXDH_NO_IP_FRAGMENT | ZXDH_TX_IP_CKSUM_CAL; + pcode = ZXDH_PCODE_TCP_PKT_TYPE; + } else if (cookie->ol_flags & RTE_MBUF_F_TX_UDP_SEG) { + mtu_or_mss = zxdh_get_mtu(vq); + mtu_or_mss = (mtu_or_mss >= ZXDH_MIN_MSS) ? mtu_or_mss : ZXDH_MIN_MSS; + pkt_flag_lw16 |= ZXDH_TX_IP_CKSUM_CAL; + pi_hdr->pkt_flag_hi8 |= ZXDH_NO_TCP_FRAGMENT | ZXDH_TX_TCPUDP_CKSUM_CAL; + pcode = ZXDH_PCODE_UDP_PKT_TYPE; + } else { + pkt_flag_lw16 |= ZXDH_NO_IP_FRAGMENT; + pi_hdr->pkt_flag_hi8 |= ZXDH_NO_TCP_FRAGMENT; + } - if (cookie->ol_flags & RTE_MBUF_F_TX_VLAN) { - ol_flag |= ZXDH_PD_OFFLOAD_CVLAN_INSERT; - hdr->pi_hdr.vlan_id = rte_be_to_cpu_16(cookie->vlan_tci); - hdr->pd_hdr.cvlan_insert = - rte_be_to_cpu_32((ZXDH_CVLAN_TPID << 16) | cookie->vlan_tci); + if (cookie->ol_flags & RTE_MBUF_F_TX_IP_CKSUM) + pkt_flag_lw16 |= ZXDH_TX_IP_CKSUM_CAL; + + if ((cookie->ol_flags & RTE_MBUF_F_TX_UDP_CKSUM) == + RTE_MBUF_F_TX_UDP_CKSUM){ + pcode = ZXDH_PCODE_UDP_PKT_TYPE; + pi_hdr->pkt_flag_hi8 |= ZXDH_TX_TCPUDP_CKSUM_CAL; + } else if ((cookie->ol_flags & RTE_MBUF_F_TX_TCP_CKSUM) == + RTE_MBUF_F_TX_TCP_CKSUM) { + pcode = ZXDH_PCODE_TCP_PKT_TYPE; + pi_hdr->pkt_flag_hi8 |= ZXDH_TX_TCPUDP_CKSUM_CAL; + } + + pkt_flag_lw16 |= (mtu_or_mss >> ZXDH_MTU_MSS_UNIT_SHIFTBIT) & ZXDH_MTU_MSS_MASK; + pi_hdr->pkt_flag_lw16 = rte_be_to_cpu_16(pkt_flag_lw16); + pi_hdr->pkt_type = l3_ptype | ZXDH_PKT_FORM_CPU | pcode; + + l3_offset = hdr_len + cookie->l2_len; + l3_offset += (cookie->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) ? + cookie->outer_l2_len + cookie->outer_l3_len : 0; + pi_hdr->l3_offset = rte_be_to_cpu_16(l3_offset); + pi_hdr->l4_offset = rte_be_to_cpu_16(l3_offset + cookie->l3_len); + if (cookie->ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) + ol_flag |= ZXDH_PD_OFFLOAD_OUTER_IPCSUM; + } else { + pd_hdr = &hdr->pd_hdr; } - if (cookie->ol_flags & RTE_MBUF_F_TX_QINQ) { - ol_flag |= ZXDH_PD_OFFLOAD_SVLAN_INSERT; - hdr->pd_hdr.svlan_insert = - rte_be_to_cpu_32((ZXDH_SVLAN_TPID << 16) | cookie->vlan_tci_outer); + + if (cookie->ol_flags & (RTE_MBUF_F_TX_VLAN | RTE_MBUF_F_TX_QINQ)) { + ol_flag |= ZXDH_PD_OFFLOAD_CVLAN_INSERT; + pd_hdr->cvlan_insert = rte_be_to_cpu_16(cookie->vlan_tci); + if (unlikely(cookie->ol_flags & RTE_MBUF_F_TX_QINQ)) { + ol_flag |= ZXDH_PD_OFFLOAD_SVLAN_INSERT; + pd_hdr->svlan_insert = rte_be_to_cpu_16(cookie->vlan_tci_outer); + } } - hdr->pd_hdr.ol_flag = rte_be_to_cpu_32(ol_flag); + pd_hdr->ol_flag = rte_be_to_cpu_16(ol_flag); } -static inline void zxdh_enqueue_xmit_packed_fast(struct zxdh_virtnet_tx *txvq, - struct rte_mbuf *cookie, int32_t in_order) +static inline void +zxdh_enqueue_xmit_packed_fast(struct zxdh_virtnet_tx *txvq, + struct rte_mbuf *cookie) { struct zxdh_virtqueue *vq = txvq->vq; - uint16_t id = in_order ? vq->vq_avail_idx : vq->vq_desc_head_idx; + uint16_t id = vq->vq_avail_idx; struct zxdh_vq_desc_extra *dxp = &vq->vq_descx[id]; uint16_t flags = vq->vq_packed.cached_flags; struct zxdh_net_hdr_dl *hdr = NULL; + uint8_t hdr_len = vq->hw->dl_net_hdr_len; + struct zxdh_vring_packed_desc *dp = &vq->vq_packed.ring.desc[id]; dxp->ndescs = 1; dxp->cookie = cookie; - hdr = rte_pktmbuf_mtod_offset(cookie, struct zxdh_net_hdr_dl *, -ZXDH_DL_NET_HDR_SIZE); - zxdh_xmit_fill_net_hdr(cookie, hdr); - - uint16_t idx = vq->vq_avail_idx; - struct zxdh_vring_packed_desc *dp = &vq->vq_packed.ring.desc[idx]; + hdr = rte_pktmbuf_mtod_offset(cookie, struct zxdh_net_hdr_dl *, -hdr_len); + zxdh_xmit_fill_net_hdr(vq, cookie, hdr); - dp->addr = rte_pktmbuf_iova(cookie) - ZXDH_DL_NET_HDR_SIZE; - dp->len = cookie->data_len + ZXDH_DL_NET_HDR_SIZE; + dp->addr = rte_pktmbuf_iova(cookie) - hdr_len; + dp->len = cookie->data_len + hdr_len; dp->id = id; if (++vq->vq_avail_idx >= vq->vq_nentries) { vq->vq_avail_idx -= vq->vq_nentries; vq->vq_packed.cached_flags ^= ZXDH_VRING_PACKED_DESC_F_AVAIL_USED; } vq->vq_free_cnt--; - if (!in_order) { - vq->vq_desc_head_idx = dxp->next; - if (vq->vq_desc_head_idx == ZXDH_VQ_RING_DESC_CHAIN_END) - vq->vq_desc_tail_idx = ZXDH_VQ_RING_DESC_CHAIN_END; - } - zxdh_queue_store_flags_packed(dp, flags, vq->hw->weak_barriers); + zxdh_queue_store_flags_packed(dp, flags); } -static inline void zxdh_enqueue_xmit_packed(struct zxdh_virtnet_tx *txvq, +static inline void +zxdh_enqueue_xmit_packed(struct zxdh_virtnet_tx *txvq, struct rte_mbuf *cookie, - uint16_t needed, - int32_t use_indirect, - int32_t in_order) + uint16_t needed) { struct zxdh_tx_region *txr = txvq->zxdh_net_hdr_mz->addr; struct zxdh_virtqueue *vq = txvq->vq; - struct zxdh_vring_packed_desc *start_dp = vq->vq_packed.ring.desc; - void *hdr = NULL; + uint16_t id = vq->vq_avail_idx; + struct zxdh_vq_desc_extra *dxp = &vq->vq_descx[id]; uint16_t head_idx = vq->vq_avail_idx; uint16_t idx = head_idx; - uint16_t prev = head_idx; - uint16_t head_flags = cookie->next ? ZXDH_VRING_DESC_F_NEXT : 0; - uint16_t seg_num = cookie->nb_segs; - uint16_t id = in_order ? vq->vq_avail_idx : vq->vq_desc_head_idx; + struct zxdh_vring_packed_desc *start_dp = vq->vq_packed.ring.desc; struct zxdh_vring_packed_desc *head_dp = &vq->vq_packed.ring.desc[idx]; - struct zxdh_vq_desc_extra *dxp = &vq->vq_descx[id]; + struct zxdh_net_hdr_dl *hdr = NULL; + + uint16_t head_flags = cookie->next ? ZXDH_VRING_DESC_F_NEXT : 0; + uint8_t hdr_len = vq->hw->dl_net_hdr_len; dxp->ndescs = needed; dxp->cookie = cookie; head_flags |= vq->vq_packed.cached_flags; - /* if offload disabled, it is not zeroed below, do it now */ - if (use_indirect) { - /** - * setup tx ring slot to point to indirect - * descriptor list stored in reserved region. - * the first slot in indirect ring is already - * preset to point to the header in reserved region - **/ - start_dp[idx].addr = - txvq->zxdh_net_hdr_mem + RTE_PTR_DIFF(&txr[idx].tx_packed_indir, txr); - start_dp[idx].len = (seg_num + 1) * sizeof(struct zxdh_vring_packed_desc); - /* Packed descriptor id needs to be restored when inorder. */ - if (in_order) - start_dp[idx].id = idx; - - /* reset flags for indirect desc */ - head_flags = ZXDH_VRING_DESC_F_INDIRECT; - head_flags |= vq->vq_packed.cached_flags; - hdr = (void *)&txr[idx].tx_hdr; - /* loop below will fill in rest of the indirect elements */ - start_dp = txr[idx].tx_packed_indir; - start_dp->len = ZXDH_DL_NET_HDR_SIZE; /* update actual net or type hdr size */ - idx = 1; - } else { - /* setup first tx ring slot to point to header stored in reserved region. */ - start_dp[idx].addr = txvq->zxdh_net_hdr_mem + RTE_PTR_DIFF(&txr[idx].tx_hdr, txr); - start_dp[idx].len = ZXDH_DL_NET_HDR_SIZE; - head_flags |= ZXDH_VRING_DESC_F_NEXT; - hdr = (void *)&txr[idx].tx_hdr; - idx++; - if (idx >= vq->vq_nentries) { - idx -= vq->vq_nentries; - vq->vq_packed.cached_flags ^= ZXDH_VRING_PACKED_DESC_F_AVAIL_USED; - } + start_dp[idx].addr = txvq->zxdh_net_hdr_mem + RTE_PTR_DIFF(&txr[idx].tx_hdr, txr); + start_dp[idx].len = hdr_len; + head_flags |= ZXDH_VRING_DESC_F_NEXT; + hdr = (void *)&txr[idx].tx_hdr; + + rte_prefetch1(hdr); + idx++; + if (idx >= vq->vq_nentries) { + idx -= vq->vq_nentries; + vq->vq_packed.cached_flags ^= ZXDH_VRING_PACKED_DESC_F_AVAIL_USED; } - zxdh_xmit_fill_net_hdr(cookie, (struct zxdh_net_hdr_dl *)hdr); + + zxdh_xmit_fill_net_hdr(vq, cookie, hdr); do { start_dp[idx].addr = rte_pktmbuf_iova(cookie); start_dp[idx].len = cookie->data_len; + start_dp[idx].id = id; if (likely(idx != head_idx)) { uint16_t flags = cookie->next ? ZXDH_VRING_DESC_F_NEXT : 0; + flags |= vq->vq_packed.cached_flags; start_dp[idx].flags = flags; } - prev = idx; + idx++; if (idx >= vq->vq_nentries) { idx -= vq->vq_nentries; vq->vq_packed.cached_flags ^= ZXDH_VRING_PACKED_DESC_F_AVAIL_USED; } } while ((cookie = cookie->next) != NULL); - start_dp[prev].id = id; - if (use_indirect) { - idx = head_idx; - if (++idx >= vq->vq_nentries) { - idx -= vq->vq_nentries; - vq->vq_packed.cached_flags ^= ZXDH_VRING_PACKED_DESC_F_AVAIL_USED; - } - } + vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed); vq->vq_avail_idx = idx; - if (!in_order) { - vq->vq_desc_head_idx = dxp->next; - if (vq->vq_desc_head_idx == ZXDH_VQ_RING_DESC_CHAIN_END) - vq->vq_desc_tail_idx = ZXDH_VQ_RING_DESC_CHAIN_END; - } - zxdh_queue_store_flags_packed(head_dp, head_flags, vq->hw->weak_barriers); + + zxdh_queue_store_flags_packed(head_dp, head_flags); } static void zxdh_update_packet_stats(struct zxdh_virtnet_stats *stats, struct rte_mbuf *mbuf) { uint32_t s = mbuf->pkt_len; - struct rte_ether_addr *ea = NULL; stats->bytes += s; - + #ifdef QUEUE_XSTAT if (s == 64) { stats->size_bins[1]++; } else if (s > 64 && s < 1024) { @@ -438,6 +450,45 @@ zxdh_update_packet_stats(struct zxdh_virtnet_stats *stats, struct rte_mbuf *mbuf else stats->multicast++; } + #endif +} + +static void +zxdh_xmit_flush(struct zxdh_virtqueue *vq) +{ + uint16_t id = 0; + uint16_t curr_id = 0; + uint16_t free_cnt = 0; + uint16_t size = vq->vq_nentries; + struct zxdh_vring_packed_desc *desc = vq->vq_packed.ring.desc; + struct zxdh_vq_desc_extra *dxp = NULL; + uint16_t used_idx = vq->vq_used_cons_idx; + + /* + * The function desc_is_used performs a load-acquire operation + * or calls rte_io_rmb to ensure memory consistency. It waits + * for a used descriptor in the virtqueue. + */ + while (desc_is_used(&desc[used_idx], vq)) { + id = desc[used_idx].id; + do { + curr_id = used_idx; + dxp = &vq->vq_descx[used_idx]; + used_idx += dxp->ndescs; + free_cnt += dxp->ndescs; + //num -= dxp->ndescs; + if (used_idx >= size) { + used_idx -= size; + vq->vq_packed.used_wrap_counter ^= 1; + } + if (dxp->cookie != NULL) { + rte_pktmbuf_free(dxp->cookie); + dxp->cookie = NULL; + } + } while (curr_id != id); + } + vq->vq_used_cons_idx = used_idx; + vq->vq_free_cnt += free_cnt; } uint16_t @@ -445,33 +496,23 @@ zxdh_xmit_pkts_packed(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkt { struct zxdh_virtnet_tx *txvq = tx_queue; struct zxdh_virtqueue *vq = txvq->vq; - struct zxdh_hw *hw = vq->hw; uint16_t nb_tx = 0; - bool in_order = zxdh_pci_with_feature(hw, ZXDH_F_IN_ORDER); + zxdh_xmit_flush(vq); - if (nb_pkts > vq->vq_free_cnt) - zxdh_xmit_cleanup_packed(vq, nb_pkts - vq->vq_free_cnt, in_order); for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { struct rte_mbuf *txm = tx_pkts[nb_tx]; int32_t can_push = 0; - int32_t use_indirect = 0; int32_t slots = 0; int32_t need = 0; + rte_prefetch0(txm); /* optimize ring usage */ - if ((zxdh_pci_with_feature(hw, ZXDH_F_ANY_LAYOUT) || - zxdh_pci_with_feature(hw, ZXDH_F_VERSION_1)) && - rte_mbuf_refcnt_read(txm) == 1 && + if (rte_mbuf_refcnt_read(txm) == 1 && RTE_MBUF_DIRECT(txm) && txm->nb_segs == 1 && - rte_pktmbuf_headroom(txm) >= ZXDH_DL_NET_HDR_SIZE && - rte_is_aligned(rte_pktmbuf_mtod(txm, char *), - alignof(struct zxdh_net_hdr_dl))) { + txm->data_off >= ZXDH_DL_NET_HDR_SIZE) { can_push = 1; - } else if (zxdh_pci_with_feature(hw, ZXDH_RING_F_INDIRECT_DESC) && - txm->nb_segs < ZXDH_MAX_TX_INDIRECT) { - use_indirect = 1; } /** * How many main ring entries are needed to this Tx? @@ -479,46 +520,50 @@ zxdh_xmit_pkts_packed(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkt * any_layout => number of segments * default => number of segments + 1 **/ - slots = use_indirect ? 1 : (txm->nb_segs + !can_push); + slots = txm->nb_segs + !can_push; need = slots - vq->vq_free_cnt; /* Positive value indicates it need free vring descriptors */ if (unlikely(need > 0)) { - zxdh_xmit_cleanup_packed(vq, need, in_order); + zxdh_xmit_cleanup_inorder_packed(vq, need); need = slots - vq->vq_free_cnt; if (unlikely(need > 0)) { - PMD_TX_LOG(ERR, "port[ep:%d, pf:%d, vf:%d, vfid:%d, pcieid:%d], queue:%d[pch:%d]. No free desc to xmit", - hw->vport.epid, hw->vport.pfid, hw->vport.vfid, - hw->vfid, hw->pcie_id, txvq->queue_id, - hw->channel_context[txvq->queue_id].ph_chno); + PMD_TX_LOG(ERR, + " No enough %d free tx descriptors to transmit." + "freecnt %d", + need, + vq->vq_free_cnt); break; } } - if (txm->nb_segs > ZXDH_TX_MAX_SEGS) { - PMD_TX_LOG(ERR, "%d segs dropped", txm->nb_segs); - txvq->stats.truncated_err += nb_pkts - nb_tx; - break; - } + /* Enqueue Packet buffers */ if (can_push) - zxdh_enqueue_xmit_packed_fast(txvq, txm, in_order); + zxdh_enqueue_xmit_packed_fast(txvq, txm); else - zxdh_enqueue_xmit_packed(txvq, txm, slots, use_indirect, in_order); + zxdh_enqueue_xmit_packed(txvq, txm, slots); zxdh_update_packet_stats(&txvq->stats, txm); } txvq->stats.packets += nb_tx; - if (likely(nb_tx)) { - if (unlikely(zxdh_queue_kick_prepare_packed(vq))) { - zxdh_queue_notify(vq); - PMD_TX_LOG(DEBUG, "Notified backend after xmit"); - } - } + if (likely(nb_tx)) + zxdh_queue_notify(vq); return nb_tx; } +static inline int dl_net_hdr_check(struct rte_mbuf *m, struct zxdh_hw *hw) +{ + if ((m->ol_flags & ZXDH_DTPOFFLOAD_MASK) && !hw->has_tx_offload) { + PMD_TX_LOG(ERR, "port:[%d], vfid[%d]. " + "not support tx_offload", hw->port_id, hw->vfid); + return -EINVAL; + } + return 0; +} + uint16_t zxdh_xmit_pkts_prepare(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { struct zxdh_virtnet_tx *txvq = tx_queue; + struct zxdh_hw *hw = txvq->vq->hw; uint16_t nb_tx; for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { @@ -544,11 +589,20 @@ uint16_t zxdh_xmit_pkts_prepare(void *tx_queue, struct rte_mbuf **tx_pkts, rte_errno = ENOMEM; break; } + + error = dl_net_hdr_check(m, hw); + if (unlikely(error)) { + rte_errno = ENOTSUP; + txvq->stats.errors += nb_pkts - nb_tx; + txvq->stats.offload_cfg_err += nb_pkts - nb_tx; + break; + } } return nb_tx; } -static uint16_t zxdh_dequeue_burst_rx_packed(struct zxdh_virtqueue *vq, +static uint16_t +zxdh_dequeue_burst_rx_packed(struct zxdh_virtqueue *vq, struct rte_mbuf **rx_pkts, uint32_t *len, uint16_t num) @@ -575,6 +629,8 @@ static uint16_t zxdh_dequeue_burst_rx_packed(struct zxdh_virtqueue *vq, "vring descriptor with no mbuf cookie at %u", vq->vq_used_cons_idx); break; } + rte_prefetch0(cookie); + rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *)); rx_pkts[i] = cookie; vq->vq_free_cnt++; vq->vq_used_cons_idx++; @@ -586,15 +642,107 @@ static uint16_t zxdh_dequeue_burst_rx_packed(struct zxdh_virtqueue *vq, return i; } -static int32_t zxdh_rx_update_mbuf(struct rte_mbuf *m, struct zxdh_net_hdr_ul *hdr) +static inline void +zxdh_rx_update_mbuf(struct rte_mbuf *m, struct zxdh_net_hdr_ul *hdr) { - struct zxdh_pd_hdr_ul *pd_hdr = &hdr->pd_hdr; - struct zxdh_pi_hdr *pi_hdr = &hdr->pi_hdr; + uint8_t has_pi = (uint64_t)(hdr->type_hdr.pd_len << 1) > ZXDH_UL_NOPI_HDR_SIZE_MAX; + struct zxdh_pd_hdr_ul *pd_hdr = has_pi ? &hdr->pipd_hdr_ul.pd_hdr : &hdr->pd_hdr; + uint32_t pkt_flag = ntohl(pd_hdr->pkt_flag); uint32_t idx = 0; + uint32_t striped_vlan_tci = rte_be_to_cpu_32(pd_hdr->striped_vlan_tci); + uint16_t pkt_type_outer = rte_be_to_cpu_16(pd_hdr->pkt_type_out); + uint16_t pkt_type_inner = rte_be_to_cpu_16(pd_hdr->pkt_type_in); - m->pkt_len = rte_be_to_cpu_16(pi_hdr->ul.pkt_len); + if (unlikely(pkt_flag & (ZXDH_MBUF_F_RX_1588_PTP | ZXDH_MBUF_F_RX_1588_TMST))) { + if (pkt_flag & ZXDH_MBUF_F_RX_1588_PTP) + m->ol_flags |= RTE_MBUF_F_RX_IEEE1588_PTP; + if (pkt_flag & ZXDH_MBUF_F_RX_1588_TMST) + m->ol_flags |= RTE_MBUF_F_RX_IEEE1588_TMST; + } - uint16_t pkt_type_outer = rte_be_to_cpu_16(pd_hdr->pkt_type_out); + if (pkt_flag & ZXDH_MBUF_F_RX_VLAN_STRIPPED) { + m->ol_flags |= (RTE_MBUF_F_RX_VLAN_STRIPPED | RTE_MBUF_F_RX_VLAN); + m->vlan_tci = (unlikely(pkt_flag & ZXDH_MBUF_F_RX_QINQ)) + ? (striped_vlan_tci >> 16) & ZXDH_VLAN_ID_MASK + : striped_vlan_tci & ZXDH_VLAN_ID_MASK; + } + + if (unlikely(pkt_flag & ZXDH_MBUF_F_RX_QINQ_STRIPPED)) { + /* + * When PKT_RX_QINQ_STRIPPED is set and PKT_RX_VLAN_STRIPPED is unset: + * - Only the outer VLAN is removed from the packet data. + * - Both TCI values are saved: the inner TCI in mbuf->vlan_tci and + * the outer TCI in mbuf->vlan_tci_outer. + * + * When PKT_RX_QINQ is set, PKT_RX_VLAN must also be set, and the inner + * TCI is saved in mbuf->vlan_tci. + */ + m->ol_flags |= (RTE_MBUF_F_RX_QINQ_STRIPPED | RTE_MBUF_F_RX_QINQ); + m->ol_flags |= (RTE_MBUF_F_RX_VLAN_STRIPPED | RTE_MBUF_F_RX_VLAN); + m->vlan_tci = striped_vlan_tci & ZXDH_VLAN_ID_MASK; + m->vlan_tci_outer = (striped_vlan_tci >> 16) & ZXDH_VLAN_ID_MASK; + } + + /* rss hash/fd handle */ + if (pkt_flag & ZXDH_MBUF_F_RX_RSS_HASH) { + m->hash.rss = rte_be_to_cpu_32(pd_hdr->rss_hash); + m->ol_flags |= RTE_MBUF_F_RX_RSS_HASH; + } + if (pkt_flag & ZXDH_MBUF_F_RX_FDIR) { + m->ol_flags |= RTE_MBUF_F_RX_FDIR; + if (pkt_flag & ZXDH_MBUF_F_RX_FDIR_ID) { + m->hash.fdir.hi = rte_be_to_cpu_32(pd_hdr->fd); + m->ol_flags |= RTE_MBUF_F_RX_FDIR_ID; + } else if ((pkt_flag & ZXDH_MBUF_F_RX_FDIR_FLX_MASK) == FELX_4BYTE) { + m->hash.fdir.hi = rte_be_to_cpu_32(pd_hdr->fd); + m->ol_flags |= RTE_MBUF_F_RX_FDIR_FLX; + } else if (((pkt_flag & ZXDH_MBUF_F_RX_FDIR_FLX_MASK) == FELX_8BYTE)) { + m->hash.fdir.hi = rte_be_to_cpu_32(pd_hdr->rss_hash); + m->hash.fdir.lo = rte_be_to_cpu_32(pd_hdr->fd); + m->ol_flags |= RTE_MBUF_F_RX_FDIR_FLX; + } + } + /* checksum handle */ + if (pkt_flag & ZXDH_MBUF_F_RX_OUTER_IP_CKSUM_BAD) + m->ol_flags |= RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD; + if (pkt_flag & ZXDH_MBUF_F_RX_OUTER_L4_CKSUM_GOOD) + m->ol_flags |= RTE_MBUF_F_RX_OUTER_L4_CKSUM_GOOD; + + if (has_pi) { + struct zxdh_pi_hdr *pi_hdr = &hdr->pipd_hdr_ul.pi_hdr; + uint16_t pkt_type_masked = pi_hdr->pkt_type & ZXDH_PCODE_MASK; + uint16_t err_code = rte_be_to_cpu_16(pi_hdr->ul.err_code); + + bool is_ip_pkt = + (pi_hdr->pkt_type == ZXDH_PCODE_IP_PKT_TYPE) || + ((pi_hdr->pkt_type & ZXDH_PI_L3TYPE_MASK) == ZXDH_PI_L3TYPE_IP); + + bool is_l4_pkt = + (pkt_type_masked == ZXDH_PCODE_UDP_PKT_TYPE) || + (pkt_type_masked == ZXDH_PCODE_NO_REASSMBLE_TCP_PKT_TYPE) || + (pkt_type_masked == ZXDH_PCODE_TCP_PKT_TYPE); + + if (is_ip_pkt && (pi_hdr->pkt_flag_hi8 & ZXDH_RX_IP_CKSUM_VERIFY)) { + if (err_code & ZXDH_IPV4_CSUM_ERR) + m->ol_flags |= RTE_MBUF_F_RX_IP_CKSUM_BAD; + else + m->ol_flags |= RTE_MBUF_F_RX_IP_CKSUM_GOOD; + } + + if (is_l4_pkt && (pi_hdr->pkt_flag_hi8 & ZXDH_RX_TCPUDP_CKSUM_VERIFY)) { + if (err_code & (ZXDH_TCP_CSUM_ERR | ZXDH_UDP_CSUM_ERR)) + m->ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_BAD; + else + m->ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_GOOD; + } + + if (ntohl(pi_hdr->ul.lro_flag) & ZXDH_PI_LRO_FALG) + m->ol_flags |= RTE_MBUF_F_RX_LRO; + + m->pkt_len = rte_be_to_cpu_16(pi_hdr->ul.pkt_len); + } else { + m->pkt_len = rte_be_to_cpu_16(pd_hdr->pkt_len); + } idx = (pkt_type_outer >> 12) & 0xF; m->packet_type = zxdh_outer_l2_type[idx]; @@ -605,8 +753,6 @@ static int32_t zxdh_rx_update_mbuf(struct rte_mbuf *m, struct zxdh_net_hdr_ul *h idx = pkt_type_outer & 0xF; m->packet_type |= zxdh_tunnel_type[idx]; - uint16_t pkt_type_inner = rte_be_to_cpu_16(pd_hdr->pkt_type_in); - if (pkt_type_inner) { idx = (pkt_type_inner >> 12) & 0xF; m->packet_type |= zxdh_inner_l2_type[idx]; @@ -616,7 +762,6 @@ static int32_t zxdh_rx_update_mbuf(struct rte_mbuf *m, struct zxdh_net_hdr_ul *h m->packet_type |= zxdh_inner_l4_type[idx]; } - return 0; } static void zxdh_discard_rxbuf(struct zxdh_virtqueue *vq, struct rte_mbuf *m) @@ -633,92 +778,67 @@ static void zxdh_discard_rxbuf(struct zxdh_virtqueue *vq, struct rte_mbuf *m) } } -uint16_t zxdh_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts, +uint16_t +zxdh_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) { struct zxdh_virtnet_rx *rxvq = rx_queue; struct zxdh_virtqueue *vq = rxvq->vq; struct zxdh_hw *hw = vq->hw; - struct rte_eth_dev *dev = hw->eth_dev; struct rte_mbuf *rxm = NULL; struct rte_mbuf *prev = NULL; - uint32_t len[ZXDH_MBUF_BURST_SZ] = {0}; + struct zxdh_net_hdr_ul *header = NULL; + uint32_t lens[ZXDH_MBUF_BURST_SZ] = {0}; struct rte_mbuf *rcv_pkts[ZXDH_MBUF_BURST_SZ] = {NULL}; - uint32_t nb_enqueued = 0; + uint16_t len = 0; uint32_t seg_num = 0; uint32_t seg_res = 0; + uint32_t error = 0; uint16_t hdr_size = 0; - int32_t error = 0; uint16_t nb_rx = 0; + uint16_t i; + uint16_t rcvd_pkt_len = 0; uint16_t num = nb_pkts; if (unlikely(num > ZXDH_MBUF_BURST_SZ)) num = ZXDH_MBUF_BURST_SZ; - num = zxdh_dequeue_burst_rx_packed(vq, rcv_pkts, len, num); - uint16_t i; - uint16_t rcvd_pkt_len = 0; + num = zxdh_dequeue_burst_rx_packed(vq, rcv_pkts, lens, num); + if (num == 0) { + rxvq->stats.idle++; + goto refill; + } for (i = 0; i < num; i++) { rxm = rcv_pkts[i]; - if (unlikely(len[i] < ZXDH_UL_NET_HDR_SIZE)) { - nb_enqueued++; - PMD_RX_LOG(ERR, "RX, len:%u err", len[i]); - zxdh_discard_rxbuf(vq, rxm); - rxvq->stats.errors++; - continue; - } - struct zxdh_net_hdr_ul *header = - (struct zxdh_net_hdr_ul *)((char *)rxm->buf_addr + - RTE_PKTMBUF_HEADROOM); + rx_pkts[nb_rx] = rxm; + prev = rxm; + len = lens[i]; + header = rte_pktmbuf_mtod(rxm, struct zxdh_net_hdr_ul *); seg_num = header->type_hdr.num_buffers; - if (seg_num == 0) { - PMD_RX_LOG(ERR, "dequeue %d pkt, No.%d pkt seg_num is %d", num, i, seg_num); - seg_num = 1; - } - if (seg_num > ZXDH_RX_MAX_SEGS) { - PMD_RX_LOG(ERR, "dequeue %d pkt, No.%d pkt seg_num is %d", num, i, seg_num); - nb_enqueued++; - zxdh_discard_rxbuf(vq, rxm); - rxvq->stats.errors++; - continue; - } - /* bit[0:6]-pd_len unit:2B */ - uint16_t pd_len = header->type_hdr.pd_len << 1; - if (pd_len > ZXDH_PD_HDR_SIZE_MAX || pd_len < ZXDH_PD_HDR_SIZE_MIN) { - PMD_RX_LOG(ERR, "pd_len:%d is invalid", pd_len); - nb_enqueued++; - zxdh_discard_rxbuf(vq, rxm); - rxvq->stats.errors++; - continue; - } + /* Private queue only handle type hdr */ - hdr_size = pd_len; - rxm->data_off = RTE_PKTMBUF_HEADROOM + hdr_size; + hdr_size = ZXDH_TYPE_HDR_SIZE; + rxm->pkt_len = ((header->type_hdr.port & 0x7f) << 8) + + header->type_hdr.pd_len; + rxm->data_off += hdr_size; rxm->nb_segs = seg_num; rxm->ol_flags = 0; - rxm->vlan_tci = 0; - rcvd_pkt_len = (uint32_t)(len[i] - hdr_size); - rxm->data_len = (uint16_t)(len[i] - hdr_size); + rcvd_pkt_len = len - hdr_size; + rxm->data_len = rcvd_pkt_len; rxm->port = rxvq->port_id; - rx_pkts[nb_rx] = rxm; - prev = rxm; + /* Update rte_mbuf according to pi/pd header */ - if (zxdh_rx_update_mbuf(rxm, header) < 0) { - zxdh_discard_rxbuf(vq, rxm); - rxvq->stats.errors++; - continue; - } + zxdh_rx_update_mbuf(rxm, header); seg_res = seg_num - 1; /* Merge remaining segments */ while (seg_res != 0 && i < (num - 1)) { i++; + len = lens[i]; rxm = rcv_pkts[i]; - rxm->data_off = RTE_PKTMBUF_HEADROOM; - rxm->data_len = (uint16_t)(len[i]); - - rcvd_pkt_len += (uint32_t)(len[i]); + rxm->data_len = len; + rcvd_pkt_len += len; prev->next = rxm; prev = rxm; rxm->next = NULL; @@ -743,27 +863,26 @@ uint16_t zxdh_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res, ZXDH_MBUF_BURST_SZ); uint16_t extra_idx = 0; - rcv_cnt = zxdh_dequeue_burst_rx_packed(vq, rcv_pkts, len, rcv_cnt); + rcv_cnt = zxdh_dequeue_burst_rx_packed(vq, rcv_pkts, lens, rcv_cnt); if (unlikely(rcv_cnt == 0)) { PMD_RX_LOG(ERR, "No enough segments for packet"); rte_pktmbuf_free(rx_pkts[nb_rx]); rxvq->stats.errors++; + rxvq->stats.no_segs_err++; break; } while (extra_idx < rcv_cnt) { rxm = rcv_pkts[extra_idx]; - rxm->data_off = RTE_PKTMBUF_HEADROOM; - rxm->pkt_len = (uint32_t)(len[extra_idx]); - rxm->data_len = (uint16_t)(len[extra_idx]); + rcvd_pkt_len += (uint16_t)(lens[extra_idx]); + rxm->data_len = lens[extra_idx]; prev->next = rxm; prev = rxm; rxm->next = NULL; - rcvd_pkt_len += len[extra_idx]; extra_idx += 1; } seg_res -= rcv_cnt; if (!seg_res) { - if (rcvd_pkt_len != rx_pkts[nb_rx]->pkt_len) { + if (unlikely(rcvd_pkt_len != rx_pkts[nb_rx]->pkt_len)) { PMD_RX_LOG(ERR, "dropped rcvd_pkt_len %d pktlen %d", rcvd_pkt_len, rx_pkts[nb_rx]->pkt_len); zxdh_discard_rxbuf(vq, rx_pkts[nb_rx]); @@ -777,6 +896,7 @@ uint16_t zxdh_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts, } rxvq->stats.packets += nb_rx; +refill: /* Allocate new mbuf for the used descriptor */ if (likely(!zxdh_queue_full(vq))) { /* free_cnt may include mrg descs */ @@ -789,16 +909,14 @@ uint16_t zxdh_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts, for (i = 0; i < free_cnt; i++) rte_pktmbuf_free(new_pkts[i]); } - nb_enqueued += free_cnt; + + if (unlikely(zxdh_queue_kick_prepare_packed(vq))) + zxdh_queue_notify(vq); } else { + struct rte_eth_dev *dev = hw->eth_dev; + dev->data->rx_mbuf_alloc_failed += free_cnt; } } - if (likely(nb_enqueued)) { - if (unlikely(zxdh_queue_kick_prepare_packed(vq))) { - zxdh_queue_notify(vq); - PMD_RX_LOG(DEBUG, "Notified"); - } - } return nb_rx; } diff --git a/drivers/net/zxdh/zxdh_rxtx.h b/drivers/net/zxdh/zxdh_rxtx.h index 79c2a882c3..16ea506517 100644 --- a/drivers/net/zxdh/zxdh_rxtx.h +++ b/drivers/net/zxdh/zxdh_rxtx.h @@ -10,13 +10,40 @@ #include <rte_common.h> #include <rte_mbuf_core.h> +#define ZXDH_PORT_NP 0 +#define ZXDH_PORT_DRS 1 +#define ZXDH_PORT_DTP 2 + +/*PI PKT FLAG */ +#define ZXDH_PKT_FORM_CPU 0x20 +#define ZXDH_NO_IP_FRAGMENT 0x2000 +#define ZXDH_NO_IPID_UPDATE 0x4000 +#define ZXDH_TX_IP_CKSUM_CAL 0x8000 +#define ZXDH_RX_IP_CKSUM_VERIFY 0x01 +#define ZXDH_RX_PSEDUO_CKSUM_VALID 0x02 +#define ZXDH_TX_TCPUDP_CKSUM_CAL 0x04 +#define ZXDH_RX_TCPUDP_CKSUM_VERIFY 0x08 +#define ZXDH_NO_TCP_FRAGMENT 0x10 +#define ZXDH_PI_FLAG 0x20 +#define ZXDH_PI_TYPE 0x40 +#define ZXDH_VERSION1 0x80 +#define ZXDH_PI_TYPE_PI 0x00 +#define ZXDH_PI_TYPE_VIRTIO95 0x40 +#define ZXDH_PI_TYPE_VIRTIO11 0xC0 + struct zxdh_virtnet_stats { uint64_t packets; uint64_t bytes; uint64_t errors; + uint64_t idle; + uint64_t full; + uint64_t norefill; uint64_t multicast; uint64_t broadcast; uint64_t truncated_err; + uint64_t offload_cfg_err; + uint64_t invalid_hdr_len_err; + uint64_t no_segs_err; uint64_t size_bins[8]; }; -- 2.27.0