[dpdk-dev] [PATCH v4 6/8] net/mlx5: introduce Tx burst routine template

[Yongseok Koh]

> On Jul 21, 2019, at 7:24 AM, Viacheslav Ovsiienko <viacheslavo at mellanox.com> wrote:
> 
> Mellanox NICs support the wide set of Tx offloads. The supported
> offloads are reported by the mlx5 PMD in rte_eth_dev_info tx_offload_capa
> field. An application may choose any combination of supported offloads
> and configure the device appropriately. Some of Tx offloads may be
> not requested by application, or ever all of them may be omitted.
> Most of the Tx offloads require some code branches in tx_burst routine
> to support ones. If Tx offload is not requested the tx_burst routine
> code may be significantly simplified and consume less CPU cycles.
> 
> For example, if application does not engage TSO offload this code
> can be omitted, if multi-segment packet is not supposed the tx_burst
> may assume single mbuf packets only, etc.
> 
> Currently, the mlx5 PMD implements multiple tx_burst subroutines
> for most common combinations of requested Tx offloads, each branch
> has its own dedicated implementation. It is not very easy to update,
> support and develop such kind of code - multiple branches impose
> the multiple points to process. Also many of frequently requested
> offload combinations are not supported yet. That leads to selecting of
> not completely matching tx_burst routine and harms the performance.
> 
> This patch introduces the new approach for tx_burst code. It is proposed
> to develop the unified template for tx_burst routine, which supports
> all the Tx offloads and takes the compile time defined parameter
> describing the supposed set of supported offloads. On the base
> of this template, the compiler is able to generate multiple tx_burst
> routines highly optimized for the statically specified set of Tx offloads.
> Next, in runtime, at Tx queue configuration the best matching optimized
> implementation of tx_burst is chosen.
> 
> This patch intentionally omits the template internal implementation,
> but just introduces the template itself to emboss the approach of
> the multiple specially tuned tx_burst routines.
> 
> Signed-off-by: Viacheslav Ovsiienko <viacheslavo at mellanox.com>
> ---

Acked-by: Yongseok Koh <yskoh at mellanox.com>

> drivers/net/mlx5/mlx5_rxtx.c | 520 ++++++++++++++++++++++++++++++++++++++++++-
> 1 file changed, 516 insertions(+), 4 deletions(-)
> 
> diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
> index 13f9431..16be493 100644
> --- a/drivers/net/mlx5/mlx5_rxtx.c
> +++ b/drivers/net/mlx5/mlx5_rxtx.c
> @@ -1,6 +1,6 @@
> /* SPDX-License-Identifier: BSD-3-Clause
>  * Copyright 2015 6WIND S.A.
> - * Copyright 2015 Mellanox Technologies, Ltd
> + * Copyright 2015-2019 Mellanox Technologies, Ltd
>  */
> 
> #include <assert.h>
> @@ -34,6 +34,57 @@
> #include "mlx5_defs.h"
> #include "mlx5_prm.h"
> 
> +/* TX burst subroutines return codes. */
> +enum mlx5_txcmp_code {
> +	MLX5_TXCMP_CODE_EXIT = 0,
> +	MLX5_TXCMP_CODE_ERROR,
> +	MLX5_TXCMP_CODE_SINGLE,
> +	MLX5_TXCMP_CODE_MULTI,
> +	MLX5_TXCMP_CODE_TSO,
> +	MLX5_TXCMP_CODE_EMPW,
> +};
> +
> +/*
> + * These defines are used to configure Tx burst routine option set
> + * supported at compile time. The not specified options are optimized out
> + * out due to if conditions can be explicitly calculated at compile time.
> + * The offloads with bigger runtime check (require more CPU cycles to
> + * skip) overhead should have the bigger index - this is needed to
> + * select the better matching routine function if no exact match and
> + * some offloads are not actually requested.
> + */
> +#define MLX5_TXOFF_CONFIG_MULTI (1u << 0) /* Multi-segment packets.*/
> +#define MLX5_TXOFF_CONFIG_TSO (1u << 1) /* TCP send offload supported.*/
> +#define MLX5_TXOFF_CONFIG_SWP (1u << 2) /* Tunnels/SW Parser offloads.*/
> +#define MLX5_TXOFF_CONFIG_CSUM (1u << 3) /* Check Sums offloaded. */
> +#define MLX5_TXOFF_CONFIG_INLINE (1u << 4) /* Data inlining supported. */
> +#define MLX5_TXOFF_CONFIG_VLAN (1u << 5) /* VLAN insertion supported.*/
> +#define MLX5_TXOFF_CONFIG_METADATA (1u << 6) /* Flow metadata. */
> +#define MLX5_TXOFF_CONFIG_EMPW (1u << 8) /* Enhanced MPW supported.*/
> +
> +/* The most common offloads groups. */
> +#define MLX5_TXOFF_CONFIG_NONE 0
> +#define MLX5_TXOFF_CONFIG_FULL (MLX5_TXOFF_CONFIG_MULTI | \
> +				MLX5_TXOFF_CONFIG_TSO | \
> +				MLX5_TXOFF_CONFIG_SWP | \
> +				MLX5_TXOFF_CONFIG_CSUM | \
> +				MLX5_TXOFF_CONFIG_INLINE | \
> +				MLX5_TXOFF_CONFIG_VLAN | \
> +				MLX5_TXOFF_CONFIG_METADATA)
> +
> +#define MLX5_TXOFF_CONFIG(mask) (olx & MLX5_TXOFF_CONFIG_##mask)
> +
> +#define MLX5_TXOFF_DECL(func, olx) \
> +static uint16_t mlx5_tx_burst_##func(void *txq, \
> +				     struct rte_mbuf **pkts, \
> +				    uint16_t pkts_n) \
> +{ \
> +	return mlx5_tx_burst_tmpl((struct mlx5_txq_data *restrict)txq, \
> +		    pkts, pkts_n, (olx)); \
> +}
> +
> +#define MLX5_TXOFF_INFO(func, olx) {mlx5_tx_burst_##func, olx},
> +
> static __rte_always_inline uint32_t
> rxq_cq_to_pkt_type(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe);
> 
> @@ -1531,7 +1582,323 @@
> }
> 
> /**
> - * Configure the TX function to use.
> + * DPDK Tx callback template. This is configured template
> + * used to generate routines optimized for specified offload setup.
> + * One of this generated functions is chosen at SQ configuration
> + * time.
> + *
> + * @param txq
> + *   Generic pointer to TX queue structure.
> + * @param[in] pkts
> + *   Packets to transmit.
> + * @param pkts_n
> + *   Number of packets in array.
> + * @param olx
> + *   Configured offloads mask, presents the bits of MLX5_TXOFF_CONFIG_xxx
> + *   values. Should be static to take compile time static configuration
> + *   advantages.
> + *
> + * @return
> + *   Number of packets successfully transmitted (<= pkts_n).
> + */
> +static __rte_always_inline uint16_t
> +mlx5_tx_burst_tmpl(struct mlx5_txq_data *restrict txq,
> +		   struct rte_mbuf **restrict pkts,
> +		   uint16_t pkts_n,
> +		   unsigned int olx)
> +{
> +	(void)txq;
> +	(void)pkts;
> +	(void)pkts_n;
> +	(void)olx;
> +	return 0;
> +}
> +
> +/* Generate routines with Enhanced Multi-Packet Write support. */
> +MLX5_TXOFF_DECL(full_empw,
> +		MLX5_TXOFF_CONFIG_FULL | MLX5_TXOFF_CONFIG_EMPW)
> +
> +MLX5_TXOFF_DECL(none_empw,
> +		MLX5_TXOFF_CONFIG_NONE | MLX5_TXOFF_CONFIG_EMPW)
> +
> +MLX5_TXOFF_DECL(md_empw,
> +		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
> +
> +MLX5_TXOFF_DECL(mt_empw,
> +		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
> +		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
> +
> +MLX5_TXOFF_DECL(mtsc_empw,
> +		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
> +		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
> +		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
> +
> +MLX5_TXOFF_DECL(mti_empw,
> +		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
> +		MLX5_TXOFF_CONFIG_INLINE |
> +		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
> +
> +MLX5_TXOFF_DECL(mtv_empw,
> +		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
> +		MLX5_TXOFF_CONFIG_VLAN |
> +		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
> +
> +MLX5_TXOFF_DECL(mtiv_empw,
> +		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
> +		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
> +		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
> +
> +MLX5_TXOFF_DECL(sc_empw,
> +		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
> +		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
> +
> +MLX5_TXOFF_DECL(sci_empw,
> +		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
> +		MLX5_TXOFF_CONFIG_INLINE |
> +		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
> +
> +MLX5_TXOFF_DECL(scv_empw,
> +		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
> +		MLX5_TXOFF_CONFIG_VLAN |
> +		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
> +
> +MLX5_TXOFF_DECL(sciv_empw,
> +		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
> +		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
> +		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
> +
> +MLX5_TXOFF_DECL(i_empw,
> +		MLX5_TXOFF_CONFIG_INLINE |
> +		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
> +
> +MLX5_TXOFF_DECL(v_empw,
> +		MLX5_TXOFF_CONFIG_VLAN |
> +		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
> +
> +MLX5_TXOFF_DECL(iv_empw,
> +		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
> +		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
> +
> +/* Generate routines without Enhanced Multi-Packet Write support. */
> +MLX5_TXOFF_DECL(full,
> +		MLX5_TXOFF_CONFIG_FULL)
> +
> +MLX5_TXOFF_DECL(none,
> +		MLX5_TXOFF_CONFIG_NONE)
> +
> +MLX5_TXOFF_DECL(md,
> +		MLX5_TXOFF_CONFIG_METADATA)
> +
> +MLX5_TXOFF_DECL(mt,
> +		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
> +		MLX5_TXOFF_CONFIG_METADATA)
> +
> +MLX5_TXOFF_DECL(mtsc,
> +		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
> +		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
> +		MLX5_TXOFF_CONFIG_METADATA)
> +
> +MLX5_TXOFF_DECL(mti,
> +		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
> +		MLX5_TXOFF_CONFIG_INLINE |
> +		MLX5_TXOFF_CONFIG_METADATA)
> +
> +
> +MLX5_TXOFF_DECL(mtv,
> +		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
> +		MLX5_TXOFF_CONFIG_VLAN |
> +		MLX5_TXOFF_CONFIG_METADATA)
> +
> +
> +MLX5_TXOFF_DECL(mtiv,
> +		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
> +		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
> +		MLX5_TXOFF_CONFIG_METADATA)
> +
> +MLX5_TXOFF_DECL(sc,
> +		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
> +		MLX5_TXOFF_CONFIG_METADATA)
> +
> +MLX5_TXOFF_DECL(sci,
> +		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
> +		MLX5_TXOFF_CONFIG_INLINE |
> +		MLX5_TXOFF_CONFIG_METADATA)
> +
> +
> +MLX5_TXOFF_DECL(scv,
> +		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
> +		MLX5_TXOFF_CONFIG_VLAN |
> +		MLX5_TXOFF_CONFIG_METADATA)
> +
> +
> +MLX5_TXOFF_DECL(sciv,
> +		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
> +		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
> +		MLX5_TXOFF_CONFIG_METADATA)
> +
> +MLX5_TXOFF_DECL(i,
> +		MLX5_TXOFF_CONFIG_INLINE |
> +		MLX5_TXOFF_CONFIG_METADATA)
> +
> +MLX5_TXOFF_DECL(v,
> +		MLX5_TXOFF_CONFIG_VLAN |
> +		MLX5_TXOFF_CONFIG_METADATA)
> +
> +MLX5_TXOFF_DECL(iv,
> +		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
> +		MLX5_TXOFF_CONFIG_METADATA)
> +
> +/*
> + * Array of declared and compiled Tx burst function and corresponding
> + * supported offloads set. The array is used to select the Tx burst
> + * function for specified offloads set at Tx queue configuration time.
> + */
> +const struct {
> +	eth_tx_burst_t func;
> +	unsigned int olx;
> +} txoff_func[] = {
> +MLX5_TXOFF_INFO(full_empw,
> +		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
> +		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
> +		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
> +		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
> +
> +MLX5_TXOFF_INFO(none_empw,
> +		MLX5_TXOFF_CONFIG_NONE | MLX5_TXOFF_CONFIG_EMPW)
> +
> +MLX5_TXOFF_INFO(md_empw,
> +		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
> +
> +MLX5_TXOFF_INFO(mt_empw,
> +		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
> +		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
> +
> +MLX5_TXOFF_INFO(mtsc_empw,
> +		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
> +		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
> +		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
> +
> +MLX5_TXOFF_INFO(mti_empw,
> +		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
> +		MLX5_TXOFF_CONFIG_INLINE |
> +		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
> +
> +MLX5_TXOFF_INFO(mtv_empw,
> +		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
> +		MLX5_TXOFF_CONFIG_VLAN |
> +		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
> +
> +MLX5_TXOFF_INFO(mtiv_empw,
> +		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
> +		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
> +		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
> +
> +MLX5_TXOFF_INFO(sc_empw,
> +		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
> +		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
> +
> +MLX5_TXOFF_INFO(sci_empw,
> +		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
> +		MLX5_TXOFF_CONFIG_INLINE |
> +		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
> +
> +MLX5_TXOFF_INFO(scv_empw,
> +		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
> +		MLX5_TXOFF_CONFIG_VLAN |
> +		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
> +
> +MLX5_TXOFF_INFO(sciv_empw,
> +		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
> +		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
> +		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
> +
> +MLX5_TXOFF_INFO(i_empw,
> +		MLX5_TXOFF_CONFIG_INLINE |
> +		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
> +
> +MLX5_TXOFF_INFO(v_empw,
> +		MLX5_TXOFF_CONFIG_VLAN |
> +		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
> +
> +MLX5_TXOFF_INFO(iv_empw,
> +		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
> +		MLX5_TXOFF_CONFIG_METADATA | MLX5_TXOFF_CONFIG_EMPW)
> +
> +MLX5_TXOFF_INFO(full,
> +		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
> +		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
> +		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
> +		MLX5_TXOFF_CONFIG_METADATA)
> +
> +MLX5_TXOFF_INFO(none,
> +		MLX5_TXOFF_CONFIG_NONE)
> +
> +MLX5_TXOFF_INFO(md,
> +		MLX5_TXOFF_CONFIG_METADATA)
> +
> +MLX5_TXOFF_INFO(mt,
> +		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
> +		MLX5_TXOFF_CONFIG_METADATA)
> +
> +MLX5_TXOFF_INFO(mtsc,
> +		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
> +		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
> +		MLX5_TXOFF_CONFIG_METADATA)
> +
> +MLX5_TXOFF_INFO(mti,
> +		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
> +		MLX5_TXOFF_CONFIG_INLINE |
> +		MLX5_TXOFF_CONFIG_METADATA)
> +
> +
> +MLX5_TXOFF_INFO(mtv,
> +		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
> +		MLX5_TXOFF_CONFIG_VLAN |
> +		MLX5_TXOFF_CONFIG_METADATA)
> +
> +MLX5_TXOFF_INFO(mtiv,
> +		MLX5_TXOFF_CONFIG_MULTI | MLX5_TXOFF_CONFIG_TSO |
> +		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
> +		MLX5_TXOFF_CONFIG_METADATA)
> +
> +MLX5_TXOFF_INFO(sc,
> +		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
> +		MLX5_TXOFF_CONFIG_METADATA)
> +
> +MLX5_TXOFF_INFO(sci,
> +		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
> +		MLX5_TXOFF_CONFIG_INLINE |
> +		MLX5_TXOFF_CONFIG_METADATA)
> +
> +MLX5_TXOFF_INFO(scv,
> +		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
> +		MLX5_TXOFF_CONFIG_VLAN |
> +		MLX5_TXOFF_CONFIG_METADATA)
> +
> +MLX5_TXOFF_INFO(sciv,
> +		MLX5_TXOFF_CONFIG_SWP |	MLX5_TXOFF_CONFIG_CSUM |
> +		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
> +		MLX5_TXOFF_CONFIG_METADATA)
> +
> +MLX5_TXOFF_INFO(i,
> +		MLX5_TXOFF_CONFIG_INLINE |
> +		MLX5_TXOFF_CONFIG_METADATA)
> +
> +MLX5_TXOFF_INFO(v,
> +		MLX5_TXOFF_CONFIG_VLAN |
> +		MLX5_TXOFF_CONFIG_METADATA)
> +
> +MLX5_TXOFF_INFO(iv,
> +		MLX5_TXOFF_CONFIG_INLINE | MLX5_TXOFF_CONFIG_VLAN |
> +		MLX5_TXOFF_CONFIG_METADATA)
> +};
> +
> +/**
> + * Configure the Tx function to use. The routine checks configured
> + * Tx offloads for the device and selects appropriate Tx burst
> + * routine. There are multiple Tx burst routines compiled from
> + * the same template in the most optimal way for the dedicated
> + * Tx offloads set.
>  *
>  * @param dev
>  *   Pointer to private data structure.
> @@ -1542,8 +1909,153 @@
> eth_tx_burst_t
> mlx5_select_tx_function(struct rte_eth_dev *dev)
> {
> -	(void)dev;
> -	return removed_tx_burst;
> +	struct mlx5_priv *priv = dev->data->dev_private;
> +	struct mlx5_dev_config *config = &priv->config;
> +	uint64_t tx_offloads = dev->data->dev_conf.txmode.offloads;
> +	unsigned int diff = 0, olx = 0, i, m;
> +
> +	static_assert(MLX5_WQE_SIZE_MAX / MLX5_WSEG_SIZE <=
> +		      MLX5_DSEG_MAX, "invalid WQE max size");
> +	static_assert(MLX5_WQE_CSEG_SIZE == MLX5_WSEG_SIZE,
> +		      "invalid WQE Control Segment size");
> +	static_assert(MLX5_WQE_ESEG_SIZE == MLX5_WSEG_SIZE,
> +		      "invalid WQE Ethernet Segment size");
> +	static_assert(MLX5_WQE_DSEG_SIZE == MLX5_WSEG_SIZE,
> +		      "invalid WQE Data Segment size");
> +	static_assert(MLX5_WQE_SIZE == 4 * MLX5_WSEG_SIZE,
> +		      "invalid WQE size");
> +	assert(priv);
> +	if (tx_offloads & DEV_TX_OFFLOAD_MULTI_SEGS) {
> +		/* We should support Multi-Segment Packets. */
> +		olx |= MLX5_TXOFF_CONFIG_MULTI;
> +	}
> +	if (tx_offloads & (DEV_TX_OFFLOAD_TCP_TSO |
> +			   DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
> +			   DEV_TX_OFFLOAD_GRE_TNL_TSO |
> +			   DEV_TX_OFFLOAD_IP_TNL_TSO |
> +			   DEV_TX_OFFLOAD_UDP_TNL_TSO)) {
> +		/* We should support TCP Send Offload. */
> +		olx |= MLX5_TXOFF_CONFIG_TSO;
> +	}
> +	if (tx_offloads & (DEV_TX_OFFLOAD_IP_TNL_TSO |
> +			   DEV_TX_OFFLOAD_UDP_TNL_TSO |
> +			   DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM)) {
> +		/* We should support Software Parser for Tunnels. */
> +		olx |= MLX5_TXOFF_CONFIG_SWP;
> +	}
> +	if (tx_offloads & (DEV_TX_OFFLOAD_IPV4_CKSUM |
> +			   DEV_TX_OFFLOAD_UDP_CKSUM |
> +			   DEV_TX_OFFLOAD_TCP_CKSUM |
> +			   DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM)) {
> +		/* We should support IP/TCP/UDP Checksums. */
> +		olx |= MLX5_TXOFF_CONFIG_CSUM;
> +	}
> +	if (tx_offloads & DEV_TX_OFFLOAD_VLAN_INSERT) {
> +		/* We should support VLAN insertion. */
> +		olx |= MLX5_TXOFF_CONFIG_VLAN;
> +	}
> +	if (priv->txqs_n && (*priv->txqs)[0]) {
> +		struct mlx5_txq_data *txd = (*priv->txqs)[0];
> +
> +		if (txd->inlen_send) {
> +			/*
> +			 * Check the data inline requirements. Data inline
> +			 * is enabled on per device basis, we can check
> +			 * the first Tx queue only.
> +			 *
> +			 * If device does not support VLAN insertion in WQE
> +			 * and some queues are requested to perform VLAN
> +			 * insertion offload than inline must be enabled.
> +			 */
> +			olx |= MLX5_TXOFF_CONFIG_INLINE;
> +		}
> +	}
> +	if (config->mps == MLX5_MPW_ENHANCED &&
> +	    config->txq_inline_min <= 0) {
> +		/*
> +		 * The NIC supports Enhanced Multi-Packet Write.
> +		 * We do not support legacy MPW due to its
> +		 * hardware related problems, so we just ignore
> +		 * legacy MLX5_MPW settings. There should be no
> +		 * minimal required inline data.
> +		 */
> +		olx |= MLX5_TXOFF_CONFIG_EMPW;
> +	}
> +	if (tx_offloads & DEV_TX_OFFLOAD_MATCH_METADATA) {
> +		/* We should support Flow metadata. */
> +		olx |= MLX5_TXOFF_CONFIG_METADATA;
> +	}
> +	/*
> +	 * Scan the routines table to find the minimal
> +	 * satisfying routine with requested offloads.
> +	 */
> +	m = RTE_DIM(txoff_func);
> +	for (i = 0; i < RTE_DIM(txoff_func); i++) {
> +		unsigned int tmp;
> +
> +		tmp = txoff_func[i].olx;
> +		if (tmp == olx) {
> +			/* Meets requested offloads exactly.*/
> +			m = i;
> +			break;
> +		}
> +		if ((tmp & olx) != olx) {
> +			/* Does not meet requested offloads at all. */
> +			continue;
> +		}
> +		if ((olx ^ tmp) & MLX5_TXOFF_CONFIG_EMPW)
> +			/* Do not enable eMPW if not configured. */
> +			continue;
> +		if ((olx ^ tmp) & MLX5_TXOFF_CONFIG_INLINE)
> +			/* Do not enable inlining if not configured. */
> +			continue;
> +		/*
> +		 * Some routine meets the requirements.
> +		 * Check whether it has minimal amount
> +		 * of not requested offloads.
> +		 */
> +		tmp = __builtin_popcountl(tmp & ~olx);
> +		if (m >= RTE_DIM(txoff_func) || tmp < diff) {
> +			/* First or better match, save and continue. */
> +			m = i;
> +			diff = tmp;
> +			continue;
> +		}
> +		if (tmp == diff) {
> +			tmp = txoff_func[i].olx ^ txoff_func[m].olx;
> +			if (__builtin_ffsl(txoff_func[i].olx & ~tmp) <
> +			    __builtin_ffsl(txoff_func[m].olx & ~tmp)) {
> +				/* Lighter not requested offload. */
> +				m = i;
> +			}
> +		}
> +	}
> +	if (m >= RTE_DIM(txoff_func)) {
> +		DRV_LOG(DEBUG, "port %u has no selected Tx function"
> +			       " for requested offloads %04X",
> +				dev->data->port_id, olx);
> +		return NULL;
> +	}
> +	DRV_LOG(DEBUG, "port %u has selected Tx function"
> +		       " supporting offloads %04X/%04X",
> +			dev->data->port_id, olx, txoff_func[m].olx);
> +	if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_MULTI)
> +		DRV_LOG(DEBUG, "\tMULTI (multi segment)");
> +	if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_TSO)
> +		DRV_LOG(DEBUG, "\tTSO   (TCP send offload)");
> +	if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_SWP)
> +		DRV_LOG(DEBUG, "\tSWP   (software parser)");
> +	if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_CSUM)
> +		DRV_LOG(DEBUG, "\tCSUM  (checksum offload)");
> +	if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_INLINE)
> +		DRV_LOG(DEBUG, "\tINLIN (inline data)");
> +	if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_VLAN)
> +		DRV_LOG(DEBUG, "\tVLANI (VLAN insertion)");
> +	if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_METADATA)
> +		DRV_LOG(DEBUG, "\tMETAD (tx Flow metadata)");
> +	if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_EMPW)
> +		DRV_LOG(DEBUG, "\tEMPW  (Enhanced MPW)");
> +	return txoff_func[m].func;
> }
> 
> 
> -- 
> 1.8.3.1
>