[dpdk-dev] [PATCH v3 13/13] examples/ipsec-secgw: make number of buffers dynamic

[Ananyev, Konstantin]

Hi Lukasz,
 
> Make number of buffers in a pool nb_mbuf_in_pool dependent on number
> of ports, cores and crypto queues. Add command line option -s which
> when used overrides dynamic calculation of number of buffers in a pool.
> 
> Signed-off-by: Anoob Joseph <anoobj at marvell.com>
> Signed-off-by: Lukasz Bartosik <lbartosik at marvell.com>
> ---
>  examples/ipsec-secgw/ipsec-secgw.c | 59 +++++++++++++++++++++++++++++++-------
>  1 file changed, 48 insertions(+), 11 deletions(-)
> 
> diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
> index 862a7f0..f7acb52 100644
> --- a/examples/ipsec-secgw/ipsec-secgw.c
> +++ b/examples/ipsec-secgw/ipsec-secgw.c
> @@ -59,8 +59,6 @@ volatile bool force_quit;
> 
>  #define MEMPOOL_CACHE_SIZE 256
> 
> -#define NB_MBUF	(32000)
> -
>  #define CDEV_QUEUE_DESC 2048
>  #define CDEV_MAP_ENTRIES 16384
>  #define CDEV_MP_NB_OBJS 1024
> @@ -163,6 +161,7 @@ static int32_t promiscuous_on = 1;
>  static int32_t numa_on = 1; /**< NUMA is enabled by default. */
>  static uint32_t nb_lcores;
>  static uint32_t single_sa;
> +static uint32_t nb_bufs_in_pool;
> 
>  /*
>   * RX/TX HW offload capabilities to enable/use on ethernet ports.
> @@ -1259,6 +1258,7 @@ print_usage(const char *prgname)
>  		" [-w REPLAY_WINDOW_SIZE]"
>  		" [-e]"
>  		" [-a]"
> +		" [-s NUMBER_OF_MBUFS_IN_PKT_POOL]"
>  		" -f CONFIG_FILE"
>  		" --config (port,queue,lcore)[,(port,queue,lcore)]"
>  		" [--single-sa SAIDX]"
> @@ -1280,6 +1280,7 @@ print_usage(const char *prgname)
>  		"     size for each SA\n"
>  		"  -e enables ESN\n"
>  		"  -a enables SA SQN atomic behaviour\n"
> +		"  -s number of mbufs in packet pool (default 8192)\n"
>  		"  -f CONFIG_FILE: Configuration file\n"
>  		"  --config (port,queue,lcore): Rx queue configuration\n"
>  		"  --single-sa SAIDX: In poll mode use single SA index for\n"
> @@ -1479,7 +1480,7 @@ parse_args(int32_t argc, char **argv, struct eh_conf *eh_conf)
> 
>  	argvopt = argv;
> 
> -	while ((opt = getopt_long(argc, argvopt, "aelp:Pu:f:j:w:",
> +	while ((opt = getopt_long(argc, argvopt, "aelp:Pu:f:j:w:s:",
>  				lgopts, &option_index)) != EOF) {
> 
>  		switch (opt) {
> @@ -1513,6 +1514,19 @@ parse_args(int32_t argc, char **argv, struct eh_conf *eh_conf)
>  			cfgfile = optarg;
>  			f_present = 1;
>  			break;
> +
> +		case 's':
> +			ret = parse_decimal(optarg);
> +			if (ret < 0) {
> +				printf("Invalid number of buffers in a pool: "
> +					"%s\n", optarg);
> +				print_usage(prgname);
> +				return -1;
> +			}
> +
> +			nb_bufs_in_pool = ret;
> +			break;
> +
>  		case 'j':
>  			ret = parse_decimal(optarg);
>  			if (ret < RTE_MBUF_DEFAULT_BUF_SIZE ||
> @@ -1876,12 +1890,12 @@ check_cryptodev_mask(uint8_t cdev_id)
>  	return -1;
>  }
> 
> -static int32_t
> +static uint16_t
>  cryptodevs_init(void)
>  {
>  	struct rte_cryptodev_config dev_conf;
>  	struct rte_cryptodev_qp_conf qp_conf;
> -	uint16_t idx, max_nb_qps, qp, i;
> +	uint16_t idx, max_nb_qps, qp, total_nb_qps, i;
>  	int16_t cdev_id;
>  	struct rte_hash_parameters params = { 0 };
> 
> @@ -1909,6 +1923,7 @@ cryptodevs_init(void)
>  	printf("lcore/cryptodev/qp mappings:\n");
> 
>  	idx = 0;
> +	total_nb_qps = 0;
>  	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
>  		struct rte_cryptodev_info cdev_info;
> 
> @@ -1942,6 +1957,7 @@ cryptodevs_init(void)
>  		if (qp == 0)
>  			continue;
> 
> +		total_nb_qps += qp;
>  		dev_conf.socket_id = rte_cryptodev_socket_id(cdev_id);
>  		dev_conf.nb_queue_pairs = qp;
>  		dev_conf.ff_disable = RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO;
> @@ -1974,7 +1990,7 @@ cryptodevs_init(void)
> 
>  	printf("\n");
> 
> -	return 0;
> +	return total_nb_qps;
>  }
> 
>  static void
> @@ -2607,16 +2623,18 @@ int32_t
>  main(int32_t argc, char **argv)
>  {
>  	int32_t ret;
> -	uint32_t lcore_id;
> +	uint32_t lcore_id, nb_txq, nb_rxq = 0;
>  	uint32_t cdev_id;
>  	uint32_t i;
>  	uint8_t socket_id;
> -	uint16_t portid;
> +	uint16_t portid, nb_crypto_qp, nb_ports = 0;
>  	uint64_t req_rx_offloads[RTE_MAX_ETHPORTS];
>  	uint64_t req_tx_offloads[RTE_MAX_ETHPORTS];
>  	struct eh_conf *eh_conf = NULL;
>  	size_t sess_sz;
> 
> +	nb_bufs_in_pool = 0;
> +
>  	/* init EAL */
>  	ret = rte_eal_init(argc, argv);
>  	if (ret < 0)
> @@ -2665,6 +2683,26 @@ main(int32_t argc, char **argv)
> 
>  	sess_sz = max_session_size();
> 
> +	nb_crypto_qp = cryptodevs_init();
> +
> +	if (nb_bufs_in_pool == 0) {
> +		RTE_ETH_FOREACH_DEV(portid) {
> +			if ((enabled_port_mask & (1 << portid)) == 0)
> +				continue;
> +			nb_ports++;
> +			nb_rxq += get_port_nb_rx_queues(portid);
> +		}
> +
> +		nb_txq = nb_lcores;
> +
> +		nb_bufs_in_pool = RTE_MAX((nb_rxq*nb_rxd +
> +					   nb_ports*nb_lcores*MAX_PKT_BURST +
> +					   nb_ports*nb_txq*nb_txd +
> +					   nb_lcores*MEMPOOL_CACHE_SIZE +
> +					   nb_crypto_qp*CDEV_QUEUE_DESC),

I think you forgot to take into account possible reassemble table:
@@ -2699,7 +2699,9 @@ main(int32_t argc, char **argv)
                                           nb_ports*nb_lcores*MAX_PKT_BURST +
                                           nb_ports*nb_txq*nb_txd +
                                           nb_lcores*MEMPOOL_CACHE_SIZE +
-                                          nb_crypto_qp*CDEV_QUEUE_DESC),
+                                          nb_crypto_qp*CDEV_QUEUE_DESC +
+                                          nb_lcores * frag_tbl_sz *
+                                          FRAG_TBL_BUCKET_ENTRIES),
                                          8192U);


Also it might be worth for better readability to put code for nb_bufs_in_pool calculation
in a separate  function (and add spaces between '*' and its' operands).
Apart from that - whole series LGTM.
Konstantin


> +					  8192U);
> +	}
> +
>  	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
>  		if (rte_lcore_is_enabled(lcore_id) == 0)
>  			continue;
> @@ -2678,11 +2716,12 @@ main(int32_t argc, char **argv)
>  		if (socket_ctx[socket_id].mbuf_pool)
>  			continue;
> 
> -		pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF);
> +		pool_init(&socket_ctx[socket_id], socket_id, nb_bufs_in_pool);
>  		session_pool_init(&socket_ctx[socket_id], socket_id, sess_sz);
>  		session_priv_pool_init(&socket_ctx[socket_id], socket_id,
>  			sess_sz);
>  	}
> +	printf("Number of mbufs in packet pool %d\n", nb_bufs_in_pool);
> 
>  	RTE_ETH_FOREACH_DEV(portid) {
>  		if ((enabled_port_mask & (1 << portid)) == 0)
> @@ -2694,8 +2733,6 @@ main(int32_t argc, char **argv)
>  				req_tx_offloads[portid]);
>  	}
> 
> -	cryptodevs_init();
> -
>  	/*
>  	 * Set the enabled port mask in helper config for use by helper
>  	 * sub-system. This will be used while initializing devices using
> --
> 2.7.4