[PATCH v1] dts: update configuration for performance test

[Andrew Bailey]

The single core forward performance test suite can run a variety of
combinations of buffer sizes and number of descriptors. It is beneficial
if any possible combination of these parameters could be tested against
a specific delta tolerance; Rather than a globally defined tolerance for
all testing. Along with this, adjusting traffic duration and test
repetitions helps users define a more accurate test run if they wish to
get a more refined average. Furthermore, the cores allotted to TREX can
vary among systems and should be defined by the user.

Signed-off-by: Andrew Bailey <abailey at iol.unh.edu>
---
 dts/configurations/test_run.example.yaml      |  1 +
 dts/framework/config/test_run.py              |  1 +
 .../testbed_model/traffic_generator/trex.py   |  3 +-
 .../TestSuite_single_core_forward_perf.py     | 52 ++++++++++++++-----
 4 files changed, 43 insertions(+), 14 deletions(-)

diff --git a/dts/configurations/test_run.example.yaml b/dts/configurations/test_run.example.yaml
index 51f9ab8237..c6eb086211 100644
--- a/dts/configurations/test_run.example.yaml
+++ b/dts/configurations/test_run.example.yaml
@@ -52,6 +52,7 @@ func_traffic_generator:
 #   type: TREX
 #   remote_path: "/opt/trex/v3.03" # The remote path of the traffic generator application.
 #   config: "/opt/trex_config/trex_config.yaml" # Additional configuration files. (Leave blank if not required)
+#   cores: 10 # The number of cores to run TRex with.
 perf: false # disable performance testing
 func: true # enable functional testing
 crypto: false # disable cryptographic testing
diff --git a/dts/framework/config/test_run.py b/dts/framework/config/test_run.py
index 81630df77d..1ffb6b2365 100644
--- a/dts/framework/config/test_run.py
+++ b/dts/framework/config/test_run.py
@@ -420,6 +420,7 @@ class TrexTrafficGeneratorConfig(TrafficGeneratorConfig):
     type: Literal[TrafficGeneratorType.TREX]
     remote_path: PurePath
     config: PurePath
+    cores: int
 
 
 #: A union type discriminating traffic generators by the `type` field.
diff --git a/dts/framework/testbed_model/traffic_generator/trex.py b/dts/framework/testbed_model/traffic_generator/trex.py
index d53791b0a6..2b52cfcbd1 100644
--- a/dts/framework/testbed_model/traffic_generator/trex.py
+++ b/dts/framework/testbed_model/traffic_generator/trex.py
@@ -82,7 +82,7 @@ class TrexTrafficGenerator(PerformanceTrafficGenerator):
 
     _streaming_mode: TrexStatelessTXModes = TrexStatelessTXModes.STLTXCont
 
-    _tg_cores: int = 10
+    _tg_cores: int
 
     _trex_app: BlockingApp
 
@@ -102,6 +102,7 @@ def __init__(self, tg_node: Node, config: TrexTrafficGeneratorConfig) -> None:
         super().__init__(tg_node=tg_node, config=config)
         self._tg_node_config = tg_node.config
         self._tg_config = config
+        self._tg_cores = config.cores
 
         self._os_session = create_session(self._tg_node.config, "TRex", self._logger)
 
diff --git a/dts/tests/TestSuite_single_core_forward_perf.py b/dts/tests/TestSuite_single_core_forward_perf.py
index 57aecdf7a4..a966d44ebd 100644
--- a/dts/tests/TestSuite_single_core_forward_perf.py
+++ b/dts/tests/TestSuite_single_core_forward_perf.py
@@ -32,14 +32,40 @@ class Config(BaseConfig):
     """Performance test metrics."""
 
     test_parameters: list[dict[str, int | float]] = [
-        {"frame_size": 64, "num_descriptors": 1024, "expected_mpps": 1.00},
-        {"frame_size": 128, "num_descriptors": 1024, "expected_mpps": 1.00},
-        {"frame_size": 256, "num_descriptors": 1024, "expected_mpps": 1.00},
-        {"frame_size": 512, "num_descriptors": 1024, "expected_mpps": 1.00},
-        {"frame_size": 1024, "num_descriptors": 1024, "expected_mpps": 1.00},
-        {"frame_size": 1518, "num_descriptors": 1024, "expected_mpps": 1.00},
+        {"frame_size": 64, "num_descriptors": 1024, "expected_mpps": 1.00, "delta_tolerance": 0.05},
+        {
+            "frame_size": 128,
+            "num_descriptors": 1024,
+            "expected_mpps": 1.00,
+            "delta_tolerance": 0.05,
+        },
+        {
+            "frame_size": 256,
+            "num_descriptors": 1024,
+            "expected_mpps": 1.00,
+            "delta_tolerance": 0.05,
+        },
+        {
+            "frame_size": 512,
+            "num_descriptors": 1024,
+            "expected_mpps": 1.00,
+            "delta_tolerance": 0.05,
+        },
+        {
+            "frame_size": 1024,
+            "num_descriptors": 1024,
+            "expected_mpps": 1.00,
+            "delta_tolerance": 0.05,
+        },
+        {
+            "frame_size": 1518,
+            "num_descriptors": 1024,
+            "expected_mpps": 1.00,
+            "delta_tolerance": 0.05,
+        },
     ]
-    delta_tolerance: float = 0.05
+    traffic_duration: int = 5
+    test_repetitions: int = 5
 
 
 @requires_link_topology(LinkTopology.TWO_LINKS)
@@ -51,7 +77,8 @@ class TestSingleCoreForwardPerf(TestSuite):
     def set_up_suite(self):
         """Set up the test suite."""
         self.test_parameters = self.config.test_parameters
-        self.delta_tolerance = self.config.delta_tolerance
+        self.traffic_duration = self.config.traffic_duration
+        self.test_repetitions = self.config.test_repetitions
 
     def _transmit(self, testpmd: TestPmd, frame_size: int, repetitions: int = 1) -> float:
         """Create a testpmd session with every rule in the given list, verify jump behavior.
@@ -76,8 +103,7 @@ def _transmit(self, testpmd: TestPmd, frame_size: int, repetitions: int = 1) ->
         rx_avg = 0.0
 
         for _ in range(repetitions):
-            # Transmit for 5 seconds.
-            stats = assess_performance_by_packet(packet=packet, duration=5)
+            stats = assess_performance_by_packet(packet=packet, duration=self.traffic_duration)
             rx_avg += stats.rx_pps
         return rx_avg / (repetitions * 1_000_000)
 
@@ -134,21 +160,21 @@ def single_core_forward_perf(self) -> None:
                 **driver_specific_testpmd_args,
             ) as testpmd:
                 params["measured_mpps"] = round(
-                    self._transmit(testpmd, frame_size, repetitions=5), 3
+                    self._transmit(testpmd, frame_size, repetitions=self.test_repetitions), 3
                 )
                 params["performance_delta"] = round(
                     (float(params["measured_mpps"]) - float(params["expected_mpps"]))
                     / float(params["expected_mpps"]),
                     3,
                 )
-                params["pass"] = float(params["performance_delta"]) >= -self.delta_tolerance
+                params["pass"] = float(params["performance_delta"]) >= -params["delta_tolerance"]
 
         self._produce_stats_table(self.test_parameters)
 
         for params in self.test_parameters:
             verify(
                 params["pass"] is True,
-                f"""Packets forwarded is less than {(1 - self.delta_tolerance) * 100}%
+                f"""Packets forwarded is less than {(1 - params["delta_tolerance"]) * 100}%
                 of the expected baseline.
                 Measured MPPS = {params["measured_mpps"]}
                 Expected MPPS = {params["expected_mpps"]}""",
-- 
2.55.0