[dts] [next][PATCH V1 1/2] tests/nic_single_core_perf: redefine accecpt_tolerance

[Lijuan Tu]

Current accecpt_tolerance is a delta value, change it to a percentage
value, as a percentage value is more make sense for tolerance.

Signed-off-by: Lijuan Tu <lijuan.tu at intel.com>
---
 tests/TestSuite_nic_single_core_perf.py | 40 ++++++++++++++++++++++++---------
 1 file changed, 30 insertions(+), 10 deletions(-)

diff --git a/tests/TestSuite_nic_single_core_perf.py b/tests/TestSuite_nic_single_core_perf.py
index 847a94c..b4d682e 100644
--- a/tests/TestSuite_nic_single_core_perf.py
+++ b/tests/TestSuite_nic_single_core_perf.py
@@ -74,6 +74,18 @@ class TestNicSingleCorePerf(TestCase):
         # determine if to save test result as a separated file
         self.save_result_flag = True
 
+    def convert_tolenrance_to_gap(self):
+        # Accepted tolerance in percentage
+        gap = dict()
+        rate = self.get_suite_cfg()['accepted_tolerance'] * 0.01
+        for frame_size in self.test_parameters.keys():
+            gap[frame_size] = dict()
+            for nb_desc in self.test_parameters[frame_size]:
+                expected_gap = rate * self.expected_throughput[frame_size][nb_desc]
+                gap[frame_size][nb_desc] = round(expected_gap, 3)
+        self.logger.info("Accept tolerance are (Mpps)%s" % gap)
+        return gap
+
     def set_up(self):
         """
         Run before each test case.
@@ -104,14 +116,27 @@ class TestNicSingleCorePerf(TestCase):
         # {'$framesize':{"$nb_desc": 'throughput'}
         self.throughput = {}
 
-        # Accepted tolerance in Mpps
-        self.gap = self.get_suite_cfg()['accepted_tolerance']
+        # self.gap has same structure as throughput.
+        self.gap = self.convert_tolenrance_to_gap()
 
         # header to print test result table
         self.table_header = ['Frame Size', 'TXD/RXD', 'Throughput', 'Rate',
                              'Expected Throughput', 'Throughput Difference']
         self.test_result = {}
 
+    def get_gap_border(self, frame_size, nb_desc):
+        return -self.gap[frame_size][nb_desc]
+
+    def verify_tolerance(self):
+        for frame_size in self.test_parameters.keys():
+            for nb_desc in self.test_parameters[frame_size]:
+                cur_gap = self.throughput[frame_size][nb_desc] - self.expected_throughput[frame_size][nb_desc]
+                gap_border = self.get_gap_border(frame_size, nb_desc)
+                show_str = "Possible regression, " +\
+                           "Frame size/Descriptors: {}/{}, " +\
+                           "Acceptable Gap/Actual Gap: {}/{}"
+                self.verify(cur_gap >= gap_border,show_str.format(frame_size, nb_desc, gap_border, cur_gap))
+
     def test_nic_single_core_perf(self):
         """
         Run nic single core performance 
@@ -124,12 +149,7 @@ class TestNicSingleCorePerf(TestCase):
 
         # check the gap between expected throughput and actual throughput
         try:
-            for frame_size in self.test_parameters.keys():
-                for nb_desc in self.test_parameters[frame_size]:
-                    cur_gap = (self.expected_throughput[frame_size][nb_desc] -
-                                self.throughput[frame_size][nb_desc])
-                    self.verify(cur_gap < self.gap,
-                                 "Beyond Gap, Possible regression")
+            self.verify_tolerance()
         except Exception as e:
             self.logger.error(e)
             self.handle_expected()
@@ -146,7 +166,7 @@ class TestNicSingleCorePerf(TestCase):
             for frame_size in self.test_parameters.keys():
                 for nb_desc in self.test_parameters[frame_size]:
                     self.expected_throughput[frame_size][nb_desc] = \
-                        round(self.throughput[frame_size][nb_desc],3)
+                        round(self.throughput[frame_size][nb_desc], 3)
 
     def perf_test(self, port_num):
         """
@@ -303,7 +323,7 @@ class TestNicSingleCorePerf(TestCase):
                     value = row_in['TXD/RXD'], unit = 'descriptors')
                 delta = (float(row_in['Throughput'].split()[0]) -
                          float(row_in['Expected Throughput'].split()[0]))
-                if delta >= -self.gap:
+                if delta >= self.get_gap_border(frame_size, nb_desc):
                     result = 'PASS'
                 else:
                     result = 'FAIL'
-- 
1.8.3.1