[dts] [PATCH V1 2/2] ethtool_stats: suite automation script

yufengx.mo at intel.com yufengx.mo at intel.com
Wed Jun 6 07:34:02 CEST 2018


From: yufengmx <yufengx.mo at intel.com>


Currently Ethtool supports a more complete list of stats for the same drivers
that DPDK supports. The idea behind this epic is two fold as following:
1. To achieve metric equivalence with what linux's ethtool provides.
2. To extend the functionality of the xstats API to enable the following options:
   ## the retrieval of aggregate stats upon request (Top level stats).
   ## the retrieval of the extended NIC stats.
   ## grouping of stats logically so they can be retrieved per logical grouping.
   ## the option to enable/disable the stats groups to retrieve similar to set
      private flags in ethtool.

Signed-off-by: yufengmx <yufengx.mo at intel.com>
---
 tests/TestSuite_ethtool_stats.py | 566 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 566 insertions(+)
 create mode 100644 tests/TestSuite_ethtool_stats.py

diff --git a/tests/TestSuite_ethtool_stats.py b/tests/TestSuite_ethtool_stats.py
new file mode 100644
index 0000000..d8cd0df
--- /dev/null
+++ b/tests/TestSuite_ethtool_stats.py
@@ -0,0 +1,566 @@
+# BSD LICENSE
+#
+# Copyright(c) 2010-2018 Intel Corporation. All rights reserved.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+#   * Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#   * Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in
+#     the documentation and/or other materials provided with the
+#     distribution.
+#   * Neither the name of Intel Corporation nor the names of its
+#     contributors may be used to endorse or promote products derived
+#     from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# 'AS IS' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+'''
+DPDK Test suite.
+
+'''
+
+import re
+import time
+import os, sys
+
+import utils
+from qemu_kvm import QEMUKvm
+from test_case import TestCase
+from pmd_output import PmdOutput
+from exception import VerifyFailure
+
+from scapy.utils import rdpcap
+
+from packet import Packet, sniff_packets, load_sniff_packets
+from settings import HEADER_SIZE
+
+import logger
+
+
+class TestEthtoolStats(TestCase):
+    def set_up_all(self):
+        self.dut_ports = self.dut.get_ports(self.nic)
+        self.verify(len(self.dut_ports) >= 1, 'Insufficient ports')
+        self.printFlag = self._enable_debug
+        # get link port pairs
+        self.link_ports_pair = {}
+        for port_num in range(len(self.dut_ports)):
+            local_port = self.tester.get_local_port(0)
+            self.src_intf =self.tester.get_interface(local_port)
+            self.src_mac =  self.tester.get_mac(local_port)
+            self.dst_mac = self.dut.get_mac_address(0)
+            self.link_ports_pair[port_num] = [self.src_intf,
+                                              self.src_mac,
+                                              self.dst_mac]
+        # 
+        self.pmd_cmds = list()
+        self.testpmd = PmdOutput(self.dut)
+        self.testpmd_flag = False 
+        # dpdk pro-info
+        ports_count = len(self.dut_ports)
+        ports_mask = reduce(lambda x, y: x | y,
+                           map(lambda x: 0x1<<x, range(ports_count)))
+        self.query_tool = os.sep.join([self.dut.base_dir,
+                      self.target,
+                      'app',
+                      'dpdk-procinfo'])
+        self.dpdk_proc_info = "%s -- -p %s"\
+                                % (self.query_tool,
+                                   ports_mask)
+        self.session_ex = self.dut.new_session()
+        # set packet sizes for testing different type 
+        self.test_frame_sizes = [64, 72, 128, 256, 512, 1024]
+        # (linux's ethtool statistics names, dpdk statistics names) 
+        self.port_options = [('',''), 
+                             ('','')]
+        #-----------------------------------------------------------------
+        if logger.log_dir.startswith("/home"):
+            self.output_dir = logger.log_dir
+        else:
+            curPath = os.path.dirname(os.path.realpath(__file__)) + "/.."
+            self.output_dir = curPath + os.sep + logger.log_dir
+        #-----------------------------------------------------------------
+        self.clear_ASLR()
+
+    def set_up(self):
+        pass
+
+    def tear_down(self):
+        self.clear_test_enviroment()
+
+    def tear_down_all(self):
+        if self.session_ex:
+            self.reset_ASLR()
+            self.session_ex.close()
+            self.session_ex = None
+
+    def clear_ASLR(self):
+        '''
+        DPDK don't support ASLR open under multiprocess status, so close it
+        '''
+        cmd = "echo 0 > /proc/sys/kernel/randomize_va_space"
+        self.session_ex.send_expect( cmd, "# ", timeout=10)
+        time.sleep(2)
+
+    def reset_ASLR(self):
+        '''
+        reset ASLR to original status
+        '''
+        cmd = "echo 2 > /proc/sys/kernel/randomize_va_space"
+        self.session_ex.send_expect(cmd, "# ", timeout=10)
+        time.sleep(4)
+
+    def send_custom_frames(self):
+        total_bytes = 0
+        total_packets = len(self.test_frame_sizes)
+        src_intf = self.src_intf
+        for frame_size in self.test_frame_sizes:
+            headers_size = sum(map(lambda x: HEADER_SIZE[x], 
+                                   ['eth', 'ip', 'udp']))
+
+            payload_size = frame_size - headers_size
+            config_layers =  {'ether': {'src': self.src_mac},
+                              'raw':   {'payload': ['58'] * payload_size}}
+            pkt_types = {'UDP': {'layer_configs': config_layers}}
+            pcap_file = self.output_dir + os.sep + "xstat_{0}.pcap".format(
+                                                            str(frame_size))
+            total_bytes += self.packet_transmission(pkt_types, pcap_file, 
+                                                    src_intf)
+
+        check_statistics = {
+            # rx
+            'rx_size_64_packets' : 1,
+            'rx_size_65_to_127_packets' : 1,
+            'rx_size_128_to_255_packets' : 1,
+            'rx_size_256_to_511_packets' : 1,
+            'rx_size_512_to_1023_packets' : 1,
+            'rx_total_bytes': total_bytes,
+            'rx_total_packets': total_packets,
+            'rx_broadcast_packets': total_packets,
+            # tx
+            'tx_size_64_packets' : 1,
+            'tx_size_65_to_127_packets' : 1,
+            'tx_size_128_to_255_packets' : 1,
+            'tx_size_256_to_511_packets' : 1,
+            'tx_size_512_to_1023_packets' : 1,
+            'tx_total_bytes': total_bytes,
+            'tx_total_packets': total_packets,
+            'tx_broadcast_packets': total_packets,
+        }
+
+        if self.kdriver == "i40e":
+            check_statistics.pop("rx_total_packets")
+            check_statistics.pop("rx_total_bytes")
+            check_statistics.pop("tx_total_packets")
+            check_statistics.pop("tx_total_bytes")
+        elif self.kdriver == "ixgbe":
+            check_statistics.pop('tx_total_bytes')
+        
+        return check_statistics
+
+    def packet_transmission(self, pkt_types, pcap_file, src_intf):
+        time.sleep(1)
+        packet_size = 0
+        self.session_ex.send_expect("ifconfig {0} up".format(src_intf),
+                                    "# ", timeout=10)
+        for pkt_type in pkt_types.keys():
+            pkt  = Packet(pkt_type=pkt_type)
+            # set packet every layer's input parameters
+            if 'layer_configs' in pkt_types[pkt_type].keys():
+                pkt_configs = pkt_types[pkt_type]['layer_configs']
+                if pkt_configs:
+                    for layer in pkt_configs.keys():
+                        pkt.config_layer(layer, pkt_configs[layer])
+            pkt.pktgen.write_pcap(pcap_file)
+            packet_size = len(pkt.pktgen.pkt[0])
+            pkt.send_pkt(tx_port=src_intf)
+            time.sleep(1)
+
+        return packet_size
+
+    def preset_testpmd(self):
+        self.testpmd.start_testpmd('1S/2C/2T', 
+                                   param='--port-topology=loop')
+        self.testpmd_flag = True
+        self.execute_testpmd_cmd(self.pmd_cmds)
+        self.pmd_cmds = list()
+        time.sleep(2)
+
+    def execute_testpmd_cmd(self, cmds):
+        if len(cmds) == 0:
+            return
+        for item in cmds:
+            if len(item) == 2:
+                self.testpmd.execute_cmd(item[0], int(item[1]))
+            else:
+                self.testpmd.execute_cmd(item[0])
+        time.sleep(2)
+
+    def get_port_xstat_data(self, msg):
+        if "statistics" not in msg:
+            self.logger.error(msg)
+            raise VerifyFailure("get port statistics data failed")
+
+        data_str = msg.splitlines()
+        port_xstat = {}
+        cur_port = None
+        pat = ".*for port (\d)+ .*"
+        for line in data_str:
+            if not line.strip():
+                continue
+            if "statistics" in line:
+                result = re.findall(pat, line.strip())
+                if len(result):
+                    cur_port = result[0]
+            elif cur_port != None and ": " in line:
+                if cur_port not in port_xstat:
+                    port_xstat[cur_port] = {}
+                result  = line.strip().split(": ")
+                if len(result) == 2 and result[0]:
+                    name, value = result
+                    port_xstat[cur_port][name] = value
+                else:
+                    raise VerifyFailure("invalid data")
+
+        return port_xstat
+
+    def query_dpdk_xstat_all(self, option="xstats"):
+        cmd = self.dpdk_proc_info + " --%s"%(option)
+        self.session_ex.send_expect(cmd,
+                                     "# ")
+        msg = self.session_ex.session.get_session_before(0.5)
+        self.logger.info(msg)
+        infos = self.get_port_xstat_data(msg)
+        return infos
+
+    def check_xstat_command_list(self):
+        output = self.session_ex.send_expect(
+                                    self.query_tool,
+                                    "# ")
+        
+        expected_command = ["xstats-reset", 
+                            "xstats-name NAME",
+                            "xstats-ids IDLIST",
+                            "xstats-reset"]
+        
+        pat = ".*--(.*):.*"
+        handle = re.compile(pat)
+        result = handle.findall(output)
+        if not result or len(result) == 0:
+            cmds = " | ".join(expected_command)
+            msg = "expected commands {0} have not been included".format(cmds)
+            raise VerifyFailure(msg)
+        missing_cmds = []
+        for cmd in expected_command:
+            if cmd not in result:
+                missing_cmds.append(cmd)
+        
+        if len(missing_cmds):
+            msg = " | ".join(missing_cmds) + " have not been included"
+            raise VerifyFailure(msg)
+        else:
+            cmds = " | ".join(expected_command)
+            msg = "expected commands {0} have been included".format(cmds)
+            self.logger.info(msg)
+
+    def check_xstat_reset_status(self):
+        all_stat_data = self.query_dpdk_xstat_all()
+        execept_msgs = []
+        for port in all_stat_data:
+            stats_info = all_stat_data[port]
+            for stat_name, value in stats_info.items():
+                if int(value) != 0:
+                    msg = "port {0} <{1}> [{2}] has not been reset"
+                    execept_msgs.append( msg.format(port,
+                                                   stat_name,
+                                                   value))
+        if len(execept_msgs):
+            self.logger.info(os.linesep.join(execept_msgs))
+            raise VerifyFailure("xstat-reset failed")
+        else:
+            self.logger.info("xstat-reset success !")
+
+    def check_xstat_statistic_intergrity(self, sub_options_ex=None):
+        all_stat_data = self.query_dpdk_xstat_all()
+        self.check_xstat_id_cmd(all_stat_data)
+        self.check_xstat_name_cmd(all_stat_data)
+
+    def check_xstat_id_cmd(self, all_stat_data):
+        option = "xstats-id"
+        ids = reduce(lambda x, y: str(x) + "," + str(y), 
+                     range(len(all_stat_data['0'].keys())))
+        _sub_options = [ids]
+        execept_msgs = []
+        for sub_option in _sub_options:
+            cmd = self.dpdk_proc_info + " --%s %s"%(option, 
+                                                    sub_option)
+            self.session_ex.send_expect(cmd,
+                                         "# ")
+            msg = self.session_ex.session.get_session_before(0.5)
+            sub_stat_data = self.get_port_xstat_data(msg)
+            if not sub_stat_data or not len(sub_stat_data):
+                execept_msgs.append([option, msg])
+            else:
+                for port, infos in sub_stat_data.items():
+                    for item in infos:
+                        if not port \
+                               or port not in all_stat_data \
+                               or item not in all_stat_data[port]:
+                            msg1 = "port {0} get [{1}] failed".format(port, 
+                                                                      item)
+                            execept_msgs.append([msg1])
+                            continue
+        if len(execept_msgs):
+            for msgs in execept_msgs:
+                self.logger.error(msgs[0])
+                #self.logger.info(msgs[1])
+            raise VerifyFailure("query data exception ")
+        else:
+            self.logger.info("all ports stat id can get")
+
+        time.sleep(1)
+
+    def check_xstat_name_cmd(self, all_stat_data):
+        option = "xstats-name"
+        _sub_options = all_stat_data['0'].keys()
+        
+        execept_msgs = []
+        for sub_option in _sub_options:
+            
+            cmd = self.dpdk_proc_info + " --%s %s"%(option, sub_option)
+            self.session_ex.send_expect(cmd, "# ")
+            msg = self.session_ex.session.get_session_before(0.5)
+            sub_stat_data = self.get_port_xstat_data(msg)
+            if sub_option not in msg or not len(sub_stat_data):
+                execept_msgs.append([option, msg])
+            else:
+                for port in sub_stat_data:
+                    if sub_option not in sub_stat_data[port]:
+                        msg = "{0} {1} data doesn't existed".format(port,
+                                                                    sub_option)
+                        self.logger.error(msg)
+                        continue
+                    if not port or \
+                           port not in all_stat_data or \
+                           sub_option not in all_stat_data[port]:
+                        msg1 = "port {0} [{1}]".format(port, sub_option)
+                        execept_msgs.append([msg1])
+                        continue
+        if len(execept_msgs):
+            for msgs in execept_msgs:
+                self.logger.error(msgs[0])
+                self.logger.info(msgs[1])
+            raise VerifyFailure("query data exception ")
+        else:
+            self.logger.info("all port's stat value can get")
+
+    def check_xstat_single_statistic(self, sub_options_ex=None):
+        all_stat_data = self.query_dpdk_xstat_all()
+        self.logger.info("total stat names [%d]"%len(all_stat_data['0']))
+        for stat_name in all_stat_data['0'].keys():
+            # firstly, get statistic name. 
+            stats, execept_msgs = self.get_xstat_statistic_id(stat_name)
+            if len(execept_msgs):
+                for msgs in execept_msgs:
+                    self.logger.error(msgs[0])
+                    self.logger.info(msgs[1])
+                continue
+
+            self.logger.info(stat_name)
+            self.get_xstat_single_statistic(stats['0'], all_stat_data)
+
+    def get_xstat_statistic_id(self, stat_name):
+        option          = "xstats-name"
+        sub_option      = stat_name
+        execept_msgs    = []
+        cmd = self.dpdk_proc_info + " --%s %s"%(option,
+                                                sub_option)
+        self.session_ex.send_expect(cmd, "# ")
+        msg = self.session_ex.session.get_session_before(0.5)
+        sub_stat_data = self.get_port_xstat_data(msg)
+        if sub_option not in msg or not len(sub_stat_data):
+            execept_msgs.append([option, msg])
+        else:
+            for port in sub_stat_data:
+                if sub_option not in sub_stat_data[port]:
+                    msg = "{0} {1} data doesn't existed".format(port,
+                                                                sub_option)
+                    self.logger.error(msg)
+                    continue
+                if not port:
+                    msg1 = "port {0} [{1}]".format(port, sub_option)
+                    execept_msgs.append([msg1, msg2])
+                    continue
+        return sub_stat_data, execept_msgs
+
+    def _xstat_single_stats_result(self, sub_stat_data, all_stat_data):
+        execept_msgs = []
+        for port, infos in sub_stat_data.items():
+            for item in infos:
+                if not port or \
+                       port not in all_stat_data or \
+                       item not in all_stat_data[port] or \
+                       sub_stat_data[port][item] != all_stat_data[port][item]:
+                    msg1 = "port {0} [{1}]".format(port, item)
+                    msg2 = "expect {0} ".format(all_stat_data[port][item]) + \
+                           "show {0}".format(sub_stat_data[port][item])
+                    execept_msgs.append([msg1, msg2])
+                    continue
+                msg2 = "expect {0} ".format(all_stat_data[port][item]) + \
+                       "show {0}".format(sub_stat_data[port][item])
+                self.logger.info(msg2)
+        return execept_msgs
+
+    def get_xstat_single_statistic(self, stat, all_stat_data):
+        option = "xstats-id"
+        execept_msgs = []
+        for id in stat.values():
+            cmd = self.dpdk_proc_info + " --%s %s"%(option, 
+                                                    id)
+            self.session_ex.send_expect(cmd, "# ")
+            msg = self.session_ex.session.get_session_before(0.5)
+            sub_stat_data = self.get_port_xstat_data(msg)
+            if not sub_stat_data or not len(sub_stat_data):
+                execept_msgs.append([option, msg])
+            else:
+                execept_msgs += self._xstat_single_stats_result(sub_stat_data,
+                                                                all_stat_data)
+            if len(execept_msgs):
+                for msgs in execept_msgs:
+                    self.logger.error(msgs[0])
+                    self.logger.info(msgs[1])
+                raise VerifyFailure("query data exception ")
+            else:
+                self.logger.info("all port is correct")
+
+        time.sleep(1)
+
+    def query_linux_stat(self):
+        options = ["-S"]
+        dev_name = self.link_ports_pair[0][0]
+        for option in options:
+            cmd = "ethtool %s %s"%(option,
+                                   dev_name)
+            msg=self.session_ex.send_expect(cmd,
+                                        "# ", 
+                                        timeout=10)
+            temp = list()
+            for item in map(lambda x: x.strip().split(": "), 
+                            msg.strip().splitlines()):
+                if len(item) == 2 and item[0]:
+                    temp.append(item)
+                else:
+                    raise VerifyFailure("invalid data")
+            
+            infos = dict(temp)
+            time.sleep(1)
+
+    def preset_test_enviroment(self):
+        self.preset_testpmd()
+        time.sleep(2)
+
+    def clear_test_enviroment(self):
+        if not self.testpmd_flag:
+            return
+        self.testpmd.quit()
+        self.testpmd_flag = False
+
+    def clear_pmd_ports_stat(self):
+        options = ["--xstats-reset ", "--stats-reset "]
+        for option in options:
+            self.session_ex.send_expect(self.dpdk_proc_info + " %s"%(option),
+                                         "# ")
+            msg = self.session_ex.session.get_session_before(0.5)
+            self.logger.info(msg)
+            time.sleep(1)
+
+    def compare_with_expected_values(self, infos, expected_infos):
+        except_msg = []
+        for port, info in infos.items():
+            if port != '0':
+                continue
+            for stat_name, value in expected_infos.items():
+                if stat_name not in info.keys():
+                    msg = "port <{0}> {1} has no <{2}>"
+                    except_msg.append(msg.format(port,
+                                                self.kdriver,
+                                                stat_name))
+                    continue
+                if value != int(info[stat_name]):
+                    msg = "port <{0}>  <{1}> value is [{2}] | expect [{3}]"
+                    except_msg.append(msg.format(port,
+                                              stat_name,
+                                              info[stat_name],
+                                              value))
+        if except_msg:
+            self.logger.warning(os.linesep.join(except_msg))
+            raise VerifyFailure("xstat statitstics exception")
+
+    def test_xstat(self):
+        '''
+        test xstat command set intergrity
+        '''
+        pmd_cmds =[ ['set fwd io'],
+                    ['set verbose 1'],
+                    ['start']]
+
+        self.preset_test_enviroment()
+        self.execute_testpmd_cmd(pmd_cmds)
+        self.check_xstat_command_list()
+
+    def test_xstat_reset(self):
+        '''
+        test xstat-reset command
+        '''
+        pmd_cmds =[ ['set fwd io'],
+                    ['set verbose 1'],
+                    ['start']]
+
+        self.preset_test_enviroment()
+        self.execute_testpmd_cmd(pmd_cmds)
+        self.send_custom_frames()
+        self.clear_pmd_ports_stat()
+        self.check_xstat_reset_status()
+
+    def test_xstat_intergrity(self):
+        '''
+        test xstat command
+        '''
+        pmd_cmds =[ ['set fwd io'],
+                    ['set verbose 1'],
+                    ['start']]
+
+        self.preset_test_enviroment()
+        self.execute_testpmd_cmd(pmd_cmds)
+        self.query_dpdk_xstat_all()
+        self.check_xstat_statistic_intergrity()
+
+    def test_xstat_single_statistic(self):
+        '''
+        '''
+        pmd_cmds =[ #['set fwd rxonly'],
+                    ['set fwd io'],
+                    ['set verbose 1'],
+                    ['start']]
+
+        self.preset_test_enviroment()
+        self.execute_testpmd_cmd(pmd_cmds)
+        self.clear_pmd_ports_stat()
+        self.send_custom_frames()
+        self.query_dpdk_xstat_all()
+        self.check_xstat_single_statistic()
-- 
1.9.3



More information about the dts mailing list