[dpdk-dev] [PATCHv3 5/5] pmdinfo.py: Add tool to query binaries for hw and other support information

Neil Horman nhorman at tuxdriver.com
Tue May 24 17:17:56 CEST 2016


On Tue, May 24, 2016 at 10:41:28AM +0300, Panu Matilainen wrote:
> On 05/20/2016 08:24 PM, Neil Horman wrote:
> > This tool searches for the primer sting PMD_DRIVER_INFO= in any ELF binary,
> > and, if found parses the remainder of the string as a json encoded string,
> > outputting the results in either a human readable or raw, script parseable
> > format
> > 
> > Note that, in the case of dynamically linked applications, pmdinfo.py will scan
> > for implicitly linked PMDs by searching the specified binaries .dynamic section
> > for DT_NEEDED entries that contain the substring librte_pmd.  The DT_RUNPATH,
> > LD_LIBRARY_PATH, /usr/lib and /lib are searched for these libraries, in that
> > order
> > 
> > If a file is specified with no path, it is assumed to be a PMD DSO, and the
> > LD_LIBRARY_PATH, /usr/lib[64]/ and /lib[64] is searched for it
> > 
> > Currently the tool can output data in 3 formats:
> > 
> > a) raw, suitable for scripting, where the raw JSON strings are dumped out
> > b) table format (default) where hex pci ids are dumped in a table format
> > c) pretty, where a user supplied pci.ids file is used to print out vendor and
> > device strings
> > 
> > Signed-off-by: Neil Horman <nhorman at tuxdriver.com>
> > CC: Bruce Richardson <bruce.richardson at intel.com>
> > CC: Thomas Monjalon <thomas.monjalon at 6wind.com>
> > CC: Stephen Hemminger <stephen at networkplumber.org>
> > CC: Panu Matilainen <pmatilai at redhat.com>
> > ---
> >  tools/pmdinfo.py | 545 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
> >  1 file changed, 545 insertions(+)
> >  create mode 100755 tools/pmdinfo.py
> > 
> > diff --git a/tools/pmdinfo.py b/tools/pmdinfo.py
> > new file mode 100755
> > index 0000000..9b4b4a4
> > --- /dev/null
> > +++ b/tools/pmdinfo.py
> > @@ -0,0 +1,545 @@
> > +#!/usr/bin/python
> > +#-------------------------------------------------------------------------------
> > +# scripts/pmd_hw_support.py
> > +#
> > +# Utility to dump PMD_INFO_STRING support from an object file
> > +#
> > +#-------------------------------------------------------------------------------
> > +import os, sys
> > +from optparse import OptionParser
> > +import string
> > +import json
> > +
> > +# For running from development directory. It should take precedence over the
> > +# installed pyelftools.
> > +sys.path.insert(0, '.')
> > +
> > +
> > +from elftools import __version__
> > +from elftools.common.exceptions import ELFError
> > +from elftools.common.py3compat import (
> > +        ifilter, byte2int, bytes2str, itervalues, str2bytes)
> > +from elftools.elf.elffile import ELFFile
> > +from elftools.elf.dynamic import DynamicSection, DynamicSegment
> > +from elftools.elf.enums import ENUM_D_TAG
> > +from elftools.elf.segments import InterpSegment
> > +from elftools.elf.sections import SymbolTableSection
> > +from elftools.elf.gnuversions import (
> > +    GNUVerSymSection, GNUVerDefSection,
> > +    GNUVerNeedSection,
> > +    )
> > +from elftools.elf.relocation import RelocationSection
> > +from elftools.elf.descriptions import (
> > +    describe_ei_class, describe_ei_data, describe_ei_version,
> > +    describe_ei_osabi, describe_e_type, describe_e_machine,
> > +    describe_e_version_numeric, describe_p_type, describe_p_flags,
> > +    describe_sh_type, describe_sh_flags,
> > +    describe_symbol_type, describe_symbol_bind, describe_symbol_visibility,
> > +    describe_symbol_shndx, describe_reloc_type, describe_dyn_tag,
> > +    describe_ver_flags,
> > +    )
> > +from elftools.elf.constants import E_FLAGS
> > +from elftools.dwarf.dwarfinfo import DWARFInfo
> > +from elftools.dwarf.descriptions import (
> > +    describe_reg_name, describe_attr_value, set_global_machine_arch,
> > +    describe_CFI_instructions, describe_CFI_register_rule,
> > +    describe_CFI_CFA_rule,
> > +    )
> > +from elftools.dwarf.constants import (
> > +    DW_LNS_copy, DW_LNS_set_file, DW_LNE_define_file)
> > +from elftools.dwarf.callframe import CIE, FDE
> > +
> > +raw_output = False
> > +pcidb = None
> > +
> > +#===========================================
> > +
> > +class Vendor:
> > +    """
> > +    Class for vendors. This is the top level class
> > +    for the devices belong to a specific vendor.
> > +    self.devices is the device dictionary
> > +    subdevices are in each device.
> > +    """
> > +    def __init__(self, vendorStr):
> > +        """
> > +        Class initializes with the raw line from pci.ids
> > +        Parsing takes place inside __init__
> > +        """
> > +        self.ID = vendorStr.split()[0]
> > +        self.name = vendorStr.replace("%s " % self.ID,"").rstrip()
> > +        self.devices = {}
> > +
> > +    def addDevice(self, deviceStr):
> > +        """
> > +        Adds a device to self.devices
> > +        takes the raw line from pci.ids
> > +        """
> > +        s = deviceStr.strip()
> > +        devID = s.split()[0]
> > +        if devID in self.devices:
> > +            pass
> > +        else:
> > +            self.devices[devID] = Device(deviceStr)
> > +
> > +    def report(self):
> > +        print self.ID, self.name
> > +        for id, dev in self.devices.items():
> > +            dev.report()
> > +
> > +    def find_device(self, devid):
> > +        # convert to a hex string and remove 0x
> > +        devid = hex(devid)[2:]
> > +        try:
> > +            return self.devices[devid]
> > +        except:
> > +            return Device("%s  Unknown Device" % devid)
> > +
> > +class Device:
> > +    def __init__(self, deviceStr):
> > +        """
> > +        Class for each device.
> > +        Each vendor has its own devices dictionary.
> > +        """
> > +        s = deviceStr.strip()
> > +        self.ID = s.split()[0]
> > +        self.name = s.replace("%s  " % self.ID,"")
> > +        self.subdevices = {}
> > +
> > +    def report(self):
> > +        print "\t%s\t%s" % (self.ID, self.name)
> > +        for subID, subdev in self.subdevices.items():
> > +            subdev.report()
> > +
> > +    def addSubDevice(self, subDeviceStr):
> > +        """
> > +        Adds a subvendor, subdevice to device.
> > +        Uses raw line from pci.ids
> > +        """
> > +        s = subDeviceStr.strip()
> > +        spl = s.split()
> > +        subVendorID  = spl[0]
> > +        subDeviceID  = spl[1]
> > +        subDeviceName = s.split("  ")[-1]
> > +        devID = "%s:%s" % (subVendorID,subDeviceID)
> > +        self.subdevices[devID] = SubDevice(subVendorID,subDeviceID,subDeviceName)
> > +
> > +    def find_subid(self, subven, subdev):
> > +        subven = hex(subven)[2:]
> > +        subdev = hex(subdev)[2:]
> > +        devid ="%s:%s" % (subven, subdev)
> > +
> > +        try:
> > +            return self.subdevices[devid]
> > +        except:
> > +            if (subven == "ffff" and subdev == "ffff"):
> > +                return SubDevice("ffff", "ffff", "(All Subdevices)");
> > +            else:
> > +                return SubDevice(subven, subdev, "(Unknown Subdevice)")
> > +
> > +
> > +class SubDevice:
> > +    """
> > +    Class for subdevices.
> > +    """
> > +    def __init__(self, vendor, device, name):
> > +        """
> > +        Class initializes with vendorid, deviceid and name
> > +        """
> > +        self.vendorID = vendor
> > +        self.deviceID = device
> > +        self.name = name
> > +
> > +    def report(self):
> > +        print "\t\t%s\t%s\t%s" % (self.vendorID, self.deviceID,self.name)
> > +
> > +class PCIIds:
> > +    """
> > +    Top class for all pci.ids entries.
> > +    All queries will be asked to this class.
> > +    PCIIds.vendors["0e11"].devices["0046"].subdevices["0e11:4091"].name  =  "Smart Array 6i"
> > +    """
> > +    def __init__(self, filename):
> > +        """
> > +        Prepares the directories.
> > +        Checks local data file.
> > +        Tries to load from local, if not found, downloads from web
> > +        """
> > +        self.version = ""
> > +        self.date = ""
> > +        self.vendors = {}
> > +        self.contents = None
> > +        self.readLocal(filename)
> > +        self.parse()
> > +
> > +    def reportVendors(self):
> > +        """Reports the vendors
> > +        """
> > +        for vid, v in self.vendors.items():
> > +            print v.ID, v.name
> > +
> > +    def report(self, vendor = None):
> > +        """
> > +        Reports everything for all vendors or a specific vendor
> > +        PCIIds.report()  reports everything
> > +        PCIIDs.report("0e11") reports only "Compaq Computer Corporation"
> > +        """
> > +        if vendor != None:
> > +            self.vendors[vendor].report()
> > +        else:
> > +            for vID, v in self.vendors.items():
> > +                v.report()
> > +
> > +    def find_vendor(self, vid):
> > +        # convert vid to a hex string and remove the 0x
> > +        vid = hex(vid)[2:]
> > +
> > +        try:
> > +            return self.vendors[vid]
> > +        except:
> > +            return Vendor("%s Unknown Vendor" % (vid))
> > +
> > +    def findDate(self, content):
> > +        for l in content:
> > +            if l.find("Date:") > -1:
> > +                return l.split()[-2].replace("-", "")
> > +        return None
> > +
> > +    def parse(self):
> > +        if len(self.contents) < 1:
> > +            print "data/%s-pci.ids not found" % self.date
> > +        else:
> > +            vendorID = ""
> > +            deviceID = ""
> > +            for l in self.contents:
> > +                if l[0] == "#":
> > +                    continue
> > +                elif len(l.strip()) == 0:
> > +                    continue
> > +                else:
> > +                    if l.find("\t\t") == 0:
> > +                        self.vendors[vendorID].devices[deviceID].addSubDevice(l)
> > +                    elif l.find("\t") == 0:
> > +                        deviceID = l.strip().split()[0]
> > +                        self.vendors[vendorID].addDevice(l)
> > +                    else:
> > +                        vendorID = l.split()[0]
> > +                        self.vendors[vendorID] = Vendor(l)
> > +
> > +    def readLocal(self, filename):
> > +        """
> > +        Reads the local file
> > +        """
> > +        self.contents = open(filename).readlines()
> > +        self.date = self.findDate(self.contents)
> > +
> > +    def loadLocal(self):
> > +        """
> > +        Loads database from local. If there is no file,
> > +        it creates a new one from web
> > +        """
> > +        self.date = idsfile[0].split("/")[1].split("-")[0]
> > +        self.readLocal()
> > +
> > +
> > +
> > +#=======================================
> > +
> > +def search_file(filename, search_path):
> > +    """ Given a search path, find file with requested name """
> > +    for path in string.split(search_path, ":"):
> > +        candidate = os.path.join(path, filename)
> > +        if os.path.exists(candidate): return os.path.abspath(candidate)
> > +    return None
> > +
> > +class ReadElf(object):
> > +    """ display_* methods are used to emit output into the output stream
> > +    """
> > +    def __init__(self, file, output):
> > +        """ file:
> > +                stream object with the ELF file to read
> > +
> > +            output:
> > +                output stream to write to
> > +        """
> > +        self.elffile = ELFFile(file)
> > +        self.output = output
> > +
> > +        # Lazily initialized if a debug dump is requested
> > +        self._dwarfinfo = None
> > +
> > +        self._versioninfo = None
> > +
> > +    def _section_from_spec(self, spec):
> > +        """ Retrieve a section given a "spec" (either number or name).
> > +            Return None if no such section exists in the file.
> > +        """
> > +        try:
> > +            num = int(spec)
> > +            if num < self.elffile.num_sections():
> > +                return self.elffile.get_section(num)
> > +            else:
> > +                return None
> > +        except ValueError:
> > +            # Not a number. Must be a name then
> > +            return self.elffile.get_section_by_name(str2bytes(spec))
> > +
> > +    def pretty_print_pmdinfo(self, pmdinfo):
> > +        global pcidb
> > +
> > +        for i in pmdinfo["pci_ids"]:
> > +            vendor = pcidb.find_vendor(i[0])
> > +            device = vendor.find_device(i[1])
> > +            subdev = device.find_subid(i[2], i[3])
> > +            print("%s (%s) : %s (%s) %s" % (vendor.name, vendor.ID, device.name, device.ID, subdev.name))
> > +
> > +    def parse_pmd_info_string(self, mystring):
> > +        global raw_output
> > +        global pcidb
> > +
> > +        optional_pmd_info = [{'id': 'params', 'tag' : 'PMD PARAMETERS'}]
> > +
> > +        i = mystring.index("=");
> > +        mystring = mystring[i+2:]
> > +        pmdinfo = json.loads(mystring)
> > +
> > +        if raw_output:
> > +            print(pmdinfo)
> > +            return
> > +
> > +        print("PMD NAME: " + pmdinfo["name"])
> > +        print("PMD TYPE: " + pmdinfo["type"])
> > +        for i in optional_pmd_info:
> > +            try:
> > +                print("%s: %s" % (i['tag'], pmdinfo[i['id']]))
> > +            except KeyError as e:
> > +                continue
> > +
> > +        if (pmdinfo["type"] == "PMD_PDEV"):
> > +            print("PMD HW SUPPORT:")
> > +            if pcidb != None:
> > +                self.pretty_print_pmdinfo(pmdinfo)
> > +            else:
> > +                print("VENDOR\t DEVICE\t SUBVENDOR\t SUBDEVICE")
> > +                for i in pmdinfo["pci_ids"]:
> > +                    print("0x%04x\t 0x%04x\t 0x%04x\t\t 0x%04x" % (i[0], i[1], i[2], i[3]))
> > +
> > +        print("")
> > +
> > +
> > +    def display_pmd_info_strings(self, section_spec):
> > +        """ Display a strings dump of a section. section_spec is either a
> > +            section number or a name.
> > +        """
> > +        section = self._section_from_spec(section_spec)
> > +        if section is None:
> > +            return
> > +
> > +
> > +        data = section.data()
> > +        dataptr = 0
> > +
> > +        while dataptr < len(data):
> > +            while ( dataptr < len(data) and
> > +                    not (32 <= byte2int(data[dataptr]) <= 127)):
> > +                dataptr += 1
> > +
> > +            if dataptr >= len(data):
> > +                break
> > +
> > +            endptr = dataptr
> > +            while endptr < len(data) and byte2int(data[endptr]) != 0:
> > +                endptr += 1
> > +
> > +            mystring = bytes2str(data[dataptr:endptr])
> > +            rc = mystring.find("PMD_INFO_STRING")
> > +            if (rc != -1):
> > +                self.parse_pmd_info_string(mystring)
> > +
> > +            dataptr = endptr
> > +
> > +    def search_for_autoload_path(self):
> > +        section = self._section_from_spec(".rodata")
> > +        if section is None:
> > +            return None
> > +
> > +        data = section.data()
> > +        dataptr = 0
> > +
> > +        while dataptr < len(data):
> > +            while ( dataptr < len(data) and
> > +                    not (32 <= byte2int(data[dataptr]) <= 127)):
> > +                dataptr += 1
> > +
> > +            if dataptr >= len(data):
> > +                break
> > +
> > +            endptr = dataptr
> > +            while endptr < len(data) and byte2int(data[endptr]) != 0:
> > +                endptr += 1
> > +
> > +            mystring = bytes2str(data[dataptr:endptr])
> > +            rc = mystring.find("DPDK_PLUGIN_PATH")
> > +            if (rc != -1):
> > +                rc = mystring.find("=")
> > +                return mystring[rc+1:]
> > +
> > +            dataptr = endptr
> > +        return None
> > +
> > +    def get_dt_runpath(self, dynsec):
> > +        for tag in dynsec.iter_tags():
> > +            if tag.entry.d_tag == 'DT_RUNPATH':
> > +                return tag.runpath
> > +        return ""
> > +
> > +
> > +    def process_dt_needed_entries(self):
> > +        """ Look to see if there are any DT_NEEDED entries in the binary
> > +            And process those if there are
> > +        """
> > +        global raw_output
> > +        runpath = ""
> > +        ldlibpath = os.environ.get('LD_LIBRARY_PATH')
> > +        if (ldlibpath == None):
> > +            ldlibpath = ""
> > +
> > +        dynsec = self._section_from_spec(".dynamic")
> > +        try:
> > +            runpath = self.get_dt_runpath(dynsec)
> > +        except AttributeError:
> > +            # dynsec is None, just return
> > +            return
> > +
> > +        for tag in dynsec.iter_tags():
> > +            if tag.entry.d_tag == 'DT_NEEDED':
> > +                rc = tag.needed.find("librte_pmd")
> > +                if (rc != -1):
> > +                    library = search_file(tag.needed,
> > +                            runpath + ":" + ldlibpath +
> > +                            ":/usr/lib64:/lib64:/usr/lib:/lib")
> > +                    if (library != None):
> > +                        if (raw_output == False):
> > +                            print("Scanning %s for pmd information" % library)
> > +                        with open(library, 'rb') as file:
> > +                            libelf = ReadElf(file, sys.stdout)
> > +                            libelf.process_dt_needed_entries()
> > +                            libelf.display_pmd_info_strings(".rodata")
> > +                            file.close()
> > +
> > +def scan_autoload_path(autoload_path):
> > +    global raw_output
> > +
> > +    if os.path.exists(autoload_path) == False:
> > +        return
> > +
> > +    for d in os.listdir(autoload_path):
> > +        dpath = os.path.join(autoload_path, d)
> > +        if os.path.isdir(dpath):
> > +            scan_autoload_path(dpath)
> > +        if os.path.isfile(dpath):
> > +            if (raw_output == False):
> > +                print("Hw Support for library %s" % d)
> > +            file = open(dpath, 'rb')
> > +            readelf = ReadElf(file, sys.stdout)
> > +            readelf.display_pmd_info_strings(".rodata")
> > +            file.close()
> 
> Might want to wrap ReadElf() in try-except ELFError to avoid traceback in
> case there are other files in the directory. Not that other files are
> expected there, but EAL ignores all non-pmd files there so it'd be good to
> match that.
> 
Yup, I can do that.

> > +
> > +
> > +def scan_for_autoload_pmds(dpdk_path):
> > +    """
> > +    search the specified application or path for a pmd autoload path
> > +    then scan said path for pmds and report hw support
> > +    """
> > +    global raw_output
> > +
> > +    if (os.path.isfile(dpdk_path) == False):
> > +        # Assume this is a directory and we need to scan librte_eal.so
> > +        dpdk_path=os.path.join(dpdk_path, "librte_eal.so")
> 
> The unversioned .so symlink is not guaranteed to be there, they're typically
> placed into -devel packages which wont be installed for "normal use"
> (developers are another story obviously)
> 
> Also assuming there might be multiple versions of DPDK runtime
> parallel-installed, it might point to any of them so its a bit of a lottery.
> 
Crap, thats right, if they're in the same directory path, then we have no way to
disambiguate that.  Maybe we do need to just specify the binary, as the
DT_NEEDED entry points to the specific version the application will use.


> > +
> > +    file = open(dpdk_path, 'rb')
> > +    readelf = ReadElf(file, sys.stdout)
> 
> Might want to wrap this in try-except ELFError, its a user-entered path.
> 
yeah, as a rule, we should. I will

> > +    autoload_path = readelf.search_for_autoload_path()
> > +    if (autoload_path == None or autoload_path == ""):
> > +        if (raw_output == False):
> > +            print("No autoload path configured in %s" % dpdk_path)
> > +        return
> > +    if (raw_output == False):
> > +        print("Found autoload path %s in %s" % (autoload_path, dpdk_path))
> > +
> > +    file.close()
> > +    if (raw_output == False):
> > +        print("Discovered Autoload HW Support:")
> > +    scan_autoload_path(autoload_path)
> > +    return
> > +
> > +
> > +def main(stream=None):
> > +    global raw_output
> > +    global pcidb
> > +
> > +    optparser = OptionParser(
> > +            usage='usage: %prog [-hrt] [-p <dpdk dir>] [-d <pci id file] <elf-file>',
> 
> Minor nit, but with the later additions the usage message isn't accurate
> anymore, <elf-file> could be a directory too so <file|dir> would be closer
> to the mark, and in plugin mode it's not looked at all. Might be simpler to
> both users + and in actual code if plugin mode is just a true/false flag and
> uses the same args[0] as argument as it too handles both a file and a
> directory case.
> 
I'll clean this up as I finish the above

> > +            description="Dump pmd hardware support info",
> > +            add_help_option=True,
> > +            prog='pmdinfo.py')
> > +    optparser.add_option('-r', '--raw',
> > +            action='store_true', dest='raw_output',
> > +            help='Dump raw json strings')
> > +    optparser.add_option("-d", "--pcidb", dest="pcifile",
> > +            help="specify a pci database to get vendor names from",
> > +            default="/usr/share/hwdata/pci.ids", metavar="FILE")
> > +    optparser.add_option("-t", "--table", dest="tblout",
> > +            help="output infomation on hw support as a hex table",
> > +            action='store_true')
> > +    optparser.add_option("-p", "--plugindir", dest="pdir",
> > +            help="scan dpdk for autoload plugins", metavar="PATH|APP")
> > +
> > +    options, args = optparser.parse_args()
> > +
> > +    if options.raw_output:
> > +        raw_output = True
> > +
> > +    if options.pcifile:
> > +        pcidb = PCIIds(options.pcifile)
> > +        if pcidb == None:
> > +            print("Pci DB file not found")
> > +            exit(1)
> > +
> > +    if options.tblout:
> > +        options.pcifile = None
> > +        pcidb = None
> > +
> > +    if options.pdir:
> > +        exit (scan_for_autoload_pmds(options.pdir))
> > +
> > +    ldlibpath = os.environ.get('LD_LIBRARY_PATH')
> > +    if (ldlibpath == None):
> > +        ldlibpath = ""
> > +
> > +    if (os.path.exists(args[0]) == True):
> > +        myelffile = args[0]
> > +    else:
> > +        myelffile = search_file(args[0], ldlibpath + ":/usr/lib64:/lib64:/usr/lib:/lib")
> 
> Minor nit, but this tracebacks when executing pmdinfo.py without arguments.
> Would be nicer to dump usage instead.
> 
Yup

> [pmatilai at sopuli dpdk]$ tools/pmdinfo.py
> Traceback (most recent call last):
>   File "tools/pmdinfo.py", line 543, in <module>
>     main()
>   File "tools/pmdinfo.py", line 520, in main
>     if (os.path.exists(args[0]) == True):
> IndexError: list index out of range
> 
> 	- Panu -
> 
> > +
> > +    if (myelffile == None):
> > +        print("File not found")
> > +        sys.exit(1)
> > +
> > +    with open(myelffile, 'rb') as file:
> > +        try:
> > +            readelf = ReadElf(file, sys.stdout)
> > +            readelf.process_dt_needed_entries()
> > +            readelf.display_pmd_info_strings(".rodata")
> > +            sys.exit(0)
> > +
> > +        except ELFError as ex:
> > +            sys.stderr.write('ELF error: %s\n' % ex)
> > +            sys.exit(1)
> > +
> > +
> > +#-------------------------------------------------------------------------------
> > +if __name__ == '__main__':
> > +    main()
> > +
> > +
> > 
> 
> 


More information about the dev mailing list