[dpdk-dev] [PATCHv3 5/5] pmdinfo.py: Add tool to query binaries for hw and other support information
Neil Horman
nhorman at tuxdriver.com
Tue May 24 17:17:56 CEST 2016
On Tue, May 24, 2016 at 10:41:28AM +0300, Panu Matilainen wrote:
> On 05/20/2016 08:24 PM, Neil Horman wrote:
> > This tool searches for the primer sting PMD_DRIVER_INFO= in any ELF binary,
> > and, if found parses the remainder of the string as a json encoded string,
> > outputting the results in either a human readable or raw, script parseable
> > format
> >
> > Note that, in the case of dynamically linked applications, pmdinfo.py will scan
> > for implicitly linked PMDs by searching the specified binaries .dynamic section
> > for DT_NEEDED entries that contain the substring librte_pmd. The DT_RUNPATH,
> > LD_LIBRARY_PATH, /usr/lib and /lib are searched for these libraries, in that
> > order
> >
> > If a file is specified with no path, it is assumed to be a PMD DSO, and the
> > LD_LIBRARY_PATH, /usr/lib[64]/ and /lib[64] is searched for it
> >
> > Currently the tool can output data in 3 formats:
> >
> > a) raw, suitable for scripting, where the raw JSON strings are dumped out
> > b) table format (default) where hex pci ids are dumped in a table format
> > c) pretty, where a user supplied pci.ids file is used to print out vendor and
> > device strings
> >
> > Signed-off-by: Neil Horman <nhorman at tuxdriver.com>
> > CC: Bruce Richardson <bruce.richardson at intel.com>
> > CC: Thomas Monjalon <thomas.monjalon at 6wind.com>
> > CC: Stephen Hemminger <stephen at networkplumber.org>
> > CC: Panu Matilainen <pmatilai at redhat.com>
> > ---
> > tools/pmdinfo.py | 545 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
> > 1 file changed, 545 insertions(+)
> > create mode 100755 tools/pmdinfo.py
> >
> > diff --git a/tools/pmdinfo.py b/tools/pmdinfo.py
> > new file mode 100755
> > index 0000000..9b4b4a4
> > --- /dev/null
> > +++ b/tools/pmdinfo.py
> > @@ -0,0 +1,545 @@
> > +#!/usr/bin/python
> > +#-------------------------------------------------------------------------------
> > +# scripts/pmd_hw_support.py
> > +#
> > +# Utility to dump PMD_INFO_STRING support from an object file
> > +#
> > +#-------------------------------------------------------------------------------
> > +import os, sys
> > +from optparse import OptionParser
> > +import string
> > +import json
> > +
> > +# For running from development directory. It should take precedence over the
> > +# installed pyelftools.
> > +sys.path.insert(0, '.')
> > +
> > +
> > +from elftools import __version__
> > +from elftools.common.exceptions import ELFError
> > +from elftools.common.py3compat import (
> > + ifilter, byte2int, bytes2str, itervalues, str2bytes)
> > +from elftools.elf.elffile import ELFFile
> > +from elftools.elf.dynamic import DynamicSection, DynamicSegment
> > +from elftools.elf.enums import ENUM_D_TAG
> > +from elftools.elf.segments import InterpSegment
> > +from elftools.elf.sections import SymbolTableSection
> > +from elftools.elf.gnuversions import (
> > + GNUVerSymSection, GNUVerDefSection,
> > + GNUVerNeedSection,
> > + )
> > +from elftools.elf.relocation import RelocationSection
> > +from elftools.elf.descriptions import (
> > + describe_ei_class, describe_ei_data, describe_ei_version,
> > + describe_ei_osabi, describe_e_type, describe_e_machine,
> > + describe_e_version_numeric, describe_p_type, describe_p_flags,
> > + describe_sh_type, describe_sh_flags,
> > + describe_symbol_type, describe_symbol_bind, describe_symbol_visibility,
> > + describe_symbol_shndx, describe_reloc_type, describe_dyn_tag,
> > + describe_ver_flags,
> > + )
> > +from elftools.elf.constants import E_FLAGS
> > +from elftools.dwarf.dwarfinfo import DWARFInfo
> > +from elftools.dwarf.descriptions import (
> > + describe_reg_name, describe_attr_value, set_global_machine_arch,
> > + describe_CFI_instructions, describe_CFI_register_rule,
> > + describe_CFI_CFA_rule,
> > + )
> > +from elftools.dwarf.constants import (
> > + DW_LNS_copy, DW_LNS_set_file, DW_LNE_define_file)
> > +from elftools.dwarf.callframe import CIE, FDE
> > +
> > +raw_output = False
> > +pcidb = None
> > +
> > +#===========================================
> > +
> > +class Vendor:
> > + """
> > + Class for vendors. This is the top level class
> > + for the devices belong to a specific vendor.
> > + self.devices is the device dictionary
> > + subdevices are in each device.
> > + """
> > + def __init__(self, vendorStr):
> > + """
> > + Class initializes with the raw line from pci.ids
> > + Parsing takes place inside __init__
> > + """
> > + self.ID = vendorStr.split()[0]
> > + self.name = vendorStr.replace("%s " % self.ID,"").rstrip()
> > + self.devices = {}
> > +
> > + def addDevice(self, deviceStr):
> > + """
> > + Adds a device to self.devices
> > + takes the raw line from pci.ids
> > + """
> > + s = deviceStr.strip()
> > + devID = s.split()[0]
> > + if devID in self.devices:
> > + pass
> > + else:
> > + self.devices[devID] = Device(deviceStr)
> > +
> > + def report(self):
> > + print self.ID, self.name
> > + for id, dev in self.devices.items():
> > + dev.report()
> > +
> > + def find_device(self, devid):
> > + # convert to a hex string and remove 0x
> > + devid = hex(devid)[2:]
> > + try:
> > + return self.devices[devid]
> > + except:
> > + return Device("%s Unknown Device" % devid)
> > +
> > +class Device:
> > + def __init__(self, deviceStr):
> > + """
> > + Class for each device.
> > + Each vendor has its own devices dictionary.
> > + """
> > + s = deviceStr.strip()
> > + self.ID = s.split()[0]
> > + self.name = s.replace("%s " % self.ID,"")
> > + self.subdevices = {}
> > +
> > + def report(self):
> > + print "\t%s\t%s" % (self.ID, self.name)
> > + for subID, subdev in self.subdevices.items():
> > + subdev.report()
> > +
> > + def addSubDevice(self, subDeviceStr):
> > + """
> > + Adds a subvendor, subdevice to device.
> > + Uses raw line from pci.ids
> > + """
> > + s = subDeviceStr.strip()
> > + spl = s.split()
> > + subVendorID = spl[0]
> > + subDeviceID = spl[1]
> > + subDeviceName = s.split(" ")[-1]
> > + devID = "%s:%s" % (subVendorID,subDeviceID)
> > + self.subdevices[devID] = SubDevice(subVendorID,subDeviceID,subDeviceName)
> > +
> > + def find_subid(self, subven, subdev):
> > + subven = hex(subven)[2:]
> > + subdev = hex(subdev)[2:]
> > + devid ="%s:%s" % (subven, subdev)
> > +
> > + try:
> > + return self.subdevices[devid]
> > + except:
> > + if (subven == "ffff" and subdev == "ffff"):
> > + return SubDevice("ffff", "ffff", "(All Subdevices)");
> > + else:
> > + return SubDevice(subven, subdev, "(Unknown Subdevice)")
> > +
> > +
> > +class SubDevice:
> > + """
> > + Class for subdevices.
> > + """
> > + def __init__(self, vendor, device, name):
> > + """
> > + Class initializes with vendorid, deviceid and name
> > + """
> > + self.vendorID = vendor
> > + self.deviceID = device
> > + self.name = name
> > +
> > + def report(self):
> > + print "\t\t%s\t%s\t%s" % (self.vendorID, self.deviceID,self.name)
> > +
> > +class PCIIds:
> > + """
> > + Top class for all pci.ids entries.
> > + All queries will be asked to this class.
> > + PCIIds.vendors["0e11"].devices["0046"].subdevices["0e11:4091"].name = "Smart Array 6i"
> > + """
> > + def __init__(self, filename):
> > + """
> > + Prepares the directories.
> > + Checks local data file.
> > + Tries to load from local, if not found, downloads from web
> > + """
> > + self.version = ""
> > + self.date = ""
> > + self.vendors = {}
> > + self.contents = None
> > + self.readLocal(filename)
> > + self.parse()
> > +
> > + def reportVendors(self):
> > + """Reports the vendors
> > + """
> > + for vid, v in self.vendors.items():
> > + print v.ID, v.name
> > +
> > + def report(self, vendor = None):
> > + """
> > + Reports everything for all vendors or a specific vendor
> > + PCIIds.report() reports everything
> > + PCIIDs.report("0e11") reports only "Compaq Computer Corporation"
> > + """
> > + if vendor != None:
> > + self.vendors[vendor].report()
> > + else:
> > + for vID, v in self.vendors.items():
> > + v.report()
> > +
> > + def find_vendor(self, vid):
> > + # convert vid to a hex string and remove the 0x
> > + vid = hex(vid)[2:]
> > +
> > + try:
> > + return self.vendors[vid]
> > + except:
> > + return Vendor("%s Unknown Vendor" % (vid))
> > +
> > + def findDate(self, content):
> > + for l in content:
> > + if l.find("Date:") > -1:
> > + return l.split()[-2].replace("-", "")
> > + return None
> > +
> > + def parse(self):
> > + if len(self.contents) < 1:
> > + print "data/%s-pci.ids not found" % self.date
> > + else:
> > + vendorID = ""
> > + deviceID = ""
> > + for l in self.contents:
> > + if l[0] == "#":
> > + continue
> > + elif len(l.strip()) == 0:
> > + continue
> > + else:
> > + if l.find("\t\t") == 0:
> > + self.vendors[vendorID].devices[deviceID].addSubDevice(l)
> > + elif l.find("\t") == 0:
> > + deviceID = l.strip().split()[0]
> > + self.vendors[vendorID].addDevice(l)
> > + else:
> > + vendorID = l.split()[0]
> > + self.vendors[vendorID] = Vendor(l)
> > +
> > + def readLocal(self, filename):
> > + """
> > + Reads the local file
> > + """
> > + self.contents = open(filename).readlines()
> > + self.date = self.findDate(self.contents)
> > +
> > + def loadLocal(self):
> > + """
> > + Loads database from local. If there is no file,
> > + it creates a new one from web
> > + """
> > + self.date = idsfile[0].split("/")[1].split("-")[0]
> > + self.readLocal()
> > +
> > +
> > +
> > +#=======================================
> > +
> > +def search_file(filename, search_path):
> > + """ Given a search path, find file with requested name """
> > + for path in string.split(search_path, ":"):
> > + candidate = os.path.join(path, filename)
> > + if os.path.exists(candidate): return os.path.abspath(candidate)
> > + return None
> > +
> > +class ReadElf(object):
> > + """ display_* methods are used to emit output into the output stream
> > + """
> > + def __init__(self, file, output):
> > + """ file:
> > + stream object with the ELF file to read
> > +
> > + output:
> > + output stream to write to
> > + """
> > + self.elffile = ELFFile(file)
> > + self.output = output
> > +
> > + # Lazily initialized if a debug dump is requested
> > + self._dwarfinfo = None
> > +
> > + self._versioninfo = None
> > +
> > + def _section_from_spec(self, spec):
> > + """ Retrieve a section given a "spec" (either number or name).
> > + Return None if no such section exists in the file.
> > + """
> > + try:
> > + num = int(spec)
> > + if num < self.elffile.num_sections():
> > + return self.elffile.get_section(num)
> > + else:
> > + return None
> > + except ValueError:
> > + # Not a number. Must be a name then
> > + return self.elffile.get_section_by_name(str2bytes(spec))
> > +
> > + def pretty_print_pmdinfo(self, pmdinfo):
> > + global pcidb
> > +
> > + for i in pmdinfo["pci_ids"]:
> > + vendor = pcidb.find_vendor(i[0])
> > + device = vendor.find_device(i[1])
> > + subdev = device.find_subid(i[2], i[3])
> > + print("%s (%s) : %s (%s) %s" % (vendor.name, vendor.ID, device.name, device.ID, subdev.name))
> > +
> > + def parse_pmd_info_string(self, mystring):
> > + global raw_output
> > + global pcidb
> > +
> > + optional_pmd_info = [{'id': 'params', 'tag' : 'PMD PARAMETERS'}]
> > +
> > + i = mystring.index("=");
> > + mystring = mystring[i+2:]
> > + pmdinfo = json.loads(mystring)
> > +
> > + if raw_output:
> > + print(pmdinfo)
> > + return
> > +
> > + print("PMD NAME: " + pmdinfo["name"])
> > + print("PMD TYPE: " + pmdinfo["type"])
> > + for i in optional_pmd_info:
> > + try:
> > + print("%s: %s" % (i['tag'], pmdinfo[i['id']]))
> > + except KeyError as e:
> > + continue
> > +
> > + if (pmdinfo["type"] == "PMD_PDEV"):
> > + print("PMD HW SUPPORT:")
> > + if pcidb != None:
> > + self.pretty_print_pmdinfo(pmdinfo)
> > + else:
> > + print("VENDOR\t DEVICE\t SUBVENDOR\t SUBDEVICE")
> > + for i in pmdinfo["pci_ids"]:
> > + print("0x%04x\t 0x%04x\t 0x%04x\t\t 0x%04x" % (i[0], i[1], i[2], i[3]))
> > +
> > + print("")
> > +
> > +
> > + def display_pmd_info_strings(self, section_spec):
> > + """ Display a strings dump of a section. section_spec is either a
> > + section number or a name.
> > + """
> > + section = self._section_from_spec(section_spec)
> > + if section is None:
> > + return
> > +
> > +
> > + data = section.data()
> > + dataptr = 0
> > +
> > + while dataptr < len(data):
> > + while ( dataptr < len(data) and
> > + not (32 <= byte2int(data[dataptr]) <= 127)):
> > + dataptr += 1
> > +
> > + if dataptr >= len(data):
> > + break
> > +
> > + endptr = dataptr
> > + while endptr < len(data) and byte2int(data[endptr]) != 0:
> > + endptr += 1
> > +
> > + mystring = bytes2str(data[dataptr:endptr])
> > + rc = mystring.find("PMD_INFO_STRING")
> > + if (rc != -1):
> > + self.parse_pmd_info_string(mystring)
> > +
> > + dataptr = endptr
> > +
> > + def search_for_autoload_path(self):
> > + section = self._section_from_spec(".rodata")
> > + if section is None:
> > + return None
> > +
> > + data = section.data()
> > + dataptr = 0
> > +
> > + while dataptr < len(data):
> > + while ( dataptr < len(data) and
> > + not (32 <= byte2int(data[dataptr]) <= 127)):
> > + dataptr += 1
> > +
> > + if dataptr >= len(data):
> > + break
> > +
> > + endptr = dataptr
> > + while endptr < len(data) and byte2int(data[endptr]) != 0:
> > + endptr += 1
> > +
> > + mystring = bytes2str(data[dataptr:endptr])
> > + rc = mystring.find("DPDK_PLUGIN_PATH")
> > + if (rc != -1):
> > + rc = mystring.find("=")
> > + return mystring[rc+1:]
> > +
> > + dataptr = endptr
> > + return None
> > +
> > + def get_dt_runpath(self, dynsec):
> > + for tag in dynsec.iter_tags():
> > + if tag.entry.d_tag == 'DT_RUNPATH':
> > + return tag.runpath
> > + return ""
> > +
> > +
> > + def process_dt_needed_entries(self):
> > + """ Look to see if there are any DT_NEEDED entries in the binary
> > + And process those if there are
> > + """
> > + global raw_output
> > + runpath = ""
> > + ldlibpath = os.environ.get('LD_LIBRARY_PATH')
> > + if (ldlibpath == None):
> > + ldlibpath = ""
> > +
> > + dynsec = self._section_from_spec(".dynamic")
> > + try:
> > + runpath = self.get_dt_runpath(dynsec)
> > + except AttributeError:
> > + # dynsec is None, just return
> > + return
> > +
> > + for tag in dynsec.iter_tags():
> > + if tag.entry.d_tag == 'DT_NEEDED':
> > + rc = tag.needed.find("librte_pmd")
> > + if (rc != -1):
> > + library = search_file(tag.needed,
> > + runpath + ":" + ldlibpath +
> > + ":/usr/lib64:/lib64:/usr/lib:/lib")
> > + if (library != None):
> > + if (raw_output == False):
> > + print("Scanning %s for pmd information" % library)
> > + with open(library, 'rb') as file:
> > + libelf = ReadElf(file, sys.stdout)
> > + libelf.process_dt_needed_entries()
> > + libelf.display_pmd_info_strings(".rodata")
> > + file.close()
> > +
> > +def scan_autoload_path(autoload_path):
> > + global raw_output
> > +
> > + if os.path.exists(autoload_path) == False:
> > + return
> > +
> > + for d in os.listdir(autoload_path):
> > + dpath = os.path.join(autoload_path, d)
> > + if os.path.isdir(dpath):
> > + scan_autoload_path(dpath)
> > + if os.path.isfile(dpath):
> > + if (raw_output == False):
> > + print("Hw Support for library %s" % d)
> > + file = open(dpath, 'rb')
> > + readelf = ReadElf(file, sys.stdout)
> > + readelf.display_pmd_info_strings(".rodata")
> > + file.close()
>
> Might want to wrap ReadElf() in try-except ELFError to avoid traceback in
> case there are other files in the directory. Not that other files are
> expected there, but EAL ignores all non-pmd files there so it'd be good to
> match that.
>
Yup, I can do that.
> > +
> > +
> > +def scan_for_autoload_pmds(dpdk_path):
> > + """
> > + search the specified application or path for a pmd autoload path
> > + then scan said path for pmds and report hw support
> > + """
> > + global raw_output
> > +
> > + if (os.path.isfile(dpdk_path) == False):
> > + # Assume this is a directory and we need to scan librte_eal.so
> > + dpdk_path=os.path.join(dpdk_path, "librte_eal.so")
>
> The unversioned .so symlink is not guaranteed to be there, they're typically
> placed into -devel packages which wont be installed for "normal use"
> (developers are another story obviously)
>
> Also assuming there might be multiple versions of DPDK runtime
> parallel-installed, it might point to any of them so its a bit of a lottery.
>
Crap, thats right, if they're in the same directory path, then we have no way to
disambiguate that. Maybe we do need to just specify the binary, as the
DT_NEEDED entry points to the specific version the application will use.
> > +
> > + file = open(dpdk_path, 'rb')
> > + readelf = ReadElf(file, sys.stdout)
>
> Might want to wrap this in try-except ELFError, its a user-entered path.
>
yeah, as a rule, we should. I will
> > + autoload_path = readelf.search_for_autoload_path()
> > + if (autoload_path == None or autoload_path == ""):
> > + if (raw_output == False):
> > + print("No autoload path configured in %s" % dpdk_path)
> > + return
> > + if (raw_output == False):
> > + print("Found autoload path %s in %s" % (autoload_path, dpdk_path))
> > +
> > + file.close()
> > + if (raw_output == False):
> > + print("Discovered Autoload HW Support:")
> > + scan_autoload_path(autoload_path)
> > + return
> > +
> > +
> > +def main(stream=None):
> > + global raw_output
> > + global pcidb
> > +
> > + optparser = OptionParser(
> > + usage='usage: %prog [-hrt] [-p <dpdk dir>] [-d <pci id file] <elf-file>',
>
> Minor nit, but with the later additions the usage message isn't accurate
> anymore, <elf-file> could be a directory too so <file|dir> would be closer
> to the mark, and in plugin mode it's not looked at all. Might be simpler to
> both users + and in actual code if plugin mode is just a true/false flag and
> uses the same args[0] as argument as it too handles both a file and a
> directory case.
>
I'll clean this up as I finish the above
> > + description="Dump pmd hardware support info",
> > + add_help_option=True,
> > + prog='pmdinfo.py')
> > + optparser.add_option('-r', '--raw',
> > + action='store_true', dest='raw_output',
> > + help='Dump raw json strings')
> > + optparser.add_option("-d", "--pcidb", dest="pcifile",
> > + help="specify a pci database to get vendor names from",
> > + default="/usr/share/hwdata/pci.ids", metavar="FILE")
> > + optparser.add_option("-t", "--table", dest="tblout",
> > + help="output infomation on hw support as a hex table",
> > + action='store_true')
> > + optparser.add_option("-p", "--plugindir", dest="pdir",
> > + help="scan dpdk for autoload plugins", metavar="PATH|APP")
> > +
> > + options, args = optparser.parse_args()
> > +
> > + if options.raw_output:
> > + raw_output = True
> > +
> > + if options.pcifile:
> > + pcidb = PCIIds(options.pcifile)
> > + if pcidb == None:
> > + print("Pci DB file not found")
> > + exit(1)
> > +
> > + if options.tblout:
> > + options.pcifile = None
> > + pcidb = None
> > +
> > + if options.pdir:
> > + exit (scan_for_autoload_pmds(options.pdir))
> > +
> > + ldlibpath = os.environ.get('LD_LIBRARY_PATH')
> > + if (ldlibpath == None):
> > + ldlibpath = ""
> > +
> > + if (os.path.exists(args[0]) == True):
> > + myelffile = args[0]
> > + else:
> > + myelffile = search_file(args[0], ldlibpath + ":/usr/lib64:/lib64:/usr/lib:/lib")
>
> Minor nit, but this tracebacks when executing pmdinfo.py without arguments.
> Would be nicer to dump usage instead.
>
Yup
> [pmatilai at sopuli dpdk]$ tools/pmdinfo.py
> Traceback (most recent call last):
> File "tools/pmdinfo.py", line 543, in <module>
> main()
> File "tools/pmdinfo.py", line 520, in main
> if (os.path.exists(args[0]) == True):
> IndexError: list index out of range
>
> - Panu -
>
> > +
> > + if (myelffile == None):
> > + print("File not found")
> > + sys.exit(1)
> > +
> > + with open(myelffile, 'rb') as file:
> > + try:
> > + readelf = ReadElf(file, sys.stdout)
> > + readelf.process_dt_needed_entries()
> > + readelf.display_pmd_info_strings(".rodata")
> > + sys.exit(0)
> > +
> > + except ELFError as ex:
> > + sys.stderr.write('ELF error: %s\n' % ex)
> > + sys.exit(1)
> > +
> > +
> > +#-------------------------------------------------------------------------------
> > +if __name__ == '__main__':
> > + main()
> > +
> > +
> >
>
>
More information about the dev
mailing list