[dpdk-dev] [PATCH v9 1/2] devtools: script to track symbols over releases

Ray Kinsella mdr at ashroe.eu
Mon Aug 9 14:53:50 CEST 2021


This script tracks the growth of stable and experimental symbols
over releases since v19.11. The script has the ability to
count the added symbols between two dpdk releases, and to
list experimental symbols present in two dpdk releases
(expired symbols).

example usages:

Count symbols added since v19.11
$ devtools/symbol-tool.py count-symbols

Count symbols added since v20.11
$ devtools/symbol-tool.py count-symbols --releases v20.11,v21.05

List experimental symbols present in v20.11 and v21.05
$ devtools/symbol-tool.py list-expired --releases v20.11,v21.05

List experimental symbols in libraries only, present since v19.11
$ devtools/symbol-tool.py list-expired --directory lib

Signed-off-by: Ray Kinsella <mdr at ashroe.eu>
---
 devtools/symbol-tool.py | 402 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 402 insertions(+)
 create mode 100755 devtools/symbol-tool.py

diff --git a/devtools/symbol-tool.py b/devtools/symbol-tool.py
new file mode 100755
index 0000000000..4a357579dc
--- /dev/null
+++ b/devtools/symbol-tool.py
@@ -0,0 +1,402 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2021 Intel Corporation
+'''Tool to count or list symbols in each DPDK release'''
+from pathlib import Path
+import sys
+import os
+import subprocess
+import argparse
+from argparse import RawTextHelpFormatter
+import re
+import datetime
+try:
+    from parsley import makeGrammar
+except ImportError:
+    print('This script uses the package Parsley to parse C Mapfiles.\n'
+          'This can be installed with \"pip install parsley".')
+    sys.exit()
+
+DESCRIPTION = '''
+This script tracks the growth of stable and experimental symbols
+over releases since v19.11. The script has the ability to
+count the added symbols between two dpdk releases, and to
+list experimental symbols present in two dpdk releases
+(expired symbols).
+
+example usages:
+
+Count symbols added since v19.11
+$ {s} count-symbols
+
+Count symbols added since v20.11
+$ {s} count-symbols --releases v20.11,v21.05
+
+List experimental symbols present in v20.11 and v21.05
+$ {s} list-expired --releases v20.11,v21.05
+
+List experimental symbols in libraries only, present since v19.11
+$ {s} list-expired --directory lib
+'''
+
+MAP_GRAMMAR = r"""
+
+ws = (' ' | '\r' | '\n' | '\t')*
+
+ABI_VER = ({})
+DPDK_VER = ('DPDK_' ABI_VER)
+ABI_NAME = ('INTERNAL' | 'EXPERIMENTAL' | DPDK_VER)
+comment = '#' (~'\n' anything)+ '\n'
+symbol = (~(';' | '}}' | '#') anything )+:c ';' -> ''.join(c)
+global = 'global:'
+local = 'local: *;'
+symbols = comment* symbol:s ws comment* -> s
+
+abi = (abi_section+):m -> dict(m)
+abi_section = (ws ABI_NAME:e ws '{{' ws global* (~local ws symbols)*:s ws local* ws '}}' ws DPDK_VER* ';' ws) -> (e,s)
+"""
+
+def get_abi_versions():
+    '''Returns a string of possible dpdk abi versions'''
+
+    year = datetime.date.today().year - 2000
+    tags = " |".join(['\'{}\''.format(i) \
+                     for i in reversed(range(21, year + 1)) ])
+    tags  = tags + ' | \'20.0.1\' | \'20.0\' | \'20\''
+
+    return tags
+
+def get_dpdk_releases():
+    '''Returns a list of dpdk release tags names  since v19.11'''
+
+    year = datetime.date.today().year - 2000
+    year_range = "|".join("{}".format(i) for i in range(19,year + 1))
+    pattern = re.compile(r'^\"v(' +  year_range + r')\.\d{2}\"$')
+
+    cmd = ['git', 'for-each-ref', '--sort=taggerdate', '--format', '"%(tag)"']
+    try:
+        result = subprocess.run(cmd, \
+                                stdout=subprocess.PIPE, \
+                                stderr=subprocess.PIPE,
+                                check=True)
+    except subprocess.CalledProcessError:
+        print("Failed to interogate git for release tags")
+        sys.exit()
+
+
+    tags = result.stdout.decode('utf-8').split('\n')
+
+    # find the non-rcs between now and v19.11
+    tags = [ tag.replace('\"','') \
+             for tag in reversed(tags) \
+             if pattern.match(tag) ][:-3]
+
+    return tags
+
+def fix_directory_name(path):
+    '''Prepend librte to the source directory name'''
+    mapfilepath1 = str(path.parent.name)
+    mapfilepath2 = str(path.parents[1])
+    mapfilepath = mapfilepath2 + '/librte_' + mapfilepath1
+
+    return mapfilepath
+
+def directory_renamed(path, rel):
+    '''Fix removal of the librte_ from the directory names'''
+
+    mapfilepath = fix_directory_name(path)
+    tagfile = '{}:{}/{}'.format(rel, mapfilepath,  path.name)
+
+    try:
+        result = subprocess.run(['git', 'show', tagfile], \
+                                stdout=subprocess.PIPE, \
+                                stderr=subprocess.PIPE,
+                                check=True)
+    except subprocess.CalledProcessError:
+        result = None
+
+    return result
+
+def mapfile_renamed(path, rel):
+    '''Fix renaming of the map file'''
+    newfile = None
+
+    result = subprocess.run(['git', 'ls-tree', \
+                             rel, str(path.parent) + '/'], \
+                            stdout=subprocess.PIPE, \
+                            stderr=subprocess.PIPE,
+                            check=True)
+    dentries = result.stdout.decode('utf-8')
+    dentries = dentries.split('\n')
+
+    # filter entries looking for the map file
+    dentries = [dentry for dentry in dentries if dentry.endswith('.map')]
+    if len(dentries) > 1 or len(dentries) == 0:
+        return None
+
+    dparts = dentries[0].split('/')
+    newfile = dparts[len(dparts) - 1]
+
+    if newfile is not None:
+        tagfile = '{}:{}/{}'.format(rel, path.parent, newfile)
+
+        try:
+            result = subprocess.run(['git', 'show', tagfile], \
+                                    stdout=subprocess.PIPE, \
+                                    stderr=subprocess.PIPE,
+                                    check=True)
+        except subprocess.CalledProcessError:
+            result = None
+
+    else:
+        result = None
+
+    return result
+
+def mapfile_and_directory_renamed(path, rel):
+    '''Fix renaming of the map file & the source directory'''
+    mapfilepath = Path("{}/{}".format(fix_directory_name(path),path.name))
+
+    return mapfile_renamed(mapfilepath, rel)
+
+FIX_STRATEGIES = [directory_renamed, \
+                  mapfile_renamed, \
+                  mapfile_and_directory_renamed]
+
+def get_symbols(map_parser, release, mapfile_path):
+    '''Count the symbols for a given release and mapfile'''
+    abi_sections = {}
+
+    tagfile = '{}:{}'.format(release,mapfile_path)
+    try:
+        result = subprocess.run(['git', 'show', tagfile], \
+                                stdout=subprocess.PIPE, \
+                                stderr=subprocess.PIPE,
+                                check=True)
+    except subprocess.CalledProcessError:
+        result = None
+
+    for fix_strategy in FIX_STRATEGIES:
+        if result is not None:
+            break
+        result = fix_strategy(mapfile_path, release)
+
+    if result is not None:
+        mapfile = result.stdout.decode('utf-8')
+        abi_sections = map_parser(mapfile).abi()
+
+    return abi_sections
+
+def get_terminal_rows():
+    '''Find the number of rows in the terminal'''
+
+    try:
+        return os.get_terminal_size().lines
+    except IOError:
+        return 0
+
+class SymbolCountOutput():
+    '''Format the output to supported formats'''
+    output_fmt = ""
+    column_fmt = ""
+
+    def __init__(self, format_output, dpdk_releases):
+        self.OUTPUT_FORMATS[format_output](self,dpdk_releases)
+        self.column_titles = ['mapfile'] +  dpdk_releases
+
+        self.terminal_rows = get_terminal_rows()
+        self.row = 0
+
+    def set_terminal_output(self,dpdk_rel):
+        '''Set the output format to Tabbed Separated Values'''
+
+        self.output_fmt = '{:<50}' + \
+            ''.join(['{:<6}{:<6}'] * (len(dpdk_rel)))
+        self.column_fmt = '{:50}' + \
+            ''.join(['{:<12}'] * (len(dpdk_rel)))
+
+    def set_csv_output(self,dpdk_rel):
+        '''Set the output format to Comma Separated Values'''
+
+        self.output_fmt = '{},' + \
+            ','.join(['{},{}'] * (len(dpdk_rel)))
+        self.column_fmt = '{},' + \
+            ','.join(['{},'] * (len(dpdk_rel)))
+
+    def print_columns(self):
+        '''Print column rows with release names'''
+        print(self.column_fmt.format(*self.column_titles))
+        self.row += 1
+
+    def print_row(self, mapfile, symbols):
+        '''Print row of symbol values'''
+        print(self.output_fmt.format(*([mapfile] + symbols)))
+        self.row += 1
+
+        if((self.terminal_rows>0) and ((self.row % self.terminal_rows) == 0)):
+            self.print_columns()
+
+    OUTPUT_FORMATS = { None: set_terminal_output, \
+                   'terminal': set_terminal_output, \
+                   'csv': set_csv_output }
+
+class ListExpiredOutput():
+    '''Format the output to supported formats'''
+    output_fmt = ""
+    column_fmt = ""
+
+    def __init__(self, format_output, dpdk_releases):
+        self.terminal = True
+        self.OUTPUT_FORMATS[format_output](self,dpdk_releases)
+        self.column_titles = ['mapfile'] +  \
+            ['expired (' + ','.join(dpdk_releases) + ')']
+
+    def set_terminal_output(self, _):
+        '''Set the output format to Tabbed Separated Values'''
+
+        self.output_fmt = '{:<50}{:<50}'
+        self.column_fmt = '{:50}{:50}'
+
+    def set_csv_output(self, _):
+        '''Set the output format to Comma Separated Values'''
+
+        self.output_fmt = '{},{}'
+        self.column_fmt = '{},{}'
+        self.terminal = False
+
+    def print_columns(self):
+        '''Print column rows with release names'''
+        print(self.column_fmt.format(*self.column_titles))
+
+    def print_row(self, mapfile, symbols):
+        '''Print row of symbol values'''
+
+        for symbol in symbols:
+            print(self.output_fmt.format(mapfile,symbol))
+            if self.terminal :
+                mapfile = ''
+
+    OUTPUT_FORMATS = { None: set_terminal_output, \
+                   'terminal': set_terminal_output, \
+                   'csv': set_csv_output }
+
+class CountSymbolsAction:
+    ''' Logic to count symbols added since a give release '''
+    IGNORE_SECTIONS = ['EXPERIMENTAL','INTERNAL']
+
+    def __init__(self, mapfile_path, mapfile_parser, format_output):
+        self.path = mapfile_path
+        self.parser = mapfile_parser
+        self.format_output = format_output
+        self.symbols_count = []
+
+    def add_mapfile(self, release):
+        ''' add a version mapfile '''
+        symbol_count = experimental_count = 0
+
+        symbols = get_symbols(self.parser, release, self.path)
+
+        # which versions are present, and we care about
+        abi_vers = [abi_ver \
+                    for abi_ver in symbols \
+                    if abi_ver not in self.IGNORE_SECTIONS]
+
+        for abi_ver in abi_vers:
+            symbol_count += len(symbols[abi_ver])
+
+        # count experimental symbols
+        if 'EXPERIMENTAL' in symbols.keys():
+            experimental_count = len(symbols['EXPERIMENTAL'])
+
+        self.symbols_count += [symbol_count, experimental_count]
+
+    def __del__(self):
+        self.format_output.print_row(self.path.parent, self.symbols_count)
+
+class ListExpiredAction:
+    ''' Logic to list expired symbols between two releases '''
+
+    def __init__(self, mapfile_path, mapfile_parser, format_output):
+        self.path = mapfile_path
+        self.parser = mapfile_parser
+        self.format_output = format_output
+        self.experimental_symbols = []
+
+    def add_mapfile(self, release):
+        ''' add a version mapfile '''
+        symbols = get_symbols(self.parser, release, self.path)
+        if 'EXPERIMENTAL' in symbols.keys():
+            self.experimental_symbols.append(symbols['EXPERIMENTAL'])
+
+    def __del__(self):
+        if len(self.experimental_symbols) != 2:
+            return
+
+        tmp = self.experimental_symbols
+        # find symbols present in both dpdk releases
+        intersect_syms = [sym for sym in tmp[0] if sym in tmp[1]]
+
+        # check for empty set
+        if intersect_syms == []:
+            return
+
+        self.format_output.print_row(self.path.parent, intersect_syms)
+
+SRC_DIRECTORIES = 'drivers,lib'
+
+ACTIONS = {None: CountSymbolsAction, \
+           'count-symbols': CountSymbolsAction, \
+           'list-expired': ListExpiredAction}
+
+ACTION_OUTPUT = {None: SymbolCountOutput, \
+                 'count-symbols': SymbolCountOutput, \
+                 'list-expired': ListExpiredOutput}
+
+def main():
+    '''Main entry point'''
+
+    dpdk_releases = get_dpdk_releases()
+
+    parser = argparse.ArgumentParser(description=DESCRIPTION.format(s=__file__), \
+                                     formatter_class=RawTextHelpFormatter
+                                     )
+    parser.add_argument('mode', choices=['count-symbols','list-expired'])
+    parser.add_argument('--format-output', choices=['terminal','csv'], \
+                        default='terminal')
+    parser.add_argument('--directory', choices=SRC_DIRECTORIES.split(','),
+                        default=SRC_DIRECTORIES)
+    parser.add_argument('--releases', \
+                        help='2 x comma separated release tags e.g. \'' \
+                        + ','.join([dpdk_releases[0],dpdk_releases[-1]]) \
+                        + '\'')
+    args = parser.parse_args()
+
+    if args.releases is not None:
+        dpdk_releases = args.releases.split(',')
+
+    if args.mode == 'list-expired':
+        if len(dpdk_releases) < 2:
+            sys.exit('Please specify two releases to compare ' \
+                     'in \'list-expired\' mode.')
+        dpdk_releases = [dpdk_releases[0], dpdk_releases[len(dpdk_releases) - 1]]
+
+    action = ACTIONS[args.mode]
+    format_output = ACTION_OUTPUT[args.mode](args.format_output, dpdk_releases)
+
+    map_grammar = MAP_GRAMMAR.format(get_abi_versions())
+    map_parser = makeGrammar(map_grammar, {})
+
+    format_output.print_columns()
+
+    for src_dir in args.directory.split(','):
+        for path in Path(src_dir).rglob('*.map'):
+            release_action = action(path, map_parser, format_output)
+
+            for release in dpdk_releases:
+                release_action.add_mapfile(release)
+
+            # all the magic happens in the destructor
+            del release_action
+
+if __name__ == '__main__':
+    main()
-- 
2.26.2



More information about the dev mailing list