new file mode 100755
@@ -0,0 +1,402 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2021 Intel Corporation
+'''Tool to count or list symbols in each DPDK release'''
+from pathlib import Path
+import sys
+import os
+import subprocess
+import argparse
+from argparse import RawTextHelpFormatter
+import re
+import datetime
+try:
+ from parsley import makeGrammar
+except ImportError:
+ print('This script uses the package Parsley to parse C Mapfiles.\n'
+ 'This can be installed with \"pip install parsley".')
+ sys.exit()
+
+DESCRIPTION = '''
+This script tracks the growth of stable and experimental symbols
+over releases since v19.11. The script has the ability to
+count the added symbols between two dpdk releases, and to
+list experimental symbols present in two dpdk releases
+(expired symbols).
+
+example usages:
+
+Count symbols added since v19.11
+$ devtools/symbol-tool.py count-symbols
+
+Count symbols added since v20.11
+$ devtools/symbol-tool.py count-symbols --releases v20.11,v21.05
+
+List experimental symbols present in v20.11 and v21.05
+$ devtools/symbol-tool.py list-expired --releases v20.11,v21.05
+
+List experimental symbols in libraries only, present since v19.11
+$ devtools/symbol-tool.py list-expired --directory lib
+'''
+
+MAP_GRAMMAR = r"""
+
+ws = (' ' | '\r' | '\n' | '\t')*
+
+ABI_VER = ({})
+DPDK_VER = ('DPDK_' ABI_VER)
+ABI_NAME = ('INTERNAL' | 'EXPERIMENTAL' | DPDK_VER)
+comment = '#' (~'\n' anything)+ '\n'
+symbol = (~(';' | '}}' | '#') anything )+:c ';' -> ''.join(c)
+global = 'global:'
+local = 'local: *;'
+symbols = comment* symbol:s ws comment* -> s
+
+abi = (abi_section+):m -> dict(m)
+abi_section = (ws ABI_NAME:e ws '{{' ws global* (~local ws symbols)*:s ws local* ws '}}' ws DPDK_VER* ';' ws) -> (e,s)
+"""
+
+def get_abi_versions():
+ '''Returns a string of possible dpdk abi versions'''
+
+ year = datetime.date.today().year - 2000
+ tags = " |".join(['\'{}\''.format(i) \
+ for i in reversed(range(21, year + 1)) ])
+ tags = tags + ' | \'20.0.1\' | \'20.0\' | \'20\''
+
+ return tags
+
+def get_dpdk_releases():
+ '''Returns a list of dpdk release tags names since v19.11'''
+
+ year = datetime.date.today().year - 2000
+ year_range = "|".join("{}".format(i) for i in range(19,year + 1))
+ pattern = re.compile(r'^\"v(' + year_range + r')\.\d{2}\"$')
+
+ cmd = ['git', 'for-each-ref', '--sort=taggerdate', '--format', '"%(tag)"']
+ try:
+ result = subprocess.run(cmd, \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ print("Failed to interogate git for release tags")
+ sys.exit()
+
+
+ tags = result.stdout.decode('utf-8').split('\n')
+
+ # find the non-rcs between now and v19.11
+ tags = [ tag.replace('\"','') \
+ for tag in reversed(tags) \
+ if pattern.match(tag) ][:-3]
+
+ return tags
+
+def fix_directory_name(path):
+ '''Prepend librte to the source directory name'''
+ mapfilepath1 = str(path.parent.name)
+ mapfilepath2 = str(path.parents[1])
+ mapfilepath = mapfilepath2 + '/librte_' + mapfilepath1
+
+ return mapfilepath
+
+def directory_renamed(path, rel):
+ '''Fix removal of the librte_ from the directory names'''
+
+ mapfilepath = fix_directory_name(path)
+ tagfile = '{}:{}/{}'.format(rel, mapfilepath, path.name)
+
+ try:
+ result = subprocess.run(['git', 'show', tagfile], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ return result
+
+def mapfile_renamed(path, rel):
+ '''Fix renaming of the map file'''
+ newfile = None
+
+ result = subprocess.run(['git', 'ls-tree', \
+ rel, str(path.parent) + '/'], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ dentries = result.stdout.decode('utf-8')
+ dentries = dentries.split('\n')
+
+ # filter entries looking for the map file
+ dentries = [dentry for dentry in dentries if dentry.endswith('.map')]
+ if len(dentries) > 1 or len(dentries) == 0:
+ return None
+
+ dparts = dentries[0].split('/')
+ newfile = dparts[len(dparts) - 1]
+
+ if newfile is not None:
+ tagfile = '{}:{}/{}'.format(rel, path.parent, newfile)
+
+ try:
+ result = subprocess.run(['git', 'show', tagfile], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ else:
+ result = None
+
+ return result
+
+def mapfile_and_directory_renamed(path, rel):
+ '''Fix renaming of the map file & the source directory'''
+ mapfilepath = Path("{}/{}".format(fix_directory_name(path),path.name))
+
+ return mapfile_renamed(mapfilepath, rel)
+
+FIX_STRATEGIES = [directory_renamed, \
+ mapfile_renamed, \
+ mapfile_and_directory_renamed]
+
+def get_symbols(map_parser, release, mapfile_path):
+ '''Count the symbols for a given release and mapfile'''
+ abi_sections = {}
+
+ tagfile = '{}:{}'.format(release,mapfile_path)
+ try:
+ result = subprocess.run(['git', 'show', tagfile], \
+ stdout=subprocess.PIPE, \
+ stderr=subprocess.PIPE,
+ check=True)
+ except subprocess.CalledProcessError:
+ result = None
+
+ for fix_strategy in FIX_STRATEGIES:
+ if result is not None:
+ break
+ result = fix_strategy(mapfile_path, release)
+
+ if result is not None:
+ mapfile = result.stdout.decode('utf-8')
+ abi_sections = map_parser(mapfile).abi()
+
+ return abi_sections
+
+def get_terminal_rows():
+ '''Find the number of rows in the terminal'''
+
+ try:
+ return os.get_terminal_size().lines
+ except IOError:
+ return 0
+
+class SymbolCountOutput():
+ '''Format the output to supported formats'''
+ output_fmt = ""
+ column_fmt = ""
+
+ def __init__(self, format_output, dpdk_releases):
+ self.OUTPUT_FORMATS[format_output](self,dpdk_releases)
+ self.column_titles = ['mapfile'] + dpdk_releases
+
+ self.terminal_rows = get_terminal_rows()
+ self.row = 0
+
+ def set_terminal_output(self,dpdk_rel):
+ '''Set the output format to Tabbed Separated Values'''
+
+ self.output_fmt = '{:<50}' + \
+ ''.join(['{:<6}{:<6}'] * (len(dpdk_rel)))
+ self.column_fmt = '{:50}' + \
+ ''.join(['{:<12}'] * (len(dpdk_rel)))
+
+ def set_csv_output(self,dpdk_rel):
+ '''Set the output format to Comma Separated Values'''
+
+ self.output_fmt = '{},' + \
+ ','.join(['{},{}'] * (len(dpdk_rel)))
+ self.column_fmt = '{},' + \
+ ','.join(['{},'] * (len(dpdk_rel)))
+
+ def print_columns(self):
+ '''Print column rows with release names'''
+ print(self.column_fmt.format(*self.column_titles))
+ self.row += 1
+
+ def print_row(self, mapfile, symbols):
+ '''Print row of symbol values'''
+ print(self.output_fmt.format(*([mapfile] + symbols)))
+ self.row += 1
+
+ if((self.terminal_rows>0) and ((self.row % self.terminal_rows) == 0)):
+ self.print_columns()
+
+ OUTPUT_FORMATS = { None: set_terminal_output, \
+ 'terminal': set_terminal_output, \
+ 'csv': set_csv_output }
+
+class ListExpiredOutput():
+ '''Format the output to supported formats'''
+ output_fmt = ""
+ column_fmt = ""
+
+ def __init__(self, format_output, dpdk_releases):
+ self.terminal = True
+ self.OUTPUT_FORMATS[format_output](self,dpdk_releases)
+ self.column_titles = ['mapfile'] + \
+ ['expired (' + ','.join(dpdk_releases) + ')']
+
+ def set_terminal_output(self, _):
+ '''Set the output format to Tabbed Separated Values'''
+
+ self.output_fmt = '{:<50}{:<50}'
+ self.column_fmt = '{:50}{:50}'
+
+ def set_csv_output(self, _):
+ '''Set the output format to Comma Separated Values'''
+
+ self.output_fmt = '{},{}'
+ self.column_fmt = '{},{}'
+ self.terminal = False
+
+ def print_columns(self):
+ '''Print column rows with release names'''
+ print(self.column_fmt.format(*self.column_titles))
+
+ def print_row(self, mapfile, symbols):
+ '''Print row of symbol values'''
+
+ for symbol in symbols:
+ print(self.output_fmt.format(mapfile,symbol))
+ if self.terminal :
+ mapfile = ''
+
+ OUTPUT_FORMATS = { None: set_terminal_output, \
+ 'terminal': set_terminal_output, \
+ 'csv': set_csv_output }
+
+class CountSymbolsAction:
+ ''' Logic to count symbols added since a give release '''
+ IGNORE_SECTIONS = ['EXPERIMENTAL','INTERNAL']
+
+ def __init__(self, mapfile_path, mapfile_parser, format_output):
+ self.path = mapfile_path
+ self.parser = mapfile_parser
+ self.format_output = format_output
+ self.symbols_count = []
+
+ def add_mapfile(self, release):
+ ''' add a version mapfile '''
+ symbol_count = experimental_count = 0
+
+ symbols = get_symbols(self.parser, release, self.path)
+
+ # which versions are present, and we care about
+ abi_vers = [abi_ver \
+ for abi_ver in symbols \
+ if abi_ver not in self.IGNORE_SECTIONS]
+
+ for abi_ver in abi_vers:
+ symbol_count += len(symbols[abi_ver])
+
+ # count experimental symbols
+ if 'EXPERIMENTAL' in symbols.keys():
+ experimental_count = len(symbols['EXPERIMENTAL'])
+
+ self.symbols_count += [symbol_count, experimental_count]
+
+ def __del__(self):
+ self.format_output.print_row(self.path.parent, self.symbols_count)
+
+class ListExpiredAction:
+ ''' Logic to list expired symbols between two releases '''
+
+ def __init__(self, mapfile_path, mapfile_parser, format_output):
+ self.path = mapfile_path
+ self.parser = mapfile_parser
+ self.format_output = format_output
+ self.experimental_symbols = []
+
+ def add_mapfile(self, release):
+ ''' add a version mapfile '''
+ symbols = get_symbols(self.parser, release, self.path)
+ if 'EXPERIMENTAL' in symbols.keys():
+ self.experimental_symbols.append(symbols['EXPERIMENTAL'])
+
+ def __del__(self):
+ if len(self.experimental_symbols) != 2:
+ return
+
+ tmp = self.experimental_symbols
+ # find symbols present in both dpdk releases
+ intersect_syms = [sym for sym in tmp[0] if sym in tmp[1]]
+
+ # check for empty set
+ if intersect_syms == []:
+ return
+
+ self.format_output.print_row(self.path.parent, intersect_syms)
+
+SRC_DIRECTORIES = 'drivers,lib'
+
+ACTIONS = {None: CountSymbolsAction, \
+ 'count-symbols': CountSymbolsAction, \
+ 'list-expired': ListExpiredAction}
+
+ACTION_OUTPUT = {None: SymbolCountOutput, \
+ 'count-symbols': SymbolCountOutput, \
+ 'list-expired': ListExpiredOutput}
+
+def main():
+ '''Main entry point'''
+
+ dpdk_releases = get_dpdk_releases()
+
+ parser = argparse.ArgumentParser(description=DESCRIPTION, \
+ formatter_class=RawTextHelpFormatter
+ )
+ parser.add_argument('mode', choices=['count-symbols','list-expired'])
+ parser.add_argument('--format-output', choices=['terminal','csv'], \
+ default='terminal')
+ parser.add_argument('--directory', choices=SRC_DIRECTORIES.split(','),
+ default=SRC_DIRECTORIES)
+ parser.add_argument('--releases', \
+ help='2 x comma separated release tags e.g. \'' \
+ + ','.join([dpdk_releases[0],dpdk_releases[-1]]) \
+ + '\'')
+ args = parser.parse_args()
+
+ if args.releases is not None:
+ dpdk_releases = args.releases.split(',')
+
+ if args.mode == 'list-expired':
+ if len(dpdk_releases) < 2:
+ sys.exit('Please specify two releases to compare ' \
+ 'in \'list-expired\' mode.')
+ dpdk_releases = [dpdk_releases[0], dpdk_releases[len(dpdk_releases) - 1]]
+
+ action = ACTIONS[args.mode]
+ format_output = ACTION_OUTPUT[args.mode](args.format_output, dpdk_releases)
+
+ map_grammar = MAP_GRAMMAR.format(get_abi_versions())
+ map_parser = makeGrammar(map_grammar, {})
+
+ format_output.print_columns()
+
+ for src_dir in args.directory.split(','):
+ for path in Path(src_dir).rglob('*.map'):
+ release_action = action(path, map_parser, format_output)
+
+ for release in dpdk_releases:
+ release_action.add_mapfile(release)
+
+ # all the magic happens in the destructor
+ del release_action
+
+if __name__ == '__main__':
+ main()