From patchwork Mon Jun 21 15:11:00 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ray Kinsella X-Patchwork-Id: 94658 Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 7AA13A0547; Mon, 21 Jun 2021 17:19:17 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id F333741158; Mon, 21 Jun 2021 17:19:16 +0200 (CEST) Received: from mga05.intel.com (mga05.intel.com [192.55.52.43]) by mails.dpdk.org (Postfix) with ESMTP id CE99540040 for ; Mon, 21 Jun 2021 17:19:15 +0200 (CEST) IronPort-SDR: wxxnKEUIPo6As1oSSbc4kGAVjfFhwmYdmlX8HBq57iDOB60XeryXFcCMBxg3rWKxJmgJa8i7KS kJk6hCfpY5nQ== X-IronPort-AV: E=McAfee;i="6200,9189,10022"; a="292495713" X-IronPort-AV: E=Sophos;i="5.83,289,1616482800"; d="scan'208";a="292495713" Received: from orsmga008.jf.intel.com ([10.7.209.65]) by fmsmga105.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 21 Jun 2021 08:19:14 -0700 IronPort-SDR: olbmnbCuz4igOq3blyWlARt9tBuuQu1dxNRODNFd618WtiysEVa4AbrEvv4Ogu9aavfB8KsLoW iMyj9owCqUPg== X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.83,289,1616482800"; d="scan'208";a="452225488" Received: from silpixa00396680.ir.intel.com (HELO silpixa00396680.ger.corp.intel.com) ([10.237.223.54]) by orsmga008.jf.intel.com with ESMTP; 21 Jun 2021 08:19:12 -0700 From: Ray Kinsella To: dev@dpdk.org Cc: stephen@networkplumber.org, ferruh.yigit@intel.com, thomas@monjalon.net, ktraynor@redhat.com, bruce.richardson@intel.com, mdr@ashroe.eu Date: Mon, 21 Jun 2021 16:11:00 +0100 Message-Id: <20210621151100.406252-1-mdr@ashroe.eu> X-Mailer: git-send-email 2.26.2 In-Reply-To: References: MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH] devtools: script to track map symbols X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Script to track growth of stable and experimental symbols over releases since v19.11. Signed-off-by: Ray Kinsella --- devtools/count_symbols.py | 230 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 230 insertions(+) create mode 100755 devtools/count_symbols.py diff --git a/devtools/count_symbols.py b/devtools/count_symbols.py new file mode 100755 index 0000000000..7b29651044 --- /dev/null +++ b/devtools/count_symbols.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2021 Intel Corporation +from pathlib import Path +import sys, os +import subprocess +import argparse +import re +import datetime + +try: + from parsley import makeGrammar +except ImportError: + print('This script uses the package Parsley to parse C Mapfiles.\n' + 'This can be installed with \"pip install parsley".') + exit() + +symbolMapGrammar = r""" + +ws = (' ' | '\r' | '\n' | '\t')* + +ABI_VER = ({}) +DPDK_VER = ('DPDK_' ABI_VER) +ABI_NAME = ('INTERNAL' | 'EXPERIMENTAL' | DPDK_VER) +comment = '#' (~'\n' anything)+ '\n' +symbol = (~(';' | '}}' | '#') anything )+:c ';' -> ''.join(c) +global = 'global:' +local = 'local: *;' +symbols = comment* symbol:s ws comment* -> s + +abi = (abi_section+):m -> dict(m) +abi_section = (ws ABI_NAME:e ws '{{' ws global* (~local ws symbols)*:s ws local* ws '}}' ws DPDK_VER* ';' ws) -> (e,s) +""" + +#abi_ver = ['21', '20.0.1', '20.0', '20'] + +def get_abi_versions(): + year = datetime.date.today().year - 2000 + s=" |".join(['\'{}\''.format(i) for i in reversed(range(21, year + 1)) ]) + s = s + ' | \'20.0.1\' | \'20.0\' | \'20\'' + + return s + +def get_dpdk_releases(): + year = datetime.date.today().year - 2000 + s="|".join("{}".format(i) for i in range(19,year + 1)) + pattern = re.compile('^\"v(' + s + ')\.\d{2}\"$') + + cmd = ['git', 'for-each-ref', '--sort=taggerdate', '--format', '"%(tag)"'] + result = subprocess.run(cmd, \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE) + if result.stderr.startswith(b'fatal'): + result = None + + tags = result.stdout.decode('utf-8').split('\n') + + # find the non-rcs between now and v19.11 + tags = [ tag.replace('\"','') \ + for tag in reversed(tags) \ + if pattern.match(tag) ][:-3] + + return tags + + +def get_terminal_rows(): + rows, _ = os.popen('stty size', 'r').read().split() + return int(rows) + +def fix_directory_name(path): + mapfilepath1 = str(path.parent.name) + mapfilepath2 = str(path.parents[1]) + mapfilepath = mapfilepath2 + '/librte_' + mapfilepath1 + + return mapfilepath + +# fix removal of the librte_ from the directory names +def directory_renamed(path, rel): + mapfilepath = fix_directory_name(path) + tagfile = '{}:{}/{}'.format(rel, mapfilepath, path.name) + + result = subprocess.run(['git', 'show', tagfile], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE) + if result.stderr.startswith(b'fatal'): + result = None + + return result + +# fix renaming of map files +def mapfile_renamed(path, rel): + newfile = None + + result = subprocess.run(['git', 'ls-tree', \ + rel, str(path.parent) + '/'], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE) + dentries = result.stdout.decode('utf-8') + dentries = dentries.split('\n') + + # filter entries looking for the map file + dentries = [dentry for dentry in dentries if dentry.endswith('.map')] + if len(dentries) > 1 or len(dentries) == 0: + return None + + dparts = dentries[0].split('/') + newfile = dparts[len(dparts) - 1] + + if(newfile is not None): + tagfile = '{}:{}/{}'.format(rel, path.parent, newfile) + + result = subprocess.run(['git', 'show', tagfile], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE) + if result.stderr.startswith(b'fatal'): + result = None + + else: + result = None + + return result + +# renaming of the map file & renaming of directory +def mapfile_and_directory_renamed(path, rel): + mapfilepath = Path("{}/{}".format(fix_directory_name(path),path.name)) + + return mapfile_renamed(mapfilepath, rel) + +fix_strategies = [directory_renamed, \ + mapfile_renamed, \ + mapfile_and_directory_renamed] + +fmt = col_fmt = "" + +def set_terminal_output(dpdk_rel): + global fmt, col_fmt + + fmt = '{:<50}' + col_fmt = fmt + for rel in dpdk_rel: + fmt += '{:<6}{:<6}' + col_fmt += '{:<12}' + +def set_csv_output(dpdk_rel): + global fmt, col_fmt + + fmt = '{},' + col_fmt = fmt + for rel in dpdk_rel: + fmt += '{},{},' + col_fmt += '{},,' + +output_formats = { None: set_terminal_output, \ + 'terminal': set_terminal_output, \ + 'csv': set_csv_output } +directories = 'drivers, lib' + +def main(): + global fmt, col_fmt, symbolMapGrammar + + parser = argparse.ArgumentParser(description='Count symbols in DPDK Libs') + parser.add_argument('--format-output', choices=['terminal','csv'], \ + default='terminal') + parser.add_argument('--directory', choices=directories, + default=directories) + args = parser.parse_args() + + dpdk_rel = get_dpdk_releases() + + # set the output format + output_formats[args.format_output](dpdk_rel) + + column_titles = ['mapfile'] + dpdk_rel + print(col_fmt.format(*column_titles)) + + symbolMapGrammar = symbolMapGrammar.format(get_abi_versions()) + MAPParser = makeGrammar(symbolMapGrammar, {}) + + terminal_rows = get_terminal_rows() + row = 0 + + for src_dir in args.directory.split(','): + for path in Path(src_dir).rglob('*.map'): + csym = [0] * 2 + relsym = [str(path)] + + for rel in dpdk_rel: + i = csym[0] = csym[1] = 0 + abi_sections = None + + tagfile = '{}:{}'.format(rel,path) + result = subprocess.run(['git', 'show', tagfile], \ + stdout=subprocess.PIPE, \ + stderr=subprocess.PIPE) + + if result.stderr.startswith(b'fatal'): + result = None + + while(result is None and i < len(fix_strategies)): + result = fix_strategies[i](path, rel) + i += 1 + + if result is not None: + mapfile = result.stdout.decode('utf-8') + abi_sections = MAPParser(mapfile).abi() + + if abi_sections is not None: + # which versions are present, and we care about + ignore = ['EXPERIMENTAL','INTERNAL'] + found_ver = [ver \ + for ver in abi_sections \ + if ver not in ignore] + + for ver in found_ver: + csym[0] += len(abi_sections[ver]) + + # count experimental symbols + if 'EXPERIMENTAL' in abi_sections: + csym[1] = len(abi_sections['EXPERIMENTAL']) + + relsym += csym + + print(fmt.format(*relsym)) + row += 1 + + if((terminal_rows>0) and ((row % terminal_rows) == 0)): + print(col_fmt.format(*column_titles)) + +if __name__ == '__main__': + main()