[v4,2/4] build: use Python pmdinfogen

  Like for other build scripts, use Python interpreter to run pmdinfogen.
Adjust wrapper script accordingly and also don't suppress stderr from ar
and pmdinfogen.

Add python3-pyelftools to CI configuration.

Signed-off-by: Dmitry Kozlyuk <dmitry.kozliuk@gmail.com>
---
 .travis.yml                     | 2 +-
 GNUmakefile                     | 2 +-
 buildtools/Makefile             | 9 ---------
 buildtools/gen-pmdinfo-cfile.sh | 6 +++---
 buildtools/meson.build          | 1 +
 drivers/meson.build             | 2 +-
 meson.build                     | 1 -
 mk/internal/rte.compile-pre.mk  | 2 +-
 mk/rte.sdkinstall.mk            | 2 --
 9 files changed, 8 insertions(+), 19 deletions(-)
 delete mode 100644 buildtools/Makefile
  

On Thu, Jul 09, 2020 at 12:23:33AM +0300, Dmitry Kozlyuk wrote:
> Like for other build scripts, use Python interpreter to run pmdinfogen.
> Adjust wrapper script accordingly and also don't suppress stderr from ar
> and pmdinfogen.
> 
> Add python3-pyelftools to CI configuration.
> 
> Signed-off-by: Dmitry Kozlyuk <dmitry.kozliuk@gmail.com>
> ---
>  .travis.yml                     | 2 +-
>  GNUmakefile                     | 2 +-
>  buildtools/Makefile             | 9 ---------
>  buildtools/gen-pmdinfo-cfile.sh | 6 +++---
>  buildtools/meson.build          | 1 +
>  drivers/meson.build             | 2 +-
>  meson.build                     | 1 -
>  mk/internal/rte.compile-pre.mk  | 2 +-
>  mk/rte.sdkinstall.mk            | 2 --
>  9 files changed, 8 insertions(+), 19 deletions(-)
>  delete mode 100644 buildtools/Makefile
> 
> diff --git a/.travis.yml b/.travis.yml
> index 14f812423..28b559a25 100644
> --- a/.travis.yml
> +++ b/.travis.yml
> @@ -14,7 +14,7 @@ addons:
>    apt:
>      update: true
>      packages: &required_packages
> -      - [libnuma-dev, python3-setuptools, python3-wheel, python3-pip, ninja-build]
> +      - [libnuma-dev, python3-setuptools, python3-wheel, python3-pip, python3-pyelftools, ninja-build]
>        - [libbsd-dev, libpcap-dev, libibverbs-dev, libcrypto++-dev, libfdt-dev, libjansson-dev]
>  
>  _aarch64_packages: &aarch64_packages
> diff --git a/GNUmakefile b/GNUmakefile
> index e8de422df..242d30d2e 100644
> --- a/GNUmakefile
> +++ b/GNUmakefile
> @@ -12,6 +12,6 @@ export RTE_SDK
>  # directory list
>  #
>  
> -ROOTDIRS-y := buildtools lib kernel drivers app
> +ROOTDIRS-y := lib kernel drivers app
>  
>  include $(RTE_SDK)/mk/rte.sdkroot.mk
> diff --git a/buildtools/Makefile b/buildtools/Makefile
> deleted file mode 100644
> index 7f76fd7d6..000000000
> --- a/buildtools/Makefile
> +++ /dev/null
> @@ -1,9 +0,0 @@
> -# SPDX-License-Identifier: BSD-3-Clause
> -# Copyright(c) 2016 Neil Horman <nhorman@tuxdriver.com>
> -# All rights reserved.
> -
> -include $(RTE_SDK)/mk/rte.vars.mk
> -
> -DIRS-y += pmdinfogen
> -
> -include $(RTE_SDK)/mk/rte.subdir.mk
> diff --git a/buildtools/gen-pmdinfo-cfile.sh b/buildtools/gen-pmdinfo-cfile.sh
> index 43059cf36..109ee461e 100755
> --- a/buildtools/gen-pmdinfo-cfile.sh
> +++ b/buildtools/gen-pmdinfo-cfile.sh
> @@ -4,11 +4,11 @@
>  
>  arfile=$1
>  output=$2
> -pmdinfogen=$3
> +shift 2
> +pmdinfogen=$*
>  
>  # The generated file must not be empty if compiled in pedantic mode
>  echo 'static __attribute__((unused)) const char *generator = "'$0'";' > $output
>  for ofile in `ar t $arfile` ; do
> -	ar p $arfile $ofile | $pmdinfogen - - >> $output 2> /dev/null
> +	ar p $arfile $ofile | $pmdinfogen - - >> $output
>  done
> -exit 0

For 20.11, since make build system is being removed, I think we should look
to merge this into the python script rather than having an extra layer of
wrapper and having meson pass the command from one script to another.
However, since out-of-the-box python doesn't seem to have support for
reading .a files, this is fine for now.


> diff --git a/buildtools/meson.build b/buildtools/meson.build
> index 04808dabc..3a64b28b7 100644
> --- a/buildtools/meson.build
> +++ b/buildtools/meson.build
> @@ -17,3 +17,4 @@ else
>  endif
>  map_to_win_cmd = py3 + files('map_to_win.py')
>  sphinx_wrapper = py3 + files('call-sphinx-build.py')
> +pmdinfogen = py3 + files('pmdinfogen.py')
> diff --git a/drivers/meson.build b/drivers/meson.build
> index 161cfda04..3c4d4b700 100644
> --- a/drivers/meson.build
> +++ b/drivers/meson.build
> @@ -119,7 +119,7 @@ foreach class:dpdk_driver_classes
>  						command: [pmdinfo, tmp_lib.full_path(),
>  							'@OUTPUT@', pmdinfogen],
>  						output: out_filename,
> -						depends: [pmdinfogen, tmp_lib])
> +						depends: [tmp_lib])
>  			endif
>  
>  			# now build the static driver
> diff --git a/meson.build b/meson.build
> index 61d9a4f5f..a6e8c09c2 100644
> --- a/meson.build
> +++ b/meson.build
> @@ -45,7 +45,6 @@ subdir('buildtools')
>  subdir('config')
>  
>  # build libs and drivers
> -subdir('buildtools/pmdinfogen')
>  subdir('lib')
>  subdir('drivers')
>  
> diff --git a/mk/internal/rte.compile-pre.mk b/mk/internal/rte.compile-pre.mk
> index df05b5576..bb2ab0725 100644
> --- a/mk/internal/rte.compile-pre.mk
> +++ b/mk/internal/rte.compile-pre.mk
> @@ -59,7 +59,7 @@ endif
>  CHECK_SYMBOLS_SCRIPT = $(RTE_SDK)/buildtools/check-symbols.sh
>  CHECK_SYMBOLS = $(CHECK_SYMBOLS_SCRIPT) $(SRCDIR)/$(EXPORT_MAP) $@
>  
> -PMDINFO_GEN = $(RTE_SDK_BIN)/app/dpdk-pmdinfogen $@ $@.pmd.c
> +PMDINFO_GEN = $(RTE_SDK)/buildtools/pmdinfogen.py $@ $@.pmd.c
>  PMDINFO_CC = $(CC) $(CPPFLAGS) $(CFLAGS) $(EXTRA_CFLAGS) -c -o $@.pmd.o $@.pmd.c
>  PMDINFO_LD = $(CROSS)ld -r $(filter-out -export-dynamic,$(LDFLAGS)) -o $@.o $@.pmd.o $@
>  PMDINFO_TO_O = if grep -q 'RTE_PMD_REGISTER_.*(.*)' $<; then \
> diff --git a/mk/rte.sdkinstall.mk b/mk/rte.sdkinstall.mk
> index 32bed5d95..875a64f04 100644
> --- a/mk/rte.sdkinstall.mk
> +++ b/mk/rte.sdkinstall.mk
> @@ -98,7 +98,6 @@ install-runtime:
>  	$(Q)cp $(CP_FLAGS)    $O/lib/* $(DESTDIR)$(libdir)
>  	$(Q)$(call rte_mkdir, $(DESTDIR)$(bindir))
>  	$(Q)tar -cf -      -C $O --exclude 'app/*.map' \
> -		--exclude app/dpdk-pmdinfogen \
>  		--exclude 'app/cmdline*' --exclude app/test \
>  		--exclude app/testacl --exclude app/testpipeline app | \
>  	    tar -xf -      -C $(DESTDIR)$(bindir) $(TAR_X_FLAGS)
> @@ -134,7 +133,6 @@ install-sdk:
>  	$(Q)cp $(CP_FLAGS)      $(RTE_SDK)/buildtools    $(DESTDIR)$(sdkdir)
>  	$(Q)$(call rte_mkdir,                            $(DESTDIR)$(targetdir)/app)
>  	$(Q)cp $(CP_FLAGS)      $O/.config               $(DESTDIR)$(targetdir)
> -	$(Q)cp $(CP_FLAGS)      $O/app/dpdk-pmdinfogen   $(DESTDIR)$(targetdir)/app
>  	$(Q)$(call rte_symlink, $(DESTDIR)$(includedir), $(DESTDIR)$(targetdir)/include)
>  	$(Q)$(call rte_symlink, $(DESTDIR)$(libdir),     $(DESTDIR)$(targetdir)/lib)
>  
> -- 
> 2.25.4
> 

For the meson build changes here:

Reviewed-by: Bruce Richardson <bruce.richardson@intel.com>
  

On Tue, 21 Jul 2020 15:04:11 +0100, Bruce Richardson wrote:
> On Thu, Jul 09, 2020 at 12:23:33AM +0300, Dmitry Kozlyuk wrote:
[snip]
> > diff --git a/buildtools/gen-pmdinfo-cfile.sh b/buildtools/gen-pmdinfo-cfile.sh
> > index 43059cf36..109ee461e 100755
> > --- a/buildtools/gen-pmdinfo-cfile.sh
> > +++ b/buildtools/gen-pmdinfo-cfile.sh
> > @@ -4,11 +4,11 @@
> >  
> >  arfile=$1
> >  output=$2
> > -pmdinfogen=$3
> > +shift 2
> > +pmdinfogen=$*
> >  
> >  # The generated file must not be empty if compiled in pedantic mode
> >  echo 'static __attribute__((unused)) const char *generator = "'$0'";' > $output
> >  for ofile in `ar t $arfile` ; do
> > -	ar p $arfile $ofile | $pmdinfogen - - >> $output 2> /dev/null
> > +	ar p $arfile $ofile | $pmdinfogen - - >> $output
> >  done
> > -exit 0  
> 
> For 20.11, since make build system is being removed, I think we should look
> to merge this into the python script rather than having an extra layer of
> wrapper and having meson pass the command from one script to another.
> However, since out-of-the-box python doesn't seem to have support for
> reading .a files, this is fine for now.

Or Meson can only tell the Python script whether to process ELF of COFF and
the path to an archiver tool (ar or lib.exe). This way we'd get rid of shell
(which is non-portable to Windows) and need no libraries to read .a or .lib.