[v6,9/9] build: generate version.map file for MingW on Windows
Checks
Commit Message
From: Tal Shnaiderman <talshn@mellanox.com>
The MingW build for Windows has special cases where exported
function contain additional prefix:
__emutls_v.per_lcore__*
To avoid adding those prefixed functions to the version.map file
the map_to_def.py script was modified to create a map file for Mingw
with the needed changed.
The file name was changed to map_to_win.py
Signed-off-by: Tal Shnaiderman <talshn@mellanox.com>
---
buildtools/{map_to_def.py => map_to_win.py} | 21 ++++++++++++++++-----
buildtools/meson.build | 4 ++--
drivers/meson.build | 12 +++++++++---
lib/meson.build | 15 ++++++++++++---
4 files changed, 39 insertions(+), 13 deletions(-)
rename buildtools/{map_to_def.py => map_to_win.py} (51%)
Comments
On Fri, 19 Jun 2020 00:15:46 +0300
talshn@mellanox.com wrote:
> From: Tal Shnaiderman <talshn@mellanox.com>
>
> The MingW build for Windows has special cases where exported
> function contain additional prefix:
>
> __emutls_v.per_lcore__*
>
> To avoid adding those prefixed functions to the version.map file
> the map_to_def.py script was modified to create a map file for Mingw
> with the needed changed.
Thanks for working on this. Your solution is even simpler than we discussed,
see some comments inline.
>
> The file name was changed to map_to_win.py
>
> Signed-off-by: Tal Shnaiderman <talshn@mellanox.com>
> ---
> buildtools/{map_to_def.py => map_to_win.py} | 21 ++++++++++++++++-----
> buildtools/meson.build | 4 ++--
> drivers/meson.build | 12 +++++++++---
> lib/meson.build | 15 ++++++++++++---
> 4 files changed, 39 insertions(+), 13 deletions(-)
> rename buildtools/{map_to_def.py => map_to_win.py} (51%)
>
> diff --git a/buildtools/map_to_def.py b/buildtools/map_to_win.py
> similarity index 51%
> rename from buildtools/map_to_def.py
> rename to buildtools/map_to_win.py
> index 6775b54a9d..dfb0748159 100644
> --- a/buildtools/map_to_def.py
> +++ b/buildtools/map_to_win.py
> @@ -13,23 +13,34 @@ def is_function_line(ln):
>
> def main(args):
> if not args[1].endswith('version.map') or \
> - not args[2].endswith('exports.def'):
> + not args[2].endswith('exports.def') and \
> + not args[2].endswith('mingw.map'):
> return 1
>
> # special case, allow override if an def file already exists alongside map file
> +# for mingw also replace per_lcore__* to __emutls_v.per_lcore__*
> override_file = join(dirname(args[1]), basename(args[2]))
> if exists(override_file):
> with open(override_file) as f_in:
> - functions = f_in.readlines()
> + lines = f_in.readlines()
> + if args[2].endswith('mingw.map'):
> + lines = [l.replace('per_lcore__', '__emutls_v.per_lcore__') for l in lines]
> + functions = lines
MinGW GCC linker doesn't fail on undefined symbols, so .map overrides will
never be needed (and we're going to get rid of overrides soon anyway).
>
> # generate def file from map file.
> -# This works taking indented lines only which end with a ";" and which don't
> +# For clang this works taking indented lines only which end with a ";" and which don't
> # have a colon in them, i.e. the lines defining functions only.
> +# mingw keeps the original .map file but replaces per_lcore__* to __emutls_v.per_lcore__*
> else:
> with open(args[1]) as f_in:
> - functions = [ln[:-2] + '\n' for ln in sorted(f_in.readlines())
> + lines = f_in.readlines()
> + if args[2].endswith('mingw.map'):
> + lines = [l.replace('per_lcore__', '__emutls_v.per_lcore__') for l in lines]
> + functions = lines
> + else:
> + functions = [ln[:-2] + '\n' for ln in sorted(lines)
> if is_function_line(ln)]
> - functions = ["EXPORTS\n"] + functions
> + functions = ["EXPORTS\n"] + functions
Considering the comment above, entire logic for mingw.map is the following:
if args[2].endswith('mingw.map'):
with open(args[1]) as f_in, open(args[2], 'w') as f_out:
f_out.writelines([...replace... for line in f_in.readlines()])
return 0
Wouldn't it be cleaner to move it to the beginning of function or even to a
separate function dispatched by filename (or even script)? Motivation: soon
.def logic will become more complicated and Windows-dependent (to analyze
dumpbin.exe output), intertwining it with .map saves nothing and will impede.
[snip]
> diff --git a/drivers/meson.build b/drivers/meson.build
> index 646a7d5eb5..b25a368531 100644
> --- a/drivers/meson.build
> +++ b/drivers/meson.build
> @@ -152,16 +152,22 @@ foreach class:dpdk_driver_classes
> implib = 'lib' + lib_name + '.dll.a'
>
> def_file = custom_target(lib_name + '_def',
> - command: [map_to_def_cmd, '@INPUT@', '@OUTPUT@'],
> + command: [map_to_win_cmd, '@INPUT@', '@OUTPUT@'],
> input: version_map,
> output: '@0@_exports.def'.format(lib_name))
> - lk_deps = [version_map, def_file]
> +
> + mingw_map = custom_target(name + '_mingw',
> + command: [map_to_win_cmd, '@INPUT@', '@OUTPUT@'],
> + input: version_map,
> + output: '@0@_mingw.map'.format(name))
Hit the following error while building a Linux-targeted DPDK:
drivers/meson.build:159:3: ERROR: Tried to create target "dpaa_mingw", but a target of that name already exists.
Probably you need "lib_name" instead of "name" or just to disable the target
for non-Windows builds.
> +
> + lk_deps = [version_map, def_file, mingw_map]
> if is_windows
> if is_ms_linker
> lk_args = ['-Wl,/def:' + def_file.full_path(),
> '-Wl,/implib:drivers\\' + implib]
> else
> - lk_args = []
> + lk_args = ['-Wl,--version-script=' + mingw_map.full_path()]
> endif
> else
> lk_args = ['-Wl,--version-script=' + version_map]
> diff --git a/lib/meson.build b/lib/meson.build
> index a8fd317a18..9f6c85a3e1 100644
> --- a/lib/meson.build
> +++ b/lib/meson.build
> @@ -150,18 +150,27 @@ foreach l:libraries
> implib = dir_name + '.dll.a'
>
> def_file = custom_target(name + '_def',
> - command: [map_to_def_cmd, '@INPUT@', '@OUTPUT@'],
> + command: [map_to_win_cmd, '@INPUT@', '@OUTPUT@'],
> input: version_map,
> output: 'rte_@0@_exports.def'.format(name))
>
> + mingw_map = custom_target(name + '_mingw',
> + command: [map_to_win_cmd, '@INPUT@', '@OUTPUT@'],
> + input: version_map,
> + output: 'rte_@0@_mingw.map'.format(name))
> +
See above on name/lib_name. Maybe it's technically irrelevant here, but
better use consistent name generation code.
Thanks Dmitry,
Using libname is the correct way in drivers/meson.build.
I'll modify lib/meson.build to use libname for consistency.
> -----Original Message-----
> From: Dmitry Kozlyuk <dmitry.kozliuk@gmail.com>
> Sent: Saturday, June 20, 2020 9:55 PM
> To: Tal Shnaiderman <talshn@mellanox.com>
> Cc: dev@dpdk.org; Thomas Monjalon <thomas@monjalon.net>;
> pallavi.kadam@intel.com; david.marchand@redhat.com; grive@u256.net;
> ranjit.menon@intel.com; navasile@linux.microsoft.com;
> harini.ramakrishnan@microsoft.com; ocardona@microsoft.com;
> anatoly.burakov@intel.com; Fady Bader <fady@mellanox.com>;
> bruce.richardson@intel.com
> Subject: Re: [PATCH v6 9/9] build: generate version.map file for MingW on
> Windows
>
> [snip]
> > diff --git a/drivers/meson.build b/drivers/meson.build index
> > 646a7d5eb5..b25a368531 100644
> > --- a/drivers/meson.build
> > +++ b/drivers/meson.build
> > @@ -152,16 +152,22 @@ foreach class:dpdk_driver_classes
> > implib = 'lib' + lib_name + '.dll.a'
> >
> > def_file = custom_target(lib_name + '_def',
> > - command: [map_to_def_cmd, '@INPUT@',
> '@OUTPUT@'],
> > + command: [map_to_win_cmd, '@INPUT@',
> '@OUTPUT@'],
> > input: version_map,
> > output:
> '@0@_exports.def'.format(lib_name))
> > - lk_deps = [version_map, def_file]
> > +
> > + mingw_map = custom_target(name + '_mingw',
> > + command: [map_to_win_cmd, '@INPUT@',
> '@OUTPUT@'],
> > + input: version_map,
> > + output: '@0@_mingw.map'.format(name))
>
> Hit the following error while building a Linux-targeted DPDK:
>
> drivers/meson.build:159:3: ERROR: Tried to create target "dpaa_mingw", but
> a target of that name already exists.
>
> Probably you need "lib_name" instead of "name" or just to disable the target
> for non-Windows builds.
>
> > +
> > + lk_deps = [version_map, def_file, mingw_map]
> > if is_windows
> > if is_ms_linker
> > lk_args = ['-Wl,/def:' +
> def_file.full_path(),
> > '-Wl,/implib:drivers\\' +
> implib]
> > else
> > - lk_args = []
> > + lk_args = ['-Wl,--version-script=' +
> mingw_map.full_path()]
> > endif
> > else
> > lk_args = ['-Wl,--version-script=' +
> version_map] diff --git
> > a/lib/meson.build b/lib/meson.build index a8fd317a18..9f6c85a3e1
> > 100644
> > --- a/lib/meson.build
> > +++ b/lib/meson.build
> > @@ -150,18 +150,27 @@ foreach l:libraries
> > implib = dir_name + '.dll.a'
> >
> > def_file = custom_target(name + '_def',
> > - command: [map_to_def_cmd, '@INPUT@',
> '@OUTPUT@'],
> > + command: [map_to_win_cmd, '@INPUT@',
> '@OUTPUT@'],
> > input: version_map,
> > output:
> 'rte_@0@_exports.def'.format(name))
> >
> > + mingw_map = custom_target(name + '_mingw',
> > + command: [map_to_win_cmd, '@INPUT@',
> '@OUTPUT@'],
> > + input: version_map,
> > + output:
> 'rte_@0@_mingw.map'.format(name))
> > +
>
> See above on name/lib_name. Maybe it's technically irrelevant here, but
> better use consistent name generation code.
>
> --
> Dmitry Kozlyuk
> Subject: Re: [PATCH v6 9/9] build: generate version.map file for MingW on
> Windows
>
> On Fri, 19 Jun 2020 00:15:46 +0300
> talshn@mellanox.com wrote:
>
> > From: Tal Shnaiderman <talshn@mellanox.com>
> >
> > The MingW build for Windows has special cases where exported function
> > contain additional prefix:
> >
> > __emutls_v.per_lcore__*
> >
> > To avoid adding those prefixed functions to the version.map file the
> > map_to_def.py script was modified to create a map file for Mingw with
> > the needed changed.
>
> Thanks for working on this. Your solution is even simpler than we discussed,
> see some comments inline.
>
> >
> > The file name was changed to map_to_win.py
> >
> > Signed-off-by: Tal Shnaiderman <talshn@mellanox.com>
> > ---
> > buildtools/{map_to_def.py => map_to_win.py} | 21 ++++++++++++++++-
> ----
> > buildtools/meson.build | 4 ++--
> > drivers/meson.build | 12 +++++++++---
> > lib/meson.build | 15 ++++++++++++---
> > 4 files changed, 39 insertions(+), 13 deletions(-) rename
> > buildtools/{map_to_def.py => map_to_win.py} (51%)
> >
> > diff --git a/buildtools/map_to_def.py b/buildtools/map_to_win.py
> > similarity index 51% rename from buildtools/map_to_def.py rename to
> > buildtools/map_to_win.py index 6775b54a9d..dfb0748159 100644
> > --- a/buildtools/map_to_def.py
> > +++ b/buildtools/map_to_win.py
> > @@ -13,23 +13,34 @@ def is_function_line(ln):
> >
> > def main(args):
> > if not args[1].endswith('version.map') or \
> > - not args[2].endswith('exports.def'):
> > + not args[2].endswith('exports.def') and \
> > + not args[2].endswith('mingw.map'):
> > return 1
> >
> > # special case, allow override if an def file already exists
> > alongside map file
> > +# for mingw also replace per_lcore__* to __emutls_v.per_lcore__*
> > override_file = join(dirname(args[1]), basename(args[2]))
> > if exists(override_file):
> > with open(override_file) as f_in:
> > - functions = f_in.readlines()
> > + lines = f_in.readlines()
> > + if args[2].endswith('mingw.map'):
> > + lines = [l.replace('per_lcore__', '__emutls_v.per_lcore__') for l in
> lines]
> > + functions = lines
>
> MinGW GCC linker doesn't fail on undefined symbols, so .map overrides will
> never be needed (and we're going to get rid of overrides soon anyway).
>
Right, this special case is irrelevant to MinGW, will remove in v7.
> >
> > # generate def file from map file.
> > -# This works taking indented lines only which end with a ";" and
> > which don't
> > +# For clang this works taking indented lines only which end with a
> > +";" and which don't
> > # have a colon in them, i.e. the lines defining functions only.
> > +# mingw keeps the original .map file but replaces per_lcore__* to
> > +__emutls_v.per_lcore__*
> > else:
> > with open(args[1]) as f_in:
> > - functions = [ln[:-2] + '\n' for ln in sorted(f_in.readlines())
> > + lines = f_in.readlines()
> > + if args[2].endswith('mingw.map'):
> > + lines = [l.replace('per_lcore__', '__emutls_v.per_lcore__') for l in
> lines]
> > + functions = lines
> > + else:
> > + functions = [ln[:-2] + '\n' for ln in sorted(lines)
> > if is_function_line(ln)]
> > - functions = ["EXPORTS\n"] + functions
> > + functions = ["EXPORTS\n"] + functions
>
> Considering the comment above, entire logic for mingw.map is the following:
>
> if args[2].endswith('mingw.map'):
> with open(args[1]) as f_in, open(args[2], 'w') as f_out:
> f_out.writelines([...replace... for line in f_in.readlines()])
> return 0
>
> Wouldn't it be cleaner to move it to the beginning of function or even to a
> separate function dispatched by filename (or even script)? Motivation: soon
> .def logic will become more complicated and Windows-dependent (to
> analyze dumpbin.exe output), intertwining it with .map saves nothing and
> will impede.
>
IMO Having a separated function for minGW is sufficient, I'll add the change to v7.
Thanks.
> --
> Dmitry Kozlyuk
similarity index 51%
rename from buildtools/map_to_def.py
rename to buildtools/map_to_win.py
@@ -13,23 +13,34 @@ def is_function_line(ln):
def main(args):
if not args[1].endswith('version.map') or \
- not args[2].endswith('exports.def'):
+ not args[2].endswith('exports.def') and \
+ not args[2].endswith('mingw.map'):
return 1
# special case, allow override if an def file already exists alongside map file
+# for mingw also replace per_lcore__* to __emutls_v.per_lcore__*
override_file = join(dirname(args[1]), basename(args[2]))
if exists(override_file):
with open(override_file) as f_in:
- functions = f_in.readlines()
+ lines = f_in.readlines()
+ if args[2].endswith('mingw.map'):
+ lines = [l.replace('per_lcore__', '__emutls_v.per_lcore__') for l in lines]
+ functions = lines
# generate def file from map file.
-# This works taking indented lines only which end with a ";" and which don't
+# For clang this works taking indented lines only which end with a ";" and which don't
# have a colon in them, i.e. the lines defining functions only.
+# mingw keeps the original .map file but replaces per_lcore__* to __emutls_v.per_lcore__*
else:
with open(args[1]) as f_in:
- functions = [ln[:-2] + '\n' for ln in sorted(f_in.readlines())
+ lines = f_in.readlines()
+ if args[2].endswith('mingw.map'):
+ lines = [l.replace('per_lcore__', '__emutls_v.per_lcore__') for l in lines]
+ functions = lines
+ else:
+ functions = [ln[:-2] + '\n' for ln in sorted(lines)
if is_function_line(ln)]
- functions = ["EXPORTS\n"] + functions
+ functions = ["EXPORTS\n"] + functions
with open(args[2], 'w') as f_out:
f_out.writelines(functions)
@@ -9,14 +9,14 @@ list_dir_globs = find_program('list-dir-globs.py')
check_symbols = find_program('check-symbols.sh')
ldflags_ibverbs_static = find_program('options-ibverbs-static.sh')
-# set up map-to-def script using python, either built-in or external
+# set up map-to-win script using python, either built-in or external
python3 = import('python').find_installation(required: false)
if python3.found()
py3 = [python3]
else
py3 = ['meson', 'runpython']
endif
-map_to_def_cmd = py3 + files('map_to_def.py')
+map_to_win_cmd = py3 + files('map_to_win.py')
sphinx_wrapper = py3 + files('call-sphinx-build.py')
# stable ABI always starts with "DPDK_"
@@ -152,16 +152,22 @@ foreach class:dpdk_driver_classes
implib = 'lib' + lib_name + '.dll.a'
def_file = custom_target(lib_name + '_def',
- command: [map_to_def_cmd, '@INPUT@', '@OUTPUT@'],
+ command: [map_to_win_cmd, '@INPUT@', '@OUTPUT@'],
input: version_map,
output: '@0@_exports.def'.format(lib_name))
- lk_deps = [version_map, def_file]
+
+ mingw_map = custom_target(name + '_mingw',
+ command: [map_to_win_cmd, '@INPUT@', '@OUTPUT@'],
+ input: version_map,
+ output: '@0@_mingw.map'.format(name))
+
+ lk_deps = [version_map, def_file, mingw_map]
if is_windows
if is_ms_linker
lk_args = ['-Wl,/def:' + def_file.full_path(),
'-Wl,/implib:drivers\\' + implib]
else
- lk_args = []
+ lk_args = ['-Wl,--version-script=' + mingw_map.full_path()]
endif
else
lk_args = ['-Wl,--version-script=' + version_map]
@@ -150,18 +150,27 @@ foreach l:libraries
implib = dir_name + '.dll.a'
def_file = custom_target(name + '_def',
- command: [map_to_def_cmd, '@INPUT@', '@OUTPUT@'],
+ command: [map_to_win_cmd, '@INPUT@', '@OUTPUT@'],
input: version_map,
output: 'rte_@0@_exports.def'.format(name))
+ mingw_map = custom_target(name + '_mingw',
+ command: [map_to_win_cmd, '@INPUT@', '@OUTPUT@'],
+ input: version_map,
+ output: 'rte_@0@_mingw.map'.format(name))
+
if is_ms_linker
lk_args = ['-Wl,/def:' + def_file.full_path(),
'-Wl,/implib:lib\\' + implib]
else
- lk_args = ['-Wl,--version-script=' + version_map]
+ if is_windows
+ lk_args = ['-Wl,--version-script=' + mingw_map.full_path()]
+ else
+ lk_args = ['-Wl,--version-script=' + version_map]
+ endif
endif
- lk_deps = [version_map, def_file]
+ lk_deps = [version_map, def_file, mingw_map]
if not is_windows
# on unix systems check the output of the
# check-symbols.sh script, using it as a