From patchwork Thu Nov 15 15:47:18 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Burakov, Anatoly" X-Patchwork-Id: 48132 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id ED6335F0D; Thu, 15 Nov 2018 16:47:41 +0100 (CET) Received: from mga05.intel.com (mga05.intel.com [192.55.52.43]) by dpdk.org (Postfix) with ESMTP id B04223238 for ; Thu, 15 Nov 2018 16:47:27 +0100 (CET) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga001.jf.intel.com ([10.7.209.18]) by fmsmga105.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 15 Nov 2018 07:47:25 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.56,236,1539673200"; d="scan'208";a="108528224" Received: from irvmail001.ir.intel.com ([163.33.26.43]) by orsmga001.jf.intel.com with ESMTP; 15 Nov 2018 07:47:22 -0800 Received: from sivswdev01.ir.intel.com (sivswdev01.ir.intel.com [10.237.217.45]) by irvmail001.ir.intel.com (8.14.3/8.13.6/MailSET/Hub) with ESMTP id wAFFlL2D024808; Thu, 15 Nov 2018 15:47:21 GMT Received: from sivswdev01.ir.intel.com (localhost [127.0.0.1]) by sivswdev01.ir.intel.com with ESMTP id wAFFlLEk028136; Thu, 15 Nov 2018 15:47:21 GMT Received: (from aburakov@localhost) by sivswdev01.ir.intel.com with LOCAL id wAFFlLsE028132; Thu, 15 Nov 2018 15:47:21 GMT From: Anatoly Burakov To: dev@dpdk.org Cc: john.mcnamara@intel.com, bruce.richardson@intel.com, pablo.de.lara.guarch@intel.com, david.hunt@intel.com, mohammad.abdul.awal@intel.com, thomas@monjalon.net, ferruh.yigit@intel.com Date: Thu, 15 Nov 2018 15:47:18 +0000 Message-Id: <087f92918853c99ef4163eb548fa046c8e536fdb.1542291869.git.anatoly.burakov@intel.com> X-Mailer: git-send-email 1.7.0.7 In-Reply-To: References: In-Reply-To: References: Subject: [dpdk-dev] [RFC v2 6/9] usertools/devbind: switch to using DPDKConfigLib X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Make devbind use DPDKConfigLib library instead of having its own logic for binding/unbinding/enumerating devices and getting device information. All semantics and output are kept the same, only the internals changed. Signed-off-by: Anatoly Burakov --- usertools/dpdk-devbind.py | 533 ++++++-------------------------------- 1 file changed, 74 insertions(+), 459 deletions(-) diff --git a/usertools/dpdk-devbind.py b/usertools/dpdk-devbind.py index 7d564634c..b1834c302 100755 --- a/usertools/dpdk-devbind.py +++ b/usertools/dpdk-devbind.py @@ -1,50 +1,18 @@ #! /usr/bin/env python # SPDX-License-Identifier: BSD-3-Clause -# Copyright(c) 2010-2014 Intel Corporation +# Copyright(c) 2010-2018 Intel Corporation # +from DPDKConfigLib import DevInfo, DevUtil import sys -import os import getopt -import subprocess -from os.path import exists, abspath, dirname, basename - -# The PCI base class for all devices -network_class = {'Class': '02', 'Vendor': None, 'Device': None, - 'SVendor': None, 'SDevice': None} -encryption_class = {'Class': '10', 'Vendor': None, 'Device': None, - 'SVendor': None, 'SDevice': None} -intel_processor_class = {'Class': '0b', 'Vendor': '8086', 'Device': None, - 'SVendor': None, 'SDevice': None} -cavium_sso = {'Class': '08', 'Vendor': '177d', 'Device': 'a04b,a04d', - 'SVendor': None, 'SDevice': None} -cavium_fpa = {'Class': '08', 'Vendor': '177d', 'Device': 'a053', - 'SVendor': None, 'SDevice': None} -cavium_pkx = {'Class': '08', 'Vendor': '177d', 'Device': 'a0dd,a049', - 'SVendor': None, 'SDevice': None} -cavium_tim = {'Class': '08', 'Vendor': '177d', 'Device': 'a051', - 'SVendor': None, 'SDevice': None} -cavium_zip = {'Class': '12', 'Vendor': '177d', 'Device': 'a037', - 'SVendor': None, 'SDevice': None} -avp_vnic = {'Class': '05', 'Vendor': '1af4', 'Device': '1110', - 'SVendor': None, 'SDevice': None} - -network_devices = [network_class, cavium_pkx, avp_vnic] -crypto_devices = [encryption_class, intel_processor_class] -eventdev_devices = [cavium_sso, cavium_tim] -mempool_devices = [cavium_fpa] -compress_devices = [cavium_zip] - -# global dict ethernet devices present. Dictionary indexed by PCI address. -# Each device within this is itself a dictionary of device properties -devices = {} -# list of supported DPDK drivers -dpdk_drivers = ["igb_uio", "vfio-pci", "uio_pci_generic"] +from os.path import basename # command-line arg flags b_flag = None status_flag = False force_flag = False +status_dev = "" args = [] @@ -119,394 +87,43 @@ def usage(): """ % locals()) # replace items from local variables -# This is roughly compatible with check_output function in subprocess module -# which is only available in python 2.7. -def check_output(args, stderr=None): - '''Run a command and capture its output''' - return subprocess.Popen(args, stdout=subprocess.PIPE, - stderr=stderr).communicate()[0] - - def check_modules(): '''Checks that igb_uio is loaded''' - global dpdk_drivers + loaded_drivers = DevUtil.get_loaded_dpdk_drivers() + supported_drivers = DevUtil.get_supported_dpdk_drivers() - # list of supported modules - mods = [{"Name": driver, "Found": False} for driver in dpdk_drivers] - - # first check if module is loaded - try: - # Get list of sysfs modules (both built-in and dynamically loaded) - sysfs_path = '/sys/module/' - - # Get the list of directories in sysfs_path - sysfs_mods = [os.path.join(sysfs_path, o) for o - in os.listdir(sysfs_path) - if os.path.isdir(os.path.join(sysfs_path, o))] - - # Extract the last element of '/sys/module/abc' in the array - sysfs_mods = [a.split('/')[-1] for a in sysfs_mods] - - # special case for vfio_pci (module is named vfio-pci, - # but its .ko is named vfio_pci) - sysfs_mods = [a if a != 'vfio_pci' else 'vfio-pci' for a in sysfs_mods] - - for mod in mods: - if mod["Name"] in sysfs_mods: - mod["Found"] = True - except: - pass + if b_flag in supported_drivers and b_flag not in loaded_drivers: + print("Error - %s driver is not loaded" % b_flag) + sys.exit(1) # check if we have at least one loaded module - if True not in [mod["Found"] for mod in mods] and b_flag is not None: - if b_flag in dpdk_drivers: - print("Error - no supported modules(DPDK driver) are loaded") - sys.exit(1) - else: - print("Warning - no supported modules(DPDK driver) are loaded") - - # change DPDK driver list to only contain drivers that are loaded - dpdk_drivers = [mod["Name"] for mod in mods if mod["Found"]] - - -def has_driver(dev_id): - '''return true if a device is assigned to a driver. False otherwise''' - return "Driver_str" in devices[dev_id] - - -def get_pci_device_details(dev_id, probe_lspci): - '''This function gets additional details for a PCI device''' - device = {} - - if probe_lspci: - extra_info = check_output(["lspci", "-vmmks", dev_id]).splitlines() - - # parse lspci details - for line in extra_info: - if len(line) == 0: - continue - name, value = line.decode().split("\t", 1) - name = name.strip(":") + "_str" - device[name] = value - # check for a unix interface name - device["Interface"] = "" - for base, dirs, _ in os.walk("/sys/bus/pci/devices/%s/" % dev_id): - if "net" in dirs: - device["Interface"] = \ - ",".join(os.listdir(os.path.join(base, "net"))) - break - # check if a port is used for ssh connection - device["Ssh_if"] = False - device["Active"] = "" - - return device - -def clear_data(): - '''This function clears any old data''' - devices = {} - -def get_device_details(devices_type): - '''This function populates the "devices" dictionary. The keys used are - the pci addresses (domain:bus:slot.func). The values are themselves - dictionaries - one for each NIC.''' - global devices - global dpdk_drivers - - # first loop through and read details for all devices - # request machine readable format, with numeric IDs and String - dev = {} - dev_lines = check_output(["lspci", "-Dvmmnnk"]).splitlines() - for dev_line in dev_lines: - if len(dev_line) == 0: - if device_type_match(dev, devices_type): - # Replace "Driver" with "Driver_str" to have consistency of - # of dictionary key names - if "Driver" in dev.keys(): - dev["Driver_str"] = dev.pop("Driver") - if "Module" in dev.keys(): - dev["Module_str"] = dev.pop("Module") - # use dict to make copy of dev - devices[dev["Slot"]] = dict(dev) - # Clear previous device's data - dev = {} - else: - name, value = dev_line.decode().split("\t", 1) - value_list = value.rsplit(' ', 1) - if len(value_list) > 1: - # String stored in _str - dev[name.rstrip(":") + '_str'] = value_list[0] - # Numeric IDs - dev[name.rstrip(":")] = value_list[len(value_list) - 1] \ - .rstrip("]").lstrip("[") - - if devices_type == network_devices: - # check what is the interface if any for an ssh connection if - # any to this host, so we can mark it later. - ssh_if = [] - route = check_output(["ip", "-o", "route"]) - # filter out all lines for 169.254 routes - route = "\n".join(filter(lambda ln: not ln.startswith("169.254"), - route.decode().splitlines())) - rt_info = route.split() - for i in range(len(rt_info) - 1): - if rt_info[i] == "dev": - ssh_if.append(rt_info[i+1]) - - # based on the basic info, get extended text details - for d in devices.keys(): - if not device_type_match(devices[d], devices_type): - continue - - # get additional info and add it to existing data - devices[d] = devices[d].copy() - # No need to probe lspci - devices[d].update(get_pci_device_details(d, False).items()) - - if devices_type == network_devices: - for _if in ssh_if: - if _if in devices[d]["Interface"].split(","): - devices[d]["Ssh_if"] = True - devices[d]["Active"] = "*Active*" - break - - # add igb_uio to list of supporting modules if needed - if "Module_str" in devices[d]: - for driver in dpdk_drivers: - if driver not in devices[d]["Module_str"]: - devices[d]["Module_str"] = \ - devices[d]["Module_str"] + ",%s" % driver - else: - devices[d]["Module_str"] = ",".join(dpdk_drivers) - - # make sure the driver and module strings do not have any duplicates - if has_driver(d): - modules = devices[d]["Module_str"].split(",") - if devices[d]["Driver_str"] in modules: - modules.remove(devices[d]["Driver_str"]) - devices[d]["Module_str"] = ",".join(modules) - - -def device_type_match(dev, devices_type): - for i in range(len(devices_type)): - param_count = len( - [x for x in devices_type[i].values() if x is not None]) - match_count = 0 - if dev["Class"][0:2] == devices_type[i]["Class"]: - match_count = match_count + 1 - for key in devices_type[i].keys(): - if key != 'Class' and devices_type[i][key]: - value_list = devices_type[i][key].split(',') - for value in value_list: - if value.strip(' ') == dev[key]: - match_count = match_count + 1 - # count must be the number of non None parameters to match - if match_count == param_count: - return True - return False - -def dev_id_from_dev_name(dev_name): - '''Take a device "name" - a string passed in by user to identify a NIC - device, and determine the device id - i.e. the domain:bus:slot.func - for - it, which can then be used to index into the devices array''' - - # check if it's already a suitable index - if dev_name in devices: - return dev_name - # check if it's an index just missing the domain part - elif "0000:" + dev_name in devices: - return "0000:" + dev_name - else: - # check if it's an interface name, e.g. eth1 - for d in devices.keys(): - if dev_name in devices[d]["Interface"].split(","): - return devices[d]["Slot"] - # if nothing else matches - error - print("Unknown device: %s. " - "Please specify device in \"bus:slot.func\" format" % dev_name) - sys.exit(1) - - -def unbind_one(dev_id, force): - '''Unbind the device identified by "dev_id" from its current driver''' - dev = devices[dev_id] - if not has_driver(dev_id): - print("%s %s %s is not currently managed by any driver\n" % - (dev["Slot"], dev["Device_str"], dev["Interface"])) - return - - # prevent us disconnecting ourselves - if dev["Ssh_if"] and not force: - print("Routing table indicates that interface %s is active. " - "Skipping unbind" % (dev_id)) - return - - # write to /sys to unbind - filename = "/sys/bus/pci/drivers/%s/unbind" % dev["Driver_str"] - try: - f = open(filename, "a") - except: - print("Error: unbind failed for %s - Cannot open %s" - % (dev_id, filename)) - sys.exit(1) - f.write(dev_id) - f.close() - - -def bind_one(dev_id, driver, force): - '''Bind the device given by "dev_id" to the driver "driver". If the device - is already bound to a different driver, it will be unbound first''' - dev = devices[dev_id] - saved_driver = None # used to rollback any unbind in case of failure - - # prevent disconnection of our ssh session - if dev["Ssh_if"] and not force: - print("Routing table indicates that interface %s is active. " - "Not modifying" % (dev_id)) - return - - # unbind any existing drivers we don't want - if has_driver(dev_id): - if dev["Driver_str"] == driver: - print("%s already bound to driver %s, skipping\n" - % (dev_id, driver)) - return - else: - saved_driver = dev["Driver_str"] - unbind_one(dev_id, force) - dev["Driver_str"] = "" # clear driver string - - # For kernels >= 3.15 driver_override can be used to specify the driver - # for a device rather than relying on the driver to provide a positive - # match of the device. The existing process of looking up - # the vendor and device ID, adding them to the driver new_id, - # will erroneously bind other devices too which has the additional burden - # of unbinding those devices - if driver in dpdk_drivers: - filename = "/sys/bus/pci/devices/%s/driver_override" % dev_id - if os.path.exists(filename): - try: - f = open(filename, "w") - except: - print("Error: bind failed for %s - Cannot open %s" - % (dev_id, filename)) - return - try: - f.write("%s" % driver) - f.close() - except: - print("Error: bind failed for %s - Cannot write driver %s to " - "PCI ID " % (dev_id, driver)) - return - # For kernels < 3.15 use new_id to add PCI id's to the driver - else: - filename = "/sys/bus/pci/drivers/%s/new_id" % driver - try: - f = open(filename, "w") - except: - print("Error: bind failed for %s - Cannot open %s" - % (dev_id, filename)) - return - try: - # Convert Device and Vendor Id to int to write to new_id - f.write("%04x %04x" % (int(dev["Vendor"],16), - int(dev["Device"], 16))) - f.close() - except: - print("Error: bind failed for %s - Cannot write new PCI ID to " - "driver %s" % (dev_id, driver)) - return - - # do the bind by writing to /sys - filename = "/sys/bus/pci/drivers/%s/bind" % driver - try: - f = open(filename, "a") - except: - print("Error: bind failed for %s - Cannot open %s" - % (dev_id, filename)) - if saved_driver is not None: # restore any previous driver - bind_one(dev_id, saved_driver, force) - return - try: - f.write(dev_id) - f.close() - except: - # for some reason, closing dev_id after adding a new PCI ID to new_id - # results in IOError. however, if the device was successfully bound, - # we don't care for any errors and can safely ignore IOError - tmp = get_pci_device_details(dev_id, True) - if "Driver_str" in tmp and tmp["Driver_str"] == driver: - return - print("Error: bind failed for %s - Cannot bind to driver %s" - % (dev_id, driver)) - if saved_driver is not None: # restore any previous driver - bind_one(dev_id, saved_driver, force) - return - - # For kernels > 3.15 driver_override is used to bind a device to a driver. - # Before unbinding it, overwrite driver_override with empty string so that - # the device can be bound to any other driver - filename = "/sys/bus/pci/devices/%s/driver_override" % dev_id - if os.path.exists(filename): - try: - f = open(filename, "w") - except: - print("Error: unbind failed for %s - Cannot open %s" - % (dev_id, filename)) - sys.exit(1) - try: - f.write("\00") - f.close() - except: - print("Error: unbind failed for %s - Cannot open %s" - % (dev_id, filename)) - sys.exit(1) + if len(loaded_drivers) == 0: + print("Warning - no supported modules(DPDK driver) are loaded") def unbind_all(dev_list, force=False): """Unbind method, takes a list of device locations""" - - if dev_list[0] == "dpdk": - for d in devices.keys(): - if "Driver_str" in devices[d]: - if devices[d]["Driver_str"] in dpdk_drivers: - unbind_one(devices[d]["Slot"], force) - return - - dev_list = map(dev_id_from_dev_name, dev_list) - for d in dev_list: - unbind_one(d, force) + try: + DevUtil.unbind(dev_list, force) + except DevUtil.UnbindException as e: + print("Unbind failed: %s" % e) + sys.exit(1) def bind_all(dev_list, driver, force=False): """Bind method, takes a list of device locations""" - global devices - dev_list = map(dev_id_from_dev_name, dev_list) + try: + DevUtil.bind(dev_list, driver, force) + except DevUtil.BindException as e: + print("Bind failed: %s" % e) + sys.exit(1) + except DevUtil.UnbindException as e: + print("Bind failed: %s" % e) + sys.exit(1) - for d in dev_list: - bind_one(d, driver, force) - # For kernels < 3.15 when binding devices to a generic driver - # (i.e. one that doesn't have a PCI ID table) using new_id, some devices - # that are not bound to any other driver could be bound even if no one has - # asked them to. hence, we check the list of drivers again, and see if - # some of the previously-unbound devices were erroneously bound. - if not os.path.exists("/sys/bus/pci/devices/%s/driver_override" % d): - for d in devices.keys(): - # skip devices that were already bound or that we know should be bound - if "Driver_str" in devices[d] or d in dev_list: - continue - - # update information about this device - devices[d] = dict(devices[d].items() + - get_pci_device_details(d, True).items()) - - # check if updated information indicates that the device was bound - if "Driver_str" in devices[d]: - unbind_one(d, force) - - -def display_devices(title, dev_list, extra_params=None): +def display_devices(title, dev_list): '''Displays to the user the details of a list of devices given in "dev_list". The "extra_params" parameter, if given, should contain a string with %()s fields in it for replacement by the named fields in each @@ -518,62 +135,74 @@ def display_devices(title, dev_list, extra_params=None): strings.append("") else: for dev in dev_list: - if extra_params is not None: - strings.append("%s '%s %s' %s" % (dev["Slot"], - dev["Device_str"], - dev["Device"], - extra_params % dev)) + if dev.devtype == DevInfo.DeviceType.DEVTYPE_NETWORK: + extra_str = "if=%s drv=%s unused=%s%s" % \ + (",".join(dev.interfaces), dev.active_driver, + ",".join(dev.available_drivers), + " *Active*" if dev.active_interface else "") else: - strings.append("%s '%s'" % (dev["Slot"], dev["Device_str"])) + extra_str = "drv=%s unused=%s" % \ + (dev.active_driver, ",".join(dev.available_drivers)) + strings.append("%s '%s %s' %s" % (dev.pci_addr, + dev.device_name, + dev.device_id, + extra_str)) # sort before printing, so that the entries appear in PCI order strings.sort() print("\n".join(strings)) # print one per line -def show_device_status(devices_type, device_name): - global dpdk_drivers + +def show_device_status(device_list, device_type): kernel_drv = [] dpdk_drv = [] no_drv = [] + typenames = { + DevInfo.DeviceType.DEVTYPE_NETWORK: "Network", + DevInfo.DeviceType.DEVTYPE_CRYPTO: "Crypto", + DevInfo.DeviceType.DEVTYPE_EVENT: "Event", + DevInfo.DeviceType.DEVTYPE_MEMPOOL: "Mempool", + DevInfo.DeviceType.DEVTYPE_COMPRESS: "Compress" + } + typename = typenames[device_type] + # split our list of network devices into the three categories above - for d in devices.keys(): - if device_type_match(devices[d], devices_type): - if not has_driver(d): - no_drv.append(devices[d]) - continue - if devices[d]["Driver_str"] in dpdk_drivers: - dpdk_drv.append(devices[d]) + for d in device_list: + if d.devtype == device_type: + if d.active_driver == "": + no_drv.append(d) + elif d.active_driver in DevInfo.get_loaded_dpdk_drivers(): + dpdk_drv.append(d) else: - kernel_drv.append(devices[d]) + kernel_drv.append(d) # print each category separately, so we can clearly see what's used by DPDK - display_devices("%s devices using DPDK-compatible driver" % device_name, - dpdk_drv, "drv=%(Driver_str)s unused=%(Module_str)s") - display_devices("%s devices using kernel driver" % device_name, kernel_drv, - "if=%(Interface)s drv=%(Driver_str)s " - "unused=%(Module_str)s %(Active)s") - display_devices("Other %s devices" % device_name, no_drv, - "unused=%(Module_str)s") + display_devices("%s devices using DPDK-compatible driver" % typename, + dpdk_drv) + display_devices("%s devices using kernel driver" % typename, kernel_drv) + display_devices("Other %s devices" % typename, no_drv) + def show_status(): '''Function called when the script is passed the "--status" option. Displays to the user what devices are bound to the igb_uio driver, the kernel driver or to no driver''' - if status_dev == "net" or status_dev == "all": - show_device_status(network_devices, "Network") - - if status_dev == "crypto" or status_dev == "all": - show_device_status(crypto_devices, "Crypto") - - if status_dev == "event" or status_dev == "all": - show_device_status(eventdev_devices, "Eventdev") - - if status_dev == "mempool" or status_dev == "all": - show_device_status(mempool_devices, "Mempool") - - if status_dev == "compress" or status_dev == "all": - show_device_status(compress_devices , "Compress") + devtypes = { + "net": DevInfo.DeviceType.DEVTYPE_NETWORK, + "crypto": DevInfo.DeviceType.DEVTYPE_CRYPTO, + "event": DevInfo.DeviceType.DEVTYPE_EVENT, + "mempool": DevInfo.DeviceType.DEVTYPE_MEMPOOL, + "compress": DevInfo.DeviceType.DEVTYPE_COMPRESS + } + device_list = DevInfo.get_devices().values() + + found_types = sorted([devtypes[t] for t in devtypes.keys() + if t == status_dev or status_dev == "all"]) + + for t in found_types: + devices = list(filter(lambda dev: dev.devtype == t, device_list)) + show_device_status(devices, t) def parse_args(): @@ -642,14 +271,6 @@ def do_arg_actions(): elif b_flag is not None: bind_all(args, b_flag, force_flag) if status_flag: - if b_flag is not None: - clear_data() - # refresh if we have changed anything - get_device_details(network_devices) - get_device_details(crypto_devices) - get_device_details(eventdev_devices) - get_device_details(mempool_devices) - get_device_details(compress_devices) show_status() @@ -657,12 +278,6 @@ def main(): '''program main function''' parse_args() check_modules() - clear_data() - get_device_details(network_devices) - get_device_details(crypto_devices) - get_device_details(eventdev_devices) - get_device_details(mempool_devices) - get_device_details(compress_devices) do_arg_actions() if __name__ == "__main__":