Patch Detail

HTTP 200 OK
Allow: GET, PUT, PATCH, HEAD, OPTIONS
Content-Type: application/json
Vary: Accept

{
    "id": 77492,
    "url": "http://patches.dpdk.org/api/patches/77492/?format=api",
    "web_url": "http://patches.dpdk.org/project/dpdk/patch/1599851920-16802-9-git-send-email-timothy.mcdaniel@intel.com/",
    "project": {
        "id": 1,
        "url": "http://patches.dpdk.org/api/projects/1/?format=api",
        "name": "DPDK",
        "link_name": "dpdk",
        "list_id": "dev.dpdk.org",
        "list_email": "dev@dpdk.org",
        "web_url": "http://core.dpdk.org",
        "scm_url": "git://dpdk.org/dpdk",
        "webscm_url": "http://git.dpdk.org/dpdk",
        "list_archive_url": "https://inbox.dpdk.org/dev",
        "list_archive_url_format": "https://inbox.dpdk.org/dev/{}",
        "commit_url_format": ""
    },
    "msgid": "<1599851920-16802-9-git-send-email-timothy.mcdaniel@intel.com>",
    "list_archive_url": "https://inbox.dpdk.org/dev/1599851920-16802-9-git-send-email-timothy.mcdaniel@intel.com",
    "date": "2020-09-11T19:18:26",
    "name": "[v4,08/22] event/dlb: add infos get and configure",
    "commit_ref": null,
    "pull_url": null,
    "state": "superseded",
    "archived": true,
    "hash": "9f30f2863190e68fd42ea1681320e4dd55cf59d8",
    "submitter": {
        "id": 826,
        "url": "http://patches.dpdk.org/api/people/826/?format=api",
        "name": "Timothy McDaniel",
        "email": "timothy.mcdaniel@intel.com"
    },
    "delegate": {
        "id": 310,
        "url": "http://patches.dpdk.org/api/users/310/?format=api",
        "username": "jerin",
        "first_name": "Jerin",
        "last_name": "Jacob",
        "email": "jerinj@marvell.com"
    },
    "mbox": "http://patches.dpdk.org/project/dpdk/patch/1599851920-16802-9-git-send-email-timothy.mcdaniel@intel.com/mbox/",
    "series": [
        {
            "id": 12163,
            "url": "http://patches.dpdk.org/api/series/12163/?format=api",
            "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=12163",
            "date": "2020-09-11T19:18:18",
            "name": "Add DLB PMD",
            "version": 4,
            "mbox": "http://patches.dpdk.org/series/12163/mbox/"
        }
    ],
    "comments": "http://patches.dpdk.org/api/patches/77492/comments/",
    "check": "warning",
    "checks": "http://patches.dpdk.org/api/patches/77492/checks/",
    "tags": {},
    "related": [],
    "headers": {
        "Return-Path": "<dev-bounces@dpdk.org>",
        "X-Original-To": "patchwork@inbox.dpdk.org",
        "Delivered-To": "patchwork@inbox.dpdk.org",
        "Received": [
            "from dpdk.org (dpdk.org [92.243.14.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id D7EF2A04C1;\n\tFri, 11 Sep 2020 21:23:31 +0200 (CEST)",
            "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id C00AA1C1D0;\n\tFri, 11 Sep 2020 21:22:20 +0200 (CEST)",
            "from mga12.intel.com (mga12.intel.com [192.55.52.136])\n by dpdk.org (Postfix) with ESMTP id 8A5161C191\n for <dev@dpdk.org>; Fri, 11 Sep 2020 21:22:09 +0200 (CEST)",
            "from fmsmga005.fm.intel.com ([10.253.24.32])\n by fmsmga106.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384;\n 11 Sep 2020 12:22:08 -0700",
            "from txasoft-yocto.an.intel.com ([10.123.72.192])\n by fmsmga005.fm.intel.com with ESMTP; 11 Sep 2020 12:22:08 -0700"
        ],
        "IronPort-SDR": [
            "\n RO9RwhA8UFQM2veCYPJVkXqep8KVyWL2hnSOeiEcojlpzENyDQl3riI69jD+Hpxy8zixWpjkmc\n 8RVOhkGnLxsw==",
            "\n /Ftp4Qp67/q3j+3bfOysGRYESNk0Lp+axgDbvknF+z/lQZHPzSQV5KF8ourYKchy8JI7wKthYb\n hefG2nUrKNZQ=="
        ],
        "X-IronPort-AV": [
            "E=McAfee;i=\"6000,8403,9741\"; a=\"138352264\"",
            "E=Sophos;i=\"5.76,416,1592895600\"; d=\"scan'208\";a=\"138352264\"",
            "E=Sophos;i=\"5.76,416,1592895600\"; d=\"scan'208\";a=\"506375664\""
        ],
        "X-Amp-Result": "SKIPPED(no attachment in message)",
        "X-Amp-File-Uploaded": "False",
        "X-ExtLoop1": "1",
        "From": "Timothy McDaniel <timothy.mcdaniel@intel.com>",
        "To": "",
        "Cc": "dev@dpdk.org, erik.g.carrillo@intel.com, gage.eads@intel.com,\n harry.van.haaren@intel.com, jerinj@marvell.com",
        "Date": "Fri, 11 Sep 2020 14:18:26 -0500",
        "Message-Id": "<1599851920-16802-9-git-send-email-timothy.mcdaniel@intel.com>",
        "X-Mailer": "git-send-email 1.7.10",
        "In-Reply-To": "<1599851920-16802-1-git-send-email-timothy.mcdaniel@intel.com>",
        "References": "<1599851920-16802-1-git-send-email-timothy.mcdaniel@intel.com>",
        "Subject": "[dpdk-dev] [PATCH v4 08/22] event/dlb: add infos get and configure",
        "X-BeenThere": "dev@dpdk.org",
        "X-Mailman-Version": "2.1.15",
        "Precedence": "list",
        "List-Id": "DPDK patches and discussions <dev.dpdk.org>",
        "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>",
        "List-Archive": "<http://mails.dpdk.org/archives/dev/>",
        "List-Post": "<mailto:dev@dpdk.org>",
        "List-Help": "<mailto:dev-request@dpdk.org?subject=help>",
        "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>",
        "Errors-To": "dev-bounces@dpdk.org",
        "Sender": "\"dev\" <dev-bounces@dpdk.org>"
    },
    "content": "Add support for configuring the DLB hardware.\n\nSigned-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>\n---\n drivers/event/dlb/dlb.c                  |  402 +++\n drivers/event/dlb/dlb_iface.c            |   11 +\n drivers/event/dlb/dlb_iface.h            |   11 +\n drivers/event/dlb/pf/base/dlb_resource.c | 4098 +++++++++++++++++++++++++++++-\n drivers/event/dlb/pf/dlb_pf.c            |   88 +\n 5 files changed, 4517 insertions(+), 93 deletions(-)",
    "diff": "diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c\nindex f89edf2..2dba396 100644\n--- a/drivers/event/dlb/dlb.c\n+++ b/drivers/event/dlb/dlb.c\n@@ -140,6 +140,23 @@ dlb_hw_query_resources(struct dlb_eventdev *dlb)\n \treturn 0;\n }\n \n+void\n+dlb_free_qe_mem(struct dlb_port *qm_port)\n+{\n+\tif (qm_port == NULL)\n+\t\treturn;\n+\n+\tif (qm_port->qe4) {\n+\t\trte_free(qm_port->qe4);\n+\t\tqm_port->qe4 = NULL;\n+\t}\n+\n+\tif (qm_port->consume_qe) {\n+\t\trte_free(qm_port->consume_qe);\n+\t\tqm_port->consume_qe = NULL;\n+\t}\n+}\n+\n /* Wrapper for string to int conversion. Substituted for atoi(...), which is\n  * unsafe.\n  */\n@@ -231,6 +248,389 @@ set_num_dir_credits(const char *key __rte_unused,\n \t\t\t    DLB_MAX_NUM_DIR_CREDITS);\n \t\treturn -EINVAL;\n \t}\n+\treturn 0;\n+}\n+\n+/* VDEV-only notes:\n+ * This function first unmaps all memory mappings and closes the\n+ * domain's file descriptor, which causes the driver to reset the\n+ * scheduling domain. Once that completes (when close() returns), we\n+ * can safely free the dynamically allocated memory used by the\n+ * scheduling domain.\n+ *\n+ * PF-only notes:\n+ * We will maintain a use count and use that to determine when\n+ * a reset is required.  In PF mode, we never mmap, or munmap\n+ * device memory,  and we own the entire physical PCI device.\n+ */\n+\n+static void\n+dlb_hw_reset_sched_domain(const struct rte_eventdev *dev, bool reconfig)\n+{\n+\tstruct dlb_eventdev *dlb = dlb_pmd_priv(dev);\n+\tenum dlb_configuration_state config_state;\n+\tint i, j;\n+\n+\t/* Close and reset the domain */\n+\tdlb_iface_domain_close(dlb);\n+\n+\t/* Free all dynamically allocated port memory */\n+\tfor (i = 0; i < dlb->num_ports; i++)\n+\t\tdlb_free_qe_mem(&dlb->ev_ports[i].qm_port);\n+\n+\t/* If reconfiguring, mark the device's queues and ports as \"previously\n+\t * configured.\" If the user does not reconfigure them, the PMD will\n+\t * reapply their previous configuration when the device is started.\n+\t */\n+\tconfig_state = (reconfig) ? DLB_PREV_CONFIGURED : DLB_NOT_CONFIGURED;\n+\n+\tfor (i = 0; i < dlb->num_ports; i++) {\n+\t\tdlb->ev_ports[i].qm_port.config_state = config_state;\n+\t\t/* Reset setup_done so ports can be reconfigured */\n+\t\tdlb->ev_ports[i].setup_done = false;\n+\t\tfor (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)\n+\t\t\tdlb->ev_ports[i].link[j].mapped = false;\n+\t}\n+\n+\tfor (i = 0; i < dlb->num_queues; i++)\n+\t\tdlb->ev_queues[i].qm_queue.config_state = config_state;\n+\n+\tfor (i = 0; i < DLB_MAX_NUM_QUEUES; i++)\n+\t\tdlb->ev_queues[i].setup_done = false;\n+\n+\tdlb->num_ports = 0;\n+\tdlb->num_ldb_ports = 0;\n+\tdlb->num_dir_ports = 0;\n+\tdlb->num_queues = 0;\n+\tdlb->num_ldb_queues = 0;\n+\tdlb->num_dir_queues = 0;\n+\tdlb->configured = false;\n+}\n+\n+static int\n+dlb_ldb_credit_pool_create(struct dlb_hw_dev *handle)\n+{\n+\tstruct dlb_create_ldb_pool_args cfg;\n+\tstruct dlb_cmd_response response;\n+\tint ret;\n+\n+\tif (handle == NULL)\n+\t\treturn -EINVAL;\n+\n+\tif (!handle->cfg.resources.num_ldb_credits) {\n+\t\thandle->cfg.ldb_credit_pool_id = 0;\n+\t\thandle->cfg.num_ldb_credits = 0;\n+\t\treturn 0;\n+\t}\n+\n+\tcfg.response = (uintptr_t)&response;\n+\tcfg.num_ldb_credits = handle->cfg.resources.num_ldb_credits;\n+\n+\tret = dlb_iface_ldb_credit_pool_create(handle,\n+\t\t\t\t\t       &cfg);\n+\tif (ret < 0) {\n+\t\tDLB_LOG_ERR(\"dlb: ldb_credit_pool_create ret=%d (driver status: %s)\\n\",\n+\t\t\t    ret, dlb_error_strings[response.status]);\n+\t}\n+\n+\thandle->cfg.ldb_credit_pool_id = response.id;\n+\thandle->cfg.num_ldb_credits = cfg.num_ldb_credits;\n+\n+\treturn ret;\n+}\n+\n+static int\n+dlb_dir_credit_pool_create(struct dlb_hw_dev *handle)\n+{\n+\tstruct dlb_create_dir_pool_args cfg;\n+\tstruct dlb_cmd_response response;\n+\tint ret;\n+\n+\tif (handle == NULL)\n+\t\treturn -EINVAL;\n+\n+\tif (!handle->cfg.resources.num_dir_credits) {\n+\t\thandle->cfg.dir_credit_pool_id = 0;\n+\t\thandle->cfg.num_dir_credits = 0;\n+\t\treturn 0;\n+\t}\n+\n+\tcfg.response = (uintptr_t)&response;\n+\tcfg.num_dir_credits = handle->cfg.resources.num_dir_credits;\n+\n+\tret = dlb_iface_dir_credit_pool_create(handle, &cfg);\n+\tif (ret < 0)\n+\t\tDLB_LOG_ERR(\"dlb: dir_credit_pool_create ret=%d (driver status: %s)\\n\",\n+\t\t\t    ret, dlb_error_strings[response.status]);\n+\n+\thandle->cfg.dir_credit_pool_id = response.id;\n+\thandle->cfg.num_dir_credits = cfg.num_dir_credits;\n+\n+\treturn ret;\n+}\n+\n+static int\n+dlb_hw_create_sched_domain(struct dlb_hw_dev *handle,\n+\t\t\t   struct dlb_eventdev *dlb,\n+\t\t\t   const struct dlb_hw_rsrcs *resources_asked)\n+{\n+\tint ret = 0;\n+\tstruct dlb_create_sched_domain_args *config_params;\n+\tstruct dlb_cmd_response response;\n+\n+\tif (resources_asked == NULL) {\n+\t\tDLB_LOG_ERR(\"dlb: dlb_create NULL parameter\\n\");\n+\t\tret = EINVAL;\n+\t\tgoto error_exit;\n+\t}\n+\n+\t/* Map generic qm resources to dlb resources */\n+\tconfig_params = &handle->cfg.resources;\n+\n+\tconfig_params->response = (uintptr_t)&response;\n+\n+\t/* DIR ports and queues */\n+\n+\tconfig_params->num_dir_ports =\n+\t\tresources_asked->num_dir_ports;\n+\n+\tconfig_params->num_dir_credits =\n+\t\tresources_asked->num_dir_credits;\n+\n+\t/* LDB ports and queues */\n+\n+\tconfig_params->num_ldb_queues =\n+\t\tresources_asked->num_ldb_queues;\n+\n+\tconfig_params->num_ldb_ports =\n+\t\tresources_asked->num_ldb_ports;\n+\n+\tconfig_params->num_ldb_credits =\n+\t\tresources_asked->num_ldb_credits;\n+\n+\tconfig_params->num_atomic_inflights =\n+\t\tdlb->num_atm_inflights_per_queue *\n+\t\tconfig_params->num_ldb_queues;\n+\n+\tconfig_params->num_hist_list_entries = config_params->num_ldb_ports *\n+\t\tDLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;\n+\n+\t/* dlb limited to 1 credit pool per queue type */\n+\tconfig_params->num_ldb_credit_pools = 1;\n+\tconfig_params->num_dir_credit_pools = 1;\n+\n+\tDLB_LOG_DBG(\"sched domain create - ldb_qs=%d, ldb_ports=%d, dir_ports=%d, atomic_inflights=%d, hist_list_entries=%d, ldb_credits=%d, dir_credits=%d, ldb_cred_pools=%d, dir-credit_pools=%d\\n\",\n+\t\t    config_params->num_ldb_queues,\n+\t\t    config_params->num_ldb_ports,\n+\t\t    config_params->num_dir_ports,\n+\t\t    config_params->num_atomic_inflights,\n+\t\t    config_params->num_hist_list_entries,\n+\t\t    config_params->num_ldb_credits,\n+\t\t    config_params->num_dir_credits,\n+\t\t    config_params->num_ldb_credit_pools,\n+\t\t    config_params->num_dir_credit_pools);\n+\n+\t/* Configure the QM */\n+\n+\tret = dlb_iface_sched_domain_create(handle, config_params);\n+\tif (ret < 0) {\n+\t\tDLB_LOG_ERR(\"dlb: domain create failed, device_id = %d, (driver ret = %d, extra status: %s)\\n\",\n+\t\t\t    handle->device_id,\n+\t\t\t    ret,\n+\t\t\t    dlb_error_strings[response.status]);\n+\t\tgoto error_exit;\n+\t}\n+\n+\thandle->domain_id = response.id;\n+\thandle->domain_id_valid = 1;\n+\n+\tconfig_params->response = 0;\n+\n+\tret = dlb_ldb_credit_pool_create(handle);\n+\tif (ret < 0) {\n+\t\tDLB_LOG_ERR(\"dlb: create ldb credit pool failed\\n\");\n+\t\tgoto error_exit2;\n+\t}\n+\n+\tret = dlb_dir_credit_pool_create(handle);\n+\tif (ret < 0) {\n+\t\tDLB_LOG_ERR(\"dlb: create dir credit pool failed\\n\");\n+\t\tgoto error_exit2;\n+\t}\n+\n+\thandle->cfg.configured = true;\n+\n+\treturn 0;\n+\n+error_exit2:\n+\tdlb_iface_domain_close(dlb);\n+\n+error_exit:\n+\treturn ret;\n+}\n+\n+/* End HW specific */\n+static void\n+dlb_eventdev_info_get(struct rte_eventdev *dev,\n+\t\t      struct rte_event_dev_info *dev_info)\n+{\n+\tstruct dlb_eventdev *dlb = dlb_pmd_priv(dev);\n+\tint ret;\n+\n+\tret = dlb_hw_query_resources(dlb);\n+\tif (ret) {\n+\t\tconst struct rte_eventdev_data *data = dev->data;\n+\n+\t\tDLB_LOG_ERR(\"get resources err=%d, devid=%d\\n\",\n+\t\t\t    ret, data->dev_id);\n+\t\t/* fn is void, so fall through and return values set up in\n+\t\t * probe\n+\t\t */\n+\t}\n+\n+\t/* Add num resources currently owned by this domain.\n+\t * These would become available if the scheduling domain were reset due\n+\t * to the application recalling eventdev_configure to *reconfigure* the\n+\t * domain.\n+\t */\n+\tevdev_dlb_default_info.max_event_ports += dlb->num_ldb_ports;\n+\tevdev_dlb_default_info.max_event_queues += dlb->num_ldb_queues;\n+\tevdev_dlb_default_info.max_num_events += dlb->num_ldb_credits;\n+\n+\n+\t/* In DLB A-stepping hardware, applications are limited to 128\n+\t * configured ports (load-balanced or directed). The reported number of\n+\t * available ports must reflect this.\n+\t */\n+\tif (dlb->revision < DLB_REV_B0) {\n+\t\tint used_ports;\n+\n+\t\tused_ports = DLB_MAX_NUM_LDB_PORTS + DLB_MAX_NUM_DIR_PORTS -\n+\t\t\tdlb->hw_rsrc_query_results.num_ldb_ports -\n+\t\t\tdlb->hw_rsrc_query_results.num_dir_ports;\n+\n+\t\tevdev_dlb_default_info.max_event_ports =\n+\t\t\tRTE_MIN(evdev_dlb_default_info.max_event_ports,\n+\t\t\t\t128 - used_ports);\n+\t}\n+\n+\tevdev_dlb_default_info.max_event_queues =\n+\t\tRTE_MIN(evdev_dlb_default_info.max_event_queues,\n+\t\t\tRTE_EVENT_MAX_QUEUES_PER_DEV);\n+\n+\tevdev_dlb_default_info.max_num_events =\n+\t\tRTE_MIN(evdev_dlb_default_info.max_num_events,\n+\t\t\tdlb->max_num_events_override);\n+\n+\t*dev_info = evdev_dlb_default_info;\n+}\n+\n+/* Note: 1 QM instance per QM device, QM instance/device == event device */\n+static int\n+dlb_eventdev_configure(const struct rte_eventdev *dev)\n+{\n+\tstruct dlb_eventdev *dlb = dlb_pmd_priv(dev);\n+\tstruct dlb_hw_dev *handle = &dlb->qm_instance;\n+\tstruct dlb_hw_rsrcs *rsrcs = &handle->info.hw_rsrc_max;\n+\tconst struct rte_eventdev_data *data = dev->data;\n+\tconst struct rte_event_dev_config *config = &data->dev_conf;\n+\tint ret;\n+\n+\t/* If this eventdev is already configured, we must release the current\n+\t * scheduling domain before attempting to configure a new one.\n+\t */\n+\tif (dlb->configured) {\n+\t\tdlb_hw_reset_sched_domain(dev, true);\n+\n+\t\tret = dlb_hw_query_resources(dlb);\n+\t\tif (ret) {\n+\t\t\tDLB_LOG_ERR(\"get resources err=%d, devid=%d\\n\",\n+\t\t\t\t    ret, data->dev_id);\n+\t\t\treturn ret;\n+\t\t}\n+\t}\n+\n+\tif (config->nb_event_queues > rsrcs->num_queues) {\n+\t\tDLB_LOG_ERR(\"nb_event_queues parameter (%d) exceeds the QM device's capabilities (%d).\\n\",\n+\t\t\t    config->nb_event_queues,\n+\t\t\t    rsrcs->num_queues);\n+\t\treturn -EINVAL;\n+\t}\n+\tif (config->nb_event_ports > (rsrcs->num_ldb_ports\n+\t\t\t+ rsrcs->num_dir_ports)) {\n+\t\tDLB_LOG_ERR(\"nb_event_ports parameter (%d) exceeds the QM device's capabilities (%d).\\n\",\n+\t\t\t    config->nb_event_ports,\n+\t\t\t    (rsrcs->num_ldb_ports + rsrcs->num_dir_ports));\n+\t\treturn -EINVAL;\n+\t}\n+\tif (config->nb_events_limit > rsrcs->nb_events_limit) {\n+\t\tDLB_LOG_ERR(\"nb_events_limit parameter (%d) exceeds the QM device's capabilities (%d).\\n\",\n+\t\t\t    config->nb_events_limit,\n+\t\t\t    rsrcs->nb_events_limit);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\tif (config->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT) {\n+\t\tdlb->global_dequeue_wait = false;\n+\t} else {\n+\t\tuint32_t timeout32;\n+\n+\t\tdlb->global_dequeue_wait = true;\n+\n+\t\ttimeout32 = config->dequeue_timeout_ns;\n+\n+\t\tdlb->global_dequeue_wait_ticks =\n+\t\t\ttimeout32 * (rte_get_timer_hz() / 1E9);\n+\t}\n+\n+\t/* Does this platform support umonitor/umwait? */\n+\tif (rte_cpu_get_flag_enabled(RTE_CPUFLAG_UMWAIT)) {\n+\t\tif (RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE != 0 &&\n+\t\t    RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE != 1) {\n+\t\t\tDLB_LOG_ERR(\"invalid value (%d) for RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE must be 0 or 1.\\n\",\n+\t\t\t\t    RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE);\n+\t\t\treturn -EINVAL;\n+\t\t}\n+\t\tdlb->umwait_allowed = true;\n+\t}\n+\n+\trsrcs->num_dir_ports = config->nb_single_link_event_port_queues;\n+\trsrcs->num_ldb_ports = config->nb_event_ports - rsrcs->num_dir_ports;\n+\t/* 1 dir queue per dir port */\n+\trsrcs->num_ldb_queues = config->nb_event_queues - rsrcs->num_dir_ports;\n+\n+\t/* Scale down nb_events_limit by 4 for directed credits, since there\n+\t * are 4x as many load-balanced credits.\n+\t */\n+\trsrcs->num_ldb_credits = 0;\n+\trsrcs->num_dir_credits = 0;\n+\n+\tif (rsrcs->num_ldb_queues)\n+\t\trsrcs->num_ldb_credits = config->nb_events_limit;\n+\tif (rsrcs->num_dir_ports)\n+\t\trsrcs->num_dir_credits = config->nb_events_limit / 4;\n+\tif (dlb->num_dir_credits_override != -1)\n+\t\trsrcs->num_dir_credits = dlb->num_dir_credits_override;\n+\n+\tif (dlb_hw_create_sched_domain(handle, dlb, rsrcs) < 0) {\n+\t\tDLB_LOG_ERR(\"dlb_hw_create_sched_domain failed\\n\");\n+\t\treturn -ENODEV;\n+\t}\n+\n+\tdlb->new_event_limit = config->nb_events_limit;\n+\t__atomic_store_n(&dlb->inflights, 0, __ATOMIC_SEQ_CST);\n+\n+\t/* Save number of ports/queues for this event dev */\n+\tdlb->num_ports = config->nb_event_ports;\n+\tdlb->num_queues = config->nb_event_queues;\n+\tdlb->num_dir_ports = rsrcs->num_dir_ports;\n+\tdlb->num_ldb_ports = dlb->num_ports - dlb->num_dir_ports;\n+\tdlb->num_ldb_queues = dlb->num_queues - dlb->num_dir_ports;\n+\tdlb->num_dir_queues = dlb->num_dir_ports;\n+\tdlb->num_ldb_credits = rsrcs->num_ldb_credits;\n+\tdlb->num_dir_credits = rsrcs->num_dir_credits;\n+\n+\tdlb->configured = true;\n \n \treturn 0;\n }\n@@ -309,6 +709,8 @@ void\n dlb_entry_points_init(struct rte_eventdev *dev)\n {\n \tstatic struct rte_eventdev_ops dlb_eventdev_entry_ops = {\n+\t\t.dev_infos_get    = dlb_eventdev_info_get,\n+\t\t.dev_configure    = dlb_eventdev_configure,\n \t\t.dump             = dlb_eventdev_dump,\n \t\t.xstats_get       = dlb_eventdev_xstats_get,\n \t\t.xstats_get_names = dlb_eventdev_xstats_get_names,\ndiff --git a/drivers/event/dlb/dlb_iface.c b/drivers/event/dlb/dlb_iface.c\nindex 3f17bd1..9f74d43 100644\n--- a/drivers/event/dlb/dlb_iface.c\n+++ b/drivers/event/dlb/dlb_iface.c\n@@ -30,12 +30,23 @@ void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);\n \n int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);\n \n+void (*dlb_iface_domain_close)(struct dlb_eventdev *dlb);\n+\n int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,\n \t\t\t\t    uint8_t *revision);\n \n int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,\n \t\t\t\t   struct dlb_get_num_resources_args *rsrcs);\n \n+int (*dlb_iface_sched_domain_create)(struct dlb_hw_dev *handle,\n+\t\t\t\t     struct dlb_create_sched_domain_args *args);\n+\n+int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,\n+\t\t\t\t\tstruct dlb_create_ldb_pool_args *cfg);\n+\n+int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,\n+\t\t\t\t\tstruct dlb_create_dir_pool_args *cfg);\n+\n int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,\n \t\t\t\t  enum dlb_cq_poll_modes *mode);\n \ndiff --git a/drivers/event/dlb/dlb_iface.h b/drivers/event/dlb/dlb_iface.h\nindex 416d1b3..d576232 100644\n--- a/drivers/event/dlb/dlb_iface.h\n+++ b/drivers/event/dlb/dlb_iface.h\n@@ -15,12 +15,23 @@ extern void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);\n \n extern int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);\n \n+extern void (*dlb_iface_domain_close)(struct dlb_eventdev *dlb);\n+\n extern int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,\n \t\t\t\t\t   uint8_t *revision);\n \n extern int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,\n \t\t\t\t   struct dlb_get_num_resources_args *rsrcs);\n \n+extern int (*dlb_iface_sched_domain_create)(struct dlb_hw_dev *handle,\n+\t\t\t\t     struct dlb_create_sched_domain_args *args);\n+\n+extern int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,\n+\t\t\t\t\tstruct dlb_create_ldb_pool_args *cfg);\n+\n+extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,\n+\t\t\t\t\tstruct dlb_create_dir_pool_args *cfg);\n+\n extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,\n \t\t\t\t\t enum dlb_cq_poll_modes *mode);\n \ndiff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c\nindex 9c4267b..0d0bbf9 100644\n--- a/drivers/event/dlb/pf/base/dlb_resource.c\n+++ b/drivers/event/dlb/pf/base/dlb_resource.c\n@@ -10,106 +10,27 @@\n #include \"dlb_osdep_types.h\"\n #include \"dlb_regs.h\"\n \n-void dlb_disable_dp_vasr_feature(struct dlb_hw *hw)\n-{\n-\tunion dlb_dp_dir_csr_ctrl r0;\n-\n-\tr0.val = DLB_CSR_RD(hw, DLB_DP_DIR_CSR_CTRL);\n-\n-\tr0.field.cfg_vasr_dis = 1;\n-\n-\tDLB_CSR_WR(hw, DLB_DP_DIR_CSR_CTRL, r0.val);\n-}\n-\n-void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw)\n-{\n-\tunion dlb_chp_cfg_chp_csr_ctrl r0;\n-\n-\tr0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);\n-\n-\tr0.val |= 1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT;\n-\n-\tDLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);\n-}\n-\n-void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw)\n-{\n-\tunion dlb_sys_cq_mode r0;\n-\n-\tr0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);\n-\n-\tr0.field.ldb_cq64 = 1;\n-\n-\tDLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);\n-}\n+#define DLB_DOM_LIST_HEAD(head, type) \\\n+\tDLB_LIST_HEAD((head), type, domain_list)\n \n-void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw)\n-{\n-\tunion dlb_sys_cq_mode r0;\n-\n-\tr0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);\n-\n-\tr0.field.dir_cq64 = 1;\n-\n-\tDLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);\n-}\n+#define DLB_FUNC_LIST_HEAD(head, type) \\\n+\tDLB_LIST_HEAD((head), type, func_list)\n \n-void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw)\n-{\n-\tunion dlb_sys_sys_alarm_int_enable r0;\n+#define DLB_DOM_LIST_FOR(head, ptr, iter) \\\n+\tDLB_LIST_FOR_EACH(head, ptr, domain_list, iter)\n \n-\tr0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);\n+#define DLB_FUNC_LIST_FOR(head, ptr, iter) \\\n+\tDLB_LIST_FOR_EACH(head, ptr, func_list, iter)\n \n-\tr0.field.pf_to_vf_isr_pend_error = 0;\n+#define DLB_DOM_LIST_FOR_SAFE(head, ptr, ptr_tmp, it, it_tmp) \\\n+\tDLB_LIST_FOR_EACH_SAFE((head), ptr, ptr_tmp, domain_list, it, it_tmp)\n \n-\tDLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);\n-}\n+#define DLB_FUNC_LIST_FOR_SAFE(head, ptr, ptr_tmp, it, it_tmp) \\\n+\tDLB_LIST_FOR_EACH_SAFE((head), ptr, ptr_tmp, func_list, it, it_tmp)\n \n-void dlb_hw_get_num_resources(struct dlb_hw *hw,\n-\t\t\t      struct dlb_get_num_resources_args *arg)\n+static inline void dlb_flush_csr(struct dlb_hw *hw)\n {\n-\tstruct dlb_function_resources *rsrcs;\n-\tstruct dlb_bitmap *map;\n-\n-\trsrcs = &hw->pf;\n-\n-\targ->num_sched_domains = rsrcs->num_avail_domains;\n-\n-\targ->num_ldb_queues = rsrcs->num_avail_ldb_queues;\n-\n-\targ->num_ldb_ports = rsrcs->num_avail_ldb_ports;\n-\n-\targ->num_dir_ports = rsrcs->num_avail_dir_pq_pairs;\n-\n-\tmap = rsrcs->avail_aqed_freelist_entries;\n-\n-\targ->num_atomic_inflights = dlb_bitmap_count(map);\n-\n-\targ->max_contiguous_atomic_inflights =\n-\t\tdlb_bitmap_longest_set_range(map);\n-\n-\tmap = rsrcs->avail_hist_list_entries;\n-\n-\targ->num_hist_list_entries = dlb_bitmap_count(map);\n-\n-\targ->max_contiguous_hist_list_entries =\n-\t\tdlb_bitmap_longest_set_range(map);\n-\n-\tmap = rsrcs->avail_qed_freelist_entries;\n-\n-\targ->num_ldb_credits = dlb_bitmap_count(map);\n-\n-\targ->max_contiguous_ldb_credits = dlb_bitmap_longest_set_range(map);\n-\n-\tmap = rsrcs->avail_dqed_freelist_entries;\n-\n-\targ->num_dir_credits = dlb_bitmap_count(map);\n-\n-\targ->max_contiguous_dir_credits = dlb_bitmap_longest_set_range(map);\n-\n-\targ->num_ldb_credit_pools = rsrcs->num_avail_ldb_credit_pools;\n-\n-\targ->num_dir_credit_pools = rsrcs->num_avail_dir_credit_pools;\n+\tDLB_CSR_RD(hw, DLB_SYS_TOTAL_VAS);\n }\n \n static void dlb_init_fn_rsrc_lists(struct dlb_function_resources *rsrc)\n@@ -290,6 +211,3997 @@ void dlb_resource_free(struct dlb_hw *hw)\n \tdlb_bitmap_free(hw->pf.avail_aqed_freelist_entries);\n }\n \n+static struct dlb_domain *dlb_get_domain_from_id(struct dlb_hw *hw, u32 id)\n+{\n+\tif (id >= DLB_MAX_NUM_DOMAINS)\n+\t\treturn NULL;\n+\n+\treturn &hw->domains[id];\n+}\n+\n+static int dlb_attach_ldb_queues(struct dlb_hw *hw,\n+\t\t\t\t struct dlb_function_resources *rsrcs,\n+\t\t\t\t struct dlb_domain *domain,\n+\t\t\t\t u32 num_queues,\n+\t\t\t\t struct dlb_cmd_response *resp)\n+{\n+\tunsigned int i, j;\n+\n+\tif (rsrcs->num_avail_ldb_queues < num_queues) {\n+\t\tresp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;\n+\t\treturn -1;\n+\t}\n+\n+\tfor (i = 0; i < num_queues; i++) {\n+\t\tstruct dlb_ldb_queue *queue;\n+\n+\t\tqueue = DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_queues,\n+\t\t\t\t\t   typeof(*queue));\n+\t\tif (!queue) {\n+\t\t\tDLB_HW_ERR(hw,\n+\t\t\t\t   \"[%s()] Internal error: domain validation failed\\n\",\n+\t\t\t\t   __func__);\n+\t\t\tgoto cleanup;\n+\t\t}\n+\n+\t\tdlb_list_del(&rsrcs->avail_ldb_queues, &queue->func_list);\n+\n+\t\tqueue->domain_id = domain->id;\n+\t\tqueue->owned = true;\n+\n+\t\tdlb_list_add(&domain->avail_ldb_queues, &queue->domain_list);\n+\t}\n+\n+\trsrcs->num_avail_ldb_queues -= num_queues;\n+\n+\treturn 0;\n+\n+cleanup:\n+\n+\t/* Return the assigned queues */\n+\tfor (j = 0; j < i; j++) {\n+\t\tstruct dlb_ldb_queue *queue;\n+\n+\t\tqueue = DLB_FUNC_LIST_HEAD(domain->avail_ldb_queues,\n+\t\t\t\t\t   typeof(*queue));\n+\t\t/* Unrecoverable internal error */\n+\t\tif (!queue)\n+\t\t\tbreak;\n+\n+\t\tqueue->owned = false;\n+\n+\t\tdlb_list_del(&domain->avail_ldb_queues, &queue->domain_list);\n+\n+\t\tdlb_list_add(&rsrcs->avail_ldb_queues, &queue->func_list);\n+\t}\n+\n+\treturn -EFAULT;\n+}\n+\n+static struct dlb_ldb_port *\n+dlb_get_next_ldb_port(struct dlb_hw *hw,\n+\t\t      struct dlb_function_resources *rsrcs,\n+\t\t      u32 domain_id)\n+{\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tstruct dlb_ldb_port *port;\n+\n+\t/* To reduce the odds of consecutive load-balanced ports mapping to the\n+\t * same queue(s), the driver attempts to allocate ports whose neighbors\n+\t * are owned by a different domain.\n+\t */\n+\tDLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {\n+\t\tu32 next, prev;\n+\t\tu32 phys_id;\n+\n+\t\tphys_id = port->id;\n+\t\tnext = phys_id + 1;\n+\t\tprev = phys_id - 1;\n+\n+\t\tif (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)\n+\t\t\tnext = 0;\n+\t\tif (phys_id == 0)\n+\t\t\tprev = DLB_MAX_NUM_LDB_PORTS - 1;\n+\n+\t\tif (!hw->rsrcs.ldb_ports[next].owned ||\n+\t\t    hw->rsrcs.ldb_ports[next].domain_id == domain_id)\n+\t\t\tcontinue;\n+\n+\t\tif (!hw->rsrcs.ldb_ports[prev].owned ||\n+\t\t    hw->rsrcs.ldb_ports[prev].domain_id == domain_id)\n+\t\t\tcontinue;\n+\n+\t\treturn port;\n+\t}\n+\n+\t/* Failing that, the driver looks for a port with one neighbor owned by\n+\t * a different domain and the other unallocated.\n+\t */\n+\tDLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {\n+\t\tu32 next, prev;\n+\t\tu32 phys_id;\n+\n+\t\tphys_id = port->id;\n+\t\tnext = phys_id + 1;\n+\t\tprev = phys_id - 1;\n+\n+\t\tif (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)\n+\t\t\tnext = 0;\n+\t\tif (phys_id == 0)\n+\t\t\tprev = DLB_MAX_NUM_LDB_PORTS - 1;\n+\n+\t\tif (!hw->rsrcs.ldb_ports[prev].owned &&\n+\t\t    hw->rsrcs.ldb_ports[next].owned &&\n+\t\t    hw->rsrcs.ldb_ports[next].domain_id != domain_id)\n+\t\t\treturn port;\n+\n+\t\tif (!hw->rsrcs.ldb_ports[next].owned &&\n+\t\t    hw->rsrcs.ldb_ports[prev].owned &&\n+\t\t    hw->rsrcs.ldb_ports[prev].domain_id != domain_id)\n+\t\t\treturn port;\n+\t}\n+\n+\t/* Failing that, the driver looks for a port with both neighbors\n+\t * unallocated.\n+\t */\n+\tDLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {\n+\t\tu32 next, prev;\n+\t\tu32 phys_id;\n+\n+\t\tphys_id = port->id;\n+\t\tnext = phys_id + 1;\n+\t\tprev = phys_id - 1;\n+\n+\t\tif (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)\n+\t\t\tnext = 0;\n+\t\tif (phys_id == 0)\n+\t\t\tprev = DLB_MAX_NUM_LDB_PORTS - 1;\n+\n+\t\tif (!hw->rsrcs.ldb_ports[prev].owned &&\n+\t\t    !hw->rsrcs.ldb_ports[next].owned)\n+\t\t\treturn port;\n+\t}\n+\n+\t/* If all else fails, the driver returns the next available port. */\n+\treturn DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_ports, typeof(*port));\n+}\n+\n+static int dlb_attach_ldb_ports(struct dlb_hw *hw,\n+\t\t\t\tstruct dlb_function_resources *rsrcs,\n+\t\t\t\tstruct dlb_domain *domain,\n+\t\t\t\tu32 num_ports,\n+\t\t\t\tstruct dlb_cmd_response *resp)\n+{\n+\tunsigned int i, j;\n+\n+\tif (rsrcs->num_avail_ldb_ports < num_ports) {\n+\t\tresp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;\n+\t\treturn -1;\n+\t}\n+\n+\tfor (i = 0; i < num_ports; i++) {\n+\t\tstruct dlb_ldb_port *port;\n+\n+\t\tport = dlb_get_next_ldb_port(hw, rsrcs, domain->id);\n+\n+\t\tif (!port) {\n+\t\t\tDLB_HW_ERR(hw,\n+\t\t\t\t   \"[%s()] Internal error: domain validation failed\\n\",\n+\t\t\t\t   __func__);\n+\t\t\tgoto cleanup;\n+\t\t}\n+\n+\t\tdlb_list_del(&rsrcs->avail_ldb_ports, &port->func_list);\n+\n+\t\tport->domain_id = domain->id;\n+\t\tport->owned = true;\n+\n+\t\tdlb_list_add(&domain->avail_ldb_ports, &port->domain_list);\n+\t}\n+\n+\trsrcs->num_avail_ldb_ports -= num_ports;\n+\n+\treturn 0;\n+\n+cleanup:\n+\n+\t/* Return the assigned ports */\n+\tfor (j = 0; j < i; j++) {\n+\t\tstruct dlb_ldb_port *port;\n+\n+\t\tport = DLB_FUNC_LIST_HEAD(domain->avail_ldb_ports,\n+\t\t\t\t\t  typeof(*port));\n+\t\t/* Unrecoverable internal error */\n+\t\tif (!port)\n+\t\t\tbreak;\n+\n+\t\tport->owned = false;\n+\n+\t\tdlb_list_del(&domain->avail_ldb_ports, &port->domain_list);\n+\n+\t\tdlb_list_add(&rsrcs->avail_ldb_ports, &port->func_list);\n+\t}\n+\n+\treturn -EFAULT;\n+}\n+\n+static int dlb_attach_dir_ports(struct dlb_hw *hw,\n+\t\t\t\tstruct dlb_function_resources *rsrcs,\n+\t\t\t\tstruct dlb_domain *domain,\n+\t\t\t\tu32 num_ports,\n+\t\t\t\tstruct dlb_cmd_response *resp)\n+{\n+\tunsigned int i, j;\n+\n+\tif (rsrcs->num_avail_dir_pq_pairs < num_ports) {\n+\t\tresp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;\n+\t\treturn -1;\n+\t}\n+\n+\tfor (i = 0; i < num_ports; i++) {\n+\t\tstruct dlb_dir_pq_pair *port;\n+\n+\t\tport = DLB_FUNC_LIST_HEAD(rsrcs->avail_dir_pq_pairs,\n+\t\t\t\t\t  typeof(*port));\n+\t\tif (!port) {\n+\t\t\tDLB_HW_ERR(hw,\n+\t\t\t\t   \"[%s()] Internal error: domain validation failed\\n\",\n+\t\t\t\t   __func__);\n+\t\t\tgoto cleanup;\n+\t\t}\n+\n+\t\tdlb_list_del(&rsrcs->avail_dir_pq_pairs, &port->func_list);\n+\n+\t\tport->domain_id = domain->id;\n+\t\tport->owned = true;\n+\n+\t\tdlb_list_add(&domain->avail_dir_pq_pairs, &port->domain_list);\n+\t}\n+\n+\trsrcs->num_avail_dir_pq_pairs -= num_ports;\n+\n+\treturn 0;\n+\n+cleanup:\n+\n+\t/* Return the assigned ports */\n+\tfor (j = 0; j < i; j++) {\n+\t\tstruct dlb_dir_pq_pair *port;\n+\n+\t\tport = DLB_FUNC_LIST_HEAD(domain->avail_dir_pq_pairs,\n+\t\t\t\t\t  typeof(*port));\n+\t\t/* Unrecoverable internal error */\n+\t\tif (!port)\n+\t\t\tbreak;\n+\n+\t\tport->owned = false;\n+\n+\t\tdlb_list_del(&domain->avail_dir_pq_pairs, &port->domain_list);\n+\n+\t\tdlb_list_add(&rsrcs->avail_dir_pq_pairs, &port->func_list);\n+\t}\n+\n+\treturn -EFAULT;\n+}\n+\n+static int dlb_attach_ldb_credits(struct dlb_function_resources *rsrcs,\n+\t\t\t\t  struct dlb_domain *domain,\n+\t\t\t\t  u32 num_credits,\n+\t\t\t\t  struct dlb_cmd_response *resp)\n+{\n+\tstruct dlb_bitmap *bitmap = rsrcs->avail_qed_freelist_entries;\n+\n+\tif (dlb_bitmap_count(bitmap) < (int)num_credits) {\n+\t\tresp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;\n+\t\treturn -1;\n+\t}\n+\n+\tif (num_credits) {\n+\t\tint base;\n+\n+\t\tbase = dlb_bitmap_find_set_bit_range(bitmap, num_credits);\n+\t\tif (base < 0)\n+\t\t\tgoto error;\n+\n+\t\tdomain->qed_freelist.base = base;\n+\t\tdomain->qed_freelist.bound = base + num_credits;\n+\t\tdomain->qed_freelist.offset = 0;\n+\n+\t\tdlb_bitmap_clear_range(bitmap, base, num_credits);\n+\t}\n+\n+\treturn 0;\n+\n+error:\n+\tresp->status = DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE;\n+\treturn -1;\n+}\n+\n+static int dlb_attach_dir_credits(struct dlb_function_resources *rsrcs,\n+\t\t\t\t  struct dlb_domain *domain,\n+\t\t\t\t  u32 num_credits,\n+\t\t\t\t  struct dlb_cmd_response *resp)\n+{\n+\tstruct dlb_bitmap *bitmap = rsrcs->avail_dqed_freelist_entries;\n+\n+\tif (dlb_bitmap_count(bitmap) < (int)num_credits) {\n+\t\tresp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;\n+\t\treturn -1;\n+\t}\n+\n+\tif (num_credits) {\n+\t\tint base;\n+\n+\t\tbase = dlb_bitmap_find_set_bit_range(bitmap, num_credits);\n+\t\tif (base < 0)\n+\t\t\tgoto error;\n+\n+\t\tdomain->dqed_freelist.base = base;\n+\t\tdomain->dqed_freelist.bound = base + num_credits;\n+\t\tdomain->dqed_freelist.offset = 0;\n+\n+\t\tdlb_bitmap_clear_range(bitmap, base, num_credits);\n+\t}\n+\n+\treturn 0;\n+\n+error:\n+\tresp->status = DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE;\n+\treturn -1;\n+}\n+\n+static int dlb_attach_ldb_credit_pools(struct dlb_hw *hw,\n+\t\t\t\t       struct dlb_function_resources *rsrcs,\n+\t\t\t\t       struct dlb_domain *domain,\n+\t\t\t\t       u32 num_credit_pools,\n+\t\t\t\t       struct dlb_cmd_response *resp)\n+{\n+\tunsigned int i, j;\n+\n+\tif (rsrcs->num_avail_ldb_credit_pools < num_credit_pools) {\n+\t\tresp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;\n+\t\treturn -1;\n+\t}\n+\n+\tfor (i = 0; i < num_credit_pools; i++) {\n+\t\tstruct dlb_credit_pool *pool;\n+\n+\t\tpool = DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_credit_pools,\n+\t\t\t\t\t  typeof(*pool));\n+\t\tif (!pool) {\n+\t\t\tDLB_HW_ERR(hw,\n+\t\t\t\t   \"[%s()] Internal error: domain validation failed\\n\",\n+\t\t\t\t   __func__);\n+\t\t\tgoto cleanup;\n+\t\t}\n+\n+\t\tdlb_list_del(&rsrcs->avail_ldb_credit_pools,\n+\t\t\t     &pool->func_list);\n+\n+\t\tpool->domain_id = domain->id;\n+\t\tpool->owned = true;\n+\n+\t\tdlb_list_add(&domain->avail_ldb_credit_pools,\n+\t\t\t     &pool->domain_list);\n+\t}\n+\n+\trsrcs->num_avail_ldb_credit_pools -= num_credit_pools;\n+\n+\treturn 0;\n+\n+cleanup:\n+\n+\t/* Return the assigned credit pools */\n+\tfor (j = 0; j < i; j++) {\n+\t\tstruct dlb_credit_pool *pool;\n+\n+\t\tpool = DLB_FUNC_LIST_HEAD(domain->avail_ldb_credit_pools,\n+\t\t\t\t\t  typeof(*pool));\n+\t\t/* Unrecoverable internal error */\n+\t\tif (!pool)\n+\t\t\tbreak;\n+\n+\t\tpool->owned = false;\n+\n+\t\tdlb_list_del(&domain->avail_ldb_credit_pools,\n+\t\t\t     &pool->domain_list);\n+\n+\t\tdlb_list_add(&rsrcs->avail_ldb_credit_pools,\n+\t\t\t     &pool->func_list);\n+\t}\n+\n+\treturn -EFAULT;\n+}\n+\n+static int dlb_attach_dir_credit_pools(struct dlb_hw *hw,\n+\t\t\t\t       struct dlb_function_resources *rsrcs,\n+\t\t\t\t       struct dlb_domain *domain,\n+\t\t\t\t       u32 num_credit_pools,\n+\t\t\t\t       struct dlb_cmd_response *resp)\n+{\n+\tunsigned int i, j;\n+\n+\tif (rsrcs->num_avail_dir_credit_pools < num_credit_pools) {\n+\t\tresp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;\n+\t\treturn -1;\n+\t}\n+\n+\tfor (i = 0; i < num_credit_pools; i++) {\n+\t\tstruct dlb_credit_pool *pool;\n+\n+\t\tpool = DLB_FUNC_LIST_HEAD(rsrcs->avail_dir_credit_pools,\n+\t\t\t\t\t  typeof(*pool));\n+\t\tif (!pool) {\n+\t\t\tDLB_HW_ERR(hw,\n+\t\t\t\t   \"[%s()] Internal error: domain validation failed\\n\",\n+\t\t\t\t   __func__);\n+\t\t\tgoto cleanup;\n+\t\t}\n+\n+\t\tdlb_list_del(&rsrcs->avail_dir_credit_pools,\n+\t\t\t     &pool->func_list);\n+\n+\t\tpool->domain_id = domain->id;\n+\t\tpool->owned = true;\n+\n+\t\tdlb_list_add(&domain->avail_dir_credit_pools,\n+\t\t\t     &pool->domain_list);\n+\t}\n+\n+\trsrcs->num_avail_dir_credit_pools -= num_credit_pools;\n+\n+\treturn 0;\n+\n+cleanup:\n+\n+\t/* Return the assigned credit pools */\n+\tfor (j = 0; j < i; j++) {\n+\t\tstruct dlb_credit_pool *pool;\n+\n+\t\tpool = DLB_FUNC_LIST_HEAD(domain->avail_dir_credit_pools,\n+\t\t\t\t\t  typeof(*pool));\n+\t\t/* Unrecoverable internal error */\n+\t\tif (!pool)\n+\t\t\tbreak;\n+\n+\t\tpool->owned = false;\n+\n+\t\tdlb_list_del(&domain->avail_dir_credit_pools,\n+\t\t\t     &pool->domain_list);\n+\n+\t\tdlb_list_add(&rsrcs->avail_dir_credit_pools,\n+\t\t\t     &pool->func_list);\n+\t}\n+\n+\treturn -EFAULT;\n+}\n+\n+static int\n+dlb_attach_domain_hist_list_entries(struct dlb_function_resources *rsrcs,\n+\t\t\t\t    struct dlb_domain *domain,\n+\t\t\t\t    u32 num_hist_list_entries,\n+\t\t\t\t    struct dlb_cmd_response *resp)\n+{\n+\tstruct dlb_bitmap *bitmap;\n+\tint base;\n+\n+\tif (num_hist_list_entries) {\n+\t\tbitmap = rsrcs->avail_hist_list_entries;\n+\n+\t\tbase = dlb_bitmap_find_set_bit_range(bitmap,\n+\t\t\t\t\t\t     num_hist_list_entries);\n+\t\tif (base < 0)\n+\t\t\tgoto error;\n+\n+\t\tdomain->total_hist_list_entries = num_hist_list_entries;\n+\t\tdomain->avail_hist_list_entries = num_hist_list_entries;\n+\t\tdomain->hist_list_entry_base = base;\n+\t\tdomain->hist_list_entry_offset = 0;\n+\n+\t\tdlb_bitmap_clear_range(bitmap, base, num_hist_list_entries);\n+\t}\n+\treturn 0;\n+\n+error:\n+\tresp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;\n+\treturn -1;\n+}\n+\n+static int dlb_attach_atomic_inflights(struct dlb_function_resources *rsrcs,\n+\t\t\t\t       struct dlb_domain *domain,\n+\t\t\t\t       u32 num_atomic_inflights,\n+\t\t\t\t       struct dlb_cmd_response *resp)\n+{\n+\tif (num_atomic_inflights) {\n+\t\tstruct dlb_bitmap *bitmap =\n+\t\t\trsrcs->avail_aqed_freelist_entries;\n+\t\tint base;\n+\n+\t\tbase = dlb_bitmap_find_set_bit_range(bitmap,\n+\t\t\t\t\t\t     num_atomic_inflights);\n+\t\tif (base < 0)\n+\t\t\tgoto error;\n+\n+\t\tdomain->aqed_freelist.base = base;\n+\t\tdomain->aqed_freelist.bound = base + num_atomic_inflights;\n+\t\tdomain->aqed_freelist.offset = 0;\n+\n+\t\tdlb_bitmap_clear_range(bitmap, base, num_atomic_inflights);\n+\t}\n+\n+\treturn 0;\n+\n+error:\n+\tresp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;\n+\treturn -1;\n+}\n+\n+\n+static int\n+dlb_domain_attach_resources(struct dlb_hw *hw,\n+\t\t\t    struct dlb_function_resources *rsrcs,\n+\t\t\t    struct dlb_domain *domain,\n+\t\t\t    struct dlb_create_sched_domain_args *args,\n+\t\t\t    struct dlb_cmd_response *resp)\n+{\n+\tint ret;\n+\n+\tret = dlb_attach_ldb_queues(hw,\n+\t\t\t\t    rsrcs,\n+\t\t\t\t    domain,\n+\t\t\t\t    args->num_ldb_queues,\n+\t\t\t\t    resp);\n+\tif (ret < 0)\n+\t\treturn ret;\n+\n+\tret = dlb_attach_ldb_ports(hw,\n+\t\t\t\t   rsrcs,\n+\t\t\t\t   domain,\n+\t\t\t\t   args->num_ldb_ports,\n+\t\t\t\t   resp);\n+\tif (ret < 0)\n+\t\treturn ret;\n+\n+\tret = dlb_attach_dir_ports(hw,\n+\t\t\t\t   rsrcs,\n+\t\t\t\t   domain,\n+\t\t\t\t   args->num_dir_ports,\n+\t\t\t\t   resp);\n+\tif (ret < 0)\n+\t\treturn ret;\n+\n+\tret = dlb_attach_ldb_credits(rsrcs,\n+\t\t\t\t     domain,\n+\t\t\t\t     args->num_ldb_credits,\n+\t\t\t\t     resp);\n+\tif (ret < 0)\n+\t\treturn ret;\n+\n+\tret = dlb_attach_dir_credits(rsrcs,\n+\t\t\t\t     domain,\n+\t\t\t\t     args->num_dir_credits,\n+\t\t\t\t     resp);\n+\tif (ret < 0)\n+\t\treturn ret;\n+\n+\tret = dlb_attach_ldb_credit_pools(hw,\n+\t\t\t\t\t  rsrcs,\n+\t\t\t\t\t  domain,\n+\t\t\t\t\t  args->num_ldb_credit_pools,\n+\t\t\t\t\t  resp);\n+\tif (ret < 0)\n+\t\treturn ret;\n+\n+\tret = dlb_attach_dir_credit_pools(hw,\n+\t\t\t\t\t  rsrcs,\n+\t\t\t\t\t  domain,\n+\t\t\t\t\t  args->num_dir_credit_pools,\n+\t\t\t\t\t  resp);\n+\tif (ret < 0)\n+\t\treturn ret;\n+\n+\tret = dlb_attach_domain_hist_list_entries(rsrcs,\n+\t\t\t\t\t\t  domain,\n+\t\t\t\t\t\t  args->num_hist_list_entries,\n+\t\t\t\t\t\t  resp);\n+\tif (ret < 0)\n+\t\treturn ret;\n+\n+\tret = dlb_attach_atomic_inflights(rsrcs,\n+\t\t\t\t\t  domain,\n+\t\t\t\t\t  args->num_atomic_inflights,\n+\t\t\t\t\t  resp);\n+\tif (ret < 0)\n+\t\treturn ret;\n+\n+\tdomain->configured = true;\n+\n+\tdomain->started = false;\n+\n+\trsrcs->num_avail_domains--;\n+\n+\treturn 0;\n+}\n+\n+static void dlb_ldb_port_cq_enable(struct dlb_hw *hw,\n+\t\t\t\t   struct dlb_ldb_port *port)\n+{\n+\tunion dlb_lsp_cq_ldb_dsbl reg;\n+\n+\t/* Don't re-enable the port if a removal is pending. The caller should\n+\t * mark this port as enabled (if it isn't already), and when the\n+\t * removal completes the port will be enabled.\n+\t */\n+\tif (port->num_pending_removals)\n+\t\treturn;\n+\n+\treg.field.disabled = 0;\n+\n+\tDLB_CSR_WR(hw, DLB_LSP_CQ_LDB_DSBL(port->id), reg.val);\n+\n+\tdlb_flush_csr(hw);\n+}\n+\n+static void dlb_dir_port_cq_enable(struct dlb_hw *hw,\n+\t\t\t\t   struct dlb_dir_pq_pair *port)\n+{\n+\tunion dlb_lsp_cq_dir_dsbl reg;\n+\n+\treg.field.disabled = 0;\n+\n+\tDLB_CSR_WR(hw, DLB_LSP_CQ_DIR_DSBL(port->id), reg.val);\n+\n+\tdlb_flush_csr(hw);\n+}\n+\n+\n+static void dlb_ldb_port_cq_disable(struct dlb_hw *hw,\n+\t\t\t\t    struct dlb_ldb_port *port)\n+{\n+\tunion dlb_lsp_cq_ldb_dsbl reg;\n+\n+\treg.field.disabled = 1;\n+\n+\tDLB_CSR_WR(hw, DLB_LSP_CQ_LDB_DSBL(port->id), reg.val);\n+\n+\tdlb_flush_csr(hw);\n+}\n+\n+static void dlb_dir_port_cq_disable(struct dlb_hw *hw,\n+\t\t\t\t    struct dlb_dir_pq_pair *port)\n+{\n+\tunion dlb_lsp_cq_dir_dsbl reg;\n+\n+\treg.field.disabled = 1;\n+\n+\tDLB_CSR_WR(hw, DLB_LSP_CQ_DIR_DSBL(port->id), reg.val);\n+\n+\tdlb_flush_csr(hw);\n+}\n+\n+\n+\n+void dlb_disable_dp_vasr_feature(struct dlb_hw *hw)\n+{\n+\tunion dlb_dp_dir_csr_ctrl r0;\n+\n+\tr0.val = DLB_CSR_RD(hw, DLB_DP_DIR_CSR_CTRL);\n+\n+\tr0.field.cfg_vasr_dis = 1;\n+\n+\tDLB_CSR_WR(hw, DLB_DP_DIR_CSR_CTRL, r0.val);\n+}\n+\n+void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw)\n+{\n+\tunion dlb_chp_cfg_chp_csr_ctrl r0;\n+\n+\tr0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);\n+\n+\tr0.val |= 1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT;\n+\n+\tDLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);\n+}\n+\n+void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw)\n+{\n+\tunion dlb_sys_cq_mode r0;\n+\n+\tr0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);\n+\n+\tr0.field.ldb_cq64 = 1;\n+\n+\tDLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);\n+}\n+\n+void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw)\n+{\n+\tunion dlb_sys_cq_mode r0;\n+\n+\tr0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);\n+\n+\tr0.field.dir_cq64 = 1;\n+\n+\tDLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);\n+}\n+\n+void dlb_hw_disable_pf_to_vf_isr_pend_err(struct dlb_hw *hw)\n+{\n+\tunion dlb_sys_sys_alarm_int_enable r0;\n+\n+\tr0.val = DLB_CSR_RD(hw, DLB_SYS_SYS_ALARM_INT_ENABLE);\n+\n+\tr0.field.pf_to_vf_isr_pend_error = 0;\n+\n+\tDLB_CSR_WR(hw, DLB_SYS_SYS_ALARM_INT_ENABLE, r0.val);\n+}\n+\n+static unsigned int\n+dlb_get_num_ports_in_use(struct dlb_hw *hw)\n+{\n+\tunsigned int i, n = 0;\n+\n+\tfor (i = 0; i < DLB_MAX_NUM_LDB_PORTS; i++)\n+\t\tif (hw->rsrcs.ldb_ports[i].owned)\n+\t\t\tn++;\n+\n+\tfor (i = 0; i < DLB_MAX_NUM_DIR_PORTS; i++)\n+\t\tif (hw->rsrcs.dir_pq_pairs[i].owned)\n+\t\t\tn++;\n+\n+\treturn n;\n+}\n+\n+static bool dlb_port_find_slot(struct dlb_ldb_port *port,\n+\t\t\t       enum dlb_qid_map_state state,\n+\t\t\t       int *slot)\n+{\n+\tint i;\n+\n+\tfor (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {\n+\t\tif (port->qid_map[i].state == state)\n+\t\t\tbreak;\n+\t}\n+\n+\t*slot = i;\n+\n+\treturn (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);\n+}\n+\n+static bool dlb_port_find_slot_queue(struct dlb_ldb_port *port,\n+\t\t\t\t     enum dlb_qid_map_state state,\n+\t\t\t\t     struct dlb_ldb_queue *queue,\n+\t\t\t\t     int *slot)\n+{\n+\tint i;\n+\n+\tfor (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {\n+\t\tif (port->qid_map[i].state == state &&\n+\t\t    port->qid_map[i].qid == queue->id)\n+\t\t\tbreak;\n+\t}\n+\n+\t*slot = i;\n+\n+\treturn (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);\n+}\n+\n+static int dlb_port_slot_state_transition(struct dlb_hw *hw,\n+\t\t\t\t\t  struct dlb_ldb_port *port,\n+\t\t\t\t\t  struct dlb_ldb_queue *queue,\n+\t\t\t\t\t  int slot,\n+\t\t\t\t\t  enum dlb_qid_map_state new_state)\n+{\n+\tenum dlb_qid_map_state curr_state = port->qid_map[slot].state;\n+\tstruct dlb_domain *domain;\n+\n+\tdomain = dlb_get_domain_from_id(hw, port->domain_id);\n+\tif (!domain) {\n+\t\tDLB_HW_ERR(hw,\n+\t\t\t   \"[%s()] Internal error: unable to find domain %d\\n\",\n+\t\t\t   __func__, port->domain_id);\n+\t\treturn -EFAULT;\n+\t}\n+\n+\tswitch (curr_state) {\n+\tcase DLB_QUEUE_UNMAPPED:\n+\t\tswitch (new_state) {\n+\t\tcase DLB_QUEUE_MAPPED:\n+\t\t\tqueue->num_mappings++;\n+\t\t\tport->num_mappings++;\n+\t\t\tbreak;\n+\t\tcase DLB_QUEUE_MAP_IN_PROGRESS:\n+\t\t\tqueue->num_pending_additions++;\n+\t\t\tdomain->num_pending_additions++;\n+\t\t\tbreak;\n+\t\tdefault:\n+\t\t\tgoto error;\n+\t\t}\n+\t\tbreak;\n+\tcase DLB_QUEUE_MAPPED:\n+\t\tswitch (new_state) {\n+\t\tcase DLB_QUEUE_UNMAPPED:\n+\t\t\tqueue->num_mappings--;\n+\t\t\tport->num_mappings--;\n+\t\t\tbreak;\n+\t\tcase DLB_QUEUE_UNMAP_IN_PROGRESS:\n+\t\t\tport->num_pending_removals++;\n+\t\t\tdomain->num_pending_removals++;\n+\t\t\tbreak;\n+\t\tcase DLB_QUEUE_MAPPED:\n+\t\t\t/* Priority change, nothing to update */\n+\t\t\tbreak;\n+\t\tdefault:\n+\t\t\tgoto error;\n+\t\t}\n+\t\tbreak;\n+\tcase DLB_QUEUE_MAP_IN_PROGRESS:\n+\t\tswitch (new_state) {\n+\t\tcase DLB_QUEUE_UNMAPPED:\n+\t\t\tqueue->num_pending_additions--;\n+\t\t\tdomain->num_pending_additions--;\n+\t\t\tbreak;\n+\t\tcase DLB_QUEUE_MAPPED:\n+\t\t\tqueue->num_mappings++;\n+\t\t\tport->num_mappings++;\n+\t\t\tqueue->num_pending_additions--;\n+\t\t\tdomain->num_pending_additions--;\n+\t\t\tbreak;\n+\t\tdefault:\n+\t\t\tgoto error;\n+\t\t}\n+\t\tbreak;\n+\tcase DLB_QUEUE_UNMAP_IN_PROGRESS:\n+\t\tswitch (new_state) {\n+\t\tcase DLB_QUEUE_UNMAPPED:\n+\t\t\tport->num_pending_removals--;\n+\t\t\tdomain->num_pending_removals--;\n+\t\t\tqueue->num_mappings--;\n+\t\t\tport->num_mappings--;\n+\t\t\tbreak;\n+\t\tcase DLB_QUEUE_MAPPED:\n+\t\t\tport->num_pending_removals--;\n+\t\t\tdomain->num_pending_removals--;\n+\t\t\tbreak;\n+\t\tcase DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP:\n+\t\t\t/* Nothing to update */\n+\t\t\tbreak;\n+\t\tdefault:\n+\t\t\tgoto error;\n+\t\t}\n+\t\tbreak;\n+\tcase DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP:\n+\t\tswitch (new_state) {\n+\t\tcase DLB_QUEUE_UNMAP_IN_PROGRESS:\n+\t\t\t/* Nothing to update */\n+\t\t\tbreak;\n+\t\tcase DLB_QUEUE_UNMAPPED:\n+\t\t\t/* An UNMAP_IN_PROGRESS_PENDING_MAP slot briefly\n+\t\t\t * becomes UNMAPPED before it transitions to\n+\t\t\t * MAP_IN_PROGRESS.\n+\t\t\t */\n+\t\t\tqueue->num_mappings--;\n+\t\t\tport->num_mappings--;\n+\t\t\tport->num_pending_removals--;\n+\t\t\tdomain->num_pending_removals--;\n+\t\t\tbreak;\n+\t\tdefault:\n+\t\t\tgoto error;\n+\t\t}\n+\t\tbreak;\n+\tdefault:\n+\t\tgoto error;\n+\t}\n+\n+\tport->qid_map[slot].state = new_state;\n+\n+\tDLB_HW_INFO(hw,\n+\t\t    \"[%s()] queue %d -> port %d state transition (%d -> %d)\\n\",\n+\t\t    __func__, queue->id, port->id, curr_state,\n+\t\t    new_state);\n+\treturn 0;\n+\n+error:\n+\tDLB_HW_ERR(hw,\n+\t\t   \"[%s()] Internal error: invalid queue %d -> port %d state transition (%d -> %d)\\n\",\n+\t\t   __func__, queue->id, port->id, curr_state,\n+\t\t   new_state);\n+\treturn -EFAULT;\n+}\n+\n+/* dlb_ldb_queue_{enable, disable}_mapped_cqs() don't operate exactly as their\n+ * function names imply, and should only be called by the dynamic CQ mapping\n+ * code.\n+ */\n+static void dlb_ldb_queue_disable_mapped_cqs(struct dlb_hw *hw,\n+\t\t\t\t\t     struct dlb_domain *domain,\n+\t\t\t\t\t     struct dlb_ldb_queue *queue)\n+{\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tstruct dlb_ldb_port *port;\n+\tint slot;\n+\n+\tDLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {\n+\t\tenum dlb_qid_map_state state = DLB_QUEUE_MAPPED;\n+\n+\t\tif (!dlb_port_find_slot_queue(port, state, queue, &slot))\n+\t\t\tcontinue;\n+\n+\t\tif (port->enabled)\n+\t\t\tdlb_ldb_port_cq_disable(hw, port);\n+\t}\n+}\n+\n+static void dlb_ldb_queue_enable_mapped_cqs(struct dlb_hw *hw,\n+\t\t\t\t\t    struct dlb_domain *domain,\n+\t\t\t\t\t    struct dlb_ldb_queue *queue)\n+{\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tstruct dlb_ldb_port *port;\n+\tint slot;\n+\n+\tDLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {\n+\t\tenum dlb_qid_map_state state = DLB_QUEUE_MAPPED;\n+\n+\t\tif (!dlb_port_find_slot_queue(port, state, queue, &slot))\n+\t\t\tcontinue;\n+\n+\t\tif (port->enabled)\n+\t\t\tdlb_ldb_port_cq_enable(hw, port);\n+\t}\n+}\n+\n+static int dlb_ldb_port_map_qid_static(struct dlb_hw *hw,\n+\t\t\t\t       struct dlb_ldb_port *p,\n+\t\t\t\t       struct dlb_ldb_queue *q,\n+\t\t\t\t       u8 priority)\n+{\n+\tunion dlb_lsp_cq2priov r0;\n+\tunion dlb_lsp_cq2qid r1;\n+\tunion dlb_atm_pipe_qid_ldb_qid2cqidx r2;\n+\tunion dlb_lsp_qid_ldb_qid2cqidx r3;\n+\tunion dlb_lsp_qid_ldb_qid2cqidx2 r4;\n+\tenum dlb_qid_map_state state;\n+\tint i;\n+\n+\t/* Look for a pending or already mapped slot, else an unused slot */\n+\tif (!dlb_port_find_slot_queue(p, DLB_QUEUE_MAP_IN_PROGRESS, q, &i) &&\n+\t    !dlb_port_find_slot_queue(p, DLB_QUEUE_MAPPED, q, &i) &&\n+\t    !dlb_port_find_slot(p, DLB_QUEUE_UNMAPPED, &i)) {\n+\t\tDLB_HW_ERR(hw,\n+\t\t\t   \"[%s():%d] Internal error: CQ has no available QID mapping slots\\n\",\n+\t\t\t   __func__, __LINE__);\n+\t\treturn -EFAULT;\n+\t}\n+\n+\tif (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {\n+\t\tDLB_HW_ERR(hw,\n+\t\t\t   \"[%s():%d] Internal error: port slot tracking failed\\n\",\n+\t\t\t   __func__, __LINE__);\n+\t\treturn -EFAULT;\n+\t}\n+\n+\t/* Read-modify-write the priority and valid bit register */\n+\tr0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(p->id));\n+\n+\tr0.field.v |= 1 << i;\n+\tr0.field.prio |= (priority & 0x7) << i * 3;\n+\n+\tDLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(p->id), r0.val);\n+\n+\t/* Read-modify-write the QID map register */\n+\tr1.val = DLB_CSR_RD(hw, DLB_LSP_CQ2QID(p->id, i / 4));\n+\n+\tif (i == 0 || i == 4)\n+\t\tr1.field.qid_p0 = q->id;\n+\tif (i == 1 || i == 5)\n+\t\tr1.field.qid_p1 = q->id;\n+\tif (i == 2 || i == 6)\n+\t\tr1.field.qid_p2 = q->id;\n+\tif (i == 3 || i == 7)\n+\t\tr1.field.qid_p3 = q->id;\n+\n+\tDLB_CSR_WR(hw, DLB_LSP_CQ2QID(p->id, i / 4), r1.val);\n+\n+\tr2.val = DLB_CSR_RD(hw,\n+\t\t\t    DLB_ATM_PIPE_QID_LDB_QID2CQIDX(q->id,\n+\t\t\t\t\t\t\t   p->id / 4));\n+\n+\tr3.val = DLB_CSR_RD(hw,\n+\t\t\t    DLB_LSP_QID_LDB_QID2CQIDX(q->id,\n+\t\t\t\t\t\t      p->id / 4));\n+\n+\tr4.val = DLB_CSR_RD(hw,\n+\t\t\t    DLB_LSP_QID_LDB_QID2CQIDX2(q->id,\n+\t\t\t\t\t\t       p->id / 4));\n+\n+\tswitch (p->id % 4) {\n+\tcase 0:\n+\t\tr2.field.cq_p0 |= 1 << i;\n+\t\tr3.field.cq_p0 |= 1 << i;\n+\t\tr4.field.cq_p0 |= 1 << i;\n+\t\tbreak;\n+\n+\tcase 1:\n+\t\tr2.field.cq_p1 |= 1 << i;\n+\t\tr3.field.cq_p1 |= 1 << i;\n+\t\tr4.field.cq_p1 |= 1 << i;\n+\t\tbreak;\n+\n+\tcase 2:\n+\t\tr2.field.cq_p2 |= 1 << i;\n+\t\tr3.field.cq_p2 |= 1 << i;\n+\t\tr4.field.cq_p2 |= 1 << i;\n+\t\tbreak;\n+\n+\tcase 3:\n+\t\tr2.field.cq_p3 |= 1 << i;\n+\t\tr3.field.cq_p3 |= 1 << i;\n+\t\tr4.field.cq_p3 |= 1 << i;\n+\t\tbreak;\n+\t}\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_ATM_PIPE_QID_LDB_QID2CQIDX(q->id,\n+\t\t\t\t\t\t  p->id / 4),\n+\t\t   r2.val);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_LSP_QID_LDB_QID2CQIDX(q->id,\n+\t\t\t\t\t     p->id / 4),\n+\t\t   r3.val);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_LSP_QID_LDB_QID2CQIDX2(q->id,\n+\t\t\t\t\t      p->id / 4),\n+\t\t   r4.val);\n+\n+\tdlb_flush_csr(hw);\n+\n+\tp->qid_map[i].qid = q->id;\n+\tp->qid_map[i].priority = priority;\n+\n+\tstate = DLB_QUEUE_MAPPED;\n+\n+\treturn dlb_port_slot_state_transition(hw, p, q, i, state);\n+}\n+\n+static int dlb_ldb_port_set_has_work_bits(struct dlb_hw *hw,\n+\t\t\t\t\t  struct dlb_ldb_port *port,\n+\t\t\t\t\t  struct dlb_ldb_queue *queue,\n+\t\t\t\t\t  int slot)\n+{\n+\tunion dlb_lsp_qid_aqed_active_cnt r0;\n+\tunion dlb_lsp_qid_ldb_enqueue_cnt r1;\n+\tunion dlb_lsp_ldb_sched_ctrl r2 = { {0} };\n+\n+\t/* Set the atomic scheduling haswork bit */\n+\tr0.val = DLB_CSR_RD(hw, DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id));\n+\n+\tr2.field.cq = port->id;\n+\tr2.field.qidix = slot;\n+\tr2.field.value = 1;\n+\tr2.field.rlist_haswork_v = r0.field.count > 0;\n+\n+\t/* Set the non-atomic scheduling haswork bit */\n+\tDLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);\n+\n+\tr1.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id));\n+\n+\tmemset(&r2, 0, sizeof(r2));\n+\n+\tr2.field.cq = port->id;\n+\tr2.field.qidix = slot;\n+\tr2.field.value = 1;\n+\tr2.field.nalb_haswork_v = (r1.field.count > 0);\n+\n+\tDLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);\n+\n+\tdlb_flush_csr(hw);\n+\n+\treturn 0;\n+}\n+\n+static void dlb_ldb_port_clear_queue_if_status(struct dlb_hw *hw,\n+\t\t\t\t\t       struct dlb_ldb_port *port,\n+\t\t\t\t\t       int slot)\n+{\n+\tunion dlb_lsp_ldb_sched_ctrl r0 = { {0} };\n+\n+\tr0.field.cq = port->id;\n+\tr0.field.qidix = slot;\n+\tr0.field.value = 0;\n+\tr0.field.inflight_ok_v = 1;\n+\n+\tDLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r0.val);\n+\n+\tdlb_flush_csr(hw);\n+}\n+\n+static void dlb_ldb_port_set_queue_if_status(struct dlb_hw *hw,\n+\t\t\t\t\t     struct dlb_ldb_port *port,\n+\t\t\t\t\t     int slot)\n+{\n+\tunion dlb_lsp_ldb_sched_ctrl r0 = { {0} };\n+\n+\tr0.field.cq = port->id;\n+\tr0.field.qidix = slot;\n+\tr0.field.value = 1;\n+\tr0.field.inflight_ok_v = 1;\n+\n+\tDLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r0.val);\n+\n+\tdlb_flush_csr(hw);\n+}\n+\n+static void dlb_ldb_queue_set_inflight_limit(struct dlb_hw *hw,\n+\t\t\t\t\t     struct dlb_ldb_queue *queue)\n+{\n+\tunion dlb_lsp_qid_ldb_infl_lim r0 = { {0} };\n+\n+\tr0.field.limit = queue->num_qid_inflights;\n+\n+\tDLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id), r0.val);\n+}\n+\n+static void dlb_ldb_queue_clear_inflight_limit(struct dlb_hw *hw,\n+\t\t\t\t\t       struct dlb_ldb_queue *queue)\n+{\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_LSP_QID_LDB_INFL_LIM(queue->id),\n+\t\t   DLB_LSP_QID_LDB_INFL_LIM_RST);\n+}\n+\n+static int dlb_ldb_port_finish_map_qid_dynamic(struct dlb_hw *hw,\n+\t\t\t\t\t       struct dlb_domain *domain,\n+\t\t\t\t\t       struct dlb_ldb_port *port,\n+\t\t\t\t\t       struct dlb_ldb_queue *queue)\n+{\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tunion dlb_lsp_qid_ldb_infl_cnt r0;\n+\tenum dlb_qid_map_state state;\n+\tint slot, ret;\n+\tu8 prio;\n+\n+\tr0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id));\n+\n+\tif (r0.field.count) {\n+\t\tDLB_HW_ERR(hw,\n+\t\t\t   \"[%s()] Internal error: non-zero QID inflight count\\n\",\n+\t\t\t   __func__);\n+\t\treturn -EFAULT;\n+\t}\n+\n+\t/* For each port with a pending mapping to this queue, perform the\n+\t * static mapping and set the corresponding has_work bits.\n+\t */\n+\tstate = DLB_QUEUE_MAP_IN_PROGRESS;\n+\tif (!dlb_port_find_slot_queue(port, state, queue, &slot))\n+\t\treturn -EINVAL;\n+\n+\tif (slot >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {\n+\t\tDLB_HW_ERR(hw,\n+\t\t\t   \"[%s():%d] Internal error: port slot tracking failed\\n\",\n+\t\t\t   __func__, __LINE__);\n+\t\treturn -EFAULT;\n+\t}\n+\n+\tprio = port->qid_map[slot].priority;\n+\n+\t/* Update the CQ2QID, CQ2PRIOV, and QID2CQIDX registers, and\n+\t * the port's qid_map state.\n+\t */\n+\tret = dlb_ldb_port_map_qid_static(hw, port, queue, prio);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\tret = dlb_ldb_port_set_has_work_bits(hw, port, queue, slot);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\t/* Ensure IF_status(cq,qid) is 0 before enabling the port to\n+\t * prevent spurious schedules to cause the queue's inflight\n+\t * count to increase.\n+\t */\n+\tdlb_ldb_port_clear_queue_if_status(hw, port, slot);\n+\n+\t/* Reset the queue's inflight status */\n+\tDLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {\n+\t\tstate = DLB_QUEUE_MAPPED;\n+\t\tif (!dlb_port_find_slot_queue(port, state, queue, &slot))\n+\t\t\tcontinue;\n+\n+\t\tdlb_ldb_port_set_queue_if_status(hw, port, slot);\n+\t}\n+\n+\tdlb_ldb_queue_set_inflight_limit(hw, queue);\n+\n+\t/* Re-enable CQs mapped to this queue */\n+\tdlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);\n+\n+\t/* If this queue has other mappings pending, clear its inflight limit */\n+\tif (queue->num_pending_additions > 0)\n+\t\tdlb_ldb_queue_clear_inflight_limit(hw, queue);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * dlb_ldb_port_map_qid_dynamic() - perform a \"dynamic\" QID->CQ mapping\n+ * @hw: dlb_hw handle for a particular device.\n+ * @port: load-balanced port\n+ * @queue: load-balanced queue\n+ * @priority: queue servicing priority\n+ *\n+ * Returns 0 if the queue was mapped, 1 if the mapping is scheduled to occur\n+ * at a later point, and <0 if an error occurred.\n+ */\n+static int dlb_ldb_port_map_qid_dynamic(struct dlb_hw *hw,\n+\t\t\t\t\tstruct dlb_ldb_port *port,\n+\t\t\t\t\tstruct dlb_ldb_queue *queue,\n+\t\t\t\t\tu8 priority)\n+{\n+\tunion dlb_lsp_qid_ldb_infl_cnt r0 = { {0} };\n+\tenum dlb_qid_map_state state;\n+\tstruct dlb_domain *domain;\n+\tint slot, ret;\n+\n+\tdomain = dlb_get_domain_from_id(hw, port->domain_id);\n+\tif (!domain) {\n+\t\tDLB_HW_ERR(hw,\n+\t\t\t   \"[%s()] Internal error: unable to find domain %d\\n\",\n+\t\t\t   __func__, port->domain_id);\n+\t\treturn -EFAULT;\n+\t}\n+\n+\t/* Set the QID inflight limit to 0 to prevent further scheduling of the\n+\t * queue.\n+\t */\n+\tDLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id), 0);\n+\n+\tif (!dlb_port_find_slot(port, DLB_QUEUE_UNMAPPED, &slot)) {\n+\t\tDLB_HW_ERR(hw,\n+\t\t\t   \"Internal error: No available unmapped slots\\n\");\n+\t\treturn -EFAULT;\n+\t}\n+\n+\tif (slot >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {\n+\t\tDLB_HW_ERR(hw,\n+\t\t\t   \"[%s():%d] Internal error: port slot tracking failed\\n\",\n+\t\t\t   __func__, __LINE__);\n+\t\treturn -EFAULT;\n+\t}\n+\n+\tport->qid_map[slot].qid = queue->id;\n+\tport->qid_map[slot].priority = priority;\n+\n+\tstate = DLB_QUEUE_MAP_IN_PROGRESS;\n+\tret = dlb_port_slot_state_transition(hw, port, queue, slot, state);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\tr0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id));\n+\n+\tif (r0.field.count) {\n+\t\t/* The queue is owed completions so it's not safe to map it\n+\t\t * yet. Schedule a kernel thread to complete the mapping later,\n+\t\t * once software has completed all the queue's inflight events.\n+\t\t */\n+\t\tif (!os_worker_active(hw))\n+\t\t\tos_schedule_work(hw);\n+\n+\t\treturn 1;\n+\t}\n+\n+\t/* Disable the affected CQ, and the CQs already mapped to the QID,\n+\t * before reading the QID's inflight count a second time. There is an\n+\t * unlikely race in which the QID may schedule one more QE after we\n+\t * read an inflight count of 0, and disabling the CQs guarantees that\n+\t * the race will not occur after a re-read of the inflight count\n+\t * register.\n+\t */\n+\tif (port->enabled)\n+\t\tdlb_ldb_port_cq_disable(hw, port);\n+\n+\tdlb_ldb_queue_disable_mapped_cqs(hw, domain, queue);\n+\n+\tr0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id));\n+\n+\tif (r0.field.count) {\n+\t\tif (port->enabled)\n+\t\t\tdlb_ldb_port_cq_enable(hw, port);\n+\n+\t\tdlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);\n+\n+\t\t/* The queue is owed completions so it's not safe to map it\n+\t\t * yet. Schedule a kernel thread to complete the mapping later,\n+\t\t * once software has completed all the queue's inflight events.\n+\t\t */\n+\t\tif (!os_worker_active(hw))\n+\t\t\tos_schedule_work(hw);\n+\n+\t\treturn 1;\n+\t}\n+\n+\treturn dlb_ldb_port_finish_map_qid_dynamic(hw, domain, port, queue);\n+}\n+\n+\n+static int dlb_ldb_port_map_qid(struct dlb_hw *hw,\n+\t\t\t\tstruct dlb_domain *domain,\n+\t\t\t\tstruct dlb_ldb_port *port,\n+\t\t\t\tstruct dlb_ldb_queue *queue,\n+\t\t\t\tu8 prio)\n+{\n+\tif (domain->started)\n+\t\treturn dlb_ldb_port_map_qid_dynamic(hw, port, queue, prio);\n+\telse\n+\t\treturn dlb_ldb_port_map_qid_static(hw, port, queue, prio);\n+}\n+\n+static int dlb_ldb_port_unmap_qid(struct dlb_hw *hw,\n+\t\t\t\t  struct dlb_ldb_port *port,\n+\t\t\t\t  struct dlb_ldb_queue *queue)\n+{\n+\tenum dlb_qid_map_state mapped, in_progress, pending_map, unmapped;\n+\tunion dlb_lsp_cq2priov r0;\n+\tunion dlb_atm_pipe_qid_ldb_qid2cqidx r1;\n+\tunion dlb_lsp_qid_ldb_qid2cqidx r2;\n+\tunion dlb_lsp_qid_ldb_qid2cqidx2 r3;\n+\tu32 queue_id;\n+\tu32 port_id;\n+\tint i;\n+\n+\t/* Find the queue's slot */\n+\tmapped = DLB_QUEUE_MAPPED;\n+\tin_progress = DLB_QUEUE_UNMAP_IN_PROGRESS;\n+\tpending_map = DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP;\n+\n+\tif (!dlb_port_find_slot_queue(port, mapped, queue, &i) &&\n+\t    !dlb_port_find_slot_queue(port, in_progress, queue, &i) &&\n+\t    !dlb_port_find_slot_queue(port, pending_map, queue, &i)) {\n+\t\tDLB_HW_ERR(hw,\n+\t\t\t   \"[%s():%d] Internal error: QID %d isn't mapped\\n\",\n+\t\t\t   __func__, __LINE__, queue->id);\n+\t\treturn -EFAULT;\n+\t}\n+\n+\tif (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {\n+\t\tDLB_HW_ERR(hw,\n+\t\t\t   \"[%s():%d] Internal error: port slot tracking failed\\n\",\n+\t\t\t   __func__, __LINE__);\n+\t\treturn -EFAULT;\n+\t}\n+\n+\tport_id = port->id;\n+\tqueue_id = queue->id;\n+\n+\t/* Read-modify-write the priority and valid bit register */\n+\tr0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(port_id));\n+\n+\tr0.field.v &= ~(1 << i);\n+\n+\tDLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port_id), r0.val);\n+\n+\tr1.val = DLB_CSR_RD(hw,\n+\t\t\t    DLB_ATM_PIPE_QID_LDB_QID2CQIDX(queue_id,\n+\t\t\t\t\t\t\t   port_id / 4));\n+\n+\tr2.val = DLB_CSR_RD(hw,\n+\t\t\t    DLB_LSP_QID_LDB_QID2CQIDX(queue_id,\n+\t\t\t\t\t\t      port_id / 4));\n+\n+\tr3.val = DLB_CSR_RD(hw,\n+\t\t\t    DLB_LSP_QID_LDB_QID2CQIDX2(queue_id,\n+\t\t\t\t\t\t       port_id / 4));\n+\n+\tswitch (port_id % 4) {\n+\tcase 0:\n+\t\tr1.field.cq_p0 &= ~(1 << i);\n+\t\tr2.field.cq_p0 &= ~(1 << i);\n+\t\tr3.field.cq_p0 &= ~(1 << i);\n+\t\tbreak;\n+\n+\tcase 1:\n+\t\tr1.field.cq_p1 &= ~(1 << i);\n+\t\tr2.field.cq_p1 &= ~(1 << i);\n+\t\tr3.field.cq_p1 &= ~(1 << i);\n+\t\tbreak;\n+\n+\tcase 2:\n+\t\tr1.field.cq_p2 &= ~(1 << i);\n+\t\tr2.field.cq_p2 &= ~(1 << i);\n+\t\tr3.field.cq_p2 &= ~(1 << i);\n+\t\tbreak;\n+\n+\tcase 3:\n+\t\tr1.field.cq_p3 &= ~(1 << i);\n+\t\tr2.field.cq_p3 &= ~(1 << i);\n+\t\tr3.field.cq_p3 &= ~(1 << i);\n+\t\tbreak;\n+\t}\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_ATM_PIPE_QID_LDB_QID2CQIDX(queue_id, port_id / 4),\n+\t\t   r1.val);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_LSP_QID_LDB_QID2CQIDX(queue_id, port_id / 4),\n+\t\t   r2.val);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_LSP_QID_LDB_QID2CQIDX2(queue_id, port_id / 4),\n+\t\t   r3.val);\n+\n+\tdlb_flush_csr(hw);\n+\n+\tunmapped = DLB_QUEUE_UNMAPPED;\n+\n+\treturn dlb_port_slot_state_transition(hw, port, queue, i, unmapped);\n+}\n+\n+static int\n+dlb_verify_create_sched_domain_args(struct dlb_hw *hw,\n+\t\t\t\t    struct dlb_function_resources *rsrcs,\n+\t\t\t\t    struct dlb_create_sched_domain_args *args,\n+\t\t\t\t    struct dlb_cmd_response *resp)\n+{\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tstruct dlb_bitmap *ldb_credit_freelist;\n+\tstruct dlb_bitmap *dir_credit_freelist;\n+\tunsigned int ldb_credit_freelist_count;\n+\tunsigned int dir_credit_freelist_count;\n+\tunsigned int max_contig_aqed_entries;\n+\tunsigned int max_contig_dqed_entries;\n+\tunsigned int max_contig_qed_entries;\n+\tunsigned int max_contig_hl_entries;\n+\tstruct dlb_bitmap *aqed_freelist;\n+\tenum dlb_dev_revision revision;\n+\n+\tldb_credit_freelist = rsrcs->avail_qed_freelist_entries;\n+\tdir_credit_freelist = rsrcs->avail_dqed_freelist_entries;\n+\taqed_freelist = rsrcs->avail_aqed_freelist_entries;\n+\n+\tldb_credit_freelist_count = dlb_bitmap_count(ldb_credit_freelist);\n+\tdir_credit_freelist_count = dlb_bitmap_count(dir_credit_freelist);\n+\n+\tmax_contig_hl_entries =\n+\t\tdlb_bitmap_longest_set_range(rsrcs->avail_hist_list_entries);\n+\tmax_contig_aqed_entries =\n+\t\tdlb_bitmap_longest_set_range(aqed_freelist);\n+\tmax_contig_qed_entries =\n+\t\tdlb_bitmap_longest_set_range(ldb_credit_freelist);\n+\tmax_contig_dqed_entries =\n+\t\tdlb_bitmap_longest_set_range(dir_credit_freelist);\n+\n+\tif (rsrcs->num_avail_domains < 1)\n+\t\tresp->status = DLB_ST_DOMAIN_UNAVAILABLE;\n+\telse if (rsrcs->num_avail_ldb_queues < args->num_ldb_queues)\n+\t\tresp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;\n+\telse if (rsrcs->num_avail_ldb_ports < args->num_ldb_ports)\n+\t\tresp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;\n+\telse if (args->num_ldb_queues > 0 && args->num_ldb_ports == 0)\n+\t\tresp->status = DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES;\n+\telse if (rsrcs->num_avail_dir_pq_pairs < args->num_dir_ports)\n+\t\tresp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;\n+\telse if (ldb_credit_freelist_count < args->num_ldb_credits)\n+\t\tresp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;\n+\telse if (dir_credit_freelist_count < args->num_dir_credits)\n+\t\tresp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;\n+\telse if (rsrcs->num_avail_ldb_credit_pools < args->num_ldb_credit_pools)\n+\t\tresp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;\n+\telse if (rsrcs->num_avail_dir_credit_pools < args->num_dir_credit_pools)\n+\t\tresp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;\n+\telse if (max_contig_hl_entries < args->num_hist_list_entries)\n+\t\tresp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;\n+\telse if (max_contig_aqed_entries < args->num_atomic_inflights)\n+\t\tresp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;\n+\telse if (max_contig_qed_entries < args->num_ldb_credits)\n+\t\tresp->status = DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE;\n+\telse if (max_contig_dqed_entries < args->num_dir_credits)\n+\t\tresp->status = DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE;\n+\n+\t/* DLB A-stepping workaround for hardware write buffer lock up issue:\n+\t * limit the maximum configured ports to less than 128 and disable CQ\n+\t * occupancy interrupts.\n+\t */\n+\trevision = os_get_dev_revision(hw);\n+\n+\tif (revision < DLB_B0) {\n+\t\tu32 n = dlb_get_num_ports_in_use(hw);\n+\n+\t\tn += args->num_ldb_ports + args->num_dir_ports;\n+\n+\t\tif (n >= DLB_A_STEP_MAX_PORTS)\n+\t\t\tresp->status = args->num_ldb_ports ?\n+\t\t\t\tDLB_ST_LDB_PORTS_UNAVAILABLE :\n+\t\t\t\tDLB_ST_DIR_PORTS_UNAVAILABLE;\n+\t}\n+\n+\tif (resp->status)\n+\t\treturn -1;\n+\n+\treturn 0;\n+}\n+\n+\n+static void\n+dlb_log_create_sched_domain_args(struct dlb_hw *hw,\n+\t\t\t\t struct dlb_create_sched_domain_args *args)\n+{\n+\tDLB_HW_INFO(hw, \"DLB create sched domain arguments:\\n\");\n+\tDLB_HW_INFO(hw, \"\\tNumber of LDB queues:        %d\\n\",\n+\t\t    args->num_ldb_queues);\n+\tDLB_HW_INFO(hw, \"\\tNumber of LDB ports:         %d\\n\",\n+\t\t    args->num_ldb_ports);\n+\tDLB_HW_INFO(hw, \"\\tNumber of DIR ports:         %d\\n\",\n+\t\t    args->num_dir_ports);\n+\tDLB_HW_INFO(hw, \"\\tNumber of ATM inflights:     %d\\n\",\n+\t\t    args->num_atomic_inflights);\n+\tDLB_HW_INFO(hw, \"\\tNumber of hist list entries: %d\\n\",\n+\t\t    args->num_hist_list_entries);\n+\tDLB_HW_INFO(hw, \"\\tNumber of LDB credits:       %d\\n\",\n+\t\t    args->num_ldb_credits);\n+\tDLB_HW_INFO(hw, \"\\tNumber of DIR credits:       %d\\n\",\n+\t\t    args->num_dir_credits);\n+\tDLB_HW_INFO(hw, \"\\tNumber of LDB credit pools:  %d\\n\",\n+\t\t    args->num_ldb_credit_pools);\n+\tDLB_HW_INFO(hw, \"\\tNumber of DIR credit pools:  %d\\n\",\n+\t\t    args->num_dir_credit_pools);\n+}\n+\n+/**\n+ * dlb_hw_create_sched_domain() - Allocate and initialize a DLB scheduling\n+ *\tdomain and its resources.\n+ * @hw:\t  Contains the current state of the DLB hardware.\n+ * @args: User-provided arguments.\n+ * @resp: Response to user.\n+ *\n+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to\n+ * satisfy a request, resp->status will be set accordingly.\n+ */\n+int dlb_hw_create_sched_domain(struct dlb_hw *hw,\n+\t\t\t       struct dlb_create_sched_domain_args *args,\n+\t\t\t       struct dlb_cmd_response *resp)\n+{\n+\tstruct dlb_domain *domain;\n+\tstruct dlb_function_resources *rsrcs;\n+\tint ret;\n+\n+\trsrcs = &hw->pf;\n+\n+\tdlb_log_create_sched_domain_args(hw, args);\n+\n+\t/* Verify that hardware resources are available before attempting to\n+\t * satisfy the request. This simplifies the error unwinding code.\n+\t */\n+\tif (dlb_verify_create_sched_domain_args(hw, rsrcs, args, resp))\n+\t\treturn -EINVAL;\n+\n+\tdomain = DLB_FUNC_LIST_HEAD(rsrcs->avail_domains, typeof(*domain));\n+\n+\t/* Verification should catch this. */\n+\tif (!domain) {\n+\t\tDLB_HW_ERR(hw,\n+\t\t\t   \"[%s():%d] Internal error: no available domains\\n\",\n+\t\t\t   __func__, __LINE__);\n+\t\treturn -EFAULT;\n+\t}\n+\n+\tif (domain->configured) {\n+\t\tDLB_HW_ERR(hw,\n+\t\t\t   \"[%s()] Internal error: avail_domains contains configured domains.\\n\",\n+\t\t\t   __func__);\n+\t\treturn -EFAULT;\n+\t}\n+\n+\tdlb_init_domain_rsrc_lists(domain);\n+\n+\t/* Verification should catch this too. */\n+\tret = dlb_domain_attach_resources(hw, rsrcs, domain, args, resp);\n+\tif (ret < 0) {\n+\t\tDLB_HW_ERR(hw,\n+\t\t\t   \"[%s()] Internal error: failed to verify args.\\n\",\n+\t\t\t   __func__);\n+\n+\t\treturn -EFAULT;\n+\t}\n+\n+\tdlb_list_del(&rsrcs->avail_domains, &domain->func_list);\n+\n+\tdlb_list_add(&rsrcs->used_domains, &domain->func_list);\n+\n+\tresp->id = domain->id;\n+\tresp->status = 0;\n+\n+\treturn 0;\n+}\n+\n+static void\n+dlb_configure_ldb_credit_pool(struct dlb_hw *hw,\n+\t\t\t      struct dlb_domain *domain,\n+\t\t\t      struct dlb_create_ldb_pool_args *args,\n+\t\t\t      struct dlb_credit_pool *pool)\n+{\n+\tunion dlb_sys_ldb_pool_enbld r0 = { {0} };\n+\tunion dlb_chp_ldb_pool_crd_lim r1 = { {0} };\n+\tunion dlb_chp_ldb_pool_crd_cnt r2 = { {0} };\n+\tunion dlb_chp_qed_fl_base  r3 = { {0} };\n+\tunion dlb_chp_qed_fl_lim r4 = { {0} };\n+\tunion dlb_chp_qed_fl_push_ptr r5 = { {0} };\n+\tunion dlb_chp_qed_fl_pop_ptr  r6 = { {0} };\n+\n+\tr1.field.limit = args->num_ldb_credits;\n+\n+\tDLB_CSR_WR(hw, DLB_CHP_LDB_POOL_CRD_LIM(pool->id), r1.val);\n+\n+\tr2.field.count = args->num_ldb_credits;\n+\n+\tDLB_CSR_WR(hw, DLB_CHP_LDB_POOL_CRD_CNT(pool->id), r2.val);\n+\n+\tr3.field.base = domain->qed_freelist.base + domain->qed_freelist.offset;\n+\n+\tDLB_CSR_WR(hw, DLB_CHP_QED_FL_BASE(pool->id), r3.val);\n+\n+\tr4.field.freelist_disable = 0;\n+\tr4.field.limit = r3.field.base + args->num_ldb_credits - 1;\n+\n+\tDLB_CSR_WR(hw, DLB_CHP_QED_FL_LIM(pool->id), r4.val);\n+\n+\tr5.field.push_ptr = r3.field.base;\n+\tr5.field.generation = 1;\n+\n+\tDLB_CSR_WR(hw, DLB_CHP_QED_FL_PUSH_PTR(pool->id), r5.val);\n+\n+\tr6.field.pop_ptr = r3.field.base;\n+\tr6.field.generation = 0;\n+\n+\tDLB_CSR_WR(hw, DLB_CHP_QED_FL_POP_PTR(pool->id), r6.val);\n+\n+\tr0.field.pool_enabled = 1;\n+\n+\tDLB_CSR_WR(hw, DLB_SYS_LDB_POOL_ENBLD(pool->id), r0.val);\n+\n+\tpool->avail_credits = args->num_ldb_credits;\n+\tpool->total_credits = args->num_ldb_credits;\n+\tdomain->qed_freelist.offset += args->num_ldb_credits;\n+\n+\tpool->configured = true;\n+}\n+\n+static int\n+dlb_verify_create_ldb_pool_args(struct dlb_hw *hw,\n+\t\t\t\tu32 domain_id,\n+\t\t\t\tstruct dlb_create_ldb_pool_args *args,\n+\t\t\t\tstruct dlb_cmd_response *resp)\n+{\n+\tstruct dlb_freelist *qed_freelist;\n+\tstruct dlb_domain *domain;\n+\n+\tdomain = dlb_get_domain_from_id(hw, domain_id);\n+\n+\tif (!domain) {\n+\t\tresp->status = DLB_ST_INVALID_DOMAIN_ID;\n+\t\treturn -1;\n+\t}\n+\n+\tif (!domain->configured) {\n+\t\tresp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;\n+\t\treturn -1;\n+\t}\n+\n+\tqed_freelist = &domain->qed_freelist;\n+\n+\tif (dlb_freelist_count(qed_freelist) < args->num_ldb_credits) {\n+\t\tresp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;\n+\t\treturn -1;\n+\t}\n+\n+\tif (dlb_list_empty(&domain->avail_ldb_credit_pools)) {\n+\t\tresp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;\n+\t\treturn -1;\n+\t}\n+\n+\tif (domain->started) {\n+\t\tresp->status = DLB_ST_DOMAIN_STARTED;\n+\t\treturn -1;\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static void\n+dlb_log_create_ldb_pool_args(struct dlb_hw *hw,\n+\t\t\t     u32 domain_id,\n+\t\t\t     struct dlb_create_ldb_pool_args *args)\n+{\n+\tDLB_HW_INFO(hw, \"DLB create load-balanced credit pool arguments:\\n\");\n+\tDLB_HW_INFO(hw, \"\\tDomain ID:             %d\\n\", domain_id);\n+\tDLB_HW_INFO(hw, \"\\tNumber of LDB credits: %d\\n\",\n+\t\t    args->num_ldb_credits);\n+}\n+\n+/**\n+ * dlb_hw_create_ldb_pool() - Allocate and initialize a DLB credit pool.\n+ * @hw:\t  Contains the current state of the DLB hardware.\n+ * @args: User-provided arguments.\n+ * @resp: Response to user.\n+ *\n+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to\n+ * satisfy a request, resp->status will be set accordingly.\n+ */\n+int dlb_hw_create_ldb_pool(struct dlb_hw *hw,\n+\t\t\t   u32 domain_id,\n+\t\t\t   struct dlb_create_ldb_pool_args *args,\n+\t\t\t   struct dlb_cmd_response *resp)\n+{\n+\tstruct dlb_credit_pool *pool;\n+\tstruct dlb_domain *domain;\n+\n+\tdlb_log_create_ldb_pool_args(hw, domain_id, args);\n+\n+\t/* Verify that hardware resources are available before attempting to\n+\t * satisfy the request. This simplifies the error unwinding code.\n+\t */\n+\tif (dlb_verify_create_ldb_pool_args(hw, domain_id, args, resp))\n+\t\treturn -EINVAL;\n+\n+\tdomain = dlb_get_domain_from_id(hw, domain_id);\n+\tif (!domain) {\n+\t\tDLB_HW_ERR(hw,\n+\t\t\t   \"[%s():%d] Internal error: domain not found\\n\",\n+\t\t\t   __func__, __LINE__);\n+\t\treturn -EFAULT;\n+\t}\n+\n+\tpool = DLB_DOM_LIST_HEAD(domain->avail_ldb_credit_pools, typeof(*pool));\n+\n+\t/* Verification should catch this. */\n+\tif (!pool) {\n+\t\tDLB_HW_ERR(hw,\n+\t\t\t   \"[%s():%d] Internal error: no available ldb credit pools\\n\",\n+\t\t\t   __func__, __LINE__);\n+\t\treturn -EFAULT;\n+\t}\n+\n+\tdlb_configure_ldb_credit_pool(hw, domain, args, pool);\n+\n+\t/* Configuration succeeded, so move the resource from the 'avail' to\n+\t * the 'used' list.\n+\t */\n+\tdlb_list_del(&domain->avail_ldb_credit_pools, &pool->domain_list);\n+\n+\tdlb_list_add(&domain->used_ldb_credit_pools, &pool->domain_list);\n+\n+\tresp->status = 0;\n+\tresp->id = pool->id;\n+\n+\treturn 0;\n+}\n+\n+static void\n+dlb_configure_dir_credit_pool(struct dlb_hw *hw,\n+\t\t\t      struct dlb_domain *domain,\n+\t\t\t      struct dlb_create_dir_pool_args *args,\n+\t\t\t      struct dlb_credit_pool *pool)\n+{\n+\tunion dlb_sys_dir_pool_enbld r0 = { {0} };\n+\tunion dlb_chp_dir_pool_crd_lim r1 = { {0} };\n+\tunion dlb_chp_dir_pool_crd_cnt r2 = { {0} };\n+\tunion dlb_chp_dqed_fl_base  r3 = { {0} };\n+\tunion dlb_chp_dqed_fl_lim r4 = { {0} };\n+\tunion dlb_chp_dqed_fl_push_ptr r5 = { {0} };\n+\tunion dlb_chp_dqed_fl_pop_ptr  r6 = { {0} };\n+\n+\tr1.field.limit = args->num_dir_credits;\n+\n+\tDLB_CSR_WR(hw, DLB_CHP_DIR_POOL_CRD_LIM(pool->id), r1.val);\n+\n+\tr2.field.count = args->num_dir_credits;\n+\n+\tDLB_CSR_WR(hw, DLB_CHP_DIR_POOL_CRD_CNT(pool->id), r2.val);\n+\n+\tr3.field.base = domain->dqed_freelist.base +\n+\t\t\tdomain->dqed_freelist.offset;\n+\n+\tDLB_CSR_WR(hw, DLB_CHP_DQED_FL_BASE(pool->id), r3.val);\n+\n+\tr4.field.freelist_disable = 0;\n+\tr4.field.limit = r3.field.base + args->num_dir_credits - 1;\n+\n+\tDLB_CSR_WR(hw, DLB_CHP_DQED_FL_LIM(pool->id), r4.val);\n+\n+\tr5.field.push_ptr = r3.field.base;\n+\tr5.field.generation = 1;\n+\n+\tDLB_CSR_WR(hw, DLB_CHP_DQED_FL_PUSH_PTR(pool->id), r5.val);\n+\n+\tr6.field.pop_ptr = r3.field.base;\n+\tr6.field.generation = 0;\n+\n+\tDLB_CSR_WR(hw, DLB_CHP_DQED_FL_POP_PTR(pool->id), r6.val);\n+\n+\tr0.field.pool_enabled = 1;\n+\n+\tDLB_CSR_WR(hw, DLB_SYS_DIR_POOL_ENBLD(pool->id), r0.val);\n+\n+\tpool->avail_credits = args->num_dir_credits;\n+\tpool->total_credits = args->num_dir_credits;\n+\tdomain->dqed_freelist.offset += args->num_dir_credits;\n+\n+\tpool->configured = true;\n+}\n+\n+static int\n+dlb_verify_create_dir_pool_args(struct dlb_hw *hw,\n+\t\t\t\tu32 domain_id,\n+\t\t\t\tstruct dlb_create_dir_pool_args *args,\n+\t\t\t\tstruct dlb_cmd_response *resp)\n+{\n+\tstruct dlb_freelist *dqed_freelist;\n+\tstruct dlb_domain *domain;\n+\n+\tdomain = dlb_get_domain_from_id(hw, domain_id);\n+\n+\tif (!domain) {\n+\t\tresp->status = DLB_ST_INVALID_DOMAIN_ID;\n+\t\treturn -1;\n+\t}\n+\n+\tif (!domain->configured) {\n+\t\tresp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;\n+\t\treturn -1;\n+\t}\n+\n+\tdqed_freelist = &domain->dqed_freelist;\n+\n+\tif (dlb_freelist_count(dqed_freelist) < args->num_dir_credits) {\n+\t\tresp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;\n+\t\treturn -1;\n+\t}\n+\n+\tif (dlb_list_empty(&domain->avail_dir_credit_pools)) {\n+\t\tresp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;\n+\t\treturn -1;\n+\t}\n+\n+\tif (domain->started) {\n+\t\tresp->status = DLB_ST_DOMAIN_STARTED;\n+\t\treturn -1;\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static void\n+dlb_log_create_dir_pool_args(struct dlb_hw *hw,\n+\t\t\t     u32 domain_id,\n+\t\t\t     struct dlb_create_dir_pool_args *args)\n+{\n+\tDLB_HW_INFO(hw, \"DLB create directed credit pool arguments:\\n\");\n+\tDLB_HW_INFO(hw, \"\\tDomain ID:             %d\\n\", domain_id);\n+\tDLB_HW_INFO(hw, \"\\tNumber of DIR credits: %d\\n\",\n+\t\t    args->num_dir_credits);\n+}\n+\n+/**\n+ * dlb_hw_create_dir_pool() - Allocate and initialize a DLB credit pool.\n+ * @hw:\t  Contains the current state of the DLB hardware.\n+ * @args: User-provided arguments.\n+ * @resp: Response to user.\n+ *\n+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to\n+ * satisfy a request, resp->status will be set accordingly.\n+ */\n+int dlb_hw_create_dir_pool(struct dlb_hw *hw,\n+\t\t\t   u32 domain_id,\n+\t\t\t   struct dlb_create_dir_pool_args *args,\n+\t\t\t   struct dlb_cmd_response *resp)\n+{\n+\tstruct dlb_credit_pool *pool;\n+\tstruct dlb_domain *domain;\n+\n+\tdlb_log_create_dir_pool_args(hw, domain_id, args);\n+\n+\t/* Verify that hardware resources are available before attempting to\n+\t * satisfy the request. This simplifies the error unwinding code.\n+\t */\n+\t/* At least one available pool */\n+\tif (dlb_verify_create_dir_pool_args(hw, domain_id, args, resp))\n+\t\treturn -EINVAL;\n+\n+\tdomain = dlb_get_domain_from_id(hw, domain_id);\n+\tif (!domain) {\n+\t\tDLB_HW_ERR(hw,\n+\t\t\t   \"[%s():%d] Internal error: domain not found\\n\",\n+\t\t\t   __func__, __LINE__);\n+\t\treturn -EFAULT;\n+\t}\n+\n+\tpool = DLB_DOM_LIST_HEAD(domain->avail_dir_credit_pools, typeof(*pool));\n+\n+\t/* Verification should catch this. */\n+\tif (!pool) {\n+\t\tDLB_HW_ERR(hw,\n+\t\t\t   \"[%s():%d] Internal error: no available dir credit pools\\n\",\n+\t\t\t   __func__, __LINE__);\n+\t\treturn -EFAULT;\n+\t}\n+\n+\tdlb_configure_dir_credit_pool(hw, domain, args, pool);\n+\n+\t/* Configuration succeeded, so move the resource from the 'avail' to\n+\t * the 'used' list.\n+\t */\n+\tdlb_list_del(&domain->avail_dir_credit_pools, &pool->domain_list);\n+\n+\tdlb_list_add(&domain->used_dir_credit_pools, &pool->domain_list);\n+\n+\tresp->status = 0;\n+\tresp->id = pool->id;\n+\n+\treturn 0;\n+}\n+\n+static u32 dlb_ldb_cq_inflight_count(struct dlb_hw *hw,\n+\t\t\t\t     struct dlb_ldb_port *port)\n+{\n+\tunion dlb_lsp_cq_ldb_infl_cnt r0;\n+\n+\tr0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_INFL_CNT(port->id));\n+\n+\treturn r0.field.count;\n+}\n+\n+static u32 dlb_ldb_cq_token_count(struct dlb_hw *hw,\n+\t\t\t\t  struct dlb_ldb_port *port)\n+{\n+\tunion dlb_lsp_cq_ldb_tkn_cnt r0;\n+\n+\tr0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_TKN_CNT(port->id));\n+\n+\treturn r0.field.token_count;\n+}\n+\n+static int dlb_drain_ldb_cq(struct dlb_hw *hw, struct dlb_ldb_port *port)\n+{\n+\tu32 infl_cnt, tkn_cnt;\n+\tunsigned int i;\n+\n+\tinfl_cnt = dlb_ldb_cq_inflight_count(hw, port);\n+\n+\t/* Account for the initial token count, which is used in order to\n+\t * provide a CQ with depth less than 8.\n+\t */\n+\ttkn_cnt = dlb_ldb_cq_token_count(hw, port) - port->init_tkn_cnt;\n+\n+\tif (infl_cnt || tkn_cnt) {\n+\t\tstruct dlb_hcw hcw_mem[8], *hcw;\n+\t\tvoid  *pp_addr;\n+\n+\t\tpp_addr = os_map_producer_port(hw, port->id, true);\n+\n+\t\t/* Point hcw to a 64B-aligned location */\n+\t\thcw = (struct dlb_hcw *)((uintptr_t)&hcw_mem[4] & ~0x3F);\n+\n+\t\t/* Program the first HCW for a completion and token return and\n+\t\t * the other HCWs as NOOPS\n+\t\t */\n+\n+\t\tmemset(hcw, 0, 4 * sizeof(*hcw));\n+\t\thcw->qe_comp = (infl_cnt > 0);\n+\t\thcw->cq_token = (tkn_cnt > 0);\n+\t\thcw->lock_id = tkn_cnt - 1;\n+\n+\t\t/* Return tokens in the first HCW */\n+\t\tos_enqueue_four_hcws(hw, hcw, pp_addr);\n+\n+\t\thcw->cq_token = 0;\n+\n+\t\t/* Issue remaining completions (if any) */\n+\t\tfor (i = 1; i < infl_cnt; i++)\n+\t\t\tos_enqueue_four_hcws(hw, hcw, pp_addr);\n+\n+\t\tos_fence_hcw(hw, pp_addr);\n+\n+\t\tos_unmap_producer_port(hw, pp_addr);\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static int dlb_domain_drain_ldb_cqs(struct dlb_hw *hw,\n+\t\t\t\t    struct dlb_domain *domain,\n+\t\t\t\t    bool toggle_port)\n+{\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tstruct dlb_ldb_port *port;\n+\tint ret;\n+\n+\t/* If the domain hasn't been started, there's no traffic to drain */\n+\tif (!domain->started)\n+\t\treturn 0;\n+\n+\tDLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {\n+\t\tif (toggle_port)\n+\t\t\tdlb_ldb_port_cq_disable(hw, port);\n+\n+\t\tret = dlb_drain_ldb_cq(hw, port);\n+\t\tif (ret < 0)\n+\t\t\treturn ret;\n+\n+\t\tif (toggle_port)\n+\t\t\tdlb_ldb_port_cq_enable(hw, port);\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static void dlb_domain_disable_ldb_queue_write_perms(struct dlb_hw *hw,\n+\t\t\t\t\t\t     struct dlb_domain *domain)\n+{\n+\tint domain_offset = domain->id * DLB_MAX_NUM_LDB_QUEUES;\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tunion dlb_sys_ldb_vasqid_v r0;\n+\tstruct dlb_ldb_queue *queue;\n+\n+\tr0.field.vasqid_v = 0;\n+\n+\tDLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {\n+\t\tint idx = domain_offset + queue->id;\n+\n+\t\tDLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(idx), r0.val);\n+\t}\n+}\n+\n+static void dlb_domain_disable_ldb_seq_checks(struct dlb_hw *hw,\n+\t\t\t\t\t      struct dlb_domain *domain)\n+{\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tunion dlb_chp_sn_chk_enbl r1;\n+\tstruct dlb_ldb_port *port;\n+\n+\tr1.field.en = 0;\n+\n+\tDLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)\n+\t\tDLB_CSR_WR(hw,\n+\t\t\t   DLB_CHP_SN_CHK_ENBL(port->id),\n+\t\t\t   r1.val);\n+}\n+\n+static void dlb_domain_disable_ldb_port_crd_updates(struct dlb_hw *hw,\n+\t\t\t\t\t\t    struct dlb_domain *domain)\n+{\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tunion dlb_chp_ldb_pp_crd_req_state r0;\n+\tstruct dlb_ldb_port *port;\n+\n+\tr0.field.no_pp_credit_update = 1;\n+\n+\tDLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)\n+\t\tDLB_CSR_WR(hw,\n+\t\t\t   DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id),\n+\t\t\t   r0.val);\n+}\n+\n+static void dlb_domain_disable_ldb_port_interrupts(struct dlb_hw *hw,\n+\t\t\t\t\t\t   struct dlb_domain *domain)\n+{\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tunion dlb_chp_ldb_cq_int_enb r0 = { {0} };\n+\tunion dlb_chp_ldb_cq_wd_enb r1 = { {0} };\n+\tstruct dlb_ldb_port *port;\n+\n+\tr0.field.en_tim = 0;\n+\tr0.field.en_depth = 0;\n+\n+\tr1.field.wd_enable = 0;\n+\n+\tDLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {\n+\t\tDLB_CSR_WR(hw,\n+\t\t\t   DLB_CHP_LDB_CQ_INT_ENB(port->id),\n+\t\t\t   r0.val);\n+\n+\t\tDLB_CSR_WR(hw,\n+\t\t\t   DLB_CHP_LDB_CQ_WD_ENB(port->id),\n+\t\t\t   r1.val);\n+\t}\n+}\n+\n+static void dlb_domain_disable_dir_queue_write_perms(struct dlb_hw *hw,\n+\t\t\t\t\t\t     struct dlb_domain *domain)\n+{\n+\tint domain_offset = domain->id * DLB_MAX_NUM_DIR_PORTS;\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tunion dlb_sys_dir_vasqid_v r0;\n+\tstruct dlb_dir_pq_pair *port;\n+\n+\tr0.field.vasqid_v = 0;\n+\n+\tDLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {\n+\t\tint idx = domain_offset + port->id;\n+\n+\t\tDLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(idx), r0.val);\n+\t}\n+}\n+\n+static void dlb_domain_disable_dir_port_interrupts(struct dlb_hw *hw,\n+\t\t\t\t\t\t   struct dlb_domain *domain)\n+{\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tunion dlb_chp_dir_cq_int_enb r0 = { {0} };\n+\tunion dlb_chp_dir_cq_wd_enb r1 = { {0} };\n+\tstruct dlb_dir_pq_pair *port;\n+\n+\tr0.field.en_tim = 0;\n+\tr0.field.en_depth = 0;\n+\n+\tr1.field.wd_enable = 0;\n+\n+\tDLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {\n+\t\tDLB_CSR_WR(hw,\n+\t\t\t   DLB_CHP_DIR_CQ_INT_ENB(port->id),\n+\t\t\t   r0.val);\n+\n+\t\tDLB_CSR_WR(hw,\n+\t\t\t   DLB_CHP_DIR_CQ_WD_ENB(port->id),\n+\t\t\t   r1.val);\n+\t}\n+}\n+\n+static void dlb_domain_disable_dir_port_crd_updates(struct dlb_hw *hw,\n+\t\t\t\t\t\t    struct dlb_domain *domain)\n+{\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tunion dlb_chp_dir_pp_crd_req_state r0;\n+\tstruct dlb_dir_pq_pair *port;\n+\n+\tr0.field.no_pp_credit_update = 1;\n+\n+\tDLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)\n+\t\tDLB_CSR_WR(hw,\n+\t\t\t   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id),\n+\t\t\t   r0.val);\n+}\n+\n+static void dlb_domain_disable_dir_cqs(struct dlb_hw *hw,\n+\t\t\t\t       struct dlb_domain *domain)\n+{\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tstruct dlb_dir_pq_pair *port;\n+\n+\tDLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {\n+\t\tport->enabled = false;\n+\n+\t\tdlb_dir_port_cq_disable(hw, port);\n+\t}\n+}\n+\n+static void dlb_domain_disable_ldb_cqs(struct dlb_hw *hw,\n+\t\t\t\t       struct dlb_domain *domain)\n+{\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tstruct dlb_ldb_port *port;\n+\n+\tDLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {\n+\t\tport->enabled = false;\n+\n+\t\tdlb_ldb_port_cq_disable(hw, port);\n+\t}\n+}\n+\n+static void dlb_domain_enable_ldb_cqs(struct dlb_hw *hw,\n+\t\t\t\t      struct dlb_domain *domain)\n+{\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tstruct dlb_ldb_port *port;\n+\n+\tDLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {\n+\t\tport->enabled = true;\n+\n+\t\tdlb_ldb_port_cq_enable(hw, port);\n+\t}\n+}\n+\n+static struct dlb_ldb_queue *dlb_get_ldb_queue_from_id(struct dlb_hw *hw,\n+\t\t\t\t\t\t       u32 id)\n+{\n+\tif (id >= DLB_MAX_NUM_LDB_QUEUES)\n+\t\treturn NULL;\n+\n+\treturn &hw->rsrcs.ldb_queues[id];\n+}\n+\n+static void dlb_ldb_port_clear_has_work_bits(struct dlb_hw *hw,\n+\t\t\t\t\t     struct dlb_ldb_port *port,\n+\t\t\t\t\t     u8 slot)\n+{\n+\tunion dlb_lsp_ldb_sched_ctrl r2 = { {0} };\n+\n+\tr2.field.cq = port->id;\n+\tr2.field.qidix = slot;\n+\tr2.field.value = 0;\n+\tr2.field.rlist_haswork_v = 1;\n+\n+\tDLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);\n+\n+\tmemset(&r2, 0, sizeof(r2));\n+\n+\tr2.field.cq = port->id;\n+\tr2.field.qidix = slot;\n+\tr2.field.value = 0;\n+\tr2.field.nalb_haswork_v = 1;\n+\n+\tDLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);\n+\n+\tdlb_flush_csr(hw);\n+}\n+\n+static void dlb_domain_finish_map_port(struct dlb_hw *hw,\n+\t\t\t\t       struct dlb_domain *domain,\n+\t\t\t\t       struct dlb_ldb_port *port)\n+{\n+\tint i;\n+\n+\tfor (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {\n+\t\tunion dlb_lsp_qid_ldb_infl_cnt r0;\n+\t\tstruct dlb_ldb_queue *queue;\n+\t\tint qid;\n+\n+\t\tif (port->qid_map[i].state != DLB_QUEUE_MAP_IN_PROGRESS)\n+\t\t\tcontinue;\n+\n+\t\tqid = port->qid_map[i].qid;\n+\n+\t\tqueue = dlb_get_ldb_queue_from_id(hw, qid);\n+\n+\t\tif (!queue) {\n+\t\t\tDLB_HW_ERR(hw,\n+\t\t\t\t   \"[%s()] Internal error: unable to find queue %d\\n\",\n+\t\t\t\t   __func__, qid);\n+\t\t\tcontinue;\n+\t\t}\n+\n+\t\tr0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(qid));\n+\n+\t\tif (r0.field.count)\n+\t\t\tcontinue;\n+\n+\t\t/* Disable the affected CQ, and the CQs already mapped to the\n+\t\t * QID, before reading the QID's inflight count a second time.\n+\t\t * There is an unlikely race in which the QID may schedule one\n+\t\t * more QE after we read an inflight count of 0, and disabling\n+\t\t * the CQs guarantees that the race will not occur after a\n+\t\t * re-read of the inflight count register.\n+\t\t */\n+\t\tif (port->enabled)\n+\t\t\tdlb_ldb_port_cq_disable(hw, port);\n+\n+\t\tdlb_ldb_queue_disable_mapped_cqs(hw, domain, queue);\n+\n+\t\tr0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(qid));\n+\n+\t\tif (r0.field.count) {\n+\t\t\tif (port->enabled)\n+\t\t\t\tdlb_ldb_port_cq_enable(hw, port);\n+\n+\t\t\tdlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);\n+\n+\t\t\tcontinue;\n+\t\t}\n+\n+\t\tdlb_ldb_port_finish_map_qid_dynamic(hw, domain, port, queue);\n+\t}\n+}\n+\n+static unsigned int\n+dlb_domain_finish_map_qid_procedures(struct dlb_hw *hw,\n+\t\t\t\t     struct dlb_domain *domain)\n+{\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tstruct dlb_ldb_port *port;\n+\n+\tif (!domain->configured || domain->num_pending_additions == 0)\n+\t\treturn 0;\n+\n+\tDLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)\n+\t\tdlb_domain_finish_map_port(hw, domain, port);\n+\n+\treturn domain->num_pending_additions;\n+}\n+\n+unsigned int dlb_finish_map_qid_procedures(struct dlb_hw *hw)\n+{\n+\tint i, num = 0;\n+\n+\t/* Finish queue map jobs for any domain that needs it */\n+\tfor (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {\n+\t\tstruct dlb_domain *domain = &hw->domains[i];\n+\n+\t\tnum += dlb_domain_finish_map_qid_procedures(hw, domain);\n+\t}\n+\n+\treturn num;\n+}\n+\n+\n+static int dlb_domain_wait_for_ldb_cqs_to_empty(struct dlb_hw *hw,\n+\t\t\t\t\t\tstruct dlb_domain *domain)\n+{\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tstruct dlb_ldb_port *port;\n+\n+\tDLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {\n+\t\tint i;\n+\n+\t\tfor (i = 0; i < DLB_MAX_CQ_COMP_CHECK_LOOPS; i++) {\n+\t\t\tif (dlb_ldb_cq_inflight_count(hw, port) == 0)\n+\t\t\t\tbreak;\n+\t\t}\n+\n+\t\tif (i == DLB_MAX_CQ_COMP_CHECK_LOOPS) {\n+\t\t\tDLB_HW_ERR(hw,\n+\t\t\t\t   \"[%s()] Internal error: failed to flush load-balanced port %d's completions.\\n\",\n+\t\t\t\t   __func__, port->id);\n+\t\t\treturn -EFAULT;\n+\t\t}\n+\t}\n+\n+\treturn 0;\n+}\n+\n+\n+static void dlb_domain_finish_unmap_port_slot(struct dlb_hw *hw,\n+\t\t\t\t\t      struct dlb_domain *domain,\n+\t\t\t\t\t      struct dlb_ldb_port *port,\n+\t\t\t\t\t      int slot)\n+{\n+\tenum dlb_qid_map_state state;\n+\tstruct dlb_ldb_queue *queue;\n+\n+\tqueue = &hw->rsrcs.ldb_queues[port->qid_map[slot].qid];\n+\n+\tstate = port->qid_map[slot].state;\n+\n+\t/* Update the QID2CQIDX and CQ2QID vectors */\n+\tdlb_ldb_port_unmap_qid(hw, port, queue);\n+\n+\t/* Ensure the QID will not be serviced by this {CQ, slot} by clearing\n+\t * the has_work bits\n+\t */\n+\tdlb_ldb_port_clear_has_work_bits(hw, port, slot);\n+\n+\t/* Reset the {CQ, slot} to its default state */\n+\tdlb_ldb_port_set_queue_if_status(hw, port, slot);\n+\n+\t/* Re-enable the CQ if it was not manually disabled by the user */\n+\tif (port->enabled)\n+\t\tdlb_ldb_port_cq_enable(hw, port);\n+\n+\t/* If there is a mapping that is pending this slot's removal, perform\n+\t * the mapping now.\n+\t */\n+\tif (state == DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP) {\n+\t\tstruct dlb_ldb_port_qid_map *map;\n+\t\tstruct dlb_ldb_queue *map_queue;\n+\t\tu8 prio;\n+\n+\t\tmap = &port->qid_map[slot];\n+\n+\t\tmap->qid = map->pending_qid;\n+\t\tmap->priority = map->pending_priority;\n+\n+\t\tmap_queue = &hw->rsrcs.ldb_queues[map->qid];\n+\t\tprio = map->priority;\n+\n+\t\tdlb_ldb_port_map_qid(hw, domain, port, map_queue, prio);\n+\t}\n+}\n+\n+static bool dlb_domain_finish_unmap_port(struct dlb_hw *hw,\n+\t\t\t\t\t struct dlb_domain *domain,\n+\t\t\t\t\t struct dlb_ldb_port *port)\n+{\n+\tunion dlb_lsp_cq_ldb_infl_cnt r0;\n+\tint i;\n+\n+\tif (port->num_pending_removals == 0)\n+\t\treturn false;\n+\n+\t/* The unmap requires all the CQ's outstanding inflights to be\n+\t * completed.\n+\t */\n+\tr0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_INFL_CNT(port->id));\n+\tif (r0.field.count > 0)\n+\t\treturn false;\n+\n+\tfor (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {\n+\t\tstruct dlb_ldb_port_qid_map *map;\n+\n+\t\tmap = &port->qid_map[i];\n+\n+\t\tif (map->state != DLB_QUEUE_UNMAP_IN_PROGRESS &&\n+\t\t    map->state != DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP)\n+\t\t\tcontinue;\n+\n+\t\tdlb_domain_finish_unmap_port_slot(hw, domain, port, i);\n+\t}\n+\n+\treturn true;\n+}\n+\n+static unsigned int\n+dlb_domain_finish_unmap_qid_procedures(struct dlb_hw *hw,\n+\t\t\t\t       struct dlb_domain *domain)\n+{\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tstruct dlb_ldb_port *port;\n+\n+\tif (!domain->configured || domain->num_pending_removals == 0)\n+\t\treturn 0;\n+\n+\tDLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)\n+\t\tdlb_domain_finish_unmap_port(hw, domain, port);\n+\n+\treturn domain->num_pending_removals;\n+}\n+\n+unsigned int dlb_finish_unmap_qid_procedures(struct dlb_hw *hw)\n+{\n+\tint i, num = 0;\n+\n+\t/* Finish queue unmap jobs for any domain that needs it */\n+\tfor (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {\n+\t\tstruct dlb_domain *domain = &hw->domains[i];\n+\n+\t\tnum += dlb_domain_finish_unmap_qid_procedures(hw, domain);\n+\t}\n+\n+\treturn num;\n+}\n+\n+/* Returns whether the queue is empty, including its inflight and replay\n+ * counts.\n+ */\n+static bool dlb_ldb_queue_is_empty(struct dlb_hw *hw,\n+\t\t\t\t   struct dlb_ldb_queue *queue)\n+{\n+\tunion dlb_lsp_qid_ldb_replay_cnt r0;\n+\tunion dlb_lsp_qid_aqed_active_cnt r1;\n+\tunion dlb_lsp_qid_atq_enqueue_cnt r2;\n+\tunion dlb_lsp_qid_ldb_enqueue_cnt r3;\n+\tunion dlb_lsp_qid_ldb_infl_cnt r4;\n+\n+\tr0.val = DLB_CSR_RD(hw,\n+\t\t\t    DLB_LSP_QID_LDB_REPLAY_CNT(queue->id));\n+\tif (r0.val)\n+\t\treturn false;\n+\n+\tr1.val = DLB_CSR_RD(hw,\n+\t\t\t    DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id));\n+\tif (r1.val)\n+\t\treturn false;\n+\n+\tr2.val = DLB_CSR_RD(hw,\n+\t\t\t    DLB_LSP_QID_ATQ_ENQUEUE_CNT(queue->id));\n+\tif (r2.val)\n+\t\treturn false;\n+\n+\tr3.val = DLB_CSR_RD(hw,\n+\t\t\t    DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id));\n+\tif (r3.val)\n+\t\treturn false;\n+\n+\tr4.val = DLB_CSR_RD(hw,\n+\t\t\t    DLB_LSP_QID_LDB_INFL_CNT(queue->id));\n+\tif (r4.val)\n+\t\treturn false;\n+\n+\treturn true;\n+}\n+\n+static bool dlb_domain_mapped_queues_empty(struct dlb_hw *hw,\n+\t\t\t\t\t   struct dlb_domain *domain)\n+{\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tstruct dlb_ldb_queue *queue;\n+\n+\tDLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {\n+\t\tif (queue->num_mappings == 0)\n+\t\t\tcontinue;\n+\n+\t\tif (!dlb_ldb_queue_is_empty(hw, queue))\n+\t\t\treturn false;\n+\t}\n+\n+\treturn true;\n+}\n+\n+static int dlb_domain_drain_mapped_queues(struct dlb_hw *hw,\n+\t\t\t\t\t  struct dlb_domain *domain)\n+{\n+\tint i, ret;\n+\n+\t/* If the domain hasn't been started, there's no traffic to drain */\n+\tif (!domain->started)\n+\t\treturn 0;\n+\n+\tif (domain->num_pending_removals > 0) {\n+\t\tDLB_HW_ERR(hw,\n+\t\t\t   \"[%s()] Internal error: failed to unmap domain queues\\n\",\n+\t\t\t   __func__);\n+\t\treturn -EFAULT;\n+\t}\n+\n+\tfor (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {\n+\t\tret = dlb_domain_drain_ldb_cqs(hw, domain, true);\n+\t\tif (ret < 0)\n+\t\t\treturn ret;\n+\n+\t\tif (dlb_domain_mapped_queues_empty(hw, domain))\n+\t\t\tbreak;\n+\t}\n+\n+\tif (i == DLB_MAX_QID_EMPTY_CHECK_LOOPS) {\n+\t\tDLB_HW_ERR(hw,\n+\t\t\t   \"[%s()] Internal error: failed to empty queues\\n\",\n+\t\t\t   __func__);\n+\t\treturn -EFAULT;\n+\t}\n+\n+\t/* Drain the CQs one more time. For the queues to go empty, they would\n+\t * have scheduled one or more QEs.\n+\t */\n+\tret = dlb_domain_drain_ldb_cqs(hw, domain, true);\n+\tif (ret < 0)\n+\t\treturn ret;\n+\n+\treturn 0;\n+}\n+\n+static int dlb_domain_drain_unmapped_queue(struct dlb_hw *hw,\n+\t\t\t\t\t   struct dlb_domain *domain,\n+\t\t\t\t\t   struct dlb_ldb_queue *queue)\n+{\n+\tstruct dlb_ldb_port *port;\n+\tint ret;\n+\n+\t/* If a domain has LDB queues, it must have LDB ports */\n+\tif (dlb_list_empty(&domain->used_ldb_ports)) {\n+\t\tDLB_HW_ERR(hw,\n+\t\t\t   \"[%s()] Internal error: No configured LDB ports\\n\",\n+\t\t\t   __func__);\n+\t\treturn -EFAULT;\n+\t}\n+\n+\tport = DLB_DOM_LIST_HEAD(domain->used_ldb_ports, typeof(*port));\n+\n+\t/* If necessary, free up a QID slot in this CQ */\n+\tif (port->num_mappings == DLB_MAX_NUM_QIDS_PER_LDB_CQ) {\n+\t\tstruct dlb_ldb_queue *mapped_queue;\n+\n+\t\tmapped_queue = &hw->rsrcs.ldb_queues[port->qid_map[0].qid];\n+\n+\t\tret = dlb_ldb_port_unmap_qid(hw, port, mapped_queue);\n+\t\tif (ret)\n+\t\t\treturn ret;\n+\t}\n+\n+\tret = dlb_ldb_port_map_qid_dynamic(hw, port, queue, 0);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\treturn dlb_domain_drain_mapped_queues(hw, domain);\n+}\n+\n+static int dlb_domain_drain_unmapped_queues(struct dlb_hw *hw,\n+\t\t\t\t\t    struct dlb_domain *domain)\n+{\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tstruct dlb_ldb_queue *queue;\n+\tint ret;\n+\n+\t/* If the domain hasn't been started, there's no traffic to drain */\n+\tif (!domain->started)\n+\t\treturn 0;\n+\n+\tDLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {\n+\t\tif (queue->num_mappings != 0 ||\n+\t\t    dlb_ldb_queue_is_empty(hw, queue))\n+\t\t\tcontinue;\n+\n+\t\tret = dlb_domain_drain_unmapped_queue(hw, domain, queue);\n+\t\tif (ret)\n+\t\t\treturn ret;\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static int dlb_domain_wait_for_ldb_pool_refill(struct dlb_hw *hw,\n+\t\t\t\t\t       struct dlb_domain *domain)\n+{\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tstruct dlb_credit_pool *pool;\n+\n+\t/* Confirm that all credits are returned to the domain's credit pools */\n+\tDLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {\n+\t\tunion dlb_chp_qed_fl_push_ptr r0;\n+\t\tunion dlb_chp_qed_fl_pop_ptr r1;\n+\t\tunsigned long pop_offs, push_offs;\n+\t\tint i;\n+\n+\t\tpush_offs = DLB_CHP_QED_FL_PUSH_PTR(pool->id);\n+\t\tpop_offs = DLB_CHP_QED_FL_POP_PTR(pool->id);\n+\n+\t\tfor (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {\n+\t\t\tr0.val = DLB_CSR_RD(hw, push_offs);\n+\n+\t\t\tr1.val = DLB_CSR_RD(hw, pop_offs);\n+\n+\t\t\t/* Break early if the freelist is replenished */\n+\t\t\tif (r1.field.pop_ptr == r0.field.push_ptr &&\n+\t\t\t    r1.field.generation != r0.field.generation) {\n+\t\t\t\tbreak;\n+\t\t\t}\n+\t\t}\n+\n+\t\t/* Error if the freelist is not full */\n+\t\tif (r1.field.pop_ptr != r0.field.push_ptr ||\n+\t\t    r1.field.generation == r0.field.generation) {\n+\t\t\treturn -EFAULT;\n+\t\t}\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static int dlb_domain_wait_for_dir_pool_refill(struct dlb_hw *hw,\n+\t\t\t\t\t       struct dlb_domain *domain)\n+{\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tstruct dlb_credit_pool *pool;\n+\n+\t/* Confirm that all credits are returned to the domain's credit pools */\n+\tDLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {\n+\t\tunion dlb_chp_dqed_fl_push_ptr r0;\n+\t\tunion dlb_chp_dqed_fl_pop_ptr r1;\n+\t\tunsigned long pop_offs, push_offs;\n+\t\tint i;\n+\n+\t\tpush_offs = DLB_CHP_DQED_FL_PUSH_PTR(pool->id);\n+\t\tpop_offs = DLB_CHP_DQED_FL_POP_PTR(pool->id);\n+\n+\t\tfor (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {\n+\t\t\tr0.val = DLB_CSR_RD(hw, push_offs);\n+\n+\t\t\tr1.val = DLB_CSR_RD(hw, pop_offs);\n+\n+\t\t\t/* Break early if the freelist is replenished */\n+\t\t\tif (r1.field.pop_ptr == r0.field.push_ptr &&\n+\t\t\t    r1.field.generation != r0.field.generation) {\n+\t\t\t\tbreak;\n+\t\t\t}\n+\t\t}\n+\n+\t\t/* Error if the freelist is not full */\n+\t\tif (r1.field.pop_ptr != r0.field.push_ptr ||\n+\t\t    r1.field.generation == r0.field.generation) {\n+\t\t\treturn -EFAULT;\n+\t\t}\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static u32 dlb_dir_queue_depth(struct dlb_hw *hw,\n+\t\t\t       struct dlb_dir_pq_pair *queue)\n+{\n+\tunion dlb_lsp_qid_dir_enqueue_cnt r0;\n+\n+\tr0.val = DLB_CSR_RD(hw, DLB_LSP_QID_DIR_ENQUEUE_CNT(queue->id));\n+\n+\treturn r0.field.count;\n+}\n+\n+static bool dlb_dir_queue_is_empty(struct dlb_hw *hw,\n+\t\t\t\t   struct dlb_dir_pq_pair *queue)\n+{\n+\treturn dlb_dir_queue_depth(hw, queue) == 0;\n+}\n+\n+static bool dlb_domain_dir_queues_empty(struct dlb_hw *hw,\n+\t\t\t\t\tstruct dlb_domain *domain)\n+{\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tstruct dlb_dir_pq_pair *queue;\n+\n+\tDLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, queue, iter) {\n+\t\tif (!dlb_dir_queue_is_empty(hw, queue))\n+\t\t\treturn false;\n+\t}\n+\n+\treturn true;\n+}\n+\n+static u32 dlb_dir_cq_token_count(struct dlb_hw *hw,\n+\t\t\t\t  struct dlb_dir_pq_pair *port)\n+{\n+\tunion dlb_lsp_cq_dir_tkn_cnt r0;\n+\n+\tr0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_DIR_TKN_CNT(port->id));\n+\n+\treturn r0.field.count;\n+}\n+\n+static void dlb_drain_dir_cq(struct dlb_hw *hw, struct dlb_dir_pq_pair *port)\n+{\n+\tunsigned int port_id = port->id;\n+\tu32 cnt;\n+\n+\t/* Return any outstanding tokens */\n+\tcnt = dlb_dir_cq_token_count(hw, port);\n+\n+\tif (cnt != 0) {\n+\t\tstruct dlb_hcw hcw_mem[8], *hcw;\n+\t\tvoid  *pp_addr;\n+\n+\t\tpp_addr = os_map_producer_port(hw, port_id, false);\n+\n+\t\t/* Point hcw to a 64B-aligned location */\n+\t\thcw = (struct dlb_hcw *)((uintptr_t)&hcw_mem[4] & ~0x3F);\n+\n+\t\t/* Program the first HCW for a batch token return and\n+\t\t * the rest as NOOPS\n+\t\t */\n+\t\tmemset(hcw, 0, 4 * sizeof(*hcw));\n+\t\thcw->cq_token = 1;\n+\t\thcw->lock_id = cnt - 1;\n+\n+\t\tos_enqueue_four_hcws(hw, hcw, pp_addr);\n+\n+\t\tos_fence_hcw(hw, pp_addr);\n+\n+\t\tos_unmap_producer_port(hw, pp_addr);\n+\t}\n+}\n+\n+static int dlb_domain_drain_dir_cqs(struct dlb_hw *hw,\n+\t\t\t\t    struct dlb_domain *domain,\n+\t\t\t\t    bool toggle_port)\n+{\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tstruct dlb_dir_pq_pair *port;\n+\n+\tDLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {\n+\t\t/* Can't drain a port if it's not configured, and there's\n+\t\t * nothing to drain if its queue is unconfigured.\n+\t\t */\n+\t\tif (!port->port_configured || !port->queue_configured)\n+\t\t\tcontinue;\n+\n+\t\tif (toggle_port)\n+\t\t\tdlb_dir_port_cq_disable(hw, port);\n+\n+\t\tdlb_drain_dir_cq(hw, port);\n+\n+\t\tif (toggle_port)\n+\t\t\tdlb_dir_port_cq_enable(hw, port);\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static int dlb_domain_drain_dir_queues(struct dlb_hw *hw,\n+\t\t\t\t       struct dlb_domain *domain)\n+{\n+\tint i;\n+\n+\t/* If the domain hasn't been started, there's no traffic to drain */\n+\tif (!domain->started)\n+\t\treturn 0;\n+\n+\tfor (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {\n+\t\tdlb_domain_drain_dir_cqs(hw, domain, true);\n+\n+\t\tif (dlb_domain_dir_queues_empty(hw, domain))\n+\t\t\tbreak;\n+\t}\n+\n+\tif (i == DLB_MAX_QID_EMPTY_CHECK_LOOPS) {\n+\t\tDLB_HW_ERR(hw,\n+\t\t\t   \"[%s()] Internal error: failed to empty queues\\n\",\n+\t\t\t   __func__);\n+\t\treturn -EFAULT;\n+\t}\n+\n+\t/* Drain the CQs one more time. For the queues to go empty, they would\n+\t * have scheduled one or more QEs.\n+\t */\n+\tdlb_domain_drain_dir_cqs(hw, domain, true);\n+\n+\treturn 0;\n+}\n+\n+static void dlb_domain_disable_dir_producer_ports(struct dlb_hw *hw,\n+\t\t\t\t\t\t  struct dlb_domain *domain)\n+{\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tstruct dlb_dir_pq_pair *port;\n+\tunion dlb_sys_dir_pp_v r1;\n+\n+\tr1.field.pp_v = 0;\n+\n+\tDLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)\n+\t\tDLB_CSR_WR(hw,\n+\t\t\t   DLB_SYS_DIR_PP_V(port->id),\n+\t\t\t   r1.val);\n+}\n+\n+static void dlb_domain_disable_ldb_producer_ports(struct dlb_hw *hw,\n+\t\t\t\t\t\t  struct dlb_domain *domain)\n+{\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tunion dlb_sys_ldb_pp_v r1;\n+\tstruct dlb_ldb_port *port;\n+\n+\tr1.field.pp_v = 0;\n+\n+\tDLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {\n+\t\tDLB_CSR_WR(hw,\n+\t\t\t   DLB_SYS_LDB_PP_V(port->id),\n+\t\t\t   r1.val);\n+\n+\t\thw->pf.num_enabled_ldb_ports--;\n+\t}\n+}\n+\n+static void dlb_domain_disable_dir_pools(struct dlb_hw *hw,\n+\t\t\t\t\t struct dlb_domain *domain)\n+{\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tunion dlb_sys_dir_pool_enbld r0 = { {0} };\n+\tstruct dlb_credit_pool *pool;\n+\n+\tDLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)\n+\t\tDLB_CSR_WR(hw,\n+\t\t\t   DLB_SYS_DIR_POOL_ENBLD(pool->id),\n+\t\t\t   r0.val);\n+}\n+\n+static void dlb_domain_disable_ldb_pools(struct dlb_hw *hw,\n+\t\t\t\t\t struct dlb_domain *domain)\n+{\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tunion dlb_sys_ldb_pool_enbld r0 = { {0} };\n+\tstruct dlb_credit_pool *pool;\n+\n+\tDLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)\n+\t\tDLB_CSR_WR(hw,\n+\t\t\t   DLB_SYS_LDB_POOL_ENBLD(pool->id),\n+\t\t\t   r0.val);\n+}\n+\n+static int dlb_reset_hw_resource(struct dlb_hw *hw, int type, int id)\n+{\n+\tunion dlb_cfg_mstr_diag_reset_sts r0 = { {0} };\n+\tunion dlb_cfg_mstr_bcast_reset_vf_start r1 = { {0} };\n+\tint i;\n+\n+\tr1.field.vf_reset_start = 1;\n+\n+\tr1.field.vf_reset_type = type;\n+\tr1.field.vf_reset_id = id;\n+\n+\tDLB_CSR_WR(hw, DLB_CFG_MSTR_BCAST_RESET_VF_START, r1.val);\n+\n+\t/* Wait for hardware to complete. This is a finite time operation,\n+\t * but wait set a loop bound just in case.\n+\t */\n+\tfor (i = 0; i < 1024 * 1024; i++) {\n+\t\tr0.val = DLB_CSR_RD(hw, DLB_CFG_MSTR_DIAG_RESET_STS);\n+\n+\t\tif (r0.field.chp_vf_reset_done &&\n+\t\t    r0.field.rop_vf_reset_done &&\n+\t\t    r0.field.lsp_vf_reset_done &&\n+\t\t    r0.field.nalb_vf_reset_done &&\n+\t\t    r0.field.ap_vf_reset_done &&\n+\t\t    r0.field.dp_vf_reset_done &&\n+\t\t    r0.field.qed_vf_reset_done &&\n+\t\t    r0.field.dqed_vf_reset_done &&\n+\t\t    r0.field.aqed_vf_reset_done)\n+\t\t\treturn 0;\n+\n+\t\tos_udelay(1);\n+\t}\n+\n+\treturn -ETIMEDOUT;\n+}\n+\n+static int dlb_domain_reset_hw_resources(struct dlb_hw *hw,\n+\t\t\t\t\t struct dlb_domain *domain)\n+{\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tstruct dlb_dir_pq_pair *dir_port;\n+\tstruct dlb_ldb_queue *ldb_queue;\n+\tstruct dlb_ldb_port *ldb_port;\n+\tstruct dlb_credit_pool *pool;\n+\tint ret;\n+\n+\tDLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {\n+\t\tret = dlb_reset_hw_resource(hw,\n+\t\t\t\t\t    VF_RST_TYPE_POOL_LDB,\n+\t\t\t\t\t    pool->id);\n+\t\tif (ret)\n+\t\t\treturn ret;\n+\t}\n+\n+\tDLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {\n+\t\tret = dlb_reset_hw_resource(hw,\n+\t\t\t\t\t    VF_RST_TYPE_POOL_DIR,\n+\t\t\t\t\t    pool->id);\n+\t\tif (ret)\n+\t\t\treturn ret;\n+\t}\n+\n+\tDLB_DOM_LIST_FOR(domain->used_ldb_queues, ldb_queue, iter) {\n+\t\tret = dlb_reset_hw_resource(hw,\n+\t\t\t\t\t    VF_RST_TYPE_QID_LDB,\n+\t\t\t\t\t    ldb_queue->id);\n+\t\tif (ret)\n+\t\t\treturn ret;\n+\t}\n+\n+\tDLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {\n+\t\tret = dlb_reset_hw_resource(hw,\n+\t\t\t\t\t    VF_RST_TYPE_QID_DIR,\n+\t\t\t\t\t    dir_port->id);\n+\t\tif (ret)\n+\t\t\treturn ret;\n+\t}\n+\n+\tDLB_DOM_LIST_FOR(domain->used_ldb_ports, ldb_port, iter) {\n+\t\tret = dlb_reset_hw_resource(hw,\n+\t\t\t\t\t    VF_RST_TYPE_CQ_LDB,\n+\t\t\t\t\t    ldb_port->id);\n+\t\tif (ret)\n+\t\t\treturn ret;\n+\t}\n+\n+\tDLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {\n+\t\tret = dlb_reset_hw_resource(hw,\n+\t\t\t\t\t    VF_RST_TYPE_CQ_DIR,\n+\t\t\t\t\t    dir_port->id);\n+\t\tif (ret)\n+\t\t\treturn ret;\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static int dlb_domain_verify_reset_success(struct dlb_hw *hw,\n+\t\t\t\t\t   struct dlb_domain *domain)\n+{\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tstruct dlb_dir_pq_pair *dir_port;\n+\tstruct dlb_ldb_port *ldb_port;\n+\tstruct dlb_credit_pool *pool;\n+\tstruct dlb_ldb_queue *queue;\n+\n+\t/* Confirm that all credits are returned to the domain's credit pools */\n+\tDLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {\n+\t\tunion dlb_chp_dqed_fl_pop_ptr r0;\n+\t\tunion dlb_chp_dqed_fl_push_ptr r1;\n+\n+\t\tr0.val = DLB_CSR_RD(hw,\n+\t\t\t\t    DLB_CHP_DQED_FL_POP_PTR(pool->id));\n+\n+\t\tr1.val = DLB_CSR_RD(hw,\n+\t\t\t\t    DLB_CHP_DQED_FL_PUSH_PTR(pool->id));\n+\n+\t\tif (r0.field.pop_ptr != r1.field.push_ptr ||\n+\t\t    r0.field.generation == r1.field.generation) {\n+\t\t\tDLB_HW_ERR(hw,\n+\t\t\t\t   \"[%s()] Internal error: failed to refill directed pool %d's credits.\\n\",\n+\t\t\t\t   __func__, pool->id);\n+\t\t\treturn -EFAULT;\n+\t\t}\n+\t}\n+\n+\t/* Confirm that all the domain's queue's inflight counts and AQED\n+\t * active counts are 0.\n+\t */\n+\tDLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {\n+\t\tif (!dlb_ldb_queue_is_empty(hw, queue)) {\n+\t\t\tDLB_HW_ERR(hw,\n+\t\t\t\t   \"[%s()] Internal error: failed to empty ldb queue %d\\n\",\n+\t\t\t\t   __func__, queue->id);\n+\t\t\treturn -EFAULT;\n+\t\t}\n+\t}\n+\n+\t/* Confirm that all the domain's CQs inflight and token counts are 0. */\n+\tDLB_DOM_LIST_FOR(domain->used_ldb_ports, ldb_port, iter) {\n+\t\tif (dlb_ldb_cq_inflight_count(hw, ldb_port) ||\n+\t\t    dlb_ldb_cq_token_count(hw, ldb_port)) {\n+\t\t\tDLB_HW_ERR(hw,\n+\t\t\t\t   \"[%s()] Internal error: failed to empty ldb port %d\\n\",\n+\t\t\t\t   __func__, ldb_port->id);\n+\t\t\treturn -EFAULT;\n+\t\t}\n+\t}\n+\n+\tDLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {\n+\t\tif (!dlb_dir_queue_is_empty(hw, dir_port)) {\n+\t\t\tDLB_HW_ERR(hw,\n+\t\t\t\t   \"[%s()] Internal error: failed to empty dir queue %d\\n\",\n+\t\t\t\t   __func__, dir_port->id);\n+\t\t\treturn -EFAULT;\n+\t\t}\n+\n+\t\tif (dlb_dir_cq_token_count(hw, dir_port)) {\n+\t\t\tDLB_HW_ERR(hw,\n+\t\t\t\t   \"[%s()] Internal error: failed to empty dir port %d\\n\",\n+\t\t\t\t   __func__, dir_port->id);\n+\t\t\treturn -EFAULT;\n+\t\t}\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static void __dlb_domain_reset_ldb_port_registers(struct dlb_hw *hw,\n+\t\t\t\t\t\t  struct dlb_ldb_port *port)\n+{\n+\tunion dlb_chp_ldb_pp_state_reset r0 = { {0} };\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id),\n+\t\t   DLB_CHP_LDB_PP_CRD_REQ_STATE_RST);\n+\n+\t/* Reset the port's load-balanced and directed credit state */\n+\tr0.field.dir_type = 0;\n+\tr0.field.reset_pp_state = 1;\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_LDB_PP_STATE_RESET(port->id),\n+\t\t   r0.val);\n+\n+\tr0.field.dir_type = 1;\n+\tr0.field.reset_pp_state = 1;\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_LDB_PP_STATE_RESET(port->id),\n+\t\t   r0.val);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_LDB_PP_DIR_PUSH_PTR(port->id),\n+\t\t   DLB_CHP_LDB_PP_DIR_PUSH_PTR_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_LDB_PP_LDB_PUSH_PTR(port->id),\n+\t\t   DLB_CHP_LDB_PP_LDB_PUSH_PTR_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(port->id),\n+\t\t   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_LDB_PP_LDB_CRD_LWM(port->id),\n+\t\t   DLB_CHP_LDB_PP_LDB_CRD_LWM_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_LDB_PP_LDB_CRD_HWM(port->id),\n+\t\t   DLB_CHP_LDB_PP_LDB_CRD_HWM_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_LDB_LDB_PP2POOL(port->id),\n+\t\t   DLB_CHP_LDB_LDB_PP2POOL_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(port->id),\n+\t\t   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_LDB_PP_DIR_CRD_LWM(port->id),\n+\t\t   DLB_CHP_LDB_PP_DIR_CRD_LWM_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_LDB_PP_DIR_CRD_HWM(port->id),\n+\t\t   DLB_CHP_LDB_PP_DIR_CRD_HWM_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_LDB_DIR_PP2POOL(port->id),\n+\t\t   DLB_CHP_LDB_DIR_PP2POOL_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_SYS_LDB_PP2LDBPOOL(port->id),\n+\t\t   DLB_SYS_LDB_PP2LDBPOOL_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_SYS_LDB_PP2DIRPOOL(port->id),\n+\t\t   DLB_SYS_LDB_PP2DIRPOOL_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_HIST_LIST_LIM(port->id),\n+\t\t   DLB_CHP_HIST_LIST_LIM_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_HIST_LIST_BASE(port->id),\n+\t\t   DLB_CHP_HIST_LIST_BASE_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_HIST_LIST_POP_PTR(port->id),\n+\t\t   DLB_CHP_HIST_LIST_POP_PTR_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_HIST_LIST_PUSH_PTR(port->id),\n+\t\t   DLB_CHP_HIST_LIST_PUSH_PTR_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_LDB_CQ_WPTR(port->id),\n+\t\t   DLB_CHP_LDB_CQ_WPTR_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_LDB_CQ_INT_DEPTH_THRSH(port->id),\n+\t\t   DLB_CHP_LDB_CQ_INT_DEPTH_THRSH_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_LDB_CQ_TMR_THRESHOLD(port->id),\n+\t\t   DLB_CHP_LDB_CQ_TMR_THRESHOLD_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_LDB_CQ_INT_ENB(port->id),\n+\t\t   DLB_CHP_LDB_CQ_INT_ENB_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_LSP_CQ_LDB_INFL_LIM(port->id),\n+\t\t   DLB_LSP_CQ_LDB_INFL_LIM_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_LSP_CQ2PRIOV(port->id),\n+\t\t   DLB_LSP_CQ2PRIOV_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL(port->id),\n+\t\t   DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(port->id),\n+\t\t   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(port->id),\n+\t\t   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_LSP_CQ_LDB_DSBL(port->id),\n+\t\t   DLB_LSP_CQ_LDB_DSBL_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_SYS_LDB_CQ2VF_PF(port->id),\n+\t\t   DLB_SYS_LDB_CQ2VF_PF_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_SYS_LDB_PP2VF_PF(port->id),\n+\t\t   DLB_SYS_LDB_PP2VF_PF_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_SYS_LDB_CQ_ADDR_L(port->id),\n+\t\t   DLB_SYS_LDB_CQ_ADDR_L_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_SYS_LDB_CQ_ADDR_U(port->id),\n+\t\t   DLB_SYS_LDB_CQ_ADDR_U_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_SYS_LDB_PP_ADDR_L(port->id),\n+\t\t   DLB_SYS_LDB_PP_ADDR_L_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_SYS_LDB_PP_ADDR_U(port->id),\n+\t\t   DLB_SYS_LDB_PP_ADDR_U_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_SYS_LDB_PP_V(port->id),\n+\t\t   DLB_SYS_LDB_PP_V_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_SYS_LDB_PP2VAS(port->id),\n+\t\t   DLB_SYS_LDB_PP2VAS_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_SYS_LDB_CQ_ISR(port->id),\n+\t\t   DLB_SYS_LDB_CQ_ISR_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_SYS_WBUF_LDB_FLAGS(port->id),\n+\t\t   DLB_SYS_WBUF_LDB_FLAGS_RST);\n+}\n+\n+static void __dlb_domain_reset_dir_port_registers(struct dlb_hw *hw,\n+\t\t\t\t\t\t  struct dlb_dir_pq_pair *port)\n+{\n+\tunion dlb_chp_dir_pp_state_reset r0 = { {0} };\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id),\n+\t\t   DLB_CHP_DIR_PP_CRD_REQ_STATE_RST);\n+\n+\t/* Reset the port's load-balanced and directed credit state */\n+\tr0.field.dir_type = 0;\n+\tr0.field.reset_pp_state = 1;\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_DIR_PP_STATE_RESET(port->id),\n+\t\t   r0.val);\n+\n+\tr0.field.dir_type = 1;\n+\tr0.field.reset_pp_state = 1;\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_DIR_PP_STATE_RESET(port->id),\n+\t\t   r0.val);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_DIR_PP_DIR_PUSH_PTR(port->id),\n+\t\t   DLB_CHP_DIR_PP_DIR_PUSH_PTR_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_DIR_PP_LDB_PUSH_PTR(port->id),\n+\t\t   DLB_CHP_DIR_PP_LDB_PUSH_PTR_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(port->id),\n+\t\t   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_DIR_PP_LDB_CRD_LWM(port->id),\n+\t\t   DLB_CHP_DIR_PP_LDB_CRD_LWM_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_DIR_PP_LDB_CRD_HWM(port->id),\n+\t\t   DLB_CHP_DIR_PP_LDB_CRD_HWM_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_DIR_LDB_PP2POOL(port->id),\n+\t\t   DLB_CHP_DIR_LDB_PP2POOL_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(port->id),\n+\t\t   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_DIR_PP_DIR_CRD_LWM(port->id),\n+\t\t   DLB_CHP_DIR_PP_DIR_CRD_LWM_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_DIR_PP_DIR_CRD_HWM(port->id),\n+\t\t   DLB_CHP_DIR_PP_DIR_CRD_HWM_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_DIR_DIR_PP2POOL(port->id),\n+\t\t   DLB_CHP_DIR_DIR_PP2POOL_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_SYS_DIR_PP2LDBPOOL(port->id),\n+\t\t   DLB_SYS_DIR_PP2LDBPOOL_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_SYS_DIR_PP2DIRPOOL(port->id),\n+\t\t   DLB_SYS_DIR_PP2DIRPOOL_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_DIR_CQ_WPTR(port->id),\n+\t\t   DLB_CHP_DIR_CQ_WPTR_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(port->id),\n+\t\t   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(port->id),\n+\t\t   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_LSP_CQ_DIR_DSBL(port->id),\n+\t\t   DLB_LSP_CQ_DIR_DSBL_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_DIR_CQ_WPTR(port->id),\n+\t\t   DLB_CHP_DIR_CQ_WPTR_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_DIR_CQ_INT_DEPTH_THRSH(port->id),\n+\t\t   DLB_CHP_DIR_CQ_INT_DEPTH_THRSH_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_DIR_CQ_TMR_THRESHOLD(port->id),\n+\t\t   DLB_CHP_DIR_CQ_TMR_THRESHOLD_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_CHP_DIR_CQ_INT_ENB(port->id),\n+\t\t   DLB_CHP_DIR_CQ_INT_ENB_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_SYS_DIR_CQ2VF_PF(port->id),\n+\t\t   DLB_SYS_DIR_CQ2VF_PF_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_SYS_DIR_PP2VF_PF(port->id),\n+\t\t   DLB_SYS_DIR_PP2VF_PF_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_SYS_DIR_CQ_ADDR_L(port->id),\n+\t\t   DLB_SYS_DIR_CQ_ADDR_L_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_SYS_DIR_CQ_ADDR_U(port->id),\n+\t\t   DLB_SYS_DIR_CQ_ADDR_U_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_SYS_DIR_PP_ADDR_L(port->id),\n+\t\t   DLB_SYS_DIR_PP_ADDR_L_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_SYS_DIR_PP_ADDR_U(port->id),\n+\t\t   DLB_SYS_DIR_PP_ADDR_U_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_SYS_DIR_PP_V(port->id),\n+\t\t   DLB_SYS_DIR_PP_V_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_SYS_DIR_PP2VAS(port->id),\n+\t\t   DLB_SYS_DIR_PP2VAS_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_SYS_DIR_CQ_ISR(port->id),\n+\t\t   DLB_SYS_DIR_CQ_ISR_RST);\n+\n+\tDLB_CSR_WR(hw,\n+\t\t   DLB_SYS_WBUF_DIR_FLAGS(port->id),\n+\t\t   DLB_SYS_WBUF_DIR_FLAGS_RST);\n+}\n+\n+static void dlb_domain_reset_dir_port_registers(struct dlb_hw *hw,\n+\t\t\t\t\t\tstruct dlb_domain *domain)\n+{\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tstruct dlb_dir_pq_pair *port;\n+\n+\tDLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)\n+\t\t__dlb_domain_reset_dir_port_registers(hw, port);\n+}\n+\n+static void dlb_domain_reset_ldb_queue_registers(struct dlb_hw *hw,\n+\t\t\t\t\t\t struct dlb_domain *domain)\n+{\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tstruct dlb_ldb_queue *queue;\n+\n+\tDLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {\n+\t\tDLB_CSR_WR(hw,\n+\t\t\t   DLB_AQED_PIPE_FL_LIM(queue->id),\n+\t\t\t   DLB_AQED_PIPE_FL_LIM_RST);\n+\n+\t\tDLB_CSR_WR(hw,\n+\t\t\t   DLB_AQED_PIPE_FL_BASE(queue->id),\n+\t\t\t   DLB_AQED_PIPE_FL_BASE_RST);\n+\n+\t\tDLB_CSR_WR(hw,\n+\t\t\t   DLB_AQED_PIPE_FL_POP_PTR(queue->id),\n+\t\t\t   DLB_AQED_PIPE_FL_POP_PTR_RST);\n+\n+\t\tDLB_CSR_WR(hw,\n+\t\t\t   DLB_AQED_PIPE_FL_PUSH_PTR(queue->id),\n+\t\t\t   DLB_AQED_PIPE_FL_PUSH_PTR_RST);\n+\n+\t\tDLB_CSR_WR(hw,\n+\t\t\t   DLB_AQED_PIPE_QID_FID_LIM(queue->id),\n+\t\t\t   DLB_AQED_PIPE_QID_FID_LIM_RST);\n+\n+\t\tDLB_CSR_WR(hw,\n+\t\t\t   DLB_LSP_QID_AQED_ACTIVE_LIM(queue->id),\n+\t\t\t   DLB_LSP_QID_AQED_ACTIVE_LIM_RST);\n+\n+\t\tDLB_CSR_WR(hw,\n+\t\t\t   DLB_LSP_QID_LDB_INFL_LIM(queue->id),\n+\t\t\t   DLB_LSP_QID_LDB_INFL_LIM_RST);\n+\n+\t\tDLB_CSR_WR(hw,\n+\t\t\t   DLB_SYS_LDB_QID_V(queue->id),\n+\t\t\t   DLB_SYS_LDB_QID_V_RST);\n+\n+\t\tDLB_CSR_WR(hw,\n+\t\t\t   DLB_SYS_LDB_QID_V(queue->id),\n+\t\t\t   DLB_SYS_LDB_QID_V_RST);\n+\n+\t\tDLB_CSR_WR(hw,\n+\t\t\t   DLB_CHP_ORD_QID_SN(queue->id),\n+\t\t\t   DLB_CHP_ORD_QID_SN_RST);\n+\n+\t\tDLB_CSR_WR(hw,\n+\t\t\t   DLB_CHP_ORD_QID_SN_MAP(queue->id),\n+\t\t\t   DLB_CHP_ORD_QID_SN_MAP_RST);\n+\n+\t\tDLB_CSR_WR(hw,\n+\t\t\t   DLB_RO_PIPE_QID2GRPSLT(queue->id),\n+\t\t\t   DLB_RO_PIPE_QID2GRPSLT_RST);\n+\t}\n+}\n+\n+static void dlb_domain_reset_dir_queue_registers(struct dlb_hw *hw,\n+\t\t\t\t\t\t struct dlb_domain *domain)\n+{\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tstruct dlb_dir_pq_pair *queue;\n+\n+\tDLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, queue, iter) {\n+\t\tDLB_CSR_WR(hw,\n+\t\t\t   DLB_SYS_DIR_QID_V(queue->id),\n+\t\t\t   DLB_SYS_DIR_QID_V_RST);\n+\t}\n+}\n+\n+static void dlb_domain_reset_ldb_pool_registers(struct dlb_hw *hw,\n+\t\t\t\t\t\tstruct dlb_domain *domain)\n+{\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tstruct dlb_credit_pool *pool;\n+\n+\tDLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {\n+\t\tDLB_CSR_WR(hw,\n+\t\t\t   DLB_CHP_LDB_POOL_CRD_LIM(pool->id),\n+\t\t\t   DLB_CHP_LDB_POOL_CRD_LIM_RST);\n+\n+\t\tDLB_CSR_WR(hw,\n+\t\t\t   DLB_CHP_LDB_POOL_CRD_CNT(pool->id),\n+\t\t\t   DLB_CHP_LDB_POOL_CRD_CNT_RST);\n+\n+\t\tDLB_CSR_WR(hw,\n+\t\t\t   DLB_CHP_QED_FL_BASE(pool->id),\n+\t\t\t   DLB_CHP_QED_FL_BASE_RST);\n+\n+\t\tDLB_CSR_WR(hw,\n+\t\t\t   DLB_CHP_QED_FL_LIM(pool->id),\n+\t\t\t   DLB_CHP_QED_FL_LIM_RST);\n+\n+\t\tDLB_CSR_WR(hw,\n+\t\t\t   DLB_CHP_QED_FL_PUSH_PTR(pool->id),\n+\t\t\t   DLB_CHP_QED_FL_PUSH_PTR_RST);\n+\n+\t\tDLB_CSR_WR(hw,\n+\t\t\t   DLB_CHP_QED_FL_POP_PTR(pool->id),\n+\t\t\t   DLB_CHP_QED_FL_POP_PTR_RST);\n+\t}\n+}\n+\n+static void dlb_domain_reset_dir_pool_registers(struct dlb_hw *hw,\n+\t\t\t\t\t\tstruct dlb_domain *domain)\n+{\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tstruct dlb_credit_pool *pool;\n+\n+\tDLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {\n+\t\tDLB_CSR_WR(hw,\n+\t\t\t   DLB_CHP_DIR_POOL_CRD_LIM(pool->id),\n+\t\t\t   DLB_CHP_DIR_POOL_CRD_LIM_RST);\n+\n+\t\tDLB_CSR_WR(hw,\n+\t\t\t   DLB_CHP_DIR_POOL_CRD_CNT(pool->id),\n+\t\t\t   DLB_CHP_DIR_POOL_CRD_CNT_RST);\n+\n+\t\tDLB_CSR_WR(hw,\n+\t\t\t   DLB_CHP_DQED_FL_BASE(pool->id),\n+\t\t\t   DLB_CHP_DQED_FL_BASE_RST);\n+\n+\t\tDLB_CSR_WR(hw,\n+\t\t\t   DLB_CHP_DQED_FL_LIM(pool->id),\n+\t\t\t   DLB_CHP_DQED_FL_LIM_RST);\n+\n+\t\tDLB_CSR_WR(hw,\n+\t\t\t   DLB_CHP_DQED_FL_PUSH_PTR(pool->id),\n+\t\t\t   DLB_CHP_DQED_FL_PUSH_PTR_RST);\n+\n+\t\tDLB_CSR_WR(hw,\n+\t\t\t   DLB_CHP_DQED_FL_POP_PTR(pool->id),\n+\t\t\t   DLB_CHP_DQED_FL_POP_PTR_RST);\n+\t}\n+}\n+\n+static void dlb_domain_reset_ldb_port_registers(struct dlb_hw *hw,\n+\t\t\t\t\t\tstruct dlb_domain *domain)\n+{\n+\tstruct dlb_list_entry *iter;\n+\tRTE_SET_USED(iter);\n+\tstruct dlb_ldb_port *port;\n+\n+\tDLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)\n+\t\t__dlb_domain_reset_ldb_port_registers(hw, port);\n+}\n+\n+static void dlb_domain_reset_registers(struct dlb_hw *hw,\n+\t\t\t\t       struct dlb_domain *domain)\n+{\n+\tdlb_domain_reset_ldb_port_registers(hw, domain);\n+\n+\tdlb_domain_reset_dir_port_registers(hw, domain);\n+\n+\tdlb_domain_reset_ldb_queue_registers(hw, domain);\n+\n+\tdlb_domain_reset_dir_queue_registers(hw, domain);\n+\n+\tdlb_domain_reset_ldb_pool_registers(hw, domain);\n+\n+\tdlb_domain_reset_dir_pool_registers(hw, domain);\n+}\n+\n+static int dlb_domain_reset_software_state(struct dlb_hw *hw,\n+\t\t\t\t\t   struct dlb_domain *domain)\n+{\n+\tstruct dlb_ldb_queue *tmp_ldb_queue;\n+\tRTE_SET_USED(tmp_ldb_queue);\n+\tstruct dlb_dir_pq_pair *tmp_dir_port;\n+\tRTE_SET_USED(tmp_dir_port);\n+\tstruct dlb_ldb_port *tmp_ldb_port;\n+\tRTE_SET_USED(tmp_ldb_port);\n+\tstruct dlb_credit_pool *tmp_pool;\n+\tRTE_SET_USED(tmp_pool);\n+\tstruct dlb_list_entry *iter1;\n+\tRTE_SET_USED(iter1);\n+\tstruct dlb_list_entry *iter2;\n+\tRTE_SET_USED(iter2);\n+\tstruct dlb_ldb_queue *ldb_queue;\n+\tstruct dlb_dir_pq_pair *dir_port;\n+\tstruct dlb_ldb_port *ldb_port;\n+\tstruct dlb_credit_pool *pool;\n+\n+\tstruct dlb_function_resources *rsrcs;\n+\tstruct dlb_list_head *list;\n+\tint ret;\n+\n+\trsrcs = domain->parent_func;\n+\n+\t/* Move the domain's ldb queues to the function's avail list */\n+\tlist = &domain->used_ldb_queues;\n+\tDLB_DOM_LIST_FOR_SAFE(*list, ldb_queue, tmp_ldb_queue, iter1, iter2) {\n+\t\tif (ldb_queue->sn_cfg_valid) {\n+\t\t\tstruct dlb_sn_group *grp;\n+\n+\t\t\tgrp = &hw->rsrcs.sn_groups[ldb_queue->sn_group];\n+\n+\t\t\tdlb_sn_group_free_slot(grp, ldb_queue->sn_slot);\n+\t\t\tldb_queue->sn_cfg_valid = false;\n+\t\t}\n+\n+\t\tldb_queue->owned = false;\n+\t\tldb_queue->num_mappings = 0;\n+\t\tldb_queue->num_pending_additions = 0;\n+\n+\t\tdlb_list_del(&domain->used_ldb_queues, &ldb_queue->domain_list);\n+\t\tdlb_list_add(&rsrcs->avail_ldb_queues, &ldb_queue->func_list);\n+\t\trsrcs->num_avail_ldb_queues++;\n+\t}\n+\n+\tlist = &domain->avail_ldb_queues;\n+\tDLB_DOM_LIST_FOR_SAFE(*list, ldb_queue, tmp_ldb_queue, iter1, iter2) {\n+\t\tldb_queue->owned = false;\n+\n+\t\tdlb_list_del(&domain->avail_ldb_queues,\n+\t\t\t     &ldb_queue->domain_list);\n+\t\tdlb_list_add(&rsrcs->avail_ldb_queues,\n+\t\t\t     &ldb_queue->func_list);\n+\t\trsrcs->num_avail_ldb_queues++;\n+\t}\n+\n+\t/* Move the domain's ldb ports to the function's avail list */\n+\tlist = &domain->used_ldb_ports;\n+\tDLB_DOM_LIST_FOR_SAFE(*list, ldb_port, tmp_ldb_port, iter1, iter2) {\n+\t\tint i;\n+\n+\t\tldb_port->owned = false;\n+\t\tldb_port->configured = false;\n+\t\tldb_port->num_pending_removals = 0;\n+\t\tldb_port->num_mappings = 0;\n+\t\tfor (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++)\n+\t\t\tldb_port->qid_map[i].state = DLB_QUEUE_UNMAPPED;\n+\n+\t\tdlb_list_del(&domain->used_ldb_ports, &ldb_port->domain_list);\n+\t\tdlb_list_add(&rsrcs->avail_ldb_ports, &ldb_port->func_list);\n+\t\trsrcs->num_avail_ldb_ports++;\n+\t}\n+\n+\tlist = &domain->avail_ldb_ports;\n+\tDLB_DOM_LIST_FOR_SAFE(*list, ldb_port, tmp_ldb_port, iter1, iter2) {\n+\t\tldb_port->owned = false;\n+\n+\t\tdlb_list_del(&domain->avail_ldb_ports, &ldb_port->domain_list);\n+\t\tdlb_list_add(&rsrcs->avail_ldb_ports, &ldb_port->func_list);\n+\t\trsrcs->num_avail_ldb_ports++;\n+\t}\n+\n+\t/* Move the domain's dir ports to the function's avail list */\n+\tlist = &domain->used_dir_pq_pairs;\n+\tDLB_DOM_LIST_FOR_SAFE(*list, dir_port, tmp_dir_port, iter1, iter2) {\n+\t\tdir_port->owned = false;\n+\t\tdir_port->port_configured = false;\n+\n+\t\tdlb_list_del(&domain->used_dir_pq_pairs,\n+\t\t\t     &dir_port->domain_list);\n+\n+\t\tdlb_list_add(&rsrcs->avail_dir_pq_pairs,\n+\t\t\t     &dir_port->func_list);\n+\t\trsrcs->num_avail_dir_pq_pairs++;\n+\t}\n+\n+\tlist = &domain->avail_dir_pq_pairs;\n+\tDLB_DOM_LIST_FOR_SAFE(*list, dir_port, tmp_dir_port, iter1, iter2) {\n+\t\tdir_port->owned = false;\n+\n+\t\tdlb_list_del(&domain->avail_dir_pq_pairs,\n+\t\t\t     &dir_port->domain_list);\n+\n+\t\tdlb_list_add(&rsrcs->avail_dir_pq_pairs,\n+\t\t\t     &dir_port->func_list);\n+\t\trsrcs->num_avail_dir_pq_pairs++;\n+\t}\n+\n+\t/* Return hist list entries to the function */\n+\tret = dlb_bitmap_set_range(rsrcs->avail_hist_list_entries,\n+\t\t\t\t   domain->hist_list_entry_base,\n+\t\t\t\t   domain->total_hist_list_entries);\n+\tif (ret) {\n+\t\tDLB_HW_ERR(hw,\n+\t\t\t   \"[%s()] Internal error: domain hist list base does not match the function's bitmap.\\n\",\n+\t\t\t   __func__);\n+\t\treturn -EFAULT;\n+\t}\n+\n+\tdomain->total_hist_list_entries = 0;\n+\tdomain->avail_hist_list_entries = 0;\n+\tdomain->hist_list_entry_base = 0;\n+\tdomain->hist_list_entry_offset = 0;\n+\n+\t/* Return QED entries to the function */\n+\tret = dlb_bitmap_set_range(rsrcs->avail_qed_freelist_entries,\n+\t\t\t\t   domain->qed_freelist.base,\n+\t\t\t\t   (domain->qed_freelist.bound -\n+\t\t\t\t\tdomain->qed_freelist.base));\n+\tif (ret) {\n+\t\tDLB_HW_ERR(hw,\n+\t\t\t   \"[%s()] Internal error: domain QED base does not match the function's bitmap.\\n\",\n+\t\t\t   __func__);\n+\t\treturn -EFAULT;\n+\t}\n+\n+\tdomain->qed_freelist.base = 0;\n+\tdomain->qed_freelist.bound = 0;\n+\tdomain->qed_freelist.offset = 0;\n+\n+\t/* Return DQED entries back to the function */\n+\tret = dlb_bitmap_set_range(rsrcs->avail_dqed_freelist_entries,\n+\t\t\t\t   domain->dqed_freelist.base,\n+\t\t\t\t   (domain->dqed_freelist.bound -\n+\t\t\t\t\tdomain->dqed_freelist.base));\n+\tif (ret) {\n+\t\tDLB_HW_ERR(hw,\n+\t\t\t   \"[%s()] Internal error: domain DQED base does not match the function's bitmap.\\n\",\n+\t\t\t   __func__);\n+\t\treturn -EFAULT;\n+\t}\n+\n+\tdomain->dqed_freelist.base = 0;\n+\tdomain->dqed_freelist.bound = 0;\n+\tdomain->dqed_freelist.offset = 0;\n+\n+\t/* Return AQED entries back to the function */\n+\tret = dlb_bitmap_set_range(rsrcs->avail_aqed_freelist_entries,\n+\t\t\t\t   domain->aqed_freelist.base,\n+\t\t\t\t   (domain->aqed_freelist.bound -\n+\t\t\t\t\tdomain->aqed_freelist.base));\n+\tif (ret) {\n+\t\tDLB_HW_ERR(hw,\n+\t\t\t   \"[%s()] Internal error: domain AQED base does not match the function's bitmap.\\n\",\n+\t\t\t   __func__);\n+\t\treturn -EFAULT;\n+\t}\n+\n+\tdomain->aqed_freelist.base = 0;\n+\tdomain->aqed_freelist.bound = 0;\n+\tdomain->aqed_freelist.offset = 0;\n+\n+\t/* Return ldb credit pools back to the function's avail list */\n+\tlist = &domain->used_ldb_credit_pools;\n+\tDLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {\n+\t\tpool->owned = false;\n+\t\tpool->configured = false;\n+\n+\t\tdlb_list_del(&domain->used_ldb_credit_pools,\n+\t\t\t     &pool->domain_list);\n+\t\tdlb_list_add(&rsrcs->avail_ldb_credit_pools,\n+\t\t\t     &pool->func_list);\n+\t\trsrcs->num_avail_ldb_credit_pools++;\n+\t}\n+\n+\tlist = &domain->avail_ldb_credit_pools;\n+\tDLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {\n+\t\tpool->owned = false;\n+\n+\t\tdlb_list_del(&domain->avail_ldb_credit_pools,\n+\t\t\t     &pool->domain_list);\n+\t\tdlb_list_add(&rsrcs->avail_ldb_credit_pools,\n+\t\t\t     &pool->func_list);\n+\t\trsrcs->num_avail_ldb_credit_pools++;\n+\t}\n+\n+\t/* Move dir credit pools back to the function */\n+\tlist = &domain->used_dir_credit_pools;\n+\tDLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {\n+\t\tpool->owned = false;\n+\t\tpool->configured = false;\n+\n+\t\tdlb_list_del(&domain->used_dir_credit_pools,\n+\t\t\t     &pool->domain_list);\n+\t\tdlb_list_add(&rsrcs->avail_dir_credit_pools,\n+\t\t\t     &pool->func_list);\n+\t\trsrcs->num_avail_dir_credit_pools++;\n+\t}\n+\n+\tlist = &domain->avail_dir_credit_pools;\n+\tDLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {\n+\t\tpool->owned = false;\n+\n+\t\tdlb_list_del(&domain->avail_dir_credit_pools,\n+\t\t\t     &pool->domain_list);\n+\t\tdlb_list_add(&rsrcs->avail_dir_credit_pools,\n+\t\t\t     &pool->func_list);\n+\t\trsrcs->num_avail_dir_credit_pools++;\n+\t}\n+\n+\tdomain->num_pending_removals = 0;\n+\tdomain->num_pending_additions = 0;\n+\tdomain->configured = false;\n+\tdomain->started = false;\n+\n+\t/* Move the domain out of the used_domains list and back to the\n+\t * function's avail_domains list.\n+\t */\n+\tdlb_list_del(&rsrcs->used_domains, &domain->func_list);\n+\tdlb_list_add(&rsrcs->avail_domains, &domain->func_list);\n+\trsrcs->num_avail_domains++;\n+\n+\treturn 0;\n+}\n+\n+static void dlb_log_reset_domain(struct dlb_hw *hw, u32 domain_id)\n+{\n+\tDLB_HW_INFO(hw, \"DLB reset domain:\\n\");\n+\tDLB_HW_INFO(hw, \"\\tDomain ID: %d\\n\", domain_id);\n+}\n+\n+/**\n+ * dlb_reset_domain() - Reset a DLB scheduling domain and its associated\n+ *\thardware resources.\n+ * @hw:\t  Contains the current state of the DLB hardware.\n+ * @args: User-provided arguments.\n+ * @resp: Response to user.\n+ *\n+ * Note: User software *must* stop sending to this domain's producer ports\n+ * before invoking this function, otherwise undefined behavior will result.\n+ *\n+ * Return: returns < 0 on error, 0 otherwise.\n+ */\n+int dlb_reset_domain(struct dlb_hw *hw, u32 domain_id)\n+{\n+\tstruct dlb_domain *domain;\n+\tint ret;\n+\n+\tdlb_log_reset_domain(hw, domain_id);\n+\n+\tdomain = dlb_get_domain_from_id(hw, domain_id);\n+\n+\tif (!domain || !domain->configured)\n+\t\treturn -EINVAL;\n+\n+\t/* For each queue owned by this domain, disable its write permissions to\n+\t * cause any traffic sent to it to be dropped. Well-behaved software\n+\t * should not be sending QEs at this point.\n+\t */\n+\tdlb_domain_disable_dir_queue_write_perms(hw, domain);\n+\n+\tdlb_domain_disable_ldb_queue_write_perms(hw, domain);\n+\n+\t/* Disable credit updates and turn off completion tracking on all the\n+\t * domain's PPs.\n+\t */\n+\tdlb_domain_disable_dir_port_crd_updates(hw, domain);\n+\n+\tdlb_domain_disable_ldb_port_crd_updates(hw, domain);\n+\n+\tdlb_domain_disable_dir_port_interrupts(hw, domain);\n+\n+\tdlb_domain_disable_ldb_port_interrupts(hw, domain);\n+\n+\tdlb_domain_disable_ldb_seq_checks(hw, domain);\n+\n+\t/* Disable the LDB CQs and drain them in order to complete the map and\n+\t * unmap procedures, which require zero CQ inflights and zero QID\n+\t * inflights respectively.\n+\t */\n+\tdlb_domain_disable_ldb_cqs(hw, domain);\n+\n+\tret = dlb_domain_drain_ldb_cqs(hw, domain, false);\n+\tif (ret < 0)\n+\t\treturn ret;\n+\n+\tret = dlb_domain_wait_for_ldb_cqs_to_empty(hw, domain);\n+\tif (ret < 0)\n+\t\treturn ret;\n+\n+\tret = dlb_domain_finish_unmap_qid_procedures(hw, domain);\n+\tif (ret < 0)\n+\t\treturn ret;\n+\n+\tret = dlb_domain_finish_map_qid_procedures(hw, domain);\n+\tif (ret < 0)\n+\t\treturn ret;\n+\n+\t/* Re-enable the CQs in order to drain the mapped queues. */\n+\tdlb_domain_enable_ldb_cqs(hw, domain);\n+\n+\tret = dlb_domain_drain_mapped_queues(hw, domain);\n+\tif (ret < 0)\n+\t\treturn ret;\n+\n+\tret = dlb_domain_drain_unmapped_queues(hw, domain);\n+\tif (ret < 0)\n+\t\treturn ret;\n+\n+\tret = dlb_domain_wait_for_ldb_pool_refill(hw, domain);\n+\tif (ret) {\n+\t\tDLB_HW_ERR(hw,\n+\t\t\t   \"[%s()] Internal error: LDB credits failed to refill\\n\",\n+\t\t\t   __func__);\n+\t\treturn ret;\n+\t}\n+\n+\t/* Done draining LDB QEs, so disable the CQs. */\n+\tdlb_domain_disable_ldb_cqs(hw, domain);\n+\n+\t/* Directed queues are reset in dlb_domain_reset_hw_resources(), but\n+\t * that process does not decrement the directed queue size counters used\n+\t * by SMON for its average DQED depth measurement. So, we manually drain\n+\t * the directed queues here.\n+\t */\n+\tdlb_domain_drain_dir_queues(hw, domain);\n+\n+\tret = dlb_domain_wait_for_dir_pool_refill(hw, domain);\n+\tif (ret) {\n+\t\tDLB_HW_ERR(hw,\n+\t\t\t   \"[%s()] Internal error: DIR credits failed to refill\\n\",\n+\t\t\t   __func__);\n+\t\treturn ret;\n+\t}\n+\n+\t/* Done draining DIR QEs, so disable the CQs. */\n+\tdlb_domain_disable_dir_cqs(hw, domain);\n+\n+\tdlb_domain_disable_dir_producer_ports(hw, domain);\n+\n+\tdlb_domain_disable_ldb_producer_ports(hw, domain);\n+\n+\tdlb_domain_disable_dir_pools(hw, domain);\n+\n+\tdlb_domain_disable_ldb_pools(hw, domain);\n+\n+\t/* Reset the QID, credit pool, and CQ hardware.\n+\t *\n+\t * Note: DLB 1.0 A0 h/w does not disarm CQ interrupts during sched\n+\t * domain reset.\n+\t * A spurious interrupt can occur on subsequent use of a reset CQ.\n+\t */\n+\tret = dlb_domain_reset_hw_resources(hw, domain);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\tret = dlb_domain_verify_reset_success(hw, domain);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\tdlb_domain_reset_registers(hw, domain);\n+\n+\t/* Hardware reset complete. Reset the domain's software state */\n+\tret = dlb_domain_reset_software_state(hw, domain);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\treturn 0;\n+}\n+\n+void dlb_hw_get_num_resources(struct dlb_hw *hw,\n+\t\t\t      struct dlb_get_num_resources_args *arg)\n+{\n+\tstruct dlb_function_resources *rsrcs;\n+\tstruct dlb_bitmap *map;\n+\n+\trsrcs = &hw->pf;\n+\n+\targ->num_sched_domains = rsrcs->num_avail_domains;\n+\n+\targ->num_ldb_queues = rsrcs->num_avail_ldb_queues;\n+\n+\targ->num_ldb_ports = rsrcs->num_avail_ldb_ports;\n+\n+\targ->num_dir_ports = rsrcs->num_avail_dir_pq_pairs;\n+\n+\tmap = rsrcs->avail_aqed_freelist_entries;\n+\n+\targ->num_atomic_inflights = dlb_bitmap_count(map);\n+\n+\targ->max_contiguous_atomic_inflights =\n+\t\tdlb_bitmap_longest_set_range(map);\n+\n+\tmap = rsrcs->avail_hist_list_entries;\n+\n+\targ->num_hist_list_entries = dlb_bitmap_count(map);\n+\n+\targ->max_contiguous_hist_list_entries =\n+\t\tdlb_bitmap_longest_set_range(map);\n+\n+\tmap = rsrcs->avail_qed_freelist_entries;\n+\n+\targ->num_ldb_credits = dlb_bitmap_count(map);\n+\n+\targ->max_contiguous_ldb_credits = dlb_bitmap_longest_set_range(map);\n+\n+\tmap = rsrcs->avail_dqed_freelist_entries;\n+\n+\targ->num_dir_credits = dlb_bitmap_count(map);\n+\n+\targ->max_contiguous_dir_credits = dlb_bitmap_longest_set_range(map);\n+\n+\targ->num_ldb_credit_pools = rsrcs->num_avail_ldb_credit_pools;\n+\n+\targ->num_dir_credit_pools = rsrcs->num_avail_dir_credit_pools;\n+}\n+\n void dlb_hw_disable_vf_to_pf_isr_pend_err(struct dlb_hw *hw)\n {\n \tunion dlb_sys_sys_alarm_int_enable r0;\ndiff --git a/drivers/event/dlb/pf/dlb_pf.c b/drivers/event/dlb/pf/dlb_pf.c\nindex 7fc85e9..57a150c 100644\n--- a/drivers/event/dlb/pf/dlb_pf.c\n+++ b/drivers/event/dlb/pf/dlb_pf.c\n@@ -78,6 +78,17 @@ dlb_pf_open(struct dlb_hw_dev *handle, const char *name)\n \treturn 0;\n }\n \n+static void\n+dlb_pf_domain_close(struct dlb_eventdev *dlb)\n+{\n+\tstruct dlb_dev *dlb_dev = (struct dlb_dev *)dlb->qm_instance.pf_dev;\n+\tint ret;\n+\n+\tret = dlb_reset_domain(&dlb_dev->hw, dlb->qm_instance.domain_id);\n+\tif (ret)\n+\t\tDLB_LOG_ERR(\"dlb_pf_reset_domain err %d\", ret);\n+}\n+\n static int\n dlb_pf_get_device_version(struct dlb_hw_dev *handle,\n \t\t\t  uint8_t *revision)\n@@ -101,6 +112,79 @@ dlb_pf_get_num_resources(struct dlb_hw_dev *handle,\n }\n \n static int\n+dlb_pf_sched_domain_create(struct dlb_hw_dev *handle,\n+\t\t\t   struct dlb_create_sched_domain_args *arg)\n+{\n+\tstruct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;\n+\tstruct dlb_cmd_response response = {0};\n+\tint ret;\n+\n+\tDLB_INFO(dev->dlb_device, \"Entering %s()\\n\", __func__);\n+\n+\tif (dlb_dev->domain_reset_failed) {\n+\t\tresponse.status = DLB_ST_DOMAIN_RESET_FAILED;\n+\t\tret = -EINVAL;\n+\t\tgoto done;\n+\t}\n+\n+\tret = dlb_hw_create_sched_domain(&dlb_dev->hw, arg, &response);\n+\tif (ret)\n+\t\tgoto done;\n+\n+done:\n+\n+\t*(struct dlb_cmd_response *)arg->response = response;\n+\n+\tDLB_INFO(dev->dlb_device, \"Exiting %s() with ret=%d\\n\", __func__, ret);\n+\n+\treturn ret;\n+}\n+\n+static int\n+dlb_pf_ldb_credit_pool_create(struct dlb_hw_dev *handle,\n+\t\t\t      struct dlb_create_ldb_pool_args *cfg)\n+{\n+\tstruct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;\n+\tstruct dlb_cmd_response response = {0};\n+\tint ret;\n+\n+\tDLB_INFO(dev->dlb_device, \"Entering %s()\\n\", __func__);\n+\n+\tret = dlb_hw_create_ldb_pool(&dlb_dev->hw,\n+\t\t\t\t     handle->domain_id,\n+\t\t\t\t     cfg,\n+\t\t\t\t     &response);\n+\n+\t*(struct dlb_cmd_response *)cfg->response = response;\n+\n+\tDLB_INFO(dev->dlb_device, \"Exiting %s() with ret=%d\\n\", __func__, ret);\n+\n+\treturn ret;\n+}\n+\n+static int\n+dlb_pf_dir_credit_pool_create(struct dlb_hw_dev *handle,\n+\t\t\t      struct dlb_create_dir_pool_args *cfg)\n+{\n+\tstruct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;\n+\tstruct dlb_cmd_response response = {0};\n+\tint ret;\n+\n+\tDLB_INFO(dev->dlb_device, \"Entering %s()\\n\", __func__);\n+\n+\tret = dlb_hw_create_dir_pool(&dlb_dev->hw,\n+\t\t\t\t     handle->domain_id,\n+\t\t\t\t     cfg,\n+\t\t\t\t     &response);\n+\n+\t*(struct dlb_cmd_response *)cfg->response = response;\n+\n+\tDLB_INFO(dev->dlb_device, \"Exiting %s() with ret=%d\\n\", __func__, ret);\n+\n+\treturn ret;\n+}\n+\n+static int\n dlb_pf_get_cq_poll_mode(struct dlb_hw_dev *handle,\n \t\t\tenum dlb_cq_poll_modes *mode)\n {\n@@ -119,8 +203,12 @@ dlb_pf_iface_fn_ptrs_init(void)\n {\n \tdlb_iface_low_level_io_init = dlb_pf_low_level_io_init;\n \tdlb_iface_open = dlb_pf_open;\n+\tdlb_iface_domain_close = dlb_pf_domain_close;\n \tdlb_iface_get_device_version = dlb_pf_get_device_version;\n \tdlb_iface_get_num_resources = dlb_pf_get_num_resources;\n+\tdlb_iface_sched_domain_create = dlb_pf_sched_domain_create;\n+\tdlb_iface_ldb_credit_pool_create = dlb_pf_ldb_credit_pool_create;\n+\tdlb_iface_dir_credit_pool_create = dlb_pf_dir_credit_pool_create;\n \tdlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;\n }\n \n",
    "prefixes": [
        "v4",
        "08/22"
    ]
}