Patch Detail

HTTP 200 OK
Allow: GET, PUT, PATCH, HEAD, OPTIONS
Content-Type: application/json
Vary: Accept

{
    "id": 75070,
    "url": "http://patches.dpdk.org/api/patches/75070/?format=api",
    "web_url": "http://patches.dpdk.org/project/dpdk/patch/1596138614-17409-20-git-send-email-timothy.mcdaniel@intel.com/",
    "project": {
        "id": 1,
        "url": "http://patches.dpdk.org/api/projects/1/?format=api",
        "name": "DPDK",
        "link_name": "dpdk",
        "list_id": "dev.dpdk.org",
        "list_email": "dev@dpdk.org",
        "web_url": "http://core.dpdk.org",
        "scm_url": "git://dpdk.org/dpdk",
        "webscm_url": "http://git.dpdk.org/dpdk",
        "list_archive_url": "https://inbox.dpdk.org/dev",
        "list_archive_url_format": "https://inbox.dpdk.org/dev/{}",
        "commit_url_format": ""
    },
    "msgid": "<1596138614-17409-20-git-send-email-timothy.mcdaniel@intel.com>",
    "list_archive_url": "https://inbox.dpdk.org/dev/1596138614-17409-20-git-send-email-timothy.mcdaniel@intel.com",
    "date": "2020-07-30T19:50:06",
    "name": "[19/27] event/dlb: add port_setup",
    "commit_ref": null,
    "pull_url": null,
    "state": "superseded",
    "archived": true,
    "hash": "99835a603a6feada591941505c60df48c34cc3e6",
    "submitter": {
        "id": 826,
        "url": "http://patches.dpdk.org/api/people/826/?format=api",
        "name": "Timothy McDaniel",
        "email": "timothy.mcdaniel@intel.com"
    },
    "delegate": {
        "id": 310,
        "url": "http://patches.dpdk.org/api/users/310/?format=api",
        "username": "jerin",
        "first_name": "Jerin",
        "last_name": "Jacob",
        "email": "jerinj@marvell.com"
    },
    "mbox": "http://patches.dpdk.org/project/dpdk/patch/1596138614-17409-20-git-send-email-timothy.mcdaniel@intel.com/mbox/",
    "series": [
        {
            "id": 11425,
            "url": "http://patches.dpdk.org/api/series/11425/?format=api",
            "web_url": "http://patches.dpdk.org/project/dpdk/list/?series=11425",
            "date": "2020-07-30T19:49:47",
            "name": "Add Intel DLM PMD to 20.11",
            "version": 1,
            "mbox": "http://patches.dpdk.org/series/11425/mbox/"
        }
    ],
    "comments": "http://patches.dpdk.org/api/patches/75070/comments/",
    "check": "fail",
    "checks": "http://patches.dpdk.org/api/patches/75070/checks/",
    "tags": {},
    "related": [],
    "headers": {
        "Return-Path": "<dev-bounces@dpdk.org>",
        "X-Original-To": "patchwork@inbox.dpdk.org",
        "Delivered-To": "patchwork@inbox.dpdk.org",
        "Received": [
            "from dpdk.org (dpdk.org [92.243.14.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 2B6C1A052B;\n\tThu, 30 Jul 2020 21:56:29 +0200 (CEST)",
            "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id AF95D1C0CE;\n\tThu, 30 Jul 2020 21:53:40 +0200 (CEST)",
            "from mga02.intel.com (mga02.intel.com [134.134.136.20])\n by dpdk.org (Postfix) with ESMTP id B7D0A1C0B3\n for <dev@dpdk.org>; Thu, 30 Jul 2020 21:53:20 +0200 (CEST)",
            "from orsmga005.jf.intel.com ([10.7.209.41])\n by orsmga101.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384;\n 30 Jul 2020 12:53:20 -0700",
            "from txasoft-yocto.an.intel.com ([10.123.72.192])\n by orsmga005.jf.intel.com with ESMTP; 30 Jul 2020 12:53:18 -0700"
        ],
        "IronPort-SDR": [
            "\n bV+zkTOzak86/x8JYMSoyMOIhEElMGlgrDLagvihEazn6HBoaMayTryJAFOeDyS3+guxL5BFUC\n L4HttgEsBiyA==",
            "\n I2BoXZeaP5/wcd+i4hLmPALCwY+wVufEvX33lUFnUk0Mb+feTbsBaOwmkCymNmSpH/EqxMxeDl\n q03B00ZUZbJQ=="
        ],
        "X-IronPort-AV": [
            "E=McAfee;i=\"6000,8403,9698\"; a=\"139672338\"",
            "E=Sophos;i=\"5.75,415,1589266800\"; d=\"scan'208\";a=\"139672338\"",
            "E=Sophos;i=\"5.75,415,1589266800\"; d=\"scan'208\";a=\"465378136\""
        ],
        "X-Amp-Result": "SKIPPED(no attachment in message)",
        "X-Amp-File-Uploaded": "False",
        "X-ExtLoop1": "1",
        "From": "\"McDaniel, Timothy\" <timothy.mcdaniel@intel.com>",
        "To": "jerinj@marvell.com",
        "Cc": "mattias.ronnblom@ericsson.com, dev@dpdk.org, gage.eads@intel.com,\n harry.van.haaren@intel.com,\n \"McDaniel, Timothy\" <timothy.mcdaniel@intel.com>",
        "Date": "Thu, 30 Jul 2020 14:50:06 -0500",
        "Message-Id": "<1596138614-17409-20-git-send-email-timothy.mcdaniel@intel.com>",
        "X-Mailer": "git-send-email 1.7.10",
        "In-Reply-To": "<1596138614-17409-1-git-send-email-timothy.mcdaniel@intel.com>",
        "References": "<1593232671-5690-0-git-send-email-timothy.mcdaniel@intel.com>\n <1596138614-17409-1-git-send-email-timothy.mcdaniel@intel.com>",
        "Subject": "[dpdk-dev] [PATCH 19/27] event/dlb: add port_setup",
        "X-BeenThere": "dev@dpdk.org",
        "X-Mailman-Version": "2.1.15",
        "Precedence": "list",
        "List-Id": "DPDK patches and discussions <dev.dpdk.org>",
        "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>",
        "List-Archive": "<http://mails.dpdk.org/archives/dev/>",
        "List-Post": "<mailto:dev@dpdk.org>",
        "List-Help": "<mailto:dev-request@dpdk.org?subject=help>",
        "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>",
        "Errors-To": "dev-bounces@dpdk.org",
        "Sender": "\"dev\" <dev-bounces@dpdk.org>"
    },
    "content": "From: \"McDaniel, Timothy\" <timothy.mcdaniel@intel.com>\n\nSigned-off-by: McDaniel, Timothy <timothy.mcdaniel@intel.com>\n---\n drivers/event/dlb/dlb.c | 1241 +++++++++++++++++++++++++++++++++++++++++++++++\n 1 file changed, 1241 insertions(+)",
    "diff": "diff --git a/drivers/event/dlb/dlb.c b/drivers/event/dlb/dlb.c\nindex ae1d19b..8bbcd03 100644\n--- a/drivers/event/dlb/dlb.c\n+++ b/drivers/event/dlb/dlb.c\n@@ -71,6 +71,29 @@\n /* These functions will vary based on processor capabilities */\n static struct dlb_port_low_level_io_functions qm_mmio_fns;\n \n+struct process_local_port_data\n+dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];\n+\n+static __rte_always_inline uint16_t\n+dlb_read_pc(struct process_local_port_data *port_data, bool ldb)\n+{\n+\tvolatile uint16_t *popcount;\n+\n+\tif (ldb)\n+\t\tpopcount = port_data->ldb_popcount;\n+\telse\n+\t\tpopcount = port_data->dir_popcount;\n+\n+\treturn *popcount;\n+}\n+\n+static __rte_always_inline void\n+dlb_pp_write(struct dlb_enqueue_qe *qe4,\n+\t     struct process_local_port_data *port_data)\n+{\n+\tqm_mmio_fns.pp_enqueue_four(port_data->pp_addr, qe4);\n+}\n+\n static int\n dlb_hw_query_resources(struct dlb_eventdev *dlb)\n {\n@@ -169,6 +192,46 @@ int dlb_string_to_int(int *result, const char *str)\n \treturn 0;\n }\n \n+static inline int\n+dlb_consume_qe_immediate(struct dlb_port *qm_port, int num)\n+{\n+\tstruct process_local_port_data *port_data;\n+\tstruct dlb_cq_pop_qe *qe;\n+\n+\tRTE_ASSERT(qm_port->config_state == DLB_CONFIGURED);\n+\n+\tif (qm_port->use_rsvd_token_scheme) {\n+\t\t/* Check if there's a deficit of reserved tokens, and return\n+\t\t * early if there are no (unreserved) tokens to consume.\n+\t\t */\n+\t\tif (num <= qm_port->cq_rsvd_token_deficit) {\n+\t\t\tqm_port->cq_rsvd_token_deficit -= num;\n+\t\t\tqm_port->owed_tokens = 0;\n+\t\t\treturn 0;\n+\t\t}\n+\t\tnum -= qm_port->cq_rsvd_token_deficit;\n+\t\tqm_port->cq_rsvd_token_deficit = 0;\n+\t}\n+\n+\tqe = qm_port->consume_qe;\n+\n+\tqe->tokens = num - 1;\n+\tqe->int_arm = 0;\n+\n+\t/* No store fence needed since no pointer is being sent, and CQ token\n+\t * pops can be safely reordered with other HCWs.\n+\t */\n+\tport_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];\n+\n+\tdlb_movntdq_single(port_data->pp_addr, qe);\n+\n+\tDLB_LOG_DBG(\"dlb: consume immediate - %d QEs\\n\", num);\n+\n+\tqm_port->owed_tokens = 0;\n+\n+\treturn 0;\n+}\n+\n int\n set_max_num_events(const char *key __rte_unused,\n \t\t   const char *value,\n@@ -221,6 +284,449 @@ int dlb_string_to_int(int *result, const char *str)\n \treturn 0;\n }\n \n+static inline uint16_t\n+dlb_event_enqueue_delayed(void *event_port,\n+\t\t\t  const struct rte_event events[]);\n+\n+static inline uint16_t\n+dlb_event_enqueue_burst_delayed(void *event_port,\n+\t\t\t\tconst struct rte_event events[],\n+\t\t\t\tuint16_t num);\n+\n+static inline uint16_t\n+dlb_event_enqueue_new_burst_delayed(void *event_port,\n+\t\t\t\t    const struct rte_event events[],\n+\t\t\t\t    uint16_t num);\n+\n+static inline uint16_t\n+dlb_event_enqueue_forward_burst_delayed(void *event_port,\n+\t\t\t\t\tconst struct rte_event events[],\n+\t\t\t\t\tuint16_t num);\n+\n+int\n+dlb_init_qe_mem(struct dlb_port *qm_port, char *mz_name)\n+{\n+\tint ret, sz;\n+\n+\tsz = DLB_NUM_QES_PER_CACHE_LINE * sizeof(struct dlb_enqueue_qe);\n+\n+\tqm_port->qe4 = rte_malloc(mz_name, sz, RTE_CACHE_LINE_SIZE);\n+\n+\tif (qm_port->qe4 == NULL) {\n+\t\tDLB_LOG_ERR(\"dlb: no qe4 memory\\n\");\n+\t\tret = -ENOMEM;\n+\t\tgoto error_exit;\n+\t}\n+\n+\tmemset(qm_port->qe4, 0, sz);\n+\n+\tret = dlb_init_consume_qe(qm_port, mz_name);\n+\tif (ret < 0) {\n+\t\tDLB_LOG_ERR(\"dlb: dlb_init_consume_qe ret=%d\\n\",\n+\t\t\t    ret);\n+\t\tgoto error_exit;\n+\t}\n+\n+\treturn 0;\n+\n+error_exit:\n+\n+\tdlb_free_qe_mem(qm_port);\n+\n+\treturn ret;\n+}\n+\n+int\n+dlb_init_consume_qe(struct dlb_port *qm_port, char *mz_name)\n+{\n+\tstruct dlb_cq_pop_qe *qe;\n+\n+\tqe = rte_malloc(mz_name,\n+\t\t\tDLB_NUM_QES_PER_CACHE_LINE *\n+\t\t\t\tsizeof(struct dlb_cq_pop_qe),\n+\t\t\tRTE_CACHE_LINE_SIZE);\n+\n+\tif (qe == NULL)\t{\n+\t\tDLB_LOG_ERR(\"dlb: no memory for consume_qe\\n\");\n+\t\treturn -ENOMEM;\n+\t}\n+\n+\tqm_port->consume_qe = qe;\n+\n+\tmemset(qe, 0, DLB_NUM_QES_PER_CACHE_LINE *\n+\t       sizeof(struct dlb_cq_pop_qe));\n+\n+\tqe->qe_valid = 0;\n+\tqe->qe_frag = 0;\n+\tqe->qe_comp = 0;\n+\tqe->cq_token = 1;\n+\t/* Tokens value is 0-based; i.e. '0' returns 1 token, '1' returns 2,\n+\t * and so on.\n+\t */\n+\tqe->tokens = 0;\t/* set at run time */\n+\tqe->meas_lat = 0;\n+\tqe->no_dec = 0;\n+\t/* Completion IDs are disabled */\n+\tqe->cmp_id = 0;\n+\n+\treturn 0;\n+}\n+\n+static int\n+dlb_hw_create_ldb_port(struct dlb_eventdev *dlb,\n+\t\t       struct dlb_eventdev_port *ev_port,\n+\t\t       uint32_t dequeue_depth,\n+\t\t       uint32_t cq_depth,\n+\t\t       uint32_t enqueue_depth,\n+\t\t       uint16_t rsvd_tokens,\n+\t\t       bool use_rsvd_token_scheme)\n+{\n+\tstruct dlb_hw_dev *handle = &dlb->qm_instance;\n+\tstruct dlb_create_ldb_port_args cfg = {0};\n+\tstruct dlb_cmd_response response = {0};\n+\tint ret;\n+\tstruct dlb_port *qm_port = NULL;\n+\tchar mz_name[RTE_MEMZONE_NAMESIZE];\n+\tuint32_t qm_port_id;\n+\n+\tif (handle == NULL)\n+\t\treturn -EINVAL;\n+\n+\tif (cq_depth < DLB_MIN_LDB_CQ_DEPTH ||\n+\t    cq_depth > DLB_MAX_INPUT_QUEUE_DEPTH) {\n+\t\tDLB_LOG_ERR(\"dlb: invalid cq_depth, must be %d-%d\\n\",\n+\t\t\tDLB_MIN_LDB_CQ_DEPTH, DLB_MAX_INPUT_QUEUE_DEPTH);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\tif (enqueue_depth < DLB_MIN_ENQUEUE_DEPTH) {\n+\t\tDLB_LOG_ERR(\"dlb: invalid enqueue_depth, must be at least %d\\n\",\n+\t\t\t    DLB_MIN_ENQUEUE_DEPTH);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\trte_spinlock_lock(&handle->resource_lock);\n+\n+\tcfg.response = (uintptr_t)&response;\n+\n+\t/* We round up to the next power of 2 if necessary */\n+\tcfg.cq_depth = rte_align32pow2(cq_depth);\n+\tcfg.cq_depth_threshold = rsvd_tokens;\n+\n+\tcfg.cq_history_list_size = DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;\n+\n+\t/* User controls the LDB high watermark via enqueue depth. The DIR high\n+\t * watermark is equal, unless the directed credit pool is too small.\n+\t */\n+\tcfg.ldb_credit_high_watermark = enqueue_depth;\n+\n+\t/* If there are no directed ports, the kernel driver will ignore this\n+\t * port's directed credit settings. Don't use enqueue_depth if it would\n+\t * require more directed credits than are available.\n+\t */\n+\tcfg.dir_credit_high_watermark =\n+\t\tRTE_MIN(enqueue_depth,\n+\t\t\thandle->cfg.num_dir_credits / dlb->num_ports);\n+\n+\tcfg.ldb_credit_quantum = cfg.ldb_credit_high_watermark / 2;\n+\tcfg.ldb_credit_low_watermark = RTE_MIN(16, cfg.ldb_credit_quantum);\n+\n+\tcfg.dir_credit_quantum = cfg.dir_credit_high_watermark / 2;\n+\tcfg.dir_credit_low_watermark = RTE_MIN(16, cfg.dir_credit_quantum);\n+\n+\t/* Per QM values */\n+\n+\tcfg.ldb_credit_pool_id = handle->cfg.ldb_credit_pool_id;\n+\tcfg.dir_credit_pool_id = handle->cfg.dir_credit_pool_id;\n+\n+\tret = dlb_iface_ldb_port_create(handle, &cfg, dlb->poll_mode);\n+\tif (ret < 0) {\n+\t\tDLB_LOG_ERR(\"dlb: dlb_ldb_port_create error, ret=%d (driver status: %s)\\n\",\n+\t\t\t    ret, dlb_error_strings[response.status]);\n+\t\tgoto error_exit;\n+\t}\n+\n+\tqm_port_id = response.id;\n+\n+\tDLB_LOG_DBG(\"dlb: ev_port %d uses qm LB port %d <<<<<\\n\",\n+\t\t    ev_port->id, qm_port_id);\n+\n+\tqm_port = &ev_port->qm_port;\n+\tqm_port->ev_port = ev_port; /* back ptr */\n+\tqm_port->dlb = dlb; /* back ptr */\n+\n+\t/*\n+\t * Allocate and init local qe struct(s).\n+\t * Note: MOVDIR64 requires the enqueue QE (qe4) to be aligned.\n+\t */\n+\n+\tsnprintf(mz_name, sizeof(mz_name), \"%s_ldb_port%d\",\n+\t\t handle->device_name,\n+\t\t ev_port->id);\n+\n+\tret = dlb_init_qe_mem(qm_port, mz_name);\n+\tif (ret < 0) {\n+\t\tDLB_LOG_ERR(\"dlb: init_qe_mem failed, ret=%d\\n\", ret);\n+\t\tgoto error_exit;\n+\t}\n+\n+\tqm_port->pp_mmio_base = DLB_LDB_PP_BASE + PAGE_SIZE * qm_port_id;\n+\tqm_port->id = qm_port_id;\n+\n+\t/* The credit window is one high water mark of QEs */\n+\tqm_port->ldb_pushcount_at_credit_expiry = 0;\n+\tqm_port->cached_ldb_credits = cfg.ldb_credit_high_watermark;\n+\t/* The credit window is one high water mark of QEs */\n+\tqm_port->dir_pushcount_at_credit_expiry = 0;\n+\tqm_port->cached_dir_credits = cfg.dir_credit_high_watermark;\n+\tqm_port->cq_depth = cfg.cq_depth;\n+\t/* CQs with depth < 8 use an 8-entry queue, but withhold credits so\n+\t * the effective depth is smaller.\n+\t */\n+\tqm_port->cq_depth = cfg.cq_depth <= 8 ? 8 : cfg.cq_depth;\n+\tqm_port->cq_idx = 0;\n+\tqm_port->cq_idx_unmasked = 0;\n+\tif (dlb->poll_mode == DLB_CQ_POLL_MODE_SPARSE)\n+\t\tqm_port->cq_depth_mask = (qm_port->cq_depth * 4) - 1;\n+\telse\n+\t\tqm_port->cq_depth_mask = qm_port->cq_depth - 1;\n+\n+\tqm_port->gen_bit_shift = __builtin_popcount(qm_port->cq_depth_mask);\n+\t/* starting value of gen bit - it toggles at wrap time */\n+\tqm_port->gen_bit = 1;\n+\n+\tqm_port->use_rsvd_token_scheme = use_rsvd_token_scheme;\n+\tqm_port->cq_rsvd_token_deficit = rsvd_tokens;\n+\tqm_port->int_armed = false;\n+\n+\t/* Save off for later use in info and lookup APIs. */\n+\tqm_port->qid_mappings = &dlb->qm_ldb_to_ev_queue_id[0];\n+\n+\t/* When using the reserved token scheme, token_pop_thresh is\n+\t * initially 2 * dequeue_depth. Once the tokens are reserved,\n+\t * the enqueue code re-assigns it to dequeue_depth.\n+\t */\n+\tqm_port->dequeue_depth = dequeue_depth;\n+\tqm_port->token_pop_thresh = cq_depth;\n+\n+\t/* When the deferred scheduling vdev arg is selected, use deferred pop\n+\t * for all single-entry CQs.\n+\t */\n+\tif (cfg.cq_depth == 1 || (cfg.cq_depth == 2 && use_rsvd_token_scheme)) {\n+\t\tif (dlb->defer_sched)\n+\t\t\tqm_port->token_pop_mode = DEFERRED_POP;\n+\t}\n+\n+\t/* The default enqueue functions do not include delayed-pop support for\n+\t * performance reasons.\n+\t */\n+\tif (qm_port->token_pop_mode == DELAYED_POP) {\n+\t\tdlb->event_dev->enqueue = dlb_event_enqueue_delayed;\n+\t\tdlb->event_dev->enqueue_burst =\n+\t\t\tdlb_event_enqueue_burst_delayed;\n+\t\tdlb->event_dev->enqueue_new_burst =\n+\t\t\tdlb_event_enqueue_new_burst_delayed;\n+\t\tdlb->event_dev->enqueue_forward_burst =\n+\t\t\tdlb_event_enqueue_forward_burst_delayed;\n+\t}\n+\n+\tqm_port->owed_tokens = 0;\n+\tqm_port->issued_releases = 0;\n+\n+\t/* Save config message too. */\n+\trte_memcpy(&qm_port->cfg.ldb, &cfg, sizeof(cfg));\n+\n+\t/* update state */\n+\tqm_port->state = PORT_STARTED; /* enabled at create time */\n+\tqm_port->config_state = DLB_CONFIGURED;\n+\n+\tqm_port->dir_credits = cfg.dir_credit_high_watermark;\n+\tqm_port->ldb_credits = cfg.ldb_credit_high_watermark;\n+\n+\tDLB_LOG_DBG(\"dlb: created ldb port %d, depth = %d, ldb credits=%d, dir credits=%d\\n\",\n+\t\t    qm_port_id,\n+\t\t    cq_depth,\n+\t\t    qm_port->ldb_credits,\n+\t\t    qm_port->dir_credits);\n+\n+\trte_spinlock_unlock(&handle->resource_lock);\n+\n+\treturn 0;\n+\n+error_exit:\n+\tif (qm_port) {\n+\t\tdlb_free_qe_mem(qm_port);\n+\t\tqm_port->pp_mmio_base = 0;\n+\t}\n+\n+\trte_spinlock_unlock(&handle->resource_lock);\n+\n+\tDLB_LOG_ERR(\"dlb: create ldb port failed!\\n\");\n+\n+\treturn ret;\n+}\n+\n+static int\n+dlb_hw_create_dir_port(struct dlb_eventdev *dlb,\n+\t\t       struct dlb_eventdev_port *ev_port,\n+\t\t       uint32_t dequeue_depth,\n+\t\t       uint32_t cq_depth,\n+\t\t       uint32_t enqueue_depth,\n+\t\t       uint16_t rsvd_tokens,\n+\t\t       bool use_rsvd_token_scheme)\n+{\n+\tstruct dlb_hw_dev *handle = &dlb->qm_instance;\n+\tstruct dlb_create_dir_port_args cfg = {0};\n+\tstruct dlb_cmd_response response = {0};\n+\tint ret;\n+\tstruct dlb_port *qm_port = NULL;\n+\tchar mz_name[RTE_MEMZONE_NAMESIZE];\n+\tuint32_t qm_port_id;\n+\n+\tif (dlb == NULL || handle == NULL)\n+\t\treturn -EINVAL;\n+\n+\tif (cq_depth < DLB_MIN_DIR_CQ_DEPTH ||\n+\t    cq_depth > DLB_MAX_INPUT_QUEUE_DEPTH) {\n+\t\tDLB_LOG_ERR(\"dlb: invalid cq_depth, must be %d-%d\\n\",\n+\t\t\t    DLB_MIN_DIR_CQ_DEPTH, DLB_MAX_INPUT_QUEUE_DEPTH);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\trte_spinlock_lock(&handle->resource_lock);\n+\n+\t/* Directed queues are configured at link time. */\n+\tcfg.queue_id = -1;\n+\n+\tcfg.response = (uintptr_t)&response;\n+\n+\t/* We round up to the next power of 2 if necessary */\n+\tcfg.cq_depth = rte_align32pow2(cq_depth);\n+\tcfg.cq_depth_threshold = rsvd_tokens;\n+\n+\t/* User controls the LDB high watermark via enqueue depth. The DIR high\n+\t * watermark is equal, unless the directed credit pool is too small.\n+\t */\n+\tcfg.ldb_credit_high_watermark = enqueue_depth;\n+\n+\t/* Don't use enqueue_depth if it would require more directed credits\n+\t * than are available.\n+\t */\n+\tcfg.dir_credit_high_watermark =\n+\t\tRTE_MIN(enqueue_depth,\n+\t\t\thandle->cfg.num_dir_credits / dlb->num_ports);\n+\n+\tcfg.ldb_credit_quantum = cfg.ldb_credit_high_watermark / 2;\n+\tcfg.ldb_credit_low_watermark = RTE_MIN(16, cfg.ldb_credit_quantum);\n+\n+\tcfg.dir_credit_quantum = cfg.dir_credit_high_watermark / 2;\n+\tcfg.dir_credit_low_watermark = RTE_MIN(16, cfg.dir_credit_quantum);\n+\n+\t/* Per QM values */\n+\n+\tcfg.ldb_credit_pool_id = handle->cfg.ldb_credit_pool_id;\n+\tcfg.dir_credit_pool_id = handle->cfg.dir_credit_pool_id;\n+\n+\tret = dlb_iface_dir_port_create(handle, &cfg, dlb->poll_mode);\n+\tif (ret < 0) {\n+\t\tDLB_LOG_ERR(\"dlb: dlb_dir_port_create error, ret=%d (driver status: %s)\\n\",\n+\t\t\t    ret, dlb_error_strings[response.status]);\n+\t\tgoto error_exit;\n+\t}\n+\n+\tqm_port_id = response.id;\n+\n+\tDLB_LOG_DBG(\"dlb: ev_port %d uses qm DIR port %d <<<<<\\n\",\n+\t\t    ev_port->id, qm_port_id);\n+\n+\tqm_port = &ev_port->qm_port;\n+\tqm_port->ev_port = ev_port; /* back ptr */\n+\tqm_port->dlb = dlb;  /* back ptr */\n+\n+\t/*\n+\t * Init local qe struct(s).\n+\t * Note: MOVDIR64 requires the enqueue QE to be aligned\n+\t */\n+\n+\tsnprintf(mz_name, sizeof(mz_name), \"%s_dir_port%d\",\n+\t\t handle->device_name,\n+\t\t ev_port->id);\n+\n+\tret = dlb_init_qe_mem(qm_port, mz_name);\n+\n+\tif (ret < 0) {\n+\t\tDLB_LOG_ERR(\"dlb: init_qe_mem failed, ret=%d\\n\", ret);\n+\t\tgoto error_exit;\n+\t}\n+\n+\tqm_port->pp_mmio_base = DLB_DIR_PP_BASE + PAGE_SIZE * qm_port_id;\n+\tqm_port->id = qm_port_id;\n+\n+\t/* The credit window is one high water mark of QEs */\n+\tqm_port->ldb_pushcount_at_credit_expiry = 0;\n+\tqm_port->cached_ldb_credits = cfg.ldb_credit_high_watermark;\n+\t/* The credit window is one high water mark of QEs */\n+\tqm_port->dir_pushcount_at_credit_expiry = 0;\n+\tqm_port->cached_dir_credits = cfg.dir_credit_high_watermark;\n+\tqm_port->cq_depth = cfg.cq_depth;\n+\tqm_port->cq_idx = 0;\n+\tqm_port->cq_idx_unmasked = 0;\n+\tif (dlb->poll_mode == DLB_CQ_POLL_MODE_SPARSE)\n+\t\tqm_port->cq_depth_mask = (cfg.cq_depth * 4) - 1;\n+\telse\n+\t\tqm_port->cq_depth_mask = cfg.cq_depth - 1;\n+\n+\tqm_port->gen_bit_shift = __builtin_popcount(qm_port->cq_depth_mask);\n+\t/* starting value of gen bit - it toggles at wrap time */\n+\tqm_port->gen_bit = 1;\n+\n+\tqm_port->use_rsvd_token_scheme = use_rsvd_token_scheme;\n+\tqm_port->cq_rsvd_token_deficit = rsvd_tokens;\n+\tqm_port->int_armed = false;\n+\n+\t/* Save off for later use in info and lookup APIs. */\n+\tqm_port->qid_mappings = &dlb->qm_dir_to_ev_queue_id[0];\n+\n+\tqm_port->dequeue_depth = dequeue_depth;\n+\n+\t/* Directed ports are auto-pop, by default. */\n+\tqm_port->token_pop_mode = AUTO_POP;\n+\tqm_port->owed_tokens = 0;\n+\tqm_port->issued_releases = 0;\n+\n+\t/* Save config message too. */\n+\trte_memcpy(&qm_port->cfg.dir, &cfg, sizeof(cfg));\n+\n+\t/* update state */\n+\tqm_port->state = PORT_STARTED; /* enabled at create time */\n+\tqm_port->config_state = DLB_CONFIGURED;\n+\n+\tqm_port->dir_credits = cfg.dir_credit_high_watermark;\n+\tqm_port->ldb_credits = cfg.ldb_credit_high_watermark;\n+\n+\tDLB_LOG_DBG(\"dlb: created dir port %d, depth = %d cr=%d,%d\\n\",\n+\t\t    qm_port_id,\n+\t\t    cq_depth,\n+\t\t    cfg.dir_credit_high_watermark,\n+\t\t    cfg.ldb_credit_high_watermark);\n+\n+\trte_spinlock_unlock(&handle->resource_lock);\n+\n+\treturn 0;\n+\n+error_exit:\n+\tif (qm_port) {\n+\t\tqm_port->pp_mmio_base = 0;\n+\t\tdlb_free_qe_mem(qm_port);\n+\t}\n+\n+\trte_spinlock_unlock(&handle->resource_lock);\n+\n+\tDLB_LOG_ERR(\"dlb: create dir port failed!\\n\");\n+\n+\treturn ret;\n+}\n+\n static int32_t\n dlb_hw_create_ldb_queue(struct dlb_eventdev *dlb,\n \t\t\tstruct dlb_queue *queue,\n@@ -280,6 +786,23 @@ int dlb_string_to_int(int *result, const char *str)\n \treturn qm_qid;\n }\n \n+static inline void\n+dlb_hw_do_enqueue(struct dlb_port *qm_port,\n+\t\t  bool do_sfence,\n+\t\t  struct process_local_port_data *port_data)\n+{\n+\tDLB_LOG_DBG(\"dlb: Flushing QE(s) to DLB\\n\");\n+\n+\t/* Since MOVDIR64B is weakly-ordered, use an SFENCE to ensure that\n+\t * application writes complete before enqueueing the release HCW.\n+\t */\n+\tif (do_sfence)\n+\t\trte_wmb();\n+\n+\n+\tdlb_pp_write(qm_port->qe4, port_data);\n+}\n+\n /* VDEV-only notes:\n  * This function first unmaps all memory mappings and closes the\n  * domain's file descriptor, which causes the driver to reset the\n@@ -557,6 +1080,596 @@ int dlb_string_to_int(int *result, const char *str)\n \t*dev_info = evdev_dlb_default_info;\n }\n \n+static inline int\n+dlb_check_enqueue_sw_credits(struct dlb_eventdev *dlb,\n+\t\t\t     struct dlb_eventdev_port *ev_port)\n+{\n+\tuint32_t sw_inflights = rte_atomic32_read(&dlb->inflights);\n+\tconst int num = 1;\n+\n+\tif (unlikely(ev_port->inflight_max < sw_inflights)) {\n+\t\tDLB_INC_STAT(ev_port->stats.traffic.tx_nospc_inflight_max, 1);\n+\t\trte_errno = -ENOSPC;\n+\t\treturn 1;\n+\t}\n+\n+\tif (ev_port->inflight_credits < num) {\n+\t\t/* check if event enqueue brings ev_port over max threshold */\n+\t\tuint32_t credit_update_quanta = ev_port->credit_update_quanta;\n+\n+\t\tif (sw_inflights + credit_update_quanta >\n+\t\t    dlb->new_event_limit) {\n+\t\t\tDLB_INC_STAT(\n+\t\t\t\tev_port->stats.traffic.tx_nospc_new_event_limit,\n+\t\t\t\t1);\n+\t\t\trte_errno = -ENOSPC;\n+\t\t\treturn 1;\n+\t\t}\n+\n+\t\trte_atomic32_add(&dlb->inflights, credit_update_quanta);\n+\t\tev_port->inflight_credits += (credit_update_quanta);\n+\n+\t\tif (ev_port->inflight_credits < num) {\n+\t\t\tDLB_INC_STAT(\n+\t\t\t    ev_port->stats.traffic.tx_nospc_inflight_credits,\n+\t\t\t    1);\n+\t\t\trte_errno = -ENOSPC;\n+\t\t\treturn 1;\n+\t\t}\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static inline void\n+dlb_replenish_sw_credits(struct dlb_eventdev *dlb,\n+\t\t\t struct dlb_eventdev_port *ev_port)\n+{\n+\tuint16_t quanta = ev_port->credit_update_quanta;\n+\n+\tif (ev_port->inflight_credits >= quanta * 2) {\n+\t\t/* Replenish credits, saving one quanta for enqueues */\n+\t\tuint16_t val = ev_port->inflight_credits - quanta;\n+\n+\t\trte_atomic32_sub(&dlb->inflights, val);\n+\t\tev_port->inflight_credits -= val;\n+\t}\n+}\n+\n+static inline int\n+dlb_check_enqueue_hw_ldb_credits(struct dlb_port *qm_port,\n+\t\t\t\t struct process_local_port_data *port_data)\n+{\n+\tif (unlikely(qm_port->cached_ldb_credits == 0)) {\n+\t\tuint16_t pc;\n+\n+\t\tpc = dlb_read_pc(port_data, true);\n+\n+\t\tqm_port->cached_ldb_credits = pc -\n+\t\t\tqm_port->ldb_pushcount_at_credit_expiry;\n+\t\tif (unlikely(qm_port->cached_ldb_credits == 0)) {\n+\t\t\tDLB_INC_STAT(\n+\t\t\tqm_port->ev_port->stats.traffic.tx_nospc_ldb_hw_credits,\n+\t\t\t1);\n+\n+\t\t\tDLB_LOG_DBG(\"ldb credits exhausted\\n\");\n+\t\t\treturn 1;\n+\t\t}\n+\t\tqm_port->ldb_pushcount_at_credit_expiry +=\n+\t\t\tqm_port->cached_ldb_credits;\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static inline int\n+dlb_check_enqueue_hw_dir_credits(struct dlb_port *qm_port,\n+\t\t\t\t struct process_local_port_data *port_data)\n+{\n+\tif (unlikely(qm_port->cached_dir_credits == 0)) {\n+\t\tuint16_t pc;\n+\n+\t\tpc = dlb_read_pc(port_data, false);\n+\n+\t\tqm_port->cached_dir_credits = pc -\n+\t\t\tqm_port->dir_pushcount_at_credit_expiry;\n+\n+\t\tif (unlikely(qm_port->cached_dir_credits == 0)) {\n+\t\t\tDLB_INC_STAT(\n+\t\t\tqm_port->ev_port->stats.traffic.tx_nospc_dir_hw_credits,\n+\t\t\t1);\n+\n+\t\t\tDLB_LOG_DBG(\"dir credits exhausted\\n\");\n+\t\t\treturn 1;\n+\t\t}\n+\t\tqm_port->dir_pushcount_at_credit_expiry +=\n+\t\t\tqm_port->cached_dir_credits;\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static inline int\n+dlb_event_enqueue_prep(struct dlb_eventdev_port *ev_port,\n+\t\t       struct dlb_port *qm_port,\n+\t\t       const struct rte_event ev[],\n+\t\t       struct process_local_port_data *port_data,\n+\t\t       uint8_t *sched_type,\n+\t\t       uint8_t *queue_id)\n+{\n+\tstruct dlb_eventdev *dlb = ev_port->dlb;\n+\tstruct dlb_eventdev_queue *ev_queue;\n+\tuint16_t *cached_credits = NULL;\n+\tstruct dlb_queue *qm_queue;\n+\n+\tev_queue = &dlb->ev_queues[ev->queue_id];\n+\tqm_queue = &ev_queue->qm_queue;\n+\t*queue_id = qm_queue->id;\n+\n+\t/* Ignore sched_type and hardware credits on release events */\n+\tif (ev->op == RTE_EVENT_OP_RELEASE)\n+\t\tgoto op_check;\n+\n+\tif (!qm_queue->is_directed) {\n+\t\t/* Load balanced destination queue */\n+\n+\t\tif (dlb_check_enqueue_hw_ldb_credits(qm_port, port_data)) {\n+\t\t\trte_errno = -ENOSPC;\n+\t\t\treturn 1;\n+\t\t}\n+\t\tcached_credits = &qm_port->cached_ldb_credits;\n+\n+\t\tswitch (ev->sched_type) {\n+\t\tcase RTE_SCHED_TYPE_ORDERED:\n+\t\t\tDLB_LOG_DBG(\"dlb: put_qe: RTE_SCHED_TYPE_ORDERED\\n\");\n+\t\t\tif (qm_queue->sched_type != RTE_SCHED_TYPE_ORDERED) {\n+\t\t\t\tDLB_LOG_ERR(\"dlb: tried to send ordered event to unordered queue %d\\n\",\n+\t\t\t\t\t    *queue_id);\n+\t\t\t\trte_errno = -EINVAL;\n+\t\t\t\treturn 1;\n+\t\t\t}\n+\t\t\t*sched_type = DLB_SCHED_ORDERED;\n+\t\t\tbreak;\n+\t\tcase RTE_SCHED_TYPE_ATOMIC:\n+\t\t\tDLB_LOG_DBG(\"dlb: put_qe: RTE_SCHED_TYPE_ATOMIC\\n\");\n+\t\t\t*sched_type = DLB_SCHED_ATOMIC;\n+\t\t\tbreak;\n+\t\tcase RTE_SCHED_TYPE_PARALLEL:\n+\t\t\tDLB_LOG_DBG(\"dlb: put_qe: RTE_SCHED_TYPE_PARALLEL\\n\");\n+\t\t\tif (qm_queue->sched_type == RTE_SCHED_TYPE_ORDERED)\n+\t\t\t\t*sched_type = DLB_SCHED_ORDERED;\n+\t\t\telse\n+\t\t\t\t*sched_type = DLB_SCHED_UNORDERED;\n+\t\t\tbreak;\n+\t\tdefault:\n+\t\t\tDLB_LOG_ERR(\"Unsupported LDB sched type in put_qe\\n\");\n+\t\t\tDLB_INC_STAT(ev_port->stats.tx_invalid, 1);\n+\t\t\trte_errno = -EINVAL;\n+\t\t\treturn 1;\n+\t\t}\n+\t} else {\n+\t\t/* Directed destination queue */\n+\n+\t\tif (dlb_check_enqueue_hw_dir_credits(qm_port, port_data)) {\n+\t\t\trte_errno = -ENOSPC;\n+\t\t\treturn 1;\n+\t\t}\n+\t\tcached_credits = &qm_port->cached_dir_credits;\n+\n+\t\tDLB_LOG_DBG(\"dlb: put_qe: RTE_SCHED_TYPE_DIRECTED\\n\");\n+\n+\t\t*sched_type = DLB_SCHED_DIRECTED;\n+\t}\n+\n+op_check:\n+\tswitch (ev->op) {\n+\tcase RTE_EVENT_OP_NEW:\n+\t\t/* Check that a sw credit is available */\n+\t\tif (dlb_check_enqueue_sw_credits(dlb, ev_port)) {\n+\t\t\trte_errno = -ENOSPC;\n+\t\t\treturn 1;\n+\t\t}\n+\t\tev_port->inflight_credits--;\n+\t\t(*cached_credits)--;\n+\t\tbreak;\n+\tcase RTE_EVENT_OP_FORWARD:\n+\t\t/* Check for outstanding_releases underflow. If this occurs,\n+\t\t * the application is not using the EVENT_OPs correctly; for\n+\t\t * example, forwarding or releasing events that were not\n+\t\t * dequeued.\n+\t\t */\n+\t\tRTE_ASSERT(ev_port->outstanding_releases > 0);\n+\t\tev_port->outstanding_releases--;\n+\t\tqm_port->issued_releases++;\n+\t\t(*cached_credits)--;\n+\t\tbreak;\n+\tcase RTE_EVENT_OP_RELEASE:\n+\t\tev_port->inflight_credits++;\n+\t\t/* Check for outstanding_releases underflow. If this occurs,\n+\t\t * the application is not using the EVENT_OPs correctly; for\n+\t\t * example, forwarding or releasing events that were not\n+\t\t * dequeued.\n+\t\t */\n+\t\tRTE_ASSERT(ev_port->outstanding_releases > 0);\n+\t\tev_port->outstanding_releases--;\n+\t\tqm_port->issued_releases++;\n+\t\t/* Replenish s/w credits if enough are cached */\n+\t\tdlb_replenish_sw_credits(dlb, ev_port);\n+\t\tbreak;\n+\t}\n+\n+\tDLB_INC_STAT(ev_port->stats.tx_op_cnt[ev->op], 1);\n+\tDLB_INC_STAT(ev_port->stats.traffic.tx_ok, 1);\n+\n+#ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS\n+\tif (ev->op != RTE_EVENT_OP_RELEASE) {\n+\t\tDLB_INC_STAT(ev_port->stats.enq_ok[ev->queue_id], 1);\n+\t\tDLB_INC_STAT(ev_port->stats.tx_sched_cnt[*sched_type], 1);\n+\t}\n+#endif\n+\n+\treturn 0;\n+}\n+\n+static uint8_t cmd_byte_map[NUM_DLB_PORT_TYPES][DLB_NUM_HW_SCHED_TYPES] = {\n+\t{\n+\t\t/* Load-balanced cmd bytes */\n+\t\t[RTE_EVENT_OP_NEW] = DLB_NEW_CMD_BYTE,\n+\t\t[RTE_EVENT_OP_FORWARD] = DLB_FWD_CMD_BYTE,\n+\t\t[RTE_EVENT_OP_RELEASE] = DLB_COMP_CMD_BYTE,\n+\t},\n+\t{\n+\t\t/* Directed cmd bytes */\n+\t\t[RTE_EVENT_OP_NEW] = DLB_NEW_CMD_BYTE,\n+\t\t[RTE_EVENT_OP_FORWARD] = DLB_NEW_CMD_BYTE,\n+\t\t[RTE_EVENT_OP_RELEASE] = DLB_NOOP_CMD_BYTE,\n+\t},\n+};\n+\n+static inline void\n+dlb_event_build_hcws(struct dlb_port *qm_port,\n+\t\t     const struct rte_event ev[],\n+\t\t     int num,\n+\t\t     uint8_t *sched_type,\n+\t\t     uint8_t *queue_id)\n+{\n+\tstruct dlb_enqueue_qe *qe;\n+\tuint16_t sched_word[4];\n+\t__m128i sse_qe[2];\n+\tint i;\n+\n+\tqe = qm_port->qe4;\n+\n+\tsse_qe[0] = _mm_setzero_si128();\n+\tsse_qe[1] = _mm_setzero_si128();\n+\n+\tswitch (num) {\n+\tcase 4:\n+\t\t/* Construct the metadata portion of two HCWs in one 128b SSE\n+\t\t * register. HCW metadata is constructed in the SSE registers\n+\t\t * like so:\n+\t\t * sse_qe[0][63:0]:   qe[0]'s metadata\n+\t\t * sse_qe[0][127:64]: qe[1]'s metadata\n+\t\t * sse_qe[1][63:0]:   qe[2]'s metadata\n+\t\t * sse_qe[1][127:64]: qe[3]'s metadata\n+\t\t */\n+\n+\t\t/* Convert the event operation into a command byte and store it\n+\t\t * in the metadata:\n+\t\t * sse_qe[0][63:56]   = cmd_byte_map[is_directed][ev[0].op]\n+\t\t * sse_qe[0][127:120] = cmd_byte_map[is_directed][ev[1].op]\n+\t\t * sse_qe[1][63:56]   = cmd_byte_map[is_directed][ev[2].op]\n+\t\t * sse_qe[1][127:120] = cmd_byte_map[is_directed][ev[3].op]\n+\t\t */\n+#define DLB_QE_CMD_BYTE 7\n+\t\tsse_qe[0] = _mm_insert_epi8(sse_qe[0],\n+\t\t\t\tcmd_byte_map[qm_port->is_directed][ev[0].op],\n+\t\t\t\tDLB_QE_CMD_BYTE);\n+\t\tsse_qe[0] = _mm_insert_epi8(sse_qe[0],\n+\t\t\t\tcmd_byte_map[qm_port->is_directed][ev[1].op],\n+\t\t\t\tDLB_QE_CMD_BYTE + 8);\n+\t\tsse_qe[1] = _mm_insert_epi8(sse_qe[1],\n+\t\t\t\tcmd_byte_map[qm_port->is_directed][ev[2].op],\n+\t\t\t\tDLB_QE_CMD_BYTE);\n+\t\tsse_qe[1] = _mm_insert_epi8(sse_qe[1],\n+\t\t\t\tcmd_byte_map[qm_port->is_directed][ev[3].op],\n+\t\t\t\tDLB_QE_CMD_BYTE + 8);\n+\n+\t\t/* Store priority, scheduling type, and queue ID in the sched\n+\t\t * word array because these values are re-used when the\n+\t\t * destination is a directed queue.\n+\t\t */\n+\t\tsched_word[0] = EV_TO_DLB_PRIO(ev[0].priority) << 10 |\n+\t\t\t\tsched_type[0] << 8 |\n+\t\t\t\tqueue_id[0];\n+\t\tsched_word[1] = EV_TO_DLB_PRIO(ev[1].priority) << 10 |\n+\t\t\t\tsched_type[1] << 8 |\n+\t\t\t\tqueue_id[1];\n+\t\tsched_word[2] = EV_TO_DLB_PRIO(ev[2].priority) << 10 |\n+\t\t\t\tsched_type[2] << 8 |\n+\t\t\t\tqueue_id[2];\n+\t\tsched_word[3] = EV_TO_DLB_PRIO(ev[3].priority) << 10 |\n+\t\t\t\tsched_type[3] << 8 |\n+\t\t\t\tqueue_id[3];\n+\n+\t\t/* Store the event priority, scheduling type, and queue ID in\n+\t\t * the metadata:\n+\t\t * sse_qe[0][31:16] = sched_word[0]\n+\t\t * sse_qe[0][95:80] = sched_word[1]\n+\t\t * sse_qe[1][31:16] = sched_word[2]\n+\t\t * sse_qe[1][95:80] = sched_word[3]\n+\t\t */\n+#define DLB_QE_QID_SCHED_WORD 1\n+\t\tsse_qe[0] = _mm_insert_epi16(sse_qe[0],\n+\t\t\t\t\t     sched_word[0],\n+\t\t\t\t\t     DLB_QE_QID_SCHED_WORD);\n+\t\tsse_qe[0] = _mm_insert_epi16(sse_qe[0],\n+\t\t\t\t\t     sched_word[1],\n+\t\t\t\t\t     DLB_QE_QID_SCHED_WORD + 4);\n+\t\tsse_qe[1] = _mm_insert_epi16(sse_qe[1],\n+\t\t\t\t\t     sched_word[2],\n+\t\t\t\t\t     DLB_QE_QID_SCHED_WORD);\n+\t\tsse_qe[1] = _mm_insert_epi16(sse_qe[1],\n+\t\t\t\t\t     sched_word[3],\n+\t\t\t\t\t     DLB_QE_QID_SCHED_WORD + 4);\n+\n+\t\t/* If the destination is a load-balanced queue, store the lock\n+\t\t * ID. If it is a directed queue, DLB places this field in\n+\t\t * bytes 10-11 of the received QE, so we format it accordingly:\n+\t\t * sse_qe[0][47:32]  = dir queue ? sched_word[0] : flow_id[0]\n+\t\t * sse_qe[0][111:96] = dir queue ? sched_word[1] : flow_id[1]\n+\t\t * sse_qe[1][47:32]  = dir queue ? sched_word[2] : flow_id[2]\n+\t\t * sse_qe[1][111:96] = dir queue ? sched_word[3] : flow_id[3]\n+\t\t */\n+#define DLB_QE_LOCK_ID_WORD 2\n+\t\tsse_qe[0] = _mm_insert_epi16(sse_qe[0],\n+\t\t\t\t(sched_type[0] == DLB_SCHED_DIRECTED) ?\n+\t\t\t\t\tsched_word[0] : ev[0].flow_id,\n+\t\t\t\tDLB_QE_LOCK_ID_WORD);\n+\t\tsse_qe[0] = _mm_insert_epi16(sse_qe[0],\n+\t\t\t\t(sched_type[1] == DLB_SCHED_DIRECTED) ?\n+\t\t\t\t\tsched_word[1] : ev[1].flow_id,\n+\t\t\t\tDLB_QE_LOCK_ID_WORD + 4);\n+\t\tsse_qe[1] = _mm_insert_epi16(sse_qe[1],\n+\t\t\t\t(sched_type[2] == DLB_SCHED_DIRECTED) ?\n+\t\t\t\t\tsched_word[2] : ev[2].flow_id,\n+\t\t\t\tDLB_QE_LOCK_ID_WORD);\n+\t\tsse_qe[1] = _mm_insert_epi16(sse_qe[1],\n+\t\t\t\t(sched_type[3] == DLB_SCHED_DIRECTED) ?\n+\t\t\t\t\tsched_word[3] : ev[3].flow_id,\n+\t\t\t\tDLB_QE_LOCK_ID_WORD + 4);\n+\n+\t\t/* Store the event type and sub event type in the metadata:\n+\t\t * sse_qe[0][15:0]  = flow_id[0]\n+\t\t * sse_qe[0][79:64] = flow_id[1]\n+\t\t * sse_qe[1][15:0]  = flow_id[2]\n+\t\t * sse_qe[1][79:64] = flow_id[3]\n+\t\t */\n+#define DLB_QE_EV_TYPE_WORD 0\n+\t\tsse_qe[0] = _mm_insert_epi16(sse_qe[0],\n+\t\t\t\t\t     ev[0].sub_event_type << 8 |\n+\t\t\t\t\t\tev[0].event_type,\n+\t\t\t\t\t     DLB_QE_EV_TYPE_WORD);\n+\t\tsse_qe[0] = _mm_insert_epi16(sse_qe[0],\n+\t\t\t\t\t     ev[1].sub_event_type << 8 |\n+\t\t\t\t\t\tev[1].event_type,\n+\t\t\t\t\t     DLB_QE_EV_TYPE_WORD + 4);\n+\t\tsse_qe[1] = _mm_insert_epi16(sse_qe[1],\n+\t\t\t\t\t     ev[2].sub_event_type << 8 |\n+\t\t\t\t\t\tev[2].event_type,\n+\t\t\t\t\t     DLB_QE_EV_TYPE_WORD);\n+\t\tsse_qe[1] = _mm_insert_epi16(sse_qe[1],\n+\t\t\t\t\t     ev[3].sub_event_type << 8 |\n+\t\t\t\t\t\tev[3].event_type,\n+\t\t\t\t\t     DLB_QE_EV_TYPE_WORD + 4);\n+\n+\t\t/* Store the metadata to memory (use the double-precision\n+\t\t * _mm_storeh_pd because there is no integer function for\n+\t\t * storing the upper 64b):\n+\t\t * qe[0] metadata = sse_qe[0][63:0]\n+\t\t * qe[1] metadata = sse_qe[0][127:64]\n+\t\t * qe[2] metadata = sse_qe[1][63:0]\n+\t\t * qe[3] metadata = sse_qe[1][127:64]\n+\t\t */\n+\t\t_mm_storel_epi64((__m128i *)&qe[0].u.opaque_data, sse_qe[0]);\n+\t\t_mm_storeh_pd((double *)&qe[1].u.opaque_data,\n+\t\t\t      (__m128d) sse_qe[0]);\n+\t\t_mm_storel_epi64((__m128i *)&qe[2].u.opaque_data, sse_qe[1]);\n+\t\t_mm_storeh_pd((double *)&qe[3].u.opaque_data,\n+\t\t\t      (__m128d) sse_qe[1]);\n+\n+\t\tqe[0].data = ev[0].u64;\n+\t\tqe[1].data = ev[1].u64;\n+\t\tqe[2].data = ev[2].u64;\n+\t\tqe[3].data = ev[3].u64;\n+\n+\t\tbreak;\n+\tcase 3:\n+\tcase 2:\n+\tcase 1:\n+\t\t/* At least one QE will be valid, so only zero out three */\n+\t\tqe[1].cmd_byte = 0;\n+\t\tqe[2].cmd_byte = 0;\n+\t\tqe[3].cmd_byte = 0;\n+\n+\t\tfor (i = 0; i < num; i++) {\n+\t\t\tqe[i].cmd_byte =\n+\t\t\t\tcmd_byte_map[qm_port->is_directed][ev[i].op];\n+\t\t\tqe[i].sched_type = sched_type[i];\n+\t\t\tqe[i].data = ev[i].u64;\n+\t\t\tqe[i].qid = queue_id[i];\n+\t\t\tqe[i].priority = EV_TO_DLB_PRIO(ev[i].priority);\n+\t\t\tqe[i].lock_id = ev[i].flow_id;\n+\t\t\tif (sched_type[i] == DLB_SCHED_DIRECTED) {\n+\t\t\t\tstruct dlb_msg_info *info =\n+\t\t\t\t\t(struct dlb_msg_info *)&qe[i].lock_id;\n+\n+\t\t\t\tinfo->qid = queue_id[i];\n+\t\t\t\tinfo->sched_type = DLB_SCHED_DIRECTED;\n+\t\t\t\tinfo->priority = qe[i].priority;\n+\t\t\t}\n+\t\t\tqe[i].u.event_type.major = ev[i].event_type;\n+\t\t\tqe[i].u.event_type.sub = ev[i].sub_event_type;\n+\t\t}\n+\t\tbreak;\n+\t}\n+}\n+\n+static inline void\n+dlb_construct_token_pop_qe(struct dlb_port *qm_port, int idx)\n+{\n+\tstruct dlb_cq_pop_qe *qe = (void *)qm_port->qe4;\n+\tint num = qm_port->owed_tokens;\n+\n+\tif (qm_port->use_rsvd_token_scheme) {\n+\t\t/* Check if there's a deficit of reserved tokens, and return\n+\t\t * early if there are no (unreserved) tokens to consume.\n+\t\t */\n+\t\tif (num <= qm_port->cq_rsvd_token_deficit) {\n+\t\t\tqm_port->cq_rsvd_token_deficit -= num;\n+\t\t\tqm_port->owed_tokens = 0;\n+\t\t\treturn;\n+\t\t}\n+\t\tnum -= qm_port->cq_rsvd_token_deficit;\n+\t\tqm_port->cq_rsvd_token_deficit = 0;\n+\t}\n+\n+\tqe[idx].cmd_byte = DLB_POP_CMD_BYTE;\n+\tqe[idx].tokens = num - 1;\n+\n+\tqm_port->owed_tokens = 0;\n+}\n+\n+static inline uint16_t\n+__dlb_event_enqueue_burst(void *event_port,\n+\t\t\t  const struct rte_event events[],\n+\t\t\t  uint16_t num,\n+\t\t\t  bool use_delayed)\n+{\n+\tstruct dlb_eventdev_port *ev_port = event_port;\n+\tstruct dlb_port *qm_port = &ev_port->qm_port;\n+\tstruct process_local_port_data *port_data;\n+\tint i, cnt;\n+\n+\tRTE_ASSERT(ev_port->enq_configured);\n+\tRTE_ASSERT(events != NULL);\n+\n+\trte_errno = 0;\n+\tcnt = 0;\n+\n+\tport_data = &dlb_port[qm_port->id][PORT_TYPE(qm_port)];\n+\n+\tif (!port_data->mmaped)\n+\t\tdlb_iface_port_mmap(qm_port);\n+\n+\tfor (i = 0; i < num; i += DLB_NUM_QES_PER_CACHE_LINE) {\n+\t\tuint8_t sched_types[DLB_NUM_QES_PER_CACHE_LINE];\n+\t\tuint8_t queue_ids[DLB_NUM_QES_PER_CACHE_LINE];\n+\t\tint j = 0;\n+\n+\t\tfor (; j < DLB_NUM_QES_PER_CACHE_LINE && (i + j) < num; j++) {\n+\t\t\tconst struct rte_event *ev = &events[i + j];\n+\n+\t\t\tif (dlb_event_enqueue_prep(ev_port, qm_port, ev,\n+\t\t\t\t\t\t   port_data, &sched_types[j],\n+\t\t\t\t\t\t   &queue_ids[j]))\n+\t\t\t\tbreak;\n+\t\t}\n+\n+\t\tif (j == 0)\n+\t\t\tbreak;\n+\n+\t\tdlb_event_build_hcws(qm_port, &events[i], j,\n+\t\t\t\t     sched_types, queue_ids);\n+\n+\t\t/* The delayed-pop code causes an unnecessary performance\n+\t\t * penalty when it is not in use. The use_delayed argument\n+\t\t * allows the compiler to create a version of this function\n+\t\t * with these checks factored out that the PMD can call\n+\t\t * when delayed-pop is not in use.\n+\t\t */\n+\t\tif (use_delayed &&\n+\t\t    qm_port->token_pop_mode == DELAYED_POP && j < 4 &&\n+\t\t    qm_port->issued_releases >= qm_port->token_pop_thresh - 1) {\n+\n+\t\t\tdlb_construct_token_pop_qe(qm_port, j);\n+\n+\t\t\t/* Reset the releases counter for the next QE batch */\n+\t\t\tqm_port->issued_releases -= qm_port->token_pop_thresh;\n+\n+\t\t\t/* When using delayed token pop mode, the initial token\n+\t\t\t * threshold is the full CQ depth. After the first\n+\t\t\t * token pop, we need to reset it to the dequeue_depth.\n+\t\t\t */\n+\t\t\tqm_port->token_pop_thresh = qm_port->dequeue_depth;\n+\t\t}\n+\n+\t\tdlb_hw_do_enqueue(qm_port, i == 0, port_data);\n+\n+\t\tcnt += j;\n+\n+\t\tif (j < DLB_NUM_QES_PER_CACHE_LINE)\n+\t\t\tbreak;\n+\t}\n+\n+\tif (use_delayed && qm_port->token_pop_mode == DELAYED_POP &&\n+\t    qm_port->issued_releases >= qm_port->token_pop_thresh - 1) {\n+\t\tdlb_consume_qe_immediate(qm_port, qm_port->owed_tokens);\n+\t\tqm_port->issued_releases -= qm_port->token_pop_thresh;\n+\t\tqm_port->token_pop_thresh = qm_port->dequeue_depth;\n+\t}\n+\n+\tRTE_ASSERT(!((cnt == 0 && rte_errno != -ENOSPC)));\n+\n+\treturn cnt;\n+}\n+\n+static inline uint16_t\n+dlb_event_enqueue_burst(void *event_port,\n+\t\t\tconst struct rte_event events[],\n+\t\t\tuint16_t num)\n+{\n+\treturn __dlb_event_enqueue_burst(event_port, events, num, false);\n+}\n+\n+static inline uint16_t\n+dlb_event_enqueue_burst_delayed(void *event_port,\n+\t\t\t\tconst struct rte_event events[],\n+\t\t\t\tuint16_t num)\n+{\n+\treturn __dlb_event_enqueue_burst(event_port, events, num, true);\n+}\n+\n+static inline uint16_t\n+dlb_event_enqueue(void *event_port,\n+\t\t  const struct rte_event events[])\n+{\n+\treturn __dlb_event_enqueue_burst(event_port, events, 1, false);\n+}\n+\n+static inline uint16_t\n+dlb_event_enqueue_delayed(void *event_port,\n+\t\t\t  const struct rte_event events[])\n+{\n+\treturn __dlb_event_enqueue_burst(event_port, events, 1, true);\n+}\n+\n+static uint16_t\n+dlb_event_enqueue_new_burst_delayed(void *event_port,\n+\t\t\t\t    const struct rte_event events[],\n+\t\t\t\t    uint16_t num)\n+{\n+\treturn __dlb_event_enqueue_burst(event_port, events, num, true);\n+}\n+\n+static uint16_t\n+dlb_event_enqueue_forward_burst_delayed(void *event_port,\n+\t\t\t\t\tconst struct rte_event events[],\n+\t\t\t\t\tuint16_t num)\n+{\n+\treturn __dlb_event_enqueue_burst(event_port, events, num, true);\n+}\n+\n /* Note: 1 QM instance per QM device, QM instance/device == event device */\n static int\n dlb_eventdev_configure(const struct rte_eventdev *dev)\n@@ -934,6 +2047,133 @@ static int dlb_num_dir_queues_setup(struct dlb_eventdev *dlb)\n \treturn ret;\n }\n \n+static void\n+dlb_port_link_teardown(struct dlb_eventdev *dlb,\n+\t\t       struct dlb_eventdev_port *ev_port)\n+{\n+\tstruct dlb_eventdev_queue *ev_queue;\n+\tint i;\n+\n+\tfor (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {\n+\t\tif (!ev_port->link[i].valid)\n+\t\t\tcontinue;\n+\n+\t\tev_queue = &dlb->ev_queues[ev_port->link[i].queue_id];\n+\n+\t\tev_port->link[i].valid = false;\n+\t\tev_port->num_links--;\n+\t\tev_queue->num_links--;\n+\t}\n+}\n+\n+static int\n+dlb_eventdev_port_setup(struct rte_eventdev *dev,\n+\t\t\tuint8_t ev_port_id,\n+\t\t\tconst struct rte_event_port_conf *port_conf)\n+{\n+\tstruct dlb_eventdev *dlb;\n+\tstruct dlb_eventdev_port *ev_port;\n+\tbool use_rsvd_token_scheme;\n+\tuint32_t adj_cq_depth;\n+\tuint16_t rsvd_tokens;\n+\tint ret;\n+\n+\tif (dev == NULL || port_conf == NULL) {\n+\t\tDLB_LOG_ERR(\"Null parameter\\n\");\n+\t\treturn -EINVAL;\n+\t}\n+\n+\tdlb = dlb_pmd_priv(dev);\n+\n+\tif (ev_port_id >= DLB_MAX_NUM_PORTS)\n+\t\treturn -EINVAL;\n+\n+\tif (port_conf->dequeue_depth >\n+\t\tevdev_dlb_default_info.max_event_port_dequeue_depth ||\n+\t    port_conf->enqueue_depth >\n+\t\tevdev_dlb_default_info.max_event_port_enqueue_depth)\n+\t\treturn -EINVAL;\n+\n+\tev_port = &dlb->ev_ports[ev_port_id];\n+\t/* configured? */\n+\tif (ev_port->setup_done) {\n+\t\tDLB_LOG_ERR(\"evport %d is already configured\\n\", ev_port_id);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\t/* The reserved token interrupt arming scheme requires that one or more\n+\t * CQ tokens be reserved by the PMD. This limits the amount of CQ space\n+\t * usable by the DLB, so in order to give an *effective* CQ depth equal\n+\t * to the user-requested value, we double CQ depth and reserve half of\n+\t * its tokens. If the user requests the max CQ depth (256) then we\n+\t * cannot double it, so we reserve one token and give an effective\n+\t * depth of 255 entries.\n+\t */\n+\tuse_rsvd_token_scheme = true;\n+\trsvd_tokens = 1;\n+\tadj_cq_depth = port_conf->dequeue_depth;\n+\n+\tif (use_rsvd_token_scheme && adj_cq_depth < 256) {\n+\t\trsvd_tokens = adj_cq_depth;\n+\t\tadj_cq_depth *= 2;\n+\t}\n+\n+\tev_port->qm_port.is_directed = port_conf->event_port_cfg &\n+\t\tRTE_EVENT_PORT_CFG_SINGLE_LINK;\n+\n+\tif (!ev_port->qm_port.is_directed) {\n+\t\tret = dlb_hw_create_ldb_port(dlb,\n+\t\t\t\t\t     ev_port,\n+\t\t\t\t\t     port_conf->dequeue_depth,\n+\t\t\t\t\t     adj_cq_depth,\n+\t\t\t\t\t     port_conf->enqueue_depth,\n+\t\t\t\t\t     rsvd_tokens,\n+\t\t\t\t\t     use_rsvd_token_scheme);\n+\t\tif (ret < 0) {\n+\t\t\tDLB_LOG_ERR(\"Failed to create the lB port ve portId=%d\\n\",\n+\t\t\t\t    ev_port_id);\n+\t\t\treturn ret;\n+\t\t}\n+\t} else {\n+\t\tret = dlb_hw_create_dir_port(dlb,\n+\t\t\t\t\t     ev_port,\n+\t\t\t\t\t     port_conf->dequeue_depth,\n+\t\t\t\t\t     adj_cq_depth,\n+\t\t\t\t\t     port_conf->enqueue_depth,\n+\t\t\t\t\t     rsvd_tokens,\n+\t\t\t\t\t     use_rsvd_token_scheme);\n+\t\tif (ret < 0) {\n+\t\t\tDLB_LOG_ERR(\"Failed to create the DIR port\\n\");\n+\t\t\treturn ret;\n+\t\t}\n+\t}\n+\n+\t/* Save off port config for reconfig */\n+\tdlb->ev_ports[ev_port_id].conf = *port_conf;\n+\n+\tdlb->ev_ports[ev_port_id].id = ev_port_id;\n+\tdlb->ev_ports[ev_port_id].enq_configured = true;\n+\tdlb->ev_ports[ev_port_id].setup_done = true;\n+\tdlb->ev_ports[ev_port_id].inflight_max =\n+\t\tport_conf->new_event_threshold;\n+\tdlb->ev_ports[ev_port_id].implicit_release =\n+\t\t!(port_conf->event_port_cfg &\n+\t\t  RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL);\n+\tdlb->ev_ports[ev_port_id].outstanding_releases = 0;\n+\tdlb->ev_ports[ev_port_id].inflight_credits = 0;\n+\tdlb->ev_ports[ev_port_id].credit_update_quanta =\n+\t\tRTE_LIBRTE_PMD_DLB_SW_CREDIT_QUANTA;\n+\tdlb->ev_ports[ev_port_id].dlb = dlb; /* reverse link */\n+\n+\t/* Tear down pre-existing port->queue links */\n+\tif (dlb->run_state == DLB_RUN_STATE_STOPPED)\n+\t\tdlb_port_link_teardown(dlb, &dlb->ev_ports[ev_port_id]);\n+\n+\tdev->data->ports[ev_port_id] = &dlb->ev_ports[ev_port_id];\n+\n+\treturn 0;\n+}\n+\n static int\n set_dev_id(const char *key __rte_unused,\n \t   const char *value,\n@@ -1013,6 +2253,7 @@ static int dlb_num_dir_queues_setup(struct dlb_eventdev *dlb)\n \t\t.queue_def_conf   = dlb_eventdev_queue_default_conf_get,\n \t\t.queue_setup      = dlb_eventdev_queue_setup,\n \t\t.port_def_conf    = dlb_eventdev_port_default_conf_get,\n+\t\t.port_setup       = dlb_eventdev_port_setup,\n \t};\n \n \t/* Expose PMD's eventdev interface */\n",
    "prefixes": [
        "19/27"
    ]
}