From patchwork Fri Feb 26 20:01:36 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Cristian Dumitrescu X-Patchwork-Id: 88267 Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id CD893A034F; Fri, 26 Feb 2021 21:01:41 +0100 (CET) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 3F6B640A4B; Fri, 26 Feb 2021 21:01:41 +0100 (CET) Received: from mga05.intel.com (mga05.intel.com [192.55.52.43]) by mails.dpdk.org (Postfix) with ESMTP id 9BB65407FF for ; Fri, 26 Feb 2021 21:01:39 +0100 (CET) IronPort-SDR: O29Uf17vufeY9lJLxUMLJrhLSkW/6kPz6kf30nmsATzaCX9r8OHO+Z7KINZ5vip2+EgVRKGZKH WSGsdX2pO4Cg== X-IronPort-AV: E=McAfee;i="6000,8403,9907"; a="270961657" X-IronPort-AV: E=Sophos;i="5.81,209,1610438400"; d="scan'208";a="270961657" Received: from orsmga002.jf.intel.com ([10.7.209.21]) by fmsmga105.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 26 Feb 2021 12:01:38 -0800 IronPort-SDR: P3ZvZ/aOyBzULO7WVaRdPy4byr/taUbsYOyEldcI1j/dxjkQ7mKVpF06vBTAN+PA79XKf3yvcM gzSF19UrU0sQ== X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.81,209,1610438400"; d="scan'208";a="382132934" Received: from silpixa00400573.ir.intel.com (HELO silpixa00400573.ger.corp.intel.com) ([10.237.223.107]) by orsmga002.jf.intel.com with ESMTP; 26 Feb 2021 12:01:36 -0800 From: Cristian Dumitrescu To: dev@dpdk.org Date: Fri, 26 Feb 2021 20:01:36 +0000 Message-Id: <20210226200136.4265-1-cristian.dumitrescu@intel.com> X-Mailer: git-send-email 2.17.1 Subject: [dpdk-dev] [PATCH] pipeline: add register array support to SWX pipeline X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Register arrays are stateful objects that can be read & modified by both the data plane and the control plane, as opposed to tables, which are read-only for data plane. One key use-case is the implementation of stats counters. Signed-off-by: Cristian Dumitrescu --- examples/pipeline/cli.c | 125 ++ lib/librte_pipeline/rte_swx_ctl.h | 79 + lib/librte_pipeline/rte_swx_pipeline.c | 1712 +++++++++++++++++-- lib/librte_pipeline/rte_swx_pipeline.h | 21 + lib/librte_pipeline/rte_swx_pipeline_spec.c | 91 + lib/librte_pipeline/version.map | 6 + 6 files changed, 1855 insertions(+), 179 deletions(-) diff --git a/examples/pipeline/cli.c b/examples/pipeline/cli.c index 30c2dd34d..cdf8f13f3 100644 --- a/examples/pipeline/cli.c +++ b/examples/pipeline/cli.c @@ -1009,6 +1009,105 @@ cmd_pipeline_table_update(char **tokens, fclose(file_default); } +static const char cmd_pipeline_regrd_help[] = +"pipeline regrd \n"; + +static void +cmd_pipeline_regrd(char **tokens, + uint32_t n_tokens, + char *out, + size_t out_size, + void *obj) +{ + struct pipeline *p; + const char *name; + uint64_t value; + uint32_t idx; + int status; + + if (n_tokens != 5) { + snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]); + return; + } + + p = pipeline_find(obj, tokens[1]); + if (!p || !p->ctl) { + snprintf(out, out_size, MSG_ARG_INVALID, "pipeline_name"); + return; + } + + if (strcmp(tokens[2], "regrd")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "regrd"); + return; + } + + name = tokens[3]; + + if (parser_read_uint32(&idx, tokens[4])) { + snprintf(out, out_size, MSG_ARG_INVALID, "index"); + return; + } + + status = rte_swx_ctl_pipeline_regarray_read(p->p, name, idx, &value); + if (status) { + snprintf(out, out_size, "Command failed.\n"); + return; + } + + snprintf(out, out_size, "0x%" PRIx64 "\n", value); +} + +static const char cmd_pipeline_regwr_help[] = +"pipeline regwr \n"; + +static void +cmd_pipeline_regwr(char **tokens, + uint32_t n_tokens, + char *out, + size_t out_size, + void *obj) +{ + struct pipeline *p; + const char *name; + uint64_t value; + uint32_t idx; + int status; + + if (n_tokens != 6) { + snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]); + return; + } + + p = pipeline_find(obj, tokens[1]); + if (!p || !p->ctl) { + snprintf(out, out_size, MSG_ARG_INVALID, "pipeline_name"); + return; + } + + if (strcmp(tokens[2], "regwr")) { + snprintf(out, out_size, MSG_ARG_NOT_FOUND, "regwr"); + return; + } + + name = tokens[3]; + + if (parser_read_uint32(&idx, tokens[4])) { + snprintf(out, out_size, MSG_ARG_INVALID, "index"); + return; + } + + if (parser_read_uint64(&value, tokens[5])) { + snprintf(out, out_size, MSG_ARG_INVALID, "value"); + return; + } + + status = rte_swx_ctl_pipeline_regarray_write(p->p, name, idx, value); + if (status) { + snprintf(out, out_size, "Command failed.\n"); + return; + } +} + static const char cmd_pipeline_stats_help[] = "pipeline stats\n"; @@ -1202,6 +1301,8 @@ cmd_help(char **tokens, "\tpipeline port out\n" "\tpipeline build\n" "\tpipeline table update\n" + "\tpipeline regrd\n" + "\tpipeline regwr\n" "\tpipeline stats\n" "\tthread pipeline enable\n" "\tthread pipeline disable\n\n"); @@ -1254,6 +1355,18 @@ cmd_help(char **tokens, return; } + if ((strcmp(tokens[0], "pipeline") == 0) && + (n_tokens == 2) && (strcmp(tokens[1], "regrd") == 0)) { + snprintf(out, out_size, "\n%s\n", cmd_pipeline_regrd_help); + return; + } + + if ((strcmp(tokens[0], "pipeline") == 0) && + (n_tokens == 2) && (strcmp(tokens[1], "regwr") == 0)) { + snprintf(out, out_size, "\n%s\n", cmd_pipeline_regwr_help); + return; + } + if ((strcmp(tokens[0], "pipeline") == 0) && (n_tokens == 2) && (strcmp(tokens[1], "stats") == 0)) { snprintf(out, out_size, "\n%s\n", cmd_pipeline_stats_help); @@ -1356,6 +1469,18 @@ cli_process(char *in, char *out, size_t out_size, void *obj) return; } + if ((n_tokens >= 3) && + (strcmp(tokens[2], "regrd") == 0)) { + cmd_pipeline_regrd(tokens, n_tokens, out, out_size, obj); + return; + } + + if ((n_tokens >= 3) && + (strcmp(tokens[2], "regwr") == 0)) { + cmd_pipeline_regwr(tokens, n_tokens, out, out_size, obj); + return; + } + if ((n_tokens >= 3) && (strcmp(tokens[2], "stats") == 0)) { cmd_pipeline_stats(tokens, n_tokens, out, out_size, diff --git a/lib/librte_pipeline/rte_swx_ctl.h b/lib/librte_pipeline/rte_swx_ctl.h index 530671db1..c0a55c6f0 100644 --- a/lib/librte_pipeline/rte_swx_ctl.h +++ b/lib/librte_pipeline/rte_swx_ctl.h @@ -46,6 +46,9 @@ struct rte_swx_ctl_pipeline_info { /** Number of tables. */ uint32_t n_tables; + + /** Number of register arrays. */ + uint32_t n_regarrays; }; /** @@ -557,6 +560,82 @@ rte_swx_ctl_pipeline_table_fprintf(FILE *f, struct rte_swx_ctl_pipeline *ctl, const char *table_name); +/* + * Register Array Query API. + */ + +/** Register array info. */ +struct rte_swx_ctl_regarray_info { + /** Register array name. */ + char name[RTE_SWX_CTL_NAME_SIZE]; + + /** Register array size. */ + uint32_t size; +}; + +/** + * Register array info get + * + * @param[in] p + * Pipeline handle. + * @param[in] regarray_id + * Register array ID (0 .. *n_regarrays* - 1). + * @param[out] regarray + * Register array info. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument. + */ +__rte_experimental +int +rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p, + uint32_t regarray_id, + struct rte_swx_ctl_regarray_info *regarray); + +/** + * Register read + * + * @param[in] p + * Pipeline handle. + * @param[in] regarray_name + * Register array name. + * @param[in] regarray_index + * Register index within the array (0 .. *size* - 1). + * @param[out] value + * Current register value. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument. + */ +__rte_experimental +int +rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p, + const char *regarray_name, + uint32_t regarray_index, + uint64_t *value); + +/** + * Register write + * + * @param[in] p + * Pipeline handle. + * @param[in] regarray_name + * Register array name. + * @param[in] regarray_index + * Register index within the array (0 .. *size* - 1). + * @param[in] value + * Value to be written to the register. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument. + */ +__rte_experimental +int +rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p, + const char *regarray_name, + uint32_t regarray_index, + uint64_t value); + /** * Pipeline control free * diff --git a/lib/librte_pipeline/rte_swx_pipeline.c b/lib/librte_pipeline/rte_swx_pipeline.c index eaaed7a0a..cb7ee48cc 100644 --- a/lib/librte_pipeline/rte_swx_pipeline.c +++ b/lib/librte_pipeline/rte_swx_pipeline.c @@ -361,6 +361,53 @@ enum instruction_type { INSTR_ALU_SHR_MI, /* dst = MEF, src = I */ INSTR_ALU_SHR_HI, /* dst = H, src = I */ + /* regprefetch REGARRAY index + * prefetch REGARRAY[index] + * index = HMEFTI + */ + INSTR_REGPREFETCH_RH, /* index = H */ + INSTR_REGPREFETCH_RM, /* index = MEFT */ + INSTR_REGPREFETCH_RI, /* index = I */ + + /* regrd dst REGARRAY index + * dst = REGARRAY[index] + * dst = HMEF, index = HMEFTI + */ + INSTR_REGRD_HRH, /* dst = H, index = H */ + INSTR_REGRD_HRM, /* dst = H, index = MEFT */ + INSTR_REGRD_HRI, /* dst = H, index = I */ + INSTR_REGRD_MRH, /* dst = MEF, index = H */ + INSTR_REGRD_MRM, /* dst = MEF, index = MEFT */ + INSTR_REGRD_MRI, /* dst = MEF, index = I */ + + /* regwr REGARRAY index src + * REGARRAY[index] = src + * index = HMEFTI, src = HMEFTI + */ + INSTR_REGWR_RHH, /* index = H, src = H */ + INSTR_REGWR_RHM, /* index = H, src = MEFT */ + INSTR_REGWR_RHI, /* index = H, src = I */ + INSTR_REGWR_RMH, /* index = MEFT, src = H */ + INSTR_REGWR_RMM, /* index = MEFT, src = MEFT */ + INSTR_REGWR_RMI, /* index = MEFT, src = I */ + INSTR_REGWR_RIH, /* index = I, src = H */ + INSTR_REGWR_RIM, /* index = I, src = MEFT */ + INSTR_REGWR_RII, /* index = I, src = I */ + + /* regadd REGARRAY index src + * REGARRAY[index] += src + * index = HMEFTI, src = HMEFTI + */ + INSTR_REGADD_RHH, /* index = H, src = H */ + INSTR_REGADD_RHM, /* index = H, src = MEFT */ + INSTR_REGADD_RHI, /* index = H, src = I */ + INSTR_REGADD_RMH, /* index = MEFT, src = H */ + INSTR_REGADD_RMM, /* index = MEFT, src = MEFT */ + INSTR_REGADD_RMI, /* index = MEFT, src = I */ + INSTR_REGADD_RIH, /* index = I, src = H */ + INSTR_REGADD_RIM, /* index = I, src = MEFT */ + INSTR_REGADD_RII, /* index = I, src = I */ + /* table TABLE */ INSTR_TABLE, @@ -495,6 +542,21 @@ struct instr_dst_src { }; }; +struct instr_regarray { + uint8_t regarray_id; + uint8_t pad[3]; + + union { + struct instr_operand idx; + uint32_t idx_val; + }; + + union { + struct instr_operand dstsrc; + uint64_t dstsrc_val; + }; +}; + struct instr_dma { struct { uint8_t header_id[8]; @@ -529,6 +591,7 @@ struct instruction { struct instr_io io; struct instr_hdr_validity valid; struct instr_dst_src mov; + struct instr_regarray regarray; struct instr_dma dma; struct instr_dst_src alu; struct instr_table table; @@ -608,6 +671,23 @@ struct table_runtime { uint8_t **key; }; +/* + * Register array. + */ +struct regarray { + TAILQ_ENTRY(regarray) node; + char name[RTE_SWX_NAME_SIZE]; + uint32_t size; + uint32_t id; +}; + +TAILQ_HEAD(regarray_tailq, regarray); + +struct regarray_runtime { + uint64_t *regarray; + uint32_t size_mask; +}; + /* * Pipeline. */ @@ -983,11 +1063,13 @@ struct rte_swx_pipeline { struct action_tailq actions; struct table_type_tailq table_types; struct table_tailq tables; + struct regarray_tailq regarrays; struct port_in_runtime *in; struct port_out_runtime *out; struct instruction **action_instructions; struct rte_swx_table_state *table_state; + struct regarray_runtime *regarray_runtime; struct instruction *instructions; struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX]; @@ -998,6 +1080,7 @@ struct rte_swx_pipeline { uint32_t n_extern_funcs; uint32_t n_actions; uint32_t n_tables; + uint32_t n_regarrays; uint32_t n_headers; uint32_t thread_id; uint32_t port_id; @@ -4622,252 +4705,1278 @@ instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p) } /* - * jmp. + * regarray. */ -static struct action * -action_find(struct rte_swx_pipeline *p, const char *name); - -static int -instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused, - struct action *action __rte_unused, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - CHECK(n_tokens == 2, EINVAL); - - strcpy(data->jmp_label, tokens[1]); - - instr->type = INSTR_JMP; - instr->jmp.ip = NULL; /* Resolved later. */ - return 0; -} +static struct regarray * +regarray_find(struct rte_swx_pipeline *p, const char *name); static int -instr_jmp_valid_translate(struct rte_swx_pipeline *p, - struct action *action __rte_unused, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) +instr_regprefetch_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data __rte_unused) { - struct header *h; + char *regarray = tokens[1], *idx = tokens[2]; + struct regarray *r; + struct field *fidx; + uint32_t idx_struct_id, idx_val; CHECK(n_tokens == 3, EINVAL); - strcpy(data->jmp_label, tokens[1]); + r = regarray_find(p, regarray); + CHECK(r, EINVAL); + + /* REGPREFETCH_RH, REGPREFETCH_RM. */ + fidx = struct_field_parse(p, action, idx, &idx_struct_id); + if (fidx) { + instr->type = INSTR_REGPREFETCH_RM; + if (idx[0] == 'h') + instr->type = INSTR_REGPREFETCH_RH; + + instr->regarray.regarray_id = r->id; + instr->regarray.idx.struct_id = (uint8_t)idx_struct_id; + instr->regarray.idx.n_bits = fidx->n_bits; + instr->regarray.idx.offset = fidx->offset / 8; + instr->regarray.dstsrc_val = 0; /* Unused. */ + return 0; + } - h = header_parse(p, tokens[2]); - CHECK(h, EINVAL); + /* REGPREFETCH_RI. */ + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); - instr->type = INSTR_JMP_VALID; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.header_id = h->id; + instr->type = INSTR_REGPREFETCH_RI; + instr->regarray.regarray_id = r->id; + instr->regarray.idx_val = idx_val; + instr->regarray.dstsrc_val = 0; /* Unused. */ return 0; } static int -instr_jmp_invalid_translate(struct rte_swx_pipeline *p, - struct action *action __rte_unused, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) +instr_regrd_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data __rte_unused) { - struct header *h; + char *dst = tokens[1], *regarray = tokens[2], *idx = tokens[3]; + struct regarray *r; + struct field *fdst, *fidx; + uint32_t dst_struct_id, idx_struct_id, idx_val; - CHECK(n_tokens == 3, EINVAL); + CHECK(n_tokens == 4, EINVAL); - strcpy(data->jmp_label, tokens[1]); + r = regarray_find(p, regarray); + CHECK(r, EINVAL); - h = header_parse(p, tokens[2]); - CHECK(h, EINVAL); + fdst = struct_field_parse(p, NULL, dst, &dst_struct_id); + CHECK(fdst, EINVAL); - instr->type = INSTR_JMP_INVALID; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.header_id = h->id; - return 0; -} + /* REGRD_HRH, REGRD_HRM, REGRD_MRH, REGRD_MRM. */ + fidx = struct_field_parse(p, action, idx, &idx_struct_id); + if (fidx) { + instr->type = INSTR_REGRD_MRM; + if (dst[0] == 'h' && idx[0] == 'm') + instr->type = INSTR_REGRD_HRM; + if (dst[0] == 'm' && idx[0] == 'h') + instr->type = INSTR_REGRD_MRH; + if (dst[0] == 'h' && idx[0] == 'h') + instr->type = INSTR_REGRD_HRH; + + instr->regarray.regarray_id = r->id; + instr->regarray.idx.struct_id = (uint8_t)idx_struct_id; + instr->regarray.idx.n_bits = fidx->n_bits; + instr->regarray.idx.offset = fidx->offset / 8; + instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id; + instr->regarray.dstsrc.n_bits = fdst->n_bits; + instr->regarray.dstsrc.offset = fdst->offset / 8; + return 0; + } -static int -instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - CHECK(!action, EINVAL); - CHECK(n_tokens == 2, EINVAL); + /* REGRD_MRI, REGRD_HRI. */ + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); - strcpy(data->jmp_label, tokens[1]); + instr->type = INSTR_REGRD_MRI; + if (dst[0] == 'h') + instr->type = INSTR_REGRD_HRI; - instr->type = INSTR_JMP_HIT; - instr->jmp.ip = NULL; /* Resolved later. */ + instr->regarray.regarray_id = r->id; + instr->regarray.idx_val = idx_val; + instr->regarray.dstsrc.struct_id = (uint8_t)dst_struct_id; + instr->regarray.dstsrc.n_bits = fdst->n_bits; + instr->regarray.dstsrc.offset = fdst->offset / 8; return 0; } static int -instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) +instr_regwr_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data __rte_unused) { - CHECK(!action, EINVAL); - CHECK(n_tokens == 2, EINVAL); + char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3]; + struct regarray *r; + struct field *fidx, *fsrc; + uint64_t src_val; + uint32_t idx_struct_id, idx_val, src_struct_id; - strcpy(data->jmp_label, tokens[1]); + CHECK(n_tokens == 4, EINVAL); - instr->type = INSTR_JMP_MISS; - instr->jmp.ip = NULL; /* Resolved later. */ - return 0; -} + r = regarray_find(p, regarray); + CHECK(r, EINVAL); -static int -instr_jmp_action_hit_translate(struct rte_swx_pipeline *p, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - struct action *a; + /* REGWR_RHH, REGWR_RHM, REGWR_RMH, REGWR_RMM. */ + fidx = struct_field_parse(p, action, idx, &idx_struct_id); + fsrc = struct_field_parse(p, action, src, &src_struct_id); + if (fidx && fsrc) { + instr->type = INSTR_REGWR_RMM; + if (idx[0] == 'h' && src[0] == 'm') + instr->type = INSTR_REGWR_RHM; + if (idx[0] == 'm' && src[0] == 'h') + instr->type = INSTR_REGWR_RMH; + if (idx[0] == 'h' && src[0] == 'h') + instr->type = INSTR_REGWR_RHH; + + instr->regarray.regarray_id = r->id; + instr->regarray.idx.struct_id = (uint8_t)idx_struct_id; + instr->regarray.idx.n_bits = fidx->n_bits; + instr->regarray.idx.offset = fidx->offset / 8; + instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id; + instr->regarray.dstsrc.n_bits = fsrc->n_bits; + instr->regarray.dstsrc.offset = fsrc->offset / 8; + return 0; + } - CHECK(!action, EINVAL); - CHECK(n_tokens == 3, EINVAL); + /* REGWR_RHI, REGWR_RMI. */ + if (fidx && !fsrc) { + src_val = strtoull(src, &src, 0); + CHECK(!src[0], EINVAL); - strcpy(data->jmp_label, tokens[1]); + instr->type = INSTR_REGWR_RMI; + if (idx[0] == 'h') + instr->type = INSTR_REGWR_RHI; - a = action_find(p, tokens[2]); - CHECK(a, EINVAL); + instr->regarray.regarray_id = r->id; + instr->regarray.idx.struct_id = (uint8_t)idx_struct_id; + instr->regarray.idx.n_bits = fidx->n_bits; + instr->regarray.idx.offset = fidx->offset / 8; + instr->regarray.dstsrc_val = src_val; + return 0; + } - instr->type = INSTR_JMP_ACTION_HIT; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.action_id = a->id; - return 0; -} + /* REGWR_RIH, REGWR_RIM. */ + if (!fidx && fsrc) { + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); -static int -instr_jmp_action_miss_translate(struct rte_swx_pipeline *p, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - struct action *a; + instr->type = INSTR_REGWR_RIM; + if (idx[0] == 'h') + instr->type = INSTR_REGWR_RIH; - CHECK(!action, EINVAL); - CHECK(n_tokens == 3, EINVAL); + instr->regarray.regarray_id = r->id; + instr->regarray.idx_val = idx_val; + instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id; + instr->regarray.dstsrc.n_bits = fsrc->n_bits; + instr->regarray.dstsrc.offset = fsrc->offset / 8; + return 0; + } - strcpy(data->jmp_label, tokens[1]); + /* REGWR_RII. */ + src_val = strtoull(src, &src, 0); + CHECK(!src[0], EINVAL); - a = action_find(p, tokens[2]); - CHECK(a, EINVAL); + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); + + instr->type = INSTR_REGWR_RII; + instr->regarray.idx_val = idx_val; + instr->regarray.dstsrc_val = src_val; - instr->type = INSTR_JMP_ACTION_MISS; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.action_id = a->id; return 0; } static int -instr_jmp_eq_translate(struct rte_swx_pipeline *p, +instr_regadd_translate(struct rte_swx_pipeline *p, struct action *action, char **tokens, int n_tokens, struct instruction *instr, - struct instruction_data *data) + struct instruction_data *data __rte_unused) { - char *a = tokens[2], *b = tokens[3]; - struct field *fa, *fb; - uint64_t b_val; - uint32_t a_struct_id, b_struct_id; + char *regarray = tokens[1], *idx = tokens[2], *src = tokens[3]; + struct regarray *r; + struct field *fidx, *fsrc; + uint64_t src_val; + uint32_t idx_struct_id, idx_val, src_struct_id; CHECK(n_tokens == 4, EINVAL); - strcpy(data->jmp_label, tokens[1]); + r = regarray_find(p, regarray); + CHECK(r, EINVAL); - fa = struct_field_parse(p, action, a, &a_struct_id); - CHECK(fa, EINVAL); + /* REGADD_RHH, REGADD_RHM, REGADD_RMH, REGADD_RMM. */ + fidx = struct_field_parse(p, action, idx, &idx_struct_id); + fsrc = struct_field_parse(p, action, src, &src_struct_id); + if (fidx && fsrc) { + instr->type = INSTR_REGADD_RMM; + if (idx[0] == 'h' && src[0] == 'm') + instr->type = INSTR_REGADD_RHM; + if (idx[0] == 'm' && src[0] == 'h') + instr->type = INSTR_REGADD_RMH; + if (idx[0] == 'h' && src[0] == 'h') + instr->type = INSTR_REGADD_RHH; + + instr->regarray.regarray_id = r->id; + instr->regarray.idx.struct_id = (uint8_t)idx_struct_id; + instr->regarray.idx.n_bits = fidx->n_bits; + instr->regarray.idx.offset = fidx->offset / 8; + instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id; + instr->regarray.dstsrc.n_bits = fsrc->n_bits; + instr->regarray.dstsrc.offset = fsrc->offset / 8; + return 0; + } - /* JMP_EQ or JMP_EQ_S. */ - fb = struct_field_parse(p, action, b, &b_struct_id); - if (fb) { - instr->type = INSTR_JMP_EQ; - if ((a[0] == 'h' && b[0] != 'h') || - (a[0] != 'h' && b[0] == 'h')) - instr->type = INSTR_JMP_EQ_S; - instr->jmp.ip = NULL; /* Resolved later. */ + /* REGADD_RHI, REGADD_RMI. */ + if (fidx && !fsrc) { + src_val = strtoull(src, &src, 0); + CHECK(!src[0], EINVAL); - instr->jmp.a.struct_id = (uint8_t)a_struct_id; - instr->jmp.a.n_bits = fa->n_bits; - instr->jmp.a.offset = fa->offset / 8; - instr->jmp.b.struct_id = (uint8_t)b_struct_id; - instr->jmp.b.n_bits = fb->n_bits; - instr->jmp.b.offset = fb->offset / 8; + instr->type = INSTR_REGADD_RMI; + if (idx[0] == 'h') + instr->type = INSTR_REGADD_RHI; + + instr->regarray.regarray_id = r->id; + instr->regarray.idx.struct_id = (uint8_t)idx_struct_id; + instr->regarray.idx.n_bits = fidx->n_bits; + instr->regarray.idx.offset = fidx->offset / 8; + instr->regarray.dstsrc_val = src_val; return 0; } - /* JMP_EQ_I. */ - b_val = strtoull(b, &b, 0); - CHECK(!b[0], EINVAL); + /* REGADD_RIH, REGADD_RIM. */ + if (!fidx && fsrc) { + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); - if (a[0] == 'h') - b_val = hton64(b_val) >> (64 - fa->n_bits); + instr->type = INSTR_REGADD_RIM; + if (idx[0] == 'h') + instr->type = INSTR_REGADD_RIH; - instr->type = INSTR_JMP_EQ_I; - instr->jmp.ip = NULL; /* Resolved later. */ - instr->jmp.a.struct_id = (uint8_t)a_struct_id; - instr->jmp.a.n_bits = fa->n_bits; - instr->jmp.a.offset = fa->offset / 8; - instr->jmp.b_val = b_val; + instr->regarray.regarray_id = r->id; + instr->regarray.idx_val = idx_val; + instr->regarray.dstsrc.struct_id = (uint8_t)src_struct_id; + instr->regarray.dstsrc.n_bits = fsrc->n_bits; + instr->regarray.dstsrc.offset = fsrc->offset / 8; + return 0; + } + + /* REGADD_RII. */ + src_val = strtoull(src, &src, 0); + CHECK(!src[0], EINVAL); + + idx_val = strtoul(idx, &idx, 0); + CHECK(!idx[0], EINVAL); + + instr->type = INSTR_REGADD_RII; + instr->regarray.idx_val = idx_val; + instr->regarray.dstsrc_val = src_val; return 0; } -static int -instr_jmp_neq_translate(struct rte_swx_pipeline *p, - struct action *action, - char **tokens, - int n_tokens, - struct instruction *instr, - struct instruction_data *data) -{ - char *a = tokens[2], *b = tokens[3]; - struct field *fa, *fb; - uint64_t b_val; - uint32_t a_struct_id, b_struct_id; +#define REGPREFETCH_RM(pipeline, thread, ip) \ +{ \ + struct regarray_runtime *r = &(pipeline)->regarray_runtime[(ip)->regarray.regarray_id]; \ + uint64_t *regarray = r->regarray; \ + \ + uint8_t *idx_struct = (thread)->structs[(ip)->regarray.idx.struct_id]; \ + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[(ip)->regarray.idx.offset]; \ + uint64_t idx64 = *idx64_ptr; \ + uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->regarray.idx.n_bits); \ + uint64_t idx = idx64 & idx64_mask & r->size_mask; \ + \ + rte_prefetch0(®array[idx]); \ +} - CHECK(n_tokens == 4, EINVAL); +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN - strcpy(data->jmp_label, tokens[1]); +#define REGPREFETCH_RH(pipeline, thread, ip) \ +{ \ + struct regarray_runtime *r = &(pipeline)->regarray_runtime[(ip)->regarray.regarray_id]; \ + uint64_t *regarray = r->regarray; \ + \ + uint8_t *idx_struct = (thread)->structs[(ip)->regarray.idx.struct_id]; \ + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[(ip)->regarray.idx.offset]; \ + uint64_t idx64 = *idx64_ptr; \ + uint64_t idx = (ntoh64(idx64) >> (64 - (ip)->regarray.idx.n_bits)) & r->size_mask; \ + \ + rte_prefetch0(®array[idx]); \ +} - fa = struct_field_parse(p, action, a, &a_struct_id); - CHECK(fa, EINVAL); +#else - /* JMP_NEQ or JMP_NEQ_S. */ - fb = struct_field_parse(p, action, b, &b_struct_id); - if (fb) { - instr->type = INSTR_JMP_NEQ; - if ((a[0] == 'h' && b[0] != 'h') || - (a[0] != 'h' && b[0] == 'h')) - instr->type = INSTR_JMP_NEQ_S; - instr->jmp.ip = NULL; /* Resolved later. */ +#define REGPREFETCH_RH REGPREFETCH_RM - instr->jmp.a.struct_id = (uint8_t)a_struct_id; - instr->jmp.a.n_bits = fa->n_bits; - instr->jmp.a.offset = fa->offset / 8; - instr->jmp.b.struct_id = (uint8_t)b_struct_id; - instr->jmp.b.n_bits = fb->n_bits; - instr->jmp.b.offset = fb->offset / 8; - return 0; - } +#endif - /* JMP_NEQ_I. */ - b_val = strtoull(b, &b, 0); +#define REGPREFETCH_RI(pipeline, thread, ip) \ +{ \ + struct regarray_runtime *r = &(pipeline)->regarray_runtime[(ip)->regarray.regarray_id]; \ + uint64_t *regarray = r->regarray; \ + \ + uint64_t idx = (ip)->regarray.idx_val & r->size_mask; \ + \ + rte_prefetch0(®array[idx]); \ +} + +static inline void +instr_regprefetch_rh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id); + + /* Structs. */ + REGPREFETCH_RH(p, t, ip); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regprefetch_rm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id); + + /* Structs. */ + REGPREFETCH_RM(p, t, ip); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regprefetch_ri_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id); + + /* Structs. */ + REGPREFETCH_RI(p, t, ip); + + /* Thread. */ + thread_ip_inc(p); +} + +#define REGRD_MRM(pipeline, thread, ip) \ +{ \ + struct regarray_runtime *r = &(pipeline)->regarray_runtime[(ip)->regarray.regarray_id]; \ + uint64_t *regarray = r->regarray; \ + \ + uint8_t *idx_struct = (thread)->structs[(ip)->regarray.idx.struct_id]; \ + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[(ip)->regarray.idx.offset]; \ + uint64_t idx64 = *idx64_ptr; \ + uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->regarray.idx.n_bits); \ + uint64_t idx = idx64 & idx64_mask & r->size_mask; \ + \ + uint8_t *dstsrc_struct = (thread)->structs[(ip)->regarray.dstsrc.struct_id]; \ + uint64_t *dstsrc64_ptr = (uint64_t *)&dstsrc_struct[(ip)->regarray.dstsrc.offset]; \ + uint64_t dstsrc64 = *dstsrc64_ptr; \ + uint64_t dstsrc64_mask = UINT64_MAX >> (64 - (ip)->regarray.dstsrc.n_bits); \ + \ + *dstsrc64_ptr = (dstsrc64 & ~dstsrc64_mask) | (regarray[idx] & dstsrc64_mask); \ +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +#define REGRD_MRH(pipeline, thread, ip) \ +{ \ + struct regarray_runtime *r = &(pipeline)->regarray_runtime[(ip)->regarray.regarray_id]; \ + uint64_t *regarray = r->regarray; \ + \ + uint8_t *idx_struct = (thread)->structs[(ip)->regarray.idx.struct_id]; \ + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[(ip)->regarray.idx.offset]; \ + uint64_t idx64 = *idx64_ptr; \ + uint64_t idx = (ntoh64(idx64) >> (64 - (ip)->regarray.idx.n_bits)) & r->size_mask; \ + \ + uint8_t *dstsrc_struct = (thread)->structs[(ip)->regarray.dstsrc.struct_id]; \ + uint64_t *dstsrc64_ptr = (uint64_t *)&dstsrc_struct[(ip)->regarray.dstsrc.offset]; \ + uint64_t dstsrc64 = *dstsrc64_ptr; \ + uint64_t dstsrc64_mask = UINT64_MAX >> (64 - (ip)->regarray.dstsrc.n_bits); \ + \ + *dstsrc64_ptr = (dstsrc64 & ~dstsrc64_mask) | (regarray[idx] & dstsrc64_mask); \ +} + +#define REGRD_HRM(pipeline, thread, ip) \ +{ \ + struct regarray_runtime *r = &(pipeline)->regarray_runtime[(ip)->regarray.regarray_id]; \ + uint64_t *regarray = r->regarray; \ + \ + uint8_t *idx_struct = (thread)->structs[(ip)->regarray.idx.struct_id]; \ + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[(ip)->regarray.idx.offset]; \ + uint64_t idx64 = *idx64_ptr; \ + uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->regarray.idx.n_bits); \ + uint64_t idx = idx64 & idx64_mask & r->size_mask; \ + \ + uint8_t *dstsrc_struct = (thread)->structs[(ip)->regarray.dstsrc.struct_id]; \ + uint64_t *dstsrc64_ptr = (uint64_t *)&dstsrc_struct[(ip)->regarray.dstsrc.offset]; \ + uint64_t dstsrc64 = *dstsrc64_ptr; \ + uint64_t dstsrc64_mask = UINT64_MAX >> (64 - (ip)->regarray.dstsrc.n_bits); \ + \ + *dstsrc64_ptr = (dstsrc64 & ~dstsrc64_mask) | (hton64(regarray[idx]) & dstsrc64_mask); \ +} + +#define REGRD_HRH(pipeline, thread, ip) \ +{ \ + struct regarray_runtime *r = &(pipeline)->regarray_runtime[(ip)->regarray.regarray_id]; \ + uint64_t *regarray = r->regarray; \ + \ + uint8_t *idx_struct = (thread)->structs[(ip)->regarray.idx.struct_id]; \ + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[(ip)->regarray.idx.offset]; \ + uint64_t idx64 = *idx64_ptr; \ + uint64_t idx = (ntoh64(idx64) >> (64 - (ip)->regarray.idx.n_bits)) & r->size_mask; \ + \ + uint8_t *dstsrc_struct = (thread)->structs[(ip)->regarray.dstsrc.struct_id]; \ + uint64_t *dstsrc64_ptr = (uint64_t *)&dstsrc_struct[(ip)->regarray.dstsrc.offset]; \ + uint64_t dstsrc64 = *dstsrc64_ptr; \ + uint64_t dstsrc64_mask = UINT64_MAX >> (64 - (ip)->regarray.dstsrc.n_bits); \ + \ + *dstsrc64_ptr = (dstsrc64 & ~dstsrc64_mask) | (hton64(regarray[idx]) & dstsrc64_mask); \ +} + +#else + +#define REGRD_MRH REGRD_MRM +#define REGRD_HRM REGRD_MRM +#define REGRD_HRH REGRD_MRM + +#endif + +#define REGRD_MRI(pipeline, thread, ip) \ +{ \ + struct regarray_runtime *r = &(pipeline)->regarray_runtime[(ip)->regarray.regarray_id]; \ + uint64_t *regarray = r->regarray; \ + \ + uint64_t idx = (ip)->regarray.idx_val & r->size_mask; \ + \ + uint8_t *dstsrc_struct = (thread)->structs[(ip)->regarray.dstsrc.struct_id]; \ + uint64_t *dstsrc64_ptr = (uint64_t *)&dstsrc_struct[(ip)->regarray.dstsrc.offset]; \ + uint64_t dstsrc64 = *dstsrc64_ptr; \ + uint64_t dstsrc64_mask = UINT64_MAX >> (64 - (ip)->regarray.dstsrc.n_bits); \ + \ + *dstsrc64_ptr = (dstsrc64 & ~dstsrc64_mask) | (regarray[idx] & dstsrc64_mask); \ +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +#define REGRD_HRI(pipeline, thread, ip) \ +{ \ + struct regarray_runtime *r = &(pipeline)->regarray_runtime[(ip)->regarray.regarray_id]; \ + uint64_t *regarray = r->regarray; \ + \ + uint64_t idx = (ip)->regarray.idx_val & r->size_mask; \ + \ + uint8_t *dstsrc_struct = (thread)->structs[(ip)->regarray.dstsrc.struct_id]; \ + uint64_t *dstsrc64_ptr = (uint64_t *)&dstsrc_struct[(ip)->regarray.dstsrc.offset]; \ + uint64_t dstsrc64 = *dstsrc64_ptr; \ + uint64_t dstsrc64_mask = UINT64_MAX >> (64 - (ip)->regarray.dstsrc.n_bits); \ + \ + *dstsrc64_ptr = (dstsrc64 & ~dstsrc64_mask) | (hton64(regarray[idx]) & dstsrc64_mask); \ +} + +#else + +#define REGRD_HRI REGRD_MRI + +#endif + +static inline void +instr_regrd_hrh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id); + + /* Structs. */ + REGRD_HRH(p, t, ip); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regrd_hrm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id); + + /* Structs. */ + REGRD_HRM(p, t, ip); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regrd_mrh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id); + + /* Structs. */ + REGRD_MRH(p, t, ip); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regrd_mrm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regrd (m = r[m])\n", p->thread_id); + + /* Structs. */ + REGRD_MRM(p, t, ip); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regrd_hri_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id); + + /* Structs. */ + REGRD_HRI(p, t, ip); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regrd_mri_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id); + + /* Structs. */ + REGRD_MRI(p, t, ip); + + /* Thread. */ + thread_ip_inc(p); +} + +#define REGWR_RMM(pipeline, thread, ip, operator) \ +{ \ + struct regarray_runtime *r = &(pipeline)->regarray_runtime[(ip)->regarray.regarray_id]; \ + uint64_t *regarray = r->regarray; \ + \ + uint8_t *idx_struct = (thread)->structs[(ip)->regarray.idx.struct_id]; \ + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[(ip)->regarray.idx.offset]; \ + uint64_t idx64 = *idx64_ptr; \ + uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->regarray.idx.n_bits); \ + uint64_t idx = idx64 & idx64_mask & r->size_mask; \ + \ + uint8_t *dstsrc_struct = (thread)->structs[(ip)->regarray.dstsrc.struct_id]; \ + uint64_t *dstsrc64_ptr = (uint64_t *)&dstsrc_struct[(ip)->regarray.dstsrc.offset]; \ + uint64_t dstsrc64 = *dstsrc64_ptr; \ + uint64_t dstsrc64_mask = UINT64_MAX >> (64 - (ip)->regarray.dstsrc.n_bits); \ + uint64_t dstsrc = dstsrc64 & dstsrc64_mask; \ + \ + regarray[idx] operator dstsrc; \ +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +#define REGWR_RMH(pipeline, thread, ip, operator) \ +{ \ + struct regarray_runtime *r = &(pipeline)->regarray_runtime[(ip)->regarray.regarray_id]; \ + uint64_t *regarray = r->regarray; \ + \ + uint8_t *idx_struct = (thread)->structs[(ip)->regarray.idx.struct_id]; \ + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[(ip)->regarray.idx.offset]; \ + uint64_t idx64 = *idx64_ptr; \ + uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->regarray.idx.n_bits); \ + uint64_t idx = idx64 & idx64_mask & r->size_mask; \ + \ + uint8_t *dstsrc_struct = (thread)->structs[(ip)->regarray.dstsrc.struct_id]; \ + uint64_t *dstsrc64_ptr = (uint64_t *)&dstsrc_struct[(ip)->regarray.dstsrc.offset]; \ + uint64_t dstsrc64 = *dstsrc64_ptr; \ + uint64_t dstsrc = ntoh64(dstsrc64) >> (64 - (ip)->regarray.dstsrc.n_bits); \ + \ + regarray[idx] operator dstsrc; \ +} + +#define REGWR_RHM(pipeline, thread, ip, operator) \ +{ \ + struct regarray_runtime *r = &(pipeline)->regarray_runtime[(ip)->regarray.regarray_id]; \ + uint64_t *regarray = r->regarray; \ + \ + uint8_t *idx_struct = (thread)->structs[(ip)->regarray.idx.struct_id]; \ + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[(ip)->regarray.idx.offset]; \ + uint64_t idx64 = *idx64_ptr; \ + uint64_t idx = (ntoh64(idx64) >> (64 - (ip)->regarray.idx.n_bits)) & r->size_mask; \ + \ + uint8_t *dstsrc_struct = (thread)->structs[(ip)->regarray.dstsrc.struct_id]; \ + uint64_t *dstsrc64_ptr = (uint64_t *)&dstsrc_struct[(ip)->regarray.dstsrc.offset]; \ + uint64_t dstsrc64 = *dstsrc64_ptr; \ + uint64_t dstsrc64_mask = UINT64_MAX >> (64 - (ip)->regarray.dstsrc.n_bits); \ + uint64_t dstsrc = dstsrc64 & dstsrc64_mask; \ + \ + regarray[idx] operator dstsrc; \ +} + +#define REGWR_RHH(pipeline, thread, ip, operator) \ +{ \ + struct regarray_runtime *r = &(pipeline)->regarray_runtime[(ip)->regarray.regarray_id]; \ + uint64_t *regarray = r->regarray; \ + \ + uint8_t *idx_struct = (thread)->structs[(ip)->regarray.idx.struct_id]; \ + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[(ip)->regarray.idx.offset]; \ + uint64_t idx64 = *idx64_ptr; \ + uint64_t idx = (ntoh64(idx64) >> (64 - (ip)->regarray.idx.n_bits)) & r->size_mask; \ + \ + uint8_t *dstsrc_struct = (thread)->structs[(ip)->regarray.dstsrc.struct_id]; \ + uint64_t *dstsrc64_ptr = (uint64_t *)&dstsrc_struct[(ip)->regarray.dstsrc.offset]; \ + uint64_t dstsrc64 = *dstsrc64_ptr; \ + uint64_t dstsrc = ntoh64(dstsrc64) >> (64 - (ip)->regarray.dstsrc.n_bits); \ + \ + regarray[idx] operator dstsrc; \ +} + +#else + +#define REGWR_RMH REGWR_RMM +#define REGWR_RHM REGWR_RMM +#define REGWR_RHH REGWR_RMM + +#endif + +#define REGWR_RMI(pipeline, thread, ip, operator) \ +{ \ + struct regarray_runtime *r = &(pipeline)->regarray_runtime[(ip)->regarray.regarray_id]; \ + uint64_t *regarray = r->regarray; \ + \ + uint8_t *idx_struct = (thread)->structs[(ip)->regarray.idx.struct_id]; \ + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[(ip)->regarray.idx.offset]; \ + uint64_t idx64 = *idx64_ptr; \ + uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->regarray.idx.n_bits); \ + uint64_t idx = idx64 & idx64_mask & r->size_mask; \ + \ + uint64_t dstsrc = (ip)->regarray.dstsrc_val; \ + \ + regarray[idx] operator dstsrc; \ +} + +#define REGWR_RIM(pipeline, thread, ip, operator) \ +{ \ + struct regarray_runtime *r = &(pipeline)->regarray_runtime[(ip)->regarray.regarray_id]; \ + uint64_t *regarray = r->regarray; \ + \ + uint64_t idx = (ip)->regarray.idx_val & r->size_mask; \ + \ + uint8_t *dstsrc_struct = (thread)->structs[(ip)->regarray.dstsrc.struct_id]; \ + uint64_t *dstsrc64_ptr = (uint64_t *)&dstsrc_struct[(ip)->regarray.dstsrc.offset]; \ + uint64_t dstsrc64 = *dstsrc64_ptr; \ + uint64_t dstsrc64_mask = UINT64_MAX >> (64 - (ip)->regarray.dstsrc.n_bits); \ + uint64_t dstsrc = dstsrc64 & dstsrc64_mask; \ + \ + regarray[idx] operator dstsrc; \ +} + +#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN + +#define REGWR_RHI(pipeline, thread, ip, operator) \ +{ \ + struct regarray_runtime *r = &(pipeline)->regarray_runtime[(ip)->regarray.regarray_id]; \ + uint64_t *regarray = r->regarray; \ + \ + uint8_t *idx_struct = (thread)->structs[(ip)->regarray.idx.struct_id]; \ + uint64_t *idx64_ptr = (uint64_t *)&idx_struct[(ip)->regarray.idx.offset]; \ + uint64_t idx64 = *idx64_ptr; \ + uint64_t idx = (ntoh64(idx64) >> (64 - (ip)->regarray.idx.n_bits)) & r->size_mask; \ + \ + uint64_t dstsrc = (ip)->regarray.dstsrc_val; \ + \ + regarray[idx] operator dstsrc; \ +} + +#define REGWR_RIH(pipeline, thread, ip, operator) \ +{ \ + struct regarray_runtime *r = &(pipeline)->regarray_runtime[(ip)->regarray.regarray_id]; \ + uint64_t *regarray = r->regarray; \ + \ + uint64_t idx = (ip)->regarray.idx_val & r->size_mask; \ + \ + uint8_t *dstsrc_struct = (thread)->structs[(ip)->regarray.dstsrc.struct_id]; \ + uint64_t *dstsrc64_ptr = (uint64_t *)&dstsrc_struct[(ip)->regarray.dstsrc.offset]; \ + uint64_t dstsrc64 = *dstsrc64_ptr; \ + uint64_t dstsrc = ntoh64(dstsrc64) >> (64 - (ip)->regarray.dstsrc.n_bits); \ + \ + regarray[idx] operator dstsrc; \ +} + +#else + +#define REGWR_RHI REGWR_RMI +#define REGWR_RIH REGWR_RIM + +#endif + +#define REGWR_RII(pipeline, thread, ip, operator) \ +{ \ + struct regarray_runtime *r = &(pipeline)->regarray_runtime[(ip)->regarray.regarray_id]; \ + uint64_t *regarray = r->regarray; \ + \ + uint64_t idx = (ip)->regarray.idx_val & r->size_mask; \ + \ + uint64_t dstsrc = (ip)->regarray.dstsrc_val; \ + \ + regarray[idx] operator dstsrc; \ +} + +static inline void +instr_regwr_rhh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id); + + /* Structs. */ + REGWR_RHH(p, t, ip, =); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rhm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id); + + /* Structs. */ + REGWR_RHM(p, t, ip, =); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rmh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id); + + /* Structs. */ + REGWR_RMH(p, t, ip, =); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rmm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id); + + /* Structs. */ + REGWR_RMM(p, t, ip, =); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rhi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id); + + /* Structs. */ + REGWR_RHI(p, t, ip, =); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rmi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id); + + /* Structs. */ + REGWR_RMI(p, t, ip, =); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rih_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id); + + /* Structs. */ + REGWR_RIH(p, t, ip, =); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rim_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id); + + /* Structs. */ + REGWR_RIM(p, t, ip, =); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regwr_rii_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id); + + /* Structs. */ + REGWR_RII(p, t, ip, =); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rhh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id); + + /* Structs. */ + REGWR_RHH(p, t, ip, +=); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rhm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id); + + /* Structs. */ + REGWR_RHM(p, t, ip, +=); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rmh_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id); + + /* Structs. */ + REGWR_RMH(p, t, ip, +=); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rmm_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id); + + /* Structs. */ + REGWR_RMM(p, t, ip, +=); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rhi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id); + + /* Structs. */ + REGWR_RHI(p, t, ip, +=); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rmi_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id); + + /* Structs. */ + REGWR_RMI(p, t, ip, +=); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rih_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id); + + /* Structs. */ + REGWR_RIH(p, t, ip, +=); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rim_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id); + + /* Structs. */ + REGWR_RIM(p, t, ip, +=); + + /* Thread. */ + thread_ip_inc(p); +} + +static inline void +instr_regadd_rii_exec(struct rte_swx_pipeline *p) +{ + struct thread *t = &p->threads[p->thread_id]; + struct instruction *ip = t->ip; + + TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id); + + /* Structs. */ + REGWR_RII(p, t, ip, +=); + + /* Thread. */ + thread_ip_inc(p); +} + +/* + * jmp. + */ +static struct action * +action_find(struct rte_swx_pipeline *p, const char *name); + +static int +instr_jmp_translate(struct rte_swx_pipeline *p __rte_unused, + struct action *action __rte_unused, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + CHECK(n_tokens == 2, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + instr->type = INSTR_JMP; + instr->jmp.ip = NULL; /* Resolved later. */ + return 0; +} + +static int +instr_jmp_valid_translate(struct rte_swx_pipeline *p, + struct action *action __rte_unused, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + struct header *h; + + CHECK(n_tokens == 3, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + h = header_parse(p, tokens[2]); + CHECK(h, EINVAL); + + instr->type = INSTR_JMP_VALID; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.header_id = h->id; + return 0; +} + +static int +instr_jmp_invalid_translate(struct rte_swx_pipeline *p, + struct action *action __rte_unused, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + struct header *h; + + CHECK(n_tokens == 3, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + h = header_parse(p, tokens[2]); + CHECK(h, EINVAL); + + instr->type = INSTR_JMP_INVALID; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.header_id = h->id; + return 0; +} + +static int +instr_jmp_hit_translate(struct rte_swx_pipeline *p __rte_unused, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + CHECK(!action, EINVAL); + CHECK(n_tokens == 2, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + instr->type = INSTR_JMP_HIT; + instr->jmp.ip = NULL; /* Resolved later. */ + return 0; +} + +static int +instr_jmp_miss_translate(struct rte_swx_pipeline *p __rte_unused, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + CHECK(!action, EINVAL); + CHECK(n_tokens == 2, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + instr->type = INSTR_JMP_MISS; + instr->jmp.ip = NULL; /* Resolved later. */ + return 0; +} + +static int +instr_jmp_action_hit_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + struct action *a; + + CHECK(!action, EINVAL); + CHECK(n_tokens == 3, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + a = action_find(p, tokens[2]); + CHECK(a, EINVAL); + + instr->type = INSTR_JMP_ACTION_HIT; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.action_id = a->id; + return 0; +} + +static int +instr_jmp_action_miss_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + struct action *a; + + CHECK(!action, EINVAL); + CHECK(n_tokens == 3, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + a = action_find(p, tokens[2]); + CHECK(a, EINVAL); + + instr->type = INSTR_JMP_ACTION_MISS; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.action_id = a->id; + return 0; +} + +static int +instr_jmp_eq_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + char *a = tokens[2], *b = tokens[3]; + struct field *fa, *fb; + uint64_t b_val; + uint32_t a_struct_id, b_struct_id; + + CHECK(n_tokens == 4, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + fa = struct_field_parse(p, action, a, &a_struct_id); + CHECK(fa, EINVAL); + + /* JMP_EQ or JMP_EQ_S. */ + fb = struct_field_parse(p, action, b, &b_struct_id); + if (fb) { + instr->type = INSTR_JMP_EQ; + if ((a[0] == 'h' && b[0] != 'h') || + (a[0] != 'h' && b[0] == 'h')) + instr->type = INSTR_JMP_EQ_S; + instr->jmp.ip = NULL; /* Resolved later. */ + + instr->jmp.a.struct_id = (uint8_t)a_struct_id; + instr->jmp.a.n_bits = fa->n_bits; + instr->jmp.a.offset = fa->offset / 8; + instr->jmp.b.struct_id = (uint8_t)b_struct_id; + instr->jmp.b.n_bits = fb->n_bits; + instr->jmp.b.offset = fb->offset / 8; + return 0; + } + + /* JMP_EQ_I. */ + b_val = strtoull(b, &b, 0); + CHECK(!b[0], EINVAL); + + if (a[0] == 'h') + b_val = hton64(b_val) >> (64 - fa->n_bits); + + instr->type = INSTR_JMP_EQ_I; + instr->jmp.ip = NULL; /* Resolved later. */ + instr->jmp.a.struct_id = (uint8_t)a_struct_id; + instr->jmp.a.n_bits = fa->n_bits; + instr->jmp.a.offset = fa->offset / 8; + instr->jmp.b_val = b_val; + return 0; +} + +static int +instr_jmp_neq_translate(struct rte_swx_pipeline *p, + struct action *action, + char **tokens, + int n_tokens, + struct instruction *instr, + struct instruction_data *data) +{ + char *a = tokens[2], *b = tokens[3]; + struct field *fa, *fb; + uint64_t b_val; + uint32_t a_struct_id, b_struct_id; + + CHECK(n_tokens == 4, EINVAL); + + strcpy(data->jmp_label, tokens[1]); + + fa = struct_field_parse(p, action, a, &a_struct_id); + CHECK(fa, EINVAL); + + /* JMP_NEQ or JMP_NEQ_S. */ + fb = struct_field_parse(p, action, b, &b_struct_id); + if (fb) { + instr->type = INSTR_JMP_NEQ; + if ((a[0] == 'h' && b[0] != 'h') || + (a[0] != 'h' && b[0] == 'h')) + instr->type = INSTR_JMP_NEQ_S; + instr->jmp.ip = NULL; /* Resolved later. */ + + instr->jmp.a.struct_id = (uint8_t)a_struct_id; + instr->jmp.a.n_bits = fa->n_bits; + instr->jmp.a.offset = fa->offset / 8; + instr->jmp.b.struct_id = (uint8_t)b_struct_id; + instr->jmp.b.n_bits = fb->n_bits; + instr->jmp.b.offset = fb->offset / 8; + return 0; + } + + /* JMP_NEQ_I. */ + b_val = strtoull(b, &b, 0); CHECK(!b[0], EINVAL); if (a[0] == 'h') @@ -5475,6 +6584,38 @@ instr_translate(struct rte_swx_pipeline *p, instr, data); + if (!strcmp(tokens[tpos], "regprefetch")) + return instr_regprefetch_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + + if (!strcmp(tokens[tpos], "regrd")) + return instr_regrd_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + + if (!strcmp(tokens[tpos], "regwr")) + return instr_regwr_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + + if (!strcmp(tokens[tpos], "regadd")) + return instr_regadd_translate(p, + action, + &tokens[tpos], + n_tokens - tpos, + instr, + data); + if (!strcmp(tokens[tpos], "table")) return instr_table_translate(p, action, @@ -6111,6 +7252,37 @@ static instr_exec_t instruction_table[] = { [INSTR_ALU_SHR_MI] = instr_alu_shr_mi_exec, [INSTR_ALU_SHR_HI] = instr_alu_shr_hi_exec, + [INSTR_REGPREFETCH_RH] = instr_regprefetch_rh_exec, + [INSTR_REGPREFETCH_RM] = instr_regprefetch_rm_exec, + [INSTR_REGPREFETCH_RI] = instr_regprefetch_ri_exec, + + [INSTR_REGRD_HRH] = instr_regrd_hrh_exec, + [INSTR_REGRD_HRM] = instr_regrd_hrm_exec, + [INSTR_REGRD_MRH] = instr_regrd_mrh_exec, + [INSTR_REGRD_MRM] = instr_regrd_mrm_exec, + [INSTR_REGRD_HRI] = instr_regrd_hri_exec, + [INSTR_REGRD_MRI] = instr_regrd_mri_exec, + + [INSTR_REGWR_RHH] = instr_regwr_rhh_exec, + [INSTR_REGWR_RHM] = instr_regwr_rhm_exec, + [INSTR_REGWR_RMH] = instr_regwr_rmh_exec, + [INSTR_REGWR_RMM] = instr_regwr_rmm_exec, + [INSTR_REGWR_RHI] = instr_regwr_rhi_exec, + [INSTR_REGWR_RMI] = instr_regwr_rmi_exec, + [INSTR_REGWR_RIH] = instr_regwr_rih_exec, + [INSTR_REGWR_RIM] = instr_regwr_rim_exec, + [INSTR_REGWR_RII] = instr_regwr_rii_exec, + + [INSTR_REGADD_RHH] = instr_regadd_rhh_exec, + [INSTR_REGADD_RHM] = instr_regadd_rhm_exec, + [INSTR_REGADD_RMH] = instr_regadd_rmh_exec, + [INSTR_REGADD_RMM] = instr_regadd_rmm_exec, + [INSTR_REGADD_RHI] = instr_regadd_rhi_exec, + [INSTR_REGADD_RMI] = instr_regadd_rmi_exec, + [INSTR_REGADD_RIH] = instr_regadd_rih_exec, + [INSTR_REGADD_RIM] = instr_regadd_rim_exec, + [INSTR_REGADD_RII] = instr_regadd_rii_exec, + [INSTR_TABLE] = instr_table_exec, [INSTR_EXTERN_OBJ] = instr_extern_obj_exec, [INSTR_EXTERN_FUNC] = instr_extern_func_exec, @@ -6823,6 +7995,119 @@ table_free(struct rte_swx_pipeline *p) } } +/* + * Register array. + */ +static struct regarray * +regarray_find(struct rte_swx_pipeline *p, const char *name) +{ + struct regarray *elem; + + TAILQ_FOREACH(elem, &p->regarrays, node) + if (!strcmp(elem->name, name)) + return elem; + + return NULL; +} + +static struct regarray * +regarray_find_by_id(struct rte_swx_pipeline *p, uint32_t id) +{ + struct regarray *elem = NULL; + + TAILQ_FOREACH(elem, &p->regarrays, node) + if (elem->id == id) + return elem; + + return NULL; +} + +int +rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p, + const char *name, + uint32_t size) +{ + struct regarray *r; + + CHECK(p, EINVAL); + + CHECK_NAME(name, EINVAL); + CHECK(!regarray_find(p, name), EEXIST); + + CHECK(size, EINVAL); + size = rte_align32pow2(size); + + /* Memory allocation. */ + r = calloc(1, sizeof(struct regarray)); + CHECK(r, ENOMEM); + + /* Node initialization. */ + strcpy(r->name, name); + r->size = size; + r->id = p->n_regarrays; + + /* Node add to tailq. */ + TAILQ_INSERT_TAIL(&p->regarrays, r, node); + p->n_regarrays++; + + return 0; +} + +static int +regarray_build(struct rte_swx_pipeline *p) +{ + struct regarray *regarray; + + p->regarray_runtime = calloc(p->n_regarrays, sizeof(struct regarray_runtime)); + CHECK(p->regarray_runtime, ENOMEM); + + TAILQ_FOREACH(regarray, &p->regarrays, node) { + struct regarray_runtime *r = &p->regarray_runtime[regarray->id]; + + r->regarray = calloc(regarray->size, sizeof(uint64_t)); + CHECK(r->regarray, ENOMEM); + + r->size_mask = regarray->size - 1; + } + + return 0; +} + +static void +regarray_build_free(struct rte_swx_pipeline *p) +{ + uint32_t i; + + if (!p->regarray_runtime) + return; + + for (i = 0; i < p->n_regarrays; i++) { + struct regarray_runtime *r = &p->regarray_runtime[i]; + + free(r->regarray); + } + + free(p->regarray_runtime); + p->regarray_runtime = NULL; +} + +static void +regarray_free(struct rte_swx_pipeline *p) +{ + regarray_build_free(p); + + for ( ; ; ) { + struct regarray *elem; + + elem = TAILQ_FIRST(&p->regarrays); + if (!elem) + break; + + TAILQ_REMOVE(&p->regarrays, elem, node); + free(elem); + } +} + /* * Pipeline. */ @@ -6851,6 +8136,7 @@ rte_swx_pipeline_config(struct rte_swx_pipeline **p, int numa_node) TAILQ_INIT(&pipeline->actions); TAILQ_INIT(&pipeline->table_types); TAILQ_INIT(&pipeline->tables); + TAILQ_INIT(&pipeline->regarrays); pipeline->n_structs = 1; /* Struct 0 is reserved for action_data. */ pipeline->numa_node = numa_node; @@ -6867,6 +8153,7 @@ rte_swx_pipeline_free(struct rte_swx_pipeline *p) free(p->instructions); + regarray_free(p); table_state_free(p); table_free(p); action_free(p); @@ -6951,10 +8238,15 @@ rte_swx_pipeline_build(struct rte_swx_pipeline *p) if (status) goto error; + status = regarray_build(p); + if (status) + goto error; + p->build_done = 1; return 0; error: + regarray_build_free(p); table_state_build_free(p); table_build_free(p); action_build_free(p); @@ -7015,6 +8307,7 @@ rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p, pipeline->n_ports_out = p->n_ports_out; pipeline->n_actions = n_actions; pipeline->n_tables = n_tables; + pipeline->n_regarrays = p->n_regarrays; return 0; } @@ -7222,3 +8515,64 @@ rte_swx_ctl_pipeline_port_out_stats_read(struct rte_swx_pipeline *p, port->type->ops.stats_read(port->obj, stats); return 0; } + +int +rte_swx_ctl_regarray_info_get(struct rte_swx_pipeline *p, + uint32_t regarray_id, + struct rte_swx_ctl_regarray_info *regarray) +{ + struct regarray *r; + + if (!p || !regarray) + return -EINVAL; + + r = regarray_find_by_id(p, regarray_id); + if (!r) + return -EINVAL; + + strcpy(regarray->name, r->name); + regarray->size = r->size; + return 0; +} + +int +rte_swx_ctl_pipeline_regarray_read(struct rte_swx_pipeline *p, + const char *regarray_name, + uint32_t regarray_index, + uint64_t *value) +{ + struct regarray *regarray; + struct regarray_runtime *r; + + if (!p || !regarray_name || !value) + return -EINVAL; + + regarray = regarray_find(p, regarray_name); + if (!regarray || (regarray_index >= regarray->size)) + return -EINVAL; + + r = &p->regarray_runtime[regarray->id]; + *value = r->regarray[regarray_index]; + return 0; +} + +int +rte_swx_ctl_pipeline_regarray_write(struct rte_swx_pipeline *p, + const char *regarray_name, + uint32_t regarray_index, + uint64_t value) +{ + struct regarray *regarray; + struct regarray_runtime *r; + + if (!p || !regarray_name) + return -EINVAL; + + regarray = regarray_find(p, regarray_name); + if (!regarray || (regarray_index >= regarray->size)) + return -EINVAL; + + r = &p->regarray_runtime[regarray->id]; + r->regarray[regarray_index] = value; + return 0; +} diff --git a/lib/librte_pipeline/rte_swx_pipeline.h b/lib/librte_pipeline/rte_swx_pipeline.h index f0a2cef77..c6e4e50a8 100644 --- a/lib/librte_pipeline/rte_swx_pipeline.h +++ b/lib/librte_pipeline/rte_swx_pipeline.h @@ -616,6 +616,27 @@ rte_swx_pipeline_table_config(struct rte_swx_pipeline *p, const char *args, uint32_t size); +/** + * Pipeline register array configure + * + * @param[in] p + * Pipeline handle. + * @param[in] name + * Register array name. + * @param[in] size + * Number of registers in the array. Each register is 64-bit in size. + * @return + * 0 on success or the following error codes otherwise: + * -EINVAL: Invalid argument; + * -ENOMEM: Not enough space/cannot allocate memory; + * -EEXIST: Register array with this name already exists. + */ +__rte_experimental +int +rte_swx_pipeline_regarray_config(struct rte_swx_pipeline *p, + const char *name, + uint32_t size); + /** * Pipeline instructions configure * diff --git a/lib/librte_pipeline/rte_swx_pipeline_spec.c b/lib/librte_pipeline/rte_swx_pipeline_spec.c index f7884491b..31dcfc4c3 100644 --- a/lib/librte_pipeline/rte_swx_pipeline_spec.c +++ b/lib/librte_pipeline/rte_swx_pipeline_spec.c @@ -940,6 +940,68 @@ table_block_parse(struct table_spec *s, return -EINVAL; } +/* + * regarray. + * + * regarray NAME size SIZE + */ +struct regarray_spec { + char *name; + uint32_t size; +}; + +static void +regarray_spec_free(struct regarray_spec *s) +{ + if (!s) + return; + + free(s->name); + s->name = NULL; +} + +static int +regarray_statement_parse(struct regarray_spec *s, + char **tokens, + uint32_t n_tokens, + uint32_t n_lines, + uint32_t *err_line, + const char **err_msg) +{ + char *p; + + /* Check format. */ + if ((n_tokens != 4) || strcmp(tokens[2], "size")) { + if (err_line) + *err_line = n_lines; + if (err_msg) + *err_msg = "Invalid regarray statement."; + return -EINVAL; + } + + /* spec. */ + s->name = strdup(tokens[1]); + if (!s->name) { + if (err_line) + *err_line = n_lines; + if (err_msg) + *err_msg = "Memory allocation failed."; + return -ENOMEM; + } + + p = tokens[3]; + s->size = strtoul(p, &p, 0); + if (p[0] || !s->size) { + if (err_line) + *err_line = n_lines; + if (err_msg) + *err_msg = "Invalid size argument."; + return -EINVAL; + } + + return 0; +} + /* * apply. * @@ -1066,6 +1128,7 @@ rte_swx_pipeline_build_from_spec(struct rte_swx_pipeline *p, struct metadata_spec metadata_spec = {0}; struct action_spec action_spec = {0}; struct table_spec table_spec = {0}; + struct regarray_spec regarray_spec = {0}; struct apply_spec apply_spec = {0}; uint32_t n_lines; uint32_t block_mask = 0; @@ -1405,6 +1468,33 @@ rte_swx_pipeline_build_from_spec(struct rte_swx_pipeline *p, continue; } + /* regarray. */ + if (!strcmp(tokens[0], "regarray")) { + status = regarray_statement_parse(®array_spec, + tokens, + n_tokens, + n_lines, + err_line, + err_msg); + if (status) + goto error; + + status = rte_swx_pipeline_regarray_config(p, + regarray_spec.name, + regarray_spec.size); + if (status) { + if (err_line) + *err_line = n_lines; + if (err_msg) + *err_msg = "Register array configuration error."; + goto error; + } + + regarray_spec_free(®array_spec); + + continue; + } + /* apply. */ if (!strcmp(tokens[0], "apply")) { status = apply_statement_parse(&block_mask, @@ -1457,6 +1547,7 @@ rte_swx_pipeline_build_from_spec(struct rte_swx_pipeline *p, metadata_spec_free(&metadata_spec); action_spec_free(&action_spec); table_spec_free(&table_spec); + regarray_spec_free(®array_spec); apply_spec_free(&apply_spec); return status; } diff --git a/lib/librte_pipeline/version.map b/lib/librte_pipeline/version.map index 87c826f1b..26a9edf47 100644 --- a/lib/librte_pipeline/version.map +++ b/lib/librte_pipeline/version.map @@ -102,4 +102,10 @@ EXPERIMENTAL { rte_swx_pipeline_table_state_get; rte_swx_pipeline_table_state_set; rte_swx_pipeline_table_type_register; + + #added in 21.05 + rte_swx_ctl_pipeline_regarray_read; + rte_swx_ctl_pipeline_regarray_write; + rte_swx_ctl_regarray_info_get; + rte_swx_pipeline_regarray_config; };