Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/70650/?format=api
{ "id": 70650, "url": "https://patches.dpdk.org/api/patches/70650/?format=api", "web_url": "https://patches.dpdk.org/project/dpdk/patch/20200527141653.15576-6-konstantin.ananyev@intel.com/", "project": { "id": 1, "url": "https://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20200527141653.15576-6-konstantin.ananyev@intel.com>", "list_archive_url": "https://inbox.dpdk.org/dev/20200527141653.15576-6-konstantin.ananyev@intel.com", "date": "2020-05-27T14:16:53", "name": "[v2,5/5] bpf: x86 JIT support for packet data load instructions", "commit_ref": null, "pull_url": null, "state": "accepted", "archived": true, "hash": "303dd854cac587666605a688693452d959ef8e81", "submitter": { "id": 33, "url": "https://patches.dpdk.org/api/people/33/?format=api", "name": "Ananyev, Konstantin", "email": "konstantin.ananyev@intel.com" }, "delegate": { "id": 1, "url": "https://patches.dpdk.org/api/users/1/?format=api", "username": "tmonjalo", "first_name": "Thomas", "last_name": "Monjalon", "email": "thomas@monjalon.net" }, "mbox": "https://patches.dpdk.org/project/dpdk/patch/20200527141653.15576-6-konstantin.ananyev@intel.com/mbox/", "series": [ { "id": 10250, "url": "https://patches.dpdk.org/api/series/10250/?format=api", "web_url": "https://patches.dpdk.org/project/dpdk/list/?series=10250", "date": "2020-05-27T14:16:48", "name": "bpf: add support for BPF_ABS/BPF_IND instructions", "version": 2, "mbox": "https://patches.dpdk.org/series/10250/mbox/" } ], "comments": "https://patches.dpdk.org/api/patches/70650/comments/", "check": "success", "checks": "https://patches.dpdk.org/api/patches/70650/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from dpdk.org (dpdk.org [92.243.14.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id B9E3BA034E;\n\tWed, 27 May 2020 16:18:09 +0200 (CEST)", "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id 31A3A1DAAF;\n\tWed, 27 May 2020 16:17:36 +0200 (CEST)", "from mga09.intel.com (mga09.intel.com [134.134.136.24])\n by dpdk.org (Postfix) with ESMTP id DBF2A1DA87\n for <dev@dpdk.org>; Wed, 27 May 2020 16:17:33 +0200 (CEST)", "from fmsmga004.fm.intel.com ([10.253.24.48])\n by orsmga102.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384;\n 27 May 2020 07:17:33 -0700", "from sivswdev08.ir.intel.com ([10.237.217.47])\n by fmsmga004.fm.intel.com with ESMTP; 27 May 2020 07:17:31 -0700" ], "IronPort-SDR": [ "\n gxRkZRCsB1ZEmZ7fvqJLFxgAfMWemtDoq/H7di7o9wTb6eobK1XiOztuHJ9xaJYGW4jC06mXzY\n xh9kZT2QYbKw==", "\n nkKJG2bJ6VawompvBGbe4A+//UIEMPjrUkbko3uzEf6VlVK7t6Eb4Ci5/0bwlMFt2/d26BNDdY\n 5uND4cG1d61w==" ], "X-Amp-Result": "SKIPPED(no attachment in message)", "X-Amp-File-Uploaded": "False", "X-ExtLoop1": "1", "X-IronPort-AV": "E=Sophos;i=\"5.73,441,1583222400\"; d=\"scan'208\";a=\"291608940\"", "From": "Konstantin Ananyev <konstantin.ananyev@intel.com>", "To": "dev@dpdk.org", "Cc": "jerinj@marvell.com, stephen@networkplumber.org, mb@smartsharesystems.com,\n Konstantin Ananyev <konstantin.ananyev@intel.com>", "Date": "Wed, 27 May 2020 15:16:53 +0100", "Message-Id": "<20200527141653.15576-6-konstantin.ananyev@intel.com>", "X-Mailer": "git-send-email 2.18.0", "In-Reply-To": "<20200527141653.15576-1-konstantin.ananyev@intel.com>", "References": "<20200518155245.11380-1-konstantin.ananyev@intel.com>\n <20200527141653.15576-1-konstantin.ananyev@intel.com>", "Subject": "[dpdk-dev] [PATCH v2 5/5] bpf: x86 JIT support for packet data load\n\tinstructions", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.15", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "Make x86 JIT to generate native code for\n(BPF_ABS | <size> | BPF_LD) and (BPF_IND | <size> | BPF_LD)\ninstructions.\n\nSigned-off-by: Konstantin Ananyev <konstantin.ananyev@intel.com>\n---\n lib/librte_bpf/bpf_jit_x86.c | 181 ++++++++++++++++++++++++++++++++++\n lib/librte_bpf/bpf_validate.c | 17 +++-\n 2 files changed, 197 insertions(+), 1 deletion(-)", "diff": "diff --git a/lib/librte_bpf/bpf_jit_x86.c b/lib/librte_bpf/bpf_jit_x86.c\nindex f70cd6be5..aa22ea78a 100644\n--- a/lib/librte_bpf/bpf_jit_x86.c\n+++ b/lib/librte_bpf/bpf_jit_x86.c\n@@ -87,6 +87,14 @@ enum {\n \tREG_TMP1 = R10,\n };\n \n+/* LD_ABS/LD_IMM offsets */\n+enum {\n+\tLDMB_FSP_OFS, /* fast-path */\n+\tLDMB_SLP_OFS, /* slow-path */\n+\tLDMB_FIN_OFS, /* final part */\n+\tLDMB_OFS_NUM\n+};\n+\n /*\n * callee saved registers list.\n * keep RBP as the last one.\n@@ -100,6 +108,9 @@ struct bpf_jit_state {\n \t\tuint32_t num;\n \t\tint32_t off;\n \t} exit;\n+\tstruct {\n+\t\tuint32_t stack_ofs;\n+\t} ldmb;\n \tuint32_t reguse;\n \tint32_t *off;\n \tuint8_t *ins;\n@@ -1024,6 +1035,166 @@ emit_div(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,\n \t\temit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP1, RDX);\n }\n \n+/*\n+ * helper function, used by emit_ld_mbuf().\n+ * generates code for 'fast_path':\n+ * calculate load offset and check is it inside first packet segment.\n+ */\n+static void\n+emit_ldmb_fast_path(struct bpf_jit_state *st, const uint32_t rg[EBPF_REG_7],\n+\tuint32_t sreg, uint32_t mode, uint32_t sz, uint32_t imm,\n+\tconst int32_t ofs[LDMB_OFS_NUM])\n+{\n+\t/* make R2 contain *off* value */\n+\n+\tif (sreg != rg[EBPF_REG_2]) {\n+\t\temit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K,\n+\t\t\trg[EBPF_REG_2], imm);\n+\t\tif (mode == BPF_IND)\n+\t\t\temit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X,\n+\t\t\t\tsreg, rg[EBPF_REG_2]);\n+\t} else\n+\t\t/* BPF_IND with sreg == R2 */\n+\t\temit_alu_imm(st, EBPF_ALU64 | BPF_ADD | BPF_K,\n+\t\t\trg[EBPF_REG_2], imm);\n+\n+\t/* R3 = mbuf->data_len */\n+\temit_ld_reg(st, BPF_LDX | BPF_MEM | BPF_H,\n+\t\trg[EBPF_REG_6], rg[EBPF_REG_3],\n+\t\toffsetof(struct rte_mbuf, data_len));\n+\n+\t/* R3 = R3 - R2 */\n+\temit_alu_reg(st, EBPF_ALU64 | BPF_SUB | BPF_X,\n+\t\trg[EBPF_REG_2], rg[EBPF_REG_3]);\n+\n+\t/* JSLT R3, <sz> <slow_path> */\n+\temit_cmp_imm(st, EBPF_ALU64, rg[EBPF_REG_3], sz);\n+\temit_abs_jcc(st, BPF_JMP | EBPF_JSLT | BPF_K, ofs[LDMB_SLP_OFS]);\n+\n+\t/* R3 = mbuf->data_off */\n+\temit_ld_reg(st, BPF_LDX | BPF_MEM | BPF_H,\n+\t\trg[EBPF_REG_6], rg[EBPF_REG_3],\n+\t\toffsetof(struct rte_mbuf, data_off));\n+\n+\t/* R0 = mbuf->buf_addr */\n+\temit_ld_reg(st, BPF_LDX | BPF_MEM | EBPF_DW,\n+\t\trg[EBPF_REG_6], rg[EBPF_REG_0],\n+\t\toffsetof(struct rte_mbuf, buf_addr));\n+\n+\t/* R0 = R0 + R3 */\n+\temit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X,\n+\t\trg[EBPF_REG_3], rg[EBPF_REG_0]);\n+\n+\t/* R0 = R0 + R2 */\n+\temit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X,\n+\t\trg[EBPF_REG_2], rg[EBPF_REG_0]);\n+\n+\t/* JMP <fin_part> */\n+\temit_abs_jmp(st, ofs[LDMB_FIN_OFS]);\n+}\n+\n+/*\n+ * helper function, used by emit_ld_mbuf().\n+ * generates code for 'slow_path':\n+ * call __rte_pktmbuf_read() and check return value.\n+ */\n+static void\n+emit_ldmb_slow_path(struct bpf_jit_state *st, const uint32_t rg[EBPF_REG_7],\n+\tuint32_t sz)\n+{\n+\t/* make R3 contain *len* value (1/2/4) */\n+\n+\temit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, rg[EBPF_REG_3], sz);\n+\n+\t/* make R4 contain (RBP - ldmb.stack_ofs) */\n+\n+\temit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RBP, rg[EBPF_REG_4]);\n+\temit_alu_imm(st, EBPF_ALU64 | BPF_SUB | BPF_K, rg[EBPF_REG_4],\n+\t\tst->ldmb.stack_ofs);\n+\n+\t/* make R1 contain mbuf ptr */\n+\n+\temit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X,\n+\t\trg[EBPF_REG_6], rg[EBPF_REG_1]);\n+\n+\t/* call rte_pktmbuf_read */\n+\temit_call(st, (uintptr_t)__rte_pktmbuf_read);\n+\n+\t/* check that return value (R0) is not zero */\n+\temit_tst_reg(st, EBPF_ALU64, rg[EBPF_REG_0], rg[EBPF_REG_0]);\n+\temit_abs_jcc(st, BPF_JMP | BPF_JEQ | BPF_K, st->exit.off);\n+}\n+\n+/*\n+ * helper function, used by emit_ld_mbuf().\n+ * generates final part of code for BPF_ABS/BPF_IND load:\n+ * perform data load and endianness conversion.\n+ * expects dreg to contain valid data pointer.\n+ */\n+static void\n+emit_ldmb_fin(struct bpf_jit_state *st, uint32_t dreg, uint32_t opsz,\n+\tuint32_t sz)\n+{\n+\temit_ld_reg(st, BPF_LDX | BPF_MEM | opsz, dreg, dreg, 0);\n+\tif (sz != sizeof(uint8_t))\n+\t\temit_be2le(st, dreg, sz * CHAR_BIT);\n+}\n+\n+/*\n+ * emit code for BPF_ABS/BPF_IND load.\n+ * generates the following construction:\n+ * fast_path:\n+ * off = ins->sreg + ins->imm\n+ * if (mbuf->data_len - off < ins->opsz)\n+ * goto slow_path;\n+ * ptr = mbuf->buf_addr + mbuf->data_off + off;\n+ * goto fin_part;\n+ * slow_path:\n+ * typeof(ins->opsz) buf; //allocate space on the stack\n+ * ptr = __rte_pktmbuf_read(mbuf, off, ins->opsz, &buf);\n+ * if (ptr == NULL)\n+ * goto exit_label;\n+ * fin_part:\n+ * res = *(typeof(ins->opsz))ptr;\n+ * res = bswap(res);\n+ */\n+static void\n+emit_ld_mbuf(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t imm)\n+{\n+\tuint32_t i, mode, opsz, sz;\n+\tuint32_t rg[EBPF_REG_7];\n+\tint32_t ofs[LDMB_OFS_NUM];\n+\n+\tmode = BPF_MODE(op);\n+\topsz = BPF_SIZE(op);\n+\tsz = bpf_size(opsz);\n+\n+\tfor (i = 0; i != RTE_DIM(rg); i++)\n+\t\trg[i] = ebpf2x86[i];\n+\n+\t/* fill with fake offsets */\n+\tfor (i = 0; i != RTE_DIM(ofs); i++)\n+\t\tofs[i] = st->sz + INT8_MAX;\n+\n+\t/* dry run first to calculate jump offsets */\n+\n+\tofs[LDMB_FSP_OFS] = st->sz;\n+\temit_ldmb_fast_path(st, rg, sreg, mode, sz, imm, ofs);\n+\tofs[LDMB_SLP_OFS] = st->sz;\n+\temit_ldmb_slow_path(st, rg, sz);\n+\tofs[LDMB_FIN_OFS] = st->sz;\n+\temit_ldmb_fin(st, rg[EBPF_REG_0], opsz, sz);\n+\n+\tRTE_VERIFY(ofs[LDMB_FIN_OFS] - ofs[LDMB_FSP_OFS] <= INT8_MAX);\n+\n+\t/* reset dry-run code and do a proper run */\n+\n+\tst->sz = ofs[LDMB_FSP_OFS];\n+\temit_ldmb_fast_path(st, rg, sreg, mode, sz, imm, ofs);\n+\temit_ldmb_slow_path(st, rg, sz);\n+\temit_ldmb_fin(st, rg[EBPF_REG_0], opsz, sz);\n+}\n+\n static void\n emit_prolog(struct bpf_jit_state *st, int32_t stack_size)\n {\n@@ -1121,6 +1292,7 @@ emit(struct bpf_jit_state *st, const struct rte_bpf *bpf)\n \t/* reset state fields */\n \tst->sz = 0;\n \tst->exit.num = 0;\n+\tst->ldmb.stack_ofs = bpf->stack_sz;\n \n \temit_prolog(st, bpf->stack_sz);\n \n@@ -1240,6 +1412,15 @@ emit(struct bpf_jit_state *st, const struct rte_bpf *bpf)\n \t\t\temit_ld_imm64(st, dr, ins[0].imm, ins[1].imm);\n \t\t\ti++;\n \t\t\tbreak;\n+\t\t/* load absolute/indirect instructions */\n+\t\tcase (BPF_LD | BPF_ABS | BPF_B):\n+\t\tcase (BPF_LD | BPF_ABS | BPF_H):\n+\t\tcase (BPF_LD | BPF_ABS | BPF_W):\n+\t\tcase (BPF_LD | BPF_IND | BPF_B):\n+\t\tcase (BPF_LD | BPF_IND | BPF_H):\n+\t\tcase (BPF_LD | BPF_IND | BPF_W):\n+\t\t\temit_ld_mbuf(st, op, sr, ins->imm);\n+\t\t\tbreak;\n \t\t/* store instructions */\n \t\tcase (BPF_STX | BPF_MEM | BPF_B):\n \t\tcase (BPF_STX | BPF_MEM | BPF_H):\ndiff --git a/lib/librte_bpf/bpf_validate.c b/lib/librte_bpf/bpf_validate.c\nindex fecdda0e1..9214f1503 100644\n--- a/lib/librte_bpf/bpf_validate.c\n+++ b/lib/librte_bpf/bpf_validate.c\n@@ -70,6 +70,7 @@ struct bpf_verifier {\n \tuint64_t stack_sz;\n \tuint32_t nb_nodes;\n \tuint32_t nb_jcc_nodes;\n+\tuint32_t nb_ldmb_nodes;\n \tuint32_t node_colour[MAX_NODE_COLOUR];\n \tuint32_t edge_type[MAX_EDGE_TYPE];\n \tstruct bpf_eval_state *evst;\n@@ -2020,6 +2021,14 @@ validate(struct bpf_verifier *bvf)\n \t\t\trc |= add_edge(bvf, node, i + 2);\n \t\t\ti++;\n \t\t\tbreak;\n+\t\tcase (BPF_LD | BPF_ABS | BPF_B):\n+\t\tcase (BPF_LD | BPF_ABS | BPF_H):\n+\t\tcase (BPF_LD | BPF_ABS | BPF_W):\n+\t\tcase (BPF_LD | BPF_IND | BPF_B):\n+\t\tcase (BPF_LD | BPF_IND | BPF_H):\n+\t\tcase (BPF_LD | BPF_IND | BPF_W):\n+\t\t\tbvf->nb_ldmb_nodes++;\n+\t\t\t/* fallthrough */\n \t\tdefault:\n \t\t\trc |= add_edge(bvf, node, i + 1);\n \t\t\tbreak;\n@@ -2320,8 +2329,14 @@ bpf_validate(struct rte_bpf *bpf)\n \tfree(bvf.in);\n \n \t/* copy collected info */\n-\tif (rc == 0)\n+\tif (rc == 0) {\n \t\tbpf->stack_sz = bvf.stack_sz;\n \n+\t\t/* for LD_ABS/LD_IND, we'll need extra space on the stack */\n+\t\tif (bvf.nb_ldmb_nodes != 0)\n+\t\t\tbpf->stack_sz = RTE_ALIGN_CEIL(bpf->stack_sz +\n+\t\t\t\tsizeof(uint64_t), sizeof(uint64_t));\n+\t}\n+\n \treturn rc;\n }\n", "prefixes": [ "v2", "5/5" ] }