From patchwork Wed May 27 14:16:49 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Ananyev, Konstantin" X-Patchwork-Id: 70646 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 4BACFA034E; Wed, 27 May 2020 16:17:21 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 366A11D9B1; Wed, 27 May 2020 16:17:21 +0200 (CEST) Received: from mga09.intel.com (mga09.intel.com [134.134.136.24]) by dpdk.org (Postfix) with ESMTP id B2C461D9AF; Wed, 27 May 2020 16:17:19 +0200 (CEST) IronPort-SDR: xpU0vPXTiSYaydMXYP+iJkLCj1avQKasBRCB0NEMuRZMAJZhI7BxzjUH+rhYY4E1F406i9qRzQ k0yMPkdulPUg== X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga004.fm.intel.com ([10.253.24.48]) by orsmga102.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 27 May 2020 07:17:18 -0700 IronPort-SDR: pNLUvvaUthhiERRs9Q8dcM3XMmf1gLv39u8Qt3BG4PXqpLRuQxWjEh62zqrHIi1hNFxzGeMuCc KPTTI6DpvwjQ== X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.73,441,1583222400"; d="scan'208";a="291608826" Received: from sivswdev08.ir.intel.com ([10.237.217.47]) by fmsmga004.fm.intel.com with ESMTP; 27 May 2020 07:17:16 -0700 From: Konstantin Ananyev To: dev@dpdk.org Cc: jerinj@marvell.com, stephen@networkplumber.org, mb@smartsharesystems.com, Konstantin Ananyev , stable@dpdk.org Date: Wed, 27 May 2020 15:16:49 +0100 Message-Id: <20200527141653.15576-2-konstantin.ananyev@intel.com> X-Mailer: git-send-email 2.18.0 In-Reply-To: <20200527141653.15576-1-konstantin.ananyev@intel.com> References: <20200518155245.11380-1-konstantin.ananyev@intel.com> <20200527141653.15576-1-konstantin.ananyev@intel.com> Subject: [dpdk-dev] [PATCH v2 1/5] test/bpf: fix few small issues X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Address for few small issues: - unreachable return statement - failed test-case can finish with 'success' status Also use unified cmp_res() function to check return value. Fixes: a9de470cc7c0 ("test: move to app directory") Cc: stable@dpdk.org Signed-off-by: Konstantin Ananyev --- app/test/test_bpf.c | 41 +++++++++++++++-------------------------- 1 file changed, 15 insertions(+), 26 deletions(-) diff --git a/app/test/test_bpf.c b/app/test/test_bpf.c index ee534687a..4a61a7d7c 100644 --- a/app/test/test_bpf.c +++ b/app/test/test_bpf.c @@ -1797,13 +1797,6 @@ test_call1_check(uint64_t rc, const void *arg) dummy_func1(arg, &v32, &v64); v64 += v32; - if (v64 != rc) { - printf("%s@%d: invalid return value " - "expected=0x%" PRIx64 ", actual=0x%" PRIx64 "\n", - __func__, __LINE__, v64, rc); - return -1; - } - return 0; return cmp_res(__func__, v64, rc, dv, dv, sizeof(*dv)); } @@ -1934,13 +1927,7 @@ test_call2_check(uint64_t rc, const void *arg) dummy_func2(&a, &b); v = a.u64 + a.u32 + b.u16 + b.u8; - if (v != rc) { - printf("%s@%d: invalid return value " - "expected=0x%" PRIx64 ", actual=0x%" PRIx64 "\n", - __func__, __LINE__, v, rc); - return -1; - } - return 0; + return cmp_res(__func__, v, rc, arg, arg, 0); } static const struct rte_bpf_xsym test_call2_xsym[] = { @@ -2429,7 +2416,6 @@ test_call5_check(uint64_t rc, const void *arg) v = 0; fail: - return cmp_res(__func__, v, rc, &v, &rc, sizeof(v)); } @@ -2458,6 +2444,7 @@ static const struct rte_bpf_xsym test_call5_xsym[] = { }, }; +/* all bpf test cases */ static const struct bpf_test tests[] = { { .name = "test_store1", @@ -2738,7 +2725,6 @@ run_test(const struct bpf_test *tst) } tst->prepare(tbuf); - rc = rte_bpf_exec(bpf, tbuf); ret = tst->check_result(rc, tbuf); if (ret != 0) { @@ -2746,17 +2732,20 @@ run_test(const struct bpf_test *tst) __func__, __LINE__, tst->name, ret, strerror(ret)); } + /* repeat the same test with jit, when possible */ rte_bpf_get_jit(bpf, &jit); - if (jit.func == NULL) - return 0; - - tst->prepare(tbuf); - rc = jit.func(tbuf); - rv = tst->check_result(rc, tbuf); - ret |= rv; - if (rv != 0) { - printf("%s@%d: check_result(%s) failed, error: %d(%s);\n", - __func__, __LINE__, tst->name, rv, strerror(ret)); + if (jit.func != NULL) { + + tst->prepare(tbuf); + rc = jit.func(tbuf); + rv = tst->check_result(rc, tbuf); + ret |= rv; + if (rv != 0) { + printf("%s@%d: check_result(%s) failed, " + "error: %d(%s);\n", + __func__, __LINE__, tst->name, + rv, strerror(ret)); + } } rte_bpf_destroy(bpf); From patchwork Wed May 27 14:16:50 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Ananyev, Konstantin" X-Patchwork-Id: 70647 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id E6368A034E; Wed, 27 May 2020 16:17:32 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id B171C1DA80; Wed, 27 May 2020 16:17:24 +0200 (CEST) Received: from mga09.intel.com (mga09.intel.com [134.134.136.24]) by dpdk.org (Postfix) with ESMTP id 1C31B1DA71; Wed, 27 May 2020 16:17:22 +0200 (CEST) IronPort-SDR: K2MjQ6HdbHYXXaMDDWFurOPCbwT9VbWSze0ql7CYEqvG9egO5OozvQD+xDXZ2CkojR1xxEgQbU HlbSctpTGAmQ== X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga004.fm.intel.com ([10.253.24.48]) by orsmga102.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 27 May 2020 07:17:22 -0700 IronPort-SDR: zN7ZubpC55EAhEK9gDSdFBaBdf3qPmVWReMc30jB2rJIidta3eA/pkuAej0LF/jfpWBa+X7j89 W9VnGVnVdW7w== X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.73,441,1583222400"; d="scan'208";a="291608845" Received: from sivswdev08.ir.intel.com ([10.237.217.47]) by fmsmga004.fm.intel.com with ESMTP; 27 May 2020 07:17:21 -0700 From: Konstantin Ananyev To: dev@dpdk.org Cc: jerinj@marvell.com, stephen@networkplumber.org, mb@smartsharesystems.com, Konstantin Ananyev , stable@dpdk.org Date: Wed, 27 May 2020 15:16:50 +0100 Message-Id: <20200527141653.15576-3-konstantin.ananyev@intel.com> X-Mailer: git-send-email 2.18.0 In-Reply-To: <20200527141653.15576-1-konstantin.ananyev@intel.com> References: <20200518155245.11380-1-konstantin.ananyev@intel.com> <20200527141653.15576-1-konstantin.ananyev@intel.com> Subject: [dpdk-dev] [PATCH v2 2/5] bpf: fix add/sub min/max estimations X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" eval_add()/eval_sub() not always correctly estimate minimum and maximum possible values of add/sub operations. Fixes: 8021917293d0 ("bpf: add extra validation for input BPF program") Cc: stable@dpdk.org Signed-off-by: Konstantin Ananyev --- lib/librte_bpf/bpf_validate.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/librte_bpf/bpf_validate.c b/lib/librte_bpf/bpf_validate.c index 6bd6f78e9..80d21fabb 100644 --- a/lib/librte_bpf/bpf_validate.c +++ b/lib/librte_bpf/bpf_validate.c @@ -226,7 +226,7 @@ eval_add(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, uint64_t msk) struct bpf_reg_val rv; rv.u.min = (rd->u.min + rs->u.min) & msk; - rv.u.max = (rd->u.min + rs->u.max) & msk; + rv.u.max = (rd->u.max + rs->u.max) & msk; rv.s.min = (rd->s.min + rs->s.min) & msk; rv.s.max = (rd->s.max + rs->s.max) & msk; @@ -254,10 +254,10 @@ eval_sub(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, uint64_t msk) { struct bpf_reg_val rv; - rv.u.min = (rd->u.min - rs->u.min) & msk; - rv.u.max = (rd->u.min - rs->u.max) & msk; - rv.s.min = (rd->s.min - rs->s.min) & msk; - rv.s.max = (rd->s.max - rs->s.max) & msk; + rv.u.min = (rd->u.min - rs->u.max) & msk; + rv.u.max = (rd->u.max - rs->u.min) & msk; + rv.s.min = (rd->s.min - rs->s.max) & msk; + rv.s.max = (rd->s.max - rs->s.min) & msk; /* * if at least one of the operands is not constant, From patchwork Wed May 27 14:16:51 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Ananyev, Konstantin" X-Patchwork-Id: 70648 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 84AF5A034E; Wed, 27 May 2020 16:17:46 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 69B991DA97; Wed, 27 May 2020 16:17:28 +0200 (CEST) Received: from mga09.intel.com (mga09.intel.com [134.134.136.24]) by dpdk.org (Postfix) with ESMTP id 1B1E61DA17 for ; Wed, 27 May 2020 16:17:26 +0200 (CEST) IronPort-SDR: +bxo3P3YUoTGrYKokki0JE3MMsbJ808U/gPIl53fONdx8b5x6lPAV/p7UMf/u/4evdP/DYkeNo 8XVSgUORNVMQ== X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga004.fm.intel.com ([10.253.24.48]) by orsmga102.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 27 May 2020 07:17:26 -0700 IronPort-SDR: LXnotw2CLxoNnuZgL7BP3wbe4vMcmXz74K3W4CCQsDjD570KoDfB/KiqG1pQlPbIRTEAG8psEb MX2IgSh3CMAg== X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.73,441,1583222400"; d="scan'208";a="291608872" Received: from sivswdev08.ir.intel.com ([10.237.217.47]) by fmsmga004.fm.intel.com with ESMTP; 27 May 2020 07:17:25 -0700 From: Konstantin Ananyev To: dev@dpdk.org Cc: jerinj@marvell.com, stephen@networkplumber.org, mb@smartsharesystems.com, Konstantin Ananyev Date: Wed, 27 May 2020 15:16:51 +0100 Message-Id: <20200527141653.15576-4-konstantin.ananyev@intel.com> X-Mailer: git-send-email 2.18.0 In-Reply-To: <20200527141653.15576-1-konstantin.ananyev@intel.com> References: <20200518155245.11380-1-konstantin.ananyev@intel.com> <20200527141653.15576-1-konstantin.ananyev@intel.com> Subject: [dpdk-dev] [PATCH v2 3/5] bpf: add support for packet data load instructions X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" To fill the gap with linux kernel eBPF implementation, add support for two non-generic instructions: (BPF_ABS | | BPF_LD) and (BPF_IND | | BPF_LD) which are used to access packet data. These instructions can only be used when BPF context is a pointer to 'struct rte_mbuf' (i.e: RTE_BPF_ARG_PTR_MBUF type). Signed-off-by: Konstantin Ananyev --- doc/guides/prog_guide/bpf_lib.rst | 30 +++++++++- doc/guides/rel_notes/release_20_08.rst | 7 +++ lib/librte_bpf/bpf_exec.c | 57 +++++++++++++++++++ lib/librte_bpf/bpf_validate.c | 77 ++++++++++++++++++++++++++ 4 files changed, 170 insertions(+), 1 deletion(-) diff --git a/doc/guides/prog_guide/bpf_lib.rst b/doc/guides/prog_guide/bpf_lib.rst index 9c728da7b..1feb7734a 100644 --- a/doc/guides/prog_guide/bpf_lib.rst +++ b/doc/guides/prog_guide/bpf_lib.rst @@ -27,6 +27,35 @@ The library API provides the following basic operations: * Load BPF program from the ELF file and install callback to execute it on given ethdev port/queue. +Packet data load instructions +----------------------------- + +DPDK supports two non-generic instructions: ``(BPF_ABS | size | BPF_LD)`` +and ``(BPF_IND | size | BPF_LD)`` which are used to access packet data. +These instructions can only be used when execution context is a pointer to +``struct rte_mbuf`` and have seven implicit operands. +Register ``R6`` is an implicit input that must contain pointer to ``rte_mbuf``. +Register ``R0`` is an implicit output which contains the data fetched from the +packet. Registers ``R1-R5`` are scratch registers +and must not be used to store the data across these instructions. +These instructions have implicit program exit condition as well. When +eBPF program is trying to access the data beyond the packet boundary, +the interpreter will abort the execution of the program. JIT compilers +therefore must preserve this property. ``src_reg`` and ``imm32`` fields are +explicit inputs to these instructions. +For example, ``(BPF_IND | BPF_W | BPF_LD)`` means: + +.. code-block:: c + + uint32_t tmp; + R0 = rte_pktmbuf_read((const struct rte_mbuf *)R6, src_reg + imm32, + sizeof(tmp), &tmp); + if (R0 == NULL) return FAILED; + R0 = ntohl(*(uint32_t *)R0); + +and ``R1-R5`` were scratched. + + Not currently supported eBPF features ------------------------------------- @@ -34,5 +63,4 @@ Not currently supported eBPF features - cBPF - tail-pointer call - eBPF MAP - - skb - external function calls for 32-bit platforms diff --git a/doc/guides/rel_notes/release_20_08.rst b/doc/guides/rel_notes/release_20_08.rst index 39064afbe..3c0824616 100644 --- a/doc/guides/rel_notes/release_20_08.rst +++ b/doc/guides/rel_notes/release_20_08.rst @@ -56,6 +56,13 @@ New Features Also, make sure to start the actual text at the margin. ========================================================= +* **Added support for BPF_ABS/BPF_IND load instructions.** + + Added support for two BPF non-generic instructions: + ``(BPF_ABS | | BPF_LD)`` and ``(BPF_IND | | BPF_LD)`` + which are used to access packet data in a safe manner. Currently JIT support + for these instructions is implemented for x86 only. + Removed Items ------------- diff --git a/lib/librte_bpf/bpf_exec.c b/lib/librte_bpf/bpf_exec.c index 1bb226643..b921112fe 100644 --- a/lib/librte_bpf/bpf_exec.c +++ b/lib/librte_bpf/bpf_exec.c @@ -74,6 +74,26 @@ (uintptr_t)((reg)[(ins)->dst_reg] + (ins)->off), \ reg[ins->src_reg])) +/* BPF_LD | BPF_ABS/BPF_IND */ + +#define NOP(x) (x) + +#define BPF_LD_ABS(bpf, reg, ins, type, op) do { \ + const type *p = bpf_ld_mbuf(bpf, reg, ins, (ins)->imm, sizeof(type)); \ + if (p == NULL) \ + return 0; \ + reg[EBPF_REG_0] = op(p[0]); \ +} while (0) + +#define BPF_LD_IND(bpf, reg, ins, type, op) do { \ + uint32_t ofs = reg[ins->src_reg] + (ins)->imm; \ + const type *p = bpf_ld_mbuf(bpf, reg, ins, ofs, sizeof(type)); \ + if (p == NULL) \ + return 0; \ + reg[EBPF_REG_0] = op(p[0]); \ +} while (0) + + static inline void bpf_alu_be(uint64_t reg[EBPF_REG_NUM], const struct ebpf_insn *ins) { @@ -112,6 +132,23 @@ bpf_alu_le(uint64_t reg[EBPF_REG_NUM], const struct ebpf_insn *ins) } } +static inline const void * +bpf_ld_mbuf(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM], + const struct ebpf_insn *ins, uint32_t off, uint32_t len) +{ + const struct rte_mbuf *mb; + const void *p; + + mb = (const struct rte_mbuf *)(uintptr_t)reg[EBPF_REG_6]; + p = rte_pktmbuf_read(mb, off, len, reg + EBPF_REG_0); + if (p == NULL) + RTE_BPF_LOG(DEBUG, "%s(bpf=%p, mbuf=%p, ofs=%u, len=%u): " + "load beyond packet boundary at pc: %#zx;\n", + __func__, bpf, mb, off, len, + (uintptr_t)(ins) - (uintptr_t)(bpf)->prm.ins); + return p; +} + static inline uint64_t bpf_exec(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM]) { @@ -296,6 +333,26 @@ bpf_exec(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM]) (uint64_t)(uint32_t)ins[1].imm << 32; ins++; break; + /* load absolute instructions */ + case (BPF_LD | BPF_ABS | BPF_B): + BPF_LD_ABS(bpf, reg, ins, uint8_t, NOP); + break; + case (BPF_LD | BPF_ABS | BPF_H): + BPF_LD_ABS(bpf, reg, ins, uint16_t, rte_be_to_cpu_16); + break; + case (BPF_LD | BPF_ABS | BPF_W): + BPF_LD_ABS(bpf, reg, ins, uint32_t, rte_be_to_cpu_32); + break; + /* load indirect instructions */ + case (BPF_LD | BPF_IND | BPF_B): + BPF_LD_IND(bpf, reg, ins, uint8_t, NOP); + break; + case (BPF_LD | BPF_IND | BPF_H): + BPF_LD_IND(bpf, reg, ins, uint16_t, rte_be_to_cpu_16); + break; + case (BPF_LD | BPF_IND | BPF_W): + BPF_LD_IND(bpf, reg, ins, uint32_t, rte_be_to_cpu_32); + break; /* store instructions */ case (BPF_STX | BPF_MEM | BPF_B): BPF_ST_REG(reg, ins, uint8_t); diff --git a/lib/librte_bpf/bpf_validate.c b/lib/librte_bpf/bpf_validate.c index 80d21fabb..fecdda0e1 100644 --- a/lib/librte_bpf/bpf_validate.c +++ b/lib/librte_bpf/bpf_validate.c @@ -102,6 +102,9 @@ struct bpf_ins_check { #define WRT_REGS RTE_LEN2MASK(EBPF_REG_10, uint16_t) #define ZERO_REG RTE_LEN2MASK(EBPF_REG_1, uint16_t) +/* For LD_IND R6 is an implicit CTX register. */ +#define IND_SRC_REGS (WRT_REGS ^ 1 << EBPF_REG_6) + /* * check and evaluate functions for particular instruction types. */ @@ -580,6 +583,42 @@ eval_neg(struct bpf_reg_val *rd, size_t opsz, uint64_t msk) rd->s.min = RTE_MIN(sx, sy); } +static const char * +eval_ld_mbuf(struct bpf_verifier *bvf, const struct ebpf_insn *ins) +{ + uint32_t i, mode; + struct bpf_reg_val *rv, ri, rs; + + mode = BPF_MODE(ins->code); + + /* R6 is an implicit input that must contain pointer to mbuf */ + if (bvf->evst->rv[EBPF_REG_6].v.type != RTE_BPF_ARG_PTR_MBUF) + return "invalid type for implicit ctx register"; + + if (mode == BPF_IND) { + rs = bvf->evst->rv[ins->src_reg]; + if (rs.v.type != RTE_BPF_ARG_RAW) + return "unexpected type for src register"; + + eval_fill_imm(&ri, UINT64_MAX, ins->imm); + eval_add(&rs, &ri, UINT64_MAX); + + if (rs.s.max < 0 || rs.u.min > UINT32_MAX) + return "mbuf boundary violation"; + } + + /* R1-R5 scratch registers */ + for (i = EBPF_REG_1; i != EBPF_REG_6; i++) + bvf->evst->rv[i].v.type = RTE_BPF_ARG_UNDEF; + + /* R0 is an implicit output, contains data fetched from the packet */ + rv = bvf->evst->rv + EBPF_REG_0; + rv->v.size = bpf_size(BPF_SIZE(ins->code)); + eval_fill_max_bound(rv, RTE_LEN2MASK(rv->v.size * CHAR_BIT, uint64_t)); + + return NULL; +} + /* * check that destination and source operand are in defined state. */ @@ -1425,6 +1464,44 @@ static const struct bpf_ins_check ins_chk[UINT8_MAX + 1] = { .imm = { .min = 0, .max = UINT32_MAX}, .eval = eval_ld_imm64, }, + /* load absolute instructions */ + [(BPF_LD | BPF_ABS | BPF_B)] = { + .mask = {. dreg = ZERO_REG, .sreg = ZERO_REG}, + .off = { .min = 0, .max = 0}, + .imm = { .min = 0, .max = INT32_MAX}, + .eval = eval_ld_mbuf, + }, + [(BPF_LD | BPF_ABS | BPF_H)] = { + .mask = {. dreg = ZERO_REG, .sreg = ZERO_REG}, + .off = { .min = 0, .max = 0}, + .imm = { .min = 0, .max = INT32_MAX}, + .eval = eval_ld_mbuf, + }, + [(BPF_LD | BPF_ABS | BPF_W)] = { + .mask = {. dreg = ZERO_REG, .sreg = ZERO_REG}, + .off = { .min = 0, .max = 0}, + .imm = { .min = 0, .max = INT32_MAX}, + .eval = eval_ld_mbuf, + }, + /* load indirect instructions */ + [(BPF_LD | BPF_IND | BPF_B)] = { + .mask = {. dreg = ZERO_REG, .sreg = IND_SRC_REGS}, + .off = { .min = 0, .max = 0}, + .imm = { .min = 0, .max = UINT32_MAX}, + .eval = eval_ld_mbuf, + }, + [(BPF_LD | BPF_IND | BPF_H)] = { + .mask = {. dreg = ZERO_REG, .sreg = IND_SRC_REGS}, + .off = { .min = 0, .max = 0}, + .imm = { .min = 0, .max = UINT32_MAX}, + .eval = eval_ld_mbuf, + }, + [(BPF_LD | BPF_IND | BPF_W)] = { + .mask = {. dreg = ZERO_REG, .sreg = IND_SRC_REGS}, + .off = { .min = 0, .max = 0}, + .imm = { .min = 0, .max = UINT32_MAX}, + .eval = eval_ld_mbuf, + }, /* store REG instructions */ [(BPF_STX | BPF_MEM | BPF_B)] = { .mask = { .dreg = ALL_REGS, .sreg = ALL_REGS}, From patchwork Wed May 27 14:16:52 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Ananyev, Konstantin" X-Patchwork-Id: 70649 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id E2F60A034E; Wed, 27 May 2020 16:17:56 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id B81301DAA9; Wed, 27 May 2020 16:17:31 +0200 (CEST) Received: from mga09.intel.com (mga09.intel.com [134.134.136.24]) by dpdk.org (Postfix) with ESMTP id 3003B1DA9A for ; Wed, 27 May 2020 16:17:30 +0200 (CEST) IronPort-SDR: Vqu8UsnjImUVY/pLC76GahYMmMox/VBAlBPru/bUScOm90+FFY8Yaawbj53yr4PaxuMgyVdVRB OMSQIwjSv5JA== X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga004.fm.intel.com ([10.253.24.48]) by orsmga102.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 27 May 2020 07:17:29 -0700 IronPort-SDR: eYmiYtlODokqBwBmIN79nnt8+m9yKWAtSUfGlCQhTeVA1HeRA7CxIubgKI/jUmpiYTW2C2hJSP xH8olcyz7izg== X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.73,441,1583222400"; d="scan'208";a="291608908" Received: from sivswdev08.ir.intel.com ([10.237.217.47]) by fmsmga004.fm.intel.com with ESMTP; 27 May 2020 07:17:28 -0700 From: Konstantin Ananyev To: dev@dpdk.org Cc: jerinj@marvell.com, stephen@networkplumber.org, mb@smartsharesystems.com, Konstantin Ananyev Date: Wed, 27 May 2020 15:16:52 +0100 Message-Id: <20200527141653.15576-5-konstantin.ananyev@intel.com> X-Mailer: git-send-email 2.18.0 In-Reply-To: <20200527141653.15576-1-konstantin.ananyev@intel.com> References: <20200518155245.11380-1-konstantin.ananyev@intel.com> <20200527141653.15576-1-konstantin.ananyev@intel.com> Subject: [dpdk-dev] [PATCH v2 4/5] test/bpf: add new test cases for mbuf load instructions X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Add new test-cases for BPF_ABS/BPF_IND load instructions. Signed-off-by: Konstantin Ananyev --- app/test/test_bpf.c | 463 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 463 insertions(+) diff --git a/app/test/test_bpf.c b/app/test/test_bpf.c index 4a61a7d7c..7c3de96c6 100644 --- a/app/test/test_bpf.c +++ b/app/test/test_bpf.c @@ -43,6 +43,14 @@ struct dummy_net { struct rte_ipv4_hdr ip_hdr; }; +#define DUMMY_MBUF_NUM 2 + +/* first mbuf in the packet, should always be at offset 0 */ +struct dummy_mbuf { + struct rte_mbuf mb[DUMMY_MBUF_NUM]; + uint8_t buf[DUMMY_MBUF_NUM][RTE_MBUF_DEFAULT_BUF_SIZE]; +}; + #define TEST_FILL_1 0xDEADBEEF #define TEST_MUL_1 21 @@ -2444,6 +2452,413 @@ static const struct rte_bpf_xsym test_call5_xsym[] = { }, }; +/* load mbuf (BPF_ABS/BPF_IND) test-cases */ +static const struct ebpf_insn test_ld_mbuf1_prog[] = { + + /* BPF_ABS/BPF_IND implicitly expect mbuf ptr in R6 */ + { + .code = (EBPF_ALU64 | EBPF_MOV | BPF_X), + .dst_reg = EBPF_REG_6, + .src_reg = EBPF_REG_1, + }, + /* load IPv4 version and IHL */ + { + .code = (BPF_LD | BPF_ABS | BPF_B), + .imm = offsetof(struct rte_ipv4_hdr, version_ihl), + }, + /* check IP version */ + { + .code = (EBPF_ALU64 | EBPF_MOV | BPF_X), + .dst_reg = EBPF_REG_2, + .src_reg = EBPF_REG_0, + }, + { + .code = (BPF_ALU | BPF_AND | BPF_K), + .dst_reg = EBPF_REG_2, + .imm = 0xf0, + }, + { + .code = (BPF_JMP | BPF_JEQ | BPF_K), + .dst_reg = EBPF_REG_2, + .imm = IPVERSION << 4, + .off = 2, + }, + /* invalid IP version, return 0 */ + { + .code = (EBPF_ALU64 | BPF_XOR | BPF_X), + .dst_reg = EBPF_REG_0, + .src_reg = EBPF_REG_0, + }, + { + .code = (BPF_JMP | EBPF_EXIT), + }, + /* load 3-rd byte of IP data */ + { + .code = (BPF_ALU | BPF_AND | BPF_K), + .dst_reg = EBPF_REG_0, + .imm = RTE_IPV4_HDR_IHL_MASK, + }, + { + .code = (BPF_ALU | BPF_LSH | BPF_K), + .dst_reg = EBPF_REG_0, + .imm = 2, + }, + { + .code = (BPF_LD | BPF_IND | BPF_B), + .src_reg = EBPF_REG_0, + .imm = 3, + }, + { + .code = (EBPF_ALU64 | EBPF_MOV | BPF_X), + .dst_reg = EBPF_REG_7, + .src_reg = EBPF_REG_0, + }, + /* load IPv4 src addr */ + { + .code = (BPF_LD | BPF_ABS | BPF_W), + .imm = offsetof(struct rte_ipv4_hdr, src_addr), + }, + { + .code = (EBPF_ALU64 | BPF_ADD | BPF_X), + .dst_reg = EBPF_REG_7, + .src_reg = EBPF_REG_0, + }, + /* load IPv4 total length */ + { + .code = (BPF_LD | BPF_ABS | BPF_H), + .imm = offsetof(struct rte_ipv4_hdr, total_length), + }, + { + .code = (EBPF_ALU64 | EBPF_MOV | BPF_X), + .dst_reg = EBPF_REG_8, + .src_reg = EBPF_REG_0, + }, + /* load last 4 bytes of IP data */ + { + .code = (BPF_LD | BPF_IND | BPF_W), + .src_reg = EBPF_REG_8, + .imm = -(int32_t)sizeof(uint32_t), + }, + { + .code = (EBPF_ALU64 | BPF_ADD | BPF_X), + .dst_reg = EBPF_REG_7, + .src_reg = EBPF_REG_0, + }, + /* load 2 bytes from the middle of IP data */ + { + .code = (EBPF_ALU64 | BPF_RSH | BPF_K), + .dst_reg = EBPF_REG_8, + .imm = 1, + }, + { + .code = (BPF_LD | BPF_IND | BPF_H), + .src_reg = EBPF_REG_8, + }, + { + .code = (EBPF_ALU64 | BPF_ADD | BPF_X), + .dst_reg = EBPF_REG_0, + .src_reg = EBPF_REG_7, + }, + { + .code = (BPF_JMP | EBPF_EXIT), + }, +}; + +static void +dummy_mbuf_prep(struct rte_mbuf *mb, uint8_t buf[], uint32_t buf_len, + uint32_t data_len) +{ + uint32_t i; + uint8_t *db; + + mb->buf_addr = buf; + mb->buf_iova = (uintptr_t)buf; + mb->buf_len = buf_len; + rte_mbuf_refcnt_set(mb, 1); + + /* set pool pointer to dummy value, test doesn't use it */ + mb->pool = (void *)buf; + + rte_pktmbuf_reset(mb); + db = (uint8_t *)rte_pktmbuf_append(mb, data_len); + + for (i = 0; i != data_len; i++) + db[i] = i; +} + +static void +test_ld_mbuf1_prepare(void *arg) +{ + struct dummy_mbuf *dm; + struct rte_ipv4_hdr *ph; + + const uint32_t plen = 400; + const struct rte_ipv4_hdr iph = { + .version_ihl = RTE_IPV4_VHL_DEF, + .total_length = rte_cpu_to_be_16(plen), + .time_to_live = IPDEFTTL, + .next_proto_id = IPPROTO_RAW, + .src_addr = rte_cpu_to_be_32(RTE_IPV4_LOOPBACK), + .dst_addr = rte_cpu_to_be_32(RTE_IPV4_BROADCAST), + }; + + dm = arg; + memset(dm, 0, sizeof(*dm)); + + dummy_mbuf_prep(&dm->mb[0], dm->buf[0], sizeof(dm->buf[0]), + plen / 2 + 1); + dummy_mbuf_prep(&dm->mb[1], dm->buf[1], sizeof(dm->buf[0]), + plen / 2 - 1); + + rte_pktmbuf_chain(&dm->mb[0], &dm->mb[1]); + + ph = rte_pktmbuf_mtod(dm->mb, typeof(ph)); + memcpy(ph, &iph, sizeof(iph)); +} + +static uint64_t +test_ld_mbuf1(const struct rte_mbuf *pkt) +{ + uint64_t n, v; + const uint8_t *p8; + const uint16_t *p16; + const uint32_t *p32; + struct dummy_offset dof; + + /* load IPv4 version and IHL */ + p8 = rte_pktmbuf_read(pkt, + offsetof(struct rte_ipv4_hdr, version_ihl), sizeof(*p8), + &dof); + if (p8 == NULL) + return 0; + + /* check IP version */ + if ((p8[0] & 0xf0) != IPVERSION << 4) + return 0; + + n = (p8[0] & RTE_IPV4_HDR_IHL_MASK) * RTE_IPV4_IHL_MULTIPLIER; + + /* load 3-rd byte of IP data */ + p8 = rte_pktmbuf_read(pkt, n + 3, sizeof(*p8), &dof); + if (p8 == NULL) + return 0; + + v = p8[0]; + + /* load IPv4 src addr */ + p32 = rte_pktmbuf_read(pkt, + offsetof(struct rte_ipv4_hdr, src_addr), sizeof(*p32), + &dof); + if (p32 == NULL) + return 0; + + v += rte_be_to_cpu_32(p32[0]); + + /* load IPv4 total length */ + p16 = rte_pktmbuf_read(pkt, + offsetof(struct rte_ipv4_hdr, total_length), sizeof(*p16), + &dof); + if (p16 == NULL) + return 0; + + n = rte_be_to_cpu_16(p16[0]); + + /* load last 4 bytes of IP data */ + p32 = rte_pktmbuf_read(pkt, n - sizeof(*p32), sizeof(*p32), &dof); + if (p32 == NULL) + return 0; + + v += rte_be_to_cpu_32(p32[0]); + + /* load 2 bytes from the middle of IP data */ + p16 = rte_pktmbuf_read(pkt, n / 2, sizeof(*p16), &dof); + if (p16 == NULL) + return 0; + + v += rte_be_to_cpu_16(p16[0]); + return v; +} + +static int +test_ld_mbuf1_check(uint64_t rc, const void *arg) +{ + const struct dummy_mbuf *dm; + uint64_t v; + + dm = arg; + v = test_ld_mbuf1(dm->mb); + return cmp_res(__func__, v, rc, arg, arg, 0); +} + +/* + * same as ld_mbuf1, but then trancate the mbuf by 1B, + * so load of last 4B fail. + */ +static void +test_ld_mbuf2_prepare(void *arg) +{ + struct dummy_mbuf *dm; + + test_ld_mbuf1_prepare(arg); + dm = arg; + rte_pktmbuf_trim(dm->mb, 1); +} + +static int +test_ld_mbuf2_check(uint64_t rc, const void *arg) +{ + return cmp_res(__func__, 0, rc, arg, arg, 0); +} + +/* same as test_ld_mbuf1, but now store intermediate results on the stack */ +static const struct ebpf_insn test_ld_mbuf3_prog[] = { + + /* BPF_ABS/BPF_IND implicitly expect mbuf ptr in R6 */ + { + .code = (EBPF_ALU64 | EBPF_MOV | BPF_X), + .dst_reg = EBPF_REG_6, + .src_reg = EBPF_REG_1, + }, + /* load IPv4 version and IHL */ + { + .code = (BPF_LD | BPF_ABS | BPF_B), + .imm = offsetof(struct rte_ipv4_hdr, version_ihl), + }, + /* check IP version */ + { + .code = (EBPF_ALU64 | EBPF_MOV | BPF_X), + .dst_reg = EBPF_REG_2, + .src_reg = EBPF_REG_0, + }, + { + .code = (BPF_ALU | BPF_AND | BPF_K), + .dst_reg = EBPF_REG_2, + .imm = 0xf0, + }, + { + .code = (BPF_JMP | BPF_JEQ | BPF_K), + .dst_reg = EBPF_REG_2, + .imm = IPVERSION << 4, + .off = 2, + }, + /* invalid IP version, return 0 */ + { + .code = (EBPF_ALU64 | BPF_XOR | BPF_X), + .dst_reg = EBPF_REG_0, + .src_reg = EBPF_REG_0, + }, + { + .code = (BPF_JMP | EBPF_EXIT), + }, + /* load 3-rd byte of IP data */ + { + .code = (BPF_ALU | BPF_AND | BPF_K), + .dst_reg = EBPF_REG_0, + .imm = RTE_IPV4_HDR_IHL_MASK, + }, + { + .code = (BPF_ALU | BPF_LSH | BPF_K), + .dst_reg = EBPF_REG_0, + .imm = 2, + }, + { + .code = (BPF_LD | BPF_IND | BPF_B), + .src_reg = EBPF_REG_0, + .imm = 3, + }, + { + .code = (BPF_STX | BPF_MEM | BPF_B), + .dst_reg = EBPF_REG_10, + .src_reg = EBPF_REG_0, + .off = (int16_t)(offsetof(struct dummy_offset, u8) - + sizeof(struct dummy_offset)), + }, + /* load IPv4 src addr */ + { + .code = (BPF_LD | BPF_ABS | BPF_W), + .imm = offsetof(struct rte_ipv4_hdr, src_addr), + }, + { + .code = (BPF_STX | BPF_MEM | BPF_W), + .dst_reg = EBPF_REG_10, + .src_reg = EBPF_REG_0, + .off = (int16_t)(offsetof(struct dummy_offset, u32) - + sizeof(struct dummy_offset)), + }, + /* load IPv4 total length */ + { + .code = (BPF_LD | BPF_ABS | BPF_H), + .imm = offsetof(struct rte_ipv4_hdr, total_length), + }, + { + .code = (EBPF_ALU64 | EBPF_MOV | BPF_X), + .dst_reg = EBPF_REG_8, + .src_reg = EBPF_REG_0, + }, + /* load last 4 bytes of IP data */ + { + .code = (BPF_LD | BPF_IND | BPF_W), + .src_reg = EBPF_REG_8, + .imm = -(int32_t)sizeof(uint32_t), + }, + { + .code = (BPF_STX | BPF_MEM | EBPF_DW), + .dst_reg = EBPF_REG_10, + .src_reg = EBPF_REG_0, + .off = (int16_t)(offsetof(struct dummy_offset, u64) - + sizeof(struct dummy_offset)), + }, + /* load 2 bytes from the middle of IP data */ + { + .code = (EBPF_ALU64 | BPF_RSH | BPF_K), + .dst_reg = EBPF_REG_8, + .imm = 1, + }, + { + .code = (BPF_LD | BPF_IND | BPF_H), + .src_reg = EBPF_REG_8, + }, + { + .code = (BPF_LDX | BPF_MEM | EBPF_DW), + .dst_reg = EBPF_REG_1, + .src_reg = EBPF_REG_10, + .off = (int16_t)(offsetof(struct dummy_offset, u64) - + sizeof(struct dummy_offset)), + }, + { + .code = (EBPF_ALU64 | BPF_ADD | BPF_X), + .dst_reg = EBPF_REG_0, + .src_reg = EBPF_REG_1, + }, + { + .code = (BPF_LDX | BPF_MEM | BPF_W), + .dst_reg = EBPF_REG_1, + .src_reg = EBPF_REG_10, + .off = (int16_t)(offsetof(struct dummy_offset, u32) - + sizeof(struct dummy_offset)), + }, + { + .code = (EBPF_ALU64 | BPF_ADD | BPF_X), + .dst_reg = EBPF_REG_0, + .src_reg = EBPF_REG_1, + }, + { + .code = (BPF_LDX | BPF_MEM | BPF_B), + .dst_reg = EBPF_REG_1, + .src_reg = EBPF_REG_10, + .off = (int16_t)(offsetof(struct dummy_offset, u8) - + sizeof(struct dummy_offset)), + }, + { + .code = (EBPF_ALU64 | BPF_ADD | BPF_X), + .dst_reg = EBPF_REG_0, + .src_reg = EBPF_REG_1, + }, + { + .code = (BPF_JMP | EBPF_EXIT), + }, +}; + /* all bpf test cases */ static const struct bpf_test tests[] = { { @@ -2704,6 +3119,54 @@ static const struct bpf_test tests[] = { /* for now don't support function calls on 32 bit platform */ .allow_fail = (sizeof(uint64_t) != sizeof(uintptr_t)), }, + { + .name = "test_ld_mbuf1", + .arg_sz = sizeof(struct dummy_mbuf), + .prm = { + .ins = test_ld_mbuf1_prog, + .nb_ins = RTE_DIM(test_ld_mbuf1_prog), + .prog_arg = { + .type = RTE_BPF_ARG_PTR_MBUF, + .buf_size = sizeof(struct dummy_mbuf), + }, + }, + .prepare = test_ld_mbuf1_prepare, + .check_result = test_ld_mbuf1_check, + /* mbuf as input argument is not supported on 32 bit platform */ + .allow_fail = (sizeof(uint64_t) != sizeof(uintptr_t)), + }, + { + .name = "test_ld_mbuf2", + .arg_sz = sizeof(struct dummy_mbuf), + .prm = { + .ins = test_ld_mbuf1_prog, + .nb_ins = RTE_DIM(test_ld_mbuf1_prog), + .prog_arg = { + .type = RTE_BPF_ARG_PTR_MBUF, + .buf_size = sizeof(struct dummy_mbuf), + }, + }, + .prepare = test_ld_mbuf2_prepare, + .check_result = test_ld_mbuf2_check, + /* mbuf as input argument is not supported on 32 bit platform */ + .allow_fail = (sizeof(uint64_t) != sizeof(uintptr_t)), + }, + { + .name = "test_ld_mbuf3", + .arg_sz = sizeof(struct dummy_mbuf), + .prm = { + .ins = test_ld_mbuf3_prog, + .nb_ins = RTE_DIM(test_ld_mbuf3_prog), + .prog_arg = { + .type = RTE_BPF_ARG_PTR_MBUF, + .buf_size = sizeof(struct dummy_mbuf), + }, + }, + .prepare = test_ld_mbuf1_prepare, + .check_result = test_ld_mbuf1_check, + /* mbuf as input argument is not supported on 32 bit platform */ + .allow_fail = (sizeof(uint64_t) != sizeof(uintptr_t)), + }, }; static int From patchwork Wed May 27 14:16:53 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: "Ananyev, Konstantin" X-Patchwork-Id: 70650 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id B9E3BA034E; Wed, 27 May 2020 16:18:09 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 31A3A1DAAF; Wed, 27 May 2020 16:17:36 +0200 (CEST) Received: from mga09.intel.com (mga09.intel.com [134.134.136.24]) by dpdk.org (Postfix) with ESMTP id DBF2A1DA87 for ; Wed, 27 May 2020 16:17:33 +0200 (CEST) IronPort-SDR: gxRkZRCsB1ZEmZ7fvqJLFxgAfMWemtDoq/H7di7o9wTb6eobK1XiOztuHJ9xaJYGW4jC06mXzY xh9kZT2QYbKw== X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga004.fm.intel.com ([10.253.24.48]) by orsmga102.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 27 May 2020 07:17:33 -0700 IronPort-SDR: nkKJG2bJ6VawompvBGbe4A+//UIEMPjrUkbko3uzEf6VlVK7t6Eb4Ci5/0bwlMFt2/d26BNDdY 5uND4cG1d61w== X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.73,441,1583222400"; d="scan'208";a="291608940" Received: from sivswdev08.ir.intel.com ([10.237.217.47]) by fmsmga004.fm.intel.com with ESMTP; 27 May 2020 07:17:31 -0700 From: Konstantin Ananyev To: dev@dpdk.org Cc: jerinj@marvell.com, stephen@networkplumber.org, mb@smartsharesystems.com, Konstantin Ananyev Date: Wed, 27 May 2020 15:16:53 +0100 Message-Id: <20200527141653.15576-6-konstantin.ananyev@intel.com> X-Mailer: git-send-email 2.18.0 In-Reply-To: <20200527141653.15576-1-konstantin.ananyev@intel.com> References: <20200518155245.11380-1-konstantin.ananyev@intel.com> <20200527141653.15576-1-konstantin.ananyev@intel.com> Subject: [dpdk-dev] [PATCH v2 5/5] bpf: x86 JIT support for packet data load instructions X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Make x86 JIT to generate native code for (BPF_ABS | | BPF_LD) and (BPF_IND | | BPF_LD) instructions. Signed-off-by: Konstantin Ananyev Acked-by: Morten Brørup --- lib/librte_bpf/bpf_jit_x86.c | 181 ++++++++++++++++++++++++++++++++++ lib/librte_bpf/bpf_validate.c | 17 +++- 2 files changed, 197 insertions(+), 1 deletion(-) diff --git a/lib/librte_bpf/bpf_jit_x86.c b/lib/librte_bpf/bpf_jit_x86.c index f70cd6be5..aa22ea78a 100644 --- a/lib/librte_bpf/bpf_jit_x86.c +++ b/lib/librte_bpf/bpf_jit_x86.c @@ -87,6 +87,14 @@ enum { REG_TMP1 = R10, }; +/* LD_ABS/LD_IMM offsets */ +enum { + LDMB_FSP_OFS, /* fast-path */ + LDMB_SLP_OFS, /* slow-path */ + LDMB_FIN_OFS, /* final part */ + LDMB_OFS_NUM +}; + /* * callee saved registers list. * keep RBP as the last one. @@ -100,6 +108,9 @@ struct bpf_jit_state { uint32_t num; int32_t off; } exit; + struct { + uint32_t stack_ofs; + } ldmb; uint32_t reguse; int32_t *off; uint8_t *ins; @@ -1024,6 +1035,166 @@ emit_div(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg, emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP1, RDX); } +/* + * helper function, used by emit_ld_mbuf(). + * generates code for 'fast_path': + * calculate load offset and check is it inside first packet segment. + */ +static void +emit_ldmb_fast_path(struct bpf_jit_state *st, const uint32_t rg[EBPF_REG_7], + uint32_t sreg, uint32_t mode, uint32_t sz, uint32_t imm, + const int32_t ofs[LDMB_OFS_NUM]) +{ + /* make R2 contain *off* value */ + + if (sreg != rg[EBPF_REG_2]) { + emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, + rg[EBPF_REG_2], imm); + if (mode == BPF_IND) + emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X, + sreg, rg[EBPF_REG_2]); + } else + /* BPF_IND with sreg == R2 */ + emit_alu_imm(st, EBPF_ALU64 | BPF_ADD | BPF_K, + rg[EBPF_REG_2], imm); + + /* R3 = mbuf->data_len */ + emit_ld_reg(st, BPF_LDX | BPF_MEM | BPF_H, + rg[EBPF_REG_6], rg[EBPF_REG_3], + offsetof(struct rte_mbuf, data_len)); + + /* R3 = R3 - R2 */ + emit_alu_reg(st, EBPF_ALU64 | BPF_SUB | BPF_X, + rg[EBPF_REG_2], rg[EBPF_REG_3]); + + /* JSLT R3, */ + emit_cmp_imm(st, EBPF_ALU64, rg[EBPF_REG_3], sz); + emit_abs_jcc(st, BPF_JMP | EBPF_JSLT | BPF_K, ofs[LDMB_SLP_OFS]); + + /* R3 = mbuf->data_off */ + emit_ld_reg(st, BPF_LDX | BPF_MEM | BPF_H, + rg[EBPF_REG_6], rg[EBPF_REG_3], + offsetof(struct rte_mbuf, data_off)); + + /* R0 = mbuf->buf_addr */ + emit_ld_reg(st, BPF_LDX | BPF_MEM | EBPF_DW, + rg[EBPF_REG_6], rg[EBPF_REG_0], + offsetof(struct rte_mbuf, buf_addr)); + + /* R0 = R0 + R3 */ + emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X, + rg[EBPF_REG_3], rg[EBPF_REG_0]); + + /* R0 = R0 + R2 */ + emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X, + rg[EBPF_REG_2], rg[EBPF_REG_0]); + + /* JMP */ + emit_abs_jmp(st, ofs[LDMB_FIN_OFS]); +} + +/* + * helper function, used by emit_ld_mbuf(). + * generates code for 'slow_path': + * call __rte_pktmbuf_read() and check return value. + */ +static void +emit_ldmb_slow_path(struct bpf_jit_state *st, const uint32_t rg[EBPF_REG_7], + uint32_t sz) +{ + /* make R3 contain *len* value (1/2/4) */ + + emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, rg[EBPF_REG_3], sz); + + /* make R4 contain (RBP - ldmb.stack_ofs) */ + + emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RBP, rg[EBPF_REG_4]); + emit_alu_imm(st, EBPF_ALU64 | BPF_SUB | BPF_K, rg[EBPF_REG_4], + st->ldmb.stack_ofs); + + /* make R1 contain mbuf ptr */ + + emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, + rg[EBPF_REG_6], rg[EBPF_REG_1]); + + /* call rte_pktmbuf_read */ + emit_call(st, (uintptr_t)__rte_pktmbuf_read); + + /* check that return value (R0) is not zero */ + emit_tst_reg(st, EBPF_ALU64, rg[EBPF_REG_0], rg[EBPF_REG_0]); + emit_abs_jcc(st, BPF_JMP | BPF_JEQ | BPF_K, st->exit.off); +} + +/* + * helper function, used by emit_ld_mbuf(). + * generates final part of code for BPF_ABS/BPF_IND load: + * perform data load and endianness conversion. + * expects dreg to contain valid data pointer. + */ +static void +emit_ldmb_fin(struct bpf_jit_state *st, uint32_t dreg, uint32_t opsz, + uint32_t sz) +{ + emit_ld_reg(st, BPF_LDX | BPF_MEM | opsz, dreg, dreg, 0); + if (sz != sizeof(uint8_t)) + emit_be2le(st, dreg, sz * CHAR_BIT); +} + +/* + * emit code for BPF_ABS/BPF_IND load. + * generates the following construction: + * fast_path: + * off = ins->sreg + ins->imm + * if (mbuf->data_len - off < ins->opsz) + * goto slow_path; + * ptr = mbuf->buf_addr + mbuf->data_off + off; + * goto fin_part; + * slow_path: + * typeof(ins->opsz) buf; //allocate space on the stack + * ptr = __rte_pktmbuf_read(mbuf, off, ins->opsz, &buf); + * if (ptr == NULL) + * goto exit_label; + * fin_part: + * res = *(typeof(ins->opsz))ptr; + * res = bswap(res); + */ +static void +emit_ld_mbuf(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t imm) +{ + uint32_t i, mode, opsz, sz; + uint32_t rg[EBPF_REG_7]; + int32_t ofs[LDMB_OFS_NUM]; + + mode = BPF_MODE(op); + opsz = BPF_SIZE(op); + sz = bpf_size(opsz); + + for (i = 0; i != RTE_DIM(rg); i++) + rg[i] = ebpf2x86[i]; + + /* fill with fake offsets */ + for (i = 0; i != RTE_DIM(ofs); i++) + ofs[i] = st->sz + INT8_MAX; + + /* dry run first to calculate jump offsets */ + + ofs[LDMB_FSP_OFS] = st->sz; + emit_ldmb_fast_path(st, rg, sreg, mode, sz, imm, ofs); + ofs[LDMB_SLP_OFS] = st->sz; + emit_ldmb_slow_path(st, rg, sz); + ofs[LDMB_FIN_OFS] = st->sz; + emit_ldmb_fin(st, rg[EBPF_REG_0], opsz, sz); + + RTE_VERIFY(ofs[LDMB_FIN_OFS] - ofs[LDMB_FSP_OFS] <= INT8_MAX); + + /* reset dry-run code and do a proper run */ + + st->sz = ofs[LDMB_FSP_OFS]; + emit_ldmb_fast_path(st, rg, sreg, mode, sz, imm, ofs); + emit_ldmb_slow_path(st, rg, sz); + emit_ldmb_fin(st, rg[EBPF_REG_0], opsz, sz); +} + static void emit_prolog(struct bpf_jit_state *st, int32_t stack_size) { @@ -1121,6 +1292,7 @@ emit(struct bpf_jit_state *st, const struct rte_bpf *bpf) /* reset state fields */ st->sz = 0; st->exit.num = 0; + st->ldmb.stack_ofs = bpf->stack_sz; emit_prolog(st, bpf->stack_sz); @@ -1240,6 +1412,15 @@ emit(struct bpf_jit_state *st, const struct rte_bpf *bpf) emit_ld_imm64(st, dr, ins[0].imm, ins[1].imm); i++; break; + /* load absolute/indirect instructions */ + case (BPF_LD | BPF_ABS | BPF_B): + case (BPF_LD | BPF_ABS | BPF_H): + case (BPF_LD | BPF_ABS | BPF_W): + case (BPF_LD | BPF_IND | BPF_B): + case (BPF_LD | BPF_IND | BPF_H): + case (BPF_LD | BPF_IND | BPF_W): + emit_ld_mbuf(st, op, sr, ins->imm); + break; /* store instructions */ case (BPF_STX | BPF_MEM | BPF_B): case (BPF_STX | BPF_MEM | BPF_H): diff --git a/lib/librte_bpf/bpf_validate.c b/lib/librte_bpf/bpf_validate.c index fecdda0e1..9214f1503 100644 --- a/lib/librte_bpf/bpf_validate.c +++ b/lib/librte_bpf/bpf_validate.c @@ -70,6 +70,7 @@ struct bpf_verifier { uint64_t stack_sz; uint32_t nb_nodes; uint32_t nb_jcc_nodes; + uint32_t nb_ldmb_nodes; uint32_t node_colour[MAX_NODE_COLOUR]; uint32_t edge_type[MAX_EDGE_TYPE]; struct bpf_eval_state *evst; @@ -2020,6 +2021,14 @@ validate(struct bpf_verifier *bvf) rc |= add_edge(bvf, node, i + 2); i++; break; + case (BPF_LD | BPF_ABS | BPF_B): + case (BPF_LD | BPF_ABS | BPF_H): + case (BPF_LD | BPF_ABS | BPF_W): + case (BPF_LD | BPF_IND | BPF_B): + case (BPF_LD | BPF_IND | BPF_H): + case (BPF_LD | BPF_IND | BPF_W): + bvf->nb_ldmb_nodes++; + /* fallthrough */ default: rc |= add_edge(bvf, node, i + 1); break; @@ -2320,8 +2329,14 @@ bpf_validate(struct rte_bpf *bpf) free(bvf.in); /* copy collected info */ - if (rc == 0) + if (rc == 0) { bpf->stack_sz = bvf.stack_sz; + /* for LD_ABS/LD_IND, we'll need extra space on the stack */ + if (bvf.nb_ldmb_nodes != 0) + bpf->stack_sz = RTE_ALIGN_CEIL(bpf->stack_sz + + sizeof(uint64_t), sizeof(uint64_t)); + } + return rc; }