From patchwork Wed Feb 3 08:32:30 2016 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Rahul Lakkireddy X-Patchwork-Id: 10346 X-Patchwork-Delegate: bruce.richardson@intel.com Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [IPv6:::1]) by dpdk.org (Postfix) with ESMTP id 6944BC300; Wed, 3 Feb 2016 09:33:08 +0100 (CET) Received: from stargate3.asicdesigners.com (stargate.chelsio.com [12.32.117.8]) by dpdk.org (Postfix) with ESMTP id 2ADE7C300 for ; Wed, 3 Feb 2016 09:33:07 +0100 (CET) Received: from localhost (scalar.blr.asicdesigners.com [10.193.185.94]) by stargate3.asicdesigners.com (8.13.8/8.13.8) with ESMTP id u138X3PQ024916; Wed, 3 Feb 2016 00:33:04 -0800 From: Rahul Lakkireddy To: dev@dpdk.org Date: Wed, 3 Feb 2016 14:02:30 +0530 Message-Id: X-Mailer: git-send-email 2.5.3 In-Reply-To: References: In-Reply-To: References: Cc: Kumar Sanghvi , Nirranjan Kirubaharan Subject: [dpdk-dev] [PATCH 09/10] cxgbe: add HASH filtering support X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: patches and discussions about DPDK List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Add support for setting HASH (Maskless) filters. Both IPv4 and IPv6 occupy only one index per filter. Also, the index returned is a hash computed by the hardware based on value in the fields being matched. During matching, the hardware computes a hash of the relevant fields in the incoming packet and compares it against the hashed indices. If a match is found, then the filter's action is taken immediately. HASH filters have higher priority over LE-TCAM filters if a packet matches rules in both the HASH filter region and the LE-TCAM filter region. This can be changed by setting the prio bit in the filter specification. Signed-off-by: Rahul Lakkireddy Signed-off-by: Kumar Sanghvi --- drivers/net/cxgbe/base/adapter.h | 11 + drivers/net/cxgbe/base/common.h | 7 + drivers/net/cxgbe/base/t4_msg.h | 198 ++++++++ drivers/net/cxgbe/base/t4_regs.h | 9 + drivers/net/cxgbe/base/t4_regs_values.h | 25 + drivers/net/cxgbe/base/t4_tcb.h | 21 + drivers/net/cxgbe/base/t4fw_interface.h | 19 + drivers/net/cxgbe/cxgbe_compat.h | 12 + drivers/net/cxgbe/cxgbe_filter.c | 812 ++++++++++++++++++++++++++++++++ drivers/net/cxgbe/cxgbe_filter.h | 7 + drivers/net/cxgbe/cxgbe_main.c | 135 +++++- drivers/net/cxgbe/cxgbe_ofld.h | 26 + 12 files changed, 1280 insertions(+), 2 deletions(-) diff --git a/drivers/net/cxgbe/base/adapter.h b/drivers/net/cxgbe/base/adapter.h index a866993..a64571d 100644 --- a/drivers/net/cxgbe/base/adapter.h +++ b/drivers/net/cxgbe/base/adapter.h @@ -629,6 +629,17 @@ static inline struct adapter *ethdev2adap(const struct rte_eth_dev *dev) return ethdev2pinfo(dev)->adapter; } +/** + * cxgbe_port_viid - get the VI id of a port + * @dev: the device for the port + * + * Return the VI id of the given port. + */ +static inline unsigned int cxgbe_port_viid(const struct rte_eth_dev *dev) +{ + return ethdev2pinfo(dev)->viid; +} + void *t4_alloc_mem(size_t size); void t4_free_mem(void *addr); #define t4_os_alloc(_size) t4_alloc_mem((_size)) diff --git a/drivers/net/cxgbe/base/common.h b/drivers/net/cxgbe/base/common.h index 21dca32..bbeca75 100644 --- a/drivers/net/cxgbe/base/common.h +++ b/drivers/net/cxgbe/base/common.h @@ -220,6 +220,8 @@ struct adapter_params { unsigned char nports; /* # of ethernet ports */ unsigned char portvec; + unsigned char hash_filter; + enum chip_type chip; /* chip code */ struct arch_specific_params arch; /* chip specific params */ @@ -255,6 +257,11 @@ static inline int t4_wait_op_done(struct adapter *adapter, int reg, u32 mask, #define for_each_port(adapter, iter) \ for (iter = 0; iter < (adapter)->params.nports; ++iter) +static inline int is_hashfilter(const struct adapter *adap) +{ + return adap->params.hash_filter; +} + void t4_read_mtu_tbl(struct adapter *adap, u16 *mtus, u8 *mtu_log); void t4_tp_wr_bits_indirect(struct adapter *adap, unsigned int addr, unsigned int mask, unsigned int val); diff --git a/drivers/net/cxgbe/base/t4_msg.h b/drivers/net/cxgbe/base/t4_msg.h index 57534f0..bffea8b 100644 --- a/drivers/net/cxgbe/base/t4_msg.h +++ b/drivers/net/cxgbe/base/t4_msg.h @@ -35,12 +35,19 @@ #define T4_MSG_H enum { + CPL_ACT_OPEN_REQ = 0x3, CPL_SET_TCB_FIELD = 0x5, + CPL_ABORT_REQ = 0xA, + CPL_ABORT_RPL = 0xB, CPL_L2T_WRITE_REQ = 0x12, CPL_SMT_WRITE_REQ = 0x14, + CPL_TID_RELEASE = 0x1A, CPL_L2T_WRITE_RPL = 0x23, + CPL_ACT_OPEN_RPL = 0x25, + CPL_ABORT_RPL_RSS = 0x2D, CPL_SMT_WRITE_RPL = 0x2E, CPL_SET_TCB_RPL = 0x3A, + CPL_ACT_OPEN_REQ6 = 0x83, CPL_SGE_EGR_UPDATE = 0xA5, CPL_FW4_MSG = 0xC0, CPL_FW6_MSG = 0xE0, @@ -53,6 +60,16 @@ enum CPL_error { CPL_ERR_TCAM_FULL = 3, }; +enum { + ULP_MODE_NONE = 0, + ULP_MODE_TCPDDP = 5, +}; + +enum { + CPL_ABORT_SEND_RST = 0, + CPL_ABORT_NO_RST, +}; + enum { /* TX_PKT_XT checksum types */ TX_CSUM_TCPIP = 8, TX_CSUM_UDPIP = 9, @@ -127,6 +144,148 @@ struct work_request_hdr { #define WR_HDR_SIZE 0 #endif +/* option 0 fields */ +#define S_TX_CHAN 2 +#define V_TX_CHAN(x) ((x) << S_TX_CHAN) + +#define S_NO_CONG 4 +#define V_NO_CONG(x) ((x) << S_NO_CONG) + +#define S_DELACK 5 +#define V_DELACK(x) ((x) << S_DELACK) + +#define S_NON_OFFLOAD 7 +#define V_NON_OFFLOAD(x) ((x) << S_NON_OFFLOAD) +#define F_NON_OFFLOAD V_NON_OFFLOAD(1U) + +#define S_ULP_MODE 8 +#define V_ULP_MODE(x) ((x) << S_ULP_MODE) + +#define S_SMAC_SEL 28 +#define V_SMAC_SEL(x) ((__u64)(x) << S_SMAC_SEL) + +#define S_L2T_IDX 36 +#define V_L2T_IDX(x) ((__u64)(x) << S_L2T_IDX) + +#define S_TCAM_BYPASS 48 +#define V_TCAM_BYPASS(x) ((__u64)(x) << S_TCAM_BYPASS) +#define F_TCAM_BYPASS V_TCAM_BYPASS(1ULL) + +#define S_NAGLE 49 +#define V_NAGLE(x) ((__u64)(x) << S_NAGLE) + +/* option 2 fields */ +#define S_RSS_QUEUE 0 +#define V_RSS_QUEUE(x) ((x) << S_RSS_QUEUE) + +#define S_RSS_QUEUE_VALID 10 +#define V_RSS_QUEUE_VALID(x) ((x) << S_RSS_QUEUE_VALID) +#define F_RSS_QUEUE_VALID V_RSS_QUEUE_VALID(1U) + +#define S_CONG_CNTRL 14 +#define V_CONG_CNTRL(x) ((x) << S_CONG_CNTRL) + +#define S_PACE 16 +#define V_PACE(x) ((x) << S_PACE) + +#define S_RX_FC_DISABLE 20 +#define V_RX_FC_DISABLE(x) ((x) << S_RX_FC_DISABLE) + +#define S_TX_QUEUE 23 +#define V_TX_QUEUE(x) ((x) << S_TX_QUEUE) + +#define S_RX_CHANNEL 26 +#define V_RX_CHANNEL(x) ((x) << S_RX_CHANNEL) +#define F_RX_CHANNEL V_RX_CHANNEL(1U) + +#define S_CCTRL_ECN 27 +#define V_CCTRL_ECN(x) ((x) << S_CCTRL_ECN) + +#define S_WND_SCALE_EN 28 +#define V_WND_SCALE_EN(x) ((x) << S_WND_SCALE_EN) + +#define S_SACK_EN 30 +#define V_SACK_EN(x) ((x) << S_SACK_EN) + +#define S_T5_OPT_2_VALID 31 +#define V_T5_OPT_2_VALID(x) ((x) << S_T5_OPT_2_VALID) +#define F_T5_OPT_2_VALID V_T5_OPT_2_VALID(1U) + +struct cpl_act_open_req { + WR_HDR; + union opcode_tid ot; + __be16 local_port; + __be16 peer_port; + __be32 local_ip; + __be32 peer_ip; + __be64 opt0; + __be32 params; + __be32 opt2; +}; + +#define S_FILTER_TUPLE 24 +#define V_FILTER_TUPLE(x) ((x) << S_FILTER_TUPLE) + +struct cpl_t5_act_open_req { + WR_HDR; + union opcode_tid ot; + __be16 local_port; + __be16 peer_port; + __be32 local_ip; + __be32 peer_ip; + __be64 opt0; + __be32 rsvd; + __be32 opt2; + __be64 params; +}; + +/* cpl_{t5,t6}_act_open_req.params field */ +struct cpl_act_open_req6 { + WR_HDR; + union opcode_tid ot; + __be16 local_port; + __be16 peer_port; + __be64 local_ip_hi; + __be64 local_ip_lo; + __be64 peer_ip_hi; + __be64 peer_ip_lo; + __be64 opt0; + __be32 params; + __be32 opt2; +}; + +struct cpl_t5_act_open_req6 { + WR_HDR; + union opcode_tid ot; + __be16 local_port; + __be16 peer_port; + __be64 local_ip_hi; + __be64 local_ip_lo; + __be64 peer_ip_hi; + __be64 peer_ip_lo; + __be64 opt0; + __be32 rsvd; + __be32 opt2; + __be64 params; +}; + +struct cpl_act_open_rpl { + RSS_HDR + union opcode_tid ot; + __be32 atid_status; +}; + +/* cpl_act_open_rpl.atid_status fields */ +#define S_AOPEN_STATUS 0 +#define M_AOPEN_STATUS 0xFF +#define V_AOPEN_STATUS(x) ((x) << S_AOPEN_STATUS) +#define G_AOPEN_STATUS(x) (((x) >> S_AOPEN_STATUS) & M_AOPEN_STATUS) + +#define S_AOPEN_ATID 8 +#define M_AOPEN_ATID 0xFFFFFF +#define V_AOPEN_ATID(x) ((x) << S_AOPEN_ATID) +#define G_AOPEN_ATID(x) (((x) >> S_AOPEN_ATID) & M_AOPEN_ATID) + /* cpl_get_tcb.reply_ctrl fields */ #define S_QUEUENO 0 #define V_QUEUENO(x) ((x) << S_QUEUENO) @@ -164,6 +323,39 @@ struct cpl_set_tcb_rpl { __be64 oldval; }; +/* cpl_abort_req status command code + */ +struct cpl_abort_req { + WR_HDR; + union opcode_tid ot; + __be32 rsvd0; + __u8 rsvd1; + __u8 cmd; + __u8 rsvd2[6]; +}; + +struct cpl_abort_rpl_rss { + RSS_HDR + union opcode_tid ot; + __u8 rsvd[3]; + __u8 status; +}; + +struct cpl_abort_rpl { + WR_HDR; + union opcode_tid ot; + __be32 rsvd0; + __u8 rsvd1; + __u8 cmd; + __u8 rsvd2[6]; +}; + +struct cpl_tid_release { + WR_HDR; + union opcode_tid ot; + __be32 rsvd; +}; + struct cpl_tx_data { union opcode_tid ot; __be32 len; @@ -411,7 +603,13 @@ struct cpl_fw6_msg { __be64 data[4]; }; +/* ULP_TX opcodes */ +enum { + ULP_TX_PKT = 4 +}; + enum { + ULP_TX_SC_NOOP = 0x80, ULP_TX_SC_IMM = 0x81, ULP_TX_SC_DSGL = 0x82, ULP_TX_SC_ISGL = 0x83 diff --git a/drivers/net/cxgbe/base/t4_regs.h b/drivers/net/cxgbe/base/t4_regs.h index 9057e40..1a7b6df 100644 --- a/drivers/net/cxgbe/base/t4_regs.h +++ b/drivers/net/cxgbe/base/t4_regs.h @@ -793,3 +793,12 @@ #define M_REV 0xfU #define V_REV(x) ((x) << S_REV) #define G_REV(x) (((x) >> S_REV) & M_REV) + +/* registers for module LE */ +#define A_LE_DB_CONFIG 0x19c04 + +#define S_HASHEN 20 +#define V_HASHEN(x) ((x) << S_HASHEN) +#define F_HASHEN V_HASHEN(1U) + +#define A_LE_DB_TID_HASHBASE 0x19df8 diff --git a/drivers/net/cxgbe/base/t4_regs_values.h b/drivers/net/cxgbe/base/t4_regs_values.h index d7d3144..8e1f6f3 100644 --- a/drivers/net/cxgbe/base/t4_regs_values.h +++ b/drivers/net/cxgbe/base/t4_regs_values.h @@ -155,6 +155,9 @@ * selects for a particular field being present. These fields, when present * in the Compressed Filter Tuple, have the following widths in bits. */ +#define S_FT_FIRST S_FCOE +#define S_FT_LAST S_FRAGMENTATION + #define W_FT_FCOE 1 #define W_FT_PORT 3 #define W_FT_VNIC_ID 17 @@ -166,4 +169,26 @@ #define W_FT_MPSHITTYPE 3 #define W_FT_FRAGMENTATION 1 +/* + * Some of the Compressed Filter Tuple fields have internal structure. These + * bit shifts/masks describe those structures. All shifts are relative to the + * base position of the fields within the Compressed Filter Tuple + */ +#define S_FT_VLAN_VLD 16 +#define V_FT_VLAN_VLD(x) ((x) << S_FT_VLAN_VLD) +#define F_FT_VLAN_VLD V_FT_VLAN_VLD(1U) + +#define S_FT_VNID_ID_VF 0 +#define M_FT_VNID_ID_VF 0x7fU +#define V_FT_VNID_ID_VF(x) ((x) << S_FT_VNID_ID_VF) +#define G_FT_VNID_ID_VF(x) (((x) >> S_FT_VNID_ID_VF) & M_FT_VNID_ID_VF) + +#define S_FT_VNID_ID_PF 7 +#define M_FT_VNID_ID_PF 0x7U +#define V_FT_VNID_ID_PF(x) ((x) << S_FT_VNID_ID_PF) +#define G_FT_VNID_ID_PF(x) (((x) >> S_FT_VNID_ID_PF) & M_FT_VNID_ID_PF) + +#define S_FT_VNID_ID_VLD 16 +#define V_FT_VNID_ID_VLD(x) ((x) << S_FT_VNID_ID_VLD) +#define F_FT_VNID_ID_VLD(x) V_FT_VNID_ID_VLD(1U) #endif /* __T4_REGS_VALUES_H__ */ diff --git a/drivers/net/cxgbe/base/t4_tcb.h b/drivers/net/cxgbe/base/t4_tcb.h index 36afd56..1a076f3 100644 --- a/drivers/net/cxgbe/base/t4_tcb.h +++ b/drivers/net/cxgbe/base/t4_tcb.h @@ -60,6 +60,27 @@ #define M_TCB_T_RTT_TS_RECENT_AGE 0xffffffffULL #define V_TCB_T_RTT_TS_RECENT_AGE(x) ((x) << S_TCB_T_RTT_TS_RECENT_AGE) +/* 347:320 */ +#define W_TCB_SND_UNA_RAW 10 + +/* 553:522 */ +#define W_TCB_RCV_NXT 16 +#define S_TCB_RCV_NXT 10 +#define M_TCB_RCV_NXT 0xffffffffULL +#define V_TCB_RCV_NXT(x) ((__u64)(x) << S_TCB_RCV_NXT) + +/* 891:875 */ +#define W_TCB_RX_FRAG2_PTR_RAW 27 + +/* 964:937 */ +#define W_TCB_RX_FRAG3_LEN_RAW 29 + +/* 992:965 */ +#define W_TCB_RX_FRAG3_START_IDX_OFFSET_RAW 30 + +/* 1000:993 */ +#define W_TCB_PDU_HDR_LEN 31 + #define S_TF_MIGRATING 0 #define V_TF_MIGRATING(x) ((x) << S_TF_MIGRATING) diff --git a/drivers/net/cxgbe/base/t4fw_interface.h b/drivers/net/cxgbe/base/t4fw_interface.h index d3e4de5..d9278ff 100644 --- a/drivers/net/cxgbe/base/t4fw_interface.h +++ b/drivers/net/cxgbe/base/t4fw_interface.h @@ -83,6 +83,7 @@ enum fw_memtype { enum fw_wr_opcodes { FW_FILTER_WR = 0x02, + FW_ULPTX_WR = 0x04, FW_TP_WR = 0x05, FW_ETH_TX_PKT_WR = 0x08, FW_ETH_TX_PKTS_WR = 0x09, @@ -1009,6 +1010,24 @@ struct fw_eq_ctrl_cmd { #define S_FW_EQ_CTRL_CMD_EQSIZE 0 #define V_FW_EQ_CTRL_CMD_EQSIZE(x) ((x) << S_FW_EQ_CTRL_CMD_EQSIZE) +/* Macros for VIID parsing: + * VIID - [10:8] PFN, [7] VI Valid, [6:0] VI number + */ +#define S_FW_VIID_PFN 8 +#define M_FW_VIID_PFN 0x7 +#define V_FW_VIID_PFN(x) ((x) << S_FW_VIID_PFN) +#define G_FW_VIID_PFN(x) (((x) >> S_FW_VIID_PFN) & M_FW_VIID_PFN) + +#define S_FW_VIID_VIVLD 7 +#define M_FW_VIID_VIVLD 0x1 +#define V_FW_VIID_VIVLD(x) ((x) << S_FW_VIID_VIVLD) +#define G_FW_VIID_VIVLD(x) (((x) >> S_FW_VIID_VIVLD) & M_FW_VIID_VIVLD) + +#define S_FW_VIID_VIN 0 +#define M_FW_VIID_VIN 0x7F +#define V_FW_VIID_VIN(x) ((x) << S_FW_VIID_VIN) +#define G_FW_VIID_VIN(x) (((x) >> S_FW_VIID_VIN) & M_FW_VIID_VIN) + enum fw_vi_func { FW_VI_FUNC_ETH, }; diff --git a/drivers/net/cxgbe/cxgbe_compat.h b/drivers/net/cxgbe/cxgbe_compat.h index e68f8f5..6649afc 100644 --- a/drivers/net/cxgbe/cxgbe_compat.h +++ b/drivers/net/cxgbe/cxgbe_compat.h @@ -263,4 +263,16 @@ static inline void writeq(u64 val, volatile void __iomem *addr) writel(val >> 32, (void *)((uintptr_t)addr + 4)); } +/* + * Multiplies an integer by a fraction, while avoiding unnecessary + * overflow or loss of precision. + */ +#define mult_frac(x, numer, denom)( \ +{ \ + typeof(x) quot = (x) / (denom); \ + typeof(x) rem = (x) % (denom); \ + (quot * (numer)) + ((rem * (numer)) / (denom)); \ +} \ +) + #endif /* _CXGBE_COMPAT_H_ */ diff --git a/drivers/net/cxgbe/cxgbe_filter.c b/drivers/net/cxgbe/cxgbe_filter.c index d4e32b1..285381b 100644 --- a/drivers/net/cxgbe/cxgbe_filter.c +++ b/drivers/net/cxgbe/cxgbe_filter.c @@ -41,6 +41,44 @@ #include "cxgbe_filter.h" /** + * Initialize Hash Filters + */ +int init_hash_filter(struct adapter *adap) +{ + unsigned int n_user_filters; + unsigned int user_filter_perc; + int ret; + u32 params[7], val[7]; + +#define FW_PARAM_DEV(param) \ + (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ + V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) + +#define FW_PARAM_PFVF(param) \ + (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ + V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param) | \ + V_FW_PARAMS_PARAM_Y(0) | \ + V_FW_PARAMS_PARAM_Z(0)) + + params[0] = FW_PARAM_DEV(NTID); + ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, + params, val); + if (ret < 0) + return ret; + adap->tids.ntids = val[0]; + adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS); + + user_filter_perc = 100; + n_user_filters = mult_frac(adap->tids.nftids, + user_filter_perc, + 100); + + adap->tids.nftids = n_user_filters; + adap->params.hash_filter = 1; + return 0; +} + +/** * Validate if the requested filter specification can be set by checking * if the requested features have been enabled */ @@ -190,6 +228,556 @@ static void set_tcb_tflag(struct adapter *adap, unsigned int ftid, } /** + * Build a CPL_SET_TCB_FIELD message as payload of a ULP_TX_PKT command. + */ +static inline void mk_set_tcb_field_ulp(struct filter_entry *f, + struct cpl_set_tcb_field *req, + unsigned int word, + u64 mask, u64 val, u8 cookie, + int no_reply) +{ + struct ulp_txpkt *txpkt = (struct ulp_txpkt *)req; + struct ulptx_idata *sc = (struct ulptx_idata *)(txpkt + 1); + + txpkt->cmd_dest = cpu_to_be32(V_ULPTX_CMD(ULP_TX_PKT) | + V_ULP_TXPKT_DEST(0)); + txpkt->len = cpu_to_be32(DIV_ROUND_UP(sizeof(*req), 16)); + sc->cmd_more = cpu_to_be32(V_ULPTX_CMD(ULP_TX_SC_IMM)); + sc->len = cpu_to_be32(sizeof(*req) - sizeof(struct work_request_hdr)); + OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_SET_TCB_FIELD, f->tid)); + req->reply_ctrl = cpu_to_be16(V_NO_REPLY(no_reply) | V_REPLY_CHAN(0) | + V_QUEUENO(0)); + req->word_cookie = cpu_to_be16(V_WORD(word) | V_COOKIE(cookie)); + req->mask = cpu_to_be64(mask); + req->val = cpu_to_be64(val); + sc = (struct ulptx_idata *)(req + 1); + sc->cmd_more = cpu_to_be32(V_ULPTX_CMD(ULP_TX_SC_NOOP)); + sc->len = cpu_to_be32(0); +} + +/** + * Set NAT parameters + */ +static void set_nat_params(struct adapter *adap, struct filter_entry *f, + unsigned int tid, bool dip, bool sip, + bool dp, bool sp) +{ + if (dip) { + if (f->fs.type) { + set_tcb_field(adap, tid, W_TCB_SND_UNA_RAW, + WORD_MASK, f->fs.nat_lip[15] | + f->fs.nat_lip[14] << 8 | + f->fs.nat_lip[13] << 16 | + f->fs.nat_lip[12] << 24, 1); + + set_tcb_field(adap, tid, W_TCB_SND_UNA_RAW + 1, + WORD_MASK, f->fs.nat_lip[11] | + f->fs.nat_lip[10] << 8 | + f->fs.nat_lip[9] << 16 | + f->fs.nat_lip[8] << 24, 1); + + set_tcb_field(adap, tid, W_TCB_SND_UNA_RAW + 2, + WORD_MASK, f->fs.nat_lip[7] | + f->fs.nat_lip[6] << 8 | + f->fs.nat_lip[5] << 16 | + f->fs.nat_lip[4] << 24, 1); + + set_tcb_field(adap, tid, W_TCB_SND_UNA_RAW + 3, + WORD_MASK, f->fs.nat_lip[3] | + f->fs.nat_lip[2] << 8 | + f->fs.nat_lip[1] << 16 | + f->fs.nat_lip[0] << 24, 1); + } else { + set_tcb_field(adap, tid, W_TCB_RX_FRAG3_LEN_RAW, + WORD_MASK, f->fs.nat_lip[3] | + f->fs.nat_lip[2] << 8 | + f->fs.nat_lip[1] << 16 | + f->fs.nat_lip[0] << 24, 1); + } + } + + if (sip) { + if (f->fs.type) { + set_tcb_field(adap, tid, W_TCB_RX_FRAG2_PTR_RAW, + WORD_MASK, f->fs.nat_fip[15] | + f->fs.nat_fip[14] << 8 | + f->fs.nat_fip[13] << 16 | + f->fs.nat_fip[12] << 24, 1); + + set_tcb_field(adap, tid, W_TCB_RX_FRAG2_PTR_RAW + 1, + WORD_MASK, f->fs.nat_fip[11] | + f->fs.nat_fip[10] << 8 | + f->fs.nat_fip[9] << 16 | + f->fs.nat_fip[8] << 24, 1); + + set_tcb_field(adap, tid, W_TCB_RX_FRAG2_PTR_RAW + 2, + WORD_MASK, f->fs.nat_fip[7] | + f->fs.nat_fip[6] << 8 | + f->fs.nat_fip[5] << 16 | + f->fs.nat_fip[4] << 24, 1); + + set_tcb_field(adap, tid, W_TCB_RX_FRAG2_PTR_RAW + 3, + WORD_MASK, f->fs.nat_fip[3] | + f->fs.nat_fip[2] << 8 | + f->fs.nat_fip[1] << 16 | + f->fs.nat_fip[0] << 24, 1); + + } else { + set_tcb_field(adap, tid, + W_TCB_RX_FRAG3_START_IDX_OFFSET_RAW, + WORD_MASK, f->fs.nat_fip[3] | + f->fs.nat_fip[2] << 8 | + f->fs.nat_fip[1] << 16 | + f->fs.nat_fip[0] << 24, 1); + } + } + + set_tcb_field(adap, tid, W_TCB_PDU_HDR_LEN, WORD_MASK, + (dp ? f->fs.nat_lport : 0) | + (sp ? f->fs.nat_fport << 16 : 0), 1); +} + +/** + * Build a CPL_ABORT_REQ message as payload of a ULP_TX_PKT command. + */ +static void mk_abort_req_ulp(struct cpl_abort_req *abort_req, + unsigned int tid) +{ + struct ulp_txpkt *txpkt = (struct ulp_txpkt *)abort_req; + struct ulptx_idata *sc = (struct ulptx_idata *)(txpkt + 1); + + txpkt->cmd_dest = cpu_to_be32(V_ULPTX_CMD(ULP_TX_PKT) | + V_ULP_TXPKT_DEST(0)); + txpkt->len = cpu_to_be32(DIV_ROUND_UP(sizeof(*abort_req), 16)); + sc->cmd_more = cpu_to_be32(V_ULPTX_CMD(ULP_TX_SC_IMM)); + sc->len = cpu_to_be32(sizeof(*abort_req) - + sizeof(struct work_request_hdr)); + OPCODE_TID(abort_req) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_REQ, tid)); + abort_req->rsvd0 = cpu_to_be32(0); + abort_req->rsvd1 = 0; + abort_req->cmd = CPL_ABORT_NO_RST; + sc = (struct ulptx_idata *)(abort_req + 1); + sc->cmd_more = cpu_to_be32(V_ULPTX_CMD(ULP_TX_SC_NOOP)); + sc->len = cpu_to_be32(0); +} + +/** + * Build a CPL_ABORT_RPL message as payload of a ULP_TX_PKT command. + */ +static void mk_abort_rpl_ulp(struct cpl_abort_rpl *abort_rpl, + unsigned int tid) +{ + struct ulp_txpkt *txpkt = (struct ulp_txpkt *)abort_rpl; + struct ulptx_idata *sc = (struct ulptx_idata *)(txpkt + 1); + + txpkt->cmd_dest = cpu_to_be32(V_ULPTX_CMD(ULP_TX_PKT) | + V_ULP_TXPKT_DEST(0)); + txpkt->len = cpu_to_be32(DIV_ROUND_UP(sizeof(*abort_rpl), 16)); + sc->cmd_more = cpu_to_be32(V_ULPTX_CMD(ULP_TX_SC_IMM)); + sc->len = cpu_to_be32(sizeof(*abort_rpl) - + sizeof(struct work_request_hdr)); + OPCODE_TID(abort_rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_RPL, tid)); + abort_rpl->rsvd0 = cpu_to_be32(0); + abort_rpl->rsvd1 = 0; + abort_rpl->cmd = CPL_ABORT_NO_RST; + sc = (struct ulptx_idata *)(abort_rpl + 1); + sc->cmd_more = cpu_to_be32(V_ULPTX_CMD(ULP_TX_SC_NOOP)); + sc->len = cpu_to_be32(0); +} + +/** + * Delete the specified hash filter. + */ +static int cxgbe_del_hash_filter(struct rte_eth_dev *dev, + unsigned int filter_id, + struct filter_ctx *ctx) +{ + struct adapter *adapter = ethdev2adap(dev); + struct tid_info *t = &adapter->tids; + struct filter_entry *f; + struct sge_ctrl_txq *ctrlq; + unsigned int port_id = ethdev2pinfo(dev)->port_id; + int ret; + + if (filter_id > adapter->tids.ntids) + return -E2BIG; + + f = lookup_tid(t, filter_id); + if (!f) { + dev_err(adapter, "%s: no filter entry for filter_id = %d\n", + __func__, filter_id); + return -EINVAL; + } + + ret = writable_filter(f); + if (ret) + return ret; + + if (f->valid) { + unsigned int wrlen; + struct rte_mbuf *mbuf; + struct work_request_hdr *wr; + struct ulptx_idata *aligner; + struct cpl_set_tcb_field *req; + struct cpl_abort_req *abort_req; + struct cpl_abort_rpl *abort_rpl; + + f->ctx = ctx; + f->pending = 1; + + wrlen = cxgbe_roundup(sizeof(*wr) + + (sizeof(*req) + sizeof(*aligner)) + + sizeof(*abort_req) + sizeof(*abort_rpl), + 16); + + ctrlq = &adapter->sge.ctrlq[port_id]; + mbuf = rte_pktmbuf_alloc(ctrlq->mb_pool); + if (!mbuf) { + dev_err(adapter, "%s: could not allocate skb ..\n", + __func__); + goto out_err; + } + + mbuf->data_len = wrlen; + mbuf->pkt_len = mbuf->data_len; + + req = rte_pktmbuf_mtod(mbuf, struct cpl_set_tcb_field *); + INIT_ULPTX_WR(req, wrlen, 0, 0); + wr = (struct work_request_hdr *)req; + wr++; + req = (struct cpl_set_tcb_field *)wr; + mk_set_tcb_field_ulp(f, req, W_TCB_RSS_INFO, + V_TCB_RSS_INFO(M_TCB_RSS_INFO), + V_TCB_RSS_INFO( + adapter->sge.fw_evtq.abs_id), + 0, 1); + aligner = (struct ulptx_idata *)(req + 1); + abort_req = (struct cpl_abort_req *)(aligner + 1); + mk_abort_req_ulp(abort_req, f->tid); + abort_rpl = (struct cpl_abort_rpl *)(abort_req + 1); + mk_abort_rpl_ulp(abort_rpl, f->tid); + t4_mgmt_tx(ctrlq, mbuf); + } + return 0; + +out_err: + return -ENOMEM; +} + +/** + * Construct hash filter ntuple. + */ +static u64 hash_filter_ntuple(const struct filter_entry *f) +{ + struct adapter *adap = ethdev2adap(f->dev); + struct tp_params *tp = &adap->params.tp; + u64 ntuple = 0; + u16 tcp_proto = 6; /* TCP Protocol Number */ + + /* + * Initialize each of the fields which we care about which are present + * in the Compressed Filter Tuple. + */ + if (tp->vlan_shift >= 0 && f->fs.mask.ivlan) + ntuple |= (F_FT_VLAN_VLD | f->fs.val.ivlan) << tp->vlan_shift; + + if (tp->port_shift >= 0 && f->fs.mask.iport) + ntuple |= (u64)f->fs.val.iport << tp->port_shift; + + if (tp->protocol_shift >= 0) { + if (!f->fs.val.proto) + ntuple |= (u64)tcp_proto << tp->protocol_shift; + else + ntuple |= (u64)f->fs.val.proto << tp->protocol_shift; + } + + if (tp->tos_shift >= 0 && f->fs.mask.tos) + ntuple |= (u64)(f->fs.val.tos) << tp->tos_shift; + + if (tp->vnic_shift >= 0 && + (f->fs.mask.ovlan || f->fs.mask.pf || f->fs.mask.vf)) { + u32 viid = cxgbe_port_viid(f->dev); + u32 vf = G_FW_VIID_VIN(viid); + u32 pf = G_FW_VIID_PFN(viid); + u32 vld = G_FW_VIID_VIVLD(viid); + + ntuple |= (u64)(V_FT_VNID_ID_VF(vf) | + V_FT_VNID_ID_PF(pf) | + V_FT_VNID_ID_VLD(vld)) << tp->vnic_shift; + } + + if (tp->ethertype_shift >= 0 && f->fs.mask.ethtype) + ntuple |= (u64)(f->fs.val.ethtype) << tp->ethertype_shift; + + return ntuple; +} + +/** + * Build a ACT_OPEN_REQ6 message for setting IPv6 hash filter. + */ +static void mk_act_open_req6(struct filter_entry *f, struct rte_mbuf *mbuf, + unsigned int qid_filterid, struct adapter *adap) +{ + struct cpl_act_open_req6 *req = NULL; + struct cpl_t5_act_open_req6 *t5req = NULL; + u64 local_lo, local_hi, peer_lo, peer_hi; + u32 *lip = (u32 *)f->fs.val.lip; + u32 *fip = (u32 *)f->fs.val.fip; + + switch (CHELSIO_CHIP_VERSION(adap->params.chip)) { + case CHELSIO_T5: + t5req = rte_pktmbuf_mtod(mbuf, struct cpl_t5_act_open_req6 *); + + INIT_TP_WR(t5req, 0); + req = (struct cpl_act_open_req6 *)t5req; + break; + default: + dev_err(adap, "%s: unsupported chip type!\n", __func__); + return; + } + + local_hi = ((u64)lip[1]) << 32 | lip[0]; + local_lo = ((u64)lip[3]) << 32 | lip[2]; + peer_hi = ((u64)fip[1]) << 32 | fip[0]; + peer_lo = ((u64)fip[3]) << 32 | fip[2]; + + OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, + qid_filterid)); + req->local_port = cpu_to_be16(f->fs.val.lport); + req->peer_port = cpu_to_be16(f->fs.val.fport); + req->local_ip_hi = local_hi; + req->local_ip_lo = local_lo; + req->peer_ip_hi = peer_hi; + req->peer_ip_lo = peer_lo; + req->opt0 = cpu_to_be64(V_NAGLE(f->fs.newvlan == VLAN_REMOVE || + f->fs.newvlan == VLAN_REWRITE) | + V_DELACK(f->fs.hitcnts) | + V_L2T_IDX(f->l2t ? f->l2t->idx : 0) | + V_SMAC_SEL( + (cxgbe_port_viid(f->dev) & 0x7F) << 1) | + V_TX_CHAN(f->fs.eport) | + V_NO_CONG(f->fs.rpttid) | + V_ULP_MODE(f->fs.nat_mode ? + ULP_MODE_TCPDDP : ULP_MODE_NONE) | + F_TCAM_BYPASS | F_NON_OFFLOAD); + + if (is_t5(adap->params.chip)) { + t5req->params = cpu_to_be64( + V_FILTER_TUPLE(hash_filter_ntuple(f))); + t5req->opt2 = cpu_to_be32(F_RSS_QUEUE_VALID | + V_RSS_QUEUE(f->fs.iq) | + V_TX_QUEUE(f->fs.nat_mode) | + V_WND_SCALE_EN(f->fs.nat_flag_chk) | + V_RX_FC_DISABLE( + f->fs.nat_seq_chk ? 1 : 0) | + F_T5_OPT_2_VALID | + F_RX_CHANNEL | + V_SACK_EN(f->fs.swapmac) | + V_CONG_CNTRL( + (f->fs.action == FILTER_DROP) | + (f->fs.dirsteer << 1)) | + V_PACE((f->fs.maskhash) | + (f->fs.dirsteerhash << 1)) | + V_CCTRL_ECN( + f->fs.action == FILTER_SWITCH)); + } +} + +/** + * Build a ACT_OPEN_REQ message for setting IPv4 hash filter. + */ +static void mk_act_open_req(struct filter_entry *f, struct rte_mbuf *mbuf, + unsigned int qid_filterid, struct adapter *adap) +{ + struct cpl_act_open_req *req = NULL; + struct cpl_t5_act_open_req *t5req = NULL; + + switch (CHELSIO_CHIP_VERSION(adap->params.chip)) { + case CHELSIO_T5: + t5req = rte_pktmbuf_mtod(mbuf, struct cpl_t5_act_open_req *); + + INIT_TP_WR(t5req, 0); + req = (struct cpl_act_open_req *)t5req; + break; + default: + dev_err(adap, "%s: unsupported chip type!\n", __func__); + return; + } + + OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, + qid_filterid)); + req->local_port = cpu_to_be16(f->fs.val.lport); + req->peer_port = cpu_to_be16(f->fs.val.fport); + req->local_ip = f->fs.val.lip[0] | f->fs.val.lip[1] << 8 | + f->fs.val.lip[2] << 16 | f->fs.val.lip[3] << 24; + req->peer_ip = f->fs.val.fip[0] | f->fs.val.fip[1] << 8 | + f->fs.val.fip[2] << 16 | f->fs.val.fip[3] << 24; + req->opt0 = cpu_to_be64(V_NAGLE(f->fs.newvlan == VLAN_REMOVE || + f->fs.newvlan == VLAN_REWRITE) | + V_DELACK(f->fs.hitcnts) | + V_L2T_IDX(f->l2t ? f->l2t->idx : 0) | + V_SMAC_SEL( + (cxgbe_port_viid(f->dev) & 0x7F) << 1) | + V_TX_CHAN(f->fs.eport) | + V_NO_CONG(f->fs.rpttid) | + V_ULP_MODE(f->fs.nat_mode ? + ULP_MODE_TCPDDP : ULP_MODE_NONE) | + F_TCAM_BYPASS | F_NON_OFFLOAD); + + if (is_t5(adap->params.chip)) { + t5req->params = cpu_to_be64( + V_FILTER_TUPLE(hash_filter_ntuple(f))); + t5req->opt2 = cpu_to_be32(F_RSS_QUEUE_VALID | + V_RSS_QUEUE(f->fs.iq) | + V_TX_QUEUE(f->fs.nat_mode) | + V_WND_SCALE_EN(f->fs.nat_flag_chk) | + V_RX_FC_DISABLE( + f->fs.nat_seq_chk ? 1 : 0) | + F_T5_OPT_2_VALID | + F_RX_CHANNEL | + V_SACK_EN(f->fs.swapmac) | + V_CONG_CNTRL( + (f->fs.action == FILTER_DROP) | + (f->fs.dirsteer << 1)) | + V_PACE((f->fs.maskhash) | + (f->fs.dirsteerhash << 1)) | + V_CCTRL_ECN( + f->fs.action == FILTER_SWITCH)); + } +} + +/** + * Set the specified hash filter. + */ +static int cxgbe_set_hash_filter(struct rte_eth_dev *dev, + struct ch_filter_specification *fs, + struct filter_ctx *ctx) +{ + struct port_info *pi = ethdev2pinfo(dev); + struct adapter *adapter = pi->adapter; + struct tid_info *t = &adapter->tids; + struct filter_entry *f; + struct rte_mbuf *mbuf; + struct sge_ctrl_txq *ctrlq; + unsigned int iq; + int atid, size; + int ret = 0; + + ret = validate_filter(adapter, fs); + if (ret) + return ret; + + iq = get_filter_steerq(dev, fs); + + ctrlq = &adapter->sge.ctrlq[pi->port_id]; + + f = t4_os_alloc(sizeof(*f)); + if (!f) + goto out_err; + + f->fs = *fs; + f->ctx = ctx; + f->dev = dev; + f->fs.iq = iq; + + /* + * If the new filter requires loopback Destination MAC and/or VLAN + * rewriting then we need to allocate a Layer 2 Table (L2T) entry for + * the filter. + */ + if (f->fs.newdmac || + ((f->fs.newvlan == VLAN_INSERT) || + (f->fs.newvlan == VLAN_REWRITE))) { + /* allocate L2T entry for new filter */ + f->l2t = cxgbe_l2t_alloc_switching(dev, f->fs.vlan, + f->fs.eport, f->fs.dmac); + if (!f->l2t) { + ret = -ENOMEM; + goto out_err; + } + } + + /* + * If the new filter requires loopback Source MAC rewriting then + * we need to allocate a SMT entry for the filter. + */ + if (f->fs.newsmac) { + f->smt = cxgbe_smt_alloc_switching(dev, f->fs.smac); + if (!f->smt) { + ret = -EAGAIN; + goto free_l2t; + } + f->smtidx = f->smt->idx; + } + + atid = cxgbe_alloc_atid(t, f); + if (atid < 0) + goto free_smt; + + if (f->fs.type) { + /* IPv6 hash filter */ + f->clipt = cxgbe_clip_alloc(f->dev, (u32 *)&f->fs.val.lip); + if (!f->clipt) + goto free_atid; + + size = sizeof(struct cpl_t5_act_open_req6); + mbuf = rte_pktmbuf_alloc(ctrlq->mb_pool); + if (!mbuf) { + ret = -ENOMEM; + goto free_clip; + } + + mbuf->data_len = size; + mbuf->pkt_len = mbuf->data_len; + + mk_act_open_req6(f, mbuf, + ((adapter->sge.fw_evtq.abs_id << 14) | atid), + adapter); + } else { + /* IPv4 hash filter */ + size = sizeof(struct cpl_t5_act_open_req); + mbuf = rte_pktmbuf_alloc(ctrlq->mb_pool); + if (!mbuf) { + ret = -ENOMEM; + goto free_atid; + } + + mbuf->data_len = size; + mbuf->pkt_len = mbuf->data_len; + + mk_act_open_req(f, mbuf, + ((adapter->sge.fw_evtq.abs_id << 14) | atid), + adapter); + } + + f->pending = 1; + t4_mgmt_tx(ctrlq, mbuf); + return 0; + +free_clip: + cxgbe_clip_release(f->dev, f->clipt); + +free_atid: + cxgbe_free_atid(t, atid); + +free_smt: + if (f->smt) { + cxgbe_smt_release(f->smt); + f->smt = NULL; + } + +free_l2t: + if (f->l2t) { + cxgbe_l2t_release(f->l2t); + f->l2t = NULL; + } + +out_err: + t4_os_free(f); + return ret; +} + +/** * Clear a filter and release any of its resources that we own. This also * clears the filter's "pending" status. */ @@ -229,6 +817,18 @@ void cxgbe_clear_all_filters(struct adapter *adapter) if (f->valid || f->pending) clear_filter(f); } + + if (is_hashfilter(adapter) && adapter->tids.tid_tab) { + struct filter_entry *f; + + for (i = adapter->tids.hash_base; i <= adapter->tids.ntids; + i++) { + f = (struct filter_entry *)adapter->tids.tid_tab[i]; + + if (f && (f->valid || f->pending)) + t4_os_free(f); + } + } } /** @@ -536,6 +1136,9 @@ int cxgbe_del_filter(struct rte_eth_dev *dev, unsigned int filter_id, struct filter_entry *f; int ret; + if (is_hashfilter(adapter) && fs->cap) + return cxgbe_del_hash_filter(dev, filter_id, ctx); + if (filter_id >= adapter->tids.nftids) return -ERANGE; @@ -591,6 +1194,9 @@ int cxgbe_set_filter(struct rte_eth_dev *dev, unsigned int filter_id, struct filter_entry *f; int ret; + if (is_hashfilter(adapter) && fs->cap) + return cxgbe_set_hash_filter(dev, fs, ctx); + if (filter_id >= adapter->tids.nftids) return -ERANGE; @@ -800,3 +1406,209 @@ void filter_rpl(struct adapter *adap, const struct cpl_set_tcb_rpl *rpl) t4_complete(&ctx->completion); } } + +/** + * Handle a Hash filter write reply. + */ +void hash_filter_rpl(struct adapter *adap, const struct cpl_act_open_rpl *rpl) +{ + struct tid_info *t = &adap->tids; + struct filter_entry *f; + struct filter_ctx *ctx = NULL; + unsigned int tid = GET_TID(rpl); + unsigned int ftid = G_TID_TID( + G_AOPEN_ATID(be32_to_cpu(rpl->atid_status))); + unsigned int status = G_AOPEN_STATUS(be32_to_cpu(rpl->atid_status)); + + f = lookup_atid(t, ftid); + if (!f) { + dev_warn(adap, "%s: could not find filter entry: %d\n", + __func__, ftid); + return; + } + + ctx = f->ctx; + f->ctx = NULL; + + switch (status) { + case CPL_ERR_NONE: { + f->tid = tid; + f->pending = 0; /* asynchronous setup completed */ + f->valid = 1; + + cxgbe_insert_tid(t, f, f->tid, 0); + cxgbe_free_atid(t, ftid); + if (ctx) { + ctx->tid = f->tid; + ctx->result = 0; + } + + if (f->fs.hitcnts) + set_tcb_field(adap, tid, + W_TCB_TIMESTAMP, + V_TCB_TIMESTAMP(M_TCB_TIMESTAMP) | + V_TCB_T_RTT_TS_RECENT_AGE( + M_TCB_T_RTT_TS_RECENT_AGE), + V_TCB_TIMESTAMP(0ULL) | + V_TCB_T_RTT_TS_RECENT_AGE(0ULL), + 1); + + if (f->fs.newdmac) + set_tcb_tflag(adap, tid, S_TF_CCTRL_ECE, 1, 1); + + if ((f->fs.newvlan == VLAN_INSERT) || + (f->fs.newvlan == VLAN_REWRITE)) + set_tcb_tflag(adap, tid, S_TF_CCTRL_RFR, 1, 1); + + if (f->fs.newsmac) { + set_tcb_tflag(adap, tid, S_TF_CCTRL_CWR, 1, 1); + set_tcb_field(adap, tid, W_TCB_SMAC_SEL, + V_TCB_SMAC_SEL(M_TCB_SMAC_SEL), + V_TCB_SMAC_SEL(f->smtidx), 1); + } + + if (f->fs.nat_mode) { + switch (f->fs.nat_mode) { + case NAT_MODE_DIP: + set_nat_params(adap, f, tid, true, + false, false, false); + break; + + case NAT_MODE_DIP_DP: + set_nat_params(adap, f, tid, true, + false, true, false); + break; + + case NAT_MODE_DIP_DP_SIP: + set_nat_params(adap, f, tid, true, + true, true, false); + break; + + case NAT_MODE_DIP_DP_SP: + set_nat_params(adap, f, tid, true, + false, true, true); + break; + + case NAT_MODE_SIP_SP: + set_nat_params(adap, f, tid, false, + true, false, true); + break; + + case NAT_MODE_DIP_SIP_SP: + set_nat_params(adap, f, tid, true, + true, false, true); + break; + + case NAT_MODE_ALL: + set_nat_params(adap, f, tid, true, + true, true, true); + break; + + default: + dev_err(adap, "%s: Invalid NAT mode: %d\n", + __func__, f->fs.nat_mode); + + if (f->l2t) + cxgbe_l2t_release(f->l2t); + + if (f->smt) + cxgbe_smt_release(f->smt); + + t4_os_free(f); + + if (ctx) { + ctx->result = -EINVAL; + t4_complete(&ctx->completion); + } + return; + } + } + + if (f->fs.nat_seq_chk) { + set_tcb_field(adap, tid, W_TCB_RCV_NXT, + V_TCB_RCV_NXT(M_TCB_RCV_NXT), + V_TCB_RCV_NXT(f->fs.nat_seq_chk), 1); + } + + if (is_t5(adap->params.chip)) { + if (f->fs.action == FILTER_DROP) { + /* + * Set Migrating bit to 1, and + * set Non-offload bit to 0 - to achieve + * Drop action with Hash filters + */ + set_tcb_field(adap, tid, + W_TCB_T_FLAGS, + V_TF_NON_OFFLOAD(1) | + V_TF_MIGRATING(1), + V_TF_MIGRATING(1), 1); + } + } + + break; + } + default: + dev_warn(adap, "%s: filter creation failed with status = %u\n", + __func__, status); + + if (ctx) { + if (status == CPL_ERR_TCAM_FULL) + ctx->result = -EAGAIN; + else + ctx->result = -EINVAL; + } + + if (f->l2t) + cxgbe_l2t_release(f->l2t); + + if (f->smt) + cxgbe_smt_release(f->smt); + + cxgbe_free_atid(t, ftid); + t4_os_free(f); + } + + if (ctx) + t4_complete(&ctx->completion); +} + +/** + * Handle a Hash filter delete reply. + */ +void hash_del_filter_rpl(struct adapter *adap, + const struct cpl_abort_rpl_rss *rpl) +{ + struct tid_info *t = &adap->tids; + struct filter_entry *f; + struct filter_ctx *ctx = NULL; + unsigned int tid = GET_TID(rpl); + + f = lookup_tid(t, tid); + if (!f) { + dev_warn(adap, "%s: could not find filter entry: %u\n", + __func__, tid); + return; + } + + ctx = f->ctx; + f->ctx = NULL; + + f->valid = 0; + + if (f->clipt) + cxgbe_clip_release(f->dev, f->clipt); + + if (f->l2t) + cxgbe_l2t_release(f->l2t); + + if (f->smt) + cxgbe_smt_release(f->smt); + + cxgbe_remove_tid(t, 0, tid, 0); + t4_os_free(f); + + if (ctx) { + ctx->result = 0; + t4_complete(&ctx->completion); + } +} diff --git a/drivers/net/cxgbe/cxgbe_filter.h b/drivers/net/cxgbe/cxgbe_filter.h index 96c15d2..cde74fc 100644 --- a/drivers/net/cxgbe/cxgbe_filter.h +++ b/drivers/net/cxgbe/cxgbe_filter.h @@ -235,6 +235,8 @@ struct filter_entry { struct ch_filter_specification fs; }; +#define WORD_MASK 0xffffffff + struct adapter; void filter_rpl(struct adapter *adap, const struct cpl_set_tcb_rpl *rpl); @@ -249,5 +251,10 @@ int cxgbe_del_filter(struct rte_eth_dev *dev, unsigned int filter_id, struct ch_filter_specification *fs, struct filter_ctx *ctx); +int init_hash_filter(struct adapter *adap); +void hash_filter_rpl(struct adapter *adap, const struct cpl_act_open_rpl *rpl); +void hash_del_filter_rpl(struct adapter *adap, + const struct cpl_abort_rpl_rss *rpl); + void cxgbe_clear_all_filters(struct adapter *adapter); #endif /* _CXGBE_FILTER_H_ */ diff --git a/drivers/net/cxgbe/cxgbe_main.c b/drivers/net/cxgbe/cxgbe_main.c index dfb6567..1f79ba3 100644 --- a/drivers/net/cxgbe/cxgbe_main.c +++ b/drivers/net/cxgbe/cxgbe_main.c @@ -123,6 +123,14 @@ static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp, const struct cpl_set_tcb_rpl *p = (const void *)rsp; filter_rpl(q->adapter, p); + } else if (opcode == CPL_ACT_OPEN_RPL) { + const struct cpl_act_open_rpl *p = (const void *)rsp; + + hash_filter_rpl(q->adapter, p); + } else if (opcode == CPL_ABORT_RPL_RSS) { + const struct cpl_abort_rpl_rss *p = (const void *)rsp; + + hash_del_filter_rpl(q->adapter, p); } else if (opcode == CPL_SMT_WRITE_RPL) { const struct cpl_smt_write_rpl *p = (const void *)rsp; @@ -272,6 +280,110 @@ int cxgb4_set_rspq_intr_params(struct sge_rspq *q, unsigned int us, } /** + * Allocate an active-open TID and set it to the supplied value. + */ +int cxgbe_alloc_atid(struct tid_info *t, void *data) +{ + int atid = -1; + + t4_os_lock(&t->atid_lock); + if (t->afree) { + union aopen_entry *p = t->afree; + + atid = p - t->atid_tab; + t->afree = p->next; + p->data = data; + t->atids_in_use++; + } + t4_os_unlock(&t->atid_lock); + return atid; +} + +/** + * Release an active-open TID. + */ +void cxgbe_free_atid(struct tid_info *t, unsigned int atid) +{ + union aopen_entry *p = &t->atid_tab[atid]; + + t4_os_lock(&t->atid_lock); + p->next = t->afree; + t->afree = p; + t->atids_in_use--; + t4_os_unlock(&t->atid_lock); +} + +/** + * Populate a TID_RELEASE WR. Caller must properly size the skb. + */ +static void mk_tid_release(struct rte_mbuf *mbuf, unsigned int tid) +{ + struct cpl_tid_release *req; + + req = rte_pktmbuf_mtod(mbuf, struct cpl_tid_release *); + INIT_TP_WR_MIT_CPL(req, CPL_TID_RELEASE, tid); +} + +/** + * Release a TID and inform HW. If we are unable to allocate the release + * message we defer to a work queue. + */ +void cxgbe_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid, + unsigned short family) +{ + struct rte_mbuf *mbuf; + struct adapter *adap = container_of(t, struct adapter, tids); + + WARN_ON(tid >= t->ntids); + + if (t->tid_tab[tid]) { + t->tid_tab[tid] = NULL; + rte_atomic32_dec(&t->conns_in_use); + if (t->hash_base && (tid >= t->hash_base)) { + if (family == FILTER_TYPE_IPV6) + rte_atomic32_sub(&t->hash_tids_in_use, 2); + else + rte_atomic32_dec(&t->hash_tids_in_use); + } else { + if (family == FILTER_TYPE_IPV6) + rte_atomic32_sub(&t->tids_in_use, 2); + else + rte_atomic32_dec(&t->tids_in_use); + } + } + + mbuf = rte_pktmbuf_alloc((&adap->sge.ctrlq[chan])->mb_pool); + if (mbuf) { + mbuf->data_len = sizeof(struct cpl_tid_release); + mbuf->pkt_len = mbuf->data_len; + mk_tid_release(mbuf, tid); + t4_mgmt_tx(&adap->sge.ctrlq[chan], mbuf); + } +} + +/** + * Insert a TID. + */ +void cxgbe_insert_tid(struct tid_info *t, void *data, unsigned int tid, + unsigned short family) +{ + t->tid_tab[tid] = data; + if (t->hash_base && (tid >= t->hash_base)) { + if (family == FILTER_TYPE_IPV6) + rte_atomic32_add(&t->hash_tids_in_use, 2); + else + rte_atomic32_inc(&t->hash_tids_in_use); + } else { + if (family == FILTER_TYPE_IPV6) + rte_atomic32_add(&t->tids_in_use, 2); + else + rte_atomic32_inc(&t->tids_in_use); + } + + rte_atomic32_inc(&t->conns_in_use); +} + +/** * Free TID tables. */ static void tid_free(struct tid_info *t) @@ -675,8 +787,7 @@ static int adap_init0_config(struct adapter *adapter, int reset) * This will allow the firmware to optimize aspects of the hardware * configuration which will result in improved performance. */ - caps_cmd.niccaps &= cpu_to_be16(~(FW_CAPS_CONFIG_NIC_HASHFILTER | - FW_CAPS_CONFIG_NIC_ETHOFLD)); + caps_cmd.niccaps &= cpu_to_be16(~FW_CAPS_CONFIG_NIC_ETHOFLD); caps_cmd.toecaps = 0; caps_cmd.iscsicaps = 0; caps_cmd.rdmacaps = 0; @@ -908,6 +1019,12 @@ static int adap_init0(struct adapter *adap) if (ret < 0) goto bye; + if ((caps_cmd.niccaps & cpu_to_be16(FW_CAPS_CONFIG_NIC_HASHFILTER)) && + is_t5(adap->params.chip)) { + if (init_hash_filter(adap) < 0) + goto bye; + } + /* query tid-related parameters */ params[0] = FW_PARAM_DEV(NTID); ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, @@ -1411,6 +1528,20 @@ allocate_mac: "filter support disabled. Continuing\n"); } + if (is_hashfilter(adapter)) { + if (t4_read_reg(adapter, A_LE_DB_CONFIG) & F_HASHEN) { + u32 hash_base, hash_reg; + + hash_reg = A_LE_DB_TID_HASHBASE; + hash_base = t4_read_reg(adapter, hash_reg); + adapter->tids.hash_base = hash_base / 4; + } + } else { + /* Disable hash filtering support */ + dev_warn(adapter, + "Maskless filter support disabled. Continuing\n"); + } + err = init_rss(adapter); if (err) goto out_free; diff --git a/drivers/net/cxgbe/cxgbe_ofld.h b/drivers/net/cxgbe/cxgbe_ofld.h index 115472e..0cddf8d 100644 --- a/drivers/net/cxgbe/cxgbe_ofld.h +++ b/drivers/net/cxgbe/cxgbe_ofld.h @@ -52,6 +52,14 @@ OPCODE_TID(w) = cpu_to_be32(MK_OPCODE_TID(cpl, tid)); \ } while (0) +#define INIT_ULPTX_WR(w, wrlen, atomic, tid) do { \ + (w)->wr.wr_hi = cpu_to_be32(V_FW_WR_OP(FW_ULPTX_WR) | \ + V_FW_WR_ATOMIC(atomic)); \ + (w)->wr.wr_mid = cpu_to_be32(V_FW_WR_LEN16(DIV_ROUND_UP(wrlen, 16)) | \ + V_FW_WR_FLOWID(tid)); \ + (w)->wr.wr_lo = cpu_to_be64(0); \ +} while (0) + /* * Max # of ATIDs. The absolute HW max is 16K but we keep it lower. */ @@ -97,4 +105,22 @@ struct tid_info { rte_atomic32_t conns_in_use; rte_spinlock_t ftid_lock; }; + +static inline void *lookup_tid(const struct tid_info *t, unsigned int tid) +{ + return tid < t->ntids ? t->tid_tab[tid] : NULL; +} + +static inline void *lookup_atid(const struct tid_info *t, unsigned int atid) +{ + return atid < t->natids ? t->atid_tab[atid].data : NULL; +} + +int cxgbe_alloc_atid(struct tid_info *t, void *data); +void cxgbe_free_atid(struct tid_info *t, unsigned int atid); + +void cxgbe_remove_tid(struct tid_info *t, unsigned int qid, unsigned int tid, + unsigned short family); +void cxgbe_insert_tid(struct tid_info *t, void *data, unsigned int tid, + unsigned short family); #endif /* _CXGBE_OFLD_H_ */