@@ -7,6 +7,8 @@
#include "ark_logs.h"
#include "ark_ddm.h"
+static_assert(sizeof(union ark_tx_meta) == 8, "Unexpected struct size ark_tx_meta");
+
/* ************************************************************************* */
int
ark_ddm_verify(struct ark_ddm_t *ddm)
@@ -19,18 +21,26 @@ ark_ddm_verify(struct ark_ddm_t *ddm)
}
hw_const = ddm->cfg.const0;
+ if (hw_const == ARK_DDM_CONST3)
+ return 0;
+
if (hw_const == ARK_DDM_CONST1) {
ARK_PMD_LOG(ERR,
"ARK: DDM module is version 1, "
"PMD expects version 2\n");
return -1;
- } else if (hw_const != ARK_DDM_CONST2) {
+ }
+
+ if (hw_const == ARK_DDM_CONST2) {
ARK_PMD_LOG(ERR,
- "ARK: DDM module not found as expected 0x%08x\n",
- ddm->cfg.const0);
+ "ARK: DDM module is version 2, "
+ "PMD expects version 3\n");
return -1;
}
- return 0;
+ ARK_PMD_LOG(ERR,
+ "ARK: DDM module not found as expected 0x%08x\n",
+ ddm->cfg.const0);
+ return -1;
}
void
@@ -16,17 +16,22 @@
* there is minimal documentation.
*/
-/* struct defining Tx meta data -- fixed in FPGA -- 16 bytes */
-struct ark_tx_meta {
+/* struct defining Tx meta data -- fixed in FPGA -- 8 bytes */
+union ark_tx_meta {
uint64_t physaddr;
- uint32_t user1;
- uint16_t data_len; /* of this MBUF */
+ struct {
+ uint32_t usermeta0;
+ uint32_t usermeta1;
+ };
+ struct {
+ uint16_t data_len; /* of this MBUF */
#define ARK_DDM_EOP 0x01
#define ARK_DDM_SOP 0x02
- uint8_t flags; /* bit 0 indicates last mbuf in chain. */
- uint8_t reserved[1];
-};
-
+ uint8_t flags;
+ uint8_t meta_cnt;
+ uint32_t user1;
+ };
+} __rte_packed;
/*
* DDM core hardware structures
@@ -35,6 +40,7 @@ struct ark_tx_meta {
*/
#define ARK_DDM_CFG 0x0000
/* Set unique HW ID for hardware version */
+#define ARK_DDM_CONST3 (0x334d4444)
#define ARK_DDM_CONST2 (0x324d4444)
#define ARK_DDM_CONST1 (0xfacecafe)
@@ -95,6 +95,8 @@ static const char * const valid_arguments[] = {
static const struct rte_pci_id pci_id_ark_map[] = {
{RTE_PCI_DEVICE(0x1d6c, 0x100d)},
{RTE_PCI_DEVICE(0x1d6c, 0x100e)},
+ {RTE_PCI_DEVICE(0x1d6c, 0x1017)},
+ {RTE_PCI_DEVICE(0x1d6c, 0x1018)},
{.vendor_id = 0, /* sentinel */ },
};
@@ -60,7 +60,6 @@ struct ark_rx_queue {
volatile uint32_t prod_index; /* step 2 filled by FPGA */
} __rte_cache_aligned;
-
/* ************************************************************************* */
static int
eth_ark_rx_hw_setup(struct rte_eth_dev *dev,
@@ -265,7 +264,6 @@ eth_ark_recv_pkts(void *rx_queue,
/* META DATA embedded in headroom */
meta = RTE_PTR_ADD(mbuf->buf_addr, ARK_RX_META_OFFSET);
- mbuf->port = meta->port;
mbuf->pkt_len = meta->pkt_len;
mbuf->data_len = meta->pkt_len;
/* set timestamp if enabled at least on one device */
@@ -346,8 +344,7 @@ eth_ark_rx_jumbo(struct ark_rx_queue *queue,
/* HW guarantees that the data does not exceed prod_index! */
while (remaining != 0) {
data_len = RTE_MIN(remaining,
- RTE_MBUF_DEFAULT_DATAROOM +
- RTE_PKTMBUF_HEADROOM);
+ RTE_MBUF_DEFAULT_DATAROOM);
remaining -= data_len;
segments += 1;
@@ -356,7 +353,6 @@ eth_ark_rx_jumbo(struct ark_rx_queue *queue,
mbuf_prev->next = mbuf;
mbuf_prev = mbuf;
mbuf->data_len = data_len;
- mbuf->data_off = 0;
cons_index += 1;
}
@@ -1,5 +1,5 @@
/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright (c) 2015-2018 Atomic Rules LLC
+ * Copyright (c) 2015-2021 Atomic Rules LLC
*/
#include <unistd.h>
@@ -23,7 +23,7 @@
/* ************************************************************************* */
struct ark_tx_queue {
- struct ark_tx_meta *meta_q;
+ union ark_tx_meta *meta_q;
struct rte_mbuf **bufs;
/* handles for hw objects */
@@ -37,8 +37,8 @@ struct ark_tx_queue {
uint32_t queue_mask;
/* 3 indexes to the paired data rings. */
- uint32_t prod_index; /* where to put the next one */
- uint32_t free_index; /* mbuf has been freed */
+ int32_t prod_index; /* where to put the next one */
+ int32_t free_index; /* mbuf has been freed */
/* The queue Id is used to identify the HW Q */
uint16_t phys_qid;
@@ -47,14 +47,15 @@ struct ark_tx_queue {
uint32_t pad[1];
- /* second cache line - fields only used in slow path */
+ /* second cache line - fields written by device */
RTE_MARKER cacheline1 __rte_cache_min_aligned;
- uint32_t cons_index; /* hw is done, can be freed */
+ volatile int32_t cons_index; /* hw is done, can be freed */
} __rte_cache_aligned;
/* Forward declarations */
-static uint32_t eth_ark_tx_jumbo(struct ark_tx_queue *queue,
- struct rte_mbuf *mbuf);
+static int eth_ark_tx_jumbo(struct ark_tx_queue *queue,
+ struct rte_mbuf *mbuf,
+ uint32_t *user_meta, uint8_t meta_cnt);
static int eth_ark_tx_hw_queue_config(struct ark_tx_queue *queue);
static void free_completed_tx(struct ark_tx_queue *queue);
@@ -66,16 +67,44 @@ ark_tx_hw_queue_stop(struct ark_tx_queue *queue)
/* ************************************************************************* */
static inline void
-eth_ark_tx_meta_from_mbuf(struct ark_tx_meta *meta,
- const struct rte_mbuf *mbuf,
- uint8_t flags)
+eth_ark_tx_desc_fill(struct ark_tx_queue *queue,
+ struct rte_mbuf *mbuf,
+ uint8_t flags,
+ uint32_t *user_meta,
+ uint8_t meta_cnt /* 0 to 5 */
+ )
{
- meta->physaddr = rte_mbuf_data_iova(mbuf);
- meta->user1 = rte_pmd_ark_mbuf_tx_userdata_get(mbuf);
+ uint32_t tx_idx;
+ union ark_tx_meta *meta;
+ uint8_t m;
+
+ /* Header */
+ tx_idx = queue->prod_index & queue->queue_mask;
+ meta = &queue->meta_q[tx_idx];
meta->data_len = rte_pktmbuf_data_len(mbuf);
meta->flags = flags;
+ meta->meta_cnt = meta_cnt / 2;
+ meta->user1 = meta_cnt ? (*user_meta++) : 0;
+ queue->prod_index++;
+
+ queue->bufs[tx_idx] = mbuf;
+
+ /* 1 or 2 user meta data entries, user words 1,2 and 3,4 */
+ for (m = 1; m < meta_cnt; m += 2) {
+ tx_idx = queue->prod_index & queue->queue_mask;
+ meta = &queue->meta_q[tx_idx];
+ meta->usermeta0 = *user_meta++;
+ meta->usermeta1 = *user_meta++;
+ queue->prod_index++;
+ }
+
+ tx_idx = queue->prod_index & queue->queue_mask;
+ meta = &queue->meta_q[tx_idx];
+ meta->physaddr = rte_mbuf_data_iova(mbuf);
+ queue->prod_index++;
}
+
/* ************************************************************************* */
uint16_t
eth_ark_xmit_pkts_noop(void *vtxq __rte_unused,
@@ -91,12 +120,12 @@ eth_ark_xmit_pkts(void *vtxq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
{
struct ark_tx_queue *queue;
struct rte_mbuf *mbuf;
- struct ark_tx_meta *meta;
+ uint32_t user_meta;
- uint32_t idx;
- uint32_t prod_index_limit;
int stat;
+ int32_t prod_index_limit;
uint16_t nb;
+ uint8_t user_len = 1;
const uint32_t min_pkt_len = ARK_MIN_TX_PKTLEN;
queue = (struct ark_tx_queue *)vtxq;
@@ -104,10 +133,11 @@ eth_ark_xmit_pkts(void *vtxq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
/* free any packets after the HW is done with them */
free_completed_tx(queue);
- prod_index_limit = queue->queue_size + queue->free_index;
+ /* leave 4 elements mpu data */
+ prod_index_limit = queue->queue_size + queue->free_index - 4;
for (nb = 0;
- (nb < nb_pkts) && (queue->prod_index != prod_index_limit);
+ (nb < nb_pkts) && (prod_index_limit - queue->prod_index) > 0;
++nb) {
mbuf = tx_pkts[nb];
@@ -133,19 +163,16 @@ eth_ark_xmit_pkts(void *vtxq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
memset(appended, 0, to_add);
}
+ user_meta = rte_pmd_ark_mbuf_tx_userdata_get(mbuf);
if (unlikely(mbuf->nb_segs != 1)) {
- stat = eth_ark_tx_jumbo(queue, mbuf);
+ stat = eth_ark_tx_jumbo(queue, mbuf,
+ &user_meta, user_len);
if (unlikely(stat != 0))
break; /* Queue is full */
} else {
- idx = queue->prod_index & queue->queue_mask;
- queue->bufs[idx] = mbuf;
- meta = &queue->meta_q[idx];
- eth_ark_tx_meta_from_mbuf(meta,
- mbuf,
- ARK_DDM_SOP |
- ARK_DDM_EOP);
- queue->prod_index++;
+ eth_ark_tx_desc_fill(queue, mbuf,
+ ARK_DDM_SOP | ARK_DDM_EOP,
+ &user_meta, user_len);
}
}
@@ -173,32 +200,28 @@ eth_ark_xmit_pkts(void *vtxq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
}
/* ************************************************************************* */
-static uint32_t
-eth_ark_tx_jumbo(struct ark_tx_queue *queue, struct rte_mbuf *mbuf)
+static int
+eth_ark_tx_jumbo(struct ark_tx_queue *queue, struct rte_mbuf *mbuf,
+ uint32_t *user_meta, uint8_t meta_cnt)
{
struct rte_mbuf *next;
- struct ark_tx_meta *meta;
- uint32_t free_queue_space;
- uint32_t idx;
+ int32_t free_queue_space;
uint8_t flags = ARK_DDM_SOP;
free_queue_space = queue->queue_mask -
(queue->prod_index - queue->free_index);
- if (unlikely(free_queue_space < mbuf->nb_segs))
+ /* We need up to 4 mbufs for first header and 2 for subsequent ones */
+ if (unlikely(free_queue_space < (2 + (2 * mbuf->nb_segs))))
return -1;
while (mbuf != NULL) {
next = mbuf->next;
-
- idx = queue->prod_index & queue->queue_mask;
- queue->bufs[idx] = mbuf;
- meta = &queue->meta_q[idx];
-
flags |= (next == NULL) ? ARK_DDM_EOP : 0;
- eth_ark_tx_meta_from_mbuf(meta, mbuf, flags);
- queue->prod_index++;
+
+ eth_ark_tx_desc_fill(queue, mbuf, flags, user_meta, meta_cnt);
flags &= ~ARK_DDM_SOP; /* drop SOP flags */
+ meta_cnt = 0; /* Meta only on SOP */
mbuf = next;
}
@@ -227,6 +250,9 @@ eth_ark_tx_queue_setup(struct rte_eth_dev *dev,
return -1;
}
+ /* Each packet requires at least 2 mpu elements - double desc count */
+ nb_desc = 2 * nb_desc;
+
/* Allocate queue struct */
queue = rte_zmalloc_socket("Ark_txqueue",
sizeof(struct ark_tx_queue),
@@ -248,7 +274,7 @@ eth_ark_tx_queue_setup(struct rte_eth_dev *dev,
queue->meta_q =
rte_zmalloc_socket("Ark_txqueue meta",
- nb_desc * sizeof(struct ark_tx_meta),
+ nb_desc * sizeof(union ark_tx_meta),
64,
socket_id);
queue->bufs =
@@ -289,7 +315,7 @@ eth_ark_tx_hw_queue_config(struct ark_tx_queue *queue)
uint32_t write_interval_ns;
/* Verify HW -- MPU */
- if (ark_mpu_verify(queue->mpu, sizeof(struct ark_tx_meta)))
+ if (ark_mpu_verify(queue->mpu, sizeof(union ark_tx_meta)))
return -1;
queue_base = rte_malloc_virt2iova(queue);
@@ -391,19 +417,19 @@ static void
free_completed_tx(struct ark_tx_queue *queue)
{
struct rte_mbuf *mbuf;
- struct ark_tx_meta *meta;
- uint32_t top_index;
+ union ark_tx_meta *meta;
+ int32_t top_index;
top_index = queue->cons_index; /* read once */
- while (queue->free_index != top_index) {
+ while ((top_index - queue->free_index) > 0) {
meta = &queue->meta_q[queue->free_index & queue->queue_mask];
- mbuf = queue->bufs[queue->free_index & queue->queue_mask];
-
if (likely((meta->flags & ARK_DDM_SOP) != 0)) {
+ mbuf = queue->bufs[queue->free_index &
+ queue->queue_mask];
/* ref count of the mbuf is checked in this call. */
rte_pktmbuf_free(mbuf);
}
- queue->free_index++;
+ queue->free_index += (meta->meta_cnt + 2);
}
}
@@ -7,6 +7,8 @@
#include "ark_logs.h"
#include "ark_udm.h"
+static_assert(sizeof(struct ark_rx_meta) == 32, "Unexpected struct size ark_rx_meta");
+
int
ark_udm_verify(struct ark_udm_t *udm)
{
@@ -15,14 +15,15 @@
* there is minimal documentation.
*/
-/* Meta data structure apssed from FPGA, must match layout in FPGA */
+/* Meta data structure passed from FPGA, must match layout in FPGA
+ * -- 32 bytes
+ */
struct ark_rx_meta {
uint64_t timestamp;
uint64_t user_data;
- uint8_t port;
- uint8_t dst_queue;
+ uint8_t reserved[14];
uint16_t pkt_len;
-};
+} __rte_packed;
/*
* UDM hardware structures
@@ -32,7 +33,9 @@ struct ark_rx_meta {
#define ARK_RX_WRITE_TIME_NS 2500
#define ARK_UDM_SETUP 0
-#define ARK_UDM_CONST 0xbACECACE
+#define ARK_UDM_CONST2 0xbACECACE
+#define ARK_UDM_CONST3 0x334d4455
+#define ARK_UDM_CONST ARK_UDM_CONST3
struct ark_udm_setup_t {
uint32_t r0;
uint32_t r4;