@@ -187,6 +187,148 @@ test_mempool_basic(struct rte_mempool *mp, int use_external_cache)
return ret;
}
+/* basic tests (done on one core) */
+static int
+test_mempool_get_cache(struct rte_mempool *mp, int use_external_cache)
+{
+ uint32_t *objnum;
+ void **objtable;
+ void *obj, *obj2;
+ char *obj_data;
+ int ret = 0;
+ unsigned i, j;
+ int offset;
+ struct rte_mempool_cache *cache;
+ void **cache_objs;
+
+ if (use_external_cache) {
+ /* Create a user-owned mempool cache. */
+ cache = rte_mempool_cache_create(RTE_MEMPOOL_CACHE_MAX_SIZE,
+ SOCKET_ID_ANY);
+ if (cache == NULL)
+ RET_ERR();
+ } else {
+ /* May be NULL if cache is disabled. */
+ cache = rte_mempool_default_cache(mp, rte_lcore_id());
+ }
+
+ /* dump the mempool status */
+ rte_mempool_dump(stdout, mp);
+
+ printf("get an object\n");
+ if (rte_mempool_generic_get(mp, &obj, 1, cache) < 0)
+ GOTO_ERR(ret, out);
+ rte_mempool_dump(stdout, mp);
+
+ /* tests that improve coverage */
+ printf("get object count\n");
+ /* We have to count the extra caches, one in this case. */
+ offset = use_external_cache ? 1 * cache->len : 0;
+ if (rte_mempool_avail_count(mp) + offset != MEMPOOL_SIZE - 1)
+ GOTO_ERR(ret, out);
+
+ printf("get private data\n");
+ if (rte_mempool_get_priv(mp) != (char *)mp +
+ RTE_MEMPOOL_HEADER_SIZE(mp, mp->cache_size))
+ GOTO_ERR(ret, out);
+
+#ifndef RTE_EXEC_ENV_FREEBSD /* rte_mem_virt2iova() not supported on bsd */
+ printf("get physical address of an object\n");
+ if (rte_mempool_virt2iova(obj) != rte_mem_virt2iova(obj))
+ GOTO_ERR(ret, out);
+#endif
+
+
+ printf("put the object back\n");
+ cache_objs = rte_mempool_get_cache(mp,1);
+ if (cache_objs != NULL) {
+ rte_memcpy(cache_objs, &obj, sizeof(void*));
+ }
+ else {
+ rte_mempool_ops_enqueue_bulk(mp, &obj, 1);
+ }
+
+ rte_mempool_dump(stdout, mp);
+
+ printf("get 2 objects\n");
+ if (rte_mempool_generic_get(mp, &obj, 1, cache) < 0)
+ GOTO_ERR(ret, out);
+ if (rte_mempool_generic_get(mp, &obj2, 1, cache) < 0) {
+ rte_mempool_generic_put(mp, &obj, 1, cache);
+ GOTO_ERR(ret, out);
+ }
+ rte_mempool_dump(stdout, mp);
+
+ printf("put the objects back\n");
+ cache_objs = rte_mempool_get_cache(mp,1);
+ if (cache_objs != NULL) {
+ rte_memcpy(mp, &obj, sizeof(void *));
+ }
+ else {
+ rte_mempool_ops_enqueue_bulk(mp, &obj, 1);
+ }
+ cache_objs = rte_mempool_get_cache(mp,1);
+ if (cache_objs != NULL) {
+ rte_memcpy(mp, &obj2, sizeof(void *));
+ }
+ else {
+ rte_mempool_ops_enqueue_bulk(mp, &obj2, 1);
+ }
+ rte_mempool_dump(stdout, mp);
+
+ /*
+ * get many objects: we cannot get them all because the cache
+ * on other cores may not be empty.
+ */
+ objtable = malloc(MEMPOOL_SIZE * sizeof(void *));
+ if (objtable == NULL)
+ GOTO_ERR(ret, out);
+
+ for (i = 0; i < MEMPOOL_SIZE; i++) {
+ if (rte_mempool_generic_get(mp, &objtable[i], 1, cache) < 0)
+ break;
+ }
+
+ /*
+ * for each object, check that its content was not modified,
+ * and put objects back in pool
+ */
+ cache_objs = rte_mempool_get_cache (mp, MEMPOOL_SIZE);
+ if (cache_objs != NULL) {
+ while (i--) {
+ obj = objtable[i];
+ obj_data = obj;
+ objnum = obj;
+ if (*objnum > MEMPOOL_SIZE) {
+ printf("bad object number(%d)\n", *objnum);
+ ret = -1;
+ break;
+ }
+ for (j = sizeof(*objnum); j < mp->elt_size; j++) {
+ if (obj_data[j] != 0)
+ ret = -1;
+ }
+
+ rte_memcpy(&cache_objs[i], &objtable[i], sizeof(void *));
+ }
+ }
+ else {
+ rte_mempool_ops_enqueue_bulk(mp, objtable, MEMPOOL_SIZE);
+ }
+
+ free(objtable);
+ if (ret == -1)
+ printf("objects were modified!\n");
+
+out:
+ if (use_external_cache) {
+ rte_mempool_cache_flush(cache, mp);
+ rte_mempool_cache_free(cache);
+ }
+
+ return ret;
+}
+
static int test_mempool_creation_with_exceeded_cache_size(void)
{
struct rte_mempool *mp_cov;
@@ -986,6 +1128,10 @@ test_mempool(void)
if (test_mempool_basic(mp_cache, 0) < 0)
GOTO_ERR(ret, err);
+ /* basic tests with get cache */
+ if (test_mempool_get_cache(mp_cache, 0) < 0)
+ GOTO_ERR(ret, err);
+
/* basic tests with user-owned cache */
if (test_mempool_basic(mp_nocache, 1) < 0)
GOTO_ERR(ret, err);
@@ -99,14 +99,27 @@ i40e_tx_free_bufs(struct i40e_tx_queue *txq)
* tx_next_dd - (tx_rs_thresh-1)
*/
txep = &txq->sw_ring[txq->tx_next_dd - (n - 1)];
+ struct rte_mempool *mp = txep[0].mbuf->pool;
if (txq->offloads & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) {
- for (i = 0; i < n; i++) {
- free[i] = txep[i].mbuf;
- /* no need to reset txep[i].mbuf in vector path */
+ void **cache_objs;
+ cache_objs = rte_mempool_get_cache(mp,n);
+
+ if (cache_objs!=NULL) {
+ for (i = 0; i < n; i++) {
+ /* no need to reset txep[i].mbuf in vector path */
+ rte_memcpy(&cache_objs[i],&txep->mbuf,sizeof(txep->mbuf));
+ txep++;
+ }
+ goto done;
+ }
+ else {
+ for (i = 0; i < n; i++) {
+ free[i] = txep->mbuf;
+ txep++;
+ }
+ rte_mempool_ops_enqueue_bulk(mp, (void **)free, n);
}
- rte_mempool_put_bulk(free[0]->pool, (void **)free, n);
- goto done;
}
m = rte_pktmbuf_prefree_seg(txep[0].mbuf);
@@ -1360,6 +1360,40 @@ rte_mempool_do_generic_put(struct rte_mempool *mp, void * const *obj_table,
rte_mempool_ops_enqueue_bulk(mp, obj_table, n);
}
+/**
+ * @internal Give pointer to per core cache; used internally.
+ * @param mp
+ * A pointer to the mempool structure.
+ * @param n
+ * The number of objects to store back in the mempool, must be strictly
+ * positive.
+ */
+static __rte_always_inline void **
+rte_mempool_get_cache(struct rte_mempool *mp, unsigned int n)
+{
+ void **cache_objs;
+
+ /* increment stat now, adding in mempool always success */
+ RTE_MEMPOOL_STAT_ADD(mp, put_bulk, 1);
+ RTE_MEMPOOL_STAT_ADD(mp, put_objs, n);
+
+ struct rte_mempool_cache *cache = rte_mempool_default_cache(mp,
+ rte_lcore_id());
+
+ /* No cache provided or the request itself is too big for the cache */
+ if (unlikely(cache == NULL || n > cache->flushthresh))
+ return NULL;
+
+ if (cache->len + n <= cache->flushthresh) {
+ cache_objs = &cache->objs[cache->len];
+ cache->len += n;
+ } else {
+ cache_objs = &cache->objs[0];
+ rte_mempool_ops_enqueue_bulk(mp, cache_objs, cache->len);
+ cache->len = n;
+ }
+ return cache_objs;
+}
/**
* Put several objects back in the mempool.