@@ -51,6 +51,8 @@
#define RTE_MAX_VFIO_CONTAINERS 64
/* bsd module defines */
+#define RTE_CONTIGMEM_DEFAULT_NUM_DEVS 1
+#define RTE_CONTIGMEM_MAX_NUM_DEVS 64
#define RTE_CONTIGMEM_MAX_NUM_BUFS 64
#define RTE_CONTIGMEM_DEFAULT_NUM_BUFS 1
#define RTE_CONTIGMEM_DEFAULT_BUF_SIZE (512*1024*1024)
@@ -1,5 +1,9 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2014 Intel Corporation
+ * Copyright(c) 2024 FreeBSD Foundation
+ *
+ * Part of this software was developed by Tom Jones <thj@freebsd.org> under
+ * sponsorship from the FreeBSD Foundation.
*/
#include <sys/cdefs.h>
@@ -37,8 +41,17 @@ struct contigmem_buffer {
struct mtx mtx;
};
+struct contigmem_device {
+ struct contigmem_buffer cm_buffers[RTE_CONTIGMEM_MAX_NUM_BUFS];
+ struct cdev *cm_cdev;
+ int cm_refcnt;
+ int cm_device_index;
+ eventhandler_tag contigmem_eh_tag;
+};
+
struct contigmem_vm_handle {
int buffer_index;
+ int device_index;
};
static int contigmem_load(void);
@@ -49,29 +62,18 @@ static d_mmap_single_t contigmem_mmap_single;
static d_open_t contigmem_open;
static d_close_t contigmem_close;
+static struct sysctl_ctx_list sysctl_ctx;
+static struct contigmem_device contigmem_device_list[RTE_CONTIGMEM_MAX_NUM_DEVS];
+
+static int contigmem_num_devices = RTE_CONTIGMEM_DEFAULT_NUM_DEVS;
static int contigmem_num_buffers = RTE_CONTIGMEM_DEFAULT_NUM_BUFS;
static int64_t contigmem_buffer_size = RTE_CONTIGMEM_DEFAULT_BUF_SIZE;
-
-static eventhandler_tag contigmem_eh_tag;
-static struct contigmem_buffer contigmem_buffers[RTE_CONTIGMEM_MAX_NUM_BUFS];
-static struct cdev *contigmem_cdev = NULL;
static int contigmem_refcnt;
+TUNABLE_INT("hw.contigmem.num_devices", &contigmem_num_devices);
TUNABLE_INT("hw.contigmem.num_buffers", &contigmem_num_buffers);
TUNABLE_QUAD("hw.contigmem.buffer_size", &contigmem_buffer_size);
-static SYSCTL_NODE(_hw, OID_AUTO, contigmem, CTLFLAG_RD, 0, "contigmem");
-
-SYSCTL_INT(_hw_contigmem, OID_AUTO, num_buffers, CTLFLAG_RD,
- &contigmem_num_buffers, 0, "Number of contigmem buffers allocated");
-SYSCTL_QUAD(_hw_contigmem, OID_AUTO, buffer_size, CTLFLAG_RD,
- &contigmem_buffer_size, 0, "Size of each contiguous buffer");
-SYSCTL_INT(_hw_contigmem, OID_AUTO, num_references, CTLFLAG_RD,
- &contigmem_refcnt, 0, "Number of references to contigmem");
-
-static SYSCTL_NODE(_hw_contigmem, OID_AUTO, physaddr, CTLFLAG_RD, 0,
- "physaddr");
-
MALLOC_DEFINE(M_CONTIGMEM, "contigmem", "contigmem(4) allocations");
static int contigmem_modevent(module_t mod, int type, void *arg)
@@ -114,16 +116,9 @@ static int
contigmem_load(void)
{
char index_string[8], description[32];
- int i, error = 0;
+ int i, j, created_devs = 0, error = 0;
void *addr;
- if (contigmem_num_buffers > RTE_CONTIGMEM_MAX_NUM_BUFS) {
- printf("%d buffers requested is greater than %d allowed\n",
- contigmem_num_buffers, RTE_CONTIGMEM_MAX_NUM_BUFS);
- error = EINVAL;
- goto error;
- }
-
if (contigmem_buffer_size < PAGE_SIZE ||
(contigmem_buffer_size & (contigmem_buffer_size - 1)) != 0) {
printf("buffer size 0x%lx is not greater than PAGE_SIZE and "
@@ -132,83 +127,156 @@ contigmem_load(void)
goto error;
}
- for (i = 0; i < contigmem_num_buffers; i++) {
- addr = contigmalloc(contigmem_buffer_size, M_CONTIGMEM, M_ZERO,
- 0, BUS_SPACE_MAXADDR, contigmem_buffer_size, 0);
- if (addr == NULL) {
- printf("contigmalloc failed for buffer %d\n", i);
- error = ENOMEM;
- goto error;
- }
+ if (contigmem_num_devices > RTE_CONTIGMEM_MAX_NUM_BUFS) {
+ printf("%d buffers requested is greater than %d allowed\n",
+ contigmem_num_buffers, RTE_CONTIGMEM_MAX_NUM_BUFS);
+ error = EINVAL;
+ goto error;
+ }
- printf("%2u: virt=%p phys=%p\n", i, addr,
- (void *)pmap_kextract((vm_offset_t)addr));
+ if (contigmem_num_buffers > RTE_CONTIGMEM_MAX_NUM_DEVS) {
+ printf("%d devices requested is greater than %d allowed\n",
+ contigmem_num_buffers, RTE_CONTIGMEM_MAX_NUM_DEVS);
+ error = EINVAL;
+ goto error;
+ }
- mtx_init(&contigmem_buffers[i].mtx, "contigmem", NULL, MTX_DEF);
- contigmem_buffers[i].addr = addr;
- contigmem_buffers[i].refcnt = 0;
+ if (contigmem_num_devices == 0) {
+ printf("contigmem_num_devices set to 0, not creating any allocations\n");
+ error = EINVAL;
+ goto error;
+ }
- snprintf(index_string, sizeof(index_string), "%d", i);
- snprintf(description, sizeof(description),
- "phys addr for buffer %d", i);
- SYSCTL_ADD_PROC(NULL,
- &SYSCTL_NODE_CHILDREN(_hw_contigmem, physaddr), OID_AUTO,
+ sysctl_ctx_init(&sysctl_ctx);
+
+ static struct sysctl_oid *sysctl_root;
+ sysctl_root = SYSCTL_ADD_NODE(&sysctl_ctx, SYSCTL_STATIC_CHILDREN(_hw),
+ OID_AUTO, "contigmem", CTLFLAG_RD, 0, "contigmem");
+
+ SYSCTL_ADD_INT(&sysctl_ctx, SYSCTL_CHILDREN(sysctl_root), OID_AUTO,
+ "num_devices", CTLFLAG_RD, &contigmem_num_devices, 0,
+ "Number of contigmem devices");
+ SYSCTL_ADD_INT(&sysctl_ctx, SYSCTL_CHILDREN(sysctl_root), OID_AUTO,
+ "num_buffers", CTLFLAG_RD, &contigmem_num_buffers, 0,
+ "Number of contigmem buffers allocated");
+ SYSCTL_ADD_QUAD(&sysctl_ctx, SYSCTL_CHILDREN(sysctl_root), OID_AUTO,
+ "buffer_size", CTLFLAG_RD, &contigmem_buffer_size,
+ "Size of each contiguous buffer");
+ SYSCTL_ADD_INT(&sysctl_ctx, SYSCTL_CHILDREN(sysctl_root), OID_AUTO,
+ "num_references", CTLFLAG_RD, &contigmem_refcnt, 0,
+ "Number of references to contigmem");
+
+ struct contigmem_device *cd;
+ for (i = 0; i < contigmem_num_devices; i++) {
+ cd = &contigmem_device_list[i];
+ struct sysctl_oid *sysctl_dev;
+ char namebuf[32];
+ snprintf(namebuf, sizeof(namebuf), "contigmem%d", i);
+
+ cd->cm_device_index = i;
+
+ printf("Adding node at index %d\n", i);
+ sysctl_dev = SYSCTL_ADD_NODE(&sysctl_ctx, SYSCTL_CHILDREN(sysctl_root),
+ OID_AUTO, namebuf, CTLFLAG_RD, 0,
+ "contigmem");
+ SYSCTL_ADD_INT(&sysctl_ctx,
+ SYSCTL_CHILDREN(sysctl_dev), OID_AUTO,
+ "num_references", CTLFLAG_RD, &cd->cm_refcnt, 0,
+ "Number of references to contigmem device");
+
+ for (j = 0; j < contigmem_num_buffers; j++) {
+ addr = contigmalloc(contigmem_buffer_size, M_CONTIGMEM, M_ZERO,
+ 0, BUS_SPACE_MAXADDR, contigmem_buffer_size, 0);
+ if (addr == NULL) {
+ printf("contigmalloc failed for device %d buffer %d\n",
+ i, j);
+ error = ENOMEM;
+ goto error;
+ }
+
+ printf("dev: %2u %2u: virt=%p phys=%p\n", i, j, addr,
+ (void *)pmap_kextract((vm_offset_t)addr));
+
+ mtx_init(&cd->cm_buffers[j].mtx, "contigmem", NULL, MTX_DEF);
+ cd->cm_buffers[j].addr = addr;
+ cd->cm_buffers[j].refcnt = 0;
+
+ snprintf(index_string, sizeof(index_string), "%d", j);
+ snprintf(description, sizeof(description),
+ "phys addr for buffer %d", j);
+
+ SYSCTL_ADD_PROC(&sysctl_ctx,
+ SYSCTL_CHILDREN(sysctl_dev), OID_AUTO,
index_string, CTLTYPE_U64 | CTLFLAG_RD,
- (void *)(uintptr_t)i, 0, contigmem_physaddr, "LU",
+ (void *)&cd->cm_buffers[j], 0, contigmem_physaddr, "LU",
description);
- }
+ }
- contigmem_cdev = make_dev_credf(0, &contigmem_ops, 0, NULL, UID_ROOT,
- GID_WHEEL, 0600, "contigmem");
+ cd->cm_cdev = make_dev_credf(0, &contigmem_ops, i, NULL,
+ UID_ROOT, GID_WHEEL, 0600, "contigmem%d", i);
+ cd->cm_cdev->si_drv1 = cd;
+ created_devs++;
+ }
return 0;
error:
- for (i = 0; i < contigmem_num_buffers; i++) {
- if (contigmem_buffers[i].addr != NULL) {
- contigfree(contigmem_buffers[i].addr,
- contigmem_buffer_size, M_CONTIGMEM);
- contigmem_buffers[i].addr = NULL;
+ for (i = 0; i < created_devs; i++) {
+ cd = &contigmem_device_list[i];
+ for (j = 0; j < contigmem_num_buffers; j++) {
+ if (cd->cm_buffers[j].addr != NULL) {
+ contigfree(cd->cm_buffers[j].addr,
+ contigmem_buffer_size, M_CONTIGMEM);
+ cd->cm_buffers[j].addr = NULL;
+ }
+ if (mtx_initialized(&cd->cm_buffers[j].mtx))
+ mtx_destroy(&cd->cm_buffers[j].mtx);
}
- if (mtx_initialized(&contigmem_buffers[i].mtx))
- mtx_destroy(&contigmem_buffers[i].mtx);
}
+ sysctl_ctx_free(&sysctl_ctx);
return error;
}
static int
contigmem_unload(void)
{
- int i;
+ struct contigmem_device *cd;
if (contigmem_refcnt > 0)
return EBUSY;
- if (contigmem_cdev != NULL)
- destroy_dev(contigmem_cdev);
+ for (int i = 0; i < contigmem_num_devices; i++) {
+ cd = &contigmem_device_list[i];
+ if (cd->cm_cdev != NULL)
+ destroy_dev(cd->cm_cdev);
- if (contigmem_eh_tag != NULL)
- EVENTHANDLER_DEREGISTER(process_exit, contigmem_eh_tag);
+ if (cd->contigmem_eh_tag != NULL)
+ EVENTHANDLER_DEREGISTER(process_exit, cd->contigmem_eh_tag);
- for (i = 0; i < RTE_CONTIGMEM_MAX_NUM_BUFS; i++) {
- if (contigmem_buffers[i].addr != NULL)
- contigfree(contigmem_buffers[i].addr,
- contigmem_buffer_size, M_CONTIGMEM);
- if (mtx_initialized(&contigmem_buffers[i].mtx))
- mtx_destroy(&contigmem_buffers[i].mtx);
+ for (int j = 0; j < RTE_CONTIGMEM_MAX_NUM_BUFS; j++) {
+ if (cd->cm_buffers[j].addr != NULL)
+ contigfree(cd->cm_buffers[j].addr,
+ contigmem_buffer_size, M_CONTIGMEM);
+ if (mtx_initialized(&cd->cm_buffers[j].mtx))
+ mtx_destroy(&cd->cm_buffers[j].mtx);
+ }
}
+ sysctl_ctx_free(&sysctl_ctx);
+
return 0;
}
static int
contigmem_physaddr(SYSCTL_HANDLER_ARGS)
{
- uint64_t physaddr;
- int index = (int)(uintptr_t)arg1;
+ uint64_t physaddr;
+ struct contigmem_buffer *buf;
- physaddr = (uint64_t)vtophys(contigmem_buffers[index].addr);
+ buf = (struct contigmem_buffer *)arg1;
+
+ physaddr = (uint64_t)vtophys(buf->addr);
return sysctl_handle_64(oidp, &physaddr, 0, req);
}
@@ -216,8 +284,11 @@ static int
contigmem_open(struct cdev *cdev, int fflags, int devtype,
struct thread *td)
{
+ struct contigmem_device *cd;
+ cd = cdev->si_drv1;
atomic_add_int(&contigmem_refcnt, 1);
+ atomic_add_int(&cd->cm_refcnt, 1);
return 0;
}
@@ -226,8 +297,11 @@ static int
contigmem_close(struct cdev *cdev, int fflags, int devtype,
struct thread *td)
{
+ struct contigmem_device *cd;
+ cd = cdev->si_drv1;
atomic_subtract_int(&contigmem_refcnt, 1);
+ atomic_subtract_int(&cd->cm_refcnt, 1);
return 0;
}
@@ -238,10 +312,14 @@ contigmem_cdev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot,
{
struct contigmem_vm_handle *vmh = handle;
struct contigmem_buffer *buf;
+ struct contigmem_device *cd;
- buf = &contigmem_buffers[vmh->buffer_index];
+ cd = &contigmem_device_list[vmh->device_index];
+ buf = &cd->cm_buffers[vmh->buffer_index];
+ vmh, vmh->buffer_index, vmh->device_index, cd, buf, buf->refcnt);
atomic_add_int(&contigmem_refcnt, 1);
+ atomic_add_int(&cd->cm_refcnt, 1);
mtx_lock(&buf->mtx);
if (buf->refcnt == 0)
@@ -257,8 +335,10 @@ contigmem_cdev_pager_dtor(void *handle)
{
struct contigmem_vm_handle *vmh = handle;
struct contigmem_buffer *buf;
+ struct contigmem_device *cd;
- buf = &contigmem_buffers[vmh->buffer_index];
+ cd = &contigmem_device_list[vmh->device_index];
+ buf = &cd->cm_buffers[vmh->buffer_index];
mtx_lock(&buf->mtx);
buf->refcnt--;
@@ -267,6 +347,7 @@ contigmem_cdev_pager_dtor(void *handle)
free(vmh, M_CONTIGMEM);
atomic_subtract_int(&contigmem_refcnt, 1);
+ atomic_subtract_int(&cd->cm_refcnt, 1);
}
static int
@@ -334,8 +415,11 @@ contigmem_mmap_single(struct cdev *cdev, vm_ooffset_t *offset, vm_size_t size,
struct vm_object **obj, int nprot)
{
struct contigmem_vm_handle *vmh;
+ struct contigmem_device *cd;
uint64_t buffer_index;
+ cd = (struct contigmem_device *)cdev->si_drv1;
+
/*
* The buffer index is encoded in the offset. Divide the offset by
* PAGE_SIZE to get the index of the buffer requested by the user
@@ -352,8 +436,9 @@ contigmem_mmap_single(struct cdev *cdev, vm_ooffset_t *offset, vm_size_t size,
if (vmh == NULL)
return ENOMEM;
vmh->buffer_index = buffer_index;
+ vmh->device_index = cd->cm_device_index;
- *offset = (vm_ooffset_t)vtophys(contigmem_buffers[buffer_index].addr);
+ *offset = (vm_ooffset_t)vtophys(cd->cm_buffers[buffer_index].addr);
*obj = cdev_pager_allocate(vmh, OBJT_DEVICE, &contigmem_cdev_pager_ops,
size, nprot, *offset, curthread->td_ucred);
@@ -313,11 +313,13 @@ eal_option_device_parse(void)
const char *
eal_get_hugefile_prefix(void)
{
+#ifndef RTE_EXEC_ENV_FREEBSD
const struct internal_config *internal_conf =
eal_get_internal_configuration();
if (internal_conf->hugefile_prefix != NULL)
return internal_conf->hugefile_prefix;
+#endif
return HUGEFILE_PREFIX_DEFAULT;
}
@@ -464,6 +464,18 @@ eal_parse_args(int argc, char **argv)
}
break;
}
+ case OPT_FILE_PREFIX_NUM:
+ {
+ char *prefix = strdup(optarg);
+ if (prefix == NULL)
+ EAL_LOG(ERR, "Could not store file prefix");
+ else {
+ /* free old prefix */
+ free(internal_conf->hugefile_prefix);
+ internal_conf->hugefile_prefix = prefix;
+ }
+ break;
+ }
case OPT_HELP_NUM:
eal_usage(prgname);
exit(EXIT_SUCCESS);
@@ -14,8 +14,6 @@
#include "eal_internal_cfg.h"
#include "eal_filesystem.h"
-#define CONTIGMEM_DEV "/dev/contigmem"
-
/*
* Uses mmap to create a shared memory area for storage of data
* Used in this file to store the hugepage file map on disk
@@ -85,9 +83,13 @@ eal_hugepage_info_init(void)
return -1;
}
- fd = open(CONTIGMEM_DEV, O_RDWR);
+ char contigmemdev[64];
+ snprintf(contigmemdev, sizeof(contigmemdev), "/dev/%s",
+ internal_conf->hugefile_prefix);
+
+ fd = open(contigmemdev, O_RDWR);
if (fd < 0) {
- EAL_LOG(ERR, "could not open "CONTIGMEM_DEV);
+ EAL_LOG(ERR, "could not open %s", contigmemdev);
return -1;
}
if (flock(fd, LOCK_EX | LOCK_NB) < 0) {
@@ -105,7 +107,7 @@ eal_hugepage_info_init(void)
EAL_LOG(INFO, "Contigmem driver has %d buffers, each of size %dKB",
num_buffers, (int)(buffer_size>>10));
- strlcpy(hpi->hugedir, CONTIGMEM_DEV, sizeof(hpi->hugedir));
+ strlcpy(hpi->hugedir, contigmemdev, sizeof(hpi->hugedir));
hpi->hugepage_sz = buffer_size;
hpi->num_pages[0] = num_buffers;
hpi->lock_descriptor = fd;
@@ -128,7 +128,8 @@ rte_eal_hugepage_init(void)
* the previous one.
*/
snprintf(physaddr_str, sizeof(physaddr_str),
- "hw.contigmem.physaddr.%d", j);
+ "hw.contigmem.%s.%d",
+ internal_conf->hugefile_prefix, j);
error = sysctlbyname(physaddr_str, &physaddr,
&sysctl_size, NULL, 0);
if (error < 0) {