@@ -50,6 +50,7 @@ test_sources = files(
'test_errno.c',
'test_ethdev_link.c',
'test_event_crypto_adapter.c',
+ 'test_event_dispatcher.c',
'test_event_eth_rx_adapter.c',
'test_event_ring.c',
'test_event_timer_adapter.c',
new file mode 100644
@@ -0,0 +1,814 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2023 Ericsson AB
+ */
+
+#include "test.h"
+
+#include <stdatomic.h>
+
+#include <rte_bus_vdev.h>
+#include <rte_event_dispatcher.h>
+#include <rte_eventdev.h>
+#include <rte_random.h>
+#include <rte_service.h>
+
+#define NUM_WORKERS 3
+
+#define NUM_PORTS (NUM_WORKERS + 1)
+#define WORKER_PORT_ID(worker_idx) (worker_idx)
+#define DRIVER_PORT_ID (NUM_PORTS - 1)
+
+#define NUM_SERVICE_CORES NUM_WORKERS
+
+/* Eventdev */
+#define NUM_QUEUES 8
+#define LAST_QUEUE_ID (NUM_QUEUES - 1)
+#define MAX_EVENTS 4096
+#define NEW_EVENT_THRESHOLD (MAX_EVENTS / 2)
+#define DEQUEUE_BURST_SIZE 32
+#define ENQUEUE_BURST_SIZE 32
+
+#define NUM_EVENTS 10000000
+#define NUM_FLOWS 16
+
+#define DSW_VDEV "event_dsw0"
+
+struct app_queue {
+ uint8_t queue_id;
+ uint64_t sn[NUM_FLOWS];
+ int dispatcher_reg_id;
+};
+
+struct test_app {
+ uint8_t event_dev_id;
+ uint8_t dispatcher_id;
+ uint32_t dispatcher_service_id;
+
+ unsigned int service_lcores[NUM_SERVICE_CORES];
+
+ struct app_queue queues[NUM_QUEUES];
+
+ bool running;
+
+ atomic_int completed_events;
+ atomic_int errors;
+};
+
+#define RETURN_ON_ERROR(rc) \
+ do { \
+ if (rc != TEST_SUCCESS) \
+ return rc; \
+ } while (0)
+
+static struct test_app *
+test_app_create(void)
+{
+ int i;
+ struct test_app *app;
+
+ app = calloc(1, sizeof(struct test_app));
+
+ if (app == NULL)
+ return NULL;
+
+ for (i = 0; i < NUM_QUEUES; i++)
+ app->queues[i].queue_id = i;
+
+ return app;
+}
+
+static void
+test_app_free(struct test_app *app)
+{
+ free(app);
+}
+
+static int
+test_app_create_vdev(struct test_app *app)
+{
+ int rc;
+
+ rc = rte_vdev_init(DSW_VDEV, NULL);
+ if (rc < 0)
+ return TEST_SKIPPED;
+
+ rc = rte_event_dev_get_dev_id(DSW_VDEV);
+
+ app->event_dev_id = (uint8_t)rc;
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_app_destroy_vdev(struct test_app *app)
+{
+ int rc;
+
+ rc = rte_event_dev_close(app->event_dev_id);
+ TEST_ASSERT_SUCCESS(rc, "Error while closing event device");
+
+ rc = rte_vdev_uninit(DSW_VDEV);
+ TEST_ASSERT_SUCCESS(rc, "Error while uninitializing virtual device");
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_app_setup_event_dev(struct test_app *app)
+{
+ int rc;
+ int i;
+
+ rc = test_app_create_vdev(app);
+ if (rc < 0)
+ return rc;
+
+ struct rte_event_dev_config config = {
+ .nb_event_queues = NUM_QUEUES,
+ .nb_event_ports = NUM_PORTS,
+ .nb_events_limit = MAX_EVENTS,
+ .nb_event_queue_flows = 64,
+ .nb_event_port_dequeue_depth = DEQUEUE_BURST_SIZE,
+ .nb_event_port_enqueue_depth = ENQUEUE_BURST_SIZE
+ };
+
+ rc = rte_event_dev_configure(app->event_dev_id, &config);
+
+ TEST_ASSERT_SUCCESS(rc, "Unable to configure event device");
+
+ struct rte_event_queue_conf queue_config = {
+ .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
+ .schedule_type = RTE_SCHED_TYPE_ATOMIC,
+ .nb_atomic_flows = 64
+ };
+
+ for (i = 0; i < NUM_QUEUES; i++) {
+ uint8_t queue_id = i;
+
+ rc = rte_event_queue_setup(app->event_dev_id, queue_id,
+ &queue_config);
+
+ TEST_ASSERT_SUCCESS(rc, "Unable to setup queue %d", queue_id);
+ }
+
+ struct rte_event_port_conf port_config = {
+ .new_event_threshold = NEW_EVENT_THRESHOLD,
+ .dequeue_depth = DEQUEUE_BURST_SIZE,
+ .enqueue_depth = ENQUEUE_BURST_SIZE
+ };
+
+ for (i = 0; i < NUM_PORTS; i++) {
+ uint8_t event_port_id = i;
+
+ rc = rte_event_port_setup(app->event_dev_id, event_port_id,
+ &port_config);
+ TEST_ASSERT_SUCCESS(rc, "Failed to create event port %d",
+ event_port_id);
+
+ if (event_port_id == DRIVER_PORT_ID)
+ continue;
+
+ rc = rte_event_port_link(app->event_dev_id, event_port_id,
+ NULL, NULL, 0);
+
+ TEST_ASSERT_EQUAL(rc, NUM_QUEUES, "Failed to link port %d",
+ event_port_id);
+ }
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_app_teardown_event_dev(struct test_app *app)
+{
+ return test_app_destroy_vdev(app);
+}
+
+static int
+test_app_start_event_dev(struct test_app *app)
+{
+ int rc;
+
+ rc = rte_event_dev_start(app->event_dev_id);
+ TEST_ASSERT_SUCCESS(rc, "Unable to start event device");
+
+ return TEST_SUCCESS;
+}
+
+static void
+test_app_stop_event_dev(struct test_app *app)
+{
+ rte_event_dev_stop(app->event_dev_id);
+}
+
+static int
+test_app_create_dispatcher(struct test_app *app)
+{
+ int rc;
+
+ app->dispatcher_id = rte_rand_max(256);
+
+ rc = rte_event_dispatcher_create(app->dispatcher_id,
+ app->event_dev_id);
+
+ TEST_ASSERT_SUCCESS(rc, "Unable to create event dispatcher");
+
+ rc = rte_event_dispatcher_service_id_get(app->dispatcher_id,
+ &app->dispatcher_service_id);
+ TEST_ASSERT_SUCCESS(rc, "Unable to get event dispatcher service ID");
+
+ rc = rte_service_set_stats_enable(app->dispatcher_service_id, 1);
+
+ TEST_ASSERT_SUCCESS(rc, "Unable to enable event dispatcher service "
+ "stats");
+
+ rc = rte_service_runstate_set(app->dispatcher_service_id, 1);
+ TEST_ASSERT_SUCCESS(rc, "Error disabling dispatcher service");
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_app_free_dispatcher(struct test_app *app)
+{
+ int rc;
+
+ rc = rte_service_runstate_set(app->dispatcher_service_id, 0);
+ TEST_ASSERT_SUCCESS(rc, "Error disabling dispatcher service");
+
+ rte_event_dispatcher_free(app->dispatcher_id);
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_app_bind_ports(struct test_app *app)
+{
+ int i;
+
+ for (i = 0; i < NUM_WORKERS; i++) {
+ unsigned int lcore_id = app->service_lcores[i];
+
+ int rc = rte_event_dispatcher_bind_port_to_lcore(
+ app->dispatcher_id, WORKER_PORT_ID(i),
+ DEQUEUE_BURST_SIZE, 0, lcore_id
+ );
+
+ TEST_ASSERT_SUCCESS(rc, "Unable to bind event device port %d "
+ "to lcore %d", WORKER_PORT_ID(i),
+ lcore_id);
+ }
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_app_unbind_ports(struct test_app *app)
+{
+ int i;
+
+ for (i = 0; i < NUM_WORKERS; i++) {
+ unsigned int lcore_id = app->service_lcores[i];
+
+ int rc = rte_event_dispatcher_unbind_port_from_lcore(
+ app->dispatcher_id,
+ WORKER_PORT_ID(i),
+ lcore_id
+ );
+
+ TEST_ASSERT_SUCCESS(rc, "Unable to unbind event device port %d "
+ "from lcore %d", WORKER_PORT_ID(i),
+ lcore_id);
+ }
+
+ return TEST_SUCCESS;
+}
+
+static bool
+match_queue(const struct rte_event *event, void *cb_data)
+{
+ uintptr_t queue_id = (uintptr_t)cb_data;
+
+ return event->queue_id == queue_id;
+}
+
+static int
+test_app_get_worker_index(struct test_app *app, unsigned int lcore_id)
+{
+ int i;
+
+ for (i = 0; i < NUM_SERVICE_CORES; i++)
+ if (app->service_lcores[i] == lcore_id)
+ return i;
+
+ return -1;
+}
+
+static int
+test_app_get_worker_port(struct test_app *app, unsigned int lcore_id)
+{
+ int worker;
+
+ worker = test_app_get_worker_index(app, lcore_id);
+
+ if (worker < 0)
+ return -1;
+
+ return WORKER_PORT_ID(worker);
+}
+
+static void
+test_app_queue_note_error(struct test_app *app)
+{
+ atomic_fetch_add_explicit(&app->errors, 1, memory_order_relaxed);
+}
+
+static void
+test_app_process_queue(uint8_t p_event_dev_id, uint8_t p_event_port_id,
+ const struct rte_event *in_events, uint16_t num,
+ void *cb_data)
+{
+ struct app_queue *app_queue = cb_data;
+ struct test_app *app = container_of(app_queue, struct test_app,
+ queues[app_queue->queue_id]);
+ unsigned int lcore_id = rte_lcore_id();
+ bool intermediate_queue = app_queue->queue_id != LAST_QUEUE_ID;
+ int event_port_id;
+ uint16_t i;
+ struct rte_event out_events[num];
+
+ event_port_id = test_app_get_worker_port(app, lcore_id);
+
+ if (event_port_id < 0 || p_event_dev_id != app->event_dev_id ||
+ p_event_port_id != event_port_id) {
+ test_app_queue_note_error(app);
+ return;
+ }
+
+ for (i = 0; i < num; i++) {
+ const struct rte_event *in_event = &in_events[i];
+ struct rte_event *out_event = &out_events[i];
+ uint64_t sn = in_event->u64;
+ uint64_t expected_sn;
+
+ if (in_event->queue_id != app_queue->queue_id) {
+ test_app_queue_note_error(app);
+ return;
+ }
+
+ expected_sn = app_queue->sn[in_event->flow_id]++;
+
+ if (expected_sn != sn) {
+ test_app_queue_note_error(app);
+ return;
+ }
+
+ if (intermediate_queue)
+ *out_event = (struct rte_event) {
+ .queue_id = in_event->queue_id + 1,
+ .flow_id = in_event->flow_id,
+ .sched_type = RTE_SCHED_TYPE_ATOMIC,
+ .op = RTE_EVENT_OP_FORWARD,
+ .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
+ .u64 = sn
+ };
+ }
+
+ if (intermediate_queue) {
+ uint16_t n = 0;
+
+ do {
+ n += rte_event_enqueue_forward_burst(p_event_dev_id,
+ p_event_port_id,
+ out_events + n,
+ num - n);
+ } while (n != num);
+ } else
+ atomic_fetch_add_explicit(&app->completed_events, num,
+ memory_order_relaxed);
+}
+
+static int
+test_app_register_callbacks(struct test_app *app)
+{
+ int i;
+
+ for (i = 0; i < NUM_QUEUES; i++) {
+ struct app_queue *app_queue = &app->queues[i];
+ uintptr_t queue_id = app_queue->queue_id;
+ int reg_id;
+
+ reg_id = rte_event_dispatcher_register(app->dispatcher_id,
+ match_queue,
+ (void *)queue_id,
+ test_app_process_queue,
+ app_queue);
+
+ TEST_ASSERT(reg_id >= 0, "Unable to register consumer "
+ "callback for queue %d", i);
+
+ app_queue->dispatcher_reg_id = reg_id;
+ }
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_app_unregister_callback(struct test_app *app, uint8_t queue_id)
+{
+ int reg_id = app->queues[queue_id].dispatcher_reg_id;
+
+ if (reg_id < 0) /* unregistered already */
+ return 0;
+
+ int rc = rte_event_dispatcher_unregister(app->dispatcher_id, reg_id);
+
+ TEST_ASSERT_SUCCESS(rc, "Unable to unregister consumer "
+ "callback for queue %d", queue_id);
+
+ app->queues[queue_id].dispatcher_reg_id = -1;
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_app_unregister_callbacks(struct test_app *app)
+{
+ int i;
+
+ for (i = 0; i < NUM_QUEUES; i++) {
+ int rc;
+
+ rc = test_app_unregister_callback(app, i);
+ RETURN_ON_ERROR(rc);
+ }
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_app_start_dispatcher(struct test_app *app)
+{
+ int rc;
+
+ rc = rte_event_dispatcher_start(app->dispatcher_id);
+
+ TEST_ASSERT_SUCCESS(rc, "Unable to start the event dispatcher");
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_app_stop_dispatcher(struct test_app *app)
+{
+ int rc;
+
+ rc = rte_event_dispatcher_stop(app->dispatcher_id);
+
+ TEST_ASSERT_SUCCESS(rc, "Unable to stop the event dispatcher");
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_app_setup_service_core(struct test_app *app, unsigned int lcore_id)
+{
+ int rc;
+
+ rc = rte_service_lcore_add(lcore_id);
+ TEST_ASSERT_SUCCESS(rc, "Unable to make lcore %d an event dispatcher "
+ "service core", lcore_id);
+
+ rc = rte_service_map_lcore_set(app->dispatcher_service_id, lcore_id, 1);
+ TEST_ASSERT_SUCCESS(rc, "Unable to map event dispatcher service");
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_app_setup_service_cores(struct test_app *app)
+{
+ int i;
+ int lcore_id = -1;
+
+ for (i = 0; i < NUM_SERVICE_CORES; i++) {
+ lcore_id = rte_get_next_lcore(lcore_id, 1, 0);
+
+ TEST_ASSERT(lcore_id != RTE_MAX_LCORE,
+ "Too few lcores. Needs at least %d worker lcores",
+ NUM_SERVICE_CORES);
+
+ app->service_lcores[i] = lcore_id;
+ }
+
+ for (i = 0; i < NUM_SERVICE_CORES; i++) {
+ int rc;
+
+ rc = test_app_setup_service_core(app, app->service_lcores[i]);
+
+ RETURN_ON_ERROR(rc);
+ }
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_app_teardown_service_core(struct test_app *app, unsigned int lcore_id)
+{
+ int rc;
+
+ rc = rte_service_map_lcore_set(app->dispatcher_service_id, lcore_id, 0);
+ TEST_ASSERT_SUCCESS(rc, "Unable to unmap event dispatcher service");
+
+ rc = rte_service_lcore_del(lcore_id);
+ TEST_ASSERT_SUCCESS(rc, "Unable change role of service lcore %d",
+ lcore_id);
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_app_teardown_service_cores(struct test_app *app)
+{
+ int i;
+
+ for (i = 0; i < NUM_SERVICE_CORES; i++) {
+ unsigned int lcore_id = app->service_lcores[i];
+ int rc;
+
+ rc = test_app_teardown_service_core(app, lcore_id);
+
+ RETURN_ON_ERROR(rc);
+ }
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_app_start_service_cores(struct test_app *app)
+{
+ int i;
+
+ for (i = 0; i < NUM_SERVICE_CORES; i++) {
+ unsigned int lcore_id = app->service_lcores[i];
+ int rc;
+
+ rc = rte_service_lcore_start(lcore_id);
+ TEST_ASSERT_SUCCESS(rc, "Unable to start service lcore %d",
+ lcore_id);
+
+ RETURN_ON_ERROR(rc);
+ }
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_app_stop_service_cores(struct test_app *app)
+{
+ int i;
+
+ for (i = 0; i < NUM_SERVICE_CORES; i++) {
+ unsigned int lcore_id = app->service_lcores[i];
+ int rc;
+
+ rc = rte_service_lcore_stop(lcore_id);
+ TEST_ASSERT_SUCCESS(rc, "Unable to stop service lcore %d",
+ lcore_id);
+
+ RETURN_ON_ERROR(rc);
+ }
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_app_start(struct test_app *app)
+{
+ int rc;
+
+ rc = test_app_start_event_dev(app);
+ RETURN_ON_ERROR(rc);
+
+ rc = test_app_start_service_cores(app);
+ RETURN_ON_ERROR(rc);
+
+ rc = test_app_start_dispatcher(app);
+
+ app->running = true;
+
+ return rc;
+}
+
+static int
+test_app_stop(struct test_app *app)
+{
+ int rc;
+
+ rc = test_app_stop_dispatcher(app);
+ RETURN_ON_ERROR(rc);
+
+ test_app_stop_service_cores(app);
+ RETURN_ON_ERROR(rc);
+
+ test_app_stop_event_dev(app);
+ RETURN_ON_ERROR(rc);
+
+ app->running = false;
+
+ return TEST_SUCCESS;
+}
+
+struct test_app *test_app;
+
+static int
+test_setup(void)
+{
+ int rc;
+
+ test_app = test_app_create();
+ TEST_ASSERT(test_app != NULL, "Unable to allocate memory");
+
+ rc = test_app_setup_event_dev(test_app);
+ RETURN_ON_ERROR(rc);
+
+ rc = test_app_create_dispatcher(test_app);
+
+ rc = test_app_setup_service_cores(test_app);
+ RETURN_ON_ERROR(rc);
+
+ rc = test_app_register_callbacks(test_app);
+ RETURN_ON_ERROR(rc);
+
+ rc = test_app_bind_ports(test_app);
+
+ return rc;
+}
+
+static void test_teardown(void)
+{
+ if (test_app->running)
+ test_app_stop(test_app);
+
+ test_app_teardown_service_cores(test_app);
+
+ test_app_unregister_callbacks(test_app);
+
+ test_app_unbind_ports(test_app);
+
+ test_app_free_dispatcher(test_app);
+
+ test_app_teardown_event_dev(test_app);
+
+ test_app_free(test_app);
+
+ test_app = NULL;
+}
+
+static int
+test_app_get_completed_events(struct test_app *app)
+{
+ return atomic_load_explicit(&app->completed_events,
+ memory_order_relaxed);
+}
+
+static int
+test_app_get_errors(struct test_app *app)
+{
+ return atomic_load_explicit(&app->errors, memory_order_relaxed);
+}
+
+static int
+test_basic(void)
+{
+ int rc;
+ int i;
+
+ rc = test_app_start(test_app);
+ RETURN_ON_ERROR(rc);
+
+ uint64_t sns[NUM_FLOWS] = { 0 };
+
+ for (i = 0; i < NUM_EVENTS;) {
+ struct rte_event events[ENQUEUE_BURST_SIZE];
+ int left;
+ int batch_size;
+ int j;
+ uint16_t n = 0;
+
+ batch_size = 1 + rte_rand_max(ENQUEUE_BURST_SIZE);
+ left = NUM_EVENTS - i;
+
+ batch_size = RTE_MIN(left, batch_size);
+
+ for (j = 0; j < batch_size; j++) {
+ struct rte_event *event = &events[j];
+ uint64_t sn;
+ uint32_t flow_id;
+
+ flow_id = rte_rand_max(NUM_FLOWS);
+
+ sn = sns[flow_id]++;
+
+ *event = (struct rte_event) {
+ .queue_id = 0,
+ .flow_id = flow_id,
+ .sched_type = RTE_SCHED_TYPE_ATOMIC,
+ .op = RTE_EVENT_OP_NEW,
+ .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
+ .u64 = sn
+ };
+ }
+
+ while (n < batch_size)
+ n += rte_event_enqueue_new_burst(test_app->event_dev_id,
+ DRIVER_PORT_ID,
+ events + n,
+ batch_size - n);
+
+ i += batch_size;
+ }
+
+ while (test_app_get_completed_events(test_app) != NUM_EVENTS)
+ rte_event_maintain(test_app->event_dev_id, DRIVER_PORT_ID, 0);
+
+ rc = test_app_get_errors(test_app);
+ TEST_ASSERT(rc == 0, "%d errors occurred", rc);
+
+ rc = test_app_stop(test_app);
+ RETURN_ON_ERROR(rc);
+
+ struct rte_event_dispatcher_stats stats;
+ rc = rte_event_dispatcher_stats_get(test_app->dispatcher_id,
+ &stats);
+
+ TEST_ASSERT_EQUAL(stats.ev_drop_count, 0, "Drop count is not zero");
+ TEST_ASSERT_EQUAL(stats.ev_dispatch_count, NUM_EVENTS * NUM_QUEUES,
+ "Invalid dispatch count");
+ TEST_ASSERT(stats.poll_count > 0, "Poll count is zero");
+
+ return TEST_SUCCESS;
+}
+
+static int
+test_drop(void)
+{
+ int rc;
+ uint8_t unhandled_queue = 1;
+ struct rte_event_dispatcher_stats stats;
+
+ rc = test_app_start(test_app);
+ RETURN_ON_ERROR(rc);
+
+ rc = test_app_unregister_callback(test_app, unhandled_queue);
+ RETURN_ON_ERROR(rc);
+
+ struct rte_event event = {
+ .queue_id = unhandled_queue,
+ .flow_id = 0,
+ .sched_type = RTE_SCHED_TYPE_ATOMIC,
+ .op = RTE_EVENT_OP_NEW,
+ .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
+ .u64 = 0
+ };
+
+ do {
+ rc = rte_event_enqueue_burst(test_app->event_dev_id,
+ DRIVER_PORT_ID, &event, 1);
+ } while (rc == 0);
+
+ do {
+ rc = rte_event_dispatcher_stats_get(test_app->dispatcher_id,
+ &stats);
+ RETURN_ON_ERROR(rc);
+
+ rte_event_maintain(test_app->event_dev_id, DRIVER_PORT_ID, 0);
+ } while (stats.ev_drop_count == 0 && stats.ev_dispatch_count == 0);
+
+ rc = test_app_stop(test_app);
+ RETURN_ON_ERROR(rc);
+
+ TEST_ASSERT_EQUAL(stats.ev_drop_count, 1, "Drop count is not one");
+ TEST_ASSERT_EQUAL(stats.ev_dispatch_count, 0,
+ "Dispatch count is not zero");
+ TEST_ASSERT(stats.poll_count > 0, "Poll count is zero");
+
+ return TEST_SUCCESS;
+}
+
+static struct unit_test_suite test_suite = {
+ .suite_name = "Event dispatcher test suite",
+ .unit_test_cases = {
+ TEST_CASE_ST(test_setup, test_teardown, test_basic),
+ TEST_CASE_ST(test_setup, test_teardown, test_drop),
+ TEST_CASES_END()
+ }
+};
+
+static int
+test_event_dispatcher(void)
+{
+ return unit_test_suite_runner(&test_suite);
+}
+
+REGISTER_TEST_COMMAND(event_dispatcher_autotest, test_event_dispatcher);