@@ -282,6 +282,11 @@ CONFIG_RTE_LIBRTE_HASH=y
CONFIG_RTE_LIBRTE_HASH_DEBUG=n
#
+# Compile librte_headroom
+#
+CONFIG_RTE_LIBRTE_HEADROOM=y
+
+#
# Compile librte_lpm
#
CONFIG_RTE_LIBRTE_LPM=y
@@ -290,6 +290,11 @@ CONFIG_RTE_LIBRTE_HASH=y
CONFIG_RTE_LIBRTE_HASH_DEBUG=n
#
+# Compile librte_headroom
+#
+CONFIG_RTE_LIBRTE_HEADROOM=y
+
+#
# Compile librte_lpm
#
CONFIG_RTE_LIBRTE_LPM=y
@@ -54,6 +54,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_VMXNET3_PMD) += librte_pmd_vmxnet3
DIRS-$(CONFIG_RTE_LIBRTE_PMD_XENVIRT) += librte_pmd_xenvirt
DIRS-$(CONFIG_RTE_LIBRTE_VHOST) += librte_vhost
DIRS-$(CONFIG_RTE_LIBRTE_HASH) += librte_hash
+DIRS-$(CONFIG_RTE_LIBRTE_HEADROOM) += librte_headroom
DIRS-$(CONFIG_RTE_LIBRTE_LPM) += librte_lpm
DIRS-$(CONFIG_RTE_LIBRTE_ACL) += librte_acl
DIRS-$(CONFIG_RTE_LIBRTE_NET) += librte_net
new file mode 100644
@@ -0,0 +1,54 @@
+# BSD LICENSE
+#
+# Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Intel Corporation nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+# library name
+LIB = librte_headroom.a
+
+CFLAGS += -O3
+CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR)
+
+EXPORT_MAP := rte_headroom_version.map
+
+LIBABIVER := 1
+
+# all source are stored in SRCS-y
+SRCS-$(CONFIG_RTE_LIBRTE_HEADROOM) := rte_headroom.c
+
+# install this header file
+SYMLINK-$(CONFIG_RTE_LIBRTE_HEADROOM)-include := rte_headroom.h
+
+# this lib needs eal
+DEPDIRS-$(CONFIG_RTE_LIBRTE_HEADROOM) += lib/librte_eal
+DEPDIRS-$(CONFIG_RTE_LIBRTE_HEADROOM) += lib/librte_mbuf
+
+include $(RTE_SDK)/mk/rte.lib.mk
new file mode 100644
@@ -0,0 +1,271 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <string.h>
+#include <stdlib.h>
+
+#include <rte_log.h>
+#include <rte_errno.h>
+#include <rte_common.h>
+#include <rte_cycles.h>
+#include <rte_branch_prediction.h>
+#include <rte_debug.h>
+#include <rte_eal.h>
+#include <rte_malloc.h>
+
+#include "rte_headroom.h"
+
+/* Those are steps used to adjust job period.
+ * Experiments show that for forwarding apps the up step must be less than down
+ * step to achieve optimal performance.
+ */
+#define JOB_UPDATE_STEP_UP 1
+#define JOB_UPDATE_STEP_DOWN 4
+
+/*
+ * Default update function that implements simple period adjustment.
+ */
+static void
+default_update_function(struct rte_headroom_job *job, int64_t result)
+{
+ int64_t err = job->target - result;
+
+ /* Job is happy. Nothing to do */
+ if (err == 0)
+ return;
+
+ if (err > 0) {
+ if (job->period + JOB_UPDATE_STEP_UP < job->max_period)
+ job->period += JOB_UPDATE_STEP_UP;
+ } else {
+ if (job->min_period + JOB_UPDATE_STEP_DOWN < job->period)
+ job->period -= JOB_UPDATE_STEP_DOWN;
+ }
+}
+
+#define HDR_ADD_TIME_MIN_MAX(obj, type, value) do { \
+ typeof(value) tmp = (value); \
+ (obj)->type ## _time += tmp; \
+ if (tmp < (obj)->min_ ## type ## _time) \
+ (obj)->min_ ## type ## _time = tmp; \
+ if (tmp > (obj)->max_ ## type ## _time) \
+ (obj)->max_ ## type ## _time = tmp; \
+} while (0)
+
+#define HDR_RESET_TIME_MIN_MAX(obj, type) do { \
+ (obj)->type ## _time = 0; \
+ (obj)->min_ ## type ## _time = UINT64_MAX; \
+ (obj)->max_ ## type ## _time = 0; \
+} while (0)
+
+int
+rte_headroom_init(struct rte_headroom *hdr)
+{
+ if (hdr == NULL)
+ return -EINVAL;
+
+ /* Init only needed parameters. Zero out everything else. */
+ memset(hdr, 0, sizeof(struct rte_headroom));
+
+ rte_headroom_reset_stats(hdr);
+
+ return 0;
+}
+
+void
+rte_headroom_start_loop(struct rte_headroom *hdr)
+{
+ uint64_t now;
+
+ hdr->loop_executed_jobs = 0;
+
+ rte_mb();
+ now = rte_get_timer_cycles();
+ HDR_ADD_TIME_MIN_MAX(hdr, management, now - hdr->state_time);
+ hdr->state_time = now;
+}
+
+void
+rte_headroom_finish_loop(struct rte_headroom *hdr)
+{
+ uint64_t now;
+
+ if (likely(hdr->loop_executed_jobs))
+ hdr->loop_cnt++;
+
+ rte_mb();
+ now = rte_get_timer_cycles();
+ HDR_ADD_TIME_MIN_MAX(hdr, management, now - hdr->state_time);
+ hdr->state_time = now;
+}
+
+void
+rte_headroom_set_job_target(struct rte_headroom_job *job, int64_t target)
+{
+ job->target = target;
+}
+
+int
+rte_headroom_start_job(struct rte_headroom *hdr, struct rte_headroom_job *job)
+{
+ uint64_t now;
+
+ /* Some sanity check. */
+ if (unlikely(hdr == NULL || job == NULL || job->headroom != NULL))
+ return -EINVAL;
+
+ /* Link job with headroom object. */
+ job->headroom = hdr;
+
+ rte_mb();
+ now = rte_get_timer_cycles();
+ HDR_ADD_TIME_MIN_MAX(hdr, management, now - hdr->state_time);
+ hdr->state_time = now;
+
+ return 0;
+}
+
+int
+rte_headroom_finish_job(struct rte_headroom_job *job, int64_t job_value)
+{
+ struct rte_headroom *hdr;
+ uint64_t now, exec_time;
+ int need_update;
+
+ /* Some sanity check. */
+ if (unlikely(job == NULL || job->headroom == NULL))
+ return -EINVAL;
+
+ need_update = job->target != job_value;
+ /* Adjust period only if job is unhappy of its current period. */
+ if (need_update)
+ (*job->update_period_cb)(job, job_value);
+
+ hdr = job->headroom;
+
+ /* Update execution time is considered as runtime so get time after it is
+ * executed. */
+ rte_mb();
+ now = rte_get_timer_cycles();
+ exec_time = now - hdr->state_time;
+ HDR_ADD_TIME_MIN_MAX(job, exec, exec_time);
+ HDR_ADD_TIME_MIN_MAX(hdr, exec, exec_time);
+
+ hdr->state_time = now;
+
+ hdr->loop_executed_jobs++;
+ hdr->job_exec_cnt++;
+
+ job->exec_cnt++;
+ job->headroom = NULL;
+
+ return need_update;
+}
+
+void
+rte_headroom_job_set_period(struct rte_headroom_job *job, uint64_t period,
+ uint8_t saturate)
+{
+ if (saturate != 0) {
+ if (period < job->min_period)
+ period = job->min_period;
+ else if (period > job->max_period)
+ period = job->max_period;
+ }
+
+ job->period = period;
+}
+
+void
+rte_headroom_set_min_period(struct rte_headroom_job *job, uint64_t period)
+{
+ job->min_period = period;
+ if (job->period < period)
+ job->period = period;
+}
+
+void
+rte_headroom_set_max_period(struct rte_headroom_job *job, uint64_t period)
+{
+ job->max_period = period;
+ if (job->period > period)
+ job->period = period;
+}
+
+int
+rte_headroom_job_init(struct rte_headroom_job *job, const char *name,
+ uint64_t min_period, uint64_t max_period, uint64_t initial_period,
+ int64_t target)
+{
+ if (job == NULL)
+ return -EINVAL;
+
+ job->period = initial_period;
+ job->min_period = min_period;
+ job->max_period = max_period;
+ job->target = target;
+ job->update_period_cb = &default_update_function;
+ rte_headroom_reset_job_stats(job);
+ snprintf(job->name, RTE_DIM(job->name), "%s", name == NULL ? "" : name);
+ job->headroom = NULL;
+
+ return 0;
+}
+
+void
+rte_headroom_set_update_period_function(struct rte_headroom_job *job,
+ rte_headroom_update_fn_t update_period_cb)
+{
+ if (update_period_cb == NULL)
+ update_period_cb = default_update_function;
+
+ job->update_period_cb = update_period_cb;
+}
+
+void
+rte_headroom_reset_job_stats(struct rte_headroom_job *job)
+{
+ HDR_RESET_TIME_MIN_MAX(job, exec);
+ job->exec_cnt = 0;
+}
+
+void
+rte_headroom_reset_stats(struct rte_headroom *hdr)
+{
+ HDR_RESET_TIME_MIN_MAX(hdr, exec);
+ HDR_RESET_TIME_MIN_MAX(hdr, management);
+ hdr->start_time = rte_get_timer_cycles();
+ hdr->state_time = hdr->start_time;
+ hdr->job_exec_cnt = 0;
+ hdr->loop_cnt = 0;
+}
new file mode 100644
@@ -0,0 +1,324 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef HEADROOM_H_
+#define HEADROOM_H_
+
+#include <stdint.h>
+
+#include <rte_memory.h>
+#include <rte_memcpy.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define RTE_HEADROOM_JOB_NAMESIZE 32
+#define RTE_HEADROOM_NAMESIZE 32
+#define RTE_HEADROOM_MZ_PREFIX "HDR_"
+
+/* Forward declarations. */
+struct rte_headroom;
+struct rte_headroom_job;
+
+/**
+ * This function should calculate new period and set it using
+ * rte_headroom_set_period() function. Time spent in this function will be
+ * added to job's runtime.
+ *
+ * @param job
+ * The job data structure handler.
+ * @param job_result
+ * Result of calling job callback.
+ */
+typedef void (*rte_headroom_update_fn_t)(struct rte_headroom_job *job,
+ int64_t job_result);
+
+struct rte_headroom_job {
+ uint64_t period;
+ /**< Estimated period of execution. */
+
+ uint64_t min_period;
+ /**< Minimum period. */
+
+ uint64_t max_period;
+ /**< Maximum period. */
+
+ int64_t target;
+ /**< Desired value for this job. */
+
+ rte_headroom_update_fn_t update_period_cb;
+ /**< Period update callback. */
+
+ uint64_t exec_time;
+ /**< Total time (sum) that this job was executing. */
+
+ uint64_t min_exec_time;
+ /**< Minimum execute time. */
+
+ uint64_t max_exec_time;
+ /**< Minimum execute time. */
+
+ uint64_t exec_cnt;
+ /**< Execute count. */
+
+ char name[RTE_HEADROOM_JOB_NAMESIZE];
+ /**< Name of this job */
+
+ struct rte_headroom *headroom;
+ /**< Headroom object that is executing this job. */
+} __rte_cache_aligned;
+
+struct rte_headroom {
+ /** Viariable holding time at different points:
+ * -# loop start time if loop was started but no job executed yet.
+ * -# job start time if job is currently executing.
+ * -# job finish time if job finished its execution.
+ * -# loop finish time if loop finished its execution. */
+ uint64_t state_time;
+
+ uint64_t loop_executed_jobs;
+ /**< Count of executed jobs in this loop. */
+
+ /* Statistics start. */
+
+ uint64_t exec_time;
+ /**< Total time taken to execute jobs, not including management time. */
+
+ uint64_t min_exec_time;
+ /**< Minimum loop execute time. */
+
+ uint64_t max_exec_time;
+ /**< Minimum loop execute time. */
+
+ /**
+ * Sum of time that is not the execute time (ex: from job finish to next
+ * job start).
+ *
+ * This time might be considered as overhead of headroom library + job
+ * scheduling.
+ */
+ uint64_t management_time;
+
+ uint64_t min_management_time;
+ /**< Minimum management time */
+
+ uint64_t max_management_time;
+ /**< Maximum management time */
+
+ uint64_t start_time;
+ /**< Time since last reset stats. */
+
+ uint64_t job_exec_cnt;
+ /**< Total count of executed jobs. */
+
+ uint64_t loop_cnt;
+ /**< Total count of executed loops with at least one executed job. */
+} __rte_cache_aligned;
+
+/**
+ * Initialize given headroom object with default values.
+ *
+ * @param hdr
+ * Headroom object to initialize.
+ *
+ * @return
+ * 0 on success
+ * -EINVAL if *hdr* is NULL
+ */
+int
+rte_headroom_init(struct rte_headroom *hdr);
+
+/**
+ * Mark that new set of jobs start executing.
+ *
+ * @param hdr
+ * Headroom object.
+ */
+void
+rte_headroom_start_loop(struct rte_headroom *hdr);
+
+/**
+ * Mark that there is no more jobs ready to execute in this turn. Calculate
+ * stats for this loop turn.
+ *
+ * @param hdr
+ * Headroom object.
+ */
+void
+rte_headroom_finish_loop(struct rte_headroom *hdr);
+
+/**
+ * Initialize given job stats object.
+ *
+ * @param job
+ * Job object.
+ * @param name
+ * Optional job name.
+ * @param min_period
+ * Minimum period that this job can accept.
+ * @param max_period
+ * Maximum period that this job can accept.
+ * @param initial_period
+ * Initial period. It will be checked against *min_period* and *max_period*.
+ * @param target
+ * Target value that this job try to achieve.
+ *
+ * @return
+ * 0 on success
+ * -EINVAL if *job* is NULL
+ */
+int
+rte_headroom_job_init(struct rte_headroom_job *job, const char *name,
+ uint64_t min_period, uint64_t max_period, uint64_t initial_period,
+ int64_t target);
+
+/**
+ * Set job desired target value. Difference between target and job callback
+ * return value must be used to properly adjust job execute period value.
+ *
+ * @param job
+ * The job object.
+ * @param target
+ * New target.
+ */
+void
+rte_headroom_set_job_target(struct rte_headroom_job *job, int64_t target);
+
+/**
+ * Mark that *job* is starting of its execution in context of *hdr* object.
+ *
+ * @param hdr
+ * Headroom object context.
+ * @param job
+ * Job object.
+ * @return
+ * 0 on success
+ * -EINVAL if *hdr* or *job* is NULL or *job* is executing in another headroom
+ * context already,
+ */
+int
+rte_headroom_start_job(struct rte_headroom *hdr, struct rte_headroom_job *job);
+
+/**
+ * Mark that *job* finished its execution. Context in which it was executing will
+ * receive stat update. After this function call *job* object is ready to be
+ * executed in other headroom context.
+ *
+ * @param job
+ * Job object.
+ * @param job_value
+ * Job value. Job should pass in this parameter a value that it try to optimize
+ * for example the number of packets it processed.
+ *
+ * @return
+ * 0 if job's period was not updated (job target equals *job_value*)
+ * 1 if job's period was updated
+ * -EINVAL if job is NULL or job was not started (it have no headroom context).
+ */
+int
+rte_headroom_finish_job(struct rte_headroom_job *job, int64_t job_value);
+
+/**
+ * Set execute period of given job.
+ *
+ * @param job
+ * The job object.
+ * @param period
+ * New period value.
+ * @param saturate
+ * If zero, skip period saturation to min, max range.
+ */
+void
+rte_headroom_job_set_period(struct rte_headroom_job *job, uint64_t period,
+ uint8_t saturate);
+/**
+ * Set minimum execute period of given job. Current period will be checked
+ * against new minimum value.
+ *
+ * @param job
+ * The job object.
+ * @param period
+ * New minimum period value.
+ */
+void
+rte_headroom_set_min_period(struct rte_headroom_job *job, uint64_t period);
+/**
+ * Set maximum execute period of given job. Current period will be checked
+ * against new maximum value.
+ *
+ * @param job
+ * The job object.
+ * @param period
+ * New maximum period value.
+ */
+void
+rte_headroom_set_max_period(struct rte_headroom_job *job, uint64_t period);
+
+/**
+ * Set update period callback that is invoked after job finish.
+ *
+ * If application wants to do more sophisticated calculations than default
+ * it can provide this handler.
+ *
+ * @param job
+ * Job object.
+ * @param update_pedriod_cb
+ * Callback to set. If NULL restore default update function.
+ */
+void
+rte_headroom_set_update_period_function(struct rte_headroom_job *job,
+ rte_headroom_update_fn_t update_period_cb);
+
+/**
+ * Function resets job statistics.
+ *
+ * @param job
+ * Job which statistics will be reset.
+ */
+void
+rte_headroom_reset_job_stats(struct rte_headroom_job *job);
+/**
+ * Function resets headroom statistics.
+ *
+ * @param hdr
+ * Headroom which statistics will be reset.
+ */
+void
+rte_headroom_reset_stats(struct rte_headroom *hdr);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* HEADROOM_H_ */
new file mode 100644
@@ -0,0 +1,19 @@
+DPDK_2.0 {
+ global:
+
+ rte_headroom_init;
+ rte_headroom_start_loop;
+ rte_headroom_finish_loop;
+ rte_headroom_job_init;
+ rte_headroom_set_job_target;
+ rte_headroom_start_job;
+ rte_headroom_finish_job;
+ rte_headroom_job_set_period;
+ rte_headroom_set_min_period;
+ rte_headroom_set_max_period;
+ rte_headroom_set_update_period_function;
+ rte_headroom_reset_job_stats;
+ rte_headroom_reset_stats;
+
+ local: *;
+};