Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/1662/?format=api
{ "id": 1662, "url": "https://patches.dpdk.org/api/patches/1662/?format=api", "web_url": "https://patches.dpdk.org/project/dpdk/patch/1417106021-22842-2-git-send-email-danielx.t.mrzyglod@intel.com/", "project": { "id": 1, "url": "https://patches.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<1417106021-22842-2-git-send-email-danielx.t.mrzyglod@intel.com>", "list_archive_url": "https://inbox.dpdk.org/dev/1417106021-22842-2-git-send-email-danielx.t.mrzyglod@intel.com", "date": "2014-11-27T16:33:40", "name": "[dpdk-dev,v4,1/2] This patch add support of mode 5 to link bonding pmd", "commit_ref": null, "pull_url": null, "state": "accepted", "archived": true, "hash": "e8739af9cfabcffe50d4b6559d42226ded9356e8", "submitter": { "id": 23, "url": "https://patches.dpdk.org/api/people/23/?format=api", "name": "Daniel Mrzyglod", "email": "danielx.t.mrzyglod@intel.com" }, "delegate": null, "mbox": "https://patches.dpdk.org/project/dpdk/patch/1417106021-22842-2-git-send-email-danielx.t.mrzyglod@intel.com/mbox/", "series": [], "comments": "https://patches.dpdk.org/api/patches/1662/comments/", "check": "pending", "checks": "https://patches.dpdk.org/api/patches/1662/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@dpdk.org", "Delivered-To": "patchwork@dpdk.org", "Received": [ "from [92.243.14.124] (localhost [IPv6:::1])\n\tby dpdk.org (Postfix) with ESMTP id 50C50803B;\n\tThu, 27 Nov 2014 17:33:53 +0100 (CET)", "from mga03.intel.com (mga03.intel.com [134.134.136.65])\n\tby dpdk.org (Postfix) with ESMTP id B1FDC803A\n\tfor <dev@dpdk.org>; Thu, 27 Nov 2014 17:33:50 +0100 (CET)", "from orsmga001.jf.intel.com ([10.7.209.18])\n\tby orsmga103.jf.intel.com with ESMTP; 27 Nov 2014 08:30:43 -0800", "from irvmail001.ir.intel.com ([163.33.26.43])\n\tby orsmga001.jf.intel.com with ESMTP; 27 Nov 2014 08:33:49 -0800", "from sivswdev02.ir.intel.com (sivswdev02.ir.intel.com\n\t[10.237.217.46])\n\tby irvmail001.ir.intel.com (8.14.3/8.13.6/MailSET/Hub) with ESMTP id\n\tsARGXmJi013332; Thu, 27 Nov 2014 16:33:48 GMT", "from sivswdev02.ir.intel.com (localhost [127.0.0.1])\n\tby sivswdev02.ir.intel.com with ESMTP id sARGXm1d022898;\n\tThu, 27 Nov 2014 16:33:48 GMT", "(from dtmrzglx@localhost)\n\tby sivswdev02.ir.intel.com with id sARGXme4022894;\n\tThu, 27 Nov 2014 16:33:48 GMT" ], "X-ExtLoop1": "1", "X-IronPort-AV": "E=Sophos;i=\"5.07,470,1413270000\"; d=\"scan'208\";a=\"615089159\"", "From": "Daniel Mrzyglod <danielx.t.mrzyglod@intel.com>", "To": "dev@dpdk.org", "Date": "Thu, 27 Nov 2014 16:33:40 +0000", "Message-Id": "<1417106021-22842-2-git-send-email-danielx.t.mrzyglod@intel.com>", "X-Mailer": "git-send-email 1.7.4.1", "In-Reply-To": "<1417106021-22842-1-git-send-email-danielx.t.mrzyglod@intel.com>", "References": "<1417106021-22842-1-git-send-email-danielx.t.mrzyglod@intel.com>", "MIME-Version": "1.0", "Content-Type": "text/plain; charset=UTF-8", "Content-Transfer-Encoding": "8bit", "Subject": "[dpdk-dev] =?utf-8?q?=5BPATCH_v4_1/2=5D_This_patch_add_support_of?=\n\t=?utf-8?q?_mode_5_to_link_bonding_pmd?=", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.15", "Precedence": "list", "List-Id": "patches and discussions about DPDK <dev.dpdk.org>", "List-Unsubscribe": "<http://dpdk.org/ml/options/dev>,\n\t<mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://dpdk.org/ml/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<http://dpdk.org/ml/listinfo/dev>,\n\t<mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "Add support for mode 5 (Transmit load balancing) into pmd driver\n\nThis patch add support for Adaptive transmit load balancing (mode 5) to the\nlibrte_pmd_bond library. This mode provides an adaptive transmit load \nbalancing. It dynamically changes the transmitting slave, according to the \ncomputed load. \n\nFurther details are described here:\nhttps://www.kernel.org/doc/Documentation/networking/bonding.txt\nIn implementation callback is used for sorting slave order - providing \nstatistics for burst function about slave bandwith usage and sort \ninterfaces due to usage.\n\nDifference in this implementation vs Linux implementation:\n- We Are trying send all pkts – If one interface hasn’t send packets we are \ntrying to send rest of packets by other slaves sorted previously by callback \nfunction.\n\nSome implementation details:\n- Every 100ms is taken obytes statistics from every slave.\n- Every 10 ms the slaves in table are sorted and updated by callback - \nbandwidth and successfully transmitted bytes from previous iteration which \nhappens every 100 ms\n- There is callback function which updates this statistics for transparency and\nfor rather intensive computation involved in this mode.\n\nSigned-off-by: Daniel Mrzyglod <danielx.t.mrzyglod@intel.com>\n---\n lib/librte_pmd_bond/rte_eth_bond.h | 6 ++\n lib/librte_pmd_bond/rte_eth_bond_args.c | 1 +\n lib/librte_pmd_bond/rte_eth_bond_pmd.c | 160 ++++++++++++++++++++++++++++-\n lib/librte_pmd_bond/rte_eth_bond_private.h | 2 +-\n 4 files changed, 166 insertions(+), 3 deletions(-)", "diff": "diff --git a/lib/librte_pmd_bond/rte_eth_bond.h b/lib/librte_pmd_bond/rte_eth_bond.h\nindex 085500b..3831f56 100644\n--- a/lib/librte_pmd_bond/rte_eth_bond.h\n+++ b/lib/librte_pmd_bond/rte_eth_bond.h\n@@ -77,6 +77,12 @@ extern \"C\" {\n * In this mode all transmitted packets will be transmitted on all available\n * active slaves of the bonded. */\n #endif\n+#define BONDING_MODE_ADAPTIVE_TRANSMIT_LOAD_BALANCING\t(5)\n+/**< Adaptive TLB (Mode 5)\n+ * This mode provides an adaptive transmit load balancing. It dynamically\n+ * changes the transmitting slave, according to the computed load. Statistics\n+ * are collected in 100ms intervals and scheduled every 10ms */\n+\n /* Balance Mode Transmit Policies */\n #define BALANCE_XMIT_POLICY_LAYER2\t\t(0)\n /**< Layer 2 (Ethernet MAC) */\ndiff --git a/lib/librte_pmd_bond/rte_eth_bond_args.c b/lib/librte_pmd_bond/rte_eth_bond_args.c\nindex d8ce681..2675cf6 100644\n--- a/lib/librte_pmd_bond/rte_eth_bond_args.c\n+++ b/lib/librte_pmd_bond/rte_eth_bond_args.c\n@@ -173,6 +173,7 @@ bond_ethdev_parse_slave_mode_kvarg(const char *key __rte_unused,\n #ifdef RTE_MBUF_REFCNT\n \tcase BONDING_MODE_BROADCAST:\n #endif\n+\tcase BONDING_MODE_ADAPTIVE_TRANSMIT_LOAD_BALANCING:\n \t\treturn 0;\n \tdefault:\n \t\tRTE_BOND_LOG(ERR, \"Invalid slave mode value (%s) specified\", value);\ndiff --git a/lib/librte_pmd_bond/rte_eth_bond_pmd.c b/lib/librte_pmd_bond/rte_eth_bond_pmd.c\nindex cf2fbab..7a5dae6 100644\n--- a/lib/librte_pmd_bond/rte_eth_bond_pmd.c\n+++ b/lib/librte_pmd_bond/rte_eth_bond_pmd.c\n@@ -30,7 +30,7 @@\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n */\n-\n+#include <stdlib.h>\n #include <rte_mbuf.h>\n #include <rte_malloc.h>\n #include <rte_ethdev.h>\n@@ -41,10 +41,15 @@\n #include <rte_kvargs.h>\n #include <rte_dev.h>\n #include <rte_alarm.h>\n+#include <rte_cycles.h>\n \n #include \"rte_eth_bond.h\"\n #include \"rte_eth_bond_private.h\"\n \n+#define REORDER_PERIOD_MS 10\n+/* Table for statistics in mode 5 TLB */\n+static uint64_t tlb_last_obytets[RTE_MAX_ETHPORTS];\n+\n static uint16_t\n bond_ethdev_rx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)\n {\n@@ -288,6 +293,144 @@ xmit_slave_hash(const struct rte_mbuf *buf, uint8_t slave_count, uint8_t policy)\n \treturn hash % slave_count;\n }\n \n+struct bwg_slave {\n+\tuint64_t bwg_left_int;\n+\tuint64_t bwg_left_remainder;\n+\tuint8_t slave;\n+};\n+\n+static int\n+bandwidth_cmp(const void *a, const void *b)\n+{\n+\tconst struct bwg_slave *bwg_a = a;\n+\tconst struct bwg_slave *bwg_b = b;\n+\tint64_t diff = (int64_t)bwg_b->bwg_left_int - (int64_t)bwg_a->bwg_left_int;\n+\tint64_t diff2 = (int64_t)bwg_b->bwg_left_remainder -\n+\t\t\t(int64_t)bwg_a->bwg_left_remainder;\n+\tif (diff > 0)\n+\t\treturn 1;\n+\telse if (diff < 0)\n+\t\treturn -1;\n+\telse if (diff2 > 0)\n+\t\treturn 1;\n+\telse if (diff2 < 0)\n+\t\treturn -1;\n+\telse\n+\t\treturn 0;\n+}\n+\n+static void\n+bandwidth_left(int port_id, uint64_t load, uint8_t update_idx,\n+\t\tstruct bwg_slave *bwg_slave)\n+{\n+\tstruct rte_eth_link link_status;\n+\n+\trte_eth_link_get(port_id, &link_status);\n+\tuint64_t link_bwg = link_status.link_speed * 1000000ULL / 8;\n+\tif (link_bwg == 0)\n+\t\treturn;\n+\tlink_bwg = (link_bwg * (update_idx+1) * REORDER_PERIOD_MS);\n+\tbwg_slave->bwg_left_int = (link_bwg - 1000*load) / link_bwg;\n+\tbwg_slave->bwg_left_remainder = (link_bwg - 1000*load) % link_bwg;\n+}\n+\n+static void\n+bond_ethdev_update_tlb_slave_cb(void *arg)\n+{\n+\tstruct bond_dev_private *internals = arg;\n+\tstruct rte_eth_stats slave_stats;\n+\tstruct bwg_slave bwg_array[RTE_MAX_ETHPORTS];\n+\tuint8_t slave_count;\n+\tuint64_t tx_bytes;\n+\n+\tuint8_t update_stats = 0;\n+\tuint8_t i, slave_id;\n+\n+\tinternals->slave_update_idx++;\n+\n+\n+\tif (internals->slave_update_idx >= REORDER_PERIOD_MS)\n+\t\tupdate_stats = 1;\n+\n+\tfor (i = 0; i < internals->active_slave_count; i++) {\n+\t\tslave_id = internals->active_slaves[i];\n+\t\trte_eth_stats_get(slave_id, &slave_stats);\n+\t\ttx_bytes = slave_stats.obytes - tlb_last_obytets[slave_id];\n+\t\tbandwidth_left(slave_id, tx_bytes,\n+\t\t\t\tinternals->slave_update_idx, &bwg_array[i]);\n+\t\tbwg_array[i].slave = slave_id;\n+\n+\t\tif (update_stats)\n+\t\t\ttlb_last_obytets[slave_id] = slave_stats.obytes;\n+\t}\n+\n+\tif (update_stats == 1)\n+\t\tinternals->slave_update_idx = 0;\n+\n+\tslave_count = i;\n+\tqsort(bwg_array, slave_count, sizeof(bwg_array[0]), bandwidth_cmp);\n+\tfor (i = 0; i < slave_count; i++)\n+\t\tinternals->active_slaves[i] = bwg_array[i].slave;\n+\n+\trte_eal_alarm_set(REORDER_PERIOD_MS * 1000, bond_ethdev_update_tlb_slave_cb,\n+\t\t\t(struct bond_dev_private *)internals);\n+}\n+\n+static uint16_t\n+bond_ethdev_tx_burst_tlb(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)\n+{\n+\tstruct bond_tx_queue *bd_tx_q = (struct bond_tx_queue *)queue;\n+\tstruct bond_dev_private *internals = bd_tx_q->dev_private;\n+\n+\tstruct rte_eth_dev *primary_port =\n+\t\t\t&rte_eth_devices[internals->primary_port];\n+\tuint16_t num_tx_total = 0;\n+\tuint8_t i, j;\n+\n+\tuint8_t num_of_slaves = internals->active_slave_count;\n+\tuint8_t slaves[RTE_MAX_ETHPORTS];\n+\n+\tstruct ether_hdr *ether_hdr;\n+\tstruct ether_addr primary_slave_addr;\n+\tstruct ether_addr active_slave_addr;\n+\n+\tif (num_of_slaves < 1)\n+\t\treturn num_tx_total;\n+\n+\tmemcpy(slaves, internals->active_slaves,\n+\t\t\t\tsizeof(internals->active_slaves[0]) * num_of_slaves);\n+\n+\n+\tether_addr_copy(primary_port->data->mac_addrs, &primary_slave_addr);\n+\n+\tif (nb_pkts > 3) {\n+\t\tfor (i = 0; i < 3; i++)\n+\t\t\trte_prefetch0(rte_pktmbuf_mtod(bufs[i], void*));\n+\t}\n+\n+\tfor (i = 0; i < num_of_slaves; i++) {\n+\t\tether_addr_copy(&internals->slaves[slaves[i]].persisted_mac_addr,\n+\t\t\t\t&active_slave_addr);\n+\n+\t\tfor (j = num_tx_total; j < nb_pkts; j++) {\n+\t\t\tif (j + 3 < nb_pkts)\n+\t\t\t\trte_prefetch0(rte_pktmbuf_mtod(bufs[j+3], void*));\n+\n+\t\t\tether_hdr = rte_pktmbuf_mtod(bufs[j], struct ether_hdr *);\n+\t\t\tif (is_same_ether_addr(ðer_hdr->s_addr, &primary_slave_addr))\n+\t\t\t\tether_addr_copy(&active_slave_addr, ðer_hdr->s_addr);\n+\t\t}\n+\n+\t\tnum_tx_total += rte_eth_tx_burst(slaves[i], bd_tx_q->queue_id,\n+\t\t\t\tbufs + num_tx_total, nb_pkts - num_tx_total);\n+\n+\t\tif (num_tx_total == nb_pkts)\n+\t\t\tbreak;\n+\t}\n+\n+\treturn num_tx_total;\n+}\n+\n static uint16_t\n bond_ethdev_tx_burst_balance(void *queue, struct rte_mbuf **bufs,\n \t\tuint16_t nb_pkts)\n@@ -500,6 +643,7 @@ mac_address_slaves_update(struct rte_eth_dev *bonded_eth_dev)\n \t\t}\n \t\tbreak;\n \tcase BONDING_MODE_ACTIVE_BACKUP:\n+\tcase BONDING_MODE_ADAPTIVE_TRANSMIT_LOAD_BALANCING:\n \tdefault:\n \t\tfor (i = 0; i < internals->slave_count; i++) {\n \t\t\tif (internals->slaves[i].port_id ==\n@@ -551,6 +695,10 @@ bond_ethdev_mode_set(struct rte_eth_dev *eth_dev, int mode)\n \t\teth_dev->rx_pkt_burst = bond_ethdev_rx_burst;\n \t\tbreak;\n #endif\n+\tcase BONDING_MODE_ADAPTIVE_TRANSMIT_LOAD_BALANCING:\n+\t\teth_dev->tx_pkt_burst = bond_ethdev_tx_burst_tlb;\n+\t\teth_dev->rx_pkt_burst = bond_ethdev_rx_burst_active_backup;\n+\t\tbreak;\n \tdefault:\n \t\treturn -1;\n \t}\n@@ -676,7 +824,7 @@ slave_add(struct bond_dev_private *internals,\n \t}\n \n \tslave_details->link_status_wait_to_complete = 0;\n-\n+\t/* clean tlb_last_obytes when adding port for bonding device */\n \tmemcpy(&(slave_details->persisted_mac_addr), slave_eth_dev->data->mac_addrs,\n \t\t\tsizeof(struct ether_addr));\n }\n@@ -762,6 +910,9 @@ bond_ethdev_start(struct rte_eth_dev *eth_dev)\n \tif (internals->user_defined_primary_port)\n \t\tbond_ethdev_primary_set(internals, internals->primary_port);\n \n+\tif (internals->mode == BONDING_MODE_ADAPTIVE_TRANSMIT_LOAD_BALANCING)\n+\t\tbond_ethdev_update_tlb_slave_cb(internals);\n+\n \treturn 0;\n }\n \n@@ -770,6 +921,9 @@ bond_ethdev_stop(struct rte_eth_dev *eth_dev)\n {\n \tstruct bond_dev_private *internals = eth_dev->data->dev_private;\n \n+\tif (internals->mode == BONDING_MODE_ADAPTIVE_TRANSMIT_LOAD_BALANCING) {\n+\t\trte_eal_alarm_cancel(bond_ethdev_update_tlb_slave_cb, internals);\n+ }\n \tinternals->active_slave_count = 0;\n \tinternals->link_status_polling_enabled = 0;\n \n@@ -1016,6 +1170,7 @@ bond_ethdev_promiscuous_enable(struct rte_eth_dev *eth_dev)\n \t\tbreak;\n \t/* Promiscuous mode is propagated only to primary slave */\n \tcase BONDING_MODE_ACTIVE_BACKUP:\n+\tcase BONDING_MODE_ADAPTIVE_TRANSMIT_LOAD_BALANCING:\n \tdefault:\n \t\trte_eth_promiscuous_enable(internals->current_primary_port);\n \n@@ -1042,6 +1197,7 @@ bond_ethdev_promiscuous_disable(struct rte_eth_dev *dev)\n \t\tbreak;\n \t/* Promiscuous mode is propagated only to primary slave */\n \tcase BONDING_MODE_ACTIVE_BACKUP:\n+\tcase BONDING_MODE_ADAPTIVE_TRANSMIT_LOAD_BALANCING:\n \tdefault:\n \t\trte_eth_promiscuous_disable(internals->current_primary_port);\n \t}\ndiff --git a/lib/librte_pmd_bond/rte_eth_bond_private.h b/lib/librte_pmd_bond/rte_eth_bond_private.h\nindex 6254c84..2a4e129 100644\n--- a/lib/librte_pmd_bond/rte_eth_bond_private.h\n+++ b/lib/librte_pmd_bond/rte_eth_bond_private.h\n@@ -102,7 +102,6 @@ struct bond_slave_details {\n \tuint8_t link_status_poll_enabled;\n \tuint8_t link_status_wait_to_complete;\n \tuint8_t last_link_status;\n-\n \t/**< Port Id of slave eth_dev */\n \tstruct ether_addr persisted_mac_addr;\n };\n@@ -145,6 +144,7 @@ struct bond_dev_private {\n \t/**< Arary of bonded slaves details */\n \n \tstruct rte_kvargs *kvlist;\n+\tuint8_t slave_update_idx;\n };\n \n extern struct eth_dev_ops default_dev_ops;\n", "prefixes": [ "dpdk-dev", "v4", "1/2" ] }