diff --git a/drivers/event/dlb/pf/base/dlb_hw_types.h b/drivers/event/dlb/pf/base/dlb_hw_types.h
new file mode 100644
index 0000000..d56590e
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_hw_types.h
@@ -0,0 +1,360 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_HW_TYPES_H
+#define __DLB_HW_TYPES_H
+
+#include "dlb_user.h"
+#include "dlb_osdep_types.h"
+#include "dlb_osdep_list.h"
+
+#define DLB_MAX_NUM_VFS 16
+#define DLB_MAX_NUM_DOMAINS 32
+#define DLB_MAX_NUM_LDB_QUEUES 128
+#define DLB_MAX_NUM_LDB_PORTS 64
+#define DLB_MAX_NUM_DIR_PORTS 128
+#define DLB_MAX_NUM_LDB_CREDITS 16384
+#define DLB_MAX_NUM_DIR_CREDITS 4096
+#define DLB_MAX_NUM_LDB_CREDIT_POOLS 64
+#define DLB_MAX_NUM_DIR_CREDIT_POOLS 64
+#define DLB_MAX_NUM_HIST_LIST_ENTRIES 5120
+#define DLB_MAX_NUM_AQOS_ENTRIES 2048
+#define DLB_MAX_NUM_TOTAL_OUTSTANDING_COMPLETIONS 4096
+#define DLB_MAX_NUM_QIDS_PER_LDB_CQ 8
+#define DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS 4
+#define DLB_MAX_NUM_SEQUENCE_NUMBER_MODES 6
+#define DLB_QID_PRIORITIES 8
+#define DLB_NUM_ARB_WEIGHTS 8
+#define DLB_MAX_WEIGHT 255
+#define DLB_MAX_PORT_CREDIT_QUANTUM 1023
+#define DLB_MAX_CQ_COMP_CHECK_LOOPS 409600
+#define DLB_MAX_QID_EMPTY_CHECK_LOOPS (32 * 64 * 1024 * (800 / 30))
+#define DLB_HZ 800000000
+
+/* Used for DLB A-stepping workaround for hardware write buffer lock up issue */
+#define DLB_A_STEP_MAX_PORTS 128
+
+#define DLB_PF_DEV_ID 0x270B
+#define DLB_VF_DEV_ID 0x270C
+
+/* Interrupt related macros */
+#define DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS 8
+#define DLB_PF_NUM_CQ_INTERRUPT_VECTORS	 64
+#define DLB_PF_TOTAL_NUM_INTERRUPT_VECTORS \
+	(DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS + \
+	 DLB_PF_NUM_CQ_INTERRUPT_VECTORS)
+#define DLB_PF_NUM_COMPRESSED_MODE_VECTORS \
+	(DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS + 1)
+#define DLB_PF_NUM_PACKED_MODE_VECTORS	 DLB_PF_TOTAL_NUM_INTERRUPT_VECTORS
+#define DLB_PF_COMPRESSED_MODE_CQ_VECTOR_ID DLB_PF_NUM_NON_CQ_INTERRUPT_VECTORS
+
+#define DLB_VF_NUM_NON_CQ_INTERRUPT_VECTORS 1
+#define DLB_VF_NUM_CQ_INTERRUPT_VECTORS 31
+#define DLB_VF_BASE_CQ_VECTOR_ID 0
+#define DLB_VF_LAST_CQ_VECTOR_ID 30
+#define DLB_VF_MBOX_VECTOR_ID 31
+#define DLB_VF_TOTAL_NUM_INTERRUPT_VECTORS \
+	(DLB_VF_NUM_NON_CQ_INTERRUPT_VECTORS + \
+	 DLB_VF_NUM_CQ_INTERRUPT_VECTORS)
+
+#define DLB_PF_NUM_ALARM_INTERRUPT_VECTORS 4
+/* DLB ALARM interrupts */
+#define DLB_INT_ALARM 0
+/* VF to PF Mailbox Service Request */
+#define DLB_INT_VF_TO_PF_MBOX 1
+/* HCW Ingress Errors */
+#define DLB_INT_INGRESS_ERROR 3
+
+#define DLB_ALARM_HW_SOURCE_SYS 0
+#define DLB_ALARM_HW_SOURCE_DLB 1
+
+#define DLB_ALARM_HW_UNIT_CHP 1
+#define DLB_ALARM_HW_UNIT_LSP 3
+
+#define DLB_ALARM_HW_CHP_AID_OUT_OF_CREDITS 6
+#define DLB_ALARM_HW_CHP_AID_ILLEGAL_ENQ 7
+#define DLB_ALARM_HW_LSP_AID_EXCESS_TOKEN_POPS 15
+#define DLB_ALARM_SYS_AID_ILLEGAL_HCW 0
+#define DLB_ALARM_SYS_AID_ILLEGAL_QID 3
+#define DLB_ALARM_SYS_AID_DISABLED_QID 4
+#define DLB_ALARM_SYS_AID_ILLEGAL_CQID 6
+
+/* Hardware-defined base addresses */
+#define DLB_LDB_PP_BASE 0x2100000
+#define DLB_LDB_PP_STRIDE 0x1000
+#define DLB_LDB_PP_BOUND \
+	(DLB_LDB_PP_BASE + DLB_LDB_PP_STRIDE * DLB_MAX_NUM_LDB_PORTS)
+#define DLB_DIR_PP_BASE 0x2000000
+#define DLB_DIR_PP_STRIDE 0x1000
+#define DLB_DIR_PP_BOUND \
+	(DLB_DIR_PP_BASE + DLB_DIR_PP_STRIDE * DLB_MAX_NUM_DIR_PORTS)
+
+struct dlb_resource_id {
+	u32 phys_id;
+	u32 virt_id;
+	u8 vf_owned;
+	u8 vf_id;
+};
+
+struct dlb_freelist {
+	u32 base;
+	u32 bound;
+	u32 offset;
+};
+
+static inline u32 dlb_freelist_count(struct dlb_freelist *list)
+{
+	return (list->bound - list->base) - list->offset;
+}
+
+struct dlb_hcw {
+	u64 data;
+	/* Word 3 */
+	u16 opaque;
+	u8 qid;
+	u8 sched_type:2;
+	u8 priority:3;
+	u8 msg_type:3;
+	/* Word 4 */
+	u16 lock_id;
+	u8 meas_lat:1;
+	u8 rsvd1:2;
+	u8 no_dec:1;
+	u8 cmp_id:4;
+	u8 cq_token:1;
+	u8 qe_comp:1;
+	u8 qe_frag:1;
+	u8 qe_valid:1;
+	u8 int_arm:1;
+	u8 error:1;
+	u8 rsvd:2;
+};
+
+struct dlb_ldb_queue {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	struct dlb_resource_id id;
+	struct dlb_resource_id domain_id;
+	u32 num_qid_inflights;
+	struct dlb_freelist aqed_freelist;
+	u8 sn_cfg_valid;
+	u32 sn_group;
+	u32 sn_slot;
+	u32 num_mappings;
+	u8 num_pending_additions;
+	u8 owned;
+	u8 configured;
+};
+
+/* Directed ports and queues are paired by nature, so the driver tracks them
+ * with a single data structure.
+ */
+struct dlb_dir_pq_pair {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	struct dlb_resource_id id;
+	struct dlb_resource_id domain_id;
+	u8 ldb_pool_used;
+	u8 dir_pool_used;
+	u8 queue_configured;
+	u8 port_configured;
+	u8 owned;
+	u8 enabled;
+	u32 ref_cnt;
+};
+
+enum dlb_qid_map_state {
+	/* The slot doesn't contain a valid queue mapping */
+	DLB_QUEUE_UNMAPPED,
+	/* The slot contains a valid queue mapping */
+	DLB_QUEUE_MAPPED,
+	/* The driver is mapping a queue into this slot */
+	DLB_QUEUE_MAP_IN_PROGRESS,
+	/* The driver is unmapping a queue from this slot */
+	DLB_QUEUE_UNMAP_IN_PROGRESS,
+	/* The driver is unmapping a queue from this slot, and once complete
+	 * will replace it with another mapping.
+	 */
+	DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP,
+};
+
+struct dlb_ldb_port_qid_map {
+	u16 qid;
+	u8 priority;
+	u16 pending_qid;
+	u8 pending_priority;
+	enum dlb_qid_map_state state;
+};
+
+struct dlb_ldb_port {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	struct dlb_resource_id id;
+	struct dlb_resource_id domain_id;
+	u8 ldb_pool_used;
+	u8 dir_pool_used;
+	u8 init_tkn_cnt;
+	u32 hist_list_entry_base;
+	u32 hist_list_entry_limit;
+	/* The qid_map represents the hardware QID mapping state. */
+	struct dlb_ldb_port_qid_map qid_map[DLB_MAX_NUM_QIDS_PER_LDB_CQ];
+	u32 ref_cnt;
+	u8 num_pending_removals;
+	u8 num_mappings;
+	u8 owned;
+	u8 enabled;
+	u8 configured;
+};
+
+struct dlb_credit_pool {
+	struct dlb_list_entry domain_list;
+	struct dlb_list_entry func_list;
+	struct dlb_resource_id id;
+	struct dlb_resource_id domain_id;
+	u32 total_credits;
+	u32 avail_credits;
+	u8 owned;
+	u8 configured;
+};
+
+struct dlb_sn_group {
+	u32 mode;
+	u32 sequence_numbers_per_queue;
+	u32 slot_use_bitmap;
+	u32 id;
+};
+
+static inline bool dlb_sn_group_full(struct dlb_sn_group *group)
+{
+	u32 mask[6] = {
+		0xffffffff,  /* 32 SNs per queue */
+		0x0000ffff,  /* 64 SNs per queue */
+		0x000000ff,  /* 128 SNs per queue */
+		0x0000000f,  /* 256 SNs per queue */
+		0x00000003,  /* 512 SNs per queue */
+		0x00000001}; /* 1024 SNs per queue */
+
+	return group->slot_use_bitmap == mask[group->mode];
+}
+
+static inline int dlb_sn_group_alloc_slot(struct dlb_sn_group *group)
+{
+	int bound[6] = {32, 16, 8, 4, 2, 1};
+	int i;
+
+	for (i = 0; i < bound[group->mode]; i++) {
+		if (!(group->slot_use_bitmap & (1 << i))) {
+			group->slot_use_bitmap |= 1 << i;
+			return i;
+		}
+	}
+
+	return -1;
+}
+
+static inline void dlb_sn_group_free_slot(struct dlb_sn_group *group, int slot)
+{
+	group->slot_use_bitmap &= ~(1 << slot);
+}
+
+static inline int dlb_sn_group_used_slots(struct dlb_sn_group *group)
+{
+	int i, cnt = 0;
+
+	for (i = 0; i < 32; i++)
+		cnt += !!(group->slot_use_bitmap & (1 << i));
+
+	return cnt;
+}
+
+struct dlb_domain {
+	struct dlb_function_resources *parent_func;
+	struct dlb_list_entry func_list;
+	struct dlb_list_head used_ldb_queues;
+	struct dlb_list_head used_ldb_ports;
+	struct dlb_list_head used_dir_pq_pairs;
+	struct dlb_list_head used_ldb_credit_pools;
+	struct dlb_list_head used_dir_credit_pools;
+	struct dlb_list_head avail_ldb_queues;
+	struct dlb_list_head avail_ldb_ports;
+	struct dlb_list_head avail_dir_pq_pairs;
+	struct dlb_list_head avail_ldb_credit_pools;
+	struct dlb_list_head avail_dir_credit_pools;
+	u32 total_hist_list_entries;
+	u32 avail_hist_list_entries;
+	u32 hist_list_entry_base;
+	u32 hist_list_entry_offset;
+	struct dlb_freelist qed_freelist;
+	struct dlb_freelist dqed_freelist;
+	struct dlb_freelist aqed_freelist;
+	struct dlb_resource_id id;
+	int num_pending_removals;
+	int num_pending_additions;
+	u8 configured;
+	u8 started;
+};
+
+struct dlb_bitmap;
+
+struct dlb_function_resources {
+	u32 num_avail_domains;
+	struct dlb_list_head avail_domains;
+	struct dlb_list_head used_domains;
+	u32 num_avail_ldb_queues;
+	struct dlb_list_head avail_ldb_queues;
+	u32 num_avail_ldb_ports;
+	struct dlb_list_head avail_ldb_ports;
+	u32 num_avail_dir_pq_pairs;
+	struct dlb_list_head avail_dir_pq_pairs;
+	struct dlb_bitmap *avail_hist_list_entries;
+	struct dlb_bitmap *avail_qed_freelist_entries;
+	struct dlb_bitmap *avail_dqed_freelist_entries;
+	struct dlb_bitmap *avail_aqed_freelist_entries;
+	u32 num_avail_ldb_credit_pools;
+	struct dlb_list_head avail_ldb_credit_pools;
+	u32 num_avail_dir_credit_pools;
+	struct dlb_list_head avail_dir_credit_pools;
+	u32 num_enabled_ldb_ports; /* (PF only) */
+	u8 locked; /* (VF only) */
+};
+
+/* After initialization, each resource in dlb_hw_resources is located in one of
+ * the following lists:
+ * -- The PF's available resources list. These are unconfigured resources owned
+ *	by the PF and not allocated to a DLB scheduling domain.
+ * -- A VF's available resources list. These are VF-owned unconfigured
+ *	resources not allocated to a DLB scheduling domain.
+ * -- A domain's available resources list. These are domain-owned unconfigured
+ *	resources.
+ * -- A domain's used resources list. These are are domain-owned configured
+ *	resources.
+ *
+ * A resource moves to a new list when a VF or domain is created or destroyed,
+ * or when the resource is configured.
+ */
+struct dlb_hw_resources {
+	struct dlb_ldb_queue ldb_queues[DLB_MAX_NUM_LDB_QUEUES];
+	struct dlb_ldb_port ldb_ports[DLB_MAX_NUM_LDB_PORTS];
+	struct dlb_dir_pq_pair dir_pq_pairs[DLB_MAX_NUM_DIR_PORTS];
+	struct dlb_credit_pool ldb_credit_pools[DLB_MAX_NUM_LDB_CREDIT_POOLS];
+	struct dlb_credit_pool dir_credit_pools[DLB_MAX_NUM_DIR_CREDIT_POOLS];
+	struct dlb_sn_group sn_groups[DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS];
+};
+
+struct dlb_hw {
+	/* BAR 0 address */
+	void  *csr_kva;
+	unsigned long csr_phys_addr;
+	/* BAR 2 address */
+	void  *func_kva;
+	unsigned long func_phys_addr;
+
+	/* Resource tracking */
+	struct dlb_hw_resources rsrcs;
+	struct dlb_function_resources pf;
+	struct dlb_function_resources vf[DLB_MAX_NUM_VFS];
+	struct dlb_domain domains[DLB_MAX_NUM_DOMAINS];
+};
+
+#endif /* __DLB_HW_TYPES_H */
diff --git a/drivers/event/dlb/pf/base/dlb_mbox.h b/drivers/event/dlb/pf/base/dlb_mbox.h
new file mode 100644
index 0000000..e195526
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_mbox.h
@@ -0,0 +1,645 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_BASE_DLB_MBOX_H
+#define __DLB_BASE_DLB_MBOX_H
+
+#include "dlb_regs.h"
+#include "dlb_osdep_types.h"
+
+#define DLB_MBOX_INTERFACE_VERSION 1
+
+/* The PF uses its PF->VF mailbox to send responses to VF requests, as well as
+ * to send requests of its own (e.g. notifying a VF of an impending FLR).
+ * To avoid communication race conditions, e.g. the PF sends a response and then
+ * sends a request before the VF reads the response, the PF->VF mailbox is
+ * divided into two sections:
+ * - Bytes 0-47: PF responses
+ * - Bytes 48-63: PF requests
+ *
+ * Partitioning the PF->VF mailbox allows responses and requests to occupy the
+ * mailbox simultaneously.
+ */
+#define DLB_PF2VF_RESP_BYTES 48
+#define DLB_PF2VF_RESP_BASE 0
+#define DLB_PF2VF_RESP_BASE_WORD (DLB_PF2VF_RESP_BASE / 4)
+
+#define DLB_PF2VF_REQ_BYTES \
+	(DLB_FUNC_PF_PF2VF_MAILBOX_BYTES - DLB_PF2VF_RESP_BYTES)
+#define DLB_PF2VF_REQ_BASE DLB_PF2VF_RESP_BYTES
+#define DLB_PF2VF_REQ_BASE_WORD (DLB_PF2VF_REQ_BASE / 4)
+
+/* Similarly, the VF->PF mailbox is divided into two sections:
+ * - Bytes 0-239: VF requests
+ * - Bytes 240-255: VF responses
+ */
+#define DLB_VF2PF_REQ_BYTES 240
+#define DLB_VF2PF_REQ_BASE 0
+#define DLB_VF2PF_REQ_BASE_WORD (DLB_VF2PF_REQ_BASE / 4)
+
+#define DLB_VF2PF_RESP_BYTES \
+	(DLB_FUNC_VF_VF2PF_MAILBOX_BYTES - DLB_VF2PF_REQ_BYTES)
+#define DLB_VF2PF_RESP_BASE DLB_VF2PF_REQ_BYTES
+#define DLB_VF2PF_RESP_BASE_WORD (DLB_VF2PF_RESP_BASE / 4)
+
+/* VF-initiated commands */
+enum dlb_mbox_cmd_type {
+	DLB_MBOX_CMD_REGISTER,
+	DLB_MBOX_CMD_UNREGISTER,
+	DLB_MBOX_CMD_GET_NUM_RESOURCES,
+	DLB_MBOX_CMD_CREATE_SCHED_DOMAIN,
+	DLB_MBOX_CMD_RESET_SCHED_DOMAIN,
+	DLB_MBOX_CMD_CREATE_LDB_POOL,
+	DLB_MBOX_CMD_CREATE_DIR_POOL,
+	DLB_MBOX_CMD_CREATE_LDB_QUEUE,
+	DLB_MBOX_CMD_CREATE_DIR_QUEUE,
+	DLB_MBOX_CMD_CREATE_LDB_PORT,
+	DLB_MBOX_CMD_CREATE_DIR_PORT,
+	DLB_MBOX_CMD_ENABLE_LDB_PORT,
+	DLB_MBOX_CMD_DISABLE_LDB_PORT,
+	DLB_MBOX_CMD_ENABLE_DIR_PORT,
+	DLB_MBOX_CMD_DISABLE_DIR_PORT,
+	DLB_MBOX_CMD_LDB_PORT_OWNED_BY_DOMAIN,
+	DLB_MBOX_CMD_DIR_PORT_OWNED_BY_DOMAIN,
+	DLB_MBOX_CMD_MAP_QID,
+	DLB_MBOX_CMD_UNMAP_QID,
+	DLB_MBOX_CMD_START_DOMAIN,
+	DLB_MBOX_CMD_ENABLE_LDB_PORT_INTR,
+	DLB_MBOX_CMD_ENABLE_DIR_PORT_INTR,
+	DLB_MBOX_CMD_ARM_CQ_INTR,
+	DLB_MBOX_CMD_GET_NUM_USED_RESOURCES,
+	DLB_MBOX_CMD_INIT_CQ_SCHED_COUNT,
+	DLB_MBOX_CMD_COLLECT_CQ_SCHED_COUNT,
+	DLB_MBOX_CMD_ACK_VF_FLR_DONE,
+	DLB_MBOX_CMD_GET_SN_ALLOCATION,
+	DLB_MBOX_CMD_GET_LDB_QUEUE_DEPTH,
+	DLB_MBOX_CMD_GET_DIR_QUEUE_DEPTH,
+	DLB_MBOX_CMD_PENDING_PORT_UNMAPS,
+	DLB_MBOX_CMD_QUERY_CQ_POLL_MODE,
+	DLB_MBOX_CMD_GET_SN_OCCUPANCY,
+
+	/* NUM_QE_CMD_TYPES must be last */
+	NUM_DLB_MBOX_CMD_TYPES,
+};
+
+static const char dlb_mbox_cmd_type_strings[][128] = {
+	"DLB_MBOX_CMD_REGISTER",
+	"DLB_MBOX_CMD_UNREGISTER",
+	"DLB_MBOX_CMD_GET_NUM_RESOURCES",
+	"DLB_MBOX_CMD_CREATE_SCHED_DOMAIN",
+	"DLB_MBOX_CMD_RESET_SCHED_DOMAIN",
+	"DLB_MBOX_CMD_CREATE_LDB_POOL",
+	"DLB_MBOX_CMD_CREATE_DIR_POOL",
+	"DLB_MBOX_CMD_CREATE_LDB_QUEUE",
+	"DLB_MBOX_CMD_CREATE_DIR_QUEUE",
+	"DLB_MBOX_CMD_CREATE_LDB_PORT",
+	"DLB_MBOX_CMD_CREATE_DIR_PORT",
+	"DLB_MBOX_CMD_ENABLE_LDB_PORT",
+	"DLB_MBOX_CMD_DISABLE_LDB_PORT",
+	"DLB_MBOX_CMD_ENABLE_DIR_PORT",
+	"DLB_MBOX_CMD_DISABLE_DIR_PORT",
+	"DLB_MBOX_CMD_LDB_PORT_OWNED_BY_DOMAIN",
+	"DLB_MBOX_CMD_DIR_PORT_OWNED_BY_DOMAIN",
+	"DLB_MBOX_CMD_MAP_QID",
+	"DLB_MBOX_CMD_UNMAP_QID",
+	"DLB_MBOX_CMD_START_DOMAIN",
+	"DLB_MBOX_CMD_ENABLE_LDB_PORT_INTR",
+	"DLB_MBOX_CMD_ENABLE_DIR_PORT_INTR",
+	"DLB_MBOX_CMD_ARM_CQ_INTR",
+	"DLB_MBOX_CMD_GET_NUM_USED_RESOURCES",
+	"DLB_MBOX_CMD_INIT_CQ_SCHED_COUNT",
+	"DLB_MBOX_CMD_COLLECT_CQ_SCHED_COUNT",
+	"DLB_MBOX_CMD_ACK_VF_FLR_DONE",
+	"DLB_MBOX_CMD_GET_SN_ALLOCATION",
+	"DLB_MBOX_CMD_GET_LDB_QUEUE_DEPTH",
+	"DLB_MBOX_CMD_GET_DIR_QUEUE_DEPTH",
+	"DLB_MBOX_CMD_PENDING_PORT_UNMAPS",
+	"DLB_MBOX_CMD_QUERY_CQ_POLL_MODE",
+	"DLB_MBOX_CMD_GET_SN_OCCUPANCY",
+};
+
+/* PF-initiated commands */
+enum dlb_mbox_vf_cmd_type {
+	DLB_MBOX_VF_CMD_DOMAIN_ALERT,
+	DLB_MBOX_VF_CMD_NOTIFICATION,
+	DLB_MBOX_VF_CMD_IN_USE,
+
+	/* NUM_DLB_MBOX_VF_CMD_TYPES must be last */
+	NUM_DLB_MBOX_VF_CMD_TYPES,
+};
+
+static const char dlb_mbox_vf_cmd_type_strings[][128] = {
+	"DLB_MBOX_VF_CMD_DOMAIN_ALERT",
+	"DLB_MBOX_VF_CMD_NOTIFICATION",
+	"DLB_MBOX_VF_CMD_IN_USE",
+};
+
+#define DLB_MBOX_CMD_TYPE(hdr) \
+	(((struct dlb_mbox_req_hdr *)hdr)->type)
+#define DLB_MBOX_CMD_STRING(hdr) \
+	dlb_mbox_cmd_type_strings[DLB_MBOX_CMD_TYPE(hdr)]
+
+enum dlb_mbox_status_type {
+	DLB_MBOX_ST_SUCCESS,
+	DLB_MBOX_ST_INVALID_CMD_TYPE,
+	DLB_MBOX_ST_VERSION_MISMATCH,
+	DLB_MBOX_ST_EXPECTED_PHASE_ONE,
+	DLB_MBOX_ST_EXPECTED_PHASE_TWO,
+	DLB_MBOX_ST_INVALID_OWNER_VF,
+};
+
+static const char dlb_mbox_status_type_strings[][128] = {
+	"DLB_MBOX_ST_SUCCESS",
+	"DLB_MBOX_ST_INVALID_CMD_TYPE",
+	"DLB_MBOX_ST_VERSION_MISMATCH",
+	"DLB_MBOX_ST_EXPECTED_PHASE_ONE",
+	"DLB_MBOX_ST_EXPECTED_PHASE_TWO",
+	"DLB_MBOX_ST_INVALID_OWNER_VF",
+};
+
+#define DLB_MBOX_ST_TYPE(hdr) \
+	(((struct dlb_mbox_resp_hdr *)hdr)->status)
+#define DLB_MBOX_ST_STRING(hdr) \
+	dlb_mbox_status_type_strings[DLB_MBOX_ST_TYPE(hdr)]
+
+/* This structure is always the first field in a request structure */
+struct dlb_mbox_req_hdr {
+	u32 type;
+};
+
+/* This structure is always the first field in a response structure */
+struct dlb_mbox_resp_hdr {
+	u32 status;
+};
+
+struct dlb_mbox_register_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u16 min_interface_version;
+	u16 max_interface_version;
+};
+
+struct dlb_mbox_register_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 interface_version;
+	u8 pf_id;
+	u8 vf_id;
+	u8 is_auxiliary_vf;
+	u8 primary_vf_id;
+	u32 padding;
+};
+
+struct dlb_mbox_unregister_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 padding;
+};
+
+struct dlb_mbox_unregister_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 padding;
+};
+
+struct dlb_mbox_get_num_resources_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 padding;
+};
+
+struct dlb_mbox_get_num_resources_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u16 num_sched_domains;
+	u16 num_ldb_queues;
+	u16 num_ldb_ports;
+	u16 num_dir_ports;
+	u16 padding0;
+	u8 num_ldb_credit_pools;
+	u8 num_dir_credit_pools;
+	u32 num_atomic_inflights;
+	u32 max_contiguous_atomic_inflights;
+	u32 num_hist_list_entries;
+	u32 max_contiguous_hist_list_entries;
+	u16 num_ldb_credits;
+	u16 max_contiguous_ldb_credits;
+	u16 num_dir_credits;
+	u16 max_contiguous_dir_credits;
+	u32 padding1;
+};
+
+struct dlb_mbox_create_sched_domain_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 num_ldb_queues;
+	u32 num_ldb_ports;
+	u32 num_dir_ports;
+	u32 num_atomic_inflights;
+	u32 num_hist_list_entries;
+	u32 num_ldb_credits;
+	u32 num_dir_credits;
+	u32 num_ldb_credit_pools;
+	u32 num_dir_credit_pools;
+};
+
+struct dlb_mbox_create_sched_domain_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 id;
+};
+
+struct dlb_mbox_reset_sched_domain_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 id;
+};
+
+struct dlb_mbox_reset_sched_domain_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+};
+
+struct dlb_mbox_create_credit_pool_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+	u32 num_credits;
+	u32 padding;
+};
+
+struct dlb_mbox_create_credit_pool_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 id;
+};
+
+struct dlb_mbox_create_ldb_queue_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+	u32 num_sequence_numbers;
+	u32 num_qid_inflights;
+	u32 num_atomic_inflights;
+	u32 padding;
+};
+
+struct dlb_mbox_create_ldb_queue_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 id;
+};
+
+struct dlb_mbox_create_dir_queue_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+	u32 port_id;
+	u32 padding0;
+};
+
+struct dlb_mbox_create_dir_queue_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 id;
+};
+
+struct dlb_mbox_create_ldb_port_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+	u32 ldb_credit_pool_id;
+	u32 dir_credit_pool_id;
+	u64 pop_count_address;
+	u16 ldb_credit_high_watermark;
+	u16 ldb_credit_low_watermark;
+	u16 ldb_credit_quantum;
+	u16 dir_credit_high_watermark;
+	u16 dir_credit_low_watermark;
+	u16 dir_credit_quantum;
+	u32 padding0;
+	u16 cq_depth;
+	u16 cq_history_list_size;
+	u32 padding1;
+	u64 cq_base_address;
+	u64 nq_base_address;
+	u32 nq_size;
+	u32 padding2;
+};
+
+struct dlb_mbox_create_ldb_port_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 id;
+};
+
+struct dlb_mbox_create_dir_port_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+	u32 ldb_credit_pool_id;
+	u32 dir_credit_pool_id;
+	u64 pop_count_address;
+	u16 ldb_credit_high_watermark;
+	u16 ldb_credit_low_watermark;
+	u16 ldb_credit_quantum;
+	u16 dir_credit_high_watermark;
+	u16 dir_credit_low_watermark;
+	u16 dir_credit_quantum;
+	u16 cq_depth;
+	u16 padding0;
+	u64 cq_base_address;
+	s32 queue_id;
+	u32 padding1;
+};
+
+struct dlb_mbox_create_dir_port_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 id;
+};
+
+struct dlb_mbox_enable_ldb_port_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+	u32 port_id;
+	u32 padding;
+};
+
+struct dlb_mbox_enable_ldb_port_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 padding;
+};
+
+struct dlb_mbox_disable_ldb_port_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+	u32 port_id;
+	u32 padding;
+};
+
+struct dlb_mbox_disable_ldb_port_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 padding;
+};
+
+struct dlb_mbox_enable_dir_port_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+	u32 port_id;
+	u32 padding;
+};
+
+struct dlb_mbox_enable_dir_port_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 padding;
+};
+
+struct dlb_mbox_disable_dir_port_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+	u32 port_id;
+	u32 padding;
+};
+
+struct dlb_mbox_disable_dir_port_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 padding;
+};
+
+struct dlb_mbox_ldb_port_owned_by_domain_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+	u32 port_id;
+	u32 padding;
+};
+
+struct dlb_mbox_ldb_port_owned_by_domain_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	s32 owned;
+};
+
+struct dlb_mbox_dir_port_owned_by_domain_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+	u32 port_id;
+	u32 padding;
+};
+
+struct dlb_mbox_dir_port_owned_by_domain_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	s32 owned;
+};
+
+struct dlb_mbox_map_qid_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+	u32 port_id;
+	u32 qid;
+	u32 priority;
+	u32 padding0;
+};
+
+struct dlb_mbox_map_qid_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 id;
+};
+
+struct dlb_mbox_unmap_qid_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+	u32 port_id;
+	u32 qid;
+};
+
+struct dlb_mbox_unmap_qid_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 padding;
+};
+
+struct dlb_mbox_start_domain_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+};
+
+struct dlb_mbox_start_domain_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 padding;
+};
+
+struct dlb_mbox_enable_ldb_port_intr_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u16 port_id;
+	u16 thresh;
+	u16 vector;
+	u16 owner_vf;
+	u16 reserved[2];
+};
+
+struct dlb_mbox_enable_ldb_port_intr_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 padding0;
+};
+
+struct dlb_mbox_enable_dir_port_intr_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u16 port_id;
+	u16 thresh;
+	u16 vector;
+	u16 owner_vf;
+	u16 reserved[2];
+};
+
+struct dlb_mbox_enable_dir_port_intr_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 padding0;
+};
+
+struct dlb_mbox_arm_cq_intr_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+	u32 port_id;
+	u32 is_ldb;
+};
+
+struct dlb_mbox_arm_cq_intr_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 padding0;
+};
+
+/* The alert_id and aux_alert_data follows the format of the alerts defined in
+ * dlb_types.h. The alert id contains an enum dlb_domain_alert_id value, and
+ * the aux_alert_data value varies depending on the alert.
+ */
+struct dlb_mbox_vf_alert_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+	u32 alert_id;
+	u32 aux_alert_data;
+};
+
+enum dlb_mbox_vf_notification_type {
+	DLB_MBOX_VF_NOTIFICATION_PRE_RESET,
+	DLB_MBOX_VF_NOTIFICATION_POST_RESET,
+
+	/* NUM_DLB_MBOX_VF_NOTIFICATION_TYPES must be last */
+	NUM_DLB_MBOX_VF_NOTIFICATION_TYPES,
+};
+
+struct dlb_mbox_vf_notification_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 notification;
+};
+
+struct dlb_mbox_vf_in_use_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 padding;
+};
+
+struct dlb_mbox_vf_in_use_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 in_use;
+};
+
+struct dlb_mbox_ack_vf_flr_done_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 padding;
+};
+
+struct dlb_mbox_ack_vf_flr_done_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 padding;
+};
+
+struct dlb_mbox_get_sn_allocation_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 group_id;
+};
+
+struct dlb_mbox_get_sn_allocation_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 num;
+};
+
+struct dlb_mbox_get_ldb_queue_depth_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+	u32 queue_id;
+	u32 padding;
+};
+
+struct dlb_mbox_get_ldb_queue_depth_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 depth;
+};
+
+struct dlb_mbox_get_dir_queue_depth_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+	u32 queue_id;
+	u32 padding;
+};
+
+struct dlb_mbox_get_dir_queue_depth_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 depth;
+};
+
+struct dlb_mbox_pending_port_unmaps_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 domain_id;
+	u32 port_id;
+	u32 padding;
+};
+
+struct dlb_mbox_pending_port_unmaps_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 num;
+};
+
+struct dlb_mbox_query_cq_poll_mode_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 padding;
+};
+
+struct dlb_mbox_query_cq_poll_mode_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 error_code;
+	u32 status;
+	u32 mode;
+};
+
+struct dlb_mbox_get_sn_occupancy_cmd_req {
+	struct dlb_mbox_req_hdr hdr;
+	u32 group_id;
+};
+
+struct dlb_mbox_get_sn_occupancy_cmd_resp {
+	struct dlb_mbox_resp_hdr hdr;
+	u32 num;
+};
+
+#endif /* __DLB_BASE_DLB_MBOX_H */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep.h b/drivers/event/dlb/pf/base/dlb_osdep.h
new file mode 100644
index 0000000..9286f3d
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep.h
@@ -0,0 +1,348 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_H__
+#define __DLB_OSDEP_H__
+
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+#include <pthread.h>
+#include <rte_string_fns.h>
+#include <rte_cycles.h>
+#include <rte_io.h>
+#include <rte_log.h>
+#include <rte_spinlock.h>
+#include "../dlb_main.h"
+#include "dlb_resource.h"
+#include "../../dlb_user.h"
+
+
+#define DLB_PCI_REG_READ(reg)        rte_read32((void *)reg)
+#define DLB_PCI_REG_WRITE(reg, val)   rte_write32(val, (void *)reg)
+
+#define DLB_CSR_REG_ADDR(a, reg) ((void *)((uintptr_t)(a)->csr_kva + (reg)))
+#define DLB_CSR_RD(hw, reg) \
+	DLB_PCI_REG_READ(DLB_CSR_REG_ADDR((hw), (reg)))
+#define DLB_CSR_WR(hw, reg, val) \
+	DLB_PCI_REG_WRITE(DLB_CSR_REG_ADDR((hw), (reg)), (val))
+
+#define DLB_FUNC_REG_ADDR(a, reg) ((void *)((uintptr_t)(a)->func_kva + (reg)))
+#define DLB_FUNC_RD(hw, reg) \
+	DLB_PCI_REG_READ(DLB_FUNC_REG_ADDR((hw), (reg)))
+#define DLB_FUNC_WR(hw, reg, val) \
+	DLB_PCI_REG_WRITE(DLB_FUNC_REG_ADDR((hw), (reg)), (val))
+
+#define READ_ONCE(x) (x)
+#define WRITE_ONCE(x, y) ((x) = (y))
+
+#define OS_READ_ONCE(x) READ_ONCE(x)
+#define OS_WRITE_ONCE(x, y) WRITE_ONCE(x, y)
+
+
+extern unsigned int dlb_unregister_timeout_s;
+/**
+ * os_queue_unregister_timeout_s() - timeout (in seconds) to wait for queue
+ *                                   unregister acknowledgments.
+ */
+static inline unsigned int os_queue_unregister_timeout_s(void)
+{
+	return dlb_unregister_timeout_s;
+}
+
+static inline size_t os_strlcpy(char *dst, const char *src, size_t sz)
+{
+	return rte_strlcpy(dst, src, sz);
+}
+
+/**
+ * os_udelay() - busy-wait for a number of microseconds
+ * @usecs: delay duration.
+ */
+static inline void os_udelay(int usecs)
+{
+	rte_delay_us(usecs);
+}
+
+/**
+ * os_msleep() - sleep for a number of milliseconds
+ * @usecs: delay duration.
+ */
+
+static inline void os_msleep(int msecs)
+{
+	rte_delay_ms(msecs);
+}
+
+/**
+ * os_curtime_s() - get the current time (in seconds)
+ * @usecs: delay duration.
+ */
+static inline unsigned long os_curtime_s(void)
+{
+	struct timespec tv;
+
+	clock_gettime(CLOCK_MONOTONIC, &tv);
+
+	return (unsigned long)tv.tv_sec;
+}
+
+#define DLB_PP_BASE(__is_ldb) ((__is_ldb) ? DLB_LDB_PP_BASE : DLB_DIR_PP_BASE)
+/**
+ * os_map_producer_port() - map a producer port into the caller's address space
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: port ID
+ * @is_ldb: true for load-balanced port, false for a directed port
+ *
+ * This function maps the requested producer port memory into the caller's
+ * address space.
+ *
+ * Return:
+ * Returns the base address at which the PP memory was mapped, else NULL.
+ */
+static inline void *os_map_producer_port(struct dlb_hw *hw,
+					 u8 port_id,
+					 bool is_ldb)
+{
+	uint64_t addr;
+	uint64_t pp_dma_base;
+
+
+	pp_dma_base = (uintptr_t)hw->func_kva + DLB_PP_BASE(is_ldb);
+	addr = (pp_dma_base + (PAGE_SIZE * port_id));
+
+	return (void *)(uintptr_t)addr;
+
+}
+/**
+ * os_unmap_producer_port() - unmap a producer port
+ * @addr: mapped producer port address
+ *
+ * This function undoes os_map_producer_port() by unmapping the producer port
+ * memory from the caller's address space.
+ *
+ * Return:
+ * Returns the base address at which the PP memory was mapped, else NULL.
+ */
+
+/* PFPMD - Nothing to do here, since memory was not actually mapped by us */
+static inline void os_unmap_producer_port(struct dlb_hw *hw, void *addr)
+{
+	RTE_SET_USED(hw);
+	RTE_SET_USED(addr);
+}
+/**
+ * os_enqueue_four_hcws() - enqueue four HCWs to DLB
+ * @hw: dlb_hw handle for a particular device.
+ * @hcw: pointer to the 64B-aligned contiguous HCW memory
+ * @addr: producer port address
+ */
+static inline void os_enqueue_four_hcws(struct dlb_hw *hw,
+					struct dlb_hcw *hcw,
+					void *addr)
+{
+	struct dlb_dev *dlb_dev;
+
+	dlb_dev = container_of(hw, struct dlb_dev, hw);
+
+	dlb_dev->enqueue_four(hcw, addr);
+}
+
+/**
+ * os_fence_hcw() - fence an HCW to ensure it arrives at the device
+ * @hw: dlb_hw handle for a particular device.
+ * @pp_addr: producer port address
+ */
+static inline void os_fence_hcw(struct dlb_hw *hw, u64 *pp_addr)
+{
+	RTE_SET_USED(hw);
+
+	/* To ensure outstanding HCWs reach the device, read the PP address. IA
+	 * memory ordering prevents reads from passing older writes, and the
+	 * mfence also ensures this.
+	 */
+	rte_mb();
+
+	*(volatile u64 *)pp_addr;
+}
+
+#define DLB_ERR(dev, fmt, args...) \
+	RTE_LOG(ERR, PMD, "%s() line %u: " fmt "\n",  \
+			__func__, __LINE__, ## args)
+
+#define DLB_INFO(dev, fmt, args...) \
+	RTE_LOG(INFO, PMD, "%s() line %u: " fmt "\n", \
+			__func__, __LINE__, ## args)
+
+#define DLB_DEBUG(dev, fmt, args...) \
+	RTE_LOG(DEBUG, PMD, "%s() line %u: " fmt "\n", \
+			__func__, __LINE__, ## args)
+
+/**
+ * DLB_HW_ERR() - log an error message
+ * @dlb: dlb_hw handle for a particular device.
+ * @...: variable string args.
+ */
+#define DLB_HW_ERR(dlb, ...) do {	\
+	RTE_SET_USED(dlb);		\
+	DLB_ERR(dlb, __VA_ARGS__);	\
+} while (0)
+
+/**
+ * DLB_HW_INFO() - log an info message
+ * @dlb: dlb_hw handle for a particular device.
+ * @...: variable string args.
+ */
+#define DLB_HW_INFO(dlb, ...) do {	\
+	RTE_SET_USED(dlb);		\
+	DLB_INFO(dlb, __VA_ARGS__);	\
+} while (0)
+
+/*** scheduling functions ***/
+
+/* The callback runs until it completes all outstanding QID->CQ
+ * map and unmap requests. To prevent deadlock, this function gives other
+ * threads a chance to grab the resource mutex and configure hardware.
+ */
+static void *dlb_complete_queue_map_unmap(void *__args)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)__args;
+	int ret;
+
+	while (1) {
+		rte_spinlock_lock(&dlb_dev->resource_mutex);
+
+		ret = dlb_finish_unmap_qid_procedures(&dlb_dev->hw);
+		ret += dlb_finish_map_qid_procedures(&dlb_dev->hw);
+
+		if (ret != 0) {
+			rte_spinlock_unlock(&dlb_dev->resource_mutex);
+			/* Relinquish the CPU so the application can process
+			 * its CQs, so this function doesn't deadlock.
+			 */
+			sched_yield();
+		} else
+			break;
+	}
+
+	dlb_dev->worker_launched = false;
+
+	rte_spinlock_unlock(&dlb_dev->resource_mutex);
+
+	return NULL;
+}
+
+
+/**
+ * os_schedule_work() - launch a thread to process pending map and unmap work
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function launches a thread that will run until all pending
+ * map and unmap procedures are complete.
+ */
+static inline void os_schedule_work(struct dlb_hw *hw)
+{
+	struct dlb_dev *dlb_dev;
+	pthread_t complete_queue_map_unmap_thread;
+	int ret;
+
+	dlb_dev = container_of(hw, struct dlb_dev, hw);
+
+	ret = pthread_create(&complete_queue_map_unmap_thread,
+			     NULL,
+			     dlb_complete_queue_map_unmap,
+			     dlb_dev);
+	if (ret)
+		DLB_ERR(dlb_dev,
+		"Could not create queue complete map /unmap thread, err=%d\n",
+			  ret);
+	else
+		dlb_dev->worker_launched = true;
+}
+
+/**
+ * os_worker_active() - query whether the map/unmap worker thread is active
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function returns a boolean indicating whether a thread (launched by
+ * os_schedule_work()) is active. This function is used to determine
+ * whether or not to launch a worker thread.
+ */
+static inline bool os_worker_active(struct dlb_hw *hw)
+{
+	struct dlb_dev *dlb_dev;
+
+	dlb_dev = container_of(hw, struct dlb_dev, hw);
+
+	return dlb_dev->worker_launched;
+}
+
+/**
+ * os_notify_user_space() - notify user space
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: ID of domain to notify.
+ * @alert_id: alert ID.
+ * @aux_alert_data: additional alert data.
+ *
+ * This function notifies user space of an alert (such as a remote queue
+ * unregister or hardware alarm).
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ */
+static inline int os_notify_user_space(struct dlb_hw *hw,
+				       u32 domain_id,
+				       u64 alert_id,
+				       u64 aux_alert_data)
+{
+	RTE_SET_USED(hw);
+	RTE_SET_USED(domain_id);
+	RTE_SET_USED(alert_id);
+	RTE_SET_USED(aux_alert_data);
+
+	rte_panic("internal_error: %s should never be called for DLB PF PMD\n",
+		  __func__);
+	return -1;
+}
+
+enum dlb_dev_revision {
+	DLB_A0,
+	DLB_A1,
+	DLB_A2,
+	DLB_A3,
+	DLB_B0,
+};
+
+#include <cpuid.h>
+
+/**
+ * os_get_dev_revision() - query the device_revision
+ * @hw: dlb_hw handle for a particular device.
+ */
+static inline enum dlb_dev_revision os_get_dev_revision(struct dlb_hw *hw)
+{
+	uint32_t a, b, c, d, stepping;
+
+	RTE_SET_USED(hw);
+
+	__cpuid(0x1, a, b, c, d);
+
+	stepping = a & 0xf;
+
+	switch (stepping) {
+	case 0:
+		return DLB_A0;
+	case 1:
+		return DLB_A1;
+	case 2:
+		return DLB_A2;
+	case 3:
+		return DLB_A3;
+	default:
+		/* Treat all revisions >= 4 as B0 */
+		return DLB_B0;
+	}
+}
+
+#endif /*  __DLB_OSDEP_H__ */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep_bitmap.h b/drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
new file mode 100644
index 0000000..8df1d59
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
@@ -0,0 +1,442 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_BITMAP_H__
+#define __DLB_OSDEP_BITMAP_H__
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <rte_bitmap.h>
+#include <rte_string_fns.h>
+#include <rte_malloc.h>
+#include <rte_errno.h>
+#include "../dlb_main.h"
+
+
+/*************************/
+/*** Bitmap operations ***/
+/*************************/
+struct dlb_bitmap {
+	struct rte_bitmap *map;
+	unsigned int len;
+	struct dlb_hw *hw;
+};
+
+/**
+ * dlb_bitmap_alloc() - alloc a bitmap data structure
+ * @bitmap: pointer to dlb_bitmap structure pointer.
+ * @len: number of entries in the bitmap.
+ *
+ * This function allocates a bitmap and initializes it with length @len. All
+ * entries are initially zero.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or len is 0.
+ * ENOMEM - could not allocate memory for the bitmap data structure.
+ */
+static inline int dlb_bitmap_alloc(struct dlb_hw *hw,
+				   struct dlb_bitmap **bitmap,
+				   unsigned int len)
+{
+	struct dlb_bitmap *bm;
+	void *mem;
+	uint32_t alloc_size;
+	uint32_t nbits = (uint32_t) len;
+	RTE_SET_USED(hw);
+
+	if (!bitmap || nbits == 0)
+		return -EINVAL;
+
+	/* Allocate DLB bitmap control struct */
+	bm = rte_malloc("DLB_PF",
+		sizeof(struct dlb_bitmap),
+		RTE_CACHE_LINE_SIZE);
+
+	if (!bm)
+		return -ENOMEM;
+
+	/* Allocate bitmap memory */
+	alloc_size = rte_bitmap_get_memory_footprint(nbits);
+	mem = rte_malloc("DLB_PF_BITMAP", alloc_size, RTE_CACHE_LINE_SIZE);
+	if (!mem) {
+		rte_free(bm);
+		return -ENOMEM;
+	}
+
+	bm->map = rte_bitmap_init(len, mem, alloc_size);
+	if (!bm->map) {
+		rte_free(mem);
+		rte_free(bm);
+		return -ENOMEM;
+	}
+
+	bm->len = len;
+
+	*bitmap = bm;
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_free() - free a previously allocated bitmap data structure
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function frees a bitmap that was allocated with dlb_bitmap_alloc().
+ */
+static inline void dlb_bitmap_free(struct dlb_bitmap *bitmap)
+{
+	if (!bitmap)
+		rte_panic("NULL dlb_bitmap in %s\n", __func__);
+
+	rte_free(bitmap->map);
+	rte_free(bitmap);
+}
+
+/**
+ * dlb_bitmap_fill() - fill a bitmap with all 1s
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function sets all bitmap values to 1.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_fill(struct dlb_bitmap *bitmap)
+{
+	unsigned int i;
+
+	if (!bitmap || !bitmap->map)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++)
+		rte_bitmap_set(bitmap->map, i);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_fill() - fill a bitmap with all 0s
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function sets all bitmap values to 0.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_zero(struct dlb_bitmap *bitmap)
+{
+	if (!bitmap || !bitmap->map)
+		return -EINVAL;
+
+	rte_bitmap_reset(bitmap->map);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_set() - set a bitmap entry
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: bit index.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or bit is larger than the
+ *	    bitmap length.
+ */
+static inline int dlb_bitmap_set(struct dlb_bitmap *bitmap,
+				 unsigned int bit)
+{
+	if (!bitmap || !bitmap->map)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	rte_bitmap_set(bitmap->map, bit);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_set_range() - set a range of bitmap entries
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: starting bit index.
+ * @len: length of the range.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or the range exceeds the bitmap
+ *	    length.
+ */
+static inline int dlb_bitmap_set_range(struct dlb_bitmap *bitmap,
+				       unsigned int bit,
+				       unsigned int len)
+{
+	unsigned int i;
+
+	if (!bitmap || !bitmap->map)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	for (i = 0; i != len; i++)
+		rte_bitmap_set(bitmap->map, bit + i);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_clear() - clear a bitmap entry
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: bit index.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or bit is larger than the
+ *	    bitmap length.
+ */
+static inline int dlb_bitmap_clear(struct dlb_bitmap *bitmap,
+				   unsigned int bit)
+{
+	if (!bitmap || !bitmap->map)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	rte_bitmap_clear(bitmap->map, bit);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_clear_range() - clear a range of bitmap entries
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @bit: starting bit index.
+ * @len: length of the range.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized, or the range exceeds the bitmap
+ *	    length.
+ */
+static inline int dlb_bitmap_clear_range(struct dlb_bitmap *bitmap,
+					 unsigned int bit,
+					 unsigned int len)
+{
+	unsigned int i;
+
+	if (!bitmap || !bitmap->map)
+		return -EINVAL;
+
+	if (bitmap->len <= bit)
+		return -EINVAL;
+
+	for (i = 0; i != len; i++)
+		rte_bitmap_clear(bitmap->map, bit + i);
+
+	return 0;
+}
+
+/**
+ * dlb_bitmap_find_set_bit_range() - find a range of set bits
+ * @bitmap: pointer to dlb_bitmap structure.
+ * @len: length of the range.
+ *
+ * This function looks for a range of set bits of length @len.
+ *
+ * Return:
+ * Returns the base bit index upon success, < 0 otherwise.
+ *
+ * Errors:
+ * ENOENT - unable to find a length *len* range of set bits.
+ * EINVAL - bitmap is NULL or is uninitialized, or len is invalid.
+ */
+static inline int dlb_bitmap_find_set_bit_range(struct dlb_bitmap *bitmap,
+						unsigned int len)
+{
+	unsigned int i, j = 0;
+
+	if (!bitmap || !bitmap->map || len == 0)
+		return -EINVAL;
+
+	if (bitmap->len < len)
+		return -ENOENT;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i)) {
+			if (++j == len)
+				return i - j + 1;
+		} else
+			j = 0;
+	}
+
+	/* No set bit range of length len? */
+	return -ENOENT;
+}
+
+/**
+ * dlb_bitmap_find_set_bit() - find the first set bit
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function looks for a single set bit.
+ *
+ * Return:
+ * Returns the base bit index upon success, < 0 otherwise.
+ *
+ * Errors:
+ * ENOENT - the bitmap contains no set bits.
+ * EINVAL - bitmap is NULL or is uninitialized, or len is invalid.
+ */
+static inline int dlb_bitmap_find_set_bit(struct dlb_bitmap *bitmap)
+{
+	unsigned int i;
+
+	if (!bitmap)
+		return -EINVAL;
+
+	if (!bitmap->map)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i))
+			return i;
+	}
+
+	return -ENOENT;
+}
+
+/**
+ * dlb_bitmap_count() - returns the number of set bits
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * This function looks for a single set bit.
+ *
+ * Return:
+ * Returns the number of set bits upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_count(struct dlb_bitmap *bitmap)
+{
+	int weight = 0;
+	unsigned int i;
+
+	if (!bitmap)
+		return -EINVAL;
+
+	if (!bitmap->map)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i))
+			weight++;
+	}
+	return weight;
+}
+
+/**
+ * dlb_bitmap_longest_set_range() - returns longest contiguous range of set bits
+ * @bitmap: pointer to dlb_bitmap structure.
+ *
+ * Return:
+ * Returns the bitmap's longest contiguous range of of set bits upon success,
+ * <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - bitmap is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_longest_set_range(struct dlb_bitmap *bitmap)
+{
+	int max_len = 0, len = 0;
+	unsigned int i;
+
+	if (!bitmap)
+		return -EINVAL;
+
+	if (!bitmap->map)
+		return -EINVAL;
+
+	for (i = 0; i != bitmap->len; i++) {
+		if  (rte_bitmap_get(bitmap->map, i)) {
+			len++;
+		} else {
+			if (len > max_len)
+				max_len = len;
+			len = 0;
+		}
+	}
+
+	if (len > max_len)
+		max_len = len;
+
+	return max_len;
+}
+
+/**
+ * dlb_bitmap_or() - store the logical 'or' of two bitmaps into a third
+ * @dest: pointer to dlb_bitmap structure, which will contain the results of
+ *	  the 'or' of src1 and src2.
+ * @src1: pointer to dlb_bitmap structure, will be 'or'ed with src2.
+ * @src2: pointer to dlb_bitmap structure, will be 'or'ed with src1.
+ *
+ * This function 'or's two bitmaps together and stores the result in a third
+ * bitmap. The source and destination bitmaps can be the same.
+ *
+ * Return:
+ * Returns the number of set bits upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - One of the bitmaps is NULL or is uninitialized.
+ */
+static inline int dlb_bitmap_or(struct dlb_bitmap *dest,
+				struct dlb_bitmap *src1,
+				struct dlb_bitmap *src2)
+{
+	unsigned int i, min;
+	int numset = 0;
+
+	if (!dest || !dest->map ||
+	    !src1 || !src1->map ||
+	    !src2 || !src2->map)
+		return -EINVAL;
+
+	min = dest->len;
+	min = (min > src1->len) ? src1->len : min;
+	min = (min > src2->len) ? src2->len : min;
+
+	for (i = 0; i != min; i++) {
+		if  (rte_bitmap_get(src1->map, i) ||
+				rte_bitmap_get(src2->map, i)) {
+			rte_bitmap_set(dest->map, i);
+			numset++;
+		} else
+			rte_bitmap_clear(dest->map, i);
+	}
+
+	return numset;
+}
+
+#endif /*  __DLB_OSDEP_BITMAP_H__ */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep_list.h b/drivers/event/dlb/pf/base/dlb_osdep_list.h
new file mode 100644
index 0000000..a53b362
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep_list.h
@@ -0,0 +1,131 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_LIST_H__
+#define __DLB_OSDEP_LIST_H__
+
+#include <rte_tailq.h>
+
+struct dlb_list_entry {
+	TAILQ_ENTRY(dlb_list_entry) node;
+};
+
+/* Dummy - just a struct definition */
+TAILQ_HEAD(dlb_list_head, dlb_list_entry);
+
+/* =================
+ * TAILQ Supplements
+ * =================
+ */
+
+#ifndef TAILQ_FOREACH_ENTRY
+#define TAILQ_FOREACH_ENTRY(ptr, head, name, iter)		\
+	for ((iter) = TAILQ_FIRST(&head);			\
+	    (iter)						\
+		&& (ptr = container_of(iter, typeof(*(ptr)), name)); \
+	    (iter) = TAILQ_NEXT((iter), node))
+#endif
+
+#ifndef TAILQ_FOREACH_ENTRY_SAFE
+#define TAILQ_FOREACH_ENTRY_SAFE(ptr, head, name, iter, tvar)	\
+	for ((iter) = TAILQ_FIRST(&head);			\
+	    (iter) &&						\
+		(ptr = container_of(iter, typeof(*(ptr)), name)) &&\
+		((tvar) = TAILQ_NEXT((iter), node), 1);	\
+	    (iter) = (tvar))
+#endif
+
+/* =========
+ * DLB Lists
+ * =========
+ */
+
+/**
+ * dlb_list_init_head() - initialize the head of a list
+ * @head: list head
+ */
+static inline void dlb_list_init_head(struct dlb_list_head *head)
+{
+	TAILQ_INIT(head);
+}
+
+/**
+ * dlb_list_add() - add an entry to a list
+ * @head: new entry will be added after this list header
+ * @entry: new list entry to be added
+ */
+static inline void dlb_list_add(struct dlb_list_head *head,
+				struct dlb_list_entry *entry)
+{
+	TAILQ_INSERT_TAIL(head, entry, node);
+}
+
+/**
+ * @head: list head
+ * @entry: list entry to be deleted
+ */
+static inline void dlb_list_del(struct dlb_list_head *head,
+				struct dlb_list_entry *entry)
+{
+	TAILQ_REMOVE(head, entry, node);
+}
+
+/**
+ * dlb_list_empty() - check if a list is empty
+ * @head: list head
+ *
+ * Return:
+ * Returns 1 if empty, 0 if not.
+ */
+static inline bool dlb_list_empty(struct dlb_list_head *head)
+{
+	return TAILQ_EMPTY(head);
+}
+
+/**
+ * dlb_list_empty() - check if a list is empty
+ * @src_head: list to be added
+ * @ head: where src_head will be inserted
+ */
+static inline void dlb_list_splice(struct dlb_list_head *src_head,
+				   struct dlb_list_head *head)
+{
+	TAILQ_CONCAT(head, src_head, node);
+}
+
+/**
+ * DLB_LIST_HEAD() - retrieve the head of the list
+ * @head: list head
+ * @type: type of the list variable
+ * @name: name of the dlb_list within the struct
+ */
+#define DLB_LIST_HEAD(head, type, name)				\
+	(TAILQ_FIRST(&head) ?					\
+		container_of(TAILQ_FIRST(&head), type, name) :	\
+		NULL)
+
+/**
+ * DLB_LIST_FOR_EACH() - iterate over a list
+ * @head: list head
+ * @ptr: pointer to struct containing a struct dlb_list_entry
+ * @name: name of the dlb_list_entry field within the containing struct
+ * @iter: iterator variable
+ */
+#define DLB_LIST_FOR_EACH(head, ptr, name, tmp_iter) \
+	TAILQ_FOREACH_ENTRY(ptr, head, name, tmp_iter)
+
+/**
+ * DLB_LIST_FOR_EACH_SAFE() - iterate over a list. This loop works even if
+ * an element is removed from the list while processing it.
+ * @ptr: pointer to struct containing a struct dlb_list_entry
+ * @ptr_tmp: pointer to struct containing a struct dlb_list_entry (temporary)
+ * @head: list head
+ * @name: name of the dlb_list_entry field within the containing struct
+ * @iter: iterator variable
+ * @iter_tmp: iterator variable (temporary)
+ */
+#define DLB_LIST_FOR_EACH_SAFE(head, ptr, ptr_tmp, name, tmp_iter, saf_iter) \
+	TAILQ_FOREACH_ENTRY_SAFE(ptr, head, name, tmp_iter, saf_iter)
+
+#endif /*  __DLB_OSDEP_LIST_H__ */
diff --git a/drivers/event/dlb/pf/base/dlb_osdep_types.h b/drivers/event/dlb/pf/base/dlb_osdep_types.h
new file mode 100644
index 0000000..2e9d7d8
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_osdep_types.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_TYPES_H
+#define __DLB_OSDEP_TYPES_H
+
+#include <linux/types.h>
+
+#include <inttypes.h>
+#include <ctype.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+
+/* Types for user mode PF PMD */
+typedef uint8_t         u8;
+typedef int8_t          s8;
+typedef uint16_t        u16;
+typedef int16_t         s16;
+typedef uint32_t        u32;
+typedef int32_t         s32;
+typedef uint64_t        u64;
+
+#define __iomem
+
+/* END types for user mode PF PMD */
+
+#endif /* __DLB_OSDEP_TYPES_H */
diff --git a/drivers/event/dlb/pf/base/dlb_regs.h b/drivers/event/dlb/pf/base/dlb_regs.h
new file mode 100644
index 0000000..3b0be23
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_regs.h
@@ -0,0 +1,2646 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_REGS_H
+#define __DLB_REGS_H
+
+#include "dlb_osdep_types.h"
+
+#define DLB_FUNC_PF_VF2PF_MAILBOX_BYTES 256
+#define DLB_FUNC_PF_VF2PF_MAILBOX(vf_id, x) \
+	(0x1000 + 0x4 * (x) + (vf_id) * 0x10000)
+#define DLB_FUNC_PF_VF2PF_MAILBOX_RST 0x0
+union dlb_func_pf_vf2pf_mailbox {
+	struct {
+		u32 msg : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_FUNC_PF_VF2PF_MAILBOX_ISR(vf_id) \
+	(0x1f00 + (vf_id) * 0x10000)
+#define DLB_FUNC_PF_VF2PF_MAILBOX_ISR_RST 0x0
+union dlb_func_pf_vf2pf_mailbox_isr {
+	struct {
+		u32 vf_isr : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_FUNC_PF_VF2PF_FLR_ISR(vf_id) \
+	(0x1f04 + (vf_id) * 0x10000)
+#define DLB_FUNC_PF_VF2PF_FLR_ISR_RST 0x0
+union dlb_func_pf_vf2pf_flr_isr {
+	struct {
+		u32 vf_isr : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_FUNC_PF_VF2PF_ISR_PEND(vf_id) \
+	(0x1f10 + (vf_id) * 0x10000)
+#define DLB_FUNC_PF_VF2PF_ISR_PEND_RST 0x0
+union dlb_func_pf_vf2pf_isr_pend {
+	struct {
+		u32 isr_pend : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_FUNC_PF_PF2VF_MAILBOX_BYTES 64
+#define DLB_FUNC_PF_PF2VF_MAILBOX(vf_id, x) \
+	(0x2000 + 0x4 * (x) + (vf_id) * 0x10000)
+#define DLB_FUNC_PF_PF2VF_MAILBOX_RST 0x0
+union dlb_func_pf_pf2vf_mailbox {
+	struct {
+		u32 msg : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_FUNC_PF_PF2VF_MAILBOX_ISR(vf_id) \
+	(0x2f00 + (vf_id) * 0x10000)
+#define DLB_FUNC_PF_PF2VF_MAILBOX_ISR_RST 0x0
+union dlb_func_pf_pf2vf_mailbox_isr {
+	struct {
+		u32 isr : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_FUNC_PF_VF_RESET_IN_PROGRESS(vf_id) \
+	(0x3000 + (vf_id) * 0x10000)
+#define DLB_FUNC_PF_VF_RESET_IN_PROGRESS_RST 0xffff
+union dlb_func_pf_vf_reset_in_progress {
+	struct {
+		u32 reset_in_progress : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_MSIX_MEM_VECTOR_CTRL(x) \
+	(0x100000c + (x) * 0x10)
+#define DLB_MSIX_MEM_VECTOR_CTRL_RST 0x1
+union dlb_msix_mem_vector_ctrl {
+	struct {
+		u32 vec_mask : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_TOTAL_VAS 0x124
+#define DLB_SYS_TOTAL_VAS_RST 0x20
+union dlb_sys_total_vas {
+	struct {
+		u32 total_vas : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_PF_SYND2 0x508
+#define DLB_SYS_ALARM_PF_SYND2_RST 0x0
+union dlb_sys_alarm_pf_synd2 {
+	struct {
+		u32 lock_id : 16;
+		u32 meas : 1;
+		u32 debug : 7;
+		u32 cq_pop : 1;
+		u32 qe_uhl : 1;
+		u32 qe_orsp : 1;
+		u32 qe_valid : 1;
+		u32 cq_int_rearm : 1;
+		u32 dsi_error : 1;
+		u32 rsvd0 : 2;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_PF_SYND1 0x504
+#define DLB_SYS_ALARM_PF_SYND1_RST 0x0
+union dlb_sys_alarm_pf_synd1 {
+	struct {
+		u32 dsi : 16;
+		u32 qid : 8;
+		u32 qtype : 2;
+		u32 qpri : 3;
+		u32 msg_type : 3;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_PF_SYND0 0x500
+#define DLB_SYS_ALARM_PF_SYND0_RST 0x0
+union dlb_sys_alarm_pf_synd0 {
+	struct {
+		u32 syndrome : 8;
+		u32 rtype : 2;
+		u32 rsvd0 : 2;
+		u32 from_dmv : 1;
+		u32 is_ldb : 1;
+		u32 cls : 2;
+		u32 aid : 6;
+		u32 unit : 4;
+		u32 source : 4;
+		u32 more : 1;
+		u32 valid : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_VF_LDB_VPP_V(x) \
+	(0xf00 + (x) * 0x1000)
+#define DLB_SYS_VF_LDB_VPP_V_RST 0x0
+union dlb_sys_vf_ldb_vpp_v {
+	struct {
+		u32 vpp_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_VF_LDB_VPP2PP(x) \
+	(0xf08 + (x) * 0x1000)
+#define DLB_SYS_VF_LDB_VPP2PP_RST 0x0
+union dlb_sys_vf_ldb_vpp2pp {
+	struct {
+		u32 pp : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_VF_DIR_VPP_V(x) \
+	(0xf10 + (x) * 0x1000)
+#define DLB_SYS_VF_DIR_VPP_V_RST 0x0
+union dlb_sys_vf_dir_vpp_v {
+	struct {
+		u32 vpp_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_VF_DIR_VPP2PP(x) \
+	(0xf18 + (x) * 0x1000)
+#define DLB_SYS_VF_DIR_VPP2PP_RST 0x0
+union dlb_sys_vf_dir_vpp2pp {
+	struct {
+		u32 pp : 7;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_VF_LDB_VQID_V(x) \
+	(0xf20 + (x) * 0x1000)
+#define DLB_SYS_VF_LDB_VQID_V_RST 0x0
+union dlb_sys_vf_ldb_vqid_v {
+	struct {
+		u32 vqid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_VF_LDB_VQID2QID(x) \
+	(0xf28 + (x) * 0x1000)
+#define DLB_SYS_VF_LDB_VQID2QID_RST 0x0
+union dlb_sys_vf_ldb_vqid2qid {
+	struct {
+		u32 qid : 7;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_QID2VQID(x) \
+	(0xf2c + (x) * 0x1000)
+#define DLB_SYS_LDB_QID2VQID_RST 0x0
+union dlb_sys_ldb_qid2vqid {
+	struct {
+		u32 vqid : 7;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_VF_DIR_VQID_V(x) \
+	(0xf30 + (x) * 0x1000)
+#define DLB_SYS_VF_DIR_VQID_V_RST 0x0
+union dlb_sys_vf_dir_vqid_v {
+	struct {
+		u32 vqid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_VF_DIR_VQID2QID(x) \
+	(0xf38 + (x) * 0x1000)
+#define DLB_SYS_VF_DIR_VQID2QID_RST 0x0
+union dlb_sys_vf_dir_vqid2qid {
+	struct {
+		u32 qid : 7;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_VASQID_V(x) \
+	(0xf60 + (x) * 0x1000)
+#define DLB_SYS_LDB_VASQID_V_RST 0x0
+union dlb_sys_ldb_vasqid_v {
+	struct {
+		u32 vasqid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_VASQID_V(x) \
+	(0xf68 + (x) * 0x1000)
+#define DLB_SYS_DIR_VASQID_V_RST 0x0
+union dlb_sys_dir_vasqid_v {
+	struct {
+		u32 vasqid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_WBUF_DIR_FLAGS(x) \
+	(0xf70 + (x) * 0x1000)
+#define DLB_SYS_WBUF_DIR_FLAGS_RST 0x0
+union dlb_sys_wbuf_dir_flags {
+	struct {
+		u32 wb_v : 4;
+		u32 cl : 1;
+		u32 busy : 1;
+		u32 opt : 1;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_WBUF_LDB_FLAGS(x) \
+	(0xf78 + (x) * 0x1000)
+#define DLB_SYS_WBUF_LDB_FLAGS_RST 0x0
+union dlb_sys_wbuf_ldb_flags {
+	struct {
+		u32 wb_v : 4;
+		u32 cl : 1;
+		u32 busy : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_VF_SYND2(x) \
+	(0x8000018 + (x) * 0x1000)
+#define DLB_SYS_ALARM_VF_SYND2_RST 0x0
+union dlb_sys_alarm_vf_synd2 {
+	struct {
+		u32 lock_id : 16;
+		u32 meas : 1;
+		u32 debug : 7;
+		u32 cq_pop : 1;
+		u32 qe_uhl : 1;
+		u32 qe_orsp : 1;
+		u32 qe_valid : 1;
+		u32 cq_int_rearm : 1;
+		u32 dsi_error : 1;
+		u32 rsvd0 : 2;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_VF_SYND1(x) \
+	(0x8000014 + (x) * 0x1000)
+#define DLB_SYS_ALARM_VF_SYND1_RST 0x0
+union dlb_sys_alarm_vf_synd1 {
+	struct {
+		u32 dsi : 16;
+		u32 qid : 8;
+		u32 qtype : 2;
+		u32 qpri : 3;
+		u32 msg_type : 3;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_VF_SYND0(x) \
+	(0x8000010 + (x) * 0x1000)
+#define DLB_SYS_ALARM_VF_SYND0_RST 0x0
+union dlb_sys_alarm_vf_synd0 {
+	struct {
+		u32 syndrome : 8;
+		u32 rtype : 2;
+		u32 rsvd0 : 2;
+		u32 from_dmv : 1;
+		u32 is_ldb : 1;
+		u32 cls : 2;
+		u32 aid : 6;
+		u32 unit : 4;
+		u32 source : 4;
+		u32 more : 1;
+		u32 valid : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_QID_V(x) \
+	(0x8000034 + (x) * 0x1000)
+#define DLB_SYS_LDB_QID_V_RST 0x0
+union dlb_sys_ldb_qid_v {
+	struct {
+		u32 qid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_QID_CFG_V(x) \
+	(0x8000030 + (x) * 0x1000)
+#define DLB_SYS_LDB_QID_CFG_V_RST 0x0
+union dlb_sys_ldb_qid_cfg_v {
+	struct {
+		u32 sn_cfg_v : 1;
+		u32 fid_cfg_v : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_QID_V(x) \
+	(0x8000040 + (x) * 0x1000)
+#define DLB_SYS_DIR_QID_V_RST 0x0
+union dlb_sys_dir_qid_v {
+	struct {
+		u32 qid_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_POOL_ENBLD(x) \
+	(0x8000070 + (x) * 0x1000)
+#define DLB_SYS_LDB_POOL_ENBLD_RST 0x0
+union dlb_sys_ldb_pool_enbld {
+	struct {
+		u32 pool_enabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_POOL_ENBLD(x) \
+	(0x8000080 + (x) * 0x1000)
+#define DLB_SYS_DIR_POOL_ENBLD_RST 0x0
+union dlb_sys_dir_pool_enbld {
+	struct {
+		u32 pool_enabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2VPP(x) \
+	(0x8000090 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2VPP_RST 0x0
+union dlb_sys_ldb_pp2vpp {
+	struct {
+		u32 vpp : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2VPP(x) \
+	(0x8000094 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2VPP_RST 0x0
+union dlb_sys_dir_pp2vpp {
+	struct {
+		u32 vpp : 7;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP_V(x) \
+	(0x8000128 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP_V_RST 0x0
+union dlb_sys_ldb_pp_v {
+	struct {
+		u32 pp_v : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_ISR(x) \
+	(0x8000124 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ_ISR_RST 0x0
+/* CQ Interrupt Modes */
+#define DLB_CQ_ISR_MODE_DIS  0
+#define DLB_CQ_ISR_MODE_MSI  1
+#define DLB_CQ_ISR_MODE_MSIX 2
+union dlb_sys_ldb_cq_isr {
+	struct {
+		u32 vector : 6;
+		u32 vf : 4;
+		u32 en_code : 2;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ2VF_PF(x) \
+	(0x8000120 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ2VF_PF_RST 0x0
+union dlb_sys_ldb_cq2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2VAS(x) \
+	(0x800011c + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2VAS_RST 0x0
+union dlb_sys_ldb_pp2vas {
+	struct {
+		u32 vas : 5;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2LDBPOOL(x) \
+	(0x8000118 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2LDBPOOL_RST 0x0
+union dlb_sys_ldb_pp2ldbpool {
+	struct {
+		u32 ldbpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2DIRPOOL(x) \
+	(0x8000114 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2DIRPOOL_RST 0x0
+union dlb_sys_ldb_pp2dirpool {
+	struct {
+		u32 dirpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP2VF_PF(x) \
+	(0x8000110 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP2VF_PF_RST 0x0
+union dlb_sys_ldb_pp2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP_ADDR_U(x) \
+	(0x800010c + (x) * 0x1000)
+#define DLB_SYS_LDB_PP_ADDR_U_RST 0x0
+union dlb_sys_ldb_pp_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_PP_ADDR_L(x) \
+	(0x8000108 + (x) * 0x1000)
+#define DLB_SYS_LDB_PP_ADDR_L_RST 0x0
+union dlb_sys_ldb_pp_addr_l {
+	struct {
+		u32 rsvd0 : 7;
+		u32 addr_l : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_ADDR_U(x) \
+	(0x8000104 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ_ADDR_U_RST 0x0
+union dlb_sys_ldb_cq_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_ADDR_L(x) \
+	(0x8000100 + (x) * 0x1000)
+#define DLB_SYS_LDB_CQ_ADDR_L_RST 0x0
+union dlb_sys_ldb_cq_addr_l {
+	struct {
+		u32 rsvd0 : 6;
+		u32 addr_l : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP_V(x) \
+	(0x8000228 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP_V_RST 0x0
+union dlb_sys_dir_pp_v {
+	struct {
+		u32 pp_v : 1;
+		u32 mb_dm : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_ISR(x) \
+	(0x8000224 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ_ISR_RST 0x0
+union dlb_sys_dir_cq_isr {
+	struct {
+		u32 vector : 6;
+		u32 vf : 4;
+		u32 en_code : 2;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ2VF_PF(x) \
+	(0x8000220 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ2VF_PF_RST 0x0
+union dlb_sys_dir_cq2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2VAS(x) \
+	(0x800021c + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2VAS_RST 0x0
+union dlb_sys_dir_pp2vas {
+	struct {
+		u32 vas : 5;
+		u32 rsvd0 : 27;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2LDBPOOL(x) \
+	(0x8000218 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2LDBPOOL_RST 0x0
+union dlb_sys_dir_pp2ldbpool {
+	struct {
+		u32 ldbpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2DIRPOOL(x) \
+	(0x8000214 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2DIRPOOL_RST 0x0
+union dlb_sys_dir_pp2dirpool {
+	struct {
+		u32 dirpool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP2VF_PF(x) \
+	(0x8000210 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP2VF_PF_RST 0x0
+union dlb_sys_dir_pp2vf_pf {
+	struct {
+		u32 vf : 4;
+		u32 is_pf : 1;
+		u32 is_hw_dsi : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP_ADDR_U(x) \
+	(0x800020c + (x) * 0x1000)
+#define DLB_SYS_DIR_PP_ADDR_U_RST 0x0
+union dlb_sys_dir_pp_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_PP_ADDR_L(x) \
+	(0x8000208 + (x) * 0x1000)
+#define DLB_SYS_DIR_PP_ADDR_L_RST 0x0
+union dlb_sys_dir_pp_addr_l {
+	struct {
+		u32 rsvd0 : 7;
+		u32 addr_l : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_ADDR_U(x) \
+	(0x8000204 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ_ADDR_U_RST 0x0
+union dlb_sys_dir_cq_addr_u {
+	struct {
+		u32 addr_u : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_ADDR_L(x) \
+	(0x8000200 + (x) * 0x1000)
+#define DLB_SYS_DIR_CQ_ADDR_L_RST 0x0
+union dlb_sys_dir_cq_addr_l {
+	struct {
+		u32 rsvd0 : 6;
+		u32 addr_l : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_INGRESS_ALARM_ENBL 0x300
+#define DLB_SYS_INGRESS_ALARM_ENBL_RST 0x0
+union dlb_sys_ingress_alarm_enbl {
+	struct {
+		u32 illegal_hcw : 1;
+		u32 illegal_pp : 1;
+		u32 disabled_pp : 1;
+		u32 illegal_qid : 1;
+		u32 disabled_qid : 1;
+		u32 illegal_ldb_qid_cfg : 1;
+		u32 illegal_cqid : 1;
+		u32 rsvd0 : 25;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_CQ_MODE 0x30c
+#define DLB_SYS_CQ_MODE_RST 0x0
+union dlb_sys_cq_mode {
+	struct {
+		u32 ldb_cq64 : 1;
+		u32 dir_cq64 : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_FUNC_VF_BAR_DSBL(x) \
+	(0x310 + (x) * 0x4)
+#define DLB_SYS_FUNC_VF_BAR_DSBL_RST 0x0
+union dlb_sys_func_vf_bar_dsbl {
+	struct {
+		u32 func_vf_bar_dis : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_MSIX_ACK 0x400
+#define DLB_SYS_MSIX_ACK_RST 0x0
+union dlb_sys_msix_ack {
+	struct {
+		u32 msix_0_ack : 1;
+		u32 msix_1_ack : 1;
+		u32 msix_2_ack : 1;
+		u32 msix_3_ack : 1;
+		u32 msix_4_ack : 1;
+		u32 msix_5_ack : 1;
+		u32 msix_6_ack : 1;
+		u32 msix_7_ack : 1;
+		u32 msix_8_ack : 1;
+		u32 rsvd0 : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_MSIX_PASSTHRU 0x404
+#define DLB_SYS_MSIX_PASSTHRU_RST 0x0
+union dlb_sys_msix_passthru {
+	struct {
+		u32 msix_0_passthru : 1;
+		u32 msix_1_passthru : 1;
+		u32 msix_2_passthru : 1;
+		u32 msix_3_passthru : 1;
+		u32 msix_4_passthru : 1;
+		u32 msix_5_passthru : 1;
+		u32 msix_6_passthru : 1;
+		u32 msix_7_passthru : 1;
+		u32 msix_8_passthru : 1;
+		u32 rsvd0 : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_MSIX_MODE 0x408
+#define DLB_SYS_MSIX_MODE_RST 0x0
+/* MSI-X Modes */
+#define DLB_MSIX_MODE_PACKED     0
+#define DLB_MSIX_MODE_COMPRESSED 1
+union dlb_sys_msix_mode {
+	struct {
+		u32 mode : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_31_0_OCC_INT_STS 0x440
+#define DLB_SYS_DIR_CQ_31_0_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_31_0_occ_int_sts {
+	struct {
+		u32 cq_0_occ_int : 1;
+		u32 cq_1_occ_int : 1;
+		u32 cq_2_occ_int : 1;
+		u32 cq_3_occ_int : 1;
+		u32 cq_4_occ_int : 1;
+		u32 cq_5_occ_int : 1;
+		u32 cq_6_occ_int : 1;
+		u32 cq_7_occ_int : 1;
+		u32 cq_8_occ_int : 1;
+		u32 cq_9_occ_int : 1;
+		u32 cq_10_occ_int : 1;
+		u32 cq_11_occ_int : 1;
+		u32 cq_12_occ_int : 1;
+		u32 cq_13_occ_int : 1;
+		u32 cq_14_occ_int : 1;
+		u32 cq_15_occ_int : 1;
+		u32 cq_16_occ_int : 1;
+		u32 cq_17_occ_int : 1;
+		u32 cq_18_occ_int : 1;
+		u32 cq_19_occ_int : 1;
+		u32 cq_20_occ_int : 1;
+		u32 cq_21_occ_int : 1;
+		u32 cq_22_occ_int : 1;
+		u32 cq_23_occ_int : 1;
+		u32 cq_24_occ_int : 1;
+		u32 cq_25_occ_int : 1;
+		u32 cq_26_occ_int : 1;
+		u32 cq_27_occ_int : 1;
+		u32 cq_28_occ_int : 1;
+		u32 cq_29_occ_int : 1;
+		u32 cq_30_occ_int : 1;
+		u32 cq_31_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_63_32_OCC_INT_STS 0x444
+#define DLB_SYS_DIR_CQ_63_32_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_63_32_occ_int_sts {
+	struct {
+		u32 cq_32_occ_int : 1;
+		u32 cq_33_occ_int : 1;
+		u32 cq_34_occ_int : 1;
+		u32 cq_35_occ_int : 1;
+		u32 cq_36_occ_int : 1;
+		u32 cq_37_occ_int : 1;
+		u32 cq_38_occ_int : 1;
+		u32 cq_39_occ_int : 1;
+		u32 cq_40_occ_int : 1;
+		u32 cq_41_occ_int : 1;
+		u32 cq_42_occ_int : 1;
+		u32 cq_43_occ_int : 1;
+		u32 cq_44_occ_int : 1;
+		u32 cq_45_occ_int : 1;
+		u32 cq_46_occ_int : 1;
+		u32 cq_47_occ_int : 1;
+		u32 cq_48_occ_int : 1;
+		u32 cq_49_occ_int : 1;
+		u32 cq_50_occ_int : 1;
+		u32 cq_51_occ_int : 1;
+		u32 cq_52_occ_int : 1;
+		u32 cq_53_occ_int : 1;
+		u32 cq_54_occ_int : 1;
+		u32 cq_55_occ_int : 1;
+		u32 cq_56_occ_int : 1;
+		u32 cq_57_occ_int : 1;
+		u32 cq_58_occ_int : 1;
+		u32 cq_59_occ_int : 1;
+		u32 cq_60_occ_int : 1;
+		u32 cq_61_occ_int : 1;
+		u32 cq_62_occ_int : 1;
+		u32 cq_63_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_95_64_OCC_INT_STS 0x448
+#define DLB_SYS_DIR_CQ_95_64_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_95_64_occ_int_sts {
+	struct {
+		u32 cq_64_occ_int : 1;
+		u32 cq_65_occ_int : 1;
+		u32 cq_66_occ_int : 1;
+		u32 cq_67_occ_int : 1;
+		u32 cq_68_occ_int : 1;
+		u32 cq_69_occ_int : 1;
+		u32 cq_70_occ_int : 1;
+		u32 cq_71_occ_int : 1;
+		u32 cq_72_occ_int : 1;
+		u32 cq_73_occ_int : 1;
+		u32 cq_74_occ_int : 1;
+		u32 cq_75_occ_int : 1;
+		u32 cq_76_occ_int : 1;
+		u32 cq_77_occ_int : 1;
+		u32 cq_78_occ_int : 1;
+		u32 cq_79_occ_int : 1;
+		u32 cq_80_occ_int : 1;
+		u32 cq_81_occ_int : 1;
+		u32 cq_82_occ_int : 1;
+		u32 cq_83_occ_int : 1;
+		u32 cq_84_occ_int : 1;
+		u32 cq_85_occ_int : 1;
+		u32 cq_86_occ_int : 1;
+		u32 cq_87_occ_int : 1;
+		u32 cq_88_occ_int : 1;
+		u32 cq_89_occ_int : 1;
+		u32 cq_90_occ_int : 1;
+		u32 cq_91_occ_int : 1;
+		u32 cq_92_occ_int : 1;
+		u32 cq_93_occ_int : 1;
+		u32 cq_94_occ_int : 1;
+		u32 cq_95_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_DIR_CQ_127_96_OCC_INT_STS 0x44c
+#define DLB_SYS_DIR_CQ_127_96_OCC_INT_STS_RST 0x0
+union dlb_sys_dir_cq_127_96_occ_int_sts {
+	struct {
+		u32 cq_96_occ_int : 1;
+		u32 cq_97_occ_int : 1;
+		u32 cq_98_occ_int : 1;
+		u32 cq_99_occ_int : 1;
+		u32 cq_100_occ_int : 1;
+		u32 cq_101_occ_int : 1;
+		u32 cq_102_occ_int : 1;
+		u32 cq_103_occ_int : 1;
+		u32 cq_104_occ_int : 1;
+		u32 cq_105_occ_int : 1;
+		u32 cq_106_occ_int : 1;
+		u32 cq_107_occ_int : 1;
+		u32 cq_108_occ_int : 1;
+		u32 cq_109_occ_int : 1;
+		u32 cq_110_occ_int : 1;
+		u32 cq_111_occ_int : 1;
+		u32 cq_112_occ_int : 1;
+		u32 cq_113_occ_int : 1;
+		u32 cq_114_occ_int : 1;
+		u32 cq_115_occ_int : 1;
+		u32 cq_116_occ_int : 1;
+		u32 cq_117_occ_int : 1;
+		u32 cq_118_occ_int : 1;
+		u32 cq_119_occ_int : 1;
+		u32 cq_120_occ_int : 1;
+		u32 cq_121_occ_int : 1;
+		u32 cq_122_occ_int : 1;
+		u32 cq_123_occ_int : 1;
+		u32 cq_124_occ_int : 1;
+		u32 cq_125_occ_int : 1;
+		u32 cq_126_occ_int : 1;
+		u32 cq_127_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_31_0_OCC_INT_STS 0x460
+#define DLB_SYS_LDB_CQ_31_0_OCC_INT_STS_RST 0x0
+union dlb_sys_ldb_cq_31_0_occ_int_sts {
+	struct {
+		u32 cq_0_occ_int : 1;
+		u32 cq_1_occ_int : 1;
+		u32 cq_2_occ_int : 1;
+		u32 cq_3_occ_int : 1;
+		u32 cq_4_occ_int : 1;
+		u32 cq_5_occ_int : 1;
+		u32 cq_6_occ_int : 1;
+		u32 cq_7_occ_int : 1;
+		u32 cq_8_occ_int : 1;
+		u32 cq_9_occ_int : 1;
+		u32 cq_10_occ_int : 1;
+		u32 cq_11_occ_int : 1;
+		u32 cq_12_occ_int : 1;
+		u32 cq_13_occ_int : 1;
+		u32 cq_14_occ_int : 1;
+		u32 cq_15_occ_int : 1;
+		u32 cq_16_occ_int : 1;
+		u32 cq_17_occ_int : 1;
+		u32 cq_18_occ_int : 1;
+		u32 cq_19_occ_int : 1;
+		u32 cq_20_occ_int : 1;
+		u32 cq_21_occ_int : 1;
+		u32 cq_22_occ_int : 1;
+		u32 cq_23_occ_int : 1;
+		u32 cq_24_occ_int : 1;
+		u32 cq_25_occ_int : 1;
+		u32 cq_26_occ_int : 1;
+		u32 cq_27_occ_int : 1;
+		u32 cq_28_occ_int : 1;
+		u32 cq_29_occ_int : 1;
+		u32 cq_30_occ_int : 1;
+		u32 cq_31_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_LDB_CQ_63_32_OCC_INT_STS 0x464
+#define DLB_SYS_LDB_CQ_63_32_OCC_INT_STS_RST 0x0
+union dlb_sys_ldb_cq_63_32_occ_int_sts {
+	struct {
+		u32 cq_32_occ_int : 1;
+		u32 cq_33_occ_int : 1;
+		u32 cq_34_occ_int : 1;
+		u32 cq_35_occ_int : 1;
+		u32 cq_36_occ_int : 1;
+		u32 cq_37_occ_int : 1;
+		u32 cq_38_occ_int : 1;
+		u32 cq_39_occ_int : 1;
+		u32 cq_40_occ_int : 1;
+		u32 cq_41_occ_int : 1;
+		u32 cq_42_occ_int : 1;
+		u32 cq_43_occ_int : 1;
+		u32 cq_44_occ_int : 1;
+		u32 cq_45_occ_int : 1;
+		u32 cq_46_occ_int : 1;
+		u32 cq_47_occ_int : 1;
+		u32 cq_48_occ_int : 1;
+		u32 cq_49_occ_int : 1;
+		u32 cq_50_occ_int : 1;
+		u32 cq_51_occ_int : 1;
+		u32 cq_52_occ_int : 1;
+		u32 cq_53_occ_int : 1;
+		u32 cq_54_occ_int : 1;
+		u32 cq_55_occ_int : 1;
+		u32 cq_56_occ_int : 1;
+		u32 cq_57_occ_int : 1;
+		u32 cq_58_occ_int : 1;
+		u32 cq_59_occ_int : 1;
+		u32 cq_60_occ_int : 1;
+		u32 cq_61_occ_int : 1;
+		u32 cq_62_occ_int : 1;
+		u32 cq_63_occ_int : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_SYS_ALARM_HW_SYND 0x50c
+#define DLB_SYS_ALARM_HW_SYND_RST 0x0
+union dlb_sys_alarm_hw_synd {
+	struct {
+		u32 syndrome : 8;
+		u32 rtype : 2;
+		u32 rsvd0 : 2;
+		u32 from_dmv : 1;
+		u32 is_ldb : 1;
+		u32 cls : 2;
+		u32 aid : 6;
+		u32 unit : 4;
+		u32 source : 4;
+		u32 more : 1;
+		u32 valid : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL(x) \
+	(0x20000000 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL_RST 0x0
+union dlb_lsp_cq_ldb_tot_sch_cnt_ctrl {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_DSBL(x) \
+	(0x20000124 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_DSBL_RST 0x1
+union dlb_lsp_cq_ldb_dsbl {
+	struct {
+		u32 disabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTH(x) \
+	(0x20000120 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTH_RST 0x0
+union dlb_lsp_cq_ldb_tot_sch_cnth {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTL(x) \
+	(0x2000011c + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TOT_SCH_CNTL_RST 0x0
+union dlb_lsp_cq_ldb_tot_sch_cntl {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(x) \
+	(0x20000118 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TKN_DEPTH_SEL_RST 0x0
+union dlb_lsp_cq_ldb_tkn_depth_sel {
+	struct {
+		u32 token_depth_select : 4;
+		u32 ignore_depth : 1;
+		u32 enab_shallow_cq : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_TKN_CNT(x) \
+	(0x20000114 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_TKN_CNT_RST 0x0
+union dlb_lsp_cq_ldb_tkn_cnt {
+	struct {
+		u32 token_count : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_INFL_LIM(x) \
+	(0x20000110 + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_INFL_LIM_RST 0x0
+union dlb_lsp_cq_ldb_infl_lim {
+	struct {
+		u32 limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_LDB_INFL_CNT(x) \
+	(0x2000010c + (x) * 0x1000)
+#define DLB_LSP_CQ_LDB_INFL_CNT_RST 0x0
+union dlb_lsp_cq_ldb_infl_cnt {
+	struct {
+		u32 count : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ2QID(x, y) \
+	(0x20000104 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_LSP_CQ2QID_RST 0x0
+union dlb_lsp_cq2qid {
+	struct {
+		u32 qid_p0 : 7;
+		u32 rsvd3 : 1;
+		u32 qid_p1 : 7;
+		u32 rsvd2 : 1;
+		u32 qid_p2 : 7;
+		u32 rsvd1 : 1;
+		u32 qid_p3 : 7;
+		u32 rsvd0 : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ2PRIOV(x) \
+	(0x20000100 + (x) * 0x1000)
+#define DLB_LSP_CQ2PRIOV_RST 0x0
+union dlb_lsp_cq2priov {
+	struct {
+		u32 prio : 24;
+		u32 v : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_DSBL(x) \
+	(0x20000310 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_DSBL_RST 0x1
+union dlb_lsp_cq_dir_dsbl {
+	struct {
+		u32 disabled : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(x) \
+	(0x2000030c + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI_RST 0x0
+union dlb_lsp_cq_dir_tkn_depth_sel_dsi {
+	struct {
+		u32 token_depth_select : 4;
+		u32 disable_wb_opt : 1;
+		u32 ignore_depth : 1;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTH(x) \
+	(0x20000308 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTH_RST 0x0
+union dlb_lsp_cq_dir_tot_sch_cnth {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTL(x) \
+	(0x20000304 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TOT_SCH_CNTL_RST 0x0
+union dlb_lsp_cq_dir_tot_sch_cntl {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CQ_DIR_TKN_CNT(x) \
+	(0x20000300 + (x) * 0x1000)
+#define DLB_LSP_CQ_DIR_TKN_CNT_RST 0x0
+union dlb_lsp_cq_dir_tkn_cnt {
+	struct {
+		u32 count : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_QID2CQIDX(x, y) \
+	(0x20000400 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_LSP_QID_LDB_QID2CQIDX_RST 0x0
+union dlb_lsp_qid_ldb_qid2cqidx {
+	struct {
+		u32 cq_p0 : 8;
+		u32 cq_p1 : 8;
+		u32 cq_p2 : 8;
+		u32 cq_p3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_QID2CQIDX2(x, y) \
+	(0x20000500 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_LSP_QID_LDB_QID2CQIDX2_RST 0x0
+union dlb_lsp_qid_ldb_qid2cqidx2 {
+	struct {
+		u32 cq_p0 : 8;
+		u32 cq_p1 : 8;
+		u32 cq_p2 : 8;
+		u32 cq_p3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_ATQ_ENQUEUE_CNT(x) \
+	(0x2000066c + (x) * 0x1000)
+#define DLB_LSP_QID_ATQ_ENQUEUE_CNT_RST 0x0
+union dlb_lsp_qid_atq_enqueue_cnt {
+	struct {
+		u32 count : 15;
+		u32 rsvd0 : 17;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_INFL_LIM(x) \
+	(0x2000064c + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_INFL_LIM_RST 0x0
+union dlb_lsp_qid_ldb_infl_lim {
+	struct {
+		u32 limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_INFL_CNT(x) \
+	(0x2000062c + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_INFL_CNT_RST 0x0
+union dlb_lsp_qid_ldb_infl_cnt {
+	struct {
+		u32 count : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_AQED_ACTIVE_LIM(x) \
+	(0x20000628 + (x) * 0x1000)
+#define DLB_LSP_QID_AQED_ACTIVE_LIM_RST 0x0
+union dlb_lsp_qid_aqed_active_lim {
+	struct {
+		u32 limit : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_AQED_ACTIVE_CNT(x) \
+	(0x20000624 + (x) * 0x1000)
+#define DLB_LSP_QID_AQED_ACTIVE_CNT_RST 0x0
+union dlb_lsp_qid_aqed_active_cnt {
+	struct {
+		u32 count : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_ENQUEUE_CNT(x) \
+	(0x20000604 + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_ENQUEUE_CNT_RST 0x0
+union dlb_lsp_qid_ldb_enqueue_cnt {
+	struct {
+		u32 count : 15;
+		u32 rsvd0 : 17;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_LDB_REPLAY_CNT(x) \
+	(0x20000600 + (x) * 0x1000)
+#define DLB_LSP_QID_LDB_REPLAY_CNT_RST 0x0
+union dlb_lsp_qid_ldb_replay_cnt {
+	struct {
+		u32 count : 15;
+		u32 rsvd0 : 17;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_QID_DIR_ENQUEUE_CNT(x) \
+	(0x20000700 + (x) * 0x1000)
+#define DLB_LSP_QID_DIR_ENQUEUE_CNT_RST 0x0
+union dlb_lsp_qid_dir_enqueue_cnt {
+	struct {
+		u32 count : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CTRL_CONFIG_0 0x2800002c
+#define DLB_LSP_CTRL_CONFIG_0_RST 0x12cc
+union dlb_lsp_ctrl_config_0 {
+	struct {
+		u32 atm_cq_qid_priority_prot : 1;
+		u32 ldb_arb_ignore_empty : 1;
+		u32 ldb_arb_mode : 2;
+		u32 ldb_arb_threshold : 18;
+		u32 cfg_cq_sla_upd_always : 1;
+		u32 cfg_cq_wcn_upd_always : 1;
+		u32 spare : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_1 0x28000028
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_1_RST 0x0
+union dlb_lsp_cfg_arb_weight_atm_nalb_qid_1 {
+	struct {
+		u32 slot4_weight : 8;
+		u32 slot5_weight : 8;
+		u32 slot6_weight : 8;
+		u32 slot7_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_0 0x28000024
+#define DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_0_RST 0x0
+union dlb_lsp_cfg_arb_weight_atm_nalb_qid_0 {
+	struct {
+		u32 slot0_weight : 8;
+		u32 slot1_weight : 8;
+		u32 slot2_weight : 8;
+		u32 slot3_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_1 0x28000020
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_1_RST 0x0
+union dlb_lsp_cfg_arb_weight_ldb_qid_1 {
+	struct {
+		u32 slot4_weight : 8;
+		u32 slot5_weight : 8;
+		u32 slot6_weight : 8;
+		u32 slot7_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_0 0x2800001c
+#define DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_0_RST 0x0
+union dlb_lsp_cfg_arb_weight_ldb_qid_0 {
+	struct {
+		u32 slot0_weight : 8;
+		u32 slot1_weight : 8;
+		u32 slot2_weight : 8;
+		u32 slot3_weight : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_LDB_SCHED_CTRL 0x28100000
+#define DLB_LSP_LDB_SCHED_CTRL_RST 0x0
+union dlb_lsp_ldb_sched_ctrl {
+	struct {
+		u32 cq : 8;
+		u32 qidix : 3;
+		u32 value : 1;
+		u32 nalb_haswork_v : 1;
+		u32 rlist_haswork_v : 1;
+		u32 slist_haswork_v : 1;
+		u32 inflight_ok_v : 1;
+		u32 aqed_nfull_v : 1;
+		u32 spare0 : 15;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_DIR_SCH_CNT_H 0x2820000c
+#define DLB_LSP_DIR_SCH_CNT_H_RST 0x0
+union dlb_lsp_dir_sch_cnt_h {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_DIR_SCH_CNT_L 0x28200008
+#define DLB_LSP_DIR_SCH_CNT_L_RST 0x0
+union dlb_lsp_dir_sch_cnt_l {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_LDB_SCH_CNT_H 0x28200004
+#define DLB_LSP_LDB_SCH_CNT_H_RST 0x0
+union dlb_lsp_ldb_sch_cnt_h {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_LSP_LDB_SCH_CNT_L 0x28200000
+#define DLB_LSP_LDB_SCH_CNT_L_RST 0x0
+union dlb_lsp_ldb_sch_cnt_l {
+	struct {
+		u32 count : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_DIR_CSR_CTRL 0x38000018
+#define DLB_DP_DIR_CSR_CTRL_RST 0xc0000000
+union dlb_dp_dir_csr_ctrl {
+	struct {
+		u32 cfg_int_dis : 1;
+		u32 cfg_int_dis_sbe : 1;
+		u32 cfg_int_dis_mbe : 1;
+		u32 spare0 : 27;
+		u32 cfg_vasr_dis : 1;
+		u32 cfg_int_dis_synd : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_1 0x38000014
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_1_RST 0xfffefdfc
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_dir_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_0 0x38000010
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_0_RST 0xfbfaf9f8
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_dir_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1 0x3800000c
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1_RST 0xfffefdfc
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_replay_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0 0x38000008
+#define DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0_RST 0xfbfaf9f8
+union dlb_dp_cfg_ctrl_arb_weights_tqpri_replay_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_1 0x6800001c
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_1_RST 0xfffefdfc
+union dlb_nalb_pipe_ctrl_arb_weights_tqpri_nalb_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_0 0x68000018
+#define DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_0_RST 0xfbfaf9f8
+union dlb_nalb_pipe_ctrl_arb_weights_tqpri_nalb_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_1 0x68000014
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_1_RST 0xfffefdfc
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_atq_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_0 0x68000010
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_0_RST 0xfbfaf9f8
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_atq_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1 0x6800000c
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1_RST 0xfffefdfc
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_replay_1 {
+	struct {
+		u32 pri4 : 8;
+		u32 pri5 : 8;
+		u32 pri6 : 8;
+		u32 pri7 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0 0x68000008
+#define DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0_RST 0xfbfaf9f8
+union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_replay_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_ATM_PIPE_QID_LDB_QID2CQIDX(x, y) \
+	(0x70000000 + (x) * 0x1000 + (y) * 0x4)
+#define DLB_ATM_PIPE_QID_LDB_QID2CQIDX_RST 0x0
+union dlb_atm_pipe_qid_ldb_qid2cqidx {
+	struct {
+		u32 cq_p0 : 8;
+		u32 cq_p1 : 8;
+		u32 cq_p2 : 8;
+		u32 cq_p3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_ATM_PIPE_CFG_CTRL_ARB_WEIGHTS_SCHED_BIN 0x7800000c
+#define DLB_ATM_PIPE_CFG_CTRL_ARB_WEIGHTS_SCHED_BIN_RST 0xfffefdfc
+union dlb_atm_pipe_cfg_ctrl_arb_weights_sched_bin {
+	struct {
+		u32 bin0 : 8;
+		u32 bin1 : 8;
+		u32 bin2 : 8;
+		u32 bin3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_ATM_PIPE_CTRL_ARB_WEIGHTS_RDY_BIN 0x78000008
+#define DLB_ATM_PIPE_CTRL_ARB_WEIGHTS_RDY_BIN_RST 0xfffefdfc
+union dlb_atm_pipe_ctrl_arb_weights_rdy_bin {
+	struct {
+		u32 bin0 : 8;
+		u32 bin1 : 8;
+		u32 bin2 : 8;
+		u32 bin3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_QID_FID_LIM(x) \
+	(0x80000014 + (x) * 0x1000)
+#define DLB_AQED_PIPE_QID_FID_LIM_RST 0x7ff
+union dlb_aqed_pipe_qid_fid_lim {
+	struct {
+		u32 qid_fid_limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_POP_PTR(x) \
+	(0x80000010 + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_POP_PTR_RST 0x0
+union dlb_aqed_pipe_fl_pop_ptr {
+	struct {
+		u32 pop_ptr : 11;
+		u32 generation : 1;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_PUSH_PTR(x) \
+	(0x8000000c + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_PUSH_PTR_RST 0x0
+union dlb_aqed_pipe_fl_push_ptr {
+	struct {
+		u32 push_ptr : 11;
+		u32 generation : 1;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_BASE(x) \
+	(0x80000008 + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_BASE_RST 0x0
+union dlb_aqed_pipe_fl_base {
+	struct {
+		u32 base : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_FL_LIM(x) \
+	(0x80000004 + (x) * 0x1000)
+#define DLB_AQED_PIPE_FL_LIM_RST 0x800
+union dlb_aqed_pipe_fl_lim {
+	struct {
+		u32 limit : 11;
+		u32 freelist_disable : 1;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_AQED_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATM_0 0x88000008
+#define DLB_AQED_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATM_0_RST 0xfffe
+union dlb_aqed_pipe_cfg_ctrl_arb_weights_tqpri_atm_0 {
+	struct {
+		u32 pri0 : 8;
+		u32 pri1 : 8;
+		u32 pri2 : 8;
+		u32 pri3 : 8;
+	} field;
+	u32 val;
+};
+
+#define DLB_RO_PIPE_QID2GRPSLT(x) \
+	(0x90000000 + (x) * 0x1000)
+#define DLB_RO_PIPE_QID2GRPSLT_RST 0x0
+union dlb_ro_pipe_qid2grpslt {
+	struct {
+		u32 slot : 5;
+		u32 rsvd1 : 3;
+		u32 group : 2;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_RO_PIPE_GRP_SN_MODE 0x98000008
+#define DLB_RO_PIPE_GRP_SN_MODE_RST 0x0
+union dlb_ro_pipe_grp_sn_mode {
+	struct {
+		u32 sn_mode_0 : 3;
+		u32 reserved0 : 5;
+		u32 sn_mode_1 : 3;
+		u32 reserved1 : 5;
+		u32 sn_mode_2 : 3;
+		u32 reserved2 : 5;
+		u32 sn_mode_3 : 3;
+		u32 reserved3 : 5;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CFG_DIR_PP_SW_ALARM_EN(x) \
+	(0xa000003c + (x) * 0x1000)
+#define DLB_CHP_CFG_DIR_PP_SW_ALARM_EN_RST 0x1
+union dlb_chp_cfg_dir_pp_sw_alarm_en {
+	struct {
+		u32 alarm_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_WD_ENB(x) \
+	(0xa0000038 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_WD_ENB_RST 0x0
+union dlb_chp_dir_cq_wd_enb {
+	struct {
+		u32 wd_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_LDB_PP2POOL(x) \
+	(0xa0000034 + (x) * 0x1000)
+#define DLB_CHP_DIR_LDB_PP2POOL_RST 0x0
+union dlb_chp_dir_ldb_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_DIR_PP2POOL(x) \
+	(0xa0000030 + (x) * 0x1000)
+#define DLB_CHP_DIR_DIR_PP2POOL_RST 0x0
+union dlb_chp_dir_dir_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_CRD_CNT(x) \
+	(0xa000002c + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_CRD_CNT_RST 0x0
+union dlb_chp_dir_pp_ldb_crd_cnt {
+	struct {
+		u32 count : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_CRD_CNT(x) \
+	(0xa0000028 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_CRD_CNT_RST 0x0
+union dlb_chp_dir_pp_dir_crd_cnt {
+	struct {
+		u32 count : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_TMR_THRESHOLD(x) \
+	(0xa0000024 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_TMR_THRESHOLD_RST 0x0
+union dlb_chp_dir_cq_tmr_threshold {
+	struct {
+		u32 timer_thrsh : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INT_ENB(x) \
+	(0xa0000020 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_INT_ENB_RST 0x0
+union dlb_chp_dir_cq_int_enb {
+	struct {
+		u32 en_tim : 1;
+		u32 en_depth : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INT_DEPTH_THRSH(x) \
+	(0xa000001c + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_INT_DEPTH_THRSH_RST 0x0
+union dlb_chp_dir_cq_int_depth_thrsh {
+	struct {
+		u32 depth_threshold : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(x) \
+	(0xa0000018 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_TKN_DEPTH_SEL_RST 0x0
+union dlb_chp_dir_cq_tkn_depth_sel {
+	struct {
+		u32 token_depth_select : 4;
+		u32 rsvd0 : 28;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(x) \
+	(0xa0000014 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT_RST 0x1
+union dlb_chp_dir_pp_ldb_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(x) \
+	(0xa0000010 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT_RST 0x1
+union dlb_chp_dir_pp_dir_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_CRD_LWM(x) \
+	(0xa000000c + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_CRD_LWM_RST 0x0
+union dlb_chp_dir_pp_ldb_crd_lwm {
+	struct {
+		u32 lwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_CRD_HWM(x) \
+	(0xa0000008 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_CRD_HWM_RST 0x0
+union dlb_chp_dir_pp_ldb_crd_hwm {
+	struct {
+		u32 hwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_CRD_LWM(x) \
+	(0xa0000004 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_CRD_LWM_RST 0x0
+union dlb_chp_dir_pp_dir_crd_lwm {
+	struct {
+		u32 lwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_CRD_HWM(x) \
+	(0xa0000000 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_CRD_HWM_RST 0x0
+union dlb_chp_dir_pp_dir_crd_hwm {
+	struct {
+		u32 hwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CFG_LDB_PP_SW_ALARM_EN(x) \
+	(0xa0000148 + (x) * 0x1000)
+#define DLB_CHP_CFG_LDB_PP_SW_ALARM_EN_RST 0x1
+union dlb_chp_cfg_ldb_pp_sw_alarm_en {
+	struct {
+		u32 alarm_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_WD_ENB(x) \
+	(0xa0000144 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_WD_ENB_RST 0x0
+union dlb_chp_ldb_cq_wd_enb {
+	struct {
+		u32 wd_enable : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_SN_CHK_ENBL(x) \
+	(0xa0000140 + (x) * 0x1000)
+#define DLB_CHP_SN_CHK_ENBL_RST 0x0
+union dlb_chp_sn_chk_enbl {
+	struct {
+		u32 en : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_BASE(x) \
+	(0xa000013c + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_BASE_RST 0x0
+union dlb_chp_hist_list_base {
+	struct {
+		u32 base : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_LIM(x) \
+	(0xa0000138 + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_LIM_RST 0x0
+union dlb_chp_hist_list_lim {
+	struct {
+		u32 limit : 13;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_LDB_PP2POOL(x) \
+	(0xa0000134 + (x) * 0x1000)
+#define DLB_CHP_LDB_LDB_PP2POOL_RST 0x0
+union dlb_chp_ldb_ldb_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_DIR_PP2POOL(x) \
+	(0xa0000130 + (x) * 0x1000)
+#define DLB_CHP_LDB_DIR_PP2POOL_RST 0x0
+union dlb_chp_ldb_dir_pp2pool {
+	struct {
+		u32 pool : 6;
+		u32 rsvd0 : 26;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_CRD_CNT(x) \
+	(0xa000012c + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_CRD_CNT_RST 0x0
+union dlb_chp_ldb_pp_ldb_crd_cnt {
+	struct {
+		u32 count : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_CRD_CNT(x) \
+	(0xa0000128 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_CRD_CNT_RST 0x0
+union dlb_chp_ldb_pp_dir_crd_cnt {
+	struct {
+		u32 count : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_TMR_THRESHOLD(x) \
+	(0xa0000124 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_TMR_THRESHOLD_RST 0x0
+union dlb_chp_ldb_cq_tmr_threshold {
+	struct {
+		u32 thrsh : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INT_ENB(x) \
+	(0xa0000120 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_INT_ENB_RST 0x0
+union dlb_chp_ldb_cq_int_enb {
+	struct {
+		u32 en_tim : 1;
+		u32 en_depth : 1;
+		u32 rsvd0 : 30;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INT_DEPTH_THRSH(x) \
+	(0xa000011c + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_INT_DEPTH_THRSH_RST 0x0
+union dlb_chp_ldb_cq_int_depth_thrsh {
+	struct {
+		u32 depth_threshold : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(x) \
+	(0xa0000118 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_TKN_DEPTH_SEL_RST 0x0
+union dlb_chp_ldb_cq_tkn_depth_sel {
+	struct {
+		u32 token_depth_select : 4;
+		u32 rsvd0 : 28;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(x) \
+	(0xa0000114 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT_RST 0x1
+union dlb_chp_ldb_pp_ldb_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(x) \
+	(0xa0000110 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT_RST 0x1
+union dlb_chp_ldb_pp_dir_min_crd_qnt {
+	struct {
+		u32 quanta : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_CRD_LWM(x) \
+	(0xa000010c + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_CRD_LWM_RST 0x0
+union dlb_chp_ldb_pp_ldb_crd_lwm {
+	struct {
+		u32 lwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_CRD_HWM(x) \
+	(0xa0000108 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_CRD_HWM_RST 0x0
+union dlb_chp_ldb_pp_ldb_crd_hwm {
+	struct {
+		u32 hwm : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_CRD_LWM(x) \
+	(0xa0000104 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_CRD_LWM_RST 0x0
+union dlb_chp_ldb_pp_dir_crd_lwm {
+	struct {
+		u32 lwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_CRD_HWM(x) \
+	(0xa0000100 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_CRD_HWM_RST 0x0
+union dlb_chp_ldb_pp_dir_crd_hwm {
+	struct {
+		u32 hwm : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_DEPTH(x) \
+	(0xa0000218 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_DEPTH_RST 0x0
+union dlb_chp_dir_cq_depth {
+	struct {
+		u32 cq_depth : 11;
+		u32 rsvd0 : 21;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_WPTR(x) \
+	(0xa0000214 + (x) * 0x1000)
+#define DLB_CHP_DIR_CQ_WPTR_RST 0x0
+union dlb_chp_dir_cq_wptr {
+	struct {
+		u32 write_pointer : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_LDB_PUSH_PTR(x) \
+	(0xa0000210 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_LDB_PUSH_PTR_RST 0x0
+union dlb_chp_dir_pp_ldb_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_DIR_PUSH_PTR(x) \
+	(0xa000020c + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_DIR_PUSH_PTR_RST 0x0
+union dlb_chp_dir_pp_dir_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_STATE_RESET(x) \
+	(0xa0000204 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_STATE_RESET_RST 0x0
+union dlb_chp_dir_pp_state_reset {
+	struct {
+		u32 rsvd1 : 7;
+		u32 dir_type : 1;
+		u32 rsvd0 : 23;
+		u32 reset_pp_state : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_PP_CRD_REQ_STATE(x) \
+	(0xa0000200 + (x) * 0x1000)
+#define DLB_CHP_DIR_PP_CRD_REQ_STATE_RST 0x0
+union dlb_chp_dir_pp_crd_req_state {
+	struct {
+		u32 dir_crd_req_active_valid : 1;
+		u32 dir_crd_req_active_check : 1;
+		u32 dir_crd_req_active_busy : 1;
+		u32 rsvd1 : 1;
+		u32 ldb_crd_req_active_valid : 1;
+		u32 ldb_crd_req_active_check : 1;
+		u32 ldb_crd_req_active_busy : 1;
+		u32 rsvd0 : 1;
+		u32 no_pp_credit_update : 1;
+		u32 crd_req_state : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_DEPTH(x) \
+	(0xa0000320 + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_DEPTH_RST 0x0
+union dlb_chp_ldb_cq_depth {
+	struct {
+		u32 depth : 11;
+		u32 reserved : 2;
+		u32 rsvd0 : 19;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_WPTR(x) \
+	(0xa000031c + (x) * 0x1000)
+#define DLB_CHP_LDB_CQ_WPTR_RST 0x0
+union dlb_chp_ldb_cq_wptr {
+	struct {
+		u32 write_pointer : 10;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_LDB_PUSH_PTR(x) \
+	(0xa0000318 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_LDB_PUSH_PTR_RST 0x0
+union dlb_chp_ldb_pp_ldb_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_DIR_PUSH_PTR(x) \
+	(0xa0000314 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_DIR_PUSH_PTR_RST 0x0
+union dlb_chp_ldb_pp_dir_push_ptr {
+	struct {
+		u32 push_pointer : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_POP_PTR(x) \
+	(0xa000030c + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_POP_PTR_RST 0x0
+union dlb_chp_hist_list_pop_ptr {
+	struct {
+		u32 pop_ptr : 13;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_HIST_LIST_PUSH_PTR(x) \
+	(0xa0000308 + (x) * 0x1000)
+#define DLB_CHP_HIST_LIST_PUSH_PTR_RST 0x0
+union dlb_chp_hist_list_push_ptr {
+	struct {
+		u32 push_ptr : 13;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_STATE_RESET(x) \
+	(0xa0000304 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_STATE_RESET_RST 0x0
+union dlb_chp_ldb_pp_state_reset {
+	struct {
+		u32 rsvd1 : 7;
+		u32 dir_type : 1;
+		u32 rsvd0 : 23;
+		u32 reset_pp_state : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_PP_CRD_REQ_STATE(x) \
+	(0xa0000300 + (x) * 0x1000)
+#define DLB_CHP_LDB_PP_CRD_REQ_STATE_RST 0x0
+union dlb_chp_ldb_pp_crd_req_state {
+	struct {
+		u32 dir_crd_req_active_valid : 1;
+		u32 dir_crd_req_active_check : 1;
+		u32 dir_crd_req_active_busy : 1;
+		u32 rsvd1 : 1;
+		u32 ldb_crd_req_active_valid : 1;
+		u32 ldb_crd_req_active_check : 1;
+		u32 ldb_crd_req_active_busy : 1;
+		u32 rsvd0 : 1;
+		u32 no_pp_credit_update : 1;
+		u32 crd_req_state : 23;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_ORD_QID_SN(x) \
+	(0xa0000408 + (x) * 0x1000)
+#define DLB_CHP_ORD_QID_SN_RST 0x0
+union dlb_chp_ord_qid_sn {
+	struct {
+		u32 sn : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_ORD_QID_SN_MAP(x) \
+	(0xa0000404 + (x) * 0x1000)
+#define DLB_CHP_ORD_QID_SN_MAP_RST 0x0
+union dlb_chp_ord_qid_sn_map {
+	struct {
+		u32 mode : 3;
+		u32 slot : 5;
+		u32 grp : 2;
+		u32 rsvd0 : 22;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_POOL_CRD_CNT(x) \
+	(0xa000050c + (x) * 0x1000)
+#define DLB_CHP_LDB_POOL_CRD_CNT_RST 0x0
+union dlb_chp_ldb_pool_crd_cnt {
+	struct {
+		u32 count : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_BASE(x) \
+	(0xa0000508 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_BASE_RST 0x0
+union dlb_chp_qed_fl_base {
+	struct {
+		u32 base : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_LIM(x) \
+	(0xa0000504 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_LIM_RST 0x8000
+union dlb_chp_qed_fl_lim {
+	struct {
+		u32 limit : 14;
+		u32 rsvd1 : 1;
+		u32 freelist_disable : 1;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_POOL_CRD_LIM(x) \
+	(0xa0000500 + (x) * 0x1000)
+#define DLB_CHP_LDB_POOL_CRD_LIM_RST 0x0
+union dlb_chp_ldb_pool_crd_lim {
+	struct {
+		u32 limit : 16;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_POP_PTR(x) \
+	(0xa0000604 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_POP_PTR_RST 0x0
+union dlb_chp_qed_fl_pop_ptr {
+	struct {
+		u32 pop_ptr : 14;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_QED_FL_PUSH_PTR(x) \
+	(0xa0000600 + (x) * 0x1000)
+#define DLB_CHP_QED_FL_PUSH_PTR_RST 0x0
+union dlb_chp_qed_fl_push_ptr {
+	struct {
+		u32 push_ptr : 14;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 16;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_POOL_CRD_CNT(x) \
+	(0xa000070c + (x) * 0x1000)
+#define DLB_CHP_DIR_POOL_CRD_CNT_RST 0x0
+union dlb_chp_dir_pool_crd_cnt {
+	struct {
+		u32 count : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_BASE(x) \
+	(0xa0000708 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_BASE_RST 0x0
+union dlb_chp_dqed_fl_base {
+	struct {
+		u32 base : 12;
+		u32 rsvd0 : 20;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_LIM(x) \
+	(0xa0000704 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_LIM_RST 0x2000
+union dlb_chp_dqed_fl_lim {
+	struct {
+		u32 limit : 12;
+		u32 rsvd1 : 1;
+		u32 freelist_disable : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_POOL_CRD_LIM(x) \
+	(0xa0000700 + (x) * 0x1000)
+#define DLB_CHP_DIR_POOL_CRD_LIM_RST 0x0
+union dlb_chp_dir_pool_crd_lim {
+	struct {
+		u32 limit : 14;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_POP_PTR(x) \
+	(0xa0000804 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_POP_PTR_RST 0x0
+union dlb_chp_dqed_fl_pop_ptr {
+	struct {
+		u32 pop_ptr : 12;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DQED_FL_PUSH_PTR(x) \
+	(0xa0000800 + (x) * 0x1000)
+#define DLB_CHP_DQED_FL_PUSH_PTR_RST 0x0
+union dlb_chp_dqed_fl_push_ptr {
+	struct {
+		u32 push_ptr : 12;
+		u32 reserved0 : 1;
+		u32 generation : 1;
+		u32 rsvd0 : 18;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CTRL_DIAG_02 0xa8000154
+#define DLB_CHP_CTRL_DIAG_02_RST 0x0
+union dlb_chp_ctrl_diag_02 {
+	struct {
+		u32 control : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_CFG_CHP_CSR_CTRL 0xa8000130
+#define DLB_CHP_CFG_CHP_CSR_CTRL_RST 0xc0003fff
+#define DLB_CHP_CFG_EXCESS_TOKENS_SHIFT 12
+union dlb_chp_cfg_chp_csr_ctrl {
+	struct {
+		u32 int_inf_alarm_enable_0 : 1;
+		u32 int_inf_alarm_enable_1 : 1;
+		u32 int_inf_alarm_enable_2 : 1;
+		u32 int_inf_alarm_enable_3 : 1;
+		u32 int_inf_alarm_enable_4 : 1;
+		u32 int_inf_alarm_enable_5 : 1;
+		u32 int_inf_alarm_enable_6 : 1;
+		u32 int_inf_alarm_enable_7 : 1;
+		u32 int_inf_alarm_enable_8 : 1;
+		u32 int_inf_alarm_enable_9 : 1;
+		u32 int_inf_alarm_enable_10 : 1;
+		u32 int_inf_alarm_enable_11 : 1;
+		u32 int_inf_alarm_enable_12 : 1;
+		u32 int_cor_alarm_enable : 1;
+		u32 csr_control_spare : 14;
+		u32 cfg_vasr_dis : 1;
+		u32 counter_clear : 1;
+		u32 blk_cor_report : 1;
+		u32 blk_cor_synd : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INTR_ARMED1 0xa8000068
+#define DLB_CHP_LDB_CQ_INTR_ARMED1_RST 0x0
+union dlb_chp_ldb_cq_intr_armed1 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_LDB_CQ_INTR_ARMED0 0xa8000064
+#define DLB_CHP_LDB_CQ_INTR_ARMED0_RST 0x0
+union dlb_chp_ldb_cq_intr_armed0 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED3 0xa8000024
+#define DLB_CHP_DIR_CQ_INTR_ARMED3_RST 0x0
+union dlb_chp_dir_cq_intr_armed3 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED2 0xa8000020
+#define DLB_CHP_DIR_CQ_INTR_ARMED2_RST 0x0
+union dlb_chp_dir_cq_intr_armed2 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED1 0xa800001c
+#define DLB_CHP_DIR_CQ_INTR_ARMED1_RST 0x0
+union dlb_chp_dir_cq_intr_armed1 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CHP_DIR_CQ_INTR_ARMED0 0xa8000018
+#define DLB_CHP_DIR_CQ_INTR_ARMED0_RST 0x0
+union dlb_chp_dir_cq_intr_armed0 {
+	struct {
+		u32 armed : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_CFG_MSTR_DIAG_RESET_STS 0xb8000004
+#define DLB_CFG_MSTR_DIAG_RESET_STS_RST 0x1ff
+union dlb_cfg_mstr_diag_reset_sts {
+	struct {
+		u32 chp_pf_reset_done : 1;
+		u32 rop_pf_reset_done : 1;
+		u32 lsp_pf_reset_done : 1;
+		u32 nalb_pf_reset_done : 1;
+		u32 ap_pf_reset_done : 1;
+		u32 dp_pf_reset_done : 1;
+		u32 qed_pf_reset_done : 1;
+		u32 dqed_pf_reset_done : 1;
+		u32 aqed_pf_reset_done : 1;
+		u32 rsvd1 : 6;
+		u32 pf_reset_active : 1;
+		u32 chp_vf_reset_done : 1;
+		u32 rop_vf_reset_done : 1;
+		u32 lsp_vf_reset_done : 1;
+		u32 nalb_vf_reset_done : 1;
+		u32 ap_vf_reset_done : 1;
+		u32 dp_vf_reset_done : 1;
+		u32 qed_vf_reset_done : 1;
+		u32 dqed_vf_reset_done : 1;
+		u32 aqed_vf_reset_done : 1;
+		u32 rsvd0 : 6;
+		u32 vf_reset_active : 1;
+	} field;
+	u32 val;
+};
+
+#define DLB_CFG_MSTR_BCAST_RESET_VF_START 0xc8100000
+#define DLB_CFG_MSTR_BCAST_RESET_VF_START_RST 0x0
+/* HW Reset Types */
+#define VF_RST_TYPE_CQ_LDB   0
+#define VF_RST_TYPE_QID_LDB  1
+#define VF_RST_TYPE_POOL_LDB 2
+#define VF_RST_TYPE_CQ_DIR   8
+#define VF_RST_TYPE_QID_DIR  9
+#define VF_RST_TYPE_POOL_DIR 10
+union dlb_cfg_mstr_bcast_reset_vf_start {
+	struct {
+		u32 vf_reset_start : 1;
+		u32 reserved : 3;
+		u32 vf_reset_type : 4;
+		u32 vf_reset_id : 24;
+	} field;
+	u32 val;
+};
+
+#define DLB_FUNC_VF_VF2PF_MAILBOX_BYTES 256
+#define DLB_FUNC_VF_VF2PF_MAILBOX(x) \
+	(0x1000 + (x) * 0x4)
+#define DLB_FUNC_VF_VF2PF_MAILBOX_RST 0x0
+union dlb_func_vf_vf2pf_mailbox {
+	struct {
+		u32 msg : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_FUNC_VF_VF2PF_MAILBOX_ISR 0x1f00
+#define DLB_FUNC_VF_VF2PF_MAILBOX_ISR_RST 0x0
+union dlb_func_vf_vf2pf_mailbox_isr {
+	struct {
+		u32 isr : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_FUNC_VF_PF2VF_MAILBOX_BYTES 64
+#define DLB_FUNC_VF_PF2VF_MAILBOX(x) \
+	(0x2000 + (x) * 0x4)
+#define DLB_FUNC_VF_PF2VF_MAILBOX_RST 0x0
+union dlb_func_vf_pf2vf_mailbox {
+	struct {
+		u32 msg : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_FUNC_VF_PF2VF_MAILBOX_ISR 0x2f00
+#define DLB_FUNC_VF_PF2VF_MAILBOX_ISR_RST 0x0
+union dlb_func_vf_pf2vf_mailbox_isr {
+	struct {
+		u32 pf_isr : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_FUNC_VF_VF_MSI_ISR_PEND 0x2f10
+#define DLB_FUNC_VF_VF_MSI_ISR_PEND_RST 0x0
+union dlb_func_vf_vf_msi_isr_pend {
+	struct {
+		u32 isr_pend : 32;
+	} field;
+	u32 val;
+};
+
+#define DLB_FUNC_VF_VF_RESET_IN_PROGRESS 0x3000
+#define DLB_FUNC_VF_VF_RESET_IN_PROGRESS_RST 0x1
+union dlb_func_vf_vf_reset_in_progress {
+	struct {
+		u32 reset_in_progress : 1;
+		u32 rsvd0 : 31;
+	} field;
+	u32 val;
+};
+
+#define DLB_FUNC_VF_VF_MSI_ISR 0x4000
+#define DLB_FUNC_VF_VF_MSI_ISR_RST 0x0
+union dlb_func_vf_vf_msi_isr {
+	struct {
+		u32 vf_msi_isr : 32;
+	} field;
+	u32 val;
+};
+
+#endif /* __DLB_REGS_H */
diff --git a/drivers/event/dlb/pf/base/dlb_resource.c b/drivers/event/dlb/pf/base/dlb_resource.c
new file mode 100644
index 0000000..4d67eb8
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_resource.c
@@ -0,0 +1,9700 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include "dlb_hw_types.h"
+#include "dlb_user.h"
+#include "dlb_resource.h"
+#include "dlb_osdep.h"
+#include "dlb_osdep_bitmap.h"
+#include "dlb_osdep_types.h"
+#include "dlb_regs.h"
+#include "dlb_mbox.h"
+
+#define DLB_DOM_LIST_HEAD(head, type) \
+	DLB_LIST_HEAD((head), type, domain_list)
+
+#define DLB_FUNC_LIST_HEAD(head, type) \
+	DLB_LIST_HEAD((head), type, func_list)
+
+#define DLB_DOM_LIST_FOR(head, ptr, iter) \
+	DLB_LIST_FOR_EACH(head, ptr, domain_list, iter)
+
+#define DLB_FUNC_LIST_FOR(head, ptr, iter) \
+	DLB_LIST_FOR_EACH(head, ptr, func_list, iter)
+
+#define DLB_DOM_LIST_FOR_SAFE(head, ptr, ptr_tmp, it, it_tmp) \
+	DLB_LIST_FOR_EACH_SAFE((head), ptr, ptr_tmp, domain_list, it, it_tmp)
+
+#define DLB_FUNC_LIST_FOR_SAFE(head, ptr, ptr_tmp, it, it_tmp) \
+	DLB_LIST_FOR_EACH_SAFE((head), ptr, ptr_tmp, func_list, it, it_tmp)
+
+/* The PF driver cannot assume that a register write will affect subsequent HCW
+ * writes. To ensure a write completes, the driver must read back a CSR. This
+ * function only need be called for configuration that can occur after the
+ * domain has started; prior to starting, applications can't send HCWs.
+ */
+static inline void dlb_flush_csr(struct dlb_hw *hw)
+{
+	DLB_CSR_RD(hw, DLB_SYS_TOTAL_VAS);
+}
+
+static void dlb_init_fn_rsrc_lists(struct dlb_function_resources *rsrc)
+{
+	dlb_list_init_head(&rsrc->avail_domains);
+	dlb_list_init_head(&rsrc->used_domains);
+	dlb_list_init_head(&rsrc->avail_ldb_queues);
+	dlb_list_init_head(&rsrc->avail_ldb_ports);
+	dlb_list_init_head(&rsrc->avail_dir_pq_pairs);
+	dlb_list_init_head(&rsrc->avail_ldb_credit_pools);
+	dlb_list_init_head(&rsrc->avail_dir_credit_pools);
+}
+
+static void dlb_init_domain_rsrc_lists(struct dlb_domain *domain)
+{
+	dlb_list_init_head(&domain->used_ldb_queues);
+	dlb_list_init_head(&domain->used_ldb_ports);
+	dlb_list_init_head(&domain->used_dir_pq_pairs);
+	dlb_list_init_head(&domain->used_ldb_credit_pools);
+	dlb_list_init_head(&domain->used_dir_credit_pools);
+	dlb_list_init_head(&domain->avail_ldb_queues);
+	dlb_list_init_head(&domain->avail_ldb_ports);
+	dlb_list_init_head(&domain->avail_dir_pq_pairs);
+	dlb_list_init_head(&domain->avail_ldb_credit_pools);
+	dlb_list_init_head(&domain->avail_dir_credit_pools);
+}
+
+int dlb_resource_init(struct dlb_hw *hw)
+{
+	struct dlb_list_entry *list;
+	unsigned int i;
+
+	/* For optimal load-balancing, ports that map to one or more QIDs in
+	 * common should not be in numerical sequence. This is application
+	 * dependent, but the driver interleaves port IDs as much as possible
+	 * to reduce the likelihood of this. This initial allocation maximizes
+	 * the average distance between an ID and its immediate neighbors (i.e.
+	 * the distance from 1 to 0 and to 2, the distance from 2 to 1 and to
+	 * 3, etc.).
+	 */
+	u32 init_ldb_port_allocation[DLB_MAX_NUM_LDB_PORTS] = {
+		0,  31, 62, 29, 60, 27, 58, 25, 56, 23, 54, 21, 52, 19, 50, 17,
+		48, 15, 46, 13, 44, 11, 42,  9, 40,  7, 38,  5, 36,  3, 34, 1,
+		32, 63, 30, 61, 28, 59, 26, 57, 24, 55, 22, 53, 20, 51, 18, 49,
+		16, 47, 14, 45, 12, 43, 10, 41,  8, 39,  6, 37,  4, 35,  2, 33
+	};
+
+	/* Zero-out resource tracking data structures */
+	memset(&hw->rsrcs, 0, sizeof(hw->rsrcs));
+	memset(&hw->pf, 0, sizeof(hw->pf));
+
+	dlb_init_fn_rsrc_lists(&hw->pf);
+
+	for (i = 0; i < DLB_MAX_NUM_VFS; i++) {
+		memset(&hw->vf[i], 0, sizeof(hw->vf[i]));
+		dlb_init_fn_rsrc_lists(&hw->vf[i]);
+	}
+
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		memset(&hw->domains[i], 0, sizeof(hw->domains[i]));
+		dlb_init_domain_rsrc_lists(&hw->domains[i]);
+		hw->domains[i].parent_func = &hw->pf;
+	}
+
+	/* Give all resources to the PF driver */
+	hw->pf.num_avail_domains = DLB_MAX_NUM_DOMAINS;
+	for (i = 0; i < hw->pf.num_avail_domains; i++) {
+		list = &hw->domains[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_domains, list);
+	}
+
+	hw->pf.num_avail_ldb_queues = DLB_MAX_NUM_LDB_QUEUES;
+	for (i = 0; i < hw->pf.num_avail_ldb_queues; i++) {
+		list = &hw->rsrcs.ldb_queues[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_ldb_queues, list);
+	}
+
+	hw->pf.num_avail_ldb_ports = DLB_MAX_NUM_LDB_PORTS;
+	for (i = 0; i < hw->pf.num_avail_ldb_ports; i++) {
+		struct dlb_ldb_port *port;
+
+		port = &hw->rsrcs.ldb_ports[init_ldb_port_allocation[i]];
+
+		dlb_list_add(&hw->pf.avail_ldb_ports, &port->func_list);
+	}
+
+	hw->pf.num_avail_dir_pq_pairs = DLB_MAX_NUM_DIR_PORTS;
+	for (i = 0; i < hw->pf.num_avail_dir_pq_pairs; i++) {
+		list = &hw->rsrcs.dir_pq_pairs[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_dir_pq_pairs, list);
+	}
+
+	hw->pf.num_avail_ldb_credit_pools = DLB_MAX_NUM_LDB_CREDIT_POOLS;
+	for (i = 0; i < hw->pf.num_avail_ldb_credit_pools; i++) {
+		list = &hw->rsrcs.ldb_credit_pools[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_ldb_credit_pools, list);
+	}
+
+	hw->pf.num_avail_dir_credit_pools = DLB_MAX_NUM_DIR_CREDIT_POOLS;
+	for (i = 0; i < hw->pf.num_avail_dir_credit_pools; i++) {
+		list = &hw->rsrcs.dir_credit_pools[i].func_list;
+
+		dlb_list_add(&hw->pf.avail_dir_credit_pools, list);
+	}
+
+	/* There are 5120 history list entries, which allows us to overprovision
+	 * the inflight limit (4096) by 1k.
+	 */
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_hist_list_entries,
+			     DLB_MAX_NUM_HIST_LIST_ENTRIES))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_hist_list_entries))
+		return -1;
+
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_qed_freelist_entries,
+			     DLB_MAX_NUM_LDB_CREDITS))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_qed_freelist_entries))
+		return -1;
+
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_dqed_freelist_entries,
+			     DLB_MAX_NUM_DIR_CREDITS))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_dqed_freelist_entries))
+		return -1;
+
+	if (dlb_bitmap_alloc(hw,
+			     &hw->pf.avail_aqed_freelist_entries,
+			     DLB_MAX_NUM_AQOS_ENTRIES))
+		return -1;
+
+	if (dlb_bitmap_fill(hw->pf.avail_aqed_freelist_entries))
+		return -1;
+
+	for (i = 0; i < DLB_MAX_NUM_VFS; i++) {
+		if (dlb_bitmap_alloc(hw,
+				     &hw->vf[i].avail_hist_list_entries,
+				     DLB_MAX_NUM_HIST_LIST_ENTRIES))
+			return -1;
+		if (dlb_bitmap_alloc(hw,
+				     &hw->vf[i].avail_qed_freelist_entries,
+				     DLB_MAX_NUM_LDB_CREDITS))
+			return -1;
+		if (dlb_bitmap_alloc(hw,
+				     &hw->vf[i].avail_dqed_freelist_entries,
+				     DLB_MAX_NUM_DIR_CREDITS))
+			return -1;
+		if (dlb_bitmap_alloc(hw,
+				     &hw->vf[i].avail_aqed_freelist_entries,
+				     DLB_MAX_NUM_AQOS_ENTRIES))
+			return -1;
+
+		if (dlb_bitmap_zero(hw->vf[i].avail_hist_list_entries))
+			return -1;
+
+		if (dlb_bitmap_zero(hw->vf[i].avail_qed_freelist_entries))
+			return -1;
+
+		if (dlb_bitmap_zero(hw->vf[i].avail_dqed_freelist_entries))
+			return -1;
+
+		if (dlb_bitmap_zero(hw->vf[i].avail_aqed_freelist_entries))
+			return -1;
+	}
+
+	/* Initialize the hardware resource IDs */
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		hw->domains[i].id.phys_id = i;
+		hw->domains[i].id.vf_owned = false;
+	}
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_QUEUES; i++) {
+		hw->rsrcs.ldb_queues[i].id.phys_id = i;
+		hw->rsrcs.ldb_queues[i].id.vf_owned = false;
+	}
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_PORTS; i++) {
+		hw->rsrcs.ldb_ports[i].id.phys_id = i;
+		hw->rsrcs.ldb_ports[i].id.vf_owned = false;
+	}
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_PORTS; i++) {
+		hw->rsrcs.dir_pq_pairs[i].id.phys_id = i;
+		hw->rsrcs.dir_pq_pairs[i].id.vf_owned = false;
+	}
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_CREDIT_POOLS; i++) {
+		hw->rsrcs.ldb_credit_pools[i].id.phys_id = i;
+		hw->rsrcs.ldb_credit_pools[i].id.vf_owned = false;
+	}
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_CREDIT_POOLS; i++) {
+		hw->rsrcs.dir_credit_pools[i].id.phys_id = i;
+		hw->rsrcs.dir_credit_pools[i].id.vf_owned = false;
+	}
+
+	for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
+		hw->rsrcs.sn_groups[i].id = i;
+		/* Default mode (0) is 32 sequence numbers per queue */
+		hw->rsrcs.sn_groups[i].mode = 0;
+		hw->rsrcs.sn_groups[i].sequence_numbers_per_queue = 32;
+		hw->rsrcs.sn_groups[i].slot_use_bitmap = 0;
+	}
+
+	return 0;
+}
+
+void dlb_resource_free(struct dlb_hw *hw)
+{
+	int i;
+
+	dlb_bitmap_free(hw->pf.avail_hist_list_entries);
+
+	dlb_bitmap_free(hw->pf.avail_qed_freelist_entries);
+
+	dlb_bitmap_free(hw->pf.avail_dqed_freelist_entries);
+
+	dlb_bitmap_free(hw->pf.avail_aqed_freelist_entries);
+
+	for (i = 0; i < DLB_MAX_NUM_VFS; i++) {
+		dlb_bitmap_free(hw->vf[i].avail_hist_list_entries);
+		dlb_bitmap_free(hw->vf[i].avail_qed_freelist_entries);
+		dlb_bitmap_free(hw->vf[i].avail_dqed_freelist_entries);
+		dlb_bitmap_free(hw->vf[i].avail_aqed_freelist_entries);
+	}
+}
+
+static struct dlb_domain *dlb_get_domain_from_id(struct dlb_hw *hw,
+						 u32 id,
+						 bool vf_request,
+						 unsigned int vf_id)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_function_resources *rsrcs;
+	struct dlb_domain *domain;
+
+	if (id >= DLB_MAX_NUM_DOMAINS)
+		return NULL;
+
+	if (!vf_request)
+		return &hw->domains[id];
+
+	rsrcs = &hw->vf[vf_id];
+
+	DLB_FUNC_LIST_FOR(rsrcs->used_domains, domain, iter)
+		if (domain->id.virt_id == id)
+			return domain;
+
+	return NULL;
+}
+
+static struct dlb_credit_pool *
+dlb_get_domain_ldb_pool(u32 id,
+			bool vf_request,
+			struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_credit_pool *pool;
+
+	if (id >= DLB_MAX_NUM_LDB_CREDIT_POOLS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
+		if ((!vf_request && pool->id.phys_id == id) ||
+		    (vf_request && pool->id.virt_id == id))
+			return pool;
+
+	return NULL;
+}
+
+static struct dlb_credit_pool *
+dlb_get_domain_dir_pool(u32 id,
+			bool vf_request,
+			struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_credit_pool *pool;
+
+	if (id >= DLB_MAX_NUM_DIR_CREDIT_POOLS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
+		if ((!vf_request && pool->id.phys_id == id) ||
+		    (vf_request && pool->id.virt_id == id))
+			return pool;
+
+	return NULL;
+}
+
+static struct dlb_ldb_port *dlb_get_ldb_port_from_id(struct dlb_hw *hw,
+						     u32 id,
+						     bool vf_request,
+						     unsigned int vf_id)
+{
+	struct dlb_list_entry *iter1 __attribute__((unused));
+	struct dlb_list_entry *iter2 __attribute__((unused));
+	struct dlb_function_resources *rsrcs;
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+
+	if (id >= DLB_MAX_NUM_LDB_PORTS)
+		return NULL;
+
+	rsrcs = (vf_request) ? &hw->vf[vf_id] : &hw->pf;
+
+	if (!vf_request)
+		return &hw->rsrcs.ldb_ports[id];
+
+	DLB_FUNC_LIST_FOR(rsrcs->used_domains, domain, iter1) {
+		DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter2)
+			if (port->id.virt_id == id)
+				return port;
+	}
+
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter1)
+		if (port->id.virt_id == id)
+			return port;
+
+	return NULL;
+}
+
+static struct dlb_ldb_port *
+dlb_get_domain_used_ldb_port(u32 id,
+			     bool vf_request,
+			     struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_ldb_port *port;
+
+	if (id >= DLB_MAX_NUM_LDB_PORTS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		if ((!vf_request && port->id.phys_id == id) ||
+		    (vf_request && port->id.virt_id == id))
+			return port;
+
+	DLB_DOM_LIST_FOR(domain->avail_ldb_ports, port, iter)
+		if ((!vf_request && port->id.phys_id == id) ||
+		    (vf_request && port->id.virt_id == id))
+			return port;
+
+	return NULL;
+}
+
+static struct dlb_ldb_port *dlb_get_domain_ldb_port(u32 id,
+						    bool vf_request,
+						    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_ldb_port *port;
+
+	if (id >= DLB_MAX_NUM_LDB_PORTS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		if ((!vf_request && port->id.phys_id == id) ||
+		    (vf_request && port->id.virt_id == id))
+			return port;
+
+	DLB_DOM_LIST_FOR(domain->avail_ldb_ports, port, iter)
+		if ((!vf_request && port->id.phys_id == id) ||
+		    (vf_request && port->id.virt_id == id))
+			return port;
+
+	return NULL;
+}
+
+static struct dlb_dir_pq_pair *dlb_get_dir_pq_from_id(struct dlb_hw *hw,
+						      u32 id,
+						      bool vf_request,
+						      unsigned int vf_id)
+{
+	struct dlb_list_entry *iter1 __attribute__((unused));
+	struct dlb_list_entry *iter2 __attribute__((unused));
+	struct dlb_function_resources *rsrcs;
+	struct dlb_dir_pq_pair *port;
+	struct dlb_domain *domain;
+
+	if (id >= DLB_MAX_NUM_DIR_PORTS)
+		return NULL;
+
+	rsrcs = (vf_request) ? &hw->vf[vf_id] : &hw->pf;
+
+	if (!vf_request)
+		return &hw->rsrcs.dir_pq_pairs[id];
+
+	DLB_FUNC_LIST_FOR(rsrcs->used_domains, domain, iter1) {
+		DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter2)
+			if (port->id.virt_id == id)
+				return port;
+	}
+
+	DLB_FUNC_LIST_FOR(rsrcs->avail_dir_pq_pairs, port, iter1)
+		if (port->id.virt_id == id)
+			return port;
+
+	return NULL;
+}
+
+static struct dlb_dir_pq_pair *
+dlb_get_domain_used_dir_pq(u32 id,
+			   bool vf_request,
+			   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_dir_pq_pair *port;
+
+	if (id >= DLB_MAX_NUM_DIR_PORTS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		if ((!vf_request && port->id.phys_id == id) ||
+		    (vf_request && port->id.virt_id == id))
+			return port;
+
+	return NULL;
+}
+
+static struct dlb_dir_pq_pair *dlb_get_domain_dir_pq(u32 id,
+						     bool vf_request,
+						     struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_dir_pq_pair *port;
+
+	if (id >= DLB_MAX_NUM_DIR_PORTS)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		if ((!vf_request && port->id.phys_id == id) ||
+		    (vf_request && port->id.virt_id == id))
+			return port;
+
+	DLB_DOM_LIST_FOR(domain->avail_dir_pq_pairs, port, iter)
+		if ((!vf_request && port->id.phys_id == id) ||
+		    (vf_request && port->id.virt_id == id))
+			return port;
+
+	return NULL;
+}
+
+static struct dlb_ldb_queue *dlb_get_ldb_queue_from_id(struct dlb_hw *hw,
+						       u32 id,
+						       bool vf_request,
+						       unsigned int vf_id)
+{
+	struct dlb_list_entry *iter1 __attribute__((unused));
+	struct dlb_list_entry *iter2 __attribute__((unused));
+	struct dlb_function_resources *rsrcs;
+	struct dlb_ldb_queue *queue;
+	struct dlb_domain *domain;
+
+	if (id >= DLB_MAX_NUM_LDB_QUEUES)
+		return NULL;
+
+	rsrcs = (vf_request) ? &hw->vf[vf_id] : &hw->pf;
+
+	if (!vf_request)
+		return &hw->rsrcs.ldb_queues[id];
+
+	DLB_FUNC_LIST_FOR(rsrcs->used_domains, domain, iter1) {
+		DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter2)
+			if (queue->id.virt_id == id)
+				return queue;
+	}
+
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_queues, queue, iter1)
+		if (queue->id.virt_id == id)
+			return queue;
+
+	return NULL;
+}
+
+static struct dlb_ldb_queue *dlb_get_domain_ldb_queue(u32 id,
+						      bool vf_request,
+						      struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_ldb_queue *queue;
+
+	if (id >= DLB_MAX_NUM_LDB_QUEUES)
+		return NULL;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter)
+		if ((!vf_request && queue->id.phys_id == id) ||
+		    (vf_request && queue->id.virt_id == id))
+			return queue;
+
+	return NULL;
+}
+
+#define DLB_XFER_LL_RSRC(dst, src, num, type_t, name) ({		    \
+	struct dlb_list_entry *it1 __attribute__((unused));		    \
+	struct dlb_list_entry *it2 __attribute__((unused));		    \
+	struct dlb_function_resources *_src = src;			    \
+	struct dlb_function_resources *_dst = dst;			    \
+	type_t *ptr, *tmp __attribute__((unused));			    \
+	unsigned int i = 0;						    \
+									    \
+	DLB_FUNC_LIST_FOR_SAFE(_src->avail_##name##s, ptr, tmp, it1, it2) { \
+		if (i++ == (num))					    \
+			break;						    \
+									    \
+		dlb_list_del(&_src->avail_##name##s, &ptr->func_list);	    \
+		dlb_list_add(&_dst->avail_##name##s,  &ptr->func_list);     \
+		_src->num_avail_##name##s--;				    \
+		_dst->num_avail_##name##s++;				    \
+	}								    \
+})
+
+#define DLB_VF_ID_CLEAR(head, type_t) ({   \
+	struct dlb_list_entry *iter __attribute__((unused)); \
+	type_t *var;					     \
+							     \
+	DLB_FUNC_LIST_FOR(head, var, iter)		     \
+		var->id.vf_owned = false;		     \
+})
+
+int dlb_update_vf_sched_domains(struct dlb_hw *hw, u32 vf_id, u32 num)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_function_resources *src, *dst;
+	struct dlb_domain *domain;
+	unsigned int orig;
+	int ret;
+
+	if (vf_id >= DLB_MAX_NUM_VFS)
+		return -EINVAL;
+
+	src = &hw->pf;
+	dst = &hw->vf[vf_id];
+
+	/* If the VF is locked, its resource assignment can't be changed */
+	if (dlb_vf_is_locked(hw, vf_id))
+		return -EPERM;
+
+	orig = dst->num_avail_domains;
+
+	/* Detach the destination VF's current resources before checking if
+	 * enough are available, and set their IDs accordingly.
+	 */
+	DLB_VF_ID_CLEAR(dst->avail_domains, struct dlb_domain);
+
+	DLB_XFER_LL_RSRC(src, dst, orig, struct dlb_domain, domain);
+
+	/* Are there enough available resources to satisfy the request? */
+	if (num > src->num_avail_domains) {
+		num = orig;
+		ret = -EINVAL;
+	} else {
+		ret = 0;
+	}
+
+	DLB_XFER_LL_RSRC(dst, src, num, struct dlb_domain, domain);
+
+	/* Set the domains' VF backpointer */
+	DLB_FUNC_LIST_FOR(dst->avail_domains, domain, iter)
+		domain->parent_func = dst;
+
+	return ret;
+}
+
+int dlb_update_vf_ldb_queues(struct dlb_hw *hw, u32 vf_id, u32 num)
+{
+	struct dlb_function_resources *src, *dst;
+	unsigned int orig;
+	int ret;
+
+	if (vf_id >= DLB_MAX_NUM_VFS)
+		return -EINVAL;
+
+	src = &hw->pf;
+	dst = &hw->vf[vf_id];
+
+	/* If the VF is locked, its resource assignment can't be changed */
+	if (dlb_vf_is_locked(hw, vf_id))
+		return -EPERM;
+
+	orig = dst->num_avail_ldb_queues;
+
+	/* Detach the destination VF's current resources before checking if
+	 * enough are available, and set their IDs accordingly.
+	 */
+	DLB_VF_ID_CLEAR(dst->avail_ldb_queues, struct dlb_ldb_queue);
+
+	DLB_XFER_LL_RSRC(src, dst, orig, struct dlb_ldb_queue, ldb_queue);
+
+	/* Are there enough available resources to satisfy the request? */
+	if (num > src->num_avail_ldb_queues) {
+		num = orig;
+		ret = -EINVAL;
+	} else {
+		ret = 0;
+	}
+
+	DLB_XFER_LL_RSRC(dst, src, num, struct dlb_ldb_queue, ldb_queue);
+
+	return ret;
+}
+
+int dlb_update_vf_ldb_ports(struct dlb_hw *hw, u32 vf_id, u32 num)
+{
+	struct dlb_function_resources *src, *dst;
+	unsigned int orig;
+	int ret;
+
+	if (vf_id >= DLB_MAX_NUM_VFS)
+		return -EINVAL;
+
+	src = &hw->pf;
+	dst = &hw->vf[vf_id];
+
+	/* If the VF is locked, its resource assignment can't be changed */
+	if (dlb_vf_is_locked(hw, vf_id))
+		return -EPERM;
+
+	orig = dst->num_avail_ldb_ports;
+
+	/* Detach the destination VF's current resources before checking if
+	 * enough are available, and set their IDs accordingly.
+	 */
+	DLB_VF_ID_CLEAR(dst->avail_ldb_ports, struct dlb_ldb_port);
+
+	DLB_XFER_LL_RSRC(src, dst, orig, struct dlb_ldb_port, ldb_port);
+
+	/* Are there enough available resources to satisfy the request? */
+	if (num > src->num_avail_ldb_ports) {
+		num = orig;
+		ret = -EINVAL;
+	} else {
+		ret = 0;
+	}
+
+	DLB_XFER_LL_RSRC(dst, src, num, struct dlb_ldb_port, ldb_port);
+
+	return ret;
+}
+
+int dlb_update_vf_dir_ports(struct dlb_hw *hw, u32 vf_id, u32 num)
+{
+	struct dlb_function_resources *src, *dst;
+	unsigned int orig;
+	int ret;
+
+	if (vf_id >= DLB_MAX_NUM_VFS)
+		return -EINVAL;
+
+	src = &hw->pf;
+	dst = &hw->vf[vf_id];
+
+	/* If the VF is locked, its resource assignment can't be changed */
+	if (dlb_vf_is_locked(hw, vf_id))
+		return -EPERM;
+
+	orig = dst->num_avail_dir_pq_pairs;
+
+	/* Detach the destination VF's current resources before checking if
+	 * enough are available, and set their IDs accordingly.
+	 */
+	DLB_VF_ID_CLEAR(dst->avail_dir_pq_pairs, struct dlb_dir_pq_pair);
+
+	DLB_XFER_LL_RSRC(src, dst, orig, struct dlb_dir_pq_pair, dir_pq_pair);
+
+	/* Are there enough available resources to satisfy the request? */
+	if (num > src->num_avail_dir_pq_pairs) {
+		num = orig;
+		ret = -EINVAL;
+	} else {
+		ret = 0;
+	}
+
+	DLB_XFER_LL_RSRC(dst, src, num, struct dlb_dir_pq_pair, dir_pq_pair);
+
+	return ret;
+}
+
+int dlb_update_vf_ldb_credit_pools(struct dlb_hw *hw,
+				   u32 vf_id,
+				   u32 num)
+{
+	struct dlb_function_resources *src, *dst;
+	unsigned int orig;
+	int ret;
+
+	if (vf_id >= DLB_MAX_NUM_VFS)
+		return -EINVAL;
+
+	src = &hw->pf;
+	dst = &hw->vf[vf_id];
+
+	/* If the VF is locked, its resource assignment can't be changed */
+	if (dlb_vf_is_locked(hw, vf_id))
+		return -EPERM;
+
+	orig = dst->num_avail_ldb_credit_pools;
+
+	/* Detach the destination VF's current resources before checking if
+	 * enough are available, and set their IDs accordingly.
+	 */
+	DLB_VF_ID_CLEAR(dst->avail_ldb_credit_pools, struct dlb_credit_pool);
+
+	DLB_XFER_LL_RSRC(src,
+			 dst,
+			 orig,
+			 struct dlb_credit_pool,
+			 ldb_credit_pool);
+
+	/* Are there enough available resources to satisfy the request? */
+	if (num > src->num_avail_ldb_credit_pools) {
+		num = orig;
+		ret = -EINVAL;
+	} else {
+		ret = 0;
+	}
+
+	DLB_XFER_LL_RSRC(dst,
+			 src,
+			 num,
+			 struct dlb_credit_pool,
+			 ldb_credit_pool);
+
+	return ret;
+}
+
+int dlb_update_vf_dir_credit_pools(struct dlb_hw *hw,
+				   u32 vf_id,
+				   u32 num)
+{
+	struct dlb_function_resources *src, *dst;
+	unsigned int orig;
+	int ret;
+
+	if (vf_id >= DLB_MAX_NUM_VFS)
+		return -EINVAL;
+
+	src = &hw->pf;
+	dst = &hw->vf[vf_id];
+
+	/* If the VF is locked, its resource assignment can't be changed */
+	if (dlb_vf_is_locked(hw, vf_id))
+		return -EPERM;
+
+	orig = dst->num_avail_dir_credit_pools;
+
+	/* Detach the VF's current resources before checking if enough are
+	 * available, and set their IDs accordingly.
+	 */
+	DLB_VF_ID_CLEAR(dst->avail_dir_credit_pools, struct dlb_credit_pool);
+
+	DLB_XFER_LL_RSRC(src,
+			 dst,
+			 orig,
+			 struct dlb_credit_pool,
+			 dir_credit_pool);
+
+	/* Are there enough available resources to satisfy the request? */
+	if (num > src->num_avail_dir_credit_pools) {
+		num = orig;
+		ret = -EINVAL;
+	} else {
+		ret = 0;
+	}
+
+	DLB_XFER_LL_RSRC(dst,
+			 src,
+			 num,
+			 struct dlb_credit_pool,
+			 dir_credit_pool);
+
+	return ret;
+}
+
+static int dlb_transfer_bitmap_resources(struct dlb_bitmap *src,
+					 struct dlb_bitmap *dst,
+					 u32 num)
+{
+	int orig, ret, base;
+
+	/* Validate bitmaps before use */
+	if (dlb_bitmap_count(dst) < 0 || dlb_bitmap_count(src) < 0)
+		return -EINVAL;
+
+	/* Reassign the dest's bitmap entries to the source's before checking
+	 * if a contiguous chunk of size 'num' is available. The reassignment
+	 * may be necessary to create a sufficiently large contiguous chunk.
+	 */
+	orig = dlb_bitmap_count(dst);
+
+	dlb_bitmap_or(src, src, dst);
+
+	dlb_bitmap_zero(dst);
+
+	/* Are there enough available resources to satisfy the request? */
+	base = dlb_bitmap_find_set_bit_range(src, num);
+
+	if (base == -ENOENT) {
+		num = orig;
+		base = dlb_bitmap_find_set_bit_range(src, num);
+		ret = -EINVAL;
+	} else {
+		ret = 0;
+	}
+
+	dlb_bitmap_set_range(dst, base, num);
+
+	dlb_bitmap_clear_range(src, base, num);
+
+	return ret;
+}
+
+int dlb_update_vf_ldb_credits(struct dlb_hw *hw, u32 vf_id, u32 num)
+{
+	struct dlb_function_resources *src, *dst;
+
+	if (vf_id >= DLB_MAX_NUM_VFS)
+		return -EINVAL;
+
+	src = &hw->pf;
+	dst = &hw->vf[vf_id];
+
+	/* If the VF is locked, its resource assignment can't be changed */
+	if (dlb_vf_is_locked(hw, vf_id))
+		return -EPERM;
+
+	return dlb_transfer_bitmap_resources(src->avail_qed_freelist_entries,
+					     dst->avail_qed_freelist_entries,
+					     num);
+}
+
+int dlb_update_vf_dir_credits(struct dlb_hw *hw, u32 vf_id, u32 num)
+{
+	struct dlb_function_resources *src, *dst;
+
+	if (vf_id >= DLB_MAX_NUM_VFS)
+		return -EINVAL;
+
+	src = &hw->pf;
+	dst = &hw->vf[vf_id];
+
+	/* If the VF is locked, its resource assignment can't be changed */
+	if (dlb_vf_is_locked(hw, vf_id))
+		return -EPERM;
+
+	return dlb_transfer_bitmap_resources(src->avail_dqed_freelist_entries,
+					     dst->avail_dqed_freelist_entries,
+					     num);
+}
+
+int dlb_update_vf_hist_list_entries(struct dlb_hw *hw,
+				    u32 vf_id,
+				    u32 num)
+{
+	struct dlb_function_resources *src, *dst;
+
+	if (vf_id >= DLB_MAX_NUM_VFS)
+		return -EINVAL;
+
+	src = &hw->pf;
+	dst = &hw->vf[vf_id];
+
+	/* If the VF is locked, its resource assignment can't be changed */
+	if (dlb_vf_is_locked(hw, vf_id))
+		return -EPERM;
+
+	return dlb_transfer_bitmap_resources(src->avail_hist_list_entries,
+					     dst->avail_hist_list_entries,
+					     num);
+}
+
+int dlb_update_vf_atomic_inflights(struct dlb_hw *hw,
+				   u32 vf_id,
+				   u32 num)
+{
+	struct dlb_function_resources *src, *dst;
+
+	if (vf_id >= DLB_MAX_NUM_VFS)
+		return -EINVAL;
+
+	src = &hw->pf;
+	dst = &hw->vf[vf_id];
+
+	/* If the VF is locked, its resource assignment can't be changed */
+	if (dlb_vf_is_locked(hw, vf_id))
+		return -EPERM;
+
+	return dlb_transfer_bitmap_resources(src->avail_aqed_freelist_entries,
+					     dst->avail_aqed_freelist_entries,
+					     num);
+}
+
+static int dlb_attach_ldb_queues(struct dlb_hw *hw,
+				 struct dlb_function_resources *rsrcs,
+				 struct dlb_domain *domain,
+				 u32 num_queues,
+				 struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_ldb_queues < num_queues) {
+		resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_queues; i++) {
+		struct dlb_ldb_queue *queue;
+
+		queue = DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_queues,
+					   typeof(*queue));
+		if (!queue) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_ldb_queues, &queue->func_list);
+
+		queue->domain_id = domain->id;
+		queue->owned = true;
+
+		dlb_list_add(&domain->avail_ldb_queues, &queue->domain_list);
+	}
+
+	rsrcs->num_avail_ldb_queues -= num_queues;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned queues */
+	for (j = 0; j < i; j++) {
+		struct dlb_ldb_queue *queue;
+
+		queue = DLB_FUNC_LIST_HEAD(domain->avail_ldb_queues,
+					   typeof(*queue));
+		/* Unrecoverable internal error */
+		if (!queue)
+			break;
+
+		queue->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_queues, &queue->domain_list);
+
+		dlb_list_add(&rsrcs->avail_ldb_queues, &queue->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static struct dlb_ldb_port *
+dlb_get_next_ldb_port(struct dlb_hw *hw,
+		      struct dlb_function_resources *rsrcs,
+		      u32 domain_id)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_ldb_port *port;
+
+	/* To reduce the odds of consecutive load-balanced ports mapping to the
+	 * same queue(s), the driver attempts to allocate ports whose neighbors
+	 * are owned by a different domain.
+	 */
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
+		u32 next, prev;
+		u32 phys_id;
+
+		phys_id = port->id.phys_id;
+		next = phys_id + 1;
+		prev = phys_id - 1;
+
+		if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
+			next = 0;
+		if (phys_id == 0)
+			prev = DLB_MAX_NUM_LDB_PORTS - 1;
+
+		if (!hw->rsrcs.ldb_ports[next].owned ||
+		    hw->rsrcs.ldb_ports[next].domain_id.phys_id == domain_id)
+			continue;
+
+		if (!hw->rsrcs.ldb_ports[prev].owned ||
+		    hw->rsrcs.ldb_ports[prev].domain_id.phys_id == domain_id)
+			continue;
+
+		return port;
+	}
+
+	/* Failing that, the driver looks for a port with one neighbor owned by
+	 * a different domain and the other unallocated.
+	 */
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
+		u32 next, prev;
+		u32 phys_id;
+
+		phys_id = port->id.phys_id;
+		next = phys_id + 1;
+		prev = phys_id - 1;
+
+		if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
+			next = 0;
+		if (phys_id == 0)
+			prev = DLB_MAX_NUM_LDB_PORTS - 1;
+
+		if (!hw->rsrcs.ldb_ports[prev].owned &&
+		    hw->rsrcs.ldb_ports[next].owned &&
+		    hw->rsrcs.ldb_ports[next].domain_id.phys_id != domain_id)
+			return port;
+
+		if (!hw->rsrcs.ldb_ports[next].owned &&
+		    hw->rsrcs.ldb_ports[prev].owned &&
+		    hw->rsrcs.ldb_ports[prev].domain_id.phys_id != domain_id)
+			return port;
+	}
+
+	/* Failing that, the driver looks for a port with both neighbors
+	 * unallocated.
+	 */
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, port, iter) {
+		u32 next, prev;
+		u32 phys_id;
+
+		phys_id = port->id.phys_id;
+		next = phys_id + 1;
+		prev = phys_id - 1;
+
+		if (phys_id == DLB_MAX_NUM_LDB_PORTS - 1)
+			next = 0;
+		if (phys_id == 0)
+			prev = DLB_MAX_NUM_LDB_PORTS - 1;
+
+		if (!hw->rsrcs.ldb_ports[prev].owned &&
+		    !hw->rsrcs.ldb_ports[next].owned)
+			return port;
+	}
+
+	/* If all else fails, the driver returns the next available port. */
+	return DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_ports, typeof(*port));
+}
+
+static int dlb_attach_ldb_ports(struct dlb_hw *hw,
+				struct dlb_function_resources *rsrcs,
+				struct dlb_domain *domain,
+				u32 num_ports,
+				struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_ldb_ports < num_ports) {
+		resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_ports; i++) {
+		struct dlb_ldb_port *port;
+
+		port = dlb_get_next_ldb_port(hw, rsrcs, domain->id.phys_id);
+
+		if (!port) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_ldb_ports, &port->func_list);
+
+		port->domain_id = domain->id;
+		port->owned = true;
+
+		dlb_list_add(&domain->avail_ldb_ports, &port->domain_list);
+	}
+
+	rsrcs->num_avail_ldb_ports -= num_ports;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned ports */
+	for (j = 0; j < i; j++) {
+		struct dlb_ldb_port *port;
+
+		port = DLB_FUNC_LIST_HEAD(domain->avail_ldb_ports,
+					  typeof(*port));
+		/* Unrecoverable internal error */
+		if (!port)
+			break;
+
+		port->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_ports, &port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_ldb_ports, &port->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_dir_ports(struct dlb_hw *hw,
+				struct dlb_function_resources *rsrcs,
+				struct dlb_domain *domain,
+				u32 num_ports,
+				struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_dir_pq_pairs < num_ports) {
+		resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_ports; i++) {
+		struct dlb_dir_pq_pair *port;
+
+		port = DLB_FUNC_LIST_HEAD(rsrcs->avail_dir_pq_pairs,
+					  typeof(*port));
+		if (!port) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_dir_pq_pairs, &port->func_list);
+
+		port->domain_id = domain->id;
+		port->owned = true;
+
+		dlb_list_add(&domain->avail_dir_pq_pairs, &port->domain_list);
+	}
+
+	rsrcs->num_avail_dir_pq_pairs -= num_ports;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned ports */
+	for (j = 0; j < i; j++) {
+		struct dlb_dir_pq_pair *port;
+
+		port = DLB_FUNC_LIST_HEAD(domain->avail_dir_pq_pairs,
+					  typeof(*port));
+		/* Unrecoverable internal error */
+		if (!port)
+			break;
+
+		port->owned = false;
+
+		dlb_list_del(&domain->avail_dir_pq_pairs, &port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_pq_pairs, &port->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_ldb_credits(struct dlb_function_resources *rsrcs,
+				  struct dlb_domain *domain,
+				  u32 num_credits,
+				  struct dlb_cmd_response *resp)
+{
+	struct dlb_bitmap *bitmap = rsrcs->avail_qed_freelist_entries;
+
+	if (dlb_bitmap_count(bitmap) < (int)num_credits) {
+		resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (num_credits) {
+		int base;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap, num_credits);
+		if (base < 0)
+			goto error;
+
+		domain->qed_freelist.base = base;
+		domain->qed_freelist.bound = base + num_credits;
+		domain->qed_freelist.offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_credits);
+	}
+
+	return 0;
+
+error:
+	resp->status = DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE;
+	return -1;
+}
+
+static int dlb_attach_dir_credits(struct dlb_function_resources *rsrcs,
+				  struct dlb_domain *domain,
+				  u32 num_credits,
+				  struct dlb_cmd_response *resp)
+{
+	struct dlb_bitmap *bitmap = rsrcs->avail_dqed_freelist_entries;
+
+	if (dlb_bitmap_count(bitmap) < (int)num_credits) {
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (num_credits) {
+		int base;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap, num_credits);
+		if (base < 0)
+			goto error;
+
+		domain->dqed_freelist.base = base;
+		domain->dqed_freelist.bound = base + num_credits;
+		domain->dqed_freelist.offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_credits);
+	}
+
+	return 0;
+
+error:
+	resp->status = DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE;
+	return -1;
+}
+
+static int dlb_attach_ldb_credit_pools(struct dlb_hw *hw,
+				       struct dlb_function_resources *rsrcs,
+				       struct dlb_domain *domain,
+				       u32 num_credit_pools,
+				       struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_ldb_credit_pools < num_credit_pools) {
+		resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_credit_pools; i++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(rsrcs->avail_ldb_credit_pools,
+					  typeof(*pool));
+		if (!pool) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+
+		pool->domain_id = domain->id;
+		pool->owned = true;
+
+		dlb_list_add(&domain->avail_ldb_credit_pools,
+			     &pool->domain_list);
+	}
+
+	rsrcs->num_avail_ldb_credit_pools -= num_credit_pools;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned credit pools */
+	for (j = 0; j < i; j++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(domain->avail_ldb_credit_pools,
+					  typeof(*pool));
+		/* Unrecoverable internal error */
+		if (!pool)
+			break;
+
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_credit_pools,
+			     &pool->domain_list);
+
+		dlb_list_add(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_dir_credit_pools(struct dlb_hw *hw,
+				       struct dlb_function_resources *rsrcs,
+				       struct dlb_domain *domain,
+				       u32 num_credit_pools,
+				       struct dlb_cmd_response *resp)
+{
+	unsigned int i, j;
+
+	if (rsrcs->num_avail_dir_credit_pools < num_credit_pools) {
+		resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	for (i = 0; i < num_credit_pools; i++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(rsrcs->avail_dir_credit_pools,
+					  typeof(*pool));
+		if (!pool) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: domain validation failed\n",
+				   __func__);
+			goto cleanup;
+		}
+
+		dlb_list_del(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+
+		pool->domain_id = domain->id;
+		pool->owned = true;
+
+		dlb_list_add(&domain->avail_dir_credit_pools,
+			     &pool->domain_list);
+	}
+
+	rsrcs->num_avail_dir_credit_pools -= num_credit_pools;
+
+	return 0;
+
+cleanup:
+
+	/* Return the assigned credit pools */
+	for (j = 0; j < i; j++) {
+		struct dlb_credit_pool *pool;
+
+		pool = DLB_FUNC_LIST_HEAD(domain->avail_dir_credit_pools,
+					  typeof(*pool));
+		/* Unrecoverable internal error */
+		if (!pool)
+			break;
+
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_dir_credit_pools,
+			     &pool->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+	}
+
+	return -EFAULT;
+}
+
+static int dlb_attach_atomic_inflights(struct dlb_function_resources *rsrcs,
+				       struct dlb_domain *domain,
+				       u32 num_atomic_inflights,
+				       struct dlb_cmd_response *resp)
+{
+	if (num_atomic_inflights) {
+		struct dlb_bitmap *bitmap =
+			rsrcs->avail_aqed_freelist_entries;
+		int base;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap,
+						     num_atomic_inflights);
+		if (base < 0)
+			goto error;
+
+		domain->aqed_freelist.base = base;
+		domain->aqed_freelist.bound = base + num_atomic_inflights;
+		domain->aqed_freelist.offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_atomic_inflights);
+	}
+
+	return 0;
+
+error:
+	resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
+	return -1;
+}
+
+static int
+dlb_attach_domain_hist_list_entries(struct dlb_function_resources *rsrcs,
+				    struct dlb_domain *domain,
+				    u32 num_hist_list_entries,
+				    struct dlb_cmd_response *resp)
+{
+	struct dlb_bitmap *bitmap;
+	int base;
+
+	if (num_hist_list_entries) {
+		bitmap = rsrcs->avail_hist_list_entries;
+
+		base = dlb_bitmap_find_set_bit_range(bitmap,
+						     num_hist_list_entries);
+		if (base < 0)
+			goto error;
+
+		domain->total_hist_list_entries = num_hist_list_entries;
+		domain->avail_hist_list_entries = num_hist_list_entries;
+		domain->hist_list_entry_base = base;
+		domain->hist_list_entry_offset = 0;
+
+		dlb_bitmap_clear_range(bitmap, base, num_hist_list_entries);
+	}
+	return 0;
+
+error:
+	resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+	return -1;
+}
+
+static unsigned int
+dlb_get_num_ports_in_use(struct dlb_hw *hw)
+{
+	unsigned int i, n = 0;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_PORTS; i++)
+		if (hw->rsrcs.ldb_ports[i].owned)
+			n++;
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_PORTS; i++)
+		if (hw->rsrcs.dir_pq_pairs[i].owned)
+			n++;
+
+	return n;
+}
+
+static int
+dlb_verify_create_sched_domain_args(struct dlb_hw *hw,
+				    struct dlb_function_resources *rsrcs,
+				    struct dlb_create_sched_domain_args *args,
+				    struct dlb_cmd_response *resp)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_bitmap *ldb_credit_freelist;
+	struct dlb_bitmap *dir_credit_freelist;
+	unsigned int ldb_credit_freelist_count;
+	unsigned int dir_credit_freelist_count;
+	unsigned int max_contig_aqed_entries;
+	unsigned int max_contig_dqed_entries;
+	unsigned int max_contig_qed_entries;
+	unsigned int max_contig_hl_entries;
+	struct dlb_bitmap *aqed_freelist;
+	enum dlb_dev_revision revision;
+
+	ldb_credit_freelist = rsrcs->avail_qed_freelist_entries;
+	dir_credit_freelist = rsrcs->avail_dqed_freelist_entries;
+	aqed_freelist = rsrcs->avail_aqed_freelist_entries;
+
+	ldb_credit_freelist_count = dlb_bitmap_count(ldb_credit_freelist);
+	dir_credit_freelist_count = dlb_bitmap_count(dir_credit_freelist);
+
+	max_contig_hl_entries =
+		dlb_bitmap_longest_set_range(rsrcs->avail_hist_list_entries);
+	max_contig_aqed_entries =
+		dlb_bitmap_longest_set_range(aqed_freelist);
+	max_contig_qed_entries =
+		dlb_bitmap_longest_set_range(ldb_credit_freelist);
+	max_contig_dqed_entries =
+		dlb_bitmap_longest_set_range(dir_credit_freelist);
+
+	if (rsrcs->num_avail_domains < 1)
+		resp->status = DLB_ST_DOMAIN_UNAVAILABLE;
+	else if (rsrcs->num_avail_ldb_queues < args->num_ldb_queues)
+		resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
+	else if (rsrcs->num_avail_ldb_ports < args->num_ldb_ports)
+		resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
+	else if (args->num_ldb_queues > 0 && args->num_ldb_ports == 0)
+		resp->status = DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES;
+	else if (rsrcs->num_avail_dir_pq_pairs < args->num_dir_ports)
+		resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
+	else if (ldb_credit_freelist_count < args->num_ldb_credits)
+		resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+	else if (dir_credit_freelist_count < args->num_dir_credits)
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+	else if (rsrcs->num_avail_ldb_credit_pools < args->num_ldb_credit_pools)
+		resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
+	else if (rsrcs->num_avail_dir_credit_pools < args->num_dir_credit_pools)
+		resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
+	else if (max_contig_hl_entries < args->num_hist_list_entries)
+		resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+	else if (max_contig_aqed_entries < args->num_atomic_inflights)
+		resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
+	else if (max_contig_qed_entries < args->num_ldb_credits)
+		resp->status = DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE;
+	else if (max_contig_dqed_entries < args->num_dir_credits)
+		resp->status = DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE;
+
+	/* DLB A-stepping workaround for hardware write buffer lock up issue:
+	 * limit the maximum configured ports to less than 128 and disable CQ
+	 * occupancy interrupts.
+	 */
+	revision = os_get_dev_revision(hw);
+
+	if (revision < DLB_B0) {
+		u32 n = dlb_get_num_ports_in_use(hw);
+
+		n += args->num_ldb_ports + args->num_dir_ports;
+
+		if (n >= DLB_A_STEP_MAX_PORTS)
+			resp->status = args->num_ldb_ports ?
+				DLB_ST_LDB_PORTS_UNAVAILABLE :
+				DLB_ST_DIR_PORTS_UNAVAILABLE;
+	}
+
+	if (resp->status)
+		return -1;
+
+	return 0;
+}
+
+static int
+dlb_verify_create_ldb_pool_args(struct dlb_hw *hw,
+				u32 domain_id,
+				struct dlb_create_ldb_pool_args *args,
+				struct dlb_cmd_response *resp,
+				bool vf_request,
+				unsigned int vf_id)
+{
+	struct dlb_freelist *qed_freelist;
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	qed_freelist = &domain->qed_freelist;
+
+	if (dlb_freelist_count(qed_freelist) < args->num_ldb_credits) {
+		resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_ldb_credit_pools)) {
+		resp->status = DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+dlb_configure_ldb_credit_pool(struct dlb_hw *hw,
+			      struct dlb_domain *domain,
+			      struct dlb_create_ldb_pool_args *args,
+			      struct dlb_credit_pool *pool)
+{
+	union dlb_sys_ldb_pool_enbld r0 = { {0} };
+	union dlb_chp_ldb_pool_crd_lim r1 = { {0} };
+	union dlb_chp_ldb_pool_crd_cnt r2 = { {0} };
+	union dlb_chp_qed_fl_base  r3 = { {0} };
+	union dlb_chp_qed_fl_lim r4 = { {0} };
+	union dlb_chp_qed_fl_push_ptr r5 = { {0} };
+	union dlb_chp_qed_fl_pop_ptr  r6 = { {0} };
+
+	r1.field.limit = args->num_ldb_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_POOL_CRD_LIM(pool->id.phys_id), r1.val);
+
+	r2.field.count = args->num_ldb_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_POOL_CRD_CNT(pool->id.phys_id), r2.val);
+
+	r3.field.base = domain->qed_freelist.base + domain->qed_freelist.offset;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_BASE(pool->id.phys_id), r3.val);
+
+	r4.field.freelist_disable = 0;
+	r4.field.limit = r3.field.base + args->num_ldb_credits - 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_LIM(pool->id.phys_id), r4.val);
+
+	r5.field.push_ptr = r3.field.base;
+	r5.field.generation = 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_PUSH_PTR(pool->id.phys_id), r5.val);
+
+	r6.field.pop_ptr = r3.field.base;
+	r6.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_QED_FL_POP_PTR(pool->id.phys_id), r6.val);
+
+	r0.field.pool_enabled = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_POOL_ENBLD(pool->id.phys_id), r0.val);
+
+	pool->avail_credits = args->num_ldb_credits;
+	pool->total_credits = args->num_ldb_credits;
+	domain->qed_freelist.offset += args->num_ldb_credits;
+
+	pool->configured = true;
+}
+
+static int
+dlb_verify_create_dir_pool_args(struct dlb_hw *hw,
+				u32 domain_id,
+				struct dlb_create_dir_pool_args *args,
+				struct dlb_cmd_response *resp,
+				bool vf_request,
+				unsigned int vf_id)
+{
+	struct dlb_freelist *dqed_freelist;
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	dqed_freelist = &domain->dqed_freelist;
+
+	if (dlb_freelist_count(dqed_freelist) < args->num_dir_credits) {
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_dir_credit_pools)) {
+		resp->status = DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+dlb_configure_dir_credit_pool(struct dlb_hw *hw,
+			      struct dlb_domain *domain,
+			      struct dlb_create_dir_pool_args *args,
+			      struct dlb_credit_pool *pool)
+{
+	union dlb_sys_dir_pool_enbld r0 = { {0} };
+	union dlb_chp_dir_pool_crd_lim r1 = { {0} };
+	union dlb_chp_dir_pool_crd_cnt r2 = { {0} };
+	union dlb_chp_dqed_fl_base  r3 = { {0} };
+	union dlb_chp_dqed_fl_lim r4 = { {0} };
+	union dlb_chp_dqed_fl_push_ptr r5 = { {0} };
+	union dlb_chp_dqed_fl_pop_ptr  r6 = { {0} };
+
+	r1.field.limit = args->num_dir_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_DIR_POOL_CRD_LIM(pool->id.phys_id), r1.val);
+
+	r2.field.count = args->num_dir_credits;
+
+	DLB_CSR_WR(hw, DLB_CHP_DIR_POOL_CRD_CNT(pool->id.phys_id), r2.val);
+
+	r3.field.base = domain->dqed_freelist.base +
+			domain->dqed_freelist.offset;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_BASE(pool->id.phys_id), r3.val);
+
+	r4.field.freelist_disable = 0;
+	r4.field.limit = r3.field.base + args->num_dir_credits - 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_LIM(pool->id.phys_id), r4.val);
+
+	r5.field.push_ptr = r3.field.base;
+	r5.field.generation = 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_PUSH_PTR(pool->id.phys_id), r5.val);
+
+	r6.field.pop_ptr = r3.field.base;
+	r6.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_DQED_FL_POP_PTR(pool->id.phys_id), r6.val);
+
+	r0.field.pool_enabled = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_POOL_ENBLD(pool->id.phys_id), r0.val);
+
+	pool->avail_credits = args->num_dir_credits;
+	pool->total_credits = args->num_dir_credits;
+	domain->dqed_freelist.offset += args->num_dir_credits;
+
+	pool->configured = true;
+}
+
+static int
+dlb_verify_create_ldb_queue_args(struct dlb_hw *hw,
+				 u32 domain_id,
+				 struct dlb_create_ldb_queue_args *args,
+				 struct dlb_cmd_response *resp,
+				 bool vf_request,
+				 unsigned int vf_id)
+{
+	struct dlb_freelist *aqed_freelist;
+	struct dlb_domain *domain;
+	int i;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_ldb_queues)) {
+		resp->status = DLB_ST_LDB_QUEUES_UNAVAILABLE;
+		return -1;
+	}
+
+	if (args->num_sequence_numbers) {
+		for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
+			struct dlb_sn_group *group = &hw->rsrcs.sn_groups[i];
+
+			if (group->sequence_numbers_per_queue ==
+			    args->num_sequence_numbers &&
+			    !dlb_sn_group_full(group))
+				break;
+		}
+
+		if (i == DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS) {
+			resp->status = DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE;
+			return -1;
+		}
+	}
+
+	if (args->num_qid_inflights > 4096) {
+		resp->status = DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION;
+		return -1;
+	}
+
+	/* Inflights must be <= number of sequence numbers if ordered */
+	if (args->num_sequence_numbers != 0 &&
+	    args->num_qid_inflights > args->num_sequence_numbers) {
+		resp->status = DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION;
+		return -1;
+	}
+
+	aqed_freelist = &domain->aqed_freelist;
+
+	if (dlb_freelist_count(aqed_freelist) < args->num_atomic_inflights) {
+		resp->status = DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+dlb_verify_create_dir_queue_args(struct dlb_hw *hw,
+				 u32 domain_id,
+				 struct dlb_create_dir_queue_args *args,
+				 struct dlb_cmd_response *resp,
+				 bool vf_request,
+				 unsigned int vf_id)
+{
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	/* If the user claims the port is already configured, validate the port
+	 * ID, its domain, and whether the port is configured.
+	 */
+	if (args->port_id != -1) {
+		struct dlb_dir_pq_pair *port;
+
+		port = dlb_get_domain_used_dir_pq(args->port_id,
+						  vf_request,
+						  domain);
+
+		if (!port || port->domain_id.phys_id != domain->id.phys_id ||
+		    !port->port_configured) {
+			resp->status = DLB_ST_INVALID_PORT_ID;
+			return -1;
+		}
+	}
+
+	/* If the queue's port is not configured, validate that a free
+	 * port-queue pair is available.
+	 */
+	if (args->port_id == -1 &&
+	    dlb_list_empty(&domain->avail_dir_pq_pairs)) {
+		resp->status = DLB_ST_DIR_QUEUES_UNAVAILABLE;
+		return -1;
+	}
+
+	return 0;
+}
+
+static void dlb_configure_ldb_queue(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    struct dlb_ldb_queue *queue,
+				    struct dlb_create_ldb_queue_args *args,
+				    bool vf_request,
+				    unsigned int vf_id)
+{
+	union dlb_sys_vf_ldb_vqid_v r0 = { {0} };
+	union dlb_sys_vf_ldb_vqid2qid r1 = { {0} };
+	union dlb_sys_ldb_qid2vqid r2 = { {0} };
+	union dlb_sys_ldb_vasqid_v r3 = { {0} };
+	union dlb_lsp_qid_ldb_infl_lim r4 = { {0} };
+	union dlb_lsp_qid_aqed_active_lim r5 = { {0} };
+	union dlb_aqed_pipe_fl_lim r6 = { {0} };
+	union dlb_aqed_pipe_fl_base r7 = { {0} };
+	union dlb_chp_ord_qid_sn_map r11 = { {0} };
+	union dlb_sys_ldb_qid_cfg_v r12 = { {0} };
+	union dlb_sys_ldb_qid_v r13 = { {0} };
+	union dlb_aqed_pipe_fl_push_ptr r14 = { {0} };
+	union dlb_aqed_pipe_fl_pop_ptr r15 = { {0} };
+	union dlb_aqed_pipe_qid_fid_lim r16 = { {0} };
+	union dlb_ro_pipe_qid2grpslt r17 = { {0} };
+	struct dlb_sn_group *sn_group;
+	unsigned int offs;
+
+	/* QID write permissions are turned on when the domain is started */
+	r3.field.vasqid_v = 0;
+
+	offs = domain->id.phys_id * DLB_MAX_NUM_LDB_QUEUES + queue->id.phys_id;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(offs), r3.val);
+
+	/* Unordered QIDs get 4K inflights, ordered get as many as the number
+	 * of sequence numbers.
+	 */
+	r4.field.limit = args->num_qid_inflights;
+
+	DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id.phys_id), r4.val);
+
+	r5.field.limit = queue->aqed_freelist.bound -
+			 queue->aqed_freelist.base;
+
+	if (r5.field.limit > DLB_MAX_NUM_AQOS_ENTRIES)
+		r5.field.limit = DLB_MAX_NUM_AQOS_ENTRIES;
+
+	/* AQOS */
+	DLB_CSR_WR(hw, DLB_LSP_QID_AQED_ACTIVE_LIM(queue->id.phys_id), r5.val);
+
+	r6.field.freelist_disable = 0;
+	r6.field.limit = queue->aqed_freelist.bound - 1;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_LIM(queue->id.phys_id), r6.val);
+
+	r7.field.base = queue->aqed_freelist.base;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_BASE(queue->id.phys_id), r7.val);
+
+	r14.field.push_ptr = r7.field.base;
+	r14.field.generation = 1;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_PUSH_PTR(queue->id.phys_id), r14.val);
+
+	r15.field.pop_ptr = r7.field.base;
+	r15.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_FL_POP_PTR(queue->id.phys_id), r15.val);
+
+	/* Configure SNs */
+	sn_group = &hw->rsrcs.sn_groups[queue->sn_group];
+	r11.field.mode = sn_group->mode;
+	r11.field.slot = queue->sn_slot;
+	r11.field.grp  = sn_group->id;
+
+	DLB_CSR_WR(hw, DLB_CHP_ORD_QID_SN_MAP(queue->id.phys_id), r11.val);
+
+	/* This register limits the number of inflight flows a queue can have
+	 * at one time.  It has an upper bound of 2048, but can be
+	 * over-subscribed. 512 is chosen so that a single queue doesn't use
+	 * the entire atomic storage, but can use a substantial portion if
+	 * needed.
+	 */
+	r16.field.qid_fid_limit = 512;
+
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_QID_FID_LIM(queue->id.phys_id), r16.val);
+
+	r17.field.group = sn_group->id;
+	r17.field.slot = queue->sn_slot;
+
+	DLB_CSR_WR(hw, DLB_RO_PIPE_QID2GRPSLT(queue->id.phys_id), r17.val);
+
+	r12.field.sn_cfg_v = (args->num_sequence_numbers != 0);
+	r12.field.fid_cfg_v = (args->num_atomic_inflights != 0);
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_QID_CFG_V(queue->id.phys_id), r12.val);
+
+	if (vf_request) {
+		unsigned int offs;
+
+		r0.field.vqid_v = 1;
+
+		offs = vf_id * DLB_MAX_NUM_LDB_QUEUES + queue->id.virt_id;
+
+		DLB_CSR_WR(hw, DLB_SYS_VF_LDB_VQID_V(offs), r0.val);
+
+		r1.field.qid = queue->id.phys_id;
+
+		DLB_CSR_WR(hw, DLB_SYS_VF_LDB_VQID2QID(offs), r1.val);
+
+		r2.field.vqid = queue->id.virt_id;
+
+		offs = vf_id * DLB_MAX_NUM_LDB_QUEUES + queue->id.phys_id;
+
+		DLB_CSR_WR(hw, DLB_SYS_LDB_QID2VQID(offs), r2.val);
+	}
+
+	r13.field.qid_v = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_QID_V(queue->id.phys_id), r13.val);
+}
+
+static void dlb_configure_dir_queue(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    struct dlb_dir_pq_pair *queue,
+				    bool vf_request,
+				    unsigned int vf_id)
+{
+	union dlb_sys_dir_vasqid_v r0 = { {0} };
+	unsigned int offs;
+
+	/* QID write permissions are turned on when the domain is started */
+	r0.field.vasqid_v = 0;
+
+	offs = (domain->id.phys_id * DLB_MAX_NUM_DIR_PORTS) + queue->id.phys_id;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(offs), r0.val);
+
+	if (vf_request) {
+		union dlb_sys_vf_dir_vqid_v   r1 = { {0} };
+		union dlb_sys_vf_dir_vqid2qid r2 = { {0} };
+
+		r1.field.vqid_v = 1;
+
+		offs = (vf_id * DLB_MAX_NUM_DIR_PORTS) + queue->id.virt_id;
+
+		DLB_CSR_WR(hw, DLB_SYS_VF_DIR_VQID_V(offs), r1.val);
+
+		r2.field.qid = queue->id.phys_id;
+
+		DLB_CSR_WR(hw, DLB_SYS_VF_DIR_VQID2QID(offs), r2.val);
+	} else {
+		union dlb_sys_dir_qid_v r3 = { {0} };
+
+		r3.field.qid_v = 1;
+
+		DLB_CSR_WR(hw, DLB_SYS_DIR_QID_V(queue->id.phys_id), r3.val);
+	}
+
+	queue->queue_configured = true;
+}
+
+static int
+dlb_verify_create_ldb_port_args(struct dlb_hw *hw,
+				u32 domain_id,
+				u64 pop_count_dma_base,
+				u64 cq_dma_base,
+				struct dlb_create_ldb_port_args *args,
+				struct dlb_cmd_response *resp,
+				bool vf_request,
+				unsigned int vf_id)
+{
+	struct dlb_domain *domain;
+	struct dlb_credit_pool *pool;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	if (dlb_list_empty(&domain->avail_ldb_ports)) {
+		resp->status = DLB_ST_LDB_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	/* If the scheduling domain has no LDB queues, we configure the
+	 * hardware to not supply the port with any LDB credits. In that
+	 * case, ignore the LDB credit arguments.
+	 */
+	if (!dlb_list_empty(&domain->used_ldb_queues) ||
+	    !dlb_list_empty(&domain->avail_ldb_queues)) {
+		pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+					       vf_request,
+					       domain);
+
+		if (!pool || !pool->configured ||
+		    pool->domain_id.phys_id != domain->id.phys_id) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_POOL_ID;
+			return -1;
+		}
+
+		if (args->ldb_credit_high_watermark > pool->avail_credits) {
+			resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+			return -1;
+		}
+
+		if (args->ldb_credit_low_watermark >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+	}
+
+	/* Likewise, if the scheduling domain has no DIR queues, we configure
+	 * the hardware to not supply the port with any DIR credits. In that
+	 * case, ignore the DIR credit arguments.
+	 */
+	if (!dlb_list_empty(&domain->used_dir_pq_pairs) ||
+	    !dlb_list_empty(&domain->avail_dir_pq_pairs)) {
+		pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+					       vf_request,
+					       domain);
+
+		if (!pool || !pool->configured ||
+		    pool->domain_id.phys_id != domain->id.phys_id) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_POOL_ID;
+			return -1;
+		}
+
+		if (args->dir_credit_high_watermark > pool->avail_credits) {
+			resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+			return -1;
+		}
+
+		if (args->dir_credit_low_watermark >=
+		    args->dir_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK;
+			return -1;
+		}
+
+		if (args->dir_credit_quantum >=
+		    args->dir_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+			return -1;
+		}
+
+		if (args->dir_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+			resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+			return -1;
+		}
+	}
+
+	/* Check cache-line alignment */
+	if ((pop_count_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_POP_COUNT_VIRT_ADDR;
+		return -1;
+	}
+
+	if ((cq_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_CQ_VIRT_ADDR;
+		return -1;
+	}
+
+	if (args->cq_depth != 1 &&
+	    args->cq_depth != 2 &&
+	    args->cq_depth != 4 &&
+	    args->cq_depth != 8 &&
+	    args->cq_depth != 16 &&
+	    args->cq_depth != 32 &&
+	    args->cq_depth != 64 &&
+	    args->cq_depth != 128 &&
+	    args->cq_depth != 256 &&
+	    args->cq_depth != 512 &&
+	    args->cq_depth != 1024) {
+		resp->status = DLB_ST_INVALID_CQ_DEPTH;
+		return -1;
+	}
+
+	/* The history list size must be >= 1 */
+	if (!args->cq_history_list_size) {
+		resp->status = DLB_ST_INVALID_HIST_LIST_DEPTH;
+		return -1;
+	}
+
+	if (args->cq_history_list_size > domain->avail_hist_list_entries) {
+		resp->status = DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+dlb_verify_create_dir_port_args(struct dlb_hw *hw,
+				u32 domain_id,
+				u64 pop_count_dma_base,
+				u64 cq_dma_base,
+				struct dlb_create_dir_port_args *args,
+				struct dlb_cmd_response *resp,
+				bool vf_request,
+				unsigned int vf_id)
+{
+	struct dlb_domain *domain;
+	struct dlb_credit_pool *pool;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	/* If the user claims the queue is already configured, validate
+	 * the queue ID, its domain, and whether the queue is configured.
+	 */
+	if (args->queue_id != -1) {
+		struct dlb_dir_pq_pair *queue;
+
+		queue = dlb_get_domain_used_dir_pq(args->queue_id,
+						   vf_request,
+						   domain);
+
+		if (!queue || queue->domain_id.phys_id != domain->id.phys_id ||
+		    !queue->queue_configured) {
+			resp->status = DLB_ST_INVALID_DIR_QUEUE_ID;
+			return -1;
+		}
+	}
+
+	/* If the port's queue is not configured, validate that a free
+	 * port-queue pair is available.
+	 */
+	if (args->queue_id == -1 &&
+	    dlb_list_empty(&domain->avail_dir_pq_pairs)) {
+		resp->status = DLB_ST_DIR_PORTS_UNAVAILABLE;
+		return -1;
+	}
+
+	/* If the scheduling domain has no LDB queues, we configure the
+	 * hardware to not supply the port with any LDB credits. In that
+	 * case, ignore the LDB credit arguments.
+	 */
+	if (!dlb_list_empty(&domain->used_ldb_queues) ||
+	    !dlb_list_empty(&domain->avail_ldb_queues)) {
+		pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+					       vf_request,
+					       domain);
+
+		if (!pool || !pool->configured ||
+		    pool->domain_id.phys_id != domain->id.phys_id) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_POOL_ID;
+			return -1;
+		}
+
+		if (args->ldb_credit_high_watermark > pool->avail_credits) {
+			resp->status = DLB_ST_LDB_CREDITS_UNAVAILABLE;
+			return -1;
+		}
+
+		if (args->ldb_credit_low_watermark >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum >=
+		    args->ldb_credit_high_watermark) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+
+		if (args->ldb_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+			resp->status = DLB_ST_INVALID_LDB_CREDIT_QUANTUM;
+			return -1;
+		}
+	}
+
+	pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+				       vf_request,
+				       domain);
+
+	if (!pool || !pool->configured ||
+	    pool->domain_id.phys_id != domain->id.phys_id) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_POOL_ID;
+		return -1;
+	}
+
+	if (args->dir_credit_high_watermark > pool->avail_credits) {
+		resp->status = DLB_ST_DIR_CREDITS_UNAVAILABLE;
+		return -1;
+	}
+
+	if (args->dir_credit_low_watermark >= args->dir_credit_high_watermark) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK;
+		return -1;
+	}
+
+	if (args->dir_credit_quantum >= args->dir_credit_high_watermark) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+		return -1;
+	}
+
+	if (args->dir_credit_quantum > DLB_MAX_PORT_CREDIT_QUANTUM) {
+		resp->status = DLB_ST_INVALID_DIR_CREDIT_QUANTUM;
+		return -1;
+	}
+
+	/* Check cache-line alignment */
+	if ((pop_count_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_POP_COUNT_VIRT_ADDR;
+		return -1;
+	}
+
+	if ((cq_dma_base & 0x3F) != 0) {
+		resp->status = DLB_ST_INVALID_CQ_VIRT_ADDR;
+		return -1;
+	}
+
+	if (args->cq_depth != 8 &&
+	    args->cq_depth != 16 &&
+	    args->cq_depth != 32 &&
+	    args->cq_depth != 64 &&
+	    args->cq_depth != 128 &&
+	    args->cq_depth != 256 &&
+	    args->cq_depth != 512 &&
+	    args->cq_depth != 1024) {
+		resp->status = DLB_ST_INVALID_CQ_DEPTH;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int dlb_verify_start_domain_args(struct dlb_hw *hw,
+					u32 domain_id,
+					struct dlb_cmd_response *resp,
+					bool vf_request,
+					unsigned int vf_id)
+{
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	if (domain->started) {
+		resp->status = DLB_ST_DOMAIN_STARTED;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int dlb_verify_map_qid_args(struct dlb_hw *hw,
+				   u32 domain_id,
+				   struct dlb_map_qid_args *args,
+				   struct dlb_cmd_response *resp,
+				   bool vf_request,
+				   unsigned int vf_id)
+{
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+	struct dlb_ldb_queue *queue;
+	int id;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, vf_request, domain);
+
+	if (!port || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	if (args->priority >= DLB_QID_PRIORITIES) {
+		resp->status = DLB_ST_INVALID_PRIORITY;
+		return -1;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, vf_request, domain);
+
+	if (!queue || !queue->configured) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -1;
+	}
+
+	if (queue->domain_id.phys_id != domain->id.phys_id) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -1;
+	}
+
+	if (port->domain_id.phys_id != domain->id.phys_id) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	return 0;
+}
+
+static bool dlb_port_find_slot(struct dlb_ldb_port *port,
+			       enum dlb_qid_map_state state,
+			       int *slot)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (port->qid_map[i].state == state)
+			break;
+	}
+
+	*slot = i;
+
+	return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
+}
+
+static bool dlb_port_find_slot_queue(struct dlb_ldb_port *port,
+				     enum dlb_qid_map_state state,
+				     struct dlb_ldb_queue *queue,
+				     int *slot)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (port->qid_map[i].state == state &&
+		    port->qid_map[i].qid == queue->id.phys_id)
+			break;
+	}
+
+	*slot = i;
+
+	return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
+}
+
+static bool
+dlb_port_find_slot_with_pending_map_queue(struct dlb_ldb_port *port,
+					  struct dlb_ldb_queue *queue,
+					  int *slot)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		struct dlb_ldb_port_qid_map *map = &port->qid_map[i];
+
+		if (map->state == DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP &&
+		    map->pending_qid == queue->id.phys_id)
+			break;
+	}
+
+	*slot = i;
+
+	return (i < DLB_MAX_NUM_QIDS_PER_LDB_CQ);
+}
+
+static int dlb_port_slot_state_transition(struct dlb_hw *hw,
+					  struct dlb_ldb_port *port,
+					  struct dlb_ldb_queue *queue,
+					  int slot,
+					  enum dlb_qid_map_state new_state)
+{
+	enum dlb_qid_map_state curr_state = port->qid_map[slot].state;
+	struct dlb_domain *domain;
+
+	domain = dlb_get_domain_from_id(hw, port->domain_id.phys_id, false, 0);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: unable to find domain %d\n",
+			   __func__, port->domain_id.phys_id);
+		return -EFAULT;
+	}
+
+	switch (curr_state) {
+	case DLB_QUEUE_UNMAPPED:
+		switch (new_state) {
+		case DLB_QUEUE_MAPPED:
+			queue->num_mappings++;
+			port->num_mappings++;
+			break;
+		case DLB_QUEUE_MAP_IN_PROGRESS:
+			queue->num_pending_additions++;
+			domain->num_pending_additions++;
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_MAPPED:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAPPED:
+			queue->num_mappings--;
+			port->num_mappings--;
+			break;
+		case DLB_QUEUE_UNMAP_IN_PROGRESS:
+			port->num_pending_removals++;
+			domain->num_pending_removals++;
+			break;
+		case DLB_QUEUE_MAPPED:
+			/* Priority change, nothing to update */
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_MAP_IN_PROGRESS:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAPPED:
+			queue->num_pending_additions--;
+			domain->num_pending_additions--;
+			break;
+		case DLB_QUEUE_MAPPED:
+			queue->num_mappings++;
+			port->num_mappings++;
+			queue->num_pending_additions--;
+			domain->num_pending_additions--;
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_UNMAP_IN_PROGRESS:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAPPED:
+			port->num_pending_removals--;
+			domain->num_pending_removals--;
+			queue->num_mappings--;
+			port->num_mappings--;
+			break;
+		case DLB_QUEUE_MAPPED:
+			port->num_pending_removals--;
+			domain->num_pending_removals--;
+			break;
+		case DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP:
+			/* Nothing to update */
+			break;
+		default:
+			goto error;
+		}
+		break;
+	case DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP:
+		switch (new_state) {
+		case DLB_QUEUE_UNMAP_IN_PROGRESS:
+			/* Nothing to update */
+			break;
+		case DLB_QUEUE_UNMAPPED:
+			/* An UNMAP_IN_PROGRESS_PENDING_MAP slot briefly
+			 * becomes UNMAPPED before it transitions to
+			 * MAP_IN_PROGRESS.
+			 */
+			queue->num_mappings--;
+			port->num_mappings--;
+			port->num_pending_removals--;
+			domain->num_pending_removals--;
+			break;
+		default:
+			goto error;
+		}
+		break;
+	default:
+		goto error;
+	}
+
+	port->qid_map[slot].state = new_state;
+
+	DLB_HW_INFO(hw,
+		    "[%s()] queue %d -> port %d state transition (%d -> %d)\n",
+		    __func__, queue->id.phys_id, port->id.phys_id, curr_state,
+		    new_state);
+	return 0;
+
+error:
+	DLB_HW_ERR(hw,
+		   "[%s()] Internal error: invalid queue %d -> port %d state transition (%d -> %d)\n",
+		   __func__, queue->id.phys_id, port->id.phys_id, curr_state,
+		   new_state);
+	return -EFAULT;
+}
+
+static int dlb_verify_map_qid_slot_available(struct dlb_ldb_port *port,
+					     struct dlb_ldb_queue *queue,
+					     struct dlb_cmd_response *resp)
+{
+	enum dlb_qid_map_state state;
+	int i;
+
+	/* Unused slot available? */
+	if (port->num_mappings < DLB_MAX_NUM_QIDS_PER_LDB_CQ)
+		return 0;
+
+	/* If the queue is already mapped (from the application's perspective),
+	 * this is simply a priority update.
+	 */
+	state = DLB_QUEUE_MAPPED;
+	if (dlb_port_find_slot_queue(port, state, queue, &i))
+		return 0;
+
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i))
+		return 0;
+
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &i))
+		return 0;
+
+	/* If the slot contains an unmap in progress, it's considered
+	 * available.
+	 */
+	state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	if (dlb_port_find_slot(port, state, &i))
+		return 0;
+
+	state = DLB_QUEUE_UNMAPPED;
+	if (dlb_port_find_slot(port, state, &i))
+		return 0;
+
+	resp->status = DLB_ST_NO_QID_SLOTS_AVAILABLE;
+	return -EINVAL;
+}
+
+static int dlb_verify_unmap_qid_args(struct dlb_hw *hw,
+				     u32 domain_id,
+				     struct dlb_unmap_qid_args *args,
+				     struct dlb_cmd_response *resp,
+				     bool vf_request,
+				     unsigned int vf_id)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+	struct dlb_ldb_queue *queue;
+	int slot;
+	int id;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, vf_request, domain);
+
+	if (!port || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	if (port->domain_id.phys_id != domain->id.phys_id) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, vf_request, domain);
+
+	if (!queue || !queue->configured) {
+		DLB_HW_ERR(hw, "[%s()] Can't unmap unconfigured queue %d\n",
+			   __func__, args->qid);
+		resp->status = DLB_ST_INVALID_QID;
+		return -1;
+	}
+
+	/* Verify that the port has the queue mapped. From the application's
+	 * perspective a queue is mapped if it is actually mapped, the map is
+	 * in progress, or the map is blocked pending an unmap.
+	 */
+	state = DLB_QUEUE_MAPPED;
+	if (dlb_port_find_slot_queue(port, state, queue, &slot))
+		return 0;
+
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &slot))
+		return 0;
+
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &slot))
+		return 0;
+
+	resp->status = DLB_ST_INVALID_QID;
+	return -1;
+}
+
+static int
+dlb_verify_enable_ldb_port_args(struct dlb_hw *hw,
+				u32 domain_id,
+				struct dlb_enable_ldb_port_args *args,
+				struct dlb_cmd_response *resp,
+				bool vf_request,
+				unsigned int vf_id)
+{
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+	int id;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, vf_request, domain);
+
+	if (!port || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+dlb_verify_enable_dir_port_args(struct dlb_hw *hw,
+				u32 domain_id,
+				struct dlb_enable_dir_port_args *args,
+				struct dlb_cmd_response *resp,
+				bool vf_request,
+				unsigned int vf_id)
+{
+	struct dlb_domain *domain;
+	struct dlb_dir_pq_pair *port;
+	int id;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_dir_pq(id, vf_request, domain);
+
+	if (!port || !port->port_configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+dlb_verify_disable_ldb_port_args(struct dlb_hw *hw,
+				 u32 domain_id,
+				 struct dlb_disable_ldb_port_args *args,
+				 struct dlb_cmd_response *resp,
+				 bool vf_request,
+				 unsigned int vf_id)
+{
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+	int id;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, vf_request, domain);
+
+	if (!port || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+dlb_verify_disable_dir_port_args(struct dlb_hw *hw,
+				 u32 domain_id,
+				 struct dlb_disable_dir_port_args *args,
+				 struct dlb_cmd_response *resp,
+				 bool vf_request,
+				 unsigned int vf_id)
+{
+	struct dlb_domain *domain;
+	struct dlb_dir_pq_pair *port;
+	int id;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -1;
+	}
+
+	if (!domain->configured) {
+		resp->status = DLB_ST_DOMAIN_NOT_CONFIGURED;
+		return -1;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_dir_pq(id, vf_request, domain);
+
+	if (!port || !port->port_configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+dlb_domain_attach_resources(struct dlb_hw *hw,
+			    struct dlb_function_resources *rsrcs,
+			    struct dlb_domain *domain,
+			    struct dlb_create_sched_domain_args *args,
+			    struct dlb_cmd_response *resp)
+{
+	int ret;
+
+	ret = dlb_attach_ldb_queues(hw,
+				    rsrcs,
+				    domain,
+				    args->num_ldb_queues,
+				    resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_ldb_ports(hw,
+				   rsrcs,
+				   domain,
+				   args->num_ldb_ports,
+				   resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_dir_ports(hw,
+				   rsrcs,
+				   domain,
+				   args->num_dir_ports,
+				   resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_ldb_credits(rsrcs,
+				     domain,
+				     args->num_ldb_credits,
+				     resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_dir_credits(rsrcs,
+				     domain,
+				     args->num_dir_credits,
+				     resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_ldb_credit_pools(hw,
+					  rsrcs,
+					  domain,
+					  args->num_ldb_credit_pools,
+					  resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_dir_credit_pools(hw,
+					  rsrcs,
+					  domain,
+					  args->num_dir_credit_pools,
+					  resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_domain_hist_list_entries(rsrcs,
+						  domain,
+						  args->num_hist_list_entries,
+						  resp);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_attach_atomic_inflights(rsrcs,
+					  domain,
+					  args->num_atomic_inflights,
+					  resp);
+	if (ret < 0)
+		return ret;
+
+	domain->configured = true;
+
+	domain->started = false;
+
+	rsrcs->num_avail_domains--;
+
+	return 0;
+}
+
+static int
+dlb_ldb_queue_attach_to_sn_group(struct dlb_hw *hw,
+				 struct dlb_ldb_queue *queue,
+				 struct dlb_create_ldb_queue_args *args)
+{
+	int slot = -1;
+	int i;
+
+	queue->sn_cfg_valid = false;
+
+	if (args->num_sequence_numbers == 0)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS; i++) {
+		struct dlb_sn_group *group = &hw->rsrcs.sn_groups[i];
+
+		if (group->sequence_numbers_per_queue ==
+		    args->num_sequence_numbers &&
+		    !dlb_sn_group_full(group)) {
+			slot = dlb_sn_group_alloc_slot(group);
+			if (slot >= 0)
+				break;
+		}
+	}
+
+	if (slot == -1) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no sequence number slots available\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue->sn_cfg_valid = true;
+	queue->sn_group = i;
+	queue->sn_slot = slot;
+	return 0;
+}
+
+static int
+dlb_ldb_queue_attach_resources(struct dlb_hw *hw,
+			       struct dlb_domain *domain,
+			       struct dlb_ldb_queue *queue,
+			       struct dlb_create_ldb_queue_args *args)
+{
+	int ret;
+
+	ret = dlb_ldb_queue_attach_to_sn_group(hw, queue, args);
+	if (ret)
+		return ret;
+
+	/* Attach QID inflights */
+	queue->num_qid_inflights = args->num_qid_inflights;
+
+	/* Attach atomic inflights */
+	queue->aqed_freelist.base = domain->aqed_freelist.base +
+				    domain->aqed_freelist.offset;
+	queue->aqed_freelist.bound = queue->aqed_freelist.base +
+				     args->num_atomic_inflights;
+	domain->aqed_freelist.offset += args->num_atomic_inflights;
+
+	return 0;
+}
+
+static void dlb_ldb_port_cq_enable(struct dlb_hw *hw,
+				   struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_dsbl reg;
+
+	/* Don't re-enable the port if a removal is pending. The caller should
+	 * mark this port as enabled (if it isn't already), and when the
+	 * removal completes the port will be enabled.
+	 */
+	if (port->num_pending_removals)
+		return;
+
+	reg.field.disabled = 0;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_DSBL(port->id.phys_id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_ldb_port_cq_disable(struct dlb_hw *hw,
+				    struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_dsbl reg;
+
+	reg.field.disabled = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_DSBL(port->id.phys_id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_dir_port_cq_enable(struct dlb_hw *hw,
+				   struct dlb_dir_pq_pair *port)
+{
+	union dlb_lsp_cq_dir_dsbl reg;
+
+	reg.field.disabled = 0;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_DIR_DSBL(port->id.phys_id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_dir_port_cq_disable(struct dlb_hw *hw,
+				    struct dlb_dir_pq_pair *port)
+{
+	union dlb_lsp_cq_dir_dsbl reg;
+
+	reg.field.disabled = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_DIR_DSBL(port->id.phys_id), reg.val);
+
+	dlb_flush_csr(hw);
+}
+
+static int dlb_ldb_port_configure_pp(struct dlb_hw *hw,
+				     struct dlb_domain *domain,
+				     struct dlb_ldb_port *port,
+				     struct dlb_create_ldb_port_args *args,
+				     bool vf_request,
+				     unsigned int vf_id)
+{
+	union dlb_sys_ldb_pp2ldbpool r0 = { {0} };
+	union dlb_sys_ldb_pp2dirpool r1 = { {0} };
+	union dlb_sys_ldb_pp2vf_pf r2 = { {0} };
+	union dlb_sys_ldb_pp2vas r3 = { {0} };
+	union dlb_sys_ldb_pp_v r4 = { {0} };
+	union dlb_sys_ldb_pp2vpp r5 = { {0} };
+	union dlb_chp_ldb_pp_ldb_crd_hwm r6 = { {0} };
+	union dlb_chp_ldb_pp_dir_crd_hwm r7 = { {0} };
+	union dlb_chp_ldb_pp_ldb_crd_lwm r8 = { {0} };
+	union dlb_chp_ldb_pp_dir_crd_lwm r9 = { {0} };
+	union dlb_chp_ldb_pp_ldb_min_crd_qnt r10 = { {0} };
+	union dlb_chp_ldb_pp_dir_min_crd_qnt r11 = { {0} };
+	union dlb_chp_ldb_pp_ldb_crd_cnt r12 = { {0} };
+	union dlb_chp_ldb_pp_dir_crd_cnt r13 = { {0} };
+	union dlb_chp_ldb_ldb_pp2pool r14 = { {0} };
+	union dlb_chp_ldb_dir_pp2pool r15 = { {0} };
+	union dlb_chp_ldb_pp_crd_req_state r16 = { {0} };
+	union dlb_chp_ldb_pp_ldb_push_ptr r17 = { {0} };
+	union dlb_chp_ldb_pp_dir_push_ptr r18 = { {0} };
+
+	struct dlb_credit_pool *ldb_pool = NULL;
+	struct dlb_credit_pool *dir_pool = NULL;
+	unsigned int offs;
+
+	if (port->ldb_pool_used) {
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   vf_request,
+						   domain);
+		if (!ldb_pool) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	if (port->dir_pool_used) {
+		dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+						   vf_request,
+						   domain);
+		if (!dir_pool) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	r0.field.ldbpool = (port->ldb_pool_used) ? ldb_pool->id.phys_id : 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2LDBPOOL(port->id.phys_id), r0.val);
+
+	r1.field.dirpool = (port->dir_pool_used) ? dir_pool->id.phys_id : 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2DIRPOOL(port->id.phys_id), r1.val);
+
+	r2.field.vf = vf_id;
+	r2.field.is_pf = !vf_request;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2VF_PF(port->id.phys_id), r2.val);
+
+	r3.field.vas = domain->id.phys_id;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2VAS(port->id.phys_id), r3.val);
+
+	r5.field.vpp = port->id.virt_id;
+
+	offs = (vf_id * DLB_MAX_NUM_LDB_PORTS) + port->id.phys_id;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP2VPP(offs), r5.val);
+
+	r6.field.hwm = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_CRD_HWM(port->id.phys_id), r6.val);
+
+	r7.field.hwm = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_CRD_HWM(port->id.phys_id), r7.val);
+
+	r8.field.lwm = args->ldb_credit_low_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_CRD_LWM(port->id.phys_id), r8.val);
+
+	r9.field.lwm = args->dir_credit_low_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_CRD_LWM(port->id.phys_id), r9.val);
+
+	r10.field.quanta = args->ldb_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(port->id.phys_id),
+		   r10.val);
+
+	r11.field.quanta = args->dir_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(port->id.phys_id),
+		   r11.val);
+
+	r12.field.count = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_CRD_CNT(port->id.phys_id), r12.val);
+
+	r13.field.count = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_CRD_CNT(port->id.phys_id), r13.val);
+
+	r14.field.pool = (port->ldb_pool_used) ? ldb_pool->id.phys_id : 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_LDB_PP2POOL(port->id.phys_id), r14.val);
+
+	r15.field.pool = (port->dir_pool_used) ? dir_pool->id.phys_id : 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_DIR_PP2POOL(port->id.phys_id), r15.val);
+
+	r16.field.no_pp_credit_update = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id.phys_id), r16.val);
+
+	r17.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_LDB_PUSH_PTR(port->id.phys_id), r17.val);
+
+	r18.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_PP_DIR_PUSH_PTR(port->id.phys_id), r18.val);
+
+	if (vf_request) {
+		union dlb_sys_vf_ldb_vpp2pp r16 = { {0} };
+		union dlb_sys_vf_ldb_vpp_v r17 = { {0} };
+
+		r16.field.pp = port->id.phys_id;
+
+		offs = vf_id * DLB_MAX_NUM_LDB_PORTS + port->id.virt_id;
+
+		DLB_CSR_WR(hw, DLB_SYS_VF_LDB_VPP2PP(offs), r16.val);
+
+		r17.field.vpp_v = 1;
+
+		DLB_CSR_WR(hw, DLB_SYS_VF_LDB_VPP_V(offs), r17.val);
+	}
+
+	r4.field.pp_v = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_V(port->id.phys_id),
+		   r4.val);
+
+	return 0;
+}
+
+static int dlb_ldb_port_configure_cq(struct dlb_hw *hw,
+				     struct dlb_ldb_port *port,
+				     u64 pop_count_dma_base,
+				     u64 cq_dma_base,
+				     struct dlb_create_ldb_port_args *args,
+				     bool vf_request,
+				     unsigned int vf_id)
+{
+	int i;
+
+	union dlb_sys_ldb_cq_addr_l r0 = { {0} };
+	union dlb_sys_ldb_cq_addr_u r1 = { {0} };
+	union dlb_sys_ldb_cq2vf_pf r2 = { {0} };
+	union dlb_chp_ldb_cq_tkn_depth_sel r3 = { {0} };
+	union dlb_chp_hist_list_lim r4 = { {0} };
+	union dlb_chp_hist_list_base r5 = { {0} };
+	union dlb_lsp_cq_ldb_infl_lim r6 = { {0} };
+	union dlb_lsp_cq2priov r7 = { {0} };
+	union dlb_chp_hist_list_push_ptr r8 = { {0} };
+	union dlb_chp_hist_list_pop_ptr r9 = { {0} };
+	union dlb_lsp_cq_ldb_tkn_depth_sel r10 = { {0} };
+	union dlb_sys_ldb_pp_addr_l r11 = { {0} };
+	union dlb_sys_ldb_pp_addr_u r12 = { {0} };
+
+	/* The CQ address is 64B-aligned, and the DLB only wants bits [63:6] */
+	r0.field.addr_l = cq_dma_base >> 6;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_L(port->id.phys_id),
+		   r0.val);
+
+	r1.field.addr_u = cq_dma_base >> 32;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_U(port->id.phys_id),
+		   r1.val);
+
+	r2.field.vf = vf_id;
+	r2.field.is_pf = !vf_request;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ2VF_PF(port->id.phys_id),
+		   r2.val);
+
+	if (args->cq_depth <= 8) {
+		r3.field.token_depth_select = 1;
+	} else if (args->cq_depth == 16) {
+		r3.field.token_depth_select = 2;
+	} else if (args->cq_depth == 32) {
+		r3.field.token_depth_select = 3;
+	} else if (args->cq_depth == 64) {
+		r3.field.token_depth_select = 4;
+	} else if (args->cq_depth == 128) {
+		r3.field.token_depth_select = 5;
+	} else if (args->cq_depth == 256) {
+		r3.field.token_depth_select = 6;
+	} else if (args->cq_depth == 512) {
+		r3.field.token_depth_select = 7;
+	} else if (args->cq_depth == 1024) {
+		r3.field.token_depth_select = 8;
+	} else {
+		DLB_HW_ERR(hw, "[%s():%d] Internal error: invalid CQ depth\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(port->id.phys_id),
+		   r3.val);
+
+	r10.field.token_depth_select = r3.field.token_depth_select;
+	r10.field.ignore_depth = 0;
+	/* TDT algorithm: DLB must be able to write CQs with depth < 4 */
+	r10.field.enab_shallow_cq = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(port->id.phys_id),
+		   r10.val);
+
+	/* To support CQs with depth less than 8, program the token count
+	 * register with a non-zero initial value. Operations such as domain
+	 * reset must take this initial value into account when quiescing the
+	 * CQ.
+	 */
+	port->init_tkn_cnt = 0;
+
+	if (args->cq_depth < 8) {
+		union dlb_lsp_cq_ldb_tkn_cnt r12 = { {0} };
+
+		port->init_tkn_cnt = 8 - args->cq_depth;
+
+		r12.field.token_count = port->init_tkn_cnt;
+
+		DLB_CSR_WR(hw,
+			   DLB_LSP_CQ_LDB_TKN_CNT(port->id.phys_id),
+			   r12.val);
+	}
+
+	r4.field.limit = port->hist_list_entry_limit - 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_LIM(port->id.phys_id), r4.val);
+
+	r5.field.base = port->hist_list_entry_base;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_BASE(port->id.phys_id), r5.val);
+
+	r8.field.push_ptr = r5.field.base;
+	r8.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_PUSH_PTR(port->id.phys_id), r8.val);
+
+	r9.field.pop_ptr = r5.field.base;
+	r9.field.generation = 0;
+
+	DLB_CSR_WR(hw, DLB_CHP_HIST_LIST_POP_PTR(port->id.phys_id), r9.val);
+
+	/* The inflight limit sets a cap on the number of QEs for which this CQ
+	 * can owe completions at one time.
+	 */
+	r6.field.limit = args->cq_history_list_size;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ_LDB_INFL_LIM(port->id.phys_id), r6.val);
+
+	/* Disable the port's QID mappings */
+	r7.field.v = 0;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port->id.phys_id), r7.val);
+
+	/* Two cache lines (128B) are dedicated for the port's pop counts */
+	r11.field.addr_l = pop_count_dma_base >> 7;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP_ADDR_L(port->id.phys_id), r11.val);
+
+	r12.field.addr_u = pop_count_dma_base >> 32;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_PP_ADDR_U(port->id.phys_id), r12.val);
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++)
+		port->qid_map[i].state = DLB_QUEUE_UNMAPPED;
+
+	return 0;
+}
+
+static void dlb_update_ldb_arb_threshold(struct dlb_hw *hw)
+{
+	union dlb_lsp_ctrl_config_0 r0 = { {0} };
+
+	/* From the hardware spec:
+	 * "The optimal value for ldb_arb_threshold is in the region of {8 *
+	 * #CQs}. It is expected therefore that the PF will change this value
+	 * dynamically as the number of active ports changes."
+	 */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CTRL_CONFIG_0);
+
+	r0.field.ldb_arb_threshold = hw->pf.num_enabled_ldb_ports * 8;
+	r0.field.ldb_arb_ignore_empty = 1;
+	r0.field.ldb_arb_mode = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_CTRL_CONFIG_0, r0.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_ldb_pool_update_credit_count(struct dlb_hw *hw,
+					     u32 pool_id,
+					     u32 count)
+{
+	hw->rsrcs.ldb_credit_pools[pool_id].avail_credits -= count;
+}
+
+static void dlb_dir_pool_update_credit_count(struct dlb_hw *hw,
+					     u32 pool_id,
+					     u32 count)
+{
+	hw->rsrcs.dir_credit_pools[pool_id].avail_credits -= count;
+}
+
+static void dlb_ldb_pool_write_credit_count_reg(struct dlb_hw *hw,
+						u32 pool_id)
+{
+	union dlb_chp_ldb_pool_crd_cnt r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	pool = &hw->rsrcs.ldb_credit_pools[pool_id];
+
+	r0.field.count = pool->avail_credits;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_POOL_CRD_CNT(pool->id.phys_id),
+		   r0.val);
+}
+
+static void dlb_dir_pool_write_credit_count_reg(struct dlb_hw *hw,
+						u32 pool_id)
+{
+	union dlb_chp_dir_pool_crd_cnt r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	pool = &hw->rsrcs.dir_credit_pools[pool_id];
+
+	r0.field.count = pool->avail_credits;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_POOL_CRD_CNT(pool->id.phys_id),
+		   r0.val);
+}
+
+static int dlb_configure_ldb_port(struct dlb_hw *hw,
+				  struct dlb_domain *domain,
+				  struct dlb_ldb_port *port,
+				  u64 pop_count_dma_base,
+				  u64 cq_dma_base,
+				  struct dlb_create_ldb_port_args *args,
+				  bool vf_request,
+				  unsigned int vf_id)
+{
+	struct dlb_credit_pool *ldb_pool, *dir_pool;
+	int ret;
+
+	port->hist_list_entry_base = domain->hist_list_entry_base +
+				     domain->hist_list_entry_offset;
+	port->hist_list_entry_limit = port->hist_list_entry_base +
+				      args->cq_history_list_size;
+
+	domain->hist_list_entry_offset += args->cq_history_list_size;
+	domain->avail_hist_list_entries -= args->cq_history_list_size;
+
+	port->ldb_pool_used = !dlb_list_empty(&domain->used_ldb_queues) ||
+			      !dlb_list_empty(&domain->avail_ldb_queues);
+	port->dir_pool_used = !dlb_list_empty(&domain->used_dir_pq_pairs) ||
+			      !dlb_list_empty(&domain->avail_dir_pq_pairs);
+
+	if (port->ldb_pool_used) {
+		u32 cnt = args->ldb_credit_high_watermark;
+
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   vf_request,
+						   domain);
+		if (!ldb_pool) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+
+		dlb_ldb_pool_update_credit_count(hw, ldb_pool->id.phys_id, cnt);
+	} else {
+		args->ldb_credit_high_watermark = 0;
+		args->ldb_credit_low_watermark = 0;
+		args->ldb_credit_quantum = 0;
+	}
+
+	if (port->dir_pool_used) {
+		u32 cnt = args->dir_credit_high_watermark;
+
+		dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+						   vf_request,
+						   domain);
+		if (!dir_pool) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+
+		dlb_dir_pool_update_credit_count(hw, dir_pool->id.phys_id, cnt);
+	} else {
+		args->dir_credit_high_watermark = 0;
+		args->dir_credit_low_watermark = 0;
+		args->dir_credit_quantum = 0;
+	}
+
+	ret = dlb_ldb_port_configure_cq(hw,
+					port,
+					pop_count_dma_base,
+					cq_dma_base,
+					args,
+					vf_request,
+					vf_id);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_ldb_port_configure_pp(hw,
+					domain,
+					port,
+					args,
+					vf_request,
+					vf_id);
+	if (ret < 0)
+		return ret;
+
+	dlb_ldb_port_cq_enable(hw, port);
+
+	port->num_mappings = 0;
+
+	port->enabled = true;
+
+	hw->pf.num_enabled_ldb_ports++;
+
+	dlb_update_ldb_arb_threshold(hw);
+
+	port->configured = true;
+
+	return 0;
+}
+
+static int dlb_dir_port_configure_pp(struct dlb_hw *hw,
+				     struct dlb_domain *domain,
+				     struct dlb_dir_pq_pair *port,
+				     struct dlb_create_dir_port_args *args,
+				     bool vf_request,
+				     unsigned int vf_id)
+{
+	union dlb_sys_dir_pp2ldbpool r0 = { {0} };
+	union dlb_sys_dir_pp2dirpool r1 = { {0} };
+	union dlb_sys_dir_pp2vf_pf r2 = { {0} };
+	union dlb_sys_dir_pp2vas r3 = { {0} };
+	union dlb_sys_dir_pp_v r4 = { {0} };
+	union dlb_sys_dir_pp2vpp r5 = { {0} };
+	union dlb_chp_dir_pp_ldb_crd_hwm r6 = { {0} };
+	union dlb_chp_dir_pp_dir_crd_hwm r7 = { {0} };
+	union dlb_chp_dir_pp_ldb_crd_lwm r8 = { {0} };
+	union dlb_chp_dir_pp_dir_crd_lwm r9 = { {0} };
+	union dlb_chp_dir_pp_ldb_min_crd_qnt r10 = { {0} };
+	union dlb_chp_dir_pp_dir_min_crd_qnt r11 = { {0} };
+	union dlb_chp_dir_pp_ldb_crd_cnt r12 = { {0} };
+	union dlb_chp_dir_pp_dir_crd_cnt r13 = { {0} };
+	union dlb_chp_dir_ldb_pp2pool r14 = { {0} };
+	union dlb_chp_dir_dir_pp2pool r15 = { {0} };
+	union dlb_chp_dir_pp_crd_req_state r16 = { {0} };
+	union dlb_chp_dir_pp_ldb_push_ptr r17 = { {0} };
+	union dlb_chp_dir_pp_dir_push_ptr r18 = { {0} };
+
+	struct dlb_credit_pool *ldb_pool = NULL;
+	struct dlb_credit_pool *dir_pool = NULL;
+
+	if (port->ldb_pool_used) {
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   vf_request,
+						   domain);
+		if (!ldb_pool) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	if (port->dir_pool_used) {
+		dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+						   vf_request,
+						   domain);
+		if (!dir_pool) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+	}
+
+	r0.field.ldbpool = (port->ldb_pool_used) ? ldb_pool->id.phys_id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2LDBPOOL(port->id.phys_id),
+		   r0.val);
+
+	r1.field.dirpool = (port->dir_pool_used) ? dir_pool->id.phys_id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2DIRPOOL(port->id.phys_id),
+		   r1.val);
+
+	r2.field.vf = vf_id;
+	r2.field.is_pf = !vf_request;
+	r2.field.is_hw_dsi = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VF_PF(port->id.phys_id),
+		   r2.val);
+
+	r3.field.vas = domain->id.phys_id;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VAS(port->id.phys_id),
+		   r3.val);
+
+	r5.field.vpp = port->id.virt_id;
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VPP((vf_id * DLB_MAX_NUM_DIR_PORTS) +
+				      port->id.phys_id),
+		   r5.val);
+
+	r6.field.hwm = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_HWM(port->id.phys_id),
+		   r6.val);
+
+	r7.field.hwm = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_HWM(port->id.phys_id),
+		   r7.val);
+
+	r8.field.lwm = args->ldb_credit_low_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_LWM(port->id.phys_id),
+		   r8.val);
+
+	r9.field.lwm = args->dir_credit_low_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_LWM(port->id.phys_id),
+		   r9.val);
+
+	r10.field.quanta = args->ldb_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(port->id.phys_id),
+		   r10.val);
+
+	r11.field.quanta = args->dir_credit_quantum;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(port->id.phys_id),
+		   r11.val);
+
+	r12.field.count = args->ldb_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_CNT(port->id.phys_id),
+		   r12.val);
+
+	r13.field.count = args->dir_credit_high_watermark;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_CNT(port->id.phys_id),
+		   r13.val);
+
+	r14.field.pool = (port->ldb_pool_used) ? ldb_pool->id.phys_id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_LDB_PP2POOL(port->id.phys_id),
+		   r14.val);
+
+	r15.field.pool = (port->dir_pool_used) ? dir_pool->id.phys_id : 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_DIR_PP2POOL(port->id.phys_id),
+		   r15.val);
+
+	r16.field.no_pp_credit_update = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id.phys_id),
+		   r16.val);
+
+	r17.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_PUSH_PTR(port->id.phys_id),
+		   r17.val);
+
+	r18.field.push_pointer = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_PUSH_PTR(port->id.phys_id),
+		   r18.val);
+
+	if (vf_request) {
+		union dlb_sys_vf_dir_vpp2pp r16 = { {0} };
+		union dlb_sys_vf_dir_vpp_v r17 = { {0} };
+		unsigned int offs;
+
+		r16.field.pp = port->id.phys_id;
+
+		offs = vf_id * DLB_MAX_NUM_DIR_PORTS + port->id.virt_id;
+
+		DLB_CSR_WR(hw, DLB_SYS_VF_DIR_VPP2PP(offs), r16.val);
+
+		r17.field.vpp_v = 1;
+
+		DLB_CSR_WR(hw, DLB_SYS_VF_DIR_VPP_V(offs), r17.val);
+	}
+
+	r4.field.pp_v = 1;
+	r4.field.mb_dm = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_PP_V(port->id.phys_id), r4.val);
+
+	return 0;
+}
+
+static int dlb_dir_port_configure_cq(struct dlb_hw *hw,
+				     struct dlb_dir_pq_pair *port,
+				     u64 pop_count_dma_base,
+				     u64 cq_dma_base,
+				     struct dlb_create_dir_port_args *args,
+				     bool vf_request,
+				     unsigned int vf_id)
+{
+	union dlb_sys_dir_cq_addr_l r0 = { {0} };
+	union dlb_sys_dir_cq_addr_u r1 = { {0} };
+	union dlb_sys_dir_cq2vf_pf r2 = { {0} };
+	union dlb_chp_dir_cq_tkn_depth_sel r3 = { {0} };
+	union dlb_lsp_cq_dir_tkn_depth_sel_dsi r4 = { {0} };
+	union dlb_sys_dir_pp_addr_l r5 = { {0} };
+	union dlb_sys_dir_pp_addr_u r6 = { {0} };
+
+	/* The CQ address is 64B-aligned, and the DLB only wants bits [63:6] */
+	r0.field.addr_l = cq_dma_base >> 6;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ_ADDR_L(port->id.phys_id), r0.val);
+
+	r1.field.addr_u = cq_dma_base >> 32;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ_ADDR_U(port->id.phys_id), r1.val);
+
+	r2.field.vf = vf_id;
+	r2.field.is_pf = !vf_request;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ2VF_PF(port->id.phys_id), r2.val);
+
+	if (args->cq_depth == 8) {
+		r3.field.token_depth_select = 1;
+	} else if (args->cq_depth == 16) {
+		r3.field.token_depth_select = 2;
+	} else if (args->cq_depth == 32) {
+		r3.field.token_depth_select = 3;
+	} else if (args->cq_depth == 64) {
+		r3.field.token_depth_select = 4;
+	} else if (args->cq_depth == 128) {
+		r3.field.token_depth_select = 5;
+	} else if (args->cq_depth == 256) {
+		r3.field.token_depth_select = 6;
+	} else if (args->cq_depth == 512) {
+		r3.field.token_depth_select = 7;
+	} else if (args->cq_depth == 1024) {
+		r3.field.token_depth_select = 8;
+	} else {
+		DLB_HW_ERR(hw, "[%s():%d] Internal error: invalid CQ depth\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(port->id.phys_id),
+		   r3.val);
+
+	r4.field.token_depth_select = r3.field.token_depth_select;
+	r4.field.disable_wb_opt = 0;
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(port->id.phys_id),
+		   r4.val);
+
+	/* Two cache lines (128B) are dedicated for the port's pop counts */
+	r5.field.addr_l = pop_count_dma_base >> 7;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_PP_ADDR_L(port->id.phys_id), r5.val);
+
+	r6.field.addr_u = pop_count_dma_base >> 32;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_PP_ADDR_U(port->id.phys_id), r6.val);
+
+	return 0;
+}
+
+static int dlb_configure_dir_port(struct dlb_hw *hw,
+				  struct dlb_domain *domain,
+				  struct dlb_dir_pq_pair *port,
+				  u64 pop_count_dma_base,
+				  u64 cq_dma_base,
+				  struct dlb_create_dir_port_args *args,
+				  bool vf_request,
+				  unsigned int vf_id)
+{
+	struct dlb_credit_pool *ldb_pool, *dir_pool;
+	int ret;
+
+	port->ldb_pool_used = !dlb_list_empty(&domain->used_ldb_queues) ||
+			      !dlb_list_empty(&domain->avail_ldb_queues);
+
+	/* Each directed port has a directed queue, hence this port requires
+	 * directed credits.
+	 */
+	port->dir_pool_used = true;
+
+	if (port->ldb_pool_used) {
+		u32 cnt = args->ldb_credit_high_watermark;
+
+		ldb_pool = dlb_get_domain_ldb_pool(args->ldb_credit_pool_id,
+						   vf_request,
+						   domain);
+		if (!ldb_pool) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: port validation failed\n",
+				   __func__);
+			return -EFAULT;
+		}
+
+		dlb_ldb_pool_update_credit_count(hw, ldb_pool->id.phys_id, cnt);
+	} else {
+		args->ldb_credit_high_watermark = 0;
+		args->ldb_credit_low_watermark = 0;
+		args->ldb_credit_quantum = 0;
+	}
+
+	dir_pool = dlb_get_domain_dir_pool(args->dir_credit_pool_id,
+					   vf_request,
+					   domain);
+	if (!dir_pool) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: port validation failed\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	dlb_dir_pool_update_credit_count(hw,
+					 dir_pool->id.phys_id,
+					 args->dir_credit_high_watermark);
+
+	ret = dlb_dir_port_configure_cq(hw,
+					port,
+					pop_count_dma_base,
+					cq_dma_base,
+					args,
+					vf_request,
+					vf_id);
+
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_dir_port_configure_pp(hw,
+					domain,
+					port,
+					args,
+					vf_request,
+					vf_id);
+	if (ret < 0)
+		return ret;
+
+	dlb_dir_port_cq_enable(hw, port);
+
+	port->enabled = true;
+
+	port->port_configured = true;
+
+	return 0;
+}
+
+static int dlb_ldb_port_map_qid_static(struct dlb_hw *hw,
+				       struct dlb_ldb_port *p,
+				       struct dlb_ldb_queue *q,
+				       u8 priority)
+{
+	union dlb_lsp_cq2priov r0;
+	union dlb_lsp_cq2qid r1;
+	union dlb_atm_pipe_qid_ldb_qid2cqidx r2;
+	union dlb_lsp_qid_ldb_qid2cqidx r3;
+	union dlb_lsp_qid_ldb_qid2cqidx2 r4;
+	enum dlb_qid_map_state state;
+	int i;
+
+	/* Look for a pending or already mapped slot, else an unused slot */
+	if (!dlb_port_find_slot_queue(p, DLB_QUEUE_MAP_IN_PROGRESS, q, &i) &&
+	    !dlb_port_find_slot_queue(p, DLB_QUEUE_MAPPED, q, &i) &&
+	    !dlb_port_find_slot(p, DLB_QUEUE_UNMAPPED, &i)) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: CQ has no available QID mapping slots\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* Read-modify-write the priority and valid bit register */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(p->id.phys_id));
+
+	r0.field.v |= 1 << i;
+	r0.field.prio |= (priority & 0x7) << i * 3;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(p->id.phys_id), r0.val);
+
+	/* Read-modify-write the QID map register */
+	r1.val = DLB_CSR_RD(hw, DLB_LSP_CQ2QID(p->id.phys_id, i / 4));
+
+	if (i == 0 || i == 4)
+		r1.field.qid_p0 = q->id.phys_id;
+	if (i == 1 || i == 5)
+		r1.field.qid_p1 = q->id.phys_id;
+	if (i == 2 || i == 6)
+		r1.field.qid_p2 = q->id.phys_id;
+	if (i == 3 || i == 7)
+		r1.field.qid_p3 = q->id.phys_id;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2QID(p->id.phys_id, i / 4), r1.val);
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_ATM_PIPE_QID_LDB_QID2CQIDX(q->id.phys_id,
+							   p->id.phys_id / 4));
+
+	r3.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX(q->id.phys_id,
+						      p->id.phys_id / 4));
+
+	r4.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX2(q->id.phys_id,
+						       p->id.phys_id / 4));
+
+	switch (p->id.phys_id % 4) {
+	case 0:
+		r2.field.cq_p0 |= 1 << i;
+		r3.field.cq_p0 |= 1 << i;
+		r4.field.cq_p0 |= 1 << i;
+		break;
+
+	case 1:
+		r2.field.cq_p1 |= 1 << i;
+		r3.field.cq_p1 |= 1 << i;
+		r4.field.cq_p1 |= 1 << i;
+		break;
+
+	case 2:
+		r2.field.cq_p2 |= 1 << i;
+		r3.field.cq_p2 |= 1 << i;
+		r4.field.cq_p2 |= 1 << i;
+		break;
+
+	case 3:
+		r2.field.cq_p3 |= 1 << i;
+		r3.field.cq_p3 |= 1 << i;
+		r4.field.cq_p3 |= 1 << i;
+		break;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_ATM_PIPE_QID_LDB_QID2CQIDX(q->id.phys_id,
+						  p->id.phys_id / 4),
+		   r2.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX(q->id.phys_id,
+					     p->id.phys_id / 4),
+		   r3.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX2(q->id.phys_id,
+					      p->id.phys_id / 4),
+		   r4.val);
+
+	dlb_flush_csr(hw);
+
+	p->qid_map[i].qid = q->id.phys_id;
+	p->qid_map[i].priority = priority;
+
+	state = DLB_QUEUE_MAPPED;
+
+	return dlb_port_slot_state_transition(hw, p, q, i, state);
+}
+
+static void dlb_ldb_port_change_qid_priority(struct dlb_hw *hw,
+					     struct dlb_ldb_port *port,
+					     int slot,
+					     struct dlb_map_qid_args *args)
+{
+	union dlb_lsp_cq2priov r0;
+
+	/* Read-modify-write the priority and valid bit register */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(port->id.phys_id));
+
+	r0.field.v |= 1 << slot;
+	r0.field.prio |= (args->priority & 0x7) << slot * 3;
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port->id.phys_id), r0.val);
+
+	dlb_flush_csr(hw);
+
+	port->qid_map[slot].priority = args->priority;
+}
+
+static int dlb_ldb_port_set_has_work_bits(struct dlb_hw *hw,
+					  struct dlb_ldb_port *port,
+					  struct dlb_ldb_queue *queue,
+					  int slot)
+{
+	union dlb_lsp_qid_aqed_active_cnt r0;
+	union dlb_lsp_qid_ldb_enqueue_cnt r1;
+	union dlb_lsp_ldb_sched_ctrl r2 = { {0} };
+
+	/* Set the atomic scheduling haswork bit */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id.phys_id));
+
+	r2.field.cq = port->id.phys_id;
+	r2.field.qidix = slot;
+	r2.field.value = 1;
+	r2.field.rlist_haswork_v = r0.field.count > 0;
+
+	/* Set the non-atomic scheduling haswork bit */
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	r1.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id.phys_id));
+
+	memset(&r2, 0, sizeof(r2));
+
+	r2.field.cq = port->id.phys_id;
+	r2.field.qidix = slot;
+	r2.field.value = 1;
+	r2.field.nalb_haswork_v = (r1.field.count > 0);
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	dlb_flush_csr(hw);
+
+	return 0;
+}
+
+static void dlb_ldb_port_clear_has_work_bits(struct dlb_hw *hw,
+					     struct dlb_ldb_port *port,
+					     u8 slot)
+{
+	union dlb_lsp_ldb_sched_ctrl r2 = { {0} };
+
+	r2.field.cq = port->id.phys_id;
+	r2.field.qidix = slot;
+	r2.field.value = 0;
+	r2.field.rlist_haswork_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	memset(&r2, 0, sizeof(r2));
+
+	r2.field.cq = port->id.phys_id;
+	r2.field.qidix = slot;
+	r2.field.value = 0;
+	r2.field.nalb_haswork_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r2.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_ldb_port_clear_queue_if_status(struct dlb_hw *hw,
+					       struct dlb_ldb_port *port,
+					       int slot)
+{
+	union dlb_lsp_ldb_sched_ctrl r0 = { {0} };
+
+	r0.field.cq = port->id.phys_id;
+	r0.field.qidix = slot;
+	r0.field.value = 0;
+	r0.field.inflight_ok_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r0.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_ldb_port_set_queue_if_status(struct dlb_hw *hw,
+					     struct dlb_ldb_port *port,
+					     int slot)
+{
+	union dlb_lsp_ldb_sched_ctrl r0 = { {0} };
+
+	r0.field.cq = port->id.phys_id;
+	r0.field.qidix = slot;
+	r0.field.value = 1;
+	r0.field.inflight_ok_v = 1;
+
+	DLB_CSR_WR(hw, DLB_LSP_LDB_SCHED_CTRL, r0.val);
+
+	dlb_flush_csr(hw);
+}
+
+static void dlb_ldb_queue_set_inflight_limit(struct dlb_hw *hw,
+					     struct dlb_ldb_queue *queue)
+{
+	union dlb_lsp_qid_ldb_infl_lim r0 = { {0} };
+
+	r0.field.limit = queue->num_qid_inflights;
+
+	DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id.phys_id), r0.val);
+}
+
+static void dlb_ldb_queue_clear_inflight_limit(struct dlb_hw *hw,
+					       struct dlb_ldb_queue *queue)
+{
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_INFL_LIM(queue->id.phys_id),
+		   DLB_LSP_QID_LDB_INFL_LIM_RST);
+}
+
+/* dlb_ldb_queue_{enable, disable}_mapped_cqs() don't operate exactly as their
+ * function names imply, and should only be called by the dynamic CQ mapping
+ * code.
+ */
+static void dlb_ldb_queue_disable_mapped_cqs(struct dlb_hw *hw,
+					     struct dlb_domain *domain,
+					     struct dlb_ldb_queue *queue)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_ldb_port *port;
+	int slot;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		enum dlb_qid_map_state state = DLB_QUEUE_MAPPED;
+
+		if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+			continue;
+
+		if (port->enabled)
+			dlb_ldb_port_cq_disable(hw, port);
+	}
+}
+
+static void dlb_ldb_queue_enable_mapped_cqs(struct dlb_hw *hw,
+					    struct dlb_domain *domain,
+					    struct dlb_ldb_queue *queue)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_ldb_port *port;
+	int slot;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		enum dlb_qid_map_state state = DLB_QUEUE_MAPPED;
+
+		if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+			continue;
+
+		if (port->enabled)
+			dlb_ldb_port_cq_enable(hw, port);
+	}
+}
+
+static int dlb_ldb_port_finish_map_qid_dynamic(struct dlb_hw *hw,
+					       struct dlb_domain *domain,
+					       struct dlb_ldb_port *port,
+					       struct dlb_ldb_queue *queue)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	union dlb_lsp_qid_ldb_infl_cnt r0;
+	enum dlb_qid_map_state state;
+	int slot, ret;
+	u8 prio;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id.phys_id));
+
+	if (r0.field.count) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: non-zero QID inflight count\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	/* For each port with a pending mapping to this queue, perform the
+	 * static mapping and set the corresponding has_work bits.
+	 */
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+		return -EINVAL;
+
+	if (slot >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	prio = port->qid_map[slot].priority;
+
+	/* Update the CQ2QID, CQ2PRIOV, and QID2CQIDX registers, and
+	 * the port's qid_map state.
+	 */
+	ret = dlb_ldb_port_map_qid_static(hw, port, queue, prio);
+	if (ret)
+		return ret;
+
+	ret = dlb_ldb_port_set_has_work_bits(hw, port, queue, slot);
+	if (ret)
+		return ret;
+
+	/* Ensure IF_status(cq,qid) is 0 before enabling the port to
+	 * prevent spurious schedules to cause the queue's inflight
+	 * count to increase.
+	 */
+	dlb_ldb_port_clear_queue_if_status(hw, port, slot);
+
+	/* Reset the queue's inflight status */
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		state = DLB_QUEUE_MAPPED;
+		if (!dlb_port_find_slot_queue(port, state, queue, &slot))
+			continue;
+
+		dlb_ldb_port_set_queue_if_status(hw, port, slot);
+	}
+
+	dlb_ldb_queue_set_inflight_limit(hw, queue);
+
+	/* Re-enable CQs mapped to this queue */
+	dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
+
+	/* If this queue has other mappings pending, clear its inflight limit */
+	if (queue->num_pending_additions > 0)
+		dlb_ldb_queue_clear_inflight_limit(hw, queue);
+
+	return 0;
+}
+
+/**
+ * dlb_ldb_port_map_qid_dynamic() - perform a "dynamic" QID->CQ mapping
+ * @hw: dlb_hw handle for a particular device.
+ * @port: load-balanced port
+ * @queue: load-balanced queue
+ * @priority: queue servicing priority
+ *
+ * Returns 0 if the queue was mapped, 1 if the mapping is scheduled to occur
+ * at a later point, and <0 if an error occurred.
+ */
+static int dlb_ldb_port_map_qid_dynamic(struct dlb_hw *hw,
+					struct dlb_ldb_port *port,
+					struct dlb_ldb_queue *queue,
+					u8 priority)
+{
+	union dlb_lsp_qid_ldb_infl_cnt r0 = { {0} };
+	enum dlb_qid_map_state state;
+	struct dlb_domain *domain;
+	int slot, ret;
+
+	domain = dlb_get_domain_from_id(hw, port->domain_id.phys_id, false, 0);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: unable to find domain %d\n",
+			   __func__, port->domain_id.phys_id);
+		return -EFAULT;
+	}
+
+	/* Set the QID inflight limit to 0 to prevent further scheduling of the
+	 * queue.
+	 */
+	DLB_CSR_WR(hw, DLB_LSP_QID_LDB_INFL_LIM(queue->id.phys_id), 0);
+
+	if (!dlb_port_find_slot(port, DLB_QUEUE_UNMAPPED, &slot)) {
+		DLB_HW_ERR(hw,
+			   "Internal error: No available unmapped slots\n");
+		return -EFAULT;
+	}
+
+	if (slot >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	port->qid_map[slot].qid = queue->id.phys_id;
+	port->qid_map[slot].priority = priority;
+
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	ret = dlb_port_slot_state_transition(hw, port, queue, slot, state);
+	if (ret)
+		return ret;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id.phys_id));
+
+	if (r0.field.count) {
+		/* The queue is owed completions so it's not safe to map it
+		 * yet. Schedule a kernel thread to complete the mapping later,
+		 * once software has completed all the queue's inflight events.
+		 */
+		if (!os_worker_active(hw))
+			os_schedule_work(hw);
+
+		return 1;
+	}
+
+	/* Disable the affected CQ, and the CQs already mapped to the QID,
+	 * before reading the QID's inflight count a second time. There is an
+	 * unlikely race in which the QID may schedule one more QE after we
+	 * read an inflight count of 0, and disabling the CQs guarantees that
+	 * the race will not occur after a re-read of the inflight count
+	 * register.
+	 */
+	if (port->enabled)
+		dlb_ldb_port_cq_disable(hw, port);
+
+	dlb_ldb_queue_disable_mapped_cqs(hw, domain, queue);
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(queue->id.phys_id));
+
+	if (r0.field.count) {
+		if (port->enabled)
+			dlb_ldb_port_cq_enable(hw, port);
+
+		dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
+
+		/* The queue is owed completions so it's not safe to map it
+		 * yet. Schedule a kernel thread to complete the mapping later,
+		 * once software has completed all the queue's inflight events.
+		 */
+		if (!os_worker_active(hw))
+			os_schedule_work(hw);
+
+		return 1;
+	}
+
+	return dlb_ldb_port_finish_map_qid_dynamic(hw, domain, port, queue);
+}
+
+static int dlb_ldb_port_map_qid(struct dlb_hw *hw,
+				struct dlb_domain *domain,
+				struct dlb_ldb_port *port,
+				struct dlb_ldb_queue *queue,
+				u8 prio)
+{
+	if (domain->started)
+		return dlb_ldb_port_map_qid_dynamic(hw, port, queue, prio);
+	else
+		return dlb_ldb_port_map_qid_static(hw, port, queue, prio);
+}
+
+static int dlb_ldb_port_unmap_qid(struct dlb_hw *hw,
+				  struct dlb_ldb_port *port,
+				  struct dlb_ldb_queue *queue)
+{
+	enum dlb_qid_map_state mapped, in_progress, pending_map, unmapped;
+	union dlb_lsp_cq2priov r0;
+	union dlb_atm_pipe_qid_ldb_qid2cqidx r1;
+	union dlb_lsp_qid_ldb_qid2cqidx r2;
+	union dlb_lsp_qid_ldb_qid2cqidx2 r3;
+	u32 queue_id;
+	u32 port_id;
+	int i;
+
+	/* Find the queue's slot */
+	mapped = DLB_QUEUE_MAPPED;
+	in_progress = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	pending_map = DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP;
+
+	if (!dlb_port_find_slot_queue(port, mapped, queue, &i) &&
+	    !dlb_port_find_slot_queue(port, in_progress, queue, &i) &&
+	    !dlb_port_find_slot_queue(port, pending_map, queue, &i)) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: QID %d isn't mapped\n",
+			   __func__, __LINE__, queue->id.phys_id);
+		return -EFAULT;
+	}
+
+	if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	port_id = port->id.phys_id;
+	queue_id = queue->id.phys_id;
+
+	/* Read-modify-write the priority and valid bit register */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ2PRIOV(port_id));
+
+	r0.field.v &= ~(1 << i);
+
+	DLB_CSR_WR(hw, DLB_LSP_CQ2PRIOV(port_id), r0.val);
+
+	r1.val = DLB_CSR_RD(hw,
+			    DLB_ATM_PIPE_QID_LDB_QID2CQIDX(queue_id,
+							   port_id / 4));
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX(queue_id,
+						      port_id / 4));
+
+	r3.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_QID2CQIDX2(queue_id,
+						       port_id / 4));
+
+	switch (port_id % 4) {
+	case 0:
+		r1.field.cq_p0 &= ~(1 << i);
+		r2.field.cq_p0 &= ~(1 << i);
+		r3.field.cq_p0 &= ~(1 << i);
+		break;
+
+	case 1:
+		r1.field.cq_p1 &= ~(1 << i);
+		r2.field.cq_p1 &= ~(1 << i);
+		r3.field.cq_p1 &= ~(1 << i);
+		break;
+
+	case 2:
+		r1.field.cq_p2 &= ~(1 << i);
+		r2.field.cq_p2 &= ~(1 << i);
+		r3.field.cq_p2 &= ~(1 << i);
+		break;
+
+	case 3:
+		r1.field.cq_p3 &= ~(1 << i);
+		r2.field.cq_p3 &= ~(1 << i);
+		r3.field.cq_p3 &= ~(1 << i);
+		break;
+	}
+
+	DLB_CSR_WR(hw,
+		   DLB_ATM_PIPE_QID_LDB_QID2CQIDX(queue_id, port_id / 4),
+		   r1.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX(queue_id, port_id / 4),
+		   r2.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_QID_LDB_QID2CQIDX2(queue_id, port_id / 4),
+		   r3.val);
+
+	dlb_flush_csr(hw);
+
+	unmapped = DLB_QUEUE_UNMAPPED;
+
+	return dlb_port_slot_state_transition(hw, port, queue, i, unmapped);
+}
+
+static void
+dlb_log_create_sched_domain_args(struct dlb_hw *hw,
+				 struct dlb_create_sched_domain_args *args,
+				 bool vf_request,
+				 unsigned int vf_id)
+{
+	DLB_HW_INFO(hw, "DLB create sched domain arguments:\n");
+	if (vf_request)
+		DLB_HW_INFO(hw, "(Request from VF %d)\n", vf_id);
+	DLB_HW_INFO(hw, "\tNumber of LDB queues:        %d\n",
+		    args->num_ldb_queues);
+	DLB_HW_INFO(hw, "\tNumber of LDB ports:         %d\n",
+		    args->num_ldb_ports);
+	DLB_HW_INFO(hw, "\tNumber of DIR ports:         %d\n",
+		    args->num_dir_ports);
+	DLB_HW_INFO(hw, "\tNumber of ATM inflights:     %d\n",
+		    args->num_atomic_inflights);
+	DLB_HW_INFO(hw, "\tNumber of hist list entries: %d\n",
+		    args->num_hist_list_entries);
+	DLB_HW_INFO(hw, "\tNumber of LDB credits:       %d\n",
+		    args->num_ldb_credits);
+	DLB_HW_INFO(hw, "\tNumber of DIR credits:       %d\n",
+		    args->num_dir_credits);
+	DLB_HW_INFO(hw, "\tNumber of LDB credit pools:  %d\n",
+		    args->num_ldb_credit_pools);
+	DLB_HW_INFO(hw, "\tNumber of DIR credit pools:  %d\n",
+		    args->num_dir_credit_pools);
+}
+
+/**
+ * dlb_hw_create_sched_domain() - Allocate and initialize a DLB scheduling
+ *	domain and its resources.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ * @vf_request: Request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_sched_domain(struct dlb_hw *hw,
+			       struct dlb_create_sched_domain_args *args,
+			       struct dlb_cmd_response *resp,
+			       bool vf_request,
+			       unsigned int vf_id)
+{
+	struct dlb_domain *domain;
+	struct dlb_function_resources *rsrcs;
+	int ret;
+
+	rsrcs = (vf_request) ? &hw->vf[vf_id] : &hw->pf;
+
+	dlb_log_create_sched_domain_args(hw, args, vf_request, vf_id);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_sched_domain_args(hw, rsrcs, args, resp))
+		return -EINVAL;
+
+	domain = DLB_FUNC_LIST_HEAD(rsrcs->avail_domains, typeof(*domain));
+
+	/* Verification should catch this. */
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available domains\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (domain->configured) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: avail_domains contains configured domains.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	dlb_init_domain_rsrc_lists(domain);
+
+	/* Verification should catch this too. */
+	ret = dlb_domain_attach_resources(hw, rsrcs, domain, args, resp);
+	if (ret < 0) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to verify args.\n",
+			   __func__);
+
+		return -EFAULT;
+	}
+
+	dlb_list_del(&rsrcs->avail_domains, &domain->func_list);
+
+	dlb_list_add(&rsrcs->used_domains, &domain->func_list);
+
+	resp->id = (vf_request) ? domain->id.virt_id : domain->id.phys_id;
+	resp->status = 0;
+
+	return 0;
+}
+
+static void
+dlb_log_create_ldb_pool_args(struct dlb_hw *hw,
+			     u32 domain_id,
+			     struct dlb_create_ldb_pool_args *args,
+			     bool vf_request,
+			     unsigned int vf_id)
+{
+	DLB_HW_INFO(hw, "DLB create load-balanced credit pool arguments:\n");
+	if (vf_request)
+		DLB_HW_INFO(hw, "(Request from VF %d)\n", vf_id);
+	DLB_HW_INFO(hw, "\tDomain ID:             %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tNumber of LDB credits: %d\n",
+		    args->num_ldb_credits);
+}
+
+/**
+ * dlb_hw_create_ldb_pool() - Allocate and initialize a DLB credit pool.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_ldb_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_pool_args *args,
+			   struct dlb_cmd_response *resp,
+			   bool vf_request,
+			   unsigned int vf_id)
+{
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+
+	dlb_log_create_ldb_pool_args(hw, domain_id, args, vf_request, vf_id);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_ldb_pool_args(hw,
+					    domain_id,
+					    args,
+					    resp,
+					    vf_request,
+					    vf_id))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	pool = DLB_DOM_LIST_HEAD(domain->avail_ldb_credit_pools, typeof(*pool));
+
+	/* Verification should catch this. */
+	if (!pool) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available ldb credit pools\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	dlb_configure_ldb_credit_pool(hw, domain, args, pool);
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_ldb_credit_pools, &pool->domain_list);
+
+	dlb_list_add(&domain->used_ldb_credit_pools, &pool->domain_list);
+
+	resp->status = 0;
+	resp->id = (vf_request) ? pool->id.virt_id : pool->id.phys_id;
+
+	return 0;
+}
+
+static void
+dlb_log_create_dir_pool_args(struct dlb_hw *hw,
+			     u32 domain_id,
+			     struct dlb_create_dir_pool_args *args,
+			     bool vf_request,
+			     unsigned int vf_id)
+{
+	DLB_HW_INFO(hw, "DLB create directed credit pool arguments:\n");
+	if (vf_request)
+		DLB_HW_INFO(hw, "(Request from VF %d)\n", vf_id);
+	DLB_HW_INFO(hw, "\tDomain ID:             %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tNumber of DIR credits: %d\n",
+		    args->num_dir_credits);
+}
+
+/**
+ * dlb_hw_create_dir_pool() - Allocate and initialize a DLB credit pool.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_dir_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_pool_args *args,
+			   struct dlb_cmd_response *resp,
+			   bool vf_request,
+			   unsigned int vf_id)
+{
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+
+	dlb_log_create_dir_pool_args(hw, domain_id, args, vf_request, vf_id);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	/* At least one available pool */
+	if (dlb_verify_create_dir_pool_args(hw,
+					    domain_id,
+					    args,
+					    resp,
+					    vf_request,
+					    vf_id))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	pool = DLB_DOM_LIST_HEAD(domain->avail_dir_credit_pools, typeof(*pool));
+
+	/* Verification should catch this. */
+	if (!pool) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available dir credit pools\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	dlb_configure_dir_credit_pool(hw, domain, args, pool);
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_dir_credit_pools, &pool->domain_list);
+
+	dlb_list_add(&domain->used_dir_credit_pools, &pool->domain_list);
+
+	resp->status = 0;
+	resp->id = (vf_request) ? pool->id.virt_id : pool->id.phys_id;
+
+	return 0;
+}
+
+static void
+dlb_log_create_ldb_queue_args(struct dlb_hw *hw,
+			      u32 domain_id,
+			      struct dlb_create_ldb_queue_args *args,
+			      bool vf_request,
+			      unsigned int vf_id)
+{
+	DLB_HW_INFO(hw, "DLB create load-balanced queue arguments:\n");
+	if (vf_request)
+		DLB_HW_INFO(hw, "(Request from VF %d)\n", vf_id);
+	DLB_HW_INFO(hw, "\tDomain ID:                  %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tNumber of sequence numbers: %d\n",
+		    args->num_sequence_numbers);
+	DLB_HW_INFO(hw, "\tNumber of QID inflights:    %d\n",
+		    args->num_qid_inflights);
+	DLB_HW_INFO(hw, "\tNumber of ATM inflights:    %d\n",
+		    args->num_atomic_inflights);
+}
+
+/**
+ * dlb_hw_create_ldb_queue() - Allocate and initialize a DLB LDB queue.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_ldb_queue_args *args,
+			    struct dlb_cmd_response *resp,
+			    bool vf_request,
+			    unsigned int vf_id)
+{
+	struct dlb_ldb_queue *queue;
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_create_ldb_queue_args(hw, domain_id, args, vf_request, vf_id);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	/* At least one available queue */
+	if (dlb_verify_create_ldb_queue_args(hw,
+					     domain_id,
+					     args,
+					     resp,
+					     vf_request,
+					     vf_id))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue = DLB_DOM_LIST_HEAD(domain->avail_ldb_queues, typeof(*queue));
+
+	/* Verification should catch this. */
+	if (!queue) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available ldb queues\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	ret = dlb_ldb_queue_attach_resources(hw, domain, queue, args);
+	if (ret < 0) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: failed to attach the ldb queue resources\n",
+			   __func__, __LINE__);
+		return ret;
+	}
+
+	dlb_configure_ldb_queue(hw, domain, queue, args, vf_request, vf_id);
+
+	queue->num_mappings = 0;
+
+	queue->configured = true;
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_ldb_queues, &queue->domain_list);
+
+	dlb_list_add(&domain->used_ldb_queues, &queue->domain_list);
+
+	resp->status = 0;
+	resp->id = (vf_request) ? queue->id.virt_id : queue->id.phys_id;
+
+	return 0;
+}
+
+static void
+dlb_log_create_dir_queue_args(struct dlb_hw *hw,
+			      u32 domain_id,
+			      struct dlb_create_dir_queue_args *args,
+			      bool vf_request,
+			      unsigned int vf_id)
+{
+	DLB_HW_INFO(hw, "DLB create directed queue arguments:\n");
+	if (vf_request)
+		DLB_HW_INFO(hw, "(Request from VF %d)\n", vf_id);
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n", args->port_id);
+}
+
+/**
+ * dlb_hw_create_dir_queue() - Allocate and initialize a DLB DIR queue.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_dir_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_dir_queue_args *args,
+			    struct dlb_cmd_response *resp,
+			    bool vf_request,
+			    unsigned int vf_id)
+{
+	struct dlb_dir_pq_pair *queue;
+	struct dlb_domain *domain;
+
+	dlb_log_create_dir_queue_args(hw, domain_id, args, vf_request, vf_id);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_dir_queue_args(hw,
+					     domain_id,
+					     args,
+					     resp,
+					     vf_request,
+					     vf_id))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (args->port_id != -1)
+		queue = dlb_get_domain_used_dir_pq(args->port_id,
+						   vf_request,
+						   domain);
+	else
+		queue = DLB_DOM_LIST_HEAD(domain->avail_dir_pq_pairs,
+					  typeof(*queue));
+
+	/* Verification should catch this. */
+	if (!queue) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available dir queues\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	dlb_configure_dir_queue(hw, domain, queue, vf_request, vf_id);
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list (if it's not already there).
+	 */
+	if (args->port_id == -1) {
+		dlb_list_del(&domain->avail_dir_pq_pairs, &queue->domain_list);
+
+		dlb_list_add(&domain->used_dir_pq_pairs, &queue->domain_list);
+	}
+
+	resp->status = 0;
+
+	resp->id = (vf_request) ? queue->id.virt_id : queue->id.phys_id;
+
+	return 0;
+}
+
+static void dlb_log_create_ldb_port_args(struct dlb_hw *hw,
+					 u32 domain_id,
+					 u64 pop_count_dma_base,
+					 u64 cq_dma_base,
+					 struct dlb_create_ldb_port_args *args,
+					 bool vf_request,
+					 unsigned int vf_id)
+{
+	DLB_HW_INFO(hw, "DLB create load-balanced port arguments:\n");
+	if (vf_request)
+		DLB_HW_INFO(hw, "(Request from VF %d)\n", vf_id);
+	DLB_HW_INFO(hw, "\tDomain ID:                 %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tLDB credit pool ID:        %d\n",
+		    args->ldb_credit_pool_id);
+	DLB_HW_INFO(hw, "\tLDB credit high watermark: %d\n",
+		    args->ldb_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit low watermark:  %d\n",
+		    args->ldb_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit quantum:        %d\n",
+		    args->ldb_credit_quantum);
+	DLB_HW_INFO(hw, "\tDIR credit pool ID:        %d\n",
+		    args->dir_credit_pool_id);
+	DLB_HW_INFO(hw, "\tDIR credit high watermark: %d\n",
+		    args->dir_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit low watermark:  %d\n",
+		    args->dir_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit quantum:        %d\n",
+		    args->dir_credit_quantum);
+	DLB_HW_INFO(hw, "\tpop_count_address:         0x%"PRIx64"\n",
+		    pop_count_dma_base);
+	DLB_HW_INFO(hw, "\tCQ depth:                  %d\n",
+		    args->cq_depth);
+	DLB_HW_INFO(hw, "\tCQ hist list size:         %d\n",
+		    args->cq_history_list_size);
+	DLB_HW_INFO(hw, "\tCQ base address:           0x%"PRIx64"\n",
+		    cq_dma_base);
+}
+
+/**
+ * dlb_hw_create_ldb_port() - Allocate and initialize a load-balanced port and
+ *	its resources.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp,
+			   bool vf_request,
+			   unsigned int vf_id)
+{
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_create_ldb_port_args(hw,
+				     domain_id,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args,
+				     vf_request,
+				     vf_id);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_ldb_port_args(hw,
+					    domain_id,
+					    pop_count_dma_base,
+					    cq_dma_base,
+					    args,
+					    resp,
+					    vf_request,
+					    vf_id))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	port = DLB_DOM_LIST_HEAD(domain->avail_ldb_ports, typeof(*port));
+
+	/* Verification should catch this. */
+	if (!port) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available ldb ports\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (port->configured) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: avail_ldb_ports contains configured ports.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	ret = dlb_configure_ldb_port(hw,
+				     domain,
+				     port,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args,
+				     vf_request,
+				     vf_id);
+	if (ret < 0)
+		return ret;
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list.
+	 */
+	dlb_list_del(&domain->avail_ldb_ports, &port->domain_list);
+
+	dlb_list_add(&domain->used_ldb_ports, &port->domain_list);
+
+	resp->status = 0;
+	resp->id = (vf_request) ? port->id.virt_id : port->id.phys_id;
+
+	return 0;
+}
+
+static void dlb_log_create_dir_port_args(struct dlb_hw *hw,
+					 u32 domain_id,
+					 u64 pop_count_dma_base,
+					 u64 cq_dma_base,
+					 struct dlb_create_dir_port_args *args,
+					 bool vf_request,
+					 unsigned int vf_id)
+{
+	DLB_HW_INFO(hw, "DLB create directed port arguments:\n");
+	if (vf_request)
+		DLB_HW_INFO(hw, "(Request from VF %d)\n", vf_id);
+	DLB_HW_INFO(hw, "\tDomain ID:                 %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tLDB credit pool ID:        %d\n",
+		    args->ldb_credit_pool_id);
+	DLB_HW_INFO(hw, "\tLDB credit high watermark: %d\n",
+		    args->ldb_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit low watermark:  %d\n",
+		    args->ldb_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tLDB credit quantum:        %d\n",
+		    args->ldb_credit_quantum);
+	DLB_HW_INFO(hw, "\tDIR credit pool ID:        %d\n",
+		    args->dir_credit_pool_id);
+	DLB_HW_INFO(hw, "\tDIR credit high watermark: %d\n",
+		    args->dir_credit_high_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit low watermark:  %d\n",
+		    args->dir_credit_low_watermark);
+	DLB_HW_INFO(hw, "\tDIR credit quantum:        %d\n",
+		    args->dir_credit_quantum);
+	DLB_HW_INFO(hw, "\tpop_count_address:         0x%"PRIx64"\n",
+		    pop_count_dma_base);
+	DLB_HW_INFO(hw, "\tCQ depth:                  %d\n",
+		    args->cq_depth);
+	DLB_HW_INFO(hw, "\tCQ base address:           0x%"PRIx64"\n",
+		    cq_dma_base);
+}
+
+/**
+ * dlb_hw_create_dir_port() - Allocate and initialize a DLB directed port and
+ *	queue. The port/queue pair have the same ID and name.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_create_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp,
+			   bool vf_request,
+			   unsigned int vf_id)
+{
+	struct dlb_dir_pq_pair *port;
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_create_dir_port_args(hw,
+				     domain_id,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args,
+				     vf_request,
+				     vf_id);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_create_dir_port_args(hw,
+					    domain_id,
+					    pop_count_dma_base,
+					    cq_dma_base,
+					    args,
+					    resp,
+					    vf_request,
+					    vf_id))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	if (args->queue_id != -1)
+		port = dlb_get_domain_used_dir_pq(args->queue_id,
+						  vf_request,
+						  domain);
+	else
+		port = DLB_DOM_LIST_HEAD(domain->avail_dir_pq_pairs,
+					 typeof(*port));
+
+	/* Verification should catch this. */
+	if (!port) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: no available dir ports\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	ret = dlb_configure_dir_port(hw,
+				     domain,
+				     port,
+				     pop_count_dma_base,
+				     cq_dma_base,
+				     args,
+				     vf_request,
+				     vf_id);
+	if (ret < 0)
+		return ret;
+
+	/* Configuration succeeded, so move the resource from the 'avail' to
+	 * the 'used' list (if it's not already there).
+	 */
+	if (args->queue_id == -1) {
+		dlb_list_del(&domain->avail_dir_pq_pairs, &port->domain_list);
+
+		dlb_list_add(&domain->used_dir_pq_pairs, &port->domain_list);
+	}
+
+	resp->status = 0;
+	resp->id = (vf_request) ? port->id.virt_id : port->id.phys_id;
+
+	return 0;
+}
+
+static void dlb_log_start_domain(struct dlb_hw *hw,
+				 u32 domain_id,
+				 bool vf_request,
+				 unsigned int vf_id)
+{
+	DLB_HW_INFO(hw, "DLB start domain arguments:\n");
+	if (vf_request)
+		DLB_HW_INFO(hw, "(Request from VF %d)\n", vf_id);
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+}
+
+/**
+ * dlb_hw_start_domain() - Lock the domain configuration
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb_hw_start_domain(struct dlb_hw *hw,
+			u32 domain_id,
+			__attribute((unused)) struct dlb_start_domain_args *arg,
+			struct dlb_cmd_response *resp,
+			bool vf_request,
+			unsigned int vf_id)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_dir_pq_pair *dir_queue;
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+
+	dlb_log_start_domain(hw, domain_id, vf_request, vf_id);
+
+	if (dlb_verify_start_domain_args(hw,
+					 domain_id,
+					 resp,
+					 vf_request,
+					 vf_id))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* Write the domain's pool credit counts, which have been updated
+	 * during port configuration. The sum of the pool credit count plus
+	 * each producer port's credit count must equal the pool's credit
+	 * allocation *before* traffic is sent.
+	 */
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
+		dlb_ldb_pool_write_credit_count_reg(hw, pool->id.phys_id);
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
+		dlb_dir_pool_write_credit_count_reg(hw, pool->id.phys_id);
+
+	/* Enable load-balanced and directed queue write permissions for the
+	 * queues this domain owns. Without this, the DLB will drop all
+	 * incoming traffic to those queues.
+	 */
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, ldb_queue, iter) {
+		union dlb_sys_ldb_vasqid_v r0 = { {0} };
+		unsigned int offs;
+
+		r0.field.vasqid_v = 1;
+
+		offs = domain->id.phys_id * DLB_MAX_NUM_LDB_QUEUES +
+			ldb_queue->id.phys_id;
+
+		DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(offs), r0.val);
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_queue, iter) {
+		union dlb_sys_dir_vasqid_v r0 = { {0} };
+		unsigned int offs;
+
+		r0.field.vasqid_v = 1;
+
+		offs = domain->id.phys_id * DLB_MAX_NUM_DIR_PORTS +
+			dir_queue->id.phys_id;
+
+		DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(offs), r0.val);
+	}
+
+	dlb_flush_csr(hw);
+
+	domain->started = true;
+
+	resp->status = 0;
+
+	return 0;
+}
+
+static void dlb_domain_finish_unmap_port_slot(struct dlb_hw *hw,
+					      struct dlb_domain *domain,
+					      struct dlb_ldb_port *port,
+					      int slot)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_ldb_queue *queue;
+
+	queue = &hw->rsrcs.ldb_queues[port->qid_map[slot].qid];
+
+	state = port->qid_map[slot].state;
+
+	/* Update the QID2CQIDX and CQ2QID vectors */
+	dlb_ldb_port_unmap_qid(hw, port, queue);
+
+	/* Ensure the QID will not be serviced by this {CQ, slot} by clearing
+	 * the has_work bits
+	 */
+	dlb_ldb_port_clear_has_work_bits(hw, port, slot);
+
+	/* Reset the {CQ, slot} to its default state */
+	dlb_ldb_port_set_queue_if_status(hw, port, slot);
+
+	/* Re-enable the CQ if it wasn't manually disabled by the user */
+	if (port->enabled)
+		dlb_ldb_port_cq_enable(hw, port);
+
+	/* If there is a mapping that is pending this slot's removal, perform
+	 * the mapping now.
+	 */
+	if (state == DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP) {
+		struct dlb_ldb_port_qid_map *map;
+		struct dlb_ldb_queue *map_queue;
+		u8 prio;
+
+		map = &port->qid_map[slot];
+
+		map->qid = map->pending_qid;
+		map->priority = map->pending_priority;
+
+		map_queue = &hw->rsrcs.ldb_queues[map->qid];
+		prio = map->priority;
+
+		dlb_ldb_port_map_qid(hw, domain, port, map_queue, prio);
+	}
+}
+
+static bool dlb_domain_finish_unmap_port(struct dlb_hw *hw,
+					 struct dlb_domain *domain,
+					 struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_infl_cnt r0;
+	int i;
+
+	if (port->num_pending_removals == 0)
+		return false;
+
+	/* The unmap requires all the CQ's outstanding inflights to be
+	 * completed.
+	 */
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_INFL_CNT(port->id.phys_id));
+	if (r0.field.count > 0)
+		return false;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		struct dlb_ldb_port_qid_map *map;
+
+		map = &port->qid_map[i];
+
+		if (map->state != DLB_QUEUE_UNMAP_IN_PROGRESS &&
+		    map->state != DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP)
+			continue;
+
+		dlb_domain_finish_unmap_port_slot(hw, domain, port, i);
+	}
+
+	return true;
+}
+
+static unsigned int
+dlb_domain_finish_unmap_qid_procedures(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_ldb_port *port;
+
+	if (!domain->configured || domain->num_pending_removals == 0)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		dlb_domain_finish_unmap_port(hw, domain, port);
+
+	return domain->num_pending_removals;
+}
+
+unsigned int dlb_finish_unmap_qid_procedures(struct dlb_hw *hw)
+{
+	int i, num = 0;
+
+	/* Finish queue unmap jobs for any domain that needs it */
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		struct dlb_domain *domain = &hw->domains[i];
+
+		num += dlb_domain_finish_unmap_qid_procedures(hw, domain);
+	}
+
+	return num;
+}
+
+static void dlb_domain_finish_map_port(struct dlb_hw *hw,
+				       struct dlb_domain *domain,
+				       struct dlb_ldb_port *port)
+{
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		union dlb_lsp_qid_ldb_infl_cnt r0;
+		struct dlb_ldb_queue *queue;
+		int qid;
+
+		if (port->qid_map[i].state != DLB_QUEUE_MAP_IN_PROGRESS)
+			continue;
+
+		qid = port->qid_map[i].qid;
+
+		queue = dlb_get_ldb_queue_from_id(hw, qid, false, 0);
+
+		if (!queue) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: unable to find queue %d\n",
+				   __func__, qid);
+			continue;
+		}
+
+		r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(qid));
+
+		if (r0.field.count)
+			continue;
+
+		/* Disable the affected CQ, and the CQs already mapped to the
+		 * QID, before reading the QID's inflight count a second time.
+		 * There is an unlikely race in which the QID may schedule one
+		 * more QE after we read an inflight count of 0, and disabling
+		 * the CQs guarantees that the race will not occur after a
+		 * re-read of the inflight count register.
+		 */
+		if (port->enabled)
+			dlb_ldb_port_cq_disable(hw, port);
+
+		dlb_ldb_queue_disable_mapped_cqs(hw, domain, queue);
+
+		r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_LDB_INFL_CNT(qid));
+
+		if (r0.field.count) {
+			if (port->enabled)
+				dlb_ldb_port_cq_enable(hw, port);
+
+			dlb_ldb_queue_enable_mapped_cqs(hw, domain, queue);
+
+			continue;
+		}
+
+		dlb_ldb_port_finish_map_qid_dynamic(hw, domain, port, queue);
+	}
+}
+
+static unsigned int
+dlb_domain_finish_map_qid_procedures(struct dlb_hw *hw,
+				     struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_ldb_port *port;
+
+	if (!domain->configured || domain->num_pending_additions == 0)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		dlb_domain_finish_map_port(hw, domain, port);
+
+	return domain->num_pending_additions;
+}
+
+unsigned int dlb_finish_map_qid_procedures(struct dlb_hw *hw)
+{
+	int i, num = 0;
+
+	/* Finish queue map jobs for any domain that needs it */
+	for (i = 0; i < DLB_MAX_NUM_DOMAINS; i++) {
+		struct dlb_domain *domain = &hw->domains[i];
+
+		num += dlb_domain_finish_map_qid_procedures(hw, domain);
+	}
+
+	return num;
+}
+
+static void dlb_log_map_qid(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_map_qid_args *args,
+			    bool vf_request,
+			    unsigned int vf_id)
+{
+	DLB_HW_INFO(hw, "DLB map QID arguments:\n");
+	if (vf_request)
+		DLB_HW_INFO(hw, "(Request from VF %d)\n", vf_id);
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n",
+		    args->port_id);
+	DLB_HW_INFO(hw, "\tQueue ID:  %d\n",
+		    args->qid);
+	DLB_HW_INFO(hw, "\tPriority:  %d\n",
+		    args->priority);
+}
+
+int dlb_hw_map_qid(struct dlb_hw *hw,
+		   u32 domain_id,
+		   struct dlb_map_qid_args *args,
+		   struct dlb_cmd_response *resp,
+		   bool vf_request,
+		   unsigned int vf_id)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_ldb_queue *queue;
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	int ret, i, id;
+	u8 prio;
+
+	dlb_log_map_qid(hw, domain_id, args, vf_request, vf_id);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_map_qid_args(hw,
+				    domain_id,
+				    args,
+				    resp,
+				    vf_request,
+				    vf_id))
+		return -EINVAL;
+
+	prio = args->priority;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, vf_request, domain);
+	if (!port) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, vf_request, domain);
+	if (!queue) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: queue not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* If there are any outstanding detach operations for this port,
+	 * attempt to complete them. This may be necessary to free up a QID
+	 * slot for this requested mapping.
+	 */
+	if (port->num_pending_removals)
+		dlb_domain_finish_unmap_port(hw, domain, port);
+
+	ret = dlb_verify_map_qid_slot_available(port, queue, resp);
+	if (ret)
+		return ret;
+
+	/* Hardware requires disabling the CQ before mapping QIDs. */
+	if (port->enabled)
+		dlb_ldb_port_cq_disable(hw, port);
+
+	/* If this is only a priority change, don't perform the full QID->CQ
+	 * mapping procedure
+	 */
+	state = DLB_QUEUE_MAPPED;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		if (prio != port->qid_map[i].priority) {
+			dlb_ldb_port_change_qid_priority(hw, port, i, args);
+			DLB_HW_INFO(hw, "DLB map: priority change only\n");
+		}
+
+		state = DLB_QUEUE_MAPPED;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto map_qid_done;
+	}
+
+	state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		if (prio != port->qid_map[i].priority) {
+			dlb_ldb_port_change_qid_priority(hw, port, i, args);
+			DLB_HW_INFO(hw, "DLB map: priority change only\n");
+		}
+
+		state = DLB_QUEUE_MAPPED;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto map_qid_done;
+	}
+
+	/* If this is a priority change on an in-progress mapping, don't
+	 * perform the full QID->CQ mapping procedure.
+	 */
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		port->qid_map[i].priority = prio;
+
+		DLB_HW_INFO(hw, "DLB map: priority change only\n");
+
+		goto map_qid_done;
+	}
+
+	/* If this is a priority change on a pending mapping, update the
+	 * pending priority
+	 */
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		port->qid_map[i].pending_priority = prio;
+
+		DLB_HW_INFO(hw, "DLB map: priority change only\n");
+
+		goto map_qid_done;
+	}
+
+	/* If all the CQ's slots are in use, then there's an unmap in progress
+	 * (guaranteed by dlb_verify_map_qid_slot_available()), so add this
+	 * mapping to pending_map and return. When the removal is completed for
+	 * the slot's current occupant, this mapping will be performed.
+	 */
+	if (!dlb_port_find_slot(port, DLB_QUEUE_UNMAPPED, &i)) {
+		if (dlb_port_find_slot(port, DLB_QUEUE_UNMAP_IN_PROGRESS, &i)) {
+			enum dlb_qid_map_state state;
+
+			if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+				DLB_HW_ERR(hw,
+					   "[%s():%d] Internal error: port slot tracking failed\n",
+					   __func__, __LINE__);
+				return -EFAULT;
+			}
+
+			port->qid_map[i].pending_qid = queue->id.phys_id;
+			port->qid_map[i].pending_priority = prio;
+
+			state = DLB_QUEUE_UNMAP_IN_PROGRESS_PENDING_MAP;
+
+			ret = dlb_port_slot_state_transition(hw, port, queue,
+							     i, state);
+			if (ret)
+				return ret;
+
+			DLB_HW_INFO(hw, "DLB map: map pending removal\n");
+
+			goto map_qid_done;
+		}
+	}
+
+	/* If the domain has started, a special "dynamic" CQ->queue mapping
+	 * procedure is required in order to safely update the CQ<->QID tables.
+	 * The "static" procedure cannot be used when traffic is flowing,
+	 * because the CQ<->QID tables cannot be updated atomically and the
+	 * scheduler won't see the new mapping unless the queue's if_status
+	 * changes, which isn't guaranteed.
+	 */
+	ret = dlb_ldb_port_map_qid(hw, domain, port, queue, prio);
+
+	/* If ret is less than zero, it's due to an internal error */
+	if (ret < 0)
+		return ret;
+
+map_qid_done:
+	if (port->enabled)
+		dlb_ldb_port_cq_enable(hw, port);
+
+	resp->status = 0;
+
+	return 0;
+}
+
+static void dlb_log_unmap_qid(struct dlb_hw *hw,
+			      u32 domain_id,
+			      struct dlb_unmap_qid_args *args,
+			      bool vf_request,
+			      unsigned int vf_id)
+{
+	DLB_HW_INFO(hw, "DLB unmap QID arguments:\n");
+	if (vf_request)
+		DLB_HW_INFO(hw, "(Request from VF %d)\n", vf_id);
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n",
+		    args->port_id);
+	DLB_HW_INFO(hw, "\tQueue ID:  %d\n",
+		    args->qid);
+	if (args->qid < DLB_MAX_NUM_LDB_QUEUES)
+		DLB_HW_INFO(hw, "\tQueue's num mappings:  %d\n",
+			    hw->rsrcs.ldb_queues[args->qid].num_mappings);
+}
+
+int dlb_hw_unmap_qid(struct dlb_hw *hw,
+		     u32 domain_id,
+		     struct dlb_unmap_qid_args *args,
+		     struct dlb_cmd_response *resp,
+		     bool vf_request,
+		     unsigned int vf_id)
+{
+	enum dlb_qid_map_state state;
+	struct dlb_ldb_queue *queue;
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	bool unmap_complete;
+	int i, ret, id;
+
+	dlb_log_unmap_qid(hw, domain_id, args, vf_request, vf_id);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_unmap_qid_args(hw,
+				      domain_id,
+				      args,
+				      resp,
+				      vf_request,
+				      vf_id))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, vf_request, domain);
+	if (!port) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->qid, vf_request, domain);
+	if (!queue) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: queue not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* If the queue hasn't been mapped yet, we need to update the slot's
+	 * state and re-enable the queue's inflights.
+	 */
+	state = DLB_QUEUE_MAP_IN_PROGRESS;
+	if (dlb_port_find_slot_queue(port, state, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		/* Since the in-progress map was aborted, re-enable the QID's
+		 * inflights.
+		 */
+		if (queue->num_pending_additions == 0)
+			dlb_ldb_queue_set_inflight_limit(hw, queue);
+
+		state = DLB_QUEUE_UNMAPPED;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto unmap_qid_done;
+	}
+
+	/* If the queue mapping is on hold pending an unmap, we simply need to
+	 * update the slot's state.
+	 */
+	if (dlb_port_find_slot_with_pending_map_queue(port, queue, &i)) {
+		if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+			DLB_HW_ERR(hw,
+				   "[%s():%d] Internal error: port slot tracking failed\n",
+				   __func__, __LINE__);
+			return -EFAULT;
+		}
+
+		state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+		ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+		if (ret)
+			return ret;
+
+		goto unmap_qid_done;
+	}
+
+	state = DLB_QUEUE_MAPPED;
+	if (!dlb_port_find_slot_queue(port, state, queue, &i)) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: no available CQ slots\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	if (i >= DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port slot tracking failed\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* QID->CQ mapping removal is an asychronous procedure. It requires
+	 * stopping the DLB from scheduling this CQ, draining all inflights
+	 * from the CQ, then unmapping the queue from the CQ. This function
+	 * simply marks the port as needing the queue unmapped, and (if
+	 * necessary) starts the unmapping worker thread.
+	 */
+	dlb_ldb_port_cq_disable(hw, port);
+
+	state = DLB_QUEUE_UNMAP_IN_PROGRESS;
+	ret = dlb_port_slot_state_transition(hw, port, queue, i, state);
+	if (ret)
+		return ret;
+
+	/* Attempt to finish the unmapping now, in case the port has no
+	 * outstanding inflights. If that's not the case, this will fail and
+	 * the unmapping will be completed at a later time.
+	 */
+	unmap_complete = dlb_domain_finish_unmap_port(hw, domain, port);
+
+	/* If the unmapping couldn't complete immediately, launch the worker
+	 * thread (if it isn't already launched) to finish it later.
+	 */
+	if (!unmap_complete && !os_worker_active(hw))
+		os_schedule_work(hw);
+
+unmap_qid_done:
+	resp->status = 0;
+
+	return 0;
+}
+
+static void dlb_log_enable_port(struct dlb_hw *hw,
+				u32 domain_id,
+				u32 port_id,
+				bool vf_request,
+				unsigned int vf_id)
+{
+	DLB_HW_INFO(hw, "DLB enable port arguments:\n");
+	if (vf_request)
+		DLB_HW_INFO(hw, "(Request from VF %d)\n", vf_id);
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n",
+		    port_id);
+}
+
+int dlb_hw_enable_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_enable_ldb_port_args *args,
+			   struct dlb_cmd_response *resp,
+			   bool vf_request,
+			   unsigned int vf_id)
+{
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	int id;
+
+	dlb_log_enable_port(hw, domain_id, args->port_id, vf_request, vf_id);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_enable_ldb_port_args(hw,
+					    domain_id,
+					    args,
+					    resp,
+					    vf_request,
+					    vf_id))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, vf_request, domain);
+	if (!port) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* Hardware requires disabling the CQ before unmapping QIDs. */
+	if (!port->enabled) {
+		dlb_ldb_port_cq_enable(hw, port);
+		port->enabled = true;
+
+		hw->pf.num_enabled_ldb_ports++;
+		dlb_update_ldb_arb_threshold(hw);
+	}
+
+	resp->status = 0;
+
+	return 0;
+}
+
+static void dlb_log_disable_port(struct dlb_hw *hw,
+				 u32 domain_id,
+				 u32 port_id,
+				 bool vf_request,
+				 unsigned int vf_id)
+{
+	DLB_HW_INFO(hw, "DLB disable port arguments:\n");
+	if (vf_request)
+		DLB_HW_INFO(hw, "(Request from VF %d)\n", vf_id);
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n",
+		    domain_id);
+	DLB_HW_INFO(hw, "\tPort ID:   %d\n",
+		    port_id);
+}
+
+int dlb_hw_disable_ldb_port(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_disable_ldb_port_args *args,
+			    struct dlb_cmd_response *resp,
+			    bool vf_request,
+			    unsigned int vf_id)
+{
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+	int id;
+
+	dlb_log_disable_port(hw, domain_id, args->port_id, vf_request, vf_id);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_disable_ldb_port_args(hw,
+					     domain_id,
+					     args,
+					     resp,
+					     vf_request,
+					     vf_id))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_ldb_port(id, vf_request, domain);
+	if (!port) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* Hardware requires disabling the CQ before unmapping QIDs. */
+	if (port->enabled) {
+		dlb_ldb_port_cq_disable(hw, port);
+		port->enabled = false;
+
+		hw->pf.num_enabled_ldb_ports--;
+		dlb_update_ldb_arb_threshold(hw);
+	}
+
+	resp->status = 0;
+
+	return 0;
+}
+
+int dlb_hw_enable_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_enable_dir_port_args *args,
+			   struct dlb_cmd_response *resp,
+			   bool vf_request,
+			   unsigned int vf_id)
+{
+	struct dlb_dir_pq_pair *port;
+	struct dlb_domain *domain;
+	int id;
+
+	dlb_log_enable_port(hw, domain_id, args->port_id, vf_request, vf_id);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_enable_dir_port_args(hw,
+					    domain_id,
+					    args,
+					    resp,
+					    vf_request,
+					    vf_id))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_dir_pq(id, vf_request, domain);
+	if (!port) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* Hardware requires disabling the CQ before unmapping QIDs. */
+	if (!port->enabled) {
+		dlb_dir_port_cq_enable(hw, port);
+		port->enabled = true;
+	}
+
+	resp->status = 0;
+
+	return 0;
+}
+
+int dlb_hw_disable_dir_port(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_disable_dir_port_args *args,
+			    struct dlb_cmd_response *resp,
+			    bool vf_request,
+			    unsigned int vf_id)
+{
+	struct dlb_dir_pq_pair *port;
+	struct dlb_domain *domain;
+	int id;
+
+	dlb_log_disable_port(hw, domain_id, args->port_id, vf_request, vf_id);
+
+	/* Verify that hardware resources are available before attempting to
+	 * satisfy the request. This simplifies the error unwinding code.
+	 */
+	if (dlb_verify_disable_dir_port_args(hw,
+					     domain_id,
+					     args,
+					     resp,
+					     vf_request,
+					     vf_id))
+		return -EINVAL;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+	if (!domain) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: domain not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	id = args->port_id;
+
+	port = dlb_get_domain_used_dir_pq(id, vf_request, domain);
+	if (!port) {
+		DLB_HW_ERR(hw,
+			   "[%s():%d] Internal error: port not found\n",
+			   __func__, __LINE__);
+		return -EFAULT;
+	}
+
+	/* Hardware requires disabling the CQ before unmapping QIDs. */
+	if (port->enabled) {
+		dlb_dir_port_cq_disable(hw, port);
+		port->enabled = false;
+	}
+
+	resp->status = 0;
+
+	return 0;
+}
+
+int dlb_notify_vf(struct dlb_hw *hw,
+		  unsigned int vf_id,
+		  enum dlb_mbox_vf_notification_type notification)
+{
+	struct dlb_mbox_vf_notification_cmd_req req;
+	int retry_cnt;
+
+	req.hdr.type = DLB_MBOX_VF_CMD_NOTIFICATION;
+	req.notification = notification;
+
+	if (dlb_pf_write_vf_mbox_req(hw, vf_id, &req, sizeof(req)))
+		return -1;
+
+	dlb_send_async_pf_to_vf_msg(hw, vf_id);
+
+	/* Timeout after 1 second of inactivity */
+	retry_cnt = 0;
+	while (!dlb_pf_to_vf_complete(hw, vf_id)) {
+		os_msleep(1);
+		if (++retry_cnt >= 1000) {
+			DLB_HW_ERR(hw,
+				   "PF driver timed out waiting for mbox response\n");
+			return -1;
+		}
+	}
+
+	/* No response data expected for notifications. */
+
+	return 0;
+}
+
+int dlb_vf_in_use(struct dlb_hw *hw, unsigned int vf_id)
+{
+	struct dlb_mbox_vf_in_use_cmd_resp resp;
+	struct dlb_mbox_vf_in_use_cmd_req req;
+	int retry_cnt;
+
+	req.hdr.type = DLB_MBOX_VF_CMD_IN_USE;
+
+	if (dlb_pf_write_vf_mbox_req(hw, vf_id, &req, sizeof(req)))
+		return -1;
+
+	dlb_send_async_pf_to_vf_msg(hw, vf_id);
+
+	/* Timeout after 1 second of inactivity */
+	retry_cnt = 0;
+	while (!dlb_pf_to_vf_complete(hw, vf_id)) {
+		os_msleep(1);
+		if (++retry_cnt >= 1000) {
+			DLB_HW_ERR(hw,
+				   "PF driver timed out waiting for mbox response\n");
+			return -1;
+		}
+	}
+
+	if (dlb_pf_read_vf_mbox_resp(hw, vf_id, &resp, sizeof(resp)))
+		return -1;
+
+	if (resp.hdr.status != DLB_MBOX_ST_SUCCESS) {
+		DLB_HW_ERR(hw,
+			   "[%s()]: failed with mailbox error: %s\n",
+			   __func__,
+			   DLB_MBOX_ST_STRING(&resp));
+
+		return -1;
+	}
+
+	return resp.in_use;
+}
+
+static int dlb_vf_domain_alert(struct dlb_hw *hw,
+			       unsigned int vf_id,
+			       u32 domain_id,
+			       u32 alert_id,
+			       u32 aux_alert_data)
+{
+	struct dlb_mbox_vf_alert_cmd_req req;
+	int retry_cnt;
+
+	req.hdr.type = DLB_MBOX_VF_CMD_DOMAIN_ALERT;
+	req.domain_id = domain_id;
+	req.alert_id = alert_id;
+	req.aux_alert_data = aux_alert_data;
+
+	if (dlb_pf_write_vf_mbox_req(hw, vf_id, &req, sizeof(req)))
+		return -1;
+
+	dlb_send_async_pf_to_vf_msg(hw, vf_id);
+
+	/* Timeout after 1 second of inactivity */
+	retry_cnt = 0;
+	while (!dlb_pf_to_vf_complete(hw, vf_id)) {
+		os_msleep(1);
+		if (++retry_cnt >= 1000) {
+			DLB_HW_ERR(hw,
+				   "PF driver timed out waiting for mbox response\n");
+			return -1;
+		}
+	}
+
+	/* No response data expected for alarm notifications. */
+
+	return 0;
+}
+
+void dlb_set_msix_mode(struct dlb_hw *hw, int mode)
+{
+	union dlb_sys_msix_mode r0 = { {0} };
+
+	r0.field.mode = mode;
+
+	DLB_CSR_WR(hw, DLB_SYS_MSIX_MODE, r0.val);
+}
+
+int dlb_configure_ldb_cq_interrupt(struct dlb_hw *hw,
+				   int port_id,
+				   int vector,
+				   int mode,
+				   unsigned int vf,
+				   unsigned int owner_vf,
+				   u16 threshold)
+{
+	union dlb_chp_ldb_cq_int_depth_thrsh r0 = { {0} };
+	union dlb_chp_ldb_cq_int_enb r1 = { {0} };
+	union dlb_sys_ldb_cq_isr r2 = { {0} };
+	struct dlb_ldb_port *port;
+	bool vf_request;
+
+	vf_request = (mode == DLB_CQ_ISR_MODE_MSI);
+
+	port = dlb_get_ldb_port_from_id(hw, port_id, vf_request, vf);
+	if (!port) {
+		DLB_HW_ERR(hw,
+			   "[%s()]: Internal error: failed to enable LDB CQ int\n\tport_id: %u, vf_req: %u, vf: %u\n",
+			   __func__, port_id, vf_request, vf);
+		return -EINVAL;
+	}
+
+	/* Trigger the interrupt when threshold or more QEs arrive in the CQ */
+	r0.field.depth_threshold = threshold - 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_INT_DEPTH_THRSH(port->id.phys_id),
+		   r0.val);
+
+	r1.field.en_depth = 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_LDB_CQ_INT_ENB(port->id.phys_id), r1.val);
+
+	r2.field.vector = vector;
+	r2.field.vf = owner_vf;
+	r2.field.en_code = mode;
+
+	DLB_CSR_WR(hw, DLB_SYS_LDB_CQ_ISR(port->id.phys_id), r2.val);
+
+	return 0;
+}
+
+int dlb_configure_dir_cq_interrupt(struct dlb_hw *hw,
+				   int port_id,
+				   int vector,
+				   int mode,
+				   unsigned int vf,
+				   unsigned int owner_vf,
+				   u16 threshold)
+{
+	union dlb_chp_dir_cq_int_depth_thrsh r0 = { {0} };
+	union dlb_chp_dir_cq_int_enb r1 = { {0} };
+	union dlb_sys_dir_cq_isr r2 = { {0} };
+	struct dlb_dir_pq_pair *port;
+	bool vf_request;
+
+	vf_request = (mode == DLB_CQ_ISR_MODE_MSI);
+
+	port = dlb_get_dir_pq_from_id(hw, port_id, vf_request, vf);
+	if (!port) {
+		DLB_HW_ERR(hw,
+			   "[%s()]: Internal error: failed to enable DIR CQ int\n\tport_id: %u, vf_req: %u, vf: %u\n",
+			   __func__, port_id, vf_request, vf);
+		return -EINVAL;
+	}
+
+	/* Trigger the interrupt when threshold or more QEs arrive in the CQ */
+	r0.field.depth_threshold = threshold - 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_INT_DEPTH_THRSH(port->id.phys_id),
+		   r0.val);
+
+	r1.field.en_depth = 1;
+
+	DLB_CSR_WR(hw, DLB_CHP_DIR_CQ_INT_ENB(port->id.phys_id), r1.val);
+
+	r2.field.vector = vector;
+	r2.field.vf = owner_vf;
+	r2.field.en_code = mode;
+
+	DLB_CSR_WR(hw, DLB_SYS_DIR_CQ_ISR(port->id.phys_id), r2.val);
+
+	return 0;
+}
+
+int dlb_arm_cq_interrupt(struct dlb_hw *hw,
+			 int port_id,
+			 bool is_ldb,
+			 bool vf_request,
+			 unsigned int vf_id)
+{
+	u32 val;
+	u32 reg;
+
+	if (vf_request && is_ldb) {
+		struct dlb_ldb_port *ldb_port;
+
+		ldb_port = dlb_get_ldb_port_from_id(hw, port_id, true, vf_id);
+
+		if (!ldb_port || !ldb_port->configured)
+			return -EINVAL;
+
+		port_id = ldb_port->id.phys_id;
+	} else if (vf_request && !is_ldb) {
+		struct dlb_dir_pq_pair *dir_port;
+
+		dir_port = dlb_get_dir_pq_from_id(hw, port_id, true, vf_id);
+
+		if (!dir_port || !dir_port->port_configured)
+			return -EINVAL;
+
+		port_id = dir_port->id.phys_id;
+	}
+
+	val = 1 << (port_id % 32);
+
+	if (is_ldb && port_id < 32)
+		reg = DLB_CHP_LDB_CQ_INTR_ARMED0;
+	else if (is_ldb && port_id < 64)
+		reg = DLB_CHP_LDB_CQ_INTR_ARMED1;
+	else if (!is_ldb && port_id < 32)
+		reg = DLB_CHP_DIR_CQ_INTR_ARMED0;
+	else if (!is_ldb && port_id < 64)
+		reg = DLB_CHP_DIR_CQ_INTR_ARMED1;
+	else if (!is_ldb && port_id < 96)
+		reg = DLB_CHP_DIR_CQ_INTR_ARMED2;
+	else
+		reg = DLB_CHP_DIR_CQ_INTR_ARMED3;
+
+	DLB_CSR_WR(hw, reg, val);
+
+	dlb_flush_csr(hw);
+
+	return 0;
+}
+
+void dlb_read_compressed_cq_intr_status(struct dlb_hw *hw,
+					u32 *ldb_interrupts,
+					u32 *dir_interrupts)
+{
+	/* Read every CQ's interrupt status */
+
+	ldb_interrupts[0] = DLB_CSR_RD(hw, DLB_SYS_LDB_CQ_31_0_OCC_INT_STS);
+	ldb_interrupts[1] = DLB_CSR_RD(hw, DLB_SYS_LDB_CQ_63_32_OCC_INT_STS);
+
+	dir_interrupts[0] = DLB_CSR_RD(hw, DLB_SYS_DIR_CQ_31_0_OCC_INT_STS);
+	dir_interrupts[1] = DLB_CSR_RD(hw, DLB_SYS_DIR_CQ_63_32_OCC_INT_STS);
+	dir_interrupts[2] = DLB_CSR_RD(hw, DLB_SYS_DIR_CQ_95_64_OCC_INT_STS);
+	dir_interrupts[3] = DLB_CSR_RD(hw, DLB_SYS_DIR_CQ_127_96_OCC_INT_STS);
+}
+
+static void dlb_ack_msix_interrupt(struct dlb_hw *hw, int vector)
+{
+	union dlb_sys_msix_ack r0 = { {0} };
+
+	switch (vector) {
+	case 0:
+		r0.field.msix_0_ack = 1;
+		break;
+	case 1:
+		r0.field.msix_1_ack = 1;
+		break;
+	case 2:
+		r0.field.msix_2_ack = 1;
+		break;
+	case 3:
+		r0.field.msix_3_ack = 1;
+		break;
+	case 4:
+		r0.field.msix_4_ack = 1;
+		break;
+	case 5:
+		r0.field.msix_5_ack = 1;
+		break;
+	case 6:
+		r0.field.msix_6_ack = 1;
+		break;
+	case 7:
+		r0.field.msix_7_ack = 1;
+		break;
+	case 8:
+		r0.field.msix_8_ack = 1;
+		/*
+		 * CSSY-1650
+		 * workaround h/w bug for lost MSI-X interrupts
+		 *
+		 * The recommended workaround for acknowledging
+		 * vector 8 interrupts is :
+		 *   1: set   MSI-X mask
+		 *   2: set   MSIX_PASSTHROUGH
+		 *   3: clear MSIX_ACK
+		 *   4: clear MSIX_PASSTHROUGH
+		 *   5: clear MSI-X mask
+		 *
+		 * The MSIX-ACK (step 3) is cleared for all vectors
+		 * below. We handle steps 1 & 2 for vector 8 here.
+		 *
+		 * The bitfields for MSIX_ACK and MSIX_PASSTHRU are
+		 * defined the same, so we just use the MSIX_ACK
+		 * value when writing to PASSTHRU.
+		 */
+
+		/* set MSI-X mask and passthrough for vector 8 */
+		DLB_FUNC_WR(hw, DLB_MSIX_MEM_VECTOR_CTRL(8), 1);
+		DLB_CSR_WR(hw, DLB_SYS_MSIX_PASSTHRU, r0.val);
+		break;
+	}
+
+	/* clear MSIX_ACK (write one to clear) */
+	DLB_CSR_WR(hw, DLB_SYS_MSIX_ACK, r0.val);
+
+	if (vector == 8) {
+		/*
+		 * finish up steps 4 & 5 of the workaround -
+		 * clear pasthrough and mask
+		 */
+		DLB_CSR_WR(hw, DLB_SYS_MSIX_PASSTHRU, 0);
+		DLB_FUNC_WR(hw, DLB_MSIX_MEM_VECTOR_CTRL(8), 0);
+	}
+
+	dlb_flush_csr(hw);
+}
+
+void dlb_ack_compressed_cq_intr(struct dlb_hw *hw,
+				u32 *ldb_interrupts,
+				u32 *dir_interrupts)
+{
+	/* Write back the status regs to ack the interrupts */
+	if (ldb_interrupts[0])
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_CQ_31_0_OCC_INT_STS,
+			   ldb_interrupts[0]);
+	if (ldb_interrupts[1])
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_CQ_63_32_OCC_INT_STS,
+			   ldb_interrupts[1]);
+
+	if (dir_interrupts[0])
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_CQ_31_0_OCC_INT_STS,
+			   dir_interrupts[0]);
+	if (dir_interrupts[1])
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_CQ_63_32_OCC_INT_STS,
+			   dir_interrupts[1]);
+	if (dir_interrupts[2])
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_CQ_95_64_OCC_INT_STS,
+			   dir_interrupts[2]);
+	if (dir_interrupts[3])
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_CQ_127_96_OCC_INT_STS,
+			   dir_interrupts[3]);
+
+	dlb_ack_msix_interrupt(hw, DLB_PF_COMPRESSED_MODE_CQ_VECTOR_ID);
+}
+
+u32 dlb_read_vf_intr_status(struct dlb_hw *hw)
+{
+	return DLB_FUNC_RD(hw, DLB_FUNC_VF_VF_MSI_ISR);
+}
+
+void dlb_ack_vf_intr_status(struct dlb_hw *hw, u32 interrupts)
+{
+	DLB_FUNC_WR(hw, DLB_FUNC_VF_VF_MSI_ISR, interrupts);
+}
+
+void dlb_ack_vf_msi_intr(struct dlb_hw *hw, u32 interrupts)
+{
+	DLB_FUNC_WR(hw, DLB_FUNC_VF_VF_MSI_ISR_PEND, interrupts);
+}
+
+void dlb_ack_pf_mbox_int(struct dlb_hw *hw)
+{
+	union dlb_func_vf_pf2vf_mailbox_isr r0;
+
+	r0.field.pf_isr = 1;
+
+	DLB_FUNC_WR(hw, DLB_FUNC_VF_PF2VF_MAILBOX_ISR, r0.val);
+}
+
+u32 dlb_read_vf_to_pf_int_bitvec(struct dlb_hw *hw)
+{
+	/* The PF has one VF->PF MBOX ISR register per VF space, but they all
+	 * alias to the same physical register.
+	 */
+	return DLB_FUNC_RD(hw, DLB_FUNC_PF_VF2PF_MAILBOX_ISR(0));
+}
+
+void dlb_ack_vf_mbox_int(struct dlb_hw *hw, u32 bitvec)
+{
+	/* The PF has one VF->PF MBOX ISR register per VF space, but they all
+	 * alias to the same physical register.
+	 */
+	DLB_FUNC_WR(hw, DLB_FUNC_PF_VF2PF_MAILBOX_ISR(0), bitvec);
+}
+
+u32 dlb_read_vf_flr_int_bitvec(struct dlb_hw *hw)
+{
+	/* The PF has one VF->PF FLR ISR register per VF space, but they all
+	 * alias to the same physical register.
+	 */
+	return DLB_FUNC_RD(hw, DLB_FUNC_PF_VF2PF_FLR_ISR(0));
+}
+
+void dlb_set_vf_reset_in_progress(struct dlb_hw *hw, int vf)
+{
+	u32 bitvec = DLB_FUNC_RD(hw, DLB_FUNC_PF_VF_RESET_IN_PROGRESS(0));
+
+	bitvec |= (1 << vf);
+
+	DLB_FUNC_WR(hw, DLB_FUNC_PF_VF_RESET_IN_PROGRESS(0), bitvec);
+}
+
+void dlb_clr_vf_reset_in_progress(struct dlb_hw *hw, int vf)
+{
+	u32 bitvec = DLB_FUNC_RD(hw, DLB_FUNC_PF_VF_RESET_IN_PROGRESS(0));
+
+	bitvec &= ~(1 << vf);
+
+	DLB_FUNC_WR(hw, DLB_FUNC_PF_VF_RESET_IN_PROGRESS(0), bitvec);
+}
+
+void dlb_ack_vf_flr_int(struct dlb_hw *hw, u32 bitvec, bool a_stepping)
+{
+	union dlb_sys_func_vf_bar_dsbl r0 = { {0} };
+	u32 clear;
+	int i;
+
+	if (!bitvec)
+		return;
+
+	/* Re-enable access to the VF BAR */
+	r0.field.func_vf_bar_dis = 0;
+	for (i = 0; i < DLB_MAX_NUM_VFS; i++) {
+		if (!(bitvec & (1 << i)))
+			continue;
+
+		DLB_CSR_WR(hw, DLB_SYS_FUNC_VF_BAR_DSBL(i), r0.val);
+	}
+
+	/* Notify the VF driver that the reset has completed. This register is
+	 * RW in A-stepping devices, WOCLR otherwise.
+	 */
+	if (a_stepping) {
+		clear = DLB_FUNC_RD(hw, DLB_FUNC_PF_VF_RESET_IN_PROGRESS(0));
+		clear &= ~bitvec;
+	} else {
+		clear = bitvec;
+	}
+
+	DLB_FUNC_WR(hw, DLB_FUNC_PF_VF_RESET_IN_PROGRESS(0), clear);
+
+	/* Mark the FLR ISR as complete */
+	DLB_FUNC_WR(hw, DLB_FUNC_PF_VF2PF_FLR_ISR(0), bitvec);
+}
+
+void dlb_ack_vf_to_pf_int(struct dlb_hw *hw,
+			  u32 mbox_bitvec,
+			  u32 flr_bitvec)
+{
+	int i;
+
+	dlb_ack_msix_interrupt(hw, DLB_INT_VF_TO_PF_MBOX);
+
+	for (i = 0; i < DLB_MAX_NUM_VFS; i++) {
+		union dlb_func_pf_vf2pf_isr_pend r0 = { {0} };
+
+		if (!((mbox_bitvec & (1 << i)) || (flr_bitvec & (1 << i))))
+			continue;
+
+		/* Unset the VF's ISR pending bit */
+		r0.field.isr_pend = 1;
+		DLB_FUNC_WR(hw, DLB_FUNC_PF_VF2PF_ISR_PEND(i), r0.val);
+	}
+}
+
+void dlb_enable_alarm_interrupts(struct dlb_hw *hw)
+{
+	union dlb_sys_ingress_alarm_enbl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_INGRESS_ALARM_ENBL);
+
+	r0.field.illegal_hcw = 1;
+	r0.field.illegal_pp = 1;
+	r0.field.disabled_pp = 1;
+	r0.field.illegal_qid = 1;
+	r0.field.disabled_qid = 1;
+	r0.field.illegal_ldb_qid_cfg = 1;
+	r0.field.illegal_cqid = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_INGRESS_ALARM_ENBL, r0.val);
+}
+
+void dlb_disable_alarm_interrupts(struct dlb_hw *hw)
+{
+	union dlb_sys_ingress_alarm_enbl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_INGRESS_ALARM_ENBL);
+
+	r0.field.illegal_hcw = 0;
+	r0.field.illegal_pp = 0;
+	r0.field.disabled_pp = 0;
+	r0.field.illegal_qid = 0;
+	r0.field.disabled_qid = 0;
+	r0.field.illegal_ldb_qid_cfg = 0;
+	r0.field.illegal_cqid = 0;
+
+	DLB_CSR_WR(hw, DLB_SYS_INGRESS_ALARM_ENBL, r0.val);
+}
+
+static void dlb_log_alarm_syndrome(struct dlb_hw *hw,
+				   const char *str,
+				   union dlb_sys_alarm_hw_synd r0)
+{
+	DLB_HW_ERR(hw, "%s:\n", str);
+	DLB_HW_ERR(hw, "\tsyndrome: 0x%x\n", r0.field.syndrome);
+	DLB_HW_ERR(hw, "\trtype:    0x%x\n", r0.field.rtype);
+	DLB_HW_ERR(hw, "\tfrom_dmv: 0x%x\n", r0.field.from_dmv);
+	DLB_HW_ERR(hw, "\tis_ldb:   0x%x\n", r0.field.is_ldb);
+	DLB_HW_ERR(hw, "\tcls:      0x%x\n", r0.field.cls);
+	DLB_HW_ERR(hw, "\taid:      0x%x\n", r0.field.aid);
+	DLB_HW_ERR(hw, "\tunit:     0x%x\n", r0.field.unit);
+	DLB_HW_ERR(hw, "\tsource:   0x%x\n", r0.field.source);
+	DLB_HW_ERR(hw, "\tmore:     0x%x\n", r0.field.more);
+	DLB_HW_ERR(hw, "\tvalid:    0x%x\n", r0.field.valid);
+}
+
+/* Note: this array's contents must match dlb_alert_id() */
+static const char dlb_alert_strings[NUM_DLB_DOMAIN_ALERTS][128] = {
+	[DLB_DOMAIN_ALERT_PP_OUT_OF_CREDITS] = "Insufficient credits",
+	[DLB_DOMAIN_ALERT_PP_ILLEGAL_ENQ] = "Illegal enqueue",
+	[DLB_DOMAIN_ALERT_PP_EXCESS_TOKEN_POPS] = "Excess token pops",
+	[DLB_DOMAIN_ALERT_ILLEGAL_HCW] = "Illegal HCW",
+	[DLB_DOMAIN_ALERT_ILLEGAL_QID] = "Illegal QID",
+	[DLB_DOMAIN_ALERT_DISABLED_QID] = "Disabled QID",
+};
+
+static void dlb_log_pf_vf_syndrome(struct dlb_hw *hw,
+				   const char *str,
+				   union dlb_sys_alarm_pf_synd0 r0,
+				   union dlb_sys_alarm_pf_synd1 r1,
+				   union dlb_sys_alarm_pf_synd2 r2,
+				   u32 alert_id)
+{
+	DLB_HW_ERR(hw, "%s:\n", str);
+	if (alert_id < NUM_DLB_DOMAIN_ALERTS)
+		DLB_HW_ERR(hw, "Alert: %s\n", dlb_alert_strings[alert_id]);
+	DLB_HW_ERR(hw, "\tsyndrome:     0x%x\n", r0.field.syndrome);
+	DLB_HW_ERR(hw, "\trtype:        0x%x\n", r0.field.rtype);
+	DLB_HW_ERR(hw, "\tfrom_dmv:     0x%x\n", r0.field.from_dmv);
+	DLB_HW_ERR(hw, "\tis_ldb:       0x%x\n", r0.field.is_ldb);
+	DLB_HW_ERR(hw, "\tcls:          0x%x\n", r0.field.cls);
+	DLB_HW_ERR(hw, "\taid:          0x%x\n", r0.field.aid);
+	DLB_HW_ERR(hw, "\tunit:         0x%x\n", r0.field.unit);
+	DLB_HW_ERR(hw, "\tsource:       0x%x\n", r0.field.source);
+	DLB_HW_ERR(hw, "\tmore:         0x%x\n", r0.field.more);
+	DLB_HW_ERR(hw, "\tvalid:        0x%x\n", r0.field.valid);
+	DLB_HW_ERR(hw, "\tdsi:          0x%x\n", r1.field.dsi);
+	DLB_HW_ERR(hw, "\tqid:          0x%x\n", r1.field.qid);
+	DLB_HW_ERR(hw, "\tqtype:        0x%x\n", r1.field.qtype);
+	DLB_HW_ERR(hw, "\tqpri:         0x%x\n", r1.field.qpri);
+	DLB_HW_ERR(hw, "\tmsg_type:     0x%x\n", r1.field.msg_type);
+	DLB_HW_ERR(hw, "\tlock_id:      0x%x\n", r2.field.lock_id);
+	DLB_HW_ERR(hw, "\tmeas:         0x%x\n", r2.field.meas);
+	DLB_HW_ERR(hw, "\tdebug:        0x%x\n", r2.field.debug);
+	DLB_HW_ERR(hw, "\tcq_pop:       0x%x\n", r2.field.cq_pop);
+	DLB_HW_ERR(hw, "\tqe_uhl:       0x%x\n", r2.field.qe_uhl);
+	DLB_HW_ERR(hw, "\tqe_orsp:      0x%x\n", r2.field.qe_orsp);
+	DLB_HW_ERR(hw, "\tqe_valid:     0x%x\n", r2.field.qe_valid);
+	DLB_HW_ERR(hw, "\tcq_int_rearm: 0x%x\n", r2.field.cq_int_rearm);
+	DLB_HW_ERR(hw, "\tdsi_error:    0x%x\n", r2.field.dsi_error);
+}
+
+static void dlb_clear_syndrome_register(struct dlb_hw *hw, u32 offset)
+{
+	union dlb_sys_alarm_hw_synd r0 = { {0} };
+
+	r0.field.valid = 1;
+	r0.field.more = 1;
+
+	DLB_CSR_WR(hw, offset, r0.val);
+}
+
+void dlb_process_alarm_interrupt(struct dlb_hw *hw)
+{
+	union dlb_sys_alarm_hw_synd r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_ALARM_HW_SYND);
+
+	dlb_log_alarm_syndrome(hw, "HW alarm syndrome", r0);
+
+	dlb_clear_syndrome_register(hw, DLB_SYS_ALARM_HW_SYND);
+
+	dlb_ack_msix_interrupt(hw, DLB_INT_ALARM);
+}
+
+static void dlb_process_ingress_error(struct dlb_hw *hw,
+				      union dlb_sys_alarm_pf_synd0 r0,
+				      u32 alert_id,
+				      bool vf_error,
+				      unsigned int vf_id)
+{
+	struct dlb_domain *domain;
+	bool is_ldb;
+	u8 port_id;
+	int ret;
+
+	port_id = r0.field.syndrome & 0x7F;
+	if (r0.field.source == DLB_ALARM_HW_SOURCE_SYS)
+		is_ldb = r0.field.is_ldb;
+	else
+		is_ldb = (r0.field.syndrome & 0x80) != 0;
+
+	/* Get the domain ID and, if it's a VF domain, the virtual port ID */
+	if (is_ldb) {
+		struct dlb_ldb_port *port;
+
+		port = dlb_get_ldb_port_from_id(hw, port_id, vf_error, vf_id);
+
+		if (!port) {
+			DLB_HW_ERR(hw,
+				   "[%s()]: Internal error: unable to find LDB port\n\tport: %u, vf_error: %u, vf_id: %u\n",
+				   __func__, port_id, vf_error, vf_id);
+			return;
+		}
+
+		domain = &hw->domains[port->domain_id.phys_id];
+	} else {
+		struct dlb_dir_pq_pair *port;
+
+		port = dlb_get_dir_pq_from_id(hw, port_id, vf_error, vf_id);
+
+		if (!port) {
+			DLB_HW_ERR(hw,
+				   "[%s()]: Internal error: unable to find DIR port\n\tport: %u, vf_error: %u, vf_id: %u\n",
+				   __func__, port_id, vf_error, vf_id);
+			return;
+		}
+
+		domain = &hw->domains[port->domain_id.phys_id];
+	}
+
+	if (vf_error)
+		ret = dlb_vf_domain_alert(hw,
+					  vf_id,
+					  domain->id.virt_id,
+					  alert_id,
+					  (is_ldb << 8) | port_id);
+	else
+		ret = os_notify_user_space(hw,
+					   domain->id.phys_id,
+					   alert_id,
+					   (is_ldb << 8) | port_id);
+
+	if (ret)
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to notify\n",
+			   __func__);
+}
+
+static u32 dlb_alert_id(union dlb_sys_alarm_pf_synd0 r0)
+{
+	if (r0.field.unit == DLB_ALARM_HW_UNIT_CHP &&
+	    r0.field.aid == DLB_ALARM_HW_CHP_AID_OUT_OF_CREDITS)
+		return DLB_DOMAIN_ALERT_PP_OUT_OF_CREDITS;
+	else if (r0.field.unit == DLB_ALARM_HW_UNIT_CHP &&
+		 r0.field.aid == DLB_ALARM_HW_CHP_AID_ILLEGAL_ENQ)
+		return DLB_DOMAIN_ALERT_PP_ILLEGAL_ENQ;
+	else if (r0.field.unit == DLB_ALARM_HW_UNIT_LSP &&
+		 r0.field.aid == DLB_ALARM_HW_LSP_AID_EXCESS_TOKEN_POPS)
+		return DLB_DOMAIN_ALERT_PP_EXCESS_TOKEN_POPS;
+	else if (r0.field.source == DLB_ALARM_HW_SOURCE_SYS &&
+		 r0.field.aid == DLB_ALARM_SYS_AID_ILLEGAL_HCW)
+		return DLB_DOMAIN_ALERT_ILLEGAL_HCW;
+	else if (r0.field.source == DLB_ALARM_HW_SOURCE_SYS &&
+		 r0.field.aid == DLB_ALARM_SYS_AID_ILLEGAL_QID)
+		return DLB_DOMAIN_ALERT_ILLEGAL_QID;
+	else if (r0.field.source == DLB_ALARM_HW_SOURCE_SYS &&
+		 r0.field.aid == DLB_ALARM_SYS_AID_DISABLED_QID)
+		return DLB_DOMAIN_ALERT_DISABLED_QID;
+	else
+		return NUM_DLB_DOMAIN_ALERTS;
+}
+
+void dlb_process_ingress_error_interrupt(struct dlb_hw *hw)
+{
+	union dlb_sys_alarm_pf_synd0 r0;
+	union dlb_sys_alarm_pf_synd1 r1;
+	union dlb_sys_alarm_pf_synd2 r2;
+	u32 alert_id;
+	int i;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_ALARM_PF_SYND0);
+
+	if (r0.field.valid) {
+		r1.val = DLB_CSR_RD(hw, DLB_SYS_ALARM_PF_SYND1);
+		r2.val = DLB_CSR_RD(hw, DLB_SYS_ALARM_PF_SYND2);
+
+		alert_id = dlb_alert_id(r0);
+
+		dlb_log_pf_vf_syndrome(hw,
+				       "PF Ingress error alarm",
+				       r0, r1, r2, alert_id);
+
+		dlb_clear_syndrome_register(hw, DLB_SYS_ALARM_PF_SYND0);
+
+		dlb_process_ingress_error(hw, r0, alert_id, false, 0);
+	}
+
+	for (i = 0; i < DLB_MAX_NUM_VFS; i++) {
+		r0.val = DLB_CSR_RD(hw, DLB_SYS_ALARM_VF_SYND0(i));
+
+		if (!r0.field.valid)
+			continue;
+
+		r1.val = DLB_CSR_RD(hw, DLB_SYS_ALARM_VF_SYND1(i));
+		r2.val = DLB_CSR_RD(hw, DLB_SYS_ALARM_VF_SYND2(i));
+
+		alert_id = dlb_alert_id(r0);
+
+		dlb_log_pf_vf_syndrome(hw,
+				       "VF Ingress error alarm",
+				       r0, r1, r2, alert_id);
+
+		dlb_clear_syndrome_register(hw,
+					    DLB_SYS_ALARM_VF_SYND0(i));
+
+		dlb_process_ingress_error(hw, r0, alert_id, true, i);
+	}
+
+	dlb_ack_msix_interrupt(hw, DLB_INT_INGRESS_ERROR);
+}
+
+int dlb_get_group_sequence_numbers(struct dlb_hw *hw, unsigned int group_id)
+{
+	if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
+		return -EINVAL;
+
+	return hw->rsrcs.sn_groups[group_id].sequence_numbers_per_queue;
+}
+
+int dlb_get_group_sequence_number_occupancy(struct dlb_hw *hw,
+					    unsigned int group_id)
+{
+	if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
+		return -EINVAL;
+
+	return dlb_sn_group_used_slots(&hw->rsrcs.sn_groups[group_id]);
+}
+
+static void dlb_log_set_group_sequence_numbers(struct dlb_hw *hw,
+					       unsigned int group_id,
+					       unsigned long val)
+{
+	DLB_HW_INFO(hw, "DLB set group sequence numbers:\n");
+	DLB_HW_INFO(hw, "\tGroup ID: %u\n", group_id);
+	DLB_HW_INFO(hw, "\tValue:    %lu\n", val);
+}
+
+int dlb_set_group_sequence_numbers(struct dlb_hw *hw,
+				   unsigned int group_id,
+				   unsigned long val)
+{
+	u32 valid_allocations[6] = {32, 64, 128, 256, 512, 1024};
+	union dlb_ro_pipe_grp_sn_mode r0 = { {0} };
+	struct dlb_sn_group *group;
+	int mode;
+
+	if (group_id >= DLB_MAX_NUM_SEQUENCE_NUMBER_GROUPS)
+		return -EINVAL;
+
+	group = &hw->rsrcs.sn_groups[group_id];
+
+	/* Once the first load-balanced queue using an SN group is configured,
+	 * the group cannot be changed.
+	 */
+	if (group->slot_use_bitmap != 0)
+		return -EPERM;
+
+	for (mode = 0; mode < DLB_MAX_NUM_SEQUENCE_NUMBER_MODES; mode++)
+		if (val == valid_allocations[mode])
+			break;
+
+	if (mode == DLB_MAX_NUM_SEQUENCE_NUMBER_MODES)
+		return -EINVAL;
+
+	group->mode = mode;
+	group->sequence_numbers_per_queue = val;
+
+	r0.field.sn_mode_0 = hw->rsrcs.sn_groups[0].mode;
+	r0.field.sn_mode_1 = hw->rsrcs.sn_groups[1].mode;
+	r0.field.sn_mode_2 = hw->rsrcs.sn_groups[2].mode;
+	r0.field.sn_mode_3 = hw->rsrcs.sn_groups[3].mode;
+
+	DLB_CSR_WR(hw, DLB_RO_PIPE_GRP_SN_MODE, r0.val);
+
+	dlb_log_set_group_sequence_numbers(hw, group_id, val);
+
+	return 0;
+}
+
+void dlb_disable_dp_vasr_feature(struct dlb_hw *hw)
+{
+	union dlb_dp_dir_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_DP_DIR_CSR_CTRL);
+
+	r0.field.cfg_vasr_dis = 1;
+
+	DLB_CSR_WR(hw, DLB_DP_DIR_CSR_CTRL, r0.val);
+}
+
+void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw)
+{
+	union dlb_chp_cfg_chp_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);
+
+	r0.val |= 1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT;
+
+	DLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);
+}
+
+void dlb_disable_excess_tokens_alarm(struct dlb_hw *hw)
+{
+	union dlb_chp_cfg_chp_csr_ctrl r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_CHP_CFG_CHP_CSR_CTRL);
+
+	r0.val &= ~(1 << DLB_CHP_CFG_EXCESS_TOKENS_SHIFT);
+
+	DLB_CSR_WR(hw, DLB_CHP_CFG_CHP_CSR_CTRL, r0.val);
+}
+
+static int dlb_reset_hw_resource(struct dlb_hw *hw, int type, int id)
+{
+	union dlb_cfg_mstr_diag_reset_sts r0 = { {0} };
+	union dlb_cfg_mstr_bcast_reset_vf_start r1 = { {0} };
+	int i;
+
+	r1.field.vf_reset_start = 1;
+
+	r1.field.vf_reset_type = type;
+	r1.field.vf_reset_id = id;
+
+	DLB_CSR_WR(hw, DLB_CFG_MSTR_BCAST_RESET_VF_START, r1.val);
+
+	/* Wait for hardware to complete. This is a finite time operation,
+	 * but wait set a loop bound just in case.
+	 */
+	for (i = 0; i < 1024 * 1024; i++) {
+		r0.val = DLB_CSR_RD(hw, DLB_CFG_MSTR_DIAG_RESET_STS);
+
+		if (r0.field.chp_vf_reset_done &&
+		    r0.field.rop_vf_reset_done &&
+		    r0.field.lsp_vf_reset_done &&
+		    r0.field.nalb_vf_reset_done &&
+		    r0.field.ap_vf_reset_done &&
+		    r0.field.dp_vf_reset_done &&
+		    r0.field.qed_vf_reset_done &&
+		    r0.field.dqed_vf_reset_done &&
+		    r0.field.aqed_vf_reset_done)
+			return 0;
+
+		os_udelay(1);
+	}
+
+	return -ETIMEDOUT;
+}
+
+static int dlb_domain_reset_hw_resources(struct dlb_hw *hw,
+					 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+	int ret;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_POOL_LDB,
+					    pool->id.phys_id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_POOL_DIR,
+					    pool->id.phys_id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, ldb_queue, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_QID_LDB,
+					    ldb_queue->id.phys_id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_QID_DIR,
+					    dir_port->id.phys_id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, ldb_port, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_CQ_LDB,
+					    ldb_port->id.phys_id);
+		if (ret)
+			return ret;
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
+		ret = dlb_reset_hw_resource(hw,
+					    VF_RST_TYPE_CQ_DIR,
+					    dir_port->id.phys_id);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static u32 dlb_ldb_cq_inflight_count(struct dlb_hw *hw,
+				     struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_infl_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_INFL_CNT(port->id.phys_id));
+
+	return r0.field.count;
+}
+
+static u32 dlb_ldb_cq_token_count(struct dlb_hw *hw,
+				  struct dlb_ldb_port *port)
+{
+	union dlb_lsp_cq_ldb_tkn_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_LDB_TKN_CNT(port->id.phys_id));
+
+	return r0.field.token_count;
+}
+
+static int dlb_drain_ldb_cq(struct dlb_hw *hw, struct dlb_ldb_port *port)
+{
+	u32 infl_cnt, tkn_cnt;
+	unsigned int i;
+
+	infl_cnt = dlb_ldb_cq_inflight_count(hw, port);
+
+	/* Account for the initial token count, which is used in order to
+	 * provide a CQ with depth less than 8.
+	 */
+	tkn_cnt = dlb_ldb_cq_token_count(hw, port) - port->init_tkn_cnt;
+
+	if (infl_cnt || tkn_cnt) {
+		struct dlb_hcw hcw_mem[8], *hcw;
+		void  *pp_addr;
+
+		pp_addr = os_map_producer_port(hw, port->id.phys_id, true);
+
+		/* Point hcw to a 64B-aligned location */
+		hcw = (struct dlb_hcw *)((uintptr_t)&hcw_mem[4] & ~0x3F);
+
+		/* Program the first HCW for a completion and token return and
+		 * the other HCWs as NOOPS
+		 */
+
+		memset(hcw, 0, 4 * sizeof(*hcw));
+		hcw->qe_comp = (infl_cnt > 0);
+		hcw->cq_token = (tkn_cnt > 0);
+		hcw->lock_id = tkn_cnt - 1;
+
+		/* Return tokens in the first HCW */
+		os_enqueue_four_hcws(hw, hcw, pp_addr);
+
+		hcw->cq_token = 0;
+
+		/* Issue remaining completions (if any) */
+		for (i = 1; i < infl_cnt; i++)
+			os_enqueue_four_hcws(hw, hcw, pp_addr);
+
+		os_fence_hcw(hw, pp_addr);
+
+		os_unmap_producer_port(hw, pp_addr);
+	}
+
+	return 0;
+}
+
+static int dlb_domain_wait_for_ldb_cqs_to_empty(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		int i;
+
+		for (i = 0; i < DLB_MAX_CQ_COMP_CHECK_LOOPS; i++) {
+			if (dlb_ldb_cq_inflight_count(hw, port) == 0)
+				break;
+		}
+
+		if (i == DLB_MAX_CQ_COMP_CHECK_LOOPS) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to flush load-balanced port %d's completions.\n",
+				   __func__, port->id.phys_id);
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static int dlb_domain_reset_software_state(struct dlb_hw *hw,
+					   struct dlb_domain *domain)
+{
+	struct dlb_ldb_queue *tmp_ldb_queue __attribute__((unused));
+	struct dlb_dir_pq_pair *tmp_dir_port __attribute__((unused));
+	struct dlb_ldb_port *tmp_ldb_port __attribute__((unused));
+	struct dlb_credit_pool *tmp_pool __attribute__((unused));
+	struct dlb_list_entry *iter1 __attribute__((unused));
+	struct dlb_list_entry *iter2 __attribute__((unused));
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+
+	struct dlb_function_resources *rsrcs;
+	struct dlb_list_head *list;
+	int ret;
+
+	rsrcs = domain->parent_func;
+
+	/* Move the domain's ldb queues to the function's avail list */
+	list = &domain->used_ldb_queues;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_queue, tmp_ldb_queue, iter1, iter2) {
+		if (ldb_queue->sn_cfg_valid) {
+			struct dlb_sn_group *grp;
+
+			grp = &hw->rsrcs.sn_groups[ldb_queue->sn_group];
+
+			dlb_sn_group_free_slot(grp, ldb_queue->sn_slot);
+			ldb_queue->sn_cfg_valid = false;
+		}
+
+		ldb_queue->owned = false;
+		ldb_queue->num_mappings = 0;
+		ldb_queue->num_pending_additions = 0;
+
+		dlb_list_del(&domain->used_ldb_queues, &ldb_queue->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_queues, &ldb_queue->func_list);
+		rsrcs->num_avail_ldb_queues++;
+	}
+
+	list = &domain->avail_ldb_queues;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_queue, tmp_ldb_queue, iter1, iter2) {
+		ldb_queue->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_queues,
+			     &ldb_queue->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_queues,
+			     &ldb_queue->func_list);
+		rsrcs->num_avail_ldb_queues++;
+	}
+
+	/* Move the domain's ldb ports to the function's avail list */
+	list = &domain->used_ldb_ports;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_port, tmp_ldb_port, iter1, iter2) {
+		int i;
+
+		ldb_port->owned = false;
+		ldb_port->configured = false;
+		ldb_port->num_pending_removals = 0;
+		ldb_port->num_mappings = 0;
+		for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++)
+			ldb_port->qid_map[i].state = DLB_QUEUE_UNMAPPED;
+
+		dlb_list_del(&domain->used_ldb_ports, &ldb_port->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_ports, &ldb_port->func_list);
+		rsrcs->num_avail_ldb_ports++;
+	}
+
+	list = &domain->avail_ldb_ports;
+	DLB_DOM_LIST_FOR_SAFE(*list, ldb_port, tmp_ldb_port, iter1, iter2) {
+		ldb_port->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_ports, &ldb_port->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_ports, &ldb_port->func_list);
+		rsrcs->num_avail_ldb_ports++;
+	}
+
+	/* Move the domain's dir ports to the function's avail list */
+	list = &domain->used_dir_pq_pairs;
+	DLB_DOM_LIST_FOR_SAFE(*list, dir_port, tmp_dir_port, iter1, iter2) {
+		dir_port->owned = false;
+		dir_port->port_configured = false;
+
+		dlb_list_del(&domain->used_dir_pq_pairs,
+			     &dir_port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_pq_pairs,
+			     &dir_port->func_list);
+		rsrcs->num_avail_dir_pq_pairs++;
+	}
+
+	list = &domain->avail_dir_pq_pairs;
+	DLB_DOM_LIST_FOR_SAFE(*list, dir_port, tmp_dir_port, iter1, iter2) {
+		dir_port->owned = false;
+
+		dlb_list_del(&domain->avail_dir_pq_pairs,
+			     &dir_port->domain_list);
+
+		dlb_list_add(&rsrcs->avail_dir_pq_pairs,
+			     &dir_port->func_list);
+		rsrcs->num_avail_dir_pq_pairs++;
+	}
+
+	/* Return hist list entries to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_hist_list_entries,
+				   domain->hist_list_entry_base,
+				   domain->total_hist_list_entries);
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain hist list base doesn't match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->total_hist_list_entries = 0;
+	domain->avail_hist_list_entries = 0;
+	domain->hist_list_entry_base = 0;
+	domain->hist_list_entry_offset = 0;
+
+	/* Return QED entries to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_qed_freelist_entries,
+				   domain->qed_freelist.base,
+				   (domain->qed_freelist.bound -
+					domain->qed_freelist.base));
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain QED base doesn't match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->qed_freelist.base = 0;
+	domain->qed_freelist.bound = 0;
+	domain->qed_freelist.offset = 0;
+
+	/* Return DQED entries back to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_dqed_freelist_entries,
+				   domain->dqed_freelist.base,
+				   (domain->dqed_freelist.bound -
+					domain->dqed_freelist.base));
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain DQED base doesn't match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->dqed_freelist.base = 0;
+	domain->dqed_freelist.bound = 0;
+	domain->dqed_freelist.offset = 0;
+
+	/* Return AQED entries back to the function */
+	ret = dlb_bitmap_set_range(rsrcs->avail_aqed_freelist_entries,
+				   domain->aqed_freelist.base,
+				   (domain->aqed_freelist.bound -
+					domain->aqed_freelist.base));
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: domain AQED base doesn't match the function's bitmap.\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	domain->aqed_freelist.base = 0;
+	domain->aqed_freelist.bound = 0;
+	domain->aqed_freelist.offset = 0;
+
+	/* Return ldb credit pools back to the function's avail list */
+	list = &domain->used_ldb_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+		pool->configured = false;
+
+		dlb_list_del(&domain->used_ldb_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_ldb_credit_pools++;
+	}
+
+	list = &domain->avail_ldb_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_ldb_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_ldb_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_ldb_credit_pools++;
+	}
+
+	/* Move dir credit pools back to the function */
+	list = &domain->used_dir_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+		pool->configured = false;
+
+		dlb_list_del(&domain->used_dir_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_dir_credit_pools++;
+	}
+
+	list = &domain->avail_dir_credit_pools;
+	DLB_DOM_LIST_FOR_SAFE(*list, pool, tmp_pool, iter1, iter2) {
+		pool->owned = false;
+
+		dlb_list_del(&domain->avail_dir_credit_pools,
+			     &pool->domain_list);
+		dlb_list_add(&rsrcs->avail_dir_credit_pools,
+			     &pool->func_list);
+		rsrcs->num_avail_dir_credit_pools++;
+	}
+
+	domain->num_pending_removals = 0;
+	domain->num_pending_additions = 0;
+	domain->configured = false;
+	domain->started = false;
+
+	/* Move the domain out of the used_domains list and back to the
+	 * function's avail_domains list.
+	 */
+	dlb_list_del(&rsrcs->used_domains, &domain->func_list);
+	dlb_list_add(&rsrcs->avail_domains, &domain->func_list);
+	rsrcs->num_avail_domains++;
+
+	return 0;
+}
+
+void dlb_resource_reset(struct dlb_hw *hw)
+{
+	struct dlb_domain *domain, *next __attribute__((unused));
+	struct dlb_list_entry *iter1 __attribute__((unused));
+	struct dlb_list_entry *iter2 __attribute__((unused));
+	int i;
+
+	for (i = 0; i < DLB_MAX_NUM_VFS; i++) {
+		DLB_FUNC_LIST_FOR_SAFE(hw->vf[i].used_domains, domain,
+				       next, iter1, iter2)
+			dlb_domain_reset_software_state(hw, domain);
+	}
+
+	DLB_FUNC_LIST_FOR_SAFE(hw->pf.used_domains, domain, next, iter1, iter2)
+		dlb_domain_reset_software_state(hw, domain);
+}
+
+static u32 dlb_dir_queue_depth(struct dlb_hw *hw,
+			       struct dlb_dir_pq_pair *queue)
+{
+	union dlb_lsp_qid_dir_enqueue_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_QID_DIR_ENQUEUE_CNT(queue->id.phys_id));
+
+	return r0.field.count;
+}
+
+static bool dlb_dir_queue_is_empty(struct dlb_hw *hw,
+				   struct dlb_dir_pq_pair *queue)
+{
+	return dlb_dir_queue_depth(hw, queue) == 0;
+}
+
+static void dlb_log_get_dir_queue_depth(struct dlb_hw *hw,
+					u32 domain_id,
+					u32 queue_id,
+					bool vf_request,
+					unsigned int vf_id)
+{
+	DLB_HW_INFO(hw, "DLB get directed queue depth:\n");
+	if (vf_request)
+		DLB_HW_INFO(hw, "(Request from VF %d)\n", vf_id);
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tQueue ID: %d\n", queue_id);
+}
+
+int dlb_hw_get_dir_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_dir_queue_depth_args *args,
+			       struct dlb_cmd_response *resp,
+			       bool vf_request,
+			       unsigned int vf_id)
+{
+	struct dlb_dir_pq_pair *queue;
+	struct dlb_domain *domain;
+	int id;
+
+	id = domain_id;
+
+	dlb_log_get_dir_queue_depth(hw, domain_id, args->queue_id,
+				    vf_request, vf_id);
+
+	domain = dlb_get_domain_from_id(hw, id, vf_request, vf_id);
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -EINVAL;
+	}
+
+	id = args->queue_id;
+
+	queue = dlb_get_domain_used_dir_pq(id, vf_request, domain);
+	if (!queue) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -EINVAL;
+	}
+
+	resp->id = dlb_dir_queue_depth(hw, queue);
+
+	return 0;
+}
+
+static void
+dlb_log_pending_port_unmaps_args(struct dlb_hw *hw,
+				 struct dlb_pending_port_unmaps_args *args,
+				 bool vf_request,
+				 unsigned int vf_id)
+{
+	DLB_HW_INFO(hw, "DLB pending port unmaps arguments:\n");
+	if (vf_request)
+		DLB_HW_INFO(hw, "(Request from VF %d)\n", vf_id);
+	DLB_HW_INFO(hw, "\tPort ID: %d\n", args->port_id);
+}
+
+int dlb_hw_pending_port_unmaps(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_pending_port_unmaps_args *args,
+			       struct dlb_cmd_response *resp,
+			       bool vf_request,
+			       unsigned int vf_id)
+{
+	struct dlb_domain *domain;
+	struct dlb_ldb_port *port;
+
+	dlb_log_pending_port_unmaps_args(hw, args, vf_request, vf_id);
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -EINVAL;
+	}
+
+	port = dlb_get_domain_used_ldb_port(args->port_id, vf_request, domain);
+	if (!port || !port->configured) {
+		resp->status = DLB_ST_INVALID_PORT_ID;
+		return -EINVAL;
+	}
+
+	resp->id = port->num_pending_removals;
+
+	return 0;
+}
+
+/* Returns whether the queue is empty, including its inflight and replay
+ * counts.
+ */
+static bool dlb_ldb_queue_is_empty(struct dlb_hw *hw,
+				   struct dlb_ldb_queue *queue)
+{
+	union dlb_lsp_qid_ldb_replay_cnt r0;
+	union dlb_lsp_qid_aqed_active_cnt r1;
+	union dlb_lsp_qid_atq_enqueue_cnt r2;
+	union dlb_lsp_qid_ldb_enqueue_cnt r3;
+	union dlb_lsp_qid_ldb_infl_cnt r4;
+
+	r0.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_REPLAY_CNT(queue->id.phys_id));
+	if (r0.val)
+		return false;
+
+	r1.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id.phys_id));
+	if (r1.val)
+		return false;
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_ATQ_ENQUEUE_CNT(queue->id.phys_id));
+	if (r2.val)
+		return false;
+
+	r3.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id.phys_id));
+	if (r3.val)
+		return false;
+
+	r4.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_INFL_CNT(queue->id.phys_id));
+	if (r4.val)
+		return false;
+
+	return true;
+}
+
+static void dlb_log_get_ldb_queue_depth(struct dlb_hw *hw,
+					u32 domain_id,
+					u32 queue_id,
+					bool vf_request,
+					unsigned int vf_id)
+{
+	DLB_HW_INFO(hw, "DLB get load-balanced queue depth:\n");
+	if (vf_request)
+		DLB_HW_INFO(hw, "(Request from VF %d)\n", vf_id);
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+	DLB_HW_INFO(hw, "\tQueue ID: %d\n", queue_id);
+}
+
+int dlb_hw_get_ldb_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_ldb_queue_depth_args *args,
+			       struct dlb_cmd_response *resp,
+			       bool vf_req,
+			       unsigned int vf_id)
+{
+	union dlb_lsp_qid_aqed_active_cnt r0;
+	union dlb_lsp_qid_atq_enqueue_cnt r1;
+	union dlb_lsp_qid_ldb_enqueue_cnt r2;
+	struct dlb_ldb_queue *queue;
+	struct dlb_domain *domain;
+
+	dlb_log_get_ldb_queue_depth(hw, domain_id, args->queue_id,
+				    vf_req, vf_id);
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_req, vf_id);
+	if (!domain) {
+		resp->status = DLB_ST_INVALID_DOMAIN_ID;
+		return -EINVAL;
+	}
+
+	queue = dlb_get_domain_ldb_queue(args->queue_id, vf_req, domain);
+	if (!queue) {
+		resp->status = DLB_ST_INVALID_QID;
+		return -EINVAL;
+	}
+
+	r0.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_AQED_ACTIVE_CNT(queue->id.phys_id));
+
+	r1.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_ATQ_ENQUEUE_CNT(queue->id.phys_id));
+
+	r2.val = DLB_CSR_RD(hw,
+			    DLB_LSP_QID_LDB_ENQUEUE_CNT(queue->id.phys_id));
+
+	resp->id = r0.val + r1.val + r2.val;
+
+	return 0;
+}
+
+static u32 dlb_dir_cq_token_count(struct dlb_hw *hw,
+				  struct dlb_dir_pq_pair *port)
+{
+	union dlb_lsp_cq_dir_tkn_cnt r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_LSP_CQ_DIR_TKN_CNT(port->id.phys_id));
+
+	return r0.field.count;
+}
+
+static int dlb_domain_verify_reset_success(struct dlb_hw *hw,
+					   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+	struct dlb_ldb_queue *queue;
+
+	/* Confirm that all credits are returned to the domain's credit pools */
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		union dlb_chp_dqed_fl_pop_ptr r0;
+		union dlb_chp_dqed_fl_push_ptr r1;
+
+		r0.val = DLB_CSR_RD(hw,
+				    DLB_CHP_DQED_FL_POP_PTR(pool->id.phys_id));
+
+		r1.val = DLB_CSR_RD(hw,
+				    DLB_CHP_DQED_FL_PUSH_PTR(pool->id.phys_id));
+
+		if (r0.field.pop_ptr != r1.field.push_ptr ||
+		    r0.field.generation == r1.field.generation) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to refill directed pool %d's credits.\n",
+				   __func__, pool->id.phys_id);
+			return -EFAULT;
+		}
+	}
+
+	/* Confirm that all the domain's queue's inflight counts and AQED
+	 * active counts are 0.
+	 */
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		if (!dlb_ldb_queue_is_empty(hw, queue)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty ldb queue %d\n",
+				   __func__, queue->id.phys_id);
+			return -EFAULT;
+		}
+	}
+
+	/* Confirm that all the domain's CQs inflight and token counts are 0. */
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, ldb_port, iter) {
+		if (dlb_ldb_cq_inflight_count(hw, ldb_port) ||
+		    dlb_ldb_cq_token_count(hw, ldb_port)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty ldb port %d\n",
+				   __func__, ldb_port->id.phys_id);
+			return -EFAULT;
+		}
+	}
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter) {
+		if (!dlb_dir_queue_is_empty(hw, dir_port)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty dir queue %d\n",
+				   __func__, dir_port->id.phys_id);
+			return -EFAULT;
+		}
+
+		if (dlb_dir_cq_token_count(hw, dir_port)) {
+			DLB_HW_ERR(hw,
+				   "[%s()] Internal error: failed to empty dir port %d\n",
+				   __func__, dir_port->id.phys_id);
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static void __dlb_domain_reset_ldb_port_registers(struct dlb_hw *hw,
+						  struct dlb_ldb_port *port)
+{
+	union dlb_chp_ldb_pp_state_reset r0 = { {0} };
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id.phys_id),
+		   DLB_CHP_LDB_PP_CRD_REQ_STATE_RST);
+
+	/* Reset the port's load-balanced and directed credit state */
+	r0.field.dir_type = 0;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_STATE_RESET(port->id.phys_id),
+		   r0.val);
+
+	r0.field.dir_type = 1;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_STATE_RESET(port->id.phys_id),
+		   r0.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_PUSH_PTR(port->id.phys_id),
+		   DLB_CHP_LDB_PP_DIR_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_PUSH_PTR(port->id.phys_id),
+		   DLB_CHP_LDB_PP_LDB_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT(port->id.phys_id),
+		   DLB_CHP_LDB_PP_LDB_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_CRD_LWM(port->id.phys_id),
+		   DLB_CHP_LDB_PP_LDB_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_LDB_CRD_HWM(port->id.phys_id),
+		   DLB_CHP_LDB_PP_LDB_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_LDB_PP2POOL(port->id.phys_id),
+		   DLB_CHP_LDB_LDB_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT(port->id.phys_id),
+		   DLB_CHP_LDB_PP_DIR_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_CRD_LWM(port->id.phys_id),
+		   DLB_CHP_LDB_PP_DIR_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_PP_DIR_CRD_HWM(port->id.phys_id),
+		   DLB_CHP_LDB_PP_DIR_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_DIR_PP2POOL(port->id.phys_id),
+		   DLB_CHP_LDB_DIR_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2LDBPOOL(port->id.phys_id),
+		   DLB_SYS_LDB_PP2LDBPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2DIRPOOL(port->id.phys_id),
+		   DLB_SYS_LDB_PP2DIRPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_LIM(port->id.phys_id),
+		   DLB_CHP_HIST_LIST_LIM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_BASE(port->id.phys_id),
+		   DLB_CHP_HIST_LIST_BASE_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_POP_PTR(port->id.phys_id),
+		   DLB_CHP_HIST_LIST_POP_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_HIST_LIST_PUSH_PTR(port->id.phys_id),
+		   DLB_CHP_HIST_LIST_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_WPTR(port->id.phys_id),
+		   DLB_CHP_LDB_CQ_WPTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_INT_DEPTH_THRSH(port->id.phys_id),
+		   DLB_CHP_LDB_CQ_INT_DEPTH_THRSH_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_TMR_THRESHOLD(port->id.phys_id),
+		   DLB_CHP_LDB_CQ_TMR_THRESHOLD_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_INT_ENB(port->id.phys_id),
+		   DLB_CHP_LDB_CQ_INT_ENB_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_INFL_LIM(port->id.phys_id),
+		   DLB_LSP_CQ_LDB_INFL_LIM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ2PRIOV(port->id.phys_id),
+		   DLB_LSP_CQ2PRIOV_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL(port->id.phys_id),
+		   DLB_LSP_CQ_LDB_TOT_SCH_CNT_CTRL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL(port->id.phys_id),
+		   DLB_LSP_CQ_LDB_TKN_DEPTH_SEL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL(port->id.phys_id),
+		   DLB_CHP_LDB_CQ_TKN_DEPTH_SEL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_LDB_DSBL(port->id.phys_id),
+		   DLB_LSP_CQ_LDB_DSBL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ2VF_PF(port->id.phys_id),
+		   DLB_SYS_LDB_CQ2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2VF_PF(port->id.phys_id),
+		   DLB_SYS_LDB_PP2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_L(port->id.phys_id),
+		   DLB_SYS_LDB_CQ_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ADDR_U(port->id.phys_id),
+		   DLB_SYS_LDB_CQ_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_ADDR_L(port->id.phys_id),
+		   DLB_SYS_LDB_PP_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_ADDR_U(port->id.phys_id),
+		   DLB_SYS_LDB_PP_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP_V(port->id.phys_id),
+		   DLB_SYS_LDB_PP_V_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_PP2VAS(port->id.phys_id),
+		   DLB_SYS_LDB_PP2VAS_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_LDB_CQ_ISR(port->id.phys_id),
+		   DLB_SYS_LDB_CQ_ISR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_WBUF_LDB_FLAGS(port->id.phys_id),
+		   DLB_SYS_WBUF_LDB_FLAGS_RST);
+}
+
+static void dlb_domain_reset_ldb_port_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		__dlb_domain_reset_ldb_port_registers(hw, port);
+}
+
+static void __dlb_domain_reset_dir_port_registers(struct dlb_hw *hw,
+						  struct dlb_dir_pq_pair *port)
+{
+	union dlb_chp_dir_pp_state_reset r0 = { {0} };
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id.phys_id),
+		   DLB_CHP_DIR_PP_CRD_REQ_STATE_RST);
+
+	/* Reset the port's load-balanced and directed credit state */
+	r0.field.dir_type = 0;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_STATE_RESET(port->id.phys_id),
+		   r0.val);
+
+	r0.field.dir_type = 1;
+	r0.field.reset_pp_state = 1;
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_STATE_RESET(port->id.phys_id),
+		   r0.val);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_PUSH_PTR(port->id.phys_id),
+		   DLB_CHP_DIR_PP_DIR_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_PUSH_PTR(port->id.phys_id),
+		   DLB_CHP_DIR_PP_LDB_PUSH_PTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT(port->id.phys_id),
+		   DLB_CHP_DIR_PP_LDB_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_LWM(port->id.phys_id),
+		   DLB_CHP_DIR_PP_LDB_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_LDB_CRD_HWM(port->id.phys_id),
+		   DLB_CHP_DIR_PP_LDB_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_LDB_PP2POOL(port->id.phys_id),
+		   DLB_CHP_DIR_LDB_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT(port->id.phys_id),
+		   DLB_CHP_DIR_PP_DIR_MIN_CRD_QNT_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_LWM(port->id.phys_id),
+		   DLB_CHP_DIR_PP_DIR_CRD_LWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_PP_DIR_CRD_HWM(port->id.phys_id),
+		   DLB_CHP_DIR_PP_DIR_CRD_HWM_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_DIR_PP2POOL(port->id.phys_id),
+		   DLB_CHP_DIR_DIR_PP2POOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2LDBPOOL(port->id.phys_id),
+		   DLB_SYS_DIR_PP2LDBPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2DIRPOOL(port->id.phys_id),
+		   DLB_SYS_DIR_PP2DIRPOOL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_WPTR(port->id.phys_id),
+		   DLB_CHP_DIR_CQ_WPTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(port->id.phys_id),
+		   DLB_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL(port->id.phys_id),
+		   DLB_CHP_DIR_CQ_TKN_DEPTH_SEL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_LSP_CQ_DIR_DSBL(port->id.phys_id),
+		   DLB_LSP_CQ_DIR_DSBL_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_WPTR(port->id.phys_id),
+		   DLB_CHP_DIR_CQ_WPTR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_INT_DEPTH_THRSH(port->id.phys_id),
+		   DLB_CHP_DIR_CQ_INT_DEPTH_THRSH_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_TMR_THRESHOLD(port->id.phys_id),
+		   DLB_CHP_DIR_CQ_TMR_THRESHOLD_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_CHP_DIR_CQ_INT_ENB(port->id.phys_id),
+		   DLB_CHP_DIR_CQ_INT_ENB_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ2VF_PF(port->id.phys_id),
+		   DLB_SYS_DIR_CQ2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VF_PF(port->id.phys_id),
+		   DLB_SYS_DIR_PP2VF_PF_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ_ADDR_L(port->id.phys_id),
+		   DLB_SYS_DIR_CQ_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ_ADDR_U(port->id.phys_id),
+		   DLB_SYS_DIR_CQ_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP_ADDR_L(port->id.phys_id),
+		   DLB_SYS_DIR_PP_ADDR_L_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP_ADDR_U(port->id.phys_id),
+		   DLB_SYS_DIR_PP_ADDR_U_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP_V(port->id.phys_id),
+		   DLB_SYS_DIR_PP_V_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_PP2VAS(port->id.phys_id),
+		   DLB_SYS_DIR_PP2VAS_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_DIR_CQ_ISR(port->id.phys_id),
+		   DLB_SYS_DIR_CQ_ISR_RST);
+
+	DLB_CSR_WR(hw,
+		   DLB_SYS_WBUF_DIR_FLAGS(port->id.phys_id),
+		   DLB_SYS_WBUF_DIR_FLAGS_RST);
+}
+
+static void dlb_domain_reset_dir_port_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_dir_pq_pair *port;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		__dlb_domain_reset_dir_port_registers(hw, port);
+}
+
+static void dlb_domain_reset_ldb_queue_registers(struct dlb_hw *hw,
+						 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_ldb_queue *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_LIM(queue->id.phys_id),
+			   DLB_AQED_PIPE_FL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_BASE(queue->id.phys_id),
+			   DLB_AQED_PIPE_FL_BASE_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_POP_PTR(queue->id.phys_id),
+			   DLB_AQED_PIPE_FL_POP_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_FL_PUSH_PTR(queue->id.phys_id),
+			   DLB_AQED_PIPE_FL_PUSH_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_AQED_PIPE_QID_FID_LIM(queue->id.phys_id),
+			   DLB_AQED_PIPE_QID_FID_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_LSP_QID_AQED_ACTIVE_LIM(queue->id.phys_id),
+			   DLB_LSP_QID_AQED_ACTIVE_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_LSP_QID_LDB_INFL_LIM(queue->id.phys_id),
+			   DLB_LSP_QID_LDB_INFL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_QID_V(queue->id.phys_id),
+			   DLB_SYS_LDB_QID_V_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_QID_V(queue->id.phys_id),
+			   DLB_SYS_LDB_QID_V_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_ORD_QID_SN(queue->id.phys_id),
+			   DLB_CHP_ORD_QID_SN_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_ORD_QID_SN_MAP(queue->id.phys_id),
+			   DLB_CHP_ORD_QID_SN_MAP_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_RO_PIPE_QID2GRPSLT(queue->id.phys_id),
+			   DLB_RO_PIPE_QID2GRPSLT_RST);
+	}
+}
+
+static void dlb_domain_reset_dir_queue_registers(struct dlb_hw *hw,
+						 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_dir_pq_pair *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, queue, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_QID_V(queue->id.phys_id),
+			   DLB_SYS_DIR_QID_V_RST);
+	}
+}
+
+static void dlb_domain_reset_ldb_pool_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_POOL_CRD_LIM(pool->id.phys_id),
+			   DLB_CHP_LDB_POOL_CRD_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_POOL_CRD_CNT(pool->id.phys_id),
+			   DLB_CHP_LDB_POOL_CRD_CNT_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_BASE(pool->id.phys_id),
+			   DLB_CHP_QED_FL_BASE_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_LIM(pool->id.phys_id),
+			   DLB_CHP_QED_FL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_PUSH_PTR(pool->id.phys_id),
+			   DLB_CHP_QED_FL_PUSH_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_QED_FL_POP_PTR(pool->id.phys_id),
+			   DLB_CHP_QED_FL_POP_PTR_RST);
+	}
+}
+
+static void dlb_domain_reset_dir_pool_registers(struct dlb_hw *hw,
+						struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_POOL_CRD_LIM(pool->id.phys_id),
+			   DLB_CHP_DIR_POOL_CRD_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_POOL_CRD_CNT(pool->id.phys_id),
+			   DLB_CHP_DIR_POOL_CRD_CNT_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_BASE(pool->id.phys_id),
+			   DLB_CHP_DQED_FL_BASE_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_LIM(pool->id.phys_id),
+			   DLB_CHP_DQED_FL_LIM_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_PUSH_PTR(pool->id.phys_id),
+			   DLB_CHP_DQED_FL_PUSH_PTR_RST);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DQED_FL_POP_PTR(pool->id.phys_id),
+			   DLB_CHP_DQED_FL_POP_PTR_RST);
+	}
+}
+
+static void dlb_domain_reset_registers(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	dlb_domain_reset_ldb_port_registers(hw, domain);
+
+	dlb_domain_reset_dir_port_registers(hw, domain);
+
+	dlb_domain_reset_ldb_queue_registers(hw, domain);
+
+	dlb_domain_reset_dir_queue_registers(hw, domain);
+
+	dlb_domain_reset_ldb_pool_registers(hw, domain);
+
+	dlb_domain_reset_dir_pool_registers(hw, domain);
+}
+
+static int dlb_domain_drain_ldb_cqs(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    bool toggle_port)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_ldb_port *port;
+	int ret;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		if (toggle_port)
+			dlb_ldb_port_cq_disable(hw, port);
+
+		ret = dlb_drain_ldb_cq(hw, port);
+		if (ret < 0)
+			return ret;
+
+		if (toggle_port)
+			dlb_ldb_port_cq_enable(hw, port);
+	}
+
+	return 0;
+}
+
+static bool dlb_domain_mapped_queues_empty(struct dlb_hw *hw,
+					   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_ldb_queue *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		if (queue->num_mappings == 0)
+			continue;
+
+		if (!dlb_ldb_queue_is_empty(hw, queue))
+			return false;
+	}
+
+	return true;
+}
+
+static int dlb_domain_drain_mapped_queues(struct dlb_hw *hw,
+					  struct dlb_domain *domain)
+{
+	int i, ret;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	if (domain->num_pending_removals > 0) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to unmap domain queues\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+		ret = dlb_domain_drain_ldb_cqs(hw, domain, true);
+		if (ret < 0)
+			return ret;
+
+		if (dlb_domain_mapped_queues_empty(hw, domain))
+			break;
+	}
+
+	if (i == DLB_MAX_QID_EMPTY_CHECK_LOOPS) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to empty queues\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	/* Drain the CQs one more time. For the queues to go empty, they would
+	 * have scheduled one or more QEs.
+	 */
+	ret = dlb_domain_drain_ldb_cqs(hw, domain, true);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int dlb_domain_drain_unmapped_queue(struct dlb_hw *hw,
+					   struct dlb_domain *domain,
+					   struct dlb_ldb_queue *queue)
+{
+	struct dlb_ldb_port *port;
+	int ret;
+
+	/* If a domain has LDB queues, it must have LDB ports */
+	if (dlb_list_empty(&domain->used_ldb_ports)) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: No configured LDB ports\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	port = DLB_DOM_LIST_HEAD(domain->used_ldb_ports, typeof(*port));
+
+	/* If necessary, free up a QID slot in this CQ */
+	if (port->num_mappings == DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		struct dlb_ldb_queue *mapped_queue;
+
+		mapped_queue = &hw->rsrcs.ldb_queues[port->qid_map[0].qid];
+
+		ret = dlb_ldb_port_unmap_qid(hw, port, mapped_queue);
+		if (ret)
+			return ret;
+	}
+
+	ret = dlb_ldb_port_map_qid_dynamic(hw, port, queue, 0);
+	if (ret)
+		return ret;
+
+	return dlb_domain_drain_mapped_queues(hw, domain);
+}
+
+static int dlb_domain_drain_unmapped_queues(struct dlb_hw *hw,
+					    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_ldb_queue *queue;
+	int ret;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		if (queue->num_mappings != 0 ||
+		    dlb_ldb_queue_is_empty(hw, queue))
+			continue;
+
+		ret = dlb_domain_drain_unmapped_queue(hw, domain, queue);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static void dlb_drain_dir_cq(struct dlb_hw *hw, struct dlb_dir_pq_pair *port)
+{
+	unsigned int port_id = port->id.phys_id;
+	u32 cnt;
+
+	/* Return any outstanding tokens */
+	cnt = dlb_dir_cq_token_count(hw, port);
+
+	if (cnt != 0) {
+		struct dlb_hcw hcw_mem[8], *hcw;
+		void  *pp_addr;
+
+		pp_addr = os_map_producer_port(hw, port_id, false);
+
+		/* Point hcw to a 64B-aligned location */
+		hcw = (struct dlb_hcw *)((uintptr_t)&hcw_mem[4] & ~0x3F);
+
+		/* Program the first HCW for a batch token return and
+		 * the rest as NOOPS
+		 */
+		memset(hcw, 0, 4 * sizeof(*hcw));
+		hcw->cq_token = 1;
+		hcw->lock_id = cnt - 1;
+
+		os_enqueue_four_hcws(hw, hcw, pp_addr);
+
+		os_fence_hcw(hw, pp_addr);
+
+		os_unmap_producer_port(hw, pp_addr);
+	}
+}
+
+static int dlb_domain_drain_dir_cqs(struct dlb_hw *hw,
+				    struct dlb_domain *domain,
+				    bool toggle_port)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_dir_pq_pair *port;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		/* Can't drain a port if it's not configured, and there's
+		 * nothing to drain if its queue is unconfigured.
+		 */
+		if (!port->port_configured || !port->queue_configured)
+			continue;
+
+		if (toggle_port)
+			dlb_dir_port_cq_disable(hw, port);
+
+		dlb_drain_dir_cq(hw, port);
+
+		if (toggle_port)
+			dlb_dir_port_cq_enable(hw, port);
+	}
+
+	return 0;
+}
+
+static bool dlb_domain_dir_queues_empty(struct dlb_hw *hw,
+					struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_dir_pq_pair *queue;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, queue, iter) {
+		if (!dlb_dir_queue_is_empty(hw, queue))
+			return false;
+	}
+
+	return true;
+}
+
+static int dlb_domain_drain_dir_queues(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	int i;
+
+	/* If the domain hasn't been started, there's no traffic to drain */
+	if (!domain->started)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+		dlb_domain_drain_dir_cqs(hw, domain, true);
+
+		if (dlb_domain_dir_queues_empty(hw, domain))
+			break;
+	}
+
+	if (i == DLB_MAX_QID_EMPTY_CHECK_LOOPS) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: failed to empty queues\n",
+			   __func__);
+		return -EFAULT;
+	}
+
+	/* Drain the CQs one more time. For the queues to go empty, they would
+	 * have scheduled one or more QEs.
+	 */
+	dlb_domain_drain_dir_cqs(hw, domain, true);
+
+	return 0;
+}
+
+static void dlb_domain_disable_dir_producer_ports(struct dlb_hw *hw,
+						  struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_dir_pq_pair *port;
+	union dlb_sys_dir_pp_v r1;
+
+	r1.field.pp_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_PP_V(port->id.phys_id),
+			   r1.val);
+}
+
+static void dlb_domain_disable_ldb_producer_ports(struct dlb_hw *hw,
+						  struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	union dlb_sys_ldb_pp_v r1;
+	struct dlb_ldb_port *port;
+
+	r1.field.pp_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_PP_V(port->id.phys_id),
+			   r1.val);
+
+		hw->pf.num_enabled_ldb_ports--;
+	}
+}
+
+static void dlb_domain_disable_dir_vpps(struct dlb_hw *hw,
+					struct dlb_domain *domain,
+					unsigned int vf_id)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	union dlb_sys_vf_dir_vpp_v r1;
+	struct dlb_dir_pq_pair *port;
+
+	r1.field.vpp_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		unsigned int offs;
+
+		offs = vf_id * DLB_MAX_NUM_DIR_PORTS + port->id.virt_id;
+
+		DLB_CSR_WR(hw, DLB_SYS_VF_DIR_VPP_V(offs), r1.val);
+	}
+}
+
+static void dlb_domain_disable_ldb_vpps(struct dlb_hw *hw,
+					struct dlb_domain *domain,
+					unsigned int vf_id)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	union dlb_sys_vf_ldb_vpp_v r1;
+	struct dlb_ldb_port *port;
+
+	r1.field.vpp_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		unsigned int offs;
+
+		offs = vf_id * DLB_MAX_NUM_LDB_PORTS + port->id.virt_id;
+
+		DLB_CSR_WR(hw, DLB_SYS_VF_LDB_VPP_V(offs), r1.val);
+	}
+}
+
+static void dlb_domain_disable_dir_pools(struct dlb_hw *hw,
+					 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	union dlb_sys_dir_pool_enbld r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter)
+		DLB_CSR_WR(hw,
+			   DLB_SYS_DIR_POOL_ENBLD(pool->id.phys_id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_ldb_pools(struct dlb_hw *hw,
+					 struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	union dlb_sys_ldb_pool_enbld r0 = { {0} };
+	struct dlb_credit_pool *pool;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter)
+		DLB_CSR_WR(hw,
+			   DLB_SYS_LDB_POOL_ENBLD(pool->id.phys_id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_ldb_seq_checks(struct dlb_hw *hw,
+					      struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	union dlb_chp_sn_chk_enbl r1;
+	struct dlb_ldb_port *port;
+
+	r1.field.en = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_CHP_SN_CHK_ENBL(port->id.phys_id),
+			   r1.val);
+}
+
+static void dlb_domain_disable_ldb_port_crd_updates(struct dlb_hw *hw,
+						    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	union dlb_chp_ldb_pp_crd_req_state r0;
+	struct dlb_ldb_port *port;
+
+	r0.field.no_pp_credit_update = 1;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_PP_CRD_REQ_STATE(port->id.phys_id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_ldb_port_interrupts(struct dlb_hw *hw,
+						   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	union dlb_chp_ldb_cq_int_enb r0 = { {0} };
+	union dlb_chp_ldb_cq_wd_enb r1 = { {0} };
+	struct dlb_ldb_port *port;
+
+	r0.field.en_tim = 0;
+	r0.field.en_depth = 0;
+
+	r1.field.wd_enable = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_CQ_INT_ENB(port->id.phys_id),
+			   r0.val);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_LDB_CQ_WD_ENB(port->id.phys_id),
+			   r1.val);
+	}
+}
+
+static void dlb_domain_disable_dir_port_interrupts(struct dlb_hw *hw,
+						   struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	union dlb_chp_dir_cq_int_enb r0 = { {0} };
+	union dlb_chp_dir_cq_wd_enb r1 = { {0} };
+	struct dlb_dir_pq_pair *port;
+
+	r0.field.en_tim = 0;
+	r0.field.en_depth = 0;
+
+	r1.field.wd_enable = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_CQ_INT_ENB(port->id.phys_id),
+			   r0.val);
+
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_CQ_WD_ENB(port->id.phys_id),
+			   r1.val);
+	}
+}
+
+static void dlb_domain_disable_dir_port_crd_updates(struct dlb_hw *hw,
+						    struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	union dlb_chp_dir_pp_crd_req_state r0;
+	struct dlb_dir_pq_pair *port;
+
+	r0.field.no_pp_credit_update = 1;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+		DLB_CSR_WR(hw,
+			   DLB_CHP_DIR_PP_CRD_REQ_STATE(port->id.phys_id),
+			   r0.val);
+}
+
+static void dlb_domain_disable_ldb_queue_write_perms(struct dlb_hw *hw,
+						     struct dlb_domain *domain)
+{
+	int domain_offset = domain->id.phys_id * DLB_MAX_NUM_LDB_QUEUES;
+	struct dlb_list_entry *iter __attribute__((unused));
+	union dlb_sys_ldb_vasqid_v r0;
+	struct dlb_ldb_queue *queue;
+
+	r0.field.vasqid_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter) {
+		int idx = domain_offset + queue->id.phys_id;
+
+		DLB_CSR_WR(hw, DLB_SYS_LDB_VASQID_V(idx), r0.val);
+	}
+}
+
+static void dlb_domain_disable_dir_queue_write_perms(struct dlb_hw *hw,
+						     struct dlb_domain *domain)
+{
+	int domain_offset = domain->id.phys_id * DLB_MAX_NUM_DIR_PORTS;
+	struct dlb_list_entry *iter __attribute__((unused));
+	union dlb_sys_dir_vasqid_v r0;
+	struct dlb_dir_pq_pair *port;
+
+	r0.field.vasqid_v = 0;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		int idx = domain_offset + port->id.phys_id;
+
+		DLB_CSR_WR(hw, DLB_SYS_DIR_VASQID_V(idx), r0.val);
+	}
+}
+
+static void dlb_domain_disable_dir_cqs(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_dir_pq_pair *port;
+
+	DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) {
+		port->enabled = false;
+
+		dlb_dir_port_cq_disable(hw, port);
+	}
+}
+
+static void dlb_domain_disable_ldb_cqs(struct dlb_hw *hw,
+				       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		port->enabled = false;
+
+		dlb_ldb_port_cq_disable(hw, port);
+	}
+}
+
+static void dlb_domain_enable_ldb_cqs(struct dlb_hw *hw,
+				      struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_ldb_port *port;
+
+	DLB_DOM_LIST_FOR(domain->used_ldb_ports, port, iter) {
+		port->enabled = true;
+
+		dlb_ldb_port_cq_enable(hw, port);
+	}
+}
+
+static int dlb_domain_wait_for_ldb_pool_refill(struct dlb_hw *hw,
+					       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_credit_pool *pool;
+
+	/* Confirm that all credits are returned to the domain's credit pools */
+	DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter) {
+		union dlb_chp_qed_fl_push_ptr r0;
+		union dlb_chp_qed_fl_pop_ptr r1;
+		unsigned long pop_offs, push_offs;
+		int i;
+
+		push_offs = DLB_CHP_QED_FL_PUSH_PTR(pool->id.phys_id);
+		pop_offs = DLB_CHP_QED_FL_POP_PTR(pool->id.phys_id);
+
+		for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+			r0.val = DLB_CSR_RD(hw, push_offs);
+
+			r1.val = DLB_CSR_RD(hw, pop_offs);
+
+			/* Break early if the freelist is replenished */
+			if (r1.field.pop_ptr == r0.field.push_ptr &&
+			    r1.field.generation != r0.field.generation) {
+				break;
+			}
+		}
+
+		/* Error if the freelist is not full */
+		if (r1.field.pop_ptr != r0.field.push_ptr ||
+		    r1.field.generation == r0.field.generation) {
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static int dlb_domain_wait_for_dir_pool_refill(struct dlb_hw *hw,
+					       struct dlb_domain *domain)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_credit_pool *pool;
+
+	/* Confirm that all credits are returned to the domain's credit pools */
+	DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter) {
+		union dlb_chp_dqed_fl_push_ptr r0;
+		union dlb_chp_dqed_fl_pop_ptr r1;
+		unsigned long pop_offs, push_offs;
+		int i;
+
+		push_offs = DLB_CHP_DQED_FL_PUSH_PTR(pool->id.phys_id);
+		pop_offs = DLB_CHP_DQED_FL_POP_PTR(pool->id.phys_id);
+
+		for (i = 0; i < DLB_MAX_QID_EMPTY_CHECK_LOOPS; i++) {
+			r0.val = DLB_CSR_RD(hw, push_offs);
+
+			r1.val = DLB_CSR_RD(hw, pop_offs);
+
+			/* Break early if the freelist is replenished */
+			if (r1.field.pop_ptr == r0.field.push_ptr &&
+			    r1.field.generation != r0.field.generation) {
+				break;
+			}
+		}
+
+		/* Error if the freelist is not full */
+		if (r1.field.pop_ptr != r0.field.push_ptr ||
+		    r1.field.generation == r0.field.generation) {
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static void dlb_log_reset_domain(struct dlb_hw *hw,
+				 u32 domain_id,
+				 bool vf_request,
+				 unsigned int vf_id)
+{
+	DLB_HW_INFO(hw, "DLB reset domain:\n");
+	if (vf_request)
+		DLB_HW_INFO(hw, "(Request from VF %d)\n", vf_id);
+	DLB_HW_INFO(hw, "\tDomain ID: %d\n", domain_id);
+}
+
+/**
+ * dlb_reset_domain() - Reset a DLB scheduling domain and its associated
+ *	hardware resources.
+ * @hw:	  Contains the current state of the DLB hardware.
+ * @args: User-provided arguments.
+ * @resp: Response to user.
+ *
+ * Note: User software *must* stop sending to this domain's producer ports
+ * before invoking this function, otherwise undefined behavior will result.
+ *
+ * Return: returns < 0 on error, 0 otherwise.
+ */
+int dlb_reset_domain(struct dlb_hw *hw,
+		     u32 domain_id,
+		     bool vf_request,
+		     unsigned int vf_id)
+{
+	struct dlb_domain *domain;
+	int ret;
+
+	dlb_log_reset_domain(hw, domain_id, vf_request, vf_id);
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+
+	if (!domain || !domain->configured)
+		return -EINVAL;
+
+	if (vf_request) {
+		dlb_domain_disable_dir_vpps(hw, domain, vf_id);
+
+		dlb_domain_disable_ldb_vpps(hw, domain, vf_id);
+	}
+
+	/* For each queue owned by this domain, disable its write permissions to
+	 * cause any traffic sent to it to be dropped. Well-behaved software
+	 * should not be sending QEs at this point.
+	 */
+	dlb_domain_disable_dir_queue_write_perms(hw, domain);
+
+	dlb_domain_disable_ldb_queue_write_perms(hw, domain);
+
+	/* Disable credit updates and turn off completion tracking on all the
+	 * domain's PPs.
+	 */
+	dlb_domain_disable_dir_port_crd_updates(hw, domain);
+
+	dlb_domain_disable_ldb_port_crd_updates(hw, domain);
+
+	dlb_domain_disable_dir_port_interrupts(hw, domain);
+
+	dlb_domain_disable_ldb_port_interrupts(hw, domain);
+
+	dlb_domain_disable_ldb_seq_checks(hw, domain);
+
+	/* Disable the LDB CQs and drain them in order to complete the map and
+	 * unmap procedures, which require zero CQ inflights and zero QID
+	 * inflights respectively.
+	 */
+	dlb_domain_disable_ldb_cqs(hw, domain);
+
+	ret = dlb_domain_drain_ldb_cqs(hw, domain, false);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_wait_for_ldb_cqs_to_empty(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_finish_unmap_qid_procedures(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_finish_map_qid_procedures(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	/* Re-enable the CQs in order to drain the mapped queues. */
+	dlb_domain_enable_ldb_cqs(hw, domain);
+
+	ret = dlb_domain_drain_mapped_queues(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_drain_unmapped_queues(hw, domain);
+	if (ret < 0)
+		return ret;
+
+	ret = dlb_domain_wait_for_ldb_pool_refill(hw, domain);
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: LDB credits failed to refill\n",
+			   __func__);
+		return ret;
+	}
+
+	/* Done draining LDB QEs, so disable the CQs. */
+	dlb_domain_disable_ldb_cqs(hw, domain);
+
+	/* Directed queues are reset in dlb_domain_reset_hw_resources(), but
+	 * that process doesn't decrement the directed queue size counters used
+	 * by SMON for its average DQED depth measurement. So, we manually drain
+	 * the directed queues here.
+	 */
+	dlb_domain_drain_dir_queues(hw, domain);
+
+	ret = dlb_domain_wait_for_dir_pool_refill(hw, domain);
+	if (ret) {
+		DLB_HW_ERR(hw,
+			   "[%s()] Internal error: DIR credits failed to refill\n",
+			   __func__);
+		return ret;
+	}
+
+	/* Done draining DIR QEs, so disable the CQs. */
+	dlb_domain_disable_dir_cqs(hw, domain);
+
+	dlb_domain_disable_dir_producer_ports(hw, domain);
+
+	dlb_domain_disable_ldb_producer_ports(hw, domain);
+
+	dlb_domain_disable_dir_pools(hw, domain);
+
+	dlb_domain_disable_ldb_pools(hw, domain);
+
+	/* Reset the QID, credit pool, and CQ hardware.
+	 *
+	 * Note: DLB 1.0 A0 h/w does not disarm CQ interrupts during VAS reset.
+	 * A spurious interrupt can occur on subsequent use of a reset CQ.
+	 */
+	ret = dlb_domain_reset_hw_resources(hw, domain);
+	if (ret)
+		return ret;
+
+	ret = dlb_domain_verify_reset_success(hw, domain);
+	if (ret)
+		return ret;
+
+	dlb_domain_reset_registers(hw, domain);
+
+	/* Hardware reset complete. Reset the domain's software state */
+	ret = dlb_domain_reset_software_state(hw, domain);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+int dlb_reset_vf(struct dlb_hw *hw, unsigned int vf_id)
+{
+	struct dlb_domain *domain, *next __attribute__((unused));
+	struct dlb_list_entry *it1 __attribute__((unused));
+	struct dlb_list_entry *it2 __attribute__((unused));
+	struct dlb_function_resources *rsrcs;
+
+	if (vf_id >= DLB_MAX_NUM_VFS) {
+		DLB_HW_ERR(hw, "[%s()] Internal error: invalid VF ID %d\n",
+			   __func__, vf_id);
+		return -EFAULT;
+	}
+
+	rsrcs = &hw->vf[vf_id];
+
+	DLB_FUNC_LIST_FOR_SAFE(rsrcs->used_domains, domain, next, it1, it2) {
+		int ret = dlb_reset_domain(hw,
+					   domain->id.virt_id,
+					   true,
+					   vf_id);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+int dlb_ldb_port_owned_by_domain(struct dlb_hw *hw,
+				 u32 domain_id,
+				 u32 port_id,
+				 bool vf_request,
+				 unsigned int vf_id)
+{
+	struct dlb_ldb_port *port;
+	struct dlb_domain *domain;
+
+	if (vf_request && vf_id >= DLB_MAX_NUM_VFS)
+		return -1;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+
+	if (!domain || !domain->configured)
+		return -EINVAL;
+
+	port = dlb_get_domain_ldb_port(port_id, vf_request, domain);
+
+	if (!port)
+		return -EINVAL;
+
+	return port->domain_id.phys_id == domain->id.phys_id;
+}
+
+int dlb_dir_port_owned_by_domain(struct dlb_hw *hw,
+				 u32 domain_id,
+				 u32 port_id,
+				 bool vf_request,
+				 unsigned int vf_id)
+{
+	struct dlb_dir_pq_pair *port;
+	struct dlb_domain *domain;
+
+	if (vf_request && vf_id >= DLB_MAX_NUM_VFS)
+		return -1;
+
+	domain = dlb_get_domain_from_id(hw, domain_id, vf_request, vf_id);
+
+	if (!domain || !domain->configured)
+		return -EINVAL;
+
+	port = dlb_get_domain_dir_pq(port_id, vf_request, domain);
+
+	if (!port)
+		return -EINVAL;
+
+	return port->domain_id.phys_id == domain->id.phys_id;
+}
+
+int dlb_hw_get_num_resources(struct dlb_hw *hw,
+			     struct dlb_get_num_resources_args *arg,
+			     bool vf_request,
+			     unsigned int vf_id)
+{
+	struct dlb_function_resources *rsrcs;
+	struct dlb_bitmap *map;
+
+	if (vf_request && vf_id >= DLB_MAX_NUM_VFS)
+		return -1;
+
+	if (vf_request)
+		rsrcs = &hw->vf[vf_id];
+	else
+		rsrcs = &hw->pf;
+
+	arg->num_sched_domains = rsrcs->num_avail_domains;
+
+	arg->num_ldb_queues = rsrcs->num_avail_ldb_queues;
+
+	arg->num_ldb_ports = rsrcs->num_avail_ldb_ports;
+
+	arg->num_dir_ports = rsrcs->num_avail_dir_pq_pairs;
+
+	map = rsrcs->avail_aqed_freelist_entries;
+
+	arg->num_atomic_inflights = dlb_bitmap_count(map);
+
+	arg->max_contiguous_atomic_inflights =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_hist_list_entries;
+
+	arg->num_hist_list_entries = dlb_bitmap_count(map);
+
+	arg->max_contiguous_hist_list_entries =
+		dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_qed_freelist_entries;
+
+	arg->num_ldb_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_ldb_credits = dlb_bitmap_longest_set_range(map);
+
+	map = rsrcs->avail_dqed_freelist_entries;
+
+	arg->num_dir_credits = dlb_bitmap_count(map);
+
+	arg->max_contiguous_dir_credits = dlb_bitmap_longest_set_range(map);
+
+	arg->num_ldb_credit_pools = rsrcs->num_avail_ldb_credit_pools;
+
+	arg->num_dir_credit_pools = rsrcs->num_avail_dir_credit_pools;
+
+	return 0;
+}
+
+int dlb_hw_get_num_used_resources(struct dlb_hw *hw,
+				  struct dlb_get_num_resources_args *arg,
+				  bool vf_request,
+				  unsigned int vf_id)
+{
+	struct dlb_list_entry *iter1 __attribute__((unused));
+	struct dlb_list_entry *iter2 __attribute__((unused));
+	struct dlb_function_resources *rsrcs;
+	struct dlb_domain *domain;
+
+	if (vf_request && vf_id >= DLB_MAX_NUM_VFS)
+		return -1;
+
+	rsrcs = (vf_request) ? &hw->vf[vf_id] : &hw->pf;
+
+	memset(arg, 0, sizeof(*arg));
+
+	DLB_FUNC_LIST_FOR(rsrcs->used_domains, domain, iter1) {
+		struct dlb_dir_pq_pair *dir_port;
+		struct dlb_ldb_port *ldb_port;
+		struct dlb_credit_pool *pool;
+		struct dlb_ldb_queue *queue;
+
+		arg->num_sched_domains++;
+
+		arg->num_atomic_inflights +=
+			domain->aqed_freelist.bound -
+			domain->aqed_freelist.base;
+
+		DLB_DOM_LIST_FOR(domain->used_ldb_queues, queue, iter2)
+			arg->num_ldb_queues++;
+		DLB_DOM_LIST_FOR(domain->avail_ldb_queues, queue, iter2)
+			arg->num_ldb_queues++;
+
+		DLB_DOM_LIST_FOR(domain->used_ldb_ports, ldb_port, iter2)
+			arg->num_ldb_ports++;
+		DLB_DOM_LIST_FOR(domain->avail_ldb_ports, ldb_port, iter2)
+			arg->num_ldb_ports++;
+
+		DLB_DOM_LIST_FOR(domain->used_dir_pq_pairs, dir_port, iter2)
+			arg->num_dir_ports++;
+		DLB_DOM_LIST_FOR(domain->avail_dir_pq_pairs, dir_port, iter2)
+			arg->num_dir_ports++;
+
+		arg->num_ldb_credits +=
+			domain->qed_freelist.bound -
+			domain->qed_freelist.base;
+
+		DLB_DOM_LIST_FOR(domain->avail_ldb_credit_pools, pool, iter2)
+			arg->num_ldb_credit_pools++;
+		DLB_DOM_LIST_FOR(domain->used_ldb_credit_pools, pool, iter2) {
+			arg->num_ldb_credit_pools++;
+			arg->num_ldb_credits += pool->total_credits;
+		}
+
+		arg->num_dir_credits +=
+			domain->dqed_freelist.bound -
+			domain->dqed_freelist.base;
+
+		DLB_DOM_LIST_FOR(domain->avail_dir_credit_pools, pool, iter2)
+			arg->num_dir_credit_pools++;
+		DLB_DOM_LIST_FOR(domain->used_dir_credit_pools, pool, iter2) {
+			arg->num_dir_credit_pools++;
+			arg->num_dir_credits += pool->total_credits;
+		}
+
+		arg->num_hist_list_entries += domain->total_hist_list_entries;
+	}
+
+	return 0;
+}
+
+static inline bool dlb_ldb_port_owned_by_vf(struct dlb_hw *hw,
+					    u32 vf_id,
+					    u32 port_id)
+{
+	return (hw->rsrcs.ldb_ports[port_id].id.vf_owned &&
+		hw->rsrcs.ldb_ports[port_id].id.vf_id == vf_id);
+}
+
+static inline bool dlb_dir_port_owned_by_vf(struct dlb_hw *hw,
+					    u32 vf_id,
+					    u32 port_id)
+{
+	return (hw->rsrcs.dir_pq_pairs[port_id].id.vf_owned &&
+		hw->rsrcs.dir_pq_pairs[port_id].id.vf_id == vf_id);
+}
+
+void dlb_send_async_pf_to_vf_msg(struct dlb_hw *hw, unsigned int vf_id)
+{
+	union dlb_func_pf_pf2vf_mailbox_isr r0 = { {0} };
+
+	r0.field.isr = 1 << vf_id;
+
+	DLB_FUNC_WR(hw, DLB_FUNC_PF_PF2VF_MAILBOX_ISR(0), r0.val);
+}
+
+bool dlb_pf_to_vf_complete(struct dlb_hw *hw, unsigned int vf_id)
+{
+	union dlb_func_pf_pf2vf_mailbox_isr r0;
+
+	r0.val = DLB_FUNC_RD(hw, DLB_FUNC_PF_PF2VF_MAILBOX_ISR(vf_id));
+
+	return (r0.val & (1 << vf_id)) == 0;
+}
+
+void dlb_send_async_vf_to_pf_msg(struct dlb_hw *hw)
+{
+	union dlb_func_vf_vf2pf_mailbox_isr r0 = { {0} };
+
+	r0.field.isr = 1;
+	DLB_FUNC_WR(hw, DLB_FUNC_VF_VF2PF_MAILBOX_ISR, r0.val);
+}
+
+bool dlb_vf_to_pf_complete(struct dlb_hw *hw)
+{
+	union dlb_func_vf_vf2pf_mailbox_isr r0;
+
+	r0.val = DLB_FUNC_RD(hw, DLB_FUNC_VF_VF2PF_MAILBOX_ISR);
+
+	return (r0.field.isr == 0);
+}
+
+bool dlb_vf_flr_complete(struct dlb_hw *hw)
+{
+	union dlb_func_vf_vf_reset_in_progress r0;
+
+	r0.val = DLB_FUNC_RD(hw, DLB_FUNC_VF_VF_RESET_IN_PROGRESS);
+
+	return (r0.field.reset_in_progress == 0);
+}
+
+int dlb_pf_read_vf_mbox_req(struct dlb_hw *hw,
+			    unsigned int vf_id,
+			    void *data,
+			    int len)
+{
+	u32 buf[DLB_VF2PF_REQ_BYTES / 4];
+	int num_words;
+	int i;
+
+	if (len > DLB_VF2PF_REQ_BYTES) {
+		DLB_HW_ERR(hw, "[%s()] len (%d) > VF->PF mailbox req size\n",
+			   __func__, len);
+		return -EINVAL;
+	}
+
+	if (len == 0) {
+		DLB_HW_ERR(hw, "[%s()] invalid len (0)\n", __func__);
+		return -EINVAL;
+	}
+
+	/* Round up len to the nearest 4B boundary, since the mailbox registers
+	 * are 32b wide.
+	 */
+	num_words = len / 4;
+	if (len % 4 != 0)
+		num_words++;
+
+	for (i = 0; i < num_words; i++) {
+		u32 idx = i + DLB_VF2PF_REQ_BASE_WORD;
+
+		buf[i] = DLB_FUNC_RD(hw, DLB_FUNC_PF_VF2PF_MAILBOX(vf_id, idx));
+	}
+
+	memcpy(data, buf, len);
+
+	return 0;
+}
+
+int dlb_pf_read_vf_mbox_resp(struct dlb_hw *hw,
+			     unsigned int vf_id,
+			     void *data,
+			     int len)
+{
+	u32 buf[DLB_VF2PF_RESP_BYTES / 4];
+	int num_words;
+	int i;
+
+	if (len > DLB_VF2PF_RESP_BYTES) {
+		DLB_HW_ERR(hw, "[%s()] len (%d) > VF->PF mailbox resp size\n",
+			   __func__, len);
+		return -EINVAL;
+	}
+
+	/* Round up len to the nearest 4B boundary, since the mailbox registers
+	 * are 32b wide.
+	 */
+	num_words = len / 4;
+	if (len % 4 != 0)
+		num_words++;
+
+	for (i = 0; i < num_words; i++) {
+		u32 idx = i + DLB_VF2PF_RESP_BASE_WORD;
+
+		buf[i] = DLB_FUNC_RD(hw, DLB_FUNC_PF_VF2PF_MAILBOX(vf_id, idx));
+	}
+
+	memcpy(data, buf, len);
+
+	return 0;
+}
+
+int dlb_pf_write_vf_mbox_resp(struct dlb_hw *hw,
+			      unsigned int vf_id,
+			      void *data,
+			      int len)
+{
+	u32 buf[DLB_PF2VF_RESP_BYTES / 4];
+	int num_words;
+	int i;
+
+	if (len > DLB_PF2VF_RESP_BYTES) {
+		DLB_HW_ERR(hw, "[%s()] len (%d) > PF->VF mailbox resp size\n",
+			   __func__, len);
+		return -EINVAL;
+	}
+
+	memcpy(buf, data, len);
+
+	/* Round up len to the nearest 4B boundary, since the mailbox registers
+	 * are 32b wide.
+	 */
+	num_words = len / 4;
+	if (len % 4 != 0)
+		num_words++;
+
+	for (i = 0; i < num_words; i++) {
+		u32 idx = i + DLB_PF2VF_RESP_BASE_WORD;
+
+		DLB_FUNC_WR(hw, DLB_FUNC_PF_PF2VF_MAILBOX(vf_id, idx), buf[i]);
+	}
+
+	return 0;
+}
+
+int dlb_pf_write_vf_mbox_req(struct dlb_hw *hw,
+			     unsigned int vf_id,
+			     void *data,
+			     int len)
+{
+	u32 buf[DLB_PF2VF_REQ_BYTES / 4];
+	int num_words;
+	int i;
+
+	if (len > DLB_PF2VF_REQ_BYTES) {
+		DLB_HW_ERR(hw, "[%s()] len (%d) > PF->VF mailbox req size\n",
+			   __func__, len);
+		return -EINVAL;
+	}
+
+	memcpy(buf, data, len);
+
+	/* Round up len to the nearest 4B boundary, since the mailbox registers
+	 * are 32b wide.
+	 */
+	num_words = len / 4;
+	if (len % 4 != 0)
+		num_words++;
+
+	for (i = 0; i < num_words; i++) {
+		u32 idx = i + DLB_PF2VF_REQ_BASE_WORD;
+
+		DLB_FUNC_WR(hw, DLB_FUNC_PF_PF2VF_MAILBOX(vf_id, idx), buf[i]);
+	}
+
+	return 0;
+}
+
+int dlb_vf_read_pf_mbox_resp(struct dlb_hw *hw, void *data, int len)
+{
+	u32 buf[DLB_PF2VF_RESP_BYTES / 4];
+	int num_words;
+	int i;
+
+	if (len > DLB_PF2VF_RESP_BYTES) {
+		DLB_HW_ERR(hw, "[%s()] len (%d) > PF->VF mailbox resp size\n",
+			   __func__, len);
+		return -EINVAL;
+	}
+
+	if (len == 0) {
+		DLB_HW_ERR(hw, "[%s()] invalid len (0)\n", __func__);
+		return -EINVAL;
+	}
+
+	/* Round up len to the nearest 4B boundary, since the mailbox registers
+	 * are 32b wide.
+	 */
+	num_words = len / 4;
+	if (len % 4 != 0)
+		num_words++;
+
+	for (i = 0; i < num_words; i++) {
+		u32 idx = i + DLB_PF2VF_RESP_BASE_WORD;
+
+		buf[i] = DLB_FUNC_RD(hw, DLB_FUNC_VF_PF2VF_MAILBOX(idx));
+	}
+
+	memcpy(data, buf, len);
+
+	return 0;
+}
+
+int dlb_vf_read_pf_mbox_req(struct dlb_hw *hw, void *data, int len)
+{
+	u32 buf[DLB_PF2VF_REQ_BYTES / 4];
+	int num_words;
+	int i;
+
+	if (len > DLB_PF2VF_REQ_BYTES) {
+		DLB_HW_ERR(hw, "[%s()] len (%d) > PF->VF mailbox req size\n",
+			   __func__, len);
+		return -EINVAL;
+	}
+
+	/* Round up len to the nearest 4B boundary, since the mailbox registers
+	 * are 32b wide.
+	 */
+	num_words = len / 4;
+	if ((len % 4) != 0)
+		num_words++;
+
+	for (i = 0; i < num_words; i++) {
+		u32 idx = i + DLB_PF2VF_REQ_BASE_WORD;
+
+		buf[i] = DLB_FUNC_RD(hw, DLB_FUNC_VF_PF2VF_MAILBOX(idx));
+	}
+
+	memcpy(data, buf, len);
+
+	return 0;
+}
+
+int dlb_vf_write_pf_mbox_req(struct dlb_hw *hw, void *data, int len)
+{
+	u32 buf[DLB_VF2PF_REQ_BYTES / 4];
+	int num_words;
+	int i;
+
+	if (len > DLB_VF2PF_REQ_BYTES) {
+		DLB_HW_ERR(hw, "[%s()] len (%d) > VF->PF mailbox req size\n",
+			   __func__, len);
+		return -EINVAL;
+	}
+
+	memcpy(buf, data, len);
+
+	/* Round up len to the nearest 4B boundary, since the mailbox registers
+	 * are 32b wide.
+	 */
+	num_words = len / 4;
+	if (len % 4 != 0)
+		num_words++;
+
+	for (i = 0; i < num_words; i++) {
+		u32 idx = i + DLB_VF2PF_REQ_BASE_WORD;
+
+		DLB_FUNC_WR(hw, DLB_FUNC_VF_VF2PF_MAILBOX(idx), buf[i]);
+	}
+
+	return 0;
+}
+
+int dlb_vf_write_pf_mbox_resp(struct dlb_hw *hw, void *data, int len)
+{
+	u32 buf[DLB_VF2PF_RESP_BYTES / 4];
+	int num_words;
+	int i;
+
+	if (len > DLB_VF2PF_RESP_BYTES) {
+		DLB_HW_ERR(hw, "[%s()] len (%d) > VF->PF mailbox resp size\n",
+			   __func__, len);
+		return -EINVAL;
+	}
+
+	memcpy(buf, data, len);
+
+	/* Round up len to the nearest 4B boundary, since the mailbox registers
+	 * are 32b wide.
+	 */
+	num_words = len / 4;
+	if (len % 4 != 0)
+		num_words++;
+
+	for (i = 0; i < num_words; i++) {
+		u32 idx = i + DLB_VF2PF_RESP_BASE_WORD;
+
+		DLB_FUNC_WR(hw, DLB_FUNC_VF_VF2PF_MAILBOX(idx), buf[i]);
+	}
+
+	return 0;
+}
+
+bool dlb_vf_is_locked(struct dlb_hw *hw, unsigned int vf_id)
+{
+	return hw->vf[vf_id].locked;
+}
+
+static void dlb_vf_set_rsrc_virt_ids(struct dlb_function_resources *rsrcs,
+				     unsigned int vf_id)
+{
+	struct dlb_list_entry *iter __attribute__((unused));
+	struct dlb_dir_pq_pair *dir_port;
+	struct dlb_ldb_queue *ldb_queue;
+	struct dlb_ldb_port *ldb_port;
+	struct dlb_credit_pool *pool;
+	struct dlb_domain *domain;
+	int i;
+
+	i = 0;
+	DLB_FUNC_LIST_FOR(rsrcs->avail_domains, domain, iter) {
+		domain->id.virt_id = i;
+		domain->id.vf_owned = true;
+		domain->id.vf_id = vf_id;
+		i++;
+	}
+
+	i = 0;
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_queues, ldb_queue, iter) {
+		ldb_queue->id.virt_id = i;
+		ldb_queue->id.vf_owned = true;
+		ldb_queue->id.vf_id = vf_id;
+		i++;
+	}
+
+	i = 0;
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_ports, ldb_port, iter) {
+		ldb_port->id.virt_id = i;
+		ldb_port->id.vf_owned = true;
+		ldb_port->id.vf_id = vf_id;
+		i++;
+	}
+
+	i = 0;
+	DLB_FUNC_LIST_FOR(rsrcs->avail_dir_pq_pairs, dir_port, iter) {
+		dir_port->id.virt_id = i;
+		dir_port->id.vf_owned = true;
+		dir_port->id.vf_id = vf_id;
+		i++;
+	}
+
+	i = 0;
+	DLB_FUNC_LIST_FOR(rsrcs->avail_ldb_credit_pools, pool, iter) {
+		pool->id.virt_id = i;
+		pool->id.vf_owned = true;
+		pool->id.vf_id = vf_id;
+		i++;
+	}
+
+	i = 0;
+	DLB_FUNC_LIST_FOR(rsrcs->avail_dir_credit_pools, pool, iter) {
+		pool->id.virt_id = i;
+		pool->id.vf_owned = true;
+		pool->id.vf_id = vf_id;
+		i++;
+	}
+}
+
+void dlb_lock_vf(struct dlb_hw *hw, unsigned int vf_id)
+{
+	struct dlb_function_resources *rsrcs = &hw->vf[vf_id];
+
+	rsrcs->locked = true;
+
+	dlb_vf_set_rsrc_virt_ids(rsrcs, vf_id);
+}
+
+void dlb_unlock_vf(struct dlb_hw *hw, unsigned int vf_id)
+{
+	hw->vf[vf_id].locked = false;
+}
+
+int dlb_reset_vf_resources(struct dlb_hw *hw, unsigned int vf_id)
+{
+	if (vf_id >= DLB_MAX_NUM_VFS)
+		return -EINVAL;
+
+	/* If the VF is locked, its resource assignment can't be changed */
+	if (dlb_vf_is_locked(hw, vf_id))
+		return -EPERM;
+
+	dlb_update_vf_sched_domains(hw, vf_id, 0);
+	dlb_update_vf_ldb_queues(hw, vf_id, 0);
+	dlb_update_vf_ldb_ports(hw, vf_id, 0);
+	dlb_update_vf_dir_ports(hw, vf_id, 0);
+	dlb_update_vf_ldb_credit_pools(hw, vf_id, 0);
+	dlb_update_vf_dir_credit_pools(hw, vf_id, 0);
+	dlb_update_vf_ldb_credits(hw, vf_id, 0);
+	dlb_update_vf_dir_credits(hw, vf_id, 0);
+	dlb_update_vf_hist_list_entries(hw, vf_id, 0);
+	dlb_update_vf_atomic_inflights(hw, vf_id, 0);
+
+	return 0;
+}
+
+void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.ldb_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw)
+{
+	union dlb_sys_cq_mode r0;
+
+	r0.val = DLB_CSR_RD(hw, DLB_SYS_CQ_MODE);
+
+	r0.field.dir_cq64 = 1;
+
+	DLB_CSR_WR(hw, DLB_SYS_CQ_MODE, r0.val);
+}
+
+void dlb_hw_set_qe_arbiter_weights(struct dlb_hw *hw, u8 weight[8])
+{
+	union dlb_atm_pipe_ctrl_arb_weights_rdy_bin r0 = { {0} };
+	union dlb_nalb_pipe_ctrl_arb_weights_tqpri_nalb_0 r1 = { {0} };
+	union dlb_nalb_pipe_ctrl_arb_weights_tqpri_nalb_1 r2 = { {0} };
+	union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_replay_0 r3 = { {0} };
+	union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_replay_1 r4 = { {0} };
+	union dlb_dp_cfg_ctrl_arb_weights_tqpri_replay_0 r5 = { {0} };
+	union dlb_dp_cfg_ctrl_arb_weights_tqpri_replay_1 r6 = { {0} };
+	union dlb_dp_cfg_ctrl_arb_weights_tqpri_dir_0 r7 =  { {0} };
+	union dlb_dp_cfg_ctrl_arb_weights_tqpri_dir_1 r8 =  { {0} };
+	union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_atq_0 r9 = { {0} };
+	union dlb_nalb_pipe_cfg_ctrl_arb_weights_tqpri_atq_1 r10 = { {0} };
+	union dlb_atm_pipe_cfg_ctrl_arb_weights_sched_bin r11 = { {0} };
+	union dlb_aqed_pipe_cfg_ctrl_arb_weights_tqpri_atm_0 r12 = { {0} };
+
+	r0.field.bin0 = weight[1];
+	r0.field.bin1 = weight[3];
+	r0.field.bin2 = weight[5];
+	r0.field.bin3 = weight[7];
+
+	r1.field.pri0 = weight[0];
+	r1.field.pri1 = weight[1];
+	r1.field.pri2 = weight[2];
+	r1.field.pri3 = weight[3];
+	r2.field.pri4 = weight[4];
+	r2.field.pri5 = weight[5];
+	r2.field.pri6 = weight[6];
+	r2.field.pri7 = weight[7];
+
+	r3.field.pri0 = weight[0];
+	r3.field.pri1 = weight[1];
+	r3.field.pri2 = weight[2];
+	r3.field.pri3 = weight[3];
+	r4.field.pri4 = weight[4];
+	r4.field.pri5 = weight[5];
+	r4.field.pri6 = weight[6];
+	r4.field.pri7 = weight[7];
+
+	r5.field.pri0 = weight[0];
+	r5.field.pri1 = weight[1];
+	r5.field.pri2 = weight[2];
+	r5.field.pri3 = weight[3];
+	r6.field.pri4 = weight[4];
+	r6.field.pri5 = weight[5];
+	r6.field.pri6 = weight[6];
+	r6.field.pri7 = weight[7];
+
+	r7.field.pri0 = weight[0];
+	r7.field.pri1 = weight[1];
+	r7.field.pri2 = weight[2];
+	r7.field.pri3 = weight[3];
+	r8.field.pri4 = weight[4];
+	r8.field.pri5 = weight[5];
+	r8.field.pri6 = weight[6];
+	r8.field.pri7 = weight[7];
+
+	r9.field.pri0 = weight[0];
+	r9.field.pri1 = weight[1];
+	r9.field.pri2 = weight[2];
+	r9.field.pri3 = weight[3];
+	r10.field.pri4 = weight[4];
+	r10.field.pri5 = weight[5];
+	r10.field.pri6 = weight[6];
+	r10.field.pri7 = weight[7];
+
+	r11.field.bin0 = weight[1];
+	r11.field.bin1 = weight[3];
+	r11.field.bin2 = weight[5];
+	r11.field.bin3 = weight[7];
+
+	r12.field.pri0 = weight[1];
+	r12.field.pri1 = weight[3];
+	r12.field.pri2 = weight[5];
+	r12.field.pri3 = weight[7];
+
+	DLB_CSR_WR(hw, DLB_ATM_PIPE_CTRL_ARB_WEIGHTS_RDY_BIN, r0.val);
+	DLB_CSR_WR(hw, DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_0, r1.val);
+	DLB_CSR_WR(hw, DLB_NALB_PIPE_CTRL_ARB_WEIGHTS_TQPRI_NALB_1, r2.val);
+	DLB_CSR_WR(hw,
+		   DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0,
+		   r3.val);
+	DLB_CSR_WR(hw,
+		   DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1,
+		   r4.val);
+	DLB_CSR_WR(hw, DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_0, r5.val);
+	DLB_CSR_WR(hw, DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_REPLAY_1, r6.val);
+	DLB_CSR_WR(hw, DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_0, r7.val);
+	DLB_CSR_WR(hw, DLB_DP_CFG_CTRL_ARB_WEIGHTS_TQPRI_DIR_1, r8.val);
+	DLB_CSR_WR(hw, DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_0, r9.val);
+	DLB_CSR_WR(hw, DLB_NALB_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATQ_1, r10.val);
+	DLB_CSR_WR(hw, DLB_ATM_PIPE_CFG_CTRL_ARB_WEIGHTS_SCHED_BIN, r11.val);
+	DLB_CSR_WR(hw, DLB_AQED_PIPE_CFG_CTRL_ARB_WEIGHTS_TQPRI_ATM_0, r12.val);
+}
+
+void dlb_hw_set_qid_arbiter_weights(struct dlb_hw *hw, u8 weight[8])
+{
+	union dlb_lsp_cfg_arb_weight_ldb_qid_0 r0 = { {0} };
+	union dlb_lsp_cfg_arb_weight_ldb_qid_1 r1 = { {0} };
+	union dlb_lsp_cfg_arb_weight_atm_nalb_qid_0 r2 = { {0} };
+	union dlb_lsp_cfg_arb_weight_atm_nalb_qid_1 r3 = { {0} };
+
+	r0.field.slot0_weight = weight[0];
+	r0.field.slot1_weight = weight[1];
+	r0.field.slot2_weight = weight[2];
+	r0.field.slot3_weight = weight[3];
+	r1.field.slot4_weight = weight[4];
+	r1.field.slot5_weight = weight[5];
+	r1.field.slot6_weight = weight[6];
+	r1.field.slot7_weight = weight[7];
+
+	r2.field.slot0_weight = weight[0];
+	r2.field.slot1_weight = weight[1];
+	r2.field.slot2_weight = weight[2];
+	r2.field.slot3_weight = weight[3];
+	r3.field.slot4_weight = weight[4];
+	r3.field.slot5_weight = weight[5];
+	r3.field.slot6_weight = weight[6];
+	r3.field.slot7_weight = weight[7];
+
+	DLB_CSR_WR(hw, DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_0, r0.val);
+	DLB_CSR_WR(hw, DLB_LSP_CFG_ARB_WEIGHT_LDB_QID_1, r1.val);
+	DLB_CSR_WR(hw, DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_0, r2.val);
+	DLB_CSR_WR(hw, DLB_LSP_CFG_ARB_WEIGHT_ATM_NALB_QID_1, r3.val);
+}
+
+void dlb_hw_enable_pp_sw_alarms(struct dlb_hw *hw)
+{
+	union dlb_chp_cfg_ldb_pp_sw_alarm_en r0 = { {0} };
+	union dlb_chp_cfg_dir_pp_sw_alarm_en r1 = { {0} };
+	int i;
+
+	r0.field.alarm_enable = 1;
+	r1.field.alarm_enable = 1;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_PORTS; i++)
+		DLB_CSR_WR(hw, DLB_CHP_CFG_LDB_PP_SW_ALARM_EN(i), r0.val);
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_PORTS; i++)
+		DLB_CSR_WR(hw, DLB_CHP_CFG_DIR_PP_SW_ALARM_EN(i), r1.val);
+}
+
+void dlb_hw_disable_pp_sw_alarms(struct dlb_hw *hw)
+{
+	union dlb_chp_cfg_ldb_pp_sw_alarm_en r0 = { {0} };
+	union dlb_chp_cfg_dir_pp_sw_alarm_en r1 = { {0} };
+	int i;
+
+	r0.field.alarm_enable = 0;
+	r1.field.alarm_enable = 0;
+
+	for (i = 0; i < DLB_MAX_NUM_LDB_PORTS; i++)
+		DLB_CSR_WR(hw, DLB_CHP_CFG_LDB_PP_SW_ALARM_EN(i), r0.val);
+
+	for (i = 0; i < DLB_MAX_NUM_DIR_PORTS; i++)
+		DLB_CSR_WR(hw, DLB_CHP_CFG_DIR_PP_SW_ALARM_EN(i), r1.val);
+}
diff --git a/drivers/event/dlb/pf/base/dlb_resource.h b/drivers/event/dlb/pf/base/dlb_resource.h
new file mode 100644
index 0000000..5500f9b
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_resource.h
@@ -0,0 +1,1625 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_RESOURCE_H
+#define __DLB_RESOURCE_H
+
+#include "dlb_hw_types.h"
+#include "dlb_osdep_types.h"
+#include "dlb_user.h"
+
+/**
+ * dlb_resource_init() - initialize the device
+ * @hw: pointer to struct dlb_hw.
+ *
+ * This function initializes the device's software state (pointed to by the hw
+ * argument) and programs global scheduling QoS registers. This function should
+ * be called during driver initialization.
+ *
+ * The dlb_hw struct must be unique per DLB device and persist until the device
+ * is reset.
+ *
+ * Return:
+ * Returns 0 upon success, -1 otherwise.
+ */
+int dlb_resource_init(struct dlb_hw *hw);
+
+/**
+ * dlb_resource_free() - free device state memory
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function frees software state pointed to by dlb_hw. This function
+ * should be called when resetting the device or unloading the driver.
+ */
+void dlb_resource_free(struct dlb_hw *hw);
+
+/**
+ * dlb_resource_reset() - reset in-use resources to their initial state
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function resets in-use resources, and makes them available for use.
+ * All resources go back to their owning function, whether a PF or a VF.
+ */
+void dlb_resource_reset(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_create_sched_domain() - create a scheduling domain
+ * @hw: dlb_hw handle for a particular device.
+ * @args: scheduling domain creation arguments.
+ * @resp: response structure.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function creates a scheduling domain containing the resources specified
+ * in args. The individual resources (queues, ports, credit pools) can be
+ * configured after creating a scheduling domain.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the domain ID.
+ *
+ * Note: resp->id contains a virtual ID if vf_request is true.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, or the requested domain name
+ *	    is already in use.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_sched_domain(struct dlb_hw *hw,
+			       struct dlb_create_sched_domain_args *args,
+			       struct dlb_cmd_response *resp,
+			       bool vf_request,
+			       unsigned int vf_id);
+
+/**
+ * dlb_hw_create_ldb_pool() - create a load-balanced credit pool
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: credit pool creation arguments.
+ * @resp: response structure.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function creates a load-balanced credit pool containing the number of
+ * requested credits.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the pool ID.
+ *
+ * Note: resp->id contains a virtual ID if vf_request is true.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_ldb_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_pool_args *args,
+			   struct dlb_cmd_response *resp,
+			   bool vf_request,
+			   unsigned int vf_id);
+
+/**
+ * dlb_hw_create_dir_pool() - create a directed credit pool
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: credit pool creation arguments.
+ * @resp: response structure.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function creates a directed credit pool containing the number of
+ * requested credits.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the pool ID.
+ *
+ * Note: resp->id contains a virtual ID if vf_request is true.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_dir_pool(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_pool_args *args,
+			   struct dlb_cmd_response *resp,
+			   bool vf_request,
+			   unsigned int vf_id);
+
+/**
+ * dlb_hw_create_ldb_queue() - create a load-balanced queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue creation arguments.
+ * @resp: response structure.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function creates a load-balanced queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the queue ID.
+ *
+ * Note: resp->id contains a virtual ID if vf_request is true.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    the domain has already been started, or the requested queue name is
+ *	    already in use.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_ldb_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_ldb_queue_args *args,
+			    struct dlb_cmd_response *resp,
+			    bool vf_request,
+			    unsigned int vf_id);
+
+/**
+ * dlb_hw_create_dir_queue() - create a directed queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue creation arguments.
+ * @resp: response structure.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function creates a directed queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the queue ID.
+ *
+ * Note: resp->id contains a virtual ID if vf_request is true.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, the domain is not configured,
+ *	    or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_dir_queue(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_create_dir_queue_args *args,
+			    struct dlb_cmd_response *resp,
+			    bool vf_request,
+			    unsigned int vf_id);
+
+/**
+ * dlb_hw_create_dir_port() - create a directed port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port creation arguments.
+ * @pop_count_dma_base: base address of the pop count memory. This can be
+ *			a PA or an IOVA.
+ * @cq_dma_base: base address of the CQ memory. This can be a PA or an IOVA.
+ * @resp: response structure.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function creates a directed port.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the port ID.
+ *
+ * Note: resp->id contains a virtual ID if vf_request is true.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, a credit setting is invalid, a
+ *	    pool ID is invalid, a pointer address is not properly aligned, the
+ *	    domain is not configured, or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_dir_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp,
+			   bool vf_request,
+			   unsigned int vf_id);
+
+/**
+ * dlb_hw_create_ldb_port() - create a load-balanced port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port creation arguments.
+ * @pop_count_dma_base: base address of the pop count memory. This can be
+ *			 a PA or an IOVA.
+ * @cq_dma_base: base address of the CQ memory. This can be a PA or an IOVA.
+ * @resp: response structure.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function creates a load-balanced port.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the port ID.
+ *
+ * Note: resp->id contains a virtual ID if vf_request is true.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, a credit setting is invalid, a
+ *	    pool ID is invalid, a pointer address is not properly aligned, the
+ *	    domain is not configured, or the domain has already been started.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_create_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_create_ldb_port_args *args,
+			   u64 pop_count_dma_base,
+			   u64 cq_dma_base,
+			   struct dlb_cmd_response *resp,
+			   bool vf_request,
+			   unsigned int vf_id);
+
+/**
+ * dlb_hw_start_domain() - start a scheduling domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: start domain arguments.
+ * @resp: response structure.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function starts a scheduling domain, which allows applications to send
+ * traffic through it. Once a domain is started, its resources can no longer be
+ * configured (besides QID remapping and port enable/disable).
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - the domain is not configured, or the domain is already started.
+ */
+int dlb_hw_start_domain(struct dlb_hw *hw,
+			u32 domain_id,
+			struct dlb_start_domain_args *args,
+			struct dlb_cmd_response *resp,
+			bool vf_request,
+			unsigned int vf_id);
+
+/**
+ * dlb_hw_map_qid() - map a load-balanced queue to a load-balanced port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: map QID arguments.
+ * @resp: response structure.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function configures the DLB to schedule QEs from the specified queue to
+ * the specified port. Each load-balanced port can be mapped to up to 8 queues;
+ * each load-balanced queue can potentially map to all the load-balanced ports.
+ *
+ * A successful return does not necessarily mean the mapping was configured. If
+ * this function is unable to immediately map the queue to the port, it will
+ * add the requested operation to a per-port list of pending map/unmap
+ * operations, and (if it's not already running) launch a kernel thread that
+ * periodically attempts to process all pending operations. In a sense, this is
+ * an asynchronous function.
+ *
+ * This asynchronicity creates two views of the state of hardware: the actual
+ * hardware state and the requested state (as if every request completed
+ * immediately). If there are any pending map/unmap operations, the requested
+ * state will differ from the actual state. All validation is performed with
+ * respect to the pending state; for instance, if there are 8 pending map
+ * operations for port X, a request for a 9th will fail because a load-balanced
+ * port can only map up to 8 queues.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, invalid port or queue ID, or
+ *	    the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_map_qid(struct dlb_hw *hw,
+		   u32 domain_id,
+		   struct dlb_map_qid_args *args,
+		   struct dlb_cmd_response *resp,
+		   bool vf_request,
+		   unsigned int vf_id);
+
+/**
+ * dlb_hw_unmap_qid() - Unmap a load-balanced queue from a load-balanced port
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: unmap QID arguments.
+ * @resp: response structure.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function configures the DLB to stop scheduling QEs from the specified
+ * queue to the specified port.
+ *
+ * A successful return does not necessarily mean the mapping was removed. If
+ * this function is unable to immediately unmap the queue from the port, it
+ * will add the requested operation to a per-port list of pending map/unmap
+ * operations, and (if it's not already running) launch a kernel thread that
+ * periodically attempts to process all pending operations. See
+ * dlb_hw_map_qid() for more details.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - A requested resource is unavailable, invalid port or queue ID, or
+ *	    the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_unmap_qid(struct dlb_hw *hw,
+		     u32 domain_id,
+		     struct dlb_unmap_qid_args *args,
+		     struct dlb_cmd_response *resp,
+		     bool vf_request,
+		     unsigned int vf_id);
+
+/**
+ * dlb_finish_unmap_qid_procedures() - finish any pending unmap procedures
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function attempts to finish any outstanding unmap procedures.
+ * This function should be called by the kernel thread responsible for
+ * finishing map/unmap procedures.
+ *
+ * Return:
+ * Returns the number of procedures that weren't completed.
+ */
+unsigned int dlb_finish_unmap_qid_procedures(struct dlb_hw *hw);
+
+/**
+ * dlb_finish_map_qid_procedures() - finish any pending map procedures
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function attempts to finish any outstanding map procedures.
+ * This function should be called by the kernel thread responsible for
+ * finishing map/unmap procedures.
+ *
+ * Return:
+ * Returns the number of procedures that weren't completed.
+ */
+unsigned int dlb_finish_map_qid_procedures(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_enable_ldb_port() - enable a load-balanced port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port enable arguments.
+ * @resp: response structure.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function configures the DLB to schedule QEs to a load-balanced port.
+ * Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_enable_ldb_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_enable_ldb_port_args *args,
+			   struct dlb_cmd_response *resp,
+			   bool vf_request,
+			   unsigned int vf_id);
+
+/**
+ * dlb_hw_disable_ldb_port() - disable a load-balanced port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port disable arguments.
+ * @resp: response structure.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function configures the DLB to stop scheduling QEs to a load-balanced
+ * port. Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_disable_ldb_port(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_disable_ldb_port_args *args,
+			    struct dlb_cmd_response *resp,
+			    bool vf_request,
+			    unsigned int vf_id);
+
+/**
+ * dlb_hw_enable_dir_port() - enable a directed port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port enable arguments.
+ * @resp: response structure.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function configures the DLB to schedule QEs to a directed port.
+ * Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_enable_dir_port(struct dlb_hw *hw,
+			   u32 domain_id,
+			   struct dlb_enable_dir_port_args *args,
+			   struct dlb_cmd_response *resp,
+			   bool vf_request,
+			   unsigned int vf_id);
+
+/**
+ * dlb_hw_disable_dir_port() - disable a directed port for scheduling
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: port disable arguments.
+ * @resp: response structure.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function configures the DLB to stop scheduling QEs to a directed port.
+ * Ports are enabled by default.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid or the domain is not configured.
+ * EFAULT - Internal error (resp->status not set).
+ */
+int dlb_hw_disable_dir_port(struct dlb_hw *hw,
+			    u32 domain_id,
+			    struct dlb_disable_dir_port_args *args,
+			    struct dlb_cmd_response *resp,
+			    bool vf_request,
+			    unsigned int vf_id);
+
+/**
+ * dlb_configure_ldb_cq_interrupt() - configure load-balanced CQ for interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: load-balancd port ID.
+ * @vector: interrupt vector ID. Should be 0 for MSI or compressed MSI-X mode,
+ *	    else a value up to 64.
+ * @mode: interrupt type (DLB_CQ_ISR_MODE_MSI or DLB_CQ_ISR_MODE_MSIX)
+ * @vf: If the port is VF-owned, the VF's ID. This is used for translating the
+ *	virtual port ID to a physical port ID. Ignored if mode is not MSI.
+ * @owner_vf: the VF to route the interrupt to. Ignore if mode is not MSI.
+ * @threshold: the minimum CQ depth at which the interrupt can fire. Must be
+ *	greater than 0.
+ *
+ * This function configures the DLB registers for load-balanced CQ's interrupts.
+ * This doesn't enable the CQ's interrupt; that can be done with
+ * dlb_arm_cq_interrupt() or through an interrupt arm QE.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid.
+ */
+int dlb_configure_ldb_cq_interrupt(struct dlb_hw *hw,
+				   int port_id,
+				   int vector,
+				   int mode,
+				   unsigned int vf,
+				   unsigned int owner_vf,
+				   u16 threshold);
+
+/**
+ * dlb_configure_dir_cq_interrupt() - configure directed CQ for interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: load-balancd port ID.
+ * @vector: interrupt vector ID. Should be 0 for MSI or compressed MSI-X mode,
+ *	    else a value up to 64.
+ * @mode: interrupt type (DLB_CQ_ISR_MODE_MSI or DLB_CQ_ISR_MODE_MSIX)
+ * @vf: If the port is VF-owned, the VF's ID. This is used for translating the
+ *	virtual port ID to a physical port ID. Ignored if mode is not MSI.
+ * @owner_vf: the VF to route the interrupt to. Ignore if mode is not MSI.
+ * @threshold: the minimum CQ depth at which the interrupt can fire. Must be
+ *	greater than 0.
+ *
+ * This function configures the DLB registers for directed CQ's interrupts.
+ * This doesn't enable the CQ's interrupt; that can be done with
+ * dlb_arm_cq_interrupt() or through an interrupt arm QE.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise.
+ *
+ * Errors:
+ * EINVAL - The port ID is invalid.
+ */
+int dlb_configure_dir_cq_interrupt(struct dlb_hw *hw,
+				   int port_id,
+				   int vector,
+				   int mode,
+				   unsigned int vf,
+				   unsigned int owner_vf,
+				   u16 threshold);
+
+/**
+ * dlb_enable_alarm_interrupts() - enable certain hardware alarm interrupts
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function configures the ingress error alarm. (Other alarms are enabled
+ * by default.)
+ */
+void dlb_enable_alarm_interrupts(struct dlb_hw *hw);
+
+/**
+ * dlb_disable_alarm_interrupts() - disable certain hardware alarm interrupts
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function configures the ingress error alarm. (Other alarms are disabled
+ * by default.)
+ */
+void dlb_disable_alarm_interrupts(struct dlb_hw *hw);
+
+/**
+ * dlb_set_msix_mode() - enable certain hardware alarm interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @mode: MSI-X mode (DLB_MSIX_MODE_PACKED or DLB_MSIX_MODE_COMPRESSED)
+ *
+ * This function configures the hardware to use either packed or compressed
+ * mode. This function should not be called if using MSI interrupts.
+ */
+void dlb_set_msix_mode(struct dlb_hw *hw, int mode);
+
+/**
+ * dlb_arm_cq_interrupt() - arm a CQ's interrupt
+ * @hw: dlb_hw handle for a particular device.
+ * @port_id: port ID
+ * @is_ldb: true for load-balanced port, false for a directed port
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function arms the CQ's interrupt. The CQ must be configured prior to
+ * calling this function.
+ *
+ * The function does no parameter validation; that is the caller's
+ * responsibility.
+ *
+ * Return: returns 0 upon success, <0 otherwise.
+ *
+ * EINVAL - Invalid port ID.
+ */
+int dlb_arm_cq_interrupt(struct dlb_hw *hw,
+			 int port_id,
+			 bool is_ldb,
+			 bool vf_request,
+			 unsigned int vf_id);
+
+/**
+ * dlb_read_compressed_cq_intr_status() - read compressed CQ interrupt status
+ * @hw: dlb_hw handle for a particular device.
+ * @ldb_interrupts: 2-entry array of u32 bitmaps
+ * @dir_interrupts: 4-entry array of u32 bitmaps
+ *
+ * This function can be called from a compressed CQ interrupt handler to
+ * determine which CQ interrupts have fired. The caller should take appropriate
+ * (such as waking threads blocked on a CQ's interrupt) then ack the interrupts
+ * with dlb_ack_compressed_cq_intr().
+ */
+void dlb_read_compressed_cq_intr_status(struct dlb_hw *hw,
+					u32 *ldb_interrupts,
+					u32 *dir_interrupts);
+
+/**
+ * dlb_ack_compressed_cq_intr_status() - ack compressed CQ interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @ldb_interrupts: 2-entry array of u32 bitmaps
+ * @dir_interrupts: 4-entry array of u32 bitmaps
+ *
+ * This function ACKs compressed CQ interrupts. Its arguments should be the
+ * same ones passed to dlb_read_compressed_cq_intr_status().
+ */
+void dlb_ack_compressed_cq_intr(struct dlb_hw *hw,
+				u32 *ldb_interrupts,
+				u32 *dir_interrupts);
+
+/**
+ * dlb_read_vf_intr_status() - read the VF interrupt status register
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function can be called from a VF's interrupt handler to determine
+ * which interrupts have fired. The first 31 bits correspond to CQ interrupt
+ * vectors, and the final bit is for the PF->VF mailbox interrupt vector.
+ *
+ * Return:
+ * Returns a bit vector indicating which interrupt vectors are active.
+ */
+u32 dlb_read_vf_intr_status(struct dlb_hw *hw);
+
+/**
+ * dlb_ack_vf_intr_status() - ack VF interrupts
+ * @hw: dlb_hw handle for a particular device.
+ * @interrupts: 32-bit bitmap
+ *
+ * This function ACKs a VF's interrupts. Its interrupts argument should be the
+ * value returned by dlb_read_vf_intr_status().
+ */
+void dlb_ack_vf_intr_status(struct dlb_hw *hw, u32 interrupts);
+
+/**
+ * dlb_ack_vf_msi_intr() - ack VF MSI interrupt
+ * @hw: dlb_hw handle for a particular device.
+ * @interrupts: 32-bit bitmap
+ *
+ * This function clears the VF's MSI interrupt pending register. Its interrupts
+ * argument should be contain the MSI vectors to ACK. For example, if MSI MME
+ * is in mode 0, then one bit 0 should ever be set.
+ */
+void dlb_ack_vf_msi_intr(struct dlb_hw *hw, u32 interrupts);
+
+/**
+ * dlb_ack_vf_mbox_int() - ack PF->VF mailbox interrupt
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * When done processing the PF mailbox request, this function unsets
+ * the PF's mailbox ISR register.
+ */
+void dlb_ack_pf_mbox_int(struct dlb_hw *hw);
+
+/**
+ * dlb_read_vf_to_pf_int_bitvec() - return a bit vector of all requesting VFs
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * When the VF->PF ISR fires, this function can be called to determine which
+ * VF(s) are requesting service. This bitvector must be passed to
+ * dlb_ack_vf_to_pf_int() when processing is complete for all requesting VFs.
+ *
+ * Return:
+ * Returns a bit vector indicating which VFs (0-15) have requested service.
+ */
+u32 dlb_read_vf_to_pf_int_bitvec(struct dlb_hw *hw);
+
+/**
+ * dlb_ack_vf_mbox_int() - ack processed VF->PF mailbox interrupt
+ * @hw: dlb_hw handle for a particular device.
+ * @bitvec: bit vector returned by dlb_read_vf_to_pf_int_bitvec()
+ *
+ * When done processing all VF mailbox requests, this function unsets the VF's
+ * mailbox ISR register.
+ */
+void dlb_ack_vf_mbox_int(struct dlb_hw *hw, u32 bitvec);
+
+/**
+ * dlb_read_vf_flr_int_bitvec() - return a bit vector of all VFs requesting FLR
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * When the VF FLR ISR fires, this function can be called to determine which
+ * VF(s) are requesting FLRs. This bitvector must passed to
+ * dlb_ack_vf_flr_int() when processing is complete for all requesting VFs.
+ *
+ * Return:
+ * Returns a bit vector indicating which VFs (0-15) have requested FLRs.
+ */
+u32 dlb_read_vf_flr_int_bitvec(struct dlb_hw *hw);
+
+/**
+ * dlb_ack_vf_flr_int() - ack processed VF<->PF interrupt(s)
+ * @hw: dlb_hw handle for a particular device.
+ * @bitvec: bit vector returned by dlb_read_vf_flr_int_bitvec()
+ * @a_stepping: device is A-stepping
+ *
+ * When done processing all VF FLR requests, this function unsets the VF's FLR
+ * ISR register.
+ *
+ * Note: The caller must ensure dlb_set_vf_reset_in_progress(),
+ * dlb_clr_vf_reset_in_progress(), and dlb_ack_vf_flr_int() are not executed in
+ * parallel, because the reset-in-progress register does not support atomic
+ * updates on A-stepping devices.
+ */
+void dlb_ack_vf_flr_int(struct dlb_hw *hw, u32 bitvec, bool a_stepping);
+
+/**
+ * dlb_ack_vf_to_pf_int() - ack processed VF mbox and FLR interrupt(s)
+ * @hw: dlb_hw handle for a particular device.
+ * @mbox_bitvec: bit vector returned by dlb_read_vf_to_pf_int_bitvec()
+ * @flr_bitvec: bit vector returned by dlb_read_vf_flr_int_bitvec()
+ *
+ * When done processing all VF requests, this function communicates to the
+ * hardware that processing is complete. When this function completes, hardware
+ * can immediately generate another VF mbox or FLR interrupt.
+ */
+void dlb_ack_vf_to_pf_int(struct dlb_hw *hw,
+			  u32 mbox_bitvec,
+			  u32 flr_bitvec);
+
+/**
+ * dlb_process_alarm_interrupt() - process an alarm interrupt
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function reads the alarm syndrome, logs its, and acks the interrupt.
+ * This function should be called from the alarm interrupt handler when
+ * interrupt vector DLB_INT_ALARM fires.
+ */
+void dlb_process_alarm_interrupt(struct dlb_hw *hw);
+
+/**
+ * dlb_process_ingress_error_interrupt() - process ingress error interrupts
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function reads the alarm syndrome, logs it, notifies user-space, and
+ * acks the interrupt. This function should be called from the alarm interrupt
+ * handler when interrupt vector DLB_INT_INGRESS_ERROR fires.
+ */
+void dlb_process_ingress_error_interrupt(struct dlb_hw *hw);
+
+/**
+ * dlb_get_group_sequence_numbers() - return a group's number of SNs per queue
+ * @hw: dlb_hw handle for a particular device.
+ * @group_id: sequence number group ID.
+ *
+ * This function returns the configured number of sequence numbers per queue
+ * for the specified group.
+ *
+ * Return:
+ * Returns -EINVAL if group_id is invalid, else the group's SNs per queue.
+ */
+int dlb_get_group_sequence_numbers(struct dlb_hw *hw, unsigned int group_id);
+
+/**
+ * dlb_get_group_sequence_number_occupancy() - return a group's in-use slots
+ * @hw: dlb_hw handle for a particular device.
+ * @group_id: sequence number group ID.
+ *
+ * This function returns the group's number of in-use slots (i.e. load-balanced
+ * queues using the specified group).
+ *
+ * Return:
+ * Returns -EINVAL if group_id is invalid, else the group's occupancy.
+ */
+int dlb_get_group_sequence_number_occupancy(struct dlb_hw *hw,
+					    unsigned int group_id);
+
+/**
+ * dlb_set_group_sequence_numbers() - assign a group's number of SNs per queue
+ * @hw: dlb_hw handle for a particular device.
+ * @group_id: sequence number group ID.
+ * @val: requested amount of sequence numbers per queue.
+ *
+ * This function configures the group's number of sequence numbers per queue.
+ * val can be a power-of-two between 32 and 1024, inclusive. This setting can
+ * be configured until the first ordered load-balanced queue is configured, at
+ * which point the configuration is locked.
+ *
+ * Return:
+ * Returns 0 upon success; -EINVAL if group_id or val is invalid, -EPERM if an
+ * ordered queue is configured.
+ */
+int dlb_set_group_sequence_numbers(struct dlb_hw *hw,
+				   unsigned int group_id,
+				   unsigned long val);
+
+/**
+ * dlb_reset_domain() - reset a scheduling domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function resets and frees a DLB scheduling domain and its associated
+ * resources.
+ *
+ * Pre-condition: the driver must ensure software has stopped sending QEs
+ * through this domain's producer ports before invoking this function, or
+ * undefined behavior will result.
+ *
+ * Return:
+ * Returns 0 upon success, -1 otherwise.
+ *
+ * EINVAL - Invalid domain ID, or the domain is not configured.
+ * EFAULT - Internal error. (Possibly caused if software is the pre-condition
+ *	    is not met.)
+ * ETIMEDOUT - Hardware component didn't reset in the expected time.
+ */
+int dlb_reset_domain(struct dlb_hw *hw,
+		     u32 domain_id,
+		     bool vf_request,
+		     unsigned int vf_id);
+
+/**
+ * dlb_ldb_port_owned_by_domain() - query whether a port is owned by a domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @port_id: indicates whether this request came from a VF.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function returns whether a load-balanced port is owned by a specified
+ * domain.
+ *
+ * Return:
+ * Returns 0 if false, 1 if true, <0 otherwise.
+ *
+ * EINVAL - Invalid domain or port ID, or the domain is not configured.
+ */
+int dlb_ldb_port_owned_by_domain(struct dlb_hw *hw,
+				 u32 domain_id,
+				 u32 port_id,
+				 bool vf_request,
+				 unsigned int vf_id);
+
+/**
+ * dlb_dir_port_owned_by_domain() - query whether a port is owned by a domain
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @port_id: indicates whether this request came from a VF.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function returns whether a directed port is owned by a specified
+ * domain.
+ *
+ * Return:
+ * Returns 0 if false, 1 if true, <0 otherwise.
+ *
+ * EINVAL - Invalid domain or port ID, or the domain is not configured.
+ */
+int dlb_dir_port_owned_by_domain(struct dlb_hw *hw,
+				 u32 domain_id,
+				 u32 port_id,
+				 bool vf_request,
+				 unsigned int vf_id);
+
+/**
+ * dlb_hw_get_num_resources() - query the PCI function's available resources
+ * @arg: pointer to resource counts.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function returns the number of available resources for the PF or for a
+ * VF.
+ *
+ * Return:
+ * Returns 0 upon success, -1 if vf_request is true and vf_id is invalid.
+ */
+int dlb_hw_get_num_resources(struct dlb_hw *hw,
+			     struct dlb_get_num_resources_args *arg,
+			     bool vf_request,
+			     unsigned int vf_id);
+
+/**
+ * dlb_hw_get_num_used_resources() - query the PCI function's used resources
+ * @arg: pointer to resource counts.
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function returns the number of resources in use by the PF or a VF. It
+ * fills in the fields that args points to, except the following:
+ * - max_contiguous_atomic_inflights
+ * - max_contiguous_hist_list_entries
+ * - max_contiguous_ldb_credits
+ * - max_contiguous_dir_credits
+ *
+ * Return:
+ * Returns 0 upon success, -1 if vf_request is true and vf_id is invalid.
+ */
+int dlb_hw_get_num_used_resources(struct dlb_hw *hw,
+				  struct dlb_get_num_resources_args *arg,
+				  bool vf_request,
+				  unsigned int vf_id);
+
+/**
+ * dlb_send_async_vf_to_pf_msg() - (VF only) send a mailbox message to the PF
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function sends a VF->PF mailbox message. It is asynchronous, so it
+ * returns once the message is sent but potentially before the PF has processed
+ * the message. The caller must call dlb_vf_to_pf_complete() to determine when
+ * the PF has finished processing the request.
+ */
+void dlb_send_async_vf_to_pf_msg(struct dlb_hw *hw);
+
+/**
+ * dlb_vf_to_pf_complete() - check the status of an asynchronous mailbox request
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function returns a boolean indicating whether the PF has finished
+ * processing a VF->PF mailbox request. It should only be called after sending
+ * an asynchronous request with dlb_send_async_vf_to_pf_msg().
+ */
+bool dlb_vf_to_pf_complete(struct dlb_hw *hw);
+
+/**
+ * dlb_vf_flr_complete() - check the status of a VF FLR
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function returns a boolean indicating whether the PF has finished
+ * executing the VF FLR. It should only be called after setting the VF's FLR
+ * bit.
+ */
+bool dlb_vf_flr_complete(struct dlb_hw *hw);
+
+/**
+ * dlb_set_vf_reset_in_progress() - set a VF's reset in progress bit
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID.
+ *
+ * Note: This function is only supported on A-stepping devices.
+ *
+ * Note: The caller must ensure dlb_set_vf_reset_in_progress(),
+ * dlb_clr_vf_reset_in_progress(), and dlb_ack_vf_flr_int() are not executed in
+ * parallel, because the reset-in-progress register does not support atomic
+ * updates on A-stepping devices.
+ */
+void dlb_set_vf_reset_in_progress(struct dlb_hw *hw, int vf_id);
+
+/**
+ * dlb_clr_vf_reset_in_progress() - clear a VF's reset in progress bit
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID.
+ *
+ * Note: This function is only supported on A-stepping devices.
+ *
+ * Note: The caller must ensure dlb_set_vf_reset_in_progress(),
+ * dlb_clr_vf_reset_in_progress(), and dlb_ack_vf_flr_int() are not executed in
+ * parallel, because the reset-in-progress register does not support atomic
+ * updates on A-stepping devices.
+ */
+void dlb_clr_vf_reset_in_progress(struct dlb_hw *hw, int vf_id);
+
+/**
+ * dlb_send_async_pf_to_vf_msg() - (PF only) send a mailbox message to the VF
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID.
+ *
+ * This function sends a PF->VF mailbox message. It is asynchronous, so it
+ * returns once the message is sent but potentially before the VF has processed
+ * the message. The caller must call dlb_pf_to_vf_complete() to determine when
+ * the VF has finished processing the request.
+ */
+void dlb_send_async_pf_to_vf_msg(struct dlb_hw *hw, unsigned int vf_id);
+
+/**
+ * dlb_pf_to_vf_complete() - check the status of an asynchronous mailbox request
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID.
+ *
+ * This function returns a boolean indicating whether the VF has finished
+ * processing a PF->VF mailbox request. It should only be called after sending
+ * an asynchronous request with dlb_send_async_pf_to_vf_msg().
+ */
+bool dlb_pf_to_vf_complete(struct dlb_hw *hw, unsigned int vf_id);
+
+/**
+ * dlb_pf_read_vf_mbox_req() - (PF only) read a VF->PF mailbox request
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID.
+ * @data: pointer to message data.
+ * @len: size, in bytes, of the data array.
+ *
+ * This function copies one of the PF's VF->PF mailboxes into the array pointed
+ * to by data.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * EINVAL - len >= DLB_VF2PF_REQ_BYTES.
+ */
+int dlb_pf_read_vf_mbox_req(struct dlb_hw *hw,
+			    unsigned int vf_id,
+			    void *data,
+			    int len);
+
+/**
+ * dlb_pf_read_vf_mbox_resp() - (PF only) read a VF->PF mailbox response
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID.
+ * @data: pointer to message data.
+ * @len: size, in bytes, of the data array.
+ *
+ * This function copies one of the PF's VF->PF mailboxes into the array pointed
+ * to by data.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * EINVAL - len >= DLB_VF2PF_RESP_BYTES.
+ */
+int dlb_pf_read_vf_mbox_resp(struct dlb_hw *hw,
+			     unsigned int vf_id,
+			     void *data,
+			     int len);
+
+/**
+ * dlb_pf_write_vf_mbox_resp() - (PF only) write a PF->VF mailbox response
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID.
+ * @data: pointer to message data.
+ * @len: size, in bytes, of the data array.
+ *
+ * This function copies the user-provided message data into of the PF's VF->PF
+ * mailboxes.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * EINVAL - len >= DLB_PF2VF_RESP_BYTES.
+ */
+int dlb_pf_write_vf_mbox_resp(struct dlb_hw *hw,
+			      unsigned int vf_id,
+			      void *data,
+			      int len);
+
+/**
+ * dlb_pf_write_vf_mbox_req() - (PF only) write a PF->VF mailbox request
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID.
+ * @data: pointer to message data.
+ * @len: size, in bytes, of the data array.
+ *
+ * This function copies the user-provided message data into of the PF's VF->PF
+ * mailboxes.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * EINVAL - len >= DLB_PF2VF_REQ_BYTES.
+ */
+int dlb_pf_write_vf_mbox_req(struct dlb_hw *hw,
+			     unsigned int vf_id,
+			     void *data,
+			     int len);
+
+/**
+ * dlb_vf_read_pf_mbox_resp() - (VF only) read a PF->VF mailbox response
+ * @hw: dlb_hw handle for a particular device.
+ * @data: pointer to message data.
+ * @len: size, in bytes, of the data array.
+ *
+ * This function copies the VF's PF->VF mailbox into the array pointed to by
+ * data.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * EINVAL - len >= DLB_PF2VF_RESP_BYTES.
+ */
+int dlb_vf_read_pf_mbox_resp(struct dlb_hw *hw, void *data, int len);
+
+/**
+ * dlb_vf_read_pf_mbox_req() - (VF only) read a PF->VF mailbox request
+ * @hw: dlb_hw handle for a particular device.
+ * @data: pointer to message data.
+ * @len: size, in bytes, of the data array.
+ *
+ * This function copies the VF's PF->VF mailbox into the array pointed to by
+ * data.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * EINVAL - len >= DLB_PF2VF_REQ_BYTES.
+ */
+int dlb_vf_read_pf_mbox_req(struct dlb_hw *hw, void *data, int len);
+
+/**
+ * dlb_vf_write_pf_mbox_req() - (VF only) write a VF->PF mailbox request
+ * @hw: dlb_hw handle for a particular device.
+ * @data: pointer to message data.
+ * @len: size, in bytes, of the data array.
+ *
+ * This function copies the user-provided message data into of the VF's PF->VF
+ * mailboxes.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * EINVAL - len >= DLB_VF2PF_REQ_BYTES.
+ */
+int dlb_vf_write_pf_mbox_req(struct dlb_hw *hw, void *data, int len);
+
+/**
+ * dlb_vf_write_pf_mbox_resp() - (VF only) write a VF->PF mailbox response
+ * @hw: dlb_hw handle for a particular device.
+ * @data: pointer to message data.
+ * @len: size, in bytes, of the data array.
+ *
+ * This function copies the user-provided message data into of the VF's PF->VF
+ * mailboxes.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * EINVAL - len >= DLB_VF2PF_RESP_BYTES.
+ */
+int dlb_vf_write_pf_mbox_resp(struct dlb_hw *hw, void *data, int len);
+
+/**
+ * dlb_reset_vf() - reset the hardware owned by a VF
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID
+ *
+ * This function resets the hardware owned by a VF (if any), by resetting the
+ * VF's domains one by one.
+ */
+int dlb_reset_vf(struct dlb_hw *hw, unsigned int vf_id);
+
+/**
+ * dlb_vf_is_locked() - check whether the VF's resources are locked
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID
+ *
+ * This function returns whether or not the VF's resource assignments are
+ * locked. If locked, no resources can be added to or subtracted from the
+ * group.
+ */
+bool dlb_vf_is_locked(struct dlb_hw *hw, unsigned int vf_id);
+
+/**
+ * dlb_lock_vf() - lock the VF's resources
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID
+ *
+ * This function sets a flag indicating that the VF is using its resources.
+ * When VF is locked, its resource assignment cannot be changed.
+ */
+void dlb_lock_vf(struct dlb_hw *hw, unsigned int vf_id);
+
+/**
+ * dlb_unlock_vf() - unlock the VF's resources
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID
+ *
+ * This function unlocks the VF's resource assignment, allowing it to be
+ * modified.
+ */
+void dlb_unlock_vf(struct dlb_hw *hw, unsigned int vf_id);
+
+/**
+ * dlb_update_vf_sched_domains() - update the domains assigned to a VF
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID
+ * @num: number of scheduling domains to assign to this VF
+ *
+ * This function assigns num scheduling domains to the specified VF. If the VF
+ * already has domains assigned, this existing assignment is adjusted
+ * accordingly.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - vf_id is invalid, or the requested number of resources are
+ *	    unavailable.
+ * EPERM  - The VF's resource assignment is locked and cannot be changed.
+ */
+int dlb_update_vf_sched_domains(struct dlb_hw *hw,
+				u32 vf_id,
+				u32 num);
+
+/**
+ * dlb_update_vf_ldb_queues() - update the LDB queues assigned to a VF
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID
+ * @num: number of LDB queues to assign to this VF
+ *
+ * This function assigns num LDB queues to the specified VF. If the VF already
+ * has LDB queues assigned, this existing assignment is adjusted
+ * accordingly.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - vf_id is invalid, or the requested number of resources are
+ *	    unavailable.
+ * EPERM  - The VF's resource assignment is locked and cannot be changed.
+ */
+int dlb_update_vf_ldb_queues(struct dlb_hw *hw, u32 vf_id, u32 num);
+
+/**
+ * dlb_update_vf_ldb_ports() - update the LDB ports assigned to a VF
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID
+ * @num: number of LDB ports to assign to this VF
+ *
+ * This function assigns num LDB ports to the specified VF. If the VF already
+ * has LDB ports assigned, this existing assignment is adjusted accordingly.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - vf_id is invalid, or the requested number of resources are
+ *	    unavailable.
+ * EPERM  - The VF's resource assignment is locked and cannot be changed.
+ */
+int dlb_update_vf_ldb_ports(struct dlb_hw *hw, u32 vf_id, u32 num);
+
+/**
+ * dlb_update_vf_dir_ports() - update the DIR ports assigned to a VF
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID
+ * @num: number of DIR ports to assign to this VF
+ *
+ * This function assigns num DIR ports to the specified VF. If the VF already
+ * has DIR ports assigned, this existing assignment is adjusted accordingly.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - vf_id is invalid, or the requested number of resources are
+ *	    unavailable.
+ * EPERM  - The VF's resource assignment is locked and cannot be changed.
+ */
+int dlb_update_vf_dir_ports(struct dlb_hw *hw, u32 vf_id, u32 num);
+
+/**
+ * dlb_update_vf_ldb_credit_pools() - update the VF's assigned LDB pools
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID
+ * @num: number of LDB credit pools to assign to this VF
+ *
+ * This function assigns num LDB credit pools to the specified VF. If the VF
+ * already has LDB credit pools assigned, this existing assignment is adjusted
+ * accordingly.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - vf_id is invalid, or the requested number of resources are
+ *	    unavailable.
+ * EPERM  - The VF's resource assignment is locked and cannot be changed.
+ */
+int dlb_update_vf_ldb_credit_pools(struct dlb_hw *hw,
+				   u32 vf_id,
+				   u32 num);
+
+/**
+ * dlb_update_vf_dir_credit_pools() - update the VF's assigned DIR pools
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID
+ * @num: number of DIR credit pools to assign to this VF
+ *
+ * This function assigns num DIR credit pools to the specified VF. If the VF
+ * already has DIR credit pools assigned, this existing assignment is adjusted
+ * accordingly.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - vf_id is invalid, or the requested number of resources are
+ *	    unavailable.
+ * EPERM  - The VF's resource assignment is locked and cannot be changed.
+ */
+int dlb_update_vf_dir_credit_pools(struct dlb_hw *hw,
+				   u32 vf_id,
+				   u32 num);
+
+/**
+ * dlb_update_vf_ldb_credits() - update the VF's assigned LDB credits
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID
+ * @num: number of LDB credits to assign to this VF
+ *
+ * This function assigns num LDB credits to the specified VF. If the VF already
+ * has LDB credits assigned, this existing assignment is adjusted accordingly.
+ * VF's are assigned a contiguous chunk of credits, so this function may fail
+ * if a sufficiently large contiguous chunk is not available.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - vf_id is invalid, or the requested number of resources are
+ *	    unavailable.
+ * EPERM  - The VF's resource assignment is locked and cannot be changed.
+ */
+int dlb_update_vf_ldb_credits(struct dlb_hw *hw, u32 vf_id, u32 num);
+
+/**
+ * dlb_update_vf_dir_credits() - update the VF's assigned DIR credits
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID
+ * @num: number of DIR credits to assign to this VF
+ *
+ * This function assigns num DIR credits to the specified VF. If the VF already
+ * has DIR credits assigned, this existing assignment is adjusted accordingly.
+ * VF's are assigned a contiguous chunk of credits, so this function may fail
+ * if a sufficiently large contiguous chunk is not available.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - vf_id is invalid, or the requested number of resources are
+ *	    unavailable.
+ * EPERM  - The VF's resource assignment is locked and cannot be changed.
+ */
+int dlb_update_vf_dir_credits(struct dlb_hw *hw, u32 vf_id, u32 num);
+
+/**
+ * dlb_update_vf_hist_list_entries() - update the VF's assigned HL entries
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID
+ * @num: number of history list entries to assign to this VF
+ *
+ * This function assigns num history list entries to the specified VF. If the
+ * VF already has history list entries assigned, this existing assignment is
+ * adjusted accordingly. VF's are assigned a contiguous chunk of entries, so
+ * this function may fail if a sufficiently large contiguous chunk is not
+ * available.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - vf_id is invalid, or the requested number of resources are
+ *	    unavailable.
+ * EPERM  - The VF's resource assignment is locked and cannot be changed.
+ */
+int dlb_update_vf_hist_list_entries(struct dlb_hw *hw,
+				    u32 vf_id,
+				    u32 num);
+
+/**
+ * dlb_update_vf_atomic_inflights() - update the VF's atomic inflights
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID
+ * @num: number of atomic inflights to assign to this VF
+ *
+ * This function assigns num atomic inflights to the specified VF. If the VF
+ * already has atomic inflights assigned, this existing assignment is adjusted
+ * accordingly. VF's are assigned a contiguous chunk of entries, so this
+ * function may fail if a sufficiently large contiguous chunk is not available.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - vf_id is invalid, or the requested number of resources are
+ *	    unavailable.
+ * EPERM  - The VF's resource assignment is locked and cannot be changed.
+ */
+int dlb_update_vf_atomic_inflights(struct dlb_hw *hw,
+				   u32 vf_id,
+				   u32 num);
+
+/**
+ * dlb_reset_vf_resources() - reassign the VF's resources to the PF
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID
+ *
+ * This function takes any resources currently assigned to the VF and reassigns
+ * them to the PF.
+ *
+ * Return:
+ * Returns 0 upon success, <0 otherwise.
+ *
+ * Errors:
+ * EINVAL - vf_id is invalid
+ * EPERM  - The VF's resource assignment is locked and cannot be changed.
+ */
+int dlb_reset_vf_resources(struct dlb_hw *hw, unsigned int vf_id);
+
+/**
+ * dlb_notify_vf() - send a notification to a VF
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID
+ * @notification: notification
+ *
+ * This function sends a notification (as defined in dlb_mbox.h) to a VF.
+ *
+ * Return:
+ * Returns 0 upon success, -1 if the VF doesn't ACK the PF->VF interrupt.
+ */
+int dlb_notify_vf(struct dlb_hw *hw,
+		  unsigned int vf_id,
+		  u32 notification);
+
+/**
+ * dlb_vf_in_use() - query whether a VF is in use
+ * @hw: dlb_hw handle for a particular device.
+ * @vf_id: VF ID
+ *
+ * This function sends a mailbox request to the VF to query whether the VF is in
+ * use.
+ *
+ * Return:
+ * Returns 0 for false, 1 for true, and -1 if the mailbox request times out or
+ * an internal error occurs.
+ */
+int dlb_vf_in_use(struct dlb_hw *hw, unsigned int vf_id);
+
+/**
+ * dlb_disable_dp_vasr_feature() - disable directed pipe VAS reset hardware
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function disables certain hardware in the directed pipe,
+ * necessary to workaround a DLB VAS reset issue.
+ */
+void dlb_disable_dp_vasr_feature(struct dlb_hw *hw);
+
+/**
+ * dlb_enable_excess_tokens_alarm() - enable interrupts for the excess token
+ * pop alarm
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function enables the PF ingress error alarm interrupt to fire when an
+ * excess token pop occurs.
+ */
+void dlb_enable_excess_tokens_alarm(struct dlb_hw *hw);
+
+/**
+ * dlb_disable_excess_tokens_alarm() - disable interrupts for the excess token
+ * pop alarm
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function disables the PF ingress error alarm interrupt to fire when an
+ * excess token pop occurs.
+ */
+void dlb_disable_excess_tokens_alarm(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_get_ldb_queue_depth() - returns the depth of a load-balanced queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue depth args
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function returns the depth of a load-balanced queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the depth.
+ *
+ * Errors:
+ * EINVAL - Invalid domain ID or queue ID.
+ */
+int dlb_hw_get_ldb_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_ldb_queue_depth_args *args,
+			       struct dlb_cmd_response *resp,
+			       bool vf_request,
+			       unsigned int vf_id);
+
+/**
+ * dlb_hw_get_dir_queue_depth() - returns the depth of a directed queue
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: queue depth args
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * This function returns the depth of a directed queue.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the depth.
+ *
+ * Errors:
+ * EINVAL - Invalid domain ID or queue ID.
+ */
+int dlb_hw_get_dir_queue_depth(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_get_dir_queue_depth_args *args,
+			       struct dlb_cmd_response *resp,
+			       bool vf_request,
+			       unsigned int vf_id);
+
+/**
+ * dlb_hw_pending_port_unmaps() - returns the number of unmap operations in
+ *	progress for a load-balanced port.
+ * @hw: dlb_hw handle for a particular device.
+ * @domain_id: domain ID.
+ * @args: number of unmaps in progress args
+ * @vf_request: indicates whether this request came from a VF.
+ * @vf_id: If vf_request is true, this contains the VF's ID.
+ *
+ * Return:
+ * Returns 0 upon success, < 0 otherwise. If an error occurs, resp->status is
+ * assigned a detailed error code from enum dlb_error. If successful, resp->id
+ * contains the number of unmaps in progress.
+ *
+ * Errors:
+ * EINVAL - Invalid port ID.
+ */
+int dlb_hw_pending_port_unmaps(struct dlb_hw *hw,
+			       u32 domain_id,
+			       struct dlb_pending_port_unmaps_args *args,
+			       struct dlb_cmd_response *resp,
+			       bool vf_request,
+			       unsigned int vf_id);
+
+/**
+ * dlb_hw_enable_sparse_ldb_cq_mode() - enable sparse mode for load-balanced
+ *	ports.
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function must be called prior to configuring scheduling domains.
+ */
+void dlb_hw_enable_sparse_ldb_cq_mode(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_enable_sparse_dir_cq_mode() - enable sparse mode for directed ports
+ * @hw: dlb_hw handle for a particular device.
+ *
+ * This function must be called prior to configuring scheduling domains.
+ */
+void dlb_hw_enable_sparse_dir_cq_mode(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_set_qe_arbiter_weights() - program QE arbiter weights
+ * @hw: dlb_hw handle for a particular device.
+ * @weight: 8-entry array of arbiter weights.
+ *
+ * weight[N] programs priority N's weight. In cases where the 8 priorities are
+ * reduced to 4 bins, the mapping is:
+ * - weight[1] programs bin 0
+ * - weight[3] programs bin 1
+ * - weight[5] programs bin 2
+ * - weight[7] programs bin 3
+ */
+void dlb_hw_set_qe_arbiter_weights(struct dlb_hw *hw, u8 weight[8]);
+
+/**
+ * dlb_hw_set_qid_arbiter_weights() - program QID arbiter weights
+ * @hw: dlb_hw handle for a particular device.
+ * @weight: 8-entry array of arbiter weights.
+ *
+ * weight[N] programs priority N's weight. In cases where the 8 priorities are
+ * reduced to 4 bins, the mapping is:
+ * - weight[1] programs bin 0
+ * - weight[3] programs bin 1
+ * - weight[5] programs bin 2
+ * - weight[7] programs bin 3
+ */
+void dlb_hw_set_qid_arbiter_weights(struct dlb_hw *hw, u8 weight[8]);
+
+/**
+ * dlb_hw_enable_pp_sw_alarms() - enable out-of-credit alarm for all producer
+ * ports
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_enable_pp_sw_alarms(struct dlb_hw *hw);
+
+/**
+ * dlb_hw_disable_pp_sw_alarms() - disable out-of-credit alarm for all producer
+ * ports
+ * @hw: dlb_hw handle for a particular device.
+ */
+void dlb_hw_disable_pp_sw_alarms(struct dlb_hw *hw);
+
+#endif /* __DLB_RESOURCE_H */
diff --git a/drivers/event/dlb/pf/base/dlb_user.h b/drivers/event/dlb/pf/base/dlb_user.h
new file mode 100644
index 0000000..6e7ee2e
--- /dev/null
+++ b/drivers/event/dlb/pf/base/dlb_user.h
@@ -0,0 +1,1084 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause)
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_USER_H
+#define __DLB_USER_H
+
+#define DLB_MAX_NAME_LEN 64
+
+#include "dlb_osdep_types.h"
+
+enum dlb_error {
+	DLB_ST_SUCCESS = 0,
+	DLB_ST_NAME_EXISTS,
+	DLB_ST_DOMAIN_UNAVAILABLE,
+	DLB_ST_LDB_PORTS_UNAVAILABLE,
+	DLB_ST_DIR_PORTS_UNAVAILABLE,
+	DLB_ST_LDB_QUEUES_UNAVAILABLE,
+	DLB_ST_LDB_CREDITS_UNAVAILABLE,
+	DLB_ST_DIR_CREDITS_UNAVAILABLE,
+	DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE,
+	DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE,
+	DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE,
+	DLB_ST_INVALID_DOMAIN_ID,
+	DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION,
+	DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE,
+	DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_INVALID_LDB_CREDIT_POOL_ID,
+	DLB_ST_INVALID_DIR_CREDIT_POOL_ID,
+	DLB_ST_INVALID_POP_COUNT_VIRT_ADDR,
+	DLB_ST_INVALID_LDB_QUEUE_ID,
+	DLB_ST_INVALID_CQ_DEPTH,
+	DLB_ST_INVALID_CQ_VIRT_ADDR,
+	DLB_ST_INVALID_PORT_ID,
+	DLB_ST_INVALID_QID,
+	DLB_ST_INVALID_PRIORITY,
+	DLB_ST_NO_QID_SLOTS_AVAILABLE,
+	DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_INVALID_DIR_QUEUE_ID,
+	DLB_ST_DIR_QUEUES_UNAVAILABLE,
+	DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK,
+	DLB_ST_INVALID_LDB_CREDIT_QUANTUM,
+	DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK,
+	DLB_ST_INVALID_DIR_CREDIT_QUANTUM,
+	DLB_ST_DOMAIN_NOT_CONFIGURED,
+	DLB_ST_PID_ALREADY_ATTACHED,
+	DLB_ST_PID_NOT_ATTACHED,
+	DLB_ST_INTERNAL_ERROR,
+	DLB_ST_DOMAIN_IN_USE,
+	DLB_ST_IOMMU_MAPPING_ERROR,
+	DLB_ST_FAIL_TO_PIN_MEMORY_PAGE,
+	DLB_ST_UNABLE_TO_PIN_POPCOUNT_PAGES,
+	DLB_ST_UNABLE_TO_PIN_CQ_PAGES,
+	DLB_ST_DISCONTIGUOUS_CQ_MEMORY,
+	DLB_ST_DISCONTIGUOUS_POP_COUNT_MEMORY,
+	DLB_ST_DOMAIN_STARTED,
+	DLB_ST_LARGE_POOL_NOT_SPECIFIED,
+	DLB_ST_SMALL_POOL_NOT_SPECIFIED,
+	DLB_ST_NEITHER_POOL_SPECIFIED,
+	DLB_ST_DOMAIN_NOT_STARTED,
+	DLB_ST_INVALID_MEASUREMENT_DURATION,
+	DLB_ST_INVALID_PERF_METRIC_GROUP_ID,
+	DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES,
+	DLB_ST_DOMAIN_RESET_FAILED,
+	DLB_ST_MBOX_ERROR,
+	DLB_ST_INVALID_HIST_LIST_DEPTH,
+	DLB_ST_NO_MEMORY,
+};
+
+static const char dlb_error_strings[][128] = {
+	"DLB_ST_SUCCESS",
+	"DLB_ST_NAME_EXISTS",
+	"DLB_ST_DOMAIN_UNAVAILABLE",
+	"DLB_ST_LDB_PORTS_UNAVAILABLE",
+	"DLB_ST_DIR_PORTS_UNAVAILABLE",
+	"DLB_ST_LDB_QUEUES_UNAVAILABLE",
+	"DLB_ST_LDB_CREDITS_UNAVAILABLE",
+	"DLB_ST_DIR_CREDITS_UNAVAILABLE",
+	"DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE",
+	"DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE",
+	"DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE",
+	"DLB_ST_INVALID_DOMAIN_ID",
+	"DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION",
+	"DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE",
+	"DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_INVALID_LDB_CREDIT_POOL_ID",
+	"DLB_ST_INVALID_DIR_CREDIT_POOL_ID",
+	"DLB_ST_INVALID_POP_COUNT_VIRT_ADDR",
+	"DLB_ST_INVALID_LDB_QUEUE_ID",
+	"DLB_ST_INVALID_CQ_DEPTH",
+	"DLB_ST_INVALID_CQ_VIRT_ADDR",
+	"DLB_ST_INVALID_PORT_ID",
+	"DLB_ST_INVALID_QID",
+	"DLB_ST_INVALID_PRIORITY",
+	"DLB_ST_NO_QID_SLOTS_AVAILABLE",
+	"DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_INVALID_DIR_QUEUE_ID",
+	"DLB_ST_DIR_QUEUES_UNAVAILABLE",
+	"DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK",
+	"DLB_ST_INVALID_LDB_CREDIT_QUANTUM",
+	"DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK",
+	"DLB_ST_INVALID_DIR_CREDIT_QUANTUM",
+	"DLB_ST_DOMAIN_NOT_CONFIGURED",
+	"DLB_ST_PID_ALREADY_ATTACHED",
+	"DLB_ST_PID_NOT_ATTACHED",
+	"DLB_ST_INTERNAL_ERROR",
+	"DLB_ST_DOMAIN_IN_USE",
+	"DLB_ST_IOMMU_MAPPING_ERROR",
+	"DLB_ST_FAIL_TO_PIN_MEMORY_PAGE",
+	"DLB_ST_UNABLE_TO_PIN_POPCOUNT_PAGES",
+	"DLB_ST_UNABLE_TO_PIN_CQ_PAGES",
+	"DLB_ST_DISCONTIGUOUS_CQ_MEMORY",
+	"DLB_ST_DISCONTIGUOUS_POP_COUNT_MEMORY",
+	"DLB_ST_DOMAIN_STARTED",
+	"DLB_ST_LARGE_POOL_NOT_SPECIFIED",
+	"DLB_ST_SMALL_POOL_NOT_SPECIFIED",
+	"DLB_ST_NEITHER_POOL_SPECIFIED",
+	"DLB_ST_DOMAIN_NOT_STARTED",
+	"DLB_ST_INVALID_MEASUREMENT_DURATION",
+	"DLB_ST_INVALID_PERF_METRIC_GROUP_ID",
+	"DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES",
+	"DLB_ST_DOMAIN_RESET_FAILED",
+	"DLB_ST_MBOX_ERROR",
+	"DLB_ST_INVALID_HIST_LIST_DEPTH",
+	"DLB_ST_NO_MEMORY",
+};
+
+struct dlb_cmd_response {
+	__u32 status; /* Interpret using enum dlb_error */
+	__u32 id;
+};
+
+/******************************/
+/* 'dlb' device file commands */
+/******************************/
+
+#define DLB_DEVICE_VERSION(x) (((x) >> 8) & 0xFF)
+#define DLB_DEVICE_REVISION(x) ((x) & 0xFF)
+
+enum dlb_revisions {
+	DLB_REV_A0 = 0,
+	DLB_REV_A1 = 1,
+	DLB_REV_A2 = 2,
+	DLB_REV_A3 = 3,
+	DLB_REV_B0 = 4,
+};
+
+/*
+ * DLB_CMD_GET_DEVICE_VERSION: Query the DLB device version.
+ *
+ *	This ioctl interface is the same in all driver versions and is always
+ *	the first ioctl.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id[7:0]: Device revision.
+ *	response.id[15:8]: Device version.
+ */
+
+struct dlb_get_device_version_args {
+	/* Output parameters */
+	__u64 response;
+};
+
+#define DLB_VERSION_MAJOR_NUMBER 10
+#define DLB_VERSION_MINOR_NUMBER 7
+#define DLB_VERSION_REVISION_NUMBER 9
+#define DLB_VERSION (DLB_VERSION_MAJOR_NUMBER << 24 | \
+		     DLB_VERSION_MINOR_NUMBER << 16 | \
+		     DLB_VERSION_REVISION_NUMBER)
+
+#define DLB_VERSION_GET_MAJOR_NUMBER(x) (((x) >> 24) & 0xFF)
+#define DLB_VERSION_GET_MINOR_NUMBER(x) (((x) >> 16) & 0xFF)
+#define DLB_VERSION_GET_REVISION_NUMBER(x) ((x) & 0xFFFF)
+
+static inline __u8 dlb_version_incompatible(__u32 version)
+{
+	__u8 inc;
+
+	inc = DLB_VERSION_GET_MAJOR_NUMBER(version) != DLB_VERSION_MAJOR_NUMBER;
+	inc |= (int)DLB_VERSION_GET_MINOR_NUMBER(version) <
+		DLB_VERSION_MINOR_NUMBER;
+
+	return inc;
+}
+
+/*
+ * DLB_CMD_GET_DRIVER_VERSION: Query the DLB driver version. The major number
+ *	is changed when there is an ABI-breaking change, the minor number is
+ *	changed if the API is changed in a backwards-compatible way, and the
+ *	revision number is changed for fixes that don't affect the API.
+ *
+ *	If the kernel driver's API version major number and the header's
+ *	DLB_VERSION_MAJOR_NUMBER differ, the two are incompatible, or if the
+ *	major numbers match but the kernel driver's minor number is less than
+ *	the header file's, they are incompatible. The DLB_VERSION_INCOMPATIBLE
+ *	macro should be used to check for compatibility.
+ *
+ *	This ioctl interface is the same in all driver versions. Applications
+ *	should check the driver version before performing any other ioctl
+ *	operations.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Driver API version. Use the DLB_VERSION_GET_MAJOR_NUMBER,
+ *		DLB_VERSION_GET_MINOR_NUMBER, and
+ *		DLB_VERSION_GET_REVISION_NUMBER macros to interpret the field.
+ */
+
+struct dlb_get_driver_version_args {
+	/* Output parameters */
+	__u64 response;
+};
+
+/*
+ * DLB_CMD_CREATE_SCHED_DOMAIN: Create a DLB scheduling domain and reserve the
+ *	resources (queues, ports, etc.) that it contains.
+ *
+ * Input parameters:
+ * - num_ldb_queues: Number of load-balanced queues.
+ * - num_ldb_ports: Number of load-balanced ports.
+ * - num_dir_ports: Number of directed ports. A directed port has one directed
+ *	queue, so no num_dir_queues argument is necessary.
+ * - num_atomic_inflights: This specifies the amount of temporary atomic QE
+ *	storage for the domain. This storage is divided among the domain's
+ *	load-balanced queues that are configured for atomic scheduling.
+ * - num_hist_list_entries: Amount of history list storage. This is divided
+ *	among the domain's CQs.
+ * - num_ldb_credits: Amount of load-balanced QE storage (QED). QEs occupy this
+ *	space until they are scheduled to a load-balanced CQ. One credit
+ *	represents the storage for one QE.
+ * - num_dir_credits: Amount of directed QE storage (DQED). QEs occupy this
+ *	space until they are scheduled to a directed CQ. One credit represents
+ *	the storage for one QE.
+ * - num_ldb_credit_pools: Number of pools into which the load-balanced credits
+ *	are placed.
+ * - num_dir_credit_pools: Number of pools into which the directed credits are
+ *	placed.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: domain ID.
+ */
+struct dlb_create_sched_domain_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_ldb_queues;
+	__u32 num_ldb_ports;
+	__u32 num_dir_ports;
+	__u32 num_atomic_inflights;
+	__u32 num_hist_list_entries;
+	__u32 num_ldb_credits;
+	__u32 num_dir_credits;
+	__u32 num_ldb_credit_pools;
+	__u32 num_dir_credit_pools;
+};
+
+/*
+ * DLB_CMD_GET_NUM_RESOURCES: Return the number of available resources
+ *	(queues, ports, etc.) that this device owns.
+ *
+ * Output parameters:
+ * - num_domains: Number of available scheduling domains.
+ * - num_ldb_queues: Number of available load-balanced queues.
+ * - num_ldb_ports: Number of available load-balanced ports.
+ * - num_dir_ports: Number of available directed ports. There is one directed
+ *	queue for every directed port.
+ * - num_atomic_inflights: Amount of available temporary atomic QE storage.
+ * - max_contiguous_atomic_inflights: When a domain is created, the temporary
+ *	atomic QE storage is allocated in a contiguous chunk. This return value
+ *	is the longest available contiguous range of atomic QE storage.
+ * - num_hist_list_entries: Amount of history list storage.
+ * - max_contiguous_hist_list_entries: History list storage is allocated in
+ *	a contiguous chunk, and this return value is the longest available
+ *	contiguous range of history list entries.
+ * - num_ldb_credits: Amount of available load-balanced QE storage.
+ * - max_contiguous_ldb_credits: QED storage is allocated in a contiguous
+ *	chunk, and this return value is the longest available contiguous range
+ *	of load-balanced credit storage.
+ * - num_dir_credits: Amount of available directed QE storage.
+ * - max_contiguous_dir_credits: DQED storage is allocated in a contiguous
+ *	chunk, and this return value is the longest available contiguous range
+ *	of directed credit storage.
+ * - num_ldb_credit_pools: Number of available load-balanced credit pools.
+ * - num_dir_credit_pools: Number of available directed credit pools.
+ * - padding0: Reserved for future use.
+ */
+struct dlb_get_num_resources_args {
+	/* Output parameters */
+	__u32 num_sched_domains;
+	__u32 num_ldb_queues;
+	__u32 num_ldb_ports;
+	__u32 num_dir_ports;
+	__u32 num_atomic_inflights;
+	__u32 max_contiguous_atomic_inflights;
+	__u32 num_hist_list_entries;
+	__u32 max_contiguous_hist_list_entries;
+	__u32 num_ldb_credits;
+	__u32 max_contiguous_ldb_credits;
+	__u32 num_dir_credits;
+	__u32 max_contiguous_dir_credits;
+	__u32 num_ldb_credit_pools;
+	__u32 num_dir_credit_pools;
+	__u32 padding0;
+};
+
+/*
+ * DLB_CMD_SET_SN_ALLOCATION: Configure a sequence number group
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - num: Number of sequence numbers per queue.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_set_sn_allocation_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 num;
+};
+
+/*
+ * DLB_CMD_GET_SN_ALLOCATION: Get a sequence number group's configuration
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Specified group's number of sequence numbers per queue.
+ */
+struct dlb_get_sn_allocation_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 padding0;
+};
+
+/*
+ * DLB_CMD_QUERY_CQ_POLL_MODE: Query the CQ poll mode the kernel driver is using
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: CQ poll mode (see enum dlb_cq_poll_modes).
+ */
+struct dlb_query_cq_poll_mode_args {
+	/* Output parameters */
+	__u64 response;
+};
+
+enum dlb_cq_poll_modes {
+	DLB_CQ_POLL_MODE_STD,
+	DLB_CQ_POLL_MODE_SPARSE,
+
+	/* NUM_DLB_CQ_POLL_MODE must be last */
+	NUM_DLB_CQ_POLL_MODE,
+};
+
+/*
+ * DLB_CMD_GET_SN_OCCUPANCY: Get a sequence number group's occupancy
+ *
+ * Each sequence number group has one or more slots, depending on its
+ * configuration. I.e.:
+ * - If configured for 1024 sequence numbers per queue, the group has 1 slot
+ * - If configured for 512 sequence numbers per queue, the group has 2 slots
+ *   ...
+ * - If configured for 32 sequence numbers per queue, the group has 32 slots
+ *
+ * This ioctl returns the group's number of in-use slots. If its occupancy is
+ * 0, the group's sequence number allocation can be reconfigured.
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Specified group's number of used slots.
+ */
+struct dlb_get_sn_occupancy_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 padding0;
+};
+
+enum dlb_user_interface_commands {
+	DLB_CMD_GET_DEVICE_VERSION,
+	DLB_CMD_CREATE_SCHED_DOMAIN,
+	DLB_CMD_GET_NUM_RESOURCES,
+	DLB_CMD_GET_DRIVER_VERSION,
+	DLB_CMD_SAMPLE_PERF_COUNTERS,
+	DLB_CMD_SET_SN_ALLOCATION,
+	DLB_CMD_GET_SN_ALLOCATION,
+	DLB_CMD_MEASURE_SCHED_COUNTS,
+	DLB_CMD_QUERY_CQ_POLL_MODE,
+	DLB_CMD_GET_SN_OCCUPANCY,
+
+	/* NUM_DLB_CMD must be last */
+	NUM_DLB_CMD,
+};
+
+/*******************************/
+/* 'domain' device file alerts */
+/*******************************/
+
+/* Scheduling domain device files can be read to receive domain-specific
+ * notifications, for alerts such as hardware errors.
+ *
+ * Each alert is encoded in a 16B message. The first 8B contains the alert ID,
+ * and the second 8B is optional and contains additional information.
+ * Applications should cast read data to a struct dlb_domain_alert, and
+ * interpret the struct's alert_id according to dlb_domain_alert_id. The read
+ * length must be 16B, or the function will return -EINVAL.
+ *
+ * Reads are destructive, and in the case of multiple file descriptors for the
+ * same domain device file, an alert will be read by only one of the file
+ * descriptors.
+ *
+ * The driver stores alerts in a fixed-size alert ring until they are read. If
+ * the alert ring fills completely, subsequent alerts will be dropped. It is
+ * recommended that DLB applications dedicate a thread to perform blocking
+ * reads on the device file.
+ */
+enum dlb_domain_alert_id {
+	/* A destination domain queue that this domain connected to has
+	 * unregistered, and can no longer be sent to. The aux alert data
+	 * contains the queue ID.
+	 */
+	DLB_DOMAIN_ALERT_REMOTE_QUEUE_UNREGISTER,
+	/* A producer port in this domain attempted to send a QE without a
+	 * credit. aux_alert_data[7:0] contains the port ID, and
+	 * aux_alert_data[15:8] contains a flag indicating whether the port is
+	 * load-balanced (1) or directed (0).
+	 */
+	DLB_DOMAIN_ALERT_PP_OUT_OF_CREDITS,
+	/* Software issued an illegal enqueue for a port in this domain. An
+	 * illegal enqueue could be:
+	 * - Illegal (excess) completion
+	 * - Illegal fragment
+	 * - Illegal enqueue command
+	 * aux_alert_data[7:0] contains the port ID, and aux_alert_data[15:8]
+	 * contains a flag indicating whether the port is load-balanced (1) or
+	 * directed (0).
+	 */
+	DLB_DOMAIN_ALERT_PP_ILLEGAL_ENQ,
+	/* Software issued excess CQ token pops for a port in this domain.
+	 * aux_alert_data[7:0] contains the port ID, and aux_alert_data[15:8]
+	 * contains a flag indicating whether the port is load-balanced (1) or
+	 * directed (0).
+	 */
+	DLB_DOMAIN_ALERT_PP_EXCESS_TOKEN_POPS,
+	/* A enqueue contained either an invalid command encoding or a REL,
+	 * REL_T, RLS, FWD, FWD_T, FRAG, or FRAG_T from a directed port.
+	 *
+	 * aux_alert_data[7:0] contains the port ID, and aux_alert_data[15:8]
+	 * contains a flag indicating whether the port is load-balanced (1) or
+	 * directed (0).
+	 */
+	DLB_DOMAIN_ALERT_ILLEGAL_HCW,
+	/* The QID must be valid and less than 128.
+	 *
+	 * aux_alert_data[7:0] contains the port ID, and aux_alert_data[15:8]
+	 * contains a flag indicating whether the port is load-balanced (1) or
+	 * directed (0).
+	 */
+	DLB_DOMAIN_ALERT_ILLEGAL_QID,
+	/* An enqueue went to a disabled QID.
+	 *
+	 * aux_alert_data[7:0] contains the port ID, and aux_alert_data[15:8]
+	 * contains a flag indicating whether the port is load-balanced (1) or
+	 * directed (0).
+	 */
+	DLB_DOMAIN_ALERT_DISABLED_QID,
+	/* The device containing this domain was reset. All applications using
+	 * the device need to exit for the driver to complete the reset
+	 * procedure.
+	 *
+	 * aux_alert_data doesn't contain any information for this alert.
+	 */
+	DLB_DOMAIN_ALERT_DEVICE_RESET,
+	/* User-space has enqueued an alert.
+	 *
+	 * aux_alert_data contains user-provided data.
+	 */
+	DLB_DOMAIN_ALERT_USER,
+
+	/* Number of DLB domain alerts */
+	NUM_DLB_DOMAIN_ALERTS
+};
+
+static const char dlb_domain_alert_strings[][128] = {
+	"DLB_DOMAIN_ALERT_REMOTE_QUEUE_UNREGISTER",
+	"DLB_DOMAIN_ALERT_PP_OUT_OF_CREDITS",
+	"DLB_DOMAIN_ALERT_PP_ILLEGAL_ENQ",
+	"DLB_DOMAIN_ALERT_PP_EXCESS_TOKEN_POPS",
+	"DLB_DOMAIN_ALERT_ILLEGAL_HCW",
+	"DLB_DOMAIN_ALERT_ILLEGAL_QID",
+	"DLB_DOMAIN_ALERT_DISABLED_QID",
+	"DLB_DOMAIN_ALERT_DEVICE_RESET",
+	"DLB_DOMAIN_ALERT_USER",
+};
+
+struct dlb_domain_alert {
+	__u64 alert_id;
+	__u64 aux_alert_data;
+};
+
+/*********************************/
+/* 'domain' device file commands */
+/*********************************/
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_POOL: Configure a load-balanced credit pool.
+ * Input parameters:
+ * - num_ldb_credits: Number of load-balanced credits (QED space) for this
+ *	pool.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: pool ID.
+ */
+struct dlb_create_ldb_pool_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_ldb_credits;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_POOL: Configure a directed credit pool.
+ * Input parameters:
+ * - num_dir_credits: Number of directed credits (DQED space) for this pool.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Pool ID.
+ */
+struct dlb_create_dir_pool_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_dir_credits;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_QUEUE: Configure a load-balanced queue.
+ * Input parameters:
+ * - num_atomic_inflights: This specifies the amount of temporary atomic QE
+ *	storage for this queue. If zero, the queue will not support atomic
+ *	scheduling.
+ * - num_sequence_numbers: This specifies the number of sequence numbers used
+ *	by this queue. If zero, the queue will not support ordered scheduling.
+ *	If non-zero, the queue will not support unordered scheduling.
+ * - num_qid_inflights: The maximum number of QEs that can be inflight
+ *	(scheduled to a CQ but not completed) at any time. If
+ *	num_sequence_numbers is non-zero, num_qid_inflights must be set equal
+ *	to num_sequence_numbers.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Queue ID.
+ */
+struct dlb_create_ldb_queue_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_sequence_numbers;
+	__u32 num_qid_inflights;
+	__u32 num_atomic_inflights;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_QUEUE: Configure a directed queue.
+ * Input parameters:
+ * - port_id: Port ID. If the corresponding directed port is already created,
+ *	specify its ID here. Else this argument must be 0xFFFFFFFF to indicate
+ *	that the queue is being created before the port.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Queue ID.
+ */
+struct dlb_create_dir_queue_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__s32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_PORT: Configure a load-balanced port.
+ * Input parameters:
+ * - ldb_credit_pool_id: Load-balanced credit pool this port will belong to.
+ * - dir_credit_pool_id: Directed credit pool this port will belong to.
+ * - ldb_credit_high_watermark: Number of load-balanced credits from the pool
+ *	that this port will own.
+ *
+ *	If this port's scheduling domain doesn't have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_high_watermark: Number of directed credits from the pool that
+ *	this port will own.
+ *
+ *	If this port's scheduling domain doesn't have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - ldb_credit_low_watermark: Load-balanced credit low watermark. When the
+ *	port's credits reach this watermark, they become eligible to be
+ *	refilled by the DLB as credits until the high watermark
+ *	(num_ldb_credits) is reached.
+ *
+ *	If this port's scheduling domain doesn't have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_low_watermark: Directed credit low watermark. When the port's
+ *	credits reach this watermark, they become eligible to be refilled by
+ *	the DLB as credits until the high watermark (num_dir_credits) is
+ *	reached.
+ *
+ *	If this port's scheduling domain doesn't have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - ldb_credit_quantum: Number of load-balanced credits for the DLB to refill
+ *	per refill operation.
+ *
+ *	If this port's scheduling domain doesn't have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_quantum: Number of directed credits for the DLB to refill per
+ *	refill operation.
+ *
+ *	If this port's scheduling domain doesn't have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - padding0: Reserved for future use.
+ * - cq_depth: Depth of the port's CQ. Must be a power-of-two between 8 and
+ *	1024, inclusive.
+ * - cq_depth_threshold: CQ depth interrupt threshold. A value of N means that
+ *	the CQ interrupt won't fire until there are N or more outstanding CQ
+ *	tokens.
+ * - cq_history_list_size: Number of history list entries. This must be greater
+ *	than or equal to cq_depth.
+ * - padding1: Reserved for future use.
+ * - padding2: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: port ID.
+ */
+struct dlb_create_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 ldb_credit_pool_id;
+	__u32 dir_credit_pool_id;
+	__u16 ldb_credit_high_watermark;
+	__u16 ldb_credit_low_watermark;
+	__u16 ldb_credit_quantum;
+	__u16 dir_credit_high_watermark;
+	__u16 dir_credit_low_watermark;
+	__u16 dir_credit_quantum;
+	__u16 padding0;
+	__u16 cq_depth;
+	__u16 cq_depth_threshold;
+	__u16 cq_history_list_size;
+	__u32 padding1;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_PORT: Configure a directed port.
+ * Input parameters:
+ * - ldb_credit_pool_id: Load-balanced credit pool this port will belong to.
+ * - dir_credit_pool_id: Directed credit pool this port will belong to.
+ * - ldb_credit_high_watermark: Number of load-balanced credits from the pool
+ *	that this port will own.
+ *
+ *	If this port's scheduling domain doesn't have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_high_watermark: Number of directed credits from the pool that
+ *	this port will own.
+ * - ldb_credit_low_watermark: Load-balanced credit low watermark. When the
+ *	port's credits reach this watermark, they become eligible to be
+ *	refilled by the DLB as credits until the high watermark
+ *	(num_ldb_credits) is reached.
+ *
+ *	If this port's scheduling domain doesn't have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_low_watermark: Directed credit low watermark. When the port's
+ *	credits reach this watermark, they become eligible to be refilled by
+ *	the DLB as credits until the high watermark (num_dir_credits) is
+ *	reached.
+ * - ldb_credit_quantum: Number of load-balanced credits for the DLB to refill
+ *	per refill operation.
+ *
+ *	If this port's scheduling domain doesn't have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_quantum: Number of directed credits for the DLB to refill per
+ *	refill operation.
+ * - cq_depth: Depth of the port's CQ. Must be a power-of-two between 8 and
+ *	1024, inclusive.
+ * - cq_depth_threshold: CQ depth interrupt threshold. A value of N means that
+ *	the CQ interrupt won't fire until there are N or more outstanding CQ
+ *	tokens.
+ * - qid: Queue ID. If the corresponding directed queue is already created,
+ *	specify its ID here. Else this argument must be 0xFFFFFFFF to indicate
+ *	that the port is being created before the queue.
+ * - padding1: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Port ID.
+ */
+struct dlb_create_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 ldb_credit_pool_id;
+	__u32 dir_credit_pool_id;
+	__u16 ldb_credit_high_watermark;
+	__u16 ldb_credit_low_watermark;
+	__u16 ldb_credit_quantum;
+	__u16 dir_credit_high_watermark;
+	__u16 dir_credit_low_watermark;
+	__u16 dir_credit_quantum;
+	__u16 cq_depth;
+	__u16 cq_depth_threshold;
+	__s32 queue_id;
+	__u32 padding1;
+};
+
+/*
+ * DLB_DOMAIN_CMD_START_DOMAIN: Mark the end of the domain configuration. This
+ *	must be called before passing QEs into the device, and no configuration
+ *	ioctls can be issued once the domain has started. Sending QEs into the
+ *	device before calling this ioctl will result in undefined behavior.
+ * Input parameters:
+ * - (None)
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_start_domain_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+};
+
+/*
+ * DLB_DOMAIN_CMD_MAP_QID: Map a load-balanced queue to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - qid: Load-balanced queue ID.
+ * - priority: Queue->port service priority.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_map_qid_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 qid;
+	__u32 priority;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_UNMAP_QID: Unmap a load-balanced queue to a load-balanced
+ *	port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - qid: Load-balanced queue ID.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_unmap_qid_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 qid;
+};
+
+/*
+ * DLB_DOMAIN_CMD_ENABLE_LDB_PORT: Enable scheduling to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_enable_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_ENABLE_DIR_PORT: Enable scheduling to a directed port.
+ * Input parameters:
+ * - port_id: Directed port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_enable_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+};
+
+/*
+ * DLB_DOMAIN_CMD_DISABLE_LDB_PORT: Disable scheduling to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_disable_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_DISABLE_DIR_PORT: Disable scheduling to a directed port.
+ * Input parameters:
+ * - port_id: Directed port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_disable_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_BLOCK_ON_CQ_INTERRUPT: Block on a CQ interrupt until a QE
+ *	arrives for the specified port. If a QE is already present, the ioctl
+ *	will immediately return.
+ *
+ *	Note: Only one thread can block on a CQ's interrupt at a time. Doing
+ *	otherwise can result in hung threads.
+ *
+ * Input parameters:
+ * - port_id: Port ID.
+ * - is_ldb: True if the port is load-balanced, false otherwise.
+ * - arm: Tell the driver to arm the interrupt.
+ * - cq_gen: Current CQ generation bit.
+ * - padding0: Reserved for future use.
+ * - cq_va: VA of the CQ entry where the next QE will be placed.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_block_on_cq_interrupt_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u8 is_ldb;
+	__u8 arm;
+	__u8 cq_gen;
+	__u8 padding0;
+	__u64 cq_va;
+};
+
+/*
+ * DLB_DOMAIN_CMD_ENQUEUE_DOMAIN_ALERT: Enqueue a domain alert that will be
+ *	read by one reader thread.
+ *
+ * Input parameters:
+ * - aux_alert_data: user-defined auxiliary data.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_enqueue_domain_alert_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u64 aux_alert_data;
+};
+
+/*
+ * DLB_DOMAIN_CMD_GET_LDB_QUEUE_DEPTH: Get a load-balanced queue's depth.
+ * Input parameters:
+ * - queue_id: The load-balanced queue ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: queue depth.
+ */
+struct dlb_get_ldb_queue_depth_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 queue_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_GET_DIR_QUEUE_DEPTH: Get a directed queue's depth.
+ * Input parameters:
+ * - queue_id: The directed queue ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: queue depth.
+ */
+struct dlb_get_dir_queue_depth_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 queue_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_PENDING_PORT_UNMAPS: Get number of queue unmap operations in
+ *	progress for a load-balanced port.
+ *
+ *	Note: This is a snapshot; the number of unmap operations in progress
+ *	is subject to change at any time.
+ *
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: number of unmaps in progress.
+ */
+struct dlb_pending_port_unmaps_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+enum dlb_domain_user_interface_commands {
+	DLB_DOMAIN_CMD_CREATE_LDB_POOL,
+	DLB_DOMAIN_CMD_CREATE_DIR_POOL,
+	DLB_DOMAIN_CMD_CREATE_LDB_QUEUE,
+	DLB_DOMAIN_CMD_CREATE_DIR_QUEUE,
+	DLB_DOMAIN_CMD_CREATE_LDB_PORT,
+	DLB_DOMAIN_CMD_CREATE_DIR_PORT,
+	DLB_DOMAIN_CMD_START_DOMAIN,
+	DLB_DOMAIN_CMD_MAP_QID,
+	DLB_DOMAIN_CMD_UNMAP_QID,
+	DLB_DOMAIN_CMD_ENABLE_LDB_PORT,
+	DLB_DOMAIN_CMD_ENABLE_DIR_PORT,
+	DLB_DOMAIN_CMD_DISABLE_LDB_PORT,
+	DLB_DOMAIN_CMD_DISABLE_DIR_PORT,
+	DLB_DOMAIN_CMD_BLOCK_ON_CQ_INTERRUPT,
+	DLB_DOMAIN_CMD_ENQUEUE_DOMAIN_ALERT,
+	DLB_DOMAIN_CMD_GET_LDB_QUEUE_DEPTH,
+	DLB_DOMAIN_CMD_GET_DIR_QUEUE_DEPTH,
+	DLB_DOMAIN_CMD_PENDING_PORT_UNMAPS,
+
+	/* NUM_DLB_DOMAIN_CMD must be last */
+	NUM_DLB_DOMAIN_CMD,
+};
+
+/*
+ * Base addresses for memory mapping the consumer queue (CQ) and popcount (PC)
+ * memory space, and producer port (PP) MMIO space. The CQ, PC, and PP
+ * addresses are per-port. Every address is page-separated (e.g. LDB PP 0 is at
+ * 0x2100000 and LDB PP 1 is at 0x2101000).
+ */
+#define DLB_LDB_CQ_BASE 0x3000000
+#define DLB_LDB_CQ_MAX_SIZE 65536
+#define DLB_LDB_CQ_OFFS(id) (DLB_LDB_CQ_BASE + (id) * DLB_LDB_CQ_MAX_SIZE)
+
+#define DLB_DIR_CQ_BASE 0x3800000
+#define DLB_DIR_CQ_MAX_SIZE 65536
+#define DLB_DIR_CQ_OFFS(id) (DLB_DIR_CQ_BASE + (id) * DLB_DIR_CQ_MAX_SIZE)
+
+#define DLB_LDB_PC_BASE 0x2300000
+#define DLB_LDB_PC_MAX_SIZE 4096
+#define DLB_LDB_PC_OFFS(id) (DLB_LDB_PC_BASE + (id) * DLB_LDB_PC_MAX_SIZE)
+
+#define DLB_DIR_PC_BASE 0x2200000
+#define DLB_DIR_PC_MAX_SIZE 4096
+#define DLB_DIR_PC_OFFS(id) (DLB_DIR_PC_BASE + (id) * DLB_DIR_PC_MAX_SIZE)
+
+#define DLB_LDB_PP_BASE 0x2100000
+#define DLB_LDB_PP_MAX_SIZE 4096
+#define DLB_LDB_PP_OFFS(id) (DLB_LDB_PP_BASE + (id) * DLB_LDB_PP_MAX_SIZE)
+
+#define DLB_DIR_PP_BASE 0x2000000
+#define DLB_DIR_PP_MAX_SIZE 4096
+#define DLB_DIR_PP_OFFS(id) (DLB_DIR_PP_BASE + (id) * DLB_DIR_PP_MAX_SIZE)
+
+#endif /* __DLB_USER_H */
