[dpdk-dev,1/2] eal/persistent: new library to hold memory region after program exit
diff mbox

Message ID 1436189297-7780-2-git-send-email-dlrmsghd@gmail.com
State Rejected, archived
Headers show

Commit Message

이근홍 July 6, 2015, 1:28 p.m. UTC
Some NICs use host memory region as their scratch area.
When DPDK user applications terminate, all the memory regions are lost,
re-initialized (memzone), which causes HW faults.
This libraray maintains shared memory regions that is persistent across
multiple execution and termination of user level applications.
It also manages physically contiguous memory regions.

Signed-off-by: leeopop <dlrmsghd@gmail.com>
---
 drivers/net/mlnx_uio/LICENSE                       |  30 ++++
 lib/Makefile                                       |   1 +
 lib/librte_eal/common/Makefile                     |   3 +
 lib/librte_eal/common/include/rte_pci.h            |   1 +
 lib/librte_eal/common/include/rte_persistent_mem.h |  26 +++
 lib/librte_eal/linuxapp/eal/Makefile               |   6 +
 lib/librte_eal/linuxapp/eal/eal.c                  |   9 +
 lib/librte_eal/linuxapp/eal/eal_persistent_mem.c   | 148 +++++++++++++++
 .../eal/include/exec-env/rte_persistent_mem.h      |  15 ++
 lib/librte_eal/linuxapp/eal/rte_eal_version.map    |   2 +
 lib/librte_persistent/Makefile                     |  55 ++++++
 lib/librte_persistent/rte_persistent.c             | 198 +++++++++++++++++++++
 lib/librte_persistent/rte_persistent.h             |  20 +++
 lib/librte_persistent/rte_persistent_version.map   |  11 ++
 14 files changed, 525 insertions(+)
 create mode 100644 drivers/net/mlnx_uio/LICENSE
 create mode 100644 lib/librte_eal/common/include/rte_persistent_mem.h
 create mode 100644 lib/librte_eal/linuxapp/eal/eal_persistent_mem.c
 create mode 100644 lib/librte_eal/linuxapp/eal/include/exec-env/rte_persistent_mem.h
 create mode 100644 lib/librte_persistent/Makefile
 create mode 100644 lib/librte_persistent/rte_persistent.c
 create mode 100644 lib/librte_persistent/rte_persistent.h
 create mode 100644 lib/librte_persistent/rte_persistent_version.map

Comments

Avi Kivity July 6, 2015, 2:34 p.m. UTC | #1
On 07/06/2015 04:28 PM, leeopop wrote:
> Some NICs use host memory region as their scratch area.
> When DPDK user applications terminate, all the memory regions are lost,
> re-initialized (memzone), which causes HW faults.
> This libraray maintains shared memory regions that is persistent across
> multiple execution and termination of user level applications.
> It also manages physically contiguous memory regions.
>
> Signed-off-by: leeopop <dlrmsghd@gmail.com>
>

Does dpdk accept anonymous signoffs?

DCO usually requires a real name.
Thomas Monjalon July 6, 2015, 2:41 p.m. UTC | #2
2015-07-06 17:34, Avi Kivity:
> On 07/06/2015 04:28 PM, leeopop wrote:
> > Some NICs use host memory region as their scratch area.
> > When DPDK user applications terminate, all the memory regions are lost,
> > re-initialized (memzone), which causes HW faults.
> > This libraray maintains shared memory regions that is persistent across
> > multiple execution and termination of user level applications.
> > It also manages physically contiguous memory regions.
> >
> > Signed-off-by: leeopop <dlrmsghd@gmail.com>
> >
> 
> Does dpdk accept anonymous signoffs?
> 
> DCO usually requires a real name.

Exact. We follow Linux guidelines for DCO:
	 http://dpdk.org/dev#send
Stephen Hemminger July 6, 2015, 7:19 p.m. UTC | #3
Am I right, this library is using Unix shared memory to get
persistent storage. That is going to be slower and more worrying,
the kernel makes no guarantees that virtual to physical mapping
will not change.

There is also an ABI issue in this patch. You are introducing a new
API here, so the symbol should go in a new 2.1 section


> +	if((len / RTE_PGSIZE_4K) > 1)
> +	{
> +		shmget_flag |= SHM_HUGETLB;
> +	}

Please follow kernel coding style. This is one of many examples that
needs to be fixed before accepting.


ERROR: "foo* bar" should be "foo *bar"
#166: FILE: lib/librte_eal/common/include/rte_pci.h:210:
+	void* priv; /**< Private data. */

ERROR: "foo* bar" should be "foo *bar"
#195: FILE: lib/librte_eal/common/include/rte_persistent_mem.h:20:
+extern void* persistent_allocated_memory[RTE_MAX_NUMA_NODES][RTE_EAL_PERSISTENT_MEM_COUNT];

ERROR: "foo* bar" should be "foo *bar"
#302: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:44:
+static void* reserve_shared_zone(int subindex, uint32_t len, int socket_id)

ERROR: do not use C99 // comments
#307: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:49:
+	int shmget_flag = IPC_CREAT | SHM_R | SHM_W | IPC_EXCL; // | SHM_LOCKED;

WARNING: Missing a blank line after declarations
#310: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:52:
+	int err;
+	if((len / RTE_PGSIZE_4K) > 1)

ERROR: that open brace { should be on the previous line
#310: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:52:
+	if((len / RTE_PGSIZE_4K) > 1)
+	{

ERROR: space required before the open parenthesis '('
#310: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:52:
+	if((len / RTE_PGSIZE_4K) > 1)

ERROR: "foo* bar" should be "foo *bar"
#316: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:58:
+	void* addr = 0;

WARNING: Missing a blank line after declarations
#318: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:60:
+	int clear = 1;
+	if(shmid < 0)

ERROR: that open brace { should be on the previous line
#318: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:60:
+	if(shmid < 0)
+	{

ERROR: space required before the open parenthesis '('
#318: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:60:
+	if(shmid < 0)

ERROR: do not use C99 // comments
#320: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:62:
+		//Reuse existing

ERROR: that open brace { should be on the previous line
#328: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:70:
+	if(socket_id != SOCKET_ID_ANY)
+	{

ERROR: space required before the open parenthesis '('
#328: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:70:
+	if(socket_id != SOCKET_ID_ANY)

ERROR: "foo * bar" should be "foo *bar"
#330: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:72:
+		struct bitmask * mask = numa_bitmask_alloc(RTE_MAX_NUMA_NODES);

WARNING: Missing a blank line after declarations
#331: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:73:
+		struct bitmask * mask = numa_bitmask_alloc(RTE_MAX_NUMA_NODES);
+		mask = numa_bitmask_clearall(mask);

ERROR: that open brace { should be on the previous line
#336: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:78:
+		if(ret < 0)
+		{

ERROR: space required before the open parenthesis '('
#336: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:78:
+		if(ret < 0)

ERROR: that open brace { should be on the previous line
#344: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:86:
+	if(clear)
+	{

ERROR: space required before the open parenthesis '('
#344: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:86:
+	if(clear)

WARNING: Missing a blank line after declarations
#350: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:92:
+	size_t size;
+	volatile uint8_t reader = 0; //this prevents from being optimized out

ERROR: do not use C99 // comments
#350: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:92:
+	volatile uint8_t reader = 0; //this prevents from being optimized out

WARNING: Use of volatile is usually wrong: see Documentation/volatile-considered-harmful.txt
#350: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:92:
+	volatile uint8_t reader = 0; //this prevents from being optimized out

ERROR: "(foo*)" should be "(foo *)"
#351: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:93:
+	volatile uint8_t* readp = (uint8_t*)addr;

ERROR: "foo* bar" should be "foo *bar"
#351: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:93:
+	volatile uint8_t* readp = (uint8_t*)addr;

WARNING: Use of volatile is usually wrong: see Documentation/volatile-considered-harmful.txt
#351: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:93:
+	volatile uint8_t* readp = (uint8_t*)addr;

WARNING: Missing a blank line after declarations
#352: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:94:
+	volatile uint8_t* readp = (uint8_t*)addr;
+	for(size = 0; size < len; size++)

ERROR: that open brace { should be on the previous line
#352: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:94:
+	for(size = 0; size < len; size++)
+	{

ERROR: space required before the open parenthesis '('
#352: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:94:
+	for(size = 0; size < len; size++)

ERROR: "foo* bar" should be "foo *bar"
#364: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:106:
+void* persistent_allocated_memory[RTE_MAX_NUMA_NODES][SHM_COUNT];

ERROR: do not initialise statics to 0 or NULL
#366: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:108:
+static int numa_count = 0;

ERROR: do not use C99 // comments
#375: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:117:
+	assert(SHM_SIZE == RTE_PGSIZE_2M); //XXX considering only 2MB pages.

WARNING: Missing a blank line after declarations
#377: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:119:
+	int num_numa = numa_num_configured_nodes();
+	if(num_numa == 0)

ERROR: space required before the open parenthesis '('
#377: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:119:
+	if(num_numa == 0)

WARNING: Missing a blank line after declarations
#382: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:124:
+	int k;
+	for(node = 0; node < RTE_MAX_NUMA_NODES; node++)

ERROR: space required before the open parenthesis '('
#382: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:124:
+	for(node = 0; node < RTE_MAX_NUMA_NODES; node++)

ERROR: spaces required around that '=' (ctx:VxV)
#383: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:125:
+		for(k=0; k<SHM_COUNT; k++)
 		     ^

ERROR: spaces required around that '<' (ctx:VxV)
#383: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:125:
+		for(k=0; k<SHM_COUNT; k++)
 		          ^

ERROR: space required before the open parenthesis '('
#383: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:125:
+		for(k=0; k<SHM_COUNT; k++)

ERROR: that open brace { should be on the previous line
#386: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:128:
+	for(node = 0; node < num_numa; node++)
+	{

ERROR: space required before the open parenthesis '('
#386: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:128:
+	for(node = 0; node < num_numa; node++)

WARNING: Missing a blank line after declarations
#389: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:131:
+		int cur_socket = num_numa > 1 ? node : SOCKET_ID_ANY;
+		for(k=0; k<SHM_COUNT/num_numa; k++)

ERROR: that open brace { should be on the previous line
#389: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:131:
+		for(k=0; k<SHM_COUNT/num_numa; k++)
+		{

ERROR: spaces required around that '=' (ctx:VxV)
#389: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:131:
+		for(k=0; k<SHM_COUNT/num_numa; k++)
 		     ^

ERROR: spaces required around that '<' (ctx:VxV)
#389: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:131:
+		for(k=0; k<SHM_COUNT/num_numa; k++)
 		          ^

ERROR: space required before the open parenthesis '('
#389: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:131:
+		for(k=0; k<SHM_COUNT/num_numa; k++)

WARNING: Missing a blank line after declarations
#392: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:134:
+			int zone_index = ((SHM_COUNT/num_numa)*node + k);
+			persistent_allocated_memory[node][k] = reserve_shared_zone(zone_index,

ERROR: that open brace { should be on the previous line
#394: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:136:
+			if(persistent_allocated_memory[node][k] == 0)
+			{

ERROR: space required before the open parenthesis '('
#394: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:136:
+			if(persistent_allocated_memory[node][k] == 0)

WARNING: quoted string split across lines
#397: FILE: lib/librte_eal/linuxapp/eal/eal_persistent_mem.c:139:
+				RTE_LOG(ERR, EAL, "Cannot allocate shared zone index %d."
+						"node: %d, local index: %d\n", zone_index, node, k);

ERROR: do not initialise statics to 0 or NULL
#533: FILE: lib/librte_persistent/rte_persistent.c:26:
+static struct rte_hash* allocated_segments = 0;

ERROR: "foo* bar" should be "foo *bar"
#533: FILE: lib/librte_persistent/rte_persistent.c:26:
+static struct rte_hash* allocated_segments = 0;

ERROR: open brace '{' following struct go on the same line
#536: FILE: lib/librte_persistent/rte_persistent.c:29:
+struct alloc_info
+{

ERROR: do not use C99 // comments
#537: FILE: lib/librte_persistent/rte_persistent.c:30:
+	void* addr; //0 if not allocated

ERROR: "foo* bar" should be "foo *bar"
#537: FILE: lib/librte_persistent/rte_persistent.c:30:
+	void* addr; //0 if not allocated

ERROR: do not initialise statics to 0 or NULL
#550: FILE: lib/librte_persistent/rte_persistent.c:43:
+static int __initialized = 0;

ERROR: that open brace { should be on the previous line
#554: FILE: lib/librte_persistent/rte_persistent.c:47:
+	if(!__initialized)
+	{

ERROR: space required before the open parenthesis '('
#554: FILE: lib/librte_persistent/rte_persistent.c:47:
+	if(!__initialized)

ERROR: that open brace { should be on the previous line
#557: FILE: lib/librte_persistent/rte_persistent.c:50:
+		struct rte_hash_parameters hash_param =
+		{

ERROR: "(foo*)" should be "(foo *)"
#561: FILE: lib/librte_persistent/rte_persistent.c:54:
+				.key_len = sizeof(void*),

ERROR: do not use C99 // comments
#562: FILE: lib/librte_persistent/rte_persistent.c:55:
+				.hash_func = 0, //DEFAULT_HASH_FUNC,

WARNING: Missing a blank line after declarations
#571: FILE: lib/librte_persistent/rte_persistent.c:64:
+		int k;
+		for(k=0; k<SEGMENT_COUNT; k++)

ERROR: spaces required around that '=' (ctx:VxV)
#571: FILE: lib/librte_persistent/rte_persistent.c:64:
+		for(k=0; k<SEGMENT_COUNT; k++)
 		     ^

ERROR: spaces required around that '<' (ctx:VxV)
#571: FILE: lib/librte_persistent/rte_persistent.c:64:
+		for(k=0; k<SEGMENT_COUNT; k++)
 		          ^

ERROR: space required before the open parenthesis '('
#571: FILE: lib/librte_persistent/rte_persistent.c:64:
+		for(k=0; k<SEGMENT_COUNT; k++)

ERROR: "foo* bar" should be "foo *bar"
#588: FILE: lib/librte_persistent/rte_persistent.c:81:
+void* rte_persistent_alloc(size_t size, int socket)

WARNING: Missing a blank line after declarations
#591: FILE: lib/librte_persistent/rte_persistent.c:84:
+	int num_numa = rte_persistent_memory_num_numa();
+	if(socket == SOCKET_ID_ANY)

ERROR: that open brace { should be on the previous line
#591: FILE: lib/librte_persistent/rte_persistent.c:84:
+	if(socket == SOCKET_ID_ANY)
+	{

ERROR: space required before the open parenthesis '('
#591: FILE: lib/librte_persistent/rte_persistent.c:84:
+	if(socket == SOCKET_ID_ANY)

WARNING: Missing a blank line after declarations
#600: FILE: lib/librte_persistent/rte_persistent.c:93:
+	int num_page = (size / ALLOC_UNIT);
+	if(size % ALLOC_UNIT)

ERROR: space required before the open parenthesis '('
#600: FILE: lib/librte_persistent/rte_persistent.c:93:
+	if(size % ALLOC_UNIT)

WARNING: Missing a blank line after declarations
#605: FILE: lib/librte_persistent/rte_persistent.c:98:
+	int k;
+	for(k=0; k<num_page; k++)

ERROR: that open brace { should be on the previous line
#605: FILE: lib/librte_persistent/rte_persistent.c:98:
+	for(k=0; k<num_page; k++)
+	{

ERROR: spaces required around that '=' (ctx:VxV)
#605: FILE: lib/librte_persistent/rte_persistent.c:98:
+	for(k=0; k<num_page; k++)
 	     ^

ERROR: spaces required around that '<' (ctx:VxV)
#605: FILE: lib/librte_persistent/rte_persistent.c:98:
+	for(k=0; k<num_page; k++)
 	          ^

ERROR: space required before the open parenthesis '('
#605: FILE: lib/librte_persistent/rte_persistent.c:98:
+	for(k=0; k<num_page; k++)

ERROR: "foo* bar" should be "foo *bar"
#611: FILE: lib/librte_persistent/rte_persistent.c:104:
+	void* found_buffer = 0;

WARNING: Missing a blank line after declarations
#612: FILE: lib/librte_persistent/rte_persistent.c:105:
+	void* found_buffer = 0;
+	for(k=l_start; k<(l_start + l_range); k++)

ERROR: that open brace { should be on the previous line
#612: FILE: lib/librte_persistent/rte_persistent.c:105:
+	for(k=l_start; k<(l_start + l_range); k++)
+	{

ERROR: spaces required around that '=' (ctx:VxV)
#612: FILE: lib/librte_persistent/rte_persistent.c:105:
+	for(k=l_start; k<(l_start + l_range); k++)
 	     ^

ERROR: spaces required around that '<' (ctx:VxV)
#612: FILE: lib/librte_persistent/rte_persistent.c:105:
+	for(k=l_start; k<(l_start + l_range); k++)
 	                ^

ERROR: space required before the open parenthesis '('
#612: FILE: lib/librte_persistent/rte_persistent.c:105:
+	for(k=l_start; k<(l_start + l_range); k++)

ERROR: "foo* bar" should be "foo *bar"
#614: FILE: lib/librte_persistent/rte_persistent.c:107:
+		char* start = alloc_array[k];

ERROR: "foo* bar" should be "foo *bar"
#615: FILE: lib/librte_persistent/rte_persistent.c:108:
+		char* found = strstr(start, find_str);

ERROR: that open brace { should be on the previous line
#617: FILE: lib/librte_persistent/rte_persistent.c:110:
+		if(found)
+		{

ERROR: space required before the open parenthesis '('
#617: FILE: lib/librte_persistent/rte_persistent.c:110:
+		if(found)

WARNING: Missing a blank line after declarations
#620: FILE: lib/librte_persistent/rte_persistent.c:113:
+			int offset = found - start;
+			found_buffer = persistent_allocated_memory[socket][k];

WARNING: Missing a blank line after declarations
#624: FILE: lib/librte_persistent/rte_persistent.c:117:
+			int j;
+			for(j=0; j<num_page; j++)

ERROR: that open brace { should be on the previous line
#624: FILE: lib/librte_persistent/rte_persistent.c:117:
+			for(j=0; j<num_page; j++)
+			{

ERROR: spaces required around that '=' (ctx:VxV)
#624: FILE: lib/librte_persistent/rte_persistent.c:117:
+			for(j=0; j<num_page; j++)
 			     ^

ERROR: spaces required around that '<' (ctx:VxV)
#624: FILE: lib/librte_persistent/rte_persistent.c:117:
+			for(j=0; j<num_page; j++)
 			          ^

ERROR: space required before the open parenthesis '('
#624: FILE: lib/librte_persistent/rte_persistent.c:117:
+			for(j=0; j<num_page; j++)

WARNING: Missing a blank line after declarations
#629: FILE: lib/librte_persistent/rte_persistent.c:122:
+			int index = rte_hash_add_key(allocated_segments, &found_buffer);
+			assert(index >= 0);

ERROR: "foo* bar" should be "foo *bar"
#639: FILE: lib/librte_persistent/rte_persistent.c:132:
+			void* user = found_buffer;

WARNING: Missing a blank line after declarations
#642: FILE: lib/librte_persistent/rte_persistent.c:135:
+			size_t diff = RTE_MAX((uint64_t)user, hw) - RTE_MIN((uint64_t)user, hw);
+			for(j = 0; j < num_page; j++)

ERROR: that open brace { should be on the previous line
#642: FILE: lib/librte_persistent/rte_persistent.c:135:
+			for(j = 0; j < num_page; j++)
+			{

ERROR: space required before the open parenthesis '('
#642: FILE: lib/librte_persistent/rte_persistent.c:135:
+			for(j = 0; j < num_page; j++)

ERROR: "(foo*)" should be "(foo *)"
#645: FILE: lib/librte_persistent/rte_persistent.c:138:
+				void* cur_user = ((char*)user + shift);

ERROR: "foo* bar" should be "foo *bar"
#645: FILE: lib/librte_persistent/rte_persistent.c:138:
+				void* cur_user = ((char*)user + shift);

WARNING: line over 120 characters
#647: FILE: lib/librte_persistent/rte_persistent.c:140:
+				size_t cur_diff = RTE_MAX((uint64_t)cur_user, cur_hw) - RTE_MIN((uint64_t)cur_user, cur_hw);

ERROR: that open brace { should be on the previous line
#649: FILE: lib/librte_persistent/rte_persistent.c:142:
+				if(cur_diff != diff)
+				{

ERROR: space required before the open parenthesis '('
#649: FILE: lib/librte_persistent/rte_persistent.c:142:
+				if(cur_diff != diff)

WARNING: line over 120 characters
#651: FILE: lib/librte_persistent/rte_persistent.c:144:
+					RTE_LOG(ERR, EAL, "Hugepage is not contiguous, curdiff: %lX, expected: %lX\n", cur_diff, diff);

ERROR: space required before the open parenthesis '('
#658: FILE: lib/librte_persistent/rte_persistent.c:151:
+	if(!found_buffer)

ERROR: "foo* bar" should be "foo *bar"
#663: FILE: lib/librte_persistent/rte_persistent.c:156:
+phys_addr_t rte_persistent_hw_addr(const void* addr)

ERROR: space required before the open parenthesis '('
#665: FILE: lib/librte_persistent/rte_persistent.c:158:
+	if(addr == 0)

ERROR: "(foo*)" should be "(foo *)"
#667: FILE: lib/librte_persistent/rte_persistent.c:160:
+	int index = rte_hash_lookup(allocated_segments, (const void*)&addr);

WARNING: Missing a blank line after declarations
#668: FILE: lib/librte_persistent/rte_persistent.c:161:
+	int index = rte_hash_lookup(allocated_segments, (const void*)&addr);
+	assert(index >= 0);

ERROR: "foo* bar" should be "foo *bar"
#674: FILE: lib/librte_persistent/rte_persistent.c:167:
+size_t rte_persistent_mem_length(const void* addr)

ERROR: "(foo*)" should be "(foo *)"
#676: FILE: lib/librte_persistent/rte_persistent.c:169:
+	int index = rte_hash_lookup(allocated_segments, (const void*)&addr);

WARNING: Missing a blank line after declarations
#677: FILE: lib/librte_persistent/rte_persistent.c:170:
+	int index = rte_hash_lookup(allocated_segments, (const void*)&addr);
+	assert(index >= 0);

ERROR: "foo* bar" should be "foo *bar"
#683: FILE: lib/librte_persistent/rte_persistent.c:176:
+void rte_persistent_free(void* addr)

ERROR: "(foo*)" should be "(foo *)"
#685: FILE: lib/librte_persistent/rte_persistent.c:178:
+	int index = rte_hash_lookup(allocated_segments, (const void*)&addr);

WARNING: Missing a blank line after declarations
#686: FILE: lib/librte_persistent/rte_persistent.c:179:
+	int index = rte_hash_lookup(allocated_segments, (const void*)&addr);
+	assert(index >= 0);

ERROR: "(foo*)" should be "(foo *)"
#700: FILE: lib/librte_persistent/rte_persistent.c:193:
+	rte_hash_del_key(allocated_segments, (const void*)&addr);

WARNING: Missing a blank line after declarations
#703: FILE: lib/librte_persistent/rte_persistent.c:196:
+	int k;
+	for(k=0; k<len; k++)

ERROR: spaces required around that '=' (ctx:VxV)
#703: FILE: lib/librte_persistent/rte_persistent.c:196:
+	for(k=0; k<len; k++)
 	     ^

ERROR: spaces required around that '<' (ctx:VxV)
#703: FILE: lib/librte_persistent/rte_persistent.c:196:
+	for(k=0; k<len; k++)
 	          ^

ERROR: space required before the open parenthesis '('
#703: FILE: lib/librte_persistent/rte_persistent.c:196:
+	for(k=0; k<len; k++)

ERROR: "foo* bar" should be "foo *bar"
#726: FILE: lib/librte_persistent/rte_persistent.h:15:
+void* rte_persistent_alloc(size_t size, int socket);

ERROR: "foo* bar" should be "foo *bar"
#727: FILE: lib/librte_persistent/rte_persistent.h:16:
+phys_addr_t rte_persistent_hw_addr(const void* addr);

ERROR: "foo* bar" should be "foo *bar"
#728: FILE: lib/librte_persistent/rte_persistent.h:17:
+void rte_persistent_free(void* addr);

ERROR: "foo* bar" should be "foo *bar"
#729: FILE: lib/librte_persistent/rte_persistent.h:18:
+size_t rte_persistent_mem_length(const void* addr);

total: 96 errors, 28 warnings, 573 lines checked

/tmp/persist.patch has style problems, please review.

NOTE: If any of the errors are false positives, please report
      them to the maintainer, see CHECKPATCH in MAINTAINERS.

Patch
diff mbox

diff --git a/drivers/net/mlnx_uio/LICENSE b/drivers/net/mlnx_uio/LICENSE
new file mode 100644
index 0000000..7ef5b4b
--- /dev/null
+++ b/drivers/net/mlnx_uio/LICENSE
@@ -0,0 +1,30 @@ 
+* Source code in kernel/ directory follows GPLv2 license.
+
+
+Copyright (c) 2015, Keunhong Lee
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of bsd nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/lib/Makefile b/lib/Makefile
index 5f480f9..7a491d3 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -57,6 +57,7 @@  DIRS-$(CONFIG_RTE_LIBRTE_PORT) += librte_port
 DIRS-$(CONFIG_RTE_LIBRTE_TABLE) += librte_table
 DIRS-$(CONFIG_RTE_LIBRTE_PIPELINE) += librte_pipeline
 DIRS-$(CONFIG_RTE_LIBRTE_REORDER) += librte_reorder
+DIRS-$(CONFIG_RTE_LIBRTE_PERSISTENT) += librte_persistent
 
 ifeq ($(CONFIG_RTE_EXEC_ENV_LINUXAPP),y)
 DIRS-$(CONFIG_RTE_LIBRTE_KNI) += librte_kni
diff --git a/lib/librte_eal/common/Makefile b/lib/librte_eal/common/Makefile
index 38772d4..ce4b0a7 100644
--- a/lib/librte_eal/common/Makefile
+++ b/lib/librte_eal/common/Makefile
@@ -40,6 +40,9 @@  INC += rte_string_fns.h rte_version.h
 INC += rte_eal_memconfig.h rte_malloc_heap.h
 INC += rte_hexdump.h rte_devargs.h rte_dev.h
 INC += rte_pci_dev_feature_defs.h rte_pci_dev_features.h
+ifeq ($(CONFIG_RTE_EAL_PERSISTENT_MEM),y)
+INC += rte_persistent_mem.h
+endif
 
 ifeq ($(CONFIG_RTE_INSECURE_FUNCTION_WARNING),y)
 INC += rte_warnings.h
diff --git a/lib/librte_eal/common/include/rte_pci.h b/lib/librte_eal/common/include/rte_pci.h
index 7801fa0..a323e74 100644
--- a/lib/librte_eal/common/include/rte_pci.h
+++ b/lib/librte_eal/common/include/rte_pci.h
@@ -207,6 +207,7 @@  struct rte_pci_driver {
 	pci_devuninit_t *devuninit;             /**< Device uninit function. */
 	const struct rte_pci_id *id_table;	/**< ID table, NULL terminated. */
 	uint32_t drv_flags;                     /**< Flags contolling handling of device. */
+	void* priv; /**< Private data. */
 };
 
 /** Device needs PCI BAR mapping (done with either IGB_UIO or VFIO) */
diff --git a/lib/librte_eal/common/include/rte_persistent_mem.h b/lib/librte_eal/common/include/rte_persistent_mem.h
new file mode 100644
index 0000000..3a8ff23
--- /dev/null
+++ b/lib/librte_eal/common/include/rte_persistent_mem.h
@@ -0,0 +1,26 @@ 
+/*
+ * rte_persistent_memory.h
+ *
+ *  Created on: Jun 22, 2015
+ *      Author: leeopop
+ */
+
+#ifndef LIBRTE_EAL_COMMON_INCLUDE_RTE_PERSISTENT_MEM_H_
+#define LIBRTE_EAL_COMMON_INCLUDE_RTE_PERSISTENT_MEM_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <exec-env/rte_persistent_mem.h>
+
+int rte_persistent_memory_init(void);
+int rte_persistent_memory_num_numa(void);
+
+extern void* persistent_allocated_memory[RTE_MAX_NUMA_NODES][RTE_EAL_PERSISTENT_MEM_COUNT];
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* LIBRTE_EAL_COMMON_INCLUDE_RTE_PERSISTENT_MEM_H_ */
diff --git a/lib/librte_eal/linuxapp/eal/Makefile b/lib/librte_eal/linuxapp/eal/Makefile
index e99d7a3..139b608 100644
--- a/lib/librte_eal/linuxapp/eal/Makefile
+++ b/lib/librte_eal/linuxapp/eal/Makefile
@@ -74,6 +74,9 @@  SRCS-$(CONFIG_RTE_LIBRTE_EAL_LINUXAPP) += eal_alarm.c
 ifeq ($(CONFIG_RTE_LIBRTE_IVSHMEM),y)
 SRCS-$(CONFIG_RTE_LIBRTE_EAL_LINUXAPP) += eal_ivshmem.c
 endif
+ifeq ($(CONFIG_RTE_EAL_PERSISTENT_MEM),y)
+SRCS-$(CONFIG_RTE_LIBRTE_EAL_LINUXAPP) += eal_persistent_mem.c
+endif
 
 # from common dir
 SRCS-$(CONFIG_RTE_LIBRTE_EAL_LINUXAPP) += eal_common_memzone.c
@@ -112,6 +115,9 @@  CFLAGS_eal_thread.o += -Wno-return-type
 endif
 
 INC := rte_interrupts.h rte_kni_common.h rte_dom0_common.h
+ifeq ($(CONFIG_RTE_EAL_PERSISTENT_MEM),y)
+INC += rte_persistent_mem.h
+endif
 
 SYMLINK-$(CONFIG_RTE_LIBRTE_EAL_LINUXAPP)-include/exec-env := \
 	$(addprefix include/exec-env/,$(INC))
diff --git a/lib/librte_eal/linuxapp/eal/eal.c b/lib/librte_eal/linuxapp/eal/eal.c
index 8809f57..b3f05a8 100644
--- a/lib/librte_eal/linuxapp/eal/eal.c
+++ b/lib/librte_eal/linuxapp/eal/eal.c
@@ -77,6 +77,10 @@ 
 #include <malloc_heap.h>
 #include <rte_eth_ring.h>
 
+#ifdef RTE_EAL_PERSISTENT_MEM
+#include <rte_persistent_mem.h>
+#endif
+
 #include "eal_private.h"
 #include "eal_thread.h"
 #include "eal_internal_cfg.h"
@@ -759,6 +763,11 @@  rte_eal_init(int argc, char **argv)
 	if (fctret < 0)
 		exit(1);
 
+#ifdef RTE_EAL_PERSISTENT_MEM
+	if (rte_persistent_memory_init() < 0)
+		rte_panic("Cannot init persistent memory\n");
+#endif
+
 	if (internal_config.no_hugetlbfs == 0 &&
 			internal_config.process_type != RTE_PROC_SECONDARY &&
 			internal_config.xen_dom0_support == 0 &&
diff --git a/lib/librte_eal/linuxapp/eal/eal_persistent_mem.c b/lib/librte_eal/linuxapp/eal/eal_persistent_mem.c
new file mode 100644
index 0000000..f72c148
--- /dev/null
+++ b/lib/librte_eal/linuxapp/eal/eal_persistent_mem.c
@@ -0,0 +1,148 @@ 
+/*
+ * eal_persistent_mem.c
+ *
+ *  Created on: Jun 22, 2015
+ *      Author: leeopop
+ */
+
+
+/*
+ * dma_memory.c
+ *
+ *  Created on: Oct 4, 2014
+ *      Author: leeopop
+ */
+
+
+#include <rte_persistent_mem.h>
+
+#include <sys/io.h>
+#include <sys/ipc.h>
+#include <sys/shm.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <assert.h>
+#include <unistd.h>
+#include <numa.h>
+#include <numaif.h>
+#include <sys/time.h>
+#include <sys/resource.h>
+
+#include <rte_log.h>
+#include <rte_eal.h>
+#include <rte_memory.h>
+#include <rte_common.h>
+#include <rte_atomic.h>
+
+#define SHM_SIZE RTE_EAL_PERSISTENT_MEM_UNIT
+#define SHM_COUNT RTE_EAL_PERSISTENT_MEM_COUNT
+
+#define SHM_KEY_BASE (0x861591B)
+#define SHM_KEY ((SHM_KEY_BASE / SHM_COUNT)*SHM_COUNT)
+
+static void* reserve_shared_zone(int subindex, uint32_t len, int socket_id)
+{
+	assert(subindex < SHM_COUNT);
+	uint32_t shared_key = SHM_KEY_BASE + subindex;
+
+	int shmget_flag = IPC_CREAT | SHM_R | SHM_W | IPC_EXCL; // | SHM_LOCKED;
+	int shmid = -1;
+	int err;
+	if((len / RTE_PGSIZE_4K) > 1)
+	{
+		shmget_flag |= SHM_HUGETLB;
+	}
+
+	shmid = shmget(shared_key, len, shmget_flag);
+	void* addr = 0;
+	int clear = 1;
+	if(shmid < 0)
+	{
+		//Reuse existing
+		shmid = shmget(shared_key, len, shmget_flag &= ~IPC_EXCL);
+		assert(shmid >= 0);
+		clear = 0;
+	}
+	addr = shmat(shmid, 0, SHM_RND);
+	assert(addr);
+
+	if(socket_id != SOCKET_ID_ANY)
+	{
+		struct bitmask * mask = numa_bitmask_alloc(RTE_MAX_NUMA_NODES);
+		mask = numa_bitmask_clearall(mask);
+		mask = numa_bitmask_setbit(mask, socket_id);
+		long ret = mbind(addr, len, MPOL_BIND,
+				mask->maskp, RTE_MAX_NUMA_NODES,
+				MPOL_MF_MOVE_ALL | MPOL_MF_STRICT);
+		if(ret < 0)
+		{
+			RTE_LOG(WARNING, EAL, "Cannot mbind memory. Are you running with root?\n");
+		}
+		numa_bitmask_free(mask);
+	}
+	rte_mb();
+
+	if(clear)
+	{
+		memset(addr, 0, len);
+	}
+
+	size_t size;
+	volatile uint8_t reader = 0; //this prevents from being optimized out
+	volatile uint8_t* readp = (uint8_t*)addr;
+	for(size = 0; size < len; size++)
+	{
+		reader += *readp;
+		readp++;
+	}
+
+	rte_mb();
+	err = shmctl(shmid, SHM_LOCK, 0);
+	assert(err == 0);
+	return addr;
+}
+
+void* persistent_allocated_memory[RTE_MAX_NUMA_NODES][SHM_COUNT];
+
+static int numa_count = 0;
+
+int rte_persistent_memory_num_numa(void)
+{
+	return numa_count;
+}
+
+int rte_persistent_memory_init(void)
+{
+	assert(SHM_SIZE == RTE_PGSIZE_2M); //XXX considering only 2MB pages.
+	int num_numa = numa_num_configured_nodes();
+	if(num_numa == 0)
+		num_numa = 1;
+	numa_count = num_numa;
+	int node;
+	int k;
+	for(node = 0; node < RTE_MAX_NUMA_NODES; node++)
+		for(k=0; k<SHM_COUNT; k++)
+			persistent_allocated_memory[node][k] = 0;
+
+	for(node = 0; node < num_numa; node++)
+	{
+		int cur_socket = num_numa > 1 ? node : SOCKET_ID_ANY;
+		for(k=0; k<SHM_COUNT/num_numa; k++)
+		{
+			int zone_index = ((SHM_COUNT/num_numa)*node + k);
+			persistent_allocated_memory[node][k] = reserve_shared_zone(zone_index,
+					SHM_SIZE, cur_socket);
+			if(persistent_allocated_memory[node][k] == 0)
+			{
+				RTE_LOG(ERR, EAL, "Cannot allocate shared zone index %d."
+						"node: %d, local index: %d\n", zone_index, node, k);
+				return -1;
+			}
+		}
+		RTE_LOG(INFO, EAL, "Initialized %lu bytes shared zone on socket %d.\n",
+				((uint64_t)(SHM_COUNT/num_numa)) * ((uint64_t)(SHM_SIZE)),
+				cur_socket);
+	}
+	return 0;
+}
diff --git a/lib/librte_eal/linuxapp/eal/include/exec-env/rte_persistent_mem.h b/lib/librte_eal/linuxapp/eal/include/exec-env/rte_persistent_mem.h
new file mode 100644
index 0000000..4038cd5
--- /dev/null
+++ b/lib/librte_eal/linuxapp/eal/include/exec-env/rte_persistent_mem.h
@@ -0,0 +1,15 @@ 
+/*
+ * rte_persistent_mem.h
+ *
+ *  Created on: Jun 22, 2015
+ *      Author: leeopop
+ */
+
+#ifndef LIBRTE_EAL_LINUXAPP_EAL_INCLUDE_EXEC_ENV_RTE_PERSISTENT_MEM_H_
+#define LIBRTE_EAL_LINUXAPP_EAL_INCLUDE_EXEC_ENV_RTE_PERSISTENT_MEM_H_
+
+#ifndef LIBRTE_EAL_COMMON_INCLUDE_RTE_PERSISTENT_MEM_H_
+#error "don't include this file directly, please include generic <rte_persistent_mem.h>"
+#endif
+
+#endif /* LIBRTE_EAL_LINUXAPP_EAL_INCLUDE_EXEC_ENV_RTE_PERSISTENT_MEM_H_ */
diff --git a/lib/librte_eal/linuxapp/eal/rte_eal_version.map b/lib/librte_eal/linuxapp/eal/rte_eal_version.map
index 7e850a9..4382a01 100644
--- a/lib/librte_eal/linuxapp/eal/rte_eal_version.map
+++ b/lib/librte_eal/linuxapp/eal/rte_eal_version.map
@@ -95,6 +95,8 @@  DPDK_2.0 {
 	rte_xen_dom0_memory_attach;
 	rte_xen_dom0_memory_init;
 	test_mp_secondary;
+	rte_persistent_memory_init;
+	rte_persistent_memory_num_numa;
 
 	local: *;
 };
diff --git a/lib/librte_persistent/Makefile b/lib/librte_persistent/Makefile
new file mode 100644
index 0000000..a233d95
--- /dev/null
+++ b/lib/librte_persistent/Makefile
@@ -0,0 +1,55 @@ 
+#   BSD LICENSE
+#
+#   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+#   All rights reserved.
+#
+#   Redistribution and use in source and binary forms, with or without
+#   modification, are permitted provided that the following conditions
+#   are met:
+#
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in
+#       the documentation and/or other materials provided with the
+#       distribution.
+#     * Neither the name of Intel Corporation nor the names of its
+#       contributors may be used to endorse or promote products derived
+#       from this software without specific prior written permission.
+#
+#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+# library name
+LIB = librte_persistent.a
+
+CFLAGS += -O3
+CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR)
+
+EXPORT_MAP := rte_persistent_version.map
+
+LIBABIVER := 1
+
+# all source are stored in SRCS-y
+SRCS-$(CONFIG_RTE_LIBRTE_PERSISTENT) := rte_persistent.c
+
+# install this header file
+SYMLINK-$(CONFIG_RTE_LIBRTE_PERSISTENT)-include := rte_persistent.h
+
+# this lib depends upon:
+DEPDIRS-$(CONFIG_RTE_LIBRTE_PERSISTENT) += lib/librte_hash
+DEPDIRS-$(CONFIG_RTE_LIBRTE_PERSISTENT) += lib/librte_mbuf
+DEPDIRS-$(CONFIG_RTE_LIBRTE_PERSISTENT) += lib/librte_eal
+
+include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/lib/librte_persistent/rte_persistent.c b/lib/librte_persistent/rte_persistent.c
new file mode 100644
index 0000000..a21f9dc
--- /dev/null
+++ b/lib/librte_persistent/rte_persistent.c
@@ -0,0 +1,198 @@ 
+/*
+ * rte_persistent.c
+ *
+ *  Created on: Jun 23, 2015
+ *      Author: leeopop
+ */
+
+#include <rte_persistent_mem.h>
+#include <rte_persistent.h>
+#include <rte_hash.h>
+#include <rte_memory.h>
+
+#include <memory.h>
+#include <string.h>
+#include <rte_common.h>
+#include <rte_random.h>
+#include <rte_log.h>
+#include <assert.h>
+
+#define ALLOC_UNIT RTE_PGSIZE_4K
+#define MAX_CONT_MEMORY RTE_EAL_PERSISTENT_MEM_UNIT
+#define MAX_ALLOC_COUNT (RTE_EAL_PERSISTENT_MEM_COUNT*(RTE_EAL_PERSISTENT_MEM_UNIT/ALLOC_UNIT))
+#define SEGMENT_COUNT (RTE_EAL_PERSISTENT_MEM_COUNT)
+#define SUBSEGMENT_COUNT (RTE_EAL_PERSISTENT_MEM_UNIT/ALLOC_UNIT)
+
+static struct rte_hash* allocated_segments = 0;
+
+struct alloc_info
+{
+	void* addr; //0 if not allocated
+	phys_addr_t hw_addr;
+	int seg_index;
+	int sub_index;
+	int seg_count;
+};
+
+struct alloc_info info_array[MAX_ALLOC_COUNT];
+char alloc_array[SEGMENT_COUNT][SUBSEGMENT_COUNT+1];
+
+#define ALLOCATED 'a'
+#define FREE 'f'
+
+static int __initialized = 0;
+
+int rte_persistent_init(void)
+{
+	if(!__initialized)
+	{
+		struct rte_hash_parameters hash_param =
+		{
+				.name = "Persistent memory segments",
+				.entries = MAX_ALLOC_COUNT,
+				.bucket_entries = RTE_HASH_BUCKET_ENTRIES_MAX,
+				.key_len = sizeof(void*),
+				.hash_func = 0, //DEFAULT_HASH_FUNC,
+				.hash_func_init_val = 0,
+				.socket_id = SOCKET_ID_ANY,
+		};
+		allocated_segments = rte_hash_create(&hash_param);
+		memset(info_array, 0, sizeof(info_array));
+		memset(alloc_array, (int)FREE, sizeof(alloc_array));
+
+		int k;
+		for(k=0; k<SEGMENT_COUNT; k++)
+			alloc_array[k][SUBSEGMENT_COUNT] = 0;
+		__initialized = 1;
+	}
+	return 0;
+}
+
+static int global_to_local_start(int total_numa, int numa)
+{
+	return ((RTE_EAL_PERSISTENT_MEM_COUNT/total_numa)*numa);
+}
+
+static int global_to_local_range(int total_numa)
+{
+	return ((RTE_EAL_PERSISTENT_MEM_COUNT/total_numa));
+}
+
+void* rte_persistent_alloc(size_t size, int socket)
+{
+	int num_numa = rte_persistent_memory_num_numa();
+	if(socket == SOCKET_ID_ANY)
+	{
+		socket = rte_rand() % num_numa;
+	}
+
+	int l_start = global_to_local_start(num_numa, socket);
+	int l_range = global_to_local_range(num_numa);
+
+	int num_page = (size / ALLOC_UNIT);
+	if(size % ALLOC_UNIT)
+		num_page++;
+
+	char find_str[SUBSEGMENT_COUNT+1];
+	int k;
+	for(k=0; k<num_page; k++)
+	{
+		find_str[k] = FREE;
+	}
+	find_str[k] = 0;
+
+	void* found_buffer = 0;
+	for(k=l_start; k<(l_start + l_range); k++)
+	{
+		char* start = alloc_array[k];
+		char* found = strstr(start, find_str);
+
+		if(found)
+		{
+			int offset = found - start;
+			found_buffer = persistent_allocated_memory[socket][k];
+			assert(found_buffer);
+			found_buffer = RTE_PTR_ADD(found_buffer, ALLOC_UNIT*offset);
+			int j;
+			for(j=0; j<num_page; j++)
+			{
+				found[j] = ALLOCATED;
+			}
+			int index = rte_hash_add_key(allocated_segments, &found_buffer);
+			assert(index >= 0);
+			assert(info_array[index].addr == 0);
+			info_array[index].addr = found_buffer;
+			info_array[index].hw_addr = rte_mem_virt2phy(found_buffer);
+			info_array[index].seg_count = num_page;
+			info_array[index].seg_index = k;
+			info_array[index].sub_index = offset;
+			memset(found_buffer, 0, num_page*ALLOC_UNIT);
+
+
+			void* user = found_buffer;
+			uint64_t hw = rte_mem_virt2phy(user);
+			size_t diff = RTE_MAX((uint64_t)user, hw) - RTE_MIN((uint64_t)user, hw);
+			for(j = 0; j < num_page; j++)
+			{
+				size_t shift = ALLOC_UNIT * j;
+				void* cur_user = ((char*)user + shift);
+				uint64_t cur_hw = rte_mem_virt2phy(cur_user);
+				size_t cur_diff = RTE_MAX((uint64_t)cur_user, cur_hw) - RTE_MIN((uint64_t)cur_user, cur_hw);
+
+				if(cur_diff != diff)
+				{
+					RTE_LOG(ERR, EAL, "Hugepage is not contiguous, curdiff: %lX, expected: %lX\n", cur_diff, diff);
+					assert(0);
+				}
+			}
+			break;
+		}
+	}
+	if(!found_buffer)
+		RTE_LOG(ERR, EAL, "Cannot allocate persistent memory, size: %lu, socket: %d\n", size, socket);
+	return found_buffer;
+}
+
+phys_addr_t rte_persistent_hw_addr(const void* addr)
+{
+	if(addr == 0)
+		return 0;
+	int index = rte_hash_lookup(allocated_segments, (const void*)&addr);
+	assert(index >= 0);
+	assert(info_array[index].addr);
+	assert(info_array[index].addr == addr);
+	return info_array[index].hw_addr;
+}
+
+size_t rte_persistent_mem_length(const void* addr)
+{
+	int index = rte_hash_lookup(allocated_segments, (const void*)&addr);
+	assert(index >= 0);
+	assert(info_array[index].addr);
+	assert(info_array[index].addr == addr);
+	return info_array[index].seg_count * ALLOC_UNIT;
+}
+
+void rte_persistent_free(void* addr)
+{
+	int index = rte_hash_lookup(allocated_segments, (const void*)&addr);
+	assert(index >= 0);
+	assert(info_array[index].addr);
+	assert(info_array[index].addr == addr);
+
+	int seg_index = info_array[index].seg_index;
+	int sub_index = info_array[index].sub_index;
+	int len = info_array[index].seg_count;
+
+	info_array[index].seg_index = 0;
+	info_array[index].sub_index = 0;
+	info_array[index].seg_count = 0;
+	info_array[index].addr = 0;
+	info_array[index].hw_addr = 0;
+
+	rte_hash_del_key(allocated_segments, (const void*)&addr);
+
+	int k;
+	for(k=0; k<len; k++)
+		alloc_array[seg_index][sub_index+k] = FREE;
+}
diff --git a/lib/librte_persistent/rte_persistent.h b/lib/librte_persistent/rte_persistent.h
new file mode 100644
index 0000000..b59bd86
--- /dev/null
+++ b/lib/librte_persistent/rte_persistent.h
@@ -0,0 +1,20 @@ 
+/*
+ * rte_persistent.h
+ *
+ *  Created on: Jun 23, 2015
+ *      Author: leeopop
+ */
+
+#ifndef LIBRTE_PERSISTENT_RTE_PERSISTENT_H_
+#define LIBRTE_PERSISTENT_RTE_PERSISTENT_H_
+
+#include <rte_common.h>
+#include <rte_memory.h>
+
+int rte_persistent_init(void);
+void* rte_persistent_alloc(size_t size, int socket);
+phys_addr_t rte_persistent_hw_addr(const void* addr);
+void rte_persistent_free(void* addr);
+size_t rte_persistent_mem_length(const void* addr);
+
+#endif /* LIBRTE_PERSISTENT_RTE_PERSISTENT_H_ */
diff --git a/lib/librte_persistent/rte_persistent_version.map b/lib/librte_persistent/rte_persistent_version.map
new file mode 100644
index 0000000..f81d505
--- /dev/null
+++ b/lib/librte_persistent/rte_persistent_version.map
@@ -0,0 +1,11 @@ 
+DPDK_2.0 {
+	global:
+
+	rte_persistent_init;
+	rte_persistent_alloc;
+	rte_persistent_hw_addr;
+	rte_persistent_free;
+	rte_persistent_mem_length;
+
+	local: *;
+};