[dpdk-dev] [RFC 09/11] malloc: allow removing memory from named heaps

[Anatoly Burakov]

Add an API to remove memory from specified heaps. This will first
check if all elements within the region are free, and that the
region is the original region that was added to the heap (by
comparing its length to length of memory addressed by the
underlying memseg list).

Signed-off-by: Anatoly Burakov <anatoly.burakov at intel.com>
---
 lib/librte_eal/common/include/rte_malloc.h | 28 ++++++++++
 lib/librte_eal/common/malloc_heap.c        | 61 ++++++++++++++++++++++
 lib/librte_eal/common/malloc_heap.h        |  4 ++
 lib/librte_eal/common/rte_malloc.c         | 28 ++++++++++
 lib/librte_eal/rte_eal_version.map         |  1 +
 5 files changed, 122 insertions(+)

diff --git a/lib/librte_eal/common/include/rte_malloc.h b/lib/librte_eal/common/include/rte_malloc.h
index 5f933993b..25d8d3f11 100644
--- a/lib/librte_eal/common/include/rte_malloc.h
+++ b/lib/librte_eal/common/include/rte_malloc.h
@@ -318,6 +318,34 @@ int __rte_experimental
 rte_malloc_heap_add_memory(const char *heap_name, void *va_addr, size_t len,
 		rte_iova_t iova_addrs[], unsigned int n_pages, size_t page_sz);
 
+/**
+ * Remove memory area from heap with specified name.
+ *
+ * @note Concurrently adding or removing memory from different heaps is not
+ *   safe.
+ *
+ * @note This function does not need to be called in multiple processes, as
+ *   multiprocess synchronization will happen automatically as far as heap data
+ *   is concerned. However, before accessing pointers to memory in this heap, it
+ *   is responsibility of the user to ensure that the heap memory is accessible
+ *   in all processes.
+ *
+ * @note Memory area must be empty to allow its removal from the heap.
+ *
+ * @param heap_name
+ *   Name of the heap to create.
+ * @param va_addr
+ *   Virtual address to remove from the heap.
+ * @param len
+ *   Length of virtual area to remove from the heap.
+ *
+ * @return
+ *   - 0 on successful creation.
+ *   - -1 on error.
+ */
+int __rte_experimental
+rte_malloc_heap_remove_memory(const char *heap_name, void *va_addr, size_t len);
+
 /**
  * If malloc debug is enabled, check a memory block for header
  * and trailer markers to indicate that all is well with the block.
diff --git a/lib/librte_eal/common/malloc_heap.c b/lib/librte_eal/common/malloc_heap.c
index 29446cac9..27dbf6e60 100644
--- a/lib/librte_eal/common/malloc_heap.c
+++ b/lib/librte_eal/common/malloc_heap.c
@@ -892,6 +892,44 @@ malloc_heap_dump(struct malloc_heap *heap, FILE *f)
 	rte_spinlock_unlock(&heap->lock);
 }
 
+static int
+destroy_seg(struct malloc_elem *elem, size_t len)
+{
+	struct malloc_heap *heap = elem->heap;
+	struct rte_memseg_list *msl;
+
+	/* check if the element is unused */
+	if (elem->state != ELEM_FREE) {
+		rte_errno = EBUSY;
+		return -1;
+	}
+
+	msl = elem->msl;
+
+	/* check if element encompasses the entire memseg list */
+	if (elem->size != len || len != (msl->page_sz * msl->memseg_arr.len)) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+
+	/* destroy the fbarray backing this memory */
+	if (rte_fbarray_destroy(&msl->memseg_arr) < 0)
+		return -1;
+
+	/* reset the memseg list */
+	memset(msl, 0, sizeof(*msl));
+
+	/* this element can be removed */
+	malloc_elem_free_list_remove(elem);
+	malloc_elem_hide_region(elem, elem, len);
+
+	memset(elem, 0, sizeof(*elem));
+
+	heap->total_size -= len;
+
+	return 0;
+}
+
 int
 malloc_heap_add_external_memory(struct malloc_heap *heap, void *va_addr,
 		rte_iova_t iova_addrs[], unsigned int n_pages, size_t page_sz)
@@ -962,6 +1000,29 @@ malloc_heap_add_external_memory(struct malloc_heap *heap, void *va_addr,
 	return 0;
 }
 
+int
+malloc_heap_remove_external_memory(struct malloc_heap *heap, void *va_addr,
+		size_t len)
+{
+	struct malloc_elem *elem = heap->first;
+
+	/* find element with specified va address */
+	while (elem != NULL && elem != va_addr) {
+		elem = elem->next;
+		/* stop if we've blown past our VA */
+		if (elem > (struct malloc_elem *)va_addr) {
+			elem = NULL;
+			break;
+		}
+	}
+	/* check if element was found */
+	if (elem == NULL) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+	return destroy_seg(elem, len);
+}
+
 int
 malloc_heap_create(struct malloc_heap *heap, const char *heap_name)
 {
diff --git a/lib/librte_eal/common/malloc_heap.h b/lib/librte_eal/common/malloc_heap.h
index 3be4656d0..000146365 100644
--- a/lib/librte_eal/common/malloc_heap.h
+++ b/lib/librte_eal/common/malloc_heap.h
@@ -42,6 +42,10 @@ int
 malloc_heap_add_external_memory(struct malloc_heap *heap, void *va_addr,
 		rte_iova_t iova_addrs[], unsigned int n_pages, size_t page_sz);
 
+int
+malloc_heap_remove_external_memory(struct malloc_heap *heap, void *va_addr,
+		size_t len);
+
 int
 malloc_heap_find_named_heap_idx(const char *name);
 
diff --git a/lib/librte_eal/common/rte_malloc.c b/lib/librte_eal/common/rte_malloc.c
index db0f604ad..8d2eb7250 100644
--- a/lib/librte_eal/common/rte_malloc.c
+++ b/lib/librte_eal/common/rte_malloc.c
@@ -313,6 +313,34 @@ rte_malloc_heap_add_memory(const char *heap_name, void *va_addr, size_t len,
 	return ret;
 }
 
+int
+rte_malloc_heap_remove_memory(const char *heap_name, void *va_addr, size_t len)
+{
+	struct malloc_heap *heap = NULL;
+	int ret;
+
+	/* iova_addrs is allowed to be NULL */
+	if (heap_name == NULL || va_addr == NULL || len == 0 ||
+			strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) == 0 ||
+			strnlen(heap_name, RTE_HEAP_NAME_MAX_LEN) ==
+				RTE_HEAP_NAME_MAX_LEN) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+	/* find our heap */
+	heap = malloc_heap_find_named_heap(heap_name);
+	if (heap == NULL) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+
+	rte_spinlock_lock(&heap->lock);
+	ret = malloc_heap_remove_external_memory(heap, va_addr, len);
+	rte_spinlock_unlock(&heap->lock);
+
+	return ret;
+}
+
 int
 rte_malloc_heap_create(const char *heap_name)
 {
diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
index 6290cc910..7ee79051f 100644
--- a/lib/librte_eal/rte_eal_version.map
+++ b/lib/librte_eal/rte_eal_version.map
@@ -282,6 +282,7 @@ EXPERIMENTAL {
 	rte_malloc_get_stats_from_heap;
 	rte_malloc_heap_add_memory;
 	rte_malloc_heap_create;
+	rte_malloc_heap_remove_memory;
 	rte_mem_alloc_validator_register;
 	rte_mem_alloc_validator_unregister;
 	rte_mem_event_callback_register;
-- 
2.17.1