userspace: application: move to vregion

Modules, running in userspace while using the "application" DP
implementation, are moved to vregion for all their private
allocations. This has the advantage of not depending on
build-time configured VMH buffers and of faster lifetime allocations.

Signed-off-by: Guennadi Liakhovetski <guennadi.liakhovetski@linux.intel.com>

Author: lyakh

src/audio/buffers/comp_buffer.c

@@ -157,14 +157,12 @@ static void comp_buffer_free(struct sof_audio_buffer *audio_buffer)
 	struct mod_alloc_ctx *alloc = buffer->audio_buffer.alloc;
 
 	rfree(buffer->stream.addr);
-	if (alloc && alloc->vreg)
+	if (alloc && alloc->vreg) {
 		vregion_free(alloc->vreg, buffer);
-	else
+		if (!vregion_put(alloc->vreg))
+			rfree(alloc);
+	} else {
 		sof_heap_free(alloc ? alloc->heap : NULL, buffer);
-
-	if (alloc && !--alloc->client_count) {
-		vregion_put(alloc->vreg);
-		rfree(alloc);
 	}
 }
 

src/audio/module_adapter/module/generic.c

@@ -17,6 +17,7 @@
 #include <sof/audio/module_adapter/module/generic.h>
 #include <sof/audio/data_blob.h>
 #include <sof/lib/fast-get.h>
+#include <sof/lib/vregion.h>
 #include <sof/schedule/dp_schedule.h>
 #if CONFIG_IPC_MAJOR_4
 #include <ipc4/header.h>
@@ -87,6 +88,7 @@ void mod_resource_init(struct processing_module *mod)
 	/* Init memory list */
 	list_init(&res->objpool.list);
 	res->objpool.heap = res->alloc->heap;
+	res->objpool.vreg = res->alloc->vreg;
 	res->heap_usage = 0;
 	res->heap_high_water_mark = 0;
 }
@@ -158,11 +160,15 @@ void mod_heap_info(struct processing_module *mod, size_t *size, uintptr_t *start
 {
 	struct module_resources *res = &mod->priv.resources;
 
-	if (size)
-		*size = res->alloc->heap->heap.init_bytes;
+	if (res->alloc->vreg) {
+		vregion_mem_info(res->alloc->vreg, size, start);
+	} else if (res->alloc->heap) {
+		if (size)
+			*size = res->alloc->heap->heap.init_bytes;
 
-	if (start)
-		*start = (uintptr_t)res->alloc->heap;
+		if (start)
+			*start = (uintptr_t)res->alloc->heap->heap.init_mem;
+	}
 }
 #endif
 
@@ -246,7 +252,16 @@ void *z_impl_mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t
 	}
 
 	/* Allocate memory for module */
-	void *ptr = sof_heap_alloc(res->alloc->heap, flags, size, alignment);
+	void *ptr;
+
+	if (!res->alloc->vreg)
+		ptr = sof_heap_alloc(res->alloc->heap, flags, size, alignment);
+	else if (flags & SOF_MEM_FLAG_COHERENT)
+		ptr = vregion_alloc_coherent_align(res->alloc->vreg, VREGION_MEM_TYPE_INTERIM,
+						   size, alignment);
+	else
+		ptr = vregion_alloc_align(res->alloc->vreg, VREGION_MEM_TYPE_INTERIM,
+					  size, alignment);
 
 	if (!ptr) {
 		comp_err(mod->dev, "Failed to alloc %zu bytes %zu alignment for comp %#x.",
@@ -347,7 +362,10 @@ static int free_contents(struct processing_module *mod, struct module_resource *
 
 	switch (container->type) {
 	case MOD_RES_HEAP:
-		sof_heap_free(res->alloc->heap, container->ptr);
+		if (res->alloc->vreg)
+			vregion_free(res->alloc->vreg, container->ptr);
+		else
+			sof_heap_free(res->alloc->heap, container->ptr);
 		res->heap_usage -= container->size;
 		return 0;
 #if CONFIG_COMP_BLOB
@@ -426,10 +444,13 @@ EXPORT_SYMBOL(z_impl_mod_free);
 const void *z_vrfy_mod_fast_get(struct processing_module *mod, const void * const dram_ptr,
 				size_t size)
 {
-	struct module_resources *res = &mod->priv.resources;
+	size_t h_size = 0;
+	uintptr_t h_start;
 
 	K_OOPS(K_SYSCALL_MEMORY_WRITE(mod, sizeof(*mod)));
-	K_OOPS(K_SYSCALL_MEMORY_WRITE(res->alloc->heap, sizeof(*res->alloc->heap)));
+	mod_heap_info(mod, &h_size, &h_start);
+	if (h_size)
+		K_OOPS(K_SYSCALL_MEMORY_WRITE(h_start, h_size));
 	K_OOPS(K_SYSCALL_MEMORY_READ(dram_ptr, size));
 
 	return z_impl_mod_fast_get(mod, dram_ptr, size);
@@ -440,21 +461,27 @@ const void *z_vrfy_mod_fast_get(struct processing_module *mod, const void * cons
 void *z_vrfy_mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t size,
 			   size_t alignment)
 {
-	struct module_resources *res = &mod->priv.resources;
+	size_t h_size = 0;
+	uintptr_t h_start;
 
 	K_OOPS(K_SYSCALL_MEMORY_WRITE(mod, sizeof(*mod)));
-	K_OOPS(K_SYSCALL_MEMORY_WRITE(res->alloc->heap, sizeof(*res->alloc->heap)));
+	mod_heap_info(mod, &h_size, &h_start);
+	if (h_size)
+		K_OOPS(K_SYSCALL_MEMORY_WRITE(h_start, h_size));
 
 	return z_impl_mod_alloc_ext(mod, flags, size, alignment);
 }
 #include <zephyr/syscalls/mod_alloc_ext_mrsh.c>
 
 int z_vrfy_mod_free(struct processing_module *mod, const void *ptr)
 {
-	struct module_resources *res = &mod->priv.resources;
+	size_t h_size = 0;
+	uintptr_t h_start;
 
 	K_OOPS(K_SYSCALL_MEMORY_WRITE(mod, sizeof(*mod)));
-	K_OOPS(K_SYSCALL_MEMORY_WRITE(res->alloc->heap, sizeof(*res->alloc->heap)));
+	mod_heap_info(mod, &h_size, &h_start);
+	if (h_size)
+		K_OOPS(K_SYSCALL_MEMORY_WRITE(h_start, h_size));
 
 	return z_impl_mod_free(mod, ptr);
 }

src/audio/module_adapter/module_adapter.c

@@ -18,6 +18,7 @@
 #include <sof/audio/source_api.h>
 #include <sof/audio/audio_buffer.h>
 #include <sof/audio/pipeline.h>
+#include <sof/lib/vregion.h>
 #include <sof/schedule/dp_schedule.h>
 #include <sof/schedule/ll_schedule_domain.h>
 #include <sof/common.h>
@@ -57,38 +58,29 @@ struct comp_dev *module_adapter_new(const struct comp_driver *drv,
 #define PAGE_SZ HOST_PAGE_SIZE
 #endif
 
-static struct k_heap *module_adapter_dp_heap_new(const struct comp_ipc_config *config,
-						 size_t *heap_size)
+static struct vregion *module_adapter_dp_heap_new(const struct comp_ipc_config *config,
+						  size_t *heap_size)
 {
 	/* src-lite with 8 channels has been seen allocating 14k in one go */
 	/* FIXME: the size will be derived from configuration */
 	const size_t buf_size = 20 * 1024;
 
-	/* Keep uncached to match the default SOF heap! */
-	uint8_t *mod_heap_mem = rballoc_align(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT,
-					      buf_size, PAGE_SZ);
-
-	if (!mod_heap_mem)
-		return NULL;
-
-	struct k_heap *mod_heap = (struct k_heap *)mod_heap_mem;
-	const size_t heap_prefix_size = ALIGN_UP(sizeof(*mod_heap), 4);
-	void *mod_heap_buf = mod_heap_mem + heap_prefix_size;
-
-	*heap_size = buf_size - heap_prefix_size;
-	k_heap_init(mod_heap, mod_heap_buf, *heap_size);
-#ifdef __ZEPHYR__
-	mod_heap->heap.init_mem = mod_heap_buf;
-	mod_heap->heap.init_bytes = *heap_size;
-#endif
-
-	return mod_heap;
+	/*
+	 * A 1-to-1 replacement of the original heap implementation would be to
+	 * have "lifetime size" equal to 0. But (1) this is invalid for
+	 * vregion_create() and (2) we gradually move objects, that are simple
+	 * to move to the lifetime buffer. Make it 1k for the beginning.
+	 */
+	return vregion_create(4096, buf_size - 4096);
 }
 
 static struct processing_module *module_adapter_mem_alloc(const struct comp_driver *drv,
 							  const struct comp_ipc_config *config)
 {
 	struct k_heap *mod_heap;
+	struct vregion *mod_vreg;
+	struct processing_module *mod;
+	struct comp_dev *dev;
 	/*
 	 * For DP shared modules the struct processing_module object must be
 	 * accessible from all cores. Unfortunately at this point there's no
@@ -102,17 +94,24 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv
 
 	if (config->proc_domain == COMP_PROCESSING_DOMAIN_DP && IS_ENABLED(CONFIG_USERSPACE) &&
 	    !IS_ENABLED(CONFIG_SOF_USERSPACE_USE_DRIVER_HEAP)) {
-		mod_heap = module_adapter_dp_heap_new(config, &heap_size);
-		if (!mod_heap) {
-			comp_cl_err(drv, "Failed to allocate DP module heap");
+		mod_vreg = module_adapter_dp_heap_new(config, &heap_size);
+		if (!mod_vreg) {
+			comp_cl_err(drv, "Failed to allocate DP module heap / vregion");
 			return NULL;
 		}
+		mod_heap = NULL;
 	} else {
 		mod_heap = drv->user_heap;
 		heap_size = 0;
+		mod_vreg = NULL;
 	}
 
-	struct processing_module *mod = sof_heap_alloc(mod_heap, flags, sizeof(*mod), 0);
+	if (!mod_vreg)
+		mod = sof_heap_alloc(mod_heap, flags, sizeof(*mod), 0);
+	else if (flags & SOF_MEM_FLAG_COHERENT)
+		mod = vregion_alloc_coherent(mod_vreg, VREGION_MEM_TYPE_LIFETIME, sizeof(*mod));
+	else
+		mod = vregion_alloc(mod_vreg, VREGION_MEM_TYPE_LIFETIME, sizeof(*mod));
 
 	if (!mod) {
 		comp_cl_err(drv, "failed to allocate memory for module");
@@ -126,8 +125,7 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv
 
 	memset(mod, 0, sizeof(*mod));
 	alloc->heap = mod_heap;
-	alloc->vreg = NULL;
-	alloc->client_count = 0;
+	alloc->vreg = mod_vreg;
 	mod->priv.resources.alloc = alloc;
 	mod_resource_init(mod);
 
@@ -137,7 +135,10 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv
 	 * then it can be cached. Effectively it can be only cached in
 	 * single-core configurations.
 	 */
-	struct comp_dev *dev = sof_heap_alloc(mod_heap, SOF_MEM_FLAG_COHERENT, sizeof(*dev), 0);
+	if (mod_vreg)
+		dev = vregion_alloc_coherent(mod_vreg, VREGION_MEM_TYPE_LIFETIME, sizeof(*dev));
+	else
+		dev = sof_heap_alloc(mod_heap, SOF_MEM_FLAG_COHERENT, sizeof(*dev), 0);
 
 	if (!dev) {
 		comp_cl_err(drv, "failed to allocate memory for comp_dev");
@@ -150,17 +151,17 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv
 	mod->dev = dev;
 	dev->mod = mod;
 
-	if (config->proc_domain == COMP_PROCESSING_DOMAIN_DP)
-		alloc->client_count++;
-
 	return mod;
 
 edev:
 	rfree(alloc);
 ealloc:
-	sof_heap_free(mod_heap, mod);
+	if (mod_vreg)
+		vregion_free(mod_vreg, mod);
+	else
+		sof_heap_free(mod_heap, mod);
 emod:
-	rfree(mod_heap);
+	vregion_put(mod_vreg);
 
 	return NULL;
 }
@@ -169,7 +170,6 @@ static void module_adapter_mem_free(struct processing_module *mod)
 {
 	struct mod_alloc_ctx *alloc = mod->priv.resources.alloc;
 	struct k_heap *mod_heap = alloc->heap;
-	unsigned int domain = mod->dev->ipc_config.proc_domain;
 
 	/*
 	 * In principle it shouldn't even be needed to free individual objects
@@ -178,14 +178,16 @@ static void module_adapter_mem_free(struct processing_module *mod)
 #if CONFIG_IPC_MAJOR_4
 	sof_heap_free(mod_heap, mod->priv.cfg.input_pins);
 #endif
-	sof_heap_free(mod_heap, mod->dev);
-	sof_heap_free(mod_heap, mod);
-	if (domain == COMP_PROCESSING_DOMAIN_DP) {
-		if (!IS_ENABLED(CONFIG_SOF_USERSPACE_USE_DRIVER_HEAP))
-			rfree(mod_heap);
-		if (!--alloc->client_count)
+	if (alloc->vreg) {
+		struct vregion *mod_vreg = alloc->vreg;
+
+		vregion_free(mod_vreg, mod->dev);
+		vregion_free(mod_vreg, mod);
+		if (!vregion_put(mod_vreg))
 			rfree(alloc);
 	} else {
+		sof_heap_free(mod_heap, mod->dev);
+		sof_heap_free(mod_heap, mod);
 		rfree(alloc);
 	}
 }
@@ -631,9 +633,7 @@ int module_adapter_prepare(struct comp_dev *dev)
 				goto free;
 			}
 
-			if (md->resources.alloc->heap &&
-			    md->resources.alloc->heap != dev->drv->user_heap)
-				md->resources.alloc->client_count++;
+			vregion_get(md->resources.alloc->vreg);
 
 			irq_local_disable(flags);
 			list_item_prepend(&buffer->buffers_list, &mod->raw_data_buffers_list);

src/include/sof/audio/component.h

@@ -583,7 +583,6 @@ struct vregion;
 struct mod_alloc_ctx {
 	struct k_heap *heap;
 	struct vregion *vreg;
-	unsigned int client_count;
 };
 
 /**

src/include/sof/objpool.h

@@ -12,9 +12,11 @@
 #include <stddef.h>
 #include <stdint.h>
 
+struct vregion;
 struct k_heap;
 struct objpool_head {
 	struct list_item list;
+	struct vregion *vreg;
 	struct k_heap *heap;
 	uint32_t flags;
 };

src/ipc/ipc4/helper.c

@@ -21,6 +21,7 @@
 #include <sof/ipc/msg.h>
 #include <sof/lib/mailbox.h>
 #include <sof/lib/memory.h>
+#include <sof/lib/vregion.h>
 #include <sof/list.h>
 #include <sof/platform.h>
 #include <sof/schedule/dp_schedule.h>
@@ -739,7 +740,7 @@ __cold int ipc_comp_connect(struct ipc *ipc, ipc_pipe_comp_connect *_connect)
 
 #if CONFIG_ZEPHYR_DP_SCHEDULER
 	if (alloc)
-		alloc->client_count++;
+		vregion_get(alloc->vreg);
 #endif
 
 	/*

src/lib/objpool.c

@@ -7,6 +7,7 @@
 #include <sof/objpool.h>
 #include <sof/common.h>
 #include <sof/list.h>
+#include <sof/lib/vregion.h>
 
 #include <errno.h>
 #include <limits.h>
@@ -37,8 +38,17 @@ static int objpool_add(struct objpool_head *head, unsigned int n, size_t size, u
 	if (!head->heap)
 		head->heap = sof_sys_heap_get();
 
-	struct objpool *pobjpool = sof_heap_alloc(head->heap, flags,
-						  aligned_size + sizeof(*pobjpool), 0);
+	struct objpool *pobjpool;
+
+	if (!head->vreg)
+		pobjpool = sof_heap_alloc(head->heap, flags,
+					  aligned_size + sizeof(*pobjpool), 0);
+	else if (flags & SOF_MEM_FLAG_COHERENT)
+		pobjpool = vregion_alloc_coherent(head->vreg, VREGION_MEM_TYPE_INTERIM,
+						  aligned_size + sizeof(*pobjpool));
+	else
+		pobjpool = vregion_alloc(head->vreg, VREGION_MEM_TYPE_INTERIM,
+					 aligned_size + sizeof(*pobjpool));
 
 	if (!pobjpool)
 		return -ENOMEM;
@@ -150,8 +160,13 @@ void objpool_prune(struct objpool_head *head)
 	struct list_item *next, *tmp;
 
 	list_for_item_safe(next, tmp, &head->list) {
+		struct objpool *pool = container_of(next, struct objpool, list);
+
 		list_item_del(next);
-		sof_heap_free(head->heap, container_of(next, struct objpool, list));
+		if (head->vreg)
+			vregion_free(head->vreg, pool);
+		else
+			sof_heap_free(head->heap, pool);
 	}
 }