diff --git a/src/audio/module_adapter/library/userspace_proxy.c b/src/audio/module_adapter/library/userspace_proxy.c index 3859fbd129f9..a45a85df8653 100644 --- a/src/audio/module_adapter/library/userspace_proxy.c +++ b/src/audio/module_adapter/library/userspace_proxy.c @@ -26,6 +26,7 @@ #include #include +#include #include #include #include @@ -163,6 +164,7 @@ static int user_work_item_init(struct userspace_context *user_ctx, struct k_heap work_item->event = &worker.event; #endif work_item->params.context = user_ctx; + work_item->params.mod = NULL; user_ctx->work_item = work_item; return 0; @@ -198,7 +200,7 @@ static int userspace_proxy_invoke(struct userspace_context *user_ctx, uint32_t c struct k_mem_partition ipc_part = { .start = ipc_req_buf, .size = MAILBOX_HOSTBOX_SIZE, - .attr = user_get_partition_attr(ipc_req_buf) | K_MEM_PARTITION_P_RO_U_RO, + .attr = user_get_partition_cache_attr(ipc_req_buf) | K_MEM_PARTITION_P_RO_U_RO, }; int ret = 0, ret2; @@ -274,8 +276,25 @@ static int userspace_proxy_memory_init(struct userspace_context *user_ctx, tr_dbg(&userspace_proxy_tr, "Heap partition %#lx + %zx, attr = %u", heap_part.start, heap_part.size, heap_part.attr); -#if !defined(CONFIG_XTENSA_MMU_DOUBLE_MAP) && defined(CONFIG_SOF_ZEPHYR_HEAP_CACHED) -#define HEAP_PART_CACHED + /* When a new memory domain is created, only the "factory" entries from the L2 page + * tables are copied. Memory that was dynamically mapped during firmware execution + * will not be accessible from the new domain. The k_heap structure (drv->user_heap) + * resides in such dynamically mapped memory, so we must explicitly add a partition + * for it to ensure that syscalls can access this structure from the userspace domain. + */ + struct k_mem_partition heap_struct_part = { + .attr = K_MEM_PARTITION_P_RW_U_NA | + user_get_partition_cache_attr(POINTER_TO_UINT(drv->user_heap)) + }; + + k_mem_region_align(&heap_struct_part.start, &heap_struct_part.size, + POINTER_TO_UINT(drv->user_heap), + sizeof(*drv->user_heap), CONFIG_MM_DRV_PAGE_SIZE); + + tr_dbg(&userspace_proxy_tr, "Heap struct partition %#lx + %zx, attr = %u", + heap_struct_part.start, heap_struct_part.size, heap_struct_part.attr); + +#if defined(CONFIG_SOF_ZEPHYR_HEAP_CACHED) /* Add cached module private heap to memory partitions */ struct k_mem_partition heap_cached_part = { .attr = K_MEM_PARTITION_P_RW_U_RW | XTENSA_MMU_CACHED_WB @@ -294,10 +313,11 @@ static int userspace_proxy_memory_init(struct userspace_context *user_ctx, * These include ops structures marked with APP_TASK_DATA. */ &common_partition, -#ifdef HEAP_PART_CACHED +#ifdef CONFIG_SOF_ZEPHYR_HEAP_CACHED &heap_cached_part, #endif - &heap_part + &heap_part, + &heap_struct_part }; tr_dbg(&userspace_proxy_tr, "Common partition %#lx + %zx, attr = %u", @@ -326,7 +346,7 @@ static int userspace_proxy_add_sections(struct userspace_context *user_ctx, uint mem_partition.start = mod->segment[idx].v_base_addr; mem_partition.size = mod->segment[idx].flags.r.length * CONFIG_MM_DRV_PAGE_SIZE; - mem_partition.attr |= user_get_partition_attr(mem_partition.start); + mem_partition.attr |= user_get_partition_cache_attr(mem_partition.start); ret = k_mem_domain_add_partition(user_ctx->comp_dom, &mem_partition); @@ -339,7 +359,7 @@ static int userspace_proxy_add_sections(struct userspace_context *user_ctx, uint lib_manager_get_instance_bss_address(instance_id, mod, &va_base, &mem_partition.size); mem_partition.start = POINTER_TO_UINT(va_base); - mem_partition.attr = user_get_partition_attr(mem_partition.start) | + mem_partition.attr = user_get_partition_cache_attr(mem_partition.start) | K_MEM_PARTITION_P_RW_U_RW; ret = k_mem_domain_add_partition(user_ctx->comp_dom, &mem_partition); @@ -382,7 +402,7 @@ static int userspace_proxy_start_agent(struct userspace_context *user_ctx, } int userspace_proxy_create(struct userspace_context **user_ctx, const struct comp_driver *drv, - const struct sof_man_module *manifest, system_agent_start_fn start_fn, + const struct sof_man_module *manifest, system_agent_start_fn agent_fn, const struct system_agent_params *agent_params, const void **agent_interface, const struct module_interface **ops) { @@ -410,7 +430,12 @@ int userspace_proxy_create(struct userspace_context **user_ctx, const struct com if (ret) goto error_dom; - ret = userspace_proxy_add_sections(context, agent_params->instance_id, manifest); + if (agent_fn) + ret = userspace_proxy_add_sections(context, agent_params->instance_id, manifest); + else + /* llext modules do not use the system agent. */ + ret = llext_manager_add_domain(agent_params->module_id, domain); + if (ret) goto error_dom; @@ -418,7 +443,7 @@ int userspace_proxy_create(struct userspace_context **user_ctx, const struct com if (ret) goto error_dom; - ret = userspace_proxy_start_agent(context, start_fn, agent_params, agent_interface); + ret = userspace_proxy_start_agent(context, agent_fn, agent_params, agent_interface); if (ret) { tr_err(&userspace_proxy_tr, "System agent failed with error %d.", ret); goto error_work_item; @@ -680,7 +705,7 @@ static int userspace_proxy_get_configuration(struct processing_module *mod, uint struct k_mem_partition ipc_resp_part = { .start = ipc_resp_buf, .size = SOF_IPC_MSG_MAX_SIZE, - .attr = user_get_partition_attr(ipc_resp_buf) | K_MEM_PARTITION_P_RW_U_RW, + .attr = user_get_partition_cache_attr(ipc_resp_buf) | K_MEM_PARTITION_P_RW_U_RW, }; int ret; diff --git a/src/include/module/module/llext.h b/src/include/module/module/llext.h index 2f05cb6c692a..2ef0d47d77cc 100644 --- a/src/include/module/module/llext.h +++ b/src/include/module/module/llext.h @@ -6,7 +6,7 @@ #ifndef MODULE_LLEXT_H #define MODULE_LLEXT_H -#define SOF_LLEXT_MODULE_MANIFEST(manifest_name, entry, affinity, mod_uuid, instances) \ +#define SOF_LLEXT_MODULE_MANIFEST(manifest_name, entry, affinity, mod_uuid, instances, ...) \ { \ .module = { \ .name = manifest_name, \ @@ -16,6 +16,8 @@ .type = { \ .load_type = SOF_MAN_MOD_TYPE_LLEXT, \ .domain_ll = 1, \ + .user_mode = COND_CODE_0(NUM_VA_ARGS_LESS_1(_, ##__VA_ARGS__), (0), \ + (GET_ARG_N(1, __VA_ARGS__))), \ }, \ .affinity_mask = (affinity), \ } \ diff --git a/src/library_manager/lib_manager.c b/src/library_manager/lib_manager.c index b81ec6bbe738..951828808b7c 100644 --- a/src/library_manager/lib_manager.c +++ b/src/library_manager/lib_manager.c @@ -543,12 +543,14 @@ static int lib_manager_start_agent(const struct comp_driver *drv, return ret; } #endif /* CONFIG_SOF_USERSPACE_PROXY */ + if (agent) { + ret = agent(&agent_params, agent_interface); + if (ret) + tr_err(&lib_manager_tr, "System agent start failed %d!", ret); + return ret; + } - ret = agent(&agent_params, agent_interface); - if (ret) - tr_err(&lib_manager_tr, "System agent start failed %d!", ret); - - return ret; + return 0; } enum buildinfo_mod_type { MOD_TYPE_INVALID, MOD_TYPE_IADK, MOD_TYPE_LMDK, MOD_TYPE_LLEXT }; @@ -654,6 +656,7 @@ static struct comp_dev *lib_manager_module_create(const struct comp_driver *drv, case MOD_TYPE_LLEXT: agent = NULL; ops = (const struct module_interface *)module_entry_point; + agent_iface = NULL; break; case MOD_TYPE_LMDK: agent = &native_system_agent_start; @@ -671,12 +674,10 @@ static struct comp_dev *lib_manager_module_create(const struct comp_driver *drv, } /* At this point module resources are allocated and it is moved to L2 memory. */ - if (agent) { - ret = lib_manager_start_agent(drv, config, mod, args, module_entry_point, agent, - agent_iface, &userspace, &ops); - if (ret) - goto err; - } + ret = lib_manager_start_agent(drv, config, mod, args, module_entry_point, agent, + agent_iface, &userspace, &ops); + if (ret) + goto err; if (comp_set_adapter_ops(drv, ops) < 0) goto err; diff --git a/tools/rimage/src/manifest.c b/tools/rimage/src/manifest.c index 6f3a161e37a7..fb30dc7ab219 100644 --- a/tools/rimage/src/manifest.c +++ b/tools/rimage/src/manifest.c @@ -212,6 +212,7 @@ static void man_get_section_manifest(struct image *image, man_module->type.domain_dp = sof_mod->module.type.domain_dp; man_module->type.domain_ll = sof_mod->module.type.domain_ll; man_module->type.load_type = sof_mod->module.type.load_type; + man_module->type.user_mode = sof_mod->module.type.user_mode; /* text segment */ segment = &man_module->segment[SOF_MAN_SEGMENT_TEXT]; diff --git a/zephyr/include/rtos/userspace_helper.h b/zephyr/include/rtos/userspace_helper.h index 29635fb942ad..8998785e3606 100644 --- a/zephyr/include/rtos/userspace_helper.h +++ b/zephyr/include/rtos/userspace_helper.h @@ -109,7 +109,7 @@ int user_access_to_mailbox(struct k_mem_domain *domain, k_tid_t thread_id); * * @param ptr Address of the partition start */ -static inline uint32_t user_get_partition_attr(uintptr_t ptr) +static inline uint32_t user_get_partition_cache_attr(uintptr_t ptr) { return sys_cache_is_ptr_cached(UINT_TO_POINTER(ptr)) ? XTENSA_MMU_CACHED_WB : 0; } diff --git a/zephyr/lib/fast-get.c b/zephyr/lib/fast-get.c index c681252ae27a..c49501f6d8ac 100644 --- a/zephyr/lib/fast-get.c +++ b/zephyr/lib/fast-get.c @@ -150,7 +150,11 @@ const void *fast_get(struct k_heap *heap, const void *dram_ptr, size_t size) alloc_align = PLATFORM_DCACHE_ALIGN; } - if (size > FAST_GET_MAX_COPY_SIZE || !IS_ENABLED(CONFIG_USERSPACE)) + /* The module driver heap is shared by all instances of a given module. + * Instances can share the allocated buffer. + */ + if (size > FAST_GET_MAX_COPY_SIZE || !IS_ENABLED(CONFIG_USERSPACE) || + IS_ENABLED(CONFIG_SOF_USERSPACE_USE_DRIVER_HEAP)) alloc_ptr = dram_ptr; else /* When userspace is enabled only share large buffers */ @@ -188,8 +192,12 @@ const void *fast_get(struct k_heap *heap, const void *dram_ptr, size_t size) /* * We only get there for large buffers, since small buffers with * enabled userspace don't create fast-get entries + * + * We also reach this point when using the module driver heap. + * Since the heap is already shared across module instances, + * we skip memory domain manipulation. */ - if (mdom->num_partitions > 1) { + if (k_current_get()->base.user_options & K_USER && size > FAST_GET_MAX_COPY_SIZE) { /* A userspace thread makes the request */ if (mdom != entry->mdom && !fast_get_partition_exists(k_current_get(), ret, diff --git a/zephyr/lib/userspace_helper.c b/zephyr/lib/userspace_helper.c index 8c4aef423e15..cce57ac8ee4e 100644 --- a/zephyr/lib/userspace_helper.c +++ b/zephyr/lib/userspace_helper.c @@ -23,8 +23,6 @@ #include #include -#define MODULE_DRIVER_HEAP_CACHED CONFIG_SOF_ZEPHYR_HEAP_CACHED - /* Zephyr includes */ #include #include