diff --git a/src/audio/buffers/comp_buffer.c b/src/audio/buffers/comp_buffer.c index 8a3d44133d4b..37d2cc07e16c 100644 --- a/src/audio/buffers/comp_buffer.c +++ b/src/audio/buffers/comp_buffer.c @@ -166,7 +166,7 @@ static void comp_buffer_free(struct sof_audio_buffer *audio_buffer) if (alloc && alloc->vreg) { vregion_free(alloc->vreg, buffer); if (!vregion_put(alloc->vreg)) - rfree(alloc); + sof_heap_free(alloc->heap, alloc); } else { sof_heap_free(alloc ? alloc->heap : NULL, buffer); } diff --git a/src/audio/module_adapter/module_adapter.c b/src/audio/module_adapter/module_adapter.c index 995501bfbe0a..0ac5de8edf1e 100644 --- a/src/audio/module_adapter/module_adapter.c +++ b/src/audio/module_adapter/module_adapter.c @@ -51,16 +51,6 @@ struct comp_dev *module_adapter_new(const struct comp_driver *drv, return module_adapter_new_ext(drv, config, spec, NULL, NULL, NULL); } -static struct vregion *module_adapter_dp_heap_new(const struct comp_ipc_config *config, - size_t *heap_size) -{ - /* src-lite with 8 channels has been seen allocating 14k in one go */ - /* FIXME: the size will be derived from configuration */ - const size_t buf_size = 28 * 1024; - - return vregion_create(buf_size); -} - static struct processing_module *module_adapter_mem_alloc(const struct comp_driver *drv, const struct comp_ipc_config *config) { @@ -77,11 +67,15 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv */ uint32_t flags = config->proc_domain == COMP_PROCESSING_DOMAIN_DP ? SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT : SOF_MEM_FLAG_USER; - size_t heap_size; + size_t vreg_size; + uintptr_t vreg_start; if (config->proc_domain == COMP_PROCESSING_DOMAIN_DP && IS_ENABLED(CONFIG_SOF_VREGIONS) && IS_ENABLED(CONFIG_USERSPACE) && !IS_ENABLED(CONFIG_SOF_USERSPACE_USE_DRIVER_HEAP)) { - mod_vreg = module_adapter_dp_heap_new(config, &heap_size); + /* src-lite with 8 channels has been seen allocating 14k in one go */ + /* FIXME: the size will be derived from configuration */ + vreg_size = 28 * 1024; + mod_vreg = vregion_create_map(&vreg_start, &vreg_size); if (!mod_vreg) { comp_cl_err(drv, "Failed to allocate DP module heap / vregion"); return NULL; @@ -98,7 +92,8 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv #else mod_heap = drv->user_heap; #endif - heap_size = 0; + vreg_size = 0; + vreg_start = 0; mod_vreg = NULL; } @@ -199,6 +194,7 @@ static void module_adapter_mem_free(struct processing_module *mod) * * Note: Use the ext version if you need to set the module's private data before calling * the create method. + * Note 2: ATM runs in privileged / kernel mode for DP modules */ struct comp_dev *module_adapter_new_ext(const struct comp_driver *drv, const struct comp_ipc_config *config, diff --git a/src/include/sof/lib/vregion.h b/src/include/sof/lib/vregion.h index 5c066c90dbc8..559c47ed3431 100644 --- a/src/include/sof/lib/vregion.h +++ b/src/include/sof/lib/vregion.h @@ -6,6 +6,8 @@ #define __SOF_LIB_VREGION_H__ #include +#include +#include #ifdef __cplusplus extern "C" { @@ -40,6 +42,13 @@ enum vregion_mem_type { */ struct vregion *vregion_create(size_t memsize); +/** + * @brief Create and map a new virtual region instance. + * + * Like above, but usable by userspace and can add domain access. + */ +__syscall struct vregion *vregion_create_map(uintptr_t *vreg_start, size_t *vreg_size); + /** * @brief Switch virtual region allocations to interim mode. * @@ -49,7 +58,7 @@ struct vregion *vregion_create(size_t memsize); * * @param[in] vr Pointer to the virtual region instance. */ -void vregion_set_interim(struct vregion *vr); +__syscall void vregion_set_interim(struct vregion *vr); /** * @brief Increment virtual region's user count. @@ -60,7 +69,7 @@ void vregion_set_interim(struct vregion *vr); * @param[in] vr Pointer to the virtual region instance to release. * @return struct vregion* Pointer to the virtual region instance. */ -struct vregion *vregion_get(struct vregion *vr); +__syscall struct vregion *vregion_get(struct vregion *vr); /** * @brief Decrement virtual region's user count or destroy it. @@ -71,39 +80,45 @@ struct vregion *vregion_get(struct vregion *vr); * @param[in] vr Pointer to the virtual region instance to release. * @return struct vregion* Pointer to the virtual region instance or NULL if it has been destroyed. */ -struct vregion *vregion_put(struct vregion *vr); +__syscall struct vregion *vregion_put(struct vregion *vr); /** - * @brief Allocate memory from the specified virtual region. + * @brief Allocate aligned memory from the specified virtual region. + * + * Allocate aligned memory from the specified virtual region using the + * current allocation mode (lifetime or interim). * * @param[in] vr Pointer to the virtual region instance. * @param[in] size Size of memory to allocate in bytes. + * @param[in] alignment Alignment of memory to allocate in bytes. * @return void* Pointer to the allocated memory, or NULL on failure. */ -void *vregion_alloc(struct vregion *vr, size_t size); +__syscall void *vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment); /** - * @brief like vregion_alloc() but allocates coherent memory + * @brief like vregion_alloc_align() but allocates coherent memory */ -void *vregion_alloc_coherent(struct vregion *vr, size_t size); +__syscall void *vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t alignment); /** - * @brief Allocate aligned memory from the specified virtual region. - * - * Allocate aligned memory from the specified virtual region using the - * current allocation mode (lifetime or interim). + * @brief Allocate memory from the specified virtual region. * * @param[in] vr Pointer to the virtual region instance. * @param[in] size Size of memory to allocate in bytes. - * @param[in] alignment Alignment of memory to allocate in bytes. * @return void* Pointer to the allocated memory, or NULL on failure. */ -void *vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment); +static inline void *vregion_alloc(struct vregion *vr, size_t size) +{ + return vregion_alloc_align(vr, size, 0); +} /** - * @brief like vregion_alloc_align() but allocates coherent memory + * @brief like vregion_alloc() but allocates coherent memory */ -void *vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t alignment); +static inline void *vregion_alloc_coherent(struct vregion *vr, size_t size) +{ + return vregion_alloc_coherent_align(vr, size, 0); +} /** * @brief Free memory allocated from the specified virtual region. @@ -113,7 +128,7 @@ void *vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t align * @param[in] vr Pointer to the virtual region instance. * @param[in] ptr Pointer to the memory to free. */ -void vregion_free(struct vregion *vr, void *ptr); +__syscall void vregion_free(struct vregion *vr, void *ptr); /** * @brief Log virtual region memory usage. @@ -131,6 +146,10 @@ void vregion_info(struct vregion *vr); */ void vregion_mem_info(struct vregion *vr, size_t *size, uintptr_t *start); +bool vregion_verify(struct vregion *vr); + +#include + #else /* CONFIG_SOF_VREGIONS */ struct vregion { @@ -141,6 +160,10 @@ static inline struct vregion *vregion_create(size_t memsize) { return NULL; } +static inline struct vregion *vregion_create_map(uintptr_t *vreg_start, size_t *vreg_size) +{ + return NULL; +} static inline void vregion_set_interim(struct vregion *vr) {} static inline struct vregion *vregion_get(struct vregion *vr) { @@ -174,6 +197,7 @@ static inline void vregion_mem_info(struct vregion *vr, size_t *size, uintptr_t if (size) *size = 0; } +static inline bool vregion_verify(struct vregion *vr) {return false;} #endif /* CONFIG_SOF_VREGIONS */ diff --git a/src/schedule/zephyr_dp_schedule_application.c b/src/schedule/zephyr_dp_schedule_application.c index daf9070ae4dc..547823ca6e6f 100644 --- a/src/schedule/zephyr_dp_schedule_application.c +++ b/src/schedule/zephyr_dp_schedule_application.c @@ -425,7 +425,7 @@ static void scheduler_dp_thread_name_set(k_tid_t thread_id, struct processing_mo #define scheduler_dp_thread_name_set(x, y) #endif -/* Called only in IPC context */ +/* Called only in IPC context in kernel mode (this can change) */ int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid, const struct task_ops *ops, struct processing_module *mod, uint16_t core, size_t stack_size, uint32_t options) @@ -437,6 +437,7 @@ int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid, /* must be called on the same core the task will be bound to */ assert(cpu_get_id() == core); + assert(!k_is_user_context()); /* * allocate memory diff --git a/zephyr/CMakeLists.txt b/zephyr/CMakeLists.txt index e0e7e8bfb302..c8f49c1f07cb 100644 --- a/zephyr/CMakeLists.txt +++ b/zephyr/CMakeLists.txt @@ -629,6 +629,8 @@ zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/schedule/ll_schedule_domain.h) zephyr_syscall_header(${SOF_SRC_PATH}/include/ipc4/handler.h) zephyr_syscall_header(include/rtos/alloc.h) zephyr_library_sources_ifdef(CONFIG_SOF_USERSPACE_INTERFACE_ALLOC syscall/alloc.c) +zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib/vregion.h) +zephyr_library_sources_ifdef(CONFIG_SOF_USERSPACE_INTERFACE_VREGION syscall/vregion.c) zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib/dai-zephyr.h) zephyr_library_sources_ifdef(CONFIG_USERSPACE syscall/dai.c) diff --git a/zephyr/Kconfig b/zephyr/Kconfig index ab88efcb9a6a..2d03a6d6f7ae 100644 --- a/zephyr/Kconfig +++ b/zephyr/Kconfig @@ -36,11 +36,20 @@ config SOF_USERSPACE_INTERFACE_ALLOC Allow user-space threads to use sof_heap_alloc/sof_heap_free as Zephyr system calls. +config SOF_USERSPACE_INTERFACE_VREGION + bool "Enable SOF vregion interface to userspace threads" + depends on USERSPACE + depends on SOF_VREGIONS + help + Allow user-space threads to use vregion_alloc/vregion_free + and their variants as Zephyr system calls. + config SOF_USERSPACE_LL bool "Run Low-Latency pipelines in userspace threads" depends on USERSPACE select SOF_USERSPACE_INTERFACE_ALLOC select SOF_USERSPACE_INTERFACE_DMA + select SOF_USERSPACE_INTERFACE_VREGION if SOF_VREGIONS help Run Low-Latency (LL) pipelines in userspace threads. This adds memory protection between operating system resources and diff --git a/zephyr/lib/userspace_helper.c b/zephyr/lib/userspace_helper.c index ea9db4181b9d..e205fc256308 100644 --- a/zephyr/lib/userspace_helper.c +++ b/zephyr/lib/userspace_helper.c @@ -110,7 +110,7 @@ int user_access_to_mailbox(struct k_mem_domain *domain, k_tid_t thread_id) if (ret < 0) return ret; -#if defined(CONFIG_SOF_USERSPACE_LL) && defined(CONFIG_IPC_MAJOR_4) +#ifdef CONFIG_IPC_MAJOR_4 /* HOSTBOX partitions for IPC4 module init parameter block reads. * comp_new_ipc4() accesses MAILBOX_HOSTBOX_BASE directly to get * the module configuration data sent by the host. @@ -141,9 +141,7 @@ int user_access_to_mailbox(struct k_mem_domain *domain, k_tid_t thread_id) ret = k_mem_domain_add_partition(domain, &hostbox_partition); if (ret < 0) return ret; -#endif /* CONFIG_IPC_MAJOR_4 */ - -#ifndef CONFIG_IPC_MAJOR_4 +#else /* CONFIG_IPC_MAJOR_4 */ /* * Next mailbox_stream (not available in IPC4). Stream access is cached, * so different mapping this time. diff --git a/zephyr/lib/vregion.c b/zephyr/lib/vregion.c index 9c8c94c23f97..e8cfb2347246 100644 --- a/zephyr/lib/vregion.c +++ b/zephyr/lib/vregion.c @@ -8,11 +8,13 @@ #include #include #include +#include #include #include #include #include #include +#include LOG_MODULE_REGISTER(vregion, CONFIG_SOF_LOG_LEVEL); @@ -87,6 +89,7 @@ struct vregion { unsigned int pages; /* size of whole region in pages */ struct k_mutex lock; /* protect vregion heaps and use-count */ unsigned int use_count; + struct k_mem_domain *domain; /* current allocation mode */ enum vregion_mem_type type; /* LIFETIME at creation, switch to INTERIM */ @@ -144,6 +147,7 @@ struct vregion *vregion_create(size_t memsize) vr->base = vregion_base; vr->size = total_size; vr->pages = pages; + vr->domain = NULL; /* lifetime linear allocator starts at the beginning of the vregion memory */ vr->lifetime.base = vregion_base; @@ -161,12 +165,12 @@ struct vregion *vregion_create(size_t memsize) /* log the new vregion */ LOG_INF("new at base %p size %#zx pages %u metadata at %p", - (void *)vr->base, total_size, pages, (void *)vr); + (void *)vregion_base, total_size, pages, (void *)vr); return vr; } -struct vregion *vregion_get(struct vregion *vr) +struct vregion *z_impl_vregion_get(struct vregion *vr) { if (!vr) return NULL; @@ -178,13 +182,79 @@ struct vregion *vregion_get(struct vregion *vr) return vr; } +struct vregion *z_impl_vregion_create_map(uintptr_t *vreg_start, size_t *vreg_size) +{ + if (!vreg_start || !vreg_size || !*vreg_size) + return NULL; + + struct vregion *vr = vregion_create(*vreg_size); + + if (!vr) + return NULL; + +#if CONFIG_USERSPACE && CONFIG_SOF_USERSPACE_LL + vregion_mem_info(vr, vreg_size, vreg_start); + + /* + * In the userspace LL case allocations are also performed by the + * userspace IPC thread, which is also the one, executing this syscall + */ + struct k_mem_domain *domain = zephyr_ll_mem_domain(); + struct k_mem_partition part = { + .start = *vreg_start, + .size = *vreg_size, + .attr = K_MEM_PARTITION_P_RW_U_RW | XTENSA_MMU_CACHED_WB, + }; + int ret = k_mem_domain_add_partition(domain, &part); + + if (ret < 0) { + vregion_put(vr); + return NULL; + } + + part.start = (uintptr_t)sys_cache_uncached_ptr_get((void *)part.start); + part.attr = K_MEM_PARTITION_P_RW_U_RW; + + ret = k_mem_domain_add_partition(domain, &part); + if (ret < 0) { + vregion_put(vr); + return NULL; + } + + vr->domain = domain; +#endif + + return vr; +} + +static void vregion_unmap(struct vregion *vr) +{ +#if CONFIG_USERSPACE && CONFIG_SOF_USERSPACE_LL + if (!vr->domain) + return; + + struct k_mem_partition part = { + .attr = K_MEM_PARTITION_P_RW_U_RW | XTENSA_MMU_CACHED_WB, + .start = (uintptr_t)vr->base, + .size = vr->size, + }; + + k_mem_domain_remove_partition(vr->domain, &part); + + part.start = (uintptr_t)sys_cache_uncached_ptr_get((void *)part.start); + part.attr = K_MEM_PARTITION_P_RW_U_RW; + + k_mem_domain_remove_partition(vr->domain, &part); +#endif +} + /** * @brief Decrement virtual region's user count or destroy it. * * @param[in] vr Pointer to the virtual region instance to release. * @return struct vregion* Pointer to the virtual region instance or NULL if it has been destroyed. */ -struct vregion *vregion_put(struct vregion *vr) +struct vregion *z_impl_vregion_put(struct vregion *vr) { unsigned int use_count; @@ -204,6 +274,8 @@ struct vregion *vregion_put(struct vregion *vr) LOG_DBG("destroy %p size %#zx pages %u", (void *)vr->base, vr->size, vr->pages); LOG_DBG(" lifetime used %zu free count %d", vr->lifetime.used, vr->lifetime.free_count); vpage_free(vr->base); + + vregion_unmap(vr); rfree(vr); return NULL; @@ -259,7 +331,7 @@ static void interim_heap_init(struct vregion *vr) vr->lifetime.used = (uint8_t *)vr->lifetime.ptr - (uint8_t *)vr->lifetime.base; } -void vregion_set_interim(struct vregion *vr) +void z_impl_vregion_set_interim(struct vregion *vr) { if (!vr) return; @@ -365,7 +437,7 @@ static void lifetime_free(struct vlinear_heap *heap, void *ptr) * @param vr Pointer to the virtual region instance. * @param ptr Pointer to the memory to free. */ -void vregion_free(struct vregion *vr, void *ptr) +void z_impl_vregion_free(struct vregion *vr, void *ptr) { if (!vr || !ptr) return; @@ -390,7 +462,7 @@ void vregion_free(struct vregion *vr, void *ptr) k_mutex_unlock(&vr->lock); } -EXPORT_SYMBOL(vregion_free); +EXPORT_SYMBOL(z_impl_vregion_free); /** * @brief Allocate memory from the virtual region. @@ -401,7 +473,8 @@ EXPORT_SYMBOL(vregion_free); * * @return void* Pointer to the allocated memory, or NULL on failure. */ -void *vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment) +void *z_impl_vregion_alloc_align(struct vregion *vr, + size_t size, size_t alignment) { void *p; @@ -429,41 +502,15 @@ void *vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment) return p; } -EXPORT_SYMBOL(vregion_alloc_align); +EXPORT_SYMBOL(z_impl_vregion_alloc_align); -/** - * @brief Allocate memory from the virtual region. - * @param[in] vr Pointer to the virtual region instance. - * @param[in] size Size of the allocation. - * @return void* Pointer to the allocated memory, or NULL on failure. - */ -void *vregion_alloc(struct vregion *vr, size_t size) -{ - return vregion_alloc_align(vr, size, 0); -} -EXPORT_SYMBOL(vregion_alloc); - -void *vregion_alloc_coherent(struct vregion *vr, size_t size) -{ - size = ALIGN_UP(size, CONFIG_DCACHE_LINE_SIZE); - - void *p = vregion_alloc_align(vr, size, CONFIG_DCACHE_LINE_SIZE); - - if (!p) - return NULL; - - sys_cache_data_invd_range(p, size); - - return sys_cache_uncached_ptr_get(p); -} - -void *vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t alignment) +void *z_impl_vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t alignment) { if (alignment < CONFIG_DCACHE_LINE_SIZE) alignment = CONFIG_DCACHE_LINE_SIZE; size = ALIGN_UP(size, CONFIG_DCACHE_LINE_SIZE); - void *p = vregion_alloc_align(vr, size, alignment); + void *p = z_impl_vregion_alloc_align(vr, size, alignment); if (!p) return NULL; @@ -472,6 +519,7 @@ void *vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t align return sys_cache_uncached_ptr_get(p); } +EXPORT_SYMBOL(z_impl_vregion_alloc_coherent_align); /** * @brief Log virtual region memory usage. @@ -488,7 +536,6 @@ void vregion_info(struct vregion *vr) LOG_INF("lifetime used %#zx free count %d", vr->lifetime.used, vr->lifetime.free_count); } -EXPORT_SYMBOL(vregion_info); void vregion_mem_info(struct vregion *vr, size_t *size, uintptr_t *start) { @@ -498,3 +545,25 @@ void vregion_mem_info(struct vregion *vr, size_t *size, uintptr_t *start) if (start) *start = (uintptr_t)vr->base; } + +#if CONFIG_SOF_VREGIONS && CONFIG_USERSPACE +#include + +bool vregion_verify(struct vregion *vr) +{ + if (!vr) + return false; + + /* vregion instances must not be accessible to the userspace. */ + K_OOPS(!K_SYSCALL_MEMORY_READ(vr, sizeof(*vr))); + + size_t vr_size = 0; + uintptr_t vr_start; + + vregion_mem_info(vr, &vr_size, &vr_start); + if (vr_size) + K_OOPS(K_SYSCALL_MEMORY_WRITE((void *)vr_start, vr_size)); + + return true; +} +#endif diff --git a/zephyr/syscall/vregion.c b/zephyr/syscall/vregion.c new file mode 100644 index 000000000000..dd02f854cf82 --- /dev/null +++ b/zephyr/syscall/vregion.c @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. + +#include +#include +#include + +static inline void *z_vrfy_vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment) +{ + if (vregion_verify(vr)) + return z_impl_vregion_alloc_align(vr, size, alignment); + + return NULL; +} +#include + +static inline void *z_vrfy_vregion_alloc_coherent_align(struct vregion *vr, + size_t size, size_t alignment) +{ + if (vregion_verify(vr)) + return z_impl_vregion_alloc_coherent_align(vr, size, alignment); + + return NULL; +} +#include + +static inline void z_vrfy_vregion_free(struct vregion *vr, void *ptr) +{ + if (vregion_verify(vr)) + z_impl_vregion_free(vr, ptr); +} +#include + +struct vregion *z_vrfy_vregion_get(struct vregion *vr) +{ + if (vregion_verify(vr)) + return z_impl_vregion_get(vr); + return NULL; +} +#include + +struct vregion *z_vrfy_vregion_put(struct vregion *vr) +{ + if (vregion_verify(vr)) + return z_impl_vregion_put(vr); + return NULL; +} +#include + +struct vregion *z_vrfy_vregion_create_map(uintptr_t *vreg_start, size_t *vreg_size) +{ + K_OOPS(K_SYSCALL_MEMORY_WRITE(vreg_start, sizeof(*vreg_start))); + K_OOPS(K_SYSCALL_MEMORY_WRITE(vreg_size, sizeof(*vreg_size))); + return z_impl_vregion_create_map(vreg_start, vreg_size); +} +#include + +void z_vrfy_vregion_set_interim(struct vregion *vr) +{ + if (vregion_verify(vr)) + z_impl_vregion_set_interim(vr); +} +#include