diff --git a/Kconfig b/Kconfig index a192aa93ab09..1c9849d9903e 100644 --- a/Kconfig +++ b/Kconfig @@ -1,7 +1,112 @@ # SPDX-License-Identifier: BSD-3-Clause -mainmenu "SOF $(PROJECTVERSION) Configuration" +config COMP_FFMPEG_DEC + tristate "FFmpeg (libavcodec) audio decoder" + select COMP_FFMPEG_DEC_STUB if COMP_STUBS + help + Select to include the FFmpeg audio decoder module. It wraps + libavcodec decoders behind the SOF module interface, decoding a + compressed elementary stream to PCM. The real backend cross-builds + libavcodec/libavutil/libswresample from the FFmpeg source pulled in + by west (see west.yml), enabling only the decoders selected below. + Without the source (or for CI) select COMP_FFMPEG_DEC_STUB to build + the dependency-free passthrough stub. -comment "Compiler: $(CC_VERSION_TEXT)" +config COMP_FFMPEG_DEC_STUB + bool "FFmpeg decoder stub backend" + depends on COMP_FFMPEG_DEC + help + Build the ffmpeg_dec module against a dependency-free passthrough + backend instead of libavcodec. Intended for testing and CI so the + SOF glue and LLEXT packaging can be validated without the FFmpeg + source or a cross-built archive. -source "Kconfig.sof" +if COMP_FFMPEG_DEC && !COMP_FFMPEG_DEC_STUB + +comment "FFmpeg decoders to build (footprint scales with selection)" + +config FFMPEG_DEC_FLAC + bool "FLAC decoder" + default y + help + Lossless integer decoder. Smallest footprint, no floating-point + math. Enables --enable-decoder=flac in the libavcodec build. + +config FFMPEG_DEC_AAC + bool "AAC-LC decoder" + help + AAC Low Complexity decoder. Pulls in the single-precision float + math layer (see FFMPEG_DEC_FLOAT_MATH). Enables + --enable-decoder=aac in the libavcodec build. + +config FFMPEG_DEC_OPUS + bool "Opus decoder" + help + Opus (SILK+CELT) decoder. Pulls in the single-precision float math + layer. Enables --enable-decoder=opus in the libavcodec build. + +config FFMPEG_DEC_MP3 + bool "MP3 decoder" + help + MPEG-1/2 Layer III decoder. Enables --enable-decoder=mp3 and the + mpegaudio parser in the libavcodec build. Uses the float math layer. + +config FFMPEG_ENC_MP3 + bool "MP3 encoder (libshine, fixed-point)" + help + MPEG-1 Layer III encoder via libshine, a small fixed-point encoder + (west-pinned; FFmpeg has no native MP3 encoder). Cross-builds libshine + and enables --enable-libshine --enable-encoder=libshine. Fixed-point, so + no float math dependency. The encoder is built into the archive; a module + encode path (PCM->MP3) is a separate build mode. + +comment "FFmpeg audio filters (libavfilter; need the avfilter-graph backend to run)" + +config FFMPEG_FILTER_AFFTDN + bool "FFT noise reduction (afftdn)" + help + Build FFmpeg's FFT-based denoiser (afftdn) into libavfilter. Pure DSP, + no external model file (unlike arnndn). Enabling any filter turns on + libavfilter in the cross-build. NOTE: a filter is only usable at runtime + once the module gains an avfilter-graph backend; selecting it here just + builds the filter into the archive. + +config FFMPEG_DEC_FILTER_MODE + bool "Build as a PCM filter effect instead of a decoder [EXPERIMENTAL]" + select FFMPEG_FILTER_AFFTDN + help + Build the module as a PCM source->sink effect that runs an FFmpeg + audio filter graph (default afftdn noise reduction) via the modern + .process interface, instead of the compressed-stream decoder. Turns on + libavfilter and the fast float math. Structurally complete and + load-ready; real-time latency/format tuning needs on-hardware bring-up. + +# Turns on libavfilter in the FFmpeg cross-build when any filter is selected. +config FFMPEG_BUILD_AVFILTER + bool + default y if FFMPEG_FILTER_AFFTDN + +# Selected automatically when a decoder or filter that needs IEEE float math is +# built. Gates compilation of the fast single-precision libm (fastmathf.c). +config FFMPEG_DEC_FLOAT_MATH + bool + default y if FFMPEG_DEC_AAC || FFMPEG_DEC_OPUS || FFMPEG_DEC_MP3 || FFMPEG_FILTER_AFFTDN + +config FFMPEG_DEC_COLD_SPLIT + bool "Place cold FFmpeg code (av_cold init/setup) in DRAM [EXPERIMENTAL]" + depends on COLD_STORE_EXECUTE_DRAM + default n + help + EXPERIMENTAL. Build libavcodec with -mtext-section-literals and rename + FFmpeg's cold functions (its av_cold init/table-generation code, which GCC + emits into .text.unlikely) into SOF's .cold section so the LLEXT loader + places them in DRAM, keeping the hot per-frame decode in fast SRAM. + + Measured cold code is small (~1 KB for FLAC, ~26 KB for AAC) because + FFmpeg only marks init functions av_cold. Placing that much cold code in a + -shared LLEXT currently hits Xtensa linker limits (cold->hot call8 range + needs -mlongcalls; the orphan .cold lands at a bad LMA and overlaps .hash), + so this is off by default pending proper cold-region placement in the LLEXT + memory map. See git history / TESTING notes. + +endif # COMP_FFMPEG_DEC && !COMP_FFMPEG_DEC_STUB diff --git a/ffmpeg.cmake b/ffmpeg.cmake new file mode 100644 index 000000000000..b705e02adf41 --- /dev/null +++ b/ffmpeg.cmake @@ -0,0 +1,199 @@ +# SPDX-License-Identifier: BSD-3-Clause +# +# Cross-build decoder-only FFmpeg static libraries for the ffmpeg_dec module. +# +# Invokes FFmpeg's autotools build as an ExternalProject, cross-compiled with the +# Zephyr target toolchain, enabling only the decoders selected in Kconfig. The +# source comes from the west-pinned FFmpeg project (see west.yml). Produces +# libavcodec/libavutil/libswresample .a under ${FFMPEG_INSTALL_DIR} for the LLEXT +# to link, and defines the 'ffmpeg_ext' target it must depend on. + +include(ExternalProject) + +# --- 1. Locate the FFmpeg source (west module), overridable for local trees --- +if(NOT DEFINED SOF_FFMPEG_SRC_DIR) + set(SOF_FFMPEG_SRC_DIR "${sof_top_dir}/../modules/audio/ffmpeg" + CACHE PATH "FFmpeg source tree (west-pinned)") +endif() +cmake_path(NORMAL_PATH SOF_FFMPEG_SRC_DIR) +if(NOT EXISTS "${SOF_FFMPEG_SRC_DIR}/configure") + message(FATAL_ERROR + "ffmpeg_dec: FFmpeg source not found at '${SOF_FFMPEG_SRC_DIR}'.\n" + "Run 'west update' (FFmpeg is pinned in west.yml) or pass " + "-DSOF_FFMPEG_SRC_DIR=.") +endif() + +# --- 2. Kconfig -> --enable-decoder / --enable-parser lists --- +set(_ff_decoders "") +set(_ff_parsers "") +if(CONFIG_FFMPEG_DEC_FLAC) + list(APPEND _ff_decoders flac) + list(APPEND _ff_parsers flac) +endif() +if(CONFIG_FFMPEG_DEC_AAC) + list(APPEND _ff_decoders aac aac_latm) + list(APPEND _ff_parsers aac aac_latm) +endif() +if(CONFIG_FFMPEG_DEC_OPUS) + list(APPEND _ff_decoders opus) + list(APPEND _ff_parsers opus) +endif() +if(CONFIG_FFMPEG_DEC_MP3) + list(APPEND _ff_decoders mp3) + list(APPEND _ff_parsers mpegaudio) +endif() +if(NOT _ff_decoders) + message(FATAL_ERROR "ffmpeg_dec: no decoder selected (Kconfig FFMPEG_DEC_*)") +endif() +list(JOIN _ff_decoders "," _ff_dec_csv) +list(JOIN _ff_parsers "," _ff_par_csv) + +# --- 2b. Kconfig -> libavfilter audio filters (off unless a filter is chosen) --- +set(_ff_avfilter_cfg --disable-avfilter) +if(CONFIG_FFMPEG_BUILD_AVFILTER) + set(_ff_filters "") + if(CONFIG_FFMPEG_FILTER_AFFTDN) + list(APPEND _ff_filters afftdn) + endif() + list(JOIN _ff_filters "," _ff_filt_csv) + # abuffer/abuffersink feed/drain a filter graph; aformat negotiates format. + set(_ff_avfilter_cfg + --enable-avfilter + --enable-filter=abuffer,abuffersink,aformat,${_ff_filt_csv}) + message(STATUS "ffmpeg_dec: enabling avfilter, filters [${_ff_filt_csv}]") +endif() + +# --- 3. Derive the cross toolchain prefix from the Zephyr target compiler --- +# e.g. .../bin/xtensa-intel_ace30_ptl_zephyr-elf-gcc +# -> prefix .../bin/xtensa-intel_ace30_ptl_zephyr-elf- +get_filename_component(_tc_dir "${CMAKE_C_COMPILER}" DIRECTORY) +get_filename_component(_tc_name "${CMAKE_C_COMPILER}" NAME) +string(REGEX REPLACE "gcc$" "" _tc_prefix_name "${_tc_name}") +set(_ff_cross_prefix "${_tc_dir}/${_tc_prefix_name}") + +# GCC 14 (Zephyr SDK) promotes these to errors; FFmpeg 7.x trips them. +set(_ff_extra_cflags "-fPIC -Wno-error=incompatible-pointer-types -Wno-error=implicit-function-declaration") + +set(FFMPEG_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/ffmpeg-install" CACHE INTERNAL "ffmpeg_dec libs") + +# Hot/cold split: GCC emits av_cold (init/setup) functions into .text.unlikely. +# We rename that to SOF's .cold (post-install, below) so the LLEXT loader places +# it in DRAM. -mtext-section-literals interleaves each function's Xtensa L32R +# literals into its text section (as SOF does for LLEXT module code) so the +# renamed section carries its literals and L32R stays in range. We deliberately +# do NOT use -ffunction-sections: keeping one .text.unlikely per object lets a +# single objcopy --rename-section catch it, and lets .cold be placed by SOF's +# existing orphan .cold handling (a hand-written linker script that forces .cold +# ahead of .text overruns the SRAM budget and overlaps .rodata for big codecs). +if(CONFIG_FFMPEG_DEC_COLD_SPLIT) + # -mlongcalls: cold code lives in DRAM (.cold, relocated by the loader) far + # from the hot SRAM .text, so calls between them exceed call8 range; longcalls + # emit an indirect L32R+CALLX with unlimited range. -mtext-section-literals + # keeps each function's literals in its own text section so the rename carries + # them. Coarse split: rename the whole FFmpeg .text (not just av_cold) into + # .cold so essentially all of libav* executes from DRAM, freeing SRAM. + set(_ff_extra_cflags "${_ff_extra_cflags} -mtext-section-literals -mlongcalls") + set(_ff_objcopy "${_ff_cross_prefix}objcopy") + set(_ff_cold_rename + COMMAND ${_ff_objcopy} --rename-section .text=.cold + ${FFMPEG_INSTALL_DIR}/lib/libavcodec.a + COMMAND ${_ff_objcopy} --rename-section .text=.cold + ${FFMPEG_INSTALL_DIR}/lib/libavutil.a + COMMAND ${_ff_objcopy} --rename-section .text=.cold + ${FFMPEG_INSTALL_DIR}/lib/libswresample.a) +endif() + +# --- 3b. MP3 encoder via libshine (fixed-point; FFmpeg has no native one) --- +set(_ff_cfg_env "PATH=${_tc_dir}:$ENV{PATH}") +set(_ff_shine_cfg "") +set(_ff_shine_dep "") +if(CONFIG_FFMPEG_ENC_MP3) + set(SHINE_SRC "${sof_top_dir}/../modules/audio/shine") + if(NOT EXISTS "${SHINE_SRC}/src/lib/layer3.h") + message(FATAL_ERROR "ffmpeg_dec: libshine source not at '${SHINE_SRC}'; run 'west update'") + endif() + set(SHINE_INSTALL "${CMAKE_CURRENT_BINARY_DIR}/shine-install") + file(MAKE_DIRECTORY "${SHINE_INSTALL}/lib/pkgconfig" "${SHINE_INSTALL}/include/shine") + # pkg-config file for FFmpeg's require_pkg_config libshine. + file(WRITE "${SHINE_INSTALL}/lib/pkgconfig/shine.pc" +"prefix=${SHINE_INSTALL} +libdir=\${prefix}/lib +includedir=\${prefix}/include +Name: shine +Description: Shine fixed-point MP3 encoder +Version: 3.1.1 +Libs: -L\${libdir} -lshine +Cflags: -I\${includedir} +") + # FFmpeg's -lshine link *test* pulls newlib malloc -> Zephyr runtime syms + # (z_errno_wrap, ...) that only exist at module load. Provide dummies so the + # configure test links; --extra-ldflags only affects tests, not the .a build. + file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/shine_cfgstub.c" +"int z_errno_wrap(void){return 0;} +void *stdout; +int open(const char *p, int f, ...){return -1;} +void *sbrk(int i){return (void *)-1;} +") + # libshine's lib needs no config.h; build the objects directly and archive + # (its autotools CLI/shared link cannot work bare-metal). + file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/shine-build.sh" +"#!/bin/sh +set -e +export PATH=${_tc_dir}:\$PATH +mkdir -p ${CMAKE_CURRENT_BINARY_DIR}/shine-obj +for f in ${SHINE_SRC}/src/lib/*.c; do + ${CMAKE_C_COMPILER} -O2 -fPIC -mtext-section-literals -DSHINE_HAVE_BSWAP_H \\ + -I${SHINE_SRC}/src/lib -c \"\$f\" \\ + -o ${CMAKE_CURRENT_BINARY_DIR}/shine-obj/\$(basename \"\$f\").o +done +${_ff_cross_prefix}ar rcs ${SHINE_INSTALL}/lib/libshine.a ${CMAKE_CURRENT_BINARY_DIR}/shine-obj/*.o +cp ${SHINE_SRC}/src/lib/layer3.h ${SHINE_INSTALL}/include/shine/layer3.h +${CMAKE_C_COMPILER} -fPIC -c ${CMAKE_CURRENT_BINARY_DIR}/shine_cfgstub.c \\ + -o ${CMAKE_CURRENT_BINARY_DIR}/shine_cfgstub.o +") + add_custom_command( + OUTPUT "${SHINE_INSTALL}/lib/libshine.a" "${CMAKE_CURRENT_BINARY_DIR}/shine_cfgstub.o" + COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/shine-build.sh" + VERBATIM) + add_custom_target(shine_ext DEPENDS "${SHINE_INSTALL}/lib/libshine.a") + + set(_ff_cfg_env "PATH=${_tc_dir}:$ENV{PATH}" "PKG_CONFIG_PATH=${SHINE_INSTALL}/lib/pkgconfig") + set(_ff_shine_cfg --enable-libshine --enable-encoder=libshine --pkg-config=pkg-config + "--extra-ldflags=${CMAKE_CURRENT_BINARY_DIR}/shine_cfgstub.o") + set(_ff_shine_dep shine_ext) + message(STATUS "ffmpeg_dec: enabling MP3 encoder via libshine (${SHINE_SRC})") +endif() + +# --- 4. Configure + make + install (out-of-tree; source must be clean) --- +ExternalProject_Add(ffmpeg_ext + DEPENDS ${_ff_shine_dep} + SOURCE_DIR "${SOF_FFMPEG_SRC_DIR}" + BUILD_IN_SOURCE 0 + CONFIGURE_COMMAND + ${CMAKE_COMMAND} -E env ${_ff_cfg_env} + /configure + --prefix=${FFMPEG_INSTALL_DIR} + --enable-cross-compile --target-os=none --arch=xtensa + --cross-prefix=${_ff_cross_prefix} --cc=${CMAKE_C_COMPILER} + --disable-asm --disable-all --disable-everything --disable-autodetect + --disable-programs --disable-doc --disable-network + --disable-avformat --disable-avdevice ${_ff_avfilter_cfg} + --disable-swscale --disable-postproc + --disable-pthreads --disable-w32threads --disable-os2threads + --disable-runtime-cpudetect --disable-debug + --enable-avcodec --enable-avutil --enable-swresample + --enable-decoder=${_ff_dec_csv} --enable-parser=${_ff_par_csv} + ${_ff_shine_cfg} + --enable-small --enable-pic + "--extra-cflags=${_ff_extra_cflags}" + BUILD_COMMAND + ${CMAKE_COMMAND} -E env "PATH=${_tc_dir}:$ENV{PATH}" make -j 8 + INSTALL_COMMAND make install + ${_ff_cold_rename} + BUILD_BYPRODUCTS + ${FFMPEG_INSTALL_DIR}/lib/libavcodec.a + ${FFMPEG_INSTALL_DIR}/lib/libavutil.a + ${FFMPEG_INSTALL_DIR}/lib/libswresample.a +) + +message(STATUS "ffmpeg_dec: cross-building FFmpeg decoders [${_ff_dec_csv}] from ${SOF_FFMPEG_SRC_DIR}") diff --git a/src/audio/CMakeLists.txt b/src/audio/CMakeLists.txt index 92002c8b7c1c..fdfd460ca6aa 100644 --- a/src/audio/CMakeLists.txt +++ b/src/audio/CMakeLists.txt @@ -41,6 +41,9 @@ if(NOT CONFIG_COMP_MODULE_SHARED_LIBRARY_BUILD) if(CONFIG_COMP_DRC) add_subdirectory(drc) endif() + if(CONFIG_COMP_FFMPEG_DEC) + add_subdirectory(ffmpeg_dec) + endif() if(CONFIG_COMP_FIR) add_subdirectory(eq_fir) endif() @@ -107,6 +110,18 @@ if(NOT CONFIG_COMP_MODULE_SHARED_LIBRARY_BUILD) if(CONFIG_COMP_VOLUME) add_subdirectory(volume) endif() + if(CONFIG_COMP_WEBRTC_VAD) + add_subdirectory(webrtc_vad) + endif() + if(CONFIG_COMP_WEBRTC_NS) + add_subdirectory(webrtc_ns) + endif() + if(CONFIG_COMP_WEBRTC_AEC) + add_subdirectory(webrtc_aec) + endif() + if(CONFIG_COMP_WEBRTC_NS2) + add_subdirectory(webrtc_ns2) + endif() if(CONFIG_DTS_CODEC) add_subdirectory(codec) endif() diff --git a/src/audio/Kconfig b/src/audio/Kconfig index 8accb25738a2..511568858384 100644 --- a/src/audio/Kconfig +++ b/src/audio/Kconfig @@ -139,6 +139,7 @@ rsource "dcblock/Kconfig" rsource "drc/Kconfig" rsource "eq_fir/Kconfig" rsource "eq_iir/Kconfig" +rsource "ffmpeg_dec/Kconfig" rsource "google/Kconfig" rsource "igo_nr/Kconfig" rsource "mfcc/Kconfig" @@ -161,6 +162,10 @@ rsource "tensorflow/Kconfig" rsource "tone/Kconfig" rsource "up_down_mixer/Kconfig" rsource "volume/Kconfig" +rsource "webrtc_vad/Kconfig" +rsource "webrtc_ns/Kconfig" +rsource "webrtc_aec/Kconfig" +rsource "webrtc_ns2/Kconfig" # --- End Kconfig Sources (alphabetical order) --- rsource "level_multiplier/Kconfig" diff --git a/src/audio/ffmpeg_dec/CMakeLists.txt b/src/audio/ffmpeg_dec/CMakeLists.txt new file mode 100644 index 000000000000..a830a1e5fead --- /dev/null +++ b/src/audio/ffmpeg_dec/CMakeLists.txt @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: BSD-3-Clause + +if(CONFIG_COMP_FFMPEG_DEC STREQUAL "m" AND DEFINED CONFIG_LLEXT) + add_subdirectory(llext ${PROJECT_BINARY_DIR}/ffmpeg_dec_llext) + add_dependencies(app ffmpeg_dec) +else() + add_local_sources(sof ffmpeg_dec.c) + + if(CONFIG_COMP_FFMPEG_DEC_STUB) + add_local_sources(sof ffmpeg_dec-stub.c) + else() + add_local_sources(sof ffmpeg_dec-ffmpeg.c) + # NOTE: ffmpeg_dec-shims.c / ffmpeg_dec-alloc.c are LLEXT-only (see + # llext/CMakeLists.txt). They provide libc/libm symbols the isolated LLEXT + # cannot import; in a built-in (=y) or native/testbench build the real libc + # provides those, so including them here would multiply-define malloc etc. + endif() +endif() diff --git a/src/audio/ffmpeg_dec/HIFI.md b/src/audio/ffmpeg_dec/HIFI.md new file mode 100644 index 000000000000..8e2a19becf53 --- /dev/null +++ b/src/audio/ffmpeg_dec/HIFI.md @@ -0,0 +1,225 @@ +# HiFi intrinsics opportunities in ffmpeg_dec + +Analysis of the hot audio-processing paths in the FFmpeg SOF module (decode / +filter / encode) and where Xtensa **HiFi** SIMD intrinsics would give the biggest +wins. It covers both the code this module owns and the FFmpeg code it links. + +## Background: why this matters on Xtensa + +- The FFmpeg archive is cross-built with **`--disable-asm`** (see `ffmpeg.cmake`), + because FFmpeg has no Xtensa assembly. Every DSP kernel therefore runs as + **scalar C** on the DSP — no SIMD, no fused multiply-add vectorisation. +- FFmpeg dispatches its DSP through function-pointer contexts with per-architecture + init hooks (`ff_*_init_aarch64/arm/x86/riscv`) — but **no `_xtensa`** variant. The + generic `_c` kernels are what execute on our target. +- SOF already ships HiFi-optimised DSP and the toolchain for it. Use it as the + template: + - ISA select + intrinsics header (`src/math/exp_fcn_hifi.c`): + ```c + #if XCHAL_HAVE_HIFI5 + #include + #elif XCHAL_HAVE_HIFI4 + #include + #else + #include + #endif + ``` + - Existing HiFi kernels to mirror: `src/math/fir_hifi{2ep,3,5}.c`, + `src/math/iir_df1_hifi{3,4,5}.c`, `src/math/fft/fft_{16,32}_hifi3.c`, + `src/math/exp_fcn_hifi.c`. `ace30` (ptl) is HiFi4-class. + +There are two distinct bodies of code: + +- **(A) Module glue we own** — the PCM format-conversion loops in + `ffmpeg_dec-*.c`. Small, per-sample, easy to hand-vectorise with HiFi, and we + can change them freely. +- **(B) FFmpeg DSP** — the actual codec compute (MDCT/FFT, LPC, windowing). Much + higher cycle cost, but upstream; adding HiFi means either FFmpeg fork patches + (an `_xtensa` DSP init, mirroring the other arch dirs) or routing to SOF's own + HiFi kernels. + +## Hot-path summary + +| # | Path | File | Kind | Now | HiFi win | Effort | +|---|------|------|------|-----|----------|--------| +| 1 | MDCT/FFT (AAC/MP3/Opus/afftdn) | FFmpeg `libavutil/tx*` | float/fixed transform | scalar C | **very high** | high | +| 2 | float vector kernels (fmul/window/overlap-add) | FFmpeg `libavutil/float_dsp` | float SIMD | scalar C | **high** | low–med | +| 3 | FLAC LPC / residual | FFmpeg `libavcodec/flacdsp` | int32/64 MAC | scalar C | high | med | +| 4 | MP3 synthesis window / imdct36 | FFmpeg `libavcodec/mpegaudiodsp` | float/fixed | scalar C | high | med | +| 5 | AAC PS / SBR DSP | FFmpeg `aacpsdsp`/`sbrdsp` | float SIMD | scalar C | med (HE-AAC only) | med | +| 6 | S32↔float PCM convert (filter) | `ffmpeg_dec-filter.c:230,255` | int↔float + scale | scalar C | med | **low** | +| 7 | planar→interleaved PCM (decode) | `ffmpeg_dec-ffmpeg.c:231` | copy/interleave | scalar C | low–med | **low** | +| 8 | S32→S16 pack (encode) | `ffmpeg_dec-encode.c:147` | narrow + interleave | scalar C | med | **low** | +| 9 | fast float libm (powf/exp2f/...) | `fastmathf.c` | scalar poly/Newton | scalar C | med (per-band) | med | + +Priority order (cost/benefit): **2 → 6/7/8 → 3 → 1 → 4 → 9 → 5**. Start with the +easy, high-use float vector kernels (2) and the conversion loops we own (6–8); +tackle the transform (1) last because it is the most work despite the highest +raw cost. + +## (A) Module conversion loops — we own these, do them first + +These run **per sample over a whole frame** (typically 1024–4608 samples × +channels per call), so they are genuinely hot and are trivial, self-contained HiFi +wins. Add HiFi versions guarded by `#if XCHAL_HAVE_HIFI*` with the current scalar +loop as the `#else` fallback. + +1. **Filter S32→float deinterleave + normalise** — `ffmpeg_dec-filter.c:230` + ```c + ((float *)in->data[c])[i] = (float)src[i * ch + c] / FFMPEG_AF_S32_SCALE; + ``` + HiFi: load 32-bit lanes, convert int32→float (`AE_FLOAT32`-family), multiply by + the reciprocal scale, store to the per-channel plane. Deinterleave with + strided/`AE_SEL` moves. + +2. **Filter float→S32 interleave + denormalise + clip** — `ffmpeg_dec-filter.c:255` + The scalar path multiplies, branches to clamp, then casts. HiFi does the + float→int32 convert **with saturation** in one op (no per-sample branch), which + is both faster and removes the mispredicted clamp branch. + +3. **Encode S32→S16 narrow + deinterleave** — `ffmpeg_dec-encode.c:147` + ```c + int16_t s = (int16_t)(in[i * ch + c] >> 16); + ``` + HiFi: pack 32→16 with rounding (`AE_ROUND16X4F32`-style) and store; SIMD handles + 4–8 samples per iteration. + +4. **Decode planar→interleaved** — `ffmpeg_dec-ffmpeg.c:231` + Per-element `memcpy`-equivalent; a HiFi interleave (vector load per plane, + `AE_SEL` interleave, vector store) removes the per-sample loop overhead. Lower + priority as many decoders already output the sink format. + +A single small `ffmpeg_dec-convert.c` with HiFi + scalar variants (like SOF's +`*_generic.c` / `*_hifi3.c` split) would hold all four and be reusable across the +three modes. + +## (B) FFmpeg DSP — the real compute + +FFmpeg's DSP contexts are function-pointer tables initialised per arch. The +Xtensa-idiomatic fix is a **fork patch** adding `ff__init_xtensa()` with HiFi +intrinsics, mirroring `libavcodec/aarch64/`, `x86/` etc. (our FFmpeg is already a +SOF fork — see `west.yml` — so such patches have a home). + +- **`AVFloatDSPContext`** (`libavutil/float_dsp.h`) — `vector_fmul`, + `vector_fmul_window`, `vector_fmul_add`, `vector_fmul_reverse`, + `scalarproduct_float`. Used for windowing / overlap-add by **AAC, MP3 and Opus**. + These are plain multiply/MAC over contiguous float arrays — the easiest and + highest-leverage HiFi target (one small `float_dsp_init_xtensa` benefits three + codecs). **Do this first on the FFmpeg side.** +- **`FLACDSPContext`** (`libavcodec/flacdsp.h`) — `lpc16`/`lpc32`/`lpc33` and + decorrelate. FLAC decode is dominated by the LPC prediction MAC loop over the + residual: a natural fit for HiFi multiply-accumulate. `ff_flacdsp_init_xtensa`. +- **`MPADSPContext`** (`libavcodec/mpegaudiodsp.h`) — `apply_window_{float,fixed}`, + `imdct36_blocks_*`, `synth_filter`. The MP3 subband synthesis window is the MP3 + decode hot loop. +- **`libavutil/tx`** (MDCT/FFT) — the single largest cost for AAC/MP3/Opus decode + and for the `afftdn` filter. It is a "codelet" system rather than one function + pointer, so it is the most work. Two routes: + 1. Add HiFi FFT/MDCT codelets to the fork (largest effort, cleanest for FFmpeg). + 2. Bridge to **SOF's existing HiFi3 FFT** (`src/math/fft/fft_*_hifi3.c`) by + replacing FFmpeg's transform behind `av_tx_init` for the sizes SOF supports. + Less code, but a semantic bridge (twiddle/scaling/layout must match) and only + covers power-of-two sizes. +- **`PSDSPContext` / SBR DSP** — only relevant if HE-AAC (SBR/PS) is enabled; skip + unless needed. + +## (C) fastmathf — the module's float libm + +`fastmathf.c` (`powf`/`exp2f`/`log2f`/`sinf`/`cosf`/`sqrtf`/`cbrtf`) is scalar +polynomial/Newton code. These are called per-band / per-frame (not the innermost +per-sample loop), so they are a secondary target. HiFi wins come from: +- `sqrtf`/`cbrtf`: HiFi `RSQRT`/reciprocal seed instructions instead of the + bit-trick + Newton. +- `exp2f`/`log2f`/`sinf`/`cosf`: process 2/4 lanes at once when a decoder needs a + vector of them (e.g. AAC scalefactor gains), and use FMA for the polynomials. + Mirror `src/math/exp_fcn_hifi.c`. + +## Appendix: AAC-LC decode hot spots (per-frame detail) + +Where the cycles actually go decoding one AAC-LC frame (1024 samples long, or +8×128 short) per channel, mapped to the code in the tree. Everything below runs as +**scalar C** on Xtensa today (`--disable-asm`, no `_xtensa` DSP init). + +### Toolchain requirement (applies to all FFmpeg-side HiFi work) + +The Cadence HiFi intrinsic headers (``) and a +`core-isa.h` with `XCHAL_HAVE_HIFI4 == 1` are **only** available through the +**LLVM/xt-clang Xtensa** toolchain (on this build host under +`llvm-project/.../clang/lib/Headers/xtensa/tie/` plus the ace30 core config at +`modules/hal/xtensa/.../soc/intel_ace30_ptl/xtensa/config/core-isa.h`). The +Zephyr-SDK GCC that `ffmpeg.cmake` currently drives does **not** ship these headers. + +Consequence: the `ff_*_init_xtensa` kernels must be written with `#if +XCHAL_HAVE_HIFI*` intrinsics **and** a scalar `#else` fallback. Under the current +GCC cross-build the guard is 0, so the fallback compiles (correct, but no speedup). +Actual HiFi codegen requires cross-building FFmpeg with the LLVM Xtensa clang and +adding the ace30 core-config include dir to the FFmpeg `CFLAGS` — the same +toolchain SOF's own `src/math/*_hifi*.c` already rely on. + +### Tier 1 — dominant, best return + +1. **IMDCT** — `mdct1024_fn` / `mdct128_fn` (short blocks) via `libavutil/tx`, + driven from `imdct_and_windowing()` (`libavcodec/aac/aacdec_dsp_template.c:341-343`). + One 1024-pt (or 8×128) inverse MDCT per channel per frame — the single largest + compute. Dispatch: `AVTXContext`. HiFi = complex-FMA butterflies, via HiFi `tx` + codelets or a bridge to SOF's HiFi3 FFT (`src/math/fft/`). Highest cost, most work. + +2. **`AVFloatDSPContext` vector kernels** — pervasive for windowing / overlap-add / + stereo, and the easiest high-leverage target: + - `vector_fmul_window` — windowed overlap-add, ~10 call sites (`:354-419`); *the* + per-sample AAC output kernel. + - `vector_fmul` / `vector_fmul_reverse` — window application (`:235-308`). + - `vector_fmul_scalar` — intensity-stereo gain (`:146`). + - `butterflies_float` — M/S stereo (`:102`). + One `ff_float_dsp_init_xtensa` (in `libavutil/`) covers all of them and also + accelerates MP3 and Opus. **Start here** (lowest effort, cross-codec). + +### Tier 2 — AAC-specific loops (in `aacdec_dsp_template.c`, HiFi'd inline in the fork) + +3. **Inverse quantization `x^(4/3)`** — `decode_spectrum_and_dequant()` + (`libavcodec/aac/aacdec.c:1768`) + `dequant_scalefactors()` (`:41`): up to 1024 + coeffs, each `ff_cbrt_tab` lookup for `|x|^(4/3)` × scalefactor gain + `2^(0.25·sf)`. HiFi = vectorised table gather + FMA (per-SFB gain is a + `vector_fmul_scalar`). +4. **TNS** — `apply_tns()` (`:164`): LPC all-pole filter (order ≤ ~20) over spectral + coeffs per window. HiFi = MAC filter (mirror `src/math/iir_df1_hifi*`). + +### Tier 3 — HE-AAC only (skip unless SBR/PS enabled) + +5. **SBR QMF** — `libavcodec/sbrdsp.c`: `sbr_qmf_pre/post_shuffle`, `sbr_hf_gen`, + `sbr_hf_g_filt`, `sbr_sum64x5` + a 64-pt QMF transform. Dispatch: + `AACSBRDSPContext` (`sbrdsp.h`, `ff_sbrdsp_init`). Nearly as heavy as the base IMDCT. +6. **PS** — `libavcodec/aacps.c` `hybrid_analysis` / `stereo_interpolate` via + `PSDSPContext` (`aacpsdsp.h`, `ff_psdsp_init`). + +### Where the HiFi hooks land + priority + +| Priority | Target | Dispatch / location | Effort | +|---|--------|---------------------|--------| +| 1 | `vector_fmul_window` + friends | `AVFloatDSPContext` → `ff_float_dsp_init_xtensa` (libavutil) | low; AAC+MP3+Opus | +| 2 | IMDCT (mdct128/1024) | `libavutil/tx` — HiFi codelets or SOF-FFT bridge | high | +| 3 | dequant `x^(4/3)` | `aac/aacdec.c` + dsp_template (cbrt gather+FMA) | med | +| 4 | TNS | `apply_tns` (LPC MAC) | med | +| 5 | SBR / PS | `sbrdsp`/`aacpsdsp` `_xtensa` inits | med, HE-AAC only | + +AAC-LC order: **float_dsp (2) → IMDCT (1) → dequant (3) → TNS (4)**; SBR/PS only if +HE-AAC. `float_dsp`/`tx`/`sbrdsp`/`psdsp` are `libavutil`/`libavcodec` and take +`_xtensa` inits mirroring the existing `aarch64`/`x86` dirs; dequant/TNS are AAC +decoder code patched in place. All on the `lgirdwood/FFmpeg` `sof-hifi` branch. + +## Recommendations + +1. **Quick wins we own** — hand-vectorise the four conversion loops (§A) with + `#if XCHAL_HAVE_HIFI*` + scalar fallback. Small, self-contained, no fork needed. +2. **Biggest FFmpeg leverage-per-effort** — add `ff_float_dsp_init_xtensa` (HiFi + `vector_fmul*`/`scalarproduct`) to the FFmpeg fork; benefits AAC + MP3 + Opus. +3. **Per-codec kernels** — `flacdsp` (FLAC), `mpegaudiodsp` (MP3) HiFi inits as + fork patches, gated on the enabled decoders. +4. **Transform** — evaluate SOF-HiFi-FFT bridge vs. HiFi tx codelets for + `libavutil/tx`; highest payoff, most work, do once the cheaper items land. +5. Always keep the scalar C path as the fallback (`#else`) and behind the same ISA + guards SOF uses, so non-HiFi / host/testbench builds still work. + +Measurement: profile on hardware per codec before and after; the ranking above is +by expected cost, but the actual hot spot depends on the codec and content (FLAC +is LPC-bound, AAC/MP3/Opus are transform+window-bound). diff --git a/src/audio/ffmpeg_dec/Kconfig b/src/audio/ffmpeg_dec/Kconfig new file mode 100644 index 000000000000..944bdc882470 --- /dev/null +++ b/src/audio/ffmpeg_dec/Kconfig @@ -0,0 +1,122 @@ +# SPDX-License-Identifier: BSD-3-Clause + +config COMP_FFMPEG_DEC + tristate "FFmpeg (libavcodec) audio decoder" + select COMP_FFMPEG_DEC_STUB if COMP_STUBS + help + Select to include the FFmpeg audio decoder module. It wraps + libavcodec decoders behind the SOF module interface, decoding a + compressed elementary stream to PCM. The real backend cross-builds + libavcodec/libavutil/libswresample from the FFmpeg source pulled in + by west (see west.yml), enabling only the decoders selected below. + Without the source (or for CI) select COMP_FFMPEG_DEC_STUB to build + the dependency-free passthrough stub. + +config COMP_FFMPEG_DEC_STUB + bool "FFmpeg decoder stub backend" + depends on COMP_FFMPEG_DEC + help + Build the ffmpeg_dec module against a dependency-free passthrough + backend instead of libavcodec. Intended for testing and CI so the + SOF glue and LLEXT packaging can be validated without the FFmpeg + source or a cross-built archive. + +if COMP_FFMPEG_DEC && !COMP_FFMPEG_DEC_STUB + +comment "FFmpeg decoders to build (footprint scales with selection)" + +config FFMPEG_DEC_FLAC + bool "FLAC decoder" + default y + help + Lossless integer decoder. Smallest footprint, no floating-point + math. Enables --enable-decoder=flac in the libavcodec build. + +config FFMPEG_DEC_AAC + bool "AAC-LC decoder" + help + AAC Low Complexity decoder. Pulls in the single-precision float + math layer (see FFMPEG_DEC_FLOAT_MATH). Enables + --enable-decoder=aac in the libavcodec build. + +config FFMPEG_DEC_OPUS + bool "Opus decoder" + help + Opus (SILK+CELT) decoder. Pulls in the single-precision float math + layer. Enables --enable-decoder=opus in the libavcodec build. + +config FFMPEG_DEC_MP3 + bool "MP3 decoder" + help + MPEG-1/2 Layer III decoder. Enables --enable-decoder=mp3 and the + mpegaudio parser in the libavcodec build. Uses the float math layer. + +config FFMPEG_ENC_MP3 + bool "MP3 encoder (libshine, fixed-point)" + help + MPEG-1 Layer III encoder via libshine, a small fixed-point encoder + (west-pinned; FFmpeg has no native MP3 encoder). Cross-builds libshine + and enables --enable-libshine --enable-encoder=libshine. Fixed-point, so + no float math dependency. The encoder is built into the archive; a module + encode path (PCM->MP3) is a separate build mode. + +comment "FFmpeg audio filters (libavfilter; need the avfilter-graph backend to run)" + +config FFMPEG_FILTER_AFFTDN + bool "FFT noise reduction (afftdn)" + help + Build FFmpeg's FFT-based denoiser (afftdn) into libavfilter. Pure DSP, + no external model file (unlike arnndn). Enabling any filter turns on + libavfilter in the cross-build. NOTE: a filter is only usable at runtime + once the module gains an avfilter-graph backend; selecting it here just + builds the filter into the archive. + +config FFMPEG_DEC_ENCODE_MODE + bool "Build as an audio encoder (PCM -> compressed) instead of a decoder [EXPERIMENTAL]" + select FFMPEG_ENC_MP3 + help + Build the module as an encoder: PCM in, compressed elementary stream out + (MP3 via libshine), driven with avcodec_send_frame/avcodec_receive_packet + (the reverse of the decoder). Selects the libshine MP3 encoder. Do not set + together with FFMPEG_DEC_FILTER_MODE. Structurally complete and load-ready; + real-time framing (MP3 fixed 1152-sample frames) needs on-hardware tuning. + +config FFMPEG_DEC_FILTER_MODE + bool "Build as a PCM filter effect instead of a decoder [EXPERIMENTAL]" + select FFMPEG_FILTER_AFFTDN + help + Build the module as a PCM source->sink effect that runs an FFmpeg + audio filter graph (default afftdn noise reduction) via the modern + .process interface, instead of the compressed-stream decoder. Turns on + libavfilter and the fast float math. Structurally complete and + load-ready; real-time latency/format tuning needs on-hardware bring-up. + +# Turns on libavfilter in the FFmpeg cross-build when any filter is selected. +config FFMPEG_BUILD_AVFILTER + bool + default y if FFMPEG_FILTER_AFFTDN + +# Selected automatically when a decoder or filter that needs IEEE float math is +# built. Gates compilation of the fast single-precision libm (fastmathf.c). +config FFMPEG_DEC_FLOAT_MATH + bool + default y if FFMPEG_DEC_AAC || FFMPEG_DEC_OPUS || FFMPEG_DEC_MP3 || FFMPEG_FILTER_AFFTDN + +config FFMPEG_DEC_COLD_SPLIT + bool "Place cold FFmpeg code (av_cold init/setup) in DRAM [EXPERIMENTAL]" + depends on COLD_STORE_EXECUTE_DRAM + default n + help + EXPERIMENTAL. Build libavcodec with -mtext-section-literals and rename + FFmpeg's cold functions (its av_cold init/table-generation code, which GCC + emits into .text.unlikely) into SOF's .cold section so the LLEXT loader + places them in DRAM, keeping the hot per-frame decode in fast SRAM. + + Measured cold code is small (~1 KB for FLAC, ~26 KB for AAC) because + FFmpeg only marks init functions av_cold. Placing that much cold code in a + -shared LLEXT currently hits Xtensa linker limits (cold->hot call8 range + needs -mlongcalls; the orphan .cold lands at a bad LMA and overlaps .hash), + so this is off by default pending proper cold-region placement in the LLEXT + memory map. See git history / TESTING notes. + +endif # COMP_FFMPEG_DEC && !COMP_FFMPEG_DEC_STUB diff --git a/src/audio/ffmpeg_dec/README.md b/src/audio/ffmpeg_dec/README.md new file mode 100644 index 000000000000..326e878b2829 --- /dev/null +++ b/src/audio/ffmpeg_dec/README.md @@ -0,0 +1,100 @@ +# FFmpeg Audio Processing Module (`ffmpeg_dec`) + +Wraps FFmpeg's `libavcodec` and `libavfilter` libraries behind the SOF `module_interface`. This module enables running on-DSP audio decoders, encoders, and real-time audio filter graphs. + +--- + +## Features + +- **Decoder Mode**: Decodes compressed audio streams (e.g., FLAC, MP3) to raw PCM. +- **Encoder Mode**: Encodes raw PCM streams to compressed formats (e.g., MP3 using `libshine`). +- **Filter Mode**: Runs FFmpeg's `libavfilter` graphs directly on raw PCM data (e.g., `afftdn` for noise reduction). +- **Embedded-Optimized**: Supports single-threaded execution, memory shims for dynamic allocation, and LLEXT packaging. +- **CI-Friendly**: Includes a pass-through stub backend to allow pipeline, IPC, and topology verification without external dependencies. + +--- + +## Architecture & Data Flow + +The following Mermaid diagram illustrates the module's internal architecture and the interaction between the SOF pipeline, the `ffmpeg_dec` core, and its backend translation units: + +```mermaid +graph TD + %% Audio Data Flow + InBuf[Input Audio Buffer] -->|Raw Bytes or PCM| Core[ffmpeg_dec.c Core] + Core -->|Pass-through| BackendStub[ffmpeg_dec-stub.c Stub] + Core -->|Process| BackendFFmpeg[ffmpeg_dec-ffmpeg.c FFmpeg Backend] + + BackendFFmpeg -->|Parse/Decode/Filter| Libavcodec[libavcodec / libavfilter] + Libavcodec -->|Output PCM Frame| Core + BackendStub -->|Pass-through S16/S32| Core + Core -->|Interleaved PCM or Encoded Bytes| OutBuf[Output Audio Buffer] + + %% Control Flow + IPC[SOF IPC set_configuration] -.->|Metadata/Extradata| Core + Core -.->|avcodec_open2 / init| Libavcodec +``` + +### Modular Design +- **`ffmpeg_dec.c` (Core Glue)**: Implements the standard `module_interface` functions (`init`, `prepare`, `process`, `reset`, `free`). +- **`ffmpeg_dec-stub.c`**: Pass-through stub backend that bypasses FFmpeg libraries. +- **`ffmpeg_dec-ffmpeg.c` / `ffmpeg_dec-filter.c` / `ffmpeg_dec-encode.c`**: Real backend implementations interfacing with `libavcodec` and `libavfilter`. +- **`ffmpeg_dec-shims.c`**: Overrides dynamic memory allocations (`malloc`, `realloc`, `free`, `calloc`) inside FFmpeg to use SOF's `rballoc` pools. + +--- + +## Build Instructions + +### 1. Stub Mode (Default / CI / Staging) +No external libraries are required. The module acts as a simple pass-through. +```ini +CONFIG_COMP_FFMPEG_DEC=y # or =m for LLEXT +CONFIG_COMP_FFMPEG_DEC_STUB=y +``` + +### 2. Real FFmpeg Backend +Requires pre-compiled static libraries and headers placed under the `third_party/` directory of the workspace. +```ini +CONFIG_COMP_FFMPEG_DEC=m +CONFIG_COMP_FFMPEG_DEC_STUB=n +``` + +#### Configuring the FFmpeg Build +When cross-compiling FFmpeg for Xtensa DSP targets, disable unnecessary components to minimize code size: +```bash +./configure \ + --target-os=none \ + --arch=xtensa \ + --enable-cross-compile \ + --disable-everything \ + --disable-avformat \ + --disable-pthreads \ + --enable-decoder=flac,mp3 \ + --enable-encoder=mp3 \ + --enable-parser=flac,mpegaudio \ + --enable-filter=afftdn \ + --enable-static \ + --disable-shared +``` + +#### Useful Kconfig Options +- `CONFIG_FFMPEG_DEC_FILTER_MODE`: Configures the module to run as a PCM filter graph instead of a decoder. +- `CONFIG_FFMPEG_DEC_FLOAT_MATH`: Automatically selected to pull in optimized floating-point shims (`fastmathf.c`). +- `CONFIG_FFMPEG_DEC_COLD_SPLIT`: Relocates initialization tables and code to DRAM to conserve fast SRAM. + +--- + +## Usage & Topology + +### 1. Decoder Mode +The topology should feed compressed bytes (e.g., from a host gateway) into the decoder input pin. The decoder outputs decoded PCM. +- **Initialization**: Provide codec-specific setup data (e.g., FLAC `STREAMINFO` or MP3 headers) via IPC bytes control `set_configuration`. +- **Buffer Period**: Keep period size large enough (typically 10ms or 20ms) to reduce parsing overhead. + +### 2. Filter Mode +The module is placed in the capture or playback pipeline as a normal 1-in / 1-out effect. +- **Routing Example**: +``` +DAI Copier (Mic Capture) ---> [ ffmpeg_dec (Filter Mode) ] ---> Host Copier (PCM) +``` +- **Control**: Filter coefficients and parameters can be set dynamically via standard TLV bytes controls. diff --git a/src/audio/ffmpeg_dec/TESTING.md b/src/audio/ffmpeg_dec/TESTING.md new file mode 100644 index 000000000000..3d50ed8d84f6 --- /dev/null +++ b/src/audio/ffmpeg_dec/TESTING.md @@ -0,0 +1,66 @@ +# Testing the ffmpeg_dec module + +This describes how to verify the FFmpeg FLAC decoder module end to end. FLAC is +lossless, so a correct decode is **bit-exact** against a reference decode. + +## Test vectors + +Generate vectors with the helper (needs `ffmpeg` on the host): + +``` +tools/test/audio/ffmpeg_dec_prepare.sh -i song.wav -o vectors/ +``` + +Produces in `vectors/`: +- `streaminfo.bin` — 34-byte FLAC STREAMINFO = the codec **extradata** blob to + load into the module's bytes control (`set_configuration`). +- `stream.raw` — the raw FLAC elementary stream fed to the decoder input. +- `ref_s32le.raw` — reference interleaved S32_LE PCM; the decoder output must + match this byte-for-byte. + +## Topology + +The component widget is `tools/topology/topology2/include/components/ffmpeg_dec.conf` +(models the DTS codec: `type "effect"`, one input/one output pin, a `bytes` +control for STREAMINFO). Its `uuid` must match the firmware's registry entry for +`ffmpeg_dec` (see `uuid-registry.txt`). + +A runnable pipeline must deliver **compressed bytes** to the module and take PCM +out. Two paths: + +1. **ALSA compress-offload** (the production path for a decoder): the host writes + the FLAC elementary stream to a compress DMA gateway; the module decodes; PCM + goes to a DAI or host capture. This is the correct model (a decoder is not a + fixed-rate PCM effect) and is the remaining integration to wire on real ptl + hardware. +2. **Direct raw-data injection** (bring-up shortcut): a minimal pipeline where a + host copier delivers `stream.raw` as an opaque byte buffer into the module's + `process_raw_data`, capturing PCM on the other side. + +## Verification paths + +### A. On target (ptl hardware) — full verification +1. Build + flash firmware with the real backend: + `CONFIG_COMP_FFMPEG_DEC=m`, `CONFIG_COMP_FFMPEG_DEC_STUB=n` (requires the + FFmpeg static libs in `third_party/`; see the module README). +2. Load the topology; set the STREAMINFO bytes control: + `amixer -c0 cset name='FFMPGDEC...' -- $(od -An -tx1 vectors/streaminfo.bin)` + (or via `tinymix`). +3. Play `stream.raw` through the pipeline, capture the PCM output. +4. `cmp captured.raw vectors/ref_s32le.raw` — must be identical. + +### B. Native testbench (no hardware) — decode-only check +The module builds native (built-in, real libc — the LLEXT-only shims/alloc are +excluded automatically, see `CMakeLists.txt`). Linking the **host** libavcodec +and driving `process_raw_data` over `stream.raw` with `ref_s32le.raw` as the +oracle gives a bit-exact check without a DSP. This harness is not yet wired (the +testbench file component is PCM-oriented; feeding compressed bytes needs a small +adapter) and is the recommended next step for CI-friendly verification. + +## Status + +- Component widget conf: **done**. +- Host vector/reference tooling: **done** (`ffmpeg_dec_prepare.sh`). +- Runnable pipeline (compress-offload) + on-target bit-exact run: **pending + hardware**. +- Native testbench harness: **pending** (small compressed-input adapter). diff --git a/src/audio/ffmpeg_dec/fastmathf.c b/src/audio/ffmpeg_dec/fastmathf.c new file mode 100644 index 000000000000..02153fd10dc1 --- /dev/null +++ b/src/audio/ffmpeg_dec/fastmathf.c @@ -0,0 +1,257 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. +// +// Fast single-precision float math for FFmpeg lossy decoders (AAC-LC / Opus). +// +// Empirically these decoders need only 7 single-precision functions on their +// decode paths: powf, sqrtf, cbrtf, exp2f, log2f, sinf, cosf. SOF's own math is +// fixed-point Q-format (range-limited) and cannot serve them, so this provides +// IEEE single-precision approximations. 24-bit mantissa precision is ample for +// 32-bit audio samples destined for 24/16-bit rendering, and these are per-frame +// / per-band calls (not the innermost per-sample loop, which is MDCT + table +// lookups), so scalar polynomial/Newton approximations are fast enough. +// +// On Xtensa HiFi, sqrtf/rsqrt and FMA map to hardware intrinsics; the scalar C +// here is the portable reference. Marked spots are where HiFi intrinsics would +// slot in. +// +// Build standalone accuracy/perf test: +// gcc -O2 -DSOFM_FASTMATH_TEST fastmathf.c -lm -o t && ./t + +#include + +/* --- helpers --- */ + +static inline float sofm_flint(int n) +{ + return (float)n; +} + +/* Build 2^n as a float via the exponent field (n in [-126,127]). */ +static inline float sofm_ldexp2(int n) +{ + union { float f; uint32_t i; } u; + + u.i = (uint32_t)(n + 127) << 23; + return u.f; +} + +/* --- sqrtf: bit-trick seed + 2 Newton steps --- */ +float sofm_sqrtf(float x) +{ + union { float f; uint32_t i; } u = { .f = x }; + float y; + + if (x <= 0.0f) + return 0.0f; + + /* HiFi: use FSQRT/RSQRT intrinsic instead of this seed. */ + u.i = 0x1fbd1df5 + (u.i >> 1); + y = u.f; + y = 0.5f * (y + x / y); + y = 0.5f * (y + x / y); + return y; +} + +/* --- cbrtf: bit-trick seed + 2 Newton steps, sign-symmetric --- */ +float sofm_cbrtf(float x) +{ + union { float f; uint32_t i; } u; + int neg = x < 0.0f; + float a = neg ? -x : x; + float y; + + if (x == 0.0f) + return 0.0f; + + u.f = a; + u.i = u.i / 3 + 0x2a514067; + y = u.f; + y = (2.0f * y + a / (y * y)) * (1.0f / 3.0f); + y = (2.0f * y + a / (y * y)) * (1.0f / 3.0f); + return neg ? -y : y; +} + +/* --- exp2f: split x = n + f, 2^x = 2^n * 2^f, poly for 2^f on [0,1) --- */ +float sofm_exp2f(float x) +{ + int n; + float f, p; + + if (x > 127.0f) + return sofm_ldexp2(127) * 2.0f; /* +inf-ish */ + if (x < -126.0f) + return 0.0f; + + n = (int)x; + if (x < 0.0f && (float)n != x) + n--; /* floor */ + f = x - (float)n; + + /* Taylor of 2^f, coeffs (ln2)^k/k!, degree 5 -> ~1e-6 rel error. */ + p = 1.0f + f * (0.6931472f + f * (0.2402265f + f * (0.0555041f + + f * (0.0096181f + f * 0.0013334f)))); + return p * sofm_ldexp2(n); +} + +/* --- log2f: x = 2^e * m, reduce m to [sqrt(2)/2, sqrt(2)), atanh series --- */ +float sofm_log2f(float x) +{ + union { float f; uint32_t i; } u = { .f = x }; + int e; + float m, a, a2, ln_m; + + if (x <= 0.0f) + return -127.0f; + + e = (int)((u.i >> 23) & 0xff) - 127; + u.i = (u.i & 0x007fffff) | 0x3f800000; /* m in [1,2) */ + m = u.f; + if (m > 1.41421356f) { /* center around 1: m in [~.707,~1.414) */ + m *= 0.5f; + e += 1; + } + + /* ln(m) = 2*atanh(a), a = (m-1)/(m+1); exact 0 at m==1. */ + a = (m - 1.0f) / (m + 1.0f); + a2 = a * a; + ln_m = 2.0f * a * (1.0f + a2 * (0.3333333f + a2 * (0.2f + a2 * 0.14285714f))); + return (float)e + ln_m * 1.44269504f; /* / ln(2) */ +} + +/* --- powf: x^y = 2^(y*log2(x)) for x > 0 (AAC uses non-negative magnitudes) --- */ +float sofm_powf(float x, float y) +{ + if (x <= 0.0f) + return 0.0f; + return sofm_exp2f(y * sofm_log2f(x)); +} + +/* --- sinf/cosf: Cody-Waite reduction by pi/2 + quadrant, polys on [-pi/4,pi/4] --- */ +static float sofm_poly_sin(float r) +{ + float r2 = r * r; + + return r * (1.0f + r2 * (-0.16666667f + r2 * (0.00833216f + + r2 * -0.00019515f))); +} + +static float sofm_poly_cos(float r) +{ + float r2 = r * r; + + return 1.0f + r2 * (-0.5f + r2 * (0.04166664f + + r2 * (-0.00138873f + r2 * 2.4433e-5f))); +} + +static float sofm_sincos_core(float x, int cos_phase) +{ + /* pi/2 split so k*PIO2_hi is exact for moderate k (Cody-Waite). */ + const float two_over_pi = 0.636619772f; + const float PIO2_hi = 1.5707855225f; + const float PIO2_lo = 1.08043006e-5f; + int k = (int)(x * two_over_pi + (x < 0.0f ? -0.5f : 0.5f)); + float r = (x - (float)k * PIO2_hi) - (float)k * PIO2_lo; + int q = (k + cos_phase) & 3; + + switch (q) { + case 0: return sofm_poly_sin(r); + case 1: return sofm_poly_cos(r); + case 2: return -sofm_poly_sin(r); + default: return -sofm_poly_cos(r); + } +} + +float sofm_sinf(float x) +{ + return sofm_sincos_core(x, 0); +} + +float sofm_cosf(float x) +{ + return sofm_sincos_core(x, 1); +} + +/* + * libc-name entry points used by libavcodec. Defined here so the LLEXT resolves + * them internally. (In a native/testbench build the real libm provides these; + * this file, like the other shims, is LLEXT-only.) + */ +#ifndef SOFM_FASTMATH_TEST +float sqrtf(float x) { return sofm_sqrtf(x); } +float cbrtf(float x) { return sofm_cbrtf(x); } +float exp2f(float x) { return sofm_exp2f(x); } +float log2f(float x) { return sofm_log2f(x); } +float powf(float x, float y) { return sofm_powf(x, y); } +float sinf(float x) { return sofm_sinf(x); } +float cosf(float x) { return sofm_cosf(x); } +#endif + +#ifdef SOFM_FASTMATH_TEST +#include +#include +#include +#include + +/* metric: 0 = relative error, 1 = absolute error */ +static double maxerr(const char *name, float (*approx)(float), + double (*ref)(double), float lo, float hi, int log_sweep, + int absolute) +{ + double worst = 0.0, at = 0.0; + int i, N = 200000; + + for (i = 0; i < N; i++) { + double t = (double)i / (N - 1); + float x = log_sweep ? (float)(lo * pow(hi / lo, t)) + : (float)(lo + (hi - lo) * t); + double r = ref((double)x), a = approx(x); + double e = absolute ? fabs(a - r) + : (fabs(r) > 1e-9 ? fabs((a - r) / r) : fabs(a - r)); + + if (e > worst) { worst = e; at = x; } + } + printf(" %-8s max_%s_err=%.3e at x=%.6g\n", name, + absolute ? "abs" : "rel", worst, at); + return worst; +} + +static double refpow_x(double x) { return pow(x, 1.3333333333); } + +int main(void) +{ + volatile float acc = 0; + int i; + clock_t t0; + + printf("accuracy (approx vs libm double reference):\n"); + maxerr("sqrtf", sofm_sqrtf, sqrt, 1e-6f, 1e6f, 1, 0); + maxerr("cbrtf", sofm_cbrtf, cbrt, 1e-6f, 1e6f, 1, 0); + maxerr("exp2f", sofm_exp2f, exp2, -30.f, 30.f, 0, 0); + maxerr("log2f", sofm_log2f, log2, 1e-6f, 1e6f, 1, 1); /* abs: log2 crosses 0 */ + maxerr("sinf", sofm_sinf, sin, -25.f, 25.f, 0, 1); /* abs: sin/cos in [-1,1] */ + maxerr("cosf", sofm_cosf, cos, -25.f, 25.f, 0, 1); + /* powf specifically over the AAC dequant shape x^(4/3), x in [0,8192] */ + { + double worst = 0, at = 0; + for (i = 1; i < 200000; i++) { + float x = (float)(8192.0 * i / 200000.0); + double r = refpow_x(x), a = sofm_powf(x, 1.3333333f); + double e = fabs(r) > 1e-9 ? fabs((a - r) / r) : fabs(a - r); + if (e > worst) { worst = e; at = x; } + } + printf(" %-8s max_rel_err=%.3e at x=%.6g (x^4/3)\n", "powf", worst, at); + } + + /* rough timing: fast vs libm for exp2f/log2f/powf */ + printf("timing (200M calls):\n"); + t0 = clock(); + for (i = 0; i < 200000000; i++) acc += sofm_exp2f((float)(i & 63) - 32.f); + printf(" sofm_exp2f: %.2fs\n", (double)(clock() - t0) / CLOCKS_PER_SEC); + t0 = clock(); + for (i = 0; i < 200000000; i++) acc += exp2f((float)(i & 63) - 32.f); + printf(" libm exp2f: %.2fs\n", (double)(clock() - t0) / CLOCKS_PER_SEC); + return (int)acc & 0; +} +#endif diff --git a/src/audio/ffmpeg_dec/ffmpeg.cmake b/src/audio/ffmpeg_dec/ffmpeg.cmake new file mode 100644 index 000000000000..e77c0d9882b2 --- /dev/null +++ b/src/audio/ffmpeg_dec/ffmpeg.cmake @@ -0,0 +1,238 @@ +# SPDX-License-Identifier: BSD-3-Clause +# +# Cross-build decoder-only FFmpeg static libraries for the ffmpeg_dec module. +# +# Invokes FFmpeg's autotools build as an ExternalProject, cross-compiled with the +# Zephyr target toolchain, enabling only the decoders selected in Kconfig. The +# source comes from the west-pinned FFmpeg project (see west.yml). Produces +# libavcodec/libavutil/libswresample .a under ${FFMPEG_INSTALL_DIR} for the LLEXT +# to link, and defines the 'ffmpeg_ext' target it must depend on. + +include(ExternalProject) + +# --- 1. Locate the FFmpeg source (west module), overridable for local trees --- +if(NOT DEFINED SOF_FFMPEG_SRC_DIR) + set(SOF_FFMPEG_SRC_DIR "${sof_top_dir}/../modules/audio/ffmpeg" + CACHE PATH "FFmpeg source tree (west-pinned)") +endif() +cmake_path(NORMAL_PATH SOF_FFMPEG_SRC_DIR) +if(NOT EXISTS "${SOF_FFMPEG_SRC_DIR}/configure") + message(FATAL_ERROR + "ffmpeg_dec: FFmpeg source not found at '${SOF_FFMPEG_SRC_DIR}'.\n" + "Run 'west update' (FFmpeg is pinned in west.yml) or pass " + "-DSOF_FFMPEG_SRC_DIR=.") +endif() + +# --- 2. Kconfig -> --enable-decoder / --enable-parser lists --- +set(_ff_decoders "") +set(_ff_parsers "") +if(CONFIG_FFMPEG_DEC_FLAC) + list(APPEND _ff_decoders flac) + list(APPEND _ff_parsers flac) +endif() +if(CONFIG_FFMPEG_DEC_AAC) + list(APPEND _ff_decoders aac aac_latm) + list(APPEND _ff_parsers aac aac_latm) +endif() +if(CONFIG_FFMPEG_DEC_OPUS) + list(APPEND _ff_decoders opus) + list(APPEND _ff_parsers opus) +endif() +if(CONFIG_FFMPEG_DEC_MP3) + list(APPEND _ff_decoders mp3) + list(APPEND _ff_parsers mpegaudio) +endif() +# A decoder is required unless this is an encoder-only build. +if(NOT _ff_decoders AND NOT CONFIG_FFMPEG_ENC_MP3) + message(FATAL_ERROR "ffmpeg_dec: no decoder or encoder selected (Kconfig FFMPEG_DEC_*/FFMPEG_ENC_*)") +endif() +set(_ff_dec_cfg "") +if(_ff_decoders) + list(JOIN _ff_decoders "," _ff_dec_csv) + list(JOIN _ff_parsers "," _ff_par_csv) + set(_ff_dec_cfg --enable-decoder=${_ff_dec_csv} --enable-parser=${_ff_par_csv}) +endif() + +# --- 2b. Kconfig -> libavfilter audio filters (off unless a filter is chosen) --- +set(_ff_avfilter_cfg --disable-avfilter) +if(CONFIG_FFMPEG_BUILD_AVFILTER) + set(_ff_filters "") + if(CONFIG_FFMPEG_FILTER_AFFTDN) + list(APPEND _ff_filters afftdn) + endif() + list(JOIN _ff_filters "," _ff_filt_csv) + # abuffer/abuffersink feed/drain a filter graph; aformat negotiates format. + set(_ff_avfilter_cfg + --enable-avfilter + --enable-filter=abuffer,abuffersink,aformat,${_ff_filt_csv}) + message(STATUS "ffmpeg_dec: enabling avfilter, filters [${_ff_filt_csv}]") +endif() + +# --- 3. Derive the cross toolchain + per-compiler cross flags --- +# GCC 14 (Zephyr SDK) promotes these to errors; FFmpeg 7.x trips them. +set(_ff_extra_cflags "-fPIC -Wno-error=incompatible-pointer-types -Wno-error=implicit-function-declaration") + +if(CMAKE_C_COMPILER_ID STREQUAL "Clang") + # LLVM/xt-clang build. clang is a generic driver, so unlike the target- + # specific GCC it needs the target/core/sysroot spelled out, it uses the + # Zephyr-SDK GNU binutils for ar/as/nm/objcopy, and it cannot link the bare- + # metal configure *test* executables itself (no default crt/linker script) -- + # so those are linked with the GNU gcc via --ld. We only ever build .a + # archives (ar), so the linker choice affects configure tests only. + # Everything is derived from CMAKE_AR (Zephyr uses GNU binutils even under + # clang), e.g. .../bin/xtensa-intel_ace30_ptl_zephyr-elf-ar: + string(REGEX REPLACE "ar$" "" _ff_cross_prefix "${CMAKE_AR}") # ...-elf- + get_filename_component(_tc_dir "${CMAKE_AR}" DIRECTORY) # .../bin + get_filename_component(_ff_gnuroot "${_tc_dir}" DIRECTORY) # gnu root + get_filename_component(_ff_triple "${_ff_cross_prefix}" NAME) # ...-elf- + string(REGEX REPLACE "-$" "" _ff_triple "${_ff_triple}") # triple + set(_ff_sysroot "${_ff_gnuroot}/${_ff_triple}") + # core name for -mcpu: triple minus the xtensa- prefix and _zephyr-elf suffix + # (xtensa-intel_ace30_ptl_zephyr-elf -> intel_ace30_ptl). + string(REGEX REPLACE "^xtensa-" "" _ff_core "${_ff_triple}") + string(REGEX REPLACE "_zephyr-elf$" "" _ff_core "${_ff_core}") + set(_ff_cc "${CMAKE_C_COMPILER} --target=xtensa -mcpu=${_ff_core} --sysroot=${_ff_sysroot}") + set(_ff_ld "${_ff_cross_prefix}gcc") + # The LLVM Xtensa backend cannot lower the SLP-vectorised v2i32 bswap that + # FFmpeg byteswap code produces ("Cannot select: v2i32 = bswap"); disable + # vectorisation to avoid it (and it buys nothing -- no packed float SIMD). + set(_ff_extra_cflags "${_ff_extra_cflags} -fno-vectorize -fno-slp-vectorize") +else() + # GCC (Zephyr SDK): target-specific driver, brings its own sysroot + crt. + # e.g. .../bin/xtensa-intel_ace30_ptl_zephyr-elf-gcc -> prefix ...-elf- + get_filename_component(_tc_dir "${CMAKE_C_COMPILER}" DIRECTORY) + get_filename_component(_tc_name "${CMAKE_C_COMPILER}" NAME) + string(REGEX REPLACE "gcc$" "" _tc_prefix_name "${_tc_name}") + set(_ff_cross_prefix "${_tc_dir}/${_tc_prefix_name}") + set(_ff_cc "${CMAKE_C_COMPILER}") + set(_ff_ld "${CMAKE_C_COMPILER}") +endif() + +set(FFMPEG_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/ffmpeg-install" CACHE INTERNAL "ffmpeg_dec libs") + +# Hot/cold split: GCC emits av_cold (init/setup) functions into .text.unlikely. +# We rename that to SOF's .cold (post-install, below) so the LLEXT loader places +# it in DRAM. -mtext-section-literals interleaves each function's Xtensa L32R +# literals into its text section (as SOF does for LLEXT module code) so the +# renamed section carries its literals and L32R stays in range. We deliberately +# do NOT use -ffunction-sections: keeping one .text.unlikely per object lets a +# single objcopy --rename-section catch it, and lets .cold be placed by SOF's +# existing orphan .cold handling (a hand-written linker script that forces .cold +# ahead of .text overruns the SRAM budget and overlaps .rodata for big codecs). +if(CONFIG_FFMPEG_DEC_COLD_SPLIT) + # -mlongcalls: cold code lives in DRAM (.cold, relocated by the loader) far + # from the hot SRAM .text, so calls between them exceed call8 range; longcalls + # emit an indirect L32R+CALLX with unlimited range. -mtext-section-literals + # keeps each function's literals in its own text section so the rename carries + # them. Coarse split: rename the whole FFmpeg .text (not just av_cold) into + # .cold so essentially all of libav* executes from DRAM, freeing SRAM. + set(_ff_extra_cflags "${_ff_extra_cflags} -mtext-section-literals -mlongcalls") + set(_ff_objcopy "${_ff_cross_prefix}objcopy") + set(_ff_cold_rename + COMMAND ${_ff_objcopy} --rename-section .text=.cold + ${FFMPEG_INSTALL_DIR}/lib/libavcodec.a + COMMAND ${_ff_objcopy} --rename-section .text=.cold + ${FFMPEG_INSTALL_DIR}/lib/libavutil.a + COMMAND ${_ff_objcopy} --rename-section .text=.cold + ${FFMPEG_INSTALL_DIR}/lib/libswresample.a) +endif() + +# --- 3b. MP3 encoder via libshine (fixed-point; FFmpeg has no native one) --- +set(_ff_cfg_env "PATH=${_tc_dir}:$ENV{PATH}") +set(_ff_shine_cfg "") +set(_ff_shine_dep "") +if(CONFIG_FFMPEG_ENC_MP3) + set(SHINE_SRC "${sof_top_dir}/../modules/audio/shine") + if(NOT EXISTS "${SHINE_SRC}/src/lib/layer3.h") + message(FATAL_ERROR "ffmpeg_dec: libshine source not at '${SHINE_SRC}'; run 'west update'") + endif() + set(SHINE_INSTALL "${CMAKE_CURRENT_BINARY_DIR}/shine-install") + file(MAKE_DIRECTORY "${SHINE_INSTALL}/lib/pkgconfig" "${SHINE_INSTALL}/include/shine") + # pkg-config file for FFmpeg's require_pkg_config libshine. + file(WRITE "${SHINE_INSTALL}/lib/pkgconfig/shine.pc" +"prefix=${SHINE_INSTALL} +libdir=\${prefix}/lib +includedir=\${prefix}/include +Name: shine +Description: Shine fixed-point MP3 encoder +Version: 3.1.1 +Libs: -L\${libdir} -lshine +Cflags: -I\${includedir} +") + # FFmpeg's -lshine link *test* pulls newlib malloc -> Zephyr runtime syms + # (z_errno_wrap, ...) that only exist at module load. Provide dummies so the + # configure test links; --extra-ldflags only affects tests, not the .a build. + file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/shine_cfgstub.c" +"int z_errno_wrap(void){return 0;} +void *stdout; +int open(const char *p, int f, ...){return -1;} +void *sbrk(int i){return (void *)-1;} +") + # libshine's lib needs no config.h; build the objects directly and archive + # (its autotools CLI/shared link cannot work bare-metal). + file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/shine-build.sh" +"#!/bin/sh +set -e +export PATH=${_tc_dir}:\$PATH +mkdir -p ${CMAKE_CURRENT_BINARY_DIR}/shine-obj +for f in ${SHINE_SRC}/src/lib/*.c; do + ${CMAKE_C_COMPILER} -O2 -fPIC -mtext-section-literals -DSHINE_HAVE_BSWAP_H \\ + -I${SHINE_SRC}/src/lib -c \"\$f\" \\ + -o ${CMAKE_CURRENT_BINARY_DIR}/shine-obj/\$(basename \"\$f\").o +done +${_ff_cross_prefix}ar rcs ${SHINE_INSTALL}/lib/libshine.a ${CMAKE_CURRENT_BINARY_DIR}/shine-obj/*.o +cp ${SHINE_SRC}/src/lib/layer3.h ${SHINE_INSTALL}/include/shine/layer3.h +${CMAKE_C_COMPILER} -fPIC -c ${CMAKE_CURRENT_BINARY_DIR}/shine_cfgstub.c \\ + -o ${CMAKE_CURRENT_BINARY_DIR}/shine_cfgstub.o +") + add_custom_command( + OUTPUT "${SHINE_INSTALL}/lib/libshine.a" "${CMAKE_CURRENT_BINARY_DIR}/shine_cfgstub.o" + COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/shine-build.sh" + VERBATIM) + add_custom_target(shine_ext DEPENDS "${SHINE_INSTALL}/lib/libshine.a") + + set(_ff_cfg_env "PATH=${_tc_dir}:$ENV{PATH}" "PKG_CONFIG_PATH=${SHINE_INSTALL}/lib/pkgconfig") + set(_ff_shine_cfg --enable-libshine --enable-encoder=libshine --pkg-config=pkg-config + "--extra-ldflags=${CMAKE_CURRENT_BINARY_DIR}/shine_cfgstub.o") + set(_ff_shine_dep shine_ext) + message(STATUS "ffmpeg_dec: enabling MP3 encoder via libshine (${SHINE_SRC})") +endif() + +# --- 4. Configure + make + install (out-of-tree; source must be clean) --- +ExternalProject_Add(ffmpeg_ext + DEPENDS ${_ff_shine_dep} + SOURCE_DIR "${SOF_FFMPEG_SRC_DIR}" + BUILD_IN_SOURCE 0 + CONFIGURE_COMMAND + ${CMAKE_COMMAND} -E env ${_ff_cfg_env} + /configure + --prefix=${FFMPEG_INSTALL_DIR} + --enable-cross-compile --target-os=none --arch=xtensa + --cross-prefix=${_ff_cross_prefix} "--cc=${_ff_cc}" "--ld=${_ff_ld}" + # NOTE: do NOT pass --disable-asm. FFmpeg treats every per-arch + # optimisation dir (incl. our C-intrinsic libavutil/xtensa/) as + # "asm"; --disable-asm forces arch=c and drops them. There is no + # Xtensa assembly, so leaving asm enabled is safe (HiFi kernels are + # C intrinsics) and is required to build ff_float_dsp_init_xtensa. + --disable-all --disable-everything --disable-autodetect + --disable-programs --disable-doc --disable-network + --disable-avformat --disable-avdevice ${_ff_avfilter_cfg} + --disable-swscale --disable-postproc + --disable-pthreads --disable-w32threads --disable-os2threads + --disable-runtime-cpudetect --disable-debug + --enable-avcodec --enable-avutil --enable-swresample + ${_ff_dec_cfg} + ${_ff_shine_cfg} + --enable-small --enable-pic + "--extra-cflags=${_ff_extra_cflags}" + BUILD_COMMAND + ${CMAKE_COMMAND} -E env "PATH=${_tc_dir}:$ENV{PATH}" make -j 8 + INSTALL_COMMAND make install + ${_ff_cold_rename} + BUILD_BYPRODUCTS + ${FFMPEG_INSTALL_DIR}/lib/libavcodec.a + ${FFMPEG_INSTALL_DIR}/lib/libavutil.a + ${FFMPEG_INSTALL_DIR}/lib/libswresample.a +) + +message(STATUS "ffmpeg_dec: cross-building FFmpeg decoders [${_ff_dec_csv}] from ${SOF_FFMPEG_SRC_DIR}") diff --git a/src/audio/ffmpeg_dec/ffmpeg_dec-alloc.c b/src/audio/ffmpeg_dec/ffmpeg_dec-alloc.c new file mode 100644 index 000000000000..817ee06c480a --- /dev/null +++ b/src/audio/ffmpeg_dec/ffmpeg_dec-alloc.c @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. +// +// malloc/free/realloc for the ffmpeg_dec libavcodec backend, backed by the SOF +// module heap. libavcodec's av_malloc() bottoms out in libc malloc; the SOF core +// does not export malloc to LLEXT modules, so we provide it here and route it to +// mod_alloc_align()/mod_free() — no large static reservation, memory comes from +// the DSP heap like any other SOF module. +// +// Each block is prefixed with a small header storing its payload size so realloc +// can copy the old contents (SOF's allocator does not expose a realloc). The +// header is 16 bytes to preserve 16-byte alignment of the returned pointer. +// +// The active processing_module is bound at the entry of every backend op (init/ +// prepare/process/reset/free) via ffmpeg_dec_libc_bind(); decode is serialized +// per module on a pipeline core, so a single global binding is safe. + +#include +#include +#include +#include +#include "ffmpeg_dec.h" + +#define FFMPEG_DEC_ALLOC_HDR 16 + +static struct processing_module *ffmpeg_dec_alloc_mod; + +void ffmpeg_dec_libc_bind(struct processing_module *mod) +{ + ffmpeg_dec_alloc_mod = mod; +} + +void *malloc(size_t size) +{ + uint8_t *base; + + if (!ffmpeg_dec_alloc_mod || !size) + return NULL; + + base = mod_alloc_align(ffmpeg_dec_alloc_mod, size + FFMPEG_DEC_ALLOC_HDR, + FFMPEG_DEC_ALLOC_HDR); + if (!base) + return NULL; + + *(size_t *)base = size; + return base + FFMPEG_DEC_ALLOC_HDR; +} + +void free(void *ptr) +{ + if (!ptr || !ffmpeg_dec_alloc_mod) + return; + + mod_free(ffmpeg_dec_alloc_mod, (uint8_t *)ptr - FFMPEG_DEC_ALLOC_HDR); +} + +void *realloc(void *ptr, size_t size) +{ + size_t old_size; + void *new_ptr; + + if (!ptr) + return malloc(size); + + if (!size) { + free(ptr); + return NULL; + } + + old_size = *(size_t *)((uint8_t *)ptr - FFMPEG_DEC_ALLOC_HDR); + new_ptr = malloc(size); + if (new_ptr) { + memcpy(new_ptr, ptr, old_size < size ? old_size : size); + free(ptr); + } + + return new_ptr; +} + +void *calloc(size_t nmemb, size_t size) +{ + size_t total = nmemb * size; + void *ptr = malloc(total); + + if (ptr) + memset(ptr, 0, total); + return ptr; +} + +/* av_malloc may use posix_memalign. Our allocator returns 16-byte-aligned blocks, + * which is sufficient for the --disable-asm (no-SIMD) build; larger alignment + * requests are satisfied at 16 bytes. */ +int posix_memalign(void **memptr, size_t alignment, size_t size) +{ + void *ptr = malloc(size); + + (void)alignment; + if (!ptr) + return 12; /* ENOMEM */ + *memptr = ptr; + return 0; +} diff --git a/src/audio/ffmpeg_dec/ffmpeg_dec-encode.c b/src/audio/ffmpeg_dec/ffmpeg_dec-encode.c new file mode 100644 index 000000000000..69d2d14807b1 --- /dev/null +++ b/src/audio/ffmpeg_dec/ffmpeg_dec-encode.c @@ -0,0 +1,207 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. +// +// libavcodec encode backend for the ffmpeg_dec module (encode mode): PCM in, +// compressed elementary stream out (MP3 via libshine). It is the mirror of the +// decoder path - avcodec_send_frame / avcodec_receive_packet instead of +// send_packet / receive_frame. +// +// SOF PCM is interleaved S32; the encoder is opened in whatever sample format it +// supports (libshine: S16), so S32 is converted to the encoder's format per +// frame. MP3 uses fixed 1152-sample frames; this feeds one encoder frame worth +// of input per process call when enough is available. + +#include +#include +#include +#include +#include +#include "ffmpeg_dec.h" + +#include +#include +#include + +LOG_MODULE_DECLARE(ffmpeg_dec, CONFIG_SOF_LOG_LEVEL); + +#define FFMPEG_ENC_DEFAULT_BITRATE 128000 + +struct ffmpeg_enc_data { + const AVCodec *codec; + AVCodecContext *avctx; + AVPacket *pkt; + AVFrame *frame; + int frame_bytes; /* SOF S32 input frame size (all channels) */ +}; + +int ffmpeg_enc_mod_init(struct processing_module *mod) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + struct comp_dev *dev = mod->dev; + struct ffmpeg_enc_data *e; + + ffmpeg_dec_libc_bind(mod); + + e = mod_zalloc(mod, sizeof(*e)); + if (!e) + return -ENOMEM; + + /* libshine is the fixed-point MP3 encoder built into the archive. */ + e->codec = avcodec_find_encoder_by_name("libshine"); + if (!e->codec) { + comp_err(dev, "libshine MP3 encoder not found"); + mod_free(mod, e); + return -ENODEV; + } + + e->avctx = avcodec_alloc_context3(e->codec); + e->pkt = av_packet_alloc(); + e->frame = av_frame_alloc(); + if (!e->avctx || !e->pkt || !e->frame) { + if (e->frame) + av_frame_free(&e->frame); + if (e->pkt) + av_packet_free(&e->pkt); + if (e->avctx) + avcodec_free_context(&e->avctx); + mod_free(mod, e); + return -ENOMEM; + } + + cd->backend_data = e; + return 0; +} + +int ffmpeg_enc_mod_prepare(struct processing_module *mod, + struct sof_source **sources, int num_of_sources, + struct sof_sink **sinks, int num_of_sinks) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + struct ffmpeg_enc_data *e = cd->backend_data; + struct comp_dev *dev = mod->dev; + int ret; + + if (num_of_sources != 1) + return -EINVAL; + + ffmpeg_dec_libc_bind(mod); + + cd->out_channels = source_get_channels(sources[0]); + cd->out_rate = source_get_rate(sources[0]); + e->frame_bytes = source_get_frame_bytes(sources[0]); + + e->avctx->sample_rate = cd->out_rate; + av_channel_layout_default(&e->avctx->ch_layout, cd->out_channels); + /* Use the encoder's native sample format (libshine: S16). */ + e->avctx->sample_fmt = e->codec->sample_fmts ? + e->codec->sample_fmts[0] : AV_SAMPLE_FMT_S16; + e->avctx->bit_rate = FFMPEG_ENC_DEFAULT_BITRATE; + e->avctx->thread_count = 1; + + ret = avcodec_open2(e->avctx, e->codec, NULL); + if (ret < 0) { + comp_err(dev, "avcodec_open2 (encoder) failed %d", ret); + return -EIO; + } + + comp_info(dev, "ffmpeg_enc: libshine MP3, rate %u ch %u frame_size %d", + cd->out_rate, cd->out_channels, e->avctx->frame_size); + return 0; +} + +/* Convert one interleaved S32 frame block to the encoder AVFrame, then encode. */ +int ffmpeg_enc_mod_process(struct processing_module *mod, + struct input_stream_buffer *input_buffers, int num_input_buffers, + struct output_stream_buffer *output_buffers, int num_output_buffers) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + struct ffmpeg_enc_data *e = cd->backend_data; + struct comp_dev *dev = mod->dev; + int ch = cd->out_channels; + int nb = e->avctx->frame_size ? e->avctx->frame_size : 1152; + const int32_t *in; + uint8_t *out = output_buffers[0].data; + size_t out_off = 0; + int planar, i, c, ret; + + if (num_input_buffers != 1 || num_output_buffers != 1) + return -EINVAL; + + ffmpeg_dec_libc_bind(mod); + + /* Need a full encoder frame of input. */ + if (input_buffers[0].size < (uint32_t)(nb * e->frame_bytes)) + return -ENODATA; + + e->frame->nb_samples = nb; + e->frame->format = e->avctx->sample_fmt; + e->frame->sample_rate = cd->out_rate; + av_channel_layout_copy(&e->frame->ch_layout, &e->avctx->ch_layout); + if (av_frame_get_buffer(e->frame, 0) < 0) + return -ENOMEM; + + /* S32 interleaved -> S16 (planar or packed per the encoder format). */ + in = input_buffers[0].data; + planar = av_sample_fmt_is_planar(e->frame->format); + for (i = 0; i < nb; i++) + for (c = 0; c < ch; c++) { + int16_t s = (int16_t)(in[i * ch + c] >> 16); + + if (planar) + ((int16_t *)e->frame->data[c])[i] = s; + else + ((int16_t *)e->frame->data[0])[i * ch + c] = s; + } + input_buffers[0].consumed = nb * e->frame_bytes; + + ret = avcodec_send_frame(e->avctx, e->frame); + av_frame_unref(e->frame); + if (ret < 0) { + comp_err(dev, "avcodec_send_frame failed %d", ret); + return -EIO; + } + + /* Drain compressed packets into the output buffer. */ + while (ret >= 0) { + ret = avcodec_receive_packet(e->avctx, e->pkt); + if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) + break; + if (ret < 0) { + comp_err(dev, "avcodec_receive_packet failed %d", ret); + return -EIO; + } + if (out_off + e->pkt->size <= output_buffers[0].size) { + memcpy_s(out + out_off, output_buffers[0].size - out_off, + e->pkt->data, e->pkt->size); + out_off += e->pkt->size; + } else { + comp_warn(dev, "output full, MP3 packet dropped"); + } + av_packet_unref(e->pkt); + ret = 0; + } + + output_buffers[0].size = out_off; + return 0; +} + +int ffmpeg_enc_mod_free(struct processing_module *mod) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + struct ffmpeg_enc_data *e = cd->backend_data; + + ffmpeg_dec_libc_bind(mod); + if (!e) + return 0; + + if (e->frame) + av_frame_free(&e->frame); + if (e->pkt) + av_packet_free(&e->pkt); + if (e->avctx) + avcodec_free_context(&e->avctx); + mod_free(mod, e); + cd->backend_data = NULL; + return 0; +} diff --git a/src/audio/ffmpeg_dec/ffmpeg_dec-ffmpeg.c b/src/audio/ffmpeg_dec/ffmpeg_dec-ffmpeg.c new file mode 100644 index 000000000000..8c087b9c4036 --- /dev/null +++ b/src/audio/ffmpeg_dec/ffmpeg_dec-ffmpeg.c @@ -0,0 +1,346 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. +// +// libavcodec decode backend for the ffmpeg_dec module. +// +// Drives the modern send-packet / receive-frame libavcodec API standalone (no +// libavformat, no file I/O). A raw compressed elementary stream is parsed on +// the DSP with an AVCodecParser (av_parser_parse2), each recovered frame is fed +// to avcodec_send_packet, and decoded PCM is drained with avcodec_receive_frame +// and interleaved into the SOF output buffer. +// +// Requires pre-compiled libavcodec / libavutil / libswresample static libraries +// and headers for the target under third_party/ (see the Phase 0 build). It is +// only compiled when CONFIG_COMP_FFMPEG_DEC_STUB is not selected. + +#include +#include +#include +#include +#include +#include "ffmpeg_dec.h" + +#include +#include +#include +#include +#include + +LOG_MODULE_DECLARE(ffmpeg_dec, CONFIG_SOF_LOG_LEVEL); + +/* Padding libavcodec's bitstream readers over-read past the end of a packet. */ +#ifndef AV_INPUT_BUFFER_PADDING_SIZE +#define AV_INPUT_BUFFER_PADDING_SIZE 64 +#endif + +/* + * Route FFmpeg's av_log() into the SOF/Zephyr logging subsystem instead of its + * default callback (which writes to stderr via fprintf). The line buffer is + * static: decode is single-threaded, and a static buffer avoids handing Zephyr's + * deferred logging a pointer into a stack frame that has since been unwound. + */ +static char ffmpeg_dec_log_line[160]; + +static void ffmpeg_dec_av_log(void *avcl, int level, const char *fmt, va_list vl) +{ + int n; + + if (level > av_log_get_level()) + return; + + n = vsnprintf(ffmpeg_dec_log_line, sizeof(ffmpeg_dec_log_line), fmt, vl); + if (n <= 0) + return; + + /* Trim the trailing newline FFmpeg conventionally appends. */ + if (n < (int)sizeof(ffmpeg_dec_log_line) && ffmpeg_dec_log_line[n - 1] == '\n') + ffmpeg_dec_log_line[n - 1] = '\0'; + + if (level <= AV_LOG_ERROR) + LOG_ERR("ffmpeg: %s", ffmpeg_dec_log_line); + else if (level <= AV_LOG_WARNING) + LOG_WRN("ffmpeg: %s", ffmpeg_dec_log_line); + else if (level <= AV_LOG_INFO) + LOG_INF("ffmpeg: %s", ffmpeg_dec_log_line); + else + LOG_DBG("ffmpeg: %s", ffmpeg_dec_log_line); +} + +/** + * struct ffmpeg_dec_ffmpeg_data - libavcodec backend private state. + * @avctx: Decoder context. + * @parser: Elementary-stream frame parser. + * @pkt: Reusable packet handed to the decoder. + * @frame: Reusable decoded PCM frame. + * @pktbuf: Padded scratch buffer for parsed packet payloads. + * @codec: Resolved AVCodec for the selected codec id. + */ +struct ffmpeg_dec_ffmpeg_data { + AVCodecContext *avctx; + AVCodecParserContext *parser; + AVPacket *pkt; + AVFrame *frame; + uint8_t *pktbuf; + const AVCodec *codec; +}; + +/* Map the SOF codec enum to a libavcodec decoder id. */ +static enum AVCodecID ffmpeg_dec_av_codec_id(enum ffmpeg_dec_codec codec) +{ + switch (codec) { +#if defined(CONFIG_FFMPEG_DEC_FLAC) + case FFMPEG_DEC_CODEC_FLAC: + return AV_CODEC_ID_FLAC; +#endif +#if defined(CONFIG_FFMPEG_DEC_AAC) + case FFMPEG_DEC_CODEC_AAC: + return AV_CODEC_ID_AAC; +#endif +#if defined(CONFIG_FFMPEG_DEC_OPUS) + case FFMPEG_DEC_CODEC_OPUS: + return AV_CODEC_ID_OPUS; +#endif +#if defined(CONFIG_FFMPEG_DEC_MP3) + case FFMPEG_DEC_CODEC_MP3: + return AV_CODEC_ID_MP3; +#endif + default: + return AV_CODEC_ID_NONE; + } +} + +static int ffmpeg_dec_ff_init(struct processing_module *mod) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + struct comp_dev *dev = mod->dev; + struct ffmpeg_dec_ffmpeg_data *ff; + enum AVCodecID id; + + /* Bind the heap that backs FFmpeg's malloc before any av_malloc runs. */ + ffmpeg_dec_libc_bind(mod); + + /* Redirect libavcodec logging into SOF/Zephyr before anything can log. */ + av_log_set_level(AV_LOG_ERROR); + av_log_set_callback(ffmpeg_dec_av_log); + + ff = mod_zalloc(mod, sizeof(*ff)); + if (!ff) + return -ENOMEM; + + id = ffmpeg_dec_av_codec_id(cd->codec); + ff->codec = avcodec_find_decoder(id); + if (!ff->codec) { + comp_err(dev, "no libavcodec decoder for codec %d", cd->codec); + mod_free(mod, ff); + return -ENODEV; + } + + ff->parser = av_parser_init(ff->codec->id); + ff->avctx = avcodec_alloc_context3(ff->codec); + ff->pkt = av_packet_alloc(); + ff->frame = av_frame_alloc(); + if (!ff->avctx || !ff->pkt || !ff->frame) { + comp_err(dev, "libavcodec object allocation failed"); + goto err; + } + + cd->backend_data = ff; + return 0; + +err: + if (ff->frame) + av_frame_free(&ff->frame); + if (ff->pkt) + av_packet_free(&ff->pkt); + if (ff->avctx) + avcodec_free_context(&ff->avctx); + if (ff->parser) + av_parser_close(ff->parser); + mod_free(mod, ff); + return -ENOMEM; +} + +static int ffmpeg_dec_ff_configure(struct processing_module *mod) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + struct ffmpeg_dec_ffmpeg_data *ff = cd->backend_data; + struct comp_dev *dev = mod->dev; + int ret; + + ffmpeg_dec_libc_bind(mod); + + /* Some codecs (FLAC, Opus, Vorbis, AAC-in-MP4) need their setup header + * in avctx->extradata before avcodec_open2(). + */ + if (cd->extradata && cd->extradata_size) { + av_freep(&ff->avctx->extradata); + ff->avctx->extradata = + av_mallocz(cd->extradata_size + AV_INPUT_BUFFER_PADDING_SIZE); + if (!ff->avctx->extradata) + return -ENOMEM; + memcpy_s(ff->avctx->extradata, cd->extradata_size, + cd->extradata, cd->extradata_size); + ff->avctx->extradata_size = cd->extradata_size; + } + + /* Force single-threaded decode: no pthreads, no frame-threading latency, + * and get_buffer2 need not be thread-safe. + */ + ff->avctx->thread_count = 1; + + ret = avcodec_open2(ff->avctx, ff->codec, NULL); + if (ret < 0) { + comp_err(dev, "avcodec_open2 failed %d", ret); + return -EIO; + } + + /* Padded scratch for a parsed packet payload, sized to OBS as a bound. */ + ff->pktbuf = mod_alloc(mod, cd->out_frame_bytes ? + cd->out_frame_bytes * 4096 : 65536); + if (!ff->pktbuf) + return -ENOMEM; + + return 0; +} + +/* + * Interleave one decoded AVFrame into @out, honouring planar vs packed layout. + * Returns bytes written, or a negative errno if @out cannot hold the frame. + * NOTE: for the first FLAC bring-up the output sample format is assumed to match + * the sink (S16/S32). Format/rate conversion (libswresample) is a follow-up. + */ +static int ffmpeg_dec_emit_frame(struct processing_module *mod, AVFrame *frame, + uint8_t *out, size_t out_size) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + int channels = cd->out_channels; + int bps = av_get_bytes_per_sample(frame->format); + int planar = av_sample_fmt_is_planar(frame->format); + size_t need = (size_t)frame->nb_samples * channels * bps; + int i, ch; + + if (need > out_size) + return -ENOSPC; + + if (!planar) { + memcpy_s(out, out_size, frame->data[0], need); + return need; + } + + for (i = 0; i < frame->nb_samples; i++) + for (ch = 0; ch < channels; ch++) + memcpy_s(out + (((size_t)i * channels + ch) * bps), + out_size, frame->data[ch] + (size_t)i * bps, bps); + + return need; +} + +static int ffmpeg_dec_ff_decode(struct processing_module *mod, + const uint8_t *in, size_t in_size, size_t *consumed, + uint8_t *out, size_t out_size, size_t *produced) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + struct ffmpeg_dec_ffmpeg_data *ff = cd->backend_data; + struct comp_dev *dev = mod->dev; + size_t out_off = 0; + int used; + int ret; + + ffmpeg_dec_libc_bind(mod); + + *consumed = 0; + *produced = 0; + + /* Parse one frame out of the raw stream. */ + used = av_parser_parse2(ff->parser, ff->avctx, + &ff->pkt->data, &ff->pkt->size, + in, in_size, + AV_NOPTS_VALUE, AV_NOPTS_VALUE, 0); + if (used < 0) { + comp_err(dev, "av_parser_parse2 failed %d", used); + return -EIO; + } + *consumed = used; + + /* Parser still buffering: no complete frame yet. */ + if (ff->pkt->size == 0) + return 0; + + ret = avcodec_send_packet(ff->avctx, ff->pkt); + if (ret < 0) { + comp_err(dev, "avcodec_send_packet failed %d", ret); + return -EIO; + } + + /* Drain all frames this packet produced. */ + while (ret >= 0) { + ret = avcodec_receive_frame(ff->avctx, ff->frame); + if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) + break; + if (ret < 0) { + comp_err(dev, "avcodec_receive_frame failed %d", ret); + return -EIO; + } + + ret = ffmpeg_dec_emit_frame(mod, ff->frame, out + out_off, + out_size - out_off); + if (ret < 0) { + comp_warn(dev, "output buffer full, PCM dropped"); + break; + } + out_off += ret; + ret = 0; + } + + *produced = out_off; + return 0; +} + +static int ffmpeg_dec_ff_reset(struct processing_module *mod) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + struct ffmpeg_dec_ffmpeg_data *ff = cd->backend_data; + + ffmpeg_dec_libc_bind(mod); + + if (ff && ff->avctx) + avcodec_flush_buffers(ff->avctx); + + return 0; +} + +static int ffmpeg_dec_ff_free(struct processing_module *mod) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + struct ffmpeg_dec_ffmpeg_data *ff = cd->backend_data; + + ffmpeg_dec_libc_bind(mod); + + if (!ff) + return 0; + + if (ff->frame) + av_frame_free(&ff->frame); + if (ff->pkt) + av_packet_free(&ff->pkt); + if (ff->avctx) + avcodec_free_context(&ff->avctx); + if (ff->parser) + av_parser_close(ff->parser); + if (ff->pktbuf) + mod_free(mod, ff->pktbuf); + + mod_free(mod, ff); + cd->backend_data = NULL; + return 0; +} + +const struct ffmpeg_dec_backend ffmpeg_dec_backend = { + .name = "libavcodec", + .init = ffmpeg_dec_ff_init, + .configure = ffmpeg_dec_ff_configure, + .decode = ffmpeg_dec_ff_decode, + .reset = ffmpeg_dec_ff_reset, + .free = ffmpeg_dec_ff_free, +}; diff --git a/src/audio/ffmpeg_dec/ffmpeg_dec-filter.c b/src/audio/ffmpeg_dec/ffmpeg_dec-filter.c new file mode 100644 index 000000000000..5b33b99692a2 --- /dev/null +++ b/src/audio/ffmpeg_dec/ffmpeg_dec-filter.c @@ -0,0 +1,281 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. +// +// libavfilter graph backend for the ffmpeg_dec module: runs an FFmpeg audio +// filter (e.g. afftdn FFT noise reduction) over PCM. Unlike the decoder path +// (send_packet/receive_frame), filters use the avfilter graph API: +// +// abuffer(src) -> -> abuffersink(sink) +// +// PCM AVFrames are pushed into the source, pulled filtered from the sink. This +// is a PCM->PCM effect path, distinct from the compressed->PCM decode path; it +// is only built when a filter is selected (CONFIG_FFMPEG_FILTER_*), which pulls +// libavfilter into the module. + +#include +#include +#include +#include +#include "ffmpeg_dec.h" + +#include +#include +#include +#include +#include +#include + +LOG_MODULE_DECLARE(ffmpeg_dec, CONFIG_SOF_LOG_LEVEL); + +/** + * struct ffmpeg_af_graph - one audio filter graph instance. + * @graph: The avfilter graph. + * @src: abuffer source context (frames pushed here). + * @sink: abuffersink context (filtered frames pulled here). + */ +struct ffmpeg_af_graph { + AVFilterGraph *graph; + AVFilterContext *src; + AVFilterContext *sink; +}; + +/** + * ffmpeg_af_open() - Build a src -> -> sink graph for the given PCM + * format. + * @g: Graph to initialise. + * @filter: Filter name, e.g. "afftdn". + * @rate: Sample rate (Hz). + * @channels: Channel count. + * @fmt: Interleaved AV sample format (e.g. AV_SAMPLE_FMT_S32). + * + * Return: Zero on success, negative errno otherwise. + */ +int ffmpeg_af_open(struct ffmpeg_af_graph *g, const char *filter, + int rate, int channels, enum AVSampleFormat fmt) +{ + const AVFilter *abuffer = avfilter_get_by_name("abuffer"); + const AVFilter *abuffersink = avfilter_get_by_name("abuffersink"); + const AVFilter *filt = avfilter_get_by_name(filter); + AVFilterContext *filt_ctx = NULL; + AVChannelLayout layout; + char chbuf[64]; + char args[256]; + int ret; + + memset(g, 0, sizeof(*g)); + if (!abuffer || !abuffersink || !filt) + return -ENODEV; + + g->graph = avfilter_graph_alloc(); + if (!g->graph) + return -ENOMEM; + + av_channel_layout_default(&layout, channels); + av_channel_layout_describe(&layout, chbuf, sizeof(chbuf)); + + snprintf(args, sizeof(args), + "time_base=1/%d:sample_rate=%d:sample_fmt=%s:channel_layout=%s", + rate, rate, av_get_sample_fmt_name(fmt), chbuf); + + ret = avfilter_graph_create_filter(&g->src, abuffer, "in", args, NULL, g->graph); + if (ret < 0) + goto err; + ret = avfilter_graph_create_filter(&filt_ctx, filt, "filter", NULL, NULL, g->graph); + if (ret < 0) + goto err; + ret = avfilter_graph_create_filter(&g->sink, abuffersink, "out", NULL, NULL, g->graph); + if (ret < 0) + goto err; + + ret = avfilter_link(g->src, 0, filt_ctx, 0); + if (!ret) + ret = avfilter_link(filt_ctx, 0, g->sink, 0); + if (ret < 0) + goto err; + + ret = avfilter_graph_config(g->graph, NULL); + if (ret < 0) + goto err; + + return 0; + +err: + avfilter_graph_free(&g->graph); + return -EIO; +} + +/** + * ffmpeg_af_filter() - Push one PCM frame through the graph, pull the result. + * @g: Graph. + * @in: Input PCM frame. + * @out: Output frame to receive the filtered PCM. + * + * Return: 0 on a produced frame, -EAGAIN if the filter needs more input, + * negative errno on error. + */ +int ffmpeg_af_filter(struct ffmpeg_af_graph *g, AVFrame *in, AVFrame *out) +{ + int ret = av_buffersrc_add_frame(g->src, in); + + if (ret < 0) + return -EIO; + + ret = av_buffersink_get_frame(g->sink, out); + if (ret == AVERROR(EAGAIN)) + return -EAGAIN; + if (ret < 0) + return -EIO; + + return 0; +} + +void ffmpeg_af_close(struct ffmpeg_af_graph *g) +{ + if (g->graph) + avfilter_graph_free(&g->graph); +} + +#if CONFIG_FFMPEG_DEC_FILTER_MODE +/* + * Filter-mode SOF module: a PCM source->sink effect that runs the graph + * (default afftdn). SOF PCM is interleaved S32; afftdn works on float planar + * (FLTP), so we deinterleave+normalize S32 -> float on the way in and the + * reverse on the way out. Bounded chunk per cycle. + * + * NOTE: afftdn has internal latency/framing, so produced samples per cycle may + * differ from consumed; the structure below drives the graph correctly but + * real-time latency/underrun tuning is left for on-hardware bring-up. + */ + +#include +#include +#include + +#define FFMPEG_AF_MAX_CHUNK 4096 +#define FFMPEG_AF_S32_SCALE 2147483648.0f /* 2^31 */ +#ifndef CONFIG_FFMPEG_AF_FILTER_NAME +#define CONFIG_FFMPEG_AF_FILTER_NAME "afftdn" +#endif + +int ffmpeg_af_mod_init(struct processing_module *mod) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + + ffmpeg_dec_libc_bind(mod); + cd->af_graph = mod_zalloc(mod, sizeof(struct ffmpeg_af_graph)); + return cd->af_graph ? 0 : -ENOMEM; +} + +int ffmpeg_af_mod_prepare(struct processing_module *mod, + struct sof_source **sources, int num_of_sources, + struct sof_sink **sinks, int num_of_sinks) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + struct comp_dev *dev = mod->dev; + + if (num_of_sources != 1 || num_of_sinks != 1) + return -EINVAL; + + ffmpeg_dec_libc_bind(mod); + cd->out_rate = source_get_rate(sources[0]); + cd->out_channels = source_get_channels(sources[0]); + cd->out_frame_bytes = source_get_frame_bytes(sources[0]); + + comp_info(dev, "ffmpeg_af: %s, rate %u ch %u", + CONFIG_FFMPEG_AF_FILTER_NAME, cd->out_rate, cd->out_channels); + + /* Graph runs in float planar; the module converts to/from S32. */ + return ffmpeg_af_open(cd->af_graph, CONFIG_FFMPEG_AF_FILTER_NAME, + cd->out_rate, cd->out_channels, AV_SAMPLE_FMT_FLTP); +} + +int ffmpeg_af_mod_process(struct processing_module *mod, + struct sof_source **sources, int num_of_sources, + struct sof_sink **sinks, int num_of_sinks) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + struct sof_source *source = sources[0]; + struct sof_sink *sink = sinks[0]; + int ch = cd->out_channels; + int in_frames = source_get_data_frames_available(source); + int n = MIN(in_frames, FFMPEG_AF_MAX_CHUNK); + AVFrame *in, *out; + const int32_t *src; + int32_t *dst; + int i, c, m, ret; + + ffmpeg_dec_libc_bind(mod); + if (n <= 0) + return 0; + + /* Build the FLTP input frame (deinterleave + normalize S32 -> float). */ + in = av_frame_alloc(); + if (!in) + return -ENOMEM; + in->format = AV_SAMPLE_FMT_FLTP; + in->nb_samples = n; + in->sample_rate = cd->out_rate; + av_channel_layout_default(&in->ch_layout, ch); + if (av_frame_get_buffer(in, 0) < 0) { + av_frame_free(&in); + return -ENOMEM; + } + + ret = source_get_data_s32(source, n * cd->out_frame_bytes, &src, NULL, NULL); + if (ret) { + av_frame_free(&in); + return ret; + } + for (i = 0; i < n; i++) + for (c = 0; c < ch; c++) + ((float *)in->data[c])[i] = (float)src[i * ch + c] / FFMPEG_AF_S32_SCALE; + source_release_data(source, n * cd->out_frame_bytes); + + /* Run the graph. */ + out = av_frame_alloc(); + if (!out) { + av_frame_free(&in); + return -ENOMEM; + } + ret = ffmpeg_af_filter(cd->af_graph, in, out); + av_frame_free(&in); + if (ret == -EAGAIN) { /* filter still priming, no output yet */ + av_frame_free(&out); + return 0; + } + if (ret) { + av_frame_free(&out); + return ret; + } + + /* Interleave + denormalize float -> S32 into the sink. */ + m = MIN(out->nb_samples, sink_get_free_frames(sink)); + if (m > 0 && sink_get_buffer_s32(sink, m * cd->out_frame_bytes, &dst, NULL, NULL) == 0) { + for (i = 0; i < m; i++) + for (c = 0; c < ch; c++) { + float f = ((const float *)out->data[c])[i] * FFMPEG_AF_S32_SCALE; + + f = f > 2147483647.0f ? 2147483647.0f : + (f < -2147483648.0f ? -2147483648.0f : f); + dst[i * ch + c] = (int32_t)f; + } + sink_commit_buffer(sink, m * cd->out_frame_bytes); + } + av_frame_free(&out); + return 0; +} + +int ffmpeg_af_mod_free(struct processing_module *mod) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + + ffmpeg_dec_libc_bind(mod); + if (cd->af_graph) { + ffmpeg_af_close(cd->af_graph); + mod_free(mod, cd->af_graph); + cd->af_graph = NULL; + } + return 0; +} +#endif /* CONFIG_FFMPEG_DEC_FILTER_MODE */ diff --git a/src/audio/ffmpeg_dec/ffmpeg_dec-shims.c b/src/audio/ffmpeg_dec/ffmpeg_dec-shims.c new file mode 100644 index 000000000000..9cbc19c5dd2d --- /dev/null +++ b/src/audio/ffmpeg_dec/ffmpeg_dec-shims.c @@ -0,0 +1,452 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. +// +// Local libc surface for the ffmpeg_dec libavcodec backend. +// +// libavcodec/libavutil reference a set of libc/libm symbols the SOF core does +// not export to LLEXT modules. This file defines them inside the module so it +// resolves at load. Categories: +// +// * stdio / time / env (file I/O, clock, getenv, stderr): NEVER used on a SOF +// DSP and never on the FLAC decode path -> no-op stubs. av_log()'s default +// stderr path is additionally bypassed by the Zephyr LOG wrapper installed in +// ffmpeg_dec-ffmpeg.c. +// * vsnprintf / snprintf: genuinely used (av_log formatting, string utils) -> +// a compact real implementation. +// * str/mem helpers not exported by the core (memchr, strchr, ...): real. +// * bsearch, __bswap*: real. +// * strto* : not on the FLAC decode path -> return-0 stubs. +// * libm (pow/sin/sqrt/...): FLAC is lossless integer and never calls these; +// SOF's fixed-point math is range-limited (sqrt 0-2, exp +/-8, 16-bit sin) so +// it cannot serve as a double libm. These are RESOLVE-ONLY STUBS that let the +// module load. THEY MUST BE REPLACED WITH A REAL DOUBLE libm before enabling +// any lossy codec (AAC/Opus/Vorbis), which would otherwise decode to garbage. +// +// NOTE: this file intentionally includes NO libc headers so these definitions +// cannot clash with newlib prototypes; only the symbol names matter to the linker. + +#include +#include +#include + +/* ============================ stdio / time / env ========================== */ + +/* stderr is referenced as a data object by FFmpeg's default log path. */ +void *stderr; + +int fprintf(void *stream, const char *fmt, ...) { (void)stream; (void)fmt; return 0; } +int fputs(const char *s, void *stream) { (void)s; (void)stream; return 0; } +unsigned long fread(void *p, unsigned long sz, unsigned long n, void *stream) +{ (void)p; (void)sz; (void)n; (void)stream; return 0; } +int fclose(void *stream) { (void)stream; return 0; } +void *fopen(const char *path, const char *mode) { (void)path; (void)mode; return NULL; } +void *fdopen(int fd, const char *mode) { (void)fd; (void)mode; return NULL; } +int setvbuf(void *s, char *buf, int mode, size_t size) +{ (void)s; (void)buf; (void)mode; (void)size; return 0; } +int open(const char *path, int flags, ...) { (void)path; (void)flags; return -1; } +int sscanf(const char *str, const char *fmt, ...) { (void)str; (void)fmt; return 0; } + +char *getenv(const char *name) { (void)name; return NULL; } +unsigned long clock(void) { return 0; } +void *gmtime_r(const void *timep, void *result) { (void)timep; (void)result; return NULL; } +void *localtime(const void *timep) { (void)timep; return NULL; } +void *localtime_r(const void *timep, void *result) { (void)timep; (void)result; return NULL; } + +/* iconv (avutil text/metadata charset conversion) - unused on the audio path. */ +void *iconv_open(const char *to, const char *from) { (void)to; (void)from; return (void *)-1; } +int iconv_close(void *cd) { (void)cd; return 0; } +size_t iconv(void *cd, char **in, size_t *inb, char **out, size_t *outb) +{ (void)cd; (void)in; (void)inb; (void)out; (void)outb; return (size_t)-1; } +int __xpg_strerror_r(int errnum, char *buf, size_t buflen) +{ (void)errnum; if (buflen) buf[0] = '\0'; return 0; } +long mktime(void *tm) { (void)tm; return -1; } +size_t strftime(char *s, size_t max, const char *fmt, const void *tm) +{ (void)fmt; (void)tm; if (max) s[0] = '\0'; return 0; } +void abort(void) { for (;;) ; } + +/* ============================ str / mem helpers =========================== */ + +void *memchr(const void *s, int c, size_t n) +{ + const unsigned char *p = s; + + while (n--) { + if (*p == (unsigned char)c) + return (void *)p; + p++; + } + return NULL; +} + +char *strchr(const char *s, int c) +{ + for (;; s++) { + if (*s == (char)c) + return (char *)s; + if (!*s) + return NULL; + } +} + +char *strrchr(const char *s, int c) +{ + const char *last = NULL; + + for (;; s++) { + if (*s == (char)c) + last = s; + if (!*s) + return (char *)last; + } +} + +char *strstr(const char *haystack, const char *needle) +{ + if (!*needle) + return (char *)haystack; + + for (; *haystack; haystack++) { + const char *h = haystack, *n = needle; + + while (*h && *n && *h == *n) { + h++; + n++; + } + if (!*n) + return (char *)haystack; + } + return NULL; +} + +static int ffmpeg_dec_in_set(char ch, const char *set) +{ + for (; *set; set++) + if (*set == ch) + return 1; + return 0; +} + +size_t strspn(const char *s, const char *accept) +{ + size_t n = 0; + + while (s[n] && ffmpeg_dec_in_set(s[n], accept)) + n++; + return n; +} + +size_t strcspn(const char *s, const char *reject) +{ + size_t n = 0; + + while (s[n] && !ffmpeg_dec_in_set(s[n], reject)) + n++; + return n; +} + +/* ============================ bsearch / bswap ============================= */ + +void *bsearch(const void *key, const void *base, size_t nmemb, size_t size, + int (*compar)(const void *, const void *)) +{ + size_t lo = 0, hi = nmemb; + + while (lo < hi) { + size_t mid = lo + (hi - lo) / 2; + const void *elem = (const char *)base + mid * size; + int r = compar(key, elem); + + if (r < 0) + hi = mid; + else if (r > 0) + lo = mid + 1; + else + return (void *)elem; + } + return NULL; +} + +uint32_t __bswapsi2(uint32_t x) +{ + return ((x & 0x000000ffu) << 24) | ((x & 0x0000ff00u) << 8) | + ((x & 0x00ff0000u) >> 8) | ((x & 0xff000000u) >> 24); +} + +uint64_t __bswapdi2(uint64_t x) +{ + return ((uint64_t)__bswapsi2((uint32_t)x) << 32) | + __bswapsi2((uint32_t)(x >> 32)); +} + +/* ============================ strto* (unused stubs) ======================= */ + +double strtod(const char *nptr, char **endptr) +{ if (endptr) *endptr = (char *)nptr; return 0; } +long strtol(const char *nptr, char **endptr, int base) +{ (void)base; if (endptr) *endptr = (char *)nptr; return 0; } +long long strtoll(const char *nptr, char **endptr, int base) +{ (void)base; if (endptr) *endptr = (char *)nptr; return 0; } +unsigned long strtoul(const char *nptr, char **endptr, int base) +{ (void)base; if (endptr) *endptr = (char *)nptr; return 0; } +unsigned long long strtoull(const char *nptr, char **endptr, int base) +{ (void)base; if (endptr) *endptr = (char *)nptr; return 0; } + +/* ============================ libm (RESOLVE-ONLY STUBS) =================== */ +/* FLAC never calls these. Replace with a real double libm for lossy codecs. */ + +double acos(double x) { (void)x; return 0; } +double asin(double x) { (void)x; return 0; } +double atan(double x) { (void)x; return 0; } +double atan2(double y, double x) { (void)y; (void)x; return 0; } +double ceil(double x) { (void)x; return 0; } +double cos(double x) { (void)x; return 0; } +double cosh(double x) { (void)x; return 0; } +double exp(double x) { (void)x; return 0; } +double exp2(double x) { (void)x; return 0; } +double fabs(double x) { return x < 0 ? -x : x; } +double floor(double x) { (void)x; return 0; } +double fmod(double x, double y) { (void)x; (void)y; return 0; } +double frexp(double x, int *e) { (void)x; if (e) *e = 0; return 0; } +double hypot(double x, double y) { (void)x; (void)y; return 0; } +long long llrint(double x) { (void)x; return 0; } +double log(double x) { (void)x; return 0; } +double pow(double x, double y) { (void)x; (void)y; return 0; } +double round(double x) { (void)x; return 0; } +double scalbn(double x, int n) { (void)x; (void)n; return 0; } +double sin(double x) { (void)x; return 0; } +double sinh(double x) { (void)x; return 0; } +double sqrt(double x) { (void)x; return 0; } +double tan(double x) { (void)x; return 0; } +double tanh(double x) { (void)x; return 0; } +double trunc(double x) { (void)x; return 0; } +double modf(double x, double *iptr) +{ long long i = (long long)x; if (iptr) *iptr = (double)i; return x - (double)i; } + +/* Small real libm helpers pulled in by libavfilter (afftdn). fmax/fmin/rint + * are exact; log10 is routed through the fast single-precision log2f (see + * fastmathf.c) so afftdn's dB math stays correct. */ +extern float sofm_log2f(float x); + +double fmax(double a, double b) { return a > b ? a : b; } +double fmin(double a, double b) { return a < b ? a : b; } +long lrint(double x) { return (long)(x < 0.0 ? x - 0.5 : x + 0.5); } +long long llrintf(float x) { return (long long)(x < 0.0f ? x - 0.5f : x + 0.5f); } +double log10(double x) { return (double)sofm_log2f((float)x) * 0.30102999566398120; } + +/* ============================ vsnprintf / snprintf ======================= */ + +static void ffmpeg_dec_putc(char *buf, size_t size, size_t *pos, char c) +{ + if (*pos < size) + buf[*pos] = c; + (*pos)++; +} + +int vsnprintf(char *buf, size_t size, const char *fmt, va_list ap) +{ + size_t pos = 0; + + for (; *fmt; fmt++) { + int left = 0, zero = 0, alt = 0, lng = 0, width = 0, prec = -1; + unsigned long long uv = 0; + int isnum = 0, base = 10, upper = 0, neg = 0; + const char *s = NULL; + char c; + + if (*fmt != '%') { + ffmpeg_dec_putc(buf, size, &pos, *fmt); + continue; + } + + fmt++; + /* flags */ + for (;; fmt++) { + if (*fmt == '-') + left = 1; + else if (*fmt == '0') + zero = 1; + else if (*fmt == '#') + alt = 1; + else if (*fmt == '+' || *fmt == ' ') + continue; + else + break; + } + /* width */ + if (*fmt == '*') { + width = va_arg(ap, int); + fmt++; + if (width < 0) { + left = 1; + width = -width; + } + } else { + while (*fmt >= '0' && *fmt <= '9') + width = width * 10 + (*fmt++ - '0'); + } + /* precision */ + if (*fmt == '.') { + fmt++; + prec = 0; + if (*fmt == '*') { + prec = va_arg(ap, int); + fmt++; + } else { + while (*fmt >= '0' && *fmt <= '9') + prec = prec * 10 + (*fmt++ - '0'); + } + } + /* length modifiers */ + for (;;) { + if (*fmt == 'l') { + lng++; + fmt++; + } else if (*fmt == 'z' || *fmt == 't' || *fmt == 'j') { + lng = 2; + fmt++; + } else if (*fmt == 'h' || *fmt == 'L') { + fmt++; + } else { + break; + } + } + + c = *fmt; + switch (c) { + case '%': + ffmpeg_dec_putc(buf, size, &pos, '%'); + continue; + case 'c': + ffmpeg_dec_putc(buf, size, &pos, (char)va_arg(ap, int)); + continue; + case 's': + s = va_arg(ap, const char *); + if (!s) + s = "(null)"; + break; + case 'd': + case 'i': { + long long v = lng >= 2 ? va_arg(ap, long long) : + lng == 1 ? va_arg(ap, long) : va_arg(ap, int); + if (v < 0) { + neg = 1; + uv = (unsigned long long)-v; + } else { + uv = (unsigned long long)v; + } + isnum = 1; + break; + } + case 'u': + uv = lng >= 2 ? va_arg(ap, unsigned long long) : + lng == 1 ? va_arg(ap, unsigned long) : va_arg(ap, unsigned int); + isnum = 1; + break; + case 'o': + uv = lng >= 2 ? va_arg(ap, unsigned long long) : + lng == 1 ? va_arg(ap, unsigned long) : va_arg(ap, unsigned int); + base = 8; + isnum = 1; + break; + case 'x': + case 'X': + uv = lng >= 2 ? va_arg(ap, unsigned long long) : + lng == 1 ? va_arg(ap, unsigned long) : va_arg(ap, unsigned int); + base = 16; + upper = (c == 'X'); + isnum = 1; + break; + case 'p': + uv = (unsigned long long)(uintptr_t)va_arg(ap, void *); + base = 16; + alt = 1; + isnum = 1; + break; + case 'e': + case 'f': + case 'g': + case 'E': + case 'F': + case 'G': + case 'a': + case 'A': + /* float formatting unsupported (see file header) */ + (void)va_arg(ap, double); + s = "0"; + break; + default: + ffmpeg_dec_putc(buf, size, &pos, '%'); + ffmpeg_dec_putc(buf, size, &pos, c); + continue; + } + + if (isnum) { + char digits[24]; + int di = 0, i; + const char *hx = upper ? "0123456789ABCDEF" : "0123456789abcdef"; + int len, pad; + + if (uv == 0) { + digits[di++] = '0'; + } else { + while (uv) { + digits[di++] = hx[uv % base]; + uv /= base; + } + } + + len = di + (neg ? 1 : 0) + (alt && base == 16 ? 2 : 0); + pad = width > len ? width - len : 0; + + if (!left && !zero) + while (pad-- > 0) + ffmpeg_dec_putc(buf, size, &pos, ' '); + if (neg) + ffmpeg_dec_putc(buf, size, &pos, '-'); + if (alt && base == 16) { + ffmpeg_dec_putc(buf, size, &pos, '0'); + ffmpeg_dec_putc(buf, size, &pos, upper ? 'X' : 'x'); + } + if (!left && zero) + while (pad-- > 0) + ffmpeg_dec_putc(buf, size, &pos, '0'); + for (i = di - 1; i >= 0; i--) + ffmpeg_dec_putc(buf, size, &pos, digits[i]); + if (left) + while (pad-- > 0) + ffmpeg_dec_putc(buf, size, &pos, ' '); + } else if (s) { + int len = 0, pad; + + while (s[len] && (prec < 0 || len < prec)) + len++; + pad = width > len ? width - len : 0; + + if (!left) + while (pad-- > 0) + ffmpeg_dec_putc(buf, size, &pos, ' '); + for (int i = 0; i < len; i++) + ffmpeg_dec_putc(buf, size, &pos, s[i]); + if (left) + while (pad-- > 0) + ffmpeg_dec_putc(buf, size, &pos, ' '); + } + } + + if (size) + buf[pos < size ? pos : size - 1] = '\0'; + + return (int)pos; +} + +int snprintf(char *buf, size_t size, const char *fmt, ...) +{ + va_list ap; + int n; + + va_start(ap, fmt); + n = vsnprintf(buf, size, fmt, ap); + va_end(ap); + return n; +} diff --git a/src/audio/ffmpeg_dec/ffmpeg_dec-stub.c b/src/audio/ffmpeg_dec/ffmpeg_dec-stub.c new file mode 100644 index 000000000000..bb7cc7408d6a --- /dev/null +++ b/src/audio/ffmpeg_dec/ffmpeg_dec-stub.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. +// +// Dependency-free stub backend for the ffmpeg_dec module. +// +// This lets the SOF module glue (init/prepare/process/config/reset/free, the +// LLEXT manifest and the topology wiring) be built and exercised in CI without +// pulling in libavcodec. It does not decode: it simply passes the input bytes +// through to the output, so the pipeline moves data end to end. The real +// libavcodec implementation lives in ffmpeg_dec-ffmpeg.c and provides the same +// struct ffmpeg_dec_backend symbol. + +#include +#include +#include +#include +#include "ffmpeg_dec.h" + +LOG_MODULE_DECLARE(ffmpeg_dec, CONFIG_SOF_LOG_LEVEL); + +static int ffmpeg_dec_stub_init(struct processing_module *mod) +{ + comp_info(mod->dev, "ffmpeg_dec stub backend: no decoder linked"); + return 0; +} + +static int ffmpeg_dec_stub_configure(struct processing_module *mod) +{ + return 0; +} + +static int ffmpeg_dec_stub_decode(struct processing_module *mod, + const uint8_t *in, size_t in_size, size_t *consumed, + uint8_t *out, size_t out_size, size_t *produced) +{ + /* Passthrough: copy as many bytes as fit, report exact consume/produce. */ + size_t n = MIN(in_size, out_size); + + if (n) + memcpy_s(out, out_size, in, n); + + *consumed = n; + *produced = n; + return 0; +} + +static int ffmpeg_dec_stub_reset(struct processing_module *mod) +{ + return 0; +} + +static int ffmpeg_dec_stub_free(struct processing_module *mod) +{ + return 0; +} + +const struct ffmpeg_dec_backend ffmpeg_dec_backend = { + .name = "stub", + .init = ffmpeg_dec_stub_init, + .configure = ffmpeg_dec_stub_configure, + .decode = ffmpeg_dec_stub_decode, + .reset = ffmpeg_dec_stub_reset, + .free = ffmpeg_dec_stub_free, +}; diff --git a/src/audio/ffmpeg_dec/ffmpeg_dec.c b/src/audio/ffmpeg_dec/ffmpeg_dec.c new file mode 100644 index 000000000000..06516a821c9d --- /dev/null +++ b/src/audio/ffmpeg_dec/ffmpeg_dec.c @@ -0,0 +1,331 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. +// +// FFmpeg (libavcodec) audio decoder wrapper - SOF module core. +// +// This translation unit contains the SOF module_interface glue only; the actual +// decoding is delegated to the backend selected at build time (see +// ffmpeg_dec-stub.c and ffmpeg_dec-ffmpeg.c). The input is a compressed audio +// elementary stream (raw data), the output is interleaved PCM. + +#include +#include +#include +#include +#include +#include "ffmpeg_dec.h" + +/* UUID identifies the component. Registered in uuid-registry.txt. */ +SOF_DEFINE_REG_UUID(ffmpeg_dec); + +/* Creates logging data for the component */ +LOG_MODULE_REGISTER(ffmpeg_dec, CONFIG_SOF_LOG_LEVEL); + +/* Decoder-mode ops (compressed -> PCM). In filter mode the module uses the + * avfilter-graph PCM effect ops (ffmpeg_dec-filter.c); in encode mode the + * PCM->compressed ops (ffmpeg_dec-encode.c) instead. + */ +#if !CONFIG_FFMPEG_DEC_FILTER_MODE && !CONFIG_FFMPEG_DEC_ENCODE_MODE +int ffmpeg_dec_store_extradata(struct processing_module *mod, + const uint8_t *data, size_t size) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + struct comp_dev *dev = mod->dev; + uint8_t *buf; + + if (!size) + return 0; + + if (size > FFMPEG_DEC_MAX_EXTRADATA) { + comp_err(dev, "extradata too large: %zu", size); + return -EINVAL; + } + + /* Replace any previously stored setup header. */ + if (cd->extradata) { + mod_free(mod, cd->extradata); + cd->extradata = NULL; + cd->extradata_size = 0; + } + + buf = mod_alloc(mod, size); + if (!buf) + return -ENOMEM; + + memcpy_s(buf, size, data, size); + cd->extradata = buf; + cd->extradata_size = size; + comp_info(dev, "stored %zu bytes of codec extradata", size); + return 0; +} + +/** + * ffmpeg_dec_init() - Initialize the ffmpeg_dec component. + * @mod: Pointer to module data. + * + * Allocates private data and hands off to the selected decode backend for its + * one-time initialization. __cold marks this non-critical path for slower DRAM. + * + * Return: Zero if success, otherwise error code. + */ +__cold static int ffmpeg_dec_init(struct processing_module *mod) +{ + struct module_data *md = &mod->priv; + struct comp_dev *dev = mod->dev; + struct ffmpeg_dec_comp_data *cd; + int ret; + + comp_info(dev, "entry"); + + cd = mod_zalloc(mod, sizeof(*cd)); + if (!cd) + return -ENOMEM; + + md->private = cd; + cd->backend = &ffmpeg_dec_backend; + /* TODO: derive codec id from topology/IPC init config. Until then use the + * default from the Kconfig decoder selection (first enabled). + */ + cd->codec = FFMPEG_DEC_DEFAULT_CODEC; + + comp_info(dev, "backend '%s'", cd->backend->name); + + if (cd->backend->init) { + ret = cd->backend->init(mod); + if (ret) { + comp_err(dev, "backend init failed %d", ret); + mod_free(mod, cd); + return ret; + } + } + + return 0; +} + +/** + * ffmpeg_dec_prepare() - Prepare the component for processing. + * @mod: Pointer to module data. + * @sources: Unused (input is a raw compressed byte stream). + * @sinks: Output PCM sink array; sinks[0] provides the target PCM format. + * + * Caches the decoded PCM output format and opens the backend decoder. + * + * Return: Zero if success, otherwise error code. + */ +static int ffmpeg_dec_prepare(struct processing_module *mod, + struct sof_source **sources, int num_of_sources, + struct sof_sink **sinks, int num_of_sinks) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + struct comp_dev *dev = mod->dev; + int ret; + + comp_dbg(dev, "entry"); + + if (num_of_sinks != 1) { + comp_err(dev, "unsupported sink count %d", num_of_sinks); + return -EINVAL; + } + + /* Cache the PCM format the decoder must produce for the pipeline. */ + cd->out_rate = sink_get_rate(sinks[0]); + cd->out_channels = sink_get_channels(sinks[0]); + cd->out_frame_fmt = sink_get_frm_fmt(sinks[0]); + cd->out_frame_bytes = sink_get_frame_bytes(sinks[0]); + + comp_info(dev, "out rate %u ch %u fmt %d frame_bytes %u", + cd->out_rate, cd->out_channels, cd->out_frame_fmt, + cd->out_frame_bytes); + + if (cd->backend->configure) { + ret = cd->backend->configure(mod); + if (ret) { + comp_err(dev, "backend configure failed %d", ret); + return ret; + } + } + + cd->configured = true; + return 0; +} + +/** + * ffmpeg_dec_process() - Decode a chunk of the compressed stream. + * @mod: Pointer to module data. + * @input_buffers: input_buffers[0].data holds compressed bytes. + * @output_buffers: output_buffers[0].data receives interleaved PCM. + * + * Raw-data processing: hand the input bytes to the backend decoder and report + * how many input bytes were consumed and how many PCM bytes were produced. + * + * Return: Zero if success, otherwise error code. + */ +static int ffmpeg_dec_process(struct processing_module *mod, + struct input_stream_buffer *input_buffers, + int num_input_buffers, + struct output_stream_buffer *output_buffers, + int num_output_buffers) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + struct comp_dev *dev = mod->dev; + size_t consumed = 0; + size_t produced = 0; + int ret; + + if (num_input_buffers != 1 || num_output_buffers != 1) + return -EINVAL; + + if (!input_buffers[0].size) + return -ENODATA; + + ret = cd->backend->decode(mod, + input_buffers[0].data, input_buffers[0].size, + &consumed, + output_buffers[0].data, output_buffers[0].size, + &produced); + if (ret) { + comp_err(dev, "decode failed %d", ret); + return ret; + } + + input_buffers[0].consumed = consumed; + output_buffers[0].size = produced; + return 0; +} + +/** + * ffmpeg_dec_set_config() - Receive the codec setup header (extradata). + * + * Reassembles a possibly fragmented binary configuration via the common + * module_set_configuration() helper, then stores the whole reassembled blob as + * codec extradata (e.g. FLAC STREAMINFO). Mirrors the DTS codec config path. + */ +__cold static int +ffmpeg_dec_set_config(struct processing_module *mod, uint32_t config_id, + enum module_cfg_fragment_position pos, uint32_t data_offset_size, + const uint8_t *fragment, size_t fragment_size, uint8_t *response, + size_t response_size) +{ + struct comp_dev *dev = mod->dev; + struct module_config *config = &mod->priv.cfg; + uint32_t header_size; + int ret; + + assert_can_be_cold(); + + ret = module_set_configuration(mod, config_id, pos, data_offset_size, fragment, + fragment_size, response, response_size); + if (ret < 0) { + comp_err(dev, "module_set_configuration failed %d", ret); + return ret; + } + + /* Wait until the whole (possibly fragmented) blob has been received. */ + if (pos != MODULE_CFG_FRAGMENT_LAST && pos != MODULE_CFG_FRAGMENT_SINGLE) + return 0; + + /* module_config prepends a {size, avail} header to the payload. */ + header_size = sizeof(config->size) + sizeof(config->avail); + if (config->size <= header_size) { + comp_warn(dev, "empty codec config"); + return 0; + } + + return ffmpeg_dec_store_extradata(mod, config->data, + config->size - header_size); +} + +/** + * ffmpeg_dec_reset() - Reset the component to a re-preparable state. + * @mod: Pointer to module data. + * + * Flushes decoder state but keeps allocations so the pipeline can restart. + * + * Return: Zero if success, otherwise error code. + */ +static int ffmpeg_dec_reset(struct processing_module *mod) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + + comp_dbg(mod->dev, "entry"); + + if (cd->backend->reset) + return cd->backend->reset(mod); + + return 0; +} + +/** + * ffmpeg_dec_free() - Free dynamic allocations. + * @mod: Pointer to module data. + * + * Return: Zero if success, otherwise error code. + */ +__cold static int ffmpeg_dec_free(struct processing_module *mod) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + + assert_can_be_cold(); + comp_dbg(mod->dev, "entry"); + + if (cd->backend->free) + cd->backend->free(mod); + + if (cd->extradata) + mod_free(mod, cd->extradata); + + mod_free(mod, cd); + return 0; +} +#endif /* !CONFIG_FFMPEG_DEC_FILTER_MODE && !CONFIG_FFMPEG_DEC_ENCODE_MODE */ + +/* This defines the module operations */ +#if CONFIG_FFMPEG_DEC_ENCODE_MODE +/* PCM -> compressed encoder (ffmpeg_dec-encode.c). */ +static const struct module_interface ffmpeg_dec_interface = { + .init = ffmpeg_enc_mod_init, + .prepare = ffmpeg_enc_mod_prepare, + .process_raw_data = ffmpeg_enc_mod_process, + .free = ffmpeg_enc_mod_free +}; +#elif CONFIG_FFMPEG_DEC_FILTER_MODE +/* PCM source/sink effect driving an avfilter graph (ffmpeg_dec-filter.c). */ +static const struct module_interface ffmpeg_dec_interface = { + .init = ffmpeg_af_mod_init, + .prepare = ffmpeg_af_mod_prepare, + .process = ffmpeg_af_mod_process, + .free = ffmpeg_af_mod_free +}; +#else +static const struct module_interface ffmpeg_dec_interface = { + .init = ffmpeg_dec_init, + .prepare = ffmpeg_dec_prepare, + .process_raw_data = ffmpeg_dec_process, + .set_configuration = ffmpeg_dec_set_config, + .reset = ffmpeg_dec_reset, + .free = ffmpeg_dec_free +}; +#endif + +/* If COMP_FFMPEG_DEC is =m in Kconfig this is built as a loadable LLEXT module. */ +#if CONFIG_COMP_FFMPEG_DEC_MODULE + +#include +#include +#include + +static const struct sof_man_module_manifest mod_manifest __section(".module") __used = + SOF_LLEXT_MODULE_MANIFEST("FFMPGDEC", &ffmpeg_dec_interface, 1, + SOF_REG_UUID(ffmpeg_dec), 40); + +SOF_LLEXT_BUILDINFO; + +#else + +/* Only used for the module adapter trace context, soon to be deprecated */ +DECLARE_TR_CTX(ffmpeg_dec_tr, SOF_UUID(ffmpeg_dec_uuid), LOG_LEVEL_INFO); +DECLARE_MODULE_ADAPTER(ffmpeg_dec_interface, ffmpeg_dec_uuid, ffmpeg_dec_tr); +SOF_MODULE_INIT(ffmpeg_dec, sys_comp_module_ffmpeg_dec_interface_init); + +#endif diff --git a/src/audio/ffmpeg_dec/ffmpeg_dec.h b/src/audio/ffmpeg_dec/ffmpeg_dec.h new file mode 100644 index 000000000000..a32d7ed70a91 --- /dev/null +++ b/src/audio/ffmpeg_dec/ffmpeg_dec.h @@ -0,0 +1,164 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright(c) 2026 Intel Corporation. + * + * FFmpeg (libavcodec) audio decoder wrapper for SOF. + * + * This module adapts SOF's module_interface to libavcodec's send-packet / + * receive-frame decode API. Compressed elementary-stream bytes arrive on the + * input (raw data) buffer, are parsed into codec frames, decoded to interleaved + * PCM and written to the output buffer. + * + * The actual decode work is delegated to a pluggable "backend" so the SOF glue + * can be validated with a dependency-free stub (ffmpeg_dec-stub.c) before the + * real libavcodec backend (ffmpeg_dec-ffmpeg.c) is linked in. + */ +#ifndef __SOF_AUDIO_FFMPEG_DEC_H__ +#define __SOF_AUDIO_FFMPEG_DEC_H__ + +#include +#include +#include +#include + +/* Maximum codec setup header (extradata) we accept, e.g. FLAC STREAMINFO. */ +#define FFMPEG_DEC_MAX_EXTRADATA 4096 + +/* Which codec this instance decodes. Kept as an explicit enum so it can be + * carried in topology/IPC config and mapped to an AVCodecID by the backend. + */ +enum ffmpeg_dec_codec { + FFMPEG_DEC_CODEC_NONE = 0, + FFMPEG_DEC_CODEC_FLAC, + FFMPEG_DEC_CODEC_AAC, + FFMPEG_DEC_CODEC_OPUS, + FFMPEG_DEC_CODEC_MP3, +}; + +/* Default codec for a new instance, picked from the Kconfig selection until the + * codec is carried per-instance from topology/IPC config. First enabled wins. + */ +#if defined(CONFIG_FFMPEG_DEC_FLAC) +#define FFMPEG_DEC_DEFAULT_CODEC FFMPEG_DEC_CODEC_FLAC +#elif defined(CONFIG_FFMPEG_DEC_AAC) +#define FFMPEG_DEC_DEFAULT_CODEC FFMPEG_DEC_CODEC_AAC +#elif defined(CONFIG_FFMPEG_DEC_OPUS) +#define FFMPEG_DEC_DEFAULT_CODEC FFMPEG_DEC_CODEC_OPUS +#elif defined(CONFIG_FFMPEG_DEC_MP3) +#define FFMPEG_DEC_DEFAULT_CODEC FFMPEG_DEC_CODEC_MP3 +#else +#define FFMPEG_DEC_DEFAULT_CODEC FFMPEG_DEC_CODEC_NONE +#endif + +struct ffmpeg_dec_comp_data; + +/** + * struct ffmpeg_dec_backend - decode backend operations. + * + * A backend owns the real decoder state (allocated by the backend itself and + * stored in ffmpeg_dec_comp_data.backend_data). All ops return 0 on success or + * a negative errno. + */ +struct ffmpeg_dec_backend { + const char *name; + + /* One-time backend init, called from module init(). */ + int (*init)(struct processing_module *mod); + + /* Open/configure the decoder once codec, extradata and the output PCM + * format (rate/channels/frame format) are known. Called from prepare(). + */ + int (*configure)(struct processing_module *mod); + + /* + * Decode from @in (in_size bytes of compressed stream) into @out (up to + * out_size bytes of interleaved PCM). On return *consumed holds input + * bytes eaten and *produced holds PCM bytes written. + */ + int (*decode)(struct processing_module *mod, + const uint8_t *in, size_t in_size, size_t *consumed, + uint8_t *out, size_t out_size, size_t *produced); + + /* Flush decoder state, keep it configured. Called from reset(). */ + int (*reset)(struct processing_module *mod); + + /* Tear down decoder state allocated by init()/configure(). */ + int (*free)(struct processing_module *mod); +}; + +/** + * struct ffmpeg_dec_comp_data - ffmpeg_dec module private data. + * @backend: Selected decode backend ops. + * @backend_data: Backend-private decoder state. + * @codec: Codec id for this instance. + * @extradata: Codec setup header (e.g. FLAC STREAMINFO), NULL if none. + * @extradata_size: Size of @extradata in bytes. + * @out_rate: Decoded PCM sample rate (Hz), from the sink stream params. + * @out_channels: Decoded PCM channel count. + * @out_frame_fmt: Decoded PCM sample format (enum sof_ipc_frame). + * @out_frame_bytes: Bytes per PCM frame (all channels). + * @configured: True once the backend decoder has been opened. + */ +struct ffmpeg_dec_comp_data { + const struct ffmpeg_dec_backend *backend; + void *backend_data; + enum ffmpeg_dec_codec codec; + uint8_t *extradata; + size_t extradata_size; + uint32_t out_rate; + uint32_t out_channels; + int out_frame_fmt; + uint32_t out_frame_bytes; + bool configured; + /* Filter mode (CONFIG_FFMPEG_DEC_FILTER_MODE): avfilter graph handle + * (struct ffmpeg_af_graph *, see ffmpeg_dec-filter.c). Unused for decode. + */ + void *af_graph; +}; + +/* Encode-mode module ops (PCM -> compressed), in ffmpeg_dec-encode.c. */ +int ffmpeg_enc_mod_init(struct processing_module *mod); +int ffmpeg_enc_mod_prepare(struct processing_module *mod, + struct sof_source **sources, int num_of_sources, + struct sof_sink **sinks, int num_of_sinks); +int ffmpeg_enc_mod_process(struct processing_module *mod, + struct input_stream_buffer *input_buffers, int num_input_buffers, + struct output_stream_buffer *output_buffers, int num_output_buffers); +int ffmpeg_enc_mod_free(struct processing_module *mod); + +/* Filter-mode module ops (PCM source/sink effect), in ffmpeg_dec-filter.c. */ +int ffmpeg_af_mod_init(struct processing_module *mod); +int ffmpeg_af_mod_prepare(struct processing_module *mod, + struct sof_source **sources, int num_of_sources, + struct sof_sink **sinks, int num_of_sinks); +int ffmpeg_af_mod_process(struct processing_module *mod, + struct sof_source **sources, int num_of_sources, + struct sof_sink **sinks, int num_of_sinks); +int ffmpeg_af_mod_free(struct processing_module *mod); + +/* Backend instance provided by the selected backend translation unit + * (ffmpeg_dec-stub.c or ffmpeg_dec-ffmpeg.c). + */ +extern const struct ffmpeg_dec_backend ffmpeg_dec_backend; + +/** + * ffmpeg_dec_store_extradata() - Copy a codec setup header into private data. + * @mod: Pointer to module data. + * @data: Setup header bytes (e.g. FLAC STREAMINFO). + * @size: Number of bytes. + * + * Return: Zero if success, otherwise error code. + */ +int ffmpeg_dec_store_extradata(struct processing_module *mod, + const uint8_t *data, size_t size); + +/** + * ffmpeg_dec_libc_bind() - Bind the module whose heap backs malloc/free/realloc. + * @mod: Pointer to module data (see ffmpeg_dec-alloc.c). + * + * Called at the entry of every libavcodec backend op so allocations made by + * FFmpeg are drawn from the current instance's SOF heap. + */ +void ffmpeg_dec_libc_bind(struct processing_module *mod); + +#endif /* __SOF_AUDIO_FFMPEG_DEC_H__ */ diff --git a/src/audio/ffmpeg_dec/ffmpeg_dec.toml b/src/audio/ffmpeg_dec/ffmpeg_dec.toml new file mode 100644 index 000000000000..6c58fcb8b731 --- /dev/null +++ b/src/audio/ffmpeg_dec/ffmpeg_dec.toml @@ -0,0 +1,25 @@ +#ifndef LOAD_TYPE +#define LOAD_TYPE "0" +#endif + + REM # FFmpeg audio decoder module config + [[module.entry]] + REM # NOTE: module name is limited to 8 chars (manifest name field); + REM # it must match the SOF_LLEXT_MODULE_MANIFEST() name in ffmpeg_dec.c. + name = "FFMPGDEC" + uuid = UUIDREG_STR_FFMPEG_DEC + affinity_mask = "0x1" + instance_count = "40" + domain_types = "0" + load_type = LOAD_TYPE + module_type = "9" + auto_start = "0" + sched_caps = [1, 0x00008000] + REM # pin = [dir, type, sample rate, size, container, channel-cfg] + pin = [0, 0, 0xfeef, 0xf, 0xf, 0x45ff, 1, 0, 0xfeef, 0xf, 0xf, 0x1ff] + REM # mod_cfg [PAR_0 PAR_1 PAR_2 PAR_3 IS_BYTES CPS IBS OBS MOD_FLAGS CPC OBLS] + REM # Decoder: input is a small compressed chunk, output is larger PCM, so + REM # OBS (output buffer size) is sized well above IBS (input buffer size). + mod_cfg = [0, 0, 0, 0, 8192, 5000000, 4096, 16384, 0, 5000, 0] + + index = __COUNTER__ diff --git a/src/audio/ffmpeg_dec/llext/CMakeLists.txt b/src/audio/ffmpeg_dec/llext/CMakeLists.txt new file mode 100644 index 000000000000..df794983f1dc --- /dev/null +++ b/src/audio/ffmpeg_dec/llext/CMakeLists.txt @@ -0,0 +1,73 @@ +# Copyright (c) 2026 Intel Corporation. +# SPDX-License-Identifier: Apache-2.0 + +if(CONFIG_COMP_FFMPEG_DEC_STUB) + # Dependency-free build: SOF glue + passthrough stub backend, no libavcodec. + sof_llext_build("ffmpeg_dec" + SOURCES ../ffmpeg_dec.c + ../ffmpeg_dec-stub.c + LIB openmodules + ) +else() + # Real decoder: cross-build the FFmpeg archive (Kconfig-selected decoders) + # and link it. libc/libm are NOT bundled (non-PIC, and would clash) — the + # module provides the needed libc symbols itself (shims/alloc), resolves the + # rest from the SOF core, and supplies fast single-precision float math + # (fastmathf.c) when a float codec (AAC/Opus) is selected. + include(${CMAKE_CURRENT_LIST_DIR}/../ffmpeg.cmake) + + set(ffmpeg_dec_srcs + ../ffmpeg_dec.c + ../ffmpeg_dec-ffmpeg.c + ../ffmpeg_dec-shims.c + ../ffmpeg_dec-alloc.c) + # fastmathf provides the single-precision libm the codecs need AND sofm_log2f + # used by the shims' log10(); always built for the real (non-stub) backend. + list(APPEND ffmpeg_dec_srcs ../fastmathf.c) + + # avfilter graph backend (PCM effects, e.g. afftdn noise reduction). Pulls + # libavfilter into the link; it must precede the libs it depends on. + set(ffmpeg_dec_libs avcodec swresample avutil) + set(ffmpeg_dec_libpaths "${FFMPEG_INSTALL_DIR}/lib") + if(CONFIG_FFMPEG_BUILD_AVFILTER) + list(APPEND ffmpeg_dec_srcs ../ffmpeg_dec-filter.c) + set(ffmpeg_dec_libs avfilter ${ffmpeg_dec_libs}) + endif() + + # Encode mode: PCM->compressed backend + libshine (linked AFTER libavcodec, + # which references it), from its own cross-build install dir. + if(CONFIG_FFMPEG_DEC_ENCODE_MODE) + list(APPEND ffmpeg_dec_srcs ../ffmpeg_dec-encode.c) + endif() + if(CONFIG_FFMPEG_ENC_MP3) + list(APPEND ffmpeg_dec_libs shine) + list(APPEND ffmpeg_dec_libpaths "${SHINE_INSTALL}/lib") + endif() + + # Cold split: module code must also use -mlongcalls to reach FFmpeg code + # that now lives in the far (DRAM) .cold region. + set(ffmpeg_dec_cflags "") + if(CONFIG_FFMPEG_DEC_COLD_SPLIT) + set(ffmpeg_dec_cflags -mlongcalls) + endif() + + sof_llext_build("ffmpeg_dec" + SOURCES ${ffmpeg_dec_srcs} + INCLUDES "${FFMPEG_INSTALL_DIR}/include" + LIBS_PATH ${ffmpeg_dec_libpaths} + LIBS ${ffmpeg_dec_libs} + CFLAGS ${ffmpeg_dec_cflags} + LIB openmodules + ) + + # The archive must be built before the module compiles/links against it. + add_dependencies(ffmpeg_dec_llext_lib ffmpeg_ext) + add_dependencies(ffmpeg_dec ffmpeg_ext) + + # Cold split: place the (renamed) FFmpeg .cold code so the loader relocates + # it to DRAM. See ffmpeg_cold.ld for why INSERT AFTER .bss. + if(CONFIG_FFMPEG_DEC_COLD_SPLIT) + llext_link_options(ffmpeg_dec + "-Wl,-T,${CMAKE_CURRENT_LIST_DIR}/ffmpeg_cold.ld") + endif() +endif() diff --git a/src/audio/ffmpeg_dec/llext/ffmpeg_cold.ld b/src/audio/ffmpeg_dec/llext/ffmpeg_cold.ld new file mode 100644 index 000000000000..1ddecd4dbfff --- /dev/null +++ b/src/audio/ffmpeg_dec/llext/ffmpeg_cold.ld @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright(c) 2026 Intel Corporation. + * + * LLEXT linker fragment for the ffmpeg_dec hot/cold split + * (CONFIG_FFMPEG_DEC_COLD_SPLIT). Collects the FFmpeg code (renamed to .cold in + * the archives by ffmpeg.cmake) plus any av_cold leftovers into an explicit + * .cold output section so the LLEXT loader (llext_manager) relocates it to DRAM. + * + * Placed with INSERT AFTER .bss so .cold gets the highest link-time address and + * cannot overlap the fixed-LMA .text/.rodata/.hash of the ace "mirrored" -shared + * layout (forcing it before/after .text overran those budgets). Its link address + * is irrelevant at runtime: the loader reads the .cold section and copies it to + * the cold/DRAM region. Cross-region .cold<->.text calls are handled by + * -mlongcalls (FFmpeg and module code). + */ +SECTIONS +{ + .cold : { + *(.cold .cold.*) + *(.literal.unlikely .literal.unlikely.*) + *(.text.unlikely .text.unlikely.*) + } +} INSERT AFTER .bss; diff --git a/src/audio/ffmpeg_dec/llext/llext.toml.h b/src/audio/ffmpeg_dec/llext/llext.toml.h new file mode 100644 index 000000000000..eda3fce26a5c --- /dev/null +++ b/src/audio/ffmpeg_dec/llext/llext.toml.h @@ -0,0 +1,6 @@ +#include +#define LOAD_TYPE "2" +#include "../ffmpeg_dec.toml" + +[module] +count = __COUNTER__ diff --git a/src/audio/webrtc_aec/CMakeLists.txt b/src/audio/webrtc_aec/CMakeLists.txt new file mode 100644 index 000000000000..e42a8644a0b2 --- /dev/null +++ b/src/audio/webrtc_aec/CMakeLists.txt @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: BSD-3-Clause + +if(CONFIG_COMP_WEBRTC_AEC STREQUAL "m" AND DEFINED CONFIG_LLEXT) + add_subdirectory(llext ${PROJECT_BINARY_DIR}/webrtc_aec_llext) + add_dependencies(app webrtc_aec) +else() + add_local_sources(sof webrtc_aec.c) + if(CONFIG_COMP_WEBRTC_AEC_STUB) + add_local_sources(sof webrtc_aec-stub.c) + else() + add_local_sources(sof webrtc_aec-aecm.c) + endif() +endif() diff --git a/src/audio/webrtc_aec/Kconfig b/src/audio/webrtc_aec/Kconfig new file mode 100644 index 000000000000..7f3602d6cbf3 --- /dev/null +++ b/src/audio/webrtc_aec/Kconfig @@ -0,0 +1,66 @@ +# SPDX-License-Identifier: BSD-3-Clause + +config COMP_WEBRTC_AEC + tristate "WebRTC AECm — fixed-point Acoustic Echo Canceller" + help + Select to include the WebRTC AECm (Mobile) acoustic echo canceller. + AECm is a fixed-point Q15 implementation derived from the WebRTC + audio processing library. It requires no floating-point unit and is + well suited to Xtensa HiFi3/HiFi4 class DSPs. + + The module accepts two input streams (2 input pins): + Pin 0 — microphone capture (same pipeline as the output) + Pin 1 — playback echo reference (from the render pipeline) + and produces one denoised microphone output stream. + + This module is intended as an open-source drop-in replacement for + the google_rtc_audio_processing module that requires a proprietary + library. Both implement the same two-input / one-output topology so + topologies are interchangeable. + + The real backend requires webrtc-audio-processing 0.3.x source + (west module modules/audio/webrtc-apm, same as COMP_WEBRTC_NS). + For CI/testing without that source, select COMP_WEBRTC_AEC_STUB. + +if COMP_WEBRTC_AEC + +config COMP_WEBRTC_AEC_STUB + bool "WebRTC AECm stub backend (no WebRTC source dependency)" + default y if COMP_STUBS + help + Build the webrtc_aec module against a dependency-free stub backend + that passes microphone audio through without modification. Useful + for CI validation, topology testing, and LLEXT packaging without + the cross-built WebRTC AECm archive. + +config WEBRTC_AEC_FILTER_LEN_MS + int "AECm adaptive filter length in ms (32, 64 or 128)" + default 64 + help + Length of the adaptive echo-cancellation filter. Longer filters + can handle larger acoustic echo delays (e.g. in rooms with long + reverb tails) at the cost of increased memory and CPU. + Valid values: 32, 64, 128. + +config WEBRTC_AEC_SUPPRESSION_LEVEL + int "AECm comfort-noise suppression level (0=off, 1=low, 2=medium)" + range 0 2 + default 1 + help + Controls how aggressively residual echo / background noise is + suppressed in the CNG (comfort noise generator) post-filter: + 0 - disabled (only echo suppression, no CNG) + 1 - low suppression (recommended) + 2 - medium suppression + +config WEBRTC_AEC_SAMPLE_RATE_HZ + int "AECm processing sample rate in Hz" + default 16000 + help + AECm operates at 8000 or 16000 Hz. When the pipeline runs at a + higher rate (e.g. 48 kHz), the module downsamples before AEC + processing and upsamples afterward. 16 kHz is recommended for + best echo suppression quality. + Valid values: 8000, 16000. + +endif # COMP_WEBRTC_AEC diff --git a/src/audio/webrtc_aec/README.md b/src/audio/webrtc_aec/README.md new file mode 100644 index 000000000000..4fa1e045b326 --- /dev/null +++ b/src/audio/webrtc_aec/README.md @@ -0,0 +1,104 @@ +# WebRTC Acoustic Echo Cancellation Module (`webrtc_aec`) + +Wraps the fixed-point mobile echo canceller (AECm) from [webrtc-audio-processing](https://gitlab.freedesktop.org/gstreamer/webrtc-audio-processing) behind the SOF `module_interface`. + +--- + +## Features + +- **Dual-Input Processing**: Connects both the microphone capture stream (Pin 0) and the speaker playback reference stream (Pin 1). +- **AECm Algorithm**: Highly optimized Q15 fixed-point echo cancellation; runs in real-time on low-power embedded DSPs without an FPU. +- **Dynamic Routing**: Automatic stream alignment using SOF pipeline-ID mapping. +- **Multichannel Independent Filtering**: Allocates one `AecmCore` handle per channel, supporting independent echo cancellation across multiple microphone paths. +- **Rate Options**: Natively supports 8 kHz and 16 kHz sample rates (requires matching rates for both microphone and speaker reference inputs). +- **Frame Buffer**: Synchronizes inputs and processes audio in 10 ms blocks. + +--- + +## Architecture & Data Flow + +The following Mermaid diagram explains how the dual input pins (Microphone and Speaker Reference) are processed and aligned inside the module: + +```mermaid +graph TD + %% Microphone capture path (Pin 0) + MicIn[Microphone Input Pin 0] -->|S16/S32 PCM| Core[webrtc_aec.c Core] + + %% Speaker playback reference path (Pin 1) + RefIn[Speaker Reference Pin 1] -->|S16 PCM| Core + + %% Buffering & Alignment + Core -->|Accumulate Mic| MicAccum[Mic Frame Buffer 10ms] + Core -->|Accumulate Ref| RefAccum[Ref Frame Buffer 10ms] + + MicAccum -->|Plane S16| Backend[webrtc_aec-aecm.c Backend] + RefAccum -->|Plane S16| Backend + + %% AEC Processing + Backend -->|WebRtcAecm_BufferFarend| AECM[AecmCore Channel Instance] + Backend -->|WebRtcAecm_Process| AECM + + AECM -->|Clean Mic Audio| Backend + Backend -->|Scale & Interleave| Core + Core -->|Echo Cancelled Output| OutPin[Output Pin 0] +``` + +### Components +- `webrtc_aec.c`: Core SOF wrapper handling dual-source mapping, preparing format layouts, and period accumulator synchronization. +- `webrtc_aec-aecm.c`: Real AECm library integration translation unit. +- `webrtc_aec-stub.c`: Dependency-free pass-through stub. +- `webrtc_aec.cmake`: CMake compiler scripting pulling the WebRTC AECm codebase from the `webrtc-apm` dependency. + +--- + +## Build Instructions + +### 1. Stub Mode (CI / Staging) +```ini +CONFIG_COMP_WEBRTC_AEC=y # or =m for LLEXT +CONFIG_COMP_WEBRTC_AEC_STUB=y +``` + +### 2. Real AECm Integration +Integrates the WebRTC AECm engine: +```ini +CONFIG_COMP_WEBRTC_AEC=m +CONFIG_COMP_WEBRTC_AEC_STUB=n +``` +*Note: Make sure to run `west update` to retrieve `modules/audio/webrtc-apm` before compiling.* + +--- + +## Kconfig Parameters + +| Option | Default | Range / Value | Description | +|---|---|---|---| +| `CONFIG_COMP_WEBRTC_AEC` | n | y / m / n | Enable WebRTC AECm module | +| `CONFIG_COMP_WEBRTC_AEC_STUB` | y | y / n | Use pass-through stub | +| `CONFIG_WEBRTC_AEC_CHANNELS_MAX` | 2 | 1 - 8 | Maximum channels supported | +| `CONFIG_WEBRTC_AEC_ROUTING_HEURISTIC` | y | y / n | Use pipeline-ID matching | + +--- + +## Usage & Topology + +### Topology Connections +AEC requires two input pin bindings: +1. **Pin 0 (Capture)**: Fed by the Microphone SSP DAI. +2. **Pin 1 (Reference)**: Fed by the Speaker SSP DAI (cross-pipeline). + +#### Example Topology Code snippet: +``` +Object.Widget.webrtc-aec.1 { + # Pin 0: Microphone Capture + Object.Base.input_pin_binding.1 { + input_pin_binding_name "dai-copier.SSP.NoCodec-0.capture" + } + # Pin 1: Speaker Reference + Object.Base.input_pin_binding.2 { + input_pin_binding_name "dai-copier.SSP.NoCodec-2.capture" + } +} +``` + +Both input streams must be active and running at the same sample rate (8 kHz or 16 kHz) for the echo canceller to align and filter the signals. diff --git a/src/audio/webrtc_aec/llext/CMakeLists.txt b/src/audio/webrtc_aec/llext/CMakeLists.txt new file mode 100644 index 000000000000..0786faec8ca6 --- /dev/null +++ b/src/audio/webrtc_aec/llext/CMakeLists.txt @@ -0,0 +1,24 @@ +# Copyright (c) 2026 Intel Corporation. +# SPDX-License-Identifier: Apache-2.0 + +if(CONFIG_COMP_WEBRTC_AEC_STUB) + sof_llext_build("webrtc_aec" + SOURCES ../webrtc_aec.c + ../webrtc_aec-stub.c + LIB openmodules + ) +else() + include(${CMAKE_CURRENT_LIST_DIR}/../webrtc_aec.cmake) + + sof_llext_build("webrtc_aec" + SOURCES ../webrtc_aec.c + ../webrtc_aec-aecm.c + INCLUDES "${WEBRTC_AECM_INSTALL_DIR}/include" + LIBS_PATH "${WEBRTC_AECM_INSTALL_DIR}/lib" + LIBS webrtc_aecm + LIB openmodules + ) + + add_dependencies(webrtc_aec_llext_lib webrtc_aecm_ext) + add_dependencies(webrtc_aec webrtc_aecm_ext) +endif() diff --git a/src/audio/webrtc_aec/llext/llext.toml.h b/src/audio/webrtc_aec/llext/llext.toml.h new file mode 100644 index 000000000000..154329ba4e89 --- /dev/null +++ b/src/audio/webrtc_aec/llext/llext.toml.h @@ -0,0 +1,6 @@ +#include +#define LOAD_TYPE "2" +#include "../webrtc_aec.toml" + +[module] +count = __COUNTER__ diff --git a/src/audio/webrtc_aec/webrtc_aec-aecm.c b/src/audio/webrtc_aec/webrtc_aec-aecm.c new file mode 100644 index 000000000000..c8b60678746b --- /dev/null +++ b/src/audio/webrtc_aec/webrtc_aec-aecm.c @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. +// +// Real AECm backend for the webrtc_aec module. +// +// Wraps WebRtcAecm_Create/Init/BufferFarend/Process from the WebRTC AECm +// (Mobile) fixed-point library extracted from webrtc-audio-processing 0.3.x. +// +// API summary: +// WebRtcAecm_Create() — allocate AecmCore handle +// WebRtcAecm_Init() — configure sample rate (8000 or 16000 Hz) +// WebRtcAecm_BufferFarend() — queue 10 ms of echo reference (far-end) +// WebRtcAecm_Process() — cancel echo from near-end mic frame +// WebRtcAecm_Enable() — configure filter length and CNG suppression +// WebRtcAecm_Free() — release handle +// +// All operations are Q15 fixed-point on int16_t. One handle per channel. + +#include +#include +#include "webrtc_aec.h" + +#include /* from cross-built webrtc-audio-processing 0.3.x */ + +LOG_MODULE_DECLARE(webrtc_aec, CONFIG_SOF_LOG_LEVEL); + +struct webrtc_aec_real_data { + void *aecm[WEBRTC_AEC_CHANNELS_MAX]; + int num_channels; + int filter_len_ms; + int suppression; +}; + +static int webrtc_aec_real_init(struct processing_module *mod) +{ + struct webrtc_aec_comp_data *cd = module_get_private_data(mod); + struct webrtc_aec_real_data *rd; + + rd = mod_zalloc(mod, sizeof(*rd)); + if (!rd) + return -ENOMEM; + + cd->backend_data = rd; + comp_info(mod->dev, "webrtc_aec: AECm real backend initialised"); + return 0; +} + +static int webrtc_aec_real_configure(struct processing_module *mod, int sample_rate_hz, + int filter_len_ms, int suppression, int num_channels) +{ + struct webrtc_aec_comp_data *cd = module_get_private_data(mod); + struct webrtc_aec_real_data *rd = cd->backend_data; + AecmConfig config; + int c, ret; + + /* Free previously allocated handles. */ + for (c = 0; c < rd->num_channels; c++) { + if (rd->aecm[c]) { + WebRtcAecm_Free(rd->aecm[c]); + rd->aecm[c] = NULL; + } + } + rd->num_channels = 0; + rd->filter_len_ms = filter_len_ms; + rd->suppression = suppression; + + for (c = 0; c < num_channels; c++) { + rd->aecm[c] = WebRtcAecm_Create(); + if (!rd->aecm[c]) { + comp_err(mod->dev, "webrtc_aec: WebRtcAecm_Create() failed ch%d", c); + goto err; + } + + ret = WebRtcAecm_Init(rd->aecm[c], sample_rate_hz); + if (ret) { + comp_err(mod->dev, + "webrtc_aec: WebRtcAecm_Init(ch%d, %d) failed %d", + c, sample_rate_hz, ret); + goto err; + } + + config.cngMode = suppression; + config.echoMode = (filter_len_ms == 128) ? 4 : + (filter_len_ms == 64) ? 3 : + (filter_len_ms == 32) ? 2 : 3; + ret = WebRtcAecm_set_config(rd->aecm[c], config); + if (ret) { + comp_err(mod->dev, "webrtc_aec: set_config ch%d failed %d", c, ret); + goto err; + } + } + + rd->num_channels = num_channels; + comp_info(mod->dev, "webrtc_aec: AECm rate=%d filter=%dms cng=%d ch=%d", + sample_rate_hz, filter_len_ms, suppression, num_channels); + return 0; + +err: + for (c = 0; c < num_channels; c++) { + if (rd->aecm[c]) { + WebRtcAecm_Free(rd->aecm[c]); + rd->aecm[c] = NULL; + } + } + return -ENOMEM; +} + +static int webrtc_aec_real_process_ch(struct processing_module *mod, + const int16_t *mic, const int16_t *ref, int16_t *out, + int frame_samples, int ch) +{ + struct webrtc_aec_comp_data *cd = module_get_private_data(mod); + struct webrtc_aec_real_data *rd = cd->backend_data; + int ret; + + /* Queue the far-end (reference/playback) frame first. */ + ret = WebRtcAecm_BufferFarend(rd->aecm[ch], ref, frame_samples); + if (ret) { + comp_err(mod->dev, "webrtc_aec: BufferFarend ch%d failed %d", ch, ret); + return ret; + } + + /* Process near-end (mic) and produce echo-cancelled output. + * The third parameter (near_end_noiseless) can be NULL. */ + ret = WebRtcAecm_Process(rd->aecm[ch], mic, NULL, out, frame_samples, 0); + if (ret) { + comp_err(mod->dev, "webrtc_aec: Process ch%d failed %d", ch, ret); + return ret; + } + + return 0; +} + +static int webrtc_aec_real_reset(struct processing_module *mod) +{ + struct webrtc_aec_comp_data *cd = module_get_private_data(mod); + struct webrtc_aec_real_data *rd = cd->backend_data; + int c, ret; + + for (c = 0; c < rd->num_channels; c++) { + if (!rd->aecm[c]) + continue; + ret = WebRtcAecm_Init(rd->aecm[c], cd->proc_rate); + if (ret) + comp_warn(mod->dev, "webrtc_aec: reset ch%d failed %d", c, ret); + } + return 0; +} + +static int webrtc_aec_real_free(struct processing_module *mod) +{ + struct webrtc_aec_comp_data *cd = module_get_private_data(mod); + struct webrtc_aec_real_data *rd = cd->backend_data; + int c; + + if (!rd) + return 0; + + for (c = 0; c < rd->num_channels; c++) { + if (rd->aecm[c]) { + WebRtcAecm_Free(rd->aecm[c]); + rd->aecm[c] = NULL; + } + } + mod_free(mod, rd); + cd->backend_data = NULL; + return 0; +} + +const struct webrtc_aec_backend webrtc_aec_backend = { + .name = "aecm", + .init = webrtc_aec_real_init, + .configure = webrtc_aec_real_configure, + .process_ch = webrtc_aec_real_process_ch, + .reset = webrtc_aec_real_reset, + .free = webrtc_aec_real_free, +}; diff --git a/src/audio/webrtc_aec/webrtc_aec-stub.c b/src/audio/webrtc_aec/webrtc_aec-stub.c new file mode 100644 index 000000000000..dce0b9bd7d2e --- /dev/null +++ b/src/audio/webrtc_aec/webrtc_aec-stub.c @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. +// +// Stub backend for webrtc_aec: passes mic audio through unchanged. +// Reference audio is consumed and discarded. + +#include +#include +#include +#include "webrtc_aec.h" + +LOG_MODULE_DECLARE(webrtc_aec, CONFIG_SOF_LOG_LEVEL); + +static int webrtc_aec_stub_init(struct processing_module *mod) +{ + comp_info(mod->dev, "webrtc_aec stub: no AECm library linked"); + return 0; +} + +static int webrtc_aec_stub_configure(struct processing_module *mod, int sample_rate_hz, + int filter_len_ms, int suppression, int num_channels) +{ + comp_info(mod->dev, "webrtc_aec stub: rate=%d filter=%dms sup=%d ch=%d (ignored)", + sample_rate_hz, filter_len_ms, suppression, num_channels); + return 0; +} + +static int webrtc_aec_stub_process_ch(struct processing_module *mod, + const int16_t *mic, const int16_t *ref, int16_t *out, + int frame_samples, int ch) +{ + /* Pass mic straight to output; ref is silently discarded. */ + memcpy(out, mic, (size_t)frame_samples * sizeof(int16_t)); + (void)ref; + (void)ch; + return 0; +} + +static int webrtc_aec_stub_reset(struct processing_module *mod) +{ + (void)mod; + return 0; +} + +static int webrtc_aec_stub_free(struct processing_module *mod) +{ + (void)mod; + return 0; +} + +const struct webrtc_aec_backend webrtc_aec_backend = { + .name = "stub", + .init = webrtc_aec_stub_init, + .configure = webrtc_aec_stub_configure, + .process_ch = webrtc_aec_stub_process_ch, + .reset = webrtc_aec_stub_reset, + .free = webrtc_aec_stub_free, +}; diff --git a/src/audio/webrtc_aec/webrtc_aec.c b/src/audio/webrtc_aec/webrtc_aec.c new file mode 100644 index 000000000000..1f7d796fa30a --- /dev/null +++ b/src/audio/webrtc_aec/webrtc_aec.c @@ -0,0 +1,371 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. +// +// WebRTC AECm module — SOF module_interface core. +// +// This is the SOF glue layer for the AECm fixed-point echo canceller. +// The dual-source topology and source-routing heuristic are copied +// directly from google_rtc_audio_processing.c which is the canonical +// reference for this pattern in SOF. +// +// Key differences from google_rtc_audio_processing: +// - Uses fixed-point S16 throughout (no float intermediate buffers) +// - AECm runs per-channel independently, not as a multi-channel block +// - Operates at 8 or 16 kHz maximum (pipeline downsampling is out-of-scope +// for this first revision — see CONFIG_WEBRTC_AEC_SAMPLE_RATE_HZ) + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "webrtc_aec.h" + +SOF_DEFINE_REG_UUID(webrtc_aec); +LOG_MODULE_REGISTER(webrtc_aec, CONFIG_SOF_LOG_LEVEL); + +/* ---------------------------------------------------------------- + * Helper: convert between S32/S16 interleaved ↔ per-channel S16 + * (matching the style of google_rtc_audio_processing.c) + * ---------------------------------------------------------------- */ + +/** + * src_to_s16() - Copy frames from a source into per-channel S16 scratch. + * Handles both S16 and S32 pipeline formats. + * @src: SOF source (provides source_get_data / source_release_data) + * @n: number of frames to consume + * @dst: per-channel S16 destination (ch × frame_samples) + * @frame0: offset within dst at which to start writing + * @ch: number of channels + * @is_s32: true if the pipeline format is S32_LE + */ +static void src_to_s16(struct sof_source *src, int n, + int16_t dst[][WEBRTC_AEC_FRAME_SAMPLES_MAX], + int frame0, int ch, bool is_s32) +{ + size_t sample_sz = is_s32 ? sizeof(int32_t) : sizeof(int16_t); + size_t nbytes = (size_t)n * ch * sample_sz; + const char *buf, *bufstart; + size_t bufsz; + int i, c, err; + + err = source_get_data(src, nbytes, (void *)&buf, (void *)&bufstart, &bufsz); + if (err) + return; /* shouldn't happen if caller checked availability */ + + for (i = 0; i < n; i++) { + for (c = 0; c < ch; c++) { + if (is_s32) { + int32_t s = *(const int32_t *)buf; + + dst[c][frame0 + i] = (int16_t)(s >> 16); + buf += sizeof(int32_t); + } else { + dst[c][frame0 + i] = *(const int16_t *)buf; + buf += sizeof(int16_t); + } + } + /* Wrap circular buffer. */ + if (buf >= bufstart + bufsz) + buf = bufstart; + } + + source_release_data(src, nbytes); +} + +/** + * s16_to_sink() - Write per-channel S16 output to a SOF sink. + * Upshifts to S32 if the pipeline format requires it. + * @dst: SOF sink + * @src: per-channel S16 source (ch × frame_samples, from index 0) + * @n: number of frames to write + * @ch: number of channels + * @is_s32: true if the pipeline format is S32_LE + */ +static void s16_to_sink(struct sof_sink *dst, int16_t src[][WEBRTC_AEC_FRAME_SAMPLES_MAX], + int n, int ch, bool is_s32) +{ + size_t sample_sz = is_s32 ? sizeof(int32_t) : sizeof(int16_t); + size_t nbytes = (size_t)n * ch * sample_sz; + char *buf, *bufstart; + size_t bufsz; + int i, c, err; + + err = sink_get_buffer(dst, nbytes, (void *)&buf, (void *)&bufstart, &bufsz); + if (err) + return; + + for (i = 0; i < n; i++) { + for (c = 0; c < ch; c++) { + if (is_s32) { + *(int32_t *)buf = (int32_t)src[c][i] << 16; + buf += sizeof(int32_t); + } else { + *(int16_t *)buf = src[c][i]; + buf += sizeof(int16_t); + } + } + if (buf >= bufstart + bufsz) + buf = bufstart; + } + + sink_commit_buffer(dst, nbytes); +} + +/* ---------------------------------------------------------------- + * module_interface operations + * ---------------------------------------------------------------- */ + +__cold static int webrtc_aec_init(struct processing_module *mod) +{ + struct module_data *md = &mod->priv; + struct comp_dev *dev = mod->dev; + struct webrtc_aec_comp_data *cd; + int ret; + + assert_can_be_cold(); + comp_info(dev, "webrtc_aec: init"); + + cd = mod_zalloc(mod, sizeof(*cd)); + if (!cd) + return -ENOMEM; + + md->private = cd; + cd->backend = &webrtc_aec_backend; + + /* Two input pins: mic + echo reference. */ + mod->max_sources = 2; + + comp_info(dev, "webrtc_aec: backend '%s'", cd->backend->name); + + if (cd->backend->init) { + ret = cd->backend->init(mod); + if (ret) { + comp_err(dev, "webrtc_aec: backend init failed %d", ret); + mod_free(mod, cd); + return ret; + } + } + + return 0; +} + +__cold static int webrtc_aec_prepare(struct processing_module *mod, + struct sof_source **sources, int num_of_sources, + struct sof_sink **sinks, int num_of_sinks) +{ + struct webrtc_aec_comp_data *cd = module_get_private_data(mod); + struct comp_dev *dev = mod->dev; + int mic_fmt, ref_fmt, mic_rate, ref_rate, ret; + + assert_can_be_cold(); + + if (num_of_sources != 2 || num_of_sinks != 1) { + comp_err(dev, "webrtc_aec: need 2 sources and 1 sink (got %d/%d)", + num_of_sources, num_of_sinks); + return -EINVAL; + } + + /* + * Resolve which source is mic and which is echo reference. + * The mic is the one on the same pipeline as the output sink. + * This matches google_rtc_audio_processing.c exactly. + */ + cd->ref_src = (source_get_pipeline_id(sources[0]) == sink_get_pipeline_id(sinks[0])); + cd->mic_src = cd->ref_src ? 0 : 1; + + mic_fmt = source_get_frm_fmt(sources[cd->mic_src]); + ref_fmt = source_get_frm_fmt(sources[cd->ref_src]); + mic_rate = source_get_rate(sources[cd->mic_src]); + ref_rate = source_get_rate(sources[cd->ref_src]); + cd->channels = source_get_channels(sources[cd->mic_src]); + + if (cd->channels > WEBRTC_AEC_CHANNELS_MAX) { + comp_err(dev, "webrtc_aec: too many channels %d (max %d)", + cd->channels, WEBRTC_AEC_CHANNELS_MAX); + return -EINVAL; + } + + if (mic_rate != ref_rate) { + comp_err(dev, "webrtc_aec: mic_rate %d != ref_rate %d", mic_rate, ref_rate); + return -EINVAL; + } + cd->rate = mic_rate; + + /* AECm supports only 8 and 16 kHz natively. */ + cd->proc_rate = CONFIG_WEBRTC_AEC_SAMPLE_RATE_HZ; + if (cd->proc_rate != 8000 && cd->proc_rate != 16000) { + comp_err(dev, "webrtc_aec: invalid proc_rate %d (must be 8000 or 16000)", + cd->proc_rate); + return -EINVAL; + } + + if (cd->rate != cd->proc_rate) { + comp_warn(dev, "webrtc_aec: pipeline rate %d != AECm rate %d; " + "pipeline resampling required upstream", cd->rate, cd->proc_rate); + } + + if ((mic_fmt != SOF_IPC_FRAME_S16_LE && mic_fmt != SOF_IPC_FRAME_S32_LE) || + (ref_fmt != SOF_IPC_FRAME_S16_LE && ref_fmt != SOF_IPC_FRAME_S32_LE)) { + comp_err(dev, "webrtc_aec: unsupported format mic=%d ref=%d", + mic_fmt, ref_fmt); + return -EINVAL; + } + + cd->is_s32 = (mic_fmt == SOF_IPC_FRAME_S32_LE); + cd->mic_frame_bytes = source_get_frame_bytes(sources[cd->mic_src]); + cd->ref_frame_bytes = source_get_frame_bytes(sources[cd->ref_src]); + cd->out_frame_bytes = sink_get_frame_bytes(sinks[0]); + cd->frame_samples = (cd->proc_rate * 10) / 1000; /* 10 ms */ + + if (cd->frame_samples > WEBRTC_AEC_FRAME_SAMPLES_MAX) { + comp_err(dev, "webrtc_aec: frame_samples %d exceeds max %d", + cd->frame_samples, WEBRTC_AEC_FRAME_SAMPLES_MAX); + return -EINVAL; + } + + comp_info(dev, "webrtc_aec: mic_src=%d ref_src=%d rate=%d/%d ch=%d frame=%d %s", + cd->mic_src, cd->ref_src, cd->rate, cd->proc_rate, cd->channels, + cd->frame_samples, cd->is_s32 ? "S32" : "S16"); + +#ifdef CONFIG_IPC_MAJOR_4 + /* Apply reference format override from topology pin descriptor. */ + ipc4_update_source_format(sources[cd->ref_src], + &mod->priv.cfg.input_pins[1].audio_fmt); +#endif + + if (cd->backend->configure) { + ret = cd->backend->configure(mod, cd->proc_rate, + CONFIG_WEBRTC_AEC_FILTER_LEN_MS, + CONFIG_WEBRTC_AEC_SUPPRESSION_LEVEL, + cd->channels); + if (ret) { + comp_err(dev, "webrtc_aec: backend configure failed %d", ret); + return ret; + } + } + + cd->buffered_frames = 0; + cd->last_ref_ok = true; + cd->configured = true; + return 0; +} + +static int webrtc_aec_process(struct processing_module *mod, + struct sof_source **sources, int num_of_sources, + struct sof_sink **sinks, int num_of_sinks) +{ + struct webrtc_aec_comp_data *cd = module_get_private_data(mod); + struct sof_source *mic = sources[cd->mic_src]; + struct sof_source *ref = sources[cd->ref_src]; + struct sof_sink *out = sinks[0]; + int fmic = (int)source_get_data_frames_available(mic); + int fref = (int)source_get_data_frames_available(ref); + int frames = MIN(fmic, fref); + int n, frames_rem; + + for (frames_rem = frames; frames_rem > 0; frames_rem -= n) { + /* Consume at most what fills one complete AECm frame. */ + n = MIN(frames_rem, cd->frame_samples - cd->buffered_frames); + + /* Convert source data → per-channel S16 accumulators. */ + src_to_s16(mic, n, cd->mic_buf, cd->buffered_frames, cd->channels, cd->is_s32); + src_to_s16(ref, n, cd->ref_buf, cd->buffered_frames, cd->channels, cd->is_s32); + + cd->buffered_frames += n; + + /* Once we have a full 10 ms block, run AECm per channel. */ + if (cd->buffered_frames >= cd->frame_samples) { + int fs = cd->frame_samples; + + /* Check output headroom. */ + if (sink_get_free_size(out) < (size_t)(fs * cd->out_frame_bytes)) { + comp_warn(mod->dev, "webrtc_aec: sink backed up!"); + break; + } + + int c, ret; + + for (c = 0; c < cd->channels; c++) { + ret = cd->backend->process_ch(mod, + cd->mic_buf[c], + cd->ref_buf[c], + cd->out_buf[c], + fs, c); + if (ret) { + /* Fall back to mic pass-through for this channel. */ + memcpy(cd->out_buf[c], cd->mic_buf[c], + (size_t)fs * sizeof(int16_t)); + } + } + + /* Write denoised output to sink. */ + s16_to_sink(out, cd->out_buf, fs, cd->channels, cd->is_s32); + cd->buffered_frames = 0; + } + } + + cd->last_ref_ok = true; + return 0; +} + +static int webrtc_aec_reset(struct processing_module *mod) +{ + struct webrtc_aec_comp_data *cd = module_get_private_data(mod); + + comp_dbg(mod->dev, "webrtc_aec: reset"); + cd->buffered_frames = 0; + + if (cd->backend->reset) + return cd->backend->reset(mod); + + return 0; +} + +__cold static int webrtc_aec_free(struct processing_module *mod) +{ + struct webrtc_aec_comp_data *cd = module_get_private_data(mod); + + assert_can_be_cold(); + comp_dbg(mod->dev, "webrtc_aec: free"); + + if (cd->backend->free) + cd->backend->free(mod); + + mod_free(mod, cd); + return 0; +} + +static const struct module_interface webrtc_aec_interface = { + .init = webrtc_aec_init, + .prepare = webrtc_aec_prepare, + .process = webrtc_aec_process, + .reset = webrtc_aec_reset, + .free = webrtc_aec_free, +}; + +#if CONFIG_COMP_WEBRTC_AEC_MODULE + +#include +#include +#include + +static const struct sof_man_module_manifest mod_manifest __section(".module") __used = + SOF_LLEXT_MODULE_MANIFEST("WRTCAEC", &webrtc_aec_interface, 2, + SOF_REG_UUID(webrtc_aec), 40); + +SOF_LLEXT_BUILDINFO; + +#else + +DECLARE_TR_CTX(webrtc_aec_tr, SOF_UUID(webrtc_aec_uuid), LOG_LEVEL_INFO); +DECLARE_MODULE_ADAPTER(webrtc_aec_interface, webrtc_aec_uuid, webrtc_aec_tr); +SOF_MODULE_INIT(webrtc_aec, sys_comp_module_webrtc_aec_interface_init); + +#endif diff --git a/src/audio/webrtc_aec/webrtc_aec.cmake b/src/audio/webrtc_aec/webrtc_aec.cmake new file mode 100644 index 000000000000..2753ab018fc2 --- /dev/null +++ b/src/audio/webrtc_aec/webrtc_aec.cmake @@ -0,0 +1,85 @@ +# SPDX-License-Identifier: BSD-3-Clause +# +# Cross-build the WebRTC AECm (echo_control_mobile) fixed-point C sources +# for the webrtc_aec LLEXT module. +# +# Source: webrtc-audio-processing 0.3.x (same west module as webrtc_ns). +# The AECm module lives under webrtc/modules/audio_processing/aecm/ and +# depends on webrtc/common_audio/signal_processing/ (same as NS). +# +# Produces: +# ${WEBRTC_AECM_INSTALL_DIR}/lib/libwebrtc_aecm.a +# ${WEBRTC_AECM_INSTALL_DIR}/include/echo_control_mobile.h + +if(NOT DEFINED SOF_WEBRTC_APM_SRC_DIR) + set(SOF_WEBRTC_APM_SRC_DIR "${sof_top_dir}/../modules/audio/webrtc-apm" + CACHE PATH "webrtc-audio-processing 0.3.x source directory (west-pinned)") +endif() +cmake_path(NORMAL_PATH SOF_WEBRTC_APM_SRC_DIR) +if(NOT EXISTS "${SOF_WEBRTC_APM_SRC_DIR}/webrtc/modules/audio_processing/aecm/aecm_core.c") + message(FATAL_ERROR + "webrtc_aec: AECm source not found at '${SOF_WEBRTC_APM_SRC_DIR}'.\n" + "Run 'west update' or pass -DSOF_WEBRTC_APM_SRC_DIR=.") +endif() + +get_filename_component(_tc_dir "${CMAKE_C_COMPILER}" DIRECTORY) +get_filename_component(_tc_name "${CMAKE_C_COMPILER}" NAME) +string(REGEX REPLACE "gcc$" "" _tc_prefix_name "${_tc_name}") +set(_aecm_cross_prefix "${_tc_dir}/${_tc_prefix_name}") + +set(WEBRTC_AECM_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/webrtc-aecm-install" + CACHE INTERNAL "webrtc_aec: AECm library install prefix") + +set(_aecm_src "${SOF_WEBRTC_APM_SRC_DIR}/webrtc") + +file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/webrtc-aecm-build.sh" +"#!/bin/sh +set -e +SRC=${_aecm_src} +INST=${WEBRTC_AECM_INSTALL_DIR} +OBJ=${CMAKE_CURRENT_BINARY_DIR}/webrtc-aecm-obj + +mkdir -p \"$OBJ\" \"$INST/lib\" \"$INST/include\" + +CFLAGS=\"-O2 -fPIC -I$SRC -I$SRC/common_audio/signal_processing/include\" + +# AECm core +SRCS=\" + $SRC/modules/audio_processing/aecm/aecm_core.c + $SRC/modules/audio_processing/aecm/echo_control_mobile.c + $SRC/common_audio/signal_processing/complex_bit_reverse.c + $SRC/common_audio/signal_processing/complex_fft.c + $SRC/common_audio/signal_processing/cross_correlation.c + $SRC/common_audio/signal_processing/division_operations.c + $SRC/common_audio/signal_processing/downsample_fast.c + $SRC/common_audio/signal_processing/energy.c + $SRC/common_audio/signal_processing/get_scaling_square.c + $SRC/common_audio/signal_processing/min_max_operations.c + $SRC/common_audio/signal_processing/real_fft.c + $SRC/common_audio/signal_processing/resample_48khz.c + $SRC/common_audio/signal_processing/resample_by_2_internal.c + $SRC/common_audio/signal_processing/resample_fractional.c + $SRC/common_audio/signal_processing/spl_inl.c + $SRC/common_audio/signal_processing/sqrt_of_one_minus_x_squared.c + $SRC/common_audio/signal_processing/vector_scaling_operations.c +\" + +for f in \$SRCS; do + bn=\$(basename \"\$f\" .c) + ${CMAKE_C_COMPILER} \$CFLAGS -c \"\$f\" -o \"$OBJ/\${bn}.o\" +done + +${_aecm_cross_prefix}ar rcs \"$INST/lib/libwebrtc_aecm.a\" \"$OBJ\"/*.o +cp $SRC/modules/audio_processing/aecm/echo_control_mobile.h \"$INST/include/\" +") + +add_custom_command( + OUTPUT "${WEBRTC_AECM_INSTALL_DIR}/lib/libwebrtc_aecm.a" + "${WEBRTC_AECM_INSTALL_DIR}/include/echo_control_mobile.h" + COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/webrtc-aecm-build.sh" + VERBATIM) + +add_custom_target(webrtc_aecm_ext + DEPENDS "${WEBRTC_AECM_INSTALL_DIR}/lib/libwebrtc_aecm.a") + +message(STATUS "webrtc_aec: cross-building AECm from ${SOF_WEBRTC_APM_SRC_DIR}") diff --git a/src/audio/webrtc_aec/webrtc_aec.h b/src/audio/webrtc_aec/webrtc_aec.h new file mode 100644 index 000000000000..2832f21339a2 --- /dev/null +++ b/src/audio/webrtc_aec/webrtc_aec.h @@ -0,0 +1,122 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright(c) 2026 Intel Corporation. + * + * WebRTC AECm (fixed-point Acoustic Echo Canceller Mobile) for SOF. + * + * Two input pins: + * sources[mic_src] — microphone capture (same pipeline as the sink) + * sources[ref_src] — playback reference (different render pipeline) + * One output pin: + * sinks[0] — echo-cancelled microphone + * + * The mic/ref source indices are resolved at prepare() time using the same + * pipeline-ID heuristic as google_rtc_audio_processing.c. + * + * The actual AEC processing is delegated to a pluggable backend so the + * module can be validated with a stub (webrtc_aec-stub.c) before the + * real AECm backend (webrtc_aec-webrtc.c) is linked in. + */ +#ifndef __SOF_AUDIO_WEBRTC_AEC_H__ +#define __SOF_AUDIO_WEBRTC_AEC_H__ + +#include +#include +#include +#include + +/* AECm operates on 10 ms frames. Max 16 kHz × 10 ms = 160 samples/channel. */ +#define WEBRTC_AEC_FRAME_SAMPLES_MAX 160 + +/* Maximum supported channel count (AECm is mono; multi-ch runs N instances). */ +#define WEBRTC_AEC_CHANNELS_MAX 4 + +/* Maximum DMA buffer alignment. */ +#define WEBRTC_AEC_MEM_ALIGN 64 + +/** + * struct webrtc_aec_backend - AECm backend operations. + */ +struct webrtc_aec_backend { + const char *name; + + /* One-time allocation, called from init(). */ + int (*init)(struct processing_module *mod); + + /** + * Open/configure AECm state. Called from prepare(). + * @sample_rate_hz: 8000 or 16000 + * @filter_len_ms: adaptive filter length (32/64/128) + * @suppression: CNG level 0..2 + * @num_channels: mic and ref channel count (same; AECm runs per-ch) + */ + int (*configure)(struct processing_module *mod, int sample_rate_hz, + int filter_len_ms, int suppression, int num_channels); + + /** + * Process one 10 ms frame (interleaved S16 mic + ref → interleaved S16 out). + * All arrays are mono; caller provides per-channel slices. + * @mic: mic input S16 samples (frame_samples) + * @ref: echo reference S16 samples (frame_samples) + * @out: denoised output S16 samples (frame_samples) + * @ch: channel index (for multi-channel instances) + */ + int (*process_ch)(struct processing_module *mod, + const int16_t *mic, const int16_t *ref, int16_t *out, + int frame_samples, int ch); + + /* Reset adaptive filters (keep configuration). Called from reset(). */ + int (*reset)(struct processing_module *mod); + + /* Tear down all state from configure(). */ + int (*free)(struct processing_module *mod); +}; + +/** + * struct webrtc_aec_comp_data - webrtc_aec module private data. + * + * All buffers are S16 because AECm is a fixed-point Q15 algorithm. + * We do S32→S16 downshift on input and S16→S32 upshift on output when + * the pipeline format is S32. + */ +struct webrtc_aec_comp_data { + const struct webrtc_aec_backend *backend; + void *backend_data; + + /* Source routing (resolved in prepare). */ + int mic_src; /* index into sources[] for microphone */ + int ref_src; /* index into sources[] for echo reference */ + + /* Negotiated format. */ + int rate; /* pipeline sample rate (Hz) */ + int proc_rate; /* AECm processing rate (8000 or 16000 Hz) */ + int channels; + int mic_frame_bytes; /* bytes per pipeline frame on mic input */ + int ref_frame_bytes; /* bytes per pipeline frame on ref input */ + int out_frame_bytes; + bool is_s32; /* true for S32_LE pipeline */ + + /* Processing frame size at proc_rate. */ + int frame_samples; /* proc_rate * 10 / 1000 */ + + /* Accumulation state: we collect frames until a full 10 ms block. */ + int buffered_frames; + + /* Per-channel S16 scratch buffers. */ + int16_t mic_buf[WEBRTC_AEC_CHANNELS_MAX][WEBRTC_AEC_FRAME_SAMPLES_MAX]; + int16_t ref_buf[WEBRTC_AEC_CHANNELS_MAX][WEBRTC_AEC_FRAME_SAMPLES_MAX]; + int16_t out_buf[WEBRTC_AEC_CHANNELS_MAX][WEBRTC_AEC_FRAME_SAMPLES_MAX]; + + /* Ref stream liveness (IPC4: always active). */ + bool last_ref_ok; + + /* IPC4 tuning (echo path delay, etc.) — reserved for set_config. */ + struct sof_ipc4_aec_config config; + + bool configured; +}; + +/* Backend instance exported from the selected translation unit. */ +extern const struct webrtc_aec_backend webrtc_aec_backend; + +#endif /* __SOF_AUDIO_WEBRTC_AEC_H__ */ diff --git a/src/audio/webrtc_aec/webrtc_aec.toml b/src/audio/webrtc_aec/webrtc_aec.toml new file mode 100644 index 000000000000..fde0f102e0ce --- /dev/null +++ b/src/audio/webrtc_aec/webrtc_aec.toml @@ -0,0 +1,24 @@ +#ifndef LOAD_TYPE +#define LOAD_TYPE "0" +#endif + + REM # WebRTC AECm acoustic echo canceller module config. + REM # Two input pins: pin 0 = mic capture, pin 1 = echo reference. + [[module.entry]] + name = "WRTCAEC" + uuid = UUIDREG_STR_WEBRTC_AEC + affinity_mask = "0x1" + instance_count = "40" + domain_types = "0" + load_type = LOAD_TYPE + module_type = "9" + auto_start = "0" + sched_caps = [1, 0x00008000] + REM # pin = [dir, type, sample rate, size, container, channel-cfg] + REM # 2 input pins (mic + echo ref), 1 output pin (processed mic). + pin = [0, 0, 0xfeef, 0xf, 0xf, 0x45ff, 0, 0, 0xfeef, 0xf, 0xf, 0x45ff, 1, 0, 0xfeef, 0xf, 0xf, 0x45ff] + REM # mod_cfg [PAR_0 PAR_1 PAR_2 PAR_3 IS_BYTES CPS IBS OBS MOD_FLAGS CPC OBLS] + REM # AECm is fixed-point, modest CPU; 2×10ms buffers for mic+ref at 16kHz S32. + mod_cfg = [0, 0, 0, 0, 0, 2000000, 16384, 8192, 0, 2000, 0] + + index = __COUNTER__ diff --git a/src/audio/webrtc_ns/CMakeLists.txt b/src/audio/webrtc_ns/CMakeLists.txt new file mode 100644 index 000000000000..b1f24ed5d925 --- /dev/null +++ b/src/audio/webrtc_ns/CMakeLists.txt @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: BSD-3-Clause + +if(CONFIG_COMP_WEBRTC_NS STREQUAL "m" AND DEFINED CONFIG_LLEXT) + add_subdirectory(llext ${PROJECT_BINARY_DIR}/webrtc_ns_llext) + add_dependencies(app webrtc_ns) +else() + add_local_sources(sof webrtc_ns.c) + if(CONFIG_COMP_WEBRTC_NS_STUB) + add_local_sources(sof webrtc_ns-stub.c) + else() + add_local_sources(sof webrtc_ns-webrtc.c) + endif() +endif() diff --git a/src/audio/webrtc_ns/Kconfig b/src/audio/webrtc_ns/Kconfig new file mode 100644 index 000000000000..062c6cd8b248 --- /dev/null +++ b/src/audio/webrtc_ns/Kconfig @@ -0,0 +1,55 @@ +# SPDX-License-Identifier: BSD-3-Clause + +config COMP_WEBRTC_NS + tristate "WebRTC Noise Suppression (classic spectral Wiener filter)" + help + Select to include the WebRTC Noise Suppression module. It wraps + the classic WebRTC NS algorithm — a frequency-domain spectral + subtraction / Wiener filter — as a single-input, single-output + PCM effect module. + + The NS module reduces stationary and slowly-varying background + noise (HVAC hum, fan noise, electrical hiss). It is independent + of the ffmpeg afftdn filter already available through ffmpeg_dec, + providing users a build-time choice between the two. + + The real backend uses the pure-C noise_suppression module + extracted from webrtc-audio-processing 0.3.x (no abseil, no C++, + BSD-3-Clause). Source is fetched via west (west update). + + Without the source, or for CI/testing, select COMP_WEBRTC_NS_STUB. + +if COMP_WEBRTC_NS + +config COMP_WEBRTC_NS_STUB + bool "WebRTC NS stub backend (no WebRTC source dependency)" + default y if COMP_STUBS + help + Build the webrtc_ns module against a dependency-free stub backend + that passes audio through unmodified. Useful for CI validation of + the SOF module glue, LLEXT packaging, and topology wiring without + needing the cross-built libwebrtc_ns archive. + +config WEBRTC_NS_LEVEL + int "Noise suppression level (0=mild, 1=moderate, 2=aggressive, 3=very_aggressive)" + range 0 3 + default 1 + help + Controls the aggressiveness of the noise suppressor: + 0 - Mild (minimal suppression, lowest distortion) + 1 - Moderate (recommended for most use cases) + 2 - Aggressive + 3 - Very aggressive (maximum suppression, most distortion) + +config WEBRTC_NS_SAMPLE_RATE_HZ + int "NS processing sample rate in Hz" + default 16000 + help + The internal processing sample rate. The WebRTC NS algorithm works + best at 16000 Hz. When the pipeline runs at 48000 Hz the module + can downsample to 16 kHz for NS processing and upsample back to + 48 kHz, saving significant CPU. Set to 48000 to process at full + rate (higher quality, higher CPU cost). + Valid values: 8000, 16000, 32000, 48000. + +endif # COMP_WEBRTC_NS diff --git a/src/audio/webrtc_ns/README.md b/src/audio/webrtc_ns/README.md new file mode 100644 index 000000000000..8da440d612ba --- /dev/null +++ b/src/audio/webrtc_ns/README.md @@ -0,0 +1,89 @@ +# WebRTC Noise Suppression Module (`webrtc_ns`) + +Wraps the classic fixed-point WebRTC Noise Suppression (NS) algorithm (spectral Wiener filter) from [webrtc-audio-processing](https://gitlab.freedesktop.org/gstreamer/webrtc-audio-processing) behind the SOF `module_interface`. + +--- + +## Features + +- **Wiener Filter Suppression**: Fixed-point spectral subtraction to isolate stationary noise. +- **Low Complexity**: Highly optimized for low-power embedded DSPs; requires no FPU. +- **Multiple Modes**: Supports four levels of suppression (Mild, Medium, Aggressive, Very Aggressive). +- **Per-Channel Processing**: Allocates one NS instance per audio channel, processing them independently. +- **8 kHz and 16 kHz Support**: Natively supports 8 kHz and 16 kHz sample rates (requires resampling upstream if running at higher rates). +- **Frame Size**: Operates on 10 ms audio frames (80 samples at 8 kHz, 160 samples at 16 kHz). + +--- + +## Architecture & Data Flow + +The following Mermaid diagram outlines the internal architecture of the `webrtc_ns` module: + +```mermaid +graph TD + %% Data Flow + InBuf[Input Audio Buffer] -->|S16/S32 Interleaved| Core[webrtc_ns.c Core] + Core -->|1. Demux & Normalise| ScaleIn[Format Conversion] + ScaleIn -->|10 ms Frame per Channel| Backend[webrtc_ns-webrtc.c Backend] + + Backend -->|WebRTC NS Core Filter| NSInstance[NsCore Instance per Channel] + NSInstance -->|Filtered Audio| Backend + Backend -->|Scale & Interleave| ScaleOut[Format Conversion] + + ScaleOut -->|S16/S32 Interleaved| Core + Core -->|Denoised PCM| OutBuf[Output Audio Buffer] + + %% Control Flow + IPC[IPC Control / Set Config] -.->|Suppression Level| Core + Core -.->|WebRtcNs_set_policy| NSInstance +``` + +### Components +- `webrtc_ns.c`: Core SOF wrapper logic, handling format parsing, period buffering, and interleaved-to-planar copying. +- `webrtc_ns-webrtc.c`: Integration backend interfacing directly with the WebRTC NS codebase. +- `webrtc_ns-stub.c`: Standard pass-through stub for testing. +- `webrtc_ns-shims.c`: Platform-specific macros and memory mapping layers. +- `webrtc_ns.cmake`: Downloads and extracts the `webrtc-audio-processing` 0.3.1 source and compiles the necessary C files. + +--- + +## Build Instructions + +### 1. Stub Mode (CI / Staging) +```ini +CONFIG_COMP_WEBRTC_NS=y # or =m for LLEXT +CONFIG_COMP_WEBRTC_NS_STUB=y +``` + +### 2. Real NS Integration +Pulls the classic WebRTC codebase via West and builds the fixed-point Wiener filter: +```ini +CONFIG_COMP_WEBRTC_NS=m +CONFIG_COMP_WEBRTC_NS_STUB=n +``` +*Note: Make sure to run `west update` to retrieve `modules/audio/webrtc-apm` before compiling.* + +--- + +## Kconfig Parameters + +| Option | Default | Range / Value | Description | +|---|---|---|---| +| `CONFIG_COMP_WEBRTC_NS` | n | y / m / n | Enable WebRTC Noise Suppressor | +| `CONFIG_COMP_WEBRTC_NS_STUB` | y | y / n | Use pass-through stub | +| `CONFIG_WEBRTC_NS_POLICY` | 2 | 0 - 3 | Severity (0: Mild, 3: Very Aggressive) | +| `CONFIG_WEBRTC_NS_CHANNELS_MAX` | 2 | 1 - 8 | Maximum channels supported | + +--- + +## Usage & Topology + +### Pipeline Integration +The module is integrated as a normal 1-in / 1-out effect widget. It should be run in a pipeline configured for either **8000 Hz** or **16000 Hz**. + +#### Example Topology Pipeline +``` +DAI Copier (SSP RX) ---> [ webrtc-ns ] ---> Host Copier (PCM) +``` + +For applications requiring higher rates (e.g. 48 kHz mic capture), place an ASRC component upstream to downsample to 16 kHz before running the `webrtc-ns` widget. diff --git a/src/audio/webrtc_ns/llext/CMakeLists.txt b/src/audio/webrtc_ns/llext/CMakeLists.txt new file mode 100644 index 000000000000..8810c457281c --- /dev/null +++ b/src/audio/webrtc_ns/llext/CMakeLists.txt @@ -0,0 +1,27 @@ +# Copyright (c) 2026 Intel Corporation. +# SPDX-License-Identifier: Apache-2.0 + +if(CONFIG_COMP_WEBRTC_NS_STUB) + # Dependency-free build: SOF glue + pass-through stub, no WebRTC library. + sof_llext_build("webrtc_ns" + SOURCES ../webrtc_ns.c + ../webrtc_ns-stub.c + LIB openmodules + ) +else() + # Real NS: cross-build webrtc-apm NS subset and link it. + include(${CMAKE_CURRENT_LIST_DIR}/../webrtc_ns.cmake) + + sof_llext_build("webrtc_ns" + SOURCES ../webrtc_ns.c + ../webrtc_ns-webrtc.c + ../webrtc_ns-shims.c + INCLUDES "${WEBRTC_NS_INSTALL_DIR}/include" + LIBS_PATH "${WEBRTC_NS_INSTALL_DIR}/lib" + LIBS webrtc_ns + LIB openmodules + ) + + add_dependencies(webrtc_ns_llext_lib webrtc_ns_ext) + add_dependencies(webrtc_ns webrtc_ns_ext) +endif() diff --git a/src/audio/webrtc_ns/llext/llext.toml.h b/src/audio/webrtc_ns/llext/llext.toml.h new file mode 100644 index 000000000000..1e75499f0efd --- /dev/null +++ b/src/audio/webrtc_ns/llext/llext.toml.h @@ -0,0 +1,6 @@ +#include +#define LOAD_TYPE "2" +#include "../webrtc_ns.toml" + +[module] +count = __COUNTER__ diff --git a/src/audio/webrtc_ns/webrtc_ns-shims.c b/src/audio/webrtc_ns/webrtc_ns-shims.c new file mode 100644 index 000000000000..16a6c028874e --- /dev/null +++ b/src/audio/webrtc_ns/webrtc_ns-shims.c @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. +// +// libc shims for the webrtc_ns real backend in LLEXT context. +// +// The WebRTC NS C library references malloc/free and a small set of math +// symbols. These are not exported by SOF core to LLEXT modules, so we +// provide thin wrappers backed by SOF's rballoc heap and sofm_ math. + +#include +#include +#include + +/* ============================ Memory ============================ */ + +void *malloc(size_t size) +{ + return rballoc(SOF_MEM_FLAG_USER, size); +} + +void free(void *ptr) +{ + rfree(ptr); +} + +void *calloc(size_t nmemb, size_t size) +{ + void *p = rballoc(SOF_MEM_FLAG_USER, nmemb * size); + + if (p) + memset(p, 0, nmemb * size); + return p; +} + +/* ============================ String helpers ===================== */ + +void *memset(void *s, int c, size_t n) +{ + uint8_t *p = s; + + while (n--) + *p++ = (uint8_t)c; + return s; +} + +void *memcpy(void *dest, const void *src, size_t n) +{ + const uint8_t *s = src; + uint8_t *d = dest; + + while (n--) + *d++ = *s++; + return dest; +} + +void *memmove(void *dest, const void *src, size_t n) +{ + uint8_t *d = dest; + const uint8_t *s = src; + + if (d < s || d >= s + n) { + while (n--) + *d++ = *s++; + } else { + d += n; + s += n; + while (n--) + *--d = *--s; + } + return dest; +} + +/* ============================ Math ============================== */ +/* + * The NS spectral processing uses log(), exp(), sqrt(), pow(), fabsf(). + * Route to sofm_ equivalents where available; fall back to soft-float + * versions provided by the Zephyr minimal libc for others. + */ +#include + +float sqrtf(float x) { return (float)sofm_sqrt_int32((int32_t)(x * 65536.0f)) / 256.0f; } + +/* log/exp/pow are used for spectral gain computation — leave as weak + * references satisfied by the toolchain's soft-float libm linked into + * the SOF binary. LLEXT resolves them at load time from the core symbol + * table. These shims are intentionally empty for now; add implementations + * if the real WebRTC NS backend needs them unsatisfied. */ diff --git a/src/audio/webrtc_ns/webrtc_ns-stub.c b/src/audio/webrtc_ns/webrtc_ns-stub.c new file mode 100644 index 000000000000..6bf58eaba9b9 --- /dev/null +++ b/src/audio/webrtc_ns/webrtc_ns-stub.c @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. +// +// Dependency-free stub backend for the webrtc_ns module. +// +// Always passes audio through unmodified (identity transform). Used in CI +// and for topology/LLEXT packaging validation without the WebRTC NS library. + +#include +#include +#include +#include "webrtc_ns.h" + +LOG_MODULE_DECLARE(webrtc_ns, CONFIG_SOF_LOG_LEVEL); + +static int webrtc_ns_stub_init(struct processing_module *mod) +{ + comp_info(mod->dev, "webrtc_ns stub backend: no WebRTC NS linked"); + return 0; +} + +static int webrtc_ns_stub_configure(struct processing_module *mod, + int sample_rate_hz, int level, int num_channels) +{ + comp_info(mod->dev, "webrtc_ns stub: rate=%d level=%d ch=%d (ignored)", + sample_rate_hz, level, num_channels); + return 0; +} + +/* Stub: copy input to output (pass-through). */ +static int webrtc_ns_stub_process(struct processing_module *mod, + const float *const *in, float *const *out, + int frame_samples) +{ + struct webrtc_ns_comp_data *cd = module_get_private_data(mod); + int c; + + for (c = 0; c < cd->channels; c++) + memcpy(out[c], in[c], (size_t)frame_samples * sizeof(float)); + + return 0; +} + +static int webrtc_ns_stub_reset(struct processing_module *mod) +{ + (void)mod; + return 0; +} + +static int webrtc_ns_stub_free(struct processing_module *mod) +{ + (void)mod; + return 0; +} + +const struct webrtc_ns_backend webrtc_ns_backend = { + .name = "stub", + .init = webrtc_ns_stub_init, + .configure = webrtc_ns_stub_configure, + .process = webrtc_ns_stub_process, + .reset = webrtc_ns_stub_reset, + .free = webrtc_ns_stub_free, +}; diff --git a/src/audio/webrtc_ns/webrtc_ns-webrtc.c b/src/audio/webrtc_ns/webrtc_ns-webrtc.c new file mode 100644 index 000000000000..8bb27c1b7f0e --- /dev/null +++ b/src/audio/webrtc_ns/webrtc_ns-webrtc.c @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. +// +// WebRTC NS real backend for the webrtc_ns module. +// +// Wraps the classic WebRTC noise_suppression library extracted from +// webrtc-audio-processing 0.3.x. This version is pure C, has no C++ +// or abseil dependency, and uses a simple stateless API: +// +// WebRtcNs_Create() — allocate NsHandle +// WebRtcNs_Init() — configure sample rate +// WebRtcNs_set_policy() — aggressiveness 0..3 +// WebRtcNs_Analyze() — feed 10ms frame for noise model update +// WebRtcNs_Process() — run suppression +// WebRtcNs_Free() — release NsHandle +// +// The library works at a single fixed sample rate; if the pipeline runs at +// a different rate, the caller (webrtc_ns.c) is responsible for downsampling +// before and upsampling after (not yet implemented in this backend — for now, +// the backend uses CONFIG_WEBRTC_NS_SAMPLE_RATE_HZ directly and requires the +// pipeline to run at that rate, or to be configured identically). +// +// Requires cross-built libwebrtc_ns.a (west module modules/audio/webrtc-apm). +// See webrtc_ns.cmake for the cross-build recipe. + +#include +#include +#include +#include "webrtc_ns.h" + +#include /* from cross-built webrtc-audio-processing 0.3.x */ + +LOG_MODULE_DECLARE(webrtc_ns, CONFIG_SOF_LOG_LEVEL); + +/* Per-channel NS state handles. */ +struct webrtc_ns_real_data { + NsHandle *ns[WEBRTC_NS_CHANNELS_MAX]; + int num_channels; +}; + +static int webrtc_ns_real_init(struct processing_module *mod) +{ + struct webrtc_ns_comp_data *cd = module_get_private_data(mod); + struct webrtc_ns_real_data *rd; + + rd = mod_zalloc(mod, sizeof(*rd)); + if (!rd) + return -ENOMEM; + + cd->backend_data = rd; + comp_info(mod->dev, "webrtc_ns: WebRTC NS real backend initialised"); + return 0; +} + +static int webrtc_ns_real_configure(struct processing_module *mod, + int sample_rate_hz, int level, int num_channels) +{ + struct webrtc_ns_comp_data *cd = module_get_private_data(mod); + struct webrtc_ns_real_data *rd = cd->backend_data; + int c, ret; + + /* Free any previously allocated handles from a re-prepare. */ + for (c = 0; c < rd->num_channels; c++) { + if (rd->ns[c]) { + WebRtcNs_Free(rd->ns[c]); + rd->ns[c] = NULL; + } + } + + for (c = 0; c < num_channels; c++) { + rd->ns[c] = WebRtcNs_Create(); + if (!rd->ns[c]) { + comp_err(mod->dev, "webrtc_ns: WebRtcNs_Create() failed ch%d", c); + goto err; + } + + ret = WebRtcNs_Init(rd->ns[c], (uint32_t)sample_rate_hz); + if (ret) { + comp_err(mod->dev, "webrtc_ns: WebRtcNs_Init(ch%d, %d) failed %d", + c, sample_rate_hz, ret); + goto err; + } + + ret = WebRtcNs_set_policy(rd->ns[c], level); + if (ret) { + comp_err(mod->dev, "webrtc_ns: set_policy(%d) failed %d", level, ret); + goto err; + } + } + + rd->num_channels = num_channels; + comp_info(mod->dev, "webrtc_ns: WebRTC NS rate=%d level=%d ch=%d", + sample_rate_hz, level, num_channels); + return 0; + +err: + for (c = 0; c < num_channels; c++) { + if (rd->ns[c]) { + WebRtcNs_Free(rd->ns[c]); + rd->ns[c] = NULL; + } + } + return -ENOMEM; +} + +static int webrtc_ns_real_process(struct processing_module *mod, + const float *const *in, float *const *out, + int frame_samples) +{ + struct webrtc_ns_comp_data *cd = module_get_private_data(mod); + struct webrtc_ns_real_data *rd = cd->backend_data; + int c; + + for (c = 0; c < rd->num_channels; c++) { + /* Analyze updates the noise model (non-destructive). */ + WebRtcNs_Analyze(rd->ns[c], in[c]); + + /* Process suppresses noise: in → out. */ + const float *in_ptrs[1] = { in[c] }; + float *out_ptrs[1] = { out[c] }; + + WebRtcNs_Process(rd->ns[c], in_ptrs, 1, out_ptrs); + } + + return 0; +} + +static int webrtc_ns_real_reset(struct processing_module *mod) +{ + struct webrtc_ns_comp_data *cd = module_get_private_data(mod); + struct webrtc_ns_real_data *rd = cd->backend_data; + int c, ret; + + /* Re-initialise each channel to reset internal state. */ + for (c = 0; c < rd->num_channels; c++) { + if (!rd->ns[c]) + continue; + ret = WebRtcNs_Init(rd->ns[c], (uint32_t)cd->proc_rate); + if (ret) + comp_warn(mod->dev, "webrtc_ns: reset ch%d failed %d", c, ret); + } + return 0; +} + +static int webrtc_ns_real_free(struct processing_module *mod) +{ + struct webrtc_ns_comp_data *cd = module_get_private_data(mod); + struct webrtc_ns_real_data *rd = cd->backend_data; + int c; + + if (!rd) + return 0; + + for (c = 0; c < rd->num_channels; c++) { + if (rd->ns[c]) { + WebRtcNs_Free(rd->ns[c]); + rd->ns[c] = NULL; + } + } + + mod_free(mod, rd); + cd->backend_data = NULL; + return 0; +} + +const struct webrtc_ns_backend webrtc_ns_backend = { + .name = "webrtc-ns", + .init = webrtc_ns_real_init, + .configure = webrtc_ns_real_configure, + .process = webrtc_ns_real_process, + .reset = webrtc_ns_real_reset, + .free = webrtc_ns_real_free, +}; diff --git a/src/audio/webrtc_ns/webrtc_ns.c b/src/audio/webrtc_ns/webrtc_ns.c new file mode 100644 index 000000000000..13bca7606be7 --- /dev/null +++ b/src/audio/webrtc_ns/webrtc_ns.c @@ -0,0 +1,373 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. +// +// WebRTC Noise Suppression module — SOF module_interface core. +// +// Single-input, single-output PCM effect (mic → denoised mic). The module +// converts interleaved S16/S32 → planar float, accumulates a full 10 ms frame, +// runs the backend NS for each channel, and converts float → S16/S32 back to +// the sink. Latency is at most 10 ms (one NS frame period). + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "webrtc_ns.h" + +/* UUID registered in uuid-registry.txt. */ +SOF_DEFINE_REG_UUID(webrtc_ns); + +LOG_MODULE_REGISTER(webrtc_ns, CONFIG_SOF_LOG_LEVEL); + +/* S32 normalisation scale factor (2^31). */ +#define WEBRTC_NS_S32_SCALE 2147483648.0f + +/* S16 normalisation scale factor (2^15). */ +#define WEBRTC_NS_S16_SCALE 32768.0f + +/** + * webrtc_ns_init() - Allocate private data and initialise the backend. + */ +__cold static int webrtc_ns_init(struct processing_module *mod) +{ + struct module_data *md = &mod->priv; + struct comp_dev *dev = mod->dev; + struct webrtc_ns_comp_data *cd; + int ret, i; + + assert_can_be_cold(); + comp_info(dev, "webrtc_ns: init"); + + cd = mod_zalloc(mod, sizeof(*cd)); + if (!cd) + return -ENOMEM; + + md->private = cd; + cd->backend = &webrtc_ns_backend; + + /* Wire up permanent planar buffer pointer arrays. */ + for (i = 0; i < WEBRTC_NS_CHANNELS_MAX; i++) { + cd->in_ptrs[i] = cd->in_buf[i]; + cd->out_ptrs[i] = cd->out_buf[i]; + } + + comp_info(dev, "webrtc_ns: backend '%s'", cd->backend->name); + + if (cd->backend->init) { + ret = cd->backend->init(mod); + if (ret) { + comp_err(dev, "webrtc_ns: backend init failed %d", ret); + mod_free(mod, cd); + return ret; + } + } + + return 0; +} + +/** + * webrtc_ns_prepare() - Configure NS for the negotiated pipeline format. + */ +__cold static int webrtc_ns_prepare(struct processing_module *mod, + struct sof_source **sources, int num_of_sources, + struct sof_sink **sinks, int num_of_sinks) +{ + struct webrtc_ns_comp_data *cd = module_get_private_data(mod); + struct comp_dev *dev = mod->dev; + int rate, fmt, ret; + + assert_can_be_cold(); + + if (num_of_sources != 1 || num_of_sinks != 1) { + comp_err(dev, "webrtc_ns: need exactly 1 source and 1 sink"); + return -EINVAL; + } + + rate = source_get_rate(sources[0]); + fmt = source_get_frm_fmt(sources[0]); + cd->channels = source_get_channels(sources[0]); + + if (cd->channels > WEBRTC_NS_CHANNELS_MAX) { + comp_err(dev, "webrtc_ns: too many channels %d (max %d)", + cd->channels, WEBRTC_NS_CHANNELS_MAX); + return -EINVAL; + } + + /* NS only accepts 8/16/32/48 kHz. */ + if (rate != 8000 && rate != 16000 && rate != 32000 && rate != 48000) { + comp_err(dev, "webrtc_ns: unsupported rate %d", rate); + return -EINVAL; + } + + switch (fmt) { + case SOF_IPC_FRAME_S16_LE: + case SOF_IPC_FRAME_S32_LE: + break; + default: + comp_err(dev, "webrtc_ns: unsupported frame format %d", fmt); + return -EINVAL; + } + + cd->in_rate = rate; + cd->proc_rate = CONFIG_WEBRTC_NS_SAMPLE_RATE_HZ; + if (cd->proc_rate != rate) { + /* Proc rate must be <= in_rate for simple downsampling. */ + if (cd->proc_rate > rate || rate % cd->proc_rate != 0) { + comp_err(dev, "webrtc_ns: proc_rate %d incompatible with in_rate %d", + cd->proc_rate, rate); + return -EINVAL; + } + } + + cd->in_frame_bytes = source_get_frame_bytes(sources[0]); + cd->proc_frame_samples = (cd->proc_rate * 10) / 1000; /* 10 ms */ + cd->in_frame_samples = (cd->in_rate * 10) / 1000; + + if (cd->proc_frame_samples > WEBRTC_NS_FRAME_SAMPLES_MAX) { + comp_err(dev, "webrtc_ns: proc_frame_samples %d exceeds max %d", + cd->proc_frame_samples, WEBRTC_NS_FRAME_SAMPLES_MAX); + return -EINVAL; + } + + comp_info(dev, "webrtc_ns: in_rate=%d proc_rate=%d ch=%d proc_frame=%d", + cd->in_rate, cd->proc_rate, cd->channels, cd->proc_frame_samples); + + if (cd->backend->configure) { + ret = cd->backend->configure(mod, cd->proc_rate, + CONFIG_WEBRTC_NS_LEVEL, + cd->channels); + if (ret) { + comp_err(dev, "webrtc_ns: backend configure failed %d", ret); + return ret; + } + } + + cd->buffered_frames = 0; + cd->configured = true; + return 0; +} + +/** + * s32_to_float() - Convert interleaved S32 samples to planar float. + * @cd: Module private data (channels, proc_frame_samples). + * @src: Source pointer (interleaved S32, raw[ch]). + * @n_frames: Number of frames (each frame has cd->channels samples). + * @frame0: Starting offset in the planar float buffer (for accumulation). + */ +static void s32_to_float(struct webrtc_ns_comp_data *cd, + const int32_t *src, int n_frames, int frame0) +{ + int i, c; + + for (i = 0; i < n_frames; i++) + for (c = 0; c < cd->channels; c++) + cd->in_buf[c][frame0 + i] = + (float)src[i * cd->channels + c] / WEBRTC_NS_S32_SCALE; +} + +/** + * s16_to_float() - Convert interleaved S16 samples to planar float. + */ +static void s16_to_float(struct webrtc_ns_comp_data *cd, + const int16_t *src, int n_frames, int frame0) +{ + int i, c; + + for (i = 0; i < n_frames; i++) + for (c = 0; c < cd->channels; c++) + cd->in_buf[c][frame0 + i] = + (float)src[i * cd->channels + c] / WEBRTC_NS_S16_SCALE; +} + +/** + * float_to_s32() - Write planar float NS output back as interleaved S32. + */ +static void float_to_s32(struct webrtc_ns_comp_data *cd, + int32_t *dst, int n_frames) +{ + int i, c; + + for (i = 0; i < n_frames; i++) { + for (c = 0; c < cd->channels; c++) { + float f = cd->out_buf[c][i] * WEBRTC_NS_S32_SCALE; + + f = f > 2147483647.0f ? 2147483647.0f : + f < -2147483648.0f ? -2147483648.0f : f; + dst[i * cd->channels + c] = (int32_t)f; + } + } +} + +/** + * float_to_s16() - Write planar float NS output back as interleaved S16. + */ +static void float_to_s16(struct webrtc_ns_comp_data *cd, + int16_t *dst, int n_frames) +{ + int i, c; + + for (i = 0; i < n_frames; i++) { + for (c = 0; c < cd->channels; c++) { + float f = cd->out_buf[c][i] * WEBRTC_NS_S16_SCALE; + + f = f > 32767.0f ? 32767.0f : + f < -32768.0f ? -32768.0f : f; + dst[i * cd->channels + c] = (int16_t)f; + } + } +} + +/** + * webrtc_ns_process() - Run NS on accumulated 10 ms frames. + * + * The pipeline period may be shorter than 10 ms, so we accumulate interleaved + * PCM → planar float until we have a full NS frame, then process and drain. + */ +static int webrtc_ns_process(struct processing_module *mod, + struct sof_source **sources, int num_of_sources, + struct sof_sink **sinks, int num_of_sinks) +{ + struct webrtc_ns_comp_data *cd = module_get_private_data(mod); + struct sof_source *src = sources[0]; + struct sof_sink *snk = sinks[0]; + enum sof_ipc_frame fmt = source_get_frm_fmt(src); + int avail = (int)source_get_data_frames_available(src); + int free = (int)sink_get_free_frames(snk); + int n = MIN(avail, free); + size_t nbytes, buf_size; + void const *rd_ptr, *buf_start; + void *wr_ptr, *wr_buf_start; + int ret; + + if (n <= 0) + return 0; + + nbytes = (size_t)n * source_get_frame_bytes(src); + + ret = source_get_data(src, nbytes, &rd_ptr, &buf_start, &buf_size); + if (ret) + return ret; + + ret = sink_get_buffer(snk, nbytes, &wr_ptr, &wr_buf_start, &buf_size); + if (ret) { + source_release_data(src, 0); + return ret; + } + + /* Convert input to planar float, accumulating into in_buf[]. */ + if (fmt == SOF_IPC_FRAME_S32_LE) + s32_to_float(cd, rd_ptr, n, cd->buffered_frames); + else + s16_to_float(cd, rd_ptr, n, cd->buffered_frames); + + source_release_data(src, nbytes); + + cd->buffered_frames += n; + + /* + * Process complete NS frames. Each call consumes proc_frame_samples. + * Any partial frame remains in in_buf[] for the next cycle. + */ + while (cd->buffered_frames >= cd->proc_frame_samples) { + int fs = cd->proc_frame_samples; + + ret = cd->backend->process(mod, + (const float *const *)cd->in_ptrs, + cd->out_ptrs, fs); + if (ret) { + comp_err(mod->dev, "webrtc_ns: backend process error %d", ret); + /* On error: fall back to pass-through for this frame. */ + int c; + + for (c = 0; c < cd->channels; c++) + memcpy(cd->out_buf[c], cd->in_buf[c], + (size_t)fs * sizeof(float)); + } + + /* Write processed frame to sink. */ + if (fmt == SOF_IPC_FRAME_S32_LE) + float_to_s32(cd, wr_ptr, fs); + else + float_to_s16(cd, wr_ptr, fs); + + /* Slide remaining samples to front of in_buf[]. */ + cd->buffered_frames -= fs; + if (cd->buffered_frames > 0) { + int c; + + for (c = 0; c < cd->channels; c++) + memmove(cd->in_buf[c], cd->in_buf[c] + fs, + (size_t)cd->buffered_frames * sizeof(float)); + } + } + + sink_commit_buffer(snk, nbytes); + return 0; +} + +/** + * webrtc_ns_reset() - Flush accumulator and reset backend state. + */ +static int webrtc_ns_reset(struct processing_module *mod) +{ + struct webrtc_ns_comp_data *cd = module_get_private_data(mod); + + comp_dbg(mod->dev, "webrtc_ns: reset"); + cd->buffered_frames = 0; + + if (cd->backend->reset) + return cd->backend->reset(mod); + + return 0; +} + +/** + * webrtc_ns_free() - Free all resources. + */ +__cold static int webrtc_ns_free(struct processing_module *mod) +{ + struct webrtc_ns_comp_data *cd = module_get_private_data(mod); + + assert_can_be_cold(); + comp_dbg(mod->dev, "webrtc_ns: free"); + + if (cd->backend->free) + cd->backend->free(mod); + + mod_free(mod, cd); + return 0; +} + +static const struct module_interface webrtc_ns_interface = { + .init = webrtc_ns_init, + .prepare = webrtc_ns_prepare, + .process = webrtc_ns_process, + .reset = webrtc_ns_reset, + .free = webrtc_ns_free, +}; + +#if CONFIG_COMP_WEBRTC_NS_MODULE + +#include +#include +#include + +static const struct sof_man_module_manifest mod_manifest __section(".module") __used = + SOF_LLEXT_MODULE_MANIFEST("WRTCNS", &webrtc_ns_interface, 1, + SOF_REG_UUID(webrtc_ns), 40); + +SOF_LLEXT_BUILDINFO; + +#else + +DECLARE_TR_CTX(webrtc_ns_tr, SOF_UUID(webrtc_ns_uuid), LOG_LEVEL_INFO); +DECLARE_MODULE_ADAPTER(webrtc_ns_interface, webrtc_ns_uuid, webrtc_ns_tr); +SOF_MODULE_INIT(webrtc_ns, sys_comp_module_webrtc_ns_interface_init); + +#endif diff --git a/src/audio/webrtc_ns/webrtc_ns.cmake b/src/audio/webrtc_ns/webrtc_ns.cmake new file mode 100644 index 000000000000..7ca121417c5f --- /dev/null +++ b/src/audio/webrtc_ns/webrtc_ns.cmake @@ -0,0 +1,80 @@ +# SPDX-License-Identifier: BSD-3-Clause +# +# Cross-build the WebRTC Noise Suppression (classic 0.3.x) C sources into +# a static library for the webrtc_ns LLEXT module. +# +# webrtc-audio-processing 0.3.x uses a pure-C noise suppressor with no C++ +# and no abseil dependency. We compile the relevant source files directly +# using the Xtensa cross-toolchain (same approach as libfvad in webrtc_vad). +# +# Source is pulled via west from the webrtc-apm west module (which contains +# the webrtc-audio-processing 0.3.1 tarball extraction). See west.yml. +# +# Produces: +# ${WEBRTC_NS_INSTALL_DIR}/lib/libwebrtc_ns.a +# ${WEBRTC_NS_INSTALL_DIR}/include/noise_suppression.h + +if(NOT DEFINED SOF_WEBRTC_APM_SRC_DIR) + set(SOF_WEBRTC_APM_SRC_DIR "${sof_top_dir}/../modules/audio/webrtc-apm" + CACHE PATH "webrtc-audio-processing 0.3.x source directory (west-pinned)") +endif() +cmake_path(NORMAL_PATH SOF_WEBRTC_APM_SRC_DIR) +if(NOT EXISTS "${SOF_WEBRTC_APM_SRC_DIR}/webrtc/modules/audio_processing/ns/ns_core.c") + message(FATAL_ERROR + "webrtc_ns: WebRTC APM NS source not found at '${SOF_WEBRTC_APM_SRC_DIR}'.\n" + "Run 'west update' (webrtc-apm is pinned in west.yml) or pass\n" + "-DSOF_WEBRTC_APM_SRC_DIR=.") +endif() + +# Derive cross-toolchain prefix from the Zephyr compiler path. +get_filename_component(_tc_dir "${CMAKE_C_COMPILER}" DIRECTORY) +get_filename_component(_tc_name "${CMAKE_C_COMPILER}" NAME) +string(REGEX REPLACE "gcc$" "" _tc_prefix_name "${_tc_name}") +set(_ns_cross_prefix "${_tc_dir}/${_tc_prefix_name}") + +set(WEBRTC_NS_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/webrtc-ns-install" + CACHE INTERNAL "webrtc_ns: NS library install prefix") + +set(_ns_src "${SOF_WEBRTC_APM_SRC_DIR}/webrtc") + +file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/webrtc-ns-build.sh" +"#!/bin/sh +set -e +SRC=${_ns_src} +INST=${WEBRTC_NS_INSTALL_DIR} +OBJ=${CMAKE_CURRENT_BINARY_DIR}/webrtc-ns-obj + +mkdir -p \"$OBJ\" \"$INST/lib\" \"$INST/include\" + +CFLAGS=\"-O2 -fPIC -I$SRC -I$SRC/common_audio/signal_processing/include\" + +SRCS=\" + $SRC/modules/audio_processing/ns/ns_core.c + $SRC/modules/audio_processing/ns/ns_fft.c + $SRC/common_audio/signal_processing/complex_fft.c + $SRC/common_audio/signal_processing/division_operations.c + $SRC/common_audio/signal_processing/energy.c + $SRC/common_audio/signal_processing/get_scaling_square.c + $SRC/common_audio/signal_processing/real_fft.c + $SRC/common_audio/signal_processing/spl_inl.c +\" + +for f in \$SRCS; do + bn=\$(basename \"\$f\" .c) + ${CMAKE_C_COMPILER} \$CFLAGS -c \"\$f\" -o \"$OBJ/\${bn}.o\" +done + +${_ns_cross_prefix}ar rcs \"$INST/lib/libwebrtc_ns.a\" \"$OBJ\"/*.o +cp $SRC/modules/audio_processing/ns/include/noise_suppression.h \"$INST/include/\" +") + +add_custom_command( + OUTPUT "${WEBRTC_NS_INSTALL_DIR}/lib/libwebrtc_ns.a" + "${WEBRTC_NS_INSTALL_DIR}/include/noise_suppression.h" + COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/webrtc-ns-build.sh" + VERBATIM) + +add_custom_target(webrtc_ns_ext + DEPENDS "${WEBRTC_NS_INSTALL_DIR}/lib/libwebrtc_ns.a") + +message(STATUS "webrtc_ns: cross-building WebRTC NS from ${SOF_WEBRTC_APM_SRC_DIR}") diff --git a/src/audio/webrtc_ns/webrtc_ns.h b/src/audio/webrtc_ns/webrtc_ns.h new file mode 100644 index 000000000000..2e215418166f --- /dev/null +++ b/src/audio/webrtc_ns/webrtc_ns.h @@ -0,0 +1,104 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright(c) 2026 Intel Corporation. + * + * WebRTC Noise Suppression module for SOF. + * + * Wraps the classic WebRTC NS algorithm (spectral subtraction / Wiener filter + * from webrtc-audio-processing 0.3.x) behind the SOF module_interface as a + * single-input, single-output PCM effect. + * + * The actual suppression is delegated to a pluggable backend so the SOF glue + * can be validated with a stub (webrtc_ns-stub.c) before the real WebRTC NS + * backend (webrtc_ns-webrtc.c) is linked in. + */ +#ifndef __SOF_AUDIO_WEBRTC_NS_H__ +#define __SOF_AUDIO_WEBRTC_NS_H__ + +#include +#include +#include + +/* + * WebRTC NS requires exactly 10 ms frames at the processing sample rate. + * Maximum: 10 ms at 48 kHz = 480 samples/channel. + */ +#define WEBRTC_NS_FRAME_SAMPLES_MAX 480 + +/* Maximum channel count supported. */ +#define WEBRTC_NS_CHANNELS_MAX 8 + +/** + * struct webrtc_ns_backend - NS backend operations. + * + * A backend owns the WebRTC NS state and the per-frame suppression call. + * All ops return 0 on success or a negative errno. + */ +struct webrtc_ns_backend { + const char *name; + + /* One-time init, called from module init(). */ + int (*init)(struct processing_module *mod); + + /** + * Configure the NS for rate/level. Called from prepare(). + * @sample_rate_hz: processing rate (8000/16000/32000/48000) + * @level: NS aggressiveness 0..3 + * @num_channels: number of channels to suppress independently + */ + int (*configure)(struct processing_module *mod, + int sample_rate_hz, int level, int num_channels); + + /** + * Process one 10 ms frame of float PCM (per channel, planar). + * @in: array of num_channels pointers, each of frame_samples floats + * @out: array of num_channels pointers, each of frame_samples floats + * @frame_samples: samples per channel (rate * 10 / 1000) + */ + int (*process)(struct processing_module *mod, + const float *const *in, float *const *out, + int frame_samples); + + /* Reset NS state (keep configuration). Called from reset(). */ + int (*reset)(struct processing_module *mod); + + /* Tear down all state from init()/configure(). */ + int (*free)(struct processing_module *mod); +}; + +/** + * struct webrtc_ns_comp_data - webrtc_ns module private data. + * @backend: Selected backend operations. + * @backend_data: Backend-private state handle. + * @in_rate: Input sample rate from pipeline (Hz). + * @proc_rate: Processing rate used by NS (may be < in_rate). + * @channels: Channel count. + * @in_frame_bytes: Bytes per frame at pipeline rate. + * @proc_frame_samples: Samples per 10 ms at proc_rate (per channel). + * @in_frame_samples: Samples per 10 ms at in_rate (per channel). + * @buffered_frames: Samples currently in accumulator. + * @in_buf: Float planar input accumulator [ch][frame_samples]. + * @out_buf: Float planar output buffer [ch][frame_samples]. + * @configured: True once backend has been opened. + */ +struct webrtc_ns_comp_data { + const struct webrtc_ns_backend *backend; + void *backend_data; + int in_rate; + int proc_rate; + int channels; + int in_frame_bytes; + int proc_frame_samples; + int in_frame_samples; + int buffered_frames; + float in_buf[WEBRTC_NS_CHANNELS_MAX][WEBRTC_NS_FRAME_SAMPLES_MAX]; + float out_buf[WEBRTC_NS_CHANNELS_MAX][WEBRTC_NS_FRAME_SAMPLES_MAX]; + const float *in_ptrs[WEBRTC_NS_CHANNELS_MAX]; + float *out_ptrs[WEBRTC_NS_CHANNELS_MAX]; + bool configured; +}; + +/* Backend instance provided by the selected translation unit. */ +extern const struct webrtc_ns_backend webrtc_ns_backend; + +#endif /* __SOF_AUDIO_WEBRTC_NS_H__ */ diff --git a/src/audio/webrtc_ns/webrtc_ns.toml b/src/audio/webrtc_ns/webrtc_ns.toml new file mode 100644 index 000000000000..293fd9dcd66f --- /dev/null +++ b/src/audio/webrtc_ns/webrtc_ns.toml @@ -0,0 +1,24 @@ +#ifndef LOAD_TYPE +#define LOAD_TYPE "0" +#endif + + REM # WebRTC Noise Suppression module config + [[module.entry]] + REM # Module name limited to 8 chars; must match SOF_LLEXT_MODULE_MANIFEST() in webrtc_ns.c. + name = "WRTCNS" + uuid = UUIDREG_STR_WEBRTC_NS + affinity_mask = "0x1" + instance_count = "40" + domain_types = "0" + load_type = LOAD_TYPE + module_type = "9" + auto_start = "0" + sched_caps = [1, 0x00008000] + REM # pin = [dir, type, sample rate, size, container, channel-cfg] + REM # Single input pin (PCM), single output pin (denoised PCM). Pass-through topology. + pin = [0, 0, 0xfeef, 0xf, 0xf, 0x45ff, 1, 0, 0xfeef, 0xf, 0xf, 0x45ff] + REM # mod_cfg [PAR_0 PAR_1 PAR_2 PAR_3 IS_BYTES CPS IBS OBS MOD_FLAGS CPC OBLS] + REM # NS adds ~10ms buffering latency; IBS/OBS sized for 2×10ms at 48kHz stereo S32. + mod_cfg = [0, 0, 0, 0, 0, 3000000, 16384, 16384, 0, 3000, 0] + + index = __COUNTER__ diff --git a/src/audio/webrtc_ns2/CMakeLists.txt b/src/audio/webrtc_ns2/CMakeLists.txt new file mode 100644 index 000000000000..fbca7ab5205d --- /dev/null +++ b/src/audio/webrtc_ns2/CMakeLists.txt @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: BSD-3-Clause + +if(CONFIG_COMP_WEBRTC_NS2 STREQUAL "m" AND DEFINED CONFIG_LLEXT) + add_subdirectory(llext ${PROJECT_BINARY_DIR}/webrtc_ns2_llext) + add_dependencies(app webrtc_ns2) +else() + add_local_sources(sof webrtc_ns2.c) + if(CONFIG_COMP_WEBRTC_NS2_STUB) + add_local_sources(sof webrtc_ns2-stub.c) + else() + add_local_sources(sof webrtc_ns2-rnn.c) + endif() +endif() diff --git a/src/audio/webrtc_ns2/Kconfig b/src/audio/webrtc_ns2/Kconfig new file mode 100644 index 000000000000..09af8b84d9d2 --- /dev/null +++ b/src/audio/webrtc_ns2/Kconfig @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: BSD-3-Clause + +config COMP_WEBRTC_NS2 + tristate "RNNoise deep-learning noise suppressor (floating-point)" + help + Select to include the RNNoise module — a recurrent neural network + noise suppressor originally developed by Jean-Marc Valin (Mozilla / + Opus). RNNoise operates at 48 kHz on 10 ms (480-sample) mono frames + and achieves significantly better suppression of non-stationary noise + (music, babble, keyboard) than the classic spectral Wiener filter + (COMP_WEBRTC_NS). + + The module is single-input, single-output and internally emits a + per-frame VAD probability (0.0–1.0) via NOTIFIER_ID_VAD, making it + a combined denoiser and activity detector. + + Requirements: + - Floating-point unit (FPU=y on ACE30/PTL; soft-float also works + but at significant CPU cost) + - ~30 KB of state per channel (DenoiseState + RNN weights) + - Pipeline sample rate exactly 48000 Hz + - Pipeline period must be a multiple of 480 samples (10 ms) + + The real backend builds RNNoise from source (west module rnnoise, + pinned at commit 70f1d256). For CI/stub builds select + COMP_WEBRTC_NS2_STUB. + + License: BSD-3-Clause (xiph/rnnoise). + +if COMP_WEBRTC_NS2 + +config COMP_WEBRTC_NS2_STUB + bool "RNNoise stub backend (no RNNoise source dependency)" + default y if COMP_STUBS + help + Build webrtc_ns2 against a dependency-free pass-through stub instead + of the real RNNoise library. The stub passes audio through unmodified + and always reports VAD probability = 1.0. Used for CI and LLEXT + packaging validation without the RNNoise source. + +config WEBRTC_NS2_CHANNELS_MAX + int "Maximum channel count (each channel gets its own RNNoise instance)" + range 1 8 + default 2 + help + RNNoise is strictly mono; the module runs one DenoiseState per + channel. Increasing this raises SRAM usage by ~30 KB per extra + channel. Set to 1 for pure mono capture pipelines. + +config WEBRTC_NS2_VAD_NOTIFY + bool "Emit NOTIFIER_ID_VAD event from RNNoise VAD probability" + default y + help + RNNoise returns a per-frame speech probability in [0.0, 1.0]. When + this option is enabled the module broadcasts a NOTIFIER_ID_VAD event + (speech=1 / silence=0) using a configurable threshold, providing a + combined denoiser + VAD without running a separate webrtc_vad module. + +config WEBRTC_NS2_VAD_THRESHOLD_PCT + int "VAD threshold as percentage of max probability (0-100)" + range 0 100 + default 50 + depends on WEBRTC_NS2_VAD_NOTIFY + help + Speech is declared when rnnoise_process_frame() returns a probability + >= this value / 100.0. 50 is a good default; lower values increase + sensitivity (more false positives), higher values reduce it. + +endif # COMP_WEBRTC_NS2 diff --git a/src/audio/webrtc_ns2/README.md b/src/audio/webrtc_ns2/README.md new file mode 100644 index 000000000000..b5276e3db5a0 --- /dev/null +++ b/src/audio/webrtc_ns2/README.md @@ -0,0 +1,97 @@ +# WebRTC Noise Suppression 2 Module (`webrtc_ns2` / RNNoise) + +Wraps Xiph's [RNNoise](https://github.com/xiph/rnnoise) — a recurrent neural network (RNN/GRU) based noise suppression algorithm — behind the SOF `module_interface`. + +--- + +## Features + +- **Deep Learning Suppression**: Uses a hybrid DSP + deep recurrent neural network structure (GRU) for suppressing non-stationary noise (such as keyboard clicks, speech babble, or music). +- **VAD Output**: Calculates speech probability per frame and broadcasts binary voice activity events via `NOTIFIER_ID_VAD`. +- **Locked Rate (48 kHz)**: Runs exclusively at **48000 Hz** (requires an ASRC upstream if the DAI runs at another rate). +- **10 ms Process Window**: Accumulates pipeline periods to process complete 480-sample mono frames. +- **Float Scaled**: normalizes standard SOF float pipelines to the internal full-scale float representation ($\pm 32768.0$) used during training. +- **Low Memory / Fast Math**: Employs a 201-entry lookup table for sigmoid and tanh functions; eliminates hot-path `expf` calls. + +--- + +## Architecture & Data Flow + +The following Mermaid diagram outlines the data normalization, period accumulator, and deep-learning inference flow inside the module: + +```mermaid +graph TD + %% Input Path + InBuf[Input Audio Buffer] -->|S16 or S32 PCM| Core[webrtc_ns2.c Core] + + %% Accumulation & Scaling + Core -->|Accumulate 480 Samples| Acc{Frame Ready?} + Acc -->|No| Core + Acc -->|Yes: Normalize| FloatScale[Scale to ±32768.0] + + %% RNNoise Processing + FloatScale -->|Full-Scale Float Frame| Backend[webrtc_ns2-rnn.c Backend] + Backend -->|rnnoise_process_frame| RNNEngine[RNNoise GRU inference] + + %% Outputs + RNNEngine -->|Denoised Frame| Backend + RNNEngine -->|Speech Probability 0.0 - 1.0| VAD{Prob >= Threshold?} + + %% Scale down and copy out + Backend -->|Scale back to ±1.0| Core + Core -->|Interleaved output| OutBuf[Output Audio Buffer] + + VAD -->|Yes: 1 / No: 0| Notifier[NOTIFIER_ID_VAD Event] +``` + +### Components +- `webrtc_ns2.c`: Handles format conversions, period packing, and the VAD event notifier. +- `webrtc_ns2-rnn.c`: Real RNNoise library integration backend. +- `webrtc_ns2-stub.c`: Pass-through stub that always outputs `1.0` VAD probability. +- `webrtc_ns2.cmake`: Controls downloading the `rnnoise` Git repository and building its files. + +--- + +## Build Instructions + +### 1. Stub Mode (CI / Staging) +```ini +CONFIG_COMP_WEBRTC_NS2=y # or =m for LLEXT +CONFIG_COMP_WEBRTC_NS2_STUB=y +``` + +### 2. Real RNNoise Integration +Cross-compiles the neural networks: +```ini +CONFIG_COMP_WEBRTC_NS2=m +CONFIG_COMP_WEBRTC_NS2_STUB=n +``` +Requires `west update` to retrieve `modules/audio/rnnoise` (SHA `70f1d256`). + +*Important: The target DSP board must support floating-point compilation. Although a hardware FPU is not strictly mandatory, software-float emulation will impose a severe CPU bottleneck.* + +--- + +## Kconfig Parameters + +| Option | Default | Range / Value | Description | +|---|---|---|---| +| `CONFIG_COMP_WEBRTC_NS2` | n | y / m / n | Enable RNNoise module | +| `CONFIG_COMP_WEBRTC_NS2_STUB` | y | y / n | Use pass-through stub | +| `CONFIG_WEBRTC_NS2_CHANNELS_MAX` | 2 | 1 - 8 | Maximum channels supported | +| `CONFIG_WEBRTC_NS2_VAD_NOTIFY` | y | y / n | Emit NOTIFIER_ID_VAD events | +| `CONFIG_WEBRTC_NS2_VAD_THRESHOLD_PCT` | 50 | 0 - 100 | VAD speech threshold percentage | + +--- + +## Usage & Topology + +### Pipeline Integration +The module is integrated as a normal 1-in / 1-out effect widget. **The sample rate must be exactly 48000 Hz**. + +#### Example Topology Route +``` +DAI Copier (SSP RX @ 48kHz) ---> [ webrtc-ns2 ] ---> Host Copier (PCM @ 48kHz) +``` + +If the capture microphone or stream runs at 16 kHz, you must insert an ASRC widget upstream of `webrtc-ns2` to upsample the stream to 48 kHz before processing. diff --git a/src/audio/webrtc_ns2/llext/CMakeLists.txt b/src/audio/webrtc_ns2/llext/CMakeLists.txt new file mode 100644 index 000000000000..51393a0c41bf --- /dev/null +++ b/src/audio/webrtc_ns2/llext/CMakeLists.txt @@ -0,0 +1,24 @@ +# Copyright (c) 2026 Intel Corporation. +# SPDX-License-Identifier: Apache-2.0 + +if(CONFIG_COMP_WEBRTC_NS2_STUB) + sof_llext_build("webrtc_ns2" + SOURCES ../webrtc_ns2.c + ../webrtc_ns2-stub.c + LIB openmodules + ) +else() + include(${CMAKE_CURRENT_LIST_DIR}/../webrtc_ns2.cmake) + + sof_llext_build("webrtc_ns2" + SOURCES ../webrtc_ns2.c + ../webrtc_ns2-rnn.c + INCLUDES "${WEBRTC_NS2_INSTALL_DIR}/include" + LIBS_PATH "${WEBRTC_NS2_INSTALL_DIR}/lib" + LIBS rnnoise + LIB openmodules + ) + + add_dependencies(webrtc_ns2_llext_lib rnnoise_ext) + add_dependencies(webrtc_ns2 rnnoise_ext) +endif() diff --git a/src/audio/webrtc_ns2/llext/llext.toml.h b/src/audio/webrtc_ns2/llext/llext.toml.h new file mode 100644 index 000000000000..ac60a66b75c0 --- /dev/null +++ b/src/audio/webrtc_ns2/llext/llext.toml.h @@ -0,0 +1,6 @@ +#include +#define LOAD_TYPE "2" +#include "../webrtc_ns2.toml" + +[module] +count = __COUNTER__ diff --git a/src/audio/webrtc_ns2/webrtc_ns2-rnn.c b/src/audio/webrtc_ns2/webrtc_ns2-rnn.c new file mode 100644 index 000000000000..c0914df0e4ac --- /dev/null +++ b/src/audio/webrtc_ns2/webrtc_ns2-rnn.c @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. +// +// Real RNNoise backend for webrtc_ns2. +// +// RNNoise public API (rnnoise.h / denoise.c): +// +// int rnnoise_get_size(void); +// Returns sizeof(DenoiseState). Use this instead of sizeof() so the +// wrapper doesn't need to know the internal layout. +// +// int rnnoise_init(DenoiseState *st, const RNNModel *model); +// Initialise state in-place. model=NULL uses the built-in weights +// (rnnoise_model_orig compiled into rnnoise_tables.c). +// +// DenoiseState *rnnoise_create(const RNNModel *model); +// Allocate + init. Calls malloc() internally. +// +// void rnnoise_destroy(DenoiseState *st); +// Free. Calls free() internally. +// +// float rnnoise_process_frame(DenoiseState *st, float *out, const float *in); +// Process one 480-sample (10 ms at 48 kHz) mono frame. +// Samples are in full-scale float (not normalised to [-1,1]). +// Returns speech probability in [0, 1]. +// +// Memory: ~30 KB per DenoiseState (including RNN state). +// Dependencies: libm (expf, tanhf, sinf, cosf, sqrtf). + +#include +#include +#include "webrtc_ns2.h" + +#include /* from cross-built rnnoise library */ + +LOG_MODULE_DECLARE(webrtc_ns2, CONFIG_SOF_LOG_LEVEL); + +/* RNNoise uses full-scale float (raw amplitude, not normalised). */ +#define RNN_FULL_SCALE 32768.0f + +struct webrtc_ns2_rnn_data { + DenoiseState *st[WEBRTC_NS2_CHANNELS_MAX]; + int num_channels; +}; + +/* Per-frame scratch buffer at full scale. */ +static float rnn_in[WEBRTC_NS2_FRAME_SAMPLES]; +static float rnn_out[WEBRTC_NS2_FRAME_SAMPLES]; + +static int webrtc_ns2_rnn_init(struct processing_module *mod) +{ + struct webrtc_ns2_comp_data *cd = module_get_private_data(mod); + struct webrtc_ns2_rnn_data *rd; + + rd = mod_zalloc(mod, sizeof(*rd)); + if (!rd) + return -ENOMEM; + + cd->backend_data = rd; + comp_info(mod->dev, "webrtc_ns2: RNNoise real backend, frame=%d rate=%d", + WEBRTC_NS2_FRAME_SAMPLES, WEBRTC_NS2_SAMPLE_RATE); + return 0; +} + +static int webrtc_ns2_rnn_configure(struct processing_module *mod, int num_channels) +{ + struct webrtc_ns2_comp_data *cd = module_get_private_data(mod); + struct webrtc_ns2_rnn_data *rd = cd->backend_data; + int c; + + /* Free any previous instances from a re-prepare. */ + for (c = 0; c < rd->num_channels; c++) { + if (rd->st[c]) { + rnnoise_destroy(rd->st[c]); + rd->st[c] = NULL; + } + } + rd->num_channels = 0; + + for (c = 0; c < num_channels; c++) { + /* Pass NULL to use the built-in model weights. */ + rd->st[c] = rnnoise_create(NULL); + if (!rd->st[c]) { + comp_err(mod->dev, + "webrtc_ns2: rnnoise_create() failed ch%d", c); + goto err; + } + } + + rd->num_channels = num_channels; + comp_info(mod->dev, "webrtc_ns2: %d RNNoise instance(s) created", num_channels); + return 0; + +err: + for (c = 0; c < num_channels; c++) { + if (rd->st[c]) { + rnnoise_destroy(rd->st[c]); + rd->st[c] = NULL; + } + } + return -ENOMEM; +} + +static float webrtc_ns2_rnn_process_ch(struct processing_module *mod, + const float *in, float *out, int ch) +{ + struct webrtc_ns2_comp_data *cd = module_get_private_data(mod); + struct webrtc_ns2_rnn_data *rd = cd->backend_data; + float vad_prob; + int i; + + /* + * RNNoise expects full-scale float (raw PCM amplitude), but the SOF + * glue normalises to [-1, +1]. Scale up before processing and back + * down afterward. + */ + for (i = 0; i < WEBRTC_NS2_FRAME_SAMPLES; i++) + rnn_in[i] = in[i] * RNN_FULL_SCALE; + + vad_prob = rnnoise_process_frame(rd->st[ch], rnn_out, rnn_in); + + for (i = 0; i < WEBRTC_NS2_FRAME_SAMPLES; i++) + out[i] = rnn_out[i] / RNN_FULL_SCALE; + + return vad_prob; +} + +static int webrtc_ns2_rnn_reset(struct processing_module *mod) +{ + struct webrtc_ns2_comp_data *cd = module_get_private_data(mod); + struct webrtc_ns2_rnn_data *rd = cd->backend_data; + int c; + + /* + * RNNoise has no reset API; re-initialise each instance in-place + * using rnnoise_init() to clear the GRU hidden state. + */ + for (c = 0; c < rd->num_channels; c++) { + if (rd->st[c]) + rnnoise_init(rd->st[c], NULL); + } + return 0; +} + +static int webrtc_ns2_rnn_free(struct processing_module *mod) +{ + struct webrtc_ns2_comp_data *cd = module_get_private_data(mod); + struct webrtc_ns2_rnn_data *rd = cd->backend_data; + int c; + + if (!rd) + return 0; + + for (c = 0; c < rd->num_channels; c++) { + if (rd->st[c]) { + rnnoise_destroy(rd->st[c]); + rd->st[c] = NULL; + } + } + mod_free(mod, rd); + cd->backend_data = NULL; + return 0; +} + +const struct webrtc_ns2_backend webrtc_ns2_backend = { + .name = "rnnoise", + .init = webrtc_ns2_rnn_init, + .configure = webrtc_ns2_rnn_configure, + .process_ch = webrtc_ns2_rnn_process_ch, + .reset = webrtc_ns2_rnn_reset, + .free = webrtc_ns2_rnn_free, +}; diff --git a/src/audio/webrtc_ns2/webrtc_ns2-stub.c b/src/audio/webrtc_ns2/webrtc_ns2-stub.c new file mode 100644 index 000000000000..0cc943319c19 --- /dev/null +++ b/src/audio/webrtc_ns2/webrtc_ns2-stub.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. +// +// Stub backend for webrtc_ns2. Passes audio through unchanged. +// Returns VAD probability = 1.0 (always speech). + +#include +#include +#include "webrtc_ns2.h" + +LOG_MODULE_DECLARE(webrtc_ns2, CONFIG_SOF_LOG_LEVEL); + +static int webrtc_ns2_stub_init(struct processing_module *mod) +{ + comp_info(mod->dev, "webrtc_ns2 stub: no RNNoise library linked"); + return 0; +} + +static int webrtc_ns2_stub_configure(struct processing_module *mod, int num_channels) +{ + comp_info(mod->dev, "webrtc_ns2 stub: ch=%d (ignored)", num_channels); + return 0; +} + +static float webrtc_ns2_stub_process_ch(struct processing_module *mod, + const float *in, float *out, int ch) +{ + memcpy(out, in, WEBRTC_NS2_FRAME_SAMPLES * sizeof(float)); + (void)ch; + return 1.0f; /* always report speech */ +} + +static int webrtc_ns2_stub_reset(struct processing_module *mod) +{ + (void)mod; + return 0; +} + +static int webrtc_ns2_stub_free(struct processing_module *mod) +{ + (void)mod; + return 0; +} + +const struct webrtc_ns2_backend webrtc_ns2_backend = { + .name = "stub", + .init = webrtc_ns2_stub_init, + .configure = webrtc_ns2_stub_configure, + .process_ch = webrtc_ns2_stub_process_ch, + .reset = webrtc_ns2_stub_reset, + .free = webrtc_ns2_stub_free, +}; diff --git a/src/audio/webrtc_ns2/webrtc_ns2.c b/src/audio/webrtc_ns2/webrtc_ns2.c new file mode 100644 index 000000000000..8642dd3b3039 --- /dev/null +++ b/src/audio/webrtc_ns2/webrtc_ns2.c @@ -0,0 +1,335 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. +// +// webrtc_ns2 — SOF module_interface glue for the RNNoise deep-learning +// noise suppressor. +// +// Pass-through PCM effect: audio flows source → sink after per-channel +// RNNoise denoising. Each 480-sample (10 ms at 48 kHz) frame is +// processed independently per channel. Partial pipeline periods are +// accumulated until a full frame is ready. +// +// As a bonus, the per-frame VAD probability returned by RNNoise is +// compared against a configurable threshold and a NOTIFIER_ID_VAD event +// is fired, making this a drop-in combined denoiser + voice detector. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "webrtc_ns2.h" + +SOF_DEFINE_REG_UUID(webrtc_ns2); +LOG_MODULE_REGISTER(webrtc_ns2, CONFIG_SOF_LOG_LEVEL); + +/* Normalisation constants for S16 ↔ float and S32 ↔ float. */ +#define NS2_S16_SCALE 32768.0f +#define NS2_S32_SCALE 2147483648.0f + +/* ---------------------------------------------------------------- + * Format conversion helpers (interleaved PCM ↔ per-channel float) + * ---------------------------------------------------------------- */ + +static inline void src_to_float(struct webrtc_ns2_comp_data *cd, + const void *raw, int n_frames, int frame0) +{ + int i, c, ch = cd->channels; + + if (cd->is_s32) { + const int32_t *p = raw; + + for (i = 0; i < n_frames; i++) + for (c = 0; c < ch; c++) + cd->in_buf[c][frame0 + i] = + (float)p[i * ch + c] / NS2_S32_SCALE; + } else { + const int16_t *p = raw; + + for (i = 0; i < n_frames; i++) + for (c = 0; c < ch; c++) + cd->in_buf[c][frame0 + i] = + (float)p[i * ch + c] / NS2_S16_SCALE; + } +} + +static inline void float_to_sink(struct webrtc_ns2_comp_data *cd, + void *raw, int n_frames) +{ + int i, c, ch = cd->channels; + + if (cd->is_s32) { + int32_t *p = raw; + + for (i = 0; i < n_frames; i++) { + for (c = 0; c < ch; c++) { + float f = cd->out_buf[c][i] * NS2_S32_SCALE; + + f = f > 2147483647.0f ? 2147483647.0f : + f < -2147483648.0f ? -2147483648.0f : f; + p[i * ch + c] = (int32_t)f; + } + } + } else { + int16_t *p = raw; + + for (i = 0; i < n_frames; i++) { + for (c = 0; c < ch; c++) { + float f = cd->out_buf[c][i] * NS2_S16_SCALE; + + f = f > 32767.0f ? 32767.0f : + f < -32768.0f ? -32768.0f : f; + p[i * ch + c] = (int16_t)f; + } + } + } +} + +/* ---------------------------------------------------------------- + * module_interface operations + * ---------------------------------------------------------------- */ + +__cold static int webrtc_ns2_init(struct processing_module *mod) +{ + struct module_data *md = &mod->priv; + struct comp_dev *dev = mod->dev; + struct webrtc_ns2_comp_data *cd; + int ret; + + assert_can_be_cold(); + comp_info(dev, "webrtc_ns2: init"); + + cd = mod_zalloc(mod, sizeof(*cd)); + if (!cd) + return -ENOMEM; + + md->private = cd; + cd->backend = &webrtc_ns2_backend; + cd->vad_threshold = (float)CONFIG_WEBRTC_NS2_VAD_THRESHOLD_PCT / 100.0f; + + comp_info(dev, "webrtc_ns2: backend '%s' vad_threshold=%.2f", + cd->backend->name, (double)cd->vad_threshold); + + if (cd->backend->init) { + ret = cd->backend->init(mod); + if (ret) { + comp_err(dev, "webrtc_ns2: backend init failed %d", ret); + mod_free(mod, cd); + return ret; + } + } + + return 0; +} + +__cold static int webrtc_ns2_prepare(struct processing_module *mod, + struct sof_source **sources, int num_of_sources, + struct sof_sink **sinks, int num_of_sinks) +{ + struct webrtc_ns2_comp_data *cd = module_get_private_data(mod); + struct comp_dev *dev = mod->dev; + int rate, fmt, ret; + + assert_can_be_cold(); + + if (num_of_sources != 1 || num_of_sinks != 1) { + comp_err(dev, "webrtc_ns2: need exactly 1 source and 1 sink"); + return -EINVAL; + } + + rate = source_get_rate(sources[0]); + fmt = source_get_frm_fmt(sources[0]); + cd->channels = source_get_channels(sources[0]); + + /* RNNoise is hard-wired to 48 kHz. */ + if (rate != WEBRTC_NS2_SAMPLE_RATE) { + comp_err(dev, "webrtc_ns2: rate %d != required %d Hz; " + "add an ASRC upstream", rate, WEBRTC_NS2_SAMPLE_RATE); + return -EINVAL; + } + + if (cd->channels > WEBRTC_NS2_CHANNELS_MAX) { + comp_err(dev, "webrtc_ns2: %d channels exceeds max %d", + cd->channels, WEBRTC_NS2_CHANNELS_MAX); + return -EINVAL; + } + + switch (fmt) { + case SOF_IPC_FRAME_S16_LE: + cd->is_s32 = false; + break; + case SOF_IPC_FRAME_S32_LE: + cd->is_s32 = true; + break; + default: + comp_err(dev, "webrtc_ns2: unsupported format %d", fmt); + return -EINVAL; + } + + comp_info(dev, "webrtc_ns2: rate=%d ch=%d %s frame=%d", + rate, cd->channels, cd->is_s32 ? "S32" : "S16", + WEBRTC_NS2_FRAME_SAMPLES); + + if (cd->backend->configure) { + ret = cd->backend->configure(mod, cd->channels); + if (ret) { + comp_err(dev, "webrtc_ns2: backend configure failed %d", ret); + return ret; + } + } + + cd->buffered_frames = 0; + cd->last_vad = -1; /* unknown */ + cd->configured = true; + return 0; +} + +static int webrtc_ns2_process(struct processing_module *mod, + struct sof_source **sources, int num_of_sources, + struct sof_sink **sinks, int num_of_sinks) +{ + struct webrtc_ns2_comp_data *cd = module_get_private_data(mod); + struct sof_source *src = sources[0]; + struct sof_sink *snk = sinks[0]; + int frame_bytes = source_get_frame_bytes(src); + int avail = (int)source_get_data_frames_available(src); + int free_f = (int)sink_get_free_frames(snk); + int n = MIN(avail, free_f); + size_t nbytes; + const void *rd_ptr, *rd_start; + void *wr_ptr, *wr_start; + size_t buf_sz; + int ret, frames_rem, chunk; + + if (n <= 0) + return 0; + + nbytes = (size_t)n * frame_bytes; + + ret = source_get_data(src, nbytes, &rd_ptr, &rd_start, &buf_sz); + if (ret) + return ret; + + ret = sink_get_buffer(snk, nbytes, &wr_ptr, &wr_start, &buf_sz); + if (ret) { + source_release_data(src, 0); + return ret; + } + + /* + * Accumulate into in_buf[], process full 480-sample frames, + * drain denoised samples from out_buf[] into the sink write pointer. + * Both rd_ptr and wr_ptr advance by frame_bytes * n total. + */ + const char *rp = rd_ptr; + char *wp = wr_ptr; + + for (frames_rem = n; frames_rem > 0; frames_rem -= chunk) { + chunk = MIN(frames_rem, + WEBRTC_NS2_FRAME_SAMPLES - cd->buffered_frames); + + src_to_float(cd, rp, chunk, cd->buffered_frames); + rp += chunk * frame_bytes; + cd->buffered_frames += chunk; + + if (cd->buffered_frames >= WEBRTC_NS2_FRAME_SAMPLES) { + float vad_prob = 0.0f, ch_prob; + int c; + + for (c = 0; c < cd->channels; c++) { + ch_prob = cd->backend->process_ch( + mod, + cd->in_buf[c], + cd->out_buf[c], c); + /* Average VAD probability across channels. */ + if (ch_prob >= 0.0f) + vad_prob += ch_prob; + } + vad_prob /= (float)cd->channels; + + /* Write denoised samples to sink. */ + float_to_sink(cd, wp, WEBRTC_NS2_FRAME_SAMPLES); + wp += WEBRTC_NS2_FRAME_SAMPLES * frame_bytes; + + cd->buffered_frames = 0; + +#if CONFIG_WEBRTC_NS2_VAD_NOTIFY + { + int vad = (vad_prob >= cd->vad_threshold) ? 1 : 0; + + if (vad != cd->last_vad) { + notifier_event(mod->dev, NOTIFIER_ID_VAD, + NOTIFIER_TARGET_CORE_LOCAL, + &vad, sizeof(vad)); + cd->last_vad = vad; + } + } +#endif + } + } + + source_release_data(src, nbytes); + sink_commit_buffer(snk, nbytes); + return 0; +} + +static int webrtc_ns2_reset(struct processing_module *mod) +{ + struct webrtc_ns2_comp_data *cd = module_get_private_data(mod); + + comp_dbg(mod->dev, "webrtc_ns2: reset"); + cd->buffered_frames = 0; + cd->last_vad = -1; + + if (cd->backend->reset) + return cd->backend->reset(mod); + + return 0; +} + +__cold static int webrtc_ns2_free(struct processing_module *mod) +{ + struct webrtc_ns2_comp_data *cd = module_get_private_data(mod); + + assert_can_be_cold(); + comp_dbg(mod->dev, "webrtc_ns2: free"); + + if (cd->backend->free) + cd->backend->free(mod); + + mod_free(mod, cd); + return 0; +} + +static const struct module_interface webrtc_ns2_interface = { + .init = webrtc_ns2_init, + .prepare = webrtc_ns2_prepare, + .process = webrtc_ns2_process, + .reset = webrtc_ns2_reset, + .free = webrtc_ns2_free, +}; + +#if CONFIG_COMP_WEBRTC_NS2_MODULE + +#include +#include +#include + +static const struct sof_man_module_manifest mod_manifest __section(".module") __used = + SOF_LLEXT_MODULE_MANIFEST("WRTCNS2", &webrtc_ns2_interface, 1, + SOF_REG_UUID(webrtc_ns2), 40); + +SOF_LLEXT_BUILDINFO; + +#else + +DECLARE_TR_CTX(webrtc_ns2_tr, SOF_UUID(webrtc_ns2_uuid), LOG_LEVEL_INFO); +DECLARE_MODULE_ADAPTER(webrtc_ns2_interface, webrtc_ns2_uuid, webrtc_ns2_tr); +SOF_MODULE_INIT(webrtc_ns2, sys_comp_module_webrtc_ns2_interface_init); + +#endif diff --git a/src/audio/webrtc_ns2/webrtc_ns2.cmake b/src/audio/webrtc_ns2/webrtc_ns2.cmake new file mode 100644 index 000000000000..f3f4604050e3 --- /dev/null +++ b/src/audio/webrtc_ns2/webrtc_ns2.cmake @@ -0,0 +1,76 @@ +# SPDX-License-Identifier: BSD-3-Clause +# +# Cross-build RNNoise from the xiph/rnnoise repository for the webrtc_ns2 +# LLEXT module. RNNoise is pure C, requires libm (expf, tanhf, sinf, cosf, +# sqrtf), and carries built-in model weights in rnnoise_tables.c. +# +# Pinned SHA: 70f1d256acd4b34a572f999a05c87bf00b67730d (Feb 2025) +# +# Produces: +# ${WEBRTC_NS2_INSTALL_DIR}/lib/librnnoise.a +# ${WEBRTC_NS2_INSTALL_DIR}/include/rnnoise.h + +if(NOT DEFINED SOF_RNNOISE_SRC_DIR) + set(SOF_RNNOISE_SRC_DIR "${sof_top_dir}/../modules/audio/rnnoise" + CACHE PATH "xiph/rnnoise source directory (west-pinned)") +endif() +cmake_path(NORMAL_PATH SOF_RNNOISE_SRC_DIR) +if(NOT EXISTS "${SOF_RNNOISE_SRC_DIR}/src/denoise.c") + message(FATAL_ERROR + "webrtc_ns2: RNNoise source not found at '${SOF_RNNOISE_SRC_DIR}'.\n" + "Run 'west update' (rnnoise is pinned in west.yml) or pass\n" + "-DSOF_RNNOISE_SRC_DIR=.") +endif() + +get_filename_component(_tc_dir "${CMAKE_C_COMPILER}" DIRECTORY) +get_filename_component(_tc_name "${CMAKE_C_COMPILER}" NAME) +string(REGEX REPLACE "gcc$" "" _tc_prefix_name "${_tc_name}") +set(_rnn_cross_prefix "${_tc_dir}/${_tc_prefix_name}") + +set(WEBRTC_NS2_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/rnnoise-install" + CACHE INTERNAL "webrtc_ns2: RNNoise library install prefix") + +set(_rnn_src "${SOF_RNNOISE_SRC_DIR}/src") +set(_rnn_inc "${SOF_RNNOISE_SRC_DIR}/include") + +file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/rnnoise-build.sh" +"#!/bin/sh +set -e +SRC=${_rnn_src} +INC=${_rnn_inc} +INST=${WEBRTC_NS2_INSTALL_DIR} +OBJ=${CMAKE_CURRENT_BINARY_DIR}/rnnoise-obj + +mkdir -p \"$OBJ\" \"$INST/lib\" \"$INST/include\" + +CFLAGS=\"-O2 -fPIC -I$SRC -I$INC -DOUTSIDE_SPEEX -DRANDOM_PREFIX=rnn\" + +# Core RNNoise sources (excludes dump_features.c and train-only files). +SRCS=\" + $SRC/denoise.c + $SRC/rnn.c + $SRC/rnn_data.c + $SRC/pitch.c + $SRC/celt_lpc.c + $SRC/kiss_fft.c +\" + +for f in \$SRCS; do + bn=\$(basename \"\$f\" .c) + ${CMAKE_C_COMPILER} \$CFLAGS -c \"\$f\" -o \"$OBJ/\${bn}.o\" +done + +${_rnn_cross_prefix}ar rcs \"$INST/lib/librnnoise.a\" \"$OBJ\"/*.o +cp \"$INC/rnnoise.h\" \"$INST/include/\" +") + +add_custom_command( + OUTPUT "${WEBRTC_NS2_INSTALL_DIR}/lib/librnnoise.a" + "${WEBRTC_NS2_INSTALL_DIR}/include/rnnoise.h" + COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/rnnoise-build.sh" + VERBATIM) + +add_custom_target(rnnoise_ext + DEPENDS "${WEBRTC_NS2_INSTALL_DIR}/lib/librnnoise.a") + +message(STATUS "webrtc_ns2: cross-building RNNoise from ${SOF_RNNOISE_SRC_DIR}") diff --git a/src/audio/webrtc_ns2/webrtc_ns2.h b/src/audio/webrtc_ns2/webrtc_ns2.h new file mode 100644 index 000000000000..51c74ffd9da6 --- /dev/null +++ b/src/audio/webrtc_ns2/webrtc_ns2.h @@ -0,0 +1,84 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright(c) 2026 Intel Corporation. + * + * webrtc_ns2 — RNNoise deep-learning noise suppressor for SOF. + * + * Single-input, single-output PCM effect. Processes interleaved S16/S32 + * at 48 kHz in 480-sample (10 ms) mono frames. One RNNoise instance is + * allocated per channel and processes channel audio independently. + * + * In addition to denoising, the module broadcasts a NOTIFIER_ID_VAD event + * from the per-frame speech probability returned by rnnoise_process_frame(). + */ +#ifndef __SOF_AUDIO_WEBRTC_NS2_H__ +#define __SOF_AUDIO_WEBRTC_NS2_H__ + +#include +#include +#include + +/* RNNoise is strictly 48 kHz, 10 ms frames. */ +#define WEBRTC_NS2_SAMPLE_RATE 48000 +#define WEBRTC_NS2_FRAME_SAMPLES 480 /* 10 ms at 48 kHz */ +#define WEBRTC_NS2_CHANNELS_MAX CONFIG_WEBRTC_NS2_CHANNELS_MAX + +/** + * struct webrtc_ns2_backend - NS2 backend operations. + */ +struct webrtc_ns2_backend { + const char *name; + + /* One-time init called from module init(). */ + int (*init)(struct processing_module *mod); + + /** + * Open and configure one RNNoise instance per channel. + * Called from prepare(). + * @num_channels: number of channels to process + */ + int (*configure)(struct processing_module *mod, int num_channels); + + /** + * Process one 480-sample mono frame for a single channel. + * @in: 480 floats in [-1.0, +1.0] (pipeline samples normalised) + * @out: 480 floats out (denoised) + * @ch: channel index + * Returns speech probability in [0.0, 1.0], or < 0 on error. + */ + float (*process_ch)(struct processing_module *mod, + const float *in, float *out, int ch); + + /* Reset per-channel state (history buffers). Called from reset(). */ + int (*reset)(struct processing_module *mod); + + /* Deallocate all state. Called from free(). */ + int (*free)(struct processing_module *mod); +}; + +/** + * struct webrtc_ns2_comp_data - webrtc_ns2 module private data. + */ +struct webrtc_ns2_comp_data { + const struct webrtc_ns2_backend *backend; + void *backend_data; + + int channels; + bool is_s32; /* true when pipeline format is S32_LE */ + int buffered_frames; /* frames accumulated toward 480 */ + + /* Per-channel float scratch buffers (480 samples each). */ + float in_buf[WEBRTC_NS2_CHANNELS_MAX][WEBRTC_NS2_FRAME_SAMPLES]; + float out_buf[WEBRTC_NS2_CHANNELS_MAX][WEBRTC_NS2_FRAME_SAMPLES]; + + /* VAD: last decision and threshold. */ + int last_vad; + float vad_threshold; + + bool configured; +}; + +/* Backend instance provided by the selected translation unit. */ +extern const struct webrtc_ns2_backend webrtc_ns2_backend; + +#endif /* __SOF_AUDIO_WEBRTC_NS2_H__ */ diff --git a/src/audio/webrtc_ns2/webrtc_ns2.toml b/src/audio/webrtc_ns2/webrtc_ns2.toml new file mode 100644 index 000000000000..8a35dfb1c38e --- /dev/null +++ b/src/audio/webrtc_ns2/webrtc_ns2.toml @@ -0,0 +1,24 @@ +#ifndef LOAD_TYPE +#define LOAD_TYPE "0" +#endif + + REM # RNNoise deep-learning noise suppressor module config. + REM # Single input, single output. 48 kHz only. Float-only. + [[module.entry]] + name = "WRTCNS2" + uuid = UUIDREG_STR_WEBRTC_NS2 + affinity_mask = "0x1" + instance_count = "40" + domain_types = "0" + load_type = LOAD_TYPE + module_type = "9" + auto_start = "0" + sched_caps = [1, 0x00008000] + REM # pin = [dir, type, sample rate, size, container, channel-cfg] + REM # Single input (48 kHz PCM), single output. + pin = [0, 0, 0xfeef, 0xf, 0xf, 0x45ff, 1, 0, 0xfeef, 0xf, 0xf, 0x45ff] + REM # mod_cfg: RNNoise ~30KB state, ~10ms latency, 48kHz stereo S32 + REM # [PAR_0 PAR_1 PAR_2 PAR_3 IS_BYTES CPS IBS OBS MOD_FLAGS CPC OBLS] + mod_cfg = [0, 0, 0, 0, 0, 5000000, 16384, 16384, 0, 5000, 0] + + index = __COUNTER__ diff --git a/src/audio/webrtc_vad/CMakeLists.txt b/src/audio/webrtc_vad/CMakeLists.txt new file mode 100644 index 000000000000..e0fc7b39ffaf --- /dev/null +++ b/src/audio/webrtc_vad/CMakeLists.txt @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: BSD-3-Clause + +if(CONFIG_COMP_WEBRTC_VAD STREQUAL "m" AND DEFINED CONFIG_LLEXT) + add_subdirectory(llext ${PROJECT_BINARY_DIR}/webrtc_vad_llext) + add_dependencies(app webrtc_vad) +else() + add_local_sources(sof webrtc_vad.c) + + if(CONFIG_COMP_WEBRTC_VAD_STUB) + add_local_sources(sof webrtc_vad-stub.c) + else() + add_local_sources(sof webrtc_vad-fvad.c) + # NOTE: the libc symbols needed by libfvad (malloc/free/memset/...) are + # already provided by the SOF core in built-in and testbench builds. + # For LLEXT the shims are in llext/CMakeLists.txt (webrtc_vad-shims.c). + endif() +endif() diff --git a/src/audio/webrtc_vad/Kconfig b/src/audio/webrtc_vad/Kconfig new file mode 100644 index 000000000000..d90a6e6a75ec --- /dev/null +++ b/src/audio/webrtc_vad/Kconfig @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: BSD-3-Clause + +config COMP_WEBRTC_VAD + tristate "WebRTC Voice Activity Detection (libfvad)" + help + Select to include the WebRTC Voice Activity Detection module. It + wraps the libfvad library (a standalone pure-C extraction of the + WebRTC GMM VAD) behind the SOF module interface. The module + inspects each 10/20/30 ms PCM frame and annotates it as speech or + non-speech via a NOTIFIER_ID_VAD notifier event, leaving the audio + stream unmodified (pass-through). + + libfvad supports 8, 16, 32 and 48 kHz; it resamples to 8 kHz + internally. The GMM classifier operates in Q15 fixed-point and + requires no FPU. + + The real backend requires libfvad cross-built for the target + (west module: modules/audio/libfvad). Without the source, or for + CI/stub builds, select COMP_WEBRTC_VAD_STUB. + +if COMP_WEBRTC_VAD + +config COMP_WEBRTC_VAD_STUB + bool "WebRTC VAD stub backend (no libfvad dependency)" + default y if COMP_STUBS + help + Build the webrtc_vad module against a dependency-free stub backend + instead of libfvad. The stub always reports speech (VAD = 1) so + the audio data flow and LLEXT packaging can be validated without + the libfvad source or a cross-built archive. + +config WEBRTC_VAD_MODE + int "VAD aggressiveness mode (0=quality ... 3=aggressive)" + range 0 3 + default 2 + help + Sets the libfvad operating mode which controls the trade-off + between false-positive rate and missed-speech rate: + 0 - highest quality (fewest false alarms, may miss quiet speech) + 1 - moderate quality + 2 - moderate aggressiveness (default) + 3 - most aggressive (detects most speech, more false alarms) + This option is also used by the stub backend (mode value is ignored + by the stub but must be a valid integer for compilation). + +config WEBRTC_VAD_FRAME_MS + int "VAD frame length in milliseconds (10, 20 or 30)" + default 10 + help + libfvad processes audio in fixed-size frames. Valid values are + 10, 20 and 30 ms. Smaller frames give lower latency; 10 ms is + the recommended default and matches the WebRTC pipeline convention. + The SOF pipeline period must be a multiple of this value (the + module accumulates sub-frames internally if needed). + +endif # COMP_WEBRTC_VAD diff --git a/src/audio/webrtc_vad/README.md b/src/audio/webrtc_vad/README.md new file mode 100644 index 000000000000..f81a69b406fa --- /dev/null +++ b/src/audio/webrtc_vad/README.md @@ -0,0 +1,88 @@ +# WebRTC Voice Activity Detection Module (`webrtc_vad`) + +Wraps [libfvad](https://github.com/dpirch/libfvad) — a standalone, pure-C, BSD-3 licensed extraction of the WebRTC GMM Voice Activity Detection algorithm — behind the SOF `module_interface`. + +--- + +## Features + +- **GMM Classifier**: Uses Gaussian Mixture Models (GMM) for robust speech vs. noise classification. +- **Pass-through Data Flow**: Does not modify PCM audio; passes it through untouched. +- **Notifier Integration**: Broadcasts binary speech/silence decisions (`1` or `0`) via SOF's `NOTIFIER_ID_VAD` event channel. +- **Low Power & Fixed-Point**: Operates entirely in Q15 fixed-point; requires no floating-point hardware (FPU). +- **Flexible Frame Sizes**: Supports 10 ms, 20 ms, and 30 ms classification frame windows. +- **Multichannel Support**: Classifies channel-0 and copies remaining channels. +- **LLEXT Packaging**: Full support for loading as a shared library module. + +--- + +## Architecture & Data Flow + +The following Mermaid diagram outlines how audio data flows through the pipeline and how the classification events are broadcast to other modules: + +```mermaid +graph TD + %% Audio Data Flow + InBuf[Input Audio Buffer] -->|S16 or S32 PCM| Core[webrtc_vad.c Core] + Core -->|Pass-through| OutBuf[Output Audio Buffer] + + %% Accumulation & Classification + Core -->|Extract Channel 0 S16| RingBuf[Ring Buffer Accumulator] + RingBuf -->|10/20/30 ms Block| Backend[webrtc_vad-fvad.c Backend] + Backend -->|libfvad GMM Classify| Decision{Speech Detected?} + + %% Notifications + Decision -->|Yes: 1 / No: 0| Notifier[NOTIFIER_ID_VAD Event] + Notifier -->|.subscribe| SubscribedModules[e.g., webrtc_ns2, Host, KWD] +``` + +### Components +- `webrtc_vad.c`: SOF module interface glue (handles pipeline buffers and state transitions). +- `webrtc_vad-fvad.c`: Standard libfvad execution backend. +- `webrtc_vad-stub.c`: Pass-through stub that always reports `1` (speech) to satisfy CI validation. +- `webrtc_vad-shims.c`: Cross-compilation memory wrapper mappings. +- `webrtc_vad.cmake`: External build driving the download and build of the `libfvad` library. + +--- + +## Build Instructions + +### 1. Stub Mode (CI / Staging) +Building without external repository dependencies: +```ini +CONFIG_COMP_WEBRTC_VAD=y # or =m for LLEXT +CONFIG_COMP_WEBRTC_VAD_STUB=y +``` + +### 2. Real VAD Integration +Pulls `libfvad` automatically via West and compiles the full GMM detector: +```ini +CONFIG_COMP_WEBRTC_VAD=m +CONFIG_COMP_WEBRTC_VAD_STUB=n +``` +Ensure you run `west update` after updating `west.yml` to retrieve `modules/audio/libfvad`. + +--- + +## Kconfig Parameters + +| Option | Default | Range / Value | Description | +|---|---|---|---| +| `CONFIG_COMP_WEBRTC_VAD` | n | y / m / n | Enable WebRTC GMM VAD module | +| `CONFIG_COMP_WEBRTC_VAD_STUB` | y | y / n | Use pass-through stub | +| `CONFIG_WEBRTC_VAD_MODE` | 2 | 0 - 3 | Aggressiveness (3 = most restrictive) | +| `CONFIG_WEBRTC_VAD_FRAME_MS` | 10 | 10, 20, 30 | Window size for classification (ms) | + +--- + +## Usage & Topology + +### Pipeline Integration +The module functions as a simple 1-in / 1-out effect widget. Connect it directly in your capture stream path. + +#### Example Topology Route +``` +DAI Copier (Mic) ---> [ webrtc-vad ] ---> [ webrtc-ns ] ---> Host Copier +``` + +In this routing model, `webrtc-vad` runs first. Any downstream module or host application subscribing to the `NOTIFIER_ID_VAD` event will receive real-time classification changes instantly without querying the module adapter. diff --git a/src/audio/webrtc_vad/llext/CMakeLists.txt b/src/audio/webrtc_vad/llext/CMakeLists.txt new file mode 100644 index 000000000000..7d12eb109978 --- /dev/null +++ b/src/audio/webrtc_vad/llext/CMakeLists.txt @@ -0,0 +1,28 @@ +# Copyright (c) 2026 Intel Corporation. +# SPDX-License-Identifier: Apache-2.0 + +if(CONFIG_COMP_WEBRTC_VAD_STUB) + # Dependency-free build: SOF glue + passthrough stub backend, no libfvad. + sof_llext_build("webrtc_vad" + SOURCES ../webrtc_vad.c + ../webrtc_vad-stub.c + LIB openmodules + ) +else() + # Real VAD: cross-build libfvad and link it. + include(${CMAKE_CURRENT_LIST_DIR}/../webrtc_vad.cmake) + + sof_llext_build("webrtc_vad" + SOURCES ../webrtc_vad.c + ../webrtc_vad-fvad.c + ../webrtc_vad-shims.c + INCLUDES "${LIBFVAD_INSTALL_DIR}/include" + LIBS_PATH "${LIBFVAD_INSTALL_DIR}/lib" + LIBS fvad + LIB openmodules + ) + + # The archive must be built before the module compiles/links against it. + add_dependencies(webrtc_vad_llext_lib fvad_ext) + add_dependencies(webrtc_vad fvad_ext) +endif() diff --git a/src/audio/webrtc_vad/llext/llext.toml.h b/src/audio/webrtc_vad/llext/llext.toml.h new file mode 100644 index 000000000000..4975162cab08 --- /dev/null +++ b/src/audio/webrtc_vad/llext/llext.toml.h @@ -0,0 +1,6 @@ +#include +#define LOAD_TYPE "2" +#include "../webrtc_vad.toml" + +[module] +count = __COUNTER__ diff --git a/src/audio/webrtc_vad/webrtc_vad-fvad.c b/src/audio/webrtc_vad/webrtc_vad-fvad.c new file mode 100644 index 000000000000..ef1988db8cd7 --- /dev/null +++ b/src/audio/webrtc_vad/webrtc_vad-fvad.c @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. +// +// libfvad backend for the webrtc_vad module. +// +// libfvad is a standalone pure-C BSD-3 extraction of the WebRTC GMM Voice +// Activity Detection algorithm. It operates in Q15 fixed-point and requires +// no FPU, making it well suited to Xtensa DSPs and Cortex-M class targets. +// +// The backend wraps the following minimal libfvad API: +// +// fvad_new() — allocate a FvadState +// fvad_set_mode() — aggressiveness 0..3 +// fvad_set_sample_rate() — 8000/16000/32000/48000 Hz +// fvad_process() — classify num_samples of int16 mono audio +// fvad_free() — release FvadState +// +// Requires cross-compiled libfvad static library and headers (west module +// modules/audio/libfvad; see webrtc_vad.cmake for the cross-build recipe). +// Only compiled when CONFIG_COMP_WEBRTC_VAD_STUB is not selected. + +#include +#include +#include +#include "webrtc_vad.h" + +#include /* from cross-built libfvad install */ + +LOG_MODULE_DECLARE(webrtc_vad, CONFIG_SOF_LOG_LEVEL); + +static int webrtc_vad_fvad_init(struct processing_module *mod) +{ + struct webrtc_vad_comp_data *cd = module_get_private_data(mod); + Fvad *fvad; + + fvad = fvad_new(); + if (!fvad) { + comp_err(mod->dev, "webrtc_vad: fvad_new() failed"); + return -ENOMEM; + } + + cd->backend_data = fvad; + comp_info(mod->dev, "webrtc_vad: libfvad backend initialised"); + return 0; +} + +static int webrtc_vad_fvad_configure(struct processing_module *mod, + int sample_rate_hz, int mode) +{ + struct webrtc_vad_comp_data *cd = module_get_private_data(mod); + Fvad *fvad = cd->backend_data; + int ret; + + ret = fvad_set_sample_rate(fvad, sample_rate_hz); + if (ret < 0) { + comp_err(mod->dev, "webrtc_vad: fvad_set_sample_rate(%d) failed %d", + sample_rate_hz, ret); + return -EINVAL; + } + + ret = fvad_set_mode(fvad, mode); + if (ret < 0) { + comp_err(mod->dev, "webrtc_vad: fvad_set_mode(%d) failed %d", + mode, ret); + return -EINVAL; + } + + comp_info(mod->dev, "webrtc_vad: libfvad rate=%d mode=%d", + sample_rate_hz, mode); + return 0; +} + +static int webrtc_vad_fvad_classify(struct processing_module *mod, + const int16_t *samples, int num_samples) +{ + struct webrtc_vad_comp_data *cd = module_get_private_data(mod); + Fvad *fvad = cd->backend_data; + int decision; + + /* fvad_process returns 1 (speech), 0 (non-speech), or <0 on error. */ + decision = fvad_process(fvad, samples, num_samples); + if (decision < 0) { + comp_err(mod->dev, "webrtc_vad: fvad_process error %d", decision); + return -EIO; + } + + return decision; +} + +static int webrtc_vad_fvad_reset(struct processing_module *mod) +{ + struct webrtc_vad_comp_data *cd = module_get_private_data(mod); + Fvad *fvad = cd->backend_data; + + /* libfvad has no explicit reset API; re-init the state via free+new. */ + int rate = cd->sample_rate; + int mode = CONFIG_WEBRTC_VAD_MODE; + + fvad_free(fvad); + fvad = fvad_new(); + if (!fvad) + return -ENOMEM; + + cd->backend_data = fvad; + return webrtc_vad_fvad_configure(mod, rate, mode); +} + +static int webrtc_vad_fvad_free(struct processing_module *mod) +{ + struct webrtc_vad_comp_data *cd = module_get_private_data(mod); + + if (cd->backend_data) { + fvad_free(cd->backend_data); + cd->backend_data = NULL; + } + return 0; +} + +const struct webrtc_vad_backend webrtc_vad_backend = { + .name = "fvad", + .init = webrtc_vad_fvad_init, + .configure = webrtc_vad_fvad_configure, + .classify = webrtc_vad_fvad_classify, + .reset = webrtc_vad_fvad_reset, + .free = webrtc_vad_fvad_free, +}; diff --git a/src/audio/webrtc_vad/webrtc_vad-shims.c b/src/audio/webrtc_vad/webrtc_vad-shims.c new file mode 100644 index 000000000000..b2b35b6fbf8d --- /dev/null +++ b/src/audio/webrtc_vad/webrtc_vad-shims.c @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. +// +// Local libc surface for the webrtc_vad libfvad backend in LLEXT context. +// +// libfvad references a small set of libc symbols that the SOF core does not +// export to LLEXT modules. This file defines them inside the module so it +// resolves at load time. Unlike the ffmpeg_dec shims, libfvad's requirements +// are minimal — it only needs memory allocation and a handful of string/math +// helpers. +// +// NOTE: this file intentionally includes NO libc headers to avoid clashing +// with newlib prototypes; only the symbol names matter to the linker. + +#include +#include +#include + +/* ============================ Memory ============================ */ + +void *malloc(size_t size) +{ + return rballoc(SOF_MEM_FLAG_USER, size); +} + +void free(void *ptr) +{ + rfree(ptr); +} + +void *calloc(size_t nmemb, size_t size) +{ + void *p = rballoc(SOF_MEM_FLAG_USER, nmemb * size); + + if (p) + memset(p, 0, nmemb * size); + return p; +} + +/* ============================ String helpers ===================== */ + +void *memset(void *s, int c, size_t n) +{ + uint8_t *p = s; + + while (n--) + *p++ = (uint8_t)c; + return s; +} + +void *memcpy(void *dest, const void *src, size_t n) +{ + const uint8_t *s = src; + uint8_t *d = dest; + + while (n--) + *d++ = *s++; + return dest; +} + +void *memmove(void *dest, const void *src, size_t n) +{ + uint8_t *d = dest; + const uint8_t *s = src; + + if (d < s || d >= s + n) { + while (n--) + *d++ = *s++; + } else { + d += n; + s += n; + while (n--) + *--d = *--s; + } + return dest; +} + +int memcmp(const void *s1, const void *s2, size_t n) +{ + const uint8_t *a = s1, *b = s2; + + while (n--) { + if (*a != *b) + return *a - *b; + a++; b++; + } + return 0; +} diff --git a/src/audio/webrtc_vad/webrtc_vad-stub.c b/src/audio/webrtc_vad/webrtc_vad-stub.c new file mode 100644 index 000000000000..278fb3545a6a --- /dev/null +++ b/src/audio/webrtc_vad/webrtc_vad-stub.c @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. +// +// Dependency-free stub backend for the webrtc_vad module. +// +// This lets the SOF module glue (init/prepare/process/reset/free, the LLEXT +// manifest and topology wiring) be built and exercised in CI without pulling +// in libfvad. The stub always reports SPEECH (decision = 1) so audio flow +// and notifier events can be validated end-to-end. The real libfvad backend +// lives in webrtc_vad-fvad.c and provides the same webrtc_vad_backend symbol. + +#include +#include +#include "webrtc_vad.h" + +LOG_MODULE_DECLARE(webrtc_vad, CONFIG_SOF_LOG_LEVEL); + +static int webrtc_vad_stub_init(struct processing_module *mod) +{ + comp_info(mod->dev, "webrtc_vad stub backend: no libfvad linked"); + return 0; +} + +static int webrtc_vad_stub_configure(struct processing_module *mod, + int sample_rate_hz, int mode) +{ + comp_info(mod->dev, "webrtc_vad stub: rate=%d mode=%d (ignored)", + sample_rate_hz, mode); + return 0; +} + +/* Stub classifier: always reports speech. */ +static int webrtc_vad_stub_classify(struct processing_module *mod, + const int16_t *samples, int num_samples) +{ + (void)mod; (void)samples; (void)num_samples; + return WEBRTC_VAD_SPEECH; +} + +static int webrtc_vad_stub_reset(struct processing_module *mod) +{ + (void)mod; + return 0; +} + +static int webrtc_vad_stub_free(struct processing_module *mod) +{ + (void)mod; + return 0; +} + +const struct webrtc_vad_backend webrtc_vad_backend = { + .name = "stub", + .init = webrtc_vad_stub_init, + .configure = webrtc_vad_stub_configure, + .classify = webrtc_vad_stub_classify, + .reset = webrtc_vad_stub_reset, + .free = webrtc_vad_stub_free, +}; diff --git a/src/audio/webrtc_vad/webrtc_vad.c b/src/audio/webrtc_vad/webrtc_vad.c new file mode 100644 index 000000000000..30b230f9b90a --- /dev/null +++ b/src/audio/webrtc_vad/webrtc_vad.c @@ -0,0 +1,352 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. +// +// WebRTC Voice Activity Detection module — SOF module_interface core. +// +// This translation unit contains the SOF module_interface glue only; the +// actual VAD classification is delegated to the backend selected at build +// time (webrtc_vad-stub.c or webrtc_vad-fvad.c). +// +// The module is a pass-through PCM effect: audio flows from source to sink +// unmodified. On each complete VAD frame (10/20/30 ms at the configured +// rate), the module classifies channel-0 audio and broadcasts the binary +// speech/non-speech decision via NOTIFIER_ID_VAD so that downstream +// consumers (keyword detection gating, host-side VAD) can react. +// +// Because the WebRTC VAD requires a fixed 10 ms frame size which may differ +// from the SOF pipeline period, the module accumulates incoming S16 samples +// in a small ring buffer, running the classifier once per full frame. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "webrtc_vad.h" + +/* UUID identifies the component. Registered in uuid-registry.txt. */ +SOF_DEFINE_REG_UUID(webrtc_vad); + +/* Logging context. */ +LOG_MODULE_REGISTER(webrtc_vad, CONFIG_SOF_LOG_LEVEL); + +/** + * webrtc_vad_init() - Allocate private data and initialise the backend. + * @mod: Pointer to the processing module. + * + * Return: 0 on success, negative errno on failure. + */ +__cold static int webrtc_vad_init(struct processing_module *mod) +{ + struct module_data *md = &mod->priv; + struct comp_dev *dev = mod->dev; + struct webrtc_vad_comp_data *cd; + int ret; + + assert_can_be_cold(); + comp_info(dev, "webrtc_vad: init"); + + cd = mod_zalloc(mod, sizeof(*cd)); + if (!cd) + return -ENOMEM; + + md->private = cd; + cd->backend = &webrtc_vad_backend; + + comp_info(dev, "webrtc_vad: backend '%s'", cd->backend->name); + + if (cd->backend->init) { + ret = cd->backend->init(mod); + if (ret) { + comp_err(dev, "webrtc_vad: backend init failed %d", ret); + mod_free(mod, cd); + return ret; + } + } + + return 0; +} + +/** + * webrtc_vad_prepare() - Configure the VAD for the negotiated audio format. + * @mod: Pointer to the processing module. + * @sources: Array of input audio sources (exactly 1 expected). + * @num_of_sources: Must be 1. + * @sinks: Array of output sinks (exactly 1 expected). + * @num_of_sinks: Must be 1. + * + * Reads the sample rate and format from the source, configures the backend, + * and calculates the frame size for the configured frame_ms. + * + * Return: 0 on success, negative errno on failure. + */ +__cold static int webrtc_vad_prepare(struct processing_module *mod, + struct sof_source **sources, int num_of_sources, + struct sof_sink **sinks, int num_of_sinks) +{ + struct webrtc_vad_comp_data *cd = module_get_private_data(mod); + struct comp_dev *dev = mod->dev; + int rate, fmt, ret; + int frame_ms = CONFIG_WEBRTC_VAD_FRAME_MS; + + assert_can_be_cold(); + + if (num_of_sources != 1 || num_of_sinks != 1) { + comp_err(dev, "webrtc_vad: need exactly 1 source and 1 sink"); + return -EINVAL; + } + + rate = source_get_rate(sources[0]); + fmt = source_get_frm_fmt(sources[0]); + cd->channels = source_get_channels(sources[0]); + cd->sample_rate = rate; + + /* libfvad accepts 8/16/32/48 kHz. */ + if (rate != 8000 && rate != 16000 && rate != 32000 && rate != 48000) { + comp_err(dev, "webrtc_vad: unsupported rate %d", rate); + return -EINVAL; + } + + switch (fmt) { + case SOF_IPC_FRAME_S16_LE: + cd->sample_bytes = sizeof(int16_t); + break; + case SOF_IPC_FRAME_S32_LE: + cd->sample_bytes = sizeof(int32_t); + break; + default: + comp_err(dev, "webrtc_vad: unsupported frame format %d", fmt); + return -EINVAL; + } + + /* Number of samples per VAD frame (mono). */ + cd->frame_samples = (rate * frame_ms) / 1000; + if (cd->frame_samples > WEBRTC_VAD_MAX_FRAME_SAMPLES) { + comp_err(dev, "webrtc_vad: frame too large (%d samples)", cd->frame_samples); + return -EINVAL; + } + + comp_info(dev, "webrtc_vad: rate=%d ch=%d frame_ms=%d frame_samples=%d", + rate, cd->channels, frame_ms, cd->frame_samples); + + if (cd->backend->configure) { + ret = cd->backend->configure(mod, rate, CONFIG_WEBRTC_VAD_MODE); + if (ret) { + comp_err(dev, "webrtc_vad: backend configure failed %d", ret); + return ret; + } + } + + cd->accum_samples = 0; + cd->last_decision = WEBRTC_VAD_SILENCE; + cd->configured = true; + return 0; +} + +/** + * webrtc_vad_accumulate() - Extract channel-0 samples into the VAD ring buffer. + * @cd: Module private data. + * @raw: Pointer to interleaved raw audio (S16 or S32). + * @n_frames: Number of frames in @raw. + * + * Downmixes to mono by picking channel 0. For S32, shifts right 16 bits to + * produce S16 for libfvad (which always classifies at 16-bit depth). + */ +static void webrtc_vad_accumulate(struct webrtc_vad_comp_data *cd, + const void *raw, int n_frames) +{ + int ch = cd->channels; + int i; + + for (i = 0; i < n_frames && cd->accum_samples < WEBRTC_VAD_MAX_FRAME_SAMPLES; i++) { + int16_t s; + + if (cd->sample_bytes == sizeof(int16_t)) { + const int16_t *p = raw; + + s = p[i * ch]; /* channel 0 */ + } else { + const int32_t *p = raw; + + s = (int16_t)(p[i * ch] >> 16); + } + cd->accumulator[cd->accum_samples++] = s; + } +} + +/** + * webrtc_vad_classify_pending() - Drain the accumulator, classifying full frames. + * @mod: Pointer to the processing module. + * + * For each complete VAD frame accumulated, calls the backend and fires a + * NOTIFIER_ID_VAD event with the binary decision. + */ +static void webrtc_vad_classify_pending(struct processing_module *mod) +{ + struct webrtc_vad_comp_data *cd = module_get_private_data(mod); + struct comp_dev *dev = mod->dev; + int decision, remaining; + + while (cd->accum_samples >= cd->frame_samples) { + decision = cd->backend->classify(mod, cd->accumulator, + cd->frame_samples); + if (decision < 0) { + comp_warn(dev, "webrtc_vad: classify error %d", decision); + decision = cd->last_decision; + } else { + cd->last_decision = decision; + } + + /* Shift consumed samples out of the accumulator. */ + remaining = cd->accum_samples - cd->frame_samples; + if (remaining > 0) + memmove(cd->accumulator, + cd->accumulator + cd->frame_samples, + (size_t)remaining * sizeof(int16_t)); + cd->accum_samples = remaining; + + /* Broadcast VAD decision to registered listeners. */ + notifier_event(dev, NOTIFIER_ID_VAD, + NOTIFIER_TARGET_CORE_LOCAL, &decision, + sizeof(decision)); + } +} + +/** + * webrtc_vad_process() - Pass audio through and classify accumulated frames. + * @mod: Pointer to the processing module. + * @sources: Input sources (1 element). + * @num_of_sources: 1. + * @sinks: Output sinks (1 element). + * @num_of_sinks: 1. + * + * Audio is passed from source to sink byte-for-byte. Simultaneously, channel-0 + * samples are extracted from the source data buffer (before it is released) + * and appended to the accumulator for VAD classification. + * + * Return: 0 on success, negative errno on error. + */ +static int webrtc_vad_process(struct processing_module *mod, + struct sof_source **sources, int num_of_sources, + struct sof_sink **sinks, int num_of_sinks) +{ + struct webrtc_vad_comp_data *cd = module_get_private_data(mod); + struct sof_source *src = sources[0]; + struct sof_sink *snk = sinks[0]; + size_t frame_bytes = source_get_frame_bytes(src); + int avail = (int)source_get_data_frames_available(src); + int free = (int)sink_get_free_frames(snk); + int n = MIN(avail, free); + size_t nbytes; + void const *rd_ptr, *buf_start; + void *wr_ptr, *wr_buf_start; + size_t buf_size; + int ret; + + if (n <= 0) + return 0; + + nbytes = (size_t)n * frame_bytes; + + /* ---- Obtain source data pointer (read-only, zero-copy) ---- */ + ret = source_get_data(src, nbytes, &rd_ptr, &buf_start, &buf_size); + if (ret) + return ret; + + /* ---- Obtain sink write pointer ---- */ + ret = sink_get_buffer(snk, nbytes, &wr_ptr, &wr_buf_start, &buf_size); + if (ret) { + source_release_data(src, 0); /* release without consuming */ + return ret; + } + + /* ---- Pass-through copy ---- */ + memcpy_s(wr_ptr, nbytes, rd_ptr, nbytes); + + /* ---- Accumulate channel-0 samples for VAD (from source read ptr) ---- */ + webrtc_vad_accumulate(cd, rd_ptr, n); + + /* ---- Commit / release ---- */ + source_release_data(src, nbytes); + sink_commit_buffer(snk, nbytes); + + /* ---- Classify any complete frames ---- */ + webrtc_vad_classify_pending(mod); + + return 0; +} + +/** + * webrtc_vad_reset() - Flush accumulator and reset backend state. + * @mod: Pointer to the processing module. + * + * Return: 0 on success. + */ +static int webrtc_vad_reset(struct processing_module *mod) +{ + struct webrtc_vad_comp_data *cd = module_get_private_data(mod); + + comp_dbg(mod->dev, "webrtc_vad: reset"); + cd->accum_samples = 0; + cd->last_decision = WEBRTC_VAD_SILENCE; + + if (cd->backend->reset) + return cd->backend->reset(mod); + + return 0; +} + +/** + * webrtc_vad_free() - Free all resources. + * @mod: Pointer to the processing module. + * + * Return: 0 on success. + */ +__cold static int webrtc_vad_free(struct processing_module *mod) +{ + struct webrtc_vad_comp_data *cd = module_get_private_data(mod); + + assert_can_be_cold(); + comp_dbg(mod->dev, "webrtc_vad: free"); + + if (cd->backend->free) + cd->backend->free(mod); + + mod_free(mod, cd); + return 0; +} + +/* Module operations vtable. */ +static const struct module_interface webrtc_vad_interface = { + .init = webrtc_vad_init, + .prepare = webrtc_vad_prepare, + .process = webrtc_vad_process, + .reset = webrtc_vad_reset, + .free = webrtc_vad_free, +}; + +/* If COMP_WEBRTC_VAD is =m in Kconfig this is built as a loadable LLEXT. */ +#if CONFIG_COMP_WEBRTC_VAD_MODULE + +#include +#include +#include + +static const struct sof_man_module_manifest mod_manifest __section(".module") __used = + SOF_LLEXT_MODULE_MANIFEST("WRTCVAD", &webrtc_vad_interface, 1, + SOF_REG_UUID(webrtc_vad), 40); + +SOF_LLEXT_BUILDINFO; + +#else + +DECLARE_TR_CTX(webrtc_vad_tr, SOF_UUID(webrtc_vad_uuid), LOG_LEVEL_INFO); +DECLARE_MODULE_ADAPTER(webrtc_vad_interface, webrtc_vad_uuid, webrtc_vad_tr); +SOF_MODULE_INIT(webrtc_vad, sys_comp_module_webrtc_vad_interface_init); + +#endif diff --git a/src/audio/webrtc_vad/webrtc_vad.cmake b/src/audio/webrtc_vad/webrtc_vad.cmake new file mode 100644 index 000000000000..0e3177b0a2ee --- /dev/null +++ b/src/audio/webrtc_vad/webrtc_vad.cmake @@ -0,0 +1,90 @@ +# SPDX-License-Identifier: BSD-3-Clause +# +# Cross-build libfvad static library for the webrtc_vad module. +# +# libfvad is a standalone pure-C extraction of the WebRTC GMM VAD. It has +# no build system of its own beyond a trivial set of C source files, so we +# compile them directly with ExternalProject using the Zephyr cross-toolchain. +# +# Source is pulled via west from github.com/dpirch/libfvad (see west.yml). +# Produces ${LIBFVAD_INSTALL_DIR}/lib/libfvad.a and +# ${LIBFVAD_INSTALL_DIR}/include/fvad.h for the LLEXT to link. + +include(ExternalProject) + +# --- 1. Locate libfvad source (west module) --- +if(NOT DEFINED SOF_LIBFVAD_SRC_DIR) + set(SOF_LIBFVAD_SRC_DIR "${sof_top_dir}/../modules/audio/libfvad" + CACHE PATH "libfvad source tree (west-pinned)") +endif() +cmake_path(NORMAL_PATH SOF_LIBFVAD_SRC_DIR) +if(NOT EXISTS "${SOF_LIBFVAD_SRC_DIR}/src/fvad.c") + message(FATAL_ERROR + "webrtc_vad: libfvad source not found at '${SOF_LIBFVAD_SRC_DIR}'.\n" + "Run 'west update' (libfvad is pinned in west.yml) or pass " + "-DSOF_LIBFVAD_SRC_DIR=.") +endif() + +# --- 2. Derive cross toolchain prefix from Zephyr target compiler --- +# e.g. .../bin/xtensa-intel_ace30_ptl_zephyr-elf-gcc +# -> prefix .../bin/xtensa-intel_ace30_ptl_zephyr-elf- +get_filename_component(_tc_dir "${CMAKE_C_COMPILER}" DIRECTORY) +get_filename_component(_tc_name "${CMAKE_C_COMPILER}" NAME) +string(REGEX REPLACE "gcc$" "" _tc_prefix_name "${_tc_name}") +set(_fvad_cross_prefix "${_tc_dir}/${_tc_prefix_name}") + +# --- 3. Output directories --- +set(LIBFVAD_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/libfvad-install" + CACHE INTERNAL "webrtc_vad: libfvad install prefix") + +# libfvad has exactly two source files (fvad.c and the internal vad/ sources). +# Rather than invoking its CMakeLists (which would need a CMake toolchain file), +# we compile the objects directly and archive them. This mirrors the libshine +# approach in ffmpeg.cmake. +file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/fvad-build.sh" +"#!/bin/sh +set -e +export PATH=${_tc_dir}:$ENV{PATH} +SRC=${SOF_LIBFVAD_SRC_DIR} +INST=${LIBFVAD_INSTALL_DIR} +OBJ=${CMAKE_CURRENT_BINARY_DIR}/fvad-obj + +mkdir -p \"$OBJ\" \"$INST/lib\" \"$INST/include\" + +SRCS=\" + $SRC/src/fvad.c + $SRC/src/signal_processing/division_operations.c + $SRC/src/signal_processing/energy.c + $SRC/src/signal_processing/get_scaling_square.c + $SRC/src/signal_processing/resample_48khz.c + $SRC/src/signal_processing/resample_by_2_internal.c + $SRC/src/signal_processing/resample_fractional.c + $SRC/src/signal_processing/spl_inl.c + $SRC/src/vad/vad_core.c + $SRC/src/vad/vad_filterbank.c + $SRC/src/vad/vad_gmm.c + $SRC/src/vad/vad_sp.c +\" + +for f in \$SRCS; do + bn=\$(basename \"\$f\" .c) + ${CMAKE_C_COMPILER} -O2 -fPIC \ + -I$SRC/include \ + -I$SRC/src \ + -c \"\$f\" -o \"$OBJ/\${bn}.o\" +done + +${_fvad_cross_prefix}ar rcs \"$INST/lib/libfvad.a\" \"$OBJ\"/*.o +cp $SRC/include/fvad.h \"$INST/include/fvad.h\" +") + +add_custom_command( + OUTPUT "${LIBFVAD_INSTALL_DIR}/lib/libfvad.a" + "${LIBFVAD_INSTALL_DIR}/include/fvad.h" + COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/fvad-build.sh" + VERBATIM) + +add_custom_target(fvad_ext + DEPENDS "${LIBFVAD_INSTALL_DIR}/lib/libfvad.a") + +message(STATUS "webrtc_vad: cross-building libfvad from ${SOF_LIBFVAD_SRC_DIR}") diff --git a/src/audio/webrtc_vad/webrtc_vad.h b/src/audio/webrtc_vad/webrtc_vad.h new file mode 100644 index 000000000000..d6fb31ae4791 --- /dev/null +++ b/src/audio/webrtc_vad/webrtc_vad.h @@ -0,0 +1,92 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright(c) 2026 Intel Corporation. + * + * WebRTC Voice Activity Detection (VAD) module for SOF. + * + * Wraps libfvad — a standalone pure-C BSD-3 port of the WebRTC GMM VAD — + * behind the SOF module_interface. The module inspects each incoming PCM + * frame and emits a binary speech/non-speech decision as a SOF notifier + * event, leaving the audio data stream unmodified (pass-through). + * + * The actual VAD work is delegated to a pluggable backend so the SOF glue + * can be validated with a dependency-free stub (webrtc_vad-stub.c) before + * the real libfvad backend (webrtc_vad-fvad.c) is linked in. + */ +#ifndef __SOF_AUDIO_WEBRTC_VAD_H__ +#define __SOF_AUDIO_WEBRTC_VAD_H__ + +#include +#include +#include + +/* Maximum accumulation buffer: 30 ms at 48 kHz, one channel of int16. */ +#define WEBRTC_VAD_MAX_FRAME_SAMPLES (30 * 48) + +/* VAD decision values emitted via notifier. */ +#define WEBRTC_VAD_SILENCE 0 +#define WEBRTC_VAD_SPEECH 1 + +struct webrtc_vad_comp_data; + +/** + * struct webrtc_vad_backend - VAD backend operations. + * + * A backend owns the real VAD state and the frame-level classification. + * All ops return 0 on success or a negative errno; process() additionally + * returns the VAD decision (0 = silence, 1 = speech) via *decision. + */ +struct webrtc_vad_backend { + const char *name; + + /* One-time backend init, called from module init(). */ + int (*init)(struct processing_module *mod); + + /* Configure the VAD for rate/mode, called from prepare(). */ + int (*configure)(struct processing_module *mod, + int sample_rate_hz, int mode); + + /** + * Classify one complete VAD frame (frame_ms worth of int16 samples, + * mono). Returns 1 for speech, 0 for non-speech, negative on error. + */ + int (*classify)(struct processing_module *mod, + const int16_t *samples, int num_samples); + + /* Reset VAD state, keep configuration. Called from reset(). */ + int (*reset)(struct processing_module *mod); + + /* Tear down any state allocated by init()/configure(). */ + int (*free)(struct processing_module *mod); +}; + +/** + * struct webrtc_vad_comp_data - webrtc_vad module private data. + * @backend: Selected VAD backend ops. + * @backend_data: Backend-private state (e.g. FvadState*). + * @sample_rate: Input sample rate (Hz). + * @channels: Input channel count (VAD runs on channel 0 only). + * @frame_samples: Samples per VAD frame (frame_ms * sample_rate / 1000). + * @sample_bytes: Bytes per sample (2 for S16, 4 for S32). + * @accumulator: S16 mono ring buffer collecting sub-frame periods. + * @accum_samples: Number of S16 samples currently in @accumulator. + * @last_decision: Most recent VAD classification (0/1). + * @configured: True once the backend has been opened. + */ +struct webrtc_vad_comp_data { + const struct webrtc_vad_backend *backend; + void *backend_data; + int sample_rate; + int channels; + int frame_samples; + int sample_bytes; + int16_t accumulator[WEBRTC_VAD_MAX_FRAME_SAMPLES]; + int accum_samples; + int last_decision; + bool configured; +}; + +/* Backend instance provided by the selected translation unit. */ +extern const struct webrtc_vad_backend webrtc_vad_backend; + +#endif /* __SOF_AUDIO_WEBRTC_VAD_H__ */ diff --git a/src/audio/webrtc_vad/webrtc_vad.toml b/src/audio/webrtc_vad/webrtc_vad.toml new file mode 100644 index 000000000000..6081d82bd278 --- /dev/null +++ b/src/audio/webrtc_vad/webrtc_vad.toml @@ -0,0 +1,25 @@ +#ifndef LOAD_TYPE +#define LOAD_TYPE "0" +#endif + + REM # WebRTC VAD (libfvad) module config + [[module.entry]] + REM # NOTE: module name is limited to 8 chars (manifest name field); + REM # it must match the SOF_LLEXT_MODULE_MANIFEST() name in webrtc_vad.c. + name = "WRTCVAD" + uuid = UUIDREG_STR_WEBRTC_VAD + affinity_mask = "0x1" + instance_count = "40" + domain_types = "0" + load_type = LOAD_TYPE + module_type = "9" + auto_start = "0" + sched_caps = [1, 0x00008000] + REM # pin = [dir, type, sample rate, size, container, channel-cfg] + REM # Single input pin (source), single output pin (sink). PCM pass-through. + pin = [0, 0, 0xfeef, 0xf, 0xf, 0x45ff, 1, 0, 0xfeef, 0xf, 0xf, 0x45ff] + REM # mod_cfg [PAR_0 PAR_1 PAR_2 PAR_3 IS_BYTES CPS IBS OBS MOD_FLAGS CPC OBLS] + REM # VAD is pure pass-through: IBS == OBS, modest CPU (fixed-point GMM). + mod_cfg = [0, 0, 0, 0, 0, 500000, 8192, 8192, 0, 500, 0] + + index = __COUNTER__ diff --git a/src/include/sof/audio/component.h b/src/include/sof/audio/component.h index db8a69e63646..11b8023c16c0 100644 --- a/src/include/sof/audio/component.h +++ b/src/include/sof/audio/component.h @@ -954,6 +954,7 @@ void sys_comp_module_drc_interface_init(void); void sys_comp_module_dts_interface_init(void); void sys_comp_module_eq_fir_interface_init(void); void sys_comp_module_eq_iir_interface_init(void); +void sys_comp_module_ffmpeg_dec_interface_init(void); void sys_comp_module_gain_interface_init(void); void sys_comp_module_google_rtc_audio_processing_interface_init(void); void sys_comp_module_google_ctc_audio_processing_interface_init(void); diff --git a/src/include/sof/lib/notifier.h b/src/include/sof/lib/notifier.h index 87ca2cd40265..0ac19a8687b9 100644 --- a/src/include/sof/lib/notifier.h +++ b/src/include/sof/lib/notifier.h @@ -32,6 +32,7 @@ enum notify_id { NOTIFIER_ID_DMA_IRQ, /* struct dma_chan_data * */ NOTIFIER_ID_DAI_TRIGGER, /* struct dai_group * */ NOTIFIER_ID_MIC_PRIVACY_STATE_CHANGE, /* struct mic_privacy_settings * */ + NOTIFIER_ID_VAD, /* int * (WEBRTC_VAD_SPEECH / WEBRTC_VAD_SILENCE) */ NOTIFIER_ID_COUNT }; diff --git a/tools/rimage/config/lnl.toml.h b/tools/rimage/config/lnl.toml.h index faefbb8acadb..ab45dd1b9902 100644 --- a/tools/rimage/config/lnl.toml.h +++ b/tools/rimage/config/lnl.toml.h @@ -101,6 +101,9 @@ #if defined(CONFIG_COMP_FIR) || defined(LLEXT_FORCE_ALL_MODULAR) #include