diff --git a/README.md b/README.md index 4e9858b..eb422cc 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,7 @@ await nodewhisper(filePath, { modelName: 'base.en', //Downloaded models name modelRootPath: '/path/to/whisper/models', // (optional) directory containing the selected ggml model file autoDownloadModelName: 'base.en', // (optional) auto download a model if model is not present + autoDownloadVadModelName: 'silero-v6.2.0', // (optional) download and enable a Silero VAD model removeWavFileAfterTranscription: false, // (optional) remove wav file once transcribed withCuda: false, // (optional) use cuda for faster processing logger: console, // (optional) Logging instance, defaults to console @@ -87,6 +88,12 @@ await nodewhisper(filePath, { timestamps_length: 20, // amount of dialogue per timestamp pair splitOnWord: true, // split on word rather than on token noGpu: false, // disable GPU inference + vadThreshold: 0.5, // speech detection probability threshold + vadMinSpeechDurationMs: 250, // discard shorter speech segments + vadMinSilenceDurationMs: 100, // silence required to split segments + vadMaxSpeechDurationS: 30, // split speech segments longer than this + vadSpeechPadMs: 30, // padding around detected speech + vadSamplesOverlap: 0.1, // overlap between speech segments in seconds }, }) @@ -158,6 +165,26 @@ await nodewhisper(filePath, { The downloaded model will be stored at `/data/whisper-models/ggml-tiny.en.bin`, while the package's internal downloader scripts remain available. +### Voice activity detection + +VAD detects speech before transcription, which can reduce work on long recordings with silence. The recommended +Silero model is less than 1 MB and can be downloaded automatically: + +```javascript +await nodewhisper(filePath, { + modelName: 'tiny.en', + autoDownloadModelName: 'tiny.en', + autoDownloadVadModelName: 'silero-v6.2.0', + whisperOptions: { + vadThreshold: 0.5, + vadMinSilenceDurationMs: 100, + }, +}) +``` + +Providing `autoDownloadVadModelName` enables VAD automatically. To use an existing or custom VAD model instead, +set `whisperOptions.vad` to `true` and provide its path through `whisperOptions.vadModelPath`. + ## Types ``` @@ -167,6 +194,7 @@ The downloaded model will be stored at `/data/whisper-models/ggml-tiny.en.bin`, removeWavFileAfterTranscription?: boolean withCuda?: boolean autoDownloadModelName?: string + autoDownloadVadModelName?: 'silero-v5.1.2' | 'silero-v6.2.0' whisperOptions?: WhisperOptions logger?: Console } @@ -185,6 +213,14 @@ The downloaded model will be stored at `/data/whisper-models/ggml-tiny.en.bin`, wordTimestamps?: boolean splitOnWord?: boolean noGpu?: boolean + vad?: boolean + vadModelPath?: string + vadThreshold?: number + vadMinSpeechDurationMs?: number + vadMinSilenceDurationMs?: number + vadMaxSpeechDurationS?: number + vadSpeechPadMs?: number + vadSamplesOverlap?: number } ``` @@ -233,8 +269,9 @@ Run the end-to-end transcription test npm run test:integration ``` -The integration test downloads and builds `tiny.en` when needed, transcribes the bundled audio sample, verifies the -returned transcript and VTT file, and checks that whisper.cpp output is routed through the configured logger. +The integration test downloads and builds `tiny.en` when needed, downloads the Silero VAD model, transcribes the +bundled audio sample with VAD enabled, verifies the returned transcript and VTT file, and checks that whisper.cpp +output is routed through the configured logger. ## Made with diff --git a/src/WhisperHelper.ts b/src/WhisperHelper.ts index 799f4c0..8911bf5 100644 --- a/src/WhisperHelper.ts +++ b/src/WhisperHelper.ts @@ -56,14 +56,6 @@ export const constructCommand = (filePath: string, args: IOptions): string => { throw new Error('[Nodejs-whisper] Error: whisper-cli executable not found') } - // Construct command with proper path escaping - const escapeArg = (arg: string) => { - if (process.platform === 'win32') { - return `"${arg.replace(/"/g, '\\"')}"` - } - return `"${arg}"` - } - const modelArg = args.modelRootPath ? modelPath : `./models/${modelFileName}` let command = `${escapeArg(executablePath)} ${constructOptionsFlags(args)} -l ${args.whisperOptions?.language || 'auto'} -m ${escapeArg(modelArg)} -f ${escapeArg(filePath)}` @@ -72,6 +64,7 @@ export const constructCommand = (filePath: string, args: IOptions): string => { } const constructOptionsFlags = (args: IOptions): string => { + const vadFlags = constructVadFlags(args) let flags = [ args.whisperOptions?.outputInCsv ? '-ocsv ' : '', args.whisperOptions?.outputInJson ? '-oj ' : '', @@ -86,7 +79,68 @@ const constructOptionsFlags = (args: IOptions): string => { args.whisperOptions?.timestamps_length ? `-ml ${args.whisperOptions.timestamps_length} ` : '', args.whisperOptions?.splitOnWord ? '-sow ' : '', args.whisperOptions?.noGpu ? '-ng ' : '', + vadFlags, ].join('') return flags.trim() } + +const constructVadFlags = (args: IOptions): string => { + const options = args.whisperOptions + if (!options?.vad) { + return '' + } + + if (!options.vadModelPath) { + throw new Error('[Nodejs-whisper] Error: VAD requires whisperOptions.vadModelPath or autoDownloadVadModelName.') + } + + const vadModelPath = path.resolve(options.vadModelPath) + if (!fs.existsSync(vadModelPath)) { + throw new Error(`[Nodejs-whisper] Error: VAD model file does not exist at ${vadModelPath}.`) + } + + validateNumber('vadThreshold', options.vadThreshold, 0, 1) + validateNumber('vadMinSpeechDurationMs', options.vadMinSpeechDurationMs, 0, undefined, true) + validateNumber('vadMinSilenceDurationMs', options.vadMinSilenceDurationMs, 0, undefined, true) + validateNumber('vadMaxSpeechDurationS', options.vadMaxSpeechDurationS, Number.MIN_VALUE) + validateNumber('vadSpeechPadMs', options.vadSpeechPadMs, 0, undefined, true) + validateNumber('vadSamplesOverlap', options.vadSamplesOverlap, 0) + + return [ + `--vad -vm ${escapeArg(vadModelPath)} `, + optionFlag('-vt', options.vadThreshold), + optionFlag('-vspd', options.vadMinSpeechDurationMs), + optionFlag('-vsd', options.vadMinSilenceDurationMs), + optionFlag('-vmsd', options.vadMaxSpeechDurationS), + optionFlag('-vp', options.vadSpeechPadMs), + optionFlag('-vo', options.vadSamplesOverlap), + ].join('') +} + +const optionFlag = (flag: string, value?: number): string => (value === undefined ? '' : `${flag} ${value} `) + +const validateNumber = (name: string, value: number | undefined, min: number, max?: number, integer = false) => { + if (value === undefined) { + return + } + + if ( + !Number.isFinite(value) || + value < min || + (max !== undefined && value > max) || + (integer && !Number.isInteger(value)) + ) { + const range = max === undefined ? `at least ${min}` : `between ${min} and ${max}` + throw new Error( + `[Nodejs-whisper] Error: whisperOptions.${name} must be ${range}${integer ? ' and an integer' : ''}.` + ) + } +} + +const escapeArg = (arg: string) => { + if (process.platform === 'win32') { + return `"${arg.replace(/"/g, '\\"')}"` + } + return `"${arg}"` +} diff --git a/src/autoDownloadVadModel.ts b/src/autoDownloadVadModel.ts new file mode 100644 index 0000000..b229699 --- /dev/null +++ b/src/autoDownloadVadModel.ts @@ -0,0 +1,112 @@ +import fs from 'fs' +import https from 'https' +import path from 'path' +import { IncomingMessage } from 'http' +import { pipeline } from 'stream' +import { promisify } from 'util' +import { isVadModelName, VAD_MODEL_OBJECT, VadModelName, WHISPER_CPP_PATH } from './constants' +import { Logger } from './types' + +const pipelineAsync = promisify(pipeline) +const VAD_MODEL_BASE_URL = 'https://huggingface.co/ggml-org/whisper-vad/resolve/main' + +export default async function autoDownloadVadModel( + logger: Logger = console, + modelName: VadModelName, + modelRootPath?: string +): Promise { + if (!isVadModelName(modelName)) { + throw new Error('[Nodejs-whisper] Error: Provide a valid VAD model name') + } + + const modelDirectory = modelRootPath ? path.resolve(modelRootPath) : path.join(WHISPER_CPP_PATH, 'models') + const modelPath = path.join(modelDirectory, VAD_MODEL_OBJECT[modelName]) + + fs.mkdirSync(modelDirectory, { recursive: true }) + + if (isValidVadModel(modelPath)) { + logger.debug(`[Nodejs-whisper] ${modelName} already exists. Skipping download.`) + return modelPath + } + + if (fs.existsSync(modelPath)) { + throw new Error(`[Nodejs-whisper] Existing VAD model is invalid: ${modelPath}`) + } + + logger.debug(`[Nodejs-whisper] Auto-download VAD model: ${modelName}`) + const modelUrl = `${VAD_MODEL_BASE_URL}/${VAD_MODEL_OBJECT[modelName]}` + + try { + await downloadFile(modelUrl, modelPath) + } catch (error) { + throw new Error(`[Nodejs-whisper] Failed to download VAD model: ${error.message}`) + } + + if (!isValidVadModel(modelPath)) { + fs.unlinkSync(modelPath) + throw new Error('[Nodejs-whisper] Failed to download VAD model: downloaded file is invalid') + } + + logger.debug(`[Nodejs-whisper] VAD model downloaded to ${modelPath}`) + return modelPath +} + +async function downloadFile(url: string, destination: string): Promise { + const temporaryPath = `${destination}.${process.pid}.download` + + try { + const response = await getResponse(url) + await pipelineAsync(response, fs.createWriteStream(temporaryPath, { flags: 'wx' })) + fs.renameSync(temporaryPath, destination) + } catch (error) { + if (fs.existsSync(temporaryPath)) { + fs.unlinkSync(temporaryPath) + } + throw error + } +} + +function getResponse(url: string, redirectsRemaining = 5): Promise { + return new Promise((resolve, reject) => { + const request = https.get(url, response => { + const statusCode = response.statusCode || 0 + const redirectUrl = response.headers.location + + if (statusCode >= 300 && statusCode < 400 && redirectUrl) { + response.resume() + if (redirectsRemaining === 0) { + reject(new Error('too many redirects')) + return + } + getResponse(new URL(redirectUrl, url).toString(), redirectsRemaining - 1).then(resolve, reject) + return + } + + if (statusCode !== 200) { + response.resume() + reject(new Error(`download request returned HTTP ${statusCode}`)) + return + } + + resolve(response) + }) + + request.setTimeout(60_000, () => request.destroy(new Error('download request timed out'))) + request.on('error', reject) + }) +} + +function isValidVadModel(modelPath: string): boolean { + if (!fs.existsSync(modelPath) || fs.statSync(modelPath).size < 4) { + return false + } + + const file = fs.openSync(modelPath, 'r') + try { + const magic = Buffer.alloc(4) + fs.readSync(file, magic, 0, magic.length, 0) + return magic.readUInt32LE(0) === 0x67676d6c + } finally { + fs.closeSync(file) + } +} diff --git a/src/constants.ts b/src/constants.ts index 3b6abcb..6281d12 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -52,6 +52,16 @@ export const isModelName = (modelName: unknown): modelName is ModelName => export const DEFAULT_MODEL = 'tiny.en' +export const VAD_MODEL_OBJECT = { + 'silero-v5.1.2': 'ggml-silero-v5.1.2.bin', + 'silero-v6.2.0': 'ggml-silero-v6.2.0.bin', +} + +export type VadModelName = keyof typeof VAD_MODEL_OBJECT + +export const isVadModelName = (modelName: unknown): modelName is VadModelName => + typeof modelName === 'string' && Object.prototype.hasOwnProperty.call(VAD_MODEL_OBJECT, modelName) + export const WHISPER_CPP_PATH = path.join(__dirname, '..', 'cpp', 'whisper.cpp') export const WHISPER_CPP_MAIN_PATH = diff --git a/src/index.d.ts b/src/index.d.ts index e13c40b..cfd60e6 100644 --- a/src/index.d.ts +++ b/src/index.d.ts @@ -8,12 +8,23 @@ export interface WhisperOptions { wordTimestamps?: boolean splitOnWord?: boolean noGpu?: boolean + vad?: boolean + vadModelPath?: string + vadThreshold?: number + vadMinSpeechDurationMs?: number + vadMinSilenceDurationMs?: number + vadMaxSpeechDurationS?: number + vadSpeechPadMs?: number + vadSamplesOverlap?: number } +export type VadModelName = 'silero-v5.1.2' | 'silero-v6.2.0' + export interface IOptions { modelName: string modelRootPath?: string autoDownloadModelName?: string + autoDownloadVadModelName?: VadModelName whisperOptions?: WhisperOptions withCuda?: boolean removeWavFileAfterTranscription?: boolean diff --git a/src/index.ts b/src/index.ts index f71cf2a..7ea0666 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,11 +4,16 @@ import fs from 'fs' import { constructCommand } from './WhisperHelper' import { checkIfFileExists, convertToWavType } from './utils' import autoDownloadModel from './autoDownloadModel' +import autoDownloadVadModel from './autoDownloadVadModel' +import { VadModelName } from './constants' + +export type { VadModelName } from './constants' export interface IOptions { modelName: string modelRootPath?: string autoDownloadModelName?: string + autoDownloadVadModelName?: VadModelName whisperOptions?: WhisperOptions withCuda?: boolean removeWavFileAfterTranscription?: boolean @@ -19,6 +24,8 @@ export async function nodewhisper(filePath: string, options: IOptions) { const { removeWavFileAfterTranscription = false, logger = console } = options try { + let runtimeOptions = options + if (options.autoDownloadModelName) { logger.debug(`[Nodejs-whisper] Checking and downloading model if needed: ${options.autoDownloadModelName}`) @@ -28,6 +35,25 @@ export async function nodewhisper(filePath: string, options: IOptions) { await autoDownloadModel(logger, options.autoDownloadModelName, options.withCuda, options.modelRootPath) } + if (options.autoDownloadVadModelName) { + logger.debug( + `[Nodejs-whisper] Checking and downloading VAD model if needed: ${options.autoDownloadVadModelName}` + ) + const vadModelPath = await autoDownloadVadModel( + logger, + options.autoDownloadVadModelName, + options.modelRootPath + ) + runtimeOptions = { + ...options, + whisperOptions: { + ...options.whisperOptions, + vad: true, + vadModelPath, + }, + } + } + logger.debug(`[Nodejs-whisper] Checking file existence: ${filePath}`) checkIfFileExists(filePath) @@ -35,7 +61,7 @@ export async function nodewhisper(filePath: string, options: IOptions) { const outputFilePath = await convertToWavType(filePath, logger) logger.debug(`[Nodejs-whisper] Constructing command for file: ${outputFilePath}`) - const command = constructCommand(outputFilePath, options) + const command = constructCommand(outputFilePath, runtimeOptions) logger.debug(`[Nodejs-whisper] Executing command: ${command}`) const transcript = await executeCppCommand(command, logger, options.withCuda) diff --git a/src/types.ts b/src/types.ts index fb70a6e..eae19a5 100644 --- a/src/types.ts +++ b/src/types.ts @@ -13,6 +13,14 @@ export interface WhisperOptions { wordTimestamps?: boolean splitOnWord?: boolean noGpu?: boolean + vad?: boolean + vadModelPath?: string + vadThreshold?: number + vadMinSpeechDurationMs?: number + vadMinSilenceDurationMs?: number + vadMaxSpeechDurationS?: number + vadSpeechPadMs?: number + vadSamplesOverlap?: number } export interface Logger { diff --git a/test/core.integration.js b/test/core.integration.js index 741d188..0781220 100644 --- a/test/core.integration.js +++ b/test/core.integration.js @@ -4,15 +4,16 @@ const path = require('node:path') const test = require('node:test') const { nodewhisper } = require('../src/index') -const { MODEL_OBJECT, WHISPER_CPP_PATH } = require('../src/constants') +const { MODEL_OBJECT, VAD_MODEL_OBJECT, WHISPER_CPP_PATH } = require('../src/constants') test( - 'transcribes audio with tiny.en and routes whisper.cpp output through the logger', + 'transcribes audio with tiny.en and VAD while routing whisper.cpp output through the logger', { timeout: 10 * 60 * 1000 }, async t => { const audioFile = path.resolve(__dirname, '../example/mother_teresa.wav') const outputFile = `${audioFile}.vtt` const modelFile = path.join(WHISPER_CPP_PATH, 'models', MODEL_OBJECT['tiny.en']) + const vadModelFile = path.join(WHISPER_CPP_PATH, 'models', VAD_MODEL_OBJECT['silero-v6.2.0']) const loggerEvents = [] const leakedWhisperOutput = [] const logger = { @@ -44,12 +45,19 @@ test( transcript = await nodewhisper(audioFile, { modelName: 'tiny.en', autoDownloadModelName: 'tiny.en', + autoDownloadVadModelName: 'silero-v6.2.0', logger, whisperOptions: { noGpu: true, outputInVtt: true, splitOnWord: true, timestamps_length: 14, + vadThreshold: 0.5, + vadMinSpeechDurationMs: 250, + vadMinSilenceDurationMs: 100, + vadMaxSpeechDurationS: 30, + vadSpeechPadMs: 30, + vadSamplesOverlap: 0.1, }, }) } finally { @@ -58,6 +66,7 @@ test( } const normalizedTranscript = transcript.replace(/\s+/g, ' ').toLowerCase() + const transcriptText = normalizedTranscript.replace(/\[[^\]]+\]/g, ' ').replace(/\s+/g, ' ') const debugOutput = loggerEvents .filter(([level]) => level === 'debug') .flatMap(([, ...args]) => args) @@ -68,12 +77,16 @@ test( .join(' ') assert.equal(fs.existsSync(modelFile), true, 'tiny.en model should be available') - assert.match(normalizedTranscript, /i do not want.*your money/) - assert.match(normalizedTranscript, /i want your.*forgiveness/) + assert.equal(fs.existsSync(vadModelFile), true, 'Silero VAD model should be available') + assert.match(transcriptText, /i do not want.*your money/) + assert.match(transcriptText, /i want your/) assert.match(loggedTranscript.toLowerCase(), /i do not want/) assert.match(debugOutput, /whisper_init_with_params_no_state:/) assert.match(debugOutput, /-sow(?:\s|$)/) assert.doesNotMatch(debugOutput, /-sow\s+true/) + assert.match(debugOutput, /--vad\s+-vm/) + assert.match(debugOutput, /ggml-silero-v6\.2\.0\.bin/) + assert.match(debugOutput, /whisper_vad_init_from_file_with_params/) assert.deepEqual(leakedWhisperOutput, []) assert.equal(fs.existsSync(outputFile), true, 'VTT output should be created')