80 lines
4.2 KiB
JavaScript
80 lines
4.2 KiB
JavaScript
import { voiceSettings } from './voice-settings.js';
|
|
import { ttsConfiguration, scaleSpeech } from './tts-config.js';
|
|
import { EventEmitter } from 'node:events';
|
|
import { acquireQvac, releaseQvac, loadAuxiliaryModel, unloadAuxiliaryModel, qvacSdk, withQvacMaster, assertSdkVersion } from './qvac-master.js';
|
|
|
|
export function pcmS16le(samples, sampleRate = 44_100) {
|
|
if (Array.isArray(samples)) return { samples: Int16Array.from(samples), sampleRate };
|
|
if (samples instanceof Int16Array) return { samples, sampleRate };
|
|
const bytes = toUint8(samples);
|
|
const even = bytes.byteLength - (bytes.byteLength % 2);
|
|
const copy = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + even);
|
|
return { samples: new Int16Array(copy), sampleRate };
|
|
}
|
|
|
|
function toUint8(samples) {
|
|
if (!samples) return new Uint8Array(0);
|
|
if (samples instanceof ArrayBuffer) return new Uint8Array(samples);
|
|
if (ArrayBuffer.isView(samples)) return new Uint8Array(samples.buffer, samples.byteOffset, samples.byteLength);
|
|
if (typeof samples === 'string') return Uint8Array.from(Buffer.from(samples));
|
|
if (Array.isArray(samples) || samples.length != null) return Uint8Array.from(Buffer.from(samples));
|
|
return new Uint8Array(0);
|
|
}
|
|
|
|
export class QvacVoiceAdapter extends EventEmitter {
|
|
constructor({ role = 'both', settings = voiceSettings(), asrModel = settings.asrModel, ttsModel } = {}) {
|
|
super(); this.role = role; this.settings = settings;
|
|
this.ttsConfig = role === 'asr' ? null : ttsConfiguration(settings);
|
|
this.asrModel = asrModel; this.ttsModel = ttsModel || this.ttsConfig?.model;
|
|
this.asrId = null; this.ttsId = null; this.asrSession = null; this.acquired = false;
|
|
}
|
|
|
|
async start() {
|
|
if (this.acquired) return;
|
|
assertSdkVersion();
|
|
await acquireQvac({ auxiliaryOnly: true }); this.acquired = true;
|
|
try {
|
|
if (this.role !== 'tts') this.asrId = await loadAuxiliaryModel(this.asrModel, { contextParams: { use_gpu: true }, vadModelSrc: 'VAD_SILERO_5_1_2', audio_format: 's16le', language: this.settings.asrLanguage, no_timestamps: true, vad_params: { threshold: this.settings.vadThreshold, min_speech_duration_ms: this.settings.vadMinSpeechMs, min_silence_duration_ms: this.settings.vadSilenceMs, max_speech_duration_s: this.settings.vadMaxSpeechSeconds, speech_pad_ms: 200 } }, 'whispercpp-transcription');
|
|
if (this.role !== 'asr') this.ttsId = await loadAuxiliaryModel(this.ttsModel, this.ttsConfig.config, 'tts-ggml');
|
|
} catch (error) { await this.stop(); throw error; }
|
|
}
|
|
|
|
writeAudio(chunk) { this.asrSession?.write(Buffer.from(chunk)); }
|
|
async transcribeAudio(audio) {
|
|
if (!this.asrId) throw new Error('Speech recognition is unavailable');
|
|
const sdk = await qvacSdk();
|
|
// Local VAD has already bounded this utterance. Avoid a second streaming
|
|
// VAD gate, which can discard a complete short push-to-talk recording.
|
|
return withQvacMaster(() => sdk.transcribe({ modelId: this.asrId, audioChunk: Buffer.from(audio) }));
|
|
}
|
|
async *transcripts() { if (!this.asrSession) throw new Error('voice adapter is not started'); yield* this.asrSession; }
|
|
endAudio() { this.asrSession?.end(); }
|
|
|
|
async speak(text) {
|
|
if (!this.ttsId) throw new Error('Speech output is unavailable');
|
|
const sdk = await qvacSdk();
|
|
const samples = await withQvacMaster(async () => {
|
|
const result = await sdk.textToSpeech({ modelId: this.ttsId, text: String(text).replace(/[`]/g, "'"), inputType: 'text', stream: false });
|
|
if (result?.buffer != null) return await result.buffer;
|
|
if (result?.bufferStream) {
|
|
const chunks = [];
|
|
for await (const chunk of result.bufferStream) chunks.push(Buffer.from(chunk));
|
|
return Buffer.concat(chunks);
|
|
}
|
|
return result;
|
|
});
|
|
const audio = pcmS16le(samples, this.ttsConfig.sampleRate);
|
|
audio.samples = scaleSpeech(audio.samples, this.settings.ttsVolume);
|
|
return audio;
|
|
}
|
|
|
|
async stop() {
|
|
try { this.asrSession?.destroy?.(); } catch {}
|
|
this.asrSession = null;
|
|
if (this.ttsId) await unloadAuxiliaryModel(this.ttsId).catch(() => {});
|
|
if (this.asrId) await unloadAuxiliaryModel(this.asrId).catch(() => {});
|
|
this.ttsId = this.asrId = null;
|
|
if (this.acquired) { this.acquired = false; releaseQvac(); }
|
|
}
|
|
}
|