metadata
license: mit
language:
- en
tags:
- audio
- text-to-speech
- matcha-tts
Matcha-TTS CommonVoice EN001
Source Audio
https://commonvoice.mozilla.org/en/datasets Common Voice Corpus 1
I called audios 42da7f26(head-audio-id)_290(files) EN001 (No plan to include audios in this repo)
Any Good point?
LJSpeech is much better quality,but it's female voice.
VCTK 107 voices are similar quality,but that is ODC-By License.
This audio is just under MIT more easy to continue training or something.
however I recommend you use VCTK,ODC-By License is not so problem.I'm going to create new voices with this.
How to Train
Train with IPA text(this folk) https://github.com/akjava/Matcha-TTS-Japanese
check this repo's config files. however there are no audio copy tools.TODO later
Files Info
checkpoints
Matcha-TTS checkpoint - epoch seems big but train with only 290 audios
see Training metrics
ONNX
onnx simplified loading speed is now 1.5 times faster.
from onnxsim import simplify
import onnx
model = onnx.load("en001_6399_T2.onnx")
model_simp, check = simplify(model)
onnx.save(model_simp, "en001_6399_T2_simplify.onnx")
timesteps is default(5) ,small time steps ;The infer speed is somewhat faster, but the quality is lower.
If you need original onnx do like official way
python -m matcha.onnx.export checkpoint_epoch=5699.ckpt en001_5699t2.onnx --vocoder-name hifigan_T2_v1 --n-timesteps 5 --vocoder-checkpoint generator_v1
python -m matcha.onnx.export checkpoint_epoch=5699.ckpt en001_5699.onnx --vocoder-name hifigan_univ_v1 --n-timesteps 5 --vocoder-checkpoint g_02500000
- T2 means Vocoder is hifigan_T2_v1
- Unif means Voder is hifigan_univ_v1
you can quantize this onnx,but 3 times smaller, but 4-5 times slower,that why I did't include that.
from onnxruntime.quantization import quantize_dynamic, QuantType
quantized_model = quantize_dynamic(src_model_path, dst_model_path, weight_type=QuantType.QUInt8)
To use onnx need something,below is old sample
const _pad = "_";
const _punctuation = ";:,.!?¡¿—…\"«»“” ";
const _letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
const _letters_ipa = "ɑɐɒæɓʙβɔɕçɗɖðʤəɘɚɛɜɝɞɟʄɡɠɢʛɦɧħɥʜɨɪʝɭɬɫɮʟɱɯɰŋɳɲɴøɵɸθœɶʘɹɺɾɻʀʁɽʂʃʈʧʉʊʋⱱʌɣɤʍχʎʏʑʐʒʔʡʕʢǀǁǂǃˈˌːˑʼʴʰʱʲʷˠˤ˞↓↑→↗↘'̩'ᵻ";
// below code called Spread syntax
const Symbols = [_pad, ..._punctuation, ..._letters, ..._letters_ipa];
const SpaceId = Symbols.indexOf(' ');
const symbolToId = {};
const idToSymbol = {};
// initialize symbolToId and idToSymbol
for (let i = 0; i < Symbols.length; i++) {
symbolToId[Symbols[i]] = i;
idToSymbol[i] = Symbols[i];
}
class MatchaOnnx {
constructor() {
}
async load_model(model_path,options={}){
this.session = await ort.InferenceSession.create(model_path,options);
}
get_output_names_html(){
if (typeof this.session=='undefined'){
return null
}
let outputNamesString = '[outputs]<br>';
const outputNames = this.session.outputNames;
for (let outputName of outputNames) {
console.log(outputName)
outputNamesString+=outputName+"<br>"
}
return outputNamesString.trim()
}
get_input_names_html(){
if (typeof this.session=='undefined'){
return null
}
let inputNamesString = '[Inputs]<br>';
const inputNames = this.session.inputNames;
for (let inputName of inputNames) {
console.log(inputName)
inputNamesString+=inputName+"<br>"
}
return inputNamesString.trim()
}
processText(text) {
const x = this.intersperse(this.textToSequence(text));
const x_phones = this.sequenceToText(x);
const textList = [];
for (let i = 1; i < x_phones.length; i += 2) {
textList.push(x_phones[i]);
}
return {
x: x,
x_length: x.length,
x_phones: x_phones,
x_phones_label: textList.join(""),
};
}
basicCleaners2(text, lowercase = false) {
if (lowercase) {
text = text.toLowerCase();
}
text = text.replace(/\s+/g, " ");
return text;
}
textToSequence(text) {
const sequenceList = [];
const clean_text = this.basicCleaners2(text);
for (let i = 0; i < clean_text.length; i++) {
const symbol = clean_text[i];
sequenceList.push(symbolToId[symbol]);
}
return sequenceList;
}
intersperse(sequence, item = 0) {
const sequenceList = [item];
for (let i = 0; i < sequence.length; i++) {
sequenceList.push(sequence[i]);
sequenceList.push(item);
}
return sequenceList;
}
sequenceToText(sequence) {
const textList = [];
for (let i = 0; i < sequence.length; i++) {
const symbol = idToSymbol[sequence[i]];
textList.push(symbol);
}
return textList.join("");
}
async infer(text, temperature, speed) {
console.log(this.session)
const dic = this.processText(text);
console.log(`x:${dic.x.join(", ")}`);
console.log(`x_length:${dic.x_length}`);
console.log(`x_phones_label:${dic.x_phones_label}`);
// Prepare input tensors (assuming your ONNX Runtime library uses similar syntax)
//const x_tensor = new this.session.Tensor('long', dic.x, [1, dic.x.length]);
//const x_length_tensor = new this.session.Tensor('long', [dic.x.length], [1]);
//const scales_tensor = new this.session.Tensor('float', [temperature, speed], [2]);
const dataX = new BigInt64Array(dic.x.length)
for (let i = 0; i < dic.x.length; i++) {
//console.log(dic.x[i])
dataX[i] = BigInt(dic.x[i]); // Convert each number to a BigInt
}
const data_x_length = new BigInt64Array(1)
data_x_length[0] = BigInt(dic.x_length)
//const dataX = Int32Array.from([dic.x_length])
const tensorX = new ort.Tensor('int64', dataX, [1, dic.x.length]);
// const data_x_length = Int32Array.from([dic.x_length])
const tensor_x_length = new ort.Tensor('int64', data_x_length, [1]);
const data_scale = Float32Array.from( [temperature, speed])
const tensor_scale = new ort.Tensor('float32', data_scale, [2]);
// Run inference
const output = await this.session.run({
x: tensorX,
x_lengths: tensor_x_length,
scales: tensor_scale,
});
console.log(output)
// Extract output (assuming your ONNX Runtime library uses similar syntax)
const wav_array = output.wav.data;
console.log(wav_array[0]);
console.log(wav_array.length);
const x_lengths_array = output.wav_lengths.data;
console.log(x_lengths_array.join(", "));
return wav_array;
}
}
convert to wav
function webWavPlay(f32array){
blob = float32ArrayToWav(f32array)
url = createObjectUrlFromBlob(blob)
console.log(url)
playAudioFromUrl(url)
}
function createObjectUrlFromBlob(blob) {
const url = URL.createObjectURL(blob);
return url;
}
function playAudioFromUrl(url) {
const audio = new Audio(url);
audio.play().catch(error => console.error('Failed to play audio:', error));
}
//I copied
//https://huggingface.co/spaces/k2-fsa/web-assembly-tts-sherpa-onnx-de/blob/main/app-tts.js
// this function is copied/modified from
// https://gist.github.com/meziantou/edb7217fddfbb70e899e
function float32ArrayToWav(floatSamples, sampleRate=22050) {
let samples = new Int16Array(floatSamples.length);
for (let i = 0; i < samples.length; ++i) {
let s = floatSamples[i];
if (s >= 1)
s = 1;
else if (s <= -1)
s = -1;
samples[i] = s * 32767;
}
let buf = new ArrayBuffer(44 + samples.length * 2);
var view = new DataView(buf);
// http://soundfile.sapp.org/doc/WaveFormat/
// F F I R
view.setUint32(0, 0x46464952, true); // chunkID
view.setUint32(4, 36 + samples.length * 2, true); // chunkSize
// E V A W
view.setUint32(8, 0x45564157, true); // format
//
// t m f
view.setUint32(12, 0x20746d66, true); // subchunk1ID
view.setUint32(16, 16, true); // subchunk1Size, 16 for PCM
view.setUint32(20, 1, true); // audioFormat, 1 for PCM
view.setUint16(22, 1, true); // numChannels: 1 channel
view.setUint32(24, sampleRate, true); // sampleRate
view.setUint32(28, sampleRate * 2, true); // byteRate
view.setUint16(32, 2, true); // blockAlign
view.setUint16(34, 16, true); // bitsPerSample
view.setUint32(36, 0x61746164, true); // Subchunk2ID
view.setUint32(40, samples.length * 2, true); // subchunk2Size
let offset = 44;
for (let i = 0; i < samples.length; ++i) {
view.setInt16(offset, samples[i], true);
offset += 2;
}
return new Blob([view], {type: 'audio/wav'});
}
Audio
I cut with VAD tools and denoise with resemble-enhance