Duplicate from neuphonic/neutts-air

Co-authored-by: Johanna Ulin <[email protected]>

.gitattributes

@@ -0,0 +1,44 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
+tokenizer_config.json filter=lfs diff=lfs merge=lfs -text
+new_tokenizer.json filter=lfs diff=lfs merge=lfs -text
+neutts-BF16.gguf filter=lfs diff=lfs merge=lfs -text
+neutts-Q8-0.gguf filter=lfs diff=lfs merge=lfs -text
+neutts-Q4_0.gguf filter=lfs diff=lfs merge=lfs -text
+neutss-air-BF16.gguf filter=lfs diff=lfs merge=lfs -text
+neutts-air-Q4-0.gguf filter=lfs diff=lfs merge=lfs -text
+neutts-air-Q8-0.gguf filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,156 @@
+---
+license: apache-2.0
+pipeline_tag: text-to-speech
+tags:
+- audio
+- speech
+- speech-language-models
+datasets:
+- amphion/Emilia-Dataset
+- neuphonic/emilia-yodas-english-neucodec
+---
+
+# NeuTTS Air ☁️ 
+
+[![NeuTTSAir_Intro](neutts-air.png)](https://www.youtube.com/watch?v=YAB3hCtu5wE)
+
+[🚀 Spaces Demo](https://huggingface.co/spaces/neuphonic/neutts-air), [🔧 Github](https://github.com/neuphonic/neutts-air)
+
+[Q8 GGUF version](https://huggingface.co/neuphonic/neutts-air-q8-gguf), [Q4 GGUF version](https://huggingface.co/neuphonic/neutts-air-q4-gguf)
+
+*Created by [Neuphonic](http://neuphonic.com/) - building faster, smaller, on-device voice AI*
+
+State-of-the-art Voice AI has been locked behind web APIs for too long. NeuTTS Air is the world’s first super-realistic, on-device, TTS speech language model with instant voice cloning. Built off a 0.5B LLM backbone, NeuTTS Air brings natural-sounding speech, real-time performance, built-in security and speaker cloning to your local device - unlocking a new category of embedded voice agents, assistants, toys, and compliance-safe apps.
+
+## Key Features
+
+- 🗣Best-in-class realism for its size - produces natural, ultra-realistic voices that sound human
+- 📱Optimised for on-device deployment - provided in GGML format, ready to run on phones, laptops, or even Raspberry Pis
+- 👫Instant voice cloning - create your own speaker with as little as 3 seconds of audio
+- 🚄Simple LM + codec architecture built off a 0.5B backbone - the sweet spot between speed, size, and quality for real-world applications
+
+
+> [!CAUTION]
+> Websites like neutts.com are popping up and they're not affliated with Neuphonic, our github or this repo.
+>
+> We are on neuphonic.com only. Please be careful out there! 🙏
+
+
+## Model Details
+
+NeuTTS Air is built off Qwen 0.5B - a lightweight yet capable language model optimised for text understanding and generation - as well as a powerful combination of technologies designed for efficiency and quality:
+
+- **Audio Codec**: [NeuCodec](https://huggingface.co/neuphonic/neucodec) - our proprietary neural audio codec that achieves exceptional audio quality at low bitrates using a single codebook
+- **Format**: Available in GGML format for efficient on-device inference
+- **Responsibility**: Watermarked outputs
+- **Inference Speed**: Real-time generation on mid-range devices
+- **Power Consumption**: Optimised for mobile and embedded devices
+
+## Get Started
+
+1. **Clone the [Git Repo](https://github.com/neuphonic/neutts-air)**
+    
+    ```bash
+    git clone https://github.com/neuphonic/neutts-air.git
+    cd neuttsair
+    ```
+    
+2. **Install `espeak` (required dependency)**
+    
+    Please refer to the following link for instructions on how to install `espeak`:
+    
+    https://github.com/espeak-ng/espeak-ng/blob/master/docs/guide.md
+    
+    ```bash
+    # Mac OS
+    brew install espeak
+    
+    # Ubuntu/Debian
+    sudo apt install espeak
+
+    # Arch Linux
+    paru -S aur/espeak
+    ```
+    
+3. **Install Python dependencies**
+    
+    The requirements file includes the dependencies needed to run the model with PyTorch. When using an ONNX decoder or a GGML model, some dependencies (such as PyTorch) are no longer required.
+    
+    The inference is compatible and tested on `python>=3.11`.
+    
+    ```
+    pip install -r requirements.txt
+    
+    ```
+    
+
+## **Basic Example**
+
+Run the basic example script to synthesize speech:
+
+```bash
+python -m examples.basic_example \
+  --input_text "My name is Dave, and um, I'm from London" \
+  --ref_audio samples/dave.wav \
+  --ref_text samples/dave.txt
+
+```
+
+To specify a particular model repo for the backbone or codec, add the `--backbone` argument. Available backbones are listed in [NeuTTS-Air huggingface collection](https://huggingface.co/collections/neuphonic/neutts-air-68cc14b7033b4c56197ef350).
+
+Several examples are available, including a Jupyter notebook in the `examples` folder.
+
+### **Simple One-Code Block Usage**
+
+```python
+from neuttsair.neutts import NeuTTSAir
+import soundfile as sf
+
+tts = NeuTTSAir( backbone_repo="neuphonic/neutts-air-q4-gguf", backbone_device="cpu", codec_repo="neuphonic/neucodec", codec_device="cpu")
+input_text = "My name is Dave, and um, I'm from London."
+
+ref_text = "samples/dave.txt"
+ref_audio_path = "samples/dave.wav"
+
+ref_text = open(ref_text, "r").read().strip()
+ref_codes = tts.encode_reference(ref_audio_path)
+
+wav = tts.infer(input_text, ref_codes, ref_text)
+sf.write("test.wav", wav, 24000)
+
+```
+
+# Tips
+
+NeuTTS Air requires two inputs:
+
+1. A reference audio sample (`.wav` file)
+2. A text string
+
+The model then synthesises the text as speech in the style of the reference audio. This is what enables NeuTTS Air’s instant voice cloning capability.
+
+### Example Reference Files
+
+You can find some ready-to-use samples in the `examples` folder:
+
+- `samples/dave.wav`
+- `samples/jo.wav`
+
+### Guidelines for Best Results
+
+For optimal performance, reference audio samples should be:
+
+1. **Mono channel**
+2. **16-44 kHz sample rate**
+3. **3–15 seconds in length**
+4. **Saved as a `.wav` file**
+5. **Clean** — minimal to no background noise
+6. **Natural, continuous speech** — like a monologue or conversation, with few pauses, so the model can capture tone effectively
+
+# **Responsibility**
+
+Every audio file generated by NeuTTS Air includes [**Perth (Perceptual Threshold) Watermarker](https://github.com/resemble-ai/perth).**
+
+# **Disclaimer**
+
+Don't use this model to do bad things… please.

config.json

@@ -0,0 +1,28 @@
+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 896,
+  "initializer_range": 0.02,
+  "intermediate_size": 4864,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 21,
+  "model_type": "qwen2",
+  "num_attention_heads": 14,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": 32768,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.50.3",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 217652
+}

generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.1,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "4.50.3"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85c7db53fbe8d62be9bc29a0743661adcb0067552488f185b5f2eb2f1ee4179f
+size 1495893752

neutss-air-BF16.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f6d562b881e64feb785a2b0a422eeadea326289fd5614990f9809ae37acd0d7
+size 1503776000

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74c466530bd698626a5b6a424d204711c58dfff0a6b3dd8b4dbac1e1e8c9aa87
+size 24140239

tokenizer_config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:364126212a294d794d83036954b0154b925c329411da93e68cdd1addeb4a5bea
+size 12065831