Merge pull request #149 from huggingface/multilingual-parler

multilingual improvements for parler

Author: andimarafioti

TTS/parler_handler.py

@@ -31,6 +31,18 @@
     )
 
 
+WHISPER_LANGUAGE_TO_PARLER_SPEAKER = {
+    "en": "Jason",
+    "fr": "Christine",
+    "es": "Steven",
+    "de": "Nicole",
+    "pt": "Sophia",
+    "pl": "Alex",
+    "it": "Richard",
+    "nl": "Mark",
+}
+
+
 class ParlerTTSHandler(BaseHandler):
     def setup(
         self,
@@ -46,13 +58,19 @@ def setup(
         ),
         play_steps_s=1,
         blocksize=512,
+        use_default_speakers_list=True,
     ):
         self.should_listen = should_listen
         self.device = device
         self.torch_dtype = getattr(torch, torch_dtype)
         self.gen_kwargs = gen_kwargs
         self.compile_mode = compile_mode
         self.max_prompt_pad_length = max_prompt_pad_length
+        self.use_default_speakers_list = use_default_speakers_list
+        if self.use_default_speakers_list:
+            description = description.replace("Jenny", "")
+
+        self.speaker = "Jason"
         self.description = description
 
         self.model = ParlerTTSForConditionalGeneration.from_pretrained(
@@ -91,8 +109,12 @@ def prepare_model_inputs(
             {"padding": "max_length", "max_length": max_length_prompt} if pad else {}
         )
 
+        description = self.description
+        if self.use_default_speakers_list:
+            description = self.speaker + " " + self.description
+
         tokenized_description = self.description_tokenizer(
-            self.description, return_tensors="pt"
+            description, return_tensors="pt"
         ).to(self.device)
         input_ids = tokenized_description.input_ids
         attention_mask = tokenized_description.attention_mask
@@ -149,7 +171,8 @@ def warmup(self):
 
     def process(self, llm_sentence):
         if isinstance(llm_sentence, tuple):
-            llm_sentence, _ = llm_sentence
+            llm_sentence, language_code = llm_sentence
+            self.speaker = WHISPER_LANGUAGE_TO_PARLER_SPEAKER.get(language_code, "Jason")
 
         console.print(f"[green]ASSISTANT: {llm_sentence}")
         nb_tokens = len(self.prompt_tokenizer(llm_sentence).input_ids)

arguments_classes/parler_tts_arguments.py

@@ -4,9 +4,9 @@
 @dataclass
 class ParlerTTSHandlerArguments:
     tts_model_name: str = field(
-        default="ylacombe/parler-tts-mini-jenny-30H",
+        default="parler-tts/parler-mini-v1-jenny",
         metadata={
-            "help": "The pretrained TTS model to use. Default is 'ylacombe/parler-tts-mini-jenny-30H'."
+            "help": "The pretrained TTS model to use. Default is 'parler-tts/parler-mini-v1-jenny'."
         },
     )
     tts_device: str = field(
@@ -41,8 +41,7 @@ class ParlerTTSHandlerArguments:
     )
     description: str = field(
         default=(
-            "A female speaker with a slightly low-pitched voice delivers her words quite expressively, in a very confined sounding environment with clear audio quality. "
-            "She speaks very fast."
+            "Jenny speaks at a slightly slow pace with an animated delivery with clear audio quality."
         ),
         metadata={
             "help": "Description of the speaker's voice and speaking style to guide the TTS model."
@@ -60,3 +59,9 @@ class ParlerTTSHandlerArguments:
             "help": "When using compilation, the prompt as to be padded to closest power of 2. This parameters sets the maximun power of 2 possible."
         },
     )
+    use_default_speakers_list: bool = field(
+        default=False,
+        metadata={
+            "help": "Whether to use the default list of speakers or not."
+        },
+    )