Merge pull request transformerlab#749 from transformerlab/add/voice-audio

Add/voice audio

Author: mina-parham

src/renderer/components/Experiment/Audio/Audio.tsx

@@ -37,6 +37,7 @@ export async function sendAndReceiveAudioPath(
   sample_rate: number,
   temperature: number,
   speed: number,
+  voice?: string,
   audioPath?: string,
 ) {
   const data: any = {
@@ -50,6 +51,10 @@ export async function sendAndReceiveAudioPath(
     speed: speed,
   };
 
+  if (voice) {
+    data.voice = voice;
+  }
+
   // Add audio path if provided
   if (audioPath) {
     data.audio_path = audioPath;
@@ -128,6 +133,23 @@ export default function Audio() {
     experimentInfo?.config?.foundation_model_architecture;
   const adaptor = experimentInfo?.config?.adaptor || '';
 
+  // Fetch model config from gallery
+  const processedModelId = currentModel
+    ? currentModel.replace(/\//g, '~~~')
+    : null;
+
+  const { data: modelData } = useAPI(
+    'models',
+    ['getModelDetailsFromGallery'],
+    {
+      modelId: processedModelId,
+    },
+    {
+      enabled: !!currentModel,
+    },
+  );
+  const modelConfigVoices = modelData?.model_config?.voices || {};
+
   const { data: audioHistory, mutate: mutateHistory } = useAPI(
     'conversations',
     ['getAudioHistory'],
@@ -145,6 +167,8 @@ export default function Audio() {
   const [filePrefix, setFilePrefix] = React.useState('output_audio');
   const [sampleRate, setSampleRate] = React.useState(24000);
   const [temperature, setTemperature] = React.useState(0.7);
+  const [selectedLanguage, setSelectedLanguage] = React.useState('');
+  const [selectedVoice, setSelectedVoice] = React.useState('');
 
   const [showSettingsModal, setShowSettingsModal] = React.useState(false);
 
@@ -173,6 +197,7 @@ export default function Audio() {
       sampleRate,
       temperature,
       speed,
+      selectedVoice || undefined,
       uploadedAudioPath || undefined,
     );
 
@@ -270,6 +295,48 @@ export default function Audio() {
             Generation Settings:
           </Typography>
           <Stack spacing={3} sx={{ py: 2 }}>
+            {/* Voice Selection */}
+            {Object.keys(modelConfigVoices).length > 0 && (
+              <>
+                <FormControl>
+                  <FormLabel>Language</FormLabel>
+                  <Select
+                    value={selectedLanguage}
+                    onChange={(_, v) => {
+                      setSelectedLanguage(v as string);
+                      setSelectedVoice(''); // Reset voice when language changes
+                    }}
+                    placeholder="Select language..."
+                  >
+                    {Object.keys(modelConfigVoices).map((language) => (
+                      <Option key={language} value={language}>
+                        {language}
+                      </Option>
+                    ))}
+                  </Select>
+                </FormControl>
+
+                {selectedLanguage && modelConfigVoices[selectedLanguage] && (
+                  <FormControl>
+                    <FormLabel>Voice</FormLabel>
+                    <Select
+                      value={selectedVoice}
+                      onChange={(_, v) => setSelectedVoice(v as string)}
+                      placeholder="Select voice..."
+                    >
+                      {modelConfigVoices[selectedLanguage].map(
+                        (voice: string) => (
+                          <Option key={voice} value={voice}>
+                            {voice}
+                          </Option>
+                        ),
+                      )}
+                    </Select>
+                  </FormControl>
+                )}
+              </>
+            )}
+
             {/* Sample Rate */}
             <FormControl>
               <FormLabel>Sample Rate</FormLabel>

src/renderer/components/Experiment/Audio/AudioHistory.tsx

@@ -13,7 +13,7 @@ import { getAPIFullPath } from 'renderer/lib/transformerlab-api-sdk';
 import AudioPlayer from '../../Data/AudioPlayer';
 
 interface AudioHistoryItem {
-  id: string; // Added id property
+  id: string;
   type: string;
   text: string;
   filename: string;
@@ -23,6 +23,7 @@ interface AudioHistoryItem {
   audio_format: string;
   sample_rate: number;
   temperature: number;
+  voice?: string;
   audio_data_url?: string; // Add audio data URL for the AudioPlayer
 }
 
@@ -103,6 +104,11 @@ const AudioHistory = React.forwardRef<HTMLDivElement, AudioHistoryProps>(
                       <> + {item.adaptor.split('/').pop()}</>
                     )}
                   </Chip>
+                  {item.voice && (
+                    <Chip size="sm" variant="soft" color="neutral">
+                      Voice: {item.voice}
+                    </Chip>
+                  )}
                   <Chip size="sm" variant="soft" color="neutral">
                     {item.audio_format.toUpperCase()}
                   </Chip>

src/renderer/lib/api-client/allEndpoints.json

@@ -81,6 +81,10 @@
     "pipeline_tag": {
       "method": "GET",
       "path": "model/pipeline_tag?model_name={modelName}"
+    },
+    "getModelDetailsFromGallery": {
+      "method": "GET",
+      "path": "model/gallery/{modelId}"
     }
   },
   "jobs": {