diff --git a/src/blackbox/cosyvoicetts.py b/src/blackbox/cosyvoicetts.py new file mode 100644 index 0000000..f62a61f --- /dev/null +++ b/src/blackbox/cosyvoicetts.py @@ -0,0 +1,93 @@ +import io +import time + +import requests +from fastapi import Request, Response, status +from fastapi.responses import JSONResponse +from injector import inject +from injector import singleton + +from ..log.logging_time import logging_time + +from ..configuration import CosyVoiceConf +from .blackbox import Blackbox + +import soundfile +import pyloudnorm as pyln +import sys +sys.path.append('/home/gpu/Workspace/CosyVoice') +from cosyvoice.cli.cosyvoice import CosyVoice +from cosyvoice.utils.file_utils import load_wav +import torchaudio + +import os +import logging +logger = logging.getLogger(__name__) + +@singleton +class CosyVoiceTTS(Blackbox): + mode: str + url: str + speed: int + device: str + language: str + speaker: str + + @logging_time(logger=logger) + def model_init(self, cosyvoice_config: CosyVoiceConf) -> None: + self.speed = cosyvoice_config.speed + self.device = cosyvoice_config.device + self.language = cosyvoice_config.language + self.speaker = cosyvoice_config.speaker + self.device = cosyvoice_config.device + self.url = '' + self.mode = cosyvoice_config.mode + self.cosyvoicetts = None + self.speaker_ids = None + os.environ['CUDA_VISIBLE_DEVICES'] = str(cosyvoice_config.device) + if self.mode == 'local': + self.cosyvoicetts = CosyVoice('/home/gpu/Workspace/Models/CosyVoice/pretrained_models/CosyVoice-300M') + + else: + self.url = cosyvoice_config.url + logging.info('#### Initializing CosyVoiceTTS Service in cuda:' + str(cosyvoice_config.device) + ' mode...') + + @inject + def __init__(self, cosyvoice_config: CosyVoiceConf) -> None: + self.model_init(cosyvoice_config) + + def __call__(self, *args, **kwargs): + return self.processing(*args, **kwargs) + + def valid(self, *args, **kwargs) -> bool: + text = args[0] + return isinstance(text, str) + + @logging_time(logger=logger) + def processing(self, *args, **kwargs) -> io.BytesIO | bytes: + text = args[0] + current_time = time.time() + if self.mode == 'local': + audio = self.cosyvoicetts.inference_sft(text, self.language) + f = io.BytesIO() + soundfile.write(f, audio['tts_speech'].cpu().numpy().squeeze(0), 22050, format='wav') + f.seek(0) + print("#### CosyVoiceTTS Service consume - local : ", (time.time() - current_time)) + return f.read() + else: + message = { + "text": text + } + response = requests.post(self.url, json=message) + print("#### CosyVoiceTTS Service consume - docker : ", (time.time()-current_time)) + return response.content + + async def fast_api_handler(self, request: Request) -> Response: + try: + data = await request.json() + except: + return JSONResponse(content={"error": "json parse error"}, status_code=status.HTTP_400_BAD_REQUEST) + text = data.get("text") + if text is None: + return JSONResponse(content={"error": "text is required"}, status_code=status.HTTP_400_BAD_REQUEST) + return Response(content=self.processing(text), media_type="audio/wav", headers={"Content-Disposition": "attachment; filename=audio.wav"}) \ No newline at end of file diff --git a/src/blackbox/melotts.py b/src/blackbox/melotts.py new file mode 100644 index 0000000..0a2e118 --- /dev/null +++ b/src/blackbox/melotts.py @@ -0,0 +1,108 @@ +import io +import time + +import requests +from fastapi import Request, Response, status +from fastapi.responses import JSONResponse +from injector import inject +from injector import singleton + +from ..log.logging_time import logging_time + +from ..configuration import MeloConf +from .blackbox import Blackbox + +import soundfile +import pyloudnorm as pyln +from melo.api import TTS + +import logging +logger = logging.getLogger(__name__) + +@singleton +class MeloTTS(Blackbox): + mode: str + url: str + speed: int + device: str + language: str + speaker: str + + @logging_time(logger=logger) + def model_init(self, melo_config: MeloConf) -> None: + self.speed = melo_config.speed + self.device = melo_config.device + self.language = melo_config.language + self.speaker = melo_config.speaker + self.device = melo_config.device + self.url = '' + self.mode = melo_config.mode + self.melotts = None + self.speaker_ids = None + if self.mode == 'local': + self.melotts = TTS(language=self.language, device=self.device) + self.speaker_ids = self.melotts.hps.data.spk2id + else: + self.url = melo_config.url + logging.info('#### Initializing MeloTTS Service in ' + self.device + ' mode...') + + @inject + def __init__(self, melo_config: MeloConf) -> None: + self.model_init(melo_config) + + def __call__(self, *args, **kwargs): + return self.processing(*args, **kwargs) + + def valid(self, *args, **kwargs) -> bool: + text = args[0] + return isinstance(text, str) + + @logging_time(logger=logger) + def processing(self, *args, **kwargs) -> io.BytesIO | bytes: + text = args[0] + current_time = time.time() + if self.mode == 'local': + audio = self.melotts.tts_to_file(text, self.speaker_ids[self.speaker], speed=self.speed) + f = io.BytesIO() + soundfile.write(f, audio, 44100, format='wav') + f.seek(0) + # print("#### MeloTTS Service consume - local : ", (time.time() - current_time)) + # return f.read() + + + # Read the audio data from the buffer + data, rate = soundfile.read(f, dtype='float32') + + # Peak normalization + peak_normalized_audio = pyln.normalize.peak(data, -1.0) + + # Integrated loudness normalization + meter = pyln.Meter(rate) + loudness = meter.integrated_loudness(peak_normalized_audio) + loudness_normalized_audio = pyln.normalize.loudness(peak_normalized_audio, loudness, -12.0) + + # Write the loudness normalized audio to an in-memory buffer + normalized_audio_buffer = io.BytesIO() + soundfile.write(normalized_audio_buffer, loudness_normalized_audio, rate, format='wav') + normalized_audio_buffer.seek(0) + + print("#### MeloTTS Service consume - local : ", (time.time() - current_time)) + return normalized_audio_buffer.read() + + else: + message = { + "text": text + } + response = requests.post(self.url, json=message) + print("#### MeloTTS Service consume - docker : ", (time.time()-current_time)) + return response.content + + async def fast_api_handler(self, request: Request) -> Response: + try: + data = await request.json() + except: + return JSONResponse(content={"error": "json parse error"}, status_code=status.HTTP_400_BAD_REQUEST) + text = data.get("text") + if text is None: + return JSONResponse(content={"error": "text is required"}, status_code=status.HTTP_400_BAD_REQUEST) + return Response(content=self.processing(text), media_type="audio/wav", headers={"Content-Disposition": "attachment; filename=audio.wav"}) \ No newline at end of file diff --git a/src/blackbox/vlms.py b/src/blackbox/vlms.py index 6765260..9ac90d5 100644 --- a/src/blackbox/vlms.py +++ b/src/blackbox/vlms.py @@ -273,7 +273,6 @@ class VLMS(Blackbox): prompt = data.get("prompt") settings: dict = data.get('settings') context = data.get("context") - if not context: user_context = [] elif isinstance(context[0], list):