From 7fc5e169e9558d5ebcfd4a94a714728a77206628 Mon Sep 17 00:00:00 2001 From: superobk Date: Fri, 24 May 2024 11:29:28 +0800 Subject: [PATCH 1/5] fix: import path --- src/blackbox/blackbox_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/blackbox/blackbox_factory.py b/src/blackbox/blackbox_factory.py index 7dd8754..82b9e71 100644 --- a/src/blackbox/blackbox_factory.py +++ b/src/blackbox/blackbox_factory.py @@ -1,4 +1,4 @@ -from blackbox.emotion import Emotion +from .emotion import Emotion from .chat import Chat from .audio_chat import AudioChat from .sentiment import Sentiment From d6d1b337980d7af4b07923b945426c0e9a5448b0 Mon Sep 17 00:00:00 2001 From: gdw6463 Date: Fri, 24 May 2024 12:19:01 +0800 Subject: [PATCH 2/5] Chroma libary readme updated --- README.md | 27 ++++++++++++++------------- install-dependencies.sh | 3 ++- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 563a917..2ec2b52 100644 --- a/README.md +++ b/README.md @@ -6,19 +6,20 @@ pip install -r sample/requirement_out_of_pytorch.txt pip install torch==2.0.1 torchvision==0.15.2 torchaudio==2.0.2 --index-url https://download.pytorch.org/whl/cu118 ``` ## More Dependencies -| System | package | web | install command | -| --- | ---- | --- | --- | -| python | filetype | https://pypi.org/project/filetype/ | pip install filetype | -| python | fastAPI | https://fastapi.tiangolo.com/ | pip install fastapi | -| python | python-multipart | https://pypi.org/project/python-multipart/ | pip install python-multipart | -| python | uvicorn | https://www.uvicorn.org/ | pip install "uvicorn[standard]" | -| python | SpeechRecognition | https://pypi.org/project/SpeechRecognition/ | pip install SpeechRecognition | -| python | gtts | https://pypi.org/project/gTTS/ | pip install gTTS | -| python | PyYAML | https://pypi.org/project/PyYAML/ | pip install PyYAML | -| python | injector | https://github.com/python-injector/injector | pip install injector | -| python | langchain | https://github.com/langchain-ai/langchain | pip install langchain | -| python | chromadb | https://docs.trychroma.com/getting-started | pip install chromadb | -| python | lagent | https://github.com/InternLM/lagent/blob/main/README.md | pip install lagent | +| System | package | web | install command | +| --- |-----------------------| --- | --- | +| python | filetype | https://pypi.org/project/filetype/ | pip install filetype | +| python | fastAPI | https://fastapi.tiangolo.com/ | pip install fastapi | +| python | python-multipart | https://pypi.org/project/python-multipart/ | pip install python-multipart | +| python | uvicorn | https://www.uvicorn.org/ | pip install "uvicorn[standard]" | +| python | SpeechRecognition | https://pypi.org/project/SpeechRecognition/ | pip install SpeechRecognition | +| python | gtts | https://pypi.org/project/gTTS/ | pip install gTTS | +| python | PyYAML | https://pypi.org/project/PyYAML/ | pip install PyYAML | +| python | injector | https://github.com/python-injector/injector | pip install injector | +| python | langchain | https://github.com/langchain-ai/langchain | pip install langchain | +| python | chromadb | https://docs.trychroma.com/getting-started | pip install chromadb | +| python | lagent | https://github.com/InternLM/lagent/blob/main/README.md | pip install lagent | +| python | sentence_transformers | https://github.com/InternLM/lagent/blob/main/README.md | pip install sentence_transformers | ## Start diff --git a/install-dependencies.sh b/install-dependencies.sh index 0f613cb..fce7084 100644 --- a/install-dependencies.sh +++ b/install-dependencies.sh @@ -9,4 +9,5 @@ pip install PyYAML pip install injector pip install landchain pip install chromadb -pip install lagent \ No newline at end of file +pip install lagent +pip install sentence_transformers \ No newline at end of file From 36239dfe6abcff2ec576a8ccb1a1d254234318c3 Mon Sep 17 00:00:00 2001 From: superobk Date: Fri, 24 May 2024 15:19:37 +0800 Subject: [PATCH 3/5] feat: models lazy loader --- src/blackbox/blackbox_factory.py | 151 +++++++++++++++++++++---------- src/configuration.py | 12 ++- 2 files changed, 114 insertions(+), 49 deletions(-) diff --git a/src/blackbox/blackbox_factory.py b/src/blackbox/blackbox_factory.py index 82b9e71..c96a393 100644 --- a/src/blackbox/blackbox_factory.py +++ b/src/blackbox/blackbox_factory.py @@ -1,57 +1,112 @@ -from .emotion import Emotion -from .chat import Chat -from .audio_chat import AudioChat -from .sentiment import Sentiment -from .tts import TTS -from .asr import ASR -from .audio_to_text import AudioToText from .blackbox import Blackbox -from .fastchat import Fastchat -from .g2e import G2E -from .text_and_image import TextAndImage -from .melotts import MeloTTS -from .vlms import VLMS -from .chroma_query import ChromaQuery -from .chroma_upsert import ChromaUpsert -from .chroma_chat import ChromaChat -from injector import inject, singleton +from injector import inject, singleton, Injector +from ..configuration import BlackboxConf + +blackboxConf = Injector().get(BlackboxConf) + +def model_loader(lazy=blackboxConf.lazyloading): + def load(init_fun): + model = None + if not lazy: + model = init_fun() + def inner(): + nonlocal model + if model is None: + model = init_fun() + return model + return inner + return load + +@model_loader(lazy=blackboxConf.lazyloading) +def text_and_image_loader(): + from .text_and_image import TextAndImage + return Injector().get(TextAndImage) + +@model_loader(lazy=blackboxConf.lazyloading) +def audio_chat_loader(): + from .audio_chat import AudioChat + return Injector().get(AudioChat) + +@model_loader(lazy=blackboxConf.lazyloading) +def sentiment_loader(): + from .sentiment import Sentiment + return Injector().get(Sentiment) + +@model_loader(lazy=blackboxConf.lazyloading) +def g2e_loader(): + from .g2e import G2E + return Injector().get(G2E) + +@model_loader(lazy=blackboxConf.lazyloading) +def asr_loader(): + from .asr import ASR + return Injector().get(ASR) + +@model_loader(lazy=blackboxConf.lazyloading) +def vlms_loader(): + from .vlms import VLMS + return Injector().get(VLMS) + +@model_loader(lazy=blackboxConf.lazyloading) +def melotts_loader(): + from .melotts import MeloTTS + return Injector().get(MeloTTS) + +@model_loader(lazy=blackboxConf.lazyloading) +def tts_loader(): + from .tts import TTS + return Injector().get(TTS) + +@model_loader(lazy=blackboxConf.lazyloading) +def emotion_loader(): + from .emotion import Emotion + return Injector().get(Emotion) + +@model_loader(lazy=blackboxConf.lazyloading) +def fastchat_loader(): + from .fastchat import Fastchat + return Injector().get(Fastchat) + +@model_loader(lazy=blackboxConf.lazyloading) +def chat_loader(): + from .chat import Chat + return Injector().get(Chat) + +@model_loader(lazy=blackboxConf.lazyloading) +def chroma_query_loader(): + from .chroma_query import ChromaQuery + return Injector().get(ChromaQuery) + +@model_loader(lazy=blackboxConf.lazyloading) +def chroma_upsert_loader(): + from .chroma_upsert import ChromaUpsert + return Injector().get(ChromaUpsert) + +@model_loader(lazy=blackboxConf.lazyloading) +def chroma_chat_load(): + from .chroma_chat import ChromaChat + return Injector().get(ChromaChat) @singleton class BlackboxFactory: models = {} @inject - def __init__(self, - audio_to_text: AudioToText, - asr: ASR, - tts: TTS, - sentiment_engine: Sentiment, - emotion: Emotion, - fastchat: Fastchat, - audio_chat: AudioChat, - g2e: G2E, - text_and_image: TextAndImage, - melotts: MeloTTS, - vlms: VLMS, - chroma_query: ChromaQuery, - chroma_upsert: ChromaUpsert, - chroma_chat: ChromaChat, - chat: Chat) -> None: - self.models["audio_to_text"] = audio_to_text - self.models["asr"] = asr - self.models["tts"] = tts - self.models["sentiment_engine"] = sentiment_engine - self.models["emotion"] = emotion - self.models["fastchat"] = fastchat - self.models["audio_chat"] = audio_chat - self.models["g2e"] = g2e - self.models["text_and_image"] = text_and_image - self.models["chroma_query"] = chroma_query - self.models["chroma_upsert"] = chroma_upsert - self.models["chroma_chat"] = chroma_chat - self.models["melotts"] = melotts - self.models["vlms"] = vlms - self.models["chat"] = chat + def __init__(self,) -> None: + self.models["asr"] = asr_loader + self.models["tts"] = tts_loader + self.models["sentiment_engine"] = sentiment_loader + self.models["emotion"] = emotion_loader + self.models["fastchat"] = fastchat_loader + self.models["audio_chat"] = audio_chat_loader + self.models["g2e"] = g2e_loader + self.models["text_and_image"] = text_and_image_loader + self.models["chroma_query"] = chroma_query_loader + self.models["chroma_upsert"] = chroma_upsert_loader + self.models["chroma_chat"] = chroma_chat_load + self.models["melotts"] = melotts_loader + self.models["vlms"] = vlms_loader + self.models["chat"] = chat_loader def __call__(self, *args, **kwargs): return self.processing(*args, **kwargs) @@ -60,4 +115,4 @@ class BlackboxFactory: model = self.models.get(blackbox_name) if model is None: raise ValueError("Invalid Blackbox Type...") - return model + return model() diff --git a/src/configuration.py b/src/configuration.py index face1e0..16fa3cc 100644 --- a/src/configuration.py +++ b/src/configuration.py @@ -1,4 +1,5 @@ +from dataclasses import dataclass from injector import inject,singleton import yaml import sys @@ -101,4 +102,13 @@ class EnvConf(): def __init__(self, config: Configuration) -> None: self.version = "0.0.1" self.host = config.get("env.host", default="0.0.0.0") - self.port = config.get("env.port", default="8080") \ No newline at end of file + self.port = config.get("env.port", default="8080") + +@singleton +@dataclass +class BlackboxConf(): + lazyloading: bool + + @inject + def __init__(self, config: Configuration) -> None: + self.lazyloading = bool(config.get("blackbox.lazyloading", default=False)) \ No newline at end of file From 28621b7169c2f565ab6bc4ec3616063126afe9c2 Mon Sep 17 00:00:00 2001 From: superobk Date: Fri, 24 May 2024 15:21:37 +0800 Subject: [PATCH 4/5] fixup --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 563a917..0bf3f77 100644 --- a/README.md +++ b/README.md @@ -86,4 +86,6 @@ Model: cudnn_conv_algo_search: EXHAUSTIVE do_copy_in_default_stream: true batch_size: 3 +blackbox: + lazyloading: true ``` From ab82aa75752e712a1ab305991153e03e8ba99da2 Mon Sep 17 00:00:00 2001 From: superobk Date: Fri, 24 May 2024 16:16:43 +0800 Subject: [PATCH 5/5] refactor: chat processing --- src/blackbox/chat.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/blackbox/chat.py b/src/blackbox/chat.py index 0ba8c68..369c228 100644 --- a/src/blackbox/chat.py +++ b/src/blackbox/chat.py @@ -1,4 +1,3 @@ -import logging from typing import Any, Coroutine from fastapi import Request, Response, status @@ -14,8 +13,6 @@ import re from injector import singleton -logger = logging.getLogger - @singleton class Chat(Blackbox): @@ -27,15 +24,14 @@ class Chat(Blackbox): return isinstance(data, list) # model_name有 Qwen1.5-14B-Chat , internlm2-chat-20b - @logging_time(logger=logger) - def processing(self, *args, **kwargs) -> str: + @logging_time() + def processing(self, prompt: str, context: list, settings: dict) -> str: - settings: dict = args[0] if settings is None: settings = {} user_model_name = settings.get("model_name") - user_context = settings.get("context") - user_question = settings.get("question") + user_context = context + user_question = prompt user_template = settings.get("template") user_temperature = settings.get("temperature") user_top_p = settings.get("top_p") @@ -44,7 +40,6 @@ class Chat(Blackbox): user_stop = settings.get("stop") user_frequency_penalty = settings.get("frequency_penalty") user_presence_penalty = settings.get("presence_penalty") - if user_context == None: user_context = [] @@ -124,5 +119,7 @@ class Chat(Blackbox): return JSONResponse(content={"error": "json parse error"}, status_code=status.HTTP_400_BAD_REQUEST) setting: dict = data.get("settings") - - return JSONResponse(content={"response": self.processing(setting)}, status_code=status.HTTP_200_OK) \ No newline at end of file + context = data.get("context") + prompt = data.get("prompt") + + return JSONResponse(content={"response": self.processing(prompt, context, setting)}, status_code=status.HTTP_200_OK) \ No newline at end of file