Merge branch 'main' of github.com:BoardWare-Genius/javis-models into main

sample/requirements_out_of_pytorch.txt

@@ -0,0 +1,82 @@
+aiohttp==3.8.4
+aiosignal==1.3.1
+anyio==3.6.2
+appdirs==1.4.4
+async-timeout==4.0.2
+async-tio==1.3.2
+attrs==23.1.0
+audioread==3.0.0
+certifi==2022.12.7
+cffi==1.15.1
+charset-normalizer==2.1.1
+cn2an==0.5.19
+colorama==0.4.6
+coloredlogs==15.0.1
+Cython==0.29.34
+decorator==5.1.1
+filelock==3.9.0
+flatbuffers==23.3.3
+frozenlist==1.3.3
+fsspec==2023.4.0
+h11==0.14.0
+httpcore==0.17.0
+httpx==0.24.0
+huggingface-hub==0.14.1
+humanfriendly==10.0
+idna==3.4
+jieba==0.42.1
+Jinja2==3.1.2
+joblib==1.2.0
+lazy_loader==0.2
+librosa==0.10.0.post2
+llvmlite==0.40.0
+MarkupSafe==2.1.2
+mpmath==1.2.1
+msgpack==1.0.5
+multidict==6.0.4
+networkx==3.0
+numba==0.57.0
+numpy==1.24.1
+onnxruntime==1.14.1
+openai==0.27.6
+OpenAIAuth==0.3.6
+packaging==23.1
+Pillow==9.3.0
+pooch==1.6.0
+proces==0.1.4
+prompt-toolkit==3.0.38
+protobuf==4.22.4
+PyAudio==0.2.13
+pycparser==2.21
+pypinyin==0.48.0
+pyreadline3==3.4.1
+PySocks==1.7.1
+pywin32==306
+PyYAML==6.0
+regex==2023.5.5
+requests==2.28.1
+revChatGPT==5.0.0
+scikit-learn==1.2.2
+scipy==1.10.1
+sniffio==1.3.0
+socksio==1.0.0
+soundfile==0.12.1
+soxr==0.3.5
+sympy==1.11.1
+threadpoolctl==3.1.0
+tiktoken==0.3.3
+tokenizers==0.13.3
+tqdm==4.65.0
+transformers==4.28.1
+typeguard==2.13.3
+typing_extensions==4.4.0
+urllib3==1.26.13
+wcwidth==0.2.6
+WMI==1.5.1
+yarl==1.9.2
+filetype
+fastapi
+python-multipart
+uvicorn[standard]
+SpeechRecognition
+gtts

src/blackbox/blackbox_factory.py

@@ -1,15 +1,14 @@
 from .audio_chat import AudioChat
-from .sum import SUM
 from .sentiment import Sentiment
 from .tts import TTS
 from .asr import ASR
 from .audio_to_text import AudioToText
 from .blackbox import Blackbox
-from .calculator import Calculator
 from .text_to_audio import TextToAudio
 from .tesou import Tesou
 from .fastchat import Fastchat
 from .g2e import G2E
+from .text_and_image import TextAndImage
 
 class BlackboxFactory:
 
@@ -17,14 +16,13 @@ class BlackboxFactory:
         self.tts = TTS()
         self.asr = ASR(".env.yaml")
         self.sentiment = Sentiment()
-        self.sum = SUM()
-        self.calculator = Calculator()
         self.audio_to_text = AudioToText()
         self.text_to_audio = TextToAudio()
         self.tesou = Tesou()
         self.fastchat = Fastchat()
         self.audio_chat = AudioChat(self.asr, self.tesou, self.tts)
         self.g2e = G2E()
+        self.text_and_image = TextAndImage()
 
     def __call__(self, *args, **kwargs):
         return self.processing(*args, **kwargs)
@@ -34,16 +32,12 @@ class BlackboxFactory:
             return self.audio_to_text
         if blackbox_name == "text_to_audio":
             return self.text_to_audio
-        if blackbox_name == "calculator":
-            return self.calculator
         if blackbox_name == "asr":
             return self.asr
         if blackbox_name == "tts":
             return self.tts
         if blackbox_name == "sentiment_engine":
             return self.sentiment
-        if blackbox_name == "sum":
-            return self.sum
         if blackbox_name == "tesou":
             return self.tesou
         if blackbox_name == "fastchat":
@@ -52,4 +46,6 @@ class BlackboxFactory:
             return self.audio_chat
         if blackbox_name == "g2e":
             return self.g2e
+        if blackbox_name == 'text_and_image':
+            return self.text_and_image
         raise ValueError("Invalid blockbox type")

src/blackbox/blackbox_sentiment.py

@@ -0,0 +1,48 @@
+from typing import Any, Coroutine
+
+from fastapi import Request, Response, status
+from fastapi.responses import JSONResponse
+
+from .blackbox import Blackbox
+from lagent.llms.lmdepoly_wrapper import LMDeployClient
+from lagent.llms.meta_template import INTERNLM2_META as META
+
+class Sentiment(Blackbox):
+
+    def __init__(self, model_name, model_url) -> None:
+        self.model = LMDeployClient(
+            model_name=model_name,
+            url=model_url,
+            meta_template=META,
+            top_p=0.8,
+            top_k=100,
+            temperature=0,
+            repetition_penalty=1.0,
+            stop_words=['<|im_end|>'])
+
+    def __call__(self, *args, **kwargs):
+        return self.processing(*args, **kwargs)
+
+    def valid(self,  *args, **kwargs) -> bool:
+        data = args[0]
+        return isinstance(data, str)
+    
+    def processing(self, *args, **kwargs) -> int:
+        text = args[0]
+        text = "Please use one word to infer the emotion of the following passage:\n" + text + "\nJust print out that signle word pls."
+        text = [{'role': 'user', 'content': text}]
+        return self.model.stream_chat(text)
+    
+    async def fast_api_handler(self, request) -> Response:
+        try:
+            data = await request.json()
+        except:
+            return JSONResponse(content={"error": "json parse error"}, status_code=status.HTTP_400_BAD_REQUEST)
+        text = data.get("text")
+        if text is None:
+            return JSONResponse(content={"error": "text is required"}, status_code=status.HTTP_400_BAD_REQUEST)
+        text = "Please use one word to infer the emotion of the following passage:\n" + text + "\nJust print out that signle word pls."
+        text = [{'role': 'user', 'content': text}]
+        sentiment = self.processing(text)
+        return JSONResponse(content={"sentiment": sentiment }, status_code=status.HTTP_200_OK)
+    

src/blackbox/calculator.py

@@ -1,42 +0,0 @@
-from fastapi import status
-from fastapi.responses import JSONResponse
-from .blackbox import Blackbox
-
-
-class Calculator(Blackbox):
-
-    """This class just for example, it show how to implement Blackbox interface."""
-    
-    def __call__(self, *args, **kwargs):
-        return self.processing(*args, **kwargs)
-
-    def valid(self, *args, **kwargs) -> bool:
-        data = args[0]
-        return isinstance(data, dict) and "op" in data and "left" in data and "right" in data
-
-    def processing(self, data: dict) -> int | float:
-        if not self.valid(data):
-            raise ValueError("Invalid data")
-        a = data["left"]
-        b = data["right"]
-        op = data["op"]
-        if op == "add":
-            return a + b
-        if op == "sub":
-            return a - b
-        if op == "mul":
-            return a * b
-        if op == "div":
-            return a / b
-        raise ValueError("Invalid operation")
-    
-    async def fast_api_handler(self, request) -> any:
-        try:
-            data = await request.json()
-        except:
-            return JSONResponse(content={"error": "json parse error"}, status_code=status.HTTP_400_BAD_REQUEST)
-        try:
-            result = self.processing(data)
-        except ValueError as e:
-            return JSONResponse(content={"error": str(e)}, status_code=status.HTTP_400_BAD_REQUEST)
-        return JSONResponse(content={"result": result}, status_code=status.HTTP_200_OK)

src/blackbox/g2e.py

@@ -22,7 +22,7 @@ class G2E(Blackbox):
         if context == None:
             context = []
         url = 'http://120.196.116.194:48890/v1'
-
+        #url = 'http://120.196.116.194:48892/v1'
 
         background_prompt = '''KOMBUKIKI是一款茶饮料，目标受众 年龄：20-35岁 性别：女性 地点：一线城市、二线城市 职业：精英中产、都市白领 收入水平：中高收入，有一定消费能力 兴趣和爱好：注重健康，有运动习惯
 
@@ -31,7 +31,7 @@ class G2E(Blackbox):
         性别：女
         姓名：琪琪
         性格特质： 勇敢、独立、坚韧、积极、正面、自爱。 具有探险精神，拥有一颗开放的心，在面对困难和挑战时能够保持积极的态度 MBTI类型：ENFP
-        背景故事： KIKI对旅行和冒险充满热情和渴望，正在寻找生活中的新体验和挑战，从而踏上追寻自我成长的旅程
+        背景故事： 琪琪对旅行和冒险充满热情和渴望，正在寻找生活中的新体验和挑战，从而踏上追寻自我成长的旅程
 
         品牌故事 KOMBUKIKI：每一口，旅行的开始在忙碌的日常中，KOMBUKIKI康普茶不仅是一种饮料，它是一个承载梦想和自由的容器。对于那些渴望旅行，寻求生活质感的20至35岁女性来说，KOMBUKIKI是她们心灵的伙伴。KOMBUKIKI的每一瓶，都是一段故事的开端。我们的康普茶瓶身设计灵感来自旅行手账，最大的特色是其邮票形状的包装贴纸。每个贴纸上描绘着不同的旅游景点，从繁华的都市到静谧的乡村，每一瓶都是对该地点的独特致敬。当顾客扫描瓶身上的二维码，将展开一系列充满诗意的故事，带领她们走进一个个迷人的旅行梦境。在KOMBUKIKI的世界里，每一口康普茶不仅是滋味的享受，更是心灵的旅行。它不仅卖给顾客一种饮料，更是卖给她们一种情绪价值，一种对自由和旅行的渴望。这是一个专为那些处于人生转变期，对成长话题感兴趣的年轻女性所设计的品牌。KOMBUKIKI，不仅是康普茶，它是每个人心中对旅行渴望的象征，是一场心灵和味觉的双重旅行。口号 每一口，启程至心中的远方 是每个人心中对旅行渴望的象征， 是一场心灵和味觉的双重旅行。
 
@@ -47,8 +47,11 @@ class G2E(Blackbox):
 
 
         prompt_template = [
-            {"role": "system", "content": background_prompt + prompt1},
+           {"role": "system", "content": background_prompt + prompt1},
         ]
+        #prompt_template = [
+        #    {"role": "system", "content": ''},
+        #]
 
 
         messages = prompt_template + context + [
@@ -57,13 +60,12 @@ class G2E(Blackbox):
                     "content": prompt + inject_prompt
                 }
             ]
-
         client = OpenAI(
             api_key='YOUR_API_KEY',
             base_url=url
         )
-
         model_name = client.models.list().data[0].id
+        print(model_name)
         response = client.chat.completions.create(
         model=model_name,
         messages=messages,

src/blackbox/sum.py

@@ -1,27 +0,0 @@
-from typing import Any, Coroutine
-
-from fastapi import Request, Response, status
-from fastapi.responses import JSONResponse
-from .blackbox import Blackbox
-
-
-class SUM(Blackbox):
-    
-    def __call__(self, *args, **kwargs):
-        return self.processing(*args, **kwargs)
-
-    def valid(self, *args, **kwargs) -> bool:
-        data = args[0]
-        return isinstance(data, list)
-
-    def processing(self, *args, **kwargs):
-        return sum(args)
-    
-    async def fast_api_handler(self, request: Request) -> Response:
-        try:
-            data = await request.json()
-        except:
-            return JSONResponse(content={"error": "json parse error"}, status_code=status.HTTP_400_BAD_REQUEST)
-        if not self.valid(data):
-            return JSONResponse(content={"error": "format error"}, status_code=status.HTTP_400_BAD_REQUEST)
-        return JSONResponse(content={"result": self.processing(data)}, status_code=status.HTTP_200_OK)

src/blackbox/text_and_image.py

@@ -0,0 +1,28 @@
+from typing import Any, Coroutine
+
+from fastapi import Request, Response, status
+from fastapi.responses import JSONResponse
+from .blackbox import Blackbox
+
+
+class TextAndImage(Blackbox):
+    
+    def __call__(self, *args, **kwargs):
+        return self.processing(*args, **kwargs)
+
+    def processing(self, text: str, image: bytes):
+        # TODO: implement the processing logic
+        return "text and image processing..."
+    
+    def valid(self, *args, **kwargs) -> bool:
+        return True
+    
+    async def fast_api_handler(self, request: Request) -> Response:
+        # If the request is a form data, you can use the following code
+        data = (await request.form()).get("image")
+        if data is None:
+            return JSONResponse(content={"error": "image is required"}, status_code=status.HTTP_400_BAD_REQUEST)
+        imageBytes = await data.read()
+        text = (await request.form()).get("text")
+        result = self.processing(text, imageBytes)
+        return JSONResponse(content={"result": result }, status_code=status.HTTP_200_OK)

swagger.yml

@@ -54,10 +54,19 @@ components:
         audio:
           type: string
           format: binary
+    TextAndImageInput:
+      type: object
+      properties:
+        text:
+          type: string
+        image:
+          type: string
+          format: binary
     Input:
       oneOf:
         - $ref: "#/components/schemas/TextToAudioInput"
         - $ref: "#/components/schemas/AudioToTextInput"
+        - $ref: "#/components/schemas/TextAndImageInput"
     TextResult:
       type: object
       properties:
@@ -71,5 +80,13 @@ components:
       type: string
       description: "Blackbox name"
       enum:
-        - "text_to_audio"
+        - text_to_audio
-        - "audio_to_text"
+        - audio_to_text
+        - asr
+        - tts
+        - sentiment_engine
+        - tesou
+        - fastchat
+        - audio_chat
+        - g2e
+        - text_and_image