modety vlms with lmdeploy

README.md

@@ -89,4 +89,7 @@ Model:
   batch_size: 3
 blackbox:
   lazyloading: true
+
+vlms:
+  url: http://10.6.80.87:23333
 ```

src/blackbox/vlms.py

@@ -1,11 +1,19 @@
 from fastapi import Request, Response, status
 from fastapi.responses import JSONResponse
-from .blackbox import Blackbox
+from injector import singleton,inject
 from typing import Optional
 
+from .blackbox import Blackbox
+from ..log.logging_time import logging_time
+from .chroma_query import ChromaQuery
+from ..configuration import VLMConf
+
 import requests
 import base64
 
+import io 
+from PIL import Image
+from lmdeploy.serve.openai.api_client import APIClient
 
 def is_base64(value) -> bool:
     try:
@@ -14,9 +22,16 @@ def is_base64(value) -> bool:
     except Exception:
         return False
 
-
+@singleton
 class VLMS(Blackbox):
 
+    @inject
+    def __init__(self, vlm_config: VLMConf):
+        # Chroma database initially set up for RAG for vision model.
+        # It could be expended to history store. 
+        # self.chroma_query = chroma_query
+        self.url = vlm_config.url
+
     def __call__(self, *args, **kwargs):
         return self.processing(*args, **kwargs)
 
@@ -33,23 +48,64 @@ class VLMS(Blackbox):
         else:
             model_name = "infer-qwen-vl"
 
-        url = 'http://120.196.116.194:48894/' + model_name + '/'
+
+        ## AutoLoad Model
+        # url = 'http://10.6.80.87:8000/' + model_name + '/'
 
         if is_base64(images):
             images_data = images
         else:
+            # print("{}Type of image data in form {}".format('#'*20,type(images)))
+            # print("{}Type of image data in form {}".format('#'*20,type(images.file)))
+            # byte_stream = io.BytesIO(images.read())      
+            # print("{}Type of image data in form {}".format('#'*20,type(byte_stream)))
+            # # roiImg = Image.open(byte_stream) 
+            # # print("{}Successful {}".format('#'*20,type(roiImg)))
+            # return str(type(byte_stream))
+            # images_data = base64.b64encode(byte_stream)
             with open(images, "rb") as img_file:
-                images_data = str(base64.b64encode(img_file.read()), 'utf-8')
+                # images_data = str(base64.b64encode(img_file.read()), 'utf-8')
+                images_data = base64.b64encode(img_file.read())
 
-        data_input = {'model': model_name, 'prompt': prompt, 'img_data': images_data}
+        # data_input = {'model': model_name, 'prompt': prompt, 'img_data': images_data}
 
-        data = requests.post(url, json=data_input)
+        # data = requests.post(url, json=data_input)
+        # print(data.text)
+        # 'https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/tests/data/tiger.jpeg'
+        ## Lmdeploy
+        api_client = APIClient(self.url)
+        # api_client = APIClient(f'http://10.6.80.87:23333')
+        model_name = api_client.available_models[0]
+        messages = [{
+                'role':
+                'user',
+                'content': [{
+                    'type': 'text',
+                    'text': prompt,
+                }, {
+                    'type': 'image_url',
+                    'image_url': {
+                        'url': f"data:image/jpeg;base64,{images_data}",
+                        # './val_data/image_5.jpg',
+                    },
+                }]
+            }
+        ]
         
-        return data.text
+        responses = ''
+        for i,item in enumerate(api_client.chat_completions_v1(model=model_name,
+                                                messages=messages#,stream = True
+                                                )):
+            print(item["choices"][0]["message"]['content'])
+            responses += item["choices"][0]["message"]['content']
+        
+        return responses
+
+        # return data.text
 
     async def fast_api_handler(self, request: Request) -> Response:
         try:
-            data = await request.json()
+            data = await request.form()
         except:
             return JSONResponse(content={"error": "json parse error"}, status_code=status.HTTP_400_BAD_REQUEST)
 
@@ -63,5 +119,6 @@ class VLMS(Blackbox):
         if model_name is None or model_name.isspace():
             model_name = "Qwen-VL-Chat"
 
-        jsonresp = str(JSONResponse(content={"response": self.processing(prompt, img_data, model_name)}).body, "utf-8")
+        # jsonresp = str(JSONResponse(content={"response": self.processing(prompt, img_data, model_name)}).body, "utf-8")
-        return JSONResponse(content={"response": jsonresp}, status_code=status.HTTP_200_OK)
+
+        return JSONResponse(content={"response": self.processing(prompt, img_data, model_name)}, status_code=status.HTTP_200_OK)

src/configuration.py

@@ -129,3 +129,10 @@ class BlackboxConf():
     @inject
     def __init__(self, config: Configuration) -> None:
         self.lazyloading = bool(config.get("blackbox.lazyloading", default=False))
+
+@singleton
+class VLMConf():
+
+    @inject
+    def __init__(self, config: Configuration) -> None:
+        self.url = config.get("vlms.url")