ivanwu/jarvis-models
1
0
Fork 0
mirror of https://github.com/BoardWare-Genius/jarvis-models.git synced 2025-12-14 00:53:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

modety vlms with lmdeploy

Browse Source
This commit is contained in:
Ivan087
2024-08-20 09:41:10 +08:00
parent 30dedc935f
commit 4c3756811d
3 changed files with 357 additions and 290 deletions
Download Patch File Download Diff File Expand all files Collapse all files

191
src/blackbox/vlms.py
View File

@ -1,67 +1,124 @@
from fastapi import Request, Response, status
from fastapi.responses import JSONResponse
from .blackbox import Blackbox
from typing import Optional
import requests
import base64
def is_base64(value) -> bool:
try:
base64.b64decode(base64.b64decode(value)) == value.encode()
return True
except Exception:
return False
class VLMS(Blackbox):
def __call__(self, *args, **kwargs):
return self.processing(*args, **kwargs)
def valid(self, *args, **kwargs) -> bool:
data = args[0]
return isinstance(data, list)
def processing(self, prompt, images, model_name: Optional[str] = None) -> str:
if model_name == "Qwen-VL-Chat":
model_name = "infer-qwen-vl"
elif model_name == "llava-llama-3-8b-v1_1-transformers":
model_name = "infer-lav-lam-v1-1"
else:
model_name = "infer-qwen-vl"
url = 'http://120.196.116.194:48894/' + model_name + '/'
if is_base64(images):
images_data = images
else:
with open(images, "rb") as img_file:
images_data = str(base64.b64encode(img_file.read()), 'utf-8')
data_input = {'model': model_name, 'prompt': prompt, 'img_data': images_data}
data = requests.post(url, json=data_input)
return data.text
async def fast_api_handler(self, request: Request) -> Response:
try:
data = await request.json()
except:
return JSONResponse(content={"error": "json parse error"}, status_code=status.HTTP_400_BAD_REQUEST)
model_name = data.get("model_name")
prompt = data.get("prompt")
img_data = data.get("img_data")
if prompt is None:
return JSONResponse(content={'error': "Question is required"}, status_code=status.HTTP_400_BAD_REQUEST)
if model_name is None or model_name.isspace():
model_name = "Qwen-VL-Chat"
jsonresp = str(JSONResponse(content={"response": self.processing(prompt, img_data, model_name)}).body, "utf-8")
return JSONResponse(content={"response": jsonresp}, status_code=status.HTTP_200_OK)
from fastapi import Request, Response, status
from fastapi.responses import JSONResponse
from injector import singleton,inject
from typing import Optional
from .blackbox import Blackbox
from ..log.logging_time import logging_time
from .chroma_query import ChromaQuery
from ..configuration import VLMConf
import requests
import base64
import io
from PIL import Image
from lmdeploy.serve.openai.api_client import APIClient
def is_base64(value) -> bool:
try:
base64.b64decode(base64.b64decode(value)) == value.encode()
return True
except Exception:
return False
@singleton
class VLMS(Blackbox):
@inject
def __init__(self, vlm_config: VLMConf):
# Chroma database initially set up for RAG for vision model.
# It could be expended to history store.
# self.chroma_query = chroma_query
self.url = vlm_config.url
def __call__(self, *args, **kwargs):
return self.processing(*args, **kwargs)
def valid(self, *args, **kwargs) -> bool:
data = args[0]
return isinstance(data, list)
def processing(self, prompt, images, model_name: Optional[str] = None) -> str:
if model_name == "Qwen-VL-Chat":
model_name = "infer-qwen-vl"
elif model_name == "llava-llama-3-8b-v1_1-transformers":
model_name = "infer-lav-lam-v1-1"
else:
model_name = "infer-qwen-vl"
## AutoLoad Model
# url = 'http://10.6.80.87:8000/' + model_name + '/'
if is_base64(images):
images_data = images
else:
# print("{}Type of image data in form {}".format('#'*20,type(images)))
# print("{}Type of image data in form {}".format('#'*20,type(images.file)))
# byte_stream = io.BytesIO(images.read())
# print("{}Type of image data in form {}".format('#'*20,type(byte_stream)))
# # roiImg = Image.open(byte_stream)
# # print("{}Successful {}".format('#'*20,type(roiImg)))
# return str(type(byte_stream))
# images_data = base64.b64encode(byte_stream)
with open(images, "rb") as img_file:
# images_data = str(base64.b64encode(img_file.read()), 'utf-8')
images_data = base64.b64encode(img_file.read())
# data_input = {'model': model_name, 'prompt': prompt, 'img_data': images_data}
# data = requests.post(url, json=data_input)
# print(data.text)
# 'https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/tests/data/tiger.jpeg'
## Lmdeploy
api_client = APIClient(self.url)
# api_client = APIClient(f'http://10.6.80.87:23333')
model_name = api_client.available_models[0]
messages = [{
'role':
'user',
'content': [{
'type': 'text',
'text': prompt,
}, {
'type': 'image_url',
'image_url': {
'url': f"data:image/jpeg;base64,{images_data}",
# './val_data/image_5.jpg',
},
}]
}
]
responses = ''
for i,item in enumerate(api_client.chat_completions_v1(model=model_name,
messages=messages#,stream = True
)):
print(item["choices"][0]["message"]['content'])
responses += item["choices"][0]["message"]['content']
return responses
# return data.text
async def fast_api_handler(self, request: Request) -> Response:
try:
data = await request.form()
except:
return JSONResponse(content={"error": "json parse error"}, status_code=status.HTTP_400_BAD_REQUEST)
model_name = data.get("model_name")
prompt = data.get("prompt")
img_data = data.get("img_data")
if prompt is None:
return JSONResponse(content={'error': "Question is required"}, status_code=status.HTTP_400_BAD_REQUEST)
if model_name is None or model_name.isspace():
model_name = "Qwen-VL-Chat"
# jsonresp = str(JSONResponse(content={"response": self.processing(prompt, img_data, model_name)}).body, "utf-8")
return JSONResponse(content={"response": self.processing(prompt, img_data, model_name)}, status_code=status.HTTP_200_OK)