ivanwu/jarvis-models
1
0
Fork 0
mirror of https://github.com/BoardWare-Genius/jarvis-models.git synced 2025-12-13 16:53:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

add asr and tts with settings

Browse Source
This commit is contained in:
0Xiao0
2024-08-22 15:26:27 +08:00
parent abb5195e55
commit c247d611a0
28 changed files with 626 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
src/blackbox/vlms.py
View File

@ -1,11 +1,19 @@
from fastapi import Request, Response, status
from fastapi.responses import JSONResponse
from .blackbox import Blackbox
from injector import singleton,inject
from typing import Optional
from .blackbox import Blackbox
from ..log.logging_time import logging_time
from .chroma_query import ChromaQuery
from ..configuration import VLMConf
import requests
import base64
import io
from PIL import Image
from lmdeploy.serve.openai.api_client import APIClient
def is_base64(value) -> bool:
try:
@ -14,9 +22,16 @@ def is_base64(value) -> bool:
except Exception:
return False
@singleton
class VLMS(Blackbox):
@inject
def __init__(self, vlm_config: VLMConf):
# Chroma database initially set up for RAG for vision model.
# It could be expended to history store.
# self.chroma_query = chroma_query
self.url = vlm_config.url
def __call__(self, *args, **kwargs):
return self.processing(*args, **kwargs)
@ -26,6 +41,7 @@ class VLMS(Blackbox):
def processing(self, prompt, images, model_name: Optional[str] = None) -> str:
# Current only Qwen-vl model
if model_name == "Qwen-VL-Chat":
model_name = "infer-qwen-vl"
elif model_name == "llava-llama-3-8b-v1_1-transformers":
@ -33,19 +49,61 @@ class VLMS(Blackbox):
else:
model_name = "infer-qwen-vl"
url = 'http://120.196.116.194:48894/' + model_name + '/'
## AutoLoad Model
# url = 'http://10.6.80.87:8000/' + model_name + '/'
if is_base64(images):
images_data = images
else:
# print("{}Type of image data in form {}".format('#'*20,type(images)))
# print("{}Type of image data in form {}".format('#'*20,type(images.file)))
# byte_stream = io.BytesIO(images.read())
# print("{}Type of image data in form {}".format('#'*20,type(byte_stream)))
# roiImg = Image.open(byte_stream)
# print("{}Successful {}".format('#'*20,type(roiImg)))
# return str(type(byte_stream))
# images_data = base64.b64encode(byte_stream)
with open(images, "rb") as img_file:
images_data = str(base64.b64encode(img_file.read()), 'utf-8')
# images_data = str(base64.b64encode(img_file.read()), 'utf-8')
images_data = base64.b64encode(img_file.read())
data_input = {'model': model_name, 'prompt': prompt, 'img_data': images_data}
# data_input = {'model': model_name, 'prompt': prompt, 'img_data': images_data}
data = requests.post(url, json=data_input)
# data = requests.post(url, json=data_input)
# 'https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/tests/data/tiger.jpeg'
return data.text
## Lmdeploy
api_client = APIClient(self.url)
model_name = api_client.available_models[0]
messages = [{
'role': 'user',
'content': [{
'type': 'text',
'text': prompt,
}, {
'type': 'image_url',
'image_url': {
'url': f"data:image/jpeg;base64,{images_data}",
# './val_data/image_5.jpg',
},
}]
}
]
responses = ''
for i,item in enumerate(api_client.chat_completions_v1(model=model_name,
messages=messages#,stream = True
)):
# print(item["choices"][0]["message"]['content'])
responses += item["choices"][0]["message"]['content']
return responses
# return data.text
async def fast_api_handler(self, request: Request) -> Response:
try:
@ -63,5 +121,6 @@ class VLMS(Blackbox):
if model_name is None or model_name.isspace():
model_name = "Qwen-VL-Chat"
jsonresp = str(JSONResponse(content={"response": self.processing(prompt, img_data, model_name)}).body, "utf-8")
return JSONResponse(content={"response": jsonresp}, status_code=status.HTTP_200_OK)
# jsonresp = str(JSONResponse(content={"response": self.processing(prompt, img_data, model_name)}).body, "utf-8")
return JSONResponse(content={"response": self.processing(prompt, img_data, model_name)}, status_code=status.HTTP_200_OK)