mirror of
https://github.com/BoardWare-Genius/jarvis-models.git
synced 2025-12-13 16:53:24 +00:00
modety vlms with lmdeploy
This commit is contained in:
Ivan087
@ -1,67 +1,124 @@
|
||||
from fastapi import Request, Response, status
|
||||
from fastapi.responses import JSONResponse
|
||||
from .blackbox import Blackbox
|
||||
from typing import Optional
|
||||
|
||||
import requests
|
||||
import base64
|
||||
|
||||
|
||||
def is_base64(value) -> bool:
|
||||
try:
|
||||
base64.b64decode(base64.b64decode(value)) == value.encode()
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class VLMS(Blackbox):
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
return self.processing(*args, **kwargs)
|
||||
|
||||
def valid(self, *args, **kwargs) -> bool:
|
||||
data = args[0]
|
||||
return isinstance(data, list)
|
||||
|
||||
def processing(self, prompt, images, model_name: Optional[str] = None) -> str:
|
||||
|
||||
if model_name == "Qwen-VL-Chat":
|
||||
model_name = "infer-qwen-vl"
|
||||
elif model_name == "llava-llama-3-8b-v1_1-transformers":
|
||||
model_name = "infer-lav-lam-v1-1"
|
||||
else:
|
||||
model_name = "infer-qwen-vl"
|
||||
|
||||
url = 'http://120.196.116.194:48894/' + model_name + '/'
|
||||
|
||||
if is_base64(images):
|
||||
images_data = images
|
||||
else:
|
||||
with open(images, "rb") as img_file:
|
||||
images_data = str(base64.b64encode(img_file.read()), 'utf-8')
|
||||
|
||||
data_input = {'model': model_name, 'prompt': prompt, 'img_data': images_data}
|
||||
|
||||
data = requests.post(url, json=data_input)
|
||||
|
||||
return data.text
|
||||
|
||||
async def fast_api_handler(self, request: Request) -> Response:
|
||||
try:
|
||||
data = await request.json()
|
||||
except:
|
||||
return JSONResponse(content={"error": "json parse error"}, status_code=status.HTTP_400_BAD_REQUEST)
|
||||
|
||||
model_name = data.get("model_name")
|
||||
prompt = data.get("prompt")
|
||||
img_data = data.get("img_data")
|
||||
|
||||
if prompt is None:
|
||||
return JSONResponse(content={'error': "Question is required"}, status_code=status.HTTP_400_BAD_REQUEST)
|
||||
|
||||
if model_name is None or model_name.isspace():
|
||||
model_name = "Qwen-VL-Chat"
|
||||
|
||||
jsonresp = str(JSONResponse(content={"response": self.processing(prompt, img_data, model_name)}).body, "utf-8")
|
||||
return JSONResponse(content={"response": jsonresp}, status_code=status.HTTP_200_OK)
|
||||
from fastapi import Request, Response, status
|
||||
from fastapi.responses import JSONResponse
|
||||
from injector import singleton,inject
|
||||
from typing import Optional
|
||||
|
||||
from .blackbox import Blackbox
|
||||
from ..log.logging_time import logging_time
|
||||
from .chroma_query import ChromaQuery
|
||||
from ..configuration import VLMConf
|
||||
|
||||
import requests
|
||||
import base64
|
||||
|
||||
import io
|
||||
from PIL import Image
|
||||
from lmdeploy.serve.openai.api_client import APIClient
|
||||
|
||||
def is_base64(value) -> bool:
|
||||
try:
|
||||
base64.b64decode(base64.b64decode(value)) == value.encode()
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
@singleton
|
||||
class VLMS(Blackbox):
|
||||
|
||||
@inject
|
||||
def __init__(self, vlm_config: VLMConf):
|
||||
# Chroma database initially set up for RAG for vision model.
|
||||
# It could be expended to history store.
|
||||
# self.chroma_query = chroma_query
|
||||
self.url = vlm_config.url
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
return self.processing(*args, **kwargs)
|
||||
|
||||
def valid(self, *args, **kwargs) -> bool:
|
||||
data = args[0]
|
||||
return isinstance(data, list)
|
||||
|
||||
def processing(self, prompt, images, model_name: Optional[str] = None) -> str:
|
||||
|
||||
if model_name == "Qwen-VL-Chat":
|
||||
model_name = "infer-qwen-vl"
|
||||
elif model_name == "llava-llama-3-8b-v1_1-transformers":
|
||||
model_name = "infer-lav-lam-v1-1"
|
||||
else:
|
||||
model_name = "infer-qwen-vl"
|
||||
|
||||
|
||||
## AutoLoad Model
|
||||
# url = 'http://10.6.80.87:8000/' + model_name + '/'
|
||||
|
||||
if is_base64(images):
|
||||
images_data = images
|
||||
else:
|
||||
# print("{}Type of image data in form {}".format('#'*20,type(images)))
|
||||
# print("{}Type of image data in form {}".format('#'*20,type(images.file)))
|
||||
# byte_stream = io.BytesIO(images.read())
|
||||
# print("{}Type of image data in form {}".format('#'*20,type(byte_stream)))
|
||||
# # roiImg = Image.open(byte_stream)
|
||||
# # print("{}Successful {}".format('#'*20,type(roiImg)))
|
||||
# return str(type(byte_stream))
|
||||
# images_data = base64.b64encode(byte_stream)
|
||||
with open(images, "rb") as img_file:
|
||||
# images_data = str(base64.b64encode(img_file.read()), 'utf-8')
|
||||
images_data = base64.b64encode(img_file.read())
|
||||
|
||||
# data_input = {'model': model_name, 'prompt': prompt, 'img_data': images_data}
|
||||
|
||||
# data = requests.post(url, json=data_input)
|
||||
# print(data.text)
|
||||
# 'https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/tests/data/tiger.jpeg'
|
||||
## Lmdeploy
|
||||
api_client = APIClient(self.url)
|
||||
# api_client = APIClient(f'http://10.6.80.87:23333')
|
||||
model_name = api_client.available_models[0]
|
||||
messages = [{
|
||||
'role':
|
||||
'user',
|
||||
'content': [{
|
||||
'type': 'text',
|
||||
'text': prompt,
|
||||
}, {
|
||||
'type': 'image_url',
|
||||
'image_url': {
|
||||
'url': f"data:image/jpeg;base64,{images_data}",
|
||||
# './val_data/image_5.jpg',
|
||||
},
|
||||
}]
|
||||
}
|
||||
]
|
||||
|
||||
responses = ''
|
||||
for i,item in enumerate(api_client.chat_completions_v1(model=model_name,
|
||||
messages=messages#,stream = True
|
||||
)):
|
||||
print(item["choices"][0]["message"]['content'])
|
||||
responses += item["choices"][0]["message"]['content']
|
||||
|
||||
return responses
|
||||
|
||||
# return data.text
|
||||
|
||||
async def fast_api_handler(self, request: Request) -> Response:
|
||||
try:
|
||||
data = await request.form()
|
||||
except:
|
||||
return JSONResponse(content={"error": "json parse error"}, status_code=status.HTTP_400_BAD_REQUEST)
|
||||
|
||||
model_name = data.get("model_name")
|
||||
prompt = data.get("prompt")
|
||||
img_data = data.get("img_data")
|
||||
|
||||
if prompt is None:
|
||||
return JSONResponse(content={'error': "Question is required"}, status_code=status.HTTP_400_BAD_REQUEST)
|
||||
|
||||
if model_name is None or model_name.isspace():
|
||||
model_name = "Qwen-VL-Chat"
|
||||
|
||||
# jsonresp = str(JSONResponse(content={"response": self.processing(prompt, img_data, model_name)}).body, "utf-8")
|
||||
|
||||
return JSONResponse(content={"response": self.processing(prompt, img_data, model_name)}, status_code=status.HTTP_200_OK)
|
||||
@ -1,131 +1,138 @@
|
||||
|
||||
from dataclasses import dataclass
|
||||
from injector import inject,singleton
|
||||
import yaml
|
||||
import sys
|
||||
import logging
|
||||
|
||||
@singleton
|
||||
class Configuration():
|
||||
|
||||
@inject
|
||||
def __init__(self) -> None:
|
||||
config_file_path = ""
|
||||
try:
|
||||
config_file_path = sys.argv[1]
|
||||
except:
|
||||
config_file_path = ".env.yaml"
|
||||
with open(config_file_path) as f:
|
||||
cfg = yaml.load(f, Loader=yaml.FullLoader)
|
||||
self.cfg = cfg
|
||||
|
||||
def getDict(self):
|
||||
return self.cfg
|
||||
|
||||
"""
|
||||
# yaml 檔中的路徑 get("aaa.bbb.ccc")
|
||||
aaa:
|
||||
bbb:
|
||||
ccc: "hello world"
|
||||
"""
|
||||
def get(self, path: str | list[str], cfg: dict = None, default=None):
|
||||
if isinstance(path, str):
|
||||
if cfg is None:
|
||||
cfg = self.cfg
|
||||
return self.get(path.split("."), cfg)
|
||||
length = len(path)
|
||||
if length == 0 or not isinstance(cfg, dict):
|
||||
return default
|
||||
if length == 1:
|
||||
return cfg.get(path[0])
|
||||
return self.get(path[1:], cfg.get(path[0]))
|
||||
|
||||
class TesouConf():
|
||||
url: str
|
||||
|
||||
@inject
|
||||
def __init__(self, config: Configuration) -> None:
|
||||
self.url = config.get("tesou.url")
|
||||
|
||||
|
||||
class MeloConf():
|
||||
mode: str
|
||||
url: str
|
||||
speed: int
|
||||
device: str
|
||||
language: str
|
||||
speaker: str
|
||||
|
||||
@inject
|
||||
def __init__(self, config: Configuration) -> None:
|
||||
self.mode = config.get("melotts.mode")
|
||||
self.url = config.get("melotts.url")
|
||||
self.speed = config.get("melotts.speed")
|
||||
self.device = config.get("melotts.device")
|
||||
self.language = config.get("melotts.language")
|
||||
self.speaker = config.get("melotts.speaker")
|
||||
|
||||
class CosyVoiceConf():
|
||||
mode: str
|
||||
url: str
|
||||
speed: int
|
||||
device: str
|
||||
language: str
|
||||
speaker: str
|
||||
|
||||
@inject
|
||||
def __init__(self, config: Configuration) -> None:
|
||||
self.mode = config.get("cosyvoicetts.mode")
|
||||
self.url = config.get("cosyvoicetts.url")
|
||||
self.speed = config.get("cosyvoicetts.speed")
|
||||
self.device = config.get("cosyvoicetts.device")
|
||||
self.language = config.get("cosyvoicetts.language")
|
||||
self.speaker = config.get("cosyvoicetts.speaker")
|
||||
|
||||
# 'CRITICAL': CRITICAL,
|
||||
# 'FATAL': FATAL,
|
||||
# 'ERROR': ERROR,
|
||||
# 'WARN': WARNING,
|
||||
# 'WARNING': WARNING,
|
||||
# 'INFO': INFO,
|
||||
# 'DEBUG': DEBUG,
|
||||
# 'NOTSET': NOTSET,
|
||||
DEFAULT_LEVEL="WARNING"
|
||||
DEFAULT_TIME_FORMAT="%Y-%m-%d %H:%M:%S"
|
||||
|
||||
@singleton
|
||||
class LogConf():
|
||||
level: int
|
||||
time_format = "%Y-%m-%d %H:%M:%S"
|
||||
filename: str | None
|
||||
@inject
|
||||
def __init__(self, config: Configuration) -> None:
|
||||
self.level = config.get("log.level")
|
||||
c = config.get("log.level", default=DEFAULT_LEVEL).upper()
|
||||
level=logging._nameToLevel.get(c)
|
||||
if level is None:
|
||||
self.level = logging.WARNING
|
||||
else:
|
||||
self.level = level
|
||||
self.filename = config.get("log.filename")
|
||||
self.time_format = config.get("log.time_format", default=DEFAULT_TIME_FORMAT)
|
||||
|
||||
@singleton
|
||||
class EnvConf():
|
||||
version: str
|
||||
host: str
|
||||
port: str
|
||||
@inject
|
||||
def __init__(self, config: Configuration) -> None:
|
||||
self.version = "0.0.1"
|
||||
self.host = config.get("env.host", default="0.0.0.0")
|
||||
self.port = config.get("env.port", default="8080")
|
||||
|
||||
@singleton
|
||||
@dataclass
|
||||
class BlackboxConf():
|
||||
lazyloading: bool
|
||||
|
||||
@inject
|
||||
def __init__(self, config: Configuration) -> None:
|
||||
self.lazyloading = bool(config.get("blackbox.lazyloading", default=False))
|
||||
|
||||
from dataclasses import dataclass
|
||||
from injector import inject,singleton
|
||||
import yaml
|
||||
import sys
|
||||
import logging
|
||||
|
||||
@singleton
|
||||
class Configuration():
|
||||
|
||||
@inject
|
||||
def __init__(self) -> None:
|
||||
config_file_path = ""
|
||||
try:
|
||||
config_file_path = sys.argv[1]
|
||||
except:
|
||||
config_file_path = ".env.yaml"
|
||||
with open(config_file_path) as f:
|
||||
cfg = yaml.load(f, Loader=yaml.FullLoader)
|
||||
self.cfg = cfg
|
||||
|
||||
def getDict(self):
|
||||
return self.cfg
|
||||
|
||||
"""
|
||||
# yaml 檔中的路徑 get("aaa.bbb.ccc")
|
||||
aaa:
|
||||
bbb:
|
||||
ccc: "hello world"
|
||||
"""
|
||||
def get(self, path: str | list[str], cfg: dict = None, default=None):
|
||||
if isinstance(path, str):
|
||||
if cfg is None:
|
||||
cfg = self.cfg
|
||||
return self.get(path.split("."), cfg)
|
||||
length = len(path)
|
||||
if length == 0 or not isinstance(cfg, dict):
|
||||
return default
|
||||
if length == 1:
|
||||
return cfg.get(path[0])
|
||||
return self.get(path[1:], cfg.get(path[0]))
|
||||
|
||||
class TesouConf():
|
||||
url: str
|
||||
|
||||
@inject
|
||||
def __init__(self, config: Configuration) -> None:
|
||||
self.url = config.get("tesou.url")
|
||||
|
||||
|
||||
class MeloConf():
|
||||
mode: str
|
||||
url: str
|
||||
speed: int
|
||||
device: str
|
||||
language: str
|
||||
speaker: str
|
||||
|
||||
@inject
|
||||
def __init__(self, config: Configuration) -> None:
|
||||
self.mode = config.get("melotts.mode")
|
||||
self.url = config.get("melotts.url")
|
||||
self.speed = config.get("melotts.speed")
|
||||
self.device = config.get("melotts.device")
|
||||
self.language = config.get("melotts.language")
|
||||
self.speaker = config.get("melotts.speaker")
|
||||
|
||||
class CosyVoiceConf():
|
||||
mode: str
|
||||
url: str
|
||||
speed: int
|
||||
device: str
|
||||
language: str
|
||||
speaker: str
|
||||
|
||||
@inject
|
||||
def __init__(self, config: Configuration) -> None:
|
||||
self.mode = config.get("cosyvoicetts.mode")
|
||||
self.url = config.get("cosyvoicetts.url")
|
||||
self.speed = config.get("cosyvoicetts.speed")
|
||||
self.device = config.get("cosyvoicetts.device")
|
||||
self.language = config.get("cosyvoicetts.language")
|
||||
self.speaker = config.get("cosyvoicetts.speaker")
|
||||
|
||||
# 'CRITICAL': CRITICAL,
|
||||
# 'FATAL': FATAL,
|
||||
# 'ERROR': ERROR,
|
||||
# 'WARN': WARNING,
|
||||
# 'WARNING': WARNING,
|
||||
# 'INFO': INFO,
|
||||
# 'DEBUG': DEBUG,
|
||||
# 'NOTSET': NOTSET,
|
||||
DEFAULT_LEVEL="WARNING"
|
||||
DEFAULT_TIME_FORMAT="%Y-%m-%d %H:%M:%S"
|
||||
|
||||
@singleton
|
||||
class LogConf():
|
||||
level: int
|
||||
time_format = "%Y-%m-%d %H:%M:%S"
|
||||
filename: str | None
|
||||
@inject
|
||||
def __init__(self, config: Configuration) -> None:
|
||||
self.level = config.get("log.level")
|
||||
c = config.get("log.level", default=DEFAULT_LEVEL).upper()
|
||||
level=logging._nameToLevel.get(c)
|
||||
if level is None:
|
||||
self.level = logging.WARNING
|
||||
else:
|
||||
self.level = level
|
||||
self.filename = config.get("log.filename")
|
||||
self.time_format = config.get("log.time_format", default=DEFAULT_TIME_FORMAT)
|
||||
|
||||
@singleton
|
||||
class EnvConf():
|
||||
version: str
|
||||
host: str
|
||||
port: str
|
||||
@inject
|
||||
def __init__(self, config: Configuration) -> None:
|
||||
self.version = "0.0.1"
|
||||
self.host = config.get("env.host", default="0.0.0.0")
|
||||
self.port = config.get("env.port", default="8080")
|
||||
|
||||
@singleton
|
||||
@dataclass
|
||||
class BlackboxConf():
|
||||
lazyloading: bool
|
||||
|
||||
@inject
|
||||
def __init__(self, config: Configuration) -> None:
|
||||
self.lazyloading = bool(config.get("blackbox.lazyloading", default=False))
|
||||
|
||||
@singleton
|
||||
class VLMConf():
|
||||
|
||||
@inject
|
||||
def __init__(self, config: Configuration) -> None:
|
||||
self.url = config.get("vlms.url")
|
||||
Reference in New Issue
Block a user