ivanwu/jarvis-models
1
0
Fork 0
mirror of https://github.com/BoardWare-Genius/jarvis-models.git synced 2025-12-14 00:53:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into veraGDI

Browse Source
This commit is contained in:
headbigsile
2025-03-19 16:07:30 +08:00
committed by GitHub
parent 5b84192ea9 bc6a17e75b
commit 14243a0388
6 changed files with 395 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

102
src/blackbox/vlms.py
View File

@ -1,5 +1,6 @@
from fastapi import Request, Response, status
from fastapi.responses import JSONResponse
from fastapi.responses import JSONResponse, StreamingResponse
from sse_starlette.sse import EventSourceResponse
from injector import singleton,inject
from typing import Optional, List
@ -12,14 +13,17 @@ import requests
import base64
import copy
import ast
import json
import random
from time import time
import io
from PIL import Image
from lmdeploy.serve.openai.api_client import APIClient
import io
from PIL import Image
from lmdeploy.serve.openai.api_client import APIClient
def is_base64(value) -> bool:
try:
@ -51,8 +55,8 @@ class VLMS(Blackbox):
- ignore_eos (bool): indicator for ignoring eos
- skip_special_tokens (bool): Whether or not to remove special tokens
in the decoding. Default to be True."""
self.url = vlm_config.url
self.model_dict = vlm_config.urls
self.model_url = None
self.temperature: float = 0.7
self.top_p:float = 1
self.max_tokens: (int |None) = 512
@ -82,7 +86,7 @@ class VLMS(Blackbox):
data = args[0]
return isinstance(data, list)
def processing(self, prompt:str, images:str | bytes, settings: dict, model_name: Optional[str] = None, user_context: List[dict] = None) -> str:
def processing(self, prompt:str | None, images:str | bytes | None, settings: dict, model_name: Optional[str] = None, user_context: List[dict] = None) -> str:
"""
Args:
prompt: a string query to the model.
@ -105,6 +109,9 @@ class VLMS(Blackbox):
else:
settings = {}
if not prompt:
prompt = '你是一个辅助机器人，请就此图做一个简短的概括性描述，包括图中的主体物品及状态，不超过50字。' if images else '你好'
# Transform the images into base64 format where openai format need.
if images:
if is_base64(images): # image as base64 str
@ -148,7 +155,11 @@ class VLMS(Blackbox):
# 'content': '图片中主要展示了一只老虎，它正在绿色的草地上休息。草地上有很多可以让人坐下的地方，而且看起来相当茂盛。背景比较模糊，可能是因为老虎的影响，让整个图片的其他部分都变得不太清晰了。'
# }
# ]
api_client = APIClient(self.url)
user_context = self.keep_last_k_images(user_context,k = 1)
if self.model_url is None: self.model_url = self._get_model_url(model_name)
api_client = APIClient(self.model_url)
# api_client = APIClient("http://10.6.80.91:23333")
model_name = api_client.available_models[0]
# Reformat input into openai format to request.
@ -187,21 +198,40 @@ class VLMS(Blackbox):
responses = ''
total_token_usage = 0 # which can be used to count the cost of a query
for i,item in enumerate(api_client.chat_completions_v1(model=model_name,
messages=messages,#stream = True,
messages=messages,stream = True,
**settings,
# session_id=,
)):
# Stream output
# print(item["choices"][0]["delta"]['content'],end='')
# responses += item["choices"][0]["delta"]['content']
print(item["choices"][0]["delta"]['content'],end='\n')
yield item["choices"][0]["delta"]['content']
responses += item["choices"][0]["delta"]['content']
print(item["choices"][0]["message"]['content'])
responses += item["choices"][0]["message"]['content']
# print(item["choices"][0]["message"]['content'])
# responses += item["choices"][0]["message"]['content']
# total_token_usage += item['usage']['total_tokens'] # 'usage': {'prompt_tokens': *, 'total_tokens': *, 'completion_tokens': *}
user_context = messages + [{'role': 'assistant', 'content': responses}]
return responses, user_context
self.custom_print(user_context)
# return responses, user_context
def _get_model_url(self,model_name:str | None):
available_models = {}
for model, url in self.model_dict.items():
try:
response = requests.get(url,timeout=3)
if response.status_code == 200:
available_models[model] = url
except Exception as e:
# print(e)
pass
if not available_models: print("There are no available running models and please check your endpoint urls.")
if model_name and model_name in available_models:
return available_models[model_name]
else:
model = random.choice(list(available_models.keys()))
print(f"No such model {model_name}, using {model} instead.") if model_name else print(f"Using random model {model}.")
return available_models[model]
def _into_openai_format(self, context:List[list]) -> List[dict]:
"""
Convert the data into openai format.
@ -255,7 +285,35 @@ class VLMS(Blackbox):
return user_context
def keep_last_k_images(self, user_context: list, k:int=2):
count = 0
result =[]
for item in user_context[::-1]:
if item['role'] == 'user' and len(item['content']) > 1:
for idx, info in enumerate(item['content']):
if info['type'] in ('image_url','image') and count >= k:
item['content'].pop(idx)
# item['content'].insert(idx, {'type': 'text', 'text': '<IMAGE>'})
elif info['type'] in ('image_url','image') and count < k:
count += 1
else:
continue
result.append(item)
return result[::-1]
def custom_print(self, user_context: list):
result = []
for item in user_context:
if item['role'] == 'user':
for idx, info in enumerate(item['content']):
if info['type'] in ('image_url','image'):
item['content'].pop(idx)
item['content'].insert(idx, {'type': 'image', 'image': '##<IMAGE>##'})
else:
continue
result.append(item)
print(result)
async def fast_api_handler(self, request: Request) -> Response:
## TODO: add support for multiple images and support image in form-data format
json_request = True
@ -278,7 +336,6 @@ class VLMS(Blackbox):
prompt = data.get("prompt")
settings: dict = data.get('settings')
context = data.get("context")
if not context:
user_context = []
elif isinstance(context[0], list):
@ -297,10 +354,13 @@ class VLMS(Blackbox):
if prompt is None:
return JSONResponse(content={'error': "Question is required"}, status_code=status.HTTP_400_BAD_REQUEST)
if model_name is None or model_name.isspace():
model_name = "Qwen-VL-Chat"
# if model_name is None or model_name.isspace():
# model_name = "Qwen-VL-Chat"
# response,_ = self.processing(prompt, img_data,settings, model_name,user_context=user_context)
# return StreamingResponse(self.processing(prompt, img_data,settings, model_name,user_context=user_context), status_code=status.HTTP_200_OK)
return EventSourceResponse(self.processing(prompt, img_data,settings, model_name,user_context=user_context), status_code=status.HTTP_200_OK)
# HTTP JsonResponse
response, history = self.processing(prompt, img_data,settings, model_name,user_context=user_context)
# jsonresp = str(JSONResponse(content={"response": self.processing(prompt, img_data, model_name)}).body, "utf-8")
return JSONResponse(content={"response": response}, status_code=status.HTTP_200_OK)
# return JSONResponse(content={"response": response}, status_code=status.HTTP_200_OK)