ivanwu/jarvis-models
1
0
Fork 0
mirror of https://github.com/BoardWare-Genius/jarvis-models.git synced 2025-12-13 16:53:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: vlm support vllm, system prompt, model selection

Browse Source
This commit is contained in:
Ivan087
2025-03-21 14:46:01 +08:00
parent 8d5be0319b
commit 6f2ff6ef77
2 changed files with 73 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
README.md
View File

@ -91,5 +91,8 @@ blackbox:
lazyloading: true
vlms:
url: http://10.6.80.87:23333
urls:
qwen_vl: http://10.6.80.87:8000
qwen2_vl: http://10.6.80.87:23333
qwen2_vl_72b: http://10.6.80.91:23333
```

105
src/blackbox/vlms.py
View File

@ -24,6 +24,8 @@ import io
from PIL import Image
from lmdeploy.serve.openai.api_client import APIClient
from openai import OpenAI
def is_base64(value) -> bool:
try:
@ -56,14 +58,15 @@ class VLMS(Blackbox):
- skip_special_tokens (bool): Whether or not to remove special tokens
in the decoding. Default to be True."""
self.model_dict = vlm_config.urls
self.model_url = None
# self.model_url = None
self.available_models = {}
self.temperature: float = 0.7
self.top_p:float = 1
self.max_tokens: (int |None) = 512
self.repetition_penalty: float = 1
self.stop: (str | List[str] |None) = ['<|endoftext|>','<|im_end|>']
self.top_k: (int) = None
self.top_k: (int) = 40
self.ignore_eos: (bool) = False
self.skip_special_tokens: (bool) = True
@ -76,11 +79,16 @@ class VLMS(Blackbox):
"top_k": self.top_k,
"ignore_eos": self.ignore_eos,
"skip_special_tokens": self.skip_special_tokens,
# "system_prompt":"",
# "vlm_model_name":" ",
}
for model, url in self.model_dict.items():
try:
response = requests.get(url+'/health',timeout=3)
if response.status_code == 200:
self.available_models[model] = url
except Exception as e:
# print(e)
pass
def __call__(self, *args, **kwargs):
return self.processing(*args, **kwargs)
@ -100,21 +108,30 @@ class VLMS(Blackbox):
response: a string
history: a list
"""
config: dict = {
"lmdeploy_infer":True,
"system_prompt":"",
"vlm_model_name":"",
}
if settings:
for k in list(settings.keys()):
if k not in self.settings:
print("Warning: '{}' is not a support argument and ignore this argment, check the arguments {}".format(k,self.settings.keys()))
settings.pop(k)
config[k] = settings.pop(k)
tmp = copy.deepcopy(self.settings)
tmp.update(settings)
settings = tmp
else:
settings = {}
config['lmdeploy_infer'] = str(config['lmdeploy_infer']).strip().lower() == 'true'
if not prompt:
prompt = '你是一个辅助机器人，请就此图做一个简短的概括性描述，包括图中的主体物品及状态，不超过50字。' if images else '你好'
# Transform the images into base64 format where openai format need.
# Transform the images into base64 format where openai url)
# print(self.config['vlm_model_name'])
# print(self.available_models)format need.
if images:
if is_base64(images): # image as base64 str
images_data = images
@ -131,7 +148,6 @@ class VLMS(Blackbox):
# url = 'http://10.6.80.87:8000/' + model_name + '/'
# data_input = {'model': model_name, 'prompt': prompt, 'img_data': images_data}
# data = requests.post(url, json=data_input)
# print(data.text)
# return data.text
# 'https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/tests/data/tiger.jpeg'
@ -157,13 +173,10 @@ class VLMS(Blackbox):
# 'content': '图片中主要展示了一只老虎，它正在绿色的草地上休息。草地上有很多可以让人坐下的地方，而且看起来相当茂盛。背景比较模糊，可能是因为老虎的影响，让整个图片的其他部分都变得不太清晰了。'
# }
# ]
if not user_context and config['system_prompt']: user_context = [{'role':'system','content': config['system_prompt']}]
user_context = self.keep_last_k_images(user_context,k = 1)
if self.model_url is None: self.model_url = self._get_model_url(model_name)
api_client = APIClient(self.model_url)
# api_client = APIClient("http://10.6.80.91:23333")
model_name = api_client.available_models[0]
# if self.model_url is None: self.model_url = self._get_model_url(model_name)
# Reformat input into openai format to request.
if images_data:
messages = user_context + [{
@ -199,40 +212,60 @@ class VLMS(Blackbox):
responses = ''
total_token_usage = 0 # which can be used to count the cost of a query
for i,item in enumerate(api_client.chat_completions_v1(model=model_name,
model_url = self._get_model_url(config['vlm_model_name'])
# print(model_url)
# print(self.config['vlm_model_name'])
# print(self.available_models)
if config['lmdeploy_infer']:
api_client = APIClient(model_url)
model_name = api_client.available_models[0]
for i,item in enumerate(api_client.chat_completions_v1(model=model_name,
messages=messages,stream = True,
**settings,
# session_id=,
)):
# Stream output
print(item["choices"][0]["delta"]['content'],end='\n')
yield item["choices"][0]["delta"]['content']
responses += item["choices"][0]["delta"]['content']
# Stream output
# print(item["choices"][0]["delta"]['content'],end='\n')
yield item["choices"][0]["delta"]['content']
responses += item["choices"][0]["delta"]['content']
# print(item["choices"][0]["message"]['content'])
# responses += item["choices"][0]["message"]['content']
# total_token_usage += item['usage']['total_tokens'] # 'usage': {'prompt_tokens': *, 'total_tokens': *, 'completion_tokens': *}
# print(item["choices"][0]["message"]['content'])
# responses += item["choices"][0]["message"]['content']
# total_token_usage += item['usage']['total_tokens'] # 'usage': {'prompt_tokens': *, 'total_tokens': *, 'completion_tokens': *}
else:
api_key = "EMPTY_API_KEY"
# print(model_url+'/v1')
api_client = OpenAI(api_key=api_key, base_url=model_url+'/v1')
model_name = api_client.models.list().data[0].id
for item in api_client.chat.completions.create(
model=model_name,
messages=messages,
temperature=0.8,
top_p=0.8,
stream=True):
yield(item.choices[0].delta.content)
responses += item.choices[0].delta.content
# response = api_client.chat.completions.create(
# model=model_name,
# messages=messages,
# temperature=0.8,
# top_p=0.8)
# print(response.choices[0].message.content)
# return response.choices[0].message.content
user_context = messages + [{'role': 'assistant', 'content': responses}]
self.custom_print(user_context)
# return responses, user_context
def _get_model_url(self,model_name:str | None):
available_models = {}
for model, url in self.model_dict.items():
try:
response = requests.get(url,timeout=3)
if response.status_code == 200:
available_models[model] = url
except Exception as e:
# print(e)
pass
if not available_models: print("There are no available running models and please check your endpoint urls.")
if model_name and model_name in available_models:
return available_models[model_name]
if not self.available_models: print("There are no available running models and please check your endpoint urls.")
if model_name and model_name in self.available_models:
return self.available_models[model_name]
else:
model = random.choice(list(available_models.keys()))
model = random.choice(list(self.available_models.keys()))
print(f"No such model {model_name}, using {model} instead.") if model_name else print(f"Using random model {model}.")
return available_models[model]
return self.available_models[model]
def _into_openai_format(self, context:List[list]) -> List[dict]:
"""