ivanwu/jarvis-models
1
0
Fork 0
mirror of https://github.com/BoardWare-Genius/jarvis-models.git synced 2025-12-13 16:53:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

support vlms streaming output text

Browse Source
This commit is contained in:
Ivan087
2025-03-05 14:46:16 +08:00
parent 2c450a5ffe
commit cdf5c214d0
1 changed files with 15 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
src/blackbox/vlms.py
View File

@ -1,5 +1,6 @@
from fastapi import Request, Response, status
from fastapi.responses import JSONResponse
from fastapi.responses import JSONResponse, StreamingResponse
from sse_starlette.sse import EventSourceResponse
from injector import singleton,inject
from typing import Optional, List
@ -194,20 +195,21 @@ class VLMS(Blackbox):
responses = ''
total_token_usage = 0 # which can be used to count the cost of a query
for i,item in enumerate(api_client.chat_completions_v1(model=model_name,
messages=messages,#stream = True,
messages=messages,stream = True,
**settings,
# session_id=,
)):
# Stream output
# print(item["choices"][0]["delta"]['content'],end='')
# responses += item["choices"][0]["delta"]['content']
print(item["choices"][0]["delta"]['content'],end='\n')
yield item["choices"][0]["delta"]['content']
responses += item["choices"][0]["delta"]['content']
print(item["choices"][0]["message"]['content'])
responses += item["choices"][0]["message"]['content']
# print(item["choices"][0]["message"]['content'])
# responses += item["choices"][0]["message"]['content']
# total_token_usage += item['usage']['total_tokens'] # 'usage': {'prompt_tokens': *, 'total_tokens': *, 'completion_tokens': *}
user_context = messages + [{'role': 'assistant', 'content': responses}]
self.custom_print(user_context)
return responses, user_context
# return responses, user_context
def _get_model_url(self,model_name:str | None):
available_models = {}
@ -346,8 +348,11 @@ class VLMS(Blackbox):
# if model_name is None or model_name.isspace():
# model_name = "Qwen-VL-Chat"
# response,_ = self.processing(prompt, img_data,settings, model_name,user_context=user_context)
# return StreamingResponse(self.processing(prompt, img_data,settings, model_name,user_context=user_context), status_code=status.HTTP_200_OK)
return EventSourceResponse(self.processing(prompt, img_data,settings, model_name,user_context=user_context), status_code=status.HTTP_200_OK)
# HTTP JsonResponse
response, history = self.processing(prompt, img_data,settings, model_name,user_context=user_context)
# jsonresp = str(JSONResponse(content={"response": self.processing(prompt, img_data, model_name)}).body, "utf-8")
return JSONResponse(content={"response": response}, status_code=status.HTTP_200_OK)
# return JSONResponse(content={"response": response}, status_code=status.HTTP_200_OK)