|
import asyncio |
|
import threading |
|
import time |
|
from typing import List |
|
|
|
from fastapi import FastAPI, Request, BackgroundTasks |
|
from fastapi.responses import StreamingResponse, JSONResponse |
|
import requests |
|
|
|
from fastchat.constants import WORKER_HEART_BEAT_INTERVAL |
|
from fastchat.conversation import Conversation |
|
from fastchat.utils import pretty_print_semaphore, build_logger |
|
|
|
|
|
worker = None |
|
logger = None |
|
|
|
app = FastAPI() |
|
|
|
|
|
def heart_beat_worker(obj): |
|
while True: |
|
time.sleep(WORKER_HEART_BEAT_INTERVAL) |
|
obj.send_heart_beat() |
|
|
|
|
|
class BaseModelWorker: |
|
def __init__( |
|
self, |
|
controller_addr: str, |
|
worker_addr: str, |
|
worker_id: str, |
|
model_path: str, |
|
model_names: List[str], |
|
limit_worker_concurrency: int, |
|
conv_template: str = None, |
|
): |
|
global logger, worker |
|
|
|
self.controller_addr = controller_addr |
|
self.worker_addr = worker_addr |
|
self.worker_id = worker_id |
|
if model_path.endswith("/"): |
|
model_path = model_path[:-1] |
|
self.model_names = model_names or [model_path.split("/")[-1]] |
|
self.limit_worker_concurrency = limit_worker_concurrency |
|
self.conv = self.make_conv_template(conv_template, model_path) |
|
self.conv.sep_style = int(self.conv.sep_style) |
|
self.tokenizer = None |
|
self.context_len = None |
|
self.call_ct = 0 |
|
self.semaphore = None |
|
|
|
self.heart_beat_thread = None |
|
|
|
if logger is None: |
|
logger = build_logger("model_worker", f"model_worker_{self.worker_id}.log") |
|
if worker is None: |
|
worker = self |
|
|
|
def make_conv_template( |
|
self, |
|
conv_template: str = None, |
|
model_path: str = None, |
|
) -> Conversation: |
|
""" |
|
can be overrided to costomize the conversation template for different model workers. |
|
""" |
|
from fastchat.conversation import get_conv_template |
|
from fastchat.model.model_adapter import get_conversation_template |
|
|
|
if conv_template: |
|
conv = get_conv_template(conv_template) |
|
else: |
|
conv = get_conversation_template(model_path) |
|
return conv |
|
|
|
def init_heart_beat(self): |
|
self.register_to_controller() |
|
self.heart_beat_thread = threading.Thread( |
|
target=heart_beat_worker, |
|
args=(self,), |
|
daemon=True, |
|
) |
|
self.heart_beat_thread.start() |
|
|
|
def register_to_controller(self): |
|
logger.info("Register to controller") |
|
|
|
url = self.controller_addr + "/register_worker" |
|
data = { |
|
"worker_name": self.worker_addr, |
|
"check_heart_beat": True, |
|
"worker_status": self.get_status(), |
|
} |
|
r = requests.post(url, json=data) |
|
assert r.status_code == 200 |
|
|
|
def send_heart_beat(self): |
|
logger.info( |
|
f"Send heart beat. Models: {self.model_names}. " |
|
f"Semaphore: {pretty_print_semaphore(self.semaphore)}. " |
|
f"call_ct: {self.call_ct}. " |
|
f"worker_id: {self.worker_id}. " |
|
) |
|
|
|
url = self.controller_addr + "/receive_heart_beat" |
|
|
|
while True: |
|
try: |
|
ret = requests.post( |
|
url, |
|
json={ |
|
"worker_name": self.worker_addr, |
|
"queue_length": self.get_queue_length(), |
|
}, |
|
timeout=5, |
|
) |
|
exist = ret.json()["exist"] |
|
break |
|
except (requests.exceptions.RequestException, KeyError) as e: |
|
logger.error(f"heart beat error: {e}") |
|
time.sleep(5) |
|
|
|
if not exist: |
|
self.register_to_controller() |
|
|
|
def get_queue_length(self): |
|
if self.semaphore is None: |
|
return 0 |
|
else: |
|
sempahore_value = ( |
|
self.semaphore._value |
|
if self.semaphore._value is not None |
|
else self.limit_worker_concurrency |
|
) |
|
waiter_count = ( |
|
0 if self.semaphore._waiters is None else len(self.semaphore._waiters) |
|
) |
|
return self.limit_worker_concurrency - sempahore_value + waiter_count |
|
|
|
def get_status(self): |
|
return { |
|
"model_names": self.model_names, |
|
"speed": 1, |
|
"queue_length": self.get_queue_length(), |
|
} |
|
|
|
def count_token(self, params): |
|
prompt = params["prompt"] |
|
|
|
try: |
|
input_ids = self.tokenizer(prompt).input_ids |
|
input_echo_len = len(input_ids) |
|
except TypeError: |
|
input_echo_len = self.tokenizer.num_tokens(prompt) |
|
|
|
ret = { |
|
"count": input_echo_len, |
|
"error_code": 0, |
|
} |
|
return ret |
|
|
|
def get_conv_template(self): |
|
return {"conv": self.conv} |
|
|
|
def generate_stream_gate(self, params): |
|
raise NotImplementedError |
|
|
|
def generate_gate(self, params): |
|
raise NotImplementedError |
|
|
|
def get_embeddings(self, params): |
|
raise NotImplementedError |
|
|
|
|
|
def release_worker_semaphore(): |
|
worker.semaphore.release() |
|
|
|
|
|
def acquire_worker_semaphore(): |
|
if worker.semaphore is None: |
|
worker.semaphore = asyncio.Semaphore(worker.limit_worker_concurrency) |
|
return worker.semaphore.acquire() |
|
|
|
|
|
def create_background_tasks(): |
|
background_tasks = BackgroundTasks() |
|
background_tasks.add_task(release_worker_semaphore) |
|
return background_tasks |
|
|
|
|
|
@app.post("/worker_generate_stream") |
|
async def api_generate_stream(request: Request): |
|
params = await request.json() |
|
await acquire_worker_semaphore() |
|
generator = worker.generate_stream_gate(params) |
|
background_tasks = create_background_tasks() |
|
return StreamingResponse(generator, background=background_tasks) |
|
|
|
|
|
@app.post("/worker_generate") |
|
async def api_generate(request: Request): |
|
params = await request.json() |
|
await acquire_worker_semaphore() |
|
output = await asyncio.to_thread(worker.generate_gate, params) |
|
release_worker_semaphore() |
|
return JSONResponse(output) |
|
|
|
|
|
@app.post("/worker_get_embeddings") |
|
async def api_get_embeddings(request: Request): |
|
params = await request.json() |
|
await acquire_worker_semaphore() |
|
embedding = worker.get_embeddings(params) |
|
release_worker_semaphore() |
|
return JSONResponse(content=embedding) |
|
|
|
|
|
@app.post("/worker_get_status") |
|
async def api_get_status(request: Request): |
|
return worker.get_status() |
|
|
|
|
|
@app.post("/count_token") |
|
async def api_count_token(request: Request): |
|
params = await request.json() |
|
return worker.count_token(params) |
|
|
|
|
|
@app.post("/worker_get_conv_template") |
|
async def api_get_conv(request: Request): |
|
return worker.get_conv_template() |
|
|
|
|
|
@app.post("/model_details") |
|
async def api_model_details(request: Request): |
|
return {"context_length": worker.context_len} |
|
|
