update space

app.py

@@ -1,63 +1,154 @@
 import gradio as gr
-from huggingface_hub import InferenceClient
-
-"""
-For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
-"""
-client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
-
-
-def respond(
-    message,
-    history: list[tuple[str, str]],
-    system_message,
-    max_tokens,
-    temperature,
-    top_p,
-):
-    messages = [{"role": "system", "content": system_message}]
-
-    for val in history:
-        if val[0]:
-            messages.append({"role": "user", "content": val[0]})
-        if val[1]:
-            messages.append({"role": "assistant", "content": val[1]})
-
-    messages.append({"role": "user", "content": message})
-
-    response = ""
-
-    for message in client.chat_completion(
-        messages,
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
-    ):
-        token = message.choices[0].delta.content
-
-        response += token
-        yield response
-
-"""
-For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
-"""
-demo = gr.ChatInterface(
-    respond,
-    additional_inputs=[
-        gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
-        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
-        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
-        gr.Slider(
-            minimum=0.1,
-            maximum=1.0,
-            value=0.95,
-            step=0.05,
-            label="Top-p (nucleus sampling)",
-        ),
-    ],
-)
+import torch
+import re
+from decord import VideoReader, cpu
+from PIL import Image
+import numpy as np
+import transformers
+from typing import Dict, Optional, Sequence, List
+
+import sys
+from oryx.conversation import conv_templates, SeparatorStyle
+from oryx.model.builder import load_pretrained_model
+from oryx.utils import disable_torch_init
+from oryx.mm_utils import tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria, process_anyres_video_genli
+from oryx.constants import IGNORE_INDEX, DEFAULT_IMAGE_TOKEN, IMAGE_TOKEN_INDEX
+
+
+model_path = "THUdyh/Oryx-7B"
+model_name = get_model_name_from_path(model_path)
+overwrite_config = {}
+overwrite_config["mm_resampler_type"] = "dynamic_compressor"
+overwrite_config["patchify_video_feature"] = False
+overwrite_config["attn_implementation"] = "sdpa" if torch.__version__ >= "2.1.2" else "eager"
+tokenizer, model, image_processor, context_len = load_pretrained_model(model_path, None, model_name, device_map="cuda:0", overwrite_config=overwrite_config)
+model.to('cuda').eval()
+
+def preprocess_qwen(sources, tokenizer: transformers.PreTrainedTokenizer, has_image: bool = False, max_len=2048, system_message: str = "You are a helpful assistant.") -> Dict:
+    roles = {"human": "<|im_start|>user", "gpt": "<|im_start|>assistant"}
+
+    im_start, im_end = tokenizer.additional_special_tokens_ids
+    nl_tokens = tokenizer("\n").input_ids
+    _system = tokenizer("system").input_ids + nl_tokens
+    _user = tokenizer("user").input_ids + nl_tokens
+    _assistant = tokenizer("assistant").input_ids + nl_tokens
+
+    # Apply prompt templates
+    input_ids, targets = [], []
+
+    source = sources
+    if roles[source[0]["from"]] != roles["human"]:
+        source = source[1:]
+
+    input_id, target = [], []
+    system = [im_start] + _system + tokenizer(system_message).input_ids + [im_end] + nl_tokens
+    input_id += system
+    target += [im_start] + [IGNORE_INDEX] * (len(system) - 3) + [im_end] + nl_tokens
+    assert len(input_id) == len(target)
+    for j, sentence in enumerate(source):
+        role = roles[sentence["from"]]
+        if has_image and sentence["value"] is not None and "<image>" in sentence["value"]:
+            num_image = len(re.findall(DEFAULT_IMAGE_TOKEN, sentence["value"]))
+            texts = sentence["value"].split('<image>')
+            _input_id = tokenizer(role).input_ids + nl_tokens 
+            for i,text in enumerate(texts):
+                _input_id += tokenizer(text).input_ids 
+                if i<len(texts)-1:
+                    _input_id += [IMAGE_TOKEN_INDEX] + nl_tokens
+            _input_id += [im_end] + nl_tokens
+            assert sum([i==IMAGE_TOKEN_INDEX for i in _input_id])==num_image
+        else:
+            if sentence["value"] is None:
+                _input_id = tokenizer(role).input_ids + nl_tokens
+            else:
+                _input_id = tokenizer(role).input_ids + nl_tokens + tokenizer(sentence["value"]).input_ids + [im_end] + nl_tokens
+        input_id += _input_id
+        if role == "<|im_start|>user":
+            _target = [im_start] + [IGNORE_INDEX] * (len(_input_id) - 3) + [im_end] + nl_tokens
+        elif role == "<|im_start|>assistant":
+            _target = [im_start] + [IGNORE_INDEX] * len(tokenizer(role).input_ids) + _input_id[len(tokenizer(role).input_ids) + 1 : -2] + [im_end] + nl_tokens
+        else:
+            raise NotImplementedError
+        target += _target
+
+    input_ids.append(input_id)
+    targets.append(target)
+    input_ids = torch.tensor(input_ids, dtype=torch.long)
+    targets = torch.tensor(targets, dtype=torch.long)
+    return input_ids
+
 
+def oryx_inference(video, text):
+    vr = VideoReader(video, ctx=cpu(0))
+    total_frame_num = len(vr)
+    fps = round(vr.get_avg_fps())
+    uniform_sampled_frames = np.linspace(0, total_frame_num - 1, 64, dtype=int)
+    frame_idx = uniform_sampled_frames.tolist()
+    spare_frames = vr.get_batch(frame_idx).asnumpy()
+    video = [Image.fromarray(frame) for frame in spare_frames]
+
+    conv_mode = "qwen_1_5"
+    
+    question = text
+    question = "<image>\n" + question
+
+    conv = conv_templates[conv_mode].copy()
+    conv.append_message(conv.roles[0], question)
+    conv.append_message(conv.roles[1], None)
+    prompt = conv.get_prompt()
+
+    input_ids = preprocess_qwen([{'from': 'human','value': question},{'from': 'gpt','value': None}], tokenizer, has_image=True).cuda()
+
+    video_processed = []
+    for idx, frame in enumerate(video):
+        image_processor.do_resize = False
+        image_processor.do_center_crop = False
+        frame = process_anyres_video_genli(frame, image_processor)
+
+        if frame_idx is not None and idx in frame_idx:
+            video_processed.append(frame.unsqueeze(0))
+        elif frame_idx is None:
+            video_processed.append(frame.unsqueeze(0))
+    
+    if frame_idx is None:
+        frame_idx = np.arange(0, len(video_processed), dtype=int).tolist()
+    
+    video_processed = torch.cat(video_processed, dim=0).bfloat16().cuda()
+    video_processed = (video_processed, video_processed)
+
+    video_data = (video_processed, (384, 384), "video")
+
+    stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
+    keywords = [stop_str]
+
+    with torch.inference_mode():
+        output_ids = model.generate(
+            inputs=input_ids,
+            images=video_data[0][0],
+            images_highres=video_data[0][1],
+            modalities=video_data[2],
+            do_sample=False,
+            temperature=0,
+            max_new_tokens=1024,
+            use_cache=True,
+        )
+
+    
+    outputs = tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0]
+    outputs = outputs.strip()
+    if outputs.endswith(stop_str):
+        outputs = outputs[:-len(stop_str)]
+    outputs = outputs.strip()
+    return outputs
+
+# Define input and output for the Gradio interface
+demo = gr.Interface(
+    fn=oryx_inference,
+    inputs=[gr.Video(label="Input Video"), gr.Textbox(label="Input Text")],
+    outputs="text",
+    title="Oryx Inference",
+    description="This is a demo for Oryx inference."
+)
 
-if __name__ == "__main__":
-    demo.launch()
+# Launch the Gradio app
+demo.launch(server_name="0.0.0.0",server_port=80)

requirements.txt

@@ -1 +1,27 @@
-huggingface_hub==0.22.2
+huggingface_hub==0.22.2
+torch
+torchvision,
+transformers==4.39.2
+tokenizers==0.15.2
+sentencepiece==0.1.99
+shortuuid
+accelerate==0.27.2
+peft==0.4.0
+bitsandbytes==0.41.0
+pydantic<2,>=1
+markdown2
+numpy
+scikit-learn==1.2.2
+gradio==3.35.2
+gradio_client==0.2.9
+requests
+httpx==0.24.0
+uvicorn
+fastapi
+einops==0.6.1
+einops-exts==0.0.4
+timm==0.9.16
+decord
+ninja
+deepspeed==0.12.2
+protobuf