adding video random sampling function

model/fetch_museum_results/__init__.py

@@ -1,5 +1,5 @@
 from .imagen_museum import TASK_DICT, DOMAIN
-from .imagen_museum import fetch_indexes
+from .imagen_museum import fetch_indexes, fetch_indexes_no_csv
 import random
 
 ARENA_TO_IG_MUSEUM = {"LCM(v1.5/XL)":"LCM", 
@@ -56,3 +56,36 @@ def draw_from_imagen_museum(task, model_name):
         return [[source_image_link, image_link], [input_caption, output_caption, instruction]]
     else:
         raise ValueError("Task not supported")
+
+def draw2_from_videogen_museum(task, model_name1, model_name2):
+    domain = "https://github.com/ChromAIca/VideoGenMuseum/raw/main/Museum/"
+    baselink = domain + "VideoGenHub_Text-Guided_VG"
+
+    matched_results = fetch_indexes_no_csv(baselink)
+    r = random.Random()
+    uid, value = r.choice(list(matched_results.items()))
+    video_link_1 = baselink + "/" + model_name1 + "/" + uid
+    video_link_2 = baselink + "/" + model_name2 + "/" + uid
+
+    if task == "t2v": # Video Gen
+        prompt = value['prompt_en']
+        return [[video_link_1, video_link_2], [prompt]]
+    else:
+        raise ValueError("Task not supported")
+
+def draw_from_videogen_museum(task, model_name):
+    domain = "https://github.com/ChromAIca/VideoGenMuseum/raw/main/Museum/"
+    baselink = domain + "VideoGenHub_Text-Guided_VG"
+
+    matched_results = fetch_indexes_no_csv(baselink)
+    r = random.Random()
+    uid, value = r.choice(list(matched_results.items()))
+    model = model_name
+    video_link = baselink + "/" + model + "/" + uid
+    print(video_link)
+
+    if task == "t2v": # Video Gen
+        prompt = value['prompt_en']
+        return [video_link, prompt]
+    else:
+        raise ValueError("Task not supported")

model/fetch_museum_results/imagen_museum/__init__.py

@@ -116,6 +116,10 @@ def fetch_indexes(baselink):
     matched_results = fetch_data_and_match(baselink+"/dataset_lookup.csv", baselink+"/dataset_lookup.json")
     return matched_results
 
+def fetch_indexes_no_csv(baselink):
+    matched_results = fetch_json_data(baselink+"/dataset_lookup.json")
+    return matched_results
+
 if __name__ == "__main__":
     domain = "https://chromaica.github.io/Museum/"
     baselink = domain + "ImagenHub_Text-Guided_IE"

model/model_manager.py

@@ -6,7 +6,7 @@ import io, base64, json
 import spaces
 from PIL import Image
 from .models import IMAGE_GENERATION_MODELS, IMAGE_EDITION_MODELS, VIDEO_GENERATION_MODELS, load_pipeline
-from .fetch_museum_results import draw_from_imagen_museum, draw2_from_imagen_museum
+from .fetch_museum_results import draw_from_imagen_museum, draw2_from_imagen_museum, draw_from_videogen_museum, draw2_from_videogen_museum
 
 class ModelManager:
     def __init__(self):
@@ -161,7 +161,12 @@ class ModelManager:
         return result
 
     def generate_video_vg_museum(self, model_name):
-        raise NotImplementedError
+        model_name = model_name.split('_')[1]
+        result_list = draw_from_videogen_museum("t2v", model_name)
+        video_link = result_list[0]
+        prompt = result_list[1]
+
+        return video_link, prompt
 
     def generate_video_vg_parallel_anony(self, prompt, model_A, model_B):
         if model_A == "" and model_B == "":
@@ -181,7 +186,14 @@ class ModelManager:
         else:
             model_names = [model_A, model_B]
 
-        raise NotImplementedError
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            model_1 = model_names[0].split('_')[1]
+            model_2 = model_names[1].split('_')[1]
+            result_list = draw2_from_videogen_museum("t2v", model_1, model_2)
+            video_links = result_list[0]
+            prompt_list = result_list[1]
+
+        return video_links[0], video_links[1], model_names[0], model_names[1], prompt_list[0]
     
     def generate_video_vg_parallel(self, prompt, model_A, model_B):
         model_names = [model_A, model_B]
@@ -193,5 +205,10 @@ class ModelManager:
 
     def generate_video_vg_museum_parallel(self, model_A, model_B):
         model_names = [model_A, model_B]
-
-        raise NotImplementedError
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            model_1 = model_A.split('_')[1]
+            model_2 = model_B.split('_')[1]
+            result_list = draw2_from_videogen_museum("t2v", model_1, model_2)
+            video_links = result_list[0]
+            prompt_list = result_list[1]
+        return video_links[0], video_links[1], prompt_list[0]

serve/gradio_web_video_generation.py

@@ -9,8 +9,11 @@ from .vote_utils import (
     bothbad_vote_last_response_vgm as bothbad_vote_last_response,
     # share_click_vgm as share_click,
     generate_vg,
+    generate_vg_museum,
     generate_vgm,
+    generate_vgm_museum,
     generate_vgm_annoy,
+    generate_vgm_annoy_museum,
     share_js
 )
 from functools import partial
@@ -38,6 +41,7 @@ Find out who is the 🥇conditional video generation models! More models are goi
     state0 = gr.State()
     state1 = gr.State()
     gen_func = partial(generate_vgm_annoy, models.generate_video_vg_parallel_anony)
+    gen_func_random = partial(generate_vgm_annoy_museum, models.generate_video_vg_museum_parallel_anony)
 
     gr.Markdown(notice_markdown, elem_id="notice_markdown")
 
@@ -79,6 +83,8 @@ Find out who is the 🥇conditional video generation models! More models are goi
             elem_id="input_box",
         )
         send_btn = gr.Button(value="Send", variant="primary", scale=0)
+        draw_btn = gr.Button(value="🎲 Random sample", variant="primary", scale=0)
+
 
     with gr.Row():
         clear_btn = gr.Button(value="🎲 New Round", interactive=False)
@@ -120,6 +126,17 @@ Find out who is the 🥇conditional video generation models! More models are goi
         outputs=btn_list
     )
 
+    draw_btn.click(
+        gen_func_random,
+        inputs=[state0, state1, model_selector_left, model_selector_right],
+        outputs=[state0, state1, chatbot_left, chatbot_right, textbox, model_selector_left, model_selector_right],
+        api_name="draw_btn_annony"
+    ).then(
+        enable_buttons_side_by_side,
+        inputs=None,
+        outputs=btn_list
+    )
+
     clear_btn.click(
         clear_history_side_by_side_anony,
         inputs=None,
@@ -190,6 +207,7 @@ def build_side_by_side_ui_named_vg(models):
     state0 = gr.State()
     state1 = gr.State()
     gen_func = partial(generate_vgm, models.generate_video_vg_parallel)
+    gen_func_random = partial(generate_vgm_museum, models.generate_video_vg_museum_parallel)
     gr.Markdown(notice_markdown, elem_id="notice_markdown")
 
     with gr.Group(elem_id="share-region-named"):
@@ -239,6 +257,8 @@ def build_side_by_side_ui_named_vg(models):
             elem_id="input_box"
         )
         send_btn = gr.Button(value="Send", variant="primary", scale=0)
+        draw_btn = gr.Button(value="🎲 Random sample", variant="primary", scale=0)
+
 
     with gr.Row():
         clear_btn = gr.Button(value="🗑️  Clear history", interactive=False)
@@ -289,6 +309,17 @@ def build_side_by_side_ui_named_vg(models):
         inputs=None,
         outputs=btn_list
     )
+    draw_btn.click(
+        gen_func_random,
+        inputs=[state0, state1, model_selector_left, model_selector_right],
+        outputs=[state0, state1, chatbot_left, chatbot_right, textbox],
+        api_name="draw_side_by_side"
+    ).then(
+        enable_buttons_side_by_side,
+        inputs=None,
+        outputs=btn_list
+    )
+
     regenerate_btn.click(
         gen_func,
         inputs=[state0, state1, textbox, model_selector_left, model_selector_right],
@@ -358,6 +389,7 @@ def build_single_model_ui_vg(models, add_promotion_links=False):
 
     state = gr.State()
     gen_func = partial(generate_vg, models.generate_video_vg)
+    gen_func_random = partial(generate_vg_museum, models.generate_video_vg_museum)
     gr.Markdown(notice_markdown, elem_id="notice_markdown")
 
     model_list = models.model_vg_list
@@ -387,6 +419,8 @@ def build_single_model_ui_vg(models, add_promotion_links=False):
         )
 
         send_btn = gr.Button(value="Send", variant="primary", scale=0)
+        draw_btn = gr.Button(value="🎲 Random sample", variant="primary", scale=0)
+
 
     with gr.Row():
         chatbot = gr.Video(width=512, autoplay=True)
@@ -442,6 +476,18 @@ def build_single_model_ui_vg(models, add_promotion_links=False):
         outputs=btn_list
     )
 
+    draw_btn.click(
+        gen_func_random,
+        inputs=[state, model_selector],
+        outputs=[state, chatbot, textbox],
+        api_name="draw_btn_single",
+        show_progress="full"
+    ).success(
+        enable_buttons,
+        inputs=None,
+        outputs=btn_list
+    )
+
     upvote_btn.click(
         upvote_last_response,
         inputs=[state, model_selector],

serve/vote_utils.py

@@ -1249,6 +1249,46 @@ def generate_vg(gen_func, state, text, model_name, request: gr.Request):
     save_video_file_on_log_server(output_file)
     yield state, output_file
 
+def generate_vg_museum(gen_func, state, model_name, request: gr.Request):
+    if state is None:
+        state = VideoStateVG(model_name)
+    ip = get_ip(request)
+    vg_logger.info(f"generate. ip: {ip}")
+    start_tstamp = time.time()
+    generated_video, text = gen_func(model_name)
+    state.prompt = text
+    state.output = generated_video
+    state.model_name = model_name
+
+    # yield state, generated_video
+
+    finish_tstamp = time.time()
+
+    with open(get_conv_log_filename(), "a") as fout:
+        data = {
+            "tstamp": round(finish_tstamp, 4),
+            "type": "chat",
+            "model": model_name,
+            "gen_params": {},
+            "start": round(start_tstamp, 4),
+            "finish": round(finish_tstamp, 4),
+            "state": state.dict(),
+            "ip": get_ip(request),
+        }
+        fout.write(json.dumps(data) + "\n")
+        append_json_item_on_log_server(data, get_conv_log_filename())
+
+    output_file = f'{VIDEO_DIR}/generation/{state.conv_id}.mp4'
+    os.makedirs(os.path.dirname(output_file), exist_ok=True)
+
+    r = requests.get(state.output)
+    with open(output_file, 'wb') as outfile:
+        outfile.write(r.content)
+
+    save_video_file_on_log_server(output_file)
+    yield state, output_file, text
+
+
 def generate_vgm(gen_func, state0, state1, text, model_name0, model_name1, request: gr.Request):
     if not text:
         raise gr.Warning("Prompt cannot be empty.")
@@ -1326,6 +1366,72 @@ def generate_vgm(gen_func, state0, state1, text, model_name0, model_name1, reque
         save_video_file_on_log_server(output_file)
     yield state0, state1, f'{VIDEO_DIR}/generation/{state0.conv_id}.mp4', f'{VIDEO_DIR}/generation/{state1.conv_id}.mp4'
 
+def generate_vgm_museum(gen_func, state0, state1, model_name0, model_name1, request: gr.Request):
+    if state0 is None:
+        state0 = VideoStateVG(model_name0)
+    if state1 is None:
+        state1 = VideoStateVG(model_name1)
+    ip = get_ip(request)
+    igm_logger.info(f"generate. ip: {ip}")
+    start_tstamp = time.time()
+    # Remove ### Model (A|B): from model name
+    model_name0 = re.sub(r"### Model A: ", "", model_name0)
+    model_name1 = re.sub(r"### Model B: ", "", model_name1)
+    generated_video0, generated_video1, text = gen_func(model_name0, model_name1)
+    state0.prompt = text
+    state1.prompt = text
+    state0.output = generated_video0
+    state1.output = generated_video1
+    state0.model_name = model_name0
+    state1.model_name = model_name1
+
+    # yield state0, state1, generated_video0, generated_video1
+    print("====== model name =========")
+    print(state0.model_name)
+    print(state1.model_name)
+
+
+    finish_tstamp = time.time()
+
+
+    with open(get_conv_log_filename(), "a") as fout:
+        data = {
+            "tstamp": round(finish_tstamp, 4),
+            "type": "chat",
+            "model": model_name0,
+            "gen_params": {},
+            "start": round(start_tstamp, 4),
+            "finish": round(finish_tstamp, 4),
+            "state": state0.dict(),
+            "ip": get_ip(request),
+        }
+        fout.write(json.dumps(data) + "\n")
+        append_json_item_on_log_server(data, get_conv_log_filename())
+        data = {
+            "tstamp": round(finish_tstamp, 4),
+            "type": "chat",
+            "model": model_name1,
+            "gen_params": {},
+            "start": round(start_tstamp, 4),
+            "finish": round(finish_tstamp, 4),
+            "state": state1.dict(),
+            "ip": get_ip(request),
+        }
+        fout.write(json.dumps(data) + "\n")
+        append_json_item_on_log_server(data, get_conv_log_filename())
+
+    for i, state in enumerate([state0, state1]):
+        output_file = f'{VIDEO_DIR}/generation/{state.conv_id}.mp4'
+        os.makedirs(os.path.dirname(output_file), exist_ok=True)
+        print(state.model_name)
+
+        r = requests.get(state.output)
+        with open(output_file, 'wb') as outfile:
+            outfile.write(r.content)
+
+        save_video_file_on_log_server(output_file)
+    yield state0, state1, f'{VIDEO_DIR}/generation/{state0.conv_id}.mp4', f'{VIDEO_DIR}/generation/{state1.conv_id}.mp4', text
+
 
 def generate_vgm_annoy(gen_func, state0, state1, text, model_name0, model_name1, request: gr.Request):
     if not text:
@@ -1393,4 +1499,67 @@ def generate_vgm_annoy(gen_func, state0, state1, text, model_name0, model_name1,
         save_video_file_on_log_server(output_file)
 
     yield state0, state1, f'{VIDEO_DIR}/generation/{state0.conv_id}.mp4', f'{VIDEO_DIR}/generation/{state1.conv_id}.mp4', \
+        gr.Markdown(f"### Model A: {model_name0}", visible=False), gr.Markdown(f"### Model B: {model_name1}", visible=False)
+
+def generate_vgm_annoy_museum(gen_func, state0, state1, model_name0, model_name1, request: gr.Request):
+    if state0 is None:
+        state0 = VideoStateVG(model_name0)
+    if state1 is None:
+        state1 = VideoStateVG(model_name1)
+    ip = get_ip(request)
+    vgm_logger.info(f"generate. ip: {ip}")
+    start_tstamp = time.time()
+    model_name0 = ""
+    model_name1 = ""
+    generated_video0, generated_video1, model_name0, model_name1, text = gen_func(model_name0, model_name1)
+    state0.prompt = text
+    state1.prompt = text
+    state0.output = generated_video0
+    state1.output = generated_video1
+    state0.model_name = model_name0
+    state1.model_name = model_name1
+    
+    # yield state0, state1, generated_video0, generated_video1, \
+    #     gr.Markdown(f"### Model A: {model_name0}"), gr.Markdown(f"### Model B: {model_name1}")
+
+    finish_tstamp = time.time()
+    # logger.info(f"===output===: {output}")
+
+    with open(get_conv_log_filename(), "a") as fout:
+        data = {
+            "tstamp": round(finish_tstamp, 4),
+            "type": "chat",
+            "model": model_name0,
+            "gen_params": {},
+            "start": round(start_tstamp, 4),
+            "finish": round(finish_tstamp, 4),
+            "state": state0.dict(),
+            "ip": get_ip(request),
+        }
+        fout.write(json.dumps(data) + "\n")
+        append_json_item_on_log_server(data, get_conv_log_filename())
+        data = {
+            "tstamp": round(finish_tstamp, 4),
+            "type": "chat",
+            "model": model_name1,
+            "gen_params": {},
+            "start": round(start_tstamp, 4),
+            "finish": round(finish_tstamp, 4),
+            "state": state1.dict(),
+            "ip": get_ip(request),
+        }
+        fout.write(json.dumps(data) + "\n")
+        append_json_item_on_log_server(data, get_conv_log_filename())
+
+    for i, state in enumerate([state0, state1]):
+        output_file = f'{VIDEO_DIR}/generation/{state.conv_id}.mp4'
+        os.makedirs(os.path.dirname(output_file), exist_ok=True)
+
+        r = requests.get(state.output)
+        with open(output_file, 'wb') as outfile:
+            outfile.write(r.content)
+
+        save_video_file_on_log_server(output_file)
+
+    yield state0, state1, f'{VIDEO_DIR}/generation/{state0.conv_id}.mp4', f'{VIDEO_DIR}/generation/{state1.conv_id}.mp4', text,\
         gr.Markdown(f"### Model A: {model_name0}", visible=False), gr.Markdown(f"### Model B: {model_name1}", visible=False)