Demo v2: pose, edge, edge+DB and text conditional generation

app.py

@@ -39,19 +39,14 @@ with gr.Blocks(css='style.css') as demo:
         """)
 
     with gr.Tab('Zero-Shot Text2Video'):
-        # pass
         create_demo_text_to_video(model)
     with gr.Tab('Video Instruct Pix2Pix'):
-        # pass
         create_demo_pix2pix_video(model)
     with gr.Tab('Pose Conditional'):
-        # pass
         create_demo_pose(model)
     with gr.Tab('Edge Conditional'):
-        # pass
         create_demo_canny(model)
     with gr.Tab('Edge Conditional and Dreambooth Specialized'):
-        # pass
         create_demo_canny_db(model)
 
     gr.HTML(
@@ -75,5 +70,5 @@ with gr.Blocks(css='style.css') as demo:
         </div>
         """)
 
-demo.launch(debug=True)
-# demo.queue(api_open=False).launch(file_directories=['temporal'], share=True)
+# demo.launch(debug=True)
+demo.queue(api_open=False).launch(file_directories=['temporal'], share=True)

app_canny.py

@@ -3,7 +3,55 @@ from model import Model
 
 def create_demo(model: Model):
 
+    examples = [
+        ["__assets__/canny_videos_edge/butterfly.mp4", "white butterfly, a high-quality, detailed, and professional photo"],
+        ["__assets__/canny_videos_edge/deer.mp4", "oil painting of a deer, a high-quality, detailed, and professional photo"],
+        ["__assets__/canny_videos_edge/fox.mp4", "wild red fox is walking on the grass, a high-quality, detailed, and professional photo"],
+        ["__assets__/canny_videos_edge/girl_dancing.mp4", "oil painting of a girl dancing close-up, masterpiece, a high-quality, detailed, and professional photo"],
+        ["__assets__/canny_videos_edge/girl_turning.mp4", "oil painting of a beautiful girl, a high-quality, detailed, and professional photo"],
+        ["__assets__/canny_videos_edge/halloween.mp4", "beautiful girl halloween style, a high-quality, detailed, and professional photo"],
+        ["__assets__/canny_videos_edge/santa.mp4", "a santa claus, a high-quality, detailed, and professional photo"],
+    ]
+
     with gr.Blocks() as demo:
         with gr.Row():
-            gr.Markdown('## Text and Canny-Edge Conditional Video Generation (coming soon)')
+            gr.Markdown('## Text and Canny-Edge Conditional Video Generation')
+        with gr.Row():
+            gr.HTML(
+                """
+                <div style="text-align: left; auto;">
+                <h2 style="font-weight: 450; font-size: 1rem; margin: 0rem">
+                    Description: TBD
+                </h3>
+                </div>
+                """)
+
+        with gr.Row():
+            gr.Markdown('### You can either use one of the below shown examples, or upload your own video from which edge-motions will be extracted. But Take into account that for now If your uploaded video has more than 8 frames, then we will uniformly select them and our method will run only on them.')
+
+        with gr.Row():
+            with gr.Column():
+                input_video = gr.Video(label="Input Video",source='upload', format="mp4", visible=True).style(height="auto")
+            with gr.Column():
+                prompt = gr.Textbox(label='Prompt')
+                run_button = gr.Button(label='Run')
+            with gr.Column():
+                result = gr.Video(label="Generated Video").style(height="auto")
+
+        inputs = [
+            input_video,
+            prompt, 
+        ]
+
+        gr.Examples(examples=examples,
+                    inputs=inputs,
+                    outputs=result,
+                    fn=model.process_controlnet_canny,
+                    # cache_examples=os.getenv('SYSTEM') == 'spaces',
+                    run_on_click=False,
+                    )
+
+        run_button.click(fn=model.process_controlnet_canny,
+                         inputs=inputs,
+                         outputs=result,)
     return demo

app_canny_db.py

@@ -2,10 +2,91 @@ import gradio as gr
 from model import Model
 import gradio_utils
 
+examples = [
+    ['Anime DB', "woman1", "Portrait of detailed 1girl, feminine, soldier cinematic shot on canon 5d ultra realistic skin intricate clothes accurate hands Rory Lewis Artgerm WLOP Jeremy Lipking Jane Ansell studio lighting"],
+    ['Arcane DB', "woman1", "Oil painting of a beautiful girl arcane style, masterpiece, a high-quality, detailed, and professional photo"],
+    ['GTA-5 DB', "man1", "gtav style"],
+    ['GTA-5 DB', "woman3", "gtav style"],
+    ['Avatar DB', "woman2", "oil painting of a beautiful girl avatar style"],
+]
+
+
+def load_db_model(evt: gr.SelectData):
+    db_name = gradio_utils.get_db_name_from_id(evt.index)
+    return db_name
+
+
+def canny_select(evt: gr.SelectData):
+    canny_name = gradio_utils.get_canny_name_from_id(evt.index)
+    return canny_name
+
 
 def create_demo(model: Model):
 
     with gr.Blocks() as demo:
         with gr.Row():
-            gr.Markdown('## Text, Canny-Edge and DreamBooth Conditional Video Generation (coming soon)')
+            gr.Markdown('## Text, Canny-Edge and DreamBooth Conditional Video Generation')
+        with gr.Row():
+            gr.HTML(
+                """
+                <div style="text-align: left; auto;">
+                <h2 style="font-weight: 450; font-size: 1rem; margin: 0rem">
+                    Description: TBD
+                </h3>
+                </div>
+                """)
+
+        with gr.Row():
+            gr.Markdown('### You must choose one DB model and one "motion edges" shown below, or use the examples')
+
+        with gr.Row():
+            with gr.Column():
+                gr.Markdown("## Selection")
+                db_text_field = gr.Markdown('DB Model: **Anime DB** ')
+                canny_text_field = gr.Markdown('Motion: **woman1**')
+                prompt = gr.Textbox(label='Prompt')
+                run_button = gr.Button(label='Run')
+            with gr.Column():
+                result = gr.Image(label="Generated Video").style(height=400)
+
+        with gr.Row():
+            gallery_db = gr.Gallery(label="Db models", value=[('__assets__/db_files/anime.jpg', "anime"), ('__assets__/db_files/arcane.jpg', "Arcane"), ('__assets__/db_files/gta.jpg', "GTA-5 (Man)"), ('__assets__/db_files/avatar.jpg', "Avatar DB")]).style(grid=[4], height=50)
+        with gr.Row():
+            gallery_canny = gr.Gallery(label="Motions", value=[('__assets__/db_files/woman1.gif', "woman1"), ('__assets__/db_files/woman2.gif', "woman2"), ('__assets__/db_files/man1.gif', "man1"), ('__assets__/db_files/woman3.gif', "woman3")]).style(grid=[4], height=50)
+            predefined_motion = gr.Textbox(visible=False, label='One of the above defined motions')
+
+        db_selection = gr.Textbox(label="DB Model", visible=False)
+        canny_selection = gr.Textbox(label="One of the above defined motions", visible=False)
+
+        gallery_db.select(load_db_model, None, db_selection)
+        gallery_canny.select(canny_select, None, canny_selection)
+
+        db_selection.change(on_db_selection_update,None,db_text_field)
+        canny_selection.change(on_canny_selection_update,None,canny_text_field)
+
+        inputs = [
+            db_selection,
+            canny_selection,
+            prompt,
+        ]
+
+        gr.Examples(examples=examples,
+                    inputs=inputs,
+                    outputs=result,
+                    fn=model.process_controlnet_canny_db,
+                    # cache_examples=os.getenv('SYSTEM') == 'spaces',
+                    )
+
+        run_button.click(fn=model.process_controlnet_canny_db,
+                         inputs=inputs,
+                         outputs=result,)
     return demo
+
+
+def on_db_selection_update(evt : gr.EventData):
+
+    return f"DB model: **{evt._data}**"
+
+
+def on_canny_selection_update(evt: gr.EventData):
+    return f"Motion: **{evt._data}**"

app_pose.py

@@ -3,9 +3,65 @@ import os
 
 from model import Model
 
+examples = [
+    ['Motion 1', "A Robot is dancing in Sahara desert"],
+    ['Motion 2', "A Robot is dancing in Sahara desert"],
+    ['Motion 3', "A Robot is dancing in Sahara desert"],
+    ['Motion 4', "A Robot is dancing in Sahara desert"],
+    ['Motion 5', "A Robot is dancing in Sahara desert"],
+]
+
 def create_demo(model: Model):
     with gr.Blocks() as demo:
         with gr.Row():
-            gr.Markdown('## Text and Pose Conditional Video Generation (coming soon)')
+            gr.Markdown('## Text and Pose Conditional Video Generation')
+        with gr.Row():
+            gr.HTML(
+                """
+                <div style="text-align: left; auto;">
+                <h2 style="font-weight: 450; font-size: 1rem; margin: 0rem">
+                    Description: TBD
+                </h3>
+                </div>
+                """)
+
+        with gr.Row():
+            gr.Markdown('### You must select one pose sequence shown below, or use the examples')
+            with gr.Column():
+                gallery_pose_sequence = gr.Gallery(label="Pose Sequence", value=[('__assets__/poses_skeleton_gifs/dance1.gif', "Motion 1"), ('__assets__/poses_skeleton_gifs/dance2.gif', "Motion 2"), ('__assets__/poses_skeleton_gifs/dance3.gif', "Motion 3"), ('__assets__/poses_skeleton_gifs/dance4.gif', "Motion 4"), ('__assets__/poses_skeleton_gifs/dance5.gif', "Motion 5")]).style(grid=[2], height="auto")
+                input_video_path = gr.Textbox(label="Pose Sequence",visible=False,value="Motion 1")
+                gr.Markdown("## Selection")
+                pose_sequence_selector = gr.Markdown('Pose Sequence: **Motion 1**')
+            with gr.Column():
+                prompt = gr.Textbox(label='Prompt')
+                run_button = gr.Button(label='Run')
+            with gr.Column():
+                result = gr.Image(label="Generated Video")
+
+        input_video_path.change(on_video_path_update, None, pose_sequence_selector)
+        gallery_pose_sequence.select(pose_gallery_callback, None, input_video_path)
+        inputs = [
+            input_video_path,
+            prompt,
+        ]
+
+        gr.Examples(examples=examples,
+                    inputs=inputs,
+                    outputs=result,
+                    # cache_examples=os.getenv('SYSTEM') == 'spaces',
+                    fn=model.process_controlnet_pose,
+                    run_on_click=False,
+                    )
+
+        run_button.click(fn=model.process_controlnet_pose,
+                         inputs=inputs,
+                         outputs=result,)
 
     return demo
+
+
+def on_video_path_update(evt: gr.EventData):
+    return f'Pose Sequence: **{evt._data}**'
+
+def pose_gallery_callback(evt: gr.SelectData):
+    return f"Motion {evt.index+1}"