updaste

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 *.wav filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text

app.py

@@ -1,12 +1,13 @@
 import gradio as gr
 import imagebind
+import soundfile as sf
 import torch
 from diffusers import StableUnCLIPImg2ImgPipeline
-import soundfile as sf
+from PIL import Image
 
 device = "cuda:0" if torch.cuda.is_available() else "cpu"
 pipe = StableUnCLIPImg2ImgPipeline.from_pretrained(
-    "stabilityai/stable-diffusion-2-1-unclip", torch_dtype=torch.float16, variation="fp16"
+    "stabilityai/stable-diffusion-2-1-unclip",
 )
 pipe = pipe.to(device)
 
@@ -15,18 +16,34 @@ model.eval()
 model.to(device)
 
 @torch.no_grad()
-def anything2img(prompt, audio):
-    sr, waveform = audio
-    audio_path = 'tmp.wav'
-    sf.write(audio_path, waveform, sr)
-    audio_paths=[audio_path]
-    embeddings = model.forward({
-        imagebind.ModalityType.AUDIO: imagebind.load_and_transform_audio_data(audio_paths, device),
-    })
-    embeddings = embeddings[imagebind.ModalityType.AUDIO]
-    images = pipe(prompt=prompt, image_embeds=embeddings.half()).images
+def anything2img(prompt, audio, image):
+    if audio is not None:
+        sr, waveform = audio
+        sf.write('tmp.wav', waveform, sr)
+        embeddings = model.forward({
+            imagebind.ModalityType.AUDIO: imagebind.load_and_transform_audio_data(['tmp.wav'], device),
+        })
+        audio_embeddings = embeddings[imagebind.ModalityType.AUDIO]
+    if image is not None:
+        Image.fromarray(image).save('tmp.png')
+        embeddings = model.forward({
+            imagebind.ModalityType.VISION: imagebind.load_and_transform_vision_data(['tmp.png'], device),
+        })
+        image_embeddings = embeddings[imagebind.ModalityType.VISION]
+
+    if audio_embeddings is not None and image_embeddings is not None:
+        embeddings = audio_embeddings + image_embeddings
+    elif image_embeddings is not None:
+        embeddings = image_embeddings
+    elif audio_embeddings is not None:
+        embeddings = audio_embeddings
+    else:
+        embeddings = None
+    
+    images = pipe(prompt=prompt, image_embeds=embeddings).images
     return images[0]
     
 
-demo = gr.Interface(fn=anything2img, inputs=["text", "audio"], outputs="image")
-demo.launch()
+demo = gr.Interface(fn=anything2img, inputs=["text", "audio", "image"], outputs="image")
+# demo.launch(server_name='0.0.0.0', server_port=10051, share=True)
+demo.launch(server_name='0.0.0.0', server_port=10047, share=True)

assets/wav/cat.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa3d926fa712eb851c4e27d81dff0804cafc72e4f1bd716800478b2a4b40a02d
+size 640044

assets/wav/fire_engine.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29165227485a6d457146dbee150610cc92a099af01c4a228172b328bcdcef3f0
+size 882078

assets/wav/motorcycle.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45cd2e6a875984813afe9097be6dae8c911e7616698ae50cdd3cdb1ab6900b17
+size 457918

assets/wav/plane.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e816bca22a06dc50e75d1cab715ca99ab3334a4097e5635c955eaee0c0b2ccf
+size 1058478

assets/wav/train.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54eaa334a44b11f8d8888cdbf7cfc0340b70a87fb57c97c09adb454f26de7f60
+size 483959

assets/wav/wave.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ce431cb79b82390adf8640c0f68536aae71cdf4b48da412516c4fee0428831a
+size 1411278