readme

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text

.idea/.gitignore

@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

.idea/CSGO.iml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,17 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="DuplicatedCode" enabled="true" level="WEAK WARNING" enabled_by_default="true">
+      <Languages>
+        <language minSize="1197" name="Python" />
+      </Languages>
+    </inspection_tool>
+    <inspection_tool class="PyPep8NamingInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
+      <option name="ignoredErrors">
+        <list>
+          <option value="N801" />
+        </list>
+      </option>
+    </inspection_tool>
+  </profile>
+</component>

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Black">
+    <option name="sdkName" value="Python 3.9" />
+  </component>
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.9" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/CSGO.iml" filepath="$PROJECT_DIR$/.idea/CSGO.iml" />
+    </modules>
+  </component>
+</project>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="" vcs="Git" />
+  </component>
+</project>

README.md

@@ -1,3 +1,257 @@
 ---
 license: apache-2.0
 ---
+<div align="center">
+
+[//]: # (<h1>CSGO: Content-Style Composition in Text-to-Image Generation</h1>)
+
+[//]: # ()
+[//]: # ([**Peng Xing**]&#40;https://github.com/xingp-ng&#41;<sup>12*</sup> · [**Haofan Wang**]&#40;https://haofanwang.github.io/&#41;<sup>1*</sup> · [**Yanpeng Sun**]&#40;https://scholar.google.com.hk/citations?user=a3FI8c4AAAAJ&hl=zh-CN&oi=ao/&#41;<sup>2</sup> · [**Qixun Wang**]&#40;https://github.com/wangqixun&#41;<sup>1</sup> · [**Xu Bai**]&#40;https://huggingface.co/baymin0220&#41;<sup>1</sup> · [**Hao Ai**]&#40;https://github.com/aihao2000&#41;<sup>13</sup> · [**Renyuan Huang**]&#40;https://github.com/DannHuang&#41;<sup>14</sup> · [**Zechao Li**]&#40;https://zechao-li.github.io/&#41;<sup>2✉</sup>)
+
+[//]: # ()
+[//]: # (<sup>1</sup>InstantX Team · <sup>2</sup>Nanjing University of Science and Technology · <sup>3</sup>Beihang University · <sup>4</sup>Peking University)
+
+<sup>*</sup>equal contributions, <sup>✉</sup>corresponding authors
+
+<a href='https://csgo-gen.github.io/'><img src='https://img.shields.io/badge/Project-Page-green'></a>
+<a href='https://arxiv.org/abs/2404.02733'><img src='https://img.shields.io/badge/Technique-Report-red'></a>
+[![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Space-red)](https://huggingface.co/spaces/InstantX/InstantStyle)
+[![ModelScope](https://img.shields.io/badge/ModelScope-Studios-blue)](https://modelscope.cn/studios/instantx/InstantStyle/summary)
+[![GitHub](https://img.shields.io/github/stars/instantX-research/CSGO?style=social)](https://github.com/instantX-research/CSGO)
+</div>
+
+
+[//]: # (##  Updates 🔥)
+
+[//]: # ()
+[//]: # ([//]: # &#40;- **`2024/07/19`**: ✨ We support 🎞️ portrait video editing &#40;aka v2v&#41;! More to see [here]&#40;assets/docs/changelog/2024-07-19.md&#41;.&#41;)
+[//]: # ()
+[//]: # ([//]: # &#40;- **`2024/07/17`**: 🍎 We support macOS with Apple Silicon, modified from [jeethu]&#40;https://github.com/jeethu&#41;'s PR [#143]&#40;https://github.com/KwaiVGI/LivePortrait/pull/143&#41;.&#41;)
+[//]: # ()
+[//]: # ([//]: # &#40;- **`2024/07/10`**: 💪 We support audio and video concatenating, driving video auto-cropping, and template making to protect privacy. More to see [here]&#40;assets/docs/changelog/2024-07-10.md&#41;.&#41;)
+[//]: # ()
+[//]: # ([//]: # &#40;- **`2024/07/09`**: 🤗 We released the [HuggingFace Space]&#40;https://huggingface.co/spaces/KwaiVGI/liveportrait&#41;, thanks to the HF team and [Gradio]&#40;https://github.com/gradio-app/gradio&#41;!&#41;)
+[//]: # ([//]: # &#40;Continuous updates, stay tuned!&#41;)
+[//]: # (- **`2024/08/30`**: 😊 We released the initial version of the inference code.)
+
+[//]: # (- **`2024/08/30`**: 😊 We released the technical report on [arXiv]&#40;https://arxiv.org/pdf/2408.16766&#41;)
+
+[//]: # (- **`2024/07/15`**: 🔥 We released the [homepage]&#40;https://csgo-gen.github.io&#41;.)
+
+[//]: # ()
+[//]: # (##   Plan 💪)
+
+[//]: # (- [x]  technical report)
+
+[//]: # (- [x]  inference code)
+
+[//]: # (- [ ]  pre-trained weight)
+
+[//]: # (- [ ]  IMAGStyle dataset)
+
+[//]: # (- [ ]  training code)
+
+## Introduction 📖
+This repo, named **CSGO**, contains the official PyTorch implementation of our paper [CSGO: Content-Style Composition in Text-to-Image Generation](https://arxiv.org/pdf/).
+We are actively updating and improving this repository. If you find any bugs or have suggestions, welcome to raise issues or submit pull requests (PR) 💖.
+
+## Pipeline 	💻
+<p align="center">
+  <img src="assets/image3_1.jpg">
+</p>
+
+## Capabilities 🚅 
+
+  🔥 Our CSGO achieves **image-driven style transfer, text-driven stylized synthesis, and text editing-driven stylized synthesis**.
+
+  🔥 For more results, visit our <a href="https://csgo-gen.github.io"><strong>homepage</strong></a> 🔥
+
+<p align="center">
+  <img src="assets/vis.jpg">
+</p>
+
+
+## Getting Started 🏁
+### 1. Clone the code and prepare the environment
+```bash
+git clone https://github.com/instantX-research/CSGO
+cd CSGO
+
+# create env using conda
+conda create -n CSGO python=3.9
+conda activate CSGO
+
+# install dependencies with pip
+# for Linux and Windows users
+pip install -r requirements.txt
+```
+
+### 2. Download pretrained weights(coming soon)
+
+The easiest way to download the pretrained weights is from HuggingFace:
+```bash
+# first, ensure git-lfs is installed, see: https://docs.github.com/en/repositories/working-with-files/managing-large-files/installing-git-large-file-storage
+git lfs install
+# clone and move the weights
+git clone https://huggingface.co/InstanX/CSGO CSGO
+```
+Our method is fully compatible with [SDXL](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0), [VAE](https://huggingface.co/madebyollin/sdxl-vae-fp16-fix), [ControlNet](https://huggingface.co/TTPlanet/TTPLanet_SDXL_Controlnet_Tile_Realistic), and [Image Encoder](https://huggingface.co/h94/IP-Adapter/tree/main/sdxl_models/image_encoder).
+Please download them and place them in the ./base_models folder.
+### 3. Inference 🚀
+
+```python
+import torch
+from ip_adapter.utils import BLOCKS as BLOCKS
+from ip_adapter.utils import controlnet_BLOCKS as controlnet_BLOCKS
+from PIL import Image
+from diffusers import (
+    AutoencoderKL,
+    ControlNetModel,
+    StableDiffusionXLControlNetPipeline,
+
+)
+from ip_adapter import CSGO
+
+
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+base_model_path =  "./base_models/stable-diffusion-xl-base-1.0"  
+image_encoder_path = "./base_models/IP-Adapter/sdxl_models/image_encoder"
+csgo_ckpt = "./CSGO/csgo.bin"
+pretrained_vae_name_or_path ='./base_models/sdxl-vae-fp16-fix'
+controlnet_path = "./base_models/TTPLanet_SDXL_Controlnet_Tile_Realistic"
+weight_dtype = torch.float16
+
+
+vae = AutoencoderKL.from_pretrained(pretrained_vae_name_or_path,torch_dtype=torch.float16)
+controlnet = ControlNetModel.from_pretrained(controlnet_path, torch_dtype=torch.float16,use_safetensors=True)
+pipe = StableDiffusionXLControlNetPipeline.from_pretrained(
+    base_model_path,
+    controlnet=controlnet,
+    torch_dtype=torch.float16,
+    add_watermarker=False,
+    vae=vae
+)
+pipe.enable_vae_tiling()
+
+
+target_content_blocks = BLOCKS['content']
+target_style_blocks = BLOCKS['style']
+controlnet_target_content_blocks = controlnet_BLOCKS['content']
+controlnet_target_style_blocks = controlnet_BLOCKS['style']
+
+csgo = CSGO(pipe, image_encoder_path, csgo_ckpt, device, num_content_tokens=4,num_style_tokens=32,
+                          target_content_blocks=target_content_blocks, target_style_blocks=target_style_blocks,controlnet=False,controlnet_adapter=True,
+                              controlnet_target_content_blocks=controlnet_target_content_blocks, 
+                              controlnet_target_style_blocks=controlnet_target_style_blocks,
+                              content_model_resampler=True,
+                              style_model_resampler=True,
+                              load_controlnet=False,
+
+                              )
+
+style_name = 'img_0.png'
+content_name = 'img_0.png'
+style_image = "../assets/{}".format(style_name)
+content_image = Image.open('../assets/{}'.format(content_name)).convert('RGB')
+
+caption ='a small house with a sheep statue on top of it'
+
+num_sample=4
+
+#image-driven style transfer
+images = csgo.generate(pil_content_image= content_image, pil_style_image=style_image,
+                           prompt=caption,
+                           negative_prompt= "text, watermark, lowres, low quality, worst quality, deformed, glitch, low contrast, noisy, saturation, blurry",
+                           content_scale=1.0,
+                           style_scale=1.0,
+                           guidance_scale=10,
+                           num_images_per_prompt=num_sample,
+                           num_samples=1,
+                           num_inference_steps=50,
+                           seed=42,
+                           image=content_image.convert('RGB'),
+                           controlnet_conditioning_scale=0.6,
+                          )
+
+#text-driven stylized synthesis
+caption='a cat'
+images = csgo.generate(pil_content_image= content_image, pil_style_image=style_image,
+                           prompt=caption,
+                           negative_prompt= "text, watermark, lowres, low quality, worst quality, deformed, glitch, low contrast, noisy, saturation, blurry",
+                           content_scale=1.0,
+                           style_scale=1.0,
+                           guidance_scale=10,
+                           num_images_per_prompt=num_sample,
+                           num_samples=1,
+                           num_inference_steps=50,
+                           seed=42,
+                           image=content_image.convert('RGB'),
+                           controlnet_conditioning_scale=0.01,
+                          )
+
+#text editing-driven stylized synthesis
+caption='a small house'
+images = csgo.generate(pil_content_image= content_image, pil_style_image=style_image,
+                           prompt=caption,
+                           negative_prompt= "text, watermark, lowres, low quality, worst quality, deformed, glitch, low contrast, noisy, saturation, blurry",
+                           content_scale=1.0,
+                           style_scale=1.0,
+                           guidance_scale=10,
+                           num_images_per_prompt=num_sample,
+                           num_samples=1,
+                           num_inference_steps=50,
+                           seed=42,
+                           image=content_image.convert('RGB'),
+                           controlnet_conditioning_scale=0.4,
+                          )
+```
+
+## Demos
+<p align="center">
+  <br>
+  🔥 For more results, visit our <a href="https://csgo-gen.github.io"><strong>homepage</strong></a> 🔥
+</p>
+
+### Content-Style Composition
+<p align="center">
+  <img src="assets/page1.png">
+</p>
+
+<p align="center">
+  <img src="assets/page4.png">
+</p>
+
+### Cycle Translation
+<p align="center">
+  <img src="assets/page8.png">
+</p>
+
+### Text-Driven Style Synthesis
+<p align="center">
+  <img src="assets/page10.png">
+</p>
+
+### Text Editing-Driven Style Synthesis
+<p align="center">
+  <img src="assets/page11.jpg">
+</p>
+
+## Star History
+[![Star History Chart](https://api.star-history.com/svg?repos=instantX-research/CSGO&type=Date)](https://star-history.com/#instantX-research/CSGO&Date)
+
+
+
+## Acknowledgements
+This project is developed by InstantX Team, all copyright reserved.
+
+## Citation 💖
+If you find CSGO useful for your research, welcome to 🌟 this repo and cite our work using the following BibTeX:
+```bibtex
+@article{xing2024csgo,
+       title={CSGO: Content-Style Composition in Text-to-Image Generation}, 
+       author={Peng Xing and Haofan Wang and Yanpeng Sun and Qixun Wang and Xu Bai and Hao Ai and Renyuan Huang and Zechao Li},
+       year={2024},
+       journal = {arXiv 2408.16766},
+}
+```