End of training

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+images_0.png filter=lfs diff=lfs merge=lfs -text
+images_2.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,83 @@
+
+---
+license: creativeml-openrail-m
+base_model: runwayml/stable-diffusion-v1-5
+tags:
+- stable-diffusion
+- stable-diffusion-diffusers
+- image-to-image
+- diffusers
+- controlnet
+- control-lora
+---
+    
+# ControlLoRA - Head3d Version
+
+ControlLoRA is a neural network structure extended from Controlnet to control diffusion models by adding extra conditions. This checkpoint corresponds to the ControlLoRA conditioned on Head3d.
+
+ControlLoRA uses the same structure as Controlnet. But its core weight comes from UNet, unmodified. Only hint image encoding layers, linear lora layers and conv2d lora layers used in weight offset are trained.
+
+The main idea is from my [ControlLoRA](https://github.com/HighCWu/ControlLoRA) and sdxl [control-lora](https://huggingface.co/stabilityai/control-lora).
+
+## Example
+
+1. Clone ControlLoRA from [Github](https://github.com/HighCWu/control-lora-v2):
+```sh
+$ git clone https://github.com/HighCWu/control-lora-v2
+```
+
+2. Enter the repo dir:
+```sh
+$ cd control-lora-v2
+```
+
+3. Run code:
+```py
+import torch
+from PIL import Image
+from diffusers import StableDiffusionControlNetPipeline, UNet2DConditionModel, UniPCMultistepScheduler
+from models.control_lora import ControlLoRAModel
+
+device = 'cuda' if torch.cuda.is_available() else 'cpu'
+dtype = torch.float16 if torch.cuda.is_available() else torch.float32
+
+image = Image.open('<Your Conditioning Image Path>')
+
+base_model = "runwayml/stable-diffusion-v1-5"
+
+unet = UNet2DConditionModel.from_pretrained(
+    base_model, subfolder="unet", torch_dtype=dtype
+)
+control_lora: ControlLoRAModel = ControlLoRAModel.from_pretrained(
+    "HighCWu/sd-latent-control-dora-rank128-head3d", torch_dtype=dtype
+)
+control_lora.tie_weights(unet)
+
+pipe = StableDiffusionControlNetPipeline.from_pretrained(
+    base_model, unet=unet, controlnet=control_lora, safety_checker=None, torch_dtype=dtype
+).to(device)
+control_lora.bind_vae(pipe.vae)
+
+pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
+
+# Remove if you do not have xformers installed
+# see https://huggingface.co/docs/diffusers/v0.13.0/en/optimization/xformers#installing-xformers
+# for installation instructions
+pipe.enable_xformers_memory_efficient_attention()
+
+# pipe.enable_model_cpu_offload()
+
+image = pipe("Girl smiling, professional dslr photograph, high quality", image, num_inference_steps=20).images[0]
+
+image.show()
+```
+
+You can find some example images below.
+
+prompt: a photography of a man with a beard and sunglasses on
+![images_0)](./images_0.png)
+prompt: worst quality , low quality , portrait , close - up , inconsistent head shape
+![images_1)](./images_1.png)
+prompt: a photography of a man with a mustache and a suit jacket
+![images_2)](./images_2.png)
+

config.json

@@ -0,0 +1,57 @@
+{
+  "_class_name": "ControlLoRAModel",
+  "_diffusers_version": "0.26.3",
+  "_name_or_path": "output/sd-latent-control-dora-rank128-head3d\\checkpoint-75000",
+  "act_fn": "silu",
+  "addition_embed_type": null,
+  "addition_embed_type_num_heads": 64,
+  "addition_time_embed_dim": null,
+  "attention_head_dim": 8,
+  "block_out_channels": [
+    320,
+    640,
+    1280,
+    1280
+  ],
+  "class_embed_type": null,
+  "conditioning_channels": 4,
+  "conditioning_embedding_out_channels": [
+    16,
+    32,
+    96,
+    256
+  ],
+  "controlnet_conditioning_channel_order": "rgb",
+  "cross_attention_dim": 768,
+  "down_block_types": [
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D",
+    "DownBlock2D"
+  ],
+  "downsample_padding": 1,
+  "encoder_hid_dim": null,
+  "encoder_hid_dim_type": null,
+  "flip_sin_to_cos": true,
+  "freq_shift": 0,
+  "global_pool_conditions": false,
+  "in_channels": 4,
+  "layers_per_block": 2,
+  "lora_conv2d_rank": 128,
+  "lora_linear_rank": 128,
+  "mid_block_scale_factor": 1,
+  "mid_block_type": "UNetMidBlock2DCrossAttn",
+  "norm_eps": 1e-05,
+  "norm_num_groups": 32,
+  "num_attention_heads": null,
+  "num_class_embeds": null,
+  "only_cross_attention": false,
+  "projection_class_embeddings_input_dim": null,
+  "resnet_time_scale_shift": "default",
+  "transformer_layers_per_block": 1,
+  "upcast_attention": false,
+  "use_conditioning_latent": true,
+  "use_dora": true,
+  "use_linear_projection": false,
+  "use_same_level_conditioning_latent": false
+}

diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d02974f69df2ec4ead727979f0cd6bf4bfa31abb7f0502fb71589a13b96807d
+size 271963378