End of training

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+images_0.png filter=lfs diff=lfs merge=lfs -text
+images_1.png filter=lfs diff=lfs merge=lfs -text
+images_2.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,80 @@
+
+---
+license: creativeml-openrail-m
+base_model: runwayml/stable-diffusion-v1-5
+tags:
+- stable-diffusion
+- stable-diffusion-diffusers
+- image-to-image
+- diffusers
+- controlnet
+- controllora
+---
+    
+# ControlLoRA - HighCWu/sd-controllora-face-landmarks
+
+ControlLoRA is a neural network structure extended from Controlnet to control diffusion models by adding extra conditions. This checkpoint corresponds to the ControlLoRA conditioned on Unknown Input.
+
+ControlLoRA uses the same structure as Controlnet. But its core weight comes from UNet, unmodified. Only hint image encoding layers, linear lora layers and conv2d lora layers used in weight offset are trained.
+
+The main idea is from my [ControlLoRA](https://github.com/HighCWu/ControlLoRA) and sdxl [control-lora](https://huggingface.co/stabilityai/control-lora).
+
+## Example
+
+1. Clone ControlLoRA from [Github](https://github.com/HighCWu/control-lora-v2):
+```sh
+$ git clone https://github.com/HighCWu/control-lora-v2
+```
+
+2. Enter the repo dir:
+```sh
+$ cd control-lora-v2
+```
+
+3. Run code:
+```py
+from PIL import Image
+from diffusers import StableDiffusionControlNetPipeline, UNet2DConditionModel, UniPCMultistepScheduler
+import torch
+from PIL import Image
+from diffusers.utils import load_image
+from models.controllora import ControlLoRAModel
+
+image = Image.open('<Your Conditioning Image Path>')
+
+unet = UNet2DConditionModel.from_pretrained(
+    "runwayml/stable-diffusion-v1-5", subfolder="unet"
+)
+controllora = ControlLoRA.from_pretrained(
+    "HighCWu/HighCWu/sd-controllora-face-landmarks", torch_dtype=torch.float16
+)
+controllora.tie_weights(unet)
+
+pipe = StableDiffusionControlNetPipeline.from_pretrained(
+    "runwayml/stable-diffusion-v1-5", unet=unet, controlnet=controlnet, safety_checker=None, torch_dtype=torch.float16
+)
+
+pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
+
+# Remove if you do not have xformers installed
+# see https://huggingface.co/docs/diffusers/v0.13.0/en/optimization/xformers#installing-xformers
+# for installation instructions
+pipe.enable_xformers_memory_efficient_attention()
+
+pipe.enable_model_cpu_offload()
+
+image = pipe("Girl smiling, professional dslr photograph, high quality", image, num_inference_steps=20).images[0]
+
+image.show()
+```
+
+Here are some validation images:
+
+You can find some example images below.
+prompt: High-quality close-up dslr photo of man wearing a hat with trees in the background
+![images_0)](./images_0.png)
+prompt: Girl smiling, professional dslr photograph, dark background, studio lights, high quality
+![images_1)](./images_1.png)
+prompt: Portrait of a clown face, oil on canvas, bittersweet expression
+![images_2)](./images_2.png)
+

config.json

@@ -0,0 +1,53 @@
+{
+  "_class_name": "ControlLoRAModel",
+  "_diffusers_version": "0.21.0.dev0",
+  "_name_or_path": "output/sd-controllora-face-landmarks\\checkpoint-75000",
+  "act_fn": "silu",
+  "addition_embed_type": null,
+  "addition_embed_type_num_heads": 64,
+  "addition_time_embed_dim": null,
+  "attention_head_dim": 8,
+  "block_out_channels": [
+    320,
+    640,
+    1280,
+    1280
+  ],
+  "class_embed_type": null,
+  "conditioning_channels": 3,
+  "conditioning_embedding_out_channels": [
+    16,
+    32,
+    96,
+    256
+  ],
+  "controlnet_conditioning_channel_order": "rgb",
+  "cross_attention_dim": 768,
+  "down_block_types": [
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D",
+    "DownBlock2D"
+  ],
+  "downsample_padding": 1,
+  "encoder_hid_dim": null,
+  "encoder_hid_dim_type": null,
+  "flip_sin_to_cos": true,
+  "freq_shift": 0,
+  "global_pool_conditions": false,
+  "in_channels": 4,
+  "layers_per_block": 2,
+  "lora_conv2d_rank": 32,
+  "lora_linear_rank": 32,
+  "mid_block_scale_factor": 1,
+  "norm_eps": 1e-05,
+  "norm_num_groups": 32,
+  "num_attention_heads": null,
+  "num_class_embeds": null,
+  "only_cross_attention": false,
+  "projection_class_embeddings_input_dim": null,
+  "resnet_time_scale_shift": "default",
+  "transformer_layers_per_block": 1,
+  "upcast_attention": false,
+  "use_linear_projection": false
+}

diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67d584ae2cba78dc9eca4a5da7f00e801d004f1625bacf3408d47aa54db8bcc5
+size 104845256