jon-pascal's picture
Update app.py
78a19b7 verified
raw
history blame
25.6 kB
import os
import gradio as gr
import argparse
import numpy as np
import torch
import einops
import copy
import math
import time
import random
import spaces
import re
import uuid
from gradio_imageslider import ImageSlider
from PIL import Image
from SUPIR.util import HWC3, upscale_image, fix_resize, convert_dtype, create_SUPIR_model, load_QF_ckpt
from huggingface_hub import hf_hub_download
from pillow_heif import register_heif_opener
register_heif_opener()
max_64_bit_int = np.iinfo(np.int32).max
hf_hub_download(repo_id="laion/CLIP-ViT-bigG-14-laion2B-39B-b160k", filename="open_clip_pytorch_model.bin", local_dir="laion_CLIP-ViT-bigG-14-laion2B-39B-b160k")
hf_hub_download(repo_id="camenduru/SUPIR", filename="sd_xl_base_1.0_0.9vae.safetensors", local_dir="yushan777_SUPIR")
hf_hub_download(repo_id="camenduru/SUPIR", filename="SUPIR-v0F.ckpt", local_dir="yushan777_SUPIR")
hf_hub_download(repo_id="camenduru/SUPIR", filename="SUPIR-v0Q.ckpt", local_dir="yushan777_SUPIR")
hf_hub_download(repo_id="RunDiffusion/Juggernaut-XL-Lightning", filename="Juggernaut_RunDiffusionPhoto2_Lightning_4Steps.safetensors", local_dir="RunDiffusion_Juggernaut-XL-Lightning")
parser = argparse.ArgumentParser()
parser.add_argument("--opt", type=str, default='options/SUPIR_v0.yaml')
parser.add_argument("--ip", type=str, default='127.0.0.1')
parser.add_argument("--port", type=int, default='6688')
parser.add_argument("--no_llava", action='store_true', default=True)
parser.add_argument("--use_image_slider", action='store_true', default=False)
parser.add_argument("--log_history", action='store_true', default=False)
parser.add_argument("--loading_half_params", action='store_true', default=False)
parser.add_argument("--use_tile_vae", action='store_true', default=True)
parser.add_argument("--encoder_tile_size", type=int, default=512)
parser.add_argument("--decoder_tile_size", type=int, default=64)
parser.add_argument("--load_8bit_llava", action='store_true', default=False)
args = parser.parse_args()
if torch.cuda.device_count() > 0:
SUPIR_device = 'cuda:0'
# Load SUPIR
model, default_setting = create_SUPIR_model(args.opt, SUPIR_sign='Q', load_default_setting=True)
if args.loading_half_params:
model = model.half()
if args.use_tile_vae:
model.init_tile_vae(encoder_tile_size=args.encoder_tile_size, decoder_tile_size=args.decoder_tile_size)
model = model.to(SUPIR_device)
model.first_stage_model.denoise_encoder_s1 = copy.deepcopy(model.first_stage_model.denoise_encoder)
model.current_model = 'v0-Q'
ckpt_Q, ckpt_F = load_QF_ckpt(args.opt)
def check_upload(input_image):
if input_image is None:
raise gr.Error("Please provide an image to restore.")
return gr.update(visible=True)
def process_uploaded_image(image_path):
image = Image.open(image_path)
width, height = image.size
max_dim = max(width, height)
if max_dim > 1024:
if width > height:
new_width = 1024
new_height = int((1024 / width) * height)
else:
new_height = 1024
new_width = int((1024 / height) * width)
image = image.resize((new_width, new_height), Image.Resampling.LANCZOS)
image.save(image_path)
return image_path
def update_seed(is_randomize_seed, seed):
if is_randomize_seed:
return random.randint(0, max_64_bit_int)
return seed
def reset():
return [
None, # input_image
"", # prompt
'Cinematic, High Contrast, highly detailed, taken using a Canon EOS R camera, hyper detailed photo - realistic maximum detail, 32k, Color Grading, ultra HD, extreme meticulous detailing, skin pore detailing, hyper sharpness, perfect without deformations.', # a_prompt
'painting, oil painting, illustration, drawing, art, sketch, anime, cartoon, CG Style, 3D render, unreal engine, blurring, aliasing, unsharp, weird textures, ugly, dirty, messy, worst quality, low quality, frames, watermark, signature, jpeg artifacts, deformed, lowres, over-smooth', # n_prompt
1, # num_samples
1024, # min_size
1, # downscale
2, # upscale
default_setting.edm_steps if torch.cuda.device_count() > 0 else 1, # edm_steps
-1.0, # s_stage1
1.0, # s_stage2
default_setting.s_cfg_Quality if torch.cuda.device_count() > 0 else 1.0, # s_cfg
random.randint(0, max_64_bit_int), # seed
5, # s_churn
1.003, # s_noise
'Wavelet', # color_fix_type
'fp32', # diff_dtype
'fp32', # ae_dtype
1.0, # gamma_correction
True, # linear_CFG
False, # linear_s_stage2
default_setting.spt_linear_CFG_Quality if torch.cuda.device_count() > 0 else 1.0, # spt_linear_CFG
0.0, # spt_linear_s_stage2
'v0-Q', # model_select
4 # allocation
]
def check(input_image):
if input_image is None:
raise gr.Error("Please provide an image to restore.")
def stage2_process(
input_image,
prompt,
a_prompt,
n_prompt,
num_samples,
min_size,
downscale,
upscale,
edm_steps,
s_stage1,
s_stage2,
s_cfg,
seed,
s_churn,
s_noise,
color_fix_type,
diff_dtype,
ae_dtype,
gamma_correction,
linear_CFG,
linear_s_stage2,
spt_linear_CFG,
spt_linear_s_stage2,
model_select,
allocation
):
try:
return restore_in_Xmin(
input_image,
prompt,
a_prompt,
n_prompt,
num_samples,
min_size,
downscale,
upscale,
edm_steps,
s_stage1,
s_stage2,
s_cfg,
seed,
s_churn,
s_noise,
color_fix_type,
diff_dtype,
ae_dtype,
gamma_correction,
linear_CFG,
linear_s_stage2,
spt_linear_CFG,
spt_linear_s_stage2,
model_select,
allocation
)
except Exception as e:
print(f"Exception occurred: {str(e)}")
raise e
def restore_in_Xmin(
input_image_path,
prompt,
a_prompt,
n_prompt,
num_samples,
min_size,
downscale,
upscale,
edm_steps,
s_stage1,
s_stage2,
s_cfg,
seed,
s_churn,
s_noise,
color_fix_type,
diff_dtype,
ae_dtype,
gamma_correction,
linear_CFG,
linear_s_stage2,
spt_linear_CFG,
spt_linear_s_stage2,
model_select,
allocation
):
print("Starting image restoration process...")
input_format = re.sub(r"^.*\.([^\.]+)$", r"\1", input_image_path)
if input_format.lower() not in ['png', 'webp', 'jpg', 'jpeg', 'gif', 'bmp', 'heic']:
gr.Warning('Invalid image format. Please use a supported image format.')
return None, None, None
if prompt is None:
prompt = ""
if a_prompt is None:
a_prompt = ""
if n_prompt is None:
n_prompt = ""
if prompt != "" and a_prompt != "":
a_prompt = prompt + ", " + a_prompt
else:
a_prompt = prompt + a_prompt
print("Final prompt: " + str(a_prompt))
denoise_image = np.array(Image.open(input_image_path))
if downscale > 1:
input_height, input_width, input_channel = denoise_image.shape
denoise_image = np.array(Image.fromarray(denoise_image).resize((input_width // downscale, input_height // downscale), Image.LANCZOS))
denoise_image = HWC3(denoise_image)
if torch.cuda.device_count() == 0:
gr.Warning('Set this space to GPU config to make it work.')
return [input_image_path, denoise_image], gr.update(value="GPU not available."), gr.update(visible=True)
if model_select != model.current_model:
print('Loading model: ' + model_select)
if model_select == 'v0-Q':
model.load_state_dict(ckpt_Q, strict=False)
elif model_select == 'v0-F':
model.load_state_dict(ckpt_F, strict=False)
model.current_model = model_select
model.ae_dtype = convert_dtype(ae_dtype)
model.model.dtype = convert_dtype(diff_dtype)
# Allocation
allocation_functions = {
1: restore_in_1min,
2: restore_in_2min,
3: restore_in_3min,
4: restore_in_4min,
5: restore_in_5min,
6: restore_in_6min,
7: restore_in_7min,
8: restore_in_8min,
9: restore_in_9min,
10: restore_in_10min,
}
restore_function = allocation_functions.get(allocation, restore_in_4min)
return restore_function(
input_image_path, prompt, a_prompt, n_prompt, num_samples, min_size, downscale, upscale,
edm_steps, s_stage1, s_stage2, s_cfg, seed, s_churn, s_noise, color_fix_type,
diff_dtype, ae_dtype, gamma_correction, linear_CFG, linear_s_stage2, spt_linear_CFG,
spt_linear_s_stage2, model_select, allocation
)
@spaces.GPU(duration=59)
def restore_in_1min(*args, **kwargs):
return restore_on_gpu(*args, **kwargs)
@spaces.GPU(duration=119)
def restore_in_2min(*args, **kwargs):
return restore_on_gpu(*args, **kwargs)
@spaces.GPU(duration=179)
def restore_in_3min(*args, **kwargs):
return restore_on_gpu(*args, **kwargs)
@spaces.GPU(duration=239)
def restore_in_4min(*args, **kwargs):
return restore_on_gpu(*args, **kwargs)
@spaces.GPU(duration=299)
def restore_in_5min(*args, **kwargs):
return restore_on_gpu(*args, **kwargs)
@spaces.GPU(duration=359)
def restore_in_6min(*args, **kwargs):
return restore_on_gpu(*args, **kwargs)
@spaces.GPU(duration=419)
def restore_in_7min(*args, **kwargs):
return restore_on_gpu(*args, **kwargs)
@spaces.GPU(duration=479)
def restore_in_8min(*args, **kwargs):
return restore_on_gpu(*args, **kwargs)
@spaces.GPU(duration=539)
def restore_in_9min(*args, **kwargs):
return restore_on_gpu(*args, **kwargs)
@spaces.GPU(duration=599)
def restore_in_10min(*args, **kwargs):
return restore_on_gpu(*args, **kwargs)
def restore_on_gpu(
input_image_path,
prompt,
a_prompt,
n_prompt,
num_samples,
min_size,
downscale,
upscale,
edm_steps,
s_stage1,
s_stage2,
s_cfg,
seed,
s_churn,
s_noise,
color_fix_type,
diff_dtype,
ae_dtype,
gamma_correction,
linear_CFG,
linear_s_stage2,
spt_linear_CFG,
spt_linear_s_stage2,
model_select,
allocation
):
start = time.time()
print('Starting GPU restoration...')
torch.cuda.set_device(SUPIR_device)
with torch.no_grad():
# Convert input image to NumPy array and ensure it has 3 channels
input_image = HWC3(np.array(Image.open(input_image_path)))
input_image = upscale_image(input_image, upscale, unit_resolution=32, min_size=min_size)
LQ = input_image / 255.0
LQ = np.power(LQ, gamma_correction)
LQ *= 255.0
LQ = LQ.round().clip(0, 255).astype(np.uint8)
LQ = LQ / 255 * 2 - 1
LQ = torch.tensor(LQ, dtype=torch.float32).permute(2, 0, 1).unsqueeze(0).to(SUPIR_device)[:, :3, :, :]
captions = ['']
samples = model.batchify_sample(
LQ, captions, num_steps=edm_steps, restoration_scale=s_stage1, s_churn=s_churn,
s_noise=s_noise, cfg_scale=s_cfg, control_scale=s_stage2, seed=seed,
num_samples=num_samples, p_p=a_prompt, n_p=n_prompt, color_fix_type=color_fix_type,
use_linear_CFG=linear_CFG, use_linear_control_scale=linear_s_stage2,
cfg_scale_start=spt_linear_CFG, control_scale_start=spt_linear_s_stage2
)
x_samples = (einops.rearrange(samples, 'b c h w -> b h w c') * 127.5 + 127.5).cpu().numpy().round().clip(
0, 255).astype(np.uint8)
results = [x_samples[i] for i in range(num_samples)]
torch.cuda.empty_cache()
input_height, input_width, input_channel = input_image.shape
result_height, result_width, result_channel = results[0].shape
print('Restoration completed.')
end = time.time()
secondes = int(end - start)
minutes = math.floor(secondes / 60)
secondes = secondes - (minutes * 60)
hours = math.floor(minutes / 60)
minutes = minutes - (hours * 60)
information = ("Start the process again if you want a different result. " if seed is not None else "") + \
"The image has been enhanced successfully."
# Save the result image to a temporary file for downloading
result_image = Image.fromarray(results[0])
result_image_path = f"result_{uuid.uuid4().hex}.png"
result_image.save(result_image_path)
# Update the result slider with the before and after images
return [input_image_path, result_image_path], gr.update(value=information, visible=True), gr.update(visible=True)
def load_and_reset(param_setting):
print('Resetting parameters...')
if torch.cuda.device_count() == 0:
gr.Warning('Set this space to GPU config to make it work.')
return None, None, None, None, None, None, None, None, None, None, None, None, None
edm_steps = default_setting.edm_steps
s_stage2 = 1.0
s_stage1 = -1.0
s_churn = 5
s_noise = 1.003
a_prompt = 'Cinematic, High Contrast, highly detailed, taken using a Canon EOS R camera, hyper detailed photo - ' \
'realistic maximum detail, 32k, Color Grading, ultra HD, extreme meticulous detailing, skin pore ' \
'detailing, hyper sharpness, perfect without deformations.'
n_prompt = 'painting, oil painting, illustration, drawing, art, sketch, anime, cartoon, CG Style, ' \
'3D render, unreal engine, blurring, dirty, messy, worst quality, low quality, frames, watermark, ' \
'signature, jpeg artifacts, deformed, lowres, over-smooth'
color_fix_type = 'Wavelet'
spt_linear_s_stage2 = 0.0
linear_s_stage2 = False
linear_CFG = True
if param_setting == "Quality":
s_cfg = default_setting.s_cfg_Quality
spt_linear_CFG = default_setting.spt_linear_CFG_Quality
model_select = "v0-Q"
elif param_setting == "Fidelity":
s_cfg = default_setting.s_cfg_Fidelity
spt_linear_CFG = default_setting.spt_linear_CFG_Fidelity
model_select = "v0-F"
else:
raise NotImplementedError
gr.Info('The parameters are reset.')
print('Parameters reset completed.')
return edm_steps, s_cfg, s_stage2, s_stage1, s_churn, s_noise, a_prompt, n_prompt, color_fix_type, linear_CFG, \
linear_s_stage2, spt_linear_CFG, spt_linear_s_stage2, model_select
def log_information(result_slider):
print('Logging information...')
if result_slider is not None:
print(result_slider)
title_html = """
<h1><center>Maree's Magical Photo Tool</center></h1>
"""
# Gradio interface
with gr.Blocks() as interface:
if torch.cuda.device_count() == 0:
with gr.Row():
gr.HTML("""
<p style="background-color: red;"><big><big><big><b>⚠️To use this tool, set a GPU with sufficient VRAM.</b></big></big></big></p>
""")
gr.HTML(title_html)
input_image = gr.Image(label="Upload your photo", show_label=True, type="filepath", height=400, elem_id="image-input")
with gr.Group():
prompt = gr.Textbox(
label="Describe your photo",
info="Tell me about your photo so I can make it better.",
value="",
placeholder="Type a description...",
lines=3
)
upscale = gr.Radio(
[["x1", 1], ["x2", 2], ["x3", 3], ["x4", 4]],
label="Upscale factor",
info="Choose how much to enlarge the photo",
value=2,
interactive=True
)
allocation = gr.Radio(
[["1 min", 1], ["2 min", 2], ["3 min", 3], ["4 min", 4], ["5 min", 5]],
label="GPU allocation time (for Jon)",
info="You can ignore this setting",
value=4,
interactive=True
)
gamma_correction = gr.Number(value=1.0, visible=False) # Hidden component with default value 1.0
with gr.Accordion("Advanced options", open=False):
a_prompt = gr.Textbox(
label="Additional image description",
info="Completes the main image description",
value='Cinematic, High Contrast, highly detailed, taken using a Canon EOS R '
'camera, hyper detailed photo - realistic maximum detail, 32k, Color '
'Grading, ultra HD, extreme meticulous detailing, skin pore detailing, '
'hyper sharpness, perfect without deformations.',
lines=3
)
n_prompt = gr.Textbox(
label="Negative image description",
info="Disambiguate by listing what the image does NOT represent",
value='painting, oil painting, illustration, drawing, art, sketch, anime, '
'cartoon, CG Style, 3D render, unreal engine, blurring, aliasing, unsharp, weird textures, ugly, dirty, messy, '
'worst quality, low quality, frames, watermark, signature, jpeg artifacts, '
'deformed, lowres, over-smooth',
lines=3
)
edm_steps = gr.Slider(
label="Steps",
info="Lower=faster, higher=more details",
minimum=1,
maximum=200,
value=default_setting.edm_steps if torch.cuda.device_count() > 0 else 1,
step=1
)
num_samples = gr.Slider(
label="Num Samples",
info="Number of generated results",
minimum=1,
maximum=4 if not args.use_image_slider else 1,
value=1,
step=1
)
min_size = gr.Slider(
label="Minimum size",
info="Minimum height, minimum width of the result",
minimum=32,
maximum=4096,
value=1024,
step=32
)
downscale = gr.Radio(
[["/1", 1], ["/2", 2], ["/3", 3], ["/4", 4]],
label="Pre-downscale factor",
info="Reducing blurred image reduces the process time",
value=1,
interactive=True
)
with gr.Row():
with gr.Column():
model_select = gr.Radio(
[["💃 Quality (v0-Q)", "v0-Q"], ["🎯 Fidelity (v0-F)", "v0-F"]],
label="Model Selection",
info="Pretrained model",
value="v0-Q",
interactive=True
)
with gr.Column():
color_fix_type = gr.Radio(
[["None", "None"], ["AdaIn (improve as a photo)", "AdaIn"], ["Wavelet (for JPEG artifacts)", "Wavelet"]],
label="Color-Fix Type",
info="AdaIn=Improve following a style, Wavelet=For JPEG artifacts",
value="AdaIn",
interactive=True
)
s_cfg = gr.Slider(
label="Text Guidance Scale",
info="Lower=follow the image, higher=follow the prompt",
minimum=1.0,
maximum=15.0,
value=default_setting.s_cfg_Quality if torch.cuda.device_count() > 0 else 1.0,
step=0.1
)
s_stage2 = gr.Slider(
label="Restoring Guidance Strength",
minimum=0.,
maximum=1.,
value=1.,
step=0.05
)
s_stage1 = gr.Slider(
label="Pre-denoising Guidance Strength",
minimum=-1.0,
maximum=6.0,
value=-1.0,
step=1.0
)
s_churn = gr.Slider(
label="S-Churn",
minimum=0,
maximum=40,
value=5,
step=1
)
s_noise = gr.Slider(
label="S-Noise",
minimum=1.0,
maximum=1.1,
value=1.003,
step=0.001
)
with gr.Row():
with gr.Column():
linear_CFG = gr.Checkbox(label="Linear CFG", value=True)
spt_linear_CFG = gr.Slider(
label="CFG Start",
minimum=1.0,
maximum=9.0,
value=default_setting.spt_linear_CFG_Quality if torch.cuda.device_count() > 0 else 1.0,
step=0.5
)
with gr.Column():
linear_s_stage2 = gr.Checkbox(label="Linear Restoring Guidance", value=False)
spt_linear_s_stage2 = gr.Slider(
label="Guidance Start",
minimum=0.,
maximum=1.,
value=0.,
step=0.05
)
with gr.Column():
diff_dtype = gr.Radio(
[["fp32 (precision)", "fp32"], ["fp16 (medium)", "fp16"], ["bf16 (speed)", "bf16"]],
label="Diffusion Data Type",
value="fp32",
interactive=True
)
with gr.Column():
ae_dtype = gr.Radio(
[["fp32 (precision)", "fp32"], ["bf16 (speed)", "bf16"]],
label="Auto-Encoder Data Type",
value="fp32",
interactive=True
)
randomize_seed = gr.Checkbox(
label="\U0001F3B2 Randomize seed",
value=True,
info="If checked, result is always different"
)
seed = gr.Slider(
label="Seed",
minimum=0,
maximum=max_64_bit_int,
step=1,
randomize=True
)
with gr.Group():
param_setting = gr.Radio(
["Quality", "Fidelity"],
interactive=True,
label="Presetting",
value="Quality"
)
restart_button = gr.Button(value="Apply presetting")
with gr.Column():
diffusion_button = gr.Button(
value="🚀 Enhance Photo",
variant="primary",
elem_id="process_button"
)
reset_btn = gr.Button(
value="🧹 Reset",
variant="stop",
elem_id="reset_button",
visible=False
)
restore_information = gr.HTML(
value="Start the process again if you want a different result.",
visible=False
)
result_slider = ImageSlider(
label='Result',
show_label=False,
interactive=False,
elem_id="slider1",
show_download_button=True # Enable the download button
)
input_image.upload(
fn=process_uploaded_image,
inputs=input_image,
outputs=input_image,
queue=False
)
input_image.upload(
fn=check_upload,
inputs=input_image,
outputs=[],
queue=False,
show_progress=False
)
diffusion_button.click(
fn=update_seed,
inputs=[randomize_seed, seed],
outputs=[seed],
queue=False,
show_progress=False
).then(
fn=check,
inputs=[input_image],
outputs=[],
queue=False,
show_progress=False
).success(
fn=stage2_process,
inputs=[
input_image,
prompt,
a_prompt,
n_prompt,
num_samples,
min_size,
downscale,
upscale,
edm_steps,
s_stage1,
s_stage2,
s_cfg,
seed,
s_churn,
s_noise,
color_fix_type,
diff_dtype,
ae_dtype,
gamma_correction, # Use the hidden gamma_correction component
linear_CFG,
linear_s_stage2,
spt_linear_CFG,
spt_linear_s_stage2,
model_select,
allocation
],
outputs=[
result_slider,
restore_information,
reset_btn
]
).success(
fn=log_information,
inputs=[result_slider],
outputs=[],
queue=False,
show_progress=False
)
restart_button.click(
fn=load_and_reset,
inputs=[param_setting],
outputs=[
edm_steps,
s_cfg,
s_stage2,
s_stage1,
s_churn,
s_noise,
a_prompt,
n_prompt,
color_fix_type,
linear_CFG,
linear_s_stage2,
spt_linear_CFG,
spt_linear_s_stage2,
model_select
]
)
reset_btn.click(
fn=reset,
inputs=[],
outputs=[
input_image,
prompt,
a_prompt,
n_prompt,
num_samples,
min_size,
downscale,
upscale,
edm_steps,
s_stage1,
s_stage2,
s_cfg,
seed,
s_churn,
s_noise,
color_fix_type,
diff_dtype,
ae_dtype,
gamma_correction, # Use the hidden gamma_correction component
linear_CFG,
linear_s_stage2,
spt_linear_CFG,
spt_linear_s_stage2,
model_select,
allocation
],
queue=False,
show_progress=False
)
interface.queue(10).launch()