OzUVGL/ips/ops.py

import cv2
import numpy as np

from fractions import Fraction
from exifread.utils import Ratio
from PIL import Image
from skimage.color import rgb2hsv, hsv2rgb
from skimage.exposure import rescale_intensity
from skimage.filters import gaussian as sk_gaussian
from skimage.restoration import denoise_tv_bregman
from scipy import signal

from colour_demosaicing import demosaicing_CFA_Bayer_Menon2007

from utils.misc import *
from utils.color import *
from ips.wb import illumination_parameters_estimation


def normalize(raw_image, black_level, white_level):
    if isinstance(black_level, list) and len(black_level) == 1:
        black_level = float(black_level[0])
    if isinstance(white_level, list) and len(white_level) == 1:
        white_level = float(white_level[0])
    black_level_mask = black_level
    if type(black_level) is list and len(black_level) == 4:
        if type(black_level[0]) is Ratio:
            black_level = ratios2floats(black_level)
        if type(black_level[0]) is Fraction:
            black_level = fractions2floats(black_level)
        black_level_mask = np.zeros(raw_image.shape)
        idx2by2 = [[0, 0], [0, 1], [1, 0], [1, 1]]
        step2 = 2
        for i, idx in enumerate(idx2by2):
            black_level_mask[idx[0]::step2, idx[1]::step2] = black_level[i]
    normalized_image = raw_image.astype(np.float32) - black_level_mask
    # if some values were smaller than black level
    normalized_image[normalized_image < 0] = 0
    normalized_image = normalized_image / (white_level - black_level_mask)
    return normalized_image


def demosaic(norm_image, cfa_pattern):
    return demosaicing_CFA_Bayer_Menon2007(norm_image, decode_cfa_pattern(cfa_pattern))


def denoise(demosaiced_image, y_noise_profile, cc_noise_profile):
    ycc_demosaiced = rgb2ycc(demosaiced_image[:, :, ::-1])
    y_demosaiced = ycc_demosaiced[:, :, 0]
    cc_demosaiced = ycc_demosaiced[:, :, 1:]
    current_image_y = y_demosaiced
    current_image_cc = gaussian(cc_demosaiced, sigma=cc_noise_profile)
    current_image_ycc = np.concatenate([
        np.expand_dims(current_image_y, -1), 
        current_image_cc
    ], axis=-1)
    return ycc2rgb(current_image_ycc)[:, :, ::-1]


def raw_color_denoise(demosaiced_image, cc_noise_profile):
    ycc_demosaiced = rgb2ycc(demosaiced_image[:, :, ::-1])
    cc_demosaiced = ycc_demosaiced[:, :, 1:]
    cc_demosaiced_denoised = sk_gaussian(cc_demosaiced, sigma=cc_noise_profile)
    ycc_demosaiced[:, :, 1:] = cc_demosaiced_denoised
    return ycc2rgb(ycc_demosaiced)[:, :, ::-1]


def luminance_denoise(tone_mapped_image, weight=20.0):
    ycc_tone_mapped = rgb2ycc(tone_mapped_image[:, :, ::-1])
    y_tone_mapped = ycc_tone_mapped[:, :, 0]
    y_tone_mapped_denoised = denoise_tv_bregman(y_tone_mapped, weight=weight)
    ycc_tone_mapped[:, :, 0] = np.clip(y_tone_mapped_denoised, 1e-4, 0.999)
    return ycc2rgb(ycc_tone_mapped)[:, :, ::-1]


def white_balance(denoised_image, metadata, max_repeat_limit=10000):
    if metadata["wb_estimation"] is not None:
        as_shot_neutral = np.array(metadata["wb_estimation"])
        white_balanced_image = np.dot(denoised_image, as_shot_neutral.T)
        return np.clip(white_balanced_image, 0.0, 1.0)
    illumuniation_estimation_algorithm = metadata["wb_method"]
    as_shot_neutral = illumination_parameters_estimation(denoised_image, illumuniation_estimation_algorithm)    
    
    if isinstance(as_shot_neutral[0], Ratio):
        as_shot_neutral = ratios2floats(as_shot_neutral)

    as_shot_neutral = np.asarray(as_shot_neutral)
    # transform vector into matrix
    if as_shot_neutral.shape == (3,):
        as_shot_neutral = np.diag(1./as_shot_neutral)

    assert as_shot_neutral.shape == (3, 3)
    repeat_count = 0
    while (as_shot_neutral[0, 0] < 2.3 and as_shot_neutral[2, 2] < 2.3) or (as_shot_neutral[0, 0] < 2.02 or as_shot_neutral[2, 2] < 1.92):
        if repeat_count < max_repeat_limit:
            as_shot_neutral = illumination_parameters_estimation(denoised_image, illumuniation_estimation_algorithm)
            if isinstance(as_shot_neutral[0], Ratio):
                as_shot_neutral = ratios2floats(as_shot_neutral)

            as_shot_neutral = np.asarray(as_shot_neutral)
            # transform vector into matrix
            if as_shot_neutral.shape == (3,):
                as_shot_neutral = np.diag(1./as_shot_neutral)

            assert as_shot_neutral.shape == (3, 3)
        else:
            print(f"WARNING! Invalid range for illumination matrix and repeated to estimate by '{illumuniation_estimation_algorithm}' so many times. Using 'gw' for illumination estimation now...")
            as_shot_neutral = illumination_parameters_estimation(denoised_image, "gw")  
            if isinstance(as_shot_neutral[0], Ratio):
                as_shot_neutral = ratios2floats(as_shot_neutral)

            as_shot_neutral = np.asarray(as_shot_neutral)
            # transform vector into matrix
            if as_shot_neutral.shape == (3,):
                as_shot_neutral = np.diag(1./as_shot_neutral)

            assert as_shot_neutral.shape == (3, 3)
            break
        repeat_count += 1
    
    white_balanced_image = np.dot(denoised_image, as_shot_neutral.T)
    metadata["wb_estimation"] = as_shot_neutral.tolist()

    # print(as_shot_neutral)
    return np.clip(white_balanced_image, 0.0, 1.0)


def xyz_transform(wb_image, color_matrix):
    if isinstance(color_matrix[0], Fraction):
        color_matrix = fractions2floats(color_matrix)
    xyz2cam = np.reshape(np.asarray(color_matrix), (3, 3))
    # normalize rows (needed?)
    xyz2cam = xyz2cam / np.sum(xyz2cam, axis=1, keepdims=True)
    # inverse
    cam2xyz = np.linalg.inv(xyz2cam)
    # for now, use one matrix  # TODO: interpolate btween both
    # simplified matrix multiplication
    xyz_image = cam2xyz[np.newaxis, np.newaxis, :, :] * wb_image[:, :, np.newaxis, :]
    xyz_image = np.sum(xyz_image, axis=-1)
    xyz_image = np.clip(xyz_image, 0.0, 1.0)
    return xyz_image


def xyz_to_srgb(xyz_image):
    # srgb2xyz = np.array([[0.4124564, 0.3575761, 0.1804375],
    #                      [0.2126729, 0.7151522, 0.0721750],
    #                      [0.0193339, 0.1191920, 0.9503041]])

    # xyz2srgb = np.linalg.inv(srgb2xyz)

    xyz2srgb = np.array([[3.2404542, -1.5371385, -0.4985314],
                         [-0.9692660, 1.8760108, 0.0415560],
                         [0.0556434, -0.2040259, 1.0572252]])

    # normalize rows (needed?)
    xyz2srgb = xyz2srgb / np.sum(xyz2srgb, axis=-1, keepdims=True)

    srgb_image = xyz2srgb[np.newaxis, np.newaxis,
                          :, :] * xyz_image[:, :, np.newaxis, :]
    srgb_image = np.sum(srgb_image, axis=-1)
    srgb_image = np.clip(srgb_image, 0.0, 1.0)
    return srgb_image


def apply_tmo_flash(Y, a):
    Y[Y == 0] = 1e-9
    return Y / (Y + a * np.exp(np.mean(np.log(Y))))


def apply_tmo_storm(Y, a, kernels):
    rows, cols = Y.shape
    Y[Y == 0] = 1e-9
    return sum([
        Y / (Y + a * np.exp(cv2.boxFilter(np.log(Y), -1, (int(min(rows // kernel, cols // kernel)),) * 2)))
        for kernel in kernels
    ]) / len(kernels)


def apply_tmo_nite(Y, CC, kernels):
    rows, cols = Y.shape
    Y[Y == 0] = 1e-9
    y_mu, y_std = max(Y.mean(), 0.001), Y.std()
    cc_std = CC.std()
    # tmo_offset = np.exp(y_mu * (cc_std / y_std) * 100)
    tmo_offset = 10. / np.sqrt(np.exp(np.log(y_mu) * (np.log(cc_std) / np.log(y_std))) * 100)
    # print(f"Y mean: {y_mu:.3f}, Y std: {y_std:.3f}, CC std: {cc_std:.3f}, Offset: {tmo_offset:.3f}")

    # tmo_scale = 8.5 + min(6.5, round(tmo_offset))
    tmo_scale = min(28., max(5., tmo_offset))
    # print(f"TMO scale: {tmo_scale}")
    return sum([
        Y / np.clip((Y + tmo_scale * np.exp(cv2.boxFilter(np.log(Y), -1, (int(min(rows // kernel, cols // kernel)),) * 2))), 0., 1.)
        for kernel in kernels
    ]) / len(kernels)


def perform_tone_mapping(source, metadata):
    ycc_source = rgb2ycc(source[:, :, ::-1])
    y_source = ycc_source[:, :, 0]
    cc_source = ycc_source[:, :, 1:]

    if metadata["tmo_type"].lower() == "flash":
        y_hat_source = apply_tmo_flash(y_source, metadata["tmo_scale"])
    elif metadata["tmo_type"].lower() == "storm":
        y_hat_source = apply_tmo_storm(y_source, metadata["tmo_scale"], metadata["tmo_kernels"])
    else:  # nite
        y_hat_source = apply_tmo_nite(y_source, cc_source, metadata["tmo_kernels"])

    ycc_nite = np.concatenate([
        np.expand_dims(y_hat_source, -1), 
        cc_source
    ], axis=-1)
    result = ycc2rgb(ycc_nite)[:, :, ::-1]
    if metadata["tmo_do_leap"]:
        target_mean_grayscale = 0.282  # 72 / 255
        result = np.clip(result, a_min=0., a_max=1.)
        grayscale = cv2.cvtColor(result * 255., cv2.COLOR_BGR2GRAY) / 255.
        result *= target_mean_grayscale / np.mean(grayscale)
    result = np.clip(result, a_min=0., a_max=1.)
    return result


def global_mean_contrast(input_im, beta=1.0):
    mu_ = input_im.mean(axis=(0, 1), keepdims=True)
    output_im = mu_ + beta * (input_im - mu_)
    output_im = np.where(0 > output_im, input_im, output_im)
    output_im = np.where(1 < output_im, input_im, output_im)
    return output_im


def s_curve_correction(input_im, alpha=0.5, lambd=0.5):
    ycc_ = rgb2ycc(input_im[:, :, ::-1])
    Y = ycc_[:, :, 0]
    Y_hat = alpha + np.where(
        Y >= alpha, 
        (1 - alpha) * np.power(((Y - alpha) / (1 - alpha)), lambd), 
        -alpha * np.power((1 - (Y / alpha)), lambd)
    )
    ycc_[:, :, 0] = Y_hat
    bgr_ = np.clip(ycc2rgb(ycc_)[:, :, ::-1], a_min=0., a_max=1.)
    return bgr_


def histogram_stretching(input_im):
    hsv = rgb2hsv(input_im[:, :, ::-1])
    V = hsv[:, :, 0]
    p0_01, p99 = np.percentile(V, (0.01, 99.99))
    if 0.7 > p99:
         _, p99 = np.percentile(V, (0.01, 99.5))

    V_hat = rescale_intensity(V, in_range=(p0_01, p99))
    hsv[:, :, 0] = V_hat
    bgr_ = np.clip(hsv2rgb(hsv), a_min=0., a_max=1.)[:, :, ::-1]
    return bgr_


def conditional_contrast_correction(input_im, threshold=0.5):
    ycc_ = rgb2ycc(input_im[:, :, ::-1])
    Y = ycc_[:, :, 0]
    y_avg = Y.mean()
    if y_avg > threshold:
        Y_hat = Y.copy()
        idx = Y_hat <= 0.0031308
        Y_hat[idx] *= 12.92
        Y_hat[idx == False] = (Y_hat[idx == False] ** (1.0 / 2.4)) * 1.055 - 0.055
    else:
        alpha = 0.5
        lambd = 1.2
        Y_hat = alpha + np.where(
            Y >= alpha, 
            (1 - alpha) * np.power(((Y - alpha) / (1 - alpha)), lambd), 
            -alpha * np.power((1 - (Y / alpha)), lambd)
        )
    ycc_[:, :, 0] = Y_hat
    bgr_ = np.clip(ycc2rgb(ycc_)[:, :, ::-1], a_min=0., a_max=1.)
    return bgr_


def memory_color_enhancement(data, color_space="srgb", illuminant="D65", clip_range=[0, 1], cie_version="1964"):
    target_hue = [30., -125., 100.]
    hue_preference = [20., -118., 130.]
    hue_sigma = [20., 10., 5.]
    is_both_side = [True, False, False]
    multiplier = [0.6, 0.6, 0.6]
    chroma_preference = [25., 14., 30.]
    chroma_sigma = [10., 10., 5.]

    # RGB to xyz
    data = rgb2xyz(data, color_space, clip_range)
    # xyz to lab
    data = xyz2lab(data, cie_version, illuminant)
    # lab to lch
    data = lab2lch(data)

    # hue squeezing
    # we are traversing through different color preferences
    height, width, _ = data.shape
    hue_correction = np.zeros((height, width), dtype=np.float32)
    for i in range(0, np.size(target_hue)):

        delta_hue = data[:, :, 2] - hue_preference[i]

        if is_both_side[i]:
            weight_temp = np.exp(-np.power(data[:, :, 2] - target_hue[i], 2) / (2 * hue_sigma[i] ** 2)) + \
                          np.exp(-np.power(data[:, :, 2] + target_hue[i], 2) / (2 * hue_sigma[i] ** 2))
        else:
            weight_temp = np.exp(-np.power(data[:, :, 2] - target_hue[i], 2) / (2 * hue_sigma[i] ** 2))

        weight_hue = multiplier[i] * weight_temp / np.max(weight_temp)

        weight_chroma = np.exp(-np.power(data[:, :, 1] - chroma_preference[i], 2) / (2 * chroma_sigma[i] ** 2))

        hue_correction = hue_correction + np.multiply(np.multiply(delta_hue, weight_hue), weight_chroma)

    # correct the hue
    data[:, :, 2] = data[:, :, 2] - hue_correction

    # lch to lab
    data = lch2lab(data)
    # lab to xyz
    data = lab2xyz(data, cie_version, illuminant)
    # xyz to rgb
    data = xyz2rgb(data, color_space, clip_range)

    data = outOfGamutClipping(data, range=clip_range[1])
    return data


def unsharp_masking(data, gaussian_kernel_size=[5, 5], gaussian_sigma=2.0, slope=1.5, tau_threshold=0.05, gamma_speed=4., clip_range=[0, 1]):
    # create gaussian kernel
    gaussian_kernel = gaussian(gaussian_kernel_size, gaussian_sigma)

    # convolve the image with the gaussian kernel
    # first input is the image
    # second input is the kernel
    # output shape will be the same as the first input
    # boundary will be padded by using symmetrical method while convolving
    if np.ndim(data) > 2:
        image_blur = np.empty(np.shape(data), dtype=np.float32)
        for i in range(0, np.shape(data)[2]):
            image_blur[:, :, i] = signal.convolve2d(data[:, :, i], gaussian_kernel, mode="same", boundary="symm")
    else:
        image_blur = signal.convolve2d(data, gaussian_kernel, mode="same", boundary="symm")

    # the high frequency component image
    image_high_pass = data - image_blur

    # soft coring (see in utility)
    # basically pass the high pass image via a slightly nonlinear function
    tau_threshold = tau_threshold * clip_range[1]

    # add the soft cored high pass image to the original and clip
    # within range and return
    def soft_coring(img_hp, slope, tau_threshold, gamma_speed):
        return slope * np.float32(img_hp) * (1. - np.exp(-((np.abs(img_hp / tau_threshold))**gamma_speed)))
    return np.clip(data + soft_coring(image_high_pass, slope, tau_threshold, gamma_speed), clip_range[0], clip_range[1])


def to_uint8(srgb):
    return (srgb * 255).astype(np.uint8)


def resize(img, width=None, height=None):
    if width is None or height is None:
        return img
    img_pil = Image.fromarray(img)
    out_size = (width, height)
    if img_pil.size == out_size:
        return img
    out_img = img_pil.resize(out_size, Image.Resampling.LANCZOS)
    out_img = np.array(out_img)
    return out_img


def fix_orientation(image, orientation):
    # 1 = Horizontal (normal)
    # 2 = Mirror horizontal
    # 3 = Rotate 180
    # 4 = Mirror vertical
    # 5 = Mirror horizontal and rotate 270 CW
    # 6 = Rotate 90 CW
    # 7 = Mirror horizontal and rotate 90 CW
    # 8 = Rotate 270 CW

    orientation_dict = {
        "Horizontal (normal)": 1,
        "Mirror horizontal": 2,
        "Rotate 180": 3,
        "Mirror vertical": 4,
        "Mirror horizontal and rotate 270 CW": 5,
        "Rotate 90 CW": 6,
        "Mirror horizontal and rotate 90 CW": 7,
        "Rotate 270 CW": 8
    }

    if type(orientation) is list:
        orientation = orientation[0]
    orientation = orientation_dict[orientation]
    if orientation == 1:
        pass
    elif orientation == 2:
        image = cv2.flip(image, 0)
    elif orientation == 3:
        image = cv2.rotate(image, cv2.ROTATE_180)
    elif orientation == 4:
        image = cv2.flip(image, 1)
    elif orientation == 5:
        image = cv2.flip(image, 0)
        image = cv2.rotate(image, cv2.ROTATE_90_COUNTERCLOCKWISE)
    elif orientation == 6:
        image = cv2.rotate(image, cv2.ROTATE_90_CLOCKWISE)
    elif orientation == 7:
        image = cv2.flip(image, 0)
        image = cv2.rotate(image, cv2.ROTATE_90_CLOCKWISE)
    elif orientation == 8:
        image = cv2.rotate(image, cv2.ROTATE_90_COUNTERCLOCKWISE)

    return image