losses/losses.py

import torch
import wandb
import cv2
import torch.nn.functional as F
import numpy as np
from facenet_pytorch import MTCNN
from torchvision import transforms
from dreamsim import dreamsim
from einops import rearrange
import kornia.augmentation as K
import lpips

from pretrained_models.arcface import Backbone
from utils.vis_utils import add_text_to_image
from utils.utils import extract_faces_and_landmarks
import clip


class Loss():
    """
    General purpose loss class. 
    Mainly handles dtype and visualize_every_k.
    keeps current iteration of loss, mainly for visualization purposes.
    """
    def __init__(self, visualize_every_k=-1, dtype=torch.float32, accelerator=None, **kwargs):
        self.visualize_every_k = visualize_every_k
        self.iteration = -1
        self.dtype=dtype
        self.accelerator = accelerator
        
    def __call__(self, **kwargs):
        self.iteration += 1
        return self.forward(**kwargs)


class L1Loss(Loss):
    """
    Simple L1 loss between predicted_pixel_values and pixel_values
    
    Args:
        predicted_pixel_values (torch.Tensor): The predicted pixel values using 1 step LCM and the VAE decoder.
        encoder_pixel_values (torch.Tesnor): The input image to the encoder
    """
    def forward(
        self, 
        predict: torch.Tensor,
        target: torch.Tensor,
        **kwargs
    ) -> torch.Tensor:
        return F.l1_loss(predict, target, reduction="mean")


class DreamSIMLoss(Loss):
    """DreamSIM loss between predicted_pixel_values and pixel_values.
    DreamSIM is similar to LPIPS (https://dreamsim-nights.github.io/) but is trained on more human defined similarity dataset
    DreamSIM expects an RGB image of size 224x224 and values between 0 and 1. So we need to normalize the input images to 0-1 range and resize them to 224x224.
    Args:
        predicted_pixel_values (torch.Tensor): The predicted pixel values using 1 step LCM and the VAE decoder.
        encoder_pixel_values (torch.Tesnor): The input image to the encoder
    """
    def __init__(self, device: str='cuda:0', **kwargs):
        super().__init__(**kwargs)
        self.model, _ = dreamsim(pretrained=True, device=device)
        self.model.to(dtype=self.dtype, device=device)
        self.model = self.accelerator.prepare(self.model)
        self.transforms = transforms.Compose([
            transforms.Lambda(lambda x: (x + 1) / 2),
            transforms.Resize((224, 224), interpolation=transforms.InterpolationMode.BICUBIC)])

    def forward(
        self,
        predicted_pixel_values: torch.Tensor,
        encoder_pixel_values: torch.Tensor,
        **kwargs,
    ) -> torch.Tensor:
        predicted_pixel_values.to(dtype=self.dtype)
        encoder_pixel_values.to(dtype=self.dtype)
        return self.model(self.transforms(predicted_pixel_values), self.transforms(encoder_pixel_values)).mean()


class LPIPSLoss(Loss):
    """LPIPS loss between predicted_pixel_values and pixel_values.
    Args:
        predicted_pixel_values (torch.Tensor): The predicted pixel values using 1 step LCM and the VAE decoder.
        encoder_pixel_values (torch.Tesnor): The input image to the encoder
    """
    def __init__(self,  **kwargs):
        super().__init__(**kwargs)
        self.model = lpips.LPIPS(net='vgg')
        self.model.to(dtype=self.dtype, device=self.accelerator.device)
        self.model = self.accelerator.prepare(self.model)

    def forward(self, predict, target, **kwargs):
        predict.to(dtype=self.dtype)
        target.to(dtype=self.dtype)
        return self.model(predict, target).mean()


class LCMVisualization(Loss):
    """Dummy loss used to visualize the LCM outputs
    Args:
        predicted_pixel_values (torch.Tensor): The predicted pixel values using 1 step LCM and the VAE decoder.
        pixel_values (torch.Tensor): The input image to the decoder
        encoder_pixel_values (torch.Tesnor): The input image to the encoder
    """
    def forward(
        self, 
        predicted_pixel_values: torch.Tensor,
        pixel_values: torch.Tensor,
        encoder_pixel_values: torch.Tensor,
        timesteps: torch.Tensor,
        **kwargs,
    ) -> None:
        if self.visualize_every_k > 0 and self.iteration % self.visualize_every_k == 0:
            predicted_pixel_values = rearrange(predicted_pixel_values, "n c h w -> (n h) w c").detach().cpu().numpy()
            pixel_values = rearrange(pixel_values, "n c h w -> (n h) w c").detach().cpu().numpy()
            encoder_pixel_values = rearrange(encoder_pixel_values, "n c h w -> (n h) w c").detach().cpu().numpy()
            image = np.hstack([encoder_pixel_values, pixel_values, predicted_pixel_values])
            for tracker in self.accelerator.trackers:
                if tracker.name == 'wandb':
                    tracker.log({"TrainVisualization": wandb.Image(image, caption=f"Encoder Input Image, Decoder Input Image, Predicted LCM Image. Timesteps {timesteps.cpu().tolist()}")})
        return torch.tensor(0.0)


class L2Loss(Loss):
    """
    Regular diffusion loss between predicted noise and target noise.
    Args:
        predicted_noise (torch.Tensor): noise predicted by the diffusion model
        target_noise (torch.Tensor): actual noise added to the image.
    """
    def forward(
        self,
        predict: torch.Tensor,
        target: torch.Tensor,
        weights: torch.Tensor = None,
        **kwargs
    ) -> torch.Tensor:
        if weights is not None:
            loss = (predict.float() - target.float()).pow(2) * weights
            return loss.mean()
        return F.mse_loss(predict.float(), target.float(), reduction="mean")


class HuberLoss(Loss):
    """Huber loss between predicted_pixel_values and pixel_values.
    Args:
        predicted_pixel_values (torch.Tensor): The predicted pixel values using 1 step LCM and the VAE decoder.
        encoder_pixel_values (torch.Tesnor): The input image to the encoder
    """
    def __init__(self, huber_c=0.001, **kwargs):
        super().__init__(**kwargs)
        self.huber_c = huber_c

    def forward(
        self,
        predict: torch.Tensor,
        target: torch.Tensor,
        weights: torch.Tensor = None,
        **kwargs
    ) -> torch.Tensor:
        loss = torch.sqrt((predict.float() - target.float()) ** 2 + self.huber_c**2) - self.huber_c
        if weights is not None:
            return (loss * weights).mean()
        return loss.mean()


class WeightedNoiseLoss(Loss):
    """
    Weighted diffusion loss between predicted noise and target noise.
    Args:
        predicted_noise (torch.Tensor): noise predicted by the diffusion model
        target_noise (torch.Tensor): actual noise added to the image.
        loss_batch_weights (torch.Tensor): weighting for each batch item. Can be used to e.g. zero-out loss for InstantID training if keypoint extraction fails.
    """
    def forward(
        self,
        predict: torch.Tensor,
        target: torch.Tensor,
        weights,
        **kwargs
    ) -> torch.Tensor:
        return F.mse_loss(predict.float() * weights, target.float() * weights, reduction="mean")


class IDLoss(Loss):
    """
    Use pretrained facenet model to extract features from the face of the predicted image and target image.
    Facenet expects 112x112 images, so we crop the face using MTCNN and resize it to 112x112.
    Then we use the cosine similarity between the features to calculate the loss. (The cosine similarity is 1 - cosine distance).
    Also notice that the outputs of facenet are normalized so the dot product is the same as cosine distance.
    """
    def __init__(self, pretrained_arcface_path: str, skip_not_found=True, **kwargs):
        super().__init__(**kwargs)
        assert pretrained_arcface_path is not None, "please pass `pretrained_arcface_path` in the losses config. You can download the pretrained model from "\
            "https://drive.google.com/file/d/1KW7bjndL3QG3sxBbZxreGHigcCCpsDgn/view?usp=sharing"
        self.mtcnn = MTCNN(device=self.accelerator.device)
        self.mtcnn.forward = self.mtcnn.detect
        self.facenet_input_size = 112  # Has to be 112, can't find weights for 224 size.
        self.facenet = Backbone(input_size=112, num_layers=50, drop_ratio=0.6, mode='ir_se')
        self.facenet.load_state_dict(torch.load(pretrained_arcface_path))
        self.face_pool = torch.nn.AdaptiveAvgPool2d((self.facenet_input_size, self.facenet_input_size))
        self.facenet.requires_grad_(False)
        self.facenet.eval()
        self.facenet.to(device=self.accelerator.device, dtype=self.dtype)  # not implemented for half precision
        self.face_pool.to(device=self.accelerator.device, dtype=self.dtype)  # not implemented for half precision
        self.visualization_resize = transforms.Resize((self.facenet_input_size, self.facenet_input_size), interpolation=transforms.InterpolationMode.BICUBIC)
        self.reference_facial_points = np.array([[38.29459953, 51.69630051],
                                                 [72.53179932, 51.50139999],
                                                 [56.02519989, 71.73660278],
                                                 [41.54930115, 92.3655014],
                                                 [70.72990036, 92.20410156]
                                                 ])  # Original points are 112 * 96 added 8 to the x axis to make it 112 * 112
        self.facenet, self.face_pool, self.mtcnn = self.accelerator.prepare(self.facenet, self.face_pool, self.mtcnn)

        self.skip_not_found = skip_not_found
    
    def extract_feats(self, x: torch.Tensor):
        """
        Extract features from the face of the image using facenet model.
        """
        x = self.face_pool(x)
        x_feats = self.facenet(x)

        return x_feats

    def forward(
        self, 
        predicted_pixel_values: torch.Tensor,
        encoder_pixel_values: torch.Tensor,
        timesteps: torch.Tensor,
        **kwargs
    ):
        encoder_pixel_values = encoder_pixel_values.to(dtype=self.dtype)
        predicted_pixel_values = predicted_pixel_values.to(dtype=self.dtype)

        predicted_pixel_values_face, predicted_invalid_indices = extract_faces_and_landmarks(predicted_pixel_values, mtcnn=self.mtcnn)
        with torch.no_grad():
            encoder_pixel_values_face, source_invalid_indices = extract_faces_and_landmarks(encoder_pixel_values, mtcnn=self.mtcnn)
        
        if self.skip_not_found:
            valid_indices = []
            for i in range(predicted_pixel_values.shape[0]):
                if i not in predicted_invalid_indices and i not in source_invalid_indices:
                    valid_indices.append(i)
        else:
            valid_indices = list(range(predicted_pixel_values))
            
        valid_indices = torch.tensor(valid_indices).to(device=predicted_pixel_values.device)

        if len(valid_indices) == 0:
            loss =  (predicted_pixel_values_face * 0.0).mean()  # It's done this way so the `backwards` will delete the computation graph of the predicted_pixel_values.
            if self.visualize_every_k > 0 and self.iteration % self.visualize_every_k == 0:
                self.visualize(predicted_pixel_values, encoder_pixel_values, predicted_pixel_values_face, encoder_pixel_values_face, timesteps, valid_indices, loss)
            return loss

        with torch.no_grad():
            pixel_values_feats = self.extract_feats(encoder_pixel_values_face[valid_indices])
            
        predicted_pixel_values_feats = self.extract_feats(predicted_pixel_values_face[valid_indices])
        loss = 1 - torch.einsum("bi,bi->b", pixel_values_feats, predicted_pixel_values_feats)

        if self.visualize_every_k > 0 and self.iteration % self.visualize_every_k == 0:
            self.visualize(predicted_pixel_values, encoder_pixel_values, predicted_pixel_values_face, encoder_pixel_values_face, timesteps, valid_indices, loss)
        return loss.mean()
    
    def visualize(
        self,
        predicted_pixel_values: torch.Tensor,
        encoder_pixel_values: torch.Tensor,
        predicted_pixel_values_face: torch.Tensor,
        encoder_pixel_values_face: torch.Tensor,
        timesteps: torch.Tensor,
        valid_indices: torch.Tensor,
        loss: torch.Tensor,
    ) -> None:
        small_predicted_pixel_values = (rearrange(self.visualization_resize(predicted_pixel_values), "n c h w -> (n h) w c").detach().cpu().numpy())
        small_pixle_values = rearrange(self.visualization_resize(encoder_pixel_values), "n c h w -> (n h) w c").detach().cpu().numpy() 
        small_predicted_pixel_values_face = rearrange(self.visualization_resize(predicted_pixel_values_face), "n c h w -> (n h) w c").detach().cpu().numpy()
        small_pixle_values_face = rearrange(self.visualization_resize(encoder_pixel_values_face), "n c h w -> (n h) w c").detach().cpu().numpy()
        
        small_predicted_pixel_values = add_text_to_image(((small_predicted_pixel_values * 0.5 + 0.5) * 255).astype(np.uint8), "Pred Images", add_below=False)
        small_pixle_values = add_text_to_image(((small_pixle_values * 0.5 + 0.5) * 255).astype(np.uint8), "Target Images", add_below=False)
        small_predicted_pixel_values_face = add_text_to_image(((small_predicted_pixel_values_face * 0.5 + 0.5) * 255).astype(np.uint8), "Pred Faces", add_below=False)
        small_pixle_values_face = add_text_to_image(((small_pixle_values_face * 0.5 + 0.5) * 255).astype(np.uint8), "Target Faces", add_below=False)


        final_image = np.hstack([small_predicted_pixel_values, small_pixle_values, small_predicted_pixel_values_face, small_pixle_values_face])
        for tracker in self.accelerator.trackers:
            if tracker.name == 'wandb':
                tracker.log({"IDLoss Visualization": wandb.Image(final_image, caption=f"loss: {loss.cpu().tolist()} timesteps: {timesteps.cpu().tolist()}, valid_indices: {valid_indices.cpu().tolist()}")})


class ImageAugmentations(torch.nn.Module):
    # Standard image augmentations used for CLIP loss to discourage adversarial outputs.
    def __init__(self, output_size, augmentations_number, p=0.7):
        super().__init__()
        self.output_size = output_size
        self.augmentations_number = augmentations_number

        self.augmentations = torch.nn.Sequential(
            K.RandomAffine(degrees=15, translate=0.1, p=p, padding_mode="border"),  # type: ignore
            K.RandomPerspective(0.7, p=p),
        )

        self.avg_pool = torch.nn.AdaptiveAvgPool2d((self.output_size, self.output_size))

        self.device = None

    def forward(self, input):
        """Extents the input batch with augmentations
        If the input is consists of images [I1, I2] the extended augmented output
        will be [I1_resized, I2_resized, I1_aug1, I2_aug1, I1_aug2, I2_aug2 ...]
        Args:
            input ([type]): input batch of shape [batch, C, H, W]
        Returns:
            updated batch: of shape [batch * augmentations_number, C, H, W]
        """
        # We want to multiply the number of images in the batch in contrast to regular augmantations
        # that do not change the number of samples in the batch)
        resized_images = self.avg_pool(input)
        resized_images = torch.tile(resized_images, dims=(self.augmentations_number, 1, 1, 1))

        batch_size = input.shape[0]
        # We want at least one non augmented image
        non_augmented_batch = resized_images[:batch_size]
        augmented_batch = self.augmentations(resized_images[batch_size:])
        updated_batch = torch.cat([non_augmented_batch, augmented_batch], dim=0)

        return updated_batch


class CLIPLoss(Loss):
    def __init__(self, augmentations_number: int = 4, **kwargs):
        super().__init__(**kwargs)

        self.clip_model, clip_preprocess = clip.load("ViT-B/16", device=self.accelerator.device, jit=False)

        self.clip_model.device = None

        self.clip_model.eval().requires_grad_(False)
        
        self.preprocess = transforms.Compose([transforms.Normalize(mean=[-1.0, -1.0, -1.0], std=[2.0, 2.0, 2.0])] + # Un-normalize from [-1.0, 1.0] (SD output) to [0, 1].
                                              clip_preprocess.transforms[:2] +                                      # to match CLIP input scale assumptions
                                              clip_preprocess.transforms[4:])                                       # + skip convert PIL to tensor

        self.clip_size = self.clip_model.visual.input_resolution

        self.clip_normalize = transforms.Normalize(
            mean=[0.48145466, 0.4578275, 0.40821073], std=[0.26862954, 0.26130258, 0.27577711]
        )

        self.image_augmentations = ImageAugmentations(output_size=self.clip_size,
                                                      augmentations_number=augmentations_number)
        
        self.clip_model, self.image_augmentations = self.accelerator.prepare(self.clip_model, self.image_augmentations)

    def forward(self, decoder_prompts, predicted_pixel_values: torch.Tensor, **kwargs) -> torch.Tensor:

        if not isinstance(decoder_prompts, list):
            decoder_prompts = [decoder_prompts]

        tokens = clip.tokenize(decoder_prompts).to(predicted_pixel_values.device)
        image  = self.preprocess(predicted_pixel_values)

        logits_per_image, _ = self.clip_model(image, tokens)

        logits_per_image = torch.diagonal(logits_per_image)

        return (1. - logits_per_image / 100).mean()


class DINOLoss(Loss):
    def __init__(
            self,
            dino_model,
            dino_preprocess,
            output_hidden_states: bool = False,
            center_momentum: float = 0.9,
            student_temp: float = 0.1,
            teacher_temp: float = 0.04,
            warmup_teacher_temp: float = 0.04,
            warmup_teacher_temp_epochs: int = 30,
            **kwargs):
        super().__init__(**kwargs)

        self.dino_model = dino_model
        self.output_hidden_states = output_hidden_states
        self.rescale_factor = dino_preprocess.rescale_factor

        # Un-normalize from [-1.0, 1.0] (SD output) to [0, 1].
        self.preprocess = transforms.Compose(
            [
                transforms.Normalize(mean=[-1.0, -1.0, -1.0], std=[2.0, 2.0, 2.0]),
                transforms.Resize(size=256),
                transforms.CenterCrop(size=(224, 224)),
                transforms.Normalize(mean=dino_preprocess.image_mean, std=dino_preprocess.image_std)
            ]
        )

        self.student_temp = student_temp
        self.teacher_temp = teacher_temp
        self.center_momentum = center_momentum
        self.center = torch.zeros(1, 257, 1024).to(self.accelerator.device, dtype=self.dtype)

        # TODO: add temp, now fixed to 0.04
        # we apply a warm up for the teacher temperature because
        # a too high temperature makes the training instable at the beginning
        # self.teacher_temp_schedule = np.concatenate((
        #     np.linspace(warmup_teacher_temp,
        #                 teacher_temp, warmup_teacher_temp_epochs),
        #     np.ones(nepochs - warmup_teacher_temp_epochs) * teacher_temp
        # ))

        self.dino_model = self.accelerator.prepare(self.dino_model)

    def forward(
            self,
            target: torch.Tensor,
            predict: torch.Tensor,
            weights: torch.Tensor = None,
            **kwargs) -> torch.Tensor:

        predict = self.preprocess(predict)
        target = self.preprocess(target)

        encoder_input = torch.cat([target, predict]).to(self.dino_model.device, dtype=self.dino_model.dtype)

        if self.output_hidden_states:
            raise ValueError("Output hidden states not supported for DINO loss.")
            image_enc_hidden_states = self.dino_model(encoder_input, output_hidden_states=True).hidden_states[-2]
        else:
            image_enc_hidden_states = self.dino_model(encoder_input).last_hidden_state

        teacher_output, student_output = image_enc_hidden_states.chunk(2, dim=0)         # [B, 257, 1024]

        student_out = student_output.float() / self.student_temp

        # teacher centering and sharpening
        # temp = self.teacher_temp_schedule[epoch]
        temp = self.teacher_temp
        teacher_out = F.softmax((teacher_output.float() - self.center) / temp, dim=-1)
        teacher_out = teacher_out.detach()

        loss = torch.sum(-teacher_out * F.log_softmax(student_out, dim=-1), dim=-1, keepdim=True)
        # self.update_center(teacher_output)

        if weights is not None:
            loss = loss * weights
            return loss.mean()
        return loss.mean()

    @torch.no_grad()
    def update_center(self, teacher_output):
        """
        Update center used for teacher output.
        """
        batch_center = torch.sum(teacher_output, dim=0, keepdim=True)
        self.accelerator.reduce(batch_center, reduction="sum")
        batch_center = batch_center / (len(teacher_output) * self.accelerator.num_processes)

        # ema update
        self.center = self.center * self.center_momentum + batch_center * (1 - self.center_momentum)