132

.gitattributes

@@ -32,3 +32,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+classifier_DeepFakeClassifier_tf_efficientnet_b7_ns_1_best_dice filter=lfs diff=lfs merge=lfs -text

app.py

@@ -1,74 +1,71 @@
 import gradio as gr
-# import argparse
-# import os
-# import re
-# import time
+import argparse
+import os
+import re
+import time
 
-# import torch
-# import pandas as pd
+import torch
+import pandas as pd
 
-# # import os, sys
-# # root_folder = os.path.abspath(
-# #     os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-# # )
-# # sys.path.append(root_folder)
-# from kernel_utils import VideoReader, FaceExtractor, confident_strategy, predict_on_video_set
-# from classifiers import DeepFakeClassifier
-# import gradio as gr
+# import os, sys
+# root_folder = os.path.abspath(
+#     os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+# )
+# sys.path.append(root_folder)
+from kernel_utils import VideoReader, FaceExtractor, confident_strategy, predict_on_video_set
+from classifiers import DeepFakeClassifier
+import gradio as gr
 
-    
-    
-# def predict(video):
+   
+   
+def predict(video):
 
-#     # video_index = int(video_index)
-    
-#     frames_per_video = 32
-#     video_reader = VideoReader()
-#     video_read_fn = lambda x: video_reader.read_frames(x, num_frames=frames_per_video)
-#     face_extractor = FaceExtractor(video_read_fn)
-#     input_size = 380
-#     strategy = confident_strategy
-    
-#     # test_videos = sorted([x for x in os.listdir(args.test_dir) if x[-4:] == ".mp4"])[video_index]
-#     # print(f"Predicting {video_index} videos")
-#     predictions = predict_on_video_set(face_extractor=face_extractor, input_size=input_size, models=models,
-#                                        strategy=strategy, frames_per_video=frames_per_video, videos=video,
-#                                        num_workers=6, test_dir=args.test_dir)
-#     return predictions
+    frames_per_video = 32
+    video_reader = VideoReader()
+    video_read_fn = lambda x: video_reader.read_frames(x, num_frames=frames_per_video)
+    face_extractor = FaceExtractor(video_read_fn)
+    input_size = 380
+    strategy = confident_strategy
+   
+    # test_videos = sorted([x for x in os.listdir(args.test_dir) if x[-4:] == ".mp4"])[video_index]
+    # print(f"Predicting {video_index} videos")
+    predictions = predict_on_video_set(face_extractor=face_extractor, input_size=input_size, models=models,
+                                       strategy=strategy, frames_per_video=frames_per_video, videos=video,
+                                       num_workers=6, test_dir=args.test_dir)
+    return predictions
 
-# def get_args_models():
-#     parser = argparse.ArgumentParser("Predict test videos")
-#     arg = parser.add_argument
-#     arg('--weights-dir', type=str, default="weights", help="path to directory with checkpoints")
-#     arg('--models', type=str, default='classifier_DeepFakeClassifier_tf_efficientnet_b7_ns_1_best_dice', help="checkpoint files")  # nargs='+',
-#     arg('--test-dir', type=str, default='test_dataset', help="path to directory with videos")
-#     arg('--output', type=str, required=False, help="path to output csv", default="submission.csv")
-#     args = parser.parse_args()
+def get_args_models():
+    parser = argparse.ArgumentParser("Predict test videos")
+    arg = parser.add_argument
+    arg('--weights-dir', type=str, default=".", help="path to directory with checkpoints")
+    arg('--models', type=str, default='classifier_DeepFakeClassifier_tf_efficientnet_b7_ns_1_best_dice', help="checkpoint files")  # nargs='+',
+    arg('--test-dir', type=str, default='test_dataset', help="path to directory with videos")
+    arg('--output', type=str, required=False, help="path to output csv", default="submission.csv")
+    args = parser.parse_args()
 
-#     models = []
-#     # model_paths = [os.path.join(args.weights_dir, model) for model in args.models]
-#     model_paths = [os.path.join(args.weights_dir, args.models)]
-#     for path in model_paths:
-#         model = DeepFakeClassifier(encoder="tf_efficientnet_b7_ns").to("cpu")
-#         print("loading state dict {}".format(path))
-#         checkpoint = torch.load(path, map_location="cpu")
-#         state_dict = checkpoint.get("state_dict", checkpoint)
-#         model.load_state_dict({re.sub("^module.", "", k): v for k, v in state_dict.items()}, strict=True)
-#         model.eval()
-#         del checkpoint
-#         models.append(model.half())
-#     return args, models
+    models = []
+    # model_paths = [os.path.join(args.weights_dir, model) for model in args.models]
+    model_paths = [os.path.join(args.weights_dir, args.models)]
+    for path in model_paths:
+        model = DeepFakeClassifier(encoder="tf_efficientnet_b7_ns").to("cpu")
+        print("loading state dict {}".format(path))
+        checkpoint = torch.load(path, map_location="cpu")
+        state_dict = checkpoint.get("state_dict", checkpoint)
+        model.load_state_dict({re.sub("^module.", "", k): v for k, v in state_dict.items()}, strict=True)
+        model.eval()
+        del checkpoint
+        models.append(model)
+    return args, models
 
 def greet(name):
     return "Hello " + name + "!!"
 
 if __name__ == '__main__':
-    # global args, models
-    # args, models = get_args_models()
-    
+    global args, models
+    args, models = get_args_models()
+   
     # stime = time.time()
     # print("Elapsed:", time.time() - stime)
-    
-    demo = gr.Interface(fn=greet, inputs="video", outputs="text")
-    demo.launch()  
-
+   
+    demo = gr.Interface(fn=predict, inputs="video", outputs="text")
+    demo.launch()  

classifier_DeepFakeClassifier_tf_efficientnet_b7_ns_1_best_dice

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8ec8b8c200d260679069d022ca396ba55ded41c7a92c6b99f5ae52406a304ba
+size 267100135

classifiers.py

@@ -0,0 +1,172 @@
+from functools import partial
+
+import numpy as np
+import torch
+from timm.models.efficientnet import tf_efficientnet_b4_ns, tf_efficientnet_b3_ns, \
+    tf_efficientnet_b5_ns, tf_efficientnet_b2_ns, tf_efficientnet_b6_ns, tf_efficientnet_b7_ns
+from torch import nn
+from torch.nn.modules.dropout import Dropout
+from torch.nn.modules.linear import Linear
+from torch.nn.modules.pooling import AdaptiveAvgPool2d
+
+encoder_params = {
+    "tf_efficientnet_b3_ns": {
+        "features": 1536,
+        "init_op": partial(tf_efficientnet_b3_ns, pretrained=True, drop_path_rate=0.2)
+    },
+    "tf_efficientnet_b2_ns": {
+        "features": 1408,
+        "init_op": partial(tf_efficientnet_b2_ns, pretrained=False, drop_path_rate=0.2)
+    },
+    "tf_efficientnet_b4_ns": {
+        "features": 1792,
+        "init_op": partial(tf_efficientnet_b4_ns, pretrained=True, drop_path_rate=0.5)
+    },
+    "tf_efficientnet_b5_ns": {
+        "features": 2048,
+        "init_op": partial(tf_efficientnet_b5_ns, pretrained=True, drop_path_rate=0.2)
+    },
+    "tf_efficientnet_b4_ns_03d": {
+        "features": 1792,
+        "init_op": partial(tf_efficientnet_b4_ns, pretrained=True, drop_path_rate=0.3)
+    },
+    "tf_efficientnet_b5_ns_03d": {
+        "features": 2048,
+        "init_op": partial(tf_efficientnet_b5_ns, pretrained=True, drop_path_rate=0.3)
+    },
+    "tf_efficientnet_b5_ns_04d": {
+        "features": 2048,
+        "init_op": partial(tf_efficientnet_b5_ns, pretrained=True, drop_path_rate=0.4)
+    },
+    "tf_efficientnet_b6_ns": {
+        "features": 2304,
+        "init_op": partial(tf_efficientnet_b6_ns, pretrained=True, drop_path_rate=0.2)
+    },
+    "tf_efficientnet_b7_ns": {
+        "features": 2560,
+        "init_op": partial(tf_efficientnet_b7_ns, pretrained=True, drop_path_rate=0.2)
+    },
+    "tf_efficientnet_b6_ns_04d": {
+        "features": 2304,
+        "init_op": partial(tf_efficientnet_b6_ns, pretrained=True, drop_path_rate=0.4)
+    },
+}
+
+
+def setup_srm_weights(input_channels: int = 3) -> torch.Tensor:
+    """Creates the SRM kernels for noise analysis."""
+    # note: values taken from Zhou et al., "Learning Rich Features for Image Manipulation Detection", CVPR2018
+    srm_kernel = torch.from_numpy(np.array([
+        [  # srm 1/2 horiz
+            [0., 0., 0., 0., 0.],  # noqa: E241,E201
+            [0., 0., 0., 0., 0.],  # noqa: E241,E201
+            [0., 1., -2., 1., 0.],  # noqa: E241,E201
+            [0., 0., 0., 0., 0.],  # noqa: E241,E201
+            [0., 0., 0., 0., 0.],  # noqa: E241,E201
+        ], [  # srm 1/4
+            [0., 0., 0., 0., 0.],  # noqa: E241,E201
+            [0., -1., 2., -1., 0.],  # noqa: E241,E201
+            [0., 2., -4., 2., 0.],  # noqa: E241,E201
+            [0., -1., 2., -1., 0.],  # noqa: E241,E201
+            [0., 0., 0., 0., 0.],  # noqa: E241,E201
+        ], [  # srm 1/12
+            [-1., 2., -2., 2., -1.],  # noqa: E241,E201
+            [2., -6., 8., -6., 2.],  # noqa: E241,E201
+            [-2., 8., -12., 8., -2.],  # noqa: E241,E201
+            [2., -6., 8., -6., 2.],  # noqa: E241,E201
+            [-1., 2., -2., 2., -1.],  # noqa: E241,E201
+        ]
+    ])).float()
+    srm_kernel[0] /= 2
+    srm_kernel[1] /= 4
+    srm_kernel[2] /= 12
+    return srm_kernel.view(3, 1, 5, 5).repeat(1, input_channels, 1, 1)
+
+
+def setup_srm_layer(input_channels: int = 3) -> torch.nn.Module:
+    """Creates a SRM convolution layer for noise analysis."""
+    weights = setup_srm_weights(input_channels)
+    conv = torch.nn.Conv2d(input_channels, out_channels=3, kernel_size=5, stride=1, padding=2, bias=False)
+    with torch.no_grad():
+        conv.weight = torch.nn.Parameter(weights, requires_grad=False)
+    return conv
+
+
+class DeepFakeClassifierSRM(nn.Module):
+    def __init__(self, encoder, dropout_rate=0.5) -> None:
+        super().__init__()
+        self.encoder = encoder_params[encoder]["init_op"]()
+        self.avg_pool = AdaptiveAvgPool2d((1, 1))
+        self.srm_conv = setup_srm_layer(3)
+        self.dropout = Dropout(dropout_rate)
+        self.fc = Linear(encoder_params[encoder]["features"], 1)
+
+    def forward(self, x):
+        noise = self.srm_conv(x)
+        x = self.encoder.forward_features(noise)
+        x = self.avg_pool(x).flatten(1)
+        x = self.dropout(x)
+        x = self.fc(x)
+        return x
+
+
+class GlobalWeightedAvgPool2d(nn.Module):
+    """
+    Global Weighted Average Pooling from paper "Global Weighted Average
+    Pooling Bridges Pixel-level Localization and Image-level Classification"
+    """
+
+    def __init__(self, features: int, flatten=False):
+        super().__init__()
+        self.conv = nn.Conv2d(features, 1, kernel_size=1, bias=True)
+        self.flatten = flatten
+
+    def fscore(self, x):
+        m = self.conv(x)
+        m = m.sigmoid().exp()
+        return m
+
+    def norm(self, x: torch.Tensor):
+        return x / x.sum(dim=[2, 3], keepdim=True)
+
+    def forward(self, x):
+        input_x = x
+        x = self.fscore(x)
+        x = self.norm(x)
+        x = x * input_x
+        x = x.sum(dim=[2, 3], keepdim=not self.flatten)
+        return x
+
+
+class DeepFakeClassifier(nn.Module):
+    def __init__(self, encoder, dropout_rate=0.0) -> None:
+        super().__init__()
+        self.encoder = encoder_params[encoder]["init_op"]()
+        self.avg_pool = AdaptiveAvgPool2d((1, 1))
+        self.dropout = Dropout(dropout_rate)
+        self.fc = Linear(encoder_params[encoder]["features"], 1)
+
+    def forward(self, x):
+        x = self.encoder.forward_features(x)
+        x = self.avg_pool(x).flatten(1)
+        x = self.dropout(x)
+        x = self.fc(x)
+        return x
+
+
+
+
+class DeepFakeClassifierGWAP(nn.Module):
+    def __init__(self, encoder, dropout_rate=0.5) -> None:
+        super().__init__()
+        self.encoder = encoder_params[encoder]["init_op"]()
+        self.avg_pool = GlobalWeightedAvgPool2d(encoder_params[encoder]["features"])
+        self.dropout = Dropout(dropout_rate)
+        self.fc = Linear(encoder_params[encoder]["features"], 1)
+
+    def forward(self, x):
+        x = self.encoder.forward_features(x)
+        x = self.avg_pool(x).flatten(1)
+        x = self.dropout(x)
+        x = self.fc(x)
+        return x

predict/kernel_utils.py → kernel_utils.py

@@ -50,7 +50,7 @@ class VideoReader:
         frame_count = int(capture.get(cv2.CAP_PROP_FRAME_COUNT))
         if frame_count <= 0: return None
 
-        frame_idxs = np.linspace(0, frame_count - 1, num_frames, endpoint=True, dtype=np.int)
+        frame_idxs = np.linspace(0, frame_count - 1, num_frames, endpoint=True, dtype=np.int32)
         if jitter > 0:
             np.random.seed(seed)
             jitter_offsets = np.random.randint(-jitter, jitter, len(frame_idxs))
@@ -201,7 +201,7 @@ class FaceExtractor:
         self.video_read_fn = video_read_fn
         self.detector = MTCNN(margin=0, thresholds=[0.7, 0.8, 0.8], device="cuda")
 
-    def process_videos(self, videos):
+    def process_videos(self, video_path):
         videos_read = []
         frames_read = []
         frames = []
@@ -211,7 +211,8 @@ class FaceExtractor:
             # filename = filenames[video_idx]
             # video_path = os.path.join(input_dir, filename)
             # result = self.video_read_fn(video_path)
-        result = videos
+        result = self.video_read_fn(video_path)
+        # result = video
         # Error? Then skip this video.
 
         # Keep track of the original frames (need them later).
@@ -241,7 +242,7 @@ class FaceExtractor:
                     faces.append(crop)
                     scores.append(score)
 
-            frame_dict = {"video_idx": video_idx,
+            frame_dict = { #"video_idx": video_idx,
                             "frame_idx": my_idxs[i],
                             "frame_w": w,
                             "frame_h": h,
@@ -255,7 +256,7 @@ class FaceExtractor:
         """Convenience method for doing face extraction on a single video."""
         input_dir = os.path.dirname(video_path)
         filenames = [os.path.basename(video_path)]
-        return self.process_videos(input_dir, filenames, [0])
+        return self.process_videos(video_path)
 
 
 
@@ -320,7 +321,7 @@ def predict_on_video(face_extractor, video_path, videos, batch_size, input_size,
                     else:
                         pass
             if n > 0:
-                x = torch.tensor(x, device="cuda").float()
+                x = torch.tensor(x, device="cpu").float()
                 # Preprocess the images.
                 x = x.permute((0, 3, 1, 2))
                 for i in range(len(x)):
@@ -329,7 +330,7 @@ def predict_on_video(face_extractor, video_path, videos, batch_size, input_size,
                 with torch.no_grad():
                     preds = []
                     for model in models:
-                        y_pred = model(x[:n].half())
+                        y_pred = model(x[:n]) # 
                         y_pred = torch.sigmoid(y_pred.squeeze())
                         bpred = y_pred[:n].cpu().numpy()
                         preds.append(strategy(bpred))
@@ -354,5 +355,4 @@ def predict_on_video_set(face_extractor, videos, input_size, num_workers, test_d
 
     with ThreadPoolExecutor(max_workers=num_workers) as ex:
         predictions = ex.map(process_file, [1])
-    return list(predictions)
-
+    return list(predictions)

predict/app.py

@@ -1,68 +0,0 @@
-import gradio as gr
-import argparse
-import os
-import re
-import time
-
-import torch
-import pandas as pd
-
-import os, sys
-root_folder = os.path.abspath(
-    os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-)
-sys.path.append(root_folder)
-from kernel_utils import VideoReader, FaceExtractor, confident_strategy, predict_on_video_set
-from training.zoo.classifiers import DeepFakeClassifier
-
-    
-    
-def predict(video):
-    # video_index = int(video_index)
-    
-    frames_per_video = 32
-    video_reader = VideoReader()
-    video_read_fn = lambda x: video_reader.read_frames(x, num_frames=frames_per_video)
-    face_extractor = FaceExtractor(video_read_fn)
-    input_size = 380
-    strategy = confident_strategy
-    
-    # test_videos = sorted([x for x in os.listdir(args.test_dir) if x[-4:] == ".mp4"])[video_index]
-    # print(f"Predicting {video_index} videos")
-    predictions = predict_on_video_set(face_extractor=face_extractor, input_size=input_size, models=models,
-                                       strategy=strategy, frames_per_video=frames_per_video, videos=video,
-                                       num_workers=6, test_dir=args.test_dir)
-    return predictions
-
-def get_args_models():
-    parser = argparse.ArgumentParser("Predict test videos")
-    arg = parser.add_argument
-    arg('--weights-dir', type=str, default="weights", help="path to directory with checkpoints")
-    arg('--models', type=str, default='classifier_DeepFakeClassifier_tf_efficientnet_b7_ns_1_best_dice', help="checkpoint files")  # nargs='+',
-    arg('--test-dir', type=str, default='test_dataset', help="path to directory with videos")
-    arg('--output', type=str, required=False, help="path to output csv", default="submission.csv")
-    args = parser.parse_args()
-
-    models = []
-    # model_paths = [os.path.join(args.weights_dir, model) for model in args.models]
-    model_paths = [os.path.join(args.weights_dir, args.models)]
-    for path in model_paths:
-        model = DeepFakeClassifier(encoder="tf_efficientnet_b7_ns").to("cpu")
-        print("loading state dict {}".format(path))
-        checkpoint = torch.load(path, map_location="cpu")
-        state_dict = checkpoint.get("state_dict", checkpoint)
-        model.load_state_dict({re.sub("^module.", "", k): v for k, v in state_dict.items()}, strict=True)
-        model.eval()
-        del checkpoint
-        models.append(model.half())
-    return args, models
-
-if __name__ == '__main__':
-    global models, args
-    stime = time.time()
-    print("Elapsed:", time.time() - stime)
-    args, models = get_args_models()
-    
-    demo = gr.Interface(fn=predict, inputs="image", outputs="text")
-    demo.launch()  
-