adding model

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+face_landmarker_v2_with_blendshapes.task filter=lfs diff=lfs merge=lfs -text

app.py

@@ -9,7 +9,9 @@ from mediapipe.tasks import python
 from mediapipe.tasks.python import vision
 from mediapipe.framework.formats import landmark_pb2
 from mediapipe import solutions
+from PIL import Image
 
+import torch, torchvision
 
 import matplotlib
 matplotlib.use("Agg")
@@ -17,6 +19,15 @@ import matplotlib.pyplot as plt
 
 cropped_image = []
 analyzed_image = []
+
+# colors for visualization
+COLORS = [[0.000, 0.447, 0.741], [0.850, 0.325, 0.098], [0.929, 0.694, 0.125],
+          [0.494, 0.184, 0.556], [0.466, 0.674, 0.188], [0.301, 0.745, 0.933]]
+     
+finetuned_classes = [
+      'iris',
+  ]
+
 # take a phone
 # run face landmark on it to crop image
 # run our model on it
@@ -30,6 +41,8 @@ options = vision.FaceLandmarkerOptions(base_options=base_options,
                                        num_faces=1)
 detector = vision.FaceLandmarker.create_from_options(options)
 
+model = torch.hub.load('facebookresearch/detr', 'detr_resnet50', pretrained=True)
+model.eval();
 
 def video_identity(video):
     return video
@@ -40,6 +53,15 @@ def video_identity(video):
 #                    "playable_video")
 
 
+import torchvision.transforms as T
+
+# standard PyTorch mean-std input image normalization
+transform = T.Compose([
+    T.Resize(800),
+    T.ToTensor(),
+    T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
+])
+
 def handle_image(input_image):
     global cropped_image, analyzed_image
     cv2.imwrite("image.jpg", input_image) 
@@ -63,15 +85,104 @@ def handle_image(input_image):
     cv2.circle(input_image, (p1[0], p1[1]), 10, (0, 0, 255), -1)
     p2 = [int(face_landmarks_proto.landmark[346].x * width), int(face_landmarks_proto.landmark[346].y * height)]
     cv2.circle(input_image, (p2[0], p2[1]), 10, (0, 0, 255), -1)
-    print(p1[0], p1[1], p2[0], p2[1], height, width)
     cropped_image = cropped_image[p1[1]:p2[1], p1[0]:p2[0]]
-    # [row starting from the top]
-    #return ([input_image, cropped_image])
+    
+    run_worflow(cropped_image, model)
+    
     return (cropped_image)
 
+def load_model():
+    print('load model')
+    '''
+    model = torch.hub.load('facebookresearch/detr',
+                       'detr_resnet50',
+                       pretrained=False,
+                       num_classes=1)
+
+    checkpoint = torch.load('outputs/checkpoint.pth',
+                            map_location='cpu')
+
+    model.load_state_dict(checkpoint['model'],
+                      strict=False)
+
+    model.eval();
+    '''
+def filter_bboxes_from_outputs(img,
+                               outputs,
+                               threshold=0.7
+                               ):
+  
+  # keep only predictions with confidence above threshold
+  probas = outputs['pred_logits'].softmax(-1)[0, :, :-1]
+  keep = probas.max(-1).values > threshold
+
+  probas_to_keep = probas[keep]
+
+  # convert boxes from [0; 1] to image scales
+  bboxes_scaled = rescale_bboxes(outputs['pred_boxes'][0, keep], img.size)
+  
+  return probas_to_keep, bboxes_scaled
+
+
+def plot_finetuned_results(pil_img, prob=None, boxes=None):
+    plt.figure(figsize=(16,10))
+    plt.imshow(pil_img)
+    ax = plt.gca()
+    colors = COLORS * 100
+    if prob is not None and boxes is not None:
+      for p, (xmin, ymin, xmax, ymax), c in zip(prob, boxes.tolist(), colors):
+          ax.add_patch(plt.Rectangle((xmin, ymin), xmax - xmin, ymax - ymin,
+                                    fill=False, color=c, linewidth=3))
+          cl = p.argmax()
+          #text = f'{finetuned_classes[cl]}: {p[cl]:0.2f}'
+          text = 'results'
+          ax.text(xmin, ymin, text, fontsize=15,
+                  bbox=dict(facecolor='yellow', alpha=0.5))
+    plt.axis('off')
+    plt.show()
+     
+def rescale_bboxes(out_bbox, size):
+    print (size)
+    img_w, img_h = size
+    b = box_cxcywh_to_xyxy(out_bbox)
+    b = b * torch.tensor([img_w, img_h, img_w, img_h], dtype=torch.float32)
+    return b     
+
+def box_cxcywh_to_xyxy(x):
+    x_c, y_c, w, h = x.unbind(1)
+    b = [(x_c - 0.5 * w), (y_c - 0.5 * h),
+         (x_c + 0.5 * w), (y_c + 0.5 * h)]
+    return torch.stack(b, dim=1)
+
+
+def run_worflow(my_image, my_model):
+  
+  # Write image to disk and read it as PIL !!!!
+  cv2.imwrite("img1.jpg", my_image)
+  my_image = Image.open("img1.jpg")
+ 
+  # mean-std normalize the input image (batch-size: 1)
+  img = transform(my_image).unsqueeze(0)
+
+  # propagate through the model
+  outputs = my_model(img)
+
+  for threshold in [0.2, 0.2]:
+    
+    probas_to_keep, bboxes_scaled = filter_bboxes_from_outputs(my_image,
+                                                               outputs,
+                                                              threshold=threshold)
+
+    plot_finetuned_results(my_image,
+                           probas_to_keep, 
+                           bboxes_scaled)
+
+
+
 
 
 with gr.Blocks() as demo:
+            
             gr.Markdown(
             """
             # Iris detection
@@ -94,6 +205,8 @@ with gr.Blocks() as demo:
             out = [cropped_image] 
             b.click(fn=handle_image, inputs=image1, outputs=out)
 
+            
+
 demo.launch()
 
 

requirements.txt

@@ -3,4 +3,7 @@ numpy
 pandas
 Pillow
 opencv-python
-mediapipe
+mediapipe
+torch
+torchvision
+scipy