Add application file

app.py

@@ -0,0 +1,232 @@
+import gradio as gr
+import mediapipe as mp
+import cv2
+import pandas as pd
+from statistics import mean
+
+# Run simple face mesh
+mp_face_mesh = mp.solutions.face_mesh
+mp_drawing = mp.solutions.drawing_utils
+drawing_spec = mp_drawing.DrawingSpec(thickness=2, circle_radius=3)
+
+global pupilLocation, movementLeft, movementRight
+pupilLocation = pd.DataFrame()
+movementLeft = pd.DataFrame(index=['Up', 'Center', 'Down'], columns=['Left', 'Center', 'Right'])
+movementRight = pd.DataFrame(index=['Up', 'Center', 'Down'], columns=['Left', 'Center', 'Right'])
+
+# TO DO:
+# 1. Calibration screen
+
+
+def findIris(input_img1, input_img2, input_img3, input_img4, input_img5):
+    global pupilLocation
+    pupilLocation = pd.DataFrame() # Make sure it is empty
+    images = [input_img1, input_img2, input_img3, input_img4, input_img5]
+    output_images = []
+    pupil_sizes = []
+    with mp_face_mesh.FaceMesh(max_num_faces=1, refine_landmarks=True,
+                               static_image_mode=True,
+                               min_detection_confidence=0.45) as face_mesh:
+        for image in images:
+            if image is None:
+                continue
+
+            results = face_mesh.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
+            if not results.multi_face_landmarks:
+                continue
+
+            annotated_image = image.copy()
+            for face_landmarks in results.multi_face_landmarks:
+                    height, width, _ = annotated_image.shape
+                    nose = [int(face_landmarks.landmark[168].x * width), int(face_landmarks.landmark[168].y * height)]
+                    cv2.circle(annotated_image, (nose[0], nose[1]), 3, (0, 0, 255), -1)
+
+                    leftIrisPoints = [474, 475, 476, 477]
+                    rightIrisPoints = [469, 470, 471, 472]
+                    # right, top, left, bottom
+
+                    left_iris = []
+                    for p in leftIrisPoints:
+                        point = [int(face_landmarks.landmark[p].x * width), int(face_landmarks.landmark[p].y * height)]
+                        left_iris.append(point)
+                        cv2.circle(annotated_image, point, 1, (255, 0, 255), 2)
+
+                    right_iris = []
+                    for p in rightIrisPoints:
+                        point = [int(face_landmarks.landmark[p].x * width), int(face_landmarks.landmark[p].y * height)]
+                        right_iris.append(point)
+                        cv2.circle(annotated_image, point, 1, (255, 0, 255), 2)
+
+                    leftIris_leftside = (int(left_iris[2][0]), int(left_iris[2][1]))
+                    leftIris_rightside = (int(left_iris[0][0]), int(left_iris[0][1]))
+                    leftIris_top = (int(left_iris[1][0]), int(left_iris[1][1]))
+                    leftIris_bottom = (int(left_iris[3][0]), int(left_iris[3][1]))
+                    rightIris_leftside = (int(right_iris[2][0]), int(right_iris[2][1]))
+                    rightIris_rightside = (int(right_iris[0][0]), int(right_iris[0][1]))
+                    rightIris_top = (int(right_iris[1][0]), int(right_iris[1][1]))
+                    rightIris_bottom = (int(right_iris[3][0]), int(right_iris[3][1]))
+
+                    '''
+                    cv2.circle(annotated_image,
+                               (int((leftIris_leftside[0] + leftIris_rightside[0]) / 2),
+                                int((leftIris_top[1] + leftIris_bottom[1]) / 2)),
+                               # int(abs(leftIris_leftside[0] - leftIris_rightside[0])/2
+                               1,
+                               (0, 255, 255), 2)
+
+                    cv2.circle(annotated_image,
+                               (int((rightIris_leftside[0] + rightIris_rightside[0]) / 2),
+                                int((rightIris_top[1] + rightIris_bottom[1]) / 2)),
+                               # int(abs(rightIris_leftside[0] - rightIris_rightside[0]) / 2
+                               1,
+                               (0, 255, 255), 2)
+                    '''
+
+                    left = leftIris_leftside[0] - 150
+                    right = rightIris_rightside[0] + 150
+                    up = leftIris_top[1] - 50
+                    down = leftIris_bottom[1] + 50
+                    annotated_image = annotated_image[up:down, left:right]
+
+                    name = 'TBD'
+                    newRow = pd.Series([name,
+                                        leftIris_leftside[0] - nose[0],
+                                        leftIris_top[1] - nose[1],
+                                        leftIris_rightside[0] - nose[0],
+                                        leftIris_bottom[1] - nose[1],
+                                        rightIris_leftside[0] - nose[0],
+                                        rightIris_top[1] - nose[1],
+                                        rightIris_rightside[0] - nose[0],
+                                        rightIris_bottom[1] - nose[1]
+                                        ])
+                    newRow = newRow.to_frame().T
+                    pupilLocation = pd.concat([pupilLocation, newRow], axis=0, ignore_index=True)
+                    #print("Inside pupil Location = ", pupilLocation)
+                    #filename = directoy_name + 'Analysis/' + name[0:-4] + '-analysis.jpg'
+                    #cv2.imwrite(filename, annotated_image)
+
+                    x1 = (leftIris_leftside[0] - nose[0] + leftIris_rightside[0] - nose[0]) / 2
+                    y1 = (leftIris_top[1] - nose[1] + leftIris_bottom[1] - nose[1]) / 2
+                    x2 = (rightIris_leftside[0] - nose[0] + rightIris_rightside[0] - nose[0]) / 2
+                    y2 = (rightIris_top[1] - nose[1] + rightIris_bottom[1] - nose[1]) / 2
+                    print("Slope=", (y2 - y1) / (x2 - x1))
+                    text = "Slope=" + str(round((y2 - y1) / (x2 - x1), 2))
+                    cv2.putText(annotated_image, text,
+                                (5, 110), cv2.FONT_HERSHEY_SIMPLEX,
+                                1, (255, 255, 0), 1, cv2.LINE_AA)
+
+                    print("left iris size in pixels = ", abs(leftIris_leftside[0] - leftIris_rightside[0]))
+                    print("Right iris size in pixels = ", abs(rightIris_leftside[0] - rightIris_rightside[0]))
+                    pupil_sizes.append(abs(leftIris_leftside[0] - leftIris_rightside[0]))
+                    pupil_sizes.append(abs(rightIris_leftside[0] - rightIris_rightside[0]))
+
+                    output_images.append(annotated_image)
+
+    # calculate final results from pupilLocations
+    pupilDiff = pupilLocation.copy()
+    pupilDiff = pupilDiff.drop(pupilDiff.columns[0], axis=1)  # Remove file name
+    for i in range(pupilDiff.shape[0] - 1): # Calculate deltas
+        pupilDiff.loc[i + 1] = abs(pupilDiff.loc[i + 1] - pupilDiff.loc[0])
+    print("pupilDiff=", pupilDiff)
+    pupilDiff = pupilDiff.drop(0, axis=0)   # Remove first row was was used as reference row
+    #print("pupilDiff (in pixels)=", pupilDiff)
+
+    # Find average pupil size
+    pupil_sizes.remove(max(pupil_sizes))
+    pupil_sizes.remove(min(pupil_sizes))
+    pupil_average = mean(pupil_sizes) # this should be 11.7 mm
+    pixels =  11.7 / pupil_average
+    print("pixels (In MM) = ", pixels)
+
+    # Left Eye movement
+    movementLeft.iloc[0, 0] = ' '
+    movementLeft.iloc[0, 2] = ' '
+    movementLeft.iloc[1, 1] = 0  # reference point
+    movementLeft.iloc[2, 0] = ' '
+    movementLeft.iloc[2, 2] = ' '
+
+    # Y movement only
+    movementLeft.iloc[0, 1] = round(abs(pupilLocation.iloc[0, 4] - pupilLocation.iloc[1, 4]) * pixels, 0) # Up
+    movementLeft.iloc[2, 1] = round(abs(pupilLocation.iloc[0, 2] - pupilLocation.iloc[3, 2]) * pixels, 0) # Down
+
+    # X movement only
+    movementLeft.iloc[1, 0] = round(abs(pupilLocation.iloc[0, 3] - pupilLocation.iloc[1, 3]) * pixels, 1) # Left
+    movementLeft.iloc[1, 2] = round(abs(pupilLocation.iloc[0, 1] - pupilLocation.iloc[2, 1]) * pixels, 1) # Right
+
+
+    # Right Eye Movement
+    movementRight.iloc[0, 0] = ' '
+    movementRight.iloc[0, 2] = ' '
+    movementRight.iloc[1, 1] = 0  # reference point
+    movementRight.iloc[2, 0] = ' '
+    movementRight.iloc[2, 2] = ' '
+
+    # Y movement only
+    movementRight.iloc[0, 1] = round(abs(pupilLocation.iloc[0, 8] - pupilLocation.iloc[1, 8]) * pixels, 0) # Up
+    movementRight.iloc[2, 1] = round(abs(pupilLocation.iloc[0, 6] - pupilLocation.iloc[3, 6]) * pixels, 0) # Down
+
+    # X movement only
+    movementRight.iloc[1, 0] = round(abs(pupilLocation.iloc[0, 7] - pupilLocation.iloc[1, 7]) * pixels, 0) # Left
+    movementRight.iloc[1, 2] = round(abs(pupilLocation.iloc[0, 5] - pupilLocation.iloc[2, 5]) * pixels, 0) # Right
+
+    return output_images[0], output_images[1], output_images[2], output_images[3], output_images[4], pupilLocation, movementLeft, movementRight
+
+
+with gr.Blocks() as demo:
+            gr.Markdown(
+            """
+            # Range of Motion Image Analysis
+            Take 5 pictures below looking stright, left, right, up & down
+            """)
+            with gr.Row():
+                with gr.Column(scale=1):
+                    img1 = gr.Image(shape=(1000, 1000), source='webcam', label='Front')
+                with gr.Column(scale=1):
+                    out1 = gr.Image(label='Out-Front')
+            with gr.Row():
+                with gr.Column(scale=1):
+                    img2 = gr.Image(shape=(1000, 1000), source='webcam', label='Left')
+                with gr.Column(scale=1):
+                    out2 = gr.Image(label='Out-Left')
+            with gr.Row():
+                with gr.Column(scale=1):
+                    img3 = gr.Image(shape=(1000, 1000), source='webcam', label='Right')
+                with gr.Column(scale=1):
+                    out3 = gr.Image(label='Out-Right')
+            with gr.Row():
+                with gr.Column(scale=1):
+                    img4 = gr.Image(shape=(1000, 1000), source='webcam', label='Up')
+                with gr.Column(scale=1):
+                    out4 = gr.Image(label='Out-Up')
+            with gr.Row():
+                with gr.Column(scale=1):
+                    img5 = gr.Image(shape=(1000, 1000), source='webcam', label='Down')
+                with gr.Column(scale=1):
+                    out5 = gr.Image(label='Down-Right')
+
+            b = gr.Button("Go!")
+
+            gr.Markdown(
+                """
+                Pupil Locations:
+                """)
+            pupilData = gr.Dataframe(pupilLocation)
+
+            gr.Markdown(
+                """
+                # Left eye results (in mm):
+                """)
+            movementDataLeft = gr.Dataframe(movementLeft)
+
+            gr.Markdown(
+                """
+                # Right eye results (in mm):
+                """)
+            movementDataRight = gr.Dataframe(movementRight)
+
+            inp = [img1, img2, img3, img4, img5]
+            out = [out1, out2, out3, out4, out5, pupilData, movementDataLeft, movementDataRight]
+            b.click(fn=findIris, inputs=inp, outputs=out)
+
+
+demo.launch(auth=("Andrew", "Andrew"), share=True)