first version

app.py

@@ -0,0 +1,230 @@
+import gradio as gr
+import pandas as pd
+import cv2
+import mediapipe as mp
+import os
+from statistics import mean
+# Record video
+# Save video?
+# Break video into images
+# Run facemesh on all images and save locations
+# Run exterme locations
+# Run analysis on those compare to the first frame
+
+# Run simple face mesh
+mp_face_mesh = mp.solutions.face_mesh
+mp_drawing = mp.solutions.drawing_utils
+drawing_spec = mp_drawing.DrawingSpec(thickness=2, circle_radius=3)
+
+global pupilLocation
+pupilLocation = pd.DataFrame()
+pupil_sizes =  []
+ExteremeDistanceLeftEye = pd.DataFrame()
+ExteremeDistanceRightEye = pd.DataFrame()
+
+def video_identity(video):
+    return video
+
+
+#demo = gr.Interface(video_identity,
+#                    gr.Video(shape = (1000,1000), source="webcam"),
+#                    "playable_video")
+
+def findIrisInFrame(image, counter):
+    global pupilLocation, pupil_sizes
+    #pupilLocation = pd.DataFrame()  # Make sure it is empty
+    with mp_face_mesh.FaceMesh(max_num_faces=1, refine_landmarks=True,
+                               static_image_mode=True,
+                               min_detection_confidence=0.45) as face_mesh:
+
+            results = face_mesh.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
+            if not results.multi_face_landmarks:
+                return None
+
+            annotated_image = image.copy()
+            for face_landmarks in results.multi_face_landmarks:
+                height, width, _ = annotated_image.shape
+                nose = [int(face_landmarks.landmark[168].x * width), int(face_landmarks.landmark[168].y * height)]
+                cv2.circle(annotated_image, (nose[0], nose[1]), 3, (0, 0, 255), -1)
+
+                leftIrisPoints = [474, 475, 476, 477]
+                rightIrisPoints = [469, 470, 471, 472]
+                # right, top, left, bottom
+
+                left_iris = []
+                for p in leftIrisPoints:
+                    point = [int(face_landmarks.landmark[p].x * width), int(face_landmarks.landmark[p].y * height)]
+                    left_iris.append(point)
+                    cv2.circle(annotated_image, point, 1, (255, 0, 255), 2)
+
+                right_iris = []
+                for p in rightIrisPoints:
+                    point = [int(face_landmarks.landmark[p].x * width), int(face_landmarks.landmark[p].y * height)]
+                    right_iris.append(point)
+                    cv2.circle(annotated_image, point, 1, (255, 0, 255), 2)
+
+                leftIris_leftside = (int(left_iris[2][0]), int(left_iris[2][1]))
+                leftIris_rightside = (int(left_iris[0][0]), int(left_iris[0][1]))
+                leftIris_top = (int(left_iris[1][0]), int(left_iris[1][1]))
+                leftIris_bottom = (int(left_iris[3][0]), int(left_iris[3][1]))
+                rightIris_leftside = (int(right_iris[2][0]), int(right_iris[2][1]))
+                rightIris_rightside = (int(right_iris[0][0]), int(right_iris[0][1]))
+                rightIris_top = (int(right_iris[1][0]), int(right_iris[1][1]))
+                rightIris_bottom = (int(right_iris[3][0]), int(right_iris[3][1]))
+
+                cv2.circle(annotated_image,
+                           (int((leftIris_leftside[0] + leftIris_rightside[0]) / 2),
+                            int((leftIris_top[1] + leftIris_bottom[1]) / 2)),
+                           # int(abs(leftIris_leftside[0] - leftIris_rightside[0])/2
+                           1,
+                           (0, 255, 255), 2)
+
+                cv2.circle(annotated_image,
+                           (int((rightIris_leftside[0] + rightIris_rightside[0]) / 2),
+                            int((rightIris_top[1] + rightIris_bottom[1]) / 2)),
+                           # int(abs(rightIris_leftside[0] - rightIris_rightside[0]) / 2
+                           1,
+                           (0, 255, 255), 2)
+
+                pupil_sizes.append(abs(leftIris_leftside[0] - leftIris_rightside[0]))
+                pupil_sizes.append(abs(rightIris_leftside[0] - rightIris_rightside[0]))
+
+                name = "frame%d.jpg" % counter
+                newRow = pd.Series([name,
+                                    leftIris_leftside[0] - nose[0],
+                                    leftIris_top[1] - nose[1],
+                                    leftIris_rightside[0] - nose[0],
+                                    leftIris_bottom[1] - nose[1],
+                                    rightIris_leftside[0] - nose[0],
+                                    rightIris_top[1] - nose[1],
+                                    rightIris_rightside[0] - nose[0],
+                                    rightIris_bottom[1] - nose[1]
+                                    ])
+                newRow = newRow.to_frame().T
+                pupilLocation = pd.concat([pupilLocation, newRow], axis=0, ignore_index=True)
+
+                return newRow
+
+
+
+def handleVideo(input_video):
+    global ExteremeDistanceLeftEye, ExteremeDistanceRightEye, pupilLocation, pupil_sizes
+    pupilLocation = pd.DataFrame() # Make sure it is empty to begin with
+    pupil_sizes =  []
+    vidcap = cv2.VideoCapture(input_video)
+    success, image = vidcap.read()
+    count = 0
+    if not os.path.exists('Images'):
+        os.makedirs('Images')
+    #os.chdir('Images')
+
+    # Slide video into frames and find iris in each frame
+    while success:
+        cv2.imwrite("Images/frame%d.jpg" % count, image)  # save frame as JPEG file
+        findIrisInFrame(image, count)
+        success, image = vidcap.read()
+        count += 1
+
+    print("file counter=", count)
+    # Convert pupilLocation to pupilDiff
+    pupilDiff = pupilLocation.copy()
+    pupilDiff = pupilDiff.drop(pupilDiff.columns[0], axis=1)  # Remove file name
+    for i in range(pupilDiff.shape[0] - 1):  # Calculate deltas
+        pupilDiff.loc[i + 1] = (pupilDiff.loc[i + 1] - pupilDiff.loc[0])
+    pupilDiff = pupilDiff.drop(0, axis=0) # Remove the first row
+    print(pupilDiff)
+    # Find extreme iris locations (images and measurements)
+
+    # Left eye
+    LeftEyeLookingRight = pd.to_numeric(pupilDiff[1]).idxmin()
+    LeftEyeLookingDown  = pd.to_numeric(pupilDiff[2]).idxmax()
+    LeftEyeLookingLeft  = pd.to_numeric(pupilDiff[3]).idxmax()
+    LeftEyeLookingUp    = pd.to_numeric(pupilDiff[4]).idxmin()
+
+    # Right eye
+    RightEyeLookingRight = pd.to_numeric(pupilDiff[5]).idxmin()
+    RightEyeLookingDown  = pd.to_numeric(pupilDiff[6]).idxmax()
+    RightEyeLookingLeft  = pd.to_numeric(pupilDiff[7]).idxmax()
+    RightEyeLookingUp    = pd.to_numeric(pupilDiff[8]).idxmin()
+
+    print("Left eye images = ", LeftEyeLookingRight, LeftEyeLookingDown, LeftEyeLookingLeft, LeftEyeLookingUp)
+    print("Right eye images = ", RightEyeLookingRight, RightEyeLookingDown, RightEyeLookingLeft, RightEyeLookingUp)
+
+    ExtermeImageLeftEye = list([cv2.cvtColor(cv2.imread("Images/frame%d.jpg" % LeftEyeLookingRight), cv2.COLOR_BGR2RGB),
+                                cv2.cvtColor(cv2.imread("Images/frame%d.jpg" % LeftEyeLookingDown), cv2.COLOR_BGR2RGB),
+                                cv2.cvtColor(cv2.imread("Images/frame%d.jpg" % LeftEyeLookingLeft), cv2.COLOR_BGR2RGB),
+                                cv2.cvtColor(cv2.imread("Images/frame%d.jpg" % LeftEyeLookingUp), cv2.COLOR_BGR2RGB)])
+
+    ExtermeImageRightEye = list([cv2.cvtColor(cv2.imread("Images/frame%d.jpg" % RightEyeLookingRight), cv2.COLOR_BGR2RGB),
+                                 cv2.cvtColor(cv2.imread("Images/frame%d.jpg" % RightEyeLookingDown), cv2.COLOR_BGR2RGB),
+                                 cv2.cvtColor(cv2.imread("Images/frame%d.jpg" % RightEyeLookingLeft), cv2.COLOR_BGR2RGB),
+                                 cv2.cvtColor(cv2.imread("Images/frame%d.jpg" % RightEyeLookingUp), cv2.COLOR_BGR2RGB)])
+
+
+    # return the distances
+    # Find average pupil size
+    for i in range(10):
+        pupil_sizes.remove(max(pupil_sizes))
+        pupil_sizes.remove(min(pupil_sizes))
+    pupil_average = mean(pupil_sizes)  # this should be 11.7 mm
+    pixels =  11.7 / pupil_average
+    print("pixels (In MM) = ", pixels)
+
+    d = { 'direction': ['Right', 'Down', 'Left', 'Up'] ,
+          'mm' : [round(pd.to_numeric(pupilDiff[1]).min() * pixels,1),
+                  round(pd.to_numeric(pupilDiff[2]).max() * pixels,1),
+                  round(pd.to_numeric(pupilDiff[3]).max() * pixels,1),
+                  round(pd.to_numeric(pupilDiff[4]).min() * pixels,1)
+                  ]}
+    ExteremeDistanceLeftEye = pd.DataFrame(data=d)
+
+    d = {'direction': ['Right', 'Down', 'Left', 'Up'],
+         'mm': [round(pd.to_numeric(pupilDiff[5]).min() * pixels, 1),
+                round(pd.to_numeric(pupilDiff[6]).max() * pixels, 1),
+                round(pd.to_numeric(pupilDiff[7]).max() * pixels, 1),
+                round(pd.to_numeric(pupilDiff[8]).min() * pixels, 1)
+                ]}
+    ExteremeDistanceRightEye = pd.DataFrame(data=d)
+
+
+
+    print() #.idxmax(axis=0))
+    # Upmost buttom limbus
+    #
+    return ExteremeDistanceLeftEye, ExteremeDistanceRightEye, ExtermeImageLeftEye, ExtermeImageRightEye
+
+with gr.Blocks() as demo:
+            gr.Markdown(
+            """
+            # Range of Motion Video Analysis
+            Capture a video of the following looks: stright, left, right, up & down
+            """)
+            video1 = gr.Video(shape = (1000,1000), source="webcam")
+
+            b = gr.Button("Analyze Video")
+
+            gr.Markdown(
+                """
+                # Left eye results (in mm):
+                """)
+            LeftEyeGallery = gr.Gallery(
+                label="Left eye", show_label=False, elem_id="left_eye_gallery"
+            ).style(grid=[4], height="auto")
+            movementDataLeft = gr.Dataframe(ExteremeDistanceLeftEye)
+
+
+            gr.Markdown(
+                """
+                # Right eye results (in mm):
+                """)
+            RightEyeGallery = gr.Gallery(
+                label="Right eye", show_label=False, elem_id="right_eye_gallery"
+            ).style(grid=[4], height="auto")
+            movementDataRight = gr.Dataframe(ExteremeDistanceRightEye)
+
+
+
+            out = [movementDataLeft, movementDataRight, LeftEyeGallery, RightEyeGallery]
+            b.click(fn=handleVideo, inputs=video1, outputs=out)
+
+demo.launch()

requirements.txt

@@ -0,0 +1,5 @@
+numpy
+pandas
+Pillow
+opencv-python-headless
+mediapipe