Upload app4.py

app4.py

@@ -0,0 +1,424 @@
+import gradio as gr
+import pandas as pd
+import cv2
+import mediapipe as mp
+import os
+from statistics import mean
+import numpy as np
+from mediapipe.tasks import python
+from mediapipe.tasks.python import vision
+from mediapipe.framework.formats import landmark_pb2
+from mediapipe import solutions
+
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+
+# Record video
+# Save video?
+# Break video into images
+# Run facemesh on all images and save locations
+# Run exterme locations
+# Run analysis on those compare to the first frame
+
+# Create a FaceLandmarker object.
+base_options = python.BaseOptions(model_asset_path='face_landmarker_v2_with_blendshapes.task')
+options = vision.FaceLandmarkerOptions(base_options=base_options,
+                                       output_face_blendshapes=True,
+                                       output_facial_transformation_matrixes=True,
+                                       num_faces=1)
+detector = vision.FaceLandmarker.create_from_options(options)
+
+
+global pupilLocation
+pupilLocation = pd.DataFrame()
+pupil_sizes =  []
+ExteremeDistanceLeftEye = pd.DataFrame()
+ExteremeDistanceRightEye = pd.DataFrame()
+
+def video_identity(video):
+    return video
+
+# To do
+# 1. Filter out closed eye frames
+# 2. Smooth persuit from video POC
+
+def isEyeOpen(image):
+    image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+    hist = cv2.calcHist([image], [0], None, [256], [0, 256])
+    colors = np.where(hist > 10)
+
+    #print ("colors=", np.mean(colors) )
+    if np.mean(colors) > 15:
+        return True
+    else:
+        return False
+
+
+
+#demo = gr.Interface(video_identity,
+#                    gr.Video(shape = (1000,1000), source="webcam"),
+#                    "playable_video")
+
+def findIrisInFrame(image, counter):
+    global pupilLocation, pupil_sizes
+    #pupilLocation = pd.DataFrame()  # Make sure it is empty
+
+    image = mp.Image.create_from_file("image.jpg")
+
+    # STEP 4: Detect face landmarks from the input image.
+    detection_result = detector.detect(image)
+
+    # STEP 5: Process the detection result. In this case, visualize it.
+    #annotated_image = draw_landmarks_on_image(image.numpy_view(), detection_result)
+
+    annotated_image = image.numpy_view().copy()
+
+    face_landmarks_list = detection_result.face_landmarks
+
+    '''
+    # Loop through the detected faces to visualize.
+    for idx in range(len(face_landmarks_list)):
+        face_landmarks = face_landmarks_list[idx]
+
+        # Draw the face landmarks.
+        face_landmarks_proto = landmark_pb2.NormalizedLandmarkList()
+        face_landmarks_proto.landmark.extend([
+            landmark_pb2.NormalizedLandmark(x=landmark.x, y=landmark.y, z=landmark.z) for landmark in face_landmarks
+        ])
+        solutions.drawing_utils.draw_landmarks(
+            image=annotated_image,
+            landmark_list=face_landmarks_proto,
+            connections=mp.solutions.face_mesh.FACEMESH_IRISES,
+            landmark_drawing_spec=None,
+            connection_drawing_spec=mp.solutions.drawing_styles
+            .get_default_face_mesh_iris_connections_style())
+    '''
+    #FACEMESH_LEFT_IRIS = (474, 475, 476, 477)
+    #FACEMESH_RIGHT_IRIS = (469, 470, 471, 472)
+    #(lm_left_iris.x, lm_left_iris.y, lm_left_iris.z) = face_landmarks.landmark[468]
+    #(lm_right_iris.x, lm_right_iris.y, lm_right_iris.z) = face_landmarks.landmark[473]
+
+    # Draw the face landmarks.
+    face_landmarks = face_landmarks_list[0]
+    face_landmarks_proto = landmark_pb2.NormalizedLandmarkList()
+    face_landmarks_proto.landmark.extend([
+        landmark_pb2.NormalizedLandmark(x=landmark.x, y=landmark.y, z=landmark.z) for landmark in face_landmarks
+    ])
+
+    height, width, _ = annotated_image.shape
+    nose = [int(face_landmarks_proto.landmark[168].x * width), int(face_landmarks_proto.landmark[168].y * height)]
+    cv2.circle(annotated_image, (nose[0], nose[1]), 3, (0, 0, 255), -1)
+
+    leftIrisPoints = [474, 475, 476, 477]
+    rightIrisPoints = [469, 470, 471, 472]
+    # right, top, left, bottom
+
+    left_iris = []
+    for p in leftIrisPoints:
+                    point = [int(face_landmarks_proto.landmark[p].x * width), int(face_landmarks_proto.landmark[p].y * height)]
+                    left_iris.append(point)
+                    cv2.circle(annotated_image, point, 1, (255, 0, 255), 1)
+
+    right_iris = []
+    for p in rightIrisPoints:
+                    point = [int(face_landmarks_proto.landmark[p].x * width), int(face_landmarks_proto.landmark[p].y * height)]
+                    right_iris.append(point)
+                    cv2.circle(annotated_image, point, 1, (255, 0, 255), 1)
+
+    leftIris_leftside = (int(left_iris[2][0]), int(left_iris[2][1]))
+    leftIris_rightside = (int(left_iris[0][0]), int(left_iris[0][1]))
+    leftIris_top = (int(left_iris[1][0]), int(left_iris[1][1]))
+    leftIris_bottom = (int(left_iris[3][0]), int(left_iris[3][1]))
+    rightIris_leftside = (int(right_iris[2][0]), int(right_iris[2][1]))
+    rightIris_rightside = (int(right_iris[0][0]), int(right_iris[0][1]))
+    rightIris_top = (int(right_iris[1][0]), int(right_iris[1][1]))
+    rightIris_bottom = (int(right_iris[3][0]), int(right_iris[3][1]))
+
+    cv2.imwrite("Images/post-image-%d.jpg" % counter, annotated_image)  # save frame as JPEG file
+
+    '''
+                sizeIncrease = 0
+                leftEye = annotated_image[leftIris_top[1] - sizeIncrease: leftIris_bottom[1] + sizeIncrease,
+                          leftIris_leftside[0] - sizeIncrease: leftIris_rightside[0] + sizeIncrease]
+                leftEyeOpen = isEyeOpen(leftEye)
+
+                rightEye = annotated_image[rightIris_top[1] - sizeIncrease: rightIris_bottom[1] + sizeIncrease,
+                           rightIris_leftside[0] - sizeIncrease: rightIris_rightside[0] + sizeIncrease]
+                rightEyeOpen = isEyeOpen(rightEye)
+
+
+                cv2.circle(image,
+                           (int((leftIris_leftside[0] + leftIris_rightside[0]) / 2),
+                            int((leftIris_top[1] + leftIris_bottom[1]) / 2)),
+                           # int(abs(leftIris_leftside[0] - leftIris_rightside[0])/2),
+                           1,
+                           (0, 255, 255), 1)
+
+                cv2.circle(image,
+                           (int((rightIris_leftside[0] + rightIris_rightside[0]) / 2),
+                            int((rightIris_top[1] + rightIris_bottom[1]) / 2)),
+                           # int(abs(rightIris_leftside[0] - rightIris_rightside[0])/2),
+                           1,
+                           (0, 255, 255), 1)
+
+                cv2.putText(image, str(counter),
+                            (rightIris_leftside[0] - 100, leftIris_top[1] - 10), cv2.FONT_HERSHEY_SIMPLEX,
+                            1, (255, 0, 0), 1, cv2.LINE_AA)
+
+
+    '''
+    pupil_sizes.append(abs(leftIris_leftside[0] - leftIris_rightside[0]))
+    pupil_sizes.append(abs(rightIris_leftside[0] - rightIris_rightside[0]))
+
+    name = "frame%d.jpg" % counter
+    newRow = pd.Series([name,
+                                    leftIris_leftside[0]   - nose[0],
+                                    leftIris_top[1]        - nose[1],
+                                    leftIris_rightside[0]  - nose[0],
+                                    leftIris_bottom[1]     - nose[1],
+                                    rightIris_leftside[0]  - nose[0],
+                                    rightIris_top[1]       - nose[1],
+                                    rightIris_rightside[0] - nose[0],
+                                    rightIris_bottom[1]    - nose[1]
+                                    ])
+    newRow = newRow.to_frame().T
+                #if (leftEyeOpen & rightEyeOpen):
+    pupilLocation = pd.concat([pupilLocation, newRow], axis=0, ignore_index=True)
+                #else:
+                #    print("Ignored frame ", counter, "." , leftEyeOpen , rightEyeOpen)
+
+    return newRow
+
+
+def handleVideo(input_video):
+    global ExteremeDistanceLeftEye, ExteremeDistanceRightEye, pupilLocation, pupil_sizes
+
+    pupilLocation = pd.DataFrame() # Make sure it is empty to begin with
+    pupil_sizes =  []
+    vidcap = cv2.VideoCapture(input_video)
+    success, image = vidcap.read()
+    fps = vidcap.get(cv2.CAP_PROP_FPS)
+    frame_count = int(vidcap.get(cv2.CAP_PROP_FRAME_COUNT))
+    width = vidcap.get(cv2.CAP_PROP_FRAME_WIDTH)  # float `width`
+    height = vidcap.get(cv2.CAP_PROP_FRAME_HEIGHT)  # float `height`
+    print('FPS =', fps)
+    print('Frame count =', frame_count)
+    print('Resolution =', width, ' X ', height)
+
+    count = 0
+    if not os.path.exists('Images'):
+        os.makedirs('Images')
+    #os.chdir('Images')
+
+    # Slide video into frames and find iris in each frame
+    while success:
+        cv2.imwrite("image.jpg", image)  # save frame as JPEG file
+        print("Image#", count)
+        image = mp.Image.create_from_file("image.jpg")
+        findIrisInFrame(image, count)
+
+        #cv2.imwrite("Images/frame%d.jpg" % count, image)  # save frame as JPEG file
+        count += 1
+        success, image = vidcap.read()
+
+
+    # Go over all the pupils. If pupil is too small expand it in all directions
+    # Find average pupil size
+    pupil_average = 11.7
+    if (len(pupil_sizes) > 100):
+        for i in range(10):
+           pupil_sizes.remove(max(pupil_sizes))
+           pupil_sizes.remove(min(pupil_sizes))
+        pupil_average = mean(pupil_sizes)  # this should be 11.7 mm
+    print("pupil_average=", pupil_average)
+
+    # pupil size need to be kept constant in all pictures
+    # we find the center of the current pupil and make a circle around it in the size we need
+
+    for index, row in pupilLocation.iterrows():
+        currentLeftSize = abs(row[1] - row[3])
+        diffFromLeftAverage = pupil_average - currentLeftSize
+        currentRightSize = abs(row[5] - row[7])
+        diffFromAverage = pupil_average - currentRightSize
+
+        #print("(frame#", index, ") left: ", row[1], " right: ", row[3])
+        #if diffFromAverage > 2:
+        if (currentRightSize - currentLeftSize > 200):
+            print("Fixed Left pupil")
+            row[1] = int(row[1] - diffFromAverage / 2)
+            row[2] = int(row[2] - diffFromAverage / 2)
+            row[3] = int(row[3] + diffFromAverage / 2)
+            row[4] = int(row[4] + diffFromAverage / 2)
+
+        if (currentLeftSize - currentRightSize > 200):
+            print("Fixed Right pupil")
+            row[5] = int(row[5] - diffFromAverage / 2)
+            row[6] = int(row[6] - diffFromAverage / 2)
+            row[7] = int(row[7] + diffFromAverage / 2)
+            row[8] = int(row[8] + diffFromAverage / 2)
+
+
+    print("file counter=", count)
+    # Convert pupilLocation to pupilDiff
+    pupilDiff = pupilLocation.copy()
+    pupilDiff = pupilDiff.drop(pupilDiff.columns[0], axis=1)  # Remove file name
+    for i in range(pupilDiff.shape[0] - 1):  # Calculate deltas
+        pupilDiff.loc[i + 1] = (pupilDiff.loc[i + 1] - pupilDiff.loc[0])
+    pupilDiff = pupilDiff.drop(0, axis=0) # Remove the first row
+
+    # Find extreme iris locations (images and measurements)
+
+    #pupilDiff.columns = ['LL', 'LT', 'LR', 'LB', 'RR', 'RT', 'RR', 'RB']
+
+    # Take just the relevant libmus
+    #if [1] is positive, take it, otherwise take 3
+
+    #pupilDiff[9]  = [pupilDiff[1] if x >= 0 else pupilDiff[3] for x in pupilDiff[1]]
+
+    pupilDiff[9] = 0
+    pupilDiff[10] = 0
+
+    pupilDiff[9]  = np.where(pupilDiff[1] >= 0, pupilDiff[1], pupilDiff[3])
+                             #pupilDiff[[1,3]].max(axis=1), pupilDiff[[1,3]].min(axis=1))
+    pupilDiff[10] = np.where(pupilDiff[1] >= 0, pupilDiff[5], pupilDiff[7])
+                             #pupilDiff[[5,7]].max(axis=1), pupilDiff[[5,7]].min(axis=1))
+    pupilDiff[11] = np.where(pupilDiff[2] >= 0, pupilDiff[2], pupilDiff[4])
+                             #pupilDiff[[2,4]].max(axis=1), pupilDiff[[2,4]].min(axis=1))
+    pupilDiff[12] = np.where(pupilDiff[2] >= 0, pupilDiff[6], pupilDiff[8])
+                             #pupilDiff[[6,8]].max(axis=1), pupilDiff[[6,8]].min(axis=1))
+    print(pupilDiff[[1,3,5,7,9,10]])
+
+    # slope
+    x1 = (pupilLocation[1] + pupilLocation[3]) / 2
+    y1 = (pupilLocation[2] + pupilLocation[4]) / 2
+    x2 = (pupilLocation[5] + pupilLocation[7]) / 2
+    y2 = (pupilLocation[6] + pupilLocation[8]) / 2
+    pupilDiff[13] = ((y2 - y1) / (0.001 + x2 - x1))
+
+
+    pupilDiff.to_csv('pupil_diff.csv')
+
+
+    pixels = 11.7 / pupil_average
+    print("pixels (In MM) = ", pixels)
+
+
+    pupilDiff = round(pupilDiff * pixels,3)
+
+    fig1 = plt.figure()
+    plt.plot(pupilDiff[[9,10]]) #1,3,5,7
+    plt.title("Pupil movement X axis")
+    plt.ylabel("MM of movement")
+    plt.xlabel("Frame")
+    plt.ylim(-10, 10)
+    plt.legend(['Left', 'Right']) #'LL', 'LR', 'RL', 'RR'])
+
+    fig2 = plt.figure()
+    plt.plot(pupilDiff[[11,12]]) #, df[countries].to_numpy())
+    plt.ylim(-10, 10)
+    plt.title("Pupil movement Y axis")
+    plt.ylabel("MM of movement")
+    plt.xlabel("Frame")
+    plt.legend(['Left', 'Right'])
+
+
+    # Left eye
+    LeftEyeLookingRight = pd.to_numeric(pupilDiff[1]).idxmax()
+    LeftEyeLookingDown  = pd.to_numeric(pupilDiff[2]).idxmax()
+    LeftEyeLookingLeft  = pd.to_numeric(pupilDiff[3]).idxmin()
+    LeftEyeLookingUp    = pd.to_numeric(pupilDiff[4]).idxmin()
+
+
+    # Right eye
+    RightEyeLookingRight = pd.to_numeric(pupilDiff[5]).idxmax()
+    RightEyeLookingDown  = pd.to_numeric(pupilDiff[6]).idxmax()
+    RightEyeLookingLeft  = pd.to_numeric(pupilDiff[7]).idxmin()
+    RightEyeLookingUp    = pd.to_numeric(pupilDiff[8]).idxmin()
+
+
+    print("Left eye images = ", LeftEyeLookingRight, LeftEyeLookingDown, LeftEyeLookingLeft, LeftEyeLookingUp)
+    print("Right eye images = ", RightEyeLookingRight, RightEyeLookingDown, RightEyeLookingLeft, RightEyeLookingUp)
+
+    ExtermeImageLeftEye = list([cv2.cvtColor(cv2.imread("Images/post-image-%d.jpg" % LeftEyeLookingRight), cv2.COLOR_BGR2RGB),
+                                cv2.cvtColor(cv2.imread("Images/post-image-%d.jpg" % LeftEyeLookingLeft), cv2.COLOR_BGR2RGB),
+                                cv2.cvtColor(cv2.imread("Images/post-image-%d.jpg" % LeftEyeLookingUp), cv2.COLOR_BGR2RGB),
+                                cv2.cvtColor(cv2.imread("Images/post-image-%d.jpg" % LeftEyeLookingDown), cv2.COLOR_BGR2RGB)])
+
+    ExtermeImageRightEye = list([cv2.cvtColor(cv2.imread("Images/post-image-%d.jpg" % RightEyeLookingRight), cv2.COLOR_BGR2RGB),
+                                 cv2.cvtColor(cv2.imread("Images/post-image-%d.jpg" % RightEyeLookingLeft), cv2.COLOR_BGR2RGB),
+                                 cv2.cvtColor(cv2.imread("Images/post-image-%d.jpg" % RightEyeLookingUp), cv2.COLOR_BGR2RGB),
+                                 cv2.cvtColor(cv2.imread("Images/post-image-%d.jpg" % RightEyeLookingDown), cv2.COLOR_BGR2RGB)])
+
+
+    # return the distances
+
+
+    d = { 'direction': ['Right', 'Left', 'Up', 'Down'] ,
+          'mm' : [abs(round(pd.to_numeric(pupilDiff[1]).max(),1)),
+                  abs(round(pd.to_numeric(pupilDiff[3]).min(),1)),
+                  abs(round(pd.to_numeric(pupilDiff[4]).min(),1)),
+                  abs(round(pd.to_numeric(pupilDiff[2]).max(),1))
+                  ]}
+    ExteremeDistanceLeftEye = pd.DataFrame(data=d)
+
+    d = {'direction': ['Right', 'Left', 'Up', 'Down'],
+         'mm': [abs(round(pd.to_numeric(pupilDiff[5]).max(), 1)),
+                abs(round(pd.to_numeric(pupilDiff[7]).min(), 1)),
+                abs(round(pd.to_numeric(pupilDiff[8]).min(), 1)),
+                abs(round(pd.to_numeric(pupilDiff[6]).max(), 1))
+                ]}
+    ExteremeDistanceRightEye = pd.DataFrame(data=d)
+
+
+
+    print() #.idxmax(axis=0))
+    # Upmost buttom limbus
+    #
+    return ExteremeDistanceLeftEye, ExteremeDistanceRightEye, ExtermeImageLeftEye, ExtermeImageRightEye, fig1, fig2 # lines
+
+
+with gr.Blocks() as demo:
+            gr.Markdown(
+            """
+            # Range of Motion Video Analysis
+            Capture a video of the following looks: stright, left, right, up & down
+            """)
+            video1 = gr.Video()#source="webcam")
+
+            b = gr.Button("Analyze Video")
+
+            gr.Markdown(
+                """
+                # Left eye results (in mm):
+                """)
+            LeftEyeGallery = gr.Gallery(
+                label="Left eye", show_label=False, elem_id="left_eye_gallery"
+            ).style(grid=[4], height="auto")
+            movementDataLeft = gr.Dataframe(ExteremeDistanceLeftEye)
+
+
+            gr.Markdown(
+                """
+                # Right eye results (in mm):
+                """)
+            RightEyeGallery = gr.Gallery(
+                label="Right eye", show_label=False, elem_id="right_eye_gallery"
+            ).style(grid=[4], height="auto")
+            movementDataRight = gr.Dataframe(ExteremeDistanceRightEye)
+
+            plot1 = gr.Plot(label="Plot1")
+
+            plot2 = gr.Plot(label="Plot2")
+
+            out = [movementDataLeft, movementDataRight, LeftEyeGallery, RightEyeGallery, plot1, plot2]
+            b.click(fn=handleVideo, inputs=video1, outputs=out)
+
+demo.launch()
+
+
+
+
+
+