app.py

import gradio as gr
import pandas as pd
import cv2
import mediapipe as mp
import os
from statistics import mean
import numpy as np
from mediapipe.tasks import python
from mediapipe.tasks.python import vision
from mediapipe.framework.formats import landmark_pb2
from mediapipe import solutions


import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt

# Record video
# Save video?
# Break video into images
# Run facemesh on all images and save locations
# Run exterme locations
# Run analysis on those compare to the first frame

# Create a FaceLandmarker object.
base_options = python.BaseOptions(model_asset_path='face_landmarker_v2_with_blendshapes.task')
options = vision.FaceLandmarkerOptions(base_options=base_options,
                                       output_face_blendshapes=True,
                                       output_facial_transformation_matrixes=True,
                                       num_faces=1)
detector = vision.FaceLandmarker.create_from_options(options)


global pupilLocation
pupilLocation = pd.DataFrame()
pupil_sizes =  []
ExteremeDistanceLeftEye = pd.DataFrame()
ExteremeDistanceRightEye = pd.DataFrame()

def video_identity(video):
    return video

# To do
# 1. Filter out closed eye frames
# 2. Smooth persuit from video POC

def isEyeOpen(image):
    image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    hist = cv2.calcHist([image], [0], None, [256], [0, 256])
    colors = np.where(hist > 10)

    #print ("colors=", np.mean(colors) )
    if np.mean(colors) > 15:
        return True
    else:
        return False



#demo = gr.Interface(video_identity,
#                    gr.Video(shape = (1000,1000), source="webcam"),
#                    "playable_video")

def findIrisInFrame(image, counter):
    global pupilLocation, pupil_sizes
    #pupilLocation = pd.DataFrame()  # Make sure it is empty

    image = mp.Image.create_from_file("image.jpg")

    # STEP 4: Detect face landmarks from the input image.
    detection_result = detector.detect(image)

    # STEP 5: Process the detection result. In this case, visualize it.
    #annotated_image = draw_landmarks_on_image(image.numpy_view(), detection_result)

    annotated_image = image.numpy_view().copy()

    face_landmarks_list = detection_result.face_landmarks

    '''
    # Loop through the detected faces to visualize.
    for idx in range(len(face_landmarks_list)):
        face_landmarks = face_landmarks_list[idx]

        # Draw the face landmarks.
        face_landmarks_proto = landmark_pb2.NormalizedLandmarkList()
        face_landmarks_proto.landmark.extend([
            landmark_pb2.NormalizedLandmark(x=landmark.x, y=landmark.y, z=landmark.z) for landmark in face_landmarks
        ])
        solutions.drawing_utils.draw_landmarks(
            image=annotated_image,
            landmark_list=face_landmarks_proto,
            connections=mp.solutions.face_mesh.FACEMESH_IRISES,
            landmark_drawing_spec=None,
            connection_drawing_spec=mp.solutions.drawing_styles
            .get_default_face_mesh_iris_connections_style())
    '''
    #FACEMESH_LEFT_IRIS = (474, 475, 476, 477)
    #FACEMESH_RIGHT_IRIS = (469, 470, 471, 472)
    #(lm_left_iris.x, lm_left_iris.y, lm_left_iris.z) = face_landmarks.landmark[468]
    #(lm_right_iris.x, lm_right_iris.y, lm_right_iris.z) = face_landmarks.landmark[473]

    # Draw the face landmarks.
    face_landmarks = face_landmarks_list[0]
    face_landmarks_proto = landmark_pb2.NormalizedLandmarkList()
    face_landmarks_proto.landmark.extend([
        landmark_pb2.NormalizedLandmark(x=landmark.x, y=landmark.y, z=landmark.z) for landmark in face_landmarks
    ])

    height, width, _ = annotated_image.shape
    nose = [int(face_landmarks_proto.landmark[168].x * width), int(face_landmarks_proto.landmark[168].y * height)]
    cv2.circle(annotated_image, (nose[0], nose[1]), 3, (0, 0, 255), -1)

    leftIrisPoints = [474, 475, 476, 477]
    rightIrisPoints = [469, 470, 471, 472]
    # right, top, left, bottom

    left_iris = []
    for p in leftIrisPoints:
                    point = [int(face_landmarks_proto.landmark[p].x * width), int(face_landmarks_proto.landmark[p].y * height)]
                    left_iris.append(point)
                    cv2.circle(annotated_image, point, 1, (255, 0, 255), 1)

    right_iris = []
    for p in rightIrisPoints:
                    point = [int(face_landmarks_proto.landmark[p].x * width), int(face_landmarks_proto.landmark[p].y * height)]
                    right_iris.append(point)
                    cv2.circle(annotated_image, point, 1, (255, 0, 255), 1)

    leftIris_leftside = (int(left_iris[2][0]), int(left_iris[2][1]))
    leftIris_rightside = (int(left_iris[0][0]), int(left_iris[0][1]))
    leftIris_top = (int(left_iris[1][0]), int(left_iris[1][1]))
    leftIris_bottom = (int(left_iris[3][0]), int(left_iris[3][1]))
    rightIris_leftside = (int(right_iris[2][0]), int(right_iris[2][1]))
    rightIris_rightside = (int(right_iris[0][0]), int(right_iris[0][1]))
    rightIris_top = (int(right_iris[1][0]), int(right_iris[1][1]))
    rightIris_bottom = (int(right_iris[3][0]), int(right_iris[3][1]))

    cv2.imwrite("Images/post-image-%d.jpg" % counter, annotated_image)  # save frame as JPEG file

    '''
                sizeIncrease = 0
                leftEye = annotated_image[leftIris_top[1] - sizeIncrease: leftIris_bottom[1] + sizeIncrease,
                          leftIris_leftside[0] - sizeIncrease: leftIris_rightside[0] + sizeIncrease]
                leftEyeOpen = isEyeOpen(leftEye)

                rightEye = annotated_image[rightIris_top[1] - sizeIncrease: rightIris_bottom[1] + sizeIncrease,
                           rightIris_leftside[0] - sizeIncrease: rightIris_rightside[0] + sizeIncrease]
                rightEyeOpen = isEyeOpen(rightEye)


                cv2.circle(image,
                           (int((leftIris_leftside[0] + leftIris_rightside[0]) / 2),
                            int((leftIris_top[1] + leftIris_bottom[1]) / 2)),
                           # int(abs(leftIris_leftside[0] - leftIris_rightside[0])/2),
                           1,
                           (0, 255, 255), 1)

                cv2.circle(image,
                           (int((rightIris_leftside[0] + rightIris_rightside[0]) / 2),
                            int((rightIris_top[1] + rightIris_bottom[1]) / 2)),
                           # int(abs(rightIris_leftside[0] - rightIris_rightside[0])/2),
                           1,
                           (0, 255, 255), 1)

                cv2.putText(image, str(counter),
                            (rightIris_leftside[0] - 100, leftIris_top[1] - 10), cv2.FONT_HERSHEY_SIMPLEX,
                            1, (255, 0, 0), 1, cv2.LINE_AA)


    '''
    pupil_sizes.append(abs(leftIris_leftside[0] - leftIris_rightside[0]))
    pupil_sizes.append(abs(rightIris_leftside[0] - rightIris_rightside[0]))

    name = "frame%d.jpg" % counter
    newRow = pd.Series([name,
                                    leftIris_leftside[0]   - nose[0],
                                    leftIris_top[1]        - nose[1],
                                    leftIris_rightside[0]  - nose[0],
                                    leftIris_bottom[1]     - nose[1],
                                    rightIris_leftside[0]  - nose[0],
                                    rightIris_top[1]       - nose[1],
                                    rightIris_rightside[0] - nose[0],
                                    rightIris_bottom[1]    - nose[1]
                                    ])
    newRow = newRow.to_frame().T
                #if (leftEyeOpen & rightEyeOpen):
    pupilLocation = pd.concat([pupilLocation, newRow], axis=0, ignore_index=True)
                #else:
                #    print("Ignored frame ", counter, "." , leftEyeOpen , rightEyeOpen)

    return newRow


def handleVideo(input_video):
    global ExteremeDistanceLeftEye, ExteremeDistanceRightEye, pupilLocation, pupil_sizes

    pupilLocation = pd.DataFrame() # Make sure it is empty to begin with
    pupil_sizes =  []
    vidcap = cv2.VideoCapture(input_video)
    success, image = vidcap.read()
    fps = vidcap.get(cv2.CAP_PROP_FPS)
    frame_count = int(vidcap.get(cv2.CAP_PROP_FRAME_COUNT))
    width = vidcap.get(cv2.CAP_PROP_FRAME_WIDTH)  # float `width`
    height = vidcap.get(cv2.CAP_PROP_FRAME_HEIGHT)  # float `height`
    print('FPS =', fps)
    print('Frame count =', frame_count)
    print('Resolution =', width, ' X ', height)

    count = 0
    if not os.path.exists('Images'):
        os.makedirs('Images')
    #os.chdir('Images')

    # Slide video into frames and find iris in each frame
    while success:
        cv2.imwrite("image.jpg", image)  # save frame as JPEG file
        print("Image#", count)
        image = mp.Image.create_from_file("image.jpg")
        findIrisInFrame(image, count)

        #cv2.imwrite("Images/frame%d.jpg" % count, image)  # save frame as JPEG file
        count += 1
        success, image = vidcap.read()


    # Go over all the pupils. If pupil is too small expand it in all directions
    # Find average pupil size
    pupil_average = 11.7
    if (len(pupil_sizes) > 100):
        for i in range(10):
           pupil_sizes.remove(max(pupil_sizes))
           pupil_sizes.remove(min(pupil_sizes))
        pupil_average = mean(pupil_sizes)  # this should be 11.7 mm
    print("pupil_average=", pupil_average)

    # pupil size need to be kept constant in all pictures
    # we find the center of the current pupil and make a circle around it in the size we need

    for index, row in pupilLocation.iterrows():
        currentLeftSize = abs(row[1] - row[3])
        diffFromLeftAverage = pupil_average - currentLeftSize
        currentRightSize = abs(row[5] - row[7])
        diffFromAverage = pupil_average - currentRightSize

        #print("(frame#", index, ") left: ", row[1], " right: ", row[3])
        #if diffFromAverage > 2:
        if (currentRightSize - currentLeftSize > 200):
            print("Fixed Left pupil")
            row[1] = int(row[1] - diffFromAverage / 2)
            row[2] = int(row[2] - diffFromAverage / 2)
            row[3] = int(row[3] + diffFromAverage / 2)
            row[4] = int(row[4] + diffFromAverage / 2)

        if (currentLeftSize - currentRightSize > 200):
            print("Fixed Right pupil")
            row[5] = int(row[5] - diffFromAverage / 2)
            row[6] = int(row[6] - diffFromAverage / 2)
            row[7] = int(row[7] + diffFromAverage / 2)
            row[8] = int(row[8] + diffFromAverage / 2)


    print("file counter=", count)
    # Convert pupilLocation to pupilDiff
    pupilDiff = pupilLocation.copy()
    pupilDiff = pupilDiff.drop(pupilDiff.columns[0], axis=1)  # Remove file name
    for i in range(pupilDiff.shape[0] - 1):  # Calculate deltas
        pupilDiff.loc[i + 1] = (pupilDiff.loc[i + 1] - pupilDiff.loc[0])
    pupilDiff = pupilDiff.drop(0, axis=0) # Remove the first row

    # Find extreme iris locations (images and measurements)

    #pupilDiff.columns = ['LL', 'LT', 'LR', 'LB', 'RR', 'RT', 'RR', 'RB']

    # Take just the relevant libmus
    #if [1] is positive, take it, otherwise take 3

    #pupilDiff[9]  = [pupilDiff[1] if x >= 0 else pupilDiff[3] for x in pupilDiff[1]]

    pupilDiff[9] = 0
    pupilDiff[10] = 0

    pupilDiff[9]  = np.where(pupilDiff[1] >= 0, pupilDiff[1], pupilDiff[3])
                             #pupilDiff[[1,3]].max(axis=1), pupilDiff[[1,3]].min(axis=1))
    pupilDiff[10] = np.where(pupilDiff[1] >= 0, pupilDiff[5], pupilDiff[7])
                             #pupilDiff[[5,7]].max(axis=1), pupilDiff[[5,7]].min(axis=1))
    pupilDiff[11] = np.where(pupilDiff[2] >= 0, pupilDiff[2], pupilDiff[4])
                             #pupilDiff[[2,4]].max(axis=1), pupilDiff[[2,4]].min(axis=1))
    pupilDiff[12] = np.where(pupilDiff[2] >= 0, pupilDiff[6], pupilDiff[8])
                             #pupilDiff[[6,8]].max(axis=1), pupilDiff[[6,8]].min(axis=1))
    print(pupilDiff[[1,3,5,7,9,10]])

    # slope
    x1 = (pupilLocation[1] + pupilLocation[3]) / 2
    y1 = (pupilLocation[2] + pupilLocation[4]) / 2
    x2 = (pupilLocation[5] + pupilLocation[7]) / 2
    y2 = (pupilLocation[6] + pupilLocation[8]) / 2
    pupilDiff[13] = ((y2 - y1) / (0.001 + x2 - x1))


    pupilDiff.to_csv('pupil_diff.csv')


    pixels = 11.7 / pupil_average
    print("pixels (In MM) = ", pixels)


    pupilDiff = round(pupilDiff * pixels,3)

    fig1 = plt.figure()
    plt.plot(pupilDiff[[9,10]]) #1,3,5,7
    plt.title("Pupil movement X axis")
    plt.ylabel("MM of movement")
    plt.xlabel("Frame")
    plt.ylim(-10, 10)
    plt.legend(['Left', 'Right']) #'LL', 'LR', 'RL', 'RR'])

    fig2 = plt.figure()
    plt.plot(pupilDiff[[11,12]]) #, df[countries].to_numpy())
    plt.ylim(-10, 10)
    plt.title("Pupil movement Y axis")
    plt.ylabel("MM of movement")
    plt.xlabel("Frame")
    plt.legend(['Left', 'Right'])


    # Left eye
    LeftEyeLookingRight = pd.to_numeric(pupilDiff[1]).idxmax()
    LeftEyeLookingDown  = pd.to_numeric(pupilDiff[2]).idxmax()
    LeftEyeLookingLeft  = pd.to_numeric(pupilDiff[3]).idxmin()
    LeftEyeLookingUp    = pd.to_numeric(pupilDiff[4]).idxmin()


    # Right eye
    RightEyeLookingRight = pd.to_numeric(pupilDiff[5]).idxmax()
    RightEyeLookingDown  = pd.to_numeric(pupilDiff[6]).idxmax()
    RightEyeLookingLeft  = pd.to_numeric(pupilDiff[7]).idxmin()
    RightEyeLookingUp    = pd.to_numeric(pupilDiff[8]).idxmin()


    print("Left eye images = ", LeftEyeLookingRight, LeftEyeLookingDown, LeftEyeLookingLeft, LeftEyeLookingUp)
    print("Right eye images = ", RightEyeLookingRight, RightEyeLookingDown, RightEyeLookingLeft, RightEyeLookingUp)

    ExtermeImageLeftEye = list([cv2.cvtColor(cv2.imread("Images/post-image-%d.jpg" % LeftEyeLookingRight), cv2.COLOR_BGR2RGB),
                                cv2.cvtColor(cv2.imread("Images/post-image-%d.jpg" % LeftEyeLookingLeft), cv2.COLOR_BGR2RGB),
                                cv2.cvtColor(cv2.imread("Images/post-image-%d.jpg" % LeftEyeLookingUp), cv2.COLOR_BGR2RGB),
                                cv2.cvtColor(cv2.imread("Images/post-image-%d.jpg" % LeftEyeLookingDown), cv2.COLOR_BGR2RGB)])

    ExtermeImageRightEye = list([cv2.cvtColor(cv2.imread("Images/post-image-%d.jpg" % RightEyeLookingRight), cv2.COLOR_BGR2RGB),
                                 cv2.cvtColor(cv2.imread("Images/post-image-%d.jpg" % RightEyeLookingLeft), cv2.COLOR_BGR2RGB),
                                 cv2.cvtColor(cv2.imread("Images/post-image-%d.jpg" % RightEyeLookingUp), cv2.COLOR_BGR2RGB),
                                 cv2.cvtColor(cv2.imread("Images/post-image-%d.jpg" % RightEyeLookingDown), cv2.COLOR_BGR2RGB)])


    # return the distances


    d = { 'direction': ['Right', 'Left', 'Up', 'Down'] ,
          'mm' : [abs(round(pd.to_numeric(pupilDiff[1]).max(),1)),
                  abs(round(pd.to_numeric(pupilDiff[3]).min(),1)),
                  abs(round(pd.to_numeric(pupilDiff[4]).min(),1)),
                  abs(round(pd.to_numeric(pupilDiff[2]).max(),1))
                  ]}
    ExteremeDistanceLeftEye = pd.DataFrame(data=d)

    d = {'direction': ['Right', 'Left', 'Up', 'Down'],
         'mm': [abs(round(pd.to_numeric(pupilDiff[5]).max(), 1)),
                abs(round(pd.to_numeric(pupilDiff[7]).min(), 1)),
                abs(round(pd.to_numeric(pupilDiff[8]).min(), 1)),
                abs(round(pd.to_numeric(pupilDiff[6]).max(), 1))
                ]}
    ExteremeDistanceRightEye = pd.DataFrame(data=d)



    print() #.idxmax(axis=0))
    # Upmost buttom limbus
    #
    return ExteremeDistanceLeftEye, ExteremeDistanceRightEye, ExtermeImageLeftEye, ExtermeImageRightEye, fig1, fig2 # lines


with gr.Blocks() as demo:
            gr.Markdown(
            """
            # Range of Motion Video Analysis
            Capture a video of the following looks: stright, left, right, up & down
            """)
            video1 = gr.Video()#source="webcam")

            b = gr.Button("Analyze Video")

            gr.Markdown(
                """
                # Left eye results (in mm):
                """)
            LeftEyeGallery = gr.Gallery(
                label="Left eye", show_label=False, elem_id="left_eye_gallery", columns=[4], rows=[1], object_fit="contain", height="auto")
        
            movementDataLeft = gr.Dataframe(ExteremeDistanceLeftEye)


            gr.Markdown(
                """
                # Right eye results (in mm):
                """)
            RightEyeGallery = gr.Gallery(
                label="Right eye", show_label=False, elem_id="right_eye_gallery", columns=[4], rows=[1], object_fit="contain", height="auto")
            movementDataRight = gr.Dataframe(ExteremeDistanceRightEye)

            plot1 = gr.Plot(label="Plot1")

            plot2 = gr.Plot(label="Plot2")

            out = [movementDataLeft, movementDataRight, LeftEyeGallery, RightEyeGallery, plot1, plot2]
            b.click(fn=handleVideo, inputs=video1, outputs=out)

demo.launch()