Delete app.py

app.py

@@ -1,424 +0,0 @@
-import gradio as gr
-import pandas as pd
-import cv2
-import mediapipe as mp
-import os
-from statistics import mean
-import numpy as np
-from mediapipe.tasks import python
-from mediapipe.tasks.python import vision
-from mediapipe.framework.formats import landmark_pb2
-from mediapipe import solutions
-
-
-import matplotlib
-matplotlib.use("Agg")
-import matplotlib.pyplot as plt
-
-# Record video
-# Save video?
-# Break video into images
-# Run facemesh on all images and save locations
-# Run exterme locations
-# Run analysis on those compare to the first frame
-
-# Create a FaceLandmarker object.
-base_options = python.BaseOptions(model_asset_path='face_landmarker_v2_with_blendshapes.task')
-options = vision.FaceLandmarkerOptions(base_options=base_options,
-                                       output_face_blendshapes=True,
-                                       output_facial_transformation_matrixes=True,
-                                       num_faces=1)
-detector = vision.FaceLandmarker.create_from_options(options)
-
-
-global pupilLocation
-pupilLocation = pd.DataFrame()
-pupil_sizes =  []
-ExteremeDistanceLeftEye = pd.DataFrame()
-ExteremeDistanceRightEye = pd.DataFrame()
-
-def video_identity(video):
-    return video
-
-# To do
-# 1. Filter out closed eye frames
-# 2. Smooth persuit from video POC
-
-def isEyeOpen(image):
-    image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
-    hist = cv2.calcHist([image], [0], None, [256], [0, 256])
-    colors = np.where(hist > 10)
-
-    #print ("colors=", np.mean(colors) )
-    if np.mean(colors) > 15:
-        return True
-    else:
-        return False
-
-
-
-#demo = gr.Interface(video_identity,
-#                    gr.Video(shape = (1000,1000), source="webcam"),
-#                    "playable_video")
-
-def findIrisInFrame(image, counter):
-    global pupilLocation, pupil_sizes
-    #pupilLocation = pd.DataFrame()  # Make sure it is empty
-
-    image = mp.Image.create_from_file("image.jpg")
-
-    # STEP 4: Detect face landmarks from the input image.
-    detection_result = detector.detect(image)
-
-    # STEP 5: Process the detection result. In this case, visualize it.
-    #annotated_image = draw_landmarks_on_image(image.numpy_view(), detection_result)
-
-    annotated_image = image.numpy_view().copy()
-
-    face_landmarks_list = detection_result.face_landmarks
-
-    '''
-    # Loop through the detected faces to visualize.
-    for idx in range(len(face_landmarks_list)):
-        face_landmarks = face_landmarks_list[idx]
-
-        # Draw the face landmarks.
-        face_landmarks_proto = landmark_pb2.NormalizedLandmarkList()
-        face_landmarks_proto.landmark.extend([
-            landmark_pb2.NormalizedLandmark(x=landmark.x, y=landmark.y, z=landmark.z) for landmark in face_landmarks
-        ])
-        solutions.drawing_utils.draw_landmarks(
-            image=annotated_image,
-            landmark_list=face_landmarks_proto,
-            connections=mp.solutions.face_mesh.FACEMESH_IRISES,
-            landmark_drawing_spec=None,
-            connection_drawing_spec=mp.solutions.drawing_styles
-            .get_default_face_mesh_iris_connections_style())
-    '''
-    #FACEMESH_LEFT_IRIS = (474, 475, 476, 477)
-    #FACEMESH_RIGHT_IRIS = (469, 470, 471, 472)
-    #(lm_left_iris.x, lm_left_iris.y, lm_left_iris.z) = face_landmarks.landmark[468]
-    #(lm_right_iris.x, lm_right_iris.y, lm_right_iris.z) = face_landmarks.landmark[473]
-
-    # Draw the face landmarks.
-    face_landmarks = face_landmarks_list[0]
-    face_landmarks_proto = landmark_pb2.NormalizedLandmarkList()
-    face_landmarks_proto.landmark.extend([
-        landmark_pb2.NormalizedLandmark(x=landmark.x, y=landmark.y, z=landmark.z) for landmark in face_landmarks
-    ])
-
-    height, width, _ = annotated_image.shape
-    nose = [int(face_landmarks_proto.landmark[168].x * width), int(face_landmarks_proto.landmark[168].y * height)]
-    cv2.circle(annotated_image, (nose[0], nose[1]), 3, (0, 0, 255), -1)
-
-    leftIrisPoints = [474, 475, 476, 477]
-    rightIrisPoints = [469, 470, 471, 472]
-    # right, top, left, bottom
-
-    left_iris = []
-    for p in leftIrisPoints:
-                    point = [int(face_landmarks_proto.landmark[p].x * width), int(face_landmarks_proto.landmark[p].y * height)]
-                    left_iris.append(point)
-                    cv2.circle(annotated_image, point, 1, (255, 0, 255), 1)
-
-    right_iris = []
-    for p in rightIrisPoints:
-                    point = [int(face_landmarks_proto.landmark[p].x * width), int(face_landmarks_proto.landmark[p].y * height)]
-                    right_iris.append(point)
-                    cv2.circle(annotated_image, point, 1, (255, 0, 255), 1)
-
-    leftIris_leftside = (int(left_iris[2][0]), int(left_iris[2][1]))
-    leftIris_rightside = (int(left_iris[0][0]), int(left_iris[0][1]))
-    leftIris_top = (int(left_iris[1][0]), int(left_iris[1][1]))
-    leftIris_bottom = (int(left_iris[3][0]), int(left_iris[3][1]))
-    rightIris_leftside = (int(right_iris[2][0]), int(right_iris[2][1]))
-    rightIris_rightside = (int(right_iris[0][0]), int(right_iris[0][1]))
-    rightIris_top = (int(right_iris[1][0]), int(right_iris[1][1]))
-    rightIris_bottom = (int(right_iris[3][0]), int(right_iris[3][1]))
-
-    cv2.imwrite("Images/post-image-%d.jpg" % counter, annotated_image)  # save frame as JPEG file
-
-    '''
-                sizeIncrease = 0
-                leftEye = annotated_image[leftIris_top[1] - sizeIncrease: leftIris_bottom[1] + sizeIncrease,
-                          leftIris_leftside[0] - sizeIncrease: leftIris_rightside[0] + sizeIncrease]
-                leftEyeOpen = isEyeOpen(leftEye)
-
-                rightEye = annotated_image[rightIris_top[1] - sizeIncrease: rightIris_bottom[1] + sizeIncrease,
-                           rightIris_leftside[0] - sizeIncrease: rightIris_rightside[0] + sizeIncrease]
-                rightEyeOpen = isEyeOpen(rightEye)
-
-
-                cv2.circle(image,
-                           (int((leftIris_leftside[0] + leftIris_rightside[0]) / 2),
-                            int((leftIris_top[1] + leftIris_bottom[1]) / 2)),
-                           # int(abs(leftIris_leftside[0] - leftIris_rightside[0])/2),
-                           1,
-                           (0, 255, 255), 1)
-
-                cv2.circle(image,
-                           (int((rightIris_leftside[0] + rightIris_rightside[0]) / 2),
-                            int((rightIris_top[1] + rightIris_bottom[1]) / 2)),
-                           # int(abs(rightIris_leftside[0] - rightIris_rightside[0])/2),
-                           1,
-                           (0, 255, 255), 1)
-
-                cv2.putText(image, str(counter),
-                            (rightIris_leftside[0] - 100, leftIris_top[1] - 10), cv2.FONT_HERSHEY_SIMPLEX,
-                            1, (255, 0, 0), 1, cv2.LINE_AA)
-
-
-    '''
-    pupil_sizes.append(abs(leftIris_leftside[0] - leftIris_rightside[0]))
-    pupil_sizes.append(abs(rightIris_leftside[0] - rightIris_rightside[0]))
-
-    name = "frame%d.jpg" % counter
-    newRow = pd.Series([name,
-                                    leftIris_leftside[0]   - nose[0],
-                                    leftIris_top[1]        - nose[1],
-                                    leftIris_rightside[0]  - nose[0],
-                                    leftIris_bottom[1]     - nose[1],
-                                    rightIris_leftside[0]  - nose[0],
-                                    rightIris_top[1]       - nose[1],
-                                    rightIris_rightside[0] - nose[0],
-                                    rightIris_bottom[1]    - nose[1]
-                                    ])
-    newRow = newRow.to_frame().T
-                #if (leftEyeOpen & rightEyeOpen):
-    pupilLocation = pd.concat([pupilLocation, newRow], axis=0, ignore_index=True)
-                #else:
-                #    print("Ignored frame ", counter, "." , leftEyeOpen , rightEyeOpen)
-
-    return newRow
-
-
-def handleVideo(input_video):
-    global ExteremeDistanceLeftEye, ExteremeDistanceRightEye, pupilLocation, pupil_sizes
-
-    pupilLocation = pd.DataFrame() # Make sure it is empty to begin with
-    pupil_sizes =  []
-    vidcap = cv2.VideoCapture(input_video)
-    success, image = vidcap.read()
-    fps = vidcap.get(cv2.CAP_PROP_FPS)
-    frame_count = int(vidcap.get(cv2.CAP_PROP_FRAME_COUNT))
-    width = vidcap.get(cv2.CAP_PROP_FRAME_WIDTH)  # float `width`
-    height = vidcap.get(cv2.CAP_PROP_FRAME_HEIGHT)  # float `height`
-    print('FPS =', fps)
-    print('Frame count =', frame_count)
-    print('Resolution =', width, ' X ', height)
-
-    count = 0
-    if not os.path.exists('Images'):
-        os.makedirs('Images')
-    #os.chdir('Images')
-
-    # Slide video into frames and find iris in each frame
-    while success:
-        cv2.imwrite("image.jpg", image)  # save frame as JPEG file
-        print("Image#", count)
-        image = mp.Image.create_from_file("image.jpg")
-        findIrisInFrame(image, count)
-
-        #cv2.imwrite("Images/frame%d.jpg" % count, image)  # save frame as JPEG file
-        count += 1
-        success, image = vidcap.read()
-
-
-    # Go over all the pupils. If pupil is too small expand it in all directions
-    # Find average pupil size
-    pupil_average = 11.7
-    if (len(pupil_sizes) > 100):
-        for i in range(10):
-           pupil_sizes.remove(max(pupil_sizes))
-           pupil_sizes.remove(min(pupil_sizes))
-        pupil_average = mean(pupil_sizes)  # this should be 11.7 mm
-    print("pupil_average=", pupil_average)
-
-    # pupil size need to be kept constant in all pictures
-    # we find the center of the current pupil and make a circle around it in the size we need
-
-    for index, row in pupilLocation.iterrows():
-        currentLeftSize = abs(row[1] - row[3])
-        diffFromLeftAverage = pupil_average - currentLeftSize
-        currentRightSize = abs(row[5] - row[7])
-        diffFromAverage = pupil_average - currentRightSize
-
-        #print("(frame#", index, ") left: ", row[1], " right: ", row[3])
-        #if diffFromAverage > 2:
-        if (currentRightSize - currentLeftSize > 200):
-            print("Fixed Left pupil")
-            row[1] = int(row[1] - diffFromAverage / 2)
-            row[2] = int(row[2] - diffFromAverage / 2)
-            row[3] = int(row[3] + diffFromAverage / 2)
-            row[4] = int(row[4] + diffFromAverage / 2)
-
-        if (currentLeftSize - currentRightSize > 200):
-            print("Fixed Right pupil")
-            row[5] = int(row[5] - diffFromAverage / 2)
-            row[6] = int(row[6] - diffFromAverage / 2)
-            row[7] = int(row[7] + diffFromAverage / 2)
-            row[8] = int(row[8] + diffFromAverage / 2)
-
-
-    print("file counter=", count)
-    # Convert pupilLocation to pupilDiff
-    pupilDiff = pupilLocation.copy()
-    pupilDiff = pupilDiff.drop(pupilDiff.columns[0], axis=1)  # Remove file name
-    for i in range(pupilDiff.shape[0] - 1):  # Calculate deltas
-        pupilDiff.loc[i + 1] = (pupilDiff.loc[i + 1] - pupilDiff.loc[0])
-    pupilDiff = pupilDiff.drop(0, axis=0) # Remove the first row
-
-    # Find extreme iris locations (images and measurements)
-
-    #pupilDiff.columns = ['LL', 'LT', 'LR', 'LB', 'RR', 'RT', 'RR', 'RB']
-
-    # Take just the relevant libmus
-    #if [1] is positive, take it, otherwise take 3
-
-    #pupilDiff[9]  = [pupilDiff[1] if x >= 0 else pupilDiff[3] for x in pupilDiff[1]]
-
-    pupilDiff[9] = 0
-    pupilDiff[10] = 0
-
-    pupilDiff[9]  = np.where(pupilDiff[1] >= 0, pupilDiff[1], pupilDiff[3])
-                             #pupilDiff[[1,3]].max(axis=1), pupilDiff[[1,3]].min(axis=1))
-    pupilDiff[10] = np.where(pupilDiff[1] >= 0, pupilDiff[5], pupilDiff[7])
-                             #pupilDiff[[5,7]].max(axis=1), pupilDiff[[5,7]].min(axis=1))
-    pupilDiff[11] = np.where(pupilDiff[2] >= 0, pupilDiff[2], pupilDiff[4])
-                             #pupilDiff[[2,4]].max(axis=1), pupilDiff[[2,4]].min(axis=1))
-    pupilDiff[12] = np.where(pupilDiff[2] >= 0, pupilDiff[6], pupilDiff[8])
-                             #pupilDiff[[6,8]].max(axis=1), pupilDiff[[6,8]].min(axis=1))
-    print(pupilDiff[[1,3,5,7,9,10]])
-
-    # slope
-    x1 = (pupilLocation[1] + pupilLocation[3]) / 2
-    y1 = (pupilLocation[2] + pupilLocation[4]) / 2
-    x2 = (pupilLocation[5] + pupilLocation[7]) / 2
-    y2 = (pupilLocation[6] + pupilLocation[8]) / 2
-    pupilDiff[13] = ((y2 - y1) / (0.001 + x2 - x1))
-
-
-    pupilDiff.to_csv('pupil_diff.csv')
-
-
-    pixels = 11.7 / pupil_average
-    print("pixels (In MM) = ", pixels)
-
-
-    pupilDiff = round(pupilDiff * pixels,3)
-
-    fig1 = plt.figure()
-    plt.plot(pupilDiff[[9,10]]) #1,3,5,7
-    plt.title("Pupil movement X axis")
-    plt.ylabel("MM of movement")
-    plt.xlabel("Frame")
-    plt.ylim(-10, 10)
-    plt.legend(['Left', 'Right']) #'LL', 'LR', 'RL', 'RR'])
-
-    fig2 = plt.figure()
-    plt.plot(pupilDiff[[11,12]]) #, df[countries].to_numpy())
-    plt.ylim(-10, 10)
-    plt.title("Pupil movement Y axis")
-    plt.ylabel("MM of movement")
-    plt.xlabel("Frame")
-    plt.legend(['Left', 'Right'])
-
-
-    # Left eye
-    LeftEyeLookingRight = pd.to_numeric(pupilDiff[1]).idxmax()
-    LeftEyeLookingDown  = pd.to_numeric(pupilDiff[2]).idxmax()
-    LeftEyeLookingLeft  = pd.to_numeric(pupilDiff[3]).idxmin()
-    LeftEyeLookingUp    = pd.to_numeric(pupilDiff[4]).idxmin()
-
-
-    # Right eye
-    RightEyeLookingRight = pd.to_numeric(pupilDiff[5]).idxmax()
-    RightEyeLookingDown  = pd.to_numeric(pupilDiff[6]).idxmax()
-    RightEyeLookingLeft  = pd.to_numeric(pupilDiff[7]).idxmin()
-    RightEyeLookingUp    = pd.to_numeric(pupilDiff[8]).idxmin()
-
-
-    print("Left eye images = ", LeftEyeLookingRight, LeftEyeLookingDown, LeftEyeLookingLeft, LeftEyeLookingUp)
-    print("Right eye images = ", RightEyeLookingRight, RightEyeLookingDown, RightEyeLookingLeft, RightEyeLookingUp)
-
-    ExtermeImageLeftEye = list([cv2.cvtColor(cv2.imread("Images/frame%d.jpg" % LeftEyeLookingRight), cv2.COLOR_BGR2RGB),
-                                cv2.cvtColor(cv2.imread("Images/frame%d.jpg" % LeftEyeLookingLeft), cv2.COLOR_BGR2RGB),
-                                cv2.cvtColor(cv2.imread("Images/frame%d.jpg" % LeftEyeLookingUp), cv2.COLOR_BGR2RGB),
-                                cv2.cvtColor(cv2.imread("Images/frame%d.jpg" % LeftEyeLookingDown), cv2.COLOR_BGR2RGB)])
-
-    ExtermeImageRightEye = list([cv2.cvtColor(cv2.imread("Images/frame%d.jpg" % RightEyeLookingRight), cv2.COLOR_BGR2RGB),
-                                 cv2.cvtColor(cv2.imread("Images/frame%d.jpg" % RightEyeLookingLeft), cv2.COLOR_BGR2RGB),
-                                 cv2.cvtColor(cv2.imread("Images/frame%d.jpg" % RightEyeLookingUp), cv2.COLOR_BGR2RGB),
-                                 cv2.cvtColor(cv2.imread("Images/frame%d.jpg" % RightEyeLookingDown), cv2.COLOR_BGR2RGB)])
-
-
-    # return the distances
-
-
-    d = { 'direction': ['Right', 'Left', 'Up', 'Down'] ,
-          'mm' : [abs(round(pd.to_numeric(pupilDiff[1]).max(),1)),
-                  abs(round(pd.to_numeric(pupilDiff[3]).min(),1)),
-                  abs(round(pd.to_numeric(pupilDiff[4]).min(),1)),
-                  abs(round(pd.to_numeric(pupilDiff[2]).max(),1))
-                  ]}
-    ExteremeDistanceLeftEye = pd.DataFrame(data=d)
-
-    d = {'direction': ['Right', 'Left', 'Up', 'Down'],
-         'mm': [abs(round(pd.to_numeric(pupilDiff[5]).max(), 1)),
-                abs(round(pd.to_numeric(pupilDiff[7]).min(), 1)),
-                abs(round(pd.to_numeric(pupilDiff[8]).min(), 1)),
-                abs(round(pd.to_numeric(pupilDiff[6]).max(), 1))
-                ]}
-    ExteremeDistanceRightEye = pd.DataFrame(data=d)
-
-
-
-    print() #.idxmax(axis=0))
-    # Upmost buttom limbus
-    #
-    return ExteremeDistanceLeftEye, ExteremeDistanceRightEye, ExtermeImageLeftEye, ExtermeImageRightEye, fig1, fig2 # lines
-
-
-with gr.Blocks() as demo:
-            gr.Markdown(
-            """
-            # Range of Motion Video Analysis
-            Capture a video of the following looks: stright, left, right, up & down
-            """)
-            video1 = gr.Video()#source="webcam")
-
-            b = gr.Button("Analyze Video")
-
-            gr.Markdown(
-                """
-                # Left eye results (in mm):
-                """)
-            LeftEyeGallery = gr.Gallery(
-                label="Left eye", show_label=False, elem_id="left_eye_gallery"
-            ).style(grid=[4], height="auto")
-            movementDataLeft = gr.Dataframe(ExteremeDistanceLeftEye)
-
-
-            gr.Markdown(
-                """
-                # Right eye results (in mm):
-                """)
-            RightEyeGallery = gr.Gallery(
-                label="Right eye", show_label=False, elem_id="right_eye_gallery"
-            ).style(grid=[4], height="auto")
-            movementDataRight = gr.Dataframe(ExteremeDistanceRightEye)
-
-            plot1 = gr.Plot(label="Plot1")
-
-            plot2 = gr.Plot(label="Plot2")
-
-            out = [movementDataLeft, movementDataRight, LeftEyeGallery, RightEyeGallery, plot1, plot2]
-            b.click(fn=handleVideo, inputs=video1, outputs=out)
-
-demo.launch()
-
-
-
-
-
-