Spaces:
Running
Running
| import gradio as gr | |
| import pandas as pd | |
| import cv2 | |
| import mediapipe as mp | |
| import os | |
| from statistics import mean | |
| import numpy as np | |
| from mediapipe.tasks import python | |
| from mediapipe.tasks.python import vision | |
| from mediapipe.framework.formats import landmark_pb2 | |
| from mediapipe import solutions | |
| import matplotlib | |
| matplotlib.use("Agg") | |
| import matplotlib.pyplot as plt | |
| # Record video | |
| # Save video? | |
| # Break video into images | |
| # Run facemesh on all images and save locations | |
| # Run exterme locations | |
| # Run analysis on those compare to the first frame | |
| # Create a FaceLandmarker object. | |
| base_options = python.BaseOptions(model_asset_path='face_landmarker_v2_with_blendshapes.task') | |
| options = vision.FaceLandmarkerOptions(base_options=base_options, | |
| output_face_blendshapes=True, | |
| output_facial_transformation_matrixes=True, | |
| num_faces=1) | |
| detector = vision.FaceLandmarker.create_from_options(options) | |
| global pupilLocation | |
| pupilLocation = pd.DataFrame() | |
| pupil_sizes = [] | |
| ExteremeDistanceLeftEye = pd.DataFrame() | |
| ExteremeDistanceRightEye = pd.DataFrame() | |
| def video_identity(video): | |
| return video | |
| # To do | |
| # 1. Filter out closed eye frames | |
| # 2. Smooth persuit from video POC | |
| def isEyeOpen(image): | |
| image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) | |
| hist = cv2.calcHist([image], [0], None, [256], [0, 256]) | |
| colors = np.where(hist > 10) | |
| #print ("colors=", np.mean(colors) ) | |
| if np.mean(colors) > 15: | |
| return True | |
| else: | |
| return False | |
| #demo = gr.Interface(video_identity, | |
| # gr.Video(shape = (1000,1000), source="webcam"), | |
| # "playable_video") | |
| def findIrisInFrame(image, counter): | |
| global pupilLocation, pupil_sizes | |
| #pupilLocation = pd.DataFrame() # Make sure it is empty | |
| image = mp.Image.create_from_file("image.jpg") | |
| # STEP 4: Detect face landmarks from the input image. | |
| detection_result = detector.detect(image) | |
| # STEP 5: Process the detection result. In this case, visualize it. | |
| #annotated_image = draw_landmarks_on_image(image.numpy_view(), detection_result) | |
| annotated_image = image.numpy_view().copy() | |
| face_landmarks_list = detection_result.face_landmarks | |
| ''' | |
| # Loop through the detected faces to visualize. | |
| for idx in range(len(face_landmarks_list)): | |
| face_landmarks = face_landmarks_list[idx] | |
| # Draw the face landmarks. | |
| face_landmarks_proto = landmark_pb2.NormalizedLandmarkList() | |
| face_landmarks_proto.landmark.extend([ | |
| landmark_pb2.NormalizedLandmark(x=landmark.x, y=landmark.y, z=landmark.z) for landmark in face_landmarks | |
| ]) | |
| solutions.drawing_utils.draw_landmarks( | |
| image=annotated_image, | |
| landmark_list=face_landmarks_proto, | |
| connections=mp.solutions.face_mesh.FACEMESH_IRISES, | |
| landmark_drawing_spec=None, | |
| connection_drawing_spec=mp.solutions.drawing_styles | |
| .get_default_face_mesh_iris_connections_style()) | |
| ''' | |
| #FACEMESH_LEFT_IRIS = (474, 475, 476, 477) | |
| #FACEMESH_RIGHT_IRIS = (469, 470, 471, 472) | |
| #(lm_left_iris.x, lm_left_iris.y, lm_left_iris.z) = face_landmarks.landmark[468] | |
| #(lm_right_iris.x, lm_right_iris.y, lm_right_iris.z) = face_landmarks.landmark[473] | |
| # Draw the face landmarks. | |
| face_landmarks = face_landmarks_list[0] | |
| face_landmarks_proto = landmark_pb2.NormalizedLandmarkList() | |
| face_landmarks_proto.landmark.extend([ | |
| landmark_pb2.NormalizedLandmark(x=landmark.x, y=landmark.y, z=landmark.z) for landmark in face_landmarks | |
| ]) | |
| height, width, _ = annotated_image.shape | |
| nose = [int(face_landmarks_proto.landmark[168].x * width), int(face_landmarks_proto.landmark[168].y * height)] | |
| cv2.circle(annotated_image, (nose[0], nose[1]), 3, (0, 0, 255), -1) | |
| leftIrisPoints = [474, 475, 476, 477] | |
| rightIrisPoints = [469, 470, 471, 472] | |
| # right, top, left, bottom | |
| left_iris = [] | |
| for p in leftIrisPoints: | |
| point = [int(face_landmarks_proto.landmark[p].x * width), int(face_landmarks_proto.landmark[p].y * height)] | |
| left_iris.append(point) | |
| cv2.circle(annotated_image, point, 1, (255, 0, 255), 1) | |
| right_iris = [] | |
| for p in rightIrisPoints: | |
| point = [int(face_landmarks_proto.landmark[p].x * width), int(face_landmarks_proto.landmark[p].y * height)] | |
| right_iris.append(point) | |
| cv2.circle(annotated_image, point, 1, (255, 0, 255), 1) | |
| leftIris_leftside = (int(left_iris[2][0]), int(left_iris[2][1])) | |
| leftIris_rightside = (int(left_iris[0][0]), int(left_iris[0][1])) | |
| leftIris_top = (int(left_iris[1][0]), int(left_iris[1][1])) | |
| leftIris_bottom = (int(left_iris[3][0]), int(left_iris[3][1])) | |
| rightIris_leftside = (int(right_iris[2][0]), int(right_iris[2][1])) | |
| rightIris_rightside = (int(right_iris[0][0]), int(right_iris[0][1])) | |
| rightIris_top = (int(right_iris[1][0]), int(right_iris[1][1])) | |
| rightIris_bottom = (int(right_iris[3][0]), int(right_iris[3][1])) | |
| cv2.imwrite("Images/post-image-%d.jpg" % counter, annotated_image) # save frame as JPEG file | |
| ''' | |
| sizeIncrease = 0 | |
| leftEye = annotated_image[leftIris_top[1] - sizeIncrease: leftIris_bottom[1] + sizeIncrease, | |
| leftIris_leftside[0] - sizeIncrease: leftIris_rightside[0] + sizeIncrease] | |
| leftEyeOpen = isEyeOpen(leftEye) | |
| rightEye = annotated_image[rightIris_top[1] - sizeIncrease: rightIris_bottom[1] + sizeIncrease, | |
| rightIris_leftside[0] - sizeIncrease: rightIris_rightside[0] + sizeIncrease] | |
| rightEyeOpen = isEyeOpen(rightEye) | |
| cv2.circle(image, | |
| (int((leftIris_leftside[0] + leftIris_rightside[0]) / 2), | |
| int((leftIris_top[1] + leftIris_bottom[1]) / 2)), | |
| # int(abs(leftIris_leftside[0] - leftIris_rightside[0])/2), | |
| 1, | |
| (0, 255, 255), 1) | |
| cv2.circle(image, | |
| (int((rightIris_leftside[0] + rightIris_rightside[0]) / 2), | |
| int((rightIris_top[1] + rightIris_bottom[1]) / 2)), | |
| # int(abs(rightIris_leftside[0] - rightIris_rightside[0])/2), | |
| 1, | |
| (0, 255, 255), 1) | |
| cv2.putText(image, str(counter), | |
| (rightIris_leftside[0] - 100, leftIris_top[1] - 10), cv2.FONT_HERSHEY_SIMPLEX, | |
| 1, (255, 0, 0), 1, cv2.LINE_AA) | |
| ''' | |
| pupil_sizes.append(abs(leftIris_leftside[0] - leftIris_rightside[0])) | |
| pupil_sizes.append(abs(rightIris_leftside[0] - rightIris_rightside[0])) | |
| name = "frame%d.jpg" % counter | |
| newRow = pd.Series([name, | |
| leftIris_leftside[0] - nose[0], | |
| leftIris_top[1] - nose[1], | |
| leftIris_rightside[0] - nose[0], | |
| leftIris_bottom[1] - nose[1], | |
| rightIris_leftside[0] - nose[0], | |
| rightIris_top[1] - nose[1], | |
| rightIris_rightside[0] - nose[0], | |
| rightIris_bottom[1] - nose[1] | |
| ]) | |
| newRow = newRow.to_frame().T | |
| #if (leftEyeOpen & rightEyeOpen): | |
| pupilLocation = pd.concat([pupilLocation, newRow], axis=0, ignore_index=True) | |
| #else: | |
| # print("Ignored frame ", counter, "." , leftEyeOpen , rightEyeOpen) | |
| return newRow | |
| def handleVideo(input_video): | |
| global ExteremeDistanceLeftEye, ExteremeDistanceRightEye, pupilLocation, pupil_sizes | |
| pupilLocation = pd.DataFrame() # Make sure it is empty to begin with | |
| pupil_sizes = [] | |
| vidcap = cv2.VideoCapture(input_video) | |
| success, image = vidcap.read() | |
| fps = vidcap.get(cv2.CAP_PROP_FPS) | |
| frame_count = int(vidcap.get(cv2.CAP_PROP_FRAME_COUNT)) | |
| width = vidcap.get(cv2.CAP_PROP_FRAME_WIDTH) # float `width` | |
| height = vidcap.get(cv2.CAP_PROP_FRAME_HEIGHT) # float `height` | |
| print('FPS =', fps) | |
| print('Frame count =', frame_count) | |
| print('Resolution =', width, ' X ', height) | |
| count = 0 | |
| if not os.path.exists('Images'): | |
| os.makedirs('Images') | |
| #os.chdir('Images') | |
| # Slide video into frames and find iris in each frame | |
| while success: | |
| cv2.imwrite("image.jpg", image) # save frame as JPEG file | |
| print("Image#", count) | |
| image = mp.Image.create_from_file("image.jpg") | |
| findIrisInFrame(image, count) | |
| #cv2.imwrite("Images/frame%d.jpg" % count, image) # save frame as JPEG file | |
| count += 1 | |
| success, image = vidcap.read() | |
| # Go over all the pupils. If pupil is too small expand it in all directions | |
| # Find average pupil size | |
| pupil_average = 11.7 | |
| if (len(pupil_sizes) > 100): | |
| for i in range(10): | |
| pupil_sizes.remove(max(pupil_sizes)) | |
| pupil_sizes.remove(min(pupil_sizes)) | |
| pupil_average = mean(pupil_sizes) # this should be 11.7 mm | |
| print("pupil_average=", pupil_average) | |
| # pupil size need to be kept constant in all pictures | |
| # we find the center of the current pupil and make a circle around it in the size we need | |
| for index, row in pupilLocation.iterrows(): | |
| currentLeftSize = abs(row[1] - row[3]) | |
| diffFromLeftAverage = pupil_average - currentLeftSize | |
| currentRightSize = abs(row[5] - row[7]) | |
| diffFromAverage = pupil_average - currentRightSize | |
| #print("(frame#", index, ") left: ", row[1], " right: ", row[3]) | |
| #if diffFromAverage > 2: | |
| if (currentRightSize - currentLeftSize > 200): | |
| print("Fixed Left pupil") | |
| row[1] = int(row[1] - diffFromAverage / 2) | |
| row[2] = int(row[2] - diffFromAverage / 2) | |
| row[3] = int(row[3] + diffFromAverage / 2) | |
| row[4] = int(row[4] + diffFromAverage / 2) | |
| if (currentLeftSize - currentRightSize > 200): | |
| print("Fixed Right pupil") | |
| row[5] = int(row[5] - diffFromAverage / 2) | |
| row[6] = int(row[6] - diffFromAverage / 2) | |
| row[7] = int(row[7] + diffFromAverage / 2) | |
| row[8] = int(row[8] + diffFromAverage / 2) | |
| print("file counter=", count) | |
| # Convert pupilLocation to pupilDiff | |
| pupilDiff = pupilLocation.copy() | |
| pupilDiff = pupilDiff.drop(pupilDiff.columns[0], axis=1) # Remove file name | |
| for i in range(pupilDiff.shape[0] - 1): # Calculate deltas | |
| pupilDiff.loc[i + 1] = (pupilDiff.loc[i + 1] - pupilDiff.loc[0]) | |
| pupilDiff = pupilDiff.drop(0, axis=0) # Remove the first row | |
| # Find extreme iris locations (images and measurements) | |
| #pupilDiff.columns = ['LL', 'LT', 'LR', 'LB', 'RR', 'RT', 'RR', 'RB'] | |
| # Take just the relevant libmus | |
| #if [1] is positive, take it, otherwise take 3 | |
| #pupilDiff[9] = [pupilDiff[1] if x >= 0 else pupilDiff[3] for x in pupilDiff[1]] | |
| pupilDiff[9] = 0 | |
| pupilDiff[10] = 0 | |
| pupilDiff[9] = np.where(pupilDiff[1] >= 0, pupilDiff[1], pupilDiff[3]) | |
| #pupilDiff[[1,3]].max(axis=1), pupilDiff[[1,3]].min(axis=1)) | |
| pupilDiff[10] = np.where(pupilDiff[1] >= 0, pupilDiff[5], pupilDiff[7]) | |
| #pupilDiff[[5,7]].max(axis=1), pupilDiff[[5,7]].min(axis=1)) | |
| pupilDiff[11] = np.where(pupilDiff[2] >= 0, pupilDiff[2], pupilDiff[4]) | |
| #pupilDiff[[2,4]].max(axis=1), pupilDiff[[2,4]].min(axis=1)) | |
| pupilDiff[12] = np.where(pupilDiff[2] >= 0, pupilDiff[6], pupilDiff[8]) | |
| #pupilDiff[[6,8]].max(axis=1), pupilDiff[[6,8]].min(axis=1)) | |
| print(pupilDiff[[1,3,5,7,9,10]]) | |
| # slope | |
| x1 = (pupilLocation[1] + pupilLocation[3]) / 2 | |
| y1 = (pupilLocation[2] + pupilLocation[4]) / 2 | |
| x2 = (pupilLocation[5] + pupilLocation[7]) / 2 | |
| y2 = (pupilLocation[6] + pupilLocation[8]) / 2 | |
| pupilDiff[13] = ((y2 - y1) / (0.001 + x2 - x1)) | |
| pupilDiff.to_csv('pupil_diff.csv') | |
| pixels = 11.7 / pupil_average | |
| print("pixels (In MM) = ", pixels) | |
| pupilDiff = round(pupilDiff * pixels,3) | |
| fig1 = plt.figure() | |
| plt.plot(pupilDiff[[9,10]]) #1,3,5,7 | |
| plt.title("Pupil movement X axis") | |
| plt.ylabel("MM of movement") | |
| plt.xlabel("Frame") | |
| plt.ylim(-10, 10) | |
| plt.legend(['Left', 'Right']) #'LL', 'LR', 'RL', 'RR']) | |
| fig2 = plt.figure() | |
| plt.plot(pupilDiff[[11,12]]) #, df[countries].to_numpy()) | |
| plt.ylim(-10, 10) | |
| plt.title("Pupil movement Y axis") | |
| plt.ylabel("MM of movement") | |
| plt.xlabel("Frame") | |
| plt.legend(['Left', 'Right']) | |
| # Left eye | |
| LeftEyeLookingRight = pd.to_numeric(pupilDiff[1]).idxmax() | |
| LeftEyeLookingDown = pd.to_numeric(pupilDiff[2]).idxmax() | |
| LeftEyeLookingLeft = pd.to_numeric(pupilDiff[3]).idxmin() | |
| LeftEyeLookingUp = pd.to_numeric(pupilDiff[4]).idxmin() | |
| # Right eye | |
| RightEyeLookingRight = pd.to_numeric(pupilDiff[5]).idxmax() | |
| RightEyeLookingDown = pd.to_numeric(pupilDiff[6]).idxmax() | |
| RightEyeLookingLeft = pd.to_numeric(pupilDiff[7]).idxmin() | |
| RightEyeLookingUp = pd.to_numeric(pupilDiff[8]).idxmin() | |
| print("Left eye images = ", LeftEyeLookingRight, LeftEyeLookingDown, LeftEyeLookingLeft, LeftEyeLookingUp) | |
| print("Right eye images = ", RightEyeLookingRight, RightEyeLookingDown, RightEyeLookingLeft, RightEyeLookingUp) | |
| ExtermeImageLeftEye = list([cv2.cvtColor(cv2.imread("Images/post-image-%d.jpg" % LeftEyeLookingRight), cv2.COLOR_BGR2RGB), | |
| cv2.cvtColor(cv2.imread("Images/post-image-%d.jpg" % LeftEyeLookingLeft), cv2.COLOR_BGR2RGB), | |
| cv2.cvtColor(cv2.imread("Images/post-image-%d.jpg" % LeftEyeLookingUp), cv2.COLOR_BGR2RGB), | |
| cv2.cvtColor(cv2.imread("Images/post-image-%d.jpg" % LeftEyeLookingDown), cv2.COLOR_BGR2RGB)]) | |
| ExtermeImageRightEye = list([cv2.cvtColor(cv2.imread("Images/post-image-%d.jpg" % RightEyeLookingRight), cv2.COLOR_BGR2RGB), | |
| cv2.cvtColor(cv2.imread("Images/post-image-%d.jpg" % RightEyeLookingLeft), cv2.COLOR_BGR2RGB), | |
| cv2.cvtColor(cv2.imread("Images/post-image-%d.jpg" % RightEyeLookingUp), cv2.COLOR_BGR2RGB), | |
| cv2.cvtColor(cv2.imread("Images/post-image-%d.jpg" % RightEyeLookingDown), cv2.COLOR_BGR2RGB)]) | |
| # return the distances | |
| d = { 'direction': ['Right', 'Left', 'Up', 'Down'] , | |
| 'mm' : [abs(round(pd.to_numeric(pupilDiff[1]).max(),1)), | |
| abs(round(pd.to_numeric(pupilDiff[3]).min(),1)), | |
| abs(round(pd.to_numeric(pupilDiff[4]).min(),1)), | |
| abs(round(pd.to_numeric(pupilDiff[2]).max(),1)) | |
| ]} | |
| ExteremeDistanceLeftEye = pd.DataFrame(data=d) | |
| d = {'direction': ['Right', 'Left', 'Up', 'Down'], | |
| 'mm': [abs(round(pd.to_numeric(pupilDiff[5]).max(), 1)), | |
| abs(round(pd.to_numeric(pupilDiff[7]).min(), 1)), | |
| abs(round(pd.to_numeric(pupilDiff[8]).min(), 1)), | |
| abs(round(pd.to_numeric(pupilDiff[6]).max(), 1)) | |
| ]} | |
| ExteremeDistanceRightEye = pd.DataFrame(data=d) | |
| print() #.idxmax(axis=0)) | |
| # Upmost buttom limbus | |
| # | |
| return ExteremeDistanceLeftEye, ExteremeDistanceRightEye, ExtermeImageLeftEye, ExtermeImageRightEye, fig1, fig2 # lines | |
| with gr.Blocks() as demo: | |
| gr.Markdown( | |
| """ | |
| # Range of Motion Video Analysis | |
| Capture a video of the following looks: stright, left, right, up & down | |
| """) | |
| video1 = gr.Video()#source="webcam") | |
| b = gr.Button("Analyze Video") | |
| gr.Markdown( | |
| """ | |
| # Left eye results (in mm): | |
| """) | |
| LeftEyeGallery = gr.Gallery( | |
| label="Left eye", show_label=False, elem_id="left_eye_gallery" | |
| ).style(grid=[4], height="auto") | |
| movementDataLeft = gr.Dataframe(ExteremeDistanceLeftEye) | |
| gr.Markdown( | |
| """ | |
| # Right eye results (in mm): | |
| """) | |
| RightEyeGallery = gr.Gallery( | |
| label="Right eye", show_label=False, elem_id="right_eye_gallery" | |
| ).style(grid=[4], height="auto") | |
| movementDataRight = gr.Dataframe(ExteremeDistanceRightEye) | |
| plot1 = gr.Plot(label="Plot1") | |
| plot2 = gr.Plot(label="Plot2") | |
| out = [movementDataLeft, movementDataRight, LeftEyeGallery, RightEyeGallery, plot1, plot2] | |
| b.click(fn=handleVideo, inputs=video1, outputs=out) | |
| demo.launch() | |