Spaces:
Sleeping
Sleeping
| import gradio as gr | |
| import pandas as pd | |
| import cv2 | |
| import mediapipe as mp | |
| import os | |
| from statistics import mean | |
| # Record video | |
| # Save video? | |
| # Break video into images | |
| # Run facemesh on all images and save locations | |
| # Run exterme locations | |
| # Run analysis on those compare to the first frame | |
| # Run simple face mesh | |
| mp_face_mesh = mp.solutions.face_mesh | |
| mp_drawing = mp.solutions.drawing_utils | |
| drawing_spec = mp_drawing.DrawingSpec(thickness=2, circle_radius=3) | |
| global pupilLocation | |
| pupilLocation = pd.DataFrame() | |
| pupil_sizes = [] | |
| ExteremeDistanceLeftEye = pd.DataFrame() | |
| ExteremeDistanceRightEye = pd.DataFrame() | |
| def video_identity(video): | |
| return video | |
| #demo = gr.Interface(video_identity, | |
| # gr.Video(shape = (1000,1000), source="webcam"), | |
| # "playable_video") | |
| def findIrisInFrame(image, counter): | |
| global pupilLocation, pupil_sizes | |
| #pupilLocation = pd.DataFrame() # Make sure it is empty | |
| with mp_face_mesh.FaceMesh(max_num_faces=1, refine_landmarks=True, | |
| static_image_mode=True, | |
| min_detection_confidence=0.45) as face_mesh: | |
| results = face_mesh.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB)) | |
| if not results.multi_face_landmarks: | |
| return None | |
| annotated_image = image.copy() | |
| for face_landmarks in results.multi_face_landmarks: | |
| height, width, _ = annotated_image.shape | |
| nose = [int(face_landmarks.landmark[168].x * width), int(face_landmarks.landmark[168].y * height)] | |
| cv2.circle(annotated_image, (nose[0], nose[1]), 3, (0, 0, 255), -1) | |
| leftIrisPoints = [474, 475, 476, 477] | |
| rightIrisPoints = [469, 470, 471, 472] | |
| # right, top, left, bottom | |
| left_iris = [] | |
| for p in leftIrisPoints: | |
| point = [int(face_landmarks.landmark[p].x * width), int(face_landmarks.landmark[p].y * height)] | |
| left_iris.append(point) | |
| cv2.circle(annotated_image, point, 1, (255, 0, 255), 2) | |
| right_iris = [] | |
| for p in rightIrisPoints: | |
| point = [int(face_landmarks.landmark[p].x * width), int(face_landmarks.landmark[p].y * height)] | |
| right_iris.append(point) | |
| cv2.circle(annotated_image, point, 1, (255, 0, 255), 2) | |
| leftIris_leftside = (int(left_iris[2][0]), int(left_iris[2][1])) | |
| leftIris_rightside = (int(left_iris[0][0]), int(left_iris[0][1])) | |
| leftIris_top = (int(left_iris[1][0]), int(left_iris[1][1])) | |
| leftIris_bottom = (int(left_iris[3][0]), int(left_iris[3][1])) | |
| rightIris_leftside = (int(right_iris[2][0]), int(right_iris[2][1])) | |
| rightIris_rightside = (int(right_iris[0][0]), int(right_iris[0][1])) | |
| rightIris_top = (int(right_iris[1][0]), int(right_iris[1][1])) | |
| rightIris_bottom = (int(right_iris[3][0]), int(right_iris[3][1])) | |
| cv2.circle(annotated_image, | |
| (int((leftIris_leftside[0] + leftIris_rightside[0]) / 2), | |
| int((leftIris_top[1] + leftIris_bottom[1]) / 2)), | |
| # int(abs(leftIris_leftside[0] - leftIris_rightside[0])/2 | |
| 1, | |
| (0, 255, 255), 2) | |
| cv2.circle(annotated_image, | |
| (int((rightIris_leftside[0] + rightIris_rightside[0]) / 2), | |
| int((rightIris_top[1] + rightIris_bottom[1]) / 2)), | |
| # int(abs(rightIris_leftside[0] - rightIris_rightside[0]) / 2 | |
| 1, | |
| (0, 255, 255), 2) | |
| pupil_sizes.append(abs(leftIris_leftside[0] - leftIris_rightside[0])) | |
| pupil_sizes.append(abs(rightIris_leftside[0] - rightIris_rightside[0])) | |
| name = "frame%d.jpg" % counter | |
| newRow = pd.Series([name, | |
| leftIris_leftside[0] - nose[0], | |
| leftIris_top[1] - nose[1], | |
| leftIris_rightside[0] - nose[0], | |
| leftIris_bottom[1] - nose[1], | |
| rightIris_leftside[0] - nose[0], | |
| rightIris_top[1] - nose[1], | |
| rightIris_rightside[0] - nose[0], | |
| rightIris_bottom[1] - nose[1] | |
| ]) | |
| newRow = newRow.to_frame().T | |
| pupilLocation = pd.concat([pupilLocation, newRow], axis=0, ignore_index=True) | |
| return newRow | |
| def handleVideo(input_video): | |
| global ExteremeDistanceLeftEye, ExteremeDistanceRightEye, pupilLocation, pupil_sizes | |
| pupilLocation = pd.DataFrame() # Make sure it is empty to begin with | |
| pupil_sizes = [] | |
| vidcap = cv2.VideoCapture(input_video) | |
| success, image = vidcap.read() | |
| count = 0 | |
| if not os.path.exists('Images'): | |
| os.makedirs('Images') | |
| #os.chdir('Images') | |
| # Slide video into frames and find iris in each frame | |
| while success: | |
| cv2.imwrite("Images/frame%d.jpg" % count, image) # save frame as JPEG file | |
| findIrisInFrame(image, count) | |
| success, image = vidcap.read() | |
| count += 1 | |
| print("file counter=", count) | |
| # Convert pupilLocation to pupilDiff | |
| pupilDiff = pupilLocation.copy() | |
| pupilDiff = pupilDiff.drop(pupilDiff.columns[0], axis=1) # Remove file name | |
| for i in range(pupilDiff.shape[0] - 1): # Calculate deltas | |
| pupilDiff.loc[i + 1] = (pupilDiff.loc[i + 1] - pupilDiff.loc[0]) | |
| pupilDiff = pupilDiff.drop(0, axis=0) # Remove the first row | |
| print(pupilDiff) | |
| # Find extreme iris locations (images and measurements) | |
| # Left eye | |
| LeftEyeLookingRight = pd.to_numeric(pupilDiff[1]).idxmin() | |
| LeftEyeLookingDown = pd.to_numeric(pupilDiff[2]).idxmax() | |
| LeftEyeLookingLeft = pd.to_numeric(pupilDiff[3]).idxmax() | |
| LeftEyeLookingUp = pd.to_numeric(pupilDiff[4]).idxmin() | |
| # Right eye | |
| RightEyeLookingRight = pd.to_numeric(pupilDiff[5]).idxmin() | |
| RightEyeLookingDown = pd.to_numeric(pupilDiff[6]).idxmax() | |
| RightEyeLookingLeft = pd.to_numeric(pupilDiff[7]).idxmax() | |
| RightEyeLookingUp = pd.to_numeric(pupilDiff[8]).idxmin() | |
| print("Left eye images = ", LeftEyeLookingRight, LeftEyeLookingDown, LeftEyeLookingLeft, LeftEyeLookingUp) | |
| print("Right eye images = ", RightEyeLookingRight, RightEyeLookingDown, RightEyeLookingLeft, RightEyeLookingUp) | |
| ExtermeImageLeftEye = list([cv2.cvtColor(cv2.imread("Images/frame%d.jpg" % LeftEyeLookingRight), cv2.COLOR_BGR2RGB), | |
| cv2.cvtColor(cv2.imread("Images/frame%d.jpg" % LeftEyeLookingDown), cv2.COLOR_BGR2RGB), | |
| cv2.cvtColor(cv2.imread("Images/frame%d.jpg" % LeftEyeLookingLeft), cv2.COLOR_BGR2RGB), | |
| cv2.cvtColor(cv2.imread("Images/frame%d.jpg" % LeftEyeLookingUp), cv2.COLOR_BGR2RGB)]) | |
| ExtermeImageRightEye = list([cv2.cvtColor(cv2.imread("Images/frame%d.jpg" % RightEyeLookingRight), cv2.COLOR_BGR2RGB), | |
| cv2.cvtColor(cv2.imread("Images/frame%d.jpg" % RightEyeLookingDown), cv2.COLOR_BGR2RGB), | |
| cv2.cvtColor(cv2.imread("Images/frame%d.jpg" % RightEyeLookingLeft), cv2.COLOR_BGR2RGB), | |
| cv2.cvtColor(cv2.imread("Images/frame%d.jpg" % RightEyeLookingUp), cv2.COLOR_BGR2RGB)]) | |
| # return the distances | |
| # Find average pupil size | |
| for i in range(10): | |
| pupil_sizes.remove(max(pupil_sizes)) | |
| pupil_sizes.remove(min(pupil_sizes)) | |
| pupil_average = mean(pupil_sizes) # this should be 11.7 mm | |
| pixels = 11.7 / pupil_average | |
| print("pixels (In MM) = ", pixels) | |
| d = { 'direction': ['Right', 'Down', 'Left', 'Up'] , | |
| 'mm' : [round(pd.to_numeric(pupilDiff[1]).min() * pixels,1), | |
| round(pd.to_numeric(pupilDiff[2]).max() * pixels,1), | |
| round(pd.to_numeric(pupilDiff[3]).max() * pixels,1), | |
| round(pd.to_numeric(pupilDiff[4]).min() * pixels,1) | |
| ]} | |
| ExteremeDistanceLeftEye = pd.DataFrame(data=d) | |
| d = {'direction': ['Right', 'Down', 'Left', 'Up'], | |
| 'mm': [round(pd.to_numeric(pupilDiff[5]).min() * pixels, 1), | |
| round(pd.to_numeric(pupilDiff[6]).max() * pixels, 1), | |
| round(pd.to_numeric(pupilDiff[7]).max() * pixels, 1), | |
| round(pd.to_numeric(pupilDiff[8]).min() * pixels, 1) | |
| ]} | |
| ExteremeDistanceRightEye = pd.DataFrame(data=d) | |
| print() #.idxmax(axis=0)) | |
| # Upmost buttom limbus | |
| # | |
| return ExteremeDistanceLeftEye, ExteremeDistanceRightEye, ExtermeImageLeftEye, ExtermeImageRightEye | |
| with gr.Blocks() as demo: | |
| gr.Markdown( | |
| """ | |
| # Range of Motion Video Analysis | |
| Capture a video of the following looks: stright, left, right, up & down | |
| """) | |
| video1 = gr.Video(shape = (1000,1000), source="webcam") | |
| b = gr.Button("Analyze Video") | |
| gr.Markdown( | |
| """ | |
| # Left eye results (in mm): | |
| """) | |
| LeftEyeGallery = gr.Gallery( | |
| label="Left eye", show_label=False, elem_id="left_eye_gallery" | |
| ).style(grid=[4], height="auto") | |
| movementDataLeft = gr.Dataframe(ExteremeDistanceLeftEye) | |
| gr.Markdown( | |
| """ | |
| # Right eye results (in mm): | |
| """) | |
| RightEyeGallery = gr.Gallery( | |
| label="Right eye", show_label=False, elem_id="right_eye_gallery" | |
| ).style(grid=[4], height="auto") | |
| movementDataRight = gr.Dataframe(ExteremeDistanceRightEye) | |
| out = [movementDataLeft, movementDataRight, LeftEyeGallery, RightEyeGallery] | |
| b.click(fn=handleVideo, inputs=video1, outputs=out) | |
| demo.launch() | |