import gradio as gr import pandas as pd import cv2 import mediapipe as mp import os from statistics import mean # Record video # Save video? # Break video into images # Run facemesh on all images and save locations # Run exterme locations # Run analysis on those compare to the first frame # Run simple face mesh mp_face_mesh = mp.solutions.face_mesh mp_drawing = mp.solutions.drawing_utils drawing_spec = mp_drawing.DrawingSpec(thickness=2, circle_radius=3) global pupilLocation pupilLocation = pd.DataFrame() pupil_sizes = [] ExteremeDistanceLeftEye = pd.DataFrame() ExteremeDistanceRightEye = pd.DataFrame() def video_identity(video): return video #demo = gr.Interface(video_identity, # gr.Video(shape = (1000,1000), source="webcam"), # "playable_video") def findIrisInFrame(image, counter): global pupilLocation, pupil_sizes #pupilLocation = pd.DataFrame() # Make sure it is empty with mp_face_mesh.FaceMesh(max_num_faces=1, refine_landmarks=True, static_image_mode=True, min_detection_confidence=0.45) as face_mesh: results = face_mesh.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB)) if not results.multi_face_landmarks: return None annotated_image = image.copy() for face_landmarks in results.multi_face_landmarks: height, width, _ = annotated_image.shape nose = [int(face_landmarks.landmark[168].x * width), int(face_landmarks.landmark[168].y * height)] cv2.circle(annotated_image, (nose[0], nose[1]), 3, (0, 0, 255), -1) leftIrisPoints = [474, 475, 476, 477] rightIrisPoints = [469, 470, 471, 472] # right, top, left, bottom left_iris = [] for p in leftIrisPoints: point = [int(face_landmarks.landmark[p].x * width), int(face_landmarks.landmark[p].y * height)] left_iris.append(point) cv2.circle(annotated_image, point, 1, (255, 0, 255), 2) right_iris = [] for p in rightIrisPoints: point = [int(face_landmarks.landmark[p].x * width), int(face_landmarks.landmark[p].y * height)] right_iris.append(point) cv2.circle(annotated_image, point, 1, (255, 0, 255), 2) leftIris_leftside = (int(left_iris[2][0]), int(left_iris[2][1])) leftIris_rightside = (int(left_iris[0][0]), int(left_iris[0][1])) leftIris_top = (int(left_iris[1][0]), int(left_iris[1][1])) leftIris_bottom = (int(left_iris[3][0]), int(left_iris[3][1])) rightIris_leftside = (int(right_iris[2][0]), int(right_iris[2][1])) rightIris_rightside = (int(right_iris[0][0]), int(right_iris[0][1])) rightIris_top = (int(right_iris[1][0]), int(right_iris[1][1])) rightIris_bottom = (int(right_iris[3][0]), int(right_iris[3][1])) cv2.circle(annotated_image, (int((leftIris_leftside[0] + leftIris_rightside[0]) / 2), int((leftIris_top[1] + leftIris_bottom[1]) / 2)), # int(abs(leftIris_leftside[0] - leftIris_rightside[0])/2 1, (0, 255, 255), 2) cv2.circle(annotated_image, (int((rightIris_leftside[0] + rightIris_rightside[0]) / 2), int((rightIris_top[1] + rightIris_bottom[1]) / 2)), # int(abs(rightIris_leftside[0] - rightIris_rightside[0]) / 2 1, (0, 255, 255), 2) pupil_sizes.append(abs(leftIris_leftside[0] - leftIris_rightside[0])) pupil_sizes.append(abs(rightIris_leftside[0] - rightIris_rightside[0])) name = "frame%d.jpg" % counter newRow = pd.Series([name, leftIris_leftside[0] - nose[0], leftIris_top[1] - nose[1], leftIris_rightside[0] - nose[0], leftIris_bottom[1] - nose[1], rightIris_leftside[0] - nose[0], rightIris_top[1] - nose[1], rightIris_rightside[0] - nose[0], rightIris_bottom[1] - nose[1] ]) newRow = newRow.to_frame().T pupilLocation = pd.concat([pupilLocation, newRow], axis=0, ignore_index=True) return newRow def handleVideo(input_video): global ExteremeDistanceLeftEye, ExteremeDistanceRightEye, pupilLocation, pupil_sizes pupilLocation = pd.DataFrame() # Make sure it is empty to begin with pupil_sizes = [] vidcap = cv2.VideoCapture(input_video) success, image = vidcap.read() count = 0 if not os.path.exists('Images'): os.makedirs('Images') #os.chdir('Images') # Slide video into frames and find iris in each frame while success: cv2.imwrite("Images/frame%d.jpg" % count, image) # save frame as JPEG file findIrisInFrame(image, count) success, image = vidcap.read() count += 1 print("file counter=", count) # Convert pupilLocation to pupilDiff pupilDiff = pupilLocation.copy() pupilDiff = pupilDiff.drop(pupilDiff.columns[0], axis=1) # Remove file name for i in range(pupilDiff.shape[0] - 1): # Calculate deltas pupilDiff.loc[i + 1] = (pupilDiff.loc[i + 1] - pupilDiff.loc[0]) pupilDiff = pupilDiff.drop(0, axis=0) # Remove the first row print(pupilDiff) # Find extreme iris locations (images and measurements) # Left eye LeftEyeLookingRight = pd.to_numeric(pupilDiff[1]).idxmin() LeftEyeLookingDown = pd.to_numeric(pupilDiff[2]).idxmax() LeftEyeLookingLeft = pd.to_numeric(pupilDiff[3]).idxmax() LeftEyeLookingUp = pd.to_numeric(pupilDiff[4]).idxmin() # Right eye RightEyeLookingRight = pd.to_numeric(pupilDiff[5]).idxmin() RightEyeLookingDown = pd.to_numeric(pupilDiff[6]).idxmax() RightEyeLookingLeft = pd.to_numeric(pupilDiff[7]).idxmax() RightEyeLookingUp = pd.to_numeric(pupilDiff[8]).idxmin() print("Left eye images = ", LeftEyeLookingRight, LeftEyeLookingDown, LeftEyeLookingLeft, LeftEyeLookingUp) print("Right eye images = ", RightEyeLookingRight, RightEyeLookingDown, RightEyeLookingLeft, RightEyeLookingUp) ExtermeImageLeftEye = list([cv2.cvtColor(cv2.imread("Images/frame%d.jpg" % LeftEyeLookingRight), cv2.COLOR_BGR2RGB), cv2.cvtColor(cv2.imread("Images/frame%d.jpg" % LeftEyeLookingDown), cv2.COLOR_BGR2RGB), cv2.cvtColor(cv2.imread("Images/frame%d.jpg" % LeftEyeLookingLeft), cv2.COLOR_BGR2RGB), cv2.cvtColor(cv2.imread("Images/frame%d.jpg" % LeftEyeLookingUp), cv2.COLOR_BGR2RGB)]) ExtermeImageRightEye = list([cv2.cvtColor(cv2.imread("Images/frame%d.jpg" % RightEyeLookingRight), cv2.COLOR_BGR2RGB), cv2.cvtColor(cv2.imread("Images/frame%d.jpg" % RightEyeLookingDown), cv2.COLOR_BGR2RGB), cv2.cvtColor(cv2.imread("Images/frame%d.jpg" % RightEyeLookingLeft), cv2.COLOR_BGR2RGB), cv2.cvtColor(cv2.imread("Images/frame%d.jpg" % RightEyeLookingUp), cv2.COLOR_BGR2RGB)]) # return the distances # Find average pupil size for i in range(10): pupil_sizes.remove(max(pupil_sizes)) pupil_sizes.remove(min(pupil_sizes)) pupil_average = mean(pupil_sizes) # this should be 11.7 mm pixels = 11.7 / pupil_average print("pixels (In MM) = ", pixels) d = { 'direction': ['Right', 'Down', 'Left', 'Up'] , 'mm' : [round(pd.to_numeric(pupilDiff[1]).min() * pixels,1), round(pd.to_numeric(pupilDiff[2]).max() * pixels,1), round(pd.to_numeric(pupilDiff[3]).max() * pixels,1), round(pd.to_numeric(pupilDiff[4]).min() * pixels,1) ]} ExteremeDistanceLeftEye = pd.DataFrame(data=d) d = {'direction': ['Right', 'Down', 'Left', 'Up'], 'mm': [round(pd.to_numeric(pupilDiff[5]).min() * pixels, 1), round(pd.to_numeric(pupilDiff[6]).max() * pixels, 1), round(pd.to_numeric(pupilDiff[7]).max() * pixels, 1), round(pd.to_numeric(pupilDiff[8]).min() * pixels, 1) ]} ExteremeDistanceRightEye = pd.DataFrame(data=d) print() #.idxmax(axis=0)) # Upmost buttom limbus # return ExteremeDistanceLeftEye, ExteremeDistanceRightEye, ExtermeImageLeftEye, ExtermeImageRightEye with gr.Blocks() as demo: gr.Markdown( """ # Range of Motion Video Analysis Capture a video of the following looks: stright, left, right, up & down """) video1 = gr.Video(shape = (1000,1000), source="webcam") b = gr.Button("Analyze Video") gr.Markdown( """ # Left eye results (in mm): """) LeftEyeGallery = gr.Gallery( label="Left eye", show_label=False, elem_id="left_eye_gallery" ).style(grid=[4], height="auto") movementDataLeft = gr.Dataframe(ExteremeDistanceLeftEye) gr.Markdown( """ # Right eye results (in mm): """) RightEyeGallery = gr.Gallery( label="Right eye", show_label=False, elem_id="right_eye_gallery" ).style(grid=[4], height="auto") movementDataRight = gr.Dataframe(ExteremeDistanceRightEye) out = [movementDataLeft, movementDataRight, LeftEyeGallery, RightEyeGallery] b.click(fn=handleVideo, inputs=video1, outputs=out) demo.launch()