import gradio as gr import mediapipe as mp import cv2 import pandas as pd from statistics import mean, stdev import numpy as np # Run simple face mesh mp_face_mesh = mp.solutions.face_mesh mp_drawing = mp.solutions.drawing_utils drawing_spec = mp_drawing.DrawingSpec(thickness=2, circle_radius=3) global pupilLocation, movementLeft, movementRight pupilLocation = pd.DataFrame() movementLeft = pd.DataFrame(index=['Up', 'Center', 'Down'], columns=['Left', 'Center', 'Right']) movementRight = pd.DataFrame(index=['Up', 'Center', 'Down'], columns=['Left', 'Center', 'Right']) # TO DO: # 1. Calibration screen def isEyeOpen(image): image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) hist = cv2.calcHist([image], [0], None, [256], [0, 256]) colors = np.where(hist > 10) return np.mean(colors) if np.mean(colors) < 15: return True else: return False def findIris(input_img1, input_img2, input_img3, input_img4, input_img5): global pupilLocation pupilLocation = pd.DataFrame() # Make sure it is empty images = [input_img1, input_img2, input_img3, input_img4, input_img5] output_images = [] pupil_sizes = [] with mp_face_mesh.FaceMesh(max_num_faces=1, refine_landmarks=True, static_image_mode=True, min_detection_confidence=0.45) as face_mesh: #for image in images: for id, image in enumerate(images): if image is None: continue results = face_mesh.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB)) if not results.multi_face_landmarks: continue annotated_image = image.copy() for face_landmarks in results.multi_face_landmarks: height, width, _ = annotated_image.shape nose = [int(face_landmarks.landmark[168].x * width), int(face_landmarks.landmark[168].y * height)] cv2.circle(annotated_image, (nose[0], nose[1]), 3, (0, 0, 255), -1) leftIrisPoints = [474, 475, 476, 477] rightIrisPoints = [469, 470, 471, 472] # right, top, left, bottom left_iris = [] for p in leftIrisPoints: point = [int(face_landmarks.landmark[p].x * width), int(face_landmarks.landmark[p].y * height)] left_iris.append(point) #cv2.circle(annotated_image, point, 1, (255, 0, 255), 2) right_iris = [] for p in rightIrisPoints: point = [int(face_landmarks.landmark[p].x * width), int(face_landmarks.landmark[p].y * height)] right_iris.append(point) #cv2.circle(annotated_image, point, 1, (255, 0, 255), 2) leftIris_leftside = (int(left_iris[2][0]), int(left_iris[2][1])) leftIris_rightside = (int(left_iris[0][0]), int(left_iris[0][1])) leftIris_top = (int(left_iris[1][0]), int(left_iris[1][1])) leftIris_bottom = (int(left_iris[3][0]), int(left_iris[3][1])) rightIris_leftside = (int(right_iris[2][0]), int(right_iris[2][1])) rightIris_rightside = (int(right_iris[0][0]), int(right_iris[0][1])) rightIris_top = (int(right_iris[1][0]), int(right_iris[1][1])) rightIris_bottom = (int(right_iris[3][0]), int(right_iris[3][1])) sizeIncrease = 0 leftEye = annotated_image[leftIris_top[1] - sizeIncrease : leftIris_bottom[1] + sizeIncrease, leftIris_leftside[0] - sizeIncrease : leftIris_rightside[0] + sizeIncrease] leftEyeOpen = isEyeOpen (leftEye) rightEye = annotated_image[rightIris_top[1] - sizeIncrease: rightIris_bottom[1] + sizeIncrease, rightIris_leftside[0] - sizeIncrease: rightIris_rightside[0] + sizeIncrease] rightEyeOpen = isEyeOpen(rightEye) ''' cv2.putText(annotated_image, "Left " + str(int(leftEyeOpen)), (rightIris_leftside[0] - 20, leftIris_top[1] - 10), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 0), 1, cv2.LINE_AA) ''' if leftEyeOpen: cv2.putText(annotated_image, "Left Open " + str(leftEyeOpen), (rightIris_leftside[0] - 20, leftIris_top[1] - 10), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 0), 1, cv2.LINE_AA) else: cv2.putText(annotated_image, "Left Closed " + str(leftEyeOpen), (rightIris_leftside[0] - 20, leftIris_top[1] - 10), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 0), 1, cv2.LINE_AA) ''' cv2.putText(annotated_image, "Right " + str(int(rightEyeOpen)), (rightIris_leftside[0] - 20, rightIris_top[1] + 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 0), 1, cv2.LINE_AA) ''' if rightEyeOpen: cv2.putText(annotated_image, "Right Open " + str(rightEyeOpen), (rightIris_leftside[0] - 20, rightIris_top[1] + 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 0), 1, cv2.LINE_AA) else: cv2.putText(annotated_image, "Right Closed " + str(rightEyeOpen), (rightIris_leftside[0] - 20, rightIris_top[1] + 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 0), 1, cv2.LINE_AA) #leftEye = cv2.cvtColor(leftEye, cv2.COLOR_BGR2GRAY) ''' ''' cv2.circle(annotated_image, (int((leftIris_leftside[0] + leftIris_rightside[0]) / 2), int((leftIris_top[1] + leftIris_bottom[1]) / 2)), # int(abs(leftIris_leftside[0] - leftIris_rightside[0])/2 1, (0, 255, 255), 2) cv2.circle(annotated_image, (int((rightIris_leftside[0] + rightIris_rightside[0]) / 2), int((rightIris_top[1] + rightIris_bottom[1]) / 2)), # int(abs(rightIris_leftside[0] - rightIris_rightside[0]) / 2 1, (0, 255, 255), 2) name = 'TBD' newRow = pd.Series([name, leftIris_leftside[0] - nose[0], leftIris_top[1] - nose[1], leftIris_rightside[0] - nose[0], leftIris_bottom[1] - nose[1], rightIris_leftside[0] - nose[0], rightIris_top[1] - nose[1], rightIris_rightside[0] - nose[0], rightIris_bottom[1] - nose[1] ]) newRow = newRow.to_frame().T pupilLocation = pd.concat([pupilLocation, newRow], axis=0, ignore_index=True) #print("Inside pupil Location = ", pupilLocation) #filename = directoy_name + 'Analysis/' + name[0:-4] + '-analysis.jpg' #cv2.imwrite(filename, annotated_image) left = leftIris_leftside[0] - 150 right = rightIris_rightside[0] + 150 + 150 up = leftIris_top[1] - 50 down = leftIris_bottom[1] + 50 boundryIncrease = 10 print("leftside = ", leftIris_leftside[0]) print("rightside =", leftIris_rightside[0]) #print("left = ", left) #print("right = ", right) ''' if (id == 1): # Looking Left so we want the right limbus print("Id == 1") annotated_image = annotated_image[ leftIris_top[1] : leftIris_bottom[1], leftIris_rightside[0] - boundryIncrease : leftIris_rightside[0] + boundryIncrease ] annotated_image = cv2.cvtColor(annotated_image, cv2.COLOR_BGR2GRAY) #annotated_image = cv2.GaussianBlur(annotated_image, (3, 3), sigmaX=0, sigmaY=0) annotated_image = cv2.Canny(annotated_image, 50, 125) #annotated_image = cv2.Sobel(src=annotated_image, ddepth=cv2.CV_64F, dx=0, dy=1, # ksize=5) # Sobel Edge Detection on the Y axis contours = cv2.findContours(image, cv2.RETR_LIST, cv2.RETR_EXTERNAL) contours = sorted(contours, key=cv2.contourArea, reverse=True) for i in range(len(contours)): contour = contours[i] random.seed(i) color = (255 * random.random(), 255 * random.random(), 255 * random.random()) cv2.drawContours(img, [contour], -1, color, 3) elif (id == 2): # Looking Right so we want the left limbus annotated_image = annotated_image[ leftIris_top[1]: leftIris_bottom[1], leftIris_leftside[0] - boundryIncrease: leftIris_leftside[0] + boundryIncrease ] annotated_image = cv2.cvtColor(annotated_image, cv2.COLOR_BGR2GRAY) # annotated_image = cv2.GaussianBlur(annotated_image, (3, 3), sigmaX=0, sigmaY=0) annotated_image = cv2.Canny(annotated_image, 50, 125) else: ''' annotated_image = annotated_image[up:down, left:right] x1 = (leftIris_leftside[0] - nose[0] + leftIris_rightside[0] - nose[0]) / 2 y1 = (leftIris_top[1] - nose[1] + leftIris_bottom[1] - nose[1]) / 2 x2 = (rightIris_leftside[0] - nose[0] + rightIris_rightside[0] - nose[0]) / 2 y2 = (rightIris_top[1] - nose[1] + rightIris_bottom[1] - nose[1]) / 2 print("Slope=", (y2 - y1) / (x2 - x1)) text = "Slope=" + str(round((y2 - y1) / (x2 - x1), 2)) #cv2.putText(annotated_image, text, # (5, 110), cv2.FONT_HERSHEY_SIMPLEX, # 1, (255, 255, 0), 1, cv2.LINE_AA) print("left iris size in pixels = ", abs(leftIris_leftside[0] - leftIris_rightside[0])) print("Right iris size in pixels = ", abs(rightIris_leftside[0] - rightIris_rightside[0])) pupil_sizes.append(abs(leftIris_leftside[0] - leftIris_rightside[0])) pupil_sizes.append(abs(rightIris_leftside[0] - rightIris_rightside[0])) output_images.append(annotated_image) # calculate final results from pupilLocations pupilDiff = pupilLocation.copy() pupilDiff = pupilDiff.drop(pupilDiff.columns[0], axis=1) # Remove file name for i in range(pupilDiff.shape[0] - 1): # Calculate deltas pupilDiff.loc[i + 1] = abs(pupilDiff.loc[i + 1] - pupilDiff.loc[0]) print("pupilDiff=", pupilDiff) pupilDiff = pupilDiff.drop(0, axis=0) # Remove first row was was used as reference row #print("pupilDiff (in pixels)=", pupilDiff) # Find average pupil size pupil_sizes.remove(max(pupil_sizes)) pupil_sizes.remove(min(pupil_sizes)) pupil_average = mean(pupil_sizes) # this should be 11.7 mm pixels = 11.7 / pupil_average print("pixels (In MM) = ", pixels) # Left Eye movement movementLeft.iloc[0, 0] = ' ' movementLeft.iloc[0, 2] = ' ' movementLeft.iloc[1, 1] = 0 # reference point movementLeft.iloc[2, 0] = ' ' movementLeft.iloc[2, 2] = ' ' # Y movement only movementLeft.iloc[0, 1] = round(abs(pupilLocation.iloc[0, 4] - pupilLocation.iloc[1, 4]) * pixels, 0) # Up movementLeft.iloc[2, 1] = round(abs(pupilLocation.iloc[0, 2] - pupilLocation.iloc[3, 2]) * pixels, 0) # Down # X movement only movementLeft.iloc[1, 0] = round(abs(pupilLocation.iloc[0, 3] - pupilLocation.iloc[1, 3]) * pixels, 1) # Left movementLeft.iloc[1, 2] = round(abs(pupilLocation.iloc[0, 1] - pupilLocation.iloc[2, 1]) * pixels, 1) # Right # Right Eye Movement movementRight.iloc[0, 0] = ' ' movementRight.iloc[0, 2] = ' ' movementRight.iloc[1, 1] = 0 # reference point movementRight.iloc[2, 0] = ' ' movementRight.iloc[2, 2] = ' ' # Y movement only movementRight.iloc[0, 1] = round(abs(pupilLocation.iloc[0, 8] - pupilLocation.iloc[1, 8]) * pixels, 0) # Up movementRight.iloc[2, 1] = round(abs(pupilLocation.iloc[0, 6] - pupilLocation.iloc[3, 6]) * pixels, 0) # Down # X movement only movementRight.iloc[1, 0] = round(abs(pupilLocation.iloc[0, 7] - pupilLocation.iloc[1, 7]) * pixels, 0) # Left movementRight.iloc[1, 2] = round(abs(pupilLocation.iloc[0, 5] - pupilLocation.iloc[2, 5]) * pixels, 0) # Right return output_images[0], output_images[1], output_images[2], output_images[3], output_images[4], pupilLocation, movementLeft, movementRight with gr.Blocks() as demo: gr.Markdown( """ # Range of Motion Image Analysis Take 5 pictures below looking stright, left, right, up & down """) with gr.Row(): with gr.Column(scale=1): img1 = gr.Image(shape=(1000, 1000), source='webcam', label='Front') with gr.Column(scale=1): out1 = gr.Image(label='Out-Front') with gr.Row(): with gr.Column(scale=1): img2 = gr.Image(shape=(1000, 1000), source='webcam', label='Left') with gr.Column(scale=1): out2 = gr.Image(label='Out-Left') with gr.Row(): with gr.Column(scale=1): img3 = gr.Image(shape=(1000, 1000), source='webcam', label='Right') with gr.Column(scale=1): out3 = gr.Image(label='Out-Right') with gr.Row(): with gr.Column(scale=1): img4 = gr.Image(shape=(1000, 1000), source='webcam', label='Up') with gr.Column(scale=1): out4 = gr.Image(label='Out-Up') with gr.Row(): with gr.Column(scale=1): img5 = gr.Image(shape=(1000, 1000), source='webcam', label='Down') with gr.Column(scale=1): out5 = gr.Image(label='Down-Right') b = gr.Button("Go!") gr.Markdown( """ Pupil Locations: """) pupilData = gr.Dataframe(pupilLocation) gr.Markdown( """ # Left eye results (in mm): """) movementDataLeft = gr.Dataframe(movementLeft) gr.Markdown( """ # Right eye results (in mm): """) movementDataRight = gr.Dataframe(movementRight) inp = [img1, img2, img3, img4, img5] out = [out1, out2, out3, out4, out5, pupilData, movementDataLeft, movementDataRight] b.click(fn=findIris, inputs=inp, outputs=out) demo.launch(auth=("Andrew", "Andrew")) #, share=True