From 7e15783553044539f8e81ddfaf3a88c86d7e87be Mon Sep 17 00:00:00 2001 From: Salvo Musumeci Date: Wed, 22 Mar 2023 16:56:30 +0000 Subject: [PATCH 01/31] style: :recycle: Improved .gitignore to accept new files and new structure --- .gitignore | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index 72fd018..15e5dc1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,14 +1,5 @@ -* -!.gitignore -!LICENSE -!README.md -!main.py -!helpers.py -!webapp -!webapp/** +__pycache__ node_modules build -!sleep_logs.csv -!*.ipynb -!requirements.txt -!.env_sample +*.log +.env \ No newline at end of file From f3972d52d3af7d40a980dfdc48d8f4c03054c2ba Mon Sep 17 00:00:00 2001 From: Salvo Musumeci Date: Wed, 22 Mar 2023 17:00:15 +0000 Subject: [PATCH 02/31] refactor: :recycle: moved SleepyBaby as package --- main.py | 511 +--------------------------------------- sleepy_baby/__init__.py | 504 +++++++++++++++++++++++++++++++++++++++ sleepy_baby/helpers.py | 164 +++++++++++++ 3 files changed, 677 insertions(+), 502 deletions(-) create mode 100644 sleepy_baby/__init__.py create mode 100644 sleepy_baby/helpers.py diff --git a/main.py b/main.py index 2c39615..cfe4641 100644 --- a/main.py +++ b/main.py @@ -1,26 +1,20 @@ import cv2 -import numpy as np -import time -from threading import Timer, Lock, Event, Thread +from threading import Thread import os -import mediapipe as mp from collections import deque import _thread import logging -import serial -import queue -import statistics from dotenv import load_dotenv # from cast_service import CastSoundService from http.server import HTTPServer, SimpleHTTPRequestHandler -from helpers import check_eyes_open, set_hatch, check_mouth_open, maintain_aspect_ratio_resize, gamma_correction -load_dotenv() +from sleepy_baby import SleepyBaby # Uncomment if want phone notifications during daytime wakings. # Configuration of telegram API key in this dir also needed. # import telegram_send +#Set-up the logger logfile = os.getenv("SLEEP_DATA_PATH") + '/sleepy_logs.log' logging.basicConfig(filename=logfile, filemode='a+', @@ -28,504 +22,17 @@ datefmt='%H:%M:%S', level=logging.INFO) +#Load configuration from .env file +load_dotenv() + # Queue shared between the frame publishing thread and the consuming thread # This is to get around an underlying bug, described at end of this file. frame_q = deque(maxlen=20) -class SleepyBaby(): - - # TODO: break up this class, so big ew - - # General high level heuristics: - # 1) no eyes -> no body found -> baby is awake - # 2) no eyes -> body found -> moving -> baby is awake - # 3) no eyes -> body found -> not moving -> baby is sleeping - # 4) eyes -> eyes open -> baby is awake (disregard body movement) - # 5) eyes -> eyes closed -> movement -> baby is awake - # 6) eyes -> eyes closed -> no movement -> baby is asleep - # 7) eyes -> eyes closed -> mouth open -> baby is awake - - def __init__(self): - self.frame_dim = (1920,1080) - self.next_frame = 0 - self.fps = 30 - self.mpPose = mp.solutions.pose - self.mpFace = mp.solutions.face_mesh - self.pose = self.mpPose.Pose(min_detection_confidence=0.7, min_tracking_confidence=0.7) - # TODO: try turning off refine_landmarks for performance, might not be needed - self.face = self.mpFace.FaceMesh(max_num_faces=1, refine_landmarks=True, min_detection_confidence=0.8, min_tracking_confidence=0.8) - self.mpDraw = mp.solutions.drawing_utils - self.mpDrawStyles = mp.solutions.drawing_styles - - self.eyes_open_q = deque(maxlen=30) - self.awake_q = deque(maxlen=40) - self.movement_q = deque(maxlen=40) - self.eyes_open_state = False - - self.multi_face_landmarks = [] - self.is_awake = False - self.ser = None # serial connection to arduino for controlling demon owl - - # If demon owl mode, setup connection to arduino and cast service for playing audio - if os.getenv("OWL", 'False').lower() in ('true', '1'): - print("\nCAWWWWWW\n") - self.cast_service = CastSoundService() - self.ser = serial.Serial('/dev/ttyACM0', 9600, timeout=0) - - self.top_lip = frozenset([ - (324, 308), (78, 191), (191, 80), (80, 81), (81, 82), - (82, 13), (13, 312), (312, 311), (311, 310), - (310, 415), (415, 308), - (375, 291), (61, 185), (185, 40), (40, 39), (39, 37), - (37, 0), (0, 267), - (267, 269), (269, 270), (270, 409), (409, 291), - ]) - self.bottom_lip = frozenset([ - (61, 146), (146, 91), (91, 181), (181, 84), (84, 17), - (17, 314), (314, 405), (405, 321), (321, 375), - (78, 95), (95, 88), (88, 178), (178, 87), (87, 14), - (14, 317), (317, 402), (402, 318), (318, 324), - ]) - - - # Decorator ensures function that can only be called once every `s` seconds. - def debounce(s): - def decorate(f): - t = None - - def wrapped(*args, **kwargs): - nonlocal t - t_ = time.time() - if t is None or t_ - t >= s: - result = f(*args, **kwargs) - t = time.time() - return result - return wrapped - return decorate - - - @debounce(1) - def throttled_handle_no_eyes_found(self): - logging.info('No face found, depreciate queue') - print('No face found, depreciate queue') - if(len(self.eyes_open_q) > 0): - self.eyes_open_q.popleft() - - - @debounce(1) - def throttled_handle_no_body_found(self): - logging.info('No body found, vote awake') - print('No body found, vote awake') - self.awake_q.append(1) - - - def process_baby_image_models(self, img, debug_img): - results = self.face.process(img) - results_pose = self.pose.process(img) - - body_found = True - if results_pose.pose_landmarks: - # 15 left-wrist, 16 right-wrist - shape = img.shape - left_wrist_coords = (shape[1] * results_pose.pose_landmarks.landmark[15].x, shape[0] * results_pose.pose_landmarks.landmark[15].y) - right_wrist_coords = (shape[1] * results_pose.pose_landmarks.landmark[16].x, shape[0] * results_pose.pose_landmarks.landmark[16].y) - - # print('left wrist: ', left_wrist_coords) - # print('right wrist: ', right_wrist_coords) - - self.movement_q.append((left_wrist_coords, right_wrist_coords)) - - debug_img = cv2.putText(debug_img, "Left wrist", (int(left_wrist_coords[0]), int(left_wrist_coords[1])), 2, 1, (255,0,0), 2, 2) - debug_img = cv2.putText(debug_img, "Right wrist", (int(right_wrist_coords[0]), int(right_wrist_coords[1])), 2, 1, (255,0,0), 2, 2) - - if os.getenv("DEBUG", 'False').lower() in ('true', '1'): - CUTOFF_THRESHOLD = 10 # head and face - MY_CONNECTIONS = frozenset([t for t in self.mpPose.POSE_CONNECTIONS if t[0] > CUTOFF_THRESHOLD and t[1] > CUTOFF_THRESHOLD]) - - # if results_pose.pose_landmarks: # if it finds the points - # for landmark_id, landmark in enumerate(results_pose.pose_landmarks): - # if landmark_id <= CUTOFF_THRESHOLD: - # landmark.visibility = 0 - # self.mpDraw.draw_landmarks(debug_img, results_pose.pose_landmarks, MY_CONNECTIONS) - - for id, lm in enumerate(results_pose.pose_landmarks.landmark): - if id <= CUTOFF_THRESHOLD: - lm.visibility = 0 - continue - h, w,c = debug_img.shape - # print(id, lm) - cx, cy = int(lm.x*w), int(lm.y*h) - cv2.circle(debug_img, (cx, cy), 5, (255,0,0), cv2.FILLED) - - self.mpDraw.draw_landmarks(debug_img, results_pose.pose_landmarks, MY_CONNECTIONS, landmark_drawing_spec=self.mpDraw.DrawingSpec(color=(255, 0, 0), thickness=2, circle_radius=2)) - - # self.mpDraw.draw_landmarks(debug_img, results_pose.pose_landmarks, MY_CONNECTIONS) - # for id, lm in enumerate(results_pose.pose_landmarks.landmark): - # if id < CUTOFF_THRESHOLD: - # continue - # self.mpDraw.draw_landmarks(debug_img, results_pose.pose_landmarks, MY_CONNECTIONS) - # h, w,c = debug_img.shape - # # print(id, lm) - # cx, cy = int(lm.x*w), int(lm.y*h) - # cv2.circle(debug_img, (cx, cy), 5, (255,0,0), cv2.FILLED) - else: - body_found = False - self.throttled_handle_no_body_found() - - LEFT_EYE = [362, 382, 381, 380, 374, 373, 390, 249, 263, 466, 388, 387, 386, 385, 384, 398] - RIGHT_EYE = [33, 7, 163, 144, 145, 153, 154, 155, 133, 173, 157, 158, 159, 160, 161, 246] - - if results.multi_face_landmarks: - self.multi_face_landmarks = results.multi_face_landmarks - - eyes_are_open = check_eyes_open(results.multi_face_landmarks[0].landmark, img, debug_img, LEFT_EYE, RIGHT_EYE) - - # Additionally check if mouth is closed. If not, consider baby crying. Can rely on queue length to ensure - # yawns don't trigger wake - - # If mouth is open, override and just consider it, "eyes open", pushing in direction of "wake vote" - if eyes_are_open == 0: # if eyes are closed, then check if mouth is open - mouth_is_open = check_mouth_open(results.multi_face_landmarks[0].landmark) - if mouth_is_open: - logging.info('Eyes closed, mouth open, crying or yawning, consider awake.') - self.eyes_open_q.append(1) - else: - logging.info('Eyes closed, mouth closed, consider sleeping.') - self.eyes_open_q.append(0) - else: - logging.info('Eyes open, consider awake.') - self.eyes_open_q.append(1) - - else: # no face results, interpret this as baby is not in crib, i.e. awake - self.throttled_handle_no_eyes_found() - - return debug_img, body_found - - - # This is placeholder until improve sensitivity of transitioning between waking and sleeping. - # Explanation: Sometimes when baby is waking up, he'll open and close his eyes for a couple of minutes... - # TODO: Fine-tune sensitivity of voting, for now, don't allow toggling between wake & sleep within N seconds - @debounce(180) - def need_to_clean_this_up(self, wake_status, img): - str_timestamp = str(int(time.time())) - sleep_data_base_path = os.getenv("SLEEP_DATA_PATH") - p = sleep_data_base_path + '/' + str_timestamp + '.png' - if wake_status: # woke up - log_string = "1," + str_timestamp + "\n" - print(log_string) - logging.info(log_string) - with open(sleep_data_base_path + '/sleep_logs.csv', 'a+', encoding="utf-8") as f: - f.write(log_string) - cv2.imwrite(p, img) # store off image of when wake/sleep event occurred. Can help with debugging issues - - # if daytime, send phone notification if baby woke up - # now = datetime.datetime.now() - # now_time = now.time() - # if now_time >= ti(7,00) or now_time <= ti(22,00): # day time - # Thread(target=telegram_send.send(messages=["Baby woke up."]), daemon=True).start() - - self.is_awake = True - - if os.getenv("OWL", 'False').lower() in ('true', '1'): - print("MOVE & MAKE NOISE") - logging.info("MOVE & MAKE NOISE") - time.sleep(5) - self.ser.write(bytes(str(999999) + "\n", "utf-8")) - self.cast_service.play_sound() - else: # fell asleep - log_string = "0," + str_timestamp + "\n" - print(log_string) - logging.info(log_string) - with open(sleep_data_base_path + '/sleep_logs.csv', 'a+', encoding="utf-8") as f: - f.write(log_string) - cv2.imwrite(p, img) - self.is_awake = False - - # now = datetime.datetime.now() - # now_time = now.time() - # if now_time >= ti(22,00) or now_time <= ti(8,00): # night time - # set_hatch(self.is_awake) - - - @debounce(10) - def set_wakeness_status(self, img): - if len(self.awake_q): - avg_awake = sum(self.awake_q) / len(self.awake_q) - if avg_awake >= 0.6 and self.is_awake == False: - self.need_to_clean_this_up(True, img) - elif avg_awake < 0.6 and self.is_awake == True: - self.need_to_clean_this_up(False, img) - - - @debounce(1) - def awake_voting_logic(self, debug_img): - if len(self.eyes_open_q) > len(self.eyes_open_q)/2: # dont vote on eyes unless queue is half full - avg = sum(self.eyes_open_q) / len(self.eyes_open_q) - if avg > 0.75: # eyes open - self.eyes_open_state = True - print("Eyes open: vote awake") - logging.info("\nvote awake") - self.awake_q.append(1) - else: # closed - self.eyes_open_state = False - self.awake_q.append(0) - print("Eyes closed: vote sleeping") - logging.info("\nvote sleeping") - else: - print("Not voting on eyes, eye queue too short.") - - - @debounce(1) - def movement_voting_logic(self, debug_img, body_found): - if not body_found: - print('No body found, depreciate movement queue.') - if len(self.movement_q): - self.movement_q.popleft() - - elif len(self.movement_q) > 5: - left_wrist_list = [c[0] for c in self.movement_q] - left_wrist_x_list = [c[0] for c in left_wrist_list] - left_wrist_y_list = [c[1] for c in left_wrist_list] - - right_wrist_list = [c[1] for c in self.movement_q] - right_wrist_x_list = [c[0] for c in right_wrist_list] - right_wrist_y_list = [c[1] for c in right_wrist_list] - - std_left_wrist_x = statistics.pstdev(left_wrist_x_list) - 1 - std_left_wrist_y = statistics.pstdev(left_wrist_y_list) - 1 - - std_right_wrist_x = statistics.pstdev(right_wrist_x_list) - 1 - std_right_wrist_y = statistics.pstdev(right_wrist_y_list) - 1 - - # average it all together and compare to movement threshold to determine if moving - avg_std = (((std_left_wrist_x + std_left_wrist_y)/2) + ((std_right_wrist_x + std_right_wrist_y)/2))/2 - # print('movement left: ', (std_left_wrist_x + std_left_wrist_y)/2) - # print('movement right: ', (std_right_wrist_x + std_right_wrist_y)/2) - # print('movement value: ', avg_std) - if int(avg_std) < 25: - print("No movement, vote sleeping") - logging.info('No movement, vote sleeping') - self.awake_q.append(0) - else: - print("Movement, vote awake") - logging.info("Movement, vote awake") - self.awake_q.append(1) - - - # every N seconds, check if baby is awake & do stuff - @debounce(5) - def periodic_wakeness_check(self): - print('\n', 'Is baby awake:', self.is_awake, '\n') - logging.info('Is baby awake: {}'.format(str(self.is_awake))) - - - def frame_logic(self, raw_img): - img = raw_img - - debug_img = img.copy() - img.flags.writeable = False - converted_img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) - - # beef - res = self.process_baby_image_models(converted_img, debug_img) - debug_img = res[0] - body_found = res[1] - - self.awake_voting_logic(debug_img) - self.movement_voting_logic(debug_img, body_found) - self.set_wakeness_status(debug_img) - self.periodic_wakeness_check() - - if os.getenv("DEBUG", 'False').lower() in ('true', '1'): - avg_awake = sum(self.awake_q) / len(self.awake_q) - - # draw progress bar - bar_y_offset = 0 - bar_y_offset = 100 - - bar_width = 200 - w = img.shape[1] - start_point = (int(w/2 - bar_width/2), 350 + bar_y_offset) - - end_point = (int(w/2 + bar_width/2), 370 + bar_y_offset) - adj_avg_awake = 1.0 if avg_awake / .6 >= 1.0 else avg_awake / .6 - progress_end_point = (int(w/2 - bar_width/2 + (bar_width*(adj_avg_awake))), 370 + bar_y_offset) - - color = (255, 255, 117) - progress_color = (0, 0, 255) - thickness = -1 - - debug_img = cv2.rectangle(debug_img, start_point, end_point, color, thickness) - debug_img = cv2.rectangle(debug_img, start_point, progress_end_point, progress_color, thickness) - display_perc = int((avg_awake * 100) / 0.6) - display_perc = 100 if display_perc >= 100 else display_perc - debug_img = cv2.putText(debug_img, str(display_perc) + "%", (int(w/2 - bar_width/2), 330 + bar_y_offset), 2, 1, (255,0,0), 2, 2) - debug_img = cv2.putText(debug_img, "Awake", (int(w/2 - bar_width/2 + 85), 330 + bar_y_offset), 2, 1, (255,0,0), 2, 2) - - return debug_img - - - # This basically does the same thing as the live version, but is very useful for testing - def recorded(self): - cap = cv2.VideoCapture(os.getenv("VIDEO_PATH")) - success, img = cap.read() - while success: - frame = None - while frame is None: - cur_time = time.time() - if cur_time > self.next_frame: - frame = img - self.next_frame = max( - self.next_frame + 1.0 / self.fps, cur_time + 0.5 / self.fps - ) - - success, img = cap.read() - - if all(e is not None for e in [frame, img]): - # bounds to actual run models/analysis on...no need to look for babies outside of the crib - x = 800 - y = 250 - h = 650 - w = 600 - - if img.shape[0] > 1080 and img.shape[1] > 1920: # max res 1080p - img = maintain_aspect_ratio_resize(img, width=self.frame_dim[0], height=self.frame_dim[1]) - - img_to_process = img[y:y+h, x:x+w] - - debug_img = self.frame_logic(img_to_process) - - # reapply cropped and modified/marked up img back to img which is displayed - img[y:y+h, x:x+w] = debug_img - - if os.getenv("DEBUG", 'False').lower() in ('true', '1'): - asleep = sum(self.awake_q) / len(self.awake_q) < 0.6 - text = 'Sleepy Baby' if asleep else 'Wakey Baby' - text_color = (255,191,0) if asleep else (0,140,255) - cv2.putText(img, text, (int(img.shape[0]/2) + 250, int(img.shape[1]/2)), 2, 3, text_color, 2, 2) - - cv2.rectangle(img=img, pt1=(x, y), pt2=(x+w, y+h), color=[153,50,204], thickness=2) - tmp = img[y:y+h, x:x+w] - img = gamma_correction(img, .4) - - for face_landmarks in self.multi_face_landmarks: - - # INDICIES: https://github.com/tensorflow/tfjs-models/blob/838611c02f51159afdd77469ce67f0e26b7bbb23/face-landmarks-detection/src/mediapipe-facemesh/keypoints.ts - # https://github.com/google/mediapipe/blob/master/mediapipe/python/solutions/face_mesh_connections.py - - self.mpDraw.draw_landmarks( - image=tmp, - landmark_list=face_landmarks, - connections=self.mpFace.FACEMESH_RIGHT_EYE, - landmark_drawing_spec=None, - connection_drawing_spec=self.mpDraw.DrawingSpec(color=(255, 150, 255), thickness=1, circle_radius=1)) - # connection_drawing_spec=self.mpDrawStyles - # .get_default_face_mesh_contours_style()) - self.mpDraw.draw_landmarks( - image=tmp, - landmark_list=face_landmarks, - connections=self.mpFace.FACEMESH_LEFT_EYE, - landmark_drawing_spec=None, - connection_drawing_spec=self.mpDraw.DrawingSpec(color=(255, 255, 0), thickness=1, circle_radius=1)) - # connection_drawing_spec=self.mpDrawStyles - # .get_default_face_mesh_contours_style()) - - self.mpDraw.draw_landmarks( - image=tmp, - landmark_list=face_landmarks, - connections=self.top_lip, - landmark_drawing_spec=None,#self.mpDraw.DrawingSpec(color=(255, 150, 255), thickness=2, circle_radius=2), - connection_drawing_spec=self.mpDraw.DrawingSpec(color=(255, 150, 255), thickness=1, circle_radius=1)) - self.mpDraw.draw_landmarks( - image=tmp, - landmark_list=face_landmarks, - connections=self.bottom_lip, - landmark_drawing_spec=None, - connection_drawing_spec=self.mpDraw.DrawingSpec(color=(255, 255, 0), thickness=1, circle_radius=1)) - - img[y:y+h, x:x+w] = tmp - - try: - img = cv2.resize(img, (960, 540)) - cv2.imshow('baby', img) - if cv2.waitKey(1) & 0xFF == ord('q'): - break - except Exception as e: - print("Something went wrong: ", e) - - - def live(self, consumer_q): - img = None - while True: - if len(consumer_q) > 0: - try: - img = consumer_q.pop() # consume image from queue - except IndexError as e: - print('No images in queue: ', e) - continue - - # bounds to actual run models/analysis on...no need to look for babies outside of the crib - x = 700 - y = 125 - h = 1000 - w = 800 - - if img.shape[0] > 1080 and img.shape[1] > 1920: # max res 1080p - img = maintain_aspect_ratio_resize(img, width=self.frame_dim[0], height=self.frame_dim[1]) - - img_to_process = img[y:y+h, x:x+w] - - debug_img = self.frame_logic(img_to_process) - - # reapply cropped and modified/marked up img back to img which is displayed - img[y:y+h, x:x+w] = debug_img - - if os.getenv("DEBUG", 'False').lower() in ('true', '1'): - try: - cv2.rectangle(img=img, pt1=(x, y), pt2=(x+w, y+h), color=[153,50,204], thickness=2) - tmp = img[y:y+h, x:x+w] - img = gamma_correction(img, .4) - - for face_landmarks in self.multi_face_landmarks: - - # INDICIES: https://github.com/tensorflow/tfjs-models/blob/838611c02f51159afdd77469ce67f0e26b7bbb23/face-landmarks-detection/src/mediapipe-facemesh/keypoints.ts - # https://github.com/google/mediapipe/blob/master/mediapipe/python/solutions/face_mesh_connections.py - - self.mpDraw.draw_landmarks( - image=tmp, - landmark_list=face_landmarks, - connections=self.mpFace.FACEMESH_RIGHT_EYE, - landmark_drawing_spec=None, - connection_drawing_spec=self.mpDraw.DrawingSpec(color=(255, 150, 255), thickness=1, circle_radius=1)) - # connection_drawing_spec=self.mpDrawStyles - # .get_default_face_mesh_contours_style()) - - self.mpDraw.draw_landmarks( - image=tmp, - landmark_list=face_landmarks, - connections=self.mpFace.FACEMESH_LEFT_EYE, - landmark_drawing_spec=None, - connection_drawing_spec=self.mpDraw.DrawingSpec(color=(255, 255, 0), thickness=1, circle_radius=1)) - # connection_drawing_spec=self.mpDrawStyles - # .get_default_face_mesh_contours_style()) - - img[y:y+h, x:x+w] = tmp - - img = cv2.resize(img, (960, 540)) - cv2.imshow('baby', maintain_aspect_ratio_resize(img, width=self.frame_dim[0], height=self.frame_dim[1])) - - if cv2.waitKey(1) & 0xFF == ord('q'): - break - except Exception as e: - print("Something went wrong: ", e) - - -#################################### -# TODO: move out of this file, break it up - -print('Initializing...') +#Load SleepyBaby +logging.info('Initializing...') sleepy_baby = SleepyBaby() -print('\nInitialization complete.') +logging.info('\nInitialization complete.') # Below http server is used for the web app to request latest sleep data diff --git a/sleepy_baby/__init__.py b/sleepy_baby/__init__.py new file mode 100644 index 0000000..e78291c --- /dev/null +++ b/sleepy_baby/__init__.py @@ -0,0 +1,504 @@ +import cv2 +import numpy as np +import time +from threading import Timer, Lock, Event, Thread +import os +import mediapipe as mp +from collections import deque +import _thread +import logging +import serial +import queue +import statistics +from dotenv import load_dotenv +# from cast_service import CastSoundService +from http.server import HTTPServer, SimpleHTTPRequestHandler +from helpers import check_eyes_open, set_hatch, check_mouth_open, maintain_aspect_ratio_resize, gamma_correction + + +class SleepyBaby(): + + # TODO: break up this class, so big ew + + # General high level heuristics: + # 1) no eyes -> no body found -> baby is awake + # 2) no eyes -> body found -> moving -> baby is awake + # 3) no eyes -> body found -> not moving -> baby is sleeping + # 4) eyes -> eyes open -> baby is awake (disregard body movement) + # 5) eyes -> eyes closed -> movement -> baby is awake + # 6) eyes -> eyes closed -> no movement -> baby is asleep + # 7) eyes -> eyes closed -> mouth open -> baby is awake + + def __init__(self): + self.frame_dim = (1920,1080) + self.next_frame = 0 + self.fps = 30 + self.mpPose = mp.solutions.pose + self.mpFace = mp.solutions.face_mesh + self.pose = self.mpPose.Pose(min_detection_confidence=0.7, min_tracking_confidence=0.7) + # TODO: try turning off refine_landmarks for performance, might not be needed + self.face = self.mpFace.FaceMesh(max_num_faces=1, refine_landmarks=True, min_detection_confidence=0.8, min_tracking_confidence=0.8) + self.mpDraw = mp.solutions.drawing_utils + self.mpDrawStyles = mp.solutions.drawing_styles + + self.eyes_open_q = deque(maxlen=30) + self.awake_q = deque(maxlen=40) + self.movement_q = deque(maxlen=40) + self.eyes_open_state = False + + self.multi_face_landmarks = [] + self.is_awake = False + self.ser = None # serial connection to arduino for controlling demon owl + + # If demon owl mode, setup connection to arduino and cast service for playing audio + if os.getenv("OWL", 'False').lower() in ('true', '1'): + print("\nCAWWWWWW\n") + self.cast_service = CastSoundService() + self.ser = serial.Serial('/dev/ttyACM0', 9600, timeout=0) + + self.top_lip = frozenset([ + (324, 308), (78, 191), (191, 80), (80, 81), (81, 82), + (82, 13), (13, 312), (312, 311), (311, 310), + (310, 415), (415, 308), + (375, 291), (61, 185), (185, 40), (40, 39), (39, 37), + (37, 0), (0, 267), + (267, 269), (269, 270), (270, 409), (409, 291), + ]) + self.bottom_lip = frozenset([ + (61, 146), (146, 91), (91, 181), (181, 84), (84, 17), + (17, 314), (314, 405), (405, 321), (321, 375), + (78, 95), (95, 88), (88, 178), (178, 87), (87, 14), + (14, 317), (317, 402), (402, 318), (318, 324), + ]) + + + # Decorator ensures function that can only be called once every `s` seconds. + def debounce(s): + def decorate(f): + t = None + + def wrapped(*args, **kwargs): + nonlocal t + t_ = time.time() + if t is None or t_ - t >= s: + result = f(*args, **kwargs) + t = time.time() + return result + return wrapped + return decorate + + + @debounce(1) + def throttled_handle_no_eyes_found(self): + logging.info('No face found, depreciate queue') + print('No face found, depreciate queue') + if(len(self.eyes_open_q) > 0): + self.eyes_open_q.popleft() + + + @debounce(1) + def throttled_handle_no_body_found(self): + logging.info('No body found, vote awake') + print('No body found, vote awake') + self.awake_q.append(1) + + + def process_baby_image_models(self, img, debug_img): + results = self.face.process(img) + results_pose = self.pose.process(img) + + body_found = True + if results_pose.pose_landmarks: + # 15 left-wrist, 16 right-wrist + shape = img.shape + left_wrist_coords = (shape[1] * results_pose.pose_landmarks.landmark[15].x, shape[0] * results_pose.pose_landmarks.landmark[15].y) + right_wrist_coords = (shape[1] * results_pose.pose_landmarks.landmark[16].x, shape[0] * results_pose.pose_landmarks.landmark[16].y) + + # print('left wrist: ', left_wrist_coords) + # print('right wrist: ', right_wrist_coords) + + self.movement_q.append((left_wrist_coords, right_wrist_coords)) + + debug_img = cv2.putText(debug_img, "Left wrist", (int(left_wrist_coords[0]), int(left_wrist_coords[1])), 2, 1, (255,0,0), 2, 2) + debug_img = cv2.putText(debug_img, "Right wrist", (int(right_wrist_coords[0]), int(right_wrist_coords[1])), 2, 1, (255,0,0), 2, 2) + + if os.getenv("DEBUG", 'False').lower() in ('true', '1'): + CUTOFF_THRESHOLD = 10 # head and face + MY_CONNECTIONS = frozenset([t for t in self.mpPose.POSE_CONNECTIONS if t[0] > CUTOFF_THRESHOLD and t[1] > CUTOFF_THRESHOLD]) + + # if results_pose.pose_landmarks: # if it finds the points + # for landmark_id, landmark in enumerate(results_pose.pose_landmarks): + # if landmark_id <= CUTOFF_THRESHOLD: + # landmark.visibility = 0 + # self.mpDraw.draw_landmarks(debug_img, results_pose.pose_landmarks, MY_CONNECTIONS) + + for id, lm in enumerate(results_pose.pose_landmarks.landmark): + if id <= CUTOFF_THRESHOLD: + lm.visibility = 0 + continue + h, w,c = debug_img.shape + # print(id, lm) + cx, cy = int(lm.x*w), int(lm.y*h) + cv2.circle(debug_img, (cx, cy), 5, (255,0,0), cv2.FILLED) + + self.mpDraw.draw_landmarks(debug_img, results_pose.pose_landmarks, MY_CONNECTIONS, landmark_drawing_spec=self.mpDraw.DrawingSpec(color=(255, 0, 0), thickness=2, circle_radius=2)) + + # self.mpDraw.draw_landmarks(debug_img, results_pose.pose_landmarks, MY_CONNECTIONS) + # for id, lm in enumerate(results_pose.pose_landmarks.landmark): + # if id < CUTOFF_THRESHOLD: + # continue + # self.mpDraw.draw_landmarks(debug_img, results_pose.pose_landmarks, MY_CONNECTIONS) + # h, w,c = debug_img.shape + # # print(id, lm) + # cx, cy = int(lm.x*w), int(lm.y*h) + # cv2.circle(debug_img, (cx, cy), 5, (255,0,0), cv2.FILLED) + else: + body_found = False + self.throttled_handle_no_body_found() + + LEFT_EYE = [362, 382, 381, 380, 374, 373, 390, 249, 263, 466, 388, 387, 386, 385, 384, 398] + RIGHT_EYE = [33, 7, 163, 144, 145, 153, 154, 155, 133, 173, 157, 158, 159, 160, 161, 246] + + if results.multi_face_landmarks: + self.multi_face_landmarks = results.multi_face_landmarks + + eyes_are_open = check_eyes_open(results.multi_face_landmarks[0].landmark, img, debug_img, LEFT_EYE, RIGHT_EYE) + + # Additionally check if mouth is closed. If not, consider baby crying. Can rely on queue length to ensure + # yawns don't trigger wake + + # If mouth is open, override and just consider it, "eyes open", pushing in direction of "wake vote" + if eyes_are_open == 0: # if eyes are closed, then check if mouth is open + mouth_is_open = check_mouth_open(results.multi_face_landmarks[0].landmark) + if mouth_is_open: + logging.info('Eyes closed, mouth open, crying or yawning, consider awake.') + self.eyes_open_q.append(1) + else: + logging.info('Eyes closed, mouth closed, consider sleeping.') + self.eyes_open_q.append(0) + else: + logging.info('Eyes open, consider awake.') + self.eyes_open_q.append(1) + + else: # no face results, interpret this as baby is not in crib, i.e. awake + self.throttled_handle_no_eyes_found() + + return debug_img, body_found + + + # This is placeholder until improve sensitivity of transitioning between waking and sleeping. + # Explanation: Sometimes when baby is waking up, he'll open and close his eyes for a couple of minutes... + # TODO: Fine-tune sensitivity of voting, for now, don't allow toggling between wake & sleep within N seconds + @debounce(180) + def need_to_clean_this_up(self, wake_status, img): + str_timestamp = str(int(time.time())) + sleep_data_base_path = os.getenv("SLEEP_DATA_PATH") + p = sleep_data_base_path + '/' + str_timestamp + '.png' + if wake_status: # woke up + log_string = "1," + str_timestamp + "\n" + print(log_string) + logging.info(log_string) + with open(sleep_data_base_path + '/sleep_logs.csv', 'a+', encoding="utf-8") as f: + f.write(log_string) + cv2.imwrite(p, img) # store off image of when wake/sleep event occurred. Can help with debugging issues + + # if daytime, send phone notification if baby woke up + # now = datetime.datetime.now() + # now_time = now.time() + # if now_time >= ti(7,00) or now_time <= ti(22,00): # day time + # Thread(target=telegram_send.send(messages=["Baby woke up."]), daemon=True).start() + + self.is_awake = True + + if os.getenv("OWL", 'False').lower() in ('true', '1'): + print("MOVE & MAKE NOISE") + logging.info("MOVE & MAKE NOISE") + time.sleep(5) + self.ser.write(bytes(str(999999) + "\n", "utf-8")) + self.cast_service.play_sound() + else: # fell asleep + log_string = "0," + str_timestamp + "\n" + print(log_string) + logging.info(log_string) + with open(sleep_data_base_path + '/sleep_logs.csv', 'a+', encoding="utf-8") as f: + f.write(log_string) + cv2.imwrite(p, img) + self.is_awake = False + + # now = datetime.datetime.now() + # now_time = now.time() + # if now_time >= ti(22,00) or now_time <= ti(8,00): # night time + # set_hatch(self.is_awake) + + + @debounce(10) + def set_wakeness_status(self, img): + if len(self.awake_q): + avg_awake = sum(self.awake_q) / len(self.awake_q) + if avg_awake >= 0.6 and self.is_awake == False: + self.need_to_clean_this_up(True, img) + elif avg_awake < 0.6 and self.is_awake == True: + self.need_to_clean_this_up(False, img) + + + @debounce(1) + def awake_voting_logic(self, debug_img): + if len(self.eyes_open_q) > len(self.eyes_open_q)/2: # dont vote on eyes unless queue is half full + avg = sum(self.eyes_open_q) / len(self.eyes_open_q) + if avg > 0.75: # eyes open + self.eyes_open_state = True + print("Eyes open: vote awake") + logging.info("\nvote awake") + self.awake_q.append(1) + else: # closed + self.eyes_open_state = False + self.awake_q.append(0) + print("Eyes closed: vote sleeping") + logging.info("\nvote sleeping") + else: + print("Not voting on eyes, eye queue too short.") + + + @debounce(1) + def movement_voting_logic(self, debug_img, body_found): + if not body_found: + print('No body found, depreciate movement queue.') + if len(self.movement_q): + self.movement_q.popleft() + + elif len(self.movement_q) > 5: + left_wrist_list = [c[0] for c in self.movement_q] + left_wrist_x_list = [c[0] for c in left_wrist_list] + left_wrist_y_list = [c[1] for c in left_wrist_list] + + right_wrist_list = [c[1] for c in self.movement_q] + right_wrist_x_list = [c[0] for c in right_wrist_list] + right_wrist_y_list = [c[1] for c in right_wrist_list] + + std_left_wrist_x = statistics.pstdev(left_wrist_x_list) - 1 + std_left_wrist_y = statistics.pstdev(left_wrist_y_list) - 1 + + std_right_wrist_x = statistics.pstdev(right_wrist_x_list) - 1 + std_right_wrist_y = statistics.pstdev(right_wrist_y_list) - 1 + + # average it all together and compare to movement threshold to determine if moving + avg_std = (((std_left_wrist_x + std_left_wrist_y)/2) + ((std_right_wrist_x + std_right_wrist_y)/2))/2 + # print('movement left: ', (std_left_wrist_x + std_left_wrist_y)/2) + # print('movement right: ', (std_right_wrist_x + std_right_wrist_y)/2) + # print('movement value: ', avg_std) + if int(avg_std) < 25: + print("No movement, vote sleeping") + logging.info('No movement, vote sleeping') + self.awake_q.append(0) + else: + print("Movement, vote awake") + logging.info("Movement, vote awake") + self.awake_q.append(1) + + + # every N seconds, check if baby is awake & do stuff + @debounce(5) + def periodic_wakeness_check(self): + print('\n', 'Is baby awake:', self.is_awake, '\n') + logging.info('Is baby awake: {}'.format(str(self.is_awake))) + + + def frame_logic(self, raw_img): + img = raw_img + + debug_img = img.copy() + img.flags.writeable = False + converted_img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + + # beef + res = self.process_baby_image_models(converted_img, debug_img) + debug_img = res[0] + body_found = res[1] + + self.awake_voting_logic(debug_img) + self.movement_voting_logic(debug_img, body_found) + self.set_wakeness_status(debug_img) + self.periodic_wakeness_check() + + if os.getenv("DEBUG", 'False').lower() in ('true', '1'): + avg_awake = sum(self.awake_q) / len(self.awake_q) + + # draw progress bar + bar_y_offset = 0 + bar_y_offset = 100 + + bar_width = 200 + w = img.shape[1] + start_point = (int(w/2 - bar_width/2), 350 + bar_y_offset) + + end_point = (int(w/2 + bar_width/2), 370 + bar_y_offset) + adj_avg_awake = 1.0 if avg_awake / .6 >= 1.0 else avg_awake / .6 + progress_end_point = (int(w/2 - bar_width/2 + (bar_width*(adj_avg_awake))), 370 + bar_y_offset) + + color = (255, 255, 117) + progress_color = (0, 0, 255) + thickness = -1 + + debug_img = cv2.rectangle(debug_img, start_point, end_point, color, thickness) + debug_img = cv2.rectangle(debug_img, start_point, progress_end_point, progress_color, thickness) + display_perc = int((avg_awake * 100) / 0.6) + display_perc = 100 if display_perc >= 100 else display_perc + debug_img = cv2.putText(debug_img, str(display_perc) + "%", (int(w/2 - bar_width/2), 330 + bar_y_offset), 2, 1, (255,0,0), 2, 2) + debug_img = cv2.putText(debug_img, "Awake", (int(w/2 - bar_width/2 + 85), 330 + bar_y_offset), 2, 1, (255,0,0), 2, 2) + + return debug_img + + + # This basically does the same thing as the live version, but is very useful for testing + def recorded(self): + cap = cv2.VideoCapture(os.getenv("VIDEO_PATH")) + success, img = cap.read() + while success: + frame = None + while frame is None: + cur_time = time.time() + if cur_time > self.next_frame: + frame = img + self.next_frame = max( + self.next_frame + 1.0 / self.fps, cur_time + 0.5 / self.fps + ) + + success, img = cap.read() + + if all(e is not None for e in [frame, img]): + # bounds to actual run models/analysis on...no need to look for babies outside of the crib + x = 800 + y = 250 + h = 650 + w = 600 + + if img.shape[0] > 1080 and img.shape[1] > 1920: # max res 1080p + img = maintain_aspect_ratio_resize(img, width=self.frame_dim[0], height=self.frame_dim[1]) + + img_to_process = img[y:y+h, x:x+w] + + debug_img = self.frame_logic(img_to_process) + + # reapply cropped and modified/marked up img back to img which is displayed + img[y:y+h, x:x+w] = debug_img + + if os.getenv("DEBUG", 'False').lower() in ('true', '1'): + asleep = sum(self.awake_q) / len(self.awake_q) < 0.6 + text = 'Sleepy Baby' if asleep else 'Wakey Baby' + text_color = (255,191,0) if asleep else (0,140,255) + cv2.putText(img, text, (int(img.shape[0]/2) + 250, int(img.shape[1]/2)), 2, 3, text_color, 2, 2) + + cv2.rectangle(img=img, pt1=(x, y), pt2=(x+w, y+h), color=[153,50,204], thickness=2) + tmp = img[y:y+h, x:x+w] + img = gamma_correction(img, .4) + + for face_landmarks in self.multi_face_landmarks: + + # INDICIES: https://github.com/tensorflow/tfjs-models/blob/838611c02f51159afdd77469ce67f0e26b7bbb23/face-landmarks-detection/src/mediapipe-facemesh/keypoints.ts + # https://github.com/google/mediapipe/blob/master/mediapipe/python/solutions/face_mesh_connections.py + + self.mpDraw.draw_landmarks( + image=tmp, + landmark_list=face_landmarks, + connections=self.mpFace.FACEMESH_RIGHT_EYE, + landmark_drawing_spec=None, + connection_drawing_spec=self.mpDraw.DrawingSpec(color=(255, 150, 255), thickness=1, circle_radius=1)) + # connection_drawing_spec=self.mpDrawStyles + # .get_default_face_mesh_contours_style()) + self.mpDraw.draw_landmarks( + image=tmp, + landmark_list=face_landmarks, + connections=self.mpFace.FACEMESH_LEFT_EYE, + landmark_drawing_spec=None, + connection_drawing_spec=self.mpDraw.DrawingSpec(color=(255, 255, 0), thickness=1, circle_radius=1)) + # connection_drawing_spec=self.mpDrawStyles + # .get_default_face_mesh_contours_style()) + + self.mpDraw.draw_landmarks( + image=tmp, + landmark_list=face_landmarks, + connections=self.top_lip, + landmark_drawing_spec=None,#self.mpDraw.DrawingSpec(color=(255, 150, 255), thickness=2, circle_radius=2), + connection_drawing_spec=self.mpDraw.DrawingSpec(color=(255, 150, 255), thickness=1, circle_radius=1)) + self.mpDraw.draw_landmarks( + image=tmp, + landmark_list=face_landmarks, + connections=self.bottom_lip, + landmark_drawing_spec=None, + connection_drawing_spec=self.mpDraw.DrawingSpec(color=(255, 255, 0), thickness=1, circle_radius=1)) + + img[y:y+h, x:x+w] = tmp + + try: + img = cv2.resize(img, (960, 540)) + cv2.imshow('baby', img) + if cv2.waitKey(1) & 0xFF == ord('q'): + break + except Exception as e: + print("Something went wrong: ", e) + + + def live(self, consumer_q): + img = None + while True: + if len(consumer_q) > 0: + try: + img = consumer_q.pop() # consume image from queue + except IndexError as e: + print('No images in queue: ', e) + continue + + # bounds to actual run models/analysis on...no need to look for babies outside of the crib + x = 700 + y = 125 + h = 1000 + w = 800 + + if img.shape[0] > 1080 and img.shape[1] > 1920: # max res 1080p + img = maintain_aspect_ratio_resize(img, width=self.frame_dim[0], height=self.frame_dim[1]) + + img_to_process = img[y:y+h, x:x+w] + + debug_img = self.frame_logic(img_to_process) + + # reapply cropped and modified/marked up img back to img which is displayed + img[y:y+h, x:x+w] = debug_img + + if os.getenv("DEBUG", 'False').lower() in ('true', '1'): + try: + cv2.rectangle(img=img, pt1=(x, y), pt2=(x+w, y+h), color=[153,50,204], thickness=2) + tmp = img[y:y+h, x:x+w] + img = gamma_correction(img, .4) + + for face_landmarks in self.multi_face_landmarks: + + # INDICIES: https://github.com/tensorflow/tfjs-models/blob/838611c02f51159afdd77469ce67f0e26b7bbb23/face-landmarks-detection/src/mediapipe-facemesh/keypoints.ts + # https://github.com/google/mediapipe/blob/master/mediapipe/python/solutions/face_mesh_connections.py + + self.mpDraw.draw_landmarks( + image=tmp, + landmark_list=face_landmarks, + connections=self.mpFace.FACEMESH_RIGHT_EYE, + landmark_drawing_spec=None, + connection_drawing_spec=self.mpDraw.DrawingSpec(color=(255, 150, 255), thickness=1, circle_radius=1)) + # connection_drawing_spec=self.mpDrawStyles + # .get_default_face_mesh_contours_style()) + + self.mpDraw.draw_landmarks( + image=tmp, + landmark_list=face_landmarks, + connections=self.mpFace.FACEMESH_LEFT_EYE, + landmark_drawing_spec=None, + connection_drawing_spec=self.mpDraw.DrawingSpec(color=(255, 255, 0), thickness=1, circle_radius=1)) + # connection_drawing_spec=self.mpDrawStyles + # .get_default_face_mesh_contours_style()) + + img[y:y+h, x:x+w] = tmp + + img = cv2.resize(img, (960, 540)) + cv2.imshow('baby', maintain_aspect_ratio_resize(img, width=self.frame_dim[0], height=self.frame_dim[1])) + + if cv2.waitKey(1) & 0xFF == ord('q'): + break + except Exception as e: + print("Something went wrong: ", e) \ No newline at end of file diff --git a/sleepy_baby/helpers.py b/sleepy_baby/helpers.py new file mode 100644 index 0000000..9b19fdf --- /dev/null +++ b/sleepy_baby/helpers.py @@ -0,0 +1,164 @@ +import math +import numpy as np +import cv2 +import os +from pyhatchbabyrest import PyHatchBabyRest +from dotenv import load_dotenv + +load_dotenv() + +def euclidean(point, point1): + x = point.x + y = point.y + x1 = point1.x + y1 = point1.y + + return math.sqrt((x1 - x)**2 + (y1 - y)**2) + + +# Given x/y coords of eyes, returns a ratio representing "openness" of eyes +def closed_ratio(img, debug_img, landmarks, left_eye_indices, right_eye_indices): + rh_right = landmarks[right_eye_indices[0]] + rh_left = landmarks[right_eye_indices[8]] + rv_top = landmarks[right_eye_indices[12]] + rv_bottom = landmarks[right_eye_indices[4]] + + lh_right = landmarks[left_eye_indices[0]] + lh_left = landmarks[left_eye_indices[8]] + lv_top = landmarks[left_eye_indices[12]] + lv_bottom = landmarks[left_eye_indices[4]] + + rhDistance = euclidean(rh_right, rh_left) + rvDistance = euclidean(rv_top, rv_bottom) + lvDistance = euclidean(lv_top, lv_bottom) + lhDistance = euclidean(lh_right, lh_left) + reRatio = rhDistance/rvDistance + leRatio = lhDistance/lvDistance + ratio = (reRatio + leRatio)/2 + + # print('reRatio: ', reRatio) + # print('leRatio: ', leRatio) + return ratio + + +def set_hatch(is_awake): + print("attempting to boost hatch brightness") + rest = PyHatchBabyRest(os.getenv('HATCH_IP')) + rest.set_brightness(5) + print("brightness: ", rest.brightness) + + return + + +def check_eyes_open(landmarks, img, debug_img, left_eye_indices, right_eye_indices): + eyes_closed_ratio = closed_ratio(img, debug_img, landmarks, left_eye_indices, right_eye_indices) + ratio_threshold = 5 + if eyes_closed_ratio > ratio_threshold: + return 0 # closed + else: + return 1 # open + + +def get_top_lip_height(landmarks): + # 39 -> 81 + # 0 -> 13 + # 269 -> 311 + + p39 = np.array([landmarks[39].x, landmarks[39].y, landmarks[39].z]) + p81 = np.array([landmarks[81].x, landmarks[81].y, landmarks[81].z]) + p0 = np.array([landmarks[0].x, landmarks[0].y, landmarks[0].z]) + p13 = np.array([landmarks[13].x, landmarks[13].y, landmarks[13].z]) + p269 = np.array([landmarks[269].x, landmarks[269].y, landmarks[269].z]) + p311 = np.array([landmarks[311].x, landmarks[311].y, landmarks[311].z]) + + d1 = np.linalg.norm(p39-p81) + d2 = np.linalg.norm(p0-p13) + d3 = np.linalg.norm(p269-p311) + + # print("average: ", (d1 + d2 + d3) / 3) + return (d1 + d2 + d3) / 3 + + +def get_bottom_lip_height(landmarks): + # 181 -> 178 + # 17 -> 14 + # 405 -> 402 + + p181 = np.array([landmarks[181].x, landmarks[181].y, landmarks[181].z]) + p178 = np.array([landmarks[178].x, landmarks[178].y, landmarks[178].z]) + p17 = np.array([landmarks[17].x, landmarks[17].y, landmarks[17].z]) + p14 = np.array([landmarks[14].x, landmarks[14].y, landmarks[14].z]) + p405 = np.array([landmarks[405].x, landmarks[405].y, landmarks[405].z]) + p402 = np.array([landmarks[402].x, landmarks[402].y, landmarks[402].z]) + + d1 = np.linalg.norm(p181-p178) + d2 = np.linalg.norm(p17-p14) + d3 = np.linalg.norm(p405-p402) + + # print("average: ", (d1 + d2 + d3) / 3) + return (d1 + d2 + d3) / 3 + + +def get_mouth_height(landmarks): + # 178 -> 81 + # 14 -> 13 + # 402 -> 311 + + p178 = np.array([landmarks[178].x, landmarks[178].y, landmarks[178].z]) + p81 = np.array([landmarks[81].x, landmarks[81].y, landmarks[81].z]) + p14 = np.array([landmarks[14].x, landmarks[14].y, landmarks[14].z]) + p13 = np.array([landmarks[13].x, landmarks[13].y, landmarks[13].z]) + p402 = np.array([landmarks[402].x, landmarks[402].y, landmarks[402].z]) + p311 = np.array([landmarks[311].x, landmarks[311].y, landmarks[311].z]) + + d1 = np.linalg.norm(p178-p81) + d2 = np.linalg.norm(p14-p13) + d3 = np.linalg.norm(p402-p311) + + # print("average: ", (d1 + d2 + d3) / 3) + return (d1 + d2 + d3) / 3 + + +def check_mouth_open(landmarks): + top_lip_height = get_top_lip_height(landmarks) + bottom_lip_height = get_bottom_lip_height(landmarks) + mouth_height = get_mouth_height(landmarks) + + # if mouth is open more than lip height * ratio, return true. + ratio = 0.8 + if mouth_height > min(top_lip_height, bottom_lip_height) * ratio: + return 1 + else: + return 0 + + +# Resizes a image and maintains aspect ratio +def maintain_aspect_ratio_resize(self, image, width=None, height=None, inter=cv2.INTER_AREA): + # Grab the image size and initialize dimensions + dim = None + (h, w) = image.shape[:2] + + # Return original image if no need to resize + if width is None and height is None: + return image + + # We are resizing height if width is none + if width is None: + # Calculate the ratio of the height and construct the dimensions + r = height / float(h) + dim = (int(w * r), height) + # We are resizing width if height is none + else: + # Calculate the ratio of the 0idth and construct the dimensions + r = width / float(w) + dim = (width, int(h * r)) + + # Return the resized image + return cv2.resize(image, dim, interpolation=inter) + + +def gamma_correction(self, og, gamma): + invGamma = 1 / gamma + table = [((i / 255) ** invGamma) * 255 for i in range(256)] + table = np.array(table, np.uint8) + return cv2.LUT(og, table) \ No newline at end of file From 50b30007fe795669645607bce51e9e55deaa3d4a Mon Sep 17 00:00:00 2001 From: Salvo Musumeci Date: Wed, 22 Mar 2023 22:59:07 +0000 Subject: [PATCH 03/31] refactor: :construction: Refactoring code in smaller and consistent classes --- helpers.py | 164 -------------------------- sleepy_baby/__init__.py | 2 +- sleepy_baby/helpers.py | 2 +- sleepy_baby/media_analysis.py | 212 ++++++++++++++++++++++++++++++++++ 4 files changed, 214 insertions(+), 166 deletions(-) delete mode 100644 helpers.py create mode 100644 sleepy_baby/media_analysis.py diff --git a/helpers.py b/helpers.py deleted file mode 100644 index 9b19fdf..0000000 --- a/helpers.py +++ /dev/null @@ -1,164 +0,0 @@ -import math -import numpy as np -import cv2 -import os -from pyhatchbabyrest import PyHatchBabyRest -from dotenv import load_dotenv - -load_dotenv() - -def euclidean(point, point1): - x = point.x - y = point.y - x1 = point1.x - y1 = point1.y - - return math.sqrt((x1 - x)**2 + (y1 - y)**2) - - -# Given x/y coords of eyes, returns a ratio representing "openness" of eyes -def closed_ratio(img, debug_img, landmarks, left_eye_indices, right_eye_indices): - rh_right = landmarks[right_eye_indices[0]] - rh_left = landmarks[right_eye_indices[8]] - rv_top = landmarks[right_eye_indices[12]] - rv_bottom = landmarks[right_eye_indices[4]] - - lh_right = landmarks[left_eye_indices[0]] - lh_left = landmarks[left_eye_indices[8]] - lv_top = landmarks[left_eye_indices[12]] - lv_bottom = landmarks[left_eye_indices[4]] - - rhDistance = euclidean(rh_right, rh_left) - rvDistance = euclidean(rv_top, rv_bottom) - lvDistance = euclidean(lv_top, lv_bottom) - lhDistance = euclidean(lh_right, lh_left) - reRatio = rhDistance/rvDistance - leRatio = lhDistance/lvDistance - ratio = (reRatio + leRatio)/2 - - # print('reRatio: ', reRatio) - # print('leRatio: ', leRatio) - return ratio - - -def set_hatch(is_awake): - print("attempting to boost hatch brightness") - rest = PyHatchBabyRest(os.getenv('HATCH_IP')) - rest.set_brightness(5) - print("brightness: ", rest.brightness) - - return - - -def check_eyes_open(landmarks, img, debug_img, left_eye_indices, right_eye_indices): - eyes_closed_ratio = closed_ratio(img, debug_img, landmarks, left_eye_indices, right_eye_indices) - ratio_threshold = 5 - if eyes_closed_ratio > ratio_threshold: - return 0 # closed - else: - return 1 # open - - -def get_top_lip_height(landmarks): - # 39 -> 81 - # 0 -> 13 - # 269 -> 311 - - p39 = np.array([landmarks[39].x, landmarks[39].y, landmarks[39].z]) - p81 = np.array([landmarks[81].x, landmarks[81].y, landmarks[81].z]) - p0 = np.array([landmarks[0].x, landmarks[0].y, landmarks[0].z]) - p13 = np.array([landmarks[13].x, landmarks[13].y, landmarks[13].z]) - p269 = np.array([landmarks[269].x, landmarks[269].y, landmarks[269].z]) - p311 = np.array([landmarks[311].x, landmarks[311].y, landmarks[311].z]) - - d1 = np.linalg.norm(p39-p81) - d2 = np.linalg.norm(p0-p13) - d3 = np.linalg.norm(p269-p311) - - # print("average: ", (d1 + d2 + d3) / 3) - return (d1 + d2 + d3) / 3 - - -def get_bottom_lip_height(landmarks): - # 181 -> 178 - # 17 -> 14 - # 405 -> 402 - - p181 = np.array([landmarks[181].x, landmarks[181].y, landmarks[181].z]) - p178 = np.array([landmarks[178].x, landmarks[178].y, landmarks[178].z]) - p17 = np.array([landmarks[17].x, landmarks[17].y, landmarks[17].z]) - p14 = np.array([landmarks[14].x, landmarks[14].y, landmarks[14].z]) - p405 = np.array([landmarks[405].x, landmarks[405].y, landmarks[405].z]) - p402 = np.array([landmarks[402].x, landmarks[402].y, landmarks[402].z]) - - d1 = np.linalg.norm(p181-p178) - d2 = np.linalg.norm(p17-p14) - d3 = np.linalg.norm(p405-p402) - - # print("average: ", (d1 + d2 + d3) / 3) - return (d1 + d2 + d3) / 3 - - -def get_mouth_height(landmarks): - # 178 -> 81 - # 14 -> 13 - # 402 -> 311 - - p178 = np.array([landmarks[178].x, landmarks[178].y, landmarks[178].z]) - p81 = np.array([landmarks[81].x, landmarks[81].y, landmarks[81].z]) - p14 = np.array([landmarks[14].x, landmarks[14].y, landmarks[14].z]) - p13 = np.array([landmarks[13].x, landmarks[13].y, landmarks[13].z]) - p402 = np.array([landmarks[402].x, landmarks[402].y, landmarks[402].z]) - p311 = np.array([landmarks[311].x, landmarks[311].y, landmarks[311].z]) - - d1 = np.linalg.norm(p178-p81) - d2 = np.linalg.norm(p14-p13) - d3 = np.linalg.norm(p402-p311) - - # print("average: ", (d1 + d2 + d3) / 3) - return (d1 + d2 + d3) / 3 - - -def check_mouth_open(landmarks): - top_lip_height = get_top_lip_height(landmarks) - bottom_lip_height = get_bottom_lip_height(landmarks) - mouth_height = get_mouth_height(landmarks) - - # if mouth is open more than lip height * ratio, return true. - ratio = 0.8 - if mouth_height > min(top_lip_height, bottom_lip_height) * ratio: - return 1 - else: - return 0 - - -# Resizes a image and maintains aspect ratio -def maintain_aspect_ratio_resize(self, image, width=None, height=None, inter=cv2.INTER_AREA): - # Grab the image size and initialize dimensions - dim = None - (h, w) = image.shape[:2] - - # Return original image if no need to resize - if width is None and height is None: - return image - - # We are resizing height if width is none - if width is None: - # Calculate the ratio of the height and construct the dimensions - r = height / float(h) - dim = (int(w * r), height) - # We are resizing width if height is none - else: - # Calculate the ratio of the 0idth and construct the dimensions - r = width / float(w) - dim = (width, int(h * r)) - - # Return the resized image - return cv2.resize(image, dim, interpolation=inter) - - -def gamma_correction(self, og, gamma): - invGamma = 1 / gamma - table = [((i / 255) ** invGamma) * 255 for i in range(256)] - table = np.array(table, np.uint8) - return cv2.LUT(og, table) \ No newline at end of file diff --git a/sleepy_baby/__init__.py b/sleepy_baby/__init__.py index e78291c..ba3d9e7 100644 --- a/sleepy_baby/__init__.py +++ b/sleepy_baby/__init__.py @@ -13,7 +13,7 @@ from dotenv import load_dotenv # from cast_service import CastSoundService from http.server import HTTPServer, SimpleHTTPRequestHandler -from helpers import check_eyes_open, set_hatch, check_mouth_open, maintain_aspect_ratio_resize, gamma_correction +from .helpers import check_eyes_open, set_hatch, check_mouth_open, maintain_aspect_ratio_resize, gamma_correction class SleepyBaby(): diff --git a/sleepy_baby/helpers.py b/sleepy_baby/helpers.py index 9b19fdf..7e4160c 100644 --- a/sleepy_baby/helpers.py +++ b/sleepy_baby/helpers.py @@ -133,7 +133,7 @@ def check_mouth_open(landmarks): # Resizes a image and maintains aspect ratio -def maintain_aspect_ratio_resize(self, image, width=None, height=None, inter=cv2.INTER_AREA): +def maintain_aspect_ratio_resize(image, width=None, height=None, inter=cv2.INTER_AREA): # Grab the image size and initialize dimensions dim = None (h, w) = image.shape[:2] diff --git a/sleepy_baby/media_analysis.py b/sleepy_baby/media_analysis.py new file mode 100644 index 0000000..1377b31 --- /dev/null +++ b/sleepy_baby/media_analysis.py @@ -0,0 +1,212 @@ +import numpy as np +import cv2 +import mediapipe as mp +import logging + +from .helpers import check_eyes_open, check_mouth_open + +LEFT_EYE = [362, 382, 381, 380, 374, 373, 390, 249, 263, 466, 388, 387, 386, 385, 384, 398] +RIGHT_EYE = [33, 7, 163, 144, 145, 153, 154, 155, 133, 173, 157, 158, 159, 160, 161, 246] + +class MediaAnalysis: + + def __init__(self, frame_width, frame_height, debug=False): + """ + __init__ Initialize Analyzer. + + Parameters + ---------- + + debug : bool, optional + show verbose log, by default False + """ + self.debug = debug + self.logger = logging.getLogger(MediaAnalysis.__name__) + self.set_working_area(0, 0, frame_width, frame_height) + + self.pose = mp.solutions.pose.Pose(min_detection_confidence=0.7, min_tracking_confidence=0.7) + # TODO: try turning off refine_landmarks for performance, might not be needed + self.face = mp.solutions.face_mesh.FaceMesh(max_num_faces=1, refine_landmarks=True, min_detection_confidence=0.8, min_tracking_confidence=0.8) + + self.pose_landmark = None + self.multi_face_landmarks = None + + def set_working_area(self, x, y, width, height): + """ + set_working_area will define a sub-area of frame to be analyze. + + This will help hardware to be faster and have a lower power consumption. + + Parameters + ---------- + x : int + offset for crop image on x-axis + y : int + offset for crop image on y-axis + width : int + width of the interesting area + height : int + height of the interesting area + """ + self.x = x + self.y = y + self.h = height + self.w = width + self.shape = [height, width] + + def process_baby_image_models(self, img): + """ + process_baby_image_models analyze frame and get information. + + Parameters + ---------- + img : numpy.ndarray + Image to be processed. It is already cropped + + Returns + ------- + dict + analysis dict that contains all findings of the image + """ + + analysis = { + "body_detected": False, + "left_wrist_coords": None, + "right_wrist_coords": None, + "face_detected": True, + "eyes_open": False, + "mouth_open": False + } + + results_pose = self.pose.process(img) + if results_pose.pose_landmarks: + analysis["body_detected"] = True + self.pose_landmarks = results_pose.pose_landmarks + # 15 left-wrist, 16 right-wrist + analysis["left_wrist_coords"] = (self.shape[1] * self.pose_landmarks.landmark[15].x, self.shape[0] * self.pose_landmarks.landmark[15].y) + analysis["right_wrist_coords"] = (self.shape[1] * self.pose_landmarks.landmark[16].x, self.shape[0] * self.pose_landmarks.landmark[16].y) + + results = self.face.process(img) + if results.multi_face_landmarks: + self.multi_face_landmarks = results.multi_face_landmarks + debug_img = img.copy() #FIXME: remove dependency in check_eyes_open + analysis["face_detected"] = True + analysis["eyes_open"] = check_eyes_open(results.multi_face_landmarks[0].landmark, img, debug_img, LEFT_EYE, RIGHT_EYE) + analysis["mouth_open"] = check_mouth_open(results.multi_face_landmarks[0].landmark) + else: + self.pose_landmark = None + self.multi_face_landmarks = None + analysis["body_found"] = False + return analysis + + def add_body_details_to_image(self, frame, analysis): + w_area = frame[self.y:self.y+self.h, self.x:self.x+self.w] + cv2.rectangle(w_area, [0, 0], self.shape, color=(0, 255, 0), thickness=5) #Draw Analysis Area + #Draw body lines + if analysis['body_detected']: + CUTOFF_THRESHOLD = 10 # head and face + MY_CONNECTIONS = [t for t in mp.solutions.pose.POSE_CONNECTIONS if t[0] > CUTOFF_THRESHOLD and t[1] > CUTOFF_THRESHOLD] + for id, lm in enumerate(self.pose_landmarks.landmark): + if id <= CUTOFF_THRESHOLD: + lm.visibility = 0 + continue + mp.solutions.drawing_utils.draw_landmarks(w_area, + self.pose_landmarks, + MY_CONNECTIONS, + landmark_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(color=(255, 0, 0), + thickness=10, + circle_radius=2), + connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(color=(0, 0, 255), + thickness=5, + circle_radius=2) + ) + w_area = cv2.putText(w_area, "Left wrist", (int(analysis["left_wrist_coords"][0]), int(analysis["left_wrist_coords"][1])), 2, 1, (255,0,0), 2, 2) + w_area = cv2.putText(w_area, "Right wrist", (int(analysis["right_wrist_coords"][0]), int(analysis["right_wrist_coords"][1])), 2, 1, (255,0,0), 2, 2) + frame[self.y:self.y+self.h, self.x:self.x+self.w] = w_area + return frame + + def add_face_details_to_image(self, frame, analysis): + """ + add_face_details_to_image adds face details to image passed in arguments + + Parameters + ---------- + frame : numpy.ndarray + starting image + analysis : dict + dictionary containing evaluation + + Returns + ------- + numpy.ndarray + image with some draws overlayed + """ + if analysis['face_detected']: + self.logger.info(f"Face Detected. Eyes are {'open' if analysis['eyes_open'] else 'close'} and month is {'open' if analysis['mouth_open'] else 'close'}") + w_area = frame[self.y:self.y+self.h, self.x:self.x+self.w] + for face_landmarks in self.multi_face_landmarks: + # INDICIES: https://github.com/tensorflow/tfjs-models/blob/838611c02f51159afdd77469ce67f0e26b7bbb23/face-landmarks-detection/src/mediapipe-facemesh/keypoints.ts + # https://github.com/google/mediapipe/blob/master/mediapipe/python/solutions/face_mesh_connections.py + + mp.solutions.drawing_utils.draw_landmarks( + image=w_area, + landmark_list=face_landmarks, + connections=mp.solutions.face_mesh.FACEMESH_RIGHT_EYE, + landmark_drawing_spec=None, + connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(color=(255, 150, 255), thickness=5, circle_radius=1)) + # connection_drawing_spec=self.mpDrawStyles + # .get_default_face_mesh_contours_style()) + + mp.solutions.drawing_utils.draw_landmarks( + image=w_area, + landmark_list=face_landmarks, + connections=mp.solutions.face_mesh.FACEMESH_LEFT_EYE, + landmark_drawing_spec=None, + connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(color=(255, 255, 0), thickness=5, circle_radius=1)) + # connection_drawing_spec=self.mpDrawStyles + # .get_default_face_mesh_contours_style()) + frame[self.y:self.y+self.h, self.x:self.x+self.w] = w_area + else: + self.logger.info("Face is not detected") + return frame + + def add_body_details_to_image(self, frame, analysis): + """ + add_body_details_to_image adds body details to image passed in arguments + + Parameters + ---------- + frame : numpy.ndarray + starting image + analysis : dict + dictionary containing evaluation + + Returns + ------- + numpy.ndarray + image with some draws overlayed + """ + w_area = frame[self.y:self.y+self.h, self.x:self.x+self.w] + cv2.rectangle(w_area, [0, 0], self.shape, color=(0, 255, 0), thickness=5) #Draw Analysis Area + #Draw body lines + if analysis['body_detected']: + CUTOFF_THRESHOLD = 10 # head and face + MY_CONNECTIONS = [t for t in mp.solutions.pose.POSE_CONNECTIONS if t[0] > CUTOFF_THRESHOLD and t[1] > CUTOFF_THRESHOLD] + for id, lm in enumerate(self.pose_landmarks.landmark): + if id <= CUTOFF_THRESHOLD: + lm.visibility = 0 + continue + mp.solutions.drawing_utils.draw_landmarks(w_area, + self.pose_landmarks, + MY_CONNECTIONS, + landmark_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(color=(255, 0, 0), + thickness=10, + circle_radius=2), + connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(color=(0, 0, 255), + thickness=5, + circle_radius=2) + ) + w_area = cv2.putText(w_area, "Left wrist", (int(analysis["left_wrist_coords"][0]), int(analysis["left_wrist_coords"][1])), 2, 1, (255,0,0), 2, 2) + w_area = cv2.putText(w_area, "Right wrist", (int(analysis["right_wrist_coords"][0]), int(analysis["right_wrist_coords"][1])), 2, 1, (255,0,0), 2, 2) + frame[self.y:self.y+self.h, self.x:self.x+self.w] = w_area + return frame From 31b07774dab212794f54b93ad5be72e659512ea4 Mon Sep 17 00:00:00 2001 From: Salvo Musumeci Date: Thu, 23 Mar 2023 10:07:56 +0000 Subject: [PATCH 04/31] perf: :zap: helper has been refactored and optimized --- sleepy_baby/helpers.py | 167 +++++++++++++++------------------- sleepy_baby/media_analysis.py | 9 +- 2 files changed, 73 insertions(+), 103 deletions(-) diff --git a/sleepy_baby/helpers.py b/sleepy_baby/helpers.py index 7e4160c..310cef0 100644 --- a/sleepy_baby/helpers.py +++ b/sleepy_baby/helpers.py @@ -1,4 +1,3 @@ -import math import numpy as np import cv2 import os @@ -7,38 +6,63 @@ load_dotenv() -def euclidean(point, point1): - x = point.x - y = point.y - x1 = point1.x - y1 = point1.y - - return math.sqrt((x1 - x)**2 + (y1 - y)**2) +def get_point_as_array(point): + """ + get_point_as_array transforms landmarks coordinate in numpy array. + + Parameters + ---------- + point : mediapipe.framework.formats.landmark_pb2.NormalizedLandmark + Landmarks coordinate + + Returns + ------- + numpy.array + array of coordinates [x, y, z] + """ + return np.array([point.x, point.y, point.z]) + +def get_distance_between_landmarks(landmark, ref0, ref1): + p0 = get_point_as_array(landmark[ref0]) + p1 = get_point_as_array(landmark[ref1]) + return np.linalg.norm(p0 - p1) + +def get_eyes_positions(landmarks): #TODO: comment it + #Indexes of landmarks are reported at https://raw.githubusercontent.com/google/mediapipe/a908d668c730da128dfa8d9f6bd25d519d006692/mediapipe/modules/face_geometry/data/canonical_face_model_uv_visualization.png + rh_right = get_point_as_array(landmarks[33]) + rh_left = get_point_as_array(landmarks[133]) + rv_top = get_point_as_array(landmarks[159]) + rv_bottom = get_point_as_array(landmarks[145]) + + lh_right = get_point_as_array(landmarks[362]) + lh_left = get_point_as_array(landmarks[263]) + lv_top = get_point_as_array(landmarks[386]) + lv_bottom = get_point_as_array(landmarks[374]) + + return {"Left Eye": {"right": lh_right, + "left": lh_left, + "top": lv_top, + "bottom": lv_bottom}, + "Right Eye": {"right": rh_right, + "left": rh_left, + "top": rv_top, + "bottom": rv_bottom} + } # Given x/y coords of eyes, returns a ratio representing "openness" of eyes -def closed_ratio(img, debug_img, landmarks, left_eye_indices, right_eye_indices): - rh_right = landmarks[right_eye_indices[0]] - rh_left = landmarks[right_eye_indices[8]] - rv_top = landmarks[right_eye_indices[12]] - rv_bottom = landmarks[right_eye_indices[4]] - - lh_right = landmarks[left_eye_indices[0]] - lh_left = landmarks[left_eye_indices[8]] - lv_top = landmarks[left_eye_indices[12]] - lv_bottom = landmarks[left_eye_indices[4]] - - rhDistance = euclidean(rh_right, rh_left) - rvDistance = euclidean(rv_top, rv_bottom) - lvDistance = euclidean(lv_top, lv_bottom) - lhDistance = euclidean(lh_right, lh_left) - reRatio = rhDistance/rvDistance - leRatio = lhDistance/lvDistance - ratio = (reRatio + leRatio)/2 - - # print('reRatio: ', reRatio) - # print('leRatio: ', leRatio) - return ratio +def closed_ratio(landmarks): #TODO: commen the function + #Indexes of landmarks are reported at https://raw.githubusercontent.com/google/mediapipe/a908d668c730da128dfa8d9f6bd25d519d006692/mediapipe/modules/face_geometry/data/canonical_face_model_uv_visualization.png + right_eye_width = get_distance_between_landmarks(landmarks, 33, 133) + right_eye_height = get_distance_between_landmarks(landmarks, 159, 145) + left_eye_width = get_distance_between_landmarks(landmarks, 362, 263) + left_eye_height = get_distance_between_landmarks(landmarks, 386, 374) + right_eye_ratio = right_eye_width / right_eye_height + left_eye_ratio = left_eye_width / left_eye_height + return (right_eye_ratio + left_eye_ratio) / 2 + +def check_eyes_open(landmarks, ratio_threshold=5): #TODO: comment it + return closed_ratio(landmarks) <= ratio_threshold def set_hatch(is_awake): @@ -49,87 +73,38 @@ def set_hatch(is_awake): return - -def check_eyes_open(landmarks, img, debug_img, left_eye_indices, right_eye_indices): - eyes_closed_ratio = closed_ratio(img, debug_img, landmarks, left_eye_indices, right_eye_indices) - ratio_threshold = 5 - if eyes_closed_ratio > ratio_threshold: - return 0 # closed - else: - return 1 # open - - def get_top_lip_height(landmarks): - # 39 -> 81 - # 0 -> 13 - # 269 -> 311 - - p39 = np.array([landmarks[39].x, landmarks[39].y, landmarks[39].z]) - p81 = np.array([landmarks[81].x, landmarks[81].y, landmarks[81].z]) - p0 = np.array([landmarks[0].x, landmarks[0].y, landmarks[0].z]) - p13 = np.array([landmarks[13].x, landmarks[13].y, landmarks[13].z]) - p269 = np.array([landmarks[269].x, landmarks[269].y, landmarks[269].z]) - p311 = np.array([landmarks[311].x, landmarks[311].y, landmarks[311].z]) - - d1 = np.linalg.norm(p39-p81) - d2 = np.linalg.norm(p0-p13) - d3 = np.linalg.norm(p269-p311) - - # print("average: ", (d1 + d2 + d3) / 3) - return (d1 + d2 + d3) / 3 + #Indexes of landmarks are reported at https://raw.githubusercontent.com/google/mediapipe/a908d668c730da128dfa8d9f6bd25d519d006692/mediapipe/modules/face_geometry/data/canonical_face_model_uv_visualization.png + top_lip_left = get_distance_between_landmarks(landmarks, 39, 81) + top_lip_center = get_distance_between_landmarks(landmarks, 0, 13) + top_lip_right = get_distance_between_landmarks(landmarks, 269, 311) + return (top_lip_left + top_lip_center + top_lip_right) / 3 def get_bottom_lip_height(landmarks): - # 181 -> 178 - # 17 -> 14 - # 405 -> 402 - - p181 = np.array([landmarks[181].x, landmarks[181].y, landmarks[181].z]) - p178 = np.array([landmarks[178].x, landmarks[178].y, landmarks[178].z]) - p17 = np.array([landmarks[17].x, landmarks[17].y, landmarks[17].z]) - p14 = np.array([landmarks[14].x, landmarks[14].y, landmarks[14].z]) - p405 = np.array([landmarks[405].x, landmarks[405].y, landmarks[405].z]) - p402 = np.array([landmarks[402].x, landmarks[402].y, landmarks[402].z]) + #Indexes of landmarks are reported at https://raw.githubusercontent.com/google/mediapipe/a908d668c730da128dfa8d9f6bd25d519d006692/mediapipe/modules/face_geometry/data/canonical_face_model_uv_visualization.png + bottom_lip_left = get_distance_between_landmarks(landmarks, 181, 178) + bottom_lip_center = get_distance_between_landmarks(landmarks, 17, 14) + bottom_lip_right = get_distance_between_landmarks(landmarks, 405, 402) + return (bottom_lip_left + bottom_lip_center + bottom_lip_right) / 3 - d1 = np.linalg.norm(p181-p178) - d2 = np.linalg.norm(p17-p14) - d3 = np.linalg.norm(p405-p402) - - # print("average: ", (d1 + d2 + d3) / 3) - return (d1 + d2 + d3) / 3 def get_mouth_height(landmarks): - # 178 -> 81 - # 14 -> 13 - # 402 -> 311 - - p178 = np.array([landmarks[178].x, landmarks[178].y, landmarks[178].z]) - p81 = np.array([landmarks[81].x, landmarks[81].y, landmarks[81].z]) - p14 = np.array([landmarks[14].x, landmarks[14].y, landmarks[14].z]) - p13 = np.array([landmarks[13].x, landmarks[13].y, landmarks[13].z]) - p402 = np.array([landmarks[402].x, landmarks[402].y, landmarks[402].z]) - p311 = np.array([landmarks[311].x, landmarks[311].y, landmarks[311].z]) + #Indexes of landmarks are reported at https://raw.githubusercontent.com/google/mediapipe/a908d668c730da128dfa8d9f6bd25d519d006692/mediapipe/modules/face_geometry/data/canonical_face_model_uv_visualization.png + open_mouth_left = get_distance_between_landmarks(landmarks, 178, 81) + open_mouth_center = get_distance_between_landmarks(landmarks, 14, 13) + open_mouth_right = get_distance_between_landmarks(landmarks, 402, 311) + return (open_mouth_left + open_mouth_center + open_mouth_right) / 3 - d1 = np.linalg.norm(p178-p81) - d2 = np.linalg.norm(p14-p13) - d3 = np.linalg.norm(p402-p311) - # print("average: ", (d1 + d2 + d3) / 3) - return (d1 + d2 + d3) / 3 - - -def check_mouth_open(landmarks): +def check_mouth_open(landmarks, ratio = 0.8): top_lip_height = get_top_lip_height(landmarks) bottom_lip_height = get_bottom_lip_height(landmarks) mouth_height = get_mouth_height(landmarks) # if mouth is open more than lip height * ratio, return true. - ratio = 0.8 - if mouth_height > min(top_lip_height, bottom_lip_height) * ratio: - return 1 - else: - return 0 + return mouth_height > min(top_lip_height, bottom_lip_height) * ratio # Resizes a image and maintains aspect ratio diff --git a/sleepy_baby/media_analysis.py b/sleepy_baby/media_analysis.py index 1377b31..88fbcba 100644 --- a/sleepy_baby/media_analysis.py +++ b/sleepy_baby/media_analysis.py @@ -5,9 +5,6 @@ from .helpers import check_eyes_open, check_mouth_open -LEFT_EYE = [362, 382, 381, 380, 374, 373, 390, 249, 263, 466, 388, 387, 386, 385, 384, 398] -RIGHT_EYE = [33, 7, 163, 144, 145, 153, 154, 155, 133, 173, 157, 158, 159, 160, 161, 246] - class MediaAnalysis: def __init__(self, frame_width, frame_height, debug=False): @@ -16,7 +13,6 @@ def __init__(self, frame_width, frame_height, debug=False): Parameters ---------- - debug : bool, optional show verbose log, by default False """ @@ -89,9 +85,8 @@ def process_baby_image_models(self, img): results = self.face.process(img) if results.multi_face_landmarks: self.multi_face_landmarks = results.multi_face_landmarks - debug_img = img.copy() #FIXME: remove dependency in check_eyes_open analysis["face_detected"] = True - analysis["eyes_open"] = check_eyes_open(results.multi_face_landmarks[0].landmark, img, debug_img, LEFT_EYE, RIGHT_EYE) + analysis["eyes_open"] = check_eyes_open(results.multi_face_landmarks[0].landmark) analysis["mouth_open"] = check_mouth_open(results.multi_face_landmarks[0].landmark) else: self.pose_landmark = None @@ -142,7 +137,7 @@ def add_face_details_to_image(self, frame, analysis): image with some draws overlayed """ if analysis['face_detected']: - self.logger.info(f"Face Detected. Eyes are {'open' if analysis['eyes_open'] else 'close'} and month is {'open' if analysis['mouth_open'] else 'close'}") + self.logger.info(f"Face Detected. Eyes are {'open' if analysis['eyes_open'] else 'close'} and mouth is {'open' if analysis['mouth_open'] else 'close'}") w_area = frame[self.y:self.y+self.h, self.x:self.x+self.w] for face_landmarks in self.multi_face_landmarks: # INDICIES: https://github.com/tensorflow/tfjs-models/blob/838611c02f51159afdd77469ce67f0e26b7bbb23/face-landmarks-detection/src/mediapipe-facemesh/keypoints.ts From ccece2aa63ffb2a714c6a32c702abc166c6f3fe9 Mon Sep 17 00:00:00 2001 From: Salvo Musumeci Date: Thu, 23 Mar 2023 10:08:50 +0000 Subject: [PATCH 05/31] chore: :bug: Ignore supporting file of jupyter --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 15e5dc1..378dfe9 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ __pycache__ node_modules build *.log -.env \ No newline at end of file +.env +.ipynb_checkpoints From fd56431c9478f324b7245cf8d94da4bc82239939 Mon Sep 17 00:00:00 2001 From: Salvo Musumeci Date: Thu, 23 Mar 2023 21:16:38 +0000 Subject: [PATCH 06/31] refactor: :art: Refactoring --- sleepy_baby/media_analysis.py | 185 +++++++++++++++++----------------- 1 file changed, 93 insertions(+), 92 deletions(-) diff --git a/sleepy_baby/media_analysis.py b/sleepy_baby/media_analysis.py index 88fbcba..10c0ab7 100644 --- a/sleepy_baby/media_analysis.py +++ b/sleepy_baby/media_analysis.py @@ -6,6 +6,12 @@ from .helpers import check_eyes_open, check_mouth_open class MediaAnalysis: + """ + It analyzes frame provided on process_frame. + + Results are saved inside the object variable "analysis". + It will be used later for decision logic to make the proper evaluation. + """ def __init__(self, frame_width, frame_height, debug=False): """ @@ -17,21 +23,36 @@ def __init__(self, frame_width, frame_height, debug=False): show verbose log, by default False """ self.debug = debug - self.logger = logging.getLogger(MediaAnalysis.__name__) + self.logger = logging.getLogger(self.__class__.__qualname__) self.set_working_area(0, 0, frame_width, frame_height) - self.pose = mp.solutions.pose.Pose(min_detection_confidence=0.7, min_tracking_confidence=0.7) + self.pose = mp.solutions.pose.Pose(min_detection_confidence=0.7, + min_tracking_confidence=0.7) # TODO: try turning off refine_landmarks for performance, might not be needed - self.face = mp.solutions.face_mesh.FaceMesh(max_num_faces=1, refine_landmarks=True, min_detection_confidence=0.8, min_tracking_confidence=0.8) + self.face = mp.solutions.face_mesh.FaceMesh(max_num_faces=1, + refine_landmarks=True, + min_detection_confidence=0.8, + min_tracking_confidence=0.8) - self.pose_landmark = None + self.pose_landmarks = None self.multi_face_landmarks = None + self._reset_analysis() + + def _reset_analysis(self): + self.analysis = { + "body_detected": False, + "left_wrist_coords": None, + "right_wrist_coords": None, + "face_detected": True, + "eyes_open": False, + "mouth_open": False + } - def set_working_area(self, x, y, width, height): + def set_working_area(self, x_offset, y_offset, width, height): """ set_working_area will define a sub-area of frame to be analyze. - This will help hardware to be faster and have a lower power consumption. + This will help hardware to be faster and have a lower power consumption Parameters ---------- @@ -44,85 +65,106 @@ def set_working_area(self, x, y, width, height): height : int height of the interesting area """ - self.x = x - self.y = y - self.h = height - self.w = width + self.x_offset = x_offset + self.y_offset = y_offset + self.height = height + self.width = width self.shape = [height, width] - def process_baby_image_models(self, img): + def get_working_image(self, img): + return img[self.y_offset:self.y_offset+self.height, self.x_offset:self.x_offset+self.width] + + def apply_working_area_to_image(self, w_area, frame): + frame[self.y_offset:self.y_offset+self.height, self.x_offset:self.x_offset+self.width] = w_area + + def process_frame(self, img): + w_area = self.get_working_image(img) + return self._process_baby_image_models(w_area) + + def _process_baby_image_models(self, img): """ process_baby_image_models analyze frame and get information. + Results are stored in analysis variable inside object + Parameters ---------- img : numpy.ndarray Image to be processed. It is already cropped - Returns - ------- - dict - analysis dict that contains all findings of the image """ - analysis = { - "body_detected": False, - "left_wrist_coords": None, - "right_wrist_coords": None, - "face_detected": True, - "eyes_open": False, - "mouth_open": False - } - + self._reset_analysis() results_pose = self.pose.process(img) if results_pose.pose_landmarks: - analysis["body_detected"] = True + self.analysis["body_detected"] = True self.pose_landmarks = results_pose.pose_landmarks # 15 left-wrist, 16 right-wrist - analysis["left_wrist_coords"] = (self.shape[1] * self.pose_landmarks.landmark[15].x, self.shape[0] * self.pose_landmarks.landmark[15].y) - analysis["right_wrist_coords"] = (self.shape[1] * self.pose_landmarks.landmark[16].x, self.shape[0] * self.pose_landmarks.landmark[16].y) + self.analysis["left_wrist_coords"] = (self.shape[1] * self.pose_landmarks.landmark[15].x, self.shape[0] * self.pose_landmarks.landmark[15].y) + self.analysis["right_wrist_coords"] = (self.shape[1] * self.pose_landmarks.landmark[16].x, self.shape[0] * self.pose_landmarks.landmark[16].y) results = self.face.process(img) if results.multi_face_landmarks: self.multi_face_landmarks = results.multi_face_landmarks - analysis["face_detected"] = True - analysis["eyes_open"] = check_eyes_open(results.multi_face_landmarks[0].landmark) - analysis["mouth_open"] = check_mouth_open(results.multi_face_landmarks[0].landmark) + self.analysis["face_detected"] = True + self.analysis["eyes_open"] = check_eyes_open(results.multi_face_landmarks[0].landmark) + self.analysis["mouth_open"] = check_mouth_open(results.multi_face_landmarks[0].landmark) else: - self.pose_landmark = None + self.pose_landmarks = None self.multi_face_landmarks = None - analysis["body_found"] = False - return analysis + self.analysis["body_found"] = False + + + def add_body_details_to_image(self, frame): + """ + add_body_details_to_image adds body details to image passed in arguments. + + Parameters + ---------- + frame : numpy.ndarray + starting image + analysis : dict + dictionary containing evaluation - def add_body_details_to_image(self, frame, analysis): - w_area = frame[self.y:self.y+self.h, self.x:self.x+self.w] + Returns + ------- + numpy.ndarray + image with some draws overlayed + """ + w_area = self.get_working_image(frame) cv2.rectangle(w_area, [0, 0], self.shape, color=(0, 255, 0), thickness=5) #Draw Analysis Area #Draw body lines - if analysis['body_detected']: + if self.analysis['body_detected']: CUTOFF_THRESHOLD = 10 # head and face MY_CONNECTIONS = [t for t in mp.solutions.pose.POSE_CONNECTIONS if t[0] > CUTOFF_THRESHOLD and t[1] > CUTOFF_THRESHOLD] for id, lm in enumerate(self.pose_landmarks.landmark): if id <= CUTOFF_THRESHOLD: lm.visibility = 0 continue - mp.solutions.drawing_utils.draw_landmarks(w_area, - self.pose_landmarks, + mp.solutions.drawing_utils.draw_landmarks(w_area, + self.pose_landmarks, MY_CONNECTIONS, - landmark_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(color=(255, 0, 0), - thickness=10, + landmark_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(color=(255, 0, 0), + thickness=10, circle_radius=2), - connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(color=(0, 0, 255), - thickness=5, + connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(color=(0, 0, 255), + thickness=5, circle_radius=2) ) - w_area = cv2.putText(w_area, "Left wrist", (int(analysis["left_wrist_coords"][0]), int(analysis["left_wrist_coords"][1])), 2, 1, (255,0,0), 2, 2) - w_area = cv2.putText(w_area, "Right wrist", (int(analysis["right_wrist_coords"][0]), int(analysis["right_wrist_coords"][1])), 2, 1, (255,0,0), 2, 2) - frame[self.y:self.y+self.h, self.x:self.x+self.w] = w_area + w_area = cv2.putText(w_area, "Left wrist", + (int(self.analysis["left_wrist_coords"][0]), + int(self.analysis["left_wrist_coords"][1])), + 2, 1, (255,0,0), 2, 2) + w_area = cv2.putText(w_area, "Right wrist", + (int(self.analysis["right_wrist_coords"][0]), + int(self.analysis["right_wrist_coords"][1])), + 2, 1, (255,0,0), 2, 2) + self.apply_working_area_to_image(w_area, frame) return frame - def add_face_details_to_image(self, frame, analysis): + def add_face_details_to_image(self, frame): """ - add_face_details_to_image adds face details to image passed in arguments + add_face_details_to_image adds face details to image passed in arguments. Parameters ---------- @@ -136,9 +178,9 @@ def add_face_details_to_image(self, frame, analysis): numpy.ndarray image with some draws overlayed """ - if analysis['face_detected']: - self.logger.info(f"Face Detected. Eyes are {'open' if analysis['eyes_open'] else 'close'} and mouth is {'open' if analysis['mouth_open'] else 'close'}") - w_area = frame[self.y:self.y+self.h, self.x:self.x+self.w] + if self.analysis['face_detected']: + self.logger.info(f"Face Detected. Eyes are {'open' if self.analysis['eyes_open'] else 'close'} and mouth is {'open' if self.analysis['mouth_open'] else 'close'}") + w_area = self.get_working_image(frame) for face_landmarks in self.multi_face_landmarks: # INDICIES: https://github.com/tensorflow/tfjs-models/blob/838611c02f51159afdd77469ce67f0e26b7bbb23/face-landmarks-detection/src/mediapipe-facemesh/keypoints.ts # https://github.com/google/mediapipe/blob/master/mediapipe/python/solutions/face_mesh_connections.py @@ -160,48 +202,7 @@ def add_face_details_to_image(self, frame, analysis): connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(color=(255, 255, 0), thickness=5, circle_radius=1)) # connection_drawing_spec=self.mpDrawStyles # .get_default_face_mesh_contours_style()) - frame[self.y:self.y+self.h, self.x:self.x+self.w] = w_area + self.apply_working_area_to_image(w_area, frame) else: self.logger.info("Face is not detected") return frame - - def add_body_details_to_image(self, frame, analysis): - """ - add_body_details_to_image adds body details to image passed in arguments - - Parameters - ---------- - frame : numpy.ndarray - starting image - analysis : dict - dictionary containing evaluation - - Returns - ------- - numpy.ndarray - image with some draws overlayed - """ - w_area = frame[self.y:self.y+self.h, self.x:self.x+self.w] - cv2.rectangle(w_area, [0, 0], self.shape, color=(0, 255, 0), thickness=5) #Draw Analysis Area - #Draw body lines - if analysis['body_detected']: - CUTOFF_THRESHOLD = 10 # head and face - MY_CONNECTIONS = [t for t in mp.solutions.pose.POSE_CONNECTIONS if t[0] > CUTOFF_THRESHOLD and t[1] > CUTOFF_THRESHOLD] - for id, lm in enumerate(self.pose_landmarks.landmark): - if id <= CUTOFF_THRESHOLD: - lm.visibility = 0 - continue - mp.solutions.drawing_utils.draw_landmarks(w_area, - self.pose_landmarks, - MY_CONNECTIONS, - landmark_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(color=(255, 0, 0), - thickness=10, - circle_radius=2), - connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(color=(0, 0, 255), - thickness=5, - circle_radius=2) - ) - w_area = cv2.putText(w_area, "Left wrist", (int(analysis["left_wrist_coords"][0]), int(analysis["left_wrist_coords"][1])), 2, 1, (255,0,0), 2, 2) - w_area = cv2.putText(w_area, "Right wrist", (int(analysis["right_wrist_coords"][0]), int(analysis["right_wrist_coords"][1])), 2, 1, (255,0,0), 2, 2) - frame[self.y:self.y+self.h, self.x:self.x+self.w] = w_area - return frame From 47719135464cfe1904dbdae7eb37b32c42014e98 Mon Sep 17 00:00:00 2001 From: Salvo Musumeci Date: Thu, 23 Mar 2023 22:49:31 +0000 Subject: [PATCH 07/31] refactor: :art: WIP: first draft of Decision logic --- sleepy_baby/__init__.py | 453 +++++++++------------------------- sleepy_baby/decision_logic.py | 177 +++++++++++++ 2 files changed, 289 insertions(+), 341 deletions(-) create mode 100644 sleepy_baby/decision_logic.py diff --git a/sleepy_baby/__init__.py b/sleepy_baby/__init__.py index ba3d9e7..5ed98e1 100644 --- a/sleepy_baby/__init__.py +++ b/sleepy_baby/__init__.py @@ -15,24 +15,56 @@ from http.server import HTTPServer, SimpleHTTPRequestHandler from .helpers import check_eyes_open, set_hatch, check_mouth_open, maintain_aspect_ratio_resize, gamma_correction +from .media_analysis import MediaAnalysis +from .decision_logic import DecisionLogic -class SleepyBaby(): - - # TODO: break up this class, so big ew - # General high level heuristics: - # 1) no eyes -> no body found -> baby is awake - # 2) no eyes -> body found -> moving -> baby is awake - # 3) no eyes -> body found -> not moving -> baby is sleeping - # 4) eyes -> eyes open -> baby is awake (disregard body movement) - # 5) eyes -> eyes closed -> movement -> baby is awake - # 6) eyes -> eyes closed -> no movement -> baby is asleep - # 7) eyes -> eyes closed -> mouth open -> baby is awake +class SleepyBaby(): - def __init__(self): + def __init__(self, frame_width, frame_height, decision_logic = DecisionLogic, debug=False): + self.media = MediaAnalysis(frame_width, frame_height, debug) + self.logic = decision_logic() + + def set_working_area(self, x_offset, y_offset, width, height): + return self.media.set_working_area(x_offset, y_offset, width, height) + + def process_frame(self, frame): + self.media.process_frame(frame) + if self.media.analysis['body_detected'] + + + + + +class old: + + def __init__(self, x, y, width, height, debug=False): + """ + __init__ _summary_ + + Parameters + ---------- + x : int, optional + offset for crop image on x-axis, by default 700 + y : int, optional + offset for crop image on y-axis, by default 125 + width : int, optional + width of the interesting area, by default 800 + height : int, optional + height of the interesting area, by default 1000 + debug : bool, optional + show verbose log, by default False + """ + self.debug = debug + self.logger = logging.getLogger(SleepyBaby.__name__) self.frame_dim = (1920,1080) self.next_frame = 0 self.fps = 30 + self.x = x + self.y = y + self.h = height + self.w = width + self.shape = [height, width] self.mpPose = mp.solutions.pose self.mpFace = mp.solutions.face_mesh self.pose = self.mpPose.Pose(min_detection_confidence=0.7, min_tracking_confidence=0.7) @@ -72,281 +104,6 @@ def __init__(self): ]) - # Decorator ensures function that can only be called once every `s` seconds. - def debounce(s): - def decorate(f): - t = None - - def wrapped(*args, **kwargs): - nonlocal t - t_ = time.time() - if t is None or t_ - t >= s: - result = f(*args, **kwargs) - t = time.time() - return result - return wrapped - return decorate - - - @debounce(1) - def throttled_handle_no_eyes_found(self): - logging.info('No face found, depreciate queue') - print('No face found, depreciate queue') - if(len(self.eyes_open_q) > 0): - self.eyes_open_q.popleft() - - - @debounce(1) - def throttled_handle_no_body_found(self): - logging.info('No body found, vote awake') - print('No body found, vote awake') - self.awake_q.append(1) - - - def process_baby_image_models(self, img, debug_img): - results = self.face.process(img) - results_pose = self.pose.process(img) - - body_found = True - if results_pose.pose_landmarks: - # 15 left-wrist, 16 right-wrist - shape = img.shape - left_wrist_coords = (shape[1] * results_pose.pose_landmarks.landmark[15].x, shape[0] * results_pose.pose_landmarks.landmark[15].y) - right_wrist_coords = (shape[1] * results_pose.pose_landmarks.landmark[16].x, shape[0] * results_pose.pose_landmarks.landmark[16].y) - - # print('left wrist: ', left_wrist_coords) - # print('right wrist: ', right_wrist_coords) - - self.movement_q.append((left_wrist_coords, right_wrist_coords)) - - debug_img = cv2.putText(debug_img, "Left wrist", (int(left_wrist_coords[0]), int(left_wrist_coords[1])), 2, 1, (255,0,0), 2, 2) - debug_img = cv2.putText(debug_img, "Right wrist", (int(right_wrist_coords[0]), int(right_wrist_coords[1])), 2, 1, (255,0,0), 2, 2) - - if os.getenv("DEBUG", 'False').lower() in ('true', '1'): - CUTOFF_THRESHOLD = 10 # head and face - MY_CONNECTIONS = frozenset([t for t in self.mpPose.POSE_CONNECTIONS if t[0] > CUTOFF_THRESHOLD and t[1] > CUTOFF_THRESHOLD]) - - # if results_pose.pose_landmarks: # if it finds the points - # for landmark_id, landmark in enumerate(results_pose.pose_landmarks): - # if landmark_id <= CUTOFF_THRESHOLD: - # landmark.visibility = 0 - # self.mpDraw.draw_landmarks(debug_img, results_pose.pose_landmarks, MY_CONNECTIONS) - - for id, lm in enumerate(results_pose.pose_landmarks.landmark): - if id <= CUTOFF_THRESHOLD: - lm.visibility = 0 - continue - h, w,c = debug_img.shape - # print(id, lm) - cx, cy = int(lm.x*w), int(lm.y*h) - cv2.circle(debug_img, (cx, cy), 5, (255,0,0), cv2.FILLED) - - self.mpDraw.draw_landmarks(debug_img, results_pose.pose_landmarks, MY_CONNECTIONS, landmark_drawing_spec=self.mpDraw.DrawingSpec(color=(255, 0, 0), thickness=2, circle_radius=2)) - - # self.mpDraw.draw_landmarks(debug_img, results_pose.pose_landmarks, MY_CONNECTIONS) - # for id, lm in enumerate(results_pose.pose_landmarks.landmark): - # if id < CUTOFF_THRESHOLD: - # continue - # self.mpDraw.draw_landmarks(debug_img, results_pose.pose_landmarks, MY_CONNECTIONS) - # h, w,c = debug_img.shape - # # print(id, lm) - # cx, cy = int(lm.x*w), int(lm.y*h) - # cv2.circle(debug_img, (cx, cy), 5, (255,0,0), cv2.FILLED) - else: - body_found = False - self.throttled_handle_no_body_found() - - LEFT_EYE = [362, 382, 381, 380, 374, 373, 390, 249, 263, 466, 388, 387, 386, 385, 384, 398] - RIGHT_EYE = [33, 7, 163, 144, 145, 153, 154, 155, 133, 173, 157, 158, 159, 160, 161, 246] - - if results.multi_face_landmarks: - self.multi_face_landmarks = results.multi_face_landmarks - - eyes_are_open = check_eyes_open(results.multi_face_landmarks[0].landmark, img, debug_img, LEFT_EYE, RIGHT_EYE) - - # Additionally check if mouth is closed. If not, consider baby crying. Can rely on queue length to ensure - # yawns don't trigger wake - - # If mouth is open, override and just consider it, "eyes open", pushing in direction of "wake vote" - if eyes_are_open == 0: # if eyes are closed, then check if mouth is open - mouth_is_open = check_mouth_open(results.multi_face_landmarks[0].landmark) - if mouth_is_open: - logging.info('Eyes closed, mouth open, crying or yawning, consider awake.') - self.eyes_open_q.append(1) - else: - logging.info('Eyes closed, mouth closed, consider sleeping.') - self.eyes_open_q.append(0) - else: - logging.info('Eyes open, consider awake.') - self.eyes_open_q.append(1) - - else: # no face results, interpret this as baby is not in crib, i.e. awake - self.throttled_handle_no_eyes_found() - - return debug_img, body_found - - - # This is placeholder until improve sensitivity of transitioning between waking and sleeping. - # Explanation: Sometimes when baby is waking up, he'll open and close his eyes for a couple of minutes... - # TODO: Fine-tune sensitivity of voting, for now, don't allow toggling between wake & sleep within N seconds - @debounce(180) - def need_to_clean_this_up(self, wake_status, img): - str_timestamp = str(int(time.time())) - sleep_data_base_path = os.getenv("SLEEP_DATA_PATH") - p = sleep_data_base_path + '/' + str_timestamp + '.png' - if wake_status: # woke up - log_string = "1," + str_timestamp + "\n" - print(log_string) - logging.info(log_string) - with open(sleep_data_base_path + '/sleep_logs.csv', 'a+', encoding="utf-8") as f: - f.write(log_string) - cv2.imwrite(p, img) # store off image of when wake/sleep event occurred. Can help with debugging issues - - # if daytime, send phone notification if baby woke up - # now = datetime.datetime.now() - # now_time = now.time() - # if now_time >= ti(7,00) or now_time <= ti(22,00): # day time - # Thread(target=telegram_send.send(messages=["Baby woke up."]), daemon=True).start() - - self.is_awake = True - - if os.getenv("OWL", 'False').lower() in ('true', '1'): - print("MOVE & MAKE NOISE") - logging.info("MOVE & MAKE NOISE") - time.sleep(5) - self.ser.write(bytes(str(999999) + "\n", "utf-8")) - self.cast_service.play_sound() - else: # fell asleep - log_string = "0," + str_timestamp + "\n" - print(log_string) - logging.info(log_string) - with open(sleep_data_base_path + '/sleep_logs.csv', 'a+', encoding="utf-8") as f: - f.write(log_string) - cv2.imwrite(p, img) - self.is_awake = False - - # now = datetime.datetime.now() - # now_time = now.time() - # if now_time >= ti(22,00) or now_time <= ti(8,00): # night time - # set_hatch(self.is_awake) - - - @debounce(10) - def set_wakeness_status(self, img): - if len(self.awake_q): - avg_awake = sum(self.awake_q) / len(self.awake_q) - if avg_awake >= 0.6 and self.is_awake == False: - self.need_to_clean_this_up(True, img) - elif avg_awake < 0.6 and self.is_awake == True: - self.need_to_clean_this_up(False, img) - - - @debounce(1) - def awake_voting_logic(self, debug_img): - if len(self.eyes_open_q) > len(self.eyes_open_q)/2: # dont vote on eyes unless queue is half full - avg = sum(self.eyes_open_q) / len(self.eyes_open_q) - if avg > 0.75: # eyes open - self.eyes_open_state = True - print("Eyes open: vote awake") - logging.info("\nvote awake") - self.awake_q.append(1) - else: # closed - self.eyes_open_state = False - self.awake_q.append(0) - print("Eyes closed: vote sleeping") - logging.info("\nvote sleeping") - else: - print("Not voting on eyes, eye queue too short.") - - - @debounce(1) - def movement_voting_logic(self, debug_img, body_found): - if not body_found: - print('No body found, depreciate movement queue.') - if len(self.movement_q): - self.movement_q.popleft() - - elif len(self.movement_q) > 5: - left_wrist_list = [c[0] for c in self.movement_q] - left_wrist_x_list = [c[0] for c in left_wrist_list] - left_wrist_y_list = [c[1] for c in left_wrist_list] - - right_wrist_list = [c[1] for c in self.movement_q] - right_wrist_x_list = [c[0] for c in right_wrist_list] - right_wrist_y_list = [c[1] for c in right_wrist_list] - - std_left_wrist_x = statistics.pstdev(left_wrist_x_list) - 1 - std_left_wrist_y = statistics.pstdev(left_wrist_y_list) - 1 - - std_right_wrist_x = statistics.pstdev(right_wrist_x_list) - 1 - std_right_wrist_y = statistics.pstdev(right_wrist_y_list) - 1 - - # average it all together and compare to movement threshold to determine if moving - avg_std = (((std_left_wrist_x + std_left_wrist_y)/2) + ((std_right_wrist_x + std_right_wrist_y)/2))/2 - # print('movement left: ', (std_left_wrist_x + std_left_wrist_y)/2) - # print('movement right: ', (std_right_wrist_x + std_right_wrist_y)/2) - # print('movement value: ', avg_std) - if int(avg_std) < 25: - print("No movement, vote sleeping") - logging.info('No movement, vote sleeping') - self.awake_q.append(0) - else: - print("Movement, vote awake") - logging.info("Movement, vote awake") - self.awake_q.append(1) - - - # every N seconds, check if baby is awake & do stuff - @debounce(5) - def periodic_wakeness_check(self): - print('\n', 'Is baby awake:', self.is_awake, '\n') - logging.info('Is baby awake: {}'.format(str(self.is_awake))) - - - def frame_logic(self, raw_img): - img = raw_img - - debug_img = img.copy() - img.flags.writeable = False - converted_img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) - - # beef - res = self.process_baby_image_models(converted_img, debug_img) - debug_img = res[0] - body_found = res[1] - - self.awake_voting_logic(debug_img) - self.movement_voting_logic(debug_img, body_found) - self.set_wakeness_status(debug_img) - self.periodic_wakeness_check() - - if os.getenv("DEBUG", 'False').lower() in ('true', '1'): - avg_awake = sum(self.awake_q) / len(self.awake_q) - - # draw progress bar - bar_y_offset = 0 - bar_y_offset = 100 - - bar_width = 200 - w = img.shape[1] - start_point = (int(w/2 - bar_width/2), 350 + bar_y_offset) - - end_point = (int(w/2 + bar_width/2), 370 + bar_y_offset) - adj_avg_awake = 1.0 if avg_awake / .6 >= 1.0 else avg_awake / .6 - progress_end_point = (int(w/2 - bar_width/2 + (bar_width*(adj_avg_awake))), 370 + bar_y_offset) - - color = (255, 255, 117) - progress_color = (0, 0, 255) - thickness = -1 - - debug_img = cv2.rectangle(debug_img, start_point, end_point, color, thickness) - debug_img = cv2.rectangle(debug_img, start_point, progress_end_point, progress_color, thickness) - display_perc = int((avg_awake * 100) / 0.6) - display_perc = 100 if display_perc >= 100 else display_perc - debug_img = cv2.putText(debug_img, str(display_perc) + "%", (int(w/2 - bar_width/2), 330 + bar_y_offset), 2, 1, (255,0,0), 2, 2) - debug_img = cv2.putText(debug_img, "Awake", (int(w/2 - bar_width/2 + 85), 330 + bar_y_offset), 2, 1, (255,0,0), 2, 2) - - return debug_img # This basically does the same thing as the live version, but is very useful for testing @@ -437,6 +194,68 @@ def recorded(self): except Exception as e: print("Something went wrong: ", e) + + + def add_progress_bar_to_image(self, frame, percent): #TODO: move to class that manage decisions + # draw progress bar + bar_y_offset = 100 + bar_width = 200 + w = frame.shape[1] + start_point = (int(w/2 - bar_width/2), 350 + bar_y_offset) + end_point = (int(w/2 + bar_width/2), 370 + bar_y_offset) + adj_percent = 1.0 if percent / .6 >= 1.0 else percent / .6 + progress_end_point = (int(w/2 - bar_width/2 + (bar_width*(adj_percent))), 370 + bar_y_offset) + color = (255, 255, 117) + progress_color = (0, 0, 255) + thickness = -1 + frame = cv2.rectangle(frame, start_point, end_point, color, thickness) + frame = cv2.rectangle(frame, start_point, progress_end_point, progress_color, thickness) + display_perc = int((percent * 100) / 0.6) + display_perc = 100 if display_perc >= 100 else display_perc + frame = cv2.putText(frame, str(display_perc) + "%", (int(w/2 - bar_width/2), 330 + bar_y_offset), 2, 1, (255,0,0), 2, 2) + frame = cv2.putText(frame, "Awake", (int(w/2 - bar_width/2 + 85), 330 + bar_y_offset), 2, 1, (255,0,0), 2, 2) + return frame + + + + def process_image(self, frame): + """ + This function will process image. + + By default, only a interesting area defined by (x,y,h,w) is considered in processing. + This avoids to waste resources to look for a baby outside crib + + Parameters + ---------- + img : numpy.ndarray + frame to be processed + + Returns + ------- + numpy.ndarray + image with overlays + """ + + #resize image if needed #FIXME: not working + #if frame.shape[0] > self.frame_dim[0] and frame.shape[1] > self.frame_dim[1]: # max res 1080p + # frame = maintain_aspect_ratio_resize(frame, width=self.frame_dim[0], height=self.frame_dim[1]) + + frame.flags.writeable = False #make the original frame read-only + self.frame = frame + working_area = cv2.cvtColor(frame[self.y:self.y+self.h, self.x:self.x+self.w], cv2.COLOR_BGR2RGB) #crop image and create a new image for processing + + analysis = self.process_baby_image_models(working_area) + debug = frame.copy() + debug = self.add_body_details_to_image(debug, analysis) + debug = self.add_progress_bar_to_image(debug, 0.5) + debug = self.add_face_details_to_image(debug, analysis) + return debug + + #self.awake_voting_logic(debug_img) + #self.movement_voting_logic(debug_img, body_found) + #self.set_wakeness_status(debug_img) + #self.periodic_wakeness_check() + def live(self, consumer_q): img = None @@ -448,57 +267,9 @@ def live(self, consumer_q): print('No images in queue: ', e) continue - # bounds to actual run models/analysis on...no need to look for babies outside of the crib - x = 700 - y = 125 - h = 1000 - w = 800 - - if img.shape[0] > 1080 and img.shape[1] > 1920: # max res 1080p - img = maintain_aspect_ratio_resize(img, width=self.frame_dim[0], height=self.frame_dim[1]) - - img_to_process = img[y:y+h, x:x+w] - - debug_img = self.frame_logic(img_to_process) - - # reapply cropped and modified/marked up img back to img which is displayed - img[y:y+h, x:x+w] = debug_img - - if os.getenv("DEBUG", 'False').lower() in ('true', '1'): - try: - cv2.rectangle(img=img, pt1=(x, y), pt2=(x+w, y+h), color=[153,50,204], thickness=2) - tmp = img[y:y+h, x:x+w] - img = gamma_correction(img, .4) - - for face_landmarks in self.multi_face_landmarks: - - # INDICIES: https://github.com/tensorflow/tfjs-models/blob/838611c02f51159afdd77469ce67f0e26b7bbb23/face-landmarks-detection/src/mediapipe-facemesh/keypoints.ts - # https://github.com/google/mediapipe/blob/master/mediapipe/python/solutions/face_mesh_connections.py - - self.mpDraw.draw_landmarks( - image=tmp, - landmark_list=face_landmarks, - connections=self.mpFace.FACEMESH_RIGHT_EYE, - landmark_drawing_spec=None, - connection_drawing_spec=self.mpDraw.DrawingSpec(color=(255, 150, 255), thickness=1, circle_radius=1)) - # connection_drawing_spec=self.mpDrawStyles - # .get_default_face_mesh_contours_style()) - - self.mpDraw.draw_landmarks( - image=tmp, - landmark_list=face_landmarks, - connections=self.mpFace.FACEMESH_LEFT_EYE, - landmark_drawing_spec=None, - connection_drawing_spec=self.mpDraw.DrawingSpec(color=(255, 255, 0), thickness=1, circle_radius=1)) - # connection_drawing_spec=self.mpDrawStyles - # .get_default_face_mesh_contours_style()) - - img[y:y+h, x:x+w] = tmp - - img = cv2.resize(img, (960, 540)) - cv2.imshow('baby', maintain_aspect_ratio_resize(img, width=self.frame_dim[0], height=self.frame_dim[1])) - - if cv2.waitKey(1) & 0xFF == ord('q'): - break - except Exception as e: - print("Something went wrong: ", e) \ No newline at end of file + img = self.process_image(img) + cv2.imshow('baby', maintain_aspect_ratio_resize(img, width=self.frame_dim[0], height=self.frame_dim[1])) + + if cv2.waitKey(1) & 0xFF == ord('q'): + break + \ No newline at end of file diff --git a/sleepy_baby/decision_logic.py b/sleepy_baby/decision_logic.py new file mode 100644 index 0000000..b39dbe4 --- /dev/null +++ b/sleepy_baby/decision_logic.py @@ -0,0 +1,177 @@ +import logging +from collections import deque +import statistics + +import cv2 +import numpy as np + + +class DecisionLogic: + + # TODO: break up this class, so big ew + + # General high level heuristics: + # 1) no eyes -> no body found -> baby is awake + # 2) no eyes -> body found -> moving -> baby is awake + # 3) no eyes -> body found -> not moving -> baby is sleeping + # 4) eyes -> eyes open -> baby is awake (disregard body movement) + # 5) eyes -> eyes closed -> movement -> baby is awake + # 6) eyes -> eyes closed -> no movement -> baby is asleep + # 7) eyes -> eyes closed -> mouth open -> baby is awake + + def __init__(self): + self.logger = logging.getLogger(self.__class__.__qualname__) + self.eyes_open_q = deque(maxlen=30) + self.awake_q = deque(maxlen=40) + self.movement_q = deque(maxlen=40) + self.eyes_open_state = False + self.is_awake = False + self.body_found = False + + def update(self, analysis:dict, eyes_threshold:float=0.75, wrist_threshold:int=25) -> None: #every second + self.body_found = analysis['body_detected'] + if self.body_found: + self.movement_q.append((analysis['left_wrist_coords'], analysis['right_wrist_coords'])) + + if (movement_list_len := len(self.movement_q)) > 5: + positions = np.reshape(self.movement_q, (movement_list_len, 4)).T + st_dev = [statistics.pstdev(pos) for pos in positions] + avg_std = sum(st_dev)/4 + + if int(avg_std) < wrist_threshold: + self.logger.info('No movement, vote sleeping') + self.awake_q.append(0) + else: + print("Movement, vote awake") + self.logger.info("Movement, vote awake") + self.awake_q.append(1) + if analysis['face_detected']: + if analysis['eyes_open'] is False: + if analysis['mouth_open']: + self.logger.info('Eyes closed, mouth open, crying or yawning, consider awake.') + self.eyes_open_q.append(1) + else: + self.logger.info('Eyes closed, mouth closed, consider sleeping.') + self.eyes_open_q.append(0) + else: + self.logger.info('Eyes open, consider awake.') + self.eyes_open_q.append(1) + else: + self.logger.info('No face found, depreciate queue') + if len(self.eyes_open_q) > 0: + self.eyes_open_q.popleft() + else: + self.logger.info('No body found, vote awake') + self.awake_q.append(1) + if len(self.movement_q): + self.movement_q.popleft() + #Evaluate eyes + if len(self.eyes_open_q)>(self.eyes_open_q.maxlen/2): + eyes_score = sum(self.eyes_open_q) / len(self.eyes_open_q) + self.eyes_open_state = eyes_score > eyes_threshold + self.logger.info("Vote {'awake' if self.eyes_open_state else 'sleep'}") + self.awake_q.append(1 if self.eyes_open_state else 0) + else: + self.logger.info("Not voting on eyes, eye queue too short.") + + + def set_wakeness_status(self, img): #each 10s + if len(self.awake_q): + avg_awake = sum(self.awake_q) / len(self.awake_q) + if avg_awake >= 0.6 and self.is_awake == False: + self.need_to_clean_this_up(True, img) + elif avg_awake < 0.6 and self.is_awake == True: + self.need_to_clean_this_up(False, img) + + # This is placeholder until improve sensitivity of transitioning between waking and sleeping. + # Explanation: Sometimes when baby is waking up, he'll open and close his eyes for a couple of minutes... + # TODO: Fine-tune sensitivity of voting, for now, don't allow toggling between wake & sleep within N seconds + @debounce(180) + def need_to_clean_this_up(self, wake_status, img): + str_timestamp = str(int(time.time())) + sleep_data_base_path = os.getenv("SLEEP_DATA_PATH") + p = sleep_data_base_path + '/' + str_timestamp + '.png' + if wake_status: # woke up + log_string = "1," + str_timestamp + "\n" + print(log_string) + logging.info(log_string) + with open(sleep_data_base_path + '/sleep_logs.csv', 'a+', encoding="utf-8") as f: + f.write(log_string) + cv2.imwrite(p, img) # store off image of when wake/sleep event occurred. Can help with debugging issues + + # if daytime, send phone notification if baby woke up + # now = datetime.datetime.now() + # now_time = now.time() + # if now_time >= ti(7,00) or now_time <= ti(22,00): # day time + # Thread(target=telegram_send.send(messages=["Baby woke up."]), daemon=True).start() + + self.is_awake = True + + if os.getenv("OWL", 'False').lower() in ('true', '1'): + print("MOVE & MAKE NOISE") + logging.info("MOVE & MAKE NOISE") + time.sleep(5) + self.ser.write(bytes(str(999999) + "\n", "utf-8")) + self.cast_service.play_sound() + else: # fell asleep + log_string = "0," + str_timestamp + "\n" + print(log_string) + logging.info(log_string) + with open(sleep_data_base_path + '/sleep_logs.csv', 'a+', encoding="utf-8") as f: + f.write(log_string) + cv2.imwrite(p, img) + self.is_awake = False + + # now = datetime.datetime.now() + # now_time = now.time() + # if now_time >= ti(22,00) or now_time <= ti(8,00): # night time + # set_hatch(self.is_awake) + + + + + + def frame_logic(self, raw_img): + img = raw_img + + debug_img = img.copy() + img.flags.writeable = False + converted_img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + + # beef + res = self.process_baby_image_models(converted_img, debug_img) + debug_img = res[0] + body_found = res[1] + + self.awake_voting_logic(debug_img) + self.movement_voting_logic(debug_img, body_found) + self.set_wakeness_status(debug_img) + self.periodic_wakeness_check() + + if os.getenv("DEBUG", 'False').lower() in ('true', '1'): + avg_awake = sum(self.awake_q) / len(self.awake_q) + + # draw progress bar + bar_y_offset = 0 + bar_y_offset = 100 + + bar_width = 200 + w = img.shape[1] + start_point = (int(w/2 - bar_width/2), 350 + bar_y_offset) + + end_point = (int(w/2 + bar_width/2), 370 + bar_y_offset) + adj_avg_awake = 1.0 if avg_awake / .6 >= 1.0 else avg_awake / .6 + progress_end_point = (int(w/2 - bar_width/2 + (bar_width*(adj_avg_awake))), 370 + bar_y_offset) + + color = (255, 255, 117) + progress_color = (0, 0, 255) + thickness = -1 + + debug_img = cv2.rectangle(debug_img, start_point, end_point, color, thickness) + debug_img = cv2.rectangle(debug_img, start_point, progress_end_point, progress_color, thickness) + display_perc = int((avg_awake * 100) / 0.6) + display_perc = 100 if display_perc >= 100 else display_perc + debug_img = cv2.putText(debug_img, str(display_perc) + "%", (int(w/2 - bar_width/2), 330 + bar_y_offset), 2, 1, (255,0,0), 2, 2) + debug_img = cv2.putText(debug_img, "Awake", (int(w/2 - bar_width/2 + 85), 330 + bar_y_offset), 2, 1, (255,0,0), 2, 2) + + return debug_img \ No newline at end of file From f5749d2e5ba16fcc2e2463b1af26d1ad795d2eef Mon Sep 17 00:00:00 2001 From: "CSI\\salvatore musumeci" Date: Fri, 24 Mar 2023 18:35:42 +0100 Subject: [PATCH 08/31] refactor: :art: Created Frame class This will be useful to better split the code. Now, Frame class will contain the frame to analyze and all methods to work on the image. Instad MediaAnalysis will be focused only on feature extraction --- sleepy_baby/frame.py | 139 +++++++++++++++++++++++ sleepy_baby/media_analysis.py | 206 ++++++---------------------------- 2 files changed, 171 insertions(+), 174 deletions(-) create mode 100644 sleepy_baby/frame.py diff --git a/sleepy_baby/frame.py b/sleepy_baby/frame.py new file mode 100644 index 0000000..362e8f9 --- /dev/null +++ b/sleepy_baby/frame.py @@ -0,0 +1,139 @@ +import logging +import numpy as np +import mediapipe as mp +import cv2 + +mp_utils = mp.solutions.drawing_utils + +class Frame: + def __init__(self, frame: cv2.Mat, x_offset: int =0, y_offset: int = 0, width:int = None, height:int= None): + self.logger = logging.getLogger(self.__class__.__qualname__) + self.frame = frame + if (width is not None) and (height is not None): + self.set_working_area(x_offset, y_offset, width, height) + else: + self.set_working_area(0,0, frame.shape[1], frame.shape[0]) + + def set_working_area(self, x_offset: int, y_offset: int, width: int, height:int): + """ + set_working_area will define a sub-area of frame to be analyze. + + This will help hardware to be faster and have a lower power consumption + + Parameters + ---------- + x_offset : int + offset for crop image on x-axis + y_offset : int + offset for crop image on y-axis + width : int + width of the interesting area + height : int + height of the interesting area + """ + self.x_offset = x_offset + self.y_offset = y_offset + self.height = height + self.width = width + self.clean_working_frame() + + def clean_working_frame(self) -> None: + """Create a new working picture""" + self.w_data = self.get_working_image() + + def get_working_image(self) -> np.ndarray: + """ + Return the subset of the frame where analysis is done + + Returns + ------- + np.ndarray + Working area + """ + return self.frame[self.y_offset:self.y_offset+self.height, self.x_offset:self.x_offset+self.width] + + def getAugmentedFrame(self) -> np.ndarray: + """ + It generates the a new frame integrating the modified working area. + + Returns + ------- + np.ndarray + Image integrated + """ + frame = self.frame.copy() + frame[self.y_offset:self.y_offset+self.height, self.x_offset:self.x_offset+self.width] = self.w_data + return frame + + + def add_body_details(self, pose_landmarks): + if pose_landmarks: + self.logger.debug("Body detected in the frame") + CUTOFF_THRESHOLD = 10 # head and face + MY_CONNECTIONS = [t for t in mp.solutions.pose.POSE_CONNECTIONS if t[0] > CUTOFF_THRESHOLD and t[1] > CUTOFF_THRESHOLD] + for id, lm in enumerate(pose_landmarks.landmark): + if id <= CUTOFF_THRESHOLD: + lm.visibility = 0 + continue + mp_utils.draw_landmarks(self.w_data, + pose_landmarks, + MY_CONNECTIONS, + landmark_drawing_spec=mp_utils.DrawingSpec( color=(255, 0, 0), + thickness=10, + circle_radius=2), + connection_drawing_spec=mp_utils.DrawingSpec(color=(0, 0, 255), + thickness=5, + circle_radius=2) + ) + self.w_data = cv2.putText(self.w_data, "Left wrist", + (int(self.width * pose_landmarks.landmark[15].x), + int(self.height * pose_landmarks.landmark[15].y)), + 2, 1, (255,0,0), 2, 2) + self.w_data = cv2.putText(self.w_data, "Right wrist", + (int(self.width * pose_landmarks.landmark[16].x), + int(self.height * pose_landmarks.landmark[16].y)), + 2, 1, (255,0,0), 2, 2) + else: + self.logger.debug("No body detected in frame") + + def add_analysis_frame(self): + self.logger.debug("Draw the analysis frame") + self.w_data = cv2.rectangle(self.w_data, [0,0], (self.w_data.shape[1], self.w_data.shape[0]), color=(0,255,0), thickness=5) + + def add_face_details(self, multi_face_landmarks): + """ + add_face_details_to_image adds face details to image passed in arguments. + + Parameters + ---------- + frame : numpy.ndarray + starting image + multi_face_landmarks : dict + dictionary containing evaluation + + Returns + ------- + numpy.ndarray + image with some draws overlayed + """ + if multi_face_landmarks: + self.logger.debug("Face found in the frame") + for face_landmarks in multi_face_landmarks: + # INDICIES: https://github.com/tensorflow/tfjs-models/blob/838611c02f51159afdd77469ce67f0e26b7bbb23/face-landmarks-detection/src/mediapipe-facemesh/keypoints.ts + # https://github.com/google/mediapipe/blob/master/mediapipe/python/solutions/face_mesh_connections.py + + mp.solutions.drawing_utils.draw_landmarks( + image=self.w_data, + landmark_list=face_landmarks, + connections=mp.solutions.face_mesh.FACEMESH_RIGHT_EYE, + landmark_drawing_spec=None, + connection_drawing_spec=mp_utils.DrawingSpec(color=(255, 150, 255), thickness=5, circle_radius=1)) + + mp.solutions.drawing_utils.draw_landmarks( + image=self.w_data, + landmark_list=face_landmarks, + connections=mp.solutions.face_mesh.FACEMESH_LEFT_EYE, + landmark_drawing_spec=None, + connection_drawing_spec=mp_utils.DrawingSpec(color=(255, 255, 0), thickness=5, circle_radius=1)) + else: + self.logger.debug("No face found") \ No newline at end of file diff --git a/sleepy_baby/media_analysis.py b/sleepy_baby/media_analysis.py index 10c0ab7..a6f893d 100644 --- a/sleepy_baby/media_analysis.py +++ b/sleepy_baby/media_analysis.py @@ -1,6 +1,5 @@ -import numpy as np -import cv2 import mediapipe as mp +import sleepy_baby import logging from .helpers import check_eyes_open, check_mouth_open @@ -13,75 +12,19 @@ class MediaAnalysis: It will be used later for decision logic to make the proper evaluation. """ - def __init__(self, frame_width, frame_height, debug=False): - """ - __init__ Initialize Analyzer. - - Parameters - ---------- - debug : bool, optional - show verbose log, by default False - """ - self.debug = debug - self.logger = logging.getLogger(self.__class__.__qualname__) - self.set_working_area(0, 0, frame_width, frame_height) - - self.pose = mp.solutions.pose.Pose(min_detection_confidence=0.7, - min_tracking_confidence=0.7) + def __init__(self): + """__init__ Initialize Analyzer.""" + self.logger = logging.getLogger(self.__class__.__qualname__) # TODO: try turning off refine_landmarks for performance, might not be needed self.face = mp.solutions.face_mesh.FaceMesh(max_num_faces=1, refine_landmarks=True, min_detection_confidence=0.8, min_tracking_confidence=0.8) + self.pose = mp.solutions.pose.Pose(min_detection_confidence=0.7, + min_tracking_confidence=0.7) - self.pose_landmarks = None - self.multi_face_landmarks = None - self._reset_analysis() - - def _reset_analysis(self): - self.analysis = { - "body_detected": False, - "left_wrist_coords": None, - "right_wrist_coords": None, - "face_detected": True, - "eyes_open": False, - "mouth_open": False - } - - def set_working_area(self, x_offset, y_offset, width, height): - """ - set_working_area will define a sub-area of frame to be analyze. - - This will help hardware to be faster and have a lower power consumption - - Parameters - ---------- - x : int - offset for crop image on x-axis - y : int - offset for crop image on y-axis - width : int - width of the interesting area - height : int - height of the interesting area - """ - self.x_offset = x_offset - self.y_offset = y_offset - self.height = height - self.width = width - self.shape = [height, width] - - def get_working_image(self, img): - return img[self.y_offset:self.y_offset+self.height, self.x_offset:self.x_offset+self.width] - - def apply_working_area_to_image(self, w_area, frame): - frame[self.y_offset:self.y_offset+self.height, self.x_offset:self.x_offset+self.width] = w_area - - def process_frame(self, img): - w_area = self.get_working_image(img) - return self._process_baby_image_models(w_area) - def _process_baby_image_models(self, img): + def process_baby_image_models(self, frame:sleepy_baby.frame.Frame): """ process_baby_image_models analyze frame and get information. @@ -89,120 +32,35 @@ def _process_baby_image_models(self, img): Parameters ---------- - img : numpy.ndarray - Image to be processed. It is already cropped + frame : sleepy_baby.frame.Frame + Get Frame object + Returns + ------- + _type_ + Returns dict with all findings and the objects for pose and face. """ - - self._reset_analysis() - results_pose = self.pose.process(img) + + analysis = { + "body_detected": False, + "left_wrist_coords": None, + "right_wrist_coords": None, + "face_detected": False, + "eyes_open": False, + "mouth_open": False + } + results_pose = self.pose.process(frame) if results_pose.pose_landmarks: - self.analysis["body_detected"] = True - self.pose_landmarks = results_pose.pose_landmarks + analysis["body_detected"] = True # 15 left-wrist, 16 right-wrist - self.analysis["left_wrist_coords"] = (self.shape[1] * self.pose_landmarks.landmark[15].x, self.shape[0] * self.pose_landmarks.landmark[15].y) - self.analysis["right_wrist_coords"] = (self.shape[1] * self.pose_landmarks.landmark[16].x, self.shape[0] * self.pose_landmarks.landmark[16].y) + analysis["left_wrist_coords"] = (frame.shape[1] * results_pose.pose_landmarks.landmark[15].x, frame.shape[0] * results_pose.pose_landmarks.landmark[15].y) + analysis["right_wrist_coords"] = (frame.shape[1] * results_pose.pose_landmarks.landmark[16].x, frame.shape[0] * results_pose.pose_landmarks.landmark[16].y) - results = self.face.process(img) + results = self.face.process(frame) if results.multi_face_landmarks: - self.multi_face_landmarks = results.multi_face_landmarks - self.analysis["face_detected"] = True - self.analysis["eyes_open"] = check_eyes_open(results.multi_face_landmarks[0].landmark) - self.analysis["mouth_open"] = check_mouth_open(results.multi_face_landmarks[0].landmark) - else: - self.pose_landmarks = None - self.multi_face_landmarks = None - self.analysis["body_found"] = False - - - def add_body_details_to_image(self, frame): - """ - add_body_details_to_image adds body details to image passed in arguments. - - Parameters - ---------- - frame : numpy.ndarray - starting image - analysis : dict - dictionary containing evaluation - - Returns - ------- - numpy.ndarray - image with some draws overlayed - """ - w_area = self.get_working_image(frame) - cv2.rectangle(w_area, [0, 0], self.shape, color=(0, 255, 0), thickness=5) #Draw Analysis Area - #Draw body lines - if self.analysis['body_detected']: - CUTOFF_THRESHOLD = 10 # head and face - MY_CONNECTIONS = [t for t in mp.solutions.pose.POSE_CONNECTIONS if t[0] > CUTOFF_THRESHOLD and t[1] > CUTOFF_THRESHOLD] - for id, lm in enumerate(self.pose_landmarks.landmark): - if id <= CUTOFF_THRESHOLD: - lm.visibility = 0 - continue - mp.solutions.drawing_utils.draw_landmarks(w_area, - self.pose_landmarks, - MY_CONNECTIONS, - landmark_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(color=(255, 0, 0), - thickness=10, - circle_radius=2), - connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(color=(0, 0, 255), - thickness=5, - circle_radius=2) - ) - w_area = cv2.putText(w_area, "Left wrist", - (int(self.analysis["left_wrist_coords"][0]), - int(self.analysis["left_wrist_coords"][1])), - 2, 1, (255,0,0), 2, 2) - w_area = cv2.putText(w_area, "Right wrist", - (int(self.analysis["right_wrist_coords"][0]), - int(self.analysis["right_wrist_coords"][1])), - 2, 1, (255,0,0), 2, 2) - self.apply_working_area_to_image(w_area, frame) - return frame - - def add_face_details_to_image(self, frame): - """ - add_face_details_to_image adds face details to image passed in arguments. - - Parameters - ---------- - frame : numpy.ndarray - starting image - analysis : dict - dictionary containing evaluation - - Returns - ------- - numpy.ndarray - image with some draws overlayed - """ - if self.analysis['face_detected']: - self.logger.info(f"Face Detected. Eyes are {'open' if self.analysis['eyes_open'] else 'close'} and mouth is {'open' if self.analysis['mouth_open'] else 'close'}") - w_area = self.get_working_image(frame) - for face_landmarks in self.multi_face_landmarks: - # INDICIES: https://github.com/tensorflow/tfjs-models/blob/838611c02f51159afdd77469ce67f0e26b7bbb23/face-landmarks-detection/src/mediapipe-facemesh/keypoints.ts - # https://github.com/google/mediapipe/blob/master/mediapipe/python/solutions/face_mesh_connections.py - - mp.solutions.drawing_utils.draw_landmarks( - image=w_area, - landmark_list=face_landmarks, - connections=mp.solutions.face_mesh.FACEMESH_RIGHT_EYE, - landmark_drawing_spec=None, - connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(color=(255, 150, 255), thickness=5, circle_radius=1)) - # connection_drawing_spec=self.mpDrawStyles - # .get_default_face_mesh_contours_style()) - - mp.solutions.drawing_utils.draw_landmarks( - image=w_area, - landmark_list=face_landmarks, - connections=mp.solutions.face_mesh.FACEMESH_LEFT_EYE, - landmark_drawing_spec=None, - connection_drawing_spec=mp.solutions.drawing_utils.DrawingSpec(color=(255, 255, 0), thickness=5, circle_radius=1)) - # connection_drawing_spec=self.mpDrawStyles - # .get_default_face_mesh_contours_style()) - self.apply_working_area_to_image(w_area, frame) + analysis["face_detected"] = True + analysis["eyes_open"] = check_eyes_open(results.multi_face_landmarks[0].landmark) + analysis["mouth_open"] = check_mouth_open(results.multi_face_landmarks[0].landmark) else: - self.logger.info("Face is not detected") - return frame + analysis["body_found"] = False + return analysis, results_pose.pose_landmarks, results.multi_face_landmarks \ No newline at end of file From 5623d52f87f605ad0fcbf639dfcdb8d66bf98eb7 Mon Sep 17 00:00:00 2001 From: nos86 Date: Fri, 24 Mar 2023 22:10:22 +0100 Subject: [PATCH 09/31] refactor: :technologist: Threshold for recognition has been moved as arguments of function for later fine-tuning --- .gitignore | 6 ++++++ sleepy_baby/frame.py | 4 ++-- sleepy_baby/media_analysis.py | 19 ++++++++++++------- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 378dfe9..00083d1 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,9 @@ build *.log .env .ipynb_checkpoints +baby +*.jpg +*.jpeg +*.png +*.avi +*.mp4 \ No newline at end of file diff --git a/sleepy_baby/frame.py b/sleepy_baby/frame.py index 362e8f9..a6d18fc 100644 --- a/sleepy_baby/frame.py +++ b/sleepy_baby/frame.py @@ -61,7 +61,7 @@ def getAugmentedFrame(self) -> np.ndarray: np.ndarray Image integrated """ - frame = self.frame.copy() + frame = self.frame * 0.7 frame[self.y_offset:self.y_offset+self.height, self.x_offset:self.x_offset+self.width] = self.w_data return frame @@ -82,7 +82,7 @@ def add_body_details(self, pose_landmarks): thickness=10, circle_radius=2), connection_drawing_spec=mp_utils.DrawingSpec(color=(0, 0, 255), - thickness=5, + thickness=3, circle_radius=2) ) self.w_data = cv2.putText(self.w_data, "Left wrist", diff --git a/sleepy_baby/media_analysis.py b/sleepy_baby/media_analysis.py index a6f893d..302d6aa 100644 --- a/sleepy_baby/media_analysis.py +++ b/sleepy_baby/media_analysis.py @@ -12,19 +12,23 @@ class MediaAnalysis: It will be used later for decision logic to make the proper evaluation. """ - def __init__(self): + def __init__(self, + body_min_detection_confidence=0.8, + body_min_tracking_confidence=0.8, + face_min_detection_confidence=0.7, + face_min_tracking_confidence=0-7): """__init__ Initialize Analyzer.""" self.logger = logging.getLogger(self.__class__.__qualname__) # TODO: try turning off refine_landmarks for performance, might not be needed self.face = mp.solutions.face_mesh.FaceMesh(max_num_faces=1, refine_landmarks=True, - min_detection_confidence=0.8, - min_tracking_confidence=0.8) - self.pose = mp.solutions.pose.Pose(min_detection_confidence=0.7, - min_tracking_confidence=0.7) + min_detection_confidence=body_min_detection_confidence, + min_tracking_confidence=body_min_tracking_confidence) + self.pose = mp.solutions.pose.Pose(min_detection_confidence=face_min_detection_confidence, + min_tracking_confidence=face_min_tracking_confidence) - def process_baby_image_models(self, frame:sleepy_baby.frame.Frame): + def process_baby_image_models(self, frame): """ process_baby_image_models analyze frame and get information. @@ -49,6 +53,7 @@ def process_baby_image_models(self, frame:sleepy_baby.frame.Frame): "eyes_open": False, "mouth_open": False } + results = None results_pose = self.pose.process(frame) if results_pose.pose_landmarks: analysis["body_detected"] = True @@ -63,4 +68,4 @@ def process_baby_image_models(self, frame:sleepy_baby.frame.Frame): analysis["mouth_open"] = check_mouth_open(results.multi_face_landmarks[0].landmark) else: analysis["body_found"] = False - return analysis, results_pose.pose_landmarks, results.multi_face_landmarks \ No newline at end of file + return analysis, results_pose.pose_landmarks, results.multi_face_landmarks if results is not None else None \ No newline at end of file From f11341993f7f1b28cbe75fda06d0e79ac5a6c0be Mon Sep 17 00:00:00 2001 From: nos86 Date: Sat, 25 Mar 2023 00:01:57 +0100 Subject: [PATCH 10/31] refactor: :art: media_analysis moved directly inside __init__ --- sleepy_baby/__init__.py | 196 +++++++++++++++++++++------------- sleepy_baby/media_analysis.py | 71 ------------ 2 files changed, 121 insertions(+), 146 deletions(-) delete mode 100644 sleepy_baby/media_analysis.py diff --git a/sleepy_baby/__init__.py b/sleepy_baby/__init__.py index 5ed98e1..98047a5 100644 --- a/sleepy_baby/__init__.py +++ b/sleepy_baby/__init__.py @@ -9,85 +9,149 @@ import logging import serial import queue -import statistics -from dotenv import load_dotenv -# from cast_service import CastSoundService -from http.server import HTTPServer, SimpleHTTPRequestHandler -from .helpers import check_eyes_open, set_hatch, check_mouth_open, maintain_aspect_ratio_resize, gamma_correction -from .media_analysis import MediaAnalysis +from .frame import Frame +from .helpers import check_eyes_open, check_mouth_open from .decision_logic import DecisionLogic -class SleepyBaby(): - def __init__(self, frame_width, frame_height, decision_logic = DecisionLogic, debug=False): - self.media = MediaAnalysis(frame_width, frame_height, debug) - self.logic = decision_logic() - def set_working_area(self, x_offset, y_offset, width, height): - return self.media.set_working_area(x_offset, y_offset, width, height) - - def process_frame(self, frame): - self.media.process_frame(frame) - if self.media.analysis['body_detected'] +class SleepyBaby: + """ + It analyzes frame provided on process_frame. + + Results are saved inside the object variable "analysis". + It will be used later for decision logic to make the proper evaluation. + """ + + def __init__(self, + body_min_detection_confidence=0.8, + body_min_tracking_confidence=0.8, + face_min_detection_confidence=0.7, + face_min_tracking_confidence=0.7): + self.logger = logging.getLogger(self.__class__.__name__) + self.logger.debug("SleepyBaby is starting") + self.processed_frame = None #It is used to produce post-processed video + self.process_t = None #Process Thread + self.face = mp.solutions.face_mesh.FaceMesh(max_num_faces=1, + refine_landmarks=True, + min_detection_confidence=body_min_detection_confidence, + min_tracking_confidence=body_min_tracking_confidence) + self.pose = mp.solutions.pose.Pose(min_detection_confidence=face_min_detection_confidence, + min_tracking_confidence=face_min_tracking_confidence) + self.set_working_area() #Set entire area as working area + self.set_output() #Set default values + + #self.logic = decision_logic() + self.logger.info("SleepyBaby is configured") + + def set_output(self, + show_frame=True, + show_wrist_position=True, + show_wrist_text=True, + show_body_details=True, + show_face_details=True, + show_progress_bar=True): + self.show_frame = show_frame + self.show_wrist_position = show_wrist_position + self.show_wrist_text = show_wrist_text + self.show_body_details = show_body_details + self.show_face_details = show_face_details + self.show_progress_bar = show_progress_bar + + def start_thread(self, frame_q, stop_event, pause=0.1, ): + def loop(self, frame_q, stop_event, pause): + while stop_event.is_set() is False: + if len(frame_q)>0: + frame = self.processFrame(frame_q.pop(), return_image = self.processed_frame is None) + if frame is not None: + self.processed_frame = frame + time.sleep(pause) + self.process_t = Thread(target=loop, args=(self, frame_q, stop_event, pause)) + self.process_t.start() + + + + def set_working_area(self, x_offset=0, y_offset=0, width=None, height=None): + self.x_offset = x_offset + self.y_offset = y_offset + self.width = width + self.height = height + self.working_area_inited = True + + def processFrame(self, image, return_image=True): + frame = Frame(image, self.x_offset, self.y_offset, self.width, self.height) + analysis, pose, face = self.process_baby_image_models(frame.w_data) + if return_image: + if self.show_frame: + frame.add_analysis_frame() + if self.show_wrist_position or self.show_wrist_text: + frame.add_wrist_position(pose, self.show_wrist_text) + if self.show_body_details: + frame.add_body_details(pose) + if self.show_face_details: + frame.add_face_details(face) + if self.show_progress_bar: + frame.add_progress_bar(0.45) #TODO: get this value from decision object + return frame.getAugmentedFrame() + + def process_baby_image_models(self, frame): + """ + process_baby_image_models analyze frame and get information. - + Results are stored in analysis variable inside object + + Parameters + ---------- + frame : sleepy_baby.frame.Frame + Get Frame object + Returns + ------- + _type_ + Returns dict with all findings and the objects for pose and face. + """ + + analysis = { + "body_detected": False, + "left_wrist_coords": None, + "right_wrist_coords": None, + "face_detected": False, + "eyes_open": False, + "mouth_open": False + } + results = None + results_pose = self.pose.process(frame) + if results_pose.pose_landmarks: + analysis["body_detected"] = True + # 15 left-wrist, 16 right-wrist + analysis["left_wrist_coords"] = (frame.shape[1] * results_pose.pose_landmarks.landmark[15].x, frame.shape[0] * results_pose.pose_landmarks.landmark[15].y) + analysis["right_wrist_coords"] = (frame.shape[1] * results_pose.pose_landmarks.landmark[16].x, frame.shape[0] * results_pose.pose_landmarks.landmark[16].y) + + results = self.face.process(frame) + if results.multi_face_landmarks: + analysis["face_detected"] = True + analysis["eyes_open"] = check_eyes_open(results.multi_face_landmarks[0].landmark) + analysis["mouth_open"] = check_mouth_open(results.multi_face_landmarks[0].landmark) + else: + analysis["body_found"] = False + return analysis, results_pose.pose_landmarks, results.multi_face_landmarks if results is not None else None class old: def __init__(self, x, y, width, height, debug=False): - """ - __init__ _summary_ - Parameters - ---------- - x : int, optional - offset for crop image on x-axis, by default 700 - y : int, optional - offset for crop image on y-axis, by default 125 - width : int, optional - width of the interesting area, by default 800 - height : int, optional - height of the interesting area, by default 1000 - debug : bool, optional - show verbose log, by default False - """ - self.debug = debug - self.logger = logging.getLogger(SleepyBaby.__name__) - self.frame_dim = (1920,1080) - self.next_frame = 0 - self.fps = 30 - self.x = x - self.y = y - self.h = height - self.w = width - self.shape = [height, width] - self.mpPose = mp.solutions.pose - self.mpFace = mp.solutions.face_mesh - self.pose = self.mpPose.Pose(min_detection_confidence=0.7, min_tracking_confidence=0.7) - # TODO: try turning off refine_landmarks for performance, might not be needed - self.face = self.mpFace.FaceMesh(max_num_faces=1, refine_landmarks=True, min_detection_confidence=0.8, min_tracking_confidence=0.8) - self.mpDraw = mp.solutions.drawing_utils - self.mpDrawStyles = mp.solutions.drawing_styles self.eyes_open_q = deque(maxlen=30) self.awake_q = deque(maxlen=40) self.movement_q = deque(maxlen=40) self.eyes_open_state = False - self.multi_face_landmarks = [] self.is_awake = False self.ser = None # serial connection to arduino for controlling demon owl - # If demon owl mode, setup connection to arduino and cast service for playing audio - if os.getenv("OWL", 'False').lower() in ('true', '1'): - print("\nCAWWWWWW\n") - self.cast_service = CastSoundService() - self.ser = serial.Serial('/dev/ttyACM0', 9600, timeout=0) - self.top_lip = frozenset([ (324, 308), (78, 191), (191, 80), (80, 81), (81, 82), (82, 13), (13, 312), (312, 311), (311, 310), @@ -196,25 +260,7 @@ def recorded(self): - def add_progress_bar_to_image(self, frame, percent): #TODO: move to class that manage decisions - # draw progress bar - bar_y_offset = 100 - bar_width = 200 - w = frame.shape[1] - start_point = (int(w/2 - bar_width/2), 350 + bar_y_offset) - end_point = (int(w/2 + bar_width/2), 370 + bar_y_offset) - adj_percent = 1.0 if percent / .6 >= 1.0 else percent / .6 - progress_end_point = (int(w/2 - bar_width/2 + (bar_width*(adj_percent))), 370 + bar_y_offset) - color = (255, 255, 117) - progress_color = (0, 0, 255) - thickness = -1 - frame = cv2.rectangle(frame, start_point, end_point, color, thickness) - frame = cv2.rectangle(frame, start_point, progress_end_point, progress_color, thickness) - display_perc = int((percent * 100) / 0.6) - display_perc = 100 if display_perc >= 100 else display_perc - frame = cv2.putText(frame, str(display_perc) + "%", (int(w/2 - bar_width/2), 330 + bar_y_offset), 2, 1, (255,0,0), 2, 2) - frame = cv2.putText(frame, "Awake", (int(w/2 - bar_width/2 + 85), 330 + bar_y_offset), 2, 1, (255,0,0), 2, 2) - return frame + diff --git a/sleepy_baby/media_analysis.py b/sleepy_baby/media_analysis.py deleted file mode 100644 index 302d6aa..0000000 --- a/sleepy_baby/media_analysis.py +++ /dev/null @@ -1,71 +0,0 @@ -import mediapipe as mp -import sleepy_baby -import logging - -from .helpers import check_eyes_open, check_mouth_open - -class MediaAnalysis: - """ - It analyzes frame provided on process_frame. - - Results are saved inside the object variable "analysis". - It will be used later for decision logic to make the proper evaluation. - """ - - def __init__(self, - body_min_detection_confidence=0.8, - body_min_tracking_confidence=0.8, - face_min_detection_confidence=0.7, - face_min_tracking_confidence=0-7): - """__init__ Initialize Analyzer.""" - self.logger = logging.getLogger(self.__class__.__qualname__) - # TODO: try turning off refine_landmarks for performance, might not be needed - self.face = mp.solutions.face_mesh.FaceMesh(max_num_faces=1, - refine_landmarks=True, - min_detection_confidence=body_min_detection_confidence, - min_tracking_confidence=body_min_tracking_confidence) - self.pose = mp.solutions.pose.Pose(min_detection_confidence=face_min_detection_confidence, - min_tracking_confidence=face_min_tracking_confidence) - - - def process_baby_image_models(self, frame): - """ - process_baby_image_models analyze frame and get information. - - Results are stored in analysis variable inside object - - Parameters - ---------- - frame : sleepy_baby.frame.Frame - Get Frame object - - Returns - ------- - _type_ - Returns dict with all findings and the objects for pose and face. - """ - - analysis = { - "body_detected": False, - "left_wrist_coords": None, - "right_wrist_coords": None, - "face_detected": False, - "eyes_open": False, - "mouth_open": False - } - results = None - results_pose = self.pose.process(frame) - if results_pose.pose_landmarks: - analysis["body_detected"] = True - # 15 left-wrist, 16 right-wrist - analysis["left_wrist_coords"] = (frame.shape[1] * results_pose.pose_landmarks.landmark[15].x, frame.shape[0] * results_pose.pose_landmarks.landmark[15].y) - analysis["right_wrist_coords"] = (frame.shape[1] * results_pose.pose_landmarks.landmark[16].x, frame.shape[0] * results_pose.pose_landmarks.landmark[16].y) - - results = self.face.process(frame) - if results.multi_face_landmarks: - analysis["face_detected"] = True - analysis["eyes_open"] = check_eyes_open(results.multi_face_landmarks[0].landmark) - analysis["mouth_open"] = check_mouth_open(results.multi_face_landmarks[0].landmark) - else: - analysis["body_found"] = False - return analysis, results_pose.pose_landmarks, results.multi_face_landmarks if results is not None else None \ No newline at end of file From 2d1ec03123f719c79417bd08ecd843e049f9ff16 Mon Sep 17 00:00:00 2001 From: nos86 Date: Sat, 25 Mar 2023 00:03:12 +0100 Subject: [PATCH 11/31] refactor: :art: Add possibility to show additional parameters on screen --- sleepy_baby/frame.py | 46 +++++++++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/sleepy_baby/frame.py b/sleepy_baby/frame.py index a6d18fc..810dffe 100644 --- a/sleepy_baby/frame.py +++ b/sleepy_baby/frame.py @@ -50,7 +50,7 @@ def get_working_image(self) -> np.ndarray: np.ndarray Working area """ - return self.frame[self.y_offset:self.y_offset+self.height, self.x_offset:self.x_offset+self.width] + return self.frame[self.y_offset:self.y_offset+self.height, self.x_offset:self.x_offset+self.width].copy() def getAugmentedFrame(self) -> np.ndarray: """ @@ -61,7 +61,7 @@ def getAugmentedFrame(self) -> np.ndarray: np.ndarray Image integrated """ - frame = self.frame * 0.7 + frame = self.frame.copy() frame[self.y_offset:self.y_offset+self.height, self.x_offset:self.x_offset+self.width] = self.w_data return frame @@ -85,17 +85,20 @@ def add_body_details(self, pose_landmarks): thickness=3, circle_radius=2) ) - self.w_data = cv2.putText(self.w_data, "Left wrist", - (int(self.width * pose_landmarks.landmark[15].x), - int(self.height * pose_landmarks.landmark[15].y)), - 2, 1, (255,0,0), 2, 2) - self.w_data = cv2.putText(self.w_data, "Right wrist", - (int(self.width * pose_landmarks.landmark[16].x), - int(self.height * pose_landmarks.landmark[16].y)), - 2, 1, (255,0,0), 2, 2) else: self.logger.debug("No body detected in frame") + def add_wrist_position(self, pose_landmarks, show_text=True): + if pose_landmarks: + left_wrist = (int(self.width * pose_landmarks.landmark[15].x), int(self.height * pose_landmarks.landmark[15].y)) + right_wrist = (int(self.width * pose_landmarks.landmark[16].x), int(self.height * pose_landmarks.landmark[16].y)) + + cv2.circle(self.w_data, left_wrist, radius=5, color=(255,255,0), thickness=-1) + cv2.circle(self.w_data, right_wrist, radius=5, color=(255,255,0), thickness=-1) + if show_text: + self.w_data = cv2.putText(self.w_data, "Left wrist", left_wrist, 2, 1, (255,0,0), 2, 2) + self.w_data = cv2.putText(self.w_data, "Right wrist", right_wrist, 2, 1, (255,0,0), 2, 2) + def add_analysis_frame(self): self.logger.debug("Draw the analysis frame") self.w_data = cv2.rectangle(self.w_data, [0,0], (self.w_data.shape[1], self.w_data.shape[0]), color=(0,255,0), thickness=5) @@ -136,4 +139,25 @@ def add_face_details(self, multi_face_landmarks): landmark_drawing_spec=None, connection_drawing_spec=mp_utils.DrawingSpec(color=(255, 255, 0), thickness=5, circle_radius=1)) else: - self.logger.debug("No face found") \ No newline at end of file + self.logger.debug("No face found") + + def add_progress_bar(self, percent): + # draw progress bar + bar_y_offset = self.height - 100 + bar_width = 500 + w = self.width + start_point = (int(w/2 - bar_width/2), bar_y_offset) + end_point = (int(w/2 + bar_width/2), bar_y_offset + 20) + adj_percent = 1.0 if percent / .6 >= 1.0 else percent / .6 + + progress_end_point = (int(w/2 - bar_width/2 + (bar_width*(adj_percent))), 20 + bar_y_offset) + color = (255, 255, 117) + progress_color = (0, 0, 255) + thickness = -1 + + self.w_data = cv2.rectangle(self.w_data, start_point, end_point, color, thickness) + self.w_data = cv2.rectangle(self.w_data, start_point, progress_end_point, progress_color, thickness) + display_perc = int((percent * 100) / 0.6) + display_perc = 100 if display_perc >= 100 else display_perc + self.w_data = cv2.putText(self.w_data, str(display_perc) + "%", (int(w/2 - bar_width/2), 10 + bar_y_offset), 2, 1, (255,0,0), 2, 2) + self.w_data = cv2.putText(self.w_data, "Awake", (int(w/2 - bar_width/2 + 85), 10 + bar_y_offset), 2, 1, (255,0,0), 2, 2) \ No newline at end of file From 5f6a2742f602a4b711a9fb6bb5e3a03782188455 Mon Sep 17 00:00:00 2001 From: nos86 Date: Sat, 25 Mar 2023 00:46:17 +0100 Subject: [PATCH 12/31] refactor: :art: implementation of decision logic --- sleepy_baby/__init__.py | 16 +++-- sleepy_baby/decision_logic.py | 112 +++++++++++++++++++--------------- 2 files changed, 75 insertions(+), 53 deletions(-) diff --git a/sleepy_baby/__init__.py b/sleepy_baby/__init__.py index 98047a5..7cce8e2 100644 --- a/sleepy_baby/__init__.py +++ b/sleepy_baby/__init__.py @@ -42,8 +42,7 @@ def __init__(self, min_tracking_confidence=face_min_tracking_confidence) self.set_working_area() #Set entire area as working area self.set_output() #Set default values - - #self.logic = decision_logic() + self.logic = DecisionLogic() self.logger.info("SleepyBaby is configured") def set_output(self, @@ -61,15 +60,21 @@ def set_output(self, self.show_progress_bar = show_progress_bar def start_thread(self, frame_q, stop_event, pause=0.1, ): - def loop(self, frame_q, stop_event, pause): + def process_loop(self, frame_q, stop_event, pause): while stop_event.is_set() is False: if len(frame_q)>0: frame = self.processFrame(frame_q.pop(), return_image = self.processed_frame is None) if frame is not None: self.processed_frame = frame time.sleep(pause) - self.process_t = Thread(target=loop, args=(self, frame_q, stop_event, pause)) + def evaluate_loop(logic, stop_event, pause=1): + while stop_event.is_set() is False: + logic.update() + time.sleep(pause) + self.process_t = Thread(target=process_loop, args=(self, frame_q, stop_event, pause)) self.process_t.start() + self.evaluate_t = Thread(target=evaluate_loop, args=(self.logic, stop_event)) + self.evaluate_t.start() @@ -83,6 +88,7 @@ def set_working_area(self, x_offset=0, y_offset=0, width=None, height=None): def processFrame(self, image, return_image=True): frame = Frame(image, self.x_offset, self.y_offset, self.width, self.height) analysis, pose, face = self.process_baby_image_models(frame.w_data) + self.logic.push(analysis) if return_image: if self.show_frame: frame.add_analysis_frame() @@ -93,7 +99,7 @@ def processFrame(self, image, return_image=True): if self.show_face_details: frame.add_face_details(face) if self.show_progress_bar: - frame.add_progress_bar(0.45) #TODO: get this value from decision object + frame.add_progress_bar(self.logic.avg_awake) return frame.getAugmentedFrame() def process_baby_image_models(self, frame): diff --git a/sleepy_baby/decision_logic.py b/sleepy_baby/decision_logic.py index b39dbe4..1fd0aa6 100644 --- a/sleepy_baby/decision_logic.py +++ b/sleepy_baby/decision_logic.py @@ -27,66 +27,82 @@ def __init__(self): self.eyes_open_state = False self.is_awake = False self.body_found = False + self.eyes_found = False + self.avg_awake = 0 - def update(self, analysis:dict, eyes_threshold:float=0.75, wrist_threshold:int=25) -> None: #every second + + def push(self, analysis): self.body_found = analysis['body_detected'] - if self.body_found: + self.eyes_found = analysis['face_detected'] + if self.body_found: self.movement_q.append((analysis['left_wrist_coords'], analysis['right_wrist_coords'])) - - if (movement_list_len := len(self.movement_q)) > 5: - positions = np.reshape(self.movement_q, (movement_list_len, 4)).T - st_dev = [statistics.pstdev(pos) for pos in positions] - avg_std = sum(st_dev)/4 - - if int(avg_std) < wrist_threshold: - self.logger.info('No movement, vote sleeping') - self.awake_q.append(0) - else: - print("Movement, vote awake") - self.logger.info("Movement, vote awake") - self.awake_q.append(1) - if analysis['face_detected']: + if self.eyes_found: if analysis['eyes_open'] is False: - if analysis['mouth_open']: - self.logger.info('Eyes closed, mouth open, crying or yawning, consider awake.') - self.eyes_open_q.append(1) - else: - self.logger.info('Eyes closed, mouth closed, consider sleeping.') - self.eyes_open_q.append(0) + self.eyes_open_q.append(1 if analysis['mouth_open'] else 0) else: - self.logger.info('Eyes open, consider awake.') self.eyes_open_q.append(1) + #no_eyes_found + #no_body_found + + + def update(self, eyes_threshold:float=0.75, wrist_threshold:int=25) -> None: #every second + + if self.body_found is False: #throttled_handle_no_body_found + self.awake_q.append(1) + if (self.eyes_found is False) and (len(self.eyes_open_q)>0): #throttled_handle_no_eyes_found + self.eyes_open_q.popleft() + + #self.awake_voting_logic() + if len(self.eyes_open_q) > self.eyes_open_q.maxlen/2: + avg = sum(self.eyes_open_q) / len(self.eyes_open_q) + if avg > 0.75: #eyes_open + self.eyes_open_state = True + self.logger.info("Eyes Open: vote awake") + self.awake_q.append(1) else: - self.logger.info('No face found, depreciate queue') - if len(self.eyes_open_q) > 0: - self.eyes_open_q.popleft() + self.eyes_open_state = False + self.awake_q.append(0) + self.logger.info("Eyes closed: vote sleeping") else: - self.logger.info('No body found, vote awake') - self.awake_q.append(1) - if len(self.movement_q): + self.logger.debug("Not voting on eyes, eye que is too short.") + + #self.movement_voting_logic(body_found) + if self.body_found is False: + self.logger.debug("No body found, depreciate movement queue.") + if len(self.movement_q)>0: self.movement_q.popleft() - #Evaluate eyes - if len(self.eyes_open_q)>(self.eyes_open_q.maxlen/2): - eyes_score = sum(self.eyes_open_q) / len(self.eyes_open_q) - self.eyes_open_state = eyes_score > eyes_threshold - self.logger.info("Vote {'awake' if self.eyes_open_state else 'sleep'}") - self.awake_q.append(1 if self.eyes_open_state else 0) - else: - self.logger.info("Not voting on eyes, eye queue too short.") - - - def set_wakeness_status(self, img): #each 10s - if len(self.awake_q): - avg_awake = sum(self.awake_q) / len(self.awake_q) - if avg_awake >= 0.6 and self.is_awake == False: - self.need_to_clean_this_up(True, img) - elif avg_awake < 0.6 and self.is_awake == True: - self.need_to_clean_this_up(False, img) - + elif (movement_list_len := len(self.movement_q)) > 5: + positions = np.reshape(self.movement_q, (movement_list_len, 4)).T + st_dev = [statistics.pstdev(pos) for pos in positions] + avg_std = sum(st_dev)/4 + + if int(avg_std) < wrist_threshold: + self.logger.info('No movement, vote sleeping') + self.awake_q.append(0) + else: + print("Movement, vote awake") + self.logger.info("Movement, vote awake") + self.awake_q.append(1) + + #self.set_wakeness_status() + if len(self.awake_q)>0: + self.avg_awake = sum(self.awake_q) / len(self.awake_q) + if self.avg_awake >= 0.6 and self.is_awake == False: + self.logger.info("Awake Event") + self.is_awake = True + #self.need_to_clean_this_up(True) #TODO + elif self.avg_awake <0.6 and self.is_awake == True: + self.logger.info("Sleep Event") + self.is_awake = False + #self.need_to_clean_this_up(True) #TODO + + #self.periodic_wakeness_check() + + # This is placeholder until improve sensitivity of transitioning between waking and sleeping. # Explanation: Sometimes when baby is waking up, he'll open and close his eyes for a couple of minutes... # TODO: Fine-tune sensitivity of voting, for now, don't allow toggling between wake & sleep within N seconds - @debounce(180) + #@debounce(180) def need_to_clean_this_up(self, wake_status, img): str_timestamp = str(int(time.time())) sleep_data_base_path = os.getenv("SLEEP_DATA_PATH") From 46ca1846b7452002bbe2771caa324ec2215b35e5 Mon Sep 17 00:00:00 2001 From: nos86 Date: Tue, 28 Mar 2023 19:19:43 +0200 Subject: [PATCH 13/31] added example for main.py --- main.py | 132 +++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 98 insertions(+), 34 deletions(-) diff --git a/main.py b/main.py index cfe4641..0927dea 100644 --- a/main.py +++ b/main.py @@ -1,14 +1,16 @@ import cv2 -from threading import Thread +from threading import Thread, Event import os from collections import deque import _thread import logging +import time from dotenv import load_dotenv # from cast_service import CastSoundService from http.server import HTTPServer, SimpleHTTPRequestHandler from sleepy_baby import SleepyBaby +from sleepy_baby import Frame # Uncomment if want phone notifications during daytime wakings. # Configuration of telegram API key in this dir also needed. @@ -27,55 +29,117 @@ # Queue shared between the frame publishing thread and the consuming thread # This is to get around an underlying bug, described at end of this file. -frame_q = deque(maxlen=20) +#frame_q = deque(maxlen=20) #Load SleepyBaby -logging.info('Initializing...') -sleepy_baby = SleepyBaby() -logging.info('\nInitialization complete.') +#logging.info('Initializing...') +#sleepy_baby = SleepyBaby() +#logging.info('\nInitialization complete.') # Below http server is used for the web app to request latest sleep data -class CORSRequestHandler(SimpleHTTPRequestHandler): - def end_headers(self): - self.send_header('Access-Control-Allow-Origin', '*') - self.send_header('Access-Control-Allow-Methods', 'GET') - self.send_header('Cache-Control', 'no-store, no-cache, must-revalidate') - return super(CORSRequestHandler, self).end_headers() +#class CORSRequestHandler(SimpleHTTPRequestHandler): +# def end_headers(self): +# self.send_header('Access-Control-Allow-Origin', '*') +# self.send_header('Access-Control-Allow-Methods', 'GET') +# self.send_header('Cache-Control', 'no-store, no-cache, must-revalidate') +# return super(CORSRequestHandler, self).end_headers() -def start_server(): - httpd = HTTPServer(('0.0.0.0', 8000), CORSRequestHandler) - httpd.serve_forever() +#def start_server(): +# httpd = HTTPServer(('0.0.0.0', 8000), CORSRequestHandler) +# httpd.serve_forever() -_thread.start_new_thread(start_server, ()) +#_thread.start_new_thread(start_server, ()) -def receive(producer_q): - print("Start receiving frames.") - cam_ip = os.environ['CAM_IP'] - cam_pw = os.environ['CAM_PW'] - connect_str = "rtsp://admin:" + cam_pw + "@" + cam_ip - connect_str2 = connect_str + ":554" + "//h264Preview_01_main" # this might be different depending on camera used +#def receive(producer_q): +# print("Start receiving frames.") +# cam_ip = os.environ['CAM_IP'] +# cam_pw = os.environ['CAM_PW'] +# connect_str = "rtsp://admin:" + cam_pw + "@" + cam_ip +# connect_str2 = connect_str + ":554" + "//h264Preview_01_main" # this might be different depending on camera used - os.environ['OPENCV_FFMPEG_CAPTURE_OPTIONS'] = 'rtsp_transport;tcp' # Use tcp instead of udp if stream is unstable - c = cv2.VideoCapture(connect_str) + # os.environ['OPENCV_FFMPEG_CAPTURE_OPTIONS'] = 'rtsp_transport;tcp' # Use tcp instead of udp if stream is unstable +# c = cv2.VideoCapture(connect_str) - next_frame = 0 - fps = 30 - while(c.isOpened()): - ret, img = c.read() - if ret: - producer_q.append(img) +# next_frame = 0 +# fps = 30 +# while(c.isOpened()): +# ret, img = c.read() +# if ret: +# producer_q.append(img) # Had to split frame receive and processing into different threads due to underlying FFMPEG issue. Read more here: # https://stackoverflow.com/questions/49233433/opencv-read-errorh264-0x8f915e0-error-while-decoding-mb-53-20-bytestream # Current solution is to insert into deque on the thread receiving images, and process on the other -p1 = Thread(target=receive, args=(frame_q,)) -p2 = Thread(target=sleepy_baby.live, args=(frame_q,)) -p1.start() -p2.start() +#p1 = Thread(target=receive, args=(frame_q,)) +#p2 = Thread(target=sleepy_baby.live, args=(frame_q,)) +#p1.start() +#p2.start() # Note: to test w/ recorded footage, comment out above threads, and uncomment next line # TODO: use command line args rather than commenting out code -# sleepy_baby.recorded() \ No newline at end of file +# sleepy_baby.recorded() + +frame_q = deque(maxlen=2) +terminate_event = Event() + +#Load SleepyBaby +sleepy_baby = SleepyBaby(body_min_detection_confidence=0.1, body_min_tracking_confidence=0.1) +sleepy_baby.set_working_area(800, 300, 1100, 550) +sleepy_baby.set_output(show_body_details=True) +sleepy_baby.start_thread(frame_q, terminate_event) + +def show_video(sb_obj): + while terminate_event.is_set() is False: + if sb_obj.processed_frame is not None: + cv2.imshow("VIDEO", cv2.resize(sb_obj.processed_frame, (960,540))) + cv2.waitKey(1) + sb_obj.processed_frame = None + time.sleep(0.3) +p2 = Thread(target=show_video, args=(sleepy_baby,)) +p2.start() + +try: + vcap = cv2.VideoCapture("rtsp://192.168.62.198/ch0_0.h264") + #vcap = cv2.VideoCapture("rtsp://192.168.62.185:1935/") + while True: + _, img = vcap.read() + frame_q.append(img) #cv2.cvtColor(img, cv2.COLOR_BGR2RGB)) +except KeyboardInterrupt: + terminate_event.set() + logging.error("User Abort") + + + + + +# video = 0 + +# if video: +# fourcc = cv2.VideoWriter_fourcc('m', 'p', '4', 'v') +# video = cv2.VideoCapture('test.mp4') +# success, image = video.read() + +# writer = cv2.VideoWriter('output.mp4', fourcc, video.get(cv2.CAP_PROP_FPS), (image.shape[0], image.shape[1])) +# while success: +# success,image = video.read() +# if image is not None: +# frame = Frame(image) +# analysis, pose, face = sleepy_baby.process_baby_image_models(frame.w_data) +# frame.add_analysis_frame() +# frame.add_body_details(pose) +# frame.add_face_details(face) +# writer.write(frame.getAugmentedFrame()) +# cv2.destroyAllWindows() +# writer.release() +# video.release() +# else: +# img = cv2.imread("test.jpg") +# frame = Frame(cv2.cvtColor(img, cv2.COLOR_BGR2RGB), 700, 300, 1220,700) +# analysis, pose, face = sleepy_baby.process_baby_image_models(frame.w_data) +# frame.add_analysis_frame() +# frame.add_body_details(pose) +# frame.add_face_details(face) +# cv2.imwrite("debug.jpg", frame.getAugmentedFrame()) \ No newline at end of file From 1e273ffa8f04ae4f24e497676e3fc26212fd733b Mon Sep 17 00:00:00 2001 From: "CSI\\salvatore musumeci" Date: Thu, 30 Mar 2023 15:25:54 +0200 Subject: [PATCH 14/31] feat(cli): :sparkles: Add command line configuration for script execution - Update main.py for improved configuration loading and parsing - Switch to using `dotenv_values` for loading configuration values - Add command line argument parsing with `argparse` - Set defaults for command line arguments based on `.env` file values - Change logging configuration to write to a file specified in the `.env` file --- main.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index 0927dea..539be4b 100644 --- a/main.py +++ b/main.py @@ -5,7 +5,7 @@ import _thread import logging import time -from dotenv import load_dotenv +from dotenv import dotenv_values # from cast_service import CastSoundService from http.server import HTTPServer, SimpleHTTPRequestHandler @@ -16,6 +16,25 @@ # Configuration of telegram API key in this dir also needed. # import telegram_send +#Load configuration from .env file +config = dotenv_values() + +#Config command ling +parser = argparse.ArgumentParser( + prog="Sleepy Baby", + description="Library to monitor the status of your baby based on image recognition" +) + +parser.add_argument('-s', '--source', type=str, default=config['VIDEO_PATH'], help="Input path for video") +parser.add_argument('-v', '--verbose', action="store_true", default=config['DEBUG'].lower()=="true", help="Activate Debug Mode") +parser.add_argument('--log-on-screen', action="store_true", help="Show logs on screen instead of saving on file") +parser.add_argument('--log-path', default=config['SLEEP_DATA_PATH'], help="Set log path") +parser.add_argument('-r', '--recorded', action="store_true", help="Input is a recorded video. delay should be applied to simulate real-time") + +args = parser.parse_args() + + + #Set-up the logger logfile = os.getenv("SLEEP_DATA_PATH") + '/sleepy_logs.log' logging.basicConfig(filename=logfile, From a9d0ac12ddc098bf1e2677a349198dc59f254b68 Mon Sep 17 00:00:00 2001 From: "CSI\\salvatore musumeci" Date: Thu, 30 Mar 2023 15:33:36 +0200 Subject: [PATCH 15/31] feat: Refactor logging setup and options - Revamped logger setup using kwargs in `main.py` - Added flags for logging to screen and writing to a file - Logging level set to DEBUG with `--verbose` flag --- main.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/main.py b/main.py index 539be4b..34a6836 100644 --- a/main.py +++ b/main.py @@ -36,15 +36,17 @@ #Set-up the logger -logfile = os.getenv("SLEEP_DATA_PATH") + '/sleepy_logs.log' -logging.basicConfig(filename=logfile, - filemode='a+', - format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s', - datefmt='%H:%M:%S', - level=logging.INFO) - -#Load configuration from .env file -load_dotenv() +logger_kwargs = { + 'format': '%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s', + 'datefmt': '%H:%M:%S', + 'level': logging.DEBUG if args.verbose else logging.INFO + } + +if args.log_on_screen: + logging.basicConfig(**logger_kwargs) +else: + logfile = args.log_path + '/sleepy_logs.log' + logging.basicConfig(filename=logfile, filemode='a+', **logger_kwargs) # Queue shared between the frame publishing thread and the consuming thread # This is to get around an underlying bug, described at end of this file. From 734d53380e52b56f2484de8002e5aaa4354c7665 Mon Sep 17 00:00:00 2001 From: "CSI\\salvatore musumeci" Date: Thu, 30 Mar 2023 15:35:52 +0200 Subject: [PATCH 16/31] refactor: Refactor video processing for multithreading. - Separate frame receiving and processing into distinct threads - Initialize `SleepyBaby` object with specified parameters - Use `output` object to display body details in processed frame - Use `SleepyBaby.start_thread` to start frame processing thread - Create `show_video` thread to display processed frames - Read and append frames from `cv2.VideoCapture` to `frame_q` for processing - Set `terminate_event` to end all threads in case of program termination or error in video source --- main.py | 98 +++++++++++++++++++++++++++++++-------------------------- 1 file changed, 54 insertions(+), 44 deletions(-) diff --git a/main.py b/main.py index 34a6836..43b2341 100644 --- a/main.py +++ b/main.py @@ -49,13 +49,61 @@ logging.basicConfig(filename=logfile, filemode='a+', **logger_kwargs) # Queue shared between the frame publishing thread and the consuming thread -# This is to get around an underlying bug, described at end of this file. -#frame_q = deque(maxlen=20) +# Had to split frame receive and processing into different threads due to underlying FFMPEG issue. Read more here: +# https://stackoverflow.com/questions/49233433/opencv-read-errorh264-0x8f915e0-error-while-decoding-mb-53-20-bytestream +# Current solution is to insert into deque on the thread receiving images, and process on the other +frame_q = deque(maxlen=2) +terminate_event = Event() #Load SleepyBaby -#logging.info('Initializing...') -#sleepy_baby = SleepyBaby() -#logging.info('\nInitialization complete.') +logging.info('Initializing...') +sleepy_baby = SleepyBaby(body_min_detection_confidence=0.1, body_min_tracking_confidence=0.1) +#sleepy_baby.set_working_area(800, 300, 1100, 550) +sleepy_baby.set_output(show_body_details=True) +sleepy_baby.start_thread(frame_q, terminate_event) +logging.info('Initialization complete.') + +#Create a thread to show the results of the processing +def show_video(sb_obj): + logging.info("show_video thread is started") + while terminate_event.is_set() is False: + if sb_obj.processed_frame is not None: + print("Thread") + cv2.imshow("VIDEO", cv2.resize(sb_obj.processed_frame, (960,540))) + cv2.waitKey(1) + sb_obj.processed_frame = None + else: + logging.debug("No image to process") + time.sleep(0.3) + logging.info("show_video thread is terminated by event") +p2 = Thread(target=show_video, args=(sleepy_baby,)) + + +try: + vcap = cv2.VideoCapture(args.source) + if vcap.isOpened(): + success = True + logging.info("Start receiving frames.") + fps = vcap.get(cv2.CAP_PROP_FPS) + p2.start() + while success: + success, img = vcap.read() + if success is False: + terminate_event.set() #Error in streaming reading + frame_q.append(img) #cv2.cvtColor(img, cv2.COLOR_BGR2RGB)) + if args.recorded: + time.sleep(1.0/fps) + logging.error("Error in frame retrieve. Program will be ended") + else: + logging.error("Unable to open the streaming") +except KeyboardInterrupt: + logging.error("User Abort") +finally: + terminate_event.set() + vcap.release() + + + # Below http server is used for the web app to request latest sleep data @@ -91,50 +139,12 @@ # producer_q.append(img) -# Had to split frame receive and processing into different threads due to underlying FFMPEG issue. Read more here: -# https://stackoverflow.com/questions/49233433/opencv-read-errorh264-0x8f915e0-error-while-decoding-mb-53-20-bytestream -# Current solution is to insert into deque on the thread receiving images, and process on the other + #p1 = Thread(target=receive, args=(frame_q,)) #p2 = Thread(target=sleepy_baby.live, args=(frame_q,)) #p1.start() #p2.start() -# Note: to test w/ recorded footage, comment out above threads, and uncomment next line -# TODO: use command line args rather than commenting out code -# sleepy_baby.recorded() - -frame_q = deque(maxlen=2) -terminate_event = Event() - -#Load SleepyBaby -sleepy_baby = SleepyBaby(body_min_detection_confidence=0.1, body_min_tracking_confidence=0.1) -sleepy_baby.set_working_area(800, 300, 1100, 550) -sleepy_baby.set_output(show_body_details=True) -sleepy_baby.start_thread(frame_q, terminate_event) - -def show_video(sb_obj): - while terminate_event.is_set() is False: - if sb_obj.processed_frame is not None: - cv2.imshow("VIDEO", cv2.resize(sb_obj.processed_frame, (960,540))) - cv2.waitKey(1) - sb_obj.processed_frame = None - time.sleep(0.3) -p2 = Thread(target=show_video, args=(sleepy_baby,)) -p2.start() - -try: - vcap = cv2.VideoCapture("rtsp://192.168.62.198/ch0_0.h264") - #vcap = cv2.VideoCapture("rtsp://192.168.62.185:1935/") - while True: - _, img = vcap.read() - frame_q.append(img) #cv2.cvtColor(img, cv2.COLOR_BGR2RGB)) -except KeyboardInterrupt: - terminate_event.set() - logging.error("User Abort") - - - - # video = 0 From 40257d6d4e102af612a9f8bba0b463dfe8c4ba71 Mon Sep 17 00:00:00 2001 From: "CSI\\salvatore musumeci" Date: Mon, 3 Apr 2023 12:40:17 +0200 Subject: [PATCH 17/31] refactor: Refactor progress bar function signature and style. - Update function signature in sleepy_baby/frame.py - Simplify progress bar drawing code - Modify progress bar position and label text - Improve progress bar text display --- sleepy_baby/frame.py | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/sleepy_baby/frame.py b/sleepy_baby/frame.py index 810dffe..4b71c3d 100644 --- a/sleepy_baby/frame.py +++ b/sleepy_baby/frame.py @@ -141,23 +141,16 @@ def add_face_details(self, multi_face_landmarks): else: self.logger.debug("No face found") - def add_progress_bar(self, percent): + def add_progress_bar(self, percent, bar_width=500, bar_height = 20, bar_y_offset=100, backcolor=(255, 255, 117), forecolor=(0,0,255), textcolor=(255,0,0)): # draw progress bar - bar_y_offset = self.height - 100 - bar_width = 500 - w = self.width - start_point = (int(w/2 - bar_width/2), bar_y_offset) - end_point = (int(w/2 + bar_width/2), bar_y_offset + 20) adj_percent = 1.0 if percent / .6 >= 1.0 else percent / .6 - - progress_end_point = (int(w/2 - bar_width/2 + (bar_width*(adj_percent))), 20 + bar_y_offset) - color = (255, 255, 117) - progress_color = (0, 0, 255) - thickness = -1 - - self.w_data = cv2.rectangle(self.w_data, start_point, end_point, color, thickness) - self.w_data = cv2.rectangle(self.w_data, start_point, progress_end_point, progress_color, thickness) - display_perc = int((percent * 100) / 0.6) - display_perc = 100 if display_perc >= 100 else display_perc - self.w_data = cv2.putText(self.w_data, str(display_perc) + "%", (int(w/2 - bar_width/2), 10 + bar_y_offset), 2, 1, (255,0,0), 2, 2) - self.w_data = cv2.putText(self.w_data, "Awake", (int(w/2 - bar_width/2 + 85), 10 + bar_y_offset), 2, 1, (255,0,0), 2, 2) \ No newline at end of file + display_perc = min(int((percent * 100) / 0.6), 100) + + start_point = (int(self.width/2 - bar_width/2), self.height - bar_y_offset) + end_point = (start_point[0] + bar_width, start_point[1] + bar_height) + mid_point = (start_point[0] + int(bar_width * adj_percent), start_point[1] + bar_height) + text_y_position = start_point[1] - int(bar_height / 5) + + self.w_data = cv2.rectangle(self.w_data, start_point, end_point, backcolor, thickness = -1) + self.w_data = cv2.rectangle(self.w_data, start_point, mid_point, forecolor, thickness = -1) + self.w_data = cv2.putText(self.w_data, str(display_perc) + "% Awake", (start_point[0], text_y_position), 2, 1, textcolor, 2, 2) \ No newline at end of file From 5c82154ab1bc0d025cea025ccfe3da3c962c0744 Mon Sep 17 00:00:00 2001 From: "CSI\\salvatore musumeci" Date: Mon, 3 Apr 2023 12:43:46 +0200 Subject: [PATCH 18/31] refactor: Refactor progress bar functionality in frame.py - Refactor `add_progress_bar` function in `frame.py` for improved readability and consistency - Update variable naming to use `adj_percent` instead of `display_perc` --- sleepy_baby/frame.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sleepy_baby/frame.py b/sleepy_baby/frame.py index 4b71c3d..f7a2cd9 100644 --- a/sleepy_baby/frame.py +++ b/sleepy_baby/frame.py @@ -143,9 +143,7 @@ def add_face_details(self, multi_face_landmarks): def add_progress_bar(self, percent, bar_width=500, bar_height = 20, bar_y_offset=100, backcolor=(255, 255, 117), forecolor=(0,0,255), textcolor=(255,0,0)): # draw progress bar - adj_percent = 1.0 if percent / .6 >= 1.0 else percent / .6 - display_perc = min(int((percent * 100) / 0.6), 100) - + adj_percent = min(1.0, percent / 0.6) start_point = (int(self.width/2 - bar_width/2), self.height - bar_y_offset) end_point = (start_point[0] + bar_width, start_point[1] + bar_height) mid_point = (start_point[0] + int(bar_width * adj_percent), start_point[1] + bar_height) @@ -153,4 +151,4 @@ def add_progress_bar(self, percent, bar_width=500, bar_height = 20, bar_y_offset self.w_data = cv2.rectangle(self.w_data, start_point, end_point, backcolor, thickness = -1) self.w_data = cv2.rectangle(self.w_data, start_point, mid_point, forecolor, thickness = -1) - self.w_data = cv2.putText(self.w_data, str(display_perc) + "% Awake", (start_point[0], text_y_position), 2, 1, textcolor, 2, 2) \ No newline at end of file + self.w_data = cv2.putText(self.w_data, str(int(adj_percent * 100)) + "% Awake", (start_point[0], text_y_position), 2, 1, textcolor, 2, 2) \ No newline at end of file From 0085a27664a51358e43d9f7af719bc5bf39c1457 Mon Sep 17 00:00:00 2001 From: "CSI\\salvatore musumeci" Date: Mon, 3 Apr 2023 12:49:08 +0200 Subject: [PATCH 19/31] fix: :bug: Add `argparse` import and remove debug `print` statement in `main.py`. --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 43b2341..469638a 100644 --- a/main.py +++ b/main.py @@ -1,3 +1,4 @@ +import argparse import cv2 from threading import Thread, Event import os @@ -68,7 +69,6 @@ def show_video(sb_obj): logging.info("show_video thread is started") while terminate_event.is_set() is False: if sb_obj.processed_frame is not None: - print("Thread") cv2.imshow("VIDEO", cv2.resize(sb_obj.processed_frame, (960,540))) cv2.waitKey(1) sb_obj.processed_frame = None From 727aa3e00d675d18b70ed7f325ce0feeb67b233e Mon Sep 17 00:00:00 2001 From: "CSI\\salvatore musumeci" Date: Mon, 3 Apr 2023 14:52:06 +0200 Subject: [PATCH 20/31] refactor: Move attributes in function signature - Add optional parameters to `add_body_details` and `add_wrist_position` methods in `frame.py` - Change colors and thickness of face details in `add_face_details` method in `frame.py` - Add `refine_landmarks` param to `SleepyBaby` class in `__init__.py` --- sleepy_baby/__init__.py | 7 ++-- sleepy_baby/frame.py | 78 +++++++++++++++++++++++++++-------------- 2 files changed, 55 insertions(+), 30 deletions(-) diff --git a/sleepy_baby/__init__.py b/sleepy_baby/__init__.py index 7cce8e2..ffae0fc 100644 --- a/sleepy_baby/__init__.py +++ b/sleepy_baby/__init__.py @@ -29,13 +29,14 @@ def __init__(self, body_min_detection_confidence=0.8, body_min_tracking_confidence=0.8, face_min_detection_confidence=0.7, - face_min_tracking_confidence=0.7): + face_min_tracking_confidence=0.7, + refine_landmarks = True): self.logger = logging.getLogger(self.__class__.__name__) self.logger.debug("SleepyBaby is starting") self.processed_frame = None #It is used to produce post-processed video self.process_t = None #Process Thread self.face = mp.solutions.face_mesh.FaceMesh(max_num_faces=1, - refine_landmarks=True, + refine_landmarks=refine_landmarks, min_detection_confidence=body_min_detection_confidence, min_tracking_confidence=body_min_tracking_confidence) self.pose = mp.solutions.pose.Pose(min_detection_confidence=face_min_detection_confidence, @@ -76,8 +77,6 @@ def evaluate_loop(logic, stop_event, pause=1): self.evaluate_t = Thread(target=evaluate_loop, args=(self.logic, stop_event)) self.evaluate_t.start() - - def set_working_area(self, x_offset=0, y_offset=0, width=None, height=None): self.x_offset = x_offset self.y_offset = y_offset diff --git a/sleepy_baby/frame.py b/sleepy_baby/frame.py index f7a2cd9..f906ad6 100644 --- a/sleepy_baby/frame.py +++ b/sleepy_baby/frame.py @@ -66,7 +66,11 @@ def getAugmentedFrame(self) -> np.ndarray: return frame - def add_body_details(self, pose_landmarks): + def add_body_details(self, + pose_landmarks, + landmark_color=(255, 150, 255), + landmark_thickness=2, + landmark_circle_radius=2): if pose_landmarks: self.logger.debug("Body detected in the frame") CUTOFF_THRESHOLD = 10 # head and face @@ -78,32 +82,36 @@ def add_body_details(self, pose_landmarks): mp_utils.draw_landmarks(self.w_data, pose_landmarks, MY_CONNECTIONS, - landmark_drawing_spec=mp_utils.DrawingSpec( color=(255, 0, 0), - thickness=10, - circle_radius=2), - connection_drawing_spec=mp_utils.DrawingSpec(color=(0, 0, 255), - thickness=3, - circle_radius=2) + landmark_drawing_spec=mp_utils.DrawingSpec( color=landmark_color, + thickness=landmark_thickness, + circle_radius=landmark_circle_radius) ) else: self.logger.debug("No body detected in frame") - def add_wrist_position(self, pose_landmarks, show_text=True): + def add_wrist_position(self, pose_landmarks, show_text=True, text_color=(255,0,0), point_color=(255,0,0), point_radius=2): if pose_landmarks: left_wrist = (int(self.width * pose_landmarks.landmark[15].x), int(self.height * pose_landmarks.landmark[15].y)) right_wrist = (int(self.width * pose_landmarks.landmark[16].x), int(self.height * pose_landmarks.landmark[16].y)) - cv2.circle(self.w_data, left_wrist, radius=5, color=(255,255,0), thickness=-1) - cv2.circle(self.w_data, right_wrist, radius=5, color=(255,255,0), thickness=-1) + cv2.circle(self.w_data, left_wrist, radius=point_radius, color=point_color, thickness=-1) + cv2.circle(self.w_data, right_wrist, radius=point_radius, color=point_color, thickness=-1) if show_text: - self.w_data = cv2.putText(self.w_data, "Left wrist", left_wrist, 2, 1, (255,0,0), 2, 2) - self.w_data = cv2.putText(self.w_data, "Right wrist", right_wrist, 2, 1, (255,0,0), 2, 2) + self.w_data = cv2.putText(self.w_data, "Left wrist", left_wrist, 2, 1, text_color, 2, 2) + self.w_data = cv2.putText(self.w_data, "Right wrist", right_wrist, 2, 1, text_color, 2, 2) def add_analysis_frame(self): self.logger.debug("Draw the analysis frame") self.w_data = cv2.rectangle(self.w_data, [0,0], (self.w_data.shape[1], self.w_data.shape[0]), color=(0,255,0), thickness=5) - def add_face_details(self, multi_face_landmarks): + def add_face_details(self, + multi_face_landmarks, + details_thickness = 1, + left_eye_color=(255, 255, 0), + right_eye_color=(255, 150, 255), + top_lip_color = (255, 150, 255), + bottom_lip_color = (255, 255, 0) + ): """ add_face_details_to_image adds face details to image passed in arguments. @@ -125,19 +133,37 @@ def add_face_details(self, multi_face_landmarks): # INDICIES: https://github.com/tensorflow/tfjs-models/blob/838611c02f51159afdd77469ce67f0e26b7bbb23/face-landmarks-detection/src/mediapipe-facemesh/keypoints.ts # https://github.com/google/mediapipe/blob/master/mediapipe/python/solutions/face_mesh_connections.py - mp.solutions.drawing_utils.draw_landmarks( - image=self.w_data, - landmark_list=face_landmarks, - connections=mp.solutions.face_mesh.FACEMESH_RIGHT_EYE, - landmark_drawing_spec=None, - connection_drawing_spec=mp_utils.DrawingSpec(color=(255, 150, 255), thickness=5, circle_radius=1)) - - mp.solutions.drawing_utils.draw_landmarks( - image=self.w_data, - landmark_list=face_landmarks, - connections=mp.solutions.face_mesh.FACEMESH_LEFT_EYE, - landmark_drawing_spec=None, - connection_drawing_spec=mp_utils.DrawingSpec(color=(255, 255, 0), thickness=5, circle_radius=1)) + if right_eye_color is not None: + mp.solutions.drawing_utils.draw_landmarks( + image=self.w_data, + landmark_list=face_landmarks, + connections=mp.solutions.face_mesh.FACEMESH_RIGHT_EYE, + landmark_drawing_spec=None, + connection_drawing_spec=mp_utils.DrawingSpec(color=right_eye_color, thickness=details_thickness, circle_radius=1)) + + if left_eye_color is not None: + mp.solutions.drawing_utils.draw_landmarks( #a[0:9]+a[20:29] + image=self.w_data, + landmark_list=face_landmarks, + connections=mp.solutions.face_mesh.FACEMESH_LEFT_EYE, + landmark_drawing_spec=None, + connection_drawing_spec=mp_utils.DrawingSpec(color=left_eye_color, thickness=details_thickness, circle_radius=1)) + + if top_lip_color is not None: + mp.solutions.drawing_utils.draw_landmarks( + image=self.w_data, + landmark_list=face_landmarks, + connections=list(map(lambda x: x[29:40]+x[9:20], [list(mp.solutions.face_mesh.FACEMESH_LIPS)]))[0], + landmark_drawing_spec=None, + connection_drawing_spec=mp_utils.DrawingSpec(color=top_lip_color, thickness=details_thickness, circle_radius=1)) + + if bottom_lip_color is not None: + mp.solutions.drawing_utils.draw_landmarks( + image=self.w_data, + landmark_list=face_landmarks, + connections=list(map(lambda x: x[0:9]+x[20:29], [list(mp.solutions.face_mesh.FACEMESH_LIPS)]))[0], + landmark_drawing_spec=None, + connection_drawing_spec=mp_utils.DrawingSpec(color=bottom_lip_color, thickness=details_thickness, circle_radius=1)) else: self.logger.debug("No face found") From c9dd314a4ac8063f8de52472bd80cdfd75e6a40a Mon Sep 17 00:00:00 2001 From: "CSI\\salvatore musumeci" Date: Tue, 4 Apr 2023 10:29:30 +0200 Subject: [PATCH 21/31] refactor: Refactor main.py into class and add CLI support via Fire. - Refactor `main.py` into a class and methods - Add `fire` to the imports and requirements.txt - Create `live` and `recorded` methods for streaming and recorded video inputs --- main.py | 322 +++++++++++++++++++++++++++-------------------- requirements.txt | 1 + 2 files changed, 184 insertions(+), 139 deletions(-) diff --git a/main.py b/main.py index 469638a..927ea2f 100644 --- a/main.py +++ b/main.py @@ -1,4 +1,5 @@ import argparse +import fire import cv2 from threading import Thread, Event import os @@ -19,51 +20,15 @@ #Load configuration from .env file config = dotenv_values() +config['DEBUG'] = (config['DEBUG'].lower()=="true") #Transform in bool -#Config command ling -parser = argparse.ArgumentParser( - prog="Sleepy Baby", - description="Library to monitor the status of your baby based on image recognition" -) - -parser.add_argument('-s', '--source', type=str, default=config['VIDEO_PATH'], help="Input path for video") -parser.add_argument('-v', '--verbose', action="store_true", default=config['DEBUG'].lower()=="true", help="Activate Debug Mode") -parser.add_argument('--log-on-screen', action="store_true", help="Show logs on screen instead of saving on file") -parser.add_argument('--log-path', default=config['SLEEP_DATA_PATH'], help="Set log path") -parser.add_argument('-r', '--recorded', action="store_true", help="Input is a recorded video. delay should be applied to simulate real-time") - -args = parser.parse_args() - - - -#Set-up the logger -logger_kwargs = { - 'format': '%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s', - 'datefmt': '%H:%M:%S', - 'level': logging.DEBUG if args.verbose else logging.INFO - } - -if args.log_on_screen: - logging.basicConfig(**logger_kwargs) -else: - logfile = args.log_path + '/sleepy_logs.log' - logging.basicConfig(filename=logfile, filemode='a+', **logger_kwargs) - -# Queue shared between the frame publishing thread and the consuming thread -# Had to split frame receive and processing into different threads due to underlying FFMPEG issue. Read more here: -# https://stackoverflow.com/questions/49233433/opencv-read-errorh264-0x8f915e0-error-while-decoding-mb-53-20-bytestream -# Current solution is to insert into deque on the thread receiving images, and process on the other +# # Queue shared between the frame publishing thread and the consuming thread +# # Had to split frame receive and processing into different threads due to underlying FFMPEG issue. Read more here: +# # https://stackoverflow.com/questions/49233433/opencv-read-errorh264-0x8f915e0-error-while-decoding-mb-53-20-bytestream +# # Current solution is to insert into deque on the thread receiving images, and process on the other frame_q = deque(maxlen=2) terminate_event = Event() -#Load SleepyBaby -logging.info('Initializing...') -sleepy_baby = SleepyBaby(body_min_detection_confidence=0.1, body_min_tracking_confidence=0.1) -#sleepy_baby.set_working_area(800, 300, 1100, 550) -sleepy_baby.set_output(show_body_details=True) -sleepy_baby.start_thread(frame_q, terminate_event) -logging.info('Initialization complete.') - #Create a thread to show the results of the processing def show_video(sb_obj): logging.info("show_video thread is started") @@ -76,101 +41,180 @@ def show_video(sb_obj): logging.debug("No image to process") time.sleep(0.3) logging.info("show_video thread is terminated by event") -p2 = Thread(target=show_video, args=(sleepy_baby,)) - - -try: - vcap = cv2.VideoCapture(args.source) - if vcap.isOpened(): - success = True - logging.info("Start receiving frames.") - fps = vcap.get(cv2.CAP_PROP_FPS) - p2.start() - while success: - success, img = vcap.read() - if success is False: - terminate_event.set() #Error in streaming reading - frame_q.append(img) #cv2.cvtColor(img, cv2.COLOR_BGR2RGB)) - if args.recorded: - time.sleep(1.0/fps) - logging.error("Error in frame retrieve. Program will be ended") - else: - logging.error("Unable to open the streaming") -except KeyboardInterrupt: - logging.error("User Abort") -finally: - terminate_event.set() - vcap.release() - - - - - -# Below http server is used for the web app to request latest sleep data -#class CORSRequestHandler(SimpleHTTPRequestHandler): -# def end_headers(self): -# self.send_header('Access-Control-Allow-Origin', '*') -# self.send_header('Access-Control-Allow-Methods', 'GET') -# self.send_header('Cache-Control', 'no-store, no-cache, must-revalidate') -# return super(CORSRequestHandler, self).end_headers() - -#def start_server(): -# httpd = HTTPServer(('0.0.0.0', 8000), CORSRequestHandler) -# httpd.serve_forever() - -#_thread.start_new_thread(start_server, ()) - - -#def receive(producer_q): -# print("Start receiving frames.") -# cam_ip = os.environ['CAM_IP'] -# cam_pw = os.environ['CAM_PW'] -# connect_str = "rtsp://admin:" + cam_pw + "@" + cam_ip -# connect_str2 = connect_str + ":554" + "//h264Preview_01_main" # this might be different depending on camera used - - # os.environ['OPENCV_FFMPEG_CAPTURE_OPTIONS'] = 'rtsp_transport;tcp' # Use tcp instead of udp if stream is unstable -# c = cv2.VideoCapture(connect_str) - -# next_frame = 0 -# fps = 30 -# while(c.isOpened()): -# ret, img = c.read() -# if ret: -# producer_q.append(img) - - - -#p1 = Thread(target=receive, args=(frame_q,)) -#p2 = Thread(target=sleepy_baby.live, args=(frame_q,)) -#p1.start() -#p2.start() - - -# video = 0 - -# if video: -# fourcc = cv2.VideoWriter_fourcc('m', 'p', '4', 'v') -# video = cv2.VideoCapture('test.mp4') -# success, image = video.read() - -# writer = cv2.VideoWriter('output.mp4', fourcc, video.get(cv2.CAP_PROP_FPS), (image.shape[0], image.shape[1])) -# while success: -# success,image = video.read() -# if image is not None: -# frame = Frame(image) -# analysis, pose, face = sleepy_baby.process_baby_image_models(frame.w_data) -# frame.add_analysis_frame() -# frame.add_body_details(pose) -# frame.add_face_details(face) -# writer.write(frame.getAugmentedFrame()) -# cv2.destroyAllWindows() -# writer.release() -# video.release() -# else: -# img = cv2.imread("test.jpg") -# frame = Frame(cv2.cvtColor(img, cv2.COLOR_BGR2RGB), 700, 300, 1220,700) -# analysis, pose, face = sleepy_baby.process_baby_image_models(frame.w_data) -# frame.add_analysis_frame() -# frame.add_body_details(pose) -# frame.add_face_details(face) -# cv2.imwrite("debug.jpg", frame.getAugmentedFrame()) \ No newline at end of file + + +class app: + """ Sleepy Baby App """ + def __init__(self, + verbose: bool = config['DEBUG'], + log_on_screen: bool = False, + log_path: str = config['SLEEP_DATA_PATH'], + body_min_detection_confidence: float = 0.1, + body_min_tracking_confidence: float = 0.1, + working_area_x: int = None, + working_area_y: int = None, + working_area_width: int = None, + working_area_height: int = None, + show_frame: bool=True, + show_wrist_position: bool=True, + show_wrist_text: bool=True, + show_body_details: bool=True, + show_face_details: bool=True, + show_progress_bar: bool=True): + logger_kwargs = { + 'format': '%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s', + 'datefmt': '%H:%M:%S', + 'level': logging.DEBUG if verbose else logging.INFO + } + if log_on_screen: + logging.basicConfig(**logger_kwargs) + else: + logfile = log_path + '/sleepy_logs.log' + logging.basicConfig(filename=logfile, filemode='a+', **logger_kwargs) + #Load SleepyBaby + logging.info('Initializing...') + self.sleepy_baby = SleepyBaby(body_min_detection_confidence=body_min_detection_confidence, + body_min_tracking_confidence=body_min_tracking_confidence) + self.sleepy_baby.set_working_area(working_area_x, working_area_y, working_area_width, working_area_height) + self.sleepy_baby.set_output(show_frame=show_frame, + show_wrist_position=show_wrist_position, + show_wrist_text=show_wrist_text, + show_body_details=show_body_details, + show_face_details=show_face_details, + show_progress_bar=show_progress_bar) + + logging.info('Initialization complete.') + + def live(self, source:str, return_image: bool = True): + """ + Run App based on streaming video + + Parameters + ---------- + source : str + url of streaming video + return_image: bool + define if post processed image should be displayed, default: True + """ + self._process_streaming(source, apply_delay_between_frames=False, return_image=return_image) + + def recorded(self, source:str, return_image: bool = True): + """ + Run App based on streaming video + + Parameters + ---------- + source : str + url of streaming video + return_image: bool + define if post processed image should be displayed, default: True + """ + self._process_streaming(source, apply_delay_between_frames=True, return_image=return_image) + + + def _process_streaming(self, source, apply_delay_between_frames=False, return_image=True): + try: + vcap = cv2.VideoCapture(source) + if vcap.isOpened(): + success = True + logging.info("Start receiving frames.") + fps = vcap.get(cv2.CAP_PROP_FPS) + self.sleepy_baby.start_thread(frame_q, terminate_event) + if return_image: + self.show_video_thread = Thread(target=show_video, args=(self.sleepy_baby,)) + self.show_video_thread.start() + while success: + success, img = vcap.read() + if success is False: + terminate_event.set() #Error in streaming reading + frame_q.append(img) #cv2.cvtColor(img, cv2.COLOR_BGR2RGB)) + if apply_delay_between_frames: + time.sleep(1.0/fps) + logging.error("Error in frame retrieve. Program will be ended") + else: + logging.error("Unable to open the streaming") + except KeyboardInterrupt: + logging.error("User Abort") + finally: + terminate_event.set() + vcap.release() + +fire.Fire(app) + + + + + + + + + + +# # Below http server is used for the web app to request latest sleep data +# #class CORSRequestHandler(SimpleHTTPRequestHandler): +# # def end_headers(self): +# # self.send_header('Access-Control-Allow-Origin', '*') +# # self.send_header('Access-Control-Allow-Methods', 'GET') +# # self.send_header('Cache-Control', 'no-store, no-cache, must-revalidate') +# # return super(CORSRequestHandler, self).end_headers() + +# #def start_server(): +# # httpd = HTTPServer(('0.0.0.0', 8000), CORSRequestHandler) +# # httpd.serve_forever() + +# #_thread.start_new_thread(start_server, ()) + + +# #def receive(producer_q): +# # print("Start receiving frames.") +# # cam_ip = os.environ['CAM_IP'] +# # cam_pw = os.environ['CAM_PW'] +# # connect_str = "rtsp://admin:" + cam_pw + "@" + cam_ip +# # connect_str2 = connect_str + ":554" + "//h264Preview_01_main" # this might be different depending on camera used + +# # os.environ['OPENCV_FFMPEG_CAPTURE_OPTIONS'] = 'rtsp_transport;tcp' # Use tcp instead of udp if stream is unstable +# # c = cv2.VideoCapture(connect_str) + +# # next_frame = 0 +# # fps = 30 +# # while(c.isOpened()): +# # ret, img = c.read() +# # if ret: +# # producer_q.append(img) + + + +# #p1 = Thread(target=receive, args=(frame_q,)) +# #p2 = Thread(target=sleepy_baby.live, args=(frame_q,)) +# #p1.start() +# #p2.start() + + +# # video = 0 + +# # if video: +# # fourcc = cv2.VideoWriter_fourcc('m', 'p', '4', 'v') +# # video = cv2.VideoCapture('test.mp4') +# # success, image = video.read() + +# # writer = cv2.VideoWriter('output.mp4', fourcc, video.get(cv2.CAP_PROP_FPS), (image.shape[0], image.shape[1])) +# # while success: +# # success,image = video.read() +# # if image is not None: +# # frame = Frame(image) +# # analysis, pose, face = sleepy_baby.process_baby_image_models(frame.w_data) +# # frame.add_analysis_frame() +# # frame.add_body_details(pose) +# # frame.add_face_details(face) +# # writer.write(frame.getAugmentedFrame()) +# # cv2.destroyAllWindows() +# # writer.release() +# # video.release() +# # else: +# # img = cv2.imread("test.jpg") +# # frame = Frame(cv2.cvtColor(img, cv2.COLOR_BGR2RGB), 700, 300, 1220,700) +# # analysis, pose, face = sleepy_baby.process_baby_image_models(frame.w_data) +# # frame.add_analysis_frame() +# # frame.add_body_details(pose) +# # frame.add_face_details(face) +# # cv2.imwrite("debug.jpg", frame.getAugmentedFrame()) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 1678df8..74ac39d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,4 @@ python-dotenv==0.21.1 scikit_learn==1.2.1 statsmodels==0.13.5 tbats==1.1.2 +fire==0.5.0 \ No newline at end of file From 5438aaaabae4cd5f2bdc2d778e8d0f652af0c0ef Mon Sep 17 00:00:00 2001 From: "CSI\\salvatore musumeci" Date: Tue, 4 Apr 2023 11:28:21 +0200 Subject: [PATCH 22/31] refactor: Refactor source parameter in app.live() method - Set default value to config['VIDEO_PATH'] for better user experience --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 927ea2f..1f849f8 100644 --- a/main.py +++ b/main.py @@ -85,7 +85,7 @@ def __init__(self, logging.info('Initialization complete.') - def live(self, source:str, return_image: bool = True): + def live(self, source:str = config['VIDEO_PATH'], return_image: bool = True): """ Run App based on streaming video From 4964d5c8ccab3e9ffebfffe35727e8f62b5d4fb5 Mon Sep 17 00:00:00 2001 From: "CSI\\salvatore musumeci" Date: Wed, 5 Apr 2023 15:17:11 +0200 Subject: [PATCH 23/31] refactor: Improve constructor functionality and readability in main.py. - Improve initialization of `working_area` in `app` class constructor - Add support for passing `working_area` as a tuple in `main.py` - Simplify `app` class constructor by using tuple unpacking for `working_area` parameter --- main.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/main.py b/main.py index 1f849f8..85ba987 100644 --- a/main.py +++ b/main.py @@ -51,10 +51,7 @@ def __init__(self, log_path: str = config['SLEEP_DATA_PATH'], body_min_detection_confidence: float = 0.1, body_min_tracking_confidence: float = 0.1, - working_area_x: int = None, - working_area_y: int = None, - working_area_width: int = None, - working_area_height: int = None, + working_area: tuple = None, show_frame: bool=True, show_wrist_position: bool=True, show_wrist_text: bool=True, @@ -76,6 +73,8 @@ def __init__(self, self.sleepy_baby = SleepyBaby(body_min_detection_confidence=body_min_detection_confidence, body_min_tracking_confidence=body_min_tracking_confidence) self.sleepy_baby.set_working_area(working_area_x, working_area_y, working_area_width, working_area_height) + if working_area: + self.sleepy_baby.set_working_area(working_area[0], working_area[1], working_area[2], working_area[3]) self.sleepy_baby.set_output(show_frame=show_frame, show_wrist_position=show_wrist_position, show_wrist_text=show_wrist_text, From ab43dc4774f23a8dbb2c4e2f614ef2562207f1aa Mon Sep 17 00:00:00 2001 From: "CSI\\salvatore musumeci" Date: Wed, 5 Apr 2023 15:21:48 +0200 Subject: [PATCH 24/31] feat: Add face detection and tracking confidence to SleepyBaby initialization --- main.py | 11 +++++++---- sleepy_baby/__init__.py | 8 ++++---- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/main.py b/main.py index 85ba987..d54c268 100644 --- a/main.py +++ b/main.py @@ -49,8 +49,10 @@ def __init__(self, verbose: bool = config['DEBUG'], log_on_screen: bool = False, log_path: str = config['SLEEP_DATA_PATH'], - body_min_detection_confidence: float = 0.1, - body_min_tracking_confidence: float = 0.1, + body_min_detection_confidence: float = 0.8, + body_min_tracking_confidence: float = 0.8, + face_min_detection_confidence: float = 0.7, + face_min_tracking_confidence: float = 0.7, working_area: tuple = None, show_frame: bool=True, show_wrist_position: bool=True, @@ -71,8 +73,9 @@ def __init__(self, #Load SleepyBaby logging.info('Initializing...') self.sleepy_baby = SleepyBaby(body_min_detection_confidence=body_min_detection_confidence, - body_min_tracking_confidence=body_min_tracking_confidence) - self.sleepy_baby.set_working_area(working_area_x, working_area_y, working_area_width, working_area_height) + body_min_tracking_confidence=body_min_tracking_confidence, + face_min_detection_confidence=face_min_detection_confidence, + face_min_tracking_confidence=face_min_tracking_confidence) if working_area: self.sleepy_baby.set_working_area(working_area[0], working_area[1], working_area[2], working_area[3]) self.sleepy_baby.set_output(show_frame=show_frame, diff --git a/sleepy_baby/__init__.py b/sleepy_baby/__init__.py index ffae0fc..ceedb1b 100644 --- a/sleepy_baby/__init__.py +++ b/sleepy_baby/__init__.py @@ -37,10 +37,10 @@ def __init__(self, self.process_t = None #Process Thread self.face = mp.solutions.face_mesh.FaceMesh(max_num_faces=1, refine_landmarks=refine_landmarks, - min_detection_confidence=body_min_detection_confidence, - min_tracking_confidence=body_min_tracking_confidence) - self.pose = mp.solutions.pose.Pose(min_detection_confidence=face_min_detection_confidence, - min_tracking_confidence=face_min_tracking_confidence) + min_detection_confidence=face_min_detection_confidence, + min_tracking_confidence=face_min_tracking_confidence) + self.pose = mp.solutions.pose.Pose(min_detection_confidence=body_min_detection_confidence, + min_tracking_confidence=body_min_tracking_confidence) self.set_working_area() #Set entire area as working area self.set_output() #Set default values self.logic = DecisionLogic() From 0673da97ddeda60de211fa2ef82fc935b2f97287 Mon Sep 17 00:00:00 2001 From: "CSI\\salvatore musumeci" Date: Wed, 5 Apr 2023 15:25:41 +0200 Subject: [PATCH 25/31] feat: implement new `photo` method for image processing --- main.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/main.py b/main.py index d54c268..70508ef 100644 --- a/main.py +++ b/main.py @@ -112,6 +112,16 @@ def recorded(self, source:str, return_image: bool = True): define if post processed image should be displayed, default: True """ self._process_streaming(source, apply_delay_between_frames=True, return_image=return_image) + + def photo(self, source:str, output_size: tuple=None): + self.sleepy_baby.show_progress_bar = False + img = cv2.imread(source) + output = self.sleepy_baby.processFrame(img) + if output_size: + output = cv2.resize(output, output_size) + cv2.imshow("Output", output) + cv2.waitKey() + def _process_streaming(self, source, apply_delay_between_frames=False, return_image=True): From fab7cd63b04fae4f2cdc40c243d61a5511667cb1 Mon Sep 17 00:00:00 2001 From: nos86 Date: Tue, 11 Apr 2023 23:27:24 +0200 Subject: [PATCH 26/31] feat: Add working area configuration Add working area configuration to the application, obtained from .env file, and set it as a parameter when initializing the app. --- .env_sample | 6 +++++- main.py | 3 ++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.env_sample b/.env_sample index 3661206..9924861 100644 --- a/.env_sample +++ b/.env_sample @@ -2,4 +2,8 @@ SLEEP_DATA_PATH=/path/to/sleep/data DEBUG=False OWL=False VIDEO_PATH=/path/to/video -HATCH_IP=192.168.HATCH.IP \ No newline at end of file +HATCH_IP=192.168.HATCH.IP +AREA_X= +AREA_Y= +AREA_WIDTH=1920 +AREA_HEIGHT=1080 \ No newline at end of file diff --git a/main.py b/main.py index 70508ef..ba8fde1 100644 --- a/main.py +++ b/main.py @@ -21,6 +21,7 @@ #Load configuration from .env file config = dotenv_values() config['DEBUG'] = (config['DEBUG'].lower()=="true") #Transform in bool +config['WORKING_AREA'] = None if config['AREA_X'] == "" else (int(config['AREA_X']), int(config['AREA_Y']), int(config['AREA_WIDTH']), int(config['AREA_HEIGHT'])) # # Queue shared between the frame publishing thread and the consuming thread # # Had to split frame receive and processing into different threads due to underlying FFMPEG issue. Read more here: @@ -53,7 +54,7 @@ def __init__(self, body_min_tracking_confidence: float = 0.8, face_min_detection_confidence: float = 0.7, face_min_tracking_confidence: float = 0.7, - working_area: tuple = None, + working_area: tuple = config['WORKING_AREA'], show_frame: bool=True, show_wrist_position: bool=True, show_wrist_text: bool=True, From 59a1a61c8729689271d26837f28660b85cceacfe Mon Sep 17 00:00:00 2001 From: nos86 Date: Tue, 11 Apr 2023 23:30:45 +0200 Subject: [PATCH 27/31] =?UTF-8?q?feat(frame):=20=E2=9C=A8=20add=20gamma=20?= =?UTF-8?q?correction=20to=20the=20augmented=20frame=20generation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In order to improve the generation of augmented frames, we added a gamma correction function to the `getAugmentedFrame` method of the `Frame` class in the `frame.py` file. This method now accepts a `gamma` parameter as input and applies the `gamma_correction` function to the augmented frame. The `gamma_correction` function was also updated to return the lookup table rather than a cv2.LUT object. It was also decorated with a caching mechanism from the `functools` library to improve performance. --- sleepy_baby/frame.py | 5 +++-- sleepy_baby/helpers.py | 8 ++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/sleepy_baby/frame.py b/sleepy_baby/frame.py index f906ad6..b16ca81 100644 --- a/sleepy_baby/frame.py +++ b/sleepy_baby/frame.py @@ -2,6 +2,7 @@ import numpy as np import mediapipe as mp import cv2 +from .helpers import gamma_correction mp_utils = mp.solutions.drawing_utils @@ -52,7 +53,7 @@ def get_working_image(self) -> np.ndarray: """ return self.frame[self.y_offset:self.y_offset+self.height, self.x_offset:self.x_offset+self.width].copy() - def getAugmentedFrame(self) -> np.ndarray: + def getAugmentedFrame(self, gamma: float =.8) -> np.ndarray: """ It generates the a new frame integrating the modified working area. @@ -61,7 +62,7 @@ def getAugmentedFrame(self) -> np.ndarray: np.ndarray Image integrated """ - frame = self.frame.copy() + frame = cv2.LUT(self.frame.copy(), gamma_correction(gamma)) frame[self.y_offset:self.y_offset+self.height, self.x_offset:self.x_offset+self.width] = self.w_data return frame diff --git a/sleepy_baby/helpers.py b/sleepy_baby/helpers.py index 310cef0..22e2b5c 100644 --- a/sleepy_baby/helpers.py +++ b/sleepy_baby/helpers.py @@ -3,6 +3,7 @@ import os from pyhatchbabyrest import PyHatchBabyRest from dotenv import load_dotenv +from functools import lru_cache load_dotenv() @@ -131,9 +132,8 @@ def maintain_aspect_ratio_resize(image, width=None, height=None, inter=cv2.INTER # Return the resized image return cv2.resize(image, dim, interpolation=inter) - -def gamma_correction(self, og, gamma): +@lru_cache(maxsize=10) +def gamma_correction(gamma): invGamma = 1 / gamma table = [((i / 255) ** invGamma) * 255 for i in range(256)] - table = np.array(table, np.uint8) - return cv2.LUT(og, table) \ No newline at end of file + return np.array(table, np.uint8) \ No newline at end of file From 711bb8c9397861d4b68a22d8c81dd0fc2820654b Mon Sep 17 00:00:00 2001 From: nos86 Date: Tue, 11 Apr 2023 23:35:59 +0200 Subject: [PATCH 28/31] =?UTF-8?q?refactor:=20=F0=9F=9A=80=20code=20to=20si?= =?UTF-8?q?mplify=20get=5Fheight=20calculation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of calculating the distances between landmarks repeatedly, a get_height method was created to calculate the distance between the desired landmarks requested by a list of tuples, iterating through each tuple and calculating the euclidean distances between them using numpy. This made the code more readable and less repetitive. --- sleepy_baby/helpers.py | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/sleepy_baby/helpers.py b/sleepy_baby/helpers.py index 22e2b5c..04c7449 100644 --- a/sleepy_baby/helpers.py +++ b/sleepy_baby/helpers.py @@ -76,28 +76,23 @@ def set_hatch(is_awake): def get_top_lip_height(landmarks): #Indexes of landmarks are reported at https://raw.githubusercontent.com/google/mediapipe/a908d668c730da128dfa8d9f6bd25d519d006692/mediapipe/modules/face_geometry/data/canonical_face_model_uv_visualization.png - top_lip_left = get_distance_between_landmarks(landmarks, 39, 81) - top_lip_center = get_distance_between_landmarks(landmarks, 0, 13) - top_lip_right = get_distance_between_landmarks(landmarks, 269, 311) - return (top_lip_left + top_lip_center + top_lip_right) / 3 - + return get_height(landmarks, [(39,81), (0,13), (269,311)]) def get_bottom_lip_height(landmarks): #Indexes of landmarks are reported at https://raw.githubusercontent.com/google/mediapipe/a908d668c730da128dfa8d9f6bd25d519d006692/mediapipe/modules/face_geometry/data/canonical_face_model_uv_visualization.png - bottom_lip_left = get_distance_between_landmarks(landmarks, 181, 178) - bottom_lip_center = get_distance_between_landmarks(landmarks, 17, 14) - bottom_lip_right = get_distance_between_landmarks(landmarks, 405, 402) - return (bottom_lip_left + bottom_lip_center + bottom_lip_right) / 3 - - + return get_height(landmarks, [(181,178), (17,14), (405,402)]) def get_mouth_height(landmarks): #Indexes of landmarks are reported at https://raw.githubusercontent.com/google/mediapipe/a908d668c730da128dfa8d9f6bd25d519d006692/mediapipe/modules/face_geometry/data/canonical_face_model_uv_visualization.png - open_mouth_left = get_distance_between_landmarks(landmarks, 178, 81) - open_mouth_center = get_distance_between_landmarks(landmarks, 14, 13) - open_mouth_right = get_distance_between_landmarks(landmarks, 402, 311) - return (open_mouth_left + open_mouth_center + open_mouth_right) / 3 + return get_height(landmarks, [(178,81), (14,13), (402,311)]) +def get_height(landmarks, tuples): + heights = [] + for tuple in tuples: + p0 = get_point_as_array(landmarks[tuple[0]]) + p1 = get_point_as_array(landmarks[tuple[1]]) + heights.append(np.linalg.norm(p0 - p1)) + return np.mean(heights) def check_mouth_open(landmarks, ratio = 0.8): top_lip_height = get_top_lip_height(landmarks) @@ -107,7 +102,6 @@ def check_mouth_open(landmarks, ratio = 0.8): # if mouth is open more than lip height * ratio, return true. return mouth_height > min(top_lip_height, bottom_lip_height) * ratio - # Resizes a image and maintains aspect ratio def maintain_aspect_ratio_resize(image, width=None, height=None, inter=cv2.INTER_AREA): # Grab the image size and initialize dimensions From a040c9e5472a2b830d6e10f4b6e1b3d85ddf4ef8 Mon Sep 17 00:00:00 2001 From: nos86 Date: Tue, 11 Apr 2023 23:41:43 +0200 Subject: [PATCH 29/31] style: Remove unused dependencies --- sleepy_baby/helpers.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/sleepy_baby/helpers.py b/sleepy_baby/helpers.py index 04c7449..e16eeb3 100644 --- a/sleepy_baby/helpers.py +++ b/sleepy_baby/helpers.py @@ -1,13 +1,8 @@ import numpy as np import cv2 -import os from pyhatchbabyrest import PyHatchBabyRest -from dotenv import load_dotenv from functools import lru_cache -load_dotenv() - - def get_point_as_array(point): """ get_point_as_array transforms landmarks coordinate in numpy array. From cc1949d3609effb2dcfc2009bc6b374fafd1afc6 Mon Sep 17 00:00:00 2001 From: nos86 Date: Wed, 12 Apr 2023 00:20:21 +0200 Subject: [PATCH 30/31] =?UTF-8?q?refactor:=20=F0=9F=94=AE=20Refactored=20a?= =?UTF-8?q?pp.photo()=20and=20app.=5Fprocess=5Fstreaming()=20to=20maintain?= =?UTF-8?q?=20image=20aspect=20ratio=20during=20resizing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The code changes include making sure that the aspect ratio of images is maintained when resizing in both the app.photo() and app._process_streaming() functions. A new parameter `max_width` and `max_height` were also included in the app.photo() function to ensure that the image does not exceed those dimensions when resizing. The maintain_aspect_ratio_resize() function in sleepy_baby/helpers.py was also refactored to accept width and height parameters and select the smallest format. --- main.py | 11 ++++++++--- sleepy_baby/helpers.py | 23 ++++++----------------- 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/main.py b/main.py index ba8fde1..618e763 100644 --- a/main.py +++ b/main.py @@ -12,7 +12,7 @@ from http.server import HTTPServer, SimpleHTTPRequestHandler from sleepy_baby import SleepyBaby -from sleepy_baby import Frame +from sleepy_baby import helpers # Uncomment if want phone notifications during daytime wakings. # Configuration of telegram API key in this dir also needed. @@ -114,9 +114,11 @@ def recorded(self, source:str, return_image: bool = True): """ self._process_streaming(source, apply_delay_between_frames=True, return_image=return_image) - def photo(self, source:str, output_size: tuple=None): + def photo(self, source:str, output_size: tuple=None, max_width:int = 1920, max_height:int = 1080): self.sleepy_baby.show_progress_bar = False img = cv2.imread(source) + if (img.shape[1]>max_width) or (img.shape[0]>max_height): + img = helpers.maintain_aspect_ratio_resize(img, max_width, max_height) output = self.sleepy_baby.processFrame(img) if output_size: output = cv2.resize(output, output_size) @@ -125,13 +127,14 @@ def photo(self, source:str, output_size: tuple=None): - def _process_streaming(self, source, apply_delay_between_frames=False, return_image=True): + def _process_streaming(self, source, apply_delay_between_frames=False, return_image=True, max_width=1920, max_height=1080): try: vcap = cv2.VideoCapture(source) if vcap.isOpened(): success = True logging.info("Start receiving frames.") fps = vcap.get(cv2.CAP_PROP_FPS) + rescale = (vcap.get(cv2.CAP_PROP_FRAME_WIDTH)>max_width) or (vcap.get(cv2.CAP_PROP_FRAME_HEIGHT)>max_height) self.sleepy_baby.start_thread(frame_q, terminate_event) if return_image: self.show_video_thread = Thread(target=show_video, args=(self.sleepy_baby,)) @@ -140,6 +143,8 @@ def _process_streaming(self, source, apply_delay_between_frames=False, return_im success, img = vcap.read() if success is False: terminate_event.set() #Error in streaming reading + if rescale: + img = helpers.maintain_aspect_ratio_resize(img, max_width, max_height) frame_q.append(img) #cv2.cvtColor(img, cv2.COLOR_BGR2RGB)) if apply_delay_between_frames: time.sleep(1.0/fps) diff --git a/sleepy_baby/helpers.py b/sleepy_baby/helpers.py index e16eeb3..99b3cdb 100644 --- a/sleepy_baby/helpers.py +++ b/sleepy_baby/helpers.py @@ -98,27 +98,16 @@ def check_mouth_open(landmarks, ratio = 0.8): return mouth_height > min(top_lip_height, bottom_lip_height) * ratio # Resizes a image and maintains aspect ratio -def maintain_aspect_ratio_resize(image, width=None, height=None, inter=cv2.INTER_AREA): - # Grab the image size and initialize dimensions - dim = None - (h, w) = image.shape[:2] - +def maintain_aspect_ratio_resize(image, width:int=None, height:int=None, inter=cv2.INTER_AREA): # Return original image if no need to resize if width is None and height is None: return image - - # We are resizing height if width is none - if width is None: - # Calculate the ratio of the height and construct the dimensions - r = height / float(h) - dim = (int(w * r), height) - # We are resizing width if height is none + (h, w) = image.shape[:2] # Grab the image size and initialize dimensions + # Select smallest format: + if ((width or w) / w) < ((height or h) / h) and (width is not None): + dim = (width, int(h * width / w)) else: - # Calculate the ratio of the 0idth and construct the dimensions - r = width / float(w) - dim = (width, int(h * r)) - - # Return the resized image + dim = (int(w * height / height), height) return cv2.resize(image, dim, interpolation=inter) @lru_cache(maxsize=10) From b666027367ab2b8bc05af0ec2f28734b3adf4bc0 Mon Sep 17 00:00:00 2001 From: nos86 Date: Wed, 12 Apr 2023 00:37:07 +0200 Subject: [PATCH 31/31] =?UTF-8?q?feat:=20=F0=9F=9A=80=20Add=20debug=20vari?= =?UTF-8?q?able=20to=20show=20baby=20status?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit adds a new `debug` variable to the `SleepyBaby` class to allow for easier debugging. When true, status messages will be added to frames to indicate whether the baby is awake or asleep. The text and color of the message will depend on the state of the baby. --- main.py | 3 ++- sleepy_baby/__init__.py | 6 +++++- sleepy_baby/frame.py | 7 ++++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index 618e763..b562c4b 100644 --- a/main.py +++ b/main.py @@ -76,7 +76,8 @@ def __init__(self, self.sleepy_baby = SleepyBaby(body_min_detection_confidence=body_min_detection_confidence, body_min_tracking_confidence=body_min_tracking_confidence, face_min_detection_confidence=face_min_detection_confidence, - face_min_tracking_confidence=face_min_tracking_confidence) + face_min_tracking_confidence=face_min_tracking_confidence, + debug=verbose) if working_area: self.sleepy_baby.set_working_area(working_area[0], working_area[1], working_area[2], working_area[3]) self.sleepy_baby.set_output(show_frame=show_frame, diff --git a/sleepy_baby/__init__.py b/sleepy_baby/__init__.py index ceedb1b..81bb8d9 100644 --- a/sleepy_baby/__init__.py +++ b/sleepy_baby/__init__.py @@ -30,9 +30,11 @@ def __init__(self, body_min_tracking_confidence=0.8, face_min_detection_confidence=0.7, face_min_tracking_confidence=0.7, - refine_landmarks = True): + refine_landmarks = True, + debug = False): self.logger = logging.getLogger(self.__class__.__name__) self.logger.debug("SleepyBaby is starting") + self.debug = debug self.processed_frame = None #It is used to produce post-processed video self.process_t = None #Process Thread self.face = mp.solutions.face_mesh.FaceMesh(max_num_faces=1, @@ -99,6 +101,8 @@ def processFrame(self, image, return_image=True): frame.add_face_details(face) if self.show_progress_bar: frame.add_progress_bar(self.logic.avg_awake) + if self.debug: + frame.add_status(self.logic.avg_awake < 0.6) return frame.getAugmentedFrame() def process_baby_image_models(self, frame): diff --git a/sleepy_baby/frame.py b/sleepy_baby/frame.py index b16ca81..c6207d2 100644 --- a/sleepy_baby/frame.py +++ b/sleepy_baby/frame.py @@ -178,4 +178,9 @@ def add_progress_bar(self, percent, bar_width=500, bar_height = 20, bar_y_offset self.w_data = cv2.rectangle(self.w_data, start_point, end_point, backcolor, thickness = -1) self.w_data = cv2.rectangle(self.w_data, start_point, mid_point, forecolor, thickness = -1) - self.w_data = cv2.putText(self.w_data, str(int(adj_percent * 100)) + "% Awake", (start_point[0], text_y_position), 2, 1, textcolor, 2, 2) \ No newline at end of file + self.w_data = cv2.putText(self.w_data, str(int(adj_percent * 100)) + "% Awake", (start_point[0], text_y_position), 2, 1, textcolor, 2, 2) + + def add_status(self, asleep): + text = 'Sleepy Baby' if asleep else 'Wakey Baby' + text_color = (255,191,0) if asleep else (0,140,255) + cv2.putText(self.w_data, text, (int(self.w_data.shape[1]/4), int(self.w_data.shape[0]/2)), 2, 3, text_color, 2, 2) \ No newline at end of file