diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fd20fdd --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ + +*.pyc diff --git a/Hand.py b/Hand.py new file mode 100644 index 0000000..704f90e --- /dev/null +++ b/Hand.py @@ -0,0 +1,921 @@ +import math +from datetime import datetime + +import numpy as np +import random + +import cv2 + +from roi import Roi, SIDE + +MAX_UNDETECTED_FRAMES = 30 * 10 # 30 FPS * 10 seconds +MAX_UNDETECTED_SECONDS = 10 +DETECTION_TRUTH_FACTOR = 2. # points of life to recover if detected in one iteration +TRACKING_TRUTH_FACTOR = .1 # points of life to recover if tracked in one iteration +UNDETECTION_TRUTH_FACTOR = 3. # points of life to recover if detected in one iteration +UNTRACKING_TRUTH_FACTOR = 2 # points of life to recover if tracked in one iteration +MAX_TRUTH_VALUE = 100. +FONT = cv2.FONT_HERSHEY_SIMPLEX + + + +class MASKMODES: + COLOR=0 + MOG2=1 + DIFF=2 + MIXED=3 + MOVEMENT_BUFFER=4 + DEPTH=5 + +def get_random_color(n=1): + """ + Generate a randonm RGB color with values between 0-255 for R & G & B + + :param n: increment over the ranadome + :return: return an array with 3 int representing the R & G & B values + """ + ret = [] + r = int(random.random() * 256) + g = int(random.random() * 256) + b = int(random.random() * 256) + step = 256 / n + for i in range(n): + r += step + g += step + b += step + r = int(r) % 256 + g = int(g) % 256 + b = int(b) % 256 + ret = [r, g, b] + return ret + +def clean_mask_noise(mask, blur=5): + """ + Given an image mask it perfoms a clean up of it with a series of erodes and dilate + + :param mask: mask to be cleaned + :param blur: blur to be applied to the mask after clean up + :return: mask cleaned + """ + # Kernel matrices for morphological transformation + kernel_square = np.ones((11, 11), np.uint8) + kernel_ellipse = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5)) + + # Perform morphological transformations to filter out the background noise + # Dilation increase skin color area + # Erosion increase skin color area + dilation = cv2.dilate(mask, kernel_ellipse, iterations=1) + erosion = cv2.erode(dilation, kernel_square, iterations=1) + dilation2 = cv2.dilate(erosion, kernel_ellipse, iterations=1) + # filtered = cv2.medianBlur(dilation2, 5) + # kernel_ellipse = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (8, 8)) + # dilation2 = cv2.dilate(filtered, kernel_ellipse, iterations=1) + # kernel_ellipse = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5)) + # dilation3 = cv2.dilate(filtered, kernel_ellipse, iterations=1) + median = cv2.medianBlur(dilation2, blur) + return median + +def get_color_mask(image, color_from=[2, 50, 50], color_to=[15, 255, 255]): + """ + Create a mask for an image with the colors between color_from to color_to + :param image: Image to get the mask from it + :param color_from: HSV values from where the colors will be got + :param color_to: HSV values to where the colors will be got + :return: mask for the matching colors on the image + """ + # Blur the image + blur_radius = 5 + blurred = cv2.GaussianBlur(image, (blur_radius, blur_radius), 0) + # Convert to HSV color space + hsv = cv2.cvtColor(blurred, cv2.COLOR_BGR2HSV) + mask = cv2.inRange(hsv, np.array(color_from), np.array(color_to)) + return mask + + +#TODO: save a roi for each hand that we want to track. This roi would be the initial where we want to look for the hand an the one we could use if the hand is lost. + +class Hand(object): + """ + This contains all the usefull information for a detected hand. + + """ + def __init__(self): + """ + Hand class attributes values. + + """ + self._id = None + self._fingertips = [] + self._intertips = [] + self._center_of_mass = None + self._finger_distances = [] + self._average_defect_distance = [] + self._contour = None + self._consecutive_tracking_fails = 0 + self._consecutive_detection_fails = 0 + self._frame_count = 0 + self._color = get_random_color() + self._confidence = 0 + self._tracked = False + self._detected = False + self._detection_status = 0 + self._position_history = [] + + + # The region of the image where the hand is expected to be located when initialized or lost + self._initial_roi = Roi() + # The region where the hand have been detected the last time + self._detection_roi = Roi() + # The region where the hand was tracked the last time + self._tracking_roi = Roi() + # Region extended from tracking_roi to a maximum of initial_roi to look for the hand + self._extended_roi = Roi() + + self._mask_mode = MASKMODES.DEPTH + self._debug = True + self._depth_threshold = -1 + self._last_frame = None + self._ever_detected = False + + +##################################################################### +########## Properties and setters ########## +##################################################################### + + @property + def initial_roi(self): + return self._initial_roi + + @initial_roi.setter + def initial_roi(self, value): + assert all(isinstance(n, (int, float)) for n in value) or isinstance(value, + Roi), "initial_roi must be of the type Roi" + if isinstance(value, Roi): + self._initial_roi = value + else: + self._initial_roi = Roi(value) + + self.extended_roi = self._initial_roi + + + @property + def tracking_roi(self): + return self._tracking_roi + + @tracking_roi.setter + def tracking_roi(self, value): + assert all(isinstance(n, (int, float)) for n in value) or isinstance(value, Roi), "tracking_roi must be of the type Roi" + if isinstance(value, Roi): + self._tracking_roi = value + else: + self._tracking_roi = Roi(value) + # Tracking_roi must be limited to the initial_roi + self._tracking_roi.limit_to_roi(self.initial_roi) + + @property + def detection_roi(self): + return self._detection_roi + + @detection_roi.setter + def detection_roi(self, value): + assert all(isinstance(n, (int, float)) for n in value) or isinstance(value, + Roi), "detection_roi must be of the type Roi" + if isinstance(value, Roi): + self._detection_roi = value + else: + self._detection_roi = Roi(value) + # Detection_roi must be limited to the initial_roi + self._detection_roi.limit_to_roi(self.initial_roi) + + @property + def extended_roi(self): + return self._extended_roi + + @extended_roi.setter + def extended_roi(self, value): + assert all(isinstance(n, (int, float)) for n in value) or isinstance(value, + Roi), "extended_roi must be of the type Roi" + if isinstance(value, Roi): + self._extended_roi = value + else: + self._extended_roi = Roi(value) + # Extended_roi must be limited to the initial_roi + self._extended_roi.limit_to_roi(self.initial_roi) + + + + + @property + def depth_threshold(self): + return self._depth_threshold + + @depth_threshold.setter + def depth_threshold(self, value): + self._depth_threshold = value + + @property + def confidence(self): + return self._confidence + + @confidence.setter + def confidence(self, value): + self._confidence = value + + @property + def valid(self): + return (self.detected or self.tracked or self._confidence > 0) + + + @property + def detected(self): + return self._detected + + @detected.setter + def detected(self, value): + self._detected = value + + @property + def tracked(self): + return self._tracked + + @tracked.setter + def tracked(self, value): + self._tracked = value + +##################################################################### +########## Probably deprecated methods # TODO: check ########## +##################################################################### + + #TODO: Check if we need a deep copy of the data. + def update_attributes_from_detected(self, other_hand): + """ + update current hand with the values of other hand + TODO: need to be checked. + :param other_hand: the hand where the values are going to be copied + :return: None + """ + self._fingertips = other_hand.fingertips + self._intertips = other_hand.intertips + self._center_of_mass = other_hand.center_of_mass + self._finger_distances = other_hand.finger_distances + self._average_defect_distance = other_hand.average_defect_distance + self._contour = other_hand.contour + self.detection_roi = other_hand.detection_roi + self._detected = True + + def update_truth_value_by_time(self): + """ + Update the truth value of the hand based on the time elapsed between two calls + and if the hand is detected and tracked + + :return: None + """ + if self.last_time_update is not None: + elapsed_time = datetime.now() - self.last_time_update + elapsed_miliseconds = int(elapsed_time.total_seconds() * 1000) + + # Calculate how much we would substract if the hand is undetected + truth_subtraction = elapsed_miliseconds * MAX_TRUTH_VALUE / MAX_UNDETECTED_SECONDS * 1000 + + # Calculate how much we should increment if the hand has been detected + detection_adition = DETECTION_TRUTH_FACTOR if self._detected is True else 0 + + # Calculate how much we should increment if the is tracked + tracking_adition = TRACKING_TRUTH_FACTOR if self._tracked is True else 0 + + # update of the truth value + self._confidence = self._confidence - truth_subtraction + detection_adition + tracking_adition + self.last_time_update = datetime.now() + + + # Deprecated: using update_truth_value_by_frame2 + def update_truth_value_by_frame(self): + """ + Update the truth value of the hand based on the frames elapsed between two calls + and if the hand is detected and tracked + + :return: None + """ + one_frame_truth_subtraction = MAX_TRUTH_VALUE / MAX_UNDETECTED_FRAMES + detection_adition = 0 + if self._detected: + detection_adition = DETECTION_TRUTH_FACTOR * one_frame_truth_subtraction + else: + self._consecutive_detection_fails += 1 + detection_adition = -1 * UNDETECTION_TRUTH_FACTOR * one_frame_truth_subtraction + tracking_adition = 0 + if self._tracked: + tracking_adition = TRACKING_TRUTH_FACTOR * one_frame_truth_subtraction + else: + self._consecutive_tracking_fails += 1 + tracking_adition = -1 * UNTRACKING_TRUTH_FACTOR * one_frame_truth_subtraction + new_truth_value = self._confidence - one_frame_truth_subtraction + detection_adition + tracking_adition + if new_truth_value <= MAX_TRUTH_VALUE: + self._confidence = new_truth_value + else: + self._confidence = MAX_TRUTH_VALUE + self._frame_count += 1 + + def update_truth_value_by_frame2(self): + substraction = 0 + one_frame_truth_subtraction = MAX_TRUTH_VALUE / MAX_UNDETECTED_FRAMES + if not self._detected: + self._consecutive_detection_fails += 1 + if not self._tracked: + self._consecutive_tracking_fails += 1 + if not self._detected and not self._tracked: + substraction = -1 * UNDETECTION_TRUTH_FACTOR * UNTRACKING_TRUTH_FACTOR * one_frame_truth_subtraction + else: + if self._tracked: + substraction = substraction + UNTRACKING_TRUTH_FACTOR * one_frame_truth_subtraction + if self._detected: + substraction = substraction + UNDETECTION_TRUTH_FACTOR * one_frame_truth_subtraction + + new_truth_value = self._confidence + substraction + if new_truth_value <= 100: + self._confidence = new_truth_value + else: + self._confidence = 100 + self._frame_count += 1 + + + def copy_main_attributes(self): + """ + Return a new hand with the main attributes of this copied into it + + :return: New Hand with the main attributes copied into it + """ + updated_hand = Hand() + updated_hand._id = self._id + updated_hand._fingertips = [] + updated_hand._intertips = [] + updated_hand._center_of_mass = None + updated_hand._finger_distances = [] + updated_hand._average_defect_distance = [] + updated_hand._contour = None + updated_hand.detection_roi = self.detection_roi + updated_hand._consecutive_tracking_fails = self._consecutive_tracking_fails + updated_hand._position_history = self._position_history + updated_hand._color = self._color + return updated_hand + +##################################################################### +########## Currently used methods ########## +##################################################################### + + def create_contours_and_mask(self, frame, roi=None): + # Create a binary image where white will be skin colors and rest is black + hands_mask = self.create_hand_mask(frame) + if hands_mask is None: + return ([], []) + + if roi is not None: + x, y, w, h = roi + else: + x, y, w, h = self.initial_roi + + roied_hands_mask = roi.apply_to_frame_as_mask(hands_mask) + if self._debug: + cv2.imshow("DEBUG: HandDetection_lib: create_contours_and_mask (Frame Mask)", hands_mask) + # to_show = cv2.resize(hands_mask, None, fx=.3, fy=.3, interpolation=cv2.INTER_CUBIC) + to_show = roied_hands_mask.copy() + # cv2.putText(to_show, (str(w)), (x + w, y), FONT, 0.3, [255, 255, 255], 1) + # cv2.putText(to_show, (str(h)), (x + w, y + h), FONT, 0.3, [100, 255, 255], 1) + # cv2.putText(to_show, (str(w * h)), (x + w / 2, y + h / 2), FONT, 0.3, [100, 100, 255], 1) + # cv2.putText(to_show, (str(x)+", "+str(y)), (x-10, y-10), FONT, 0.3, [255, 255, 255], 1) + to_show = roi.draw_on_frame(to_show) + cv2.imshow("DEBUG: HandDetection_lib: create_contours_and_mask (current_roi_mask)", roi.extract_from_frame(frame)) + cv2.imshow("DEBUG: HandDetection_lib: create_contours_and_mask (ROIed Mask)", to_show) + + ret, thresh = cv2.threshold(roied_hands_mask, 127, 255, 0) + + # Find contours of the filtered frame + _, contours, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + return (contours, hands_mask) + + + def create_hand_mask(self, image, mode=None): + if mode is None: + mode = self._mask_mode + # print "create_hands_mask %s" % mode + mask = None + if mode == MASKMODES.COLOR: + mask = get_color_mask(image) + elif mode == MASKMODES.MOG2: + mask = self.get_MOG2_mask(image) + elif mode == MASKMODES.DIFF: + mask = self.get_simple_diff_mask2(image) + elif mode == MASKMODES.MIXED: + diff_mask = self.get_simple_diff_mask(image) + + color_mask = get_color_mask(image) + color_mask = clean_mask_noise(color_mask) + + if diff_mask is not None and color_mask is not None: + mask = cv2.bitwise_and(diff_mask, color_mask) + if self._debug: + cv2.imshow("DEBUG: HandDetection_lib: diff_mask", diff_mask) + cv2.imshow("DEBUG: HandDetection_lib: color_mask", color_mask) + elif mode == MASKMODES.MOVEMENT_BUFFER: + # Absolutly unusefull + mask = self.get_movement_buffer_mask(image) + elif mode == MASKMODES.DEPTH: + if self._debug: + print("Mode depth") + assert self._depth_threshold != -1, "Depth threshold must be set with set_depth_mask method. Use this method only with RGBD cameras" + assert len(image.shape) == 2 or image.shape[2] == 1, "Depth image should have only one channel and it have %d" % image.shape[2] + #TODO: ENV_DEPENDENCE: the second value depends on the distance from the camera to the maximum depth where it can be found in a scale of 0-255 + + mask = image + mask[mask>self._depth_threshold]= 0 + mask = self.depth_mask_to_image(mask) + + # Kernel matrices for morphological transformation + kernel_square = np.ones((5, 5), np.uint8) + kernel_ellipse = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3)) + + # cv2.imwrite("/home/robolab/robocomp/components/robocomp-robolab/components/handDetection/src/images/"+str(datetime.now().strftime("%Y%m%d%H%M%S"))+".png", mask) + # + # dilation = cv2.dilate(mask, kernel_ellipse, iterations=1) + # erosion = cv2.erode(dilation, kernel_square, iterations=1) + mask = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel_square) + mask = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, kernel_square) + + # dilation2 = cv2.dilate(erosion, kernel_ellipse, iterations=1) + # filtered = cv2.medianBlur(dilation2, 5) + # kernel_ellipse = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (8, 8)) + # dilation2 = cv2.dilate(filtered, kernel_ellipse, iterations=1) + # kernel_ellipse = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5)) + # dilation3 = cv2.dilate(filtered, kernel_ellipse, iterations=1) + mask = cv2.medianBlur(mask, 3) + # _, mask = cv2.threshold(mask, 100, 255, cv2.THRESH_BINARY) + + return mask + + def get_hand_bounding_rect_from_fingers(self, hand_contour, fingers_contour): + (x, y), radius = cv2.minEnclosingCircle(fingers_contour) + center = (int(x), int(y)) + radius = int(radius) + 10 + new_hand_contour = extract_contour_inside_circle(hand_contour, (center, radius)) + hand_bounding_rect = cv2.boundingRect(new_hand_contour) + return hand_bounding_rect, ((int(x), int(y)), radius), new_hand_contour + + def get_hand_bounding_rect_from_rect(self, hand_contour, bounding_rect): + hand_contour = extract_contour_inside_rect(hand_contour, bounding_rect) + hand_bounding_rect = cv2.boundingRect(hand_contour) + return hand_bounding_rect, hand_contour + + def get_hand_bounding_rect_from_center_of_mass(self, hand_contour, center_of_mass, average_distance): + (x, y) = center_of_mass + radius = average_distance + center = (int(x), int(y)) + radius = int(radius) + 10 + hand_contour = extract_contour_inside_circle(hand_contour, (center, radius)) + hand_bounding_rect = cv2.boundingRect(hand_contour) + return hand_bounding_rect, ((int(x), int(y)), radius), hand_contour + + + + # TODO: Move to Utils file + @staticmethod + def depth_mask_to_image(depth): + depth_min = np.min(depth) + depth_max = np.max(depth) + if depth_max!= depth_min and depth_max>0: + image = np.interp(depth, [depth_min, depth_max], [0.0, 255.0], right=255, left=0) + else: + image = np.zeros(depth.shape, dtype=np.uint8) + + image = np.array(image, dtype=np.uint8) + image = image.reshape(480, 640, 1) + return image + + def _detect_in_frame(self, frame): + self._last_frame = frame + search_roi = self.get_roi_to_use(frame) + + # Create contours and mask + self._frame_contours, self._frame_mask = self.create_contours_and_mask(frame, search_roi) + + # get the maximum contour + if len(self._frame_contours) > 0 and len(self._frame_mask) > 0: + # Get the maximum area contour + min_area = 100 + hand_contour = None + for i in range(len(self._frame_contours)): + cnt = self._frame_contours[i] + area = cv2.contourArea(cnt) + if area > min_area: + min_area = area + hand_contour = self._frame_contours[i] + + if hand_contour is not None: + # cv2.drawContours(frame, [hand_contour], -1, (0, 255, 255), 2) + detected_hand_bounding_rect = cv2.boundingRect(hand_contour) + detected_hand_x, detected_hand_y, detected_hand_w, detected_hand_h = detected_hand_bounding_rect + frame_mask_roi_image = self._frame_mask[search_roi.y:search_roi.y+ search_roi.height, + search_roi.x:search_roi.x + search_roi.width] + frame_mask_roi_image_contour, _, _ = self.calculate_max_contour(frame_mask_roi_image, to_binary=False) + # self._detected is updated inside + self.update_hand_with_contour(hand_contour) + else: + self._detected = False + self._detection_status = -1 + else: + self._detected = False + self._detection_status = -2 + + + def calculate_max_contour(self, image, to_binary=True): + if self._debug: + cv2.imshow("Hand: calculate_max_contour, image", image) + bounding_rect = None + image_roi = None + if to_binary: + gray_diff = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) + _, mask = cv2.threshold(gray_diff, 40, 255, cv2.THRESH_BINARY) + else: + mask = image + # kernel_square = np.ones((11, 11), np.uint8) + # kernel_ellipse = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5)) + # + # # Perform morphological transformations to filter out the background noise + # # Dilation increase skin color area + # # Erosion increase skin color area + # dilation = cv2.dilate(mask, kernel_ellipse, iterations=1) + # erosion = cv2.erode(dilation, kernel_square, iterations=1) + # dilation2 = cv2.dilate(erosion, kernel_ellipse, iterations=1) + # filtered = cv2.medianBlur(dilation2.astype(np.uint8), 5) + # kernel_ellipse = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (8, 8)) + # dilation2 = cv2.dilate(filtered, kernel_ellipse, iterations=1) + # kernel_ellipse = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5)) + # dilation3 = cv2.dilate(filtered, kernel_ellipse, iterations=1) + # median = cv2.medianBlur(dilation2, 5) + # if self._debug: + # cv2.imshow("Hand: calculate_max_contour, median", median) + ret, thresh = cv2.threshold(mask, 127, 255, 0) + if self._debug: + cv2.imshow("Hand: calculate_max_contour, thresh", thresh) + cnts = None + max_area = 100 + ci = 0 + # Find contours of the filtered frame + _, contours, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + if contours: + for i in range(len(contours)): + cnt = contours[i] + area = cv2.contourArea(cnt) + if area > max_area: + max_area = area + ci = i + cnts = contours[ci] + bounding_rect = cv2.boundingRect(cnts) + x, y, w, h = bounding_rect + image_roi = mask[y:y + h, x:x + w] + return cnts, bounding_rect, image_roi + + + + def update_hand_with_contour(self, hand_contour): + """ + Attributes of the hand are calculated from the hand contour. + TODO: calculate a truth value + A score of 100 is the maximum value for the hand truth. + This value is calculated like this: + A hand is expected to have 5 finger tips, 4 intertips, a center of mass + + :param hand_contour: calculated contour that is expected to describe a hand + :return: None + """ + hull2 = cv2.convexHull(hand_contour, returnPoints=False) + # Get defect points + defects = cv2.convexityDefects(hand_contour, hull2) + + if defects is not None: + estimated_fingertips_coords, \ + estimated_fingertips_indexes, \ + estimated_intertips_coords, \ + estimated_intertips_indexes = self._calculate_fingertips(hand_contour, defects) + + is_hand = self.is_hand(estimated_fingertips_coords, estimated_intertips_coords, strict=True) + if is_hand: + self._fingertips = estimated_fingertips_coords + self._intertips = estimated_intertips_coords + + if len(estimated_fingertips_coords) == 5: + fingers_contour = np.take(hand_contour, + estimated_fingertips_indexes + estimated_intertips_indexes, + axis=0, + mode="wrap") + bounding_rect, hand_circle, self._contour = self.get_hand_bounding_rect_from_fingers( + hand_contour, + fingers_contour) + # detection roi is set to the bounding rect of the fingers upscaled 20 pixels + # self.detection_roi = Roi(bounding_rect) + self.detection_roi = Roi(bounding_rect).upscaled(Roi.from_frame(self._last_frame, SIDE.CENTER, 100), 10) + if self._debug: + to_show = self._last_frame.copy() + cv2.drawContours(to_show, [hand_contour], -1, (255, 255, 255), 2) + cv2.drawContours(to_show, [fingers_contour], -1, (200, 200, 200), 2) + to_show = self.detection_roi.draw_on_frame(to_show) + # cv2.rectangle(to_show, (self.detection_roi.y, self.detection_roi.x), (self.detection_roi.y + self.detection_roi.height, self.detection_roi.x + self.detection_roi.width), [255, 255, 0]) + # (x, y, w, h) = cv2.boundingRect(hand_contour) + # cv2.rectangle(to_show, (self.detection_roi.y, self.detection_roi.x), (self.detection_roi.x + self.detection_roi.height, self.detection_roi.x + self.detection_roi.width), [255, 255, 0]) + cv2.imshow("update_hand_with_contour", to_show) + + + self._detected = True + self._detection_status = 1 + self._ever_detected = True + self._confidence = 100 + else: + self._detection_status = -1 + self._detected = False + self._confidence = 0 + return + else: + self._detection_status = -1 + self._detected = False + self._confidence = 0 + return + # Find moments of the largest contour + moments = cv2.moments(hand_contour) + center_of_mass = None + finger_distances = [] + average_defect_distance = None + # Central mass of first order moments + if moments['m00'] != 0: + cx = int(moments['m10'] / moments['m00']) # cx = M10/M00 + cy = int(moments['m01'] / moments['m00']) # cy = M01/M00 + center_of_mass = (cx, cy) + self._center_of_mass = center_of_mass + self._position_history.append(center_of_mass) + + if center_of_mass is not None and len(estimated_intertips_coords) > 0: + # Distance from each finger defect(finger webbing) to the center mass + distance_between_defects_to_center = [] + for far in estimated_intertips_coords: + x = np.array(far) + center_mass_array = np.array(center_of_mass) + distance = np.sqrt( + np.power(x[0] - center_mass_array[0], + 2) + np.power(x[1] - center_mass_array[1], 2) + ) + distance_between_defects_to_center.append(distance) + + # Get an average of three shortest distances from finger webbing to center mass + sorted_defects_distances = sorted(distance_between_defects_to_center) + average_defect_distance = np.mean(sorted_defects_distances[0:2]) + self._average_defect_distance = average_defect_distance + # # Get fingertip points from contour hull + # # If points are in proximity of 80 pixels, consider as a single point in the group + # finger = [] + # for i in range(0, len(hull) - 1): + # if (np.absolute(hull[i][0][0] - hull[i + 1][0][0]) > 10) or ( + # np.absolute(hull[i][0][1] - hull[i + 1][0][1]) > 10): + # if hull[i][0][1] < 500: + # finger.append(hull[i][0]) + # + # + # # The fingertip points are 5 hull points with largest y coordinates + # finger = sorted(finger, key=lambda x: x[1]) + # fingers = finger[0:5] + if center_of_mass is not None and len(estimated_fingertips_coords) > 0: + # Calculate distance of each finger tip to the center mass + finger_distances = [] + for i in range(0, len(estimated_fingertips_coords)): + distance = np.sqrt( + np.power(estimated_fingertips_coords[i][0] - center_of_mass[0], 2) + np.power( + estimated_fingertips_coords[i][1] - center_of_mass[0], 2)) + finger_distances.append(distance) + self._finger_distances = finger_distances + else: + self._detection_status = -2 + self._detected = False + self._confidence = 0 + return + + + def _calculate_fingertips(self, hand_contour, defects): + intertips_coords = [] + intertips_indexes = [] + far_defect = [] + fingertips_coords = [] + fingertips_indexes = [] + defect_indices = [] + for defect_index in range(defects.shape[0]): + s, e, f, d = defects[defect_index, 0] + start = tuple(hand_contour[s][0]) + end = tuple(hand_contour[e][0]) + far = tuple(hand_contour[f][0]) + far_defect.append(far) + # cv2.line(frame, start, end, [0, 255, 0], 1) + a = math.sqrt((end[0] - start[0]) ** 2 + (end[1] - start[1]) ** 2) + b = math.sqrt((far[0] - start[0]) ** 2 + (far[1] - start[1]) ** 2) + c = math.sqrt((end[0] - far[0]) ** 2 + (end[1] - far[1]) ** 2) + angle = math.acos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) # cosine theorem + # cv2.circle(frame, far, 8, [211, 84, 125], -1) + # cv2.circle(frame, start, 8, [0, 84, 125], -1) + # cv2.circle(frame, end, 8, [0, 84, 125], -1) + # Get tips and intertips coordinates + # TODO: ENV_DEPENDENCE: this angle > 90degrees determinate if two points are considered fingertips or not and 90 make thumb to fail in some occasions + intertips_max_angle = math.pi / 1.7 + if angle <= intertips_max_angle: # angle less than 90 degree, treat as fingers + defect_indices.append(defect_index) + # cv2.circle(frame, far, 8, [211, 84, 0], -1) + intertips_coords.append(far) + intertips_indexes.append(f) + # cv2.putText(frame, str(s), start, FONT, 0.7, (255, 255, 255), 1) + # cv2.putText(frame, str(e), end, FONT, 0.7, (255, 255, 200), 1) + if len(fingertips_coords) > 0: + from scipy.spatial import distance + # calculate distances from start and end to the already known tips + start_distance, end_distance = tuple( + distance.cdist(fingertips_coords, [start, end]).min(axis=0)) + # TODO: ENV_DEPENDENCE: it determinate the pixels distance to consider two points the same. It depends on camera resolution and distance from the hand to the camera + same_fingertip_radius = 10 + if start_distance > same_fingertip_radius: + fingertips_coords.append(start) + fingertips_indexes.append(s) + + # cv2.circle(frame, start, 10, [255, 100, 255], 3) + if end_distance > same_fingertip_radius: + fingertips_coords.append(end) + fingertips_indexes.append(e) + + # cv2.circle(frame, end, 10, [255, 100, 255], 3) + else: + fingertips_coords.append(start) + fingertips_indexes.append(s) + + # cv2.circle(frame, start, 10, [255, 100, 255], 3) + fingertips_coords.append(end) + fingertips_indexes.append(e) + + # cv2.circle(frame, end, 10, [255, 100, 255], 3) + + # cv2.circle(frame, far, 10, [100, 255, 255], 3) + return fingertips_coords, fingertips_indexes, intertips_coords, intertips_indexes + + # TODO: modify to use a calculated confidence + def is_hand(self, fingertips, intertips, strict=True): + if strict: + return len(fingertips) == 5 and len(intertips) > 2 + else: + return 5 >= len(fingertips) > 2 + + + def detect_and_track(self, frame): + """ + Try to detect and track the hand on the given frame + + If the hand is not detected the extended_roi is updated which will be used in the next detection + :param frame: + :return: + """ + self._detect_in_frame(frame) + if self._detected: + self._consecutive_detection_fails = 0 + else: + self._consecutive_detection_fails += 1 + + self._track_in_frame(frame) + print(self._detected, self._tracked) + + # if it's the first time we don't detect in a row... + if self._consecutive_detection_fails == 1: + # if we have a tracking roi we use it + if self._tracked: + self.extended_roi = self.tracking_roi + else: + # if we don't, we use the last detected roi + self.extended_roi = self.detection_roi + elif self._consecutive_detection_fails > 1: + # if it's not the first time we don't detect we just extend the extended roi. + # it's autolimited to the initial Roi + self.extended_roi = self.extended_roi.upscaled(self.initial_roi, 10) + + if self._tracked: + self._consecutive_tracking_fails = 0 + else: + self._consecutive_tracking_fails += 1 + + # self._update_truth_value_by_frame2() + + def get_roi_to_use(self, frame): + """ + Calculate the roi to be used depending on the situation of the hand (initial, detected, tracked) + :param frame: + :return: + """ + current_roi = None + if self._detected: + current_roi = self.detection_roi + else: + # if we already have failed to detect we use the extended_roi + if self._consecutive_detection_fails > 0: + if self._tracked: + current_roi = self.tracking_roi + else: + current_roi = self.extended_roi + else: + # Not detected and not consecutive fails on detection. + # It's probably the first time we try to detect. + # If no initial_roi is given an square of 200 x 200 is taken on the center + if self.initial_roi is not None and self.initial_roi != Roi(): + current_roi = self.initial_roi + else: + current_roi = Roi.from_frame(frame, SIDE.CENTER, 50) + assert current_roi != Roi(), "hand can't be detected on a %s roi of the frame" % str(current_roi) + return current_roi + + def _track_in_frame(self, frame, method="camshift"): + self._last_frame = frame + if self._ever_detected: + roi_for_tracking = self.get_roi_to_use(frame) + + mask = self.create_hand_mask(frame) + x, y, w, h = roi_for_tracking + track_window = tuple(roi_for_tracking) + # set up the ROI for tracking + roi = roi_for_tracking.extract_from_frame(frame) + + if self._debug: + print roi_for_tracking + cv2.imshow("DEBUG: HandDetection_lib: _track_in_frame (frame_roied)", roi) + + # fi masked frame is only 1 channel + if len(frame.shape) == 2 or (len(frame.shape) == 3 and frame.shape[2] == 1): + hsv_roi = cv2.cvtColor(roi, cv2.COLOR_GRAY2RGB) + hsv_roi = cv2.cvtColor(hsv_roi, cv2.COLOR_RGB2HSV) + hsv = cv2.cvtColor(frame, cv2.COLOR_GRAY2BGR) + hsv = cv2.cvtColor(hsv, cv2.COLOR_BGR2HSV) + else: + hsv_roi = cv2.cvtColor(roi, cv2.COLOR_BGR2HSV) + hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) + # mask = cv2.inRange(hsv_roi, np.array((0., 60., 32.)), np.array((180., 255., 255.))) + roi_mask = mask[y:y + h, x:x + w] + if self._debug: + cv2.imshow("DEBUG: HandDetection_lib: follow (ROI extracted mask)", roi_mask) + roi_hist = cv2.calcHist([hsv_roi], [0], roi_mask, [180], [0, 180]) + cv2.normalize(roi_hist, roi_hist, 0, 255, cv2.NORM_MINMAX) + # Setup the termination criteria, either 10 iteration or move by atleast 1 pt + term_crit = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10, 1) + + dst = cv2.calcBackProject([hsv], [0], roi_hist, [0, 180], 1) + # apply meanshift to get the new location + if method == "meanshift": + tracked, new_track_window = cv2.meanShift(dst, track_window, term_crit) + self._tracked = (tracked != 0) + else: + rotated_rect, new_track_window = cv2.CamShift(dst, track_window, term_crit) + + intersection_rate = roi_for_tracking.intersection_rate(Roi(new_track_window)) + if intersection_rate and roi_for_tracking != Roi(new_track_window): + self._tracked = True + else: + self._tracked = False + if self._tracked: + self.tracking_roi = Roi(new_track_window) + else: + self._tracked = False + + + +# TODO: move to a utils file +def extract_contour_inside_circle(full_contour, circle): + """ + Get the intersection of a contour and a circle + + :param full_contour: Contour to be intersected + :param circle: circle to be intersected with the contour + :return: contour that is inside the given circle + """ + center, radius = circle + new_contour = [] + for point in full_contour: + if (point[0][0] - center[0]) ** 2 + (point[0][1] - center[1]) ** 2 < radius ** 2: + new_contour.append(point) + return np.array(new_contour) + +# TODO: move to a utils file +def extract_contour_inside_rect(full_contour, rect): + """ + Get the intersection of a contour and a rectangle + + :param full_contour: Contour to be intersected + :param rect: rectangle to be intersected with the contour + :return: ontour that is inside the given rectangle + """ + x1, y1, w, h = rect + x2 = x1 + w + y2 = y1 + h + new_contour = [] + for point in full_contour: + if x1 < point[0][0] < x2 and y1 < point[0][1] < y2: + new_contour.append(point) + return np.array(new_contour) + +if __name__ == '__main__': + hand = Hand() + print(hand.initial_roi, hand.depth_threshold) \ No newline at end of file diff --git a/HandDetection.py b/HandDetection.py index 55ad790..7a15ecc 100644 --- a/HandDetection.py +++ b/HandDetection.py @@ -1,197 +1,654 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import copy + +import os +from collections import deque +from Hand import Hand + import cv2 import numpy as np -import time - -#Open Camera object -cap = cv2.VideoCapture(0) - -#Decrease frame size -cap.set(cv2.cv.CV_CAP_PROP_FRAME_WIDTH, 1000) -cap.set(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT, 600) - -def nothing(x): - pass - -# Function to find angle between two vectors -def Angle(v1,v2): - dot = np.dot(v1,v2) - x_modulus = np.sqrt((v1*v1).sum()) - y_modulus = np.sqrt((v2*v2).sum()) - cos_angle = dot / x_modulus / y_modulus - angle = np.degrees(np.arccos(cos_angle)) - return angle - -# Function to find distance between two points in a list of lists -def FindDistance(A,B): - return np.sqrt(np.power((A[0][0]-B[0][0]),2) + np.power((A[0][1]-B[0][1]),2)) - - -# Creating a window for HSV track bars -cv2.namedWindow('HSV_TrackBar') - -# Starting with 100's to prevent error while masking -h,s,v = 100,100,100 - -# Creating track bar -cv2.createTrackbar('h', 'HSV_TrackBar',0,179,nothing) -cv2.createTrackbar('s', 'HSV_TrackBar',0,255,nothing) -cv2.createTrackbar('v', 'HSV_TrackBar',0,255,nothing) - -while(1): - - #Measure execution time - start_time = time.time() - - #Capture frames from the camera - ret, frame = cap.read() - - #Blur the image - blur = cv2.blur(frame,(3,3)) - - #Convert to HSV color space - hsv = cv2.cvtColor(blur,cv2.COLOR_BGR2HSV) - - #Create a binary image with where white will be skin colors and rest is black - mask2 = cv2.inRange(hsv,np.array([2,50,50]),np.array([15,255,255])) - - #Kernel matrices for morphological transformation - kernel_square = np.ones((11,11),np.uint8) - kernel_ellipse= cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(5,5)) - - #Perform morphological transformations to filter out the background noise - #Dilation increase skin color area - #Erosion increase skin color area - dilation = cv2.dilate(mask2,kernel_ellipse,iterations = 1) - erosion = cv2.erode(dilation,kernel_square,iterations = 1) - dilation2 = cv2.dilate(erosion,kernel_ellipse,iterations = 1) - filtered = cv2.medianBlur(dilation2,5) - kernel_ellipse= cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(8,8)) - dilation2 = cv2.dilate(filtered,kernel_ellipse,iterations = 1) - kernel_ellipse= cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(5,5)) - dilation3 = cv2.dilate(filtered,kernel_ellipse,iterations = 1) - median = cv2.medianBlur(dilation2,5) - ret,thresh = cv2.threshold(median,127,255,0) - - #Find contours of the filtered frame - contours, hierarchy = cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - - #Draw Contours - #cv2.drawContours(frame, cnt, -1, (122,122,0), 3) - #cv2.imshow('Dilation',median) - - #Find Max contour area (Assume that hand is in the frame) - max_area=100 - ci=0 - for i in range(len(contours)): - cnt=contours[i] - area = cv2.contourArea(cnt) - if(area>max_area): - max_area=area - ci=i - - #Largest area contour - cnts = contours[ci] - - #Find convex hull - hull = cv2.convexHull(cnts) - - #Find convex defects - hull2 = cv2.convexHull(cnts,returnPoints = False) - defects = cv2.convexityDefects(cnts,hull2) - - #Get defect points and draw them in the original image - FarDefect = [] - for i in range(defects.shape[0]): - s,e,f,d = defects[i,0] - start = tuple(cnts[s][0]) - end = tuple(cnts[e][0]) - far = tuple(cnts[f][0]) - FarDefect.append(far) - cv2.line(frame,start,end,[0,255,0],1) - cv2.circle(frame,far,10,[100,255,255],3) - - #Find moments of the largest contour - moments = cv2.moments(cnts) - - #Central mass of first order moments - if moments['m00']!=0: - cx = int(moments['m10']/moments['m00']) # cx = M10/M00 - cy = int(moments['m01']/moments['m00']) # cy = M01/M00 - centerMass=(cx,cy) - - #Draw center mass - cv2.circle(frame,centerMass,7,[100,0,255],2) - font = cv2.FONT_HERSHEY_SIMPLEX - cv2.putText(frame,'Center',tuple(centerMass),font,2,(255,255,255),2) - - #Distance from each finger defect(finger webbing) to the center mass - distanceBetweenDefectsToCenter = [] - for i in range(0,len(FarDefect)): - x = np.array(FarDefect[i]) - centerMass = np.array(centerMass) - distance = np.sqrt(np.power(x[0]-centerMass[0],2)+np.power(x[1]-centerMass[1],2)) - distanceBetweenDefectsToCenter.append(distance) - - #Get an average of three shortest distances from finger webbing to center mass - sortedDefectsDistances = sorted(distanceBetweenDefectsToCenter) - AverageDefectDistance = np.mean(sortedDefectsDistances[0:2]) - - #Get fingertip points from contour hull - #If points are in proximity of 80 pixels, consider as a single point in the group - finger = [] - for i in range(0,len(hull)-1): - if (np.absolute(hull[i][0][0] - hull[i+1][0][0]) > 80) or ( np.absolute(hull[i][0][1] - hull[i+1][0][1]) > 80): - if hull[i][0][1] <500 : - finger.append(hull[i][0]) - - #The fingertip points are 5 hull points with largest y coordinates - finger = sorted(finger,key=lambda x: x[1]) - fingers = finger[0:5] - - #Calculate distance of each finger tip to the center mass - fingerDistance = [] - for i in range(0,len(fingers)): - distance = np.sqrt(np.power(fingers[i][0]-centerMass[0],2)+np.power(fingers[i][1]-centerMass[0],2)) - fingerDistance.append(distance) - - #Finger is pointed/raised if the distance of between fingertip to the center mass is larger - #than the distance of average finger webbing to center mass by 130 pixels - result = 0 - for i in range(0,len(fingers)): - if fingerDistance[i] > AverageDefectDistance+130: - result = result +1 - - #Print number of pointed fingers - cv2.putText(frame,str(result),(100,100),font,2,(255,255,255),2) - - #show height raised fingers - #cv2.putText(frame,'finger1',tuple(finger[0]),font,2,(255,255,255),2) - #cv2.putText(frame,'finger2',tuple(finger[1]),font,2,(255,255,255),2) - #cv2.putText(frame,'finger3',tuple(finger[2]),font,2,(255,255,255),2) - #cv2.putText(frame,'finger4',tuple(finger[3]),font,2,(255,255,255),2) - #cv2.putText(frame,'finger5',tuple(finger[4]),font,2,(255,255,255),2) - #cv2.putText(frame,'finger6',tuple(finger[5]),font,2,(255,255,255),2) - #cv2.putText(frame,'finger7',tuple(finger[6]),font,2,(255,255,255),2) - #cv2.putText(frame,'finger8',tuple(finger[7]),font,2,(255,255,255),2) - - #Print bounding rectangle - x,y,w,h = cv2.boundingRect(cnts) - img = cv2.rectangle(frame,(x,y),(x+w,y+h),(0,255,0),2) - - cv2.drawContours(frame,[hull],-1,(255,255,255),2) - - ##### Show final image ######## - cv2.imshow('Dilation',frame) - ############################### - - #Print execution time - #print time.time()-start_time - - #close the output video by pressing 'ESC' - k = cv2.waitKey(5) & 0xFF - if k == 27: - break - - -cap.release() -cv2.destroyAllWindows() + + + +def calculate_angle(v1, v2): + """ + Calculate an return the angle between two vectors in degrees + TODO: move to a utils file. + + :param v1: input vector 1 + :param v2: input vector + :return: Degree angle between the two vectors + """ + dot = np.dot(v1, v2) + x_modulus = np.sqrt((v1 * v1).sum()) + y_modulus = np.sqrt((v2 * v2).sum()) + cos_angle = dot / x_modulus / y_modulus + angle = np.degrees(np.arccos(cos_angle)) + return angle + + + + + + +class HandDetector: + """ + Class to detect hands on a image. It keeps an array with the expected hands. + + """ + def __init__(self, source=0): + + # Open Camera object + + # TODO: For testing only + if source != -1: + self.capture = cv2.VideoCapture(source) + else: + self.capture = None + + self.hands = [] + self.first_frame = None + self.next_hand_id = 0 + # TODO: ENV_DEPENDENCE: depending on the environment and camera it would be more or less frames to discard + self.discarded_frames = 10 + self.last_frames = deque(maxlen=self.discarded_frames) + self.debug = False + self.mask_mode = "rgbd" + + # Only used with RGBD cameras to create the mask. + self.depth_threshold = -1 + + # Decrease frame size + # self.capture.set(cv2.CAP_PROP_FRAME_WIDTH, 1000) + # self.capture.set(cv2.CAP_PROP_FRAME_HEIGHT, 600) + + + # TODO: Extend to return multiple hands? + # TODO: create a static method on Hand to return all the hands detected on a frame + def hand_from_frame(self, frame, roi = None): + """ + Get a Hand class from a roi in a frame + TODO: Check if it can be removed. It have been replaced by update_expected_hands + :param frame: frame where we want to look for the hand + :param roi: roi in frame to look for the hand + :return: Hand if found on frame, None if not + """ + hand = Hand() + + + hand.initial_roi = roi + #TODO: only to test. Replace + hand.depth_threshold = self.depth_threshold + hand._detect_in_frame(frame) + return hand + # hand.depth_threshold = self.depth_threshold + + + + def add_new_expected_hand(self, frame_roi): + """ + Add a new expected hand to the list of the detector. + :param frame_roi: initial roi where the hand is expected to be find and that will limit the area of the frame where it's expected to be found + :return: + """ + hand = Hand() + hand.initial_roi = frame_roi + # TODO: add only if mode is Depth + hand.depth_threshold = self.depth_threshold + self.hands.append(hand) + + def update_expected_hands(self, frame): + """ + Update with a frame the list of expected hands. + :param frame: Frame to update the hands. + :return: + """ + if len(self.hands) > 0: + for existing_hand in self.hands: + existing_hand.detect_and_track(frame) + # TODO: determine if hands are automatically removed or must be removed by the user + # if existing_hand.confidence <= 0: + # print "removing hand" + # self.hands.remove(existing_hand) + + # TODO: add a new method to know if we are tracking with overlaping rois. + + + def add_hand2(self, frame, roi = None): + """ + # TODO: Deprecated. Check if it can be removed. + Add a new hand to the system to be tracked. + :param frame: + :param roi: + :return: + """ + + new_hand = self.hand_from_frame(frame, roi) + new_hand._track_in_frame(frame) + + if new_hand.valid: + new_hand.id = len(self.hands) + self.hands.append(new_hand) + # cv2.putText(masked_frame, "HAND FOUND", + # (template_x, template_y + template_h + 10), + # self.font, 1, [0, 0, 0], 2) + # print("add_hand2:", "HAND FOUND") + else: + # cv2.putText(masked_frame, "CENTER YOUR HAND", + # (template_x, template_y + template_h + 10), + # self.font, 1, [0, 0, 0], 2) + print("CENTER THE HAND") + # if ret == -2: + # cv2.putText(masked_frame, "PLEASE PUT YOUR HAND HERE", (template_x - 100, template_y + template_h + 10), + # self.font, 1, [0, 0, 0], 2) + # masked_frame = cv2.rectangle(masked_frame, (template_x, template_y), + # (template_x + template_w, template_y + template_h), [0, 0, 0]) + return + + + def calculate_bounding_rects_intersection(self, bounding_rect_1, bounding_rect_2): + b1_x_left = bounding_rect_1[0] + b1_y_top = bounding_rect_1[1] + b1_x_right = bounding_rect_1[0] + bounding_rect_1[2] + b1_y_bottom = bounding_rect_1[1] + bounding_rect_1[3] + b2_x_left = bounding_rect_2[0] + b2_y_top = bounding_rect_2[1] + b2_x_right = bounding_rect_2[0] + bounding_rect_2[2] + b2_y_bottom = bounding_rect_2[1] + bounding_rect_2[3] + x_left = max(b1_x_left, b2_x_left) + y_top = max(b1_y_top, b2_y_top) + x_right = min(b1_x_right, b2_x_right) + y_bottom = min(b1_y_bottom, b2_y_bottom) + + if x_left > x_right or y_top > y_bottom: + return 0, None + + # compute the area of intersection rectangle + intersection_area = (x_right - x_left + 1) * (y_bottom - y_top + 1) + + # compute the area of both the prediction and ground-truth + # rectangles + box_a_area = (b1_x_right - b1_x_left + 1) * (b1_y_bottom - b1_y_top + 1) + box_b_area = (b2_x_right - b2_x_left + 1) * (b2_y_bottom - b2_y_top + 1) + + # compute the intersection over union by taking the intersection + # area and dividing it by the sum of prediction + ground-truth + # areas - the interesection area + iou = intersection_area / float(box_a_area + box_b_area - intersection_area) + + # return the intersection over union value + return iou, (x_left, y_top, x_right, y_bottom) + + + def set_mask_mode(self, mode): + self.mask_mode = mode + + + def set_depth_threshold(self, threshold): + self.depth_threshold = threshold + #TODO: Set to all available hands + + + def capture_and_compute2(self): + """ + Method to run the lib independently capturing frames from camera an computing. + # TODO: Need to be refactored. Most of the current functionality have been moved to the Hand class... + :return: + """ + if self.capture.isOpened(): + while self.capture.isOpened(): + + # Measure execution time + # start_time = time.time() + + # Capture frames from the camera + ret, frame = self.capture.read() + + if ret is True: + if len(self.hands) != 1: + self.add_hand2(frame) + + self.update_detection_and_tracking(frame) + + overlayed_frame = self.draw_hands_on_frame(frame) + + ##### Show final image ######## + if self.debug: + cv2.imshow('DEBUG: HandDetection_lib: Detection', overlayed_frame) + ############################### + # Print execution time + # print time.time()-start_time + else: + print "No video detected" + + # close the output video by pressing 'ESC' + if self.debug and len(self.hands) > 0: + k = cv2.waitKey(0) & 0xFF + else: + k = cv2.waitKey(5) & 0xFF + if k == 27: + break + else: + print("Capture device is not opened.") + + + def draw_hands_on_frame(self, frame): + """ + Draw the detected / tracked hands on the frame + :param frame: frame to be overlayed with hands + :return: overlayed frame + """ + overlayed_frame = frame.copy() + for hand in self.hands: + overlayed_frame = self.draw_hand_overlay(frame, hand) + return overlayed_frame + + + def draw_hand_overlay(self, frame, hand): + """ + TODO: Move to hand class. It could have an static + :param frame: frame to be overlayed with hand information + :param hand: hand to be drawn on the frame. + :return: + """ + if hand.detected: + for finger_number, fingertip in enumerate(hand.fingertips): + cv2.circle(frame, tuple(fingertip), 10, [255, 100, 255], 3) + cv2.putText(frame, 'finger' + str(finger_number), tuple(fingertip), self.font, 0.5, + (255, 255, 255), + 1) + for defect in hand.intertips: + cv2.circle(frame, tuple(defect), 8, [211, 84, 0], -1) + # self.draw_contour_features(frame, hand.contour) + x, y, w, h = hand.bounding_rect + # cv2.putText(frame, (str(w)), (x + w, y), self.font, 0.3, [255, 255, 255], 1) + # cv2.putText(frame, (str(h)), (x + w, y + h), self.font, 0.3, [255, 255, 255], 1) + # cv2.putText(frame, (str(w * h)), (x + w / 2, y + h / 2), self.font, 0.3, [255, 255, 255], 1) + # cv2.putText(frame, (str(x)+", "+str(y)), (x-10, y-10), self.font, 0.3, [255, 255, 255], 1) + cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2) + + if hand.detected or hand.tracked: + cv2.drawContours(frame, [hand.contour], -1, (255, 255, 255), 2) + + points = np.array(hand.position_history) + cv2.polylines(img=frame, pts=np.int32([points]), isClosed=False, color=hand.color) + tail_length = 15 + if len(points) > tail_length: + for i in np.arange(1, tail_length): + ci = len(points) - tail_length + i + thickness = int((float(i) / tail_length) * 13) + 1 + cv2.line(frame, tuple(points[ci - 1]), tuple(points[ci]), (0, 0, 255), thickness) + + if hand.center_of_mass is not None: + # Draw center mass + cv2.circle(frame, hand.center_of_mass, 7, [100, 0, 255], 2) + cv2.putText(frame, 'Center', tuple(hand.center_of_mass), self.font, 0.5, (255, 255, 255), 1) + + hand_string = "hand %d %s: D=%s|T=%s|L=%s|F=%d" % ( + hand.id, str(hand.center_of_mass), str(hand.detected), str(hand.tracked), str(hand.confidence), + hand.frame_count) + cv2.putText(frame, hand_string, (10, 30 + 15 * int(hand.id)), self.font, 0.5, (255, 255, 255), 1) + return frame + + def draw_contour_features(self, to_show, hand_contour): + """ + Draw features of the hand on the frame + TODO: Move to to Hand class. + TODO: Refactor to get the info from the Hand class and not calculate it again + + :param to_show: frame to be overlayed with hand information + :param hand_contour: hand contour to get the features from it + :return: TODO: return frame with overlayed information + """ + perimeter = cv2.arcLength(hand_contour, True) + # print perimeter + hull = cv2.convexHull(hand_contour, returnPoints=False) + new_contour = [] + # for index in hull: + # cv2.circle(to_show,tuple(hand_contour[index][0][0]),5,(255,255,255),2) + defects = cv2.convexityDefects(hand_contour, hull) + for defect_index in range(defects.shape[0]): + s, e, f, d = defects[defect_index, 0] + # cv2.circle(to_show,start,5,(0,99,255),2) + # cv2.circle(to_show, end, 5, (0, 99, 255), 2) + # cv2.circle(to_show, far, 5, (0, 99, 255), 2) + new_contour.append(hand_contour[s]) + new_contour.append(hand_contour[f]) + new_contour.append(hand_contour[e]) + + x, y, w, h = cv2.boundingRect(hand_contour) + cv2.rectangle(to_show, (x, y), (x + w, y + h), (0, 255, 0), 2) + rect = cv2.minAreaRect(hand_contour) + box = cv2.boxPoints(rect) + box = np.int0(box) + cv2.drawContours(to_show, [box], 0, (0, 0, 255), 2) + + (x, y), radius = cv2.minEnclosingCircle(hand_contour) + center = (int(x), int(y)) + radius = int(radius) + cv2.circle(to_show, center, radius, (255, 0, 0), 2) + ellipse = cv2.fitEllipse(hand_contour) + cv2.ellipse(to_show, ellipse, (100, 48, 170), 2) + + rows, cols = to_show.shape[:2] + [vx, vy, x, y] = cv2.fitLine(hand_contour, cv2.DIST_L2, 0, 0.01, 0.01) + lefty = int((-x * vy / vx) + y) + righty = int(((cols - x) * vy / vx) + y) + cv2.line(to_show, (cols - 1, righty), (0, lefty), (0, 255, 0), 2) + + cv2.imshow("Hand with contour", to_show) + + def exit(self): + """ + Method for the self execution of the lib to end the opencv windows. + :return: + """ + self.capture.release() + cv2.destroyAllWindows() + + def get_MOG2_mask(self, image): + """ + TODO: Move to Hand class + :param image: + :return: + """ + # TODO: not working (it considers everything shadows or moves (too dark background) + fgbg = cv2.createBackgroundSubtractorMOG2(detectShadows=False) + # fgbg= cv2.BackgroundSubtractorMOG2(0, 50) + # fgbg=cv2.BackgroundSubtractor() + # fgbg = cv2.BackgroundSubtractorMOG() + gray_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) + # TODO: ENV_DEPENDENCE: it could depend on the camera quality + # blur = cv2.medianBlur(gray_image, 100) + blur_radius = 5 + blurred = cv2.GaussianBlur(image, (blur_radius, blur_radius), 0) + # cv2.imshow("blurred", blur) + mask = fgbg.apply(blurred) + return mask + + + def get_movement_buffer_mask(self, image): + """ + Calculate a mask for the image based on the movement respect the previous frames. + TODO: Move to hand Class + :param image: image to get the mask from. + :return: mask for the frame mased on the movement of this image related with previous ones. + """ + mask = None + self.last_frames.append(image) + mean_image = np.zeros(image.shape, dtype=np.float32) + for old_image in self.last_frames: + cv2.accumulate(old_image, mean_image) + mean_image = mean_image / len(self.last_frames) + + blur_radius = 5 + blurred_image = cv2.GaussianBlur(image, (blur_radius, blur_radius), 0) + blurred_background = cv2.GaussianBlur(mean_image, (blur_radius, blur_radius), 0) + diff = cv2.absdiff(blurred_image, blurred_background.astype(np.uint8)) + # print "diff" + gray_diff = cv2.cvtColor(diff, cv2.COLOR_BGR2GRAY) + if self.debug: + cv2.imshow("DEBUG: HandDetection_lib: diff", gray_diff) + # TODO: ENV_DEPENDENCE: it could depend on the lighting and environment + _, mask = cv2.threshold(gray_diff, 40, 255, cv2.THRESH_BINARY) + return mask + + def get_simple_diff_mask(self, image): + """ + Calculate a mask for the image based on the difference with previous ones. + TODO: Move to Hand class + :param image: image to get the mask + :return: mask based on the difference with previous frames + """ + mask = None + if len(self.last_frames) < self.discarded_frames: + self.last_frames.append(image) + mean_image = np.zeros(image.shape, dtype=np.float32) + for old_image in self.last_frames: + cv2.accumulate(old_image, mean_image) + self.first_frame = mean_image / len(self.last_frames) + else: + blur_radius = 5 + blurred_image = cv2.GaussianBlur(image, (blur_radius, blur_radius), 0) + blurred_background = cv2.GaussianBlur(self.first_frame, (blur_radius, blur_radius), 0) + diff = cv2.absdiff(blurred_image, blurred_background.astype(np.uint8)) + # print "diff" + gray_diff = cv2.cvtColor(diff, cv2.COLOR_BGR2GRAY) + if self.debug: + cv2.imshow("DEBUG: HandDetection_lib: diff", gray_diff) + # TODO: ENV_DEPENDENCE: it could depend on the lighting and environment + _, mask = cv2.threshold(gray_diff, 40, 255, cv2.THRESH_BINARY) + return mask + + def get_simple_diff_mask2(self, image): + """ + TODO: Move to Hand class + :param image: + :return: + """ + mask = None + # TODO: ENV_DEPENDENCE: it could depend on the lighting and environment + blur_radius = 5 + if len(self.last_frames) > 0: + mean_image = np.zeros(image.shape, dtype=np.float32) + for old_image in self.last_frames: + cv2.accumulate(old_image, mean_image) + result_image = mean_image / len(self.last_frames) + result_image = result_image.astype(np.uint8) + # cv2.imshow("mean_image", result_image.astype(np.uint8)) + # cv2.imshow("the image", image) + blurred_image = cv2.GaussianBlur(image, (blur_radius, blur_radius), 0) + blurred_background = cv2.GaussianBlur(result_image, (blur_radius, blur_radius), 0) + # cv2.imshow("b_mean_image", blurred_background) + # cv2.imshow("b_the image", blurred_image) + diff = cv2.absdiff(blurred_image, blurred_background.astype(np.uint8)) + # cv2.imshow("diff", diff) + # print "diff" + gray_diff = cv2.cvtColor(diff, cv2.COLOR_BGR2GRAY) + # cv2.imshow("gray_diff", gray_diff) + # TODO: ENV_DEPENDENCE: it could depend on the lighting and environment + _, mask = cv2.threshold(gray_diff, 40, 255, cv2.THRESH_BINARY) + non_zero_pixels = cv2.countNonZero(mask) + non_zero_percent = (non_zero_pixels * 100.) / (image.shape[0] * image.shape[1]) + if non_zero_percent < 1: + self.first_frame = result_image + elif non_zero_percent > 95: + self.reset_background() + # print "Non zero pixel percentage %d" % non_zero_percent + + if self.first_frame is not None: + + blurred_image = cv2.GaussianBlur(image, (blur_radius, blur_radius), 0) + blurred_background = cv2.GaussianBlur(self.first_frame, (blur_radius, blur_radius), 0) + diff = cv2.absdiff(blurred_image, blurred_background.astype(np.uint8)) + # print "diff" + gray_diff = cv2.cvtColor(diff, cv2.COLOR_BGR2GRAY) + # cv2.imshow("diff", gray_diff) + # TODO: ENV_DEPENDENCE: it could depend on the lighting and environment + _, mask = cv2.threshold(gray_diff, 40, 255, cv2.THRESH_BINARY) + else: + self.last_frames.append(image) + return mask + + + def detect_hands_in_frame(self, frame, roi=None): + """ + Detect multiple hands on a frame and return an array with these. + + TODO: move to hand class as an staticmethod + :param frame: frame to be processed to find the hands on it. + :param roi: region to look for the hands. + :return: list of hands found as Class + """ + new_hands = [] + + contours, _ = self.create_contours_and_mask(frame, roi) + + # Draw Contours + if self.debug: + to_show = frame.copy() + cv2.drawContours(to_show, contours, -1, (122, 122, 0), 1) + if roi is not None: + cv2.rectangle(to_show, (roi[0], roi[1]), (roi[0] + roi[2], roi[1] + roi[3]), (122, 122, 255)) + cv2.imshow("DEBUG: HandDetection_lib: detect_hands_in_frame (Detected Contours)", to_show) + k = cv2.waitKey(1) + + if len(contours) > 0: + # Find Max contour area (Assume that hand is in the frame) + # TODO: ENV_DEPENDENCE: depends on the camera resolution, distance to the background, noisy areas sizes + min_area = 100 + + for contour in contours: + area = cv2.contourArea(contour) + if area > min_area: + hand_contour = contour + hand = self.contour_to_new_hand(frame, hand_contour) + if hand is not None: + new_hands.append(hand) + return new_hands + + + + def reset_background(self): + """ + Method usefull only for the get_simple_diff_mask2. + TODO: If it's going to be used probably would be moved to a buffer class with the related methods for the background. + TODO: test it better + :return: None + """ + self.last_frames = deque(maxlen=self.discarded_frames) + self.first_frame = None + + def update_detection(self, frame): + """ + Method to update the detection of the existing hands. + This method look for all new hands detected on the frame. Then it looks for the intersection with the existing ones. + + TODO: Deprecated. Check if it would be usefull to adapt to the new way of detecting tracking hands. + + + :param frame: Image used to look for hands. + :return: None + """ + for hand in self.hands: + hand.detected = False + detected_hands = self.detect_hands_in_frame(frame) + for detected_hand in detected_hands: + if len(self.hands) > 0: + hand_exists = False + for existing_hand_index, existing_hand in enumerate(self.hands): + intersection_value, _ = self.calculate_bounding_rects_intersection( + detected_hand._bounding_rect, + existing_hand.bounding_rect) + if intersection_value > 0.1: + hand_exists = True + existing_hand.update_attributes_from_detected(detected_hand) + break + if not hand_exists: + detected_hand._id = str(self.next_hand_id) + detected_hand._detected = True + self.next_hand_id += 1 + self.hands.append(detected_hand) + print "New hand" + else: + detected_hand._id = str(self.next_hand_id) + detected_hand._detected = True + self.next_hand_id += 1 + self.hands.append(detected_hand) + print("New hand") + + + def detect_fist(self, frame, roi): + """ + Method to detect fist. + TODO: Deprecated. Move to Hand class and review if it's working. + TODO: There's too much code similar to detect hand. It' would probably be unified or common code extracted. + + :param frame: Image to be processed looking for a fist. + :param roi: Roi of the image to be used to look for the fist. + :return: None + """ + contours, _ = self.create_contours_and_mask(frame, roi) + hand_contour = None + if len(contours) > 0: + # Get the maximum area contour + min_area = 100 + hand_contour = None + for i in range(len(contours)): + cnt = contours[i] + area = cv2.contourArea(cnt) + if area > min_area: + min_area = area + hand_contour = contours[i] + + if hand_contour is not None: + if self.debug: + to_show = frame.copy() + cv2.drawContours(to_show, [hand_contour], -1, (0, 255, 255), 1) + cv2.imshow("DEBUG: HandDetection_lib: detect_fist (Hand with contour)", to_show) + hull = cv2.convexHull(hand_contour, returnPoints=False) + new_contour = [] + # for index in hull: + # cv2.circle(to_show,tuple(hand_contour[index][0][0]),5,(255,255,255),2) + defects = cv2.convexityDefects(hand_contour, hull) + for defect_index in range(defects.shape[0]): + s, e, f, d = defects[defect_index, 0] + start = tuple(hand_contour[s][0]) + end = tuple(hand_contour[e][0]) + far = tuple(hand_contour[f][0]) + # cv2.circle(to_show,start,5,(0,99,255),2) + # cv2.circle(to_show, end, 5, (0, 99, 255), 2) + # cv2.circle(to_show, far, 5, (0, 99, 255), 2) + new_contour.append(hand_contour[s]) + new_contour.append(hand_contour[f]) + new_contour.append(hand_contour[e]) + + # define criteria and apply kmeans() + criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 10, 1.0) + new_contour_2d_points = np.float32(np.array(new_contour).reshape(len(new_contour), 2)) + + #Try to separate point into 2 groups. + ret, label, center = cv2.kmeans(new_contour_2d_points, 2, None, criteria, 10, cv2.KMEANS_RANDOM_CENTERS) + + # Now separate the data, Note the ravel() + max_group_len = 0 + max_group = None + + for label_number in range(2): + group = new_contour_2d_points[label.ravel() == label_number] + rand_color = get_random_color() + for point in group: + cv2.circle(to_show, tuple(point), 5, rand_color, 2) + if len(group) > max_group_len: + max_group_len = len(group) + max_group = group + + # get the circle enclosing the bigger group of points in the contour of the fist + (x, y), radius = cv2.minEnclosingCircle(max_group) + center = (int(x), int(y)) + radius = int(radius) + 20 + if self.debug: + cv2.circle(to_show, center, radius, (122, 122, 0), 1) + for point in max_group: + cv2.circle(to_show, tuple(point), 5, (0, 255, 255), 2) + cv2.imshow("DEBUG: HandDetection_lib: detect_fist (Fist_ring)", to_show) + # create the countour with only the points inside that circle + new_contour = extract_contour_inside_circle(hand_contour, (center, radius)) + return cv2.boundingRect(new_contour), new_contour + else: + return None, None + + + +def main(): + hand_detector = HandDetector() + # hand_detector = HandDetector('./resources/testing_hand_video2.mp4') + hand_detector.debug = True + + hand_detector.capture_and_compute2() + hand_detector.exit() + + +if __name__ == "__main__": + main() diff --git a/README.md b/README.md index a055ba1..d3bb6f0 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,33 @@ Real-Time Hand Gesture Detection ================================ -The code captures frames using a web camera (tested on mac's camera) and outputs a video with a number designates the number of pointed finger. For example, a fist corresponds to 0 and an open hand to 5. +Python library offering a class for hand detection mixing diverse methods using opencv and inspired on this sources: +https://github.com/lzane/Fingers-Detection-using-OpenCV-and-Python/tree/py2_opencv2 +https://github.com/sashagaz/Hand_Detection + +The HandDetection class let the user get the hand detection from an image or the own class can get the camera capture +task. +Currently it only draw the points of the fingers on the original image. This project is written in Python 2.7. The following libraries are used in this project and neccessary to be add to your computer: 1) Time - usually comes with Python 2.7 2) OpenCV (2.4.9) - http://docs.opencv.org/trunk/doc/py_tutorials/py_setup/py_setup_in_windows/py_setup_in_windows.html 3) NumPy (1.8.0rc1) - http://www.numpy.org -The code is consist of a single file and can be executed from the commandline or terminal by calling: python HandDetection.py +The library can be executed for it own test with: +```python HandDetection.py``` When running the python file, you can expect to see the real time frame from your camera with a bounding rectangular framing your hand. The bounding rectangular will contain yellow circles corresponding to the fingertips and finger webbing. The center mass of your hand will also appear in the bounding rectangular. A number on the upper left side of the frame corresponds to the number of pointed fingers. To increase accuracy of the gesture recognition, it is recommended to run the code in a bright light room. Additionally, the code can analyze one hand (either left or right), and the hand needs to be in front of the camera. References: -opencv documentation http://docs.opencv.org/ -[2] Skin Detection using HSV color space - V. A. Oliveira, A. Conci -[3] OpenCv Documentation - Miscellaneous Image Transformation -http://docs.opencv.org/modules/imgproc/doc/miscellaneous_transformations.html -[4] OpenCv Documentation - Morphological Operations http://docs.opencv.org/trunk/doc/py_tutorials/py_imgproc/py_morphological_ops/py_morphological_ops.html -[5] Suzuki, S. and Abe, K., Topological Structural Analysis of Digitized Binary Images by Border Following. CVGIP 30 1, pp 32-46 (1985) - -For any question related to the code please contact via email: -sasha7064570@gmail.com - -Sasha Gazman - + +https://github.com/lzane/Fingers-Detection-using-OpenCV-and-Python/tree/py2_opencv2 + +https://github.com/sashagaz/Hand_Detection + +http://www.tmroyal.com/a-high-level-description-of-two-fingertip-tracking-techniques-k-curvature-and-convexity-defects.html + +http://fivedots.coe.psu.ac.th/~ad/jg/nui055/ + +https://link.springer.com/article/10.1007/s10489-015-0680-z diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/resources/right_hand_mask.png b/resources/right_hand_mask.png new file mode 100644 index 0000000..61f7c5d Binary files /dev/null and b/resources/right_hand_mask.png differ diff --git a/resources/testing_hand_video.mp4 b/resources/testing_hand_video.mp4 new file mode 100644 index 0000000..9f0d3f1 Binary files /dev/null and b/resources/testing_hand_video.mp4 differ diff --git a/rgbdframe.py b/rgbdframe.py new file mode 100644 index 0000000..c07372f --- /dev/null +++ b/rgbdframe.py @@ -0,0 +1,20 @@ +import numpy as np + + + +class RGBDFrame(np.ndarray): + def __init__(self, *args, **kwargs): + super(RGBDFrame, self).__init__(*args, **kwargs) + self.__depth = None + + def __new__(cls, a): + obj = np.asarray(a).view(cls) + return obj + + @property + def depth(self): + return self.__depth + + @depth.setter + def depth(self, value): + self.__depth = value \ No newline at end of file diff --git a/roi.py b/roi.py new file mode 100644 index 0000000..f698d47 --- /dev/null +++ b/roi.py @@ -0,0 +1,428 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import numpy as np +import cv2 + + +class SIDE: + """ + Class containing the possible options of sides of a roi to be selected as relative areas of a frame. + """ + RIGHT = 0 + LEFT = 1 + TOP = 2 + BOTTOM = 3 + CENTER = 4 + +class Roi(list): + """ + Class containing the attributes and methods needed to keep the information of a ROI + The Roi is expected to be created based on the initial point (top right) and the width and height. + This class can create Roi with a size related to a frame. + Multiple methods are provided to do operations with the ROI like upscaling, intersect or draw on a frame. + Also convenience properties are implemented to return points of the ROI. + """ + def __init__(self, in_list=None): + """ + Roi class is based on a list of 4 int or floats. It can be initialized with the initial point and width and height. + :param in_list: initialization values. Tuple or List is expected to be received with (x, y, width, height) + """ + if in_list is not None: + assert isinstance(in_list, (list, tuple)), "in_list must be of the python list tupe. in_list is %s type" % type(in_list) + assert len(in_list) == 4, "in_list must 4 len. in_list is %d len" % len(in_list) + self.extend(list(in_list)) + else: + self.extend([0,0,0,0]) + # self._x = 0 + # self._y = 0 + # self._width = 0 + # self._height = 0 + + @property + def x(self): + return self[0] + + @x.setter + def x(self, x): + self[0]=x + + @property + def y(self): + return self[1] + + @y.setter + def y(self, y): + self[1]=y + + @property + def width(self): + return self[2] + + @width.setter + def width(self, width): + self[2]=width + + @property + def height(self): + return self[3] + + @height.setter + def height(self, height): + self[3]=height + + @property + def x1(self): + """ + Convenience property to return the initial point x coord. + :return: initial point x coord + """ + return self.x + + @x1.setter + def x1(self, value): + """ + Convenience property to set the initial point x coord. + :param value: value init x coord + :return: None + """ + self.x = value + + @property + def x2(self): + """ + Convenience property to return the end point x coord. + :return: end point x coord + """ + return self.x+self.width + + @x2.setter + def x2(self, value): + """ + Convenience property to set the end point x coord. + :param value: valueend x coord + :return: + """ + width = value-self.x + assert width > 0, "Not a valid x2 value %d. It results in a width value of %d" % (value, width) + self.width = width + + @property + def y1(self): + """ + Convenience property to return the initial point y coord. + :return: initial point y coord + """ + return self.y + + @y1.setter + def y1(self, value): + """ + Convenience property to set the initial point y coord. + :param value: value init y coord + :return: None + """ + self.y = value + + @property + def y2(self): + """ + Convenience property to return the end point y coord. + :return: end point y coord + """ + return self.y +self.height + + @y2.setter + def y2(self, value): + """ + Convenience property to set the end point y coord. + :param value: valueend y coord + :return: + """ + height = value - self.y + assert height > 0, "Not a valid y2 value %d. It results in a height value of %d" % (value, height) + self.height = height + + @property + def p1(self): + """ + Convenience property to return the initial point + :return: initial point + """ + return (self.x, self.y) + + @p1.setter + def p1(self, value): + """ + Convenience setter to set the initial point of the Roi + :param value: List or tupple with the coords of the initial point + :return: None + """ + assert isinstance(value, (tuple, list)), "p1 need to be a tuple or list. type is %s" % (type(value)) + assert len(value)==2, "p1 need to have a len of 2. %r have a len of %d" % (value, len(value)) + self.x, self.y = value + + @property + def p2(self): + """ + Convenience property to return the end point + :return: end point + """ + return (self.x2, self.y2) + + @p2.setter + def p2(self, value): + """ + Convenience setter to set the end point of the Roi + :param value: List or tupple with the coords of the end point + :return: None + """ + assert isinstance(value, (tuple, list)), "p1 need to be a tuple or list. type is %s" % (type(value)) + assert len(value) == 2, "p1 need to have a len of 2. %r have a len of %d" % (value, len(value)) + self.x2, self.y2 = value + + @property + def init_coords(self): + """ + Convenience property to get initial point. + :return: initial point + """ + return self.p1 + + @property + def top_left(self): + """ + Convenience property to return the top left point of the ROI + :return: top left point of the ROI + """ + return (self.x, self.y) + + @property + def top_right(self): + """ + Convenience property to return the top right point of the ROI + :return: top right point of the ROI + """ + return (self.x, self.x + self.width) + + @property + def bottom_left(self): + """ + Convenience property to return the bottom left point of the ROI + :return: bottom left point of the ROI + """ + return (self.x, self.y + self.height) + + @property + def bottom_right(self): + """ + Convenience property to return the bottom right point of the ROI + :return: bottom right point of the ROI + """ + return (self.x + self.width, self.y + self.height) + + @staticmethod + def from_frame(frame, side=SIDE.TOP, percent=100): + """ + Calculate a Roi based on a percentage of a source frame. + The user can set the side of the frame where the ROI is set. + :param frame: source frame to get the size of the ROI. Opencv frame is expected. + :param side: Predefined Side form the Side class specifying the side of the frame. + :param percent: Percentage of the frame that the ROI will cover. + :return: ROI based on the size of the frame + """ + assert isinstance(side, int) and 0 <= side < 5, "Side must be a value between 0 and 3 but %d given. Use the SIDE class to get valid values." % side + assert isinstance(percent, (int, float)) , "Percent must be must be Integer or Float but %r given" % percent + assert (0 < percent <= 100), "Percent must be a value between 0 and 100 but %d given" % percent + frame_height, frame_width = frame.shape[:2] + # calculating percentage of the frame width and height + frame_width_percent = frame_width * percent / 100 + frame_height_percent = frame_height * percent / 100 + if side == SIDE.TOP: + x, y = (0,0) + width, height = (frame_width, frame_height_percent) + elif side == SIDE.BOTTOM: + x, y = (0, frame_height-frame_height_percent) + width, height = (frame_width, frame_height_percent) + elif side == SIDE.RIGHT: + x, y = (frame_width-frame_width_percent, 0) + width, height = (frame_width_percent, frame_height) + elif side == SIDE.LEFT: + x, y = (0, 0) + width, height = (frame_width_percent, frame_height) + elif side == SIDE.CENTER: + # TODO: End implementation + x = int(frame_width/2) -int(frame_width_percent/2) + y = int(frame_height/2) - int(frame_height_percent/2) + width, height = (frame_width_percent, frame_height_percent) + new_roi = Roi([x, y, width, height]) + return new_roi + + def limit_to_roi(self, other): + """ + Method to limit the current ROI to another given + :param other: ROI setting the limits for the current one. + :return: None + """ + a, b = self, other + x1 = max(min(a.x1, a.x2), min(b.x1, b.x2)) + y1 = max(min(a.y1, a.y2), min(b.y1, b.y2)) + x2 = min(max(a.x1, a.x2), max(b.x1, b.x2)) + y2 = min(max(a.y1, a.y2), max(b.y1, b.y2)) + if x1 < x2 and y1 < y2: + width = abs(x2 - x1) + height = abs(y2 - y1) + self.x = x1 + self.y = y1 + self.width = width + self.height = height + + def apply_to_frame_as_mask(self, frame): + """ + + :param frame: Apply the current ROI to the frame as a mask setting to black all the information outside the ROI. + :return: + """ + # TODO: check if a copy of the frame is needed. + mask = np.zeros(frame.shape, dtype='uint8') + mask[self.y:self.y + self.height, self.x:self.x + self.width] = 255 + roied_frame = cv2.bitwise_and(frame, mask) + return roied_frame + + def extract_from_frame(self, frame): + """ + Extract form the imput frame the image correspoding with the current ROI + :param frame: source frame to extract the ROI + :return: Part of the frame containded on the current ROI + """ + return frame[self.y:self.y + self.height, self.x:self.x + self.width] + + def draw_on_frame(self, frame, color = [255, 255, 255], copy = True): + """ + Draw a retangle on the input frame with the color specified and anotate the initial coords of the frame. + :param frame: frame to be overlayed by the drawn ROI + :param color: Color to be set for the drawn ROI rectangle + :param copy: Determine if a copy of the frame is needed or not. + :return: frame with the ROI drawn on it. + """ + if copy: + new_frame = frame.copy() + else: + new_frame = frame + cv2.rectangle(new_frame, self.top_left, self.bottom_right, color) + cv2.circle(new_frame, self.top_left,3,color,3,1) + font = cv2.FONT_HERSHEY_SIMPLEX + fontScale = 0.3 + lineType = 1 + cv2.putText(new_frame, str(self.top_left), + self.top_left, + font, + fontScale, + color, + lineType) + return new_frame + + def intersection_rate(self, roi2): + """ + Return a float value representing the intersaction rate for this ROI with roi2 + + TODO: Check the value returned for a full inetersection and none + :param roi2: input roi to calculate the intersection ROI + :return: intersection rate float value + """ + + x1, y1 = self.top_left + x2, y2 = self.bottom_right + s_1 = np.array([[x1, y1], [x2, y1], [x2, y2], [x1, y2]]) + + x1, y1 = roi2.top_left + x2, y2 = roi2.bottom_right + s_2 = np.array([[x1, y1], [x2, y1], [x2, y2], [x1, y2]]) + + area, _intersection = cv2.intersectConvexConvex(s_1, s_2) + return 2 * area / (cv2.contourArea(s_1) + cv2.contourArea(s_2)) + + + + def upscaled(self, limiting_roi, upscaled_pixels): + """ + Create an upscaled version of an input ROI restricted to the size of limiting roi + + :param limiting_roi: Roi that limit the possible upscale of the bounding rect + :param upscaled_pixels: Number of pixels to add to both the height and the with from the center + :return: newroi upscaled + """ + x, y, w, h = self + + new_x = max(max(x - int(upscaled_pixels / 2), limiting_roi.x), 0) + new_y = max(max(y - int(upscaled_pixels / 2), limiting_roi.y), 0) + + # add the needed pixels to the width and height checking the frame limits + if x + w + upscaled_pixels < limiting_roi.x+limiting_roi.width: + new_w = w + upscaled_pixels + else: + new_w = limiting_roi.width + + if y + h + upscaled_pixels < limiting_roi.height: + new_h = h + upscaled_pixels + else: + new_h = limiting_roi.height + upscaled_roi = Roi([new_x, new_y, new_w, new_h]) + return upscaled_roi + + # def downscaled(self, limiting_roi, downscaled_pixels): + # """ + # Create an downscaled version of an input bounding rect restricted to the size of a frame + # + # :param limiting_roi: frame that limit the possible downscale of the bounding rect + # :param downscaled_pixels: Number of pixels to substracted to both the height and the with from the center + # :return: + # """ + # x, y, w, h = self + # new_x = min(x + int(downscaled_pixels / 2), limiting_roi[1]) + # + # new_y = min(y + int(downscaled_pixels / 2), limiting_roi[0]) + # + # if w - downscaled_pixels > 0: + # new_w = w - downscaled_pixels + # else: + # new_w = 0 + # + # if h - downscaled_pixels > 0: + # new_h = h + downscaled_pixels + # else: + # new_h = 0 + # downscaled_roi = Roi() + # downscaled_roi = Roi([new_x, new_y, new_w, new_h]) + # return downscaled_roi + + +if __name__ == '__main__': + class Frame: pass + frame = Frame + frame.shape =[640, 480, 3] + r = Roi.from_frame(frame, SIDE.RIGHT, 100) + print(r) + r = Roi.from_frame(frame, SIDE.LEFT, 100) + print(r) + r = Roi.from_frame(frame, SIDE.TOP, 100) + print(r) + r = Roi.from_frame(frame, SIDE.BOTTOM, 100) + print(r) + print("-----------") + r = Roi.from_frame(frame, SIDE.RIGHT, 90) + print(r) + r = Roi.from_frame(frame, SIDE.LEFT, 90) + print(r) + r = Roi.from_frame(frame, SIDE.TOP, 90) + print(r) + r = Roi.from_frame(frame, SIDE.BOTTOM, 90) + print(r) + print("-----------") + r = Roi.from_frame(frame, SIDE.RIGHT, 10) + print(r) + r = Roi.from_frame(frame, SIDE.LEFT, 10) + print(r) + r = Roi.from_frame(frame, SIDE.TOP, 10) + print(r) + r = Roi.from_frame(frame, SIDE.BOTTOM, 10) + print(r) \ No newline at end of file diff --git a/test_hand.py b/test_hand.py new file mode 100644 index 0000000..2539b29 --- /dev/null +++ b/test_hand.py @@ -0,0 +1,68 @@ +import os +import unittest + +import cv2 + +from HandDetection.Hand import Hand +from HandDetection.rgbdframe import RGBDFrame +from HandDetection.roi import Roi, SIDE + + +class Frame: pass +frame = Frame +frame.shape = [640, 480, 3] + +class TestRoi(unittest.TestCase): + def test_hand_detect(self): + """ + Test that check the creation of roi from frames + """ + hand = Hand() + hand.depth_threshold = 130 + expected_results = { + "20190725130248.png": [False, False], + "20190725130249.png": [False, False], + "20190725130250.png": [True, True], + "20190725130251.png": [False, True], # fingers too close and it's considered not a hand + "20190725130252.png": [True, True], + "20190725130253.png": [True, True], + "20190725130254.png": [False, True], + "20190725130255.png": [False, True], + "20190725130256.png": [False, True], + "20190725130257.png": [False, True], + "20190725130258.png": [False, True] + + + } + full_path = "/home/robolab/robocomp/components/robocomp-robolab/components/handDetection/src/images/depth_images" + for file in sorted(os.listdir(full_path)): + if file.endswith(".png") and file in expected_results: + frame = RGBDFrame(cv2.imread(os.path.join(full_path, file),0)) + hand.initial_roi = Roi.from_frame(frame, SIDE.CENTER, 50) + hand.detect_and_track(frame) + print("testing file %s"%file) + self.assertEqual(hand.detected , expected_results[file][0]) + self.assertEqual(hand.tracked, expected_results[file][1]) + frame = self.draw_in_frame(hand, frame) + cv2.imshow("final", frame) + key = cv2.waitKey(5000) + if key == 112: + while cv2.waitKey(1000) != 112: + pass + + + def draw_in_frame(self, hand, frame): + frame = cv2.cvtColor(frame, cv2.COLOR_GRAY2RGB) + frame = hand.initial_roi.draw_on_frame(frame,[0,0,255]) + frame = hand.detection_roi.draw_on_frame(frame, [0, 255, 0]) + frame = hand.tracking_roi.draw_on_frame(frame, [255, 0, 0]) + # frame = hand._extended_roi.draw_on_frame(frame, [0, 0, 255]) + return frame + # def test_roi_upscale(self): + # """ + # Test that roi upscale right + # """ + # self.assertEqual(r, [0, 480 - 48, 640, 48]) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/test_roi.py b/test_roi.py new file mode 100644 index 0000000..bd4bf24 --- /dev/null +++ b/test_roi.py @@ -0,0 +1,100 @@ +import unittest + +from roi import Roi, SIDE + + +class Frame: pass + + +frame = Frame +frame.shape = [640, 480, 3] + + +class TestRoi(unittest.TestCase): + + def test_roi_from_frame_top(self): + """ + Test that check the creation of roi from frames + """ + + + r = Roi.from_frame(frame, SIDE.TOP, 100) + self.assertEqual(r, [0, 0, 640, 480]) + r = Roi.from_frame(frame, SIDE.TOP, 90) + self.assertEqual(r, [0, 0, 640, 480-48]) + r = Roi.from_frame(frame, SIDE.TOP, 10) + self.assertEqual(r, [0, 0, 640, 48]) + + + def test_roi_from_frame_bottom(self): + + r = Roi.from_frame(frame, SIDE.BOTTOM, 100) + self.assertEqual(r, [0, 0, 640, 480]) + r = Roi.from_frame(frame, SIDE.BOTTOM, 90) + self.assertEqual(r, [0, 48, 640, 480-48]) + r = Roi.from_frame(frame, SIDE.BOTTOM, 10) + self.assertEqual(r, [0, 480-48, 640, 48]) + + + def test_roi_from_frame_left(self): + + r = Roi.from_frame(frame, SIDE.LEFT, 100) + self.assertEqual(r, [0, 0, 640, 480]) + r = Roi.from_frame(frame, SIDE.LEFT, 90) + self.assertEqual(r, [0, 0, 640-64, 480]) + r = Roi.from_frame(frame, SIDE.LEFT, 10) + self.assertEqual(r, [0, 0, 64, 480]) + + + def test_roi_from_frame_right(self): + + r = Roi.from_frame(frame, SIDE.RIGHT, 100) + self.assertEqual(r, [0, 0, 640, 480]) + + r = Roi.from_frame(frame, SIDE.RIGHT, 90) + self.assertEqual(r, [64, 0, 640 - 64, 480]) + + r = Roi.from_frame(frame, SIDE.RIGHT, 10) + self.assertEqual(r, [640 - 64, 0, 64, 480]) + + + def test_roi_from_frame_center(self): + + r = Roi.from_frame(frame, SIDE.CENTER, 100) + self.assertEqual(r, [0, 0, 640, 480]) + r = Roi.from_frame(frame, SIDE.CENTER, 90) + self.assertEqual(r, [(64/2), (48/2), 640 - 64, 480-48]) + r = Roi.from_frame(frame, SIDE.CENTER, 10) + self.assertEqual(r, [640/2 - 64/2, 480/2 - 48/2, 64, 48]) + + + def test_roi_upscale(self): + """ + Test that roi upscale right + """ + roi = Roi([0,0,640,480]) + limit = Roi([0,0, 640,480]) + a = roi.upscaled(limit, 10) + self.assertEqual(a, [0, 0, 640, 480]) + + roi = Roi([0, 0, 320, 240]) + a = roi.upscaled(limit, 10) + self.assertEqual(a, [0, 0, 330, 250]) + + roi = Roi([10, 10, 320, 240]) + a = roi.upscaled(limit, 10) + self.assertEqual(a, [5, 5, 330, 250]) + + roi = Roi([40, 40, 50, 50]) + limit = Roi([30, 30, 60, 60]) + a = roi.upscaled(limit, 10) + self.assertEqual(a, [35, 35, 60, 60]) + + # def test_roi_upscale(self): + # """ + # Test that roi upscale right + # """ + # self.assertEqual(r, [0, 480 - 48, 640, 48]) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file