Differnet chenggui

Readme.md

@@ -20,14 +20,49 @@ Install packages with:
 $ pip install -r requirements.txt
 ```
 
+Or install with for Windows as per [PyTorch official site](https://pytorch.org/get-started/locally/):
+
+```
+$ pip install torch===1.6.0 torchvision===0.7.0 -f https://download.pytorch.org/whl/torch_s
+table.html
+$ pip install -r requirements.txt
+```
+
 ## Configure and Run
 
 All configurations concerning data, model, training, visualization etc. can be made in _config.py_. The default configuration will run a training with paper-given parameters on the provided dummy dataset. This dataset contains images of 4 squares as normal examples and 4 circles as anomaly.
 
-To start the training, just run _main.py_! If training on the dummy data does not lead to an AUROC of 1.0, something seems to be wrong.
+If you encounter GPU Out of Memory issue, you can reduce the neuron numbers in _config.py_
+```
+fc_internal = 1536 # number of neurons in hidden layers of s-t-networks
+```
+
+To start the training, just run _main.py_ as follows! If training on the dummy data does not lead to an AUROC of 1.0, something seems to be wrong.
 Please report us if you have issues when using the code.
 
+```
+$ python main.py
+```
+
 ## Data
+How to use Data extraction tool to extract data from video clips:
+ 1. Create folder structure like the example shows in the picture below.
+
+  ![1](https://github.com/zerobox-ai/differnet/blob/zijian/dataset/data-generation/annotations/structure1.png)
+
+ 2. Dump the videos and annotations (rename them use 1.xml, 1.avi as one pair annotation and video) into the folders under data-generation folder.
+
+  ![2](https://github.com/zerobox-ai/differnet/blob/zijian/dataset/data-generation/annotations/structure2.png)
+
+ 3. Modify the annotation files: Since the annotation uses label "defect" to indicate the defect area, while, both good and defective bottles are labeled as "bottle" which is confusing. To indicate which "bottle" is defective, we need to find the frames that labeled with defect, and then manully update the group's label from "bottle" to "defective" for the groups that falling in to those frames. 
+  ![3](https://github.com/zerobox-ai/differnet/blob/zijian/dataset/data-generation/annotations/structure3.png)
+
+ - For example: in the example image above, the frame 15 and 16 are labeled as "defect" which indicates those 2 frames has defect areas on the bottles. So we need to find the group that contains frame 15 and 16, and then manully update the label from "bottle" to "defective". and then delete the whole \<track\> group that labeled as "defect" (since we don't care about the defect area in data extraction).
+
+ 4. Modify the config.py, fill in appropriate value for num_videos, save_cropped_image_to and save_original_image_to
+
+ 5. run the data extraction: python data_extraction.py
+
 
 The given dummy dataset shows how the implementation expects the construction of a dataset. Coincidentally, the [MVTec AD dataset](https://www.mvtec.com/de/unternehmen/forschung/datasets/mvtec-ad/) is constructed in this way.
 

apply_mask.py

@@ -0,0 +1,22 @@
+import cv2
+import os
+
+
+def load_images_from_folder(folder):
+    images = []
+    for filename in os.listdir(folder):
+        img = cv2.imread(os.path.join(folder, filename))
+        if img is not None:
+            images.append(img)
+    return images
+
+
+path = 'dataset/Experiment_4.1/validate/good'
+mask = cv2.imread(os.path.join('dataset/Mask/', 'Mask_shrink.jpg'))
+mask = mask / 255  # make the mask into 0/1 matrix for multiplication
+imgs = load_images_from_folder(path)
+
+for i, img in enumerate(imgs):
+    img = cv2.resize(img, (400, 700), interpolation=cv2.INTER_AREA)
+    masked_img = img * mask
+    cv2.imwrite('dataset/Experiment_5.1/validate/good/good-Masked-' + str(i) + '.jpg', masked_img)

config.py

@@ -1,49 +1,65 @@
-'''This file configures the training procedure because handling arguments in every single function is so exhaustive for
-research purposes. Don't try this code if you are a software engineer.'''
-
-# device settings
-device = 'cuda' # or 'cpu'
-import torch
-torch.cuda.set_device(0)
-
-# data settings
-dataset_path = "dummy_dataset"
-class_name = "dummy_class"
-modelname = "dummy_test"
-
-img_size = (448, 448)
-img_dims = [3] + list(img_size)
-add_img_noise = 0.01
-
-# transformation settings
-transf_rotations = True
-transf_brightness = 0.0
-transf_contrast = 0.0
-transf_saturation = 0.0
-norm_mean, norm_std = [0.485, 0.456, 0.406], [0.229, 0.224, 0.225]
-
-# network hyperparameters
-n_scales = 3 # number of scales at which features are extracted, img_size is the highest - others are //2, //4,...
-clamp_alpha = 3 # see paper equation 2 for explanation
-n_coupling_blocks = 8
-fc_internal = 2048 # number of neurons in hidden layers of s-t-networks
-dropout = 0.0 # dropout in s-t-networks
-lr_init = 2e-4
-n_feat = 256 * n_scales # do not change except you change the feature extractor
-
-# dataloader parameters
-n_transforms = 4 # number of transformations per sample in training
-n_transforms_test = 64 # number of transformations per sample in testing
-batch_size = 24 # actual batch size is this value multiplied by n_transforms(_test)
-batch_size_test = batch_size * n_transforms // n_transforms_test
-
-# total epochs = meta_epochs * sub_epochs
-# evaluation after <sub_epochs> epochs
-meta_epochs = 1
-sub_epochs = 8
-
-# output settings
-verbose = True
-grad_map_viz = True
-hide_tqdm_bar = True
-save_model = True
+'''This file configures the training procedure because handling arguments in every single function is so exhaustive for
+research purposes. Don't try this code if you are a software engineer.'''
+
+# data extraction settings
+num_videos = 21
+save_cropped_image_to = "dataset/zerobox-2010-1/"
+save_original_image_to = "dataset/zerobox-2010-1-original/"
+
+# device settings
+device = 'cuda' # 'cuda' or 'cpu'
+import torch
+torch.cuda.set_device(0)
+
+# data settings
+dataset_path = "dataset"
+class_name = "Experiment_6.1"
+modelname = "Experiment_6.1_10epoch_239tainingdata_0.5BCS"
+
+img_size = (448, 448)
+img_dims = [3] + list(img_size)
+add_img_noise = 0.01
+
+# transformation settings
+transf_rotations = True
+transf_brightness = 0.5
+transf_contrast = 0.5
+transf_saturation = 0.5
+norm_mean, norm_std = [0.485, 0.456, 0.406], [0.229, 0.224, 0.225]
+
+rotation_degree = 0
+crop_top = 0.10
+crop_left = 0.10
+crop_bottom = 0.10
+crop_right = 0.10
+
+# network hyperparameters
+n_scales = 3 # number of scales at which features are extracted, img_size is the highest - others are //2, //4,...
+clamp_alpha = 3 # see paper equation 2 for explanation
+n_coupling_blocks = 8
+# fc_internal = 2048 # number of neurons in hidden layers of s-t-networks
+fc_internal = 1536  # number of neurons in hidden layers of s-t-networks
+dropout = 0.0 # dropout in s-t-networks
+lr_init = 2e-4
+n_feat = 256 * n_scales # do not change except you change the feature extractor
+
+# dataloader parameters
+n_transforms = 4 # number of transformations per sample in training
+n_transforms_test = 1 # number of transformations per sample in testing
+batch_size = 4 # actual batch size is this value multiplied by n_transforms(_test)
+batch_size_test = 1
+
+# total epochs = meta_epochs * sub_epochs
+# evaluation after <sub_epochs> epochs
+meta_epochs = 5
+sub_epochs = 8
+
+# output settings
+verbose = True
+grad_map_viz = True
+hide_tqdm_bar = True
+save_model = True
+save_transformed_image = True
+visualization = False
+frame_name_is_given = False
+target_tpr = 0.85

data_extraction.py

@@ -0,0 +1,74 @@
+import cv2
+from xml.dom import minidom
+import config as c
+
+# Load videos one by one
+for i in range(c.num_videos):
+    print('Data generation on video-' + str(i+1))
+    filename = str(i+1)
+
+    # Opens the Video file
+    cap = cv2.VideoCapture('dataset/data-generation/videos/' + filename + '.avi')
+
+    # Read annotations
+    annotation = minidom.parse('dataset/data-generation/annotations/' + filename + '.xml')
+    boxes = annotation.getElementsByTagName('box')
+
+    frameList = []
+    labelList = []
+    boxesList = []
+
+    # Store the bounding box info along with frame number info into list
+    for i in range(boxes.length):
+
+        # make sure not select the bounding box that outside the frame
+        if (boxes[i].attributes['outside'].value != '1'):
+            frame = int(boxes[i].attributes['frame'].value)
+            frameList.append(frame)
+
+            labelList.append(boxes[i].parentNode.attributes['label'].value)
+
+            ytl = int(float(boxes[i].attributes['ytl'].value))
+            ybr = int(float(boxes[i].attributes['ybr'].value))
+            xtl = int(float(boxes[i].attributes['xtl'].value))
+            xbr = int(float(boxes[i].attributes['xbr'].value))
+            boxesList.append([ytl, ybr, xtl, xbr])
+
+    # Set up shrink percentage
+    shrink_percentage = 0.02
+    j = 0
+    while(cap.isOpened()):
+        ret, frame = cap.read()
+        if(frame is not None and j in frameList):
+            ytl = boxesList[frameList.index(j)][0]
+            ybr = boxesList[frameList.index(j)][1]
+            xtl = boxesList[frameList.index(j)][2]
+            xbr = boxesList[frameList.index(j)][3]
+            label = 'good' if labelList[frameList.index(j)] == 'bottle' else 'defect'
+
+            # draw bounding box on original frames
+            if label != 'defect':
+                cv2.rectangle(frame, (xtl, ytl), (xbr, ybr), (0, 255, 0), 5)
+            else:
+                cv2.rectangle(frame, (xtl, ytl), (xbr, ybr), (0, 0, 255), 5)
+                #cv2.imshow("Show", frame)
+                #cv2.waitKey()
+                #cv2.destroyAllWindows()
+            # Crop the frames with the bounding box position info
+            crop_frame = frame[int(ytl*(1+shrink_percentage)):int(ybr*(1-shrink_percentage)),
+                         int(xtl*(1+shrink_percentage)):int(xbr*(1-shrink_percentage))]
+
+            # output file formatting example "video1-frame4-defect.jpg"
+            print('Successfully generated: ' +c.save_cropped_image_to + label + '/video-' + filename + '-frame' + str(j) +
+                  '-' + label + '.jpg')
+            cv2.imwrite(c.save_original_image_to + label + '/original-video-' + filename + '-frame' + str(j) + '-' + label + '.jpg',
+                        frame)
+            cv2.imwrite(c.save_cropped_image_to + label + '/video-' + filename + '-frame' + str(j) + '-' + label + '.jpg',
+                        crop_frame)
+
+        if ret == False:
+            break
+        j += 1
+
+    cap.release()
+    cv2.destroyAllWindows()