Implement mixed precision

.gitignore

@@ -134,3 +134,4 @@ scripts/log/
 src/data
 src/othermodels/*.py
 src/othermodels/state_dicts/*.pt
+goldeneye.sif

src/goldeneye.py

@@ -1,11 +1,14 @@
 import random, sys
+import string
 import logging
 import numpy as np
 import torch
 import csv
 
 # sys.path.append("./pytorchfi")
-from pytorchfi import core
+from ..pytorchfi.pytorchfi import core
+from .num_sys_class import *
+
 # from num_sys_class import *
 
 
@@ -40,8 +43,10 @@ def __init__(
         self.precision = precision
 
         # for simulated number system
-        (self.num_sys, self.num_sys_name) = num_sys
-        # self.num_sys = num_sys
+
+        # Processing input
+        self.process_numsys_input(num_sys)
+
         self.signed = kwargs.get("signed", True)
 
         # for quantization
@@ -52,6 +57,24 @@ def __init__(
         # the order of injecting within the goldeneye transformation
         # 0 -> no injection, 1 -> between quantization and de-quantization, 2 -> after converting to the number system, 3 -> after dequantization, 4 -> after converting num sys
 
+    def process_numsys_input(self, num_sys):
+        # Options:
+        # 1. num_sys is a string
+        # 2. num_sys is a tuple (numsys_obj, name)
+        # 3. num_sys is a list of configurations or strings(eg. [("fp", 8, 23), "bfloat16",("fxp", 5, 10)])
+
+        if type(num_sys) is str:
+            (self.cur_num_sys, self.cur_num_sys_name) = string_to_numsys(num_sys)
+            self.num_sys = num_sys
+        elif type(num_sys) is tuple:  # i.e one numsys_obj, name tuple
+            (self.cur_num_sys, self.cur_num_sys_name) = num_sys
+            self.num_sys = num_sys
+        else:  # type(num_sys) is list:
+            assert type(num_sys) is list
+
+            self.num_sys = to_numsys_list(num_sys)
+            (self.cur_num_sys, self.cur_num_sys_name) = self.num_sys[0]
+
     def set_layer_max(self, data):
         self.LayerRanges = data
 
@@ -77,7 +100,7 @@ def _twos_comp(self, val, bits):
     def hook1_num_sys_inj1(self, in_num, bit_pos, to_inj):
         # Transform to num sys and bit flip if inj_order == 1
         flip_val = to_inj and (self.inj_order == 1 or not self.quant)
-        out_num = self.num_sys.convert_numsys_flip(in_num, bit_pos, flip=flip_val)
+        out_num = self.cur_num_sys.convert_numsys_flip(in_num, bit_pos, flip=flip_val)
         return out_num
 
     def hook2_quant(self, in_num, max_value):
@@ -142,7 +165,7 @@ def hook4_dequant(self, in_num, max_value):
 
     def hook5_num_sys_inj3(self, in_num, bit_pos, to_inj):
         # TODO only enter here if inj_order is 3. But check logic if this is the only time! Min: set flip = False
-        return self.num_sys.convert_numsys_flip(
+        return self.cur_num_sys.convert_numsys_flip(
             in_num, bit_pos, flip=(to_inj and (self.quant and self.inj_order == 6))
         )
 
@@ -203,13 +226,27 @@ def quantizeUnsigned(self, X, B, R=2.0):
         S = 2.0 / R
 
         if getMode == "FP32":
-            return 0.5 * R * torch.min(torch.pow(torch.tensor(2.0).cuda(), 1.0 - B) * torch.round(
-                X * S * torch.pow(torch.tensor(2.0).cuda(), B - 1.0)),
-                                       2.0 - torch.pow(torch.tensor(2.0).cuda(), 1.0 - B))
+            return (
+                0.5
+                * R
+                * torch.min(
+                    torch.pow(torch.tensor(2.0).cuda(), 1.0 - B)
+                    * torch.round(X * S * torch.pow(torch.tensor(2.0).cuda(), B - 1.0)),
+                    2.0 - torch.pow(torch.tensor(2.0).cuda(), 1.0 - B),
+                )
+            )
         elif getMode == "FP16":
-            return 0.5 * R * torch.min(torch.pow(torch.tensor(2.0).cuda().half(), 1.0 - B) * torch.round(
-                X * S * torch.pow(torch.tensor(2.0).cuda().half(), B - 1.0)),
-                                       2.0 - torch.pow(torch.tensor(2.0).cuda().half(), 1.0 - B))
+            return (
+                0.5
+                * R
+                * torch.min(
+                    torch.pow(torch.tensor(2.0).cuda().half(), 1.0 - B)
+                    * torch.round(
+                        X * S * torch.pow(torch.tensor(2.0).cuda().half(), B - 1.0)
+                    ),
+                    2.0 - torch.pow(torch.tensor(2.0).cuda().half(), 1.0 - B),
+                )
+            )
 
     # operates on a single value
     def _flip_bit_goldeneye(self, orig_value, max_value, bit_pos=-1, to_inj=False):
@@ -233,38 +270,49 @@ def _flip_bit_goldeneye(self, orig_value, max_value, bit_pos=-1, to_inj=False):
                     return torch.tensor(max_value)
 
         # Quantization (num_sys to int8)
-        if self.num_sys_name == "INT":
+        if self.cur_num_sys_name == "INT":
             print("QUANTIZATION!")
             out_num = self.hook2_quant(out_num, max_value)
-            if torch.isnan(torch.tensor(out_num)) or torch.isinf(torch.tensor(out_num)): return torch.tensor(out_num)
+            if torch.isnan(torch.tensor(out_num)) or torch.isinf(torch.tensor(out_num)):
+                return torch.tensor(out_num)
 
             # Bit flip if inj_order == 2
             if to_inj == 1:
                 out_num = self.hook3_inj2(out_num, bit_pos)
-            if torch.isnan(torch.tensor(out_num)) or torch.isinf(torch.tensor(out_num)): return torch.tensor(out_num)
+            if torch.isnan(torch.tensor(out_num)) or torch.isinf(torch.tensor(out_num)):
+                return torch.tensor(out_num)
 
             # Dequant
             out_num = self.hook4_dequant(out_num, max_value)
-            if torch.isnan(torch.tensor(out_num)) or torch.isinf(torch.tensor(out_num)): return torch.tensor(out_num)
+            if torch.isnan(torch.tensor(out_num)) or torch.isinf(torch.tensor(out_num)):
+                return torch.tensor(out_num)
 
         # Num sys conversion + flip if inj_order = 3
-        if to_inj == 6: # TODO Future datapath bitflip
+        if to_inj == 6:  # TODO Future datapath bitflip
             print("HOOK 6!")
             out_num = self.hook5_num_sys_inj3(out_num, bit_pos, to_inj)
-            if torch.isnan(torch.tensor(out_num)) or torch.isinf(torch.tensor(out_num)): return torch.tensor(out_num)
+            if torch.isnan(torch.tensor(out_num)) or torch.isinf(torch.tensor(out_num)):
+                return torch.tensor(out_num)
 
         # return torch.tensor(out_num, dtype=save_type)
         return torch.tensor(out_num)
 
     def apply_goldeneye_transformation(self, module, input, output):
         corrupt_layer_set = self.get_corrupt_layer()
-        range_max = self.get_layer_max(self.get_current_layer())
-        logging.info("curr_conv", self.get_current_layer())
+        range_max = self.get_layer_max(self.get_curr_layer())
+
+        # Setting current num-sys for mixed-precision
+        if type(self.num_sys) is list:
+            (self.cur_num_sys, self.cur_num_sys_name) = self.num_sys[
+                self.get_curr_layer()
+            ]
+
+        logging.info("curr_conv", self.get_curr_layer())
         logging.info("range_max", range_max)
 
         inj_list = list(
             filter(
-                lambda x: corrupt_layer_set[x] == self.get_current_layer(),
+                lambda x: corrupt_layer_set[x] == self.get_curr_layer(),
                 range(len(corrupt_layer_set)),
             )
         )
@@ -282,10 +330,10 @@ def apply_goldeneye_transformation(self, module, input, output):
                     self.corrupt_dim3[i],
                 )
 
-                if self.num_sys_name == "block_fp" and self.inj_order == 1:
+                if self.cur_num_sys_name == "block_fp" and self.inj_order == 1:
                     # only select mantissa or sign bit
-                    rand_bit = random.randint(0, self.num_sys.mant_len)
-                    if rand_bit == self.num_sys.mant_len:
+                    rand_bit = random.randint(0, self.cur_num_sys.mant_len)
+                    if rand_bit == self.cur_num_sys.mant_len:
                         rand_bit = self.bits - 1
                 else:
                     rand_bit = self.bits - 1 - random.randint(0, self.bits - 1)
@@ -304,7 +352,7 @@ def apply_goldeneye_transformation(self, module, input, output):
                 )
 
         # meta injections (tensor level)
-        if self.get_current_layer() in corrupt_layer_set and self.inj_order == 2:
+        if self.get_curr_layer() in corrupt_layer_set and self.inj_order == 2:
             # print("META INJECTION")
             meta_inj_en = True
         else:
@@ -313,8 +361,10 @@ def apply_goldeneye_transformation(self, module, input, output):
         # tensor conversions (FP32 -> Num Sys)
         # number system emulation (with meta injection support)
         if self.quant is False:
-            output[:] = self.num_sys.convert_numsys_tensor(output, meta_inj=meta_inj_en)
-            # output = self.num_sys.convert_numsys_tensor(output, meta_inj=meta_inj_en)
+            output[:] = self.cur_num_sys.convert_numsys_tensor(
+                output, meta_inj=meta_inj_en
+            )
+            # output = self.cur_num_sys.convert_numsys_tensor(output, meta_inj=meta_inj_en)
             torch.cuda.empty_cache()
         else:
             # quantize (scale and quant NumSys)
@@ -325,10 +375,8 @@ def apply_goldeneye_transformation(self, module, input, output):
 
         # always do this before proceeding
         self.updateLayer()
-        if self.get_current_layer() >= self.get_total_layers():
-            self.reset_current_layer()
-
-
+        if self.get_curr_layer() >= self.get_total_layers():
+            self.reset_curr_layer()
 
         # baseDevice = output.get_device()
         # TO OPTIMIZE (??). Must move to CPU, then back to_device
@@ -342,8 +390,8 @@ def apply_goldeneye_transformation(self, module, input, output):
         # )
 
         # apply is too slow, we replaced it by tensor-operations-based functions
-        # print("Layer: ", self.get_current_layer(), ", Shape: ", output.dim(), ", Size: ", output.size())
-        # print("Layer: ", self.get_current_layer(), "Value: ", output[-1][24][12][12])
+        # print("Layer: ", self.get_curr_layer(), ", Shape: ", output.dim(), ", Size: ", output.size())
+        # print("Layer: ", self.get_curr_layer(), "Value: ", output[-1][24][12][12])
         # print("New Value:", output[-1][24][12][12])
 
         # if self.use_cuda: