Draft: Managed table testing

nise/__init__.py

@@ -1,4 +1,3 @@
 __version__ = "5.0.3"
 
-
 VERSION = __version__.split(".")

nise/generators/azure/azure_generator.py

@@ -23,6 +23,7 @@
 from random import choice
 from random import randint
 from random import uniform
+from nise.util import NUMBER_OF_REPLICAS, pseudo_random_uuid
 
 from nise.generators.generator import AbstractGenerator
 
@@ -526,8 +527,30 @@ def _generate_daily_data(self):
             start = day.get("start")
             end = day.get("end")
             row = self._init_data_row(start, end)
-            row = self._update_data(row, start, end)
-            data.append(row)
+            for i in range(NUMBER_OF_REPLICAS):
+                id_suffix = tag_suffix = ""
+                if NUMBER_OF_REPLICAS > 1:
+                    id_suffix = f"_{pseudo_random_uuid(i)}"
+                    tag_suffix = id_suffix.split("-")[0]
+
+                row = self._update_data(row, start, end)
+                if self.azure_columns is AZURE_COLUMNS_V2_SUBSCRIPTION:
+                    row["ResourceId"] += id_suffix
+                else:
+                    row["InstanceName"] += id_suffix
+
+                row["ResourceName"] += id_suffix
+
+                if azure_tags := json.loads(row.get("Tags")):
+                    for tag_key in azure_tags:
+                        if tag_key in ["openshift_node", "openshift_project"]:
+                            azure_tags[tag_key] += id_suffix
+                        elif tag_key == "managed_tables_matching":
+                            azure_tags[tag_key] += tag_suffix
+                    azure_tags = json.dumps(azure_tags)
+                    row["Tags"] = azure_tags
+                data.append(row.copy())
+
         return data
 
     def generate_data(self, report_type=None):

nise/generators/gcp/gcp_generator.py

@@ -24,6 +24,7 @@
 from random import choice
 from random import randint
 from random import uniform
+from nise.util import NUMBER_OF_REPLICAS, pseudo_random_uuid
 
 from nise.generators.generator import AbstractGenerator
 
@@ -311,6 +312,46 @@ def _generate_hourly_data(self, **kwargs):
         for hour in self.hours:
             start = hour.get("start")
             end = hour.get("end")
-            row = self._init_data_row(start, end)
-            row = self._update_data(row)
-            yield row
+
+            # each line items will be generated {NUMBER_OF_REPLICAS} times
+            for i in range(NUMBER_OF_REPLICAS):
+                row = self._init_data_row(start, end)
+                row = self._update_data(row)
+
+                id_suffix = tag_suffix = ""
+                if NUMBER_OF_REPLICAS > 1:
+                    id_suffix = f"_{pseudo_random_uuid(i)}"
+                    tag_suffix = id_suffix.split("-")[0]
+
+                # csv flow
+                if row.get("resource.global_name"):
+                    row["resource.global_name"] += id_suffix
+                # json flow
+                elif row.get("resource", {}).get("global_name"):
+                    row["resource"]["global_name"] += id_suffix
+
+                # csv flow
+                if row.get("resource.name"):
+                    row["resource.name"] += id_suffix
+                # json flow
+                elif row.get("resource", {}).get("name"):
+                    row["resource"]["name"] += id_suffix
+
+                if gcp_labels := row.get("labels"):
+                    is_label_str = isinstance(gcp_labels, str)  # True - csv flow / False - json flow
+
+                    if is_label_str:
+                        gcp_labels = json.loads(gcp_labels)
+
+                    for tag in gcp_labels:
+                        if tag["key"] in ["openshift_node", "openshift_project"]:
+                            tag["value"] += id_suffix
+                        elif tag["key"] == "managed_tables_matching":
+                            tag["value"] += tag_suffix
+
+                    if is_label_str:
+                        gcp_labels = json.dumps(gcp_labels)
+
+                    row["labels"] = gcp_labels
+
+                yield row

nise/generators/ocp/ocp_generator.py

@@ -17,6 +17,7 @@
 """Defines the abstract generator."""
 
 import datetime
+from nise.util import pseudo_random_uuid
 from copy import deepcopy
 from random import choice
 from random import choices
@@ -27,6 +28,8 @@
 from dateutil import parser
 from nise.generators.generator import AbstractGenerator
 from nise.generators.generator import REPORT_TYPE
+from nise.util import NUMBER_OF_REPLICAS
+
 
 GIGABYTE = 1024 * 1024 * 1024
 HOUR = 60 * 60
@@ -239,7 +242,6 @@ def __init__(self, start_date, end_date, attributes, ros_ocp_info=False, constan
         self.nodes = self._gen_nodes()
         self.namespaces = self._gen_namespaces(self.nodes)
         self.pods, self.namespace2pods, self.ros_data = self._gen_pods(self.namespaces)
-
         self.volumes = self._gen_volumes(self.namespaces, self.namespace2pods)
 
         self.ocp_report_generation = {
@@ -280,21 +282,33 @@ def timestamp(in_date):
 
     def _gen_nodes(self):
         """Create nodes for report."""
+        num_replicas = NUMBER_OF_REPLICAS
         nodes = []
         if self._nodes:
             for item in self._nodes:
-                memory_gig = item.get("memory_gig", randint(2, 8))
-                memory_bytes = memory_gig * GIGABYTE
-                resource_id = str(item.get("resource_id", self.fake.word()))
-                node = {
-                    "name": item.get("node_name", "node_" + self.fake.word()),
-                    "cpu_cores": item.get("cpu_cores", randint(2, 16)),
-                    "memory_bytes": memory_bytes,
-                    "resource_id": "i-" + resource_id,
-                    "namespaces": item.get("namespaces"),
-                    "node_labels": item.get("node_labels"),
-                }
-                nodes.append(node)
+                for i in range(num_replicas):
+                    id_suffix = ""
+                    memory_gig = item.get("memory_gig", randint(2, 8))
+                    memory_bytes = memory_gig * GIGABYTE
+                    resource_id = str(item.get("resource_id", self.fake.word()))
+                    node_name = item.get("node_name", "node_" + self.fake.word())
+                    if num_replicas > 1:
+                        id_suffix = f"_{pseudo_random_uuid(i)}"
+                        resource_id += id_suffix
+                        node_name += id_suffix
+
+                    namespaces = item.get("namespaces")
+                    namespaces_renamed = {f"{project}{id_suffix}": values for project, values in namespaces.items()}
+
+                    node = {
+                        "name": node_name,
+                        "cpu_cores": item.get("cpu_cores", randint(2, 16)),
+                        "memory_bytes": memory_bytes,
+                        "resource_id": "i-" + resource_id,
+                        "namespaces": namespaces_renamed,
+                        "node_labels": item.get("node_labels"),
+                    }
+                    nodes.append(node)
         else:
             num_nodes = randint(2, 6)
             seeded_labels = {"node-role.kubernetes.io/master": [""], "node-role.kubernetes.io/infra": [""]}
@@ -319,12 +333,14 @@ def _gen_namespaces(self, nodes):
                 for name, _ in node.get("namespaces").items():
                     namespace = name
                     namespaces[namespace] = node
+
             else:
                 num_namespaces = randint(2, 12)
                 for _ in range(num_namespaces):
                     namespace_suffix = choice(("ci", "qa", "prod", "proj", "dev", "staging"))
                     namespace = self.fake.word() + "_" + namespace_suffix
                     namespaces[namespace] = node
+
         return namespaces
 
     def _gen_openshift_labels(self, seeding=None):
@@ -369,12 +385,17 @@ def _gen_pods(self, namespaces):
         ros_ocp_data_pods = {}
         namespace2pod = {}
         for namespace, node in namespaces.items():
+            if "_" in node.get("resource_id"):
+                pod_suffix = f"_{node.get('resource_id').split('_')[-1]}"
+                tag_suffix = pod_suffix.split("-")[0]
+            else:
+                pod_suffix = tag_suffix = ""
+
             namespace2pod[namespace] = []
             if node.get("namespaces"):
                 specified_pods = node.get("namespaces").get(namespace).get("pods") or []
                 for specified_pod in specified_pods:
-                    pod = specified_pod.get("pod_name", self.fake.word())
-                    namespace2pod[namespace].append(pod)
+                    pod = f"{specified_pod.get('pod_name', self.fake.word())}{pod_suffix}"
                     cpu_cores = node.get("cpu_cores")
                     memory_bytes = node.get("memory_bytes")
 
@@ -395,6 +416,11 @@ def _gen_pods(self, namespaces):
                         if value > mem_limit_gig:
                             memory_usage_gig[key] = mem_limit_gig
 
+                    pod_labels = specified_pod.get("labels", None)
+                    if pod_labels:
+                        pod_labels = pod_labels.replace(
+                            "label_managed_tables_matching:today", f"label_managed_tables_matching:today{tag_suffix}"
+                        )
                     pods[pod] = {
                         "namespace": namespace,
                         "node": node.get("name"),
@@ -408,7 +434,7 @@ def _gen_pods(self, namespaces):
                         "cpu_limit": cpu_limit,
                         "mem_request_gig": mem_request_gig,
                         "mem_limit_gig": mem_limit_gig,
-                        "pod_labels": specified_pod.get("labels", None),
+                        "pod_labels": pod_labels,
                         "cpu_usage": cpu_usage,
                         "mem_usage_gig": memory_usage_gig,
                         "pod_seconds": specified_pod.get("pod_seconds"),
@@ -547,6 +573,11 @@ def _gen_volumes(self, namespaces, namespace2pods):  # noqa: R0914,C901
         """Create volumes on specific namespaces and keep relationship."""
         volumes = []
         for namespace, node in namespaces.items():
+            if "_" in node.get("resource_id"):
+                pod_suffix = volume_suffix = f"_{node.get('resource_id').split('_')[-1]}"
+                tag_suffix = pod_suffix.split("-")[0]
+            else:
+                pod_suffix = volume_suffix = tag_suffix = ""
             storage_class_default, csi_default = choice(
                 (
                     ("gp3-csi", "ebs.csi.aws.com"),
@@ -558,7 +589,8 @@ def _gen_volumes(self, namespaces, namespace2pods):  # noqa: R0914,C901
             if node.get("namespaces"):
                 specified_volumes = node.get("namespaces").get(namespace).get("volumes", [])
                 for specified_volume in specified_volumes:
-                    volume = specified_volume.get("volume_name", self.fake.word())
+                    volume = f"{specified_volume.get('volume_name', self.fake.word())}{volume_suffix}"
+
                     volume_request_gig = specified_volume.get("volume_request_gig")
                     volume_request = volume_request_gig * GIGABYTE
                     specified_vol_claims = specified_volume.get("volume_claims", [])
@@ -567,8 +599,8 @@ def _gen_volumes(self, namespaces, namespace2pods):  # noqa: R0914,C901
                     for specified_vc in specified_vol_claims:
                         if volume_request - total_claims <= GIGABYTE:
                             break
-                        vol_claim = specified_vc.get("volume_claim_name", self.fake.word())
-                        pod = specified_vc.get("pod_name")
+                        vol_claim = f"{specified_vc.get('volume_claim_name', self.fake.word())}{volume_suffix}"
+                        pod = f"{specified_vc.get('pod_name')}{pod_suffix}"
                         claim_capacity = max(
                             specified_vc.get("capacity_gig") * GIGABYTE, (volume_request_gig * GIGABYTE - total_claims)
                         )
@@ -586,6 +618,12 @@ def _gen_volumes(self, namespaces, namespace2pods):  # noqa: R0914,C901
                             "volume_claim_usage_gig": usage_gig,
                         }
                         total_claims += claim_capacity
+
+                    volume_labels = specified_volume.get("labels", None)
+                    if volume_labels:
+                        volume_labels = volume_labels.replace(
+                            "label_managed_tables_matching:today", f"label_managed_tables_matching:today{tag_suffix}"
+                        )
                     volumes.append(
                         {
                             volume: {
@@ -594,11 +632,11 @@ def _gen_volumes(self, namespaces, namespace2pods):  # noqa: R0914,C901
                                 "volume": volume,
                                 "storage_class": specified_volume.get("storage_class", storage_class_default),
                                 "csi_driver": specified_volume.get("csi_driver", csi_default),
-                                "csi_volume_handle": specified_volume.get(
-                                    "csi_volume_handle", f"vol-{self.fake.word()}"
-                                ),
+                                "csi_volume_handle": f"{
+                                    specified_volume.get('csi_volume_handle', f'vol-{self.fake.word()}')
+                                }{volume_suffix}",
                                 "volume_request": volume_request,
-                                "labels": specified_volume.get("labels", None),
+                                "labels": volume_labels,
                                 "volume_claims": volume_claims,
                             }
                         }

nise/report.py

@@ -88,6 +88,8 @@
 from nise.upload import upload_to_gcp_storage
 from nise.upload import upload_to_s3
 from nise.util import LOG
+from nise.util import pseudo_random_uuid
+from nise.util import NUMBER_OF_REPLICAS
 
 
 def create_temporary_copy(path, temp_file_name, temp_dir_name="None"):
@@ -593,7 +595,6 @@ def aws_create_marketplace_report(options):  # noqa: C901
 
 
 def aws_create_report(options):  # noqa: C901
-    """Create a cost usage report file."""
     start_date = options.get("start_date")
     end_date = options.get("end_date")
     aws_finalize_report = options.get("aws_finalize_report")
@@ -655,9 +656,33 @@ def aws_create_report(options):  # noqa: C901
                 attributes,
                 options.get("aws_tags"),
             )
-            num_instances = 1 if attributes else randint(2, 60)
-            for _ in range(num_instances):
+            # TODO PERF_NOTE: update NUMBER_OF_REPLICAS - how many times you want to multiple the whole yaml file
+            if attributes:
+                num_instances = NUMBER_OF_REPLICAS
+            else:
+                num_instances = randint(2, 60)
+
+            for i in range(num_instances):
+                id_suffix = tag_suffix = ""
+                if attributes and attributes.get("resource_id"):
+                    if num_instances > 1:
+                        id_suffix = f"_{pseudo_random_uuid(i)}"
+                        tag_suffix = id_suffix.split("-")[0]
+                        resource_id = f"i-{attributes.get('resource_id')}{id_suffix}"
+                    else:
+                        resource_id = f"i-{attributes.get('resource_id')}"
+                else:
+                    resource_id = f"i-{fake.ean8()}"
+
                 for hour in gen.generate_data():
+                    if attributes and num_instances > 1 or not attributes:
+                        hour["lineItem/ResourceId"] = resource_id
+                        if orig_node_tag_value := hour.get("resourceTags/user:openshift_node"):
+                            hour["resourceTags/user:openshift_node"] = f"{orig_node_tag_value}{id_suffix}"
+                        if orig_project_tag_value := hour.get("resourceTags/user:openshift_project"):
+                            hour["resourceTags/user:openshift_project"] = f"{orig_project_tag_value}{id_suffix}"
+                        if orig_tag_match_value := hour.get("resourceTags/user:managed_tables_matching"):
+                            hour["resourceTags/user:managed_tables_matching"] = f"{orig_tag_match_value}{tag_suffix}"
                     data += [hour]
                     if len(data) == options.get("row_limit"):
                         file_number += 1

nise/util/__init__.py

@@ -17,13 +17,17 @@
 """Utility functions."""
 
 from collections import abc
-
+import random
+import uuid
 import yaml
 
 from .log import LOG  # noqa: F401
 from .log import LOG_FORMAT  # noqa: F401
 from .log import LOG_VERBOSITY  # noqa: F401
 
+# TODO UPDATE THIS: Num. of times to replicate each resource
+NUMBER_OF_REPLICAS = 10
+
 
 def load_yaml(objekt):
     """Load a yaml document.
@@ -56,3 +60,9 @@ def deepupdate(original, update):
         else:
             original[key] = value
     return original
+
+
+def pseudo_random_uuid(rand_seed):
+    """Generate pseudo-random uuid"""
+    random.seed(rand_seed)
+    return uuid.UUID(int=random.getrandbits(128), version=4)