diff --git a/nise/__init__.py b/nise/__init__.py index a89dbb0e1..334c39d76 100644 --- a/nise/__init__.py +++ b/nise/__init__.py @@ -1,4 +1,3 @@ __version__ = "5.0.3" - VERSION = __version__.split(".") diff --git a/nise/generators/azure/azure_generator.py b/nise/generators/azure/azure_generator.py index 36851eb4b..9a829eee4 100644 --- a/nise/generators/azure/azure_generator.py +++ b/nise/generators/azure/azure_generator.py @@ -23,6 +23,7 @@ from random import choice from random import randint from random import uniform +from nise.util import NUMBER_OF_REPLICAS, pseudo_random_uuid from nise.generators.generator import AbstractGenerator @@ -526,8 +527,30 @@ def _generate_daily_data(self): start = day.get("start") end = day.get("end") row = self._init_data_row(start, end) - row = self._update_data(row, start, end) - data.append(row) + for i in range(NUMBER_OF_REPLICAS): + id_suffix = tag_suffix = "" + if NUMBER_OF_REPLICAS > 1: + id_suffix = f"_{pseudo_random_uuid(i)}" + tag_suffix = id_suffix.split("-")[0] + + row = self._update_data(row, start, end) + if self.azure_columns is AZURE_COLUMNS_V2_SUBSCRIPTION: + row["ResourceId"] += id_suffix + else: + row["InstanceName"] += id_suffix + + row["ResourceName"] += id_suffix + + if azure_tags := json.loads(row.get("Tags")): + for tag_key in azure_tags: + if tag_key in ["openshift_node", "openshift_project"]: + azure_tags[tag_key] += id_suffix + elif tag_key == "managed_tables_matching": + azure_tags[tag_key] += tag_suffix + azure_tags = json.dumps(azure_tags) + row["Tags"] = azure_tags + data.append(row.copy()) + return data def generate_data(self, report_type=None): diff --git a/nise/generators/gcp/gcp_generator.py b/nise/generators/gcp/gcp_generator.py index 7196c1f61..4a5178658 100644 --- a/nise/generators/gcp/gcp_generator.py +++ b/nise/generators/gcp/gcp_generator.py @@ -24,6 +24,7 @@ from random import choice from random import randint from random import uniform +from nise.util import NUMBER_OF_REPLICAS, pseudo_random_uuid from nise.generators.generator import AbstractGenerator @@ -311,6 +312,46 @@ def _generate_hourly_data(self, **kwargs): for hour in self.hours: start = hour.get("start") end = hour.get("end") - row = self._init_data_row(start, end) - row = self._update_data(row) - yield row + + # each line items will be generated {NUMBER_OF_REPLICAS} times + for i in range(NUMBER_OF_REPLICAS): + row = self._init_data_row(start, end) + row = self._update_data(row) + + id_suffix = tag_suffix = "" + if NUMBER_OF_REPLICAS > 1: + id_suffix = f"_{pseudo_random_uuid(i)}" + tag_suffix = id_suffix.split("-")[0] + + # csv flow + if row.get("resource.global_name"): + row["resource.global_name"] += id_suffix + # json flow + elif row.get("resource", {}).get("global_name"): + row["resource"]["global_name"] += id_suffix + + # csv flow + if row.get("resource.name"): + row["resource.name"] += id_suffix + # json flow + elif row.get("resource", {}).get("name"): + row["resource"]["name"] += id_suffix + + if gcp_labels := row.get("labels"): + is_label_str = isinstance(gcp_labels, str) # True - csv flow / False - json flow + + if is_label_str: + gcp_labels = json.loads(gcp_labels) + + for tag in gcp_labels: + if tag["key"] in ["openshift_node", "openshift_project"]: + tag["value"] += id_suffix + elif tag["key"] == "managed_tables_matching": + tag["value"] += tag_suffix + + if is_label_str: + gcp_labels = json.dumps(gcp_labels) + + row["labels"] = gcp_labels + + yield row diff --git a/nise/generators/ocp/ocp_generator.py b/nise/generators/ocp/ocp_generator.py index b7c21a7da..b4cbc1c16 100644 --- a/nise/generators/ocp/ocp_generator.py +++ b/nise/generators/ocp/ocp_generator.py @@ -17,6 +17,7 @@ """Defines the abstract generator.""" import datetime +from nise.util import pseudo_random_uuid from copy import deepcopy from random import choice from random import choices @@ -27,6 +28,8 @@ from dateutil import parser from nise.generators.generator import AbstractGenerator from nise.generators.generator import REPORT_TYPE +from nise.util import NUMBER_OF_REPLICAS + GIGABYTE = 1024 * 1024 * 1024 HOUR = 60 * 60 @@ -239,7 +242,6 @@ def __init__(self, start_date, end_date, attributes, ros_ocp_info=False, constan self.nodes = self._gen_nodes() self.namespaces = self._gen_namespaces(self.nodes) self.pods, self.namespace2pods, self.ros_data = self._gen_pods(self.namespaces) - self.volumes = self._gen_volumes(self.namespaces, self.namespace2pods) self.ocp_report_generation = { @@ -280,21 +282,33 @@ def timestamp(in_date): def _gen_nodes(self): """Create nodes for report.""" + num_replicas = NUMBER_OF_REPLICAS nodes = [] if self._nodes: for item in self._nodes: - memory_gig = item.get("memory_gig", randint(2, 8)) - memory_bytes = memory_gig * GIGABYTE - resource_id = str(item.get("resource_id", self.fake.word())) - node = { - "name": item.get("node_name", "node_" + self.fake.word()), - "cpu_cores": item.get("cpu_cores", randint(2, 16)), - "memory_bytes": memory_bytes, - "resource_id": "i-" + resource_id, - "namespaces": item.get("namespaces"), - "node_labels": item.get("node_labels"), - } - nodes.append(node) + for i in range(num_replicas): + id_suffix = "" + memory_gig = item.get("memory_gig", randint(2, 8)) + memory_bytes = memory_gig * GIGABYTE + resource_id = str(item.get("resource_id", self.fake.word())) + node_name = item.get("node_name", "node_" + self.fake.word()) + if num_replicas > 1: + id_suffix = f"_{pseudo_random_uuid(i)}" + resource_id += id_suffix + node_name += id_suffix + + namespaces = item.get("namespaces") + namespaces_renamed = {f"{project}{id_suffix}": values for project, values in namespaces.items()} + + node = { + "name": node_name, + "cpu_cores": item.get("cpu_cores", randint(2, 16)), + "memory_bytes": memory_bytes, + "resource_id": "i-" + resource_id, + "namespaces": namespaces_renamed, + "node_labels": item.get("node_labels"), + } + nodes.append(node) else: num_nodes = randint(2, 6) seeded_labels = {"node-role.kubernetes.io/master": [""], "node-role.kubernetes.io/infra": [""]} @@ -319,12 +333,14 @@ def _gen_namespaces(self, nodes): for name, _ in node.get("namespaces").items(): namespace = name namespaces[namespace] = node + else: num_namespaces = randint(2, 12) for _ in range(num_namespaces): namespace_suffix = choice(("ci", "qa", "prod", "proj", "dev", "staging")) namespace = self.fake.word() + "_" + namespace_suffix namespaces[namespace] = node + return namespaces def _gen_openshift_labels(self, seeding=None): @@ -369,12 +385,17 @@ def _gen_pods(self, namespaces): ros_ocp_data_pods = {} namespace2pod = {} for namespace, node in namespaces.items(): + if "_" in node.get("resource_id"): + pod_suffix = f"_{node.get('resource_id').split('_')[-1]}" + tag_suffix = pod_suffix.split("-")[0] + else: + pod_suffix = tag_suffix = "" + namespace2pod[namespace] = [] if node.get("namespaces"): specified_pods = node.get("namespaces").get(namespace).get("pods") or [] for specified_pod in specified_pods: - pod = specified_pod.get("pod_name", self.fake.word()) - namespace2pod[namespace].append(pod) + pod = f"{specified_pod.get('pod_name', self.fake.word())}{pod_suffix}" cpu_cores = node.get("cpu_cores") memory_bytes = node.get("memory_bytes") @@ -395,6 +416,11 @@ def _gen_pods(self, namespaces): if value > mem_limit_gig: memory_usage_gig[key] = mem_limit_gig + pod_labels = specified_pod.get("labels", None) + if pod_labels: + pod_labels = pod_labels.replace( + "label_managed_tables_matching:today", f"label_managed_tables_matching:today{tag_suffix}" + ) pods[pod] = { "namespace": namespace, "node": node.get("name"), @@ -408,7 +434,7 @@ def _gen_pods(self, namespaces): "cpu_limit": cpu_limit, "mem_request_gig": mem_request_gig, "mem_limit_gig": mem_limit_gig, - "pod_labels": specified_pod.get("labels", None), + "pod_labels": pod_labels, "cpu_usage": cpu_usage, "mem_usage_gig": memory_usage_gig, "pod_seconds": specified_pod.get("pod_seconds"), @@ -547,6 +573,11 @@ def _gen_volumes(self, namespaces, namespace2pods): # noqa: R0914,C901 """Create volumes on specific namespaces and keep relationship.""" volumes = [] for namespace, node in namespaces.items(): + if "_" in node.get("resource_id"): + pod_suffix = volume_suffix = f"_{node.get('resource_id').split('_')[-1]}" + tag_suffix = pod_suffix.split("-")[0] + else: + pod_suffix = volume_suffix = tag_suffix = "" storage_class_default, csi_default = choice( ( ("gp3-csi", "ebs.csi.aws.com"), @@ -558,7 +589,8 @@ def _gen_volumes(self, namespaces, namespace2pods): # noqa: R0914,C901 if node.get("namespaces"): specified_volumes = node.get("namespaces").get(namespace).get("volumes", []) for specified_volume in specified_volumes: - volume = specified_volume.get("volume_name", self.fake.word()) + volume = f"{specified_volume.get('volume_name', self.fake.word())}{volume_suffix}" + volume_request_gig = specified_volume.get("volume_request_gig") volume_request = volume_request_gig * GIGABYTE specified_vol_claims = specified_volume.get("volume_claims", []) @@ -567,8 +599,8 @@ def _gen_volumes(self, namespaces, namespace2pods): # noqa: R0914,C901 for specified_vc in specified_vol_claims: if volume_request - total_claims <= GIGABYTE: break - vol_claim = specified_vc.get("volume_claim_name", self.fake.word()) - pod = specified_vc.get("pod_name") + vol_claim = f"{specified_vc.get('volume_claim_name', self.fake.word())}{volume_suffix}" + pod = f"{specified_vc.get('pod_name')}{pod_suffix}" claim_capacity = max( specified_vc.get("capacity_gig") * GIGABYTE, (volume_request_gig * GIGABYTE - total_claims) ) @@ -586,6 +618,12 @@ def _gen_volumes(self, namespaces, namespace2pods): # noqa: R0914,C901 "volume_claim_usage_gig": usage_gig, } total_claims += claim_capacity + + volume_labels = specified_volume.get("labels", None) + if volume_labels: + volume_labels = volume_labels.replace( + "label_managed_tables_matching:today", f"label_managed_tables_matching:today{tag_suffix}" + ) volumes.append( { volume: { @@ -594,11 +632,11 @@ def _gen_volumes(self, namespaces, namespace2pods): # noqa: R0914,C901 "volume": volume, "storage_class": specified_volume.get("storage_class", storage_class_default), "csi_driver": specified_volume.get("csi_driver", csi_default), - "csi_volume_handle": specified_volume.get( - "csi_volume_handle", f"vol-{self.fake.word()}" - ), + "csi_volume_handle": f"{ + specified_volume.get('csi_volume_handle', f'vol-{self.fake.word()}') + }{volume_suffix}", "volume_request": volume_request, - "labels": specified_volume.get("labels", None), + "labels": volume_labels, "volume_claims": volume_claims, } } diff --git a/nise/report.py b/nise/report.py index aba141b56..e88ea8109 100644 --- a/nise/report.py +++ b/nise/report.py @@ -88,6 +88,8 @@ from nise.upload import upload_to_gcp_storage from nise.upload import upload_to_s3 from nise.util import LOG +from nise.util import pseudo_random_uuid +from nise.util import NUMBER_OF_REPLICAS def create_temporary_copy(path, temp_file_name, temp_dir_name="None"): @@ -593,7 +595,6 @@ def aws_create_marketplace_report(options): # noqa: C901 def aws_create_report(options): # noqa: C901 - """Create a cost usage report file.""" start_date = options.get("start_date") end_date = options.get("end_date") aws_finalize_report = options.get("aws_finalize_report") @@ -655,9 +656,33 @@ def aws_create_report(options): # noqa: C901 attributes, options.get("aws_tags"), ) - num_instances = 1 if attributes else randint(2, 60) - for _ in range(num_instances): + # TODO PERF_NOTE: update NUMBER_OF_REPLICAS - how many times you want to multiple the whole yaml file + if attributes: + num_instances = NUMBER_OF_REPLICAS + else: + num_instances = randint(2, 60) + + for i in range(num_instances): + id_suffix = tag_suffix = "" + if attributes and attributes.get("resource_id"): + if num_instances > 1: + id_suffix = f"_{pseudo_random_uuid(i)}" + tag_suffix = id_suffix.split("-")[0] + resource_id = f"i-{attributes.get('resource_id')}{id_suffix}" + else: + resource_id = f"i-{attributes.get('resource_id')}" + else: + resource_id = f"i-{fake.ean8()}" + for hour in gen.generate_data(): + if attributes and num_instances > 1 or not attributes: + hour["lineItem/ResourceId"] = resource_id + if orig_node_tag_value := hour.get("resourceTags/user:openshift_node"): + hour["resourceTags/user:openshift_node"] = f"{orig_node_tag_value}{id_suffix}" + if orig_project_tag_value := hour.get("resourceTags/user:openshift_project"): + hour["resourceTags/user:openshift_project"] = f"{orig_project_tag_value}{id_suffix}" + if orig_tag_match_value := hour.get("resourceTags/user:managed_tables_matching"): + hour["resourceTags/user:managed_tables_matching"] = f"{orig_tag_match_value}{tag_suffix}" data += [hour] if len(data) == options.get("row_limit"): file_number += 1 diff --git a/nise/util/__init__.py b/nise/util/__init__.py index 46b9f9bda..d0ad8493c 100644 --- a/nise/util/__init__.py +++ b/nise/util/__init__.py @@ -17,13 +17,17 @@ """Utility functions.""" from collections import abc - +import random +import uuid import yaml from .log import LOG # noqa: F401 from .log import LOG_FORMAT # noqa: F401 from .log import LOG_VERBOSITY # noqa: F401 +# TODO UPDATE THIS: Num. of times to replicate each resource +NUMBER_OF_REPLICAS = 10 + def load_yaml(objekt): """Load a yaml document. @@ -56,3 +60,9 @@ def deepupdate(original, update): else: original[key] = value return original + + +def pseudo_random_uuid(rand_seed): + """Generate pseudo-random uuid""" + random.seed(rand_seed) + return uuid.UUID(int=random.getrandbits(128), version=4)