From 412990460c8a17272695585e52175e4083a76b60 Mon Sep 17 00:00:00 2001 From: Eva Sebestova Date: Thu, 20 Mar 2025 16:54:58 +0100 Subject: [PATCH 01/18] OCP/AWS --- nise/__main__.py | 9 +++++ nise/generators/ocp/ocp_generator.py | 58 +++++++++++++++++++--------- nise/report.py | 26 +++++++++++-- nise/util/__init__.py | 8 +++- 4 files changed, 78 insertions(+), 23 deletions(-) diff --git a/nise/__main__.py b/nise/__main__.py index 9d511c31d..a58ca66af 100644 --- a/nise/__main__.py +++ b/nise/__main__.py @@ -372,6 +372,15 @@ def create_parser(): help="Writes the monthly files.", ) + parent_parser.add_argument( + "--resource-replicas", + dest="resource_replicas", + required=False, + type=int, + default=1, + help="Num. of times to replicate each resource", + ) + report_subparser = report_parser.add_subparsers(dest="provider") aws_parser = report_subparser.add_parser( "aws", parents=[parent_parser], add_help=False, description="The AWS parser", help="create the AWS reports" diff --git a/nise/generators/ocp/ocp_generator.py b/nise/generators/ocp/ocp_generator.py index b7c21a7da..ecda9027f 100644 --- a/nise/generators/ocp/ocp_generator.py +++ b/nise/generators/ocp/ocp_generator.py @@ -17,6 +17,7 @@ """Defines the abstract generator.""" import datetime +from nise.util import pseudo_random_uuid from copy import deepcopy from random import choice from random import choices @@ -239,7 +240,6 @@ def __init__(self, start_date, end_date, attributes, ros_ocp_info=False, constan self.nodes = self._gen_nodes() self.namespaces = self._gen_namespaces(self.nodes) self.pods, self.namespace2pods, self.ros_data = self._gen_pods(self.namespaces) - self.volumes = self._gen_volumes(self.namespaces, self.namespace2pods) self.ocp_report_generation = { @@ -280,21 +280,32 @@ def timestamp(in_date): def _gen_nodes(self): """Create nodes for report.""" + num_replicas = 10 nodes = [] if self._nodes: for item in self._nodes: - memory_gig = item.get("memory_gig", randint(2, 8)) - memory_bytes = memory_gig * GIGABYTE - resource_id = str(item.get("resource_id", self.fake.word())) - node = { - "name": item.get("node_name", "node_" + self.fake.word()), - "cpu_cores": item.get("cpu_cores", randint(2, 16)), - "memory_bytes": memory_bytes, - "resource_id": "i-" + resource_id, - "namespaces": item.get("namespaces"), - "node_labels": item.get("node_labels"), - } - nodes.append(node) + for i in range(num_replicas): + memory_gig = item.get("memory_gig", randint(2, 8)) + memory_bytes = memory_gig * GIGABYTE + resource_id = str(item.get("resource_id", self.fake.word())) + node_name = item.get("node_name", "node_" + self.fake.word()) + if num_replicas > 1: + id_suffix = pseudo_random_uuid(i) + resource_id += f"_{id_suffix}" + node_name += f"_{id_suffix}" + + namespaces = item.get("namespaces") + namespaces_renamed = {f"{project}_{id_suffix}": values for project, values in namespaces.items()} + + node = { + "name": node_name, + "cpu_cores": item.get("cpu_cores", randint(2, 16)), + "memory_bytes": memory_bytes, + "resource_id": "i-" + resource_id, + "namespaces": namespaces_renamed, + "node_labels": item.get("node_labels"), + } + nodes.append(node) else: num_nodes = randint(2, 6) seeded_labels = {"node-role.kubernetes.io/master": [""], "node-role.kubernetes.io/infra": [""]} @@ -319,12 +330,14 @@ def _gen_namespaces(self, nodes): for name, _ in node.get("namespaces").items(): namespace = name namespaces[namespace] = node + else: num_namespaces = randint(2, 12) for _ in range(num_namespaces): namespace_suffix = choice(("ci", "qa", "prod", "proj", "dev", "staging")) namespace = self.fake.word() + "_" + namespace_suffix namespaces[namespace] = node + return namespaces def _gen_openshift_labels(self, seeding=None): @@ -369,12 +382,16 @@ def _gen_pods(self, namespaces): ros_ocp_data_pods = {} namespace2pod = {} for namespace, node in namespaces.items(): + if "_" in node.get('resource_id'): + pod_suffix = f"_{node.get('resource_id').split("_")[-1]}" + else: + pod_suffix = "" + namespace2pod[namespace] = [] if node.get("namespaces"): specified_pods = node.get("namespaces").get(namespace).get("pods") or [] for specified_pod in specified_pods: - pod = specified_pod.get("pod_name", self.fake.word()) - namespace2pod[namespace].append(pod) + pod = f"{specified_pod.get("pod_name", self.fake.word())}{pod_suffix}" cpu_cores = node.get("cpu_cores") memory_bytes = node.get("memory_bytes") @@ -547,6 +564,10 @@ def _gen_volumes(self, namespaces, namespace2pods): # noqa: R0914,C901 """Create volumes on specific namespaces and keep relationship.""" volumes = [] for namespace, node in namespaces.items(): + if "_" in node.get('resource_id'): + pod_suffix = volume_suffix = f"_{node.get('resource_id').split("_")[-1]}" + else: + volume_suffix = "" storage_class_default, csi_default = choice( ( ("gp3-csi", "ebs.csi.aws.com"), @@ -558,7 +579,8 @@ def _gen_volumes(self, namespaces, namespace2pods): # noqa: R0914,C901 if node.get("namespaces"): specified_volumes = node.get("namespaces").get(namespace).get("volumes", []) for specified_volume in specified_volumes: - volume = specified_volume.get("volume_name", self.fake.word()) + volume = f"{specified_volume.get("volume_name", self.fake.word())}{volume_suffix}" + volume_request_gig = specified_volume.get("volume_request_gig") volume_request = volume_request_gig * GIGABYTE specified_vol_claims = specified_volume.get("volume_claims", []) @@ -567,8 +589,8 @@ def _gen_volumes(self, namespaces, namespace2pods): # noqa: R0914,C901 for specified_vc in specified_vol_claims: if volume_request - total_claims <= GIGABYTE: break - vol_claim = specified_vc.get("volume_claim_name", self.fake.word()) - pod = specified_vc.get("pod_name") + vol_claim = f"{specified_vc.get("volume_claim_name", self.fake.word())}{volume_suffix}" + pod = f"{specified_vc.get("pod_name")}{pod_suffix}" claim_capacity = max( specified_vc.get("capacity_gig") * GIGABYTE, (volume_request_gig * GIGABYTE - total_claims) ) diff --git a/nise/report.py b/nise/report.py index aba141b56..79cbe0e99 100644 --- a/nise/report.py +++ b/nise/report.py @@ -88,7 +88,7 @@ from nise.upload import upload_to_gcp_storage from nise.upload import upload_to_s3 from nise.util import LOG - +from nise.util import pseudo_random_uuid def create_temporary_copy(path, temp_file_name, temp_dir_name="None"): """Create temporary copy of a file.""" @@ -593,7 +593,11 @@ def aws_create_marketplace_report(options): # noqa: C901 def aws_create_report(options): # noqa: C901 - """Create a cost usage report file.""" + + num_instances = options.get("resource_replicas", 1) + # TODO remove this? -> use constant? + num_instances = 10 + start_date = options.get("start_date") end_date = options.get("end_date") aws_finalize_report = options.get("aws_finalize_report") @@ -655,9 +659,23 @@ def aws_create_report(options): # noqa: C901 attributes, options.get("aws_tags"), ) - num_instances = 1 if attributes else randint(2, 60) - for _ in range(num_instances): + # TODO PERF_NOTE: update num_instances - how many times you want to multiple the whole yaml file + if not attributes: + num_instances = randint(2, 60) + + for i in range(num_instances): + if attributes and attributes.get("resource_id"): + if num_instances > 1: + id_suffix = pseudo_random_uuid(i) + resource_id = f"{attributes.get('resource_id')}_{id_suffix}" + else: + resource_id = f"{attributes.get("resource_id")}" + else: + resource_id = "i-{}".format(fake.ean8()) + for hour in gen.generate_data(): + if attributes and num_instances > 1 or not attributes: + hour["lineItem/ResourceId"] = resource_id data += [hour] if len(data) == options.get("row_limit"): file_number += 1 diff --git a/nise/util/__init__.py b/nise/util/__init__.py index 46b9f9bda..e97c57166 100644 --- a/nise/util/__init__.py +++ b/nise/util/__init__.py @@ -17,7 +17,8 @@ """Utility functions.""" from collections import abc - +import random +import uuid import yaml from .log import LOG # noqa: F401 @@ -56,3 +57,8 @@ def deepupdate(original, update): else: original[key] = value return original + +def pseudo_random_uuid(rand_seed): + """ Generate pseudo-random uuid """ + random.seed(rand_seed) + return uuid.UUID(int=random.getrandbits(128), version=4) \ No newline at end of file From 58d86f76488654e56af55f31a68b4e3f51a56c33 Mon Sep 17 00:00:00 2001 From: Eva Sebestova Date: Fri, 21 Mar 2025 09:47:07 +0100 Subject: [PATCH 02/18] fixes --- nise/report.py | 4 ++-- nise/util/__init__.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/nise/report.py b/nise/report.py index 79cbe0e99..063fa9c4d 100644 --- a/nise/report.py +++ b/nise/report.py @@ -667,9 +667,9 @@ def aws_create_report(options): # noqa: C901 if attributes and attributes.get("resource_id"): if num_instances > 1: id_suffix = pseudo_random_uuid(i) - resource_id = f"{attributes.get('resource_id')}_{id_suffix}" + resource_id = f"i-{attributes.get('resource_id')}_{id_suffix}" else: - resource_id = f"{attributes.get("resource_id")}" + resource_id = f"i-{attributes.get("resource_id")}" else: resource_id = "i-{}".format(fake.ean8()) diff --git a/nise/util/__init__.py b/nise/util/__init__.py index e97c57166..16adb0bed 100644 --- a/nise/util/__init__.py +++ b/nise/util/__init__.py @@ -61,4 +61,5 @@ def deepupdate(original, update): def pseudo_random_uuid(rand_seed): """ Generate pseudo-random uuid """ random.seed(rand_seed) - return uuid.UUID(int=random.getrandbits(128), version=4) \ No newline at end of file + return uuid.UUID(int=random.getrandbits(128), version=4) + From 8f953ce48aaf44989f8ec8d0fb05fb5946609ee6 Mon Sep 17 00:00:00 2001 From: Eva Sebestova Date: Fri, 21 Mar 2025 10:57:52 +0100 Subject: [PATCH 03/18] csi handlers --- nise/generators/ocp/ocp_generator.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nise/generators/ocp/ocp_generator.py b/nise/generators/ocp/ocp_generator.py index ecda9027f..8cc84f974 100644 --- a/nise/generators/ocp/ocp_generator.py +++ b/nise/generators/ocp/ocp_generator.py @@ -567,7 +567,7 @@ def _gen_volumes(self, namespaces, namespace2pods): # noqa: R0914,C901 if "_" in node.get('resource_id'): pod_suffix = volume_suffix = f"_{node.get('resource_id').split("_")[-1]}" else: - volume_suffix = "" + pod_suffix = volume_suffix = "" storage_class_default, csi_default = choice( ( ("gp3-csi", "ebs.csi.aws.com"), @@ -616,9 +616,9 @@ def _gen_volumes(self, namespaces, namespace2pods): # noqa: R0914,C901 "volume": volume, "storage_class": specified_volume.get("storage_class", storage_class_default), "csi_driver": specified_volume.get("csi_driver", csi_default), - "csi_volume_handle": specified_volume.get( - "csi_volume_handle", f"vol-{self.fake.word()}" - ), + "csi_volume_handle": f"{specified_volume.get( + 'csi_volume_handle', f'vol-{self.fake.word()}' + )}{volume_suffix}", "volume_request": volume_request, "labels": specified_volume.get("labels", None), "volume_claims": volume_claims, From 98ebb178ccbfdddcc8323bad2fc004725959d72f Mon Sep 17 00:00:00 2001 From: Eva Sebestova Date: Mon, 24 Mar 2025 09:26:44 +0100 Subject: [PATCH 04/18] tag matching support --- nise/report.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nise/report.py b/nise/report.py index 063fa9c4d..3fcdaa5c5 100644 --- a/nise/report.py +++ b/nise/report.py @@ -676,6 +676,12 @@ def aws_create_report(options): # noqa: C901 for hour in gen.generate_data(): if attributes and num_instances > 1 or not attributes: hour["lineItem/ResourceId"] = resource_id + if orig_node_tag_value := hour.get("resourceTags/user:openshift_node"): + hour["resourceTags/user:openshift_node"] = f"{orig_node_tag_value}_{id_suffix}" + if orig_project_tag_value := hour.get("resourceTags/user:openshift_project"): + hour["resourceTags/user:openshift_project"] = f"{orig_project_tag_value}_{id_suffix}" + if orig_tag_match_value := hour.get("resourceTags/user:tag_matching"): + hour["resourceTags/user:tag_matching"] = f"{orig_tag_match_value}_{id_suffix}" data += [hour] if len(data) == options.get("row_limit"): file_number += 1 From 62ea8d16b4e163132ed681cfc0b6e560e8608b21 Mon Sep 17 00:00:00 2001 From: Eva Sebestova Date: Mon, 24 Mar 2025 11:16:31 +0100 Subject: [PATCH 05/18] num. replicas as a constant --- nise/__main__.py | 9 --------- nise/generators/ocp/ocp_generator.py | 4 +++- nise/report.py | 12 +++++------- nise/util/__init__.py | 2 ++ 4 files changed, 10 insertions(+), 17 deletions(-) diff --git a/nise/__main__.py b/nise/__main__.py index a58ca66af..9d511c31d 100644 --- a/nise/__main__.py +++ b/nise/__main__.py @@ -372,15 +372,6 @@ def create_parser(): help="Writes the monthly files.", ) - parent_parser.add_argument( - "--resource-replicas", - dest="resource_replicas", - required=False, - type=int, - default=1, - help="Num. of times to replicate each resource", - ) - report_subparser = report_parser.add_subparsers(dest="provider") aws_parser = report_subparser.add_parser( "aws", parents=[parent_parser], add_help=False, description="The AWS parser", help="create the AWS reports" diff --git a/nise/generators/ocp/ocp_generator.py b/nise/generators/ocp/ocp_generator.py index 8cc84f974..e6531e340 100644 --- a/nise/generators/ocp/ocp_generator.py +++ b/nise/generators/ocp/ocp_generator.py @@ -28,6 +28,8 @@ from dateutil import parser from nise.generators.generator import AbstractGenerator from nise.generators.generator import REPORT_TYPE +from nise.util import NUMBER_OF_REPLICAS + GIGABYTE = 1024 * 1024 * 1024 HOUR = 60 * 60 @@ -280,7 +282,7 @@ def timestamp(in_date): def _gen_nodes(self): """Create nodes for report.""" - num_replicas = 10 + num_replicas = NUMBER_OF_REPLICAS nodes = [] if self._nodes: for item in self._nodes: diff --git a/nise/report.py b/nise/report.py index 3fcdaa5c5..bd96b8c11 100644 --- a/nise/report.py +++ b/nise/report.py @@ -89,6 +89,7 @@ from nise.upload import upload_to_s3 from nise.util import LOG from nise.util import pseudo_random_uuid +from nise.util import NUMBER_OF_REPLICAS def create_temporary_copy(path, temp_file_name, temp_dir_name="None"): """Create temporary copy of a file.""" @@ -593,11 +594,6 @@ def aws_create_marketplace_report(options): # noqa: C901 def aws_create_report(options): # noqa: C901 - - num_instances = options.get("resource_replicas", 1) - # TODO remove this? -> use constant? - num_instances = 10 - start_date = options.get("start_date") end_date = options.get("end_date") aws_finalize_report = options.get("aws_finalize_report") @@ -659,8 +655,10 @@ def aws_create_report(options): # noqa: C901 attributes, options.get("aws_tags"), ) - # TODO PERF_NOTE: update num_instances - how many times you want to multiple the whole yaml file - if not attributes: + # TODO PERF_NOTE: update NUMBER_OF_REPLICAS - how many times you want to multiple the whole yaml file + if attributes: + num_instances = NUMBER_OF_REPLICAS + else: num_instances = randint(2, 60) for i in range(num_instances): diff --git a/nise/util/__init__.py b/nise/util/__init__.py index 16adb0bed..39dbdd556 100644 --- a/nise/util/__init__.py +++ b/nise/util/__init__.py @@ -25,6 +25,8 @@ from .log import LOG_FORMAT # noqa: F401 from .log import LOG_VERBOSITY # noqa: F401 +# TODO UPDATE THIS: Num. of times to replicate each resource +NUMBER_OF_REPLICAS = 1 def load_yaml(objekt): """Load a yaml document. From cb9c258c695bcabfeb83534a7dfa46db107c8abe Mon Sep 17 00:00:00 2001 From: Eva Sebestova Date: Mon, 24 Mar 2025 11:24:58 +0100 Subject: [PATCH 06/18] update tag name --- nise/report.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nise/report.py b/nise/report.py index bd96b8c11..1d2c72029 100644 --- a/nise/report.py +++ b/nise/report.py @@ -678,8 +678,8 @@ def aws_create_report(options): # noqa: C901 hour["resourceTags/user:openshift_node"] = f"{orig_node_tag_value}_{id_suffix}" if orig_project_tag_value := hour.get("resourceTags/user:openshift_project"): hour["resourceTags/user:openshift_project"] = f"{orig_project_tag_value}_{id_suffix}" - if orig_tag_match_value := hour.get("resourceTags/user:tag_matching"): - hour["resourceTags/user:tag_matching"] = f"{orig_tag_match_value}_{id_suffix}" + if orig_tag_match_value := hour.get("resourceTags/user:managed_tables_matching"): + hour["resourceTags/user:managed_tables_matching"] = f"{orig_tag_match_value}_{id_suffix}" data += [hour] if len(data) == options.get("row_limit"): file_number += 1 From f5491c54a6650797957ebf3926a44f7caa71d27f Mon Sep 17 00:00:00 2001 From: Eva Sebestova Date: Mon, 24 Mar 2025 11:38:03 +0100 Subject: [PATCH 07/18] set replicas --- nise/util/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nise/util/__init__.py b/nise/util/__init__.py index 39dbdd556..a60ae1c05 100644 --- a/nise/util/__init__.py +++ b/nise/util/__init__.py @@ -26,7 +26,7 @@ from .log import LOG_VERBOSITY # noqa: F401 # TODO UPDATE THIS: Num. of times to replicate each resource -NUMBER_OF_REPLICAS = 1 +NUMBER_OF_REPLICAS = 10 def load_yaml(objekt): """Load a yaml document. From 20b15b1ba8bbfe7fe23544f0628f0dad34d142bb Mon Sep 17 00:00:00 2001 From: Eva Sebestova Date: Mon, 24 Mar 2025 17:41:28 +0100 Subject: [PATCH 08/18] Fix tag matching --- nise/generators/ocp/ocp_generator.py | 26 ++++++++++++++++++++------ nise/report.py | 11 ++++++----- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/nise/generators/ocp/ocp_generator.py b/nise/generators/ocp/ocp_generator.py index e6531e340..1a3d123f1 100644 --- a/nise/generators/ocp/ocp_generator.py +++ b/nise/generators/ocp/ocp_generator.py @@ -287,17 +287,18 @@ def _gen_nodes(self): if self._nodes: for item in self._nodes: for i in range(num_replicas): + id_suffix = "" memory_gig = item.get("memory_gig", randint(2, 8)) memory_bytes = memory_gig * GIGABYTE resource_id = str(item.get("resource_id", self.fake.word())) node_name = item.get("node_name", "node_" + self.fake.word()) if num_replicas > 1: - id_suffix = pseudo_random_uuid(i) - resource_id += f"_{id_suffix}" - node_name += f"_{id_suffix}" + id_suffix = f"_{pseudo_random_uuid(i)}" + resource_id += id_suffix + node_name += id_suffix namespaces = item.get("namespaces") - namespaces_renamed = {f"{project}_{id_suffix}": values for project, values in namespaces.items()} + namespaces_renamed = {f"{project}{id_suffix}": values for project, values in namespaces.items()} node = { "name": node_name, @@ -414,6 +415,12 @@ def _gen_pods(self, namespaces): if value > mem_limit_gig: memory_usage_gig[key] = mem_limit_gig + pod_labels = specified_pod.get("labels", None) + if pod_labels: + pod_labels = pod_labels.replace( + "label_managed_tables_matching:today", + f"label_managed_tables_matching:today{pod_suffix}" + ) pods[pod] = { "namespace": namespace, "node": node.get("name"), @@ -427,7 +434,7 @@ def _gen_pods(self, namespaces): "cpu_limit": cpu_limit, "mem_request_gig": mem_request_gig, "mem_limit_gig": mem_limit_gig, - "pod_labels": specified_pod.get("labels", None), + "pod_labels": pod_labels, "cpu_usage": cpu_usage, "mem_usage_gig": memory_usage_gig, "pod_seconds": specified_pod.get("pod_seconds"), @@ -610,6 +617,13 @@ def _gen_volumes(self, namespaces, namespace2pods): # noqa: R0914,C901 "volume_claim_usage_gig": usage_gig, } total_claims += claim_capacity + + volume_labels = specified_volume.get("labels", None) + if volume_labels: + volume_labels = volume_labels.replace( + "label_managed_tables_matching:today", + f"label_managed_tables_matching:today{volume_suffix}" + ) volumes.append( { volume: { @@ -622,7 +636,7 @@ def _gen_volumes(self, namespaces, namespace2pods): # noqa: R0914,C901 'csi_volume_handle', f'vol-{self.fake.word()}' )}{volume_suffix}", "volume_request": volume_request, - "labels": specified_volume.get("labels", None), + "labels": volume_labels, "volume_claims": volume_claims, } } diff --git a/nise/report.py b/nise/report.py index 1d2c72029..2d31d770e 100644 --- a/nise/report.py +++ b/nise/report.py @@ -662,10 +662,11 @@ def aws_create_report(options): # noqa: C901 num_instances = randint(2, 60) for i in range(num_instances): + id_suffix = "" if attributes and attributes.get("resource_id"): if num_instances > 1: - id_suffix = pseudo_random_uuid(i) - resource_id = f"i-{attributes.get('resource_id')}_{id_suffix}" + id_suffix = f"_{pseudo_random_uuid(i)}" + resource_id = f"i-{attributes.get('resource_id')}{id_suffix}" else: resource_id = f"i-{attributes.get("resource_id")}" else: @@ -675,11 +676,11 @@ def aws_create_report(options): # noqa: C901 if attributes and num_instances > 1 or not attributes: hour["lineItem/ResourceId"] = resource_id if orig_node_tag_value := hour.get("resourceTags/user:openshift_node"): - hour["resourceTags/user:openshift_node"] = f"{orig_node_tag_value}_{id_suffix}" + hour["resourceTags/user:openshift_node"] = f"{orig_node_tag_value}{id_suffix}" if orig_project_tag_value := hour.get("resourceTags/user:openshift_project"): - hour["resourceTags/user:openshift_project"] = f"{orig_project_tag_value}_{id_suffix}" + hour["resourceTags/user:openshift_project"] = f"{orig_project_tag_value}{id_suffix}" if orig_tag_match_value := hour.get("resourceTags/user:managed_tables_matching"): - hour["resourceTags/user:managed_tables_matching"] = f"{orig_tag_match_value}_{id_suffix}" + hour["resourceTags/user:managed_tables_matching"] = f"{orig_tag_match_value}{id_suffix}" data += [hour] if len(data) == options.get("row_limit"): file_number += 1 From 7591e8ffed579ff8e76b5ce977c680d8bc1c482b Mon Sep 17 00:00:00 2001 From: Eva Sebestova Date: Thu, 27 Mar 2025 11:18:44 +0100 Subject: [PATCH 09/18] fix tag matching --- nise/generators/ocp/ocp_generator.py | 10 ++++++---- nise/report.py | 5 +++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/nise/generators/ocp/ocp_generator.py b/nise/generators/ocp/ocp_generator.py index 1a3d123f1..48097763d 100644 --- a/nise/generators/ocp/ocp_generator.py +++ b/nise/generators/ocp/ocp_generator.py @@ -387,8 +387,9 @@ def _gen_pods(self, namespaces): for namespace, node in namespaces.items(): if "_" in node.get('resource_id'): pod_suffix = f"_{node.get('resource_id').split("_")[-1]}" + tag_suffix = pod_suffix.split("-")[0] else: - pod_suffix = "" + pod_suffix = tag_suffix = "" namespace2pod[namespace] = [] if node.get("namespaces"): @@ -419,7 +420,7 @@ def _gen_pods(self, namespaces): if pod_labels: pod_labels = pod_labels.replace( "label_managed_tables_matching:today", - f"label_managed_tables_matching:today{pod_suffix}" + f"label_managed_tables_matching:today{tag_suffix}" ) pods[pod] = { "namespace": namespace, @@ -575,8 +576,9 @@ def _gen_volumes(self, namespaces, namespace2pods): # noqa: R0914,C901 for namespace, node in namespaces.items(): if "_" in node.get('resource_id'): pod_suffix = volume_suffix = f"_{node.get('resource_id').split("_")[-1]}" + tag_suffix = pod_suffix.split("-")[0] else: - pod_suffix = volume_suffix = "" + pod_suffix = volume_suffix = tag_suffix = "" storage_class_default, csi_default = choice( ( ("gp3-csi", "ebs.csi.aws.com"), @@ -622,7 +624,7 @@ def _gen_volumes(self, namespaces, namespace2pods): # noqa: R0914,C901 if volume_labels: volume_labels = volume_labels.replace( "label_managed_tables_matching:today", - f"label_managed_tables_matching:today{volume_suffix}" + f"label_managed_tables_matching:today{tag_suffix}" ) volumes.append( { diff --git a/nise/report.py b/nise/report.py index 2d31d770e..232b7b601 100644 --- a/nise/report.py +++ b/nise/report.py @@ -662,10 +662,11 @@ def aws_create_report(options): # noqa: C901 num_instances = randint(2, 60) for i in range(num_instances): - id_suffix = "" + id_suffix = tag_suffix = "" if attributes and attributes.get("resource_id"): if num_instances > 1: id_suffix = f"_{pseudo_random_uuid(i)}" + tag_suffix = id_suffix.split("-")[0] resource_id = f"i-{attributes.get('resource_id')}{id_suffix}" else: resource_id = f"i-{attributes.get("resource_id")}" @@ -680,7 +681,7 @@ def aws_create_report(options): # noqa: C901 if orig_project_tag_value := hour.get("resourceTags/user:openshift_project"): hour["resourceTags/user:openshift_project"] = f"{orig_project_tag_value}{id_suffix}" if orig_tag_match_value := hour.get("resourceTags/user:managed_tables_matching"): - hour["resourceTags/user:managed_tables_matching"] = f"{orig_tag_match_value}{id_suffix}" + hour["resourceTags/user:managed_tables_matching"] = f"{orig_tag_match_value}{tag_suffix}" data += [hour] if len(data) == options.get("row_limit"): file_number += 1 From 1b8ab99d8f45835d24e406eceee0559f431db4ef Mon Sep 17 00:00:00 2001 From: Eva Sebestova Date: Wed, 30 Apr 2025 18:32:34 +0200 Subject: [PATCH 10/18] rebase --- nise/__init__.py | 1 - nise/generators/ocp/ocp_generator.py | 28 +++++++++++++--------------- nise/report.py | 5 +++-- nise/util/__init__.py | 5 +++-- 4 files changed, 19 insertions(+), 20 deletions(-) diff --git a/nise/__init__.py b/nise/__init__.py index a89dbb0e1..334c39d76 100644 --- a/nise/__init__.py +++ b/nise/__init__.py @@ -1,4 +1,3 @@ __version__ = "5.0.3" - VERSION = __version__.split(".") diff --git a/nise/generators/ocp/ocp_generator.py b/nise/generators/ocp/ocp_generator.py index 48097763d..b4cbc1c16 100644 --- a/nise/generators/ocp/ocp_generator.py +++ b/nise/generators/ocp/ocp_generator.py @@ -385,8 +385,8 @@ def _gen_pods(self, namespaces): ros_ocp_data_pods = {} namespace2pod = {} for namespace, node in namespaces.items(): - if "_" in node.get('resource_id'): - pod_suffix = f"_{node.get('resource_id').split("_")[-1]}" + if "_" in node.get("resource_id"): + pod_suffix = f"_{node.get('resource_id').split('_')[-1]}" tag_suffix = pod_suffix.split("-")[0] else: pod_suffix = tag_suffix = "" @@ -395,7 +395,7 @@ def _gen_pods(self, namespaces): if node.get("namespaces"): specified_pods = node.get("namespaces").get(namespace).get("pods") or [] for specified_pod in specified_pods: - pod = f"{specified_pod.get("pod_name", self.fake.word())}{pod_suffix}" + pod = f"{specified_pod.get('pod_name', self.fake.word())}{pod_suffix}" cpu_cores = node.get("cpu_cores") memory_bytes = node.get("memory_bytes") @@ -419,8 +419,7 @@ def _gen_pods(self, namespaces): pod_labels = specified_pod.get("labels", None) if pod_labels: pod_labels = pod_labels.replace( - "label_managed_tables_matching:today", - f"label_managed_tables_matching:today{tag_suffix}" + "label_managed_tables_matching:today", f"label_managed_tables_matching:today{tag_suffix}" ) pods[pod] = { "namespace": namespace, @@ -574,8 +573,8 @@ def _gen_volumes(self, namespaces, namespace2pods): # noqa: R0914,C901 """Create volumes on specific namespaces and keep relationship.""" volumes = [] for namespace, node in namespaces.items(): - if "_" in node.get('resource_id'): - pod_suffix = volume_suffix = f"_{node.get('resource_id').split("_")[-1]}" + if "_" in node.get("resource_id"): + pod_suffix = volume_suffix = f"_{node.get('resource_id').split('_')[-1]}" tag_suffix = pod_suffix.split("-")[0] else: pod_suffix = volume_suffix = tag_suffix = "" @@ -590,7 +589,7 @@ def _gen_volumes(self, namespaces, namespace2pods): # noqa: R0914,C901 if node.get("namespaces"): specified_volumes = node.get("namespaces").get(namespace).get("volumes", []) for specified_volume in specified_volumes: - volume = f"{specified_volume.get("volume_name", self.fake.word())}{volume_suffix}" + volume = f"{specified_volume.get('volume_name', self.fake.word())}{volume_suffix}" volume_request_gig = specified_volume.get("volume_request_gig") volume_request = volume_request_gig * GIGABYTE @@ -600,8 +599,8 @@ def _gen_volumes(self, namespaces, namespace2pods): # noqa: R0914,C901 for specified_vc in specified_vol_claims: if volume_request - total_claims <= GIGABYTE: break - vol_claim = f"{specified_vc.get("volume_claim_name", self.fake.word())}{volume_suffix}" - pod = f"{specified_vc.get("pod_name")}{pod_suffix}" + vol_claim = f"{specified_vc.get('volume_claim_name', self.fake.word())}{volume_suffix}" + pod = f"{specified_vc.get('pod_name')}{pod_suffix}" claim_capacity = max( specified_vc.get("capacity_gig") * GIGABYTE, (volume_request_gig * GIGABYTE - total_claims) ) @@ -623,8 +622,7 @@ def _gen_volumes(self, namespaces, namespace2pods): # noqa: R0914,C901 volume_labels = specified_volume.get("labels", None) if volume_labels: volume_labels = volume_labels.replace( - "label_managed_tables_matching:today", - f"label_managed_tables_matching:today{tag_suffix}" + "label_managed_tables_matching:today", f"label_managed_tables_matching:today{tag_suffix}" ) volumes.append( { @@ -634,9 +632,9 @@ def _gen_volumes(self, namespaces, namespace2pods): # noqa: R0914,C901 "volume": volume, "storage_class": specified_volume.get("storage_class", storage_class_default), "csi_driver": specified_volume.get("csi_driver", csi_default), - "csi_volume_handle": f"{specified_volume.get( - 'csi_volume_handle', f'vol-{self.fake.word()}' - )}{volume_suffix}", + "csi_volume_handle": f"{ + specified_volume.get('csi_volume_handle', f'vol-{self.fake.word()}') + }{volume_suffix}", "volume_request": volume_request, "labels": volume_labels, "volume_claims": volume_claims, diff --git a/nise/report.py b/nise/report.py index 232b7b601..e88ea8109 100644 --- a/nise/report.py +++ b/nise/report.py @@ -91,6 +91,7 @@ from nise.util import pseudo_random_uuid from nise.util import NUMBER_OF_REPLICAS + def create_temporary_copy(path, temp_file_name, temp_dir_name="None"): """Create temporary copy of a file.""" temp_dir = gettempdir() @@ -669,9 +670,9 @@ def aws_create_report(options): # noqa: C901 tag_suffix = id_suffix.split("-")[0] resource_id = f"i-{attributes.get('resource_id')}{id_suffix}" else: - resource_id = f"i-{attributes.get("resource_id")}" + resource_id = f"i-{attributes.get('resource_id')}" else: - resource_id = "i-{}".format(fake.ean8()) + resource_id = f"i-{fake.ean8()}" for hour in gen.generate_data(): if attributes and num_instances > 1 or not attributes: diff --git a/nise/util/__init__.py b/nise/util/__init__.py index a60ae1c05..d0ad8493c 100644 --- a/nise/util/__init__.py +++ b/nise/util/__init__.py @@ -28,6 +28,7 @@ # TODO UPDATE THIS: Num. of times to replicate each resource NUMBER_OF_REPLICAS = 10 + def load_yaml(objekt): """Load a yaml document. @@ -60,8 +61,8 @@ def deepupdate(original, update): original[key] = value return original + def pseudo_random_uuid(rand_seed): - """ Generate pseudo-random uuid """ + """Generate pseudo-random uuid""" random.seed(rand_seed) return uuid.UUID(int=random.getrandbits(128), version=4) - From 511520cd2df155508994c9e8563df2142ceca41a Mon Sep 17 00:00:00 2001 From: Eva Sebestova Date: Tue, 13 May 2025 09:46:14 +0200 Subject: [PATCH 11/18] Tweak GCP --- nise/generators/gcp/gcp_generator.py | 29 +++++++++++++++++++++++++--- nise/util/__init__.py | 2 +- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/nise/generators/gcp/gcp_generator.py b/nise/generators/gcp/gcp_generator.py index 7196c1f61..dc788b1ac 100644 --- a/nise/generators/gcp/gcp_generator.py +++ b/nise/generators/gcp/gcp_generator.py @@ -24,6 +24,7 @@ from random import choice from random import randint from random import uniform +from nise.util import NUMBER_OF_REPLICAS, pseudo_random_uuid from nise.generators.generator import AbstractGenerator @@ -311,6 +312,28 @@ def _generate_hourly_data(self, **kwargs): for hour in self.hours: start = hour.get("start") end = hour.get("end") - row = self._init_data_row(start, end) - row = self._update_data(row) - yield row + + for i in range(NUMBER_OF_REPLICAS): + row = self._init_data_row(start, end) + row = self._update_data(row) + + id_suffix = tag_suffix = "" + if NUMBER_OF_REPLICAS > 1: + id_suffix = f"_{pseudo_random_uuid(i)}" + tag_suffix = id_suffix.split("-")[0] + + if row.get("resource.global_name"): + row["resource.global_name"] += id_suffix + if row.get("resource.name"): + row["resource.name"] += id_suffix + if gcp_labels := row.get("labels"): + gcp_labels_list = json.loads(gcp_labels) + for tag in gcp_labels_list: + if tag["key"] in ["openshift_node", "openshift_project"]: + tag["value"] += id_suffix + elif tag["key"] == "managed_tables_matching": + tag["value"] += tag_suffix + + row["labels"] = json.dumps(gcp_labels_list) + raise ValueError(f"evadebug4{row['resource.name'], row['labels']}") + yield row diff --git a/nise/util/__init__.py b/nise/util/__init__.py index d0ad8493c..e2ce6259d 100644 --- a/nise/util/__init__.py +++ b/nise/util/__init__.py @@ -26,7 +26,7 @@ from .log import LOG_VERBOSITY # noqa: F401 # TODO UPDATE THIS: Num. of times to replicate each resource -NUMBER_OF_REPLICAS = 10 +NUMBER_OF_REPLICAS = 1 def load_yaml(objekt): From ed3f0f2ce24e87f772d7d759f590061651c5909e Mon Sep 17 00:00:00 2001 From: Eva Sebestova Date: Tue, 13 May 2025 09:48:34 +0200 Subject: [PATCH 12/18] fix --- nise/generators/gcp/gcp_generator.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nise/generators/gcp/gcp_generator.py b/nise/generators/gcp/gcp_generator.py index dc788b1ac..cae763d5b 100644 --- a/nise/generators/gcp/gcp_generator.py +++ b/nise/generators/gcp/gcp_generator.py @@ -313,6 +313,7 @@ def _generate_hourly_data(self, **kwargs): start = hour.get("start") end = hour.get("end") + # each line items will be generated {NUMBER_OF_REPLICAS} times for i in range(NUMBER_OF_REPLICAS): row = self._init_data_row(start, end) row = self._update_data(row) @@ -333,7 +334,5 @@ def _generate_hourly_data(self, **kwargs): tag["value"] += id_suffix elif tag["key"] == "managed_tables_matching": tag["value"] += tag_suffix - row["labels"] = json.dumps(gcp_labels_list) - raise ValueError(f"evadebug4{row['resource.name'], row['labels']}") yield row From 4641d09c8a21a055c5ae3f6d0838c0dc77b2722f Mon Sep 17 00:00:00 2001 From: Eva Sebestova Date: Tue, 13 May 2025 11:38:06 +0200 Subject: [PATCH 13/18] update replicas --- nise/util/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nise/util/__init__.py b/nise/util/__init__.py index e2ce6259d..d0ad8493c 100644 --- a/nise/util/__init__.py +++ b/nise/util/__init__.py @@ -26,7 +26,7 @@ from .log import LOG_VERBOSITY # noqa: F401 # TODO UPDATE THIS: Num. of times to replicate each resource -NUMBER_OF_REPLICAS = 1 +NUMBER_OF_REPLICAS = 10 def load_yaml(objekt): From f7779ff778a03d05f14ce6b913a3fac3c65f8513 Mon Sep 17 00:00:00 2001 From: Eva Sebestova Date: Tue, 13 May 2025 19:34:30 +0200 Subject: [PATCH 14/18] Fix json flow --- nise/generators/gcp/gcp_generator.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/nise/generators/gcp/gcp_generator.py b/nise/generators/gcp/gcp_generator.py index cae763d5b..4a5178658 100644 --- a/nise/generators/gcp/gcp_generator.py +++ b/nise/generators/gcp/gcp_generator.py @@ -323,16 +323,35 @@ def _generate_hourly_data(self, **kwargs): id_suffix = f"_{pseudo_random_uuid(i)}" tag_suffix = id_suffix.split("-")[0] + # csv flow if row.get("resource.global_name"): row["resource.global_name"] += id_suffix + # json flow + elif row.get("resource", {}).get("global_name"): + row["resource"]["global_name"] += id_suffix + + # csv flow if row.get("resource.name"): row["resource.name"] += id_suffix + # json flow + elif row.get("resource", {}).get("name"): + row["resource"]["name"] += id_suffix + if gcp_labels := row.get("labels"): - gcp_labels_list = json.loads(gcp_labels) - for tag in gcp_labels_list: + is_label_str = isinstance(gcp_labels, str) # True - csv flow / False - json flow + + if is_label_str: + gcp_labels = json.loads(gcp_labels) + + for tag in gcp_labels: if tag["key"] in ["openshift_node", "openshift_project"]: tag["value"] += id_suffix elif tag["key"] == "managed_tables_matching": tag["value"] += tag_suffix - row["labels"] = json.dumps(gcp_labels_list) + + if is_label_str: + gcp_labels = json.dumps(gcp_labels) + + row["labels"] = gcp_labels + yield row From 106e634854f4dc0a78c5969578c026dd86e81477 Mon Sep 17 00:00:00 2001 From: Eva Sebestova Date: Thu, 12 Jun 2025 13:10:05 +0200 Subject: [PATCH 15/18] Add support for Azure --- nise/generators/azure/azure_generator.py | 28 +++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/nise/generators/azure/azure_generator.py b/nise/generators/azure/azure_generator.py index 36851eb4b..afddd21a9 100644 --- a/nise/generators/azure/azure_generator.py +++ b/nise/generators/azure/azure_generator.py @@ -23,6 +23,7 @@ from random import choice from random import randint from random import uniform +from nise.util import NUMBER_OF_REPLICAS, pseudo_random_uuid from nise.generators.generator import AbstractGenerator @@ -454,7 +455,7 @@ def _update_data(self, row, start, end, **kwargs): # NOTE: Commented out columns exist in the report, but we don't have enough # information to date to accurately simulate values. - row["ResourceName"] = resource_name + row["ResourceName"] = resource_name # TODO UPDATE THIS row["IsAzureCreditEligible"] = "TRUE" row["ServiceFamily"] = service_family row["Frequency"] = "UsageBased" @@ -526,8 +527,29 @@ def _generate_daily_data(self): start = day.get("start") end = day.get("end") row = self._init_data_row(start, end) - row = self._update_data(row, start, end) - data.append(row) + for i in range(NUMBER_OF_REPLICAS): + id_suffix = tag_suffix = "" + if NUMBER_OF_REPLICAS > 1: + id_suffix = f"_{pseudo_random_uuid(i)}" + tag_suffix = id_suffix.split("-")[0] + + row = self._update_data(row, start, end) + if self.azure_columns is AZURE_COLUMNS_V2_SUBSCRIPTION: + row["ResourceId"] += id_suffix + else: + row["InstanceName"] += id_suffix + + row["ResourceName"] += id_suffix + + if azure_tags := json.loads(row.get("Tags")): + for tag_key in azure_tags: + if tag_key in ["openshift_node", "openshift_project"]: + azure_tags[tag_key] += id_suffix + elif tag_key == "managed_tables_matching": + azure_tags[tag_key] += tag_suffix + azure_tags = json.dumps(azure_tags) + row["Tags"] = azure_tags + data.append(row) return data def generate_data(self, report_type=None): From 1d782c7c0e09933c831cb92722b7ddf60e9b1f31 Mon Sep 17 00:00:00 2001 From: Eva Sebestova Date: Thu, 12 Jun 2025 13:11:02 +0200 Subject: [PATCH 16/18] remove comment --- nise/generators/azure/azure_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nise/generators/azure/azure_generator.py b/nise/generators/azure/azure_generator.py index afddd21a9..a4e85f705 100644 --- a/nise/generators/azure/azure_generator.py +++ b/nise/generators/azure/azure_generator.py @@ -455,7 +455,7 @@ def _update_data(self, row, start, end, **kwargs): # NOTE: Commented out columns exist in the report, but we don't have enough # information to date to accurately simulate values. - row["ResourceName"] = resource_name # TODO UPDATE THIS + row["ResourceName"] = resource_name row["IsAzureCreditEligible"] = "TRUE" row["ServiceFamily"] = service_family row["Frequency"] = "UsageBased" From ce46e62ac1ac6e946a85bc21e34d475c68bdb60a Mon Sep 17 00:00:00 2001 From: Eva Sebestova Date: Wed, 18 Jun 2025 12:25:40 +0200 Subject: [PATCH 17/18] fix resource_ids --- nise/generators/azure/azure_generator.py | 3 ++- nise/util/__init__.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/nise/generators/azure/azure_generator.py b/nise/generators/azure/azure_generator.py index a4e85f705..9a829eee4 100644 --- a/nise/generators/azure/azure_generator.py +++ b/nise/generators/azure/azure_generator.py @@ -549,7 +549,8 @@ def _generate_daily_data(self): azure_tags[tag_key] += tag_suffix azure_tags = json.dumps(azure_tags) row["Tags"] = azure_tags - data.append(row) + data.append(row.copy()) + return data def generate_data(self, report_type=None): diff --git a/nise/util/__init__.py b/nise/util/__init__.py index d0ad8493c..94f675ac1 100644 --- a/nise/util/__init__.py +++ b/nise/util/__init__.py @@ -26,7 +26,7 @@ from .log import LOG_VERBOSITY # noqa: F401 # TODO UPDATE THIS: Num. of times to replicate each resource -NUMBER_OF_REPLICAS = 10 +NUMBER_OF_REPLICAS = 3 def load_yaml(objekt): From 942e7bc347ad8add70f69d5fa25b7c136797ff6c Mon Sep 17 00:00:00 2001 From: Eva Sebestova Date: Wed, 18 Jun 2025 12:26:44 +0200 Subject: [PATCH 18/18] fix replicas --- nise/util/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nise/util/__init__.py b/nise/util/__init__.py index 94f675ac1..d0ad8493c 100644 --- a/nise/util/__init__.py +++ b/nise/util/__init__.py @@ -26,7 +26,7 @@ from .log import LOG_VERBOSITY # noqa: F401 # TODO UPDATE THIS: Num. of times to replicate each resource -NUMBER_OF_REPLICAS = 3 +NUMBER_OF_REPLICAS = 10 def load_yaml(objekt):