diff --git a/nise/__init__.py b/nise/__init__.py index d4e7f5c0..a6591a43 100644 --- a/nise/__init__.py +++ b/nise/__init__.py @@ -1,5 +1,5 @@ from .helpers import gcp_calculate_persistent_disk_usage_amount from .helpers import gcp_calculate_usage_amount_in_pricing -__version__ = "5.4.0" +__version__ = "5.4.1" VERSION = __version__.split(".") diff --git a/nise/__main__.py b/nise/__main__.py index 5009c418..e628a650 100644 --- a/nise/__main__.py +++ b/nise/__main__.py @@ -320,6 +320,13 @@ def add_ocp_parser_args(parser): action="store_true", help="Flag to generate constant values for ROS for Openshift", ) + parser.add_argument( + "--ros-only", + dest="ros_only", + required=False, + action="store_true", + help="Generate ONLY ROS for Openshift data", + ) def create_parser(): diff --git a/nise/generators/ocp/__init__.py b/nise/generators/ocp/__init__.py index 8edd60bd..720886c0 100644 --- a/nise/generators/ocp/__init__.py +++ b/nise/generators/ocp/__init__.py @@ -21,6 +21,8 @@ from nise.generators.ocp.ocp_generator import OCP_NODE_LABEL # noqa: F401 from nise.generators.ocp.ocp_generator import OCP_POD_USAGE # noqa: F401 from nise.generators.ocp.ocp_generator import OCP_REPORT_TYPE_TO_COLS # noqa: F401 +from nise.generators.ocp.ocp_generator import COST_OCP_REPORT_TYPE_TO_COLS # noqa: F401 +from nise.generators.ocp.ocp_generator import ROS_OCP_REPORT_TYPE_TO_COLS # noqa: F401 from nise.generators.ocp.ocp_generator import OCP_ROS_USAGE # noqa: F401 from nise.generators.ocp.ocp_generator import OCP_ROS_NAMESPACE_USAGE # noqa: F401 from nise.generators.ocp.ocp_generator import OCP_STORAGE_USAGE # noqa: F401 diff --git a/nise/generators/ocp/ocp_generator.py b/nise/generators/ocp/ocp_generator.py index 45ef4712..f991976b 100644 --- a/nise/generators/ocp/ocp_generator.py +++ b/nise/generators/ocp/ocp_generator.py @@ -464,11 +464,14 @@ def get_vm_from_label(labels): class OCPGenerator(AbstractGenerator): """Defines a abstract class for generators.""" - def __init__(self, start_date, end_date, attributes, ros_ocp_info=False, constant_values_ros_ocp=False): + def __init__( + self, start_date, end_date, attributes, ros_ocp_info=False, constant_values_ros_ocp=False, ros_only=False + ): """Initialize the generator.""" self._nodes = None self.ros_ocp_info = ros_ocp_info self.constant_values_ros_ocp = constant_values_ros_ocp + self.ros_only = ros_only if attributes: self._nodes = attributes.get("nodes") @@ -508,46 +511,50 @@ def __init__(self, start_date, end_date, attributes, ros_ocp_info=False, constan self.vms, self.namespace2vm = self._gen_virtual_machines(self.namespaces) self.gpus = self._gen_gpus() - self.ocp_report_generation = { - OCP_POD_USAGE: { - "_generate_hourly_data": self._gen_hourly_pods_usage, - "_update_data": self._update_pod_data, + ros_reports = { + OCP_ROS_USAGE: { + "_generate_hourly_data": self._gen_quarter_hourly_ros_ocp_pods_usage, + "_update_data": self._update_ros_ocp_pod_data, }, - OCP_STORAGE_USAGE: { - "_generate_hourly_data": self._gen_hourly_storage_usage, - "_update_data": self._update_storage_data, - }, - OCP_NODE_LABEL: { - "_generate_hourly_data": self._gen_hourly_node_label_usage, - "_update_data": self._update_node_label_data, - }, - OCP_NAMESPACE_LABEL: { - "_generate_hourly_data": self._gen_hourly_namespace_label_usage, - "_update_data": self._update_namespace_label_data, - }, - OCP_VM_USAGE: { - "_generate_hourly_data": self._gen_hourly_vm_usage, - "_update_data": self._update_vm_data, - }, - OCP_GPU_USAGE: { - "_generate_hourly_data": self._gen_hourly_gpu_usage, - "_update_data": self._update_gpu_data, + OCP_ROS_NAMESPACE_USAGE: { + "_generate_hourly_data": self._gen_quarter_hourly_ros_ocp_namespace_usage, + "_update_data": self._update_ros_ocp_namespace_data, }, } - if self.ros_ocp_info: - self.ocp_report_generation.update( - { - OCP_ROS_USAGE: { - "_generate_hourly_data": self._gen_quarter_hourly_ros_ocp_pods_usage, - "_update_data": self._update_ros_ocp_pod_data, - }, - OCP_ROS_NAMESPACE_USAGE: { - "_generate_hourly_data": self._gen_quarter_hourly_ros_ocp_namespace_usage, - "_update_data": self._update_ros_ocp_namespace_data, - }, - } - ) + if self.ros_only: + # ONLY ROS reports + self.ocp_report_generation = ros_reports + else: + self.ocp_report_generation = { + OCP_POD_USAGE: { + "_generate_hourly_data": self._gen_hourly_pods_usage, + "_update_data": self._update_pod_data, + }, + OCP_STORAGE_USAGE: { + "_generate_hourly_data": self._gen_hourly_storage_usage, + "_update_data": self._update_storage_data, + }, + OCP_NODE_LABEL: { + "_generate_hourly_data": self._gen_hourly_node_label_usage, + "_update_data": self._update_node_label_data, + }, + OCP_NAMESPACE_LABEL: { + "_generate_hourly_data": self._gen_hourly_namespace_label_usage, + "_update_data": self._update_namespace_label_data, + }, + OCP_VM_USAGE: { + "_generate_hourly_data": self._gen_hourly_vm_usage, + "_update_data": self._update_vm_data, + }, + OCP_GPU_USAGE: { + "_generate_hourly_data": self._gen_hourly_gpu_usage, + "_update_data": self._update_gpu_data, + }, + } + + if self.ros_ocp_info: + self.ocp_report_generation.update(ros_reports) @staticmethod def timestamp(in_date): diff --git a/nise/report.py b/nise/report.py index 49b08208..7332993d 100644 --- a/nise/report.py +++ b/nise/report.py @@ -79,15 +79,11 @@ from nise.generators.gcp import JSONLHCSGenerator from nise.generators.gcp import JSONLProjectGenerator from nise.generators.gcp import ProjectGenerator -from nise.generators.ocp import OCP_GPU_USAGE -from nise.generators.ocp import OCP_NAMESPACE_LABEL -from nise.generators.ocp import OCP_NODE_LABEL -from nise.generators.ocp import OCP_POD_USAGE from nise.generators.ocp import OCP_REPORT_TYPE_TO_COLS +from nise.generators.ocp import COST_OCP_REPORT_TYPE_TO_COLS +from nise.generators.ocp import ROS_OCP_REPORT_TYPE_TO_COLS from nise.generators.ocp import OCP_ROS_USAGE from nise.generators.ocp import OCP_ROS_NAMESPACE_USAGE -from nise.generators.ocp import OCP_STORAGE_USAGE -from nise.generators.ocp import OCP_VM_USAGE from nise.generators.ocp import OCPGenerator from nise.manifest import aws_generate_manifest from nise.manifest import ocp_generate_manifest @@ -904,6 +900,7 @@ def ocp_create_report(options): # noqa: C901 static_report_data = options.get("static_report_data") ros_ocp_info = options.get("ros_ocp_info") constant_values_ros_ocp = options.get("constant_values_ros_ocp") + ros_only = options.get("ros_only") if static_report_data: generators = _get_generators(static_report_data.get("generators")) @@ -915,25 +912,16 @@ def ocp_create_report(options): # noqa: C901 minio_upload = options.get("minio_upload") write_monthly = options.get("write_monthly", False) for month in months: - data = { - OCP_POD_USAGE: [], - OCP_STORAGE_USAGE: [], - OCP_NODE_LABEL: [], - OCP_NAMESPACE_LABEL: [], - OCP_VM_USAGE: [], - OCP_GPU_USAGE: [], - } - file_numbers = { - OCP_POD_USAGE: 0, - OCP_STORAGE_USAGE: 0, - OCP_NODE_LABEL: 0, - OCP_NAMESPACE_LABEL: 0, - OCP_VM_USAGE: 0, - OCP_GPU_USAGE: 0, - } - if ros_ocp_info: - data.update({OCP_ROS_USAGE: [], OCP_ROS_NAMESPACE_USAGE: []}) - file_numbers.update({OCP_ROS_USAGE: 0, OCP_ROS_NAMESPACE_USAGE: 0}) + if ros_only: + report_types = ROS_OCP_REPORT_TYPE_TO_COLS + elif ros_ocp_info: + report_types = OCP_REPORT_TYPE_TO_COLS + else: + report_types = COST_OCP_REPORT_TYPE_TO_COLS + + data = {rt: [] for rt in report_types} + file_numbers = {rt: 0 for rt in report_types} + monthly_files = [] monthly_ros_files = [] for generator in generators: @@ -950,7 +938,9 @@ def ocp_create_report(options): # noqa: C901 gen_start_date, gen_end_date = _create_generator_dates_from_yaml(attributes, month) - gen = generator_cls(gen_start_date, gen_end_date, attributes, ros_ocp_info, constant_values_ros_ocp) + gen = generator_cls( + gen_start_date, gen_end_date, attributes, ros_ocp_info, constant_values_ros_ocp, ros_only + ) for report_type in gen.ocp_report_generation.keys(): LOG.info(f"Generating data for {report_type} for {month}") for hour in gen.generate_data(report_type): diff --git a/tests/test_ocp_generator.py b/tests/test_ocp_generator.py index 746974d3..bbea6a9e 100644 --- a/tests/test_ocp_generator.py +++ b/tests/test_ocp_generator.py @@ -33,6 +33,7 @@ from nise.generators.ocp.ocp_generator import GPU_VENDOR from nise.generators.ocp.ocp_generator import OCP_GPU_USAGE from nise.generators.ocp.ocp_generator import OCP_GPU_USAGE_COLUMNS +from nise.generators.ocp.ocp_generator import OCP_NAMESPACE_LABEL from nise.generators.ocp.ocp_generator import OCP_NODE_LABEL from nise.generators.ocp.ocp_generator import OCP_NODE_LABEL_COLUMNS from nise.generators.ocp.ocp_generator import OCP_POD_USAGE @@ -42,6 +43,7 @@ from nise.generators.ocp.ocp_generator import OCP_ROS_USAGE from nise.generators.ocp.ocp_generator import OCP_STORAGE_COLUMNS from nise.generators.ocp.ocp_generator import OCP_STORAGE_USAGE +from nise.generators.ocp.ocp_generator import OCP_VM_USAGE from nise.generators.ocp.ocp_generator import OCP_REPORT_TYPE_TO_COLS from nise.generators.ocp.ocp_generator import COST_OCP_REPORT_TYPE_TO_COLS from nise.generators.ocp.ocp_generator import ROS_OCP_REPORT_TYPE_TO_COLS @@ -1115,6 +1117,31 @@ def test_update_ros_ocp_namespace_data(self): self.assertEqual(updated_row["cpu_request_namespace_sum"], 10.0) self.assertEqual(updated_row["memory_limit_namespace_sum"], 2048) + def test_init_with_ros_only(self): + """Test that generator initializes correctly with ros_only enabled.""" + generator = OCPGenerator(self.two_hours_ago, self.now, {}, ros_only=True) + + # Should have ONLY ROS reports in ocp_report_generation + self.assertEqual(len(generator.ocp_report_generation), 2) + self.assertIn(OCP_ROS_USAGE, generator.ocp_report_generation) + self.assertIn(OCP_ROS_NAMESPACE_USAGE, generator.ocp_report_generation) + + # Should NOT have standard reports + self.assertNotIn(OCP_POD_USAGE, generator.ocp_report_generation) + self.assertNotIn(OCP_STORAGE_USAGE, generator.ocp_report_generation) + self.assertNotIn(OCP_NODE_LABEL, generator.ocp_report_generation) + self.assertNotIn(OCP_NAMESPACE_LABEL, generator.ocp_report_generation) + self.assertNotIn(OCP_VM_USAGE, generator.ocp_report_generation) + self.assertNotIn(OCP_GPU_USAGE, generator.ocp_report_generation) + + ros_usage_config = generator.ocp_report_generation[OCP_ROS_USAGE] + self.assertIn("_generate_hourly_data", ros_usage_config) + self.assertIn("_update_data", ros_usage_config) + + ros_namespace_config = generator.ocp_report_generation[OCP_ROS_NAMESPACE_USAGE] + self.assertIn("_generate_hourly_data", ros_namespace_config) + self.assertIn("_update_data", ros_namespace_config) + def test_ocp_report_type_to_cols_includes_namespace_usage(self): """Test that OCP_REPORT_TYPE_TO_COLS includes namespace usage mapping and verify separation.""" # Test merged dictionary includes namespace usage diff --git a/tests/test_report.py b/tests/test_report.py index 7c3aec15..b1ec4d02 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -33,6 +33,16 @@ from dateutil.relativedelta import relativedelta from nise.__main__ import fix_dates +from nise.generators.ocp import ( + OCP_ROS_USAGE, + OCP_ROS_NAMESPACE_USAGE, + OCP_POD_USAGE, + OCP_STORAGE_USAGE, + OCP_NODE_LABEL, + OCP_NAMESPACE_LABEL, + OCP_VM_USAGE, + OCP_GPU_USAGE, +) from nise.generators.ocp.ocp_generator import COST_OCP_REPORT_TYPE_TO_COLS from nise.generators.ocp.ocp_generator import OCP_REPORT_TYPE_TO_COLS from nise.report import _convert_bytes @@ -1060,6 +1070,49 @@ def test_ocp_create_report(self): self.assertTrue(os.path.isfile(expected_month_output_file)) os.remove(expected_month_output_file) + def test_ocp_create_report_ros_only(self): + """Test the ocp report creation with ros_only flag - should create ONLY ROS reports.""" + now = datetime.datetime.now().replace(microsecond=0, second=0, minute=0, hour=0) + one_day = datetime.timedelta(days=1) + yesterday = now - one_day + cluster_id = "11112222" + options = { + "start_date": yesterday, + "end_date": now, + "ocp_cluster_id": cluster_id, + "write_monthly": True, + "ros_only": True, + } + fix_dates(options, "ocp") + ocp_create_report(options) + + # Verify ONLY ROS reports were created + ros_report_types = [OCP_ROS_USAGE, OCP_ROS_NAMESPACE_USAGE] + + for report_type in ros_report_types: + month_output_file_name = f"{calendar.month_name[now.month]}-{now.year}-{cluster_id}-{report_type}" + expected_month_output_file = f"{os.getcwd()}/{month_output_file_name}.csv" + self.assertTrue(os.path.isfile(expected_month_output_file), f"Expected ROS report {report_type} to exist") + os.remove(expected_month_output_file) + + # Verify standard reports were NOT created + standard_report_types = [ + OCP_POD_USAGE, + OCP_STORAGE_USAGE, + OCP_NODE_LABEL, + OCP_NAMESPACE_LABEL, + OCP_VM_USAGE, + OCP_GPU_USAGE, + ] + + for report_type in standard_report_types: + month_output_file_name = f"{calendar.month_name[now.month]}-{now.year}-{cluster_id}-{report_type}" + expected_month_output_file = f"{os.getcwd()}/{month_output_file_name}.csv" + self.assertFalse( + os.path.isfile(expected_month_output_file), + f"Standard report {report_type} should NOT exist with ros_only=True", + ) + def test_ocp_create_report_ros_ocp_constant_data_generation(self): """Test the ocp report creation method with constant_values_ros_ocp enabled.""" now = datetime.datetime.now().replace(microsecond=0, second=0, minute=0, hour=0)