diff --git a/minio-report-tracker/scripts/update_csv.py b/minio-report-tracker/scripts/update_csv.py index 1d16bec..b200901 100644 --- a/minio-report-tracker/scripts/update_csv.py +++ b/minio-report-tracker/scripts/update_csv.py @@ -1,217 +1,300 @@ +import json import os import re -import json import subprocess +from datetime import datetime, timedelta, timezone + import pandas as pd -from datetime import datetime, timezone, timedelta + try: - from zoneinfo import ZoneInfo + from zoneinfo import ZoneInfo + IST = ZoneInfo("Asia/Kolkata") except Exception: - # Fallback if zoneinfo/tzdata is unavailable IST = timezone(timedelta(hours=5, minutes=30)) -MINIO_ALIASES = ["cellbox21", "collab", "released", "qa-base", "synergy1", "dev01","dev", "devint1", "qa21", "dev11", "qa11new", "qaj21", "qa11"] + +env_name = os.getenv("MINIO_ENV") or os.getenv("MINIO_ALIAS") +env_list = os.getenv("MINIO_ENVS") + +if env_name: + MINIO_ALIASES = [env_name] +elif env_list: + MINIO_ALIASES = json.loads(env_list) +else: + MINIO_ALIASES = [] + + MINIO_BUCKETS = ["apitestrig", "automation", "dslreports", "uitestrig"] columns = ["Date", "Module", "T", "P", "S", "F", "I", "KI"] +failed_aliases = [] +successful_aliases = [] +failure_reasons = {} + + +def log(alias, message): + print(f"[{alias}] {message}") + + def format_date_str(dt): return dt.strftime("%d-%B-%Y") + def date_key_from_minio_ts(ts: str) -> str: - """ - Convert MinIO 'lastModified' (UTC, e.g. '2025-08-13T21:25:10.000Z') - to an IST date key like '14-August-2025'. - """ - # Normalize 'Z' to '+00:00' so fromisoformat can parse - ts_norm = ts.replace('Z', '+00:00') + ts_norm = ts.replace("Z", "+00:00") try: dt = datetime.fromisoformat(ts_norm) except ValueError: - # Fallback: parse without fractional seconds dt = datetime.strptime(ts_norm[:19], "%Y-%m-%dT%H:%M:%S").replace(tzinfo=timezone.utc) dt_ist = dt.astimezone(IST) return format_date_str(dt_ist) -for alias in MINIO_ALIASES: - csv_filename = f"{alias}.csv" - all_data_by_date = {} - for bucket in MINIO_BUCKETS: - folders = [] +def run_mc_json_lines(alias, command): + output = subprocess.getoutput(command) + lines = output.splitlines() + entries = [] - # --- Special handling for dslreports --- - if bucket == "dslreports": - cmd = f"mc ls --json {alias}/dslreports/full/" - output = subprocess.getoutput(cmd) - for line in output.splitlines(): - try: - info = json.loads(line) - fn = info["key"] - if fn.startswith("ExtentReport-"): - continue - ts = info["lastModified"] - date_key = date_key_from_minio_ts(ts) - except (json.JSONDecodeError, KeyError, ValueError): - continue + for line in lines: + try: + entries.append(json.loads(line)) + except json.JSONDecodeError: + if line.strip(): + log(alias, f"Skipping non-JSON mc output: {line[:200]}") - if not re.search(r"report_T-\d+_P-\d+(?:_KI-\d+)?(?:_I-\d+)?_S-\d+_F-\d+", fn): - continue + return entries - m = re.search(r"report_T-(\d+)_P-(\d+)(?:_KI-(\d+))?(?:_I-(\d+))?_S-(\d+)_F-(\d+)", fn) - if not m: - continue - T, P, KI, I, S, F = m.groups() - I = I or "0" - KI = KI or "0" +def append_row(all_data_by_date, date_key, module, row): + all_data_by_date.setdefault(date_key, []) + if not any(existing[1] == module for existing in all_data_by_date[date_key]): + all_data_by_date[date_key].append(row) - if date_key not in all_data_by_date: - all_data_by_date[date_key] = [] - if not any(row[1] == "dsl" for row in all_data_by_date[date_key]): - all_data_by_date[date_key].append([date_key, "dsl", T, P, S, F, I, KI]) - continue - # --- Special handling for uitestrig - ADMINUI, RESIDENT directly --- - if bucket == "uitestrig": - cmd = f"mc ls --json {alias}/{bucket}/" - lines = subprocess.getoutput(cmd).splitlines() - for line in lines: - try: - info = json.loads(line) - fn = info["key"] - ts = info["lastModified"] - date_key = date_key_from_minio_ts(ts) - except (json.JSONDecodeError, KeyError, ValueError): - continue +for alias in MINIO_ALIASES: + try: + log(alias, "Starting MinIO scan") + csv_filename = f"{alias}.csv" + all_data_by_date = {} + matched_rows = 0 + bucket_stats = {bucket: 0 for bucket in MINIO_BUCKETS} - if not re.search(r"-report_T-\d+_P-\d+_S-\d+_F-\d+\.html$", fn): - continue + for bucket in MINIO_BUCKETS: + folders = [] - mod_match = re.match(r"(ADMINUI|RESIDENT)-api-", fn) - if mod_match: - mod_raw = mod_match.group(1).lower() - mod = "residentui" if mod_raw == "resident" else mod_raw - m = re.search(r"report_T-(\d+)_P-(\d+)_S-(\d+)_F-(\d+)", fn) - if not m: + if bucket == "dslreports": + entries = run_mc_json_lines(alias, f"mc ls --json {alias}/dslreports/full/") + log(alias, f"dslreports/full entries scanned: {len(entries)}") + + for info in entries: + try: + fn = info["key"] + if fn.startswith("ExtentReport-"): + continue + ts = info["lastModified"] + date_key = date_key_from_minio_ts(ts) + except (KeyError, ValueError): continue - T, P, S, F = m.groups() - I, KI = "0", "0" - - if date_key not in all_data_by_date: - all_data_by_date[date_key] = [] - if not any(row[1] == mod for row in all_data_by_date[date_key]): - all_data_by_date[date_key].append([date_key, mod, T, P, S, F, I, KI]) - - # Do not 'continue' here — allow folder logic to run for PMPUI - - # --- List folders --- - cmd = f"mc ls --json {alias}/{bucket}/" - output = subprocess.getoutput(cmd) - for line in output.splitlines(): - try: - folders.append(json.loads(line)["key"].strip("/")) - except (json.JSONDecodeError, KeyError): + + if not re.search(r"report_T-\d+_P-\d+(?:_KI-\d+)?(?:_I-\d+)?_S-\d+_F-\d+", fn): + continue + + match = re.search(r"report_T-(\d+)_P-(\d+)(?:_KI-(\d+))?(?:_I-(\d+))?_S-(\d+)_F-(\d+)", fn) + if not match: + continue + + t_val, p_val, ki_val, i_val, s_val, f_val = match.groups() + i_val = i_val or "0" + ki_val = ki_val or "0" + + append_row( + all_data_by_date, + date_key, + "dsl", + [date_key, "dsl", t_val, p_val, s_val, f_val, i_val, ki_val], + ) + matched_rows += 1 + bucket_stats[bucket] += 1 + continue - if not folders: - continue + if bucket == "uitestrig": + root_entries = run_mc_json_lines(alias, f"mc ls --json {alias}/{bucket}/") + log(alias, f"uitestrig root entries scanned: {len(root_entries)}") - for folder in folders: - # --- Special case: PMPUI --- - if bucket == "uitestrig" and folder.lower() == "pmpui": - folder_path = f"{bucket}/{folder}" - cmd = f"mc ls --json {alias}/{folder_path}/" - lines = subprocess.getoutput(cmd).splitlines() - for line in lines: + for info in root_entries: try: - info = json.loads(line) fn = info["key"] ts = info["lastModified"] date_key = date_key_from_minio_ts(ts) - except (json.JSONDecodeError, KeyError, ValueError): + except (KeyError, ValueError): continue - if not re.search(r"PMPUI-.*-report_T-\d+_P-\d+_S-\d+_F-\d+", fn): + if not re.search(r"-report_T-\d+_P-\d+_S-\d+_F-\d+\.html$", fn): continue - m = re.search(r"report_T-(\d+)_P-(\d+)_S-(\d+)_F-(\d+)", fn) - if not m: + + mod_match = re.match(r"(ADMINUI|RESIDENT)-api-", fn) + if not mod_match: continue - T, P, S, F = m.groups() - I, KI = "0", "0" - mod = "pmpui" - - if date_key not in all_data_by_date: - all_data_by_date[date_key] = [] - if not any(row[1] == mod for row in all_data_by_date[date_key]): - all_data_by_date[date_key].append([date_key, mod, T, P, S, F, I, KI]) - continue - # --- Default logic for other folders --- - folder_path = f"{bucket}/{folder}" - cmd = f"mc ls --json {alias}/{folder_path}/" - lines = subprocess.getoutput(cmd).splitlines() - for line in lines: + mod_raw = mod_match.group(1).lower() + module = "residentui" if mod_raw == "resident" else mod_raw + + match = re.search(r"report_T-(\d+)_P-(\d+)_S-(\d+)_F-(\d+)", fn) + if not match: + continue + + t_val, p_val, s_val, f_val = match.groups() + append_row( + all_data_by_date, + date_key, + module, + [date_key, module, t_val, p_val, s_val, f_val, "0", "0"], + ) + matched_rows += 1 + bucket_stats[bucket] += 1 + + folder_entries = run_mc_json_lines(alias, f"mc ls --json {alias}/{bucket}/") + for info in folder_entries: try: - info = json.loads(line) - fn = info["key"] - ts = info["lastModified"] - date_key = date_key_from_minio_ts(ts) - except (json.JSONDecodeError, KeyError, ValueError): + folders.append(info["key"].strip("/")) + except KeyError: continue - if "error-report" in fn: - continue - if not re.search(r"(full-)?report_T-\d+_P-\d+_S-\d+_F-", fn): - continue - m = re.search(r"report_T-(\d+)_P-(\d+)_S-(\d+)_F-(\d+)(?:_I-(\d+))?(?:_KI-(\d+))?", fn) - if not m: + log(alias, f"{bucket} folders discovered: {len(folders)}") + if not folders: + continue + + for folder in folders: + if bucket == "uitestrig" and folder.lower() == "pmpui": + folder_entries = run_mc_json_lines(alias, f"mc ls --json {alias}/{bucket}/{folder}/") + for info in folder_entries: + try: + fn = info["key"] + ts = info["lastModified"] + date_key = date_key_from_minio_ts(ts) + except (KeyError, ValueError): + continue + + if not re.search(r"PMPUI-.*-report_T-\d+_P-\d+_S-\d+_F-\d+", fn): + continue + + match = re.search(r"report_T-(\d+)_P-(\d+)_S-(\d+)_F-(\d+)", fn) + if not match: + continue + + t_val, p_val, s_val, f_val = match.groups() + append_row( + all_data_by_date, + date_key, + "pmpui", + [date_key, "pmpui", t_val, p_val, s_val, f_val, "0", "0"], + ) + matched_rows += 1 + bucket_stats[bucket] += 1 continue - T, P, S, F, I, KI = m.groups() - I = I or "0" - KI = KI or "0" - - if folder == "masterdata": - lang = re.search(r"masterdata-([a-z]{3})", fn) - mod = f"{folder}-{lang.group(1)}" if lang else folder - else: - mod = folder - - if date_key not in all_data_by_date: - all_data_by_date[date_key] = [] - if not any(row[1] == mod for row in all_data_by_date[date_key]): - all_data_by_date[date_key].append([date_key, mod, T, P, S, F, I, KI]) - - # === Only include latest 5 working days (Mon–Fri) === - sorted_dates = sorted( - all_data_by_date.keys(), - key=lambda x: datetime.strptime(x, "%d-%B-%Y"), - reverse=True - ) - weekdays_only = [] - for date_str in sorted_dates: - dt = datetime.strptime(date_str, "%d-%B-%Y") - if dt.weekday() < 5: # Monday–Friday - weekdays_only.append(date_str) - if len(weekdays_only) == 5: - break - - latest_dates = weekdays_only - dfs = [] - for date in latest_dates: - df = pd.DataFrame(all_data_by_date[date], columns=columns) - dfs.append(df) - - if not dfs: + + folder_entries = run_mc_json_lines(alias, f"mc ls --json {alias}/{bucket}/{folder}/") + for info in folder_entries: + try: + fn = info["key"] + ts = info["lastModified"] + date_key = date_key_from_minio_ts(ts) + except (KeyError, ValueError): + continue + + if "error-report" in fn: + continue + if not re.search(r"(full-)?report_T-\d+_P-\d+_S-\d+_F-", fn): + continue + + match = re.search(r"report_T-(\d+)_P-(\d+)_S-(\d+)_F-(\d+)(?:_I-(\d+))?(?:_KI-(\d+))?", fn) + if not match: + continue + + t_val, p_val, s_val, f_val, i_val, ki_val = match.groups() + i_val = i_val or "0" + ki_val = ki_val or "0" + + if folder == "masterdata": + lang = re.search(r"masterdata-([a-z]{3})", fn) + module = f"{folder}-{lang.group(1)}" if lang else folder + else: + module = folder + + append_row( + all_data_by_date, + date_key, + module, + [date_key, module, t_val, p_val, s_val, f_val, i_val, ki_val], + ) + matched_rows += 1 + bucket_stats[bucket] += 1 + + sorted_dates = sorted( + all_data_by_date.keys(), + key=lambda value: datetime.strptime(value, "%d-%B-%Y"), + reverse=True, + ) + weekdays_only = [] + for date_str in sorted_dates: + dt = datetime.strptime(date_str, "%d-%B-%Y") + if dt.weekday() < 5: + weekdays_only.append(date_str) + if len(weekdays_only) == 5: + break + + log(alias, f"Matched rows before dedupe/write: {matched_rows}") + log(alias, f"Bucket stats: {bucket_stats}") + log(alias, f"Available date buckets (IST): {sorted_dates[:10]}") + log(alias, f"Selected latest 5 working days: {weekdays_only}") + + dfs = [] + for date_key in weekdays_only: + dfs.append(pd.DataFrame(all_data_by_date[date_key], columns=columns)) + + if not dfs: + reason = ( + "No usable report data found after parsing MinIO objects and applying the latest 5 working days filter." + ) + log(alias, reason) + failure_reasons[alias] = { + "reason": reason, + "available_dates": sorted_dates[:10], + "bucket_stats": bucket_stats, + } + failed_aliases.append(alias) + continue + + max_len = max(len(df) for df in dfs) + for index in range(len(dfs)): + dfs[index] = dfs[index].reindex(range(max_len)) + if index < len(dfs) - 1: + dfs[index][""] = "" + dfs[index][" "] = "" + + os.makedirs("../csv", exist_ok=True) + final_df = pd.concat(dfs, axis=1) + final_df.to_csv(f"../csv/{csv_filename}", index=False) + successful_aliases.append(alias) + log(alias, f"CSV written to ../csv/{csv_filename}") + + except Exception as exc: + log(alias, f"Failed with exception: {exc}") + failed_aliases.append(alias) + failure_reasons[alias] = {"reason": str(exc)} continue - max_len = max(len(df) for df in dfs) - for i in range(len(dfs)): - dfs[i] = dfs[i].reindex(range(max_len)) - if i < len(dfs) - 1: - dfs[i][""] = "" - dfs[i][" "] = "" - os.makedirs("../csv", exist_ok=True) - final_df = pd.concat(dfs, axis=1) - final_df.to_csv(f"../csv/{csv_filename}", index=False) +os.makedirs("../status", exist_ok=True) + +status = { + "success": successful_aliases, + "failed": failed_aliases, + "details": failure_reasons, +} + +alias_name = env_name if env_name else "all" +with open(f"../status/status_{alias_name}.json", "w", encoding="utf-8") as handle: + json.dump(status, handle, indent=2)