Skip to content

Report writing utility file #978

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kyle-hoffmeyer wants to merge 9 commits into
base: main
Choose a base branch
from
Open

Report writing utility file #978

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
eb0d698
create report writing utility file
kyle-hoffmeyer Aug 26, 2026
234e536
add columns to hide param
kyle-hoffmeyer Aug 28, 2026
841fa1d
add skip empty df flag
kyle-hoffmeyer Aug 28, 2026
f034025
move imports
kyle-hoffmeyer Aug 28, 2026
3567ebb
add test file
kyle-hoffmeyer Aug 28, 2026
c376aaf
remove comment
kyle-hoffmeyer Aug 28, 2026
8ecf927
remove unused imports
kyle-hoffmeyer Aug 31, 2026
e06194b
codecov
kyle-hoffmeyer Sep 1, 2026
d90f5c1
Merge branch 'main' into cleanup/report_writing_utility
kyle-hoffmeyer Sep 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 8 additions & 26 deletions TraceLens/Reporting/compare_perf_reports_pytorch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@
from typing import Dict, List, Optional, Sequence

import pandas as pd
from openpyxl.utils import get_column_letter

from TraceLens.Reporting.reporting_utils import write_report_outputs

# ──────────────────────────────────────────────────────────────────────────────
# Configuration
Expand Down Expand Up @@ -588,31 +589,12 @@ def generate_compare_perf_reports_pytorch(
cols_to_hide_xl[sheet_name] = cols_to_hide

# ── Write workbook ────────────────────────────────────────────────────────
if output_csvs_dir:
os.makedirs(output_csvs_dir, exist_ok=True)
for sheet_name, df in results.items():
csv_path = os.path.join(output_csvs_dir, f"{sheet_name}.csv")
df.to_csv(csv_path, index=False)
print(
f"Wrote '{sheet_name}.csv' with {len(df)} rows × {len(df.columns)} columns"
)

if output is not None:
with pd.ExcelWriter(output, engine="openpyxl") as xls:
for sheet_name, df in results.items():
if df.empty:
print(f"Sheet '{sheet_name}' is empty (no matching rows)")
df.to_excel(
xls, sheet_name=sheet_name[:31], index=False
) # Excel 31-char limit
for col in cols_to_hide_xl.get(sheet_name, []):
col_idx = df.columns.get_loc(col) + 1
col_letter = get_column_letter(col_idx)
worksheet = xls.sheets[sheet_name[:31]]
worksheet.column_dimensions[col_letter].hidden = True
print(
f"Wrote sheet '{sheet_name}' with {len(df)} rows × {len(df.columns)} columns"
)
write_report_outputs(
results,
xlsx_path=output,
csvs_dir=output_csvs_dir,
hide_columns=cols_to_hide_xl,
)

return results

Expand Down
25 changes: 4 additions & 21 deletions TraceLens/Reporting/generate_multi_rank_collective_report_pytorch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
# See LICENSE for license information.
###############################################################################

import importlib.util
import os
import re
import argparse
Expand All @@ -16,7 +15,7 @@
from TraceLens.Reporting.reporting_utils import (
add_node_span_columns,
detect_gpus_per_node,
request_install,
write_report_outputs,
)

DEFAULT_RANK_REGEX = r"rank[\[\-_/]?(?P<rank>\d+)"
Expand Down Expand Up @@ -257,25 +256,9 @@ def generate_collective_report(
)

# Export DataFrames
if output_csvs_dir:
os.makedirs(output_csvs_dir, exist_ok=True)
for sheet_name, df in report_dfs.items():
csv_path = os.path.join(output_csvs_dir, f"{sheet_name}.csv")
df.to_csv(csv_path, index=False)
print(f"DataFrame '{sheet_name}' written to {csv_path}")

if output_xlsx_path:
if importlib.util.find_spec("openpyxl") is None:
print("Error importing openpyxl")
request_install("openpyxl")

print(f"Writing Excel report to {output_xlsx_path}...")
with pd.ExcelWriter(output_xlsx_path, engine="openpyxl") as writer:
for sheet_name, df in report_dfs.items():
df.to_excel(
writer, sheet_name=sheet_name[:31], index=False
) # Excel limits sheet names to 31 chars
print(f"Excel report successfully written to {output_xlsx_path}")
write_report_outputs(
report_dfs, xlsx_path=output_xlsx_path, csvs_dir=output_csvs_dir
)

return report_dfs

Expand Down
30 changes: 7 additions & 23 deletions TraceLens/Reporting/generate_perf_report_genesis.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
pftrace_to_json,
resolve_profile_json,
)
from TraceLens.Reporting.reporting_utils import write_report_outputs

logging.basicConfig(
stream=sys.stdout,
Expand All @@ -45,30 +46,13 @@
logger = logging.getLogger(__name__)


def _safe_sheet(name: str, used: set) -> str:
base = name[:31]
n = 0
while base in used:
n += 1
suffix = f"_{n}"
base = name[: 31 - len(suffix)] + suffix
used.add(base)
return base


def write_excel(path: Path, sections: Dict[str, Dict[str, pd.DataFrame]]) -> None:
used: set = set()
with pd.ExcelWriter(path, engine="openpyxl") as writer:
for prefix, dfs in sections.items():
for sheet, df in dfs.items():
if df is None or df.empty:
continue
# rocprof sheets use short names (no prefix); pftrace keeps prefix for clarity
sheet_label = sheet if prefix == "rocprof" else f"{prefix}_{sheet}"
df.to_excel(
writer, sheet_name=_safe_sheet(sheet_label, used), index=False
)
logger.info("Wrote %s", path)
flat: Dict[str, pd.DataFrame] = {}
for prefix, dfs in sections.items():
for sheet, df in dfs.items():
label = sheet if prefix == "rocprof" else f"{prefix}_{sheet}"
flat[label] = df
write_report_outputs(flat, xlsx_path=str(path), skip_empty=True)


def _rocprof_sheets_for_excel(
Expand Down
32 changes: 9 additions & 23 deletions TraceLens/Reporting/generate_perf_report_jax.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,6 @@
###############################################################################

import argparse
import importlib.util
import os
import sys
from typing import Optional, Dict
import pandas as pd
Expand All @@ -22,8 +20,8 @@
from TraceLens.TreePerf import JaxTreePerfAnalyzer
from TraceLens.Reporting.reporting_utils import (
add_gpu_arch_cli_args,
request_install,
resolve_gpu_arch,
write_report_outputs,
)
from TraceLens.util import TraceEventUtils

Expand Down Expand Up @@ -165,26 +163,14 @@ def generate_perf_report_jax(
)

# Write all DataFrames to separate sheets in an Excel workbook
if output_csvs_dir:
# Ensure the output directory exists
os.makedirs(output_csvs_dir, exist_ok=True)
for sheet_name, df in dict_name2df.items():
csv_path = os.path.join(output_csvs_dir, f"{sheet_name}.csv")
df.to_csv(csv_path, index=False)
print(f"DataFrame '{sheet_name}' written to {csv_path}")
else:
if output_xlsx_path is None:
# split input path at 'xplane.pb' and take the first part and append '.xlsx'
base_path = profile_path.rsplit(".xplane.pb", 1)[0]
output_xlsx_path = base_path + "_perf_report.xlsx"
if importlib.util.find_spec("openpyxl") is None:
print("Error importing openpyxl")
request_install("openpyxl")

with pd.ExcelWriter(output_xlsx_path, engine="openpyxl") as writer:
for sheet_name, df in dict_name2df.items():
df.to_excel(writer, sheet_name=sheet_name, index=False)
print(f"DataFrames successfully written to {output_xlsx_path}")
if not output_csvs_dir and output_xlsx_path is None:
base_path = profile_path.rsplit(".xplane.pb", 1)[0]
output_xlsx_path = base_path + "_perf_report.xlsx"
write_report_outputs(
dict_name2df,
xlsx_path=output_xlsx_path,
csvs_dir=output_csvs_dir,
)

return dict_name2df

Expand Down
43 changes: 14 additions & 29 deletions TraceLens/Reporting/generate_perf_report_pftrace_hip_activity.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
API↔kernel correlation (see generate_perf_report_pftrace_hip_api for that).
"""

import importlib.util
import os
import argparse
import sys
Expand All @@ -29,7 +28,11 @@
logger = logging.getLogger(__name__)

from TraceLens.util import PftraceParser
from TraceLens.Reporting.pftrace_utils import ensure_trace_json
from TraceLens.Reporting.pftrace_utils import (
derive_pftrace_output_path,
ensure_trace_json,
)
from TraceLens.Reporting.reporting_utils import write_report_outputs
from TraceLens.Reporting.pftrace_hip_activity_analysis import (
PftraceHipActivityAnalyzer,
ns_to_ms,
Expand Down Expand Up @@ -154,33 +157,15 @@ def generate_perf_report_pftrace_hip_activity(
dict_name2df["hip_summary"] = analyzer.get_df_hip_summary()
logger.info(" - hip_summary (%d rows)", len(dict_name2df["hip_summary"]))

if output_csvs_dir:
logger.info("Writing CSV files to: %s", output_csvs_dir)
os.makedirs(output_csvs_dir, exist_ok=True)
for sheet_name, df in dict_name2df.items():
csv_path = os.path.join(output_csvs_dir, f"{sheet_name}.csv")
df.to_csv(csv_path, index=False)
logger.info(" - %s.csv (%d rows)", sheet_name, len(df))
else:
if output_xlsx_path is None:
base = Path(trace_path).resolve()
if base.suffix.lower() == ".pftrace":
base = base.with_suffix("")
elif base.suffix.lower() == ".gz" and base.name.endswith(".json.gz"):
base = base.parent / base.name.replace(".json.gz", "")
else:
base = base.with_suffix("")
output_xlsx_path = str(base) + "_pftrace_activity_report.xlsx"
logger.info("Writing Excel to: %s", output_xlsx_path)
if importlib.util.find_spec("openpyxl") is None:
logger.error("openpyxl required for Excel output")
raise ImportError("openpyxl is required for Excel output")
with pd.ExcelWriter(output_xlsx_path, engine="openpyxl") as writer:
for sheet_name, df in dict_name2df.items():
sn = sheet_name[:31]
df.to_excel(writer, sheet_name=sn, index=False)
logger.info(" - Sheet '%s' (%d rows)", sn, len(df))
logger.info("Successfully written to %s", output_xlsx_path)
if not output_csvs_dir and output_xlsx_path is None:
output_xlsx_path = derive_pftrace_output_path(
trace_path, "_pftrace_activity_report.xlsx"
)
write_report_outputs(
dict_name2df,
xlsx_path=output_xlsx_path,
csvs_dir=output_csvs_dir,
)

if output_md_path:
logger.info("Writing Markdown to: %s", output_md_path)
Expand Down
48 changes: 14 additions & 34 deletions TraceLens/Reporting/generate_perf_report_pftrace_hip_api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,10 @@
# See LICENSE for license information.
###############################################################################

import importlib.util
import os
import re
import argparse
import sys
from pathlib import Path
from typing import Optional, Dict

import pandas as pd
Expand All @@ -24,7 +22,11 @@
logger = logging.getLogger(__name__)

from TraceLens.util import PftraceParser
from TraceLens.Reporting.pftrace_utils import ensure_trace_json
from TraceLens.Reporting.pftrace_utils import (
derive_pftrace_output_path,
ensure_trace_json,
)
from TraceLens.Reporting.reporting_utils import write_report_outputs
from TraceLens.Reporting.pftrace_hip_api_analysis import PftraceHipApiAnalyzer


Expand Down Expand Up @@ -99,37 +101,15 @@ def generate_perf_report_pftrace_hip_api(
)

# Write output
if output_csvs_dir:
logger.info(f"Writing CSV files to: {output_csvs_dir}")
os.makedirs(output_csvs_dir, exist_ok=True)
for sheet_name, df in dict_name2df.items():
csv_path = os.path.join(output_csvs_dir, f"{sheet_name}.csv")
df.to_csv(csv_path, index=False)
logger.info(f" - {sheet_name}.csv ({len(df)} rows)")
else:
if output_xlsx_path is None:
base = Path(trace_path).resolve()
if base.suffix.lower() == ".pftrace":
base = base.with_suffix("")
elif base.suffix.lower() == ".gz" and base.name.endswith(".json.gz"):
base = base.parent / base.name.replace(".json.gz", "")
else:
base = base.with_suffix("")
output_xlsx_path = str(base) + "_pftrace_hip_api_report.xlsx"

logger.info(f"Writing Excel file to: {output_xlsx_path}")
if importlib.util.find_spec("openpyxl") is None:
logger.error(
"Error importing openpyxl. Please install: pip install openpyxl"
)
raise ImportError("openpyxl is required for Excel output")

with pd.ExcelWriter(output_xlsx_path, engine="openpyxl") as writer:
for sheet_name, df in dict_name2df.items():
df.to_excel(writer, sheet_name=sheet_name, index=False)
logger.info(f" - Sheet '{sheet_name}' ({len(df)} rows)")

logger.info(f"Successfully written to {output_xlsx_path}")
if not output_csvs_dir and output_xlsx_path is None:
output_xlsx_path = derive_pftrace_output_path(
trace_path, "_pftrace_hip_api_report.xlsx"
)
write_report_outputs(
dict_name2df,
xlsx_path=output_xlsx_path,
csvs_dir=output_csvs_dir,
)

return dict_name2df

Expand Down
43 changes: 14 additions & 29 deletions TraceLens/Reporting/generate_perf_report_pftrace_memory_copy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,9 @@
Uses shared pftrace_utils (traceconv) and PftraceParser.
"""

import importlib.util
import os
import argparse
import sys
from pathlib import Path
from typing import Optional, Dict, List, Any, Tuple

import pandas as pd
Expand All @@ -29,7 +27,11 @@
logger = logging.getLogger(__name__)

from TraceLens.util import PftraceParser
from TraceLens.Reporting.pftrace_utils import ensure_trace_json
from TraceLens.Reporting.pftrace_utils import (
derive_pftrace_output_path,
ensure_trace_json,
)
from TraceLens.Reporting.reporting_utils import write_report_outputs

# Event name substrings for direction (ROCm/Perfetto)
NAME_HOST_TO_DEVICE = "MEMORY_COPY_HOST_TO_DEVICE"
Expand Down Expand Up @@ -146,32 +148,15 @@ def generate_perf_report_pftrace_memory_copy(
count_df = build_memory_copy_count_df(events)
dfs = {"memory_copy_by_copy_bytes": count_df}

if output_csvs_dir:
logger.info("Writing CSV files to: %s", output_csvs_dir)
os.makedirs(output_csvs_dir, exist_ok=True)
for sheet_name, df in dfs.items():
csv_path = os.path.join(output_csvs_dir, f"{sheet_name}.csv")
df.to_csv(csv_path, index=False)
logger.info(" - %s.csv (%d rows)", sheet_name, len(df))
else:
if output_xlsx_path is None:
base = Path(trace_path).resolve()
if base.suffix.lower() == ".pftrace":
base = base.with_suffix("")
elif base.suffix.lower() == ".gz" and base.name.endswith(".json.gz"):
base = base.parent / base.name.replace(".json.gz", "")
else:
base = base.with_suffix("")
output_xlsx_path = str(base) + "_pftrace_memory_copy_report.xlsx"
logger.info("Writing Excel file to: %s", output_xlsx_path)
if importlib.util.find_spec("openpyxl") is None:
logger.error("openpyxl required for Excel output. pip install openpyxl")
raise ImportError("openpyxl is required for Excel output")
with pd.ExcelWriter(output_xlsx_path, engine="openpyxl") as writer:
for sheet_name, df in dfs.items():
df.to_excel(writer, sheet_name=sheet_name, index=False)
logger.info(" - Sheet '%s' (%d rows)", sheet_name, len(df))
logger.info("Successfully written to %s", output_xlsx_path)
if not output_csvs_dir and output_xlsx_path is None:
output_xlsx_path = derive_pftrace_output_path(
trace_path, "_pftrace_memory_copy_report.xlsx"
)
write_report_outputs(
dfs,
xlsx_path=output_xlsx_path,
csvs_dir=output_csvs_dir,
)

return dfs

Expand Down
Loading
Loading