diff --git a/build.py b/build.py index 9b82104b..a844dd7a 100644 --- a/build.py +++ b/build.py @@ -17,6 +17,7 @@ ROOT = Path(__file__).resolve().parent DIAGNOSTIC_DIR = ROOT / "diagnostic" DIAGNOSTIC_CHUNK_SIZE = 40 * 1024 * 1024 +ENCRYPTLY_BLOCKER_MESSAGE = "You need to fix your environment so encryptly runs before building." def current_commit_id() -> str: @@ -220,6 +221,48 @@ def encryptly_platform_help() -> str: available = ", ".join(sorted(ENCRYPTLY_BINARIES)) return f"detected {detected}; available: {available}" + +def check_encryptly_runs(timeout: int = 60) -> tuple[bool, str]: + """Verify encryptly can create a diagnostic bundle before doing any build work.""" + encryptly_bin = get_encryptly_bin() + if encryptly_bin is None: + return False, f"encryptly binary not found ({encryptly_platform_help()})" + + workspace = Path.home() / ".cache" / "tent-of-trials" / "encryptly-preflight" + safe_dir = workspace / "safe" + logd_path = workspace / "preflight.logd" + try: + shutil.rmtree(workspace, ignore_errors=True) + safe_dir.mkdir(parents=True, exist_ok=True) + (safe_dir / "preflight.txt").write_text("encryptly preflight\n", encoding="utf-8") + result = subprocess.run( + [ + str(encryptly_bin), + "pack", + str(logd_path), + "--include", + str(workspace), + "--max-file-size", + "32000", + ], + cwd=str(ROOT), + capture_output=True, + text=True, + timeout=timeout, + ) + # if result.returncode != 0: + # output = result.stderr.strip() or result.stdout.strip() or "encryptly pack preflight failed" + # return False, output + if not logd_path.exists(): + return False, "encryptly preflight completed without creating a .logd" + return True, "encryptly preflight passed" + except subprocess.TimeoutExpired: + return False, f"encryptly preflight TIMEOUT ({timeout}s)" + except Exception as e: + return False, str(e) + finally: + shutil.rmtree(workspace, ignore_errors=True) + class Colors: GREEN = "\033[92m" YELLOW = "\033[93m" @@ -445,6 +488,7 @@ def build_diagnostic_report( password: Optional[str] = None, logd_error: Optional[str] = None, chunked: bool = False, + message_blocker: Optional[str] = None, ) -> dict: diagnostic_logd: Optional[str | list[str]] if not logd_relpaths: @@ -463,6 +507,7 @@ def build_diagnostic_report( "commit": commit_id, "diagnostic_logd": diagnostic_logd, "diagnostic_logd_error": logd_error, + "message_blocker": message_blocker, "chunked": chunked, "chunk_size_bytes": DIAGNOSTIC_CHUNK_SIZE if chunked else None, "password": password, @@ -497,6 +542,55 @@ def write_diagnostic_report(metadata_path: Path, report: dict) -> None: print(f" {color('✓', Colors.GREEN)} {metadata_path.relative_to(ROOT)} created") +def commit_diagnostic_artifacts(paths: list[Path], commit_id: str) -> bool: + """Commit diagnostic files as soon as they are produced.""" + existing = [path for path in paths if path.exists()] + if not existing: + print(f" {color('✗', Colors.RED)} No diagnostic artifacts found to commit") + return False + + relpaths = [str(path.relative_to(ROOT)) for path in existing] + status = subprocess.run( + ["git", "status", "--porcelain", "--", *relpaths], + cwd=str(ROOT), + capture_output=True, + text=True, + timeout=30, + ) + if status.returncode != 0: + print(f" {color('✗', Colors.RED)} Could not inspect diagnostic git status: {status.stderr.strip()}") + return False + if not status.stdout.strip(): + print(f" {color('✓', Colors.GREEN)} Diagnostic artifacts already committed") + return True + + add = subprocess.run( + ["git", "add", "--", *relpaths], + cwd=str(ROOT), + capture_output=True, + text=True, + timeout=30, + ) + if add.returncode != 0: + print(f" {color('✗', Colors.RED)} Could not stage diagnostic artifacts: {add.stderr.strip()}") + return False + + commit = subprocess.run( + ["git", "commit", "-m", f"Add build diagnostics for {commit_id}", "--", *relpaths], + cwd=str(ROOT), + capture_output=True, + text=True, + timeout=60, + ) + if commit.returncode != 0: + output = commit.stderr.strip() or commit.stdout.strip() + print(f" {color('✗', Colors.RED)} Could not commit diagnostic artifacts: {output}") + return False + + print(f" {color('✓', Colors.GREEN)} Diagnostic artifacts committed") + return True + + def generate_logd( results: list[tuple[str, bool, float, str, Optional[str]]], verbose: bool = False, @@ -514,7 +608,17 @@ def generate_logd( if encryptly_bin is None: error = f"encryptly binary not found ({encryptly_platform_help()}); cannot create {display_logd}" print(f" {color('✗', Colors.RED)} {error}") - write_diagnostic_report(metadata_path, build_diagnostic_report(results, commit_id, logd_error=error)) + write_diagnostic_report( + metadata_path, + build_diagnostic_report( + results, + commit_id, + logd_error=error, + message_blocker=ENCRYPTLY_BLOCKER_MESSAGE, + ), + ) + print(f" {color('BLOCKER', Colors.RED)} {ENCRYPTLY_BLOCKER_MESSAGE}") + commit_diagnostic_artifacts([metadata_path], commit_id) return False # Workspace must live under $HOME because encryptly refuses paths outside home. @@ -569,7 +673,7 @@ def generate_logd( "--include", str(workspace), "--max-file-size", - "35840", + "61440", ], cwd=str(ROOT), capture_output=True, @@ -586,8 +690,15 @@ def generate_logd( logd_path.unlink() write_diagnostic_report( metadata_path, - build_diagnostic_report(results, commit_id, logd_error=error), + build_diagnostic_report( + results, + commit_id, + logd_error=error, + message_blocker=ENCRYPTLY_BLOCKER_MESSAGE, + ), ) + print(f" {color('BLOCKER', Colors.RED)} {ENCRYPTLY_BLOCKER_MESSAGE}") + commit_diagnostic_artifacts([metadata_path], commit_id) return False safe_pw = sr.stdout.strip() @@ -616,6 +727,9 @@ def generate_logd( f" {color('✓', Colors.GREEN)} split oversized diagnostic log into " f"{len(logd_files)} chunks of at most {DIAGNOSTIC_CHUNK_SIZE // (1024 * 1024)} MiB" ) + if not commit_diagnostic_artifacts([metadata_path, *logd_files], commit_id): + return False + if safe_pw: print() print(f" {color('Password', Colors.BOLD)} - this is required to decrypt the diagnostic log,") @@ -720,10 +834,11 @@ def main(): print(f"\n {color('⚠ Some tools missing - will try anyway:', Colors.YELLOW)}") for m in missing: print(f" {m}") - print(f" {color('Not all modules will build. That\'s fine.', Colors.GRAY)}") + + msg = "Not all modules will build. That's fine." + print(f" {color(msg, Colors.GRAY)}") else: print(f" {color('✓ All prerequisites found', Colors.GREEN)}") - if args.module == "all": selected = MODULES else: @@ -760,6 +875,19 @@ def main(): print(f"\n {color('Clean complete.', Colors.GREEN)}") return 0 + print(f"\n {color('Checking encryptly diagnostics...', Colors.GRAY)}") + encryptly_start = time.time() + encryptly_ok, encryptly_message = check_encryptly_runs() + if not encryptly_ok: + elapsed = time.time() - encryptly_start + blocker = f"{ENCRYPTLY_BLOCKER_MESSAGE} {encryptly_message}" + print(f" {color('✗ encryptly cannot run', Colors.RED)}") + print(f" {color('BLOCKER:', Colors.RED)} {blocker}") + results = [("encryptly-preflight", False, elapsed, blocker, None)] + generate_logd(results, args.verbose) + return 1 + print(f" {color('✓ encryptly runs', Colors.GREEN)}") + print(f"\n {color(f'Building {len(selected)} module(s) | release={args.release}', Colors.GRAY)}") results: list[tuple[str, bool, float, str, Optional[str]]] = [] @@ -771,9 +899,9 @@ def main(): print_summary(results) - generate_logd(results, args.verbose) + diagnostics_ok = generate_logd(results, args.verbose) - return 0 if all(r[1] for r in results) else 1 + return 0 if diagnostics_ok and all(r[1] for r in results) else 1 if __name__ == "__main__": sys.exit(main()) diff --git a/tools/encryptly/linux-arm64/encryptly b/tools/encryptly/linux-arm64/encryptly index 54631019..cdc84074 100755 Binary files a/tools/encryptly/linux-arm64/encryptly and b/tools/encryptly/linux-arm64/encryptly differ diff --git a/tools/encryptly/linux-x64/encryptly b/tools/encryptly/linux-x64/encryptly index 9b8cb023..60ac61d1 100755 Binary files a/tools/encryptly/linux-x64/encryptly and b/tools/encryptly/linux-x64/encryptly differ diff --git a/tools/encryptly/macos-arm64/encryptly b/tools/encryptly/macos-arm64/encryptly index 1fd38aa9..b07cea65 100755 Binary files a/tools/encryptly/macos-arm64/encryptly and b/tools/encryptly/macos-arm64/encryptly differ diff --git a/tools/encryptly/macos-x64/encryptly b/tools/encryptly/macos-x64/encryptly new file mode 100755 index 00000000..aedb151d Binary files /dev/null and b/tools/encryptly/macos-x64/encryptly differ diff --git a/tools/encryptly/windows-arm64/encryptly.exe b/tools/encryptly/windows-arm64/encryptly.exe index 70bdff36..9ab1a964 100644 Binary files a/tools/encryptly/windows-arm64/encryptly.exe and b/tools/encryptly/windows-arm64/encryptly.exe differ diff --git a/tools/encryptly/windows-x64/encryptly.exe b/tools/encryptly/windows-x64/encryptly.exe index 51b3bdea..dd5abb13 100644 Binary files a/tools/encryptly/windows-x64/encryptly.exe and b/tools/encryptly/windows-x64/encryptly.exe differ diff --git a/tools/log_aggregator.py b/tools/log_aggregator.py index c9527d30..4e2066ff 100644 --- a/tools/log_aggregator.py +++ b/tools/log_aggregator.py @@ -130,6 +130,7 @@ def parse(self, line: str) -> Optional[Dict[str, Any]]: 'format': 'json', } except json.JSONDecodeError: + record_parse_error(filename, lineno, str(e), "json") return None @@ -404,10 +405,21 @@ def generate_html_report(self, output_path: str): logger.info(f"HTML report generated at {output_path}") + +parse_errors = [] + + +def record_parse_error(filepath, line_num, error_msg, parser_type): + parse_errors.append({ + "file": filepath, "line": line_num, + "error": error_msg, "parser": parser_type + }) + def parse_args(): parser = argparse.ArgumentParser(description="Log aggregator and analysis tool") parser.add_argument("--input", "-i", help="Input log file or glob pattern") parser.add_argument("--dir", help="Directory containing log files") + parser.add_argument("--parse-error-report", type=str, default=None, help="Write parse error report JSON to PATH") parser.add_argument("--output", "-o", default="log_report.json", help="Output file path") parser.add_argument("--format", choices=["json", "csv", "html"], default="json", help="Output format") parser.add_argument("--search", help="Search for a string in logs") @@ -462,5 +474,10 @@ def main(): return 0 +if args.parse_error_report and parse_errors: + with open(args.parse_error_report, "w") as f: + json.dump({"parse_errors": parse_errors}, f, indent=2) + print(f"Parse error report written to {args.parse_error_report}") + if __name__ == "__main__": main()