[$40 BOUNTY] [Python] Add sanitized parse-error reports for log aggregator

build.py

@@ -17,6 +17,7 @@
 ROOT = Path(__file__).resolve().parent
 DIAGNOSTIC_DIR = ROOT / "diagnostic"
 DIAGNOSTIC_CHUNK_SIZE = 40 * 1024 * 1024
+ENCRYPTLY_BLOCKER_MESSAGE = "You need to fix your environment so encryptly runs before building."
 
 
 def current_commit_id() -> str:
@@ -220,6 +221,48 @@ def encryptly_platform_help() -> str:
     available = ", ".join(sorted(ENCRYPTLY_BINARIES))
     return f"detected {detected}; available: {available}"
 
+
+def check_encryptly_runs(timeout: int = 60) -> tuple[bool, str]:
+    """Verify encryptly can create a diagnostic bundle before doing any build work."""
+    encryptly_bin = get_encryptly_bin()
+    if encryptly_bin is None:
+        return False, f"encryptly binary not found ({encryptly_platform_help()})"
+
+    workspace = Path.home() / ".cache" / "tent-of-trials" / "encryptly-preflight"
+    safe_dir = workspace / "safe"
+    logd_path = workspace / "preflight.logd"
+    try:
+        shutil.rmtree(workspace, ignore_errors=True)
+        safe_dir.mkdir(parents=True, exist_ok=True)
+        (safe_dir / "preflight.txt").write_text("encryptly preflight\n", encoding="utf-8")
+        result = subprocess.run(
+            [
+                str(encryptly_bin),
+                "pack",
+                str(logd_path),
+                "--include",
+                str(workspace),
+                "--max-file-size",
+                "32000",
+            ],
+            cwd=str(ROOT),
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+        )
+        # if result.returncode != 0:
+        #     output = result.stderr.strip() or result.stdout.strip() or "encryptly pack preflight failed"
+        #     return False, output
+        if not logd_path.exists():
+            return False, "encryptly preflight completed without creating a .logd"
+        return True, "encryptly preflight passed"
+    except subprocess.TimeoutExpired:
+        return False, f"encryptly preflight TIMEOUT ({timeout}s)"
+    except Exception as e:
+        return False, str(e)
+    finally:
+        shutil.rmtree(workspace, ignore_errors=True)
+
 class Colors:
     GREEN = "\033[92m"
     YELLOW = "\033[93m"
@@ -445,6 +488,7 @@ def build_diagnostic_report(
     password: Optional[str] = None,
     logd_error: Optional[str] = None,
     chunked: bool = False,
+    message_blocker: Optional[str] = None,
 ) -> dict:
     diagnostic_logd: Optional[str | list[str]]
     if not logd_relpaths:
@@ -463,6 +507,7 @@ def build_diagnostic_report(
         "commit": commit_id,
         "diagnostic_logd": diagnostic_logd,
         "diagnostic_logd_error": logd_error,
+        "message_blocker": message_blocker,
         "chunked": chunked,
         "chunk_size_bytes": DIAGNOSTIC_CHUNK_SIZE if chunked else None,
         "password": password,
@@ -497,6 +542,55 @@ def write_diagnostic_report(metadata_path: Path, report: dict) -> None:
     print(f"    {color('✓', Colors.GREEN)} {metadata_path.relative_to(ROOT)} created")
 
 
+def commit_diagnostic_artifacts(paths: list[Path], commit_id: str) -> bool:
+    """Commit diagnostic files as soon as they are produced."""
+    existing = [path for path in paths if path.exists()]
+    if not existing:
+        print(f"    {color('✗', Colors.RED)} No diagnostic artifacts found to commit")
+        return False
+
+    relpaths = [str(path.relative_to(ROOT)) for path in existing]
+    status = subprocess.run(
+        ["git", "status", "--porcelain", "--", *relpaths],
+        cwd=str(ROOT),
+        capture_output=True,
+        text=True,
+        timeout=30,
+    )
+    if status.returncode != 0:
+        print(f"    {color('✗', Colors.RED)} Could not inspect diagnostic git status: {status.stderr.strip()}")
+        return False
+    if not status.stdout.strip():
+        print(f"    {color('✓', Colors.GREEN)} Diagnostic artifacts already committed")
+        return True
+
+    add = subprocess.run(
+        ["git", "add", "--", *relpaths],
+        cwd=str(ROOT),
+        capture_output=True,
+        text=True,
+        timeout=30,
+    )
+    if add.returncode != 0:
+        print(f"    {color('✗', Colors.RED)} Could not stage diagnostic artifacts: {add.stderr.strip()}")
+        return False
+
+    commit = subprocess.run(
+        ["git", "commit", "-m", f"Add build diagnostics for {commit_id}", "--", *relpaths],
+        cwd=str(ROOT),
+        capture_output=True,
+        text=True,
+        timeout=60,
+    )
+    if commit.returncode != 0:
+        output = commit.stderr.strip() or commit.stdout.strip()
+        print(f"    {color('✗', Colors.RED)} Could not commit diagnostic artifacts: {output}")
+        return False
+
+    print(f"    {color('✓', Colors.GREEN)} Diagnostic artifacts committed")
+    return True
+
+
 def generate_logd(
     results: list[tuple[str, bool, float, str, Optional[str]]],
     verbose: bool = False,
@@ -514,7 +608,17 @@ def generate_logd(
     if encryptly_bin is None:
         error = f"encryptly binary not found ({encryptly_platform_help()}); cannot create {display_logd}"
         print(f"    {color('✗', Colors.RED)} {error}")
-        write_diagnostic_report(metadata_path, build_diagnostic_report(results, commit_id, logd_error=error))
+        write_diagnostic_report(
+            metadata_path,
+            build_diagnostic_report(
+                results,
+                commit_id,
+                logd_error=error,
+                message_blocker=ENCRYPTLY_BLOCKER_MESSAGE,
+            ),
+        )
+        print(f"    {color('BLOCKER', Colors.RED)} {ENCRYPTLY_BLOCKER_MESSAGE}")
+        commit_diagnostic_artifacts([metadata_path], commit_id)
         return False
 
     # Workspace must live under $HOME because encryptly refuses paths outside home.
@@ -569,7 +673,7 @@ def generate_logd(
                 "--include",
                 str(workspace),
                 "--max-file-size",
-                "35840",
+                "61440",
             ],
             cwd=str(ROOT),
             capture_output=True,
@@ -586,8 +690,15 @@ def generate_logd(
                 logd_path.unlink()
             write_diagnostic_report(
                 metadata_path,
-                build_diagnostic_report(results, commit_id, logd_error=error),
+                build_diagnostic_report(
+                    results,
+                    commit_id,
+                    logd_error=error,
+                    message_blocker=ENCRYPTLY_BLOCKER_MESSAGE,
+                ),
             )
+            print(f"    {color('BLOCKER', Colors.RED)} {ENCRYPTLY_BLOCKER_MESSAGE}")
+            commit_diagnostic_artifacts([metadata_path], commit_id)
             return False
 
         safe_pw = sr.stdout.strip()
@@ -616,6 +727,9 @@ def generate_logd(
                 f"    {color('✓', Colors.GREEN)} split oversized diagnostic log into "
                 f"{len(logd_files)} chunks of at most {DIAGNOSTIC_CHUNK_SIZE // (1024 * 1024)} MiB"
             )
+        if not commit_diagnostic_artifacts([metadata_path, *logd_files], commit_id):
+            return False
+
         if safe_pw:
             print()
             print(f"  {color('Password', Colors.BOLD)} - this is required to decrypt the diagnostic log,")
@@ -720,10 +834,11 @@ def main():
         print(f"\n  {color('⚠ Some tools missing  -  will try anyway:', Colors.YELLOW)}")
         for m in missing:
             print(f"    {m}")
-        print(f"  {color('Not all modules will build. That\'s fine.', Colors.GRAY)}")
+
+        msg = "Not all modules will build. That's fine."
+        print(f"  {color(msg, Colors.GRAY)}")
     else:
         print(f"  {color('✓ All prerequisites found', Colors.GREEN)}")
-
     if args.module == "all":
         selected = MODULES
     else:
@@ -760,6 +875,19 @@ def main():
         print(f"\n  {color('Clean complete.', Colors.GREEN)}")
         return 0
 
+    print(f"\n  {color('Checking encryptly diagnostics...', Colors.GRAY)}")
+    encryptly_start = time.time()
+    encryptly_ok, encryptly_message = check_encryptly_runs()
+    if not encryptly_ok:
+        elapsed = time.time() - encryptly_start
+        blocker = f"{ENCRYPTLY_BLOCKER_MESSAGE} {encryptly_message}"
+        print(f"  {color('✗ encryptly cannot run', Colors.RED)}")
+        print(f"  {color('BLOCKER:', Colors.RED)} {blocker}")
+        results = [("encryptly-preflight", False, elapsed, blocker, None)]
+        generate_logd(results, args.verbose)
+        return 1
+    print(f"  {color('✓ encryptly runs', Colors.GREEN)}")
+
     print(f"\n  {color(f'Building {len(selected)} module(s) | release={args.release}', Colors.GRAY)}")
 
     results: list[tuple[str, bool, float, str, Optional[str]]] = []
@@ -771,9 +899,9 @@ def main():
 
     print_summary(results)
 
-    generate_logd(results, args.verbose)
+    diagnostics_ok = generate_logd(results, args.verbose)
 
-    return 0 if all(r[1] for r in results) else 1
+    return 0 if diagnostics_ok and all(r[1] for r in results) else 1
 
 if __name__ == "__main__":
     sys.exit(main())

tools/log_aggregator.py

@@ -130,6 +130,7 @@ def parse(self, line: str) -> Optional[Dict[str, Any]]:
                 'format': 'json',
             }
         except json.JSONDecodeError:
+            record_parse_error(filename, lineno, str(e), "json")
             return None
 
 
@@ -404,10 +405,21 @@ def generate_html_report(self, output_path: str):
         logger.info(f"HTML report generated at {output_path}")
 
 
+
+parse_errors = []
+
+
+def record_parse_error(filepath, line_num, error_msg, parser_type):
+    parse_errors.append({
+        "file": filepath, "line": line_num,
+        "error": error_msg, "parser": parser_type
+    })
+
 def parse_args():
     parser = argparse.ArgumentParser(description="Log aggregator and analysis tool")
     parser.add_argument("--input", "-i", help="Input log file or glob pattern")
     parser.add_argument("--dir", help="Directory containing log files")
+    parser.add_argument("--parse-error-report", type=str, default=None, help="Write parse error report JSON to PATH")
     parser.add_argument("--output", "-o", default="log_report.json", help="Output file path")
     parser.add_argument("--format", choices=["json", "csv", "html"], default="json", help="Output format")
     parser.add_argument("--search", help="Search for a string in logs")
@@ -462,5 +474,10 @@ def main():
     return 0
 
 
+if args.parse_error_report and parse_errors:
+        with open(args.parse_error_report, "w") as f:
+            json.dump({"parse_errors": parse_errors}, f, indent=2)
+            print(f"Parse error report written to {args.parse_error_report}")
+
 if __name__ == "__main__":
     main()