diff --git a/.gitignore b/.gitignore index 2c1cab6..5186c73 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,8 @@ **/.wrangler/ pr_clones/ code-review-benchmark/ +pr_review_agent/output/ +online_eval_results*.json +online_eval.db* .env plan.md diff --git a/eval_results.md b/eval_results.md new file mode 100644 index 0000000..7c720a3 --- /dev/null +++ b/eval_results.md @@ -0,0 +1,85 @@ +# Online Eval Results + +Evaluating prompt changes on the same set of ~91 PRs (skip-discover, skip-enrich). +Using 10 workers, gpt-5.4, skip_post=True. + +## Baseline (Run 2 — 0.70 thresholds, no borderline line, extensive search line) + +| Metric | All PRs | PRs w/ GT (52) | +|--------|---------|----------------| +| Suggestions | 208 | 125 | +| Matched suggestions | 36 | 36 | +| Ground truth | 300 | 300 | +| Precision | 0.173 | 0.288 | +| Recall | 0.120 | 0.120 | +| Mean F1 | — | 0.370 (26 PRs) | + +## Attempt 1 — Add code quality paragraph + restore borderline bug line + +**Changes:** +- Restored "it is better to report a borderline real bug than to miss one" in system.py and reviewer.py +- Added paragraph: "ALSO LOOK FOR CODE QUALITY ISSUES THE AUTHOR WOULD FIX" — inconsistent naming/style, unnecessarily complex logic, duplicated code, dead code, unused imports +- Kept 0.70 thresholds and extensive search line + +**Results:** + +| Metric | All PRs (90) | PRs w/ GT (51) | +|--------|--------------|----------------| +| Suggestions | 195 | 123 | +| Matched suggestions | 30 | 30 | +| Ground truth | 271 | 271 | +| Precision | 0.154 | 0.244 | +| Recall | 0.111 | 0.111 | +| Mean F1 | — | 0.369 (21 PRs) | + +**Analysis:** Code quality paragraph was net negative. Generated 5 more non-bug suggestions (21 vs 16) but only 1 matched. Crowded out bug-finding: 18 fewer bug suggestions, 6 lost matches. Lost matches on 12 PRs vs gained on 8. + +## Attempt 2 — Remove code quality, add completeness principle + +**Changes:** +- Removed "ALSO LOOK FOR CODE QUALITY ISSUES" paragraph (proven harmful) +- Added investigation principle #5: "VERIFY COMPLETENESS OF NEW ADDITIONS" — resource cleanup/teardown, DB constraints matching business logic, API endpoint authorization, read/write pair consistency, feature flag both-paths +- Kept borderline bug line, 0.70 thresholds, extensive search line + +**Results:** + +| Metric | All PRs (85) | PRs w/ GT (51) | +|--------|--------------|----------------| +| Suggestions | 195 | 123 | +| Matched suggestions | 37 | 37 | +| Ground truth | 290 | 290 | +| Precision | 0.190 | 0.301 | +| Recall | 0.128 | 0.128 | +| Mean F1 | — | 0.418 (25 PRs) | + +**Analysis:** Best run so far. Removing code quality paragraph refocused the model on bugs. Completeness principle helped find resource cleanup, constraint, and lifecycle bugs. +13 gained matches vs -9 lost on shared PR set. Bug match rate improved from 16% to 19%, security from 10% to 18%. + +## Attempt 3 — Add config/infrastructure category + investigation budget emphasis + +**Changes:** +- Added bug category #15: "Configuration / infrastructure" — missing env var mappings, Dockerfile steps, CI workflow permissions, version pins +- Strengthened "DON'T STOP AT THE FIRST FINDING" with "Pay equal attention to config files as to source code" + +**Results:** + +| Metric | All PRs (91) | PRs w/ GT (52) | +|--------|--------------|----------------| +| Suggestions | 209 | — | +| Matched suggestions | 31 | 31 | +| Ground truth | 299 | 299 | +| Precision | 0.148 | 0.263 | +| Recall | 0.104 | 0.104 | +| Mean F1 | — | 0.363 (24 PRs) | + +**Analysis:** Regressed vs Attempt 2. Same pattern as Attempt 1 — broadening scope dilutes bug-finding focus. Generated 14 more suggestions but 6 fewer matches. Lost on 15 shared PRs vs gained on 8. Config emphasis caused the model to spend cognitive budget on config analysis at the expense of core logic bugs. + +## Summary + +| Run | P (GT) | Recall | Mean F1 | Key Change | +|-----|--------|--------|---------|------------| +| Baseline | 0.288 | 0.120 | 0.370 | 0.70 thresholds, no borderline, extensive search | +| Attempt 1 | 0.244 | 0.111 | 0.369 | + code quality paragraph, + borderline bug line | +| **Attempt 2** | **0.301** | **0.128** | **0.418** | - code quality, + completeness principle | +| Attempt 3 | 0.263 | 0.104 | 0.363 | + config/infra category, + config emphasis | + +**Best: Attempt 2.** Key insight: the model performs best when its scope is tightly focused on runtime defects + structural completeness. Any instruction that broadens scope to non-bug categories (code quality, config/infra) dilutes focus and hurts precision without meaningfully improving recall. diff --git a/github_app/app.py b/github_app/app.py index fc51c61..2eff0bc 100644 --- a/github_app/app.py +++ b/github_app/app.py @@ -103,13 +103,17 @@ class ReviewRequest(BaseModel): model: str = "" # Override model name (e.g. "gpt-5.4", "gemini-3.1-pro-preview") openai_api_key: str = "" # Override OpenAI API key (optional, falls back to env) google_api_key: str = "" # Override Google API key (optional, falls back to env) + skip_post: bool = False # Run review but don't post to GitHub (for evals) REVIEW_API_SECRET = os.environ.get("REVIEW_API_SECRET", "") or os.environ.get("GHAPP_INTERNAL_SECRET", "") -async def _run_review_from_api(req: ReviewRequest): - """Run the review pipeline from an API request (not a webhook).""" +async def _run_review_from_api(req: ReviewRequest) -> list | None: + """Run the review pipeline from an API request (not a webhook). + + Returns the list of review comments when skip_post=True, None otherwise. + """ import time import uuid from github_app.telemetry import make_event_emitter @@ -187,32 +191,39 @@ async def _run_review_from_api(req: ReviewRequest): t_post = time.monotonic() num_issues = len(comments) if comments else 0 - summary_line = f"Found {num_issues} issue{'s' if num_issues != 1 else ''}" - if req.personality and req.github_username: - review_body = ( - f"@{req.github_username}'s review twin\n\n" - f"{summary_line}\n\n---\n" - f"*a2a-review based on @{req.github_username}'s coding preferences*" - ) - else: - review_body = f"## Morph Code Review\n\n{summary_line}" - if comments: - await client.post_review( - req.owner, req.repo, req.pr_number, req.head_sha, - comments, diff, review_body, + if req.skip_post: + logger.info( + "skip_post=True, skipping GitHub post for %s PR #%d (%d comments)", + full_name, req.pr_number, num_issues, ) + duration_post = 0.0 else: - # Post summary even with 0 issues so callers can detect completion - await client.post_issue_comment( - req.owner, req.repo, req.pr_number, review_body, - ) - duration_post = round(time.monotonic() - t_post, 1) - on_event("review.post", { - "comments_posted": len(comments) if comments else 0, - "duration_s": duration_post, - "success": True, - }) + summary_line = f"Found {num_issues} issue{'s' if num_issues != 1 else ''}" + if req.personality and req.github_username: + review_body = ( + f"@{req.github_username}'s review twin\n\n" + f"{summary_line}\n\n---\n" + f"*a2a-review based on @{req.github_username}'s coding preferences*" + ) + else: + review_body = f"## Morph Code Review\n\n{summary_line}" + + if comments: + await client.post_review( + req.owner, req.repo, req.pr_number, req.head_sha, + comments, diff, review_body, + ) + else: + await client.post_issue_comment( + req.owner, req.repo, req.pr_number, review_body, + ) + duration_post = round(time.monotonic() - t_post, 1) + on_event("review.post", { + "comments_posted": num_issues, + "duration_s": duration_post, + "success": True, + }) logger.info( "API review completed for %s PR #%d: %d comments", @@ -240,6 +251,9 @@ async def _run_review_from_api(req: ReviewRequest): if req.callback_url: await _callback(req.callback_url, agent_run_id, "completed") + if req.skip_post: + return comments or [] + except Exception as exc: logger.exception("API review failed for %s PR #%d", full_name, req.pr_number) on_event("review.failed", { @@ -278,6 +292,15 @@ async def review_api(req: ReviewRequest, request: Request): if REVIEW_API_SECRET and auth != f"Bearer {REVIEW_API_SECRET}": raise HTTPException(status_code=401, detail="Unauthorized") + if req.skip_post: + # Run synchronously and return comments directly (for evals) + comments = await _run_review_from_api(req) + return { + "status": "completed", + "agent_run_id": req.agent_run_id or "generated-server-side", + "comments": comments or [], + } + asyncio.create_task(_run_review_from_api(req)) return {"status": "accepted", "agent_run_id": req.agent_run_id or "generated-server-side"} diff --git a/online_eval.py b/online_eval.py new file mode 100644 index 0000000..4b11fef --- /dev/null +++ b/online_eval.py @@ -0,0 +1,678 @@ +#!/usr/bin/env python3 +"""Online evaluation for the Morph PR review service. + +Finds recent PRs from GitHub (via the code-review-benchmark ETL), dispatches +them to the Morph review service on Fly, retrieves the posted review, then +judges the review quality with a 3-step LLM pipeline. + +Pipeline: + 1. Discover + enrich PRs (delegates to benchmark ETL via subprocess) + 2. Dispatch each PR to the Morph review service + 3. Poll GitHub for the posted review + 4. Run the 3-step LLM judge (extract suggestions → extract fixes → match) + 5. Print aggregate precision / recall / F1 + +Usage: + # Full pipeline (needs GCP_PROJECT for BigQuery) + uv run python online_eval.py --max-prs 20 + + # Skip discovery, use PRs already in the benchmark DB + uv run python online_eval.py --skip-discover --max-prs 20 + + # Skip reviews too — just re-judge existing results + uv run python online_eval.py --skip-discover --skip-enrich --skip-review + +Env vars: + GITHUB_TOKEN GitHub PAT with repo access + REVIEW_SERVICE_URL Fly review service (default: https://morph-ghapp.fly.dev) + REVIEW_SERVICE_SECRET Auth secret for the review service + OPENAI_API_KEY For the LLM judge + GCP_PROJECT For BigQuery discovery (skip with --skip-discover) + JUDGE_MODEL Judge model (default: gpt-4.1) +""" +from __future__ import annotations + +import argparse +import asyncio +import json +import logging +import os +import subprocess +import sys +import time +from dataclasses import dataclass, field +from pathlib import Path + +import httpx +from openai import AsyncOpenAI +from pydantic import BaseModel, Field + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", +) +logger = logging.getLogger("online_eval") + +SCRIPT_DIR = Path(__file__).parent +ETL_DIR = SCRIPT_DIR / "code-review-benchmark" / "online" / "etl" + + +# --------------------------------------------------------------------------- +# Pydantic schemas (mirrored from benchmark for self-containedness) +# --------------------------------------------------------------------------- + +class BotSuggestion(BaseModel): + issue_id: str + description: str + category: str + file_path: str | None = None + line_number: int | None = None + severity: str = "medium" + +class BotSuggestionsResponse(BaseModel): + suggestions: list[BotSuggestion] + +class HumanAction(BaseModel): + action_id: str + description: str + category: str + file_path: str | None = None + commit_sha: str | None = None + action_type: str + +class HumanActionsResponse(BaseModel): + actions: list[HumanAction] + +class MatchResult(BaseModel): + bot_issue_id: str + human_action_id: str | None = None + matched: bool + confidence: float + reasoning: str + +class MatchingResponse(BaseModel): + matches: list[MatchResult] + + +# --------------------------------------------------------------------------- +# LLM prompts (from benchmark) +# --------------------------------------------------------------------------- + +EXTRACT_BOT_SUGGESTIONS = """You are analyzing a pull request to extract all actionable suggestions made by a code review bot. + +The bot's username is: {bot_username} + +Below you will see: +1. The commits that were under review (the code state the bot saw), including full diffs +2. The bot's review comments on those commits + +For each actionable suggestion the bot made, extract: +- A unique ID (S1, S2, ...) +- A description of what was suggested +- The category (bug, style, performance, security, refactor, documentation, other) +- The file path and line number if available +- Severity (low, medium, high, critical) + +Only include ACTIONABLE suggestions — skip generic praise, summaries, or "looks good" comments. + +PR Title: {pr_title} +PR Author: {pr_author} +Repository: {repo_name} + +=== Commits Under Review === +{commits_under_review} + +=== Bot Review Comments === +{bot_comments} +""" + +EXTRACT_HUMAN_ACTIONS = """You are analyzing post-review commit diffs to extract every concrete code issue that was fixed AFTER the bot reviewed the PR. + +The bot's username is: {bot_username} + +For each distinct code issue fixed in the post-review commits, extract: +- A unique ID (A1, A2, ...) +- A description of the issue that was fixed (what was wrong, not what was done) +- The category (bug, style, performance, security, refactor, documentation, other) +- The file path +- The commit SHA +- Action type (fix, improvement, cleanup, new_feature, other) + +Focus on the DIFFS. One action per distinct issue. + +PR Title: {pr_title} +PR Author: {pr_author} +Repository: {repo_name} + +=== Post-Review Commits === +{post_review_commits} + +=== Post-Review Activity === +{post_review_activity} +""" + +JUDGE_MATCHING = """You are judging whether a bot's code review suggestions correspond to actual code issues that were later fixed. + +The bot's username is: {bot_username} + +For EACH bot suggestion, determine: +1. Does it match any code fix? (matched: true/false) +2. Which fix? (human_action_id) +3. Confidence (0.0-1.0) +4. Brief reasoning + +A suggestion is "matched" if it identified the same issue that was later fixed, even partially. + +=== Bot Suggestions === +{bot_suggestions} + +=== Code Fixes (ground truth) === +{human_actions} +""" + + +# --------------------------------------------------------------------------- +# Config +# --------------------------------------------------------------------------- + +@dataclass +class EvalConfig: + github_token: str = "" + review_service_url: str = "https://morph-ghapp.fly.dev" + review_service_secret: str = "" + judge_base_url: str = "https://api.openai.com/v1" + judge_api_key: str = "" + judge_model: str = "gpt-4.1" + gcp_project: str = "" + max_prs: int = 50 + days_back: int = 7 + reference_bot: str = "coderabbitai[bot]" + concurrency: int = 5 + db_path: str = "online_eval.db" + output: str = "online_eval_results.json" + + @classmethod + def from_env(cls, args: argparse.Namespace) -> EvalConfig: + return cls( + github_token=os.environ.get("GITHUB_TOKEN", ""), + review_service_url=os.environ.get("REVIEW_SERVICE_URL", "https://morph-ghapp.fly.dev"), + review_service_secret=os.environ.get("REVIEW_SERVICE_SECRET", ""), + judge_base_url=os.environ.get("JUDGE_BASE_URL", "https://api.openai.com/v1"), + judge_api_key=os.environ.get("OPENAI_API_KEY", ""), + judge_model=os.environ.get("JUDGE_MODEL", "gpt-4.1"), + gcp_project=os.environ.get("GCP_PROJECT", ""), + max_prs=args.max_prs, + days_back=args.days_back, + reference_bot=args.reference_bot, + concurrency=args.concurrency, + db_path=args.db, + output=args.output, + ) + + +# --------------------------------------------------------------------------- +# LLM client +# --------------------------------------------------------------------------- + +class LLMJudge: + def __init__(self, base_url: str, api_key: str, model: str): + self.model = model + self._client = AsyncOpenAI(base_url=base_url, api_key=api_key) + + async def structured(self, prompt: str, schema: type[BaseModel]): + resp = await self._client.beta.chat.completions.parse( + model=self.model, + messages=[{"role": "user", "content": prompt}], + response_format=schema, + temperature=1.0, + ) + return resp.choices[0].message.parsed + + async def close(self): + await self._client.close() + + +# --------------------------------------------------------------------------- +# Step 1: Discover + Enrich via benchmark ETL subprocess +# --------------------------------------------------------------------------- + +def run_etl_step(step: str, cfg: EvalConfig, extra_args: list[str] | None = None): + """Run a benchmark ETL step via uv in the ETL directory.""" + cmd = ["uv", "run", "python", "main.py", step] + cmd += ["--chatbot", cfg.reference_bot] + if extra_args: + cmd += extra_args + + env = { + **os.environ, + "DATABASE_URL": f"sqlite:///{Path(cfg.db_path).resolve()}", + "GITHUB_TOKEN": cfg.github_token, + "GCP_PROJECT": cfg.gcp_project, + } + + logger.info(f"Running ETL: {' '.join(cmd)}") + result = subprocess.run(cmd, cwd=str(ETL_DIR), env=env, capture_output=False) + if result.returncode != 0: + logger.error(f"ETL step '{step}' failed with code {result.returncode}") + sys.exit(1) + + +def discover_and_enrich(cfg: EvalConfig, skip_discover: bool, skip_enrich: bool): + if not skip_discover: + # Limit discovery to avoid enriching thousands of PRs we'll never use + max_per_day = max(cfg.max_prs * 2, 10) + run_etl_step("discover", cfg, [ + "--days-back", str(cfg.days_back), + "--max-prs-per-day", str(max_per_day), + ]) + if not skip_enrich: + run_etl_step("enrich", cfg, ["--one-shot", "--max-prs", str(cfg.max_prs)]) + + +# --------------------------------------------------------------------------- +# Step 2: Fetch assembled PRs from the SQLite DB directly +# --------------------------------------------------------------------------- + +async def load_assembled_prs(db_path: str, reference_bot: str, limit: int) -> list[dict]: + """Load assembled PRs from the benchmark SQLite DB.""" + import sqlite3 + + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + cur = conn.cursor() + + # Get chatbot_id + cur.execute("SELECT id FROM chatbots WHERE github_username = ?", (reference_bot,)) + row = cur.fetchone() + if not row: + logger.error(f"Chatbot {reference_bot} not found in DB") + conn.close() + return [] + chatbot_id = row["id"] + + # Get assembled PRs (status = 'assembled') that haven't been analyzed yet + cur.execute( + """ + SELECT p.* FROM prs p + WHERE p.chatbot_id = ? AND p.status = 'assembled' + ORDER BY p.discovered_at DESC + LIMIT ? + """, + (chatbot_id, limit), + ) + prs = [dict(r) for r in cur.fetchall()] + conn.close() + + logger.info(f"Loaded {len(prs)} assembled PRs from {db_path}") + return prs + + +# --------------------------------------------------------------------------- +# Step 3: Dispatch Morph reviews +# --------------------------------------------------------------------------- + +async def dispatch_and_collect(cfg: EvalConfig, prs: list[dict]) -> list[dict]: + """Dispatch PRs to the Morph review service and collect results.""" + if not prs: + return [] + + logger.info(f"Dispatching {len(prs)} PRs to Morph (concurrency={cfg.concurrency})...") + sem = asyncio.Semaphore(cfg.concurrency) + results = [] + + async with httpx.AsyncClient(timeout=60) as http: + async def _do_one(pr: dict) -> dict | None: + async with sem: + repo_name = pr["repo_name"] + pr_number = pr["pr_number"] + owner, repo_short = repo_name.split("/", 1) + + commits = json.loads(pr.get("commits") or "[]") + if not commits: + logger.warning(f"No commits for {repo_name}#{pr_number}") + return None + head_sha = commits[-1].get("sha", "") + + # Dispatch + headers = {"Content-Type": "application/json"} + if cfg.review_service_secret: + headers["Authorization"] = f"Bearer {cfg.review_service_secret}" + + payload = { + "owner": owner, + "repo": repo_short, + "pr_number": pr_number, + "head_sha": head_sha, + "github_token": cfg.github_token, + "provider": "openai", + "model": "gpt-5.4", + "skip_post": True, # Don't post to GitHub — return comments in response + } + + try: + resp = await http.post( + f"{cfg.review_service_url}/review", + json=payload, + headers=headers, + timeout=300, # Reviews can take a few minutes + ) + if resp.status_code != 200: + logger.warning(f" {repo_name}#{pr_number}: service returned {resp.status_code}") + return None + except Exception as e: + logger.error(f" {repo_name}#{pr_number}: dispatch failed: {e}") + return None + + data = resp.json() + comments = data.get("comments", []) + logger.info(f" Got {len(comments)} comments for {repo_name}#{pr_number}") + + # Convert service comments to the format the judge expects + morph_reviews = [{ + "submitted_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "state": "COMMENTED", + "body": f"Found {len(comments)} issue{'s' if len(comments) != 1 else ''}", + "inline_comments": [ + { + "path": c.get("file_path", ""), + "line": c.get("line_number"), + "body": c.get("body", ""), + "diff_hunk": "", + } + for c in comments + ], + }] + return {"pr": pr, "morph_reviews": morph_reviews} + + tasks = [asyncio.create_task(_do_one(pr)) for pr in prs] + for coro in asyncio.as_completed(tasks): + r = await coro + if r: + results.append(r) + + logger.info(f"Collected {len(results)}/{len(prs)} reviews") + return results + + +# --------------------------------------------------------------------------- +# Step 4: Judge +# --------------------------------------------------------------------------- + +def _fmt_morph(reviews: list[dict]) -> str: + lines = [] + for rv in reviews: + ts = rv.get("submitted_at", "") + body = rv.get("body", "") + lines.append(f"[{ts}] REVIEW ({rv.get('state', '')}):") + if body: + lines.append(f" {body}") + for c in rv.get("inline_comments", []): + path = c.get("path", "") + line = c.get("original_line") or c.get("line", "") + lines.append(f"[{ts}] REVIEW_COMMENT ({path}:{line}):") + if c.get("diff_hunk"): + lines.append(f" ```\n{c['diff_hunk']}\n ```") + if c.get("body"): + lines.append(f" {c['body']}") + lines.append("") + return "\n".join(lines) or "(no comments)" + + +def _fmt_commits(commits: list[dict], details: dict) -> str: + if not commits: + return "(no commits)" + lines = [] + for c in commits: + sha = c.get("sha", "")[:12] + lines.append(f"COMMIT {sha} by {c.get('author', '?')} [{c.get('date', '')}]") + lines.append(f" {c.get('message', '')}") + d = details.get(c.get("sha", ""), {}) + for f in d.get("files", []): + lines.append(f" {f.get('status', '').upper()} {f.get('filename', '')} (+{f.get('additions', 0)}/-{f.get('deletions', 0)})") + if f.get("patch"): + lines.append(f" ```diff\n{f['patch']}\n ```") + lines.append("") + return "\n".join(lines) + + +def _fmt_suggestions(suggestions: list[dict]) -> str: + lines = [] + for s in suggestions: + loc = "" + if s.get("file_path"): + loc = f" ({s['file_path']}:{s.get('line_number', '')})" + lines.append(f"- [{s['issue_id']}] ({s['category']}/{s['severity']}){loc}: {s['description']}") + return "\n".join(lines) or "(none)" + + +def _fmt_actions(actions: list[dict]) -> str: + lines = [] + for a in actions: + loc = f" ({a['file_path']})" if a.get("file_path") else "" + lines.append(f"- [{a['action_id']}] ({a['category']}/{a['action_type']}){loc}: {a['description']}") + return "\n".join(lines) or "(none)" + + +async def judge_all(cfg: EvalConfig, results: list[dict]) -> list[dict]: + if not results: + return [] + + llm = LLMJudge(cfg.judge_base_url, cfg.judge_api_key, cfg.judge_model) + sem = asyncio.Semaphore(cfg.concurrency) + judgments = [] + + async def _judge(item: dict) -> dict | None: + async with sem: + pr = item["pr"] + repo_name = pr["repo_name"] + pr_number = pr["pr_number"] + + assembled = json.loads(pr.get("assembled") or "{}") + commits = json.loads(pr.get("commits") or "[]") + commit_details = json.loads(pr.get("commit_details") or "[]") + reviews_raw = json.loads(pr.get("reviews") or "[]") + events = assembled.get("events", []) + + # Find split point + bot_lower = cfg.reference_bot.lower() + hash_x = None + for r in reviews_raw: + author = (r.get("author") or r.get("user", {}).get("login", "")).lower() + if author == bot_lower and r.get("commit_id"): + hash_x = r["commit_id"] + break + if not hash_x and commits: + hash_x = commits[-1].get("sha") + + pre, post = [], [] + if hash_x: + for i, c in enumerate(commits): + sha = c.get("sha", "") + if sha == hash_x or sha.startswith(hash_x) or hash_x.startswith(sha): + pre, post = commits[:i + 1], commits[i + 1:] + break + else: + pre, post = commits, [] + else: + pre, post = commits, [] + + details = {d["sha"]: d for d in commit_details if d.get("sha")} + + morph_text = _fmt_morph(item["morph_reviews"]) + pre_text = _fmt_commits(pre, details) + post_text = _fmt_commits(post, details) + + pr_title = assembled.get("pr_title", "") + pr_author = assembled.get("pr_author", "unknown") + + try: + # Step 1: extract morph suggestions + s_resp = await llm.structured( + EXTRACT_BOT_SUGGESTIONS.format( + bot_username="morph-subagents[bot]", + pr_title=pr_title, pr_author=pr_author, repo_name=repo_name, + commits_under_review=pre_text, bot_comments=morph_text, + ), + BotSuggestionsResponse, + ) + suggestions = [s.model_dump() for s in s_resp.suggestions] + + # Step 2: extract ground truth + a_resp = await llm.structured( + EXTRACT_HUMAN_ACTIONS.format( + bot_username="morph-subagents[bot]", + pr_title=pr_title, pr_author=pr_author, repo_name=repo_name, + post_review_commits=post_text, post_review_activity="(see commits above)", + ), + HumanActionsResponse, + ) + actions = [a.model_dump() for a in a_resp.actions] + + # Step 3: match + m_resp = await llm.structured( + JUDGE_MATCHING.format( + bot_username="morph-subagents[bot]", + bot_suggestions=_fmt_suggestions(suggestions), + human_actions=_fmt_actions(actions), + ), + MatchingResponse, + ) + matches = [m.model_dump() for m in m_resp.matches] + + # Metrics + s_ids = {s["issue_id"] for s in suggestions} + a_ids = {a["action_id"] for a in actions} + matched_s = {m["bot_issue_id"] for m in matches if m["matched"] and m.get("bot_issue_id") in s_ids} + matched_a = {m["human_action_id"] for m in matches if m["matched"] and m.get("human_action_id") in a_ids} + + n_s, n_ms = len(suggestions), len(matched_s) + n_a, n_ma = len(actions), len(matched_a) + p = n_ms / n_s if n_s else None + r = n_ma / n_a if n_a else None + f1 = 2 * p * r / (p + r) if p and r and (p + r) > 0 else None + + ps = f"{p:.2f}" if p is not None else "N/A" + rs = f"{r:.2f}" if r is not None else "N/A" + fs = f"{f1:.2f}" if f1 is not None else "N/A" + logger.info(f" {repo_name}#{pr_number}: {n_s} sug, P={ps} R={rs} F1={fs}") + + return { + "repo_name": repo_name, "pr_number": pr_number, + "total_suggestions": n_s, "matched_suggestions": n_ms, + "total_actions": n_a, "matched_actions": n_ma, + "precision": p, "recall": r, "f1": f1, + "suggestions": suggestions, "actions": actions, "matches": matches, + } + except Exception as e: + logger.error(f" Judge failed {repo_name}#{pr_number}: {e}") + return None + + tasks = [asyncio.create_task(_judge(r)) for r in results] + for coro in asyncio.as_completed(tasks): + j = await coro + if j: + judgments.append(j) + + await llm.close() + return judgments + + +# --------------------------------------------------------------------------- +# Summary +# --------------------------------------------------------------------------- + +def print_summary(judgments: list[dict], output_path: str): + if not judgments: + print("\nNo judgments to summarize.") + return + + n_s = sum(j["total_suggestions"] for j in judgments) + n_ms = sum(j["matched_suggestions"] for j in judgments) + n_a = sum(j["total_actions"] for j in judgments) + n_ma = sum(j["matched_actions"] for j in judgments) + ps = [j["precision"] for j in judgments if j["precision"] is not None] + rs = [j["recall"] for j in judgments if j["recall"] is not None] + fs = [j["f1"] for j in judgments if j["f1"] is not None] + + print("\n" + "=" * 60) + print("MORPH ONLINE EVAL RESULTS") + print("=" * 60) + print(f"PRs evaluated: {len(judgments)}") + print(f"Total suggestions: {n_s}") + print(f"Matched suggestions: {n_ms}") + print(f"Total ground-truth: {n_a}") + print(f"Matched ground-truth: {n_ma}") + print() + if n_s: + print(f"Aggregate precision: {n_ms / n_s:.3f}") + if n_a: + print(f"Aggregate recall: {n_ma / n_a:.3f}") + if ps: + print(f"Mean PR precision: {sum(ps) / len(ps):.3f}") + if rs: + print(f"Mean PR recall: {sum(rs) / len(rs):.3f}") + if fs: + print(f"Mean PR F1: {sum(fs) / len(fs):.3f}") + print("=" * 60) + + Path(output_path).write_text(json.dumps(judgments, indent=2, default=str)) + print(f"\nDetailed results saved to {output_path}") + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +async def async_main(args: argparse.Namespace): + cfg = EvalConfig.from_env(args) + + if not cfg.github_token: + print("ERROR: GITHUB_TOKEN required", file=sys.stderr); sys.exit(1) + if not cfg.judge_api_key: + print("ERROR: OPENAI_API_KEY required", file=sys.stderr); sys.exit(1) + + # Step 1: Discover + enrich (via benchmark ETL subprocess) + if not args.skip_discover: + if not cfg.gcp_project: + print("ERROR: GCP_PROJECT required for discovery (or use --skip-discover)", file=sys.stderr) + sys.exit(1) + + discover_and_enrich(cfg, args.skip_discover, args.skip_enrich) + + # Step 2: Load PRs from DB + prs = await load_assembled_prs(cfg.db_path, cfg.reference_bot, cfg.max_prs) + if not prs: + print("No assembled PRs found. Run without --skip-discover/--skip-enrich.") + return + + # Step 3: Dispatch reviews + if args.skip_review: + logger.info("Skipping review dispatch (--skip-review)") + results = [] + else: + results = await dispatch_and_collect(cfg, prs) + + # Step 4: Judge + judgments = await judge_all(cfg, results) + + # Step 5: Summary + print_summary(judgments, cfg.output) + + +def main(): + parser = argparse.ArgumentParser(description="Morph online eval") + parser.add_argument("--max-prs", type=int, default=50) + parser.add_argument("--days-back", type=int, default=7) + parser.add_argument("--reference-bot", default="coderabbitai[bot]") + parser.add_argument("--concurrency", type=int, default=5) + parser.add_argument("--skip-discover", action="store_true") + parser.add_argument("--skip-enrich", action="store_true") + parser.add_argument("--skip-review", action="store_true") + parser.add_argument("--db", default="online_eval.db") + parser.add_argument("--output", default="online_eval_results.json") + args = parser.parse_args() + asyncio.run(async_main(args)) + + +if __name__ == "__main__": + main() diff --git a/pr_review_agent/config.py b/pr_review_agent/config.py index f87b91a..88cb72c 100644 --- a/pr_review_agent/config.py +++ b/pr_review_agent/config.py @@ -52,15 +52,15 @@ class Config: "type_error": 0.60, "security": 0.60, "null_reference": 0.70, # FP-prone - # Low-value / FP-prone: suppress - "missing_validation": 0.99, - "resource_leak": 0.99, - "resource_leaks": 0.99, - "performance": 0.99, - "documentation": 0.99, - "style": 0.99, - "naming": 0.99, - "refactor": 0.99, + # Lower-value / FP-prone: higher threshold but not suppressed + "missing_validation": 0.70, + "resource_leak": 0.70, + "resource_leaks": 0.70, + "performance": 0.70, + "documentation": 0.70, + "style": 0.70, + "naming": 0.70, + "refactor": 0.70, }) # WarpGrep settings diff --git a/pr_review_agent/main.py b/pr_review_agent/main.py index 7c8d55e..efe0d00 100644 --- a/pr_review_agent/main.py +++ b/pr_review_agent/main.py @@ -165,7 +165,7 @@ def run_benchmark(config: Config, args: argparse.Namespace) -> None: from concurrent.futures import ThreadPoolExecutor, as_completed - parallel = min(10, len(pr_urls)) # cap at 10 concurrent + parallel = min(30, len(pr_urls)) # cap at 30 concurrent def _review_one(idx_and_url): i, pr_url = idx_and_url @@ -321,7 +321,7 @@ def main(): parser.add_argument("--random", action="store_true", help="13 PRs, 2 per source_repo, seed=42") parser.add_argument("--threshold", type=float, help="Override confidence threshold") parser.add_argument("--no-warpgrep", action="store_true", help="Skip WarpGrep") - parser.add_argument("--provider", help="LLM provider (anthropic, openai, google)") + parser.add_argument("--provider", help="LLM provider (openai, anthropic, google)") parser.add_argument("--model", help="Model name override") parser.add_argument("--organism", help="Path to organism JSON (evolved prompts)") parser.add_argument("--evolve", action="store_true", help="Run darwinian evolution") diff --git a/pr_review_agent/output/evaluation_results.json b/pr_review_agent/output/evaluation_results.json deleted file mode 100644 index 3ebc395..0000000 --- a/pr_review_agent/output/evaluation_results.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "precision": 0.5758, - "recall": 0.5429, - "f1": 0.5588, - "tp": 19, - "fp": 16, - "fn": 16, - "total_golden": 35, - "total_candidates": 33, - "per_repo": { - "sentry": { - "tp": 6, - "fp": 5, - "fn": 8, - "golden": 14, - "candidates": 9 - }, - "grafana": { - "tp": 3, - "fp": 1, - "fn": 1, - "golden": 4, - "candidates": 4 - }, - "keycloak": { - "tp": 3, - "fp": 3, - "fn": 5, - "golden": 8, - "candidates": 6 - }, - "discourse": { - "tp": 3, - "fp": 5, - "fn": 2, - "golden": 5, - "candidates": 8 - }, - "cal.com": { - "tp": 4, - "fp": 2, - "fn": 0, - "golden": 4, - "candidates": 6 - } - } -} \ No newline at end of file diff --git a/pr_review_agent/output/iter10_mini15.log b/pr_review_agent/output/iter10_mini15.log deleted file mode 100644 index df238d2..0000000 --- a/pr_review_agent/output/iter10_mini15.log +++ /dev/null @@ -1,197 +0,0 @@ -============================================================ -PR Review Agent - Benchmark Pipeline -============================================================ -Loaded 50 PRs -WarpGrep: ENABLED -Mini eval: 15 PRs (3 per repo) -Reviewing 15 PRs - -Running 15 PRs with parallelism=15 - -[1/15] keycloak PR#37429 (4 golden) -[2/15] keycloak PR#37634 (4 golden) -[3/15] keycloak PR#38446 (2 golden) -[4/15] sentry PR#93824 (5 golden) -[5/15] sentry-greptile PR#5 (3 golden) -[6/15] sentry-greptile PR#1 (4 golden) -[8/15] grafana PR#103633 (2 golden) -[7/15] grafana PR#97529 (2 golden) -[9/15] grafana PR#94942 (2 golden) -[11/15] discourse-graphite PR#10 (4 golden) -[10/15] discourse-graphite PR#9 (2 golden) -[12/15] discourse-graphite PR#7 (3 golden) -[13/15] cal.com PR#22532 (2 golden) -[14/15] cal.com PR#8330 (2 golden) -[15/15] cal.com PR#14943 (2 golden) - [15/15] cal.com PR#14943 3 files, 38 added lines - [10/15] discourse-graphite PR#9 6 files, 31 added lines - [7/15] grafana PR#97529 5 files, 25 added lines - [8/15] grafana PR#103633 4 files, 240 added lines - [13/15] cal.com PR#22532 17 files, 379 added lines - [3/15] keycloak PR#38446 8 files, 256 added lines - [9/15] grafana PR#94942 4 files, 41 added lines - [4/15] sentry PR#93824 6 files, 199 added lines - [11/15] discourse-graphite PR#10 36 files, 449 added lines - [14/15] cal.com PR#8330 4 files, 111 added lines - [2/15] keycloak PR#37634 28 files, 722 added lines - [1/15] keycloak PR#37429 48 files, 343 added lines - [6/15] sentry-greptile PR#1 3 files, 128 added lines - [12/15] discourse-graphite PR#7 32 files, 115 added lines - [5/15] sentry-greptile PR#5 105 files, 2312 added lines - WarpGrep: How is the deletion of WorkflowReminder records handled? What conditions determi - WarpGrep: How does the store.find method work for singular resources vs collections in the - WarpGrep: How is set_locale implemented in ApplicationController and what does it return? - WarpGrep: How is checkIfIsAvailable called and what parameters are passed to it in slots.t - WarpGrep: Who calls the Context copy constructor of OAuth2GrantType.Context? - WarpGrep: How does Prisma updateMany behave when data is an empty object? Does it update @ - WarpGrep: How is SpanFlusher.main called and what arguments does it receive? Look at how t - WarpGrep: How does Django QuerySet handle negative indexing or negative slicing - WarpGrep: How is the bleveBackend cache used concurrently? What reads and writes to b.cach - WarpGrep: How does getSlots work with activeOverrides and working hours in packages/lib/sl - WarpGrep: How does the Check method in the RBAC service handle permission checking, includ - WarpGrep: How does RecoveryAuthnCodesCredentialModel.createFromValues work and what parame - WarpGrep: How does the VerifyMessageProperties class resolve the English properties file f - WarpGrep: How does fetch_error_details use nodestore.backend.get_multi and zip error_ids w - WarpGrep: How is enableSqlExpressions used and what is the expected behavior when the Flag - WarpGrep: SelectedCalendarRepository updateManyByCredentialId callers and usage - WarpGrep: How does _lookupSubType work in the store to hydrate embedded objects - WarpGrep: How is I18n.ensure_loaded! defined and used in the translate_accelerator freedom - WarpGrep: dark-light-choose SCSS function definition and how it works - WarpGrep: How does self.paginate pass extra keyword arguments to paginator_cls constructor - WarpGrep: SpansBuffer constructor and how assigned_shards is determined from partitions - WarpGrep: Who calls NewResourceServer and how is the returned server used? Is Init called - WarpGrep: PRCommentWorkflow get_merged_pr_single_issue_template and format_comment_subtitl - WarpGrep: How is santizeAnchors called and what does it do with the value replacement - WarpGrep: How does user.credentialManager().updateCredential work for federated users and - WarpGrep: How is isAccessTokenId or isUUID used in test assertions for token IDs? - WarpGrep: connectedCalendars handler return type and callers that access cacheUpdatedAt - WarpGrep: Where is load_locale defined and how does it track @loaded_locales? - WarpGrep: Who calls the SQL expression Execute method and how is QueryTypeSQL handled in t - WarpGrep connection error, retrying in 3s (attempt 1/3): SSLError - WarpGrep: How is checkPermission implemented in the RBAC service? What does it return for - WarpGrep: How does ProcessSpansStrategyFactory create_with_partitions pass parameters to S - WarpGrep: How does the paginate method on the endpoint base class instantiate the paginato - WarpGrep: WorkflowReminder retryCount field usage and scheduling logic - WarpGrep: How does the Visualize class fromJSON method work and who calls it - WarpGrep: getFederatedCredentialsStream method definition and what it returns for differen - WarpGrep: How does server.Init work with sync.Once pattern for lazy initialization? - WarpGrep: What callers exist for newRBACClient and how is it used across the codebase? - WarpGrep: RecoveryAuthnCodesFormAuthenticator authenticate method and how it validates rec - WarpGrep: How does basePath use type parameter and where does underscore vs hyphen convers - WarpGrep: TableWidgetVisualization component usage with columns and tableData props in das - WarpGrep: What happens when _hydrateEmbedded processes an array of ids but _lookupSubType - WarpGrep: How does run_with_initialized_sentry work with process args? Does it consume the - WarpGrep: Where is checkPermission called in the RBAC service with how many arguments? Wha - WarpGrep: RecoveryAuthnCodesCredentialProvider isValid method how it validates a recovery - WarpGrep: BackwardsCompatibilityUserStorage implements CredentialInputUpdater getCredentia - WarpGrep: NullPointerException or nil dereference in destroy action when record not found - WarpGrep: nodestore get_multi return type - does it return an ordered dict or regular dict - WarpGrep: How are activeOverrides used in getSlots and how are working hours computed with - WarpGrep: How does the EmbeddableHost model handle the before_validation stripping of path - WarpGrep: dayjs object comparison with === operator, should use isSame instead - WarpGrep: How does the CredentialHelper createOTPCredential method work and what does upda - WarpGrep: RecoveryAuthnCodesCredentialModel createFromValues hashes codes and what format - WarpGrep: How does the event attachments FlexCenter component work with overflowEllipsis t - WarpGrep: How is slotEndTime calculated and used in availability checking in the slots han - WarpGrep: SpanFlusher constructor in the diff - how does SpansBuffer constructor handle sh - WarpGrep: How does SelectedCalendarsSettingsWebWrapper receive and use connectedCalendar.c - WarpGrep: OrganizationAuditPermission class definition and what permissions it requires - WarpGrep: FlexCenter overflowEllipsis in eventAttachments component that was replaced by F - WarpGrep: searchSupport struct definition and its fields including tracer and log - WarpGrep: How does the WhatsApp reminder scheduler handle the deleteMany cleanup of past r - WarpGrep: How is the Context.grantType set for grants that call createTokenResponse in OAu - WarpGrep: How does the FlexCenter in eventAttachments work with overflowEllipsis - what st - WarpGrep: How do email workflow reminders handle retry count increments on scheduling fail - WarpGrep connection error, retrying in 3s (attempt 1/3): SSLError - WarpGrep: How is ExpressionQueryReader defined and what is the features field type? - WarpGrep: analytics record preprod_artifact before feature flag check - should analytics b - WarpGrep: Where is ResourceOwnerPasswordCredentialsGrantType.process creating the token re - WarpGrep: What is checkReq.UserUID and how is it set during validateCheckRequest? Is it th - WarpGrep: What happens when analytics.record is called before the feature flag check - doe - Loop done round=9 (tools: codebase_search=1, grep=4, read_file=4, glob=2, bash=6) - WarpGrep: How is the checkRequest struct defined and how is UserUID populated from the req - WarpGrep: How does the Attributes component filter with HIDDEN_ATTRIBUTES when searchQuery - WarpGrep: Where is the Paginator class get_result method that does queryset offset slicing - WarpGrep: How does Prisma updateMany handle an empty data object - does it skip @updatedAt - Review complete: 1 issues - [12/15] discourse-graphite PR#7 1 raw -> 1 kept - [12/15] discourse-graphite PR#7 [0.95] incorrect_value: The light/dark mode lightness values are swapped in `.custom-message-length`. The code use - Loop done round=9 (tools: codebase_search=5, grep=4, read_file=6, list_directory=1, glob=3) - WarpGrep: How does the store _hydrateEmbedded handle the case when _lookupSubType returns - WarpGrep: nodestore get_multi dict ordering - does it preserve order of input id_list - Review complete: 2 issues - [14/15] cal.com PR#8330 2 raw -> 2 kept - [14/15] cal.com PR#8330 [0.97] api_misuse: `===` reference comparison on two dayjs objects always returns `false` because JavaScript - [14/15] cal.com PR#8330 [0.97] incorrect_value: Copy-paste bug: `end` is computed from `slotStartTime` instead of `slotEndTime`, making `s - WarpGrep: GoogleCalendarService fetchAvailabilityAndSetCache method - what does setAvailab - WarpGrep: Where does the email reminder scheduler increment retryCount on failure? - WarpGrep: How does the email workflow reminder scheduler handle failures and update retryC - WarpGrep: Where is the translate method in translate_accelerator and how does it handle fa - WarpGrep: Where does ClientCredentialsGrantType set the GRANT_TYPE attribute? Does it use - WarpGrep: How does the EmbeddableHost before_validation handle nil host values - WarpGrep: scheduleEmailReminder retryCount update on failure in workflow email scheduler - Loop done round=9 (tools: codebase_search=3, grep=10, read_file=8, glob=1) - Review complete: 1 issues - [9/15] grafana PR#94942 1 raw -> 1 kept - [9/15] grafana PR#94942 [0.95] logic_error: The `enableSqlExpressions` function always returns `false` due to two compounding errors: - WarpGrep: Where is spans.buffer.flusher.wait_produce metric consumed or queried? Dashboard - WarpGrep: scheduleEmailReminders cron handler that processes unscheduled email workflow re - Loop done round=10 (tools: codebase_search=5, grep=6, read_file=11) - Review complete: 2 issues - [2/15] keycloak PR#37634 2 raw -> 2 kept - [2/15] keycloak PR#37634 [0.97] null_reference: The second `Objects.requireNonNull` checks `grantType` again instead of `rawTokenId` (copy - [2/15] keycloak PR#37634 [0.96] logic_error: Two bugs on this line: (1) Inverted condition — returns false when the grant shortcut matc - WarpGrep: RecoveryAuthnCodesUtils generateRawCodes format of generated recovery codes and - WarpGrep: How does the CalendarCacheRepository get instantiated? Is it singleton or uses D - WarpGrep: How does the consumer __init__.py click_options get passed to the strategy facto - WarpGrep: SetupRecoveryAuthnCodesPage getRecoveryAuthnCodes method retrieves codes from UI - WarpGrep: How does the I18n Fallbacks backend module use fallbacks when translating? What - WarpGrep: How does the account-ui or admin-ui process message property placeholders - do t - WarpGrep: How are admin-ui message properties with double curly braces processed for the f - WarpGrep: How does the BrowserReportSerializer validate_age and validate_timestamp work wi - WarpGrep: Store pluralize method implementation in discourse store service - Loop done round=13 (tools: read_file=8, codebase_search=8, grep=7) - WarpGrep: How does the embedding serializer handle the embeddable_hosts has_many relations - Review complete: 2 issues - [15/15] cal.com PR#14943 2 raw -> 2 kept - [15/15] cal.com PR#14943 [0.92] logic_error: The `deleteMany` query's second OR branch `{ retryCount: { gt: 1 } }` has no `method` filt - [15/15] cal.com PR#14943 [0.60] race_condition: The retryCount update uses a non-atomic read-modify-write pattern (`retryCount: reminder.r - WarpGrep: How does the Keycloak account-ui React app load and use i18n message properties - Loop done round=13 (tools: codebase_search=7, grep=6, read_file=9, bash=4) - WarpGrep: What did set_locale method do in application_controller before the with_resolved - Review complete: 3 issues - [4/15] sentry PR#93824 3 raw -> 3 kept - [4/15] sentry PR#93824 [0.95] type_error: isinstance(process, multiprocessing.Process) is always False for SpawnProcess created via - [4/15] sentry PR#93824 [0.95] type_error: Same isinstance(process, multiprocessing.Process) check in join() is always False for Spaw - [4/15] sentry PR#93824 [0.90] incorrect_value: Metric tag key is 'shards' (plural) but the two nearby metrics on lines 309 and 319 use 's - WarpGrep: Prisma updateMany with empty data object, does @updatedAt get set - Loop done round=18 (tools: codebase_search=10, grep=18, read_file=10, glob=3, bash=1) - Review complete: 3 issues - [11/15] discourse-graphite PR#10 3 raw -> 3 kept - [11/15] discourse-graphite PR#10 [0.95] incorrect_value: The file contents are swapped between category_fabricator.rb and embeddable_host_fabricato - [11/15] discourse-graphite PR#10 [0.92] null_reference: When no `embed_category` site setting exists (the common case for instances that never con - [11/15] discourse-graphite PR#10 [0.82] security: The host value `h` comes from user-supplied data (the `embeddable_hosts` site setting) and - WarpGrep: RecoveryAuthnCodesCredentialProvider createCredential method - does it delete ex - WarpGrep: How does setUserProfileServerError format translation parameters when calling t( - WarpGrep: cache.Cache interface definition in authlib, what methods does it require? - Loop done round=22 (tools: codebase_search=5, grep=16, read_file=18, glob=1, bash=3) - Review complete: 2 issues - [6/15] sentry-greptile PR#1 2 raw -> 2 kept - [6/15] sentry-greptile PR#1 [0.95] type_error: OptimizedCursorPaginator.get_item_key calls math.floor()/math.ceil() on a datetime object, - [6/15] sentry-greptile PR#1 [0.95] api_misuse: Django QuerySets do not support negative indexing. When enable_advanced_features=True and - Max tool rounds reached (tools: codebase_search=8, grep=20, read_file=13, glob=3, bash=9) - WarpGrep: How does the CredentialHelper createOTPCredential differ from createRecoveryCode - WarpGrep: How does the admin-ui load internationalized messages from properties files - WarpGrep: How does the admin console serve localized message properties to the frontend as - WarpGrep: How does the Flex.Item component work, does it accept grow prop - Max tool rounds reached (tools: codebase_search=14, read_file=25, grep=15, glob=1, bash=4) - Review complete: 3 issues - [5/15] sentry-greptile PR#5 3 raw -> 3 kept - [5/15] sentry-greptile PR#5 [0.95] logic_error: Dict ordering from `nodestore.backend.get_multi()` does not preserve the order of input `n - [5/15] sentry-greptile PR#5 [0.85] logic_error: Both `validate_timestamp` and `validate_age` use `.get()` truthiness to check for mutual e - [5/15] sentry-greptile PR#5 [0.75] incorrect_value: When the `use-table-widget-visualization` feature flag is enabled, `TableWidgetVisualizati - Max tool rounds reached (tools: codebase_search=8, grep=33, read_file=10, glob=2, bash=1) - Review complete: 0 issues - [1/15] keycloak PR#37429 0 raw -> 0 kept - Review complete: 0 issues - [13/15] cal.com PR#22532 0 raw -> 0 kept diff --git a/pr_review_agent/output/iter11_eval.log b/pr_review_agent/output/iter11_eval.log deleted file mode 100644 index cee0d27..0000000 --- a/pr_review_agent/output/iter11_eval.log +++ /dev/null @@ -1,91 +0,0 @@ - -keycloak PR#37429: 4 golden, 3 candidates - TP: [Medium] The translation is in Italian instead of Lithuanian. This should be translated t... - TP: [Medium] The totpStep1 value uses Traditional Chinese terms in the Simplified Chinese fil... - FN: [Low] The anchor sanitization logic has a potential issue where it consumes English ma... - FN: [Low] The method name 'santizeAnchors' should be 'sanitizeAnchors' (missing 'i').... - -keycloak PR#37634: 4 golden, 2 candidates - TP: [Critical] Wrong parameter in null check (grantType vs. rawTokenId)... - TP: [High] In isAccessTokenId, the substring for the grant shortcut and the equality check ... - FN: [Low] Javadoc mentions "usually like 3-letters shortcut" but some implementations use ... - FN: [Low] Catching generic RuntimeException is too broad. The implementation throws Illeg... - -keycloak PR#38446: 2 golden, 3 candidates - FN: [Medium] Unsafe raw List deserialization without type safety. Calling Optional.get() dire... - FN: [Low] After creating the RecoveryAuthnCodesCredentialModel, consider setting its id fr... - -sentry PR#93824: 5 golden, 3 candidates - TP: [Medium] Inconsistent metric tagging with 'shard' and 'shards'... - TP: [Low] Fixed sleep in tests can be flaky; wait on condition instead... - TP: [High] Because flusher processes are created via multiprocessing.get_context('spawn').P... - TP: [Medium] Sleep in test_consumer.py won’t actually wait because time.sleep was monkeypatch... - FN: [Medium] Breaking out of the loop when the deadline has elapsed can skip terminating rema... - -sentry-greptile PR#5: 3 golden, 5 candidates - TP: [Medium] Detector validator uses wrong key when updating type... - TP: [Low] Using zip(error_ids, events.values()) assumes the get_multi result preserves the... - FN: [Medium] Breaking changes in error response format... - -sentry-greptile PR#1: 4 golden, 4 candidates - TP: [High] Django querysets do not support negative slicing... - TP: [High] When requests are authenticated with API keys or org auth tokens (which have use... - TP: [High] get_item_key assumes a numeric key, but the paginator is used with order_by=-dat... - FN: [Low] Importing non-existent OptimizedCursorPaginator... - -grafana PR#103633: 2 golden, 1 candidates - TP: [Low] The test comment says the cached permissions 'allow access', but the map stores ... - FN: [High] The Check operation exhibits asymmetric cache trust logic: cached permission gra... - -grafana PR#94942: 2 golden, 1 candidates - TP: [Critical] The enableSqlExpressions function has flawed logic that always returns false, ef... - TP: [High] Several methods such as NewInMemoryDB().RunCommands and db.QueryFramesInto retur... - -discourse-graphite PR#9: 2 golden, 2 candidates - TP: [Low] Thread-safety issue with lazy @loaded_locales... - TP: [Low] Consider normalizing the input locale (e.g., to a symbol) when checking/loading ... - -discourse-graphite PR#10: 4 golden, 6 candidates - TP: [Critical] NoMethodError before_validation in EmbeddableHost... - FN: [Medium] The update and destroy methods in Admin::EmbeddableHostsController do not valida... - FN: [Medium] record_for_host compares lower(host) = ? but does not normalize the parameter’s ... - FN: [High] Because this migration inserts embeddable_hosts rows with raw SQL, any existing ... - -discourse-graphite PR#7: 3 golden, 2 candidates - TP: [Low] In .topic-meta-data h5 a, the original code had color: scale-color($primary, $li... - TP: [Low] This change for desktop/user.css changes $primary from 30% to 50% for the light ... - TP: [Low] In topic-post.css the original code used $lightness: 70% but the replacement use... - -cal.com PR#22532: 2 golden, 5 candidates - TP: [Medium] The updateManyByCredentialId call uses an empty data object, which prevents Pris... - TP: [Low] logic: macOS-specific sed syntax with empty string after -i flag will fail on Li... - -cal.com PR#8330: 2 golden, 2 candidates - TP: [Medium] Incorrect end time calculation using slotStartTime instead of slotEndTime... - TP: [Medium] Using === for dayjs object comparison will always return false as it compares ob... - -cal.com PR#14943: 2 golden, 2 candidates - TP: [High] Using retryCount: reminder.retryCount + 1 reads a possibly stale value and can l... - TP: [High] The deletion logic in scheduleSMSReminders.ts incorrectly deletes non-SMS workfl... - -============================================================ -OVERALL RESULTS -============================================================ - True Positives: 28/43 - False Positives: 17 - False Negatives: 15 - Total Candidates: 41 - Precision: 68.3% - Recall: 65.1% - F1: 66.7% - -Per-repo breakdown: -Repo Prec Recall F1 TP FP FN -------------------------------------------------------- -cal.com 66.7% 100.0% 80.0% 6 3 0 -discourse 60.0% 66.7% 63.2% 6 5 3 -grafana 150.0% 75.0% 100.0% 3 0 1 -keycloak 50.0% 40.0% 44.4% 4 4 6 -sentry 75.0% 75.0% 75.0% 9 5 3 - -Results saved to /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/evaluation_results.json diff --git a/pr_review_agent/output/iter11_mini15.log b/pr_review_agent/output/iter11_mini15.log deleted file mode 100644 index 647760f..0000000 --- a/pr_review_agent/output/iter11_mini15.log +++ /dev/null @@ -1,223 +0,0 @@ -============================================================ -PR Review Agent - Benchmark Pipeline -============================================================ -Loaded 50 PRs -WarpGrep: ENABLED -Mini eval: 15 PRs (3 per repo) -Reviewing 15 PRs - -Running 15 PRs with parallelism=15 - -[1/15] keycloak PR#37429 (4 golden) -[2/15] keycloak PR#37634 (4 golden) -[3/15] keycloak PR#38446 (2 golden) -[4/15] sentry PR#93824 (5 golden) -[5/15] sentry-greptile PR#5 (3 golden) -[6/15] sentry-greptile PR#1 (4 golden) -[7/15] grafana PR#97529 (2 golden) -[8/15] grafana PR#103633 (2 golden) -[9/15] grafana PR#94942 (2 golden) -[10/15] discourse-graphite PR#9 (2 golden) -[11/15] discourse-graphite PR#10 (4 golden) -[12/15] discourse-graphite PR#7 (3 golden) -[13/15] cal.com PR#22532 (2 golden) -[14/15] cal.com PR#8330 (2 golden) -[15/15] cal.com PR#14943 (2 golden) - [6/15] sentry-greptile PR#1 3 files, 128 added lines - [7/15] grafana PR#97529 5 files, 25 added lines - [9/15] grafana PR#94942 4 files, 41 added lines - [15/15] cal.com PR#14943 3 files, 38 added lines - [10/15] discourse-graphite PR#9 6 files, 31 added lines - [12/15] discourse-graphite PR#7 32 files, 115 added lines - [3/15] keycloak PR#38446 8 files, 256 added lines - [4/15] sentry PR#93824 6 files, 199 added lines - [2/15] keycloak PR#37634 28 files, 722 added lines - [11/15] discourse-graphite PR#10 36 files, 449 added lines - [14/15] cal.com PR#8330 4 files, 111 added lines - [5/15] sentry-greptile PR#5 105 files, 2312 added lines - [8/15] grafana PR#103633 4 files, 240 added lines - [13/15] cal.com PR#22532 17 files, 379 added lines - [1/15] keycloak PR#37429 48 files, 343 added lines - WarpGrep: How is the delete of workflow reminders filtered - what methods are deleted and - WarpGrep: Definition of dark-light-choose SCSS function or mixin - WarpGrep: SpanFlusher main method signature and how it's called with arguments - WarpGrep: Does Django QuerySet support negative indexing or negative slicing? - WarpGrep: How does the Check method in the RBAC service work, and what is the flow for che - WarpGrep: How is checkIfIsAvailable called and what parameters does it receive? - WarpGrep: How does Prisma updateMany behave when data is an empty object? Does it still tr - WarpGrep: How does set_locale work in application_controller.rb and what does it return? - WarpGrep: How is the cache mutex used in bleveBackend to protect concurrent access to the - WarpGrep: How does the santizeAnchors method work and what does it do with anchor tags in - WarpGrep: How is I18n.ensure_loaded! defined and used across the codebase? - WarpGrep: How does the REST adapter basePath handle type names with underscores vs hyphens - WarpGrep: How does getFederatedCredentialsStream work and what does it return for user cre - WarpGrep: How is enableSqlExpressions used and what is the expected behavior of the FlagSq - WarpGrep: How does dayjs comparison work with === operator between two dayjs objects? - WarpGrep: SpansBuffer constructor and assigned_shards property - WarpGrep: get_merged_pr_single_issue_template method definition in PRCommentWorkflow - WarpGrep: SelectedCalendarRepository.updateManyByCredentialId callers and usage - WarpGrep: How is organization_context.member used in API endpoints? Can member be None? - WarpGrep: RecoveryAuthnCodesCredentialModel.createFromValues method signature and paramete - WarpGrep: How is permissionCacheUsage metric tracked in the RBAC service? Where are cache - WarpGrep: How does _lookupSubType work in the store and what happens when it returns null - WarpGrep: How is translate_accelerator load_locale used and what is @loaded_locales? - WarpGrep: Lithuanian translation for totpStep1 in login messages - what language should it - WarpGrep: Who calls server.Init and how is the sync.Once pattern used for initialization? - WarpGrep: fetch_error_details function and how nodestore.backend.get_multi returns data or - WarpGrep: How is connectedCalendar.cacheUpdatedAt used in the frontend components - WarpGrep: Where is FlagSqlExpressions defined and how is it used across the codebase? - WarpGrep: How does multiprocessing.Process pass target args vs threading.Thread for pickli - WarpGrep: UserCredentialModel constructor and what getChallengeResponse returns - WarpGrep: category_fabricator.rb and embeddable_host_fabricator.rb fabricators for categor - WarpGrep: What does checkPermission do in the RBAC service? How does it determine if a req - WarpGrep: FlexCenter component replaced with Flex, overflowEllipsis theme mixin usage - WarpGrep: Who calls NewResourceServer and how do callers use the returned server? - WarpGrep: How does Prisma @updatedAt work with updateMany when data is empty object - WarpGrep: How does BackwardsCompatibilityUserStorage implement CredentialInputUpdater and - WarpGrep: What does bleveBackend BuildIndex do and who calls it concurrently? - WarpGrep: RecoveryAuthnCodesFormAuthenticator authenticate method and how it checks creden - WarpGrep: PRCommentWorkflow class definition and format_comment_subtitle method - WarpGrep: analytics.record preprod_artifact with user_id when user might be anonymous - WarpGrep: Visualize class fromJSON method and how it's called - WarpGrep: SelectedCalendar model definition in Prisma schema with updatedAt field - WarpGrep: Who calls the copy constructor Context(Context context) in OAuth2GrantType? - WarpGrep: run_with_initialized_sentry function signature and how it wraps target functions - WarpGrep: nodestore backend get_multi ordering of returned results dictionary - WarpGrep: How does the CalendarService fetchAvailabilityAndSetCache method use credential - WarpGrep: TableWidgetVisualization used in chart.tsx with empty data and columns - WarpGrep: How are workflow reminders deleted in bulk - what filters are applied to deleteM - WarpGrep: Where is set_locale defined as a before_action or method in ApplicationControlle - WarpGrep: How does RecoveryAuthnCodesCredentialProvider validate recovery codes - isValid - WarpGrep: Does Prisma updateMany support @updatedAt automatic update or is it only for upd - WarpGrep: nodestore get_multi dictionary key ordering guarantees - WarpGrep: How does self.paginate pass extra kwargs to the paginator constructor in API end - WarpGrep: UserCredentialManager isValid method implementation that routes to user storage - WarpGrep: Event.generate_node_id method returns node ID format for nodestore - WarpGrep: How is the scheduleSMSReminders cron job invoked - is there protection against c - WarpGrep: How does the store call basePath and what type format is used (underscore vs hyp - WarpGrep: What other workflow methods exist besides SMS - WhatsApp and Email workflow remi - WarpGrep: How does dayjs isSame work for comparing two dayjs date objects in this codebase - WarpGrep: TableWidgetVisualization component props interface columns tableData - Loop done round=10 (tools: codebase_search=1, grep=2, read_file=6, glob=1, bash=8) - Review complete: 2 issues - [12/15] discourse-graphite PR#7 2 raw -> 2 kept - [12/15] discourse-graphite PR#7 [0.95] incorrect_value: The lightness values are swapped between light and dark theme arguments. The original code - [12/15] discourse-graphite PR#7 [0.82] incorrect_value: The light-theme lightness value was changed from the original 20% to 50%, breaking the lig - WarpGrep: Where is ReadQuery called in the expression query pipeline and what happens with - WarpGrep: WhatsApp or Email workflow reminders that use retryCount to control scheduling r - WarpGrep: How does the deleteCache handler verify the credential belongs to the user for a - WarpGrep: Where is getSlots called with organizerTimeZone and how are dateOverrides passed - WarpGrep connection error, retrying in 3s (attempt 1/3): SSLError - Loop done round=6 (tools: read_file=5, codebase_search=5, grep=5) - Review complete: 2 issues - [15/15] cal.com PR#14943 2 raw -> 2 kept - [15/15] cal.com PR#14943 [0.95] logic_error: The second `OR` branch in the `deleteMany` query — `{ retryCount: { gt: 1 } }` — is missin - [15/15] cal.com PR#14943 [0.60] race_condition: The retry count increment uses a non-atomic read-modify-write pattern: `retryCount: remind - WarpGrep: request.user.id when user might be AnonymousUser in Django REST framework - WarpGrep: EnterRecoveryAuthnCodePage getRecoveryAuthnCodeToEnterNumber implementation - WarpGrep: How does EmbeddableHost.record_for_host handle when called from TopicEmbed.impor - WarpGrep: How are dateOverrides and workingHours passed to the availability check in getSc - Loop done round=18 (tools: read_file=23, grep=11, codebase_search=1) - Review complete: 2 issues - [2/15] keycloak PR#37634 2 raw -> 2 kept - [2/15] keycloak PR#37634 [0.98] incorrect_value: Copy-paste bug: The second `Objects.requireNonNull` checks `grantType` instead of `rawToke - [2/15] keycloak PR#37634 [0.97] test_correctness: Two bugs on a single line in the `isAccessTokenId` matcher: - -1. **Wrong substring indices* - WarpGrep: RecoveryAuthnCodesCredentialProviderFactory PROVIDER_ID constant definition - WarpGrep: How is CalendarCacheRepository instantiated throughout the codebase? Is there a - Loop done round=14 (tools: codebase_search=3, grep=14, read_file=7, list_directory=1) - Review complete: 1 issues - [9/15] grafana PR#94942 1 raw -> 1 kept - [9/15] grafana PR#94942 [0.97] logic_error: The `enableSqlExpressions` function always returns `false`, regardless of the feature flag - WarpGrep: SpansBuffer constructor with shards parameter and slice_id - how does it set up - WarpGrep: How does SpansBuffer.process_spans work with sharding - does it write to specifi - WarpGrep: How is the cache interface defined for authzlib clients? What does WithCacheClie - Max tool rounds reached (tools: codebase_search=12, read_file=29, grep=9, glob=1, bash=1) - WarpGrep: How does PG::Result indexing work when accessing rows from empty result sets in - WarpGrep: What type is ThemeField.name - is it a string or symbol database attribute? - Loop done round=21 (tools: codebase_search=5, read_file=8, glob=4, grep=9, bash=2) - Review complete: 2 issues - [14/15] cal.com PR#8330 2 raw -> 2 kept - [14/15] cal.com PR#8330 [0.97] api_misuse: Using `===` to compare two dayjs objects compares object references, not values. Two separ - [14/15] cal.com PR#8330 [0.97] incorrect_value: Both `start` and `end` are computed from `slotStartTime`, making them identical values. Th - Loop done round=16 (tools: codebase_search=6, grep=12, read_file=7, bash=4) - Review complete: 3 issues - [4/15] sentry PR#93824 3 raw -> 3 kept - [4/15] sentry PR#93824 [0.97] type_error: `isinstance(process, multiprocessing.Process)` is always `False` for processes created via - [4/15] sentry PR#93824 [0.95] incorrect_value: Inconsistent metric tag key: `"shards"` (plural) is used here, while all other nearby metr - [4/15] sentry PR#93824 [0.85] test_correctness: The newly added `time.sleep(0.1)` on line 61 is a no-op because `time.sleep` was already m - WarpGrep: SetupRecoveryAuthnCodesPage implementation including checkLogoutSessions and unc - WarpGrep: How does the EmbeddableHost before_validation handle nil host values? - WarpGrep: How are message properties with double curly braces placeholders like {{0}} proc - WarpGrep: Does Prisma updateMany support @updatedAt automatic field update or does it need - WarpGrep: recovery-codes-setup-form.ftl or login-recovery-authn-code-config.ftl logout-ses - WarpGrep: Where is expandable_first_post used and what was the embeddable_hosts check purp - WarpGrep: How does the Keycloak admin UI or account UI handle error-invalid-multivalued-si - Loop done round=22 (tools: codebase_search=3, grep=21, read_file=15, glob=1, bash=3) - Review complete: 4 issues - [6/15] sentry-greptile PR#1 4 raw -> 4 kept - [6/15] sentry-greptile PR#1 [0.97] type_error: `OptimizedCursorPaginator.get_item_key` will raise `TypeError` at runtime when used with t - [6/15] sentry-greptile PR#1 [0.95] api_misuse: Django QuerySets do NOT support negative indexing. The code `queryset[start_offset:stop]` - [6/15] sentry-greptile PR#1 [0.85] null_reference: `organization_context.member` can be `None` per the model definition (`RpcUserOrganization - [6/15] sentry-greptile PR#1 [0.82] api_misuse: The change to `BasePaginator.get_result` introduces a conditional that allows negative off - WarpGrep: How does the Keycloak React frontend (admin-ui or account-ui) interpolate messag - Max tool rounds reached (tools: codebase_search=4, read_file=23, grep=25, bash=2, glob=1) - WarpGrep: How are user profile validation errors displayed in the admin-ui React frontend, - Review complete: 5 issues - [5/15] sentry-greptile PR#5 5 raw -> 5 kept - [5/15] sentry-greptile PR#5 [0.93] logic_error: The `fetch_error_details` function uses `zip(error_ids, events.values())` to pair error ID - [5/15] sentry-greptile PR#5 [0.92] logic_error: The `fetch_error_details` function uses `zip(error_ids, events.values())` which assumes th - [5/15] sentry-greptile PR#5 [0.85] logic_error: The `HIDDEN_ATTRIBUTES` filter is only applied in the search query branch. When `searchQue - [5/15] sentry-greptile PR#5 [0.82] incorrect_value: When the `use-table-widget-visualization` feature flag is enabled, the `TableWidgetVisuali - [5/15] sentry-greptile PR#5 [0.78] logic_error: The cross-field validators `validate_timestamp` and `validate_age` use truthiness checks ( - Loop done round=18 (tools: codebase_search=8, grep=13, read_file=17, glob=3) - Review complete: 6 issues - [11/15] discourse-graphite PR#10 6 raw -> 6 kept - [11/15] discourse-graphite PR#10 [0.95] incorrect_value: The fabricator file contents were swapped. `category_fabricator.rb` now contains the `Fabr - [11/15] discourse-graphite PR#10 [0.95] null_reference: The migration crashes when the `embed_category` site setting doesn't exist or doesn't matc - [11/15] discourse-graphite PR#10 [0.85] logic_error: In the new plural IDs branch of `_hydrateEmbedded`, undefined values from failed lookups a - [11/15] discourse-graphite PR#10 [0.82] security: SQL injection via string interpolation of host values in the migration: -```ruby -execute "I - [11/15] discourse-graphite PR#10 [0.80] logic_error: The `update` action doesn't actually persist any changes — it just re-renders the current - [11/15] discourse-graphite PR#10 [0.70] logic_error: JavaScript's `String.replace()` with a string argument only replaces the **first** occurre - Loop done round=16 (tools: codebase_search=10, grep=19, read_file=9, bash=2, list_directory=1, glob=2) - Review complete: 5 issues - [13/15] cal.com PR#22532 5 raw -> 5 kept - [13/15] cal.com PR#22532 [0.85] portability: `sed -i '' -E "s|...|...|"` uses macOS/BSD `sed` syntax. On GNU/Linux `sed`, the `-i` flag - [13/15] cal.com PR#22532 [0.85] portability: `sed -i '' -E "s|^GOOGLE_WEBHOOK_URL=.*|GOOGLE_WEBHOOK_URL=$TUNNEL_URL|" "$ENV_FILE"` uses - [13/15] cal.com PR#22532 [0.70] api_misuse: The call `await SelectedCalendarRepository.updateManyByCredentialId(this.credential.id, {} - [13/15] cal.com PR#22532 [0.70] api_misuse: The `updateManyByCredentialId` method signature uses `Prisma.SelectedCalendarUpdateInput` - [13/15] cal.com PR#22532 [0.60] incorrect_value: The code `cacheStatusMap.get(calendar.credentialId) || null` uses `||` (logical OR) to con - Review complete: 0 issues - [7/15] grafana PR#97529 0 raw -> 0 kept - Max tool rounds reached (tools: codebase_search=5, grep=22, read_file=21, glob=2, bash=2) - Review complete: 2 issues - [10/15] discourse-graphite PR#9 2 raw -> 2 kept - [10/15] discourse-graphite PR#9 [0.90] type_error: `FallbackLocaleList#[]` does not convert its `locale` parameter to a symbol. In Ruby, `"en - [10/15] discourse-graphite PR#9 [0.72] type_error: `ensure_loaded!` does not convert its `locale` parameter to a symbol before checking `@loa - Max tool rounds reached (tools: read_file=16, codebase_search=6, grep=30, glob=1, bash=8) - Review complete: 3 issues - [1/15] keycloak PR#37429 3 raw -> 3 kept - [1/15] keycloak PR#37429 [0.97] localization: The Lithuanian (lt) translation for `totpStep1` was replaced with Italian text: "Installa - [1/15] keycloak PR#37429 [0.95] localization: The Simplified Chinese (`zh_CN`) translation for `totpStep1` was replaced with Traditional - [1/15] keycloak PR#37429 [0.95] incorrect_value: The `error-invalid-multivalued-size` message is changed from `{{0}}`/`{{1}}`/`{{2}}` (i18n - Loop done round=21 (tools: codebase_search=4, read_file=23, grep=7, bash=4) - Review complete: 1 issues - [8/15] grafana PR#103633 1 raw -> 1 kept - [8/15] grafana PR#103633 [0.92] test_correctness: The test "Should deny on explicit cache deny entry" has a setup that contradicts its state - WarpGrep: Where is hashRawCode defined in RecoveryAuthnCodesUtils and what type does it ex - Loop done round=25 (tools: codebase_search=12, grep=20, read_file=11, bash=1) - Review complete: 3 issues - [3/15] keycloak PR#38446 3 raw -> 3 kept - [3/15] keycloak PR#38446 [0.85] null_reference: The new `getCredentials()` method does not null-check the return value of `getMyUser(user) - [3/15] keycloak PR#38446 [0.82] logic_error: The `setupRecoveryKeysForUserWithRequiredAction` method has inverted logout checkbox logic - [3/15] keycloak PR#38446 [0.75] logic_error: In `isValid()` for recovery codes, the validation checks if the user's input matches ANY s -Wrote candidates to /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/candidates.json - -============================================================ -DONE: 15 reviewed, 41 raw -> 41 filtered -Avg/PR: 2.7, Time: 760s -Candidates: /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/candidates.json -Benchmark data updated: /Users/tejas/personal/applymodel/zmisc/examples/code-review-benchmark/offline/results/benchmark_data.json diff --git a/pr_review_agent/output/iter12_mini15.log b/pr_review_agent/output/iter12_mini15.log deleted file mode 100644 index 53408c5..0000000 --- a/pr_review_agent/output/iter12_mini15.log +++ /dev/null @@ -1,171 +0,0 @@ -============================================================ -PR Review Agent - Benchmark Pipeline -============================================================ -Loaded 50 PRs -WarpGrep: ENABLED -Mini eval: 15 PRs (3 per repo) -Reviewing 15 PRs - -Running 15 PRs with parallelism=15 - -[1/15] keycloak PR#37429 (4 golden) -[2/15] keycloak PR#37634 (4 golden) -[3/15] keycloak PR#38446 (2 golden) -[4/15] sentry PR#93824 (5 golden) -[5/15] sentry-greptile PR#5 (3 golden) -[6/15] sentry-greptile PR#1 (4 golden) -[7/15] grafana PR#97529 (2 golden) -[8/15] grafana PR#103633 (2 golden) -[9/15] grafana PR#94942 (2 golden) -[10/15] discourse-graphite PR#9 (2 golden) -[11/15] discourse-graphite PR#10 (4 golden) -[12/15] discourse-graphite PR#7 (3 golden) -[13/15] cal.com PR#22532 (2 golden) -[14/15] cal.com PR#8330 (2 golden) -[15/15] cal.com PR#14943 (2 golden) - [10/15] discourse-graphite PR#9 6 files, 31 added lines - [9/15] grafana PR#94942 4 files, 41 added lines - [6/15] sentry-greptile PR#1 3 files, 128 added lines - [2/15] keycloak PR#37634 28 files, 722 added lines - [15/15] cal.com PR#14943 3 files, 38 added lines - [11/15] discourse-graphite PR#10 36 files, 449 added lines - [8/15] grafana PR#103633 4 files, 240 added lines - [4/15] sentry PR#93824 6 files, 199 added lines - [13/15] cal.com PR#22532 17 files, 379 added lines - [3/15] keycloak PR#38446 8 files, 256 added lines - [7/15] grafana PR#97529 5 files, 25 added lines - [14/15] cal.com PR#8330 4 files, 111 added lines - [12/15] discourse-graphite PR#7 32 files, 115 added lines - [1/15] keycloak PR#37429 48 files, 343 added lines - [5/15] sentry-greptile PR#5 105 files, 2312 added lines - WarpGrep: How is the deleteMany for workflowReminder used and what conditions protect agai - WarpGrep: SpanFlusher class initialization and how it's created with buffer and shards - WarpGrep: How is AccessTokenContext constructor called and what null checks does it perfor - WarpGrep: Does Django QuerySet support negative indexing or negative slicing? - WarpGrep: How is checkIfIsAvailable called and what parameters does it receive in the slot - WarpGrep: How does I18n.ensure_loaded! work and where is it defined? - WarpGrep: How is the santizeAnchors method used in VerifyMessageProperties and what anchor - WarpGrep: How is the server Init method called and what does it do? Look at the sync.Once - WarpGrep: How does the category_fabricator.rb define fabricators for categories? - WarpGrep: How does Prisma updateMany behave when data object is empty? Does it still trigg - WarpGrep: Where is the category Fabricator defined in spec/fabricators? - WarpGrep: How does RecoveryAuthnCodesCredentialModel.createFromValues work and what parame - WarpGrep: How does the getSlots function use activeOverrides and dateOverrides for schedul - WarpGrep: What is the PartialWorkflowReminder type definition and what fields does it incl - WarpGrep: isAccessTokenId matcher used in test assertions for token ID validation - WarpGrep: How is enableSqlExpressions called and what does the feature flag FlagSqlExpress - WarpGrep: PRCommentWorkflow get_merged_pr_single_issue_template method definition - WarpGrep: How does the Check method in the RBAC service work, including checkPermission an - WarpGrep: SpansBuffer class constructor and assigned_shards property - WarpGrep: Definition of dark-light-choose SCSS function or mixin - WarpGrep: SelectedCalendarRepository updateManyByCredentialId callers and usage - WarpGrep: getFederatedCredentialsStream method definition in credential manager - WarpGrep: How is set_locale method in application_controller used and what does it return? - WarpGrep: scheduleEmailReminders handler that uses deleteMany and workflowReminder with re - WarpGrep: fetch_error_details nodestore get_multi ordering guarantee - WarpGrep: How does UserCredentialModel constructor work and what does the challengeRespons - WarpGrep: Who calls QueryTypeSQL in the expression reader and how is it used in alerts? - WarpGrep: How is connectedCalendars handler return type used by callers? What type does th - WarpGrep: What concurrency protections exist around the bleveBackend cache map? Who reads - WarpGrep: How is the permDenialCache invalidated or cleared when permissions change? - WarpGrep: updateCredential method on user credential manager that takes CredentialInput, r - WarpGrep: FlexBox height 100% usage in traceWaterfall - WarpGrep: Where is cacheUpdatedAt accessed in the connected calendar components? - WarpGrep: How is the SpansBuffer constructor called, what parameters does it take for shar - WarpGrep: How does the copy constructor for OAuth2GrantType.Context get used, and what cal - WarpGrep: overflowEllipsis usage in FlexCenter for event attachments - WarpGrep: Prisma updateMany with empty data object - does it trigger @updatedAt? - WarpGrep: How does the Check method in the diff version of rbac/service.go handle cached p - WarpGrep: RecoveryAuthnCodesFormAuthenticator authenticate method and how it looks up cred - WarpGrep: Who calls BuildIndex on bleveBackend and can it be called concurrently for the s - WarpGrep: analytics record preprod_artifact before feature check - WarpGrep: get_slow_conditions_for_groups return type and behavior - WarpGrep: What does SiteSetting.default_locale return, is it a string or symbol? - WarpGrep: BackwardsCompatibilityUserStorage getCredentials method before this change, does - WarpGrep: TableWidgetVisualization columns and tableData props used in dashboard chart - WarpGrep: set_locale method definition in application controller, around_action or before_ - WarpGrep: Attributes filtering with HIDDEN_ATTRIBUTES and empty searchQuery - WarpGrep: How does the santizeAnchors method handle the matcher after replacing parts of t - WarpGrep: How are dateOverrides and workingHours used in the checkIfIsAvailable function f - WarpGrep: RecoveryAuthnCodesCredentialProvider isValid method implementation for validatin - WarpGrep: hashRawCode method in RecoveryAuthnCodesUtils that hashes recovery codes - WarpGrep: nodestore get_multi dictionary ordering guarantee - does it preserve insertion o - WarpGrep: Where is the RefreshTokenGrantType shortcut "rt" defined and does it collide wit - WarpGrep: Where is FallbackLocaleList class defined and what module is it in? - WarpGrep: _lookupSubType implementation in the store model - WarpGrep: ClientCredentialsGrantType process method where it creates access token and sets - WarpGrep: How does dayjs equality comparison work with === versus .isSame() for dayjs obje - WarpGrep: How does the deleteCache trpc route verify credential ownership? Is there an aut - WarpGrep: How is the server's storageEnabled field used and set? What happens when storage - WarpGrep: Where is createTokenResponse called and which grant types use it vs have their o - WarpGrep: UserCredentialManager isValid method implementation for validating credential in - WarpGrep: SpansBuffer slice_id how is it used in buffer operations like flush_segments - WarpGrep: ExpressionQueryReader struct definition and its features field - WarpGrep connection error, retrying in 3s (attempt 1/3): SSLError - WarpGrep: Where is ReadQuery called on ExpressionQueryReader and how is the result used fo - WarpGrep: How does the EmbeddableHost model validate host format and handle before_validat - WarpGrep: Prisma increment operator for atomic counter updates, retryCount increment - Loop done round=5 (tools: codebase_search=1, grep=2, read_file=6, glob=4) - Review complete: 1 issues - [12/15] discourse-graphite PR#7 1 raw -> 1 kept - [12/15] discourse-graphite PR#7 [0.95] incorrect_value: The light-mode value for `.reply-details a` color was changed from `$lightness: 30%` to `$ - WarpGrep: How does the RefreshTokenGrantType set the grant type attribute on clientSession - Loop done round=5 (tools: read_file=4, codebase_search=4, grep=4) - Review complete: 2 issues - [15/15] cal.com PR#14943 2 raw -> 2 kept - [15/15] cal.com PR#14943 [0.95] logic_error: The `deleteMany` OR condition's second branch `{ retryCount: { gt: 1 } }` has NO `method: - [15/15] cal.com PR#14943 [0.65] race_condition: The retry count is incremented non-atomically using `reminder.retryCount + 1` (a read-modi - WarpGrep: How is checkRequest.UserUID populated in validateCheckRequest? Is it the same as - WarpGrep connection error, retrying in 3s (attempt 1/3): SSLError - WarpGrep: Where is CalendarCacheRepository class instantiated vs where is the mock used? H - WarpGrep: How does the account-ui or admin-ui JavaScript frontend handle message format pl - Loop done round=9 (tools: codebase_search=4, grep=8, read_file=6, glob=1, bash=1) - Review complete: 2 issues - [9/15] grafana PR#94942 2 raw -> 2 kept - [9/15] grafana PR#94942 [0.97] logic_error: The `enableSqlExpressions` function always returns `false`, regardless of the feature flag - [9/15] grafana PR#94942 [0.85] logic_error: All methods on the new stub `DB` (`TablesList`, `RunCommands`, `QueryFramesInto`) uncondit - WarpGrep: How does the account-ui frontend resolve message placeholders with double curly - WarpGrep: How does the Discourse store handle find without an id parameter - store.find wi - WarpGrep: How is embed_category site setting used for TopicEmbed import to assign category - Loop done round=15 (tools: codebase_search=4, grep=10, read_file=11, glob=2, list_directory=1) - Review complete: 2 issues - [14/15] cal.com PR#8330 2 raw -> 2 kept - [14/15] cal.com PR#8330 [0.97] api_misuse: Bug: `===` compares object references, not values, for dayjs objects. The expression `dayj - [14/15] cal.com PR#8330 [0.97] incorrect_value: Bug: Copy-paste error — `end` is computed identically to `start`, both using `slotStartTim - WarpGrep: How does getOrCreateIndex protect against concurrent calls for the same key in t - WarpGrep: How does the Keycloak server format error messages for user profile validation e - WarpGrep: What interface does the cache option for authzlib.NewClient expect? What methods - WarpGrep: How does the Keycloak frontend account-ui or admin-ui handle error messages with - WarpGrep: BrowserReportSerializer validate_age validate_timestamp mutual exclusion logic - WarpGrep: store.find method implementation in Discourse store service - how it handles fin - WarpGrep: How are freedom patches loaded and what initializer loads translate_accelerator? - Loop done round=17 (tools: codebase_search=7, grep=9, read_file=18) - Review complete: 2 issues - [2/15] keycloak PR#37634 2 raw -> 2 kept - [2/15] keycloak PR#37634 [0.97] incorrect_value: Copy-paste bug in null check: `Objects.requireNonNull(grantType, "Null rawTokenId not allo - [2/15] keycloak PR#37634 [0.97] test_correctness: Two bugs in the `isAccessTokenId` matcher on the same line: - -1. **Wrong substring indices* - Max tool rounds reached (tools: codebase_search=1, grep=20, read_file=19, glob=1, bash=4) - WarpGrep: RecoveryAuthnCodesAuthenticatorTest enterRecoveryCodes method for how existing t - Max tool rounds reached (tools: codebase_search=10, read_file=25, grep=11, glob=2) - Review complete: 3 issues - [6/15] sentry-greptile PR#1 3 raw -> 3 kept - [6/15] sentry-greptile PR#1 [0.97] api_misuse: Django QuerySets do NOT support negative indexing/slicing. The code `results = list(querys - [6/15] sentry-greptile PR#1 [0.92] logic_error: The change to `BasePaginator.get_result()` introduces asymmetric offset handling: `start_o - [6/15] sentry-greptile PR#1 [0.70] null_reference: `organization_context.member` is typed as `RpcOrganizationMember | None` (see `model.py` l - WarpGrep: How does Keycloak serialize ValidationError for REST API responses, does it form - WarpGrep: How recovery authn codes are formatted for display with dashes in the FTL templa - WarpGrep: How does Keycloak REST API serialize user profile validation errors to JSON resp - Loop done round=7 (tools: codebase_search=4, grep=5, read_file=5, bash=2) - Review complete: 2 issues - [4/15] sentry PR#93824 2 raw -> 2 kept - [4/15] sentry PR#93824 [0.97] api_misuse: `isinstance(process, multiprocessing.Process)` is always `False` for processes created via - [4/15] sentry PR#93824 [0.95] incorrect_value: Inconsistent metric tag key: `"shards"` (plural) is used here, while all other nearby metr - Review complete: 4 issues - [5/15] sentry-greptile PR#5 4 raw -> 4 kept - [5/15] sentry-greptile PR#5 [0.93] incorrect_value: When the `use-table-widget-visualization` feature flag is enabled, `TableWidgetVisualizati - [5/15] sentry-greptile PR#5 [0.92] logic_error: Bug: `zip(error_ids, events.values())` assumes the dict returned by `nodestore.backend.get - [5/15] sentry-greptile PR#5 [0.88] logic_error: The `HIDDEN_ATTRIBUTES` filter is only added to the branch where a search query is present - [5/15] sentry-greptile PR#5 [0.75] logic_error: The `analytics.record("preprod_artifact.api.assemble", ...)` call is placed before the fea - WarpGrep: How does the before_validation callback in EmbeddableHost handle nil host values diff --git a/pr_review_agent/output/iter3_eval.log b/pr_review_agent/output/iter3_eval.log deleted file mode 100644 index ff92578..0000000 --- a/pr_review_agent/output/iter3_eval.log +++ /dev/null @@ -1,76 +0,0 @@ - -keycloak PR#37634: 4 golden, 3 candidates - TP: [Critical] Wrong parameter in null check (grantType vs. rawTokenId)... - TP: [High] In isAccessTokenId, the substring for the grant shortcut and the equality check ... - FN: [Low] Javadoc mentions "usually like 3-letters shortcut" but some implementations use ... - FN: [Low] Catching generic RuntimeException is too broad. The implementation throws Illeg... - -keycloak PR#38446: 2 golden, 1 candidates - TP: [Low] After creating the RecoveryAuthnCodesCredentialModel, consider setting its id fr... - FN: [Medium] Unsafe raw List deserialization without type safety. Calling Optional.get() dire... - -sentry PR#93824: 5 golden, 2 candidates - TP: [Medium] Inconsistent metric tagging with 'shard' and 'shards'... - TP: [Low] Fixed sleep in tests can be flaky; wait on condition instead... - TP: [Medium] Sleep in test_consumer.py won’t actually wait because time.sleep was monkeypatch... - FN: [High] Because flusher processes are created via multiprocessing.get_context('spawn').P... - FN: [Medium] Breaking out of the loop when the deadline has elapsed can skip terminating rema... - -sentry-greptile PR#1: 4 golden, 2 candidates - TP: [High] Django querysets do not support negative slicing... - TP: [High] When requests are authenticated with API keys or org auth tokens (which have use... - FN: [Low] Importing non-existent OptimizedCursorPaginator... - FN: [High] get_item_key assumes a numeric key, but the paginator is used with order_by=-dat... - -grafana PR#103633: 2 golden, 2 candidates - TP: [Low] The test comment says the cached permissions 'allow access', but the map stores ... - FN: [High] The Check operation exhibits asymmetric cache trust logic: cached permission gra... - -grafana PR#94942: 2 golden, 1 candidates - TP: [Critical] The enableSqlExpressions function has flawed logic that always returns false, ef... - FN: [High] Several methods such as NewInMemoryDB().RunCommands and db.QueryFramesInto retur... - -discourse-graphite PR#9: 2 golden, 2 candidates - TP: [Low] Consider normalizing the input locale (e.g., to a symbol) when checking/loading ... - FN: [Low] Thread-safety issue with lazy @loaded_locales... - -discourse-graphite PR#10: 4 golden, 3 candidates - FN: [Critical] NoMethodError before_validation in EmbeddableHost... - FN: [Medium] The update and destroy methods in Admin::EmbeddableHostsController do not valida... - FN: [Medium] record_for_host compares lower(host) = ? but does not normalize the parameter’s ... - FN: [High] Because this migration inserts embeddable_hosts rows with raw SQL, any existing ... - -discourse-graphite PR#7: 3 golden, 4 candidates - TP: [Low] In .topic-meta-data h5 a, the original code had color: scale-color($primary, $li... - TP: [Low] This change for desktop/user.css changes $primary from 30% to 50% for the light ... - TP: [Low] In topic-post.css the original code used $lightness: 70% but the replacement use... - -cal.com PR#8330: 2 golden, 2 candidates - TP: [Medium] Incorrect end time calculation using slotStartTime instead of slotEndTime... - TP: [Medium] Using === for dayjs object comparison will always return false as it compares ob... - -cal.com PR#14943: 2 golden, 1 candidates - TP: [High] The deletion logic in scheduleSMSReminders.ts incorrectly deletes non-SMS workfl... - FN: [High] Using retryCount: reminder.retryCount + 1 reads a possibly stale value and can l... - -============================================================ -OVERALL RESULTS -============================================================ - True Positives: 17/43 - False Positives: 7 - False Negatives: 26 - Total Candidates: 23 - Precision: 73.9% - Recall: 39.5% - F1: 51.5% - -Per-repo breakdown: -Repo Prec Recall F1 TP FP FN -------------------------------------------------------- -cal.com 100.0% 75.0% 85.7% 3 0 1 -discourse 44.4% 44.4% 44.4% 4 5 5 -grafana 66.7% 50.0% 57.1% 2 1 2 -keycloak 75.0% 50.0% 60.0% 3 1 3 -sentry 125.0% 55.6% 76.9% 5 0 4 - -Results saved to /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/evaluation_results.json diff --git a/pr_review_agent/output/iter3_mini15.log b/pr_review_agent/output/iter3_mini15.log deleted file mode 100644 index b4575b1..0000000 --- a/pr_review_agent/output/iter3_mini15.log +++ /dev/null @@ -1,228 +0,0 @@ -============================================================ -PR Review Agent - Benchmark Pipeline -============================================================ -Loaded 50 PRs -WarpGrep: ENABLED -Mini eval: 15 PRs (3 per repo) -Reviewing 15 PRs - -Running 15 PRs with parallelism=10 - -[1/15] keycloak PR#37429 (4 golden) -[2/15] keycloak PR#37634 (4 golden) -[3/15] keycloak PR#38446 (2 golden) -[4/15] sentry PR#93824 (5 golden) -[6/15] sentry-greptile PR#1 (4 golden) -[5/15] sentry-greptile PR#5 (3 golden) -[8/15] grafana PR#103633 (2 golden) -[9/15] grafana PR#94942 (2 golden) -[10/15] discourse-graphite PR#9 (2 golden) -[7/15] grafana PR#97529 (2 golden) - [9/15] grafana PR#94942 4 files, 41 added lines - [7/15] grafana PR#97529 5 files, 25 added lines - [10/15] discourse-graphite PR#9 6 files, 31 added lines - [4/15] sentry PR#93824 6 files, 199 added lines - [2/15] keycloak PR#37634 28 files, 722 added lines - [8/15] grafana PR#103633 4 files, 240 added lines - [3/15] keycloak PR#38446 8 files, 256 added lines - [6/15] sentry-greptile PR#1 3 files, 128 added lines - [1/15] keycloak PR#37429 48 files, 343 added lines - [5/15] sentry-greptile PR#5 105 files, 2312 added lines - WarpGrep: SpansBuffer constructor and assigned_shards property - WarpGrep: Django QuerySet negative slicing behavior with negative indexes - WarpGrep: Who calls getTokenContextFromClientSessionContext and how is AccessTokenContext - WarpGrep: How does RecoveryAuthnCodesUtils.getCredential work and who calls it - WarpGrep: Who calls NewResourceServer and how do they handle initialization - WarpGrep: How does the Check method in rbac service work, and what is checkPermission doin - WarpGrep: How is set_locale used in ApplicationController and what does it return - WarpGrep: enableSqlExpressions function and its callers in reader.go - WarpGrep: How is the santizeAnchors method used and what does it do with anchor tags in tr - WarpGrep: How is get_merged_pr_single_issue_template called and what parameters does it ex - WarpGrep: FlagSqlExpressions feature flag definition and usage - WarpGrep: bleveBackend BuildIndex concurrency: who calls BuildIndex and what protects the - WarpGrep: Find where messages_lt.properties totpStep1 or loginTotpStep1 is used in Lithuan - WarpGrep: OptimizedCursorPaginator usage and callers - WarpGrep: SpanFlusher.main method signature and how it's called as a process target - WarpGrep: I18n.ensure_loaded! method definition and callers - WarpGrep: All implementations of OAuth2GrantTypeFactory interface getShortcut method - WarpGrep: fetch_error_details function and how nodestore.backend.get_multi returns results - WarpGrep: getFederatedCredentialsStream method definition and implementations in credentia - WarpGrep: getCachedIdentityPermissions callers and how it interacts with getIdentityPermis - WarpGrep: How does BasePaginator.get_result handle offset and cursor parameters - WarpGrep: createRecoveryCodesCredential callers and usage - WarpGrep: Who calls TablesList in sql/parser.go and how is it used - WarpGrep API error (turn 2): 400 Client Error: Bad Request for url: https://api.morphllm.com/v1/chat/completions - WarpGrep: translate_accelerator load_locale and @loaded_locales thread safety - WarpGrep: Find the Italian locale message for totpStep1 or loginTotpStep1 to verify "Insta - WarpGrep: ProcessSpansStrategyFactory create_with_partitions implementation - WarpGrep: bleveBackend cache map reads and writes, concurrent access to b.cache - WarpGrep: UserCredentialModel constructor and how challengeResponse is set - WarpGrep: PRCommentWorkflow class definition and _truncate_title method - WarpGrep: organization_context.member attribute and has_global_access in organization endp - WarpGrep: checkPermission function signature and how it's called with getTree parameter - WarpGrep: Find Chinese Simplified vs Traditional locale files for account messages to chec - WarpGrep: updateCredential method in credential manager that takes UserCredentialModel, wh - WarpGrep: run_with_initialized_sentry function signature and how it wraps target with extr - WarpGrep: server Init method sync.Once pattern and initErr handling - WarpGrep: nodestore get_multi return value ordering - does it preserve insertion order or - WarpGrep: SpansBuffer flush_segments method implementation - WarpGrep: newFolderTreeGetter usage in Check method - WarpGrep: isAccessTokenId method usage in tests, who calls isAccessTokenId - WarpGrep: organization_auditlogs endpoint get method with organization_context parameter - WarpGrep: FlexBox in traceWaterfall height flex-grow behavior when replaced with Flex comp - WarpGrep: RecoveryAuthnCodesCredentialModel.createFromValues method signature and what it - WarpGrep: _import_and_run function unpickle and call main with args - WarpGrep: RecoveryAuthnCodesFormAuthenticator authenticate method and how it validates rec - WarpGrep: fetch_error_details in project_replay_summarize_breadcrumbs nodestore get_multi - WarpGrep: searchSupport init method and how it reads resources during initialization - WarpGrep: Flex.Item component grow prop definition - WarpGrep connection error, retrying in 3s (attempt 1/3): SSLError - WarpGrep API error (turn 2): 400 Client Error: Bad Request for url: https://api.morphllm.com/v1/chat/completions - WarpGrep: How Cursor.from_string parses cursor parameter and validates offset values - WarpGrep: getIdentityPermissions function signature and how it differs from getCachedIdent - WarpGrep: nodestore backend get_multi key ordering guarantee - does the returned dict have - WarpGrep: RecoveryAuthnCodesCredentialProvider class definition and what interfaces it imp - WarpGrep: BackwardsCompatibilityUserStorage getCredentials method and how federated creden - WarpGrep: self.paginate method in endpoint base class, how it instantiates the paginator_c - WarpGrep: Where is setContext called on OAuth2GrantTypeBase, and where does grantType fiel - WarpGrep: AbstractUserAdapterFederatedStorage credentialManager and getFederatedCredential - WarpGrep: Where is set_locale defined as a method in ApplicationController and what before - WarpGrep: validateCheckRequest and how checkReq.UserUID is set - WarpGrep: userPermDenialCacheKey and how UserUID relates to userIdentifiers.UID in the den - WarpGrep: RpcUserOrganizationContext definition and member field type, can member be None - WarpGrep: permDenialCache invalidation, when is the denial cache cleared or evicted - WarpGrep connection error, retrying in 3s (attempt 1/3): SSLError - WarpGrep: NewResourceServer return type - does it return ResourceServer interface or *serv - Loop done round=10 (tools: warpgrep_codebase_search=3, read_file=6, grep=7) - WarpGrep: SpansBuffer slice_id usage and how it affects Redis keys or behavior - Review complete: 1 issues - [9/15] grafana PR#94942 1 raw -> 1 kept - [9/15] grafana PR#94942 [0.97] logic_error: The `enableSqlExpressions` function always returns `false` due to two compounding bugs: (1 -[11/15] discourse-graphite PR#10 (4 golden) - [11/15] discourse-graphite PR#10 36 files, 449 added lines - WarpGrep: Who uses the category fabricator and embeddable_host fabricator in tests? - Loop done round=8 (tools: warpgrep_codebase_search=4, read_file=8, grep=2) - WarpGrep: How does the rest adapter basePath work with type names containing underscores? - Review complete: 3 issues - [2/15] keycloak PR#37634 3 raw -> 3 kept - [2/15] keycloak PR#37634 [0.97] null_reference: Copy-paste error: the second Objects.requireNonNull checks `grantType` instead of `rawToke - [2/15] keycloak PR#37634 [0.97] incorrect_value: Wrong substring indices: `substring(3, 5)` extracts a cross-field slice (last char of toke - [2/15] keycloak PR#37634 [0.97] logic_error: Inverted condition logic in Hamcrest `matchesSafely`: returns `false` when the extracted v -[12/15] discourse-graphite PR#7 (3 golden) - [12/15] discourse-graphite PR#7 32 files, 115 added lines - WarpGrep: How does I18n::Backend::Fallbacks module use I18n.fallbacks in its translate met - Review failed: invalid literal for int() with base 10: '3145">' - [1/15] keycloak PR#37429 0 raw -> 0 kept -[13/15] cal.com PR#22532 (2 golden) - [13/15] cal.com PR#22532 17 files, 379 added lines - WarpGrep: How does the store's _hydrateEmbedded handle plural embedded IDs (e.g. _ids suff - WarpGrep: who calls BuildIndex on bleveBackend and is there a singleflight or other dedupl - WarpGrep: GetUserIdentifiers function implementation and how it maps userID to UID - WarpGrep: Definition and behavior of dark-light-choose SCSS function or mixin - WarpGrep: How is updateManyByCredentialId used in SelectedCalendarRepository and what does - WarpGrep: How are $primary and $secondary color variables defined for light and dark theme - WarpGrep: How does connectedCalendars handler return type get used by callers, especially - WarpGrep: overflowEllipsis theme mixin definition - WarpGrep: SelectedCalendar model schema definition in Prisma, does it have updatedAt field - WarpGrep: What happens when Prisma updateMany is called with empty data object {} - WarpGrep: Flex.Item component definition and grow prop - WarpGrep: Where is connectedCalendarsHandler called and how does it handle the result, spe - WarpGrep: How is type passed to basePath in the rest adapter? What format are type names i - WarpGrep: EnterRecoveryAuthnCodePage getRecoveryAuthnCodeToEnterNumber method implementati - WarpGrep: RecoveryAuthnCodesFormAuthenticator authenticate method full implementation and - WarpGrep: BackwardsCompatibilityUserStorage implements which interfaces including Credenti - WarpGrep connection error, retrying in 3s (attempt 1/3): ConnectionError - WarpGrep: How does EmbeddableHost.record_for_host work and where is it called? - WarpGrep: is_active_superuser vs request.user.is_superuser difference and when they diverg - WarpGrep: How is CalendarCacheRepository instantiated and does it require constructor argu - WarpGrep: resource.IndexMetrics initialization - can it be nil when accessed - WarpGrep: Where is DiscourseI18n backend instantiated and configured - Loop done round=10 (tools: warpgrep_codebase_search=2, read_file=5, glob=4, bash=4) - Review complete: 4 issues - [12/15] discourse-graphite PR#7 4 raw -> 4 kept - [12/15] discourse-graphite PR#7 [0.95] incorrect_value: Swapped light/dark arguments in dark-light-choose for .custom-message-length. The light-th - [12/15] discourse-graphite PR#7 [0.92] incorrect_value: Light-theme regression in embedded topic reply link (.row > a). The original light-theme v - [12/15] discourse-graphite PR#7 [0.85] incorrect_value: Light-theme regression in .name color (same as desktop/user.scss). The original lightness - [12/15] discourse-graphite PR#7 [0.85] incorrect_value: Light-theme regression in mobile h3 heading color. The original $lightness: 20% (near-blac -[14/15] cal.com PR#8330 (2 golden) - [14/15] cal.com PR#8330 4 files, 111 added lines - WarpGrep: SetupRecoveryAuthnCodesPage getRecoveryAuthnCodes method implementation - WarpGrep: How is checkIfIsAvailable called and what parameters does it receive - WarpGrep: getSlots function and how activeOverrides are processed with timezone - WarpGrep: How does the store pluralize method work for plural type names? - WarpGrep: authzlib.NewClient WithCacheClientOption how cache is used in the authz client l - WarpGrep: checkIfIsAvailable function definition and its callers - Loop done round=24 (tools: warpgrep_codebase_search=9, grep=10, read_file=14, bash=5) - Review complete: 2 issues - [6/15] sentry-greptile PR#1 2 raw -> 2 kept - [6/15] sentry-greptile PR#1 [0.95] api_misuse: Django QuerySets do not support negative indexing. The OptimizedCursorPaginator attempts ` - [6/15] sentry-greptile PR#1 [0.72] null_reference: `organization_context.member` can be `None` (typed as `RpcOrganizationMember | None`). The -[15/15] cal.com PR#14943 (2 golden) - [15/15] cal.com PR#14943 3 files, 38 added lines - WarpGrep: How is scheduleSMSReminders handler called and what does the full handler look l - WarpGrep: CredentialInputUpdater interface getCredentials default method - WarpGrep: WorkflowReminder deleteMany method filter conditions - WarpGrep: How does Discourse handle initializer ordering with "order: after" comments - WarpGrep: How are initializers ordered in Discourse - does "order: after" comment work or - WarpGrep: scheduleEmailReminders handler retryCount workflow reminders - WarpGrep: dayjs object comparison with triple equals === in scheduling availability slots - WarpGrep: WorkflowReminder retryCount usage across all workflow scheduling handlers - WarpGrep: use-table-widget-visualization feature flag definition or usage - WarpGrep: How is the analytics record call ordered relative to feature flag checks in sent - WarpGrep: How consumer click options are passed to ProcessSpansStrategyFactory constructor - Loop done round=12 (tools: warpgrep_codebase_search=4, read_file=4, glob=2, grep=8) - Review complete: 2 issues - [14/15] cal.com PR#8330 2 raw -> 2 kept - [14/15] cal.com PR#8330 [0.97] api_misuse: Object reference comparison (`===`) on dayjs objects always returns false. Two independent - [14/15] cal.com PR#8330 [0.96] logic_error: Copy-paste error: `end` is calculated from `slotStartTime` instead of `slotEndTime`. Both - WarpGrep: How is the embedding controller update action supposed to work and what paramete - WarpGrep: searchSupport type definition and struct fields - the old version before searchS - Loop done round=11 (tools: warpgrep_codebase_search=8, read_file=9, grep=5) - Review complete: 2 issues - [4/15] sentry PR#93824 2 raw -> 2 kept - [4/15] sentry PR#93824 [0.95] test_correctness: `time.sleep(0.1)` is called after `monkeypatch.setattr("time.sleep", lambda _: None)` has - [4/15] sentry PR#93824 [0.92] incorrect_value: The metric tag key `"shards"` (plural) is used in `flusher.wait_produce` timer, while all - Loop done round=5 (tools: warpgrep_codebase_search=4, read_file=2, grep=1) - Loop done round=15 (tools: warpgrep_codebase_search=7, read_file=13, grep=7) - Review complete: 1 issues - [15/15] cal.com PR#14943 1 raw -> 1 kept - [15/15] cal.com PR#14943 [0.95] logic_error: The `deleteMany` query's second OR branch `{ retryCount: { gt: 1 } }` lacks a `method: Wor - WarpGrep: cache.Cache interface definition with Get Set Delete methods - Review complete: 3 issues - [11/15] discourse-graphite PR#10 3 raw -> 3 kept - [11/15] discourse-graphite PR#10 [0.95] null_reference: When the `embed_category` site setting doesn't exist, the query returns zero rows. `execut - [11/15] discourse-graphite PR#10 [0.95] test_correctness: The file contents of `category_fabricator.rb` and `embeddable_host_fabricator.rb` are swap - [11/15] discourse-graphite PR#10 [0.72] logic_error: In the plural `_ids` hydration branch: (1) Failed lookups from `_lookupSubType` returning - Max tool rounds reached (tools: warpgrep_codebase_search=12, read_file=18, grep=4, bash=18) - Review complete: 0 issues - [5/15] sentry-greptile PR#5 0 raw -> 0 kept - WarpGrep: getConnectedDestinationCalendars function and how it uses selectedCalendar updat - Loop done round=22 (tools: warpgrep_codebase_search=11, read_file=11, grep=7, bash=3) - Review complete: 2 issues - [8/15] grafana PR#103633 2 raw -> 2 kept - [8/15] grafana PR#103633 [0.93] test_correctness: The permCache scope map value is `false` instead of `true`, making the test vacuously pass - [8/15] grafana PR#103633 [0.50] logic_error: The `userPermDenialCacheKey` function uses `_` as delimiter between `name` and `parent` fi - WarpGrep: Where is SelectedCalendarsSettingsWebWrapper rendered and what is passed as onCh - Loop done round=19 (tools: warpgrep_codebase_search=8, read_file=13, grep=8, bash=5) - Review complete: 2 issues - [10/15] discourse-graphite PR#9 2 raw -> 2 kept - [10/15] discourse-graphite PR#9 [0.75] type_error: `FallbackLocaleList#[]` does not call `.to_sym` on the `locale` parameter before placing i - [10/15] discourse-graphite PR#9 [0.75] type_error: `ensure_loaded!` does not normalize `locale` to a symbol, but `@loaded_locales` stores sym - Loop done round=17 (tools: warpgrep_codebase_search=15, grep=3, read_file=12, glob=1) - Review complete: 1 issues - [3/15] keycloak PR#38446 1 raw -> 1 kept - [3/15] keycloak PR#38446 [0.82] logic_error: Missing credential ID on model returned by getCredentials(). RecoveryAuthnCodesCredentialM - WarpGrep: searchServer init starts background goroutines that need cleanup - what context - Max tool rounds reached (tools: warpgrep_codebase_search=8, read_file=12, glob=5, grep=12, bash=9) - Review complete: 0 issues - [13/15] cal.com PR#22532 0 raw -> 0 kept - Max tool rounds reached (tools: warpgrep_codebase_search=10, read_file=17, grep=21, bash=2) - Review complete: 0 issues - [7/15] grafana PR#97529 0 raw -> 0 kept -Wrote candidates to /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/candidates.json - -============================================================ -DONE: 15 reviewed, 23 raw -> 23 filtered -Avg/PR: 1.5, Time: 704s -Candidates: /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/candidates.json -Benchmark data updated: /Users/tejas/personal/applymodel/zmisc/examples/code-review-benchmark/offline/results/benchmark_data.json diff --git a/pr_review_agent/output/iter4_eval.log b/pr_review_agent/output/iter4_eval.log deleted file mode 100644 index 1360ebd..0000000 --- a/pr_review_agent/output/iter4_eval.log +++ /dev/null @@ -1,86 +0,0 @@ - -keycloak PR#37429: 4 golden, 3 candidates - TP: [Medium] The translation is in Italian instead of Lithuanian. This should be translated t... - TP: [Medium] The totpStep1 value uses Traditional Chinese terms in the Simplified Chinese fil... - FN: [Low] The anchor sanitization logic has a potential issue where it consumes English ma... - FN: [Low] The method name 'santizeAnchors' should be 'sanitizeAnchors' (missing 'i').... - -keycloak PR#37634: 4 golden, 2 candidates - TP: [Critical] Wrong parameter in null check (grantType vs. rawTokenId)... - TP: [High] In isAccessTokenId, the substring for the grant shortcut and the equality check ... - FN: [Low] Javadoc mentions "usually like 3-letters shortcut" but some implementations use ... - FN: [Low] Catching generic RuntimeException is too broad. The implementation throws Illeg... - -keycloak PR#38446: 2 golden, 1 candidates - FN: [Medium] Unsafe raw List deserialization without type safety. Calling Optional.get() dire... - FN: [Low] After creating the RecoveryAuthnCodesCredentialModel, consider setting its id fr... - -sentry PR#93824: 5 golden, 3 candidates - TP: [Medium] Inconsistent metric tagging with 'shard' and 'shards'... - TP: [Low] Fixed sleep in tests can be flaky; wait on condition instead... - TP: [Medium] Sleep in test_consumer.py won’t actually wait because time.sleep was monkeypatch... - FN: [High] Because flusher processes are created via multiprocessing.get_context('spawn').P... - FN: [Medium] Breaking out of the loop when the deadline has elapsed can skip terminating rema... - -sentry-greptile PR#1: 4 golden, 3 candidates - TP: [High] Django querysets do not support negative slicing... - TP: [High] When requests are authenticated with API keys or org auth tokens (which have use... - FN: [Low] Importing non-existent OptimizedCursorPaginator... - FN: [High] get_item_key assumes a numeric key, but the paginator is used with order_by=-dat... - -grafana PR#97529: 2 golden, 1 candidates - FN: [High] A race condition in BuildIndex allows multiple goroutines to concurrently build ... - FN: [High] Calling s.search.TotalDocs() here may race with concurrent index creation: Total... - -grafana PR#103633: 2 golden, 1 candidates - FN: [High] The Check operation exhibits asymmetric cache trust logic: cached permission gra... - FN: [Low] The test comment says the cached permissions 'allow access', but the map stores ... - -grafana PR#94942: 2 golden, 1 candidates - TP: [Critical] The enableSqlExpressions function has flawed logic that always returns false, ef... - TP: [High] Several methods such as NewInMemoryDB().RunCommands and db.QueryFramesInto retur... - -discourse-graphite PR#9: 2 golden, 2 candidates - TP: [Low] Consider normalizing the input locale (e.g., to a symbol) when checking/loading ... - FN: [Low] Thread-safety issue with lazy @loaded_locales... - -discourse-graphite PR#10: 4 golden, 3 candidates - FN: [Critical] NoMethodError before_validation in EmbeddableHost... - FN: [Medium] The update and destroy methods in Admin::EmbeddableHostsController do not valida... - FN: [Medium] record_for_host compares lower(host) = ? but does not normalize the parameter’s ... - FN: [High] Because this migration inserts embeddable_hosts rows with raw SQL, any existing ... - -discourse-graphite PR#7: 3 golden, 3 candidates - TP: [Low] In .topic-meta-data h5 a, the original code had color: scale-color($primary, $li... - TP: [Low] This change for desktop/user.css changes $primary from 30% to 50% for the light ... - TP: [Low] In topic-post.css the original code used $lightness: 70% but the replacement use... - -cal.com PR#8330: 2 golden, 2 candidates - TP: [Medium] Incorrect end time calculation using slotStartTime instead of slotEndTime... - TP: [Medium] Using === for dayjs object comparison will always return false as it compares ob... - -cal.com PR#14943: 2 golden, 1 candidates - TP: [High] The deletion logic in scheduleSMSReminders.ts incorrectly deletes non-SMS workfl... - FN: [High] Using retryCount: reminder.retryCount + 1 reads a possibly stale value and can l... - -============================================================ -OVERALL RESULTS -============================================================ - True Positives: 18/43 - False Positives: 11 - False Negatives: 25 - Total Candidates: 26 - Precision: 69.2% - Recall: 41.9% - F1: 52.2% - -Per-repo breakdown: -Repo Prec Recall F1 TP FP FN -------------------------------------------------------- -cal.com 100.0% 75.0% 85.7% 3 0 1 -discourse 50.0% 44.4% 47.1% 4 5 5 -grafana 66.7% 33.3% 44.4% 2 2 4 -keycloak 66.7% 40.0% 50.0% 4 2 6 -sentry 83.3% 55.6% 66.7% 5 2 4 - -Results saved to /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/evaluation_results.json diff --git a/pr_review_agent/output/iter4_mini15.log b/pr_review_agent/output/iter4_mini15.log deleted file mode 100644 index 7afe31f..0000000 --- a/pr_review_agent/output/iter4_mini15.log +++ /dev/null @@ -1,215 +0,0 @@ -============================================================ -PR Review Agent - Benchmark Pipeline -============================================================ -Loaded 50 PRs -WarpGrep: ENABLED -Mini eval: 15 PRs (3 per repo) -Reviewing 15 PRs - -Running 15 PRs with parallelism=15 - -[1/15] keycloak PR#37429 (4 golden) -[2/15] keycloak PR#37634 (4 golden) -[3/15] keycloak PR#38446 (2 golden) -[4/15] sentry PR#93824 (5 golden) -[5/15] sentry-greptile PR#5 (3 golden) -[6/15] sentry-greptile PR#1 (4 golden) -[8/15] grafana PR#103633 (2 golden) -[7/15] grafana PR#97529 (2 golden) -[10/15] discourse-graphite PR#9 (2 golden) -[11/15] discourse-graphite PR#10 (4 golden) -[9/15] grafana PR#94942 (2 golden) -[13/15] cal.com PR#22532 (2 golden) -[12/15] discourse-graphite PR#7 (3 golden) -[14/15] cal.com PR#8330 (2 golden) -[15/15] cal.com PR#14943 (2 golden) - [7/15] grafana PR#97529 5 files, 25 added lines - [14/15] cal.com PR#8330 4 files, 111 added lines - [15/15] cal.com PR#14943 3 files, 38 added lines - [2/15] keycloak PR#37634 28 files, 722 added lines - [8/15] grafana PR#103633 4 files, 240 added lines - [9/15] grafana PR#94942 4 files, 41 added lines - [10/15] discourse-graphite PR#9 6 files, 31 added lines - [12/15] discourse-graphite PR#7 32 files, 115 added lines - [6/15] sentry-greptile PR#1 3 files, 128 added lines - [13/15] cal.com PR#22532 17 files, 379 added lines - [4/15] sentry PR#93824 6 files, 199 added lines - [11/15] discourse-graphite PR#10 36 files, 449 added lines - [3/15] keycloak PR#38446 8 files, 256 added lines - [1/15] keycloak PR#37429 48 files, 343 added lines - [5/15] sentry-greptile PR#5 105 files, 2312 added lines - WarpGrep: checkIfIsAvailable function callers and how dateOverrides and workingHours are p - WarpGrep: How is the delete of workflow reminders used and what methods are covered (SMS, - WarpGrep: SpanFlusher.main method signature and how it's called as a process target - WarpGrep: Find where category_fabricator.rb is used or where category fabricator is define - WarpGrep: Django QuerySet negative slicing behavior and how it's handled in paginator - WarpGrep: How is I18n.ensure_loaded! defined and called - what module does it belong to - WarpGrep: Who calls the Context copy constructor of OAuth2GrantType.Context - WarpGrep: How is bleveBackend.BuildIndex called and what concurrency protections exist for - WarpGrep: How is checkPermission called and what does it return in the authz rbac service - WarpGrep: How does getFederatedCredentialsStream work and what does it return for recovery - WarpGrep: How dayjs objects are compared for equality in the codebase - WarpGrep: Definition of dark-light-choose SCSS mixin or function - WarpGrep: How is updateManyByCredentialId used in SelectedCalendarRepository and what does - WarpGrep: How does santizeAnchors work in VerifyMessageProperties and what is the anchor s - WarpGrep: enableSqlExpressions function and how SQL expressions feature flag is checked - WarpGrep: scheduleSMSReminders handler and how retryCount is used in workflow reminders - WarpGrep: SpansBuffer constructor and assigned_shards property - WarpGrep: How does the store.find method work for singular resources vs collections in the - WarpGrep: How is set_locale method used in ApplicationController and what does it return - WarpGrep: Find callers of get_merged_pr_single_issue_template and format_comment_subtitle - WarpGrep: OptimizedCursorPaginator usage and callers - WarpGrep: How does the dark-light-choose function work in the SCSS color system - WarpGrep: How is updateCredential used for user credential storage - what does it return ( - WarpGrep: Find where the Lithuanian locale messages_lt.properties totpStep1 is used and wh - WarpGrep: How does connectedCalendars type flow from handler to the UI component, what typ - WarpGrep connection error, retrying in 3s (attempt 1/3): SSLError - WarpGrep: organization_context.member attribute in audit logs endpoint - WarpGrep: Who calls newRBACClient and how is it used - WarpGrep: FlagSqlExpressions feature flag definition and usage - WarpGrep: Where is cacheUpdatedAt accessed on connectedCalendar objects in components - WarpGrep: How does _lookupSubType work in the store and what does it return when lookup fa - WarpGrep: How getSlots in packages/lib/slots.ts uses timeZone parameter and activeOverride - WarpGrep: set_locale method in ApplicationController - what is its return value used for, - WarpGrep: scheduleWhatsappReminders handler delete deleteMany workflow reminders with retr - WarpGrep: Who calls server.Init and how is it used in NewResourceServer - WarpGrep: Find how fetch_error_details uses nodestore.backend.get_multi and how the return - WarpGrep: ProcessSpansStrategyFactory create_with_partitions implementation - WarpGrep: What are $primary and $secondary color variables in Discourse SCSS themes - WarpGrep: Who calls QueryTypeSQL and how SQL expression queries are processed - WarpGrep: RecoveryAuthnCodesCredentialModel createFromValues method signature and paramete - WarpGrep: Find the Chinese zh_CN account messages totpStep1 and verify whether it uses Sim - WarpGrep: checkIfIsAvailable function definition in slots - WarpGrep: PRCommentWorkflow class definition and _truncate_title method location - WarpGrep: RecoveryAuthnCodesCredentialModel createFromCredentialModel method - how does it - WarpGrep: nodestore backend get_multi return type - does it preserve insertion order of ke - WarpGrep: server.Init sync.Once initialization pattern in resource server - WarpGrep: How does basePath replace underscores with dashes and does it handle multiple un - WarpGrep: Where is ensure_loaded! called on I18n.fallbacks, what is ensure_loaded! on Fall - WarpGrep: How does the TableWidgetVisualization pass empty columns and data when feature f - WarpGrep: checkPermission function signature and all call sites in authz rbac service - WarpGrep: JavaScript String.replace with string first argument only replaces first occurre - WarpGrep: Where does the EmbeddableHost destroy method get called and is null check needed - WarpGrep: Find where FlexCenter had overflowEllipsis styling that was removed in eventAtta - WarpGrep: SelectedCalendar model schema definition with updatedAt field - WarpGrep: RecoveryAuthnCodesFormAuthenticator authenticate method and how it checks if rec - WarpGrep: What reminder methods exist (EMAIL, SMS, WHATSAPP, AI_PHONE_CALL) and do any of - WarpGrep: Find where overflowEllipsis was applied to FlexCenter that was replaced with Fle - WarpGrep: Who calls BuildIndex on bleveBackend and can it be called concurrently for the s - WarpGrep: Find how nodestore get_multi returns its results - is the dictionary keyed by no - WarpGrep: UserCredentialModel buildFromBackupAuthnCode method definition - WarpGrep: ExpressionQueryReader struct definition and ReadQuery method - WarpGrep: RecoveryAuthnCodesCredentialProvider isValid method - how does it validate recov - WarpGrep: callers of setFormParams on OAuth2GrantType Context - WarpGrep: Find the Flex component - does it handle the height: 100% that was on FlexBox in - WarpGrep: UserCredentialManager isValid method implementation for federated users - Loop done round=3 (tools: warpgrep_codebase_search=4, read_file=1, grep=1) - WarpGrep: Does Prisma updateMany with empty data object actually update @updatedAt fields - Review complete: 1 issues - [15/15] cal.com PR#14943 1 raw -> 1 kept - [15/15] cal.com PR#14943 [0.95] logic_error: The `deleteMany` query's second OR branch `{ retryCount: { gt: 1 } }` lacks a `method` fil - WarpGrep: Find the santizeAnchors method - does it have a bug where replaceFirst modifies - WarpGrep: _hydrateEmbedded implementation for handling _ids plural embedded objects in sto - WarpGrep: How are Rails initializers ordered in config/initializers, what order do they ru - WarpGrep: getIdentityPermissions and actionSets parameter usage in getCachedIdentityPermis - Loop done round=11 (tools: read_file=13, warpgrep_codebase_search=2, grep=6) - WarpGrep: SpansBuffer slice_id how it's used for Redis connection or key routing - Review complete: 2 issues - [2/15] keycloak PR#37634 2 raw -> 2 kept - [2/15] keycloak PR#37634 [0.97] logic_error: Copy-paste error: Line 114 checks `grantType` a second time instead of `rawTokenId`. `Obje - [2/15] keycloak PR#37634 [0.97] test_correctness: Two compounded bugs: (1) Wrong substring indices — uses `.substring(3, 5)` but the grant t - WarpGrep: SQL injection in migration via string interpolation in INSERT INTO embeddable_ho - WarpGrep: run_with_initialized_sentry function signature and how it wraps arguments for mu - WarpGrep: How does embedding controller saveChanges actually persist settings to the datab - Loop done round=10 (tools: warpgrep_codebase_search=4, read_file=5, glob=1, grep=3, bash=3) - WarpGrep: How are delegation credentials associated with users - do they have userId set - Review complete: 1 issues - [9/15] grafana PR#94942 1 raw -> 1 kept - [9/15] grafana PR#94942 [0.97] logic_error: The function `enableSqlExpressions` always returns `false` in every code path due to two c - WarpGrep: Find how analytics.record handles user_id when request.user is AnonymousUser or - WarpGrep: _import_and_run how pickled args and additional process args are combined - Loop done round=19 (tools: warpgrep_codebase_search=4, read_file=5, glob=5, grep=12, list_directory=1) - WarpGrep: How does SelectedCalendarsSettingsWebWrapper get connectedCalendars data and wha - Review complete: 2 issues - [14/15] cal.com PR#8330 2 raw -> 2 kept - [14/15] cal.com PR#8330 [0.95] api_misuse: `===` reference comparison on dayjs objects always evaluates to `false` since `dayjs(...)` - [14/15] cal.com PR#8330 [0.95] incorrect_value: Copy-paste error: `end` is computed from `slotStartTime` instead of `slotEndTime`. Both `s - WarpGrep: createFromValues vs createFromCredentialModel recovery authn codes - what data f - Loop done round=7 (tools: warpgrep_codebase_search=3, read_file=3, glob=2, grep=2, bash=1) - Review complete: 3 issues - [12/15] discourse-graphite PR#7 3 raw -> 3 kept - [12/15] discourse-graphite PR#7 [0.95] incorrect_value: The arguments to `dark-light-choose` are swapped for `.custom-message-length`. The origina - [12/15] discourse-graphite PR#7 [0.90] incorrect_value: The `h3` light-theme lightness value was changed from the original `20%` to `50%`, making - [12/15] discourse-graphite PR#7 [0.70] incorrect_value: The `.name` light-theme lightness value was changed from the original `30%` to `50%`, redu - WarpGrep: store.find for singular resource without id, how does REST adapter handle it - WarpGrep: cacheWrap struct definition and Get/Set methods in authz rbac - WarpGrep: CalendarCacheRepository constructor and how it is instantiated, is it a singleto - WarpGrep: getRecoveryAuthnCodeToEnterNumber method implementation in EnterRecoveryAuthnCod - Loop done round=27 (tools: warpgrep_codebase_search=3, grep=16, read_file=15, glob=1, bash=3) - WarpGrep: BackwardsCompatibilityUserStorage getCredentials method - does it already overri - Review complete: 3 issues - [6/15] sentry-greptile PR#1 3 raw -> 3 kept - [6/15] sentry-greptile PR#1 [0.97] api_misuse: Django QuerySets do not support negative indexing. When `cursor.offset < 0` and `enable_ad - [6/15] sentry-greptile PR#1 [0.75] api_misuse: In modified `BasePaginator.get_result()`, when `cursor.is_prev` is True, `offset` is used - [6/15] sentry-greptile PR#1 [0.60] null_reference: `organization_context.member` can be `None` (typed as `RpcOrganizationMember | None`). For - WarpGrep: Find the original FlexBox and Container styled components that were removed from - Loop done round=14 (tools: warpgrep_codebase_search=4, read_file=11, grep=13, glob=1) - Review complete: 3 issues - [1/15] keycloak PR#37429 3 raw -> 3 kept - [1/15] keycloak PR#37429 [0.97] localization: The totpStep1/loginTotpStep1 text is in Italian ("Installa una delle seguenti applicazioni - [1/15] keycloak PR#37429 [0.95] localization: The totpStep1 replacement uses Traditional Chinese characters (手機, 安裝, 應用程式) in a Simplifi - [1/15] keycloak PR#37429 [0.82] logic_error: In verifySafeHtml(), the end-trimming loop doesn't account for `start`, allowing `start + - WarpGrep: How does CredentialActionsDropdown handle the case where integrationType prop is - WarpGrep: How are recovery authn codes generated and formatted - are dashes included in ge - WarpGrep: Find the Flex.Item component definition with grow prop - WarpGrep: DiscourseI18n class definition and its parent/superclass - Loop done round=10 (tools: warpgrep_codebase_search=6, read_file=7, grep=6) - Loop done round=17 (tools: warpgrep_codebase_search=10, read_file=11, grep=8, bash=2) - WarpGrep: RecoveryAuthnCodesCredentialProviderFactory PROVIDER_ID constant definition - Review complete: 3 issues - [4/15] sentry PR#93824 3 raw -> 3 kept - [4/15] sentry PR#93824 [0.95] logic_error: Missing `slice_id` propagation to per-shard `SpansBuffer` instances. In `_create_process_f - [4/15] sentry PR#93824 [0.92] test_correctness: `time.sleep(0.1)` is a no-op because `time.sleep` was monkeypatched to `lambda _: None` ea - [4/15] sentry PR#93824 [0.88] incorrect_value: Inconsistent metrics tag key: `wait_produce` timer uses `"shards"` (plural) while adjacent - Review complete: 3 issues - [11/15] discourse-graphite PR#10 3 raw -> 3 kept - [11/15] discourse-graphite PR#10 [0.95] null_reference: When no 'embed_category' site setting exists, the SQL query returns an empty PG::Result. A - [11/15] discourse-graphite PR#10 [0.92] logic_error: File contents are swapped between category_fabricator.rb and embeddable_host_fabricator.rb - [11/15] discourse-graphite PR#10 [0.85] logic_error: Two issues in _hydrateEmbedded for plural _ids case: (1) `hydrated || []` is dead code sin - Max tool rounds reached (tools: warpgrep_codebase_search=9, grep=14, read_file=14, glob=5, bash=6) - Review complete: 0 issues - [13/15] cal.com PR#22532 0 raw -> 0 kept - WarpGrep: Grafana dashboard UID validation rules or allowed characters - WarpGrep: How does I18n::Backend::Fallbacks use I18n.fallbacks in its translate method - WarpGrep: How does the BrowserReportSerializer validate_age work when age is 0 (falsy valu - Max tool rounds reached (tools: warpgrep_codebase_search=13, read_file=23, grep=10, glob=1, bash=3) - Review complete: 0 issues - [5/15] sentry-greptile PR#5 0 raw -> 0 kept - Loop done round=20 (tools: warpgrep_codebase_search=4, read_file=15, grep=6, bash=4) - Review complete: 1 issues - [7/15] grafana PR#97529 1 raw -> 1 kept - [7/15] grafana PR#97529 [0.90] type_error: `err :=` should be `err =` in `NewResourceServer`. The variable `err` is already declared - WarpGrep: SetupRecoveryAuthnCodesPage getRecoveryAuthnCodes method - what format are the c - WarpGrep: GetUserIdentifiers implementation and caching in authz rbac store - WarpGrep: cache.Cache interface definition in authlib with Get Set Delete methods - Loop done round=25 (tools: warpgrep_codebase_search=7, read_file=13, grep=15, list_directory=1, bash=5) - Review complete: 2 issues - [10/15] discourse-graphite PR#9 2 raw -> 2 kept - [10/15] discourse-graphite PR#9 [0.75] api_misuse: The PR sets `I18n.fallbacks = FallbackLocaleList.new` (which defines `ensure_loaded!`) in - [10/15] discourse-graphite PR#9 [0.50] type_error: Bug in `ensure_loaded!`: the `locale` parameter is not converted to a Symbol via `.to_sym` - Loop done round=24 (tools: warpgrep_codebase_search=8, read_file=16, grep=7, bash=5) - Review complete: 1 issues - [8/15] grafana PR#103633 1 raw -> 1 kept - [8/15] grafana PR#103633 [0.75] security: Cache key collision in `userPermDenialCacheKey` due to using `_` as separator between `nam - WarpGrep: RecoveryAuthnCodesAuthenticatorTest setupRecoveryKeys logoutOtherSessions how is - Loop done round=25 (tools: warpgrep_codebase_search=15, read_file=13, grep=9, glob=1, bash=1) - Review complete: 1 issues - [3/15] keycloak PR#38446 1 raw -> 1 kept - [3/15] keycloak PR#38446 [0.85] logic_error: The `logoutOtherSessions` condition is inverted. When `logoutOtherSessions=true`, the `if -Wrote candidates to /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/candidates.json - -============================================================ -DONE: 15 reviewed, 26 raw -> 26 filtered -Avg/PR: 1.7, Time: 1045s -Candidates: /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/candidates.json -Benchmark data updated: /Users/tejas/personal/applymodel/zmisc/examples/code-review-benchmark/offline/results/benchmark_data.json diff --git a/pr_review_agent/output/iter5_eval.log b/pr_review_agent/output/iter5_eval.log deleted file mode 100644 index 72ff2db..0000000 --- a/pr_review_agent/output/iter5_eval.log +++ /dev/null @@ -1,91 +0,0 @@ - -keycloak PR#37429: 4 golden, 2 candidates - TP: [Medium] The translation is in Italian instead of Lithuanian. This should be translated t... - TP: [Medium] The totpStep1 value uses Traditional Chinese terms in the Simplified Chinese fil... - FN: [Low] The anchor sanitization logic has a potential issue where it consumes English ma... - FN: [Low] The method name 'santizeAnchors' should be 'sanitizeAnchors' (missing 'i').... - -keycloak PR#37634: 4 golden, 2 candidates - TP: [Critical] Wrong parameter in null check (grantType vs. rawTokenId)... - TP: [High] In isAccessTokenId, the substring for the grant shortcut and the equality check ... - FN: [Low] Javadoc mentions "usually like 3-letters shortcut" but some implementations use ... - FN: [Low] Catching generic RuntimeException is too broad. The implementation throws Illeg... - -keycloak PR#38446: 2 golden, 1 candidates - TP: [Low] After creating the RecoveryAuthnCodesCredentialModel, consider setting its id fr... - FN: [Medium] Unsafe raw List deserialization without type safety. Calling Optional.get() dire... - -sentry PR#93824: 5 golden, 2 candidates - TP: [Low] Fixed sleep in tests can be flaky; wait on condition instead... - TP: [Medium] Sleep in test_consumer.py won’t actually wait because time.sleep was monkeypatch... - FN: [Medium] Inconsistent metric tagging with 'shard' and 'shards'... - FN: [High] Because flusher processes are created via multiprocessing.get_context('spawn').P... - FN: [Medium] Breaking out of the loop when the deadline has elapsed can skip terminating rema... - -sentry-greptile PR#5: 3 golden, 1 candidates - FN: [Medium] Breaking changes in error response format... - FN: [Medium] Detector validator uses wrong key when updating type... - FN: [Low] Using zip(error_ids, events.values()) assumes the get_multi result preserves the... - -sentry-greptile PR#1: 4 golden, 2 candidates - TP: [High] Django querysets do not support negative slicing... - TP: [High] get_item_key assumes a numeric key, but the paginator is used with order_by=-dat... - FN: [Low] Importing non-existent OptimizedCursorPaginator... - FN: [High] When requests are authenticated with API keys or org auth tokens (which have use... - -grafana PR#97529: 2 golden, 1 candidates - FN: [High] A race condition in BuildIndex allows multiple goroutines to concurrently build ... - FN: [High] Calling s.search.TotalDocs() here may race with concurrent index creation: Total... - -grafana PR#103633: 2 golden, 1 candidates - TP: [Low] The test comment says the cached permissions 'allow access', but the map stores ... - FN: [High] The Check operation exhibits asymmetric cache trust logic: cached permission gra... - -grafana PR#94942: 2 golden, 1 candidates - TP: [Critical] The enableSqlExpressions function has flawed logic that always returns false, ef... - FN: [High] Several methods such as NewInMemoryDB().RunCommands and db.QueryFramesInto retur... - -discourse-graphite PR#9: 2 golden, 1 candidates - TP: [Low] Consider normalizing the input locale (e.g., to a symbol) when checking/loading ... - FN: [Low] Thread-safety issue with lazy @loaded_locales... - -discourse-graphite PR#10: 4 golden, 4 candidates - TP: [Medium] The update and destroy methods in Admin::EmbeddableHostsController do not valida... - FN: [Critical] NoMethodError before_validation in EmbeddableHost... - FN: [Medium] record_for_host compares lower(host) = ? but does not normalize the parameter’s ... - FN: [High] Because this migration inserts embeddable_hosts rows with raw SQL, any existing ... - -discourse-graphite PR#7: 3 golden, 4 candidates - TP: [Low] In .topic-meta-data h5 a, the original code had color: scale-color($primary, $li... - TP: [Low] This change for desktop/user.css changes $primary from 30% to 50% for the light ... - TP: [Low] In topic-post.css the original code used $lightness: 70% but the replacement use... - -cal.com PR#8330: 2 golden, 2 candidates - TP: [Medium] Incorrect end time calculation using slotStartTime instead of slotEndTime... - TP: [Medium] Using === for dayjs object comparison will always return false as it compares ob... - -cal.com PR#14943: 2 golden, 1 candidates - TP: [High] The deletion logic in scheduleSMSReminders.ts incorrectly deletes non-SMS workfl... - FN: [High] Using retryCount: reminder.retryCount + 1 reads a possibly stale value and can l... - -============================================================ -OVERALL RESULTS -============================================================ - True Positives: 19/43 - False Positives: 7 - False Negatives: 24 - Total Candidates: 25 - Precision: 76.0% - Recall: 44.2% - F1: 55.9% - -Per-repo breakdown: -Repo Prec Recall F1 TP FP FN -------------------------------------------------------- -cal.com 100.0% 75.0% 85.7% 3 0 1 -discourse 55.6% 55.6% 55.6% 5 4 4 -grafana 66.7% 33.3% 44.4% 2 1 4 -keycloak 100.0% 50.0% 66.7% 5 0 5 -sentry 80.0% 33.3% 47.1% 4 2 8 - -Results saved to /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/evaluation_results.json diff --git a/pr_review_agent/output/iter5_mini15.log b/pr_review_agent/output/iter5_mini15.log deleted file mode 100644 index 5e1c98e..0000000 --- a/pr_review_agent/output/iter5_mini15.log +++ /dev/null @@ -1,213 +0,0 @@ -============================================================ -PR Review Agent - Benchmark Pipeline -============================================================ -Loaded 50 PRs -WarpGrep: ENABLED -Mini eval: 15 PRs (3 per repo) -Reviewing 15 PRs - -Running 15 PRs with parallelism=15 - -[1/15] keycloak PR#37429 (4 golden) -[2/15] keycloak PR#37634 (4 golden) -[3/15] keycloak PR#38446 (2 golden) -[4/15] sentry PR#93824 (5 golden) -[5/15] sentry-greptile PR#5 (3 golden) -[6/15] sentry-greptile PR#1 (4 golden) -[7/15] grafana PR#97529 (2 golden) -[8/15] grafana PR#103633 (2 golden) -[9/15] grafana PR#94942 (2 golden) -[10/15] discourse-graphite PR#9 (2 golden) -[11/15] discourse-graphite PR#10 (4 golden) -[12/15] discourse-graphite PR#7 (3 golden) -[13/15] cal.com PR#22532 (2 golden) -[14/15] cal.com PR#8330 (2 golden) -[15/15] cal.com PR#14943 (2 golden) - [15/15] cal.com PR#14943 3 files, 38 added lines - [6/15] sentry-greptile PR#1 3 files, 128 added lines - [1/15] keycloak PR#37429 48 files, 343 added lines - [9/15] grafana PR#94942 4 files, 41 added lines - [2/15] keycloak PR#37634 28 files, 722 added lines - [5/15] sentry-greptile PR#5 105 files, 2312 added lines - [4/15] sentry PR#93824 6 files, 199 added lines - [10/15] discourse-graphite PR#9 6 files, 31 added lines - [3/15] keycloak PR#38446 8 files, 256 added lines - [11/15] discourse-graphite PR#10 36 files, 449 added lines - [14/15] cal.com PR#8330 4 files, 111 added lines - [13/15] cal.com PR#22532 17 files, 379 added lines - [8/15] grafana PR#103633 4 files, 240 added lines - [12/15] discourse-graphite PR#7 32 files, 115 added lines - [7/15] grafana PR#97529 5 files, 25 added lines - WarpGrep: SpansBuffer class definition and constructor parameters - WarpGrep: How is the scheduleSMSReminders handler called and what does the full flow look - WarpGrep: Django QuerySet negative indexing or negative slicing behavior - WarpGrep: enableSqlExpressions function and its callers in reader.go - WarpGrep: How is get_merged_pr_single_issue_template called and what arguments does it exp - WarpGrep: How is I18n.ensure_loaded! defined and used - is it a method on I18n module or I - WarpGrep: Who calls isAccessTokenId and how is the grant shortcut matched in the token ID - WarpGrep: What happens when Prisma updateMany is called with an empty data object? Does it - WarpGrep: getFederatedCredentialsStream method definition and its callers - WarpGrep: How is BuildIndex called and what concurrency protections exist for the bleve ca - WarpGrep: Find all usages of the category fabricator (Fabricate(:category), Fabricate(:dif - WarpGrep: How is the Check method in authz rbac service used, and what is the flow for per - WarpGrep: How does santizeAnchors work in VerifyMessageProperties and what does it do with - WarpGrep: FlagSqlExpressions feature flag definition and usage - WarpGrep: How is checkIfIsAvailable called and what parameters are passed to it - WarpGrep: How is SpanFlusher.main called as process target, what arguments does it receive - WarpGrep: WorkflowReminder deleteMany with method filter, what types of reminders exist (S - WarpGrep: createRecoveryCodesCredential callers and usage - WarpGrep: How is SelectedCalendarRepository.updateManyByCredentialId called and what does - WarpGrep: fetch_error_details and how nodestore.backend.get_multi returns results, orderin - WarpGrep: Find callers of basePath in the rest adapter and how type parameter is passed wi - WarpGrep: Where is @loaded_locales used in translate_accelerator and how is it managed for - WarpGrep: organization_context.member in OrganizationAuditPermission or ControlSiloOrganiz - WarpGrep: getSlots function and how activeOverrides are used with timezone - WarpGrep: Find the totpStep1 and loginTotpStep1 messages in Lithuanian (messages_lt) local - WarpGrep: How is dark-light-choose SCSS function defined and what do its arguments mean - WarpGrep: RecoveryAuthnCodesCredentialModel createFromValues method signature and what par - WarpGrep: QueryTypeSQL handling in expression query reader - WarpGrep: run_with_initialized_sentry function definition and how it wraps target with arg - WarpGrep: totpStep1 message in messages_zh_CN account properties - check if Traditional or - WarpGrep: Who calls newRBACClient and how is the returned client used - WarpGrep: Who calls NewResourceServer and how is Init called on the resource server - WarpGrep: UserCredentialManager getFederatedCredentialsStream implementation for federated - WarpGrep: custom-message-length styling in modal scss - WarpGrep API error (turn 3): 400 Client Error: Bad Request for url: https://api.morphllm.com/v1/chat/completions - WarpGrep: Find where _hydrateEmbedded is called and how _lookupSubType works for arrays an - WarpGrep: SpansBuffer assigned_shards property and how shards are determined from partitio - WarpGrep: SelectedCalendar model in Prisma schema, does it have @updatedAt field - WarpGrep: scheduleEmailReminders handler that also uses WorkflowReminder and retryCount - WarpGrep: OptimizedCursorPaginator usage or enable_advanced_features in paginate method - WarpGrep: PRCommentWorkflow class hierarchy and _truncate_title method - WarpGrep: checkPermission method in authz rbac service - what does it do and how does it d - WarpGrep: RecoveryAuthnCodesFormAuthenticator authenticate method and how it checks if use - WarpGrep: set_locale method in application_controller and how it's used as before_action o - WarpGrep: CredentialInputUpdater getCredentials method interface definition - WarpGrep: How does nodestore get_multi return keys - are they node_ids or event_ids? Does - WarpGrep: asymmetric cache behavior: denial cache checked but permissions cache invalidati - WarpGrep: How does self.paginate pass extra kwargs like enable_advanced_features to pagina - WarpGrep: ExpressionQueryReader struct definition with features field - WarpGrep: Find all callers of isAccessTokenId in test files - WarpGrep: FlexBox in traceWaterfall with height 100%, does Flex direction=column flex=1 in - WarpGrep: getOrCreateIndex function in search.go how is it called and what concurrency pro - WarpGrep: Where is set_locale defined in application_controller and what calls it - WarpGrep: Find all implementations of OAuth2GrantTypeFactory interface and their getShortc - WarpGrep: UserCredentialModel constructor with three parameters id type challengeResponse - WarpGrep: overflowEllipsis theme mixin and how FlexCenter was used in eventAttachments for - WarpGrep: availabilityCheckProps definition and what properties it contains in slots.ts - WarpGrep: Prisma updateMany with empty data object behavior, does @updatedAt get set - WarpGrep: RecoveryAuthnCodesCredentialProvider isValid method how it validates a recovery - WarpGrep: connectedCalendars type definition, what shape does each connected calendar have - WarpGrep: Flex component definition and what CSS properties does flex prop map to - WarpGrep: NewResourceServer return type change - who calls it and what type does it expect - WarpGrep: getConnectedDestinationCalendarsAndEnsureDefaultsInDb return type and connectedC - WarpGrep: SpansBuffer slice_id usage in Redis key generation and methods - WarpGrep: WhatsApp reminder retryCount increment on failure in scheduleWhatsappReminders - WarpGrep: RpcUserOrganizationContext member attribute can be None - WarpGrep: Attributes filtering in span EAP sections, is the filter only applied when searc - WarpGrep: Find where Context copy constructor was used before it was removed - WarpGrep: checkPermission function signature and its callers in authz rbac service - WarpGrep: validateCheckRequest method in authz rbac service, what does checkReq.UserUID co - WarpGrep: UserCredentialManager updateCredential implementation for federated users and lo - WarpGrep: TableWidgetVisualization component props and how columns and tableData are used - WarpGrep: getIdentityPermissions method signature and parameters including actionSets in a - WarpGrep: Who calls BasePaginator get_result or inherits from BasePaginator - Loop done round=11 (tools: warpgrep_codebase_search=4, read_file=5, grep=5, glob=1, bash=3) - Loop done round=7 (tools: warpgrep_codebase_search=2, read_file=3, glob=1, grep=2, bash=3) - WarpGrep: GetUserIdentifiers method in authz rbac - does it return the same UID that was p - Review complete: 1 issues - [9/15] grafana PR#94942 1 raw -> 1 kept - [9/15] grafana PR#94942 [0.97] logic_error: The `enableSqlExpressions` function always returns `false` regardless of the feature flag - WarpGrep: WidgetCardChartProps interface, how is organization passed to WidgetCardChart - Loop done round=14 (tools: read_file=16, warpgrep_codebase_search=4, grep=3) - Review complete: 4 issues - [12/15] discourse-graphite PR#7 4 raw -> 4 kept - [12/15] discourse-graphite PR#7 [0.95] incorrect_value: Swapped light/dark values for `.custom-message-length` color. The light-theme value was ch - [12/15] discourse-graphite PR#7 [0.95] incorrect_value: Swapped light/dark values for embedded post reply link color. The original light-theme val - [12/15] discourse-graphite PR#7 [0.85] incorrect_value: Light-theme value for h3 in the topic map area changed from original `$lightness: 20%` to - [12/15] discourse-graphite PR#7 [0.70] incorrect_value: Light-theme value for `.name` changed from original `$lightness: 30%` (dark, readable) to - Review complete: 2 issues - [2/15] keycloak PR#37634 2 raw -> 2 kept - [2/15] keycloak PR#37634 [0.97] logic_error: The 4th Objects.requireNonNull checks `grantType` instead of `rawTokenId`. The error messa - [2/15] keycloak PR#37634 [0.97] test_correctness: Two bugs on this line: (1) Wrong substring indices — `substring(3, 5)` extracts the wrong - WarpGrep: get_cursor_from_request parsing cursor string to create Cursor object - Loop done round=10 (tools: warpgrep_codebase_search=3, read_file=4, glob=2, grep=7) - Review complete: 2 issues - [14/15] cal.com PR#8330 2 raw -> 2 kept - [14/15] cal.com PR#8330 [0.97] api_misuse: Reference comparison `===` on dayjs objects always evaluates to `false`. `dayjs(date.start - [14/15] cal.com PR#8330 [0.97] incorrect_value: `end` is computed from `slotStartTime` instead of `slotEndTime`: `const end = slotStartTim - WarpGrep: Discourse initializer ordering mechanism - how does "order: after" comment work - WarpGrep: How does paginate pass paginator_kwargs to paginator_cls constructor, what kwarg - WarpGrep: RecoveryAuthnCodesAction processAction method full implementation showing how cr - WarpGrep API error (turn 2): 400 Client Error: Bad Request for url: https://api.morphllm.com/v1/chat/completions - WarpGrep: How does Prisma updateMany handle @updatedAt when data is empty object - WarpGrep: nodestore get_multi dict ordering guarantee, does it return keys in input order - Loop done round=9 (tools: warpgrep_codebase_search=4, grep=4, read_file=7) - Review complete: 1 issues - [15/15] cal.com PR#14943 1 raw -> 1 kept - [15/15] cal.com PR#14943 [0.90] logic_error: The second OR branch `{ retryCount: { gt: 1 } }` has no `method` filter, so every time the - WarpGrep: How does I18n::Backend::Fallbacks#translate use I18n.fallbacks - what methods do - WarpGrep: EnterRecoveryAuthnCodePage getRecoveryAuthnCodeToEnterNumber implementation - WarpGrep: Where is kafka_slice_id set or passed to ProcessSpansStrategyFactory - Loop done round=12 (tools: warpgrep_codebase_search=3, grep=14, read_file=3, glob=2) - Loop done round=17 (tools: warpgrep_codebase_search=3, grep=7, read_file=15, glob=3) - Review complete: 2 issues - [1/15] keycloak PR#37429 2 raw -> 2 kept - [1/15] keycloak PR#37429 [0.97] localization: The `loginTotpStep1` value contains Italian text ("Installa una delle seguenti applicazion - [1/15] keycloak PR#37429 [0.95] localization: The `totpStep1` value `在您的手機上安裝以下應用程式之一：` uses Traditional Chinese characters (手機, 安裝, 應用程 - Review complete: 4 issues - [11/15] discourse-graphite PR#10 4 raw -> 4 kept - [11/15] discourse-graphite PR#10 [0.97] null_reference: When no `embed_category` site setting exists (e.g. fresh installs), `execute(...)[0]` retu - [11/15] discourse-graphite PR#10 [0.95] logic_error: File contents are swapped with `embeddable_host_fabricator.rb`. The `category_fabricator.r - [11/15] discourse-graphite PR#10 [0.82] logic_error: `_lookupSubType` returns `undefined` when a referenced ID isn't found in sideloaded data. - [11/15] discourse-graphite PR#10 [0.72] security: SQL injection via string interpolation: variable `h` (from splitting the old `embeddable_h - WarpGrep: CalendarCacheRepository import path, how is it imported across the project - WarpGrep: How is disableConnectionModification used in the second ConnectedCalendarList re - WarpGrep: How click_options and strategy_factory keyword arguments are passed to ProcessSp - WarpGrep: How did set_locale work in application_controller before it became with_resolved - Max tool rounds reached (tools: warpgrep_codebase_search=11, read_file=16, grep=12, glob=1) - WarpGrep: SelectedCalendarRepository.updateManyByCredentialId callers, where is it called - Review complete: 1 issues - [5/15] sentry-greptile PR#5 1 raw -> 1 kept - [5/15] sentry-greptile PR#5 [0.80] type_error: `justify="flex-start"` is not a valid value according to the component's TypeScript union - WarpGrep: What type does SiteSetting.default_locale return - string or symbol? - Loop done round=16 (tools: warpgrep_codebase_search=8, grep=5, read_file=13, bash=3) - Review complete: 2 issues - [6/15] sentry-greptile PR#1 2 raw -> 2 kept - [6/15] sentry-greptile PR#1 [0.97] type_error: OptimizedCursorPaginator.get_item_key crashes with TypeError on datetime fields. It calls - [6/15] sentry-greptile PR#1 [0.95] api_misuse: OptimizedCursorPaginator.get_result uses negative indexing on a Django QuerySet (queryset[ - WarpGrep: How is CalendarCacheRepository instantiated, is there a factory or dependency in - Max tool rounds reached (tools: warpgrep_codebase_search=11, grep=11, read_file=11, bash=7, glob=1) - Review complete: 0 issues - [13/15] cal.com PR#22532 0 raw -> 0 kept - Loop done round=14 (tools: warpgrep_codebase_search=7, read_file=12, grep=6) - Review complete: 2 issues - [4/15] sentry PR#93824 2 raw -> 2 kept - [4/15] sentry PR#93824 [0.95] logic_error: Missing `slice_id` in per-shard SpansBuffer creation. The diff removes `self.buffer = buff - [4/15] sentry PR#93824 [0.75] test_correctness: Test `time.sleep(0.1)` is a no-op due to monkeypatch. The test monkeypatches `time.sleep` - WarpGrep: mapper Scope method for dashboard resource in authz rbac, how does t.Scope work - WarpGrep: Where is the tracer field on searchSupport or searchServer defined - WarpGrep: generatedRecoveryAuthnCodes hidden field form template recovery codes generation - Loop done round=24 (tools: warpgrep_codebase_search=8, read_file=11, grep=16, bash=3, glob=2) - Review complete: 1 issues - [10/15] discourse-graphite PR#9 1 raw -> 1 kept - [10/15] discourse-graphite PR#9 [0.50] type_error: The `ensure_loaded!` method does not normalize the `locale` parameter to a symbol before c - Loop done round=16 (tools: warpgrep_codebase_search=9, read_file=12, grep=3) - Review complete: 1 issues - [8/15] grafana PR#103633 1 raw -> 1 kept - [8/15] grafana PR#103633 [0.92] test_correctness: The test "Should deny on explicit cache deny entry" sets the permission cache value to `fa - Loop done round=25 (tools: warpgrep_codebase_search=5, read_file=21, grep=10, bash=1) - Review complete: 1 issues - [7/15] grafana PR#97529 1 raw -> 1 kept - [7/15] grafana PR#97529 [0.55] logic_error: When `Init(ctx)` fails after `s.search.init(ctx)` has already succeeded and started backgr - Loop done round=25 (tools: warpgrep_codebase_search=12, read_file=15, grep=11) - Review complete: 1 issues - [3/15] keycloak PR#38446 1 raw -> 1 kept - [3/15] keycloak PR#38446 [0.82] logic_error: BackwardsCompatibilityUserStorage.getCredentials() creates a RecoveryAuthnCodesCredentialM -Wrote candidates to /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/candidates.json - -============================================================ -DONE: 15 reviewed, 25 raw -> 25 filtered -Avg/PR: 1.7, Time: 861s -Candidates: /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/candidates.json -Benchmark data updated: /Users/tejas/personal/applymodel/zmisc/examples/code-review-benchmark/offline/results/benchmark_data.json diff --git a/pr_review_agent/output/iter6_eval.log b/pr_review_agent/output/iter6_eval.log deleted file mode 100644 index 5ffa0fd..0000000 --- a/pr_review_agent/output/iter6_eval.log +++ /dev/null @@ -1,76 +0,0 @@ - -keycloak PR#37429: 4 golden, 2 candidates - TP: [Medium] The translation is in Italian instead of Lithuanian. This should be translated t... - TP: [Medium] The totpStep1 value uses Traditional Chinese terms in the Simplified Chinese fil... - FN: [Low] The anchor sanitization logic has a potential issue where it consumes English ma... - FN: [Low] The method name 'santizeAnchors' should be 'sanitizeAnchors' (missing 'i').... - -keycloak PR#37634: 4 golden, 2 candidates - TP: [Critical] Wrong parameter in null check (grantType vs. rawTokenId)... - TP: [High] In isAccessTokenId, the substring for the grant shortcut and the equality check ... - FN: [Low] Javadoc mentions "usually like 3-letters shortcut" but some implementations use ... - FN: [Low] Catching generic RuntimeException is too broad. The implementation throws Illeg... - -keycloak PR#38446: 2 golden, 2 candidates - FN: [Medium] Unsafe raw List deserialization without type safety. Calling Optional.get() dire... - FN: [Low] After creating the RecoveryAuthnCodesCredentialModel, consider setting its id fr... - -sentry PR#93824: 5 golden, 2 candidates - TP: [Medium] Inconsistent metric tagging with 'shard' and 'shards'... - TP: [Low] Fixed sleep in tests can be flaky; wait on condition instead... - TP: [Medium] Sleep in test_consumer.py won’t actually wait because time.sleep was monkeypatch... - FN: [High] Because flusher processes are created via multiprocessing.get_context('spawn').P... - FN: [Medium] Breaking out of the loop when the deadline has elapsed can skip terminating rema... - -sentry-greptile PR#1: 4 golden, 2 candidates - TP: [High] Django querysets do not support negative slicing... - TP: [High] When requests are authenticated with API keys or org auth tokens (which have use... - FN: [Low] Importing non-existent OptimizedCursorPaginator... - FN: [High] get_item_key assumes a numeric key, but the paginator is used with order_by=-dat... - -grafana PR#103633: 2 golden, 1 candidates - TP: [Low] The test comment says the cached permissions 'allow access', but the map stores ... - FN: [High] The Check operation exhibits asymmetric cache trust logic: cached permission gra... - -grafana PR#94942: 2 golden, 1 candidates - TP: [Critical] The enableSqlExpressions function has flawed logic that always returns false, ef... - FN: [High] Several methods such as NewInMemoryDB().RunCommands and db.QueryFramesInto retur... - -discourse-graphite PR#9: 2 golden, 3 candidates - TP: [Low] Consider normalizing the input locale (e.g., to a symbol) when checking/loading ... - FN: [Low] Thread-safety issue with lazy @loaded_locales... - -discourse-graphite PR#7: 3 golden, 3 candidates - TP: [Low] In .topic-meta-data h5 a, the original code had color: scale-color($primary, $li... - TP: [Low] This change for desktop/user.css changes $primary from 30% to 50% for the light ... - TP: [Low] In topic-post.css the original code used $lightness: 70% but the replacement use... - -cal.com PR#8330: 2 golden, 2 candidates - TP: [Medium] Incorrect end time calculation using slotStartTime instead of slotEndTime... - TP: [Medium] Using === for dayjs object comparison will always return false as it compares ob... - -cal.com PR#14943: 2 golden, 1 candidates - TP: [High] The deletion logic in scheduleSMSReminders.ts incorrectly deletes non-SMS workfl... - FN: [High] Using retryCount: reminder.retryCount + 1 reads a possibly stale value and can l... - -============================================================ -OVERALL RESULTS -============================================================ - True Positives: 18/43 - False Positives: 5 - False Negatives: 25 - Total Candidates: 21 - Precision: 85.7% - Recall: 41.9% - F1: 56.3% - -Per-repo breakdown: -Repo Prec Recall F1 TP FP FN -------------------------------------------------------- -cal.com 100.0% 75.0% 85.7% 3 0 1 -discourse 66.7% 80.0% 72.7% 4 3 1 -grafana 100.0% 50.0% 66.7% 2 0 2 -keycloak 66.7% 40.0% 50.0% 4 2 6 -sentry 125.0% 55.6% 76.9% 5 0 4 - -Results saved to /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/evaluation_results.json diff --git a/pr_review_agent/output/iter6_mini15.log b/pr_review_agent/output/iter6_mini15.log deleted file mode 100644 index 8786296..0000000 --- a/pr_review_agent/output/iter6_mini15.log +++ /dev/null @@ -1,204 +0,0 @@ -============================================================ -PR Review Agent - Benchmark Pipeline -============================================================ -Loaded 50 PRs -WarpGrep: ENABLED -Mini eval: 15 PRs (3 per repo) -Reviewing 15 PRs - -Running 15 PRs with parallelism=15 - -[1/15] keycloak PR#37429 (4 golden) -[2/15] keycloak PR#37634 (4 golden) -[3/15] keycloak PR#38446 (2 golden) -[4/15] sentry PR#93824 (5 golden) -[5/15] sentry-greptile PR#5 (3 golden) -[6/15] sentry-greptile PR#1 (4 golden) -[7/15] grafana PR#97529 (2 golden) -[9/15] grafana PR#94942 (2 golden) -[8/15] grafana PR#103633 (2 golden) -[10/15] discourse-graphite PR#9 (2 golden) -[11/15] discourse-graphite PR#10 (4 golden) -[12/15] discourse-graphite PR#7 (3 golden) -[13/15] cal.com PR#22532 (2 golden) -[14/15] cal.com PR#8330 (2 golden) -[15/15] cal.com PR#14943 (2 golden) - [6/15] sentry-greptile PR#1 3 files, 128 added lines - [10/15] discourse-graphite PR#9 6 files, 31 added lines - [7/15] grafana PR#97529 5 files, 25 added lines - [3/15] keycloak PR#38446 8 files, 256 added lines - [11/15] discourse-graphite PR#10 36 files, 449 added lines - [4/15] sentry PR#93824 6 files, 199 added lines - [1/15] keycloak PR#37429 48 files, 343 added lines - [15/15] cal.com PR#14943 3 files, 38 added lines - [14/15] cal.com PR#8330 4 files, 111 added lines - [13/15] cal.com PR#22532 17 files, 379 added lines - [2/15] keycloak PR#37634 28 files, 722 added lines - [8/15] grafana PR#103633 4 files, 240 added lines - [9/15] grafana PR#94942 4 files, 41 added lines - [12/15] discourse-graphite PR#7 32 files, 115 added lines - [5/15] sentry-greptile PR#5 105 files, 2312 added lines - WarpGrep: SpansBuffer constructor and assigned_shards property - WarpGrep: How does scheduleSMSReminders work and what is the full flow of scheduling and d - WarpGrep: enableSqlExpressions function and how it's used in reader.go - WarpGrep: Who calls the copy constructor Context(Context context) that was removed from OA - WarpGrep: How is checkPermission used in the RBAC service and what does it return - WarpGrep: checkIfIsAvailable function callers and how dateOverrides and workingHours are p - WarpGrep: Django QuerySet negative indexing or slicing behavior - WarpGrep: How is I18n.ensure_loaded! defined and called throughout the codebase - WarpGrep: getFederatedCredentialsStream method definition and implementations in credentia - WarpGrep: Find all usages of the category fabricator (Fabricate(:category), Fabricate(:dif - WarpGrep: How is get_merged_pr_single_issue_template defined and called in PRCommentWorkfl - WarpGrep: How does santizeAnchors work in VerifyMessageProperties and what is the logic fo - WarpGrep: How is BuildIndex called and what concurrency model protects the bleve cache map - WarpGrep: updateManyByCredentialId in SelectedCalendarRepository and how Prisma updateMany - WarpGrep: FlagSqlExpressions feature flag definition and usage - WarpGrep: createRecoveryCodesCredential callers and how updateCredential is used for recov - WarpGrep: Find callers of _hydrateEmbedded and _lookupSubType in the store model - WarpGrep: How dayjs objects are compared for equality in the scheduling system - WarpGrep: WorkflowReminder deletion logic and what methods are used (SMS, EMAIL, WHATSAPP) - WarpGrep: How does the permDenialCache interact with cache invalidation or TTL expiry in R - WarpGrep: fetch_error_details using zip with dict.values() in replay summarize breadcrumbs - WarpGrep: SpanFlusher.main function signature and how it's called with arguments - WarpGrep: Find where the test fixture files for VerifyMessageProperties are loaded, specif - WarpGrep: How connectedCalendars return type is used by callers, especially cacheUpdatedAt - WarpGrep: How does self.paginate pass extra keyword arguments to paginator_cls constructor - WarpGrep: How is set_locale implemented in application_controller and what does it return - WarpGrep: Who calls NewResourceServer and how do callers handle its return value - WarpGrep: How does the Attributes component filter hidden attributes when searchQuery is e - WarpGrep: Find all implementations of OAuth2GrantTypeFactory interface getShortcut method - WarpGrep: RecoveryAuthnCodesCredentialModel createFromValues and how recovery codes are va - WarpGrep: run_with_initialized_sentry function signature and how it passes arguments to ta - WarpGrep: How is @loaded_locales used in translate_accelerator.rb - WarpGrep: How does verifySafeHtml resolve the English file path from a community resource - WarpGrep: SelectedCalendar schema definition in Prisma, does it have updatedAt field - WarpGrep: checkPermission function signature in RBAC service - how many parameters does it - WarpGrep: How is server.Init called and what does the once.Do pattern look like - WarpGrep: How does get_paginator pass kwargs to paginator constructor, specifically enable - WarpGrep: ProcessSpansStrategyFactory create_with_partitions how partitions become buffer - WarpGrep: PRCommentWorkflow._truncate_title static method definition and class hierarchy - WarpGrep: RecoveryAuthnCodesFormAuthenticator authenticate method and how it validates rec - WarpGrep: Lithuanian translation for totpStep1 in login messages, what language should it - WarpGrep: Prisma updateMany with empty data object behavior, does @updatedAt trigger with - WarpGrep: definition of dark-light-choose SCSS mixin or function - WarpGrep: Where is set_locale called in application_controller and what is it used as (bef - WarpGrep: nodestore.backend.get_multi return value order guarantees dict ordering - WarpGrep: connectedCalendar.cacheUpdatedAt usage in components, how is the connectedCalend - WarpGrep: _import_and_run function that unpickles main_fn and args, then calls with additi - WarpGrep API error (turn 2): 400 Client Error: Bad Request for url: https://api.morphllm.com/v1/chat/completions - WarpGrep: FlexBox styled component used in traceWaterfall with height 100% and flex direct - WarpGrep: getSchedule function in slots that calls checkIfIsAvailable with availabilityChe - WarpGrep: getTimeSlots function and how organizerTimeZone is used in slot generation with - WarpGrep: UserCredentialManager getFederatedCredentialsStream implementation and how it de - WarpGrep: DisconnectIntegration component usage and how credentialId is passed to it, espe - WarpGrep: nodestore get_multi return dict key ordering vs input id_list ordering - WarpGrep: ExpressionQueryReader ReadQuery function handling QueryTypeSQL case - WarpGrep: Who calls BuildIndex on SearchBackend and what concurrency protection do they ha - WarpGrep: BackwardsCompatibilityUserStorage getCredentials method and how it returns crede - WarpGrep: TableWidgetVisualization usage in chart.tsx with empty columns and data - WarpGrep connection error, retrying in 3s (attempt 1/3): SSLError - WarpGrep: Find all callers of isAccessTokenId matcher in test files - Loop done round=6 (tools: warpgrep_codebase_search=1, grep=1, read_file=3, glob=2, bash=3) - WarpGrep: viewer.credentials.delete mutation handler to check what parameters it expects - WarpGrep: fetch_error_details function in replay summarize breadcrumbs with zip error_ids - Review complete: 3 issues - [12/15] discourse-graphite PR#7 3 raw -> 3 kept - [12/15] discourse-graphite PR#7 [0.95] incorrect_value: Swapped arguments in dark-light-choose for embedded-posts link color. Original lightness w - [12/15] discourse-graphite PR#7 [0.85] incorrect_value: Light-theme lightness changed from original 30% to 50% for .name color, making it the same - [12/15] discourse-graphite PR#7 [0.85] incorrect_value: Light-theme lightness changed from original 20% to 50% for h3 color, making headings signi - WarpGrep: How is slotEndTime calculated in checkIfIsAvailable or similar functions for ava - WarpGrep: Django QuerySet __getitem__ assertion error on negative indexing or slicing - WarpGrep: UserCredentialModel buildFromBackupAuthnCode definition and what challengeRespon - WarpGrep: WithCacheClientOption cache interface definition for authzlib client - Loop done round=12 (tools: warpgrep_codebase_search=3, read_file=6, grep=7, glob=2) - Review complete: 1 issues - [9/15] grafana PR#94942 1 raw -> 1 kept - [9/15] grafana PR#94942 [0.97] logic_error: The `enableSqlExpressions` function always returns `false` regardless of the feature flag - WarpGrep: FlexCenter with overflowEllipsis in eventAttachments for attachment names - WarpGrep: searchSupport init function that builds indexes during startup with worker threa - Loop done round=11 (tools: warpgrep_codebase_search=5, read_file=5, glob=2, grep=4) - Review complete: 2 issues - [14/15] cal.com PR#8330 2 raw -> 2 kept - [14/15] cal.com PR#8330 [0.97] incorrect_value: Copy-paste error: `end` is computed from `slotStartTime` instead of `slotEndTime`, making - [14/15] cal.com PR#8330 [0.95] api_misuse: `===` comparison between two dayjs objects compares references, not values, so it always r - WarpGrep: Flex component definition with flex prop and CSS mapping - WarpGrep: How is the i18n.rb initializer ordered relative to freedom_patches and translate - Loop done round=7 (tools: warpgrep_codebase_search=2, read_file=5, grep=5) - Loop done round=16 (tools: warpgrep_codebase_search=4, read_file=5, grep=15, glob=1, bash=3) - Review complete: 1 issues - [15/15] cal.com PR#14943 1 raw -> 1 kept - [15/15] cal.com PR#14943 [0.92] logic_error: The `deleteMany` query's second OR branch `{ retryCount: { gt: 1 } }` lacks a `method: Wor - Rate limited, retrying in 10s (attempt 1/5) - Review complete: 2 issues - [1/15] keycloak PR#37429 2 raw -> 2 kept - [1/15] keycloak PR#37429 [0.97] localization: Lithuanian (lt) translation file contains Italian text instead of Lithuanian for totpStep1 - [1/15] keycloak PR#37429 [0.95] localization: Simplified Chinese (zh_CN) file contains Traditional Chinese characters for totpStep1. The - WarpGrep: How does the store pluralize subType in _hydrateEmbedded for embedded relationsh - Loop done round=13 (tools: warpgrep_codebase_search=3, grep=7, read_file=13) - WarpGrep: RecoveryAuthnCodesCredentialProvider class definition and what interfaces it imp - WarpGrep: Where is GOOGLE_CALENDAR_TYPE constant defined, what value does it have - WarpGrep: RecoveryAuthnCodeInputLoginBean callers and how it's constructed in the forms re - Rate limited, retrying in 20s (attempt 2/5) - Review complete: 2 issues - [2/15] keycloak PR#37634 2 raw -> 2 kept - [2/15] keycloak PR#37634 [0.97] logic_error: Copy-paste error: the fourth Objects.requireNonNull checks `grantType` a second time inste - [2/15] keycloak PR#37634 [0.97] test_correctness: Two bugs in one line: (1) Wrong substring indices — grant type shortcut is at positions 4- - WarpGrep: _lookupSubType implementation in the store, what does it do with the subType par - WarpGrep: SpansBuffer process_spans how it hashes or assigns spans to shards - Rate limited, retrying in 10s (attempt 1/5) - Rate limited, retrying in 10s (attempt 1/5) - WarpGrep: EnterRecoveryAuthnCodePage getRecoveryAuthnCodeToEnterNumber implementation - WarpGrep: BackwardsCompatibilityUserStorage CredentialInputUpdater interface implementatio - WarpGrep: chartActionDropdown referrer parameter added to function call - Max tool rounds reached (tools: warpgrep_codebase_search=12, read_file=11, grep=12, glob=1) - WarpGrep: connectedCalendars return type, what fields does each connected calendar have, d - Review complete: 0 issues - [5/15] sentry-greptile PR#5 0 raw -> 0 kept - Loop done round=16 (tools: warpgrep_codebase_search=4, grep=9, read_file=14, glob=1, bash=2) - WarpGrep: Prisma updateMany with empty data object, does @updatedAt get set - Review complete: 2 issues - [6/15] sentry-greptile PR#1 2 raw -> 2 kept - [6/15] sentry-greptile PR#1 [0.97] api_misuse: Django QuerySets do not support negative indexing. The OptimizedCursorPaginator attempts t - [6/15] sentry-greptile PR#1 [0.75] null_reference: AttributeError when organization_context.member is None. The member field can be None when - WarpGrep: How does the Discourse store.find handle singleton resources when called with ju - WarpGrep: GetUserIdentifiers function definition and what UID it returns vs input userID - Loop done round=12 (tools: warpgrep_codebase_search=6, read_file=7, grep=6, bash=1) - Review complete: 2 issues - [4/15] sentry PR#93824 2 raw -> 2 kept - [4/15] sentry PR#93824 [0.92] incorrect_value: The metric tag key is `"shards"` (plural) but all other metrics in the same function use ` - [4/15] sentry PR#93824 [0.85] test_correctness: `time.sleep(0.1)` is a no-op because `time.sleep` was monkeypatched to `lambda _: None` ea - WarpGrep: AppListCard component how does it render the actions prop - WarpGrep: RecoveryAuthnCodesSecretData class definition and how codes are stored and retri - Max tool rounds reached (tools: warpgrep_codebase_search=5, grep=10, bash=10, read_file=14, glob=2, list_directory=1) - Review complete: 0 issues - [11/15] discourse-graphite PR#10 0 raw -> 0 kept - Max tool rounds reached (tools: warpgrep_codebase_search=11, grep=7, read_file=12, glob=7, bash=9) - Review complete: 0 issues - [13/15] cal.com PR#22532 0 raw -> 0 kept - Loop done round=14 (tools: warpgrep_codebase_search=5, read_file=14, grep=1) - Review complete: 1 issues - [8/15] grafana PR#103633 1 raw -> 1 kept - [8/15] grafana PR#103633 [0.92] test_correctness: In the "Should deny on explicit cache deny entry" test, the permCache is set with `"dashbo - Loop done round=20 (tools: warpgrep_codebase_search=5, read_file=15, grep=10) - Review complete: 0 issues - [7/15] grafana PR#97529 0 raw -> 0 kept - WarpGrep: Where is set_locale used as a before_action or around_action filter in applicati - WarpGrep: SetupRecoveryAuthnCodesPage class definition and checkLogoutSessions uncheckLogo - Loop done round=23 (tools: warpgrep_codebase_search=6, read_file=11, grep=10, bash=8, glob=1) - Review complete: 3 issues - [10/15] discourse-graphite PR#9 3 raw -> 3 kept - [10/15] discourse-graphite PR#9 [0.60] localization: FallbackLocaleList#[] does not short-circuit for English locale. When user locale is :en a - [10/15] discourse-graphite PR#9 [0.50] type_error: FallbackLocaleList#[] does not normalize the locale parameter to a symbol. If a string loc - [10/15] discourse-graphite PR#9 [0.50] type_error: ensure_loaded! does not normalize the locale parameter to a symbol. Since load_locale inte - Loop done round=20 (tools: warpgrep_codebase_search=13, read_file=13, grep=7) - Review complete: 2 issues - [3/15] keycloak PR#38446 2 raw -> 2 kept - [3/15] keycloak PR#38446 [0.82] logic_error: The logout-sessions checkbox logic is inverted in setupRecoveryKeysForUserWithRequiredActi - [3/15] keycloak PR#38446 [0.55] security: In createRecoveryCodesCredential, when userStorageCreated is true (federated user), the me -Wrote candidates to /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/candidates.json - -============================================================ -DONE: 15 reviewed, 21 raw -> 21 filtered -Avg/PR: 1.4, Time: 845s -Candidates: /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/candidates.json -Benchmark data updated: /Users/tejas/personal/applymodel/zmisc/examples/code-review-benchmark/offline/results/benchmark_data.json diff --git a/pr_review_agent/output/iter7_eval.log b/pr_review_agent/output/iter7_eval.log deleted file mode 100644 index db51269..0000000 --- a/pr_review_agent/output/iter7_eval.log +++ /dev/null @@ -1,87 +0,0 @@ - -keycloak PR#37429: 4 golden, 2 candidates - TP: [Medium] The translation is in Italian instead of Lithuanian. This should be translated t... - TP: [Medium] The totpStep1 value uses Traditional Chinese terms in the Simplified Chinese fil... - FN: [Low] The anchor sanitization logic has a potential issue where it consumes English ma... - FN: [Low] The method name 'santizeAnchors' should be 'sanitizeAnchors' (missing 'i').... - -keycloak PR#37634: 4 golden, 3 candidates - TP: [Critical] Wrong parameter in null check (grantType vs. rawTokenId)... - TP: [High] In isAccessTokenId, the substring for the grant shortcut and the equality check ... - FN: [Low] Javadoc mentions "usually like 3-letters shortcut" but some implementations use ... - FN: [Low] Catching generic RuntimeException is too broad. The implementation throws Illeg... - -keycloak PR#38446: 2 golden, 2 candidates - TP: [Low] After creating the RecoveryAuthnCodesCredentialModel, consider setting its id fr... - FN: [Medium] Unsafe raw List deserialization without type safety. Calling Optional.get() dire... - -sentry PR#93824: 5 golden, 2 candidates - TP: [Medium] Inconsistent metric tagging with 'shard' and 'shards'... - TP: [Low] Fixed sleep in tests can be flaky; wait on condition instead... - TP: [Medium] Sleep in test_consumer.py won’t actually wait because time.sleep was monkeypatch... - FN: [High] Because flusher processes are created via multiprocessing.get_context('spawn').P... - FN: [Medium] Breaking out of the loop when the deadline has elapsed can skip terminating rema... - -sentry-greptile PR#5: 3 golden, 1 candidates - TP: [Medium] Detector validator uses wrong key when updating type... - TP: [Low] Using zip(error_ids, events.values()) assumes the get_multi result preserves the... - FN: [Medium] Breaking changes in error response format... - -sentry-greptile PR#1: 4 golden, 2 candidates - TP: [High] Django querysets do not support negative slicing... - TP: [High] When requests are authenticated with API keys or org auth tokens (which have use... - FN: [Low] Importing non-existent OptimizedCursorPaginator... - FN: [High] get_item_key assumes a numeric key, but the paginator is used with order_by=-dat... - -grafana PR#103633: 2 golden, 1 candidates - TP: [Low] The test comment says the cached permissions 'allow access', but the map stores ... - FN: [High] The Check operation exhibits asymmetric cache trust logic: cached permission gra... - -grafana PR#94942: 2 golden, 1 candidates - TP: [Critical] The enableSqlExpressions function has flawed logic that always returns false, ef... - FN: [High] Several methods such as NewInMemoryDB().RunCommands and db.QueryFramesInto retur... - -discourse-graphite PR#9: 2 golden, 1 candidates - TP: [Low] Consider normalizing the input locale (e.g., to a symbol) when checking/loading ... - FN: [Low] Thread-safety issue with lazy @loaded_locales... - -discourse-graphite PR#10: 4 golden, 3 candidates - FN: [Critical] NoMethodError before_validation in EmbeddableHost... - FN: [Medium] The update and destroy methods in Admin::EmbeddableHostsController do not valida... - FN: [Medium] record_for_host compares lower(host) = ? but does not normalize the parameter’s ... - FN: [High] Because this migration inserts embeddable_hosts rows with raw SQL, any existing ... - -discourse-graphite PR#7: 3 golden, 1 candidates - TP: [Low] In topic-post.css the original code used $lightness: 70% but the replacement use... - FN: [Low] In .topic-meta-data h5 a, the original code had color: scale-color($primary, $li... - FN: [Low] This change for desktop/user.css changes $primary from 30% to 50% for the light ... - -cal.com PR#8330: 2 golden, 2 candidates - TP: [Medium] Incorrect end time calculation using slotStartTime instead of slotEndTime... - TP: [Medium] Using === for dayjs object comparison will always return false as it compares ob... - -cal.com PR#14943: 2 golden, 1 candidates - TP: [High] The deletion logic in scheduleSMSReminders.ts incorrectly deletes non-SMS workfl... - FN: [High] Using retryCount: reminder.retryCount + 1 reads a possibly stale value and can l... - -============================================================ -OVERALL RESULTS -============================================================ - True Positives: 19/43 - False Positives: 5 - False Negatives: 24 - Total Candidates: 22 - Precision: 86.4% - Recall: 44.2% - F1: 58.5% - -Per-repo breakdown: -Repo Prec Recall F1 TP FP FN -------------------------------------------------------- -cal.com 100.0% 75.0% 85.7% 3 0 1 -discourse 40.0% 22.2% 28.6% 2 3 7 -grafana 100.0% 50.0% 66.7% 2 0 2 -keycloak 71.4% 50.0% 58.8% 5 2 5 -sentry 140.0% 58.3% 82.4% 7 0 5 - -Results saved to /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/evaluation_results.json diff --git a/pr_review_agent/output/iter7_mini15.log b/pr_review_agent/output/iter7_mini15.log deleted file mode 100644 index 286f0f2..0000000 --- a/pr_review_agent/output/iter7_mini15.log +++ /dev/null @@ -1,206 +0,0 @@ -============================================================ -PR Review Agent - Benchmark Pipeline -============================================================ -Loaded 50 PRs -WarpGrep: ENABLED -Mini eval: 15 PRs (3 per repo) -Reviewing 15 PRs - -Running 15 PRs with parallelism=15 - -[1/15] keycloak PR#37429 (4 golden) -[2/15] keycloak PR#37634 (4 golden) -[3/15] keycloak PR#38446 (2 golden) -[4/15] sentry PR#93824 (5 golden) -[5/15] sentry-greptile PR#5 (3 golden) -[6/15] sentry-greptile PR#1 (4 golden) -[7/15] grafana PR#97529 (2 golden) -[8/15] grafana PR#103633 (2 golden) -[9/15] grafana PR#94942 (2 golden) -[10/15] discourse-graphite PR#9 (2 golden) -[11/15] discourse-graphite PR#10 (4 golden) -[12/15] discourse-graphite PR#7 (3 golden) -[13/15] cal.com PR#22532 (2 golden) -[14/15] cal.com PR#8330 (2 golden) -[15/15] cal.com PR#14943 (2 golden) - [14/15] cal.com PR#8330 4 files, 111 added lines - [15/15] cal.com PR#14943 3 files, 38 added lines - [10/15] discourse-graphite PR#9 6 files, 31 added lines - [9/15] grafana PR#94942 4 files, 41 added lines - [7/15] grafana PR#97529 5 files, 25 added lines - [4/15] sentry PR#93824 6 files, 199 added lines - [6/15] sentry-greptile PR#1 3 files, 128 added lines - [13/15] cal.com PR#22532 17 files, 379 added lines - [8/15] grafana PR#103633 4 files, 240 added lines - [12/15] discourse-graphite PR#7 32 files, 115 added lines - [2/15] keycloak PR#37634 28 files, 722 added lines - [3/15] keycloak PR#38446 8 files, 256 added lines - [11/15] discourse-graphite PR#10 36 files, 449 added lines - [5/15] sentry-greptile PR#5 105 files, 2312 added lines - [1/15] keycloak PR#37429 48 files, 343 added lines - WarpGrep: checkIfIsAvailable function callers and how dateOverrides and workingHours are p - WarpGrep: SpanFlusher main method signature and how it's called - WarpGrep: Django QuerySet negative indexing or negative slicing behavior - WarpGrep: scheduleSMSReminders handler logic for scheduling SMS workflow reminders - WarpGrep: updateManyByCredentialId in SelectedCalendarRepository and what Prisma updateMan - WarpGrep: getFederatedCredentialsStream method definition and implementations - WarpGrep: fetch_error_details function and how nodestore.backend.get_multi returns data, o - WarpGrep: enableSqlExpressions function and how it's used with feature flag - WarpGrep: How is bleveBackend.cache used concurrently? Find all reads and writes to the ca - WarpGrep: Fabricator for category in spec/fabricators - WarpGrep: How does santizeAnchors method work in VerifyMessageProperties and what callers - WarpGrep: Who calls the Context copy constructor (new Context(context)) in OAuth2GrantType - WarpGrep: How does set_locale work in application_controller and what is its return value - WarpGrep: SpansBuffer constructor and assigned_shards property - WarpGrep: How is permissionCacheUsage metric used in the Check and List methods of the RBA - WarpGrep: getSlots function and how activeOverrides timeZone property is used - WarpGrep: OptimizedCursorPaginator usage and callers - WarpGrep: How is connectedCalendars return type used by consumers, especially cacheUpdated - WarpGrep: WorkflowReminder deleteMany cleanup logic for past reminders - WarpGrep: How is basePath used with type parameter containing underscores in rest adapter - WarpGrep: RecoveryAuthnCodesCredentialModel.createFromValues method signature and paramete - WarpGrep: PRCommentWorkflow class definition and get_merged_pr_single_issue_template metho - WarpGrep: Who calls NewResourceServer and how is Init called on the server? - WarpGrep: Find where messages_lt.properties totpStep1 is used for Lithuanian locale login - WarpGrep: ProcessSpansStrategyFactory callers and how flusher_processes parameter is passe - WarpGrep: FlagSqlExpressions feature flag definition and usage - WarpGrep: BasePaginator get_result method and how self.paginate passes enable_advanced_fea - WarpGrep: How is I18n.ensure_loaded! defined and called - is it a method on I18n module di - WarpGrep: How does nodestore backend get_multi return results - is the order preserved or - WarpGrep: What callers use newRBACClient and how does ProvideAuthZClient use the rbac clie - WarpGrep API 520, retrying in 3s (attempt 1/3) - WarpGrep: scheduleEmailReminders deleteMany cleanup past reminders retryCount - WarpGrep: Definition and usage of dark-light-choose SCSS function/mixin - WarpGrep: connectedCalendars type definition and what properties it includes like credenti - WarpGrep: QueryTypeSQL handling in expression reader - WarpGrep: messages_zh_CN totpStep1 translation for Simplified Chinese account messages - WarpGrep: How self.paginate method in API base passes extra keyword arguments to paginator - WarpGrep: translate_accelerator.rb load_locale and @loaded_locales thread safety - WarpGrep: checkIfIsAvailable function definition with busy and eventLength parameters - WarpGrep: format_comment_subtitle method in PRCommentWorkflow or CommitContextIntegration - WarpGrep: How does searchSupport.build call BuildIndex and what concurrency protection exi - WarpGrep: run_with_initialized_sentry function definition and how it passes arguments - WarpGrep: RecoveryAuthnCodesFormAuthenticator action method that validates user input reco - WarpGrep: checkPermission function in the RBAC service, what does it check and return - WarpGrep: Attributes component filtering when searchQuery is empty - hidden attributes not - WarpGrep: CredentialInputUpdater getCredentials method implementations for user storage pr - WarpGrep: SelectedCalendar model in prisma schema, updatedAt field definition - WarpGrep: Find all implementations of OAuth2GrantTypeFactory getShortcut method - WarpGrep: _import_and_run function definition that unpickles and runs main_fn with additio - WarpGrep: set_locale method definition in application_controller with before_action or aro - WarpGrep: impersonateTitleHtml in messages_sk.properties and what HTML tags it should cont - WarpGrep: overflowEllipsis theme mixin in FlexCenter component that was removed in eventAt - WarpGrep: How cursor offset is parsed from request, can cursor offset be negative - WarpGrep: TableWidgetVisualization usage in chart.tsx with empty columns and empty data - WarpGrep: How does getOrCreateIndex in searchSupport call build and BuildIndex? What concu - WarpGrep: nodestore get_multi dictionary ordering - does it preserve insertion order or ke - WarpGrep: analytics record preprod_artifact.api.assemble and where request.user.id might b - WarpGrep: scheduleWhatsAppReminders handler deleteMany cleanup retryCount - WarpGrep: FlexBox or FlexContainer in traceWaterfall.tsx - height 100% flex-grow behavior - WarpGrep: RecoveryAuthnCodesCredentialProvider isValid method that validates backup authen - WarpGrep: How does _lookupSubType work in the store for hydrating embedded objects - WarpGrep: How does checkIfIsAvailable use slotEndTime and slotStartTime to check if a slot - WarpGrep: Prisma updateMany with empty data object behavior, does @updatedAt trigger - WarpGrep: UserCredentialModel buildFromBackupAuthnCode method definition - WarpGrep: Where is getShouldUseLightweightToken called and how is it accessed, instance vs - WarpGrep: HIDDEN_ATTRIBUTES filtering when searchQuery is empty in attributes.tsx - does f - WarpGrep: preprod analytics old-style Event class with attributes tuple vs new eventclass - WarpGrep: expandable_first_post method in Topic model and how it checks for embeddable hos - WarpGrep: CalendarCacheRepository constructor and how it's instantiated - Loop done round=10 (tools: warpgrep_codebase_search=4, read_file=2, glob=2, grep=5) - Review complete: 2 issues - [14/15] cal.com PR#8330 2 raw -> 2 kept - [14/15] cal.com PR#8330 [0.95] logic_error: Using `===` to compare two dayjs objects compares references, not values, so the condition - [14/15] cal.com PR#8330 [0.95] incorrect_value: Copy-paste error: `end` is computed from `slotStartTime` instead of `slotEndTime`. Both `s - Loop done round=10 (tools: warpgrep_codebase_search=3, read_file=9, grep=6, glob=1) - WarpGrep: config.i18n.fallbacks in environment configuration files (development, test, pro - Review complete: 1 issues - [9/15] grafana PR#94942 1 raw -> 1 kept - [9/15] grafana PR#94942 [0.95] logic_error: `enableSqlExpressions` always returns `false` due to two compounding issues: (1) inverted - Loop done round=12 (tools: read_file=15, warpgrep_codebase_search=3, grep=6) - WarpGrep: organization_context.member None check in ControlSiloOrganizationEndpoint or per - Review complete: 3 issues - [2/15] keycloak PR#37634 3 raw -> 3 kept - [2/15] keycloak PR#37634 [0.97] null_reference: The second `Objects.requireNonNull` checks `grantType` again instead of `rawTokenId` (copy - [2/15] keycloak PR#37634 [0.97] incorrect_value: Wrong substring indices: The grant type shortcut is at indices 4-6 (0-based) in the encode - [2/15] keycloak PR#37634 [0.97] logic_error: Inverted condition: The method returns `false` when the shortcut equals the expected value - Loop done round=7 (tools: warpgrep_codebase_search=4, read_file=3, grep=5) - Loop done round=10 (tools: warpgrep_codebase_search=1, grep=5, read_file=5, glob=5, list_directory=2) - Review complete: 1 issues - [15/15] cal.com PR#14943 1 raw -> 1 kept - [15/15] cal.com PR#14943 [0.95] logic_error: The `deleteMany` query's `OR` clause is missing a `method: WorkflowMethods.SMS` filter on - Review complete: 1 issues - [12/15] discourse-graphite PR#7 1 raw -> 1 kept - [12/15] discourse-graphite PR#7 [0.97] incorrect_value: Light and dark mode values are swapped in dark-light-choose for `.custom-message-length`. - WarpGrep: DiscourseI18n backend class definition that includes Fallbacks module - WarpGrep: RecoveryAuthnCodesCredentialProviderFactory PROVIDER_ID constant definition - WarpGrep: How does getIdentityPermissions call getUserPermissions and getAnonymousPermissi - WarpGrep: RecoveryAuthnCodesCredentialProvider createCredential method that stores recover - WarpGrep: generateRawCodes method in RecoveryAuthnCodesUtils how codes are generated with - Max tool rounds reached (tools: warpgrep_codebase_search=12, read_file=17, grep=13, bash=3) - WarpGrep: connectedCalendars credentialId type in getConnectedDestinationCalendars return - Loop done round=21 (tools: warpgrep_codebase_search=6, grep=8, read_file=9, glob=1, bash=5) - WarpGrep: How does I18n::Backend::Fallbacks resolve_entry or translate with fallback chain - Review complete: 2 issues - [6/15] sentry-greptile PR#1 2 raw -> 2 kept - [6/15] sentry-greptile PR#1 [0.95] api_misuse: Django QuerySets do not support negative indexing. The OptimizedCursorPaginator.get_result - [6/15] sentry-greptile PR#1 [0.70] null_reference: AttributeError when organization_context.member is None. The expression `organization_cont - WarpGrep: SetupRecoveryAuthnCodesPage getRecoveryAuthnCodes method in test pages - WarpGrep: Where does the searchSupport struct have a tracer field vs searchServer struct t - Review complete: 1 issues - [5/15] sentry-greptile PR#5 1 raw -> 1 kept - [5/15] sentry-greptile PR#5 [0.95] logic_error: Bug in `fetch_error_details`: The code zips `error_ids` with `events.values()`, but `event - WarpGrep: EnterRecoveryAuthnCodePage enterRecoveryAuthnCode and getRecoveryAuthnCodeToEnte - Loop done round=10 (tools: warpgrep_codebase_search=5, read_file=5, grep=9) - Review complete: 2 issues - [4/15] sentry PR#93824 2 raw -> 2 kept - [4/15] sentry PR#93824 [0.95] incorrect_value: Metric tag key is `"shards"` (plural) instead of `"shard"` (singular), inconsistent with a - [4/15] sentry PR#93824 [0.82] test_correctness: `time.sleep(0.1)` is called to give flusher threads time to process after drift change, bu - WarpGrep: How does initWatcher interact with search index handleEvent? What happens to eve - Loop done round=13 (tools: warpgrep_codebase_search=4, read_file=7, grep=15, bash=1) - Loop done round=20 (tools: warpgrep_codebase_search=4, read_file=15, grep=9, glob=5) - Review complete: 2 issues - [1/15] keycloak PR#37429 2 raw -> 2 kept - [1/15] keycloak PR#37429 [0.97] localization: Lithuanian (lt) `loginTotpStep1` value was replaced with Italian text "Installa una delle - [1/15] keycloak PR#37429 [0.96] localization: Simplified Chinese (zh_CN) `totpStep1` value uses Traditional Chinese characters (手機, 安裝, - WarpGrep: How does the denial cache interact with permission updates, is the denial cache - Review complete: 3 issues - [11/15] discourse-graphite PR#10 3 raw -> 3 kept - [11/15] discourse-graphite PR#10 [0.95] test_correctness: Fabricator file contents are swapped: category_fabricator.rb contains the Fabricator(:embe - [11/15] discourse-graphite PR#10 [0.95] null_reference: When the `embed_category` site setting is not configured, the SQL query returns zero rows. - [11/15] discourse-graphite PR#10 [0.60] api_misuse: JavaScript's `String.prototype.replace()` with a string first argument only replaces the f - WarpGrep: AbstractUserAdapterFederatedStorage credentialManager getFederatedCredentialsStr - WarpGrep: CredentialProvider interface createCredential method signature with type paramet - WarpGrep: How does the search index get updated with new events? How does UpdateIndex work - WarpGrep: validateCheckRequest function in the RBAC service, how it sets UserUID and Ident - WarpGrep: cache.Cache interface definition in grafana authlib cache package - WarpGrep: SiteSetting default_locale definition and default value - WarpGrep: recovery codes display template kc-recovery-codes-list freemarker formatting - WarpGrep: How does searchServer.buildIndexes work during init? Can it build the same key c - Max tool rounds reached (tools: warpgrep_codebase_search=7, grep=19, read_file=13, glob=5, list_directory=1) - Review complete: 0 issues - [13/15] cal.com PR#22532 0 raw -> 0 kept - Loop done round=14 (tools: warpgrep_codebase_search=7, read_file=13, grep=4) - Review complete: 1 issues - [8/15] grafana PR#103633 1 raw -> 1 kept - [8/15] grafana PR#103633 [0.92] test_correctness: The test "Should deny on explicit cache deny entry" sets the permission cache value to `fa - Loop done round=13 (tools: warpgrep_codebase_search=8, read_file=11, grep=6) - Review complete: 0 issues - [7/15] grafana PR#97529 0 raw -> 0 kept - Loop done round=25 (tools: warpgrep_codebase_search=8, read_file=8, grep=13, bash=9, glob=1) - Review complete: 1 issues - [10/15] discourse-graphite PR#9 1 raw -> 1 kept - [10/15] discourse-graphite PR#9 [0.55] type_error: Missing `.to_sym` conversion in `ensure_loaded!` method. Since `@loaded_locales` stores Sy - WarpGrep: RecoveryAuthnCodesConfigBean generatedRecoveryAuthnCodesAsString method definiti - Loop done round=16 (tools: warpgrep_codebase_search=15, grep=3, read_file=12) - Review complete: 2 issues - [3/15] keycloak PR#38446 2 raw -> 2 kept - [3/15] keycloak PR#38446 [0.72] logic_error: getCredentials() reconstructs the credential via RecoveryAuthnCodesCredentialModel.createF - [3/15] keycloak PR#38446 [0.55] test_correctness: isValid validates recovery codes against ANY code using plaintext comparison and never con -Wrote candidates to /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/candidates.json - -============================================================ -DONE: 15 reviewed, 22 raw -> 22 filtered -Avg/PR: 1.5, Time: 862s -Candidates: /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/candidates.json -Benchmark data updated: /Users/tejas/personal/applymodel/zmisc/examples/code-review-benchmark/offline/results/benchmark_data.json diff --git a/pr_review_agent/output/iter8_eval.log b/pr_review_agent/output/iter8_eval.log deleted file mode 100644 index ec4a3e0..0000000 --- a/pr_review_agent/output/iter8_eval.log +++ /dev/null @@ -1,86 +0,0 @@ - -keycloak PR#37429: 4 golden, 3 candidates - TP: [Medium] The translation is in Italian instead of Lithuanian. This should be translated t... - TP: [Medium] The totpStep1 value uses Traditional Chinese terms in the Simplified Chinese fil... - FN: [Low] The anchor sanitization logic has a potential issue where it consumes English ma... - FN: [Low] The method name 'santizeAnchors' should be 'sanitizeAnchors' (missing 'i').... - -keycloak PR#37634: 4 golden, 2 candidates - TP: [Critical] Wrong parameter in null check (grantType vs. rawTokenId)... - TP: [High] In isAccessTokenId, the substring for the grant shortcut and the equality check ... - FN: [Low] Javadoc mentions "usually like 3-letters shortcut" but some implementations use ... - FN: [Low] Catching generic RuntimeException is too broad. The implementation throws Illeg... - -keycloak PR#38446: 2 golden, 1 candidates - TP: [Low] After creating the RecoveryAuthnCodesCredentialModel, consider setting its id fr... - FN: [Medium] Unsafe raw List deserialization without type safety. Calling Optional.get() dire... - -sentry PR#93824: 5 golden, 3 candidates - TP: [Medium] Inconsistent metric tagging with 'shard' and 'shards'... - TP: [Low] Fixed sleep in tests can be flaky; wait on condition instead... - TP: [Medium] Sleep in test_consumer.py won’t actually wait because time.sleep was monkeypatch... - FN: [High] Because flusher processes are created via multiprocessing.get_context('spawn').P... - FN: [Medium] Breaking out of the loop when the deadline has elapsed can skip terminating rema... - -sentry-greptile PR#1: 4 golden, 2 candidates - TP: [High] Django querysets do not support negative slicing... - TP: [High] When requests are authenticated with API keys or org auth tokens (which have use... - FN: [Low] Importing non-existent OptimizedCursorPaginator... - FN: [High] get_item_key assumes a numeric key, but the paginator is used with order_by=-dat... - -grafana PR#103633: 2 golden, 1 candidates - TP: [Low] The test comment says the cached permissions 'allow access', but the map stores ... - FN: [High] The Check operation exhibits asymmetric cache trust logic: cached permission gra... - -grafana PR#94942: 2 golden, 1 candidates - TP: [Critical] The enableSqlExpressions function has flawed logic that always returns false, ef... - FN: [High] Several methods such as NewInMemoryDB().RunCommands and db.QueryFramesInto retur... - -discourse-graphite PR#9: 2 golden, 1 candidates - TP: [Low] Consider normalizing the input locale (e.g., to a symbol) when checking/loading ... - FN: [Low] Thread-safety issue with lazy @loaded_locales... - -discourse-graphite PR#10: 4 golden, 3 candidates - FN: [Critical] NoMethodError before_validation in EmbeddableHost... - FN: [Medium] The update and destroy methods in Admin::EmbeddableHostsController do not valida... - FN: [Medium] record_for_host compares lower(host) = ? but does not normalize the parameter’s ... - FN: [High] Because this migration inserts embeddable_hosts rows with raw SQL, any existing ... - -discourse-graphite PR#7: 3 golden, 2 candidates - TP: [Low] In .topic-meta-data h5 a, the original code had color: scale-color($primary, $li... - TP: [Low] This change for desktop/user.css changes $primary from 30% to 50% for the light ... - TP: [Low] In topic-post.css the original code used $lightness: 70% but the replacement use... - -cal.com PR#22532: 2 golden, 2 candidates - TP: [Medium] The updateManyByCredentialId call uses an empty data object, which prevents Pris... - TP: [Low] logic: macOS-specific sed syntax with empty string after -i flag will fail on Li... - -cal.com PR#8330: 2 golden, 2 candidates - TP: [Medium] Incorrect end time calculation using slotStartTime instead of slotEndTime... - TP: [Medium] Using === for dayjs object comparison will always return false as it compares ob... - -cal.com PR#14943: 2 golden, 1 candidates - TP: [High] The deletion logic in scheduleSMSReminders.ts incorrectly deletes non-SMS workfl... - FN: [High] Using retryCount: reminder.retryCount + 1 reads a possibly stale value and can l... - -============================================================ -OVERALL RESULTS -============================================================ - True Positives: 21/43 - False Positives: 6 - False Negatives: 22 - Total Candidates: 24 - Precision: 87.5% - Recall: 48.8% - F1: 62.7% - -Per-repo breakdown: -Repo Prec Recall F1 TP FP FN -------------------------------------------------------- -cal.com 100.0% 83.3% 90.9% 5 0 1 -discourse 66.7% 44.4% 53.3% 4 4 5 -grafana 100.0% 50.0% 66.7% 2 0 2 -keycloak 83.3% 50.0% 62.5% 5 1 5 -sentry 100.0% 55.6% 71.4% 5 1 4 - -Results saved to /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/evaluation_results.json diff --git a/pr_review_agent/output/iter8_mini15.log b/pr_review_agent/output/iter8_mini15.log deleted file mode 100644 index a7ea32e..0000000 --- a/pr_review_agent/output/iter8_mini15.log +++ /dev/null @@ -1,203 +0,0 @@ -============================================================ -PR Review Agent - Benchmark Pipeline -============================================================ -Loaded 50 PRs -WarpGrep: ENABLED -Mini eval: 15 PRs (3 per repo) -Reviewing 15 PRs - -Running 15 PRs with parallelism=15 - -[1/15] keycloak PR#37429 (4 golden) -[2/15] keycloak PR#37634 (4 golden) -[3/15] keycloak PR#38446 (2 golden) -[4/15] sentry PR#93824 (5 golden) -[5/15] sentry-greptile PR#5 (3 golden) -[6/15] sentry-greptile PR#1 (4 golden) -[7/15] grafana PR#97529 (2 golden) -[8/15] grafana PR#103633 (2 golden) -[9/15] grafana PR#94942 (2 golden) -[10/15] discourse-graphite PR#9 (2 golden) -[11/15] discourse-graphite PR#10 (4 golden) -[12/15] discourse-graphite PR#7 (3 golden) -[13/15] cal.com PR#22532 (2 golden) -[14/15] cal.com PR#8330 (2 golden) -[15/15] cal.com PR#14943 (2 golden) - [10/15] discourse-graphite PR#9 6 files, 31 added lines - [15/15] cal.com PR#14943 3 files, 38 added lines - [8/15] grafana PR#103633 4 files, 240 added lines - [12/15] discourse-graphite PR#7 32 files, 115 added lines - [7/15] grafana PR#97529 5 files, 25 added lines - [14/15] cal.com PR#8330 4 files, 111 added lines - [6/15] sentry-greptile PR#1 3 files, 128 added lines - [13/15] cal.com PR#22532 17 files, 379 added lines - [9/15] grafana PR#94942 4 files, 41 added lines - [2/15] keycloak PR#37634 28 files, 722 added lines - [1/15] keycloak PR#37429 48 files, 343 added lines - [11/15] discourse-graphite PR#10 36 files, 449 added lines - [5/15] sentry-greptile PR#5 105 files, 2312 added lines - [3/15] keycloak PR#38446 8 files, 256 added lines - [4/15] sentry PR#93824 6 files, 199 added lines - WarpGrep: How is the deleteMany for workflowReminder used and what does the OR condition w - WarpGrep: How does the VerifyMessageProperties class resolve English file paths for non-En - WarpGrep: How is SpansBuffer constructed and what does the assigned_shards property look l - WarpGrep: How is enableSqlExpressions used and what is the expected behavior of the FlagSq - WarpGrep: How does Prisma updateMany behave when given an empty data object? Does it still - WarpGrep: How is checkIfIsAvailable called and what parameters does it receive? Look for a - WarpGrep: Does Django QuerySet support negative indexing or negative slicing? How does que - WarpGrep: How is the cacheMu mutex used in bleveBackend to protect the cache map? What met - WarpGrep: Where is the category fabricator defined and used in tests? - WarpGrep: How does getFederatedCredentialsStream work in UserCredentialManager? What does - WarpGrep: How does the Check method in the RBAC service work, including checkPermission an - WarpGrep: What does the availabilityCheckProps object contain? How is it constructed befor - WarpGrep: Lithuanian locale messages_lt totpStep1 translation text across all locale files - WarpGrep: What is the SelectedCalendar model schema and does it have an @updatedAt field? - WarpGrep: How does SpanFlusher.main call buffer.flush_segments and what arguments does it - WarpGrep: How does the paginate method on ControlSiloOrganizationEndpoint or base endpoint - WarpGrep: Who calls connectedCalendars handler and how is the return type used, especially - WarpGrep: How does dayjs comparison with === work? Are dayjs objects compared by reference - WarpGrep: How is get_merged_pr_single_issue_template defined and who calls it? Looking at - WarpGrep: How is QueryTypeSQL handled in ReadQuery and what callers depend on it succeedin - WarpGrep connection error, retrying in 3s (attempt 1/3): SSLError - WarpGrep: Who calls NewResourceServer and how is the returned server used? Is Init called - WarpGrep: How is ProcessSpansStrategyFactory instantiated and what keyword arguments does - WarpGrep: How is RecoveryAuthnCodesCredentialModel.createFromValues used? What parameters - WarpGrep connection error, retrying in 3s (attempt 1/3): SSLError - WarpGrep: Who calls newRBACClient or ProvideAuthZClient and how is the cache used in the a - WarpGrep: What are the different WorkflowMethods enum values? Are there EMAIL and WHATSAPP - WarpGrep: Definition of dark-light-choose SCSS function/mixin - WarpGrep: How does the store's _hydrateEmbedded handle plural ids (e.g. color_ids) and wha - WarpGrep: How is the SpanFlusher.main static method called as a target for multiprocessing - WarpGrep connection error, retrying in 3s (attempt 1/3): SSLError - WarpGrep: How does fetch_error_details use nodestore.backend.get_multi and zip error_ids w - WarpGrep: How does server.Init work with sync.Once? What is the concurrency model for init - WarpGrep: How does updateCredential work in user credential manager for federated/storage - WarpGrep: How does Prisma updateMany handle an empty data object {} with @updatedAt? Does - WarpGrep: How is the SessionType enum used in the actual file and what are all the shortcu - WarpGrep: How is dark-light-choose used to handle light vs dark themes in SCSS - WarpGrep: How is the cursor parsed from the HTTP request? Can cursor offset be negative? H - WarpGrep: How does the REST adapter basePath method handle types with underscores? What ty - WarpGrep: How does the permDenialCache interact with permCache - asymmetric caching of all - WarpGrep: How does the Attributes component filter attributes when searchQuery is empty? T - WarpGrep: Where is the connectedCalendars handler called from the web app, specifically ho - WarpGrep: What is organization_context.member? Can member be None for superusers? How does - WarpGrep: How is the RefreshTokenGrantType shortcut "rt" used and could it conflict with t - WarpGrep: What does UserCredentialManager.updateCredential return? Is it a boolean? How is - WarpGrep: How does checkPermission work in the RBAC service - what does it do when a permi - WarpGrep: Who calls BuildIndex on bleveBackend and what concurrency protection exists arou - WarpGrep: nodestore backend get_multi return type - does it preserve key order or return a - WarpGrep: How does the recovery codes validation work in RecoveryAuthnCodesFormAuthenticat - WarpGrep: How is the FlexContainer in traceWaterfall styled with height: 100%? After repla - WarpGrep: How does the newRemoteRBACClient function signature change - does it accept reg - WarpGrep: Where is the OrganizationAuditLogsEndpoint class defined? The actual implementat - WarpGrep: How are two dayjs objects compared for equality in the codebase? Do they use isS - WarpGrep: Where is the HIDDEN_ATTRIBUTES filtering applied - is it applied when searchQuer - WarpGrep: Does Prisma updateMany with empty data object actually update @updatedAt fields? - WarpGrep: UserCredentialManager updateCredential implementation - how does it delegate to - WarpGrep: Where does ClientCredentialsGrantType set the grant type attribute on the client - WarpGrep: What is the English value of identityProviderLinkBody (non-HTML version) in emai - WarpGrep: What does the dateOverrides parameter contain when passed to checkIfIsAvailable? - Loop done round=8 (tools: read_file=7, warpgrep_codebase_search=2, grep=5, glob=1) - Review complete: 1 issues - [15/15] cal.com PR#14943 1 raw -> 1 kept - [15/15] cal.com PR#14943 [0.95] logic_error: The second branch of the `deleteMany` OR condition (`{ retryCount: { gt: 1 } }`) is missin - WarpGrep: How does BackwardsCompatibilityUserStorage.getCredentials interact with the reco - WarpGrep: How does the embedding controller's update action save changes? Does it process - WarpGrep: How does processBatchCheckGroup work in the RBAC service - does it use permDenia - WarpGrep connection error, retrying in 3s (attempt 1/3): SSLError - WarpGrep: How does the overflowEllipsis theme mixin in FlexCenter interact with event atta - Loop done round=12 (tools: warpgrep_codebase_search=5, read_file=3, grep=6, glob=4) - Review complete: 2 issues - [14/15] cal.com PR#8330 2 raw -> 2 kept - [14/15] cal.com PR#8330 [0.97] api_misuse: Comparison of two dayjs objects using `===` always returns `false` because `===` compares - [14/15] cal.com PR#8330 [0.96] logic_error: Copy-paste bug: `end` is calculated from `slotStartTime` instead of `slotEndTime`, making - WarpGrep: Where is the topic-map buttons btn a link color defined in desktop and mobile to - WarpGrep: How does OrganizationAuditPermission work? Does it require membership or can API - Loop done round=8 (tools: read_file=9, grep=4, warpgrep_codebase_search=3) - WarpGrep: How does getOrCreateIndex coordinate concurrent access to prevent duplicate inde - Review complete: 2 issues - [2/15] keycloak PR#37634 2 raw -> 2 kept - [2/15] keycloak PR#37634 [0.97] null_reference: Copy-paste error: line 114 checks `grantType` a second time instead of `rawTokenId`. `Obje - [2/15] keycloak PR#37634 [0.97] test_correctness: Two independent bugs in `isAccessTokenId` matcher: (1) Wrong substring indices — `substrin - Loop done round=11 (tools: warpgrep_codebase_search=2, grep=3, read_file=9, glob=2, bash=1, list_directory=1) - Review complete: 1 issues - [9/15] grafana PR#94942 1 raw -> 1 kept - [9/15] grafana PR#94942 [0.95] logic_error: The function `enableSqlExpressions` always returns `false` on every code path, making SQL - WarpGrep: What happens when Prisma updateMany is called with empty data object and @update - WarpGrep: PostCreator initialize method - how does it handle the category parameter? Does - WarpGrep: EmbeddableHost record_for_host method - what does it return when no host is foun - WarpGrep: What does UserCredentialModel.buildFromBackupAuthnCode do? What type and challen - WarpGrep: How is getConnectedDestinationCalendarsAndEnsureDefaultsInDb defined and what do - WarpGrep: How do SentryApp or internal integration API tokens pass OrganizationPermission - WarpGrep: How is the TableWidgetVisualization used in chart.tsx? Is this empty data placeh - Loop done round=10 (tools: warpgrep_codebase_search=3, read_file=4, glob=2, list_directory=1, bash=4, grep=2) - WarpGrep: How does the Visualize class fromJSON and toJSON work, and how is it used in upd - Review complete: 2 issues - [12/15] discourse-graphite PR#7 2 raw -> 2 kept - [12/15] discourse-graphite PR#7 [0.95] incorrect_value: Swapped `dark-light-choose` arguments. The original light-mode value was `$lightness: 30%` - [12/15] discourse-graphite PR#7 [0.75] incorrect_value: Light-mode `h3` color changed from original `$lightness: 20%` to `$lightness: 50%`, making - WarpGrep: How does the account-ui frontend handle error-invalid-multivalued-size and other - WarpGrep: How are user profile validation error messages like error-invalid-multivalued-si - Loop done round=12 (tools: warpgrep_codebase_search=4, read_file=7, grep=11) - Review complete: 3 issues - [4/15] sentry PR#93824 3 raw -> 3 kept - [4/15] sentry PR#93824 [0.95] logic_error: Missing `slice_id` when creating per-shard `SpansBuffer` instances in `_create_process_for - [4/15] sentry PR#93824 [0.93] test_correctness: `time.sleep(0.1)` intended to give flusher threads time to process is a no-op because `tim - [4/15] sentry PR#93824 [0.92] incorrect_value: Inconsistent metric tag key `"shard"` vs `"shards"` in `SpanFlusher.main`. The `wait_produ - WarpGrep: How is the prisma client extended or configured with middleware? Look for extens - WarpGrep: Where is the findAllFieldRefs function defined and how does it handle writablePa - WarpGrep: Where is the set_locale method defined in older versions of ApplicationControlle - Loop done round=17 (tools: warpgrep_codebase_search=7, grep=9, read_file=12, bash=2) - WarpGrep: How does the JavaScript store's pluralize method work? What does it return for ' - Review complete: 2 issues - [6/15] sentry-greptile PR#1 2 raw -> 2 kept - [6/15] sentry-greptile PR#1 [0.95] api_misuse: Django QuerySet does not support negative indexing — `queryset[start_offset:stop]` with a - [6/15] sentry-greptile PR#1 [0.85] null_reference: `organization_context.member` can be `None` (e.g., for SentryApp proxy users with no real - WarpGrep: How does the I18n translate method handle fallback locales when a translation is - WarpGrep: Where is the createOTPCredential method in CredentialHelper and how does it comp - Loop done round=14 (tools: warpgrep_codebase_search=9, grep=7, read_file=8, glob=4, bash=2) - WarpGrep: How does the frontend account-ui or admin-ui display user profile validation err - Review complete: 2 issues - [13/15] cal.com PR#22532 2 raw -> 2 kept - [13/15] cal.com PR#22532 [0.80] api_misuse: Prisma's `updateMany` with an empty `data: {}` object is a no-op — no SQL UPDATE is genera - [13/15] cal.com PR#22532 [0.70] portability: `sed -i ''` syntax is macOS/BSD-only. On GNU/Linux, `sed -i` does not take a separate back - WarpGrep: What does PG::Result return when accessing [0] on an empty result set in Rails m - WarpGrep: How does RecoveryAuthnCodesCredentialModel.createFromCredentialModel handle cred - Loop done round=25 (tools: warpgrep_codebase_search=6, grep=27, read_file=9) - Max tool rounds reached (tools: warpgrep_codebase_search=10, read_file=23, grep=12, glob=2) - Review complete: 3 issues - [1/15] keycloak PR#37429 3 raw -> 3 kept - [1/15] keycloak PR#37429 [0.97] localization: Lithuanian locale file contains Italian text for `totpStep1` ("Installa una delle seguenti - [1/15] keycloak PR#37429 [0.96] localization: The `totpStep1` value uses Traditional Chinese characters (手機, 安裝, 應用程式之一) instead of Simp - [1/15] keycloak PR#37429 [0.93] api_misuse: The `error-invalid-multivalued-size` key was changed from i18next interpolation format (`{ - Loop done round=27 (tools: warpgrep_codebase_search=8, read_file=15, grep=8, bash=3, glob=2) - Review complete: 0 issues - [5/15] sentry-greptile PR#5 0 raw -> 0 kept - Review complete: 3 issues - [11/15] discourse-graphite PR#10 3 raw -> 3 kept - [11/15] discourse-graphite PR#10 [0.95] test_correctness: The contents of category_fabricator.rb and embeddable_host_fabricator.rb have been swapped - [11/15] discourse-graphite PR#10 [0.92] null_reference: When the 'embed_category' site setting doesn't exist (e.g., fresh installs), the SQL query - [11/15] discourse-graphite PR#10 [0.80] logic_error: In the plural ID branch of _hydrateEmbedded, _lookupSubType returns undefined when it can' - Loop done round=21 (tools: warpgrep_codebase_search=6, read_file=17, grep=9, bash=4) - WarpGrep: What does SiteSetting.default_locale return? Can it be nil? What is the default - Review complete: 1 issues - [8/15] grafana PR#103633 1 raw -> 1 kept - [8/15] grafana PR#103633 [0.92] test_correctness: The test "Should deny on explicit cache deny entry" uses `false` in the permission cache m - Loop done round=29 (tools: read_file=18, grep=14, bash=7, glob=3, warpgrep_codebase_search=3) - Review complete: 1 issues - [10/15] discourse-graphite PR#9 1 raw -> 1 kept - [10/15] discourse-graphite PR#9 [0.50] type_error: The `ensure_loaded!` method does not convert its `locale` argument to a symbol before chec - Max tool rounds reached (tools: warpgrep_codebase_search=5, read_file=23, grep=20, glob=1) - Review complete: 0 issues - [7/15] grafana PR#97529 0 raw -> 0 kept - Loop done round=25 (tools: warpgrep_codebase_search=10, grep=14, read_file=17) - Review complete: 1 issues - [3/15] keycloak PR#38446 1 raw -> 1 kept - [3/15] keycloak PR#38446 [0.80] logic_error: The `getCredentials()` method creates a `RecoveryAuthnCodesCredentialModel` via `createFro -Wrote candidates to /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/candidates.json - -============================================================ -DONE: 15 reviewed, 24 raw -> 24 filtered -Avg/PR: 1.6, Time: 829s -Candidates: /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/candidates.json -Benchmark data updated: /Users/tejas/personal/applymodel/zmisc/examples/code-review-benchmark/offline/results/benchmark_data.json diff --git a/pr_review_agent/output/iter8_random13.log b/pr_review_agent/output/iter8_random13.log deleted file mode 100644 index 95751b8..0000000 --- a/pr_review_agent/output/iter8_random13.log +++ /dev/null @@ -1,94 +0,0 @@ -============================================================ -PR Review Agent - Benchmark Pipeline -============================================================ -Loaded 50 PRs -WarpGrep: ENABLED -Random eval: 13 PRs (2 per source_repo, seed=42) -Reviewing 13 PRs - -Running 13 PRs with parallelism=13 - -[1/13] cal.com PR#8330 (2 golden) -[2/13] cal.com PR#22532 (2 golden) -[3/13] discourse-graphite PR#3 (2 golden) -[4/13] discourse-graphite PR#8 (3 golden) -[5/13] grafana PR#90939 (2 golden) -[6/13] grafana PR#94942 (2 golden) -[8/13] keycloak PR#32918 (2 golden) -[7/13] keycloak PR#37634 (4 golden) -[9/13] keycloak-greptile PR#1 (2 golden) -[10/13] sentry PR#93824 (5 golden) -[11/13] sentry PR#67876 (3 golden) -[12/13] sentry-greptile PR#5 (3 golden) -[13/13] sentry-greptile PR#3 (3 golden) - [10/13] sentry PR#93824 6 files, 199 added lines - [1/13] cal.com PR#8330 4 files, 111 added lines - [2/13] cal.com PR#22532 17 files, 379 added lines - [6/13] grafana PR#94942 4 files, 41 added lines - [5/13] grafana PR#90939 1 files, 13 added lines - [3/13] discourse-graphite PR#3 10 files, 155 added lines - [9/13] keycloak-greptile PR#1 9 files, 407 added lines - [12/13] sentry-greptile PR#5 105 files, 2312 added lines - [8/13] keycloak PR#32918 4 files, 268 added lines - [13/13] sentry-greptile PR#3 8 files, 480 added lines - [11/13] sentry PR#67876 3 files, 247 added lines - [7/13] keycloak PR#37634 28 files, 722 added lines - [4/13] discourse-graphite PR#8 16 files, 389 added lines - WarpGrep: How is the upsampled_count function defined in the errors dataset? Does it exist - WarpGrep: SpansBuffer constructor and how assigned_shards property is set - WarpGrep: How does getForLogin work in IdentityProviderStorageProvider? What are all imple - WarpGrep: How is email_in_restriction_setting used and what does the email_domains_whiteli - WarpGrep: How does checkIfIsAvailable work and who calls it? - WarpGrep: Who calls GetWebAssets and how is it used? - WarpGrep: How does PRCommentWorkflow.get_merged_pr_single_issue_template work and who call - WarpGrep: What happens when Prisma updateMany is called with an empty data object? Does it - WarpGrep: How does PipelineView.determine_active_organization work and what does self.acti - WarpGrep: Where is isConditionalPasskeysEnabled defined on UsernameForm or its parent clas - WarpGrep: How does the remove_member action in the groups controller work? What does group - WarpGrep: How does SpanFlusher.main() use the buffer parameter and what is the first posit - WarpGrep: How does dayjs object comparison with === work? Are dayjs objects compared by re - WarpGrep: What does the SnubaParams.project_ids property return? Can it be None? - WarpGrep: How is enableSqlExpressions used and what does FlagSqlExpressions feature flag c - WarpGrep: How is updateManyByCredentialId used in the codebase and what does passing an em - WarpGrep: How does the Enum class work? What does Enum.new(:block, :do_nothing) return for - WarpGrep: How does UsernameForm relate to UsernamePasswordForm in the class hierarchy? Doe - WarpGrep: What is IdentityProviderListQuery and how does it store multiple search keys? Ho - WarpGrep: What is the dateOverrides structure passed to checkIfIsAvailable and how are dat - WarpGrep: What does the connectedCalendars query return, specifically does it include cach - WarpGrep: How does fetch_error_details use nodestore.backend.get_multi and what ordering g - WarpGrep: Is upsampled_count defined as a SnQLFunction in the errors dataset (errors.py)? - WarpGrep: How is ProcessSpansStrategyFactory.create_with_partitions called and what parame - WarpGrep: Who calls the enableSqlExpressions function and how is the return value used? - WarpGrep: What is pipeline.signature and how is it generated in IntegrationPipeline? - WarpGrep: What HTTP method does the test use for remove_member action in admin groups cont - WarpGrep: How does LoginFilter enum work in IdentityProviderStorageProvider? What are the - WarpGrep: What does the Attributes component filter logic look like when searchQuery is em - WarpGrep connection error, retrying in 3s (attempt 1/3): SSLError - WarpGrep: How does GitHubIdentityProvider get_oauth_client_id and get_oauth_client_secret - WarpGrep: What format does email_domains_whitelist or email_domains_blacklist SiteSetting - WarpGrep: How does the errors dataset resolve function names? Does it inherit from the dis - WarpGrep: What does nodestore.backend.get_multi return? Is it an ordered dict or unordered - WarpGrep: How is BlockedEmail.should_block? used in the email validator? Does it get calle - WarpGrep: How does the determine_active_organization method work on PipelineView? Is it a - WarpGrep connection error, retrying in 3s (attempt 1/3): SSLError - WarpGrep: What does integration.metadata look like for GitHub integrations? What fields do - WarpGrep: Does the SelectedCalendar model have an updatedAt field in the Prisma schema? - WarpGrep: How does the FlexBox/FlexContainer replacement with Flex component affect the he - WarpGrep: Who calls the Context copy constructor in OAuth2GrantType, specifically "new Con - WarpGrep: What does group.users.delete do with an integer user_id in ActiveRecord? Does it - Loop done round=5 (tools: read_file=2, warpgrep_codebase_search=1, grep=4) - WarpGrep: What implementations of OAuth2GrantTypeFactory exist and do they all implement g - WarpGrep: How is QueryFramesInto called and what does it do? Is there another implementati - WarpGrep: What does PRCommentWorkflow._truncate_title refer to? Is _truncate_title a stati - Review complete: 1 issues - [5/13] grafana PR#90939 1 raw -> 1 kept - [5/13] grafana PR#90939 [0.92] race_condition: Incomplete double-checked locking pattern: after acquiring the exclusive write lock, the c - WarpGrep: What is the relationship between CommitContextIntegration and PRCommentWorkflow? - WarpGrep: How does the deleteCache trpc mutation verify credential ownership, especially f - WarpGrep: How does the group members template reference members in the group/members.hbs? - WarpGrep: Where does GitHubInstallation (PipelineView) get the determine_active_organizati - WarpGrep: Does Prisma updateMany with empty data object {} automatically populate @updated - WarpGrep: Where is BROKER_PUBLIC filtered in getForLogin or getLoginSearchOptions? Does ge - WarpGrep: What does _get_bytes_multi return? Is it an ordered dict that preserves the orde - WarpGrep: How does Pipeline dispatch to pipeline views? How is PipelineView.dispatch calle - WarpGrep: How is ExpressionQueryReader.ReadQuery called? What is the call chain that leads diff --git a/pr_review_agent/output/iter9_cal14943.log b/pr_review_agent/output/iter9_cal14943.log deleted file mode 100644 index 6d49fe0..0000000 --- a/pr_review_agent/output/iter9_cal14943.log +++ /dev/null @@ -1,26 +0,0 @@ -============================================================ -PR Review Agent - Benchmark Pipeline -============================================================ -Loaded 50 PRs -WarpGrep: ENABLED -Reviewing 1 PRs - -Running 1 PRs with parallelism=1 - -[1/1] cal.com PR#14943 (2 golden) - [1/1] cal.com PR#14943 3 files, 38 added lines - WarpGrep: How does the deletion of workflow reminders interact with the OR condition for m - WarpGrep: How are workflow reminders deleted in the scheduling cron jobs for Whatsapp and - WarpGrep: Where are scheduled email workflow reminders cleaned up or deleted? Is there a s - WarpGrep: How does the scheduleWhatsappReminders handler clean up or delete old Whatsapp w - Loop done round=9 (tools: read_file=5, codebase_search=4, grep=12) - Review complete: 1 issues - [1/1] cal.com PR#14943 1 raw -> 1 kept - [1/1] cal.com PR#14943 [0.95] logic_error: The second condition in the OR clause of the deleteMany query (`{ retryCount: { gt: 1 } }` -Wrote candidates to /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/candidates.json - -============================================================ -DONE: 1 reviewed, 1 raw -> 1 filtered -Avg/PR: 1.0, Time: 151s -Candidates: /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/candidates.json -Benchmark data updated: /Users/tejas/personal/applymodel/zmisc/examples/code-review-benchmark/offline/results/benchmark_data.json diff --git a/pr_review_agent/output/iter9_discourse10.log b/pr_review_agent/output/iter9_discourse10.log deleted file mode 100644 index 7f4eb70..0000000 --- a/pr_review_agent/output/iter9_discourse10.log +++ /dev/null @@ -1,31 +0,0 @@ -============================================================ -PR Review Agent - Benchmark Pipeline -============================================================ -Loaded 50 PRs -WarpGrep: ENABLED -Reviewing 1 PRs - -Running 1 PRs with parallelism=1 - -[1/1] discourse-graphite PR#10 (4 golden) - [1/1] discourse-graphite PR#10 36 files, 449 added lines - WarpGrep: How does the store.find method work for singleton resources vs collections in Di - WarpGrep: How does _hydrateEmbedded and _lookupSubType work in the Discourse store - WarpGrep: How does the rest adapter basePath method handle type names with underscores vs - WarpGrep: How does the migration handle the case when embed_category site setting doesn't - WarpGrep: Where is basePath called and what type values get passed to it in the REST adapt - WarpGrep: How does PG::Result handle indexing with [0] when there are no rows in PostgreSQ - Loop done round=32 (tools: codebase_search=6, grep=18, read_file=20, glob=6) - Review complete: 4 issues - [1/1] discourse-graphite PR#10 4 raw -> 4 kept - [1/1] discourse-graphite PR#10 [0.95] null_reference: Migration unconditionally dereferences `execute(...)[0]['id']` without checking if any row - [1/1] discourse-graphite PR#10 [0.95] logic_error: The file contents of `category_fabricator.rb` and `embeddable_host_fabricator.rb` are swap - [1/1] discourse-graphite PR#10 [0.85] logic_error: The `update` action is a no-op — it re-renders the existing `@embedding` without processin - [1/1] discourse-graphite PR#10 [0.80] logic_error: In the `_ids` plural hydration branch of `_hydrateEmbedded`, `_lookupSubType` can return ` -Wrote candidates to /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/candidates.json - -============================================================ -DONE: 1 reviewed, 4 raw -> 4 filtered -Avg/PR: 4.0, Time: 352s -Candidates: /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/candidates.json -Benchmark data updated: /Users/tejas/personal/applymodel/zmisc/examples/code-review-benchmark/offline/results/benchmark_data.json diff --git a/pr_review_agent/output/iter9_eval.log b/pr_review_agent/output/iter9_eval.log deleted file mode 100644 index 9690bb0..0000000 --- a/pr_review_agent/output/iter9_eval.log +++ /dev/null @@ -1,95 +0,0 @@ - -keycloak PR#37429: 4 golden, 2 candidates - TP: [Medium] The translation is in Italian instead of Lithuanian. This should be translated t... - TP: [Medium] The totpStep1 value uses Traditional Chinese terms in the Simplified Chinese fil... - FN: [Low] The anchor sanitization logic has a potential issue where it consumes English ma... - FN: [Low] The method name 'santizeAnchors' should be 'sanitizeAnchors' (missing 'i').... - -keycloak PR#37634: 4 golden, 2 candidates - TP: [Critical] Wrong parameter in null check (grantType vs. rawTokenId)... - TP: [High] In isAccessTokenId, the substring for the grant shortcut and the equality check ... - FN: [Low] Javadoc mentions "usually like 3-letters shortcut" but some implementations use ... - FN: [Low] Catching generic RuntimeException is too broad. The implementation throws Illeg... - -keycloak PR#38446: 2 golden, 2 candidates - TP: [Low] After creating the RecoveryAuthnCodesCredentialModel, consider setting its id fr... - FN: [Medium] Unsafe raw List deserialization without type safety. Calling Optional.get() dire... - -sentry PR#93824: 5 golden, 2 candidates - TP: [Medium] Inconsistent metric tagging with 'shard' and 'shards'... - TP: [High] Because flusher processes are created via multiprocessing.get_context('spawn').P... - FN: [Low] Fixed sleep in tests can be flaky; wait on condition instead... - FN: [Medium] Sleep in test_consumer.py won’t actually wait because time.sleep was monkeypatch... - FN: [Medium] Breaking out of the loop when the deadline has elapsed can skip terminating rema... - -sentry-greptile PR#5: 3 golden, 2 candidates - TP: [Low] Using zip(error_ids, events.values()) assumes the get_multi result preserves the... - FN: [Medium] Breaking changes in error response format... - FN: [Medium] Detector validator uses wrong key when updating type... - -sentry-greptile PR#1: 4 golden, 2 candidates - TP: [High] Django querysets do not support negative slicing... - TP: [High] When requests are authenticated with API keys or org auth tokens (which have use... - FN: [Low] Importing non-existent OptimizedCursorPaginator... - FN: [High] get_item_key assumes a numeric key, but the paginator is used with order_by=-dat... - -grafana PR#97529: 2 golden, 1 candidates - FN: [High] A race condition in BuildIndex allows multiple goroutines to concurrently build ... - FN: [High] Calling s.search.TotalDocs() here may race with concurrent index creation: Total... - -grafana PR#103633: 2 golden, 2 candidates - TP: [Low] The test comment says the cached permissions 'allow access', but the map stores ... - FN: [High] The Check operation exhibits asymmetric cache trust logic: cached permission gra... - -grafana PR#94942: 2 golden, 1 candidates - TP: [Critical] The enableSqlExpressions function has flawed logic that always returns false, ef... - FN: [High] Several methods such as NewInMemoryDB().RunCommands and db.QueryFramesInto retur... - -discourse-graphite PR#9: 2 golden, 1 candidates - TP: [Low] Consider normalizing the input locale (e.g., to a symbol) when checking/loading ... - FN: [Low] Thread-safety issue with lazy @loaded_locales... - -discourse-graphite PR#10: 4 golden, 4 candidates - FN: [Critical] NoMethodError before_validation in EmbeddableHost... - FN: [Medium] The update and destroy methods in Admin::EmbeddableHostsController do not valida... - FN: [Medium] record_for_host compares lower(host) = ? but does not normalize the parameter’s ... - FN: [High] Because this migration inserts embeddable_hosts rows with raw SQL, any existing ... - -discourse-graphite PR#7: 3 golden, 1 candidates - TP: [Low] In .topic-meta-data h5 a, the original code had color: scale-color($primary, $li... - TP: [Low] This change for desktop/user.css changes $primary from 30% to 50% for the light ... - TP: [Low] In topic-post.css the original code used $lightness: 70% but the replacement use... - -cal.com PR#22532: 2 golden, 2 candidates - TP: [Medium] The updateManyByCredentialId call uses an empty data object, which prevents Pris... - TP: [Low] logic: macOS-specific sed syntax with empty string after -i flag will fail on Li... - -cal.com PR#8330: 2 golden, 3 candidates - TP: [Medium] Incorrect end time calculation using slotStartTime instead of slotEndTime... - TP: [Medium] Using === for dayjs object comparison will always return false as it compares ob... - -cal.com PR#14943: 2 golden, 1 candidates - TP: [High] The deletion logic in scheduleSMSReminders.ts incorrectly deletes non-SMS workfl... - FN: [High] Using retryCount: reminder.retryCount + 1 reads a possibly stale value and can l... - -============================================================ -OVERALL RESULTS -============================================================ - True Positives: 21/43 - False Positives: 9 - False Negatives: 22 - Total Candidates: 28 - Precision: 75.0% - Recall: 48.8% - F1: 59.2% - -Per-repo breakdown: -Repo Prec Recall F1 TP FP FN -------------------------------------------------------- -cal.com 83.3% 83.3% 83.3% 5 1 1 -discourse 66.7% 44.4% 53.3% 4 4 5 -grafana 50.0% 33.3% 40.0% 2 2 4 -keycloak 83.3% 50.0% 62.5% 5 1 5 -sentry 83.3% 41.7% 55.6% 5 1 7 - -Results saved to /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/evaluation_results.json diff --git a/pr_review_agent/output/iter9_grafana103633.log b/pr_review_agent/output/iter9_grafana103633.log deleted file mode 100644 index b754ff7..0000000 --- a/pr_review_agent/output/iter9_grafana103633.log +++ /dev/null @@ -1,30 +0,0 @@ -============================================================ -PR Review Agent - Benchmark Pipeline -============================================================ -Loaded 50 PRs -WarpGrep: ENABLED -Reviewing 1 PRs - -Running 1 PRs with parallelism=1 - -[1/1] grafana PR#103633 (2 golden) - [1/1] grafana PR#103633 4 files, 240 added lines - WarpGrep: How does the Check method in the RBAC service handle permissions caching and wha - WarpGrep: What callers use newRBACClient and how is it referenced across the codebase? - WarpGrep: How does the permDenialCache interact with the regular permCache invalidation? W - WarpGrep: What does getCachedIdentityPermissions return and how does checkPermission use t - WarpGrep: How is the checkPermission function signature defined and how many parameters do - WarpGrep: What is the asymmetric cache pattern risk: denial cache is checked first but per - WarpGrep: What interface does WithCacheClientOption expect? What methods must the cache im - WarpGrep: How does checkPermission evaluate scopeMap entries with false values? Does scope - Loop done round=13 (tools: codebase_search=8, read_file=13, grep=8) - Review complete: 1 issues - [1/1] grafana PR#103633 1 raw -> 1 kept - [1/1] grafana PR#103633 [0.95] test_correctness: In the test "Should deny on explicit cache deny entry", the permCache value for "dashboard -Wrote candidates to /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/candidates.json - -============================================================ -DONE: 1 reviewed, 1 raw -> 1 filtered -Avg/PR: 1.0, Time: 459s -Candidates: /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/candidates.json -Benchmark data updated: /Users/tejas/personal/applymodel/zmisc/examples/code-review-benchmark/offline/results/benchmark_data.json diff --git a/pr_review_agent/output/iter9_grafana94942.log b/pr_review_agent/output/iter9_grafana94942.log deleted file mode 100644 index 861f9bd..0000000 --- a/pr_review_agent/output/iter9_grafana94942.log +++ /dev/null @@ -1,25 +0,0 @@ -============================================================ -PR Review Agent - Benchmark Pipeline -============================================================ -Loaded 50 PRs -WarpGrep: ENABLED -Reviewing 1 PRs - -Running 1 PRs with parallelism=1 - -[1/1] grafana PR#94942 (2 golden) - [1/1] grafana PR#94942 4 files, 41 added lines - WarpGrep: How is enableSqlExpressions used and what does the FlagSqlExpressions feature fl - WarpGrep: Who calls ReadQuery on ExpressionQueryReader and how are SQL expression queries - WarpGrep: Where is ExpressionQueryReader ReadQuery called and how does it handle the SQL q - Loop done round=11 (tools: codebase_search=3, grep=12, read_file=6, list_directory=1, glob=3) - Review complete: 1 issues - [1/1] grafana PR#94942 1 raw -> 1 kept - [1/1] grafana PR#94942 [0.97] logic_error: The `enableSqlExpressions` function always returns `false` due to two compounding errors: -Wrote candidates to /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/candidates.json - -============================================================ -DONE: 1 reviewed, 1 raw -> 1 filtered -Avg/PR: 1.0, Time: 157s -Candidates: /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/candidates.json -Benchmark data updated: /Users/tejas/personal/applymodel/zmisc/examples/code-review-benchmark/offline/results/benchmark_data.json diff --git a/pr_review_agent/output/iter9_mini15.log b/pr_review_agent/output/iter9_mini15.log deleted file mode 100644 index 685275d..0000000 --- a/pr_review_agent/output/iter9_mini15.log +++ /dev/null @@ -1,221 +0,0 @@ -============================================================ -PR Review Agent - Benchmark Pipeline -============================================================ -Loaded 50 PRs -WarpGrep: ENABLED -Mini eval: 15 PRs (3 per repo) -Reviewing 15 PRs - -Running 15 PRs with parallelism=15 - -[1/15] keycloak PR#37429 (4 golden) -[2/15] keycloak PR#37634 (4 golden) -[3/15] keycloak PR#38446 (2 golden) -[4/15] sentry PR#93824 (5 golden) -[5/15] sentry-greptile PR#5 (3 golden) -[6/15] sentry-greptile PR#1 (4 golden) -[7/15] grafana PR#97529 (2 golden) -[8/15] grafana PR#103633 (2 golden) -[9/15] grafana PR#94942 (2 golden) -[10/15] discourse-graphite PR#9 (2 golden) -[11/15] discourse-graphite PR#10 (4 golden) -[12/15] discourse-graphite PR#7 (3 golden) -[13/15] cal.com PR#22532 (2 golden) -[14/15] cal.com PR#8330 (2 golden) -[15/15] cal.com PR#14943 (2 golden) - [10/15] discourse-graphite PR#9 6 files, 31 added lines - [15/15] cal.com PR#14943 3 files, 38 added lines - [6/15] sentry-greptile PR#1 3 files, 128 added lines - [7/15] grafana PR#97529 5 files, 25 added lines - [8/15] grafana PR#103633 4 files, 240 added lines - [13/15] cal.com PR#22532 17 files, 379 added lines - [3/15] keycloak PR#38446 8 files, 256 added lines - [14/15] cal.com PR#8330 4 files, 111 added lines - [12/15] discourse-graphite PR#7 32 files, 115 added lines - [11/15] discourse-graphite PR#10 36 files, 449 added lines - [9/15] grafana PR#94942 4 files, 41 added lines - [2/15] keycloak PR#37634 28 files, 722 added lines - [1/15] keycloak PR#37429 48 files, 343 added lines - [5/15] sentry-greptile PR#5 105 files, 2312 added lines - [4/15] sentry PR#93824 6 files, 199 added lines - WarpGrep: How does the delete logic for WorkflowReminder interact with the OR condition an - WarpGrep: SpanFlusher main method - how does it flush segments from buffer - WarpGrep: How does checkIfIsAvailable work and who calls it in the slots router - WarpGrep: How is enableSqlExpressions used and what does the FlagSqlExpressions feature fl - WarpGrep: Who calls the Context copy constructor of OAuth2GrantType.Context that was remov - WarpGrep: Does Django QuerySet support negative indexing or negative slicing? - WarpGrep: How is the bleveBackend cache protected by cacheMu? What are all readers and wri - WarpGrep: How does the Check method in the RBAC service work, including permission checkin - WarpGrep: How is I18n.fallbacks used throughout the codebase? What code depends on fallbac - WarpGrep: How are fabricators loaded in the spec test suite? Where are category fabricator - WarpGrep: How does getFederatedCredentialsStream work for UserModel credential manager? Wh - WarpGrep: How is getSlots called and what parameters does it accept for date overrides and - WarpGrep: How does Prisma updateMany behave when given an empty data object? Does @updated - WarpGrep: How does the VerifyMessageProperties class resolve English file from community f - WarpGrep: fetch_error_details function using zip with dict.values() for nodestore events - WarpGrep: How is BasePaginator.get_result used and what does offset represent in cursor pa - WarpGrep: SpansBuffer constructor and assigned_shards property - WarpGrep: santizeAnchors method logic for removing anchor tags from translated values - WarpGrep: SelectedCalendarRepository updateManyByCredentialId callers and usage - WarpGrep: Definition of the dark-light-choose SCSS function/mixin - WarpGrep: JavaScript String replace method replacing only first occurrence vs replaceAll - WarpGrep: Who calls QueryTypeSQL and how is the SQL expression query type handled? - WarpGrep: Who calls NewResourceServer and how do callers handle the error? Is Init called - WarpGrep: How is getShouldUseLightweightToken called across the codebase? - WarpGrep: PRCommentWorkflow base class get_merged_pr_single_issue_template and format_comm - WarpGrep: What is the asymmetric caching behavior between denial cache and permission cach - WarpGrep: How does self.paginate pass extra kwargs to paginator_cls constructor in Sentry' - WarpGrep: ProcessSpansStrategyFactory create_with_partitions how partitions map to shards - WarpGrep: How does updateCredential work on user credential manager? What does it return - - WarpGrep: connectedCalendars type definition and cacheUpdatedAt property - WarpGrep: How does the translate_accelerator freedom patch handle locale loading and threa - WarpGrep: FlexCenter styled component with overflowEllipsis in eventAttachments - WarpGrep: How is the permDenialCache invalidated or expired in the RBAC authorization serv - WarpGrep: How does server.Init work with sync.Once and what happens if it's called concurr - WarpGrep: deleteCache handler and calendar cache deletion with credential ownership check - WarpGrep: RecoveryAuthnCodesCredentialModel createFromValues method signature and paramete - WarpGrep: Where are WorkflowReminder records deleted based on retryCount or method in work - WarpGrep: TraceWaterfall FlexBox styled component with height 100% and flex-grow - WarpGrep: How is ReadQuery for QueryTypeSQL called and where does the reader.go SQL case p - WarpGrep: Where is set_locale called in application_controller and what does it return? - WarpGrep: RecoveryAuthnCodesCredentialModel createFromCredentialModel behavior when creden - WarpGrep: multiprocessing.get_context spawn Process isinstance check - is SpawnProcess a s - WarpGrep: How is the permission denial cache cleared or invalidated when permissions chang - WarpGrep connection error, retrying in 3s (attempt 1/3): SSLError - WarpGrep: sed -i platform compatibility macOS vs Linux in shell scripts - WarpGrep: What does searchSupport.build do and who calls it? How does it interact with ble - WarpGrep: nodestore backend get_multi return value ordering guarantee - WarpGrep: How does the availabilityCheckProps get spread into checkIfIsAvailable in slots - WarpGrep: How is CredentialInputUpdater.getCredentials implemented for BackwardsCompatibil - WarpGrep: nodestore get_multi return type and ordering - does it return ordered dict? - WarpGrep: RecoveryAuthnCodesFormAuthenticator authenticate method - how does it check if u - WarpGrep: dayjs object comparison with === operator in TypeScript, when should isSame be u - WarpGrep: Prisma updateMany with empty data object - does it update @updatedAt field autom - WarpGrep: How do grants set the GRANT_TYPE attribute on clientSessionCtx, especially Clien - WarpGrep: How is the cursor string parsed from the request query parameter into a Cursor o - WarpGrep: How does scheduleEmailReminders handle deletion of old or past WorkflowReminders - WarpGrep: How is the schedule object with dateOverrides and workingHours passed to checkIf - WarpGrep: What is organization_context parameter in ControlSiloOrganizationEndpoint get me - WarpGrep connection error, retrying in 3s (attempt 1/3): SSLError - WarpGrep: What happens when Prisma updateMany is called with empty data object - does @upd - WarpGrep: Flex component definition in core layout - what CSS properties does it support - WarpGrep: checkPermission function implementation that checks if a user has permission bas - WarpGrep: How does server.Init use sync.Once and what does initWatcher do? - WarpGrep: How does checkIfIsAvailable handle busy time checking in original code before da - WarpGrep: Attributes section filtering hidden attributes in EAP span details - WarpGrep: How does searchSupport init work and what does it do with the watcher events dur - WarpGrep: RecoveryAuthnCodesCredentialProvider isValid method - how does it validate recov - WarpGrep connection error, retrying in 3s (attempt 1/3): SSLError - WarpGrep: How does RecoveryAuthnCodesAction.processAction build the credentialModel and ge - WarpGrep: _hydrateEmbedded implementation in store service for handling embedded records w - WarpGrep: UserCredentialModel constructor with three parameters: id, type, challengeRespon - WarpGrep: How does Discourse set the I18n locale in controllers? Is set_locale a before_ac - WarpGrep: How are email workflow reminders retried and what retryCount values can they rea - WarpGrep: How does the connectedCalendars handler import CalendarCacheRepository and where - WarpGrep: How does the embed controller ensure_embeddable method work with EmbeddableHost. - WarpGrep: How is slotEndTime computed from time and eventLength in checkIfIsAvailable - Loop done round=8 (tools: read_file=8, codebase_search=4, grep=8, list_directory=1) - WarpGrep: Who calls bleveBackend.BuildIndex and can it be called concurrently for the same - Review complete: 1 issues - [15/15] cal.com PR#14943 1 raw -> 1 kept - [15/15] cal.com PR#14943 [0.92] logic_error: The `deleteMany` OR condition's second branch `{ retryCount: { gt: 1 } }` lacks a `method: - Loop done round=10 (tools: codebase_search=3, grep=7, read_file=16) - WarpGrep: dashboard widget chart tableResultComponent and organization prop passing - Review complete: 2 issues - [2/15] keycloak PR#37634 2 raw -> 2 kept - [2/15] keycloak PR#37634 [0.97] null_reference: Copy-paste error: the second Objects.requireNonNull checks `grantType` again instead of `r - [2/15] keycloak PR#37634 [0.97] test_correctness: The `isAccessTokenId` matcher has two independent bugs: (1) Wrong substring indices — `sub - WarpGrep: What happens when dateOverrideExist is true in checkIfIsAvailable, does it skip - WarpGrep: Where is SiteSetting.default_locale defined and what type does it return - strin - WarpGrep: TableWidgetVisualization component passing empty data and columns when use-table - Loop done round=13 (tools: codebase_search=3, grep=12, read_file=10, glob=2) - Review complete: 1 issues - [9/15] grafana PR#94942 1 raw -> 1 kept - [9/15] grafana PR#94942 [0.95] logic_error: The `enableSqlExpressions` function always returns `false` regardless of the feature flag - Loop done round=10 (tools: grep=3, codebase_search=1, read_file=7, glob=4, bash=3) - Review complete: 1 issues - [12/15] discourse-graphite PR#7 1 raw -> 1 kept - [12/15] discourse-graphite PR#7 [0.95] incorrect_value: Swapped `dark-light-choose` arguments. The light theme value was changed from 30% to 70% l - WarpGrep: How does nodestore get_multi handle missing keys? Does it return them as None or - Loop done round=13 (tools: codebase_search=8, grep=11, read_file=6, glob=1) - Review complete: 3 issues - [14/15] cal.com PR#8330 3 raw -> 3 kept - [14/15] cal.com PR#8330 [0.97] api_misuse: Object reference comparison with `===` on dayjs objects always evaluates to `false`. `dayj - [14/15] cal.com PR#8330 [0.97] incorrect_value: `end` variable is incorrectly computed from `slotStartTime` instead of `slotEndTime`. Both - [14/15] cal.com PR#8330 [0.92] logic_error: When `dateOverrideExist` is true, the function returns `true` (available) immediately with - WarpGrep: get_slow_conditions_for_groups function definition - WarpGrep: fire_actions_for_groups function in delayed_workflow with event_data parameter - WarpGrep: How does the connectedCalendars handler currently access ctx.user and ctx.prisma - WarpGrep: Where is has_global_access defined on organization member or access objects? - WarpGrep: SetupRecoveryAuthnCodesPage getRecoveryAuthnCodes method implementation - WarpGrep: EnterRecoveryAuthnCodePage enterRecoveryAuthnCode and getRecoveryAuthnCodeToEnte - WarpGrep: How does the Discourse store _hydrate handle _ids fields for has-many relationsh - WarpGrep: run_top_events_timeseries_query function signature with equations parameter in r - WarpGrep: Where is the before_validation in EmbeddableHost and can host be nil when sub! i - Loop done round=11 (tools: codebase_search=6, grep=11, read_file=11, bash=1) - WarpGrep: Where is SiteSetting.default_locale called? Does it return a String or Symbol? - Review complete: 2 issues - [6/15] sentry-greptile PR#1 2 raw -> 2 kept - [6/15] sentry-greptile PR#1 [0.95] api_misuse: Django QuerySet does not support negative indexing — it raises `AssertionError: Negative i - [6/15] sentry-greptile PR#1 [0.60] null_reference: `organization_context.member` can be `None` (typed as `RpcOrganizationMember | None`). In - Loop done round=12 (tools: codebase_search=2, grep=21, read_file=6, glob=1) - Review complete: 2 issues - [1/15] keycloak PR#37429 2 raw -> 2 kept - [1/15] keycloak PR#37429 [0.97] localization: The `loginTotpStep1` key contains Italian text ("Installa una delle seguenti applicazioni - [1/15] keycloak PR#37429 [0.96] localization: The `totpStep1` key uses Traditional Chinese characters (手機, 安裝, 應用程式) in the zh_CN (Simpl - Loop done round=11 (tools: codebase_search=4, grep=10, read_file=8, bash=5) - WarpGrep: RecoveryAuthnCodesCredentialProviderFactory class and its PROVIDER_ID constant - Review complete: 2 issues - [4/15] sentry PR#93824 2 raw -> 2 kept - [4/15] sentry PR#93824 [0.95] type_error: isinstance(process, multiprocessing.Process) is always False for SpawnProcess objects crea - [4/15] sentry PR#93824 [0.92] incorrect_value: The wait_produce timer uses tag key "shards" (plural) instead of "shard" (singular) which - WarpGrep: What is the I18n::Backend::Fallbacks translate method's contract for fallback lo - WarpGrep: How is I18n.backend initialized and replaced during application boot? What class - WarpGrep: How does checkPermission use the scope map to determine if access is allowed for - Loop done round=25 (tools: codebase_search=6, grep=15, read_file=18, glob=6, bash=2) - Review complete: 4 issues - [11/15] discourse-graphite PR#10 4 raw -> 4 kept - [11/15] discourse-graphite PR#10 [0.95] incorrect_value: The contents of category_fabricator.rb and embeddable_host_fabricator.rb are swapped. cate - [11/15] discourse-graphite PR#10 [0.95] logic_error: The update action is identical to the show action — it just renders the serialized embeddi - [11/15] discourse-graphite PR#10 [0.90] logic_error: The _hydrateEmbedded method's plural _ids branch unconditionally deletes the original IDs - [11/15] discourse-graphite PR#10 [0.70] api_misuse: String.prototype.replace() with a string first argument only replaces the first occurrence - Loop done round=30 (tools: codebase_search=14, read_file=24, grep=15, bash=2, glob=1) - Review complete: 2 issues - [5/15] sentry-greptile PR#5 2 raw -> 2 kept - [5/15] sentry-greptile PR#5 [0.85] logic_error: In `fetch_error_details`, `zip(error_ids, events.values())` assumes dict ordering matches - [5/15] sentry-greptile PR#5 [0.70] logic_error: In `BrowserReportSerializer`, `validate_timestamp` and `validate_age` use `self.initial_da - WarpGrep: fakeStore implementation in the RBAC service tests including GetUserPermissions - WarpGrep: Where is SelectedCalendar.updatedAt used or read after being set by updateManyBy - WarpGrep: How does RecoveryAuthnCodesCredentialProvider.isValid update the credential afte - WarpGrep: What is the type field on connectedCalendar.integration - is it 'type' or 'slug' - Loop done round=12 (tools: codebase_search=8, read_file=11, grep=14, list_directory=1) - Review complete: 1 issues - [10/15] discourse-graphite PR#9 1 raw -> 1 kept - [10/15] discourse-graphite PR#9 [0.50] type_error: The `ensure_loaded!` method checks `@loaded_locales.include?(locale)` without first conver - Loop done round=10 (tools: codebase_search=7, read_file=10, grep=6) - Review complete: 2 issues - [8/15] grafana PR#103633 2 raw -> 2 kept - [8/15] grafana PR#103633 [0.82] test_correctness: The comment says "Allow access to the dashboard to prove this is not checked" but the perm - [8/15] grafana PR#103633 [0.55] security: userPermDenialCacheKey concatenates `name` and `parent` using `_` as the separator. Since - Loop done round=34 (tools: codebase_search=11, grep=32, read_file=26, glob=8, bash=2) - Review complete: 2 issues - [13/15] cal.com PR#22532 2 raw -> 2 kept - [13/15] cal.com PR#22532 [0.80] portability: `sed -i '' -E "s|...|...|"` uses BSD/macOS-specific syntax. On GNU/Linux, `sed` treats `'' - [13/15] cal.com PR#22532 [0.50] api_misuse: ORM `updateMany` is called with an empty `data: {}` object to trigger `@updatedAt` auto-ti - Loop done round=25 (tools: codebase_search=7, read_file=22, grep=21, bash=2) - Review complete: 1 issues - [7/15] grafana PR#97529 1 raw -> 1 kept - [7/15] grafana PR#97529 [0.85] logic_error: When Init() is called eagerly in NewResourceServer and partially fails (e.g., s.search.ini - WarpGrep: UserCredentialStore removeStoredCredential implementation, what happens when cre - Loop done round=13 (tools: codebase_search=14, grep=9, read_file=11) - Review complete: 2 issues - [3/15] keycloak PR#38446 2 raw -> 2 kept - [3/15] keycloak PR#38446 [0.82] null_reference: getCredentials() creates a RecoveryAuthnCodesCredentialModel via createFromValues() withou - [3/15] keycloak PR#38446 [0.72] test_correctness: isValid() for recovery codes validates against ANY stored code using stream().anyMatch() i -Wrote candidates to /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/candidates.json - -============================================================ -DONE: 15 reviewed, 28 raw -> 28 filtered -Avg/PR: 1.9, Time: 884s -Candidates: /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/candidates.json -Benchmark data updated: /Users/tejas/personal/applymodel/zmisc/examples/code-review-benchmark/offline/results/benchmark_data.json diff --git a/pr_review_agent/output/iter9_sentry93824.log b/pr_review_agent/output/iter9_sentry93824.log deleted file mode 100644 index edc43f3..0000000 --- a/pr_review_agent/output/iter9_sentry93824.log +++ /dev/null @@ -1,27 +0,0 @@ -============================================================ -PR Review Agent - Benchmark Pipeline -============================================================ -Loaded 50 PRs -WarpGrep: ENABLED -Reviewing 1 PRs - -Running 1 PRs with parallelism=1 - -[1/1] sentry PR#93824 (5 golden) - [1/1] sentry PR#93824 6 files, 199 added lines - WarpGrep: SpanFlusher class and how it's instantiated with buffer and shards - WarpGrep: SpansBuffer constructor and assigned_shards property - WarpGrep: Does SpawnProcess from multiprocessing.get_context('spawn') inherit from multipr - WarpGrep: run_with_initialized_sentry how does it work with partial and pickled arguments - Loop done round=11 (tools: codebase_search=4, grep=8, read_file=8, bash=3) - Review complete: 2 issues - [1/1] sentry PR#93824 2 raw -> 2 kept - [1/1] sentry PR#93824 [0.95] type_error: `isinstance(process, multiprocessing.Process)` is always `False` for processes created via - [1/1] sentry PR#93824 [0.92] incorrect_value: The metric `spans.buffer.flusher.wait_produce` uses tag key `"shards"` (plural) while all -Wrote candidates to /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/candidates.json - -============================================================ -DONE: 1 reviewed, 2 raw -> 2 filtered -Avg/PR: 2.0, Time: 270s -Candidates: /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/candidates.json -Benchmark data updated: /Users/tejas/personal/applymodel/zmisc/examples/code-review-benchmark/offline/results/benchmark_data.json diff --git a/pr_review_agent/output/iter9_sentrygreptile1.log b/pr_review_agent/output/iter9_sentrygreptile1.log deleted file mode 100644 index f0d0780..0000000 --- a/pr_review_agent/output/iter9_sentrygreptile1.log +++ /dev/null @@ -1,28 +0,0 @@ -============================================================ -PR Review Agent - Benchmark Pipeline -============================================================ -Loaded 50 PRs -WarpGrep: ENABLED -Reviewing 1 PRs - -Running 1 PRs with parallelism=1 - -[1/1] sentry-greptile PR#1 (4 golden) - [1/1] sentry-greptile PR#1 3 files, 128 added lines - WarpGrep: Does Django QuerySet support negative indexing or negative slicing? - WarpGrep: How is BasePaginator.get_result called and what does the offset parameter contro - WarpGrep: ControlSiloOrganizationEndpoint get method with organization_context parameter - WarpGrep: How does the paginate method on API endpoints pass extra kwargs to the paginator - Loop done round=19 (tools: codebase_search=4, grep=15, read_file=12, bash=4) - Review complete: 3 issues - [1/1] sentry-greptile PR#1 3 raw -> 3 kept - [1/1] sentry-greptile PR#1 [0.97] api_misuse: When `cursor.is_prev` is True, the offset is not clamped to 0, allowing negative values fr - [1/1] sentry-greptile PR#1 [0.97] api_misuse: In `OptimizedCursorPaginator.get_result`, when `enable_advanced_features` is True and `cur - [1/1] sentry-greptile PR#1 [0.85] null_reference: `organization_context.member` can be `None` (as documented in the model definition), but t -Wrote candidates to /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/candidates.json - -============================================================ -DONE: 1 reviewed, 3 raw -> 3 filtered -Avg/PR: 3.0, Time: 219s -Candidates: /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/candidates.json -Benchmark data updated: /Users/tejas/personal/applymodel/zmisc/examples/code-review-benchmark/offline/results/benchmark_data.json diff --git a/pr_review_agent/output/traces/cal.com_pr14740_trace.json b/pr_review_agent/output/traces/cal.com_pr14740_trace.json deleted file mode 100644 index 63ad5d3..0000000 --- a/pr_review_agent/output/traces/cal.com_pr14740_trace.json +++ /dev/null @@ -1,290 +0,0 @@ -[ - { - "round": 0, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "isTeamAdmin and isTeamOwner usage pattern for authorization checks" - }, - "output_len": 16016, - "is_error": false - }, - { - "round": 0, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "How is the editLocation handler checking permissions for booking modification" - }, - "output_len": 23393, - "is_error": false - }, - { - "round": 1, - "tool": "grep", - "input": { - "pattern": "isTeamAdmin", - "path": "packages/lib/server/queries/teams" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 1, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "isTeamAdmin function definition that checks admin or owner role" - }, - "output_len": 116, - "is_error": false - }, - { - "round": 1, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "EventManager updateCalendarAttendees method definition" - }, - "output_len": 2454, - "is_error": false - }, - { - "round": 2, - "tool": "grep", - "input": { - "pattern": "export async function isTeamAdmin" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 2, - "tool": "grep", - "input": { - "pattern": "export async function isTeamOwner" - }, - "output_len": 452, - "is_error": false - }, - { - "round": 3, - "tool": "grep", - "input": { - "pattern": "isTeamAdmin", - "path": "packages/features/ee/teams/lib/queries.ts" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 3, - "tool": "grep", - "input": { - "pattern": "export.*isTeamAdmin" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 4, - "tool": "grep", - "input": { - "pattern": "isTeamAdmin", - "glob": "*.ts" - }, - "output_len": 1681, - "is_error": false - }, - { - "round": 5, - "tool": "grep", - "input": { - "pattern": "from.*isTeamAdmin", - "path": "packages/trpc/server/routers/viewer/bookings/addGuests.handler.ts" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 6, - "tool": "read_file", - "input": { - "path": "packages/trpc/server/routers/viewer/bookings/addGuests.handler.ts" - }, - "output_len": 15433, - "is_error": false - }, - { - "round": 7, - "tool": "grep", - "input": { - "pattern": "isTeamAdmin", - "path": "packages/lib/server/queries" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 7, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "isTeamAdmin function definition checking ADMIN or OWNER role in server/queries/teams" - }, - "output_len": 72, - "is_error": false - }, - { - "round": 8, - "tool": "glob", - "input": { - "pattern": "**/queries/teams/**" - }, - "output_len": 17, - "is_error": false - }, - { - "round": 9, - "tool": "grep", - "input": { - "pattern": "export.*isTeamAdmin", - "glob": "*.ts" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 10, - "tool": "grep", - "input": { - "pattern": "isTeamAdmin", - "glob": "*.ts" - }, - "output_len": 1681, - "is_error": false - }, - { - "round": 11, - "tool": "grep", - "input": { - "pattern": "function isTeamAdmin", - "glob": "*.ts" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 12, - "tool": "read_file", - "input": { - "path": "packages/features/ee/teams/lib/queries.ts", - "lines": "380-420" - }, - "output_len": 1291, - "is_error": false - }, - { - "round": 13, - "tool": "grep", - "input": { - "pattern": "isTeamAdmin", - "glob": "*.{ts,tsx}" - }, - "output_len": 1681, - "is_error": false - }, - { - "round": 14, - "tool": "glob", - "input": { - "pattern": "packages/lib/server/queries/teams/*" - }, - "output_len": 17, - "is_error": false - }, - { - "round": 15, - "tool": "glob", - "input": { - "pattern": "packages/lib/server/queries/**" - }, - "output_len": 17, - "is_error": false - }, - { - "round": 16, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "sendLocationChangeEmails function and how it receives modified emails list vs original input" - }, - "output_len": 1768, - "is_error": false - }, - { - "round": 16, - "tool": "grep", - "input": { - "pattern": "AttendeeScheduledEmail", - "path": "packages/emails", - "glob": "*.ts" - }, - "output_len": 18717, - "is_error": false - }, - { - "round": 17, - "tool": "read_file", - "input": { - "path": "packages/emails/email-manager.ts", - "lines": "695-720" - }, - "output_len": 1009, - "is_error": false - }, - { - "round": 18, - "tool": "read_file", - "input": { - "path": "packages/emails/templates/attendee-add-guests-email.ts" - }, - "output_len": 1315, - "is_error": false - }, - { - "round": 19, - "tool": "glob", - "input": { - "pattern": "**/server/queries/**" - }, - "output_len": 17, - "is_error": false - }, - { - "round": 19, - "tool": "grep", - "input": { - "pattern": "isTeamAdmin|isTeamOwner", - "path": "packages/lib", - "glob": "*.ts" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 20, - "tool": "grep", - "input": { - "pattern": "isTeamOwner|isTeamAdmin", - "path": "packages", - "glob": "**/index.ts" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 21, - "tool": "grep", - "input": { - "pattern": "@calcom/lib/server/queries/teams", - "glob": "*.ts" - }, - "output_len": 10, - "is_error": false - } -] \ No newline at end of file diff --git a/pr_review_agent/output/traces/cal.com_pr14943_trace.json b/pr_review_agent/output/traces/cal.com_pr14943_trace.json deleted file mode 100644 index 37c1e64..0000000 --- a/pr_review_agent/output/traces/cal.com_pr14943_trace.json +++ /dev/null @@ -1,115 +0,0 @@ -[ - { - "round": 0, - "tool": "read_file", - "input": { - "path": "packages/features/ee/workflows/api/scheduleSMSReminders.ts" - }, - "output_len": 11097, - "is_error": false - }, - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "How is the deleteMany for workflowReminder used and what conditions protect against deleting reminders that haven't been retried yet?" - }, - "output_len": 3776, - "is_error": false - }, - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "What is the PartialWorkflowReminder type definition and what fields does it include?" - }, - "output_len": 375, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "packages/features/ee/workflows/lib/getWorkflowReminders.ts" - }, - "output_len": 7391, - "is_error": false - }, - { - "round": 1, - "tool": "codebase_search", - "input": { - "search_string": "scheduleEmailReminders handler that uses deleteMany and workflowReminder with retryCount" - }, - "output_len": 368, - "is_error": false - }, - { - "round": 1, - "tool": "grep", - "input": { - "pattern": "retryCount", - "glob": "*.ts" - }, - "output_len": 10097, - "is_error": false - }, - { - "round": 2, - "tool": "read_file", - "input": { - "path": "packages/features/ee/workflows/api/scheduleWhatsappReminders.ts" - }, - "output_len": 7266, - "is_error": false - }, - { - "round": 2, - "tool": "read_file", - "input": { - "path": "packages/features/ee/workflows/api/scheduleEmailReminders.ts" - }, - "output_len": 23338, - "is_error": false - }, - { - "round": 2, - "tool": "grep", - "input": { - "pattern": "deleteMany.*workflowReminder|workflowReminder\\.deleteMany", - "glob": "*.ts" - }, - "output_len": 468, - "is_error": false - }, - { - "round": 3, - "tool": "grep", - "input": { - "pattern": "method.*WorkflowMethods|WorkflowMethods\\.\\w+", - "path": "packages/features/ee/workflows/api" - }, - "output_len": 934, - "is_error": false - }, - { - "round": 3, - "tool": "codebase_search", - "input": { - "search_string": "Prisma increment operator for atomic counter updates, retryCount increment" - }, - "output_len": 111, - "is_error": false - }, - { - "round": 4, - "tool": "grep", - "input": { - "pattern": "\\bincrement:\\s*1\\b", - "glob": "*.ts", - "path": "packages/features/ee" - }, - "output_len": 503, - "is_error": false - } -] \ No newline at end of file diff --git a/pr_review_agent/output/traces/cal.com_pr22532_trace.json b/pr_review_agent/output/traces/cal.com_pr22532_trace.json deleted file mode 100644 index 28c6464..0000000 --- a/pr_review_agent/output/traces/cal.com_pr22532_trace.json +++ /dev/null @@ -1,405 +0,0 @@ -[ - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "How does Prisma updateMany behave when data is an empty object? Does it still trigger @updatedAt?" - }, - "output_len": 1549, - "is_error": false - }, - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "SelectedCalendarRepository.updateManyByCredentialId callers and usage" - }, - "output_len": 486, - "is_error": false - }, - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "How is connectedCalendar.cacheUpdatedAt used in the frontend components" - }, - "output_len": 2234, - "is_error": false - }, - { - "round": 0, - "tool": "grep", - "input": { - "pattern": "cacheUpdatedAt" - }, - "output_len": 2028, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "packages/trpc/server/routers/viewer/calendars/connectedCalendars.handler.ts" - }, - "output_len": 1791, - "is_error": false - }, - { - "round": 1, - "tool": "codebase_search", - "input": { - "search_string": "How does Prisma @updatedAt work with updateMany when data is empty object" - }, - "output_len": 5161, - "is_error": false - }, - { - "round": 1, - "tool": "grep", - "input": { - "pattern": "updateManyByCredentialId" - }, - "output_len": 604, - "is_error": false - }, - { - "round": 1, - "tool": "grep", - "input": { - "pattern": "deleteCacheHandler|deleteCache\\.handler" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 2, - "tool": "grep", - "input": { - "pattern": "model SelectedCalendar", - "glob": "*.prisma" - }, - "output_len": 379, - "is_error": false - }, - { - "round": 2, - "tool": "codebase_search", - "input": { - "search_string": "SelectedCalendar model definition in Prisma schema with updatedAt field" - }, - "output_len": 3689, - "is_error": false - }, - { - "round": 2, - "tool": "grep", - "input": { - "pattern": "sed -i ''", - "glob": "*.sh" - }, - "output_len": 439, - "is_error": false - }, - { - "round": 2, - "tool": "codebase_search", - "input": { - "search_string": "How does the CalendarService fetchAvailabilityAndSetCache method use credential id" - }, - "output_len": 965, - "is_error": false - }, - { - "round": 3, - "tool": "read_file", - "input": { - "path": "packages/app-store/googlecalendar/lib/CalendarService.ts", - "lines": "1010-1030" - }, - "output_len": 0, - "is_error": false - }, - { - "round": 3, - "tool": "grep", - "input": { - "pattern": "import.*SelectedCalendarRepository", - "path": "packages/app-store/googlecalendar/lib/CalendarService.ts" - }, - "output_len": 306, - "is_error": false - }, - { - "round": 3, - "tool": "codebase_search", - "input": { - "search_string": "Does Prisma updateMany support @updatedAt automatic update or is it only for update and upsert" - }, - "output_len": 86110, - "is_error": false - }, - { - "round": 3, - "tool": "grep", - "input": { - "pattern": "deleteCache", - "path": "packages/trpc/server/routers/viewer/calendars" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 4, - "tool": "bash", - "input": { - "command": "cd /Users/tejas/personal/applymodel/zmisc/examples/pr_clones/cal.com && cat package.json | grep -i prisma | head -10" - }, - "output_len": 249, - "is_error": false - }, - { - "round": 4, - "tool": "grep", - "input": { - "pattern": "\"prisma\"", - "glob": "packages/prisma/package.json" - }, - "output_len": 735, - "is_error": false - }, - { - "round": 4, - "tool": "codebase_search", - "input": { - "search_string": "How does the deleteCache handler verify the credential belongs to the user for authorization check" - }, - "output_len": 309, - "is_error": false - }, - { - "round": 4, - "tool": "grep", - "input": { - "pattern": "connectedCalendarsHandler" - }, - "output_len": 1405, - "is_error": false - }, - { - "round": 5, - "tool": "codebase_search", - "input": { - "search_string": "How is CalendarCacheRepository instantiated throughout the codebase? Is there a DI container or factory?" - }, - "output_len": 111, - "is_error": false - }, - { - "round": 5, - "tool": "grep", - "input": { - "pattern": "new CalendarCacheRepository" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 5, - "tool": "grep", - "input": { - "pattern": "CalendarCacheRepository.*import|import.*CalendarCacheRepository", - "glob": "*.ts" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 6, - "tool": "grep", - "input": { - "pattern": "CalendarCacheRepository" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 7, - "tool": "grep", - "input": { - "pattern": "CalendarCache", - "path": "packages/features/calendar-cache" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 7, - "tool": "list_directory", - "input": { - "path": "packages/features/calendar-cache" - }, - "output_len": 60, - "is_error": false - }, - { - "round": 8, - "tool": "glob", - "input": { - "pattern": "**/calendar-cache*" - }, - "output_len": 17, - "is_error": false - }, - { - "round": 9, - "tool": "grep", - "input": { - "pattern": "calendar.cache", - "glob": "*.ts" - }, - "output_len": 13074, - "is_error": false - }, - { - "round": 9, - "tool": "grep", - "input": { - "pattern": "calendarCache", - "path": "packages/features" - }, - "output_len": 21764, - "is_error": false - }, - { - "round": 9, - "tool": "glob", - "input": { - "pattern": "**/calendarCache*\"" - }, - "output_len": 17, - "is_error": false - }, - { - "round": 10, - "tool": "read_file", - "input": { - "path": "packages/features/apps/components/CredentialActionsDropdown.tsx" - }, - "output_len": 86, - "is_error": false - }, - { - "round": 10, - "tool": "read_file", - "input": { - "path": "packages/trpc/server/routers/viewer/calendars/deleteCache.handler.ts" - }, - "output_len": 91, - "is_error": false - }, - { - "round": 11, - "tool": "codebase_search", - "input": { - "search_string": "Does Prisma updateMany support @updatedAt automatic field update or does it need explicit data" - }, - "output_len": 86110, - "is_error": false - }, - { - "round": 11, - "tool": "grep", - "input": { - "pattern": "updateMany.*data:\\s*\\{\\s*\\}" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 11, - "tool": "read_file", - "input": { - "path": "packages/app-store/googlecalendar/lib/CalendarService.ts", - "lines": "1000-1040" - }, - "output_len": 0, - "is_error": false - }, - { - "round": 12, - "tool": "bash", - "input": { - "command": "cd /Users/tejas/personal/applymodel/zmisc/examples/pr_clones/cal.com && wc -l packages/app-store/googlecalendar/lib/CalendarService.ts" - }, - "output_len": 60, - "is_error": false - }, - { - "round": 12, - "tool": "grep", - "input": { - "pattern": "fetchAvailabilityAndSetCache", - "path": "packages/app-store/googlecalendar/lib/CalendarService.ts" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 13, - "tool": "grep", - "input": { - "pattern": "@updatedAt", - "path": "packages/prisma/schema.prisma" - }, - "output_len": 6753, - "is_error": false - }, - { - "round": 13, - "tool": "read_file", - "input": { - "path": "packages/prisma/migrations/20250715160635_add_calendar_cache_updated_at/migration.sql" - }, - "output_len": 353, - "is_error": false - }, - { - "round": 13, - "tool": "read_file", - "input": { - "path": "packages/features/calendar-cache/calendar-cache.repository.ts", - "lines": "169-190" - }, - "output_len": 84, - "is_error": false - }, - { - "round": 14, - "tool": "grep", - "input": { - "pattern": "SelectedCalendarUpdateManyMutationInput|SelectedCalendarUpdateInput" - }, - "output_len": 5754, - "is_error": false - }, - { - "round": 14, - "tool": "read_file", - "input": { - "path": "packages/lib/server/repository/selectedCalendar.ts", - "lines": "390-415" - }, - "output_len": 73, - "is_error": false - }, - { - "round": 15, - "tool": "read_file", - "input": { - "path": "packages/features/selectedCalendar/repositories/SelectedCalendarRepository.ts", - "lines": "465-490" - }, - "output_len": 798, - "is_error": false - } -] \ No newline at end of file diff --git a/pr_review_agent/output/traces/cal.com_pr8087_trace.json b/pr_review_agent/output/traces/cal.com_pr8087_trace.json deleted file mode 100644 index 41ba8d6..0000000 --- a/pr_review_agent/output/traces/cal.com_pr8087_trace.json +++ /dev/null @@ -1,348 +0,0 @@ -[ - { - "round": 0, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "forEach with async callback that uses await inside - find all instances where forEach is used with async in the changed files" - }, - "output_len": 1085, - "is_error": false - }, - { - "round": 0, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "getCalendarCredentials function and its callers - how is the calendar property used after getCalendarCredentials returns" - }, - "output_len": 16482, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "packages/trpc/server/routers/viewer/bookings.tsx", - "lines": "545-575" - }, - "output_len": 71, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "packages/app-store/vital/lib/reschedule.ts", - "lines": "115-145" - }, - "output_len": 1463, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "packages/app-store/wipemycalother/lib/reschedule.ts", - "lines": "115-145" - }, - "output_len": 1531, - "is_error": false - }, - { - "round": 2, - "tool": "read_file", - "input": { - "path": "packages/app-store/vital/lib/reschedule.ts", - "lines": "145-175" - }, - "output_len": 1186, - "is_error": false - }, - { - "round": 2, - "tool": "read_file", - "input": { - "path": "packages/app-store/wipemycalother/lib/reschedule.ts", - "lines": "145-175" - }, - "output_len": 1089, - "is_error": false - }, - { - "round": 3, - "tool": "read_file", - "input": { - "path": "packages/core/CalendarManager.ts", - "lines": "20-50" - }, - "output_len": 55, - "is_error": false - }, - { - "round": 3, - "tool": "read_file", - "input": { - "path": "packages/features/bookings/lib/handleCancelBooking.ts", - "lines": "445-485" - }, - "output_len": 1664, - "is_error": false - }, - { - "round": 4, - "tool": "glob", - "input": { - "pattern": "packages/core/CalendarManager.ts" - }, - "output_len": 17, - "is_error": false - }, - { - "round": 4, - "tool": "grep", - "input": { - "pattern": "forEach\\(async", - "glob": "packages/features/bookings/lib/handleCancelBooking.ts" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 5, - "tool": "grep", - "input": { - "pattern": "forEach\\(async", - "glob": "packages/trpc/server/routers/viewer/bookings.tsx" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 5, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "bookingRefsFiltered.forEach in bookings router - how are calendar deletion promises handled" - }, - "output_len": 1360, - "is_error": false - }, - { - "round": 6, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "handleCancelBooking forEach async credential calendar - how are recurring event calendar updates handled" - }, - "output_len": 17, - "is_error": false - }, - { - "round": 6, - "tool": "read_file", - "input": { - "path": "packages/features/bookings/lib/handleCancelBooking.ts", - "lines": "1-30" - }, - "output_len": 2180, - "is_error": false - }, - { - "round": 7, - "tool": "grep", - "input": { - "pattern": "forEach\\(async", - "glob": "packages/features/bookings/lib/handleCancelBooking.ts" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 8, - "tool": "glob", - "input": { - "pattern": "**/CalendarManager.ts" - }, - "output_len": 111, - "is_error": false - }, - { - "round": 8, - "tool": "grep", - "input": { - "pattern": "getCalendarCredentials", - "glob": "packages/core/CalendarManager.ts" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 9, - "tool": "read_file", - "input": { - "path": "packages/features/calendars/lib/CalendarManager.ts", - "lines": "80-110" - }, - "output_len": 1199, - "is_error": false - }, - { - "round": 10, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "All callers of getCalendar function from app-store _utils - find every place that calls getCalendar" - }, - "output_len": 111, - "is_error": false - }, - { - "round": 10, - "tool": "grep", - "input": { - "pattern": "getCalendar\\(", - "glob": "packages/core/**" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 11, - "tool": "grep", - "input": { - "pattern": "getCalendar\\(", - "glob": "*.ts" - }, - "output_len": 8498, - "is_error": false - }, - { - "round": 12, - "tool": "read_file", - "input": { - "path": "packages/app-store/_utils/getCalendar.ts" - }, - "output_len": 5371, - "is_error": false - }, - { - "round": 13, - "tool": "grep", - "input": { - "pattern": "appStore\\[", - "glob": "*.ts" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 14, - "tool": "grep", - "input": { - "pattern": "appStore\\[", - "glob": "**/*.ts" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 15, - "tool": "grep", - "input": { - "pattern": "appStore\\[" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 16, - "tool": "grep", - "input": { - "pattern": "appStore\\[\"" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 17, - "tool": "bash", - "input": { - "command": "cd /Users/tejas/personal/applymodel/zmisc/examples/pr_clones/cal.com && grep -rn 'appStore\\[' --include=\"*.ts\" --include=\"*.tsx\" | head -40" - }, - "output_len": 11, - "is_error": false - }, - { - "round": 18, - "tool": "bash", - "input": { - "command": "cd /Users/tejas/personal/applymodel/zmisc/examples/pr_clones/cal.com && grep -rn 'from \".*app-store\"' --include=\"*.ts\" --include=\"*.tsx\" | head -20" - }, - "output_len": 2445, - "is_error": false - }, - { - "round": 19, - "tool": "read_file", - "input": { - "path": "packages/app-store/vital/lib/reschedule.ts", - "lines": "120-165" - }, - "output_len": 1969, - "is_error": false - }, - { - "round": 19, - "tool": "read_file", - "input": { - "path": "packages/app-store/wipemycalother/lib/reschedule.ts", - "lines": "120-165" - }, - "output_len": 1972, - "is_error": false - }, - { - "round": 20, - "tool": "glob", - "input": { - "pattern": "packages/trpc/server/routers/viewer/bookings*" - }, - "output_len": 17, - "is_error": false - }, - { - "round": 21, - "tool": "glob", - "input": { - "pattern": "packages/trpc/server/routers/viewer/bookings/**" - }, - "output_len": 3013, - "is_error": false - }, - { - "round": 22, - "tool": "read_file", - "input": { - "path": "packages/trpc/server/routers/viewer/bookings/requestReschedule.handler.ts", - "lines": "195-240" - }, - "output_len": 1806, - "is_error": false - }, - { - "round": 23, - "tool": "grep", - "input": { - "pattern": "deleteMeeting", - "glob": "packages/app-store/vital/lib/reschedule.ts" - }, - "output_len": 1063, - "is_error": false - }, - { - "round": 23, - "tool": "grep", - "input": { - "pattern": "deleteMeeting", - "glob": "packages/app-store/wipemycalother/lib/reschedule.ts" - }, - "output_len": 1117, - "is_error": false - } -] \ No newline at end of file diff --git a/pr_review_agent/output/traces/cal.com_pr8330_trace.json b/pr_review_agent/output/traces/cal.com_pr8330_trace.json deleted file mode 100644 index 06b4c65..0000000 --- a/pr_review_agent/output/traces/cal.com_pr8330_trace.json +++ /dev/null @@ -1,273 +0,0 @@ -[ - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "How is checkIfIsAvailable called and what parameters does it receive in the slots router" - }, - "output_len": 111, - "is_error": false - }, - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "How does the getSlots function use activeOverrides and dateOverrides for scheduling" - }, - "output_len": 10224, - "is_error": false - }, - { - "round": 0, - "tool": "grep", - "input": { - "pattern": "checkIfIsAvailable", - "glob": "*.ts" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "packages/trpc/server/routers/viewer/slots.ts" - }, - "output_len": 67, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "packages/lib/slots.ts" - }, - "output_len": 44, - "is_error": false - }, - { - "round": 2, - "tool": "glob", - "input": { - "pattern": "**/slots.ts" - }, - "output_len": 77, - "is_error": false - }, - { - "round": 2, - "tool": "glob", - "input": { - "pattern": "**/getSchedule.test.ts" - }, - "output_len": 37, - "is_error": false - }, - { - "round": 3, - "tool": "list_directory", - "input": { - "path": "." - }, - "output_len": 82, - "is_error": false - }, - { - "round": 3, - "tool": "grep", - "input": { - "pattern": "checkIfIsAvailable", - "path": "." - }, - "output_len": 10, - "is_error": false - }, - { - "round": 4, - "tool": "read_file", - "input": { - "path": "packages/features/schedules/lib/slots.ts" - }, - "output_len": 10175, - "is_error": false - }, - { - "round": 4, - "tool": "codebase_search", - "input": { - "search_string": "How are dateOverrides and workingHours used in the checkIfIsAvailable function for slot availability" - }, - "output_len": 6913, - "is_error": false - }, - { - "round": 5, - "tool": "grep", - "input": { - "pattern": "availabilityCheckProps", - "glob": "*.ts" - }, - "output_len": 2751, - "is_error": false - }, - { - "round": 5, - "tool": "codebase_search", - "input": { - "search_string": "How does dayjs equality comparison work with === versus .isSame() for dayjs objects in TypeScript" - }, - "output_len": 2101, - "is_error": false - }, - { - "round": 5, - "tool": "grep", - "input": { - "pattern": "slotEndTime|slotStartTime", - "glob": "**/slots.ts" - }, - "output_len": 13505, - "is_error": false - }, - { - "round": 6, - "tool": "read_file", - "input": { - "path": "packages/trpc/server/routers/viewer/slots/util.ts", - "lines": "1340-1420" - }, - "output_len": 3090, - "is_error": false - }, - { - "round": 6, - "tool": "grep", - "input": { - "pattern": "getSchedule", - "glob": "apps/web/test/lib/getSchedule.test.ts", - "path": "apps/web/test/lib" - }, - "output_len": 3448, - "is_error": false - }, - { - "round": 7, - "tool": "grep", - "input": { - "pattern": "checkForConflicts", - "glob": "*.ts" - }, - "output_len": 21656, - "is_error": false - }, - { - "round": 7, - "tool": "read_file", - "input": { - "path": "apps/web/test/lib/getSchedule.test.ts", - "lines": "750-810" - }, - "output_len": 1869, - "is_error": false - }, - { - "round": 8, - "tool": "read_file", - "input": { - "path": "apps/web/test/lib/getSchedule.test.ts", - "lines": "770-810" - }, - "output_len": 1165, - "is_error": false - }, - { - "round": 8, - "tool": "grep", - "input": { - "pattern": "getSchedule|getAvailableSlots", - "glob": "apps/web/test/lib/getSchedule.test.ts", - "path": "apps/web/test/lib" - }, - "output_len": 27538, - "is_error": false - }, - { - "round": 9, - "tool": "read_file", - "input": { - "path": "apps/web/test/lib/getSchedule.test.ts", - "lines": "1480-1560" - }, - "output_len": 2677, - "is_error": false - }, - { - "round": 9, - "tool": "read_file", - "input": { - "path": "packages/trpc/server/routers/viewer/slots/util.ts", - "lines": "1-50" - }, - "output_len": 3616, - "is_error": false - }, - { - "round": 10, - "tool": "read_file", - "input": { - "path": "packages/features/bookings/lib/conflictChecker/checkForConflicts.ts" - }, - "output_len": 1469, - "is_error": false - }, - { - "round": 11, - "tool": "grep", - "input": { - "pattern": "getSchedule", - "path": "apps/web/test/lib/getSchedule.test.ts" - }, - "output_len": 1056, - "is_error": false - }, - { - "round": 11, - "tool": "read_file", - "input": { - "path": "apps/web/test/lib/getSchedule.test.ts", - "lines": "1-40" - }, - "output_len": 1430, - "is_error": false - }, - { - "round": 12, - "tool": "read_file", - "input": { - "path": "apps/web/test/lib/getSchedule.test.ts", - "lines": "774-810" - }, - "output_len": 1055, - "is_error": false - }, - { - "round": 13, - "tool": "grep", - "input": { - "pattern": "slotEndTime|slotStartTime", - "path": "packages/trpc/server/routers/viewer/slots" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 14, - "tool": "grep", - "input": { - "pattern": "slotEndTime|slotStartTime", - "glob": "**/slots*.ts" - }, - "output_len": 17041, - "is_error": false - } -] \ No newline at end of file diff --git a/pr_review_agent/output/traces/discourse_graphite_pr10_trace.json b/pr_review_agent/output/traces/discourse_graphite_pr10_trace.json deleted file mode 100644 index 7d3e9eb..0000000 --- a/pr_review_agent/output/traces/discourse_graphite_pr10_trace.json +++ /dev/null @@ -1,386 +0,0 @@ -[ - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "How does the REST adapter basePath handle type names with underscores vs hyphens?" - }, - "output_len": 4017, - "is_error": false - }, - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "How does _lookupSubType work in the store and what happens when it returns null or undefined?" - }, - "output_len": 1685, - "is_error": false - }, - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "category_fabricator.rb and embeddable_host_fabricator.rb fabricators for categories and embeddable hosts" - }, - "output_len": 113, - "is_error": false - }, - { - "round": 0, - "tool": "grep", - "input": { - "pattern": "Fabricator\\(:category\\)" - }, - "output_len": 414, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "app/assets/javascripts/discourse/models/store.js.es6" - }, - "output_len": 75, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "app/assets/javascripts/discourse/adapters/rest.js.es6" - }, - "output_len": 76, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "app/models/embeddable_host.rb" - }, - "output_len": 2490, - "is_error": false - }, - { - "round": 2, - "tool": "grep", - "input": { - "pattern": "basePath", - "glob": "*.es6" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 2, - "tool": "grep", - "input": { - "pattern": "_hydrateEmbedded", - "glob": "*.es6" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 2, - "tool": "read_file", - "input": { - "path": "spec/fabricators/category_fabricator.rb" - }, - "output_len": 1409, - "is_error": false - }, - { - "round": 2, - "tool": "read_file", - "input": { - "path": "spec/fabricators/embeddable_host_fabricator.rb" - }, - "output_len": 112, - "is_error": false - }, - { - "round": 3, - "tool": "grep", - "input": { - "pattern": "replace\\('_'", - "glob": "*.es6" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 3, - "tool": "codebase_search", - "input": { - "search_string": "How does the store call basePath and what type format is used (underscore vs hyphen)?" - }, - "output_len": 4017, - "is_error": false - }, - { - "round": 3, - "tool": "read_file", - "input": { - "path": "db/migrate/20150818190757_create_embeddable_hosts.rb" - }, - "output_len": 1514, - "is_error": false - }, - { - "round": 4, - "tool": "read_file", - "input": { - "path": "test/javascripts/helpers/create-pretender.js.es6", - "lines": "39-55" - }, - "output_len": 71, - "is_error": false - }, - { - "round": 4, - "tool": "read_file", - "input": { - "path": "test/javascripts/models/store-test.js.es6", - "lines": "106-140" - }, - "output_len": 64, - "is_error": false - }, - { - "round": 4, - "tool": "grep", - "input": { - "pattern": "_lookupSubType", - "glob": "*.es6" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 5, - "tool": "glob", - "input": { - "pattern": "**/store-test*" - }, - "output_len": 52, - "is_error": false - }, - { - "round": 5, - "tool": "glob", - "input": { - "pattern": "**/create-pretender*" - }, - "output_len": 52, - "is_error": false - }, - { - "round": 5, - "tool": "glob", - "input": { - "pattern": "**/store.js*" - }, - "output_len": 79, - "is_error": false - }, - { - "round": 6, - "tool": "read_file", - "input": { - "path": "frontend/discourse/app/services/store.js", - "lines": "350-400" - }, - "output_len": 1550, - "is_error": false - }, - { - "round": 6, - "tool": "read_file", - "input": { - "path": "frontend/discourse/tests/unit/services/store-test.js", - "lines": "100-150" - }, - "output_len": 1952, - "is_error": false - }, - { - "round": 7, - "tool": "codebase_search", - "input": { - "search_string": "How does EmbeddableHost.record_for_host handle when called from TopicEmbed.import?" - }, - "output_len": 6291, - "is_error": false - }, - { - "round": 7, - "tool": "grep", - "input": { - "pattern": "record_for_host" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 8, - "tool": "codebase_search", - "input": { - "search_string": "How does PG::Result indexing work when accessing rows from empty result sets in Rails migrations?" - }, - "output_len": 5862, - "is_error": false - }, - { - "round": 8, - "tool": "grep", - "input": { - "pattern": "store\\.find\\('fruit'" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 9, - "tool": "grep", - "input": { - "pattern": "find\\('fruit'" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 9, - "tool": "grep", - "input": { - "pattern": "fruits/:id" - }, - "output_len": 496, - "is_error": false - }, - { - "round": 10, - "tool": "read_file", - "input": { - "path": "app/controllers/admin/embeddable_hosts_controller.rb" - }, - "output_len": 1902, - "is_error": false - }, - { - "round": 10, - "tool": "read_file", - "input": { - "path": "app/controllers/admin/embedding_controller.rb" - }, - "output_len": 905, - "is_error": false - }, - { - "round": 11, - "tool": "read_file", - "input": { - "path": "frontend/discourse/app/services/store.js", - "lines": "315-390" - }, - "output_len": 2381, - "is_error": false - }, - { - "round": 11, - "tool": "read_file", - "input": { - "path": "frontend/discourse/tests/helpers/store-pretender.js" - }, - "output_len": 4217, - "is_error": false - }, - { - "round": 12, - "tool": "grep", - "input": { - "pattern": "embed_category", - "glob": "*.yml" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 12, - "tool": "codebase_search", - "input": { - "search_string": "How does the EmbeddableHost before_validation handle nil host values?" - }, - "output_len": 2528, - "is_error": false - }, - { - "round": 13, - "tool": "grep", - "input": { - "pattern": "expandable_first_post" - }, - "output_len": 4684, - "is_error": false - }, - { - "round": 13, - "tool": "codebase_search", - "input": { - "search_string": "Where is expandable_first_post used and what was the embeddable_hosts check purpose?" - }, - "output_len": 17, - "is_error": false - }, - { - "round": 14, - "tool": "read_file", - "input": { - "path": "frontend/discourse/app/services/store.js", - "lines": "1-80" - }, - "output_len": 2354, - "is_error": false - }, - { - "round": 15, - "tool": "read_file", - "input": { - "path": "frontend/discourse/app/services/store.js", - "lines": "90-160" - }, - "output_len": 2515, - "is_error": false - }, - { - "round": 16, - "tool": "grep", - "input": { - "pattern": "cmd_tuples", - "path": "db/migrate" - }, - "output_len": 6466, - "is_error": false - }, - { - "round": 16, - "tool": "grep", - "input": { - "pattern": "uncategorized_category_id", - "glob": "*.yml" - }, - "output_len": 355, - "is_error": false - }, - { - "round": 17, - "tool": "read_file", - "input": { - "path": "frontend/discourse/tests/unit/services/store-test.js", - "lines": "160-220" - }, - "output_len": 2181, - "is_error": false - } -] \ No newline at end of file diff --git a/pr_review_agent/output/traces/discourse_graphite_pr3_trace.json b/pr_review_agent/output/traces/discourse_graphite_pr3_trace.json deleted file mode 100644 index 7713e96..0000000 --- a/pr_review_agent/output/traces/discourse_graphite_pr3_trace.json +++ /dev/null @@ -1,187 +0,0 @@ -[ - { - "round": 0, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "email_in_restriction_setting domain matching regex whitelist blacklist" - }, - "output_len": 17, - "is_error": false - }, - { - "round": 0, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "How is email_domains_whitelist and email_domains_blacklist setting formatted? Multiple domains separated by what delimiter?" - }, - "output_len": 2375, - "is_error": false - }, - { - "round": 0, - "tool": "grep", - "input": { - "pattern": "email_domains_whitelist|email_domains_blacklist" - }, - "output_len": 1554, - "is_error": false - }, - { - "round": 1, - "tool": "grep", - "input": { - "pattern": "email_domains_whitelist|email_domains_blacklist", - "glob": "config/site_settings.yml" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 1, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "SiteSetting email_domains_whitelist email_domains_blacklist definition type" - }, - "output_len": 4053, - "is_error": false - }, - { - "round": 1, - "tool": "grep", - "input": { - "pattern": "email_domains", - "glob": "config/site_settings.yml" - }, - "output_len": 1459, - "is_error": false - }, - { - "round": 2, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "Enum class definition used for actions block do_nothing" - }, - "output_len": 2566, - "is_error": false - }, - { - "round": 2, - "tool": "grep", - "input": { - "pattern": "class Enum" - }, - "output_len": 2712, - "is_error": false - }, - { - "round": 2, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "BlockedEmail should_block callers - where is should_block? called from" - }, - "output_len": 1212, - "is_error": false - }, - { - "round": 3, - "tool": "grep", - "input": { - "pattern": "be_within_one_second_of" - }, - "output_len": 13642, - "is_error": false - }, - { - "round": 3, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "email_domains_whitelist SiteSetting definition default value" - }, - "output_len": 4033, - "is_error": false - }, - { - "round": 3, - "tool": "read_file", - "input": { - "path": "lib/validators/email_validator.rb" - }, - "output_len": 1896, - "is_error": false - }, - { - "round": 4, - "tool": "read_file", - "input": { - "path": "app/models/blocked_email.rb" - }, - "output_len": 50, - "is_error": false - }, - { - "round": 4, - "tool": "read_file", - "input": { - "path": "app/models/user.rb", - "lines": "38-50" - }, - "output_len": 596, - "is_error": false - }, - { - "round": 5, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "how is email_domains_whitelist or email_domains_blacklist setting value formatted - pipe separated or comma" - }, - "output_len": 2999, - "is_error": false - }, - { - "round": 5, - "tool": "grep", - "input": { - "pattern": "email_domains_whitelist", - "glob": "*.yml" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 6, - "tool": "grep", - "input": { - "pattern": "host_list", - "glob": "*.rb" - }, - "output_len": 1443, - "is_error": false - }, - { - "round": 7, - "tool": "grep", - "input": { - "pattern": "class HostListSettingValidator" - }, - "output_len": 407, - "is_error": false - }, - { - "round": 8, - "tool": "read_file", - "input": { - "path": "lib/validators/host_list_setting_validator.rb" - }, - "output_len": 312, - "is_error": false - }, - { - "round": 9, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "how are host_list site settings stored - pipe separated format" - }, - "output_len": 2142, - "is_error": false - } -] \ No newline at end of file diff --git a/pr_review_agent/output/traces/discourse_graphite_pr7_trace.json b/pr_review_agent/output/traces/discourse_graphite_pr7_trace.json deleted file mode 100644 index 6847a69..0000000 --- a/pr_review_agent/output/traces/discourse_graphite_pr7_trace.json +++ /dev/null @@ -1,126 +0,0 @@ -[ - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "Definition of dark-light-choose SCSS function or mixin" - }, - "output_len": 473, - "is_error": false - }, - { - "round": 0, - "tool": "grep", - "input": { - "pattern": "dark-light-choose", - "glob": "**/color_transformations.scss" - }, - "output_len": 11688, - "is_error": false - }, - { - "round": 0, - "tool": "grep", - "input": { - "pattern": "@function dark-light-choose" - }, - "output_len": 487, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "app/assets/stylesheets/desktop/topic-post.scss", - "lines": "285-295" - }, - "output_len": 69, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "app/assets/stylesheets/mobile/modal.scss", - "lines": "96-106" - }, - "output_len": 63, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "app/assets/stylesheets/desktop/user.scss", - "lines": "516-530" - }, - "output_len": 63, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "app/assets/stylesheets/mobile/user.scss", - "lines": "490-505" - }, - "output_len": 62, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "app/assets/stylesheets/mobile/topic-post.scss", - "lines": "175-190" - }, - "output_len": 68, - "is_error": false - }, - { - "round": 2, - "tool": "glob", - "input": { - "pattern": "**/desktop/topic-post*" - }, - "output_len": 17, - "is_error": false - }, - { - "round": 2, - "tool": "glob", - "input": { - "pattern": "**/mobile/modal*" - }, - "output_len": 17, - "is_error": false - }, - { - "round": 3, - "tool": "glob", - "input": { - "pattern": "**/*topic-post*scss" - }, - "output_len": 50, - "is_error": false - }, - { - "round": 3, - "tool": "glob", - "input": { - "pattern": "**/*modal*scss" - }, - "output_len": 1635, - "is_error": false - }, - { - "round": 4, - "tool": "read_file", - "input": { - "path": "app/assets/stylesheets/common/foundation/variables.scss", - "lines": "195-215" - }, - "output_len": 513, - "is_error": false - } -] \ No newline at end of file diff --git a/pr_review_agent/output/traces/discourse_graphite_pr8_trace.json b/pr_review_agent/output/traces/discourse_graphite_pr8_trace.json deleted file mode 100644 index 3262112..0000000 --- a/pr_review_agent/output/traces/discourse_graphite_pr8_trace.json +++ /dev/null @@ -1,247 +0,0 @@ -[ - { - "round": 0, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "How is group.users.delete used in Rails, and what does it do with an integer argument vs a User object" - }, - "output_len": 495, - "is_error": false - }, - { - "round": 0, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "remove_member route definition for admin groups" - }, - "output_len": 1366, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "app/controllers/admin/groups_controller.rb" - }, - "output_len": 4595, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "config/routes.rb", - "lines": "40-60" - }, - "output_len": 959, - "is_error": false - }, - { - "round": 2, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "totalPages or total pages calculation with Math.floor and Math.ceil for pagination" - }, - "output_len": 17, - "is_error": false - }, - { - "round": 2, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "group.users.delete with user_id integer in Rails has_many through association" - }, - "output_len": 45596, - "is_error": false - }, - { - "round": 3, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "group-members controller in Discourse frontend that handles members list display" - }, - "output_len": 33376, - "is_error": false - }, - { - "round": 3, - "tool": "grep", - "input": { - "pattern": "showingLast|showingFirst|totalPages|currentPage", - "glob": "*.js*" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 4, - "tool": "grep", - "input": { - "pattern": "xhr :put.*remove_member|xhr :delete.*remove_member", - "glob": "*spec*" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 4, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "How does Rails controller spec dispatch actions: does xhr method honor HTTP verb for routing or just dispatch by action name?" - }, - "output_len": 279, - "is_error": false - }, - { - "round": 5, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "group/members.hbs template controller model members property binding in Ember" - }, - "output_len": 24857, - "is_error": false - }, - { - "round": 5, - "tool": "grep", - "input": { - "pattern": "GroupMembersController|group-members.*controller|group_members.*controller", - "glob": "*.{js,es6}" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 6, - "tool": "grep", - "input": { - "pattern": "class BasicGroupSerializer" - }, - "output_len": 405, - "is_error": false - }, - { - "round": 6, - "tool": "grep", - "input": { - "pattern": "render_serialized", - "glob": "*admin/groups_controller*" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 7, - "tool": "read_file", - "input": { - "path": "app/serializers/basic_group_serializer.rb" - }, - "output_len": 2895, - "is_error": false - }, - { - "round": 7, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "render_serialized method definition and how it generates the root key in JSON" - }, - "output_len": 196, - "is_error": false - }, - { - "round": 8, - "tool": "grep", - "input": { - "pattern": "def render_serialized" - }, - "output_len": 1061, - "is_error": false - }, - { - "round": 9, - "tool": "read_file", - "input": { - "path": "app/controllers/application_controller.rb", - "lines": "508-530" - }, - "output_len": 760, - "is_error": false - }, - { - "round": 10, - "tool": "grep", - "input": { - "pattern": "def serialize_data" - }, - "output_len": 475, - "is_error": false - }, - { - "round": 11, - "tool": "read_file", - "input": { - "path": "app/controllers/application_controller.rb", - "lines": "493-510" - }, - "output_len": 890, - "is_error": false - }, - { - "round": 12, - "tool": "read_file", - "input": { - "path": "app/assets/javascripts/admin/controllers/admin-group.js.es6" - }, - "output_len": 82, - "is_error": false - }, - { - "round": 13, - "tool": "read_file", - "input": { - "path": "app/assets/javascripts/discourse/models/group.js" - }, - "output_len": 71, - "is_error": false - }, - { - "round": 14, - "tool": "grep", - "input": { - "pattern": "class.*badges|className.*badges", - "glob": "*admin*badge*" - }, - "output_len": 1389, - "is_error": false - }, - { - "round": 14, - "tool": "grep", - "input": { - "pattern": "section.details", - "glob": "*admin*badge*hbs" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 15, - "tool": "grep", - "input": { - "pattern": "xhr :(get|put|post|delete|patch), :(remove_member|add_members)", - "glob": "*spec*" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 16, - "tool": "grep", - "input": { - "pattern": "remove_member|add_members", - "glob": "*routes*" - }, - "output_len": 1410, - "is_error": false - } -] \ No newline at end of file diff --git a/pr_review_agent/output/traces/discourse_graphite_pr9_trace.json b/pr_review_agent/output/traces/discourse_graphite_pr9_trace.json deleted file mode 100644 index 6429c69..0000000 --- a/pr_review_agent/output/traces/discourse_graphite_pr9_trace.json +++ /dev/null @@ -1,507 +0,0 @@ -[ - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "How does set_locale work in application_controller.rb and what does it return?" - }, - "output_len": 17, - "is_error": false - }, - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "How is I18n.ensure_loaded! defined and used across the codebase?" - }, - "output_len": 8903, - "is_error": false - }, - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "How is translate_accelerator load_locale used and what is @loaded_locales?" - }, - "output_len": 8903, - "is_error": false - }, - { - "round": 0, - "tool": "grep", - "input": { - "pattern": "def set_locale", - "glob": "*.rb" - }, - "output_len": 450, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "app/controllers/application_controller.rb", - "lines": "130-170" - }, - "output_len": 1451, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "lib/freedom_patches/translate_accelerator.rb", - "lines": "55-70" - }, - "output_len": 538, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "config/initializers/i18n.rb" - }, - "output_len": 50, - "is_error": false - }, - { - "round": 2, - "tool": "glob", - "input": { - "pattern": "config/initializers/*i18n*" - }, - "output_len": 31, - "is_error": false - }, - { - "round": 2, - "tool": "grep", - "input": { - "pattern": "def set_locale", - "path": "app/controllers/application_controller.rb" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 3, - "tool": "read_file", - "input": { - "path": "config/initializers/100-i18n.rb" - }, - "output_len": 776, - "is_error": false - }, - { - "round": 3, - "tool": "grep", - "input": { - "pattern": "set_locale", - "path": "app/controllers/application_controller.rb" - }, - "output_len": 200, - "is_error": false - }, - { - "round": 4, - "tool": "grep", - "input": { - "pattern": "set_locale|ensure_loaded", - "path": "app/controllers/application_controller.rb" - }, - "output_len": 200, - "is_error": false - }, - { - "round": 4, - "tool": "read_file", - "input": { - "path": "lib/freedom_patches/translate_accelerator.rb", - "lines": "59-70" - }, - "output_len": 332, - "is_error": false - }, - { - "round": 4, - "tool": "grep", - "input": { - "pattern": "ensure_loaded", - "glob": "*.rb" - }, - "output_len": 857, - "is_error": false - }, - { - "round": 5, - "tool": "read_file", - "input": { - "path": "lib/freedom_patches/translate_accelerator.rb", - "lines": "105-115" - }, - "output_len": 442, - "is_error": false - }, - { - "round": 5, - "tool": "grep", - "input": { - "pattern": "FallbackLocaleList", - "glob": "*.rb" - }, - "output_len": 1923, - "is_error": false - }, - { - "round": 6, - "tool": "read_file", - "input": { - "path": "lib/i18n/backend/fallback_locale_list.rb" - }, - "output_len": 571, - "is_error": false - }, - { - "round": 6, - "tool": "codebase_search", - "input": { - "search_string": "Where is set_locale defined as a before_action or method in ApplicationController for locale setting" - }, - "output_len": 1225, - "is_error": false - }, - { - "round": 7, - "tool": "grep", - "input": { - "pattern": "def set_locale|def with_resolved_locale", - "path": "app/controllers/application_controller.rb" - }, - "output_len": 94, - "is_error": false - }, - { - "round": 7, - "tool": "grep", - "input": { - "pattern": "set_locale", - "path": "app/controllers" - }, - "output_len": 530, - "is_error": false - }, - { - "round": 8, - "tool": "grep", - "input": { - "pattern": "i18n.fallbacks", - "path": "config" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 8, - "tool": "read_file", - "input": { - "path": "config/environments/development.rb" - }, - "output_len": 4509, - "is_error": false - }, - { - "round": 9, - "tool": "grep", - "input": { - "pattern": "i18n.fallbacks", - "glob": "*.rb" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 9, - "tool": "grep", - "input": { - "pattern": "I18n.fallbacks", - "glob": "*.rb" - }, - "output_len": 7281, - "is_error": false - }, - { - "round": 10, - "tool": "grep", - "input": { - "pattern": "before_action.*set_locale|before_filter.*set_locale", - "path": "app/controllers" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 10, - "tool": "grep", - "input": { - "pattern": ":set_locale", - "path": "app/controllers" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 11, - "tool": "grep", - "input": { - "pattern": "set_locale", - "glob": "*.rb" - }, - "output_len": 15454, - "is_error": false - }, - { - "round": 12, - "tool": "grep", - "input": { - "pattern": "02-freedom_patches", - "glob": "*.rb" - }, - "output_len": 344, - "is_error": false - }, - { - "round": 12, - "tool": "glob", - "input": { - "pattern": "config/initializers/*" - }, - "output_len": 1958, - "is_error": false - }, - { - "round": 13, - "tool": "grep", - "input": { - "pattern": "fallbacks", - "path": "config/environments" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 13, - "tool": "read_file", - "input": { - "path": "config/environments/test.rb" - }, - "output_len": 3733, - "is_error": false - }, - { - "round": 14, - "tool": "read_file", - "input": { - "path": "app/models/theme.rb", - "lines": "565-615" - }, - "output_len": 2068, - "is_error": false - }, - { - "round": 14, - "tool": "read_file", - "input": { - "path": "app/models/theme_field.rb", - "lines": "300-315" - }, - "output_len": 670, - "is_error": false - }, - { - "round": 15, - "tool": "grep", - "input": { - "pattern": "def name", - "path": "app/models/theme_field.rb" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 15, - "tool": "codebase_search", - "input": { - "search_string": "What type is ThemeField.name - is it a string or symbol database attribute?" - }, - "output_len": 113, - "is_error": false - }, - { - "round": 16, - "tool": "grep", - "input": { - "pattern": "t\\.string.*name|name.*:string", - "path": "db/migrate", - "glob": "*theme_field*" - }, - "output_len": 941, - "is_error": false - }, - { - "round": 17, - "tool": "read_file", - "input": { - "path": "app/models/theme.rb", - "lines": "530-580" - }, - "output_len": 1900, - "is_error": false - }, - { - "round": 17, - "tool": "read_file", - "input": { - "path": "app/models/theme.rb", - "lines": "730-745" - }, - "output_len": 633, - "is_error": false - }, - { - "round": 18, - "tool": "grep", - "input": { - "pattern": "def filter_locale_fields|scope.*filter_locale_fields", - "glob": "*.rb" - }, - "output_len": 350, - "is_error": false - }, - { - "round": 19, - "tool": "read_file", - "input": { - "path": "app/models/theme_field.rb", - "lines": "60-80" - }, - "output_len": 899, - "is_error": false - }, - { - "round": 20, - "tool": "read_file", - "input": { - "path": "config/initializers/002-freedom_patches.rb" - }, - "output_len": 544, - "is_error": false - }, - { - "round": 21, - "tool": "bash", - "input": { - "command": "cd /Users/tejas/personal/applymodel/zmisc/examples/pr_clones/discourse && ruby -e '\n# Simulate the FallbackLocaleList behavior from the PR\nlocale = \"en\" # String from ThemeField.name\nresult = [locale, :en, :en].uniq.compact\nputs \"With string locale \\\"en\\\": #{result.inspect}\"\nputs \"Length: #{result.length}\"\nputs \"\\\"en\\\" == :en => #{\"en\" == :en}\"\n\nlocale2 = :en # Symbol from I18n.locale\nresult2 = [locale2, :en, :en].uniq.compact\nputs \"\\nWith symbol locale :en: #{result2.inspect}\"\nputs \"Length: #{result2.length}\"\n\nlocale3 = \"fr\"\nresult3 = [locale3, :en, :en].uniq.compact\nputs \"\\nWith string locale \\\"fr\\\": #{result3.inspect}\"\nputs \"Length: #{result3.length}\"\n'\n" - }, - "output_len": 156, - "is_error": false - }, - { - "round": 22, - "tool": "bash", - "input": { - "command": "cd /Users/tejas/personal/applymodel/zmisc/examples/pr_clones/discourse && ruby -e '\n# Simulate the ensure_loaded! issue \nloaded_locales = [:en, :fr] # symbols from load_locale\nlocale = \"en\" # string passed to ensure_loaded!\nputs \"loaded_locales.include?(\\\"en\\\"): #{loaded_locales.include?(\"en\")}\"\nputs \"loaded_locales.include?(:en): #{loaded_locales.include?(:en)}\"\n'\n" - }, - "output_len": 71, - "is_error": false - }, - { - "round": 23, - "tool": "read_file", - "input": { - "path": "lib/freedom_patches/translate_accelerator.rb", - "lines": "33-66" - }, - "output_len": 1207, - "is_error": false - }, - { - "round": 24, - "tool": "grep", - "input": { - "pattern": "Regexp.escape locale", - "path": "lib/freedom_patches/translate_accelerator.rb" - }, - "output_len": 413, - "is_error": false - }, - { - "round": 25, - "tool": "read_file", - "input": { - "path": "lib/i18n/backend/discourse_i18n.rb" - }, - "output_len": 4147, - "is_error": false - }, - { - "round": 26, - "tool": "grep", - "input": { - "pattern": "around_action|around_filter|:set_locale|before_action", - "path": "app/controllers/application_controller.rb" - }, - "output_len": 1647, - "is_error": false - }, - { - "round": 27, - "tool": "read_file", - "input": { - "path": "lib/js_locale_helper.rb", - "lines": "5-15" - }, - "output_len": 402, - "is_error": false - }, - { - "round": 27, - "tool": "read_file", - "input": { - "path": "lib/js_locale_helper.rb", - "lines": "130-150" - }, - "output_len": 670, - "is_error": false - }, - { - "round": 28, - "tool": "read_file", - "input": { - "path": "lib/js_locale_helper.rb", - "lines": "225-240" - }, - "output_len": 449, - "is_error": false - }, - { - "round": 28, - "tool": "read_file", - "input": { - "path": "lib/js_locale_helper.rb", - "lines": "340-355" - }, - "output_len": 424, - "is_error": false - }, - { - "round": 29, - "tool": "grep", - "input": { - "pattern": "output_MF|output_client_overrides|remove_message_formats!", - "glob": "*.rb" - }, - "output_len": 4228, - "is_error": false - } -] \ No newline at end of file diff --git a/pr_review_agent/output/traces/grafana_pr103633_trace.json b/pr_review_agent/output/traces/grafana_pr103633_trace.json deleted file mode 100644 index c13affc..0000000 --- a/pr_review_agent/output/traces/grafana_pr103633_trace.json +++ /dev/null @@ -1,375 +0,0 @@ -[ - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "How does the Check method in the RBAC service work, and what is the flow for checking permissions?" - }, - "output_len": 111, - "is_error": false - }, - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "How is permissionCacheUsage metric tracked in the RBAC service? Where are cache hit/miss labels incremented?" - }, - "output_len": 1934, - "is_error": false - }, - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "What does checkPermission do in the RBAC service? How does it determine if a request is allowed?" - }, - "output_len": 4141, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "pkg/services/authz/rbac/service.go", - "lines": "100-175" - }, - "output_len": 3937, - "is_error": false - }, - { - "round": 1, - "tool": "grep", - "input": { - "pattern": "func.*checkPermission", - "glob": "pkg/services/authz/rbac/service.go" - }, - "output_len": 556, - "is_error": false - }, - { - "round": 2, - "tool": "read_file", - "input": { - "path": "pkg/services/authz/rbac/service.go", - "lines": "175-220" - }, - "output_len": 1748, - "is_error": false - }, - { - "round": 2, - "tool": "read_file", - "input": { - "path": "pkg/services/authz/rbac/service.go", - "lines": "370-450" - }, - "output_len": 3358, - "is_error": false - }, - { - "round": 2, - "tool": "grep", - "input": { - "pattern": "getIdentityPermissions", - "glob": "pkg/services/authz/rbac/service.go" - }, - "output_len": 2547, - "is_error": false - }, - { - "round": 3, - "tool": "read_file", - "input": { - "path": "pkg/services/authz/rbac/service.go", - "lines": "1-70" - }, - "output_len": 2103, - "is_error": false - }, - { - "round": 3, - "tool": "read_file", - "input": { - "path": "pkg/services/authz/rbac/service.go", - "lines": "600-680" - }, - "output_len": 3114, - "is_error": false - }, - { - "round": 3, - "tool": "read_file", - "input": { - "path": "pkg/services/authz/rbac/service.go", - "lines": "680-720" - }, - "output_len": 1383, - "is_error": false - }, - { - "round": 4, - "tool": "read_file", - "input": { - "path": "pkg/services/authz/rbac/service.go", - "lines": "406-480" - }, - "output_len": 3064, - "is_error": false - }, - { - "round": 4, - "tool": "read_file", - "input": { - "path": "pkg/services/authz/rbac/service.go", - "lines": "144-195" - }, - "output_len": 2622, - "is_error": false - }, - { - "round": 5, - "tool": "read_file", - "input": { - "path": "pkg/services/authz/rbac.go", - "lines": "90-115" - }, - "output_len": 950, - "is_error": false - }, - { - "round": 5, - "tool": "grep", - "input": { - "pattern": "cache\\.ErrNotFound", - "glob": "pkg/services/authz/rbac/*.go" - }, - "output_len": 1475, - "is_error": false - }, - { - "round": 5, - "tool": "codebase_search", - "input": { - "search_string": "How is the cache interface defined for authzlib clients? What does WithCacheClientOption expect?" - }, - "output_len": 13181, - "is_error": false - }, - { - "round": 6, - "tool": "grep", - "input": { - "pattern": "func.*GetUserIdentifiers", - "glob": "pkg/services/authz/rbac/*.go" - }, - "output_len": 1037, - "is_error": false - }, - { - "round": 6, - "tool": "read_file", - "input": { - "path": "pkg/services/authz/rbac/service.go", - "lines": "720-800" - }, - "output_len": 3056, - "is_error": false - }, - { - "round": 7, - "tool": "read_file", - "input": { - "path": "pkg/services/authz/rbac/service.go", - "lines": "470-560" - }, - "output_len": 2916, - "is_error": false - }, - { - "round": 7, - "tool": "read_file", - "input": { - "path": "pkg/services/authz/rbac/service_test.go", - "lines": "890-1010" - }, - "output_len": 3752, - "is_error": false - }, - { - "round": 8, - "tool": "read_file", - "input": { - "path": "pkg/services/authz/rbac/service.go", - "lines": "204-395" - }, - "output_len": 7534, - "is_error": false - }, - { - "round": 9, - "tool": "grep", - "input": { - "pattern": "newFolderTreeGetter", - "glob": "pkg/services/authz/rbac/*.go" - }, - "output_len": 2682, - "is_error": false - }, - { - "round": 9, - "tool": "read_file", - "input": { - "path": "pkg/services/authz/rbac/service.go", - "lines": "830-900" - }, - "output_len": 2240, - "is_error": false - }, - { - "round": 10, - "tool": "grep", - "input": { - "pattern": "type Cache interface", - "path": "." - }, - "output_len": 10, - "is_error": false - }, - { - "round": 10, - "tool": "bash", - "input": { - "command": "find . -path \"*/grafana/authlib/cache\" -type d 2>/dev/null | head -5; find . -path \"*/authlib/cache/*.go\" 2>/dev/null | head -10" - }, - "output_len": 11, - "is_error": false - }, - { - "round": 11, - "tool": "bash", - "input": { - "command": "grep -r \"Cache interface\" vendor/github.com/grafana/authlib/cache/ 2>/dev/null || find . -name \"*.go\" -path \"*/authlib/*cache*\" 2>/dev/null | head -5" - }, - "output_len": 11, - "is_error": false - }, - { - "round": 11, - "tool": "grep", - "input": { - "pattern": "WithCacheClientOption", - "path": ".", - "glob": "*.go" - }, - "output_len": 812, - "is_error": false - }, - { - "round": 12, - "tool": "read_file", - "input": { - "path": "pkg/services/authz/rbac/service_test.go", - "lines": "968-1006" - }, - "output_len": 1087, - "is_error": false - }, - { - "round": 12, - "tool": "read_file", - "input": { - "path": "pkg/services/authz/rbac/service.go", - "lines": "853-900" - }, - "output_len": 1756, - "is_error": false - }, - { - "round": 13, - "tool": "read_file", - "input": { - "path": "pkg/services/authz/rbac/service_test.go", - "lines": "1072-1120" - }, - "output_len": 1628, - "is_error": false - }, - { - "round": 14, - "tool": "read_file", - "input": { - "path": "pkg/services/authz/rbac/service_test.go", - "lines": "1190-1260" - }, - "output_len": 2339, - "is_error": false - }, - { - "round": 15, - "tool": "read_file", - "input": { - "path": "pkg/services/authz/rbac/service_test.go", - "lines": "1145-1200" - }, - "output_len": 2018, - "is_error": false - }, - { - "round": 15, - "tool": "read_file", - "input": { - "path": "pkg/services/authz/rbac/service.go", - "lines": "900-920" - }, - "output_len": 977, - "is_error": false - }, - { - "round": 16, - "tool": "read_file", - "input": { - "path": "pkg/services/authz/rbac/service_test.go", - "lines": "1308-1370" - }, - "output_len": 2390, - "is_error": false - }, - { - "round": 17, - "tool": "read_file", - "input": { - "path": "pkg/services/authz/rbac/service_test.go", - "lines": "1380-1430" - }, - "output_len": 1820, - "is_error": false - }, - { - "round": 18, - "tool": "read_file", - "input": { - "path": "pkg/services/authz/rbac/service.go", - "lines": "434-470" - }, - "output_len": 1614, - "is_error": false - }, - { - "round": 19, - "tool": "bash", - "input": { - "command": "grep -r \"type Cache interface\" $(find . -name \"*.go\" -path \"*cache*\" | head -50) 2>/dev/null; grep -r \"Cache\\b\" go.sum 2>/dev/null | grep authlib | head -5; grep -r \"github.com/grafana/authlib\" go.mod 2>/dev/null | head -3" - }, - "output_len": 211, - "is_error": false - }, - { - "round": 20, - "tool": "bash", - "input": { - "command": "find . -path \"*/grafana/authlib\" -prune -name \"*.go\" 2>/dev/null | head -5; go doc github.com/grafana/authlib/cache Cache 2>/dev/null || echo \"Could not get go doc\"; find . -name \"cache.go\" -path \"*grafana/authlib*\" 2>/dev/null | head -5" - }, - "output_len": 529, - "is_error": false - } -] \ No newline at end of file diff --git a/pr_review_agent/output/traces/grafana_pr76186_trace.json b/pr_review_agent/output/traces/grafana_pr76186_trace.json deleted file mode 100644 index 54d3f94..0000000 --- a/pr_review_agent/output/traces/grafana_pr76186_trace.json +++ /dev/null @@ -1,390 +0,0 @@ -[ - { - "round": 0, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "Who implements the Logger interface in pkg/plugins/log and how is FromContext used" - }, - "output_len": 1584, - "is_error": false - }, - { - "round": 0, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "Find all callers of NewInstrumentationMiddleware to check if they were updated to NewMetricsMiddleware" - }, - "output_len": 13866, - "is_error": false - }, - { - "round": 0, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "How is instrumentContext function used and where is it defined" - }, - "output_len": 5009, - "is_error": false - }, - { - "round": 1, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "How does log.FromContext or logger.FromContext work - does it include traceID automatically" - }, - "output_len": 111, - "is_error": false - }, - { - "round": 1, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "WithContextualAttributes and how context attributes are added to logger output" - }, - "output_len": 3078, - "is_error": false - }, - { - "round": 1, - "tool": "grep", - "input": { - "pattern": "traceID|TraceIDFromContext", - "path": "pkg/services/pluginsintegration/clientmiddleware" - }, - "output_len": 1780, - "is_error": false - }, - { - "round": 2, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "FromContext implementation in pkg/infra/log that returns logger with contextual attributes" - }, - "output_len": 111, - "is_error": false - }, - { - "round": 2, - "tool": "grep", - "input": { - "pattern": "func.*FromContext", - "path": "pkg/infra/log" - }, - "output_len": 1083, - "is_error": false - }, - { - "round": 3, - "tool": "read_file", - "input": { - "path": "pkg/infra/log/log.go", - "lines": "220-250" - }, - "output_len": 876, - "is_error": false - }, - { - "round": 3, - "tool": "read_file", - "input": { - "path": "pkg/plugins/log/logger.go" - }, - "output_len": 2114, - "is_error": false - }, - { - "round": 4, - "tool": "grep", - "input": { - "pattern": "ctxLogProviders", - "path": "pkg/infra/log" - }, - "output_len": 1875, - "is_error": false - }, - { - "round": 4, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "RegisterContextualLogProvider or ctxLogProviders - how are contextual log providers registered including traceID" - }, - "output_len": 1764, - "is_error": false - }, - { - "round": 5, - "tool": "read_file", - "input": { - "path": "pkg/plugins/log/logger.go" - }, - "output_len": 2114, - "is_error": false - }, - { - "round": 5, - "tool": "read_file", - "input": { - "path": "pkg/plugins/log/fake.go" - }, - "output_len": 1899, - "is_error": false - }, - { - "round": 6, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "Tests for LoggerMiddleware that check log output or InfoLogs on TestLogger" - }, - "output_len": 12311, - "is_error": false - }, - { - "round": 6, - "tool": "grep", - "input": { - "pattern": "LoggerMiddleware|NewLoggerMiddleware", - "path": "pkg/services/pluginsintegration/clientmiddleware" - }, - "output_len": 17358, - "is_error": false - }, - { - "round": 7, - "tool": "glob", - "input": { - "pattern": "pkg/services/pluginsintegration/clientmiddleware/logger_middleware_test*" - }, - "output_len": 17, - "is_error": false - }, - { - "round": 7, - "tool": "grep", - "input": { - "pattern": "TestLogger|NewTestLogger", - "path": "pkg/services/pluginsintegration/clientmiddleware" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 8, - "tool": "grep", - "input": { - "pattern": "TestLogger|NewTestLogger", - "path": "pkg/services/pluginsintegration" - }, - "output_len": 2439, - "is_error": false - }, - { - "round": 8, - "tool": "grep", - "input": { - "pattern": "FromContext", - "path": "pkg/plugins/log" - }, - "output_len": 1254, - "is_error": false - }, - { - "round": 9, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "Where is grafanaInfraLogWrapper created and how is NewLoggerMiddleware called with a logger" - }, - "output_len": 10635, - "is_error": false - }, - { - "round": 9, - "tool": "grep", - "input": { - "pattern": "NewLoggerMiddleware" - }, - "output_len": 1506, - "is_error": false - }, - { - "round": 10, - "tool": "read_file", - "input": { - "path": "pkg/services/pluginsintegration/pluginsintegration.go", - "lines": "1-30" - }, - "output_len": 1595, - "is_error": false - }, - { - "round": 10, - "tool": "read_file", - "input": { - "path": "pkg/services/pluginsintegration/clientmiddleware/logger_middleware.go", - "lines": "1-40" - }, - "output_len": 1218, - "is_error": false - }, - { - "round": 11, - "tool": "glob", - "input": { - "pattern": "pkg/services/pluginsintegration/clientmiddleware/*logger*test*" - }, - "output_len": 17, - "is_error": false - }, - { - "round": 11, - "tool": "grep", - "input": { - "pattern": "logger.*InfoLogs|logger.*ErrorLogs|InfoLogs\\.Calls|ErrorLogs\\.Calls", - "path": "pkg/services/pluginsintegration/clientmiddleware" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 12, - "tool": "grep", - "input": { - "pattern": "LoggerMiddleware|logRequest|TestLogger.*FromContext", - "glob": "*_test.go" - }, - "output_len": 3706, - "is_error": false - }, - { - "round": 13, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "All implementations of pkg/plugins/log.Logger interface - types that implement New Debug Info Warn Error methods" - }, - "output_len": 5453, - "is_error": false - }, - { - "round": 13, - "tool": "grep", - "input": { - "pattern": "plugins/log.*Logger", - "glob": "*.go", - "path": "pkg/tsdb" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 14, - "tool": "grep", - "input": { - "pattern": "endpointQueryData|endpointCallResource|endpointCheckHealth|endpointCollectMetrics", - "path": "pkg/services/pluginsintegration/clientmiddleware" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 15, - "tool": "grep", - "input": { - "pattern": "endpoint(QueryData|CallResource|CheckHealth|CollectMetrics)\"", - "path": "pkg/services/pluginsintegration/clientmiddleware" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 16, - "tool": "grep", - "input": { - "pattern": "\"endpointQueryData|const.*endpoint|endpointQuery\"", - "path": "pkg/services/pluginsintegration/clientmiddleware" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 17, - "tool": "grep", - "input": { - "pattern": "endpoint.*=.*\\\"", - "path": "pkg/services/pluginsintegration/clientmiddleware" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 18, - "tool": "bash", - "input": { - "command": "grep -r \"endpointQueryData\\|endpointCallResource\\|endpointCheckHealth\\|endpointCollectMetrics\\|statusOK\\|statusError\" pkg/services/pluginsintegration/clientmiddleware/ | head -20" - }, - "output_len": 11, - "is_error": false - }, - { - "round": 19, - "tool": "glob", - "input": { - "pattern": "pkg/services/pluginsintegration/clientmiddleware/const*" - }, - "output_len": 17, - "is_error": false - }, - { - "round": 19, - "tool": "bash", - "input": { - "command": "grep -r \"statusOK\\|statusError\\|endpointQuery\" pkg/services/pluginsintegration/clientmiddleware/ 2>/dev/null | head -20" - }, - "output_len": 11, - "is_error": false - }, - { - "round": 20, - "tool": "list_directory", - "input": { - "path": "pkg/services/pluginsintegration/clientmiddleware" - }, - "output_len": 887, - "is_error": false - }, - { - "round": 21, - "tool": "grep", - "input": { - "pattern": "statusOK|statusError|endpointQuery\"", - "path": "pkg/services/pluginsintegration/clientmiddleware/testing.go" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 22, - "tool": "bash", - "input": { - "command": "grep -r \"statusOK\\|statusError\" pkg/services/pluginsintegration/ 2>/dev/null | head -20" - }, - "output_len": 11, - "is_error": false - }, - { - "round": 23, - "tool": "bash", - "input": { - "command": "grep -rn \"statusOK\\|statusError\\|endpointQueryData\\|endpointCallResource\\|endpointCheckHealth\\|endpointCollectMetrics\" pkg/ 2>/dev/null | grep -v \"_test.go\" | grep -v \".git\" | head -30" - }, - "output_len": 1344, - "is_error": false - }, - { - "round": 24, - "tool": "read_file", - "input": { - "path": "pkg/services/pluginsintegration/clientmiddleware/testing.go" - }, - "output_len": 1104, - "is_error": false - } -] \ No newline at end of file diff --git a/pr_review_agent/output/traces/grafana_pr90939_trace.json b/pr_review_agent/output/traces/grafana_pr90939_trace.json deleted file mode 100644 index 1d3bcb1..0000000 --- a/pr_review_agent/output/traces/grafana_pr90939_trace.json +++ /dev/null @@ -1,72 +0,0 @@ -[ - { - "round": 0, - "tool": "read_file", - "input": { - "path": "pkg/api/webassets/webassets.go" - }, - "output_len": 6398, - "is_error": false - }, - { - "round": 0, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "Who calls GetWebAssets and how is it used?" - }, - "output_len": 7471, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "pkg/services/frontend/webassets/webassets.go" - }, - "output_len": 2237, - "is_error": false - }, - { - "round": 1, - "tool": "grep", - "input": { - "pattern": "entryPointAssetsCache", - "path": "pkg/api/webassets" - }, - "output_len": 2283, - "is_error": false - }, - { - "round": 2, - "tool": "grep", - "input": { - "pattern": "setting\\.Dev", - "path": "pkg/setting", - "glob": "*.go" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 3, - "tool": "grep", - "input": { - "pattern": "Env.*=.*Dev|Dev\\s*=\"", - "path": "pkg/setting", - "glob": "*.go" - }, - "output_len": 328, - "is_error": false - }, - { - "round": 4, - "tool": "grep", - "input": { - "pattern": "Dev\\s*=\\s*\\\"", - "path": "pkg/setting", - "glob": "*.go" - }, - "output_len": 372, - "is_error": false - } -] \ No newline at end of file diff --git a/pr_review_agent/output/traces/grafana_pr94942_trace.json b/pr_review_agent/output/traces/grafana_pr94942_trace.json deleted file mode 100644 index 6c4bfd8..0000000 --- a/pr_review_agent/output/traces/grafana_pr94942_trace.json +++ /dev/null @@ -1,188 +0,0 @@ -[ - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "How is enableSqlExpressions called and what does the feature flag FlagSqlExpressions control?" - }, - "output_len": 11012, - "is_error": false - }, - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "Who calls QueryTypeSQL in the expression reader and how is it used in alerts?" - }, - "output_len": 3436, - "is_error": false - }, - { - "round": 0, - "tool": "grep", - "input": { - "pattern": "FlagSqlExpressions" - }, - "output_len": 4836, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "pkg/expr/reader.go", - "lines": "120-200" - }, - "output_len": 41, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "pkg/expr/sql_command.go" - }, - "output_len": 16300, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "pkg/expr/sql/parser.go" - }, - "output_len": 2448, - "is_error": false - }, - { - "round": 2, - "tool": "read_file", - "input": { - "path": "pkg/expr/sql/db.go" - }, - "output_len": 3546, - "is_error": false - }, - { - "round": 2, - "tool": "grep", - "input": { - "pattern": "go-duck/duck" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 2, - "tool": "grep", - "input": { - "pattern": "enableSqlExpressions" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 3, - "tool": "glob", - "input": { - "pattern": "pkg/expr/reader*.go" - }, - "output_len": 17, - "is_error": false - }, - { - "round": 3, - "tool": "bash", - "input": { - "command": "find pkg/expr/sql/ -name \"*.go\" | head -20" - }, - "output_len": 334, - "is_error": false - }, - { - "round": 4, - "tool": "read_file", - "input": { - "path": "pkg/expr/nodes.go", - "lines": "120-135" - }, - "output_len": 631, - "is_error": false - }, - { - "round": 4, - "tool": "codebase_search", - "input": { - "search_string": "ExpressionQueryReader struct definition and its features field" - }, - "output_len": 111, - "is_error": false - }, - { - "round": 4, - "tool": "codebase_search", - "input": { - "search_string": "Where is ReadQuery called on ExpressionQueryReader and how is the result used for SQL expressions?" - }, - "output_len": 312, - "is_error": false - }, - { - "round": 5, - "tool": "grep", - "input": { - "pattern": "ExpressionQueryReader", - "glob": "*.go" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 6, - "tool": "grep", - "input": { - "pattern": "ExpressionQueryReader" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 7, - "tool": "grep", - "input": { - "pattern": "ReadQuery", - "glob": "pkg/expr/*.go" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 7, - "tool": "grep", - "input": { - "pattern": "QueryTypeSQL", - "glob": "pkg/expr/*.go" - }, - "output_len": 736, - "is_error": false - }, - { - "round": 8, - "tool": "grep", - "input": { - "pattern": "QueryFramesInto", - "glob": "*.go" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 8, - "tool": "read_file", - "input": { - "path": "pkg/expr/sql/dummy_arm.go" - }, - "output_len": 844, - "is_error": false - } -] \ No newline at end of file diff --git a/pr_review_agent/output/traces/grafana_pr97529_trace.json b/pr_review_agent/output/traces/grafana_pr97529_trace.json deleted file mode 100644 index e239760..0000000 --- a/pr_review_agent/output/traces/grafana_pr97529_trace.json +++ /dev/null @@ -1,544 +0,0 @@ -[ - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "How is the cache mutex used in bleveBackend to protect concurrent access to the cache map?" - }, - "output_len": 4637, - "is_error": false - }, - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "Who calls server.Init and how is the sync.Once pattern used for initialization?" - }, - "output_len": 17, - "is_error": false - }, - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "Who calls NewResourceServer and how do callers use the returned server?" - }, - "output_len": 5071, - "is_error": false - }, - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "What does bleveBackend BuildIndex do and who calls it concurrently?" - }, - "output_len": 17, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "pkg/storage/unified/search/bleve.go", - "lines": "80-145" - }, - "output_len": 2201, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "pkg/storage/unified/resource/server.go", - "lines": "285-330" - }, - "output_len": 1452, - "is_error": false - }, - { - "round": 1, - "tool": "grep", - "input": { - "pattern": "cacheMu", - "glob": "pkg/storage/unified/search/bleve.go" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 2, - "tool": "read_file", - "input": { - "path": "pkg/storage/unified/search/bleve.go", - "lines": "145-175" - }, - "output_len": 909, - "is_error": false - }, - { - "round": 2, - "tool": "grep", - "input": { - "pattern": "cacheMu\\b", - "glob": "pkg/storage/unified/search/bleve.go" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 2, - "tool": "read_file", - "input": { - "path": "pkg/storage/unified/resource/server.go", - "lines": "245-270" - }, - "output_len": 725, - "is_error": false - }, - { - "round": 3, - "tool": "grep", - "input": { - "pattern": "BuildIndex", - "glob": "*.go" - }, - "output_len": 25161, - "is_error": false - }, - { - "round": 3, - "tool": "read_file", - "input": { - "path": "pkg/storage/unified/resource/search.go", - "lines": "300-360" - }, - "output_len": 2254, - "is_error": false - }, - { - "round": 3, - "tool": "read_file", - "input": { - "path": "pkg/storage/unified/resource/server.go", - "lines": "370-420" - }, - "output_len": 1845, - "is_error": false - }, - { - "round": 4, - "tool": "read_file", - "input": { - "path": "pkg/storage/unified/resource/server.go", - "lines": "425-465" - }, - "output_len": 1493, - "is_error": false - }, - { - "round": 4, - "tool": "read_file", - "input": { - "path": "pkg/storage/unified/resource/server.go", - "lines": "130-180" - }, - "output_len": 2448, - "is_error": false - }, - { - "round": 4, - "tool": "grep", - "input": { - "pattern": "func.*server.*Init\\(", - "glob": "pkg/storage/unified/resource/server.go" - }, - "output_len": 432, - "is_error": false - }, - { - "round": 5, - "tool": "read_file", - "input": { - "path": "pkg/storage/unified/resource/server.go", - "lines": "467-530" - }, - "output_len": 1783, - "is_error": false - }, - { - "round": 5, - "tool": "read_file", - "input": { - "path": "pkg/storage/unified/resource/search.go", - "lines": "160-230" - }, - "output_len": 2368, - "is_error": false - }, - { - "round": 6, - "tool": "read_file", - "input": { - "path": "pkg/storage/unified/resource/server.go", - "lines": "735-780" - }, - "output_len": 1795, - "is_error": false - }, - { - "round": 6, - "tool": "read_file", - "input": { - "path": "pkg/storage/unified/resource/search.go", - "lines": "170-220" - }, - "output_len": 1535, - "is_error": false - }, - { - "round": 6, - "tool": "grep", - "input": { - "pattern": "func.*server.*initWatcher", - "glob": "pkg/storage/unified/resource/server.go" - }, - "output_len": 472, - "is_error": false - }, - { - "round": 7, - "tool": "read_file", - "input": { - "path": "pkg/storage/unified/resource/server.go", - "lines": "1327-1375" - }, - "output_len": 1657, - "is_error": false - }, - { - "round": 7, - "tool": "read_file", - "input": { - "path": "pkg/storage/unified/resource/search.go", - "lines": "230-290" - }, - "output_len": 2270, - "is_error": false - }, - { - "round": 8, - "tool": "grep", - "input": { - "pattern": "func \\(s \\*searchSupport\\) init", - "glob": "pkg/storage/unified/resource/search.go" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 8, - "tool": "grep", - "input": { - "pattern": "func \\(s \\*searchServer\\) init", - "glob": "pkg/storage/unified/resource/search.go" - }, - "output_len": 400, - "is_error": false - }, - { - "round": 9, - "tool": "read_file", - "input": { - "path": "pkg/storage/unified/resource/search.go", - "lines": "640-750" - }, - "output_len": 3422, - "is_error": false - }, - { - "round": 10, - "tool": "grep", - "input": { - "pattern": "searchSupport", - "glob": "pkg/storage/unified/resource/search.go" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 10, - "tool": "grep", - "input": { - "pattern": "type searchSupport struct", - "glob": "*.go" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 11, - "tool": "read_file", - "input": { - "path": "pkg/storage/unified/resource/search.go", - "lines": "920-970" - }, - "output_len": 2041, - "is_error": false - }, - { - "round": 11, - "tool": "grep", - "input": { - "pattern": "getOrCreateIndex", - "glob": "pkg/storage/unified/resource/search.go" - }, - "output_len": 3644, - "is_error": false - }, - { - "round": 12, - "tool": "grep", - "input": { - "pattern": "IndexMetrics\\b", - "glob": "pkg/storage/unified/resource/*.go" - }, - "output_len": 3578, - "is_error": false - }, - { - "round": 12, - "tool": "read_file", - "input": { - "path": "pkg/storage/unified/search/bleve.go", - "lines": "350-580" - }, - "output_len": 9418, - "is_error": false - }, - { - "round": 13, - "tool": "grep", - "input": { - "pattern": "IndexMetrics\\s*(!=|==)\\s*nil", - "glob": "*.go" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 13, - "tool": "grep", - "input": { - "pattern": "var IndexMetrics", - "glob": "*.go" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 14, - "tool": "grep", - "input": { - "pattern": "IndexMetrics", - "glob": "pkg/storage/unified/resource/*.go" - }, - "output_len": 3578, - "is_error": false - }, - { - "round": 15, - "tool": "grep", - "input": { - "pattern": "^var IndexMetrics|^\tIndexMetrics\\s*\\*|resource\\.IndexMetrics", - "glob": "*.go" - }, - "output_len": 807, - "is_error": false - }, - { - "round": 16, - "tool": "bash", - "input": { - "command": "head -3 pkg/storage/unified/search/bleve.go" - }, - "output_len": 24, - "is_error": false - }, - { - "round": 16, - "tool": "bash", - "input": { - "command": "head -3 pkg/storage/unified/resource/search.go" - }, - "output_len": 26, - "is_error": false - }, - { - "round": 17, - "tool": "grep", - "input": { - "pattern": "^var IndexMetrics|IndexMetrics\\s*=|IndexMetrics \\*", - "glob": "pkg/storage/unified/resource/*.go" - }, - "output_len": 361, - "is_error": false - }, - { - "round": 18, - "tool": "grep", - "input": { - "pattern": "IndexMetrics", - "glob": "pkg/storage/unified/resource/metrics.go" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 18, - "tool": "glob", - "input": { - "pattern": "pkg/storage/unified/resource/metrics*" - }, - "output_len": 39, - "is_error": false - }, - { - "round": 19, - "tool": "read_file", - "input": { - "path": "pkg/storage/unified/resource/metrics.go" - }, - "output_len": 1845, - "is_error": false - }, - { - "round": 20, - "tool": "grep", - "input": { - "pattern": "&server\\{", - "glob": "pkg/storage/unified/resource/*.go" - }, - "output_len": 4018, - "is_error": false - }, - { - "round": 20, - "tool": "grep", - "input": { - "pattern": "tracingPrexfixSearch|tracingPrexfixBleve", - "glob": "*.go" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 21, - "tool": "grep", - "input": { - "pattern": "tracingPrexfix", - "glob": "*.go" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 22, - "tool": "grep", - "input": { - "pattern": "tracingPre[fx]", - "glob": "*.go" - }, - "output_len": 830, - "is_error": false - }, - { - "round": 22, - "tool": "grep", - "input": { - "pattern": "tracingPrefix", - "glob": "*.go" - }, - "output_len": 830, - "is_error": false - }, - { - "round": 23, - "tool": "grep", - "input": { - "pattern": "func.*TotalDocs", - "glob": "*.go" - }, - "output_len": 1333, - "is_error": false - }, - { - "round": 24, - "tool": "read_file", - "input": { - "path": "pkg/storage/unified/search/bleve.go", - "lines": "637-660" - }, - "output_len": 762, - "is_error": false - }, - { - "round": 25, - "tool": "read_file", - "input": { - "path": "pkg/storage/unified/resource/server.go", - "lines": "280-315" - }, - "output_len": 1245, - "is_error": false - }, - { - "round": 26, - "tool": "grep", - "input": { - "pattern": "func NewSearchServer", - "glob": "pkg/storage/unified/sql/*.go" - }, - "output_len": 503, - "is_error": false - }, - { - "round": 27, - "tool": "read_file", - "input": { - "path": "pkg/storage/unified/sql/server.go", - "lines": "74-100" - }, - "output_len": 1083, - "is_error": false - }, - { - "round": 28, - "tool": "read_file", - "input": { - "path": "pkg/storage/unified/sql/backend.go", - "lines": "120-155" - }, - "output_len": 1382, - "is_error": false - }, - { - "round": 29, - "tool": "grep", - "input": { - "pattern": "func \\(b \\*backend\\) GetResourceStats", - "glob": "pkg/storage/unified/sql/backend.go" - }, - "output_len": 579, - "is_error": false - }, - { - "round": 29, - "tool": "read_file", - "input": { - "path": "pkg/storage/unified/sql/backend.go", - "lines": "125-175" - }, - "output_len": 1942, - "is_error": false - } -] \ No newline at end of file diff --git a/pr_review_agent/output/traces/keycloak_greptile_pr1_trace.json b/pr_review_agent/output/traces/keycloak_greptile_pr1_trace.json deleted file mode 100644 index 3fbda5e..0000000 --- a/pr_review_agent/output/traces/keycloak_greptile_pr1_trace.json +++ /dev/null @@ -1,246 +0,0 @@ -[ - { - "round": 0, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "Find the isConditionalPasskeysEnabled method in UsernameForm and its parent classes" - }, - "output_len": 5315, - "is_error": false - }, - { - "round": 0, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "Who calls isConditionalPasskeysEnabled and where is it defined in the class hierarchy of UsernameForm, UsernamePasswordForm, AbstractUsernameFormAuthenticator" - }, - "output_len": 22293, - "is_error": false - }, - { - "round": 1, - "tool": "grep", - "input": { - "pattern": "isConditionalPasskeysEnabled", - "glob": "*.java" - }, - "output_len": 4131, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "services/src/main/java/org/keycloak/authentication/authenticators/browser/UsernameForm.java" - }, - "output_len": 4675, - "is_error": false - }, - { - "round": 2, - "tool": "grep", - "input": { - "pattern": "isConditionalPasskeysEnabled\\(\\)", - "glob": "*.java" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 2, - "tool": "grep", - "input": { - "pattern": "boolean isConditionalPasskeysEnabled", - "glob": "*.java" - }, - "output_len": 664, - "is_error": false - }, - { - "round": 3, - "tool": "read_file", - "input": { - "path": "services/src/main/java/org/keycloak/authentication/authenticators/browser/AbstractUsernameFormAuthenticator.java" - }, - "output_len": 13501, - "is_error": false - }, - { - "round": 3, - "tool": "read_file", - "input": { - "path": "services/src/main/java/org/keycloak/authentication/authenticators/browser/UsernamePasswordForm.java" - }, - "output_len": 7302, - "is_error": false - }, - { - "round": 4, - "tool": "grep", - "input": { - "pattern": "isConditionalPasskeysEnabled" - }, - "output_len": 4131, - "is_error": false - }, - { - "round": 4, - "tool": "read_file", - "input": { - "path": "services/src/main/java/org/keycloak/authentication/AbstractFormAuthenticator.java" - }, - "output_len": 1101, - "is_error": false - }, - { - "round": 5, - "tool": "read_file", - "input": { - "path": "services/src/main/java/org/keycloak/authentication/authenticators/browser/WebAuthnAuthenticator.java", - "lines": "85-135" - }, - "output_len": 2595, - "is_error": false - }, - { - "round": 5, - "tool": "read_file", - "input": { - "path": "services/src/main/java/org/keycloak/authentication/authenticators/browser/WebAuthnConditionalUIAuthenticator.java" - }, - "output_len": 2914, - "is_error": false - }, - { - "round": 6, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "Where is shouldDisplayAuthenticators defined and used in WebAuthnAuthenticator and WebAuthnConditionalUIAuthenticator" - }, - "output_len": 5061, - "is_error": false - }, - { - "round": 6, - "tool": "read_file", - "input": { - "path": "services/src/main/java/org/keycloak/authentication/authenticators/browser/WebAuthnAuthenticator.java", - "lines": "85-165" - }, - "output_len": 4030, - "is_error": false - }, - { - "round": 7, - "tool": "grep", - "input": { - "pattern": "configureTestRealm", - "glob": "**/PasskeysUsernamePasswordFormTest.java" - }, - "output_len": 727, - "is_error": false - }, - { - "round": 7, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "Find configureTestRealm method in the passkeys test base classes or AbstractPasskeysTest" - }, - "output_len": 1228, - "is_error": false - }, - { - "round": 8, - "tool": "read_file", - "input": { - "path": "testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/webauthn/passwordless/PasskeysUsernamePasswordFormTest.java", - "lines": "1-80" - }, - "output_len": 3275, - "is_error": false - }, - { - "round": 8, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "Find configureTestRealm in AbstractPasskeysTest or parent class of PasskeysUsernamePasswordFormTest" - }, - "output_len": 19627, - "is_error": false - }, - { - "round": 9, - "tool": "grep", - "input": { - "pattern": "isConditionalPasskeysEnabled\\b", - "glob": "*.java" - }, - "output_len": 4131, - "is_error": false - }, - { - "round": 9, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "Find all implementations and overloads of isConditionalPasskeysEnabled method in the entire codebase" - }, - "output_len": 17449, - "is_error": false - }, - { - "round": 10, - "tool": "grep", - "input": { - "pattern": "class UsernameForm extends", - "glob": "*.java" - }, - "output_len": 550, - "is_error": false - }, - { - "round": 10, - "tool": "grep", - "input": { - "pattern": "protected boolean isConditionalPasskeysEnabled\\(", - "glob": "*.java" - }, - "output_len": 664, - "is_error": false - }, - { - "round": 11, - "tool": "read_file", - "input": { - "path": "services/src/main/java/org/keycloak/authentication/authenticators/browser/WebAuthnPasswordlessAuthenticator.java" - }, - "output_len": 5366, - "is_error": false - }, - { - "round": 12, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "Find the FTL template that renders the webauth form with id='webauth' for conditional passkeys login" - }, - "output_len": 11650, - "is_error": false - }, - { - "round": 13, - "tool": "bash", - "input": { - "command": "cd /Users/tejas/personal/applymodel/zmisc/examples/pr_clones/keycloak && grep -rn \"void isConditionalPasskeysEnabled\\|boolean isConditionalPasskeysEnabled\" --include=\"*.java\" | head -20" - }, - "output_len": 181, - "is_error": false - }, - { - "round": 14, - "tool": "bash", - "input": { - "command": "cd /Users/tejas/personal/applymodel/zmisc/examples/pr_clones/keycloak && grep -rn \"isConditionalPasskeysEnabled()\" --include=\"*.java\" | head -20" - }, - "output_len": 11, - "is_error": false - } -] \ No newline at end of file diff --git a/pr_review_agent/output/traces/keycloak_pr33832_trace.json b/pr_review_agent/output/traces/keycloak_pr33832_trace.json deleted file mode 100644 index 5f08d4e..0000000 --- a/pr_review_agent/output/traces/keycloak_pr33832_trace.json +++ /dev/null @@ -1,153 +0,0 @@ -[ - { - "round": 0, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "Who implements CryptoProvider interface and what order() method do they return" - }, - "output_len": 6589, - "is_error": false - }, - { - "round": 0, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "How does CryptoIntegration.init work and who calls it" - }, - "output_len": 17, - "is_error": false - }, - { - "round": 0, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "concatenatedRSToASN1DER implementation in existing crypto providers" - }, - "output_len": 24903, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "authz/client/src/main/java/org/keycloak/authorization/client/util/crypto/AuthzClientCryptoProvider.java", - "lines": "100-125" - }, - "output_len": 1186, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "authz/client/src/main/java/org/keycloak/authorization/client/util/crypto/ASN1Decoder.java", - "lines": "50-90" - }, - "output_len": 1344, - "is_error": false - }, - { - "round": 2, - "tool": "read_file", - "input": { - "path": "common/src/main/java/org/keycloak/common/crypto/CryptoIntegration.java" - }, - "output_len": 4724, - "is_error": false - }, - { - "round": 2, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "ASN1Decoder readSequence and how readNext tracks byte count with mark/reset" - }, - "output_len": 6389, - "is_error": false - }, - { - "round": 3, - "tool": "read_file", - "input": { - "path": "authz/client/src/main/java/org/keycloak/authorization/client/AuthzClient.java", - "lines": "40-100" - }, - "output_len": 2487, - "is_error": false - }, - { - "round": 3, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "AuthzClient.create called in the codebase to understand usage patterns" - }, - "output_len": 3938, - "is_error": false - }, - { - "round": 4, - "tool": "read_file", - "input": { - "path": "authz/client/src/main/java/org/keycloak/authorization/client/util/crypto/ASN1Encoder.java" - }, - "output_len": 2935, - "is_error": false - }, - { - "round": 4, - "tool": "read_file", - "input": { - "path": "authz/client/src/main/java/org/keycloak/authorization/client/util/crypto/ASN1Decoder.java", - "lines": "76-83" - }, - "output_len": 236, - "is_error": false - }, - { - "round": 5, - "tool": "read_file", - "input": { - "path": "authz/client/src/main/java/org/keycloak/authorization/client/util/crypto/AuthzClientCryptoProvider.java", - "lines": "103-123" - }, - "output_len": 1028, - "is_error": false - }, - { - "round": 5, - "tool": "read_file", - "input": { - "path": "authz/client/src/test/java/org/keycloak/authorization/client/test/ECDSAAlgorithmTest.java" - }, - "output_len": 2709, - "is_error": false - }, - { - "round": 6, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "getBouncyCastleProvider usage in keycloak codebase and what happens if provider is null or incorrect" - }, - "output_len": 3877, - "is_error": false - }, - { - "round": 6, - "tool": "grep", - "input": { - "pattern": "default int order\\(\\)", - "glob": "*.java" - }, - "output_len": 1508, - "is_error": false - }, - { - "round": 7, - "tool": "grep", - "input": { - "pattern": "implements CryptoProvider", - "glob": "*.java" - }, - "output_len": 2155, - "is_error": false - } -] \ No newline at end of file diff --git a/pr_review_agent/output/traces/keycloak_pr36880_trace.json b/pr_review_agent/output/traces/keycloak_pr36880_trace.json deleted file mode 100644 index 3c31725..0000000 --- a/pr_review_agent/output/traces/keycloak_pr36880_trace.json +++ /dev/null @@ -1,232 +0,0 @@ -[ - { - "round": 0, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "ClientPermissions class definition and its canMapRoles, canMapCompositeRoles, canMapClientScopeRoles methods in V1" - }, - "output_len": 32116, - "is_error": false - }, - { - "round": 1, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "ClientPermissionsV2 parent class - the old ClientPermissions class that V2 extends from in services/src/main/java/org/keycloak/services/resources/admin/permissions" - }, - "output_len": 38372, - "is_error": false - }, - { - "round": 2, - "tool": "grep", - "input": { - "pattern": "class ClientPermissions", - "glob": "**/permissions/ClientPermissions.java" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 3, - "tool": "grep", - "input": { - "pattern": "class ClientPermissions", - "path": "services/src/main/java/org/keycloak/services/resources/admin/permissions" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 4, - "tool": "glob", - "input": { - "pattern": "**/admin/permissions/ClientPermissions*.java" - }, - "output_len": 17, - "is_error": false - }, - { - "round": 5, - "tool": "grep", - "input": { - "pattern": "class ClientPermissions" - }, - "output_len": 1198, - "is_error": false - }, - { - "round": 6, - "tool": "list_directory", - "input": { - "path": "services/src/main/java/org/keycloak/services/resources/admin/permissions" - }, - "output_len": 100, - "is_error": false - }, - { - "round": 7, - "tool": "grep", - "input": { - "pattern": "package org.keycloak.services.resources.admin.permissions" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 8, - "tool": "glob", - "input": { - "pattern": "**/admin/permissions/AdminPermissions.java" - }, - "output_len": 17, - "is_error": false - }, - { - "round": 9, - "tool": "bash", - "input": { - "command": "find . -name \"AdminPermissions.java\" -type f 2>/dev/null | head -10" - }, - "output_len": 89, - "is_error": false - }, - { - "round": 10, - "tool": "bash", - "input": { - "command": "find . -name \"MgmtPermissionsV2.java\" -type f 2>/dev/null | head -10" - }, - "output_len": 90, - "is_error": false - }, - { - "round": 11, - "tool": "read_file", - "input": { - "path": "services/src/main/java/org/keycloak/services/resources/admin/fgap/MgmtPermissionsV2.java" - }, - "output_len": 3274, - "is_error": false - }, - { - "round": 12, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "getResourceTypeResource method definition in AdminPermissionsSchema" - }, - "output_len": 837, - "is_error": false - }, - { - "round": 13, - "tool": "grep", - "input": { - "pattern": "getClientsWithPermission" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 14, - "tool": "grep", - "input": { - "pattern": "getClientIdsByScope|getClientsWithPermission" - }, - "output_len": 2591, - "is_error": false - }, - { - "round": 15, - "tool": "read_file", - "input": { - "path": "services/src/main/java/org/keycloak/services/resources/admin/fgap/ClientPermissionEvaluator.java" - }, - "output_len": 8142, - "is_error": false - }, - { - "round": 16, - "tool": "read_file", - "input": { - "path": "services/src/main/java/org/keycloak/services/resources/admin/fgap/AdminPermissions.java", - "lines": "60-100" - }, - "output_len": 1683, - "is_error": false - }, - { - "round": 17, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "PolicyStore findByResource method definition signature" - }, - "output_len": 9493, - "is_error": false - }, - { - "round": 18, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "UserConfigBuilder class definition with id method" - }, - "output_len": 5648, - "is_error": false - }, - { - "round": 19, - "tool": "grep", - "input": { - "pattern": "userAlice", - "path": "tests/base/src/test/java/org/keycloak/tests/admin/authz/fgap/UserResourceTypeEvaluationTest.java" - }, - "output_len": 8811, - "is_error": false - }, - { - "round": 20, - "tool": "grep", - "input": { - "pattern": "getClientsWithPermission" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 21, - "tool": "read_file", - "input": { - "path": "services/src/main/java/org/keycloak/services/resources/admin/fgap/ClientPermissionsV2.java", - "lines": "95-145" - }, - "output_len": 1916, - "is_error": false - }, - { - "round": 22, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "getClientsWithPermission method definition or declaration" - }, - "output_len": 112, - "is_error": false - }, - { - "round": 23, - "tool": "grep", - "input": { - "pattern": "getClientsWithPermission|getClientsWith" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 24, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "How is canViewClientDefault() method used and defined - parent class of ClientPermissionsV2" - }, - "output_len": 8512, - "is_error": false - } -] \ No newline at end of file diff --git a/pr_review_agent/output/traces/keycloak_pr36882_trace.json b/pr_review_agent/output/traces/keycloak_pr36882_trace.json deleted file mode 100644 index e2ae015..0000000 --- a/pr_review_agent/output/traces/keycloak_pr36882_trace.json +++ /dev/null @@ -1,362 +0,0 @@ -[ - { - "round": 0, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "Who uses RECREATE_UPGRADE_EXIT_CODE and how is exit code 4 or 3 referenced" - }, - "output_len": 5181, - "is_error": false - }, - { - "round": 0, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "callers of picocli.exit and what arguments they pass" - }, - "output_len": 566, - "is_error": false - }, - { - "round": 1, - "tool": "grep", - "input": { - "pattern": "RECREATE_UPGRADE_EXIT_CODE" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 1, - "tool": "grep", - "input": { - "pattern": "FEATURE_DISABLED_EXIT_CODE" - }, - "output_len": 625, - "is_error": false - }, - { - "round": 2, - "tool": "read_file", - "input": { - "path": "quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/compatibility/CompatibilityResult.java" - }, - "output_len": 120, - "is_error": false - }, - { - "round": 2, - "tool": "grep", - "input": { - "pattern": "RECREATE_UPGRADE" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 3, - "tool": "glob", - "input": { - "pattern": "**/CompatibilityResult.java" - }, - "output_len": 84, - "is_error": false - }, - { - "round": 3, - "tool": "glob", - "input": { - "pattern": "**/compatibility/*.java" - }, - "output_len": 1536, - "is_error": false - }, - { - "round": 4, - "tool": "read_file", - "input": { - "path": "server-spi-private/src/main/java/org/keycloak/compatibility/CompatibilityResult.java" - }, - "output_len": 2735, - "is_error": false - }, - { - "round": 5, - "tool": "grep", - "input": { - "pattern": "FEATURE_DISABLED" - }, - "output_len": 625, - "is_error": false - }, - { - "round": 5, - "tool": "grep", - "input": { - "pattern": "CompatibilityResult\\.FEATURE_DISABLED" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 6, - "tool": "bash", - "input": { - "command": "find . -name \"CompatibilityResult.java\" -type f 2>/dev/null" - }, - "output_len": 86, - "is_error": false - }, - { - "round": 7, - "tool": "grep", - "input": { - "pattern": "import.*CompatibilityResult", - "glob": "**/UpdateCompatibilityCheck.java" - }, - "output_len": 608, - "is_error": false - }, - { - "round": 7, - "tool": "grep", - "input": { - "pattern": "import.*CompatibilityResult", - "glob": "**/UpdateCompatibilityMetadata.java" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 8, - "tool": "read_file", - "input": { - "path": "quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/cli/command/UpdateCompatibilityCheck.java" - }, - "output_len": 4244, - "is_error": false - }, - { - "round": 8, - "tool": "read_file", - "input": { - "path": "quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/cli/command/UpdateCompatibilityMetadata.java" - }, - "output_len": 4333, - "is_error": false - }, - { - "round": 9, - "tool": "read_file", - "input": { - "path": "quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/cli/command/AbstractUpdatesCommand.java" - }, - "output_len": 3643, - "is_error": false - }, - { - "round": 9, - "tool": "grep", - "input": { - "pattern": "picocli\\.exit" - }, - "output_len": 2294, - "is_error": false - }, - { - "round": 10, - "tool": "read_file", - "input": { - "path": "common/src/main/java/org/keycloak/common/Profile.java", - "lines": "80-180" - }, - "output_len": 5209, - "is_error": false - }, - { - "round": 10, - "tool": "grep", - "input": { - "pattern": "getKey\\(\\)", - "glob": "**/Profile.java" - }, - "output_len": 2970, - "is_error": false - }, - { - "round": 11, - "tool": "read_file", - "input": { - "path": "common/src/main/java/org/keycloak/common/Profile.java", - "lines": "180-240" - }, - "output_len": 2915, - "is_error": false - }, - { - "round": 12, - "tool": "grep", - "input": { - "pattern": "public static boolean isFeatureEnabled", - "glob": "**/Profile.java" - }, - "output_len": 489, - "is_error": false - }, - { - "round": 13, - "tool": "read_file", - "input": { - "path": "common/src/main/java/org/keycloak/common/Profile.java", - "lines": "485-495" - }, - "output_len": 390, - "is_error": false - }, - { - "round": 14, - "tool": "grep", - "input": { - "pattern": "void exit\\(", - "glob": "**/Picocli.java" - }, - "output_len": 509, - "is_error": false - }, - { - "round": 15, - "tool": "grep", - "input": { - "pattern": "printError", - "glob": "**/AbstractUpdatesCommand.java" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 15, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "printError method in AbstractUpdatesCommand or AbstractCommand for picocli commands" - }, - "output_len": 8349, - "is_error": false - }, - { - "round": 16, - "tool": "grep", - "input": { - "pattern": "void printError", - "glob": "**/quarkus/runtime/**/*.java" - }, - "output_len": 621, - "is_error": false - }, - { - "round": 16, - "tool": "grep", - "input": { - "pattern": "printError", - "glob": "**/AbstractUpdatesCommand.java" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 17, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "How does the operator handle update compatibility exit codes and what values does it check" - }, - "output_len": 9879, - "is_error": false - }, - { - "round": 17, - "tool": "grep", - "input": { - "pattern": "exitCode|exit_code|ExitCode", - "glob": "**/operator/**/*.java" - }, - "output_len": 14669, - "is_error": false - }, - { - "round": 18, - "tool": "read_file", - "input": { - "path": "operator/src/main/java/org/keycloak/operator/controllers/KeycloakUpdateJobDependentResource.java", - "lines": "60-100" - }, - "output_len": 2307, - "is_error": false - }, - { - "round": 19, - "tool": "read_file", - "input": { - "path": "operator/src/main/java/org/keycloak/operator/controllers/KeycloakUpdateJobDependentResource.java", - "lines": "100-170" - }, - "output_len": 4039, - "is_error": false - }, - { - "round": 20, - "tool": "read_file", - "input": { - "path": "operator/src/main/java/org/keycloak/operator/controllers/KeycloakUpdateJobDependentResource.java", - "lines": "170-230" - }, - "output_len": 3354, - "is_error": false - }, - { - "round": 21, - "tool": "read_file", - "input": { - "path": "operator/src/main/java/org/keycloak/operator/update/impl/AutoUpdateLogic.java", - "lines": "106-110" - }, - "output_len": 289, - "is_error": false - }, - { - "round": 22, - "tool": "grep", - "input": { - "pattern": "FEATURE_DISABLED", - "glob": "**/*.java" - }, - "output_len": 625, - "is_error": false - }, - { - "round": 22, - "tool": "grep", - "input": { - "pattern": "rolling-update[^s]", - "glob": "**/operator/**/*.java" - }, - "output_len": 1608, - "is_error": false - }, - { - "round": 23, - "tool": "grep", - "input": { - "pattern": "ROLLING_UPGRADE_EXIT_CODE|RECREATE_UPGRADE_EXIT_CODE" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 24, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "Where are ROLLING_UPGRADE_EXIT_CODE and RECREATE_UPGRADE_EXIT_CODE used in the codebase" - }, - "output_len": 112, - "is_error": false - } -] \ No newline at end of file diff --git a/pr_review_agent/output/traces/keycloak_pr37038_trace.json b/pr_review_agent/output/traces/keycloak_pr37038_trace.json deleted file mode 100644 index bd6f305..0000000 --- a/pr_review_agent/output/traces/keycloak_pr37038_trace.json +++ /dev/null @@ -1,276 +0,0 @@ -[ - { - "round": 0, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "How is canManage() used for groups, what does it protect against? What are the callers of GroupPermissionEvaluator.canManage()?" - }, - "output_len": 41004, - "is_error": false - }, - { - "round": 0, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "resolveUser and resolveClient methods in AdminPermissionsSchema - what do they return, name or id?" - }, - "output_len": 2450, - "is_error": false - }, - { - "round": 1, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "resolveUser resolveClient in AdminPermissionsSchema - what format is the resource name? Is it the entity name or the entity id?" - }, - "output_len": 26545, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "server-spi-private/src/main/java/org/keycloak/authorization/AdminPermissionsSchema.java", - "lines": "85-130" - }, - "output_len": 110, - "is_error": false - }, - { - "round": 2, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "Who calls getOrCreateResource in AdminPermissionsSchema and how is the returned resource name used?" - }, - "output_len": 28885, - "is_error": false - }, - { - "round": 2, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "GroupPermissionsV2 getGroupIdsWithViewPermission - how is the result used to filter users by group membership?" - }, - "output_len": 17, - "is_error": false - }, - { - "round": 3, - "tool": "grep", - "input": { - "pattern": "resolveUser|resolveClient|resolveGroup", - "glob": "**/authorization/AdminPermissionsSchema.java" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 3, - "tool": "read_file", - "input": { - "path": "services/src/main/java/org/keycloak/services/resources/admin/permissions/GroupPermissionsV2.java", - "lines": "118-155" - }, - "output_len": 119, - "is_error": false - }, - { - "round": 4, - "tool": "glob", - "input": { - "pattern": "**/AdminPermissionsSchema.java" - }, - "output_len": 92, - "is_error": false - }, - { - "round": 4, - "tool": "glob", - "input": { - "pattern": "**/GroupPermissionsV2.java" - }, - "output_len": 89, - "is_error": false - }, - { - "round": 5, - "tool": "read_file", - "input": { - "path": "services/src/main/java/org/keycloak/services/resources/admin/fgap/GroupPermissionsV2.java" - }, - "output_len": 5528, - "is_error": false - }, - { - "round": 5, - "tool": "grep", - "input": { - "pattern": "resolveUser|resolveClient|resolveGroup", - "glob": "**/fgap/AdminPermissionsSchema.java" - }, - "output_len": 5424, - "is_error": false - }, - { - "round": 6, - "tool": "bash", - "input": { - "command": "find . -name \"AdminPermissionsSchema.java\" -type f 2>/dev/null" - }, - "output_len": 94, - "is_error": false - }, - { - "round": 6, - "tool": "bash", - "input": { - "command": "find . -name \"GroupPermissionsV2.java\" -type f 2>/dev/null" - }, - "output_len": 91, - "is_error": false - }, - { - "round": 7, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "How does searchForUserStream handle the GROUPS session attribute to filter users by group membership?" - }, - "output_len": 17, - "is_error": false - }, - { - "round": 7, - "tool": "read_file", - "input": { - "path": "services/src/main/java/org/keycloak/services/resources/admin/fgap/GroupPermissionsV2.java", - "lines": "57-105" - }, - "output_len": 1689, - "is_error": false - }, - { - "round": 8, - "tool": "bash", - "input": { - "command": "ls services/src/main/java/org/keycloak/services/resources/admin/permissions/ 2>/dev/null | head -20" - }, - "output_len": 11, - "is_error": false - }, - { - "round": 9, - "tool": "bash", - "input": { - "command": "find . -path \"*/admin/permissions/GroupPermissions*\" -o -path \"*/admin/permissions/UserPermissions*\" -o -path \"*/admin/permissions/MgmtPermissions*\" 2>/dev/null | head -20" - }, - "output_len": 11, - "is_error": false - }, - { - "round": 10, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "What is the difference between Resource.getId() and Resource.getName() in the authorization model?" - }, - "output_len": 16509, - "is_error": false - }, - { - "round": 10, - "tool": "grep", - "input": { - "pattern": "interface Resource", - "glob": "**/authorization/model/Resource.java" - }, - "output_len": 482, - "is_error": false - }, - { - "round": 11, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "resolveUser resolveClient in the V1 AdminPermissionsSchema (non-fgap package) - what do they return (username vs id)?" - }, - "output_len": 1065, - "is_error": false - }, - { - "round": 11, - "tool": "grep", - "input": { - "pattern": "canManageDefault|canViewDefault", - "glob": "**/*.java" - }, - "output_len": 3182, - "is_error": false - }, - { - "round": 12, - "tool": "grep", - "input": { - "pattern": "root\\.users\\(\\)\\.canManageDefault|root\\.users\\(\\)\\.canViewDefault|users\\(\\)\\.canManageDefault|users\\(\\)\\.canViewDefault", - "glob": "**/*.java" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 12, - "tool": "grep", - "input": { - "pattern": "canManageDefault\\(\\)|canViewDefault\\(\\)", - "glob": "**/*.java" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 13, - "tool": "grep", - "input": { - "pattern": "getGroupsWithViewPermission", - "glob": "**/*.java" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 14, - "tool": "read_file", - "input": { - "path": "services/src/main/java/org/keycloak/services/resources/admin/fgap/GroupPermissions.java", - "lines": "296-320" - }, - "output_len": 828, - "is_error": false - }, - { - "round": 15, - "tool": "grep", - "input": { - "pattern": "auth\\.groups\\(\\)\\.requireManage\\(\\)", - "glob": "**/*.java" - }, - "output_len": 559, - "is_error": false - }, - { - "round": 16, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "Where is auth.groups().canManage() (without parameter) called to check permission?" - }, - "output_len": 1134, - "is_error": false - }, - { - "round": 17, - "tool": "read_file", - "input": { - "path": "services/src/main/java/org/keycloak/services/DefaultKeycloakContext.java", - "lines": "330-360" - }, - "output_len": 1425, - "is_error": false - } -] \ No newline at end of file diff --git a/pr_review_agent/output/traces/keycloak_pr37429_trace.json b/pr_review_agent/output/traces/keycloak_pr37429_trace.json deleted file mode 100644 index 6d798cd..0000000 --- a/pr_review_agent/output/traces/keycloak_pr37429_trace.json +++ /dev/null @@ -1,591 +0,0 @@ -[ - { - "round": 0, - "tool": "read_file", - "input": { - "path": "misc/theme-verifier/src/main/java/org/keycloak/themeverifier/VerifyMessageProperties.java" - }, - "output_len": 16509, - "is_error": false - }, - { - "round": 0, - "tool": "read_file", - "input": { - "path": "misc/theme-verifier/src/test/java/org/keycloak/themeverifier/VerifyMessagePropertiesTest.java" - }, - "output_len": 7549, - "is_error": false - }, - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "How does the santizeAnchors method work and what does it do with anchor tags in translation values" - }, - "output_len": 112, - "is_error": false - }, - { - "round": 1, - "tool": "codebase_search", - "input": { - "search_string": "Lithuanian translation for totpStep1 in login messages - what language should it be in" - }, - "output_len": 6733, - "is_error": false - }, - { - "round": 1, - "tool": "grep", - "input": { - "pattern": "totpStep1=", - "glob": "**/messages_lt.properties" - }, - "output_len": 635, - "is_error": false - }, - { - "round": 1, - "tool": "grep", - "input": { - "pattern": "totpStep1=", - "glob": "**/messages_it.properties" - }, - "output_len": 635, - "is_error": false - }, - { - "round": 2, - "tool": "grep", - "input": { - "pattern": "totpStep1=", - "glob": "**/login/messages/messages_lt.properties" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 2, - "tool": "grep", - "input": { - "pattern": "totpStep1=", - "glob": "**/login/messages/messages_it.properties" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 2, - "tool": "grep", - "input": { - "pattern": "totpStep1=.*\u5728\u60a8\u7684\u624b\u6a5f\u4e0a\u5b89\u88dd\u4ee5\u4e0b\u61c9\u7528\u7a0b\u5f0f\u4e4b\u4e00", - "glob": "**/messages_zh_*.properties" - }, - "output_len": 547, - "is_error": false - }, - { - "round": 3, - "tool": "grep", - "input": { - "pattern": "loginTotpStep1=", - "glob": "**/login/messages/messages_lt.properties" - }, - "output_len": 622, - "is_error": false - }, - { - "round": 3, - "tool": "grep", - "input": { - "pattern": "loginTotpStep1=", - "glob": "**/login/messages/messages_it.properties" - }, - "output_len": 734, - "is_error": false - }, - { - "round": 3, - "tool": "grep", - "input": { - "pattern": "totpStep1=", - "glob": "**/account/messages/messages_zh_CN.properties" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 4, - "tool": "read_file", - "input": { - "path": "themes/src/main/resources-community/theme/base/account/messages/messages_zh_CN.properties", - "lines": "107-115" - }, - "output_len": 112, - "is_error": false - }, - { - "round": 4, - "tool": "grep", - "input": { - "pattern": "totpStep1=", - "glob": "**/account/messages/messages_zh_TW.properties" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 4, - "tool": "grep", - "input": { - "pattern": "\u5728\u60a8\u7684\u624b\u6a5f\u4e0a\u5b89\u88dd\u4ee5\u4e0b\u61c9\u7528\u7a0b\u5f0f\u4e4b\u4e00" - }, - "output_len": 547, - "is_error": false - }, - { - "round": 5, - "tool": "glob", - "input": { - "pattern": "**/account/messages/messages_zh*" - }, - "output_len": 399, - "is_error": false - }, - { - "round": 5, - "tool": "read_file", - "input": { - "path": "themes/src/main/resources-community/theme/base/account/messages/messages_zh_Hans.properties", - "lines": "135-145" - }, - "output_len": 335, - "is_error": false - }, - { - "round": 6, - "tool": "bash", - "input": { - "command": "find /Users/tejas/personal/applymodel/zmisc/examples/pr_clones/keycloak/themes/src/main/resources-community/theme/base/account/messages/ -name \"*zh*\"" - }, - "output_len": 319, - "is_error": false - }, - { - "round": 6, - "tool": "grep", - "input": { - "pattern": "totpStep1=", - "glob": "**/account/messages/messages_zh_CN.properties" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 7, - "tool": "bash", - "input": { - "command": "find /Users/tejas/personal/applymodel/zmisc/examples/pr_clones/keycloak -path \"*/account/messages/messages_zh_CN.properties\" 2>/dev/null" - }, - "output_len": 11, - "is_error": false - }, - { - "round": 7, - "tool": "read_file", - "input": { - "path": "themes/src/main/resources-community/theme/base/account/messages/messages_zh_Hans.properties", - "lines": "135-140" - }, - "output_len": 185, - "is_error": false - }, - { - "round": 8, - "tool": "grep", - "input": { - "pattern": "impersonateTitleHtml=", - "glob": "**/login/messages/messages_en.properties" - }, - "output_len": 549, - "is_error": false - }, - { - "round": 8, - "tool": "grep", - "input": { - "pattern": "error-invalid-multivalued-size=", - "glob": "**/login/messages/messages_sk.properties" - }, - "output_len": 745, - "is_error": false - }, - { - "round": 9, - "tool": "grep", - "input": { - "pattern": "totpStep1=", - "glob": "**/account/messages/messages_zh_Hant.properties" - }, - "output_len": 560, - "is_error": false - }, - { - "round": 9, - "tool": "grep", - "input": { - "pattern": "loginTotpStep1=", - "glob": "**/login/messages/messages_zh_CN.properties" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 10, - "tool": "read_file", - "input": { - "path": "themes/src/main/resources-community/theme/base/login/messages/messages_lt.properties", - "lines": "66-75" - }, - "output_len": 489, - "is_error": false - }, - { - "round": 10, - "tool": "grep", - "input": { - "pattern": "loginTotpStep1=", - "glob": "**/login/messages/messages_zh_CN.properties" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 11, - "tool": "bash", - "input": { - "command": "find /Users/tejas/personal/applymodel/zmisc/examples/pr_clones/keycloak -name \"messages_zh_CN.properties\" -path \"*/account/*\" 2>/dev/null" - }, - "output_len": 11, - "is_error": false - }, - { - "round": 11, - "tool": "grep", - "input": { - "pattern": "loginTotpStep1=", - "glob": "**/login/messages/messages_zh_Hans.properties" - }, - "output_len": 581, - "is_error": false - }, - { - "round": 12, - "tool": "read_file", - "input": { - "path": "themes/src/main/resources-community/theme/base/email/messages/messages_no.properties", - "lines": "1-10" - }, - "output_len": 2323, - "is_error": false - }, - { - "round": 12, - "tool": "grep", - "input": { - "pattern": "identityProviderLinkBody=", - "glob": "**/email/messages/messages_en.properties" - }, - "output_len": 1274, - "is_error": false - }, - { - "round": 13, - "tool": "read_file", - "input": { - "path": "themes/src/main/resources-community/theme/base/account/messages/messages_zh_Hans.properties", - "lines": "1-10" - }, - "output_len": 176, - "is_error": false - }, - { - "round": 13, - "tool": "grep", - "input": { - "pattern": "identityProviderLinkBody=.*.*.*\\{0\\}", - "glob": "**/email/messages/messages_no.properties" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 14, - "tool": "read_file", - "input": { - "path": "themes/src/main/resources-community/theme/base/login/messages/messages_sk.properties", - "lines": "25-32" - }, - "output_len": 282, - "is_error": false - }, - { - "round": 14, - "tool": "grep", - "input": { - "pattern": "impersonateTitleHtml=", - "glob": "**/login/messages/messages_sk.properties" - }, - "output_len": 571, - "is_error": false - }, - { - "round": 15, - "tool": "grep", - "input": { - "pattern": "error-invalid-multivalued-size", - "glob": "**/messages_en.properties" - }, - "output_len": 2717, - "is_error": false - }, - { - "round": 15, - "tool": "grep", - "input": { - "pattern": "error-invalid-multivalued-size", - "glob": "**/*.java" - }, - "output_len": 1381, - "is_error": false - }, - { - "round": 16, - "tool": "codebase_search", - "input": { - "search_string": "How are message properties with double curly braces placeholders like {{0}} processed in the admin-ui and account-ui React frontend" - }, - "output_len": 17, - "is_error": false - }, - { - "round": 16, - "tool": "grep", - "input": { - "pattern": "\"\\\\{\\\\{[0-9]+\\\\}\\\\}\"", - "glob": "js/apps/**/*.{ts,tsx,js}" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 17, - "tool": "grep", - "input": { - "pattern": "i18n|useTranslation|t\\(|formatMessage", - "glob": "js/apps/admin-ui/src/**/*.{ts,tsx}", - "path": "js/apps/admin-ui/src" - }, - "output_len": 146, - "is_error": false - }, - { - "round": 17, - "tool": "bash", - "input": { - "command": "head -40 /Users/tejas/personal/applymodel/zmisc/examples/pr_clones/keycloak/js/apps/admin-ui/maven-resources/theme/keycloak.v2/admin/messages/messages_en.properties" - }, - "output_len": 2550, - "is_error": false - }, - { - "round": 18, - "tool": "codebase_search", - "input": { - "search_string": "How does the Keycloak admin UI or account UI handle error-invalid-multivalued-size error message from the backend" - }, - "output_len": 6172, - "is_error": false - }, - { - "round": 18, - "tool": "grep", - "input": { - "pattern": "error-invalid-multivalued-size", - "glob": "**/*.{ts,tsx,js}" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 19, - "tool": "bash", - "input": { - "command": "grep \"error-invalid-multivalued-size\" /Users/tejas/personal/applymodel/zmisc/examples/pr_clones/keycloak/js/apps/admin-ui/maven-resources/theme/keycloak.v2/admin/messages/messages_en.properties" - }, - "output_len": 99, - "is_error": false - }, - { - "round": 19, - "tool": "bash", - "input": { - "command": "grep \"error-invalid-multivalued-size\" /Users/tejas/personal/applymodel/zmisc/examples/pr_clones/keycloak/js/apps/account-ui/maven-resources/theme/keycloak.v3/account/messages/messages_en.properties" - }, - "output_len": 99, - "is_error": false - }, - { - "round": 20, - "tool": "codebase_search", - "input": { - "search_string": "How does the Keycloak React frontend (admin-ui or account-ui) interpolate message properties with {{0}} placeholders" - }, - "output_len": 5271, - "is_error": false - }, - { - "round": 20, - "tool": "grep", - "input": { - "pattern": "interpolation|i18n|i18next", - "glob": "js/apps/admin-ui/src/**/i18n*.{ts,tsx}" - }, - "output_len": 2491, - "is_error": false - }, - { - "round": 21, - "tool": "codebase_search", - "input": { - "search_string": "How are user profile validation errors displayed in the admin-ui React frontend, specifically the error messages from user profile attributes" - }, - "output_len": 17, - "is_error": false - }, - { - "round": 22, - "tool": "grep", - "input": { - "pattern": "error-invalid|translat.*error|user.*profile.*error", - "glob": "js/apps/admin-ui/src/**/*.{ts,tsx}" - }, - "output_len": 1067, - "is_error": false - }, - { - "round": 23, - "tool": "grep", - "input": { - "pattern": "error.*message|responseError|errorMessage|fieldError", - "glob": "js/apps/admin-ui/src/user/**/*.{ts,tsx}" - }, - "output_len": 5176, - "is_error": false - }, - { - "round": 24, - "tool": "read_file", - "input": { - "path": "js/apps/admin-ui/src/user/utils/user-profile.ts" - }, - "output_len": 2715, - "is_error": false - }, - { - "round": 25, - "tool": "read_file", - "input": { - "path": "themes/src/main/resources-community/theme/base/login/messages/messages_sk.properties", - "lines": "495-500" - }, - "output_len": 433, - "is_error": false - }, - { - "round": 25, - "tool": "read_file", - "input": { - "path": "themes/src/main/resources-community/theme/base/login/messages/messages_lt.properties", - "lines": "66-75" - }, - "output_len": 489, - "is_error": false - }, - { - "round": 26, - "tool": "read_file", - "input": { - "path": "themes/src/main/resources-community/theme/base/account/messages/messages_zh_Hans.properties", - "lines": "135-142" - }, - "output_len": 262, - "is_error": false - }, - { - "round": 26, - "tool": "grep", - "input": { - "pattern": "totpStep1=", - "glob": "**/account/messages/messages_fi.properties" - }, - "output_len": 619, - "is_error": false - }, - { - "round": 27, - "tool": "read_file", - "input": { - "path": "misc/theme-verifier/src/test/resources/changedAnchor_de.properties" - }, - "output_len": 748, - "is_error": false - }, - { - "round": 27, - "tool": "read_file", - "input": { - "path": "misc/theme-verifier/src/test/resources/changedAnchor_en.properties" - }, - "output_len": 746, - "is_error": false - }, - { - "round": 27, - "tool": "read_file", - "input": { - "path": "misc/theme-verifier/src/test/resources/noHtml_en.properties" - }, - "output_len": 711, - "is_error": false - }, - { - "round": 28, - "tool": "bash", - "input": { - "command": "ls -la /Users/tejas/personal/applymodel/zmisc/examples/pr_clones/keycloak/misc/theme-verifier/src/test/resources/" - }, - "output_len": 1739, - "is_error": false - }, - { - "round": 29, - "tool": "bash", - "input": { - "command": "git -C /Users/tejas/personal/applymodel/zmisc/examples/pr_clones/keycloak log --oneline -5 -- \"themes/src/main/resources-community/theme/base/account/messages/messages_zh_Hans.properties\" 2>/dev/null || echo \"Not a git repo or file not tracked\"" - }, - "output_len": 54, - "is_error": false - }, - { - "round": 29, - "tool": "grep", - "input": { - "pattern": "loginTotpStep1=", - "glob": "**/login/messages/messages_no.properties" - }, - "output_len": 611, - "is_error": false - } -] \ No newline at end of file diff --git a/pr_review_agent/output/traces/keycloak_pr37634_trace.json b/pr_review_agent/output/traces/keycloak_pr37634_trace.json deleted file mode 100644 index 2d3f55e..0000000 --- a/pr_review_agent/output/traces/keycloak_pr37634_trace.json +++ /dev/null @@ -1,336 +0,0 @@ -[ - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "How is AccessTokenContext constructor called and what null checks does it perform on rawTokenId" - }, - "output_len": 9579, - "is_error": false - }, - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "isAccessTokenId matcher used in test assertions for token ID validation" - }, - "output_len": 2264, - "is_error": false - }, - { - "round": 0, - "tool": "grep", - "input": { - "pattern": "getShortcut\\(\\)", - "glob": "*GrantTypeFactory.java" - }, - "output_len": 6228, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "services/src/main/java/org/keycloak/protocol/oidc/encode/AccessTokenContext.java", - "lines": "69-80" - }, - "output_len": 370, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/AssertEvents.java", - "lines": "473-500" - }, - "output_len": 1325, - "is_error": false - }, - { - "round": 1, - "tool": "codebase_search", - "input": { - "search_string": "How does the copy constructor for OAuth2GrantType.Context get used, and what callers depend on it" - }, - "output_len": 17, - "is_error": false - }, - { - "round": 2, - "tool": "grep", - "input": { - "pattern": "new Context\\(context\\)|Context\\(Context", - "glob": "*.java" - }, - "output_len": 537, - "is_error": false - }, - { - "round": 2, - "tool": "read_file", - "input": { - "path": "services/src/main/java/org/keycloak/protocol/oidc/encode/DefaultTokenContextEncoderProvider.java", - "lines": "70-102" - }, - "output_len": 1965, - "is_error": false - }, - { - "round": 2, - "tool": "grep", - "input": { - "pattern": "substring\\(3, 5\\)|substring\\(4, 6\\)", - "glob": "*.java" - }, - "output_len": 2047, - "is_error": false - }, - { - "round": 3, - "tool": "codebase_search", - "input": { - "search_string": "Where is the RefreshTokenGrantType shortcut \"rt\" defined and does it collide with AccessTokenContext.TokenType.REGULAR shortcut \"rt\"" - }, - "output_len": 6182, - "is_error": false - }, - { - "round": 3, - "tool": "read_file", - "input": { - "path": "services/src/main/java/org/keycloak/protocol/oidc/grants/ResourceOwnerPasswordCredentialsGrantType.java", - "lines": "125-145" - }, - "output_len": 1323, - "is_error": false - }, - { - "round": 3, - "tool": "codebase_search", - "input": { - "search_string": "ClientCredentialsGrantType process method where it creates access token and sets grant type in clientSessionCtx" - }, - "output_len": 210, - "is_error": false - }, - { - "round": 4, - "tool": "grep", - "input": { - "pattern": "class ClientCredentialsGrantType", - "glob": "*.java" - }, - "output_len": 1140, - "is_error": false - }, - { - "round": 4, - "tool": "read_file", - "input": { - "path": "services/src/main/java/org/keycloak/protocol/oidc/grants/OAuth2GrantTypeBase.java", - "lines": "95-120" - }, - "output_len": 1574, - "is_error": false - }, - { - "round": 4, - "tool": "codebase_search", - "input": { - "search_string": "Where is createTokenResponse called and which grant types use it vs have their own token creation" - }, - "output_len": 17, - "is_error": false - }, - { - "round": 5, - "tool": "read_file", - "input": { - "path": "services/src/main/java/org/keycloak/protocol/oidc/grants/ClientCredentialsGrantType.java", - "lines": "55-130" - }, - "output_len": 4830, - "is_error": false - }, - { - "round": 5, - "tool": "read_file", - "input": { - "path": "services/src/main/java/org/keycloak/protocol/oidc/grants/ResourceOwnerPasswordCredentialsGrantType.java", - "lines": "130-155" - }, - "output_len": 1328, - "is_error": false - }, - { - "round": 6, - "tool": "read_file", - "input": { - "path": "services/src/main/java/org/keycloak/protocol/oidc/grants/ClientCredentialsGrantType.java", - "lines": "130-170" - }, - "output_len": 1989, - "is_error": false - }, - { - "round": 6, - "tool": "grep", - "input": { - "pattern": "createTokenResponse\\(|createTokenResponseBuilder\\(", - "glob": "*.java" - }, - "output_len": 7404, - "is_error": false - }, - { - "round": 7, - "tool": "grep", - "input": { - "pattern": "new Context\\(", - "glob": "*.java", - "path": "services/src/main/java/org/keycloak/protocol/oidc" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 7, - "tool": "read_file", - "input": { - "path": "services/src/main/java/org/keycloak/protocol/oidc/grants/OAuth2GrantTypeBase.java", - "lines": "105-120" - }, - "output_len": 1135, - "is_error": false - }, - { - "round": 7, - "tool": "codebase_search", - "input": { - "search_string": "How does the RefreshTokenGrantType set the grant type attribute on clientSessionContext before token creation" - }, - "output_len": 17, - "is_error": false - }, - { - "round": 8, - "tool": "read_file", - "input": { - "path": "services/src/main/java/org/keycloak/protocol/oidc/TokenManager.java", - "lines": "240-260" - }, - "output_len": 1164, - "is_error": false - }, - { - "round": 8, - "tool": "read_file", - "input": { - "path": "services/src/main/java/org/keycloak/protocol/oidc/grants/PreAuthorizedCodeGrantType.java", - "lines": "75-100" - }, - "output_len": 1587, - "is_error": false - }, - { - "round": 9, - "tool": "read_file", - "input": { - "path": "services/src/main/java/org/keycloak/protocol/oidc/grants/PreAuthorizedCodeGrantType.java", - "lines": "100-130" - }, - "output_len": 1709, - "is_error": false - }, - { - "round": 9, - "tool": "read_file", - "input": { - "path": "services/src/test/java/org/keycloak/protocol/oidc/encode/DefaultTokenContextEncoderProviderTest.java", - "lines": "55-114" - }, - "output_len": 2792, - "is_error": false - }, - { - "round": 10, - "tool": "read_file", - "input": { - "path": "services/src/main/java/org/keycloak/protocol/oidc/encode/DefaultTokenContextEncoderProviderFactory.java", - "lines": "65-100" - }, - "output_len": 1816, - "is_error": false - }, - { - "round": 10, - "tool": "read_file", - "input": { - "path": "services/src/main/java/org/keycloak/protocol/oidc/tokenexchange/StandardTokenExchangeProvider.java", - "lines": "230-245" - }, - "output_len": 1145, - "is_error": false - }, - { - "round": 11, - "tool": "read_file", - "input": { - "path": "services/src/main/java/org/keycloak/protocol/oidc/tokenexchange/StandardTokenExchangeProvider.java", - "lines": "245-275" - }, - "output_len": 2276, - "is_error": false - }, - { - "round": 12, - "tool": "grep", - "input": { - "pattern": "getShouldUseLightweightToken", - "glob": "*.java" - }, - "output_len": 4750, - "is_error": false - }, - { - "round": 13, - "tool": "read_file", - "input": { - "path": "server-spi-private/src/main/java/org/keycloak/protocol/oidc/grants/OAuth2GrantType.java", - "lines": "85-110" - }, - "output_len": 1191, - "is_error": false - }, - { - "round": 14, - "tool": "read_file", - "input": { - "path": "server-spi-private/src/main/java/org/keycloak/protocol/oidc/grants/OAuth2GrantType.java", - "lines": "110-155" - }, - "output_len": 2159, - "is_error": false - }, - { - "round": 15, - "tool": "grep", - "input": { - "pattern": "setFormParams", - "glob": "*.java" - }, - "output_len": 602, - "is_error": false - }, - { - "round": 16, - "tool": "grep", - "input": { - "pattern": "\\.setFormParams\\(", - "glob": "*.java" - }, - "output_len": 10, - "is_error": false - } -] \ No newline at end of file diff --git a/pr_review_agent/output/traces/keycloak_pr38446_trace.json b/pr_review_agent/output/traces/keycloak_pr38446_trace.json deleted file mode 100644 index 56eeeef..0000000 --- a/pr_review_agent/output/traces/keycloak_pr38446_trace.json +++ /dev/null @@ -1,414 +0,0 @@ -[ - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "How does getFederatedCredentialsStream work and what does it return for user credential manager?" - }, - "output_len": 6887, - "is_error": false - }, - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "RecoveryAuthnCodesCredentialModel.createFromValues method signature and parameters" - }, - "output_len": 5414, - "is_error": false - }, - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "UserCredentialModel constructor and what getChallengeResponse returns" - }, - "output_len": 7568, - "is_error": false - }, - { - "round": 1, - "tool": "codebase_search", - "input": { - "search_string": "How does BackwardsCompatibilityUserStorage implement CredentialInputUpdater and is getCredentials method used by federated credential stream" - }, - "output_len": 22077, - "is_error": false - }, - { - "round": 1, - "tool": "codebase_search", - "input": { - "search_string": "RecoveryAuthnCodesFormAuthenticator authenticate method and how it checks credentials for federated users" - }, - "output_len": 10173, - "is_error": false - }, - { - "round": 1, - "tool": "grep", - "input": { - "pattern": "class BackwardsCompatibilityUserStorage" - }, - "output_len": 3162, - "is_error": false - }, - { - "round": 2, - "tool": "read_file", - "input": { - "path": "services/src/main/java/org/keycloak/authentication/requiredactions/RecoveryAuthnCodesAction.java" - }, - "output_len": 8581, - "is_error": false - }, - { - "round": 2, - "tool": "codebase_search", - "input": { - "search_string": "How does RecoveryAuthnCodesCredentialProvider validate recovery codes - isValid method implementation" - }, - "output_len": 7292, - "is_error": false - }, - { - "round": 2, - "tool": "grep", - "input": { - "pattern": "createRecoveryCodesCredential" - }, - "output_len": 2017, - "is_error": false - }, - { - "round": 3, - "tool": "codebase_search", - "input": { - "search_string": "UserCredentialManager isValid method implementation that routes to user storage providers or credential providers" - }, - "output_len": 24083, - "is_error": false - }, - { - "round": 3, - "tool": "grep", - "input": { - "pattern": "updateCredential.*CredentialInput", - "path": "model/storage/src/main/java/org/keycloak/credential" - }, - "output_len": 609, - "is_error": false - }, - { - "round": 4, - "tool": "grep", - "input": { - "pattern": "getRecoveryAuthnCodeToEnterNumber" - }, - "output_len": 3328, - "is_error": false - }, - { - "round": 4, - "tool": "codebase_search", - "input": { - "search_string": "EnterRecoveryAuthnCodePage getRecoveryAuthnCodeToEnterNumber implementation" - }, - "output_len": 2437, - "is_error": false - }, - { - "round": 4, - "tool": "grep", - "input": { - "pattern": "RecoveryAuthnCodesCredentialProviderFactory.PROVIDER_ID" - }, - "output_len": 817, - "is_error": false - }, - { - "round": 5, - "tool": "grep", - "input": { - "pattern": "class RecoveryAuthnCodesCredentialProviderFactory" - }, - "output_len": 674, - "is_error": false - }, - { - "round": 5, - "tool": "grep", - "input": { - "pattern": "PROVIDER_ID.*=.*keycloak-recovery" - }, - "output_len": 577, - "is_error": false - }, - { - "round": 5, - "tool": "codebase_search", - "input": { - "search_string": "RecoveryAuthnCodesCredentialProviderFactory PROVIDER_ID constant definition" - }, - "output_len": 1105, - "is_error": false - }, - { - "round": 6, - "tool": "read_file", - "input": { - "path": "testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/federation/storage/BackwardsCompatibilityUserStorageTest.java", - "lines": "407-500" - }, - "output_len": 4471, - "is_error": false - }, - { - "round": 6, - "tool": "read_file", - "input": { - "path": "server-spi-private/src/main/java/org/keycloak/utils/CredentialHelper.java", - "lines": "95-140" - }, - "output_len": 2603, - "is_error": false - }, - { - "round": 7, - "tool": "grep", - "input": { - "pattern": "class SetupRecoveryAuthnCodesPage" - }, - "output_len": 641, - "is_error": false - }, - { - "round": 7, - "tool": "grep", - "input": { - "pattern": "uncheckLogoutSessions|checkLogoutSessions", - "glob": "**/SetupRecoveryAuthnCodesPage.java" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 7, - "tool": "codebase_search", - "input": { - "search_string": "SetupRecoveryAuthnCodesPage implementation including checkLogoutSessions and uncheckLogoutSessions methods" - }, - "output_len": 5240, - "is_error": false - }, - { - "round": 8, - "tool": "grep", - "input": { - "pattern": "logout-sessions.*check", - "glob": "**/*.ftl" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 8, - "tool": "codebase_search", - "input": { - "search_string": "recovery-codes-setup-form.ftl or login-recovery-authn-code-config.ftl logout-sessions checkbox default state" - }, - "output_len": 9819, - "is_error": false - }, - { - "round": 9, - "tool": "grep", - "input": { - "pattern": "logout-sessions.*checked|checked.*logout-sessions", - "glob": "**/*.ftl" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 9, - "tool": "grep", - "input": { - "pattern": "logoutOtherSessions", - "glob": "**/*.ftl" - }, - "output_len": 7577, - "is_error": false - }, - { - "round": 10, - "tool": "grep", - "input": { - "pattern": "setupOTPForUserWithRequiredAction" - }, - "output_len": 3851, - "is_error": false - }, - { - "round": 10, - "tool": "grep", - "input": { - "pattern": "field\\.checkbox.*name.*logout-sessions", - "glob": "**/*.ftl" - }, - "output_len": 598, - "is_error": false - }, - { - "round": 11, - "tool": "grep", - "input": { - "pattern": "macro checkbox", - "glob": "**/*.ftl" - }, - "output_len": 495, - "is_error": false - }, - { - "round": 12, - "tool": "read_file", - "input": { - "path": "themes/src/main/resources/theme/keycloak.v2/login/field.ftl", - "lines": "146-165" - }, - "output_len": 641, - "is_error": false - }, - { - "round": 13, - "tool": "grep", - "input": { - "pattern": "class LoginConfigTotpPage" - }, - "output_len": 612, - "is_error": false - }, - { - "round": 14, - "tool": "read_file", - "input": { - "path": "testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/federation/storage/BackwardsCompatibilityUserStorageTest.java", - "lines": "380-406" - }, - "output_len": 1235, - "is_error": false - }, - { - "round": 15, - "tool": "read_file", - "input": { - "path": "testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/federation/storage/BackwardsCompatibilityUserStorageTest.java", - "lines": "408-435" - }, - "output_len": 1268, - "is_error": false - }, - { - "round": 16, - "tool": "bash", - "input": { - "command": "cd /Users/tejas/personal/applymodel/zmisc/examples/pr_clones/keycloak && sed -n '418,425p' testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/federation/storage/BackwardsCompatibilityUserStorageTest.java" - }, - "output_len": 358, - "is_error": false - }, - { - "round": 17, - "tool": "grep", - "input": { - "pattern": "delayed-suthenticator" - }, - "output_len": 1939, - "is_error": false - }, - { - "round": 17, - "tool": "grep", - "input": { - "pattern": "delayed-authenticator" - }, - "output_len": 2871, - "is_error": false - }, - { - "round": 18, - "tool": "grep", - "input": { - "pattern": "RecoveryAuthnCodesUtils\\.getCredential" - }, - "output_len": 1520, - "is_error": false - }, - { - "round": 18, - "tool": "read_file", - "input": { - "path": "services/src/main/java/org/keycloak/forms/login/freemarker/model/RecoveryAuthnCodeInputLoginBean.java" - }, - "output_len": 1084, - "is_error": false - }, - { - "round": 19, - "tool": "read_file", - "input": { - "path": "testsuite/integration-arquillian/servers/auth-server/services/testsuite-providers/src/main/java/org/keycloak/testsuite/federation/BackwardsCompatibilityUserStorage.java", - "lines": "217-255" - }, - "output_len": 1620, - "is_error": false - }, - { - "round": 20, - "tool": "read_file", - "input": { - "path": "testsuite/integration-arquillian/servers/auth-server/services/testsuite-providers/src/main/java/org/keycloak/testsuite/federation/BackwardsCompatibilityUserStorage.java", - "lines": "265-285" - }, - "output_len": 907, - "is_error": false - }, - { - "round": 21, - "tool": "read_file", - "input": { - "path": "server-spi/src/main/java/org/keycloak/models/credential/RecoveryAuthnCodesCredentialModel.java", - "lines": "57-84" - }, - "output_len": 1661, - "is_error": false - }, - { - "round": 22, - "tool": "codebase_search", - "input": { - "search_string": "Where is hashRawCode defined in RecoveryAuthnCodesUtils and what type does it expect" - }, - "output_len": 3607, - "is_error": false - }, - { - "round": 23, - "tool": "read_file", - "input": { - "path": "server-spi-private/src/main/java/org/keycloak/utils/CredentialHelper.java", - "lines": "78-135" - }, - "output_len": 3578, - "is_error": false - }, - { - "round": 24, - "tool": "grep", - "input": { - "pattern": "default Stream getCredentials" - }, - "output_len": 1162, - "is_error": false - } -] \ No newline at end of file diff --git a/pr_review_agent/output/traces/keycloak_pr40940_trace.json b/pr_review_agent/output/traces/keycloak_pr40940_trace.json deleted file mode 100644 index 6324010..0000000 --- a/pr_review_agent/output/traces/keycloak_pr40940_trace.json +++ /dev/null @@ -1,142 +0,0 @@ -[ - { - "round": 0, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "GroupAdapter getGroupModel method and modelSupplier field definition" - }, - "output_len": 12255, - "is_error": false - }, - { - "round": 0, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "Who calls getSubGroupsCount on GroupModel and how do they handle null return" - }, - "output_len": 17622, - "is_error": false - }, - { - "round": 1, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "How are other getSubGroupsStream methods in infinispan GroupAdapter handling modelSupplier.get() null check" - }, - "output_len": 2201, - "is_error": false - }, - { - "round": 1, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "GroupRepresentation setSubGroupCount method and what happens when null Long is passed" - }, - "output_len": 17, - "is_error": false - }, - { - "round": 2, - "tool": "grep", - "input": { - "pattern": "setSubGroupCount", - "glob": "*.java" - }, - "output_len": 3047, - "is_error": false - }, - { - "round": 2, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "GroupRepresentation subGroupCount field type and getter setter" - }, - "output_len": 6254, - "is_error": false - }, - { - "round": 3, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "LazyModel class implementation in cache infinispan - how does it cache the result of the supplier" - }, - "output_len": 635, - "is_error": false - }, - { - "round": 3, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "Thread join in keycloak tests to ensure background thread completes before assertions" - }, - "output_len": 1925, - "is_error": false - }, - { - "round": 4, - "tool": "read_file", - "input": { - "path": "tests/base/src/test/java/org/keycloak/tests/admin/group/GroupTest.java", - "lines": "114-162" - }, - "output_len": 1970, - "is_error": false - }, - { - "round": 5, - "tool": "read_file", - "input": { - "path": "model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/entities/CachedGroup.java" - }, - "output_len": 4594, - "is_error": false - }, - { - "round": 5, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "AbstractRevisioned or parent class of CachedGroup that has getRealm method" - }, - "output_len": 1596, - "is_error": false - }, - { - "round": 6, - "tool": "grep", - "input": { - "pattern": "groupMatchesSearchOrIsPathElement", - "glob": "*.java" - }, - "output_len": 1365, - "is_error": false - }, - { - "round": 7, - "tool": "read_file", - "input": { - "path": "model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/entities/CachedGroup.java", - "lines": "56-58" - }, - "output_len": 160, - "is_error": false - }, - { - "round": 8, - "tool": "read_file", - "input": { - "path": "model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/GroupAdapter.java", - "lines": "255-278" - }, - "output_len": 1252, - "is_error": false - }, - { - "round": 9, - "tool": "read_file", - "input": { - "path": "model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/LazyModel.java" - }, - "output_len": 477, - "is_error": false - } -] \ No newline at end of file diff --git a/pr_review_agent/output/traces/sentry_greptile_pr1_trace.json b/pr_review_agent/output/traces/sentry_greptile_pr1_trace.json deleted file mode 100644 index 1d1ed42..0000000 --- a/pr_review_agent/output/traces/sentry_greptile_pr1_trace.json +++ /dev/null @@ -1,442 +0,0 @@ -[ - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "Does Django QuerySet support negative indexing or negative slicing?" - }, - "output_len": 110, - "is_error": false - }, - { - "round": 0, - "tool": "grep", - "input": { - "pattern": "^import math|^from math", - "glob": "src/sentry/api/paginator.py" - }, - "output_len": 368, - "is_error": false - }, - { - "round": 0, - "tool": "read_file", - "input": { - "path": "src/sentry/api/paginator.py", - "lines": "1-30" - }, - "output_len": 862, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "src/sentry/api/paginator.py", - "lines": "160-200" - }, - "output_len": 1894, - "is_error": false - }, - { - "round": 1, - "tool": "grep", - "input": { - "pattern": "organization_context", - "glob": "src/sentry/api/endpoints/organization_auditlogs.py" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "src/sentry/api/endpoints/organization_auditlogs.py" - }, - "output_len": 201, - "is_error": false - }, - { - "round": 2, - "tool": "grep", - "input": { - "pattern": "organization_auditlogs", - "glob": "**/core/endpoints/*.py" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 2, - "tool": "glob", - "input": { - "pattern": "**/core/endpoints/organization_auditlogs.py" - }, - "output_len": 51, - "is_error": false - }, - { - "round": 3, - "tool": "read_file", - "input": { - "path": "src/sentry/core/endpoints/organization_auditlogs.py" - }, - "output_len": 3309, - "is_error": false - }, - { - "round": 4, - "tool": "grep", - "input": { - "pattern": "class RpcUserOrganizationContext" - }, - "output_len": 427, - "is_error": false - }, - { - "round": 4, - "tool": "grep", - "input": { - "pattern": "Negative indexing is not supported", - "glob": "*.py" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 4, - "tool": "grep", - "input": { - "pattern": "has_global_access", - "glob": "**/organization/model.py" - }, - "output_len": 983, - "is_error": false - }, - { - "round": 5, - "tool": "read_file", - "input": { - "path": "src/sentry/organizations/services/organization/model.py", - "lines": "330-355" - }, - "output_len": 1523, - "is_error": false - }, - { - "round": 5, - "tool": "read_file", - "input": { - "path": "src/sentry/api/paginator.py", - "lines": "170-200" - }, - "output_len": 1501, - "is_error": false - }, - { - "round": 6, - "tool": "read_file", - "input": { - "path": "src/sentry/api/paginator.py", - "lines": "125-185" - }, - "output_len": 2477, - "is_error": false - }, - { - "round": 6, - "tool": "grep", - "input": { - "pattern": "class DateTimePaginator", - "glob": "src/sentry/api/paginator.py" - }, - "output_len": 353, - "is_error": false - }, - { - "round": 7, - "tool": "read_file", - "input": { - "path": "src/sentry/api/paginator.py", - "lines": "76-100" - }, - "output_len": 1310, - "is_error": false - }, - { - "round": 7, - "tool": "grep", - "input": { - "pattern": "class BasePaginator", - "glob": "src/sentry/api/paginator.py" - }, - "output_len": 327, - "is_error": false - }, - { - "round": 8, - "tool": "read_file", - "input": { - "path": "src/sentry/api/paginator.py", - "lines": "137-190" - }, - "output_len": 2484, - "is_error": false - }, - { - "round": 8, - "tool": "read_file", - "input": { - "path": "src/sentry/utils/cursors.py" - }, - "output_len": 10237, - "is_error": false - }, - { - "round": 9, - "tool": "grep", - "input": { - "pattern": "def paginate", - "glob": "src/sentry/api/base.py" - }, - "output_len": 643, - "is_error": false - }, - { - "round": 9, - "tool": "grep", - "input": { - "pattern": "enable_advanced_features", - "glob": "*.py" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 9, - "tool": "grep", - "input": { - "pattern": "AssertionError.*Negative indexing", - "glob": "*.py" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 10, - "tool": "read_file", - "input": { - "path": "src/sentry/api/base.py", - "lines": "516-590" - }, - "output_len": 2871, - "is_error": false - }, - { - "round": 11, - "tool": "grep", - "input": { - "pattern": "def get_paginator", - "glob": "*.py" - }, - "output_len": 384, - "is_error": false - }, - { - "round": 12, - "tool": "read_file", - "input": { - "path": "src/sentry/utils/pagination_factory.py", - "lines": "56-80" - }, - "output_len": 777, - "is_error": false - }, - { - "round": 13, - "tool": "read_file", - "input": { - "path": "src/sentry/api/paginator.py", - "lines": "60-76" - }, - "output_len": 616, - "is_error": false - }, - { - "round": 14, - "tool": "grep", - "input": { - "pattern": "class ControlSiloOrganizationEndpoint", - "glob": "*.py" - }, - "output_len": 375, - "is_error": false - }, - { - "round": 14, - "tool": "grep", - "input": { - "pattern": "class OrganizationAuditPermission", - "glob": "*.py" - }, - "output_len": 416, - "is_error": false - }, - { - "round": 15, - "tool": "read_file", - "input": { - "path": "src/sentry/api/bases/organization.py", - "lines": "251-320" - }, - "output_len": 3289, - "is_error": false - }, - { - "round": 16, - "tool": "read_file", - "input": { - "path": "src/sentry/api/paginator.py", - "lines": "227-290" - }, - "output_len": 2414, - "is_error": false - }, - { - "round": 17, - "tool": "grep", - "input": { - "pattern": "OptimizedCursorPaginator", - "glob": "*.py" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 17, - "tool": "grep", - "input": { - "pattern": "optimized_pagination", - "glob": "*.py" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 18, - "tool": "grep", - "input": { - "pattern": "def get_cursor_from_request", - "glob": "*.py" - }, - "output_len": 365, - "is_error": false - }, - { - "round": 19, - "tool": "read_file", - "input": { - "path": "src/sentry/api/base.py", - "lines": "510-520" - }, - "output_len": 384, - "is_error": false - }, - { - "round": 20, - "tool": "grep", - "input": { - "pattern": "def get_cursor", - "glob": "*.py" - }, - "output_len": 1169, - "is_error": false - }, - { - "round": 21, - "tool": "read_file", - "input": { - "path": "src/sentry/utils/pagination_factory.py", - "lines": "46-55" - }, - "output_len": 306, - "is_error": false - }, - { - "round": 22, - "tool": "read_file", - "input": { - "path": "src/sentry/api/paginator.py", - "lines": "815-912" - }, - "output_len": 1449, - "is_error": false - }, - { - "round": 23, - "tool": "read_file", - "input": { - "path": "src/sentry/api/bases/organization.py", - "lines": "37-110" - }, - "output_len": 2576, - "is_error": false - }, - { - "round": 24, - "tool": "grep", - "input": { - "pattern": "def determine_access", - "glob": "src/sentry/api/bases/organization.py" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 25, - "tool": "grep", - "input": { - "pattern": "def determine_access", - "glob": "src/sentry/api/base.py" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 26, - "tool": "bash", - "input": { - "command": "cd /Users/tejas/personal/applymodel/zmisc/examples/pr_clones/sentry && grep -r \"Negative indexing\" --include=\"*.py\" | head -5" - }, - "output_len": 11, - "is_error": false - }, - { - "round": 27, - "tool": "bash", - "input": { - "command": "cd /Users/tejas/personal/applymodel/zmisc/examples/pr_clones/sentry && python -c \"\nfrom django.db.models.query import QuerySet\nimport inspect\nsource = inspect.getsource(QuerySet.__getitem__)\nprint(source[:500])\n\" 2>/dev/null || echo \"Cannot run Django in this env\"" - }, - "output_len": 29, - "is_error": false - }, - { - "round": 28, - "tool": "bash", - "input": { - "command": "find /Users/tejas/personal/applymodel/zmisc/examples/pr_clones/sentry -path \"*/django/db/models/query.py\" 2>/dev/null | head -1" - }, - "output_len": 11, - "is_error": false - }, - { - "round": 29, - "tool": "bash", - "input": { - "command": "pip show django 2>/dev/null | head -5; python3 -c \"\nimport django.db.models.sql.query\nimport inspect\n# Find the __getitem__ on QuerySet\nfrom django.db.models.query import QuerySet\nsrc = inspect.getsource(QuerySet.__getitem__)\n# Look for negative indexing assertion\nfor line in src.split('\\n'):\n if 'Negative' in line or 'assert' in line.lower() or 'negative' in line.lower():\n print(line)\n\" 2>/dev/null || echo \"Django not available, but this is well-documented behavior\"" - }, - "output_len": 58, - "is_error": false - } -] \ No newline at end of file diff --git a/pr_review_agent/output/traces/sentry_greptile_pr5_trace.json b/pr_review_agent/output/traces/sentry_greptile_pr5_trace.json deleted file mode 100644 index 30c810f..0000000 --- a/pr_review_agent/output/traces/sentry_greptile_pr5_trace.json +++ /dev/null @@ -1,466 +0,0 @@ -[ - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "PRCommentWorkflow get_merged_pr_single_issue_template method definition" - }, - "output_len": 439, - "is_error": false - }, - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "fetch_error_details nodestore get_multi ordering guarantee" - }, - "output_len": 4547, - "is_error": false - }, - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "FlexBox height 100% usage in traceWaterfall" - }, - "output_len": 3546, - "is_error": false - }, - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "overflowEllipsis usage in FlexCenter for event attachments" - }, - "output_len": 3567, - "is_error": false - }, - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "analytics record preprod_artifact before feature check" - }, - "output_len": 110, - "is_error": false - }, - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "get_slow_conditions_for_groups return type and behavior" - }, - "output_len": 755, - "is_error": false - }, - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "TableWidgetVisualization columns and tableData props used in dashboard chart" - }, - "output_len": 17032, - "is_error": false - }, - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "Attributes filtering with HIDDEN_ATTRIBUTES and empty searchQuery" - }, - "output_len": 9683, - "is_error": false - }, - { - "round": 1, - "tool": "codebase_search", - "input": { - "search_string": "nodestore get_multi dictionary ordering guarantee - does it preserve insertion order or return in different order" - }, - "output_len": 2023, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "src/sentry/replays/endpoints/project_replay_summarize_breadcrumbs.py", - "lines": "82-110" - }, - "output_len": 91, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "src/sentry/integrations/source_code_management/commit_context.py", - "lines": "570-650" - }, - "output_len": 3521, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "src/sentry/preprod/api/endpoints/organization_preprod_artifact_assemble.py", - "lines": "70-100" - }, - "output_len": 1382, - "is_error": false - }, - { - "round": 2, - "tool": "grep", - "input": { - "pattern": "for event_id, data in zip\\(error_ids, events\\.values\\(\\)\\)", - "glob": "*.py" - }, - "output_len": 476, - "is_error": false - }, - { - "round": 2, - "tool": "read_file", - "input": { - "path": "src/sentry/preprod/api/endpoints/organization_preprod_artifact_assemble.py", - "lines": "76-105" - }, - "output_len": 1380, - "is_error": false - }, - { - "round": 2, - "tool": "grep", - "input": { - "pattern": "class PRCommentWorkflow", - "glob": "*.py" - }, - "output_len": 495, - "is_error": false - }, - { - "round": 2, - "tool": "grep", - "input": { - "pattern": "format_comment_subtitle", - "glob": "*.py" - }, - "output_len": 947, - "is_error": false - }, - { - "round": 3, - "tool": "read_file", - "input": { - "path": "src/sentry/services/nodestore/base.py", - "lines": "174-210" - }, - "output_len": 1597, - "is_error": false - }, - { - "round": 3, - "tool": "grep", - "input": { - "pattern": "for event_id, data in zip\\(error_ids, events\\.values" - }, - "output_len": 476, - "is_error": false - }, - { - "round": 4, - "tool": "glob", - "input": { - "pattern": "src/sentry/replays/endpoints/project_replay_summarize_breadcrumbs.py" - }, - "output_len": 17, - "is_error": false - }, - { - "round": 4, - "tool": "grep", - "input": { - "pattern": "zip\\(error_ids, events\\.values\\(\\)\\)" - }, - "output_len": 476, - "is_error": false - }, - { - "round": 5, - "tool": "read_file", - "input": { - "path": "src/sentry/preprod/api/endpoints/organization_preprod_artifact_assemble.py", - "lines": "60-100" - }, - "output_len": 1713, - "is_error": false - }, - { - "round": 6, - "tool": "read_file", - "input": { - "path": "src/sentry/preprod/api/endpoints/organization_preprod_artifact_assemble.py", - "lines": "115-160" - }, - "output_len": 1791, - "is_error": false - }, - { - "round": 7, - "tool": "read_file", - "input": { - "path": "src/sentry/integrations/source_code_management/commit_context.py", - "lines": "139-145" - }, - "output_len": 194, - "is_error": false - }, - { - "round": 7, - "tool": "read_file", - "input": { - "path": "src/sentry/integrations/source_code_management/commit_context.py", - "lines": "460-470" - }, - "output_len": 306, - "is_error": false - }, - { - "round": 8, - "tool": "grep", - "input": { - "pattern": "_truncate_title", - "glob": "*.py" - }, - "output_len": 1188, - "is_error": false - }, - { - "round": 9, - "tool": "read_file", - "input": { - "path": "src/sentry/integrations/source_code_management/commit_context.py", - "lines": "145-165" - }, - "output_len": 592, - "is_error": false - }, - { - "round": 9, - "tool": "read_file", - "input": { - "path": "src/sentry/integrations/source_code_management/commit_context.py", - "lines": "615-655" - }, - "output_len": 1481, - "is_error": false - }, - { - "round": 10, - "tool": "read_file", - "input": { - "path": "src/sentry/integrations/source_code_management/commit_context.py", - "lines": "455-475" - }, - "output_len": 597, - "is_error": false - }, - { - "round": 11, - "tool": "read_file", - "input": { - "path": "src/sentry/integrations/github/integration.py", - "lines": "395-420" - }, - "output_len": 1106, - "is_error": false - }, - { - "round": 11, - "tool": "read_file", - "input": { - "path": "src/sentry/integrations/gitlab/integration.py", - "lines": "240-265" - }, - "output_len": 982, - "is_error": false - }, - { - "round": 12, - "tool": "grep", - "input": { - "pattern": "def get_comment_body", - "glob": "*.py" - }, - "output_len": 1490, - "is_error": false - }, - { - "round": 13, - "tool": "read_file", - "input": { - "path": "static/app/views/performance/newTraceDetails/traceDrawer/details/span/eapSections/attributes.tsx", - "lines": "55-75" - }, - "output_len": 488, - "is_error": false - }, - { - "round": 14, - "tool": "read_file", - "input": { - "path": "static/app/views/performance/newTraceDetails/traceDrawer/details/span/eapSections/attributes.tsx", - "lines": "55-100" - }, - "output_len": 1352, - "is_error": false - }, - { - "round": 15, - "tool": "read_file", - "input": { - "path": "static/app/views/dashboards/widgetCard/chart.tsx", - "lines": "138-200" - }, - "output_len": 2163, - "is_error": false - }, - { - "round": 16, - "tool": "read_file", - "input": { - "path": "static/app/views/dashboards/widgetCard/chart.tsx", - "lines": "148-200" - }, - "output_len": 1816, - "is_error": false - }, - { - "round": 17, - "tool": "grep", - "input": { - "pattern": "flex={1}", - "glob": "static/app/views/performance/newTraceDetails/traceWaterfall.tsx" - }, - "output_len": 470, - "is_error": false - }, - { - "round": 18, - "tool": "grep", - "input": { - "pattern": "export.*Flex", - "glob": "static/app/components/core/layout/*.tsx" - }, - "output_len": 2200, - "is_error": false - }, - { - "round": 19, - "tool": "read_file", - "input": { - "path": "static/app/components/core/layout/flex.tsx" - }, - "output_len": 3890, - "is_error": false - }, - { - "round": 20, - "tool": "read_file", - "input": { - "path": "static/app/components/core/layout/container.tsx" - }, - "output_len": 10545, - "is_error": false - }, - { - "round": 21, - "tool": "read_file", - "input": { - "path": "static/app/components/events/eventAttachments.tsx", - "lines": "195-210" - }, - "output_len": 343, - "is_error": false - }, - { - "round": 22, - "tool": "codebase_search", - "input": { - "search_string": "BrowserReportSerializer validate_age validate_timestamp mutual exclusion logic" - }, - "output_len": 110, - "is_error": false - }, - { - "round": 23, - "tool": "read_file", - "input": { - "path": "static/app/views/explore/contexts/pageParamsContext/visualizes.tsx", - "lines": "80-110" - }, - "output_len": 1078, - "is_error": false - }, - { - "round": 24, - "tool": "read_file", - "input": { - "path": "static/app/views/explore/contexts/pageParamsContext/visualizes.tsx", - "lines": "85-120" - }, - "output_len": 1446, - "is_error": false - }, - { - "round": 25, - "tool": "read_file", - "input": { - "path": "static/app/views/explore/charts/index.tsx", - "lines": "147-215" - }, - "output_len": 64, - "is_error": false - }, - { - "round": 26, - "tool": "glob", - "input": { - "pattern": "**/explore/charts/index.tsx" - }, - "output_len": 17, - "is_error": false - }, - { - "round": 27, - "tool": "grep", - "input": { - "pattern": "replay\\.view.html", - "glob": "*.{ts,tsx}" - }, - "output_len": 1423, - "is_error": false - }, - { - "round": 28, - "tool": "read_file", - "input": { - "path": "static/app/views/feedback/feedbackListPage.tsx", - "lines": "155-175" - }, - "output_len": 808, - "is_error": false - }, - { - "round": 29, - "tool": "grep", - "input": { - "pattern": "log_context\\.get_logger", - "glob": "*.py" - }, - "output_len": 4756, - "is_error": false - } -] \ No newline at end of file diff --git a/pr_review_agent/output/traces/sentry_pr80528_trace.json b/pr_review_agent/output/traces/sentry_pr80528_trace.json deleted file mode 100644 index 9faca81..0000000 --- a/pr_review_agent/output/traces/sentry_pr80528_trace.json +++ /dev/null @@ -1,122 +0,0 @@ -[ - { - "round": 0, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "Who calls get_monitor_environment_context and how is its return value used" - }, - "output_len": 4683, - "is_error": false - }, - { - "round": 0, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "Who calls mark_failed_threshold or try_incident_threshold" - }, - "output_len": 4674, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "src/sentry/monitors/logic/incident_occurrence.py" - }, - "output_len": 11925, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "src/sentry/monitors/logic/incidents.py" - }, - "output_len": 7833, - "is_error": false - }, - { - "round": 2, - "tool": "read_file", - "input": { - "path": "src/sentry/monitors/logic/mark_failed.py" - }, - "output_len": 657, - "is_error": false - }, - { - "round": 2, - "tool": "read_file", - "input": { - "path": "src/sentry/monitors/types.py" - }, - "output_len": 3450, - "is_error": false - }, - { - "round": 3, - "tool": "grep", - "input": { - "pattern": "mark_failed_threshold|create_issue_platform_occurrence" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 3, - "tool": "grep", - "input": { - "pattern": "from sentry\\.monitors\\.logic\\.mark_failed import" - }, - "output_len": 2596, - "is_error": false - }, - { - "round": 4, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "How is mark_failed called with its arguments in clock_tasks and consumers" - }, - "output_len": 110, - "is_error": false - }, - { - "round": 4, - "tool": "grep", - "input": { - "pattern": "SimpleCheckIn", - "glob": "*.py" - }, - "output_len": 2196, - "is_error": false - }, - { - "round": 5, - "tool": "grep", - "input": { - "pattern": "\"config\": monitor_environment\\.monitor\\.config", - "glob": "*.py" - }, - "output_len": 517, - "is_error": false - }, - { - "round": 6, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "callers of mark_failed function including arguments passed to it" - }, - "output_len": 110, - "is_error": false - }, - { - "round": 7, - "tool": "grep", - "input": { - "pattern": "mark_failed\\(", - "glob": "*.py" - }, - "output_len": 7083, - "is_error": false - } -] \ No newline at end of file diff --git a/pr_review_agent/output/traces/sentry_pr93824_trace.json b/pr_review_agent/output/traces/sentry_pr93824_trace.json deleted file mode 100644 index a95bd77..0000000 --- a/pr_review_agent/output/traces/sentry_pr93824_trace.json +++ /dev/null @@ -1,152 +0,0 @@ -[ - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "SpanFlusher class initialization and how it's created with buffer and shards" - }, - "output_len": 3733, - "is_error": false - }, - { - "round": 0, - "tool": "codebase_search", - "input": { - "search_string": "SpansBuffer class constructor and assigned_shards property" - }, - "output_len": 1080, - "is_error": false - }, - { - "round": 0, - "tool": "grep", - "input": { - "pattern": "isinstance.*multiprocessing\\.Process", - "glob": "*.py" - }, - "output_len": 907, - "is_error": false - }, - { - "round": 1, - "tool": "read_file", - "input": { - "path": "src/sentry/spans/consumers/process/flusher.py" - }, - "output_len": 21773, - "is_error": false - }, - { - "round": 2, - "tool": "bash", - "input": { - "command": "python3 -c \"import multiprocessing; ctx = multiprocessing.get_context('spawn'); p = ctx.Process(target=lambda: None); print(type(p)); print(isinstance(p, multiprocessing.Process)); print(type(p).__mro__)\"" - }, - "output_len": 166, - "is_error": false - }, - { - "round": 2, - "tool": "codebase_search", - "input": { - "search_string": "How is the SpansBuffer constructor called, what parameters does it take for shards and slice_id" - }, - "output_len": 1405, - "is_error": false - }, - { - "round": 2, - "tool": "grep", - "input": { - "pattern": "\"shard\".*:.*shard_tag|\"shards\".*:.*shard_tag", - "glob": "*.py" - }, - "output_len": 1495, - "is_error": false - }, - { - "round": 3, - "tool": "codebase_search", - "input": { - "search_string": "SpansBuffer slice_id how is it used in buffer operations like flush_segments" - }, - "output_len": 13886, - "is_error": false - }, - { - "round": 3, - "tool": "read_file", - "input": { - "path": "src/sentry/spans/buffer.py", - "lines": "146-200" - }, - "output_len": 2698, - "is_error": false - }, - { - "round": 3, - "tool": "grep", - "input": { - "pattern": "self\\.slice_id", - "path": "src/sentry/spans/buffer.py" - }, - "output_len": 449, - "is_error": false - }, - { - "round": 4, - "tool": "bash", - "input": { - "command": "python3 -c \"\nimport multiprocessing\nctx = multiprocessing.get_context('spawn')\n\n# Check SpawnProcess hierarchy\nprint('SpawnProcess MRO:', ctx.Process.__mro__)\nprint('multiprocessing.Process MRO:', multiprocessing.Process.__mro__)\nprint('Is SpawnProcess subclass of multiprocessing.Process?', issubclass(ctx.Process, multiprocessing.Process))\nprint('Is SpawnProcess subclass of BaseProcess?', issubclass(ctx.Process, multiprocessing.process.BaseProcess))\n\"" - }, - "output_len": 374, - "is_error": false - }, - { - "round": 4, - "tool": "read_file", - "input": { - "path": "tests/sentry/spans/consumers/process/test_flusher.py" - }, - "output_len": 7404, - "is_error": false - }, - { - "round": 4, - "tool": "grep", - "input": { - "pattern": "process_healthy_since.*Value.*\\(.*0\\)|process_healthy_since.*Value.*time", - "glob": "*.py" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 5, - "tool": "read_file", - "input": { - "path": "src/sentry/spans/consumers/process/factory.py" - }, - "output_len": 10636, - "is_error": false - }, - { - "round": 5, - "tool": "grep", - "input": { - "pattern": "ProcessSpansStrategyFactory", - "glob": "*.py" - }, - "output_len": 3674, - "is_error": false - }, - { - "round": 6, - "tool": "read_file", - "input": { - "path": "tests/sentry/spans/consumers/process/test_consumer.py" - }, - "output_len": 10828, - "is_error": false - } -] \ No newline at end of file diff --git a/pr_review_agent/output/traces/sentry_pr95633_trace.json b/pr_review_agent/output/traces/sentry_pr95633_trace.json deleted file mode 100644 index fb9a4c1..0000000 --- a/pr_review_agent/output/traces/sentry_pr95633_trace.json +++ /dev/null @@ -1,316 +0,0 @@ -[ - { - "round": 0, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "How does OffsetTracker._get_partition_lock handle concurrent creation of locks for the same partition" - }, - "output_len": 110, - "is_error": false - }, - { - "round": 0, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "queue.ShutDown exception in Python standard library queue module" - }, - "output_len": 42, - "is_error": false - }, - { - "round": 0, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "FixedQueuePool.shutdown method and attribute name conflict with worker shutdown" - }, - "output_len": 17279, - "is_error": false - }, - { - "round": 1, - "tool": "bash", - "input": { - "command": "python3 -c \"import queue; hasattr(queue, 'ShutDown') and print('ShutDown exists') or print('ShutDown does not exist')\"" - }, - "output_len": 39, - "is_error": false - }, - { - "round": 1, - "tool": "grep", - "input": { - "pattern": "def shutdown", - "path": "src/sentry/remote_subscriptions/consumers/queue_consumer.py" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 2, - "tool": "bash", - "input": { - "command": "python3 --version 2>/dev/null; cat .python-version 2>/dev/null; grep python_requires setup.cfg 2>/dev/null; grep python_requires setup.py 2>/dev/null; grep -i python pyproject.toml 2>/dev/null | head -5" - }, - "output_len": 205, - "is_error": false - }, - { - "round": 2, - "tool": "grep", - "input": { - "pattern": "class ResultsStrategyFactory", - "path": "src/sentry/remote_subscriptions/consumers/result_consumer.py" - }, - "output_len": 151, - "is_error": false - }, - { - "round": 3, - "tool": "read_file", - "input": { - "path": "src/sentry/remote_subscriptions/consumers/result_consumer.py" - }, - "output_len": 13280, - "is_error": false - }, - { - "round": 4, - "tool": "grep", - "input": { - "pattern": "mock.patch.object.*__call__", - "glob": "*.py" - }, - "output_len": 1239, - "is_error": false - }, - { - "round": 4, - "tool": "warpgrep_codebase_search", - "input": { - "search_string": "How does UptimeResultsStrategyFactory create_with_partitions work and what is its __init__ signature" - }, - "output_len": 3722, - "is_error": false - }, - { - "round": 5, - "tool": "read_file", - "input": { - "path": "src/sentry/uptime/consumers/results_consumer.py", - "lines": "695-720" - }, - "output_len": 531, - "is_error": false - }, - { - "round": 5, - "tool": "grep", - "input": { - "pattern": "result_processor_cls|UptimeResultProcessor", - "path": "src/sentry/uptime/consumers/results_consumer.py" - }, - "output_len": 335, - "is_error": false - }, - { - "round": 6, - "tool": "read_file", - "input": { - "path": "src/sentry/uptime/consumers/results_consumer.py", - "lines": "474-500" - }, - "output_len": 1359, - "is_error": false - }, - { - "round": 6, - "tool": "grep", - "input": { - "pattern": "def send_result.*consumer", - "path": "tests/sentry/uptime/consumers/test_results_consumer.py" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 7, - "tool": "grep", - "input": { - "pattern": "def send_result", - "path": "tests/sentry/uptime/consumers/test_results_consumer.py" - }, - "output_len": 122, - "is_error": false - }, - { - "round": 8, - "tool": "read_file", - "input": { - "path": "tests/sentry/uptime/consumers/test_results_consumer.py", - "lines": "85-130" - }, - "output_len": 2060, - "is_error": false - }, - { - "round": 9, - "tool": "grep", - "input": { - "pattern": "patch\\.object.*__call__.*assert", - "glob": "**/*.py" - }, - "output_len": 10, - "is_error": false - }, - { - "round": 9, - "tool": "read_file", - "input": { - "path": "src/sentry/remote_subscriptions/consumers/queue_consumer.py", - "lines": "100-145" - }, - "output_len": 82, - "is_error": false - }, - { - "round": 10, - "tool": "bash", - "input": { - "command": "python3 -c \"\nfrom unittest import mock\n\nclass MyClass:\n def __call__(self, a, b):\n return a + b\n\nobj = MyClass()\nwith mock.patch.object(type(obj), '__call__') as m:\n m.return_value = None\n obj('hello', 'world')\n print('call_args:', m.call_args)\n print('call_args_list:', m.call_args_list)\n\"" - }, - "output_len": 74, - "is_error": false - }, - { - "round": 11, - "tool": "bash", - "input": { - "command": "python3 -c \"\nfrom unittest import mock\n\nclass MyClass:\n def __call__(self, a, b):\n return a + b\n\nobj = MyClass()\nwith mock.patch.object(type(obj), '__call__') as m:\n m.side_effect = lambda a, b: ('called', a, b)\n result = obj('hello', 'world')\n print('result:', result)\n print('call_args:', m.call_args)\n\"" - }, - "output_len": 70, - "is_error": false - }, - { - "round": 12, - "tool": "bash", - "input": { - "command": "python3 -c \"\n# Simulating the OffsetTracker.get_committable_offsets logic after a gap scenario:\n# Offsets received: 100, 102, 103 (gap at 101)\n# All completed. First commit at 100. Then mark_committed(100).\n\n# After mark_committed(100):\nall_offsets = {102, 103} # 100 removed\noutstanding = set() # all completed\nlast_committed = 100\n\nmin_offset = min(all_offsets) # 102\nmax_offset = max(all_offsets) # 103\nstart = max(last_committed + 1, min_offset) # max(101, 102) = 102\nprint(f'start = max({last_committed + 1}, {min_offset}) = {start}')\n\nhighest_committable = last_committed # 100\nfor offset in range(start, max_offset + 1):\n if offset in all_offsets and offset not in outstanding:\n highest_committable = offset\n print(f'offset {offset}: committable -> highest = {highest_committable}')\n else:\n print(f'offset {offset}: gap/outstanding -> break')\n break\n\nprint(f'highest_committable = {highest_committable}, last_committed = {last_committed}')\nif highest_committable > last_committed:\n print(f'BUG: Would commit offset {highest_committable} even though offset 101 was never received!')\n print(f'With +1 adjustment, would tell Kafka we processed up to offset {highest_committable + 1}')\n\"\n" - }, - "output_len": 296, - "is_error": false - }, - { - "round": 13, - "tool": "read_file", - "input": { - "path": "tests/sentry/uptime/consumers/test_results_consumer.py", - "lines": "1850-1920" - }, - "output_len": 103, - "is_error": false - }, - { - "round": 14, - "tool": "read_file", - "input": { - "path": "tests/sentry/uptime/consumers/test_results_consumer.py", - "lines": "1686-1750" - }, - "output_len": 2677, - "is_error": false - }, - { - "round": 15, - "tool": "read_file", - "input": { - "path": "tests/sentry/uptime/consumers/test_results_consumer.py", - "lines": "1750-1849" - }, - "output_len": 4710, - "is_error": false - }, - { - "round": 16, - "tool": "bash", - "input": { - "command": "cd /Users/tejas/personal/applymodel/zmisc/examples/pr_clones/sentry && git log --oneline -3 2>/dev/null; git diff HEAD~1..HEAD -- src/sentry/remote_subscriptions/consumers/result_consumer.py 2>/dev/null | head -80" - }, - "output_len": 83, - "is_error": false - }, - { - "round": 17, - "tool": "bash", - "input": { - "command": "cd /Users/tejas/personal/applymodel/zmisc/examples/pr_clones/sentry && grep -n 'def __init__' src/sentry/remote_subscriptions/consumers/result_consumer.py | head -5" - }, - "output_len": 85, - "is_error": false - }, - { - "round": 18, - "tool": "bash", - "input": { - "command": "cd /Users/tejas/personal/applymodel/zmisc/examples/pr_clones/sentry && sed -n '123,140p' src/sentry/remote_subscriptions/consumers/result_consumer.py" - }, - "output_len": 810, - "is_error": false - }, - { - "round": 19, - "tool": "bash", - "input": { - "command": "python3 -c \"\nfrom collections import defaultdict\n\n# Simulate OffsetTracker behavior with gap scenario\nall_offsets = defaultdict(set)\noutstanding = defaultdict(set)\nlast_committed = {}\n\nP = 'partition0'\n\n# Add offsets 100, 102, 103 (gap at 101)\nfor o in [100, 102, 103]:\n all_offsets[P].add(o)\n outstanding[P].add(o)\n\n# Complete all\nfor o in [100, 102, 103]:\n outstanding[P].discard(o)\n\n# First get_committable_offsets\nao = all_offsets[P]\nos_set = outstanding[P]\nlc = last_committed.get(P, -1)\nmin_o = min(ao)\nmax_o = max(ao)\nstart = max(lc + 1, min_o)\nhighest = lc\nfor off in range(start, max_o + 1):\n if off in ao and off not in os_set:\n highest = off\n else:\n break\nprint(f'First call: committable offset = {highest}')\nassert highest == 100, 'First call should only commit up to 100'\n\n# mark_committed(P, 100)\nlast_committed[P] = 100\nall_offsets[P] = {o for o in all_offsets[P] if o > 100}\nprint(f'After mark_committed(100): all_offsets = {all_offsets[P]}')\n\n# Second get_committable_offsets - THIS IS WHERE THE BUG IS\nao = all_offsets[P]\nos_set = outstanding[P]\nlc = last_committed.get(P, -1)\nmin_o = min(ao)\nmax_o = max(ao)\nstart = max(lc + 1, min_o)\nprint(f'Second call: start = max({lc+1}, {min_o}) = {start}')\nhighest = lc\nfor off in range(start, max_o + 1):\n if off in ao and off not in os_set:\n highest = off\n else:\n break\nprint(f'Second call: committable offset = {highest}')\nprint(f'BUG: Offset 101 was never seen, but we would commit up to {highest}!')\nprint(f'With +1 adjustment, Kafka would think we processed through offset {highest + 1}')\n\n# What the CORRECT behavior should be:\nstart_correct = lc + 1 # = 101\nprint(f'\\\\nCorrect start: {start_correct}')\nhighest_correct = lc\nfor off in range(start_correct, max_o + 1):\n if off in ao and off not in os_set:\n highest_correct = off\n else:\n break\nprint(f'Correct: would try offset {start_correct}, not in all_offsets -> break')\nprint(f'Correct: no new committable offsets (highest={highest_correct} == last_committed={lc})')\n\"\n" - }, - "output_len": 451, - "is_error": false - }, - { - "round": 20, - "tool": "read_file", - "input": { - "path": "tests/sentry/remote_subscriptions/consumers/test_queue_consumer.py", - "lines": "30-65" - }, - "output_len": 89, - "is_error": false - }, - { - "round": 21, - "tool": "bash", - "input": { - "command": "python3 -c \"\n# Verify the test_thread_queue_parallel_error_handling assertion is wrong\n# The test has mock_processor_call.side_effect = [Exception('Processing failed'), None]\n# Both offsets (100, 101) for same subscription -> same queue, same worker\n\n# Worker behavior:\n# First call: Exception raised -> caught by except Exception -> offset 100 completed in finally\n# Second call: returns None (success) -> offset 101 completed in finally\n\n# Both offsets completed -> commit loop commits both (offset 101 + 1 = 102)\n# But test assertion is:\n# assert len(committed_offsets) == 0 or test_partition not in committed_offsets\n# This says nothing was committed, which contradicts the code behavior.\n\n# The test might PASS due to timing (commit loop hasn't fired yet, waits 1s)\n# But the assertion is fundamentally wrong - it contradicts the docstring:\n# 'Test that errors in processing don't block offset commits for other messages'\n\nprint('Test docstring says: errors should NOT block commits')\nprint('Test assertion checks: nothing was committed (contradicts docstring)')\nprint('Code behavior: offsets always completed in finally block, both get committed')\nprint('Test passes only due to timing: commit loop waits 1s, test exits before commit')\n\"\n" - }, - "output_len": 277, - "is_error": false - }, - { - "round": 22, - "tool": "read_file", - "input": { - "path": "tests/sentry/uptime/consumers/test_results_consumer.py", - "lines": "1868-1940" - }, - "output_len": 0, - "is_error": false - }, - { - "round": 23, - "tool": "read_file", - "input": { - "path": "tests/sentry/uptime/consumers/test_results_consumer.py", - "lines": "1900-1960" - }, - "output_len": 0, - "is_error": false - }, - { - "round": 24, - "tool": "bash", - "input": { - "command": "wc -l tests/sentry/uptime/consumers/test_results_consumer.py 2>/dev/null; tail -20 tests/sentry/uptime/consumers/test_results_consumer.py 2>/dev/null" - }, - "output_len": 953, - "is_error": false - } -] \ No newline at end of file diff --git a/pr_review_agent/output/v24_plan_calibrate.log b/pr_review_agent/output/v24_plan_calibrate.log deleted file mode 100644 index 8aab903..0000000 --- a/pr_review_agent/output/v24_plan_calibrate.log +++ /dev/null @@ -1,121 +0,0 @@ -============================================================ -PR Review Agent - Benchmark Pipeline -============================================================ -Loaded 50 PRs -WarpGrep: ENABLED -Calibration: 5 PRs (1 per repo) -Reviewing 5 PRs - -Running 5 PRs with parallelism=5 - -[1/5] keycloak PR#37429 (4 golden) -[2/5] sentry PR#93824 (5 golden) -[3/5] grafana PR#97529 (2 golden) -[4/5] discourse-graphite PR#9 (2 golden) -[5/5] cal.com PR#22532 (2 golden) - [4/5] discourse-graphite PR#9 6 files, 31 added lines - [3/5] grafana PR#97529 5 files, 25 added lines - [2/5] sentry PR#93824 6 files, 199 added lines - [5/5] cal.com PR#22532 17 files, 379 added lines - [1/5] keycloak PR#37429 48 files, 343 added lines - WarpGrep: SpansBuffer constructor and assigned_shards property - WarpGrep: How does Prisma updateMany behave when data is an empty object {} - WarpGrep: How does the santizeAnchors method work and what is its purpose in VerifyMessage - WarpGrep: SpanFlusher.main method signature and how it's called as process target - WarpGrep: Lithuanian locale messages_lt.properties totpStep1 translation - WarpGrep: updateManyByCredentialId usage and callers - WarpGrep: How is BuildIndex called and what concurrency protection exists around it? Who c - WarpGrep: SelectedCalendar model schema definition with updatedAt field - WarpGrep: ProcessSpansStrategyFactory create_with_partitions implementation - WarpGrep: set_locale method definition in application_controller - WarpGrep: Italian text "Installa una delle seguenti applicazioni sul tuo cellulare" in Lit - WarpGrep: server Init method implementation with sync.Once pattern - WarpGrep: How is set_locale used as a before_action or around_action in ApplicationControl - WarpGrep: SelectedCalendar model in Prisma schema with updatedAt field - WarpGrep: run_with_initialized_sentry function signature and how it wraps target functions - WarpGrep connection error, retrying in 3s (attempt 1/3): SSLError - WarpGrep API error (turn 2): 400 Client Error: Bad Request for url: https://api.morphllm.com/v1/chat/completions - WarpGrep: How is the bleveBackend cache field accessed? Who reads from cache in bleveBacke - WarpGrep: cacheUpdatedAt property in connectedCalendars type or interface - WarpGrep: Traditional Chinese characters in zh_CN simplified Chinese locale file totpStep1 - WarpGrep: metrics tag name "shard" vs "shards" inconsistency in spans buffer flusher - WarpGrep: I18n.fallbacks usage across the codebase - WarpGrep: messages_zh_CN account totpStep1 translation installation application - WarpGrep: _import_and_run function how it unpickles and calls the main function with args - WarpGrep: Prisma updateMany with empty data object, does @updatedAt still trigger - WarpGrep: set_locale method called or used as before_action or around_action - WarpGrep: multiprocessing.Process vs multiprocessing.context.SpawnProcess kill method - WarpGrep: SiteSetting.default_locale returns string or symbol - WarpGrep: messages_zh_Hans.properties totpStep1 simplified Chinese installation mobile - WarpGrep: multiprocessing.context.SpawnProcess isinstance check with multiprocessing.Proce - WarpGrep: CalendarService cache method that stores availability data in cache - WarpGrep: server Init sync.Once initOnce initErr concurrency pattern in resource server - WarpGrep: Google Calendar CalendarService class method around line 1019 - WarpGrep: SpansBuffer process_spans and flush_segments methods and how they use assigned_s - WarpGrep: SpansBuffer slice_id property used in buffer operations - WarpGrep: How does I18n::Backend::Fallbacks module use the fallbacks hash for translation - WarpGrep: santizeAnchors replaceFirst anchor pattern mutation loop bug - Max tool rounds reached (tools: warpgrep_codebase_search=8, read_file=8, grep_pattern=9) - WarpGrep: NewResourceServer return type signature and callers of NewResourceServer - WarpGrep: def set_locale in application_controller with I18n.locale assignment and SiteSet - WarpGrep: searchSupport init function building indexes with worker threads concurrently - Review complete (tools: warpgrep_codebase_search=7, read_file=4, grep_pattern=3) - Plan complete (6314 chars) - WarpGrep: updateManyByCredentialId in SelectedCalendarRepository and how Prisma updateMany - WarpGrep: FallbackLocaleList ensure_loaded method definition - WarpGrep: Prisma updateMany with empty data object - does it update @updatedAt fields? - WarpGrep: server struct tracer field initialization in resource server - WarpGrep: verifySafeHtml method in VerifyMessageProperties showing the substring trimming - WarpGrep: searchSupport struct definition with tracer field - WarpGrep: I18n.locale returns symbol or string, locale= setter converts to symbol - WarpGrep: Prisma updateMany with empty data object behavior - WarpGrep: SelectedCalendarRepository import in GoogleCalendarService - Review complete (tools: read_file=14, grep_pattern=11, list_directory=2, warpgrep_codebase_search=1) - WarpGrep: connectedCalendar.cacheUpdatedAt property and how connected calendars type is de - WarpGrep: cacheUpdatedAt in connected calendars data flow - Review complete: 3 issues - [1/5] keycloak PR#37429 3 raw -> 3 kept - [1/5] keycloak PR#37429 [0.97] localization: Italian text in Lithuanian locale file. The `loginTotpStep1` value was replaced with Itali - [1/5] keycloak PR#37429 [0.97] localization: Italian text in Lithuanian account locale file. The `totpStep1` value contains Italian tex - [1/5] keycloak PR#37429 [0.95] localization: Traditional Chinese characters used in Simplified Chinese (zh_Hans) locale file. The `totp - WarpGrep: LocaleSiteSetting.fallback_locale method definition - WarpGrep: Prisma updateMany with @updatedAt behavior and empty data object - Review complete (tools: read_file=11, warpgrep_codebase_search=10, grep_pattern=6) - Plan complete (4973 chars) - Review complete (tools: warpgrep_codebase_search=10, read_file=8, grep_pattern=3) - Plan complete (4380 chars) - WarpGrep: FallbackLocaleList class definition - WarpGrep: ThemeField name column type or attribute definition - WarpGrep: CalendarCacheRepository import and instantiation in connectedCalendars handler - Review complete (tools: read_file=9, grep_pattern=4) - Review complete: 2 issues - [2/5] sentry PR#93824 2 raw -> 2 kept - [2/5] sentry PR#93824 [0.95] incorrect_value: Inconsistent metric tag key `"shards"` (plural) used for the `spans.buffer.flusher.wait_pr - [2/5] sentry PR#93824 [0.92] test_correctness: The test monkeypatches `time.sleep` to a no-op at the top, then later calls `time.sleep(0. - WarpGrep: method that calls fetchAvailability and setAvailabilityInCache in GoogleCalendar - Review complete (tools: read_file=11, warpgrep_codebase_search=2, grep_pattern=5) - Review complete: 2 issues - [4/5] discourse-graphite PR#9 2 raw -> 2 kept - [4/5] discourse-graphite PR#9 [0.92] type_error: The `FallbackLocaleList#[]` method doesn't convert its `locale` parameter to a symbol. In - [4/5] discourse-graphite PR#9 [0.75] type_error: The new `ensure_loaded!` method doesn't convert `locale` to a symbol before the `@loaded_l - Max tool rounds reached (tools: warpgrep_codebase_search=9, read_file=9, grep_pattern=14) - Review complete: 0 issues - [5/5] cal.com PR#22532 0 raw -> 0 kept - WarpGrep: callers of server.Init method for resource server after creation - Max tool rounds reached (tools: warpgrep_codebase_search=9, read_file=12, grep_pattern=8) - WarpGrep: who calls BuildIndex in the bleveBackend and how is concurrency handled - WarpGrep connection error, retrying in 3s (attempt 1/3): SSLError - WarpGrep: getOrCreateIndex function in searchSupport or searchServer, how is concurrency h - WarpGrep: findIndexesToRebuild function definition in search.go - WarpGrep: rebuildQueue processing, consuming rebuild requests to build indexes - WarpGrep: callers of NewResourceServer function - Review complete (tools: read_file=19, warpgrep_codebase_search=5, grep_pattern=7) - Review complete: 0 issues - [3/5] grafana PR#97529 0 raw -> 0 kept -Wrote candidates to /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/candidates.json - -============================================================ -DONE: 5 reviewed, 7 raw -> 7 filtered -Avg/PR: 1.4, Time: 1042s -Candidates: /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/candidates.json -Benchmark data updated: /Users/tejas/personal/applymodel/zmisc/examples/code-review-benchmark/offline/results/benchmark_data.json diff --git a/pr_review_agent/output/v25_xml_calibrate.log b/pr_review_agent/output/v25_xml_calibrate.log deleted file mode 100644 index 0de0198..0000000 --- a/pr_review_agent/output/v25_xml_calibrate.log +++ /dev/null @@ -1,124 +0,0 @@ -============================================================ -PR Review Agent - Benchmark Pipeline -============================================================ -Loaded 50 PRs -WarpGrep: ENABLED -Calibration: 5 PRs (1 per repo) -Reviewing 5 PRs - -Running 5 PRs with parallelism=5 - -[1/5] keycloak PR#37429 (4 golden) -[2/5] sentry PR#93824 (5 golden) -[3/5] grafana PR#97529 (2 golden) -[4/5] discourse-graphite PR#9 (2 golden) -[5/5] cal.com PR#22532 (2 golden) - [2/5] sentry PR#93824 6 files, 199 added lines - [1/5] keycloak PR#37429 48 files, 343 added lines - [5/5] cal.com PR#22532 17 files, 379 added lines - [3/5] grafana PR#97529 5 files, 25 added lines - [4/5] discourse-graphite PR#9 6 files, 31 added lines - WarpGrep: SpansBuffer constructor and assigned_shards property - WarpGrep: How does updateManyByCredentialId work and what happens when empty data object i - WarpGrep: How is BuildIndex called and what concurrency protection does the cache mutex pr - WarpGrep: Lithuanian locale messages_lt totpStep1 text content - WarpGrep: How is SpanFlusher.main called as a process target with arguments - WarpGrep: How is I18n.ensure_loaded! defined and called? Is it a method on I18n module dir - WarpGrep: How is connectedCalendar.cacheUpdatedAt used in the UI components, and where is - WarpGrep: zh_CN Simplified Chinese totpStep1 translation - WarpGrep: metric tag key "shard" vs "shards" in spans.buffer.flusher metrics - WarpGrep: How does SelectedCalendarRepository.updateManyByCredentialId get called and what - WarpGrep: How is server.Init called and what does the sync.Once pattern look like for serv - WarpGrep: santizeAnchors method and how anchor tags are matched and replaced - WarpGrep: Who calls NewResourceServer and how do callers handle the returned error - WarpGrep: run_with_initialized_sentry function signature and how it handles arguments - WarpGrep: Italian locale messages_it totpStep1 "Installa una delle seguenti" - WarpGrep: ProcessSpansStrategyFactory create_with_partitions implementation and how buffer - WarpGrep: SelectedCalendar model in Prisma schema, does it have @updatedAt field - WarpGrep: Prisma updateMany with empty data object, does @updatedAt get triggered - WarpGrep: _import_and_run function that unpickles and runs the main function with args - WarpGrep: set_locale method definition in ApplicationController - WarpGrep: VerifyMessageProperties santizeAnchors replaceFirst after value is modified, mat - WarpGrep: SiteSetting.default_locale return type - is it a string or symbol? - WarpGrep: How is BuildIndex called concurrently, who calls it and from what goroutines in - WarpGrep: How is @loaded_locales populated and what types does it contain - symbols or str - WarpGrep: Where is SelectedCalendarRepository imported in google calendar service, which r - WarpGrep: set_locale before_action definition in ApplicationController, how is it used as - WarpGrep: getOrCreateIndex and build method in searchSupport, how do they call BuildIndex - WarpGrep: impersonateTitleHtml in English locale login messages - WarpGrep: searchSupport init method that builds indexes during startup, how are indexes bu - WarpGrep: How does I18n::Backend::Fallbacks use the fallbacks object? What interface does - WarpGrep: config.i18n.fallbacks usage in development or test environments - WarpGrep: How is updateManyByCredentialId called in GoogleCalendarService, which SelectedC - WarpGrep: SpansBuffer process_spans how it determines which shard a span belongs to - WarpGrep: spans.buffer.flusher.wait_produce metric tag usage shard or shards - WarpGrep: FallbackLocaleList class definition - WarpGrep: How monkeypatch.setattr("time.sleep") affects flusher thread time.sleep calls - WarpGrep: getCachedIndex and GetIndex methods on bleveBackend, do they use read locks on t - WarpGrep: CalendarCacheRepository constructor and how it's instantiated - does it need arg - WarpGrep: SpansBuffer slice_id how it affects Redis client routing or behavior - Review complete (tools: read_file=5, warpgrep_codebase_search=8, grep=4, glob=1) - Plan complete (3312 chars) - WarpGrep connection error, retrying in 3s (attempt 1/3): SSLError - WarpGrep: FallbackLocaleList class definition - WarpGrep: set_locale method in application_controller - WarpGrep: connectedCalendar type definition that includes credentialId, delegationCredenti - WarpGrep: Prisma updateMany with empty data object behavior, or @updatedAt with updateMany - WarpGrep: SpansBuffer record_stored_segments implementation - WarpGrep: SpansBuffer get_memory_info implementation and how slice_id affects it - WarpGrep: set_locale before_action or around_action in application controller - Review complete (tools: warpgrep_codebase_search=6, read_file=7, grep=7, glob=2) - Plan complete (5412 chars) - WarpGrep: search init method initializes indexes during startup, how does it handle the in - WarpGrep: searchServer init method that builds indexes at startup and starts watcher or re - WarpGrep: How does the getConnectedCalendars function structure its return value, specific - WarpGrep: SiteSetting.default_locale return type string or symbol - WarpGrep: SiteSetting default_locale type definition, is it a string setting - WarpGrep: Where is the tRPC transformer configured, does it use superjson for date seriali - Review complete (tools: warpgrep_codebase_search=12, read_file=4, grep=1) - Plan complete (4716 chars) - WarpGrep: SpansBuffer __init__ slice_id parameter - Review complete (tools: read_file=4, glob=2, grep=5, bash=6) - Review complete: 3 issues - [1/5] keycloak PR#37429 3 raw -> 3 kept - [1/5] keycloak PR#37429 [0.99] localization: Italian text "Installa una delle seguenti applicazioni sul tuo cellulare:" was inserted in - [1/5] keycloak PR#37429 [0.99] localization: Italian text "Installa una delle seguenti applicazioni sul tuo cellulare:" was inserted in - [1/5] keycloak PR#37429 [0.97] localization: Traditional Chinese characters (手機, 安裝, 應用程式) were used in the Simplified Chinese locale f - Max tool rounds reached (tools: warpgrep_codebase_search=12, read_file=6, grep=5, glob=4, bash=5) - WarpGrep: updateManyByCredentialId definition in selectedCalendar repository - WarpGrep: TotalDocs method on SearchBackend interface or bleveBackend - Review complete (tools: read_file=9, warpgrep_codebase_search=1, grep=3) - Review complete: 1 issues - [2/5] sentry PR#93824 1 raw -> 1 kept - [2/5] sentry PR#93824 [0.97] incorrect_value: The metric tag key is `"shards"` (plural) but should be `"shard"` (singular) to match all - WarpGrep: Prisma updateMany with empty data object - what happens when data is {} - WarpGrep: DiscourseI18n class definition and inheritance, what does it inherit from - WarpGrep: SelectedCalendar model definition in Prisma schema with updatedAt - WarpGrep: Prisma @updatedAt behavior with updateMany - does updateMany trigger @updatedAt - WarpGrep: bleveBackend cache map readers - where is b.cache read outside of BuildIndex, su - WarpGrep: connectedCalendars type definition including cacheUpdatedAt property - WarpGrep: CalendarCacheRepository import in connectedCalendars handler - Review complete (tools: warpgrep_codebase_search=11, read_file=7) - Plan complete (3720 chars) - WarpGrep: who calls BuildIndex in the resource/search package - WarpGrep: rebuildIndex function in resource/search.go that calls build directly without si - WarpGrep: singleflight DoChan in searchSupport build function - WarpGrep: CalendarCacheRepository class constructor and how it's instantiated - Max tool rounds reached (tools: read_file=9, warpgrep_codebase_search=6, grep=15, bash=1, glob=2) - Review complete: 0 issues - [4/5] discourse-graphite PR#9 0 raw -> 0 kept - WarpGrep: rebuildQueue dispatch workers processing rebuild requests from queue - Max tool rounds reached (tools: read_file=10, glob=3, warpgrep_codebase_search=7, grep=4, bash=8) - WarpGrep: searchSupport struct definition with its fields in resource package - Review complete: 0 issues - [5/5] cal.com PR#22532 0 raw -> 0 kept - Review complete (tools: read_file=11, warpgrep_codebase_search=5, grep=10) - Review complete: 0 issues - [3/5] grafana PR#97529 0 raw -> 0 kept -Wrote candidates to /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/candidates.json - -============================================================ -DONE: 5 reviewed, 4 raw -> 4 filtered -Avg/PR: 0.8, Time: 829s -Candidates: /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/candidates.json -Benchmark data updated: /Users/tejas/personal/applymodel/zmisc/examples/code-review-benchmark/offline/results/benchmark_data.json diff --git a/pr_review_agent/output/v25_xml_eval.log b/pr_review_agent/output/v25_xml_eval.log deleted file mode 100644 index ff698a8..0000000 --- a/pr_review_agent/output/v25_xml_eval.log +++ /dev/null @@ -1,240 +0,0 @@ - -keycloak PR#37429: 4 golden, 3 candidates - TP: [Medium] The translation is in Italian instead of Lithuanian. This should be translated t... - TP: [Medium] The totpStep1 value uses Traditional Chinese terms in the Simplified Chinese fil... - FN: [Low] The anchor sanitization logic has a potential issue where it consumes English ma... - FN: [Low] The method name 'santizeAnchors' should be 'sanitizeAnchors' (missing 'i').... - -keycloak PR#37634: 4 golden, 2 candidates - TP: [Critical] Wrong parameter in null check (grantType vs. rawTokenId)... - TP: [High] In isAccessTokenId, the substring for the grant shortcut and the equality check ... - FN: [Low] Javadoc mentions "usually like 3-letters shortcut" but some implementations use ... - FN: [Low] Catching generic RuntimeException is too broad. The implementation throws Illeg... - -keycloak PR#38446: 2 golden, 2 candidates - TP: [Low] After creating the RecoveryAuthnCodesCredentialModel, consider setting its id fr... - FN: [Medium] Unsafe raw List deserialization without type safety. Calling Optional.get() dire... - -keycloak PR#36880: 3 golden, 3 candidates - FN: [High] Inconsistent feature flag bug causing orphaned permissions. The AdminPermissions... - FN: [High] In hasPermission(ClientModel client, String scope), the resource lookup uses fin... - FN: [High] In getClientsWithPermission(String scope), iterating resourceStore.findByType(se... - -keycloak PR#37038: 2 golden, 2 candidates - TP: [High] Incorrect permission check in canManage() method... - TP: [High] In getGroupIdsWithViewPermission, hasPermission is called with groupResource.get... - -keycloak PR#33832: 2 golden, 1 candidates - TP: [Low] Dead code exists where ASN1Encoder instances are created and written to, but the... - FN: [High] Returns wrong provider (default keystore instead of BouncyCastle)... - -keycloak PR#40940: 2 golden, 2 candidates - TP: [Critical] Returning null from getSubGroupsCount() violates the GroupModel contract (Javado... - TP: [Medium] The reader thread isn’t waited for; flipping deletedAll to true and asserting im... - -sentry PR#93824: 5 golden, 1 candidates - TP: [Medium] Inconsistent metric tagging with 'shard' and 'shards'... - FN: [Low] Fixed sleep in tests can be flaky; wait on condition instead... - FN: [High] Because flusher processes are created via multiprocessing.get_context('spawn').P... - FN: [Medium] Sleep in test_consumer.py won’t actually wait because time.sleep was monkeypatch... - FN: [Medium] Breaking out of the loop when the deadline has elapsed can skip terminating rema... - -sentry-greptile PR#5: 3 golden, 4 candidates - TP: [Low] Using zip(error_ids, events.values()) assumes the get_multi result preserves the... - FN: [Medium] Breaking changes in error response format... - FN: [Medium] Detector validator uses wrong key when updating type... - -sentry-greptile PR#1: 4 golden, 4 candidates - TP: [High] Django querysets do not support negative slicing... - TP: [High] When requests are authenticated with API keys or org auth tokens (which have use... - TP: [High] get_item_key assumes a numeric key, but the paginator is used with order_by=-dat... - FN: [Low] Importing non-existent OptimizedCursorPaginator... - -sentry PR#80168: 2 golden, 3 candidates - FN: [High] MetricAlertDetectorHandler inherits from StatefulDetectorHandler but only contai... - FN: [Low] Docstring says this returns a list of DetectorEvaluationResult, but the method n... - -sentry PR#80528: 2 golden, 1 candidates - TP: [High] The function modifies the config variable to include display values but then ret... - FN: [Low] The code fetches MonitorCheckIn objects by ID when the required data already exi... - -sentry PR#77754: 4 golden, 1 candidates - TP: [Medium] Shared mutable default in dataclass timestamp... - FN: [Low] The method name has a typo: test_from_dict_inalid_data should be test_from_dict_... - FN: [Low] Method name says 'empty_array' but tests empty dict - consider renaming to 'test... - FN: [Medium] to_dict() returns a datetime for queued; if this dict is passed in task kwargs (... - -sentry PR#95633: 3 golden, 3 candidates - TP: [Low] The test test_thread_queue_parallel_error_handling has a docstring that doesn't ... - FN: [High] The queue.shutdown() method with 'immediate=False' parameter may not exist in th... - FN: [Low] The magic number 50 for max_wait is used repeatedly throughout the tests. Consid... - -sentry-greptile PR#2: 3 golden, 4 candidates - TP: [Critical] OptimizedCursorPaginator negative-offset branch slices QuerySet with a negative ... - TP: [High] BasePaginator negative-offset branch slices QuerySet with a negative start index... - TP: [High] OptimizedCursorPaginator.get_item_key uses floor/ceil on a datetime key (order_b... - -sentry-greptile PR#3: 3 golden, 3 candidates - FN: [Low] sample_rate = 0.0 is falsy and skipped... - FN: [Low] Using Python’s built-in hash() to build cache keys is non-deterministic across p... - FN: [Medium] The upsampling eligibility check passes the outer dataset instead of the actual ... - -grafana PR#103633: 2 golden, 1 candidates - TP: [Low] The test comment says the cached permissions 'allow access', but the map stores ... - FN: [High] The Check operation exhibits asymmetric cache trust logic: cached permission gra... - -sentry PR#67876: 3 golden, 3 candidates - TP: [High] The code attempts to access integration.metadata[sender][login] without checking... - FN: [Medium] Null reference if github_authenticated_user state is missing... - FN: [Medium] OAuth state uses pipeline.signature (static) instead of a per-request random val... - -keycloak PR#32918: 2 golden, 3 candidates - TP: [Medium] Cleanup reference uses incorrect alias - should be 'idp-alias-' + i instead of '... - FN: [Critical] Recursive caching call using session instead of delegate... - -grafana PR#94942: 2 golden, 1 candidates - TP: [Critical] The enableSqlExpressions function has flawed logic that always returns false, ef... - TP: [High] Several methods such as NewInMemoryDB().RunCommands and db.QueryFramesInto retur... - -grafana PR#90939: 2 golden, 1 candidates - TP: [Medium] The GetWebAssets function implements an incomplete double-checked locking patter... - FN: [High] In addition to the missing double-check, the function has a critical flaw in its... - -grafana PR#80329: 1 golden, 2 candidates - TP: [Low] The code uses Error log level for what appears to be debugging information. This... - -grafana PR#90045: 3 golden, 5 candidates - TP: [Medium] The context is being created with d.Log instead of the log variable that was ini... - TP: [High] Bug: calling recordLegacyDuration when storage operation fails should be recordS... - TP: [Medium] Inconsistency: using name instead of options.Kind for metrics recording differs ... - -grafana PR#106778: 2 golden, 2 candidates - TP: [Medium] The rendered GrafanaRuleListItem is missing the required key prop for React list... - FN: [High] RuleActionsButtons is invoked with only promRule, but SilenceGrafanaRuleDrawer i... - -grafana PR#107534: 1 golden, 2 candidates - FN: [Low] The applyTemplateVariables method is called with request.filters as the third pa... - -grafana PR#79265: 5 golden, 2 candidates - FN: [High] Race condition: Multiple concurrent requests could pass the device count check s... - FN: [Medium] Anonymous authentication now fails entirely if anonDeviceService.TagDevice retur... - FN: [Medium] This call won’t compile: dbSession.Exec(args...) is given a []interface{} where ... - FN: [Low] Returning ErrDeviceLimitReached when no rows were updated is misleading; the dev... - FN: [Low] Time window calculation inconsistency: Using device.UpdatedAt.UTC().Add(-anonymo... - -grafana PR#76186: 2 golden, 1 candidates - FN: [High] The ContextualLoggerMiddleware methods (QueryData, CallResource, CheckHealth, Co... - FN: [Low] The traceID is no longer logged for plugin requests. During a refactoring, the t... - -discourse-graphite PR#10: 4 golden, 6 candidates - TP: [Critical] NoMethodError before_validation in EmbeddableHost... - FN: [Medium] The update and destroy methods in Admin::EmbeddableHostsController do not valida... - FN: [Medium] record_for_host compares lower(host) = ? but does not normalize the parameter’s ... - FN: [High] Because this migration inserts embeddable_hosts rows with raw SQL, any existing ... - -discourse-graphite PR#7: 3 golden, 5 candidates - TP: [Low] This change for desktop/user.css changes $primary from 30% to 50% for the light ... - TP: [Low] In topic-post.css the original code used $lightness: 70% but the replacement use... - FN: [Low] In .topic-meta-data h5 a, the original code had color: scale-color($primary, $li... - -discourse-graphite PR#8: 3 golden, 3 candidates - TP: [Medium] In the next action, capping the next offset at user_count can produce an empty p... - TP: [Medium] HTTP method mismatch in .remove_member - test uses PUT but remove_member action ... - FN: [High] The findMembers() call is now asynchronous and unhandled. The controller may no... - -discourse-graphite PR#3: 2 golden, 2 candidates - TP: [Medium] BlockedEmail.should_block_email? method has side effects during a read operation... - TP: [Medium] Regex pattern @(#{domains}) only matches domain suffixes, not full domains. evil... - -discourse-graphite PR#5: 2 golden, 2 candidates - TP: [Low] -ms-align-items never existed in any version of IE/Edge; the correct legacy prop... - FN: [Low] Mixing float: left with flexbox causes layout issues. Further this PR removes th... - -discourse-graphite PR#6: 1 golden, 1 candidates - TP: [Medium] The include_website_name method is missing the required ? suffix. Rails serializ... - -discourse-graphite PR#4: 6 golden, 5 candidates - TP: [Critical] SSRF vulnerability using open(url) without validation... - TP: [Medium] The current origin validation using indexOf is insufficient and can be bypassed.... - TP: [Medium] The ERB block closes with end if, which is invalid Ruby/ERB and will raise at re... - FN: [Medium] postMessage targetOrigin should be the origin (scheme+host+port), not the full r... - FN: [Medium] The code sets X-Frame-Options: ALLOWALL which completely disables clickjacking p... - FN: [Medium] The TopicEmbed.import method is susceptible to a NoMethodError if the contents p... - -discourse-graphite PR#1: 3 golden, 3 candidates - TP: [Medium] The downsize method is defined twice. The second definition, which expects a sin... - TP: [Low] Hardcoding maxSizeKB = 10 * 1024 ignores Discourse.SiteSettings['max_' + type + ... - FN: [Medium] Passing 80% as the dimensions can fail for animated GIFs when allow_animated_thu... - -discourse-graphite PR#2: 2 golden, 3 candidates - TP: [High] logic: Potential nil pointer exception - if no TopicUser record exists, tu will ... - TP: [Low] Typo in property name: 'stopNotificiationsText' should be 'stopNotificationsText... - -cal.com PR#8330: 2 golden, 2 candidates - TP: [Medium] Incorrect end time calculation using slotStartTime instead of slotEndTime... - TP: [Medium] Using === for dayjs object comparison will always return false as it compares ob... - -cal.com PR#14943: 2 golden, 1 candidates - TP: [High] The deletion logic in scheduleSMSReminders.ts incorrectly deletes non-SMS workfl... - FN: [High] Using retryCount: reminder.retryCount + 1 reads a possibly stale value and can l... - -cal.com PR#22345: 2 golden, 2 candidates - FN: [Low] In getBaseConditions(), the else if (filterConditions) and final else branches a... - FN: [Medium] Fetching userIdsFromOrg only when teamsFromOrg.length > 0 can exclude org-level ... - -cal.com PR#11059: 5 golden, 3 candidates - TP: [High] Invalid Zod schema syntax. Computed property keys like [z.string().toString()] a... - TP: [High] parseRefreshTokenResponse returns a Zod safeParse result ({ success, data, error... - FN: [High] The parseRefreshTokenResponse function incorrectly sets refresh_token to the har... - FN: [High] When APP_CREDENTIAL_SHARING_ENABLED and CALCOM_CREDENTIAL_SYNC_ENDPOINT are set,... - FN: [High] When the sync endpoint path is used, res is a fetch Response and has no .data; r... - -cal.com PR#7232: 2 golden, 4 candidates - TP: [Medium] Asynchronous functions deleteScheduledEmailReminder and deleteScheduledSMSRemind... - TP: [High] When immediateDelete is true, the deleteScheduledEmailReminder function cancels ... - -cal.com PR#14740: 5 golden, 3 candidates - TP: [High] Case sensitivity bypass in email blacklist... - TP: [Critical] The logic for checking team admin/owner permissions is incorrect. This condition... - TP: [Medium] This calls the email sender with the original guests, so existing attendees incl... - FN: [Medium] uniqueGuests filters out existing attendees and blacklisted emails but does not ... - FN: [Low] Starting with an array containing an empty string may cause validation issues. C... - -cal.com PR#10600: 4 golden, 4 candidates - TP: [Medium] Backup code validation is case-sensitive due to the use of indexOf(). This cause... - FN: [Low] The exported function TwoFactor handles backup codes and is in BackupCode.tsx. I... - FN: [Low] Error message mentions 'backup code login' but this is a disable endpoint, not l... - FN: [High] Because backupCodes are decrypted and mutated in memory before being written bac... - -cal.com PR#10967: 5 golden, 4 candidates - TP: [High] Potential null reference if mainHostDestinationCalendar is undefined if evt.dest... - TP: [High] Logic error: when externalCalendarId is provided, you're searching for a calenda... - TP: [Medium] Logic inversion in organization creation: The slug property is now conditionally... - FN: [Low] The optional chaining on mainHostDestinationCalendar?.integration is redundant s... - FN: [Low] The Calendar interface now requires createEvent(event, credentialId), but some i... - -cal.com PR#8087: 2 golden, 4 candidates - TP: [Critical] The code uses forEach with async callbacks, which causes asynchronous operations... - FN: [Low] Consider adding try-catch around the await to handle import failures gracefully... - -============================================================ -OVERALL RESULTS -============================================================ - True Positives: 63/137 - False Positives: 58 - False Negatives: 74 - Total Candidates: 119 - Precision: 52.9% - Recall: 46.0% - F1: 49.2% - -Per-repo breakdown: -Repo Prec Recall F1 TP FP FN -------------------------------------------------------- -cal.com 55.6% 51.7% 53.6% 15 12 14 -discourse 53.3% 61.5% 57.1% 16 14 10 -grafana 52.9% 45.0% 48.6% 9 9 11 -keycloak 61.1% 52.4% 56.4% 11 7 10 -sentry 44.4% 37.5% 40.7% 12 16 20 - -Results saved to /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/evaluation_results.json diff --git a/pr_review_agent/output/v26_single_loop.log b/pr_review_agent/output/v26_single_loop.log deleted file mode 100644 index 4716be7..0000000 --- a/pr_review_agent/output/v26_single_loop.log +++ /dev/null @@ -1,157 +0,0 @@ -============================================================ -PR Review Agent - Benchmark Pipeline -============================================================ -Loaded 50 PRs -WarpGrep: ENABLED -Reviewing 10 PRs - -Running 10 PRs with parallelism=10 - -[1/10] keycloak PR#37429 (4 golden) -[2/10] keycloak PR#37634 (4 golden) -[3/10] keycloak PR#38446 (2 golden) -[4/10] keycloak PR#36882 (1 golden) -[5/10] keycloak PR#36880 (3 golden) -[6/10] keycloak PR#37038 (2 golden) -[7/10] keycloak PR#33832 (2 golden) -[8/10] keycloak PR#40940 (2 golden) -[9/10] keycloak-greptile PR#1 (2 golden) -[10/10] sentry PR#93824 (5 golden) - [3/10] keycloak PR#38446 8 files, 256 added lines - [8/10] keycloak PR#40940 4 files, 51 added lines - [4/10] keycloak PR#36882 10 files, 56 added lines - [6/10] keycloak PR#37038 19 files, 831 added lines - [9/10] keycloak-greptile PR#1 9 files, 407 added lines - [2/10] keycloak PR#37634 28 files, 722 added lines - [7/10] keycloak PR#33832 12 files, 673 added lines - [5/10] keycloak PR#36880 10 files, 866 added lines - [10/10] sentry PR#93824 6 files, 199 added lines - [1/10] keycloak PR#37429 48 files, 343 added lines - WarpGrep: CompatibilityResult exit codes and how they are used with picocli.exit - WarpGrep: SpansBuffer constructor and assigned_shards property - WarpGrep: How does ClientPermissions class implement canMapRoles, canMapCompositeRoles, ca - WarpGrep: implementations of CryptoProvider interface order method - WarpGrep: getSubGroupsCount method definition and callers in GroupModel interface - WarpGrep: Lithuanian locale totpStep1 translation across different locale files - WarpGrep: isConditionalPasskeysEnabled method definition across all classes - WarpGrep: santizeAnchors method and how it processes anchor tags from value and englishVal - WarpGrep: SpanFlusher.main method signature and how it's called with partial - WarpGrep: canViewClientDefault method definition in ClientPermissions - WarpGrep: getFederatedCredentialsStream method definition on credential manager - WarpGrep: resolveUser and resolveClient methods in AdminPermissionsSchema - what do they r - WarpGrep: RECREATE_UPGRADE_EXIT_CODE usage and references - WarpGrep: concatenatedRSToASN1DER method implementations across the codebase - WarpGrep: UsernameForm class hierarchy and parent class - WarpGrep: MgmtPermissionsV2 class definition and its parent class methods - WarpGrep: UserCredentialManager updateCredential return type boolean - WarpGrep: getGroupModel method in GroupAdapter - WarpGrep: picocli.exit method and how exit codes are handled - WarpGrep: ProcessSpansStrategyFactory create_with_partitions method - WarpGrep: searchForUser method in UsersResource, how is canView filter applied to user sea - WarpGrep: modelSupplier field in GroupAdapter - WarpGrep: canManageDefault and canViewDefault in UserPermissions, who calls these methods - WarpGrep: run_with_initialized_sentry function signature and how it handles arguments - WarpGrep: canViewClientDefault method in ClientPermissions base class - WarpGrep: ClientPermissions class that ClientPermissionsV2 extends, getClientsWithPermissi - WarpGrep: setSubGroupCount callers and how they handle null from getSubGroupsCount - WarpGrep: resolveGroup resolveUser resolveClient in AdminPermissionsSchema, what do they r - WarpGrep: _import_and_run function definition that unpickles and runs main_fn - WarpGrep: RecoveryAuthnCodesCredentialProvider implements CredentialInputValidator isValid - WarpGrep: CryptoIntegration.init callers and how it handles multiple initializations - WarpGrep: RecoveryAuthnCodesFormAuthenticator action validate recovery code input challeng - WarpGrep: isConditionalPasskeysEnabled no-argument zero-argument overload - WarpGrep: how does process_healthy_since get initialized in SpanFlusher __init__ - WarpGrep: GroupPermissionsV2 canManage method checking VIEW scope - should it check only M - WarpGrep: LazyModel class implementation and how it handles null from supplier - WarpGrep: getClientsWithPermission method definition or callers - WarpGrep: AdminPermissions registerListener with Profile feature check ADMIN_FINE_GRAINED_ - Loop done round=7 (tools: read_file=10) - WarpGrep: Profile.isFeatureEnabled static method definition - WarpGrep: Feature getKey method definition and how feature names are converted to keys - WarpGrep: RecoveryAuthnCodesCredentialModel createFromValues and createFromCredentialModel - Review complete: 2 issues - [2/10] keycloak PR#37634 2 raw -> 2 kept - [2/10] keycloak PR#37634 [0.97] null_reference: Copy-paste error: the second Objects.requireNonNull checks `grantType` again instead of `r - [2/10] keycloak PR#37634 [0.97] test_correctness: Two bugs on one line: (1) Wrong substring indices — grant type shortcut is at indices 4–6 - WarpGrep: RecoveryAuthnCodesCredentialModel allCodesUsed getNextRecoveryAuthnCode credenti - WarpGrep: AdminPermissions class in permissions package, management method returning MgmtP - Loop done round=4 (tools: warpgrep_codebase_search=5, read_file=2, grep=1) - Review complete: 2 issues - [8/10] keycloak PR#40940 2 raw -> 2 kept - [8/10] keycloak PR#40940 [0.92] null_reference: `getSubGroupsCount()` returns `null` when `modelSupplier.get()` returns `null`, violating - [8/10] keycloak PR#40940 [0.88] test_correctness: The reader thread is started but never stored in a variable and never `join()`ed before th - WarpGrep: When is Profile initialized before CLI commands run in Keycloak quarkus runtime - WarpGrep: picocli.exit method signature and what type it accepts - WarpGrep connection error, retrying in 3s (attempt 1/3): SSLError - WarpGrep: BackwardsCompatibilityUserStorage implements interfaces CredentialInputUpdater - WarpGrep: UserCredentialModel buildFromBackupAuthnCode method definition - WarpGrep: RecoveryAuthnCodesUtils verifyRecoveryCodeInput hashRawCode method - WarpGrep: spans.buffer.flusher.wait_produce metric tag shard or shards usage - WarpGrep: AdminPermissions.java in permissions package, management method and registerList - WarpGrep: How operator handles update-compatibility check exit codes for rolling vs recrea - WarpGrep: Authenticator interface definition with isConditionalPasskeysEnabled - WarpGrep: resourceStore.findByType and granted.add in GroupPermissions, is resource.getId( - Loop done round=8 (tools: warpgrep_codebase_search=7, read_file=6, grep=3) - WarpGrep: configureTestRealm method in PasskeysUsernamePasswordFormTest or its parent clas - Review complete: 2 issues - [10/10] sentry PR#93824 2 raw -> 2 kept - [10/10] sentry PR#93824 [0.95] incorrect_value: The metric tag key `"shards"` (plural) is used for the `wait_produce` metric, while all ot - [10/10] sentry PR#93824 [0.95] test_correctness: `time.sleep` is monkeypatched to a no-op (`lambda _: None`) at the top of the test, but la - WarpGrep: Resource model interface getId getName in authorization, resource store internal - Loop done round=7 (tools: warpgrep_codebase_search=3, read_file=8, grep=1) - Review complete: 1 issues - [7/10] keycloak PR#33832 1 raw -> 1 kept - [7/10] keycloak PR#33832 [0.90] logic_error: Two ASN1Encoder instances are created and their results discarded on lines 115-116. These - Loop done round=15 (tools: warpgrep_codebase_search=2, read_file=6, grep=16, glob=2) - WarpGrep: KeycloakAdminPermissionsServerConfig class definition and realm config for Permi - Review complete: 4 issues - [1/10] keycloak PR#37429 4 raw -> 4 kept - [1/10] keycloak PR#37429 [0.98] localization: Italian text was placed in the Lithuanian locale file. The `totpStep1` value "Installa una - [1/10] keycloak PR#37429 [0.98] localization: Italian text was placed in the Lithuanian locale file. The `loginTotpStep1` value "Install - [1/10] keycloak PR#37429 [0.97] localization: Traditional Chinese characters (手機, 安裝, 應用程式) were used instead of Simplified Chinese (手机, - [1/10] keycloak PR#37429 [0.92] logic_error: StringIndexOutOfBoundsException in `verifySafeHtml` end-loop. The `end` loop does not acco - WarpGrep: RecoveryAuthnCodesCredentialProviderFactory PROVIDER_ID constant value - WarpGrep: JPAResourceStore create method implementation, how is id assigned when null - Loop done round=15 (tools: warpgrep_codebase_search=5, read_file=7, grep=5, glob=1, bash=2) - WarpGrep: FeatureSpec setEnabledFeatures method definition and what format features should - Review complete: 2 issues - [9/10] keycloak-greptile PR#1 2 raw -> 2 kept - [9/10] keycloak-greptile PR#1 [0.95] api_misuse: isConditionalPasskeysEnabled() is called with zero arguments, but the only definition of t - [9/10] keycloak-greptile PR#1 [0.92] logic_error: The isConditionalPasskeysEnabled method includes `&& user != null` which inverts the prior - WarpGrep: getTestKeycloakDeployment method definition for operator integration tests - WarpGrep: CredentialInputUpdater getCredentials default method interface - WarpGrep: where is groups().canManage() without arguments called in the admin REST resourc - WarpGrep: How operator converts FeatureSpec enabled features to Keycloak server configurat - WarpGrep: mapOptionFromCollection implementation in KeycloakDistConfigurator - Max tool rounds reached (tools: warpgrep_codebase_search=12, read_file=11, grep=9, glob=3) - WarpGrep: UserPermissionsV2 hasPermission method checking both specific user resource and - Review complete: 0 issues - [4/10] keycloak PR#36882 0 raw -> 0 kept - WarpGrep: delayed-authenticator provider implementation for testing - Loop done round=22 (tools: warpgrep_codebase_search=9, read_file=10, glob=3, grep=11) - WarpGrep: How does the authorization engine use ResourcePermission.getResourceType during - Review complete: 2 issues - [6/10] keycloak PR#37038 2 raw -> 2 kept - [6/10] keycloak PR#37038 [0.95] security: The no-arg `canManage()` method checks for both VIEW and MANAGE scopes via `hasPermission( - [6/10] keycloak PR#37038 [0.93] api_misuse: `groupResource.getId()` returns the internal resource store UUID, but `groupResource.getNa - WarpGrep: RecoveryAuthnCodeInputLoginBean constructor is called from where createLoginReco - WarpGrep: DefaultPolicyEvaluator evaluateResourceTypePolicies method implementation - Max tool rounds reached (tools: warpgrep_codebase_search=13, read_file=7, grep=10, list_directory=1, glob=4) - WarpGrep: EnterRecoveryAuthnCodePage getRecoveryAuthnCodeToEnterNumber method definition - Review complete: 0 issues - [5/10] keycloak PR#36880 0 raw -> 0 kept - WarpGrep: TestAppHelper completeLogin startLogin methods definition - WarpGrep: AppPage assertCurrent isCurrent implementation - WarpGrep: SetupRecoveryAuthnCodesPage getRecoveryAuthnCodes method implementation - Loop done round=24 (tools: warpgrep_codebase_search=17, read_file=12, grep=7) - Review complete: 2 issues - [3/10] keycloak PR#38446 2 raw -> 2 kept - [3/10] keycloak PR#38446 [0.95] test_correctness: Missing `testAppHelper.completeLogin()` call after recovery code sign-in. Without it, the - [3/10] keycloak PR#38446 [0.60] logic_error: `getCredentials()` reconstructs the credential model via `RecoveryAuthnCodesCredentialMode -Wrote candidates to /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/candidates.json - -============================================================ -DONE: 10 reviewed, 17 raw -> 17 filtered -Avg/PR: 1.7, Time: 882s -Candidates: /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/candidates.json -Benchmark data updated: /Users/tejas/personal/applymodel/zmisc/examples/code-review-benchmark/offline/results/benchmark_data.json diff --git a/pr_review_agent/output/v26_single_loop_eval.log b/pr_review_agent/output/v26_single_loop_eval.log deleted file mode 100644 index e69de29..0000000 diff --git a/pr_review_agent/output/v27_10random.log b/pr_review_agent/output/v27_10random.log deleted file mode 100644 index c5f7151..0000000 --- a/pr_review_agent/output/v27_10random.log +++ /dev/null @@ -1,4 +0,0 @@ - File "", line 111 - bdata[pr_url]['reviews'] = [r for r in bdata[pr_url].get('reviews', []) if r['tool'] \!= TOOL_NAME] - ^ -SyntaxError: unexpected character after line continuation character diff --git a/pr_review_agent/output/v27_guidelines.log b/pr_review_agent/output/v27_guidelines.log deleted file mode 100644 index 7b771c8..0000000 --- a/pr_review_agent/output/v27_guidelines.log +++ /dev/null @@ -1,145 +0,0 @@ -============================================================ -PR Review Agent - Benchmark Pipeline -============================================================ -Loaded 50 PRs -WarpGrep: ENABLED -Reviewing 10 PRs - -Running 10 PRs with parallelism=10 - -[1/10] keycloak PR#37429 (4 golden) -[2/10] keycloak PR#37634 (4 golden) -[3/10] keycloak PR#38446 (2 golden) -[5/10] keycloak PR#36880 (3 golden) -[6/10] keycloak PR#37038 (2 golden) -[4/10] keycloak PR#36882 (1 golden) -[8/10] keycloak PR#40940 (2 golden) -[7/10] keycloak PR#33832 (2 golden) -[9/10] keycloak-greptile PR#1 (2 golden) -[10/10] sentry PR#93824 (5 golden) - [8/10] keycloak PR#40940 4 files, 51 added lines - [2/10] keycloak PR#37634 28 files, 722 added lines - [4/10] keycloak PR#36882 10 files, 56 added lines - [10/10] sentry PR#93824 6 files, 199 added lines - [5/10] keycloak PR#36880 10 files, 866 added lines - [1/10] keycloak PR#37429 48 files, 343 added lines - [7/10] keycloak PR#33832 12 files, 673 added lines - [9/10] keycloak-greptile PR#1 9 files, 407 added lines - [6/10] keycloak PR#37038 19 files, 831 added lines - [3/10] keycloak PR#38446 8 files, 256 added lines - WarpGrep: ClientPermissions class definition and its canMapRoles, canMapCompositeRoles, ca - WarpGrep: SpanFlusher.main method signature and how it's called - WarpGrep: GroupAdapter getGroupModel method and modelSupplier field definition - WarpGrep: Who uses RECREATE_UPGRADE_EXIT_CODE and how is exit code 4 or 3 referenced - WarpGrep: santizeAnchors method usage and definition in VerifyMessageProperties - WarpGrep: Who implements CryptoProvider interface and what order() method do they return - WarpGrep: isConditionalPasskeysEnabled method definition in UsernameForm and its parent cl - WarpGrep: getFederatedCredentialsStream method definition on credential manager - WarpGrep: How is canManage() used for groups, what does it protect against? What are the c - WarpGrep: SpansBuffer constructor and assigned_shards property - WarpGrep: callers of picocli.exit and what arguments they pass - WarpGrep: Lithuanian locale messages_lt totpStep1 translation - WarpGrep: Who calls getSubGroupsCount on GroupModel and how do they handle null return - WarpGrep: updateCredential method on user credential manager returns boolean - WarpGrep: ClientPermissionsV2 parent class - the old ClientPermissions class that V2 exten - WarpGrep: How does CryptoIntegration.init work and who calls it - WarpGrep: UsernameForm class hierarchy and parent class AbstractUsernameFormAuthenticator - WarpGrep: resolveUser and resolveClient methods in AdminPermissionsSchema - what do they r - WarpGrep: run_with_initialized_sentry function signature and how it wraps target with addi - WarpGrep: messages_zh_CN.properties totpStep1 simplified chinese traditional chinese chara - WarpGrep: How are other getSubGroupsStream methods in infinispan GroupAdapter handling mod - WarpGrep: RecoveryAuthnCodesCredentialModel createFromValues method signature - WarpGrep: RecoveryAuthnCodesFormAuthenticator isRecoveryAuthnCodeInputValid how it validat - WarpGrep: concatenatedRSToASN1DER implementation in existing crypto providers - WarpGrep: santizeAnchors replaceFirst with value mutation while iterating with matcher - WarpGrep: resolveUser resolveClient in AdminPermissionsSchema - what format is the resourc - WarpGrep: GroupRepresentation setSubGroupCount method and what happens when null Long is p - WarpGrep: BackwardsCompatibilityUserStorage getCredentials method and how federated creden - WarpGrep: GroupRepresentation subGroupCount field type and getter setter - WarpGrep: RecoveryAuthnCodesCredentialProvider isValid method how recovery codes are valid - WarpGrep: UserCredentialManager getFederatedCredentialsStream implementation for federated - WarpGrep: Find all implementations of OAuth2GrantTypeFactory interface - WarpGrep: CredentialInputUpdater supportsCredentialType for recovery authn codes in user s - WarpGrep: LazyModel class implementation in cache infinispan - how does it cache the resul - WarpGrep: SpansBuffer slice_id usage and how it affects Redis operations - WarpGrep: Thread join in keycloak tests to ensure background thread completes before asser - WarpGrep: ASN1Decoder readSequence and how readNext tracks byte count with mark/reset - Loop done round=9 (tools: read_file=10, grep=3, warpgrep_codebase_search=1) - WarpGrep: getResourceTypeResource method definition in AdminPermissionsSchema - Review complete: 3 issues - [2/10] keycloak PR#37634 3 raw -> 3 kept - [2/10] keycloak PR#37634 [0.98] null_reference: Copy-paste error: the second Objects.requireNonNull checks `grantType` again instead of `r - [2/10] keycloak PR#37634 [0.97] incorrect_value: Wrong substring indices: `substring(3, 5)` extracts characters at indices 3-4, but the gra - [2/10] keycloak PR#37634 [0.97] logic_error: Inverted condition: the method returns false (does not match) when the grant shortcut equa - WarpGrep: AbstractRevisioned or parent class of CachedGroup that has getRealm method - WarpGrep: configureTestRealm method in PasskeysUsernamePasswordFormTest or its superclass - WarpGrep: AuthzClient.create called in the codebase to understand usage patterns - WarpGrep: printError method in AbstractUpdatesCommand or AbstractCommand for picocli comma - WarpGrep: UserCredentialModel constructor that takes id, type, challengeResponse - WarpGrep: Who calls getOrCreateResource in AdminPermissionsSchema and how is the returned - WarpGrep: ProcessSpansStrategyFactory constructor and kafka_slice_id parameter - WarpGrep: GroupPermissionsV2 getGroupIdsWithViewPermission - how is the result used to fil - WarpGrep: How does the operator handle update compatibility exit codes and what values doe - Loop done round=10 (tools: warpgrep_codebase_search=8, grep=2, read_file=5) - Review complete: 1 issues - [8/10] keycloak PR#40940 1 raw -> 1 kept - [8/10] keycloak PR#40940 [0.93] race_condition: The spawned reader thread is never saved to a variable nor .join()'d before the assertion. - Loop done round=11 (tools: warpgrep_codebase_search=4, read_file=4, grep=12, glob=1) - WarpGrep: PolicyStore findByResource method definition signature - Review complete: 4 issues - [1/10] keycloak PR#37429 4 raw -> 4 kept - [1/10] keycloak PR#37429 [0.98] localization: Italian text was inserted into the Lithuanian locale file. The value for `totpStep1` is "I - [1/10] keycloak PR#37429 [0.98] localization: Italian text was inserted into the Lithuanian locale file. The value for `loginTotpStep1` - [1/10] keycloak PR#37429 [0.97] localization: Traditional Chinese characters were used in the Simplified Chinese (zh_CN) locale file. Th - [1/10] keycloak PR#37429 [0.85] incorrect_value: Method name typo: `santizeAnchors` should be `sanitizeAnchors` (missing 'i' in "sanitize") - WarpGrep: getBouncyCastleProvider usage in keycloak codebase and what happens if provider - WarpGrep: How does searchForUserStream handle the GROUPS session attribute to filter users - WarpGrep: generatedRecoveryAuthnCodes hidden field in recovery codes form template how cod - WarpGrep: Where are ROLLING_UPGRADE_EXIT_CODE and RECREATE_UPGRADE_EXIT_CODE used in the c - WarpGrep: UserConfigBuilder class definition with id method - Loop done round=9 (tools: warpgrep_codebase_search=3, grep=5, read_file=7) - Max tool rounds reached (tools: warpgrep_codebase_search=5, grep=18, read_file=12, glob=2, bash=1) - WarpGrep: RecoveryAuthnCodesConfigBean generatedRecoveryAuthnCodesAsString how codes are j - Loop done round=8 (tools: warpgrep_codebase_search=6, read_file=8, grep=2) - Review complete: 2 issues - [9/10] keycloak-greptile PR#1 2 raw -> 2 kept - [9/10] keycloak-greptile PR#1 [0.95] api_misuse: `isConditionalPasskeysEnabled()` is called without the required `UserModel` argument. The - [9/10] keycloak-greptile PR#1 [0.92] logic_error: `isConditionalPasskeysEnabled` uses `user != null` which inverts the original behavior. Th - Review complete: 1 issues - [7/10] keycloak PR#33832 1 raw -> 1 kept - [7/10] keycloak PR#33832 [0.93] logic_error: Dead code: Lines 115-116 create two ASN1Encoder instances, write rBigInteger and sBigInteg - WarpGrep: How kafka_slice_id is passed to ProcessSpansStrategyFactory from consumer config - WarpGrep: RecoveryAuthnCodesConfigBean generatedRecoveryAuthnCodesAsString and generatedRe - Review complete: 0 issues - [4/10] keycloak PR#36882 0 raw -> 0 kept - WarpGrep: SetupRecoveryAuthnCodesPage getRecoveryAuthnCodes method implementation - WarpGrep: EnterRecoveryAuthnCodePage enterRecoveryAuthnCode method implementation - WarpGrep: What is the difference between Resource.getId() and Resource.getName() in the au - WarpGrep: getClientsWithPermission method definition or declaration - WarpGrep: How is canViewClientDefault() method used and defined - parent class of ClientPe - WarpGrep: resolveUser resolveClient in the V1 AdminPermissionsSchema (non-fgap package) - - Max tool rounds reached (tools: warpgrep_codebase_search=7, grep=9, glob=2, list_directory=1, bash=2, read_file=4) - WarpGrep: CredentialModel getUserLabel method and setUserLabel - Review complete: 0 issues - [5/10] keycloak PR#36880 0 raw -> 0 kept - Loop done round=16 (tools: warpgrep_codebase_search=6, grep=7, read_file=11) - Review complete: 2 issues - [10/10] sentry PR#93824 2 raw -> 2 kept - [10/10] sentry PR#93824 [0.95] incorrect_value: The `wait_produce` metric timer uses `tags={"shards": shard_tag}` (plural) while the `prod - [10/10] sentry PR#93824 [0.85] logic_error: When creating per-shard `SpansBuffer` instances in `_create_process_for_shards`, `SpansBuf - WarpGrep: Where is auth.groups().canManage() (without parameter) called to check permissio - Loop done round=18 (tools: warpgrep_codebase_search=9, read_file=6, grep=8, glob=2, bash=4) - Review complete: 2 issues - [6/10] keycloak PR#37038 2 raw -> 2 kept - [6/10] keycloak PR#37038 [0.95] security: The parameterless `canManage()` checks for both VIEW and MANAGE scopes via `hasPermission( - [6/10] keycloak PR#37038 [0.92] api_misuse: `getGroupIdsWithViewPermission()` uses `groupResource.getId()` (the authorization Resource - Max tool rounds reached (tools: warpgrep_codebase_search=15, grep=9, read_file=11) - Review complete: 0 issues - [3/10] keycloak PR#38446 0 raw -> 0 kept -Wrote candidates to /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/candidates.json - -============================================================ -DONE: 10 reviewed, 15 raw -> 15 filtered -Avg/PR: 1.5, Time: 533s -Candidates: /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/candidates.json -Benchmark data updated: /Users/tejas/personal/applymodel/zmisc/examples/code-review-benchmark/offline/results/benchmark_data.json diff --git a/pr_review_agent/output/v27_guidelines_eval.log b/pr_review_agent/output/v27_guidelines_eval.log deleted file mode 100644 index e69de29..0000000 diff --git a/pr_review_agent/output/v27_keycloak_eval.log b/pr_review_agent/output/v27_keycloak_eval.log deleted file mode 100644 index 624bc16..0000000 --- a/pr_review_agent/output/v27_keycloak_eval.log +++ /dev/null @@ -1,50 +0,0 @@ - -keycloak PR#37429: 4 golden, 4 candidates - TP: [Medium] The translation is in Italian instead of Lithuanian. This should be translated t... - TP: [Medium] The totpStep1 value uses Traditional Chinese terms in the Simplified Chinese fil... - TP: [Low] The method name 'santizeAnchors' should be 'sanitizeAnchors' (missing 'i').... - FN: [Low] The anchor sanitization logic has a potential issue where it consumes English ma... - -keycloak PR#37634: 4 golden, 3 candidates - TP: [Critical] Wrong parameter in null check (grantType vs. rawTokenId)... - TP: [High] In isAccessTokenId, the substring for the grant shortcut and the equality check ... - FN: [Low] Javadoc mentions "usually like 3-letters shortcut" but some implementations use ... - FN: [Low] Catching generic RuntimeException is too broad. The implementation throws Illeg... - -keycloak PR#37038: 2 golden, 2 candidates - TP: [High] Incorrect permission check in canManage() method... - TP: [High] In getGroupIdsWithViewPermission, hasPermission is called with groupResource.get... - -keycloak PR#33832: 2 golden, 1 candidates - TP: [Low] Dead code exists where ASN1Encoder instances are created and written to, but the... - FN: [High] Returns wrong provider (default keystore instead of BouncyCastle)... - -keycloak PR#40940: 2 golden, 1 candidates - TP: [Medium] The reader thread isn’t waited for; flipping deletedAll to true and asserting im... - FN: [Critical] Returning null from getSubGroupsCount() violates the GroupModel contract (Javado... - -keycloak-greptile PR#1: 2 golden, 2 candidates - TP: [Medium] ConditionalPasskeysEnabled() called without UserModel parameter... - TP: [Medium] With isConditionalPasskeysEnabled(UserModel user) requiring user != null, authen... - -keycloak PR#32918: 2 golden, 3 candidates - TP: [Medium] Cleanup reference uses incorrect alias - should be 'idp-alias-' + i instead of '... - FN: [Critical] Recursive caching call using session instead of delegate... - -============================================================ -OVERALL RESULTS -============================================================ - True Positives: 12/24 - False Positives: 4 - False Negatives: 12 - Total Candidates: 16 - Precision: 75.0% - Recall: 50.0% - F1: 60.0% - -Per-repo breakdown: -Repo Prec Recall F1 TP FP FN -------------------------------------------------------- -keycloak 75.0% 66.7% 70.6% 12 4 6 - -Results saved to /Users/tejas/personal/applymodel/zmisc/examples/pr_review_agent/output/evaluation_results.json diff --git a/pr_review_agent/pipeline/providers.py b/pr_review_agent/pipeline/providers.py index 3c4d4c9..8cfa4c4 100644 --- a/pr_review_agent/pipeline/providers.py +++ b/pr_review_agent/pipeline/providers.py @@ -229,13 +229,17 @@ def chat( model: str | None = None, ) -> LLMResponse: oai_messages = self._inject_system(messages, system) + model_name = model or "gpt-5.4" kwargs: dict[str, Any] = dict( - model=model or "gpt-5.4", + model=model_name, max_completion_tokens=max_tokens, messages=oai_messages, ) + # we have to do this because v1 chat completions API rejects function tools combined with reasoning_effort for GPT-5 models if tools: kwargs["tools"] = self.convert_tools(tools) + else: + kwargs["reasoning_effort"] = "high" raw = self._client.chat.completions.create(**kwargs) return self._parse(raw) @@ -686,7 +690,7 @@ def create_provider(config) -> LLMProvider: Returns: An LLMProvider implementation. """ - name = getattr(config, "provider", "anthropic").lower() + name = getattr(config, "provider", "openai").lower() if name == "anthropic": if not config.anthropic_api_key: diff --git a/pr_review_agent/prompts/system.py b/pr_review_agent/prompts/system.py index 01f7d17..9078421 100644 --- a/pr_review_agent/prompts/system.py +++ b/pr_review_agent/prompts/system.py @@ -42,7 +42,15 @@ - Test setup that contradicts the scenario being tested (e.g., test data configured to produce outcome A but the test claims to verify outcome B) - IMPORTANT: When you find a test bug, always trace backward — "What production behavior was this test verifying? Is that production code actually correct?" A test with wrong values often reveals the production code has the same confusion. -5. DEDUPLICATE BY ROOT CAUSE, NOT BY FILE +5. VERIFY COMPLETENESS OF NEW ADDITIONS +When a PR adds something new, check if all necessary counterparts exist: +- New resource creation (files, S3 objects, timers, subscriptions) → verify cleanup/teardown on error and on component unmount/disposal. If a `useEffect` creates a timer or subscription, the cleanup function must clear it. +- New database table or column → verify constraints (unique, not-null, foreign key) match the business logic. If the code assumes at most one row per (X, Y), there should be a unique constraint on (X, Y). +- New API endpoint → verify authorization, input validation, and error responses match sibling endpoints in the same controller. +- Changed one side of a read/write pair → verify the other side still works. If the write format changes, grep for all reads. +- New feature flag or config → verify both the enabled AND disabled paths work correctly. + +6. DEDUPLICATE BY ROOT CAUSE, NOT BY FILE When the same bug pattern appears in multiple files, report it ONCE for the most critical instance. In your comment, mention "this same pattern also appears in [file2, file3]." Two reports about the same root cause — even in different files — is one report. Before finalizing, review all your findings and merge any that share the same underlying cause. ## BUG CATEGORIES @@ -106,5 +114,6 @@ IMPORTANT: - Front load a lot of your search. Fire multiple concurrent warpgrep requests at the start. Be overly thorough. +- Conduct a very extensive search across multiple terms to find all possible bugs. Do not stop after a few queries — vary your search terms and cover every changed file. - ALWAYS cite code as a source in your comments, the code you cite must be from the diff this PR introduced. The code you cite, along with the bug description should be self-contained and should not require additional context to understand. Do not cite code outside the diff, and do not forget to cite code for every issue you find. """