first commit

2026-09-19 12:54:45 +08:00
commit 6fc5b64077
126 changed files with 8601 additions and 0 deletions
@@ -0,0 +1,112 @@
+"""Batch-judge all completed runs of an experiment with the LLM judge.
+
+Usage: python judge_all.py <experiment_id> [judge_model] [concurrency]
+
+For every done run: the judge decides detection + false alarms (semantic,
+level-agnostic), while line coverage is computed rule-based from parsed
+findings. Idempotent: runs whose result already carries a judge verdict
+(judge_verdict key in parsed_findings) are skipped.
+"""
+
+import asyncio
+import sys
+import uuid
+
+from app.analysis.judge import (
+    build_judge_prompt,
+    coverage_from_verdict,
+    parse_verdict,
+    verdict_to_metrics,
+)
+from app.analysis.parser import parse_output
+from app.db import SessionLocal, init_db
+from app.model_adapters.factory import create_adapter
+from app.models import ExperimentRun
+
+
+async def main() -> None:
+    experiment_id = uuid.UUID(sys.argv[1])
+    judge_model = sys.argv[2] if len(sys.argv) > 2 else "deepseek"
+    concurrency = int(sys.argv[3]) if len(sys.argv) > 3 else 5
+
+    init_db()
+    db = SessionLocal()
+    adapter = create_adapter(judge_model)
+
+    runs = (
+        db.query(ExperimentRun)
+        .filter(ExperimentRun.experiment_id == experiment_id, ExperimentRun.status == "done")
+        .all()
+    )
+    todo = []
+    for run in runs:
+        if not run.result or not run.result.raw_output:
+            continue
+        pf = run.result.parsed_findings
+        if isinstance(pf, dict) and pf.get("judge_verdict"):
+            continue
+        todo.append(run)
+    print(f"{len(todo)} runs to judge (judge={judge_model}, concurrency={concurrency})")
+
+    semaphore = asyncio.Semaphore(concurrency)
+
+    async def judge_one(run):
+        async with semaphore:
+            gt_list = [
+                {
+                    "defect_type": d.defect_type,
+                    "line_start": d.line_start,
+                    "line_end": d.line_end,
+                    "description": d.description,
+                    "reference_fix": d.reference_fix,
+                }
+                for d in run.sample.defects
+            ]
+            gt = gt_list[0] if gt_list else {}
+            prompt = build_judge_prompt(run.result.raw_output, gt)
+            last_err = None
+            for _ in range(3):
+                try:
+                    resp = await adapter.chat(prompt, {"temperature": 0.0, "max_tokens": 2048})
+                    verdict = parse_verdict(resp.text)
+                    if verdict:
+                        break
+                except Exception as e:  # noqa: BLE001
+                    last_err = e
+                    await asyncio.sleep(2)
+            else:
+                print(f"WARN judge failed for run {run.id}: {last_err or 'unparseable verdict'}")
+                return None
+
+            metrics = verdict_to_metrics(verdict, n_ground_truth=len(gt_list) or 1)
+            coverage = coverage_from_verdict(verdict, gt_list)
+            level = run.template_version.template.level
+            findings = parse_output(run.result.raw_output, level)
+
+            run.result.detection_rate = metrics["detection_rate"]
+            run.result.false_positive_rate = metrics["false_positive_rate"]
+            run.result.coverage_rate = coverage
+            run.result.parsed_findings = {
+                "judge_model": judge_model,
+                "judge_verdict": verdict,
+                "findings": [
+                    {
+                        "defect_type": f.defect_type,
+                        "line_start": f.line_start,
+                        "line_end": f.line_end,
+                        "description": f.description,
+                    }
+                    for f in findings
+                ],
+            }
+            db.commit()
+            return verdict
+
+    results = await asyncio.gather(*(judge_one(run) for run in todo))
+    done = sum(1 for r in results if r)
+    detected = sum(1 for r in results if r and r["detected"])
+    print(f"judged {done}/{len(todo)}; detected {detected}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())