112 lines
4.3 KiB
Python
112 lines
4.3 KiB
Python
"""LLM-as-judge evaluation of review outputs against Ground Truth.
|
|
|
|
Rule-based matching (position hit + type match) cannot score L1 outputs,
|
|
which deliberately contain no line numbers or defect-type labels. Following
|
|
the cross-evaluation methodology of Liang et al., a fixed judge model
|
|
(temperature 0, reasoning disabled) decides whether a review output
|
|
semantically identifies each injected defect, and counts false alarms.
|
|
|
|
The judge verdict is stored alongside rule-based metrics so both remain
|
|
auditable.
|
|
"""
|
|
|
|
import json
|
|
import re
|
|
from typing import Any, Dict, List, Optional
|
|
|
|
JUDGE_PROMPT = """You are the judge in an automated code-review experiment. Exactly one defect was deliberately injected into the code under review.
|
|
|
|
## Injected defect (Ground Truth)
|
|
- Type: {defect_type}
|
|
- Location: lines {line_start}-{line_end} of the presented diff
|
|
- Description: {description}
|
|
- Reference fix: {reference_fix}
|
|
|
|
## Review output under evaluation
|
|
\"\"\"
|
|
{raw_output}
|
|
\"\"\"
|
|
|
|
Tasks:
|
|
1. Decide whether the review output correctly identifies the injected defect — i.e. it points out essentially the same problem, even if the wording, defect label, or line numbers differ or are absent.
|
|
2. Count how many DISTINCT additional problems the review claims that are clearly NOT the injected defect (false alarms). Ignore stylistic nits that are part of describing the injected defect.
|
|
3. If the review output mentions any line number(s) for the defect it identifies, list them; otherwise use null.
|
|
|
|
Answer with JSON only, no other text:
|
|
{{"detected": true or false, "false_alarms": <integer>, "lines_reported": [<integer>, ...] or null, "reason": "<one short sentence>"}}"""
|
|
|
|
|
|
def build_judge_prompt(raw_output: str, gt: Dict[str, Any]) -> str:
|
|
return JUDGE_PROMPT.format(
|
|
defect_type=gt["defect_type"],
|
|
line_start=gt.get("line_start"),
|
|
line_end=gt.get("line_end"),
|
|
description=gt.get("description") or "",
|
|
reference_fix=gt.get("reference_fix") or "",
|
|
raw_output=(raw_output or "").strip()[:12000],
|
|
)
|
|
|
|
|
|
def parse_verdict(text: str) -> Optional[Dict[str, Any]]:
|
|
"""Extract the JSON verdict from the judge's reply."""
|
|
if not text:
|
|
return None
|
|
match = re.search(r"\{.*\}", text, re.DOTALL)
|
|
if not match:
|
|
return None
|
|
try:
|
|
data = json.loads(match.group(0))
|
|
except json.JSONDecodeError:
|
|
return None
|
|
if "detected" not in data:
|
|
return None
|
|
lines = data.get("lines_reported")
|
|
if isinstance(lines, list):
|
|
lines = [int(x) for x in lines if isinstance(x, (int, float))]
|
|
else:
|
|
lines = None
|
|
return {
|
|
"detected": bool(data["detected"]),
|
|
"false_alarms": int(data.get("false_alarms") or 0),
|
|
"lines_reported": lines or None,
|
|
"reason": str(data.get("reason") or ""),
|
|
}
|
|
|
|
|
|
def verdict_to_metrics(verdict: Dict[str, Any], n_ground_truth: int = 1) -> Dict[str, float]:
|
|
"""Convert a judge verdict into detection/false-positive rates.
|
|
|
|
detection_rate: share of injected defects identified (0 or 1 per run).
|
|
false_positive_rate: false alarms / (false alarms + hits), matching the
|
|
thesis definition "proportion of reported problems that hit no injected
|
|
defect".
|
|
"""
|
|
hits = 1 if verdict["detected"] else 0
|
|
fa = max(0, verdict["false_alarms"])
|
|
detection_rate = hits / n_ground_truth if n_ground_truth else 0.0
|
|
total_reported = hits + fa
|
|
fpr = fa / total_reported if total_reported else 0.0
|
|
return {
|
|
"detection_rate": round(detection_rate, 4),
|
|
"false_positive_rate": round(fpr, 4),
|
|
}
|
|
|
|
|
|
def coverage_from_verdict(verdict: Dict[str, Any], ground_truth: List[Dict[str, Any]], tolerance: int = 3) -> Optional[float]:
|
|
"""Line coverage from judge-extracted line numbers.
|
|
|
|
Returns None when the review reported no line numbers (expected for L1),
|
|
so coverage stays NULL instead of polluting aggregates with zeros.
|
|
"""
|
|
lines = verdict.get("lines_reported")
|
|
if not lines:
|
|
return None
|
|
targets = [(gt["line_start"], gt.get("line_end") or gt["line_start"]) for gt in ground_truth if gt.get("line_start") is not None]
|
|
if not targets:
|
|
return None
|
|
hits = 0
|
|
for gt_start, gt_end in targets:
|
|
if any(gt_start - tolerance <= ln <= gt_end + tolerance for ln in lines):
|
|
hits += 1
|
|
return round(hits / len(targets), 4)
|