#!/usr/bin/env python3 """Parse the AutoQA research corpus (9 domain files + critic-and-gapfill) into a master claims JSON.""" import json import re from pathlib import Path RESEARCH = Path(__file__).resolve().parent.parent / "site" / "documents" / "raw" / "research" OUT = Path(__file__).resolve().parent / "master_claims.json" DOMAIN_IDS = { "domain-academic-judges.md": ("academic-judges", "AJ"), "domain-annotation-quality.md": ("annotation-quality", "AQ"), "domain-critique-models.md": ("critique-models", "CM"), "domain-grounding-faithfulness.md": ("grounding-faithfulness", "GF"), "domain-hybrid-statistical.md": ("hybrid-statistical", "HS"), "domain-rubrics-recent.md": ("rubrics-recent", "RR"), "domain-industry-practice.md": ("industry-practice", "IP"), "domain-contrarian.md": ("contrarian", "CT"), "domain-frontier-2026.md": ("frontier-2026", "F26"), "critic-and-gapfill.md": ("critic-and-gapfill", "CG"), } META_HEADINGS = ("Tools & artifacts", "Disagreements", "Gaps", "Verifications", "Weak spots", "Contents") def parse_source_line(line): body = line[len("SOURCE:"):].strip() parts = [p.strip() for p in body.split("|")] src = {"raw": body, "title": None, "url": None, "date": None, "type": None} if len(parts) >= 4: src["title"] = parts[0] src["url"] = parts[1] src["date"] = parts[2] src["type"] = parts[3] else: # fallback: find first URL token m = re.search(r"https?://\S+", body) src["title"] = parts[0] if parts else body src["url"] = m.group(0) if m else None if len(parts) > 2: src["date"] = parts[2] if len(parts) > 3: src["type"] = parts[3] return src def parse_file(path, domain, prefix): text = path.read_text(encoding="utf-8") lines = text.split("\n") claims = [] disagreements, gaps, weak_spots = [], [], [] verifications = [] current = None section = None # meta section name when inside one area = None def close_current(): nonlocal current if current and current.get("source"): claims.append(current) current = None i = 0 while i < len(lines): line = lines[i] h = re.match(r"^(#{1,3}) (.*)$", line) if h: level, title = len(h.group(1)), h.group(2).strip() plain = title.rstrip() meta = next((m for m in META_HEADINGS if plain.startswith(m)), None) if meta: close_current() section = meta i += 1 continue if plain.startswith("Area: "): close_current() area = plain[len("Area: "):] section = None i += 1 continue if level == 1: close_current() section = None if plain.startswith("Gap-fill"): area = None i += 1 continue # claim heading (## in domains, ### in gap-fill) close_current() section = None load_bearing = "[LOAD-BEARING]" in plain claim_text = plain.replace("[LOAD-BEARING]", "").strip() current = { "id": None, "domain": domain, "area": area, "claim": claim_text, "load_bearing": load_bearing, "evidence": None, "source": None, "implication": None, } i += 1 continue if section == "Verifications": m = re.match(r"^- CLAIM: (.*)$", line) if m: verifications.append({"claim": m.group(1).strip(), "verdict": None, "transcript": None, "corrected": None}) else: v = re.match(r"^\s+VERDICT: ([a-z_]+) \| (.*)$", line) c = re.match(r"^\s+CORRECTED: (.*)$", line) if v and verifications: verifications[-1]["verdict"] = v.group(1) verifications[-1]["transcript"] = v.group(2).strip() elif c and verifications: verifications[-1]["corrected"] = c.group(1).strip() elif verifications and line.strip() and verifications[-1]["transcript"] is not None and not line.startswith("- "): verifications[-1]["transcript"] += " " + line.strip() elif section == "Disagreements": if line.startswith("- "): disagreements.append(line[2:].strip()) elif line.strip() and disagreements and not line.startswith("#"): disagreements[-1] += " " + line.strip() elif section == "Gaps": if line.startswith("- "): gaps.append(line[2:].strip()) elif line.strip() and gaps and not line.startswith("#"): gaps[-1] += " " + line.strip() elif section == "Weak spots": if line.startswith("- "): weak_spots.append(line[2:].strip()) elif line.strip() and weak_spots and not line.startswith("#"): weak_spots[-1] += " " + line.strip() elif current is not None: if line.startswith("EVIDENCE:"): current["evidence"] = line[len("EVIDENCE:"):].strip() elif line.startswith("SOURCE:"): current["source"] = parse_source_line(line) elif line.startswith("IMPLICATION:"): current["implication"] = line[len("IMPLICATION:"):].strip() elif line.strip() and current.get("implication") is not None: current["implication"] += " " + line.strip() elif line.strip() and current.get("evidence") is not None and current.get("source") is None: current["evidence"] += " " + line.strip() i += 1 close_current() # attach verifications by fuzzy prefix match def norm(s): return re.sub(r"\W+", "", s.lower())[:100] for v in verifications: key = norm(v["claim"]) best = None for c in claims: if norm(c["claim"]) == key or norm(c["claim"]).startswith(key[:80]) or key.startswith(norm(c["claim"])[:80]): best = c break if best is not None: best["in_corpus_verification"] = { "verdict": v["verdict"], "transcript": v["transcript"], "corrected": v["corrected"], } else: v["_unmatched"] = True for n, c in enumerate(claims, 1): c["id"] = f"{prefix}-{n:02d}" unmatched = [v for v in verifications if v.get("_unmatched")] return { "domain": domain, "claims": claims, "disagreements": disagreements, "gaps": gaps, "weak_spots": weak_spots, "n_verifications": len(verifications), "unmatched_verifications": [v["claim"][:90] for v in unmatched], } def main(): master = {"generated": "2026-07-15", "files": {}, "claims": []} totals = {"claims": 0, "with_source_url": 0, "load_bearing": 0, "verified_in_corpus": 0} for fname, (domain, prefix) in DOMAIN_IDS.items(): parsed = parse_file(RESEARCH / fname, domain, prefix) master["files"][fname] = { "domain": domain, "n_claims": len(parsed["claims"]), "disagreements": parsed["disagreements"], "gaps": parsed["gaps"], "weak_spots": parsed["weak_spots"], "n_verifications": parsed["n_verifications"], "unmatched_verifications": parsed["unmatched_verifications"], } master["claims"].extend(parsed["claims"]) totals["claims"] += len(parsed["claims"]) totals["with_source_url"] += sum(1 for c in parsed["claims"] if c["source"] and c["source"].get("url")) totals["load_bearing"] += sum(1 for c in parsed["claims"] if c["load_bearing"]) totals["verified_in_corpus"] += sum(1 for c in parsed["claims"] if c.get("in_corpus_verification")) master["totals"] = totals OUT.write_text(json.dumps(master, indent=2, ensure_ascii=False), encoding="utf-8") print(json.dumps(totals, indent=2)) for fname, info in master["files"].items(): print(f"{fname}: {info['n_claims']} claims, {info['n_verifications']} verifications, {len(info['unmatched_verifications'])} unmatched, {len(info['disagreements'])} disagreements, {len(info['gaps'])} gaps") if __name__ == "__main__": main()