#!/usr/bin/env python3
"""Merge master_claims + external verification results + usage map into site-v2/assets/claims.json."""
import json
import glob
import re
import collections
from pathlib import Path

EX = Path(__file__).resolve().parent
SITE = Path(__file__).resolve().parent.parent / "site"

master = json.load(open(EX / "master_claims.json"))

# usage recorded by the site generator (single source of truth for back-links)
su_path = EX / "site_usage.json"
if su_path.exists():
    usage = json.load(open(su_path))
else:
    usage = {}

# ---- external verification results -------------------------------------------------
ext_by_id = {}
src_meta_by_url = {}
missing_batches = []
for i in range(1, 13):
    p = EX / f"results_{i:02d}.json"
    if not p.exists():
        missing_batches.append(i)
        continue
    data = json.load(open(p))
    for s in data.get("sources", []):
        src_meta_by_url[s.get("url")] = {
            "retrieval": s.get("retrieval"),
            "fetched": s.get("fetched"),
            "resolved_title": s.get("resolved_title"),
            "resolved_date": s.get("resolved_date"),
            "title_match": s.get("title_match"),
        }
        for c in s.get("claims", []):
            ext_by_id[c["id"]] = {
                "verdict": c.get("verdict"),
                "checked": c.get("checked") or [],
                "not_visible": c.get("not_visible") or [],
                "quote": c.get("quote"),
                "notes": c.get("notes"),
                "checked_at": data.get("checked_at", "2026-07-15"),
                "retrieval": s.get("retrieval"),
            }

# ---- decision-use grades -------------------------------------------------------------
GRADES = json.load(open(EX / "decision_grades.json"))
MODEL_RE = re.compile(
    r"(GPT-5[\.\d]*|GPT-4o-mini|GPT-4o|GPT-4\.1[\w.-]*|GPT-4|GPT-3\.5|o1-preview|o1-mini|o1\b|o3-mini|o3\b|o4[\w.-]*|"
    r"Claude[- ]?3\.5[\w. -]*|Claude-Sonnet-4\.5|Claude Sonnet 4\.5|Gemini[- ]?[\d\.]+[\w-]*|gemini-2\.5-flash|"
    r"DeepSeek[\w.-]*|Llama[\w.-]*|Qwen[\w.-]*|PandaLM|Skywork[\w.-]*|MiniCheck|Lynx)")

def models_of(claim):
    txt = (claim.get("claim") or "") + " " + (claim.get("evidence") or "")
    seen = []
    for m in MODEL_RE.findall(txt):
        if m not in seen:
            seen.append(m)
    return seen[:8]

# ---- venue heuristic ----------------------------------------------------------------
PEER_MARKERS = [
    "ICLR", "NeurIPS", "ACL", "NAACL", "EMNLP", "AAAI", "CSCW", "UIST", "IUI",
    "PNAS", "Nature", "Cochrane", "Language Testing", "J. Occup", "Annual Review",
    "CHI", "LAK", "Findings",
]

def venue_of(claim):
    src = claim.get("source") or {}
    hay = " ".join(filter(None, [src.get("title"), src.get("date"), src.get("url"), claim.get("evidence")]))[:600]
    for mk in PEER_MARKERS:
        if mk.lower() in hay.lower():
            return {"peer_reviewed": True, "marker": mk}
    url = src.get("url") or ""
    if "arxiv.org" in url or "alphaxiv" in url or "openreview" in url:
        return {"peer_reviewed": False, "marker": "preprint"}
    if "eur-lex" in url or "iso.org" in url:
        return {"peer_reviewed": None, "marker": "regulation/standard"}
    if src.get("type") in ("practitioner", "community"):
        return {"peer_reviewed": None, "marker": src.get("type")}
    return {"peer_reviewed": None, "marker": src.get("type") or "unknown"}

# ---- same-source cross-citation index ------------------------------------------------
by_url = collections.defaultdict(list)
for c in master["claims"]:
    by_url[(c["source"] or {}).get("url")].append(c["id"])

# arXiv-id level grouping catches same paper cited via different URLs
def arxiv_id(url):
    m = re.search(r"(\d{4}\.\d{4,5})", url or "")
    return m.group(1) if m else None

by_paper = collections.defaultdict(list)
for c in master["claims"]:
    key = arxiv_id((c["source"] or {}).get("url")) or (c["source"] or {}).get("url")
    by_paper[key].append(c["id"])

# ---- assemble ------------------------------------------------------------------------
out_claims = []
for c in master["claims"]:
    url = (c["source"] or {}).get("url")
    paper_key = arxiv_id(url) or url
    siblings = [x for x in by_paper[paper_key] if x != c["id"]]
    entry = {
        "id": c["id"],
        "domain": c["domain"],
        "area": c.get("area"),
        "claim": c["claim"],
        "load_bearing": c["load_bearing"],
        "evidence": c["evidence"],
        "implication": c["implication"],
        "source": c["source"],
        "venue": venue_of(c),
        "same_source_claims": siblings,
        "in_corpus_verification": c.get("in_corpus_verification"),
        "external_verification": ext_by_id.get(c["id"]),
        "source_retrieval_meta": src_meta_by_url.get(url),
        "used_on": usage.get(c["id"], []),
        "decision_use": {**GRADES[c["id"]], "models_measured": models_of(c)},
    }
    out_claims.append(entry)

# ---- round-2 durable anchors (admitted only if verified; Owen's bar: relevant and certain)
da_candidates = []
da_admitted = 0
da_spec_path = EX / "da_anchors.json"
da_res_path = EX / "results_da.json"
if da_spec_path.exists() and da_res_path.exists():
    spec = {a["id"]: a for a in json.load(open(da_spec_path))["anchors"]}
    res = {r["id"]: r for r in json.load(open(da_res_path))["anchors"]}
    for aid, a in spec.items():
        r = res.get(aid)
        if r and r.get("verdict") in ("confirmed", "partially_confirmed"):
            basis = {"law": "law", "standard": "standard", "academic": "theorem (canonical economics)"}.get(a["source"]["type"], a["source"]["type"])
            out_claims.append({
                "id": aid,
                "domain": "durable-anchors",
                "area": None,
                "claim": a["claim"],
                "load_bearing": True,
                "evidence": a["evidence"],
                "implication": a["implication"],
                "source": a["source"],
                "venue": {"peer_reviewed": True if a["source"]["type"] == "academic" else None, "marker": a["source"]["type"]},
                "same_source_claims": [],
                "in_corpus_verification": None,
                "external_verification": {
                    "verdict": r["verdict"],
                    "checked": r.get("checked") or [],
                    "not_visible": r.get("not_visible") or [],
                    "quote": r.get("quote"),
                    "notes": r.get("notes"),
                    "checked_at": "2026-07-15",
                    "retrieval": r.get("retrieval") or "landing",
                },
                "source_retrieval_meta": {"fetched": r.get("fetched"), "resolved_title": r.get("resolved_title")},
                "used_on": usage.get(aid, a.get("used_on", [])),
                "decision_use": {"grade": "A", "basis": basis, "measured_on": "structural",
                                 "note": "Round-2 anchor: proposed from model knowledge 2026-07-15 under the 'relevant and certain' bar, then source-verified the same day. Not part of the original archive corpus; provenance disclosed by design.",
                                 "models_measured": []},
            })
            da_admitted += 1
        else:
            da_candidates.append({"id": aid, "title": a["source"]["title"],
                                  "reason": (r or {}).get("verdict", "no verification result"),
                                  "notes": (r or {}).get("notes")})

verdicts = collections.Counter((e["external_verification"] or {}).get("verdict") for e in out_claims)
grade_dist = collections.Counter(e["decision_use"]["grade"] for e in out_claims)
regime_dist = collections.Counter(e["decision_use"]["measured_on"] for e in out_claims)
DEPLOY_RE = re.compile(r"GPT-5\.[56]|Fable|Opus[- ]4\.8", re.I)
deploy_hits = [e["id"] for e in out_claims if DEPLOY_RE.search((e["claim"] or "") + " " + (e["evidence"] or ""))]
summary = {
    "generated": "2026-07-15",
    "n_claims": len(out_claims),
    "n_corpus_claims": 132,
    "n_da_admitted": da_admitted,
    "da_candidates_not_admitted": da_candidates,
    "n_sources": len([u for u in by_url if u]) + da_admitted,
    "n_load_bearing": sum(1 for e in out_claims if e["load_bearing"]),
    "n_used_on_site": sum(1 for e in out_claims if e["used_on"]),
    "external_verdicts": dict(verdicts),
    "in_corpus_verdicts": dict(collections.Counter(
        (e["in_corpus_verification"] or {}).get("verdict") for e in out_claims if e["in_corpus_verification"]
    )),
    "decision_grades": dict(grade_dist),
    "measured_on": dict(regime_dist),
    "deployment_tier_measurements": {"count": len(deploy_hits), "ids": deploy_hits,
        "definition": "claims whose measurement includes the deployment tier this program would run on (Claude Fable/Opus 4.8 class, GPT-5.5/5.6 class)"},
    "missing_batches": missing_batches,
    "method": {
        "layer1": "In-corpus adversarial spot-checks performed 2026-07-14 during the research sweep (27 load-bearing claims; transcripts preserved per claim).",
        "layer2": "Independent re-verification 2026-07-15: every claim checked against ONLY the source URL named in the corpus (plus served redirects and same-document arXiv variants). Verdicts derive exclusively from retrieved text; numbers not visible in retrievable text are listed per claim rather than assumed.",
        "constraint": "No substitute sources, no search engines, no background knowledge. Abstract-level retrieval cannot audit full-PDF-only numbers; those remain attributed to the corpus's own layer-1 transcripts where they exist.",
    },
    "corpus_meta": {
        "files": master["files"],
    },
}

payload = {"summary": summary, "claims": out_claims}
(SITE / "assets").mkdir(parents=True, exist_ok=True)
(SITE / "assets" / "claims.json").write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8")
print(json.dumps({k: v for k, v in summary.items() if k not in ("corpus_meta", "method")}, indent=2))