{ "batch": 5, "checked_at": "2026-07-15", "sources": [ { "url": "https://arxiv.org/abs/2606.03043", "retrieval": "abstract", "fetched": ["https://arxiv.org/abs/2606.03043"], "resolved_title": "The Geometry of LLM-as-Judge: Why Inter-LLM Consensus Is Not Human Alignment", "resolved_date": "2026-06-02", "title_match": true, "claims": [ { "id": "AJ-05", "verdict": "confirmed", "checked": [ "41 LLM judges, 4 community-built Indic datasets, 8 Indic languages", "judge/human score-spread ratio sigma_J/sigma_H approx 0.3-0.5", "principal angle to human axis 87-89 deg vs human-human 78-81 deg", "inter-LLM r_LL approx 0.35 vs LLM-human r_LH approx 0.27-0.32", "verifiable factual rubric: axis 58.5 deg, r_LH = 0.519", "fine-tuning recovers spread 0.32 -> 1.08 but axis stays 87-88 deg", "geometry measured with bootstrap confidence intervals; post-hoc calibration on a small human-anchored set improves all rubrics" ], "not_visible": [], "quote": "87 deg--89 deg versus 78 deg--81 deg ... r_LL approx 0.35 versus r_LH approx 0.27--0.32 ... axis 58.5 deg; r_LH = 0.519 ... sigma_J / sigma_H approx 0.3--0.5", "notes": "Every headline number in the claim is present verbatim in the abstract, including the verifiable-vs-subjective contrast (58.5 deg / r=0.519) and the fine-tuning spread recovery (0.32->1.08) with a near-stationary axis. Scope caveat (Indic community-health data) also matches." } ] }, { "url": "https://aclanthology.org/2022.tacl-1.78/", "retrieval": "abstract", "fetched": ["https://aclanthology.org/2022.tacl-1.78/"], "resolved_title": "Investigating Reasons for Disagreement in Natural Language Inference", "resolved_date": "TACL Volume 10 (2022), pp. 1357-1374", "title_match": true, "claims": [ { "id": "AQ-05", "verdict": "partially_confirmed", "checked": [ "taxonomy of disagreement sources with 10 categories spanning 3 high-level classes", "one class is uncertainty in the sentence meaning", "task is NLI (natural language inference) annotation disagreement" ], "not_visible": [ "'underspecification in guidelines' as a named high-level class (abstract instead names 'annotator biases and task artifacts')", "'survives perfect rubric operationalization' framing", "disagreement persisting specifically among trained annotators", "D3CODE (EMNLP 2024; 4.5K sentences, 4K+ annotators, 21 countries) - a separate source not on this page" ], "quote": "We developed a taxonomy of disagreement sources with 10 categories spanning 3 high-level classes. We found that some disagreements are due to uncertainty in the sentence meaning, others to annotator biases and task artifacts", "notes": "Core (10-category / 3-class taxonomy of intrinsic NLI disagreement) confirmed verbatim. Two of the claim's three named high-level sources map to the abstract ('uncertainty in sentence meaning'; 'annotator behavior' ~ 'annotator biases and task artifacts'); the third, 'underspecification in guidelines', is not named in the abstract. The D3CODE material cited in the evidence is a different paper, unverifiable from this URL. Only the abstract page is accessible (PDF/other same-site pages disallowed)." } ] }, { "url": "https://arxiv.org/abs/2506.11440", "retrieval": "fulltext", "fetched": ["https://arxiv.org/abs/2506.11440", "https://arxiv.org/html/2506.11440v1"], "resolved_title": "AbsenceBench: Language Models Can't Tell What's Missing", "resolved_date": "2025-06-13", "title_match": true, "claims": [ { "id": "CM-06", "verdict": "partially_confirmed", "checked": [ "'a massive 56.9% drop in F1-score on average' (omission vs insertion)", "best overall average F1 71.2 (Gemini-2.5-flash, thinking)", "highest GitHub-PR/code-diff score only 40.0% (by Claude-3.7-Sonnet thinking)", "nearly 99.5% F1 on poetry under the insertion (NIAH-style) setting", "'' placeholders boost performance 35.7% on average; +81.8% for Claude-3.7-Sonnet on GitHub PRs", "4302 instances total, average context length 5K tokens", "Mixtral-8x7B: perfect NIAH score but only 14.7% F1 on AbsenceBench", "inference-time compute (thinking) yields modest 7.9% improvement" ], "not_visible": [ "NeurIPS 2025 Datasets & Benchmarks acceptance (not in the abstract or HTML full text)" ], "quote": "AbsenceBench contains 4302 instances in total, with an average context length of 5K tokens. ... observing a massive 56.9% drop in F1-score on average ... This boosts the performance by a dramatic 35.7% on average", "notes": "All headline numbers are confirmed verbatim in the HTML full text. Minor attribution nuance: the claim pairs '71.2% overall' and '40.0% on code diffs' as one 'best model', but 71.2 is Gemini-2.5-flash's average while 40.0% is the domain-best set by Claude-3.7-Sonnet (thinking) (Gemini's GitHub-PR score is 30.9). Downgraded to partially_confirmed only because the asserted NeurIPS 2025 venue is not present in retrieved text." } ] }, { "url": "https://arxiv.org/abs/2512.10791", "retrieval": "fulltext", "fetched": ["https://arxiv.org/html/2512.10791v1"], "resolved_title": "The FACTS Leaderboard: A Comprehensive Benchmark for Large Language Model Factuality", "resolved_date": "2025-12", "title_match": true, "claims": [ { "id": "GF-07", "verdict": "confirmed", "checked": [ "top combo gemini-2.5-flash + v2 prompt: Macro-F1 65.33", "F1(+) 84.51 vs F1(-) 46.15 (positive/negative class asymmetry, ~85 vs ~46)", "held-out human-adjudicated evaluation set N=320 (class ratio 79:19)", "eligibility gate: ineligible/non-responsive answers deemed inaccurate", "explicit rationale that grounding metrics can be hacked by vague, short responses" ], "not_visible": [], "quote": "gemini-2.5-flash with the v2 prompt, scores Macro-F1 65.33, with F1(+) of 84.51 but F1(-) of only 46.15 ... the final factuality score is adjusted such that ineligible responses are deemed as inaccurate", "notes": "All four asserted specifics confirmed in the full text. The claim's mapping of F1(+)/F1(-) to 'grounded'/'ungrounded' class is a reasonable interpretation (majority positive class 79:19); paper labels them F1(+)/F1(-). The ~85/~46 figures are the top combo's, which the claim uses to characterize grounding judges generally." } ] }, { "url": "https://arxiv.org/abs/2601.20913", "retrieval": "abstract", "fetched": ["https://arxiv.org/abs/2601.20913"], "resolved_title": "Noisy but Valid: Robust Statistical Evaluation of LLMs with Imperfect Judges", "resolved_date": "2026-01-28", "title_match": true, "claims": [ { "id": "HS-07", "verdict": "confirmed", "checked": [ "small human-labelled calibration set estimates judge True Positive and False Positive Rates", "variance-corrected critical threshold applied to large judge-labelled dataset", "finite-sample Type-I error control (validity) despite calibration uncertainty", "derives exact conditions under which noisy testing yields higher power than direct evaluation", "validated on Jigsaw Comment, Hate Speech, SafeRLHF", "quantified 'Oracle' gap (cost of estimating judge parameters)", "Accepted to ICLR 2026" ], "not_visible": [], "quote": "leveraging a small human-labelled calibration set to estimate the judge's True Positive and False Positive Rates ... the exact conditions under which noisy testing yields higher statistical power than direct evaluation ... Accepted to ICLR2026", "notes": "Every element of the claim is present in the abstract, including the low-base-rate certification framing, finite-sample Type-I control under calibration uncertainty, the three datasets, the oracle gap, and ICLR 2026 acceptance. Authors match (Feng, Shen, Balashankar, Gerner-Beuerle, Rodrigues)." } ] }, { "url": "https://arxiv.org/html/2504.01848v2", "retrieval": "fulltext", "fetched": ["https://arxiv.org/html/2504.01848v2"], "resolved_title": "PaperBench: Evaluating AI's Ability to Replicate AI Research", "resolved_date": "2025-04-02", "title_match": true, "claims": [ { "id": "RR-08", "verdict": "confirmed", "checked": [ "8,316 leaf nodes / individually gradable tasks across 20 papers", "each leaf scored binary (1 pass / 0 fail)", "node weight = importance relative to siblings, not implementation difficulty", "rubrics co-developed with the paper's author(s); took multiple weeks per paper", "o3-mini SimpleJudge: F1 0.83 at $66 USD/paper (most cost-effective)", "human expert grading on the order of tens of hours per paper (cost modeled at 12 hrs x $100/hr)", "JudgeEval is a separate benchmark for evaluating automated judges", "o1 judge: 0.84 F1 at $830 USD/paper" ], "not_visible": [], "quote": "Across the 20 papers in PaperBench there are 8,316 leaf nodes. ... o3-mini with the SimpleJudge scaffolding is the most cost-effective, with an F1 score of 0.83 at $66 USD per paper", "notes": "All decomposition and cost figures confirmed verbatim in the full text, including sibling-relative weighting, weeks-long author co-development, the o3-mini 0.83/$66 vs o1 0.84/$830 tradeoff, JudgeEval as a separate judge benchmark, and both the 'tens of hours' and '12 hours at $100/hr' human-grading figures." } ] }, { "url": "https://www.braintrust.dev/articles/llm-as-a-judge-vs-human-in-the-loop-evals", "retrieval": "landing", "fetched": ["https://www.braintrust.dev/articles/llm-as-a-judge-vs-human-in-the-loop-evals"], "resolved_title": "LLM-as-a-judge vs human-in-the-loop evals: When to use each", "resolved_date": "2026-04-03", "title_match": true, "claims": [ { "id": "IP-08", "verdict": "confirmed", "checked": [ "LLM judge on 10,000 outputs (Claude Sonnet) might cost $5-15", "domain expert reviewing 500 outputs at $50-75/hr, 2-3 min each, runs $800-1,800", "roughly 100x more expensive per output for human review", "three-tier architecture: deterministic checks on every output, LLM judges for continuous coverage, humans only on flagged/low-confidence/disagreement cases + random spot-checks", "biggest mistake: 'Treating judge scores as ground truth without validating them against human labels'", "untrained reviewers / vague rubrics can yield labels 'worse than a decent LLM judge'" ], "not_visible": [], "quote": "Running an LLM judge on 10,000 outputs using a model like Claude Sonnet might cost $5-15 ... review 500 of those same outputs at $50-75/hour, spending 2-3 minutes per output, runs $800-1,800. That is roughly 100x more expensive per output", "notes": "Cost arithmetic, tiered architecture (deterministic + judge on everything, humans on flagged/disagreement slices plus random spot-checks), the named biggest-mistake, and the weak-human-review caveat all confirmed. Published date 3 April 2026 matches." } ] }, { "url": "https://aclanthology.org/2025.emnlp-main.144.pdf", "retrieval": "pdf", "fetched": ["https://aclanthology.org/2025.emnlp-main.144.pdf"], "resolved_title": "NUTMEG: Separating Signal From Noise in Annotator Disagreement", "resolved_date": "EMNLP 2025 (Nov 4-9, 2025), pp. 2874-2887", "title_match": true, "claims": [ { "id": "CT-11", "verdict": "partially_confirmed", "checked": [ "NUTMEG is a Bayesian model incorporating annotator backgrounds to remove noisy annotations while preserving systematic disagreements", "downstream models trained on NUTMEG-aggregated data significantly outperform models trained with traditional (majority-vote) aggregation", "evaluated on politeness and offensiveness tasks", "compared against MACE / Majority Vote and against disaggregated (no-aggregation) training", "measured by replicating the ground-truth label distribution by subgroup (Figure 7)", "authors Ivey (JHU), Gauch (Arkansas), Jurgens (Michigan); pp. 2874-2887; EMNLP 2025" ], "not_visible": [ "a uniform 'significantly outperform ... fully disaggregated training' result (the abstract's own outperformance claim is only vs 'traditional aggregation methods')" ], "quote": "downstream models trained on NUTMEG-aggregated data significantly outperform models trained on data from traditionally aggregation methods", "notes": "Core thesis (disagreement carries recoverable systematic signal; separating competence-noise from subpopulation disagreement; single-label aggregation destroys information) is confirmed, as is beating majority vote. But the claim's 'significantly outperform BOTH majority-vote AND fully disaggregated training' overstates the disaggregated arm: on politeness NUTMEG beats disaggregated on all splits except race, while on offensiveness it is only 'as or better than no-aggregation' (neutral, no significant gain). Hence partially_confirmed." } ] }, { "url": "https://doi.org/10.1177/026553229801500205", "retrieval": "failed", "fetched": ["https://doi.org/10.1177/026553229801500205", "https://journals.sagepub.com/doi/10.1177/026553229801500205"], "resolved_title": "unavailable (Cloudflare challenge; no article content served)", "resolved_date": "unavailable", "title_match": false, "claims": [ { "id": "CG-02", "verdict": "not_retrievable", "checked": [], "not_visible": [ "the asserted verbatim quote about intra- vs inter-rater reliability", "the 16-raters / operational ESL writing exam detail", "the finding that between-rater severity differences remained after training" ], "quote": "", "notes": "DOI redirect resolved (server 302: doi.org -> journals.sagepub.com/doi/10.1177/026553229801500205), confirming the DOI is valid and maps to the expected SAGE article path. However the SAGE landing page is behind a Cloudflare managed challenge: WebFetch returned only the SAGE homepage shell (no article text) and curl with browser headers returned HTTP 403 with a 'Just a moment...' JS challenge. No abstract or article content could be retrieved, so none of the claim's specifics can be verified from retrieved text. Per protocol, both WebFetch and curl were attempted." } ] }, { "url": "https://arxiv.org/html/2603.18053", "retrieval": "fulltext", "fetched": ["https://arxiv.org/html/2603.18053"], "resolved_title": "Auditing the Auditors: Does Community-based Moderation Get It Right?", "resolved_date": "arXiv:2603.18053v2 dated 2026-05-15 (v1 submission month March 2026 per the 2603 arXiv ID; source_date 2026-03-17)", "title_match": true, "claims": [ { "id": "CG-16", "verdict": "confirmed", "checked": [ "'consensus-based auditing' = rewarding agreement with the final aggregate outcome", "in September 2022 a platform (Community Notes) adopted consensus-based auditing tying eligibility for participation to agreement", "strategic conformity: minority contributors' evaluations drift toward the majority", "their participation share falls on controversial topics", "proposed two-stage algorithm weights contributors by the stability of their past residuals (predictability vs a latent-factor model) rather than majority-agreement", "consistently-informative contributors get greater influence even when they disagree; improves out-of-sample predictive performance" ], "not_visible": [ "author h-indices (Borgs h-54, Chayes h-60) - bibliometric metadata not on the page" ], "quote": "We find evidence of strategic conformity: minority contributors' evaluations drift toward the majority ... weights contributors by the stability of their past residuals ... even when they disagree ... improves out-of-sample predictive performance while avoiding penalization of disagreement", "notes": "Every element of the claim is confirmed in the HTML full text, including the Sept 2022 eligibility-tied consensus auditing, the strategic-conformity/minority-drift finding on controversial items, and the two-stage residual-stability aggregation. Retrieved page is v2 (2026-05-15); the arXiv ID and source_date (2026-03-17) correspond to the v1 submission. The reported two-stage improvement (mean abs residual ~5.73%, median ~27.99% on one-week-ahead predictions) is additional detail beyond the claim." } ] } ] }