[ { "id": "P01", "type": "proof", "title": "Record-identifiability boundary", "claim": "A record-only evaluator cannot be perfect on a population that contains two cases with identical admitted inputs but different correct outcomes.", "supports": "Expand the admitted evidence boundary, revise the task, or allow an insufficient-evidence outcome when the record does not identify the answer.", "does_not_support": "It does not imply that useful imperfect classification is impossible, or that every AutoQA lane is unidentifiable.", "source": "Internal proof in this edition", "url": "limits.html#record-identifiability", "version": "1.0-grounded", "source_type": "Logical derivation", "status": "Derived and checked", "checked": "2026-07-22" }, { "id": "P02", "type": "proof", "title": "Pairwise agreement is not an accuracy ceiling", "claim": "Pairwise agreement between experts is not, in general, a numerical upper bound on a model’s true accuracy or on its agreement with either expert.", "supports": "Use inter-rater agreement as evidence about reference reliability and task uncertainty, not as a universal accuracy ceiling.", "does_not_support": "It does not make low agreement irrelevant. Low agreement may indicate ambiguity, rater error, heterogeneous standards, or an unstable reference process.", "source": "Constructive counterexample in this edition", "url": "limits.html#reference-uncertainty", "version": "1.0-grounded", "source_type": "Logical counterexample", "status": "Derived and checked", "checked": "2026-07-22" }, { "id": "P03", "type": "proof", "title": "Finite instructions can be total", "claim": "A finite instruction can completely determine every case in a specified domain; for example, “pass exactly when field X equals yes” is total over a defined record schema.", "supports": "Instruction ambiguity must be demonstrated or measured for the actual contract; it does not follow merely from the instructions being finite.", "does_not_support": "It does not imply that open-ended natural-language policies are complete or unambiguous.", "source": "Constructive counterexample in this edition", "url": "limits.html#instruction-closure", "version": "1.0-grounded", "source_type": "Logical counterexample", "status": "Derived and checked", "checked": "2026-07-22" }, { "id": "P04", "type": "proof", "title": "Exhaustive finite search can certify absence", "claim": "For a finite, correctly parsed domain and a decidable predicate, exhaustive checking can certify that no matching item exists in that domain.", "supports": "Exact missing-field, exact-token, and other enumerable checks can support genuine absence claims when their domain and parser are controlled.", "does_not_support": "A non-exhaustive semantic search result does not establish that a concept is absent; it establishes only that the configured detector did not find it.", "source": "Direct finite-enumeration argument in this edition", "url": "limits.html#presence-absence", "version": "1.0-grounded", "source_type": "Logical derivation", "status": "Derived and checked", "checked": "2026-07-22" }, { "id": "P05", "type": "proof", "title": "Candidate-gated recall bound", "claim": "When every true positive must first be emitted by a candidate extractor, end-to-end recall cannot exceed that extractor’s recall.", "supports": "Evaluate extraction and retrieval recall separately; downstream judging cannot recover targets that never enter its candidate set.", "does_not_support": "The bound does not apply if the downstream system has an independent path that can discover omitted targets.", "source": "Set-inclusion derivation in this edition", "url": "limits.html#pipeline", "version": "1.0-grounded", "source_type": "Logical derivation", "status": "Derived and checked", "checked": "2026-07-22" }, { "id": "S01", "type": "stat", "title": "One-sided zero-failure binomial bound", "claim": "For n independent, representative Bernoulli trials with zero observed failures, the exact one-sided 1−α upper confidence bound is pᵤ = 1 − α^(1/n). At 95% confidence, n = 59, 299, and 2,995 put the upper bound below 5%, 1%, and 0.1%, respectively.", "supports": "A transparent sample-size illustration for a prespecified binary failure event under the stated sampling model.", "does_not_support": "It does not establish representativeness, independence, stationarity, class coverage, or a universal launch threshold.", "source": "Clopper & Pearson, “The Use of Confidence or Fiducial Limits Illustrated in the Case of the Binomial”", "url": "https://doi.org/10.1093/biomet/26.4.404", "version": "Biometrika 26(4), 1934", "source_type": "Peer-reviewed statistical paper plus direct derivation", "status": "Verified", "checked": "2026-07-22" }, { "id": "E01", "type": "empirical", "title": "Human uncertainty can be obscured by one aggregate agreement score", "claim": "Elangovan et al. show that a single aggregate correlation or agreement score can hide differences between human-label variation and machine labels, and recommend stratifying evaluation by human-label uncertainty.", "supports": "Report judge performance across reference-uncertainty strata rather than relying on one aggregate agreement number.", "does_not_support": "The paper does not establish pairwise human agreement as a mathematical ceiling on machine accuracy.", "source": "Elangovan et al., “Beyond correlation: The Impact of Human Uncertainty in Measuring the Effectiveness of Automatic Evaluation and LLM-as-a-Judge”", "url": "https://arxiv.org/abs/2410.03775v3", "version": "arXiv v3; accepted at ICLR 2025", "source_type": "Peer-reviewed conference paper", "status": "Verified against source version", "checked": "2026-07-22" }, { "id": "E02", "type": "empirical", "title": "Forced-choice reference construction can change judge selection", "claim": "Across 11 rating tasks and 9 commercial LLMs, Guerdan et al. report that forced-choice validation can select substantially suboptimal judges; in one toxicity example, the selected judge was 31% less consistent with human downstream decisions than the judge selected with response-set ratings.", "supports": "Where multiple ratings can reasonably apply, collect response sets or distributions and pin the source version used for numerical claims.", "does_not_support": "The 31% result is empirical and task-specific, not a proof that every forced-choice evaluation is biased by that amount.", "source": "Guerdan et al., “Validating LLM-as-a-Judge Systems under Rating Indeterminacy”", "url": "https://arxiv.org/abs/2503.05965v4", "version": "NeurIPS 2025 / arXiv v4", "source_type": "Peer-reviewed conference paper", "status": "Verified against v4", "checked": "2026-07-22" }, { "id": "E03", "type": "empirical", "title": "Aggregate judge scores can conceal large class differences", "claim": "On a held-out FACTS Grounding v2 judge test set of 320 cases with a 79:19 class ratio, Gemini 2.5 Flash with the v2 prompt had Macro-F1 65.33 and class-specific F1 values 84.51 and 46.15. The benchmark also uses an eligibility screen to reject responses that are grounded but too vague to answer the request.", "supports": "Report class-specific metrics and task completion or eligibility separately from factual grounding.", "does_not_support": "These values are not universal performance figures for all grounding judges, datasets, prompts, or AutoQA records.", "source": "The FACTS Leaderboard: A Comprehensive Benchmark for Large Language Model Factuality", "url": "https://arxiv.org/abs/2512.10791v1", "version": "arXiv v1, December 2025", "source_type": "Benchmark preprint", "status": "Verified against v1", "checked": "2026-07-22" }, { "id": "E04", "type": "empirical", "title": "Hard rubric-level judgments remain benchmark-sensitive", "claim": "RubricEval contains 3,486 rubric instances. On its Hard subset, GPT-4o-2024-11-20 is reported at 55.97% balanced accuracy; results vary materially by model and instruction type.", "supports": "Evaluate each rubric lane on task-specific hard cases and avoid extrapolating from easy aggregate results.", "does_not_support": "One model result on a preprint benchmark is not a permanent capability ceiling or an estimate of AutoQA performance.", "source": "RubricEval: A Rubric-Level Meta-Evaluation Benchmark for LLM Judges in Instruction Following", "url": "https://arxiv.org/abs/2603.25133v1", "version": "arXiv v1, March 2026", "source_type": "Preprint", "status": "Verified against v1", "checked": "2026-07-22" }, { "id": "E05", "type": "empirical", "title": "Long-form judge reliability varies across scenarios", "claim": "LongJudgeBench reports a reliability gap for long-form evaluation: tested LLM judges were unstable across scenarios, while rubrics and references helped but were not always sufficient.", "supports": "Use long-record challenge sets that vary scenario, evidence position, required coverage, and cross-section consistency.", "does_not_support": "The preprint does not estimate AutoQA’s long-record performance or prove that long-form evaluation is impossible.", "source": "Chen et al., “Benchmarking LLM-as-a-Judge for Long-Form Output Evaluation”", "url": "https://arxiv.org/abs/2606.01629v2", "version": "arXiv v2, June 2026", "source_type": "Preprint", "status": "Verified against v2 abstract", "checked": "2026-07-22" }, { "id": "E06", "type": "empirical", "title": "Reliability metrics and rankings can change across evaluation protocols", "claim": "A 2026 preprint evaluates 21 judge models from nine providers across three benchmarks, 118 runs, and 541,000 judgments. It reports a 33–41 percentage-point gap between raw agreement and Cohen’s kappa on MT-Bench and rank shifts of up to 14 places across benchmarks.", "supports": "Use chance-corrected and task-specific metrics, test repeatability and position effects, and do not assume judge rankings transfer across benchmarks.", "does_not_support": "The study is a v1 preprint, does not characterize every judge configuration, and states that reasoning-enabled configurations could differ.", "source": "Norman, Rivera & Hughes, “Reliability without Validity”", "url": "https://arxiv.org/abs/2606.19544v1", "version": "arXiv v1, June 2026", "source_type": "Preprint", "status": "Verified against v1", "checked": "2026-07-22" }, { "id": "E07", "type": "empirical", "title": "Evidence-only assistance helped in one fact-verification study, then stopped helping as raters improved", "claim": "In a fact-verification study, AI alone scored 87.7%, average individual humans 75.1%, an unassisted hybrid 89.3%, and an evidence-assisted hybrid 91.3%. In a later wave with more practiced raters, the paper reports that evidence-only assistance ceased to help and leading assistance forms became harmful.", "supports": "A randomized AutoQA human-loop pilot should compare evidence-only and verdict-visible interfaces and should test performance over time.", "does_not_support": "This adjacent preprint does not show that AutoQA helps its intended reviewers, records, or decisions.", "source": "“Human-AI Complementarity: A Goal for Amplified Oversight”", "url": "https://arxiv.org/abs/2510.26518v2", "version": "arXiv v2, 2026", "source_type": "Preprint", "status": "Verified against v2", "checked": "2026-07-22" }, { "id": "E08", "type": "empirical", "title": "Optional LLM feedback changed peer reviews in a large randomized study", "claim": "At ICLR 2025, optional feedback was offered to more than 20,000 randomly selected reviews. Twenty-seven percent of recipients updated their reviews, more than 12,000 suggestions were incorporated, updated reviews grew by an average of 80 words, and blinded researchers rated them as more informative.", "supports": "LLM-generated feedback can change reviewer behavior and can be evaluated with randomized deployment designs.", "does_not_support": "Longer or more informative reviews are not the same as correct AutoQA judgments, and transfer to another review domain is unestablished.", "source": "Thakkar et al., “Can LLM feedback enhance review quality? A randomized study of 20K reviews at ICLR 2025”", "url": "https://arxiv.org/abs/2504.09737v1", "version": "arXiv v1, April 2025", "source_type": "Randomized field-study preprint", "status": "Verified against v1 abstract", "checked": "2026-07-22" }, { "id": "E09", "type": "empirical", "title": "Human–AI combinations are heterogeneous and do not automatically beat the better component", "claim": "A preregistered meta-analysis covering 106 experiments and 370 effect sizes found that human–AI combinations were, on average, worse than the better of the human-only and AI-only baselines, while better than humans alone. Decision tasks showed average losses; creation tasks differed in direction.", "supports": "Evaluate the combined workflow against both human-only and system-only baselines rather than assuming synergy.", "does_not_support": "The average does not determine the result for a specific AutoQA interface, task, or population.", "source": "Vaccaro, Almaatouq & Malone, “When combinations of humans and AI are useful”", "url": "https://www.nature.com/articles/s41562-024-02024-1", "version": "Nature Human Behaviour 8, 2293–2303 (2024)", "source_type": "Peer-reviewed systematic review and meta-analysis", "status": "Verified", "checked": "2026-07-22" }, { "id": "E10", "type": "empirical", "title": "Selective prediction trades coverage for risk under a validated method", "claim": "Geifman and El-Yaniv present a selective-classification method that rejects some cases to target a desired risk level with high probability under its evaluation setup.", "supports": "Measure risk as a function of coverage and validate the abstention rule on held-out data.", "does_not_support": "The existence of an abstain label does not by itself guarantee low risk, calibration, or safe transfer to a changed population.", "source": "Geifman & El-Yaniv, “Selective Classification for Deep Neural Networks”", "url": "https://arxiv.org/abs/1705.08500v2", "version": "ICML 2017 / arXiv v2", "source_type": "Peer-reviewed conference paper", "status": "Verified", "checked": "2026-07-22" }, { "id": "E11", "type": "empirical", "title": "A checklist assistant was useful to many participants and manipulable in experiments", "claim": "In the NeurIPS 2024 experiment, 234 papers were voluntarily submitted; more than 70% of surveyed authors found the assistant useful. The authors also report that fabricated justifications could manipulate scores.", "supports": "Test both usefulness and gaming resistance, and avoid using the tool as an unreviewed automated gate.", "does_not_support": "The paper does not conclude that the assistant was a “poor substitute,” nor does its abstract establish that it was “gamed in a single shot.”", "source": "Goldberg et al., “Usefulness of LLMs as an Author Checklist Assistant for Scientific Papers: NeurIPS’24 Experiment”", "url": "https://arxiv.org/abs/2411.03417", "version": "arXiv, November 2024", "source_type": "Conference field experiment preprint", "status": "Verified against abstract", "checked": "2026-07-22" }, { "id": "E12", "type": "empirical", "title": "FaithBench numbers require dataset- and aggregation-specific labels", "claim": "In FaithJudge Table 1, Bespoke-MiniCheck has 60.1% claim-wise and 55.7% summary-wise balanced accuracy on FaithBench; 71.2% is its claim-wise average across four datasets. FaithJudge with o3-mini-high is reported at 84.0% balanced accuracy and 82.1 macro-F1 on the paper’s FaithBench setup.", "supports": "Name the exact dataset, unit of analysis, metric, model, prompt, and aggregation column when reporting benchmark numbers.", "does_not_support": "The result does not establish a general “grounding ceiling,” and the 71.2% average is not a FaithBench-specific score.", "source": "Tamber et al., “Benchmarking LLM Faithfulness in RAG with Evolving Leaderboards”", "url": "https://arxiv.org/abs/2505.04847v2", "version": "EMNLP 2025 Industry Track / arXiv v2", "source_type": "Peer-reviewed conference paper", "status": "Verified against v2 tables", "checked": "2026-07-22" }, { "id": "E13", "type": "empirical", "title": "NUTMEG’s downstream advantage is task-dependent", "claim": "On the paper’s politeness task, NUTMEG-trained models significantly outperformed traditional aggregation and disaggregated training for all tested subpopulation splits except race. On offensiveness, the authors report no significant differences among aggregation approaches or no aggregation.", "supports": "Model systematic disagreement rather than collapsing it by default, and verify whether the benefit holds for the actual task.", "does_not_support": "The paper does not show uniform significant superiority over both majority voting and disaggregated training across all tasks.", "source": "Ivey, Gauch & Jurgens, “NUTMEG: Separating Signal From Noise in Annotator Disagreement”", "url": "https://aclanthology.org/2025.emnlp-main.144/", "version": "EMNLP 2025, pp. 2874–2887", "source_type": "Peer-reviewed conference paper", "status": "Verified against Section 6.2", "checked": "2026-07-22" }, { "id": "E14", "type": "empirical", "title": "Rice’s theorem concerns universal semantic decision procedures", "claim": "Rice’s theorem establishes undecidability for every nontrivial semantic property of the partial functions computed by arbitrary programs; it rules out a universal deciding algorithm over that unrestricted class.", "supports": "Do not claim that one unrestricted analyzer can decide every nontrivial semantic property for every arbitrary program.", "does_not_support": "It does not show that no specific program can be proved correct relative to a formal specification and explicit assumptions.", "source": "H. G. Rice, “Classes of Recursively Enumerable Sets and Their Decision Problems”", "url": "https://www.jstor.org/stable/1990888", "version": "Transactions of the AMS 74(2), 1953", "source_type": "Peer-reviewed theorem paper", "status": "Scope checked", "checked": "2026-07-22" }, { "id": "E15", "type": "empirical", "title": "Specific implementation guarantees can be machine-checked", "claim": "The CompCert project provides a machine-checked semantic-preservation proof for a realistic C compiler within its specified languages, semantics, and assumptions.", "supports": "Use scoped language such as “proved relative to specification S and assumptions A” when a formal proof exists.", "does_not_support": "A formal proof does not automatically cover unspecified environments, hardware faults, requirements errors, or properties outside the theorem statement.", "source": "CompCert project, official description and publications", "url": "https://compcert.org/", "version": "Official project page, checked 2026-07-22", "source_type": "Primary project documentation with peer-reviewed publications", "status": "Verified", "checked": "2026-07-22" } ]