{ "batch": 1, "checked_at": "2026-07-15", "sources": [ { "url": "https://arxiv.org/abs/2410.12784", "retrieval": "pdf", "fetched": [ "https://arxiv.org/abs/2410.12784", "https://arxiv.org/pdf/2410.12784v2" ], "resolved_title": "JudgeBench: A Benchmark for Evaluating LLM-based Judges", "resolved_date": "arXiv v1 2024-10-16, v2 2025-04-05; Published as a conference paper at ICLR 2025", "title_match": true, "claims": [ { "id": "AJ-01", "verdict": "confirmed", "checked": [ "o1-preview Overall 75.43 (Table 2, Arena-Hard Judge with o1-preview backbone)", "o1-preview Math 85.71 and Coding 85.71 (Table 2) = 85.7% math/coding", "Vanilla (GPT-4o) Overall 50.86; Arena-Hard Judge (GPT-4o) Overall 56.57 (Table 1/2) = 50.9-56.6%", "Skywork-Reward-Gemma-2-27B Overall 64.29, highest reward model (Table 3)", "PandaLM Overall 13.14 (Table 1) = well below random", "ChatEval Overall 34.00 (Table 1)", "350 questions: 154 Knowledge, 98 Reasoning, 56 Mathematics, 42 Coding (p.6)", "p.8: GPT-4o 'achieving accuracy no better than random guessing when using the vanilla prompt'" ], "not_visible": [], "quote": "our dataset consists of a total of 350 questions: 154 in Knowledge, 98 in Reasoning, 56 in Mathematics, and 42 in Coding.", "notes": "All headline numbers in the claim match the tables exactly. Minor: the paper's reward-model overall range is stated as 'approximately 59% to 64%' (lowest = 59.43), so the evidence field's '60-64%' is a slight low-end rounding; does not affect the claim body. arXiv HTML full text 404'd for both v1/v2, so numbers were read from the PDF (allowed variant b)." } ] }, { "url": "https://arxiv.org/abs/2310.15683", "retrieval": "pdf", "fetched": [ "https://arxiv.org/abs/2310.15683", "https://arxiv.org/pdf/2310.15683" ], "resolved_title": "Prevalence and prevention of large language model use in crowd work", "resolved_date": "arXiv:2310.15683v1, 24 Oct 2023", "title_match": true, "claims": [ { "id": "AQ-01", "verdict": "partially_confirmed", "checked": [ "Study #1: n=168 workers on Prolific, 3 July 2023, summarizing abstracts", "Prevalence estimates 33.3% [25.9%,40.1%], 35.2% [29.8%,40.6%], 35.4% [27.8%,43.0%]", "Study #2: n=720 users, 23 July 2023, 3x3 factorial (request x hurdle)", "Table 1a baseline 27.6% dropping to 15.9% (direct request + image, preventing copy-pasting)", "'almost halved, dropping from 27.6% to 15.9%'", "abstract: mitigations 'significantly reduce, but not eliminate' LLM use", "directly requesting workers not to use LLMs decreased keyword retention by 6.2% (p=0.009)", "task = summarize medical paper abstracts (NEJM), per Materials 4.1 and Appendix A" ], "not_visible": [ "~34% Prolific self-report in a 2025 follow-up by Zhang et al. (external source, not this paper)", "June 2026 community survey arXiv 2606.04924 corroboration (external source)", "the word 'preregistered'" ], "quote": "almost halved, dropping from 27.6% to 15.9%", "notes": "The Veselovsky-attributable core and every headline number (33-35% baseline, 27.6%->15.9%, 6.2% keyword retention, n=168/n=720, medical abstracts, halve-not-eliminate) are fully confirmed by the body. Note the ABSTRACT rounds prevalence to 'around 30%', but Study #1 body reports 33.3-35.4%, so the claim's '33-35%' is supported by the body. Downgraded to partially_confirmed only because the claim additionally cites two external sources (Zhang et al. 2025; arXiv 2606.04924) not verifiable from this document. Also: source_date 2025-02-18 refers to the CACM publication; the fetched arXiv preprint is dated 2023-10-24." } ] }, { "url": "https://arxiv.org/abs/2407.00215", "retrieval": "fulltext", "fetched": [ "https://arxiv.org/abs/2407.00215", "https://arxiv.org/html/2407.00215v1" ], "resolved_title": "LLM Critics Help Catch LLM Bugs", "resolved_date": "28 Jun 2024 (v1)", "title_match": true, "claims": [ { "id": "CM-01", "verdict": "confirmed", "checked": [ "'model-written critiques are preferred over human critiques in 63% of cases' (naturally occurring LLM errors)", "'model critiques are preferred over human critiques more than 80% of the time' on Human Inserted Bugs, scale 'linear in Elo'", "'Human+CriticGPT teams move beyond the model-only Pareto frontier'", "FSBS selects critiques by 'rm_score + LENGTH_MODIFIER * num_highlights'; 'explored 4 values of LENGTH_MODIFIER' (inference-time dial)", "contractors median ~5 years Python experience, ~50 min per critique", "human-machine teams 'hallucinating less than LLMs alone'" ], "not_visible": [], "quote": "Human+CriticGPT teams move beyond the model-only Pareto frontier.", "notes": "63% preference, >80% Elo on inserted bugs, the FSBS length-penalty dial, and the beyond-the-frontier team result all confirmed from arXiv HTML full text. The critic model is named 'CriticGPT' in the body though the arXiv title is 'LLM Critics Help Catch LLM Bugs'." }, { "id": "CM-02", "verdict": "confirmed", "checked": [ "With critique: 'In 24% of cases contractors indicated that the critique found a problem that substantially decreased the rating'", "Without critique: completions first rated 'flawless' were 'rated similarly poorly by a second only 6% of the time'", "Run over ChatGPT training data first rated 'flawless'", "Held out-of-distribution ('out-of-distribution for the critic model'), mostly non-code" ], "not_visible": [], "quote": "In 24% of cases contractors indicated that the critique found a problem that substantially decreased the rating", "notes": "The 24% (critic-assisted) vs 6% (unassisted second rater) comparison on the 'flawless' slice is confirmed verbatim in Section 3.6 - a 4x lift as the claim states." } ] }, { "url": "https://aclanthology.org/2025.naacl-long.320/", "retrieval": "landing", "fetched": [ "https://aclanthology.org/2025.naacl-long.320/" ], "resolved_title": "Decomposition Dilemmas: Does Claim Decomposition Boost or Burden Fact-Checking Performance?", "resolved_date": "NAACL 2025", "title_match": true, "claims": [ { "id": "GF-03", "verdict": "partially_confirmed", "checked": [ "'Decompose-Then-Verify paradigm' present", "authors 'introduce a categorization of decomposition errors and reveal a trade-off between accuracy gains and the noise introduced'" ], "not_visible": [ "MiniCheck; WiCE; 80.01 vs 71.11 balanced accuracy", "AlignScore; FActScore; VeriScore; FELM", "F1 48.1->~68 and 71.6->54.3", "the specific 'helps weak verifiers / degrades strong ones' direction", "the four-way A/B/C/D decomposition-error taxonomy", "'best results when sub-claim count does not exceed input complexity'" ], "quote": "introduce a categorization of decomposition errors and reveal a trade-off between accuracy gains and the noise introduced", "notes": "Fetch constraints allow only the given ACL landing page (no PDF or other same-site page, and the URL contains no arXiv ID). The abstract confirms the general trade-off framing and that decomposition errors are categorized, but none of the specific verifiers, datasets, or numbers appear, and the abstract does not explicitly state the weak-vs-strong-verifier direction. Everything specific in the claim is not visible in the retrieved text." } ] }, { "url": "https://arxiv.org/html/2605.16354v1", "retrieval": "fulltext", "fetched": [ "https://arxiv.org/html/2605.16354v1" ], "resolved_title": "Augmenting Human Evaluation with LLM Judges: How Many Human Reviews Do You Need?", "resolved_date": "arXiv 2605.16354 (May 2026, per arXiv ID; source_date 2026-05-08 consistent; exact day not surfaced in fetched HTML)", "title_match": true, "claims": [ { "id": "HS-03", "verdict": "confirmed", "checked": [ "floor formula n*(1-rho^2): 'converges to a floor of n*(1-rho2)'", "worked example: n*=200, R2=0.70, N=2000 -> 'only 65 human samples are needed'", "N=400 -> 100: 'If the budget can allow for 100 human ratings, then the LLM sample size N can be reduced to 400'", "doubly robust two-stage sampling design (missingness known by design)", "stratification 'reduces the human budget by up to 12.9%' (claim's ~13%)", "caveats: pilot overestimation of R2; human gold standard a 'pragmatic choice rather than a settled fact'; single-rater scope", "author Jane Paik Kim, Stanford Dept. of Psychiatry" ], "not_visible": [], "quote": "With an LLM sample size of N=2000, only 65 human samples are needed.", "notes": "All specifics verified from the given HTML full text. Claim's '~13%' stratified savings equals the paper's precise 12.9%. Formula, both worked examples (65 and 100 human labels), diminishing returns, and all three caveats match." } ] }, { "url": "https://arxiv.org/html/2603.01562v1", "retrieval": "fulltext", "fetched": [ "https://arxiv.org/html/2603.01562v1" ], "resolved_title": "RubricBench: Aligning Model-Generated Rubrics with Human Standards", "resolved_date": "arXiv 2603.01562 (March 2026, per arXiv ID; source_date 2026-03-02 consistent)", "title_match": true, "claims": [ { "id": "RR-04", "verdict": "confirmed", "checked": [ "gap 'remains stable at ~26%' / 'a severe 27% accuracy gap' (Table 3 Delta range +22.1 to +28.3)", "DeepSeek-v3.2: Self-Gen 57.8 vs Human 84.9 (Delta +27.1)", "Gemini-3-Flash: Self-Gen 58.0 vs Human 85.3 (Delta +27.3)", "1,147 adversarial pairwise comparisons", "unnecessary rules N=1 17.9% vs 10.1%; overly rigid R=5 12.8% vs 7.7%", "checklists ~13.2/15.4/14.6 items; 'Attention Displacement'", "'even with human rubrics, accuracy plateaus around 85%'", "safety self-generated ~25-30% vs human >90%", "test-time compute does NOT close gap (GPT-4o-mini 48.0%->46.8%; refinement 46.7->46.4->45.7)" ], "not_visible": [], "quote": "even with human rubrics, accuracy plateaus around 85% rather than approaching 100%", "notes": "Every specific number in the claim matches the full-text tables. The claim's '~26-27 point... across all judge backbones' aligns with the paper's own ~26%/27% summary, though per-backbone Delta ranges +22.1 to +28.3. This is stronger than the batch's in_corpus 'partially_confirmed' - full text confirms all specifics, so verdict is confirmed." } ] }, { "url": "https://futurism.com/scale-ai-zuckerberg-incompetence", "retrieval": "fulltext", "fetched": [ "https://futurism.com/scale-ai-zuckerberg-incompetence" ], "resolved_title": "(headline not verbatim-captured; article content is Scale AI / Meta $14B investment / Bulba Experts, consistent with source_title)", "resolved_date": "not surfaced on page; source_date 2025-06 (internal documents cover 2023-03 to 2024-04)", "title_match": true, "claims": [ { "id": "IP-04", "verdict": "partially_confirmed", "checked": [ "'Bulba Experts' program to train Google's AI; platforms 'Remotasks and Outlier'", "'chaotic 11 months between March 2023 and April 2024'", "spam = 'writing gibberish, writing incorrect information, GPT-generated thought processes'", "spammers 'could get away with just totally submitting garbage and there weren't enough people to track them down'", "Allocations team 'dumped 800 spammers'", "supervisors told to use 'a tool called ZeroGPT'", "memos to remove/deny contributors from 'Egypt, Pakistan, Kenya, and Venezuela'", "caught spammers 'would come back by simply using a VPN'", "'There were no background checks whatsoever'" ], "not_visible": [ "a black market selling verified annotator accounts (Outlier, CrowdGen, Prolific) - no mention of account resale, CrowdGen, or Prolific in this article; the evidence attributes this to AlgorithmWatch (a separate source)", "community reports (Indeed/Glassdoor/Trustpilot) and the 33-46% MTurk figure (external)" ], "quote": "could get away with just totally submitting garbage and there weren't enough people to track them down", "notes": "Every Scale-AI/Futurism-attributable specific is confirmed. Only the black-market-accounts assertion (which the evidence itself attributes to AlgorithmWatch, not Futurism) is absent from this source, hence partially_confirmed." }, { "id": "CT-10", "verdict": "partially_confirmed", "checked": [ "'Bulba Experts' program for Google, March 2023 to April 2024", "spam = 'writing gibberish, writing incorrect information, GPT-generated thought processes'", "paid anyway - 'could get away with just totally submitting garbage and there weren't enough people to track them down'", "banned workers returned via VPN ('would come back by simply using a VPN')", "'There were no background checks whatsoever'" ], "not_visible": [ "'academic estimates put LLM use at 33-46% of crowdworkers on text tasks' (external - Veselovsky et al. 2306.07899, not in this article)", "NeurIPS 2025 'Evaluating LLM-Contaminated Crowdsourcing Data Without Ground Truth' (external)" ], "quote": "would come back by simply using a VPN", "notes": "The Futurism-attributable spam-economics narrative is fully confirmed. The 33-46% academic prevalence figure and the NeurIPS 2025 peer-prediction paper are separate cited sources not present in this article, hence partially_confirmed." } ] }, { "url": "https://arxiv.org/abs/2509.20293", "retrieval": "abstract", "fetched": [ "https://arxiv.org/abs/2509.20293" ], "resolved_title": "When Judgment Becomes Noise: How Design Failures in LLM Judge Benchmarks Silently Undermine Validity", "resolved_date": "24 Sep 2025 (v1); revised 8 Oct 2025 (v3)", "title_match": true, "claims": [ { "id": "CT-06", "verdict": "confirmed", "checked": [ "'Schematic adherence quantifies how much of a judge's overall verdict is explained by the explicit evaluation schema'", "'unexplained variance exceeding 90 percent for DeepSeek-R1-32B'", "'factor correlations above 0.93 for most criteria'", "'severe schema incoherence and factor collapse across popular judges'", "'Arena-Hard Auto'", "'ELO-style aggregation... collapses and masks genuine ranking uncertainty'", "code at github.com/penfever/judgment-to-noise" ], "not_visible": [ "the exact word 'halo' - paper says 'factor collapse', the claim's 'single halo factor' is an interpretive gloss", "the 'under 10% of variance' phrasing - paper states it as '>90% unexplained' (equivalent)" ], "quote": "factor correlations above 0.93 for most criteria", "notes": "All headline numbers match: <10% schema-explained variance = >90% unexplained for DeepSeek-R1-32B; factor correlations >0.93; factor collapse; ELO masking ranking uncertainty; repo confirmed. Verified from the abstract page (given URL); full text not needed as the abstract covers every asserted figure." } ] }, { "url": "https://doi.org/10.1609/aaai.v40i44.41123", "retrieval": "landing", "fetched": [ "https://doi.org/10.1609/aaai.v40i44.41123", "https://ojs.aaai.org/index.php/AAAI/article/view/41123" ], "resolved_title": "FindTheFlaws: Annotated Errors for Detecting Flawed Reasoning and Scalable Oversight Research", "resolved_date": "2026-03-14; Proceedings of the AAAI Conference on Artificial Intelligence, 40(44)", "title_match": true, "claims": [ { "id": "F26-10", "verdict": "confirmed", "checked": [ "'we present FindTheFlaws, a group of five diverse datasets spanning medicine, mathematics, science, coding, and the Lojban language'", "'(1) long-form expert-verified correct solutions and (2) long-form flawed solutions with annotations highlighting specific errors'", "scalable oversight with weaker evaluator: 'models performing more poorly on particular datasets can serve as judges/verifiers for more capable models'", "author Gabriel Recchia (Recchia, G., Mangat, C.S., Li, I., Krishnakumar, G.)", "published 2026-03-14, AAAI 40(44)" ], "not_visible": [], "quote": "we present FindTheFlaws, a group of five diverse datasets spanning medicine, mathematics, science, coding, and the Lojban language", "notes": "DOI resolves (302) to the AAAI OJS landing page (allowed redirect variant a). Claim says 'five datasets... across medicine, math, and science' - accurate, though the full set also includes coding and Lojban. The weaker-evaluator scalable-oversight design and expert-annotated flawed/correct solutions are confirmed." } ] }, { "url": "https://www.iso.org/publication/PUB200525.html", "retrieval": "landing", "fetched": [ "https://www.iso.org/publication/PUB200525.html" ], "resolved_title": "ISO - AI data quality management bundle", "resolved_date": "2025 (bundle; component parts dated 2024 and 2025)", "title_match": true, "claims": [ { "id": "CG-12", "verdict": "confirmed", "checked": [ "'This bundle includes five essential components'", "ISO/IEC 5259-1:2024 - 'Part 1: Overview, terminology, and examples'", "5259-2:2024 - 'data quality model, measures, and guidance on reporting data quality'", "5259-3:2024 - 'management requirements and guidelines'", "5259-4:2024 - 'process framework'", "5259-5:2025 - 'governance framework to enable organizations to direct and oversee data quality'", "page title 'AI data quality management bundle'" ], "not_visible": [ "internal ISO catalog IDs 81088/81860/81092/81093/84150 (from evidence, not shown on this page)", "Part 4 'covering data labeling' specifically (page shows 'process framework... guidance on data quality processes' but does not surface 'data labeling')" ], "quote": "This bundle includes five essential components: ISO/IEC 5259-1:2024 - Artificial intelligence — Data quality for analytics and machine learning (ML) — Part 1: Overview, terminology, and examples", "notes": "Retrieved via curl with browser user-agent (WebFetch returned 403). Five-part series with Parts 1-4 (2024) and Part 5 (2025), each part's subject, and the 'AI data quality management bundle' packaging all confirmed. Full standard text is paywalled; the claim's own contract-mandate observation is explicitly hedged." } ] }, { "url": "https://www.sciencedirect.com/science/article/abs/pii/S1048984321000394", "retrieval": "failed", "fetched": [ "https://www.sciencedirect.com/science/article/abs/pii/S1048984321000394" ], "resolved_title": null, "resolved_date": null, "title_match": false, "claims": [ { "id": "CG-24", "verdict": "not_retrievable", "checked": [], "not_visible": [ "feedback quality and performance in organisations", "relative rank feedback raising output (Charness et al. 2014; Tafkov 2013)", "sabotage/cheating offsetting positive effects", "Barankay 2012 - removing rank feedback improving performance" ], "quote": "", "notes": "WebFetch returned HTTP 403 Forbidden. curl -sL with a browser user-agent returned a Cloudflare bot-challenge shell (title 'ScienceDirect'; contains __cf_chl_tk tokens, 'Captcha', 'Enable JavaScript', meta http-equiv refresh 360, noscript) with zero article content - no abstract, authors, journal, or any mention of rank feedback / Charness / Barankay / Tafkov / Leadership Quarterly. Claim could not be verified against the source per the WebFetch->curl->not_retrievable fallback chain." } ] } ] }