{ "batch": 2, "checked_at": "2026-07-15", "sources": [ { "url": "https://neurips.cc/virtual/2025/poster/117308", "retrieval": "abstract", "fetched": ["https://neurips.cc/virtual/2025/poster/117308"], "resolved_title": "Validating LLM-as-a-Judge Systems under Rating Indeterminacy", "resolved_date": "NeurIPS 2025 (no specific day shown on page)", "title_match": true, "claims": [ { "id": "AJ-02", "verdict": "confirmed", "checked": [ "11 real-world rating tasks and 9 commercial LLMs", "forced-choice validation selects judge systems performing as much as 31% worse than the paper's response-set approach", "rating indeterminacy defined as criteria admitting multiple valid interpretations", "forced-choice differences 'can heavily bias LLM-as-a-judge validation'", "solution uses multi-label 'response set' ratings" ], "not_visible": ["exact source date 2025-09-19 (page shows only 'NeurIPS 2025 Poster')"], "quote": "The experiments involve \"11 real-world rating tasks and 9 commercial LLMs\" ... forced-choice validation selects judge systems \"performing as much as 31% worse than judge systems selected by our approach.\"", "notes": "Core assertion and both headline numbers (11 tasks, 9 LLMs, up-to-31% gap) match. Claim's '31% worse than multi-label response-set ratings' is consistent with retrieved '31% worse than judge systems selected by our approach' (their approach = multi-label response set). Authors match: Guerdan, Barocas, Holstein, Wallach, Wu, Chouldechova." } ] }, { "url": "https://www.pnas.org/doi/10.1073/pnas.2518075122", "retrieval": "failed", "fetched": ["https://www.pnas.org/doi/10.1073/pnas.2518075122"], "resolved_title": null, "resolved_date": null, "title_match": false, "claims": [ { "id": "AQ-02", "verdict": "not_retrievable", "checked": [], "not_visible": [ "99.8% pass rate on attention/quality checks", "6,000 trials / 20 check types", "43,800 evaluations across 9 LLMs", "97.7% reverse-shibboleth refusal", "$0.05 per survey vs $1.50 paid", "'there is no magical fix' and provenance-based defenses" ], "quote": "", "notes": "Page is protected by a Cloudflare JS challenge ('Just a moment...'). WebFetch returned HTTP 403 Forbidden; curl -sL with two different browser user-agents both returned the Cloudflare interstitial (HTTP 403, no redirect). Constraints forbid substitute sources or other pages on pnas.org, so none of the claimed figures could be verified." } ] }, { "url": "https://arxiv.org/abs/2510.26518", "retrieval": "pdf", "fetched": ["https://arxiv.org/abs/2510.26518", "https://arxiv.org/pdf/2510.26518"], "resolved_title": "Human-AI Complementarity: A Goal for Amplified Oversight", "resolved_date": "v1 2025-10-30; PDF served is v2 (2510.26518v2, 2026-06-25)", "title_match": true, "claims": [ { "id": "CM-03", "verdict": "confirmed", "checked": [ "AI rater 87.7% vs human raters 75.1%", "human majority vote 80.6% (vs 75.1%)", "N = 1918 Evaluation Set examples", "unassisted hybridization 89.3% (higher than AI-alone 87.7%)", "confidence threshold T = 0.62", "on low-confidence slice AI 60.5% vs human 71.3%", "evidence-assisted hybridization achieves 91.3%", "evidence-only helps on items AI gets correct: 79.3% vs 71.3%", "over-reliance coefficient -0.768, SE=0.174, z=-4.413, p<.001" ], "not_visible": [], "quote": "rater achieves 87.7% accuracy, performing above human raters who achieved 75.1% accuracy ... confidence threshold of 0.62, results in an accuracy of 91.3%, compared to 89.3% using unassisted ... −0.768, SE = 0.174, z = −4.413, p < .001", "notes": "Every headline number in the claim verified verbatim from the PDF full text via pdftotext. AI-alone 87.7%, human-alone 75.1%, majority 80.6%, N=1918, threshold 0.62, unassisted hybrid 89.3%, evidence-assisted 91.3%, low-conf slice 60.5% AI / 71.3% human, evidence-only 79.3% vs 71.3%, beta=-0.768 p<.001 all present. Note: PDF served at /pdf/ is v2 (June 2026); source_date is v1 (Oct 30 2025). All cited numbers match the retrieved v2 text. Authors: Jain, Bridgers, Janzer, Greig, Teh, Mikulik." } ] }, { "url": "https://aclanthology.org/2025.emnlp-main.1205/", "retrieval": "abstract", "fetched": ["https://aclanthology.org/2025.emnlp-main.1205/"], "resolved_title": "DnDScore: Decontextualization and Decomposition for Factuality Verification in Long-Form Text Generation", "resolved_date": "EMNLP 2025 (November 2025), pp. 23609-23626", "title_match": true, "claims": [ { "id": "GF-04", "verdict": "confirmed", "checked": [ "decomposition and decontextualization have conflicting purposes", "decomposition isolates atomic facts while decontextualization inserts relevant information", "adding context back creates multi-fact text ('what part of the augmented text should be verified')", "DnDScore validates subclaims in the context of contextual information (context, not content)", "strategy choice materially changes factuality scores", "pages 23609-23626; authors Wanner, Van Durme, Dredze" ], "not_visible": [], "quote": "decomposition isolates atomic facts while decontextualization inserts relevant information ... they present \"DnDScore, a decontextualization aware verification method that validates subclaims in the context of contextual information\" ... \"the choice of strategy matters in the resulting factuality scores.\"", "notes": "Core tension and the resolution (verify subclaim WITH added info as context rather than content) confirmed verbatim from the ACL Anthology abstract. Title, authors, pages all match." } ] }, { "url": "https://arxiv.org/abs/2408.15204", "retrieval": "abstract", "fetched": ["https://arxiv.org/abs/2408.15204"], "resolved_title": "Can Unconfident LLM Annotations Be Used for Confident Conclusions?", "resolved_date": "v1 2024-08-27; v2 2025-02-08 (NAACL 2025)", "title_match": true, "claims": [ { "id": "HS-04", "verdict": "partially_confirmed", "checked": [ "Confidence-Driven Inference combines LLM annotations and LLM confidence to pick which items get human annotation", "reduces needed human annotations by over 25% in each of three settings (politeness, stance, bias)", "safeguards against poor-quality LLM annotations; conclusions valid and no less accurate than human-only", "provably valid confidence intervals" ], "not_visible": [ "'Active Statistical Inference' foundation (attributed to separate paper arXiv:2403.03208, not on this page)", "2026 ICLR counter-finding (arXiv:2604.18569) that near-uniform sampling beats uncertainty-driven querying" ], "quote": "The paper introduces \"Confidence-Driven Inference: a method that combines LLM annotations and LLM confidence indicators\" ... it reduces \"the needed number of human annotations by over 25% in each\" setting.", "notes": "The core claim attributable to this URL (>25% reduction in all three settings, validity regardless of LLM quality) is fully confirmed verbatim. Marked partially_confirmed because two asserted specifics are external to this source by the claim's own admission (Active Statistical Inference foundation = arXiv:2403.03208; the ICLR-2026 counter-finding = arXiv:2604.18569) and neither is visible on the fetched abstract. Authors: Gligoric, Zrnic, Lee, Candes, Jurafsky." } ] }, { "url": "https://arxiv.org/abs/2404.12272", "retrieval": "abstract", "fetched": ["https://arxiv.org/abs/2404.12272"], "resolved_title": "Who Validates the Validators? Aligning LLM-Assisted Evaluation of LLM Outputs with Human Preferences", "resolved_date": "v1 2024-04-18 (arXiv page shows no venue; source_date 2024-10-13 not confirmed on page)", "title_match": true, "claims": [ { "id": "RR-05", "verdict": "partially_confirmed", "checked": [ "criteria drift established as a named phenomenon", "'users need criteria to grade outputs, but grading outputs helps users define criteria'", "some criteria are dependent on the specific LLM outputs observed", "alignment is subjective and iterative; EvalGen generates candidate assertions/judge prompts" ], "not_visible": ["'UIST 2024' venue attribution (arXiv page lists no conference/journal venue)"], "quote": "\"users need criteria to grade outputs, but grading outputs helps users define criteria\" ... \"some criteria appears dependent on the specific LLM outputs observed\" ... the study \"underscores the subjectivity and iterative process of alignment\".", "notes": "Criteria-drift core confirmed verbatim. Only the venue label 'UIST 2024' is not visible on the arXiv abstract page (Comments: 16 pages, 4 figures, 2 tables; no venue listed)." }, { "id": "CT-04", "verdict": "partially_confirmed", "checked": [ "criteria drift: grading outputs changes graders' criteria", "'users need criteria to grade outputs, but grading outputs helps users define criteria'", "criteria appear output-dependent, not definable in advance", "authors Shankar, Zamfirescu-Pereira, Hartmann, Parameswaran, Arawjo" ], "not_visible": [ "'UIST 2024' venue attribution (not on arXiv page)", "graders 'retroactively revise earlier grades' (specific mechanism not stated in the fetched abstract)" ], "quote": "\"we identify a phenomenon we dub criteria drift: users need criteria to grade outputs, but grading outputs helps users define criteria\"; \"some criteria appears dependent on the specific LLM outputs observed\".", "notes": "Core (output-dependent, unstable criteria; criteria drift) confirmed verbatim. Two specifics not visible in the retrieved abstract: the 'UIST 2024' venue and the explicit 'retroactively revise earlier grades' behavior (may appear in the paper body, not the abstract)." } ] }, { "url": "[link withheld]", "retrieval": "landing", "fetched": ["[link withheld]"], "resolved_title": "Report on an AI data vendor's acquisition of a data-quality startup", "resolved_date": "2026-01-28 (11:00 AM PST), by Marina Temkin", "title_match": true, "claims": [ { "id": "IP-05", "verdict": "partially_confirmed", "checked": [ "An AI data vendor acquired a data-quality startup (announced 2026-01-28)", "The startup's algorithms 'flag incorrect data without a second human reviewer'", "competing acquisition interest from other AI data-labeling companies", "Nine key employees of the startup join the vendor's research team", "Three co-founders earned PhDs from MIT", "An AI data vendor provided data for eight top AI labs, including OpenAI", "$300 million annualized revenue run-rate (end of 2025)" ], "not_visible": [ "the phrases 'confident learning', 'data-centric AI', and 'LLM evaluation' (not used in the TechCrunch article; claim attributes them to the startup's CEO letter / the vendor's announcement)" ], "quote": "\"The startup's researchers are experts in developing algorithms that flag incorrect data without a second human reviewer.\" ... \"The company has provided data for eight top AI labs, including OpenAI.\"", "notes": "Core assertion and every headline number/fact drawn from TechCrunch confirmed verbatim, including the key 'flag incorrect data without a second human reviewer' quote, competing bids, 9 staff, 3 MIT co-founders, 8 labs, $300M ARR. Only the capability-name phrases in the claim's evidence ('confident learning', 'data-centric AI', 'LLM evaluation') are absent from the article; the claim itself attributes those to other documents." } ] }, { "url": "https://aclanthology.org/2025.findings-emnlp.1361.pdf", "retrieval": "pdf", "fetched": ["https://aclanthology.org/2025.findings-emnlp.1361.pdf"], "resolved_title": "Rating Roulette: Self-Inconsistency in LLM-As-A-Judge Frameworks", "resolved_date": "Findings of EMNLP 2025, pages 24986-25004", "title_match": true, "claims": [ { "id": "CT-07", "verdict": "confirmed", "checked": [ "LLM judges have low intra-rater reliability across runs; ratings 'almost arbitrary in the worst case'", "disabling sampling (deterministic) degrades performance/agreement with human judgment (trade-off)", "phenomenon spans SummaC, SummEval, and MT-Bench", "SummEval kappa 0.492 (crowd) and 0.413 (expert first round)", "0.7127 after second round of expert annotation", "pages 24986-25004; authors Haldar & Hockenmaier" ], "not_visible": [], "quote": "\"LLM judges have low intra-rater reliability in their assigned scores across different runs. This variance makes their ratings inconsistent, almost arbitrary in the worst case\" ... kappa \"0.492 and 0.413 for the crowd-sourced workers and the first round of expert annotations ... second round ... 0.7127\".", "notes": "Fully confirmed verbatim from the PDF via pdftotext. Intra-rater-reliability core, the 'almost arbitrary in the worst case' phrase, the no-sampling degradation ('degradation in performance if run without sampling ... trade-off between self-reliability and performance'), all three benchmarks, and all three kappa figures with the exact crowd/expert-first/second-round mapping match. Page range 24986-25004 confirmed." } ] }, { "url": "https://www.langchain.com/resources/llm-as-a-judge", "retrieval": "landing", "fetched": ["https://www.langchain.com/resources/llm-as-a-judge"], "resolved_title": "How to Calibrate LLM-as-Judge with Human Corrections", "resolved_date": "2026-03-10", "title_match": true, "claims": [ { "id": "F26-11", "verdict": "partially_confirmed", "checked": [ "closed calibration loop: humans review a sample of judge verdicts and correct disagreements", "corrections become ground truth for measuring/improving evaluator alignment", "corrections recycled as few-shot examples into the judge prompt", "LLM-judge-first pattern (judges produce initial scores; humans correct disagreements)", "dated March 10, 2026" ], "not_visible": [ "'5-10%' sampled human verification rate (claim attributes this to FutureAGI's 2026 guide, not LangChain)", "periodic/monthly kappa re-checks (claim attributes this to FutureAGI; LangChain page mentions no kappa)" ], "quote": "\"human reviewers examine a sample of those judgments and correct any they disagree with\" ... \"These corrections become the ground truth for measuring and improving evaluator alignment.\" ... \"From those corrections, you build few-shot examples that calibrate the judge.\"", "notes": "The LangChain-attributable core (correction-collection -> alignment-measurement -> few-shot-rebuild loop, dated 2026-03-10) confirmed verbatim. The '5-10% human verification' default and 'monthly kappa re-sampling' are attributed by the claim to FutureAGI's guide (a different source) and are not present on the LangChain page." } ] }, { "url": "https://www.theverge.com/2019/2/25/18229714/cognizant-facebook-content-moderator-interviews-trauma-working-conditions-arizona", "retrieval": "fulltext", "fetched": ["https://www.theverge.com/2019/2/25/18229714/cognizant-facebook-content-moderator-interviews-trauma-working-conditions-arizona"], "resolved_title": "The secret lives of Facebook moderators in America (on-page headline: 'The Trauma Floor')", "resolved_date": "2019-02-25 (matches URL path; byline Casey Newton)", "title_match": true, "claims": [ { "id": "CG-13", "verdict": "partially_confirmed", "checked": [ "three-layer audit: moderator -> QA re-review -> full-time Facebook employees audit a subset of QA decisions", "~50-60 of ~1,500 weekly decisions randomly audited (~3-4%)", "QA worker makes $1 per hour more", "accuracy = agreement with auditor; target 95 percent", "actual scores in high 80s / low 90s, ~92 at press time", "misses lead to coaching/remedial program, often a pretext for managing workers out (termination)", "moderators prohibited from lobbying QAs to reverse decisions but do so regularly" ], "not_visible": [ "'other sites reported 98%' (string '98' appears 0 times in the article)", "the descriptor 'blind' (word 'blind' appears 0 times; audit structure described but not called blind)" ], "quote": "\"From Miguel's 1,500 or so weekly decisions, Facebook will randomly select 50 or 60 to audit ... a quality assurance worker, known internally as a QA, who also makes $1 per hour more ... Full-time Facebook employees then audit a subset of QA decisions.\" ... target \"95 percent ... floats in the high 80s or low 90s ... around 92\".", "notes": "Every material fact confirmed verbatim, including the '~1,500 weekly / 50-60 audited', '$1 per hour more' QA, Facebook-audits-a-subset-of-QA third layer, 95% target, high-80s-to-92 actuals, the 'Accuracy is only judged by agreement / obvious sale of heroin / This number is fake' quotes, off-book QA lobbying, and coaching-as-pretext-for-termination. Marked partially_confirmed only because the peripheral '98% at other sites' comparison is absent (no '98' anywhere) and the audit is not literally called 'blind'." } ] } ] }