{ "batch": 9, "checked_at": "2026-07-15", "sources": [ { "url": "https://arxiv.org/abs/2505.10320", "retrieval": "abstract", "fetched": ["https://arxiv.org/abs/2505.10320"], "resolved_title": "J1: Incentivizing Thinking in LLM-as-a-Judge via Reinforcement Learning", "resolved_date": "2025-05-15 (v1); v3 2025-10-13", "title_match": true, "claims": [ { "id": "AJ-09", "verdict": "partially_confirmed", "checked": [ "J1-Qwen-32B outperforms o1-mini, o3, and a much larger 671B DeepSeek-R1 on some benchmarks", "RL framework teaching LLM judges to think before deciding (thinking-judge)", "unified format with verifiable rewards; reduces positional bias", "trained at scales of 8B, 32B, and 70B", "trained only on synthetic data; v1 submitted May 15 2025" ], "not_visible": [ "'22K synthetic pairs' (abstract says only 'synthetic data', no count)", "'test-time self-consistency' (abstract mentions iterative self-correction, not self-consistency)", "attribution to 'Meta' (authors are Meta FAIR but abstract does not state Meta)", "CompassJudger-2-7B claim and 80.9% JudgerBenchV2 (cites arXiv 2507.09104, a different URL not fetched)", "Skywork-Reward-V2-Llama-3.1-8B, 26M pairs, topping seven benchmarks incl JudgeBench (cites arXiv 2507.01352, a different URL not fetched)" ], "quote": "J1-Qwen-32B, our multitasked pointwise and pairwise judge also outperforms o1-mini, o3, and a much larger 671B DeepSeek-R1", "notes": "The J1-specific core assertion (32B beats o1-mini/o3/671B DeepSeek-R1, RL thinking-judge, verifiable rewards, position-bias mitigation) is confirmed. The claim bundles two other papers (CompassJudger-2, Skywork-Reward-V2) with their own arXiv IDs that the hard constraints forbid fetching; those portions are not verifiable from this source." } ] }, { "url": "https://arxiv.org/pdf/2510.08460", "retrieval": "abstract", "fetched": ["https://arxiv.org/pdf/2510.08460", "https://arxiv.org/abs/2510.08460"], "resolved_title": "LeWiDi-2025 at NLPerspectives: Third Edition of the Learning with Disagreements Shared Task", "resolved_date": "2025-10-09 (v1); v2 2026-02-06", "title_match": true, "claims": [ { "id": "AQ-09", "verdict": "partially_confirmed", "checked": [ "Third edition of the Learning With Disagreements (LEWIDI) shared task", "NLPerspectives workshop at EMNLP 2025", "soft-label approach: models predict population-level distributions of judgments", "perspectivist approach: models predict interpretations of individual annotators", "four datasets (paraphrase, irony, sarcasm, NLI)", "went beyond standard metrics such as cross-entropy with new metrics for both paradigms" ], "not_visible": [ "'Wasserstein/Manhattan distance' as the soft-label/perspectivist metrics (not named in abstract)", "'public leaderboard' (not in abstract; a Codabench competition link appears in the PDF structure but term not confirmed)", "'per-annotator labels rather than adjudicated gold' (implied by perspectivist paradigm, not stated explicitly)", "lineage claims: DICES (NeurIPS 2023), jury learning (Gordon et al. CHI 2022), Frenda et al. survey (LREV 2024) - cite other works not fetchable here; Aroyo & Welty citation anchor was present in the PDF" ], "quote": "the soft-label approach, in which models predict population-level distributions of judgments ... the perspectivist approach, in which models predict the interpretations of individual annotators", "notes": "Given PDF was FlateDecode-compressed (no readable abstract); fetched the same-arXiv-ID abs page (allowed variant) for the abstract. Core dual-evaluation design confirmed; specific distance metrics and leaderboard not visible." } ] }, { "url": "https://arxiv.org/abs/2206.05802", "retrieval": "abstract", "fetched": ["https://arxiv.org/abs/2206.05802"], "resolved_title": "Self-critiquing models for assisting human evaluators", "resolved_date": "2022-06-12 (v1); v2 2022-06-14", "title_match": true, "claims": [ { "id": "CM-11", "verdict": "confirmed", "checked": [ "model-written critiques help humans find flaws in summaries they would otherwise have missed", "surface intentional flaws in summaries humans wrote to be deliberately misleading", "framework comparing generation, discrimination, and critique ability", "'even large models may still have relevant knowledge they cannot or do not articulate as critiques' (supports discriminator-critique gap)", "larger models write more helpful critiques and self-refine better (topic-based summarization)" ], "not_visible": [], "quote": "even large models may still have relevant knowledge they cannot or do not articulate as critiques", "notes": "All elements of the claim and evidence confirmed from the abstract, including the verbatim quote. The 'discriminator-critique gap' is not named by that phrase but is the framework's finding." } ] }, { "url": "https://aclanthology.org/2026.findings-acl.689/", "retrieval": "landing", "fetched": ["https://aclanthology.org/2026.findings-acl.689/"], "resolved_title": "FaithLens: Detecting and Explaining Faithfulness Hallucination", "resolved_date": "2026-07 (Findings of ACL 2026, San Diego), pp. 14068-14099", "title_match": true, "claims": [ { "id": "GF-11", "verdict": "confirmed", "checked": [ "8B faithfulness-hallucination detection model", "jointly outputs binary predictions plus explanations", "fine-tuned on filtered LLM-synthesized data as a cold start", "further optimized with rule-based reinforcement learning rewarding prediction correctness and explanation quality", "beats GPT-5.2 and o3 across 12 diverse tasks", "pages 14068-14099, Findings of ACL 2026, Si et al." ], "not_visible": ["per-task numeric scores (require full PDF; abstract only, consistent with the claim's own caveat)"], "quote": "a distinctive balance of trustworthiness, efficiency, and effectiveness", "notes": "Every headline assertion in the claim is present in the ACL Anthology abstract; page range and authorship match the evidence." } ] }, { "url": "https://arxiv.org/abs/2604.18569", "retrieval": "abstract", "fetched": ["https://arxiv.org/abs/2604.18569"], "resolved_title": "Revisiting Active Sequential Prediction-Powered Mean Estimation", "resolved_date": "2026-04-20", "title_match": true, "claims": [ { "id": "HS-11", "verdict": "confirmed", "checked": [ "smallest confidence width occurs when the weight on the constant probability is close to one (uncertainty-driven part diminishes)", "non-asymptotic analysis yielding a data-dependent CI bound", "with a no-regret approach the query probability converges to the maximum-value constraint when set obliviously to covariates", "Published as a conference paper at ICLR 2026", "submitted 20 Apr 2026 (Sfyraki & Wang)" ], "not_visible": [], "quote": "the smallest confidence width tends to occur when the weight on the constant probability is close to one", "notes": "Core claim confirmed. The claim's 'near-constant at the budget ceiling' maps to the paper's 'maximum-value constraint' the oblivious query probability converges to; consistent phrasing." } ] }, { "url": "https://www.langchain.com/blog/introducing-align-evals", "retrieval": "landing", "fetched": ["https://www.langchain.com/blog/introducing-align-evals"], "resolved_title": "Introducing Align Evals: Streamlining LLM Application Evaluation", "resolved_date": "2025-07-29", "title_match": true, "claims": [ { "id": "RR-12", "verdict": "confirmed", "checked": [ "Align Evals is a LangSmith feature (launched Jul 29 2025)", "four-step workflow: select criteria; select representative good+bad examples; human-grade expected scores (golden set); iterate evaluator prompt against alignment score", "'alignment score' measures evaluator vs human match; unaligned cases surfaced by sorting", "saved baseline alignment score to compare new prompt versions", "inspired by Eugene Yan's AlignEval", "roadmap: analytics for tracking over time; automatic prompt optimization" ], "not_visible": [], "quote": "Our evaluation scores don't match what we'd expect a human on our team to say", "notes": "Every element of the claim and evidence confirmed from the launch blog." } ] }, { "url": "https://kili-technology.com/blog/data-annotation-guide-how-to-achieve-high-quality-data-in-complex-ai-data-operations", "retrieval": "landing", "fetched": ["https://kili-technology.com/blog/data-annotation-guide-how-to-achieve-high-quality-data-in-complex-ai-data-operations"], "resolved_title": "Data Annotation Guide: How to Achieve High Quality Data in Complex AI Data Operations (Kili Technology)", "resolved_date": "2026-06-24 (page-stated)", "title_match": false, "claims": [ { "id": "IP-12", "verdict": "partially_confirmed", "checked": [ "Kili guide attributes low agreement to guideline defects: 'An annotator who consistently disagrees with others on a specific label category is often revealing a guideline gap'", "RLHF preference-annotation noise 'commonly exceeds 20% in real datasets'", "guidelines as living documents, versioned; calibration rounds; annotators needing training or reassignment", "general thesis (low inter-annotator agreement diagnosed to instructions/rubric, not annotator failure) supported" ], "not_visible": [ "The entire 'an annotation platform's quality playbook' - an annotation platform does not appear anywhere on the Kili page (separate URL not in allowed fetch set)", "'poor instructions, weak edge-case coverage' as an annotation platform's low-precision starting point", "An annotation platform scenario: high precision + low accuracy -> refine the ground truth itself + check for systematic bias in training materials" ], "quote": "An annotator who consistently disagrees with others on a specific label category is often revealing a guideline gap", "notes": "The composite source_title bundles a an annotation platform playbook (a different document) with the Kili guide. Only the Kili guide is at this URL; its portions of the claim (guideline-gap quote, >20% RLHF noise, calibration) are confirmed. Vendor-specific playbook steps are not present. Page-stated date (Jun 24 2026) also differs from the claimed '2024-2025' vintage." } ] }, { "url": "https://arxiv.org/abs/2602.02550", "retrieval": "abstract", "fetched": ["https://arxiv.org/abs/2602.02550"], "resolved_title": "HyPAC: Cost-Efficient LLMs-Human Hybrid Annotation with PAC Error Guarantees", "resolved_date": "2026-01-30", "title_match": true, "claims": [ { "id": "F26-04", "verdict": "confirmed", "checked": [ "calibrates two decision thresholds using importance sampling and upper confidence bounds", "splits inputs into three regions based on uncertainty", "routes to fast LLMs / slow reasoning models / human experts", "PAC error guarantee free of data distribution and pre-trained models (distribution-free)", "reduces the annotation cost by 78.51%", "submitted 30 Jan 2026" ], "not_visible": ["'one-human-touch' phrasing (author's editorial framing, not in the paper's abstract)"], "quote": "calibrates two decision thresholds using importance sampling and upper confidence bounds ... reduces the annotation cost by 78.51%", "notes": "All headline numbers and mechanics confirmed. 'Cheap LLM' is stated as 'fast LLMs' but same meaning; 'expensive reasoning model' as 'slow reasoning models'. 78.51% exact match." } ] }, { "url": "https://arxiv.org/abs/2505.18486", "retrieval": "abstract", "fetched": ["https://arxiv.org/abs/2505.18486"], "resolved_title": "Comparing Human and AI Rater Effects Using the Many-Facet Rasch Model", "resolved_date": "2025-05-24 (v1); v2 2025-05-28", "title_match": true, "claims": [ { "id": "CG-06", "verdict": "partially_confirmed", "checked": [ "fit Many-Facet Rasch Model to ten LLMs plus human expert raters on the same writing tasks", "two types of writing tasks (holistic and analytic scores)", "GPT-4o (ChatGPT 4o), Gemini 1.5 Pro, and Claude 3.5 Sonnet recommended with high accuracy and fewest rater effects", "QWK vs human scores and Cronbach alpha across prompts used", "authors Jiao, Song & Lee; submitted May 24 2025" ], "not_visible": [ "'severity/centrality' as the specific rater-effect types (abstract says 'rater effects' generally; severity/centrality not named)", "'rater effects that differ by model' (differ-by-model direction not stated verbatim in abstract)", "Parallel Wang et al. (Computers and Education: AI, Sept 2025) framework - a separate source not fetched" ], "quote": "high scoring accuracy, better rater reliability, and less rater effects", "notes": "Core (10 LLMs + human raters via MFRM; GPT-4o/Gemini 1.5 Pro/Claude 3.5 Sonnet as top performers) confirmed. The specific severity/centrality effect vocabulary is not visible in the retrieved abstract." } ] }, { "url": "https://www.cochranelibrary.com/cdsr/doi/10.1002/14651858.CD000259.pub4/full", "retrieval": "fulltext", "fetched": ["https://www.cochranelibrary.com/cdsr/doi/10.1002/14651858.CD000259.pub4/full", "https://www.cochranelibrary.com/cdsr/doi/10.1002/14651858.CD000259.pub4/full?cookiesEnabled"], "resolved_title": "Audit and feedback: effects on professional practice (Ivers, N - 2025)", "resolved_date": "2025-03-25 (Version published 25 March 2025)", "title_match": true, "claims": [ { "id": "CG-20", "verdict": "confirmed", "checked": [ "292 studies with 678 arms", "median absolute improvement in desired practice of 2.7%, IQR 0.0 to 8.6 (177 studies)", "mean absolute increase 6.2% (95% CI 4.1 to 8.2; moderate-certainty)", "OR 1.47 (95% CI 1.31 to 1.64)", "Lower baseline performance associated with larger intervention effects", "individual-recipient-level data rather than team-level data", "compares performance to top peers or a benchmark", "comparison to average performance of all peers did NOT find significant effects", "local champion with existing relationship (trusted local source)", "actionable plan with specific advice for improvement", "Contrary to expectations, repeated delivery was associated with lower effect size", "Version published 25 March 2025" ], "not_visible": [], "quote": "mean absolute increase in desired practice of 6.2% (95% confidence interval (CI) 4.1 to 8.2; moderate-certainty evidence) and an OR of 1.47 (95% CI 1.31 to 1.64; moderate-certainty evidence)", "notes": "Every number and every moderator in the claim confirmed verbatim from the full-text abstract and results, including the counterintuitive repeated-delivery finding and the peer-average null. Page required a cookie-jar two-step (WebFetch 403; curl 412) to bypass the cookie gate." } ] } ] }