{ "batch": 12, "checked_at": "2026-07-15", "sources": [ { "url": "https://arxiv.org/abs/2606.01629", "retrieval": "abstract", "fetched": ["https://arxiv.org/abs/2606.01629"], "resolved_title": "Benchmarking LLM-as-a-Judge for Long-Form Output Evaluation", "resolved_date": "2026-06-01 (v1); 2026-06-02 (v2)", "title_match": true, "claims": [ { "id": "AJ-12", "verdict": "confirmed", "checked": [ "judges unstable across long-form scenarios: 'a substantial reliability gap: current LLM judges remain unstable across scenarios'", "document-level assessment of 'overall organization, task-relevant coverage and depth, cross-section consistency'", "rubrics/references 'are helpful but not always sufficient'", "code public at github.com/cjj826/LongJudgeBench", "authors Chen et al.; submitted June 1 2026 (matches source_date 2026-06-01)" ], "not_visible": [ "evidence-field characterization 'First meta-evaluation benchmark targeting long-form judging specifically' -- abstract calls it 'a comprehensive benchmark' and does not explicitly claim to be first" ], "quote": "current LLM judges remain unstable across scenarios ... more complex document-level assessments of overall organization, task-relevant coverage and depth, cross-section consistency ... helpful but not always sufficient", "notes": "Claim text fully supported by the abstract. The only unsupported specific is the 'first benchmark' framing, which appears in the evidence field, not the claim; abstract says 'comprehensive benchmark' without a primacy claim." } ] }, { "url": "https://arxiv.org/abs/2506.19467", "retrieval": "abstract", "fetched": ["https://arxiv.org/abs/2506.19467"], "resolved_title": "Can Reasoning Help Large Language Models Capture Human Annotator Disagreement?", "resolved_date": "2025-06-24 (v1); 2026-01-12 (v3)", "title_match": true, "claims": [ { "id": "AQ-12", "verdict": "confirmed", "checked": [ "RLVR degrades disagreement modeling: 'RLVR-style reasoning degrades performance in disagreement modeling'", "naive CoT improves RLHF models: 'naive Chain-of-Thought (CoT) reasoning improves the performance of RLHF LLMs'", "'60 experimental setups across 3 tasks' -- matches asserted 60 setups / 3 tasks", "warning: 'the potential risk of replacing human annotators with reasoning LLMs'", "EACL 2026 Main listed; v3 dated Jan 12 2026 (matches source_date 2026-01-12)" ], "not_visible": [ "evidence-field 'author communication' that small fine-tuned ModernBERT-class models with human labels beat large reasoning models -- explicitly external to the paper, not in the retrieved abstract", "specific metric names 'variance-correlation' and 'distributional-alignment' not verbatim in retrieved abstract (though 'distribution expression methods, and steering methods' are confirmed)" ], "quote": "RLVR-style reasoning degrades performance in disagreement modeling ... naive Chain-of-Thought (CoT) reasoning improves the performance of RLHF LLMs ... resulting in 60 experimental setups across 3 tasks", "notes": "All core claim assertions and the 60-setups/3-tasks figures are confirmed. The ModernBERT comparison is flagged in the evidence itself as author communication, so it cannot be checked against the source and is not weighed against the claim." } ] }, { "url": "https://arxiv.org/abs/2505.04847", "retrieval": "fulltext", "fetched": ["https://arxiv.org/abs/2505.04847", "https://arxiv.org/html/2505.04847v2"], "resolved_title": "Benchmarking LLM Faithfulness in RAG with Evolving Leaderboards", "resolved_date": "2025-05-07 (v1); 2025-11-06 (v2)", "title_match": true, "claims": [ { "id": "GF-02", "verdict": "confirmed", "checked": [ "prior detectors ~50% on FaithBench: 'current methods ... achieved near 50% accuracy, suggesting negligible ability'", "HHEM-2.1-Open 66.7 (FaithBench claim-wise, asterisked value)", "Bespoke-MiniCheck 71.2 balanced accuracy (Table 1)", "zero-shot frontier judges: 'balanced accuracy below 78% and F1-macro below 72%'", "FaithJudge (o3-mini-high) 84.0 balanced accuracy / 82.1 F1-macro (Table 2 and text)", "beats FACTS Grounding prompt on all four splits; FaithBench-Summary F1 54.3 (FACTS) vs 70.8 (FaithJudge) (Table 4)", "failure modes: underprediction for Command-R/Mistral/Qwen; Benign/Questionable ternary unreliable so binary only; specificity slightly decreases as more examples given", "EMNLP Industry Track 2025; v2 Nov 6 2025 (matches source_date 2025-11)" ], "not_visible": [], "quote": "The highest effectiveness is achieved using the o3-mini-high judge, reaching a balanced accuracy of 84% and an F1-macro of 82.1%", "notes": "Full-text verification confirms every asserted number, upgrading the prior in_corpus_verdict (partially_confirmed) to confirmed. One nuance: the 66.7 for HHEM-2.1-Open is its FaithBench-specific claim-wise value (asterisked because HHEM helped select FaithBench's adversarial articles); its cross-dataset average is 67.1. Claim asserted 66.7, which matches the FaithBench value." } ] }, { "url": "https://arxiv.org/abs/2407.18370", "retrieval": "fulltext", "fetched": ["https://arxiv.org/abs/2407.18370", "https://arxiv.org/pdf/2407.18370", "https://arxiv.org/html/2407.18370v1"], "resolved_title": "Trust or Escalate: LLM Judges with Provable Guarantees for Human Agreement", "resolved_date": "2024-07-25 (v1, only version listed on arXiv)", "title_match": true, "claims": [ { "id": "HS-02", "verdict": "confirmed", "checked": [ "Cascaded Selective Evaluation and Simulated Annotators confirmed (abstract)", "provable, user-specified human-agreement guarantee confirmed", ">80% agreement at ~80% coverage: 'guarantees over 80% human agreement with almost 80% test coverage'; precise 'covering 79.1% of all samples'", "GPT-4 alone almost never reaches 80% agreement on ChatArena subset (abstract)", "~88% by cheaper models: '88.1% are evaluated by substantially cheaper Mistral-7B or GPT-3.5 instead of GPT-4'", "fixed-sequence testing: 'we adopt fixed-sequence testing instead of Bonferroni correction'", "small human calibration set: 'given access to a small calibration set' of human preferences (size 500 / 392 in experiments)" ], "not_visible": [ "'ICLR 2025 Oral' venue/status -- arXiv page carries no journal-ref or comments field and lists only v1 (July 2024); the evidence cited iclr.cc, which is outside the allowed fetch scope, so venue cannot be confirmed from the source" ], "quote": "among which 88.1% are evaluated by substantially cheaper Mistral-7B or GPT-3.5 instead of GPT-4 ... we adopt fixed-sequence testing instead of Bonferroni correction", "notes": "Every substantive mechanism and number in the claim is confirmed from the arXiv HTML full text (the PDF returned only compressed binary). The sole unverifiable item is the ICLR 2025 Oral label, which the arXiv record does not carry." } ] }, { "url": "https://arxiv.org/html/2507.17746v1", "retrieval": "fulltext", "fetched": ["https://arxiv.org/html/2507.17746v1"], "resolved_title": "Rubrics as Rewards (RaR) -- content-confirmed (RaR-Implicit tables, four desiderata); exact string not separately captured", "resolved_date": "2025-07 (arXiv 2507; source_date 2025-07-23)", "title_match": true, "claims": [ { "id": "RR-03", "verdict": "confirmed", "checked": [ "synthetic-with-references on par with human rubrics: RaR-Implicit-Synthetic 0.359 vs RaR-Implicit-Human 0.348; 'yield comparable performance'", "prompt-specific beats generic: generic 0.2898 vs 0.3723; 'predefined generic rubrics substantially underperform'", "Pitfall criteria hard to synthesize: 'often demands human intuition and domain expertise'", "four design principles: Grounded in Expert-guidance, Comprehensive coverage, Semantic weighting, Self-contained evaluation", "semantic weighting Essential 1.0 / Important 0.7 / Optional 0.3 / Pitfall 0.8 penalty", "'7-20' criteria per rubric", "implicit aggregation beats explicit weighted-sum; 'up to a 28% relative improvement on HealthBench-1k'", "small judges benefit most; rubric scores more tightly clustered (reduced variance)" ], "not_visible": [ "evidence-field detail that rubrics were 'generated by o3-mini/GPT-4o from reference answers' -- generator model names not captured in the retrieved excerpts" ], "quote": "human-authored rubrics and synthetic rubrics with access to references yield comparable performance ... synthetically generating effective pitfall criteria ... often demands human intuition and domain expertise", "notes": "All headline numbers and the four principles verified verbatim from the HTML full text. The <title> element was not separately extracted, but the RaR-Implicit-Synthetic/Human table rows and desiderata make identity unambiguous." } ] }, { "url": "https://docs.langchain.com/langsmith/improve-judge-evaluator-feedback", "retrieval": "fulltext", "fetched": ["https://docs.langchain.com/langsmith/improve-judge-evaluator-feedback"], "resolved_title": "Improve LLM-as-judge evaluators using human feedback (LangSmith docs)", "resolved_date": "no publication date visible on the doc page", "title_match": true, "claims": [ { "id": "IP-03", "verdict": "partially_confirmed", "checked": [ "annotation queue -> reference dataset workflow confirmed", "'We recommend starting with at least 20 examples' and examples 'balanced in both 0 and 1 labels'", "alignment score = 'the percentage of examples where the evaluator's judgment matches that of the human expert' (raw percent agreement, no chance correction)", "Evaluator Playground confirmed ('Start Alignment')", "cluster misaligned examples 'into common failure modes' then encode as prompt instructions" ], "not_visible": [ "Braintrust specifics (50-100 labels per scoring dimension, 3-tier deterministic/judge/human stack, 'silent overconfidence' failure mode, human-review-not-automatically-gold-standard, 2026-04-03 date) -- on a separate Braintrust URL not in scope for this source", "Galileo CLHF auto-adaptation from expert corrections (2025-02-10 blog) -- separate URL, not fetchable under the source-only constraint", "cross-vendor generalization 'none of them ships chance-corrected statistics' -- only LangSmith's raw-percent-agreement design is verifiable from the single given URL" ], "quote": "The alignment score is the percentage of examples where the evaluator's judgment matches that of the human expert. ... We recommend starting with at least 20 examples", "notes": "The LangSmith half of the claim (>=20 balanced examples, alignment score = raw percent agreement, iterate prompt, cluster failure modes, route disagreements) is fully confirmed. The Braintrust and Galileo assertions live on other pages the hard constraint forbids fetching, so the multi-vendor convergence claim is only partially verifiable." } ] }, { "url": "https://arxiv.org/abs/2502.04313", "retrieval": "abstract", "fetched": ["https://arxiv.org/abs/2502.04313"], "resolved_title": "Great Models Think Alike and this Undermines AI Oversight", "resolved_date": "2025-02-06 (v1); 2025-06-12 (v2)", "title_match": true, "claims": [ { "id": "CT-05", "verdict": "confirmed", "checked": [ "judge self-preference generalization: 'LLM-as-a-judge scores favor models similar to the judge, generalizing recent self-preference results'", "CAPA = 'Chance Adjusted Probabilistic Agreement (CAPA): a metric for LM similarity based on overlap in model mistakes' (matches 'chance-adjusted mistake-overlap')", "correlated failures: 'model mistakes are becoming more similar with increasing capabilities, pointing to risks from correlated failures'", "undermines AI oversight paradigm", "v1 Feb 6 2025 (matches source_date 2025-02-06)" ], "not_visible": [ "'ICML 2025' venue -- not present on the arXiv page (comments read '60 pages, 20 figures'; no journal-ref)", "cross-reference 'Preference Leakage (arXiv 2502.01534)' -- a different paper, not fetchable under the source-only constraint" ], "quote": "LLM-as-a-judge scores favor models similar to the judge, generalizing recent self-preference results ... model mistakes are becoming more similar with increasing capabilities, pointing to risks from correlated failures", "notes": "Claim core (similar-model judge bias, CAPA mistake-overlap metric, rising error correlation, oversight risk) fully confirmed verbatim. ICML 2025 and the Preference Leakage cross-reference appear only in the evidence field and cannot be checked from this URL." } ] }, { "url": "https://alignment.openai.com/coval", "retrieval": "fulltext", "fetched": ["https://alignment.openai.com/coval"], "resolved_title": "CoVal: Learning values-aware rubrics from the crowd", "resolved_date": "2026-01-14", "title_match": true, "claims": [ { "id": "F26-08", "verdict": "partially_confirmed", "checked": [ "'crowd-written, prompt-specific rubrics' and 'Our results are based on ~1,000 participants and ~1,000 prompts'", "reliable only at large gaps: 'small gaps correspond to near-ties, while large gaps more consistently pick out the human-preferred completion'; 'For small gaps, preference prediction is only slightly better than chance'", "gaming warning: 'Treating the score as an optimization target can also incentivize gaming (e.g., verbosity or checklist-style responses'", "CoVal-full preserves conflicting criteria; CoVal-core 'retains 4 highly rated, mutually compatible criteria per prompt'", "'no single stable target to predict'", "dataset released in two complementary forms (Hugging Face openai/coval)" ], "not_visible": [], "quote": "CoVal-full achieves 0.61 concordance with the crowd and CoVal-core achieves 0.58 ... [out-of-sample] CoVal-full achieves .75 and CoVal-core achieves .76", "notes": "Discrepancy on the headline number: the claim states rubric scores 'reach only 0.58-0.61 pairwise concordance.' Those values are the IN-SAMPLE appendix benchmark; the paper's OUT-OF-SAMPLE validation concordance is higher (CoVal-full .75 / CoVal-core .76). The 0.58-0.61 figures are genuinely in the paper and the 'reliable only at large gaps' framing is accurate, but citing the in-sample number as the concordance understates the reported result. All other specifics confirmed verbatim." } ] }, { "url": "https://academic.oup.com/ilj/article/53/4/840/7745471", "retrieval": "fulltext", "fetched": ["https://academic.oup.com/ilj/article/53/4/840/7745471"], "resolved_title": "Scores as Decisions? Article 22 GDPR and the Judgment of the CJEU in SCHUFA Holding (Scoring) in the Labour Context (Asymina Aza)", "resolved_date": "Industrial Law Journal 53(4):840-858; published online 2024-08-29 (Dec 2024 issue)", "title_match": true, "claims": [ { "id": "CG-11", "verdict": "partially_confirmed", "checked": [ "CJEU SCHUFA C-634/21, 7 Dec 2023 cited (footnote 1)", "core doctrine present: a probability value is a decision where a third party 'draws strongly' on it to 'establish, implement or terminate a contractual relationship' (paras 48, 73)", "general requirement of 'meaningful human involvement' discussed (citing Article 29 Working Party)", "Art 22(1) 'lays down a prohibition in principle' (para 52); 'lacuna in legal protection' if scores were merely a 'preparatory act'" ], "not_visible": [ "exact phrase 'determining role' -- the source uses 'draws strongly', not 'determining role' (wording discrepancy)", "the three-part meaningful-human-involvement test (reviewer with real authority, data access, competence to override) -- not stated in this article as a formulation", "'rubber-stamping' term -- not used; closest material is the Amazon/Hannover Administrative Court ruling", "Italy DPA sanction of automated rider deactivations -- not in this article", "Hamburg DPA ~EUR 490,000 fine, September 2025, for automated rejections without adequate explanation -- not in this 2024 article (and chronologically outside its scope)" ], "quote": "the CJEU held a probability value is a decision where a third party 'draws strongly' on it to 'establish, implement or terminate a contractual relationship'", "notes": "The underlying legal doctrine (a strongly-relied-upon automated score can itself be an Art 22 decision; human involvement must be meaningful) is supported. But the claim's signature phrase 'determining role' is not in this source (it says 'draws strongly'), the three-part reviewer test and 'rubber-stamping' framing are absent, and BOTH enforcement facts (Italian rider deactivations; Hamburg EUR 490k Sept 2025 fine) do not appear in this 2024 academic article -- the evidence field itself attributes them to separate 2025 enforcement roundups." } ] }, { "url": "https://www.annualreviews.org/content/journals/10.1146/annurev-orgpsych-110622-031927", "retrieval": "failed", "fetched": ["https://www.annualreviews.org/content/journals/10.1146/annurev-orgpsych-110622-031927 (WebFetch: HTTP 403; curl browser-UA: Cloudflare managed JS challenge 'Just a moment...'; Playwright same URL: HTTP 403, challenge unresolved after ~14s)"], "resolved_title": "not retrievable", "resolved_date": "not retrievable", "title_match": false, "claims": [ { "id": "CG-23", "verdict": "not_retrievable", "checked": [], "not_visible": [ "'is not yet a story of coherent and cumulative progress'", "insights 'often appear disconnected from the way feedback is practiced and experienced in organizations'", "diverging assumptions across six research substreams", "citation Annu. Rev. Organ. Psychol. Organ. Behav. 12:19-43, published 2025-01-21" ], "quote": "", "notes": "Page is protected by a Cloudflare managed challenge that requires JavaScript execution. WebFetch returned 403; curl with a browser user-agent received only the 'Just a moment...' challenge HTML; the Playwright browser loaded the exact same authorized URL but the managed challenge never cleared (HTTP 403) within ~14s. No article text retrieved; no substitute source permitted, so the claim cannot be verified." } ] } ] }