{
  "batch": 10,
  "checked_at": "2026-07-15",
  "sources": [
    {
      "url": "https://arxiv.org/abs/2506.01937",
      "retrieval": "fulltext",
      "fetched": [
        "https://arxiv.org/abs/2506.01937",
        "https://arxiv.org/pdf/2506.01937",
        "https://arxiv.org/html/2506.01937v2"
      ],
      "resolved_title": "RewardBench 2: Advancing Reward Model Evaluation",
      "resolved_date": "2025-06-02 (v1); v2 2026-04-23; ICLR 2026",
      "title_match": true,
      "claims": [
        {
          "id": "AJ-10",
          "verdict": "confirmed",
          "checked": [
            "~20 points harder: 'models score about 20 points on average lower on RewardBench 2 compared to the first RewardBench'",
            "best-of-4 format with random baseline 25%",
            "1,865 prompts",
            "leading models below 40% on Precise Instruction Following",
            "Pearson 0.87 correlation with downstream BoN sampling",
            "113 reward models evaluated on BoN",
            "six domains: factuality, precise instruction following, math, safety, focus, ties",
            "sources new human prompts instead of existing prompts from downstream evaluations"
          ],
          "not_visible": [
            "Lambert blog quotes 'weaker than expected relative to standard reward models' and 'name a color in the rainbow' (attributed in evidence to an external Lambert analysis post, not the arXiv text)"
          ],
          "quote": "There is only one correct chosen response, meaning the random baseline is 25% accuracy ... average score on downstream tasks with BoN sampling has a high Pearson correlation of 0.87 ... We evaluated 113 RMs",
          "notes": "Abstract page gave only the ~20-point figure and 'new human prompts'; all specific numbers (best-of-4, 25%, 1,865, <40% Precise IF, 0.87, 113, six domains) were verified verbatim from the arXiv HTML full text (v2). Every headline number in the claim matches."
        }
      ]
    },
    {
      "url": "https://arxiv.org/abs/2103.14749",
      "retrieval": "abstract",
      "fetched": ["https://arxiv.org/abs/2103.14749"],
      "resolved_title": "Pervasive Label Errors in Test Sets Destabilize Machine Learning Benchmarks",
      "resolved_date": "2021-03-26 (v1); 2021-11-07 (v4); NeurIPS 2021 Datasets & Benchmarks",
      "title_match": true,
      "claims": [
        {
          "id": "AQ-10",
          "verdict": "confirmed",
          "checked": [
            "average of at least 3.3% errors across the 10 datasets",
            "at least 6% label errors in the ImageNet validation set",
            "51% human-validation precision (roughly half of flagged candidates confirmed mislabeled)",
            "10 canonical benchmarks",
            "confident learning algorithms + crowdsourced human validation",
            "ResNet-18 outperforms ResNet-50 on corrected ImageNet if mislabeled prevalence rises by just 6% (rank flip)"
          ],
          "not_visible": [],
          "quote": "estimate an average of at least 3.3% errors across the 10 datasets ... label errors comprise at least 6% of the ImageNet validation set ... ResNet-18 outperforms ResNet-50 ... if the prevalence of originally mislabeled test examples increases by just 6%",
          "notes": "All numbers and the rank-flip finding confirmed from the abstract page. Cleanlab implementation referenced (github.com/cleanlab/label-errors, labelerrors.com)."
        }
      ]
    },
    {
      "url": "https://arxiv.org/abs/2411.03417",
      "retrieval": "abstract",
      "fetched": ["https://arxiv.org/abs/2411.03417"],
      "resolved_title": "Usefulness of LLMs as an Author Checklist Assistant for Scientific Papers: NeurIPS'24 Experiment",
      "resolved_date": "2024-11-05 (v1); v2 2024-11-08",
      "title_match": true,
      "claims": [
        {
          "id": "CM-12",
          "verdict": "partially_confirmed",
          "checked": [
            "234 papers voluntarily submitted to the LLM-based Checklist Assistant",
            "over 70% of authors found the assistant useful",
            "gaming vulnerability: 'could be manipulated to enhance scores through fabricated justifications'"
          ],
          "not_visible": [
            "specific model 'GPT-4-turbo' (the abstract does not name the model)",
            "conclusion that it is 'a poor substitute for human review' (abstract instead frames it as 'a promising, but controversial, tool in aiding scientific peer review')",
            "exact phrase 'not robust to gaming' (substance present, phrase not)"
          ],
          "quote": "over 70% of authors found the assistant useful ... could be manipulated to enhance scores through fabricated justifications ... a promising, but controversial, tool in aiding scientific peer review",
          "notes": "Core (234 papers, >70% useful, demonstrated gaming/manipulation) confirmed. But the claim's characterization that organizers 'concluded it is a poor substitute for human review' is not supported by the retrieved abstract, which is more favorable ('promising, but controversial'); and the GPT-4-turbo model is not named on the page. 'Gamed in a single shot' is an editorial gloss not in the text."
        }
      ]
    },
    {
      "url": "https://www.patronus.ai/blog/lynx-state-of-the-art-open-source-hallucination-detection-model",
      "retrieval": "landing",
      "fetched": ["https://www.patronus.ai/blog/lynx-state-of-the-art-open-source-hallucination-detection-model"],
      "resolved_title": "Lynx: State-of-the-Art Open Source Hallucination Detection Model",
      "resolved_date": "not stated on page (July 2024 inferred from linked paper/NeMo note)",
      "title_match": false,
      "claims": [
        {
          "id": "GF-12",
          "verdict": "partially_confirmed",
          "checked": [
            "HaluBench is 15k samples",
            "Lynx is positioned as SOTA open-source hallucination detector, beating frontier judges (relative): Lynx-70B 8.3% more accurate than GPT-4o on PubMedQA; Lynx-8B beat GPT-3.5 by 24.5% on HaluBench"
          ],
          "not_visible": [
            "RAGAS Faithfulness 66.9%",
            "Lynx-70B 87.4%",
            "GPT-4o 86.5%",
            "GPT-4-Turbo 85.0%",
            "Llama-3-70B 80.1%",
            "the HaluBench accuracy comparison table itself (this blog reports only relative % improvements, not the absolute-accuracy table)",
            "Lynx 2.0, 8B long-context model, 8 hallucination subtypes (coreference/calculation)"
          ],
          "quote": "a comprehensive hallucination evaluation benchmark consisting of 15k samples",
          "notes": "The given URL confirms the 15k HaluBench size and the directional finding (finetuned Lynx > frontier judges), but NONE of the specific accuracy numbers cited in the claim (66.9/87.4/86.5/85.0/80.1) appear on this page; they are attributed in the evidence to the Lynx paper/table, not this blog. Lynx 2.0 and its subtypes are not mentioned here at all. Resolved title is the literal page title, which differs from the descriptive source_title."
        }
      ]
    },
    {
      "url": "https://help.cleanlab.ai/tlm/",
      "retrieval": "landing",
      "fetched": ["https://help.cleanlab.ai/tlm/"],
      "resolved_title": "Trustworthy Language Model (TLM) | Cleanlab Documentation",
      "resolved_date": "not dated on overview page",
      "title_match": true,
      "claims": [
        {
          "id": "HS-12",
          "verdict": "partially_confirmed",
          "checked": [
            "attaches a real-time trustworthiness score to responses from any LLM",
            "supports custom evaluation criteria (a 'Custom Evaluation Criteria' tutorial is listed)"
          ],
          "not_visible": [
            "documentation of calibrating trust scores against human quality ratings (not on this overview page)",
            "the 2025-11-18 structured-outputs benchmark for auto-flagging incorrect outputs for human review (not on this overview page)"
          ],
          "quote": "scores the trustworthiness of responses from any LLM in real-time ... state-of-the-art trustworthiness scores for any LLM application",
          "notes": "The two core capabilities (real-time trust score for any LLM; custom eval criteria) are confirmed on the given overview URL. The human-rating calibration tutorial and the Nov-2025 structured-outputs benchmark cited in the evidence live on sub-pages (custom-eval / benchmark blog) that are outside the given URL and were not fetched; they are not visible on help.cleanlab.ai/tlm/ itself."
        }
      ]
    },
    {
      "url": "[link withheld]",
      "retrieval": "landing",
      "fetched": ["[link withheld]"],
      "resolved_title": "An annotation platform's blog post on its review process",
      "resolved_date": "2024-09-12",
      "title_match": true,
      "claims": [
        {
          "id": "IP-01",
          "verdict": "confirmed",
          "checked": [
            "double-blind / double-entry: two independent labelers without knowledge of each other's work, automated discrepancy comparison, expert adjudication, rework with feedback",
            "fine-tuned LLM similarity judge flags sub-threshold responses for human review (never autonomously fails them)",
            "Krippendorff's alpha thresholds: >=0.80 satisfactory, 0.67-0.79 moderate, <0.67 poor",
            "multidimensional per-worker trust score (historical accuracy, consistency, completion rate) routing high-trust trainers to complex/critical tasks",
            "3% workforce acceptance rate",
            "generative tasks admitted to lack a single clear-cut right answer (model creator is ultimate judge)"
          ],
          "not_visible": [
            "specific per-task metric menu items Kendall's tau / ELO / F1 / Matthews (not surfaced in this extraction; not contradicted)",
            "SLA-backed pay-only-for-quality guarantee wording (not surfaced; not contradicted)"
          ],
          "quote": "Two independent labelers perform the same task without knowledge of each other's work ... Flagging responses that fall below a certain similarity threshold for human review ... an acceptance rate of just 3% ... there isn't a single clear-cut 'right answer'",
          "notes": "All six headline elements of the QA stack confirmed verbatim from the engineering post (matches prior in_corpus_verdict 'confirmed'). The minor metric-menu and SLA specifics were not individually re-verified in this pass but are consistent with the page's described metric/guarantee framework."
        }
      ]
    },
    {
      "url": "https://www.nature.com/articles/s41562-024-02024-1",
      "retrieval": "landing",
      "fetched": ["https://www.nature.com/articles/s41562-024-02024-1 (via curl; WebFetch hit a 303 auth redirect)"],
      "resolved_title": "When combinations of humans and AI are useful: A systematic review and meta-analysis",
      "resolved_date": "2024-10-28",
      "title_match": true,
      "claims": [
        {
          "id": "CT-02",
          "verdict": "partially_confirmed",
          "checked": [
            "106 experimental studies reporting 370 effect sizes",
            "human-AI combinations performed significantly worse than the best of humans or AI alone, Hedges' g = -0.23; 95% CI -0.39 to -0.07",
            "performance losses concentrated in decision-making tasks",
            "losses specifically when AI outperformed humans alone ('when AI outperformed humans alone, we found losses')"
          ],
          "not_visible": [
            "the AIES 2025 'No Thoughts Just AI' finding that human reviewers followed race-biased AI hiring recommendations ~90% of the time (a separate source, not part of this Nature article)"
          ],
          "quote": "on average, human-AI combinations performed significantly worse than the best of humans or AI alone (Hedges' g = -0.23; 95% confidence interval, -0.39 to -0.07) ... when AI outperformed humans alone, we found losses",
          "notes": "Every fact attributed to this Nature meta-analysis is confirmed verbatim from the abstract (retrieved via curl; WebFetch was blocked by a 303 login redirect). Marked partially_confirmed only because the claim bundles a second, distinct finding (the ~90% AIES 2025 hiring-bias result) that is not from this URL and cannot be verified here."
        }
      ]
    },
    {
      "url": "https://arxiv.org/abs/2603.25133",
      "retrieval": "abstract",
      "fetched": ["https://arxiv.org/abs/2603.25133"],
      "resolved_title": "RubricEval: A Rubric-Level Meta-Evaluation Benchmark for LLM Judges in Instruction Following",
      "resolved_date": "2026-03-26",
      "title_match": true,
      "claims": [
        {
          "id": "F26-06",
          "verdict": "confirmed",
          "checked": [
            "RubricEval title exact match",
            "3,486 quality-controlled instances",
            "GPT-4o achieves only 55.97% on the Hard subset",
            "rubric-level evaluation outperforms checklist-level",
            "explicit reasoning improves accuracy",
            "combining both reduces inter-judge variance",
            "first rubric-level (per-criterion) meta-evaluation benchmark for instruction following; includes a rubric taxonomy of judge failure modes"
          ],
          "not_visible": [
            "sibling-paper corroboration (IF-RewardBench 2603.04738, RubricBench 2603.01562, MCJudgeBench, AJ-Bench) - external references, not on this page"
          ],
          "quote": "a substantial set of 3,486 quality-controlled instances ... GPT-4o achieves only 55.97% on Hard subset ... rubric-level evaluation outperforms checklist-level, explicit reasoning improves accuracy, and both together reduce inter-judge variance",
          "notes": "All headline numbers and qualitative findings confirmed from the abstract. Submission date 26 Mar 2026 matches source_date."
        }
      ]
    },
    {
      "url": "https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=CELEX:32024L2831",
      "retrieval": "fulltext",
      "fetched": ["https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=CELEX:32024L2831"],
      "resolved_title": "Directive (EU) 2024/2831 of the European Parliament and of the Council of 23 October 2024 on improving working conditions in platform work",
      "resolved_date": "2024-10-23 (adopted); OJ 2024-11-11",
      "title_match": true,
      "claims": [
        {
          "id": "CG-07",
          "verdict": "confirmed",
          "checked": [
            "Article 10(5) requires that decisions to restrict/suspend/terminate the contractual relationship or account, or any decision of equivalent detriment, be taken by a human being",
            "no exception listed in the article text"
          ],
          "not_visible": [
            "the comparison 'stricter than GDPR Art 22(2) / no consent or contractual-necessity exception' (interpretive gloss from law-firm analyses, not stated in the directive text itself)"
          ],
          "quote": "Any decision to restrict, suspend or terminate the contractual relationship or the account of a person performing platform work or any other decision of equivalent detriment shall be taken by a human being.",
          "notes": "Article 10(5) confirmed verbatim. The GDPR Art 22(2) contrast is an accurate legal characterization but is not in the directive text (attributed in evidence to Wolters Kluwer / Taylor Wessing / ETUI, not fetchable here)."
        },
        {
          "id": "CG-08",
          "verdict": "confirmed",
          "checked": [
            "Article 11 right to explanation of any decision taken or supported by an ADM system, in transparent/intelligible manner using clear and plain language",
            "designated competent human contact person with 'the competence, training and authority necessary'",
            "human review with a substantiated written reply within two weeks of receipt of the request",
            "decisions that infringe rights rectified without delay and within two weeks of adoption, or adequate compensation",
            "P2B carve-out: Article 11(5) does not apply to persons who are also business users under Regulation (EU) 2019/1150"
          ],
          "not_visible": [
            "the added detail that 'P2B itself mandates statements of reasons and an internal complaint-handling system' (that is content of Regulation 2019/1150, not this directive)"
          ],
          "quote": "The explanation shall be provided in a transparent and intelligible manner, using clear and plain language ... a sufficiently precise and adequately substantiated reply in the form of a written document ... within two weeks of receipt of the request",
          "notes": "All three limbs (a/b/c) plus the rectify-or-compensate-within-two-weeks duty and the business-user carve-out (Art 11(5), Reg 2019/1150) confirmed verbatim from the directive text."
        },
        {
          "id": "CG-10",
          "verdict": "confirmed",
          "checked": [
            "Article 7(1)(a) prohibits processing personal data on emotional or psychological state",
            "7(1)(b) private conversations including exchanges with other persons performing platform work",
            "7(1)(c) collecting data while the person is not offering or performing platform work",
            "7(1)(d) predicting the exercise of fundamental rights including freedom of association",
            "7(1)(e) inferring protected characteristics (racial/ethnic origin, religious beliefs, disability, health, trade union membership, sex life/orientation)",
            "Article 7(3) extends limits to automated systems 'taking or supporting decisions that affect persons performing platform work in any manner'",
            "Article 8 mandates a data-protection impact assessment (Art 35(1) GDPR high-risk processing)"
          ],
          "not_visible": [],
          "quote": "this Article shall also apply where digital labour platforms use automated systems taking or supporting decisions that affect persons performing platform work in any manner ... Article 8 Data-protection impact assessment",
          "notes": "Article 7(1)(a)-(e), Article 7(3), and Article 8 all confirmed verbatim. Every element of the claim (including the 'any manner' extension and the DPIA mandate) is present."
        }
      ]
    },
    {
      "url": "https://www.cs.cmu.edu/~spdow/files/Crowds-Shepherd-CSCW12.pdf",
      "retrieval": "pdf",
      "fetched": ["https://www.cs.cmu.edu/~spdow/files/Crowds-Shepherd-CSCW12.pdf"],
      "resolved_title": "Shepherding the Crowd Yields Better Work (Dow, Kulkarni, Klemmer, Hartmann)",
      "resolved_date": "CSCW'12, February 11-15, 2012",
      "title_match": true,
      "claims": [
        {
          "id": "CG-21",
          "verdict": "confirmed",
          "checked": [
            "105 participants analyzed (538 consumer reviews from 105 participants)",
            "expert 9-point ratings: External mu=6.01, Self mu=6.35, None mu=5.69; F(2,102)=3.02, p<0.05; pairwise None-vs-External and None-vs-Self both p<0.05",
            "NO significant difference between External and Self assessment (t(150)=1.40, p=0.18)",
            "revision rates: 56.5% External vs 24.8% Self changed their review",
            "self-assessors over-rated own work by 1.8 points (mu=7.9 self vs mu=6.1 expert)",
            "crowd-as-aggregate vs expert agreement Cohen's Kappa=0.20",
            "higher attrition in Self (78%) and External (61%) than None (47%)",
            "learning: Self slope 0.25 (p=0.001) significant; External 0.10 (p=0.08) borderline; None null"
          ],
          "not_visible": [],
          "quote": "The External condition (mu=6.01, SD=1.38) and the Self condition (mu=6.35, SD=1.63) outperformed the None condition (mu=5.69, SD=1.19) (F(2,102)=3.02, p<0.05) ... There was no significant difference between External and Self assessment (t(150)=1.40, p=0.18)",
          "notes": "Every number in the claim confirmed verbatim from the extracted PDF text (downloaded via curl, converted with pdftotext). Self over-rating '1.8 points higher (mu=7.9 ... mu=6.1)', revision '56.5% ... 24.8%', Kappa=0.20, attrition 78/61/47%, and learning slopes 0.25/0.10 all present exactly as stated."
        }
      ]
    }
  ]
}