{
  "batch": 10,
  "sources": [
    {
      "url": "https://arxiv.org/abs/2506.01937",
      "source_title": "RewardBench 2: Advancing Reward Model Evaluation",
      "source_date": "2025-06-02",
      "source_type": "academic",
      "claims": [
        {
          "id": "AJ-10",
          "domain": "academic-judges",
          "claim": "RewardBench 2 (Ai2, June 2025) made reward-model evaluation ~20 points harder via best-of-4 format (random = 25%) and unseen human prompts; top models score below 40% on Precise Instruction Following, yet benchmark scores correlate 0.87 (Pearson) with downstream best-of-N performance across 113 reward models.",
          "evidence": "arXiv 2506.01937 (verified via abstract, Ai2 dataset page, and Lambert's analysis post). 1,865 cases across Factuality, Focus, Math, Precise IF, Safety, Ties. Lambert (author) notes generative LLM-as-judge approaches remain 'weaker than expected relative to standard reward models' at output ranking, and that frontier models still fail trivial rankings ('name a color in the rainbow').",
          "in_corpus_verdict": null
        }
      ]
    },
    {
      "url": "https://arxiv.org/abs/2103.14749",
      "source_title": "Pervasive Label Errors in Test Sets Destabilize Machine Learning Benchmarks (NeurIPS 2021 D&B)",
      "source_date": "2021-11-01",
      "source_type": "academic",
      "claims": [
        {
          "id": "AQ-10",
          "domain": "annotation-quality",
          "claim": "Gold/test labels themselves are wrong at material rates - a lower-bound average of 3.3% label errors across 10 canonical benchmarks (at least 6% in the ImageNet validation set), validated by human review of confident-learning-flagged candidates (51% flag precision) - and error rates this size are enough to flip model rankings.",
          "evidence": "Northcutt, Athalye & Mueller (NeurIPS 2021 Datasets & Benchmarks; arXiv 2103.14749, opened and verified): algorithmic flagging via confident learning + crowdsourced verification; ResNet-18 beats ResNet-50 on corrected ImageNet if mislabeled prevalence rises just 6%. Cleanlab is the maintained open-source implementation.",
          "in_corpus_verdict": null
        }
      ]
    },
    {
      "url": "https://arxiv.org/abs/2411.03417",
      "source_title": "Usefulness of LLMs as an Author Checklist Assistant for Scientific Papers: NeurIPS'24 Experiment",
      "source_date": "2024-11-05",
      "source_type": "academic",
      "claims": [
        {
          "id": "CM-12",
          "domain": "critique-models",
          "claim": "A deployed LLM compliance-checker was gamed by its subjects in a single shot: the NeurIPS 2024 author checklist assistant (234 papers) was rated useful by >70% of authors, but the organizers found the system 'not robust to gaming' by authors and concluded it is a poor substitute for human review.",
          "evidence": "Goldberg, Ullah, Guyon, Shah et al. (arXiv 2411.03417; NeurIPS blog Dec 2024): GPT-4-turbo checklist verification offered pre-submission; surveys plus qualitative analysis showed genuine improvements in some submissions alongside demonstrated manipulation of the assistant (authors could satisfy the checker without satisfying the standard); access was restricted to authors partly to avoid biasing review. Verified via arXiv abstract and organizer blog summary.",
          "in_corpus_verdict": null
        }
      ]
    },
    {
      "url": "https://www.patronus.ai/blog/lynx-state-of-the-art-open-source-hallucination-detection-model",
      "source_title": "Patronus AI Lynx / HaluBench results",
      "source_date": "2024-07 (Lynx 1.0, foundational); Lynx 2.0 later update",
      "source_type": "practitioner",
      "claims": [
        {
          "id": "GF-12",
          "domain": "grounding-faithfulness",
          "claim": "RAGAS-style prompt-chain faithfulness scoring materially underperforms finetuned/frontier judges on hallucination detection: on the 15K-sample HaluBench, RAGAS Faithfulness scored 66.9% accuracy versus 87.4% for finetuned Lynx-70B and 86.5% for GPT-4o; Patronus has since shipped Lynx 2.0 (8B, long-context, 8 hallucination subtypes including coreference and calculation errors).",
          "evidence": "Confirmed via Patronus AI announcement and Lynx paper coverage (July 11, 2024) with the HaluBench table (Lynx-70B 87.4, GPT-4o 86.5, GPT-4-Turbo 85.0, Llama-3-70B 80.1, RAGAS 66.9); Lynx 2.0 details from Patronus docs. Caveat: vendor-reported numbers on the vendor's own benchmark, though HaluBench is public on HuggingFace. RAGAS remains the most widely adopted OSS RAG-eval framework per 2026 practitioner surveys (atlan.com RAG evaluation guide, 2026).",
          "in_corpus_verdict": null
        }
      ]
    },
    {
      "url": "https://help.cleanlab.ai/tlm/",
      "source_title": "Cleanlab Trustworthy Language Model (TLM) documentation and benchmark",
      "source_date": "2025-11-18 (benchmark); docs current 2026",
      "source_type": "practitioner",
      "claims": [
        {
          "id": "HS-12",
          "domain": "hybrid-statistical",
          "claim": "Commercial tooling for confidence-scored LLM judgments with human calibration is mature: Cleanlab's Trustworthy Language Model attaches a real-time trustworthiness score to every LLM response, supports custom evaluation criteria, explicitly documents calibrating trust scores against human quality ratings, and publishes benchmarks (Nov 2025) for auto-flagging incorrect structured outputs for human review.",
          "evidence": "Cleanlab TLM documentation (help.cleanlab.ai/tlm/: quickstart, custom-eval calibration-against-human-ratings tutorial, advanced usage with low-score explanations) and 2025-11-18 structured-outputs benchmark blog post, all fetched via search with page content confirmed. Vendor source: benchmark numbers are self-reported.",
          "in_corpus_verdict": null
        }
      ]
    },
    {
      "url": "[link withheld]",
      "source_title": "An annotation platform's blog post on its review process",
      "source_date": "c. 2024, verified live 2026-07-14",
      "source_type": "primary",
      "claims": [
        {
          "id": "IP-01",
          "domain": "industry-practice",
          "claim": "An annotation platform's production QA stack is double-blind independent labeling, automated discrepancy detection, expert adjudication, and rework, with a fine-tuned-LLM similarity judge that only FLAGS below-threshold responses for human review (never autonomously fails them), Krippendorff's alpha as the published reliability bar (>=0.80 satisfactory, 0.67-0.79 moderate, <0.67 poor), and a multidimensional per-worker 'trust score' that routes complex tasks to high-trust raters.",
          "evidence": "Fetched and read the full annotation platform's blog post on its review process. It details the 4-step double-entry review flow, LLM-assisted QC (fine-tune on expert-verified responses -> similarity score -> flag sub-threshold for human review), explicit Krippendorff alpha thresholds preferred over Cohen/Fleiss kappa, per-task metric menu (Kendall's tau/ELO for rankings, F1/Matthews for binary), trust-score routing, 3% workforce acceptance rate, SLA-backed pay-only-for-quality guarantee, and admitted failure modes: for generative tasks 'there isn't a single clear-cut right answer' so accuracy is relative to the client's worldview, and low precision is attributed to poor instructions/edge-case coverage as often as to workers.",
          "in_corpus_verdict": "confirmed"
        }
      ]
    },
    {
      "url": "https://www.nature.com/articles/s41562-024-02024-1",
      "source_title": "When combinations of humans and AI are useful: A systematic review and meta-analysis",
      "source_date": "2024-10-28",
      "source_type": "academic",
      "claims": [
        {
          "id": "CT-02",
          "domain": "contrarian",
          "claim": "Across 106 experiments and 370 effect sizes, human-AI combinations performed significantly worse than the best of human or AI alone (Hedges g = -0.23, 95% CI -0.39 to -0.07), with losses concentrated in decision-making tasks and specifically when the AI outperforms the human alone - and a 2025 AIES study found human reviewers followed severely race-biased AI hiring recommendations ~90% of the time.",
          "evidence": "Vaccaro, Almaatouq & Malone (Nature Human Behaviour, preregistered meta-analysis; direction, decision-task losses, and AI-better-than-human losses verified from the arXiv abstract; g value corroborated across MIT Sloan, Nature's own summary, and ResearchGate). UW's 'No Thoughts Just AI' (AIES 2025, DOI 10.1609/aies.v8i3.36749, 528 participants, 16 job types) found participants matched biased AI picks even at moderate bias and ~90% at severe bias, vs equal selection rates with no/neutral AI. The 2026 practitioner literature (e.g., tianpan.co HITL rubber-stamp essay, 2026-04-15) reports the same mechanism in production review pipelines.",
          "in_corpus_verdict": "confirmed"
        }
      ]
    },
    {
      "url": "https://arxiv.org/abs/2603.25133",
      "source_title": "RubricEval: A Rubric-Level Meta-Evaluation Benchmark for LLM Judges in Instruction Following (arXiv 2603.25133)",
      "source_date": "2026-03-26",
      "source_type": "academic",
      "claims": [
        {
          "id": "F26-06",
          "domain": "frontier-2026",
          "claim": "Rubric-level judging is far from solved as of March 2026: on RubricEval, the first rubric-level (per-criterion) meta-evaluation benchmark for instruction following (3,486 quality-controlled instances), GPT-4o scores only 55.97% on the Hard subset; rubric-level evaluation outperforms checklist-level, explicit reasoning improves accuracy, and combining both reduces inter-judge variance.",
          "evidence": "Abstract opened and verified on arXiv (2603.25133, Mar 26, 2026). Prior meta-evals scored judges at the response level only; RubricEval scores the fine-grained per-criterion judgments our AutoQA verdicts would be built from, and includes a rubric taxonomy of common judge failure modes. Part of a Q1 2026 wave of criterion-granularity meta-evals (IF-RewardBench arXiv 2603.04738, RubricBench arXiv 2603.01562, MCJudgeBench, AJ-Bench).",
          "in_corpus_verdict": null
        }
      ]
    },
    {
      "url": "https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=CELEX:32024L2831",
      "source_title": "Directive (EU) 2024/2831, Official Journal 11 Nov 2024 (EUR-Lex), corroborated by Wolters Kluwer Global Workplace Law & Policy analysis",
      "source_date": "2024-11-11",
      "source_type": "primary",
      "claims": [
        {
          "id": "CG-07",
          "domain": "critic-and-gapfill",
          "claim": "Platform Work Directive Article 10(5) requires that any decision to restrict, suspend, or terminate the contractual relationship or account of a person performing platform work - or any other decision of equivalent detriment - be taken by a human being, with no consent or contractual-necessity exception (stricter than GDPR Art 22(2)).",
          "evidence": "Verified against the directive text and multiple law-firm/academic analyses (Wolters Kluwer, Taylor Wessing, ETUI). Article 10 covers decisions 'taken or supported' by automated systems, deliberately closing the GDPR gap for hybrid human+algorithm pipelines. Unlike GDPR Art 22, the PWD permits no consent-based or contract-based exception for these decisions.",
          "in_corpus_verdict": null
        },
        {
          "id": "CG-08",
          "domain": "critic-and-gapfill",
          "claim": "Platform Work Directive Article 11 grants persons performing platform work the right to (a) a plain-language explanation of ANY decision taken or supported by an automated decision-making system, (b) a designated competent human contact person to discuss the decision, and (c) human review with a substantiated written reply within two weeks of request; decisions that infringed rights must be rectified within two weeks or compensated.",
          "evidence": "Verified verbatim from directive text via extraction of the EUR-Lex page: explanation 'in a transparent and intelligible manner, using clear and plain language'; contact persons must have 'the competence, training and authority necessary'; substantiated reply 'in any event within two weeks of receipt of the request'; rectification/compensation duty plus obligation to modify or discontinue the offending automated system. Carve-out: for workers who are 'business users' under the P2B Regulation (EU) 2019/1150, that regulation's human-review provisions prevail - but P2B itself mandates statements of reasons and an internal complaint-handling system for restriction/suspension/termination.",
          "in_corpus_verdict": null
        },
        {
          "id": "CG-10",
          "domain": "critic-and-gapfill",
          "claim": "Platform Work Directive Article 7 prohibits automated systems from processing personal data on workers' emotional or psychological state, private conversations (including worker-to-worker exchanges), any data collected while the person is not offering or performing platform work, data predicting exercise of fundamental rights (e.g., organizing), or inferring protected characteristics; Article 8 additionally mandates a data protection impact assessment.",
          "evidence": "Verified verbatim from the directive text via EUR-Lex extraction (Art 7(1)(a)-(c) and recital 40). Art 7(3) extends these limits to ANY automated system 'taking or supporting decisions that affect persons performing platform work in any manner' - not just formally designated monitoring systems.",
          "in_corpus_verdict": null
        }
      ]
    },
    {
      "url": "https://www.cs.cmu.edu/~spdow/files/Crowds-Shepherd-CSCW12.pdf",
      "source_title": "Shepherding the Crowd Yields Better Work (Dow, Kulkarni, Klemmer, Hartmann; CSCW 2012)",
      "source_date": "2012-02-11 (foundational; only direct crowdwork feedback RCT found)",
      "source_type": "academic",
      "claims": [
        {
          "id": "CG-21",
          "domain": "critic-and-gapfill",
          "claim": "In the closest annotation-analog RCT (105 analyzed Mechanical Turk workers writing product reviews), both timely external expert feedback and rubric-based self-assessment significantly improved work quality vs no feedback (expert ratings 6.01 and 6.35 vs 5.69 on a 9-point scale, p<0.05) with NO quality difference between external and self-assessment; external feedback uniquely drove revision behavior (56.5% revised vs 24.8% for self-assessment) and more output, and self-assessors over-rated their own work by 1.8 points (7.9 self vs 6.1 expert).",
          "evidence": "Read the full Dow, Kulkarni, Klemmer & Hartmann CSCW 2012 PDF. Between-subjects, blind-to-condition expert grading; self-assessment condition showed significant learning over the task series (slope 0.25, p=0.001) vs borderline for external (0.10, p=0.08) and null for none. Also: crowd-peer assessors had low agreement with the expert (Kappa=0.20 aggregated), and the paper explicitly did not measure long-term learning. Attrition was higher under assessment (Self 78%, External 61%, vs None 47% incompleteness), so gains partly reflect weak performers dropping out plus learning.",
          "in_corpus_verdict": null
        }
      ]
    }
  ]
}