{
  "batch": 1,
  "sources": [
    {
      "url": "https://arxiv.org/abs/2410.12784",
      "source_title": "JudgeBench: A Benchmark for Evaluating LLM-Based Judges (ICLR 2025) + EmergentMind topic synthesis (updated Jan 2026)",
      "source_date": "2024-10 (ICLR 2025; synthesis updated 2026-01)",
      "source_type": "academic",
      "claims": [
        {
          "id": "AJ-01",
          "domain": "academic-judges",
          "claim": "On hard, objectively-verifiable judging tasks (JudgeBench, ICLR 2025), frontier reasoning models dominate dedicated judge models: o1-preview scored 75.4% overall while GPT-4o scored 50.9-56.6% (near random), the best reward model (Skywork-Reward-Gemma-2-27B) hit 64.3%, and fine-tuned judges like PandaLM fell at or below random.",
          "evidence": "JudgeBench uses 350 rigorously verified response pairs across knowledge/reasoning/math/coding where correctness is checkable. Verified numbers: GPT-4o vanilla 50.9% overall, Arena-Hard pipeline 56.6%, o1-preview 75.4% (85.7% math/coding), Skywork-Reward-27B 64.3%, reward models 60-64%, ChatEval multi-agent ~34%. Fine-tuned preference judges plateau near random on knowledge/reasoning.",
          "in_corpus_verdict": "confirmed"
        }
      ]
    },
    {
      "url": "https://arxiv.org/abs/2310.15683",
      "source_title": "Prevalence and Prevention of Large Language Model Use in Crowd Work (CACM; arXiv 2310.15683)",
      "source_date": "2025-02-18",
      "source_type": "academic",
      "claims": [
        {
          "id": "AQ-01",
          "domain": "annotation-quality",
          "claim": "Roughly one-third of crowdworkers used LLMs on an LLM-advantaged text-production task (33-35% on Prolific, July 2023; ~34% Prolific self-report in a 2025 follow-up by Zhang et al.), and the best tested mitigations (direct request + copy-paste disable or image-only presentation) only cut usage roughly in half (27.6% to ~15.9%), never to zero, while also degrading response quality (direct requests reduced keyword retention 6.2%).",
          "evidence": "Veselovsky et al. ran two preregistered Prolific studies (n=168 and n=720, medical-abstract summarization) with a calibrated classifier plus self-report and keystroke heuristics; prevalence 33.3-35.4% baseline (95% CIs ~[26%,43%]); a 3x3 factorial of requests x hurdles halved but did not eliminate use. The June 2026 community survey (arXiv 2606.04924) corroborates: 30-40% estimated reliance and 34% Prolific self-report, and notes mitigations 'reduce rather than eliminate' LLM-assisted responses.",
          "in_corpus_verdict": "partially_confirmed"
        }
      ]
    },
    {
      "url": "https://arxiv.org/abs/2407.00215",
      "source_title": "LLM Critics Help Catch LLM Bugs (CriticGPT, OpenAI)",
      "source_date": "2024-06-28",
      "source_type": "primary",
      "claims": [
        {
          "id": "CM-01",
          "domain": "critique-models",
          "claim": "Human+critic-model teams occupy a strictly better operating point than either alone: in OpenAI's CriticGPT study, Human+CriticGPT teams wrote more comprehensive critiques than unassisted humans while hallucinating and nitpicking less than the model alone, and the comprehensiveness-vs-spurious-claims tradeoff is a tunable inference-time dial (FSBS length penalty), not a fixed property.",
          "evidence": "CriticGPT critiques were preferred over human contractor critiques in 63% of cases on naturally occurring LLM errors (>80% Elo win rate on inserted bugs); both ChatGPT and CriticGPT caught substantially more inserted bugs than paid contractors (median ~50 min/review, ~5 yrs Python experience). Humans alone had far fewer hallucinated bugs/nitpicks; Human+CriticGPT teams landed between, 'moving beyond the model-only Pareto frontier.' Force Sampling Beam Search scores candidates by rm_score + LENGTH_MODIFIER x num_highlights, tracing a precision/comprehensiveness Pareto curve selectable at deployment without retraining. Verified directly from the arXiv HTML full text.",
          "in_corpus_verdict": "confirmed"
        },
        {
          "id": "CM-02",
          "domain": "critique-models",
          "claim": "Critic-assisted review of items already rated 'flawless' by a first human found real, rating-changing problems in 24% of flagged cases, versus 6% found by an unassisted second human review of the same data - a 4x lift on the exact 'confident pass' slice.",
          "evidence": "CriticGPT (RL-only) was run over ChatGPT training data rated flawless by a first annotator; where the critique flagged a problem, reviewers agreed it substantially lowered the rating in 24% of cases, vs 6% in a no-critique second-rater replication - and this held out-of-distribution (mostly non-code tasks). Prioritizing by critique reward-model score further raised catch rates and cut hallucinations. Verified from arXiv HTML full text.",
          "in_corpus_verdict": null
        }
      ]
    },
    {
      "url": "https://aclanthology.org/2025.naacl-long.320/",
      "source_title": "Decomposition Dilemmas: Does Claim Decomposition Boost or Burden Fact-Checking Performance?",
      "source_date": "2025-05",
      "source_type": "academic",
      "claims": [
        {
          "id": "GF-03",
          "domain": "grounding-faithfulness",
          "claim": "Claim decomposition helps weak verifiers but actively degrades strong ones: with MiniCheck as verifier on WiCE, no-decomposition scores 80.01 balanced accuracy while FActScore-style atomic decomposition drops it to 71.11; with the weaker AlignScore verifier the same decomposition improves results - and gains only reappear as input complexity grows, with best results when sub-claim count does not exceed input complexity.",
          "evidence": "Fetched full text of 'Decomposition Dilemmas' (NAACL 2025, arXiv:2411.02400). Four-way decomposition-error taxonomy from manual inspection: (A) omission of context/logical relations, (B) ambiguity (unclear pronouns/references), (C) over-decomposition, (D) alteration of original meaning. FActScore-style atomicity produces the most over-decomposition errors; VeriScore-style tends to omit context. On FELM, decomposition raised MiniCheck F1 48.1->~68 but dropped GPT-4o-mini F1 71.6->54.3.",
          "in_corpus_verdict": "confirmed"
        }
      ]
    },
    {
      "url": "https://arxiv.org/html/2605.16354v1",
      "source_title": "Augmenting Human Evaluation with LLM Judges: How Many Human Reviews Do You Need?",
      "source_date": "2026-05-08",
      "source_type": "academic",
      "claims": [
        {
          "id": "HS-03",
          "domain": "hybrid-statistical",
          "claim": "The minimum viable human gold set is computable in closed form: with a doubly-robust two-stage design, required human labels converge to a floor of n*(1-rho^2) where n* is the target effective sample size and rho the judge-human correlation - e.g., a target effective n=200 with R^2=0.70 and 2,000 judge ratings needs only 65 human labels - with diminishing returns from adding more judge labels and up to ~13% further savings from stratified allocation when judge reliability varies across evaluation axes.",
          "evidence": "arXiv:2605.16354v1 (Jane Paik Kim, Stanford Psychiatry, 2026-05-08) HTML fetched: derives sample-size formulas from the doubly-robust estimator's asymptotic variance; worked examples confirmed (n*=200, R^2=0.7, N=2000 -> 65 human labels; N=400 -> 100). Explicit caveats: pilot overestimation of R^2 breaks precision guarantees (use conservative values), framework assumes human ratings are a reliable gold standard, single-rater scope.",
          "in_corpus_verdict": "confirmed"
        }
      ]
    },
    {
      "url": "https://arxiv.org/html/2603.01562v1",
      "source_title": "RubricBench: Aligning Model-Generated Rubrics with Human Standards",
      "source_date": "2026-03-02",
      "source_type": "academic",
      "claims": [
        {
          "id": "RR-04",
          "domain": "rubrics-recent",
          "claim": "RubricBench (Mar 2026) measures a stable ~26-27 point preference-accuracy gap between self-generated and human-annotated rubrics across all judge backbones (e.g., DeepSeek-v3.2 57.8% vs 84.9%; Gemini-3-Flash 58.0% vs 85.3%), and scaling test-time compute (more sampled rubrics, refinement) does not close it - 'rubric formation', not judge reasoning, is the bottleneck.",
          "evidence": "Full HTML (arXiv 2603.01562) confirmed: 1,147 adversarial pairwise comparisons where surface cues mislead (rejected responses longer/better-formatted/more confident); expert rubrics are 2-10 atomic binary checks derived only from the instruction; LLM-generated rubrics over-produce unnecessary (17.9% vs 10.1%) and overly rigid (12.8% vs 7.7%) rules; checklists of 13+ items add noise ('attention displacement'); even with human rubrics accuracy plateaus ~85% because judges treat must-have constraints as soft signals; safety domain worst for self-generated rubrics (~25-30% vs >90% human).",
          "in_corpus_verdict": "partially_confirmed"
        }
      ]
    },
    {
      "url": "https://futurism.com/scale-ai-zuckerberg-incompetence",
      "source_title": "Futurism: The AI Company Zuckerberg Just Poured $14 Billion Into... (quoting Inc. internal-documents exclusive by Sam Blum)",
      "source_date": "2025-06 (documents cover 2023-03 to 2024-04)",
      "source_type": "practitioner",
      "claims": [
        {
          "id": "IP-04",
          "domain": "industry-practice",
          "claim": "At Scale AI's Outlier/Remotasks (Bulba Experts program for Google, March 2023-April 2024 per internal documents), spam - defined internally as 'writing gibberish, writing incorrect information, GPT-generated thought processes' - routinely got paid because per-item detection could not keep up; defenses degraded to ZeroGPT screening, crude country-level bans (Egypt, Pakistan, Kenya, Venezuela), and were circumvented by VPNs, while a black market sells verified annotator accounts (Outlier, CrowdGen, Prolific).",
          "evidence": "Opened Futurism's 2025-06 piece quoting Inc.'s internal-document exclusive verbatim: spammers 'could get away with just totally submitting garbage and there weren't enough people to track them down'; an allocations team once 'dumped 800 spammers' into one queue; supervisors advised to vet entries with ZeroGPT; Scale's spokesperson disputed the story claiming safeguards existed. AlgorithmWatch separately documented account-resale markets. Prior research (cited in the same reporting thread) estimated 33-46% of MTurk crowdworkers used LLMs on a text task. Community reports (Indeed/Glassdoor/Trustpilot 2025) confirm the attempter->reviewer->senior-reviewer pyramid with timed capped pay and opaque removals.",
          "in_corpus_verdict": null
        },
        {
          "id": "CT-10",
          "domain": "contrarian",
          "claim": "Industrial-scale annotation QA has already lost to spam economics once: Scale AI's 'Bulba Experts' program for Google (Mar 2023-Apr 2024) was flooded with contributors submitting 'writing gibberish, writing incorrect information, GPT-generated thought processes' who were paid anyway because per-item catching was infeasible and banned workers returned via VPN - while academic estimates put LLM use at 33-46% of crowdworkers on text tasks.",
          "evidence": "Futurism (2025-06-28) reporting Inc. Magazine's review of internal Scale AI documents and former queue-manager interviews; article text read in full via crawl ('spammers could get away with just totally submitting garbage and there weren't enough people to track them down'; 'no background checks whatsoever'). Prevalence: Veselovsky et al. arXiv 2306.07899; NeurIPS 2025 'Evaluating LLM-Contaminated Crowdsourcing Data Without Ground Truth' proposes peer-prediction mechanisms precisely because text-detector approaches fail on structured annotations.",
          "in_corpus_verdict": null
        }
      ]
    },
    {
      "url": "https://arxiv.org/abs/2509.20293",
      "source_title": "When Judgment Becomes Noise: How Design Failures in LLM Judge Benchmarks Silently Undermine Validity",
      "source_date": "2025-09-24",
      "source_type": "contrarian",
      "claims": [
        {
          "id": "CT-06",
          "domain": "contrarian",
          "claim": "LLM judges do not follow their own rubrics: on Arena-Hard Auto, the explicit evaluation schema explains under 10% of verdict variance for some judges (unexplained variance >90% for DeepSeek-R1-32B), and factor correlations above 0.93 across nominally distinct criteria show per-axis scores collapse into a single halo factor.",
          "evidence": "Feuer et al. introduce 'schematic adherence' (how much of the verdict the stated rubric explains) and psychometric validity checks; abstract verified via arXiv crawl: 'severe schema incoherence and factor collapse across popular judges'; ELO-style aggregation additionally masks genuine ranking uncertainty. Code at github.com/penfever/judgment-to-noise.",
          "in_corpus_verdict": null
        }
      ]
    },
    {
      "url": "https://doi.org/10.1609/aaai.v40i44.41123",
      "source_title": "FindTheFlaws: Annotated Errors for Detecting Flawed Reasoning and Scalable Oversight Research (AAAI 2026)",
      "source_date": "2026-03-14",
      "source_type": "academic",
      "claims": [
        {
          "id": "F26-10",
          "domain": "frontier-2026",
          "claim": "FindTheFlaws (AAAI 2026, published March 2026) provides five datasets of long-form expert-annotated flawed vs correct solutions across medicine, math, and science, built specifically to test whether models can detect flawed reasoning and to support scalable-oversight experiments where the evaluator is weaker than the solution author.",
          "evidence": "AAAI-40 proceedings record confirmed (DOI 10.1609/aaai.v40i44.41123, Recchia et al.). This is the 2026-venue instantiation of the seeded-error-set methodology: known-flaw items with expert annotations of where and why the reasoning fails, enabling measurement of flaw-detection recall rather than agreement-with-consensus.",
          "in_corpus_verdict": null
        }
      ]
    },
    {
      "url": "https://www.iso.org/publication/PUB200525.html",
      "source_title": "ISO/IEC 5259 series catalog pages (ISO.org) / AI data quality management bundle",
      "source_date": "2025",
      "source_type": "primary",
      "claims": [
        {
          "id": "CG-12",
          "domain": "critic-and-gapfill",
          "claim": "ISO/IEC 5259 'Data quality for analytics and ML' is now a complete five-part published series - Parts 1-4 published 2024 (overview/terminology; data quality measures; data quality management requirements & guidelines; process framework) and Part 5 (data quality governance framework) published 2025 - sold by ISO as an 'AI data quality management bundle'.",
          "evidence": "Confirmed from ISO.org catalog pages: ISO/IEC 5259-1:2024 (81088), 5259-2:2024 (81860, 'specifies a data quality model, data quality measures and guidance on reporting data quality'), 5259-3:2024 (81092, management requirements), 5259-4:2024 (81093, process framework covering data labeling among ML data processes), 5259-5:2025 (84150, governance). Full text paywalled; no direct evidence found of large AI-data buyers contractually mandating 5259 yet, but 5259-3/-4 are certifiable-style requirements documents a buyer can reference in contract.",
          "in_corpus_verdict": null
        }
      ]
    },
    {
      "url": "https://www.sciencedirect.com/science/article/abs/pii/S1048984321000394",
      "source_title": "Feedback quality and performance in organisations (Leadership Quarterly; synthesizing Charness et al. 2014, Tafkov 2013, Barankay 2012)",
      "source_date": "2021 (synthesizing 2012-2016 primaries)",
      "source_type": "academic",
      "claims": [
        {
          "id": "CG-24",
          "domain": "critic-and-gapfill",
          "claim": "Under incentives, relative-rank feedback is a double-edged lever: lab and field economics find rank feedback raises output in flat-wage settings (Charness et al. 2014; Tafkov 2013) but induces costly sabotage and cheating to improve rank that offsets the gains, and in at least one field experiment (Barankay 2012) REMOVING rank feedback improved performance.",
          "evidence": "Confirmed via the literature synthesis in a peer-reviewed Leadership Quarterly article ('Feedback quality and performance in organisations', 2021), which states: Charness et al. (2014) found 'offering relative rank feedback increases output... and subjects are willing to engage in costly sabotage and cheating activities to improve their relative rank, thus offsetting the positive effects'; Barankay (2012) is cited as the exception where rank feedback hurt. Primary Charness/Barankay texts not independently opened, so treated as well-sourced secondary evidence.",
          "in_corpus_verdict": null
        }
      ]
    }
  ]
}