{
  "batch": 12,
  "sources": [
    {
      "url": "https://arxiv.org/abs/2606.01629",
      "source_title": "Benchmarking LLM-as-a-Judge for Long-Form Output Evaluation (LongJudgeBench)",
      "source_date": "2026-06-01",
      "source_type": "academic",
      "claims": [
        {
          "id": "AJ-12",
          "domain": "academic-judges",
          "claim": "LLM judges are measurably less reliable on long-form outputs: LongJudgeBench (June 2026) finds current judges unstable across real-world long-form scenarios requiring document-level assessment of organization, coverage, and cross-section consistency, and rubrics or references help but are 'not always sufficient'.",
          "evidence": "Chen et al., arXiv 2606.01629 (abstract opened and confirmed). First meta-evaluation benchmark targeting long-form judging specifically; systematically evaluates judges across base models and judging protocols; code public. Complements the short-form-centric older benchmarks (MT-Bench, LLMBar, JudgeBench).",
          "in_corpus_verdict": null
        }
      ]
    },
    {
      "url": "https://arxiv.org/abs/2506.19467",
      "source_title": "Can Reasoning Help Large Language Models Capture Human Annotator Disagreement? (EACL 2026)",
      "source_date": "2026-01-12",
      "source_type": "academic",
      "claims": [
        {
          "id": "AQ-12",
          "domain": "annotation-quality",
          "claim": "RLVR-style reasoning training degrades an LLM's ability to model human annotator disagreement, while naive chain-of-thought on RLHF models improves it - evaluated across 60 setups on 3 tasks - meaning stronger 'reasoning' judges are measurably worse at knowing when humans legitimately diverge.",
          "evidence": "Ni et al., EACL 2026 main (arXiv 2506.19467 v3, Jan 12 2026, opened and verified): systematic evaluation over model sizes, distribution-expression and steering methods using variance-correlation and distributional-alignment metrics; authors explicitly warn about 'the potential risk of replacing human annotators with reasoning LLMs' where disagreement carries signal; author communication adds that small fine-tuned models (ModernBERT-class) with human labels often beat large reasoning models at this.",
          "in_corpus_verdict": null
        }
      ]
    },
    {
      "url": "https://arxiv.org/abs/2505.04847",
      "source_title": "Benchmarking LLM Faithfulness in RAG with Evolving Leaderboards (FaithJudge)",
      "source_date": "2025-11",
      "source_type": "academic",
      "claims": [
        {
          "id": "GF-02",
          "domain": "grounding-faithfulness",
          "claim": "On adversarially-hard hallucination sets, specialized detectors collapse and few-shot anchoring with human-annotated exemplars is the measured fix: on FaithBench, prior detectors hit ~50% accuracy (negligible), HHEM-2.1-Open 66.7% and Bespoke-MiniCheck 71.2% balanced accuracy, zero-shot frontier judges stay below 78%, while FaithJudge - prompting o3-mini-high with human-annotated peer responses to the same source document - reaches 84.0% balanced accuracy / 82.1 F1.",
          "evidence": "Fetched full text of arXiv:2505.04847v2 (Vectara, EMNLP 2025 Industry Track, v2 Nov 2025). Table 1: baseline detector numbers; Table 2: FaithJudge results; FaithJudge beats the FACTS Grounding judging prompt head-to-head on all four RAGTruth/FaithBench splits (e.g., FaithBench 70.8 vs 54.3 F1). Documented failure modes: judge underpredicts hallucinations for some generator families; 'benign'/'questionable' ternary labels classified unreliably so only binary is used; specificity drops as more in-context examples are added.",
          "in_corpus_verdict": "partially_confirmed"
        }
      ]
    },
    {
      "url": "https://arxiv.org/abs/2407.18370",
      "source_title": "Trust or Escalate: LLM Judges with Provable Guarantees for Human Agreement (ICLR 2025 Oral)",
      "source_date": "2024-07-25 (ICLR 2025-04)",
      "source_type": "academic",
      "claims": [
        {
          "id": "HS-02",
          "domain": "hybrid-statistical",
          "claim": "Cascaded Selective Evaluation (Trust or Escalate, ICLR 2025 Oral) delivers a provable, user-specified human-agreement guarantee for LLM judges by calibrating a confidence threshold with fixed-sequence testing on a small human calibration set, escalating low-confidence items up a judge cascade and abstaining (to humans) when even the strongest judge is unconfident - achieving guaranteed >80% human agreement at ~80% coverage on a Chatbot Arena subset where GPT-4 alone almost never reaches 80% agreement, with ~88% of judged items handled by much cheaper models.",
          "evidence": "arXiv:2407.18370 abstract fetched directly (80% agreement / ~80% coverage claim confirmed); ICLR 2025 Oral status confirmed at iclr.cc/virtual/2025/oral/31838; fixed-sequence-testing calibration mechanism confirmed via OpenReview PDF snippets and secondary reviews (83 citations as of 2026-07). Its 'Simulated Annotators' method (in-context simulation of diverse annotators) is the confidence estimator that makes the guarantee achievable at high coverage.",
          "in_corpus_verdict": "confirmed"
        }
      ]
    },
    {
      "url": "https://arxiv.org/html/2507.17746v1",
      "source_title": "Rubrics as Rewards: Reinforcement Learning Beyond Verifiable Domains (Scale AI)",
      "source_date": "2025-07-23",
      "source_type": "primary",
      "claims": [
        {
          "id": "RR-03",
          "domain": "rubrics-recent",
          "claim": "Scale AI's Rubrics-as-Rewards found that LLM-synthesized rubrics grounded in expert reference answers performed on par with human-authored rubrics as reward signals (0.359 vs 0.348), that prompt-specific rubrics substantially beat generic ones, and that the one component synthetic generation could NOT produce well was Pitfall criteria - anticipating failure modes 'often demands human intuition and domain expertise'.",
          "evidence": "Full paper (arXiv 2507.17746) confirmed: four design principles (grounded in expert guidance; comprehensive coverage; semantic weighting Essential 1.0 / Important 0.7 / Optional 0.3 / Pitfall 0.8 penalty; self-contained per-criterion evaluability); rubrics generated by o3-mini/GPT-4o from reference answers, ~7-20 criteria per prompt; implicit aggregation (judge sees all criteria+weights and emits one score) consistently beat explicit per-criterion binary + weighted-sum, up to 28% relative on HealthBench-1k; rubrics helped small judges most and reduced cross-judge variance.",
          "in_corpus_verdict": "confirmed"
        }
      ]
    },
    {
      "url": "https://docs.langchain.com/langsmith/improve-judge-evaluator-feedback",
      "source_title": "LangSmith: Improve LLM-as-judge evaluators using human feedback (+ Braintrust: LLM-as-a-judge vs human-in-the-loop, 2026-04-03)",
      "source_date": "retrieved 2026-07-14 (Braintrust article 2026-04-03)",
      "source_type": "primary",
      "claims": [
        {
          "id": "IP-03",
          "domain": "industry-practice",
          "claim": "The 2026 eval-tooling vendors (LangSmith, Braintrust, Galileo) have converged on one judge-calibration pattern - small human-labeled reference sets (LangSmith: start at ~20 balanced examples; Braintrust: 50-100 per scoring dimension) with an 'alignment score' defined as RAW PERCENT AGREEMENT with the human expert, iterate the judge prompt against it, and route low-confidence or judge-disagreement items to humans - and none of them ships chance-corrected statistics.",
          "evidence": "Fetched LangSmith's official 'Improve LLM-as-judge evaluators using human feedback' doc: annotation queue -> reference dataset (>=20 examples, balanced 0/1 labels) -> Evaluator Playground -> alignment score = 'percentage of examples where the evaluator's judgment matches that of the human expert' -> cluster misalignments into failure modes and encode as prompt instructions. Fetched Braintrust's 2026-04-03 article: 3-tier stack (deterministic checks -> LLM judge -> human), 50-100 labels per dimension before trusting a judge, multi-judge disagreement routed to humans, worst failure mode named as 'silent overconfidence' (judge always returns a score even without adequate context), and explicit warning that human review is not automatically gold standard (fatigue, vague rubrics can be worse than a decent judge). Galileo's CLHF (2025-02-10 blog) auto-adapts metrics from expert corrections.",
          "in_corpus_verdict": "partially_confirmed"
        }
      ]
    },
    {
      "url": "https://arxiv.org/abs/2502.04313",
      "source_title": "Great Models Think Alike and this Undermines AI Oversight",
      "source_date": "2025-02-06",
      "source_type": "academic",
      "claims": [
        {
          "id": "CT-05",
          "domain": "contrarian",
          "claim": "LLM judges favor models whose mistakes resemble their own (a generalization of self-preference, measured by chance-adjusted mistake-overlap CAPA), and model errors across the industry are becoming MORE correlated as capabilities improve - undermining the assumption that AI oversight of AI-assisted work catches failures.",
          "evidence": "Goel et al., ICML 2025; abstract verified via arXiv crawl: 'LLM-as-a-judge scores favor models similar to the judge' and 'model mistakes are becoming more similar with increasing capabilities, pointing to risks from correlated failures'. Related: 'Preference Leakage' (arXiv 2502.01534) documents contamination when generator and judge are related.",
          "in_corpus_verdict": null
        }
      ]
    },
    {
      "url": "https://alignment.openai.com/coval",
      "source_title": "CoVal: Learning values-aware rubrics from the crowd (OpenAI Alignment blog)",
      "source_date": "2026-01-14",
      "source_type": "primary",
      "claims": [
        {
          "id": "F26-08",
          "domain": "frontier-2026",
          "claim": "OpenAI's CoVal (Jan 14, 2026) operationalizes 'disagreement is signal, not noise' at lab scale: crowd-written, prompt-specific rubrics (~1,000 participants, ~1,000 prompts) make WHY evaluators disagree auditable; rubric-derived scores reach only 0.58-0.61 pairwise concordance with crowd preference, are reliable only when score gaps are large, and OpenAI explicitly warns that treating rubric scores as optimization targets incentivizes checklist-style gaming.",
          "evidence": "Full blog post opened and read. CoVal-full preserves conflicting criteria as a record of legitimate disagreement; CoVal-core distills 4 mutually compatible criteria per prompt. OpenAI states lower alignment scores can indicate 'no single stable target to predict' rather than model failure, and that for split preferences any single aggregated score encodes a particular compromise. Dataset released in both forms.",
          "in_corpus_verdict": null
        }
      ]
    },
    {
      "url": "https://academic.oup.com/ilj/article/53/4/840/7745471",
      "source_title": "CJEU Case C-634/21 SCHUFA analysis + 'Scores as Decisions? Article 22 GDPR ... in the Labour Context', Industrial Law Journal",
      "source_date": "2024-09",
      "source_type": "academic",
      "claims": [
        {
          "id": "CG-11",
          "domain": "critic-and-gapfill",
          "claim": "Under GDPR Article 22 as interpreted in CJEU SCHUFA (C-634/21, 7 Dec 2023), an automated score that plays a 'determining role' in a downstream decision is itself an Article 22 decision, and human involvement only removes a decision from Art 22 scope if it is meaningful - a reviewer with real authority, data access, and competence to override; rubber-stamping does not count. Enforcement is active: Italy's DPA sanctioned automated rider deactivations, and Hamburg's DPA fined a company ~EUR 490,000 in September 2025 for automated rejections without adequate explanation of the logic.",
          "evidence": "SCHUFA three-condition test and 'determining role' doctrine confirmed across TLT, Bird & Bird, IAPP, and Oxford Industrial Law Journal ('Scores as Decisions?', 2024, applying it to the labour context and Uber litigation). EDPB-endorsed WP29 guidance holds that consent is generally an invalid basis in work contexts due to power imbalance, and that human involvement must be substantive. Hamburg fine and Italian rider cases reported in 2025 enforcement roundups; regulator attention to automated rejections continuing into mid-2026.",
          "in_corpus_verdict": null
        }
      ]
    },
    {
      "url": "https://www.annualreviews.org/content/journals/10.1146/annurev-orgpsych-110622-031927",
      "source_title": "A 25-Year Review of Research on Feedback in Organizations: From Simple Rules to Complex Realities (Anseel & Sherf)",
      "source_date": "2025-01-21",
      "source_type": "academic",
      "claims": [
        {
          "id": "CG-23",
          "domain": "critic-and-gapfill",
          "claim": "The 2025 Annual Review of Organizational Psychology's 25-year retrospective concludes the science of workplace feedback 'is not yet a story of coherent and cumulative progress': definitions are generic, assumptions diverge across six disconnected research substreams, and simple universal rules about feedback effectiveness do not survive contact with organizational reality.",
          "evidence": "Crawled the full open-access review (Anseel & Sherf, Annu. Rev. Organ. Psychol. Organ. Behav. 12:19-43, published 2025-01-21). Abstract states insights 'often appear disconnected from the way feedback is practiced and experienced in organizations'; the review calls for explicated assumptions and paradigms mirroring complex realities. The companion 2025 systematic review (Heine, Stouten & Liden, J. Organ. Behav., 2025-10-26) reaches the same verdict for supervisor performance feedback: most studies fail even to specify feedback valence, and feedback quality/accuracy findings rest on inconsistent constructs.",
          "in_corpus_verdict": null
        }
      ]
    }
  ]
}