{
  "batch": 3,
  "checked_at": "2026-07-15",
  "sources": [
    {
      "url": "https://arxiv.org/abs/2606.19544",
      "retrieval": "fulltext",
      "fetched": [
        "https://arxiv.org/abs/2606.19544",
        "https://arxiv.org/html/2606.19544v1"
      ],
      "resolved_title": "Reliability without Validity: A Systematic, Large-Scale Evaluation of LLM-as-a-Judge Models Across Agreement, Consistency, and Bias",
      "resolved_date": "2026-06-17",
      "title_match": true,
      "claims": [
        {
          "id": "AJ-03",
          "verdict": "confirmed",
          "checked": [
            "21 judges from nine providers",
            "approximately 541,000 individual judgments",
            "118 runs",
            "kappa deflation ... universal (33--41 pp on MT-Bench)",
            "judge rankings shift by up to 14 positions across benchmarks",
            "high test--retest reliability (>0.95) with severe position bias (>0.10) in two production-deployed judges",
            "verbosity bias is small (<0.011)",
            "consistency--bias paradox",
            "Minimum Viable Validation Protocol",
            "April 2026 frontier"
          ],
          "not_visible": [],
          "quote": "kappa deflation between exact match and Cohen's kappa is universal (33--41 pp on MT-Bench) ... judge rankings shift by up to 14 positions across benchmarks",
          "notes": "All four cohort-wide findings and every headline number verified in the abstract; verbosity bias <0.011, MVVP, and consistency-bias paradox all present verbatim."
        },
        {
          "id": "CM-09",
          "verdict": "confirmed",
          "checked": [
            "exact match overstates chance-corrected agreement by between 33.8 and 41.3 percentage points across the 21 models",
            "a judge reporting '85% agreement' on MT-Bench has kappa approximately 0.48",
            "Llama 3.3 70B which shifts 15 positions, from 5 on MT-Bench to 20 on JudgeBench",
            "Qwen 3 8B (test-retest 0.992, position bias 0.192)",
            "Gemini 2.5 Flash (test-retest 0.988, position bias 0.125)",
            "All evaluations used temperature 0",
            "UC Berkeley School of Information",
            "118 runs",
            "MVVP: report Cohen's kappa or Krippendorff's alpha alongside exact-match, AB+BA position swaps, >=3 replicates, >=2 benchmarks spanning preference- and correctness-style labels, audit that high stability is not high bias"
          ],
          "not_visible": [
            "'March 2026' as an eval start month (page states results hold across 'the April 2026 frontier'; March not explicitly seen)"
          ],
          "quote": "exact match overstates chance-corrected agreement by between 33.8 and 41.3 percentage points across the 21 models",
          "notes": "Every precise figure the claim attributes to the HTML full text is present verbatim. The claim's '15 positions' matches Section 4.3 ('shifts 15 positions'), though the abstract and Figure 2 say 14 positions -- the paper is internally inconsistent on this number. Claim's rounded 0.99/0.19 corresponds to the paper's 0.992/0.192 for Qwen 3 8B."
        },
        {
          "id": "CT-01",
          "verdict": "confirmed",
          "checked": [
            "21 judges from nine providers",
            "approximately 541,000 individual judgments",
            "kappa deflation ... 33--41 pp on MT-Bench",
            "judge rankings shift by up to 14 positions across benchmarks",
            "test--retest reliability (>0.95) with severe position bias (>0.10) in two production-deployed judges",
            "consistency--bias paradox",
            "118 runs across MT-Bench, JudgeBench, RewardBench",
            "three protocols (agreement, consistency, bias)",
            "Minimum Viable Validation Protocol",
            "April 2026 frontier"
          ],
          "not_visible": [
            "corroborating paper arXiv 2508.18076 (Chehbouni et al., 'Neither Valid nor Reliable?') -- a separate source outside the fetched URL; not verifiable under fetch constraints"
          ],
          "quote": "we evaluate 21 judges from nine providers across MT-Bench, JudgeBench, and RewardBench ... 118 runs and approximately 541,000 individual judgments",
          "notes": "Core assertion and all headline numbers confirmed. The claim's secondary corroborating citation (2508.18076) is a different paper and was not fetched per the single-URL constraint."
        }
      ]
    },
    {
      "url": "https://arxiv.org/abs/2410.03775",
      "retrieval": "abstract",
      "fetched": [
        "https://arxiv.org/abs/2410.03775"
      ],
      "resolved_title": "Beyond correlation: The Impact of Human Uncertainty in Measuring the Effectiveness of Automatic Evaluation and LLM-as-a-Judge",
      "resolved_date": "2024-10-03 (v1); v3 2025-01-27; Accepted at ICLR 2025",
      "title_match": true,
      "claims": [
        {
          "id": "AQ-03",
          "verdict": "confirmed",
          "checked": [
            "a single aggregate correlation score can obscure fundamental differences between human labels and those from automatic evaluation",
            "high uncertain-label proportion: machine labels may superficially appear to have similar or better correlation with the human majority label (an illusion)",
            "as consistent-label samples increase, correlation between machine and human labels fall well below HH correlation",
            "propose stratifying data by human label uncertainty",
            "binned Jensen-Shannon Divergence for perception",
            "perception charts",
            "code open-sourced at github.com/amazon-science/BeyondCorrelation",
            "Accepted at ICLR 2025"
          ],
          "not_visible": [
            "explicit 'Amazon' author affiliation -- page lists no affiliations; Amazon is only implied by the 'amazon-science' GitHub org name"
          ],
          "quote": "we propose ... a new metric - binned Jensen-Shannon Divergence for perception ... perception charts, to contextualize correlation measures appropriately",
          "notes": "Core claim and all three prescribed fixes (stratify by uncertainty, binned JS divergence, perception charts) confirmed verbatim. Only the 'Amazon' affiliation label is not explicitly on the page."
        }
      ]
    },
    {
      "url": "https://arxiv.org/abs/2405.06087",
      "retrieval": "abstract",
      "fetched": [
        "https://arxiv.org/abs/2405.06087"
      ],
      "resolved_title": "When combinations of humans and AI are useful: A systematic review and meta-analysis",
      "resolved_date": "2024-05-09 (v1); v2 2024-10-29; Nat Hum Behav (2024)",
      "title_match": true,
      "claims": [
        {
          "id": "CM-04",
          "verdict": "confirmed",
          "checked": [
            "meta-analysis of over 100 recent experimental studies reporting over 300 effect sizes",
            "human-AI combinations performed significantly worse than the best of humans or AI alone",
            "performance losses in tasks that involved making decisions",
            "significantly greater gains in tasks that involved creating content",
            "when humans outperformed AI alone, we found performance gains in the combination",
            "Nat Hum Behav (2024) / Nature Human Behaviour"
          ],
          "not_visible": [
            "exact counts 106 studies / 370 effect sizes (page says 'over 100' and 'over 300')",
            "DeepMind complementarity paper framing this result as its motivation -- an external editorial connection, not on this page"
          ],
          "quote": "human-AI combinations performed significantly worse than the best of humans or AI alone ... when humans outperformed AI alone, we found performance gains in the combination",
          "notes": "Nature Human Behaviour venue, 100+ studies / 300+ effect sizes, decision-task losses, creation-task gains, and the humans-beat-AI condition all confirmed verbatim."
        }
      ]
    },
    {
      "url": "https://arxiv.org/abs/2406.19276",
      "retrieval": "abstract",
      "fetched": [
        "https://arxiv.org/abs/2406.19276"
      ],
      "resolved_title": "VERISCORE: Evaluating the factuality of verifiable claims in long-form text generation",
      "resolved_date": "2024-06-27",
      "title_match": false,
      "claims": [
        {
          "id": "GF-05",
          "verdict": "partially_confirmed",
          "checked": [
            "prior methods FActScore/SAFE 'assume that every claim is verifiable (i.e., can plausibly be proven true or false)'",
            "VERISCORE handles 'both verifiable and unverifiable content'",
            "human evaluation confirms that VERISCORE's extracted claims are more sensible than those from competing methods",
            "across eight different long-form tasks"
          ],
          "not_visible": [
            "FactBench and its VERIFY pipeline -- the words 'FactBench' and 'VERIFY' do not appear on this page",
            "three-way supported/unsupported/UNDECIDABLE labeling against retrieval",
            "4,467 human-annotated content units",
            "ACL 2025 venue for FactBench",
            "'EMNLP Findings 2024' venue -- the arXiv page shows no conference venue, only arXiv"
          ],
          "quote": "they assume that every claim is verifiable (i.e., can plausibly be proven true or false) ... human evaluation confirms that VERISCORE's extracted claims are more sensible than those from competing methods",
          "notes": "The VeriScore half of this compound claim is fully confirmed. The FactBench/VERIFY half (three-way labeling, 4,467 units, ACL 2025) comes from a separate paper (2410.22257) that the single-URL constraint prohibits fetching, so it is not present in retrieved text. Minor: claim says VeriScore scores 'ONLY verifiable claims' while the abstract frames it as handling 'both verifiable and unverifiable content'. title_match=false because source_title is a composite two-paper label, not the actual paper title, and asserts an EMNLP venue not shown on the page."
        }
      ]
    },
    {
      "url": "https://arxiv.org/html/2605.31278v2",
      "retrieval": "fulltext",
      "fetched": [
        "https://arxiv.org/html/2605.31278v2"
      ],
      "resolved_title": "Industrializing Prediction-Powered Inference: The GLIDE Library for Reliable GenAI and Agentic Systems Evaluation",
      "resolved_date": "2026-06-04 (v2)",
      "title_match": true,
      "claims": [
        {
          "id": "HS-05",
          "verdict": "confirmed",
          "checked": [
            "unifies state-of-the-art PPI estimators (PPI++, Stratified PPI, Predict-Then-Debias and its stratified variants, Active Statistical Inference)",
            "an empirically grounded decision tree for method selection",
            "effective sample size grows from ~500 at rho=0.1 to ~1100 at rho=0.9 (a 2.2x effective gain), from 500 human + 1000 proxy labels",
            "at least fifty labeled samples per stratum for the asymptotic intervals to be reliable",
            "R-Judge case study; proxy is claude-sonnet-4-6 run as zero-shot LLM-as-judge",
            "Pearson correlation with expert labels rho approximately 0.59",
            "overshoots the true rate by about 13 percentage points",
            "statistically equivalent to roughly 157, 148, and 143 purely human-labeled trajectories",
            "https://github.com/EmertonData/glide",
            "limitations: mean estimation only, single proxy / i.i.d. assumption, does not provide anytime-valid constructions"
          ],
          "not_visible": [],
          "quote": "the effective sample size grows from approximately 500 at rho=0.1 to approximately 1100 at rho=0.9 ... a 2.2x effective gain at no cost to validity",
          "notes": "Every headline number confirmed verbatim in the HTML full text: the ~1.0x-to-2.2x effective-sample gain, >=50 labels/stratum, and the R-Judge case (rho~0.59, ~13pp bias, 100 labels worth 143-157). Authors Martinon/Merad/Raki, Emerton Data affiliation, and the GitHub repo all present."
        }
      ]
    },
    {
      "url": "https://arxiv.org/html/2604.01375v2",
      "retrieval": "fulltext",
      "fetched": [
        "https://arxiv.org/html/2604.01375v2"
      ],
      "resolved_title": "RIFT: A RubrIc Failure Mode Taxonomy and Automated Diagnostics",
      "resolved_date": "2026-04-20 (v2)",
      "title_match": true,
      "claims": [
        {
          "id": "RR-06",
          "verdict": "confirmed",
          "checked": [
            "8 modes in 3 categories: Subjective, Non-Atomic, Ungrounded (Reliability); Misaligned/Rigid, Missing Criteria (Content Validity); Hackable, Low Signal, Redundant Criteria (Consequential)",
            "Pearson's r = 0.162, p = 0.0021",
            "Table 2: Subjective Human 52.6% vs Synthetic 86.7%",
            "Table 2: Misaligned/Rigid Human 63.2% vs Synthetic 20.0%",
            "Table 3: Subjective F1 0.925; Hackable F1 0.000",
            "85 rubrics with 255 expert annotations drawn from five data sources",
            "AdvancedIF, ResearchRubrics, WildChecklists, OpenRubrics, and AutoRubrics",
            "developed using grounded theory",
            "0.64 average Cohen's kappa",
            "classifiers GPT-5.2 and Gemini 3 Pro"
          ],
          "not_visible": [],
          "quote": "Pearson's r = 0.162, p = 0.0021 ... we analyze 85 rubrics with 255 expert annotations drawn from five data sources",
          "notes": "All 8 modes/3 categories, both correlation stats, both Table 2 prevalence pairs, both Table 3 F1 extremes, the 85/255 counts, five sources, grounded theory, kappa 0.64, and both classifier models confirmed. Minor internal inconsistency in the paper: abstract says 'up to 0.925 F1' while a contributions bullet says 'up to 0.86 F1'; the claim uses the 0.925 (Table 3) figure."
        }
      ]
    },
    {
      "url": "https://sacra.com/c/surge-ai",
      "retrieval": "landing",
      "fetched": [
        "https://sacra.com/c/surge-ai"
      ],
      "resolved_title": "Surge AI revenue, funding & news | Sacra",
      "resolved_date": "not shown on page",
      "title_match": true,
      "claims": [
        {
          "id": "IP-06",
          "verdict": "confirmed",
          "checked": [
            "Sacra estimates that Surge AI hit $1.2B in annualized revenue in 2024",
            "approximately 50,000 expert contractors and 130 full-time employees",
            "bootstrapped company",
            "Real-time dashboards monitor annotation quality using ... gold-standard accuracy, inter-annotator agreement ... and per-worker trust ratings",
            "Labels identified as low quality are automatically reassigned to other annotators",
            "AdvancedIF ... in partnership with Meta Superintelligence Labs",
            "its verifier achieved 0.728 F1 versus human judgments",
            "using human-written rubrics as RL reward signals yields 13% performance gains",
            "premium rates of 30-40 cents per working minute",
            "rigorous vetting, including domain-specific tests, background checks, and ongoing performance evaluations",
            "Hemingway-bench ... built from 5,000+ blind pairwise comparisons by expert human judges",
            "reliance on 12 customers for over $1 billion in revenue",
            "class action lawsuit filed in California ... alleges Surge misclassified data annotators"
          ],
          "not_visible": [
            "source_date 2026-04-21 -- no last-updated date is shown on the page",
            "the lawsuit's specific allegations of 'unpaid training time and tight task timers' (not surfaced in retrieved text)"
          ],
          "quote": "Real-time dashboards monitor annotation quality using metrics such as gold-standard accuracy, inter-annotator agreement ... per-worker trust ratings ... Labels identified as low quality are automatically reassigned to other annotators",
          "notes": "All revenue/contractor/QA/AdvancedIF figures confirmed. Minor: page states '13% performance gains' (no literal '+' sign). Figures are explicitly Sacra estimates. Page shows no date, so the 2026-04-21 source_date could not be corroborated."
        }
      ]
    },
    {
      "url": "https://arxiv.org/abs/2503.05061",
      "retrieval": "abstract",
      "fetched": [
        "https://arxiv.org/abs/2503.05061"
      ],
      "resolved_title": "No Free Labels: Limitations of LLM-as-a-Judge Without Human Grounding",
      "resolved_date": "2025-03-07 (v1); last revised 2026-04-01 (v2)",
      "title_match": true,
      "claims": [
        {
          "id": "CT-08",
          "verdict": "partially_confirmed",
          "checked": [
            "judges show high agreement with human experts only on questions the judges were able to correctly answer themselves",
            "providing the judges with expert-written references largely mitigates this issue"
          ],
          "not_visible": [
            "the 'poor signal on the hardest / most difficult items in a benchmark' framing -- page frames the limitation around questions the judge cannot answer, not a difficulty ranking",
            "64% (mental health) and 68% (dietetics) SME agreement -- these are attributed in the claim to Szymanski et al. (ACM IUI 2025), a separate source; '64%', '68%', 'Szymanski', 'dietetics', 'mental health' are all absent from this page",
            "'reduce self-preference / self-bias' -- the page never mentions self-preference or self-bias"
          ],
          "quote": "judges show high agreement with human experts only on questions the judges were able to correctly answer themselves ... providing the judges with expert-written references largely mitigates this issue",
          "notes": "The portion of the claim sourced to 2503.05061 (judges unreliable on questions they can't answer; expert references improve agreement) is confirmed. The 64/68% SME figures belong to a different citation (Szymanski et al.) outside the fetched URL, and the 'reduce self-preference' specific is not stated on this page. Abstract centers on BFF-Bench (160 questions) and the VERDICTS annotation set."
        }
      ]
    },
    {
      "url": "https://arxiv.org/html/2603.00077v1",
      "retrieval": "fulltext",
      "fetched": [
        "https://arxiv.org/html/2603.00077v1"
      ],
      "resolved_title": "Autorubric: A Unified Framework for Rubric-Based LLM Evaluation",
      "resolved_date": "not shown in fetched HTML",
      "title_match": true,
      "claims": [
        {
          "id": "F26-12",
          "verdict": "confirmed",
          "checked": [
            "Autorubric, an open-source Python library",
            "per-criterion atomic evaluation with natural language explanations",
            "binary, ordinal, and nominal criteria with configurable weights",
            "multi-judge ensemble evaluation with majority, weighted, unanimous, and any-vote aggregation",
            "few-shot calibration with verdict-balanced sampling",
            "mitigations for position bias (option shuffling)",
            "verbosity bias (length penalties)",
            "reduced criterion conflation (independent scoring prevents halo effects)",
            "three benchmarks spanning educational assessment, deep research evaluation, and chatbot quality",
            "CHARM-100, a 100-sample chatbot evaluation dataset with per-sample ground truth labels"
          ],
          "not_visible": [
            "An AI data vendor's banking meta-eval using Autorubric as the representative text-only rubric-judge family -- the vendor and its meta-eval are not named on this page (external editorial claim)",
            "submission date -- no date present in the fetched HTML"
          ],
          "quote": "Autorubric supports binary, ordinal, and nominal criteria with configurable weights ... single-judge and multi-judge ensemble evaluation with majority, weighted, unanimous, and any-vote aggregation ... CHARM-100, a 100-sample chatbot evaluation dataset",
          "notes": "Every feature the claim attributes to Autorubric, plus the three-benchmark validation and the contributed CHARM-100 dataset, confirmed verbatim. The vendor meta-eval 'measured ceiling' connection is an external editorial addition not present in this paper."
        }
      ]
    },
    {
      "url": "https://www.tspa.org/curriculum/ts-fundamentals/content-moderation-and-operations/content-moderation-quality-assurance/",
      "retrieval": "landing",
      "fetched": [
        "https://www.tspa.org/curriculum/ts-fundamentals/content-moderation-and-operations/content-moderation-quality-assurance/"
      ],
      "resolved_title": "Content Moderation Quality Assurance (TSPA T&S Curriculum)",
      "resolved_date": "not shown on page",
      "title_match": true,
      "claims": [
        {
          "id": "CG-14",
          "verdict": "partially_confirmed",
          "checked": [
            "reviews by a dedicated quality team (and peer reviews); Samples are often weighted",
            "Reverse quality sampling involves taking a set of pre-reviewed examples ... sending them through the regular review process",
            "Examples must look exactly like regular reviews to be effective",
            "appeals surface false positives in particular at a low cost",
            "the population of appeals is often different from the population of all decisions",
            "four-way error taxonomy: False positives, False negatives, Wrong selection, Technical errors"
          ],
          "not_visible": [
            "overturn rate being 'a difficult metric to interpret' -- absent from this QA page (on the separate 'Metrics for Content Moderation' page, which the fetch constraint prohibits)",
            "'consistency' (multi-review mismatch rate) as a distinct tracked metric -- absent from this QA page",
            "prevalence measurement called a 'premium metric' -- absent from this QA page"
          ],
          "quote": "Reverse quality sampling involves taking a set of pre-reviewed examples ... Examples must look exactly like regular reviews to be effective",
          "notes": "The two QA channels, the seeded-item realism requirement, the appeals cost/population-bias points, and the four-way error taxonomy are all confirmed on the fetched QA page. Discrepancy: the claim quotes golden seeding as giving 'full control', but the page actually says 'complete control' (same meaning, different word). The three overturn/consistency/premium-metric items are attributed to a separate 'Metrics for Content Moderation' page (a different URL) and are not present on this page; the single-URL constraint bars fetching it."
        }
      ]
    }
  ]
}