{ "batch": 6, "checked_at": "2026-07-15", "sources": [ { "url": "https://arxiv.org/abs/2606.13685", "retrieval": "abstract", "fetched": ["https://arxiv.org/abs/2606.13685"], "resolved_title": "The Coin Flip Judge? Reliability and Bias in LLM-as-a-Judge Evaluation", "resolved_date": "2026-04-23", "title_match": true, "claims": [ { "id": "AJ-06", "verdict": "confirmed", "checked": [ "29 tasks spanning 10 categories", "50 pairwise trials (and 50 pointwise) per question", "pairwise preferences flip on average 13.6%", "28% of questions exceed a 20% flip rate (one hits 56%)", "equivalent templates change majority outcomes in 25% of tested cases", "11 repeated trials needed for majority vote to match 50-trial reference at 95%", "first-position bias 72% A-majority, p=0.024 (GPT-4o-mini)", "cross-judge agreement 76%, kappa 0.51", "judges: GPT-4o-mini and GPT-4.1-mini", "pairwise-pointwise gap: small non-significant scalar gaps despite winner picks" ], "not_visible": [ "the claim's own caveat that a 'Reliability-without-Validity cohort' shows some production judges reach test-retest >0.95 (external context, not from this paper)" ], "quote": "pairwise preferences flip on average 13.6% of the time ... 28% of questions exceeding a 20% flip rate ... two OpenAI judge models (GPT-4o-mini and GPT-4.1-mini) ... a significant first-position bias (72% A-majority, p = 0.024)", "notes": "Every headline number in the claim matches the abstract verbatim. Single-provider mini-tier caveat is stated by the paper (recommends cross-provider replication as future work)." } ] }, { "url": "https://doi.org/10.1016/0895-4356(90)90158-L", "retrieval": "landing", "fetched": ["https://doi.org/10.1016/0895-4356(90)90158-L", "https://linkinghub.elsevier.com/retrieve/pii/089543569090158L", "http://jclinepi.com/retrieve/pii/089543569090158L"], "resolved_title": "High agreement but low Kappa: I. the problems of two paradoxes", "resolved_date": "1990 (J Clin Epidemiol; exact date not shown)", "title_match": true, "claims": [ { "id": "AQ-06", "verdict": "partially_confirmed", "checked": [ "resolved article title confirms the core thesis: high agreement, low kappa, two paradoxes" ], "not_visible": [ "abstract-level mechanism: kappa near zero under extreme prevalence (prevalence-driven)", "second paradox: asymmetric marginal distributions inflating/affecting kappa", "the 'opposite direction from raw percent agreement' framing tied to QA pass rates", "external replications cited in the claim's evidence (Quarfoot & Levine 2016; Gwet AC1 literature) - not this source" ], "quote": "articleName : 'High agreement but low Kappa: I. the problems of two paradoxes'", "notes": "Server redirect chain doi.org -> linkinghub.elsevier.com -> jclinepi.com. Abstract body is behind a Cloudflare JS challenge (curl) and the Elsevier landing page returns only 'Redirecting'; a full WebFetch of the DOI/landing was flagged by a stochastic model safeguard. Title confirmed via the linkinghub siteCatalyst metadata, which locks the core high-agreement/low-kappa/two-paradoxes assertion; the specific paradox mechanisms could not be retrieved from the abstract text." } ] }, { "url": "https://arxiv.org/abs/2401.04518", "retrieval": "fulltext", "fetched": ["https://arxiv.org/abs/2401.04518", "https://arxiv.org/html/2401.04518"], "resolved_title": "The Critique of Critique", "resolved_date": "2024-01-09 (v2 2024-06-01; Findings of ACL 2024)", "title_match": true, "claims": [ { "id": "CM-07", "verdict": "confirmed", "checked": [ "critiques decomposed into Atomic Information Units (AIUs), precision/recall scored against references with NL rationales", "GPT-4 AIU-level accuracy 85-89% (Table 3: 89.12 / 87.96 precision task; 85.47 / 86.82 recall task; 'nearly 90%')", "Meta-F1 Pearson with human gold 0.841 (human-written) / 0.886 (LLM-generated)", "human critiques high-precision/low-recall 87.61 / 48.72", "LLM critiques 71.85 / 53.28", "refinement by Meta-F1: 51% Better Critique Wins / 22% Tie / 27% Loses (human eval)", "average AIUs 8.10 (LLM/Hypo.l) vs 3.31 (human/Hypo.h) = 2.4x", "non-factual AIUs ~28% (LLM) vs ~12% (human), derived from precision 71.85 / 87.61" ], "not_visible": [ "the specific comparative that GPT-4 pairwise picks 'lose more than they win' vs MetaCritique (a separate sub-figure not surfaced in retrieval)" ], "quote": "GPT-4 ... achieves an impressive performance (nearly 90%) ... MetaCritique-GPT4-F1 scores 0.841 ... 0.886 ... 51% Better Critique Wins, 22% Tie, 27% Loses ... Hypo.l = 8.10, Hypo.h = 3.31", "notes": "All headline numbers verified verbatim from arXiv HTML full text (Tables 1,3,4,6, Figure 4b). 8.10/3.31 = 2.45 confirms the '2.4x information volume' claim. Non-factual percentages are correctly derived from precision as the claim itself states." } ] }, { "url": "https://arxiv.org/abs/2501.03200", "retrieval": "fulltext", "fetched": ["https://arxiv.org/abs/2501.03200", "https://arxiv.org/html/2501.03200"], "resolved_title": "The FACTS Grounding Leaderboard: Benchmarking LLMs' Ability to Ground Responses to Long-Form Input", "resolved_date": "2025-01-06", "title_match": true, "claims": [ { "id": "GF-08", "verdict": "partially_confirmed", "checked": [ "v1 uses three judge models from three providers: Gemini 1.5 Pro (Google), GPT-4o (OpenAI), Claude 3.5 Sonnet (Anthropic)", "rationale is self-preference: 'models have been shown to be biased towards favorably judging their own outputs'", "aggregate of multiple judge models to mitigate evaluation bias", "v1 dated Jan 2025" ], "not_visible": [ "FACTS Grounding v2 (Dec 2025) two-family ensemble Gemini 2.5 Flash + GPT-5 (asserted from arXiv:2512.10791, a different paper not fetchable under the constraint)", "the DeepMind blog (Dec 17, 2024) v1 design write-up", "FACTS Parametric single-judge-preserves-rankings validation (Gemini 2.5 Pro, o3, Grok 4)" ], "quote": "We use three different judge models in order to reduce the bias of a particular judge model ... as models have been shown to be biased towards favorably judging their own outputs", "notes": "The v1 core (cross-provider ensemble as self-preference mitigation) is fully confirmed from the given URL's full text. Paper says 'three different judge models' from three providers rather than literally 'different families' - substantively equivalent. All v2 / Parametric specifics require arXiv:2512.10791 and external pages, which the fetch constraint excludes, so they are marked not_visible rather than checked." } ] }, { "url": "https://arxiv.org/abs/2511.21140", "retrieval": "fulltext", "fetched": ["https://arxiv.org/abs/2511.21140", "https://arxiv.org/html/2511.21140"], "resolved_title": "How to Correctly Report LLM-as-a-Judge Evaluations", "resolved_date": "2025-11-26 (v4 2026-05-31; ICML 2026)", "title_match": true, "claims": [ { "id": "HS-08", "verdict": "confirmed", "checked": [ "imperfect judge sensitivity/specificity biases naive evaluation scores", "plug-in framework corrects the bias", "confidence intervals propagate uncertainty from BOTH test dataset and human-labeled calibration dataset", "remains unbiased under distribution shift between test and calibration datasets", "adaptive strategy to allocate calibration samples for tighter intervals", "characterizes regimes (by true score, sensitivity, specificity) where judge-based eval beats human-only", "accepted ICML 2026; v1 2025-11-26, v4 2026-05-31" ], "not_visible": [], "quote": "imperfect sensitivity and specificity of the LLM judges induce bias in naive evaluation scores ... uncertainty from both the test dataset and a human-labeled calibration dataset ... remains unbiased under distribution shift between the test and calibration datasets", "notes": "Every core element of the claim is present in the abstract and confirmed. Distribution-shift robustness (vs standard PPI i.i.d.) is stated explicitly." }, { "id": "F26-09", "verdict": "partially_confirmed", "checked": [ "imperfect judge (sensitivity q1, specificity q0) biases naive pass rate", "direction: positive bias at low true rate, negative bias at high (overestimates below theta=0.75, underestimates above)", "Rogan-Gladen (1978) corrected estimator: theta_hat = (p_hat + q0_hat - 1)/(q0_hat + q1_hat - 1)", "CI propagating both test-set and calibration-set uncertainty", "adaptive allocation formula m0 ~ (1/p - 1)*sqrt(kappa)*m1 with kappa=(1-q0)/(1-q1)", "Monte Carlo: naive CI near-zero coverage; corrected CI holds ~95% across true accuracies", "released Python implementation github.com/UW-Madison-Lee-Lab/LLM-judge-reporting" ], "not_visible": [ "alphaXiv feature May 2026 and 'no 2026 successor supersedes it' (external, unverifiable from this source)" ], "quote": "achieving an interval shorter than 0.1 requires m approximately 200 calibration examples ... m_0 approximately (1/p - 1)*sqrt(kappa)*m_1 ... p_hat has positive bias at low values of theta and negative bias at high values of theta", "notes": "DISCREPANCY on one number: the claim says '~200 labels PER CLASS for CI width <0.1', but the paper's worked regime (p_hat=0.3, q0_hat=0.7, q1_hat=0.9) requires m approximately 200 calibration examples in TOTAL under symmetric allocation (~100 per class), not 200 per class. Every other specific (bias direction, q1/q0, Rogan-Gladen estimator, the m0 allocation formula, Monte Carlo coverage, GitHub repo) is confirmed verbatim from the full text." } ] }, { "url": "https://developers.openai.com/api/docs/guides/graders", "retrieval": "landing", "fetched": ["https://developers.openai.com/api/docs/guides/graders"], "resolved_title": "Graders (OpenAI API guide)", "resolved_date": "living doc; no publication date shown, accessed 2026-07-14", "title_match": true, "claims": [ { "id": "RR-09", "verdict": "confirmed", "checked": [ "grader taxonomy: string check, text similarity, score model (LLM judge, numeric), python (code execution), multigrader (combine/nest, reinforcement fine-tuning only)", "validate a model grader on its own eval built from expert-graded answers", "grader scores must reproduce the expert ranking (answer_1 > answer_2 > answer_3)", "reward/grader hacking signature: scores high on model grader evals but poorly on expert human evaluations", "guidance: smooth score not pass/fail; few-shot great/fair/poor examples; add edge cases over time; guard against reward hacking", "deprecation note: OpenAI deprecating graders within evals/fine-tuning workflows" ], "not_visible": [], "quote": "A model that's hacked the grader will score highly on model grader evals but score poorly on expert human evaluations ... Produce a smooth score, not a pass/fail stamp ... In reinforcement fine-tuning, you can nest and combine graders by using multigraders", "notes": "All prescribed elements of the claim are present verbatim in the live doc, including the named reward-hacking signature and the deprecation note." } ] }, { "url": "https://arxiv.org/abs/2602.18583", "retrieval": "abstract", "fetched": ["https://arxiv.org/abs/2602.18583"], "resolved_title": "Luna-2: Scalable Single-Token Evaluation with Small Language Models", "resolved_date": "2026-02-20", "title_match": true, "claims": [ { "id": "IP-09", "verdict": "partially_confirmed", "checked": [ "Luna-2: single-token SLM evaluation vs multi-token LLM-as-judge", "each metric = a lightweight LoRA/PEFT head on a shared SLM backbone (hundreds concurrently)", "cost reduced by over 80x, latency by over 20x", "matches accuracy of state-of-the-art LLM-based evaluators (parity)", "production scale: 100M+ AI sessions, 100B+ tokens/month", "eval cost savings of over $30M annually" ], "not_visible": [ "all Patronus Lynx claims: Lynx-70B beats GPT-4o on HaluBench (+8.3% PubMedQA), Lynx-8B +24.5% vs GPT-3.5", "Algomo customer 0.375 -> 0.69 hallucination detection", "Gamma 1,000+ manual eval hours saved; Percival tracing", "SiliconANGLE / Patronus launch-post sourcing (separate URLs, not fetched)" ], "quote": "matches the accuracy of state-of-the-art LLM-based evaluators ... by over 80x ... by over 20x ... protecting 100M+ AI sessions ... eval cost savings of over $30M annually", "notes": "The Luna-2 half of the claim is fully confirmed from the given arXiv abstract. The Patronus Lynx half is sourced to Patronus/SiliconANGLE pages that are not the given URL and thus not verifiable under the fetch constraint; those specifics are marked not_visible, not checked." } ] }, { "url": "https://mackinstitute.wharton.upenn.edu/2025/new-in-nature-chatgpt-decreases-idea-diversity-in-brainstorming/", "retrieval": "landing", "fetched": ["https://mackinstitute.wharton.upenn.edu/2025/new-in-nature-chatgpt-decreases-idea-diversity-in-brainstorming/"], "resolved_title": "New in Nature: ChatGPT Decreases Idea Diversity in Brainstorming", "resolved_date": "2025-05-14", "title_match": true, "claims": [ { "id": "CT-12", "verdict": "partially_confirmed", "checked": [ "94% of ChatGPT-assisted participants' ideas shared overlapping concepts", "nine participants independently produced the same product name ('Build-a-Breeze Castle')", "human-only ideas were entirely unique", "individual quality up while population diversity down (ChatGPT enhances individual creativity but significantly reduces idea diversity)", "Meincke, Nave & Terwiesch, Nature Human Behaviour 2025" ], "not_visible": [ "'Artificial Hivemind' ~81% average similarity between DeepSeek-V3 and GPT-4o (Jiang et al., reported by The Decoder)", "Doshi & Hauser (Science Advances 2024) and Wan et al. 2026 corroboration" ], "quote": "94% of ideas shared overlapping concepts ... nine participants independently naming their toy 'Build-a-Breeze Castle' ... human-generated ideas were entirely unique ... while ChatGPT can enhance the creativity of individual ideas, it significantly reduces the diversity of ideas", "notes": "The Nature Human Behaviour brainstorming portion is fully confirmed from the Wharton summary. The inter-model 'Artificial Hivemind' 81% figure and the Doshi & Hauser / Wan et al. citations are from other sources not on this page, so marked not_visible." } ] }, { "url": "https://doi.org/10.1111/j.2044-8325.2011.02045.x", "retrieval": "failed", "fetched": ["https://doi.org/10.1111/j.2044-8325.2011.02045.x", "https://bpspsychub.onlinelibrary.wiley.com/doi/10.1111/j.2044-8325.2011.02045.x"], "resolved_title": "not retrievable (DOI resolves to Wiley/BPS article page; content blocked)", "resolved_date": "not retrievable", "title_match": false, "claims": [ { "id": "CG-03", "verdict": "not_retrievable", "checked": [], "not_visible": [ "overall FOR-training effect size Cohen's d approximately 0.50 on rating accuracy", "comparison to Woehr & Huffcutt 1994 d approximately 0.83", "~4x the studies of the earlier meta-analysis", "differential accuracy and behavioural accuracy most improved", "resolved title / authors / journal / year" ], "quote": "", "notes": "The DOI issues a server 302 to bpspsychub.onlinelibrary.wiley.com (correct Wiley/BPS host for J. Occup. Organ. Psychol.), confirming the DOI resolves to the intended article. But WebFetch returned HTTP 402 Payment Required and curl hit a Cloudflare 'Just a moment... Enable JavaScript' challenge, so no title, abstract, or numbers could be retrieved. None of the claim's figures (d=0.50, d=0.83, 4x studies) are verifiable from allowed sources." } ] }, { "url": "https://arxiv.org/abs/2306.07466", "retrieval": "pdf", "fetched": ["https://arxiv.org/abs/2306.07466", "https://arxiv.org/pdf/2306.07466"], "resolved_title": "Statistical Methods for Auditing the Quality of Manual Content Reviews", "resolved_date": "2023-06-12 (Published as a Tiny Paper at ICLR 2023)", "title_match": true, "claims": [ { "id": "CG-17", "verdict": "confirmed", "checked": [ "authors Xuan Yang, Andrew Smart, Daniel Theron; affiliation Google LLC", "Published as a Tiny Paper at ICLR 2023", "per-question Fleiss kappa locates the ambiguous criterion", "Table 1: Question 3 Fleiss kappa 0.093997 vs Overall Fleiss Kappa 0.475072", "Question 3 flagged as systematically higher disagreement", "sequential non-blind review (one reviewer sees previous reviewers' results) named as audit risk", "testable via A/B test or Difference-in-Difference", "synthetic dataset: three reviewers, nine-question rubric, 1,528 products", "chi-square for criterion-verdict linkage; t-test/ANOVA for deviating review teams", "binomial-distribution confidence interval of the error rate", "code at github.com/xuanyang0607/openreviewpaper" ], "not_visible": [], "quote": "Table 1 ... Question 3 0.093997 ... Overall Fleiss Kappa 0.475072 ... synthetic data set consisting of three reviewers and their answers regarding a nine question rubric on 1,528 products ... reviewers conduct reviews sequentially ... A/B test ... Difference-in-Difference", "notes": "Extracted full PDF text via pdftotext (WebFetch saved the binary; local extraction succeeded). Every element of the claim confirmed verbatim: 0.094 vs 0.475 (0.093997 / 0.475072), Google LLC, ICLR 2023 Tiny Paper, sequential-review/A-B/DiD, dataset dimensions, chi-square/t-test/ANOVA, binomial CI (line 283), and the GitHub URL (line 182)." } ] } ] }