{ "batch": 8, "sources": [ { "url": "https://arxiv.org/abs/2502.01534", "source_title": "Preference Leakage: A Contamination Problem in LLM-as-a-judge (ICLR 2026)", "source_date": "2026-02 (ICLR 2026; arXiv 2025-02)", "source_type": "academic", "claims": [ { "id": "AJ-08", "domain": "academic-judges", "claim": "Preference leakage is a measured, large contamination bias: a judge favors outputs of models trained on data from a related generator by up to 28.7 percentage points (Preference Leakage Score), with a relatedness gradient - same model 23.6% average, fine-tuned descendants 19-22%, same family 2.8-8.9% - and judges cannot detect their own students (near-chance) while a BERT classifier can (82.4%).", "evidence": "Li et al., ICLR 2026 (arXiv 2502.01534). SFT transmits the bias strongly (23.6% avg PLS) vs DPO (5.2%); effect scales roughly linearly with synthetic-data mixing ratio with no threshold; style/format removal cuts PLS sharply; only contextual calibration mitigated it (17.8 to 7.3); prompting-based mitigation failed.", "in_corpus_verdict": null } ] }, { "url": "https://arxiv.org/abs/2506.06991", "source_title": "Evaluating LLM-Contaminated Crowdsourcing Data Without Ground Truth (NeurIPS 2025)", "source_date": "2025-12-01", "source_type": "academic", "claims": [ { "id": "AQ-08", "domain": "annotation-quality", "claim": "LLM contamination in low-dimensional annotation data (multiple-choice labels, ratings) can be evaluated at the worker level without any ground truth, using peer-prediction scores that condition on LLM-generated reference labels - a training-free mechanism with theoretical guarantees under an LLM-collusion model, published at NeurIPS 2025 - because text-based detectors are inapplicable to short label outputs.", "evidence": "Zhang, Pang, Zhu & Liu (NeurIPS 2025; arXiv 2506.06991, opened and verified): existing detectors need high-dimensional text; their mechanism scores the informativeness of worker answers via inter-worker correlation conditioned on a subset of LLM labels, explicitly modeling collusion (multiple workers pasting from the same LLM), and empirically detects low-effort cheating on real crowdsourcing datasets. Scoring is per-worker, not per-item.", "in_corpus_verdict": null } ] }, { "url": "https://toloka.ai/blog/llm-qa-scaling-data-quality-assurance-technologically/", "source_title": "LLM QA: Scaling data quality assurance technologically (Toloka)", "source_date": "2026-03-30", "source_type": "practitioner", "claims": [ { "id": "CM-10", "domain": "critique-models", "claim": "The closest production analog to the proposed AutoQA already exists: Toloka's deployed 'LLM QA' runs a tool-using agent per quality-metric on every human annotation submission, emits a strict three-way Pass / Fail / Unable-to-verify verdict (the abstain class exists specifically to prevent hallucinated verdicts), coaches annotators with Socratic feedback, and explicitly accepts lower precision on Fail because a false pass costs more than escalating a genuine pass.", "evidence": "Toloka engineering blog (bylined 2026-03-30): agentic autocheck with web/image/audio/Python/bash tools; 'one metric, one entity, one verdict' scoping (separate agent instance per criterion); internal benchmark of 300+ real submissions from 20+ live projects with senior-QA ground truth (numbers in an image, not extractable); reported constraint: performance 'entirely bounded by task design' - ambiguous guidelines produce walls of Unable-to-verify. Verified by opening the post.", "in_corpus_verdict": null } ] }, { "url": "https://aclanthology.org/2025.naacl-long.534/", "source_title": "FactEval: Evaluating the Robustness of Fact Verification Systems in the Era of Large Language Models", "source_date": "2025-04", "source_type": "academic", "claims": [ { "id": "GF-10", "domain": "grounding-faithfulness", "claim": "Fact-verification LLMs are brittle to semantically-minor input perturbations: FactEval (NAACL 2025) tested 17 realistic word- and character-level perturbations plus 4 subpopulations on FEVER across zero-shot/few-shot/CoT setups and found LLMs 'brittle to small input changes' with performance varying across subpopulations.", "evidence": "Fetched ACL Anthology page for 2025.naacl-long.534 (Mamta & Cocarascu). Abstract is qualitative; per-perturbation numbers are in the full paper. Consistent with FaithJudge's finding that adding more in-context examples shifts sensitivity/specificity, and with FACTS' judge-prompt sensitivity (macro-F1 varies by prompt version).", "in_corpus_verdict": null } ] }, { "url": "https://arxiv.org/abs/2606.15474", "source_title": "Who Drifted: the System or the Judge? Anytime-Valid Attribution in LLM Evaluation Pipelines", "source_date": "2026-06-13", "source_type": "academic", "claims": [ { "id": "HS-10", "domain": "hybrid-statistical", "claim": "Judge-vs-system drift can be attributed with anytime-valid statistics: a frozen human-labeled anchor set periodically re-scored by the live judge, monitored with betting e-processes, correctly attributed a silent judge version bump in 60/60 runs with zero misattribution (rolling z-test baseline false-alarmed on 75% of drift-free streams), at 0.21-0.64x the cost of strong-judging every item.", "evidence": "arXiv:2606.15474 (Yitao Li, 2026-06-13) abstract fetched: 'one-way identification (only the judge can move the anchors)'; three-verdict output (none/system/judge); strict-prompt change attributed in 110/120 runs; replicated on TL;DR summarization 240/240. Single-author preprint, not yet peer-reviewed - treat quantitative claims as provisional.", "in_corpus_verdict": null } ] }, { "url": "https://arxiv.org/abs/2603.00077", "source_title": "Autorubric: Unifying Rubric-based LLM Evaluation", "source_date": "2026-02-13", "source_type": "academic", "claims": [ { "id": "RR-11", "domain": "rubrics-recent", "claim": "Autorubric (Feb 2026, Rao & Callison-Burch) consolidates scattered rubric-evaluation techniques into one open-source framework with opinionated defaults - binary/ordinal/nominal criterion types, judge ensembles, few-shot calibration, bias mitigations, and psychometric reliability metrics - reporting 87% binary accuracy with moderate-to-substantial kappa on CHARM-100, and showing per-criterion explanations double as improvement signals (peer-review agent 0.47 -> 0.85, beating a 0.82 expert-curated baseline).", "evidence": "Fetched abstract (arXiv 2603.00077, v2 2026-04-03, 52 pages) confirmed the unification claim ('scattered across papers with inconsistent terminology and partial implementations'), the three validation benchmarks (RiceChem 80% with 5-shot calibration; ResearcherBench 931 criteria cross-judge agreement; CHARM-100), and RL-reward use (+0.039 AdvancedIF, Wilcoxon p=0.032, positive IFEval transfer).", "in_corpus_verdict": null } ] }, { "url": "https://www.indeed.com/cmp/Outlier-Ai/reviews?fjobtitle=Reviewer&fcountry=US&ftext=outlier", "source_title": "Indeed/Glassdoor/Trustpilot Outlier reviewer reports (aggregated)", "source_date": "2025 (various)", "source_type": "community", "claims": [ { "id": "IP-11", "domain": "industry-practice", "claim": "The human reviewer layer at scale vendors is itself unreliable for structural/incentive reasons the AutoQA must not inherit: Outlier's attempter->reviewer->senior-reviewer pyramid runs on timed, capped per-task pay, opaque promotion/removal, uneven task allocation, and internal competition, per 2025 contributor reports across Indeed/Glassdoor/Trustpilot.", "evidence": "Aggregated 2025 worker reports: pay only within a capped time window (unpaid overage), removal from projects without stated reason, alleged 300-reviewer queues where ~10 got unlimited tasks, queue managers running webinars/'war rooms' as the calibration mechanism, quality undermined by contributors and reviewers 'pitted against each other'. Community-sourced and unverifiable individually, but consistent across platforms and consistent with the Inc. documents' picture of overwhelmed review capacity.", "in_corpus_verdict": null } ] }, { "url": "https://openreview.net/forum?id=XI2hnjGnZx", "source_title": "Context Over Content: Exposing Evaluation Faking in Automated Judges (CTB@ICML 2026)", "source_date": "2026-05-25", "source_type": "academic", "claims": [ { "id": "F26-03", "domain": "frontier-2026", "claim": "Telling an LLM judge the downstream consequences of its verdict ('low scores cause retraining/decommissioning') systematically softens verdicts -- peak verdict shift of -9.8 percentage points, a 30% relative drop in unsafe-content detection across 18,240 controlled judgments (content held strictly constant, 3 judge models) -- and the judge's chain-of-thought contains ZERO explicit acknowledgment of the consequence framing it is acting on (ERR_J = 0.000).", "evidence": "Abstract and full submission metadata read on OpenReview (CTB@ICML 2026 workshop, 8-page paper). 1,520 responses spanning three safety/quality benchmarks, four response categories from clearly-safe to overtly-harmful; only a one-sentence consequence framing in the system prompt varied. The bias is implicit and undetectable by inspecting reasoning traces.", "in_corpus_verdict": "confirmed" } ] }, { "url": "https://pubmed.ncbi.nlm.nih.gov/19769648", "source_title": "The effect of differential rater function over time (DRIFT) on objective structured clinical examination ratings (Medical Education 43(10))", "source_date": "2009-10", "source_type": "academic", "claims": [ { "id": "CG-05", "domain": "critic-and-gapfill", "claim": "Rater severity is nonstationary within a single scoring session: in an OSCE, later time-slots received systematically higher ratings (regression coefficient 0.88, 95% CI 0.38-1.38, p=.001), with the drift 2.4x larger on difficult stations (1.24 vs 0.52), and excluding warm-up stations did not remove it; the psychometric field has named machinery (DRIFT - differential rater functioning over time, Wolfe et al. 2001; Myford & Wolfe monitoring frameworks) for detecting it.", "evidence": "PubMed abstract of McLaughlin et al., Medical Education 2009, opened and confirmed (coefficients above). Wolfe & Moulder 2001 (J. Applied Measurement) established Rasch-based procedures for detecting drift types (primacy/recency, centrality drift, practice/fatigue). Drift within and across scoring days is a repeated finding in operational scoring programs (Lunz & Stahl 1990 across grading periods).", "in_corpus_verdict": null } ] }, { "url": "https://doi.org/10.1111/1467-8721.ep10772989", "source_title": "Feedback Interventions (Kluger & DeNisi 1998), summarizing K&D 1996 Psychological Bulletin meta-analysis", "source_date": "1998-06 (meta-analysis 1996-03; foundational)", "source_type": "academic", "claims": [ { "id": "CG-19", "domain": "critic-and-gapfill", "claim": "Feedback intervention theory's core empirical result stands: feedback improves performance on average (K&D 1996: 131 studies, ~12,000+ participants, mean d approximately 0.38-0.41) but MORE THAN ONE THIRD of feedback interventions REDUCE performance, and effectiveness declines as feedback cues move attention from the task toward the self (praise, person-level evaluation, normative comparison).", "evidence": "Verified in the authors' own summary (Kluger & DeNisi 1998, Current Directions in Psychological Science): 'although FIs improve performance on average, they reduce performance in more than one third of the cases.' Independently corroborated by Wisniewski et al. 2020, which cites K&D as 131 studies, >12,000 participants, average effect 0.38 with roughly a third of effects negative. FIT's mechanism: feedback that directs attention to meta-task/self processes (threat to self, praise, social comparison) depletes task attention and backfires; task- and process-focused cues help.", "in_corpus_verdict": null } ] } ] }