{ "batch": 4, "sources": [ { "url": "https://www.alphaxiv.org/overview/2603.25133", "source_title": "RubricEval: A Rubric-Level Meta-Evaluation Benchmark for LLM Judges in Instruction Following", "source_date": "2026-03-26", "source_type": "academic", "claims": [ { "id": "AJ-04", "domain": "academic-judges", "claim": "At the rubric level - the exact granularity AutoQA would operate at - even frontier judges achieve only ~55-56% balanced accuracy on hard rubric-verification instances (GPT-4o 55.97%, Claude-Sonnet-4.5 55.65%), but rubric-level evaluation with explicit chain-of-thought reasoning beats checklist-level evaluation by 7-12 percentage points and reduces cross-judge variance.", "evidence": "RubricEval (arXiv 2603.25133, full alphaXiv overview read). Binary satisfied/not-satisfied verification of atomic rubrics against responses; EASY subset ~90% accuracy, HARD subset ~55%. Compositional (conditional-logic) instructions, Role Persona, and Format Structure rubrics hardest. Their multi-model arbitration labeling pipeline (RAF) reached 85.0% agreement with humans, Cohen's kappa 0.702. Reasoning-intensive models (o3) notably outperform standard instruct models.", "in_corpus_verdict": null } ] }, { "url": "https://arxiv.org/abs/2410.14632", "source_title": "Diverging Preferences: When do Annotators Disagree and do Models Know? (ICML 2025)", "source_date": "2025-07-15", "source_type": "academic", "claims": [ { "id": "AQ-04", "domain": "annotation-quality", "claim": "In human preference/rating data, the majority of annotator disagreements are attributable to task underspecification and response-style preferences - not random noise and not primarily expertise gaps - and standard aggregation (Bradley-Terry reward modeling) plus LLM-as-judge evaluation both fail to account for this divergence.", "evidence": "Zhang et al., ICML 2025 (arXiv 2410.14632, opened and verified): built a 10-category, 4-class taxonomy of disagreement sources on preference datasets; abstract states 'the majority of disagreements are due to factors such as task underspecification or response style,' explicitly challenging the noise assumption; they show LLM evaluations are heavily swayed by divisive style features and develop divergence-identification methods. Converges with Jiang & de Marneffe (TACL 2022) who found the same structure in NLI.", "in_corpus_verdict": null } ] }, { "url": "https://arxiv.org/abs/2504.09737", "source_title": "Can LLM feedback enhance review quality? A randomized study of 20K reviews at ICLR 2025", "source_date": "2025-04-13", "source_type": "academic", "claims": [ { "id": "CM-05", "domain": "critique-models", "claim": "In the largest deployed RCT of AI critiquing human evaluative work (ICLR 2025, feedback on >20,000 peer reviews), 27% of reviewers who received LLM feedback revised their reviews, incorporating >12,000 suggestions, producing reviews +80 words that blinded raters judged more informative, plus higher author-rebuttal engagement - and feedback was only delivered if it passed a suite of automated LLM reliability tests.", "evidence": "Randomized controlled study at ICLR 2025 (~22.5k feedback-arm vs ~22.4k control reviews): Review Feedback Agent flagged vague comments, content misunderstandings, unprofessional remarks; guardrail LLM tests gated delivery; 26.6-27% update rate, avg +80 words among updaters, blinded informativeness gains, longer author-reviewer discussions. Verified from arXiv abstract page; corroborated by the official ICLR blog.", "in_corpus_verdict": "confirmed" } ] }, { "url": "https://arxiv.org/abs/2503.15620", "source_title": "Does Context Matter? ContextualJudgeBench for Evaluating LLM-based Judges in Contextual Settings", "source_date": "2025-03", "source_type": "academic", "claims": [ { "id": "GF-06", "domain": "grounding-faithfulness", "claim": "Holistic LLM judging of context-grounded outputs is far weaker than claim-level checking: on ContextualJudgeBench (2,000 pairs, 8 splits over RAG/summarization with conditional criteria like 'faithfulness first, then completeness'), the best of 20 judge models tested (OpenAI o1) barely reaches 55% consistent accuracy.", "evidence": "Fetched arXiv:2503.15620 (Salesforce; ACL 2025). 11 specialized judge models + 9 general-purpose models evaluated; the benchmark encodes criteria hierarchies and finds contextual assessment poses a significant challenge even to SOTA models. Contrast: the same model class hits 77-84% on single-claim grounding tasks (LLM-AggreFact, FaithJudge).", "in_corpus_verdict": null } ] }, { "url": "https://arxiv.org/abs/2601.05420", "source_title": "Efficient Inference for Noisy LLM-as-a-Judge Evaluation", "source_date": "2026-01-08", "source_type": "academic", "claims": [ { "id": "HS-06", "domain": "hybrid-statistical", "claim": "For estimating scores from noisy judges, PPI++/efficient-influence-function estimators produce near-identical, shortest confidence intervals - reported 3-15x narrower than Rogan-Gladen-style misclassification-correction estimators depending on the human-labeling ratio, with the advantage largest when the judge is close to random guessing.", "evidence": "arXiv:2601.05420 (Chen, Lu, Li, Guo, Li; 2026-01-08) abstract fetched: unifies measurement-error correction and PPI classes via semiparametric efficiency theory, derives EIF-based estimators, characterizes when PPI-style strictly dominates; the 3-15x interval-width figure comes from the paper's empirical section as surfaced in search-indexed full text (not independently recomputed). Code: github.com/yiqunchen/debias-llm-as-a-judge.", "in_corpus_verdict": null } ] }, { "url": "https://arxiv.org/abs/2606.08077", "source_title": "Support Vector Rubrics: Closing the Gap Between Self-Generated and Human Rubrics", "source_date": "2026-06-06", "source_type": "academic", "claims": [ { "id": "RR-07", "domain": "rubrics-recent", "claim": "Support Vector Rubrics (Jun 2026) identifies the mechanism behind the LLM-vs-human rubric gap - 'self-generated rubrics describe good responses, whereas effective criteria must discriminate between close candidates' - and closes the RubricBench gap from 24.1 to 0.3 points by mining contrastive rubrics from preference pairs (support-pair selection plus adversarial probing of hard negatives).", "evidence": "Fetched abstract (arXiv 2606.08077, Sun et al.) confirmed the objective-mismatch framing, the max-margin method (rubric bank mined from preference data, prompt-conditioned selector, iterative refinement on hard negatives), transfer of the rubric bank across judges without retraining, and competitiveness with dedicated reward models on RewardBench 1/2 and RM-Bench.", "in_corpus_verdict": null } ] }, { "url": "https://sacra.com/c/micro1", "source_title": "Sacra: micro1 revenue, valuation & funding (+ Forbes, 2026-02-03)", "source_date": "2026 (Forbes 2026-02-03)", "source_type": "practitioner", "claims": [ { "id": "IP-07", "domain": "industry-practice", "claim": "micro1 (~$300M annualized revenue by 2026-04) embeds real-time QA verification layers directly in its annotation interface and operates a closed rubric-feedback loop: labs submit updated rubrics and corner-case prompts, which are routed to the highest-performing experts and used to retrain its vetting algorithms.", "evidence": "Sacra company report on micro1 (2026) describes the embedded QA layers, lab-facing dashboards with daily output and pass/fail rates, and the rubric-update routing loop; Forbes (2026-02-03) interview with CEO Ansari frames evaluation as a permanent top-level function ('how well does it work, and what are the thresholds to go into production'). Screening is via AI interviewer (Zara) at thousands of interviews/day; an expert-labeler network and Mercor use the same AI-interview screening pattern (an expert-labeler network: 3% acceptance, 4-stage vetting).", "in_corpus_verdict": null } ] }, { "url": "https://aclanthology.org/2024.emnlp-main.427/", "source_title": "Is LLM-as-a-Judge Robust? Investigating Universal Adversarial Attacks on Zero-shot LLM Assessment", "source_date": "2024-11-12", "source_type": "academic", "claims": [ { "id": "CT-09", "domain": "contrarian", "claim": "Judge verdicts are gameable through content-independent artifacts: short universal adversarial phrases learned on a surrogate model transfer to unseen judge LLMs and inflate scores toward the maximum regardless of the assessed text, with absolute scoring far more vulnerable than comparative assessment.", "evidence": "Raina, Liusie & Gales, EMNLP 2024 (main, pp. 6920+); abstract verified via OpenAlex/ACL record: attackers need no access to the deployed judge - surrogate attack then transfer; 'irrespective of the assessed text, maximum scores are predicted'. Complementary: 'Cheating Automatic LLM Benchmarks' (arXiv 2410.07137) showed null models emitting constant responses achieve top win rates on judged benchmarks.", "in_corpus_verdict": null } ] }, { "url": "https://arxiv.org/html/2602.22585v1", "source_title": "Correcting Human Labels for Rater Effects in AI Evaluation: An Item Response Theory Approach (arXiv 2602.22585)", "source_date": "2026-02", "source_type": "academic", "claims": [ { "id": "CG-01", "domain": "critic-and-gapfill", "claim": "Adjusting for rater severity/centrality with a Many-Facet Rasch Model (MFRM) materially changed which AI systems ranked best in the OpenAI RLHF summarization dataset: trained raters' agreement was only QWK .31-.50, and MFRM-adjusted scores flipped raw-mean rankings so two human-feedback policies rose above human-written reference summaries.", "evidence": "Casabianca & Beiting-Parrish (LAK 2026 LLM Psychometrics workshop) fit MFRM to 6,312 ratings from 15 trained raters on 639 summaries across 19 policies. Full text opened and confirmed: raters R02/R08/R06 systematically lenient, R04/R10/R12 severe, R10 aberrantly central; raw-score rankings 'misrepresent the relative standing of models.' Paper also specifies the estimability requirement: 3-5 ratings per output with rater-item overlap/linkage, and proposes percentile-based flagging (top/bottom 2.5%) for rater remediation.", "in_corpus_verdict": null } ] }, { "url": "https://arxiv.org/abs/2602.15809", "source_title": "Decision Quality Evaluation Framework at Pinterest (arXiv 2602.15809)", "source_date": "2026-02-17", "source_type": "primary", "claims": [ { "id": "CG-15", "domain": "critic-and-gapfill", "claim": "Pinterest's deployed Decision Quality Evaluation Framework (Feb 2026) scores both human moderators and LLM agents against an SME-adjudicated, versioned, immutable golden set selected by inverse-propensity sampling (XGBoost on PinCLIP embeddings, prioritizing low-propensity/novel items), using a two-axis diagnostic - Cohen's kappa for reliability plus correctness vs the golden set - where 'high reliability + low correctness' is read as systematic policy misunderstanding; measured results show 3x-human majority vote adds only +3.6pp accuracy over a single non-expert human, and current LLMs perform on par with a single non-expert human (GPT-5 the only config with positive accuracy delta, +0.9pp).", "evidence": "Fetched full HTML of arXiv 2602.15809. Additional verified mechanics: policy changes handled by dual-labeling the existing golden set under old and new guidelines and visualizing the 'policy delta' as label flips; QA-of-the-QA is two continuous monitors - content-drift (evaluate on newest golden items each release) and system-stability (re-run fixed prompt on fixed golden version to catch pipeline non-determinism); switching prevalence measurement from 3x-human to LLM gave 'over 30x cost savings and 10x turnaround reduction.' Gemini 2.5 configs showed +22pp recall but +48-57pp false-positive rate.", "in_corpus_verdict": null } ] } ] }