{ "batch": 9, "sources": [ { "url": "https://arxiv.org/abs/2505.10320", "source_title": "J1: Incentivizing Thinking in LLM-as-a-Judge via Reinforcement Learning (with CompassJudger-2, Skywork-Reward-V2)", "source_date": "2025-05 to 2025-07", "source_type": "academic", "claims": [ { "id": "AJ-09", "domain": "academic-judges", "claim": "Dedicated trained judges advanced substantially in 2025: Meta's J1 (RL-trained thinking-judge, May 2025) at 32B outperforms o1-mini, o3, and 671B DeepSeek-R1 on some judge benchmarks; CompassJudger-2-7B (July 2025) matches far larger generalists on judge benchmarks; Skywork-Reward-V2-Llama-3.1-8B (July 2025) topped seven reward benchmarks including JudgeBench.", "evidence": "J1 arXiv 2505.10320 (verified): unified verifiable-reward RL over 22K synthetic pairs, mitigates position bias by construction, gains from test-time self-consistency. CompassJudger-2 arXiv 2507.09104: verifiable-reward + margin-loss training; 32B scores 80.9% accuracy on JudgerBenchV2. Skywork-Reward-V2 arXiv 2507.01352: 26M human-AI-curated preference pairs; 8B model led RewardBench v1/v2, PPE, RM-Bench, JudgeBench among reward models.", "in_corpus_verdict": null } ] }, { "url": "https://arxiv.org/pdf/2510.08460", "source_title": "LeWiDi-2025 at NLPerspectives: Third Edition of the Learning with Disagreements Shared Task", "source_date": "2025-10-09", "source_type": "academic", "claims": [ { "id": "AQ-09", "domain": "annotation-quality", "claim": "The NLP field has institutionalized disagreement-as-signal: the third Learning-With-Disagreements shared task (LeWiDi-2025, EMNLP 2025) standardizes dual evaluation - soft-label (predict the population label distribution, Wasserstein/Manhattan distance) and perspectivist (recover individual annotators' labels) - across four datasets that all ship per-annotator labels rather than adjudicated gold.", "evidence": "Leonardelli et al. (arXiv 2510.08460, Oct 2025; le-wi-di.github.io) describe the two complementary paradigms and public leaderboard; lineage runs from Aroyo & Welty's 'truth is a lie' perspectivism through DICES (NeurIPS 2023 safety dataset with up to 100+ raters/item) and jury learning (Gordon et al., CHI 2022, which composes verdicts from explicitly selected rater subpopulations); Frenda et al.'s survey (LREV 2024) documents the dataset ecosystem.", "in_corpus_verdict": null } ] }, { "url": "https://arxiv.org/abs/2206.05802", "source_title": "Self-critiquing models for assisting human evaluators (OpenAI)", "source_date": "2022-06-12", "source_type": "primary", "claims": [ { "id": "CM-11", "domain": "critique-models", "claim": "The foundational 2022 result behind the whole lineage: model-written critiques helped human evaluators find flaws in summaries they would otherwise have missed (including planted flaws in deliberately misleading human-written summaries), and models exhibit a discriminator-critique gap - they can often detect that something is wrong better than they can articulate why.", "evidence": "Saunders et al. (OpenAI, arXiv 2206.05802): topic-based summarization assistance experiments; critiques helped on both model- and human-written summaries; framework comparing generation, discrimination, and critique ability found 'even large models may still have relevant knowledge they cannot or do not articulate as critiques'; larger models critique and self-refine better. Verified from arXiv abstract.", "in_corpus_verdict": null } ] }, { "url": "https://aclanthology.org/2026.findings-acl.689/", "source_title": "FaithLens: Detecting and Explaining Faithfulness Hallucination", "source_date": "2026-07 (ACL 2026)", "source_type": "academic", "claims": [ { "id": "GF-11", "domain": "grounding-faithfulness", "claim": "2026 SOTA joins verdict and explanation in one cheap model: FaithLens (ACL 2026 Findings) is an 8B faithfulness-hallucination detector that jointly outputs a binary prediction AND an explanation, trained via cold-start fine-tuning on filtered synthetic data plus rule-based RL rewarding both prediction correctness and explanation quality, and outperforms GPT-5.2 and o3 across 12 tasks.", "evidence": "Fetched ACL Anthology entry 2026.findings-acl.689 (Si et al., Findings of ACL 2026, pp. 14068-14099). Abstract claims superiority over GPT-5.2/o3 and 'a distinctive balance of trustworthiness, efficiency, and effectiveness'; per-task numbers require the full PDF.", "in_corpus_verdict": null } ] }, { "url": "https://arxiv.org/abs/2604.18569", "source_title": "Revisiting Active Sequential Prediction-Powered Mean Estimation", "source_date": "2026-04-20", "source_type": "contrarian", "claims": [ { "id": "HS-11", "domain": "hybrid-statistical", "claim": "In sequential prediction-powered mean estimation, uncertainty-based active querying contributed little: the smallest confidence widths occurred when query probabilities were near-constant at the budget ceiling, and theory shows optimized query probabilities converge to the maximum allowed constant rate when chosen without reference to covariates.", "evidence": "arXiv:2604.18569 (Sfyraki & Wang, submitted 2026-04-20, accepted ICLR 2026 per arXiv page) abstract fetched: empirical finding that the uncertainty-blend weight near the constant term minimizes CI width, plus non-asymptotic analysis and no-regret query-probability selection corroborating near-uniform optimality in this regime.", "in_corpus_verdict": null } ] }, { "url": "https://www.langchain.com/blog/introducing-align-evals", "source_title": "Introducing Align Evals: Streamlining LLM Application Evaluation (LangChain)", "source_date": "2025-07-29", "source_type": "practitioner", "claims": [ { "id": "RR-12", "domain": "rubrics-recent", "claim": "LangChain's Align Evals (LangSmith, Jul 2025) productized judge validation as a standard workflow: humans grade a representative golden set per criterion, an 'alignment score' measures judge-vs-human match, unaligned cases are surfaced for prompt iteration, and each judge-prompt version is compared against a saved baseline alignment score.", "evidence": "Fetched launch blog confirmed the four-step workflow (select criteria; select representative good+bad examples; human-grade expected scores; iterate evaluator prompt against alignment score), the stated problem ('Our evaluation scores don't match what we'd expect a human on our team to say'), inspiration from Eugene Yan's AlignEval, and roadmap items (alignment tracking over time, automatic prompt optimization).", "in_corpus_verdict": null } ] }, { "url": "https://kili-technology.com/blog/data-annotation-guide-how-to-achieve-high-quality-data-in-complex-ai-data-operations", "source_title": "An annotation platform's quality playbook + Kili Technology annotation quality guide", "source_date": "2024-2025, verified live 2026-07-14", "source_type": "practitioner", "claims": [ { "id": "IP-12", "domain": "industry-practice", "claim": "Vendor materials repeatedly attribute low inter-annotator precision to instruction/rubric defects rather than annotator failure - an annotation platform's own remediation playbook for low precision starts with 'poor instructions, weak edge-case coverage' and prescribes guideline revision and calibration sessions before retraining or replacing people.", "evidence": "An annotation platform's quality post's scenario playbook: low precision -> better training, clearer guidelines, calibration sessions, edge-case examples; high precision + low accuracy -> refine the ground truth itself and check for systematic bias in training materials. Kili's industry guide (cited in 2025 coverage) similarly notes an annotator consistently disagreeing on one label category 'is often revealing a guideline gap', and industry estimates put RLHF preference-annotation noise above 20%.", "in_corpus_verdict": null } ] }, { "url": "https://arxiv.org/abs/2602.02550", "source_title": "HyPAC: Cost-Efficient LLMs-Human Hybrid Annotation with PAC Error Guarantees (arXiv 2602.02550)", "source_date": "2026-01-30", "source_type": "academic", "claims": [ { "id": "F26-04", "domain": "frontier-2026", "claim": "HyPAC (Jan 30, 2026) gives the one-human-touch routing problem a formal solution: calibrate two uncertainty thresholds (via importance sampling + upper confidence bounds) that partition items into three regions routed to cheap LLM / expensive reasoning model / human, achieving a distribution-free PAC guarantee on annotation error while cutting annotation cost 78.51% in experiments.", "evidence": "Abstract opened and verified on arXiv. The guarantee is distribution-free and pre-trained-model-free; routing is by calibrated uncertainty, not fixed rules. This is the first 2026-vintage method that turns 'which items get the human?' from a launch-time heuristic into a statistically certified, closed-loop calibration -- exactly the Q6 fork (fixed routing vs closed-loop on observed error).", "in_corpus_verdict": null } ] }, { "url": "https://arxiv.org/abs/2505.18486", "source_title": "Comparing Human and AI Rater Effects Using the Many-Facet Rasch Model (Jiao, Song, Lee)", "source_date": "2025-05-24", "source_type": "academic", "claims": [ { "id": "CG-06", "domain": "critic-and-gapfill", "claim": "The MFRM toolkit is now being applied symmetrically to LLM judges: a 2025 study fit MFRM to 10 LLMs plus human expert raters scoring the same writing tasks and found LLMs exhibit measurable severity/centrality rater effects that differ by model, with GPT-4o, Gemini 1.5 Pro, and Claude 3.5 Sonnet showing the highest accuracy and fewest rater effects.", "evidence": "Jiao, Song & Lee, arXiv 2505.18486 (May 2025), abstract opened and confirmed: QWK vs human scores, Cronbach alpha across prompts, and MFRM rater-effect estimates computed identically for human and LLM raters. Parallel 2025 work (Wang et al., Computers and Education: AI, Sept 2025) builds a full psychometric reliability/validity framework for LLM raters in large-scale writing assessment. This is the literature bridge: the same model audits both layers.", "in_corpus_verdict": null } ] }, { "url": "https://www.cochranelibrary.com/cdsr/doi/10.1002/14651858.CD000259.pub4/full", "source_title": "Audit and feedback: effects on professional practice (Cochrane review update, Ivers et al.)", "source_date": "2025-03-25", "source_type": "academic", "claims": [ { "id": "CG-20", "domain": "critic-and-gapfill", "claim": "The March 2025 Cochrane update on audit-and-feedback (292 studies, 678 arms, healthcare professionals) finds median absolute improvement in desired practice of only 2.7% (IQR 0.0 to 8.6; weighted meta-analytic mean +6.2%, 95% CI 4.1-8.2, moderate certainty), with effects larger for low baseline performers, individual-level (not team-level) data, comparison to TOP peers or a benchmark (comparison to peer average showed no significant effect), a trusted local source, and action plans with specific advice - while repeated delivery was associated with LOWER effect size.", "evidence": "Fetched the Cochrane summary (updated review of CD000259, published 2025-03-25): '292 studies with 678 arms'; median absolute improvement '2.7%, with an IQR of 0.0 to 8.6'; weighted mean '6.2% (95% CI 4.1 to 8.2; moderate-certainty evidence)'; OR 1.47. Moderator list quoted directly, including the counterintuitive 'repeated delivery was associated with lower effect size' and 'comparison to top-peers or a benchmark increased effects; comparing against the average of all peers did not.'", "in_corpus_verdict": null } ] } ] }