{ "batch": 4, "checked_at": "2026-07-15", "sources": [ { "url": "https://www.alphaxiv.org/overview/2603.25133", "retrieval": "fulltext", "fetched": [ "https://www.alphaxiv.org/overview/2603.25133", "https://arxiv.org/abs/2603.25133", "https://arxiv.org/html/2603.25133v1" ], "resolved_title": "RubricEval: A Rubric-Level Meta-Evaluation Benchmark for LLM Judges in Instruction Following", "resolved_date": "2026-03-26", "title_match": true, "claims": [ { "id": "AJ-04", "verdict": "confirmed", "checked": [ "GPT-4o 55.97% balanced accuracy on Hard subset", "Claude-Sonnet-4.5 55.65% on Hard subset", "rubric-level with reasoning beats checklist-level by a gap of 7-12 points", "combining rubric+checklist reduces inter-judge (cross-judge) variance", "Easy subset ~90% (Qwen3-235B / gpt-oss-120b ~90%, Table 2: 89.87 / 89.55)", "RAF human agreement 85.0% accuracy, Cohen's kappa 0.702", "3,486 quality-controlled instances with Easy/Hard subsets" ], "not_visible": [], "quote": "GPT-4o achieves merely 55.97% balanced accuracy, and Claude-Sonnet-4.5 reaches 55.65% ... while checklist-level achieves only 69.90% and 70.44%--a gap of 7-12 points ... Human-RAF agreement reaches 85.0% accuracy with Cohen's kappa = 0.702", "notes": "Given alphaXiv URL returned HTTP 403 to WebFetch; curl returned only the SPA shell (page title and meta description confirm the exact title/authors). Verified against the allowed arXiv variant (abs + HTML full text) since the URL contains arXiv ID 2603.25133. All core numbers confirmed verbatim in the HTML full text. Two evidence-field nuances: (a) the paper does NOT explicitly assert reasoning models like o3 outperform instruct models as a class, though o3 leads the Hard split at 84.81 BAcc and is the best single judge at 79.9% on disputed cases; (b) evidence equates 'Compositional' with 'conditional-logic', but the paper shows Compositional *instructions* are hardest while the Conditional Logic *rubric type* actually scores relatively well (Table 4). Neither touches the core claim." } ] }, { "url": "https://arxiv.org/abs/2410.14632", "retrieval": "abstract", "fetched": ["https://arxiv.org/abs/2410.14632"], "resolved_title": "Diverging Preferences: When do Annotators Disagree and do Models Know?", "resolved_date": "2024-10-18 (v1); v3 2026-03-03; venue ICML 2025", "title_match": true, "claims": [ { "id": "AQ-04", "verdict": "confirmed", "checked": [ "majority of disagreements due to task underspecification or response style (not simple noise)", "standard reward modeling (Bradley-Terry) and LLM-as-Judge fail to account for divergence", "develops methods for identifying diverging preferences", "ten-category taxonomy across four high-level classes", "ICML 2025 venue note" ], "not_visible": [ "explicit statement that expertise gaps are NOT a main cause (excluded only by implication, since the majority is attributed to underspecification/style)", "Jiang & de Marneffe TACL 2022 corroboration (a separate citation, not part of this source)" ], "quote": "the majority of disagreements are due to factors such as task underspecification or response style ... standard reward modeling (e.g., Bradley-Terry) and LLM-as-Judge evaluation methods fail to account for divergence between annotators.", "notes": "Core assertion and all listed mechanisms confirmed from the abstract. The 'not primarily expertise gaps' element is supported by implication rather than an explicit exclusionary sentence." } ] }, { "url": "https://arxiv.org/abs/2504.09737", "retrieval": "abstract", "fetched": ["https://arxiv.org/abs/2504.09737"], "resolved_title": "Can LLM feedback enhance review quality? A randomized study of 20K reviews at ICLR 2025", "resolved_date": "2025-04-13", "title_match": true, "claims": [ { "id": "CM-05", "verdict": "confirmed", "checked": [ "feedback to more than 20,000 randomly selected reviews", "27% of reviewers who received feedback updated their reviews", "over 12,000 feedback suggestions incorporated", "average increase of 80 words among updaters", "more informative reviews as judged by blinded researchers", "longer author-reviewer discussions / more rebuttal engagement", "a suite of automated LLM-powered reliability tests acted as guardrails gating delivery" ], "not_visible": [ "the specific '26.6%' figure (abstract states 27%; 26.6% appears only in the claim's evidence field)" ], "quote": "27% of reviewers who received feedback updated their reviews and over 12,000 feedback suggestions from the agent were incorporated by those reviewers ... an average increase of 80 words among those who updated after receiving feedback.", "notes": "All headline numbers in the core claim confirmed verbatim from the abstract." } ] }, { "url": "https://arxiv.org/abs/2503.15620", "retrieval": "abstract", "fetched": ["https://arxiv.org/abs/2503.15620"], "resolved_title": "Does Context Matter? ContextualJudgeBench for Evaluating LLM-based Judges in Contextual Settings", "resolved_date": "2025-03-19", "title_match": true, "claims": [ { "id": "GF-06", "verdict": "confirmed", "checked": [ "2,000 challenging response pairs", "eight (8) splits", "20 judge models (11 judge models + 9 general-purpose models)", "OpenAI o1 best-performing, barely reaches 55% consistent accuracy", "conditional evaluation criteria (factuality then completeness)", "RAG and summarization settings" ], "not_visible": [ "Salesforce affiliation (not stated on the abstract page)", "ACL 2025 venue (not listed on the abstract page)" ], "quote": "we propose ContextualJudgeBench, a judge benchmark with 2,000 challenging response pairs across eight splits ... Our comprehensive study across 11 judge models and 9 general purpose models ... OpenAI's o1, the best-performing model, barely reaches 55% consistent accuracy.", "notes": "All core numbers confirmed. Minor terminology: the claim's illustrative criterion 'faithfulness first, then completeness' appears in the abstract as 'factuality and then considering completeness' (factuality vs faithfulness). The evidence field's 'Salesforce; ACL 2025' is not visible on the abstract page but is not part of the core claim." } ] }, { "url": "https://arxiv.org/abs/2601.05420", "retrieval": "fulltext", "fetched": [ "https://arxiv.org/abs/2601.05420", "https://arxiv.org/pdf/2601.05420", "https://arxiv.org/html/2601.05420v1" ], "resolved_title": "Efficient Inference for Noisy LLM-as-a-Judge Evaluation", "resolved_date": "2026-01-08", "title_match": true, "claims": [ { "id": "HS-06", "verdict": "confirmed", "checked": [ "EIF and PPI++ produce nearly identical and shortest intervals", "outperform Rogan-Gladen by a factor of 3-15x", "3-15x factor depends on the labeling ratio", "advantage most pronounced when judge is closer to random guessing (q0+q1-1 small)", "at q0=q1=0.6, RG intervals ~10x wider than EIF/PPI++", "code repo github.com/yiqunchen/debias-llm-as-a-judge", "authors Chen, Lu, Li, Guo, Li" ], "not_visible": [], "quote": "EIF and PPI++ produce nearly identical and shortest intervals, outperforming Rogan-Gladen by a factor of 3-15x depending on the labeling ratio. The advantage is most pronounced when q0+q1-1 is small (i.e., when the LLM-judge is closer to random guessing).", "notes": "The 3-15x figure and the near-random-guessing / labeling-ratio dependence are NOT in the abstract (and the PDF was FlateDecode-compressed and unreadable), but the allowed HTML full text (Figure 4 caption + Section 5.3) confirms them verbatim, including 'PPI++' specifically (the abstract only said 'PPI'). Fully confirmed." } ] }, { "url": "https://arxiv.org/abs/2606.08077", "retrieval": "abstract", "fetched": ["https://arxiv.org/abs/2606.08077"], "resolved_title": "Support Vector Rubrics: Closing the Gap Between Self-Generated and Human Rubrics", "resolved_date": "2026-06-06", "title_match": true, "claims": [ { "id": "RR-07", "verdict": "confirmed", "checked": [ "objective mismatch: self-generated rubrics describe good responses, effective criteria must discriminate between close candidates", "narrows gap to human reference rubrics from 24.1 to 0.3 points", "mines contrastive features from preference pairs into a rubric bank", "support-pair selection and adversarial probing of hard negatives", "max-margin boundary learning over preference data", "learned bank transfers across judges without retraining", "competitive with dedicated reward models on RewardBench 1&2 and RM-Bench" ], "not_visible": [], "quote": "self-generated rubrics describe good responses, whereas effective criteria must discriminate between close candidates ... SVR narrows the gap to human reference rubrics from 24.1 to 0.3 points ... the learned bank transfers across judges without retraining.", "notes": "Every element of the claim confirmed verbatim from the abstract, including the exact 24.1 -> 0.3 figure. Authors Sun et al. match." } ] }, { "url": "https://sacra.com/c/micro1", "retrieval": "landing", "fetched": ["https://sacra.com/c/micro1"], "resolved_title": "micro1 revenue, valuation & funding | Sacra", "resolved_date": "revenue figure labeled 2026 (April 2026 estimate on-page); site footer 2025", "title_match": true, "claims": [ { "id": "IP-07", "verdict": "partially_confirmed", "checked": [ "micro1 hit $300M annualized revenue in April 2026 (up from $125M end of 2025)", "secure annotation interface with embedded quality assurance layers verifying tasks in real time", "feedback loop: labs submit updated rubrics or corner-case prompts", "micro1 routes these updates to highest-performing experts and retrains its vetting algorithms on new criteria", "lab dashboard displaying daily output, pass/fail rates, per-task turnaround" ], "not_visible": [ "Forbes 2026-02-03 interview with CEO Ansari", "Ansari framing evaluation as a permanent top-level function ('how well does it work, thresholds to go into production')", "An expert-labeler network's 3% acceptance / 4-stage vetting (the network is not named on page)", "Mercor using AI-interview screening (Mercor not mentioned on page)" ], "quote": "Sacra estimates that Micro1 hit $300M in annualized revenue in April 2026, up from $125M at the end of 2025 ... The system incorporates a feedback loop where labs can submit updated rubrics or corner-case prompts. Micro1 routes these updates to the highest-performing experts and retrains its vetting algorithms on new criteria.", "notes": "The micro1-specific core (revenue, embedded real-time QA layers, closed rubric-feedback loop with expert routing + vetting-algorithm retraining, lab dashboards) is fully confirmed on the Sacra page. Downgraded to partially_confirmed because the source_title bundles Forbes (2026-02-03), but only the Sacra URL is allowed: the Forbes interview/Ansari framing, an expert-labeler network, and Mercor items are not present on this page. Also a conflation: the page attributes 'thousands of candidates daily' to micro1's proprietary AI system, while 'Zara' is described separately as an AI recruiter (20+ languages, auto-scoring); and the page states ~1% of applicants pass (not the 3% figure the claim attributes to an expert-labeler network)." } ] }, { "url": "https://aclanthology.org/2024.emnlp-main.427/", "retrieval": "abstract", "fetched": ["https://aclanthology.org/2024.emnlp-main.427/"], "resolved_title": "Is LLM-as-a-Judge Robust? Investigating Universal Adversarial Attacks on Zero-shot LLM Assessment", "resolved_date": "EMNLP 2024, November 2024 (pages 7499-7517)", "title_match": true, "claims": [ { "id": "CT-09", "verdict": "confirmed", "checked": [ "short universal adversarial phrases appended to text inflate judge scores", "phrase learned on a surrogate model then transferred to unseen judge LLMs", "irrespective of the assessed text, maximum scores are predicted", "absolute scoring significantly more susceptible than comparative assessment", "authors Raina, Liusie, Gales; EMNLP 2024" ], "not_visible": [], "quote": "when transferred to unseen models, scores can be drastically inflated such that irrespective of the assessed text, maximum scores are predicted ... judge-LLMs are significantly more susceptible to these adversarial attacks when used for absolute scoring, as opposed to comparative assessment.", "notes": "Core claim confirmed verbatim from the ACL Anthology abstract. One evidence-field discrepancy (not part of the core claim): the citation lists 'pp. 6920+', but the ACL Anthology record gives pages 7499-7517 for this paper." } ] }, { "url": "https://arxiv.org/html/2602.22585v1", "retrieval": "fulltext", "fetched": ["https://arxiv.org/html/2602.22585v1"], "resolved_title": "Correcting Human Labels for Rater Effects in AI Evaluation: An Item Response Theory Approach", "resolved_date": "2026-02 (arXiv 2602; LAK 2026 LLM Psychometrics workshop, April 2026)", "title_match": true, "claims": [ { "id": "CG-01", "verdict": "confirmed", "checked": [ "Many-Facet Rasch Model (MFRM) fit to OpenAI RLHF summarization dataset", "6,312 ratings, 639 unique articles/summaries, 19 policies, 15 raters", "trained raters' agreement QWK ranged from .31 to .50", "MFRM adjustment flips raw-mean rankings: two human-feedback policies (sup4_6b_ppo_rm4_6b, sup4_ppo_rm4) outperform human-generated reference summaries", "R02/R08/R06 lenient (negative estimates); R04/R10/R12 severe; R10 aberrantly central", "3-5 ratings per output with rater-item overlap/linkage for estimability", "flag raters below 2.5th / above 97.5th percentile" ], "not_visible": [], "quote": "The dataset contained 6,312 ratings based on 639 unique articles, summarized by 19 policies which were evaluated by 15 raters ... QWK ranged from .31 to .50 ... two human feedback models, sup4_6b_ppo_rm4_6b and sup4_ppo_rm4, outperform all other models and the human-generated reference summaries.", "notes": "Full HTML retrieved; all numbers, rater profiles, ranking flip, estimability requirement, and percentile flagging confirmed verbatim." } ] }, { "url": "https://arxiv.org/abs/2602.15809", "retrieval": "fulltext", "fetched": [ "https://arxiv.org/abs/2602.15809", "https://arxiv.org/html/2602.15809v1" ], "resolved_title": "Decision Quality Evaluation Framework at Pinterest", "resolved_date": "2026-02-17", "title_match": true, "claims": [ { "id": "CG-15", "verdict": "confirmed", "checked": [ "3x-human majority vote adds +3.6pp accuracy over single non-expert human", "LLMs demonstrate quality on par with a single non-expert human (gap to SME-level remains)", "GPT-5 (balanced) +0.9 accuracy delta - only LLM config in the positive", "inverse propensity sampling; XGBoost trained on PinCLIP embeddings to predict propensity", "Cohen's Kappa for reliability + correctness vs GDS; high reliability + low correctness = systematic policy misunderstanding", "shift from 3x-human to LLM gave over 30x cost savings and 10x turnaround reduction", "Gemini/flash +22.6 recall (22.5% gain) but +57.2 FPR / +47.7% FPR", "GDS published as immutable and versioned dataset; SMEs relabel under new policy to produce 'policy delta'", "content-drift monitor + system-stability monitor" ], "not_visible": [], "quote": "This shift resulted in over 30x in cost savings and a 10x reduction in labeling turnaround time ... the LLMs demonstrate quality on par with a single non-expert human, but a gap still remains to reach SME-level quality ... High reliability paired with low correctness indicates that labelers are all making the same mistake consistently.", "notes": "Abstract alone did not carry the numbers; the allowed HTML full text confirms all nine specifics verbatim (Table 1 + prose). Minor nuance: '+0.9pp' GPT-5 is the only *LLM* config with a positive accuracy delta; the 3x-human-majority row is also positive (+3.6) but is a human baseline, so 'only config' is slightly imprecise. Gemini FPR stated as +47.7% in prose and +57.2 in Table 1, consistent with the claimed +48-57pp range." } ] } ] }