{ "batch": 8, "checked_at": "2026-07-15", "sources": [ { "url": "https://arxiv.org/abs/2502.01534", "retrieval": "fulltext", "fetched": [ "https://arxiv.org/abs/2502.01534", "https://arxiv.org/html/2502.01534" ], "resolved_title": "Preference Leakage: A Contamination Problem in LLM-as-a-judge", "resolved_date": "arXiv v1 2025-02-03; v3 2026-03-04; accepted ICLR 2026", "title_match": true, "claims": [ { "id": "AJ-08", "verdict": "confirmed", "checked": [ "up to 28.7% preference leakage score (Table 1/2)", "same model 23.6% average PLS (Table 2)", "inheritance/fine-tuned 19.3% and 22.3% (in 19-22% band)", "same family 8.9% same-series vs 2.8% different-series", "SFT 23.6% avg vs DPO 5.2%", "judge self-detection near random (e.g. 41.0%, 52.0%)", "BERT classifier 82.4%", "leakage proportional to synthetic-data amount, no clear threshold", "style/format removal is largest decrease (e.g. 17.5% to 9.0%)", "contextual calibration Error Bias 17.8 to 7.3", "prompting mitigation failed (17.8 to 18.3, slightly worse)", "accepted ICLR 2026 (abstract page)" ], "not_visible": [], "quote": "contextual calibration with an additional held-out set for bias adjustment is the most effective, reducing Error Bias from 17.8 to 7.3.", "notes": "Abstract page confirms core + ICLR 2026 acceptance but shows no numbers; all 11 headline figures verified verbatim in the arxiv.org/html full text. Authors: Dawei Li et al. (Huan Liu group)." } ] }, { "url": "https://arxiv.org/abs/2506.06991", "retrieval": "abstract", "fetched": [ "https://arxiv.org/abs/2506.06991" ], "resolved_title": "Evaluating LLM-Contaminated Crowdsourcing Data Without Ground Truth", "resolved_date": "arXiv v1 2025-06-08; v2 2025-11-06", "title_match": true, "claims": [ { "id": "AQ-08", "verdict": "partially_confirmed", "checked": [ "evaluates contaminated data without ground truth via peer prediction", "quantifies correlations between worker answers conditioning on (a subset of) LLM-generated labels", "training-free scoring mechanism with theoretical guarantees under a crowdsourcing model that accounts for LLM collusion", "existing text-based detectors rely on high-dimensional text, unsuitable for annotation tasks", "multiple-choice labeling named as target task", "empirically robust in detecting low-effort cheating on real-world crowdsourcing datasets", "authors Yichi Zhang, Jinlong Pang, Zhaowei Zhu, Yang Liu (matches Zhang, Pang, Zhu & Liu)" ], "not_visible": [ "NeurIPS 2025 venue (no venue stated anywhere on the abstract page)", "'ratings' as a target task type (only multiple-choice explicitly named)", "explicit 'per-worker not per-item' framing (implied by inter-worker correlation, not stated)", "explicit 'multiple workers pasting from the same LLM' scenario", "source_date 2025-12-01 (page shows Jun/Nov 2025 only)" ], "quote": "a training-free scoring mechanism with theoretical guarantees under a crowdsourcing model that accounts for LLM collusion", "notes": "Core method claims fully match the abstract. The asserted NeurIPS 2025 publication and the 2025-12-01 date are not visible on the page; abstract dates are v1 2025-06-08, v2 2025-11-06." } ] }, { "url": "https://toloka.ai/blog/llm-qa-scaling-data-quality-assurance-technologically/", "retrieval": "fulltext", "fetched": [ "https://toloka.ai/blog/llm-qa-scaling-data-quality-assurance-technologically/" ], "resolved_title": "LLM QA: Scaling data quality assurance technologically", "resolved_date": "byline 2026-03-30 (page metadata also shows a 2026-07-15 timestamp)", "title_match": true, "claims": [ { "id": "CM-10", "verdict": "confirmed", "checked": [ "agentic autocheck running on every submission", "separate instance launched for every single quality metric", "rigid three-way scale: Pass, Fail, or Unable to verify", "abstain state exists because otherwise a model will hallucinate a guess", "Socratic-style feedback that asks questions rather than pointing to the error", "lower precision on Fail is an acceptable tradeoff; false pass costlier than sending a genuine pass to human review", "tools: download web pages, view images, process audio/video, run Python, execute bash", "'one metric, one entity, one verdict' constraint", "internal benchmark of over 300 real submissions from 20-plus live projects", "ground truth from Toloka senior QA reviewers", "performance entirely bounded by task design; ambiguity yields a wall of Unable to verify" ], "not_visible": [ "the actual benchmark performance numbers (claim itself notes these are in an image, not extractable)" ], "quote": "We enforce a deliberate constraint: one metric, one entity, one verdict.", "notes": "All eight sub-claims supported verbatim. Byline Vitaly Moiseev and Mariya Shmatova, dated March 30, 2026. Vendor self-description; benchmark scores are referenced but not displayed as text, consistent with the claim's own caveat." } ] }, { "url": "https://aclanthology.org/2025.naacl-long.534/", "retrieval": "abstract", "fetched": [ "https://aclanthology.org/2025.naacl-long.534/" ], "resolved_title": "FactEval: Evaluating the Robustness of Fact Verification Systems in the Era of Large Language Models", "resolved_date": "NAACL 2025, April 2025, pp. 10647-10660", "title_match": true, "claims": [ { "id": "GF-10", "verdict": "confirmed", "checked": [ "17 realistic word-level and character-level perturbations and 4 types of subpopulations", "built on FEVER", "zero-shot, few-shot, and chain-of-thought prompting", "LLMs brittle to small input changes and performance variations across subpopulations", "authors Mamta and Oana Cocarascu", "NAACL 2025" ], "not_visible": [ "per-perturbation quantitative results (abstract is qualitative; claim does not assert specific numbers)" ], "quote": "LLMs are brittle to small input changes and also exhibit performance variations across different subpopulations.", "notes": "Abstract confirms every asserted specific (17 perturbations, 4 subpopulations, FEVER, three prompting setups, brittleness finding). The claim explicitly defers per-perturbation numbers to the full paper, so nothing overclaimed." } ] }, { "url": "https://arxiv.org/abs/2606.15474", "retrieval": "abstract", "fetched": [ "https://arxiv.org/abs/2606.15474" ], "resolved_title": "Who Drifted: the System or the Judge? Anytime-Valid Attribution in LLM Evaluation Pipelines", "resolved_date": "arXiv submitted 2026-06-13", "title_match": true, "claims": [ { "id": "HS-10", "verdict": "confirmed", "checked": [ "fixed human-labeled anchor set re-scored by the current judge at a steady interleave", "betting e-process on the judge-versus-human gap", "silent version bump detected as judge drift in 60/60 runs with zero judge-to-system misattribution", "rolling z-test false-alarms on 75% of drift-free streams", "approximately 0.64 of the cost of strong-judging every item, or 0.21 in a cheaper-but-deafer regime", "one-way identification (only the judge can move the anchors)", "verdict in {none, system, judge}", "strict-prompt change attributed on 110 of 120 runs", "TL;DR replication attribution perfect (240/240)" ], "not_visible": [], "quote": "a silent version bump is detected as judge drift in 60/60 runs with zero judge-to-system misattribution.", "notes": "All 11 asserted specifics verified verbatim in the abstract. Single-author preprint (Yitao Li), submitted 2026-06-13, matching the claim's own not-yet-peer-reviewed caveat." } ] }, { "url": "https://arxiv.org/abs/2603.00077", "retrieval": "abstract", "fetched": [ "https://arxiv.org/abs/2603.00077" ], "resolved_title": "Autorubric: Unifying Rubric-based LLM Evaluation", "resolved_date": "arXiv v1 2026-02-13; v2 2026-04-03; 52 pages", "title_match": true, "claims": [ { "id": "RR-11", "verdict": "confirmed", "checked": [ "unifies techniques scattered across papers with inconsistent terminology and partial implementations", "binary, ordinal, and nominal criteria; single-judge and ensemble; few-shot calibration; bias mitigations; psychometric reliability metrics", "CHARM-100: 87% binary accuracy, moderate-to-substantial kappa", "peer review agent 0.47 to 0.85, above the 0.82 expert-curated baseline", "RiceChem 80% accuracy with 5-shot calibration", "ResearcherBench 931 criteria, cross-judge agreement", "RL reward AdvancedIF +0.039, Wilcoxon p=0.032, positive transfer to IFEval", "authors Delip Rao, Chris Callison-Burch" ], "not_visible": [], "quote": "raise a peer review agent's score from 0.47 to 0.85 (above the 0.82 expert-curated baseline)", "notes": "All nine asserted specifics verified verbatim in the abstract. v1 2026-02-13 matches source_date; v2 2026-04-03, 52 pages, both confirmed." } ] }, { "url": "https://www.indeed.com/cmp/Outlier-Ai/reviews?fjobtitle=Reviewer&fcountry=US&ftext=outlier", "retrieval": "fulltext", "fetched": [ "https://www.indeed.com/cmp/Outlier-Ai/reviews?fjobtitle=Reviewer&fcountry=US&ftext=outlier" ], "resolved_title": "Outlier AI Employee Reviews (Indeed, Reviewer filter) - 2.4/5 from 774 reviews", "resolved_date": "reviews span 2024-2026", "title_match": false, "claims": [ { "id": "IP-11", "verdict": "partially_confirmed", "checked": [ "removal from projects without stated reason (multiple reviews 2024-2026)", "uneven / unpredictable task allocation (feast-or-famine workload)", "opaque eligibility/onboarding decisions felt arbitrary with no transparency", "no communication from higher-ups; contact only via Slack/Discord", "pay deterioration after Outlier merger; unpaid training", "overall 2.4/5 from 774 reviews" ], "not_visible": [ "explicit attempter -> reviewer -> senior-reviewer pyramid", "timed / capped per-task pay with unpaid overage window", "specific '300-reviewer queues where ~10 got unlimited tasks'", "queue managers running webinars / 'war rooms' as calibration", "contributors and reviewers 'pitted against each other' / internal competition" ], "quote": "you can also be cut from a project with no warning or reason, placed on another completely different project", "notes": "Only the Indeed page could be fetched; source aggregates Indeed/Glassdoor/Trustpilot but constraints forbid the other two sites. The general thrust (opaque removal, unstable allocation, opacity, pay complaints) is supported, but the specific structural mechanisms in the claim (pyramid, capped timed pay, 300/10 queues, war rooms, pitted-against-each-other) are not present on this page. title_match false: source_title is a synthetic three-platform aggregate label, not this single Indeed page's title; reviews span 2024-2026, not solely 2025." } ] }, { "url": "https://openreview.net/forum?id=XI2hnjGnZx", "retrieval": "failed", "fetched": [ "https://openreview.net/forum?id=XI2hnjGnZx" ], "resolved_title": null, "resolved_date": null, "title_match": false, "claims": [ { "id": "F26-03", "verdict": "not_retrievable", "checked": [], "not_visible": [ "peak verdict shift of -9.8 percentage points", "30% relative drop in unsafe-content detection", "18,240 controlled judgments", "3 judge models", "ERR_J = 0.000 (zero CoT acknowledgment)", "1,520 responses across three benchmarks", "four response categories clearly-safe to overtly-harmful", "CTB@ICML 2026 venue / 8-page paper", "title and authors" ], "quote": "Verifying your browser | OpenReview", "notes": "Both WebFetch and Bash curl (browser UA) returned only the Cloudflare Turnstile 'Verifying your browser' interstitial. The forum content loads client-side from api2.openreview.net, which is outside the allowed URL set (no other pages/hosts). No paper content could be retrieved, so the claim cannot be independently verified. Batch file carried a pre-set in_corpus_verdict of 'confirmed', but that is not confirmable from retrieved text here." } ] }, { "url": "https://pubmed.ncbi.nlm.nih.gov/19769648", "retrieval": "abstract", "fetched": [ "https://pubmed.ncbi.nlm.nih.gov/19769648" ], "resolved_title": "The effect of differential rater function over time (DRIFT) on objective structured clinical examination ratings", "resolved_date": "Med Educ. 2009 Oct;43(10):989-92", "title_match": true, "claims": [ { "id": "CG-05", "verdict": "confirmed", "checked": [ "DRIFT in OSCE ratings; later time-slots rated higher", "regression coefficient 0.88, 95% CI 0.38-1.38, P=0.001", "difficult stations coefficient 1.24 vs 0.52 for less difficult (ratio ~2.4x)", "removing the first two (warm-up) stations did not correct DRIFT", "Med Educ 2009 Oct;43(10):989-92", "authors McLaughlin, Ainslie, Coderre, Wright, Violato" ], "not_visible": [ "secondary citations in the claim's evidence (Wolfe et al. 2001; Myford & Wolfe; Lunz & Stahl 1990) - external to this source, not on this page" ], "quote": "Removing the first two stations from our analyses did not correct DRIFT.", "notes": "All primary-source specifics confirmed. Minor caveat: the 0.52 difficult-vs-easy comparison coefficient (less difficult stations) was not statistically significant (P=0.09); the '2.4x' is the claimant's derived ratio of the two reported coefficients (1.24/0.52). Paper title uses 'differential rater function over time'." } ] }, { "url": "https://doi.org/10.1111/1467-8721.ep10772989", "retrieval": "landing", "fetched": [ "https://doi.org/10.1111/1467-8721.ep10772989", "https://journals.sagepub.com/doi/10.1111/1467-8721.ep10772989" ], "resolved_title": "Feedback Interventions: Toward the Understanding of a Double-Edged Sword", "resolved_date": "Current Directions in Psychological Science, Vol. 7(3), pp. 67-72, June 1998", "title_match": true, "claims": [ { "id": "CG-19", "verdict": "partially_confirmed", "checked": [ "Kluger & DeNisi 1998 article in Current Directions in Psychological Science (Vol 7, Issue 3, pp 67-72, June 1998)", "cites the 1996 Psychological Bulletin meta-analysis (Kluger & DeNisi 1996, vol 119, pp 254-284)" ], "not_visible": [ "131 studies", "~12,000+ participants", "mean d approximately 0.38-0.41", "phrase 'reduce performance in more than one third of the cases'", "attention shifting task->self mechanism (praise, normative comparison)" ], "quote": "Feedback Interventions: Toward the Understanding of a Double-Edged Sword", "notes": "DOI 302-redirects (server-side) to journals.sagepub.com, followed per allowed-variant rule. Page is restricted access: only title/authors/metadata and the 1996 citation are visible; no abstract or body text. The article's identity and the 1996 meta-analysis citation are confirmed, but every headline number and the 'more than one third reduce performance' phrase are behind the paywall and not visible in retrieved text." } ] } ] }