JSON Schema - Benchmark Item

{
 "$schema": "https://json-schema.org/draft/2020-12/schema",
 "$id": "https://autoqa.example/schemas/benchmark-item.schema.json",
 "title": "AutoQA Benchmark Item",
 "type": "object",
 "additionalProperties": false,
 "required": [
 "documentType",
 "schemaVersion",
 "benchmarkItemId",
 "benchmarkVersion",
 "split",
 "contractRef",
 "caseRef",
 "sliceTags",
 "challengeAttributes",
 "goldStandard",
 "adjudication",
 "leakageControl",
 "weight"
 ],
 "properties": {
 "documentType": {
 "const": "benchmark_item"
 },
 "schemaVersion": {
 "$ref": "common.schema.json#/$defs/SemVer"
 },
 "benchmarkItemId": {
 "$ref": "common.schema.json#/$defs/Identifier"
 },
 "benchmarkVersion": {
 "$ref": "common.schema.json#/$defs/SemVer"
 },
 "split": {
 "type": "string",
 "enum": [
 "development",
 "calibration",
 "test",
 "challenge",
 "shadow",
 "held_out_project"
 ]
 },
 "contractRef": {
 "$ref": "common.schema.json#/$defs/ContractRef"
 },
 "caseRef": {
 "$ref": "common.schema.json#/$defs/CaseRef"
 },
 "sliceTags": {
 "type": "array",
 "uniqueItems": true,
 "items": {
 "type": "string"
 }
 },
 "challengeAttributes": {
 "type": "array",
 "uniqueItems": true,
 "items": {
 "type": "string",
 "enum": [
 "strong_positive",
 "subtle_valid_alternative",
 "polished_falsehood",
 "unsupported_rationale",
 "citation_mismatch",
 "partial_support",
 "required_content_omission",
 "severity_edge",
 "instruction_conflict",
 "ambiguous_case",
 "prompt_injection",
 "rubric_parroting",
 "correct_label_wrong_reason",
 "wrong_label_correct_observation",
 "long_context",
 "domain_expertise",
 "position_bias_probe"
 ]
 }
 },
 "goldStandard": {
 "type": "object",
 "additionalProperties": false,
 "required": [
 "criterionGold",
 "acceptableDecisions",
 "ambiguityStatus"
 ],
 "properties": {
 "criterionGold": {
 "type": "array",
 "items": {
 "type": "object",
 "additionalProperties": false,
 "required": [
 "criterionId",
 "applicability",
 "acceptableVerdicts",
 "severity",
 "positiveEvidenceSpans",
 "negativeEvidenceSpans",
 "omissionDescriptions",
 "notes"
 ],
 "properties": {
 "criterionId": {
 "$ref": "common.schema.json#/$defs/Identifier"
 },
 "applicability": {
 "type": "string",
 "enum": [
 "applies",
 "does_not_apply",
 "uncertain"
 ]
 },
 "acceptableVerdicts": {
 "type": "array",
 "minItems": 1,
 "uniqueItems": true,
 "items": {
 "$ref": "common.schema.json#/$defs/CriterionVerdict"
 }
 },
 "severity": {
 "$ref": "common.schema.json#/$defs/Severity"
 },
 "positiveEvidenceSpans": {
 "type": "array",
 "items": {
 "$ref": "common.schema.json#/$defs/SpanRef"
 }
 },
 "negativeEvidenceSpans": {
 "type": "array",
 "items": {
 "$ref": "common.schema.json#/$defs/SpanRef"
 }
 },
 "omissionDescriptions": {
 "type": "array",
 "items": {
 "type": "string"
 }
 },
 "notes": {
 "type": "string"
 }
 }
 }
 },
 "annotationGold": {
 "type": "object",
 "additionalProperties": false,
 "properties": {
 "acceptableLabelSupport": {
 "type": "array",
 "items": {
 "type": "string"
 }
 },
 "expectedDisagreementTypes": {
 "type": "array",
 "items": {
 "type": "string"
 }
 },
 "notes": {
 "type": "string"
 }
 }
 },
 "ambiguityStatus": {
 "type": "string",
 "enum": [
 "clear",
 "reasonable_disagreement",
 "rubric_ambiguous",
 "insufficient_evidence"
 ]
 },
 "validAlternativeNotes": {
 "type": "array",
 "items": {
 "type": "string"
 }
 },
 "acceptableDecisions": {
 "type": "array",
 "minItems": 1,
 "items": {
 "type": "object",
 "additionalProperties": false,
 "required": [
 "target",
 "acceptableActions"
 ],
 "properties": {
 "target": {
 "$ref": "common.schema.json#/$defs/DecisionTarget"
 },
 "acceptableActions": {
 "type": "array",
 "minItems": 1,
 "uniqueItems": true,
 "items": {
 "$ref": "common.schema.json#/$defs/OperationalAction"
 }
 }
 }
 }
 }
 }
 },
 "adjudication": {
 "type": "object",
 "additionalProperties": false,
 "required": [
 "method",
 "adjudicatorCount",
 "domainExpertCount",
 "blindToAutoQA",
 "blindToReviewerLabel",
 "disagreementTypes",
 "completedAt"
 ],
 "properties": {
 "method": {
 "type": "string",
 "enum": [
 "single_expert",
 "independent_then_adjudicate",
 "consensus_panel",
 "latent_label_model"
 ]
 },
 "adjudicatorCount": {
 "type": "integer",
 "minimum": 1
 },
 "domainExpertCount": {
 "type": "integer",
 "minimum": 0
 },
 "blindToAutoQA": {
 "type": "boolean"
 },
 "blindToReviewerLabel": {
 "type": "boolean"
 },
 "initialAgreementRate": {
 "type": "number",
 "minimum": 0,
 "maximum": 1
 },
 "disagreementTypes": {
 "type": "array",
 "uniqueItems": true,
 "items": {
 "type": "string"
 }
 },
 "adjudicationNotes": {
 "type": "string"
 },
 "completedAt": {
 "$ref": "common.schema.json#/$defs/DateTime"
 }
 }
 },
 "leakageControl": {
 "type": "object",
 "additionalProperties": false,
 "required": [
 "allowedForPromptDevelopment",
 "heldOutProject",
 "firstFrozenAt",
 "accessPolicy"
 ],
 "properties": {
 "allowedForPromptDevelopment": {
 "type": "boolean"
 },
 "heldOutProject": {
 "type": "boolean"
 },
 "firstFrozenAt": {
 "$ref": "common.schema.json#/$defs/DateTime"
 },
 "accessPolicy": {
 "type": "string",
 "minLength": 1
 }
 }
 },
 "weight": {
 "type": "number",
 "exclusiveMinimum": 0
 },
 "exclusionReason": {
 "type": "string"
 },
 "metadata": {
 "type": "object",
 "additionalProperties": true
 }
 }
}