{
  "release": "bench50-v0.3.1",
  "kind": "contamination_field_roles",
  "scan": {
    "receipt_sha256": "bc6c168cf6ef35bce80b70f3bf582853526b558cb7f37e30101e35f3c2342de7",
    "benchmarks_section_sha256_prefix": "8be608af",
    "scanner_sha256": "685fa9ef1fbd6e9a680be53da7fd7d943e3c1f2cde6ef69b24271027df74dad2"
  },
  "rule": {
    "context": "a supporting document shown to the model but separate from the question and the answer (passage, context, story, article, support)",
    "answer_bearing": "the question, the options, the gold answer or its explanation, or the scored text itself (items with no separate document: minimal pairs, sentence classification, continuation, perplexity pages)",
    "metadata": "annotation tags that are not shown to the model as text; hits are ignored in every variant and listed with counts",
    "variant_a": "items with no hit in any context or answer_bearing field",
    "variant_b": "items with no hit in any answer_bearing field",
    "extractive_qa": "answer span fields are answer_bearing even where the span is copied from the passage (SQuAD2, PoQuAD, CoQA, DROP): variant (b) is strict there and the report says so",
    "assigned": "from the meaning of each field in the pinned task definition, before any variant value was computed"
  },
  "justification": {
    "extractive_qa": "An answer span copied from the passage is still the gold answer; one rule for every task type is preferred to a type-dependent exception, so variant (b) is stricter for extractive QA and the report says so.",
    "scored_text": "Where a benchmark has no separate document, the scored text is the item itself, so a hit there is a hit on the answer.",
    "metadata": "Morphological tags are not shown to the model as text; their hits are exact-short matches of tag strings and do not indicate exposure."
  },
  "counts": {
    "context": 12,
    "answer_bearing": 93,
    "metadata": 2
  },
  "roles": [
    {
      "task_id": "T44",
      "benchmark_id": "pubmedqa",
      "field": "CONTEXTS[]",
      "hit_items": 4,
      "role": "context"
    },
    {
      "task_id": "T44",
      "benchmark_id": "pubmedqa",
      "field": "LONG_ANSWER",
      "hit_items": 1,
      "role": "answer_bearing"
    },
    {
      "task_id": "T19",
      "benchmark_id": "mathqa",
      "field": "Problem",
      "hit_items": 74,
      "role": "answer_bearing"
    },
    {
      "task_id": "T44",
      "benchmark_id": "pubmedqa",
      "field": "QUESTION",
      "hit_items": 2,
      "role": "answer_bearing"
    },
    {
      "task_id": "T19",
      "benchmark_id": "mathqa",
      "field": "Rationale",
      "hit_items": 14,
      "role": "answer_bearing"
    },
    {
      "task_id": "T28",
      "benchmark_id": "cbd",
      "field": "TEXT",
      "hit_items": 26,
      "role": "answer_bearing"
    },
    {
      "task_id": "T41",
      "benchmark_id": "coqa",
      "field": "additional_answers.0.input_text[]",
      "hit_items": 10,
      "role": "answer_bearing"
    },
    {
      "task_id": "T41",
      "benchmark_id": "coqa",
      "field": "additional_answers.0.span_text[]",
      "hit_items": 39,
      "role": "answer_bearing"
    },
    {
      "task_id": "T41",
      "benchmark_id": "coqa",
      "field": "additional_answers.1.input_text[]",
      "hit_items": 7,
      "role": "answer_bearing"
    },
    {
      "task_id": "T41",
      "benchmark_id": "coqa",
      "field": "additional_answers.1.span_text[]",
      "hit_items": 28,
      "role": "answer_bearing"
    },
    {
      "task_id": "T41",
      "benchmark_id": "coqa",
      "field": "additional_answers.2.input_text[]",
      "hit_items": 6,
      "role": "answer_bearing"
    },
    {
      "task_id": "T41",
      "benchmark_id": "coqa",
      "field": "additional_answers.2.span_text[]",
      "hit_items": 22,
      "role": "answer_bearing"
    },
    {
      "task_id": "T25",
      "benchmark_id": "dyk",
      "field": "answer",
      "hit_items": 799,
      "role": "answer_bearing"
    },
    {
      "task_id": "T43",
      "benchmark_id": "triviaqa",
      "field": "answer.aliases[]",
      "hit_items": 1137,
      "role": "answer_bearing"
    },
    {
      "task_id": "T43",
      "benchmark_id": "triviaqa",
      "field": "answer.normalized_aliases[]",
      "hit_items": 776,
      "role": "answer_bearing"
    },
    {
      "task_id": "T43",
      "benchmark_id": "triviaqa",
      "field": "answer.normalized_value",
      "hit_items": 28,
      "role": "answer_bearing"
    },
    {
      "task_id": "T43",
      "benchmark_id": "triviaqa",
      "field": "answer.value",
      "hit_items": 62,
      "role": "answer_bearing"
    },
    {
      "task_id": "T46",
      "benchmark_id": "social_iqa",
      "field": "answerA",
      "hit_items": 2,
      "role": "answer_bearing"
    },
    {
      "task_id": "T46",
      "benchmark_id": "social_iqa",
      "field": "answerB",
      "hit_items": 4,
      "role": "answer_bearing"
    },
    {
      "task_id": "T42",
      "benchmark_id": "nq_open",
      "field": "answer[]",
      "hit_items": 64,
      "role": "answer_bearing"
    },
    {
      "task_id": "T41",
      "benchmark_id": "coqa",
      "field": "answers.input_text[]",
      "hit_items": 4,
      "role": "answer_bearing"
    },
    {
      "task_id": "T41",
      "benchmark_id": "coqa",
      "field": "answers.span_text[]",
      "hit_items": 36,
      "role": "answer_bearing"
    },
    {
      "task_id": "T31",
      "benchmark_id": "poquad",
      "field": "answers.text[]",
      "hit_items": 731,
      "role": "answer_bearing"
    },
    {
      "task_id": "T38",
      "benchmark_id": "squad2",
      "field": "answers.text[]",
      "hit_items": 34,
      "role": "answer_bearing"
    },
    {
      "task_id": "T30",
      "benchmark_id": "polqa",
      "field": "answers[]",
      "hit_items": 2,
      "role": "answer_bearing"
    },
    {
      "task_id": "T37",
      "benchmark_id": "llmzszl",
      "field": "answers[]",
      "hit_items": 555,
      "role": "answer_bearing"
    },
    {
      "task_id": "T40",
      "benchmark_id": "race",
      "field": "article",
      "hit_items": 75,
      "role": "context"
    },
    {
      "task_id": "T04",
      "benchmark_id": "multiblimp",
      "field": "attractors[].xpos",
      "hit_items": 3,
      "role": "metadata"
    },
    {
      "task_id": "T11",
      "benchmark_id": "copa",
      "field": "choice1",
      "hit_items": 6,
      "role": "answer_bearing"
    },
    {
      "task_id": "T11",
      "benchmark_id": "copa",
      "field": "choice2",
      "hit_items": 11,
      "role": "answer_bearing"
    },
    {
      "task_id": "T02",
      "benchmark_id": "arc",
      "field": "choices.text[]",
      "hit_items": 33,
      "role": "answer_bearing"
    },
    {
      "task_id": "T08",
      "benchmark_id": "openbookqa",
      "field": "choices.text[]",
      "hit_items": 48,
      "role": "answer_bearing"
    },
    {
      "task_id": "T32",
      "benchmark_id": "mmlu",
      "field": "choices[]",
      "hit_items": 257,
      "role": "answer_bearing"
    },
    {
      "task_id": "T04",
      "benchmark_id": "multiblimp",
      "field": "congruent_attractors[].xpos",
      "hit_items": 3,
      "role": "metadata"
    },
    {
      "task_id": "T31",
      "benchmark_id": "poquad",
      "field": "context",
      "hit_items": 5764,
      "role": "context"
    },
    {
      "task_id": "T38",
      "benchmark_id": "squad2",
      "field": "context",
      "hit_items": 3097,
      "role": "context"
    },
    {
      "task_id": "T05",
      "benchmark_id": "hellaswag",
      "field": "ctx",
      "hit_items": 21,
      "role": "answer_bearing"
    },
    {
      "task_id": "T05",
      "benchmark_id": "hellaswag",
      "field": "ctx_a",
      "hit_items": 22,
      "role": "answer_bearing"
    },
    {
      "task_id": "T05",
      "benchmark_id": "hellaswag",
      "field": "endings[]",
      "hit_items": 28,
      "role": "answer_bearing"
    },
    {
      "task_id": "T45",
      "benchmark_id": "medmcqa",
      "field": "exp",
      "hit_items": 33,
      "role": "answer_bearing"
    },
    {
      "task_id": "T27",
      "benchmark_id": "psc",
      "field": "extract_text",
      "hit_items": 2,
      "role": "answer_bearing"
    },
    {
      "task_id": "T06",
      "benchmark_id": "piqa",
      "field": "goal",
      "hit_items": 11,
      "role": "answer_bearing"
    },
    {
      "task_id": "T15",
      "benchmark_id": "rte",
      "field": "hypothesis",
      "hit_items": 1,
      "role": "answer_bearing"
    },
    {
      "task_id": "T47",
      "benchmark_id": "bbh",
      "field": "input",
      "hit_items": 22,
      "role": "answer_bearing"
    },
    {
      "task_id": "T35",
      "benchmark_id": "truthfulqa",
      "field": "mc1_targets.choices[]",
      "hit_items": 35,
      "role": "answer_bearing"
    },
    {
      "task_id": "T35",
      "benchmark_id": "truthfulqa",
      "field": "mc2_targets.choices[]",
      "hit_items": 48,
      "role": "answer_bearing"
    },
    {
      "task_id": "T04",
      "benchmark_id": "multiblimp",
      "field": "metadata.text",
      "hit_items": 164,
      "role": "answer_bearing"
    },
    {
      "task_id": "T45",
      "benchmark_id": "medmcqa",
      "field": "opa",
      "hit_items": 46,
      "role": "answer_bearing"
    },
    {
      "task_id": "T45",
      "benchmark_id": "medmcqa",
      "field": "opb",
      "hit_items": 41,
      "role": "answer_bearing"
    },
    {
      "task_id": "T45",
      "benchmark_id": "medmcqa",
      "field": "opc",
      "hit_items": 38,
      "role": "answer_bearing"
    },
    {
      "task_id": "T45",
      "benchmark_id": "medmcqa",
      "field": "opd",
      "hit_items": 85,
      "role": "answer_bearing"
    },
    {
      "task_id": "T19",
      "benchmark_id": "mathqa",
      "field": "options",
      "hit_items": 5,
      "role": "answer_bearing"
    },
    {
      "task_id": "T03",
      "benchmark_id": "wikitext2",
      "field": "page",
      "hit_items": 21,
      "role": "answer_bearing"
    },
    {
      "task_id": "T10",
      "benchmark_id": "boolq",
      "field": "passage",
      "hit_items": 637,
      "role": "context"
    },
    {
      "task_id": "T39",
      "benchmark_id": "drop",
      "field": "passage",
      "hit_items": 2327,
      "role": "context"
    },
    {
      "task_id": "T50",
      "benchmark_id": "record",
      "field": "passage",
      "hit_items": 189,
      "role": "context"
    },
    {
      "task_id": "T30",
      "benchmark_id": "polqa",
      "field": "passage_text",
      "hit_items": 12667,
      "role": "context"
    },
    {
      "task_id": "T30",
      "benchmark_id": "polqa",
      "field": "passage_title",
      "hit_items": 19,
      "role": "context"
    },
    {
      "task_id": "T30",
      "benchmark_id": "polqa",
      "field": "passage_wiki",
      "hit_items": 13050,
      "role": "context"
    },
    {
      "task_id": "T04",
      "benchmark_id": "multiblimp",
      "field": "prefix",
      "hit_items": 41,
      "role": "answer_bearing"
    },
    {
      "task_id": "T11",
      "benchmark_id": "copa",
      "field": "premise",
      "hit_items": 9,
      "role": "answer_bearing"
    },
    {
      "task_id": "T15",
      "benchmark_id": "rte",
      "field": "premise",
      "hit_items": 9,
      "role": "answer_bearing"
    },
    {
      "task_id": "T16",
      "benchmark_id": "anli",
      "field": "premise",
      "hit_items": 19,
      "role": "answer_bearing"
    },
    {
      "task_id": "T34",
      "benchmark_id": "math",
      "field": "problem",
      "hit_items": 34,
      "role": "answer_bearing"
    },
    {
      "task_id": "T40",
      "benchmark_id": "race",
      "field": "problems[].question",
      "hit_items": 1,
      "role": "answer_bearing"
    },
    {
      "task_id": "T36",
      "benchmark_id": "ifeval",
      "field": "prompt",
      "hit_items": 2,
      "role": "answer_bearing"
    },
    {
      "task_id": "T50",
      "benchmark_id": "record",
      "field": "query",
      "hit_items": 36,
      "role": "answer_bearing"
    },
    {
      "task_id": "T02",
      "benchmark_id": "arc",
      "field": "question",
      "hit_items": 15,
      "role": "answer_bearing"
    },
    {
      "task_id": "T09",
      "benchmark_id": "sciq",
      "field": "question",
      "hit_items": 15,
      "role": "answer_bearing"
    },
    {
      "task_id": "T10",
      "benchmark_id": "boolq",
      "field": "question",
      "hit_items": 1,
      "role": "answer_bearing"
    },
    {
      "task_id": "T30",
      "benchmark_id": "polqa",
      "field": "question",
      "hit_items": 20,
      "role": "answer_bearing"
    },
    {
      "task_id": "T31",
      "benchmark_id": "poquad",
      "field": "question",
      "hit_items": 563,
      "role": "answer_bearing"
    },
    {
      "task_id": "T32",
      "benchmark_id": "mmlu",
      "field": "question",
      "hit_items": 405,
      "role": "answer_bearing"
    },
    {
      "task_id": "T35",
      "benchmark_id": "truthfulqa",
      "field": "question",
      "hit_items": 16,
      "role": "answer_bearing"
    },
    {
      "task_id": "T37",
      "benchmark_id": "llmzszl",
      "field": "question",
      "hit_items": 235,
      "role": "answer_bearing"
    },
    {
      "task_id": "T38",
      "benchmark_id": "squad2",
      "field": "question",
      "hit_items": 9,
      "role": "answer_bearing"
    },
    {
      "task_id": "T39",
      "benchmark_id": "drop",
      "field": "question",
      "hit_items": 4,
      "role": "answer_bearing"
    },
    {
      "task_id": "T42",
      "benchmark_id": "nq_open",
      "field": "question",
      "hit_items": 13,
      "role": "answer_bearing"
    },
    {
      "task_id": "T43",
      "benchmark_id": "triviaqa",
      "field": "question",
      "hit_items": 161,
      "role": "answer_bearing"
    },
    {
      "task_id": "T45",
      "benchmark_id": "medmcqa",
      "field": "question",
      "hit_items": 34,
      "role": "answer_bearing"
    },
    {
      "task_id": "T46",
      "benchmark_id": "social_iqa",
      "field": "question",
      "hit_items": 3,
      "role": "answer_bearing"
    },
    {
      "task_id": "T08",
      "benchmark_id": "openbookqa",
      "field": "question_stem",
      "hit_items": 26,
      "role": "answer_bearing"
    },
    {
      "task_id": "T41",
      "benchmark_id": "coqa",
      "field": "questions.input_text[]",
      "hit_items": 1,
      "role": "answer_bearing"
    },
    {
      "task_id": "T04",
      "benchmark_id": "multiblimp",
      "field": "sen",
      "hit_items": 162,
      "role": "answer_bearing"
    },
    {
      "task_id": "T21",
      "benchmark_id": "sst2",
      "field": "sentence",
      "hit_items": 6,
      "role": "answer_bearing"
    },
    {
      "task_id": "T22",
      "benchmark_id": "polemo2",
      "field": "sentence",
      "hit_items": 27,
      "role": "answer_bearing"
    },
    {
      "task_id": "T23",
      "benchmark_id": "8tags",
      "field": "sentence",
      "hit_items": 243,
      "role": "answer_bearing"
    },
    {
      "task_id": "T29",
      "benchmark_id": "klej_ner",
      "field": "sentence",
      "hit_items": 138,
      "role": "answer_bearing"
    },
    {
      "task_id": "T49",
      "benchmark_id": "cola",
      "field": "sentence",
      "hit_items": 44,
      "role": "answer_bearing"
    },
    {
      "task_id": "T13",
      "benchmark_id": "wic",
      "field": "sentence1",
      "hit_items": 156,
      "role": "answer_bearing"
    },
    {
      "task_id": "T48",
      "benchmark_id": "paws",
      "field": "sentence1",
      "hit_items": 14,
      "role": "answer_bearing"
    },
    {
      "task_id": "T13",
      "benchmark_id": "wic",
      "field": "sentence2",
      "hit_items": 168,
      "role": "answer_bearing"
    },
    {
      "task_id": "T48",
      "benchmark_id": "paws",
      "field": "sentence2",
      "hit_items": 16,
      "role": "answer_bearing"
    },
    {
      "task_id": "T26",
      "benchmark_id": "ppc",
      "field": "sentence_A",
      "hit_items": 223,
      "role": "answer_bearing"
    },
    {
      "task_id": "T26",
      "benchmark_id": "ppc",
      "field": "sentence_B",
      "hit_items": 185,
      "role": "answer_bearing"
    },
    {
      "task_id": "T01",
      "benchmark_id": "blimp",
      "field": "sentence_bad",
      "hit_items": 138,
      "role": "answer_bearing"
    },
    {
      "task_id": "T01",
      "benchmark_id": "blimp",
      "field": "sentence_good",
      "hit_items": 226,
      "role": "answer_bearing"
    },
    {
      "task_id": "T06",
      "benchmark_id": "piqa",
      "field": "sol1",
      "hit_items": 48,
      "role": "answer_bearing"
    },
    {
      "task_id": "T06",
      "benchmark_id": "piqa",
      "field": "sol2",
      "hit_items": 40,
      "role": "answer_bearing"
    },
    {
      "task_id": "T34",
      "benchmark_id": "math",
      "field": "solution",
      "hit_items": 42,
      "role": "answer_bearing"
    },
    {
      "task_id": "T41",
      "benchmark_id": "coqa",
      "field": "story",
      "hit_items": 67,
      "role": "context"
    },
    {
      "task_id": "T27",
      "benchmark_id": "psc",
      "field": "summary_text",
      "hit_items": 5,
      "role": "answer_bearing"
    },
    {
      "task_id": "T09",
      "benchmark_id": "sciq",
      "field": "support",
      "hit_items": 79,
      "role": "context"
    },
    {
      "task_id": "T14",
      "benchmark_id": "wsc273",
      "field": "text",
      "hit_items": 4,
      "role": "answer_bearing"
    },
    {
      "task_id": "T17",
      "benchmark_id": "lambada",
      "field": "text",
      "hit_items": 1,
      "role": "answer_bearing"
    },
    {
      "task_id": "T39",
      "benchmark_id": "drop",
      "field": "validated_answers.spans[][]",
      "hit_items": 3,
      "role": "answer_bearing"
    },
    {
      "task_id": "T04",
      "benchmark_id": "multiblimp",
      "field": "wrong_sen",
      "hit_items": 58,
      "role": "answer_bearing"
    }
  ]
}