{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "urn:fabryka-ai:benchmark-results:2:protocol-0.3.1",
  "type": "object",
  "properties": {
    "schema_version": {
      "const": 2
    },
    "protocol_id": {
      "enum": [
        "bench50-v0.3.1-panel200",
        "bench50-v0.3.1-fullB"
      ]
    },
    "run_id": {
      "type": "string",
      "minLength": 1
    },
    "models": {
      "type": "array",
      "items": {
        "type": "object",
        "properties": {
          "model_id": {
            "type": "string",
            "minLength": 1
          },
          "repo": {
            "type": "string",
            "minLength": 1
          },
          "revision": {
            "type": [
              "string",
              "null"
            ]
          },
          "weights_sha256": {
            "anyOf": [
              {
                "type": "string",
                "pattern": "^[0-9a-f]{64}$"
              },
              {
                "type": "null"
              }
            ]
          }
        },
        "required": [
          "model_id",
          "repo",
          "revision",
          "weights_sha256"
        ],
        "additionalProperties": false
      }
    },
    "tasks": {
      "type": "array",
      "items": {
        "type": "object",
        "properties": {
          "task_id": {
            "type": "string",
            "pattern": "^T(0[1-9]|[1-4][0-9]|50)$"
          },
          "task": {
            "type": "string",
            "minLength": 1
          },
          "family": {
            "type": "string",
            "minLength": 1
          },
          "language": {
            "enum": [
              "PL",
              "EN"
            ]
          },
          "split": {
            "type": "string",
            "minLength": 1
          }
        },
        "required": [
          "task_id",
          "task",
          "family",
          "language",
          "split"
        ],
        "additionalProperties": false
      }
    },
    "observations": {
      "type": "array",
      "items": {
        "$ref": "#/$defs/observation"
      }
    },
    "contrasts": {
      "type": "array",
      "items": {
        "$ref": "#/$defs/contrast"
      }
    },
    "provenance": {
      "type": "object",
      "properties": {
        "inventory_commit": {
          "type": "string",
          "pattern": "^[0-9a-f]{40}$"
        },
        "kind": {
          "enum": [
            "measurement",
            "contract_example"
          ]
        },
        "sources": {
          "type": "array",
          "minItems": 1,
          "items": {
            "type": "object",
            "properties": {
              "run_id": {
                "type": "string",
                "minLength": 1
              },
              "results_sha256": {
                "type": "string",
                "pattern": "^[0-9a-f]{64}$"
              }
            },
            "required": [
              "run_id",
              "results_sha256"
            ],
            "additionalProperties": false
          }
        },
        "qualification_sha256": {
          "type": "string",
          "pattern": "^[0-9a-f]{64}$"
        },
        "contamination_sha256": {
          "type": "string",
          "pattern": "^[0-9a-f]{64}$"
        },
        "references_sha256": {
          "type": "string",
          "pattern": "^[0-9a-f]{64}$"
        },
        "superseded": {
          "type": "array",
          "items": {
            "type": "object",
            "properties": {
              "model_id": {
                "type": "string",
                "minLength": 1
              },
              "task_id": {
                "type": "string",
                "pattern": "^T(0[1-9]|[1-4][0-9]|50)$"
              },
              "by_results_sha256": {
                "type": "string",
                "pattern": "^[0-9a-f]{64}$"
              },
              "replaced": {
                "type": "array",
                "minItems": 1,
                "items": {
                  "type": "object",
                  "properties": {
                    "state": {
                      "enum": [
                        "not_run",
                        "unsupported",
                        "blocked_access",
                        "blocked_runtime",
                        "blocked_security",
                        "blocked_judge"
                      ]
                    },
                    "results_sha256": {
                      "type": "string",
                      "pattern": "^[0-9a-f]{64}$"
                    }
                  },
                  "required": [
                    "state",
                    "results_sha256"
                  ],
                  "additionalProperties": false
                }
              }
            },
            "required": [
              "model_id",
              "task_id",
              "by_results_sha256",
              "replaced"
            ],
            "additionalProperties": false
          }
        },
        "contrasts_sha256": {
          "type": "string",
          "pattern": "^[0-9a-f]{64}$"
        },
        "amendments": {
          "type": "array",
          "items": {
            "type": "object",
            "properties": {
              "model_id": {
                "type": "string",
                "minLength": 1
              },
              "task_id": {
                "type": "string",
                "pattern": "^T(0[1-9]|[1-4][0-9]|50)$"
              },
              "code": {
                "type": "string",
                "minLength": 1
              },
              "base_items_sha256": {
                "type": "string",
                "pattern": "^[0-9a-f]{64}$"
              },
              "artifact_sha256": {
                "type": "string",
                "pattern": "^[0-9a-f]{64}$"
              }
            },
            "required": [
              "model_id",
              "task_id",
              "code",
              "base_items_sha256",
              "artifact_sha256"
            ],
            "additionalProperties": false
          }
        }
      },
      "required": [
        "inventory_commit",
        "kind"
      ],
      "additionalProperties": false,
      "allOf": [
        {
          "if": {
            "properties": {
              "kind": {
                "const": "measurement"
              }
            }
          },
          "then": {
            "required": [
              "sources",
              "qualification_sha256",
              "contamination_sha256",
              "references_sha256"
            ]
          }
        }
      ]
    }
  },
  "required": [
    "schema_version",
    "protocol_id",
    "run_id",
    "models",
    "tasks",
    "observations",
    "contrasts",
    "provenance"
  ],
  "additionalProperties": false,
  "$defs": {
    "ci95": {
      "type": "object",
      "properties": {
        "lower": {
          "type": "number"
        },
        "upper": {
          "type": "number"
        },
        "level": {
          "const": 0.95
        },
        "method": {
          "enum": [
            "paired_bootstrap",
            "paired_cluster_bootstrap"
          ]
        },
        "replicates": {
          "const": 10000
        },
        "seed": {
          "const": 20261028
        }
      },
      "required": [
        "lower",
        "upper",
        "level",
        "method",
        "replicates",
        "seed"
      ],
      "additionalProperties": false
    },
    "observation": {
      "type": "object",
      "properties": {
        "task_id": {
          "type": "string",
          "pattern": "^T(0[1-9]|[1-4][0-9]|50)$"
        },
        "model_id": {
          "type": "string",
          "minLength": 1
        },
        "protocol_id": {
          "enum": [
            "bench50-v0.3.1-panel200",
            "bench50-v0.3.1-fullB"
          ]
        },
        "metric": {
          "type": "string",
          "minLength": 1
        },
        "unit": {
          "enum": [
            "percent",
            "score",
            "byte_ppl"
          ]
        },
        "higher_is_better": {
          "type": "boolean"
        },
        "n_population": {
          "type": [
            "integer",
            "null"
          ],
          "minimum": 0
        },
        "n_selected": {
          "type": [
            "integer",
            "null"
          ],
          "minimum": 0
        },
        "n_scored": {
          "type": "integer",
          "minimum": 0
        },
        "state": {
          "enum": [
            "ok",
            "not_run",
            "blocked_access",
            "blocked_runtime",
            "blocked_security",
            "blocked_judge",
            "unsupported",
            "failed",
            "skipped"
          ]
        },
        "value": {
          "type": [
            "number",
            "null"
          ]
        },
        "ci95": {
          "anyOf": [
            {
              "type": "object",
              "properties": {
                "lower": {
                  "type": "number"
                },
                "upper": {
                  "type": "number"
                },
                "level": {
                  "const": 0.95
                },
                "method": {
                  "enum": [
                    "paired_bootstrap",
                    "paired_cluster_bootstrap"
                  ]
                },
                "replicates": {
                  "const": 10000
                },
                "seed": {
                  "const": 20261028
                }
              },
              "required": [
                "lower",
                "upper",
                "level",
                "method",
                "replicates",
                "seed"
              ],
              "additionalProperties": false
            },
            {
              "type": "null"
            }
          ]
        },
        "source_id_digest": {
          "anyOf": [
            {
              "type": "string",
              "pattern": "^[0-9a-f]{64}$"
            },
            {
              "type": "null"
            }
          ]
        },
        "results_sha256": {
          "anyOf": [
            {
              "type": "string",
              "pattern": "^[0-9a-f]{64}$"
            },
            {
              "type": "null"
            }
          ]
        },
        "runner_sha256": {
          "anyOf": [
            {
              "type": "string",
              "pattern": "^[0-9a-f]{64}$"
            },
            {
              "type": "null"
            }
          ]
        },
        "reason": {
          "type": [
            "string",
            "null"
          ],
          "maxLength": 200
        },
        "contamination": {
          "enum": [
            "clean",
            "flagged",
            "partially_scanned",
            "unscanned",
            "unknown"
          ]
        },
        "contamination_receipt_sha256": {
          "type": [
            "string",
            "null"
          ],
          "pattern": "^[0-9a-f]{64}$"
        },
        "value_hit_free": {
          "type": [
            "number",
            "null"
          ]
        },
        "n_hit_free": {
          "type": [
            "integer",
            "null"
          ],
          "minimum": 0
        },
        "hit_rate": {
          "type": [
            "number",
            "null"
          ],
          "minimum": 0,
          "maximum": 1
        },
        "reference": {
          "type": [
            "object",
            "null"
          ],
          "properties": {
            "chance_value": {
              "type": [
                "number",
                "null"
              ],
              "description": "expected metric of uniform random choice on the frozen sample (mean 1/n_options for accuracy); null if not defined"
            },
            "majority_value": {
              "type": [
                "number",
                "null"
              ],
              "description": "same metric for always predicting the majority gold class of the frozen sample; null if classes are not fixed"
            },
            "majority_prediction_share": {
              "type": [
                "number",
                "null"
              ],
              "minimum": 0,
              "maximum": 1,
              "description": "share of the model predictions falling on its most frequent predicted class"
            },
            "constant_prediction": {
              "type": [
                "boolean",
                "null"
              ],
              "description": "true when the model predicts one class for every scored item"
            }
          },
          "required": [
            "chance_value",
            "majority_value",
            "majority_prediction_share",
            "constant_prediction"
          ],
          "additionalProperties": false
        },
        "value_hit_free_answer_fields": {
          "type": [
            "number",
            "null"
          ]
        },
        "n_hit_free_answer_fields": {
          "type": [
            "integer",
            "null"
          ],
          "minimum": 0
        },
        "hit_rate_answer_fields": {
          "type": [
            "number",
            "null"
          ],
          "minimum": 0,
          "maximum": 1
        },
        "ci95_hit_free": {
          "anyOf": [
            {
              "type": "object",
              "properties": {
                "lower": {
                  "type": "number"
                },
                "upper": {
                  "type": "number"
                },
                "level": {
                  "const": 0.95
                },
                "method": {
                  "enum": [
                    "paired_bootstrap",
                    "paired_cluster_bootstrap"
                  ]
                },
                "replicates": {
                  "const": 10000
                },
                "seed": {
                  "const": 20261028
                }
              },
              "required": [
                "lower",
                "upper",
                "level",
                "method",
                "replicates",
                "seed"
              ],
              "additionalProperties": false
            },
            {
              "type": "null"
            }
          ]
        },
        "ci95_hit_free_answer_fields": {
          "anyOf": [
            {
              "type": "object",
              "properties": {
                "lower": {
                  "type": "number"
                },
                "upper": {
                  "type": "number"
                },
                "level": {
                  "const": 0.95
                },
                "method": {
                  "enum": [
                    "paired_bootstrap",
                    "paired_cluster_bootstrap"
                  ]
                },
                "replicates": {
                  "const": 10000
                },
                "seed": {
                  "const": 20261028
                }
              },
              "required": [
                "lower",
                "upper",
                "level",
                "method",
                "replicates",
                "seed"
              ],
              "additionalProperties": false
            },
            {
              "type": "null"
            }
          ]
        }
      },
      "required": [
        "task_id",
        "model_id",
        "protocol_id",
        "metric",
        "unit",
        "higher_is_better",
        "n_population",
        "n_selected",
        "n_scored",
        "state",
        "value",
        "ci95",
        "source_id_digest",
        "results_sha256",
        "runner_sha256",
        "contamination",
        "contamination_receipt_sha256"
      ],
      "additionalProperties": false,
      "allOf": [
        {
          "if": {
            "properties": {
              "state": {
                "const": "ok"
              }
            }
          },
          "then": {
            "properties": {
              "value": {
                "type": "number"
              },
              "n_scored": {
                "type": "integer",
                "minimum": 1
              },
              "source_id_digest": {
                "type": "string",
                "pattern": "^[0-9a-f]{64}$"
              },
              "results_sha256": {
                "type": "string",
                "pattern": "^[0-9a-f]{64}$"
              },
              "runner_sha256": {
                "type": "string",
                "pattern": "^[0-9a-f]{64}$"
              }
            }
          },
          "else": {
            "properties": {
              "value": {
                "type": "null"
              },
              "ci95": {
                "type": "null"
              }
            }
          }
        },
        {
          "if": {
            "properties": {
              "contamination": {
                "const": "flagged"
              },
              "state": {
                "const": "ok"
              }
            },
            "required": [
              "contamination",
              "state"
            ]
          },
          "then": {
            "required": [
              "value_hit_free",
              "n_hit_free",
              "hit_rate",
              "value_hit_free_answer_fields",
              "n_hit_free_answer_fields",
              "hit_rate_answer_fields"
            ],
            "properties": {
              "n_hit_free": {
                "type": "integer"
              },
              "hit_rate": {
                "type": "number"
              },
              "n_hit_free_answer_fields": {
                "type": "integer"
              },
              "hit_rate_answer_fields": {
                "type": "number"
              }
            }
          }
        },
        {
          "if": {
            "properties": {
              "contamination": {
                "enum": [
                  "clean",
                  "flagged",
                  "partially_scanned"
                ]
              }
            }
          },
          "then": {
            "properties": {
              "contamination_receipt_sha256": {
                "type": "string"
              }
            }
          }
        },
        {
          "if": {
            "properties": {
              "n_hit_free": {
                "const": 0
              }
            },
            "required": [
              "n_hit_free"
            ]
          },
          "then": {
            "properties": {
              "value_hit_free": {
                "type": "null"
              }
            }
          },
          "else": {
            "if": {
              "properties": {
                "contamination": {
                  "const": "flagged"
                },
                "state": {
                  "const": "ok"
                }
              },
              "required": [
                "contamination",
                "state"
              ]
            },
            "then": {
              "properties": {
                "value_hit_free": {
                  "type": "number"
                }
              }
            }
          }
        },
        {
          "if": {
            "properties": {
              "n_hit_free_answer_fields": {
                "const": 0
              }
            },
            "required": [
              "n_hit_free_answer_fields"
            ]
          },
          "then": {
            "properties": {
              "value_hit_free_answer_fields": {
                "type": "null"
              }
            }
          },
          "else": {
            "if": {
              "properties": {
                "contamination": {
                  "const": "flagged"
                },
                "state": {
                  "const": "ok"
                }
              },
              "required": [
                "contamination",
                "state"
              ]
            },
            "then": {
              "properties": {
                "value_hit_free_answer_fields": {
                  "type": "number"
                }
              }
            }
          }
        }
      ]
    },
    "contrast": {
      "type": "object",
      "properties": {
        "task_id": {
          "type": "string",
          "pattern": "^T(0[1-9]|[1-4][0-9]|50)$"
        },
        "model_id": {
          "type": "string",
          "minLength": 1
        },
        "protocol_id": {
          "enum": [
            "bench50-v0.3.1-panel200",
            "bench50-v0.3.1-fullB"
          ]
        },
        "model_ref": {
          "type": "string",
          "minLength": 1
        },
        "delta": {
          "type": [
            "number",
            "null"
          ]
        },
        "ci95": {
          "anyOf": [
            {
              "type": "object",
              "properties": {
                "lower": {
                  "type": "number"
                },
                "upper": {
                  "type": "number"
                },
                "level": {
                  "const": 0.95
                },
                "method": {
                  "enum": [
                    "paired_bootstrap",
                    "paired_cluster_bootstrap"
                  ]
                },
                "replicates": {
                  "const": 10000
                },
                "seed": {
                  "const": 20261028
                }
              },
              "required": [
                "lower",
                "upper",
                "level",
                "method",
                "replicates",
                "seed"
              ],
              "additionalProperties": false
            },
            {
              "type": "null"
            }
          ]
        },
        "p_raw": {
          "type": [
            "number",
            "null"
          ],
          "minimum": 0,
          "maximum": 1
        },
        "p_holm": {
          "type": [
            "number",
            "null"
          ],
          "minimum": 0,
          "maximum": 1
        },
        "family_size": {
          "type": "integer",
          "minimum": 1
        },
        "verdict": {
          "enum": [
            "withheld",
            "better",
            "worse",
            "inconclusive"
          ]
        },
        "review": {
          "type": "object",
          "properties": {
            "status": {
              "enum": [
                "pending",
                "accepted",
                "rejected"
              ]
            },
            "actor": {
              "type": [
                "string",
                "null"
              ]
            },
            "artifact_sha256": {
              "anyOf": [
                {
                  "type": "string",
                  "pattern": "^[0-9a-f]{64}$"
                },
                {
                  "type": "null"
                }
              ]
            }
          },
          "required": [
            "status",
            "actor",
            "artifact_sha256"
          ],
          "additionalProperties": false,
          "allOf": [
            {
              "if": {
                "properties": {
                  "status": {
                    "const": "accepted"
                  }
                }
              },
              "then": {
                "properties": {
                  "actor": {
                    "type": "string",
                    "minLength": 1
                  },
                  "artifact_sha256": {
                    "type": "string",
                    "pattern": "^[0-9a-f]{64}$"
                  }
                }
              }
            }
          ]
        },
        "subset": {
          "enum": [
            "full",
            "hit_free",
            "hit_free_answer_fields"
          ],
          "description": "items on which the paired contrast was computed; hit-free subsets are defined by the training corpus of model_id and are identical for both models"
        },
        "n": {
          "type": [
            "integer",
            "null"
          ],
          "minimum": 0
        },
        "metric": {
          "type": "string",
          "minLength": 1
        },
        "unit": {
          "type": "string",
          "minLength": 1
        }
      },
      "required": [
        "task_id",
        "model_id",
        "protocol_id",
        "model_ref",
        "delta",
        "ci95",
        "p_raw",
        "p_holm",
        "family_size",
        "verdict",
        "review",
        "subset",
        "n",
        "metric",
        "unit"
      ],
      "additionalProperties": false,
      "allOf": [
        {
          "if": {
            "properties": {
              "verdict": {
                "enum": [
                  "better",
                  "worse",
                  "inconclusive"
                ]
              }
            }
          },
          "then": {
            "properties": {
              "review": {
                "properties": {
                  "status": {
                    "const": "accepted"
                  }
                }
              },
              "delta": {
                "type": "number"
              },
              "p_raw": {
                "type": "number"
              },
              "p_holm": {
                "type": "number"
              }
            }
          }
        },
        {
          "if": {
            "properties": {
              "review": {
                "properties": {
                  "status": {
                    "enum": [
                      "pending",
                      "rejected"
                    ]
                  }
                }
              }
            }
          },
          "then": {
            "properties": {
              "verdict": {
                "const": "withheld"
              }
            }
          }
        }
      ]
    }
  }
}