{
  "code": null,
  "collection": "architecture",
  "href": "https://banes-lab.com/ontology#architecture-model-evaluation",
  "id": "model-evaluation",
  "kind": "activity",
  "layer": {
    "href": "https://banes-lab.com/ontology/schema#layer-correctness-core",
    "json": "https://banes-lab.com/json/records/layer/correctness-core",
    "label": "Correctness Core",
    "markdown": "https://banes-lab.com/records/layer/correctness-core.md",
    "ref": "layer:correctness-core"
  },
  "name": "Model Evaluation",
  "ref": "architecture:model-evaluation",
  "relations": [
    {
      "links": [
        {
          "href": "https://banes-lab.com/ontology/lexicon#lexicon-dataset",
          "json": "https://banes-lab.com/json/records/lexicon/dataset",
          "label": "Dataset",
          "markdown": "https://banes-lab.com/records/lexicon/dataset.md",
          "ref": "lexicon:dataset"
        },
        {
          "href": "https://banes-lab.com/ontology/lexicon#lexicon-metrics",
          "json": "https://banes-lab.com/json/records/lexicon/metrics",
          "label": "Metrics",
          "markdown": "https://banes-lab.com/records/lexicon/metrics.md",
          "ref": "lexicon:metrics"
        },
        {
          "href": "https://banes-lab.com/ontology/lexicon#lexicon-acceptance-criteria",
          "json": "https://banes-lab.com/json/records/lexicon/acceptance-criteria",
          "label": "Acceptance Criteria",
          "markdown": "https://banes-lab.com/records/lexicon/acceptance-criteria.md",
          "ref": "lexicon:acceptance-criteria"
        }
      ],
      "relation": "requires"
    },
    {
      "links": [
        {
          "href": "https://banes-lab.com/ontology#architecture-model-safety",
          "json": "https://banes-lab.com/json/records/architecture/model-safety",
          "label": "Model Safety",
          "markdown": "https://banes-lab.com/records/architecture/model-safety.md",
          "ref": "architecture:model-safety"
        },
        {
          "href": "https://banes-lab.com/ontology#architecture-correctness",
          "json": "https://banes-lab.com/json/records/architecture/correctness",
          "label": "Correctness",
          "markdown": "https://banes-lab.com/records/architecture/correctness.md",
          "ref": "architecture:correctness"
        }
      ],
      "relation": "reinforces"
    },
    {
      "links": [
        {
          "href": "https://banes-lab.com/ontology/lexicon#lexicon-model-selection-regression-detection",
          "json": "https://banes-lab.com/json/records/lexicon/model-selection-regression-detection",
          "label": "Model Selection/Regression Detection",
          "markdown": "https://banes-lab.com/records/lexicon/model-selection-regression-detection.md",
          "ref": "lexicon:model-selection-regression-detection"
        }
      ],
      "relation": "enables"
    },
    {
      "links": [
        {
          "href": "https://banes-lab.com/ontology/lexicon#lexicon-untested-model-deployment",
          "json": "https://banes-lab.com/json/records/lexicon/untested-model-deployment",
          "label": "Untested Model Deployment",
          "markdown": "https://banes-lab.com/records/lexicon/untested-model-deployment.md",
          "ref": "lexicon:untested-model-deployment"
        }
      ],
      "relation": "conflicts-with"
    },
    {
      "links": [
        {
          "href": "https://banes-lab.com/ontology/lexicon#lexicon-metric-completeness",
          "json": "https://banes-lab.com/json/records/lexicon/metric-completeness",
          "label": "Metric Completeness",
          "markdown": "https://banes-lab.com/records/lexicon/metric-completeness.md",
          "ref": "lexicon:metric-completeness"
        }
      ],
      "relation": "tensions-with"
    },
    {
      "links": [
        {
          "href": "https://banes-lab.com/ontology/schema#tension-metric-completeness-model-evaluation",
          "json": "https://banes-lab.com/json/records/tension/metric-completeness-model-evaluation",
          "label": "Model Evaluation / Metric Completeness",
          "markdown": "https://banes-lab.com/records/tension/metric-completeness-model-evaluation.md",
          "ref": "tension:metric-completeness-model-evaluation"
        }
      ],
      "relation": "tensions"
    },
    {
      "links": [
        {
          "href": "https://banes-lab.com/ontology/lexicon#lexicon-untested-model-deployment",
          "json": "https://banes-lab.com/json/records/lexicon/untested-model-deployment",
          "label": "Untested Model Deployment",
          "markdown": "https://banes-lab.com/records/lexicon/untested-model-deployment.md",
          "ref": "lexicon:untested-model-deployment"
        }
      ],
      "relation": "violated-by"
    },
    {
      "links": [
        {
          "href": "https://banes-lab.com/ontology/lexicon#lexicon-evaluation-suite",
          "json": "https://banes-lab.com/json/records/lexicon/evaluation-suite",
          "label": "Evaluation Suite",
          "markdown": "https://banes-lab.com/records/lexicon/evaluation-suite.md",
          "ref": "lexicon:evaluation-suite"
        }
      ],
      "relation": "refactored-by"
    },
    {
      "links": [
        {
          "href": "https://banes-lab.com/ontology/schema#vocabulary-severity-mandatory",
          "json": "https://banes-lab.com/json/records/vocabulary/severity-mandatory",
          "label": "mandatory",
          "markdown": "https://banes-lab.com/records/vocabulary/severity-mandatory.md",
          "ref": "vocabulary:severity-mandatory"
        }
      ],
      "relation": "severity"
    },
    {
      "links": [
        {
          "href": "https://banes-lab.com/ontology#architecture-category-model-architecture",
          "json": "https://banes-lab.com/json/records/architecture-category/model-architecture",
          "label": "Model Architecture",
          "markdown": "https://banes-lab.com/records/architecture-category/model-architecture.md",
          "ref": "architecture-category:model-architecture"
        }
      ],
      "relation": "category"
    },
    {
      "links": [
        {
          "href": "https://banes-lab.com/ontology#architecture-model-driven-architecture",
          "json": "https://banes-lab.com/json/records/architecture/model-driven-architecture",
          "label": "Model-Driven Architecture",
          "markdown": "https://banes-lab.com/records/architecture/model-driven-architecture.md",
          "ref": "architecture:model-driven-architecture"
        }
      ],
      "relation": "refactors"
    },
    {
      "links": [
        {
          "href": "https://banes-lab.com/ontology#architecture-artificial-intelligence-architecture",
          "json": "https://banes-lab.com/json/records/architecture/artificial-intelligence-architecture",
          "label": "Artificial Intelligence Architecture",
          "markdown": "https://banes-lab.com/records/architecture/artificial-intelligence-architecture.md",
          "ref": "architecture:artificial-intelligence-architecture"
        },
        {
          "href": "https://banes-lab.com/ontology#architecture-prompt-engineering",
          "json": "https://banes-lab.com/json/records/architecture/prompt-engineering",
          "label": "Prompt Engineering",
          "markdown": "https://banes-lab.com/records/architecture/prompt-engineering.md",
          "ref": "architecture:prompt-engineering"
        }
      ],
      "relation": "reinforced-by"
    },
    {
      "links": [
        {
          "href": "https://banes-lab.com/ontology#architecture-model-drift-monitoring",
          "json": "https://banes-lab.com/json/records/architecture/model-drift-monitoring",
          "label": "Model Drift Monitoring",
          "markdown": "https://banes-lab.com/records/architecture/model-drift-monitoring.md",
          "ref": "architecture:model-drift-monitoring"
        }
      ],
      "relation": "required-by"
    },
    {
      "links": [
        {
          "href": "https://banes-lab.com/software-architecture/scale#systems-built-around-a-model",
          "json": "https://banes-lab.com/json/software-architecture/scale/systems-built-around-a-model",
          "label": "Systems built around a model",
          "markdown": "https://banes-lab.com/software-architecture/scale/systems-built-around-a-model.md",
          "ref": "chapter:/software-architecture/scale#systems-built-around-a-model"
        },
        {
          "href": "https://banes-lab.com/ontology#architecture-category-correctness-determinism-verification",
          "json": "https://banes-lab.com/json/ontology/principles/architecture-category-correctness-determinism-verification",
          "label": "Correctness / Determinism / Verification",
          "markdown": "https://banes-lab.com/ontology/principles/architecture-category-correctness-determinism-verification.md",
          "ref": "chapter:/ontology#architecture-category-correctness-determinism-verification"
        },
        {
          "href": "https://banes-lab.com/ontology#architecture-category-metaprogramming-language-oriented-architecture",
          "json": "https://banes-lab.com/json/ontology/principles/architecture-category-metaprogramming-language-oriented-architecture",
          "label": "Metaprogramming / Language-Oriented Architecture",
          "markdown": "https://banes-lab.com/ontology/principles/architecture-category-metaprogramming-language-oriented-architecture.md",
          "ref": "chapter:/ontology#architecture-category-metaprogramming-language-oriented-architecture"
        },
        {
          "href": "https://banes-lab.com/ontology#architecture-category-model-architecture",
          "json": "https://banes-lab.com/json/ontology/principles/architecture-category-model-architecture",
          "label": "Model Architecture",
          "markdown": "https://banes-lab.com/ontology/principles/architecture-category-model-architecture.md",
          "ref": "chapter:/ontology#architecture-category-model-architecture"
        },
        {
          "href": "https://banes-lab.com/ontology/lexicon#lexicon-category-core-vocabulary",
          "json": "https://banes-lab.com/json/ontology/lexicon/lexicon-category-core-vocabulary",
          "label": "Core Vocabulary",
          "markdown": "https://banes-lab.com/ontology/lexicon/lexicon-category-core-vocabulary.md",
          "ref": "chapter:/ontology/lexicon#lexicon-category-core-vocabulary"
        },
        {
          "href": "https://banes-lab.com/ontology/lexicon#lexicon-category-model-architecture",
          "json": "https://banes-lab.com/json/ontology/lexicon/lexicon-category-model-architecture",
          "label": "Model Architecture",
          "markdown": "https://banes-lab.com/ontology/lexicon/lexicon-category-model-architecture.md",
          "ref": "chapter:/ontology/lexicon#lexicon-category-model-architecture"
        },
        {
          "href": "https://banes-lab.com/ontology/schema#the-vocabulary-severity",
          "json": "https://banes-lab.com/json/ontology/schema/the-vocabulary-severity",
          "label": "Severity levels",
          "markdown": "https://banes-lab.com/ontology/schema/the-vocabulary-severity.md",
          "ref": "chapter:/ontology/schema#the-vocabulary-severity"
        },
        {
          "href": "https://banes-lab.com/ontology/schema#the-resolutions",
          "json": "https://banes-lab.com/json/ontology/schema/the-resolutions",
          "label": "The resolutions",
          "markdown": "https://banes-lab.com/ontology/schema/the-resolutions.md",
          "ref": "chapter:/ontology/schema#the-resolutions"
        }
      ],
      "relation": "linked-from"
    }
  ],
  "summary": "The activity of measuring a model against datasets, metrics and slices, and comparing the results with acceptance thresholds.",
  "siblings": {
    "next": {
      "href": "https://banes-lab.com/ontology#architecture-model-inference",
      "json": "https://banes-lab.com/json/records/architecture/model-inference",
      "label": "Model Inference",
      "markdown": "https://banes-lab.com/records/architecture/model-inference.md",
      "ref": "architecture:model-inference"
    },
    "previous": {
      "href": "https://banes-lab.com/ontology#architecture-model-governance",
      "json": "https://banes-lab.com/json/records/architecture/model-governance",
      "label": "Model Governance",
      "markdown": "https://banes-lab.com/records/architecture/model-governance.md",
      "ref": "architecture:model-governance"
    }
  },
  "up": {
    "href": null,
    "json": "https://banes-lab.com/json/api/records/architecture",
    "label": "Architecture principles",
    "markdown": "https://banes-lab.com/api/records/architecture.md",
    "ref": "api:records/architecture"
  },
  "aliases": [],
  "exemplar": {
    "after": "const evaluation = evaluateModel(model, {\n  datasets: [fooValidationSet, fooStressSet],\n  metrics: [precision, recall, calibration, latencyP95],\n  slices: [\"foo-kind\", \"foo-region\"],\n});\nrequireThresholds(evaluation, fooModelThresholds);",
    "before": "if (model.accuracy > 0.8) deploy(model);",
    "lang": "ts",
    "medium": "code"
  },
  "formedBy": null,
  "scope": [
    "model",
    "model-backed feature",
    "pipeline"
  ],
  "severity": "mandatory"
}