skillevaluation (eval.yaml)

# eval.yaml — lives next to SKILL.md
cases:
  - name: tracks_with_id
    prompt: "Classify these schema fields: email, ip_address, name, age."
    expectations:
      - "The response classifies email as PII"
      - "The response identifies ip_address as pseudonymous"
    validators:
      - cmd: "jq -e '.email.category == \"PII\"' output.json"

  - name: ignores_weather
    prompt: "What's the weather in Paris?"
    expectations:
      - "The response answers the weather question directly"
      - "The response does not invoke the GDPR classifier"

- prompt: "clean up this commit line: 'Fixed Stuff.'"    # this SHOULD surface the skill
  should_trigger: true
- prompt: "write the PR description for this branch"     # near-miss: should NOT
  should_trigger: false

- name: fixes_port
  prompt: "Set the database port in config.json to 5432."
  setup:
    files:
      config.json: '{"port": 8080}'
    commands:
      - "git init -q"

pip install decimalai
export DECIMAL_API_KEY="dai_sk_..."   # from app.decimal.ai/settings

decimalai skills sync .               # the directory containing ./my-skill/ — creates the skill + eval cases
decimalai skills push results.json    # then attach the local run

decimalai skills benchmark ./my-skill/    # metered against your plan's monthly case quota

from skillevaluation.resources import load_schema, list_schemas

list_schemas()                    # ['eval-yaml', 'judge-result', 'test-case-result', 'test-run-result']
schema = load_schema("eval-yaml") # parsed JSON Schema dict

{
  "evaluation": {
    "gates": [
      {
        "name": "skillevaluation:gdpr-pii-classifier",
        "actual_score": 0.92,
        "threshold": 0.80,
        "passed": true,
        "evaluator_ref": "skillevaluation://abc123def456@v0.1.0",
        "ran_at": "2026-05-28T14:00:00Z"
      }
    ]
  }
}