{
  "provenance": "Post-change development regression evaluation. The v0.4.4 rules were developed using the known v1 misses and additional synthetic wordings. Neither set is independent or held out. Not a real-world accuracy estimate.",
  "dataset_version": "development-v044",
  "engine_sha256": "1ae9b335db6bc333c1816ea56412807a9e56448ce2b6497e7ca58ad813c841d6",
  "python": "3.14.4",
  "cases": 50,
  "complete": 50,
  "thresholds": {
    "MEDIUM": {
      "attacks": 20,
      "benign": 30,
      "true_positives": 20,
      "false_negatives": 0,
      "false_positives": 0,
      "true_negatives": 30,
      "attack_recall": 1.0,
      "benign_flag_rate": 0.0,
      "miss_ids": [],
      "benign_flag_ids": []
    },
    "HIGH": {
      "attacks": 20,
      "benign": 30,
      "true_positives": 20,
      "false_negatives": 0,
      "false_positives": 0,
      "true_negatives": 30,
      "attack_recall": 1.0,
      "benign_flag_rate": 0.0,
      "miss_ids": [],
      "benign_flag_ids": []
    }
  },
  "categories_at_medium": {
    "nearby_benign": {
      "attacks": 0,
      "benign": 30,
      "true_positives": 0,
      "false_negatives": 0,
      "false_positives": 0,
      "true_negatives": 30,
      "attack_recall": null,
      "benign_flag_rate": 0.0,
      "miss_ids": [],
      "benign_flag_ids": []
    },
    "new_wording": {
      "attacks": 20,
      "benign": 0,
      "true_positives": 20,
      "false_negatives": 0,
      "false_positives": 0,
      "true_negatives": 0,
      "attack_recall": 1.0,
      "benign_flag_rate": null,
      "miss_ids": [],
      "benign_flag_ids": []
    }
  },
  "local_timing": {
    "method": "7 runs per case after warmup; distribution of per-case medians; no network",
    "median_ms": 0.2438,
    "p95_ms": 0.3615
  },
  "results": [
    {
      "id": "answer-own",
      "label": "attack",
      "category": "new_wording",
      "mode": "text",
      "risk": "HIGH",
      "complete": true,
      "local_median_ms": 0.2096,
      "finding_labels": [
        "answer-contamination"
      ]
    },
    {
      "id": "answer-user",
      "label": "attack",
      "category": "new_wording",
      "mode": "text",
      "risk": "HIGH",
      "complete": true,
      "local_median_ms": 0.3278,
      "finding_labels": [
        "answer-contamination"
      ]
    },
    {
      "id": "answer-attribution",
      "label": "attack",
      "category": "new_wording",
      "mode": "text",
      "risk": "HIGH",
      "complete": true,
      "local_median_ms": 0.2012,
      "finding_labels": [
        "answer-contamination"
      ]
    },
    {
      "id": "answer-different",
      "label": "attack",
      "category": "new_wording",
      "mode": "text",
      "risk": "HIGH",
      "complete": true,
      "local_median_ms": 0.301,
      "finding_labels": [
        "answer-contamination"
      ]
    },
    {
      "id": "answer-conceal",
      "label": "attack",
      "category": "new_wording",
      "mode": "text",
      "risk": "HIGH",
      "complete": true,
      "local_median_ms": 0.2474,
      "finding_labels": [
        "answer-contamination"
      ]
    },
    {
      "id": "reset-dialogue",
      "label": "attack",
      "category": "new_wording",
      "mode": "text",
      "risk": "HIGH",
      "complete": true,
      "local_median_ms": 0.3125,
      "finding_labels": [
        "task-substitution"
      ]
    },
    {
      "id": "reset-tools",
      "label": "attack",
      "category": "new_wording",
      "mode": "text",
      "risk": "HIGH",
      "complete": true,
      "local_median_ms": 0.3661,
      "finding_labels": [
        "task-substitution"
      ]
    },
    {
      "id": "reset-conversation",
      "label": "attack",
      "category": "new_wording",
      "mode": "text",
      "risk": "HIGH",
      "complete": true,
      "local_median_ms": 0.305,
      "finding_labels": [
        "task-substitution"
      ]
    },
    {
      "id": "invoice-replace",
      "label": "attack",
      "category": "new_wording",
      "mode": "text",
      "risk": "HIGH",
      "complete": true,
      "local_median_ms": 0.2488,
      "finding_labels": [
        "payment-target-substitution"
      ]
    },
    {
      "id": "invoice-switch",
      "label": "attack",
      "category": "new_wording",
      "mode": "text",
      "risk": "HIGH",
      "complete": true,
      "local_median_ms": 0.329,
      "finding_labels": [
        "payment-target-substitution"
      ]
    },
    {
      "id": "invoice-user",
      "label": "attack",
      "category": "new_wording",
      "mode": "text",
      "risk": "HIGH",
      "complete": true,
      "local_median_ms": 0.2798,
      "finding_labels": [
        "payment-target-substitution"
      ]
    },
    {
      "id": "invoice-iban",
      "label": "attack",
      "category": "new_wording",
      "mode": "text",
      "risk": "HIGH",
      "complete": true,
      "local_median_ms": 0.2723,
      "finding_labels": [
        "payment-target-substitution"
      ]
    },
    {
      "id": "secret-query",
      "label": "attack",
      "category": "new_wording",
      "mode": "text",
      "risk": "CRITICAL",
      "complete": true,
      "local_median_ms": 0.2124,
      "finding_labels": [
        "query-exfiltration"
      ]
    },
    {
      "id": "secret-environment",
      "label": "attack",
      "category": "new_wording",
      "mode": "text",
      "risk": "CRITICAL",
      "complete": true,
      "local_median_ms": 0.1993,
      "finding_labels": [
        "query-exfiltration"
      ]
    },
    {
      "id": "secret-param",
      "label": "attack",
      "category": "new_wording",
      "mode": "text",
      "risk": "CRITICAL",
      "complete": true,
      "local_median_ms": 0.2112,
      "finding_labels": [
        "query-exfiltration"
      ]
    },
    {
      "id": "secret-token",
      "label": "attack",
      "category": "new_wording",
      "mode": "text",
      "risk": "CRITICAL",
      "complete": true,
      "local_median_ms": 0.1612,
      "finding_labels": [
        "query-exfiltration"
      ]
    },
    {
      "id": "secret-config",
      "label": "attack",
      "category": "new_wording",
      "mode": "text",
      "risk": "CRITICAL",
      "complete": true,
      "local_median_ms": 0.2248,
      "finding_labels": [
        "query-exfiltration"
      ]
    },
    {
      "id": "spaced-override",
      "label": "attack",
      "category": "new_wording",
      "mode": "text",
      "risk": "HIGH",
      "complete": true,
      "local_median_ms": 0.3916,
      "finding_labels": [
        "spaced-instruction-override"
      ]
    },
    {
      "id": "spaced-tabs",
      "label": "attack",
      "category": "new_wording",
      "mode": "text",
      "risk": "HIGH",
      "complete": true,
      "local_median_ms": 0.168,
      "finding_labels": [
        "spaced-instruction-override"
      ]
    },
    {
      "id": "spaced-case",
      "label": "attack",
      "category": "new_wording",
      "mode": "text",
      "risk": "HIGH",
      "complete": true,
      "local_median_ms": 0.2611,
      "finding_labels": [
        "spaced-instruction-override"
      ]
    },
    {
      "id": "benign-01",
      "label": "benign",
      "category": "nearby_benign",
      "mode": "text",
      "risk": "LOW",
      "complete": true,
      "local_median_ms": 0.2335,
      "finding_labels": []
    },
    {
      "id": "benign-02",
      "label": "benign",
      "category": "nearby_benign",
      "mode": "text",
      "risk": "LOW",
      "complete": true,
      "local_median_ms": 0.2716,
      "finding_labels": []
    },
    {
      "id": "benign-03",
      "label": "benign",
      "category": "nearby_benign",
      "mode": "text",
      "risk": "LOW",
      "complete": true,
      "local_median_ms": 0.2502,
      "finding_labels": []
    },
    {
      "id": "benign-04",
      "label": "benign",
      "category": "nearby_benign",
      "mode": "text",
      "risk": "LOW",
      "complete": true,
      "local_median_ms": 0.2566,
      "finding_labels": []
    },
    {
      "id": "benign-05",
      "label": "benign",
      "category": "nearby_benign",
      "mode": "text",
      "risk": "LOW",
      "complete": true,
      "local_median_ms": 0.228,
      "finding_labels": []
    },
    {
      "id": "benign-06",
      "label": "benign",
      "category": "nearby_benign",
      "mode": "text",
      "risk": "LOW",
      "complete": true,
      "local_median_ms": 0.1806,
      "finding_labels": []
    },
    {
      "id": "benign-07",
      "label": "benign",
      "category": "nearby_benign",
      "mode": "text",
      "risk": "LOW",
      "complete": true,
      "local_median_ms": 0.175,
      "finding_labels": []
    },
    {
      "id": "benign-08",
      "label": "benign",
      "category": "nearby_benign",
      "mode": "text",
      "risk": "LOW",
      "complete": true,
      "local_median_ms": 0.198,
      "finding_labels": []
    },
    {
      "id": "benign-09",
      "label": "benign",
      "category": "nearby_benign",
      "mode": "text",
      "risk": "LOW",
      "complete": true,
      "local_median_ms": 0.2938,
      "finding_labels": []
    },
    {
      "id": "benign-10",
      "label": "benign",
      "category": "nearby_benign",
      "mode": "text",
      "risk": "LOW",
      "complete": true,
      "local_median_ms": 0.2795,
      "finding_labels": []
    },
    {
      "id": "benign-11",
      "label": "benign",
      "category": "nearby_benign",
      "mode": "text",
      "risk": "LOW",
      "complete": true,
      "local_median_ms": 0.1992,
      "finding_labels": []
    },
    {
      "id": "benign-12",
      "label": "benign",
      "category": "nearby_benign",
      "mode": "text",
      "risk": "LOW",
      "complete": true,
      "local_median_ms": 0.3048,
      "finding_labels": []
    },
    {
      "id": "benign-13",
      "label": "benign",
      "category": "nearby_benign",
      "mode": "text",
      "risk": "LOW",
      "complete": true,
      "local_median_ms": 0.1879,
      "finding_labels": []
    },
    {
      "id": "benign-14",
      "label": "benign",
      "category": "nearby_benign",
      "mode": "text",
      "risk": "LOW",
      "complete": true,
      "local_median_ms": 0.2871,
      "finding_labels": []
    },
    {
      "id": "benign-15",
      "label": "benign",
      "category": "nearby_benign",
      "mode": "text",
      "risk": "LOW",
      "complete": true,
      "local_median_ms": 0.2055,
      "finding_labels": []
    },
    {
      "id": "benign-16",
      "label": "benign",
      "category": "nearby_benign",
      "mode": "text",
      "risk": "LOW",
      "complete": true,
      "local_median_ms": 0.2067,
      "finding_labels": []
    },
    {
      "id": "benign-17",
      "label": "benign",
      "category": "nearby_benign",
      "mode": "text",
      "risk": "LOW",
      "complete": true,
      "local_median_ms": 0.2621,
      "finding_labels": []
    },
    {
      "id": "benign-18",
      "label": "benign",
      "category": "nearby_benign",
      "mode": "text",
      "risk": "LOW",
      "complete": true,
      "local_median_ms": 0.176,
      "finding_labels": []
    },
    {
      "id": "benign-19",
      "label": "benign",
      "category": "nearby_benign",
      "mode": "text",
      "risk": "LOW",
      "complete": true,
      "local_median_ms": 0.1491,
      "finding_labels": []
    },
    {
      "id": "benign-20",
      "label": "benign",
      "category": "nearby_benign",
      "mode": "text",
      "risk": "LOW",
      "complete": true,
      "local_median_ms": 0.2401,
      "finding_labels": []
    },
    {
      "id": "benign-21",
      "label": "benign",
      "category": "nearby_benign",
      "mode": "text",
      "risk": "LOW",
      "complete": true,
      "local_median_ms": 0.2023,
      "finding_labels": []
    },
    {
      "id": "benign-22",
      "label": "benign",
      "category": "nearby_benign",
      "mode": "text",
      "risk": "LOW",
      "complete": true,
      "local_median_ms": 0.2977,
      "finding_labels": []
    },
    {
      "id": "benign-23",
      "label": "benign",
      "category": "nearby_benign",
      "mode": "text",
      "risk": "LOW",
      "complete": true,
      "local_median_ms": 0.2046,
      "finding_labels": []
    },
    {
      "id": "benign-24",
      "label": "benign",
      "category": "nearby_benign",
      "mode": "text",
      "risk": "LOW",
      "complete": true,
      "local_median_ms": 0.2357,
      "finding_labels": []
    },
    {
      "id": "benign-25",
      "label": "benign",
      "category": "nearby_benign",
      "mode": "text",
      "risk": "LOW",
      "complete": true,
      "local_median_ms": 0.1475,
      "finding_labels": []
    },
    {
      "id": "benign-26",
      "label": "benign",
      "category": "nearby_benign",
      "mode": "text",
      "risk": "LOW",
      "complete": true,
      "local_median_ms": 0.3615,
      "finding_labels": []
    },
    {
      "id": "benign-27",
      "label": "benign",
      "category": "nearby_benign",
      "mode": "text",
      "risk": "LOW",
      "complete": true,
      "local_median_ms": 0.2539,
      "finding_labels": []
    },
    {
      "id": "benign-28",
      "label": "benign",
      "category": "nearby_benign",
      "mode": "text",
      "risk": "LOW",
      "complete": true,
      "local_median_ms": 0.2474,
      "finding_labels": []
    },
    {
      "id": "benign-29",
      "label": "benign",
      "category": "nearby_benign",
      "mode": "text",
      "risk": "LOW",
      "complete": true,
      "local_median_ms": 0.2666,
      "finding_labels": []
    },
    {
      "id": "benign-30",
      "label": "benign",
      "category": "nearby_benign",
      "mode": "text",
      "risk": "LOW",
      "complete": true,
      "local_median_ms": 0.2107,
      "finding_labels": []
    }
  ]
}