{
  "schema_version": 1,
  "generated_at": "2026-07-03T15:23:56Z",
  "rows": [
    {
      "benchmark_key": "rogue",
      "runner": "bastion-prompt-protection (70M)",
      "benchmark": "rogue-security/prompt-injections-benchmark",
      "n_samples": 5000,
      "n_attack": 2003,
      "n_benign": 2997,
      "auc": 0.9859013615046232,
      "f1": 0.916010498687664,
      "precision": 0.965688987271721,
      "recall": 0.871193210184723,
      "fpr_at_tpr_99": 0.20553887220553888,
      "fpr_at_tpr_95": 0.08508508508508508,
      "p50_latency_ms": 4.163936187500994,
      "p95_latency_ms": 4.326755656249404,
      "total_seconds": 20.389668347999987
    },
    {
      "benchmark_key": "jailbreakbench",
      "runner": "bastion-prompt-protection (70M)",
      "benchmark": "JailbreakBench",
      "n_samples": 200,
      "n_attack": 100,
      "n_benign": 100,
      "auc": 0.9855,
      "f1": 0.96,
      "precision": 0.96,
      "recall": 0.96,
      "fpr_at_tpr_99": 0.16,
      "fpr_at_tpr_95": 0.04,
      "p50_latency_ms": 0.9174575312496103,
      "p95_latency_ms": 1.1737482187488268,
      "total_seconds": 0.2093440630000032
    },
    {
      "benchmark_key": "xtram1_test",
      "runner": "bastion-prompt-protection (70M)",
      "benchmark": "xTRam1/test",
      "n_samples": 2060,
      "n_attack": 650,
      "n_benign": 1410,
      "auc": 0.9978832515002728,
      "f1": 0.9406099518459069,
      "precision": 0.9832214765100671,
      "recall": 0.9015384615384615,
      "fpr_at_tpr_99": 0.04113475177304964,
      "fpr_at_tpr_95": 0.009219858156028368,
      "p50_latency_ms": 4.115845343749314,
      "p95_latency_ms": 4.254997031249985,
      "total_seconds": 7.219275881999977
    },
    {
      "benchmark_key": "slabs_test",
      "runner": "bastion-prompt-protection (70M)",
      "benchmark": "S-Labs/test",
      "n_samples": 2101,
      "n_attack": 1051,
      "n_benign": 1050,
      "auc": 0.9956970685514929,
      "f1": 0.9548387096774194,
      "precision": 0.9979253112033195,
      "recall": 0.9153187440532826,
      "fpr_at_tpr_99": 0.11523809523809524,
      "fpr_at_tpr_95": 0.012380952380952381,
      "p50_latency_ms": 0.8464810937507394,
      "p95_latency_ms": 0.9107955625005104,
      "total_seconds": 1.8005451619999917
    },
    {
      "benchmark_key": "rogue",
      "runner": "wolf-defender (0.3B)",
      "benchmark": "rogue-security/prompt-injections-benchmark",
      "n_samples": 5000,
      "n_attack": 2003,
      "n_benign": 2997,
      "auc": 0.9883782934207297,
      "f1": 0.9398439466398187,
      "precision": 0.9477157360406091,
      "recall": 0.9321018472291562,
      "fpr_at_tpr_99": 0.17717717717717718,
      "fpr_at_tpr_95": 0.05472138805472139,
      "p50_latency_ms": 8.54724731249945,
      "p95_latency_ms": 8.683718250001249,
      "total_seconds": 39.14312533100002
    },
    {
      "benchmark_key": "jailbreakbench",
      "runner": "wolf-defender (0.3B)",
      "benchmark": "JailbreakBench",
      "n_samples": 200,
      "n_attack": 100,
      "n_benign": 100,
      "auc": 0.84725,
      "f1": 0.7892376681614349,
      "precision": 0.7154471544715447,
      "recall": 0.88,
      "fpr_at_tpr_99": 0.85,
      "fpr_at_tpr_95": 0.46,
      "p50_latency_ms": 0.7114075000007603,
      "p95_latency_ms": 0.7651395312500142,
      "total_seconds": 0.1608136849999937
    },
    {
      "benchmark_key": "xtram1_test",
      "runner": "wolf-defender (0.3B)",
      "benchmark": "xTRam1/test",
      "n_samples": 2060,
      "n_attack": 650,
      "n_benign": 1410,
      "auc": 0.9957326786688488,
      "f1": 0.9762815608263198,
      "precision": 0.9710806697108066,
      "recall": 0.9815384615384616,
      "fpr_at_tpr_99": 0.02695035460992908,
      "fpr_at_tpr_95": 0.00425531914893617,
      "p50_latency_ms": 8.505322187501108,
      "p95_latency_ms": 8.625873312499976,
      "total_seconds": 15.204028660999995
    },
    {
      "benchmark_key": "slabs_test",
      "runner": "wolf-defender (0.3B)",
      "benchmark": "S-Labs/test",
      "n_samples": 2101,
      "n_attack": 1051,
      "n_benign": 1050,
      "auc": 0.9859172670019483,
      "f1": 0.8648357565966612,
      "precision": 0.9962779156327544,
      "recall": 0.764034253092293,
      "fpr_at_tpr_99": 0.24666666666666667,
      "fpr_at_tpr_95": 0.06285714285714286,
      "p50_latency_ms": 0.7001066874998685,
      "p95_latency_ms": 0.7327260625000775,
      "total_seconds": 1.4845177830000011
    },
    {
      "benchmark_key": "rogue",
      "runner": "wolf-defender-small (0.1B)",
      "benchmark": "rogue-security/prompt-injections-benchmark",
      "n_samples": 5000,
      "n_attack": 2003,
      "n_benign": 2997,
      "auc": 0.9771930359382515,
      "f1": 0.9108264255212258,
      "precision": 0.916582406471183,
      "recall": 0.9051422865701447,
      "fpr_at_tpr_99": 0.31698365031698367,
      "fpr_at_tpr_95": 0.11611611611611612,
      "p50_latency_ms": 4.090529718750346,
      "p95_latency_ms": 4.215445187499611,
      "total_seconds": 18.790741190000006
    },
    {
      "benchmark_key": "jailbreakbench",
      "runner": "wolf-defender-small (0.1B)",
      "benchmark": "JailbreakBench",
      "n_samples": 200,
      "n_attack": 100,
      "n_benign": 100,
      "auc": 0.8105999999999999,
      "f1": 0.7443946188340808,
      "precision": 0.6747967479674797,
      "recall": 0.83,
      "fpr_at_tpr_99": 0.84,
      "fpr_at_tpr_95": 0.61,
      "p50_latency_ms": 0.7119806875000023,
      "p95_latency_ms": 0.7348729687510769,
      "total_seconds": 0.15889457299999776
    },
    {
      "benchmark_key": "xtram1_test",
      "runner": "wolf-defender-small (0.1B)",
      "benchmark": "xTRam1/test",
      "n_samples": 2060,
      "n_attack": 650,
      "n_benign": 1410,
      "auc": 0.9940327332242225,
      "f1": 0.9569811320754718,
      "precision": 0.9392592592592592,
      "recall": 0.9753846153846154,
      "fpr_at_tpr_99": 0.07304964539007092,
      "fpr_at_tpr_95": 0.012056737588652482,
      "p50_latency_ms": 4.050200062499343,
      "p95_latency_ms": 4.150291781250814,
      "total_seconds": 7.301580000000001
    },
    {
      "benchmark_key": "slabs_test",
      "runner": "wolf-defender-small (0.1B)",
      "benchmark": "S-Labs/test",
      "n_samples": 2101,
      "n_attack": 1051,
      "n_benign": 1050,
      "auc": 0.9819903946354945,
      "f1": 0.8963730569948186,
      "precision": 0.9840728100113766,
      "recall": 0.8230256898192198,
      "fpr_at_tpr_99": 0.3523809523809524,
      "fpr_at_tpr_95": 0.08666666666666667,
      "p50_latency_ms": 0.703109156249937,
      "p95_latency_ms": 0.8797003750000698,
      "total_seconds": 1.5101991330000146
    },
    {
      "benchmark_key": "rogue",
      "runner": "sentinel (qualifire, 395M)",
      "benchmark": "rogue-security/prompt-injections-benchmark",
      "n_samples": 5000,
      "n_attack": 2003,
      "n_benign": 2997,
      "auc": 0.9947941784353834,
      "f1": 0.975743935983996,
      "precision": 0.9774549098196392,
      "recall": 0.9740389415876186,
      "fpr_at_tpr_99": 0.03837170503837171,
      "fpr_at_tpr_95": 0.005005005005005005,
      "p50_latency_ms": 22.408726500000142,
      "p95_latency_ms": 22.55208887500082,
      "total_seconds": 102.96111130899999
    },
    {
      "benchmark_key": "jailbreakbench",
      "runner": "sentinel (qualifire, 395M)",
      "benchmark": "JailbreakBench",
      "n_samples": 200,
      "n_attack": 100,
      "n_benign": 100,
      "auc": 0.8925000000000001,
      "f1": 0.718562874251497,
      "precision": 0.8955223880597015,
      "recall": 0.6,
      "fpr_at_tpr_99": 0.67,
      "fpr_at_tpr_95": 0.41,
      "p50_latency_ms": 1.5398011562481884,
      "p95_latency_ms": 1.6989255000012804,
      "total_seconds": 0.30242132800003674
    },
    {
      "benchmark_key": "xtram1_test",
      "runner": "sentinel (qualifire, 395M)",
      "benchmark": "xTRam1/test",
      "n_samples": 2060,
      "n_attack": 650,
      "n_benign": 1410,
      "auc": 0.9876743044189852,
      "f1": 0.9268680445151033,
      "precision": 0.9588815789473685,
      "recall": 0.8969230769230769,
      "fpr_at_tpr_99": 0.1574468085106383,
      "fpr_at_tpr_95": 0.031914893617021274,
      "p50_latency_ms": 22.374326281248358,
      "p95_latency_ms": 22.488916812502424,
      "total_seconds": 40.13902029099995
    },
    {
      "benchmark_key": "slabs_test",
      "runner": "sentinel (qualifire, 395M)",
      "benchmark": "S-Labs/test",
      "n_samples": 2101,
      "n_attack": 1051,
      "n_benign": 1050,
      "auc": 0.9436754111730324,
      "f1": 0.8103638368246968,
      "precision": 0.963302752293578,
      "recall": 0.6993339676498572,
      "fpr_at_tpr_99": 1.0,
      "fpr_at_tpr_95": 1.0,
      "p50_latency_ms": 1.1864598750008781,
      "p95_latency_ms": 1.5572479062520017,
      "total_seconds": 2.569165044999977
    },
    {
      "benchmark_key": "rogue",
      "runner": "proventra mdeberta (280M)",
      "benchmark": "rogue-security/prompt-injections-benchmark",
      "n_samples": 5000,
      "n_attack": 2003,
      "n_benign": 2997,
      "auc": 0.8666540562862747,
      "f1": 0.7344292660338041,
      "precision": 0.6848013816925734,
      "recall": 0.7918122815776335,
      "fpr_at_tpr_99": 0.45045045045045046,
      "fpr_at_tpr_95": 0.3763763763763764,
      "p50_latency_ms": 9.666428031248842,
      "p95_latency_ms": 9.795325000002464,
      "total_seconds": 45.93710187800002
    },
    {
      "benchmark_key": "jailbreakbench",
      "runner": "proventra mdeberta (280M)",
      "benchmark": "JailbreakBench",
      "n_samples": 200,
      "n_attack": 100,
      "n_benign": 100,
      "auc": 0.6455,
      "f1": 0.40540540540540543,
      "precision": 0.625,
      "recall": 0.3,
      "fpr_at_tpr_99": 0.86,
      "fpr_at_tpr_95": 0.75,
      "p50_latency_ms": 0.9483319062510986,
      "p95_latency_ms": 0.9888348749989007,
      "total_seconds": 0.2016352979999283
    },
    {
      "benchmark_key": "xtram1_test",
      "runner": "proventra mdeberta (280M)",
      "benchmark": "xTRam1/test",
      "n_samples": 2060,
      "n_attack": 650,
      "n_benign": 1410,
      "auc": 0.9056808510638298,
      "f1": 0.8141745894554884,
      "precision": 0.9289940828402367,
      "recall": 0.7246153846153847,
      "fpr_at_tpr_99": 0.8879432624113475,
      "fpr_at_tpr_95": 0.6581560283687943,
      "p50_latency_ms": 9.623875687498895,
      "p95_latency_ms": 9.72396493749983,
      "total_seconds": 17.829733467999972
    },
    {
      "benchmark_key": "slabs_test",
      "runner": "proventra mdeberta (280M)",
      "benchmark": "S-Labs/test",
      "n_samples": 2101,
      "n_attack": 1051,
      "n_benign": 1050,
      "auc": 0.9539789769380635,
      "f1": 0.640926640926641,
      "precision": 0.9900596421471173,
      "recall": 0.4738344433872502,
      "fpr_at_tpr_99": 0.5876190476190476,
      "fpr_at_tpr_95": 0.30666666666666664,
      "p50_latency_ms": 0.8256551562482173,
      "p95_latency_ms": 0.9340570625013811,
      "total_seconds": 1.778308175999996
    },
    {
      "benchmark_key": "rogue",
      "runner": "fmops distilbert (67M)",
      "benchmark": "rogue-security/prompt-injections-benchmark",
      "n_samples": 5000,
      "n_attack": 2003,
      "n_benign": 2997,
      "auc": 0.7875583188447226,
      "f1": 0.6597222222222222,
      "precision": 0.4932014833127318,
      "recall": 0.9960059910134798,
      "fpr_at_tpr_99": 0.6389723056389723,
      "fpr_at_tpr_95": 0.5395395395395396,
      "p50_latency_ms": 3.2719071562503643,
      "p95_latency_ms": 3.3683451562502853,
      "total_seconds": 15.01450972200007
    },
    {
      "benchmark_key": "jailbreakbench",
      "runner": "fmops distilbert (67M)",
      "benchmark": "JailbreakBench",
      "n_samples": 200,
      "n_attack": 100,
      "n_benign": 100,
      "auc": 0.5911,
      "f1": 0.6690140845070423,
      "precision": 0.5163043478260869,
      "recall": 0.95,
      "fpr_at_tpr_99": 0.95,
      "fpr_at_tpr_95": 0.89,
      "p50_latency_ms": 0.2867544687497059,
      "p95_latency_ms": 0.29733696874956195,
      "total_seconds": 0.05884319400001914
    },
    {
      "benchmark_key": "xtram1_test",
      "runner": "fmops distilbert (67M)",
      "benchmark": "xTRam1/test",
      "n_samples": 2060,
      "n_attack": 650,
      "n_benign": 1410,
      "auc": 0.5144920894708129,
      "f1": 0.5332775219757221,
      "precision": 0.3663024726854514,
      "recall": 0.98,
      "fpr_at_tpr_99": 0.7921985815602837,
      "fpr_at_tpr_95": 0.7673758865248227,
      "p50_latency_ms": 3.2453976250010896,
      "p95_latency_ms": 3.3125104062499133,
      "total_seconds": 5.8313948390000405
    },
    {
      "benchmark_key": "slabs_test",
      "runner": "fmops distilbert (67M)",
      "benchmark": "S-Labs/test",
      "n_samples": 2101,
      "n_attack": 1051,
      "n_benign": 1050,
      "auc": 0.9070726292419918,
      "f1": 0.7757575757575758,
      "precision": 0.9214659685863874,
      "recall": 0.6698382492863939,
      "fpr_at_tpr_99": 0.7857142857142857,
      "fpr_at_tpr_95": 0.4895238095238095,
      "p50_latency_ms": 0.24902809375149104,
      "p95_latency_ms": 0.31389434375128644,
      "total_seconds": 0.5229381670000066
    },
    {
      "benchmark_key": "rogue",
      "runner": "protectai v2 (184M)",
      "benchmark": "rogue-security/prompt-injections-benchmark",
      "n_samples": 5000,
      "n_attack": 2003,
      "n_benign": 2997,
      "auc": 0.8210442094615833,
      "f1": 0.655713218820015,
      "precision": 0.653922542204568,
      "recall": 0.6575137294058911,
      "fpr_at_tpr_99": 1.0,
      "fpr_at_tpr_95": 0.4427761094427761,
      "p50_latency_ms": 9.643840468747555,
      "p95_latency_ms": 9.738016468748611,
      "total_seconds": 43.00502403100006
    },
    {
      "benchmark_key": "jailbreakbench",
      "runner": "protectai v2 (184M)",
      "benchmark": "JailbreakBench",
      "n_samples": 200,
      "n_attack": 100,
      "n_benign": 100,
      "auc": 0.5132,
      "f1": 0.0,
      "precision": 0.0,
      "recall": 0.0,
      "fpr_at_tpr_99": 1.0,
      "fpr_at_tpr_95": 1.0,
      "p50_latency_ms": 0.8447427187512346,
      "p95_latency_ms": 0.9047524062495427,
      "total_seconds": 0.18998501699991266
    },
    {
      "benchmark_key": "xtram1_test",
      "runner": "protectai v2 (184M)",
      "benchmark": "xTRam1/test",
      "n_samples": 2060,
      "n_attack": 650,
      "n_benign": 1410,
      "auc": 0.9878685215493725,
      "f1": 0.9124270225187656,
      "precision": 0.9963570127504554,
      "recall": 0.8415384615384616,
      "fpr_at_tpr_99": 1.0,
      "fpr_at_tpr_95": 0.02624113475177305,
      "p50_latency_ms": 9.604241125000357,
      "p95_latency_ms": 9.702388000000894,
      "total_seconds": 16.68136674499999
    },
    {
      "benchmark_key": "slabs_test",
      "runner": "protectai v2 (184M)",
      "benchmark": "S-Labs/test",
      "n_samples": 2101,
      "n_attack": 1051,
      "n_benign": 1050,
      "auc": 0.9596420642472022,
      "f1": 0.8264738598442715,
      "precision": 0.9946452476572959,
      "recall": 0.7069457659372027,
      "fpr_at_tpr_99": 1.0,
      "fpr_at_tpr_95": 1.0,
      "p50_latency_ms": 0.8217232812484099,
      "p95_latency_ms": 0.8490401562504246,
      "total_seconds": 1.738765398000055
    },
    {
      "benchmark_key": "rogue",
      "runner": "deepset injection (184M)",
      "benchmark": "rogue-security/prompt-injections-benchmark",
      "n_samples": 5000,
      "n_attack": 2003,
      "n_benign": 2997,
      "auc": 0.786916472138639,
      "f1": 0.6587458745874587,
      "precision": 0.4919891545476953,
      "recall": 0.9965052421367948,
      "fpr_at_tpr_99": 0.6379713046379714,
      "fpr_at_tpr_95": 0.5085085085085085,
      "p50_latency_ms": 9.644963187501077,
      "p95_latency_ms": 9.762028749999985,
      "total_seconds": 43.05011200800004
    },
    {
      "benchmark_key": "jailbreakbench",
      "runner": "deepset injection (184M)",
      "benchmark": "JailbreakBench",
      "n_samples": 200,
      "n_attack": 100,
      "n_benign": 100,
      "auc": 0.64955,
      "f1": 0.7012987012987013,
      "precision": 0.6183206106870229,
      "recall": 0.81,
      "fpr_at_tpr_99": 0.87,
      "fpr_at_tpr_95": 0.81,
      "p50_latency_ms": 0.8289240312500112,
      "p95_latency_ms": 0.8545525312513291,
      "total_seconds": 0.18309958200006804
    },
    {
      "benchmark_key": "xtram1_test",
      "runner": "deepset injection (184M)",
      "benchmark": "xTRam1/test",
      "n_samples": 2060,
      "n_attack": 650,
      "n_benign": 1410,
      "auc": 0.6660851063829787,
      "f1": 0.5470012760527435,
      "precision": 0.37801293356848914,
      "recall": 0.9892307692307692,
      "fpr_at_tpr_99": 0.7531914893617021,
      "fpr_at_tpr_95": 0.7035460992907802,
      "p50_latency_ms": 9.614579843749027,
      "p95_latency_ms": 9.717370312500861,
      "total_seconds": 16.691153098999962
    },
    {
      "benchmark_key": "slabs_test",
      "runner": "deepset injection (184M)",
      "benchmark": "S-Labs/test",
      "n_samples": 2101,
      "n_attack": 1051,
      "n_benign": 1050,
      "auc": 0.9608948393819944,
      "f1": 0.8767550702028081,
      "precision": 0.966743119266055,
      "recall": 0.80209324452902,
      "fpr_at_tpr_99": 0.6390476190476191,
      "fpr_at_tpr_95": 0.23904761904761904,
      "p50_latency_ms": 0.8085714687489087,
      "p95_latency_ms": 0.8708830625003827,
      "total_seconds": 1.713860244999978
    },
    {
      "benchmark_key": "rogue",
      "runner": "hlyn judge (70M)",
      "benchmark": "rogue-security/prompt-injections-benchmark",
      "n_samples": 5000,
      "n_attack": 2003,
      "n_benign": 2997,
      "auc": 0.9800312877363967,
      "f1": 0.8333333333333334,
      "precision": 0.9682749504296101,
      "recall": 0.7314028956565152,
      "fpr_at_tpr_99": 0.4431097764431098,
      "fpr_at_tpr_95": 0.06072739406072739,
      "p50_latency_ms": 4.136701531251674,
      "p95_latency_ms": 4.254401937497931,
      "total_seconds": 18.550888746000055
    },
    {
      "benchmark_key": "jailbreakbench",
      "runner": "hlyn judge (70M)",
      "benchmark": "JailbreakBench",
      "n_samples": 200,
      "n_attack": 100,
      "n_benign": 100,
      "auc": 0.9335500000000001,
      "f1": 0.8287292817679558,
      "precision": 0.9259259259259259,
      "recall": 0.75,
      "fpr_at_tpr_99": 0.69,
      "fpr_at_tpr_95": 0.28,
      "p50_latency_ms": 0.8803835000001925,
      "p95_latency_ms": 1.07597490625011,
      "total_seconds": 0.1994431740000664
    },
    {
      "benchmark_key": "xtram1_test",
      "runner": "hlyn judge (70M)",
      "benchmark": "xTRam1/test",
      "n_samples": 2060,
      "n_attack": 650,
      "n_benign": 1410,
      "auc": 0.9953627932351337,
      "f1": 0.8440528634361234,
      "precision": 0.9876288659793815,
      "recall": 0.7369230769230769,
      "fpr_at_tpr_99": 0.07021276595744681,
      "fpr_at_tpr_95": 0.00851063829787234,
      "p50_latency_ms": 4.089759437501783,
      "p95_latency_ms": 4.199710062501083,
      "total_seconds": 7.163698913000076
    },
    {
      "benchmark_key": "slabs_test",
      "runner": "hlyn judge (70M)",
      "benchmark": "S-Labs/test",
      "n_samples": 2101,
      "n_attack": 1051,
      "n_benign": 1050,
      "auc": 0.891223324724752,
      "f1": 0.32643312101910826,
      "precision": 1.0,
      "recall": 0.1950523311132255,
      "fpr_at_tpr_99": 0.7476190476190476,
      "fpr_at_tpr_95": 0.41714285714285715,
      "p50_latency_ms": 0.8153510312496337,
      "p95_latency_ms": 0.8526170937486199,
      "total_seconds": 1.7299410709999847
    },
    {
      "benchmark_key": "rogue",
      "runner": "meta prompt-guard (86M)",
      "benchmark": "rogue-security/prompt-injections-benchmark",
      "n_samples": 5000,
      "n_attack": 2003,
      "n_benign": 2997,
      "auc": 0.31872661478253095,
      "f1": 0.5554890816631768,
      "precision": 0.39654067905188983,
      "recall": 0.927109335996006,
      "fpr_at_tpr_99": 0.986986986986987,
      "fpr_at_tpr_95": 0.9606272939606273,
      "p50_latency_ms": 9.671374749999018,
      "p95_latency_ms": 9.784587218753416,
      "total_seconds": 45.926830945000006
    },
    {
      "benchmark_key": "jailbreakbench",
      "runner": "meta prompt-guard (86M)",
      "benchmark": "JailbreakBench",
      "n_samples": 200,
      "n_attack": 100,
      "n_benign": 100,
      "auc": 0.35440000000000005,
      "f1": 0.6666666666666666,
      "precision": 0.5,
      "recall": 1.0,
      "fpr_at_tpr_99": 0.99,
      "fpr_at_tpr_95": 0.97,
      "p50_latency_ms": 0.9381964062527004,
      "p95_latency_ms": 0.9760032812486941,
      "total_seconds": 0.19909485999994558
    },
    {
      "benchmark_key": "xtram1_test",
      "runner": "meta prompt-guard (86M)",
      "benchmark": "xTRam1/test",
      "n_samples": 2060,
      "n_attack": 650,
      "n_benign": 1410,
      "auc": 0.21509929078014187,
      "f1": 0.4843283582089552,
      "precision": 0.31970443349753697,
      "recall": 0.9984615384615385,
      "fpr_at_tpr_99": 0.9652482269503546,
      "fpr_at_tpr_95": 0.9560283687943263,
      "p50_latency_ms": 9.65547375000142,
      "p95_latency_ms": 9.735066656247682,
      "total_seconds": 17.877265561999934
    },
    {
      "benchmark_key": "slabs_test",
      "runner": "meta prompt-guard (86M)",
      "benchmark": "S-Labs/test",
      "n_samples": 2101,
      "n_attack": 1051,
      "n_benign": 1050,
      "auc": 0.36881473426668476,
      "f1": 0.6709594333547971,
      "precision": 0.5070559610705596,
      "recall": 0.9914367269267365,
      "fpr_at_tpr_99": 0.9504761904761905,
      "fpr_at_tpr_95": 0.9047619047619048,
      "p50_latency_ms": 0.8231535000007284,
      "p95_latency_ms": 0.9256850937511274,
      "total_seconds": 1.7689094229999682
    },
    {
      "benchmark_key": "rogue",
      "runner": "meta prompt-guard v2 (86M)",
      "benchmark": "rogue-security/prompt-injections-benchmark",
      "n_samples": 5000,
      "n_attack": 2003,
      "n_benign": 2997,
      "auc": 0.8818980404934806,
      "f1": 0.656985187336625,
      "precision": 0.7854166666666667,
      "recall": 0.5646530204692961,
      "fpr_at_tpr_99": 0.4444444444444444,
      "fpr_at_tpr_95": 0.3663663663663664,
      "p50_latency_ms": 9.664981843751974,
      "p95_latency_ms": 9.755997156251084,
      "total_seconds": 45.89414875600005
    },
    {
      "benchmark_key": "jailbreakbench",
      "runner": "meta prompt-guard v2 (86M)",
      "benchmark": "JailbreakBench",
      "n_samples": 200,
      "n_attack": 100,
      "n_benign": 100,
      "auc": 0.75445,
      "f1": 0.44776119402985076,
      "precision": 0.8823529411764706,
      "recall": 0.3,
      "fpr_at_tpr_99": 0.92,
      "fpr_at_tpr_95": 0.76,
      "p50_latency_ms": 0.938904562499232,
      "p95_latency_ms": 0.9808294375019955,
      "total_seconds": 0.19749907799996436
    },
    {
      "benchmark_key": "xtram1_test",
      "runner": "meta prompt-guard v2 (86M)",
      "benchmark": "xTRam1/test",
      "n_samples": 2060,
      "n_attack": 650,
      "n_benign": 1410,
      "auc": 0.9734440807419531,
      "f1": 0.6808510638297872,
      "precision": 0.9970326409495549,
      "recall": 0.5169230769230769,
      "fpr_at_tpr_99": 0.5148936170212766,
      "fpr_at_tpr_95": 0.11773049645390071,
      "p50_latency_ms": 9.630202218747996,
      "p95_latency_ms": 9.737579312499633,
      "total_seconds": 17.84583567200002
    },
    {
      "benchmark_key": "slabs_test",
      "runner": "meta prompt-guard v2 (86M)",
      "benchmark": "S-Labs/test",
      "n_samples": 2101,
      "n_attack": 1051,
      "n_benign": 1050,
      "auc": 0.9219056680712246,
      "f1": 0.5965608465608465,
      "precision": 0.9783080260303688,
      "recall": 0.4291151284490961,
      "fpr_at_tpr_99": 0.7590476190476191,
      "fpr_at_tpr_95": 0.38571428571428573,
      "p50_latency_ms": 0.8351704687505901,
      "p95_latency_ms": 0.9295489374991917,
      "total_seconds": 1.792164171999957
    },
    {
      "benchmark_key": "rogue",
      "runner": "meta prompt-guard v2 (22M)",
      "benchmark": "rogue-security/prompt-injections-benchmark",
      "n_samples": 5000,
      "n_attack": 2003,
      "n_benign": 2997,
      "auc": 0.8200980144731185,
      "f1": 0.1846989588048891,
      "precision": 0.9902912621359223,
      "recall": 0.1018472291562656,
      "fpr_at_tpr_99": 0.5305305305305306,
      "fpr_at_tpr_95": 0.4457791124457791,
      "p50_latency_ms": 4.142712406249416,
      "p95_latency_ms": 4.261734187501531,
      "total_seconds": 18.564147724999998
    },
    {
      "benchmark_key": "jailbreakbench",
      "runner": "meta prompt-guard v2 (22M)",
      "benchmark": "JailbreakBench",
      "n_samples": 200,
      "n_attack": 100,
      "n_benign": 100,
      "auc": 0.75575,
      "f1": 0.2833333333333333,
      "precision": 0.85,
      "recall": 0.17,
      "fpr_at_tpr_99": 0.92,
      "fpr_at_tpr_95": 0.76,
      "p50_latency_ms": 0.8332400312518473,
      "p95_latency_ms": 0.838523156250659,
      "total_seconds": 0.1854891469999984
    },
    {
      "benchmark_key": "xtram1_test",
      "runner": "meta prompt-guard v2 (22M)",
      "benchmark": "xTRam1/test",
      "n_samples": 2060,
      "n_attack": 650,
      "n_benign": 1410,
      "auc": 0.8927975995635571,
      "f1": 0.43937575030012005,
      "precision": 1.0,
      "recall": 0.2815384615384615,
      "fpr_at_tpr_99": 0.5588652482269504,
      "fpr_at_tpr_95": 0.36028368794326243,
      "p50_latency_ms": 4.110161375002974,
      "p95_latency_ms": 4.262562875002374,
      "total_seconds": 7.1938010949999125
    },
    {
      "benchmark_key": "slabs_test",
      "runner": "meta prompt-guard v2 (22M)",
      "benchmark": "S-Labs/test",
      "n_samples": 2101,
      "n_attack": 1051,
      "n_benign": 1050,
      "auc": 0.7716129763037469,
      "f1": 0.3403921568627451,
      "precision": 0.96875,
      "recall": 0.20647002854424357,
      "fpr_at_tpr_99": 0.9866666666666667,
      "fpr_at_tpr_95": 0.86,
      "p50_latency_ms": 0.8115839999973673,
      "p95_latency_ms": 0.889232437501164,
      "total_seconds": 1.7222538729999997
    }
  ]
}