{ "paired": [ { "corpus": "gretel-finance-en", "path": "docs/eval/paired/gretel-finance-en-50-complementarity-bcd3f281.json", "quantized": false, "selection_sha256": "e0c16f7df0a7e0a3c8f1c7154f80c15d2f520aec4c5fe14db6439a67d426f8ae", "sha256": "deb945157c8080b5d756d8b7693aa1f1398f3367908a2b225bbb7fae87c909e6", "single_span_support_counts": { "ettin-native": { "ADDRESS <- street_address -> ADDRESS": 5, "EMAIL <- email -> EMAIL": 1, "IP_ADDRESS <- ipv4 -> IP_ADDRESS": 1, "PERSON <- first_name -> PERSON": 8, "PERSON <- last_name -> PERSON": 1, "PHONE <- phone_number -> PHONE": 8 }, "horizon-native": { "ADDRESS <- SECRET -> SECRET": 2, "ADDRESS <- STREET_ADDRESS -> ADDRESS": 28, "EMAIL <- EMAIL -> EMAIL": 1, "GOV_ID <- NATIONAL_ID -> GOV_ID": 2, "GOV_ID <- PASSPORT -> GOV_ID": 1, "IP_ADDRESS <- IP_ADDRESS -> IP_ADDRESS": 3, "PAYMENT <- BANK_ACCOUNT -> PAYMENT": 1, "PAYMENT <- CREDIT_CARD -> PAYMENT": 2, "PERSON <- LOCATION -> None": 1, "PERSON <- PERSON -> PERSON": 66, "PHONE <- PHONE -> PHONE": 7, "SECRET <- PASSWORD -> SECRET": 1, "USERNAME <- EMAIL -> EMAIL": 1 }, "masker-mini-native": { "GOV_ID <- GOVERNMENT_ID -> GOV_ID": 5, "IP_ADDRESS <- PHONE -> PHONE": 1, "PAYMENT <- PHONE -> PHONE": 2, "PERSON <- GIVEN_NAME -> None": 3, "PERSON <- SURNAME -> None": 4, "PHONE <- PHONE -> PHONE": 14, "USERNAME <- EMAIL -> EMAIL": 1 }, "tracer-native": { "ADDRESS <- account_number -> None": 2, "ADDRESS <- private_address -> None": 40, "EMAIL <- private_email -> None": 2, "GOV_ID <- account_number -> None": 5, "IP_ADDRESS <- private_address -> None": 1, "IP_ADDRESS <- private_url -> None": 2, "PAYMENT <- account_number -> None": 5, "PERSON <- account_number -> None": 1, "PERSON <- private_address -> None": 1, "PERSON <- private_person -> None": 71, "PHONE <- private_phone -> None": 15, "SECRET <- secret -> SECRET": 1, "USERNAME <- private_email -> None": 1 } }, "summary": { "documents": 50, "gold_spans": 151, "high_severity_gold": 11, "models": { "ettin-native": { "character_recall": 0.9011648429226968, "contained": 100, "containment": 0.6622516556291391, "covered_characters": 2553, "documents_with_high_severity_miss": 4, "high_severity_misses": 6, "masked_characters": 5806, "masking_precision": 0.43971753358594556 }, "horizon-native": { "character_recall": 0.905400635368867, "contained": 117, "containment": 0.7748344370860927, "covered_characters": 2565, "documents_with_high_severity_miss": 3, "high_severity_misses": 4, "masked_characters": 5371, "masking_precision": 0.4775646993111152 }, "masker-mini-native": { "character_recall": 0.7942110836569009, "contained": 93, "containment": 0.6158940397350994, "covered_characters": 2250, "documents_with_high_severity_miss": 4, "high_severity_misses": 4, "masked_characters": 3629, "masking_precision": 0.6200055111600992 }, "tracer-native": { "character_recall": 0.9957642075538299, "contained": 147, "containment": 0.9735099337748344, "covered_characters": 2821, "documents_with_high_severity_miss": 0, "high_severity_misses": 0, "masked_characters": 5909, "masking_precision": 0.4774073447283804 } }, "pairs": { "ettin-native+horizon-native": { "contained": 123, "containment": 0.8145695364238411, "covered_characters": 2702, "documents_with_high_severity_miss": 3, "high_severity_misses": 4, "left": "ettin-native", "left_recovers_right_misses": 6, "masked_characters": 6639, "masking_precision": 0.40698900436812774, "native_span_parity_documents": 1, "neither_alone_but_union_covers": 0, "range_parity_documents": 3, "right": "horizon-native", "right_recovers_left_misses": 23 }, "ettin-native+masker-mini-native": { "contained": 111, "containment": 0.7350993377483444, "covered_characters": 2709, "documents_with_high_severity_miss": 3, "high_severity_misses": 3, "left": "ettin-native", "left_recovers_right_misses": 17, "masked_characters": 6347, "masking_precision": 0.4268158184969277, "native_span_parity_documents": 2, "neither_alone_but_union_covers": 1, "range_parity_documents": 3, "right": "masker-mini-native", "right_recovers_left_misses": 10 }, "ettin-native+tracer-native": { "contained": 147, "containment": 0.9735099337748344, "covered_characters": 2823, "documents_with_high_severity_miss": 0, "high_severity_misses": 0, "left": "ettin-native", "left_recovers_right_misses": 0, "masked_characters": 7516, "masking_precision": 0.3755987227248537, "native_span_parity_documents": 0, "neither_alone_but_union_covers": 0, "range_parity_documents": 0, "right": "tracer-native", "right_recovers_left_misses": 47 }, "horizon-native+masker-mini-native": { "contained": 128, "containment": 0.847682119205298, "covered_characters": 2722, "documents_with_high_severity_miss": 2, "high_severity_misses": 2, "left": "horizon-native", "left_recovers_right_misses": 35, "masked_characters": 5973, "masking_precision": 0.4557173949439143, "native_span_parity_documents": 2, "neither_alone_but_union_covers": 0, "range_parity_documents": 2, "right": "masker-mini-native", "right_recovers_left_misses": 11 }, "horizon-native+tracer-native": { "contained": 147, "containment": 0.9735099337748344, "covered_characters": 2823, "documents_with_high_severity_miss": 0, "high_severity_misses": 0, "left": "horizon-native", "left_recovers_right_misses": 0, "masked_characters": 7189, "masking_precision": 0.39268326610098764, "native_span_parity_documents": 1, "neither_alone_but_union_covers": 0, "range_parity_documents": 1, "right": "tracer-native", "right_recovers_left_misses": 30 }, "masker-mini-native+tracer-native": { "contained": 147, "containment": 0.9735099337748344, "covered_characters": 2823, "documents_with_high_severity_miss": 0, "high_severity_misses": 0, "left": "masker-mini-native", "left_recovers_right_misses": 0, "masked_characters": 6260, "masking_precision": 0.4509584664536741, "native_span_parity_documents": 6, "neither_alone_but_union_covers": 0, "range_parity_documents": 6, "right": "tracer-native", "right_recovers_left_misses": 54 } } } }, { "corpus": "gretel-finance-en", "path": "docs/eval/paired/gretel-finance-en-50-quantized-449da1bc.json", "quantized": true, "selection_sha256": "e0c16f7df0a7e0a3c8f1c7154f80c15d2f520aec4c5fe14db6439a67d426f8ae", "sha256": "4345998d97e7f871d8cb668a79a116ef41ee9399bda1ecac60281fe409c430a0", "single_span_support_counts": { "masker-int4-native": { "GOV_ID <- GOVERNMENT_ID -> GOV_ID": 4, "IP_ADDRESS <- PHONE -> PHONE": 1, "PAYMENT <- PHONE -> PHONE": 1, "PERSON <- GIVEN_NAME -> None": 3, "PERSON <- SURNAME -> None": 2, "PHONE <- PHONE -> PHONE": 14, "USERNAME <- EMAIL -> EMAIL": 1 }, "masker-mini-native": { "GOV_ID <- GOVERNMENT_ID -> GOV_ID": 5, "IP_ADDRESS <- PHONE -> PHONE": 1, "PAYMENT <- PHONE -> PHONE": 2, "PERSON <- GIVEN_NAME -> None": 3, "PERSON <- SURNAME -> None": 4, "PHONE <- PHONE -> PHONE": 14, "USERNAME <- EMAIL -> EMAIL": 1 } }, "summary": { "documents": 50, "gold_spans": 151, "high_severity_gold": 11, "models": { "masker-int4-native": { "character_recall": 0.7786798446876103, "contained": 91, "containment": 0.6026490066225165, "covered_characters": 2206, "documents_with_high_severity_miss": 4, "high_severity_misses": 5, "masked_characters": 3548, "masking_precision": 0.6217587373167982 }, "masker-mini-native": { "character_recall": 0.7942110836569009, "contained": 93, "containment": 0.6158940397350994, "covered_characters": 2250, "documents_with_high_severity_miss": 4, "high_severity_misses": 4, "masked_characters": 3629, "masking_precision": 0.6200055111600992 } }, "pairs": { "masker-int4-native+masker-mini-native": { "contained": 93, "containment": 0.6158940397350994, "covered_characters": 2251, "documents_with_high_severity_miss": 4, "high_severity_misses": 4, "left": "masker-int4-native", "left_recovers_right_misses": 0, "masked_characters": 3648, "masking_precision": 0.6170504385964912, "native_span_parity_documents": 29, "neither_alone_but_union_covers": 0, "range_parity_documents": 32, "right": "masker-mini-native", "right_recovers_left_misses": 2 } } } }, { "corpus": "spy-medical", "path": "docs/eval/paired/spy-medical-50-complementarity-2f132a4b.json", "quantized": false, "selection_sha256": "a6fbe20649c5291ea6056f1483304a74aed9edddc5ba33cf93547c5b3be4db39", "sha256": "0e127560e8c793859dee5304731e6862868643c194cbbbae3a8bad53cc94f1a5", "single_span_support_counts": { "ettin-native": { "ADDRESS <- street_address -> ADDRESS": 25, "PHONE <- phone_number -> PHONE": 8 }, "horizon-native": { "ADDRESS <- STREET_ADDRESS -> ADDRESS": 61, "EMAIL <- EMAIL -> EMAIL": 56, "GOV_ID <- MEDICAL_ID -> None": 4, "GOV_ID <- NATIONAL_ID -> GOV_ID": 33, "PERSON <- PERSON -> PERSON": 54, "PHONE <- PHONE -> PHONE": 47, "USERNAME <- USERNAME -> USERNAME": 44 }, "masker-mini-native": { "EMAIL <- EMAIL -> EMAIL": 36, "GOV_ID <- GOVERNMENT_ID -> GOV_ID": 41, "PHONE <- GOVERNMENT_ID -> GOV_ID": 6, "PHONE <- PHONE -> PHONE": 25, "USERNAME <- GOVERNMENT_ID -> GOV_ID": 1 }, "tracer-native": { "ADDRESS <- private_address -> None": 61, "EMAIL <- private_email -> None": 56, "GOV_ID <- account_number -> None": 42, "GOV_ID <- private_phone -> None": 1, "PERSON <- private_person -> None": 54, "PHONE <- private_phone -> None": 47, "USERNAME <- other_pii -> None": 38, "USERNAME <- private_person -> None": 1 } }, "summary": { "documents": 50, "gold_spans": 311, "high_severity_gold": 43, "models": { "ettin-native": { "character_recall": 0.9458520637515325, "contained": 260, "containment": 0.8360128617363344, "covered_characters": 4629, "documents_with_high_severity_miss": 14, "high_severity_misses": 15, "masked_characters": 14959, "masking_precision": 0.3094458185707601 }, "horizon-native": { "character_recall": 0.9928483857785043, "contained": 305, "containment": 0.9807073954983923, "covered_characters": 4859, "documents_with_high_severity_miss": 0, "high_severity_misses": 0, "masked_characters": 15800, "masking_precision": 0.30753164556962026 }, "masker-mini-native": { "character_recall": 0.8308132407029015, "contained": 202, "containment": 0.6495176848874598, "covered_characters": 4066, "documents_with_high_severity_miss": 0, "high_severity_misses": 0, "masked_characters": 9335, "masking_precision": 0.4355650776647027 }, "tracer-native": { "character_recall": 0.9818144666939109, "contained": 300, "containment": 0.9646302250803859, "covered_characters": 4805, "documents_with_high_severity_miss": 0, "high_severity_misses": 0, "masked_characters": 16675, "masking_precision": 0.2881559220389805 } }, "pairs": { "ettin-native+horizon-native": { "contained": 305, "containment": 0.9807073954983923, "covered_characters": 4859, "documents_with_high_severity_miss": 0, "high_severity_misses": 0, "left": "ettin-native", "left_recovers_right_misses": 0, "masked_characters": 16986, "masking_precision": 0.28605910750029434, "native_span_parity_documents": 0, "neither_alone_but_union_covers": 0, "range_parity_documents": 0, "right": "horizon-native", "right_recovers_left_misses": 45 }, "ettin-native+masker-mini-native": { "contained": 294, "containment": 0.9453376205787781, "covered_characters": 4787, "documents_with_high_severity_miss": 0, "high_severity_misses": 0, "left": "ettin-native", "left_recovers_right_misses": 86, "masked_characters": 15442, "masking_precision": 0.30999870483098046, "native_span_parity_documents": 0, "neither_alone_but_union_covers": 6, "range_parity_documents": 0, "right": "masker-mini-native", "right_recovers_left_misses": 28 }, "ettin-native+tracer-native": { "contained": 302, "containment": 0.9710610932475884, "covered_characters": 4830, "documents_with_high_severity_miss": 0, "high_severity_misses": 0, "left": "ettin-native", "left_recovers_right_misses": 2, "masked_characters": 17865, "masking_precision": 0.2703610411418976, "native_span_parity_documents": 0, "neither_alone_but_union_covers": 0, "range_parity_documents": 0, "right": "tracer-native", "right_recovers_left_misses": 42 }, "horizon-native+masker-mini-native": { "contained": 305, "containment": 0.9807073954983923, "covered_characters": 4859, "documents_with_high_severity_miss": 0, "high_severity_misses": 0, "left": "horizon-native", "left_recovers_right_misses": 103, "masked_characters": 16447, "masking_precision": 0.29543381771751687, "native_span_parity_documents": 0, "neither_alone_but_union_covers": 0, "range_parity_documents": 0, "right": "masker-mini-native", "right_recovers_left_misses": 0 }, "horizon-native+tracer-native": { "contained": 307, "containment": 0.9871382636655949, "covered_characters": 4872, "documents_with_high_severity_miss": 0, "high_severity_misses": 0, "left": "horizon-native", "left_recovers_right_misses": 7, "masked_characters": 17349, "masking_precision": 0.28082310219609197, "native_span_parity_documents": 0, "neither_alone_but_union_covers": 0, "range_parity_documents": 0, "right": "tracer-native", "right_recovers_left_misses": 2 }, "masker-mini-native+tracer-native": { "contained": 300, "containment": 0.9646302250803859, "covered_characters": 4805, "documents_with_high_severity_miss": 0, "high_severity_misses": 0, "left": "masker-mini-native", "left_recovers_right_misses": 0, "masked_characters": 17312, "masking_precision": 0.27755314232902034, "native_span_parity_documents": 0, "neither_alone_but_union_covers": 0, "range_parity_documents": 0, "right": "tracer-native", "right_recovers_left_misses": 98 } } } }, { "corpus": "spy-medical", "path": "docs/eval/paired/spy-medical-50-quantized-f59f4ca4.json", "quantized": true, "selection_sha256": "a6fbe20649c5291ea6056f1483304a74aed9edddc5ba33cf93547c5b3be4db39", "sha256": "951247218d8d3cac11d344ca7660a4c369209c92b7c75d8e91d7610745fb0226", "single_span_support_counts": { "masker-int4-native": { "EMAIL <- EMAIL -> EMAIL": 36, "GOV_ID <- GOVERNMENT_ID -> GOV_ID": 41, "GOV_ID <- PHONE -> PHONE": 1, "PHONE <- GOVERNMENT_ID -> GOV_ID": 5, "PHONE <- PHONE -> PHONE": 25 }, "masker-mini-native": { "EMAIL <- EMAIL -> EMAIL": 36, "GOV_ID <- GOVERNMENT_ID -> GOV_ID": 41, "PHONE <- GOVERNMENT_ID -> GOV_ID": 6, "PHONE <- PHONE -> PHONE": 25, "USERNAME <- GOVERNMENT_ID -> GOV_ID": 1 } }, "summary": { "documents": 50, "gold_spans": 311, "high_severity_gold": 43, "models": { "masker-int4-native": { "character_recall": 0.825091949325705, "contained": 200, "containment": 0.6430868167202572, "covered_characters": 4038, "documents_with_high_severity_miss": 0, "high_severity_misses": 0, "masked_characters": 9252, "masking_precision": 0.4364461738002594 }, "masker-mini-native": { "character_recall": 0.8308132407029015, "contained": 202, "containment": 0.6495176848874598, "covered_characters": 4066, "documents_with_high_severity_miss": 0, "high_severity_misses": 0, "masked_characters": 9335, "masking_precision": 0.4355650776647027 } }, "pairs": { "masker-int4-native+masker-mini-native": { "contained": 203, "containment": 0.6527331189710611, "covered_characters": 4074, "documents_with_high_severity_miss": 0, "high_severity_misses": 0, "left": "masker-int4-native", "left_recovers_right_misses": 1, "masked_characters": 9391, "masking_precision": 0.43381961452454476, "native_span_parity_documents": 8, "neither_alone_but_union_covers": 0, "range_parity_documents": 16, "right": "masker-mini-native", "right_recovers_left_misses": 3 } } } } ], "paired_count": 4, "report_count": 16, "reports": [ { "containment": { "character_recall": 0.8371698570390114, "covered_characters": 3455, "fully_contained_gold_spans": 158, "fully_contained_repeated_identifiers": 0, "gold_characters": 4127, "high_severity_gold_spans": 37, "masked_character_fraction": 0.06530318572975437, "masked_characters": 9167, "masking_precision": 0.376895385622341, "repeated_identifiers": 0, "scope": "declared gold policy; zero gold does not establish PII-free text", "source_characters": 140376, "span_recall": 0.6220472440944882, "zero_gold_characters": 0, "zero_gold_document_mask_rate": null, "zero_gold_documents": 0, "zero_gold_masked_character_fraction": null, "zero_gold_masked_characters": 0, "zero_gold_masked_documents": 0 }, "corpus": "spy-legal", "device": "cuda", "gold_spans": 254, "gpu_allocated_bytes": 159212544, "high_severity_misses": 0, "p95_ms": 28.947205999998005, "path": "docs/eval/modal/spy-legal-50-5fdf7be4/spy-legal__masker-mini-native-50.json", "preset": "masker-mini-native", "process_peak_rss_bytes": 5603835904, "revision": "beb7235d454aa4488b6ee1cd47f417caa12a6d15", "selection_sha256": "8923318576cb7efaa919b3e04a121401f7cc36387f66ce6c564d38ed86bc866c", "sha256": "96b5fafd53db1d77ffdaa0e442f63ae45eb32d23204ea919a0f99d8d3921ee8c" }, { "containment": { "character_recall": 0.9297310394960019, "covered_characters": 3837, "fully_contained_gold_spans": 207, "fully_contained_repeated_identifiers": 0, "gold_characters": 4127, "high_severity_gold_spans": 37, "masked_character_fraction": 0.11199207841796319, "masked_characters": 15721, "masking_precision": 0.24406844348323897, "repeated_identifiers": 0, "scope": "declared gold policy; zero gold does not establish PII-free text", "source_characters": 140376, "span_recall": 0.8149606299212598, "zero_gold_characters": 0, "zero_gold_document_mask_rate": null, "zero_gold_documents": 0, "zero_gold_masked_character_fraction": null, "zero_gold_masked_characters": 0, "zero_gold_masked_documents": 0 }, "corpus": "spy-legal", "device": "cuda", "gold_spans": 254, "gpu_allocated_bytes": 120010752, "high_severity_misses": 6, "p95_ms": 58.434759000000724, "path": "docs/eval/modal/spy-legal-50-5fdf7be4/spy-legal__ettin-native-50.json", "preset": "ettin-native", "process_peak_rss_bytes": 5718396928, "revision": "84ce02cac78d9cee27a125629ba195f12c75fe5b", "selection_sha256": "8923318576cb7efaa919b3e04a121401f7cc36387f66ce6c564d38ed86bc866c", "sha256": "cf55c8d62f06928f30b7d71f5ada1a659fef4f43b04b662937faa3ae460d41ea" }, { "containment": { "character_recall": 0.990550036346014, "covered_characters": 4088, "fully_contained_gold_spans": 247, "fully_contained_repeated_identifiers": 0, "gold_characters": 4127, "high_severity_gold_spans": 37, "masked_character_fraction": 0.11332421496552117, "masked_characters": 15908, "masking_precision": 0.256977621322605, "repeated_identifiers": 0, "scope": "declared gold policy; zero gold does not establish PII-free text", "source_characters": 140376, "span_recall": 0.9724409448818898, "zero_gold_characters": 0, "zero_gold_document_mask_rate": null, "zero_gold_documents": 0, "zero_gold_masked_character_fraction": null, "zero_gold_masked_characters": 0, "zero_gold_masked_documents": 0 }, "corpus": "spy-legal", "device": "cuda", "gold_spans": 254, "gpu_allocated_bytes": 597732864, "high_severity_misses": 0, "p95_ms": 60.44804200000087, "path": "docs/eval/modal/spy-legal-50-5fdf7be4/spy-legal__horizon-native-50.json", "preset": "horizon-native", "process_peak_rss_bytes": 5817741312, "revision": "a20835db9e671267b21a2951f5bf2c76e4901708", "selection_sha256": "8923318576cb7efaa919b3e04a121401f7cc36387f66ce6c564d38ed86bc866c", "sha256": "d362397eab6e503f855f1e35293562e493931709e35f954b47aa7acb9c92d46b" }, { "containment": { "character_recall": 0.9970923188756966, "covered_characters": 4115, "fully_contained_gold_spans": 252, "fully_contained_repeated_identifiers": 0, "gold_characters": 4127, "high_severity_gold_spans": 37, "masked_character_fraction": 0.11214880036473471, "masked_characters": 15743, "masking_precision": 0.2613860128310995, "repeated_identifiers": 0, "scope": "declared gold policy; zero gold does not establish PII-free text", "source_characters": 140376, "span_recall": 0.9921259842519685, "zero_gold_characters": 0, "zero_gold_document_mask_rate": null, "zero_gold_documents": 0, "zero_gold_masked_character_fraction": null, "zero_gold_masked_characters": 0, "zero_gold_masked_documents": 0 }, "corpus": "spy-legal", "device": "cuda", "gold_spans": 254, "gpu_allocated_bytes": 1343067136, "high_severity_misses": 0, "p95_ms": 191.279214999998, "path": "docs/eval/modal/spy-legal-50-5fdf7be4/spy-legal__tracer-native-50.json", "preset": "tracer-native", "process_peak_rss_bytes": 7347294208, "revision": "d25c16f2e57e321f6d2527715c01df9112f956f5", "selection_sha256": "8923318576cb7efaa919b3e04a121401f7cc36387f66ce6c564d38ed86bc866c", "sha256": "dc5e0f4f270f7078bcf744fac850778282938e813241756509729fb41db9029c" }, { "containment": { "character_recall": 0.8308132407029015, "covered_characters": 4066, "fully_contained_gold_spans": 202, "fully_contained_repeated_identifiers": 0, "gold_characters": 4894, "high_severity_gold_spans": 43, "masked_character_fraction": 0.07450417015842611, "masked_characters": 9335, "masking_precision": 0.4355650776647027, "repeated_identifiers": 0, "scope": "declared gold policy; zero gold does not establish PII-free text", "source_characters": 125295, "span_recall": 0.6495176848874598, "zero_gold_characters": 0, "zero_gold_document_mask_rate": null, "zero_gold_documents": 0, "zero_gold_masked_character_fraction": null, "zero_gold_masked_characters": 0, "zero_gold_masked_documents": 0 }, "corpus": "spy-medical", "device": "cuda", "gold_spans": 311, "gpu_allocated_bytes": 159094272, "high_severity_misses": 0, "p95_ms": 23.81890999999925, "path": "docs/eval/modal/spy-medical-50-55290bac/spy-medical__masker-mini-native-50.json", "preset": "masker-mini-native", "process_peak_rss_bytes": 5645430784, "revision": "beb7235d454aa4488b6ee1cd47f417caa12a6d15", "selection_sha256": "a6fbe20649c5291ea6056f1483304a74aed9edddc5ba33cf93547c5b3be4db39", "sha256": "92007faeb9f90e44c3ff1bc0917e3846bbed8851080c98e4601899492a23b5b3" }, { "containment": { "character_recall": 0.9458520637515325, "covered_characters": 4629, "fully_contained_gold_spans": 260, "fully_contained_repeated_identifiers": 0, "gold_characters": 4894, "high_severity_gold_spans": 43, "masked_character_fraction": 0.11939023903587534, "masked_characters": 14959, "masking_precision": 0.3094458185707601, "repeated_identifiers": 0, "scope": "declared gold policy; zero gold does not establish PII-free text", "source_characters": 125295, "span_recall": 0.8360128617363344, "zero_gold_characters": 0, "zero_gold_document_mask_rate": null, "zero_gold_documents": 0, "zero_gold_masked_character_fraction": null, "zero_gold_masked_characters": 0, "zero_gold_masked_documents": 0 }, "corpus": "spy-medical", "device": "cuda", "gold_spans": 311, "gpu_allocated_bytes": 118784000, "high_severity_misses": 15, "p95_ms": 54.07803299999969, "path": "docs/eval/modal/spy-medical-50-55290bac/spy-medical__ettin-native-50.json", "preset": "ettin-native", "process_peak_rss_bytes": 5755674624, "revision": "84ce02cac78d9cee27a125629ba195f12c75fe5b", "selection_sha256": "a6fbe20649c5291ea6056f1483304a74aed9edddc5ba33cf93547c5b3be4db39", "sha256": "7da3b849bb0c1cdf014ee7a3134b1d3ba2c24a710e36e20163cb9af86512c26c" }, { "containment": { "character_recall": 0.9928483857785043, "covered_characters": 4859, "fully_contained_gold_spans": 305, "fully_contained_repeated_identifiers": 0, "gold_characters": 4894, "high_severity_gold_spans": 43, "masked_character_fraction": 0.1261023983399178, "masked_characters": 15800, "masking_precision": 0.30753164556962026, "repeated_identifiers": 0, "scope": "declared gold policy; zero gold does not establish PII-free text", "source_characters": 125295, "span_recall": 0.9807073954983923, "zero_gold_characters": 0, "zero_gold_document_mask_rate": null, "zero_gold_documents": 0, "zero_gold_masked_character_fraction": null, "zero_gold_masked_characters": 0, "zero_gold_masked_documents": 0 }, "corpus": "spy-medical", "device": "cuda", "gold_spans": 311, "gpu_allocated_bytes": 597680640, "high_severity_misses": 0, "p95_ms": 60.70141800000073, "path": "docs/eval/modal/spy-medical-50-55290bac/spy-medical__horizon-native-50.json", "preset": "horizon-native", "process_peak_rss_bytes": 5821841408, "revision": "a20835db9e671267b21a2951f5bf2c76e4901708", "selection_sha256": "a6fbe20649c5291ea6056f1483304a74aed9edddc5ba33cf93547c5b3be4db39", "sha256": "4379619d874319be59c514cc697ff55e4aa7057fbfafb2b7655cfaf862aa4a69" }, { "containment": { "character_recall": 0.9818144666939109, "covered_characters": 4805, "fully_contained_gold_spans": 300, "fully_contained_repeated_identifiers": 0, "gold_characters": 4894, "high_severity_gold_spans": 43, "masked_character_fraction": 0.13308591723532465, "masked_characters": 16675, "masking_precision": 0.2881559220389805, "repeated_identifiers": 0, "scope": "declared gold policy; zero gold does not establish PII-free text", "source_characters": 125295, "span_recall": 0.9646302250803859, "zero_gold_characters": 0, "zero_gold_document_mask_rate": null, "zero_gold_documents": 0, "zero_gold_masked_character_fraction": null, "zero_gold_masked_characters": 0, "zero_gold_masked_documents": 0 }, "corpus": "spy-medical", "device": "cuda", "gold_spans": 311, "gpu_allocated_bytes": 1339804160, "high_severity_misses": 0, "p95_ms": 192.84651099999905, "path": "docs/eval/modal/spy-medical-50-55290bac/spy-medical__tracer-native-50.json", "preset": "tracer-native", "process_peak_rss_bytes": 7338708992, "revision": "d25c16f2e57e321f6d2527715c01df9112f956f5", "selection_sha256": "a6fbe20649c5291ea6056f1483304a74aed9edddc5ba33cf93547c5b3be4db39", "sha256": "5097bdae8317d84a19f99422453eaadd2a538b90655a217e7545fa4a8272353a" }, { "containment": { "character_recall": 0.7942110836569009, "covered_characters": 2250, "fully_contained_gold_spans": 93, "fully_contained_repeated_identifiers": 15, "gold_characters": 2833, "high_severity_gold_spans": 11, "masked_character_fraction": 0.05720095203568558, "masked_characters": 3629, "masking_precision": 0.6200055111600992, "repeated_identifiers": 21, "scope": "declared gold policy; zero gold does not establish PII-free text", "source_characters": 63443, "span_recall": 0.6158940397350994, "zero_gold_characters": 8758, "zero_gold_document_mask_rate": 0.1, "zero_gold_documents": 10, "zero_gold_masked_character_fraction": 0.0005709065996802923, "zero_gold_masked_characters": 5, "zero_gold_masked_documents": 1 }, "corpus": "gretel-finance-en", "device": "cuda", "gold_spans": 151, "gpu_allocated_bytes": 159094272, "high_severity_misses": 4, "p95_ms": 19.179351000001787, "path": "docs/eval/modal/gretel-finance-en-50-9418dddc/gretel-finance-en__masker-mini-native-50.json", "preset": "masker-mini-native", "process_peak_rss_bytes": 5521039360, "revision": "beb7235d454aa4488b6ee1cd47f417caa12a6d15", "selection_sha256": "e0c16f7df0a7e0a3c8f1c7154f80c15d2f520aec4c5fe14db6439a67d426f8ae", "sha256": "b084882701f9a48361e4ecb4000e2279a946066a4979f89fa183a9bb0a6aaeea" }, { "containment": { "character_recall": 0.9011648429226968, "covered_characters": 2553, "fully_contained_gold_spans": 100, "fully_contained_repeated_identifiers": 15, "gold_characters": 2833, "high_severity_gold_spans": 11, "masked_character_fraction": 0.09151521838500701, "masked_characters": 5806, "masking_precision": 0.43971753358594556, "repeated_identifiers": 21, "scope": "declared gold policy; zero gold does not establish PII-free text", "source_characters": 63443, "span_recall": 0.6622516556291391, "zero_gold_characters": 8758, "zero_gold_document_mask_rate": 0.8, "zero_gold_documents": 10, "zero_gold_masked_character_fraction": 0.06097282484585522, "zero_gold_masked_characters": 534, "zero_gold_masked_documents": 8 }, "corpus": "gretel-finance-en", "device": "cuda", "gold_spans": 151, "gpu_allocated_bytes": 92366336, "high_severity_misses": 6, "p95_ms": 28.60203500000047, "path": "docs/eval/modal/gretel-finance-en-50-9418dddc/gretel-finance-en__ettin-native-50.json", "preset": "ettin-native", "process_peak_rss_bytes": 5623123968, "revision": "84ce02cac78d9cee27a125629ba195f12c75fe5b", "selection_sha256": "e0c16f7df0a7e0a3c8f1c7154f80c15d2f520aec4c5fe14db6439a67d426f8ae", "sha256": "54faf28f8d04a608f92a338fa067efcefb8a10429209851f4a502abe69446726" }, { "containment": { "character_recall": 0.905400635368867, "covered_characters": 2565, "fully_contained_gold_spans": 117, "fully_contained_repeated_identifiers": 17, "gold_characters": 2833, "high_severity_gold_spans": 11, "masked_character_fraction": 0.08465866998723263, "masked_characters": 5371, "masking_precision": 0.4775646993111152, "repeated_identifiers": 21, "scope": "declared gold policy; zero gold does not establish PII-free text", "source_characters": 63443, "span_recall": 0.7748344370860927, "zero_gold_characters": 8758, "zero_gold_document_mask_rate": 0.8, "zero_gold_documents": 10, "zero_gold_masked_character_fraction": 0.07992692395524092, "zero_gold_masked_characters": 700, "zero_gold_masked_documents": 8 }, "corpus": "gretel-finance-en", "device": "cuda", "gold_spans": 151, "gpu_allocated_bytes": 588815872, "high_severity_misses": 4, "p95_ms": 36.660017999999184, "path": "docs/eval/modal/gretel-finance-en-50-9418dddc/gretel-finance-en__horizon-native-50.json", "preset": "horizon-native", "process_peak_rss_bytes": 5818445824, "revision": "a20835db9e671267b21a2951f5bf2c76e4901708", "selection_sha256": "e0c16f7df0a7e0a3c8f1c7154f80c15d2f520aec4c5fe14db6439a67d426f8ae", "sha256": "aabab6162c7fde7f073356d8194dcfcd8178734d48420417428b9a23aaf3f2ad" }, { "containment": { "character_recall": 0.9957642075538299, "covered_characters": 2821, "fully_contained_gold_spans": 147, "fully_contained_repeated_identifiers": 20, "gold_characters": 2833, "high_severity_gold_spans": 11, "masked_character_fraction": 0.09313872294815818, "masked_characters": 5909, "masking_precision": 0.4774073447283804, "repeated_identifiers": 21, "scope": "declared gold policy; zero gold does not establish PII-free text", "source_characters": 63443, "span_recall": 0.9735099337748344, "zero_gold_characters": 8758, "zero_gold_document_mask_rate": 0.4, "zero_gold_documents": 10, "zero_gold_masked_character_fraction": 0.05457867092943595, "zero_gold_masked_characters": 478, "zero_gold_masked_documents": 4 }, "corpus": "gretel-finance-en", "device": "cuda", "gold_spans": 151, "gpu_allocated_bytes": 1266969088, "high_severity_misses": 0, "p95_ms": 101.0636869999999, "path": "docs/eval/modal/gretel-finance-en-50-9418dddc/gretel-finance-en__tracer-native-50.json", "preset": "tracer-native", "process_peak_rss_bytes": 7345606656, "revision": "d25c16f2e57e321f6d2527715c01df9112f956f5", "selection_sha256": "e0c16f7df0a7e0a3c8f1c7154f80c15d2f520aec4c5fe14db6439a67d426f8ae", "sha256": "268deeebcf28d4aa207f655b5b59de255578cea8845a3d400c38f45e85921ae8" }, { "containment": { "character_recall": 0.8308132407029015, "covered_characters": 4066, "fully_contained_gold_spans": 202, "fully_contained_repeated_identifiers": 0, "gold_characters": 4894, "high_severity_gold_spans": 43, "masked_character_fraction": 0.07450417015842611, "masked_characters": 9335, "masking_precision": 0.4355650776647027, "repeated_identifiers": 0, "scope": "declared gold policy; zero gold does not establish PII-free text", "source_characters": 125295, "span_recall": 0.6495176848874598, "zero_gold_characters": 0, "zero_gold_document_mask_rate": null, "zero_gold_documents": 0, "zero_gold_masked_character_fraction": null, "zero_gold_masked_characters": 0, "zero_gold_masked_documents": 0 }, "corpus": "spy-medical", "device": "cpu", "gold_spans": 311, "gpu_allocated_bytes": null, "high_severity_misses": 0, "p95_ms": 152.6900849999997, "path": "docs/eval/modal/spy-medical-50-733d5cc0/spy-medical__masker-mini-native-50.json", "preset": "masker-mini-native", "process_peak_rss_bytes": 5283160064, "revision": "beb7235d454aa4488b6ee1cd47f417caa12a6d15", "selection_sha256": "a6fbe20649c5291ea6056f1483304a74aed9edddc5ba33cf93547c5b3be4db39", "sha256": "fc66894c711d425ab9f5376a1c03a51553e867cfebbba2d8d36342da5cc37d9c" }, { "containment": { "character_recall": 0.825091949325705, "covered_characters": 4038, "fully_contained_gold_spans": 200, "fully_contained_repeated_identifiers": 0, "gold_characters": 4894, "high_severity_gold_spans": 43, "masked_character_fraction": 0.07384173350891896, "masked_characters": 9252, "masking_precision": 0.4364461738002594, "repeated_identifiers": 0, "scope": "declared gold policy; zero gold does not establish PII-free text", "source_characters": 125295, "span_recall": 0.6430868167202572, "zero_gold_characters": 0, "zero_gold_document_mask_rate": null, "zero_gold_documents": 0, "zero_gold_masked_character_fraction": null, "zero_gold_masked_characters": 0, "zero_gold_masked_documents": 0 }, "corpus": "spy-medical", "device": "cpu", "gold_spans": 311, "gpu_allocated_bytes": null, "high_severity_misses": 0, "p95_ms": 165.62800500000208, "path": "docs/eval/modal/spy-medical-50-733d5cc0/spy-medical__masker-int4-native-50.json", "preset": "masker-int4-native", "process_peak_rss_bytes": 5012381696, "revision": "beb7235d454aa4488b6ee1cd47f417caa12a6d15", "selection_sha256": "a6fbe20649c5291ea6056f1483304a74aed9edddc5ba33cf93547c5b3be4db39", "sha256": "7e2d498ea4419ea8be08b9c2cbc6f23a835338dcb5e484d0a8389c1d53efb8a1" }, { "containment": { "character_recall": 0.945034736411933, "covered_characters": 4625, "fully_contained_gold_spans": 261, "fully_contained_repeated_identifiers": 0, "gold_characters": 4894, "high_severity_gold_spans": 43, "masked_character_fraction": 0.11915080410231853, "masked_characters": 14929, "masking_precision": 0.3097997186683636, "repeated_identifiers": 0, "scope": "declared gold policy; zero gold does not establish PII-free text", "source_characters": 125295, "span_recall": 0.8392282958199357, "zero_gold_characters": 0, "zero_gold_document_mask_rate": null, "zero_gold_documents": 0, "zero_gold_masked_character_fraction": null, "zero_gold_masked_characters": 0, "zero_gold_masked_documents": 0 }, "corpus": "spy-medical", "device": "cpu", "gold_spans": 311, "gpu_allocated_bytes": null, "high_severity_misses": 14, "p95_ms": 283.85335300000133, "path": "docs/eval/modal/spy-medical-50-733d5cc0/spy-medical__ettin-native-50.json", "preset": "ettin-native", "process_peak_rss_bytes": 5318787072, "revision": "84ce02cac78d9cee27a125629ba195f12c75fe5b", "selection_sha256": "a6fbe20649c5291ea6056f1483304a74aed9edddc5ba33cf93547c5b3be4db39", "sha256": "abe5953ecd94a979059a06f0569849d99206cd28cafbe104506d07e264a91bc1" }, { "containment": { "character_recall": 0.9928483857785043, "covered_characters": 4859, "fully_contained_gold_spans": 305, "fully_contained_repeated_identifiers": 0, "gold_characters": 4894, "high_severity_gold_spans": 43, "masked_character_fraction": 0.1261023983399178, "masked_characters": 15800, "masking_precision": 0.30753164556962026, "repeated_identifiers": 0, "scope": "declared gold policy; zero gold does not establish PII-free text", "source_characters": 125295, "span_recall": 0.9807073954983923, "zero_gold_characters": 0, "zero_gold_document_mask_rate": null, "zero_gold_documents": 0, "zero_gold_masked_character_fraction": null, "zero_gold_masked_characters": 0, "zero_gold_masked_documents": 0 }, "corpus": "spy-medical", "device": "cpu", "gold_spans": 311, "gpu_allocated_bytes": null, "high_severity_misses": 0, "p95_ms": 457.7599780000021, "path": "docs/eval/modal/spy-medical-50-733d5cc0/spy-medical__horizon-native-50.json", "preset": "horizon-native", "process_peak_rss_bytes": 5775089664, "revision": "a20835db9e671267b21a2951f5bf2c76e4901708", "selection_sha256": "a6fbe20649c5291ea6056f1483304a74aed9edddc5ba33cf93547c5b3be4db39", "sha256": "c6cbf2e2327b4d8b7188553b34bbb1712fab802efc5415c2d83a2f0725f65eab" } ], "runtime_precision": [ { "models": [ { "actual_parameter_dtype": "torch.bfloat16", "backend": "transformers-pipeline-torch.bfloat16", "device": "cuda", "packages": { "torch": "2.8.0", "transformers": "5.22.1" }, "parameters": 32072171, "preset": "ettin-native", "revision": "84ce02cac78d9cee27a125629ba195f12c75fe5b" }, { "actual_parameter_dtype": "torch.float32", "backend": "transformers-pipeline-torch.float32", "device": "cuda", "packages": { "torch": "2.8.0", "transformers": "5.22.1" }, "parameters": 140664251, "preset": "horizon-native", "revision": "a20835db9e671267b21a2951f5bf2c76e4901708" }, { "actual_parameter_dtype": "torch.float32", "backend": "torch-fp32", "device": "cuda", "packages": { "torch": "2.8.0", "transformers": "5.22.1" }, "parameters": 35399857, "preset": "masker-mini-native", "revision": "beb7235d454aa4488b6ee1cd47f417caa12a6d15" } ], "path": "docs/research/compact-runtime-precision-2026-11-05.json", "sha256": "33b7b88bbb1f4171dee16e141cd8b69768619fef0298b7548ef85cf01524e41f" }, { "models": [ { "actual_parameter_dtype": "torch.bfloat16", "backend": "transformers-pipeline-torch.bfloat16", "device": "cpu", "packages": { "torch": "2.8.0", "transformers": "5.22.1" }, "parameters": 32072171, "preset": "ettin-native", "revision": "84ce02cac78d9cee27a125629ba195f12c75fe5b" }, { "actual_parameter_dtype": "torch.float32", "backend": "transformers-pipeline-torch.float32", "device": "cpu", "packages": { "torch": "2.8.0", "transformers": "5.22.1" }, "parameters": 140664251, "preset": "horizon-native", "revision": "a20835db9e671267b21a2951f5bf2c76e4901708" }, { "actual_parameter_dtype": "torch.float32", "backend": "torch-fp32", "device": "cpu", "packages": { "torch": "2.8.0", "transformers": "5.22.1" }, "parameters": 35399857, "preset": "masker-mini-native", "revision": "beb7235d454aa4488b6ee1cd47f417caa12a6d15" } ], "path": "docs/research/compact-runtime-precision-cpu-2026-11-05.json", "sha256": "7e32c0dd1a23fa33c02c75c8f2ceb173db7152f7b4f0fa0f01a6ad373a2ca0cd" } ], "scope": "exploratory native containment; integration passes are not zero-leak promotion" }