From 04d34e458426b658d3555269b9ee3113dacd32b5 Mon Sep 17 00:00:00 2001 From: memadi Date: Mon, 10 Aug 2026 16:52:02 -0700 Subject: [PATCH 01/30] feat(config): add entity_label_denylist to Detect config Adds entity_label_denylist: list[str] | None to the Detect model, with the same normalisation (strip, lowercase, deduplicate) as entity_labels. A model_validator warns at config construction time when entity_labels and entity_label_denylist share labels that would never be detected. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- src/anonymizer/config/anonymizer_config.py | 32 ++++++++++ tests/config/test_anonymizer_config.py | 73 +++++++++++++++++++++- 2 files changed, 104 insertions(+), 1 deletion(-) diff --git a/src/anonymizer/config/anonymizer_config.py b/src/anonymizer/config/anonymizer_config.py index 3afdea1c..77e3b87e 100644 --- a/src/anonymizer/config/anonymizer_config.py +++ b/src/anonymizer/config/anonymizer_config.py @@ -79,6 +79,14 @@ class Detect(BaseModel): "To inspect the default set, use `from anonymizer import DEFAULT_ENTITY_LABELS`." ), ) + entity_label_denylist: list[str] | None = Field( + default=None, + description=( + "Entity labels to never detect, even if present in entity_labels or the default set. " + "Denied labels are excluded before GLiNER and LLM prompts run, and are also filtered " + "from the final entity output as a safety net." + ), + ) gliner_threshold: float = Field( default=0.3, ge=0.0, le=1.0, description="GLiNER detection confidence threshold (0.0-1.0)." ) @@ -114,6 +122,30 @@ def validate_entity_labels(cls, value: list[str] | None) -> list[str] | None: logger.warning("entity_labels contained duplicates, removed automatically.") return deduped + @field_validator("entity_label_denylist") + @classmethod + def validate_entity_label_denylist(cls, value: list[str] | None) -> list[str] | None: + if value is None: + return value + cleaned = [label.strip().lower() for label in value if label.strip()] + if not cleaned: + raise ValueError("entity_label_denylist must not be empty. Use None to disable the deny list.") + deduped = sorted(set(cleaned)) + if len(deduped) != len(cleaned): + logger.warning("entity_label_denylist contained duplicates, removed automatically.") + return deduped + + @model_validator(mode="after") + def warn_on_allowlist_denylist_overlap(self) -> "Detect": + if self.entity_labels is not None and self.entity_label_denylist is not None: + overlap = sorted(set(self.entity_labels) & set(self.entity_label_denylist)) + if overlap: + logger.warning( + "entity_labels and entity_label_denylist share labels that will never be detected: %s", + overlap, + ) + return self + class Rewrite(BaseModel): """Configuration for rewrite-mode execution.""" diff --git a/tests/config/test_anonymizer_config.py b/tests/config/test_anonymizer_config.py index 0738208c..1381eade 100644 --- a/tests/config/test_anonymizer_config.py +++ b/tests/config/test_anonymizer_config.py @@ -8,7 +8,9 @@ import pytest from pydantic import ValidationError -from anonymizer.config.anonymizer_config import AnonymizerConfig, AnonymizerInput, Rewrite, infer_input_source_suffix +import logging + +from anonymizer.config.anonymizer_config import AnonymizerConfig, AnonymizerInput, Detect, Rewrite, infer_input_source_suffix from anonymizer.config.replace_strategies import ( Annotate, Hash, @@ -147,3 +149,72 @@ def test_detect_validation_max_entities_per_call_must_be_positive() -> None: def test_detect_validation_excerpt_window_chars_must_be_positive() -> None: with pytest.raises(ValidationError): AnonymizerConfig(detect={"validation_excerpt_window_chars": 0}, replace=Redact()) + + +# ── entity_label_denylist ───────────────────────────────────────────────────── + + +def test_entity_label_denylist_defaults_to_none() -> None: + config = AnonymizerConfig(replace=Redact()) + assert config.detect.entity_label_denylist is None + + +def test_entity_label_denylist_accepts_list() -> None: + config = AnonymizerConfig(detect={"entity_label_denylist": ["EMAIL", "city"]}, replace=Redact()) + assert config.detect.entity_label_denylist is not None + assert set(config.detect.entity_label_denylist) == {"email", "city"} + + +def test_entity_label_denylist_strips_whitespace_and_lowercases() -> None: + config = AnonymizerConfig(detect={"entity_label_denylist": [" FIRST_NAME ", "Email"]}, replace=Redact()) + assert config.detect.entity_label_denylist is not None + assert "first_name" in config.detect.entity_label_denylist + assert "email" in config.detect.entity_label_denylist + + +def test_entity_label_denylist_deduplicates(caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING, logger="anonymizer"): + config = AnonymizerConfig(detect={"entity_label_denylist": ["email", "email"]}, replace=Redact()) + assert config.detect.entity_label_denylist == ["email"] + assert "duplicates" in caplog.text + + +def test_entity_label_denylist_empty_list_raises() -> None: + with pytest.raises(ValidationError, match="must not be empty"): + AnonymizerConfig(detect={"entity_label_denylist": []}, replace=Redact()) + + +def test_entity_label_denylist_whitespace_only_raises() -> None: + with pytest.raises(ValidationError, match="must not be empty"): + AnonymizerConfig(detect={"entity_label_denylist": [" ", ""]}, replace=Redact()) + + +def test_entity_label_denylist_overlap_with_entity_labels_warns(caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING, logger="anonymizer"): + AnonymizerConfig( + detect={"entity_labels": ["email", "city"], "entity_label_denylist": ["email"]}, + replace=Redact(), + ) + assert "email" in caplog.text + assert "will never be detected" in caplog.text + + +def test_entity_label_denylist_no_overlap_does_not_warn(caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING, logger="anonymizer"): + AnonymizerConfig( + detect={"entity_labels": ["email", "city"], "entity_label_denylist": ["first_name"]}, + replace=Redact(), + ) + assert "will never be detected" not in caplog.text + + +def test_entity_label_denylist_overlap_warning_only_fires_when_allowlist_explicit( + caplog: pytest.LogCaptureFixture, +) -> None: + """No warning when entity_labels=None (defaults) even if denylist is set.""" + with caplog.at_level(logging.WARNING, logger="anonymizer"): + AnonymizerConfig( + detect={"entity_label_denylist": ["email"]}, + replace=Redact(), + ) + assert "will never be detected" not in caplog.text From 54285891624134b06e17fec4820689635b41f28f Mon Sep 17 00:00:00 2001 From: memadi Date: Mon, 10 Aug 2026 16:52:46 -0700 Subject: [PATCH 02/30] feat(engine): thread entity_label_denylist through detection pipeline Applies the denylist at two points: - _resolve_detection_labels: subtracts denied labels before they reach GLiNER and the LLM augmenter/validator prompts - _materialize_final_entities: safety-net filter that drops any entity whose label is in the denylist from COL_FINAL_ENTITIES Threads entity_label_denylist through detect_and_validate_entities, _build_detection_spec, identify_latent_entities, and run on EntityDetectionWorkflow, and wires it from Anonymizer._run_internal via config.detect.entity_label_denylist. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .../engine/detection/detection_workflow.py | 49 ++++-- src/anonymizer/interface/anonymizer.py | 3 + tests/engine/test_detection_workflow.py | 164 ++++++++++++++++++ 3 files changed, 204 insertions(+), 12 deletions(-) diff --git a/src/anonymizer/engine/detection/detection_workflow.py b/src/anonymizer/engine/detection/detection_workflow.py index a577a47b..620de03b 100644 --- a/src/anonymizer/engine/detection/detection_workflow.py +++ b/src/anonymizer/engine/detection/detection_workflow.py @@ -94,6 +94,7 @@ def detect_and_validate_entities( validation_excerpt_window_chars: int = _DEFAULT_VALIDATION_EXCERPT_WINDOW_CHARS, validation_single_chunk_full_text: bool = True, entity_labels: list[str] | None = None, + entity_label_denylist: list[str] | None = None, data_summary: str | None = None, preview_num_records: int | None = None, ) -> EntityDetectionResult: @@ -113,6 +114,7 @@ def detect_and_validate_entities( validation_excerpt_window_chars=validation_excerpt_window_chars, validation_single_chunk_full_text=validation_single_chunk_full_text, entity_labels=entity_labels, + entity_label_denylist=entity_label_denylist, data_summary=data_summary, ) detection_result = self._adapter.run_workflow( @@ -135,6 +137,7 @@ def _build_detection_spec( validation_excerpt_window_chars: int = _DEFAULT_VALIDATION_EXCERPT_WINDOW_CHARS, validation_single_chunk_full_text: bool = True, entity_labels: list[str] | None = None, + entity_label_denylist: list[str] | None = None, data_summary: str | None = None, ) -> tuple[list[ModelConfig], list[ColumnConfigT]]: """Build the (model_configs, columns) for the core detection workflow. @@ -143,7 +146,7 @@ def _build_detection_spec( and :meth:`build_detection_config` (which exports it for an external runtime), so both paths run exactly the same workflow. """ - labels = _resolve_detection_labels(entity_labels) + labels = _resolve_detection_labels(entity_labels, set(entity_label_denylist) if entity_label_denylist else None) workflow_model_configs = self._inject_detector_params( model_configs=model_configs, selected_models=selected_models, @@ -240,6 +243,7 @@ def build_detection_config( validation_excerpt_window_chars: int = _DEFAULT_VALIDATION_EXCERPT_WINDOW_CHARS, validation_single_chunk_full_text: bool = True, entity_labels: list[str] | None = None, + entity_label_denylist: list[str] | None = None, data_summary: str | None = None, ) -> DataDesignerConfigBuilder: """Build (without executing) the core detection workflow as a DataDesigner @@ -255,6 +259,7 @@ def build_detection_config( validation_excerpt_window_chars=validation_excerpt_window_chars, validation_single_chunk_full_text=validation_single_chunk_full_text, entity_labels=entity_labels, + entity_label_denylist=entity_label_denylist, data_summary=data_summary, ) return self._adapter.build_config( @@ -275,6 +280,7 @@ def build_detection_builder_for_seed( validation_excerpt_window_chars: int = _DEFAULT_VALIDATION_EXCERPT_WINDOW_CHARS, validation_single_chunk_full_text: bool = True, entity_labels: list[str] | None = None, + entity_label_denylist: list[str] | None = None, data_summary: str | None = None, job_index: int = 0, num_jobs: int = 1, @@ -295,6 +301,7 @@ def build_detection_builder_for_seed( validation_excerpt_window_chars=validation_excerpt_window_chars, validation_single_chunk_full_text=validation_single_chunk_full_text, entity_labels=entity_labels, + entity_label_denylist=entity_label_denylist, data_summary=data_summary, ) return self._adapter.build_config_for_seed( @@ -313,6 +320,7 @@ def identify_latent_entities( selected_models: DetectionModelSelection, gliner_detection_threshold: float, entity_labels: list[str] | None = None, + entity_label_denylist: list[str] | None = None, privacy_goal: PrivacyGoal | None, data_summary: str | None = None, preview_num_records: int | None = None, @@ -322,7 +330,7 @@ def identify_latent_entities( Runs after ``detect_and_validate_entities`` when rewrite mode is enabled. Uses an LLM to identify entities inferable from context. """ - labels = _resolve_detection_labels(entity_labels) + labels = _resolve_detection_labels(entity_labels, set(entity_label_denylist) if entity_label_denylist else None) workflow_model_configs = self._inject_detector_params( model_configs=model_configs, selected_models=selected_models, @@ -360,6 +368,7 @@ def run( validation_excerpt_window_chars: int = _DEFAULT_VALIDATION_EXCERPT_WINDOW_CHARS, validation_single_chunk_full_text: bool = True, entity_labels: list[str] | None = None, + entity_label_denylist: list[str] | None = None, privacy_goal: PrivacyGoal | None = None, data_summary: str | None = None, tag_latent_entities: bool = True, @@ -390,6 +399,7 @@ def run( validation_excerpt_window_chars=validation_excerpt_window_chars, validation_single_chunk_full_text=validation_single_chunk_full_text, entity_labels=entity_labels, + entity_label_denylist=entity_label_denylist, data_summary=data_summary, preview_num_records=preview_num_records, ) @@ -401,6 +411,7 @@ def run( selected_models=selected_models, gliner_detection_threshold=gliner_detection_threshold, entity_labels=entity_labels, + entity_label_denylist=entity_label_denylist, privacy_goal=privacy_goal, data_summary=data_summary, preview_num_records=preview_num_records, @@ -417,8 +428,11 @@ def run( # TODO(docs): document this None-vs-explicit contract in user-facing docs. if COL_DETECTED_ENTITIES in final_df.columns: allowed = set(entity_labels) if entity_labels is not None else None + entity_label_denylist_set = set(entity_label_denylist) if entity_label_denylist else None final_df[COL_FINAL_ENTITIES] = final_df[COL_DETECTED_ENTITIES].apply( - lambda raw: _materialize_final_entities(raw, allowed_labels=allowed) + lambda raw: _materialize_final_entities( + raw, allowed_labels=allowed, entity_label_denylist=entity_label_denylist_set + ) ) if compute_grouped: final_df[COL_ENTITIES_BY_VALUE] = final_df[COL_FINAL_ENTITIES].apply(_build_entities_by_value) @@ -455,18 +469,29 @@ def _inject_detector_params( return resolved -def _resolve_detection_labels(entity_labels: list[str] | None) -> list[str]: - if entity_labels is None: - return list(DEFAULT_ENTITY_LABELS) - return list(entity_labels) +def _resolve_detection_labels( + entity_labels: list[str] | None, + entity_label_denylist: set[str] | None = None, +) -> list[str]: + labels = list(DEFAULT_ENTITY_LABELS) if entity_labels is None else list(entity_labels) + if entity_label_denylist: + labels = [label for label in labels if label not in entity_label_denylist] + return labels -def _materialize_final_entities(raw: object, *, allowed_labels: set[str] | None) -> dict: - """Build COL_FINAL_ENTITIES, optionally filtering to *allowed_labels*.""" +def _materialize_final_entities( + raw: object, + *, + allowed_labels: set[str] | None, + entity_label_denylist: set[str] | None, +) -> dict: + """Build COL_FINAL_ENTITIES, optionally filtering to *allowed_labels* and excluding *entity_label_denylist*.""" parsed = EntitiesSchema.from_raw(raw) - if allowed_labels is None: - return parsed.model_dump() - kept = [e for e in parsed.entities if e.label in allowed_labels] + kept = [ + e for e in parsed.entities + if (allowed_labels is None or e.label in allowed_labels) + and (entity_label_denylist is None or e.label not in entity_label_denylist) + ] return EntitiesSchema(entities=kept).model_dump() diff --git a/src/anonymizer/interface/anonymizer.py b/src/anonymizer/interface/anonymizer.py index 1df2351f..8fab72d0 100644 --- a/src/anonymizer/interface/anonymizer.py +++ b/src/anonymizer/interface/anonymizer.py @@ -306,6 +306,7 @@ def export_detection_config( validation_max_entities_per_call=config.detect.validation_max_entities_per_call, validation_excerpt_window_chars=config.detect.validation_excerpt_window_chars, entity_labels=config.detect.entity_labels, + entity_label_denylist=config.detect.entity_label_denylist, data_summary=data.data_summary, ) @@ -338,6 +339,7 @@ def export_detection_builder_for_seed( validation_max_entities_per_call=config.detect.validation_max_entities_per_call, validation_excerpt_window_chars=config.detect.validation_excerpt_window_chars, entity_labels=config.detect.entity_labels, + entity_label_denylist=config.detect.entity_label_denylist, data_summary=data_summary, job_index=job_index, num_jobs=num_jobs, @@ -715,6 +717,7 @@ def _run_internal_impl( validation_max_entities_per_call=config.detect.validation_max_entities_per_call, validation_excerpt_window_chars=config.detect.validation_excerpt_window_chars, entity_labels=config.detect.entity_labels, + entity_label_denylist=config.detect.entity_label_denylist, privacy_goal=config.rewrite.privacy_goal if config.rewrite else None, data_summary=data.data_summary, tag_latent_entities=config.rewrite is not None, diff --git a/tests/engine/test_detection_workflow.py b/tests/engine/test_detection_workflow.py index aed0928a..08a3e710 100644 --- a/tests/engine/test_detection_workflow.py +++ b/tests/engine/test_detection_workflow.py @@ -462,6 +462,170 @@ def test_default_entity_labels_preserves_novel_augmented_entities( assert "ipv4" in final_labels +# ── entity_label_denylist ───────────────────────────────────────────────────── + + +def test_resolve_detection_labels_denylist_removes_labels() -> None: + labels = _resolve_detection_labels(["first_name", "email", "city"], entity_label_denylist={"email"}) + assert "email" not in labels + assert "first_name" in labels + assert "city" in labels + + +def test_resolve_detection_labels_denylist_on_defaults() -> None: + labels = _resolve_detection_labels(None, entity_label_denylist={"ssn", "first_name"}) + assert "ssn" not in labels + assert "first_name" not in labels + assert "email" in labels + + +def test_resolve_detection_labels_none_denylist_is_noop() -> None: + labels = _resolve_detection_labels(["email", "city"], entity_label_denylist=None) + assert labels == ["email", "city"] + + +def test_denylist_filters_entities_from_final_entities( + stub_detector_model_configs: list[ModelConfig], + stub_detection_model_selection: DetectionModelSelection, +) -> None: + adapter = Mock() + adapter.run_workflow.return_value = WorkflowRunResult( + dataframe=pd.DataFrame( + { + COL_TEXT: ["Alice works at Acme, her email is alice@example.com"], + COL_DETECTED_ENTITIES: [ + { + "entities": [ + {"value": "Alice", "label": "first_name", "start_position": 0, "end_position": 5}, + {"value": "alice@example.com", "label": "email", "start_position": 33, "end_position": 50}, + ] + } + ], + } + ), + failed_records=[], + ) + workflow = EntityDetectionWorkflow(adapter=adapter) + + result = workflow.run( + pd.DataFrame({COL_TEXT: ["Alice works at Acme, her email is alice@example.com"]}), + model_configs=stub_detector_model_configs, + selected_models=stub_detection_model_selection, + gliner_detection_threshold=0.5, + entity_label_denylist=["email"], + tag_latent_entities=False, + ) + + final = EntitiesSchema.from_raw(result.dataframe[COL_FINAL_ENTITIES].iloc[0]) + final_labels = {e.label for e in final.entities} + assert "email" not in final_labels + assert "first_name" in final_labels + + +def test_denylist_does_not_affect_col_detected_entities( + stub_detector_model_configs: list[ModelConfig], + stub_detection_model_selection: DetectionModelSelection, +) -> None: + """COL_DETECTED_ENTITIES is the raw pre-filter output and must be untouched.""" + adapter = Mock() + adapter.run_workflow.return_value = WorkflowRunResult( + dataframe=pd.DataFrame( + { + COL_TEXT: ["Alice, alice@example.com"], + COL_DETECTED_ENTITIES: [ + { + "entities": [ + {"value": "Alice", "label": "first_name", "start_position": 0, "end_position": 5}, + {"value": "alice@example.com", "label": "email", "start_position": 7, "end_position": 24}, + ] + } + ], + } + ), + failed_records=[], + ) + workflow = EntityDetectionWorkflow(adapter=adapter) + + result = workflow.run( + pd.DataFrame({COL_TEXT: ["Alice, alice@example.com"]}), + model_configs=stub_detector_model_configs, + selected_models=stub_detection_model_selection, + gliner_detection_threshold=0.5, + entity_label_denylist=["email"], + tag_latent_entities=False, + ) + + detected = EntitiesSchema.from_raw(result.dataframe[COL_DETECTED_ENTITIES].iloc[0]) + assert "email" in {e.label for e in detected.entities} + + +def test_denylist_combined_with_allowlist_allowlist_wins_for_non_denied( + stub_detector_model_configs: list[ModelConfig], + stub_detection_model_selection: DetectionModelSelection, +) -> None: + """entity_labels restricts to an allowlist; denylist further removes from that set.""" + adapter = Mock() + adapter.run_workflow.return_value = WorkflowRunResult( + dataframe=pd.DataFrame( + { + COL_TEXT: ["Alice in Houston, alice@example.com"], + COL_DETECTED_ENTITIES: [ + { + "entities": [ + {"value": "Alice", "label": "first_name", "start_position": 0, "end_position": 5}, + {"value": "Houston", "label": "city", "start_position": 9, "end_position": 16}, + {"value": "alice@example.com", "label": "email", "start_position": 18, "end_position": 35}, + ] + } + ], + } + ), + failed_records=[], + ) + workflow = EntityDetectionWorkflow(adapter=adapter) + + result = workflow.run( + pd.DataFrame({COL_TEXT: ["Alice in Houston, alice@example.com"]}), + model_configs=stub_detector_model_configs, + selected_models=stub_detection_model_selection, + gliner_detection_threshold=0.5, + entity_labels=["first_name", "city", "email"], + entity_label_denylist=["email"], + tag_latent_entities=False, + ) + + final = EntitiesSchema.from_raw(result.dataframe[COL_FINAL_ENTITIES].iloc[0]) + final_labels = {e.label for e in final.entities} + assert final_labels == {"first_name", "city"} + + +def test_denylist_passed_to_gliner_via_labels( + stub_detector_model_configs: list[ModelConfig], + stub_detection_model_selection: DetectionModelSelection, +) -> None: + """Denied labels must be absent from the label list injected into GLiNER.""" + adapter = Mock() + adapter.run_workflow.return_value = WorkflowRunResult( + dataframe=pd.DataFrame({COL_TEXT: ["Alice"]}), failed_records=[] + ) + workflow = EntityDetectionWorkflow(adapter=adapter) + + workflow.run( + pd.DataFrame({COL_TEXT: ["Alice"]}), + model_configs=stub_detector_model_configs, + selected_models=stub_detection_model_selection, + gliner_detection_threshold=0.5, + entity_labels=["first_name", "email", "city"], + entity_label_denylist=["email"], + tag_latent_entities=False, + ) + + injected_configs = adapter.run_workflow.call_args.kwargs["model_configs"] + gliner_labels = injected_configs[0].inference_parameters.extra_body["labels"] + assert "email" not in gliner_labels + assert "first_name" in gliner_labels + assert "city" in gliner_labels + # --------------------------------------------------------------------------- # Workflow column wiring # --------------------------------------------------------------------------- From 287e67e7a9dd325071cebeaad708965900c76524 Mon Sep 17 00:00:00 2001 From: memadi Date: Mon, 10 Aug 2026 16:53:02 -0700 Subject: [PATCH 03/30] test(engine): verify entity_label_denylist respected in export detection paths Adds tests for build_detection_config and build_detection_builder_for_seed confirming that denied labels are subtracted from the GLiNER label list in the serialized workflow config, so external runtimes see the same effective label set as the in-process path. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .../test_detection_config_serialization.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/tests/engine/test_detection_config_serialization.py b/tests/engine/test_detection_config_serialization.py index 0acda66a..710f1247 100644 --- a/tests/engine/test_detection_config_serialization.py +++ b/tests/engine/test_detection_config_serialization.py @@ -93,6 +93,57 @@ def test_detection_builder_round_trips_through_native_data_designer_config(tmp_p assert "generator_params" not in serialized_text +def _get_gliner_labels_from_builder(builder: DataDesignerConfigBuilder) -> list[str]: + serialized = json.loads(builder.get_builder_config().to_json()) + model_configs = serialized["data_designer"]["model_configs"] + gliner = next(m for m in model_configs if m.get("alias") == "gliner-pii-detector") + return gliner["inference_parameters"]["extra_body"]["labels"] + + +def test_build_detection_builder_for_seed_respects_entity_label_denylist(tmp_path: Path) -> None: + seed_path = tmp_path / "seed.parquet" + pd.DataFrame({COL_TEXT: ["Alice"]}).to_parquet(seed_path, index=False) + + parsed_models = parse_model_configs(None) + workflow = EntityDetectionWorkflow(adapter=NddAdapter(data_designer=cast(DataDesigner, Mock()))) + builder = workflow.build_detection_builder_for_seed( + seed_path=seed_path, + model_configs=parsed_models.model_configs, + selected_models=parsed_models.selected_models.detection, + gliner_detection_threshold=0.3, + entity_labels=["first_name", "email", "city"], + entity_label_denylist=["email"], + ) + + labels = _get_gliner_labels_from_builder(builder) + assert "email" not in labels + assert "first_name" in labels + assert "city" in labels + + +def test_build_detection_config_respects_entity_label_denylist(tmp_path: Path) -> None: + seed_path = tmp_path / "seed.parquet" + input_df = pd.DataFrame({COL_TEXT: ["Alice"]}) + input_df.to_parquet(seed_path, index=False) + + parsed_models = parse_model_configs(None) + workflow = EntityDetectionWorkflow(adapter=NddAdapter(data_designer=cast(DataDesigner, Mock()))) + builder = workflow.build_detection_config( + input_df, + seed_path=seed_path, + model_configs=parsed_models.model_configs, + selected_models=parsed_models.selected_models.detection, + gliner_detection_threshold=0.3, + entity_labels=["first_name", "email", "city"], + entity_label_denylist=["email"], + ) + + labels = _get_gliner_labels_from_builder(builder) + assert "email" not in labels + assert "first_name" in labels + assert "city" in labels + + def test_fresh_process_discovers_plugins_when_loading_native_config(tmp_path: Path) -> None: seed_path = tmp_path / "seed.parquet" pd.DataFrame({COL_TEXT: ["Alice"]}).to_parquet(seed_path, index=False) From e2cd4f086b5e05fd2fd700873488128f2c50488b Mon Sep 17 00:00:00 2001 From: memadi Date: Mon, 10 Aug 2026 16:53:57 -0700 Subject: [PATCH 04/30] feat(engine): warn when entity_label_denylist empties the detection label set When the denylist subtracts all labels from the effective detection set, _resolve_detection_labels now emits a warning instead of silently passing an empty list to GLiNER (which returns no detections, not the default set). Co-Authored-By: Claude Sonnet 4.6 (1M context) --- src/anonymizer/engine/detection/detection_workflow.py | 5 +++++ tests/engine/test_detection_workflow.py | 8 ++++++++ 2 files changed, 13 insertions(+) diff --git a/src/anonymizer/engine/detection/detection_workflow.py b/src/anonymizer/engine/detection/detection_workflow.py index 620de03b..19d20f95 100644 --- a/src/anonymizer/engine/detection/detection_workflow.py +++ b/src/anonymizer/engine/detection/detection_workflow.py @@ -476,6 +476,11 @@ def _resolve_detection_labels( labels = list(DEFAULT_ENTITY_LABELS) if entity_labels is None else list(entity_labels) if entity_label_denylist: labels = [label for label in labels if label not in entity_label_denylist] + if not labels: + logger.warning( + "entity_label_denylist removed all labels from the effective detection set. " + "No entities will be detected." + ) return labels diff --git a/tests/engine/test_detection_workflow.py b/tests/engine/test_detection_workflow.py index 08a3e710..158798d6 100644 --- a/tests/engine/test_detection_workflow.py +++ b/tests/engine/test_detection_workflow.py @@ -4,6 +4,7 @@ from __future__ import annotations import json +import logging from unittest.mock import Mock import pandas as pd @@ -484,6 +485,13 @@ def test_resolve_detection_labels_none_denylist_is_noop() -> None: assert labels == ["email", "city"] +def test_resolve_detection_labels_empty_result_warns(caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING, logger="anonymizer.detection"): + labels = _resolve_detection_labels(["email"], entity_label_denylist={"email"}) + assert labels == [] + assert "No entities will be detected" in caplog.text + + def test_denylist_filters_entities_from_final_entities( stub_detector_model_configs: list[ModelConfig], stub_detection_model_selection: DetectionModelSelection, From 594ccbc225233b234b874af51426b2c4708affd3 Mon Sep 17 00:00:00 2001 From: memadi Date: Mon, 10 Aug 2026 17:22:05 -0700 Subject: [PATCH 05/30] feat(evaluate): propagate entity_label_denylist through evaluation pipeline Stores entity_label_denylist on AnonymizerResult and PreviewResult so evaluate() can scope its judges to the same label set that was active during detection. Changes: - AnonymizerResult / PreviewResult: add entity_label_denylist field - Anonymizer.evaluate(): extract entity_label_denylist from the result and pass it to EntityCoverageWorkflow (rewrite path) and ReplacementWorkflow.evaluate() (replace path) - EntityCoverageWorkflow: accept entity_label_denylist, pass to _filter_out_of_scope_entities in postprocess - _filter_out_of_scope_entities: exclude entities whose label is in the denylist so the judge does not penalise the output for not anonymizing denied labels - ReplacementWorkflow.evaluate(): thread entity_label_denylist through to EntityCoverageWorkflow Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .../evaluation/entity_coverage_judge.py | 13 +++- .../engine/replace/replace_runner.py | 2 + src/anonymizer/interface/anonymizer.py | 7 ++ src/anonymizer/interface/results.py | 2 + tests/engine/test_entity_coverage_judge.py | 67 +++++++++++++++++++ 5 files changed, 88 insertions(+), 3 deletions(-) diff --git a/src/anonymizer/engine/evaluation/entity_coverage_judge.py b/src/anonymizer/engine/evaluation/entity_coverage_judge.py index a260a122..0554b34f 100644 --- a/src/anonymizer/engine/evaluation/entity_coverage_judge.py +++ b/src/anonymizer/engine/evaluation/entity_coverage_judge.py @@ -345,12 +345,14 @@ def _normalize_literal_text(value: object) -> str: def _filter_out_of_scope_entities( entities: list[_CandidateT], entity_labels: list[str] | None, + entity_label_denylist: list[str] | None = None, ) -> list[_CandidateT]: """Drop entities with empty labels or labels outside the configured scope. When ``entity_labels`` is None all labels are in scope; only empty labels - are dropped. This mirrors the prompt's scope instruction deterministically - so a model that returns out-of-scope labels does not lower the coverage score. + are dropped. Labels present in ``entity_label_denylist`` are always excluded + regardless of ``entity_labels``. This mirrors the detection-time scope so the + judge does not penalise the output for not anonymizing denied labels. Label drift (e.g. the model returning ``"given_name"`` instead of ``"first_name"``) is unlikely in practice — the prompt explicitly instructs @@ -360,6 +362,7 @@ def _filter_out_of_scope_entities( meaningfully risking false negatives on well-formed responses. """ allowed = {label.casefold() for label in entity_labels} if entity_labels is not None else None + denied = {label.casefold() for label in entity_label_denylist} if entity_label_denylist is not None else None result = [] for entity in entities: label = str(entity.get("label", "")).strip() @@ -367,6 +370,8 @@ def _filter_out_of_scope_entities( continue if allowed is not None and label.casefold() not in allowed: continue + if denied is not None and label.casefold() in denied: + continue result.append(entity) return result @@ -428,10 +433,12 @@ def __init__( adapter: NddAdapter, *, entity_labels: list[str] | None = None, + entity_label_denylist: list[str] | None = None, data_summary: str | None = None, ) -> None: super().__init__(adapter) self._entity_labels = entity_labels + self._entity_label_denylist = entity_label_denylist self._data_summary = data_summary # ------------------------------------------------------------------ hooks @@ -492,7 +499,7 @@ def postprocess(self, dataframe: pd.DataFrame) -> pd.DataFrame: missed_entities_list.append([]) n_candidates_list.append(None) else: - candidates = _filter_out_of_scope_entities(candidates, self._entity_labels) + candidates = _filter_out_of_scope_entities(candidates, self._entity_labels, self._entity_label_denylist) candidates = _filter_nonliteral_entities(candidates, out[COL_TEXT].loc[idx]) candidates = _deduplicate_candidate_values(candidates) n_candidates = len(candidates) diff --git a/src/anonymizer/engine/replace/replace_runner.py b/src/anonymizer/engine/replace/replace_runner.py index 8da79241..20f734df 100644 --- a/src/anonymizer/engine/replace/replace_runner.py +++ b/src/anonymizer/engine/replace/replace_runner.py @@ -119,6 +119,7 @@ def evaluate( selected_models: EvaluateModelSelection, preview_num_records: int | None = None, entity_labels: list[str] | None = None, + entity_label_denylist: list[str] | None = None, compute_detection_validity: bool = False, data_summary: str | None = None, ) -> ReplacementResult: @@ -151,6 +152,7 @@ def evaluate( entity_coverage_judge = EntityCoverageWorkflow( adapter=self._adapter, # type: ignore[arg-type] entity_labels=entity_labels, + entity_label_denylist=entity_label_denylist, data_summary=data_summary, ) failed_records: list[FailedRecord] = [] diff --git a/src/anonymizer/interface/anonymizer.py b/src/anonymizer/interface/anonymizer.py index 8fab72d0..9feade6d 100644 --- a/src/anonymizer/interface/anonymizer.py +++ b/src/anonymizer/interface/anonymizer.py @@ -379,6 +379,7 @@ def preview( replace_method=config.replace, rewrite_config=config.rewrite.privacy_goal if config.rewrite is not None else None, entity_labels=config.detect.entity_labels, + entity_label_denylist=config.detect.entity_label_denylist, data_summary=result.data_summary, ) except KeyboardInterrupt: @@ -461,6 +462,7 @@ def evaluate( raise InvalidConfigError(str(exc)) from exc entity_labels = getattr(output, "entity_labels", None) + entity_label_denylist = getattr(output, "entity_label_denylist", None) data_summary = getattr(output, "data_summary", None) num_records = len(output.trace_dataframe) mode_name = "rewrite" if is_rewrite else type(replace_method).__name__ @@ -524,6 +526,7 @@ def evaluate( coverage_wf = EntityCoverageWorkflow( adapter=self._adapter, entity_labels=entity_labels, + entity_label_denylist=entity_label_denylist, data_summary=data_summary, ) logger.info(LOG_INDENT + "🔎 Running entity coverage") @@ -555,6 +558,7 @@ def evaluate( failed_records=all_failed, rewrite_config=rewrite_config, entity_labels=entity_labels, + entity_label_denylist=entity_label_denylist, data_summary=data_summary, ) else: @@ -568,6 +572,7 @@ def evaluate( model_configs=self._model_configs, selected_models=self._selected_models.evaluate, entity_labels=entity_labels, + entity_label_denylist=entity_label_denylist, compute_detection_validity=evaluate_config.compute_detection_validity, data_summary=data_summary, ) @@ -603,6 +608,7 @@ def evaluate( failed_records=replace_result.failed_records, replace_method=replace_method, entity_labels=entity_labels, + entity_label_denylist=entity_label_denylist, data_summary=data_summary, ) @@ -813,6 +819,7 @@ def _run_internal_impl( replace_method=config.replace, rewrite_config=config.rewrite.privacy_goal if config.rewrite is not None else None, entity_labels=config.detect.entity_labels, + entity_label_denylist=config.detect.entity_label_denylist, data_summary=data.data_summary, ) diff --git a/src/anonymizer/interface/results.py b/src/anonymizer/interface/results.py index 7bfe13f0..2d2b1ecb 100644 --- a/src/anonymizer/interface/results.py +++ b/src/anonymizer/interface/results.py @@ -76,6 +76,7 @@ class AnonymizerResult(_DisplayMixin): replace_method: ReplaceMethod | None = None rewrite_config: PrivacyGoal | None = None entity_labels: list[str] | None = None + entity_label_denylist: list[str] | None = None data_summary: str | None = None _display_cycle_index: int = field(default=0, init=False, repr=False) @@ -123,6 +124,7 @@ class PreviewResult(_DisplayMixin): replace_method: ReplaceMethod | None = None rewrite_config: PrivacyGoal | None = None entity_labels: list[str] | None = None + entity_label_denylist: list[str] | None = None data_summary: str | None = None _display_cycle_index: int = field(default=0, init=False, repr=False) diff --git a/tests/engine/test_entity_coverage_judge.py b/tests/engine/test_entity_coverage_judge.py index 7aee837e..355c483b 100644 --- a/tests/engine/test_entity_coverage_judge.py +++ b/tests/engine/test_entity_coverage_judge.py @@ -440,3 +440,70 @@ def test_filter_out_of_scope_entities_is_case_insensitive() -> None: entities = [{"value": "Alice", "label": "First_Name", "reasoning": "..."}] result = _filter_out_of_scope_entities(entities, entity_labels=["first_name"]) assert result == entities + + +# ── entity_label_denylist ───────────────────────────────────────────────────── + + +def test_filter_out_of_scope_entities_denylist_excludes_denied_label() -> None: + entities = [ + {"value": "Alice", "label": "first_name", "reasoning": "..."}, + {"value": "alice@example.com", "label": "email", "reasoning": "..."}, + ] + result = _filter_out_of_scope_entities(entities, entity_labels=None, entity_label_denylist=["email"]) + assert result == [{"value": "Alice", "label": "first_name", "reasoning": "..."}] + + +def test_filter_out_of_scope_entities_denylist_combined_with_allowlist() -> None: + entities = [ + {"value": "Alice", "label": "first_name", "reasoning": "..."}, + {"value": "alice@example.com", "label": "email", "reasoning": "..."}, + {"value": "Houston", "label": "city", "reasoning": "..."}, + ] + result = _filter_out_of_scope_entities( + entities, + entity_labels=["first_name", "email", "city"], + entity_label_denylist=["email"], + ) + assert {e["label"] for e in result} == {"first_name", "city"} + + +def test_filter_out_of_scope_entities_denylist_is_case_insensitive() -> None: + entities = [{"value": "alice@example.com", "label": "Email", "reasoning": "..."}] + result = _filter_out_of_scope_entities(entities, entity_labels=None, entity_label_denylist=["email"]) + assert result == [] + + +def test_filter_out_of_scope_entities_none_denylist_is_noop() -> None: + entities = [ + {"value": "Alice", "label": "first_name", "reasoning": "..."}, + {"value": "alice@example.com", "label": "email", "reasoning": "..."}, + ] + result = _filter_out_of_scope_entities(entities, entity_labels=None, entity_label_denylist=None) + assert result == entities + + +def test_entity_coverage_workflow_passes_denylist_to_filter() -> None: + """Denied labels must be excluded from candidate entities in postprocess.""" + raw_judge_output = [ + {"value": "Alice", "label": "first_name", "reasoning": "not replaced"}, + {"value": "alice@example.com", "label": "email", "reasoning": "not replaced"}, + ] + entities_by_value = {"entities_by_value": [{"value": "Alice", "label": "first_name", "mentions": []}]} + input_df = pd.DataFrame( + { + COL_TEXT: ["Alice alice@example.com"], + COL_ENTITIES_BY_VALUE: [entities_by_value], + "_raw_entity_coverage_judge": [{"candidate_entities": raw_judge_output}], + } + ) + + workflow = EntityCoverageWorkflow( + adapter=Mock(), + entity_labels=None, + entity_label_denylist=["email"], + ) + result_df = workflow.postprocess(workflow.prepare(input_df)) + missed = result_df[COL_MISSED_ENTITIES].iloc[0] + missed_labels = {e["label"] for e in missed} + assert "email" not in missed_labels From 2933a18fe3336df04d66e0d1ea17feb40d9642b0 Mon Sep 17 00:00:00 2001 From: memadi Date: Mon, 10 Aug 2026 17:33:45 -0700 Subject: [PATCH 06/30] feat(evaluate): exclude denied labels from coverage judge prompt and filter Signed-off-by: memadi --- .../evaluation/entity_coverage_judge.py | 34 ++++++++--- tests/engine/test_entity_coverage_judge.py | 59 +++++++++---------- 2 files changed, 53 insertions(+), 40 deletions(-) diff --git a/src/anonymizer/engine/evaluation/entity_coverage_judge.py b/src/anonymizer/engine/evaluation/entity_coverage_judge.py index 0554b34f..1fd7389e 100644 --- a/src/anonymizer/engine/evaluation/entity_coverage_judge.py +++ b/src/anonymizer/engine/evaluation/entity_coverage_judge.py @@ -68,6 +68,24 @@ class EntityCoverageSchema(BaseModel): # --------------------------------------------------------------------------- +def _effective_entity_labels( + entity_labels: list[str] | None, + entity_label_denylist: list[str] | None, +) -> list[str] | None: + """Return the effective label set for prompt and filter scope. + + Subtracts denied labels from entity_labels (or the full default set when + entity_labels is None). Returns None only when entity_labels is None and + no denylist is set, preserving the "evaluate all PII types" prompt path. + """ + if not entity_label_denylist: + return entity_labels + denied = {label.casefold() for label in entity_label_denylist} + base = entity_labels if entity_labels is not None else list(DEFAULT_ENTITY_LABELS) + effective = [label for label in base if label.casefold() not in denied] + return effective + + def _entity_type_scope_block(entity_labels: list[str] | None) -> str: if entity_labels is None: return "\nEvaluate for all PII and sensitive entity types.\n" @@ -345,14 +363,12 @@ def _normalize_literal_text(value: object) -> str: def _filter_out_of_scope_entities( entities: list[_CandidateT], entity_labels: list[str] | None, - entity_label_denylist: list[str] | None = None, ) -> list[_CandidateT]: """Drop entities with empty labels or labels outside the configured scope. When ``entity_labels`` is None all labels are in scope; only empty labels - are dropped. Labels present in ``entity_label_denylist`` are always excluded - regardless of ``entity_labels``. This mirrors the detection-time scope so the - judge does not penalise the output for not anonymizing denied labels. + are dropped. This mirrors the prompt's scope instruction deterministically + so a model that returns out-of-scope labels does not lower the coverage score. Label drift (e.g. the model returning ``"given_name"`` instead of ``"first_name"``) is unlikely in practice — the prompt explicitly instructs @@ -362,7 +378,6 @@ def _filter_out_of_scope_entities( meaningfully risking false negatives on well-formed responses. """ allowed = {label.casefold() for label in entity_labels} if entity_labels is not None else None - denied = {label.casefold() for label in entity_label_denylist} if entity_label_denylist is not None else None result = [] for entity in entities: label = str(entity.get("label", "")).strip() @@ -370,8 +385,6 @@ def _filter_out_of_scope_entities( continue if allowed is not None and label.casefold() not in allowed: continue - if denied is not None and label.casefold() in denied: - continue result.append(entity) return result @@ -473,10 +486,11 @@ def _extract_invalid(cls, parsed: BaseModel) -> list[dict[str, object]]: def column_config(self, selected_models: EvaluateModelSelection) -> LLMStructuredColumnConfig: """Override to inject instance-specific entity_labels and data_summary.""" + effective_labels = _effective_entity_labels(self._entity_labels, self._entity_label_denylist) return LLMStructuredColumnConfig( name=self.RAW_COL, prompt=_coverage_prompt( - entity_labels=self._entity_labels, + entity_labels=effective_labels, data_summary=self._data_summary, ), model_alias=resolve_model_alias(self.MODEL_ROLE, selected_models), @@ -499,7 +513,9 @@ def postprocess(self, dataframe: pd.DataFrame) -> pd.DataFrame: missed_entities_list.append([]) n_candidates_list.append(None) else: - candidates = _filter_out_of_scope_entities(candidates, self._entity_labels, self._entity_label_denylist) + candidates = _filter_out_of_scope_entities( + candidates, _effective_entity_labels(self._entity_labels, self._entity_label_denylist) + ) candidates = _filter_nonliteral_entities(candidates, out[COL_TEXT].loc[idx]) candidates = _deduplicate_candidate_values(candidates) n_candidates = len(candidates) diff --git a/tests/engine/test_entity_coverage_judge.py b/tests/engine/test_entity_coverage_judge.py index 355c483b..2b134b6f 100644 --- a/tests/engine/test_entity_coverage_judge.py +++ b/tests/engine/test_entity_coverage_judge.py @@ -21,6 +21,7 @@ _FINAL_ENTITIES_FOR_COVERAGE_COL, EntityCoverageWorkflow, _coverage_prompt, + _effective_entity_labels, _filter_out_of_scope_entities, _find_missed_candidates, _is_candidate_value_covered, @@ -445,45 +446,41 @@ def test_filter_out_of_scope_entities_is_case_insensitive() -> None: # ── entity_label_denylist ───────────────────────────────────────────────────── -def test_filter_out_of_scope_entities_denylist_excludes_denied_label() -> None: - entities = [ - {"value": "Alice", "label": "first_name", "reasoning": "..."}, - {"value": "alice@example.com", "label": "email", "reasoning": "..."}, - ] - result = _filter_out_of_scope_entities(entities, entity_labels=None, entity_label_denylist=["email"]) - assert result == [{"value": "Alice", "label": "first_name", "reasoning": "..."}] +def test_effective_entity_labels_no_denylist_returns_entity_labels_unchanged() -> None: + assert _effective_entity_labels(["email", "city"], None) == ["email", "city"] -def test_filter_out_of_scope_entities_denylist_combined_with_allowlist() -> None: - entities = [ - {"value": "Alice", "label": "first_name", "reasoning": "..."}, - {"value": "alice@example.com", "label": "email", "reasoning": "..."}, - {"value": "Houston", "label": "city", "reasoning": "..."}, - ] - result = _filter_out_of_scope_entities( - entities, - entity_labels=["first_name", "email", "city"], - entity_label_denylist=["email"], - ) - assert {e["label"] for e in result} == {"first_name", "city"} +def test_effective_entity_labels_none_labels_none_denylist_returns_none() -> None: + assert _effective_entity_labels(None, None) is None -def test_filter_out_of_scope_entities_denylist_is_case_insensitive() -> None: - entities = [{"value": "alice@example.com", "label": "Email", "reasoning": "..."}] - result = _filter_out_of_scope_entities(entities, entity_labels=None, entity_label_denylist=["email"]) - assert result == [] +def test_effective_entity_labels_subtracts_denylist_from_explicit_labels() -> None: + result = _effective_entity_labels(["first_name", "email", "city"], ["email"]) + assert result == ["first_name", "city"] -def test_filter_out_of_scope_entities_none_denylist_is_noop() -> None: - entities = [ - {"value": "Alice", "label": "first_name", "reasoning": "..."}, - {"value": "alice@example.com", "label": "email", "reasoning": "..."}, - ] - result = _filter_out_of_scope_entities(entities, entity_labels=None, entity_label_denylist=None) - assert result == entities +def test_effective_entity_labels_subtracts_denylist_from_defaults() -> None: + result = _effective_entity_labels(None, ["ssn", "first_name"]) + assert result is not None + assert "ssn" not in result + assert "first_name" not in result + assert "email" in result + + +def test_effective_entity_labels_is_case_insensitive() -> None: + result = _effective_entity_labels(["first_name", "Email"], ["email"]) + assert result == ["first_name"] + + +def test_coverage_prompt_excludes_denied_labels_from_scope() -> None: + effective = _effective_entity_labels(["first_name", "email", "city"], ["email"]) + prompt = _coverage_prompt(entity_labels=effective) + assert "email" not in prompt + assert "first_name" in prompt + assert "city" in prompt -def test_entity_coverage_workflow_passes_denylist_to_filter() -> None: +def test_entity_coverage_workflow_excludes_denied_labels_from_postprocess() -> None: """Denied labels must be excluded from candidate entities in postprocess.""" raw_judge_output = [ {"value": "Alice", "label": "first_name", "reasoning": "not replaced"}, From baf0ac410be602f0775616b1e86455a8b6e63c88 Mon Sep 17 00:00:00 2001 From: memadi Date: Mon, 10 Aug 2026 17:38:55 -0700 Subject: [PATCH 07/30] update docstring Signed-off-by: memadi --- .../engine/evaluation/entity_coverage_judge.py | 5 +++++ src/anonymizer/engine/replace/replace_runner.py | 4 ++++ src/anonymizer/interface/results.py | 14 ++++++++++++++ 3 files changed, 23 insertions(+) diff --git a/src/anonymizer/engine/evaluation/entity_coverage_judge.py b/src/anonymizer/engine/evaluation/entity_coverage_judge.py index 1fd7389e..0a1e3f28 100644 --- a/src/anonymizer/engine/evaluation/entity_coverage_judge.py +++ b/src/anonymizer/engine/evaluation/entity_coverage_judge.py @@ -426,6 +426,11 @@ class EntityCoverageWorkflow(_BaseJudgeWorkflow): The judge independently extracts candidates from the original text and entity-type scope. Deterministic postprocessing removes nonliteral and already-covered findings. + ``entity_labels`` scopes evaluation to a specific allowlist of labels (``None`` means + all default labels). ``entity_label_denylist`` further excludes specific labels from + scope regardless of ``entity_labels``. Both are applied to the LLM prompt and the + postprocess filter so denied labels are never penalised in the coverage score. + Output columns: ``COL_ENTITY_COVERAGE`` (float|None) — covered / total unique candidate values ``COL_MISSED_ENTITIES`` (list) — missed entities with value, label, reasoning diff --git a/src/anonymizer/engine/replace/replace_runner.py b/src/anonymizer/engine/replace/replace_runner.py index 20f734df..61b8941d 100644 --- a/src/anonymizer/engine/replace/replace_runner.py +++ b/src/anonymizer/engine/replace/replace_runner.py @@ -130,6 +130,10 @@ def evaluate( Detection validity runs only when ``compute_detection_validity=True``. All active judges are submitted as columns of one DataDesigner workflow. + ``entity_labels`` and ``entity_label_denylist`` together define the label + scope passed to the coverage judge — only entities whose labels were in + scope during detection are evaluated. + Raises ``ValueError`` if the workflow has no adapter wired up or if the dataframe is missing the columns the judges read. """ diff --git a/src/anonymizer/interface/results.py b/src/anonymizer/interface/results.py index 2d2b1ecb..8f82791e 100644 --- a/src/anonymizer/interface/results.py +++ b/src/anonymizer/interface/results.py @@ -64,6 +64,13 @@ class AnonymizerResult(_DisplayMixin): mode was used. Set by ``run()`` / ``preview()``; consumed by ``evaluate()`` to dispatch the rewrite judges. Mutually exclusive with ``replace_method``. + entity_labels: Allowlist of entity labels that were in scope during + detection. Preserved for ``evaluate()`` so the coverage judge scopes + its evaluation to the same label set. ``None`` means all default + labels were in scope. + entity_label_denylist: Labels that were explicitly excluded from + detection. Preserved for ``evaluate()`` so the coverage judge does + not penalise the output for not anonymizing denied labels. data_summary: Optional dataset context supplied with the original input. Preserved for ``evaluate()`` so entity-coverage judging uses the same context as detection. @@ -111,6 +118,13 @@ class PreviewResult(_DisplayMixin): rewrite_config: The privacy goal that produced this preview when rewrite mode was used. Set by ``preview()``; consumed by ``evaluate()`` to dispatch the rewrite judges. Mutually exclusive with ``replace_method``. + entity_labels: Allowlist of entity labels that were in scope during + detection. Preserved for ``evaluate()`` so the coverage judge scopes + its evaluation to the same label set. ``None`` means all default + labels were in scope. + entity_label_denylist: Labels that were explicitly excluded from + detection. Preserved for ``evaluate()`` so the coverage judge does + not penalise the output for not anonymizing denied labels. data_summary: Optional dataset context supplied with the original input. Preserved for ``evaluate()`` so entity-coverage judging uses the same context as detection. From 06db3cea0edb825ea34876219a9eea0c589bddc3 Mon Sep 17 00:00:00 2001 From: memadi Date: Mon, 10 Aug 2026 17:43:03 -0700 Subject: [PATCH 08/30] update docs accroding to deny entity list Signed-off-by: memadi --- docs/concepts/choosing-a-strategy.md | 15 ++++++++++++++- docs/concepts/detection.md | 16 ++++++++++++++++ docs/concepts/evaluation.md | 1 + docs/troubleshooting.md | 10 +++++++--- skills/anonymizer/SKILL.md | 1 + .../engine/detection/detection_workflow.py | 6 +++--- tests/config/test_anonymizer_config.py | 10 +++++++--- tests/engine/test_detection_workflow.py | 1 + 8 files changed, 50 insertions(+), 10 deletions(-) diff --git a/docs/concepts/choosing-a-strategy.md b/docs/concepts/choosing-a-strategy.md index 50f5a6e7..835704d3 100644 --- a/docs/concepts/choosing-a-strategy.md +++ b/docs/concepts/choosing-a-strategy.md @@ -43,11 +43,12 @@ What to include: - The domain (clinical, legal, financial, customer support, etc.) - The genre (notes, transcripts, opinions, biographies) - Anything about the source the engine couldn't infer from a single record (e.g. "transcribed phone calls — expect disfluencies") -- `data_summary` is the only way to provide a soft do-not-tag list for the augmenter when `entity_labels=None` — the augmenter is free to invent labels beyond `DEFAULT_ENTITY_LABELS`, so use it to tell the LLM what *not* to tag (e.g. "do not tag generic anatomical terms, medication class names, or job titles as PII"). +- `data_summary` is a soft way to guide the augmenter when `entity_labels=None` — the augmenter is free to invent labels beyond `DEFAULT_ENTITY_LABELS`, so use it to tell the LLM what *not* to tag (e.g. "do not tag generic anatomical terms, medication class names, or job titles as PII"). For a hard exclusion of specific label types, use `Detect.entity_label_denylist` instead. What to leave out: - Lists of entity types **you want detected** (those go in `Detect.entity_labels`) +- Lists of entity types **you never want detected** (those go in `Detect.entity_label_denylist`) - Privacy/utility goals (those go in `Rewrite.privacy_goal`) - Substitute behavior instructions (e.g. "names should remain Portuguese", "preserve numeric magnitude") — those go in `Substitute(instructions)` - Generic phrasing ("text data" adds no signal) @@ -78,6 +79,18 @@ from anonymizer import DEFAULT_ENTITY_LABELS, Detect detect = Detect(entity_labels=[*DEFAULT_ENTITY_LABELS, "clinical_facility", "diagnosis_code", "medication_name"]) ``` +### `entity_label_denylist` + +Use when you want to **exclude** specific label types from detection without enumerating the entire allowlist. Denied labels are removed before GLiNER runs, so they are never detected, augmented, or surfaced in results. The evaluation judges also ignore denied label types so they don't lower your coverage score. + +```python +# Never detect occupation or gender, keep everything else +Detect(entity_label_denylist=["occupation", "gender"]) + +# Combine with an explicit allowlist — denylist always wins +Detect(entity_labels=["first_name", "email", "city"], entity_label_denylist=["city"]) +``` + ### `gliner_threshold` Default `0.3`. The validator catches false positives downstream, so erring low is safe. diff --git a/docs/concepts/detection.md b/docs/concepts/detection.md index 83fbd68d..949bbce9 100644 --- a/docs/concepts/detection.md +++ b/docs/concepts/detection.md @@ -44,6 +44,7 @@ config = AnonymizerConfig( | Field | Default | Description | |-------|---------|-------------| | `entity_labels` | `None` (all defaults) | List of labels to detect. Leave unset (or pass `None`) to use the full default set. | +| `entity_label_denylist` | `None` | List of labels to **never** detect, even if present in `entity_labels` or the default set. Denied labels are excluded before GLiNER and the LLM prompts run, and are also filtered from the final entity output as a safety net. | | `gliner_threshold` | `0.3` | GLiNER confidence threshold (0.0--1.0). Lower values detect more entities but may increase false positives. | | `validation_max_entities_per_call` | `100` | Maximum candidate entities per validator LLM call. Rows with more candidates are split into chunks. See [Chunked validation](#chunked-validation). | | `validation_excerpt_window_chars` | `500` | Characters of context included before and after a chunk's entity spans in the validator prompt. Bounds per-chunk prompt size; not the model's context-window limit. | @@ -104,6 +105,21 @@ Detect(entity_labels=["first_name", "last_name", "email"]) # Permissive: detect all defaults + LLM can infer new label types Detect() # entity_labels=None ``` + +### Excluding labels with a deny list + +Use `entity_label_denylist` to exclude specific labels from detection without having to enumerate the entire allowlist. Denied labels are removed before GLiNER runs and before the LLM prompts are built, so they are never detected or augmented. + +```python +# Detect all defaults except occupation and gender +Detect(entity_label_denylist=["occupation", "gender"]) + +# Combine with an explicit allowlist — denylist always wins +Detect(entity_labels=["first_name", "email", "city"], entity_label_denylist=["city"]) +``` + +!!! warning + If every label in `entity_labels` is also in `entity_label_denylist`, the effective detection set is empty and no entities will be detected. Anonymizer logs a warning when this happens. ## Tuning the threshold For `gliner_threshold`, start with the default `0.3`. If you're seeing too many false positives, raise it to `0.5`. If entities are being missed, try lowering to `0.2`. The LLM validation step catches many false positives, so erring on the side of lower thresholds is usually safe. diff --git a/docs/concepts/evaluation.md b/docs/concepts/evaluation.md index fb04588f..68c83435 100644 --- a/docs/concepts/evaluation.md +++ b/docs/concepts/evaluation.md @@ -66,6 +66,7 @@ Note: the judge measures detection recall, not output leakage. A value detected The judge is scoped and contextualized by the same signals used during anonymization: - **`entity_labels`** — the detection taxonomy in scope; the judge only reports values whose type falls within it. +- **`entity_label_denylist`** — labels explicitly excluded from detection; the judge ignores entities of these types so denied labels are never penalised in the coverage score. - **`data_summary`** — used purely to interpret literal values and their semantic types, never to invent entities absent from the text. | Output column | Type | Description | diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 7bf655bb..b98467ab 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -116,9 +116,13 @@ Verify by re-running `preview` with `Annotate` and confirming the entity now app Symptoms: detected entities include obvious common words, dates that aren't dates, etc. -1. **Raise `gliner_threshold`** to `0.5`. The augmenter will pick up real misses, so this rarely costs recall. -2. **Lower `validation_excerpt_window_chars`** (default `500`) if context-driven validation is being misled by far-away sentences. Smaller per-chunk prompts trade context for precision. -3. **Sanity-check the validator with an `Annotate` preview.** A flaky validator (or a misconfigured alias) returns "keep" on almost everything, which presents as recall going way up — easiest spotted by eyeballing the entity list on a handful of rows. +1. **Use `Detect.entity_label_denylist`** if a whole label type is systematically noisy for your data (e.g. `occupation` tagging generic job words, `age` tagging durations). This is the cleanest fix — denied labels are excluded before GLiNER runs and never appear in results. + ```python + Detect(entity_label_denylist=["occupation", "age"]) + ``` +2. **Raise `gliner_threshold`** to `0.5`. The augmenter will pick up real misses, so this rarely costs recall. +3. **Lower `validation_excerpt_window_chars`** (default `500`) if context-driven validation is being misled by far-away sentences. Smaller per-chunk prompts trade context for precision. +4. **Sanity-check the validator with an `Annotate` preview.** A flaky validator (or a misconfigured alias) returns "keep" on almost everything, which presents as recall going way up — easiest spotted by eyeballing the entity list on a handful of rows. ### A new domain isn't being detected well diff --git a/skills/anonymizer/SKILL.md b/skills/anonymizer/SKILL.md index 30642118..bb7b332c 100644 --- a/skills/anonymizer/SKILL.md +++ b/skills/anonymizer/SKILL.md @@ -43,6 +43,7 @@ regulatory and business context. # Usage Tips and Common Pitfalls - **`Detect.entity_labels=None` (the default) is permissive** — the augmenter LLM may invent labels not in `DEFAULT_ENTITY_LABELS`. Setting an explicit list switches to **strict mode** where *only* the listed labels are detected. To add domain labels, *extend* the default, don't replace it: `entity_labels=[*DEFAULT_ENTITY_LABELS, "clinical_facility", ...]` (`DEFAULT_ENTITY_LABELS` is a tuple, so unpack it into a list). Match the snake_case convention of `DEFAULT_ENTITY_LABELS`. +- **`Detect.entity_label_denylist`** excludes specific label types from detection entirely — denied labels are removed before GLiNER runs and are never detected, augmented, or penalised in evaluation scores. Use it when a label type is systematically noisy for your data or should never be anonymized (e.g. `Detect(entity_label_denylist=["occupation", "gender"])`). The denylist takes precedence over `entity_labels` — a label in both is never detected. - **GLiNER is zero-shot** — entity labels are natural-language concept names (e.g. `"clinical_facility"`, `"internal_project_codename"`), not codes or enum values. Any concept you can name in English is a label GLiNER can detect. - **`Rewrite.instructions` is a dead field today** — it exists on the model but the rewrite engine never reads it. Do not use it. Put rewriter guidance in `privacy_goal.protect` / `privacy_goal.preserve` instead. - **`risk_tolerance` only applies to Rewrite mode**, not Replace. diff --git a/src/anonymizer/engine/detection/detection_workflow.py b/src/anonymizer/engine/detection/detection_workflow.py index 19d20f95..30a28be6 100644 --- a/src/anonymizer/engine/detection/detection_workflow.py +++ b/src/anonymizer/engine/detection/detection_workflow.py @@ -478,8 +478,7 @@ def _resolve_detection_labels( labels = [label for label in labels if label not in entity_label_denylist] if not labels: logger.warning( - "entity_label_denylist removed all labels from the effective detection set. " - "No entities will be detected." + "entity_label_denylist removed all labels from the effective detection set. No entities will be detected." ) return labels @@ -493,7 +492,8 @@ def _materialize_final_entities( """Build COL_FINAL_ENTITIES, optionally filtering to *allowed_labels* and excluding *entity_label_denylist*.""" parsed = EntitiesSchema.from_raw(raw) kept = [ - e for e in parsed.entities + e + for e in parsed.entities if (allowed_labels is None or e.label in allowed_labels) and (entity_label_denylist is None or e.label not in entity_label_denylist) ] diff --git a/tests/config/test_anonymizer_config.py b/tests/config/test_anonymizer_config.py index 1381eade..02e7daa8 100644 --- a/tests/config/test_anonymizer_config.py +++ b/tests/config/test_anonymizer_config.py @@ -3,14 +3,18 @@ from __future__ import annotations +import logging from pathlib import Path import pytest from pydantic import ValidationError -import logging - -from anonymizer.config.anonymizer_config import AnonymizerConfig, AnonymizerInput, Detect, Rewrite, infer_input_source_suffix +from anonymizer.config.anonymizer_config import ( + AnonymizerConfig, + AnonymizerInput, + Rewrite, + infer_input_source_suffix, +) from anonymizer.config.replace_strategies import ( Annotate, Hash, diff --git a/tests/engine/test_detection_workflow.py b/tests/engine/test_detection_workflow.py index 158798d6..9c636d65 100644 --- a/tests/engine/test_detection_workflow.py +++ b/tests/engine/test_detection_workflow.py @@ -634,6 +634,7 @@ def test_denylist_passed_to_gliner_via_labels( assert "first_name" in gliner_labels assert "city" in gliner_labels + # --------------------------------------------------------------------------- # Workflow column wiring # --------------------------------------------------------------------------- From 78a57827cd95d8575af648c8e5ec7455055b056d Mon Sep 17 00:00:00 2001 From: memadi Date: Mon, 10 Aug 2026 17:48:23 -0700 Subject: [PATCH 09/30] add entity_label_denylist to telemetry Signed-off-by: memadi --- src/anonymizer/measurement/records/run.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/anonymizer/measurement/records/run.py b/src/anonymizer/measurement/records/run.py index 4a9e95b9..fcff24d9 100644 --- a/src/anonymizer/measurement/records/run.py +++ b/src/anonymizer/measurement/records/run.py @@ -20,11 +20,13 @@ def _detect_config_metadata(detect: Any | None) -> dict[str, Any]: entity_label_count = len(DEFAULT_ENTITY_LABELS) else: entity_label_count = len(entity_labels) + entity_label_denylist = getattr(detect, "entity_label_denylist", None) return { "gliner_threshold": getattr(detect, "gliner_threshold", None), "entity_label_source": "custom" if entity_labels is not None else "default", "entity_label_count": entity_label_count, "entity_labels": list(entity_labels) if entity_labels is not None else None, + "entity_label_denylist": list(entity_label_denylist) if entity_label_denylist is not None else None, "validation_max_entities_per_call": getattr(detect, "validation_max_entities_per_call", None), "validation_excerpt_window_chars": getattr(detect, "validation_excerpt_window_chars", None), } From 7d46c3d7a61bc4dc2952692cf3944c39d37cda98 Mon Sep 17 00:00:00 2001 From: memadi Date: Mon, 10 Aug 2026 17:50:58 -0700 Subject: [PATCH 10/30] add test for entity_label_denylist to telemetry Signed-off-by: memadi --- tests/test_measurement.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_measurement.py b/tests/test_measurement.py index 5224bc8b..e3129e93 100644 --- a/tests/test_measurement.py +++ b/tests/test_measurement.py @@ -472,6 +472,7 @@ def test_anonymizer_records_per_record_measurement_without_raw_pii(tmp_path: Pat assert run_record["input_has_data_summary"] is False assert run_record["detect"]["entity_label_source"] == "default" assert run_record["detect"]["entity_label_count"] > 0 + assert run_record["detect"]["entity_label_denylist"] is None assert run_record["replace"]["strategy"] == "Redact" assert run_record["replace"]["normalize_label"] is True assert len(run_record["source_hash"]) == 64 @@ -482,6 +483,23 @@ def test_anonymizer_records_per_record_measurement_without_raw_pii(tmp_path: Pat assert str(input_csv) not in serialized +def test_detect_config_metadata_includes_entity_label_denylist() -> None: + from anonymizer.measurement.records.run import _detect_config_metadata + + detect = Detect(entity_labels=["first_name", "email"], entity_label_denylist=["email"]) + metadata = _detect_config_metadata(detect) + assert metadata["entity_label_denylist"] == ["email"] + assert metadata["entity_labels"] == ["email", "first_name"] + + +def test_detect_config_metadata_denylist_none_when_not_set() -> None: + from anonymizer.measurement.records.run import _detect_config_metadata + + detect = Detect() + metadata = _detect_config_metadata(detect) + assert metadata["entity_label_denylist"] is None + + def test_anonymizer_measurement_config_writes_jsonl(tmp_path: Path) -> None: input_csv = tmp_path / "input.csv" output_jsonl = tmp_path / "measurements.jsonl" From 29cb7f73c10e597317a3fd3ba9046b38d2a41324 Mon Sep 17 00:00:00 2001 From: memadi Date: Mon, 10 Aug 2026 18:05:50 -0700 Subject: [PATCH 11/30] nit-CI test pass Signed-off-by: memadi --- tests/engine/test_detection_config_serialization.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/engine/test_detection_config_serialization.py b/tests/engine/test_detection_config_serialization.py index 710f1247..976c7913 100644 --- a/tests/engine/test_detection_config_serialization.py +++ b/tests/engine/test_detection_config_serialization.py @@ -94,7 +94,9 @@ def test_detection_builder_round_trips_through_native_data_designer_config(tmp_p def _get_gliner_labels_from_builder(builder: DataDesignerConfigBuilder) -> list[str]: - serialized = json.loads(builder.get_builder_config().to_json()) + payload = builder.get_builder_config().to_json() + assert payload is not None + serialized = json.loads(payload) model_configs = serialized["data_designer"]["model_configs"] gliner = next(m for m in model_configs if m.get("alias") == "gliner-pii-detector") return gliner["inference_parameters"]["extra_body"]["labels"] From 1a6b5a61f1e5b3c9cbe070178858798856700a5f Mon Sep 17 00:00:00 2001 From: memadi Date: Tue, 11 Aug 2026 16:01:50 -0700 Subject: [PATCH 12/30] address greptile feedback Signed-off-by: memadi --- .../engine/detection/detection_workflow.py | 8 +- .../evaluation/entity_coverage_judge.py | 88 +++++++++++++------ tests/engine/test_detection_workflow.py | 25 ++++++ tests/engine/test_entity_coverage_judge.py | 36 ++++++-- 4 files changed, 119 insertions(+), 38 deletions(-) diff --git a/src/anonymizer/engine/detection/detection_workflow.py b/src/anonymizer/engine/detection/detection_workflow.py index 30a28be6..fc7b0218 100644 --- a/src/anonymizer/engine/detection/detection_workflow.py +++ b/src/anonymizer/engine/detection/detection_workflow.py @@ -475,7 +475,8 @@ def _resolve_detection_labels( ) -> list[str]: labels = list(DEFAULT_ENTITY_LABELS) if entity_labels is None else list(entity_labels) if entity_label_denylist: - labels = [label for label in labels if label not in entity_label_denylist] + denied = {label.casefold() for label in entity_label_denylist} + labels = [label for label in labels if label.casefold() not in denied] if not labels: logger.warning( "entity_label_denylist removed all labels from the effective detection set. No entities will be detected." @@ -491,11 +492,12 @@ def _materialize_final_entities( ) -> dict: """Build COL_FINAL_ENTITIES, optionally filtering to *allowed_labels* and excluding *entity_label_denylist*.""" parsed = EntitiesSchema.from_raw(raw) + allowed = {label.casefold() for label in allowed_labels} if allowed_labels is not None else None + denied = {label.casefold() for label in entity_label_denylist or []} kept = [ e for e in parsed.entities - if (allowed_labels is None or e.label in allowed_labels) - and (entity_label_denylist is None or e.label not in entity_label_denylist) + if (allowed is None or e.label.casefold() in allowed) and e.label.casefold() not in denied ] return EntitiesSchema(entities=kept).model_dump() diff --git a/src/anonymizer/engine/evaluation/entity_coverage_judge.py b/src/anonymizer/engine/evaluation/entity_coverage_judge.py index 0a1e3f28..292427fc 100644 --- a/src/anonymizer/engine/evaluation/entity_coverage_judge.py +++ b/src/anonymizer/engine/evaluation/entity_coverage_judge.py @@ -72,23 +72,33 @@ def _effective_entity_labels( entity_labels: list[str] | None, entity_label_denylist: list[str] | None, ) -> list[str] | None: - """Return the effective label set for prompt and filter scope. + """Return the effective allowlist for prompt and filter scope. - Subtracts denied labels from entity_labels (or the full default set when - entity_labels is None). Returns None only when entity_labels is None and - no denylist is set, preserving the "evaluate all PII types" prompt path. + ``None`` remains permissive so coverage includes novel labels introduced by + augmentation. Denied labels are applied independently by the prompt and + postprocessing filter. """ + if entity_labels is None: + return None if not entity_label_denylist: return entity_labels denied = {label.casefold() for label in entity_label_denylist} - base = entity_labels if entity_labels is not None else list(DEFAULT_ENTITY_LABELS) - effective = [label for label in base if label.casefold() not in denied] + effective = [label for label in entity_labels if label.casefold() not in denied] return effective -def _entity_type_scope_block(entity_labels: list[str] | None) -> str: +def _entity_type_scope_block( + entity_labels: list[str] | None, + entity_label_denylist: list[str] | None = None, +) -> str: if entity_labels is None: - return "\nEvaluate for all PII and sensitive entity types.\n" + denied = sorted({label.strip().casefold() for label in entity_label_denylist or [] if label.strip()}) + exclusion = ( + f"\nDo NOT report candidates with these explicitly excluded entity labels: {', '.join(denied)}." + if denied + else "" + ) + return f"\nEvaluate for all PII and sensitive entity types.{exclusion}\n" labels_str = ", ".join(entity_labels) return ( "\n" @@ -112,18 +122,39 @@ def _data_summary_block(data_summary: str | None) -> str: ) -def _coverage_prompt(*, entity_labels: list[str] | None, data_summary: str | None = None) -> str: - entity_scope_block = _entity_type_scope_block(entity_labels) +def _coverage_prompt( + *, + entity_labels: list[str] | None, + entity_label_denylist: list[str] | None = None, + data_summary: str | None = None, +) -> str: + entity_scope_block = _entity_type_scope_block(entity_labels, entity_label_denylist) data_context_section = f"\n\n{_data_summary_block(data_summary)}" if data_summary and data_summary.strip() else "" - entity_scope_guidance = ( - "- Respect the entity_type_scope: do not report candidate values outside the configured scope." - if entity_labels is not None - else "" - ) - active_labels = entity_labels if entity_labels is not None else DEFAULT_ENTITY_LABELS labels_str = ", ".join(active_labels) + if entity_labels is None: + taxonomy_guidance = ( + f"Common entity types include: {labels_str}. This list is not exhaustive. " + "Also report other direct or quasi-identifier types supported by the original-text context." + ) + label_interpretation = ( + "Use a concise snake_case label that describes each candidate's semantic entity category. " + "Do not use an explicitly excluded label." + ) + entity_scope_guidance = ( + "- Respect the entity_type_scope: report any supported sensitive entity type except explicit exclusions." + ) + else: + taxonomy_guidance = f"These entity types are in scope: {labels_str}." + label_interpretation = ( + "Treat each configured label as a semantic entity category. Labels may use compact, compound, " + "or abbreviated names; interpret their intended meaning from the label and the original-text " + "context. Return labels exactly as they appear in the entity_type_scope." + ) + entity_scope_guidance = ( + "- Respect the entity_type_scope: do not report candidate values outside the configured scope." + ) prompt = f"""You are a privacy-entity span extractor. Your task is defined below. @@ -139,7 +170,7 @@ def _coverage_prompt(*, entity_labels: list[str] | None, data_summary: str | Non -These entity types are in scope: {labels_str}. +{taxonomy_guidance} Quasi-identifiers: combinations of values that together re-identify someone \ (e.g. job title + employer + city appearing together). Time values (specific timestamps, \ times of day, schedules) can act as quasi-identifiers when combined with other attributes \ @@ -149,9 +180,7 @@ def _coverage_prompt(*, entity_labels: list[str] | None, data_summary: str | Non {entity_scope_block} -Treat each configured label as a semantic entity category. Labels may use compact, compound, \ -or abbreviated names; interpret their intended meaning from the label and the original-text \ -context. Return labels exactly as they appear in the entity_type_scope. +{label_interpretation} @@ -186,7 +215,7 @@ def _coverage_prompt(*, entity_labels: list[str] | None, data_summary: str | Non Do flag: - `reasoning` MUST be one sentence explaining which in-scope semantic type the value represents. -- A value that fills the role of a listed sensitive type in context, even when it is +- A value that fills the role of an in-scope sensitive type in context, even when it is short, a single token, an unfamiliar or foreign-looking word, or resembles an ordinary word or number. Decide by the value's role in the surrounding text, not by its length, rarity, or familiarity. (This still excludes pronouns and generic references that only @@ -363,12 +392,14 @@ def _normalize_literal_text(value: object) -> str: def _filter_out_of_scope_entities( entities: list[_CandidateT], entity_labels: list[str] | None, + entity_label_denylist: list[str] | None = None, ) -> list[_CandidateT]: """Drop entities with empty labels or labels outside the configured scope. When ``entity_labels`` is None all labels are in scope; only empty labels - are dropped. This mirrors the prompt's scope instruction deterministically - so a model that returns out-of-scope labels does not lower the coverage score. + and explicitly denied labels are dropped. This mirrors the prompt's scope + instruction deterministically so a model that returns out-of-scope labels + does not lower the coverage score. Label drift (e.g. the model returning ``"given_name"`` instead of ``"first_name"``) is unlikely in practice — the prompt explicitly instructs @@ -378,12 +409,14 @@ def _filter_out_of_scope_entities( meaningfully risking false negatives on well-formed responses. """ allowed = {label.casefold() for label in entity_labels} if entity_labels is not None else None + denied = {label.casefold() for label in entity_label_denylist or []} result = [] for entity in entities: label = str(entity.get("label", "")).strip() if not label: continue - if allowed is not None and label.casefold() not in allowed: + normalized_label = label.casefold() + if (allowed is not None and normalized_label not in allowed) or normalized_label in denied: continue result.append(entity) return result @@ -427,7 +460,7 @@ class EntityCoverageWorkflow(_BaseJudgeWorkflow): scope. Deterministic postprocessing removes nonliteral and already-covered findings. ``entity_labels`` scopes evaluation to a specific allowlist of labels (``None`` means - all default labels). ``entity_label_denylist`` further excludes specific labels from + all labels). ``entity_label_denylist`` further excludes specific labels from scope regardless of ``entity_labels``. Both are applied to the LLM prompt and the postprocess filter so denied labels are never penalised in the coverage score. @@ -496,6 +529,7 @@ def column_config(self, selected_models: EvaluateModelSelection) -> LLMStructure name=self.RAW_COL, prompt=_coverage_prompt( entity_labels=effective_labels, + entity_label_denylist=self._entity_label_denylist, data_summary=self._data_summary, ), model_alias=resolve_model_alias(self.MODEL_ROLE, selected_models), @@ -519,7 +553,9 @@ def postprocess(self, dataframe: pd.DataFrame) -> pd.DataFrame: n_candidates_list.append(None) else: candidates = _filter_out_of_scope_entities( - candidates, _effective_entity_labels(self._entity_labels, self._entity_label_denylist) + candidates, + _effective_entity_labels(self._entity_labels, self._entity_label_denylist), + self._entity_label_denylist, ) candidates = _filter_nonliteral_entities(candidates, out[COL_TEXT].loc[idx]) candidates = _deduplicate_candidate_values(candidates) diff --git a/tests/engine/test_detection_workflow.py b/tests/engine/test_detection_workflow.py index 9c636d65..556b6590 100644 --- a/tests/engine/test_detection_workflow.py +++ b/tests/engine/test_detection_workflow.py @@ -38,6 +38,7 @@ _get_augment_prompt, _get_latent_prompt, _get_validation_prompt, + _materialize_final_entities, _resolve_detection_labels, ) from anonymizer.engine.ndd.adapter import FailedRecord, WorkflowRunResult @@ -473,6 +474,11 @@ def test_resolve_detection_labels_denylist_removes_labels() -> None: assert "city" in labels +def test_resolve_detection_labels_denylist_is_case_insensitive() -> None: + labels = _resolve_detection_labels(["first_name", "Email"], entity_label_denylist={"EMAIL"}) + assert labels == ["first_name"] + + def test_resolve_detection_labels_denylist_on_defaults() -> None: labels = _resolve_detection_labels(None, entity_label_denylist={"ssn", "first_name"}) assert "ssn" not in labels @@ -492,6 +498,25 @@ def test_resolve_detection_labels_empty_result_warns(caplog: pytest.LogCaptureFi assert "No entities will be detected" in caplog.text +def test_materialize_final_entities_applies_label_filters_case_insensitively() -> None: + raw = { + "entities": [ + {"value": "Alice", "label": "First_Name", "start_position": 0, "end_position": 5}, + {"value": "alice@example.com", "label": "Email", "start_position": 7, "end_position": 24}, + {"value": "Houston", "label": "City", "start_position": 28, "end_position": 35}, + ] + } + + result = _materialize_final_entities( + raw, + allowed_labels={"first_name", "email"}, + entity_label_denylist={"EMAIL"}, + ) + + final = EntitiesSchema.from_raw(result) + assert [entity.label for entity in final.entities] == ["First_Name"] + + def test_denylist_filters_entities_from_final_entities( stub_detector_model_configs: list[ModelConfig], stub_detection_model_selection: DetectionModelSelection, diff --git a/tests/engine/test_entity_coverage_judge.py b/tests/engine/test_entity_coverage_judge.py index 2b134b6f..acee219e 100644 --- a/tests/engine/test_entity_coverage_judge.py +++ b/tests/engine/test_entity_coverage_judge.py @@ -459,12 +459,9 @@ def test_effective_entity_labels_subtracts_denylist_from_explicit_labels() -> No assert result == ["first_name", "city"] -def test_effective_entity_labels_subtracts_denylist_from_defaults() -> None: +def test_effective_entity_labels_preserves_permissive_scope_with_denylist() -> None: result = _effective_entity_labels(None, ["ssn", "first_name"]) - assert result is not None - assert "ssn" not in result - assert "first_name" not in result - assert "email" in result + assert result is None def test_effective_entity_labels_is_case_insensitive() -> None: @@ -480,18 +477,38 @@ def test_coverage_prompt_excludes_denied_labels_from_scope() -> None: assert "city" in prompt +def test_coverage_prompt_keeps_permissive_scope_and_names_denied_labels() -> None: + prompt = _coverage_prompt(entity_labels=None, entity_label_denylist=["email"]) + assert "Evaluate for all PII and sensitive entity types." in prompt + assert "explicitly excluded entity labels: email" in prompt + assert "This list is not exhaustive." in prompt + assert "other direct or quasi-identifier types" in prompt + assert "Use a concise snake_case label" in prompt + assert "Return labels exactly as they appear" not in prompt + + +def test_filter_out_of_scope_entities_keeps_novel_non_denied_labels() -> None: + entities = [ + {"value": "Example Clinic", "label": "clinic_name", "reasoning": "clinic"}, + {"value": "alice@example.com", "label": "Email", "reasoning": "email"}, + ] + result = _filter_out_of_scope_entities(entities, entity_labels=None, entity_label_denylist=["email"]) + assert result == [entities[0]] + + def test_entity_coverage_workflow_excludes_denied_labels_from_postprocess() -> None: - """Denied labels must be excluded from candidate entities in postprocess.""" + """Permissive postprocessing keeps novel labels while excluding denied labels.""" raw_judge_output = [ {"value": "Alice", "label": "first_name", "reasoning": "not replaced"}, + {"value": "Example Clinic", "label": "clinic_name", "reasoning": "not replaced"}, {"value": "alice@example.com", "label": "email", "reasoning": "not replaced"}, ] - entities_by_value = {"entities_by_value": [{"value": "Alice", "label": "first_name", "mentions": []}]} + entities_by_value = {"entities_by_value": [{"value": "Alice", "labels": ["first_name"]}]} input_df = pd.DataFrame( { - COL_TEXT: ["Alice alice@example.com"], + COL_TEXT: ["Alice visited Example Clinic and used alice@example.com"], COL_ENTITIES_BY_VALUE: [entities_by_value], - "_raw_entity_coverage_judge": [{"candidate_entities": raw_judge_output}], + COL_ENTITY_COVERAGE_JUDGE: [{"candidate_entities": raw_judge_output}], } ) @@ -503,4 +520,5 @@ def test_entity_coverage_workflow_excludes_denied_labels_from_postprocess() -> N result_df = workflow.postprocess(workflow.prepare(input_df)) missed = result_df[COL_MISSED_ENTITIES].iloc[0] missed_labels = {e["label"] for e in missed} + assert "clinic_name" in missed_labels assert "email" not in missed_labels From 007dc6d149db96405ca5df3b4dc7ae04b682db2c Mon Sep 17 00:00:00 2001 From: memadi Date: Tue, 11 Aug 2026 16:52:46 -0700 Subject: [PATCH 13/30] address greptile feedbacl Signed-off-by: memadi --- .../engine/detection/detection_workflow.py | 59 +++++++++++++++++- tests/engine/test_detection_workflow.py | 61 +++++++++++++++++++ 2 files changed, 118 insertions(+), 2 deletions(-) diff --git a/src/anonymizer/engine/detection/detection_workflow.py b/src/anonymizer/engine/detection/detection_workflow.py index fc7b0218..c695af96 100644 --- a/src/anonymizer/engine/detection/detection_workflow.py +++ b/src/anonymizer/engine/detection/detection_workflow.py @@ -347,6 +347,7 @@ def identify_latent_entities( prompt=_get_latent_prompt( data_summary=data_summary, privacy_goal=privacy_goal, + entity_label_denylist=entity_label_denylist, ), model_alias=latent_alias, output_format=LatentEntitiesSchema, @@ -355,7 +356,12 @@ def identify_latent_entities( workflow_name="latent-entity-detection", preview_num_records=preview_num_records, ) - return EntityDetectionResult(dataframe=latent_result.dataframe, failed_records=latent_result.failed_records) + latent_df = latent_result.dataframe.copy() + if COL_LATENT_ENTITIES in latent_df.columns: + latent_df[COL_LATENT_ENTITIES] = latent_df[COL_LATENT_ENTITIES].apply( + lambda raw: _filter_denied_latent_entities(raw, entity_label_denylist) + ) + return EntityDetectionResult(dataframe=latent_df, failed_records=latent_result.failed_records) def run( self, @@ -502,6 +508,40 @@ def _materialize_final_entities( return EntitiesSchema(entities=kept).model_dump() +def _filter_denied_latent_entities(raw: object, entity_label_denylist: list[str] | None) -> object: + """Remove denied latent labels while preserving the structured payload shape.""" + denied = {label.casefold() for label in entity_label_denylist or []} + if not denied: + return raw + + if isinstance(raw, LatentEntitiesSchema): + kept = [entity for entity in raw.latent_entities if entity.label.strip().casefold() not in denied] + return LatentEntitiesSchema(latent_entities=kept).model_dump(mode="json") + + if isinstance(raw, dict): + entities = raw.get("latent_entities") + if not isinstance(entities, list): + return raw + return { + **raw, + "latent_entities": [ + entity + for entity in entities + if not isinstance(entity, dict) or str(entity.get("label", "")).strip().casefold() not in denied + ], + } + + # Retain support for legacy list-shaped traces. + if isinstance(raw, list): + return [ + entity + for entity in raw + if not isinstance(entity, dict) or str(entity.get("label", "")).strip().casefold() not in denied + ] + + return raw + + def _build_entities_by_value(final_entities_raw: object) -> dict: """Derive COL_ENTITIES_BY_VALUE from COL_FINAL_ENTITIES.""" parsed = EntitiesSchema.from_raw(final_entities_raw) @@ -727,12 +767,26 @@ def _get_augment_prompt(*, data_summary: str | None, labels: list[str], strict_l ) -def _get_latent_prompt(*, data_summary: str | None, privacy_goal: PrivacyGoal | None) -> str: +def _get_latent_prompt( + *, + data_summary: str | None, + privacy_goal: PrivacyGoal | None, + entity_label_denylist: list[str] | None = None, +) -> str: summary_line = data_summary.strip() if data_summary else "Not provided" privacy_goal_text = _format_privacy_goal(privacy_goal) + denied_labels = sorted({label.strip().casefold() for label in entity_label_denylist or [] if label.strip()}) + denylist_block = ( + "\n\n" + f"Do NOT return latent entities with these labels: {', '.join(denied_labels)}.\n" + "\n" + if denied_labels + else "" + ) prompt = """You are performing: LATENT ENTITY & INFERENCE ANALYSIS for privacy protection. The text will be rewritten according to this privacy goal: <> +<> Goal: Identify sensitive information that is NOT explicitly stated in the text, \ but is reasonably inferable from context and could materially increase re-identification \ @@ -820,6 +874,7 @@ def _get_latent_prompt(*, data_summary: str | None, privacy_goal: PrivacyGoal | "<>": privacy_goal_text, "<>": summary_line, "<>": _jinja(COL_TAGGED_TEXT), + "<>": denylist_block, }, ) diff --git a/tests/engine/test_detection_workflow.py b/tests/engine/test_detection_workflow.py index 556b6590..380b6440 100644 --- a/tests/engine/test_detection_workflow.py +++ b/tests/engine/test_detection_workflow.py @@ -34,6 +34,7 @@ ) from anonymizer.engine.detection.detection_workflow import ( EntityDetectionWorkflow, + _filter_denied_latent_entities, _format_label_examples, _get_augment_prompt, _get_latent_prompt, @@ -159,6 +160,66 @@ def test_latent_prompt_includes_summary_and_goal() -> None: assert COL_TAGGED_TEXT in prompt +def test_latent_prompt_excludes_denied_labels() -> None: + prompt = _get_latent_prompt( + data_summary=None, + privacy_goal=None, + entity_label_denylist=["Health_Condition", "occupation"], + ) + assert "Do NOT return latent entities with these labels: health_condition, occupation." in prompt + + +def test_filter_denied_latent_entities_is_case_insensitive() -> None: + raw = { + "latent_entities": [ + {"label": "Health_Condition", "value": "diabetes"}, + {"label": "employer", "value": "Acme"}, + ] + } + result = _filter_denied_latent_entities(raw, ["health_condition"]) + assert result == {"latent_entities": [{"label": "employer", "value": "Acme"}]} + + +def test_identify_latent_entities_filters_denied_labels( + stub_detector_model_configs: list[ModelConfig], + stub_detection_model_selection: DetectionModelSelection, +) -> None: + adapter = Mock() + adapter.run_workflow.return_value = WorkflowRunResult( + dataframe=pd.DataFrame( + { + COL_TEXT: ["The patient works at Acme."], + COL_LATENT_ENTITIES: [ + { + "latent_entities": [ + {"label": "Health_Condition", "value": "diabetes"}, + {"label": "employer", "value": "Acme"}, + ] + } + ], + } + ), + failed_records=[], + ) + workflow = EntityDetectionWorkflow(adapter=adapter) + + result = workflow.identify_latent_entities( + pd.DataFrame({COL_TEXT: ["The patient works at Acme."]}), + model_configs=stub_detector_model_configs, + selected_models=stub_detection_model_selection, + gliner_detection_threshold=0.5, + entity_label_denylist=["health_condition"], + privacy_goal=PrivacyGoal( + protect="Protect inferred sensitive attributes.", + preserve="Preserve non-sensitive facts.", + ), + ) + + assert result.dataframe[COL_LATENT_ENTITIES].iloc[0] == { + "latent_entities": [{"label": "employer", "value": "Acme"}] + } + + def test_run_without_latent_detection_materializes_final_entities( stub_detector_model_configs: list[ModelConfig], stub_detection_model_selection: DetectionModelSelection, From a38770663f788b9a196dac4425cf184d11293639 Mon Sep 17 00:00:00 2001 From: Marjan Emadi Date: Tue, 11 Aug 2026 17:01:17 -0700 Subject: [PATCH 14/30] Update src/anonymizer/engine/detection/detection_workflow.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- src/anonymizer/engine/detection/detection_workflow.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/anonymizer/engine/detection/detection_workflow.py b/src/anonymizer/engine/detection/detection_workflow.py index c695af96..a3452697 100644 --- a/src/anonymizer/engine/detection/detection_workflow.py +++ b/src/anonymizer/engine/detection/detection_workflow.py @@ -503,7 +503,8 @@ def _materialize_final_entities( kept = [ e for e in parsed.entities - if (allowed is None or e.label.casefold() in allowed) and e.label.casefold() not in denied + if (allowed is None or e.label.strip().casefold() in allowed) + and e.label.strip().casefold() not in denied ] return EntitiesSchema(entities=kept).model_dump() From 53e69fd62a1194bf9d030a85a6910b9565a2fe68 Mon Sep 17 00:00:00 2001 From: memadi Date: Tue, 11 Aug 2026 17:10:16 -0700 Subject: [PATCH 15/30] nit Signed-off-by: memadi --- src/anonymizer/engine/detection/detection_workflow.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/anonymizer/engine/detection/detection_workflow.py b/src/anonymizer/engine/detection/detection_workflow.py index a3452697..a92253a7 100644 --- a/src/anonymizer/engine/detection/detection_workflow.py +++ b/src/anonymizer/engine/detection/detection_workflow.py @@ -503,8 +503,7 @@ def _materialize_final_entities( kept = [ e for e in parsed.entities - if (allowed is None or e.label.strip().casefold() in allowed) - and e.label.strip().casefold() not in denied + if (allowed is None or e.label.strip().casefold() in allowed) and e.label.strip().casefold() not in denied ] return EntitiesSchema(entities=kept).model_dump() From af0b5078daa8458e2a8a08651cd26985111d683e Mon Sep 17 00:00:00 2001 From: nvskills-svc-account Date: Wed, 12 Aug 2026 21:57:51 +0000 Subject: [PATCH 16/30] Attach NVSkills validation signatures Signed-off-by: nvskills-svc-account --- skills/anonymizer/BENCHMARK.md | 134 +++++++++++++----------- skills/anonymizer/skill-card.md | 180 ++++++++++++-------------------- skills/anonymizer/skill.oms.sig | 1 + 3 files changed, 139 insertions(+), 176 deletions(-) create mode 100644 skills/anonymizer/skill.oms.sig diff --git a/skills/anonymizer/BENCHMARK.md b/skills/anonymizer/BENCHMARK.md index 027917f8..f592adb7 100644 --- a/skills/anonymizer/BENCHMARK.md +++ b/skills/anonymizer/BENCHMARK.md @@ -1,85 +1,99 @@ - - +# Skill Benchmark: anonymizer -# Evaluation Report +> ✅ **Overall verdict: PASS — Recommended for publication** -Evaluation report for the `anonymizer` skill before publication through -NVSkills-Eval. +## Publication Recommendation -This benchmark file records the publication-ready evaluation plan and task -composition for NeMo Anonymizer. The external NVSkills-Eval run has not been -executed in this local workspace, so this branch intentionally reports no -Anonymizer scores. +Recommended for publication based on the completed evaluation evidence in this report. -## Evaluation Summary +## Evaluation Metadata - Skill: `anonymizer` -- Evaluation date: pending external `/nvskills-ci` run -- NVSkills-Eval profile: external -- Environment: external NVSkills-Eval runner -- Dataset: 6 evaluation tasks -- Attempts per task: recorded by external NVSkills-Eval after execution -- Pass threshold: recorded by external NVSkills-Eval after execution -- Overall verdict: pending external NVSkills-Eval run +- Evaluation date: 2026-08-12 +- Evaluator version: `1.2.4` +- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`), Codex (`openai/openai/gpt-5.5`) +- Tasks: 6 evaluation tasks (4 positive, 2 negative) +- Dataset digest: `sha256:c2c13b2d794c6117dac0402f1261bd2d80972085c5e716426bedff6b3d59b8ae` (skill-evaluator-dataset-snapshot/1) +- Attempts per task: 1 +- Environment: `k8s-sandbox` +- Tier 3 evidence: required for publication -## Agents Used +Each task attempt ran in its own isolated sandbox pod. -Agent-level measured results are pending the external NVSkills-Eval run. +## What This Report Answers -## Metrics Used +The three-tier evaluation checks whether the skill: -Reported benchmark dimensions: +- is safe to use; +- produces correct answers; +- is discovered and activated when needed; +- helps the agent complete the user's goal and expected workflow; and +- avoids wasted skill and tool usage. -- Security: checks whether skill-assisted execution avoids unsafe behavior such - as secret leakage, destructive commands, or unauthorized access. -- Correctness: checks whether the agent follows the expected workflow and - produces the correct final output. -- Discoverability: checks whether the agent loads the skill when relevant and - avoids using it when irrelevant. -- Effectiveness: checks whether the agent performs measurably better with the - skill than without it. -- Efficiency: checks whether the agent uses fewer tokens and avoids redundant - work. +## Results at a Glance -Underlying evaluation signals will be recorded from the external -NVSkills-Eval output after execution. +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 58% → 94% (+36 points) | 68% → 93% (+25 points) | +| Security | 100% → 92% (-8 points) | 83% → 83% (±0 points) | +| Correctness | 50% → 100% (+50 points) | 83% → 97% (+13 points) | +| Discoverability | 50% → 99% (+49 points) | 67% → 94% (+27 points) | +| Effectiveness | 52% → 89% (+38 points) | 65% → 93% (+28 points) | +| Efficiency | 38% → 89% (+51 points) | 42% → 97% (+55 points) | -## Test Tasks +**How to read this table:** baseline is the same task attempted without the target skill. Uplift is `skill score - baseline score`, shown in percentage points. -The benchmark dataset contains 6 evaluation tasks: +Example: `47% → 92% (+45 points)` means the skill-assisted run scored 92%, 45 percentage points above its 47% no-skill baseline. -- Positive tasks: 4 tasks where the skill is expected to activate. -- Negative tasks: 2 tasks where no skill is expected. -- Unlabeled tasks: 0 tasks where positive/negative intent cannot be inferred. +## Tier Status -Entries with `should_trigger: true` and `expected_skill: "anonymizer"` are -positive skill-activation cases. Entries with `should_trigger: false` and -`expected_skill: null` are negative activation cases. +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 1 validator(s); 2 finding(s) | +| Tier 2 | Semantic deduplication | **NOT RUN** | No result was recorded | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 6 task(s) | -## Results +## Findings and Observations -External NVSkills-Eval execution is pending. No copied or locally inferred -Anonymizer results are reported here. +
+Show detailed findings and successful checks -| Dimension | Tasks | Result | -|---|---:|---| -| Security | 6 | Pending external NVSkills-Eval run | -| Correctness | 6 | Pending external NVSkills-Eval run | -| Discoverability | 6 | Pending external NVSkills-Eval run | -| Effectiveness | 6 | Pending external NVSkills-Eval run | -| Efficiency | 6 | Pending external NVSkills-Eval run | +- **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Instructions' (`skills/anonymizer/SKILL.md`) +- **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Examples' (`skills/anonymizer/SKILL.md`) -## Tier 1: Static Validation Summary +
-Local static validation is covered by this branch's validation evidence. The -external NVSkills-Eval Tier 1 result is pending the `/nvskills-ci` run. +## Scoring Methodology -## Tier 2: Deduplication Summary +
+Show dimension definitions, source signals, and thresholds -External NVSkills-Eval deduplication results are pending. +| Dimension | Question | Scored signals | +|---|---|---| +| Security | Is it safe to use? | `security` (100%) | +| Correctness | Is the answer correct? | `accuracy` (100%) | +| Discoverability | Was the right skill loaded when needed? | `skill_execution` (100%) | +| Effectiveness | Did the skill help complete the task? | `goal_accuracy` (50%) + `behavior_check` (50%) | +| Efficiency | Did it avoid wasted tool or skill usage? | `skill_efficiency` (100%) | -## Publication Recommendation +- Dimension bands: PASS at 50% or above; NEUTRAL from 40% to below 50%; FAIL below 40%. +- Overall Tier 3 lift: PASS at +5 points or more; FAIL at -10 points or less; values between those bands are NEUTRAL. +- Overall verdict: PASS only when every configured dimension passes for at least one supported agent. Lift is reported as diagnostic evidence and does not override this gate. +- The 50% attempt pass threshold is a separate per-task gate; it is not the dimension pass threshold. +- Effectiveness is the equal-weight mean of goal completion (`goal_accuracy`) and expected workflow adherence (`behavior_check`). +- Token efficiency is a separate report-only signal. It does not change a dimension score or the overall verdict. + +Signals present in this run: + +- `security` (Security): unsafe operations, secret leakage, and unauthorized access. +- `skill_execution` (Skill Execution): whether the expected skill was found and executed. +- `skill_efficiency` (Efficiency): routing quality, workspace-aware skill reads, and productive tool use. +- `accuracy` (Accuracy): final-answer correctness against the reference answer. +- `goal_accuracy` (Goal Accuracy): whether the user's goal was achieved. +- `behavior_check` (Behavior Check): whether the expected workflow behavior was followed. + +
+ +## Freshness -Proceed to external NVSkills-Eval and signing. Publication should depend on the -external evaluation and signing results rather than this local preparation -branch alone. +Regenerate this benchmark when the skill, evaluation dataset, target agent/model, evaluator version, environment, or scoring policy changes. diff --git a/skills/anonymizer/skill-card.md b/skills/anonymizer/skill-card.md index 57b5aa7c..f1b2ddaa 100644 --- a/skills/anonymizer/skill-card.md +++ b/skills/anonymizer/skill-card.md @@ -1,139 +1,87 @@ - - +## Description:
+Use when the user wants to anonymize a text dataset, redact PII, de-identify free-text data, or rewrite text to remove sensitive or inferable identifying information. Produces a runnable Python script that calls the NeMo Anonymizer pipeline (detection → replace or rewrite).
-## Description - -Use NeMo Anonymizer through an interactive agent workflow: inspect text data, -choose Replace or Rewrite, select a replacement strategy, draft a runnable -Python script, preview before full execution, diagnose failed records first, and -configure self-hosted GLiNER when detection must stay local. - -This skill package is prepared for NVSkills publication review. External -NVSkills-Eval results are pending and no Anonymizer scores are reported in this -branch. +This skill is ready for commercial/non-commercial use.
## Owner +NVIDIA
-NVIDIA - -### License/Terms of Use - -Apache 2.0 - -## Use Case - -Developers, privacy engineers, and data practitioners using NeMo Anonymizer to -detect, replace, redact, hash, annotate, or rewrite sensitive entities in text -datasets while keeping a durable script for review and reruns. - -### Deployment Geography for Use - -Global - -## Known Risks and Mitigations - -Risk: Users may overinterpret anonymized output as a privacy guarantee. - -Mitigation: The skill instructs agents to describe Anonymizer as best-effort, -preview before full execution, inspect failed records, and call out human review -for rewrite outputs that need it. - -Risk: Agent-generated scripts may target the wrong source file, text column, or -model-provider configuration. - -Mitigation: The workflow requires data inspection, explicit user confirmation -of mode and key configuration choices, and preview execution before a full run. - -Risk: An incorrect provider or model alias may send detection requests to an -unintended endpoint. - -Mitigation: The skill directs agents to configure the local GLiNER provider -explicitly, keep the full model pool, verify the endpoint, preview, and consult -the self-hosting documentation. - -## Reference(s) - -- [Interactive workflow](references/interactive.md) -- [Choosing a Strategy](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/choosing-a-strategy/) -- [Detection](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/detection/) -- [Evaluation](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/evaluation/) -- [Models](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/models/) -- [Self-hosting GLiNER](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/self-hosting-gliner/) -- [Troubleshooting](https://nvidia-nemo.github.io/Anonymizer/dev/troubleshooting/) - -## Skill Output - -**Output Type(s):** Python scripts, shell commands, configuration guidance, -diagnostic guidance +### License/Terms of Use:
+Apache 2.0
+## Use Case:
+Developers and engineers who need to anonymize text datasets, redact PII, de-identify free-text data, or rewrite text to remove sensitive or inferable personal information for privacy compliance.
-**Output Format:** A runnable Python script plus concise Markdown guidance for -previewing, diagnosing failures, and running the full pipeline +### Deployment Geography for Use:
+Global
-**Output Parameters:** Dataset path, text column, data summary, mode -(`Replace` or `Rewrite`), replacement strategy when applicable, privacy goal, -risk tolerance, entity labels, and optional model-provider paths +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Yes]
+**Credential Type(s):** [API key]
-**Other Properties Related to Output:** The generated script previews by -default, exits on failed records, optionally evaluates output with -LLM-as-judge, and leaves full dataset execution under explicit user control. +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate.
-## Evaluation Agents Used +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
-The external NVSkills-Eval run is pending. Agent-level measured results will be -reported from the external `/nvskills-ci` evaluation output after it runs. +## Reference(s):
+- [NeMo Anonymizer Documentation](https://nvidia-nemo.github.io/Anonymizer/)
+- [Choosing a Strategy](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/choosing-a-strategy/)
+- [Detection Concepts](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/detection/)
+- [Evaluation Concepts](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/evaluation/)
+- [Self-hosting GLiNER](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/self-hosting-gliner/)
+- [Troubleshooting](https://nvidia-nemo.github.io/Anonymizer/dev/troubleshooting/)
+- [GitHub Repository](https://github.com/NVIDIA-NeMo/Anonymizer.git)
-## Evaluation Tasks -The prepared evaluation dataset contains 6 NVSkills-Eval tasks: 4 positive -activation cases and 2 negative activation cases. The positive tasks cover mode -choice, stable cross-record replacement with `Hash`, failed-record-first -diagnosis, and self-hosted GLiNER. The negative tasks cover a general privacy -explainer and repository source development. +## Skill Output:
+**Output Type(s):** [Code]
+**Output Format:** [Python script]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [None]
-## Evaluation Metrics Used +## Evaluation Agents Used:
+- Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`)
+- Codex (`openai/openai/gpt-5.5`)
-Metrics will be reported by the external NVSkills-Eval run. Expected benchmark -dimensions are: -- Security: Checks whether skill-assisted execution avoids unsafe behavior such - as secret leakage, destructive commands, or unauthorized access. -- Correctness: Checks whether the agent follows the expected workflow and - produces the correct final output. -- Discoverability: Checks whether the agent loads the skill when relevant and - avoids using it when irrelevant. -- Effectiveness: Checks whether the agent performs measurably better with the - skill than without it. -- Efficiency: Checks whether the agent uses fewer tokens and avoids redundant - work. -## Evaluation Results +## Evaluation Tasks:
+6 evaluation tasks (4 positive, 2 negative) run in isolated sandbox pods with 1 attempt per task.
-External NVSkills-Eval execution is pending. This publication branch does not -include local or copied Anonymizer benchmark scores. +## Evaluation Metrics Used:
+Reported benchmark dimensions:
+- Security: Checks for unsafe operations, secret leakage, and unauthorized access.
+- Correctness: Checks final-answer correctness against the reference answer.
+- Discoverability: Checks whether the expected skill was found and executed when needed.
+- Effectiveness: Checks goal completion (50%) and expected workflow adherence (50%).
+- Efficiency: Checks routing quality, workspace-aware skill reads, and productive tool use.
-| Dimension | Tasks | Result | -|---|---:|---| -| Security | 6 | Pending external NVSkills-Eval run | -| Correctness | 6 | Pending external NVSkills-Eval run | -| Discoverability | 6 | Pending external NVSkills-Eval run | -| Effectiveness | 6 | Pending external NVSkills-Eval run | -| Efficiency | 6 | Pending external NVSkills-Eval run | +Underlying evaluation signals used in this run:
+- `security`: Unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Whether the expected skill was found and executed.
+- `skill_efficiency`: Routing quality, workspace-aware skill reads, and productive tool use.
+- `accuracy`: Final-answer correctness against the reference answer.
+- `goal_accuracy`: Whether the user's goal was achieved.
+- `behavior_check`: Whether the expected workflow behavior was followed.
-## Skill Version(s) -Publication candidate from this repository branch. The released skill version -should be recorded after review, external evaluation, and signing. -## Ethical Considerations +## Evaluation Results:
+| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 58% → 94% (+36 points) | 68% → 93% (+25 points) | +| Security | 100% → 92% (-8 points) | 83% → 83% (±0 points) | +| Correctness | 50% → 100% (+50 points) | 83% → 97% (+13 points) | +| Discoverability | 50% → 99% (+49 points) | 67% → 94% (+27 points) | +| Effectiveness | 52% → 89% (+38 points) | 65% → 93% (+28 points) | +| Efficiency | 38% → 89% (+51 points) | 42% → 97% (+55 points) | -NVIDIA believes Trustworthy AI is a shared responsibility and has established -policies and practices to enable development for a wide array of AI -applications. When downloaded or used in accordance with our terms of service, -developers should work with their internal team to ensure this skill meets -requirements for the relevant industry and use case and addresses foreseeable -product misuse. +## Skill Version(s):
+e3b99da (source: git SHA, committed 2026-08-11)
-(For Release on NVIDIA Platforms Only) +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
-Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns -[here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail). +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/skills/anonymizer/skill.oms.sig b/skills/anonymizer/skill.oms.sig new file mode 100644 index 00000000..76a40d0f --- /dev/null +++ b/skills/anonymizer/skill.oms.sig @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiYW5vbnltaXplciIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICI2OWFmZTdiMzlkYjdmZmYyNzEzZTJiZDgwZmI0MTZkYzcwOTY1NjMzYWU3ZDRiMGE0YWQzZmJkNjA2MDc3YjA4IgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdCIsCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXRpZ25vcmUiCiAgICAgIF0sCiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiCiAgICB9LAogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICI5ZjJmMDU4ZGQ3M2U3ZmRiNWNiZjkyN2M1NDcyZjJhNGE5NTE5NWU1ZWU5OTZjZjBjYTUxNTM5YTdmMDY4YzdjIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIiwKICAgICAgICAiZGlnZXN0IjogImFiNThiYzY4YjQzZTZkMTAzNzk5OGQxMDNkNzlhYTVkM2VmNjUyMWRhY2FiOWYyMzY3YzQ1YWIxMDBiZTIwODQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIsCiAgICAgICAgImRpZ2VzdCI6ICJjYTQ3YjgyZGMzMTJmYzY0MDc3MGJmNjczM2JhNDYyNGRjYzhmNzA4MDdlZDM2ZjczZTllZTUwYTFkNDdlMGMyIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvaW50ZXJhY3RpdmUubWQiLAogICAgICAgICJkaWdlc3QiOiAiZDc1NWFhNDU3NDA3ZTM5MDE1YzcxMWEwY2I4MDI4MmRlZTkyNzZhYWJiYzRhMTYzM2YxZDUzZDExMGUzM2YwOCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiZGlnZXN0IjogImYwMDNlZTUyODcwZmJhZTg3NjVmYWIxN2E5ODc4MmEzNDIyYjY0ODkzNmY0NWZmM2JmOTUwNzUwYWZiYjkwOGMiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9CiAgICBdCiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQDvD8g+PIjUDUKUdto3XmHbHJx3SJ/yC8BJRlglJMKI9yMrgUSeWNWkjVj5RF6j1i8CMQDvIThge9dCTqmL2Z7U406/AfDeWclej4gysAh/UKPkwmFCZV/RaWRWTIU+TGbOHgI=","keyid":""}]}} \ No newline at end of file From e27bf5788b6265d7b9d30e35d858e7a4f3dcb4eb Mon Sep 17 00:00:00 2001 From: memadi Date: Wed, 19 Aug 2026 10:29:07 -0700 Subject: [PATCH 17/30] address feedback Signed-off-by: memadi --- docs/concepts/choosing-a-strategy.md | 14 +-- docs/concepts/detection.md | 14 +-- docs/concepts/evaluation.md | 2 +- docs/troubleshooting.md | 4 +- skills/anonymizer/SKILL.md | 2 +- src/anonymizer/config/anonymizer_config.py | 20 ++--- .../engine/detection/custom_columns.py | 7 +- .../engine/detection/detection_workflow.py | 89 ++++++++++--------- .../engine/detection/postprocess.py | 6 +- .../evaluation/entity_coverage_judge.py | 48 +++++----- .../engine/replace/replace_runner.py | 6 +- .../workflow_columns/detection/config.py | 1 + .../engine/workflow_columns/detection/impl.py | 5 ++ src/anonymizer/interface/anonymizer.py | 20 ++--- src/anonymizer/interface/results.py | 12 +-- src/anonymizer/measurement/records/run.py | 4 +- tests/config/test_anonymizer_config.py | 52 +++++------ .../test_detection_config_serialization.py | 15 +++- tests/engine/test_detection_custom_columns.py | 33 +++++++ tests/engine/test_detection_postprocess.py | 22 +++++ tests/engine/test_detection_workflow.py | 56 ++++++------ tests/engine/test_entity_coverage_judge.py | 26 +++--- tests/engine/test_replace_runner.py | 12 +-- tests/interface/test_anonymizer_interface.py | 28 +++++- tests/test_measurement.py | 12 +-- 25 files changed, 309 insertions(+), 201 deletions(-) diff --git a/docs/concepts/choosing-a-strategy.md b/docs/concepts/choosing-a-strategy.md index 835704d3..2b34814d 100644 --- a/docs/concepts/choosing-a-strategy.md +++ b/docs/concepts/choosing-a-strategy.md @@ -43,12 +43,12 @@ What to include: - The domain (clinical, legal, financial, customer support, etc.) - The genre (notes, transcripts, opinions, biographies) - Anything about the source the engine couldn't infer from a single record (e.g. "transcribed phone calls — expect disfluencies") -- `data_summary` is a soft way to guide the augmenter when `entity_labels=None` — the augmenter is free to invent labels beyond `DEFAULT_ENTITY_LABELS`, so use it to tell the LLM what *not* to tag (e.g. "do not tag generic anatomical terms, medication class names, or job titles as PII"). For a hard exclusion of specific label types, use `Detect.entity_label_denylist` instead. +- `data_summary` is a soft way to guide the augmenter when `entity_labels=None` — the augmenter is free to invent labels beyond `DEFAULT_ENTITY_LABELS`, so use it to tell the LLM what *not* to tag (e.g. "do not tag generic anatomical terms, medication class names, or job titles as PII"). For a hard exclusion of specific label types, use `Detect.excluded_entity_labels` instead. What to leave out: - Lists of entity types **you want detected** (those go in `Detect.entity_labels`) -- Lists of entity types **you never want detected** (those go in `Detect.entity_label_denylist`) +- Lists of entity types **you never want detected** (those go in `Detect.excluded_entity_labels`) - Privacy/utility goals (those go in `Rewrite.privacy_goal`) - Substitute behavior instructions (e.g. "names should remain Portuguese", "preserve numeric magnitude") — those go in `Substitute(instructions)` - Generic phrasing ("text data" adds no signal) @@ -79,16 +79,16 @@ from anonymizer import DEFAULT_ENTITY_LABELS, Detect detect = Detect(entity_labels=[*DEFAULT_ENTITY_LABELS, "clinical_facility", "diagnosis_code", "medication_name"]) ``` -### `entity_label_denylist` +### `excluded_entity_labels` -Use when you want to **exclude** specific label types from detection without enumerating the entire allowlist. Denied labels are removed before GLiNER runs, so they are never detected, augmented, or surfaced in results. The evaluation judges also ignore denied label types so they don't lower your coverage score. +Use when you want to **exclude** specific label types from detection without enumerating the entire allowlist. Excluded labels are removed before GLiNER runs, so they are never detected, augmented, or surfaced in results. The evaluation judges also ignore excluded label types so they don't lower your coverage score. ```python # Never detect occupation or gender, keep everything else -Detect(entity_label_denylist=["occupation", "gender"]) +Detect(excluded_entity_labels=["occupation", "gender"]) -# Combine with an explicit allowlist — denylist always wins -Detect(entity_labels=["first_name", "email", "city"], entity_label_denylist=["city"]) +# Combine with an explicit allowlist — exclusions always win +Detect(entity_labels=["first_name", "email", "city"], excluded_entity_labels=["city"]) ``` ### `gliner_threshold` diff --git a/docs/concepts/detection.md b/docs/concepts/detection.md index 949bbce9..bd1e2e02 100644 --- a/docs/concepts/detection.md +++ b/docs/concepts/detection.md @@ -44,7 +44,7 @@ config = AnonymizerConfig( | Field | Default | Description | |-------|---------|-------------| | `entity_labels` | `None` (all defaults) | List of labels to detect. Leave unset (or pass `None`) to use the full default set. | -| `entity_label_denylist` | `None` | List of labels to **never** detect, even if present in `entity_labels` or the default set. Denied labels are excluded before GLiNER and the LLM prompts run, and are also filtered from the final entity output as a safety net. | +| `excluded_entity_labels` | `None` | List of labels to **never** detect, even if present in `entity_labels` or the default set. Excluded labels are removed before GLiNER and the LLM prompts run, and are also filtered from the final entity output as a safety net. | | `gliner_threshold` | `0.3` | GLiNER confidence threshold (0.0--1.0). Lower values detect more entities but may increase false positives. | | `validation_max_entities_per_call` | `100` | Maximum candidate entities per validator LLM call. Rows with more candidates are split into chunks. See [Chunked validation](#chunked-validation). | | `validation_excerpt_window_chars` | `500` | Characters of context included before and after a chunk's entity spans in the validator prompt. Bounds per-chunk prompt size; not the model's context-window limit. | @@ -106,20 +106,20 @@ Detect(entity_labels=["first_name", "last_name", "email"]) Detect() # entity_labels=None ``` -### Excluding labels with a deny list +### Excluding entity labels -Use `entity_label_denylist` to exclude specific labels from detection without having to enumerate the entire allowlist. Denied labels are removed before GLiNER runs and before the LLM prompts are built, so they are never detected or augmented. +Use `excluded_entity_labels` to omit specific labels from detection without having to enumerate the entire allowlist. Excluded labels are removed before GLiNER runs and before the LLM prompts are built, so they are never detected or augmented. ```python # Detect all defaults except occupation and gender -Detect(entity_label_denylist=["occupation", "gender"]) +Detect(excluded_entity_labels=["occupation", "gender"]) -# Combine with an explicit allowlist — denylist always wins -Detect(entity_labels=["first_name", "email", "city"], entity_label_denylist=["city"]) +# Combine with an explicit allowlist — exclusions always win +Detect(entity_labels=["first_name", "email", "city"], excluded_entity_labels=["city"]) ``` !!! warning - If every label in `entity_labels` is also in `entity_label_denylist`, the effective detection set is empty and no entities will be detected. Anonymizer logs a warning when this happens. + If every label in `entity_labels` is also in `excluded_entity_labels`, the effective detection set is empty and no entities will be detected. Anonymizer logs a warning when this happens. ## Tuning the threshold For `gliner_threshold`, start with the default `0.3`. If you're seeing too many false positives, raise it to `0.5`. If entities are being missed, try lowering to `0.2`. The LLM validation step catches many false positives, so erring on the side of lower thresholds is usually safe. diff --git a/docs/concepts/evaluation.md b/docs/concepts/evaluation.md index 68c83435..57394912 100644 --- a/docs/concepts/evaluation.md +++ b/docs/concepts/evaluation.md @@ -66,7 +66,7 @@ Note: the judge measures detection recall, not output leakage. A value detected The judge is scoped and contextualized by the same signals used during anonymization: - **`entity_labels`** — the detection taxonomy in scope; the judge only reports values whose type falls within it. -- **`entity_label_denylist`** — labels explicitly excluded from detection; the judge ignores entities of these types so denied labels are never penalised in the coverage score. +- **`excluded_entity_labels`** — labels explicitly excluded from detection; the judge ignores entities of these types so excluded labels are never penalised in the coverage score. - **`data_summary`** — used purely to interpret literal values and their semantic types, never to invent entities absent from the text. | Output column | Type | Description | diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index b98467ab..5c1ce353 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -116,9 +116,9 @@ Verify by re-running `preview` with `Annotate` and confirming the entity now app Symptoms: detected entities include obvious common words, dates that aren't dates, etc. -1. **Use `Detect.entity_label_denylist`** if a whole label type is systematically noisy for your data (e.g. `occupation` tagging generic job words, `age` tagging durations). This is the cleanest fix — denied labels are excluded before GLiNER runs and never appear in results. +1. **Use `Detect.excluded_entity_labels`** if a whole label type is systematically noisy for your data (e.g. `occupation` tagging generic job words, `age` tagging durations). This is the cleanest fix — excluded labels are removed before GLiNER runs and never appear in results. ```python - Detect(entity_label_denylist=["occupation", "age"]) + Detect(excluded_entity_labels=["occupation", "age"]) ``` 2. **Raise `gliner_threshold`** to `0.5`. The augmenter will pick up real misses, so this rarely costs recall. 3. **Lower `validation_excerpt_window_chars`** (default `500`) if context-driven validation is being misled by far-away sentences. Smaller per-chunk prompts trade context for precision. diff --git a/skills/anonymizer/SKILL.md b/skills/anonymizer/SKILL.md index bb7b332c..f208148b 100644 --- a/skills/anonymizer/SKILL.md +++ b/skills/anonymizer/SKILL.md @@ -43,7 +43,7 @@ regulatory and business context. # Usage Tips and Common Pitfalls - **`Detect.entity_labels=None` (the default) is permissive** — the augmenter LLM may invent labels not in `DEFAULT_ENTITY_LABELS`. Setting an explicit list switches to **strict mode** where *only* the listed labels are detected. To add domain labels, *extend* the default, don't replace it: `entity_labels=[*DEFAULT_ENTITY_LABELS, "clinical_facility", ...]` (`DEFAULT_ENTITY_LABELS` is a tuple, so unpack it into a list). Match the snake_case convention of `DEFAULT_ENTITY_LABELS`. -- **`Detect.entity_label_denylist`** excludes specific label types from detection entirely — denied labels are removed before GLiNER runs and are never detected, augmented, or penalised in evaluation scores. Use it when a label type is systematically noisy for your data or should never be anonymized (e.g. `Detect(entity_label_denylist=["occupation", "gender"])`). The denylist takes precedence over `entity_labels` — a label in both is never detected. +- **`Detect.excluded_entity_labels`** excludes specific label types from detection entirely — excluded labels are removed before GLiNER runs and are never detected, augmented, or penalised in evaluation scores. Use it when a label type is systematically noisy for your data or should never be anonymized (e.g. `Detect(excluded_entity_labels=["occupation", "gender"])`). Exclusions take precedence over `entity_labels` — a label in both is never detected. - **GLiNER is zero-shot** — entity labels are natural-language concept names (e.g. `"clinical_facility"`, `"internal_project_codename"`), not codes or enum values. Any concept you can name in English is a label GLiNER can detect. - **`Rewrite.instructions` is a dead field today** — it exists on the model but the rewrite engine never reads it. Do not use it. Put rewriter guidance in `privacy_goal.protect` / `privacy_goal.preserve` instead. - **`risk_tolerance` only applies to Rewrite mode**, not Replace. diff --git a/src/anonymizer/config/anonymizer_config.py b/src/anonymizer/config/anonymizer_config.py index 77e3b87e..bf8be0f3 100644 --- a/src/anonymizer/config/anonymizer_config.py +++ b/src/anonymizer/config/anonymizer_config.py @@ -79,11 +79,11 @@ class Detect(BaseModel): "To inspect the default set, use `from anonymizer import DEFAULT_ENTITY_LABELS`." ), ) - entity_label_denylist: list[str] | None = Field( + excluded_entity_labels: list[str] | None = Field( default=None, description=( "Entity labels to never detect, even if present in entity_labels or the default set. " - "Denied labels are excluded before GLiNER and LLM prompts run, and are also filtered " + "Excluded labels are removed before GLiNER and LLM prompts run, and are also filtered " "from the final entity output as a safety net." ), ) @@ -122,26 +122,26 @@ def validate_entity_labels(cls, value: list[str] | None) -> list[str] | None: logger.warning("entity_labels contained duplicates, removed automatically.") return deduped - @field_validator("entity_label_denylist") + @field_validator("excluded_entity_labels") @classmethod - def validate_entity_label_denylist(cls, value: list[str] | None) -> list[str] | None: + def validate_excluded_entity_labels(cls, value: list[str] | None) -> list[str] | None: if value is None: return value cleaned = [label.strip().lower() for label in value if label.strip()] if not cleaned: - raise ValueError("entity_label_denylist must not be empty. Use None to disable the deny list.") + raise ValueError("excluded_entity_labels must not be empty. Use None to disable exclusions.") deduped = sorted(set(cleaned)) if len(deduped) != len(cleaned): - logger.warning("entity_label_denylist contained duplicates, removed automatically.") + logger.warning("excluded_entity_labels contained duplicates, removed automatically.") return deduped @model_validator(mode="after") - def warn_on_allowlist_denylist_overlap(self) -> "Detect": - if self.entity_labels is not None and self.entity_label_denylist is not None: - overlap = sorted(set(self.entity_labels) & set(self.entity_label_denylist)) + def warn_on_entity_label_overlap(self) -> "Detect": + if self.entity_labels is not None and self.excluded_entity_labels is not None: + overlap = sorted(set(self.entity_labels) & set(self.excluded_entity_labels)) if overlap: logger.warning( - "entity_labels and entity_label_denylist share labels that will never be detected: %s", + "entity_labels and excluded_entity_labels share labels that will never be detected: %s", overlap, ) return self diff --git a/src/anonymizer/engine/detection/custom_columns.py b/src/anonymizer/engine/detection/custom_columns.py index 059d82ac..7127356e 100644 --- a/src/anonymizer/engine/detection/custom_columns.py +++ b/src/anonymizer/engine/detection/custom_columns.py @@ -75,7 +75,11 @@ def parse_detected_entities(row: dict[str, Any]) -> dict[str, Any]: required_columns=[COL_TEXT, COL_VALIDATED_SEED_ENTITIES, COL_AUGMENTED_ENTITIES], side_effect_columns=[COL_MERGED_TAGGED_TEXT, COL_VALIDATION_CANDIDATES], ) -def merge_and_build_candidates(row: dict[str, Any]) -> dict[str, Any]: +def merge_and_build_candidates( + row: dict[str, Any], + *, + excluded_entity_labels: list[str] | None = None, +) -> dict[str, Any]: """Merge validated seed + augmented entities, then build tagged text and validation candidates. Contract: @@ -88,6 +92,7 @@ def merge_and_build_candidates(row: dict[str, Any]) -> dict[str, Any]: text=text, entities=seed_spans, augmented_output=row.get(COL_AUGMENTED_ENTITIES, {}), + excluded_entity_labels=set(excluded_entity_labels or []), ) merged_entities = [entity.as_dict() for entity in merged] row[COL_MERGED_ENTITIES] = EntitiesSchema(entities=merged_entities).model_dump(mode="json") diff --git a/src/anonymizer/engine/detection/detection_workflow.py b/src/anonymizer/engine/detection/detection_workflow.py index a92253a7..b146e22d 100644 --- a/src/anonymizer/engine/detection/detection_workflow.py +++ b/src/anonymizer/engine/detection/detection_workflow.py @@ -94,7 +94,7 @@ def detect_and_validate_entities( validation_excerpt_window_chars: int = _DEFAULT_VALIDATION_EXCERPT_WINDOW_CHARS, validation_single_chunk_full_text: bool = True, entity_labels: list[str] | None = None, - entity_label_denylist: list[str] | None = None, + excluded_entity_labels: list[str] | None = None, data_summary: str | None = None, preview_num_records: int | None = None, ) -> EntityDetectionResult: @@ -114,7 +114,7 @@ def detect_and_validate_entities( validation_excerpt_window_chars=validation_excerpt_window_chars, validation_single_chunk_full_text=validation_single_chunk_full_text, entity_labels=entity_labels, - entity_label_denylist=entity_label_denylist, + excluded_entity_labels=excluded_entity_labels, data_summary=data_summary, ) detection_result = self._adapter.run_workflow( @@ -137,7 +137,7 @@ def _build_detection_spec( validation_excerpt_window_chars: int = _DEFAULT_VALIDATION_EXCERPT_WINDOW_CHARS, validation_single_chunk_full_text: bool = True, entity_labels: list[str] | None = None, - entity_label_denylist: list[str] | None = None, + excluded_entity_labels: list[str] | None = None, data_summary: str | None = None, ) -> tuple[list[ModelConfig], list[ColumnConfigT]]: """Build the (model_configs, columns) for the core detection workflow. @@ -146,7 +146,10 @@ def _build_detection_spec( and :meth:`build_detection_config` (which exports it for an external runtime), so both paths run exactly the same workflow. """ - labels = _resolve_detection_labels(entity_labels, set(entity_label_denylist) if entity_label_denylist else None) + labels = _resolve_detection_labels( + entity_labels, + set(excluded_entity_labels) if excluded_entity_labels else None, + ) workflow_model_configs = self._inject_detector_params( model_configs=model_configs, selected_models=selected_models, @@ -222,6 +225,7 @@ def _build_detection_spec( DetectionTransformConfig( name=COL_MERGED_ENTITIES, operation=DetectionTransformOperation.MERGE_AND_BUILD_CANDIDATES, + excluded_entity_labels=list(excluded_entity_labels or []), ), DetectionTransformConfig( name=COL_DETECTED_ENTITIES, @@ -243,7 +247,7 @@ def build_detection_config( validation_excerpt_window_chars: int = _DEFAULT_VALIDATION_EXCERPT_WINDOW_CHARS, validation_single_chunk_full_text: bool = True, entity_labels: list[str] | None = None, - entity_label_denylist: list[str] | None = None, + excluded_entity_labels: list[str] | None = None, data_summary: str | None = None, ) -> DataDesignerConfigBuilder: """Build (without executing) the core detection workflow as a DataDesigner @@ -259,7 +263,7 @@ def build_detection_config( validation_excerpt_window_chars=validation_excerpt_window_chars, validation_single_chunk_full_text=validation_single_chunk_full_text, entity_labels=entity_labels, - entity_label_denylist=entity_label_denylist, + excluded_entity_labels=excluded_entity_labels, data_summary=data_summary, ) return self._adapter.build_config( @@ -280,7 +284,7 @@ def build_detection_builder_for_seed( validation_excerpt_window_chars: int = _DEFAULT_VALIDATION_EXCERPT_WINDOW_CHARS, validation_single_chunk_full_text: bool = True, entity_labels: list[str] | None = None, - entity_label_denylist: list[str] | None = None, + excluded_entity_labels: list[str] | None = None, data_summary: str | None = None, job_index: int = 0, num_jobs: int = 1, @@ -301,7 +305,7 @@ def build_detection_builder_for_seed( validation_excerpt_window_chars=validation_excerpt_window_chars, validation_single_chunk_full_text=validation_single_chunk_full_text, entity_labels=entity_labels, - entity_label_denylist=entity_label_denylist, + excluded_entity_labels=excluded_entity_labels, data_summary=data_summary, ) return self._adapter.build_config_for_seed( @@ -320,7 +324,7 @@ def identify_latent_entities( selected_models: DetectionModelSelection, gliner_detection_threshold: float, entity_labels: list[str] | None = None, - entity_label_denylist: list[str] | None = None, + excluded_entity_labels: list[str] | None = None, privacy_goal: PrivacyGoal | None, data_summary: str | None = None, preview_num_records: int | None = None, @@ -330,7 +334,10 @@ def identify_latent_entities( Runs after ``detect_and_validate_entities`` when rewrite mode is enabled. Uses an LLM to identify entities inferable from context. """ - labels = _resolve_detection_labels(entity_labels, set(entity_label_denylist) if entity_label_denylist else None) + labels = _resolve_detection_labels( + entity_labels, + set(excluded_entity_labels) if excluded_entity_labels else None, + ) workflow_model_configs = self._inject_detector_params( model_configs=model_configs, selected_models=selected_models, @@ -347,7 +354,7 @@ def identify_latent_entities( prompt=_get_latent_prompt( data_summary=data_summary, privacy_goal=privacy_goal, - entity_label_denylist=entity_label_denylist, + excluded_entity_labels=excluded_entity_labels, ), model_alias=latent_alias, output_format=LatentEntitiesSchema, @@ -359,7 +366,7 @@ def identify_latent_entities( latent_df = latent_result.dataframe.copy() if COL_LATENT_ENTITIES in latent_df.columns: latent_df[COL_LATENT_ENTITIES] = latent_df[COL_LATENT_ENTITIES].apply( - lambda raw: _filter_denied_latent_entities(raw, entity_label_denylist) + lambda raw: _filter_excluded_latent_entities(raw, excluded_entity_labels) ) return EntityDetectionResult(dataframe=latent_df, failed_records=latent_result.failed_records) @@ -374,7 +381,7 @@ def run( validation_excerpt_window_chars: int = _DEFAULT_VALIDATION_EXCERPT_WINDOW_CHARS, validation_single_chunk_full_text: bool = True, entity_labels: list[str] | None = None, - entity_label_denylist: list[str] | None = None, + excluded_entity_labels: list[str] | None = None, privacy_goal: PrivacyGoal | None = None, data_summary: str | None = None, tag_latent_entities: bool = True, @@ -405,7 +412,7 @@ def run( validation_excerpt_window_chars=validation_excerpt_window_chars, validation_single_chunk_full_text=validation_single_chunk_full_text, entity_labels=entity_labels, - entity_label_denylist=entity_label_denylist, + excluded_entity_labels=excluded_entity_labels, data_summary=data_summary, preview_num_records=preview_num_records, ) @@ -417,7 +424,7 @@ def run( selected_models=selected_models, gliner_detection_threshold=gliner_detection_threshold, entity_labels=entity_labels, - entity_label_denylist=entity_label_denylist, + excluded_entity_labels=excluded_entity_labels, privacy_goal=privacy_goal, data_summary=data_summary, preview_num_records=preview_num_records, @@ -434,10 +441,12 @@ def run( # TODO(docs): document this None-vs-explicit contract in user-facing docs. if COL_DETECTED_ENTITIES in final_df.columns: allowed = set(entity_labels) if entity_labels is not None else None - entity_label_denylist_set = set(entity_label_denylist) if entity_label_denylist else None + excluded_entity_labels_set = set(excluded_entity_labels) if excluded_entity_labels else None final_df[COL_FINAL_ENTITIES] = final_df[COL_DETECTED_ENTITIES].apply( lambda raw: _materialize_final_entities( - raw, allowed_labels=allowed, entity_label_denylist=entity_label_denylist_set + raw, + allowed_labels=allowed, + excluded_entity_labels=excluded_entity_labels_set, ) ) if compute_grouped: @@ -477,15 +486,15 @@ def _inject_detector_params( def _resolve_detection_labels( entity_labels: list[str] | None, - entity_label_denylist: set[str] | None = None, + excluded_entity_labels: set[str] | None = None, ) -> list[str]: labels = list(DEFAULT_ENTITY_LABELS) if entity_labels is None else list(entity_labels) - if entity_label_denylist: - denied = {label.casefold() for label in entity_label_denylist} - labels = [label for label in labels if label.casefold() not in denied] + if excluded_entity_labels: + excluded = {label.casefold() for label in excluded_entity_labels} + labels = [label for label in labels if label.casefold() not in excluded] if not labels: logger.warning( - "entity_label_denylist removed all labels from the effective detection set. No entities will be detected." + "excluded_entity_labels removed all labels from the effective detection set. No entities will be detected." ) return labels @@ -494,28 +503,28 @@ def _materialize_final_entities( raw: object, *, allowed_labels: set[str] | None, - entity_label_denylist: set[str] | None, + excluded_entity_labels: set[str] | None, ) -> dict: - """Build COL_FINAL_ENTITIES, optionally filtering to *allowed_labels* and excluding *entity_label_denylist*.""" + """Build COL_FINAL_ENTITIES, applying the configured label scope.""" parsed = EntitiesSchema.from_raw(raw) allowed = {label.casefold() for label in allowed_labels} if allowed_labels is not None else None - denied = {label.casefold() for label in entity_label_denylist or []} + excluded = {label.casefold() for label in excluded_entity_labels or []} kept = [ e for e in parsed.entities - if (allowed is None or e.label.strip().casefold() in allowed) and e.label.strip().casefold() not in denied + if (allowed is None or e.label.strip().casefold() in allowed) and e.label.strip().casefold() not in excluded ] return EntitiesSchema(entities=kept).model_dump() -def _filter_denied_latent_entities(raw: object, entity_label_denylist: list[str] | None) -> object: - """Remove denied latent labels while preserving the structured payload shape.""" - denied = {label.casefold() for label in entity_label_denylist or []} - if not denied: +def _filter_excluded_latent_entities(raw: object, excluded_entity_labels: list[str] | None) -> object: + """Remove excluded latent labels while preserving the structured payload shape.""" + excluded = {label.casefold() for label in excluded_entity_labels or []} + if not excluded: return raw if isinstance(raw, LatentEntitiesSchema): - kept = [entity for entity in raw.latent_entities if entity.label.strip().casefold() not in denied] + kept = [entity for entity in raw.latent_entities if entity.label.strip().casefold() not in excluded] return LatentEntitiesSchema(latent_entities=kept).model_dump(mode="json") if isinstance(raw, dict): @@ -527,7 +536,7 @@ def _filter_denied_latent_entities(raw: object, entity_label_denylist: list[str] "latent_entities": [ entity for entity in entities - if not isinstance(entity, dict) or str(entity.get("label", "")).strip().casefold() not in denied + if not isinstance(entity, dict) or str(entity.get("label", "")).strip().casefold() not in excluded ], } @@ -536,7 +545,7 @@ def _filter_denied_latent_entities(raw: object, entity_label_denylist: list[str] return [ entity for entity in raw - if not isinstance(entity, dict) or str(entity.get("label", "")).strip().casefold() not in denied + if not isinstance(entity, dict) or str(entity.get("label", "")).strip().casefold() not in excluded ] return raw @@ -771,22 +780,22 @@ def _get_latent_prompt( *, data_summary: str | None, privacy_goal: PrivacyGoal | None, - entity_label_denylist: list[str] | None = None, + excluded_entity_labels: list[str] | None = None, ) -> str: summary_line = data_summary.strip() if data_summary else "Not provided" privacy_goal_text = _format_privacy_goal(privacy_goal) - denied_labels = sorted({label.strip().casefold() for label in entity_label_denylist or [] if label.strip()}) - denylist_block = ( + excluded_labels = sorted({label.strip().casefold() for label in excluded_entity_labels or [] if label.strip()}) + exclusion_block = ( "\n\n" - f"Do NOT return latent entities with these labels: {', '.join(denied_labels)}.\n" + f"Do NOT return latent entities with these labels: {', '.join(excluded_labels)}.\n" "\n" - if denied_labels + if excluded_labels else "" ) prompt = """You are performing: LATENT ENTITY & INFERENCE ANALYSIS for privacy protection. The text will be rewritten according to this privacy goal: <> -<> +<> Goal: Identify sensitive information that is NOT explicitly stated in the text, \ but is reasonably inferable from context and could materially increase re-identification \ @@ -874,7 +883,7 @@ def _get_latent_prompt( "<>": privacy_goal_text, "<>": summary_line, "<>": _jinja(COL_TAGGED_TEXT), - "<>": denylist_block, + "<>": exclusion_block, }, ) diff --git a/src/anonymizer/engine/detection/postprocess.py b/src/anonymizer/engine/detection/postprocess.py index 2af2b300..aeae876c 100644 --- a/src/anonymizer/engine/detection/postprocess.py +++ b/src/anonymizer/engine/detection/postprocess.py @@ -160,12 +160,14 @@ def apply_augmented_entities( text: str, entities: list[EntitySpan], augmented_output: dict | str, + excluded_entity_labels: set[str] | None = None, ) -> list[EntitySpan]: - """Add augmented entities, split full names, and resolve overlaps on merged set.""" + """Add allowed augmented entities, split full names, and resolve overlaps.""" payload = _safe_json_loads(augmented_output) if isinstance(augmented_output, str) else augmented_output augmented = payload.get("entities", []) if isinstance(payload, dict) else [] if not isinstance(augmented, list): augmented = [] + excluded = {label.strip().casefold() for label in excluded_entity_labels or set()} merged = list(entities) for idx, suggestion in enumerate(augmented): @@ -173,7 +175,7 @@ def apply_augmented_entities( continue value = str(suggestion.get("value", "")).strip() label = str(suggestion.get("label", "")).strip() - if not value or not label: + if not value or not label or label.casefold() in excluded: continue for start, end in _find_all_occurrences(text=text, needle=value): entity_id = _build_entity_id(label=label, start=start, end=end) diff --git a/src/anonymizer/engine/evaluation/entity_coverage_judge.py b/src/anonymizer/engine/evaluation/entity_coverage_judge.py index 292427fc..4fda8495 100644 --- a/src/anonymizer/engine/evaluation/entity_coverage_judge.py +++ b/src/anonymizer/engine/evaluation/entity_coverage_judge.py @@ -70,32 +70,32 @@ class EntityCoverageSchema(BaseModel): def _effective_entity_labels( entity_labels: list[str] | None, - entity_label_denylist: list[str] | None, + excluded_entity_labels: list[str] | None, ) -> list[str] | None: """Return the effective allowlist for prompt and filter scope. ``None`` remains permissive so coverage includes novel labels introduced by - augmentation. Denied labels are applied independently by the prompt and + augmentation. Excluded labels are applied independently by the prompt and postprocessing filter. """ if entity_labels is None: return None - if not entity_label_denylist: + if not excluded_entity_labels: return entity_labels - denied = {label.casefold() for label in entity_label_denylist} - effective = [label for label in entity_labels if label.casefold() not in denied] + excluded = {label.casefold() for label in excluded_entity_labels} + effective = [label for label in entity_labels if label.casefold() not in excluded] return effective def _entity_type_scope_block( entity_labels: list[str] | None, - entity_label_denylist: list[str] | None = None, + excluded_entity_labels: list[str] | None = None, ) -> str: if entity_labels is None: - denied = sorted({label.strip().casefold() for label in entity_label_denylist or [] if label.strip()}) + excluded = sorted({label.strip().casefold() for label in excluded_entity_labels or [] if label.strip()}) exclusion = ( - f"\nDo NOT report candidates with these explicitly excluded entity labels: {', '.join(denied)}." - if denied + f"\nDo NOT report candidates with these explicitly excluded entity labels: {', '.join(excluded)}." + if excluded else "" ) return f"\nEvaluate for all PII and sensitive entity types.{exclusion}\n" @@ -125,10 +125,10 @@ def _data_summary_block(data_summary: str | None) -> str: def _coverage_prompt( *, entity_labels: list[str] | None, - entity_label_denylist: list[str] | None = None, + excluded_entity_labels: list[str] | None = None, data_summary: str | None = None, ) -> str: - entity_scope_block = _entity_type_scope_block(entity_labels, entity_label_denylist) + entity_scope_block = _entity_type_scope_block(entity_labels, excluded_entity_labels) data_context_section = f"\n\n{_data_summary_block(data_summary)}" if data_summary and data_summary.strip() else "" active_labels = entity_labels if entity_labels is not None else DEFAULT_ENTITY_LABELS @@ -392,12 +392,12 @@ def _normalize_literal_text(value: object) -> str: def _filter_out_of_scope_entities( entities: list[_CandidateT], entity_labels: list[str] | None, - entity_label_denylist: list[str] | None = None, + excluded_entity_labels: list[str] | None = None, ) -> list[_CandidateT]: """Drop entities with empty labels or labels outside the configured scope. When ``entity_labels`` is None all labels are in scope; only empty labels - and explicitly denied labels are dropped. This mirrors the prompt's scope + and explicitly excluded labels are dropped. This mirrors the prompt's scope instruction deterministically so a model that returns out-of-scope labels does not lower the coverage score. @@ -409,14 +409,14 @@ def _filter_out_of_scope_entities( meaningfully risking false negatives on well-formed responses. """ allowed = {label.casefold() for label in entity_labels} if entity_labels is not None else None - denied = {label.casefold() for label in entity_label_denylist or []} + excluded = {label.casefold() for label in excluded_entity_labels or []} result = [] for entity in entities: label = str(entity.get("label", "")).strip() if not label: continue normalized_label = label.casefold() - if (allowed is not None and normalized_label not in allowed) or normalized_label in denied: + if (allowed is not None and normalized_label not in allowed) or normalized_label in excluded: continue result.append(entity) return result @@ -460,9 +460,9 @@ class EntityCoverageWorkflow(_BaseJudgeWorkflow): scope. Deterministic postprocessing removes nonliteral and already-covered findings. ``entity_labels`` scopes evaluation to a specific allowlist of labels (``None`` means - all labels). ``entity_label_denylist`` further excludes specific labels from - scope regardless of ``entity_labels``. Both are applied to the LLM prompt and the - postprocess filter so denied labels are never penalised in the coverage score. + all labels). ``excluded_entity_labels`` removes specific labels from scope + regardless of ``entity_labels``. Both are applied to the LLM prompt and the + postprocess filter so excluded labels are never penalised in the coverage score. Output columns: ``COL_ENTITY_COVERAGE`` (float|None) — covered / total unique candidate values @@ -484,12 +484,12 @@ def __init__( adapter: NddAdapter, *, entity_labels: list[str] | None = None, - entity_label_denylist: list[str] | None = None, + excluded_entity_labels: list[str] | None = None, data_summary: str | None = None, ) -> None: super().__init__(adapter) self._entity_labels = entity_labels - self._entity_label_denylist = entity_label_denylist + self._excluded_entity_labels = excluded_entity_labels self._data_summary = data_summary # ------------------------------------------------------------------ hooks @@ -524,12 +524,12 @@ def _extract_invalid(cls, parsed: BaseModel) -> list[dict[str, object]]: def column_config(self, selected_models: EvaluateModelSelection) -> LLMStructuredColumnConfig: """Override to inject instance-specific entity_labels and data_summary.""" - effective_labels = _effective_entity_labels(self._entity_labels, self._entity_label_denylist) + effective_labels = _effective_entity_labels(self._entity_labels, self._excluded_entity_labels) return LLMStructuredColumnConfig( name=self.RAW_COL, prompt=_coverage_prompt( entity_labels=effective_labels, - entity_label_denylist=self._entity_label_denylist, + excluded_entity_labels=self._excluded_entity_labels, data_summary=self._data_summary, ), model_alias=resolve_model_alias(self.MODEL_ROLE, selected_models), @@ -554,8 +554,8 @@ def postprocess(self, dataframe: pd.DataFrame) -> pd.DataFrame: else: candidates = _filter_out_of_scope_entities( candidates, - _effective_entity_labels(self._entity_labels, self._entity_label_denylist), - self._entity_label_denylist, + _effective_entity_labels(self._entity_labels, self._excluded_entity_labels), + self._excluded_entity_labels, ) candidates = _filter_nonliteral_entities(candidates, out[COL_TEXT].loc[idx]) candidates = _deduplicate_candidate_values(candidates) diff --git a/src/anonymizer/engine/replace/replace_runner.py b/src/anonymizer/engine/replace/replace_runner.py index 61b8941d..58ff1ad4 100644 --- a/src/anonymizer/engine/replace/replace_runner.py +++ b/src/anonymizer/engine/replace/replace_runner.py @@ -119,7 +119,7 @@ def evaluate( selected_models: EvaluateModelSelection, preview_num_records: int | None = None, entity_labels: list[str] | None = None, - entity_label_denylist: list[str] | None = None, + excluded_entity_labels: list[str] | None = None, compute_detection_validity: bool = False, data_summary: str | None = None, ) -> ReplacementResult: @@ -130,7 +130,7 @@ def evaluate( Detection validity runs only when ``compute_detection_validity=True``. All active judges are submitted as columns of one DataDesigner workflow. - ``entity_labels`` and ``entity_label_denylist`` together define the label + ``entity_labels`` and ``excluded_entity_labels`` together define the label scope passed to the coverage judge — only entities whose labels were in scope during detection are evaluated. @@ -156,7 +156,7 @@ def evaluate( entity_coverage_judge = EntityCoverageWorkflow( adapter=self._adapter, # type: ignore[arg-type] entity_labels=entity_labels, - entity_label_denylist=entity_label_denylist, + excluded_entity_labels=excluded_entity_labels, data_summary=data_summary, ) failed_records: list[FailedRecord] = [] diff --git a/src/anonymizer/engine/workflow_columns/detection/config.py b/src/anonymizer/engine/workflow_columns/detection/config.py index a9661818..8a9180f9 100644 --- a/src/anonymizer/engine/workflow_columns/detection/config.py +++ b/src/anonymizer/engine/workflow_columns/detection/config.py @@ -41,6 +41,7 @@ class DetectionTransformOperation(str, Enum): class DetectionTransformConfig(SingleColumnConfig): column_type: Literal["anonymizer-detection-transform"] = "anonymizer-detection-transform" operation: DetectionTransformOperation + excluded_entity_labels: list[str] = Field(default_factory=list) _REQUIRED_COLUMNS: ClassVar[dict[DetectionTransformOperation, list[str]]] = { DetectionTransformOperation.PARSE_DETECTED_ENTITIES: [COL_TEXT, COL_RAW_DETECTED], diff --git a/src/anonymizer/engine/workflow_columns/detection/impl.py b/src/anonymizer/engine/workflow_columns/detection/impl.py index 54a92405..533d6ff8 100644 --- a/src/anonymizer/engine/workflow_columns/detection/impl.py +++ b/src/anonymizer/engine/workflow_columns/detection/impl.py @@ -100,6 +100,11 @@ def __getattr__(self, name: str) -> Any: class DetectionTransformGenerator(ColumnGeneratorCellByCell[DetectionTransformConfig]): def generate(self, data: dict[str, Any]) -> dict[str, Any]: operation = DetectionTransformOperation(self.config.operation) + if operation == DetectionTransformOperation.MERGE_AND_BUILD_CANDIDATES: + return merge_and_build_candidates( + data, + excluded_entity_labels=self.config.excluded_entity_labels, + ) return _TRANSFORMS[operation](data) diff --git a/src/anonymizer/interface/anonymizer.py b/src/anonymizer/interface/anonymizer.py index 9feade6d..6915a69b 100644 --- a/src/anonymizer/interface/anonymizer.py +++ b/src/anonymizer/interface/anonymizer.py @@ -306,7 +306,7 @@ def export_detection_config( validation_max_entities_per_call=config.detect.validation_max_entities_per_call, validation_excerpt_window_chars=config.detect.validation_excerpt_window_chars, entity_labels=config.detect.entity_labels, - entity_label_denylist=config.detect.entity_label_denylist, + excluded_entity_labels=config.detect.excluded_entity_labels, data_summary=data.data_summary, ) @@ -339,7 +339,7 @@ def export_detection_builder_for_seed( validation_max_entities_per_call=config.detect.validation_max_entities_per_call, validation_excerpt_window_chars=config.detect.validation_excerpt_window_chars, entity_labels=config.detect.entity_labels, - entity_label_denylist=config.detect.entity_label_denylist, + excluded_entity_labels=config.detect.excluded_entity_labels, data_summary=data_summary, job_index=job_index, num_jobs=num_jobs, @@ -379,7 +379,7 @@ def preview( replace_method=config.replace, rewrite_config=config.rewrite.privacy_goal if config.rewrite is not None else None, entity_labels=config.detect.entity_labels, - entity_label_denylist=config.detect.entity_label_denylist, + excluded_entity_labels=config.detect.excluded_entity_labels, data_summary=result.data_summary, ) except KeyboardInterrupt: @@ -462,7 +462,7 @@ def evaluate( raise InvalidConfigError(str(exc)) from exc entity_labels = getattr(output, "entity_labels", None) - entity_label_denylist = getattr(output, "entity_label_denylist", None) + excluded_entity_labels = getattr(output, "excluded_entity_labels", None) data_summary = getattr(output, "data_summary", None) num_records = len(output.trace_dataframe) mode_name = "rewrite" if is_rewrite else type(replace_method).__name__ @@ -526,7 +526,7 @@ def evaluate( coverage_wf = EntityCoverageWorkflow( adapter=self._adapter, entity_labels=entity_labels, - entity_label_denylist=entity_label_denylist, + excluded_entity_labels=excluded_entity_labels, data_summary=data_summary, ) logger.info(LOG_INDENT + "🔎 Running entity coverage") @@ -558,7 +558,7 @@ def evaluate( failed_records=all_failed, rewrite_config=rewrite_config, entity_labels=entity_labels, - entity_label_denylist=entity_label_denylist, + excluded_entity_labels=excluded_entity_labels, data_summary=data_summary, ) else: @@ -572,7 +572,7 @@ def evaluate( model_configs=self._model_configs, selected_models=self._selected_models.evaluate, entity_labels=entity_labels, - entity_label_denylist=entity_label_denylist, + excluded_entity_labels=excluded_entity_labels, compute_detection_validity=evaluate_config.compute_detection_validity, data_summary=data_summary, ) @@ -608,7 +608,7 @@ def evaluate( failed_records=replace_result.failed_records, replace_method=replace_method, entity_labels=entity_labels, - entity_label_denylist=entity_label_denylist, + excluded_entity_labels=excluded_entity_labels, data_summary=data_summary, ) @@ -723,7 +723,7 @@ def _run_internal_impl( validation_max_entities_per_call=config.detect.validation_max_entities_per_call, validation_excerpt_window_chars=config.detect.validation_excerpt_window_chars, entity_labels=config.detect.entity_labels, - entity_label_denylist=config.detect.entity_label_denylist, + excluded_entity_labels=config.detect.excluded_entity_labels, privacy_goal=config.rewrite.privacy_goal if config.rewrite else None, data_summary=data.data_summary, tag_latent_entities=config.rewrite is not None, @@ -819,7 +819,7 @@ def _run_internal_impl( replace_method=config.replace, rewrite_config=config.rewrite.privacy_goal if config.rewrite is not None else None, entity_labels=config.detect.entity_labels, - entity_label_denylist=config.detect.entity_label_denylist, + excluded_entity_labels=config.detect.excluded_entity_labels, data_summary=data.data_summary, ) diff --git a/src/anonymizer/interface/results.py b/src/anonymizer/interface/results.py index 8f82791e..ed7493e8 100644 --- a/src/anonymizer/interface/results.py +++ b/src/anonymizer/interface/results.py @@ -68,9 +68,9 @@ class AnonymizerResult(_DisplayMixin): detection. Preserved for ``evaluate()`` so the coverage judge scopes its evaluation to the same label set. ``None`` means all default labels were in scope. - entity_label_denylist: Labels that were explicitly excluded from + excluded_entity_labels: Labels that were explicitly excluded from detection. Preserved for ``evaluate()`` so the coverage judge does - not penalise the output for not anonymizing denied labels. + not penalise the output for not anonymizing excluded labels. data_summary: Optional dataset context supplied with the original input. Preserved for ``evaluate()`` so entity-coverage judging uses the same context as detection. @@ -83,7 +83,7 @@ class AnonymizerResult(_DisplayMixin): replace_method: ReplaceMethod | None = None rewrite_config: PrivacyGoal | None = None entity_labels: list[str] | None = None - entity_label_denylist: list[str] | None = None + excluded_entity_labels: list[str] | None = None data_summary: str | None = None _display_cycle_index: int = field(default=0, init=False, repr=False) @@ -122,9 +122,9 @@ class PreviewResult(_DisplayMixin): detection. Preserved for ``evaluate()`` so the coverage judge scopes its evaluation to the same label set. ``None`` means all default labels were in scope. - entity_label_denylist: Labels that were explicitly excluded from + excluded_entity_labels: Labels that were explicitly excluded from detection. Preserved for ``evaluate()`` so the coverage judge does - not penalise the output for not anonymizing denied labels. + not penalise the output for not anonymizing excluded labels. data_summary: Optional dataset context supplied with the original input. Preserved for ``evaluate()`` so entity-coverage judging uses the same context as detection. @@ -138,7 +138,7 @@ class PreviewResult(_DisplayMixin): replace_method: ReplaceMethod | None = None rewrite_config: PrivacyGoal | None = None entity_labels: list[str] | None = None - entity_label_denylist: list[str] | None = None + excluded_entity_labels: list[str] | None = None data_summary: str | None = None _display_cycle_index: int = field(default=0, init=False, repr=False) diff --git a/src/anonymizer/measurement/records/run.py b/src/anonymizer/measurement/records/run.py index fcff24d9..8cc721af 100644 --- a/src/anonymizer/measurement/records/run.py +++ b/src/anonymizer/measurement/records/run.py @@ -20,13 +20,13 @@ def _detect_config_metadata(detect: Any | None) -> dict[str, Any]: entity_label_count = len(DEFAULT_ENTITY_LABELS) else: entity_label_count = len(entity_labels) - entity_label_denylist = getattr(detect, "entity_label_denylist", None) + excluded_entity_labels = getattr(detect, "excluded_entity_labels", None) return { "gliner_threshold": getattr(detect, "gliner_threshold", None), "entity_label_source": "custom" if entity_labels is not None else "default", "entity_label_count": entity_label_count, "entity_labels": list(entity_labels) if entity_labels is not None else None, - "entity_label_denylist": list(entity_label_denylist) if entity_label_denylist is not None else None, + "excluded_entity_labels": list(excluded_entity_labels) if excluded_entity_labels is not None else None, "validation_max_entities_per_call": getattr(detect, "validation_max_entities_per_call", None), "validation_excerpt_window_chars": getattr(detect, "validation_excerpt_window_chars", None), } diff --git a/tests/config/test_anonymizer_config.py b/tests/config/test_anonymizer_config.py index 02e7daa8..11683d18 100644 --- a/tests/config/test_anonymizer_config.py +++ b/tests/config/test_anonymizer_config.py @@ -155,70 +155,70 @@ def test_detect_validation_excerpt_window_chars_must_be_positive() -> None: AnonymizerConfig(detect={"validation_excerpt_window_chars": 0}, replace=Redact()) -# ── entity_label_denylist ───────────────────────────────────────────────────── +# ── excluded_entity_labels ──────────────────────────────────────────────────── -def test_entity_label_denylist_defaults_to_none() -> None: +def test_excluded_entity_labels_defaults_to_none() -> None: config = AnonymizerConfig(replace=Redact()) - assert config.detect.entity_label_denylist is None + assert config.detect.excluded_entity_labels is None -def test_entity_label_denylist_accepts_list() -> None: - config = AnonymizerConfig(detect={"entity_label_denylist": ["EMAIL", "city"]}, replace=Redact()) - assert config.detect.entity_label_denylist is not None - assert set(config.detect.entity_label_denylist) == {"email", "city"} +def test_excluded_entity_labels_accepts_list() -> None: + config = AnonymizerConfig(detect={"excluded_entity_labels": ["EMAIL", "city"]}, replace=Redact()) + assert config.detect.excluded_entity_labels is not None + assert set(config.detect.excluded_entity_labels) == {"email", "city"} -def test_entity_label_denylist_strips_whitespace_and_lowercases() -> None: - config = AnonymizerConfig(detect={"entity_label_denylist": [" FIRST_NAME ", "Email"]}, replace=Redact()) - assert config.detect.entity_label_denylist is not None - assert "first_name" in config.detect.entity_label_denylist - assert "email" in config.detect.entity_label_denylist +def test_excluded_entity_labels_strips_whitespace_and_lowercases() -> None: + config = AnonymizerConfig(detect={"excluded_entity_labels": [" FIRST_NAME ", "Email"]}, replace=Redact()) + assert config.detect.excluded_entity_labels is not None + assert "first_name" in config.detect.excluded_entity_labels + assert "email" in config.detect.excluded_entity_labels -def test_entity_label_denylist_deduplicates(caplog: pytest.LogCaptureFixture) -> None: +def test_excluded_entity_labels_deduplicates(caplog: pytest.LogCaptureFixture) -> None: with caplog.at_level(logging.WARNING, logger="anonymizer"): - config = AnonymizerConfig(detect={"entity_label_denylist": ["email", "email"]}, replace=Redact()) - assert config.detect.entity_label_denylist == ["email"] + config = AnonymizerConfig(detect={"excluded_entity_labels": ["email", "email"]}, replace=Redact()) + assert config.detect.excluded_entity_labels == ["email"] assert "duplicates" in caplog.text -def test_entity_label_denylist_empty_list_raises() -> None: +def test_excluded_entity_labels_empty_list_raises() -> None: with pytest.raises(ValidationError, match="must not be empty"): - AnonymizerConfig(detect={"entity_label_denylist": []}, replace=Redact()) + AnonymizerConfig(detect={"excluded_entity_labels": []}, replace=Redact()) -def test_entity_label_denylist_whitespace_only_raises() -> None: +def test_excluded_entity_labels_whitespace_only_raises() -> None: with pytest.raises(ValidationError, match="must not be empty"): - AnonymizerConfig(detect={"entity_label_denylist": [" ", ""]}, replace=Redact()) + AnonymizerConfig(detect={"excluded_entity_labels": [" ", ""]}, replace=Redact()) -def test_entity_label_denylist_overlap_with_entity_labels_warns(caplog: pytest.LogCaptureFixture) -> None: +def test_excluded_entity_labels_overlap_with_entity_labels_warns(caplog: pytest.LogCaptureFixture) -> None: with caplog.at_level(logging.WARNING, logger="anonymizer"): AnonymizerConfig( - detect={"entity_labels": ["email", "city"], "entity_label_denylist": ["email"]}, + detect={"entity_labels": ["email", "city"], "excluded_entity_labels": ["email"]}, replace=Redact(), ) assert "email" in caplog.text assert "will never be detected" in caplog.text -def test_entity_label_denylist_no_overlap_does_not_warn(caplog: pytest.LogCaptureFixture) -> None: +def test_excluded_entity_labels_no_overlap_does_not_warn(caplog: pytest.LogCaptureFixture) -> None: with caplog.at_level(logging.WARNING, logger="anonymizer"): AnonymizerConfig( - detect={"entity_labels": ["email", "city"], "entity_label_denylist": ["first_name"]}, + detect={"entity_labels": ["email", "city"], "excluded_entity_labels": ["first_name"]}, replace=Redact(), ) assert "will never be detected" not in caplog.text -def test_entity_label_denylist_overlap_warning_only_fires_when_allowlist_explicit( +def test_excluded_entity_labels_overlap_warning_only_fires_when_allowlist_explicit( caplog: pytest.LogCaptureFixture, ) -> None: - """No warning when entity_labels=None (defaults) even if denylist is set.""" + """No warning when entity_labels=None (defaults) even if exclusions are set.""" with caplog.at_level(logging.WARNING, logger="anonymizer"): AnonymizerConfig( - detect={"entity_label_denylist": ["email"]}, + detect={"excluded_entity_labels": ["email"]}, replace=Redact(), ) assert "will never be detected" not in caplog.text diff --git a/tests/engine/test_detection_config_serialization.py b/tests/engine/test_detection_config_serialization.py index 976c7913..dd8c83ea 100644 --- a/tests/engine/test_detection_config_serialization.py +++ b/tests/engine/test_detection_config_serialization.py @@ -53,6 +53,7 @@ def test_detection_builder_round_trips_through_native_data_designer_config(tmp_p validation_max_entities_per_call=7, validation_excerpt_window_chars=321, entity_labels=["first_name", "email"], + excluded_entity_labels=["email"], data_summary="Customer support messages", job_index=1, num_jobs=3, @@ -77,6 +78,12 @@ def test_detection_builder_round_trips_through_native_data_designer_config(tmp_p transforms = [column for column in columns if isinstance(column, DetectionTransformConfig)] assert {DetectionTransformOperation(column.operation) for column in transforms} == set(DetectionTransformOperation) + merge_transform = next( + column + for column in transforms + if DetectionTransformOperation(column.operation) == DetectionTransformOperation.MERGE_AND_BUILD_CANDIDATES + ) + assert merge_transform.excluded_entity_labels == ["email"] validation = next(column for column in columns if column.name == COL_VALIDATION_DECISIONS) assert isinstance(validation, ChunkedValidationConfig) @@ -102,7 +109,7 @@ def _get_gliner_labels_from_builder(builder: DataDesignerConfigBuilder) -> list[ return gliner["inference_parameters"]["extra_body"]["labels"] -def test_build_detection_builder_for_seed_respects_entity_label_denylist(tmp_path: Path) -> None: +def test_build_detection_builder_for_seed_respects_excluded_entity_labels(tmp_path: Path) -> None: seed_path = tmp_path / "seed.parquet" pd.DataFrame({COL_TEXT: ["Alice"]}).to_parquet(seed_path, index=False) @@ -114,7 +121,7 @@ def test_build_detection_builder_for_seed_respects_entity_label_denylist(tmp_pat selected_models=parsed_models.selected_models.detection, gliner_detection_threshold=0.3, entity_labels=["first_name", "email", "city"], - entity_label_denylist=["email"], + excluded_entity_labels=["email"], ) labels = _get_gliner_labels_from_builder(builder) @@ -123,7 +130,7 @@ def test_build_detection_builder_for_seed_respects_entity_label_denylist(tmp_pat assert "city" in labels -def test_build_detection_config_respects_entity_label_denylist(tmp_path: Path) -> None: +def test_build_detection_config_respects_excluded_entity_labels(tmp_path: Path) -> None: seed_path = tmp_path / "seed.parquet" input_df = pd.DataFrame({COL_TEXT: ["Alice"]}) input_df.to_parquet(seed_path, index=False) @@ -137,7 +144,7 @@ def test_build_detection_config_respects_entity_label_denylist(tmp_path: Path) - selected_models=parsed_models.selected_models.detection, gliner_detection_threshold=0.3, entity_labels=["first_name", "email", "city"], - entity_label_denylist=["email"], + excluded_entity_labels=["email"], ) labels = _get_gliner_labels_from_builder(builder) diff --git a/tests/engine/test_detection_custom_columns.py b/tests/engine/test_detection_custom_columns.py index 1a42f226..84395f5b 100644 --- a/tests/engine/test_detection_custom_columns.py +++ b/tests/engine/test_detection_custom_columns.py @@ -99,6 +99,39 @@ def test_merge_and_build_candidates_writes_schema_shaped_payloads() -> None: assert isinstance(result[COL_VALIDATION_CANDIDATES]["candidates"], list) +def test_merge_filters_denied_augmentation_before_overlap_resolution() -> None: + row: dict[str, Any] = { + COL_TEXT: "Alice Johnson", + COL_VALIDATED_SEED_ENTITIES: { + "entities": [ + { + "id": "first_name_0_5", + "value": "Alice", + "label": "first_name", + "start_position": 0, + "end_position": 5, + "score": 0.95, + "source": "detector", + } + ] + }, + COL_AUGMENTED_ENTITIES: { + "entities": [ + { + "value": "Alice Johnson", + "label": " Full_Name ", + "reason": "longer overlapping span", + } + ] + }, + } + + result = merge_and_build_candidates(row, excluded_entity_labels=["full_name"]) + + merged = result[COL_MERGED_ENTITIES]["entities"] + assert [(entity["value"], entity["label"]) for entity in merged] == [("Alice", "first_name")] + + def test_enrich_validation_decisions_adds_value_from_candidates() -> None: row = { COL_VALIDATION_DECISIONS: { diff --git a/tests/engine/test_detection_postprocess.py b/tests/engine/test_detection_postprocess.py index a51eddc9..e6d7a659 100644 --- a/tests/engine/test_detection_postprocess.py +++ b/tests/engine/test_detection_postprocess.py @@ -117,6 +117,28 @@ def test_apply_augmented_entities_adds_occurrences() -> None: assert all(entity.source == "augmenter" for entity in merged) +def test_apply_augmented_entities_filters_exclusions_before_overlap_resolution() -> None: + text = "Alice Johnson" + allowed = EntitySpan("first_name_0_5", "Alice", "first_name", 0, 5, 0.95, "detector") + + merged = apply_augmented_entities( + text=text, + entities=[allowed], + augmented_output={ + "entities": [ + { + "value": "Alice Johnson", + "label": " Full_Name ", + "reason": "longer overlapping span", + } + ] + }, + excluded_entity_labels={"full_name"}, + ) + + assert merged == [allowed] + + def test_apply_augmented_entities_avoids_substring_matches() -> None: text = "Annex contains Ann." merged = apply_augmented_entities( diff --git a/tests/engine/test_detection_workflow.py b/tests/engine/test_detection_workflow.py index 380b6440..2123458f 100644 --- a/tests/engine/test_detection_workflow.py +++ b/tests/engine/test_detection_workflow.py @@ -34,7 +34,7 @@ ) from anonymizer.engine.detection.detection_workflow import ( EntityDetectionWorkflow, - _filter_denied_latent_entities, + _filter_excluded_latent_entities, _format_label_examples, _get_augment_prompt, _get_latent_prompt, @@ -160,27 +160,27 @@ def test_latent_prompt_includes_summary_and_goal() -> None: assert COL_TAGGED_TEXT in prompt -def test_latent_prompt_excludes_denied_labels() -> None: +def test_latent_prompt_excludes_configured_labels() -> None: prompt = _get_latent_prompt( data_summary=None, privacy_goal=None, - entity_label_denylist=["Health_Condition", "occupation"], + excluded_entity_labels=["Health_Condition", "occupation"], ) assert "Do NOT return latent entities with these labels: health_condition, occupation." in prompt -def test_filter_denied_latent_entities_is_case_insensitive() -> None: +def test_filter_excluded_latent_entities_is_case_insensitive() -> None: raw = { "latent_entities": [ {"label": "Health_Condition", "value": "diabetes"}, {"label": "employer", "value": "Acme"}, ] } - result = _filter_denied_latent_entities(raw, ["health_condition"]) + result = _filter_excluded_latent_entities(raw, ["health_condition"]) assert result == {"latent_entities": [{"label": "employer", "value": "Acme"}]} -def test_identify_latent_entities_filters_denied_labels( +def test_identify_latent_entities_filters_excluded_labels( stub_detector_model_configs: list[ModelConfig], stub_detection_model_selection: DetectionModelSelection, ) -> None: @@ -208,7 +208,7 @@ def test_identify_latent_entities_filters_denied_labels( model_configs=stub_detector_model_configs, selected_models=stub_detection_model_selection, gliner_detection_threshold=0.5, - entity_label_denylist=["health_condition"], + excluded_entity_labels=["health_condition"], privacy_goal=PrivacyGoal( protect="Protect inferred sensitive attributes.", preserve="Preserve non-sensitive facts.", @@ -525,36 +525,36 @@ def test_default_entity_labels_preserves_novel_augmented_entities( assert "ipv4" in final_labels -# ── entity_label_denylist ───────────────────────────────────────────────────── +# ── excluded_entity_labels ──────────────────────────────────────────────────── -def test_resolve_detection_labels_denylist_removes_labels() -> None: - labels = _resolve_detection_labels(["first_name", "email", "city"], entity_label_denylist={"email"}) +def test_resolve_detection_labels_exclusions_remove_labels() -> None: + labels = _resolve_detection_labels(["first_name", "email", "city"], excluded_entity_labels={"email"}) assert "email" not in labels assert "first_name" in labels assert "city" in labels -def test_resolve_detection_labels_denylist_is_case_insensitive() -> None: - labels = _resolve_detection_labels(["first_name", "Email"], entity_label_denylist={"EMAIL"}) +def test_resolve_detection_labels_exclusions_are_case_insensitive() -> None: + labels = _resolve_detection_labels(["first_name", "Email"], excluded_entity_labels={"EMAIL"}) assert labels == ["first_name"] -def test_resolve_detection_labels_denylist_on_defaults() -> None: - labels = _resolve_detection_labels(None, entity_label_denylist={"ssn", "first_name"}) +def test_resolve_detection_labels_exclusions_apply_to_defaults() -> None: + labels = _resolve_detection_labels(None, excluded_entity_labels={"ssn", "first_name"}) assert "ssn" not in labels assert "first_name" not in labels assert "email" in labels -def test_resolve_detection_labels_none_denylist_is_noop() -> None: - labels = _resolve_detection_labels(["email", "city"], entity_label_denylist=None) +def test_resolve_detection_labels_none_exclusions_is_noop() -> None: + labels = _resolve_detection_labels(["email", "city"], excluded_entity_labels=None) assert labels == ["email", "city"] def test_resolve_detection_labels_empty_result_warns(caplog: pytest.LogCaptureFixture) -> None: with caplog.at_level(logging.WARNING, logger="anonymizer.detection"): - labels = _resolve_detection_labels(["email"], entity_label_denylist={"email"}) + labels = _resolve_detection_labels(["email"], excluded_entity_labels={"email"}) assert labels == [] assert "No entities will be detected" in caplog.text @@ -571,14 +571,14 @@ def test_materialize_final_entities_applies_label_filters_case_insensitively() - result = _materialize_final_entities( raw, allowed_labels={"first_name", "email"}, - entity_label_denylist={"EMAIL"}, + excluded_entity_labels={"EMAIL"}, ) final = EntitiesSchema.from_raw(result) assert [entity.label for entity in final.entities] == ["First_Name"] -def test_denylist_filters_entities_from_final_entities( +def test_excluded_labels_are_removed_from_final_entities( stub_detector_model_configs: list[ModelConfig], stub_detection_model_selection: DetectionModelSelection, ) -> None: @@ -606,7 +606,7 @@ def test_denylist_filters_entities_from_final_entities( model_configs=stub_detector_model_configs, selected_models=stub_detection_model_selection, gliner_detection_threshold=0.5, - entity_label_denylist=["email"], + excluded_entity_labels=["email"], tag_latent_entities=False, ) @@ -616,7 +616,7 @@ def test_denylist_filters_entities_from_final_entities( assert "first_name" in final_labels -def test_denylist_does_not_affect_col_detected_entities( +def test_excluded_labels_do_not_affect_col_detected_entities( stub_detector_model_configs: list[ModelConfig], stub_detection_model_selection: DetectionModelSelection, ) -> None: @@ -645,7 +645,7 @@ def test_denylist_does_not_affect_col_detected_entities( model_configs=stub_detector_model_configs, selected_models=stub_detection_model_selection, gliner_detection_threshold=0.5, - entity_label_denylist=["email"], + excluded_entity_labels=["email"], tag_latent_entities=False, ) @@ -653,11 +653,11 @@ def test_denylist_does_not_affect_col_detected_entities( assert "email" in {e.label for e in detected.entities} -def test_denylist_combined_with_allowlist_allowlist_wins_for_non_denied( +def test_exclusions_combined_with_allowlist_preserve_other_allowed_labels( stub_detector_model_configs: list[ModelConfig], stub_detection_model_selection: DetectionModelSelection, ) -> None: - """entity_labels restricts to an allowlist; denylist further removes from that set.""" + """entity_labels restricts to an allowlist; exclusions further remove from that set.""" adapter = Mock() adapter.run_workflow.return_value = WorkflowRunResult( dataframe=pd.DataFrame( @@ -684,7 +684,7 @@ def test_denylist_combined_with_allowlist_allowlist_wins_for_non_denied( selected_models=stub_detection_model_selection, gliner_detection_threshold=0.5, entity_labels=["first_name", "city", "email"], - entity_label_denylist=["email"], + excluded_entity_labels=["email"], tag_latent_entities=False, ) @@ -693,7 +693,7 @@ def test_denylist_combined_with_allowlist_allowlist_wins_for_non_denied( assert final_labels == {"first_name", "city"} -def test_denylist_passed_to_gliner_via_labels( +def test_excluded_labels_are_removed_from_gliner_labels( stub_detector_model_configs: list[ModelConfig], stub_detection_model_selection: DetectionModelSelection, ) -> None: @@ -710,7 +710,7 @@ def test_denylist_passed_to_gliner_via_labels( selected_models=stub_detection_model_selection, gliner_detection_threshold=0.5, entity_labels=["first_name", "email", "city"], - entity_label_denylist=["email"], + excluded_entity_labels=["email"], tag_latent_entities=False, ) @@ -766,6 +766,7 @@ def test_detection_workflow_uses_plugin_transform_columns( model_configs=stub_detector_model_configs, selected_models=stub_detection_model_selection, gliner_detection_threshold=0.5, + excluded_entity_labels=["email"], tag_latent_entities=False, ) columns = adapter.run_workflow.call_args.kwargs["columns"] @@ -781,6 +782,7 @@ def test_detection_workflow_uses_plugin_transform_columns( column = _find_column(columns, name) assert isinstance(column, DetectionTransformConfig) assert DetectionTransformOperation(column.operation) == operation + assert _find_column(columns, COL_MERGED_ENTITIES).excluded_entity_labels == ["email"] assert all(getattr(column, "column_type", None) != "custom" for column in columns) diff --git a/tests/engine/test_entity_coverage_judge.py b/tests/engine/test_entity_coverage_judge.py index acee219e..b5e9d6e7 100644 --- a/tests/engine/test_entity_coverage_judge.py +++ b/tests/engine/test_entity_coverage_judge.py @@ -443,23 +443,23 @@ def test_filter_out_of_scope_entities_is_case_insensitive() -> None: assert result == entities -# ── entity_label_denylist ───────────────────────────────────────────────────── +# ── excluded_entity_labels ──────────────────────────────────────────────────── -def test_effective_entity_labels_no_denylist_returns_entity_labels_unchanged() -> None: +def test_effective_entity_labels_no_exclusions_returns_entity_labels_unchanged() -> None: assert _effective_entity_labels(["email", "city"], None) == ["email", "city"] -def test_effective_entity_labels_none_labels_none_denylist_returns_none() -> None: +def test_effective_entity_labels_none_labels_none_exclusions_returns_none() -> None: assert _effective_entity_labels(None, None) is None -def test_effective_entity_labels_subtracts_denylist_from_explicit_labels() -> None: +def test_effective_entity_labels_subtracts_exclusions_from_explicit_labels() -> None: result = _effective_entity_labels(["first_name", "email", "city"], ["email"]) assert result == ["first_name", "city"] -def test_effective_entity_labels_preserves_permissive_scope_with_denylist() -> None: +def test_effective_entity_labels_preserves_permissive_scope_with_exclusions() -> None: result = _effective_entity_labels(None, ["ssn", "first_name"]) assert result is None @@ -469,7 +469,7 @@ def test_effective_entity_labels_is_case_insensitive() -> None: assert result == ["first_name"] -def test_coverage_prompt_excludes_denied_labels_from_scope() -> None: +def test_coverage_prompt_excludes_configured_labels_from_scope() -> None: effective = _effective_entity_labels(["first_name", "email", "city"], ["email"]) prompt = _coverage_prompt(entity_labels=effective) assert "email" not in prompt @@ -477,8 +477,8 @@ def test_coverage_prompt_excludes_denied_labels_from_scope() -> None: assert "city" in prompt -def test_coverage_prompt_keeps_permissive_scope_and_names_denied_labels() -> None: - prompt = _coverage_prompt(entity_labels=None, entity_label_denylist=["email"]) +def test_coverage_prompt_keeps_permissive_scope_and_names_excluded_labels() -> None: + prompt = _coverage_prompt(entity_labels=None, excluded_entity_labels=["email"]) assert "Evaluate for all PII and sensitive entity types." in prompt assert "explicitly excluded entity labels: email" in prompt assert "This list is not exhaustive." in prompt @@ -487,17 +487,17 @@ def test_coverage_prompt_keeps_permissive_scope_and_names_denied_labels() -> Non assert "Return labels exactly as they appear" not in prompt -def test_filter_out_of_scope_entities_keeps_novel_non_denied_labels() -> None: +def test_filter_out_of_scope_entities_keeps_novel_non_excluded_labels() -> None: entities = [ {"value": "Example Clinic", "label": "clinic_name", "reasoning": "clinic"}, {"value": "alice@example.com", "label": "Email", "reasoning": "email"}, ] - result = _filter_out_of_scope_entities(entities, entity_labels=None, entity_label_denylist=["email"]) + result = _filter_out_of_scope_entities(entities, entity_labels=None, excluded_entity_labels=["email"]) assert result == [entities[0]] -def test_entity_coverage_workflow_excludes_denied_labels_from_postprocess() -> None: - """Permissive postprocessing keeps novel labels while excluding denied labels.""" +def test_entity_coverage_workflow_excludes_configured_labels_from_postprocess() -> None: + """Permissive postprocessing keeps novel labels while applying exclusions.""" raw_judge_output = [ {"value": "Alice", "label": "first_name", "reasoning": "not replaced"}, {"value": "Example Clinic", "label": "clinic_name", "reasoning": "not replaced"}, @@ -515,7 +515,7 @@ def test_entity_coverage_workflow_excludes_denied_labels_from_postprocess() -> N workflow = EntityCoverageWorkflow( adapter=Mock(), entity_labels=None, - entity_label_denylist=["email"], + excluded_entity_labels=["email"], ) result_df = workflow.postprocess(workflow.prepare(input_df)) missed = result_df[COL_MISSED_ENTITIES].iloc[0] diff --git a/tests/engine/test_replace_runner.py b/tests/engine/test_replace_runner.py index 1630ac8d..736ea8e7 100644 --- a/tests/engine/test_replace_runner.py +++ b/tests/engine/test_replace_runner.py @@ -267,13 +267,12 @@ def fake_attach_ids(df: pd.DataFrame) -> pd.DataFrame: assert bool(result.dataframe[col].iloc[0]) is True -def test_evaluate_threads_entity_labels_and_data_summary_into_coverage_prompt( +def test_evaluate_threads_detection_context_into_coverage_prompt( stub_model_configs: list[ModelConfig], stub_evaluate_model_selection: EvaluateModelSelection, ) -> None: - """Replace-mode ``evaluate()`` must forward ``entity_labels`` and ``data_summary`` - all the way into the coverage judge's prompt (the same context the rewrite path - supplies), so the judge scopes and interprets leaks against the run's taxonomy. + """Replace-mode ``evaluate()`` must forward detection context into the + coverage prompt so it uses the run's effective taxonomy. """ saved_trace = pd.DataFrame( { @@ -306,13 +305,14 @@ def fake_attach_ids(df: pd.DataFrame) -> pd.DataFrame: replace_method=Redact(), model_configs=stub_model_configs, selected_models=stub_evaluate_model_selection, - entity_labels=["first_name", "organization"], + entity_labels=["first_name", "organization", "email"], + excluded_entity_labels=["email"], data_summary="Employee HR records.", ) call_columns = adapter.run_workflow.call_args.kwargs["columns"] coverage_col = next(c for c in call_columns if c.name == COL_ENTITY_COVERAGE_JUDGE) - assert "first_name, organization" in coverage_col.prompt + assert "ONLY these entity types: first_name, organization." in coverage_col.prompt assert "Employee HR records." in coverage_col.prompt diff --git a/tests/interface/test_anonymizer_interface.py b/tests/interface/test_anonymizer_interface.py index d0a7db56..e3efcbff 100644 --- a/tests/interface/test_anonymizer_interface.py +++ b/tests/interface/test_anonymizer_interface.py @@ -134,6 +134,26 @@ def test_run_passes_detect_entity_labels_to_detection_workflow(stub_input: Anony assert detection_wf.run.call_args.kwargs["entity_labels"] == ["server_name"] +def test_run_propagates_excluded_entity_labels(stub_input: AnonymizerInput) -> None: + config = AnonymizerConfig(detect={"excluded_entity_labels": ["email"]}, replace=Redact()) + anonymizer, detection_wf, _, _ = _make_anonymizer() + + result = anonymizer.run(config=config, data=stub_input) + + assert detection_wf.run.call_args.kwargs["excluded_entity_labels"] == ["email"] + assert result.excluded_entity_labels == ["email"] + + +def test_preview_propagates_excluded_entity_labels(stub_input: AnonymizerInput) -> None: + config = AnonymizerConfig(detect={"excluded_entity_labels": ["email"]}, replace=Redact()) + anonymizer, detection_wf, _, _ = _make_anonymizer() + + result = anonymizer.preview(config=config, data=stub_input, num_records=1) + + assert detection_wf.run.call_args.kwargs["excluded_entity_labels"] == ["email"] + assert result.excluded_entity_labels == ["email"] + + def test_resolve_model_providers_raises_on_invalid_yaml(tmp_path: Path) -> None: yaml_path = tmp_path / "bad.yaml" yaml_path.write_text("not_providers: []") @@ -1052,10 +1072,10 @@ def test_run_and_preview_persist_data_summary(stub_input: AnonymizerInput) -> No assert preview.data_summary == "Customer support transcripts." -def test_evaluate_passes_data_summary_to_coverage_judge(stub_input: AnonymizerInput) -> None: - """evaluate() must forward the input summary to EntityCoverageWorkflow.""" +def test_evaluate_passes_detection_context_to_coverage_judge(stub_input: AnonymizerInput) -> None: + """evaluate() must forward persisted detection context to EntityCoverageWorkflow.""" data = stub_input.model_copy(update={"data_summary": "Customer support transcripts."}) - config = AnonymizerConfig(rewrite=Rewrite()) + config = AnonymizerConfig(detect={"excluded_entity_labels": ["email"]}, rewrite=Rewrite()) anonymizer, _, _, rewrite_runner = _make_anonymizer() run_result = anonymizer.run(config=config, data=data) @@ -1079,4 +1099,6 @@ def test_evaluate_passes_data_summary_to_coverage_judge(stub_input: AnonymizerIn evaluated = anonymizer.evaluate(run_result) assert mock_coverage_wf.call_args.kwargs["data_summary"] == "Customer support transcripts." + assert mock_coverage_wf.call_args.kwargs["excluded_entity_labels"] == ["email"] assert evaluated.data_summary == "Customer support transcripts." + assert evaluated.excluded_entity_labels == ["email"] diff --git a/tests/test_measurement.py b/tests/test_measurement.py index e3129e93..936753d8 100644 --- a/tests/test_measurement.py +++ b/tests/test_measurement.py @@ -472,7 +472,7 @@ def test_anonymizer_records_per_record_measurement_without_raw_pii(tmp_path: Pat assert run_record["input_has_data_summary"] is False assert run_record["detect"]["entity_label_source"] == "default" assert run_record["detect"]["entity_label_count"] > 0 - assert run_record["detect"]["entity_label_denylist"] is None + assert run_record["detect"]["excluded_entity_labels"] is None assert run_record["replace"]["strategy"] == "Redact" assert run_record["replace"]["normalize_label"] is True assert len(run_record["source_hash"]) == 64 @@ -483,21 +483,21 @@ def test_anonymizer_records_per_record_measurement_without_raw_pii(tmp_path: Pat assert str(input_csv) not in serialized -def test_detect_config_metadata_includes_entity_label_denylist() -> None: +def test_detect_config_metadata_includes_excluded_entity_labels() -> None: from anonymizer.measurement.records.run import _detect_config_metadata - detect = Detect(entity_labels=["first_name", "email"], entity_label_denylist=["email"]) + detect = Detect(entity_labels=["first_name", "email"], excluded_entity_labels=["email"]) metadata = _detect_config_metadata(detect) - assert metadata["entity_label_denylist"] == ["email"] + assert metadata["excluded_entity_labels"] == ["email"] assert metadata["entity_labels"] == ["email", "first_name"] -def test_detect_config_metadata_denylist_none_when_not_set() -> None: +def test_detect_config_metadata_exclusions_none_when_not_set() -> None: from anonymizer.measurement.records.run import _detect_config_metadata detect = Detect() metadata = _detect_config_metadata(detect) - assert metadata["entity_label_denylist"] is None + assert metadata["excluded_entity_labels"] is None def test_anonymizer_measurement_config_writes_jsonl(tmp_path: Path) -> None: From 6ef4589d034cf510d9475c9f44f078aac2832311 Mon Sep 17 00:00:00 2001 From: memadi Date: Wed, 19 Aug 2026 10:50:13 -0700 Subject: [PATCH 18/30] address feedback Signed-off-by: memadi --- src/anonymizer/interface/results.py | 16 +++++++------- tests/interface/test_results.py | 33 +++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/src/anonymizer/interface/results.py b/src/anonymizer/interface/results.py index ed7493e8..3c9635ea 100644 --- a/src/anonymizer/interface/results.py +++ b/src/anonymizer/interface/results.py @@ -68,12 +68,12 @@ class AnonymizerResult(_DisplayMixin): detection. Preserved for ``evaluate()`` so the coverage judge scopes its evaluation to the same label set. ``None`` means all default labels were in scope. - excluded_entity_labels: Labels that were explicitly excluded from - detection. Preserved for ``evaluate()`` so the coverage judge does - not penalise the output for not anonymizing excluded labels. data_summary: Optional dataset context supplied with the original input. Preserved for ``evaluate()`` so entity-coverage judging uses the same context as detection. + excluded_entity_labels: Labels that were explicitly excluded from + detection. Preserved for ``evaluate()`` so the coverage judge does + not penalise the output for not anonymizing excluded labels. """ dataframe: pd.DataFrame @@ -83,8 +83,8 @@ class AnonymizerResult(_DisplayMixin): replace_method: ReplaceMethod | None = None rewrite_config: PrivacyGoal | None = None entity_labels: list[str] | None = None - excluded_entity_labels: list[str] | None = None data_summary: str | None = None + excluded_entity_labels: list[str] | None = None _display_cycle_index: int = field(default=0, init=False, repr=False) def __repr__(self) -> str: @@ -122,12 +122,12 @@ class PreviewResult(_DisplayMixin): detection. Preserved for ``evaluate()`` so the coverage judge scopes its evaluation to the same label set. ``None`` means all default labels were in scope. - excluded_entity_labels: Labels that were explicitly excluded from - detection. Preserved for ``evaluate()`` so the coverage judge does - not penalise the output for not anonymizing excluded labels. data_summary: Optional dataset context supplied with the original input. Preserved for ``evaluate()`` so entity-coverage judging uses the same context as detection. + excluded_entity_labels: Labels that were explicitly excluded from + detection. Preserved for ``evaluate()`` so the coverage judge does + not penalise the output for not anonymizing excluded labels. """ dataframe: pd.DataFrame @@ -138,8 +138,8 @@ class PreviewResult(_DisplayMixin): replace_method: ReplaceMethod | None = None rewrite_config: PrivacyGoal | None = None entity_labels: list[str] | None = None - excluded_entity_labels: list[str] | None = None data_summary: str | None = None + excluded_entity_labels: list[str] | None = None _display_cycle_index: int = field(default=0, init=False, repr=False) def __repr__(self) -> str: diff --git a/tests/interface/test_results.py b/tests/interface/test_results.py index b32ee5e6..a8926e60 100644 --- a/tests/interface/test_results.py +++ b/tests/interface/test_results.py @@ -55,3 +55,36 @@ def test_preview_result_repr_is_compact() -> None: assert "preview_num_records=10" in rendered assert "__nemo_anonymizer_text_input__" not in rendered assert "bio_replaced" not in rendered + + +def test_anonymizer_result_preserves_positional_data_summary_contract() -> None: + result = AnonymizerResult( + pd.DataFrame(), + pd.DataFrame(), + "bio", + [], + None, + None, + ["first_name"], + "Customer support transcripts.", + ) + + assert result.data_summary == "Customer support transcripts." + assert result.excluded_entity_labels is None + + +def test_preview_result_preserves_positional_data_summary_contract() -> None: + result = PreviewResult( + pd.DataFrame(), + pd.DataFrame(), + "bio", + [], + 3, + None, + None, + ["first_name"], + "Customer support transcripts.", + ) + + assert result.data_summary == "Customer support transcripts." + assert result.excluded_entity_labels is None From 15715bfe61b7974323bbcfc3bb0d5028e75e7c8d Mon Sep 17 00:00:00 2001 From: memadi Date: Wed, 19 Aug 2026 15:21:28 -0700 Subject: [PATCH 19/30] ci: temporarily exclude generated skill artifacts NVSkills rewrites these files without SPDX preambles, so exclude them until the generator preserves existing headers. Upstream: https://github.com/NVIDIA/nvskills-ci/issues/66 Signed-off-by: memadi --- .copyrightignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.copyrightignore b/.copyrightignore index f72bdbad..7a83eb59 100644 --- a/.copyrightignore +++ b/.copyrightignore @@ -12,3 +12,5 @@ CHANGELOG.md .cursor/ .claude/ .agent/ +skills/anonymizer/BENCHMARK.md +skills/anonymizer/skill-card.md From dc23ebce7d063be9476b0173cfa76b26b6cf74aa Mon Sep 17 00:00:00 2001 From: memadi Date: Wed, 19 Aug 2026 15:36:30 -0700 Subject: [PATCH 20/30] nit Signed-off-by: memadi --- src/anonymizer/engine/detection/detection_workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/anonymizer/engine/detection/detection_workflow.py b/src/anonymizer/engine/detection/detection_workflow.py index b146e22d..05a8e6a6 100644 --- a/src/anonymizer/engine/detection/detection_workflow.py +++ b/src/anonymizer/engine/detection/detection_workflow.py @@ -503,7 +503,7 @@ def _materialize_final_entities( raw: object, *, allowed_labels: set[str] | None, - excluded_entity_labels: set[str] | None, + excluded_entity_labels: set[str] | None = None, ) -> dict: """Build COL_FINAL_ENTITIES, applying the configured label scope.""" parsed = EntitiesSchema.from_raw(raw) From 91ed207516cceb5c35e14145b80425f59354e0f8 Mon Sep 17 00:00:00 2001 From: memadi Date: Wed, 19 Aug 2026 16:34:37 -0700 Subject: [PATCH 21/30] normalize excluded labels in workflow filters Signed-off-by: memadi --- .../engine/detection/detection_workflow.py | 10 +++++----- tests/engine/test_detection_workflow.py | 14 +++++++------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/anonymizer/engine/detection/detection_workflow.py b/src/anonymizer/engine/detection/detection_workflow.py index 05a8e6a6..9b9e2b31 100644 --- a/src/anonymizer/engine/detection/detection_workflow.py +++ b/src/anonymizer/engine/detection/detection_workflow.py @@ -490,8 +490,8 @@ def _resolve_detection_labels( ) -> list[str]: labels = list(DEFAULT_ENTITY_LABELS) if entity_labels is None else list(entity_labels) if excluded_entity_labels: - excluded = {label.casefold() for label in excluded_entity_labels} - labels = [label for label in labels if label.casefold() not in excluded] + excluded = {label.strip().casefold() for label in excluded_entity_labels} + labels = [label for label in labels if label.strip().casefold() not in excluded] if not labels: logger.warning( "excluded_entity_labels removed all labels from the effective detection set. No entities will be detected." @@ -507,8 +507,8 @@ def _materialize_final_entities( ) -> dict: """Build COL_FINAL_ENTITIES, applying the configured label scope.""" parsed = EntitiesSchema.from_raw(raw) - allowed = {label.casefold() for label in allowed_labels} if allowed_labels is not None else None - excluded = {label.casefold() for label in excluded_entity_labels or []} + allowed = {label.strip().casefold() for label in allowed_labels} if allowed_labels is not None else None + excluded = {label.strip().casefold() for label in excluded_entity_labels or []} kept = [ e for e in parsed.entities @@ -519,7 +519,7 @@ def _materialize_final_entities( def _filter_excluded_latent_entities(raw: object, excluded_entity_labels: list[str] | None) -> object: """Remove excluded latent labels while preserving the structured payload shape.""" - excluded = {label.casefold() for label in excluded_entity_labels or []} + excluded = {label.strip().casefold() for label in excluded_entity_labels or []} if not excluded: return raw diff --git a/tests/engine/test_detection_workflow.py b/tests/engine/test_detection_workflow.py index 2123458f..dc4cc314 100644 --- a/tests/engine/test_detection_workflow.py +++ b/tests/engine/test_detection_workflow.py @@ -169,14 +169,14 @@ def test_latent_prompt_excludes_configured_labels() -> None: assert "Do NOT return latent entities with these labels: health_condition, occupation." in prompt -def test_filter_excluded_latent_entities_is_case_insensitive() -> None: +def test_filter_excluded_latent_entities_normalizes_configured_labels() -> None: raw = { "latent_entities": [ {"label": "Health_Condition", "value": "diabetes"}, {"label": "employer", "value": "Acme"}, ] } - result = _filter_excluded_latent_entities(raw, ["health_condition"]) + result = _filter_excluded_latent_entities(raw, [" HEALTH_CONDITION "]) assert result == {"latent_entities": [{"label": "employer", "value": "Acme"}]} @@ -535,8 +535,8 @@ def test_resolve_detection_labels_exclusions_remove_labels() -> None: assert "city" in labels -def test_resolve_detection_labels_exclusions_are_case_insensitive() -> None: - labels = _resolve_detection_labels(["first_name", "Email"], excluded_entity_labels={"EMAIL"}) +def test_resolve_detection_labels_exclusions_normalize_configured_labels() -> None: + labels = _resolve_detection_labels(["first_name", " Email "], excluded_entity_labels={" EMAIL "}) assert labels == ["first_name"] @@ -559,7 +559,7 @@ def test_resolve_detection_labels_empty_result_warns(caplog: pytest.LogCaptureFi assert "No entities will be detected" in caplog.text -def test_materialize_final_entities_applies_label_filters_case_insensitively() -> None: +def test_materialize_final_entities_normalizes_configured_labels() -> None: raw = { "entities": [ {"value": "Alice", "label": "First_Name", "start_position": 0, "end_position": 5}, @@ -570,8 +570,8 @@ def test_materialize_final_entities_applies_label_filters_case_insensitively() - result = _materialize_final_entities( raw, - allowed_labels={"first_name", "email"}, - excluded_entity_labels={"EMAIL"}, + allowed_labels={" first_name ", " email "}, + excluded_entity_labels={" EMAIL "}, ) final = EntitiesSchema.from_raw(result) From ca639786b84655778deb737edcc8ac8150df095f Mon Sep 17 00:00:00 2001 From: nvskills-svc-account Date: Thu, 20 Aug 2026 15:42:02 +0000 Subject: [PATCH 22/30] Attach NVSkills validation signatures Signed-off-by: nvskills-svc-account --- skills/anonymizer/BENCHMARK.md | 31 +++++++++++++++------- skills/anonymizer/skill-card.md | 46 +++++++++++++++++---------------- skills/anonymizer/skill.oms.sig | 2 +- 3 files changed, 46 insertions(+), 33 deletions(-) diff --git a/skills/anonymizer/BENCHMARK.md b/skills/anonymizer/BENCHMARK.md index f592adb7..ab281a63 100644 --- a/skills/anonymizer/BENCHMARK.md +++ b/skills/anonymizer/BENCHMARK.md @@ -9,16 +9,27 @@ Recommended for publication based on the completed evaluation evidence in this r ## Evaluation Metadata - Skill: `anonymizer` -- Evaluation date: 2026-08-12 -- Evaluator version: `1.2.4` +- Evaluation date: 2026-08-20 +- Evaluator version: `1.3.2` - Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`), Codex (`openai/openai/gpt-5.5`) - Tasks: 6 evaluation tasks (4 positive, 2 negative) - Dataset digest: `sha256:c2c13b2d794c6117dac0402f1261bd2d80972085c5e716426bedff6b3d59b8ae` (skill-evaluator-dataset-snapshot/1) - Attempts per task: 1 -- Environment: `k8s-sandbox` +- Environment: `local` - Tier 3 evidence: required for publication -Each task attempt ran in its own isolated sandbox pod. +Tasks ran on the trusted local host; local mode is not sandboxed. + +## Execution and Provenance + +- Validation status: `passed` +- Report generation: `complete` +- Evaluator version: `1.3.2` +- Git commit: `0117bc2e3e54da4244a656466526c5b1b5a559ea` +- Content type: requested `auto`, detected `skill` +- Container image: `gitlab-master.nvidia.com:5005/nvcarps/ci-group/nvcarps-ci/skillevaluator-ci:sha-0117bc2e3e54da4244a656466526c5b1b5a559ea` +- Container image digest: `not recorded` +- Tier 3: requested `true`, executed `true`, status `succeeded` ## What This Report Answers @@ -34,12 +45,12 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 58% → 94% (+36 points) | 68% → 93% (+25 points) | -| Security | 100% → 92% (-8 points) | 83% → 83% (±0 points) | -| Correctness | 50% → 100% (+50 points) | 83% → 97% (+13 points) | -| Discoverability | 50% → 99% (+49 points) | 67% → 94% (+27 points) | -| Effectiveness | 52% → 89% (+38 points) | 65% → 93% (+28 points) | -| Efficiency | 38% → 89% (+51 points) | 42% → 97% (+55 points) | +| Overall | 65% → 95% (+30 points) | 72% → 93% (+21 points) | +| Security | 100% → 100% (±0 points) | 83% → 83% (±0 points) | +| Correctness | 73% → 97% (+23 points) | 90% → 100% (+10 points) | +| Discoverability | 50% → 98% (+48 points) | 67% → 92% (+25 points) | +| Effectiveness | 54% → 87% (+33 points) | 69% → 92% (+23 points) | +| Efficiency | 50% → 94% (+44 points) | 50% → 100% (+50 points) | **How to read this table:** baseline is the same task attempted without the target skill. Uplift is `skill score - baseline score`, shown in percentage points. diff --git a/skills/anonymizer/skill-card.md b/skills/anonymizer/skill-card.md index f1b2ddaa..7bca132a 100644 --- a/skills/anonymizer/skill-card.md +++ b/skills/anonymizer/skill-card.md @@ -9,7 +9,7 @@ NVIDIA
### License/Terms of Use:
Apache 2.0
## Use Case:
-Developers and engineers who need to anonymize text datasets, redact PII, de-identify free-text data, or rewrite text to remove sensitive or inferable personal information for privacy compliance.
+Developers and data engineers who need to anonymize text datasets containing PII for downstream model training, analytics, or data sharing.
### Deployment Geography for Use:
Global
@@ -25,20 +25,22 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [NeMo Anonymizer Documentation](https://nvidia-nemo.github.io/Anonymizer/)
-- [Choosing a Strategy](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/choosing-a-strategy/)
-- [Detection Concepts](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/detection/)
-- [Evaluation Concepts](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/evaluation/)
-- [Self-hosting GLiNER](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/self-hosting-gliner/)
+- [Interactive workflow reference](references/interactive.md)
+- [NeMo Anonymizer documentation](https://nvidia-nemo.github.io/Anonymizer/)
+- [Choosing a strategy](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/choosing-a-strategy/)
+- [Detection guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/detection/)
+- [Evaluation guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/evaluation/)
+- [Models guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/models/)
- [Troubleshooting](https://nvidia-nemo.github.io/Anonymizer/dev/troubleshooting/)
-- [GitHub Repository](https://github.com/NVIDIA-NeMo/Anonymizer.git)
+- [Self-hosting GLiNER](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/self-hosting-gliner/)
+- [GitHub repository](https://github.com/NVIDIA-NeMo/Anonymizer.git)
## Skill Output:
**Output Type(s):** [Code]
**Output Format:** [Python script]
**Output Parameters:** [1D]
-**Other Properties Related to Output:** [None]
+**Other Properties Related to Output:** [Runnable script with CLI flags for preview, full run, and evaluation]
## Evaluation Agents Used:
- Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`)
@@ -47,18 +49,18 @@ Mitigation: Review and scan skill before deployment.
## Evaluation Tasks:
-6 evaluation tasks (4 positive, 2 negative) run in isolated sandbox pods with 1 attempt per task.
+6 evaluation tasks (4 positive, 2 negative) from skill-evaluator-dataset-snapshot/1.
## Evaluation Metrics Used:
Reported benchmark dimensions:
-- Security: Checks for unsafe operations, secret leakage, and unauthorized access.
-- Correctness: Checks final-answer correctness against the reference answer.
-- Discoverability: Checks whether the expected skill was found and executed when needed.
-- Effectiveness: Checks goal completion (50%) and expected workflow adherence (50%).
-- Efficiency: Checks routing quality, workspace-aware skill reads, and productive tool use.
+- Security: Whether the skill avoids unsafe operations, secret leakage, and unauthorized access.
+- Correctness: Whether the final answer is correct against the reference answer.
+- Discoverability: Whether the right skill is found and executed when needed.
+- Effectiveness: Whether the skill helps complete the user's goal (goal completion + expected workflow adherence).
+- Efficiency: Whether the skill avoids wasted tool or skill usage (routing quality and productive tool use).
Underlying evaluation signals used in this run:
-- `security`: Unsafe operations, secret leakage, and unauthorized access.
+- `security`: Checks for unsafe operations, secret leakage, and unauthorized access.
- `skill_execution`: Whether the expected skill was found and executed.
- `skill_efficiency`: Routing quality, workspace-aware skill reads, and productive tool use.
- `accuracy`: Final-answer correctness against the reference answer.
@@ -70,15 +72,15 @@ Underlying evaluation signals used in this run:
## Evaluation Results:
| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 58% → 94% (+36 points) | 68% → 93% (+25 points) | -| Security | 100% → 92% (-8 points) | 83% → 83% (±0 points) | -| Correctness | 50% → 100% (+50 points) | 83% → 97% (+13 points) | -| Discoverability | 50% → 99% (+49 points) | 67% → 94% (+27 points) | -| Effectiveness | 52% → 89% (+38 points) | 65% → 93% (+28 points) | -| Efficiency | 38% → 89% (+51 points) | 42% → 97% (+55 points) | +| Overall | 65% → 95% (+30 points) | 72% → 93% (+21 points) | +| Security | 100% → 100% (±0 points) | 83% → 83% (±0 points) | +| Correctness | 73% → 97% (+23 points) | 90% → 100% (+10 points) | +| Discoverability | 50% → 98% (+48 points) | 67% → 92% (+25 points) | +| Effectiveness | 54% → 87% (+33 points) | 69% → 92% (+23 points) | +| Efficiency | 50% → 94% (+44 points) | 50% → 100% (+50 points) | ## Skill Version(s):
-e3b99da (source: git SHA, committed 2026-08-11)
+318cc15 (source: git SHA, committed 2026-08-19)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/anonymizer/skill.oms.sig b/skills/anonymizer/skill.oms.sig index 76a40d0f..119a021c 100644 --- a/skills/anonymizer/skill.oms.sig +++ b/skills/anonymizer/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiYW5vbnltaXplciIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICI2OWFmZTdiMzlkYjdmZmYyNzEzZTJiZDgwZmI0MTZkYzcwOTY1NjMzYWU3ZDRiMGE0YWQzZmJkNjA2MDc3YjA4IgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdCIsCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXRpZ25vcmUiCiAgICAgIF0sCiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiCiAgICB9LAogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIsCiAgICAgICAgImRpZ2VzdCI6ICI5ZjJmMDU4ZGQ3M2U3ZmRiNWNiZjkyN2M1NDcyZjJhNGE5NTE5NWU1ZWU5OTZjZjBjYTUxNTM5YTdmMDY4YzdjIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIiwKICAgICAgICAiZGlnZXN0IjogImFiNThiYzY4YjQzZTZkMTAzNzk5OGQxMDNkNzlhYTVkM2VmNjUyMWRhY2FiOWYyMzY3YzQ1YWIxMDBiZTIwODQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIsCiAgICAgICAgImRpZ2VzdCI6ICJjYTQ3YjgyZGMzMTJmYzY0MDc3MGJmNjczM2JhNDYyNGRjYzhmNzA4MDdlZDM2ZjczZTllZTUwYTFkNDdlMGMyIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvaW50ZXJhY3RpdmUubWQiLAogICAgICAgICJkaWdlc3QiOiAiZDc1NWFhNDU3NDA3ZTM5MDE1YzcxMWEwY2I4MDI4MmRlZTkyNzZhYWJiYzRhMTYzM2YxZDUzZDExMGUzM2YwOCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiZGlnZXN0IjogImYwMDNlZTUyODcwZmJhZTg3NjVmYWIxN2E5ODc4MmEzNDIyYjY0ODkzNmY0NWZmM2JmOTUwNzUwYWZiYjkwOGMiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9CiAgICBdCiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQDvD8g+PIjUDUKUdto3XmHbHJx3SJ/yC8BJRlglJMKI9yMrgUSeWNWkjVj5RF6j1i8CMQDvIThge9dCTqmL2Z7U406/AfDeWclej4gysAh/UKPkwmFCZV/RaWRWTIU+TGbOHgI=","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiYW5vbnltaXplciIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICJlN2FmOWQ3MzM2NWU4MzU3ZDQxNDdiZjdlODQ1OTMyMjk5YTYyNmQ3Nzk4ZDlhZjA1NTEzNjUxMzBkZDkwMjk0IgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGh1YiIsCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXQiCiAgICAgIF0sCiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IiwKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIsCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlCiAgICB9LAogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiYjlhNmRhNmVjNmZiZTZhNjNlZWFiODIxMDcyYWRmZjkxNGQ2ZjZmNDUxNzlkNWFhY2Q3Y2ZiZjE5M2EzMzNhNiIsCiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiN2ZjOTQ2NDlhN2ZkMjBiZTBmNDUzNDQzMmQ2MjFhYTliYzBlYmYxYWJlOTJkNDJhNDQ1YzI5ZDQ2MGM2NWE4MSIsCiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJjYTQ3YjgyZGMzMTJmYzY0MDc3MGJmNjczM2JhNDYyNGRjYzhmNzA4MDdlZDM2ZjczZTllZTUwYTFkNDdlMGMyIiwKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiZDc1NWFhNDU3NDA3ZTM5MDE1YzcxMWEwY2I4MDI4MmRlZTkyNzZhYWJiYzRhMTYzM2YxZDUzZDExMGUzM2YwOCIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9pbnRlcmFjdGl2ZS5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjkxYTcxM2U5NWI5YzgwOGM2YWQyZDgzNjlmY2UyYTBmMzk1ZjMzMTcwMTVhNWI4NGY4MzUxNTVjNmQ0MTZhY2IiLAogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9CiAgICBdCiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQCMaSVpacY57xcRHqNUKyt4gV+ODfkD3GIaKfW316ZDIUmnF+7uwFCbxheCDvKDo4ACMQDecoAhlXnOmV8ca3qaA+HktQuA8yZzuAOflyHy+UVgZTKjk897md3R2vyz2KZxqAw=","keyid":""}]}} \ No newline at end of file From fe5aee92dee5d528f1b57897c275c36c8757c22f Mon Sep 17 00:00:00 2001 From: memadi Date: Mon, 24 Aug 2026 12:09:16 -0700 Subject: [PATCH 23/30] add filter after entity validator Signed-off-by: memadi --- .../engine/detection/custom_columns.py | 15 ++- .../engine/detection/detection_workflow.py | 2 + .../engine/detection/postprocess.py | 14 ++- .../engine/workflow_columns/detection/impl.py | 10 ++ .../test_detection_config_serialization.py | 13 +++ tests/engine/test_detection_custom_columns.py | 104 ++++++++++++++++++ 6 files changed, 155 insertions(+), 3 deletions(-) diff --git a/src/anonymizer/engine/detection/custom_columns.py b/src/anonymizer/engine/detection/custom_columns.py index 7127356e..b58ba0a8 100644 --- a/src/anonymizer/engine/detection/custom_columns.py +++ b/src/anonymizer/engine/detection/custom_columns.py @@ -42,6 +42,7 @@ build_tagged_text, build_validation_candidates, expand_entity_occurrences, + filter_excluded_entity_spans, get_tag_notation, parse_raw_entities, ) @@ -107,7 +108,11 @@ def merge_and_build_candidates( required_columns=[COL_TEXT, COL_SEED_ENTITIES, COL_VALIDATED_ENTITIES], side_effect_columns=[COL_INITIAL_TAGGED_TEXT, COL_SEED_ENTITIES_JSON, COL_VALIDATED_SEED_ENTITIES], ) -def apply_validation_to_seed_entities(row: dict[str, Any]) -> dict[str, Any]: +def apply_validation_to_seed_entities( + row: dict[str, Any], + *, + excluded_entity_labels: list[str] | None = None, +) -> dict[str, Any]: """Apply validation decisions to detector entities before augmentation.""" text = str(row.get(COL_TEXT, "")) seed_spans = _parse_entity_spans(row.get(COL_SEED_ENTITIES, {})) @@ -115,6 +120,7 @@ def apply_validation_to_seed_entities(row: dict[str, Any]) -> dict[str, Any]: entities=seed_spans, validation_output=row.get(COL_VALIDATED_ENTITIES, {}), ) + validated_seed = filter_excluded_entity_spans(validated_seed, excluded_entity_labels) seed_entities = [entity.as_dict() for entity in validated_seed] row[COL_VALIDATED_SEED_ENTITIES] = EntitiesSchema(entities=seed_entities).model_dump(mode="json") row[COL_SEED_ENTITIES_JSON] = json.dumps(seed_entities) @@ -169,7 +175,11 @@ def enrich_validation_decisions(row: dict[str, Any]) -> dict[str, Any]: required_columns=[COL_TEXT, COL_MERGED_ENTITIES, COL_VALIDATED_ENTITIES], side_effect_columns=[COL_TAGGED_TEXT], ) -def apply_validation_and_finalize(row: dict[str, Any]) -> dict[str, Any]: +def apply_validation_and_finalize( + row: dict[str, Any], + *, + excluded_entity_labels: list[str] | None = None, +) -> dict[str, Any]: """Apply keep/reclass/drop decisions, expand to all occurrences, and produce final outputs.""" text = str(row.get(COL_TEXT, "")) merged = _parse_entity_spans(row.get(COL_MERGED_ENTITIES, {})) @@ -177,6 +187,7 @@ def apply_validation_and_finalize(row: dict[str, Any]) -> dict[str, Any]: entities=merged, validation_output=row.get(COL_VALIDATED_ENTITIES, {}), ) + validated = filter_excluded_entity_spans(validated, excluded_entity_labels) expanded = expand_entity_occurrences(text=text, entities=validated) row[COL_DETECTED_ENTITIES] = EntitiesSchema(entities=[entity.as_dict() for entity in expanded]).model_dump( mode="json" diff --git a/src/anonymizer/engine/detection/detection_workflow.py b/src/anonymizer/engine/detection/detection_workflow.py index 9b9e2b31..31e08c91 100644 --- a/src/anonymizer/engine/detection/detection_workflow.py +++ b/src/anonymizer/engine/detection/detection_workflow.py @@ -213,6 +213,7 @@ def _build_detection_spec( DetectionTransformConfig( name=COL_SEED_ENTITIES_JSON, operation=DetectionTransformOperation.APPLY_VALIDATION_TO_SEED_ENTITIES, + excluded_entity_labels=list(excluded_entity_labels or []), ), LLMStructuredColumnConfig( name=COL_AUGMENTED_ENTITIES, @@ -230,6 +231,7 @@ def _build_detection_spec( DetectionTransformConfig( name=COL_DETECTED_ENTITIES, operation=DetectionTransformOperation.APPLY_VALIDATION_AND_FINALIZE, + excluded_entity_labels=list(excluded_entity_labels or []), ), ], ) diff --git a/src/anonymizer/engine/detection/postprocess.py b/src/anonymizer/engine/detection/postprocess.py index aeae876c..1c8d70e6 100644 --- a/src/anonymizer/engine/detection/postprocess.py +++ b/src/anonymizer/engine/detection/postprocess.py @@ -6,6 +6,7 @@ import json import logging import re +from collections.abc import Iterable from dataclasses import dataclass from enum import Enum from typing import SupportsFloat, SupportsIndex, SupportsInt @@ -39,6 +40,17 @@ def as_dict(self) -> dict[str, str | int | float]: } +def filter_excluded_entity_spans( + entities: list[EntitySpan], + excluded_entity_labels: Iterable[str] | None, +) -> list[EntitySpan]: + """Remove entity spans whose normalized labels are explicitly excluded.""" + excluded = {label.strip().casefold() for label in excluded_entity_labels or []} + if not excluded: + return list(entities) + return [entity for entity in entities if entity.label.strip().casefold() not in excluded] + + class TagNotation(str, Enum): xml = "xml" bracket = "bracket" @@ -169,7 +181,7 @@ def apply_augmented_entities( augmented = [] excluded = {label.strip().casefold() for label in excluded_entity_labels or set()} - merged = list(entities) + merged = filter_excluded_entity_spans(entities, excluded) for idx, suggestion in enumerate(augmented): if not isinstance(suggestion, dict): continue diff --git a/src/anonymizer/engine/workflow_columns/detection/impl.py b/src/anonymizer/engine/workflow_columns/detection/impl.py index 533d6ff8..5af1e855 100644 --- a/src/anonymizer/engine/workflow_columns/detection/impl.py +++ b/src/anonymizer/engine/workflow_columns/detection/impl.py @@ -100,11 +100,21 @@ def __getattr__(self, name: str) -> Any: class DetectionTransformGenerator(ColumnGeneratorCellByCell[DetectionTransformConfig]): def generate(self, data: dict[str, Any]) -> dict[str, Any]: operation = DetectionTransformOperation(self.config.operation) + if operation == DetectionTransformOperation.APPLY_VALIDATION_TO_SEED_ENTITIES: + return apply_validation_to_seed_entities( + data, + excluded_entity_labels=self.config.excluded_entity_labels, + ) if operation == DetectionTransformOperation.MERGE_AND_BUILD_CANDIDATES: return merge_and_build_candidates( data, excluded_entity_labels=self.config.excluded_entity_labels, ) + if operation == DetectionTransformOperation.APPLY_VALIDATION_AND_FINALIZE: + return apply_validation_and_finalize( + data, + excluded_entity_labels=self.config.excluded_entity_labels, + ) return _TRANSFORMS[operation](data) diff --git a/tests/engine/test_detection_config_serialization.py b/tests/engine/test_detection_config_serialization.py index dd8c83ea..3db453dc 100644 --- a/tests/engine/test_detection_config_serialization.py +++ b/tests/engine/test_detection_config_serialization.py @@ -83,7 +83,20 @@ def test_detection_builder_round_trips_through_native_data_designer_config(tmp_p for column in transforms if DetectionTransformOperation(column.operation) == DetectionTransformOperation.MERGE_AND_BUILD_CANDIDATES ) + seed_validation_transform = next( + column + for column in transforms + if DetectionTransformOperation(column.operation) + == DetectionTransformOperation.APPLY_VALIDATION_TO_SEED_ENTITIES + ) + finalize_transform = next( + column + for column in transforms + if DetectionTransformOperation(column.operation) == DetectionTransformOperation.APPLY_VALIDATION_AND_FINALIZE + ) + assert seed_validation_transform.excluded_entity_labels == ["email"] assert merge_transform.excluded_entity_labels == ["email"] + assert finalize_transform.excluded_entity_labels == ["email"] validation = next(column for column in columns if column.name == COL_VALIDATION_DECISIONS) assert isinstance(validation, ChunkedValidationConfig) diff --git a/tests/engine/test_detection_custom_columns.py b/tests/engine/test_detection_custom_columns.py index 84395f5b..a515f2c2 100644 --- a/tests/engine/test_detection_custom_columns.py +++ b/tests/engine/test_detection_custom_columns.py @@ -16,11 +16,15 @@ from anonymizer.engine.constants import ( COL_AUGMENTED_ENTITIES, COL_DETECTED_ENTITIES, + COL_INITIAL_TAGGED_TEXT, COL_MERGED_ENTITIES, + COL_MERGED_TAGGED_TEXT, COL_RAW_DETECTED, COL_SEED_ENTITIES, + COL_SEED_ENTITIES_JSON, COL_SEED_VALIDATION_CANDIDATES, COL_TAG_NOTATION, + COL_TAGGED_TEXT, COL_TEXT, COL_VALIDATED_ENTITIES, COL_VALIDATED_SEED_ENTITIES, @@ -30,6 +34,7 @@ from anonymizer.engine.detection.custom_columns import ( _parse_entity_spans, apply_validation_and_finalize, + apply_validation_to_seed_entities, enrich_validation_decisions, merge_and_build_candidates, parse_detected_entities, @@ -132,6 +137,105 @@ def test_merge_filters_denied_augmentation_before_overlap_resolution() -> None: assert [(entity["value"], entity["label"]) for entity in merged] == [("Alice", "first_name")] +def test_validation_reclassification_to_excluded_label_is_filtered_before_augmentation() -> None: + row: dict[str, Any] = { + COL_TEXT: "San Diego", + COL_SEED_ENTITIES: { + "entities": [ + { + "id": "country_0_9", + "value": "San Diego", + "label": "country", + "start_position": 0, + "end_position": 9, + "score": 0.95, + "source": "detector", + } + ] + }, + COL_VALIDATED_ENTITIES: { + "decisions": [ + { + "id": "country_0_9", + "value": "San Diego", + "label": "country", + "decision": "reclass", + "proposed_label": "city", + "reason": "San Diego is a city", + } + ] + }, + } + + result = apply_validation_to_seed_entities(row, excluded_entity_labels=[" CITY "]) + + assert result[COL_VALIDATED_SEED_ENTITIES]["entities"] == [] + assert json.loads(result[COL_SEED_ENTITIES_JSON]) == [] + assert result[COL_INITIAL_TAGGED_TEXT] == "San Diego" + + +def test_merge_filters_excluded_validated_seed_entities() -> None: + row: dict[str, Any] = { + COL_TEXT: "San Diego", + COL_VALIDATED_SEED_ENTITIES: { + "entities": [ + { + "id": "country_0_9", + "value": "San Diego", + "label": " City ", + "start_position": 0, + "end_position": 9, + "score": 0.95, + "source": "detector", + } + ] + }, + COL_AUGMENTED_ENTITIES: {"entities": []}, + } + + result = merge_and_build_candidates(row, excluded_entity_labels=["city"]) + + assert result[COL_MERGED_ENTITIES]["entities"] == [] + assert result[COL_VALIDATION_CANDIDATES]["candidates"] == [] + assert result[COL_MERGED_TAGGED_TEXT] == "San Diego" + + +def test_finalize_filters_reclassification_to_excluded_label() -> None: + row: dict[str, Any] = { + COL_TEXT: "San Diego", + COL_MERGED_ENTITIES: { + "entities": [ + { + "id": "country_0_9", + "value": "San Diego", + "label": "country", + "start_position": 0, + "end_position": 9, + "score": 0.95, + "source": "augmenter", + } + ] + }, + COL_VALIDATED_ENTITIES: { + "decisions": [ + { + "id": "country_0_9", + "value": "San Diego", + "label": "country", + "decision": "reclass", + "proposed_label": "city", + "reason": "San Diego is a city", + } + ] + }, + } + + result = apply_validation_and_finalize(row, excluded_entity_labels=["city"]) + + assert result[COL_DETECTED_ENTITIES]["entities"] == [] + assert result[COL_TAGGED_TEXT] == "San Diego" + + def test_enrich_validation_decisions_adds_value_from_candidates() -> None: row = { COL_VALIDATION_DECISIONS: { From baafb80ff83038b24df239e7e380b200ef1c618e Mon Sep 17 00:00:00 2001 From: memadi Date: Wed, 9 Sep 2026 11:30:06 -0700 Subject: [PATCH 24/30] fix: handle JSON-string COL_LATENT_ENTITIES payload in exclusion filter _filter_excluded_latent_entities only handled schema/dict/list shapes, so a JSON-string payload (the shape every other schema's from_raw() already anticipates via _parse_raw_wrapper) fell through unfiltered, letting excluded latent labels leak into the rewrite prompt. Also fixes a missing blank line that broke the MkDocs warning admonition in docs/concepts/detection.md. Co-Authored-By: Claude Sonnet 5 --- docs/concepts/detection.md | 1 + .../engine/detection/detection_workflow.py | 8 ++++++++ tests/engine/test_detection_workflow.py | 13 +++++++++++++ 3 files changed, 22 insertions(+) diff --git a/docs/concepts/detection.md b/docs/concepts/detection.md index bd1e2e02..901a6250 100644 --- a/docs/concepts/detection.md +++ b/docs/concepts/detection.md @@ -120,6 +120,7 @@ Detect(entity_labels=["first_name", "email", "city"], excluded_entity_labels=["c !!! warning If every label in `entity_labels` is also in `excluded_entity_labels`, the effective detection set is empty and no entities will be detected. Anonymizer logs a warning when this happens. + ## Tuning the threshold For `gliner_threshold`, start with the default `0.3`. If you're seeing too many false positives, raise it to `0.5`. If entities are being missed, try lowering to `0.2`. The LLM validation step catches many false positives, so erring on the side of lower thresholds is usually safe. diff --git a/src/anonymizer/engine/detection/detection_workflow.py b/src/anonymizer/engine/detection/detection_workflow.py index 31e08c91..05ee3732 100644 --- a/src/anonymizer/engine/detection/detection_workflow.py +++ b/src/anonymizer/engine/detection/detection_workflow.py @@ -3,6 +3,7 @@ from __future__ import annotations +import json import logging from copy import deepcopy from dataclasses import dataclass @@ -529,6 +530,13 @@ def _filter_excluded_latent_entities(raw: object, excluded_entity_labels: list[s kept = [entity for entity in raw.latent_entities if entity.label.strip().casefold() not in excluded] return LatentEntitiesSchema(latent_entities=kept).model_dump(mode="json") + if isinstance(raw, str): + try: + parsed = json.loads(raw) + except (json.JSONDecodeError, ValueError): + return raw + return _filter_excluded_latent_entities(parsed, excluded_entity_labels) + if isinstance(raw, dict): entities = raw.get("latent_entities") if not isinstance(entities, list): diff --git a/tests/engine/test_detection_workflow.py b/tests/engine/test_detection_workflow.py index dc4cc314..2291ba85 100644 --- a/tests/engine/test_detection_workflow.py +++ b/tests/engine/test_detection_workflow.py @@ -180,6 +180,19 @@ def test_filter_excluded_latent_entities_normalizes_configured_labels() -> None: assert result == {"latent_entities": [{"label": "employer", "value": "Acme"}]} +def test_filter_excluded_latent_entities_handles_json_string_payload() -> None: + raw = json.dumps( + { + "latent_entities": [ + {"label": "Health_Condition", "value": "diabetes"}, + {"label": "employer", "value": "Acme"}, + ] + } + ) + result = _filter_excluded_latent_entities(raw, [" HEALTH_CONDITION "]) + assert result == {"latent_entities": [{"label": "employer", "value": "Acme"}]} + + def test_identify_latent_entities_filters_excluded_labels( stub_detector_model_configs: list[ModelConfig], stub_detection_model_selection: DetectionModelSelection, From a16c9b3a50a7f59df0c3db1eaae0efb380e7c7c2 Mon Sep 17 00:00:00 2001 From: memadi Date: Wed, 9 Sep 2026 11:37:41 -0700 Subject: [PATCH 25/30] refactor: extract shared label normalization helper The strip+casefold idiom for comparing/filtering entity labels was independently reimplemented in ~8 places across detection_workflow.py, postprocess.py, and entity_coverage_judge.py, with at least one site (entity_coverage_judge._effective_entity_labels) silently missing the .strip() step. Consolidate into normalize_label()/normalize_labels() in postprocess.py so the normalization rule only needs to change once. Co-Authored-By: Claude Sonnet 5 --- .../engine/detection/detection_workflow.py | 25 +++++++++++-------- .../engine/detection/postprocess.py | 18 ++++++++++--- .../evaluation/entity_coverage_judge.py | 13 +++++----- tests/engine/test_detection_postprocess.py | 14 +++++++++++ 4 files changed, 50 insertions(+), 20 deletions(-) diff --git a/src/anonymizer/engine/detection/detection_workflow.py b/src/anonymizer/engine/detection/detection_workflow.py index 05ee3732..13edc921 100644 --- a/src/anonymizer/engine/detection/detection_workflow.py +++ b/src/anonymizer/engine/detection/detection_workflow.py @@ -42,7 +42,12 @@ ENTITY_LABEL_EXAMPLES, _jinja, ) -from anonymizer.engine.detection.postprocess import EntitySpan, group_entities_by_value +from anonymizer.engine.detection.postprocess import ( + EntitySpan, + group_entities_by_value, + normalize_label, + normalize_labels, +) from anonymizer.engine.ndd.adapter import FailedRecord, NddAdapter from anonymizer.engine.ndd.model_loader import resolve_model_alias, resolve_model_aliases from anonymizer.engine.prompt_utils import substitute_placeholders @@ -493,8 +498,8 @@ def _resolve_detection_labels( ) -> list[str]: labels = list(DEFAULT_ENTITY_LABELS) if entity_labels is None else list(entity_labels) if excluded_entity_labels: - excluded = {label.strip().casefold() for label in excluded_entity_labels} - labels = [label for label in labels if label.strip().casefold() not in excluded] + excluded = normalize_labels(excluded_entity_labels) + labels = [label for label in labels if normalize_label(label) not in excluded] if not labels: logger.warning( "excluded_entity_labels removed all labels from the effective detection set. No entities will be detected." @@ -510,24 +515,24 @@ def _materialize_final_entities( ) -> dict: """Build COL_FINAL_ENTITIES, applying the configured label scope.""" parsed = EntitiesSchema.from_raw(raw) - allowed = {label.strip().casefold() for label in allowed_labels} if allowed_labels is not None else None - excluded = {label.strip().casefold() for label in excluded_entity_labels or []} + allowed = normalize_labels(allowed_labels) if allowed_labels is not None else None + excluded = normalize_labels(excluded_entity_labels) kept = [ e for e in parsed.entities - if (allowed is None or e.label.strip().casefold() in allowed) and e.label.strip().casefold() not in excluded + if (allowed is None or normalize_label(e.label) in allowed) and normalize_label(e.label) not in excluded ] return EntitiesSchema(entities=kept).model_dump() def _filter_excluded_latent_entities(raw: object, excluded_entity_labels: list[str] | None) -> object: """Remove excluded latent labels while preserving the structured payload shape.""" - excluded = {label.strip().casefold() for label in excluded_entity_labels or []} + excluded = normalize_labels(excluded_entity_labels) if not excluded: return raw if isinstance(raw, LatentEntitiesSchema): - kept = [entity for entity in raw.latent_entities if entity.label.strip().casefold() not in excluded] + kept = [entity for entity in raw.latent_entities if normalize_label(entity.label) not in excluded] return LatentEntitiesSchema(latent_entities=kept).model_dump(mode="json") if isinstance(raw, str): @@ -546,7 +551,7 @@ def _filter_excluded_latent_entities(raw: object, excluded_entity_labels: list[s "latent_entities": [ entity for entity in entities - if not isinstance(entity, dict) or str(entity.get("label", "")).strip().casefold() not in excluded + if not isinstance(entity, dict) or normalize_label(str(entity.get("label", ""))) not in excluded ], } @@ -794,7 +799,7 @@ def _get_latent_prompt( ) -> str: summary_line = data_summary.strip() if data_summary else "Not provided" privacy_goal_text = _format_privacy_goal(privacy_goal) - excluded_labels = sorted({label.strip().casefold() for label in excluded_entity_labels or [] if label.strip()}) + excluded_labels = sorted(normalize_labels(excluded_entity_labels)) exclusion_block = ( "\n\n" f"Do NOT return latent entities with these labels: {', '.join(excluded_labels)}.\n" diff --git a/src/anonymizer/engine/detection/postprocess.py b/src/anonymizer/engine/detection/postprocess.py index 1c8d70e6..fd228245 100644 --- a/src/anonymizer/engine/detection/postprocess.py +++ b/src/anonymizer/engine/detection/postprocess.py @@ -40,15 +40,25 @@ def as_dict(self) -> dict[str, str | int | float]: } +def normalize_label(label: str) -> str: + """Canonical normalization for entity label comparisons: strip + casefold.""" + return label.strip().casefold() + + +def normalize_labels(labels: Iterable[str] | None) -> set[str]: + """Normalize a collection of labels, dropping empty/whitespace-only entries.""" + return {normalized for label in labels or [] if (normalized := normalize_label(label))} + + def filter_excluded_entity_spans( entities: list[EntitySpan], excluded_entity_labels: Iterable[str] | None, ) -> list[EntitySpan]: """Remove entity spans whose normalized labels are explicitly excluded.""" - excluded = {label.strip().casefold() for label in excluded_entity_labels or []} + excluded = normalize_labels(excluded_entity_labels) if not excluded: return list(entities) - return [entity for entity in entities if entity.label.strip().casefold() not in excluded] + return [entity for entity in entities if normalize_label(entity.label) not in excluded] class TagNotation(str, Enum): @@ -179,7 +189,7 @@ def apply_augmented_entities( augmented = payload.get("entities", []) if isinstance(payload, dict) else [] if not isinstance(augmented, list): augmented = [] - excluded = {label.strip().casefold() for label in excluded_entity_labels or set()} + excluded = normalize_labels(excluded_entity_labels) merged = filter_excluded_entity_spans(entities, excluded) for idx, suggestion in enumerate(augmented): @@ -187,7 +197,7 @@ def apply_augmented_entities( continue value = str(suggestion.get("value", "")).strip() label = str(suggestion.get("label", "")).strip() - if not value or not label or label.casefold() in excluded: + if not value or not label or normalize_label(label) in excluded: continue for start, end in _find_all_occurrences(text=text, needle=value): entity_id = _build_entity_id(label=label, start=start, end=end) diff --git a/src/anonymizer/engine/evaluation/entity_coverage_judge.py b/src/anonymizer/engine/evaluation/entity_coverage_judge.py index 36179c69..b32feb05 100644 --- a/src/anonymizer/engine/evaluation/entity_coverage_judge.py +++ b/src/anonymizer/engine/evaluation/entity_coverage_judge.py @@ -24,6 +24,7 @@ DEFAULT_ENTITY_LABELS, _jinja, ) +from anonymizer.engine.detection.postprocess import normalize_label, normalize_labels from anonymizer.engine.evaluation.judge_base import JudgeResult, _BaseJudgeWorkflow from anonymizer.engine.ndd.adapter import RECORD_ID_COLUMN, FailedRecord, NddAdapter from anonymizer.engine.ndd.model_loader import resolve_model_alias @@ -83,8 +84,8 @@ def _effective_entity_labels( return None if not excluded_entity_labels: return entity_labels - excluded = {label.casefold() for label in excluded_entity_labels} - effective = [label for label in entity_labels if label.casefold() not in excluded] + excluded = normalize_labels(excluded_entity_labels) + effective = [label for label in entity_labels if normalize_label(label) not in excluded] return effective @@ -93,7 +94,7 @@ def _entity_type_scope_block( excluded_entity_labels: list[str] | None = None, ) -> str: if entity_labels is None: - excluded = sorted({label.strip().casefold() for label in excluded_entity_labels or [] if label.strip()}) + excluded = sorted(normalize_labels(excluded_entity_labels)) exclusion = ( f"\nDo NOT report candidates with these explicitly excluded entity labels: {', '.join(excluded)}." if excluded @@ -409,14 +410,14 @@ def _filter_out_of_scope_entities( descriptions. The filter therefore drops genuine hallucinated labels without meaningfully risking false negatives on well-formed responses. """ - allowed = {label.casefold() for label in entity_labels} if entity_labels is not None else None - excluded = {label.casefold() for label in excluded_entity_labels or []} + allowed = normalize_labels(entity_labels) if entity_labels is not None else None + excluded = normalize_labels(excluded_entity_labels) result = [] for entity in entities: label = str(entity.get("label", "")).strip() if not label: continue - normalized_label = label.casefold() + normalized_label = normalize_label(label) if (allowed is not None and normalized_label not in allowed) or normalized_label in excluded: continue result.append(entity) diff --git a/tests/engine/test_detection_postprocess.py b/tests/engine/test_detection_postprocess.py index e6d7a659..754afa9e 100644 --- a/tests/engine/test_detection_postprocess.py +++ b/tests/engine/test_detection_postprocess.py @@ -17,11 +17,25 @@ expand_entity_occurrences, get_tag_notation, group_entities_by_value, + normalize_label, + normalize_labels, parse_raw_entities, resolve_overlaps, ) +def test_normalize_label_strips_and_casefolds() -> None: + assert normalize_label(" Health_Condition ") == "health_condition" + + +def test_normalize_labels_dedupes_and_drops_empty_entries() -> None: + assert normalize_labels([" Email ", "email", " ", "City"]) == {"email", "city"} + + +def test_normalize_labels_none_returns_empty_set() -> None: + assert normalize_labels(None) == set() + + def test_parse_raw_entities_parses_valid_spans() -> None: text = "Call me at (555) 123-4567" raw = '{"entities":[{"text":"(555) 123-4567","label":"phone_number","start":11,"end":25,"score":0.9}]}' From 52975e899052227c2e819a8ebc2277dada92bd66 Mon Sep 17 00:00:00 2001 From: memadi Date: Wed, 9 Sep 2026 14:34:04 -0700 Subject: [PATCH 26/30] feat(config): raise when excluded_entity_labels fully overlaps entity_labels Previously, entity_labels being entirely canceled out by excluded_entity_labels only logged a warning and silently produced a config that detects nothing at runtime. Fail early instead: raise a ValueError at config-construction time when excluded_entity_labels entirely overlaps an explicit entity_labels, leaving an empty effective detection set. Partial overlaps still only warn, and entity_labels=None (the default set) is unaffected by this check. Co-Authored-By: Claude Sonnet 5 --- docs/concepts/detection.md | 2 +- skills/anonymizer/SKILL.md | 2 +- src/anonymizer/config/anonymizer_config.py | 23 ++++++++++---- tests/config/test_anonymizer_config.py | 36 ++++++++++++++++++++++ 4 files changed, 55 insertions(+), 8 deletions(-) diff --git a/docs/concepts/detection.md b/docs/concepts/detection.md index 901a6250..55004a4a 100644 --- a/docs/concepts/detection.md +++ b/docs/concepts/detection.md @@ -119,7 +119,7 @@ Detect(entity_labels=["first_name", "email", "city"], excluded_entity_labels=["c ``` !!! warning - If every label in `entity_labels` is also in `excluded_entity_labels`, the effective detection set is empty and no entities will be detected. Anonymizer logs a warning when this happens. + If `entity_labels` and `excluded_entity_labels` partially overlap, the shared labels are dropped and Anonymizer logs a warning. If `excluded_entity_labels` entirely overlaps `entity_labels` — leaving an empty effective detection set — `Detect` raises a `ValueError` at config time instead of silently detecting nothing. This only applies when `entity_labels` is explicitly set; `entity_labels=None` (the default set) can never be fully excluded this way, since removing all default labels via `excluded_entity_labels` still only logs a warning. ## Tuning the threshold diff --git a/skills/anonymizer/SKILL.md b/skills/anonymizer/SKILL.md index f208148b..216d0506 100644 --- a/skills/anonymizer/SKILL.md +++ b/skills/anonymizer/SKILL.md @@ -43,7 +43,7 @@ regulatory and business context. # Usage Tips and Common Pitfalls - **`Detect.entity_labels=None` (the default) is permissive** — the augmenter LLM may invent labels not in `DEFAULT_ENTITY_LABELS`. Setting an explicit list switches to **strict mode** where *only* the listed labels are detected. To add domain labels, *extend* the default, don't replace it: `entity_labels=[*DEFAULT_ENTITY_LABELS, "clinical_facility", ...]` (`DEFAULT_ENTITY_LABELS` is a tuple, so unpack it into a list). Match the snake_case convention of `DEFAULT_ENTITY_LABELS`. -- **`Detect.excluded_entity_labels`** excludes specific label types from detection entirely — excluded labels are removed before GLiNER runs and are never detected, augmented, or penalised in evaluation scores. Use it when a label type is systematically noisy for your data or should never be anonymized (e.g. `Detect(excluded_entity_labels=["occupation", "gender"])`). Exclusions take precedence over `entity_labels` — a label in both is never detected. +- **`Detect.excluded_entity_labels`** excludes specific label types from detection entirely — excluded labels are removed before GLiNER runs and are never detected, augmented, or penalised in evaluation scores. Use it when a label type is systematically noisy for your data or should never be anonymized (e.g. `Detect(excluded_entity_labels=["occupation", "gender"])`). Exclusions take precedence over `entity_labels` — a label in both is never detected. If `excluded_entity_labels` entirely overlaps an explicit `entity_labels`, `Detect` raises a `ValueError` at config time instead of silently building a config that detects nothing. - **GLiNER is zero-shot** — entity labels are natural-language concept names (e.g. `"clinical_facility"`, `"internal_project_codename"`), not codes or enum values. Any concept you can name in English is a label GLiNER can detect. - **`Rewrite.instructions` is a dead field today** — it exists on the model but the rewrite engine never reads it. Do not use it. Put rewriter guidance in `privacy_goal.protect` / `privacy_goal.preserve` instead. - **`risk_tolerance` only applies to Rewrite mode**, not Replace. diff --git a/src/anonymizer/config/anonymizer_config.py b/src/anonymizer/config/anonymizer_config.py index bf8be0f3..58ce6325 100644 --- a/src/anonymizer/config/anonymizer_config.py +++ b/src/anonymizer/config/anonymizer_config.py @@ -136,14 +136,25 @@ def validate_excluded_entity_labels(cls, value: list[str] | None) -> list[str] | return deduped @model_validator(mode="after") - def warn_on_entity_label_overlap(self) -> "Detect": + def validate_entity_label_overlap(self) -> "Detect": if self.entity_labels is not None and self.excluded_entity_labels is not None: - overlap = sorted(set(self.entity_labels) & set(self.excluded_entity_labels)) - if overlap: - logger.warning( - "entity_labels and excluded_entity_labels share labels that will never be detected: %s", - overlap, + entity_labels_set = set(self.entity_labels) + excluded_set = set(self.excluded_entity_labels) + overlap = sorted(entity_labels_set & excluded_set) + if not overlap: + return self + if entity_labels_set <= excluded_set: + raise ValueError( + "excluded_entity_labels entirely overlaps entity_labels, leaving an empty " + f"effective detection set. Overlapping labels: {overlap}. Remove these labels from " + "excluded_entity_labels, add other labels to entity_labels, or unset entity_labels " + "(use None) to fall back to the default detection set — note excluded_entity_labels " + "still applies against it." ) + logger.warning( + "entity_labels and excluded_entity_labels share labels that will never be detected: %s", + overlap, + ) return self diff --git a/tests/config/test_anonymizer_config.py b/tests/config/test_anonymizer_config.py index 11683d18..d58c1966 100644 --- a/tests/config/test_anonymizer_config.py +++ b/tests/config/test_anonymizer_config.py @@ -222,3 +222,39 @@ def test_excluded_entity_labels_overlap_warning_only_fires_when_allowlist_explic replace=Redact(), ) assert "will never be detected" not in caplog.text + + +def test_excluded_entity_labels_fully_overlapping_entity_labels_raises() -> None: + with pytest.raises(ValidationError, match="entirely overlaps"): + AnonymizerConfig( + detect={"entity_labels": ["email", "city"], "excluded_entity_labels": ["email", "city"]}, + replace=Redact(), + ) + + +def test_excluded_entity_labels_superset_of_entity_labels_raises() -> None: + """excluded_entity_labels covering entity_labels plus extra labels still empties the set.""" + with pytest.raises(ValidationError, match="entirely overlaps"): + AnonymizerConfig( + detect={ + "entity_labels": ["email", "city"], + "excluded_entity_labels": ["email", "city", "bank_account"], + }, + replace=Redact(), + ) + + +def test_entity_labels_superset_of_excluded_entity_labels_only_warns( + caplog: pytest.LogCaptureFixture, +) -> None: + """entity_labels covering excluded_entity_labels plus extra labels still detects something.""" + with caplog.at_level(logging.WARNING, logger="anonymizer"): + config = AnonymizerConfig( + detect={ + "entity_labels": ["email", "city", "bank_account"], + "excluded_entity_labels": ["email", "city"], + }, + replace=Redact(), + ) + assert config.detect.entity_labels == ["bank_account", "city", "email"] + assert "will never be detected" in caplog.text From adc15a144acf415febd9257607ef8a12937ff5e5 Mon Sep 17 00:00:00 2001 From: memadi Date: Wed, 9 Sep 2026 14:37:05 -0700 Subject: [PATCH 27/30] docs: document excluded_entity_labels overlap raise in remaining spots Extend the ValueError-on-full-overlap documentation added in 52975e8 to the two other places that described this field: the field's own description= (feeds the mkdocstrings API reference) and the duplicate excluded_entity_labels section in choosing-a-strategy.md, which never had the overlap warning at all. Co-Authored-By: Claude Sonnet 5 --- docs/concepts/choosing-a-strategy.md | 3 +++ src/anonymizer/config/anonymizer_config.py | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/concepts/choosing-a-strategy.md b/docs/concepts/choosing-a-strategy.md index 2b34814d..2d5c9226 100644 --- a/docs/concepts/choosing-a-strategy.md +++ b/docs/concepts/choosing-a-strategy.md @@ -91,6 +91,9 @@ Detect(excluded_entity_labels=["occupation", "gender"]) Detect(entity_labels=["first_name", "email", "city"], excluded_entity_labels=["city"]) ``` +!!! warning + If `entity_labels` and `excluded_entity_labels` partially overlap, the shared labels are dropped and Anonymizer logs a warning. If `excluded_entity_labels` entirely overlaps `entity_labels` — leaving an empty effective detection set — `Detect` raises a `ValueError` at config time instead of silently detecting nothing. + ### `gliner_threshold` Default `0.3`. The validator catches false positives downstream, so erring low is safe. diff --git a/src/anonymizer/config/anonymizer_config.py b/src/anonymizer/config/anonymizer_config.py index 58ce6325..1bd692cd 100644 --- a/src/anonymizer/config/anonymizer_config.py +++ b/src/anonymizer/config/anonymizer_config.py @@ -84,7 +84,9 @@ class Detect(BaseModel): description=( "Entity labels to never detect, even if present in entity_labels or the default set. " "Excluded labels are removed before GLiNER and LLM prompts run, and are also filtered " - "from the final entity output as a safety net." + "from the final entity output as a safety net. If this entirely overlaps an explicit " + "entity_labels, leaving an empty effective detection set, Detect raises a ValueError " + "at construction time." ), ) gliner_threshold: float = Field( From bf1cfbf10640f557831470e199f5e21f20fac88f Mon Sep 17 00:00:00 2001 From: memadi Date: Wed, 9 Sep 2026 14:40:55 -0700 Subject: [PATCH 28/30] docs: say "at config time" instead of "at construction time" Matches the phrasing already used in detection.md and choosing-a-strategy.md for the same behavior. Co-Authored-By: Claude Sonnet 5 --- src/anonymizer/config/anonymizer_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/anonymizer/config/anonymizer_config.py b/src/anonymizer/config/anonymizer_config.py index 1bd692cd..853c09f9 100644 --- a/src/anonymizer/config/anonymizer_config.py +++ b/src/anonymizer/config/anonymizer_config.py @@ -86,7 +86,7 @@ class Detect(BaseModel): "Excluded labels are removed before GLiNER and LLM prompts run, and are also filtered " "from the final entity output as a safety net. If this entirely overlaps an explicit " "entity_labels, leaving an empty effective detection set, Detect raises a ValueError " - "at construction time." + "at config time." ), ) gliner_threshold: float = Field( From f3cffb591c7d06c007e55a50cb7082b1a888d991 Mon Sep 17 00:00:00 2001 From: nvskills-svc-account Date: Thu, 10 Sep 2026 17:52:26 +0000 Subject: [PATCH 29/30] Attach NVSkills validation signatures Signed-off-by: nvskills-svc-account --- skills/anonymizer/BENCHMARK.md | 83 +++++++++++++++++++-------------- skills/anonymizer/skill-card.md | 45 +++++++++--------- skills/anonymizer/skill.oms.sig | 2 +- 3 files changed, 72 insertions(+), 58 deletions(-) diff --git a/skills/anonymizer/BENCHMARK.md b/skills/anonymizer/BENCHMARK.md index ab281a63..690f8654 100644 --- a/skills/anonymizer/BENCHMARK.md +++ b/skills/anonymizer/BENCHMARK.md @@ -1,35 +1,23 @@ # Skill Benchmark: anonymizer -> ✅ **Overall verdict: PASS — Recommended for publication** +> ⚠️ **Overall verdict: INCOMPLETE — Required evidence is missing** -## Publication Recommendation - -Recommended for publication based on the completed evaluation evidence in this report. +One or more required evaluation tiers did not complete, so this benchmark is not publication-complete. ## Evaluation Metadata - Skill: `anonymizer` -- Evaluation date: 2026-08-20 -- Evaluator version: `1.3.2` +- Evaluation date: 2026-09-10 +- Evaluator version: `1.5.6` - Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`), Codex (`openai/openai/gpt-5.5`) - Tasks: 6 evaluation tasks (4 positive, 2 negative) - Dataset digest: `sha256:c2c13b2d794c6117dac0402f1261bd2d80972085c5e716426bedff6b3d59b8ae` (skill-evaluator-dataset-snapshot/1) -- Attempts per task: 1 -- Environment: `local` +- Attempts per task: 3 +- Environment: `k8s-sandbox` +- Tier 2 evidence: required for publication - Tier 3 evidence: required for publication -Tasks ran on the trusted local host; local mode is not sandboxed. - -## Execution and Provenance - -- Validation status: `passed` -- Report generation: `complete` -- Evaluator version: `1.3.2` -- Git commit: `0117bc2e3e54da4244a656466526c5b1b5a559ea` -- Content type: requested `auto`, detected `skill` -- Container image: `gitlab-master.nvidia.com:5005/nvcarps/ci-group/nvcarps-ci/skillevaluator-ci:sha-0117bc2e3e54da4244a656466526c5b1b5a559ea` -- Container image digest: `not recorded` -- Tier 3: requested `true`, executed `true`, status `succeeded` +Each task attempt ran in its own isolated sandbox pod. ## What This Report Answers @@ -45,16 +33,42 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 65% → 95% (+30 points) | 72% → 93% (+21 points) | -| Security | 100% → 100% (±0 points) | 83% → 83% (±0 points) | -| Correctness | 73% → 97% (+23 points) | 90% → 100% (+10 points) | -| Discoverability | 50% → 98% (+48 points) | 67% → 92% (+25 points) | -| Effectiveness | 54% → 87% (+33 points) | 69% → 92% (+23 points) | -| Efficiency | 50% → 94% (+44 points) | 50% → 100% (+50 points) | - -**How to read this table:** baseline is the same task attempted without the target skill. Uplift is `skill score - baseline score`, shown in percentage points. - -Example: `47% → 92% (+45 points)` means the skill-assisted run scored 92%, 45 percentage points above its 47% no-skill baseline. +| Overall | 89.8% — baseline ran, but no comparable score was available; uplift unavailable | 89.0% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | 95.0% → 100.0% (+5.0 points) | 100.0% → 83.3% (-16.7 points) | +| Correctness | 46.0% → 86.7% (+40.7 points) | 75.0% → 100.0% (+25.0 points) | +| Discoverability | 97.5% — baseline ran, but no comparable score was available; uplift unavailable | 93.8% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | 37.6% → 80.3% (+42.7 points) | 51.3% → 87.8% (+36.5 points) | +| Efficiency | 84.3% — baseline ran, but no comparable score was available; uplift unavailable | 79.8% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. + +## Token Usage + +Actual Tier 3 execution usage is reported for every observed agent/case pair and both conditions. + +| Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | +|---|---|---:|---:|---:|---:|---| +| claude-code | All cases | 1,699,362 | 2,863,530 | N/A | N/A | skill 6/6; base 10/10 | +| claude-code | anonymizer-negative-general-privacy-explainer | 30,472 | 30,464 | +8 | +0.03% | skill 1/1; base 1/1 | +| claude-code | anonymizer-negative-repository-source-development | 261,745 | 150,049 | +111,696 | +74.44% | skill 1/1; base 1/1 | +| claude-code | anonymizer-positive-failed-records-first | 185,130 | 431,085 | N/A | N/A | skill 1/1; base 3/3 | +| claude-code | anonymizer-positive-hash-cross-record-consistency | 456,894 | 31,819 | +425,075 | +1335.92% | skill 1/1; base 1/1 | +| claude-code | anonymizer-positive-mode-choice | 361,402 | 32,805 | +328,597 | +1001.67% | skill 1/1; base 1/1 | +| claude-code | anonymizer-positive-self-hosted-gliner | 403,719 | 2,187,308 | N/A | N/A | skill 1/1; base 3/3 | +| codex | All cases | 1,045,950 | 846,940 | N/A | N/A | skill 6/6; base 8/8 | +| codex | anonymizer-negative-general-privacy-explainer | 13,778 | 13,555 | +223 | +1.65% | skill 1/1; base 1/1 | +| codex | anonymizer-negative-repository-source-development | 853,740 | 578,858 | +274,882 | +47.49% | skill 1/1; base 1/1 | +| codex | anonymizer-positive-failed-records-first | 66,683 | 151,908 | N/A | N/A | skill 1/1; base 3/3 | +| codex | anonymizer-positive-hash-cross-record-consistency | 29,917 | 24,789 | +5,128 | +20.69% | skill 1/1; base 1/1 | +| codex | anonymizer-positive-mode-choice | 34,490 | 25,537 | +8,953 | +35.06% | skill 1/1; base 1/1 | +| codex | anonymizer-positive-self-hosted-gliner | 47,342 | 52,293 | -4,951 | -9.47% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 2,745,312 | 3,710,470 | N/A | N/A | skill 12/12; base 18/18 | + +Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. N/A means the relevant trajectory counters were not available; coverage is never estimated. ## Tier Status @@ -85,23 +99,24 @@ Example: `47% → 92% (+45 points)` means the skill-assisted run scored 92%, 45 | Correctness | Is the answer correct? | `accuracy` (100%) | | Discoverability | Was the right skill loaded when needed? | `skill_execution` (100%) | | Effectiveness | Did the skill help complete the task? | `goal_accuracy` (50%) + `behavior_check` (50%) | -| Efficiency | Did it avoid wasted tool or skill usage? | `skill_efficiency` (100%) | +| Efficiency | Did it avoid wasted tool calls and token usage? | `skill_efficiency` (50%) + `token_efficiency` (50%) | - Dimension bands: PASS at 50% or above; NEUTRAL from 40% to below 50%; FAIL below 40%. - Overall Tier 3 lift: PASS at +5 points or more; FAIL at -10 points or less; values between those bands are NEUTRAL. - Overall verdict: PASS only when every configured dimension passes for at least one supported agent. Lift is reported as diagnostic evidence and does not override this gate. - The 50% attempt pass threshold is a separate per-task gate; it is not the dimension pass threshold. - Effectiveness is the equal-weight mean of goal completion (`goal_accuracy`) and expected workflow adherence (`behavior_check`). -- Token efficiency is a separate report-only signal. It does not change a dimension score or the overall verdict. +- Efficiency is 50% tool-call productivity (the backward-compatible `skill_efficiency` wire id) and 50% `token_efficiency`. Positive-case skill routing is scored under Discoverability, not Efficiency; a negative case without a routing target is N/A. N/A sources are omitted, remaining weights are renormalized, and the dimension is marked partial. Signals present in this run: - `security` (Security): unsafe operations, secret leakage, and unauthorized access. -- `skill_execution` (Skill Execution): whether the expected skill was found and executed. -- `skill_efficiency` (Efficiency): routing quality, workspace-aware skill reads, and productive tool use. +- `skill_execution` (Skill Execution): whether the expected skill was selected, decoys were avoided, and the workflow executed. +- `skill_efficiency` (Tool Productivity): tool-call productivity (legacy wire id; routing is scored under Discoverability). - `accuracy` (Accuracy): final-answer correctness against the reference answer. - `goal_accuracy` (Goal Accuracy): whether the user's goal was achieved. - `behavior_check` (Behavior Check): whether the expected workflow behavior was followed. +- `token_efficiency` (Token Efficiency): actual uncached prompt plus completion usage (50% of Efficiency). diff --git a/skills/anonymizer/skill-card.md b/skills/anonymizer/skill-card.md index 7bca132a..0019a330 100644 --- a/skills/anonymizer/skill-card.md +++ b/skills/anonymizer/skill-card.md @@ -9,7 +9,7 @@ NVIDIA
### License/Terms of Use:
Apache 2.0
## Use Case:
-Developers and data engineers who need to anonymize text datasets containing PII for downstream model training, analytics, or data sharing.
+Developers and data engineers who need to anonymize text datasets — redact PII, de-identify free-text data, or rewrite text to remove sensitive or inferable identifying information before sharing or analysis.
### Deployment Geography for Use:
Global
@@ -25,22 +25,20 @@ Risk: Review before execution as proposals could introduce incorrect or misleadi Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [Interactive workflow reference](references/interactive.md)
-- [NeMo Anonymizer documentation](https://nvidia-nemo.github.io/Anonymizer/)
-- [Choosing a strategy](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/choosing-a-strategy/)
-- [Detection guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/detection/)
-- [Evaluation guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/evaluation/)
-- [Models guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/models/)
+- [NeMo Anonymizer GitHub Repository](https://github.com/NVIDIA-NeMo/Anonymizer)
+- [Choosing a Strategy](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/choosing-a-strategy/)
+- [Detection](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/detection/)
+- [Evaluation](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/evaluation/)
+- [Models](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/models/)
+- [Self-Hosting GLiNER](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/self-hosting-gliner/)
- [Troubleshooting](https://nvidia-nemo.github.io/Anonymizer/dev/troubleshooting/)
-- [Self-hosting GLiNER](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/self-hosting-gliner/)
-- [GitHub repository](https://github.com/NVIDIA-NeMo/Anonymizer.git)
## Skill Output:
**Output Type(s):** [Code]
**Output Format:** [Python script]
**Output Parameters:** [1D]
-**Other Properties Related to Output:** [Runnable script with CLI flags for preview, full run, and evaluation]
+**Other Properties Related to Output:** [None]
## Evaluation Agents Used:
- Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`)
@@ -49,38 +47,39 @@ Mitigation: Review and scan skill before deployment.
## Evaluation Tasks:
-6 evaluation tasks (4 positive, 2 negative) from skill-evaluator-dataset-snapshot/1.
+6 evaluation tasks (4 positive, 2 negative), 3 attempts per task, each in an isolated sandbox pod.
## Evaluation Metrics Used:
Reported benchmark dimensions:
- Security: Whether the skill avoids unsafe operations, secret leakage, and unauthorized access.
- Correctness: Whether the final answer is correct against the reference answer.
-- Discoverability: Whether the right skill is found and executed when needed.
-- Effectiveness: Whether the skill helps complete the user's goal (goal completion + expected workflow adherence).
-- Efficiency: Whether the skill avoids wasted tool or skill usage (routing quality and productive tool use).
+- Discoverability: Whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- Effectiveness: Whether the skill helped complete the user's goal (50% goal accuracy + 50% behavior check).
+- Efficiency: Whether the skill avoided wasted tool calls and token usage (50% tool productivity + 50% token efficiency).
Underlying evaluation signals used in this run:
- `security`: Checks for unsafe operations, secret leakage, and unauthorized access.
-- `skill_execution`: Whether the expected skill was found and executed.
-- `skill_efficiency`: Routing quality, workspace-aware skill reads, and productive tool use.
+- `skill_execution`: Whether the expected skill was selected, decoys were avoided, and the workflow executed.
+- `skill_efficiency`: Tool-call productivity; routing is scored under Discoverability.
- `accuracy`: Final-answer correctness against the reference answer.
- `goal_accuracy`: Whether the user's goal was achieved.
- `behavior_check`: Whether the expected workflow behavior was followed.
+- `token_efficiency`: Actual uncached prompt plus completion token usage.
## Evaluation Results:
| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 65% → 95% (+30 points) | 72% → 93% (+21 points) | -| Security | 100% → 100% (±0 points) | 83% → 83% (±0 points) | -| Correctness | 73% → 97% (+23 points) | 90% → 100% (+10 points) | -| Discoverability | 50% → 98% (+48 points) | 67% → 92% (+25 points) | -| Effectiveness | 54% → 87% (+33 points) | 69% → 92% (+23 points) | -| Efficiency | 50% → 94% (+44 points) | 50% → 100% (+50 points) | +| Overall | 89.8% | 89.0% | +| Security | 95.0% → 100.0% (+5.0 points) | 100.0% → 83.3% (-16.7 points) | +| Correctness | 46.0% → 86.7% (+40.7 points) | 75.0% → 100.0% (+25.0 points) | +| Discoverability | 97.5% | 93.8% | +| Effectiveness | 37.6% → 80.3% (+42.7 points) | 51.3% → 87.8% (+36.5 points) | +| Efficiency | 84.3% | 79.8% | ## Skill Version(s):
-318cc15 (source: git SHA, committed 2026-08-19)
+bf1cfbf (source: git SHA, committed 2026-09-09)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/anonymizer/skill.oms.sig b/skills/anonymizer/skill.oms.sig index 119a021c..0d958260 100644 --- a/skills/anonymizer/skill.oms.sig +++ b/skills/anonymizer/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiYW5vbnltaXplciIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICJlN2FmOWQ3MzM2NWU4MzU3ZDQxNDdiZjdlODQ1OTMyMjk5YTYyNmQ3Nzk4ZDlhZjA1NTEzNjUxMzBkZDkwMjk0IgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGh1YiIsCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXQiCiAgICAgIF0sCiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IiwKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIsCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlCiAgICB9LAogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiYjlhNmRhNmVjNmZiZTZhNjNlZWFiODIxMDcyYWRmZjkxNGQ2ZjZmNDUxNzlkNWFhY2Q3Y2ZiZjE5M2EzMzNhNiIsCiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiN2ZjOTQ2NDlhN2ZkMjBiZTBmNDUzNDQzMmQ2MjFhYTliYzBlYmYxYWJlOTJkNDJhNDQ1YzI5ZDQ2MGM2NWE4MSIsCiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJjYTQ3YjgyZGMzMTJmYzY0MDc3MGJmNjczM2JhNDYyNGRjYzhmNzA4MDdlZDM2ZjczZTllZTUwYTFkNDdlMGMyIiwKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiZDc1NWFhNDU3NDA3ZTM5MDE1YzcxMWEwY2I4MDI4MmRlZTkyNzZhYWJiYzRhMTYzM2YxZDUzZDExMGUzM2YwOCIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9pbnRlcmFjdGl2ZS5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjkxYTcxM2U5NWI5YzgwOGM2YWQyZDgzNjlmY2UyYTBmMzk1ZjMzMTcwMTVhNWI4NGY4MzUxNTVjNmQ0MTZhY2IiLAogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9CiAgICBdCiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQCMaSVpacY57xcRHqNUKyt4gV+ODfkD3GIaKfW316ZDIUmnF+7uwFCbxheCDvKDo4ACMQDecoAhlXnOmV8ca3qaA+HktQuA8yZzuAOflyHy+UVgZTKjk897md3R2vyz2KZxqAw=","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiYW5vbnltaXplciIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICI1YTg0NTFkNDU5MzJhYTEyNmFmYjg0YzY2YmVkYmExMjVjM2M4ZDc3YjNlMTNhYmU4YmJkMTU5YWMwYzViMjFiIgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGlnbm9yZSIsCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0IiwKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiCiAgICAgIF0sCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJtZXRob2QiOiAiZmlsZXMiCiAgICB9LAogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjUzZmI3MDNkNThmMDYzNWYwMjVlM2Y2YmY2NDNhMTY1Zjk1MzdkYjYyZTRkYTJmMTRhOWNmZDcyNGY3Nzg0MmYiLAogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjJkZWY2OWFkZjFkZjM3MTNkNDk2MzU4MWVjZjcwOWViYTI4ZmYxZjNmZDhlMDlhM2E3MjRiMTE2YmM4YWQ1MzIiLAogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiY2E0N2I4MmRjMzEyZmM2NDA3NzBiZjY3MzNiYTQ2MjRkY2M4ZjcwODA3ZWQzNmY3M2U5ZWU1MGExZDQ3ZTBjMiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImQ3NTVhYTQ1NzQwN2UzOTAxNWM3MTFhMGNiODAyODJkZWU5Mjc2YWFiYmM0YTE2MzNmMWQ1M2QxMTBlMzNmMDgiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvaW50ZXJhY3RpdmUubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJiODcxYjgxZjYxODI5OTc2ZDdjZWNjNjI5ZDFiOTJmZjFiOWIxNGJmNjZjYjIyYjQ5MGE3NjYxZDk3YTM2ZmJkIiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIgogICAgICB9CiAgICBdCiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMF1GQHK3JoN6B1PNz5OnklGjo/IEBc+WoSXbXQsZ8tER2PNfk1RFV6gR3x7xgPJwCwIxAKlG+ZakKn0RLM6z90EBNj00Ei7HvR6qHAokrcDFGqYwKDz2UpJhZ1dS2Yv6AIdSbg==","keyid":""}]}} \ No newline at end of file From 7bd386431ef8f89ed5378d92669fad440c82864f Mon Sep 17 00:00:00 2001 From: memadi Date: Thu, 10 Sep 2026 11:43:40 -0700 Subject: [PATCH 30/30] feat(config): also guard entity_labels=None against full-default exclusion The earlier full-overlap guard only checked an explicit entity_labels against excluded_entity_labels. Reviewer feedback: entity_labels=None falls back to DEFAULT_ENTITY_LABELS, and excluded_entity_labels could still cancel that out entirely, leaving _resolve_detection_labels to silently return [] at runtime with only a warning. Detect now raises in that case too, and docs/SKILL.md are reworded to state the rule once against "the effective allowlist" instead of covering the two cases with separate sentences. Co-Authored-By: Claude Sonnet 5 --- docs/concepts/choosing-a-strategy.md | 2 +- docs/concepts/detection.md | 2 +- skills/anonymizer/SKILL.md | 2 +- src/anonymizer/config/anonymizer_config.py | 24 +++++++++++++++++----- tests/config/test_anonymizer_config.py | 19 +++++++++++++++++ 5 files changed, 41 insertions(+), 8 deletions(-) diff --git a/docs/concepts/choosing-a-strategy.md b/docs/concepts/choosing-a-strategy.md index 2d5c9226..2b838bb3 100644 --- a/docs/concepts/choosing-a-strategy.md +++ b/docs/concepts/choosing-a-strategy.md @@ -92,7 +92,7 @@ Detect(entity_labels=["first_name", "email", "city"], excluded_entity_labels=["c ``` !!! warning - If `entity_labels` and `excluded_entity_labels` partially overlap, the shared labels are dropped and Anonymizer logs a warning. If `excluded_entity_labels` entirely overlaps `entity_labels` — leaving an empty effective detection set — `Detect` raises a `ValueError` at config time instead of silently detecting nothing. + `excluded_entity_labels` is always checked against the effective allowlist — `entity_labels` if set, otherwise `DEFAULT_ENTITY_LABELS`. A partial overlap just drops the shared labels and logs a warning. If the overlap is total, leaving an empty effective detection set, `Detect` raises a `ValueError` at config time instead of silently detecting nothing. ### `gliner_threshold` diff --git a/docs/concepts/detection.md b/docs/concepts/detection.md index 55004a4a..fae0295b 100644 --- a/docs/concepts/detection.md +++ b/docs/concepts/detection.md @@ -119,7 +119,7 @@ Detect(entity_labels=["first_name", "email", "city"], excluded_entity_labels=["c ``` !!! warning - If `entity_labels` and `excluded_entity_labels` partially overlap, the shared labels are dropped and Anonymizer logs a warning. If `excluded_entity_labels` entirely overlaps `entity_labels` — leaving an empty effective detection set — `Detect` raises a `ValueError` at config time instead of silently detecting nothing. This only applies when `entity_labels` is explicitly set; `entity_labels=None` (the default set) can never be fully excluded this way, since removing all default labels via `excluded_entity_labels` still only logs a warning. + `excluded_entity_labels` is always checked against the effective allowlist — `entity_labels` if set, otherwise `DEFAULT_ENTITY_LABELS`. A partial overlap just drops the shared labels and logs a warning. If the overlap is total, leaving an empty effective detection set, `Detect` raises a `ValueError` at config time instead of silently detecting nothing. ## Tuning the threshold diff --git a/skills/anonymizer/SKILL.md b/skills/anonymizer/SKILL.md index 216d0506..7bef44a1 100644 --- a/skills/anonymizer/SKILL.md +++ b/skills/anonymizer/SKILL.md @@ -43,7 +43,7 @@ regulatory and business context. # Usage Tips and Common Pitfalls - **`Detect.entity_labels=None` (the default) is permissive** — the augmenter LLM may invent labels not in `DEFAULT_ENTITY_LABELS`. Setting an explicit list switches to **strict mode** where *only* the listed labels are detected. To add domain labels, *extend* the default, don't replace it: `entity_labels=[*DEFAULT_ENTITY_LABELS, "clinical_facility", ...]` (`DEFAULT_ENTITY_LABELS` is a tuple, so unpack it into a list). Match the snake_case convention of `DEFAULT_ENTITY_LABELS`. -- **`Detect.excluded_entity_labels`** excludes specific label types from detection entirely — excluded labels are removed before GLiNER runs and are never detected, augmented, or penalised in evaluation scores. Use it when a label type is systematically noisy for your data or should never be anonymized (e.g. `Detect(excluded_entity_labels=["occupation", "gender"])`). Exclusions take precedence over `entity_labels` — a label in both is never detected. If `excluded_entity_labels` entirely overlaps an explicit `entity_labels`, `Detect` raises a `ValueError` at config time instead of silently building a config that detects nothing. +- **`Detect.excluded_entity_labels`** excludes specific label types from detection entirely — excluded labels are removed before GLiNER runs and are never detected, augmented, or penalised in evaluation scores. Use it when a label type is systematically noisy for your data or should never be anonymized (e.g. `Detect(excluded_entity_labels=["occupation", "gender"])`). Exclusions take precedence over `entity_labels` — a label in both is never detected. If `excluded_entity_labels` entirely overlaps the effective allowlist (`entity_labels` if set, otherwise `DEFAULT_ENTITY_LABELS`), `Detect` raises a `ValueError` at config time instead of silently building a config that detects nothing. - **GLiNER is zero-shot** — entity labels are natural-language concept names (e.g. `"clinical_facility"`, `"internal_project_codename"`), not codes or enum values. Any concept you can name in English is a label GLiNER can detect. - **`Rewrite.instructions` is a dead field today** — it exists on the model but the rewrite engine never reads it. Do not use it. Put rewriter guidance in `privacy_goal.protect` / `privacy_goal.preserve` instead. - **`risk_tolerance` only applies to Rewrite mode**, not Replace. diff --git a/src/anonymizer/config/anonymizer_config.py b/src/anonymizer/config/anonymizer_config.py index 853c09f9..bd97b518 100644 --- a/src/anonymizer/config/anonymizer_config.py +++ b/src/anonymizer/config/anonymizer_config.py @@ -17,6 +17,7 @@ PrivacyGoal, RiskTolerance, ) +from anonymizer.engine.constants import DEFAULT_ENTITY_LABELS logger = logging.getLogger(__name__) @@ -84,9 +85,9 @@ class Detect(BaseModel): description=( "Entity labels to never detect, even if present in entity_labels or the default set. " "Excluded labels are removed before GLiNER and LLM prompts run, and are also filtered " - "from the final entity output as a safety net. If this entirely overlaps an explicit " - "entity_labels, leaving an empty effective detection set, Detect raises a ValueError " - "at config time." + "from the final entity output as a safety net. If this entirely overlaps the effective " + "allowlist (entity_labels if set, otherwise the default label set), leaving an empty " + "effective detection set, Detect raises a ValueError at config time." ), ) gliner_threshold: float = Field( @@ -139,9 +140,12 @@ def validate_excluded_entity_labels(cls, value: list[str] | None) -> list[str] | @model_validator(mode="after") def validate_entity_label_overlap(self) -> "Detect": - if self.entity_labels is not None and self.excluded_entity_labels is not None: + if self.excluded_entity_labels is None: + return self + excluded_set = set(self.excluded_entity_labels) + + if self.entity_labels is not None: entity_labels_set = set(self.entity_labels) - excluded_set = set(self.excluded_entity_labels) overlap = sorted(entity_labels_set & excluded_set) if not overlap: return self @@ -157,6 +161,16 @@ def validate_entity_label_overlap(self) -> "Detect": "entity_labels and excluded_entity_labels share labels that will never be detected: %s", overlap, ) + return self + + # entity_labels=None falls back to DEFAULT_ENTITY_LABELS; guard that path too. + if set(DEFAULT_ENTITY_LABELS) <= excluded_set: + raise ValueError( + "excluded_entity_labels entirely overlaps DEFAULT_ENTITY_LABELS, leaving an empty " + "effective detection set (entity_labels is unset, so the default label set applies). " + "Set entity_labels explicitly to a non-empty subset of labels you still want detected, " + "or remove some labels from excluded_entity_labels." + ) return self diff --git a/tests/config/test_anonymizer_config.py b/tests/config/test_anonymizer_config.py index d58c1966..a9d4c327 100644 --- a/tests/config/test_anonymizer_config.py +++ b/tests/config/test_anonymizer_config.py @@ -20,6 +20,7 @@ Hash, Redact, ) +from anonymizer.engine.constants import DEFAULT_ENTITY_LABELS def test_hash_is_deterministic() -> None: @@ -224,6 +225,24 @@ def test_excluded_entity_labels_overlap_warning_only_fires_when_allowlist_explic assert "will never be detected" not in caplog.text +def test_excluded_entity_labels_covering_all_defaults_raises() -> None: + """entity_labels=None falls back to DEFAULT_ENTITY_LABELS; excluding all of it must also raise.""" + with pytest.raises(ValidationError, match="entirely overlaps DEFAULT_ENTITY_LABELS"): + AnonymizerConfig( + detect={"excluded_entity_labels": list(DEFAULT_ENTITY_LABELS)}, + replace=Redact(), + ) + + +def test_excluded_entity_labels_partial_default_coverage_does_not_raise() -> None: + """Excluding some — but not all — default labels is the documented common case.""" + config = AnonymizerConfig( + detect={"excluded_entity_labels": ["occupation", "gender"]}, + replace=Redact(), + ) + assert config.detect.excluded_entity_labels == ["gender", "occupation"] + + def test_excluded_entity_labels_fully_overlapping_entity_labels_raises() -> None: with pytest.raises(ValidationError, match="entirely overlaps"): AnonymizerConfig(