diff --git a/docs/analyzer/adding_recognizers.md b/docs/analyzer/adding_recognizers.md index aa574dd46..d7e84a85a 100644 --- a/docs/analyzer/adding_recognizers.md +++ b/docs/analyzer/adding_recognizers.md @@ -98,6 +98,42 @@ engine = AnalyzerEngine(registry=registry) engine.analyze(...) ``` +### Detecting part of a regex match + +By default, a pattern detects the whole regex match. +To detect only part of it, such as a password without the `password:` label before it, +set `capture_group` on the `Pattern` to the number or the name of a capture group in the regex: + +```python +from presidio_analyzer import Pattern, PatternRecognizer + +password_pattern = Pattern( + name="password (value only)", + regex=r"password:\s*(?P\S+)", + score=0.5, + capture_group="value", +) +password_recognizer = PatternRecognizer( + supported_entity="PASSWORD", patterns=[password_pattern] +) + +# Detects "hunter2" (start=13, end=20) instead of "password: hunter2" +results = password_recognizer.analyze(text="my password: hunter2", entities=["PASSWORD"]) +print(results) +``` + +Notes: + +- `capture_group` is optional. It accepts a group number (`0` is the whole match) or a group name, and is checked against the regex when the `Pattern` is created. +- This check does not apply regex flags. If the flags used at analysis time change the groups of the regex (for example, with `re.VERBOSE` the text after `#` is a comment), a pattern whose group no longer exists detects nothing and logs a warning, and a group number can refer to a different group. Group names are never renumbered, so prefer them in this case. +- Matches in which the group does not participate, such as an optional group that did not match, are skipped. +- A pattern detects a single group. To detect several groups of the same regex, define one pattern per group. +- `validate_result` and `invalidate_result` receive the text of the group. +- Words matched by the regex outside the group count as surrounding text, so they can raise the score of the result if they are context words of the recognizer. +- Recognizers that replace the pattern matching logic of `PatternRecognizer` (for example, `IbanRecognizer`) ignore `capture_group`. + +The same field can be set in [ad-hoc recognizers](#creating-ad-hoc-recognizers) and in [recognizers loaded from YAML](./recognizer_registry_provider.md#the-recognizer-parameters). + ### Creating a new `EntityRecognizer` in code To create a new recognizer via code: diff --git a/docs/analyzer/recognizer_registry_provider.md b/docs/analyzer/recognizer_registry_provider.md index 7e299543d..1e01aa0c1 100644 --- a/docs/analyzer/recognizer_registry_provider.md +++ b/docs/analyzer/recognizer_registry_provider.md @@ -105,7 +105,19 @@ The recognizer list comprises of both the predefined and custom recognizers, for In addition to the language code, this field also contains a list of context words, which increases confidence in the detection in case it is found in the surroundings of a detected entity (as seen in the credit card example above). - `type`: this could be either predefined or custom. As this is optional, if not stated otherwise, the default type is custom. - `name`: Different per the type of the recognizer. For predefined recognizers, this is the class name as defined in presidio, while for custom recognizers, it will be set as the name of the recognizer. - - `patterns`: a list of objects of type `Pattern` that contains a name, score and regex that define matching patterns. + - `patterns`: a list of objects of type `Pattern` that contains a name, score and regex that define matching patterns. A pattern can also set an optional `capture_group`: the number or the name of a capture group in the regex. When set, only the text matched by this group is detected instead of the whole match, and matches in which the group does not participate are skipped. An unknown group fails validation when the configuration is loaded. This check does not apply `global_regex_flags`. See [detecting part of a regex match](./adding_recognizers.md#detecting-part-of-a-regex-match). Example: + + ```yaml + - name: PasswordRecognizer + type: custom + supported_language: en + supported_entity: PASSWORD + patterns: + - name: password (value only) + regex: "password:\\s*(?P\\S+)" + score: 0.5 + capture_group: value + ``` - `enabled`: enables or disables the recognizer. - `supported_entity`: the detected entity associated by the recognizer. - `deny_list`: A list of words to detect, in case the recognizer uses a predefined list of words. diff --git a/docs/api-docs/api-docs.yml b/docs/api-docs/api-docs.yml index 16b91920f..4482aeee3 100644 --- a/docs/api-docs/api-docs.yml +++ b/docs/api-docs/api-docs.yml @@ -570,6 +570,12 @@ components: type: number format: double description: "Detection confidence of this pattern (0.01 if very noisy, 0.6-1.0 if very specific)" + capture_group: + oneOf: + - type: integer + minimum: 0 + - type: string + description: "Number or name of a capture group in the regex. If set, only the text matched by this group is returned as the detected entity, and matches in which the group does not participate are skipped. If omitted, the whole match is returned. The group is checked against the regex without regex flags: if the flags used at analysis time (for example re.VERBOSE) remove the group, the pattern detects nothing." PatternRecognizer: diff --git a/e2e-tests/tests/test_api_analyzer.py b/e2e-tests/tests/test_api_analyzer.py index 730179ac4..34625a41e 100644 --- a/e2e-tests/tests/test_api_analyzer.py +++ b/e2e-tests/tests/test_api_analyzer.py @@ -444,6 +444,45 @@ def test_given_ad_hoc_pattern_recognizer_context_raises_confidence(): ) +@pytest.mark.api +def test_given_ad_hoc_pattern_recognizer_with_capture_group_then_only_the_group_is_returned(): + request_body = r""" + { + "text": "John Smith drivers license is AC432223. Zip code: 10023", + "language": "en", + "ad_hoc_recognizers":[ + { + "name": "Zip code Recognizer", + "supported_language": "en", + "patterns": [ + { + "name": "zip code (after label)", + "regex": "zip code: (\\d{5})", + "score": 0.01, + "capture_group": 1 + } + ], + "supported_entity":"ZIP" + } + ] + } + """ + + response_status, response_content = analyze(request_body) + + expected_response = """ + [ + {"entity_type": "PERSON", "start": 0, "end": 10, "score": 0.85, "analysis_explanation":null}, + {"entity_type": "US_DRIVER_LICENSE", "start": 30, "end": 38, "score": 0.6499999999999999, "analysis_explanation":null}, + {"entity_type": "ZIP", "start": 50, "end": 55, "score": 0.01, "analysis_explanation":null} + ] + """ + assert response_status == 200 + assert equal_json_strings( + expected_response, response_content + ) + + @pytest.mark.api def test_given_ad_hoc_deny_list_recognizer_the_right_entities_are_returned(): request_body = r""" diff --git a/presidio-analyzer/presidio_analyzer/input_validation/yaml_recognizer_models.py b/presidio-analyzer/presidio_analyzer/input_validation/yaml_recognizer_models.py index cab80d12f..a0fbe7ed6 100644 --- a/presidio-analyzer/presidio_analyzer/input_validation/yaml_recognizer_models.py +++ b/presidio-analyzer/presidio_analyzer/input_validation/yaml_recognizer_models.py @@ -4,6 +4,7 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from presidio_analyzer import Pattern from presidio_analyzer.input_validation import validate_language_codes from presidio_analyzer.recognizer_registry.recognizers_loader_utils import ( PredefinedRecognizerNotFoundError, @@ -446,6 +447,12 @@ def validate_patterns(cls, patterns: Optional[List[Dict]]) -> Optional[List[Dict raise ValueError(f"Pattern score should be a float: {pattern}") if not (0.0 <= pattern["score"] <= 1.0): raise ValueError(f"Pattern score should be between 0 and 1: {pattern}") + if pattern.get("capture_group") is not None: + # Build the Pattern to check the group against the compiled regex + try: + Pattern.from_dict(pattern) + except (TypeError, ValueError) as e: + raise ValueError(f"Invalid pattern {pattern['name']!r}: {e}") from e return patterns @model_validator(mode="after") diff --git a/presidio-analyzer/presidio_analyzer/pattern.py b/presidio-analyzer/presidio_analyzer/pattern.py index a4a3909bb..ea212f340 100644 --- a/presidio-analyzer/presidio_analyzer/pattern.py +++ b/presidio-analyzer/presidio_analyzer/pattern.py @@ -1,5 +1,5 @@ import json -from typing import Dict +from typing import Dict, Mapping, Optional, Union import regex as re @@ -11,26 +11,69 @@ class Pattern: :param name: the name of the pattern :param regex: the regex pattern to detect :param score: the pattern's strength (values varies 0-1) + :param capture_group: optional number or name of a capture group in the + regex. If set, the span of this group is detected instead of the whole + match, and matches in which the group does not participate are skipped. + Defaults to None (the whole match). """ - def __init__(self, name: str, regex: str, score: float): + def __init__( + self, + name: str, + regex: str, + score: float, + capture_group: Optional[Union[int, str]] = None, + ): self.name = name self.regex = regex self.score = score + self.capture_group = capture_group self.compiled_regex = None self.compiled_with_flags = None - self.__validate_regex(self.regex) + self.__validate_regex(self.regex, self.capture_group) self.__validate_score(self.score) @staticmethod - def __validate_regex(pattern: str) -> None: - """Validate that the regex pattern is valid.""" + def __validate_regex( + pattern: str, capture_group: Optional[Union[int, str]] + ) -> None: + """Validate that the regex pattern is valid and defines the capture group.""" try: - re.compile(pattern) + compiled_regex = re.compile(pattern) except re.error as e: raise ValueError(f"Invalid regex pattern: {e}") + if capture_group is not None: + Pattern.__validate_capture_group( + capture_group, compiled_regex.groups, compiled_regex.groupindex + ) + + @staticmethod + def __validate_capture_group( + capture_group: Union[int, str], groups: int, groupindex: Mapping[str, int] + ) -> None: + if isinstance(capture_group, bool) or not isinstance(capture_group, (int, str)): + raise ValueError( + "capture_group must be an int or a str, " + f"got {type(capture_group).__name__}" + ) + if isinstance(capture_group, str): + if capture_group not in groupindex: + raise ValueError( + f"capture_group {capture_group!r} is not a named group " + f"in the regex. Named groups: {list(groupindex)}" + ) + elif capture_group < 0: + raise ValueError( + f"capture_group must be a non-negative integer, got {capture_group}" + ) + elif capture_group > groups: + raise ValueError( + f"capture_group {capture_group} is out of range: " + f"regex defines {groups} capture group(s)" + ) + @staticmethod def __validate_score(score: float) -> None: if score < 0 or score > 1: @@ -45,6 +88,8 @@ def to_dict(self) -> Dict: :return: a dictionary """ return_dict = {"name": self.name, "score": self.score, "regex": self.regex} + if self.capture_group is not None: + return_dict["capture_group"] = self.capture_group return return_dict @classmethod diff --git a/presidio-analyzer/presidio_analyzer/pattern_recognizer.py b/presidio-analyzer/presidio_analyzer/pattern_recognizer.py index eee0ee07a..a201416c9 100644 --- a/presidio-analyzer/presidio_analyzer/pattern_recognizer.py +++ b/presidio-analyzer/presidio_analyzer/pattern_recognizer.py @@ -1,7 +1,7 @@ import datetime import logging import os -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING, Dict, List, Optional, Union import regex as re @@ -139,6 +139,7 @@ def validate_result(self, pattern_text: str) -> Optional[bool]: :param pattern_text: the text to validated. Only the part in text that was detected by the regex engine + (the pattern's capture group if set, otherwise the whole match) :return: A bool indicating whether the validation was successful. """ return None @@ -151,6 +152,7 @@ def invalidate_result(self, pattern_text: str) -> Optional[bool]: :param pattern_text: the text to validated. Only the part in text that was detected by the regex engine + (the pattern's capture group if set, otherwise the whole match) :return: A bool indicating whether the result is invalidated """ return None @@ -190,6 +192,12 @@ def build_regex_explanation( ) return explanation + @staticmethod + def __regex_has_group(compiled_regex, capture_group: Union[int, str]) -> bool: + if isinstance(capture_group, str): + return capture_group in compiled_regex.groupindex + return capture_group <= compiled_regex.groups + def __analyze_patterns( self, text: str, flags: int = None ) -> List[RecognizerResult]: @@ -212,6 +220,19 @@ def __analyze_patterns( pattern.compiled_with_flags = flags pattern.compiled_regex = re.compile(pattern.regex, flags=flags) + # Flags such as re.VERBOSE can change the groups in a regex + if pattern.capture_group is not None and not self.__regex_has_group( + pattern.compiled_regex, pattern.capture_group + ): + logger.warning( + "Regex pattern '%s' has no capture group %r " + "when compiled with the regex flags in use, skipping.", + pattern.name, + pattern.capture_group, + ) + continue + group = 0 if pattern.capture_group is None else pattern.capture_group + try: matches = pattern.compiled_regex.finditer( text, timeout=REGEX_TIMEOUT_SECONDS @@ -224,10 +245,11 @@ def __analyze_patterns( ) for match in matches: - start, end = match.span() + start, end = match.span(group) current_match = text[start:end] - # Skip empty results + # Skip empty results, including a capture group that did not + # participate in the match (its span is (-1, -1)) if current_match == "": continue diff --git a/presidio-analyzer/tests/test_analyzer_request.py b/presidio-analyzer/tests/test_analyzer_request.py index c704ac9d5..fd78c2331 100644 --- a/presidio-analyzer/tests/test_analyzer_request.py +++ b/presidio-analyzer/tests/test_analyzer_request.py @@ -190,6 +190,34 @@ def test_analyzer_request_with_ad_hoc_recognizers(self): assert isinstance(request.ad_hoc_recognizers[0], PatternRecognizer) assert request.ad_hoc_recognizers[0].supported_entities == ["CUSTOM_ID"] + def test_analyzer_request_with_ad_hoc_recognizer_capture_group(self): + """Test that a pattern capture_group reaches the ad-hoc recognizer.""" + req_data = { + "text": "Employee ID-12345", + "language": "en", + "ad_hoc_recognizers": [ + { + "supported_entity": "CUSTOM_ID", + "supported_language": "en", + "patterns": [ + { + "name": "id_pattern", + "regex": r"ID-(\d{5})", + "score": 0.8, + "capture_group": 1 + } + ] + } + ] + } + + request = AnalyzerRequest(req_data) + + recognizer = request.ad_hoc_recognizers[0] + assert recognizer.patterns[0].capture_group == 1 + results = recognizer.analyze(request.text, ["CUSTOM_ID"]) + assert [(result.start, result.end) for result in results] == [(12, 17)] + def test_analyzer_request_without_ad_hoc_recognizers(self): """Test that ad_hoc_recognizers is empty list when not provided.""" req_data = { diff --git a/presidio-analyzer/tests/test_context_support.py b/presidio-analyzer/tests/test_context_support.py index f3f94307e..3c5f11b2d 100644 --- a/presidio-analyzer/tests/test_context_support.py +++ b/presidio-analyzer/tests/test_context_support.py @@ -216,3 +216,47 @@ def test_when_context_custom_recognizer_then_succeed(spacy_nlp_engine, mock_nlp_ assert len(results_without_context) == len(results_with_context) for res_wo, res_w in zip(results_without_context, results_with_context): assert res_wo.score < res_w.score + + +def test_when_capture_group_narrows_span_then_matched_words_before_group_are_context( + spacy_nlp_engine, lemma_context +): + """Words matched by the regex before the capture group act as context. + + The result starts at the capture group, so "password" (matched by the + regex but outside the group) is a preceding word for context enhancement. + """ + text = "my password: hunter2" + patterns = [ + Pattern("password value", r"password:\s*(\S+)", 0.4, capture_group=1) + ] + recognizer_with_context = PatternRecognizer( + supported_entity="PASSWORD", patterns=patterns, context=["password"] + ) + recognizer_without_context = PatternRecognizer( + supported_entity="PASSWORD", patterns=patterns + ) + nlp_artifacts = spacy_nlp_engine.process_text(text, "en") + + results_with_context = lemma_context.enhance_using_context( + text, + recognizer_with_context.analyze(text, ["PASSWORD"], nlp_artifacts), + nlp_artifacts, + [recognizer_with_context], + ) + results_without_context = lemma_context.enhance_using_context( + text, + recognizer_without_context.analyze(text, ["PASSWORD"], nlp_artifacts), + nlp_artifacts, + [recognizer_without_context], + ) + + assert [(result.start, result.end) for result in results_with_context] == [ + (13, 20) + ] + assert results_without_context[0].score == 0.4 + assert results_with_context[0].score == pytest.approx(0.75) + assert ( + results_with_context[0].analysis_explanation.supportive_context_word + == "password" + ) diff --git a/presidio-analyzer/tests/test_pattern.py b/presidio-analyzer/tests/test_pattern.py index 255aa9e7b..766c92952 100644 --- a/presidio-analyzer/tests/test_pattern.py +++ b/presidio-analyzer/tests/test_pattern.py @@ -73,3 +73,84 @@ def test_backward_compatibility_pattern_to_dict(): expected = {"name": "test", "regex": r"\btest\b", "score": 0.5} assert pattern_dict == expected + + +def test_when_capture_group_not_set_then_it_defaults_to_none_and_is_not_serialized(): + pattern = Pattern(name="test", regex=r"id: (\d+)", score=0.5) + + assert pattern.capture_group is None + assert "capture_group" not in pattern.to_dict() + + +@pytest.mark.parametrize( + "regex, capture_group", + [ + (r"id: (\d+)", 0), + (r"id: (\d+)", 1), + (r"id: (?P\d+)", "value"), + ], +) +def test_when_capture_group_set_then_round_trips_through_dict(regex, capture_group): + pattern = Pattern( + name="test", regex=regex, score=0.5, capture_group=capture_group + ) + + pattern_dict = pattern.to_dict() + restored = Pattern.from_dict(pattern_dict) + + assert pattern_dict == { + "name": "test", + "score": 0.5, + "regex": regex, + "capture_group": capture_group, + } + assert restored.capture_group == capture_group + + +@pytest.mark.parametrize( + "regex, capture_group, expected_message", + [ + ( + r"id: (\d+)", + 2, + r"capture_group 2 is out of range: regex defines 1 capture group\(s\)", + ), + ( + r"id: \d+", + 1, + r"capture_group 1 is out of range: regex defines 0 capture group\(s\)", + ), + ( + r"id: (?P\d+)", + "number", + r"capture_group 'number' is not a named group in the regex. " + r"Named groups: \['value'\]", + ), + ( + r"id: (\d+)", + "value", + r"capture_group 'value' is not a named group in the regex. " + r"Named groups: \[\]", + ), + ( + r"id: (\d+)", + -1, + r"capture_group must be a non-negative integer, got -1", + ), + ( + r"id: (\d+)", + True, + r"capture_group must be an int or a str, got bool", + ), + ( + r"id: (\d+)", + 1.0, + r"capture_group must be an int or a str, got float", + ), + ], +) +def test_when_capture_group_invalid_then_raises_value_error( + regex, capture_group, expected_message +): + with pytest.raises(ValueError, match=expected_message): + Pattern(name="test", regex=regex, score=0.5, capture_group=capture_group) diff --git a/presidio-analyzer/tests/test_pattern_recognizer.py b/presidio-analyzer/tests/test_pattern_recognizer.py index 61eebb6f4..feeac2acb 100644 --- a/presidio-analyzer/tests/test_pattern_recognizer.py +++ b/presidio-analyzer/tests/test_pattern_recognizer.py @@ -565,3 +565,159 @@ def test_regex_timeout_seconds_env_var_override(): # Restore the module to default state importlib.reload(pr_module) + +@pytest.mark.parametrize( + "regex, capture_group, expected_start, expected_end", + [ + (r"Password: (\w+)", None, 0, 14), + (r"Password: (\w+)", 0, 0, 14), + (r"Password: (\w+)", 1, 10, 14), + (r"Password: (?P\w+)", "password", 10, 14), + ], +) +def test_when_capture_group_given_then_result_spans_that_group_or_whole_match( + regex, capture_group, expected_start, expected_end +): + recognizer = PatternRecognizer( + supported_entity="PASSWORD", + patterns=[ + Pattern( + name="password", regex=regex, score=0.5, capture_group=capture_group + ) + ], + ) + + results = recognizer.analyze("Password: 1234", ["PASSWORD"]) + + assert len(results) == 1 + assert_result(results[0], "PASSWORD", expected_start, expected_end, 0.5) + + +@pytest.mark.parametrize( + "text, expected_spans", + [ + ("id", []), + ("id:", []), + ("id: 42", [(4, 6)]), + ], +) +def test_when_capture_group_does_not_participate_or_is_empty_then_match_is_skipped( + text, expected_spans +): + recognizer = PatternRecognizer( + supported_entity="ID", + patterns=[ + Pattern(name="id", regex=r"\bid(?:(:) ?(\d*))?", score=0.5, capture_group=2) + ], + ) + + results = recognizer.analyze(text, ["ID"]) + + assert [(result.start, result.end) for result in results] == expected_spans + + +def test_when_capture_group_set_then_validation_hooks_receive_group_text(): + received = [] + + class RecordingRecognizer(PatternRecognizer): + def validate_result(self, pattern_text): + received.append(("validate", pattern_text)) + return None + + def invalidate_result(self, pattern_text): + received.append(("invalidate", pattern_text)) + return None + + recognizer = RecordingRecognizer( + supported_entity="PASSWORD", + patterns=[ + Pattern( + name="password", regex=r"Password: (\w+)", score=0.5, capture_group=1 + ) + ], + ) + + recognizer.analyze("Password: 1234", ["PASSWORD"]) + + assert received == [("validate", "1234"), ("invalidate", "1234")] + + +def test_when_same_regex_uses_different_capture_groups_then_each_group_is_reported(): + regex = r"(\w+)@(\w+)\.com" + recognizer = PatternRecognizer( + supported_entity="EMAIL_PART", + patterns=[ + Pattern(name="user", regex=regex, score=0.4, capture_group=1), + Pattern(name="domain", regex=regex, score=0.4, capture_group=2), + ], + ) + + results = recognizer.analyze("mail john@example.com", ["EMAIL_PART"]) + + assert sorted((result.start, result.end) for result in results) == [ + (5, 9), + (10, 17), + ] + + +@pytest.mark.parametrize("capture_group", [2, "b"]) +def test_when_regex_flags_remove_capture_group_then_pattern_is_skipped_with_warning( + caplog, capture_group +): + # With re.VERBOSE, "# (?Pb)" is a comment, so the regex has only one group + verbose_dependent = Pattern( + name="verbose_dependent", + regex=r"(secret\w*) # (?Pb)", + score=0.5, + capture_group=capture_group, + ) + plain = Pattern(name="plain", regex=r"\b\d+\b", score=0.6) + recognizer = PatternRecognizer( + supported_entity="TEST", + patterns=[verbose_dependent, plain], + global_regex_flags=re.VERBOSE, + ) + + with caplog.at_level("WARNING", logger="presidio-analyzer"): + results = recognizer.analyze("secretvalue 123 secretother", ["TEST"]) + + assert [(result.start, result.end, result.score) for result in results] == [ + (12, 15, 0.6) + ] + # Logged once per analyze call, not once per match + warning = ( + f"Regex pattern 'verbose_dependent' has no capture group {capture_group!r} " + "when compiled with the regex flags in use, skipping." + ) + assert caplog.text.count(warning) == 1 + assert "secretvalue" not in caplog.text + assert "secretother" not in caplog.text + + # The warning does not depend on the text containing a candidate match + caplog.clear() + with caplog.at_level("WARNING", logger="presidio-analyzer"): + assert recognizer.analyze("no candidates here", ["TEST"]) == [] + assert caplog.text.count(warning) == 1 + + +def test_when_capture_group_set_then_recognizer_round_trips_through_dict(): + recognizer_dict = { + "supported_entity": "PASSWORD", + "patterns": [ + { + "name": "password", + "regex": r"Password: (?P\w+)", + "score": 0.5, + "capture_group": "password", + } + ], + } + + recognizer = PatternRecognizer.from_dict(recognizer_dict) + restored = PatternRecognizer.from_dict(recognizer.to_dict()) + + assert recognizer.patterns[0].capture_group == "password" + assert recognizer.to_dict()["patterns"] == recognizer_dict["patterns"] + results = restored.analyze("Password: 1234", ["PASSWORD"]) + assert len(results) == 1 + assert_result(results[0], "PASSWORD", 10, 14, 0.5) diff --git a/presidio-analyzer/tests/test_recognizer_registry_provider.py b/presidio-analyzer/tests/test_recognizer_registry_provider.py index f18f89b87..7699874eb 100644 --- a/presidio-analyzer/tests/test_recognizer_registry_provider.py +++ b/presidio-analyzer/tests/test_recognizer_registry_provider.py @@ -8,7 +8,8 @@ from presidio_analyzer.predefined_recognizers import SpacyRecognizer from presidio_analyzer.recognizer_registry import RecognizerRegistryProvider from presidio_analyzer.recognizer_registry.recognizers_loader_utils import RecognizerConfigurationLoader -from presidio_analyzer import RecognizerRegistry +from presidio_analyzer import AnalyzerEngine, RecognizerRegistry +from tests.mocks import NlpEngineMock def assert_default_configuration( @@ -117,6 +118,67 @@ def test_recognizer_registry_provider_explicit_deny_list_score_is_honored(): assert all(pattern.score == 0.3 for pattern in recognizer.patterns) +def test_recognizer_registry_provider_yaml_pattern_capture_group_reaches_recognizer( + tmp_path, +): + conf = tmp_path / "recognizers.yaml" + conf.write_text( + r""" +supported_languages: + - en +recognizers: + - name: PasswordRecognizer + type: custom + supported_language: en + supported_entity: PASSWORD + patterns: + - name: password value + regex: "password:\\s*(?P\\S+)" + score: 0.5 + capture_group: value +""" + ) + + registry = RecognizerRegistryProvider(conf_file=conf).create_recognizer_registry() + analyzer = AnalyzerEngine(registry=registry, nlp_engine=NlpEngineMock()) + + results = analyzer.analyze("my password: hunter2", language="en") + + assert registry.recognizers[0].patterns[0].capture_group == "value" + assert [ + (result.entity_type, result.start, result.end, result.score) + for result in results + ] == [("PASSWORD", 13, 20, 0.5)] + + +def test_recognizer_registry_provider_rejects_invalid_pattern_capture_group(): + with pytest.raises(ValueError) as exc_info: + RecognizerRegistryProvider( + registry_configuration={ + "supported_languages": ["en"], + "recognizers": [ + { + "name": "PasswordRecognizer", + "supported_entity": "PASSWORD", + "patterns": [ + { + "name": "password value", + "regex": r"password:\s*(\S+)", + "score": 0.5, + "capture_group": 2, + } + ], + } + ], + } + ) + + assert ( + "Invalid pattern 'password value': capture_group 2 is out of range: " + "regex defines 1 capture group(s)" + ) in str(exc_info.value.__cause__) + + def test_recognizer_registry_provider_omitted_thresholds_default_to_empty(): provider = RecognizerRegistryProvider( registry_configuration={ diff --git a/presidio-analyzer/tests/test_yaml_recognizer_models.py b/presidio-analyzer/tests/test_yaml_recognizer_models.py index 8add06b5d..4c5cf03ae 100644 --- a/presidio-analyzer/tests/test_yaml_recognizer_models.py +++ b/presidio-analyzer/tests/test_yaml_recognizer_models.py @@ -479,6 +479,84 @@ def test_custom_recognizer_config_invalid_score_range(): ) +@pytest.mark.parametrize( + "regex, capture_group", + [ + (r"id: (\d+)", 1), + (r"id: (?P\d+)", "value"), + (r"id: (\d+)", None), + ], +) +def test_custom_recognizer_config_pattern_capture_group_valid(regex, capture_group): + """A capture_group that exists in the regex is accepted and kept in the dump.""" + pattern = { + "name": "test", + "regex": regex, + "score": 0.5, + "capture_group": capture_group, + } + config = CustomRecognizerConfig( + name="test", + supported_entity="TEST", + patterns=[pattern] + ) + + assert config.model_dump()["patterns"] == [pattern] + + +@pytest.mark.parametrize( + "regex, capture_group, expected_message", + [ + ( + r"id: (\d+)", + 2, + "Invalid pattern 'test': capture_group 2 is out of range: " + "regex defines 1 capture group(s)", + ), + ( + r"id: (?P\d+)", + "number", + "Invalid pattern 'test': capture_group 'number' is not a named group " + "in the regex. Named groups: ['value']", + ), + ( + r"id: (\d+)", + -1, + "Invalid pattern 'test': capture_group must be a non-negative integer, " + "got -1", + ), + ( + r"id: (\d+)", + True, + "Invalid pattern 'test': capture_group must be an int or a str, got bool", + ), + ( + r"id: (\d+)", + [1], + "Invalid pattern 'test': capture_group must be an int or a str, got list", + ), + ], +) +def test_custom_recognizer_config_pattern_capture_group_invalid( + regex, capture_group, expected_message +): + """An invalid capture_group fails at parse time with an actionable message.""" + pattern = { + "name": "test", + "regex": regex, + "score": 0.5, + "capture_group": capture_group, + } + with pytest.raises(ValidationError) as exc_info: + CustomRecognizerConfig( + name="test", + supported_entity="TEST", + patterns=[pattern] + ) + + assert expected_message in str(exc_info.value) + + def test_custom_recognizer_config_no_patterns_or_deny_list(): """Test that custom recognizer must have patterns or deny_list.""" with pytest.raises(ValidationError) as exc_info: