diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/tlds.txt b/presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/tlds.txt new file mode 100644 index 0000000000..ed6e853fd3 --- /dev/null +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/tlds.txt @@ -0,0 +1,1441 @@ +# IANA Root Zone Database - Version 2026090900, Last Updated Wed Sep 9 07:07:01 2026 UTC +# Source: https://data.iana.org/TLD/tlds-alpha-by-domain.txt +# Regenerate with: python -m presidio_analyzer.predefined_recognizers.generic.update_tlds +aaa +aarp +abb +abbott +abbvie +abc +able +abogado +abudhabi +ac +academy +accenture +accountant +accountants +aco +actor +ad +ads +adult +ae +aeg +aero +aetna +af +afl +africa +ag +agakhan +agency +ai +aig +airbus +airforce +airtel +akdn +al +alibaba +alipay +allfinanz +allstate +ally +alsace +alstom +am +amazon +americanexpress +americanfamily +amex +amfam +amica +amsterdam +analytics +android +anquan +anz +ao +aol +apartments +app +apple +aq +aquarelle +ar +arab +aramco +archi +army +arpa +art +arte +as +asda +asia +associates +at +athleta +attorney +au +auction +audi +audible +audio +auspost +author +auto +autos +aw +aws +ax +axa +az +azure +ba +baby +baidu +banamex +band +bank +bar +barcelona +barclaycard +barclays +barefoot +bargains +baseball +basketball +bauhaus +bayern +bb +bbc +bbt +bbva +bcg +bcn +bd +be +beats +beauty +beer +berlin +best +bestbuy +bet +bf +bg +bh +bharti +bi +bible +bid +bike +bing +bingo +bio +biz +bj +black +blackfriday +blockbuster +blog +bloomberg +blue +bm +bms +bmw +bn +bnpparibas +bo +boats +boehringer +bofa +bom +bond +boo +book +booking +bosch +bostik +boston +bot +boutique +box +br +bradesco +bridgestone +broadway +broker +brother +brussels +bs +bt +build +builders +business +buy +buzz +bv +bw +by +bz +bzh +ca +cab +cafe +cal +call +calvinklein +cam +camera +camp +canon +capetown +capital +capitalone +car +caravan +cards +care +career +careers +cars +casa +case +cash +casino +cat +catering +catholic +cba +cbn +cbre +cc +cd +center +ceo +cern +cf +cfa +cfd +cg +ch +chanel +channel +charity +chase +chat +cheap +chintai +christmas +chrome +church +ci +cipriani +circle +cisco +citadel +citi +citic +city +ck +cl +claims +cleaning +click +clinic +clinique +clothing +cloud +club +clubmed +cm +cn +co +coach +codes +coffee +college +cologne +com +commbank +community +company +compare +computer +comsec +condos +construction +consulting +contact +contractors +cooking +cool +coop +corsica +country +coupon +coupons +courses +cpa +cr +credit +creditcard +creditunion +cricket +crown +crs +cruise +cruises +cu +cuisinella +cv +cw +cx +cy +cymru +cyou +cz +dad +dance +data +date +dating +datsun +day +dclk +dds +de +deal +dealer +deals +degree +delivery +dell +deloitte +delta +democrat +dental +dentist +desi +design +dev +dhl +diamonds +diet +digital +direct +directory +discount +discover +dish +diy +dj +dk +dm +dnp +do +docs +doctor +dog +domains +dot +download +drive +dtv +dubai +dupont +durban +dvag +dvr +dz +earth +eat +ec +eco +edeka +edu +education +ee +eg +email +emerck +energy +engineer +engineering +enterprises +epson +equipment +er +ericsson +erni +es +esq +estate +et +eu +eurovision +eus +events +exchange +expert +exposed +express +extraspace +fage +fail +fairwinds +faith +family +fan +fans +farm +farmers +fashion +fast +fedex +feedback +ferrari +ferrero +fi +fidelity +fido +film +final +finance +financial +fire +firestone +firmdale +fish +fishing +fit +fitness +fj +fk +flickr +flights +flir +florist +flowers +fly +fm +fo +foo +food +football +ford +forex +forsale +forum +foundation +fox +fr +free +fresenius +frl +frogans +frontier +ftr +fujitsu +fun +fund +furniture +futbol +fyi +ga +gal +gallery +gallo +gallup +game +games +gap +garden +gay +gb +gbiz +gd +gdn +ge +gea +gent +genting +george +gf +gg +ggee +gh +gi +gift +gifts +gives +giving +gl +glass +gle +global +globo +gm +gmail +gmbh +gmo +gmx +gn +godaddy +gold +goldpoint +golf +goodyear +goog +google +gop +got +gov +gp +gq +gr +grainger +graphics +gratis +green +gripe +grocery +group +gs +gt +gu +gucci +guge +guide +guitars +guru +gw +gy +hair +hamburg +hangout +haus +hbo +hdfc +hdfcbank +health +healthcare +help +helsinki +here +hermes +hiphop +hisamitsu +hitachi +hiv +hk +hkt +hm +hn +hockey +holdings +holiday +homedepot +homegoods +homes +homesense +honda +horse +hospital +host +hosting +hot +hotels +hotmail +house +how +hr +hsbc +ht +hu +hughes +hyatt +hyundai +ibm +icbc +ice +icu +id +ie +ieee +ifm +ikano +il +im +imamat +imdb +immo +immobilien +in +inc +industries +infiniti +info +ing +ink +institute +insurance +insure +int +international +intuit +investments +io +ipiranga +iq +ir +irish +is +ismaili +ist +istanbul +it +itau +itv +jaguar +java +jcb +je +jeep +jetzt +jewelry +jio +jll +jm +jmp +jnj +jo +jobs +joburg +jot +joy +jp +jpmorgan +jprs +juegos +juniper +kaufen +kddi +ke +kerryhotels +kerryproperties +kfh +kg +kh +ki +kia +kids +kim +kindle +kitchen +kiwi +km +kn +koeln +komatsu +kosher +kp +kpmg +kpn +kr +krd +kred +kuokgroup +kw +ky +kyoto +kz +la +lacaixa +lamborghini +lamer +land +landrover +lanxess +lasalle +lat +latino +latrobe +law +lawyer +lb +lc +lds +lease +leclerc +lefrak +legal +lego +lexus +lgbt +li +lidl +life +lifeinsurance +lifestyle +lighting +like +lilly +limited +limo +lincoln +link +live +living +lk +llc +llp +loan +loans +locker +locus +lol +london +lotte +lotto +love +lpl +lplfinancial +lr +ls +lt +ltd +ltda +lu +lundbeck +luxe +luxury +lv +ly +ma +madrid +maif +maison +makeup +man +management +mango +map +market +marketing +markets +marriott +marshalls +mattel +mba +mc +mckinsey +md +me +med +media +meet +melbourne +meme +memorial +men +menu +merck +merckmsd +mg +mh +miami +microsoft +mil +mini +mint +mit +mitsubishi +mk +ml +mlb +mls +mm +mma +mn +mo +mobi +mobile +moda +moe +moi +mom +monash +money +monster +mormon +mortgage +moscow +moto +motorcycles +mov +movie +mp +mq +mr +ms +msd +mt +mtn +mtr +mu +museum +music +mv +mw +mx +my +mz +na +nab +nagoya +name +navy +nba +nc +ne +nec +net +netbank +netflix +network +neustar +new +news +next +nextdirect +nexus +nf +nfl +ng +ngo +nhk +ni +nico +nike +nikon +ninja +nissan +nissay +nl +no +nokia +norton +now +nowruz +nowtv +np +nr +nra +nrw +ntt +nu +nyc +nz +obi +observer +office +okinawa +olayan +olayangroup +ollo +om +omega +one +ong +onl +online +ooo +open +oracle +orange +org +organic +origins +osaka +otsuka +ott +ovh +pa +page +panasonic +paris +pars +partners +parts +party +pay +pccw +pe +pet +pf +pfizer +pg +ph +pharmacy +phd +philips +phone +photo +photography +photos +physio +pics +pictet +pictures +pid +pin +ping +pink +pioneer +pizza +pk +pl +place +play +playstation +plumbing +plus +pm +pn +pnc +pohl +poker +politie +porn +post +pr +praxi +press +prime +pro +prod +productions +prof +progressive +promo +properties +property +protection +pru +prudential +ps +pt +pub +pw +pwc +py +qa +qpon +quebec +quest +racing +radio +re +read +realestate +realtor +realty +recipes +red +redumbrella +rehab +reise +reisen +reit +reliance +ren +rent +rentals +repair +report +republican +rest +restaurant +review +reviews +rexroth +rich +richardli +ricoh +ril +rio +rip +ro +rocks +rodeo +rogers +room +rs +rsvp +ru +rugby +ruhr +run +rw +rwe +ryukyu +sa +saarland +safe +safety +sakura +sale +salon +samsclub +samsung +sandvik +sandvikcoromant +sanofi +sap +sarl +sas +save +saxo +sb +sbi +sbs +sc +scb +schaeffler +schmidt +scholarships +school +schule +schwarz +science +scot +sd +se +search +seat +secure +security +seek +select +sener +services +seven +sew +sex +sexy +sfr +sg +sh +shangrila +sharp +shell +shia +shiksha +shoes +shop +shopping +shouji +show +si +silk +sina +singles +site +sj +sk +ski +skin +sky +skype +sl +sling +sm +smart +smile +sn +sncf +so +soccer +social +softbank +software +sohu +solar +solutions +song +sony +soy +spa +space +sport +spot +sr +srl +ss +st +stada +staples +star +statebank +statefarm +stc +stcgroup +stockholm +storage +store +stream +studio +study +style +su +sucks +supplies +supply +support +surf +surgery +suzuki +sv +swatch +swiss +sx +sy +sydney +systems +sz +tab +taipei +talk +taobao +target +tatamotors +tatar +tattoo +tax +taxi +tc +tci +td +tdk +team +tech +technology +tel +temasek +tennis +teva +tf +tg +th +thd +theater +theatre +tiaa +tickets +tienda +tips +tires +tirol +tj +tjmaxx +tjx +tk +tkmaxx +tl +tm +tmall +tn +to +today +tokyo +tools +top +toray +toshiba +total +tours +town +toyota +toys +tr +trade +trading +training +travel +travelers +travelersinsurance +trust +trv +tt +tube +tui +tunes +tushu +tv +tvs +tw +tz +ua +ubank +ubs +ug +uk +unicom +university +uno +uol +ups +us +uy +uz +va +vacations +vana +vanguard +vc +ve +vegas +ventures +verisign +versicherung +vet +vg +vi +viajes +video +vig +viking +villas +vin +vip +virgin +visa +vision +viva +vivo +vlaanderen +vn +vodka +volvo +vote +voting +voto +voyage +vu +wales +walmart +walter +wang +wanggou +watch +watches +weather +weatherchannel +web +webcam +weber +website +wed +wedding +weibo +weir +wf +whoswho +wien +wiki +williamhill +win +windows +wine +winners +wme +woodside +work +works +world +wow +ws +wtc +wtf +xbox +xerox +xihuan +xin +xn--11b4c3d +xn--1ck2e1b +xn--1qqw23a +xn--2scrj9c +xn--30rr7y +xn--3bst00m +xn--3ds443g +xn--3e0b707e +xn--3hcrj9c +xn--3pxu8k +xn--42c2d9a +xn--45br5cyl +xn--45brj9c +xn--45q11c +xn--4dbrk0ce +xn--4gbrim +xn--54b7fta0cc +xn--55qw42g +xn--55qx5d +xn--5su34j936bgsg +xn--5tzm5g +xn--6frz82g +xn--6qq986b3xl +xn--80adxhks +xn--80ao21a +xn--80aqecdr1a +xn--80asehdb +xn--80aswg +xn--8y0a063a +xn--90a3ac +xn--90ae +xn--90ais +xn--9dbq2a +xn--9et52u +xn--9krt00a +xn--b4w605ferd +xn--bck1b9a5dre4c +xn--c1avg +xn--c2br7g +xn--cck2b3b +xn--cckwcxetd +xn--cg4bki +xn--clchc0ea0b2g2a9gcd +xn--czr694b +xn--czrs0t +xn--czru2d +xn--d1acj3b +xn--d1alf +xn--e1a4c +xn--eckvdtc9d +xn--efvy88h +xn--fct429k +xn--fhbei +xn--fiq228c5hs +xn--fiq64b +xn--fiqs8s +xn--fiqz9s +xn--fjq720a +xn--flw351e +xn--fpcrj9c3d +xn--fzc2c9e2c +xn--fzys8d69uvgm +xn--g2xx48c +xn--gckr3f0f +xn--gecrj9c +xn--gk3at1e +xn--h2breg3eve +xn--h2brj9c +xn--h2brj9c8c +xn--hxt814e +xn--i1b6b1a6a2e +xn--imr513n +xn--io0a7i +xn--j1aef +xn--j1amh +xn--j6w193g +xn--jlq480n2rg +xn--jvr189m +xn--kcrx77d1x4a +xn--kprw13d +xn--kpry57d +xn--kput3i +xn--l1acc +xn--lgbbat1ad8j +xn--mgb9awbf +xn--mgba3a3ejt +xn--mgba3a4f16a +xn--mgba7c0bbn0a +xn--mgbaam7a8h +xn--mgbab2bd +xn--mgbah1a3hjkrd +xn--mgbai9azgqp6j +xn--mgbayh7gpa +xn--mgbbh1a +xn--mgbbh1a71e +xn--mgbc0a9azcg +xn--mgbca7dzdo +xn--mgbcpq6gpa1a +xn--mgberp4a5d4ar +xn--mgbgu82a +xn--mgbi4ecexp +xn--mgbpl2fh +xn--mgbt3dhd +xn--mgbtx2b +xn--mgbx4cd0ab +xn--mix891f +xn--mk1bu44c +xn--mxtq1m +xn--ngbc5azd +xn--ngbe9e0a +xn--ngbrx +xn--node +xn--nqv7f +xn--nqv7fs00ema +xn--nyqy26a +xn--o3cw4h +xn--ogbpf8fl +xn--otu796d +xn--p1acf +xn--p1ai +xn--pgbs0dh +xn--pssy2u +xn--q7ce6a +xn--q9jyb4c +xn--qcka1pmc +xn--qxa6a +xn--qxam +xn--rhqv96g +xn--rovu88b +xn--rvc1e0am3e +xn--s9brj9c +xn--ses554g +xn--t60b56a +xn--tckwe +xn--tiq49xqyj +xn--unup4y +xn--vermgensberater-ctb +xn--vermgensberatung-pwb +xn--vhquv +xn--vuq861b +xn--w4r85el8fhu5dnra +xn--w4rs40l +xn--wgbh1c +xn--wgbl6a +xn--xhq521b +xn--xkc2al3hye2a +xn--xkc2dl3a5ee0h +xn--y9a3aq +xn--yfro4i67o +xn--ygbi2ammx +xn--zfr164b +xxx +xyz +yachts +yahoo +yamaxun +yandex +ye +yodobashi +yoga +yokohama +you +youtube +yt +yun +za +zappos +zara +zero +zip +zm +zone +zuerich +zw diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/update_tlds.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/update_tlds.py new file mode 100644 index 0000000000..faa38ca499 --- /dev/null +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/update_tlds.py @@ -0,0 +1,41 @@ +"""Refresh the vendored IANA root zone TLD list used by ``UrlRecognizer``.""" + +import sys +from pathlib import Path +from urllib.request import urlopen + +IANA_URL = "https://data.iana.org/TLD/tlds-alpha-by-domain.txt" +TLD_FILE = Path(__file__).parent / "tlds.txt" + + +def update(url: str = IANA_URL, path: Path = TLD_FILE) -> int: + """ + Download the IANA root zone list and rewrite the vendored copy. + + :param url: Location of the IANA ``tlds-alpha-by-domain.txt`` file + :param path: File to rewrite + :return: Number of TLDs written + """ + with urlopen(url) as response: # noqa: S310 - fixed https URL + lines = response.read().decode("utf-8").splitlines() + + version = next((line for line in lines if line.startswith("#")), "# unknown") + tlds = sorted( + line.strip().lower() for line in lines if line.strip() and "#" not in line + ) + if len(tlds) < 1000: + raise ValueError(f"Refusing to write a suspiciously short list ({len(tlds)})") + + header = [ + f"# IANA Root Zone Database - {version.lstrip('# ')}", + f"# Source: {url}", + "# Regenerate with: " + "python -m presidio_analyzer.predefined_recognizers.generic.update_tlds", + ] + path.write_text("\n".join(header + tlds) + "\n", encoding="utf-8") + return len(tlds) + + +if __name__ == "__main__": + count = update() + print(f"Wrote {count} TLDs to {TLD_FILE}", file=sys.stderr) diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/url_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/url_recognizer.py index 310fa78d90..86bb82c270 100644 --- a/presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/url_recognizer.py +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/generic/url_recognizer.py @@ -1,7 +1,98 @@ -from typing import List, Optional +from pathlib import Path +from typing import Dict, Iterable, List, Optional + +import regex as re from presidio_analyzer import Pattern, PatternRecognizer +TLD_FILE = Path(__file__).parent / "tlds.txt" + +# Sentinel key marking a complete TLD inside the prefix tree built by +# ``tlds_to_regex``. The empty string can never be a trie edge, since edges are +# single characters. +_TERMINAL = "" + + +def load_tlds(path: Path = TLD_FILE) -> List[str]: + """ + Read the vendored IANA root zone TLD list. + + :param path: File holding one lowercase TLD per line, ``#`` for comments + :return: The list of TLDs, in file order + """ + with open(path, encoding="utf-8") as tld_file: + return [ + line.strip().lower() + for line in tld_file + if line.strip() and not line.startswith("#") + ] + + +def tlds_to_regex(tlds: Iterable[str]) -> str: + """ + Compress a list of TLDs into a prefix-tree (trie) regex alternation. + + A flat ``(?:aaa)|(?:aarp)|...`` alternation over the whole root zone forces + the engine to retry every branch at the same position. Sharing common + prefixes instead - ``bar(?:c(?:elona|lay(?:card|s))|efoot|gains)?`` - keeps + the pattern smaller and lets the engine discard whole subtrees on a single + character. + + :param tlds: TLDs to match, case-insensitively + :return: A non-capturing regex group matching exactly those TLDs + """ + root: Dict = {} + for tld in tlds: + node = root + for char in tld.lower(): + node = node.setdefault(char, {}) + node[_TERMINAL] = None + + return _branch_to_regex(root) + + +def _branch_to_regex(node: Dict) -> str: + """ + Render a single trie node as a regex fragment. + + :param node: Trie node to render + :return: Regex fragment, or an empty string for a leaf node + """ + edges = sorted(char for char in node if char != _TERMINAL) + if not edges: + return "" + + # Edges leading straight to a leaf collapse into one character class. + alternatives = [] + leaf_chars = [] + for char in edges: + suffix = _branch_to_regex(node[char]) + if suffix: + alternatives.append(re.escape(char) + suffix) + else: + leaf_chars.append(char) + if leaf_chars: + alternatives.append(_to_char_class(leaf_chars)) + + # A node that is both terminal and has edges (e.g. `bar` and `barclays`) + # must stay optional, so it always needs the enclosing group. + if len(alternatives) == 1 and _TERMINAL not in node: + return alternatives[0] + closing = ")?" if _TERMINAL in node else ")" + return "(?:" + "|".join(alternatives) + closing + + +def _to_char_class(chars: List[str]) -> str: + """ + Turn single characters into a character class, or a literal if there is one. + + :param chars: Characters to match + :return: Regex fragment matching any one of them + """ + if len(chars) == 1: + return re.escape(chars[0]) + return "[" + "".join(re.escape(char) for char in chars) + "]" + class UrlRecognizer(PatternRecognizer): """ @@ -18,7 +109,16 @@ class UrlRecognizer(PatternRecognizer): :param supported_entity: The entity this recognizer can detect """ - BASE_URL_REGEX = r"((www\d{0,3}[.])?[a-z0-9.\-]{1,253}[.](?:(?:com)|(?:edu)|(?:gov)|(?:int)|(?:mil)|(?:net)|(?:onl)|(?:org)|(?:pro)|(?:red)|(?:tel)|(?:uno)|(?:xxx)|(?:academy)|(?:accountant)|(?:accountants)|(?:actor)|(?:adult)|(?:africa)|(?:agency)|(?:airforce)|(?:apartments)|(?:app)|(?:archi)|(?:army)|(?:art)|(?:asia)|(?:associates)|(?:attorney)|(?:auction)|(?:audio)|(?:auto)|(?:autos)|(?:baby)|(?:band)|(?:bar)|(?:bargains)|(?:beer)|(?:berlin)|(?:best)|(?:bet)|(?:bid)|(?:bike)|(?:bio)|(?:black)|(?:blackfriday)|(?:blog)|(?:blue)|(?:boats)|(?:bond)|(?:boo)|(?:boston)|(?:bot)|(?:boutique)|(?:build)|(?:builders)|(?:business)|(?:buzz)|(?:cab)|(?:cafe)|(?:cam)|(?:camera)|(?:camp)|(?:capital)|(?:car)|(?:cards)|(?:care)|(?:careers)|(?:cars)|(?:casa)|(?:cash)|(?:casino)|(?:catering)|(?:center)|(?:ceo)|(?:cfd)|(?:charity)|(?:chat)|(?:cheap)|(?:christmas)|(?:church)|(?:city)|(?:claims)|(?:cleaning)|(?:click)|(?:clinic)|(?:clothing)|(?:cloud)|(?:club)|(?:codes)|(?:coffee)|(?:college)|(?:com)|(?:community)|(?:company)|(?:computer)|(?:condos)|(?:construction)|(?:consulting)|(?:contact)|(?:contractors)|(?:cooking)|(?:cool)|(?:coupons)|(?:courses)|(?:credit)|(?:creditcard)|(?:cricket)|(?:cruises)|(?:cyou)|(?:dad)|(?:dance)|(?:date)|(?:dating)|(?:day)|(?:degree)|(?:delivery)|(?:democrat)|(?:dental)|(?:dentist)|(?:desi)|(?:design)|(?:dev)|(?:diamonds)|(?:diet)|(?:digital)|(?:direct)|(?:directory)|(?:discount)|(?:doctor)|(?:dog)|(?:domains)|(?:download)|(?:earth)|(?:eco)|(?:education)|(?:email)|(?:energy)|(?:engineer)|(?:engineering)|(?:enterprises)|(?:equipment)|(?:esq)|(?:estate)|(?:events)|(?:exchange)|(?:expert)|(?:exposed)|(?:express)|(?:fail)|(?:faith)|(?:family)|(?:fans)|(?:farm)|(?:fashion)|(?:feedback)|(?:film)|(?:finance)|(?:financial)|(?:fish)|(?:fishing)|(?:fit)|(?:fitness)|(?:flights)|(?:florist)|(?:flowers)|(?:football)|(?:forsale)|(?:foundation)|(?:fun)|(?:fund)|(?:furniture)|(?:futbol)|(?:fyi)|(?:gallery)|(?:game)|(?:games)|(?:garden)|(?:gay)|(?:gdn)|(?:gifts)|(?:gives)|(?:giving)|(?:glass)|(?:global)|(?:gmbh)|(?:gold)|(?:golf)|(?:graphics)|(?:gratis)|(?:green)|(?:gripe)|(?:group)|(?:guide)|(?:guitars)|(?:guru)|(?:hair)|(?:hamburg)|(?:haus)|(?:health)|(?:healthcare)|(?:help)|(?:hiphop)|(?:hockey)|(?:holdings)|(?:holiday)|(?:homes)|(?:horse)|(?:hospital)|(?:host)|(?:hosting)|(?:house)|(?:how)|(?:icu)|(?:info)|(?:ink)|(?:institute)|(?:insure)|(?:international)|(?:investments)|(?:irish)|(?:jewelry)|(?:jetzt)|(?:juegos)|(?:kaufen)|(?:kids)|(?:kitchen)|(?:kiwi)|(?:krd)|(?:kyoto)|(?:land)|(?:lat)|(?:law)|(?:lawyer)|(?:lease)|(?:legal)|(?:lgbt)|(?:life)|(?:lighting)|(?:limited)|(?:limo)|(?:link)|(?:live)|(?:loan)|(?:loans)|(?:lol)|(?:london)|(?:love)|(?:ltd)|(?:ltda)|(?:luxury)|(?:maison)|(?:management)|(?:market)|(?:marketing)|(?:markets)|(?:mba)|(?:media)|(?:melbourne)|(?:meme)|(?:memorial)|(?:men)|(?:miami)|(?:mobi)|(?:moda)|(?:moe)|(?:mom)|(?:money)|(?:monster)|(?:mortgage)|(?:motorcycles)|(?:mov)|(?:movie)|(?:nagoya)|(?:name)|(?:navy)|(?:network)|(?:new)|(?:news)|(?:ngo)|(?:ninja)|(?:now)|(?:nyc)|(?:observer)|(?:okinawa)|(?:one)|(?:ong)|(?:onl)|(?:online)|(?:organic)|(?:osaka)|(?:page)|(?:paris)|(?:partners)|(?:parts)|(?:party)|(?:pet)|(?:phd)|(?:photo)|(?:photography)|(?:photos)|(?:pics)|(?:pictures)|(?:pink)|(?:pizza)|(?:place)|(?:plumbing)|(?:plus)|(?:poker)|(?:porn)|(?:press)|(?:pro)|(?:productions)|(?:prof)|(?:promo)|(?:properties)|(?:property)|(?:protection)|(?:pub)|(?:quest)|(?:racing)|(?:recipes)|(?:red)|(?:rehab)|(?:reise)|(?:reisen)|(?:rent)|(?:rentals)|(?:repair)|(?:report)|(?:republican)|(?:rest)|(?:restaurant)|(?:review)|(?:reviews)|(?:rip)|(?:rocks)|(?:rodeo)|(?:rsvp)|(?:run)|(?:saarland)|(?:sale)|(?:salon)|(?:sarl)|(?:sbs)|(?:school)|(?:schule)|(?:science)|(?:services)|(?:sex)|(?:sexy)|(?:sh)|(?:shoes)|(?:shop)|(?:shopping)|(?:show)|(?:singles)|(?:site)|(?:skin)|(?:soccer)|(?:social)|(?:software)|(?:solar)|(?:solutions)|(?:soy)|(?:space)|(?:spiegel)|(?:study)|(?:style)|(?:sucks)|(?:supply)|(?:support)|(?:surf)|(?:surgery)|(?:systems)|(?:tax)|(?:taxi)|(?:team)|(?:tech)|(?:technology)|(?:tel)|(?:theater)|(?:tips)|(?:tires)|(?:today)|(?:tools)|(?:top)|(?:tours)|(?:town)|(?:toys)|(?:trade)|(?:training)|(?:tube)|(?:uk)|(?:university)|(?:uno)|(?:vacations)|(?:ventures)|(?:vet)|(?:video)|(?:villas)|(?:vin)|(?:vip)|(?:vision)|(?:vlaanderen)|(?:vodka)|(?:vote)|(?:voting)|(?:voyage)|(?:wales)|(?:wang)|(?:watch)|(?:webcam)|(?:website)|(?:wedding)|(?:wiki)|(?:wine)|(?:work)|(?:works)|(?:world)|(?:wtf)|(?:xyz)|(?:yoga)|(?:yokohama)|(?:you)|(?:zone)|(?:ac)|(?:ad)|(?:ae)|(?:af)|(?:ag)|(?:ai)|(?:al)|(?:am)|(?:an)|(?:ao)|(?:aq)|(?:ar)|(?:as)|(?:at)|(?:au)|(?:aw)|(?:ax)|(?:az)|(?:ba)|(?:bb)|(?:bd)|(?:be)|(?:bf)|(?:bg)|(?:bh)|(?:bi)|(?:bj)|(?:bm)|(?:bn)|(?:bo)|(?:br)|(?:bs)|(?:bt)|(?:bv)|(?:bw)|(?:by)|(?:bz)|(?:ca)|(?:cc)|(?:cd)|(?:cf)|(?:cg)|(?:ch)|(?:ci)|(?:ck)|(?:cl)|(?:cm)|(?:cn)|(?:co)|(?:cr)|(?:cu)|(?:cv)|(?:cw)|(?:cx)|(?:cy)|(?:cz)|(?:de)|(?:dj)|(?:dk)|(?:dm)|(?:do)|(?:dz)|(?:ec)|(?:ee)|(?:eg)|(?:er)|(?:es)|(?:et)|(?:eu)|(?:fi)|(?:fj)|(?:fk)|(?:fm)|(?:fo)|(?:fr)|(?:ga)|(?:gb)|(?:gd)|(?:ge)|(?:gf)|(?:gg)|(?:gh)|(?:gi)|(?:gl)|(?:gm)|(?:gn)|(?:gp)|(?:gq)|(?:gr)|(?:gs)|(?:gt)|(?:gu)|(?:gw)|(?:gy)|(?:hk)|(?:hm)|(?:hn)|(?:hr)|(?:ht)|(?:hu)|(?:id)|(?:ie)|(?:il)|(?:im)|(?:in)|(?:io)|(?:iq)|(?:ir)|(?:is)|(?:it)|(?:je)|(?:jm)|(?:jo)|(?:jp)|(?:ke)|(?:kg)|(?:kh)|(?:ki)|(?:km)|(?:kn)|(?:kp)|(?:kr)|(?:kw)|(?:ky)|(?:kz)|(?:la)|(?:lb)|(?:lc)|(?:li)|(?:lk)|(?:lr)|(?:ls)|(?:lt)|(?:lu)|(?:lv)|(?:ly)|(?:ma)|(?:mc)|(?:md)|(?:me)|(?:mg)|(?:mh)|(?:mk)|(?:ml)|(?:mm)|(?:mn)|(?:mo)|(?:mp)|(?:mq)|(?:mr)|(?:ms)|(?:mt)|(?:mu)|(?:mv)|(?:mw)|(?:mx)|(?:my)|(?:mz)|(?:na)|(?:nc)|(?:ne)|(?:nf)|(?:ng)|(?:ni)|(?:nl)|(?:no)|(?:np)|(?:nr)|(?:nu)|(?:nz)|(?:om)|(?:pa)|(?:pe)|(?:pf)|(?:pg)|(?:ph)|(?:pk)|(?:pl)|(?:pm)|(?:pn)|(?:pr)|(?:ps)|(?:pt)|(?:pw)|(?:py)|(?:qa)|(?:re)|(?:ro)|(?:rs)|(?:ru)|(?:rw)|(?:sa)|(?:sb)|(?:sc)|(?:sd)|(?:se)|(?:sg)|(?:sh)|(?:si)|(?:sj)|(?:sk)|(?:sl)|(?:sm)|(?:sn)|(?:so)|(?:sr)|(?:st)|(?:su)|(?:sv)|(?:sx)|(?:sy)|(?:sz)|(?:tc)|(?:td)|(?:tf)|(?:tg)|(?:th)|(?:tj)|(?:tk)|(?:tl)|(?:tm)|(?:tn)|(?:to)|(?:tp)|(?:tr)|(?:tt)|(?:tv)|(?:tw)|(?:tz)|(?:ua)|(?:ug)|(?:uk)|(?:us)|(?:uy)|(?:uz)|(?:va)|(?:vc)|(?:ve)|(?:vg)|(?:vi)|(?:vn)|(?:vu)|(?:wf)|(?:ws)|(?:ye)|(?:yt)|(?:za)|(?:zm)|(?:zw))(?:/[^\s()<>\"']*)?)" # noqa: E501 + TLDS = load_tlds() + + BASE_URL_REGEX = ( + r"((www\d{0,3}[.])?[a-z0-9.\-]{1,253}[.]" + + tlds_to_regex(TLDS) + # Require the TLD to end the host. Without this the engine happily + # stops on a shorter TLD that merely prefixes the real one. + + r"(?![a-z0-9\-])" + + r"(?:/[^\s()<>\"']*)?)" + ) PATTERNS = [ Pattern("Standard Url", "(?i)(?:https?://)" + BASE_URL_REGEX, 0.6), diff --git a/presidio-analyzer/pyproject.toml b/presidio-analyzer/pyproject.toml index 116203466c..11ba912c2f 100644 --- a/presidio-analyzer/pyproject.toml +++ b/presidio-analyzer/pyproject.toml @@ -20,7 +20,7 @@ classifiers = [ keywords = ["presidio_analyzer"] urls = {Homepage = "https://github.com/data-privacy-stack/presidio"} readme = "README.md" -include = ["conf/*",] +include = ["conf/*", "predefined_recognizers/generic/tlds.txt"] requires-python = ">=3.10,<3.15" dependencies = [ diff --git a/presidio-analyzer/tests/test_url_recognizer.py b/presidio-analyzer/tests/test_url_recognizer.py index 80c4cb68fc..2122ad43b8 100644 --- a/presidio-analyzer/tests/test_url_recognizer.py +++ b/presidio-analyzer/tests/test_url_recognizer.py @@ -1,9 +1,14 @@ import time import pytest +import regex as re from tests import assert_result from presidio_analyzer.predefined_recognizers import UrlRecognizer +from presidio_analyzer.predefined_recognizers.generic.url_recognizer import ( + load_tlds, + tlds_to_regex, +) @pytest.fixture(scope="module") @@ -35,10 +40,24 @@ def entities(): ('"https://presidio.dataprivacystack.org/"', 1, ((0, 40),), 0.6), ("'https://presidio.dataprivacystack.org/'", 1, ((0, 40),), 0.6), + # Brand TLDs, incl. ones that share a prefix with a shorter TLD + # (.bank/.ba, .barclays/.bar, .americanexpress/.am) + ("https://www.chase.bank/login", 1, ((0, 28),), 0.6,), + ("chase.bank", 1, ((0, 10),), 0.5,), + ("https://home.barclays", 1, ((0, 21),), 0.6,), + ("www.americanexpress", 1, ((0, 19),), 0.5,), + ("wellsfargo.insurance", 1, ((0, 20),), 0.5,), + ("navy.creditunion", 1, ((0, 16),), 0.5,), + ("www.microsoft", 1, ((0, 13),), 0.5,), + ("'www.microsoft'", 1, ((0, 15),), 0.5,), + + # A TLD must end the host, so a trailing non-TLD label is excluded + ("Visit www.example.com.Then call", 1, ((6, 21),), 0.5,), + # Invalid URLs - ("www.microsoft", 0, (), 0), ("http://microsoft", 0, (), 0), - ("'www.microsoft'", 0, (), 0), + ("archive.tar.gz", 0, (), 0), + ("nosuch.invalidtld", 0, (), 0), # fmt: on ], ) @@ -71,3 +90,19 @@ def test_repeated_dot_input_does_not_backtrack(recognizer, entities): elapsed = time.time() - start assert results == [] assert elapsed < 15 + +def test_when_trie_regex_then_matches_exactly_the_tld_list(): + tlds = load_tlds() + anchored = re.compile("^(?:" + tlds_to_regex(tlds) + ")$") + tld_set = set(tlds) + + assert all(anchored.match(tld) for tld in tlds) + # Every proper prefix and one-character extension of a real TLD must only + # match if it is itself a TLD - this is the .bank/.ba failure mode. + for tld in tlds: + for candidate in [tld[:i] for i in range(1, len(tld))] + [ + tld + char for char in "abz0-" + ]: + assert (anchored.match(candidate) is not None) == (candidate in tld_set), ( + candidate + )