fix: dedupe personal website candidates (#198)

* Fix website candidate dedupe

* Suppress heuristic URL backfill

* Refine website dedupe helpers

Author: michaelmwu

apps/worker/src/five08/worker/crm/resume_profile_processor.py

@@ -12,6 +12,7 @@
 from five08.clients.espo import EspoAPIError, EspoClient
 from five08.crm_normalization import (
     ROLE_NORMALIZATION_MAP,
+    normalized_website_identity_key,
     normalize_city,
     normalize_country,
     normalize_role,
@@ -1046,8 +1047,8 @@ def _coerce_website_links(self, value: Any) -> list[str]:
             normalized_link = self._normalize_website_url(raw_value.strip())
             if normalized_link is None:
                 continue
-            dedupe_key = normalized_link.casefold()
-            if dedupe_key in seen:
+            dedupe_key = normalized_website_identity_key(normalized_link)
+            if dedupe_key is None or dedupe_key in seen:
                 continue
             seen.add(dedupe_key)
             normalized.append(normalized_link)
@@ -1066,8 +1067,8 @@ def _merge_website_links(
             normalized = self._normalize_website_url(value)
             if not normalized:
                 continue
-            dedupe_key = normalized.casefold()
-            if dedupe_key in seen:
+            dedupe_key = normalized_website_identity_key(normalized)
+            if dedupe_key is None or dedupe_key in seen:
                 continue
             seen.add(dedupe_key)
             merged.append(normalized)
@@ -1078,8 +1079,8 @@ def _merge_website_links(
             normalized = self._normalize_website_url(value)
             if not normalized:
                 continue
-            dedupe_key = normalized.casefold()
-            if dedupe_key in seen:
+            dedupe_key = normalized_website_identity_key(normalized)
+            if dedupe_key is None or dedupe_key in seen:
                 continue
             seen.add(dedupe_key)
             merged.append(normalized)

packages/shared/src/five08/crm_normalization.py

@@ -707,3 +707,36 @@ def normalize_website_url(
     elif normalized.startswith("http://www."):
         normalized = normalized.replace("http://www.", "http://", 1)
     return normalized
+
+
+def normalized_website_identity_key(normalized_url: str) -> str | None:
+    try:
+        parsed = urlsplit(normalized_url)
+    except Exception:
+        return normalized_url.casefold()
+
+    netloc = parsed.netloc.casefold()
+    if netloc.startswith("www."):
+        netloc = netloc[4:]
+    path = re.sub(r"/+", "/", parsed.path or "").rstrip("/").casefold()
+    query = parsed.query.casefold()
+    key = f"{netloc}{path}"
+    if query:
+        key = f"{key}?{query}"
+    return key
+
+
+def website_identity_key(
+    value: str,
+    *,
+    allow_scheme_less: bool = True,
+    disallowed_host_predicate: Callable[[str], bool] | None = None,
+) -> str | None:
+    normalized = normalize_website_url(
+        value,
+        allow_scheme_less=allow_scheme_less,
+        disallowed_host_predicate=disallowed_host_predicate,
+    )
+    if normalized is None:
+        return None
+    return normalized_website_identity_key(normalized)

packages/shared/src/five08/resume_extractor.py

@@ -10,6 +10,7 @@
 
 from pydantic import BaseModel, ConfigDict, Field, ValidationError
 from five08.crm_normalization import (
+    normalized_website_identity_key as shared_normalized_website_identity_key,
     normalize_city as shared_normalize_city,
     normalize_country as shared_normalize_country,
     normalize_role as shared_normalize_role,
@@ -1120,10 +1121,10 @@ def _normalize_website_links(value: Any) -> list[str]:
         normalized_link = _normalize_website_url(candidate)
         if not normalized_link:
             continue
-        lower = normalized_link.lower()
-        if lower in seen:
+        dedupe_key = _website_identity_key(normalized_link)
+        if dedupe_key is None or dedupe_key in seen:
             continue
-        seen.add(lower)
+        seen.add(dedupe_key)
         normalized_links.append(normalized_link)
 
     return normalized_links
@@ -1187,7 +1188,9 @@ def _extract_website_url_candidates(
         if confidence < LLM_WEBSITE_URL_MIN_CONFIDENCE:
             continue
 
-        key = normalized_url.casefold()
+        key = _website_identity_key(normalized_url)
+        if key is None:
+            continue
         prior = normalized.get(key)
         if prior is not None and prior[2] >= confidence:
             continue
@@ -1250,6 +1253,10 @@ def _normalized_host(host: str | None) -> str:
     return normalized
 
 
+def _website_identity_key(value: str) -> str | None:
+    return shared_normalized_website_identity_key(value)
+
+
 def _host_matches_domain(host: str | None, domain: str) -> bool:
     normalized_host = _normalized_host(host)
     return normalized_host == domain or normalized_host.endswith(f".{domain}")
@@ -1587,6 +1594,7 @@ def _build_website_and_social_from_candidates(
 ) -> tuple[list[str], list[str]]:
     urls_to_consider: list[str] = []
     seen: set[str] = set()
+    has_llm_url_candidates = False
 
     for candidate_url, candidate_kind, candidate_confidence in llm_candidates:
         if candidate_kind == LLM_URL_CANDIDATE_KIND_PERSONAL:
@@ -1608,18 +1616,21 @@ def _build_website_and_social_from_candidates(
         ):
             continue
 
-        candidate_key = candidate_url.casefold()
-        if candidate_key in seen:
+        candidate_key = _website_identity_key(candidate_url)
+        if candidate_key is None or candidate_key in seen:
             continue
         seen.add(candidate_key)
         urls_to_consider.append(candidate_url)
+        has_llm_url_candidates = True
 
     for candidate_url, candidate_confidence in heuristic_candidates:
         if candidate_confidence < MIDDLE_WEBSITE_POSITION_SCALE:
             continue
+        if has_llm_url_candidates:
+            continue
 
-        candidate_key = candidate_url.casefold()
-        if candidate_key in seen:
+        candidate_key = _website_identity_key(candidate_url)
+        if candidate_key is None or candidate_key in seen:
             continue
         seen.add(candidate_key)
         urls_to_consider.append(candidate_url)
@@ -1647,8 +1658,8 @@ def _split_social_and_website_links(
             continue
         social_profile = _normalize_social_profile_url(candidate)
         if social_profile:
-            social_key = social_profile.casefold()
-            if social_key in seen_social:
+            social_key = _website_identity_key(social_profile)
+            if social_key is None or social_key in seen_social:
                 continue
             seen_social.add(social_key)
             social_links.append(social_profile)
@@ -1658,8 +1669,8 @@ def _split_social_and_website_links(
         if _is_personal_website_disallowed(candidate):
             continue
 
-        normal_key = candidate.casefold()
-        if normal_key in seen_normal:
+        normal_key = _website_identity_key(candidate)
+        if normal_key is None or normal_key in seen_normal:
             continue
         seen_normal.add(normal_key)
         normal_links.append(candidate)
@@ -3408,8 +3419,8 @@ def _extract_website_link_candidates(
                     end_index,
                 ):
                     continue
-            normalized_key = normalized_link.casefold()
-            if normalized_key in seen:
+            normalized_key = _website_identity_key(normalized_link)
+            if normalized_key is None or normalized_key in seen:
                 continue
             seen.add(normalized_key)
             normalized_links.append((normalized_link, confidence))

tests/unit/test_crm_normalization.py

@@ -9,6 +9,7 @@
     normalize_state,
     normalize_timezone,
     normalize_website_url,
+    website_identity_key,
 )
 
 
@@ -47,6 +48,12 @@ def test_normalize_website_url_respects_disallowed_host_predicate() -> None:
     )
 
 
+def test_website_identity_key_ignores_scheme_and_path_casing() -> None:
+    assert website_identity_key("http://Example.com/About") == website_identity_key(
+        "https://example.com/about/"
+    )
+
+
 def test_normalize_country_and_city() -> None:
     assert normalize_country(" united states ") == "United States"
     assert normalize_country("Taiwan") == "Taiwan"