fix(run): survive malformed links and explain unreachable companies

prospector/extract.py

@@ -78,10 +78,17 @@ def discover_extra_pages(homepage_html: str, base_url: str) -> list[tuple[str, s
         href = (node.attributes.get("href") or "").strip()
         if not href or href.startswith(("#", "mailto:", "tel:", "javascript:")):
             continue
-        absolute = urljoin(base_url.rstrip("/") + "/", href)
-        if urlparse(absolute).hostname != base_host:
+        try:
+            absolute = urljoin(base_url.rstrip("/") + "/", href)
+            if urlparse(absolute).hostname != base_host:
+                continue
+            path = urlparse(absolute).path.lower()
+        except ValueError:
+            # A malformed href is the page author's mistake, not a reason to
+            # lose the company. An unfilled template placeholder such as
+            # "http://[BookingLink]" reads as an IPv6 literal to urlsplit
+            # (3.11.4+), which then raises from ipaddress. Skip the link.
             continue
-        path = urlparse(absolute).path.lower()
         for kind, keywords in PAGE_KEYWORDS.items():
             if kind not in found and any(k in path for k in keywords):
                 found[kind] = absolute

prospector/models.py

@@ -61,6 +61,10 @@ class ResearchResult:
     # pages rather than supplied in the input row.
     email_evidence: "Evidence | None" = None
     city: str | None = None
+    # How many of the company's own pages were actually read. Distinguishes
+    # "no site to read" from "read the site, nothing published" when explaining
+    # an unreachable company (008 FR-010).
+    pages_fetched: int = 0
     sources_consulted: list[str] = field(default_factory=list)
     failures: list[str] = field(default_factory=list)
 

prospector/pipeline.py

@@ -128,7 +128,7 @@ def _process_company(
 ) -> tuple[Prospect, Draft | None]:
     research = _research(company, settings, fetcher, verbose=verbose)
     if not company.email:
-        raise NoEmailFound(company.bucket_reason or "no email address found")
+        raise NoEmailFound(_no_email_reason(research))
     prospect = _score(company, research, settings)
     draft: Draft | None = None
     if no_llm:
@@ -147,6 +147,21 @@ def _process_company(
     return prospect, draft
 
 
+def _no_email_reason(research: ResearchResult) -> str:
+    """Why this company is unreachable, judged AFTER recovery ran (008 FR-010).
+
+    The ingest-time reason describes the input row ("blank email"), so reporting
+    it here reads as though recovery never happened. What an operator needs is
+    which stage ran out of road: a keyword returning listings with no websites
+    is a sourcing problem, while sites that publish no address is a vertical
+    that simply prefers contact forms."""
+    if not research.website:
+        return "no website could be resolved"
+    if not research.pages_fetched:
+        return f"no page could be fetched from {research.website}"
+    return "no published address on any fetched page"
+
+
 def _research(company: Company, settings: Settings, fetcher: Fetcher, *, verbose: bool) -> ResearchResult:
     research = ResearchResult(website=company.website)
     info = resolve.resolve(company, settings, fetcher)
@@ -165,6 +180,8 @@ def _research(company: Company, settings: Settings, fetcher: Fetcher, *, verbose
                 if html is not None:
                     pages.append(extracting.PageContent(kind, url, html))
 
+    research.pages_fetched = len(pages)
+
     # 008 FR-006: no supplied address -> look for one the company publishes on
     # the pages we already fetched. No new request is made (FR-011).
     if not company.email and pages:

tests/integration/test_email_recovery_batch.py

@@ -76,6 +76,12 @@ def test_unreachable_company_is_named(self, tmp_path, stubs):
         summary, _ = run(tmp_path, stubs)
         assert [name for name, _ in summary.skipped_companies] == ["Unreachable Ducts"]
 
+    def test_skip_reason_describes_the_outcome_not_the_input_row(self, tmp_path, stubs):
+        """FR-010: "blank email" restates the CSV. The operator needs to know
+        which stage ran out of road — the site was read and published nothing."""
+        summary, _ = run(tmp_path, stubs)
+        assert summary.skipped_companies[0][1] == "no published address on any fetched page"
+
     def test_recovered_address_lands_in_the_note(self, tmp_path, stubs):
         _, vault_dir = run(tmp_path, stubs)
         text = (vault_dir / "recoverable-ducts.md").read_text(encoding="utf-8")

tests/unit/test_extract.py

@@ -25,6 +25,14 @@ def test_external_links_ignored(self):
         html = '<a href="https://other.com/about">About</a>'
         assert discover_extra_pages(html, "https://acmeduct.com") == []
 
+    def test_malformed_href_does_not_lose_the_page(self):
+        # An unfilled template placeholder reads as an IPv6 literal to urlsplit
+        # and raises from ipaddress. One bad link must not cost the company.
+        html = '<a href="http://[BookingLink]">Book</a><a href="/contact">c</a>'
+        assert discover_extra_pages(html, "https://acmeduct.com") == [
+            ("contact", "https://acmeduct.com/contact")
+        ]
+
     def test_capped_at_three(self):
         html = (
             '<a href="/about">a</a><a href="/team">t</a>'

tests/unit/test_pipeline_reasons.py

@@ -0,0 +1,20 @@
+"""008 FR-010: an unreachable company is explained by what research found, not
+by what the input row looked like. The three reasons separate a sourcing problem
+(listings with no websites) from a vertical that publishes no addresses.
+"""
+
+from prospector.models import ResearchResult
+from prospector.pipeline import _no_email_reason
+
+
+class TestNoEmailReason:
+    def test_no_website_resolved(self):
+        assert _no_email_reason(ResearchResult()) == "no website could be resolved"
+
+    def test_website_resolved_but_unreadable(self):
+        research = ResearchResult(website="https://acmeduct.com", pages_fetched=0)
+        assert _no_email_reason(research) == "no page could be fetched from https://acmeduct.com"
+
+    def test_pages_read_but_nothing_published(self):
+        research = ResearchResult(website="https://acmeduct.com", pages_fetched=3)
+        assert _no_email_reason(research) == "no published address on any fetched page"