diff --git a/prospector/extract.py b/prospector/extract.py index 6c3faa3..cdb8342 100644 --- a/prospector/extract.py +++ b/prospector/extract.py @@ -78,10 +78,17 @@ def discover_extra_pages(homepage_html: str, base_url: str) -> list[tuple[str, s href = (node.attributes.get("href") or "").strip() if not href or href.startswith(("#", "mailto:", "tel:", "javascript:")): continue - absolute = urljoin(base_url.rstrip("/") + "/", href) - if urlparse(absolute).hostname != base_host: + try: + absolute = urljoin(base_url.rstrip("/") + "/", href) + if urlparse(absolute).hostname != base_host: + continue + path = urlparse(absolute).path.lower() + except ValueError: + # A malformed href is the page author's mistake, not a reason to + # lose the company. An unfilled template placeholder such as + # "http://[BookingLink]" reads as an IPv6 literal to urlsplit + # (3.11.4+), which then raises from ipaddress. Skip the link. continue - path = urlparse(absolute).path.lower() for kind, keywords in PAGE_KEYWORDS.items(): if kind not in found and any(k in path for k in keywords): found[kind] = absolute diff --git a/prospector/models.py b/prospector/models.py index 5bc94ee..a482b26 100644 --- a/prospector/models.py +++ b/prospector/models.py @@ -61,6 +61,10 @@ class ResearchResult: # pages rather than supplied in the input row. email_evidence: "Evidence | None" = None city: str | None = None + # How many of the company's own pages were actually read. Distinguishes + # "no site to read" from "read the site, nothing published" when explaining + # an unreachable company (008 FR-010). + pages_fetched: int = 0 sources_consulted: list[str] = field(default_factory=list) failures: list[str] = field(default_factory=list) diff --git a/prospector/pipeline.py b/prospector/pipeline.py index cb7dd8a..cb38b98 100644 --- a/prospector/pipeline.py +++ b/prospector/pipeline.py @@ -128,7 +128,7 @@ def _process_company( ) -> tuple[Prospect, Draft | None]: research = _research(company, settings, fetcher, verbose=verbose) if not company.email: - raise NoEmailFound(company.bucket_reason or "no email address found") + raise NoEmailFound(_no_email_reason(research)) prospect = _score(company, research, settings) draft: Draft | None = None if no_llm: @@ -147,6 +147,21 @@ def _process_company( return prospect, draft +def _no_email_reason(research: ResearchResult) -> str: + """Why this company is unreachable, judged AFTER recovery ran (008 FR-010). + + The ingest-time reason describes the input row ("blank email"), so reporting + it here reads as though recovery never happened. What an operator needs is + which stage ran out of road: a keyword returning listings with no websites + is a sourcing problem, while sites that publish no address is a vertical + that simply prefers contact forms.""" + if not research.website: + return "no website could be resolved" + if not research.pages_fetched: + return f"no page could be fetched from {research.website}" + return "no published address on any fetched page" + + def _research(company: Company, settings: Settings, fetcher: Fetcher, *, verbose: bool) -> ResearchResult: research = ResearchResult(website=company.website) info = resolve.resolve(company, settings, fetcher) @@ -165,6 +180,8 @@ def _research(company: Company, settings: Settings, fetcher: Fetcher, *, verbose if html is not None: pages.append(extracting.PageContent(kind, url, html)) + research.pages_fetched = len(pages) + # 008 FR-006: no supplied address -> look for one the company publishes on # the pages we already fetched. No new request is made (FR-011). if not company.email and pages: diff --git a/tests/integration/test_email_recovery_batch.py b/tests/integration/test_email_recovery_batch.py index 2f8f679..a5443a9 100644 --- a/tests/integration/test_email_recovery_batch.py +++ b/tests/integration/test_email_recovery_batch.py @@ -76,6 +76,12 @@ def test_unreachable_company_is_named(self, tmp_path, stubs): summary, _ = run(tmp_path, stubs) assert [name for name, _ in summary.skipped_companies] == ["Unreachable Ducts"] + def test_skip_reason_describes_the_outcome_not_the_input_row(self, tmp_path, stubs): + """FR-010: "blank email" restates the CSV. The operator needs to know + which stage ran out of road — the site was read and published nothing.""" + summary, _ = run(tmp_path, stubs) + assert summary.skipped_companies[0][1] == "no published address on any fetched page" + def test_recovered_address_lands_in_the_note(self, tmp_path, stubs): _, vault_dir = run(tmp_path, stubs) text = (vault_dir / "recoverable-ducts.md").read_text(encoding="utf-8") diff --git a/tests/unit/test_extract.py b/tests/unit/test_extract.py index 0d3a58d..0adcfdc 100644 --- a/tests/unit/test_extract.py +++ b/tests/unit/test_extract.py @@ -25,6 +25,14 @@ def test_external_links_ignored(self): html = 'About' assert discover_extra_pages(html, "https://acmeduct.com") == [] + def test_malformed_href_does_not_lose_the_page(self): + # An unfilled template placeholder reads as an IPv6 literal to urlsplit + # and raises from ipaddress. One bad link must not cost the company. + html = 'Bookc' + assert discover_extra_pages(html, "https://acmeduct.com") == [ + ("contact", "https://acmeduct.com/contact") + ] + def test_capped_at_three(self): html = ( 'at' diff --git a/tests/unit/test_pipeline_reasons.py b/tests/unit/test_pipeline_reasons.py new file mode 100644 index 0000000..a158c99 --- /dev/null +++ b/tests/unit/test_pipeline_reasons.py @@ -0,0 +1,20 @@ +"""008 FR-010: an unreachable company is explained by what research found, not +by what the input row looked like. The three reasons separate a sourcing problem +(listings with no websites) from a vertical that publishes no addresses. +""" + +from prospector.models import ResearchResult +from prospector.pipeline import _no_email_reason + + +class TestNoEmailReason: + def test_no_website_resolved(self): + assert _no_email_reason(ResearchResult()) == "no website could be resolved" + + def test_website_resolved_but_unreadable(self): + research = ResearchResult(website="https://acmeduct.com", pages_fetched=0) + assert _no_email_reason(research) == "no page could be fetched from https://acmeduct.com" + + def test_pages_read_but_nothing_published(self): + research = ResearchResult(website="https://acmeduct.com", pages_fetched=3) + assert _no_email_reason(research) == "no published address on any fetched page"