feat: Add send_request_enqueue_strategy option to BasicCrawler

Author: vdusek

src/crawlee/crawlers/_basic/_basic_crawler.py

@@ -204,6 +204,12 @@ class _BasicCrawlerOptions(TypedDict):
     """If set to `True`, the crawler will automatically try to fetch the robots.txt file for each domain,
     and skip those that are not allowed. This also prevents disallowed URLs to be added via `EnqueueLinksFunction`."""
 
+    send_request_enqueue_strategy: NotRequired[EnqueueStrategy]
+    """Strategy applied by `BasicCrawlingContext.send_request` to validate the target URL against the current request's
+    URL. Defaults to `'all'`, which preserves the historical behaviour of allowing arbitrary URLs. Set to e.g.
+    `'same-hostname'` to restrict `send_request` to the same host as the current request, for example when handlers
+    pass URLs extracted from page content to `send_request` directly."""
+
     status_message_logging_interval: NotRequired[timedelta]
     """Interval for logging the crawler status messages."""
 
@@ -299,6 +305,7 @@ def __init__(
         configure_logging: bool = True,
         statistics_log_format: Literal['table', 'inline'] = 'table',
         respect_robots_txt_file: bool = False,
+        send_request_enqueue_strategy: EnqueueStrategy = 'all',
         status_message_logging_interval: timedelta = timedelta(seconds=10),
         status_message_callback: Callable[[StatisticsState, StatisticsState | None, str], Awaitable[str | None]]
         | None = None,
@@ -352,6 +359,11 @@ def __init__(
             respect_robots_txt_file: If set to `True`, the crawler will automatically try to fetch the robots.txt file
                 for each domain, and skip those that are not allowed. This also prevents disallowed URLs to be added
                 via `EnqueueLinksFunction`
+            send_request_enqueue_strategy: Strategy applied by `BasicCrawlingContext.send_request` to validate the
+                target URL against the current request's URL (`loaded_url` if available, otherwise `url`). Defaults
+                to `'all'`, preserving the historical behaviour of allowing arbitrary URLs. Set to e.g.
+                `'same-hostname'` when handlers extract URLs from page content and pass them straight to
+                `send_request`, to keep cross-host fetches intentional.
             status_message_logging_interval: Interval for logging the crawler status messages.
             status_message_callback: Allows overriding the default status message. The default status message is
                 provided in the parameters. Returning `None` suppresses the status message.
@@ -432,6 +444,7 @@ def __init__(
         self._max_session_rotations = max_session_rotations
         self._max_crawl_depth = max_crawl_depth
         self._respect_robots_txt_file = respect_robots_txt_file
+        self._send_request_enqueue_strategy = send_request_enqueue_strategy
 
         # Timeouts
         self._request_handler_timeout = request_handler_timeout
@@ -1275,14 +1288,28 @@ def _prepare_send_request_function(
         self,
         session: Session | None,
         proxy_info: ProxyInfo | None,
+        request: Request,
     ) -> SendRequestFunction:
+        strategy = self._send_request_enqueue_strategy
+        origin_url = request.loaded_url or request.url
+        origin_parsed = urlparse(origin_url) if strategy != 'all' else None
+
         async def send_request(
             url: str,
             *,
             method: HttpMethod = 'GET',
             payload: HttpPayload | None = None,
             headers: HttpHeaders | dict[str, str] | None = None,
         ) -> HttpResponse:
+            if origin_parsed is not None and not self._check_enqueue_strategy(
+                strategy,
+                target_url=urlparse(url),
+                origin_url=origin_parsed,
+            ):
+                raise ValueError(
+                    f'send_request() refusing to fetch {url!r}: does not match enqueue strategy '
+                    f'{strategy!r} relative to {origin_url!r}.'
+                )
             return await self._http_client.send_request(
                 url=url,
                 method=method,
@@ -1428,7 +1455,7 @@ async def __run_task_function(self) -> None:
             request=result.request,
             session=session,
             proxy_info=proxy_info,
-            send_request=self._prepare_send_request_function(session, proxy_info),
+            send_request=self._prepare_send_request_function(session, proxy_info, request),
             add_requests=result.add_requests,
             push_data=result.push_data,
             get_key_value_store=result.get_key_value_store,

tests/unit/crawlers/_basic/test_basic_crawler.py

@@ -19,7 +19,7 @@
 
 import pytest
 
-from crawlee import ConcurrencySettings, Glob, service_locator
+from crawlee import ConcurrencySettings, EnqueueStrategy, Glob, service_locator
 from crawlee._request import Request, RequestState
 from crawlee._types import BasicCrawlingContext, EnqueueLinksKwargs, HttpMethod
 from crawlee._utils.robots import RobotsTxtFile
@@ -340,6 +340,43 @@ async def failed_request_handler(context: BasicCrawlingContext, error: Exception
         await crawler.run(['https://a.placeholder.com', 'https://b.placeholder.com', 'https://c.placeholder.com'])
 
 
+@pytest.mark.parametrize(
+    ('strategy', 'target_path', 'should_succeed'),
+    [
+        pytest.param('all', 'get', True, id='default-all-allows-same-host'),
+        pytest.param('same-hostname', 'get', True, id='same-hostname-allows-same-host'),
+        pytest.param('same-hostname', 'http://other.test/payload', False, id='same-hostname-rejects-cross-host'),
+    ],
+)
+async def test_send_request_enqueue_strategy(
+    server_url: URL, strategy: EnqueueStrategy, target_path: str, *, should_succeed: bool
+) -> None:
+    bodies: list[bytes] = []
+    errors: list[Exception] = []
+
+    crawler = BasicCrawler(max_request_retries=1, send_request_enqueue_strategy=strategy)
+    target_url = target_path if target_path.startswith('http') else str(server_url / target_path)
+
+    @crawler.router.default_handler
+    async def handler(context: BasicCrawlingContext) -> None:
+        try:
+            response = await context.send_request(target_url)
+        except ValueError as exc:
+            errors.append(exc)
+        else:
+            bodies.append(await response.read())
+
+    await crawler.run([str(server_url / 'a/page')])
+
+    if should_succeed:
+        assert bodies, 'expected the handler to receive a response'
+        assert not errors
+    else:
+        assert errors, 'expected send_request to refuse the target URL'
+        assert strategy in str(errors[0])
+        assert target_url in str(errors[0])
+
+
 @pytest.mark.parametrize(
     ('method', 'path', 'payload'),
     [