Skip to content

Commit 2144a1f

Browse files
vdusekclaude
vdusek
and
claude
committed
test: merge server.py files and add max_crawl_depth to crawlee crawler e2e tests
Consolidate the two separate server.py files (actor_source_base and test_crawlee_crawlers/actor_source) into a single base server with a category-based depth structure and an infinite /deep/N chain. Add max_crawl_depth=2 to all crawler constructors to test depth limiting. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent f8b2b45 commit 2144a1f

15 files changed

Lines changed: 102 additions & 157 deletions

‎tests/e2e/actor_source_base/server.py‎

Lines changed: 93 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -1,94 +1,128 @@
1-
"""
2-
Test server is infinite server http://localhost:8080/{any_number} and each page has links to the next 10 pages.
3-
For example:
4-
http://localhost:8080/ contains links:
5-
http://localhost:8080/0, http://localhost:8080/1, ..., http://localhost:8080/9
1+
"""Test HTTP server for e2e tests.
2+
3+
Serves an e-commerce test website with a category-based structure for testing crawl depth:
64
7-
http://localhost:8080/1 contains links:
8-
http://localhost:8080/10, http://localhost:8080/11, ..., http://localhost:8080/19
5+
/ (depth 0) - Homepage with links to categories, about page, and deep chain
6+
/categories/electronics (depth 1) - Links to products 1 and 2
7+
/categories/home (depth 1) - Links to product 3
8+
/about (depth 1) - About page
9+
/deep/1 (depth 1) -> /deep/2 (depth 2) -> /deep/3 (depth 3) -> ... (infinite chain)
10+
/products/1 (depth 2) - Widget A
11+
/products/2 (depth 2) - Widget B
12+
/products/3 (depth 2) - Widget C
913
10-
... and so on.
14+
With max_crawl_depth=2, the crawler reaches all products but does not go beyond /deep/2.
1115
"""
1216

17+
from __future__ import annotations
18+
1319
import asyncio
1420
import logging
1521
from collections.abc import Awaitable, Callable, Coroutine
16-
from socket import socket
1722
from typing import Any
1823

1924
from uvicorn import Config
2025
from uvicorn.server import Server
21-
from yarl import URL
2226

2327
Receive = Callable[[], Awaitable[dict[str, Any]]]
2428
Send = Callable[[dict[str, Any]], Coroutine[None, None, None]]
2529

30+
_PRODUCTS = {
31+
'1': {'name': 'Widget A', 'price': '$19.99', 'description': 'A basic widget for everyday use'},
32+
'2': {'name': 'Widget B', 'price': '$29.99', 'description': 'An advanced widget with extra features'},
33+
'3': {'name': 'Widget C', 'price': '$39.99', 'description': 'A premium widget for professionals'},
34+
}
35+
2636

27-
async def send_html_response(send: Send, html_content: bytes, status: int = 200) -> None:
28-
"""Send an HTML response to the client."""
37+
async def _send_html(send: Send, html: str, status: int = 200) -> None:
2938
await send(
3039
{
3140
'type': 'http.response.start',
3241
'status': status,
3342
'headers': [[b'content-type', b'text/html; charset=utf-8']],
3443
}
3544
)
36-
await send({'type': 'http.response.body', 'body': html_content})
45+
await send({'type': 'http.response.body', 'body': html.encode()})
3746

3847

39-
async def app(scope: dict[str, Any], _: Receive, send: Send) -> None:
40-
"""Main ASGI application handler that routes requests to specific handlers.
41-
42-
Args:
43-
scope: The ASGI connection scope.
44-
_: The ASGI receive function.
45-
send: The ASGI send function.
46-
"""
48+
async def app(scope: dict[str, Any], _receive: Receive, send: Send) -> None:
4749
assert scope['type'] == 'http'
4850
path = scope['path']
4951

50-
links = '\n'.join(f'<a href="{path}{i}">{path}{i}</a>' for i in range(10))
51-
await send_html_response(
52-
send,
53-
f"""\
54-
<html><head>
55-
<title>Title for {path} </title>
56-
</head>
57-
<body>
58-
{links}
59-
</body></html>""".encode(),
60-
)
61-
62-
63-
class TestServer(Server):
64-
"""A test HTTP server implementation based on Uvicorn Server."""
65-
66-
@property
67-
def url(self) -> URL:
68-
"""Get the base URL of the server.
69-
70-
Returns:
71-
A URL instance with the server's base URL.
72-
"""
73-
protocol = 'https' if self.config.is_ssl else 'http'
74-
return URL(f'{protocol}://{self.config.host}:{self.config.port}/')
75-
76-
async def serve(self, sockets: list[socket] | None = None) -> None:
77-
"""Run the server."""
78-
if sockets:
79-
raise RuntimeError('Simple TestServer does not support custom sockets')
80-
self.restart_requested = asyncio.Event()
81-
82-
loop = asyncio.get_event_loop()
83-
tasks = {
84-
loop.create_task(super().serve()),
85-
}
86-
await asyncio.wait(tasks)
52+
if path == '/':
53+
await _send_html(
54+
send,
55+
'<html><head><title>E-commerce Test Store</title></head><body>'
56+
'<h1>Welcome to Test Store</h1>'
57+
'<a href="/categories/electronics">Electronics</a>'
58+
'<a href="/categories/home">Home &amp; Garden</a>'
59+
'<a href="/about">About Us</a>'
60+
'<a href="/deep/1">Explore More</a>'
61+
'</body></html>',
62+
)
63+
elif path == '/categories/electronics':
64+
await _send_html(
65+
send,
66+
'<html><head><title>Electronics</title></head><body>'
67+
'<h1>Electronics</h1>'
68+
'<a href="/products/1">Widget A</a>'
69+
'<a href="/products/2">Widget B</a>'
70+
'<a href="/">Back to Home</a>'
71+
'</body></html>',
72+
)
73+
elif path == '/categories/home':
74+
await _send_html(
75+
send,
76+
'<html><head><title>Home &amp; Garden</title></head><body>'
77+
'<h1>Home &amp; Garden</h1>'
78+
'<a href="/products/3">Widget C</a>'
79+
'<a href="/">Back to Home</a>'
80+
'</body></html>',
81+
)
82+
elif path.startswith('/products/'):
83+
product = _PRODUCTS.get(path.split('/')[-1])
84+
if product:
85+
await _send_html(
86+
send,
87+
f'<html><head><title>{product["name"]}</title></head><body>'
88+
f'<h1>{product["name"]}</h1>'
89+
f'<span class="price">{product["price"]}</span>'
90+
f'<p class="description">{product["description"]}</p>'
91+
f'<a href="/">Back to Home</a>'
92+
f'</body></html>',
93+
)
94+
else:
95+
await _send_html(send, '<html><body>Not Found</body></html>', 404)
96+
elif path == '/about':
97+
await _send_html(
98+
send,
99+
'<html><head><title>About Us</title></head><body>'
100+
'<h1>About Test Store</h1>'
101+
'<p class="description">We sell the best widgets in the world.</p>'
102+
'<a href="/">Back to Home</a>'
103+
'</body></html>',
104+
)
105+
elif path.startswith('/deep/'):
106+
try:
107+
n = int(path.split('/')[-1])
108+
except ValueError:
109+
await _send_html(send, '<html><body>Not Found</body></html>', 404)
110+
return
111+
await _send_html(
112+
send,
113+
f'<html><head><title>Deep Page {n}</title></head><body>'
114+
f'<h1>Deep Page {n}</h1>'
115+
f'<a href="/deep/{n + 1}">Go Deeper</a>'
116+
f'<a href="/">Back to Home</a>'
117+
f'</body></html>',
118+
)
119+
else:
120+
await _send_html(send, '<html><body>Not Found</body></html>', 404)
87121

88122

89123
if __name__ == '__main__':
90124
asyncio.run(
91-
TestServer(
125+
Server(
92126
config=Config(
93127
app=app,
94128
lifespan='off',

‎tests/e2e/test_crawlee_crawlers/actor_source/main_adaptive_playwright_crawler.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
async def main() -> None:
99
async with Actor:
1010
pages_visited: list[str] = []
11-
crawler = AdaptivePlaywrightCrawler.with_parsel_static_parser()
11+
crawler = AdaptivePlaywrightCrawler.with_parsel_static_parser(max_crawl_depth=2)
1212

1313
@crawler.router.default_handler
1414
async def handler(context: AdaptivePlaywrightCrawlingContext) -> None:

‎tests/e2e/test_crawlee_crawlers/actor_source/main_basic_crawler.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ def handle_data(self, data: str) -> None:
4545
async def main() -> None:
4646
async with Actor:
4747
pages_visited: list[str] = []
48-
crawler = BasicCrawler()
48+
crawler = BasicCrawler(max_crawl_depth=2)
4949

5050
@crawler.router.default_handler
5151
async def handler(context: BasicCrawlingContext) -> None:

‎tests/e2e/test_crawlee_crawlers/actor_source/main_beautifulsoup_crawler.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
async def main() -> None:
99
async with Actor:
1010
pages_visited: list[str] = []
11-
crawler = BeautifulSoupCrawler()
11+
crawler = BeautifulSoupCrawler(max_crawl_depth=2)
1212

1313
@crawler.router.default_handler
1414
async def handler(context: BeautifulSoupCrawlingContext) -> None:

‎tests/e2e/test_crawlee_crawlers/actor_source/main_http_crawler.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
async def main() -> None:
1111
async with Actor:
1212
pages_visited: list[str] = []
13-
crawler = HttpCrawler()
13+
crawler = HttpCrawler(max_crawl_depth=2)
1414

1515
@crawler.router.default_handler
1616
async def handler(context: HttpCrawlingContext) -> None:

‎tests/e2e/test_crawlee_crawlers/actor_source/main_parsel_crawler.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
async def main() -> None:
99
async with Actor:
1010
pages_visited: list[str] = []
11-
crawler = ParselCrawler()
11+
crawler = ParselCrawler(max_crawl_depth=2)
1212

1313
@crawler.router.default_handler
1414
async def handler(context: ParselCrawlingContext) -> None:

‎tests/e2e/test_crawlee_crawlers/actor_source/main_playwright_crawler.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
async def main() -> None:
99
async with Actor:
1010
pages_visited: list[str] = []
11-
crawler = PlaywrightCrawler()
11+
crawler = PlaywrightCrawler(max_crawl_depth=2)
1212

1313
@crawler.router.default_handler
1414
async def handler(context: PlaywrightCrawlingContext) -> None:

‎tests/e2e/test_crawlee_crawlers/actor_source/server.py‎

Lines changed: 0 additions & 86 deletions
This file was deleted.

‎tests/e2e/test_crawlee_crawlers/conftest.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,4 +58,7 @@ async def verify_crawler_results(
5858
assert kvs_record is not None
5959
result = kvs_record['value']
6060
assert result['crawler_type'] == expected_crawler_type
61+
# With max_crawl_depth=2, the server has 9 pages reachable (homepage, 2 categories, about, /deep/1,
62+
# 3 products, /deep/2). The crawler should visit most of them but not go beyond /deep/2.
6163
assert result['pages_visited_count'] >= 5
64+
assert result['pages_visited_count'] <= 15

‎tests/e2e/test_crawlee_crawlers/test_adaptive_playwright_crawler.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ async def test_adaptive_playwright_crawler(make_actor: MakeActorFunction, run_ac
1212
actor = await make_actor(
1313
label='crawl-adaptive',
1414
source_files={
15-
'server.py': read_actor_source('server.py'),
1615
'src/main.py': read_actor_source('main_adaptive_playwright_crawler.py'),
1716
'Dockerfile': get_playwright_dockerfile(),
1817
},

0 commit comments

Comments
 (0)