Skip to content

Commit db86ee7

Browse files
committed
refactor: Re-export Crawlee FileSystemStorageClient and clarify Apify FS docstring
1 parent a5c9dd2 commit db86ee7

2 files changed

Lines changed: 16 additions & 8 deletions

File tree

src/apify/storage_clients/__init__.py

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
1-
# `MemoryStorageClient` is re-exported from Crawlee unchanged. Unlike `ApifyFileSystemStorageClient`, it does
2-
# not preserve the Actor `INPUT` record on purge, and unlike the Apify dataset clients it does not charge for
3-
# the `PAY_PER_EVENT` pricing model, so prefer the Apify clients when either behavior matters.
4-
from crawlee.storage_clients import MemoryStorageClient, StorageClient
1+
# `FileSystemStorageClient`, `MemoryStorageClient`, and `StorageClient` are re-exported unchanged from Crawlee.
2+
# The file-system client also has an Apify variant, `ApifyFileSystemStorageClient` (the default local client),
3+
# which preserves the Actor `INPUT` record on purge and charges for the `PAY_PER_EVENT` pricing model. There is
4+
# no Apify variant of `MemoryStorageClient`, so it is re-exported as-is and provides neither behavior. Prefer
5+
# the Apify clients when either behavior matters.
6+
from crawlee.storage_clients import FileSystemStorageClient, MemoryStorageClient, StorageClient
57

68
from ._apify import ApifyStorageClient
79
from ._file_system import ApifyFileSystemStorageClient
@@ -10,6 +12,7 @@
1012
__all__ = [
1113
'ApifyFileSystemStorageClient',
1214
'ApifyStorageClient',
15+
'FileSystemStorageClient',
1316
'MemoryStorageClient',
1417
'SmartApifyStorageClient',
1518
'StorageClient',

src/apify/storage_clients/_file_system/_storage_client.py

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,16 @@
1919

2020
@docs_group('Storage clients')
2121
class ApifyFileSystemStorageClient(FileSystemStorageClient):
22-
"""Apify-specific implementation of the file system storage client.
22+
"""Apify SDK variant of Crawlee's `FileSystemStorageClient`, used as the default local storage client.
2323
24-
The only difference is that it uses `ApifyFileSystemKeyValueStoreClient` for key-value stores,
25-
which overrides the `purge` method to delete all files in the key-value store directory
26-
except for the metadata file and the `INPUT.json` file.
24+
It extends the Crawlee file-system client with Apify-specific behavior that keeps local runs consistent
25+
with the Apify platform:
26+
27+
- Key-value stores use `ApifyFileSystemKeyValueStoreClient`, which preserves the Actor input file (e.g.
28+
`INPUT.json`) and the metadata file when purging, and maps the logical `INPUT` key to the input file on
29+
disk.
30+
- Datasets use `ApifyFileSystemDatasetClient`, which charges for the `PAY_PER_EVENT` pricing model so it
31+
can be exercised locally.
2732
"""
2833

2934
@override

0 commit comments

Comments
 (0)