From f58b61e2fd73732463f0b99a166fe6eb9a2193e0 Mon Sep 17 00:00:00 2001 From: Gyubong Date: Thu, 22 Jan 2026 11:08:27 +0900 Subject: [PATCH 1/9] docs: Add BEP-1036 for Artifact Storage Usage Tracking and Quota Enforcement --- proposals/BEP-1036-artifact-storage-quota.md | 208 +++++++++++++ proposals/BEP-1036/quota-flow.md | 108 +++++++ proposals/BEP-1036/storage_namespace.md | 256 ++++++++++++++++ proposals/BEP-1036/vfolder_storage.md | 296 +++++++++++++++++++ 4 files changed, 868 insertions(+) create mode 100644 proposals/BEP-1036-artifact-storage-quota.md create mode 100644 proposals/BEP-1036/quota-flow.md create mode 100644 proposals/BEP-1036/storage_namespace.md create mode 100644 proposals/BEP-1036/vfolder_storage.md diff --git a/proposals/BEP-1036-artifact-storage-quota.md b/proposals/BEP-1036-artifact-storage-quota.md new file mode 100644 index 00000000000..6eb72a51749 --- /dev/null +++ b/proposals/BEP-1036-artifact-storage-quota.md @@ -0,0 +1,208 @@ +--- +Author: Gyubong Lee (gbl@lablup.com) +Status: Draft +Created: 2026-01-22 +Created-Version: 26.2.0 +Target-Version: +Implemented-Version: +--- + +# Artifact Storage Usage Tracking and Quota Enforcement + +## Related Issues + +- JIRA: BA-3989 (Epic), BA-3990, BA-3991, BA-3992, BA-3994 +- GitHub: #8194 + +## Motivation + +Currently, artifact storage has no usage tracking or capacity limits. When artifacts are imported, they are stored without any visibility into how much space is being consumed or any mechanism to prevent storage exhaustion. + +This creates operational risks: + +- Storage can be exhausted without warning +- No visibility into storage utilization +- No way to enforce capacity planning or cost control + +The artifact import flow supports two storage destinations, and both need quota enforcement: + +1. **Default (StorageNamespace)**: No quota system exists +2. **VFolder destination**: Quota system exists (`max_quota_scope_size`) but not integrated into artifact import pre-check + +## Current Design + +### Existing Data Model + +- **`StorageNamespaceRow`**: Tracks basic namespace information (`id`, `storage_id`, `namespace`). No quota-related fields. +- **`ArtifactRevisionRow`**: Already tracks `size` for individual revisions. +- **`AssociationArtifactsStorageRow`**: Links artifact revisions to storage namespaces. + +### Dual Storage Destination + +The artifact import flow (`import_revision()`) supports two destinations: + +| Destination | When | Current Quota | +|-------------|------|---------------| +| StorageNamespace | `vfolder_id` is None | **None** | +| VFolder | `vfolder_id` is provided | `max_quota_scope_size` in resource policy (enforced by storage proxy at write time) | + +### VFolder Quota System + +VFolders have an existing quota system: + +- `VFolderRow.quota_scope_id` → Links to user or project +- Resource policies define `max_quota_scope_size` +- Storage proxy enforces quota at filesystem write time + +**Problem**: The artifact import does not pre-check VFolder quota before starting the import. Large imports can fail mid-way when the storage proxy rejects writes due to quota limits. + +## Proposed Design + +### Overview + +Create a unified quota enforcement layer that performs pre-validation before artifact import begins, regardless of storage destination. + +### Unified Quota Service + +Create `ArtifactStorageQuotaService` that handles both storage destinations with a single entry point: + +```python +class ArtifactStorageQuotaService: + """Unified quota service for artifact storage.""" + + async def check_quota( + self, + storage_destination: StorageDestination, + additional_size: int, + ) -> None: + match storage_destination: + case StorageNamespaceDestination(namespace_id): + await self._check_storage_namespace_quota(namespace_id, additional_size) + case VFolderDestination(vfolder_id, quota_scope_id): + await self._check_vfolder_quota(vfolder_id, quota_scope_id, additional_size) +``` + +### Storage Destination Details + +Each storage destination has different quota mechanisms: + +| Destination | Quota Source | Usage Source | Details | +|-------------|-------------|--------------|---------| +| StorageNamespace | `StorageNamespaceRow.max_size` (NEW) | Aggregated from `artifact_revisions` via association table | [storage_namespace.md](BEP-1036/storage_namespace.md) | +| VFolder | `max_quota_scope_size` from resource policy | Storage proxy API | [vfolder_storage.md](BEP-1036/vfolder_storage.md) | + +### Import Flow Integration + +The quota check is integrated into the import flow before any file transfer begins: + +```python +async def import_revision(self, action: ImportArtifactRevisionAction) -> ...: + revision_data = await self._artifact_repository.get_artifact_revision_by_id(...) + + # Determine storage destination and check quota + if action.vfolder_id: + destination = VFolderDestination(...) + else: + destination = StorageNamespaceDestination(namespace_id=namespace_id) + + if revision_data.size: + await self._quota_service.check_quota(destination, revision_data.size) + + # Proceed with import only if quota check passes + ... +``` + +See [quota-flow.md](BEP-1036/quota-flow.md) for the complete flow diagram. + +### Error Types + +```python +class StorageQuotaExceededError(BackendError): + """Base class for quota exceeded errors.""" + pass + +class StorageNamespaceQuotaExceededError(StorageQuotaExceededError): + """Raised when StorageNamespace quota would be exceeded.""" + namespace_id: uuid.UUID + current_size: int + max_size: int + requested_size: int + +class VFolderQuotaExceededError(StorageQuotaExceededError): + """Raised when VFolder quota scope limit would be exceeded.""" + vfolder_id: VFolderID + quota_scope_id: QuotaScopeID + current_size: int + max_size: int + requested_size: int +``` + +### REST API Endpoints + +#### GET /storage-namespaces/{id}/usage + +Returns storage namespace usage statistics. + +```json +{ + "namespace_id": "...", + "total_size": 10737418240, + "max_size": 107374182400, + "revision_count": 42, + "utilization_percent": 10.0 +} +``` + +#### PATCH /storage-namespaces/{id}/quota + +Updates quota for a storage namespace. Admin-only. + +```json +{ "max_size": 107374182400 } // or null for unlimited +``` + +## Migration / Compatibility + +### Backward Compatibility + +- Existing storage namespaces will have `max_size = NULL` (unlimited) +- No changes to existing import behavior unless quota is explicitly set +- VFolder quota integration uses existing `max_quota_scope_size` from resource policies + +### Breaking Changes + +None. This is an additive feature. + +## Testing Scenarios + +### StorageNamespace Quota Tests +- Quota check passes when under limit +- Quota check passes when `max_size` is NULL (unlimited) +- Quota check raises error when limit would be exceeded +- Import flow rejects artifact when quota exceeded + +### VFolder Quota Tests +- Pre-check queries storage proxy for current usage +- Quota check uses `max_quota_scope_size` from resource policy +- Import is rejected before starting if quota would be exceeded +- Handles case when resource policy limit is unlimited (-1) + +### Unified Flow Tests +- Correct quota system is selected based on `vfolder_id` presence +- Error messages clearly indicate which quota was exceeded + +## Future Ideas + +### Quota Threshold Notifications + +When storage usage approaches the configured limit, the system could notify administrators: + +- Threshold levels: 80%, 90%, 95% utilization +- Integration with existing notification system + +## References + +- [BEP-1019: MinIO Artifact Registry Storage](BEP-1019-minio-artifact-registry-storage.md) +- [StorageNamespace Quota Details](BEP-1036/storage_namespace.md) +- [VFolder Storage Quota Details](BEP-1036/vfolder_storage.md) +- [Quota Flow Diagram](BEP-1036/quota-flow.md) diff --git a/proposals/BEP-1036/quota-flow.md b/proposals/BEP-1036/quota-flow.md new file mode 100644 index 00000000000..633cac2209a --- /dev/null +++ b/proposals/BEP-1036/quota-flow.md @@ -0,0 +1,108 @@ +# Artifact Storage Quota Flow Diagram + +## Unified Import Flow with Quota Check + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ import_revision(action) │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ │ +│ 1. Get revision_data (includes size) │ +│ │ +│ 2. Determine storage destination │ +│ ┌─────────────────────────────┬────────────────────────────────────┐ │ +│ │ vfolder_id provided? │ │ │ +│ └──────────┬──────────────────┴────────────────────┬───────────────┘ │ +│ │ YES │ NO │ +│ ▼ ▼ │ +│ ┌─────────────────────────┐ ┌─────────────────────────────┐ │ +│ │ VFolderDestination │ │ StorageNamespaceDestination│ │ +│ │ - vfolder_id │ │ - namespace_id │ │ +│ │ - quota_scope_id │ │ │ │ +│ └───────────┬─────────────┘ └───────────────┬─────────────┘ │ +│ │ │ │ +│ └──────────────┬──────────────────────────┘ │ +│ ▼ │ +│ 3. ┌──────────────────────────────────────────────────────────────────┐ │ +│ │ ArtifactStorageQuotaService.check_quota() │ │ +│ └──────────────────────────────────────────────────────────────────┘ │ +│ │ │ +│ ┌──────────────┴──────────────┐ │ +│ ▼ ▼ │ +│ ┌─────────────────────────┐ ┌─────────────────────────────────┐ │ +│ │ _check_vfolder_quota() │ │ _check_storage_namespace_quota()│ │ +│ │ │ │ │ │ +│ │ See: vfolder_storage.md │ │ See: storage_namespace.md │ │ +│ └───────────┬─────────────┘ └───────────────┬─────────────────┘ │ +│ │ │ │ +│ └──────────────┬────────────────────┘ │ +│ ▼ │ +│ ┌──────────────────────────────┐ │ +│ │ Quota Exceeded? │ │ +│ └──────────────┬───────────────┘ │ +│ YES │ │ NO │ +│ ▼ ▼ │ +│ ┌─────────────────────────┐ ┌─────────────────────────────────┐ │ +│ │ Raise appropriate error │ │ 4. Proceed with import │ │ +│ │ - VFolderQuotaExceeded │ │ - Call storage proxy │ │ +│ │ - StorageNamespace │ │ - Update status │ │ +│ │ QuotaExceeded │ │ - Associate with storage │ │ +│ └─────────────────────────┘ └─────────────────────────────────┘ │ +│ │ +└──────────────────────────────────────────────────────────────────────────────┘ +``` + +## Data Sources for Quota Check + +### StorageNamespace Quota + +Usage is aggregated from artifact revisions linked to the namespace via the association table. + +``` +┌─────────────────────┐ ┌─────────────────────────────────┐ +│ storage_namespace │ │ association_artifacts_storages │ +├─────────────────────┤ ├─────────────────────────────────┤ +│ id │◄────│ storage_namespace_id │ +│ max_size (NEW) │ │ artifact_revision_id ───────────┼──┐ +└─────────────────────┘ └─────────────────────────────────┘ │ + │ + ┌─────────────────────────────────┐ │ + │ artifact_revisions │ │ + ├─────────────────────────────────┤ │ + │ id ◄────────────────────────────┼──┘ + │ size │ + └─────────────────────────────────┘ + +Usage = SUM(artifact_revisions.size) WHERE linked to namespace +``` + +### VFolder Quota + +Usage is queried from the storage proxy, and limit comes from resource policies. + +``` +┌─────────────────────┐ ┌─────────────────────────────────┐ +│ vfolders │ │ user_resource_policies / │ +├─────────────────────┤ │ project_resource_policies │ +│ id │ ├─────────────────────────────────┤ +│ quota_scope_id ─────┼────►│ max_quota_scope_size │ +└─────────────────────┘ └─────────────────────────────────┘ + │ + │ VFolderID + ▼ +┌─────────────────────┐ +│ Storage Proxy │ +├─────────────────────┤ +│ get_quota_scope_ │ +│ usage(quota_scope) │──► Current usage in bytes +└─────────────────────┘ +``` + +## Quota Check Comparison + +| Aspect | StorageNamespace | VFolder | +|--------|------------------|---------| +| Limit Source | `storage_namespace.max_size` | `resource_policy.max_quota_scope_size` | +| Usage Source | DB aggregation via association table | Storage proxy API | +| Unlimited Value | `NULL` | `-1` | +| Scope | Per namespace | Per quota scope (user/project) | diff --git a/proposals/BEP-1036/storage_namespace.md b/proposals/BEP-1036/storage_namespace.md new file mode 100644 index 00000000000..773edc60f7b --- /dev/null +++ b/proposals/BEP-1036/storage_namespace.md @@ -0,0 +1,256 @@ +# StorageNamespace Quota + +Quota system for default artifact storage (Reservoir archive storage). + +## Overview + +When `vfolder_id` is not provided, artifacts are stored in the configured Reservoir archive storage. +This storage is managed by `StorageNamespaceRow` and supports two backend types: + +- **Object Storage** (MinIO, S3, etc.) +- **VFS Storage** (local/network filesystem) + +## Data Model + +### Current Structure + +``` +┌─────────────────────────┐ +│ object_storage │ +├─────────────────────────┤ +│ id │◄─────┐ +│ name │ │ +│ host │ │ +│ endpoint │ │ +│ access_key │ │ +│ secret_key │ │ +└─────────────────────────┘ │ + │ storage_id (object_storage) +┌─────────────────────────┐ │ +│ storage_namespace │──────┘ +├─────────────────────────┤ +│ id │◄─────────────────────────────────────┐ +│ storage_id │ │ +│ namespace (bucket name) │ │ +└─────────────────────────┘ │ + │ +┌─────────────────────────┐ ┌─────────────────────────┐ │ +│ vfs_storage │ │ association_artifacts_ │ │ +├─────────────────────────┤ │ storages │ │ +│ id │◄──┐ ├─────────────────────────┤ │ +│ name │ │ │ id │ │ +│ host │ │ │ artifact_revision_id ───┼──┐ │ +│ base_path │ │ │ storage_namespace_id ───┼──┼──┘ +└─────────────────────────┘ │ │ storage_type │ │ + │ └─────────────────────────┘ │ + │ │ │ + │ │ "object_storage" │ + │ │ or "vfs_storage" │ + │ │ │ + └─────────┘ │ + storage_id (vfs) │ + │ +┌─────────────────────────┐ │ +│ artifact_revisions │ │ +├─────────────────────────┤ │ +│ id ◄────────────────────┼───────────────────────────────────┘ +│ artifact_id │ +│ version │ +│ size │ ◄── Individual revision size +│ status │ +└─────────────────────────┘ +``` + +### Proposed: Add max_size + +``` +┌─────────────────────────┐ +│ storage_namespace │ +├─────────────────────────┤ +│ id │ +│ storage_id │ +│ namespace │ +│ max_size (NEW) │ ◄── NULL = unlimited, in bytes +└─────────────────────────┘ +``` + +## Scenarios by Storage Type + +### Object Storage (MinIO/S3) + +``` +┌─────────────────────────────────────────────────────────────────────────┐ +│ Object Storage Scenario │ +├─────────────────────────────────────────────────────────────────────────┤ +│ │ +│ Reservoir Config │ +│ ┌────────────────────────────────────────┐ │ +│ │ storage_type: "object_storage" │ │ +│ │ archive_storage: "minio-main" │ │ +│ │ bucket_name: "artifacts" │ ─── namespace │ +│ └────────────────────────────────────────┘ │ +│ │ +│ import_revision() flow │ +│ ┌──────────────────────────────────────────────────────────────────┐ │ +│ │ 1. namespace = reservoir_config.bucket_name │ │ +│ │ │ │ +│ │ 2. object_storage = get_by_name("minio-main") │ │ +│ │ → ObjectStorageRow { id, host, endpoint, ... } │ │ +│ │ │ │ +│ │ 3. storage_namespace = get_by_storage_and_namespace( │ │ +│ │ storage_id=object_storage.id, │ │ +│ │ namespace="artifacts" │ │ +│ │ ) │ │ +│ │ → StorageNamespaceRow { id, max_size, ... } │ │ +│ │ │ │ +│ │ 4. quota_check(storage_namespace.id, revision.size) │ │ +│ │ │ │ +│ │ 5. storage_proxy.import_huggingface_models(...) │ │ +│ │ → Files stored in MinIO bucket │ │ +│ │ │ │ +│ │ 6. associate_artifact_with_storage( │ │ +│ │ revision_id, namespace_id, "object_storage" │ │ +│ │ ) │ │ +│ └──────────────────────────────────────────────────────────────────┘ │ +│ │ +│ Physical storage location │ +│ ┌──────────────────────────────────────────────────────────────────┐ │ +│ │ MinIO: s3://artifacts/{revision_id}/model.safetensors │ │ +│ │ ▲ │ │ +│ │ │ │ │ +│ │ bucket_name (namespace) │ │ +│ └──────────────────────────────────────────────────────────────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────────────────┘ +``` + +### VFS Storage (Filesystem) + +``` +┌─────────────────────────────────────────────────────────────────────────┐ +│ VFS Storage Scenario │ +├─────────────────────────────────────────────────────────────────────────┤ +│ │ +│ Reservoir Config │ +│ ┌────────────────────────────────────────┐ │ +│ │ storage_type: "vfs_storage" │ │ +│ │ archive_storage: "nfs-models" │ │ +│ │ subpath: "reservoir/artifacts" │ ─── namespace │ +│ └────────────────────────────────────────┘ │ +│ │ +│ import_revision() flow │ +│ ┌──────────────────────────────────────────────────────────────────┐ │ +│ │ 1. namespace = reservoir_config.subpath │ │ +│ │ │ │ +│ │ 2. vfs_storage = get_by_name("nfs-models") │ │ +│ │ → VFSStorageRow { id, host, base_path, ... } │ │ +│ │ │ │ +│ │ 3. storage_namespace = get_by_storage_and_namespace( │ │ +│ │ storage_id=vfs_storage.id, │ │ +│ │ namespace="reservoir/artifacts" │ │ +│ │ ) │ │ +│ │ → StorageNamespaceRow { id, max_size, ... } │ │ +│ │ │ │ +│ │ 4. quota_check(storage_namespace.id, revision.size) │ │ +│ │ │ │ +│ │ 5. storage_proxy.import_huggingface_models(...) │ │ +│ │ → Files stored on NFS │ │ +│ │ │ │ +│ │ 6. associate_artifact_with_storage( │ │ +│ │ revision_id, namespace_id, "vfs_storage" │ │ +│ │ ) │ │ +│ └──────────────────────────────────────────────────────────────────┘ │ +│ │ +│ Physical storage location │ +│ ┌──────────────────────────────────────────────────────────────────┐ │ +│ │ NFS: /mnt/nfs/reservoir/artifacts/{revision_id}/model.safetensors│ │ +│ │ ▲ │ │ +│ │ │ │ │ +│ │ base_path + subpath (namespace) │ │ +│ └──────────────────────────────────────────────────────────────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────────────────┘ +``` + +## Quota Check Logic + +```python +async def check_storage_namespace_quota( + self, + namespace_id: uuid.UUID, + additional_size: int, +) -> None: + """ + Check the quota for a StorageNamespace. + + Same logic applies regardless of storage_type (object_storage/vfs_storage). + """ + # 1. Get max_size from namespace + namespace = await self._storage_namespace_repo.get_by_id(namespace_id) + + # NULL means unlimited + if namespace.max_size is None: + return + + # 2. Aggregate current usage (sum of revision sizes linked via association table) + usage = await self._storage_namespace_repo.get_usage(namespace_id) + + # 3. Check if limit would be exceeded + if usage.total_size + additional_size > namespace.max_size: + raise StorageNamespaceQuotaExceededError( + namespace_id=namespace_id, + current_size=usage.total_size, + max_size=namespace.max_size, + requested_size=additional_size, + ) +``` + +### Usage Aggregation Query + +```sql +SELECT + COALESCE(SUM(ar.size), 0) as total_size, + COUNT(ar.id) as revision_count +FROM association_artifacts_storages aas +JOIN artifact_revisions ar ON aas.artifact_revision_id = ar.id +WHERE aas.storage_namespace_id = :namespace_id +``` + +## Object Storage vs VFS Storage Comparison + +| Aspect | Object Storage | VFS Storage | +|--------|---------------|-------------| +| Config key | `bucket_name` | `subpath` | +| Reference table | `object_storage` | `vfs_storage` | +| `storage_type` value | `"object_storage"` | `"vfs_storage"` | +| Physical storage | S3 bucket | Filesystem path | +| Quota management | **Same** (StorageNamespaceRow.max_size) | +| Usage aggregation | **Same** (association table based) | + +## API + +### GET /storage-namespaces/{id}/usage + +```json +{ + "namespace_id": "550e8400-e29b-41d4-a716-446655440000", + "storage_type": "object_storage", + "namespace": "artifacts", + "total_size": 10737418240, + "max_size": 107374182400, + "revision_count": 42, + "utilization_percent": 10.0 +} +``` + +### PATCH /storage-namespaces/{id}/quota + +```json +{ "max_size": 107374182400 } +``` + +Or set to unlimited: + +```json +{ "max_size": null } +``` diff --git a/proposals/BEP-1036/vfolder_storage.md b/proposals/BEP-1036/vfolder_storage.md new file mode 100644 index 00000000000..ba71fa9a9e8 --- /dev/null +++ b/proposals/BEP-1036/vfolder_storage.md @@ -0,0 +1,296 @@ +# VFolder Storage Quota + +Quota system for artifacts stored in a specific VFolder when the user provides a `vfolder_id`. + +## Overview + +When `vfolder_id` is provided in `import_revision()`, the artifact is stored in the specified VFolder. +VFolders already have an existing quota system via `quota_scope`, but there is **no pre-validation** during artifact import. + +**Current Problems:** +- No quota check before import starts +- Large artifact imports can fail mid-way when storage proxy detects quota exceeded +- Failed imports may leave partial downloads + +## Data Model + +### VFolder Quota Structure + +``` +┌─────────────────────────────────────────────────────────────────────────┐ +│ VFolder Quota Structure │ +├─────────────────────────────────────────────────────────────────────────┤ +│ │ +│ quota_scope_id determination based on Ownership Type │ +│ │ +│ ┌─────────────────────┐ ┌─────────────────────┐ │ +│ │ User-owned VFolder │ │ Project-owned VFolder│ │ +│ ├─────────────────────┤ ├─────────────────────┤ │ +│ │ ownership_type: │ │ ownership_type: │ │ +│ │ "user" │ │ "group" │ │ +│ │ │ │ │ │ +│ │ quota_scope_id: │ │ quota_scope_id: │ │ +│ │ "user:{user_uuid}"│ │ "project:{group_id}" │ +│ └──────────┬──────────┘ └──────────┬──────────┘ │ +│ │ │ │ +│ ▼ ▼ │ +│ ┌─────────────────────┐ ┌─────────────────────┐ │ +│ │ users │ │ groups │ │ +│ ├─────────────────────┤ ├─────────────────────┤ │ +│ │ uuid │ │ id │ │ +│ │ resource_policy ────┼──┐ │ resource_policy ────┼──┐ │ +│ └─────────────────────┘ │ └─────────────────────┘ │ │ +│ │ │ │ +│ ▼ ▼ │ +│ ┌─────────────────────────────┐ ┌─────────────────────────────┐ │ +│ │ user_resource_policies │ │ project_resource_policies │ │ +│ ├─────────────────────────────┤ ├─────────────────────────────┤ │ +│ │ name │ │ name │ │ +│ │ max_vfolder_count │ │ max_vfolder_count │ │ +│ │ max_quota_scope_size ◄─────┼──┼── Quota limit (bytes) │ │ +│ │ ... │ │ max_network_count │ │ +│ └─────────────────────────────┘ └─────────────────────────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────────────────┘ +``` + +### VFolder Row Structure + +``` +┌─────────────────────────┐ +│ vfolders │ +├─────────────────────────┤ +│ id │ +│ name │ +│ host │ ─── storage proxy host (e.g., "local:volume1") +│ quota_scope_id │ ─── "user:{uuid}" or "project:{uuid}" +│ usage_mode │ +│ permission │ +│ ownership_type │ ─── "user" or "group" +│ user │ ─── owner user UUID (for user-owned) +│ group │ ─── owner group ID (for project-owned) +│ ... │ +└─────────────────────────┘ +``` + +## Import Flow + +``` +┌─────────────────────────────────────────────────────────────────────────┐ +│ VFolder Destination Import Flow │ +├─────────────────────────────────────────────────────────────────────────┤ +│ │ +│ import_revision(action) where action.vfolder_id is provided │ +│ │ +│ ┌────────────────────────────────────────────────────────────────────┐ │ +│ │ 1. Get VFolder info │ │ +│ │ vfolder = await vfolder_repository.get_by_id(action.vfolder_id) │ │ +│ │ │ │ +│ │ vfolder = { │ │ +│ │ id: "...", │ │ +│ │ host: "local:volume1", │ │ +│ │ quota_scope_id: "user:abc123...", │ │ +│ │ } │ │ +│ └────────────────────────────────────────────────────────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌────────────────────────────────────────────────────────────────────┐ │ +│ │ 2. Build VFolderID │ │ +│ │ vfolder_id = VFolderID(vfolder.quota_scope_id, vfolder.id) │ │ +│ │ volume_name = parse_host(vfolder.host) # "volume1" │ │ +│ └────────────────────────────────────────────────────────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌────────────────────────────────────────────────────────────────────┐ │ +│ │ 3. Quota Check (NEW) │ │ +│ │ │ │ +│ │ ┌──────────────────────────────────────────────────────────┐ │ │ +│ │ │ a. Get current usage from storage proxy │ │ │ +│ │ │ usage = await storage_client.get_quota_scope_usage( │ │ │ +│ │ │ volume_name, quota_scope_id │ │ │ +│ │ │ ) │ │ │ +│ │ │ # returns: { used_bytes: 5368709120 } │ │ │ +│ │ └──────────────────────────────────────────────────────────┘ │ │ +│ │ │ │ │ +│ │ ▼ │ │ +│ │ ┌──────────────────────────────────────────────────────────┐ │ │ +│ │ │ b. Get quota limit from resource policy │ │ │ +│ │ │ max_size = await get_quota_scope_limit(quota_scope_id)│ │ │ +│ │ │ # Query user_resource_policies or │ │ │ +│ │ │ # project_resource_policies based on scope type │ │ │ +│ │ │ # returns: 10737418240 (10GB) or -1 (unlimited) │ │ │ +│ │ └──────────────────────────────────────────────────────────┘ │ │ +│ │ │ │ │ +│ │ ▼ │ │ +│ │ ┌──────────────────────────────────────────────────────────┐ │ │ +│ │ │ c. Compare │ │ │ +│ │ │ if max_size > 0: # -1 means unlimited │ │ │ +│ │ │ if usage.used_bytes + revision.size > max_size: │ │ │ +│ │ │ raise VFolderQuotaExceededError(...) │ │ │ +│ │ └──────────────────────────────────────────────────────────┘ │ │ +│ └────────────────────────────────────────────────────────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌────────────────────────────────────────────────────────────────────┐ │ +│ │ 4. Proceed with import │ │ +│ │ vfolder_target = VFolderStorageTarget( │ │ +│ │ vfolder_id=vfolder_id, │ │ +│ │ volume_name=volume_name, │ │ +│ │ ) │ │ +│ │ │ │ +│ │ await storage_proxy.import_huggingface_models( │ │ +│ │ storage_step_target_mappings={ │ │ +│ │ step: vfolder_target for step in ArtifactStorageImportStep │ +│ │ } │ │ +│ │ ) │ │ +│ └────────────────────────────────────────────────────────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌────────────────────────────────────────────────────────────────────┐ │ +│ │ 5. Storage Proxy writes to VFolder path │ │ +│ │ /mnt/vfhost/quota_scope_id/vfolder_id/{artifact_files} │ │ +│ └────────────────────────────────────────────────────────────────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────────────────┘ +``` + +## Quota Check Logic + +```python +async def check_vfolder_quota( + self, + vfolder_data: VFolderData, + additional_size: int, +) -> None: + """ + Check the quota scope limit for a VFolder. + """ + quota_scope_id = vfolder_data.quota_scope_id + _, volume_name = self._storage_manager.get_proxy_and_volume(vfolder_data.host) + + # 1. Get current quota scope usage from storage proxy + storage_client = self._storage_manager.get_manager_facing_client(vfolder_data.host) + usage = await storage_client.get_quota_scope_usage(volume_name, str(quota_scope_id)) + + # 2. Get limit from resource policy + max_size = await self._get_quota_scope_limit(quota_scope_id) + + # -1 means unlimited + if max_size < 0: + return + + # 3. Check if limit would be exceeded + if usage.used_bytes + additional_size > max_size: + raise VFolderQuotaExceededError( + vfolder_id=VFolderID(quota_scope_id, vfolder_data.id), + quota_scope_id=quota_scope_id, + current_size=usage.used_bytes, + max_size=max_size, + requested_size=additional_size, + ) + + +async def _get_quota_scope_limit(self, quota_scope_id: QuotaScopeID) -> int: + """ + Parse scope type from QuotaScopeID and query the appropriate resource policy. + """ + scope_type, scope_uuid = quota_scope_id.split(":", 1) + + match scope_type: + case "user": + user = await self._user_repo.get_by_uuid(UUID(scope_uuid)) + policy = await self._user_policy_repo.get_by_name(user.resource_policy) + return policy.max_quota_scope_size + + case "project": + group = await self._group_repo.get_by_id(UUID(scope_uuid)) + policy = await self._project_policy_repo.get_by_name(group.resource_policy) + return policy.max_quota_scope_size +``` + +## Storage Proxy API + +### GET /volumes/{volume}/quota-scopes/{quota_scope_id} + +Returns current usage of the quota scope used by the VFolder. + +**Request:** +``` +GET /volumes/volume1/quota-scopes/user:abc123... +``` + +**Response:** +```json +{ + "used_bytes": 5368709120, + "limit_bytes": 10737418240 +} +``` + +> Note: `limit_bytes` is the limit set at the storage proxy level and may be managed +> separately from the manager's resource policy. This BEP prioritizes the manager's +> resource policy value. + +## User vs Project VFolder Comparison + +| Aspect | User VFolder | Project VFolder | +|--------|-------------|-----------------| +| `ownership_type` | `"user"` | `"group"` | +| `quota_scope_id` format | `"user:{user_uuid}"` | `"project:{group_id}"` | +| Resource Policy table | `user_resource_policies` | `project_resource_policies` | +| Quota field | `max_quota_scope_size` | `max_quota_scope_size` | +| Owner reference | `vfolders.user` | `vfolders.group` | + +## Current vs Proposed Behavior + +### Current Behavior + +``` +import_revision(vfolder_id=...) + │ + ▼ +storage_proxy.import_models(vfolder_target=...) + │ + ▼ +Storage Proxy detects quota exceeded during file write + │ + ▼ +Error returned (import fails mid-way) + │ + ▼ +May leave partial download state +``` + +### Proposed Behavior + +``` +import_revision(vfolder_id=...) + │ + ▼ +quota_service.check_vfolder_quota(...) ◄── NEW: Pre-validation + │ + ├── If exceeded: Return VFolderQuotaExceededError immediately + │ + ▼ (if passed) +storage_proxy.import_models(vfolder_target=...) + │ + ▼ +Complete successfully +``` + +## Error Response Example + +```json +{ + "error": "VFolderQuotaExceededError", + "message": "VFolder quota scope limit would be exceeded", + "details": { + "vfolder_id": "user:abc123.../vf-123...", + "quota_scope_id": "user:abc123...", + "current_size_bytes": 5368709120, + "max_size_bytes": 10737418240, + "requested_size_bytes": 8589934592, + "available_bytes": 5368709120 + } +} +``` From f07109f3d20c94675515c666f74bff3d0224ad2b Mon Sep 17 00:00:00 2001 From: Gyubong Date: Thu, 22 Jan 2026 11:11:11 +0900 Subject: [PATCH 2/9] docs: Add news fragment --- changes/8205.docs.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/8205.docs.md diff --git a/changes/8205.docs.md b/changes/8205.docs.md new file mode 100644 index 00000000000..c49455f0b61 --- /dev/null +++ b/changes/8205.docs.md @@ -0,0 +1 @@ +Add BEP-1036 for Artifact Storage Usage Tracking and Quota Enforcement From 7d0f2aa4a40ca2c026880baeeda14113029d2d88 Mon Sep 17 00:00:00 2001 From: Gyubong Date: Thu, 22 Jan 2026 11:14:13 +0900 Subject: [PATCH 3/9] docs: Update --- proposals/BEP-1036-artifact-storage-quota.md | 109 ++++++++++++++++--- proposals/BEP-1036/quota-flow.md | 108 ------------------ 2 files changed, 91 insertions(+), 126 deletions(-) delete mode 100644 proposals/BEP-1036/quota-flow.md diff --git a/proposals/BEP-1036-artifact-storage-quota.md b/proposals/BEP-1036-artifact-storage-quota.md index 6eb72a51749..47cd445abc2 100644 --- a/proposals/BEP-1036-artifact-storage-quota.md +++ b/proposals/BEP-1036-artifact-storage-quota.md @@ -91,28 +91,104 @@ Each storage destination has different quota mechanisms: | StorageNamespace | `StorageNamespaceRow.max_size` (NEW) | Aggregated from `artifact_revisions` via association table | [storage_namespace.md](BEP-1036/storage_namespace.md) | | VFolder | `max_quota_scope_size` from resource policy | Storage proxy API | [vfolder_storage.md](BEP-1036/vfolder_storage.md) | -### Import Flow Integration +### Import Flow with Quota Check -The quota check is integrated into the import flow before any file transfer begins: +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ import_revision(action) │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ │ +│ 1. Get revision_data (includes size) │ +│ │ +│ 2. Determine storage destination │ +│ ┌─────────────────────────────┬────────────────────────────────────┐ │ +│ │ vfolder_id provided? │ │ │ +│ └──────────┬──────────────────┴────────────────────┬───────────────┘ │ +│ │ YES │ NO │ +│ ▼ ▼ │ +│ ┌─────────────────────────┐ ┌─────────────────────────────┐ │ +│ │ VFolderDestination │ │ StorageNamespaceDestination│ │ +│ │ - vfolder_id │ │ - namespace_id │ │ +│ │ - quota_scope_id │ │ │ │ +│ └───────────┬─────────────┘ └───────────────┬─────────────┘ │ +│ │ │ │ +│ └──────────────┬──────────────────────────┘ │ +│ ▼ │ +│ 3. ┌──────────────────────────────────────────────────────────────────┐ │ +│ │ ArtifactStorageQuotaService.check_quota() │ │ +│ └──────────────────────────────────────────────────────────────────┘ │ +│ │ │ +│ ┌──────────────┴──────────────┐ │ +│ ▼ ▼ │ +│ ┌─────────────────────────┐ ┌─────────────────────────────────┐ │ +│ │ _check_vfolder_quota() │ │ _check_storage_namespace_quota()│ │ +│ └───────────┬─────────────┘ └───────────────┬─────────────────┘ │ +│ │ │ │ +│ └──────────────┬────────────────────┘ │ +│ ▼ │ +│ ┌──────────────────────────────┐ │ +│ │ Quota Exceeded? │ │ +│ └──────────────┬───────────────┘ │ +│ YES │ │ NO │ +│ ▼ ▼ │ +│ ┌─────────────────────────┐ ┌─────────────────────────────────┐ │ +│ │ Raise appropriate error │ │ 4. Proceed with import │ │ +│ │ - VFolderQuotaExceeded │ │ - Call storage proxy │ │ +│ │ - StorageNamespace │ │ - Update status │ │ +│ │ QuotaExceeded │ │ - Associate with storage │ │ +│ └─────────────────────────┘ └─────────────────────────────────┘ │ +│ │ +└──────────────────────────────────────────────────────────────────────────────┘ +``` -```python -async def import_revision(self, action: ImportArtifactRevisionAction) -> ...: - revision_data = await self._artifact_repository.get_artifact_revision_by_id(...) +### Data Sources for Quota Check + +**StorageNamespace**: Usage is aggregated from artifact revisions linked via association table. - # Determine storage destination and check quota - if action.vfolder_id: - destination = VFolderDestination(...) - else: - destination = StorageNamespaceDestination(namespace_id=namespace_id) +``` +┌─────────────────────┐ ┌─────────────────────────────────┐ +│ storage_namespace │ │ association_artifacts_storages │ +├─────────────────────┤ ├─────────────────────────────────┤ +│ id │◄────│ storage_namespace_id │ +│ max_size (NEW) │ │ artifact_revision_id ───────────┼──┐ +└─────────────────────┘ └─────────────────────────────────┘ │ + │ + ┌─────────────────────────────────┐ │ + │ artifact_revisions │ │ + ├─────────────────────────────────┤ │ + │ id ◄────────────────────────────┼──┘ + │ size │ + └─────────────────────────────────┘ +``` - if revision_data.size: - await self._quota_service.check_quota(destination, revision_data.size) +**VFolder**: Usage is queried from storage proxy, limit comes from resource policies. - # Proceed with import only if quota check passes - ... ``` +┌─────────────────────┐ ┌─────────────────────────────────┐ +│ vfolders │ │ user_resource_policies / │ +├─────────────────────┤ │ project_resource_policies │ +│ id │ ├─────────────────────────────────┤ +│ quota_scope_id ─────┼────►│ max_quota_scope_size │ +└─────────────────────┘ └─────────────────────────────────┘ + │ + │ VFolderID + ▼ +┌─────────────────────┐ +│ Storage Proxy │ +├─────────────────────┤ +│ get_quota_scope_ │ +│ usage(quota_scope) │──► Current usage in bytes +└─────────────────────┘ +``` + +### Quota Check Comparison -See [quota-flow.md](BEP-1036/quota-flow.md) for the complete flow diagram. +| Aspect | StorageNamespace | VFolder | +|--------|------------------|---------| +| Limit Source | `storage_namespace.max_size` | `resource_policy.max_quota_scope_size` | +| Usage Source | DB aggregation via association table | Storage proxy API | +| Unlimited Value | `NULL` | `-1` | +| Scope | Per namespace | Per quota scope (user/project) | ### Error Types @@ -203,6 +279,3 @@ When storage usage approaches the configured limit, the system could notify admi ## References - [BEP-1019: MinIO Artifact Registry Storage](BEP-1019-minio-artifact-registry-storage.md) -- [StorageNamespace Quota Details](BEP-1036/storage_namespace.md) -- [VFolder Storage Quota Details](BEP-1036/vfolder_storage.md) -- [Quota Flow Diagram](BEP-1036/quota-flow.md) diff --git a/proposals/BEP-1036/quota-flow.md b/proposals/BEP-1036/quota-flow.md deleted file mode 100644 index 633cac2209a..00000000000 --- a/proposals/BEP-1036/quota-flow.md +++ /dev/null @@ -1,108 +0,0 @@ -# Artifact Storage Quota Flow Diagram - -## Unified Import Flow with Quota Check - -``` -┌─────────────────────────────────────────────────────────────────────────────┐ -│ import_revision(action) │ -├─────────────────────────────────────────────────────────────────────────────┤ -│ │ -│ 1. Get revision_data (includes size) │ -│ │ -│ 2. Determine storage destination │ -│ ┌─────────────────────────────┬────────────────────────────────────┐ │ -│ │ vfolder_id provided? │ │ │ -│ └──────────┬──────────────────┴────────────────────┬───────────────┘ │ -│ │ YES │ NO │ -│ ▼ ▼ │ -│ ┌─────────────────────────┐ ┌─────────────────────────────┐ │ -│ │ VFolderDestination │ │ StorageNamespaceDestination│ │ -│ │ - vfolder_id │ │ - namespace_id │ │ -│ │ - quota_scope_id │ │ │ │ -│ └───────────┬─────────────┘ └───────────────┬─────────────┘ │ -│ │ │ │ -│ └──────────────┬──────────────────────────┘ │ -│ ▼ │ -│ 3. ┌──────────────────────────────────────────────────────────────────┐ │ -│ │ ArtifactStorageQuotaService.check_quota() │ │ -│ └──────────────────────────────────────────────────────────────────┘ │ -│ │ │ -│ ┌──────────────┴──────────────┐ │ -│ ▼ ▼ │ -│ ┌─────────────────────────┐ ┌─────────────────────────────────┐ │ -│ │ _check_vfolder_quota() │ │ _check_storage_namespace_quota()│ │ -│ │ │ │ │ │ -│ │ See: vfolder_storage.md │ │ See: storage_namespace.md │ │ -│ └───────────┬─────────────┘ └───────────────┬─────────────────┘ │ -│ │ │ │ -│ └──────────────┬────────────────────┘ │ -│ ▼ │ -│ ┌──────────────────────────────┐ │ -│ │ Quota Exceeded? │ │ -│ └──────────────┬───────────────┘ │ -│ YES │ │ NO │ -│ ▼ ▼ │ -│ ┌─────────────────────────┐ ┌─────────────────────────────────┐ │ -│ │ Raise appropriate error │ │ 4. Proceed with import │ │ -│ │ - VFolderQuotaExceeded │ │ - Call storage proxy │ │ -│ │ - StorageNamespace │ │ - Update status │ │ -│ │ QuotaExceeded │ │ - Associate with storage │ │ -│ └─────────────────────────┘ └─────────────────────────────────┘ │ -│ │ -└──────────────────────────────────────────────────────────────────────────────┘ -``` - -## Data Sources for Quota Check - -### StorageNamespace Quota - -Usage is aggregated from artifact revisions linked to the namespace via the association table. - -``` -┌─────────────────────┐ ┌─────────────────────────────────┐ -│ storage_namespace │ │ association_artifacts_storages │ -├─────────────────────┤ ├─────────────────────────────────┤ -│ id │◄────│ storage_namespace_id │ -│ max_size (NEW) │ │ artifact_revision_id ───────────┼──┐ -└─────────────────────┘ └─────────────────────────────────┘ │ - │ - ┌─────────────────────────────────┐ │ - │ artifact_revisions │ │ - ├─────────────────────────────────┤ │ - │ id ◄────────────────────────────┼──┘ - │ size │ - └─────────────────────────────────┘ - -Usage = SUM(artifact_revisions.size) WHERE linked to namespace -``` - -### VFolder Quota - -Usage is queried from the storage proxy, and limit comes from resource policies. - -``` -┌─────────────────────┐ ┌─────────────────────────────────┐ -│ vfolders │ │ user_resource_policies / │ -├─────────────────────┤ │ project_resource_policies │ -│ id │ ├─────────────────────────────────┤ -│ quota_scope_id ─────┼────►│ max_quota_scope_size │ -└─────────────────────┘ └─────────────────────────────────┘ - │ - │ VFolderID - ▼ -┌─────────────────────┐ -│ Storage Proxy │ -├─────────────────────┤ -│ get_quota_scope_ │ -│ usage(quota_scope) │──► Current usage in bytes -└─────────────────────┘ -``` - -## Quota Check Comparison - -| Aspect | StorageNamespace | VFolder | -|--------|------------------|---------| -| Limit Source | `storage_namespace.max_size` | `resource_policy.max_quota_scope_size` | -| Usage Source | DB aggregation via association table | Storage proxy API | -| Unlimited Value | `NULL` | `-1` | -| Scope | Per namespace | Per quota scope (user/project) | From b0f3a0fb10e195b2f0e13686c3a83d94a07cafc9 Mon Sep 17 00:00:00 2001 From: Gyubong Date: Thu, 22 Jan 2026 11:20:11 +0900 Subject: [PATCH 4/9] docs: Update --- proposals/BEP-1036/storage_namespace.md | 129 +++--------------------- 1 file changed, 12 insertions(+), 117 deletions(-) diff --git a/proposals/BEP-1036/storage_namespace.md b/proposals/BEP-1036/storage_namespace.md index 773edc60f7b..a03e356766e 100644 --- a/proposals/BEP-1036/storage_namespace.md +++ b/proposals/BEP-1036/storage_namespace.md @@ -10,6 +10,8 @@ This storage is managed by `StorageNamespaceRow` and supports two backend types: - **Object Storage** (MinIO, S3, etc.) - **VFS Storage** (local/network filesystem) +Both types share the same quota mechanism via `StorageNamespaceRow.max_size`. + ## Data Model ### Current Structure @@ -74,103 +76,15 @@ This storage is managed by `StorageNamespaceRow` and supports two backend types: └─────────────────────────┘ ``` -## Scenarios by Storage Type - -### Object Storage (MinIO/S3) - -``` -┌─────────────────────────────────────────────────────────────────────────┐ -│ Object Storage Scenario │ -├─────────────────────────────────────────────────────────────────────────┤ -│ │ -│ Reservoir Config │ -│ ┌────────────────────────────────────────┐ │ -│ │ storage_type: "object_storage" │ │ -│ │ archive_storage: "minio-main" │ │ -│ │ bucket_name: "artifacts" │ ─── namespace │ -│ └────────────────────────────────────────┘ │ -│ │ -│ import_revision() flow │ -│ ┌──────────────────────────────────────────────────────────────────┐ │ -│ │ 1. namespace = reservoir_config.bucket_name │ │ -│ │ │ │ -│ │ 2. object_storage = get_by_name("minio-main") │ │ -│ │ → ObjectStorageRow { id, host, endpoint, ... } │ │ -│ │ │ │ -│ │ 3. storage_namespace = get_by_storage_and_namespace( │ │ -│ │ storage_id=object_storage.id, │ │ -│ │ namespace="artifacts" │ │ -│ │ ) │ │ -│ │ → StorageNamespaceRow { id, max_size, ... } │ │ -│ │ │ │ -│ │ 4. quota_check(storage_namespace.id, revision.size) │ │ -│ │ │ │ -│ │ 5. storage_proxy.import_huggingface_models(...) │ │ -│ │ → Files stored in MinIO bucket │ │ -│ │ │ │ -│ │ 6. associate_artifact_with_storage( │ │ -│ │ revision_id, namespace_id, "object_storage" │ │ -│ │ ) │ │ -│ └──────────────────────────────────────────────────────────────────┘ │ -│ │ -│ Physical storage location │ -│ ┌──────────────────────────────────────────────────────────────────┐ │ -│ │ MinIO: s3://artifacts/{revision_id}/model.safetensors │ │ -│ │ ▲ │ │ -│ │ │ │ │ -│ │ bucket_name (namespace) │ │ -│ └──────────────────────────────────────────────────────────────────┘ │ -│ │ -└─────────────────────────────────────────────────────────────────────────┘ -``` +## Backend Type Differences -### VFS Storage (Filesystem) +| Aspect | Object Storage | VFS Storage | +|--------|---------------|-------------| +| Config key for namespace | `bucket_name` | `subpath` | +| Reference table | `object_storage` | `vfs_storage` | +| Physical storage | S3 bucket (e.g., `s3://artifacts/`) | Filesystem path (e.g., `/mnt/nfs/artifacts/`) | -``` -┌─────────────────────────────────────────────────────────────────────────┐ -│ VFS Storage Scenario │ -├─────────────────────────────────────────────────────────────────────────┤ -│ │ -│ Reservoir Config │ -│ ┌────────────────────────────────────────┐ │ -│ │ storage_type: "vfs_storage" │ │ -│ │ archive_storage: "nfs-models" │ │ -│ │ subpath: "reservoir/artifacts" │ ─── namespace │ -│ └────────────────────────────────────────┘ │ -│ │ -│ import_revision() flow │ -│ ┌──────────────────────────────────────────────────────────────────┐ │ -│ │ 1. namespace = reservoir_config.subpath │ │ -│ │ │ │ -│ │ 2. vfs_storage = get_by_name("nfs-models") │ │ -│ │ → VFSStorageRow { id, host, base_path, ... } │ │ -│ │ │ │ -│ │ 3. storage_namespace = get_by_storage_and_namespace( │ │ -│ │ storage_id=vfs_storage.id, │ │ -│ │ namespace="reservoir/artifacts" │ │ -│ │ ) │ │ -│ │ → StorageNamespaceRow { id, max_size, ... } │ │ -│ │ │ │ -│ │ 4. quota_check(storage_namespace.id, revision.size) │ │ -│ │ │ │ -│ │ 5. storage_proxy.import_huggingface_models(...) │ │ -│ │ → Files stored on NFS │ │ -│ │ │ │ -│ │ 6. associate_artifact_with_storage( │ │ -│ │ revision_id, namespace_id, "vfs_storage" │ │ -│ │ ) │ │ -│ └──────────────────────────────────────────────────────────────────┘ │ -│ │ -│ Physical storage location │ -│ ┌──────────────────────────────────────────────────────────────────┐ │ -│ │ NFS: /mnt/nfs/reservoir/artifacts/{revision_id}/model.safetensors│ │ -│ │ ▲ │ │ -│ │ │ │ │ -│ │ base_path + subpath (namespace) │ │ -│ └──────────────────────────────────────────────────────────────────┘ │ -│ │ -└─────────────────────────────────────────────────────────────────────────┘ -``` +Quota management and usage aggregation are identical for both types. ## Quota Check Logic @@ -182,27 +96,19 @@ async def check_storage_namespace_quota( ) -> None: """ Check the quota for a StorageNamespace. - - Same logic applies regardless of storage_type (object_storage/vfs_storage). + Same logic applies regardless of storage_type. """ - # 1. Get max_size from namespace namespace = await self._storage_namespace_repo.get_by_id(namespace_id) # NULL means unlimited if namespace.max_size is None: return - # 2. Aggregate current usage (sum of revision sizes linked via association table) + # Aggregate current usage via association table usage = await self._storage_namespace_repo.get_usage(namespace_id) - # 3. Check if limit would be exceeded if usage.total_size + additional_size > namespace.max_size: - raise StorageNamespaceQuotaExceededError( - namespace_id=namespace_id, - current_size=usage.total_size, - max_size=namespace.max_size, - requested_size=additional_size, - ) + raise StorageNamespaceQuotaExceededError(...) ``` ### Usage Aggregation Query @@ -216,17 +122,6 @@ JOIN artifact_revisions ar ON aas.artifact_revision_id = ar.id WHERE aas.storage_namespace_id = :namespace_id ``` -## Object Storage vs VFS Storage Comparison - -| Aspect | Object Storage | VFS Storage | -|--------|---------------|-------------| -| Config key | `bucket_name` | `subpath` | -| Reference table | `object_storage` | `vfs_storage` | -| `storage_type` value | `"object_storage"` | `"vfs_storage"` | -| Physical storage | S3 bucket | Filesystem path | -| Quota management | **Same** (StorageNamespaceRow.max_size) | -| Usage aggregation | **Same** (association table based) | - ## API ### GET /storage-namespaces/{id}/usage From a50b4200804921d24fedad866002f663f52a8331 Mon Sep 17 00:00:00 2001 From: Gyubong Date: Thu, 22 Jan 2026 11:25:00 +0900 Subject: [PATCH 5/9] docs: Update --- proposals/BEP-1036-artifact-storage-quota.md | 17 ----------------- proposals/BEP-1036/storage_namespace.md | 18 ++---------------- 2 files changed, 2 insertions(+), 33 deletions(-) diff --git a/proposals/BEP-1036-artifact-storage-quota.md b/proposals/BEP-1036-artifact-storage-quota.md index 47cd445abc2..4d1130f1b6b 100644 --- a/proposals/BEP-1036-artifact-storage-quota.md +++ b/proposals/BEP-1036-artifact-storage-quota.md @@ -9,11 +9,6 @@ Implemented-Version: # Artifact Storage Usage Tracking and Quota Enforcement -## Related Issues - -- JIRA: BA-3989 (Epic), BA-3990, BA-3991, BA-3992, BA-3994 -- GitHub: #8194 - ## Motivation Currently, artifact storage has no usage tracking or capacity limits. When artifacts are imported, they are stored without any visibility into how much space is being consumed or any mechanism to prevent storage exhaustion. @@ -237,18 +232,6 @@ Updates quota for a storage namespace. Admin-only. { "max_size": 107374182400 } // or null for unlimited ``` -## Migration / Compatibility - -### Backward Compatibility - -- Existing storage namespaces will have `max_size = NULL` (unlimited) -- No changes to existing import behavior unless quota is explicitly set -- VFolder quota integration uses existing `max_quota_scope_size` from resource policies - -### Breaking Changes - -None. This is an additive feature. - ## Testing Scenarios ### StorageNamespace Quota Tests diff --git a/proposals/BEP-1036/storage_namespace.md b/proposals/BEP-1036/storage_namespace.md index a03e356766e..28114aed51f 100644 --- a/proposals/BEP-1036/storage_namespace.md +++ b/proposals/BEP-1036/storage_namespace.md @@ -14,8 +14,6 @@ Both types share the same quota mechanism via `StorageNamespaceRow.max_size`. ## Data Model -### Current Structure - ``` ┌─────────────────────────┐ │ object_storage │ @@ -33,7 +31,8 @@ Both types share the same quota mechanism via `StorageNamespaceRow.max_size`. ├─────────────────────────┤ │ id │◄─────────────────────────────────────┐ │ storage_id │ │ -│ namespace (bucket name) │ │ +│ namespace │ │ +│ max_size (NEW) │ ◄── NULL = unlimited, in bytes │ └─────────────────────────┘ │ │ ┌─────────────────────────┐ ┌─────────────────────────┐ │ @@ -63,19 +62,6 @@ Both types share the same quota mechanism via `StorageNamespaceRow.max_size`. └─────────────────────────┘ ``` -### Proposed: Add max_size - -``` -┌─────────────────────────┐ -│ storage_namespace │ -├─────────────────────────┤ -│ id │ -│ storage_id │ -│ namespace │ -│ max_size (NEW) │ ◄── NULL = unlimited, in bytes -└─────────────────────────┘ -``` - ## Backend Type Differences | Aspect | Object Storage | VFS Storage | From dc95e3b38d207b9ec63f8e7706fb48c070288db1 Mon Sep 17 00:00:00 2001 From: jopemachine Date: Thu, 5 Mar 2026 05:07:48 +0000 Subject: [PATCH 6/9] docs: Add news fragment --- changes/{8205.docs.md => 8565.docs.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changes/{8205.docs.md => 8565.docs.md} (100%) diff --git a/changes/8205.docs.md b/changes/8565.docs.md similarity index 100% rename from changes/8205.docs.md rename to changes/8565.docs.md From 1215d6361570998f83af50c21159de582a4de9b9 Mon Sep 17 00:00:00 2001 From: jopemachine Date: Thu, 5 Mar 2026 05:09:08 +0000 Subject: [PATCH 7/9] docs: Add news fragment --- changes/{8565.docs.md => 8565.doc.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changes/{8565.docs.md => 8565.doc.md} (100%) diff --git a/changes/8565.docs.md b/changes/8565.doc.md similarity index 100% rename from changes/8565.docs.md rename to changes/8565.doc.md From 98391e802eced30d98d79339cea0c84fa15dcb5d Mon Sep 17 00:00:00 2001 From: jopemachine Date: Thu, 5 Mar 2026 05:15:54 +0000 Subject: [PATCH 8/9] docs: Add news fragment --- proposals/BEP-1036-artifact-storage-quota.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proposals/BEP-1036-artifact-storage-quota.md b/proposals/BEP-1036-artifact-storage-quota.md index 4d1130f1b6b..92ceefea385 100644 --- a/proposals/BEP-1036-artifact-storage-quota.md +++ b/proposals/BEP-1036-artifact-storage-quota.md @@ -2,7 +2,7 @@ Author: Gyubong Lee (gbl@lablup.com) Status: Draft Created: 2026-01-22 -Created-Version: 26.2.0 +Created-Version: 26.3.0 Target-Version: Implemented-Version: --- From 42100167bf264ecc645e1c5eb89ae1ef8d9b46d2 Mon Sep 17 00:00:00 2001 From: jopemachine Date: Thu, 5 Mar 2026 05:19:11 +0000 Subject: [PATCH 9/9] WIP --- proposals/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proposals/README.md b/proposals/README.md index df3cf67ee96..a5922a42148 100644 --- a/proposals/README.md +++ b/proposals/README.md @@ -106,7 +106,7 @@ BEP numbers start from 1000. | [1033](BEP-1033-sokovan-handler-test-scenarios.md) | Sokovan Handler Test Scenarios | HyeokJin Kim | Draft | | [1034](BEP-1034-kernel-v2-gql-implementation.md) | KernelV2 GQL Implementation | Gyubong Lee | Draft | | [1035](BEP-1035-request-id-tracing.md) | Distributed Request ID Propagation | Gyubong Lee | Draft | -| [1036](BEP-1036-artifact-storage-quota.md) | Artifact Storage Usage Tracking and Quota Enforcement | Gyubong Lee | Rejected | +| [1036](BEP-1036-artifact-storage-quota.md) | Artifact Storage Usage Tracking and Quota Enforcement | Gyubong Lee | Draft | | [1037](BEP-1037-storage-proxy-health-monitoring.md) | Volume Host Availability Check | Gyubong Lee | Draft | | [1038](BEP-1038-image-v2-gql-implementation.md) | ImageV2 GQL Implementation | Gyubong Lee | Draft | | [1039](BEP-1039-image-id-based-service-logic-migration.md) | Image ID Based Service Logic Migration | Gyubong Lee | Draft |