From 5345e345dedcaa4f719d330afba8bc402a21ca9c Mon Sep 17 00:00:00 2001 From: Alisson Patricio Date: Wed, 20 Sep 2023 10:29:38 -0300 Subject: [PATCH 01/20] baseapp-files: initial package, ported to plugin architecture - AbstractFile/AbstractFileTarget rebuilt on DocumentIdMixin + django-swappable-models (drops the removed CommentableModel/ReactableModel/ReportableModel mixins) - FilesPlugin registered via the plugin registry (GraphQL queries/mutations, v1 files router, FilesPermissionsBackend auth slot) + pyproject entry point - apps.py uses BaseAppConfig and registers FilesInterface as a shared GraphQL interface - File ObjectType implements FileInterface and composes Comments/Reactions via graphql_shared_interfaces.get() - comments opt into files as a target (guarded FileableModel mixin + FilesInterface) - baseapp_core.graphql: File type + ThumbnailField return the URL string directly (no nested { url }); image-field consumers updated to the string contract - testproject migrations regenerated for the ported concrete models - rebased onto origin/master Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01XyuGcRJryi9hQiWRVXU7Js --- .../tests/test_graphql_mutations.py | 8 +- .../tests/test_graphql_mutations_create.py | 4 +- baseapp/files/README.md | 827 ++++++++++++++++++ baseapp/files/UPLOAD_FLOW.md | 193 ++++ baseapp/files/__init__.py | 0 baseapp/files/admin.py | 24 + baseapp/files/apps.py | 14 + baseapp/files/base.py | 37 + baseapp/files/graphql/__init__.py | 0 baseapp/files/graphql/interfaces.py | 42 + baseapp/files/graphql/mutations.py | 174 ++++ baseapp/files/graphql/object_types.py | 50 ++ baseapp/files/graphql/queries.py | 12 + baseapp/files/graphql/utils.py | 41 + baseapp/files/models.py | 352 ++++++++ baseapp/files/permissions.py | 55 ++ baseapp/files/plugin.py | 54 ++ baseapp/files/rest_framework/__init__.py | 0 .../files/rest_framework/files/__init__.py | 0 .../files/rest_framework/files/serializers.py | 72 ++ baseapp/files/rest_framework/files/views.py | 98 +++ baseapp/files/rest_framework/routers.py | 13 + .../files/rest_framework/uploads/__init__.py | 0 .../rest_framework/uploads/permissions.py | 33 + .../rest_framework/uploads/presigned_views.py | 142 +++ .../rest_framework/uploads/serializers.py | 94 ++ baseapp/files/rest_framework/uploads/views.py | 164 ++++ baseapp/files/services/__init__.py | 4 + baseapp/files/services/cleanup.py | 64 ++ baseapp/files/services/upload_service.py | 163 ++++ baseapp/files/storage/__init__.py | 18 + baseapp/files/storage/base.py | 51 ++ baseapp/files/storage/local.py | 125 +++ baseapp/files/storage/s3.py | 130 +++ baseapp/files/tests/__init__.py | 0 baseapp/files/tests/conftest.py | 2 + baseapp/files/tests/test_graphql.py | 558 ++++++++++++ baseapp/files/tests/test_models.py | 288 ++++++ baseapp/files/tests/test_rest_api.py | 577 ++++++++++++ baseapp/files/tests/test_storage.py | 280 ++++++ baseapp/files/utils.py | 73 ++ .../test_auth_without_baseapp_profiles.py | 4 +- baseapp_chats/tests/test_graphql_mutations.py | 12 +- baseapp_chats/tests/test_graphql_queries.py | 12 +- .../tests/test_graphql_subscriptions.py | 10 +- baseapp_comments/graphql/object_types.py | 1 + baseapp_comments/models.py | 6 + .../tests/test_graphql_mutations_update.py | 7 + baseapp_core/graphql/__init__.py | 2 +- baseapp_core/graphql/fields.py | 46 +- baseapp_core/graphql/views.py | 15 + .../tests/test_graphql_mutations_create.py | 4 +- baseapp_profiles/rest_framework/README.md | 72 ++ baseapp_profiles/rest_framework/__init__.py | 3 +- baseapp_profiles/rest_framework/mixins.py | 50 ++ ..._profiles_queries_without_baseapp_pages.py | 4 +- baseapp_profiles/tests/test_get_queries.py | 6 +- .../tests/test_graphql_mutations_update.py | 14 +- pyproject.toml | 1 + testproject/files/__init__.py | 0 testproject/files/apps.py | 7 + testproject/files/migrations/0001_initial.py | 244 ++++++ testproject/files/migrations/__init__.py | 0 testproject/files/models.py | 15 + testproject/settings.py | 7 + 65 files changed, 5315 insertions(+), 63 deletions(-) create mode 100644 baseapp/files/README.md create mode 100644 baseapp/files/UPLOAD_FLOW.md create mode 100644 baseapp/files/__init__.py create mode 100644 baseapp/files/admin.py create mode 100644 baseapp/files/apps.py create mode 100644 baseapp/files/base.py create mode 100644 baseapp/files/graphql/__init__.py create mode 100644 baseapp/files/graphql/interfaces.py create mode 100644 baseapp/files/graphql/mutations.py create mode 100644 baseapp/files/graphql/object_types.py create mode 100644 baseapp/files/graphql/queries.py create mode 100644 baseapp/files/graphql/utils.py create mode 100644 baseapp/files/models.py create mode 100644 baseapp/files/permissions.py create mode 100644 baseapp/files/plugin.py create mode 100644 baseapp/files/rest_framework/__init__.py create mode 100644 baseapp/files/rest_framework/files/__init__.py create mode 100644 baseapp/files/rest_framework/files/serializers.py create mode 100644 baseapp/files/rest_framework/files/views.py create mode 100644 baseapp/files/rest_framework/routers.py create mode 100644 baseapp/files/rest_framework/uploads/__init__.py create mode 100644 baseapp/files/rest_framework/uploads/permissions.py create mode 100644 baseapp/files/rest_framework/uploads/presigned_views.py create mode 100644 baseapp/files/rest_framework/uploads/serializers.py create mode 100644 baseapp/files/rest_framework/uploads/views.py create mode 100644 baseapp/files/services/__init__.py create mode 100644 baseapp/files/services/cleanup.py create mode 100644 baseapp/files/services/upload_service.py create mode 100644 baseapp/files/storage/__init__.py create mode 100644 baseapp/files/storage/base.py create mode 100644 baseapp/files/storage/local.py create mode 100644 baseapp/files/storage/s3.py create mode 100644 baseapp/files/tests/__init__.py create mode 100644 baseapp/files/tests/conftest.py create mode 100644 baseapp/files/tests/test_graphql.py create mode 100644 baseapp/files/tests/test_models.py create mode 100644 baseapp/files/tests/test_rest_api.py create mode 100644 baseapp/files/tests/test_storage.py create mode 100644 baseapp/files/utils.py create mode 100644 baseapp_profiles/rest_framework/README.md create mode 100644 baseapp_profiles/rest_framework/mixins.py create mode 100644 testproject/files/__init__.py create mode 100644 testproject/files/apps.py create mode 100644 testproject/files/migrations/0001_initial.py create mode 100644 testproject/files/migrations/__init__.py create mode 100644 testproject/files/models.py diff --git a/baseapp/activity_log/tests/test_graphql_mutations.py b/baseapp/activity_log/tests/test_graphql_mutations.py index d2281d2b..b3cddad1 100644 --- a/baseapp/activity_log/tests/test_graphql_mutations.py +++ b/baseapp/activity_log/tests/test_graphql_mutations.py @@ -57,12 +57,8 @@ def test_add_comment_is_public(django_user_client, graphql_user_client, celery_c id name biography - image(width: 100, height: 100) { - url - } - bannerImage(width: 100, height: 100) { - url - } + image(width: 100, height: 100) + bannerImage(width: 100, height: 100) } errors { field diff --git a/baseapp/content_feed/tests/test_graphql_mutations_create.py b/baseapp/content_feed/tests/test_graphql_mutations_create.py index 3e2e853e..52ec137e 100644 --- a/baseapp/content_feed/tests/test_graphql_mutations_create.py +++ b/baseapp/content_feed/tests/test_graphql_mutations_create.py @@ -28,9 +28,7 @@ node { id pk - image(width:600, height:0){ - url - } + image(width:600, height:0) } } } diff --git a/baseapp/files/README.md b/baseapp/files/README.md new file mode 100644 index 00000000..2d37a0eb --- /dev/null +++ b/baseapp/files/README.md @@ -0,0 +1,827 @@ +# BaseApp Files - S3 Direct Multipart Upload + +REST API endpoints for S3 direct multipart uploads, allowing clients to upload large files directly to S3 without going through the Django backend. + +## Table of Contents + +- [Overview](#overview) +- [Installation](#installation) +- [Configuration](#configuration) + - [Production (S3)](#production-s3) + - [Development (Local Storage)](#development-local-storage) +- [API Usage](#api-usage) + - [Complete Upload Flow](#complete-upload-flow) + - [API Endpoints](#api-endpoints) +- [Client Implementation](#client-implementation) +- [Background Tasks](#background-tasks) +- [Troubleshooting](#troubleshooting) +- [Advanced Topics](#advanced-topics) + +## Overview + +This module provides S3-compatible multipart upload functionality with: + +- **Direct S3 uploads** - Files upload directly to S3, not through Django +- **Presigned URLs** - Secure, time-limited upload URLs +- **Automatic storage detection** - Works with S3 in production, local storage in dev/test +- **Parent object support** - Attach files to any model (posts, comments, etc.) +- **Automatic cleanup** - Celery task cleans up expired/abandoned uploads + +### Architecture + +``` +Client → Backend (initiate) → S3 Presigned URLs +Client → S3 (upload parts directly) +Client → Backend (complete) → File record created +``` + +## Installation + +### 1. Run Migration + +After deploying the code, run the migration to add upload tracking fields: + +```bash +# Using docker compose +docker compose exec backend python manage.py migrate files + +# Or directly +python manage.py migrate files +``` + +This adds the following fields to the File model: +- `upload_status` - Track upload state +- `upload_id` - S3 multipart upload ID +- `total_parts` - Number of parts +- `uploaded_parts` - JSON tracking ETags +- `upload_expires_at` - Expiration timestamp + +### 2. URLs Already Configured + +The URLs are automatically included in your project via `testproject/urls.py`. + +Available endpoints: +``` +POST /v1/files/uploads - Initiate upload +POST /v1/files/uploads/{id}/complete - Complete upload +DELETE /v1/files/uploads/{id} - Abort upload +GET /v1/files - List files +GET /v1/files/{id} - Retrieve file +PATCH /v1/files/{id} - Update file metadata +DELETE /v1/files/{id} - Delete file +POST /v1/files/{id}/set-parent - Set parent relationship +``` + +## Configuration + +### Production (S3) + +#### 1. Install boto3 (if not already installed) + +```bash +pip install boto3 +``` + +#### 2. Configure Django Settings + +Add to your `settings.py` or environment variables: + +```python +# AWS S3 Configuration +AWS_ACCESS_KEY_ID = env('AWS_ACCESS_KEY_ID') +AWS_SECRET_ACCESS_KEY = env('AWS_SECRET_ACCESS_KEY') +AWS_STORAGE_BUCKET_NAME = env('AWS_STORAGE_BUCKET_NAME') +AWS_S3_REGION_NAME = env('AWS_S3_REGION_NAME', default='us-east-1') + +# File upload settings +MAX_FILE_UPLOAD_SIZE = 5 * 1024 * 1024 * 1024 # 5GB default +FILE_UPLOAD_PRESIGNED_URL_EXPIRATION = 3600 # 1 hour + +# Optional: Use S3 as default storage +STORAGES = { + 'default': { + 'BACKEND': 'storages.backends.s3boto3.S3Boto3Storage', + }, + 'staticfiles': { + 'BACKEND': 'django.contrib.staticfiles.storage.StaticFilesStorage', + }, +} +``` + +#### 3. Environment Variables + +Create or update your `.env` file: + +```bash +AWS_ACCESS_KEY_ID=your-access-key-id +AWS_SECRET_ACCESS_KEY=your-secret-access-key +AWS_STORAGE_BUCKET_NAME=your-bucket-name +AWS_S3_REGION_NAME=us-east-1 +``` + +#### 4. S3 Bucket Configuration + +Ensure your S3 bucket has the correct CORS configuration: + +```json +[ + { + "AllowedHeaders": ["*"], + "AllowedMethods": ["GET", "PUT", "POST", "DELETE"], + "AllowedOrigins": ["https://yourdomain.com"], + "ExposeHeaders": ["ETag"], + "MaxAgeSeconds": 3000 + } +] +``` + +**Important**: The `ExposeHeaders: ["ETag"]` is required so clients can read ETags from S3 responses. + +### Development (Local Storage) + +No additional configuration needed! The system automatically detects local storage and provides a fallback implementation. + +**How it works:** +- Storage factory checks if `default_storage` is `S3Boto3Storage` +- If not S3, uses `LocalUploadHandler` automatically +- Local handler stores parts in `MEDIA_ROOT/temp_uploads/` +- Parts are combined on completion + +**Note**: Local storage doesn't support true direct uploads - it's a simplified fallback for development/testing only. + +## API Usage + +### Complete Upload Flow + +#### 1. Initiate Upload + +**Request:** +```bash +POST /v1/files/uploads +Authorization: Bearer {jwt_token} +Content-Type: application/json + +{ + "file_name": "large-video.mp4", + "file_size": 104857600, + "file_content_type": "video/mp4", + "num_parts": 20, + "part_size": 5242880, + "parent_content_type": "testapp.post", // optional + "parent_object_id": 123 // optional +} +``` + +**Response:** +```json +{ + "id": 456, + "upload_id": "abc123xyz...", + "presigned_urls": [ + { + "part_number": 1, + "url": "https://bucket.s3.amazonaws.com/...?signature=..." + }, + { + "part_number": 2, + "url": "https://bucket.s3.amazonaws.com/...?signature=..." + } + // ... up to 20 parts + ], + "expires_in": 3600, + "upload_status": "uploading" +} +``` + +**Calculating Parts:** +```python +import math + +file_size = 104857600 # 100 MB +part_size = 5242880 # 5 MB (minimum for S3) +num_parts = math.ceil(file_size / part_size) # 20 parts +``` + +#### 2. Upload Parts Directly to S3 + +For each presigned URL, upload the corresponding file chunk using HTTP PUT: + +```javascript +// Example with fetch API +const part_number = 1; +const presigned_url = presigned_urls[0].url; +const chunk = file.slice(start, end); + +const response = await fetch(presigned_url, { + method: 'PUT', + body: chunk, + headers: { + 'Content-Type': file_content_type + } +}); + +// IMPORTANT: Save the ETag from response headers +const etag = response.headers.get('ETag').replace(/"/g, ''); +``` + +**Important Notes:** +- Upload to presigned URLs with HTTP PUT (not POST) +- S3 returns an ETag header for each part - you MUST save these +- ETags are required to complete the upload +- Upload parts in any order (parallel uploads supported) + +#### 3. Complete Upload + +Once all parts are uploaded, send the completion request with ETags: + +**Request:** +```bash +POST /v1/files/uploads/456/complete +Authorization: Bearer {jwt_token} +Content-Type: application/json + +{ + "parts": [ + {"part_number": 1, "etag": "abc123..."}, + {"part_number": 2, "etag": "def456..."}, + {"part_number": 3, "etag": "ghi789..."} + // ... all parts with their ETags + ] +} +``` + +**Response:** +```json +{ + "id": 456, + "file_name": "large-video.mp4", + "file_size": 104857600, + "file_content_type": "video/mp4", + "url": "https://bucket.s3.amazonaws.com/files/uuid.mp4", + "upload_status": "completed", + "created_by": 1, + "created_by_name": "John Doe", + "created": "2025-12-29T10:00:00Z", + "parent_content_type": 23, + "parent_object_id": 123 +} +``` + +#### 4. (Optional) Set Parent Later + +If you didn't specify a parent during initiation, you can set it later: + +**Request:** +```bash +POST /v1/files/456/set-parent +Authorization: Bearer {jwt_token} +Content-Type: application/json + +{ + "parent_content_type": "testapp.comment", + "parent_object_id": 789 +} +``` + +### API Endpoints + +#### Initiate Upload +``` +POST /v1/files/uploads +``` + +**Parameters:** +- `file_name` (string, required) - Original filename +- `file_size` (integer, required) - Total file size in bytes +- `file_content_type` (string, required) - MIME type (e.g., "video/mp4") +- `num_parts` (integer, required) - Number of parts (1-10,000) +- `part_size` (integer, required) - Size of each part in bytes (min 5MB for multipart) +- `parent_content_type` (string, optional) - e.g., "testapp.post" +- `parent_object_id` (integer, optional) - ID of parent object + +**Validation:** +- File size must not exceed `MAX_FILE_UPLOAD_SIZE` (default 5GB) +- Part size must be at least 5MB for multipart uploads (S3 requirement) +- Number of parts must be between 1 and 10,000 +- File size must match parts: `(num_parts - 1) * part_size <= file_size <= num_parts * part_size` + +#### Complete Upload +``` +POST /v1/files/uploads/{id}/complete +``` + +**Parameters:** +- `parts` (array, required) - Array of objects with: + - `part_number` (integer) - Part number (1-indexed) + - `etag` (string) - ETag from S3 response + +**Requirements:** +- All parts must be included +- Part numbers must be sequential (1, 2, 3, ...) +- ETags must match S3-uploaded parts +- Only file owner can complete upload + +#### Abort Upload +``` +DELETE /v1/files/uploads/{id} +``` + +**Notes:** +- Cleans up S3 multipart upload +- Marks file as "aborted" +- Only file owner can abort +- Prevents charges for incomplete S3 uploads + +#### List Files +``` +GET /v1/files +``` + +**Query Parameters:** +- `status` (string, optional) - Filter by upload_status (default: "completed") +- `parent_content_type` (string, optional) - Filter by parent type +- `parent_object_id` (integer, optional) - Filter by parent ID + +**Notes:** +- Returns only user's own files (unless staff) +- Includes only completed files by default +- Use `?status=uploading` to see in-progress uploads + +#### Retrieve File +``` +GET /v1/files/{id} +``` + +#### Update File Metadata +``` +PATCH /v1/files/{id} +``` + +**Editable fields:** +- `name` - Display name +- `description` - File description + +**Read-only fields:** +- `file_name`, `file_size`, `file_content_type` - Set during upload +- `upload_status` - Managed by system + +#### Delete File +``` +DELETE /v1/files/{id} +``` + +**Notes:** +- Only file owner can delete +- Deletes database record +- Does NOT delete S3 object (by design, for safety) + +#### Set Parent +``` +POST /v1/files/{id}/set-parent +``` + +**Parameters:** +- `parent_content_type` (string, required) - e.g., "testapp.post" +- `parent_object_id` (integer, required) - Parent object ID + +## Client Implementation + +### JavaScript Example (Complete Flow) + +```javascript +class MultipartUploader { + constructor(apiUrl, token) { + this.apiUrl = apiUrl; + this.token = token; + } + + async uploadFile(file, parentContentType = null, parentObjectId = null) { + // 1. Calculate parts + const partSize = 5 * 1024 * 1024; // 5MB + const numParts = Math.ceil(file.size / partSize); + + // 2. Initiate upload + const initResponse = await fetch(`${this.apiUrl}/v1/files/uploads`, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${this.token}`, + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + file_name: file.name, + file_size: file.size, + file_content_type: file.type, + num_parts: numParts, + part_size: partSize, + parent_content_type: parentContentType, + parent_object_id: parentObjectId + }) + }); + + const { id, upload_id, presigned_urls } = await initResponse.json(); + + // 3. Upload parts to S3 + const parts = []; + for (let i = 0; i < numParts; i++) { + const start = i * partSize; + const end = Math.min(start + partSize, file.size); + const chunk = file.slice(start, end); + + const partResponse = await fetch(presigned_urls[i].url, { + method: 'PUT', + body: chunk + }); + + // Extract ETag (remove quotes) + const etag = partResponse.headers.get('ETag').replace(/"/g, ''); + + parts.push({ + part_number: i + 1, + etag: etag + }); + + // Optional: Report progress + const progress = ((i + 1) / numParts) * 100; + console.log(`Upload progress: ${progress.toFixed(1)}%`); + } + + // 4. Complete upload + const completeResponse = await fetch( + `${this.apiUrl}/v1/files/uploads/${id}/complete`, + { + method: 'POST', + headers: { + 'Authorization': `Bearer ${this.token}`, + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ parts }) + } + ); + + return await completeResponse.json(); + } + + async abortUpload(uploadId) { + await fetch(`${this.apiUrl}/v1/files/uploads/${uploadId}`, { + method: 'DELETE', + headers: { + 'Authorization': `Bearer ${this.token}` + } + }); + } +} + +// Usage +const uploader = new MultipartUploader('https://api.example.com', userToken); + +try { + const fileInput = document.querySelector('input[type="file"]'); + const file = fileInput.files[0]; + + const result = await uploader.uploadFile(file, 'testapp.post', 123); + console.log('Upload complete:', result.url); +} catch (error) { + console.error('Upload failed:', error); +} +``` + +### Python Example (for testing) + +```python +import requests +import math + +def upload_file_multipart(api_url, token, file_path, parent_content_type=None, parent_object_id=None): + """Upload a file using multipart upload.""" + + # 1. Read file info + with open(file_path, 'rb') as f: + file_data = f.read() + + file_size = len(file_data) + file_name = file_path.split('/')[-1] + + # 2. Calculate parts + part_size = 5 * 1024 * 1024 # 5MB + num_parts = math.ceil(file_size / part_size) + + # 3. Initiate upload + headers = {'Authorization': f'Bearer {token}'} + init_response = requests.post( + f'{api_url}/v1/files/uploads', + headers=headers, + json={ + 'file_name': file_name, + 'file_size': file_size, + 'file_content_type': 'application/octet-stream', + 'num_parts': num_parts, + 'part_size': part_size, + 'parent_content_type': parent_content_type, + 'parent_object_id': parent_object_id + } + ) + init_data = init_response.json() + + # 4. Upload parts + parts = [] + for i, presigned_url_data in enumerate(init_data['presigned_urls']): + start = i * part_size + end = min(start + part_size, file_size) + chunk = file_data[start:end] + + # Upload to S3 + part_response = requests.put( + presigned_url_data['url'], + data=chunk + ) + + # Extract ETag + etag = part_response.headers['ETag'].strip('"') + + parts.append({ + 'part_number': i + 1, + 'etag': etag + }) + + # 5. Complete upload + complete_response = requests.post( + f"{api_url}/v1/files/uploads/{init_data['id']}/complete", + headers=headers, + json={'parts': parts} + ) + + return complete_response.json() + +# Usage +result = upload_file_multipart( + 'https://api.example.com', + 'your-jwt-token', + '/path/to/large-file.mp4', + parent_content_type='testapp.post', + parent_object_id=123 +) +print(f"File uploaded: {result['url']}") +``` + +## Background Tasks + +### Automatic Cleanup of Expired Uploads + +Uploads that aren't completed within 24 hours are automatically cleaned up. + +#### Setup Celery Beat + +Add to your `settings.py`: + +```python +from celery.schedules import crontab + +CELERY_BEAT_SCHEDULE = { + 'cleanup-expired-uploads': { + 'task': 'baseapp.files.services.cleanup.cleanup_expired_uploads', + 'schedule': crontab(minute=0), # Run every hour + }, +} +``` + +#### What It Does + +The cleanup task: +1. Finds files with status "pending" or "uploading" where `upload_expires_at < now()` +2. Aborts the S3 multipart upload (prevents storage charges) +3. Marks the file as "aborted" +4. Clears the `upload_id` + +#### Manual Cleanup + +You can also run cleanup manually: + +```python +from baseapp.files.services.cleanup import cleanup_expired_uploads, cleanup_failed_uploads + +# Cleanup expired uploads +cleanup_expired_uploads() + +# Cleanup old failed/aborted uploads (older than 7 days) +cleanup_failed_uploads(days_old=7) +``` + +Or via management command: + +```bash +docker compose exec backend python manage.py shell +>>> from baseapp.files.services.cleanup import cleanup_expired_uploads +>>> cleanup_expired_uploads() +``` + +## Troubleshooting + +### "Part size must be at least 5MB" + +**Problem:** S3 requires multipart uploads to have parts of at least 5MB (except the last part). + +**Solution:** +- Use `part_size >= 5242880` (5MB) for files larger than 5MB +- For files smaller than 5MB, use `num_parts = 1` and set `part_size = file_size` + +### "CORS policy" errors when uploading to S3 + +**Problem:** Browser blocks S3 upload due to CORS restrictions. + +**Solution:** Configure CORS on your S3 bucket (see [Configuration](#4-s3-bucket-configuration) above). + +### ETags don't match / Upload completion fails + +**Problem:** ETags sent to complete endpoint don't match S3's records. + +**Solution:** +- Ensure you're extracting ETags from S3 response headers correctly +- Remove quotes from ETags: `etag.replace(/"/g, '')` +- Upload parts in the correct order (part 1, 2, 3...) +- Don't modify file chunks before uploading + +### Uploads stuck in "uploading" status + +**Problem:** Client uploaded parts but never called complete endpoint. + +**Solution:** +- Ensure your client code calls the complete endpoint +- Check for JavaScript errors during upload +- Abandoned uploads are automatically cleaned up after 24 hours + +### "File size doesn't match parts" + +**Problem:** The validation `(num_parts - 1) * part_size <= file_size <= num_parts * part_size` failed. + +**Solution:** +```javascript +// Correct calculation +const partSize = 5 * 1024 * 1024; // 5MB +const numParts = Math.ceil(fileSize / partSize); + +// Incorrect - don't round down! +const numParts = Math.floor(fileSize / partSize); // ❌ Wrong +``` + +### Local storage fallback not working + +**Problem:** Getting S3 errors in development. + +**Solution:** +- Local fallback only works if NOT using S3Boto3Storage +- Check `default_storage` in Django shell: + ```python + from django.core.files.storage import default_storage + print(type(default_storage)) + # Should NOT be S3Boto3Storage for local fallback + ``` + +## Advanced Topics + +### Custom Storage Backends + +You can add support for other storage backends (GCS, Azure, etc.) by: + +1. Create a new handler in `baseapp/files/storage/`: + +```python +# baseapp/files/storage/gcs.py +from .base import BaseUploadHandler + +class GCSMultipartUploadHandler(BaseUploadHandler): + def supports_multipart(self) -> bool: + return True + + # Implement other methods... +``` + +2. Update the factory in `baseapp/files/storage/__init__.py`: + +```python +def get_upload_handler(): + storage_class = type(default_storage) + + if "S3Boto3Storage" in str(storage_class): + from .s3 import S3MultipartUploadHandler + return S3MultipartUploadHandler() + elif "GoogleCloudStorage" in str(storage_class): + from .gcs import GCSMultipartUploadHandler + return GCSMultipartUploadHandler() + else: + from .local import LocalUploadHandler + return LocalUploadHandler() +``` + +### Resumable Uploads + +The current implementation doesn't support resuming interrupted uploads. To add this: + +1. Client should track which parts were successfully uploaded +2. Add an endpoint to query uploaded parts: + ```python + @action(detail=True, methods=['get']) + def uploaded_parts(self, request, pk=None): + file_obj = self.get_object() + # Return list of uploaded part numbers + return Response(file_obj.uploaded_parts or {}) + ``` +3. Client resumes by only uploading missing parts + +### File Size Quotas + +Add per-user file size quotas: + +```python +# In UploadService._validate_file_params +def _validate_file_params(self, file_size, num_parts, part_size, user): + # Existing validation... + + # Check user quota + user_total = File.objects.filter( + created_by=user, + upload_status='completed' + ).aggregate(total=Sum('file_size'))['total'] or 0 + + user_quota = getattr(settings, 'FILE_UPLOAD_QUOTA_PER_USER', 10 * 1024 * 1024 * 1024) # 10GB + + if user_total + file_size > user_quota: + raise ValueError(f"Upload would exceed your quota of {user_quota} bytes") +``` + +### File Type Restrictions + +Restrict allowed file types: + +```python +# In settings.py +ALLOWED_FILE_TYPES = [ + 'image/jpeg', + 'image/png', + 'video/mp4', + 'application/pdf', +] + +# In InitiateUploadSerializer.validate_file_content_type +def validate_file_content_type(self, value): + allowed = getattr(settings, 'ALLOWED_FILE_TYPES', None) + if allowed and value not in allowed: + raise serializers.ValidationError( + f"File type {value} not allowed. Allowed types: {', '.join(allowed)}" + ) + return value +``` + +### Progress Notifications + +Add real-time progress updates via WebSockets: + +```python +# When parts are uploaded (would need additional endpoint) +from channels.layers import get_channel_layer +from asgiref.sync import async_to_sync + +def notify_upload_progress(file_id, uploaded_parts, total_parts): + channel_layer = get_channel_layer() + async_to_sync(channel_layer.group_send)( + f'upload_{file_id}', + { + 'type': 'upload_progress', + 'uploaded': uploaded_parts, + 'total': total_parts, + 'progress': (uploaded_parts / total_parts) * 100 + } + ) +``` + +### Parallel Part Uploads + +The API supports parallel part uploads out of the box. Example: + +```javascript +// Upload all parts in parallel +const uploadPromises = presigned_urls.map(async (urlData, index) => { + const start = index * partSize; + const end = Math.min(start + partSize, file.size); + const chunk = file.slice(start, end); + + const response = await fetch(urlData.url, { + method: 'PUT', + body: chunk + }); + + return { + part_number: index + 1, + etag: response.headers.get('ETag').replace(/"/g, '') + }; +}); + +const parts = await Promise.all(uploadPromises); +``` + +## Support + +For issues or questions: +1. Check the [Troubleshooting](#troubleshooting) section +2. Review API error responses (they include detailed error messages) +3. Check Django logs for server-side errors +4. Verify S3 bucket permissions and CORS configuration diff --git a/baseapp/files/UPLOAD_FLOW.md b/baseapp/files/UPLOAD_FLOW.md new file mode 100644 index 00000000..015c30e4 --- /dev/null +++ b/baseapp/files/UPLOAD_FLOW.md @@ -0,0 +1,193 @@ +# File Upload Flow + +## Overview + +The file upload system supports both direct S3 uploads (production) and local storage uploads (dev/test) with multipart upload capabilities. + +## Multipart Upload Workflow + +### 1. Initiate Upload +**POST** `/v1/files/uploads/` + +```json +{ + "file_name": "large-video.mp4", + "file_size": 104857600, + "file_content_type": "video/mp4", + "num_parts": 20, + "part_size": 5242880, + "parent_content_type": "testapp.post", // optional + "parent_object_id": 123 // optional +} +``` + +**Response:** +```json +{ + "file_obj": { + "id": "7", + "file_name": "large-video.mp4", + "upload_status": "uploading" + }, + "upload_id": "abc-123-def-456", + "presigned_urls": [ + { + "part_number": 1, + "url": "https://s3.amazonaws.com/...", // or local endpoint + "method": "PUT" + }, + ... + ], + "expires_in": 3600 +} +``` + +### 2. Upload Parts + +#### For S3 (Production) +Upload directly to the presigned URLs: +```bash +PUT https://s3.amazonaws.com/bucket/path?uploadId=...&partNumber=1 +Content-Type: application/octet-stream +Body: +``` + +#### For Local Storage (Dev/Test) +**PUT** `/v1/files/presigned-uploads/{file_id}/upload-part/{part_number}/?token=` + +The presigned URLs returned from the initiate endpoint include a signed token for authentication. + +```bash +PUT /v1/files/presigned-uploads/7/upload-part/1/?token= +Content-Type: application/octet-stream +Body: +``` + +**Response:** +```json +{ + "part_number": 1, + "etag": "abc123def456..." +} +``` + +**Response Headers:** +``` +ETag: abc123def456... +``` + +**Note:** The token is generated using Django signing and is valid for 1 hour. No session authentication or CSRF protection is required for this endpoint. The ETag is returned both in the response body and as an HTTP header for S3 compatibility. + +### 3. Complete Upload +**POST** `/v1/files/uploads/{id}/complete/` + +```json +{ + "parts": [ + {"part_number": 1, "etag": "abc123..."}, + {"part_number": 2, "etag": "def456..."}, + ... + ] +} +``` + +**Response:** +```json +{ + "id": "7", + "file_name": "large-video.mp4", + "file_size": 104857600, + "file_content_type": "video/mp4", + "upload_status": "completed", + "url": "https://cdn.example.com/files/...", + "created": "2025-01-01T12:00:00Z" +} +``` + +### 4. Abort Upload (Optional) +**DELETE** `/v1/files/uploads/{id}/` + +Cancels the upload and cleans up temporary files. + +## Storage Handlers + +### S3 (Production) +- Generates real presigned URLs pointing to S3 +- Client uploads directly to S3 +- No backend bandwidth used for file data +- Supports true multipart uploads + +### Local (Dev/Test) +- Generates "presigned URLs" with signed tokens pointing to backend endpoint +- Client uploads to backend via `/presigned-uploads/{id}/upload-part/{part_number}/` endpoint +- Token authentication using Django signing (no session auth or CSRF required) +- Backend stores parts in temp directory +- On complete, parts are concatenated into final file + +## Endpoints Summary + +| Method | Endpoint | Description | +|--------|----------|-------------| +| POST | `/v1/files/uploads/` | Initiate multipart upload | +| PUT | `/v1/files/presigned-uploads/{id}/upload-part/{part_number}/?token=...` | Upload a single part (local storage, token-authenticated) | +| POST | `/v1/files/uploads/{id}/complete/` | Complete the upload | +| DELETE | `/v1/files/uploads/{id}/` | Abort the upload | +| GET | `/v1/files/` | List files | +| GET | `/v1/files/{id}/` | Get file details | +| PATCH | `/v1/files/{id}/` | Update file metadata | +| DELETE | `/v1/files/{id}/` | Delete file | + +## Configuration + +### Settings +```python +# Maximum file size (default: 5GB) +MAX_FILE_UPLOAD_SIZE = 5 * 1024 * 1024 * 1024 + +# Storage backend +UPLOAD_STORAGE_HANDLER = "local" # or "s3" +``` + +### Local Storage +```python +UPLOAD_STORAGE_HANDLER = "local" +``` +- Uses `LocalUploadHandler` +- Stores files in `MEDIA_ROOT/files/` +- Temp parts in `MEDIA_ROOT/temp_uploads/{upload_id}/` + +### S3 Storage +```python +UPLOAD_STORAGE_HANDLER = "s3" +AWS_ACCESS_KEY_ID = "..." +AWS_SECRET_ACCESS_KEY = "..." +AWS_STORAGE_BUCKET_NAME = "my-bucket" +AWS_S3_REGION_NAME = "us-east-1" +``` +- Uses `S3UploadHandler` +- Generates presigned URLs for direct S3 upload +- No backend file handling + +## Error Handling + +### Common Errors +- **400 Bad Request**: Invalid parameters, part numbers, or file size +- **403 Forbidden**: User doesn't own the file/upload +- **404 Not Found**: File or upload not found +- **413 Payload Too Large**: File exceeds `MAX_FILE_UPLOAD_SIZE` + +### Upload Status +- `pending`: Upload initiated but not started +- `uploading`: Parts being uploaded +- `completed`: Upload finished successfully +- `failed`: Upload failed (error during completion) +- `aborted`: Upload cancelled by user + +## Best Practices + +1. **Chunk Size**: Use 5MB minimum for parts (S3 requirement) +2. **Part Count**: Keep between 1-10,000 parts +3. **Error Handling**: Always store ETags from part uploads +4. **Cleanup**: Call abort endpoint if upload fails +5. **Expiration**: Complete uploads within 24 hours (tokens expire in 1 hour) +6. **Local Dev**: Use the presigned URLs with tokens returned from the initiate endpoint diff --git a/baseapp/files/__init__.py b/baseapp/files/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/baseapp/files/admin.py b/baseapp/files/admin.py new file mode 100644 index 00000000..89d93631 --- /dev/null +++ b/baseapp/files/admin.py @@ -0,0 +1,24 @@ +import swapper +from django.contrib import admin + +File = swapper.load_model("baseapp_files", "File") +FileTarget = swapper.load_model("baseapp_files", "FileTarget") + + +@admin.register(File) +class FileAdmin(admin.ModelAdmin): + list_display = ("pk", "name", "file_content_type", "parent", "created_by", "created") + search_fields = ("name",) + raw_id_fields = ("created_by",) + + +@admin.register(FileTarget) +class FileTargetAdmin(admin.ModelAdmin): + list_display = ( + "pk", + "target", + "is_files_enabled", + "files_count", + ) + list_filter = ("is_files_enabled",) + search_fields = ("target_id",) diff --git a/baseapp/files/apps.py b/baseapp/files/apps.py new file mode 100644 index 00000000..7b02248d --- /dev/null +++ b/baseapp/files/apps.py @@ -0,0 +1,14 @@ +from baseapp_core.plugins import BaseAppConfig, GraphQLContributor + + +class FilesConfig(BaseAppConfig, GraphQLContributor): + default = True + name = "baseapp.files" + label = "baseapp_files" + verbose_name = "BaseApp Files" + default_auto_field = "django.db.models.BigAutoField" + + def register_graphql_shared_interfaces(self, registry): + from .graphql.interfaces import get_files_interface + + registry.register("FilesInterface", get_files_interface) diff --git a/baseapp/files/base.py b/baseapp/files/base.py new file mode 100644 index 00000000..786f4ea1 --- /dev/null +++ b/baseapp/files/base.py @@ -0,0 +1,37 @@ +import swapper +from django.contrib.contenttypes.models import ContentType +from django.db import models + +from baseapp_core.models import DocumentId + +from .utils import get_or_create_file_target + + +class FileableModel(models.Model): + class Meta: + abstract = True + + @property + def files(self): + """Returns files related to this object through DocumentId.""" + File = swapper.load_model("baseapp_files", "File") + content_type = ContentType.objects.get_for_model(self, for_concrete_model=False) + try: + document_id = DocumentId.objects.get( + content_type=content_type, + object_id=self.pk, + ) + return File.objects.filter(parent=document_id) + except DocumentId.DoesNotExist: + return File.objects.none() + + def get_file_target(self): + return get_or_create_file_target(self) + + @property + def files_count(self): + return self.get_file_target().files_count + + @property + def is_files_enabled(self): + return self.get_file_target().is_files_enabled diff --git a/baseapp/files/graphql/__init__.py b/baseapp/files/graphql/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/baseapp/files/graphql/interfaces.py b/baseapp/files/graphql/interfaces.py new file mode 100644 index 00000000..f4525807 --- /dev/null +++ b/baseapp/files/graphql/interfaces.py @@ -0,0 +1,42 @@ +import swapper +from django.contrib.contenttypes.models import ContentType +from graphene import Boolean, Interface +from graphene.types.generic import GenericScalar +from graphene_django.filter import DjangoFilterConnectionField + +from baseapp_core.models import DocumentId + + +class FilesInterface(Interface): + files_count = GenericScalar() + is_files_enabled = Boolean() + files = DjangoFilterConnectionField( + lambda: swapper.load_model("baseapp_files", "File").get_graphql_object_type() + ) + + def resolve_files_count(self, info, **kwargs): + if hasattr(self, "get_file_target"): + return self.get_file_target().files_count + return {"total": 0} + + def resolve_is_files_enabled(self, info, **kwargs): + if hasattr(self, "get_file_target"): + return self.get_file_target().is_files_enabled + return True + + def resolve_files(self, info, **kwargs): + File = swapper.load_model("baseapp_files", "File") + # Get the DocumentId for this object + content_type = ContentType.objects.get_for_model(self) + try: + document_id = DocumentId.objects.get( + content_type=content_type, + object_id=self.pk, + ) + return File.objects.filter(parent=document_id).order_by("-created") + except DocumentId.DoesNotExist: + return File.objects.none() + + +def get_files_interface(): + return FilesInterface diff --git a/baseapp/files/graphql/mutations.py b/baseapp/files/graphql/mutations.py new file mode 100644 index 00000000..012c0a8b --- /dev/null +++ b/baseapp/files/graphql/mutations.py @@ -0,0 +1,174 @@ +import graphene +import swapper +from django.db import transaction +from django.utils.translation import gettext_lazy as _ +from graphql.error import GraphQLError +from graphql_relay.connection.arrayconnection import offset_to_cursor + +from baseapp_core.graphql import RelayMutation, get_obj_from_relay_id, login_required +from baseapp_core.models import DocumentId + +from .interfaces import FilesInterface + +File = swapper.load_model("baseapp_files", "File") +app_label = File._meta.app_label +file_model_name = File._meta.model_name +FileObjectType = File.get_graphql_object_type() + + +class FileAttachToTarget(RelayMutation): + """ + Attach one or more files to a target object. + + Usage: + mutation { + fileAttachToTarget(input: { + fileRelayIds: ["RmlsZU9iamVjdFR5cGU6MQ==", "RmlsZU9iamVjdFR5cGU6Mg=="] + targetObjectId: "Q29tbWVudE9iamVjdFR5cGU6MQ==" + }) { + attachedFiles { + node { + id + fileName + } + } + target { + ... on CommentsInterface { + filesCount + isFilesEnabled + } + } + } + } + """ + + attached_files = graphene.List( + graphene.NonNull(FileObjectType._meta.connection.Edge), + required=True, + ) + target = graphene.Field(FilesInterface, required=True) + + class Input: + file_relay_ids = graphene.List(graphene.ID, required=True) + target_object_id = graphene.ID(required=True) + + @classmethod + @login_required + @transaction.atomic + def mutate_and_get_payload(cls, root, info, **input): + from django.contrib.contenttypes.models import ContentType + + file_relay_ids = input.get("file_relay_ids", []) + target_object_id = input.get("target_object_id") + + if not file_relay_ids: + raise GraphQLError( + str(_("At least one file must be provided")), + extensions={"code": "invalid_input"}, + ) + + # Get the target object + target = get_obj_from_relay_id(info, target_object_id) + if not target: + raise GraphQLError( + str(_("Target object not found")), + extensions={"code": "not_found"}, + ) + + # Get or create DocumentId for the target + target_content_type = ContentType.objects.get_for_model(target) + target_document_id, created = DocumentId.objects.get_or_create( + content_type=target_content_type, + object_id=target.pk, + ) + + # Check if files are enabled for this target + if hasattr(target, "get_file_target"): + file_target = target.get_file_target() + if not file_target.is_files_enabled: + raise GraphQLError( + str(_("Files are not enabled for this target")), + extensions={"code": "files_disabled"}, + ) + + # Get all files and verify ownership + files = [] + for file_relay_id in file_relay_ids: + file_obj = get_obj_from_relay_id(info, file_relay_id) + if not file_obj: + raise GraphQLError( + str(_("File not found: {file_id}")).format(file_id=file_relay_id), + extensions={"code": "not_found"}, + ) + + # Check ownership - only file owner can attach it + if file_obj.created_by != info.context.user: + raise GraphQLError( + str(_("You don't have permission to attach this file")), + extensions={"code": "permission_required"}, + ) + + # Check if file is already attached to another parent + if file_obj.parent_id: + raise GraphQLError( + str(_("File {file_name} is already attached to another object")).format( + file_name=file_obj.file_name + ), + extensions={"code": "already_attached"}, + ) + + files.append(file_obj) + + # Attach all files to the target + for file_obj in files: + # TO DO: Check permission + # if not info.context.user.has_perm("baseapp_files.attach_file", file_obj): + file_obj.parent = target_document_id + file_obj.save(update_fields=["parent"]) + + # Refresh target to get updated files_count + target.refresh_from_db() + + # Create file edges for response + attached_files = [ + FileObjectType._meta.connection.Edge(node=file_obj, cursor=offset_to_cursor(i)) + for i, file_obj in enumerate(files) + ] + + return FileAttachToTarget(attached_files=attached_files, target=target) + + +class FileDelete(RelayMutation): + deleted_id = graphene.ID() + parent = graphene.Field(FilesInterface) + + class Input: + id = graphene.ID(required=True) + + @classmethod + @login_required + def mutate_and_get_payload(cls, root, info, **input): + relay_id = input.get("id") + + error_exception = GraphQLError( + str(_("You don't have permission to perform this action")), + extensions={"code": "permission_required"}, + ) + + obj = get_obj_from_relay_id(info, relay_id) + if not obj: + raise error_exception + + if not info.context.user.has_perm(f"{app_label}.delete_{file_model_name}", obj): + raise error_exception + + parent = obj.parent.content_object if obj.parent else None + + obj.delete() + + return FileDelete(deleted_id=relay_id, parent=parent) + + +class FilesMutations(object): + file_attach_to_target = FileAttachToTarget.Field() + file_delete = FileDelete.Field() diff --git a/baseapp/files/graphql/object_types.py b/baseapp/files/graphql/object_types.py new file mode 100644 index 00000000..dbeece7c --- /dev/null +++ b/baseapp/files/graphql/object_types.py @@ -0,0 +1,50 @@ +import django_filters +import graphene +import swapper + +from baseapp_auth.graphql.permissions import PermissionsInterface +from baseapp_core.graphql import DjangoObjectType, FileInterface +from baseapp_core.graphql import Node as RelayNode +from baseapp_core.graphql import ThumbnailField +from baseapp_core.plugins import graphql_shared_interfaces + +File = swapper.load_model("baseapp_files", "File") + + +class FileFilter(django_filters.FilterSet): + no_parent = django_filters.BooleanFilter(field_name="parent", lookup_expr="isnull") + + class Meta: + model = File + fields = ["no_parent"] + + +class AbstractFileObjectType: + parent = graphene.Field(RelayNode) + thumbnail = ThumbnailField() + + class Meta: + interfaces = graphql_shared_interfaces.get( + RelayNode, + PermissionsInterface, + FileInterface, + "CommentsInterface", + "ReactionsInterface", + ) + model = File + filterset_class = FileFilter + + def resolve_url(self, info, **kwargs): + if not self.file: + if hasattr(self, "url"): + return self.url + return None + return info.context.build_absolute_uri(self.file.url) + + def resolve_thumbnail(self, *args, **kwargs): + return self.file + + +class FileObjectType(AbstractFileObjectType, DjangoObjectType): + class Meta(AbstractFileObjectType.Meta): + pass diff --git a/baseapp/files/graphql/queries.py b/baseapp/files/graphql/queries.py new file mode 100644 index 00000000..4ad18c47 --- /dev/null +++ b/baseapp/files/graphql/queries.py @@ -0,0 +1,12 @@ +import swapper +from graphene_django.filter import DjangoFilterConnectionField + +from baseapp_core.graphql import Node + +File = swapper.load_model("baseapp_files", "File") +FileObjectType = File.get_graphql_object_type() + + +class FilesQueries: + my_files = DjangoFilterConnectionField(FileObjectType) + file = Node.Field(FileObjectType) diff --git a/baseapp/files/graphql/utils.py b/baseapp/files/graphql/utils.py new file mode 100644 index 00000000..7ee331b0 --- /dev/null +++ b/baseapp/files/graphql/utils.py @@ -0,0 +1,41 @@ +import swapper +from django.utils.translation import gettext_lazy as _ +from graphql.error import GraphQLError + +from baseapp_core.graphql import get_pk_from_relay_id + +from ..utils import set_files_parent + + +def attach_files_from_relay_ids(parent, file_ids, user): + if file_ids in (None, []): + return + + File = swapper.load_model("baseapp_files", "File") + file_pks = [get_pk_from_relay_id(file_id) for file_id in file_ids] + pk_lookup = [str(pk) for pk in file_pks] + files_qs = File.objects.filter(pk__in=file_pks) + files_map = {str(file.pk): file for file in files_qs} + + missing = [pk for pk in pk_lookup if pk not in files_map] + if missing: + raise GraphQLError( + str(_("One or more files could not be found")), + extensions={"code": "not_found"}, + ) + + files = [files_map[pk] for pk in pk_lookup] + + if user and not user.is_superuser: + unauthorized = [ + file_obj + for file_obj in files + if file_obj.created_by_id and file_obj.created_by_id != user.id + ] + if unauthorized: + raise GraphQLError( + str(_("You don't have permission to perform this action")), + extensions={"code": "permission_required"}, + ) + + set_files_parent(parent, files) diff --git a/baseapp/files/models.py b/baseapp/files/models.py new file mode 100644 index 00000000..f0413d84 --- /dev/null +++ b/baseapp/files/models.py @@ -0,0 +1,352 @@ +import pgtrigger +import swapper +from django.apps import apps +from django.conf import settings +from django.core.exceptions import ValidationError +from django.db import models +from django.db.models.signals import class_prepared +from django.dispatch import receiver +from django.utils.translation import gettext_lazy as _ +from model_utils.models import TimeStampedModel + +from baseapp_core.graphql import RelayModel +from baseapp_core.models import DocumentId, DocumentIdMixin, random_name_in + +from .utils import default_files_count + + +class FileTargetCountFunc(pgtrigger.Func): + """ + Reusable pgtrigger function for updating FileTarget counts. + Dynamically resolves table names for swappable models. + """ + + def render( + self, + meta=None, + fields=None, + columns=None, + **kwargs, + ) -> str: + FileTarget = swapper.load_model("baseapp_files", "FileTarget") + return self.func.format( + file_table=meta.db_table, + file_target_table=FileTarget._meta.db_table, + ) + + +def file_target_insert_trigger(): + """Trigger to update FileTarget counts when a file is inserted.""" + return pgtrigger.Trigger( + name="update_file_target_on_insert", + level=pgtrigger.Row, + when=pgtrigger.After, + operation=pgtrigger.Insert, + func=FileTargetCountFunc(""" + DECLARE + new_counts JSONB; + BEGIN + IF NEW.parent_id IS NOT NULL THEN + -- Calculate new counts + SELECT COALESCE( + jsonb_object_agg( + file_content_type, + count + ) || jsonb_build_object('total', SUM(count)), + '{{"total": 0}}'::jsonb + ) INTO new_counts + FROM ( + SELECT + COALESCE(file_content_type, 'unknown') as file_content_type, + COUNT(*)::int as count + FROM {file_table} + WHERE parent_id = NEW.parent_id + GROUP BY file_content_type + ) counts; + + -- Insert or update FileTarget + INSERT INTO {file_target_table} (target_id, files_count, is_files_enabled) + VALUES (NEW.parent_id, new_counts, true) + ON CONFLICT (target_id) + DO UPDATE SET files_count = new_counts; + END IF; + RETURN NEW; + END; + """), + ) + + +def file_target_update_trigger(): + """Trigger to update FileTarget counts when a file's parent is updated.""" + return pgtrigger.Trigger( + name="update_file_target_on_update", + level=pgtrigger.Row, + when=pgtrigger.After, + operation=pgtrigger.Update, + func=FileTargetCountFunc(""" + DECLARE + new_counts JSONB; + old_counts JSONB; + BEGIN + -- Update old parent if it changed + IF OLD.parent_id IS DISTINCT FROM NEW.parent_id + AND OLD.parent_id IS NOT NULL THEN + + SELECT COALESCE( + jsonb_object_agg( + file_content_type, + count + ) || jsonb_build_object('total', SUM(count)), + '{{"total": 0}}'::jsonb + ) INTO old_counts + FROM ( + SELECT + COALESCE(file_content_type, 'unknown') as file_content_type, + COUNT(*)::int as count + FROM {file_table} + WHERE parent_id = OLD.parent_id + GROUP BY file_content_type + ) counts; + + UPDATE {file_target_table} SET files_count = old_counts + WHERE target_id = OLD.parent_id; + END IF; + + -- Update new parent + IF NEW.parent_id IS NOT NULL THEN + SELECT COALESCE( + jsonb_object_agg( + file_content_type, + count + ) || jsonb_build_object('total', SUM(count)), + '{{"total": 0}}'::jsonb + ) INTO new_counts + FROM ( + SELECT + COALESCE(file_content_type, 'unknown') as file_content_type, + COUNT(*)::int as count + FROM {file_table} + WHERE parent_id = NEW.parent_id + GROUP BY file_content_type + ) counts; + + INSERT INTO {file_target_table} (target_id, files_count, is_files_enabled) + VALUES (NEW.parent_id, new_counts, true) + ON CONFLICT (target_id) + DO UPDATE SET files_count = new_counts; + END IF; + + RETURN NEW; + END; + """), + ) + + +def file_target_delete_trigger(): + """Trigger to update FileTarget counts when a file is deleted.""" + return pgtrigger.Trigger( + name="update_file_target_on_delete", + level=pgtrigger.Row, + when=pgtrigger.After, + operation=pgtrigger.Delete, + func=FileTargetCountFunc(""" + DECLARE + new_counts JSONB; + BEGIN + IF OLD.parent_id IS NOT NULL THEN + SELECT COALESCE( + jsonb_object_agg( + file_content_type, + count + ) || jsonb_build_object('total', SUM(count)), + '{{"total": 0}}'::jsonb + ) INTO new_counts + FROM ( + SELECT + COALESCE(file_content_type, 'unknown') as file_content_type, + COUNT(*)::int as count + FROM {file_table} + WHERE parent_id = OLD.parent_id + GROUP BY file_content_type + ) counts; + + UPDATE {file_target_table} SET files_count = new_counts + WHERE target_id = OLD.parent_id; + END IF; + + RETURN OLD; + END; + """), + ) + + +@receiver(class_prepared) +def add_file_target_triggers(sender, **kwargs): + """ + Add the FileTarget count triggers to the model when it is prepared. + This handles swappable models by adding triggers to the concrete model. + """ + # Only models that inherit from AbstractFile + if not issubclass(sender, AbstractFile): + return + + # Skip non-schema models + if sender._meta.abstract or sender._meta.proxy: + return + + # Skip swapped-out models + if sender._meta.swapped: + return + + if not hasattr(sender._meta, "triggers"): + sender._meta.triggers = [] + + existing = [t.name for t in sender._meta.triggers] + if "update_file_target_on_insert" not in existing: + sender._meta.triggers.append(file_target_insert_trigger()) + if "update_file_target_on_update" not in existing: + sender._meta.triggers.append(file_target_update_trigger()) + if "update_file_target_on_delete" not in existing: + sender._meta.triggers.append(file_target_delete_trigger()) + + +class AbstractFileTarget(models.Model): + target = models.OneToOneField( + DocumentId, + on_delete=models.CASCADE, + primary_key=True, + related_name="file_target", + help_text=_("The document this file target belongs to"), + ) + + files_count = models.JSONField(default=default_files_count) + is_files_enabled = models.BooleanField(default=True, verbose_name=_("is files enabled")) + + class Meta: + abstract = True + + def __str__(self): + return f"FileTarget for {self.target}" + + +file_inheritances = [] + +if apps.is_installed("baseapp_profiles"): + + class ProfileMixin(models.Model): + profile = models.ForeignKey( + swapper.get_model_name("baseapp_profiles", "Profile"), + verbose_name=_("profile"), + related_name="files", + on_delete=models.SET_NULL, + null=True, + blank=True, + ) + + class Meta: + abstract = True + + file_inheritances.append(ProfileMixin) + + +class AbstractFile(*file_inheritances, DocumentIdMixin, RelayModel, TimeStampedModel): + parent = models.ForeignKey( + DocumentId, + null=True, + blank=True, + on_delete=models.CASCADE, + related_name="files", + help_text=_("The parent document this file belongs to"), + ) + + file_content_type = models.CharField(max_length=150, null=True, blank=True) + file_name = models.CharField(max_length=512, null=True, blank=True) + file_size = models.PositiveIntegerField(null=True, help_text=_("File size in bytes")) + file = models.FileField(max_length=512, upload_to=random_name_in("files")) + + name = models.CharField(max_length=512, null=True, blank=True) + description = models.TextField(null=True, blank=True) + + created_by = models.ForeignKey( + settings.AUTH_USER_MODEL, + related_name="files_created", + on_delete=models.CASCADE, + null=True, + ) + + # Upload status tracking + class UploadStatus(models.TextChoices): + PENDING = "pending", _("Pending Upload") + UPLOADING = "uploading", _("Upload in Progress") + COMPLETED = "completed", _("Upload Completed") + FAILED = "failed", _("Upload Failed") + ABORTED = "aborted", _("Upload Aborted") + + upload_status = models.CharField( + max_length=20, + choices=UploadStatus.choices, + default=UploadStatus.COMPLETED, + db_index=True, + help_text=_("Status of the file upload"), + ) + + # S3 multipart upload tracking + upload_id = models.CharField( + max_length=512, + null=True, + blank=True, + help_text=_("S3 multipart upload ID for in-progress uploads"), + ) + + # Metadata for multipart uploads + total_parts = models.PositiveIntegerField( + null=True, + blank=True, + help_text=_("Total number of parts for multipart upload"), + ) + + uploaded_parts = models.JSONField( + null=True, + blank=True, + default=dict, + help_text=_("Tracking uploaded parts with ETags"), + ) + + # Upload expiration for cleanup + upload_expires_at = models.DateTimeField( + null=True, + blank=True, + db_index=True, + help_text=_("Expiration time for pending uploads"), + ) + + def save(self, *args, **kwargs): + """ + Validate that files are enabled for the parent object before saving. + """ + if self.parent_id: + FileTarget = swapper.load_model("baseapp_files", "FileTarget") + try: + file_target = FileTarget.objects.get(target_id=self.parent_id) + if not file_target.is_files_enabled: + raise ValidationError( + _("Files are not enabled for this object."), + code="files_disabled", + ) + except FileTarget.DoesNotExist: + # If FileTarget doesn't exist, it will be created by the trigger + # and is_files_enabled defaults to True, so we allow the save + pass + + super().save(*args, **kwargs) + + class Meta: + abstract = True + # Note: Triggers are registered via the class_prepared signal in this module + # to properly handle swappable models. + + @classmethod + def get_graphql_object_type(cls): + from .graphql.object_types import FileObjectType + + return FileObjectType diff --git a/baseapp/files/permissions.py b/baseapp/files/permissions.py new file mode 100644 index 00000000..b6e647b5 --- /dev/null +++ b/baseapp/files/permissions.py @@ -0,0 +1,55 @@ +import swapper +from django.contrib.auth.backends import BaseBackend + +File = swapper.load_model("baseapp_files", "File") +file_app_label = File._meta.app_label +file_model_name = File._meta.model_name.lower() + + +class FilesPermissionsBackend(BaseBackend): + def authenticate(self, request, **kwargs): + return None + + def has_perm(self, user_obj, perm, obj=None): + if perm == f"{file_app_label}.attach_{file_model_name}": + if obj: + return obj.created_by == user_obj + return False + + if perm == f"{file_app_label}.view_{file_model_name}": + if obj: + # Owner can view their own files + if obj.created_by == user_obj: + return True + # Check if user has global permission (for admins with view_file perm) + # This is needed because ModelBackend doesn't grant global perms with obj + if user_obj.has_perm(perm): + return True + return False + return False + + if perm == f"{file_app_label}.change_{file_model_name}": + if obj: + # Owner can change their own files + if obj.created_by == user_obj: + return True + # Check if user has global permission (for admins with change_file perm) + # This is needed because ModelBackend doesn't grant global perms with obj + if user_obj.has_perm(perm): + return True + return False + return False + + if perm == f"{file_app_label}.delete_{file_model_name}": + if obj: + # Owner can delete their own files + if obj.created_by == user_obj: + return True + # Check if user has global permission (for admins with delete_file perm) + # This is needed because ModelBackend doesn't grant global perms with obj + if user_obj.has_perm(perm): + return True + return False + return False + + return False diff --git a/baseapp/files/plugin.py b/baseapp/files/plugin.py new file mode 100644 index 00000000..67d2ab4d --- /dev/null +++ b/baseapp/files/plugin.py @@ -0,0 +1,54 @@ +from baseapp_core.plugins.base import BaseAppPlugin, PackageSettings + + +class FilesPlugin(BaseAppPlugin): + """ + The Files plugin provides file uploads, file management, and the ability to attach + files to any object exposing a ``DocumentId`` (via ``FilesInterface``). + """ + + @property + def name(self) -> str: + return "baseapp_files" + + @property + def package_name(self) -> str: + return "baseapp.files" + + def get_settings(self) -> PackageSettings: + return PackageSettings( + INSTALLED_APPS=[], + AUTHENTICATION_BACKENDS={ + "baseapp_files": [ + "baseapp.files.permissions.FilesPermissionsBackend", + ], + }, + graphql_queries=[ + "baseapp.files.graphql.queries.FilesQueries", + ], + graphql_mutations=[ + "baseapp.files.graphql.mutations.FilesMutations", + ], + v1_urlpatterns=self.v1_urlpatterns, + required_packages=[ + {"baseapp_core": "Core shared models (DocumentId) and plugin infrastructure"}, + ], + optional_packages=[ + { + "baseapp_profiles": "If enabled, File exposes the authoring Profile via the `profile` field." + }, + { + "baseapp_comments": "If enabled, files expose comments through the CommentsInterface." + }, + { + "baseapp_reactions": "If enabled, files expose reactions through the ReactionsInterface." + }, + ], + ) + + def v1_urlpatterns(self, include, path, re_path): + from baseapp.files.rest_framework.routers import files_router + + return [ + re_path(r"", include(files_router.urls)), + ] diff --git a/baseapp/files/rest_framework/__init__.py b/baseapp/files/rest_framework/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/baseapp/files/rest_framework/files/__init__.py b/baseapp/files/rest_framework/files/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/baseapp/files/rest_framework/files/serializers.py b/baseapp/files/rest_framework/files/serializers.py new file mode 100644 index 00000000..cde12db2 --- /dev/null +++ b/baseapp/files/rest_framework/files/serializers.py @@ -0,0 +1,72 @@ +import swapper +from rest_framework import serializers + +from baseapp_core.rest_framework.serializers import ModelSerializer + +File = swapper.load_model("baseapp_files", "File") + + +class FileSerializer(ModelSerializer): + """Standard file serializer for CRUD operations.""" + + # Use public_id instead of primary key + id = serializers.SerializerMethodField() + url = serializers.SerializerMethodField() + relay_id = serializers.SerializerMethodField() + created_by_name = serializers.CharField(source="created_by.get_full_name", read_only=True) + + # Parent DocumentId public_id + parent_id = serializers.SerializerMethodField() + + class Meta: + model = File + fields = [ + "id", + "relay_id", + "file_name", + "file_size", + "file_content_type", + "name", + "description", + "url", + "upload_status", + "created_by", + "created_by_name", + "created", + "modified", + "parent_id", + ] + read_only_fields = [ + "id", + "relay_id", + "upload_status", + "created_by", + "created", + "modified", + "file_size", + "parent_id", + ] + + def get_id(self, obj): + """Get public_id instead of primary key.""" + return obj.public_id + + def get_url(self, obj): + """Get file URL if upload completed.""" + if obj.upload_status == "completed" and obj.file: + request = self.context.get("request") + url = obj.file.url + if request: + return request.build_absolute_uri(url) + return url + return None + + def get_relay_id(self, obj): + """Get GraphQL relay ID for the file.""" + return obj.relay_id + + def get_parent_id(self, obj): + """Get parent DocumentId public_id.""" + if obj.parent: + return obj.parent.public_id + return None diff --git a/baseapp/files/rest_framework/files/views.py b/baseapp/files/rest_framework/files/views.py new file mode 100644 index 00000000..1b870884 --- /dev/null +++ b/baseapp/files/rest_framework/files/views.py @@ -0,0 +1,98 @@ +import swapper +from rest_framework import mixins, permissions, status, viewsets +from rest_framework.response import Response + +from baseapp_core.models import DocumentId +from baseapp_core.rest_framework.decorators import action +from baseapp_profiles.rest_framework import CurrentProfileMixin + +from ..uploads.permissions import IsOwnerOrReadOnly +from ..uploads.serializers import SetParentSerializer +from .serializers import FileSerializer + +File = swapper.load_model("baseapp_files", "File") + + +class FilesViewSet( + CurrentProfileMixin, + mixins.RetrieveModelMixin, + mixins.UpdateModelMixin, + mixins.DestroyModelMixin, + mixins.ListModelMixin, + viewsets.GenericViewSet, +): + """ + ViewSet for file CRUD operations. + + Endpoints: + GET /v1/files/ - List files + GET /v1/files/{public_id} - Retrieve file + PATCH /v1/files/{public_id} - Update file metadata + DELETE /v1/files/{public_id} - Delete file + POST /v1/files/{public_id}/set-parent - Set parent after upload + """ + + serializer_class = FileSerializer + permission_classes = [permissions.IsAuthenticated, IsOwnerOrReadOnly] + lookup_field = "pk" + lookup_url_kwarg = "pk" + + def get_object(self): + """Look up file by public_id instead of primary key.""" + from rest_framework.exceptions import NotFound + + lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field + public_id = self.kwargs[lookup_url_kwarg] + + obj = File.get_by_public_id(public_id) + if obj is None: + raise NotFound("File not found") + + self.check_object_permissions(self.request, obj) + return obj + + def get_queryset(self): + """Filter files based on user and query params.""" + qs = File.objects.all() + + # Only show completed files by default + status_filter = self.request.query_params.get("status", File.UploadStatus.COMPLETED) + if status_filter: + qs = qs.filter(upload_status=status_filter) + + # Filter by parent using DocumentId public_id + parent_id = self.request.query_params.get("parent_id") + if parent_id: + try: + document_id = DocumentId.objects.get(public_id=parent_id) + qs = qs.filter(parent=document_id) + except DocumentId.DoesNotExist: + qs = qs.none() + + # Filter by owner (users can see their own files) + if not self.request.user.is_staff: + qs = qs.filter(created_by=self.request.user) + + return qs.select_related("created_by", "parent__content_type").order_by("-created") + + @action(detail=True, methods=["post"], serializer_class=SetParentSerializer) + def set_parent(self, request, pk=None): + """ + Set parent for a standalone file. + + POST /v1/files/{id}/set-parent + { + "parent_id": "uuid-of-document-id" + } + """ + file_obj = self.get_object() + + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + + file_obj.parent_id = serializer.validated_data["parent_id"] + file_obj.save() + + return Response( + FileSerializer(file_obj, context={"request": request}).data, status=status.HTTP_200_OK + ) diff --git a/baseapp/files/rest_framework/routers.py b/baseapp/files/rest_framework/routers.py new file mode 100644 index 00000000..32011d90 --- /dev/null +++ b/baseapp/files/rest_framework/routers.py @@ -0,0 +1,13 @@ +from baseapp_core.rest_framework.routers import DefaultRouter + +from .files.views import FilesViewSet +from .uploads.presigned_views import PresignedUploadViewSet +from .uploads.views import FileUploadViewSet + +files_router = DefaultRouter(trailing_slash=False) + +files_router.register( + r"files/presigned-uploads", PresignedUploadViewSet, basename="presigned-uploads" +) +files_router.register(r"files/uploads", FileUploadViewSet, basename="file-uploads") +files_router.register(r"files", FilesViewSet, basename="files") diff --git a/baseapp/files/rest_framework/uploads/__init__.py b/baseapp/files/rest_framework/uploads/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/baseapp/files/rest_framework/uploads/permissions.py b/baseapp/files/rest_framework/uploads/permissions.py new file mode 100644 index 00000000..12e698d6 --- /dev/null +++ b/baseapp/files/rest_framework/uploads/permissions.py @@ -0,0 +1,33 @@ +import swapper +from rest_framework import permissions + +File = swapper.load_model("baseapp_files", "File") +file_app_label = File._meta.app_label +file_model_name = File._meta.model_name.lower() + + +class IsOwner(permissions.BasePermission): + """ + Permission to only allow owners to access their files. + Uses has_perm for easy override at project level. + """ + + def has_object_permission(self, request, view, obj): + return request.user.has_perm(f"{file_app_label}.view_{file_model_name}", obj) + + +class IsOwnerOrReadOnly(permissions.BasePermission): + """ + Permission to only allow owners of a file to modify it. + Read permissions check view_file permission. + Write permissions check change_file permission. + Uses has_perm for easy override at project level. + """ + + def has_object_permission(self, request, view, obj): + # Read permissions - check view permission + if request.method in permissions.SAFE_METHODS: + return request.user.has_perm(f"{file_app_label}.view_{file_model_name}", obj) + + # Write permissions - check change permission + return request.user.has_perm(f"{file_app_label}.change_{file_model_name}", obj) diff --git a/baseapp/files/rest_framework/uploads/presigned_views.py b/baseapp/files/rest_framework/uploads/presigned_views.py new file mode 100644 index 00000000..6a030292 --- /dev/null +++ b/baseapp/files/rest_framework/uploads/presigned_views.py @@ -0,0 +1,142 @@ +import swapper +from django.core import signing +from django.views.decorators.csrf import csrf_exempt +from rest_framework import status, viewsets +from rest_framework.decorators import action +from rest_framework.exceptions import ValidationError +from rest_framework.permissions import AllowAny +from rest_framework.response import Response + +from ...services.upload_service import UploadService + +File = swapper.load_model("baseapp_files", "File") + + +class PresignedUploadViewSet(viewsets.GenericViewSet): + """ + ViewSet for handling presigned upload URLs (local storage fallback). + + This ViewSet does NOT use session authentication or CSRF protection. + Instead, it validates signed tokens embedded in the URL. + + Endpoints: + PUT /v1/files/presigned-uploads/{file_id}/upload-part/{part_number}/?token=... + """ + + queryset = File.objects.all() + permission_classes = [AllowAny] # Authentication via signed token + authentication_classes = [] # No authentication classes + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.upload_service = UploadService() + + @csrf_exempt + def dispatch(self, request, *args, **kwargs): + """Override dispatch to disable CSRF protection.""" + return super().dispatch(request, *args, **kwargs) + + def perform_authentication(self, request): + """ + Skip DRF's authentication process entirely. + Authentication is handled via signed tokens in the URL. + """ + pass + + @action(detail=True, methods=["put"], url_path="upload-part/(?P[0-9]+)") + def upload_part(self, request, pk=None, part_number=None): + """ + Upload a single part using a presigned URL. + + PUT /v1/files/presigned-uploads/{file_id}/upload-part/{part_number}/?token= + + Body: raw binary data of the part + + Returns: + { + "part_number": 1, + "etag": "abc123..." + } + """ + # Get and validate the signed token + token = request.query_params.get("token") + if not token: + return Response( + {"error": "Missing token parameter"}, + status=status.HTTP_401_UNAUTHORIZED, + ) + + try: + # Verify the token (max age: 1 hour) + token_data = signing.loads(token, max_age=3600) + + # Validate token matches request + if str(token_data.get("file_id")) != str(pk) or str( + token_data.get("part_number") + ) != str(part_number): + return Response( + {"error": "Invalid token for this file/part"}, + status=status.HTTP_401_UNAUTHORIZED, + ) + + except signing.SignatureExpired: + return Response( + {"error": "Token has expired"}, + status=status.HTTP_401_UNAUTHORIZED, + ) + except signing.BadSignature: + return Response( + {"error": "Invalid token signature"}, + status=status.HTTP_401_UNAUTHORIZED, + ) + + # Get the file object + try: + file_obj = File.objects.get(id=pk) + except File.DoesNotExist: + return Response( + {"error": "File not found"}, + status=status.HTTP_404_NOT_FOUND, + ) + + # Validate state + if file_obj.upload_status not in [File.UploadStatus.PENDING, File.UploadStatus.UPLOADING]: + return Response( + {"error": f"Cannot upload parts in status: {file_obj.upload_status}"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + try: + part_number = int(part_number) + + # Validate part number + if part_number < 1 or part_number > file_obj.total_parts: + return Response( + {"error": f"Invalid part number. Must be between 1 and {file_obj.total_parts}"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + # Get the raw binary data from request + data = request.body + + # Upload the part using the storage handler + etag = self.upload_service.handler.upload_part( + file_obj=file_obj, + upload_id=file_obj.upload_id, + part_number=part_number, + data=data, + ) + + # Return response with ETag header (S3-compatible) + response = Response( + {"part_number": part_number, "etag": etag}, + status=status.HTTP_200_OK, + ) + response["Access-Control-Expose-Headers"] = "*" + response["ETag"] = etag + return response + + except ValueError as e: + raise ValidationError(str(e)) + except Exception as e: + raise ValidationError(f"Failed to upload part: {str(e)}") diff --git a/baseapp/files/rest_framework/uploads/serializers.py b/baseapp/files/rest_framework/uploads/serializers.py new file mode 100644 index 00000000..3fa8900c --- /dev/null +++ b/baseapp/files/rest_framework/uploads/serializers.py @@ -0,0 +1,94 @@ +import swapper +from rest_framework import serializers + +from baseapp_core.models import DocumentId + +File = swapper.load_model("baseapp_files", "File") + + +class InitiateUploadSerializer(serializers.Serializer): + """Serializer for initiating a multipart upload.""" + + file_name = serializers.CharField(max_length=512) + file_size = serializers.IntegerField(min_value=1) + file_content_type = serializers.CharField(max_length=150) + num_parts = serializers.IntegerField(min_value=1, max_value=10000) + part_size = serializers.IntegerField(min_value=1) + + # Optional parent relationship via DocumentId public_id + parent_id = serializers.UUIDField(required=False, allow_null=True) + + def validate_parent_id(self, value): + """Validate parent_id exists as a DocumentId public_id.""" + if not value: + return None + + try: + document_id = DocumentId.objects.get(public_id=value) + return document_id.pk + except DocumentId.DoesNotExist: + raise serializers.ValidationError("Invalid parent_id: DocumentId not found") + + def validate(self, data): + """Validate file size matches parts.""" + num_parts = data["num_parts"] + part_size = data["part_size"] + file_size = data["file_size"] + + min_size = (num_parts - 1) * part_size + max_size = num_parts * part_size + + if file_size < min_size or file_size > max_size: + raise serializers.ValidationError( + f"File size {file_size} doesn't match {num_parts} parts of {part_size} bytes" + ) + + return data + + +class UploadResponseSerializer(serializers.Serializer): + """Response serializer for initiated upload.""" + + id = serializers.UUIDField(source="file_obj.public_id") + relay_id = serializers.CharField(source="file_obj.relay_id", read_only=True) + upload_id = serializers.CharField() + presigned_urls = serializers.ListField() + expires_in = serializers.IntegerField() + upload_status = serializers.CharField(source="file_obj.upload_status") + + +class CompleteUploadSerializer(serializers.Serializer): + """Serializer for completing a multipart upload.""" + + parts = serializers.ListField( + child=serializers.DictField(), + min_length=1, + ) + + def validate_parts(self, value): + """Validate parts structure.""" + for part in value: + if "part_number" not in part or "etag" not in part: + raise serializers.ValidationError("Each part must have 'part_number' and 'etag'") + + if not isinstance(part["part_number"], int) or part["part_number"] < 1: + raise serializers.ValidationError("part_number must be a positive integer") + + if not isinstance(part["etag"], str) or not part["etag"]: + raise serializers.ValidationError("etag must be a non-empty string") + + return value + + +class SetParentSerializer(serializers.Serializer): + """Serializer for setting parent after upload.""" + + parent_id = serializers.UUIDField() + + def validate_parent_id(self, value): + """Validate parent_id exists as a DocumentId public_id.""" + try: + document_id = DocumentId.objects.get(public_id=value) + return document_id.pk + except DocumentId.DoesNotExist: + raise serializers.ValidationError("Invalid parent_id: DocumentId not found") diff --git a/baseapp/files/rest_framework/uploads/views.py b/baseapp/files/rest_framework/uploads/views.py new file mode 100644 index 00000000..8766b615 --- /dev/null +++ b/baseapp/files/rest_framework/uploads/views.py @@ -0,0 +1,164 @@ +import swapper +from rest_framework import permissions, status, viewsets +from rest_framework.exceptions import ValidationError +from rest_framework.response import Response + +from baseapp_core.rest_framework.decorators import action +from baseapp_profiles.rest_framework import CurrentProfileMixin + +from ...services.upload_service import UploadService +from .serializers import ( + CompleteUploadSerializer, + InitiateUploadSerializer, + UploadResponseSerializer, +) + +File = swapper.load_model("baseapp_files", "File") +file_app_label = File._meta.app_label +file_model_name = File._meta.model_name.lower() + + +class FileUploadViewSet(CurrentProfileMixin, viewsets.GenericViewSet): + """ + ViewSet for managing multipart file uploads. + + Endpoints: + POST /v1/files/uploads/ - Initiate upload + POST /v1/files/uploads/{public_id}/complete - Complete upload + DELETE /v1/files/uploads/{public_id} - Abort upload + + Note: Part uploads for local storage use presigned URLs handled by PresignedUploadViewSet + """ + + queryset = File.objects.all() + permission_classes = [permissions.IsAuthenticated] + lookup_field = "pk" + lookup_url_kwarg = "pk" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.upload_service = UploadService() + + def get_object(self): + """Look up file by public_id instead of primary key.""" + lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field + public_id = self.kwargs[lookup_url_kwarg] + + obj = File.get_by_public_id(public_id) + if obj is None: + from rest_framework.exceptions import NotFound + + raise NotFound("File not found") + + self.check_object_permissions(self.request, obj) + return obj + + def get_serializer_class(self): + if self.action == "create": + return InitiateUploadSerializer + elif self.action == "complete": + return CompleteUploadSerializer + return InitiateUploadSerializer + + def create(self, request): + """ + Initiate a multipart upload. + + POST /v1/files/uploads/ + { + "file_name": "large-video.mp4", + "file_size": 104857600, + "file_content_type": "video/mp4", + "num_parts": 20, + "part_size": 5242880, + "parent_id": "ab12" // optional + } + + Returns presigned URLs for each part. + """ + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + + try: + # Get user's profile if available + profile = request.user.current_profile + + file_obj, upload_data = self.upload_service.initiate_multipart_upload( + user=request.user, profile=profile, **serializer.validated_data + ) + + # Prepare response + response_data = {"file_obj": file_obj, **upload_data} + + response_serializer = UploadResponseSerializer(response_data) + return Response(response_serializer.data, status=status.HTTP_201_CREATED) + + except ValueError as e: + raise ValidationError(str(e)) + except Exception as e: + raise ValidationError(f"Failed to initiate upload: {str(e)}") + + @action(detail=True, methods=["post"]) + def complete(self, request, pk=None): + """ + Complete a multipart upload. + + POST /v1/files/uploads/{id}/complete + { + "parts": [ + {"part_number": 1, "etag": "abc123..."}, + {"part_number": 2, "etag": "def456..."}, + ... + ] + } + """ + file_obj = self.get_object() + + # Check permission (owner or has change_file permission) + if not request.user.has_perm(f"{file_app_label}.change_{file_model_name}", file_obj): + return Response( + {"error": "You don't have permission to complete this upload"}, + status=status.HTTP_403_FORBIDDEN, + ) + + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + + try: + completed_file = self.upload_service.complete_multipart_upload( + file_id=file_obj.id, parts=serializer.validated_data["parts"] + ) + + from ..files.serializers import FileSerializer + + return Response( + FileSerializer(completed_file, context={"request": request}).data, + status=status.HTTP_200_OK, + ) + + except ValueError as e: + raise ValidationError(str(e)) + except Exception as e: + raise ValidationError(f"Failed to complete upload: {str(e)}") + + def destroy(self, request, pk=None): + """ + Abort a multipart upload. + + DELETE /v1/files/uploads/{id} + """ + file_obj = self.get_object() + + # Check permission (owner or has delete_file permission) + if not request.user.has_perm(f"{file_app_label}.delete_{file_model_name}", file_obj): + return Response( + {"error": "You don't have permission to abort this upload"}, + status=status.HTTP_403_FORBIDDEN, + ) + + try: + self.upload_service.abort_multipart_upload(file_obj.id) + return Response(status=status.HTTP_204_NO_CONTENT) + + except Exception as e: + raise ValidationError(f"Failed to abort upload: {str(e)}") diff --git a/baseapp/files/services/__init__.py b/baseapp/files/services/__init__.py new file mode 100644 index 00000000..54d075b6 --- /dev/null +++ b/baseapp/files/services/__init__.py @@ -0,0 +1,4 @@ +from .cleanup import cleanup_expired_uploads, cleanup_failed_uploads +from .upload_service import UploadService + +__all__ = ["UploadService", "cleanup_expired_uploads", "cleanup_failed_uploads"] diff --git a/baseapp/files/services/cleanup.py b/baseapp/files/services/cleanup.py new file mode 100644 index 00000000..64ffc41d --- /dev/null +++ b/baseapp/files/services/cleanup.py @@ -0,0 +1,64 @@ +import logging +from datetime import timedelta + +import swapper +from celery import shared_task +from django.utils import timezone + +from ..storage import get_upload_handler + +File = swapper.load_model("baseapp_files", "File") + +logger = logging.getLogger(__name__) + + +@shared_task +def cleanup_expired_uploads(): + """ + Celery task to cleanup expired/abandoned uploads. + + Run this periodically (e.g., hourly via celery beat). + """ + handler = get_upload_handler() + now = timezone.now() + + # Find expired uploads + expired_files = File.objects.filter( + upload_status__in=[File.UploadStatus.PENDING, File.UploadStatus.UPLOADING], + upload_expires_at__lt=now, + ) + + cleaned_count = 0 + for file_obj in expired_files: + try: + if file_obj.upload_id: + handler.abort_upload(file_obj, file_obj.upload_id) + + file_obj.upload_status = File.UploadStatus.ABORTED + file_obj.upload_id = None + file_obj.save() + + cleaned_count += 1 + except Exception as e: + # Log error but continue + logger.error(f"Failed to cleanup file {file_obj.id}: {e}") + + logger.info(f"Cleaned up {cleaned_count} expired uploads") + return f"Cleaned up {cleaned_count} expired uploads" + + +def cleanup_failed_uploads(days_old=7): + """ + Cleanup failed/aborted uploads older than specified days. + + Can be called manually or via management command. + """ + cutoff = timezone.now() - timedelta(days=days_old) + + deleted_count, _ = File.objects.filter( + upload_status__in=[File.UploadStatus.FAILED, File.UploadStatus.ABORTED], + modified__lt=cutoff, + ).delete() + + logger.info(f"Deleted {deleted_count} old failed/aborted uploads") + return f"Deleted {deleted_count} old failed/aborted uploads" diff --git a/baseapp/files/services/upload_service.py b/baseapp/files/services/upload_service.py new file mode 100644 index 00000000..0e7293d8 --- /dev/null +++ b/baseapp/files/services/upload_service.py @@ -0,0 +1,163 @@ +from datetime import timedelta + +import swapper +from django.conf import settings +from django.db import transaction +from django.utils import timezone + +from ..storage import get_upload_handler + +File = swapper.load_model("baseapp_files", "File") + + +class UploadService: + """Business logic for file uploads.""" + + def __init__(self): + self.handler = get_upload_handler() + + @transaction.atomic + def initiate_multipart_upload( + self, + user, + file_name: str, + file_size: int, + file_content_type: str, + num_parts: int, + part_size: int, + parent_id=None, + profile=None, + ): + """ + Initiate a multipart upload. + + Args: + user: The user initiating the upload + file_name: Original file name + file_size: Size in bytes + file_content_type: MIME type + num_parts: Number of parts for multipart upload + part_size: Size of each part in bytes + parent_id: DocumentId primary key (optional) + profile: Profile object (optional) + + Returns: + (file_obj, upload_data) + """ + + # Validate inputs + self._validate_file_params(file_size, num_parts, part_size) + + # Create File record in pending state + file_obj = File.objects.create( + file_name=file_name, + file_size=file_size, + file_content_type=file_content_type, + upload_status=File.UploadStatus.PENDING, + total_parts=num_parts, + created_by=user, + profile=profile, + parent_id=parent_id, + uploaded_parts={}, + upload_expires_at=timezone.now() + timedelta(hours=24), + ) + + # Get presigned URLs from storage handler + try: + upload_data = self.handler.initiate_upload(file_obj, num_parts, part_size) + + # Store upload_id + file_obj.upload_id = upload_data["upload_id"] + file_obj.upload_status = File.UploadStatus.UPLOADING + file_obj.save(update_fields=["upload_id", "upload_status"]) + + return file_obj, upload_data + + except Exception: + # Cleanup file record if upload initiation fails + file_obj.delete() + raise + + @transaction.atomic + def complete_multipart_upload(self, file_id: int, parts: list): + """ + Complete a multipart upload. + """ + file_obj = File.objects.select_for_update().get(id=file_id) + + # Validate state + if file_obj.upload_status not in [File.UploadStatus.PENDING, File.UploadStatus.UPLOADING]: + raise ValueError(f"Cannot complete upload in status: {file_obj.upload_status}") + + if not file_obj.upload_id: + raise ValueError("No upload_id found") + + # Validate parts + self._validate_parts(file_obj, parts) + + # Complete with storage handler + try: + file_path = self.handler.complete_upload(file_obj, file_obj.upload_id, parts) + + # Update file record + file_obj.file = file_path + file_obj.upload_status = File.UploadStatus.COMPLETED + file_obj.uploaded_parts = {str(p["part_number"]): p["etag"] for p in parts} + file_obj.upload_id = None + file_obj.upload_expires_at = None + file_obj.save() + + return file_obj + + except Exception: + file_obj.upload_status = File.UploadStatus.FAILED + file_obj.save(update_fields=["upload_status"]) + raise + + @transaction.atomic + def abort_multipart_upload(self, file_id: int): + """ + Abort a multipart upload and cleanup. + """ + file_obj = File.objects.select_for_update().get(id=file_id) + + if file_obj.upload_id: + try: + self.handler.abort_upload(file_obj, file_obj.upload_id) + except Exception: + # Log but continue to mark as aborted + pass + + file_obj.upload_status = File.UploadStatus.ABORTED + file_obj.upload_id = None + file_obj.save() + + return file_obj + + def _validate_file_params(self, file_size, num_parts, part_size): + """Validate file upload parameters.""" + max_file_size = getattr(settings, "MAX_FILE_UPLOAD_SIZE", 5 * 1024 * 1024 * 1024) # 5GB + + if file_size > max_file_size: + raise ValueError(f"File size exceeds maximum {max_file_size} bytes") + + if num_parts < 1 or num_parts > 10000: + raise ValueError("Number of parts must be between 1 and 10,000") + + if num_parts > 1 and part_size < 5 * 1024 * 1024: + raise ValueError("Part size must be at least 5MB for multipart uploads") + + expected_size = (num_parts - 1) * part_size + if expected_size > file_size: + raise ValueError("Part size and count don't match file size") + + def _validate_parts(self, file_obj, parts): + """Validate uploaded parts.""" + if len(parts) != file_obj.total_parts: + raise ValueError(f"Expected {file_obj.total_parts} parts, received {len(parts)}") + + part_numbers = {p["part_number"] for p in parts} + expected = set(range(1, file_obj.total_parts + 1)) + + if part_numbers != expected: + raise ValueError("Missing or duplicate part numbers") diff --git a/baseapp/files/storage/__init__.py b/baseapp/files/storage/__init__.py new file mode 100644 index 00000000..fd1e805e --- /dev/null +++ b/baseapp/files/storage/__init__.py @@ -0,0 +1,18 @@ +from django.core.files.storage import default_storage + + +def get_upload_handler(): + """ + Factory to get appropriate upload handler based on storage backend. + """ + storage_class = type(default_storage) + + # Check if using S3 storage + if "S3Boto3Storage" in str(storage_class): + from .s3 import S3MultipartUploadHandler + + return S3MultipartUploadHandler() + else: + from .local import LocalUploadHandler + + return LocalUploadHandler() diff --git a/baseapp/files/storage/base.py b/baseapp/files/storage/base.py new file mode 100644 index 00000000..dfba8d84 --- /dev/null +++ b/baseapp/files/storage/base.py @@ -0,0 +1,51 @@ +from abc import ABC, abstractmethod +from typing import Any, Dict, List + + +class BaseUploadHandler(ABC): + """Abstract interface for upload handlers (S3 vs Local).""" + + @abstractmethod + def supports_multipart(self) -> bool: + """Whether this handler supports true multipart uploads.""" + pass + + @abstractmethod + def initiate_upload(self, file_obj, num_parts: int, part_size: int) -> Dict[str, Any]: + """ + Initiate an upload and return presigned URLs. + + Returns: + { + 'upload_id': str, + 'presigned_urls': [ + {'part_number': 1, 'url': 'https://...'}, + {'part_number': 2, 'url': 'https://...'}, + ], + 'expires_in': int (seconds) + } + """ + pass + + @abstractmethod + def complete_upload(self, file_obj, upload_id: str, parts: List[Dict[str, Any]]) -> str: + """ + Complete the upload. + + Args: + parts: [{'part_number': 1, 'etag': 'xxx'}, ...] + + Returns: + final_url: str (file URL) + """ + pass + + @abstractmethod + def abort_upload(self, file_obj, upload_id: str) -> None: + """Abort and cleanup an upload.""" + pass + + @abstractmethod + def get_file_url(self, file_obj) -> str: + """Get the URL for accessing the file.""" + pass diff --git a/baseapp/files/storage/local.py b/baseapp/files/storage/local.py new file mode 100644 index 00000000..2e9bd3bd --- /dev/null +++ b/baseapp/files/storage/local.py @@ -0,0 +1,125 @@ +import os +import shutil +import uuid +from typing import Any, Dict, List + +from django.conf import settings +from django.core import signing + +from baseapp_core.models import random_name_in + +from .base import BaseUploadHandler + + +class LocalUploadHandler(BaseUploadHandler): + """ + Simplified local storage handler for dev/test. + + This doesn't implement true multipart upload - instead it provides + "presigned URLs" that are actually backend endpoints with signed tokens + that accept the file parts and store them locally. + """ + + def supports_multipart(self) -> bool: + return False # Simplified multipart + + def initiate_upload(self, file_obj, num_parts: int, part_size: int) -> Dict[str, Any]: + """ + Generate presigned URLs with signed tokens for secure uploads. + """ + + # Generate a simple upload token + upload_id = str(uuid.uuid4()) + + # Create temp directory for parts if needed + temp_dir = os.path.join(settings.MEDIA_ROOT, "temp_uploads", upload_id) + os.makedirs(temp_dir, exist_ok=True) + + # Get the base URL from settings or use default + base_url = getattr(settings, "BASE_URL", "http://localhost:8000") + + # Generate presigned URLs with signed tokens + presigned_urls = [] + for part_num in range(1, num_parts + 1): + # Create a signed token for this specific file and part + token_data = { + "file_id": file_obj.id, + "part_number": part_num, + "upload_id": upload_id, + } + token = signing.dumps(token_data) + + presigned_urls.append( + { + "part_number": part_num, + "url": f"{base_url}/v1/files/presigned-uploads/{file_obj.id}/upload-part/{part_num}?token={token}", + "method": "PUT", + } + ) + + return { + "upload_id": upload_id, + "presigned_urls": presigned_urls, + "expires_in": 3600, + } + + def upload_part(self, file_obj, upload_id: str, part_number: int, data: bytes) -> str: + """ + Upload a part to local storage. + This method is called by the backend when handling local uploads. + + Returns: + etag: str (hash of the data) + """ + import hashlib + + temp_dir = os.path.join(settings.MEDIA_ROOT, "temp_uploads", upload_id) + os.makedirs(temp_dir, exist_ok=True) + + part_file = os.path.join(temp_dir, f"part_{part_number}") + + # Write the part + with open(part_file, "wb") as f: + f.write(data) + + # Generate ETag (MD5 hash like S3) + etag = hashlib.md5(data).hexdigest() + return etag + + def complete_upload(self, file_obj, upload_id: str, parts: List[Dict]) -> str: + """ + Combine uploaded parts into final file. + """ + temp_dir = os.path.join(settings.MEDIA_ROOT, "temp_uploads", upload_id) + + # Generate final file path + final_path = random_name_in("files")(file_obj, file_obj.file_name or "file") + full_path = os.path.join(settings.MEDIA_ROOT, final_path) + + # Ensure directory exists + os.makedirs(os.path.dirname(full_path), exist_ok=True) + + # Combine parts in order + with open(full_path, "wb") as outfile: + for part in sorted(parts, key=lambda x: x["part_number"]): + part_file = os.path.join(temp_dir, f"part_{part['part_number']}") + if os.path.exists(part_file): + with open(part_file, "rb") as infile: + outfile.write(infile.read()) + + # Cleanup temp files + shutil.rmtree(temp_dir, ignore_errors=True) + + # Return the relative path (FileField will handle this) + return final_path + + def abort_upload(self, file_obj, upload_id: str): + """Cleanup temp files.""" + temp_dir = os.path.join(settings.MEDIA_ROOT, "temp_uploads", upload_id) + shutil.rmtree(temp_dir, ignore_errors=True) + + def get_file_url(self, file_obj) -> str: + """Get the local file URL.""" + if file_obj.file: + return file_obj.file.url + return None diff --git a/baseapp/files/storage/s3.py b/baseapp/files/storage/s3.py new file mode 100644 index 00000000..fdfa906b --- /dev/null +++ b/baseapp/files/storage/s3.py @@ -0,0 +1,130 @@ +from typing import Any, Dict, List + +import boto3 +from botocore.exceptions import ClientError +from django.conf import settings + +from baseapp_core.models import random_name_in + +from .base import BaseUploadHandler + + +class S3MultipartUploadHandler(BaseUploadHandler): + """Production S3 multipart upload with presigned URLs.""" + + def __init__(self): + self.s3_client = boto3.client( + "s3", + aws_access_key_id=settings.AWS_ACCESS_KEY_ID, + aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, + region_name=getattr(settings, "AWS_S3_REGION_NAME", "us-east-1"), + ) + self.bucket = settings.AWS_STORAGE_BUCKET_NAME + self.url_expiration = getattr(settings, "FILE_UPLOAD_PRESIGNED_URL_EXPIRATION", 3600) + + def supports_multipart(self) -> bool: + return True + + def initiate_upload(self, file_obj, num_parts: int, part_size: int) -> Dict[str, Any]: + """Initiate S3 multipart upload and generate presigned URLs.""" + + # Validate constraints + if num_parts > 10000: + raise ValueError("S3 supports maximum 10,000 parts") + if num_parts > 1 and part_size < 5 * 1024 * 1024: # 5MB minimum + raise ValueError("S3 requires minimum 5MB per part (except last)") + + # Get the S3 key for this file + key = self._get_s3_key(file_obj) + + # Initiate multipart upload + response = self.s3_client.create_multipart_upload( + Bucket=self.bucket, + Key=key, + ContentType=file_obj.file_content_type or "application/octet-stream", + Metadata={ + "original_filename": file_obj.file_name or "", + "created_by": str(file_obj.created_by_id) if file_obj.created_by_id else "", + }, + ) + + upload_id = response["UploadId"] + + # Generate presigned URLs for each part + presigned_urls = [] + for part_num in range(1, num_parts + 1): + url = self.s3_client.generate_presigned_url( + "upload_part", + Params={ + "Bucket": self.bucket, + "Key": key, + "UploadId": upload_id, + "PartNumber": part_num, + }, + ExpiresIn=self.url_expiration, + ) + presigned_urls.append( + { + "part_number": part_num, + "url": url, + } + ) + + return { + "upload_id": upload_id, + "presigned_urls": presigned_urls, + "expires_in": self.url_expiration, + } + + def complete_upload(self, file_obj, upload_id: str, parts: List[Dict]) -> str: + """Complete S3 multipart upload.""" + + key = self._get_s3_key(file_obj) + + # Format parts for S3 API + multipart_upload = { + "Parts": [ + { + "PartNumber": part["part_number"], + "ETag": part["etag"], + } + for part in sorted(parts, key=lambda x: x["part_number"]) + ] + } + + # Complete the upload + self.s3_client.complete_multipart_upload( + Bucket=self.bucket, + Key=key, + UploadId=upload_id, + MultipartUpload=multipart_upload, + ) + + # Return the file key (not full URL, FileField will handle that) + return key + + def abort_upload(self, file_obj, upload_id: str): + """Abort S3 multipart upload and cleanup parts.""" + key = self._get_s3_key(file_obj) + + try: + self.s3_client.abort_multipart_upload( + Bucket=self.bucket, + Key=key, + UploadId=upload_id, + ) + except ClientError as e: + # Log but don't fail if already aborted/completed + if e.response["Error"]["Code"] != "NoSuchUpload": + raise + + def get_file_url(self, file_obj) -> str: + """Get the S3 URL for the file.""" + if file_obj.file: + return file_obj.file.url + return None + + def _get_s3_key(self, file_obj) -> str: + """Generate S3 key from file object.""" + # Use the same upload_to logic from the FileField + return random_name_in("files")(file_obj, file_obj.file_name or "file") diff --git a/baseapp/files/tests/__init__.py b/baseapp/files/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/baseapp/files/tests/conftest.py b/baseapp/files/tests/conftest.py new file mode 100644 index 00000000..d91acda5 --- /dev/null +++ b/baseapp/files/tests/conftest.py @@ -0,0 +1,2 @@ +from baseapp_core.graphql.testing.fixtures import * # noqa: F401,F403 +from baseapp_core.tests.fixtures import * # noqa: F401,F403 diff --git a/baseapp/files/tests/test_graphql.py b/baseapp/files/tests/test_graphql.py new file mode 100644 index 00000000..34640863 --- /dev/null +++ b/baseapp/files/tests/test_graphql.py @@ -0,0 +1,558 @@ +import pytest +import swapper +from django.contrib.contenttypes.models import ContentType + +from baseapp_comments.tests.factories import CommentFactory +from baseapp_core.models import DocumentId + +File = swapper.load_model("baseapp_files", "File") +FileTarget = swapper.load_model("baseapp_files", "FileTarget") + +pytestmark = pytest.mark.django_db + +COMMENT_FILES_QUERY = """ + query CommentFiles($id: ID!) { + node(id: $id) { + ... on FilesInterface { + files { + edges { + node { + id + } + } + } + } + } + } +""" + +COMMENT_FILES_COUNT_QUERY = """ + query CommentFilesCount($id: ID!) { + node(id: $id) { + ... on FilesInterface { + filesCount + } + } + } +""" + +COMMENT_IS_FILES_ENABLED_QUERY = """ + query CommentIsFilesEnabled($id: ID!) { + node(id: $id) { + ... on FilesInterface { + isFilesEnabled + } + } + } +""" + +COMMENT_FILES_FULL_QUERY = """ + query CommentFilesFull($id: ID!) { + node(id: $id) { + ... on FilesInterface { + filesCount + isFilesEnabled + files { + edges { + node { + id + } + } + } + } + } + } +""" + + +def test_comment_files_interface(graphql_client): + comment = CommentFactory() + comment_content_type = ContentType.objects.get_for_model(comment) + comment_document_id, _ = DocumentId.objects.get_or_create( + content_type=comment_content_type, + object_id=comment.pk, + ) + file = File.objects.create( + parent=comment_document_id, + file_name="example.txt", + ) + + response = graphql_client(COMMENT_FILES_QUERY, variables={"id": comment.relay_id}) + content = response.json() + + edges = content["data"]["node"]["files"]["edges"] + assert len(edges) == 1 + assert edges[0]["node"]["id"] == file.relay_id + + +def test_comment_files_count_interface(graphql_client): + comment = CommentFactory() + comment_content_type = ContentType.objects.get_for_model(comment) + comment_document_id, _ = DocumentId.objects.get_or_create( + content_type=comment_content_type, + object_id=comment.pk, + ) + + File.objects.create( + parent=comment_document_id, + file_content_type="image/png", + ) + File.objects.create( + parent=comment_document_id, + file_content_type="image/jpeg", + ) + + response = graphql_client(COMMENT_FILES_COUNT_QUERY, variables={"id": comment.relay_id}) + content = response.json() + + files_count = content["data"]["node"]["filesCount"] + assert files_count["total"] == 2 + assert files_count["image/png"] == 1 + assert files_count["image/jpeg"] == 1 + + +def test_comment_files_count_interface_empty(graphql_client): + comment = CommentFactory() + + response = graphql_client(COMMENT_FILES_COUNT_QUERY, variables={"id": comment.relay_id}) + content = response.json() + + files_count = content["data"]["node"]["filesCount"] + assert files_count["total"] == 0 + + +def test_comment_is_files_enabled_interface(graphql_client): + comment = CommentFactory() + + response = graphql_client(COMMENT_IS_FILES_ENABLED_QUERY, variables={"id": comment.relay_id}) + content = response.json() + + is_files_enabled = content["data"]["node"]["isFilesEnabled"] + assert is_files_enabled is True + + +def test_comment_is_files_enabled_interface_disabled(graphql_client): + comment = CommentFactory() + file_target = comment.get_file_target() + file_target.is_files_enabled = False + file_target.save() + + response = graphql_client(COMMENT_IS_FILES_ENABLED_QUERY, variables={"id": comment.relay_id}) + content = response.json() + + is_files_enabled = content["data"]["node"]["isFilesEnabled"] + assert is_files_enabled is False + + +def test_comment_files_full_interface(graphql_client): + comment = CommentFactory() + comment_content_type = ContentType.objects.get_for_model(comment) + comment_document_id, _ = DocumentId.objects.get_or_create( + content_type=comment_content_type, + object_id=comment.pk, + ) + + file = File.objects.create( + parent=comment_document_id, + file_content_type="image/png", + file_name="example.png", + ) + + response = graphql_client(COMMENT_FILES_FULL_QUERY, variables={"id": comment.relay_id}) + content = response.json() + + data = content["data"]["node"] + assert data["filesCount"]["total"] == 1 + assert data["isFilesEnabled"] is True + assert len(data["files"]["edges"]) == 1 + assert data["files"]["edges"][0]["node"]["id"] == file.relay_id + + +def test_comment_files_interface_multiple_files(graphql_client): + comment = CommentFactory() + comment_content_type = ContentType.objects.get_for_model(comment) + comment_document_id, _ = DocumentId.objects.get_or_create( + content_type=comment_content_type, + object_id=comment.pk, + ) + + for i in range(5): + File.objects.create( + parent=comment_document_id, + file_name=f"file_{i}.txt", + ) + + response = graphql_client(COMMENT_FILES_QUERY, variables={"id": comment.relay_id}) + content = response.json() + + edges = content["data"]["node"]["files"]["edges"] + assert len(edges) == 5 + + +FILE_ATTACH_TO_TARGET_MUTATION = """ + mutation FileAttachToTarget($fileRelayIds: [ID]!, $targetObjectId: ID!) { + fileAttachToTarget(input: { + fileRelayIds: $fileRelayIds + targetObjectId: $targetObjectId + }) { + attachedFiles { + node { + id + fileName + } + } + target { + ... on FilesInterface { + filesCount + isFilesEnabled + } + } + } + } +""" + + +def test_file_attach_to_target_success(graphql_user_client, django_user_client): + user = django_user_client.user + comment = CommentFactory(user=user) + + # Create standalone files (no parent) + file1 = File.objects.create( + file_name="file1.txt", + file_size=1024, + upload_status=File.UploadStatus.COMPLETED, + created_by=user, + ) + file2 = File.objects.create( + file_name="file2.txt", + file_size=2048, + upload_status=File.UploadStatus.COMPLETED, + created_by=user, + ) + + response = graphql_user_client( + FILE_ATTACH_TO_TARGET_MUTATION, + variables={ + "fileRelayIds": [file1.relay_id, file2.relay_id], + "targetObjectId": comment.relay_id, + }, + ) + content = response.json() + + assert "errors" not in content + data = content["data"]["fileAttachToTarget"] + + # Check attached files + assert len(data["attachedFiles"]) == 2 + assert data["attachedFiles"][0]["node"]["id"] == file1.relay_id + assert data["attachedFiles"][0]["node"]["fileName"] == "file1.txt" + assert data["attachedFiles"][1]["node"]["id"] == file2.relay_id + + # Check target + assert data["target"]["filesCount"]["total"] == 2 + assert data["target"]["isFilesEnabled"] is True + + # Verify files are attached in database + file1.refresh_from_db() + file2.refresh_from_db() + comment_ct = ContentType.objects.get_for_model(comment) + comment_doc_id = DocumentId.objects.get(content_type=comment_ct, object_id=comment.pk) + assert file1.parent == comment_doc_id + assert file2.parent == comment_doc_id + + +def test_file_attach_to_target_single_file(graphql_user_client, django_user_client): + user = django_user_client.user + comment = CommentFactory(user=user) + + file = File.objects.create( + file_name="single.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=user, + ) + + response = graphql_user_client( + FILE_ATTACH_TO_TARGET_MUTATION, + variables={ + "fileRelayIds": [file.relay_id], + "targetObjectId": comment.relay_id, + }, + ) + content = response.json() + + assert "errors" not in content + data = content["data"]["fileAttachToTarget"] + assert len(data["attachedFiles"]) == 1 + assert data["target"]["filesCount"]["total"] == 1 + + +def test_file_attach_to_target_requires_authentication(graphql_client): + comment = CommentFactory() + file = File.objects.create(file_name="test.txt") + + response = graphql_client( + FILE_ATTACH_TO_TARGET_MUTATION, + variables={ + "fileRelayIds": [file.relay_id], + "targetObjectId": comment.relay_id, + }, + ) + content = response.json() + + assert "errors" in content + assert content["errors"][0]["extensions"]["code"] == "authentication_required" + + +def test_file_attach_to_target_requires_ownership(graphql_user_client, django_user_client): + user = django_user_client.user + comment = CommentFactory(user=user) + + # Create file owned by different user + from django.contrib.auth import get_user_model + + User = get_user_model() + other_user = User.objects.create_user(email="other@test.com", password="pass123") + file = File.objects.create( + file_name="test.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=other_user, + ) + + response = graphql_user_client( + FILE_ATTACH_TO_TARGET_MUTATION, + variables={ + "fileRelayIds": [file.relay_id], + "targetObjectId": comment.relay_id, + }, + ) + content = response.json() + + assert "errors" in content + assert content["errors"][0]["extensions"]["code"] == "permission_required" + + +def test_file_attach_to_target_already_attached(graphql_user_client, django_user_client): + user = django_user_client.user + comment1 = CommentFactory(user=user) + comment2 = CommentFactory(user=user) + + comment1_ct = ContentType.objects.get_for_model(comment1) + comment1_doc_id, _ = DocumentId.objects.get_or_create( + content_type=comment1_ct, + object_id=comment1.pk, + ) + + # Create file already attached to comment1 + file = File.objects.create( + file_name="attached.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=user, + parent=comment1_doc_id, + ) + + # Try to attach to comment2 + response = graphql_user_client( + FILE_ATTACH_TO_TARGET_MUTATION, + variables={ + "fileRelayIds": [file.relay_id], + "targetObjectId": comment2.relay_id, + }, + ) + content = response.json() + + assert "errors" in content + assert content["errors"][0]["extensions"]["code"] == "already_attached" + + +def test_file_attach_to_target_empty_list(graphql_user_client, django_user_client): + user = django_user_client.user + comment = CommentFactory(user=user) + + response = graphql_user_client( + FILE_ATTACH_TO_TARGET_MUTATION, + variables={ + "fileRelayIds": [], + "targetObjectId": comment.relay_id, + }, + ) + content = response.json() + + assert "errors" in content + assert content["errors"][0]["extensions"]["code"] == "invalid_input" + + +def test_file_attach_to_target_invalid_file_id(graphql_user_client, django_user_client): + user = django_user_client.user + comment = CommentFactory(user=user) + + response = graphql_user_client( + FILE_ATTACH_TO_TARGET_MUTATION, + variables={ + "fileRelayIds": ["RmlsZU9iamVjdFR5cGU6OTk5OTk="], # Non-existent file ID + "targetObjectId": comment.relay_id, + }, + ) + content = response.json() + + assert "errors" in content + # Error might not have extensions if it's a relay ID parsing error + if "extensions" in content["errors"][0]: + assert content["errors"][0]["extensions"]["code"] == "not_found" + + +FILE_DELETE_MUTATION = """ + mutation FileDeleteMutation($input: FileDeleteInput!) { + fileDelete(input: $input) { + deletedId + parent { + ... on FilesInterface { + filesCount + } + } + errors { + field + messages + } + } + } +""" + + +def test_anon_cant_delete_file(graphql_client, django_user_client): + user = django_user_client.user + file = File.objects.create( + file_name="test.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=user, + ) + + response = graphql_client( + FILE_DELETE_MUTATION, + variables={"input": {"id": file.relay_id}}, + ) + content = response.json() + assert content["errors"][0]["message"] == "authentication required" + assert File.objects.count() == 1 + + +def test_user_cant_delete_any_file(graphql_user_client, django_user_client): + from django.contrib.auth import get_user_model + + User = get_user_model() + other_user = User.objects.create_user(email="other@test.com", password="pass123") + + file = File.objects.create( + file_name="test.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=other_user, + ) + + response = graphql_user_client( + FILE_DELETE_MUTATION, + variables={"input": {"id": file.relay_id}}, + ) + content = response.json() + assert content["errors"][0]["extensions"]["code"] == "permission_required" + assert File.objects.count() == 1 + + +def test_owner_can_delete_file(django_user_client, graphql_user_client): + user = django_user_client.user + file = File.objects.create( + file_name="test.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=user, + ) + + response = graphql_user_client( + FILE_DELETE_MUTATION, + variables={"input": {"id": file.relay_id}}, + ) + content = response.json() + assert content["data"]["fileDelete"]["deletedId"] == file.relay_id + assert File.objects.count() == 0 + + +def test_superuser_can_delete_file(django_user_client, graphql_user_client): + from django.contrib.auth import get_user_model + + User = get_user_model() + other_user = User.objects.create_user(email="other@test.com", password="pass123") + + django_user_client.user.is_superuser = True + django_user_client.user.save() + + file = File.objects.create( + file_name="test.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=other_user, + ) + + response = graphql_user_client( + FILE_DELETE_MUTATION, + variables={"input": {"id": file.relay_id}}, + ) + content = response.json() + assert content["data"]["fileDelete"]["deletedId"] == file.relay_id + assert File.objects.count() == 0 + + +def test_user_with_permission_can_delete_file(django_user_client, graphql_user_client): + from django.contrib.auth import get_user_model + from django.contrib.auth.models import Permission + + User = get_user_model() + other_user = User.objects.create_user(email="other@test.com", password="pass123") + + app_label = File._meta.app_label + perm = Permission.objects.get(content_type__app_label=app_label, codename="delete_file") + django_user_client.user.user_permissions.add(perm) + + file = File.objects.create( + file_name="test.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=other_user, + ) + + response = graphql_user_client( + FILE_DELETE_MUTATION, + variables={"input": {"id": file.relay_id}}, + ) + content = response.json() + assert content["data"]["fileDelete"]["deletedId"] == file.relay_id + assert File.objects.count() == 0 + + +def test_update_files_count_after_delete_file(django_user_client, graphql_user_client): + user = django_user_client.user + comment = CommentFactory(user=user) + comment_content_type = ContentType.objects.get_for_model(comment) + comment_document_id, _ = DocumentId.objects.get_or_create( + content_type=comment_content_type, + object_id=comment.pk, + ) + + file = File.objects.create( + file_name="test.txt", + file_content_type="text/plain", + upload_status=File.UploadStatus.COMPLETED, + created_by=user, + parent=comment_document_id, + ) + + # Refresh to get updated files_count + comment.refresh_from_db() + assert comment.files_count["total"] == 1 + assert comment.files_count["text/plain"] == 1 + + response = graphql_user_client( + FILE_DELETE_MUTATION, + variables={"input": {"id": file.relay_id}}, + ) + content = response.json() + + assert content["data"]["fileDelete"]["parent"]["filesCount"]["total"] == 0 + assert File.objects.count() == 0 + + comment.refresh_from_db() + assert comment.files_count["total"] == 0 diff --git a/baseapp/files/tests/test_models.py b/baseapp/files/tests/test_models.py new file mode 100644 index 00000000..8d21731c --- /dev/null +++ b/baseapp/files/tests/test_models.py @@ -0,0 +1,288 @@ +import swapper +from django.contrib.contenttypes.models import ContentType +from django.core.exceptions import ValidationError +from django.test import TestCase + +from baseapp.files.utils import ( + default_files_count, + get_or_create_file_target, + recalculate_files_count, + set_files_parent, +) +from baseapp_comments.tests.factories import CommentFactory +from baseapp_core.models import DocumentId + +File = swapper.load_model("baseapp_files", "File") +FileTarget = swapper.load_model("baseapp_files", "FileTarget") +Comment = swapper.load_model("baseapp_comments", "Comment") + + +def get_or_create_document_id(obj): + """Helper to get or create a DocumentId for an object.""" + ct = ContentType.objects.get_for_model(obj, for_concrete_model=False) + document_id, _ = DocumentId.objects.get_or_create( + content_type=ct, + object_id=obj.pk, + ) + return document_id + + +class FileModelTest(TestCase): + def test_create_file(self): + File.objects.create() + self.assertEqual(File.objects.count(), 1) + + def test_comment_files_count(self): + comment = CommentFactory() + parent_doc_id = get_or_create_document_id(comment) + + File.objects.create(parent=parent_doc_id) + + self.assertEqual(comment.files.count(), 1) + + +class FileTargetTest(TestCase): + def test_file_target_creation(self): + comment = CommentFactory() + file_target = comment.get_file_target() + + self.assertIsNotNone(file_target) + self.assertEqual(file_target.target.object_id, comment.pk) + self.assertTrue(file_target.is_files_enabled) + self.assertEqual(file_target.files_count, {"total": 0}) + + def test_file_target_str_method(self): + comment = CommentFactory() + file_target = comment.get_file_target() + + expected_str = f"FileTarget for {file_target.target}" + self.assertEqual(str(file_target), expected_str) + + def test_file_target_unique_constraint(self): + comment = CommentFactory() + doc_id = get_or_create_document_id(comment) + + FileTarget.objects.create(target=doc_id) + + file_target2, created = FileTarget.objects.get_or_create(target=doc_id) + + self.assertFalse(created) + self.assertEqual(FileTarget.objects.count(), 1) + + def test_file_target_files_count_update_single_type(self): + comment = CommentFactory() + parent_doc_id = get_or_create_document_id(comment) + + File.objects.create( + parent=parent_doc_id, + file_content_type="image/png", + ) + + file_target = comment.get_file_target() + self.assertEqual(file_target.files_count["total"], 1) + self.assertEqual(file_target.files_count.get("image/png"), 1) + + def test_file_target_files_count_update_multiple_types(self): + comment = CommentFactory() + parent_doc_id = get_or_create_document_id(comment) + + File.objects.create( + parent=parent_doc_id, + file_content_type="image/png", + ) + File.objects.create( + parent=parent_doc_id, + file_content_type="image/jpeg", + ) + File.objects.create( + parent=parent_doc_id, + file_content_type="image/png", + ) + + file_target = comment.get_file_target() + self.assertEqual(file_target.files_count["total"], 3) + self.assertEqual(file_target.files_count.get("image/png"), 2) + self.assertEqual(file_target.files_count.get("image/jpeg"), 1) + + def test_file_target_is_files_enabled_can_be_disabled(self): + comment = CommentFactory() + file_target = comment.get_file_target() + + file_target.is_files_enabled = False + file_target.save() + + self.assertFalse(comment.is_files_enabled) + + def test_cannot_attach_file_when_files_disabled(self): + comment = CommentFactory() + parent_doc_id = get_or_create_document_id(comment) + + # Create FileTarget with is_files_enabled=False + file_target = comment.get_file_target() + file_target.is_files_enabled = False + file_target.save() + + # Try to create a file for this comment + with self.assertRaises(ValidationError) as cm: + File.objects.create( + parent=parent_doc_id, + file_content_type="image/png", + ) + + self.assertEqual(cm.exception.code, "files_disabled") + self.assertIn("Files are not enabled", str(cm.exception)) + + def test_can_attach_file_when_files_enabled(self): + comment = CommentFactory() + parent_doc_id = get_or_create_document_id(comment) + + # Create FileTarget with is_files_enabled=True (default) + file_target = comment.get_file_target() + self.assertTrue(file_target.is_files_enabled) + + # This should work without raising an error + file_obj = File.objects.create( + parent=parent_doc_id, + file_content_type="image/png", + ) + + self.assertIsNotNone(file_obj.pk) + self.assertEqual(file_obj.parent.object_id, comment.pk) + + def test_fileable_model_properties(self): + comment = CommentFactory() + + self.assertTrue(comment.is_files_enabled) + self.assertEqual(comment.files_count, {"total": 0}) + + def test_fileable_model_properties_multiple_accesses(self): + comment = CommentFactory() + + files_count_1 = comment.files_count + files_count_2 = comment.files_count + + self.assertEqual(files_count_1, files_count_2) + self.assertEqual(FileTarget.objects.count(), 1) + + +class FileUtilsTest(TestCase): + def test_default_files_count(self): + result = default_files_count() + self.assertEqual(result, {"total": 0}) + + def test_get_or_create_file_target_creates_new(self): + comment = CommentFactory() + + initial_count = FileTarget.objects.count() + file_target = get_or_create_file_target(comment) + + self.assertEqual(FileTarget.objects.count(), initial_count + 1) + self.assertEqual(file_target.target.object_id, comment.pk) + + def test_get_or_create_file_target_gets_existing(self): + comment = CommentFactory() + + file_target1 = get_or_create_file_target(comment) + file_target2 = get_or_create_file_target(comment) + + self.assertEqual(file_target1.pk, file_target2.pk) + self.assertEqual(FileTarget.objects.count(), 1) + + def test_recalculate_files_count_with_none(self): + recalculate_files_count(None) + + def test_recalculate_files_count_empty(self): + comment = CommentFactory() + recalculate_files_count(comment) + + file_target = comment.get_file_target() + self.assertEqual(file_target.files_count, {"total": 0}) + + def test_recalculate_files_count_with_files(self): + comment = CommentFactory() + parent_doc_id = get_or_create_document_id(comment) + + File.objects.create( + parent=parent_doc_id, + file_content_type="image/png", + ) + + recalculate_files_count(comment) + + file_target = comment.get_file_target() + self.assertEqual(file_target.files_count["total"], 1) + + def test_set_files_parent_with_empty_list(self): + comment = CommentFactory() + set_files_parent(comment, []) + + def test_set_files_parent_with_none(self): + comment = CommentFactory() + set_files_parent(comment, None) + + def test_set_files_parent_moves_file(self): + comment1 = CommentFactory() + comment2 = CommentFactory() + parent_doc_id1 = get_or_create_document_id(comment1) + + file_obj = File.objects.create( + parent=parent_doc_id1, + file_content_type="image/png", + ) + + set_files_parent(comment2, [file_obj]) + + file_obj.refresh_from_db() + self.assertEqual(file_obj.parent.object_id, comment2.pk) + + file_target1 = comment1.get_file_target() + file_target2 = comment2.get_file_target() + self.assertEqual(file_target1.files_count["total"], 0) + self.assertEqual(file_target2.files_count["total"], 1) + + def test_set_files_parent_same_parent(self): + comment = CommentFactory() + parent_doc_id = get_or_create_document_id(comment) + + file_obj = File.objects.create(parent=parent_doc_id) + + set_files_parent(comment, [file_obj]) + + file_obj.refresh_from_db() + self.assertEqual(file_obj.parent.object_id, comment.pk) + + +class FileTargetSignalsTest(TestCase): + def test_file_creation_updates_file_target(self): + comment = CommentFactory() + comment_content_type = ContentType.objects.get_for_model(Comment) + parent_doc_id = get_or_create_document_id(comment) + + File.objects.create( + parent=parent_doc_id, + file_content_type="image/png", + ) + + file_target = FileTarget.objects.get( + target__content_type=comment_content_type, + target__object_id=comment.pk, + ) + self.assertEqual(file_target.files_count["total"], 1) + + def test_file_deletion_updates_file_target(self): + comment = CommentFactory() + comment_content_type = ContentType.objects.get_for_model(Comment) + parent_doc_id = get_or_create_document_id(comment) + + file_obj = File.objects.create( + parent=parent_doc_id, + file_content_type="image/png", + ) + + file_obj.delete() + + file_target = FileTarget.objects.get( + target__content_type=comment_content_type, + target__object_id=comment.pk, + ) + self.assertEqual(file_target.files_count["total"], 0) diff --git a/baseapp/files/tests/test_rest_api.py b/baseapp/files/tests/test_rest_api.py new file mode 100644 index 00000000..326b2687 --- /dev/null +++ b/baseapp/files/tests/test_rest_api.py @@ -0,0 +1,577 @@ +from datetime import timedelta +from unittest.mock import MagicMock, patch + +import pytest +import swapper +from django.contrib.auth import get_user_model +from django.contrib.contenttypes.models import ContentType +from django.utils import timezone +from rest_framework import status + +from baseapp_core.models import DocumentId + +File = swapper.load_model("baseapp_files", "File") +User = get_user_model() + + +@pytest.fixture +def mock_s3_handler(): + """Mock S3 upload handler to avoid actual S3 calls.""" + with patch("baseapp.files.services.upload_service.get_upload_handler") as mock_get_handler: + mock_handler = MagicMock() + mock_handler.supports_multipart.return_value = True + mock_handler.initiate_upload.return_value = { + "upload_id": "test-upload-id-123", + "presigned_urls": [ + {"part_number": 1, "url": "https://s3.amazonaws.com/bucket/part1?signature=abc"}, + {"part_number": 2, "url": "https://s3.amazonaws.com/bucket/part2?signature=def"}, + ], + "expires_in": 3600, + } + mock_handler.complete_upload.return_value = "files/test-file.mp4" + mock_handler.abort_upload.return_value = None + mock_get_handler.return_value = mock_handler + yield mock_handler + + +@pytest.mark.django_db +class TestFileUploadInitiation: + """Tests for POST /v1/files/uploads/ (initiate upload).""" + + def test_initiate_upload_success(self, user_client, mock_s3_handler): + """Test successful upload initiation.""" + response = user_client.post( + "/v1/files/uploads", + { + "file_name": "test-video.mp4", + "file_size": 10485760, # 10MB + "file_content_type": "video/mp4", + "num_parts": 2, + "part_size": 5242880, # 5MB + }, + format="json", + ) + + assert response.status_code == status.HTTP_201_CREATED + data = response.json() + + # Check response structure + assert "id" in data + assert data["upload_id"] == "test-upload-id-123" + assert len(data["presigned_urls"]) == 2 + assert data["presigned_urls"][0]["part_number"] == 1 + assert data["expires_in"] == 3600 + assert data["upload_status"] == File.UploadStatus.UPLOADING + + # Check file was created in database (id is now public_id UUID) + file_obj = File.get_by_public_id(data["id"]) + assert file_obj.file_name == "test-video.mp4" + assert file_obj.file_size == 10485760 + assert file_obj.upload_status == File.UploadStatus.UPLOADING + assert file_obj.total_parts == 2 + assert file_obj.created_by == user_client.user + + # Verify S3 handler was called + mock_s3_handler.initiate_upload.assert_called_once() + + def test_initiate_upload_with_parent(self, user_client, mock_s3_handler): + """Test upload initiation with parent object.""" + # Get or create DocumentId for the user + user_ct = ContentType.objects.get_for_model(User) + document_id, _ = DocumentId.objects.get_or_create( + content_type=user_ct, + object_id=user_client.user.id, + ) + + response = user_client.post( + "/v1/files/uploads", + { + "file_name": "avatar.jpg", + "file_size": 1048576, # 1MB + "file_content_type": "image/jpeg", + "num_parts": 1, + "part_size": 1048576, + "parent_id": str(document_id.public_id), + }, + format="json", + ) + + # Print response for debugging if it fails + if response.status_code != status.HTTP_201_CREATED: + print(f"Response status: {response.status_code}") + print(f"Response data: {response.json()}") + + assert response.status_code == status.HTTP_201_CREATED + data = response.json() + + file_obj = File.get_by_public_id(data["id"]) + # Parent is now a ForeignKey to DocumentId + assert file_obj.parent is not None + assert file_obj.parent.content_type_id == user_ct.id + assert file_obj.parent.object_id == user_client.user.id + + def test_initiate_upload_requires_authentication(self, client): + """Test that upload initiation requires authentication.""" + response = client.post( + "/v1/files/uploads", + { + "file_name": "test.mp4", + "file_size": 1048576, + "file_content_type": "video/mp4", + "num_parts": 1, + "part_size": 1048576, + }, + format="json", + ) + + assert response.status_code == status.HTTP_401_UNAUTHORIZED + + def test_initiate_upload_invalid_part_size(self, user_client, mock_s3_handler): + """Test validation for part size (must be at least 5MB for multipart).""" + response = user_client.post( + "/v1/files/uploads", + { + "file_name": "test.mp4", + "file_size": 10485760, + "file_content_type": "video/mp4", + "num_parts": 3, + "part_size": 1048576, # Too small (1MB) + }, + format="json", + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + def test_initiate_upload_file_size_mismatch(self, user_client, mock_s3_handler): + """Test validation for file size matching parts.""" + response = user_client.post( + "/v1/files/uploads", + { + "file_name": "test.mp4", + "file_size": 100000, # 100KB - doesn't match parts + "file_content_type": "video/mp4", + "num_parts": 2, + "part_size": 5242880, # 5MB + }, + format="json", + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + response_data = response.json() + # DRF returns validation errors as dict with field names or non_field_errors + error_message = str(response_data) + assert "doesn't match" in error_message + + def test_initiate_upload_too_many_parts(self, user_client, mock_s3_handler): + """Test validation for maximum parts (10,000).""" + response = user_client.post( + "/v1/files/uploads", + { + "file_name": "test.mp4", + "file_size": 52428800000, # 50GB + "file_content_type": "video/mp4", + "num_parts": 10001, # Too many + "part_size": 5242880, + }, + format="json", + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + def test_initiate_upload_invalid_parent_id(self, user_client, mock_s3_handler): + """Test that invalid parent_id returns error.""" + import uuid + + response = user_client.post( + "/v1/files/uploads", + { + "file_name": "test.mp4", + "file_size": 1048576, + "file_content_type": "video/mp4", + "num_parts": 1, + "part_size": 1048576, + "parent_id": str(uuid.uuid4()), # Non-existent DocumentId + }, + format="json", + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + +@pytest.mark.django_db +class TestFileUploadCompletion: + """Tests for POST /v1/files/uploads/{id}/complete.""" + + @pytest.fixture + def pending_upload(self, user_client): + """Create a pending upload for testing.""" + return File.objects.create( + file_name="test.mp4", + file_size=10485760, + file_content_type="video/mp4", + upload_status=File.UploadStatus.UPLOADING, + upload_id="test-upload-id", + total_parts=2, + created_by=user_client.user, + upload_expires_at=timezone.now() + timedelta(hours=24), + ) + + def test_complete_upload_success(self, user_client, pending_upload, mock_s3_handler): + """Test successful upload completion.""" + response = user_client.post( + f"/v1/files/uploads/{pending_upload.public_id}/complete", + { + "parts": [ + {"part_number": 1, "etag": "abc123"}, + {"part_number": 2, "etag": "def456"}, + ] + }, + format="json", + ) + + assert response.status_code == status.HTTP_200_OK + data = response.json() + + assert data["upload_status"] == File.UploadStatus.COMPLETED + assert "url" in data + + # Check file was updated in database + pending_upload.refresh_from_db() + assert pending_upload.upload_status == File.UploadStatus.COMPLETED + assert pending_upload.upload_id is None + assert pending_upload.uploaded_parts == {"1": "abc123", "2": "def456"} + + # Verify S3 handler was called + mock_s3_handler.complete_upload.assert_called_once() + + def test_complete_upload_wrong_number_of_parts( + self, user_client, pending_upload, mock_s3_handler + ): + """Test validation for incorrect number of parts.""" + response = user_client.post( + f"/v1/files/uploads/{pending_upload.public_id}/complete", + { + "parts": [ + {"part_number": 1, "etag": "abc123"}, + # Missing part 2 + ] + }, + format="json", + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert "Expected 2 parts" in str(response.json()) + + def test_complete_upload_missing_part_number( + self, user_client, pending_upload, mock_s3_handler + ): + """Test validation for missing part numbers.""" + response = user_client.post( + f"/v1/files/uploads/{pending_upload.public_id}/complete", + { + "parts": [ + {"part_number": 1, "etag": "abc123"}, + {"part_number": 3, "etag": "ghi789"}, # Skipped part 2 + ] + }, + format="json", + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert "Missing or duplicate" in str(response.json()) + + def test_complete_upload_requires_ownership(self, client, pending_upload, mock_s3_handler): + """Test that only the file owner can complete upload.""" + other_user = User.objects.create_user(email="other@example.com", password="pass123") + client.force_authenticate(user=other_user) + + response = client.post( + f"/v1/files/uploads/{pending_upload.public_id}/complete", + { + "parts": [ + {"part_number": 1, "etag": "abc123"}, + {"part_number": 2, "etag": "def456"}, + ] + }, + format="json", + ) + + assert response.status_code == status.HTTP_403_FORBIDDEN + + def test_complete_upload_invalid_status(self, user_client, mock_s3_handler): + """Test that completed uploads cannot be completed again.""" + completed_file = File.objects.create( + file_name="test.mp4", + file_size=10485760, + file_content_type="video/mp4", + upload_status=File.UploadStatus.COMPLETED, + created_by=user_client.user, + ) + + response = user_client.post( + f"/v1/files/uploads/{completed_file.public_id}/complete", + { + "parts": [ + {"part_number": 1, "etag": "abc123"}, + ] + }, + format="json", + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + def test_complete_upload_invalid_part_structure( + self, user_client, pending_upload, mock_s3_handler + ): + """Test validation for part structure.""" + response = user_client.post( + f"/v1/files/uploads/{pending_upload.public_id}/complete", + { + "parts": [ + {"part_number": 1}, # Missing etag + {"etag": "def456"}, # Missing part_number + ] + }, + format="json", + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + +@pytest.mark.django_db +class TestFileUploadAbort: + """Tests for DELETE /v1/files/uploads/{id}.""" + + @pytest.fixture + def uploading_file(self, user_client): + """Create an uploading file for testing.""" + return File.objects.create( + file_name="test.mp4", + file_size=10485760, + file_content_type="video/mp4", + upload_status=File.UploadStatus.UPLOADING, + upload_id="test-upload-id", + total_parts=2, + created_by=user_client.user, + ) + + def test_abort_upload_success(self, user_client, uploading_file, mock_s3_handler): + """Test successful upload abort.""" + response = user_client.delete(f"/v1/files/uploads/{uploading_file.public_id}") + + assert response.status_code == status.HTTP_204_NO_CONTENT + + # Check file was updated + uploading_file.refresh_from_db() + assert uploading_file.upload_status == File.UploadStatus.ABORTED + assert uploading_file.upload_id is None + + # Verify S3 handler was called + mock_s3_handler.abort_upload.assert_called_once_with(uploading_file, "test-upload-id") + + def test_abort_upload_requires_ownership(self, client, uploading_file, mock_s3_handler): + """Test that only file owner can abort upload.""" + other_user = User.objects.create_user(email="other@example.com", password="pass123") + client.force_authenticate(user=other_user) + + response = client.delete(f"/v1/files/uploads/{uploading_file.public_id}") + + assert response.status_code == status.HTTP_403_FORBIDDEN + + def test_abort_upload_without_upload_id(self, user_client, mock_s3_handler): + """Test aborting upload without upload_id still marks as aborted.""" + file_obj = File.objects.create( + file_name="test.mp4", + file_size=10485760, + file_content_type="video/mp4", + upload_status=File.UploadStatus.PENDING, + upload_id=None, # No upload_id + total_parts=2, + created_by=user_client.user, + ) + + response = user_client.delete(f"/v1/files/uploads/{file_obj.public_id}") + + assert response.status_code == status.HTTP_204_NO_CONTENT + + file_obj.refresh_from_db() + assert file_obj.upload_status == File.UploadStatus.ABORTED + + +@pytest.mark.django_db +class TestFileCRUD: + """Tests for file CRUD endpoints.""" + + @pytest.fixture + def completed_file(self, user_client): + """Create a completed file for testing.""" + return File.objects.create( + file_name="test.mp4", + file_size=10485760, + file_content_type="video/mp4", + upload_status=File.UploadStatus.COMPLETED, + created_by=user_client.user, + ) + + def test_list_files(self, user_client, completed_file): + """Test listing files.""" + response = user_client.get("/v1/files") + + assert response.status_code == status.HTTP_200_OK + data = response.json() + + assert "results" in data + assert len(data["results"]) == 1 + assert str(data["results"][0]["id"]) == str(completed_file.public_id) + + def test_list_files_filter_by_status(self, user_client): + """Test filtering files by status.""" + File.objects.create( + file_name="completed.mp4", + file_size=100, + file_content_type="video/mp4", + upload_status=File.UploadStatus.COMPLETED, + created_by=user_client.user, + ) + File.objects.create( + file_name="uploading.mp4", + file_size=100, + file_content_type="video/mp4", + upload_status=File.UploadStatus.UPLOADING, + created_by=user_client.user, + ) + + # Default: only completed + response = user_client.get("/v1/files") + assert len(response.json()["results"]) == 1 + + # Filter by uploading + response = user_client.get(f"/v1/files?status={File.UploadStatus.UPLOADING}") + assert len(response.json()["results"]) == 1 + assert response.json()["results"][0]["upload_status"] == File.UploadStatus.UPLOADING + + def test_list_files_only_own_files(self, user_client): + """Test that users only see their own files.""" + other_user = User.objects.create_user(email="other@example.com", password="pass123") + + # Create file for other user + File.objects.create( + file_name="other.mp4", + file_size=100, + file_content_type="video/mp4", + upload_status=File.UploadStatus.COMPLETED, + created_by=other_user, + ) + + response = user_client.get("/v1/files") + + assert response.status_code == status.HTTP_200_OK + assert len(response.json()["results"]) == 0 + + def test_retrieve_file(self, user_client, completed_file): + """Test retrieving a single file.""" + response = user_client.get(f"/v1/files/{completed_file.public_id}") + + assert response.status_code == status.HTTP_200_OK + data = response.json() + + assert str(data["id"]) == str(completed_file.public_id) + assert data["file_name"] == "test.mp4" + assert data["upload_status"] == File.UploadStatus.COMPLETED + + def test_update_file_metadata(self, user_client, completed_file): + """Test updating file metadata.""" + response = user_client.patch( + f"/v1/files/{completed_file.public_id}", + {"name": "My Video", "description": "Test description"}, + format="json", + ) + + assert response.status_code == status.HTTP_200_OK + + completed_file.refresh_from_db() + assert completed_file.name == "My Video" + assert completed_file.description == "Test description" + + def test_delete_file(self, user_client, completed_file): + """Test deleting a file.""" + response = user_client.delete(f"/v1/files/{completed_file.public_id}") + + assert response.status_code == status.HTTP_204_NO_CONTENT + assert not File.objects.filter(id=completed_file.id).exists() + + +@pytest.mark.django_db +class TestSetParent: + """Tests for POST /v1/files/{id}/set-parent.""" + + @pytest.fixture + def standalone_file(self, user_client): + """Create a standalone file without parent.""" + return File.objects.create( + file_name="test.mp4", + file_size=10485760, + file_content_type="video/mp4", + upload_status=File.UploadStatus.COMPLETED, + created_by=user_client.user, + ) + + def test_set_parent_success(self, user_client, standalone_file): + """Test setting parent on standalone file.""" + user_ct = ContentType.objects.get_for_model(User) + document_id, _ = DocumentId.objects.get_or_create( + content_type=user_ct, + object_id=user_client.user.id, + ) + + response = user_client.post( + f"/v1/files/{standalone_file.public_id}/set-parent", + { + "parent_id": str(document_id.public_id), + }, + format="json", + ) + + assert response.status_code == status.HTTP_200_OK + + standalone_file.refresh_from_db() + # Parent is now a ForeignKey to DocumentId + assert standalone_file.parent is not None + assert standalone_file.parent.content_type_id == user_ct.id + assert standalone_file.parent.object_id == user_client.user.id + + def test_set_parent_invalid_parent_id(self, user_client, standalone_file): + """Test setting invalid parent_id.""" + import uuid + + response = user_client.post( + f"/v1/files/{standalone_file.public_id}/set-parent", + { + "parent_id": str(uuid.uuid4()), # Non-existent DocumentId + }, + format="json", + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + def test_set_parent_requires_ownership(self, client, standalone_file): + """Test that only file owner can set parent.""" + other_user = User.objects.create_user(email="other@example.com", password="pass123") + client.force_authenticate(user=other_user) + + user_ct = ContentType.objects.get_for_model(User) + document_id, _ = DocumentId.objects.get_or_create( + content_type=user_ct, + object_id=other_user.id, + ) + + response = client.post( + f"/v1/files/{standalone_file.public_id}/set-parent", + { + "parent_id": str(document_id.public_id), + }, + format="json", + ) + + # Returns 403 because permission check fails (user doesn't own the file) + assert response.status_code == status.HTTP_403_FORBIDDEN diff --git a/baseapp/files/tests/test_storage.py b/baseapp/files/tests/test_storage.py new file mode 100644 index 00000000..34344300 --- /dev/null +++ b/baseapp/files/tests/test_storage.py @@ -0,0 +1,280 @@ +from unittest.mock import MagicMock, patch + +import pytest +import swapper +from botocore.exceptions import ClientError +from django.conf import settings +from django.contrib.auth import get_user_model + +from baseapp.files.storage.base import BaseUploadHandler +from baseapp.files.storage.local import LocalUploadHandler +from baseapp.files.storage.s3 import S3MultipartUploadHandler + +File = swapper.load_model("baseapp_files", "File") +User = get_user_model() + + +@pytest.fixture +def user(db): + return User.objects.create_user( + email="test@example.com", + password="testpass123", + ) + + +@pytest.fixture +def file_obj(user): + """Create a file object for testing.""" + return File.objects.create( + file_name="test.mp4", + file_size=10485760, + file_content_type="video/mp4", + upload_status=File.UploadStatus.PENDING, + total_parts=2, + created_by=user, + ) + + +@pytest.mark.django_db +class TestBaseUploadHandler: + """Tests for the base upload handler interface.""" + + def test_base_handler_is_abstract(self): + """Test that base handler cannot be instantiated.""" + with pytest.raises(TypeError): + BaseUploadHandler() + + +@pytest.mark.django_db +@pytest.mark.skipif( + not hasattr(settings, "AWS_ACCESS_KEY_ID"), + reason="AWS settings not configured - skipping S3 tests", +) +class TestS3MultipartUploadHandler: + """Tests for S3 multipart upload handler.""" + + @pytest.fixture + def mock_s3_client(self): + """Mock boto3 S3 client.""" + with patch("baseapp.files.storage.s3.boto3.client") as mock_client: + s3 = MagicMock() + mock_client.return_value = s3 + + # Mock create_multipart_upload + s3.create_multipart_upload.return_value = {"UploadId": "test-upload-id-123"} + + # Mock generate_presigned_url + s3.generate_presigned_url.return_value = ( + "https://bucket.s3.amazonaws.com/test?signature=abc" + ) + + # Mock complete_multipart_upload + s3.complete_multipart_upload.return_value = { + "Location": "https://bucket.s3.amazonaws.com/files/test.mp4" + } + + yield s3 + + @pytest.fixture + def s3_handler(self, mock_s3_client): + """Create S3 handler with mocked client.""" + handler = S3MultipartUploadHandler() + handler.s3_client = mock_s3_client + yield handler + + def test_s3_handler_supports_multipart(self, s3_handler): + """Test that S3 handler supports multipart.""" + assert s3_handler.supports_multipart() is True + + def test_s3_initiate_upload(self, s3_handler, file_obj, mock_s3_client): + """Test S3 multipart upload initiation.""" + result = s3_handler.initiate_upload(file_obj, num_parts=2, part_size=5242880) + + assert result["upload_id"] == "test-upload-id-123" + assert len(result["presigned_urls"]) == 2 + assert result["presigned_urls"][0]["part_number"] == 1 + assert result["presigned_urls"][1]["part_number"] == 2 + assert result["expires_in"] == 3600 + + # Verify S3 calls + mock_s3_client.create_multipart_upload.assert_called_once() + assert mock_s3_client.generate_presigned_url.call_count == 2 + + def test_s3_initiate_upload_validates_part_count(self, s3_handler, file_obj): + """Test that S3 handler validates max part count.""" + with pytest.raises(ValueError, match="maximum 10,000 parts"): + s3_handler.initiate_upload(file_obj, num_parts=10001, part_size=5242880) + + def test_s3_initiate_upload_validates_part_size(self, s3_handler, file_obj): + """Test that S3 handler validates minimum part size.""" + with pytest.raises(ValueError, match="minimum 5MB"): + s3_handler.initiate_upload(file_obj, num_parts=2, part_size=1048576) + + def test_s3_complete_upload(self, s3_handler, file_obj, mock_s3_client): + """Test S3 multipart upload completion.""" + parts = [ + {"part_number": 1, "etag": "abc123"}, + {"part_number": 2, "etag": "def456"}, + ] + + result = s3_handler.complete_upload(file_obj, "test-upload-id", parts) + + assert result # Returns the S3 key + + # Verify S3 call + mock_s3_client.complete_multipart_upload.assert_called_once() + call_args = mock_s3_client.complete_multipart_upload.call_args + assert call_args.kwargs["UploadId"] == "test-upload-id" + assert len(call_args.kwargs["MultipartUpload"]["Parts"]) == 2 + + def test_s3_abort_upload(self, s3_handler, file_obj, mock_s3_client): + """Test S3 multipart upload abort.""" + s3_handler.abort_upload(file_obj, "test-upload-id") + + mock_s3_client.abort_multipart_upload.assert_called_once() + + def test_s3_abort_upload_handles_no_such_upload(self, s3_handler, file_obj, mock_s3_client): + """Test that abort handles NoSuchUpload error gracefully.""" + mock_s3_client.abort_multipart_upload.side_effect = ClientError( + {"Error": {"Code": "NoSuchUpload"}}, "abort_multipart_upload" + ) + + # Should not raise exception + s3_handler.abort_upload(file_obj, "test-upload-id") + + def test_s3_abort_upload_raises_other_errors(self, s3_handler, file_obj, mock_s3_client): + """Test that abort raises other errors.""" + mock_s3_client.abort_multipart_upload.side_effect = ClientError( + {"Error": {"Code": "AccessDenied"}}, "abort_multipart_upload" + ) + + with pytest.raises(ClientError): + s3_handler.abort_upload(file_obj, "test-upload-id") + + +@pytest.mark.django_db +class TestLocalUploadHandler: + """Tests for local upload handler.""" + + @pytest.fixture + def local_handler(self): + """Create local handler.""" + return LocalUploadHandler() + + @pytest.fixture + def temp_media_root(self, tmp_path): + """Use temporary directory for media root.""" + with patch.object(settings, "MEDIA_ROOT", str(tmp_path)): + yield tmp_path + + def test_local_handler_does_not_support_true_multipart(self, local_handler): + """Test that local handler doesn't support true multipart.""" + assert local_handler.supports_multipart() is False + + def test_local_initiate_upload(self, local_handler, file_obj, temp_media_root): + """Test local upload initiation.""" + result = local_handler.initiate_upload(file_obj, num_parts=2, part_size=5242880) + + assert "upload_id" in result + assert len(result["presigned_urls"]) == 2 + assert result["presigned_urls"][0]["part_number"] == 1 + assert "url" in result["presigned_urls"][0] + + # Check temp directory was created + temp_dir = temp_media_root / "temp_uploads" / result["upload_id"] + assert temp_dir.exists() + + def test_local_upload_part(self, local_handler, file_obj, temp_media_root): + """Test uploading a part to local storage.""" + upload_id = "test-upload-123" + data = b"test file content" + + etag = local_handler.upload_part(file_obj, upload_id, part_number=1, data=data) + + assert etag # Should return MD5 hash + assert len(etag) == 32 # MD5 hash length + + # Check part file was created + part_file = temp_media_root / "temp_uploads" / upload_id / "part_1" + assert part_file.exists() + assert part_file.read_bytes() == data + + def test_local_complete_upload(self, local_handler, file_obj, temp_media_root): + """Test completing local upload.""" + upload_id = "test-upload-123" + + # Create temp parts + temp_dir = temp_media_root / "temp_uploads" / upload_id + temp_dir.mkdir(parents=True) + + (temp_dir / "part_1").write_bytes(b"part 1 content") + (temp_dir / "part_2").write_bytes(b"part 2 content") + + parts = [ + {"part_number": 1, "etag": "abc"}, + {"part_number": 2, "etag": "def"}, + ] + + result = local_handler.complete_upload(file_obj, upload_id, parts) + + assert result # Returns file path + + # Check final file exists + final_file = temp_media_root / result + assert final_file.exists() + assert final_file.read_bytes() == b"part 1 contentpart 2 content" + + # Check temp directory was cleaned up + assert not temp_dir.exists() + + def test_local_abort_upload(self, local_handler, file_obj, temp_media_root): + """Test aborting local upload.""" + upload_id = "test-upload-123" + + # Create temp directory + temp_dir = temp_media_root / "temp_uploads" / upload_id + temp_dir.mkdir(parents=True) + (temp_dir / "part_1").write_bytes(b"test") + + # Abort + local_handler.abort_upload(file_obj, upload_id) + + # Check temp directory was cleaned up + assert not temp_dir.exists() + + +@pytest.mark.django_db +class TestStorageFactory: + """Tests for storage handler factory.""" + + @pytest.mark.skipif( + not hasattr(settings, "AWS_ACCESS_KEY_ID"), + reason="AWS settings not configured - skipping S3 factory test", + ) + def test_factory_returns_s3_handler_for_s3_storage(self): + """Test that factory returns S3 handler when using S3 storage.""" + with patch("baseapp.files.storage.default_storage") as mock_storage: + # Mock S3 storage + mock_storage.__class__.__name__ = "S3Boto3Storage" + + with patch("baseapp.files.storage.s3.S3MultipartUploadHandler") as mock_s3: + from baseapp.files.storage import get_upload_handler + + get_upload_handler() + + # Should instantiate S3 handler + mock_s3.assert_called_once() + + def test_factory_returns_local_handler_for_file_system_storage(self): + """Test that factory returns local handler for non-S3 storage.""" + with patch("baseapp.files.storage.default_storage") as mock_storage: + # Mock file system storage + mock_storage.__class__.__name__ = "FileSystemStorage" + + with patch("baseapp.files.storage.local.LocalUploadHandler") as mock_local: + from baseapp.files.storage import get_upload_handler + + get_upload_handler() + + # Should instantiate local handler + mock_local.assert_called_once() diff --git a/baseapp/files/utils.py b/baseapp/files/utils.py new file mode 100644 index 00000000..e2e4a401 --- /dev/null +++ b/baseapp/files/utils.py @@ -0,0 +1,73 @@ +from collections.abc import Iterable + +import swapper +from django.contrib.contenttypes.models import ContentType +from django.db import models + +from baseapp_core.models import DocumentId + + +def default_files_count(): + return {"total": 0} + + +def get_or_create_file_target(target_obj): + """Get or create a FileTarget for the given object.""" + FileTarget = swapper.load_model("baseapp_files", "FileTarget") + + content_type = ContentType.objects.get_for_model(target_obj, for_concrete_model=False) + document_id, created = DocumentId.objects.get_or_create( + content_type=content_type, + object_id=target_obj.pk, + ) + file_target, created = FileTarget.objects.get_or_create(target=document_id) + return file_target + + +def recalculate_files_count(parent): + if not parent: + return + + file_target = get_or_create_file_target(parent) + + files_count_qs = parent.files.values("file_content_type").annotate( + count=models.Count("file_content_type") + ) + file_target.files_count = default_files_count() + for item in files_count_qs: + count = item["count"] + file_target.files_count[item["file_content_type"]] = count + file_target.files_count["total"] += count + file_target.save(update_fields=["files_count"]) + + +def set_files_parent(parent, files: Iterable): + if not files: + return + + parent_content_type = ContentType.objects.get_for_model(parent, for_concrete_model=False) + parent_document_id, created = DocumentId.objects.get_or_create( + content_type=parent_content_type, + object_id=parent.pk, + ) + previous_parents = {} + + for file_obj in files: + previous_parent_doc_id = file_obj.parent + if previous_parent_doc_id and previous_parent_doc_id == parent_document_id: + continue + + if previous_parent_doc_id: + previous_parent = previous_parent_doc_id.content_object + key = (previous_parent_doc_id.content_type_id, previous_parent_doc_id.object_id) + if key not in previous_parents: + previous_parents[key] = previous_parent + + file_obj.parent = parent_document_id + file_obj.save(update_fields=["parent"]) + + for (content_type_id, object_id), previous_parent in previous_parents.items(): + if content_type_id == parent_content_type.pk and object_id == parent.pk: + continue + + recalculate_files_count(previous_parent) diff --git a/baseapp_auth/tests/integration/test_auth_without_baseapp_profiles.py b/baseapp_auth/tests/integration/test_auth_without_baseapp_profiles.py index 16b46fc7..7ffb7ab8 100644 --- a/baseapp_auth/tests/integration/test_auth_without_baseapp_profiles.py +++ b/baseapp_auth/tests/integration/test_auth_without_baseapp_profiles.py @@ -10,9 +10,7 @@ query { me { fullName - avatar(width: 32, height: 32) { - url - } + avatar(width: 32, height: 32) } } """ diff --git a/baseapp_chats/tests/test_graphql_mutations.py b/baseapp_chats/tests/test_graphql_mutations.py index d0c562b3..6aa2544c 100644 --- a/baseapp_chats/tests/test_graphql_mutations.py +++ b/baseapp_chats/tests/test_graphql_mutations.py @@ -57,9 +57,7 @@ } } } - image(width: 100, height: 100) { - url - } + image(width: 100, height: 100) allMessages { edges { node { @@ -98,9 +96,7 @@ } } } - image(width: 100, height: 100) { - url - } + image(width: 100, height: 100) } } removedParticipants { @@ -839,7 +835,7 @@ def test_user_can_create_group(django_user_client, graphql_user_client, image_dj assert content["data"]["chatRoomCreate"]["room"]["node"]["title"] == "group" assert content["data"]["chatRoomCreate"]["room"]["node"]["isGroup"] assert len(content["data"]["chatRoomCreate"]["room"]["node"]["participants"]["edges"]) == 3 - assert content["data"]["chatRoomCreate"]["room"]["node"]["image"]["url"].startswith( + assert content["data"]["chatRoomCreate"]["room"]["node"]["image"].startswith( "http://" # NOSONAR ) @@ -1096,7 +1092,7 @@ def test_admin_user_can_update_group_image( extra={"image": image_djangofile}, ) content = response.json() - assert content["data"]["chatRoomUpdate"]["room"]["node"]["image"]["url"].startswith( + assert content["data"]["chatRoomUpdate"]["room"]["node"]["image"].startswith( "http://" # NOSONAR ) diff --git a/baseapp_chats/tests/test_graphql_queries.py b/baseapp_chats/tests/test_graphql_queries.py index 295e06dc..ab6c354a 100644 --- a/baseapp_chats/tests/test_graphql_queries.py +++ b/baseapp_chats/tests/test_graphql_queries.py @@ -104,9 +104,7 @@ query GetRoom($roomId: ID!) { chatRoom(id: $roomId) { id - image(width: 100, height: 100) { - url - } + image(width: 100, height: 100) } } """ @@ -915,8 +913,8 @@ def test_resolve_image_for_1on1_chat_returns_other_participant_image( content = response.json() assert content["data"]["chatRoom"]["image"] is not None - assert content["data"]["chatRoom"]["image"]["url"] is not None - assert "100x100" in content["data"]["chatRoom"]["image"]["url"] + assert content["data"]["chatRoom"]["image"] is not None + assert "100x100" in content["data"]["chatRoom"]["image"] def test_resolve_image_for_group_chat_returns_group_image(django_client, image_djangofile): @@ -939,8 +937,8 @@ def test_resolve_image_for_group_chat_returns_group_image(django_client, image_d content = response.json() assert content["data"]["chatRoom"]["image"] is not None - assert content["data"]["chatRoom"]["image"]["url"] is not None - assert "100x100" in content["data"]["chatRoom"]["image"]["url"] + assert content["data"]["chatRoom"]["image"] is not None + assert "100x100" in content["data"]["chatRoom"]["image"] def test_resolve_image_returns_none_when_other_participant_has_no_image(django_client): diff --git a/baseapp_chats/tests/test_graphql_subscriptions.py b/baseapp_chats/tests/test_graphql_subscriptions.py index ed942077..a0906485 100644 --- a/baseapp_chats/tests/test_graphql_subscriptions.py +++ b/baseapp_chats/tests/test_graphql_subscriptions.py @@ -168,9 +168,7 @@ async def test_build_absolute_uri_on_graphql_subscription( message { node { profile { - image(width: 100, height: 100) { - url - } + image(width: 100, height: 100) } } } @@ -198,9 +196,9 @@ async def test_build_absolute_uri_on_graphql_subscription( # Check that subscription message were sent. resp = await client.receive(assert_id=sub_id, assert_type="next") - assert resp["data"]["chatRoomOnMessage"]["message"]["node"]["profile"]["image"][ - "url" - ].startswith("http") + assert resp["data"]["chatRoomOnMessage"]["message"]["node"]["profile"]["image"].startswith( + "http" + ) # Disconnect and wait the application to finish gracefully. await client.finalize() diff --git a/baseapp_comments/graphql/object_types.py b/baseapp_comments/graphql/object_types.py index 5d88bcb9..fee5946d 100644 --- a/baseapp_comments/graphql/object_types.py +++ b/baseapp_comments/graphql/object_types.py @@ -121,6 +121,7 @@ class Meta: "ReactionsInterface", "MentionsInterface", "NodeActivityLogInterface", + "FilesInterface", ) model = Comment fields = ( diff --git a/baseapp_comments/models.py b/baseapp_comments/models.py index 2c0d373d..8258dfe9 100644 --- a/baseapp_comments/models.py +++ b/baseapp_comments/models.py @@ -77,6 +77,12 @@ class Meta: comment_inheritances.append(ProfileMixin) +if apps.is_installed("baseapp.files"): + from baseapp.files.base import FileableModel + + comment_inheritances.append(FileableModel) + + class AbstractComment( *comment_inheritances, DocumentIdTargetMixin, diff --git a/baseapp_comments/tests/test_graphql_mutations_update.py b/baseapp_comments/tests/test_graphql_mutations_update.py index 6a4ef111..e994deae 100644 --- a/baseapp_comments/tests/test_graphql_mutations_update.py +++ b/baseapp_comments/tests/test_graphql_mutations_update.py @@ -14,6 +14,13 @@ comment { id body + files { + edges { + node { + pk + } + } + } } errors { field diff --git a/baseapp_core/graphql/__init__.py b/baseapp_core/graphql/__init__.py index 1b5c0950..765a40cf 100644 --- a/baseapp_core/graphql/__init__.py +++ b/baseapp_core/graphql/__init__.py @@ -1,7 +1,7 @@ from .connections import CountedConnection # noqa from .decorators import login_required, user_passes_test # noqa from .errors import Errors, ErrorType # noqa -from .fields import File, ThumbnailField # noqa +from .fields import FileInterface, ThumbnailField, get_file_object_type # noqa from .middlewares import ( # noqa JWTAuthentication, LogExceptionMiddleware, diff --git a/baseapp_core/graphql/fields.py b/baseapp_core/graphql/fields.py index c369d87e..2e12e6b6 100644 --- a/baseapp_core/graphql/fields.py +++ b/baseapp_core/graphql/fields.py @@ -1,6 +1,8 @@ import graphene +from django.apps import apps from django.conf import settings from django.core.cache import InvalidCacheBackendError, caches +from easy_thumbnails.engine import NoSourceGenerator from easy_thumbnails.files import get_thumbnailer try: @@ -9,14 +11,37 @@ cache = None -class File(graphene.ObjectType): - url = graphene.String(required=True) - # contentType = graphene.String() - # bytes = graphene.Int() +class FileInterface(graphene.Interface): + url = graphene.String() + + class Meta: + name = "FileInterface" + + +class ThumbnailObjectType(graphene.ObjectType): + class Meta: + name = "Thumbnail" + interfaces = (FileInterface,) + + +def get_file_object_type(): + if apps.is_installed("baseapp.files"): + import swapper + + File = swapper.load_model("baseapp_files", "File") + FileObjectType = File.get_graphql_object_type() + else: + + class FileObjectType(graphene.ObjectType): + class Meta: + interfaces = (FileInterface,) + name = "File" + + return FileObjectType class ThumbnailField(graphene.Field): - def __init__(self, type=File, **kwargs): + def __init__(self, type=graphene.String, **kwargs): kwargs.update( { "args": { @@ -40,16 +65,19 @@ def built_thumbnail(instance, info, width, height, **kwargs): cache_key = self._get_cache_key(instance, width, height) value_from_cache = cache.get(cache_key) if value_from_cache: - return File(url=value_from_cache) + return value_from_cache thumbnailer = get_thumbnailer(instance) - url = thumbnailer.get_thumbnail({"size": (width, height)}).url - absolute_url = info.context.build_absolute_uri(url) + try: + url = thumbnailer.get_thumbnail({"size": (width, height)}).url + absolute_url = info.context.build_absolute_uri(url) + except NoSourceGenerator: + absolute_url = None if cache: cache.set(cache_key, absolute_url, timeout=None) - return File(url=absolute_url) + return absolute_url return built_thumbnail diff --git a/baseapp_core/graphql/views.py b/baseapp_core/graphql/views.py index 86653289..aee37cf6 100644 --- a/baseapp_core/graphql/views.py +++ b/baseapp_core/graphql/views.py @@ -1,3 +1,4 @@ +import json import logging from contextlib import contextmanager @@ -60,3 +61,17 @@ def execute_graphql_request( return super().execute_graphql_request( request, data, query, variables, operation_name, show_graphiql ) + + def parse_body(self, request): + """Handle multipart request spec for multipart/form-data""" + content_type = self.get_content_type(request) + # logging.info('content_type: %s' % content_type) + # import pdb; pdb.set_trace() + if content_type == "multipart/form-data" and "operations" in request.POST: + operations = json.loads(request.POST.get("operations", "{}")) + # import pdb; pdb.set_trace() + files_map = json.loads(request.POST.get("map", "{}")) + from graphene_file_upload.utils import place_files_in_operations + + return place_files_in_operations(operations, files_map, request.FILES) + return super(GraphQLView, self).parse_body(request) diff --git a/baseapp_organizations/tests/test_graphql_mutations_create.py b/baseapp_organizations/tests/test_graphql_mutations_create.py index f34e1931..ad086417 100644 --- a/baseapp_organizations/tests/test_graphql_mutations_create.py +++ b/baseapp_organizations/tests/test_graphql_mutations_create.py @@ -17,9 +17,7 @@ node { id name - image(width: 100, height: 100) { - url - } + image(width: 100, height: 100) urlPath { path } diff --git a/baseapp_profiles/rest_framework/README.md b/baseapp_profiles/rest_framework/README.md new file mode 100644 index 00000000..9b35541d --- /dev/null +++ b/baseapp_profiles/rest_framework/README.md @@ -0,0 +1,72 @@ +# CurrentProfile REST Framework Integration + +## Overview + +This module provides the `CurrentProfileMixin` to enable the same current profile functionality in REST framework views that exists in Django middleware and GraphQL middleware. + +## How It Works + +The `CurrentProfileMixin` reads the `Current-Profile` header from the request and sets `request.user.current_profile` based on: + +1. **No header provided**: Uses the user's default profile (`request.user.profile`) +2. **Header provided**: Attempts to load the profile with the specified ID (relay ID format) + - Only sets the profile if the user has permission to use it (`use_profile` permission) + - Falls back to `None` if permission check fails + +This replicates the logic from: +- `baseapp_profiles.middleware.CurrentProfileMiddleware` (Django middleware) +- `baseapp_profiles.graphql.middleware.CurrentProfileMiddleware` (GraphQL middleware) + +## Usage + +### In ViewSets + +Add the mixin to your viewset classes: + +```python +from baseapp_profiles.rest_framework import CurrentProfileMixin +from rest_framework import viewsets + +class MyViewSet(CurrentProfileMixin, viewsets.ModelViewSet): + queryset = MyModel.objects.all() + serializer_class = MySerializer +``` + +### Making Requests + +Send the `Current-Profile` header with the relay ID of the profile you want to use: + +```bash +curl -H "Authorization: Bearer " \ + -H "Current-Profile: UHJvZmlsZU9iamVjdFR5cGU6MQ==" \ + https://api.example.com/v1/my-endpoint/ +``` + +### Accessing Current Profile + +In your view methods, access the current profile via: + +```python +def list(self, request): + current_profile = request.user.current_profile + # Use the profile... +``` + +## Example Implementation + +See `baseapp.files.rest_framework.files.views.FilesViewSet` for an example: + +```python +from baseapp_profiles.rest_framework import CurrentProfileMixin +from rest_framework import viewsets + +class FilesViewSet(CurrentProfileMixin, mixins.RetrieveModelMixin, ...): + """ViewSet for file CRUD operations with current profile support.""" + pass +``` + +## Notes + +- The mixin executes in the `initial()` method, which runs before any view logic +- Authentication must happen before this mixin runs (ensure authentication classes are properly configured) +- The `Current-Profile` header must be whitelisted in CORS settings diff --git a/baseapp_profiles/rest_framework/__init__.py b/baseapp_profiles/rest_framework/__init__.py index ee5a3f30..e1e1e5d4 100644 --- a/baseapp_profiles/rest_framework/__init__.py +++ b/baseapp_profiles/rest_framework/__init__.py @@ -1,3 +1,4 @@ +from .mixins import CurrentProfileMixin # noqa from .serializers import JWTProfileSerializer -__all__ = ["JWTProfileSerializer"] +__all__ = ["CurrentProfileMixin", "JWTProfileSerializer"] diff --git a/baseapp_profiles/rest_framework/mixins.py b/baseapp_profiles/rest_framework/mixins.py new file mode 100644 index 00000000..dfd5f12f --- /dev/null +++ b/baseapp_profiles/rest_framework/mixins.py @@ -0,0 +1,50 @@ +import swapper + +from baseapp_core.graphql import get_pk_from_relay_id + +Profile = swapper.load_model("baseapp_profiles", "Profile") +profile_app_label = Profile._meta.app_label + + +class CurrentProfileMixin: + """ + Mixin to set current_profile on request.user for REST framework views. + + This replicates the logic from baseapp_profiles.middleware.CurrentProfileMiddleware + and baseapp_profiles.graphql.middleware.CurrentProfileMiddleware for DRF views. + + Usage: + class MyViewSet(CurrentProfileMixin, viewsets.ModelViewSet): + ... + """ + + def initial(self, request, *args, **kwargs): + """ + Runs anything that needs to occur prior to calling the method handler. + This is where we set the current_profile. + """ + super().initial(request, *args, **kwargs) + self.set_current_profile(request) + + def set_current_profile(self, request): + """ + Set current_profile on request.user based on Current-Profile header. + """ + current_profile_header = request.headers.get("Current-Profile") + request.user.current_profile = None + + if not request.user.is_authenticated: + return + + if not current_profile_header: + # No header provided, use the user's default profile + request.user.current_profile = request.user.profile + else: + # Header provided, try to load the specified profile + pk = get_pk_from_relay_id(current_profile_header) + if pk: + profile = Profile.objects.filter(pk=pk).first() + + # Check if user has permission to use this profile + if profile and request.user.has_perm(f"{profile_app_label}.use_profile", profile): + request.user.current_profile = profile diff --git a/baseapp_profiles/tests/integration/test_profiles_queries_without_baseapp_pages.py b/baseapp_profiles/tests/integration/test_profiles_queries_without_baseapp_pages.py index 895a1711..09e7f56a 100644 --- a/baseapp_profiles/tests/integration/test_profiles_queries_without_baseapp_pages.py +++ b/baseapp_profiles/tests/integration/test_profiles_queries_without_baseapp_pages.py @@ -12,9 +12,7 @@ name metadata { metaTitle - metaOgImage(width: 100, height: 100) { - url - } + metaOgImage(width: 100, height: 100) } } } diff --git a/baseapp_profiles/tests/test_get_queries.py b/baseapp_profiles/tests/test_get_queries.py index 259d7998..9c50ce66 100644 --- a/baseapp_profiles/tests/test_get_queries.py +++ b/baseapp_profiles/tests/test_get_queries.py @@ -20,9 +20,7 @@ name metadata { metaTitle - metaOgImage(width: 100, height: 100) { - url - } + metaOgImage(width: 100, height: 100) } } } @@ -49,7 +47,7 @@ def test_profile_metadata(graphql_user_client, image_djangofile): assert content["data"]["profile"]["id"] == profile.relay_id assert content["data"]["profile"]["name"] == profile.name assert content["data"]["profile"]["metadata"]["metaTitle"] == profile.name - assert content["data"]["profile"]["metadata"]["metaOgImage"]["url"].startswith("http") is True + assert content["data"]["profile"]["metadata"]["metaOgImage"].startswith("http") is True def test_owner_can_change_profile(django_user_client, graphql_user_client): diff --git a/baseapp_profiles/tests/test_graphql_mutations_update.py b/baseapp_profiles/tests/test_graphql_mutations_update.py index 2195471c..59f2313c 100644 --- a/baseapp_profiles/tests/test_graphql_mutations_update.py +++ b/baseapp_profiles/tests/test_graphql_mutations_update.py @@ -20,12 +20,8 @@ id name biography - image(width: 100, height: 100) { - url - } - bannerImage(width: 100, height: 100) { - url - } + image(width: 100, height: 100) + bannerImage(width: 100, height: 100) } errors { field @@ -137,7 +133,7 @@ def test_owner_can_update_profile_image(django_user_client, graphql_user_client, ) content = response.json() - assert content["data"]["profileUpdate"]["profile"]["image"]["url"].startswith("http") + assert content["data"]["profileUpdate"]["profile"]["image"].startswith("http") def test_owner_can_update_profile_banner_image( @@ -152,7 +148,7 @@ def test_owner_can_update_profile_banner_image( ) content = response.json() - assert content["data"]["profileUpdate"]["profile"]["bannerImage"]["url"].startswith( + assert content["data"]["profileUpdate"]["profile"]["bannerImage"].startswith( "http://" # NOSONAR ) @@ -191,7 +187,7 @@ def test_owner_can_update_profile_banner_image_camel_case( ) content = response.json() - assert content["data"]["profileUpdate"]["profile"]["bannerImage"]["url"].startswith( + assert content["data"]["profileUpdate"]["profile"]["bannerImage"].startswith( "http://" # NOSONAR ) diff --git a/pyproject.toml b/pyproject.toml index 29a7973f..f7af5804 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -215,6 +215,7 @@ baseapp_ratings = "baseapp_ratings.plugin:RatingsPlugin" baseapp_reactions = "baseapp_reactions.plugin:ReactionsPlugin" baseapp_blocks = "baseapp_blocks.plugin:BlocksPlugin" baseapp_content_feed = "baseapp.content_feed.plugin:ContentFeedPlugin" +baseapp_files = "baseapp.files.plugin:FilesPlugin" baseapp_chats = "baseapp_chats.plugin:ChatsPlugin" [tool.black] diff --git a/testproject/files/__init__.py b/testproject/files/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testproject/files/apps.py b/testproject/files/apps.py new file mode 100644 index 00000000..d4647971 --- /dev/null +++ b/testproject/files/apps.py @@ -0,0 +1,7 @@ +from django.apps import AppConfig + + +class FilesConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "testproject.files" + label = "files" diff --git a/testproject/files/migrations/0001_initial.py b/testproject/files/migrations/0001_initial.py new file mode 100644 index 00000000..7b6958e9 --- /dev/null +++ b/testproject/files/migrations/0001_initial.py @@ -0,0 +1,244 @@ +# Generated by Django 5.2.14 on 2026-06-26 21:23 + +import django.db.models.deletion +import django.utils.timezone +import model_utils.fields +import pgtrigger.compiler +import pgtrigger.migrations +from django.conf import settings +from django.db import migrations, models + +import baseapp.files.utils +import baseapp_core.models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ("baseapp_core", "0001_initial"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + migrations.swappable_dependency(settings.BASEAPP_PROFILES_PROFILE_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="FileTarget", + fields=[ + ( + "target", + models.OneToOneField( + help_text="The document this file target belongs to", + on_delete=django.db.models.deletion.CASCADE, + primary_key=True, + related_name="file_target", + serialize=False, + to="baseapp_core.documentid", + ), + ), + ("files_count", models.JSONField(default=baseapp.files.utils.default_files_count)), + ( + "is_files_enabled", + models.BooleanField(default=True, verbose_name="is files enabled"), + ), + ], + options={ + "abstract": False, + "swappable": "BASEAPP_FILES_FILETARGET_MODEL", + }, + ), + migrations.CreateModel( + name="File", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ( + "created", + model_utils.fields.AutoCreatedField( + default=django.utils.timezone.now, editable=False, verbose_name="created" + ), + ), + ( + "modified", + model_utils.fields.AutoLastModifiedField( + default=django.utils.timezone.now, editable=False, verbose_name="modified" + ), + ), + ("file_content_type", models.CharField(blank=True, max_length=150, null=True)), + ("file_name", models.CharField(blank=True, max_length=512, null=True)), + ( + "file_size", + models.PositiveIntegerField(help_text="File size in bytes", null=True), + ), + ( + "file", + models.FileField( + max_length=512, upload_to=baseapp_core.models.random_name_in("files") + ), + ), + ("name", models.CharField(blank=True, max_length=512, null=True)), + ("description", models.TextField(blank=True, null=True)), + ( + "upload_status", + models.CharField( + choices=[ + ("pending", "Pending Upload"), + ("uploading", "Upload in Progress"), + ("completed", "Upload Completed"), + ("failed", "Upload Failed"), + ("aborted", "Upload Aborted"), + ], + db_index=True, + default="completed", + help_text="Status of the file upload", + max_length=20, + ), + ), + ( + "upload_id", + models.CharField( + blank=True, + help_text="S3 multipart upload ID for in-progress uploads", + max_length=512, + null=True, + ), + ), + ( + "total_parts", + models.PositiveIntegerField( + blank=True, + help_text="Total number of parts for multipart upload", + null=True, + ), + ), + ( + "uploaded_parts", + models.JSONField( + blank=True, + default=dict, + help_text="Tracking uploaded parts with ETags", + null=True, + ), + ), + ( + "upload_expires_at", + models.DateTimeField( + blank=True, + db_index=True, + help_text="Expiration time for pending uploads", + null=True, + ), + ), + ( + "created_by", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="files_created", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "parent", + models.ForeignKey( + blank=True, + help_text="The parent document this file belongs to", + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="files", + to="baseapp_core.documentid", + ), + ), + ( + "profile", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="files", + to=settings.BASEAPP_PROFILES_PROFILE_MODEL, + verbose_name="profile", + ), + ), + ], + options={ + "abstract": False, + "swappable": "BASEAPP_FILES_FILE_MODEL", + }, + ), + pgtrigger.migrations.AddTrigger( + model_name="file", + trigger=pgtrigger.compiler.Trigger( + name="insert_document_id", + sql=pgtrigger.compiler.UpsertTriggerSql( + func="\n INSERT INTO baseapp_core_documentid (public_id, content_type_id, object_id, created, modified)\n VALUES (\n gen_random_uuid(),\n (SELECT id FROM django_content_type WHERE app_label = 'files' AND model = 'file'),\n NEW.id,\n NOW(),\n NOW()\n )\n ON CONFLICT (content_type_id, object_id) DO NOTHING;\n RETURN NULL;\n ", + hash="6f43d1588d53b00ddfa0d5a563a4cd0dbb9e6f62", + operation="INSERT", + pgid="pgtrigger_insert_document_id_d067b", + table="files_file", + when="AFTER", + ), + ), + ), + pgtrigger.migrations.AddTrigger( + model_name="file", + trigger=pgtrigger.compiler.Trigger( + name="delete_document_id", + sql=pgtrigger.compiler.UpsertTriggerSql( + func="\n DELETE FROM baseapp_core_documentid\n WHERE\n content_type_id = (SELECT id FROM django_content_type WHERE app_label = 'files' AND model = 'file')\n AND object_id = OLD.id;\n RETURN NULL;\n ", + hash="877efe613614ca4ef5d93a264fde512958f34df2", + operation="DELETE", + pgid="pgtrigger_delete_document_id_23af1", + table="files_file", + when="AFTER", + ), + ), + ), + pgtrigger.migrations.AddTrigger( + model_name="file", + trigger=pgtrigger.compiler.Trigger( + name="update_file_target_on_insert", + sql=pgtrigger.compiler.UpsertTriggerSql( + func="\n DECLARE\n new_counts JSONB;\n BEGIN\n IF NEW.parent_id IS NOT NULL THEN\n -- Calculate new counts\n SELECT COALESCE(\n jsonb_object_agg(\n file_content_type,\n count\n ) || jsonb_build_object('total', SUM(count)),\n '{\"total\": 0}'::jsonb\n ) INTO new_counts\n FROM (\n SELECT\n COALESCE(file_content_type, 'unknown') as file_content_type,\n COUNT(*)::int as count\n FROM files_file\n WHERE parent_id = NEW.parent_id\n GROUP BY file_content_type\n ) counts;\n\n -- Insert or update FileTarget\n INSERT INTO files_filetarget (target_id, files_count, is_files_enabled)\n VALUES (NEW.parent_id, new_counts, true)\n ON CONFLICT (target_id)\n DO UPDATE SET files_count = new_counts;\n END IF;\n RETURN NEW;\n END;\n ", + hash="fe7e8e6f14c341e981b6cec3920eb6ba96222706", + operation="INSERT", + pgid="pgtrigger_update_file_target_on_insert_8a1d9", + table="files_file", + when="AFTER", + ), + ), + ), + pgtrigger.migrations.AddTrigger( + model_name="file", + trigger=pgtrigger.compiler.Trigger( + name="update_file_target_on_update", + sql=pgtrigger.compiler.UpsertTriggerSql( + func="\n DECLARE\n new_counts JSONB;\n old_counts JSONB;\n BEGIN\n -- Update old parent if it changed\n IF OLD.parent_id IS DISTINCT FROM NEW.parent_id\n AND OLD.parent_id IS NOT NULL THEN\n\n SELECT COALESCE(\n jsonb_object_agg(\n file_content_type,\n count\n ) || jsonb_build_object('total', SUM(count)),\n '{\"total\": 0}'::jsonb\n ) INTO old_counts\n FROM (\n SELECT\n COALESCE(file_content_type, 'unknown') as file_content_type,\n COUNT(*)::int as count\n FROM files_file\n WHERE parent_id = OLD.parent_id\n GROUP BY file_content_type\n ) counts;\n\n UPDATE files_filetarget SET files_count = old_counts\n WHERE target_id = OLD.parent_id;\n END IF;\n\n -- Update new parent\n IF NEW.parent_id IS NOT NULL THEN\n SELECT COALESCE(\n jsonb_object_agg(\n file_content_type,\n count\n ) || jsonb_build_object('total', SUM(count)),\n '{\"total\": 0}'::jsonb\n ) INTO new_counts\n FROM (\n SELECT\n COALESCE(file_content_type, 'unknown') as file_content_type,\n COUNT(*)::int as count\n FROM files_file\n WHERE parent_id = NEW.parent_id\n GROUP BY file_content_type\n ) counts;\n\n INSERT INTO files_filetarget (target_id, files_count, is_files_enabled)\n VALUES (NEW.parent_id, new_counts, true)\n ON CONFLICT (target_id)\n DO UPDATE SET files_count = new_counts;\n END IF;\n\n RETURN NEW;\n END;\n ", + hash="4e27b58158d9658648a4a79af0d8bfed613196ef", + operation="UPDATE", + pgid="pgtrigger_update_file_target_on_update_89baa", + table="files_file", + when="AFTER", + ), + ), + ), + pgtrigger.migrations.AddTrigger( + model_name="file", + trigger=pgtrigger.compiler.Trigger( + name="update_file_target_on_delete", + sql=pgtrigger.compiler.UpsertTriggerSql( + func="\n DECLARE\n new_counts JSONB;\n BEGIN\n IF OLD.parent_id IS NOT NULL THEN\n SELECT COALESCE(\n jsonb_object_agg(\n file_content_type,\n count\n ) || jsonb_build_object('total', SUM(count)),\n '{\"total\": 0}'::jsonb\n ) INTO new_counts\n FROM (\n SELECT\n COALESCE(file_content_type, 'unknown') as file_content_type,\n COUNT(*)::int as count\n FROM files_file\n WHERE parent_id = OLD.parent_id\n GROUP BY file_content_type\n ) counts;\n\n UPDATE files_filetarget SET files_count = new_counts\n WHERE target_id = OLD.parent_id;\n END IF;\n\n RETURN OLD;\n END;\n ", + hash="3ccd45f3c662961c3959e628d80113b1a4203f1b", + operation="DELETE", + pgid="pgtrigger_update_file_target_on_delete_3c3fa", + table="files_file", + when="AFTER", + ), + ), + ), + ] diff --git a/testproject/files/migrations/__init__.py b/testproject/files/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testproject/files/models.py b/testproject/files/models.py new file mode 100644 index 00000000..2fb2a62e --- /dev/null +++ b/testproject/files/models.py @@ -0,0 +1,15 @@ +import swapper + +from baseapp.files.models import AbstractFile, AbstractFileTarget + + +class FileTarget(AbstractFileTarget): + class Meta(AbstractFileTarget.Meta): + abstract = False + swappable = swapper.swappable_setting("baseapp_files", "FileTarget") + + +class File(AbstractFile): + class Meta(AbstractFile.Meta): + abstract = False + swappable = swapper.swappable_setting("baseapp_files", "File") diff --git a/testproject/settings.py b/testproject/settings.py index 8e1e6460..f07b570a 100644 --- a/testproject/settings.py +++ b/testproject/settings.py @@ -42,6 +42,7 @@ "baseapp_social_auth", "baseapp_social_auth.cache", "baseapp.content_feed", + "baseapp.files", "baseapp_pdf", "baseapp_api_key", *WAGTAIL_INSTALLED_APPS, @@ -53,6 +54,7 @@ "testproject.profiles", "testproject.reactions", "testproject.content_feed", + "testproject.files", "testproject.follows", "testproject.blocks", "testproject.base", @@ -138,6 +140,7 @@ *plugin_registry.get("AUTHENTICATION_BACKENDS", "baseapp_blocks"), *plugin_registry.get("AUTHENTICATION_BACKENDS", "baseapp_chats"), *plugin_registry.get("AUTHENTICATION_BACKENDS", "baseapp_organizations"), + *plugin_registry.get("AUTHENTICATION_BACKENDS", "baseapp_files"), ] ADMIN_TIME_ZONE = "UTC" @@ -238,6 +241,10 @@ BASEAPP_CHATS_MESSAGE_MODEL = "social_chats.Message" BASEAPP_CHATS_MESSAGESTATUS_MODEL = "social_chats.MessageStatus" +# Files +BASEAPP_FILES_FILE_MODEL = "files.File" +BASEAPP_FILES_FILETARGET_MODEL = "files.FileTarget" + # Notifications NOTIFICATIONS_NOTIFICATION_MODEL = "notifications.Notification" BASEAPP_NOTIFICATIONS_NOTIFICATIONSETTING_MODEL = "notifications.NotificationSetting" From 637521c9fb7f8e6c25b3242b0ff39227033f2516 Mon Sep 17 00:00:00 2001 From: Alisson Patricio Date: Sat, 27 Jun 2026 01:28:01 +0000 Subject: [PATCH 02/20] files admin: use baseapp_core.admin_helpers.ModelAdmin Replace stock django.contrib.admin.ModelAdmin with the project's baseapp_core.admin_helpers.ModelAdmin for FileAdmin and FileTargetAdmin. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01XyuGcRJryi9hQiWRVXU7Js --- baseapp/files/admin.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/baseapp/files/admin.py b/baseapp/files/admin.py index 89d93631..dce2dc7d 100644 --- a/baseapp/files/admin.py +++ b/baseapp/files/admin.py @@ -1,19 +1,21 @@ import swapper from django.contrib import admin +from baseapp_core.admin_helpers import ModelAdmin + File = swapper.load_model("baseapp_files", "File") FileTarget = swapper.load_model("baseapp_files", "FileTarget") @admin.register(File) -class FileAdmin(admin.ModelAdmin): +class FileAdmin(ModelAdmin): list_display = ("pk", "name", "file_content_type", "parent", "created_by", "created") search_fields = ("name",) raw_id_fields = ("created_by",) @admin.register(FileTarget) -class FileTargetAdmin(admin.ModelAdmin): +class FileTargetAdmin(ModelAdmin): list_display = ( "pk", "target", From 0b8e044ceb384b3f5b1221e4eaf33a33897db762 Mon Sep 17 00:00:00 2001 From: Alisson Patricio Date: Sat, 11 Jul 2026 02:00:55 +0000 Subject: [PATCH 03/20] files: port the reactions performance pattern (annotate_queryset + shared service) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - AbstractFileTarget.annotate_queryset: correlated Subquery annotations (_file_target_files_count, _file_target_is_files_enabled, files_count_total) mirroring ReactableMetadata.annotate_queryset, plus get_for_object helper. - FilesMetadataService (shared service "files_metadata") with annotation-aware getters and annotate_queryset delegate; registered via ServicesContributor. - FilesInterface resolvers now go through the service and filter files via the DocumentId join instead of per-row get_file_target()/DocumentId.get — no more N+1 on lists; disabled targets return an empty queryset. - Comments pre_optimization_hook applies files annotations alongside commentable/reactable. - Align ContentType lookups with baseapp_core (concrete model) and reuse DocumentId.get_or_create_for_object; for_concrete_model=False would miss trigger-created DocumentId rows for proxy models. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146oeiGsELS6QVpVUQHDQXd --- baseapp/files/apps.py | 9 ++- baseapp/files/base.py | 24 +++----- baseapp/files/graphql/interfaces.py | 38 +++++++------ baseapp/files/models.py | 50 +++++++++++++++++ baseapp/files/services/__init__.py | 8 ++- baseapp/files/services/metadata.py | 71 ++++++++++++++++++++++++ baseapp/files/utils.py | 13 +---- baseapp_comments/graphql/object_types.py | 4 ++ 8 files changed, 172 insertions(+), 45 deletions(-) create mode 100644 baseapp/files/services/metadata.py diff --git a/baseapp/files/apps.py b/baseapp/files/apps.py index 7b02248d..d5f10a75 100644 --- a/baseapp/files/apps.py +++ b/baseapp/files/apps.py @@ -1,13 +1,18 @@ -from baseapp_core.plugins import BaseAppConfig, GraphQLContributor +from baseapp_core.plugins import BaseAppConfig, GraphQLContributor, ServicesContributor -class FilesConfig(BaseAppConfig, GraphQLContributor): +class FilesConfig(BaseAppConfig, ServicesContributor, GraphQLContributor): default = True name = "baseapp.files" label = "baseapp_files" verbose_name = "BaseApp Files" default_auto_field = "django.db.models.BigAutoField" + def register_shared_services(self, registry): + from .services.metadata import FilesMetadataService + + registry.register(FilesMetadataService()) + def register_graphql_shared_interfaces(self, registry): from .graphql.interfaces import get_files_interface diff --git a/baseapp/files/base.py b/baseapp/files/base.py index 786f4ea1..d973d3f5 100644 --- a/baseapp/files/base.py +++ b/baseapp/files/base.py @@ -2,8 +2,6 @@ from django.contrib.contenttypes.models import ContentType from django.db import models -from baseapp_core.models import DocumentId - from .utils import get_or_create_file_target @@ -12,26 +10,22 @@ class Meta: abstract = True @property - def files(self): - """Returns files related to this object through DocumentId.""" + def files(self) -> models.QuerySet: + """Returns files related to this object through the DocumentId join.""" File = swapper.load_model("baseapp_files", "File") - content_type = ContentType.objects.get_for_model(self, for_concrete_model=False) - try: - document_id = DocumentId.objects.get( - content_type=content_type, - object_id=self.pk, - ) - return File.objects.filter(parent=document_id) - except DocumentId.DoesNotExist: - return File.objects.none() + content_type = ContentType.objects.get_for_model(self) + return File.objects.filter( + parent__content_type=content_type, + parent__object_id=self.pk, + ) def get_file_target(self): return get_or_create_file_target(self) @property - def files_count(self): + def files_count(self) -> dict: return self.get_file_target().files_count @property - def is_files_enabled(self): + def is_files_enabled(self) -> bool: return self.get_file_target().is_files_enabled diff --git a/baseapp/files/graphql/interfaces.py b/baseapp/files/graphql/interfaces.py index f4525807..ef6bd74f 100644 --- a/baseapp/files/graphql/interfaces.py +++ b/baseapp/files/graphql/interfaces.py @@ -4,7 +4,9 @@ from graphene.types.generic import GenericScalar from graphene_django.filter import DjangoFilterConnectionField -from baseapp_core.models import DocumentId +from baseapp_core.plugins import shared_services + +from ..utils import default_files_count class FilesInterface(Interface): @@ -14,29 +16,31 @@ class FilesInterface(Interface): lambda: swapper.load_model("baseapp_files", "File").get_graphql_object_type() ) - def resolve_files_count(self, info, **kwargs): - if hasattr(self, "get_file_target"): - return self.get_file_target().files_count - return {"total": 0} + def resolve_files_count(self, info, **kwargs) -> dict: + if service := shared_services.get("files_metadata"): + return service.get_files_count(self) + return default_files_count() - def resolve_is_files_enabled(self, info, **kwargs): - if hasattr(self, "get_file_target"): - return self.get_file_target().is_files_enabled + def resolve_is_files_enabled(self, info, **kwargs) -> bool: + if service := shared_services.get("files_metadata"): + return service.is_files_enabled(self) return True def resolve_files(self, info, **kwargs): File = swapper.load_model("baseapp_files", "File") - # Get the DocumentId for this object - content_type = ContentType.objects.get_for_model(self) - try: - document_id = DocumentId.objects.get( - content_type=content_type, - object_id=self.pk, - ) - return File.objects.filter(parent=document_id).order_by("-created") - except DocumentId.DoesNotExist: + + service = shared_services.get("files_metadata") + if service is not None and not service.is_files_enabled(self): return File.objects.none() + # Filter through the DocumentId join instead of fetching the DocumentId + # row per object — keeps the connection a single query per list. + content_type = ContentType.objects.get_for_model(self) + return File.objects.filter( + parent__content_type=content_type, + parent__object_id=self.pk, + ).order_by("-created") + def get_files_interface(): return FilesInterface diff --git a/baseapp/files/models.py b/baseapp/files/models.py index f0413d84..9d4a7943 100644 --- a/baseapp/files/models.py +++ b/baseapp/files/models.py @@ -2,8 +2,12 @@ import swapper from django.apps import apps from django.conf import settings +from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ValidationError from django.db import models +from django.db.models import IntegerField, OuterRef, Subquery, Value +from django.db.models.fields.json import KeyTextTransform +from django.db.models.functions import Cast, Coalesce from django.db.models.signals import class_prepared from django.dispatch import receiver from django.utils.translation import gettext_lazy as _ @@ -228,6 +232,52 @@ class Meta: def __str__(self): return f"FileTarget for {self.target}" + @classmethod + def get_for_object(cls, obj: models.Model | None) -> "AbstractFileTarget | None": + """Return the FileTarget for the given object, or `None` if not found.""" + if not obj or not getattr(obj, "pk", None): + return None + try: + ct = ContentType.objects.get_for_model(obj) + return cls.objects.get(target__content_type=ct, target__object_id=obj.pk) + except cls.DoesNotExist: + return None + + @classmethod + def annotate_queryset(cls, queryset: models.QuerySet) -> models.QuerySet: + """ + Annotate `queryset` with file-target metadata so resolvers don't N+1. + Adds `_file_target_files_count`, `_file_target_is_files_enabled`, and a + flat public `files_count_total` (integer cast of the `files_count->total` + JSON key) so consumer-side filtering/ORDER BY can use a real expression. + Mirrors `ReactableMetadata.annotate_queryset`. + """ + model_cls = queryset.model + ct_id = ContentType.objects.get_for_model(model_cls).pk + file_target_qs = cls.objects.filter( + target__content_type_id=ct_id, + target__object_id=OuterRef("pk"), + ) + total_subq = Subquery( + file_target_qs.annotate(total=KeyTextTransform("total", "files_count")).values( + "total" + )[:1] + ) + return queryset.annotate( + _file_target_files_count=Subquery(file_target_qs.values("files_count")[:1]), + _file_target_is_files_enabled=Coalesce( + Subquery( + file_target_qs.values("is_files_enabled")[:1], + output_field=models.BooleanField(), + ), + Value(True), + ), + files_count_total=Coalesce( + Cast(total_subq, output_field=IntegerField()), + Value(0), + ), + ) + file_inheritances = [] diff --git a/baseapp/files/services/__init__.py b/baseapp/files/services/__init__.py index 54d075b6..ba060fc4 100644 --- a/baseapp/files/services/__init__.py +++ b/baseapp/files/services/__init__.py @@ -1,4 +1,10 @@ from .cleanup import cleanup_expired_uploads, cleanup_failed_uploads +from .metadata import FilesMetadataService from .upload_service import UploadService -__all__ = ["UploadService", "cleanup_expired_uploads", "cleanup_failed_uploads"] +__all__ = [ + "FilesMetadataService", + "UploadService", + "cleanup_expired_uploads", + "cleanup_failed_uploads", +] diff --git a/baseapp/files/services/metadata.py b/baseapp/files/services/metadata.py new file mode 100644 index 00000000..4e655c24 --- /dev/null +++ b/baseapp/files/services/metadata.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import swapper +from django.apps import apps +from django.db import models + +from baseapp_core.plugins import SharedServiceProvider + + +class FilesMetadataService(SharedServiceProvider): + """ + Service that provides files metadata (per-content-type counts dict + enabled + flag) for any object that has a `DocumentId`. Registered in `apps.py` + `ready()`. Mirrors `ReactableMetadataService`. + """ + + @property + def service_name(self) -> str: + return "files_metadata" + + def is_available(self) -> bool: + return apps.is_installed("baseapp.files") + + def _get_model(self): + return swapper.load_model("baseapp_files", "FileTarget") + + def _default_counts(self) -> dict: + from ..utils import default_files_count + + return default_files_count() + + def get_file_target(self, obj): + """Return `FileTarget` for `obj`, or `None` if not found.""" + return self._get_model().get_for_object(obj) + + def get_or_create_file_target(self, obj): + """Return or create `FileTarget` for `obj`.""" + from ..utils import get_or_create_file_target + + return get_or_create_file_target(obj) + + def get_files_count(self, obj) -> dict: + """Return per-content-type files count dict for `obj`. Uses annotation if + available; when the annotation is present but NULL (no FileTarget row exists + yet) return the default counts dict immediately — no extra round-trip.""" + if hasattr(obj, "_file_target_files_count"): + val = obj._file_target_files_count + return val if val is not None else self._default_counts() + file_target = self.get_file_target(obj) + if file_target is not None: + return file_target.files_count + return self._default_counts() + + def is_files_enabled(self, obj) -> bool: + """Return whether files are enabled for `obj`. Uses annotation if available.""" + if hasattr(obj, "_file_target_is_files_enabled"): + val = obj._file_target_is_files_enabled + return val if val is not None else True + file_target = self.get_file_target(obj) + return file_target.is_files_enabled if file_target else True + + def set_is_files_enabled(self, obj, value: bool) -> None: + """Write-through setter for the `is_files_enabled` flag, creating the + FileTarget row if needed.""" + file_target = self.get_or_create_file_target(obj) + if file_target is not None: + file_target.is_files_enabled = bool(value) + file_target.save(update_fields=["is_files_enabled"]) + + def annotate_queryset(self, queryset: models.QuerySet) -> models.QuerySet: + return self._get_model().annotate_queryset(queryset) diff --git a/baseapp/files/utils.py b/baseapp/files/utils.py index e2e4a401..35913c9c 100644 --- a/baseapp/files/utils.py +++ b/baseapp/files/utils.py @@ -15,11 +15,7 @@ def get_or_create_file_target(target_obj): """Get or create a FileTarget for the given object.""" FileTarget = swapper.load_model("baseapp_files", "FileTarget") - content_type = ContentType.objects.get_for_model(target_obj, for_concrete_model=False) - document_id, created = DocumentId.objects.get_or_create( - content_type=content_type, - object_id=target_obj.pk, - ) + document_id = DocumentId.get_or_create_for_object(target_obj) file_target, created = FileTarget.objects.get_or_create(target=document_id) return file_target @@ -45,11 +41,8 @@ def set_files_parent(parent, files: Iterable): if not files: return - parent_content_type = ContentType.objects.get_for_model(parent, for_concrete_model=False) - parent_document_id, created = DocumentId.objects.get_or_create( - content_type=parent_content_type, - object_id=parent.pk, - ) + parent_content_type = ContentType.objects.get_for_model(parent) + parent_document_id = DocumentId.get_or_create_for_object(parent) previous_parents = {} for file_obj in files: diff --git a/baseapp_comments/graphql/object_types.py b/baseapp_comments/graphql/object_types.py index fee5946d..7918e845 100644 --- a/baseapp_comments/graphql/object_types.py +++ b/baseapp_comments/graphql/object_types.py @@ -181,6 +181,10 @@ def pre_optimization_hook(cls, queryset, optimizer): if service := shared_services.get("reactable_metadata"): queryset = service.annotate_queryset(queryset) + # Annotate files metadata so FilesInterface resolvers don't N+1. + if service := shared_services.get("files_metadata"): + queryset = service.annotate_queryset(queryset) + return queryset @classmethod From e09e8c65c1ed03772e3825b53b6f2bbba285e5da Mon Sep 17 00:00:00 2001 From: Alisson Patricio Date: Sat, 11 Jul 2026 02:04:19 +0000 Subject: [PATCH 04/20] files: enforce attach permission and files-enabled check in FileAttachToTarget Replace the hardcoded created_by comparison with the attach_ object permission (FilesPermissionsBackend grants owners; superusers pass via has_perm), resolve the enabled flag through the files_metadata service instead of get_file_target() (which created rows as a side effect and skipped non-FileableModel targets), and reuse DocumentId.get_or_create_for_object. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146oeiGsELS6QVpVUQHDQXd --- baseapp/files/graphql/mutations.py | 24 +++++------- baseapp/files/tests/test_graphql.py | 59 +++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 15 deletions(-) diff --git a/baseapp/files/graphql/mutations.py b/baseapp/files/graphql/mutations.py index 012c0a8b..fdf42110 100644 --- a/baseapp/files/graphql/mutations.py +++ b/baseapp/files/graphql/mutations.py @@ -7,6 +7,7 @@ from baseapp_core.graphql import RelayMutation, get_obj_from_relay_id, login_required from baseapp_core.models import DocumentId +from baseapp_core.plugins import shared_services from .interfaces import FilesInterface @@ -56,8 +57,6 @@ class Input: @login_required @transaction.atomic def mutate_and_get_payload(cls, root, info, **input): - from django.contrib.contenttypes.models import ContentType - file_relay_ids = input.get("file_relay_ids", []) target_object_id = input.get("target_object_id") @@ -76,22 +75,17 @@ def mutate_and_get_payload(cls, root, info, **input): ) # Get or create DocumentId for the target - target_content_type = ContentType.objects.get_for_model(target) - target_document_id, created = DocumentId.objects.get_or_create( - content_type=target_content_type, - object_id=target.pk, - ) + target_document_id = DocumentId.get_or_create_for_object(target) # Check if files are enabled for this target - if hasattr(target, "get_file_target"): - file_target = target.get_file_target() - if not file_target.is_files_enabled: + if service := shared_services.get("files_metadata"): + if not service.is_files_enabled(target): raise GraphQLError( str(_("Files are not enabled for this target")), extensions={"code": "files_disabled"}, ) - # Get all files and verify ownership + # Get all files and verify the attach permission files = [] for file_relay_id in file_relay_ids: file_obj = get_obj_from_relay_id(info, file_relay_id) @@ -101,8 +95,10 @@ def mutate_and_get_payload(cls, root, info, **input): extensions={"code": "not_found"}, ) - # Check ownership - only file owner can attach it - if file_obj.created_by != info.context.user: + # Object-level permission (FilesPermissionsBackend grants the owner) + if not info.context.user.has_perm( + f"{app_label}.attach_{file_model_name}", file_obj + ): raise GraphQLError( str(_("You don't have permission to attach this file")), extensions={"code": "permission_required"}, @@ -121,8 +117,6 @@ def mutate_and_get_payload(cls, root, info, **input): # Attach all files to the target for file_obj in files: - # TO DO: Check permission - # if not info.context.user.has_perm("baseapp_files.attach_file", file_obj): file_obj.parent = target_document_id file_obj.save(update_fields=["parent"]) diff --git a/baseapp/files/tests/test_graphql.py b/baseapp/files/tests/test_graphql.py index 34640863..355ad4c2 100644 --- a/baseapp/files/tests/test_graphql.py +++ b/baseapp/files/tests/test_graphql.py @@ -331,6 +331,65 @@ def test_file_attach_to_target_requires_ownership(graphql_user_client, django_us assert content["errors"][0]["extensions"]["code"] == "permission_required" +def test_file_attach_to_target_superuser_can_attach_any_file( + graphql_user_client, django_user_client +): + from django.contrib.auth import get_user_model + + User = get_user_model() + other_user = User.objects.create_user(email="other@test.com", password="pass123") + + django_user_client.user.is_superuser = True + django_user_client.user.save() + + comment = CommentFactory(user=other_user) + file = File.objects.create( + file_name="test.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=other_user, + ) + + response = graphql_user_client( + FILE_ATTACH_TO_TARGET_MUTATION, + variables={ + "fileRelayIds": [file.relay_id], + "targetObjectId": comment.relay_id, + }, + ) + content = response.json() + + assert "errors" not in content + assert len(content["data"]["fileAttachToTarget"]["attachedFiles"]) == 1 + + +def test_file_attach_to_target_files_disabled(graphql_user_client, django_user_client): + user = django_user_client.user + comment = CommentFactory(user=user) + file_target = comment.get_file_target() + file_target.is_files_enabled = False + file_target.save() + + file = File.objects.create( + file_name="test.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=user, + ) + + response = graphql_user_client( + FILE_ATTACH_TO_TARGET_MUTATION, + variables={ + "fileRelayIds": [file.relay_id], + "targetObjectId": comment.relay_id, + }, + ) + content = response.json() + + assert "errors" in content + assert content["errors"][0]["extensions"]["code"] == "files_disabled" + file.refresh_from_db() + assert file.parent_id is None + + def test_file_attach_to_target_already_attached(graphql_user_client, django_user_client): user = django_user_client.user comment1 = CommentFactory(user=user) From cd3643918167bd06af759b6af021f8787fb9bf83 Mon Sep 17 00:00:00 2001 From: Alisson Patricio Date: Sat, 11 Jul 2026 02:05:30 +0000 Subject: [PATCH 05/20] files docs: multipart-only scope, fix parent_id examples, real storage selection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Mark resumable uploads explicitly not planned (in-session pause/resume only). - Docs showed parent_content_type/parent_object_id; the API takes parent_id (DocumentId public_id) — fix all request/response/client examples. - UPLOAD_STORAGE_HANDLER was documented but never read; document the actual default_storage sniffing in storage/__init__.py. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146oeiGsELS6QVpVUQHDQXd --- baseapp/files/README.md | 50 ++++++++++++------------------------ baseapp/files/UPLOAD_FLOW.md | 21 +++++++-------- 2 files changed, 26 insertions(+), 45 deletions(-) diff --git a/baseapp/files/README.md b/baseapp/files/README.md index 2d37a0eb..4df7767d 100644 --- a/baseapp/files/README.md +++ b/baseapp/files/README.md @@ -167,8 +167,7 @@ Content-Type: application/json "file_content_type": "video/mp4", "num_parts": 20, "part_size": 5242880, - "parent_content_type": "testapp.post", // optional - "parent_object_id": 123 // optional + "parent_id": "b3c5…-document-public-id" // optional (DocumentId public_id of the parent) } ``` @@ -262,8 +261,7 @@ Content-Type: application/json "created_by": 1, "created_by_name": "John Doe", "created": "2025-12-29T10:00:00Z", - "parent_content_type": 23, - "parent_object_id": 123 + "parent_id": "b3c5…-document-public-id" } ``` @@ -278,8 +276,7 @@ Authorization: Bearer {jwt_token} Content-Type: application/json { - "parent_content_type": "testapp.comment", - "parent_object_id": 789 + "parent_id": "b3c5…-document-public-id" } ``` @@ -296,8 +293,7 @@ POST /v1/files/uploads - `file_content_type` (string, required) - MIME type (e.g., "video/mp4") - `num_parts` (integer, required) - Number of parts (1-10,000) - `part_size` (integer, required) - Size of each part in bytes (min 5MB for multipart) -- `parent_content_type` (string, optional) - e.g., "testapp.post" -- `parent_object_id` (integer, optional) - ID of parent object +- `parent_id` (UUID, optional) - `DocumentId` public_id of the parent object **Validation:** - File size must not exceed `MAX_FILE_UPLOAD_SIZE` (default 5GB) @@ -339,8 +335,7 @@ GET /v1/files **Query Parameters:** - `status` (string, optional) - Filter by upload_status (default: "completed") -- `parent_content_type` (string, optional) - Filter by parent type -- `parent_object_id` (integer, optional) - Filter by parent ID +- `parent_id` (UUID, optional) - Filter by parent `DocumentId` public_id **Notes:** - Returns only user's own files (unless staff) @@ -381,8 +376,7 @@ POST /v1/files/{id}/set-parent ``` **Parameters:** -- `parent_content_type` (string, required) - e.g., "testapp.post" -- `parent_object_id` (integer, required) - Parent object ID +- `parent_id` (UUID, required) - `DocumentId` public_id of the parent object ## Client Implementation @@ -395,7 +389,7 @@ class MultipartUploader { this.token = token; } - async uploadFile(file, parentContentType = null, parentObjectId = null) { + async uploadFile(file, parentId = null) { // 1. Calculate parts const partSize = 5 * 1024 * 1024; // 5MB const numParts = Math.ceil(file.size / partSize); @@ -413,8 +407,7 @@ class MultipartUploader { file_content_type: file.type, num_parts: numParts, part_size: partSize, - parent_content_type: parentContentType, - parent_object_id: parentObjectId + parent_id: parentId }) }); @@ -491,7 +484,7 @@ try { import requests import math -def upload_file_multipart(api_url, token, file_path, parent_content_type=None, parent_object_id=None): +def upload_file_multipart(api_url, token, file_path, parent_id=None): """Upload a file using multipart upload.""" # 1. Read file info @@ -516,8 +509,7 @@ def upload_file_multipart(api_url, token, file_path, parent_content_type=None, p 'file_content_type': 'application/octet-stream', 'num_parts': num_parts, 'part_size': part_size, - 'parent_content_type': parent_content_type, - 'parent_object_id': parent_object_id + 'parent_id': parent_id } ) init_data = init_response.json() @@ -557,8 +549,7 @@ result = upload_file_multipart( 'https://api.example.com', 'your-jwt-token', '/path/to/large-file.mp4', - parent_content_type='testapp.post', - parent_object_id=123 + parent_id='b3c5…-document-public-id' ) print(f"File uploaded: {result['url']}") ``` @@ -712,20 +703,13 @@ def get_upload_handler(): return LocalUploadHandler() ``` -### Resumable Uploads +### Resumable Uploads (not planned) -The current implementation doesn't support resuming interrupted uploads. To add this: - -1. Client should track which parts were successfully uploaded -2. Add an endpoint to query uploaded parts: - ```python - @action(detail=True, methods=['get']) - def uploaded_parts(self, request, pk=None): - file_obj = self.get_object() - # Return list of uploaded part numbers - return Response(file_obj.uploaded_parts or {}) - ``` -3. Client resumes by only uploading missing parts +Resuming interrupted uploads across sessions is intentionally **out of scope** — the +package provides multipart upload for large files only. If an upload is interrupted, +the client aborts (or lets the expiration cleanup task abort) and re-initiates. +Clients may still pause/resume within a session by holding on to the presigned URLs +returned at initiation, uploading only the remaining parts while the URLs are valid. ### File Size Quotas diff --git a/baseapp/files/UPLOAD_FLOW.md b/baseapp/files/UPLOAD_FLOW.md index 015c30e4..c66938ca 100644 --- a/baseapp/files/UPLOAD_FLOW.md +++ b/baseapp/files/UPLOAD_FLOW.md @@ -16,8 +16,7 @@ The file upload system supports both direct S3 uploads (production) and local st "file_content_type": "video/mp4", "num_parts": 20, "part_size": 5242880, - "parent_content_type": "testapp.post", // optional - "parent_object_id": 123 // optional + "parent_id": "b3c5…-document-public-id" // optional (DocumentId public_id) } ``` @@ -143,28 +142,26 @@ Cancels the upload and cleans up temporary files. ```python # Maximum file size (default: 5GB) MAX_FILE_UPLOAD_SIZE = 5 * 1024 * 1024 * 1024 - -# Storage backend -UPLOAD_STORAGE_HANDLER = "local" # or "s3" ``` -### Local Storage -```python -UPLOAD_STORAGE_HANDLER = "local" -``` +The storage handler is selected automatically from Django's `default_storage` +(`baseapp/files/storage/__init__.py`): an `S3Boto3Storage` backend selects the +S3 multipart handler, anything else falls back to the local handler. There is +no dedicated setting. + +### Local Storage (default_storage is not S3) - Uses `LocalUploadHandler` - Stores files in `MEDIA_ROOT/files/` - Temp parts in `MEDIA_ROOT/temp_uploads/{upload_id}/` -### S3 Storage +### S3 Storage (default_storage is S3Boto3Storage) ```python -UPLOAD_STORAGE_HANDLER = "s3" AWS_ACCESS_KEY_ID = "..." AWS_SECRET_ACCESS_KEY = "..." AWS_STORAGE_BUCKET_NAME = "my-bucket" AWS_S3_REGION_NAME = "us-east-1" ``` -- Uses `S3UploadHandler` +- Uses `S3MultipartUploadHandler` - Generates presigned URLs for direct S3 upload - No backend file handling From 7a744360693c0cc3c93d456d7dd0158dc4de32fb Mon Sep 17 00:00:00 2001 From: Alisson Patricio Date: Sat, 11 Jul 2026 02:07:47 +0000 Subject: [PATCH 06/20] files: query-count tests locking in the annotate_queryset optimisation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flat-query assertions for FilesInterface (filesCount invariant to file volume, zero-files same budget), comments-with-files listing invariant to list size, and a single-SELECT test for FileTarget.annotate_queryset — mirrors the reactions/comments query-count test style. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146oeiGsELS6QVpVUQHDQXd --- .../test_graphql_queries_object_files.py | 196 ++++++++++++++++++ 1 file changed, 196 insertions(+) create mode 100644 baseapp/files/tests/test_graphql_queries_object_files.py diff --git a/baseapp/files/tests/test_graphql_queries_object_files.py b/baseapp/files/tests/test_graphql_queries_object_files.py new file mode 100644 index 00000000..edf1d2c2 --- /dev/null +++ b/baseapp/files/tests/test_graphql_queries_object_files.py @@ -0,0 +1,196 @@ +""" +Query-count tests for the `FilesInterface`. + +Mirrors `baseapp_reactions/tests/test_graphql_queries_object_reactions.py`: assert +the GraphQL `FilesInterface` resolves in a flat number of database queries +regardless of how many `File` rows point at the target — locks in the +`FilesMetadataService.annotate_queryset` optimisation. +""" + +import pytest +import swapper +from constance.test import override_config +from django.contrib.contenttypes.models import ContentType + +from baseapp_comments.tests.factories import CommentFactory +from baseapp_core.models import DocumentId + +pytestmark = pytest.mark.django_db + +File = swapper.load_model("baseapp_files", "File") +FileTarget = swapper.load_model("baseapp_files", "FileTarget") + +# Baseline for `filesCount` + `isFilesEnabled` on one Comment node: 1× relay +# `node()` (documentid + content_type JOIN) + 1× `comments_comment` load with +# `FileTarget` inlined as a Subquery via `pre_optimization_hook`. +# Update this number deliberately if you change the resolver / annotation path. +EXPECTED_FILES_INTERFACE_QUERY_COUNT = 2 + +COUNTS_ONLY_QUERY = """ + query GetTarget($id: ID!) { + node(id: $id) { + ... on FilesInterface { + filesCount + isFilesEnabled + } + } + } +""" + +COMMENTS_WITH_FILES_LIST_QUERY = """ + query GetTarget($id: ID!) { + node(id: $id) { + ... on CommentsInterface { + comments(first: 50) { + edges { + node { + id + ... on FilesInterface { + filesCount + isFilesEnabled + } + } + } + } + } + } + } +""" + + +def attach_files(target, amount: int) -> None: + document_id = DocumentId.get_or_create_for_object(target) + for i in range(amount): + File.objects.create( + parent=document_id, + file_name=f"file-{i}.png", + file_content_type="image/png", + upload_status=File.UploadStatus.COMPLETED, + ) + + +@override_config(ENABLE_PUBLIC_ID_LOGIC=True) +def test_files_count_is_flat_regardless_of_file_volume(graphql_client_with_queries): + """`filesCount` should be a flat query path: regardless of how many `File` + rows point at the target, the GraphQL query should make the same number of + DB round-trips.""" + target_small = CommentFactory() + attach_files(target_small, 3) + + ContentType.objects.clear_cache() + response_small, queries_small = graphql_client_with_queries( + COUNTS_ONLY_QUERY, variables={"id": target_small.relay_id} + ) + assert response_small.json()["data"]["node"]["filesCount"]["total"] == 3 + small_count = queries_small.count + + target_big = CommentFactory() + attach_files(target_big, 20) + + ContentType.objects.clear_cache() + response_big, queries_big = graphql_client_with_queries( + COUNTS_ONLY_QUERY, variables={"id": target_big.relay_id} + ) + assert response_big.json()["data"]["node"]["filesCount"]["total"] == 20 + + assert queries_big.count == small_count + # Absolute baseline, catches regressions that bump both sides together. + assert small_count == EXPECTED_FILES_INTERFACE_QUERY_COUNT + + +@override_config(ENABLE_PUBLIC_ID_LOGIC=True) +def test_files_count_zero_when_no_files_does_not_extra_query(graphql_client_with_queries): + """A target with no files should resolve in the same query budget as one + with files — the FileTarget row simply doesn't exist yet and the annotation + falls back to the defaults without an extra round-trip.""" + target_with_files = CommentFactory() + attach_files(target_with_files, 3) + + ContentType.objects.clear_cache() + response_with, queries_with = graphql_client_with_queries( + COUNTS_ONLY_QUERY, variables={"id": target_with_files.relay_id} + ) + assert response_with.json()["data"]["node"]["filesCount"]["total"] == 3 + + target_empty = CommentFactory() + + ContentType.objects.clear_cache() + response_empty, queries_empty = graphql_client_with_queries( + COUNTS_ONLY_QUERY, variables={"id": target_empty.relay_id} + ) + assert response_empty.json()["data"]["node"]["filesCount"]["total"] == 0 + + assert queries_empty.count == queries_with.count + assert queries_empty.count == EXPECTED_FILES_INTERFACE_QUERY_COUNT + + +@override_config(ENABLE_PUBLIC_ID_LOGIC=True) +def test_listing_comments_with_files_is_flat(graphql_client_with_queries): + """Listing comments that each have files must not fan out one FileTarget + query per comment — the files annotations ride along in the comment list + SELECT via `pre_optimization_hook`.""" + target_small = CommentFactory() + for _ in range(2): + reply = CommentFactory(target=target_small) + attach_files(reply, 2) + + ContentType.objects.clear_cache() + response_small, queries_small = graphql_client_with_queries( + COMMENTS_WITH_FILES_LIST_QUERY, variables={"id": target_small.relay_id} + ) + edges_small = response_small.json()["data"]["node"]["comments"]["edges"] + assert len(edges_small) == 2 + assert all(edge["node"]["filesCount"]["total"] == 2 for edge in edges_small) + small_count = queries_small.count + + target_big = CommentFactory() + for _ in range(8): + reply = CommentFactory(target=target_big) + attach_files(reply, 3) + + ContentType.objects.clear_cache() + response_big, queries_big = graphql_client_with_queries( + COMMENTS_WITH_FILES_LIST_QUERY, variables={"id": target_big.relay_id} + ) + edges_big = response_big.json()["data"]["node"]["comments"]["edges"] + assert len(edges_big) == 8 + assert all(edge["node"]["filesCount"]["total"] == 3 for edge in edges_big) + + # The query count must not scale with the number of comments or files. + assert queries_big.count == small_count + + +def test_annotate_queryset_evaluates_in_single_query(django_assert_num_queries): + """`FileTarget.annotate_queryset` inlines the metadata as Subqueries — the + annotated queryset must evaluate in exactly one SELECT.""" + Comment = CommentFactory._meta.model + + comment_with_files = CommentFactory() + attach_files(comment_with_files, 2) + comment_disabled = CommentFactory() + file_target = comment_disabled.get_file_target() + file_target.is_files_enabled = False + file_target.save() + comment_bare = CommentFactory() + + ContentType.objects.get_for_model(Comment) # warm the CT cache + + with django_assert_num_queries(1): + rows = { + row.pk: row + for row in FileTarget.annotate_queryset( + Comment.objects.filter( + pk__in=[comment_with_files.pk, comment_disabled.pk, comment_bare.pk] + ) + ) + } + + assert rows[comment_with_files.pk]._file_target_files_count["total"] == 2 + assert rows[comment_with_files.pk]._file_target_is_files_enabled is True + assert rows[comment_with_files.pk].files_count_total == 2 + + assert rows[comment_disabled.pk]._file_target_is_files_enabled is False + + assert rows[comment_bare.pk]._file_target_files_count is None + assert rows[comment_bare.pk]._file_target_is_files_enabled is True + assert rows[comment_bare.pk].files_count_total == 0 From 3c33b0f2b08f00f1ce1145e4806e47a555dd18f3 Mon Sep 17 00:00:00 2001 From: Alisson Patricio Date: Sat, 11 Jul 2026 02:20:18 +0000 Subject: [PATCH 07/20] content_feed: expose CommentsInterface on ContentPost Posts can now be commented (with files via the comments module); the commentable metadata annotation keeps post lists flat, mirroring reactable. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146oeiGsELS6QVpVUQHDQXd --- baseapp/content_feed/graphql/object_types.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/baseapp/content_feed/graphql/object_types.py b/baseapp/content_feed/graphql/object_types.py index 97cfe272..4d49f18a 100644 --- a/baseapp/content_feed/graphql/object_types.py +++ b/baseapp/content_feed/graphql/object_types.py @@ -38,7 +38,7 @@ class ContentPostObjectType(DjangoObjectType): class Meta: interfaces = graphql_shared_interfaces.get( - RelayNode, "MentionsInterface", "ReactionsInterface" + RelayNode, "MentionsInterface", "ReactionsInterface", "CommentsInterface" ) model = ContentPost fields = ( @@ -61,4 +61,7 @@ def get_queryset(cls, queryset, info): # (`reactions_count`, `is_reactions_enabled`) don't N+1 per post. if service := shared_services.get("reactable_metadata"): queryset = service.annotate_queryset(queryset) + # Same for CommentsInterface (`comments_count`, `is_comments_enabled`). + if service := shared_services.get("commentable_metadata"): + queryset = service.annotate_queryset(queryset) return queryset From 0c46112857c8c65bfee2e8ebb884e88e70c24d76 Mon Sep 17 00:00:00 2001 From: Alisson Patricio Date: Sat, 11 Jul 2026 02:35:36 +0000 Subject: [PATCH 08/20] files: backfill test coverage; fix myFiles scoping and attach not_found path New tests: cleanup tasks (expired/failed uploads), local presigned token flow (happy path, tampered/expired/reused tokens, completed-file and bad part numbers), myFiles/file GraphQL queries, attach_files_from_relay_ids unit coverage. Package coverage now ~85%. Fixes surfaced by the new tests: - myFiles had no resolver: it returned every user's files and was listable anonymously. Now scoped to the authenticated user, empty for anonymous. - attach_files_from_relay_ids raised hashids' NoInstanceFound for relay ids whose DocumentId was deleted; now surfaces the intended not_found GraphQLError. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146oeiGsELS6QVpVUQHDQXd --- baseapp/files/graphql/mutations.py | 4 +- baseapp/files/graphql/queries.py | 11 +- baseapp/files/graphql/utils.py | 16 +- baseapp/files/models.py | 6 +- baseapp/files/tests/test_cleanup.py | 190 +++++++++++++++++ baseapp/files/tests/test_graphql.py | 129 ++++++++++++ baseapp/files/tests/test_graphql_utils.py | 146 +++++++++++++ baseapp/files/tests/test_presigned_uploads.py | 198 ++++++++++++++++++ 8 files changed, 688 insertions(+), 12 deletions(-) create mode 100644 baseapp/files/tests/test_cleanup.py create mode 100644 baseapp/files/tests/test_graphql_utils.py create mode 100644 baseapp/files/tests/test_presigned_uploads.py diff --git a/baseapp/files/graphql/mutations.py b/baseapp/files/graphql/mutations.py index fdf42110..e1ca8363 100644 --- a/baseapp/files/graphql/mutations.py +++ b/baseapp/files/graphql/mutations.py @@ -96,9 +96,7 @@ def mutate_and_get_payload(cls, root, info, **input): ) # Object-level permission (FilesPermissionsBackend grants the owner) - if not info.context.user.has_perm( - f"{app_label}.attach_{file_model_name}", file_obj - ): + if not info.context.user.has_perm(f"{app_label}.attach_{file_model_name}", file_obj): raise GraphQLError( str(_("You don't have permission to attach this file")), extensions={"code": "permission_required"}, diff --git a/baseapp/files/graphql/queries.py b/baseapp/files/graphql/queries.py index 4ad18c47..1a108528 100644 --- a/baseapp/files/graphql/queries.py +++ b/baseapp/files/graphql/queries.py @@ -1,4 +1,6 @@ +import graphene import swapper +from django.db import models from graphene_django.filter import DjangoFilterConnectionField from baseapp_core.graphql import Node @@ -8,5 +10,12 @@ class FilesQueries: - my_files = DjangoFilterConnectionField(FileObjectType) + my_files = DjangoFilterConnectionField( + FileObjectType, description="Files created by the current user." + ) file = Node.Field(FileObjectType) + + def resolve_my_files(self, info: graphene.ResolveInfo, **kwargs) -> models.QuerySet: + if not info.context.user.is_authenticated: + return File.objects.none() + return File.objects.filter(created_by=info.context.user) diff --git a/baseapp/files/graphql/utils.py b/baseapp/files/graphql/utils.py index 7ee331b0..7dff48b1 100644 --- a/baseapp/files/graphql/utils.py +++ b/baseapp/files/graphql/utils.py @@ -12,17 +12,23 @@ def attach_files_from_relay_ids(parent, file_ids, user): return File = swapper.load_model("baseapp_files", "File") - file_pks = [get_pk_from_relay_id(file_id) for file_id in file_ids] + not_found_error = GraphQLError( + str(_("One or more files could not be found")), + extensions={"code": "not_found"}, + ) + try: + file_pks = [get_pk_from_relay_id(file_id) for file_id in file_ids] + except Exception: + # Unresolvable relay ids (e.g. the DocumentId row was deleted) must + # surface as the same not_found error as a missing File row. + raise not_found_error from None pk_lookup = [str(pk) for pk in file_pks] files_qs = File.objects.filter(pk__in=file_pks) files_map = {str(file.pk): file for file in files_qs} missing = [pk for pk in pk_lookup if pk not in files_map] if missing: - raise GraphQLError( - str(_("One or more files could not be found")), - extensions={"code": "not_found"}, - ) + raise not_found_error files = [files_map[pk] for pk in pk_lookup] diff --git a/baseapp/files/models.py b/baseapp/files/models.py index 9d4a7943..6503f666 100644 --- a/baseapp/files/models.py +++ b/baseapp/files/models.py @@ -259,9 +259,9 @@ def annotate_queryset(cls, queryset: models.QuerySet) -> models.QuerySet: target__object_id=OuterRef("pk"), ) total_subq = Subquery( - file_target_qs.annotate(total=KeyTextTransform("total", "files_count")).values( - "total" - )[:1] + file_target_qs.annotate(total=KeyTextTransform("total", "files_count")).values("total")[ + :1 + ] ) return queryset.annotate( _file_target_files_count=Subquery(file_target_qs.values("files_count")[:1]), diff --git a/baseapp/files/tests/test_cleanup.py b/baseapp/files/tests/test_cleanup.py new file mode 100644 index 00000000..9ade79ba --- /dev/null +++ b/baseapp/files/tests/test_cleanup.py @@ -0,0 +1,190 @@ +from datetime import timedelta +from typing import Optional +from unittest.mock import MagicMock + +import pytest +import swapper +from django.contrib.auth import get_user_model +from django.utils import timezone + +from baseapp.files.services.cleanup import ( + cleanup_expired_uploads, + cleanup_failed_uploads, +) + +File = swapper.load_model("baseapp_files", "File") +User = get_user_model() + +pytestmark = pytest.mark.django_db + + +@pytest.fixture +def user() -> "User": + return User.objects.create_user(email="cleanup@example.com", password="testpass123") + + +@pytest.fixture +def mock_handler(monkeypatch: pytest.MonkeyPatch) -> MagicMock: + """Replace the storage handler used by the cleanup task.""" + handler = MagicMock() + monkeypatch.setattr("baseapp.files.services.cleanup.get_upload_handler", lambda: handler) + return handler + + +def create_file( + user: "User", + upload_status: str, + expires_delta: Optional[timedelta] = None, + upload_id: Optional[str] = "upload-id-123", +) -> "File": + """Create a File in the given upload status with an optional expiration offset.""" + upload_expires_at = timezone.now() + expires_delta if expires_delta is not None else None + return File.objects.create( + file_name="test.mp4", + file_size=1024, + file_content_type="video/mp4", + upload_status=upload_status, + upload_id=upload_id, + total_parts=1, + created_by=user, + upload_expires_at=upload_expires_at, + ) + + +class TestCleanupExpiredUploads: + """Tests for the cleanup_expired_uploads celery task.""" + + def test_aborts_expired_pending_and_uploading(self, user, mock_handler): + """Expired PENDING and UPLOADING uploads are aborted on storage and in the DB.""" + pending = create_file(user, File.UploadStatus.PENDING, expires_delta=timedelta(hours=-1)) + uploading = create_file( + user, File.UploadStatus.UPLOADING, expires_delta=timedelta(hours=-1) + ) + + result = cleanup_expired_uploads() + + assert result == "Cleaned up 2 expired uploads" + assert mock_handler.abort_upload.call_count == 2 + + for file_obj in (pending, uploading): + file_obj.refresh_from_db() + assert file_obj.upload_status == File.UploadStatus.ABORTED + assert file_obj.upload_id is None + + def test_leaves_non_expired_uploads_alone(self, user, mock_handler): + """Uploads that have not expired yet are untouched.""" + file_obj = create_file(user, File.UploadStatus.UPLOADING, expires_delta=timedelta(hours=1)) + + result = cleanup_expired_uploads() + + assert result == "Cleaned up 0 expired uploads" + mock_handler.abort_upload.assert_not_called() + + file_obj.refresh_from_db() + assert file_obj.upload_status == File.UploadStatus.UPLOADING + assert file_obj.upload_id == "upload-id-123" + + def test_leaves_completed_files_alone(self, user, mock_handler): + """COMPLETED files are never aborted, even with a past expiration date.""" + file_obj = create_file( + user, File.UploadStatus.COMPLETED, expires_delta=timedelta(hours=-1), upload_id=None + ) + + result = cleanup_expired_uploads() + + assert result == "Cleaned up 0 expired uploads" + mock_handler.abort_upload.assert_not_called() + + file_obj.refresh_from_db() + assert file_obj.upload_status == File.UploadStatus.COMPLETED + + def test_skips_storage_abort_without_upload_id(self, user, mock_handler): + """Expired uploads without an upload_id are aborted in the DB only.""" + file_obj = create_file( + user, File.UploadStatus.PENDING, expires_delta=timedelta(hours=-1), upload_id=None + ) + + result = cleanup_expired_uploads() + + assert result == "Cleaned up 1 expired uploads" + mock_handler.abort_upload.assert_not_called() + + file_obj.refresh_from_db() + assert file_obj.upload_status == File.UploadStatus.ABORTED + + def test_continues_past_handler_exception(self, user, mock_handler): + """A storage failure on one file doesn't stop cleanup of the others.""" + failing = create_file(user, File.UploadStatus.UPLOADING, expires_delta=timedelta(hours=-1)) + succeeding = create_file( + user, File.UploadStatus.UPLOADING, expires_delta=timedelta(hours=-1) + ) + + def abort_upload(file_obj, upload_id): + if file_obj.pk == failing.pk: + raise Exception("S3 is down") + + mock_handler.abort_upload.side_effect = abort_upload + + result = cleanup_expired_uploads() + + assert result == "Cleaned up 1 expired uploads" + assert mock_handler.abort_upload.call_count == 2 + + failing.refresh_from_db() + assert failing.upload_status == File.UploadStatus.UPLOADING + + succeeding.refresh_from_db() + assert succeeding.upload_status == File.UploadStatus.ABORTED + + +class TestCleanupFailedUploads: + """Tests for the cleanup_failed_uploads helper.""" + + def _age_file(self, file_obj: "File", days: int) -> None: + """Backdate the modified timestamp bypassing auto_now.""" + File.objects.filter(pk=file_obj.pk).update(modified=timezone.now() - timedelta(days=days)) + + def test_deletes_old_failed_and_aborted(self, user): + """FAILED/ABORTED files older than the cutoff are deleted.""" + failed = create_file(user, File.UploadStatus.FAILED, upload_id=None) + aborted = create_file(user, File.UploadStatus.ABORTED, upload_id=None) + self._age_file(failed, days=8) + self._age_file(aborted, days=8) + + result = cleanup_failed_uploads(days_old=7) + + assert result == "Deleted 2 old failed/aborted uploads" + assert not File.objects.filter(pk__in=[failed.pk, aborted.pk]).exists() + + def test_keeps_recent_failed_uploads(self, user): + """FAILED/ABORTED files newer than the cutoff are kept.""" + failed = create_file(user, File.UploadStatus.FAILED, upload_id=None) + aborted = create_file(user, File.UploadStatus.ABORTED, upload_id=None) + self._age_file(failed, days=3) + + result = cleanup_failed_uploads(days_old=7) + + assert result == "Deleted 0 old failed/aborted uploads" + assert File.objects.filter(pk__in=[failed.pk, aborted.pk]).count() == 2 + + def test_keeps_old_files_in_other_statuses(self, user): + """Old COMPLETED/PENDING/UPLOADING files are never deleted.""" + completed = create_file(user, File.UploadStatus.COMPLETED, upload_id=None) + pending = create_file(user, File.UploadStatus.PENDING) + uploading = create_file(user, File.UploadStatus.UPLOADING) + for file_obj in (completed, pending, uploading): + self._age_file(file_obj, days=30) + + result = cleanup_failed_uploads(days_old=7) + + assert result == "Deleted 0 old failed/aborted uploads" + assert File.objects.count() == 3 + + def test_respects_custom_cutoff(self, user): + """The days_old argument moves the deletion cutoff.""" + failed = create_file(user, File.UploadStatus.FAILED, upload_id=None) + self._age_file(failed, days=2) + + assert cleanup_failed_uploads(days_old=7) == "Deleted 0 old failed/aborted uploads" + assert cleanup_failed_uploads(days_old=1) == "Deleted 1 old failed/aborted uploads" + assert not File.objects.filter(pk=failed.pk).exists() diff --git a/baseapp/files/tests/test_graphql.py b/baseapp/files/tests/test_graphql.py index 355ad4c2..5a53b554 100644 --- a/baseapp/files/tests/test_graphql.py +++ b/baseapp/files/tests/test_graphql.py @@ -615,3 +615,132 @@ def test_update_files_count_after_delete_file(django_user_client, graphql_user_c comment.refresh_from_db() assert comment.files_count["total"] == 0 + + +MY_FILES_QUERY = """ + query MyFiles($noParent: Boolean) { + myFiles(noParent: $noParent) { + edges { + node { + id + fileName + } + } + } + } +""" + +FILE_NODE_QUERY = """ + query GetFile($id: ID!) { + file(id: $id) { + id + fileName + } + } +""" + + +def test_my_files_includes_own_files(graphql_user_client, django_user_client): + own_file = File.objects.create( + file_name="mine.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=django_user_client.user, + ) + + response = graphql_user_client(MY_FILES_QUERY) + content = response.json() + + assert "errors" not in content + edges = content["data"]["myFiles"]["edges"] + assert len(edges) == 1 + assert edges[0]["node"]["id"] == own_file.relay_id + assert edges[0]["node"]["fileName"] == "mine.txt" + + +def test_my_files_excludes_other_users_files(graphql_user_client, django_user_client): + """myFiles is scoped to the authenticated user — other users' files must + never be returned.""" + from django.contrib.auth import get_user_model + + User = get_user_model() + other_user = User.objects.create_user(email="other@test.com", password="pass123") + + File.objects.create( + file_name="theirs.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=other_user, + ) + own_file = File.objects.create( + file_name="mine.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=django_user_client.user, + ) + + response = graphql_user_client(MY_FILES_QUERY) + content = response.json() + + assert "errors" not in content + edges = content["data"]["myFiles"]["edges"] + assert len(edges) == 1 + assert edges[0]["node"]["id"] == own_file.relay_id + + +def test_my_files_no_parent_filter(graphql_user_client, django_user_client): + user = django_user_client.user + comment = CommentFactory(user=user) + comment_content_type = ContentType.objects.get_for_model(comment) + comment_document_id, _ = DocumentId.objects.get_or_create( + content_type=comment_content_type, + object_id=comment.pk, + ) + + standalone = File.objects.create( + file_name="standalone.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=user, + ) + File.objects.create( + file_name="attached.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=user, + parent=comment_document_id, + ) + + response = graphql_user_client(MY_FILES_QUERY, variables={"noParent": True}) + content = response.json() + + assert "errors" not in content + edges = content["data"]["myFiles"]["edges"] + assert len(edges) == 1 + assert edges[0]["node"]["id"] == standalone.relay_id + + +def test_my_files_empty_for_anonymous(graphql_client, django_user_client): + """Anonymous visitors must not be able to list any files via myFiles.""" + File.objects.create( + file_name="mine.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=django_user_client.user, + ) + + response = graphql_client(MY_FILES_QUERY) + content = response.json() + + assert "errors" not in content + edges = content["data"]["myFiles"]["edges"] + assert len(edges) == 0 + + +def test_file_node_query_returns_file_for_owner(graphql_user_client, django_user_client): + file = File.objects.create( + file_name="mine.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=django_user_client.user, + ) + + response = graphql_user_client(FILE_NODE_QUERY, variables={"id": file.relay_id}) + content = response.json() + + assert "errors" not in content + assert content["data"]["file"]["id"] == file.relay_id + assert content["data"]["file"]["fileName"] == "mine.txt" diff --git a/baseapp/files/tests/test_graphql_utils.py b/baseapp/files/tests/test_graphql_utils.py new file mode 100644 index 00000000..2dcb966f --- /dev/null +++ b/baseapp/files/tests/test_graphql_utils.py @@ -0,0 +1,146 @@ +"""Unit tests for baseapp.files.graphql.utils.attach_files_from_relay_ids.""" + +import pytest +import swapper +from django.contrib.auth import get_user_model +from graphql.error import GraphQLError + +from baseapp.files.graphql.utils import attach_files_from_relay_ids +from baseapp_comments.tests.factories import CommentFactory +from baseapp_core.models import DocumentId + +File = swapper.load_model("baseapp_files", "File") +FileTarget = swapper.load_model("baseapp_files", "FileTarget") +User = get_user_model() + +pytestmark = pytest.mark.django_db + + +@pytest.fixture +def user() -> "User": + return User.objects.create_user(email="owner@example.com", password="testpass123") + + +def create_file(user: "User", parent: DocumentId = None, file_name: str = "file.txt") -> "File": + """Create a completed file owned by the given user.""" + return File.objects.create( + file_name=file_name, + upload_status=File.UploadStatus.COMPLETED, + created_by=user, + parent=parent, + ) + + +def test_attaches_files_to_parent(user): + """Provided relay ids are attached to the parent's DocumentId.""" + comment = CommentFactory(user=user) + file1 = create_file(user, file_name="a.txt") + file2 = create_file(user, file_name="b.txt") + + attach_files_from_relay_ids(comment, [file1.relay_id, file2.relay_id], user) + + parent_document_id = DocumentId.get_or_create_for_object(comment) + file1.refresh_from_db() + file2.refresh_from_db() + assert file1.parent == parent_document_id + assert file2.parent == parent_document_id + + +def test_noop_for_none_and_empty_list(user): + """None or an empty list of ids is a no-op.""" + comment = CommentFactory(user=user) + + assert attach_files_from_relay_ids(comment, None, user) is None + assert attach_files_from_relay_ids(comment, [], user) is None + + +def test_skips_files_already_attached_to_same_parent(user): + """Files already attached to the parent are left untouched.""" + comment = CommentFactory(user=user) + parent_document_id = DocumentId.get_or_create_for_object(comment) + file_obj = create_file(user, parent=parent_document_id) + + attach_files_from_relay_ids(comment, [file_obj.relay_id], user) + + file_obj.refresh_from_db() + assert file_obj.parent == parent_document_id + assert comment.files.count() == 1 + + +def test_moves_file_from_previous_parent_and_recounts(user): + """A file attached to another parent is moved and the old parent's count updates.""" + old_comment = CommentFactory(user=user) + new_comment = CommentFactory(user=user) + old_document_id = DocumentId.get_or_create_for_object(old_comment) + file_obj = create_file(user, parent=old_document_id) + + attach_files_from_relay_ids(new_comment, [file_obj.relay_id], user) + + file_obj.refresh_from_db() + assert file_obj.parent == DocumentId.get_or_create_for_object(new_comment) + + old_file_target = FileTarget.objects.get(target=old_document_id) + assert old_file_target.files_count["total"] == 0 + + +def test_missing_file_raises_not_found(user): + """A relay id that doesn't resolve to a File raises a not_found error.""" + comment = CommentFactory(user=user) + # The comment's relay id resolves to a pk with no matching File row. + with pytest.raises(GraphQLError) as exc_info: + attach_files_from_relay_ids(comment, [comment.relay_id], user) + + assert exc_info.value.extensions["code"] == "not_found" + + +def test_deleted_file_relay_id_raises_not_found(user): + """A relay id whose file (and DocumentId) was deleted raises the same + not_found GraphQLError as a missing File row.""" + comment = CommentFactory(user=user) + file_obj = create_file(user) + relay_id = file_obj.relay_id + file_obj.delete() + + with pytest.raises(GraphQLError) as exc_info: + attach_files_from_relay_ids(comment, [relay_id], user) + + assert exc_info.value.extensions["code"] == "not_found" + + +def test_non_owner_cannot_attach(user): + """A user cannot attach files created by someone else.""" + other_user = User.objects.create_user(email="other@example.com", password="testpass123") + comment = CommentFactory(user=user) + file_obj = create_file(other_user) + + with pytest.raises(GraphQLError) as exc_info: + attach_files_from_relay_ids(comment, [file_obj.relay_id], user) + + assert exc_info.value.extensions["code"] == "permission_required" + file_obj.refresh_from_db() + assert file_obj.parent is None + + +def test_superuser_can_attach_any_file(user): + """A superuser can attach files owned by other users.""" + superuser = User.objects.create_user( + email="admin@example.com", password="testpass123", is_superuser=True + ) + comment = CommentFactory(user=user) + file_obj = create_file(user) + + attach_files_from_relay_ids(comment, [file_obj.relay_id], superuser) + + file_obj.refresh_from_db() + assert file_obj.parent == DocumentId.get_or_create_for_object(comment) + + +def test_without_user_skips_ownership_check(user): + """System calls without a user bypass the ownership check.""" + comment = CommentFactory(user=user) + file_obj = create_file(user) + + attach_files_from_relay_ids(comment, [file_obj.relay_id], None) + + file_obj.refresh_from_db() + assert file_obj.parent == DocumentId.get_or_create_for_object(comment) diff --git a/baseapp/files/tests/test_presigned_uploads.py b/baseapp/files/tests/test_presigned_uploads.py new file mode 100644 index 00000000..56af512c --- /dev/null +++ b/baseapp/files/tests/test_presigned_uploads.py @@ -0,0 +1,198 @@ +"""Tests for the presigned upload-part endpoint backed by local storage. + +Exercises the full local upload flow: the initiate REST endpoint (which uses +LocalUploadHandler in tests since default storage is FileSystemStorage) +returns presigned URLs containing signed tokens, and the token-authenticated +PUT endpoint stores the part data. +""" + +from datetime import timedelta +from typing import Any, Dict +from unittest.mock import patch +from urllib.parse import urlparse + +import pytest +import swapper +from django.conf import settings +from django.core import signing +from django.utils import timezone +from freezegun import freeze_time +from rest_framework import status + +File = swapper.load_model("baseapp_files", "File") + +pytestmark = pytest.mark.django_db + +PART_DATA = b"hello world" + + +@pytest.fixture +def temp_media_root(tmp_path): + """Use a temporary directory for media root.""" + with patch.object(settings, "MEDIA_ROOT", str(tmp_path)): + yield tmp_path + + +@pytest.fixture +def initiated_upload(user_client, temp_media_root) -> Dict[str, Any]: + """Initiate a single-part upload through the REST endpoint with local storage.""" + response = user_client.post( + "/v1/files/uploads", + { + "file_name": "doc.txt", + "file_size": len(PART_DATA), + "file_content_type": "text/plain", + "num_parts": 1, + "part_size": len(PART_DATA), + }, + format="json", + ) + assert response.status_code == status.HTTP_201_CREATED + return response.json() + + +def presigned_path(presigned_url: Dict[str, Any]) -> str: + """Extract the relative path (with token query string) from a presigned URL.""" + parsed = urlparse(presigned_url["url"]) + return f"{parsed.path}?{parsed.query}" + + +def put_part(client, path: str, data: bytes = PART_DATA): + """PUT raw binary data to a presigned upload-part path.""" + return client.put(path, data=data, content_type="application/octet-stream") + + +class TestPresignedUploadPart: + """Tests for PUT /v1/files/presigned-uploads/{id}/upload-part/{part}?token=...""" + + def test_upload_part_success(self, client, temp_media_root, initiated_upload): + """A valid token uploads the part and returns the etag in body and header.""" + path = presigned_path(initiated_upload["presigned_urls"][0]) + + response = put_part(client, path) + + assert response.status_code == status.HTTP_200_OK + data = response.json() + assert data["part_number"] == 1 + assert len(data["etag"]) == 32 # MD5 hex digest + assert response["ETag"] == data["etag"] + + part_file = temp_media_root / "temp_uploads" / initiated_upload["upload_id"] / "part_1" + assert part_file.read_bytes() == PART_DATA + + def test_full_upload_flow(self, client, user_client, temp_media_root, initiated_upload): + """Upload a part via presigned URL, then complete the upload.""" + path = presigned_path(initiated_upload["presigned_urls"][0]) + etag = put_part(client, path).json()["etag"] + + response = user_client.post( + f"/v1/files/uploads/{initiated_upload['id']}/complete", + {"parts": [{"part_number": 1, "etag": etag}]}, + format="json", + ) + + assert response.status_code == status.HTTP_200_OK + assert response.json()["upload_status"] == File.UploadStatus.COMPLETED + + file_obj = File.get_by_public_id(initiated_upload["id"]) + assert file_obj.upload_status == File.UploadStatus.COMPLETED + assert (temp_media_root / file_obj.file.name).read_bytes() == PART_DATA + + def test_missing_token(self, client, initiated_upload): + """A request without a token is rejected.""" + path = presigned_path(initiated_upload["presigned_urls"][0]).split("?")[0] + + response = put_part(client, path) + + assert response.status_code == status.HTTP_401_UNAUTHORIZED + assert "Missing token" in response.json()["error"] + + def test_tampered_token(self, client, initiated_upload): + """A token with an invalid signature is rejected.""" + path = presigned_path(initiated_upload["presigned_urls"][0]) + + response = put_part(client, path + "tampered") + + assert response.status_code == status.HTTP_401_UNAUTHORIZED + assert "Invalid token signature" in response.json()["error"] + + def test_token_for_wrong_part(self, client, initiated_upload): + """A valid token cannot be reused for a different part number.""" + parsed = urlparse(initiated_upload["presigned_urls"][0]["url"]) + wrong_part_path = parsed.path.replace("/upload-part/1", "/upload-part/2") + + response = put_part(client, f"{wrong_part_path}?{parsed.query}") + + assert response.status_code == status.HTTP_401_UNAUTHORIZED + assert "Invalid token for this file/part" in response.json()["error"] + + def test_expired_token(self, client, initiated_upload): + """A token older than 1 hour is rejected.""" + file_obj = File.get_by_public_id(initiated_upload["id"]) + + with freeze_time(timezone.now() - timedelta(hours=2)): + token = signing.dumps( + { + "file_id": file_obj.id, + "part_number": 1, + "upload_id": file_obj.upload_id, + } + ) + + response = put_part( + client, f"/v1/files/presigned-uploads/{file_obj.id}/upload-part/1?token={token}" + ) + + assert response.status_code == status.HTTP_401_UNAUTHORIZED + assert "Token has expired" in response.json()["error"] + + def test_upload_part_invalid_status(self, client, initiated_upload): + """Parts cannot be uploaded once the file is no longer pending/uploading.""" + file_obj = File.get_by_public_id(initiated_upload["id"]) + file_obj.upload_status = File.UploadStatus.COMPLETED + file_obj.save(update_fields=["upload_status"]) + + path = presigned_path(initiated_upload["presigned_urls"][0]) + response = put_part(client, path) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert "Cannot upload parts" in response.json()["error"] + + def test_upload_part_number_out_of_range(self, client, initiated_upload): + """A part number above total_parts is rejected even with a valid token.""" + file_obj = File.get_by_public_id(initiated_upload["id"]) + token = signing.dumps( + { + "file_id": file_obj.id, + "part_number": 5, + "upload_id": file_obj.upload_id, + } + ) + + response = put_part( + client, f"/v1/files/presigned-uploads/{file_obj.id}/upload-part/5?token={token}" + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert "Invalid part number" in response.json()["error"] + + def test_token_for_other_file(self, client, user_client, temp_media_root, initiated_upload): + """A token signed for one file cannot be used on another file's URL.""" + other_file = File.objects.create( + file_name="other.txt", + file_size=len(PART_DATA), + file_content_type="text/plain", + upload_status=File.UploadStatus.UPLOADING, + upload_id="other-upload-id", + total_parts=1, + created_by=user_client.user, + ) + + parsed = urlparse(initiated_upload["presigned_urls"][0]["url"]) + response = put_part( + client, + f"/v1/files/presigned-uploads/{other_file.id}/upload-part/1?{parsed.query}", + ) + + assert response.status_code == status.HTTP_401_UNAUTHORIZED + assert "Invalid token for this file/part" in response.json()["error"] From 2d037cf42c905771116cf642f80d048deca90d01 Mon Sep 17 00:00:00 2001 From: Alisson Patricio Date: Sat, 11 Jul 2026 12:39:37 +0000 Subject: [PATCH 09/20] files: address code-review findings (authz, ordering, dedup, dead subquery) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - FileAttachToTarget: authorize the target — reject non-FileableModel targets (invalid_target) and delegate 'may attach to this object' to a new add_ permission (default: authenticated + files enabled) so projects can override it. Previously any authenticated user could attach files to any relay id, and a non-FilesInterface target committed writes then errored on serialization. - FileAttachToTarget: require each file_relay_id to resolve to a File (isinstance) — a non-File relay id (e.g. a Comment) previously hit an uncaught AttributeError/500 in the permission backend. - FileAttachToTarget: materialize the target DocumentId only after validation, so rejected attaches don't create-then-rollback a row. - attach_files_from_relay_ids: a NULL creator is no longer an implicit grant — fixes a file-stealing path that diverged from the mutation's owner-only rule. - resolve_my_files: order by (-created, -pk) — cursor pagination over an unordered queryset duplicated/dropped rows. - AbstractFileTarget: inherit DocumentIdUniqueTargetMixin instead of copying get_for_object + the target field; get_or_create_file_target now returns None for unsaved objects (was IntegrityError). - annotate_queryset: drop the unused files_count_total subquery (no consumer; every comments list paid for it) — keep it available to add when a filter needs it. Tests added for each: non-File / non-fileable-target rejection, NULL-creator steal, unsaved-object guard. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146oeiGsELS6QVpVUQHDQXd --- baseapp/files/graphql/mutations.py | 27 +++++++--- baseapp/files/graphql/queries.py | 4 +- baseapp/files/graphql/utils.py | 10 ++-- baseapp/files/models.py | 49 +++++++----------- baseapp/files/permissions.py | 12 +++++ baseapp/files/tests/test_graphql.py | 51 +++++++++++++++++++ .../test_graphql_queries_object_files.py | 2 - baseapp/files/tests/test_graphql_utils.py | 20 ++++++++ baseapp/files/tests/test_models.py | 9 ++++ baseapp/files/utils.py | 10 ++-- 10 files changed, 145 insertions(+), 49 deletions(-) diff --git a/baseapp/files/graphql/mutations.py b/baseapp/files/graphql/mutations.py index e1ca8363..8039216c 100644 --- a/baseapp/files/graphql/mutations.py +++ b/baseapp/files/graphql/mutations.py @@ -9,6 +9,7 @@ from baseapp_core.models import DocumentId from baseapp_core.plugins import shared_services +from ..base import FileableModel from .interfaces import FilesInterface File = swapper.load_model("baseapp_files", "File") @@ -74,22 +75,35 @@ def mutate_and_get_payload(cls, root, info, **input): extensions={"code": "not_found"}, ) - # Get or create DocumentId for the target - target_document_id = DocumentId.get_or_create_for_object(target) + # Only objects that opt into files (FileableModel) can be targets — this + # also guarantees the payload's `target` field resolves as FilesInterface. + if not isinstance(target, FileableModel): + raise GraphQLError( + str(_("This object does not support file attachments")), + extensions={"code": "invalid_target"}, + ) - # Check if files are enabled for this target + # Target-level authorization: whether this user may attach files to this + # object is delegated to the permission backend so projects can override + # it (default: authenticated + files enabled). Keep the explicit enabled + # check for a precise error code. if service := shared_services.get("files_metadata"): if not service.is_files_enabled(target): raise GraphQLError( str(_("Files are not enabled for this target")), extensions={"code": "files_disabled"}, ) + if not info.context.user.has_perm(f"{app_label}.add_{file_model_name}", target): + raise GraphQLError( + str(_("You don't have permission to attach files to this object")), + extensions={"code": "permission_required"}, + ) - # Get all files and verify the attach permission + # Get all files and verify each is a File the user is allowed to attach. files = [] for file_relay_id in file_relay_ids: file_obj = get_obj_from_relay_id(info, file_relay_id) - if not file_obj: + if not isinstance(file_obj, File): raise GraphQLError( str(_("File not found: {file_id}")).format(file_id=file_relay_id), extensions={"code": "not_found"}, @@ -113,7 +127,8 @@ def mutate_and_get_payload(cls, root, info, **input): files.append(file_obj) - # Attach all files to the target + # All validation passed — now materialize the target DocumentId and attach. + target_document_id = DocumentId.get_or_create_for_object(target) for file_obj in files: file_obj.parent = target_document_id file_obj.save(update_fields=["parent"]) diff --git a/baseapp/files/graphql/queries.py b/baseapp/files/graphql/queries.py index 1a108528..36647cae 100644 --- a/baseapp/files/graphql/queries.py +++ b/baseapp/files/graphql/queries.py @@ -18,4 +18,6 @@ class FilesQueries: def resolve_my_files(self, info: graphene.ResolveInfo, **kwargs) -> models.QuerySet: if not info.context.user.is_authenticated: return File.objects.none() - return File.objects.filter(created_by=info.context.user) + # Explicit total ordering — AbstractFile has no Meta.ordering, and cursor + # pagination over an unordered queryset yields duplicated/missing rows. + return File.objects.filter(created_by=info.context.user).order_by("-created", "-pk") diff --git a/baseapp/files/graphql/utils.py b/baseapp/files/graphql/utils.py index 7dff48b1..89231777 100644 --- a/baseapp/files/graphql/utils.py +++ b/baseapp/files/graphql/utils.py @@ -33,11 +33,11 @@ def attach_files_from_relay_ids(parent, file_ids, user): files = [files_map[pk] for pk in pk_lookup] if user and not user.is_superuser: - unauthorized = [ - file_obj - for file_obj in files - if file_obj.created_by_id and file_obj.created_by_id != user.id - ] + # Only the file's own creator may attach it. A NULL creator is not an + # implicit grant — treating it as one would let any user re-parent + # (steal) system/orphan files, diverging from FileAttachToTarget which + # enforces owner-only via the attach permission. + unauthorized = [file_obj for file_obj in files if file_obj.created_by_id != user.id] if unauthorized: raise GraphQLError( str(_("You don't have permission to perform this action")), diff --git a/baseapp/files/models.py b/baseapp/files/models.py index 6503f666..6e4efa0a 100644 --- a/baseapp/files/models.py +++ b/baseapp/files/models.py @@ -5,16 +5,20 @@ from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ValidationError from django.db import models -from django.db.models import IntegerField, OuterRef, Subquery, Value -from django.db.models.fields.json import KeyTextTransform -from django.db.models.functions import Cast, Coalesce +from django.db.models import OuterRef, Subquery, Value +from django.db.models.functions import Coalesce from django.db.models.signals import class_prepared from django.dispatch import receiver from django.utils.translation import gettext_lazy as _ from model_utils.models import TimeStampedModel from baseapp_core.graphql import RelayModel -from baseapp_core.models import DocumentId, DocumentIdMixin, random_name_in +from baseapp_core.models import ( + DocumentId, + DocumentIdMixin, + DocumentIdUniqueTargetMixin, + random_name_in, +) from .utils import default_files_count @@ -214,7 +218,10 @@ def add_file_target_triggers(sender, **kwargs): sender._meta.triggers.append(file_target_delete_trigger()) -class AbstractFileTarget(models.Model): +class AbstractFileTarget(DocumentIdUniqueTargetMixin): + # Override only to keep the existing related_name/help_text (avoids a no-op + # AlterField migration on the concrete swappable models); get_for_object and + # get_or_create_for_object are inherited from the mixin. target = models.OneToOneField( DocumentId, on_delete=models.CASCADE, @@ -232,25 +239,16 @@ class Meta: def __str__(self): return f"FileTarget for {self.target}" - @classmethod - def get_for_object(cls, obj: models.Model | None) -> "AbstractFileTarget | None": - """Return the FileTarget for the given object, or `None` if not found.""" - if not obj or not getattr(obj, "pk", None): - return None - try: - ct = ContentType.objects.get_for_model(obj) - return cls.objects.get(target__content_type=ct, target__object_id=obj.pk) - except cls.DoesNotExist: - return None - @classmethod def annotate_queryset(cls, queryset: models.QuerySet) -> models.QuerySet: """ - Annotate `queryset` with file-target metadata so resolvers don't N+1. - Adds `_file_target_files_count`, `_file_target_is_files_enabled`, and a - flat public `files_count_total` (integer cast of the `files_count->total` - JSON key) so consumer-side filtering/ORDER BY can use a real expression. - Mirrors `ReactableMetadata.annotate_queryset`. + Annotate `queryset` with file-target metadata so the FilesInterface + resolvers (`files_count`, `is_files_enabled`) don't N+1. Reads the + annotated values via `FilesMetadataService`. Mirrors + `ReactableMetadata.annotate_queryset`, minus a flat count expression — + no consumer filters/orders files by count, so that extra subquery is + omitted; add it here (as reactions does with `reactions_count_total`) + if a FileFilter ever needs it. """ model_cls = queryset.model ct_id = ContentType.objects.get_for_model(model_cls).pk @@ -258,11 +256,6 @@ def annotate_queryset(cls, queryset: models.QuerySet) -> models.QuerySet: target__content_type_id=ct_id, target__object_id=OuterRef("pk"), ) - total_subq = Subquery( - file_target_qs.annotate(total=KeyTextTransform("total", "files_count")).values("total")[ - :1 - ] - ) return queryset.annotate( _file_target_files_count=Subquery(file_target_qs.values("files_count")[:1]), _file_target_is_files_enabled=Coalesce( @@ -272,10 +265,6 @@ def annotate_queryset(cls, queryset: models.QuerySet) -> models.QuerySet: ), Value(True), ), - files_count_total=Coalesce( - Cast(total_subq, output_field=IntegerField()), - Value(0), - ), ) diff --git a/baseapp/files/permissions.py b/baseapp/files/permissions.py index b6e647b5..701d327b 100644 --- a/baseapp/files/permissions.py +++ b/baseapp/files/permissions.py @@ -11,6 +11,18 @@ def authenticate(self, request, **kwargs): return None def has_perm(self, user_obj, perm, obj=None): + if perm == f"{file_app_label}.add_{file_model_name}": + # Target-level: may this user attach files to `obj`? Default policy + # mirrors comments' add_comment — authenticated and files enabled. + # Projects can override this backend to enforce target ownership. + if not obj: + return False + from baseapp_core.plugins import shared_services + + service = shared_services.get("files_metadata") + enabled = service.is_files_enabled(obj) if service else True + return user_obj.is_authenticated and enabled + if perm == f"{file_app_label}.attach_{file_model_name}": if obj: return obj.created_by == user_obj diff --git a/baseapp/files/tests/test_graphql.py b/baseapp/files/tests/test_graphql.py index 5a53b554..5350a419 100644 --- a/baseapp/files/tests/test_graphql.py +++ b/baseapp/files/tests/test_graphql.py @@ -286,6 +286,57 @@ def test_file_attach_to_target_single_file(graphql_user_client, django_user_clie assert data["target"]["filesCount"]["total"] == 1 +def test_file_attach_to_target_rejects_non_file_relay_id(graphql_user_client, django_user_client): + """Passing a non-File relay id (e.g. a Comment) is rejected as not_found + rather than crashing on the permission check / parent assignment.""" + user = django_user_client.user + comment = CommentFactory(user=user) + not_a_file = CommentFactory(user=user) + + response = graphql_user_client( + FILE_ATTACH_TO_TARGET_MUTATION, + variables={ + "fileRelayIds": [not_a_file.relay_id], + "targetObjectId": comment.relay_id, + }, + ) + content = response.json() + + assert "errors" in content + assert content["errors"][0]["extensions"]["code"] == "not_found" + + +def test_file_attach_to_target_rejects_non_fileable_target(graphql_user_client, django_user_client): + """A target that does not opt into files (not a FileableModel) is rejected + with invalid_target instead of committing writes then erroring on the + FilesInterface payload.""" + from baseapp_profiles.tests.factories import ProfileFactory + + user = django_user_client.user + # A Profile is a resolvable node but not a FileableModel, so it cannot be a + # files target. + profile = ProfileFactory() + file = File.objects.create( + file_name="test.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=user, + ) + + response = graphql_user_client( + FILE_ATTACH_TO_TARGET_MUTATION, + variables={ + "fileRelayIds": [file.relay_id], + "targetObjectId": profile.relay_id, + }, + ) + content = response.json() + + assert "errors" in content + assert content["errors"][0]["extensions"]["code"] == "invalid_target" + file.refresh_from_db() + assert file.parent_id is None + + def test_file_attach_to_target_requires_authentication(graphql_client): comment = CommentFactory() file = File.objects.create(file_name="test.txt") diff --git a/baseapp/files/tests/test_graphql_queries_object_files.py b/baseapp/files/tests/test_graphql_queries_object_files.py index edf1d2c2..17550b81 100644 --- a/baseapp/files/tests/test_graphql_queries_object_files.py +++ b/baseapp/files/tests/test_graphql_queries_object_files.py @@ -187,10 +187,8 @@ def test_annotate_queryset_evaluates_in_single_query(django_assert_num_queries): assert rows[comment_with_files.pk]._file_target_files_count["total"] == 2 assert rows[comment_with_files.pk]._file_target_is_files_enabled is True - assert rows[comment_with_files.pk].files_count_total == 2 assert rows[comment_disabled.pk]._file_target_is_files_enabled is False assert rows[comment_bare.pk]._file_target_files_count is None assert rows[comment_bare.pk]._file_target_is_files_enabled is True - assert rows[comment_bare.pk].files_count_total == 0 diff --git a/baseapp/files/tests/test_graphql_utils.py b/baseapp/files/tests/test_graphql_utils.py index 2dcb966f..1b01c153 100644 --- a/baseapp/files/tests/test_graphql_utils.py +++ b/baseapp/files/tests/test_graphql_utils.py @@ -144,3 +144,23 @@ def test_without_user_skips_ownership_check(user): file_obj.refresh_from_db() assert file_obj.parent == DocumentId.get_or_create_for_object(comment) + + +def test_non_owner_cannot_attach_null_creator_file(user): + """A NULL creator is not an implicit grant — a non-owner cannot re-parent + (steal) a system/orphan file whose created_by is NULL.""" + comment = CommentFactory(user=user) + other_parent = CommentFactory(user=user) + orphan = File.objects.create( + file_name="orphan.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=None, + parent=DocumentId.get_or_create_for_object(other_parent), + ) + + with pytest.raises(GraphQLError) as exc_info: + attach_files_from_relay_ids(comment, [orphan.relay_id], user) + + assert exc_info.value.extensions["code"] == "permission_required" + orphan.refresh_from_db() + assert orphan.parent == DocumentId.get_or_create_for_object(other_parent) diff --git a/baseapp/files/tests/test_models.py b/baseapp/files/tests/test_models.py index 8d21731c..6b9b637d 100644 --- a/baseapp/files/tests/test_models.py +++ b/baseapp/files/tests/test_models.py @@ -188,6 +188,15 @@ def test_get_or_create_file_target_gets_existing(self): self.assertEqual(file_target1.pk, file_target2.pk) self.assertEqual(FileTarget.objects.count(), 1) + def test_get_or_create_file_target_returns_none_for_unsaved_object(self): + """An unsaved object has no pk, so the mixin guards against a NULL + primary key instead of raising IntegrityError.""" + Comment = CommentFactory._meta.model + + self.assertIsNone(get_or_create_file_target(Comment())) + self.assertIsNone(get_or_create_file_target(None)) + self.assertEqual(FileTarget.objects.count(), 0) + def test_recalculate_files_count_with_none(self): recalculate_files_count(None) diff --git a/baseapp/files/utils.py b/baseapp/files/utils.py index 35913c9c..2109a8a1 100644 --- a/baseapp/files/utils.py +++ b/baseapp/files/utils.py @@ -12,12 +12,12 @@ def default_files_count(): def get_or_create_file_target(target_obj): - """Get or create a FileTarget for the given object.""" - FileTarget = swapper.load_model("baseapp_files", "FileTarget") + """Get or create a FileTarget for the given object. - document_id = DocumentId.get_or_create_for_object(target_obj) - file_target, created = FileTarget.objects.get_or_create(target=document_id) - return file_target + Returns None for an unsaved/None object (the mixin guards against a NULL + primary key rather than raising IntegrityError).""" + FileTarget = swapper.load_model("baseapp_files", "FileTarget") + return FileTarget.get_or_create_for_object(target_obj) def recalculate_files_count(parent): From 140cc44914e9f02f5240ae56c95fdaffdb4fbe92 Mon Sep 17 00:00:00 2001 From: Alisson Patricio Date: Sat, 11 Jul 2026 23:59:27 +0000 Subject: [PATCH 10/20] files: address SonarCloud new-code findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - permissions.py: reduce FilesPermissionsBackend.has_perm cognitive complexity 36->well under 15 (S3776) by extracting _can_add_to_target and _is_owner_or_has_global; behavior unchanged. - cleanup.py: logger.exception() with a lazy %-arg instead of logger.error on an f-string in an except block (S8572). - storage base/local/s3: get_file_url annotated Optional[str] since it returns None when no file is stored (S5886). - storage/local.py: drop the unused file_obj param from upload_part (S1172) — a local-only method (S3 uploads go direct); updated the one caller and test. Left intentionally: the S117 'rename File/FileTarget' hits are module-level swapper.load_model() model aliases (PascalCase is correct; the whole codebase uses this idiom), and the S6553 null=True flags are on already-deployed AbstractFile string fields where dropping null=True is a schema/data-migration decision, not a lint fix. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146oeiGsELS6QVpVUQHDQXd --- baseapp/files/permissions.py | 77 +++++++------------ .../rest_framework/uploads/presigned_views.py | 1 - baseapp/files/services/cleanup.py | 6 +- baseapp/files/storage/base.py | 6 +- baseapp/files/storage/local.py | 8 +- baseapp/files/storage/s3.py | 6 +- baseapp/files/tests/test_storage.py | 2 +- 7 files changed, 43 insertions(+), 63 deletions(-) diff --git a/baseapp/files/permissions.py b/baseapp/files/permissions.py index 701d327b..8e893157 100644 --- a/baseapp/files/permissions.py +++ b/baseapp/files/permissions.py @@ -10,58 +10,39 @@ class FilesPermissionsBackend(BaseBackend): def authenticate(self, request, **kwargs): return None - def has_perm(self, user_obj, perm, obj=None): - if perm == f"{file_app_label}.add_{file_model_name}": - # Target-level: may this user attach files to `obj`? Default policy - # mirrors comments' add_comment — authenticated and files enabled. - # Projects can override this backend to enforce target ownership. - if not obj: - return False - from baseapp_core.plugins import shared_services - - service = shared_services.get("files_metadata") - enabled = service.is_files_enabled(obj) if service else True - return user_obj.is_authenticated and enabled - - if perm == f"{file_app_label}.attach_{file_model_name}": - if obj: - return obj.created_by == user_obj + @staticmethod + def _can_add_to_target(user_obj, obj): + """Target-level: may this user attach files to `obj`? Default policy + mirrors comments' add_comment — authenticated and files enabled. + Projects can override this backend to enforce target ownership.""" + if not obj: return False + from baseapp_core.plugins import shared_services - if perm == f"{file_app_label}.view_{file_model_name}": - if obj: - # Owner can view their own files - if obj.created_by == user_obj: - return True - # Check if user has global permission (for admins with view_file perm) - # This is needed because ModelBackend doesn't grant global perms with obj - if user_obj.has_perm(perm): - return True - return False - return False + service = shared_services.get("files_metadata") + enabled = service.is_files_enabled(obj) if service else True + return user_obj.is_authenticated and enabled - if perm == f"{file_app_label}.change_{file_model_name}": - if obj: - # Owner can change their own files - if obj.created_by == user_obj: - return True - # Check if user has global permission (for admins with change_file perm) - # This is needed because ModelBackend doesn't grant global perms with obj - if user_obj.has_perm(perm): - return True - return False + @staticmethod + def _is_owner_or_has_global(user_obj, perm, obj): + """Owner acts on their own file; otherwise fall back to the global perm + (ModelBackend doesn't grant global perms when an obj is passed).""" + if not obj: return False + return obj.created_by == user_obj or user_obj.has_perm(perm) - if perm == f"{file_app_label}.delete_{file_model_name}": - if obj: - # Owner can delete their own files - if obj.created_by == user_obj: - return True - # Check if user has global permission (for admins with delete_file perm) - # This is needed because ModelBackend doesn't grant global perms with obj - if user_obj.has_perm(perm): - return True - return False - return False + def has_perm(self, user_obj, perm, obj=None): + if perm == f"{file_app_label}.add_{file_model_name}": + return self._can_add_to_target(user_obj, obj) + + if perm == f"{file_app_label}.attach_{file_model_name}": + return bool(obj) and obj.created_by == user_obj + + if perm in ( + f"{file_app_label}.view_{file_model_name}", + f"{file_app_label}.change_{file_model_name}", + f"{file_app_label}.delete_{file_model_name}", + ): + return self._is_owner_or_has_global(user_obj, perm, obj) return False diff --git a/baseapp/files/rest_framework/uploads/presigned_views.py b/baseapp/files/rest_framework/uploads/presigned_views.py index 6a030292..cb0f759f 100644 --- a/baseapp/files/rest_framework/uploads/presigned_views.py +++ b/baseapp/files/rest_framework/uploads/presigned_views.py @@ -121,7 +121,6 @@ def upload_part(self, request, pk=None, part_number=None): # Upload the part using the storage handler etag = self.upload_service.handler.upload_part( - file_obj=file_obj, upload_id=file_obj.upload_id, part_number=part_number, data=data, diff --git a/baseapp/files/services/cleanup.py b/baseapp/files/services/cleanup.py index 64ffc41d..7fc696ac 100644 --- a/baseapp/files/services/cleanup.py +++ b/baseapp/files/services/cleanup.py @@ -39,9 +39,9 @@ def cleanup_expired_uploads(): file_obj.save() cleaned_count += 1 - except Exception as e: - # Log error but continue - logger.error(f"Failed to cleanup file {file_obj.id}: {e}") + except Exception: + # Log with traceback but continue with the next file. + logger.exception("Failed to cleanup file %s", file_obj.id) logger.info(f"Cleaned up {cleaned_count} expired uploads") return f"Cleaned up {cleaned_count} expired uploads" diff --git a/baseapp/files/storage/base.py b/baseapp/files/storage/base.py index dfba8d84..9e1fc61e 100644 --- a/baseapp/files/storage/base.py +++ b/baseapp/files/storage/base.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional class BaseUploadHandler(ABC): @@ -46,6 +46,6 @@ def abort_upload(self, file_obj, upload_id: str) -> None: pass @abstractmethod - def get_file_url(self, file_obj) -> str: - """Get the URL for accessing the file.""" + def get_file_url(self, file_obj) -> Optional[str]: + """Get the URL for accessing the file, or None if not available yet.""" pass diff --git a/baseapp/files/storage/local.py b/baseapp/files/storage/local.py index 2e9bd3bd..08267c7b 100644 --- a/baseapp/files/storage/local.py +++ b/baseapp/files/storage/local.py @@ -1,7 +1,7 @@ import os import shutil import uuid -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional from django.conf import settings from django.core import signing @@ -63,7 +63,7 @@ def initiate_upload(self, file_obj, num_parts: int, part_size: int) -> Dict[str, "expires_in": 3600, } - def upload_part(self, file_obj, upload_id: str, part_number: int, data: bytes) -> str: + def upload_part(self, upload_id: str, part_number: int, data: bytes) -> str: """ Upload a part to local storage. This method is called by the backend when handling local uploads. @@ -118,8 +118,8 @@ def abort_upload(self, file_obj, upload_id: str): temp_dir = os.path.join(settings.MEDIA_ROOT, "temp_uploads", upload_id) shutil.rmtree(temp_dir, ignore_errors=True) - def get_file_url(self, file_obj) -> str: - """Get the local file URL.""" + def get_file_url(self, file_obj) -> Optional[str]: + """Get the local file URL, or None if the file isn't stored yet.""" if file_obj.file: return file_obj.file.url return None diff --git a/baseapp/files/storage/s3.py b/baseapp/files/storage/s3.py index fdfa906b..4af94420 100644 --- a/baseapp/files/storage/s3.py +++ b/baseapp/files/storage/s3.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional import boto3 from botocore.exceptions import ClientError @@ -118,8 +118,8 @@ def abort_upload(self, file_obj, upload_id: str): if e.response["Error"]["Code"] != "NoSuchUpload": raise - def get_file_url(self, file_obj) -> str: - """Get the S3 URL for the file.""" + def get_file_url(self, file_obj) -> Optional[str]: + """Get the S3 URL for the file, or None if the file isn't stored yet.""" if file_obj.file: return file_obj.file.url return None diff --git a/baseapp/files/tests/test_storage.py b/baseapp/files/tests/test_storage.py index 34344300..4def9892 100644 --- a/baseapp/files/tests/test_storage.py +++ b/baseapp/files/tests/test_storage.py @@ -189,7 +189,7 @@ def test_local_upload_part(self, local_handler, file_obj, temp_media_root): upload_id = "test-upload-123" data = b"test file content" - etag = local_handler.upload_part(file_obj, upload_id, part_number=1, data=data) + etag = local_handler.upload_part(upload_id, part_number=1, data=data) assert etag # Should return MD5 hash assert len(etag) == 32 # MD5 hash length From 5c39f5bec6a5459242213701d4180fd2d0f9f204 Mon Sep 17 00:00:00 2001 From: Alisson Patricio Date: Sun, 12 Jul 2026 00:58:13 +0000 Subject: [PATCH 11/20] files: drop null=True on AbstractFile string fields (Sonar S6553) String fields now use blank=True + default="" instead of null=True, removing the NULL-vs-"" ambiguity Sonar flags. upload_id follows suit ("" means no active multipart upload); its three None assignments and truthiness checks already work with "". testproject migration converts any existing NULLs to "" before enforcing NOT NULL, so it is safe on populated databases too. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146oeiGsELS6QVpVUQHDQXd --- baseapp/files/models.py | 12 ++-- baseapp/files/services/cleanup.py | 2 +- baseapp/files/services/upload_service.py | 4 +- baseapp/files/tests/test_cleanup.py | 18 +++--- baseapp/files/tests/test_rest_api.py | 6 +- ...n_alter_file_file_content_type_and_more.py | 55 +++++++++++++++++++ 6 files changed, 76 insertions(+), 21 deletions(-) create mode 100644 testproject/files/migrations/0002_alter_file_description_alter_file_file_content_type_and_more.py diff --git a/baseapp/files/models.py b/baseapp/files/models.py index 6e4efa0a..34fb241d 100644 --- a/baseapp/files/models.py +++ b/baseapp/files/models.py @@ -298,13 +298,13 @@ class AbstractFile(*file_inheritances, DocumentIdMixin, RelayModel, TimeStampedM help_text=_("The parent document this file belongs to"), ) - file_content_type = models.CharField(max_length=150, null=True, blank=True) - file_name = models.CharField(max_length=512, null=True, blank=True) + file_content_type = models.CharField(max_length=150, blank=True, default="") + file_name = models.CharField(max_length=512, blank=True, default="") file_size = models.PositiveIntegerField(null=True, help_text=_("File size in bytes")) file = models.FileField(max_length=512, upload_to=random_name_in("files")) - name = models.CharField(max_length=512, null=True, blank=True) - description = models.TextField(null=True, blank=True) + name = models.CharField(max_length=512, blank=True, default="") + description = models.TextField(blank=True, default="") created_by = models.ForeignKey( settings.AUTH_USER_MODEL, @@ -329,11 +329,11 @@ class UploadStatus(models.TextChoices): help_text=_("Status of the file upload"), ) - # S3 multipart upload tracking + # S3 multipart upload tracking ("" when there is no active multipart upload) upload_id = models.CharField( max_length=512, - null=True, blank=True, + default="", help_text=_("S3 multipart upload ID for in-progress uploads"), ) diff --git a/baseapp/files/services/cleanup.py b/baseapp/files/services/cleanup.py index 7fc696ac..6e73e628 100644 --- a/baseapp/files/services/cleanup.py +++ b/baseapp/files/services/cleanup.py @@ -35,7 +35,7 @@ def cleanup_expired_uploads(): handler.abort_upload(file_obj, file_obj.upload_id) file_obj.upload_status = File.UploadStatus.ABORTED - file_obj.upload_id = None + file_obj.upload_id = "" file_obj.save() cleaned_count += 1 diff --git a/baseapp/files/services/upload_service.py b/baseapp/files/services/upload_service.py index 0e7293d8..db23da42 100644 --- a/baseapp/files/services/upload_service.py +++ b/baseapp/files/services/upload_service.py @@ -103,7 +103,7 @@ def complete_multipart_upload(self, file_id: int, parts: list): file_obj.file = file_path file_obj.upload_status = File.UploadStatus.COMPLETED file_obj.uploaded_parts = {str(p["part_number"]): p["etag"] for p in parts} - file_obj.upload_id = None + file_obj.upload_id = "" file_obj.upload_expires_at = None file_obj.save() @@ -129,7 +129,7 @@ def abort_multipart_upload(self, file_id: int): pass file_obj.upload_status = File.UploadStatus.ABORTED - file_obj.upload_id = None + file_obj.upload_id = "" file_obj.save() return file_obj diff --git a/baseapp/files/tests/test_cleanup.py b/baseapp/files/tests/test_cleanup.py index 9ade79ba..c0fb62cb 100644 --- a/baseapp/files/tests/test_cleanup.py +++ b/baseapp/files/tests/test_cleanup.py @@ -69,7 +69,7 @@ def test_aborts_expired_pending_and_uploading(self, user, mock_handler): for file_obj in (pending, uploading): file_obj.refresh_from_db() assert file_obj.upload_status == File.UploadStatus.ABORTED - assert file_obj.upload_id is None + assert file_obj.upload_id == "" def test_leaves_non_expired_uploads_alone(self, user, mock_handler): """Uploads that have not expired yet are untouched.""" @@ -87,7 +87,7 @@ def test_leaves_non_expired_uploads_alone(self, user, mock_handler): def test_leaves_completed_files_alone(self, user, mock_handler): """COMPLETED files are never aborted, even with a past expiration date.""" file_obj = create_file( - user, File.UploadStatus.COMPLETED, expires_delta=timedelta(hours=-1), upload_id=None + user, File.UploadStatus.COMPLETED, expires_delta=timedelta(hours=-1), upload_id="" ) result = cleanup_expired_uploads() @@ -101,7 +101,7 @@ def test_leaves_completed_files_alone(self, user, mock_handler): def test_skips_storage_abort_without_upload_id(self, user, mock_handler): """Expired uploads without an upload_id are aborted in the DB only.""" file_obj = create_file( - user, File.UploadStatus.PENDING, expires_delta=timedelta(hours=-1), upload_id=None + user, File.UploadStatus.PENDING, expires_delta=timedelta(hours=-1), upload_id="" ) result = cleanup_expired_uploads() @@ -146,8 +146,8 @@ def _age_file(self, file_obj: "File", days: int) -> None: def test_deletes_old_failed_and_aborted(self, user): """FAILED/ABORTED files older than the cutoff are deleted.""" - failed = create_file(user, File.UploadStatus.FAILED, upload_id=None) - aborted = create_file(user, File.UploadStatus.ABORTED, upload_id=None) + failed = create_file(user, File.UploadStatus.FAILED, upload_id="") + aborted = create_file(user, File.UploadStatus.ABORTED, upload_id="") self._age_file(failed, days=8) self._age_file(aborted, days=8) @@ -158,8 +158,8 @@ def test_deletes_old_failed_and_aborted(self, user): def test_keeps_recent_failed_uploads(self, user): """FAILED/ABORTED files newer than the cutoff are kept.""" - failed = create_file(user, File.UploadStatus.FAILED, upload_id=None) - aborted = create_file(user, File.UploadStatus.ABORTED, upload_id=None) + failed = create_file(user, File.UploadStatus.FAILED, upload_id="") + aborted = create_file(user, File.UploadStatus.ABORTED, upload_id="") self._age_file(failed, days=3) result = cleanup_failed_uploads(days_old=7) @@ -169,7 +169,7 @@ def test_keeps_recent_failed_uploads(self, user): def test_keeps_old_files_in_other_statuses(self, user): """Old COMPLETED/PENDING/UPLOADING files are never deleted.""" - completed = create_file(user, File.UploadStatus.COMPLETED, upload_id=None) + completed = create_file(user, File.UploadStatus.COMPLETED, upload_id="") pending = create_file(user, File.UploadStatus.PENDING) uploading = create_file(user, File.UploadStatus.UPLOADING) for file_obj in (completed, pending, uploading): @@ -182,7 +182,7 @@ def test_keeps_old_files_in_other_statuses(self, user): def test_respects_custom_cutoff(self, user): """The days_old argument moves the deletion cutoff.""" - failed = create_file(user, File.UploadStatus.FAILED, upload_id=None) + failed = create_file(user, File.UploadStatus.FAILED, upload_id="") self._age_file(failed, days=2) assert cleanup_failed_uploads(days_old=7) == "Deleted 0 old failed/aborted uploads" diff --git a/baseapp/files/tests/test_rest_api.py b/baseapp/files/tests/test_rest_api.py index 326b2687..fbcfa935 100644 --- a/baseapp/files/tests/test_rest_api.py +++ b/baseapp/files/tests/test_rest_api.py @@ -238,7 +238,7 @@ def test_complete_upload_success(self, user_client, pending_upload, mock_s3_hand # Check file was updated in database pending_upload.refresh_from_db() assert pending_upload.upload_status == File.UploadStatus.COMPLETED - assert pending_upload.upload_id is None + assert pending_upload.upload_id == "" assert pending_upload.uploaded_parts == {"1": "abc123", "2": "def456"} # Verify S3 handler was called @@ -364,7 +364,7 @@ def test_abort_upload_success(self, user_client, uploading_file, mock_s3_handler # Check file was updated uploading_file.refresh_from_db() assert uploading_file.upload_status == File.UploadStatus.ABORTED - assert uploading_file.upload_id is None + assert uploading_file.upload_id == "" # Verify S3 handler was called mock_s3_handler.abort_upload.assert_called_once_with(uploading_file, "test-upload-id") @@ -385,7 +385,7 @@ def test_abort_upload_without_upload_id(self, user_client, mock_s3_handler): file_size=10485760, file_content_type="video/mp4", upload_status=File.UploadStatus.PENDING, - upload_id=None, # No upload_id + upload_id="", # No upload_id total_parts=2, created_by=user_client.user, ) diff --git a/testproject/files/migrations/0002_alter_file_description_alter_file_file_content_type_and_more.py b/testproject/files/migrations/0002_alter_file_description_alter_file_file_content_type_and_more.py new file mode 100644 index 00000000..c3b43f22 --- /dev/null +++ b/testproject/files/migrations/0002_alter_file_description_alter_file_file_content_type_and_more.py @@ -0,0 +1,55 @@ +# Generated by Django 5.2.14 on 2026-07-12 00:54 + +from django.db import migrations, models + +# These string fields dropped null=True in favour of default="". Convert any +# existing NULLs to "" before the AlterField enforces NOT NULL, so the +# migration is safe on databases populated by the previous schema. +NULLABLE_STRING_FIELDS = ("description", "file_content_type", "file_name", "name", "upload_id") + + +def fill_null_strings(apps, schema_editor): + File = apps.get_model("files", "File") + for field in NULLABLE_STRING_FIELDS: + File.objects.filter(**{f"{field}__isnull": True}).update(**{field: ""}) + + +class Migration(migrations.Migration): + + dependencies = [ + ("files", "0001_initial"), + ] + + operations = [ + migrations.RunPython(fill_null_strings, migrations.RunPython.noop), + migrations.AlterField( + model_name="file", + name="description", + field=models.TextField(blank=True, default=""), + ), + migrations.AlterField( + model_name="file", + name="file_content_type", + field=models.CharField(blank=True, default="", max_length=150), + ), + migrations.AlterField( + model_name="file", + name="file_name", + field=models.CharField(blank=True, default="", max_length=512), + ), + migrations.AlterField( + model_name="file", + name="name", + field=models.CharField(blank=True, default="", max_length=512), + ), + migrations.AlterField( + model_name="file", + name="upload_id", + field=models.CharField( + blank=True, + default="", + help_text="S3 multipart upload ID for in-progress uploads", + max_length=512, + ), + ), + ] From 65af75b8c4fc0a8c5ca02511da84366f9d0c9712 Mon Sep 17 00:00:00 2001 From: Alisson Patricio Date: Sun, 12 Jul 2026 01:03:32 +0000 Subject: [PATCH 12/20] core graphql: remove commented-out debug code in parse_body (Sonar S125) Dead logging/pdb.set_trace comment lines left in the multipart parse_body. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146oeiGsELS6QVpVUQHDQXd --- baseapp_core/graphql/views.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/baseapp_core/graphql/views.py b/baseapp_core/graphql/views.py index aee37cf6..a29f4357 100644 --- a/baseapp_core/graphql/views.py +++ b/baseapp_core/graphql/views.py @@ -65,11 +65,8 @@ def execute_graphql_request( def parse_body(self, request): """Handle multipart request spec for multipart/form-data""" content_type = self.get_content_type(request) - # logging.info('content_type: %s' % content_type) - # import pdb; pdb.set_trace() if content_type == "multipart/form-data" and "operations" in request.POST: operations = json.loads(request.POST.get("operations", "{}")) - # import pdb; pdb.set_trace() files_map = json.loads(request.POST.get("map", "{}")) from graphene_file_upload.utils import place_files_in_operations From 1e6728bd7a62d768a84437b2456d568bf28b0ea6 Mon Sep 17 00:00:00 2001 From: Alisson Patricio Date: Sun, 12 Jul 2026 02:53:50 +0000 Subject: [PATCH 13/20] =?UTF-8?q?files:=20address=20PR=20review=20?= =?UTF-8?q?=E2=80=94=20REST=20target=20authz,=20error=20hygiene,=20validat?= =?UTF-8?q?ion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Copilot / CodeQL review on PR #442: - REST initiate (parent_id) and files set-parent now enforce the same target authorization as fileAttachToTarget (FileableModel + add_ permission) via a shared enforce_can_attach_to_parent helper — previously these could attach files to arbitrary objects, bypassing the mutation's checks. Tests updated to use a files-enabled Comment target + new tests asserting non-fileable targets are rejected. - Stop leaking exception text to clients: the generic except blocks in the upload views now logger.exception() and return a generic message (CodeQL 'information exposure through an exception'). - presigned upload_part: bind the signed token to the file's current upload_id (block reuse of a token from a prior initiation) and return a clear error when the active storage handler has no upload_part (S3). - upload_service._validate_file_params: enforce the upper file-size bound too, mirroring InitiateUploadSerializer, so non-REST callers are protected. - recalculate_files_count: Count('id') (not the nullable content type) and bucket NULL content types under 'unknown', matching the pgtrigger. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146oeiGsELS6QVpVUQHDQXd --- baseapp/files/rest_framework/files/views.py | 7 +- .../rest_framework/uploads/presigned_views.py | 25 ++++++- baseapp/files/rest_framework/uploads/views.py | 26 +++++-- baseapp/files/rest_framework/utils.py | 31 ++++++++ baseapp/files/services/upload_service.py | 8 +- baseapp/files/storage/local.py | 2 +- baseapp/files/tests/test_rest_api.py | 75 ++++++++++++++----- baseapp/files/utils.py | 10 ++- 8 files changed, 150 insertions(+), 34 deletions(-) create mode 100644 baseapp/files/rest_framework/utils.py diff --git a/baseapp/files/rest_framework/files/views.py b/baseapp/files/rest_framework/files/views.py index 1b870884..5938dde7 100644 --- a/baseapp/files/rest_framework/files/views.py +++ b/baseapp/files/rest_framework/files/views.py @@ -8,6 +8,7 @@ from ..uploads.permissions import IsOwnerOrReadOnly from ..uploads.serializers import SetParentSerializer +from ..utils import enforce_can_attach_to_parent from .serializers import FileSerializer File = swapper.load_model("baseapp_files", "File") @@ -90,7 +91,11 @@ def set_parent(self, request, pk=None): serializer = self.get_serializer(data=request.data) serializer.is_valid(raise_exception=True) - file_obj.parent_id = serializer.validated_data["parent_id"] + parent_pk = serializer.validated_data["parent_id"] + # Enforce target-level authorization, mirroring fileAttachToTarget. + enforce_can_attach_to_parent(request.user, parent_pk) + + file_obj.parent_id = parent_pk file_obj.save() return Response( diff --git a/baseapp/files/rest_framework/uploads/presigned_views.py b/baseapp/files/rest_framework/uploads/presigned_views.py index cb0f759f..9956b805 100644 --- a/baseapp/files/rest_framework/uploads/presigned_views.py +++ b/baseapp/files/rest_framework/uploads/presigned_views.py @@ -1,3 +1,5 @@ +import logging + import swapper from django.core import signing from django.views.decorators.csrf import csrf_exempt @@ -9,6 +11,8 @@ from ...services.upload_service import UploadService +logger = logging.getLogger(__name__) + File = swapper.load_model("baseapp_files", "File") @@ -99,6 +103,14 @@ def upload_part(self, request, pk=None, part_number=None): status=status.HTTP_404_NOT_FOUND, ) + # Bind the token to the current upload session: a token minted for a + # previous initiation (different upload_id) must not upload into a new one. + if str(token_data.get("upload_id")) != str(file_obj.upload_id): + return Response( + {"error": "Token does not match the current upload session"}, + status=status.HTTP_401_UNAUTHORIZED, + ) + # Validate state if file_obj.upload_status not in [File.UploadStatus.PENDING, File.UploadStatus.UPLOADING]: return Response( @@ -106,6 +118,14 @@ def upload_part(self, request, pk=None, part_number=None): status=status.HTTP_400_BAD_REQUEST, ) + # Only the local handler stores parts through this endpoint; S3 uploads + # go straight to S3, so its handler has no upload_part. + if not hasattr(self.upload_service.handler, "upload_part"): + return Response( + {"error": "The active storage backend does not support presigned part uploads"}, + status=status.HTTP_400_BAD_REQUEST, + ) + try: part_number = int(part_number) @@ -137,5 +157,6 @@ def upload_part(self, request, pk=None, part_number=None): except ValueError as e: raise ValidationError(str(e)) - except Exception as e: - raise ValidationError(f"Failed to upload part: {str(e)}") + except Exception: + logger.exception("Failed to upload part") + raise ValidationError("Failed to upload part.") diff --git a/baseapp/files/rest_framework/uploads/views.py b/baseapp/files/rest_framework/uploads/views.py index 8766b615..d34c7f36 100644 --- a/baseapp/files/rest_framework/uploads/views.py +++ b/baseapp/files/rest_framework/uploads/views.py @@ -1,3 +1,5 @@ +import logging + import swapper from rest_framework import permissions, status, viewsets from rest_framework.exceptions import ValidationError @@ -7,12 +9,15 @@ from baseapp_profiles.rest_framework import CurrentProfileMixin from ...services.upload_service import UploadService +from ..utils import enforce_can_attach_to_parent from .serializers import ( CompleteUploadSerializer, InitiateUploadSerializer, UploadResponseSerializer, ) +logger = logging.getLogger(__name__) + File = swapper.load_model("baseapp_files", "File") file_app_label = File._meta.app_label file_model_name = File._meta.model_name.lower() @@ -79,6 +84,12 @@ def create(self, request): serializer = self.get_serializer(data=request.data) serializer.is_valid(raise_exception=True) + # Authorize the target before creating anything, mirroring the + # fileAttachToTarget mutation so this endpoint can't bypass it. + parent_pk = serializer.validated_data.get("parent_id") + if parent_pk: + enforce_can_attach_to_parent(request.user, parent_pk) + try: # Get user's profile if available profile = request.user.current_profile @@ -95,8 +106,9 @@ def create(self, request): except ValueError as e: raise ValidationError(str(e)) - except Exception as e: - raise ValidationError(f"Failed to initiate upload: {str(e)}") + except Exception: + logger.exception("Failed to initiate upload") + raise ValidationError("Failed to initiate upload.") @action(detail=True, methods=["post"]) def complete(self, request, pk=None): @@ -138,8 +150,9 @@ def complete(self, request, pk=None): except ValueError as e: raise ValidationError(str(e)) - except Exception as e: - raise ValidationError(f"Failed to complete upload: {str(e)}") + except Exception: + logger.exception("Failed to complete upload") + raise ValidationError("Failed to complete upload.") def destroy(self, request, pk=None): """ @@ -160,5 +173,6 @@ def destroy(self, request, pk=None): self.upload_service.abort_multipart_upload(file_obj.id) return Response(status=status.HTTP_204_NO_CONTENT) - except Exception as e: - raise ValidationError(f"Failed to abort upload: {str(e)}") + except Exception: + logger.exception("Failed to abort upload") + raise ValidationError("Failed to abort upload.") diff --git a/baseapp/files/rest_framework/utils.py b/baseapp/files/rest_framework/utils.py new file mode 100644 index 00000000..52280bc9 --- /dev/null +++ b/baseapp/files/rest_framework/utils.py @@ -0,0 +1,31 @@ +import swapper +from rest_framework.exceptions import PermissionDenied, ValidationError + +from baseapp_core.models import DocumentId + +from ..base import FileableModel + +File = swapper.load_model("baseapp_files", "File") +file_app_label = File._meta.app_label +file_model_name = File._meta.model_name.lower() + + +def enforce_can_attach_to_parent(user, parent_document_pk) -> None: + """ + Authorize attaching a file to the object behind ``parent_document_pk`` (a + ``DocumentId`` pk). Mirrors the ``FileAttachToTarget`` GraphQL mutation so + the REST upload/set-parent paths cannot bypass target-level authorization: + the target must opt into files (``FileableModel``) and the user must hold + the ``add_`` permission on it. + """ + try: + document = DocumentId.objects.get(pk=parent_document_pk) + except DocumentId.DoesNotExist: + raise ValidationError("Invalid parent: target not found.") + + target = document.content_object + if not isinstance(target, FileableModel): + raise ValidationError("This object does not support file attachments.") + + if not user.has_perm(f"{file_app_label}.add_{file_model_name}", target): + raise PermissionDenied("You don't have permission to attach files to this object.") diff --git a/baseapp/files/services/upload_service.py b/baseapp/files/services/upload_service.py index db23da42..31214070 100644 --- a/baseapp/files/services/upload_service.py +++ b/baseapp/files/services/upload_service.py @@ -147,8 +147,12 @@ def _validate_file_params(self, file_size, num_parts, part_size): if num_parts > 1 and part_size < 5 * 1024 * 1024: raise ValueError("Part size must be at least 5MB for multipart uploads") - expected_size = (num_parts - 1) * part_size - if expected_size > file_size: + # file_size must fit within the declared parts: strictly greater than + # the first (num_parts - 1) full parts, and no larger than all of them. + # Mirrors InitiateUploadSerializer so non-REST callers are protected too. + min_size = (num_parts - 1) * part_size + max_size = num_parts * part_size + if file_size < min_size or file_size > max_size: raise ValueError("Part size and count don't match file size") def _validate_parts(self, file_obj, parts): diff --git a/baseapp/files/storage/local.py b/baseapp/files/storage/local.py index 08267c7b..7c9370cf 100644 --- a/baseapp/files/storage/local.py +++ b/baseapp/files/storage/local.py @@ -36,7 +36,7 @@ def initiate_upload(self, file_obj, num_parts: int, part_size: int) -> Dict[str, os.makedirs(temp_dir, exist_ok=True) # Get the base URL from settings or use default - base_url = getattr(settings, "BASE_URL", "http://localhost:8000") + base_url = settings.URL # Generate presigned URLs with signed tokens presigned_urls = [] diff --git a/baseapp/files/tests/test_rest_api.py b/baseapp/files/tests/test_rest_api.py index fbcfa935..161e8640 100644 --- a/baseapp/files/tests/test_rest_api.py +++ b/baseapp/files/tests/test_rest_api.py @@ -8,6 +8,7 @@ from django.utils import timezone from rest_framework import status +from baseapp_comments.tests.factories import CommentFactory from baseapp_core.models import DocumentId File = swapper.load_model("baseapp_files", "File") @@ -75,12 +76,12 @@ def test_initiate_upload_success(self, user_client, mock_s3_handler): mock_s3_handler.initiate_upload.assert_called_once() def test_initiate_upload_with_parent(self, user_client, mock_s3_handler): - """Test upload initiation with parent object.""" - # Get or create DocumentId for the user - user_ct = ContentType.objects.get_for_model(User) + """Test upload initiation with a files-enabled parent object.""" + comment = CommentFactory(user=user_client.user) + comment_ct = ContentType.objects.get_for_model(comment) document_id, _ = DocumentId.objects.get_or_create( - content_type=user_ct, - object_id=user_client.user.id, + content_type=comment_ct, + object_id=comment.id, ) response = user_client.post( @@ -96,19 +97,38 @@ def test_initiate_upload_with_parent(self, user_client, mock_s3_handler): format="json", ) - # Print response for debugging if it fails - if response.status_code != status.HTTP_201_CREATED: - print(f"Response status: {response.status_code}") - print(f"Response data: {response.json()}") - assert response.status_code == status.HTTP_201_CREATED data = response.json() file_obj = File.get_by_public_id(data["id"]) # Parent is now a ForeignKey to DocumentId assert file_obj.parent is not None - assert file_obj.parent.content_type_id == user_ct.id - assert file_obj.parent.object_id == user_client.user.id + assert file_obj.parent.content_type_id == comment_ct.id + assert file_obj.parent.object_id == comment.id + + def test_initiate_upload_rejects_non_fileable_parent(self, user_client, mock_s3_handler): + """A parent that does not support files (not a FileableModel) is rejected, + mirroring the fileAttachToTarget mutation.""" + user_ct = ContentType.objects.get_for_model(User) + document_id, _ = DocumentId.objects.get_or_create( + content_type=user_ct, + object_id=user_client.user.id, + ) + + response = user_client.post( + "/v1/files/uploads", + { + "file_name": "avatar.jpg", + "file_size": 1048576, + "file_content_type": "image/jpeg", + "num_parts": 1, + "part_size": 1048576, + "parent_id": str(document_id.public_id), + }, + format="json", + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST def test_initiate_upload_requires_authentication(self, client): """Test that upload initiation requires authentication.""" @@ -517,11 +537,12 @@ def standalone_file(self, user_client): ) def test_set_parent_success(self, user_client, standalone_file): - """Test setting parent on standalone file.""" - user_ct = ContentType.objects.get_for_model(User) + """Test setting a files-enabled parent on a standalone file.""" + comment = CommentFactory(user=user_client.user) + comment_ct = ContentType.objects.get_for_model(comment) document_id, _ = DocumentId.objects.get_or_create( - content_type=user_ct, - object_id=user_client.user.id, + content_type=comment_ct, + object_id=comment.id, ) response = user_client.post( @@ -537,8 +558,26 @@ def test_set_parent_success(self, user_client, standalone_file): standalone_file.refresh_from_db() # Parent is now a ForeignKey to DocumentId assert standalone_file.parent is not None - assert standalone_file.parent.content_type_id == user_ct.id - assert standalone_file.parent.object_id == user_client.user.id + assert standalone_file.parent.content_type_id == comment_ct.id + assert standalone_file.parent.object_id == comment.id + + def test_set_parent_rejects_non_fileable_target(self, user_client, standalone_file): + """set-parent enforces the same target authorization as the mutation.""" + user_ct = ContentType.objects.get_for_model(User) + document_id, _ = DocumentId.objects.get_or_create( + content_type=user_ct, + object_id=user_client.user.id, + ) + + response = user_client.post( + f"/v1/files/{standalone_file.public_id}/set-parent", + { + "parent_id": str(document_id.public_id), + }, + format="json", + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST def test_set_parent_invalid_parent_id(self, user_client, standalone_file): """Test setting invalid parent_id.""" diff --git a/baseapp/files/utils.py b/baseapp/files/utils.py index 2109a8a1..5ec7c081 100644 --- a/baseapp/files/utils.py +++ b/baseapp/files/utils.py @@ -26,13 +26,15 @@ def recalculate_files_count(parent): file_target = get_or_create_file_target(parent) - files_count_qs = parent.files.values("file_content_type").annotate( - count=models.Count("file_content_type") - ) + # Count by a non-null column: Count("file_content_type") ignores NULL rows, + # under-counting files with no content type. Bucket NULLs under "unknown" + # to match the pgtrigger that maintains this same JSON on insert/update. + files_count_qs = parent.files.values("file_content_type").annotate(count=models.Count("id")) file_target.files_count = default_files_count() for item in files_count_qs: count = item["count"] - file_target.files_count[item["file_content_type"]] = count + key = item["file_content_type"] or "unknown" + file_target.files_count[key] = count file_target.files_count["total"] += count file_target.save(update_fields=["files_count"]) From 290fbaf23d216a0f8555c286468e22388f64aca5 Mon Sep 17 00:00:00 2001 From: Alisson Patricio Date: Sun, 12 Jul 2026 03:37:35 +0000 Subject: [PATCH 14/20] =?UTF-8?q?files:=20second=20review=20round=20(CodeR?= =?UTF-8?q?abbit/CodeQL)=20=E2=80=94=20hardening=20+=20fixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Storage backend detection: use default_storage.__class__ (LazyObject proxies it) instead of type(default_storage), which returned the wrapper and never matched S3 — S3 was never selected. - Genericize the remaining except ValueError responses in the upload views (log server-side, return a generic message) — clears the residual CodeQL 'information exposure' alerts on those lines. - Local complete_upload now raises on a missing part file instead of silently producing a truncated file. - MD5 ETag marked usedforsecurity=False (content checksum, not security). - cleanup tasks load the swappable model lazily (avoid AppRegistryNotReady on Celery startup); log the swallowed storage-abort failure in upload_service. - FilesInterface.resolve_files orders by (-created, -pk) for stable cursor pagination, matching my_files. - files list: empty ?status= no longer bypasses the upload_status filter. - DRF ValidationError messages wrapped in arrays / non_field_errors per guidelines; FilesPermissionsBackend.has_perm typed + documented. - Fix README multipart JS example to the single parent_id argument. Tests updated for the generic completion error; new tests already cover the non-fileable-target rejection. Files suite green (120), migrations clean. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146oeiGsELS6QVpVUQHDQXd --- baseapp/files/README.md | 2 +- baseapp/files/graphql/interfaces.py | 5 ++++- baseapp/files/permissions.py | 17 +++++++++++++---- baseapp/files/rest_framework/files/views.py | 5 +++-- .../rest_framework/uploads/presigned_views.py | 5 +++-- .../files/rest_framework/uploads/serializers.py | 17 +++++++++++------ baseapp/files/rest_framework/uploads/views.py | 10 ++++++---- baseapp/files/services/cleanup.py | 6 ++++-- baseapp/files/services/upload_service.py | 12 ++++++++++-- baseapp/files/storage/__init__.py | 7 +++++-- baseapp/files/storage/local.py | 15 +++++++++------ baseapp/files/tests/test_rest_api.py | 6 ++++-- 12 files changed, 73 insertions(+), 34 deletions(-) diff --git a/baseapp/files/README.md b/baseapp/files/README.md index 4df7767d..cd9d4f68 100644 --- a/baseapp/files/README.md +++ b/baseapp/files/README.md @@ -471,7 +471,7 @@ try { const fileInput = document.querySelector('input[type="file"]'); const file = fileInput.files[0]; - const result = await uploader.uploadFile(file, 'testapp.post', 123); + const result = await uploader.uploadFile(file, 'b3c5…-document-public-id'); console.log('Upload complete:', result.url); } catch (error) { console.error('Upload failed:', error); diff --git a/baseapp/files/graphql/interfaces.py b/baseapp/files/graphql/interfaces.py index ef6bd74f..5cee98a9 100644 --- a/baseapp/files/graphql/interfaces.py +++ b/baseapp/files/graphql/interfaces.py @@ -36,10 +36,13 @@ def resolve_files(self, info, **kwargs): # Filter through the DocumentId join instead of fetching the DocumentId # row per object — keeps the connection a single query per list. content_type = ContentType.objects.get_for_model(self) + # Total ordering (-created, -pk): AbstractFile has no Meta.ordering, and + # cursor pagination over a non-total order can duplicate/drop rows when + # timestamps collide. Matches FilesQueries.resolve_my_files. return File.objects.filter( parent__content_type=content_type, parent__object_id=self.pk, - ).order_by("-created") + ).order_by("-created", "-pk") def get_files_interface(): diff --git a/baseapp/files/permissions.py b/baseapp/files/permissions.py index 8e893157..fa7a6b27 100644 --- a/baseapp/files/permissions.py +++ b/baseapp/files/permissions.py @@ -1,17 +1,22 @@ +from typing import Any, Optional + import swapper from django.contrib.auth.backends import BaseBackend +from django.contrib.auth.models import AbstractBaseUser, AnonymousUser File = swapper.load_model("baseapp_files", "File") file_app_label = File._meta.app_label file_model_name = File._meta.model_name.lower() +UserType = AbstractBaseUser | AnonymousUser + class FilesPermissionsBackend(BaseBackend): - def authenticate(self, request, **kwargs): + def authenticate(self, request, **kwargs) -> None: return None @staticmethod - def _can_add_to_target(user_obj, obj): + def _can_add_to_target(user_obj: UserType, obj: Optional[Any]) -> bool: """Target-level: may this user attach files to `obj`? Default policy mirrors comments' add_comment — authenticated and files enabled. Projects can override this backend to enforce target ownership.""" @@ -24,14 +29,18 @@ def _can_add_to_target(user_obj, obj): return user_obj.is_authenticated and enabled @staticmethod - def _is_owner_or_has_global(user_obj, perm, obj): + def _is_owner_or_has_global(user_obj: UserType, perm: str, obj: Optional[Any]) -> bool: """Owner acts on their own file; otherwise fall back to the global perm (ModelBackend doesn't grant global perms when an obj is passed).""" if not obj: return False return obj.created_by == user_obj or user_obj.has_perm(perm) - def has_perm(self, user_obj, perm, obj=None): + def has_perm(self, user_obj: UserType, perm: str, obj: Optional[Any] = None) -> bool: + """Resolve object-level file permissions: ``add_`` (attach to a + target), ``attach_`` (owner-only), and view/change/delete + (owner or global perm). Unknown perms return False so other backends + can decide.""" if perm == f"{file_app_label}.add_{file_model_name}": return self._can_add_to_target(user_obj, obj) diff --git a/baseapp/files/rest_framework/files/views.py b/baseapp/files/rest_framework/files/views.py index 5938dde7..f5760f61 100644 --- a/baseapp/files/rest_framework/files/views.py +++ b/baseapp/files/rest_framework/files/views.py @@ -56,9 +56,10 @@ def get_queryset(self): """Filter files based on user and query params.""" qs = File.objects.all() - # Only show completed files by default + # Only show completed files by default. Use `is not None` so an empty + # `?status=` doesn't fall through and expose files in every state. status_filter = self.request.query_params.get("status", File.UploadStatus.COMPLETED) - if status_filter: + if status_filter is not None and status_filter != "": qs = qs.filter(upload_status=status_filter) # Filter by parent using DocumentId public_id diff --git a/baseapp/files/rest_framework/uploads/presigned_views.py b/baseapp/files/rest_framework/uploads/presigned_views.py index 9956b805..5b8c4557 100644 --- a/baseapp/files/rest_framework/uploads/presigned_views.py +++ b/baseapp/files/rest_framework/uploads/presigned_views.py @@ -155,8 +155,9 @@ def upload_part(self, request, pk=None, part_number=None): response["ETag"] = etag return response - except ValueError as e: - raise ValidationError(str(e)) + except ValueError: + logger.warning("Upload part rejected", exc_info=True) + raise ValidationError("Invalid upload part request.") except Exception: logger.exception("Failed to upload part") raise ValidationError("Failed to upload part.") diff --git a/baseapp/files/rest_framework/uploads/serializers.py b/baseapp/files/rest_framework/uploads/serializers.py index 3fa8900c..d8ad40cb 100644 --- a/baseapp/files/rest_framework/uploads/serializers.py +++ b/baseapp/files/rest_framework/uploads/serializers.py @@ -27,7 +27,7 @@ def validate_parent_id(self, value): document_id = DocumentId.objects.get(public_id=value) return document_id.pk except DocumentId.DoesNotExist: - raise serializers.ValidationError("Invalid parent_id: DocumentId not found") + raise serializers.ValidationError(["Invalid parent_id: DocumentId not found"]) def validate(self, data): """Validate file size matches parts.""" @@ -40,7 +40,12 @@ def validate(self, data): if file_size < min_size or file_size > max_size: raise serializers.ValidationError( - f"File size {file_size} doesn't match {num_parts} parts of {part_size} bytes" + { + "non_field_errors": [ + f"File size {file_size} doesn't match {num_parts} parts of " + f"{part_size} bytes" + ] + } ) return data @@ -69,13 +74,13 @@ def validate_parts(self, value): """Validate parts structure.""" for part in value: if "part_number" not in part or "etag" not in part: - raise serializers.ValidationError("Each part must have 'part_number' and 'etag'") + raise serializers.ValidationError(["Each part must have 'part_number' and 'etag'"]) if not isinstance(part["part_number"], int) or part["part_number"] < 1: - raise serializers.ValidationError("part_number must be a positive integer") + raise serializers.ValidationError(["part_number must be a positive integer"]) if not isinstance(part["etag"], str) or not part["etag"]: - raise serializers.ValidationError("etag must be a non-empty string") + raise serializers.ValidationError(["etag must be a non-empty string"]) return value @@ -91,4 +96,4 @@ def validate_parent_id(self, value): document_id = DocumentId.objects.get(public_id=value) return document_id.pk except DocumentId.DoesNotExist: - raise serializers.ValidationError("Invalid parent_id: DocumentId not found") + raise serializers.ValidationError(["Invalid parent_id: DocumentId not found"]) diff --git a/baseapp/files/rest_framework/uploads/views.py b/baseapp/files/rest_framework/uploads/views.py index d34c7f36..362c7108 100644 --- a/baseapp/files/rest_framework/uploads/views.py +++ b/baseapp/files/rest_framework/uploads/views.py @@ -104,8 +104,9 @@ def create(self, request): response_serializer = UploadResponseSerializer(response_data) return Response(response_serializer.data, status=status.HTTP_201_CREATED) - except ValueError as e: - raise ValidationError(str(e)) + except ValueError: + logger.warning("Upload initiation rejected", exc_info=True) + raise ValidationError("Invalid upload parameters.") except Exception: logger.exception("Failed to initiate upload") raise ValidationError("Failed to initiate upload.") @@ -148,8 +149,9 @@ def complete(self, request, pk=None): status=status.HTTP_200_OK, ) - except ValueError as e: - raise ValidationError(str(e)) + except ValueError: + logger.warning("Upload completion rejected", exc_info=True) + raise ValidationError("Invalid or incomplete upload parts.") except Exception: logger.exception("Failed to complete upload") raise ValidationError("Failed to complete upload.") diff --git a/baseapp/files/services/cleanup.py b/baseapp/files/services/cleanup.py index 6e73e628..4b85343a 100644 --- a/baseapp/files/services/cleanup.py +++ b/baseapp/files/services/cleanup.py @@ -7,8 +7,6 @@ from ..storage import get_upload_handler -File = swapper.load_model("baseapp_files", "File") - logger = logging.getLogger(__name__) @@ -19,6 +17,9 @@ def cleanup_expired_uploads(): Run this periodically (e.g., hourly via celery beat). """ + # Loaded lazily (not at import time): this module is imported during Celery + # startup, which can precede Django's app registry being ready. + File = swapper.load_model("baseapp_files", "File") handler = get_upload_handler() now = timezone.now() @@ -53,6 +54,7 @@ def cleanup_failed_uploads(days_old=7): Can be called manually or via management command. """ + File = swapper.load_model("baseapp_files", "File") cutoff = timezone.now() - timedelta(days=days_old) deleted_count, _ = File.objects.filter( diff --git a/baseapp/files/services/upload_service.py b/baseapp/files/services/upload_service.py index 31214070..ffc385ba 100644 --- a/baseapp/files/services/upload_service.py +++ b/baseapp/files/services/upload_service.py @@ -1,3 +1,4 @@ +import logging from datetime import timedelta import swapper @@ -7,6 +8,8 @@ from ..storage import get_upload_handler +logger = logging.getLogger(__name__) + File = swapper.load_model("baseapp_files", "File") @@ -125,8 +128,13 @@ def abort_multipart_upload(self, file_id: int): try: self.handler.abort_upload(file_obj, file_obj.upload_id) except Exception: - # Log but continue to mark as aborted - pass + # Continue to mark as aborted, but record the storage-side + # failure so orphaned remote uploads are diagnosable. + logger.exception( + "Storage abort failed for file %s (upload_id=%s)", + file_obj.id, + file_obj.upload_id, + ) file_obj.upload_status = File.UploadStatus.ABORTED file_obj.upload_id = "" diff --git a/baseapp/files/storage/__init__.py b/baseapp/files/storage/__init__.py index fd1e805e..7185d70a 100644 --- a/baseapp/files/storage/__init__.py +++ b/baseapp/files/storage/__init__.py @@ -5,10 +5,13 @@ def get_upload_handler(): """ Factory to get appropriate upload handler based on storage backend. """ - storage_class = type(default_storage) + # `default_storage` is a LazyObject: type() returns the wrapper class, so + # use `__class__`, which LazyObject proxies to the wrapped backend, to + # detect the real storage backend. + storage_class_name = default_storage.__class__.__name__ # Check if using S3 storage - if "S3Boto3Storage" in str(storage_class): + if "S3Boto3Storage" in storage_class_name: from .s3 import S3MultipartUploadHandler return S3MultipartUploadHandler() diff --git a/baseapp/files/storage/local.py b/baseapp/files/storage/local.py index 7c9370cf..affb5d2c 100644 --- a/baseapp/files/storage/local.py +++ b/baseapp/files/storage/local.py @@ -82,8 +82,9 @@ def upload_part(self, upload_id: str, part_number: int, data: bytes) -> str: with open(part_file, "wb") as f: f.write(data) - # Generate ETag (MD5 hash like S3) - etag = hashlib.md5(data).hexdigest() + # Generate ETag (MD5 hash like S3). Content checksum, not a security + # primitive — usedforsecurity=False documents intent and satisfies linters. + etag = hashlib.md5(data, usedforsecurity=False).hexdigest() return etag def complete_upload(self, file_obj, upload_id: str, parts: List[Dict]) -> str: @@ -99,13 +100,15 @@ def complete_upload(self, file_obj, upload_id: str, parts: List[Dict]) -> str: # Ensure directory exists os.makedirs(os.path.dirname(full_path), exist_ok=True) - # Combine parts in order + # Combine parts in order. A missing part would silently truncate the + # assembled file, so fail loudly instead. with open(full_path, "wb") as outfile: for part in sorted(parts, key=lambda x: x["part_number"]): part_file = os.path.join(temp_dir, f"part_{part['part_number']}") - if os.path.exists(part_file): - with open(part_file, "rb") as infile: - outfile.write(infile.read()) + if not os.path.exists(part_file): + raise ValueError(f"Missing uploaded part {part['part_number']}") + with open(part_file, "rb") as infile: + outfile.write(infile.read()) # Cleanup temp files shutil.rmtree(temp_dir, ignore_errors=True) diff --git a/baseapp/files/tests/test_rest_api.py b/baseapp/files/tests/test_rest_api.py index 161e8640..2d1fb6a2 100644 --- a/baseapp/files/tests/test_rest_api.py +++ b/baseapp/files/tests/test_rest_api.py @@ -280,7 +280,9 @@ def test_complete_upload_wrong_number_of_parts( ) assert response.status_code == status.HTTP_400_BAD_REQUEST - assert "Expected 2 parts" in str(response.json()) + # The specific reason is logged server-side; the client gets a generic + # message so internal exception details are not exposed. + assert "Invalid or incomplete upload parts" in str(response.json()) def test_complete_upload_missing_part_number( self, user_client, pending_upload, mock_s3_handler @@ -298,7 +300,7 @@ def test_complete_upload_missing_part_number( ) assert response.status_code == status.HTTP_400_BAD_REQUEST - assert "Missing or duplicate" in str(response.json()) + assert "Invalid or incomplete upload parts" in str(response.json()) def test_complete_upload_requires_ownership(self, client, pending_upload, mock_s3_handler): """Test that only the file owner can complete upload.""" From 4fa95794c1b5ab74692720cbe663ccbed6e63d14 Mon Sep 17 00:00:00 2001 From: Alisson Patricio Date: Sun, 12 Jul 2026 04:13:38 +0000 Subject: [PATCH 15/20] files: wrap REST ValidationError messages in arrays + annotate authz helper Follow-up to CodeRabbit: the upload view except-blocks now raise ValidationError([...]) ... from None (array form per DRF guideline, and from None clears Ruff B904), and enforce_can_attach_to_parent gains parameter type annotations. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146oeiGsELS6QVpVUQHDQXd --- .../files/rest_framework/uploads/presigned_views.py | 4 ++-- baseapp/files/rest_framework/uploads/views.py | 10 +++++----- baseapp/files/rest_framework/utils.py | 9 ++++++--- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/baseapp/files/rest_framework/uploads/presigned_views.py b/baseapp/files/rest_framework/uploads/presigned_views.py index 5b8c4557..1142c073 100644 --- a/baseapp/files/rest_framework/uploads/presigned_views.py +++ b/baseapp/files/rest_framework/uploads/presigned_views.py @@ -157,7 +157,7 @@ def upload_part(self, request, pk=None, part_number=None): except ValueError: logger.warning("Upload part rejected", exc_info=True) - raise ValidationError("Invalid upload part request.") + raise ValidationError(["Invalid upload part request."]) from None except Exception: logger.exception("Failed to upload part") - raise ValidationError("Failed to upload part.") + raise ValidationError(["Failed to upload part."]) from None diff --git a/baseapp/files/rest_framework/uploads/views.py b/baseapp/files/rest_framework/uploads/views.py index 362c7108..2935b129 100644 --- a/baseapp/files/rest_framework/uploads/views.py +++ b/baseapp/files/rest_framework/uploads/views.py @@ -106,10 +106,10 @@ def create(self, request): except ValueError: logger.warning("Upload initiation rejected", exc_info=True) - raise ValidationError("Invalid upload parameters.") + raise ValidationError(["Invalid upload parameters."]) from None except Exception: logger.exception("Failed to initiate upload") - raise ValidationError("Failed to initiate upload.") + raise ValidationError(["Failed to initiate upload."]) from None @action(detail=True, methods=["post"]) def complete(self, request, pk=None): @@ -151,10 +151,10 @@ def complete(self, request, pk=None): except ValueError: logger.warning("Upload completion rejected", exc_info=True) - raise ValidationError("Invalid or incomplete upload parts.") + raise ValidationError(["Invalid or incomplete upload parts."]) from None except Exception: logger.exception("Failed to complete upload") - raise ValidationError("Failed to complete upload.") + raise ValidationError(["Failed to complete upload."]) from None def destroy(self, request, pk=None): """ @@ -177,4 +177,4 @@ def destroy(self, request, pk=None): except Exception: logger.exception("Failed to abort upload") - raise ValidationError("Failed to abort upload.") + raise ValidationError(["Failed to abort upload."]) from None diff --git a/baseapp/files/rest_framework/utils.py b/baseapp/files/rest_framework/utils.py index 52280bc9..c8dc2bae 100644 --- a/baseapp/files/rest_framework/utils.py +++ b/baseapp/files/rest_framework/utils.py @@ -1,4 +1,5 @@ import swapper +from django.contrib.auth.models import AbstractBaseUser, AnonymousUser from rest_framework.exceptions import PermissionDenied, ValidationError from baseapp_core.models import DocumentId @@ -9,8 +10,10 @@ file_app_label = File._meta.app_label file_model_name = File._meta.model_name.lower() +UserType = AbstractBaseUser | AnonymousUser -def enforce_can_attach_to_parent(user, parent_document_pk) -> None: + +def enforce_can_attach_to_parent(user: UserType, parent_document_pk: int) -> None: """ Authorize attaching a file to the object behind ``parent_document_pk`` (a ``DocumentId`` pk). Mirrors the ``FileAttachToTarget`` GraphQL mutation so @@ -21,11 +24,11 @@ def enforce_can_attach_to_parent(user, parent_document_pk) -> None: try: document = DocumentId.objects.get(pk=parent_document_pk) except DocumentId.DoesNotExist: - raise ValidationError("Invalid parent: target not found.") + raise ValidationError(["Invalid parent: target not found."]) from None target = document.content_object if not isinstance(target, FileableModel): - raise ValidationError("This object does not support file attachments.") + raise ValidationError(["This object does not support file attachments."]) if not user.has_perm(f"{file_app_label}.add_{file_model_name}", target): raise PermissionDenied("You don't have permission to attach files to this object.") From bd7ec376c0ebf89e183873d966183b542fb385d1 Mon Sep 17 00:00:00 2001 From: Alisson Patricio Date: Sun, 12 Jul 2026 13:02:11 +0000 Subject: [PATCH 16/20] files: move storage I/O out of transactions, i18n-ready fields + messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - upload_service: the storage handler's initiate/complete/abort network calls now run OUTSIDE the DB transaction — initiate creates the row, calls the handler, then updates; complete/abort validate first, do the remote call unlocked, then flip status under a short select_for_update guard (no row lock held across network I/O). Addresses the two 'Heavy lift' review notes. - Models: verbose_name + help_text (gettext_lazy) on every AbstractFile and AbstractFileTarget field + Meta verbose_name/plural (migration 0003 in testproject; state-only). - i18n: wrap all user-facing REST strings (view/serializer/permission/util errors, 403/404 messages) in gettext_lazy; GraphQL layer already did. Files suite green (120), migrations clean. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146oeiGsELS6QVpVUQHDQXd --- baseapp/files/models.py | 69 +++++- baseapp/files/rest_framework/files/views.py | 3 +- .../rest_framework/uploads/presigned_views.py | 29 ++- .../rest_framework/uploads/serializers.py | 17 +- baseapp/files/rest_framework/uploads/views.py | 17 +- baseapp/files/rest_framework/utils.py | 7 +- baseapp/files/services/upload_service.py | 78 ++++--- ...tions_alter_filetarget_options_and_more.py | 204 ++++++++++++++++++ 8 files changed, 352 insertions(+), 72 deletions(-) create mode 100644 testproject/files/migrations/0003_alter_file_options_alter_filetarget_options_and_more.py diff --git a/baseapp/files/models.py b/baseapp/files/models.py index 34fb241d..7074a27a 100644 --- a/baseapp/files/models.py +++ b/baseapp/files/models.py @@ -227,14 +227,25 @@ class AbstractFileTarget(DocumentIdUniqueTargetMixin): on_delete=models.CASCADE, primary_key=True, related_name="file_target", + verbose_name=_("target"), help_text=_("The document this file target belongs to"), ) - files_count = models.JSONField(default=default_files_count) - is_files_enabled = models.BooleanField(default=True, verbose_name=_("is files enabled")) + files_count = models.JSONField( + default=default_files_count, + verbose_name=_("files count"), + help_text=_("Per-content-type file counts for the target"), + ) + is_files_enabled = models.BooleanField( + default=True, + verbose_name=_("is files enabled"), + help_text=_("Whether files can be attached to the target"), + ) class Meta: abstract = True + verbose_name = _("file target") + verbose_name_plural = _("file targets") def __str__(self): return f"FileTarget for {self.target}" @@ -295,22 +306,57 @@ class AbstractFile(*file_inheritances, DocumentIdMixin, RelayModel, TimeStampedM blank=True, on_delete=models.CASCADE, related_name="files", + verbose_name=_("parent"), help_text=_("The parent document this file belongs to"), ) - file_content_type = models.CharField(max_length=150, blank=True, default="") - file_name = models.CharField(max_length=512, blank=True, default="") - file_size = models.PositiveIntegerField(null=True, help_text=_("File size in bytes")) - file = models.FileField(max_length=512, upload_to=random_name_in("files")) + file_content_type = models.CharField( + max_length=150, + blank=True, + default="", + verbose_name=_("content type"), + help_text=_("MIME type of the uploaded file"), + ) + file_name = models.CharField( + max_length=512, + blank=True, + default="", + verbose_name=_("file name"), + help_text=_("Original name of the uploaded file"), + ) + file_size = models.PositiveIntegerField( + null=True, + verbose_name=_("file size"), + help_text=_("File size in bytes"), + ) + file = models.FileField( + max_length=512, + upload_to=random_name_in("files"), + verbose_name=_("file"), + help_text=_("The stored file"), + ) - name = models.CharField(max_length=512, blank=True, default="") - description = models.TextField(blank=True, default="") + name = models.CharField( + max_length=512, + blank=True, + default="", + verbose_name=_("name"), + help_text=_("Display name for the file"), + ) + description = models.TextField( + blank=True, + default="", + verbose_name=_("description"), + help_text=_("Optional description of the file"), + ) created_by = models.ForeignKey( settings.AUTH_USER_MODEL, related_name="files_created", on_delete=models.CASCADE, null=True, + verbose_name=_("created by"), + help_text=_("The user who uploaded the file"), ) # Upload status tracking @@ -326,6 +372,7 @@ class UploadStatus(models.TextChoices): choices=UploadStatus.choices, default=UploadStatus.COMPLETED, db_index=True, + verbose_name=_("upload status"), help_text=_("Status of the file upload"), ) @@ -334,6 +381,7 @@ class UploadStatus(models.TextChoices): max_length=512, blank=True, default="", + verbose_name=_("upload ID"), help_text=_("S3 multipart upload ID for in-progress uploads"), ) @@ -341,6 +389,7 @@ class UploadStatus(models.TextChoices): total_parts = models.PositiveIntegerField( null=True, blank=True, + verbose_name=_("total parts"), help_text=_("Total number of parts for multipart upload"), ) @@ -348,6 +397,7 @@ class UploadStatus(models.TextChoices): null=True, blank=True, default=dict, + verbose_name=_("uploaded parts"), help_text=_("Tracking uploaded parts with ETags"), ) @@ -356,6 +406,7 @@ class UploadStatus(models.TextChoices): null=True, blank=True, db_index=True, + verbose_name=_("upload expires at"), help_text=_("Expiration time for pending uploads"), ) @@ -381,6 +432,8 @@ def save(self, *args, **kwargs): class Meta: abstract = True + verbose_name = _("file") + verbose_name_plural = _("files") # Note: Triggers are registered via the class_prepared signal in this module # to properly handle swappable models. diff --git a/baseapp/files/rest_framework/files/views.py b/baseapp/files/rest_framework/files/views.py index f5760f61..aaf23fb8 100644 --- a/baseapp/files/rest_framework/files/views.py +++ b/baseapp/files/rest_framework/files/views.py @@ -1,4 +1,5 @@ import swapper +from django.utils.translation import gettext_lazy as _ from rest_framework import mixins, permissions, status, viewsets from rest_framework.response import Response @@ -47,7 +48,7 @@ def get_object(self): obj = File.get_by_public_id(public_id) if obj is None: - raise NotFound("File not found") + raise NotFound(_("File not found")) self.check_object_permissions(self.request, obj) return obj diff --git a/baseapp/files/rest_framework/uploads/presigned_views.py b/baseapp/files/rest_framework/uploads/presigned_views.py index 1142c073..2ea66dc0 100644 --- a/baseapp/files/rest_framework/uploads/presigned_views.py +++ b/baseapp/files/rest_framework/uploads/presigned_views.py @@ -2,6 +2,7 @@ import swapper from django.core import signing +from django.utils.translation import gettext_lazy as _ from django.views.decorators.csrf import csrf_exempt from rest_framework import status, viewsets from rest_framework.decorators import action @@ -66,7 +67,7 @@ def upload_part(self, request, pk=None, part_number=None): token = request.query_params.get("token") if not token: return Response( - {"error": "Missing token parameter"}, + {"error": _("Missing token parameter")}, status=status.HTTP_401_UNAUTHORIZED, ) @@ -79,18 +80,18 @@ def upload_part(self, request, pk=None, part_number=None): token_data.get("part_number") ) != str(part_number): return Response( - {"error": "Invalid token for this file/part"}, + {"error": _("Invalid token for this file/part")}, status=status.HTTP_401_UNAUTHORIZED, ) except signing.SignatureExpired: return Response( - {"error": "Token has expired"}, + {"error": _("Token has expired")}, status=status.HTTP_401_UNAUTHORIZED, ) except signing.BadSignature: return Response( - {"error": "Invalid token signature"}, + {"error": _("Invalid token signature")}, status=status.HTTP_401_UNAUTHORIZED, ) @@ -99,7 +100,7 @@ def upload_part(self, request, pk=None, part_number=None): file_obj = File.objects.get(id=pk) except File.DoesNotExist: return Response( - {"error": "File not found"}, + {"error": _("File not found")}, status=status.HTTP_404_NOT_FOUND, ) @@ -107,14 +108,17 @@ def upload_part(self, request, pk=None, part_number=None): # previous initiation (different upload_id) must not upload into a new one. if str(token_data.get("upload_id")) != str(file_obj.upload_id): return Response( - {"error": "Token does not match the current upload session"}, + {"error": _("Token does not match the current upload session")}, status=status.HTTP_401_UNAUTHORIZED, ) # Validate state if file_obj.upload_status not in [File.UploadStatus.PENDING, File.UploadStatus.UPLOADING]: return Response( - {"error": f"Cannot upload parts in status: {file_obj.upload_status}"}, + { + "error": _("Cannot upload parts in status: %(status)s") + % {"status": file_obj.upload_status} + }, status=status.HTTP_400_BAD_REQUEST, ) @@ -122,7 +126,7 @@ def upload_part(self, request, pk=None, part_number=None): # go straight to S3, so its handler has no upload_part. if not hasattr(self.upload_service.handler, "upload_part"): return Response( - {"error": "The active storage backend does not support presigned part uploads"}, + {"error": _("The active storage backend does not support presigned part uploads")}, status=status.HTTP_400_BAD_REQUEST, ) @@ -132,7 +136,10 @@ def upload_part(self, request, pk=None, part_number=None): # Validate part number if part_number < 1 or part_number > file_obj.total_parts: return Response( - {"error": f"Invalid part number. Must be between 1 and {file_obj.total_parts}"}, + { + "error": _("Invalid part number. Must be between 1 and %(total)s") + % {"total": file_obj.total_parts} + }, status=status.HTTP_400_BAD_REQUEST, ) @@ -157,7 +164,7 @@ def upload_part(self, request, pk=None, part_number=None): except ValueError: logger.warning("Upload part rejected", exc_info=True) - raise ValidationError(["Invalid upload part request."]) from None + raise ValidationError([_("Invalid upload part request.")]) from None except Exception: logger.exception("Failed to upload part") - raise ValidationError(["Failed to upload part."]) from None + raise ValidationError([_("Failed to upload part.")]) from None diff --git a/baseapp/files/rest_framework/uploads/serializers.py b/baseapp/files/rest_framework/uploads/serializers.py index d8ad40cb..cc4eb5ca 100644 --- a/baseapp/files/rest_framework/uploads/serializers.py +++ b/baseapp/files/rest_framework/uploads/serializers.py @@ -1,4 +1,5 @@ import swapper +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers from baseapp_core.models import DocumentId @@ -27,7 +28,7 @@ def validate_parent_id(self, value): document_id = DocumentId.objects.get(public_id=value) return document_id.pk except DocumentId.DoesNotExist: - raise serializers.ValidationError(["Invalid parent_id: DocumentId not found"]) + raise serializers.ValidationError([_("Invalid parent_id: DocumentId not found")]) def validate(self, data): """Validate file size matches parts.""" @@ -42,8 +43,8 @@ def validate(self, data): raise serializers.ValidationError( { "non_field_errors": [ - f"File size {file_size} doesn't match {num_parts} parts of " - f"{part_size} bytes" + _("File size %(size)s doesn't match %(parts)s parts of %(part_size)s bytes") + % {"size": file_size, "parts": num_parts, "part_size": part_size} ] } ) @@ -74,13 +75,15 @@ def validate_parts(self, value): """Validate parts structure.""" for part in value: if "part_number" not in part or "etag" not in part: - raise serializers.ValidationError(["Each part must have 'part_number' and 'etag'"]) + raise serializers.ValidationError( + [_("Each part must have 'part_number' and 'etag'")] + ) if not isinstance(part["part_number"], int) or part["part_number"] < 1: - raise serializers.ValidationError(["part_number must be a positive integer"]) + raise serializers.ValidationError([_("part_number must be a positive integer")]) if not isinstance(part["etag"], str) or not part["etag"]: - raise serializers.ValidationError(["etag must be a non-empty string"]) + raise serializers.ValidationError([_("etag must be a non-empty string")]) return value @@ -96,4 +99,4 @@ def validate_parent_id(self, value): document_id = DocumentId.objects.get(public_id=value) return document_id.pk except DocumentId.DoesNotExist: - raise serializers.ValidationError(["Invalid parent_id: DocumentId not found"]) + raise serializers.ValidationError([_("Invalid parent_id: DocumentId not found")]) diff --git a/baseapp/files/rest_framework/uploads/views.py b/baseapp/files/rest_framework/uploads/views.py index 2935b129..45c33a42 100644 --- a/baseapp/files/rest_framework/uploads/views.py +++ b/baseapp/files/rest_framework/uploads/views.py @@ -1,6 +1,7 @@ import logging import swapper +from django.utils.translation import gettext_lazy as _ from rest_framework import permissions, status, viewsets from rest_framework.exceptions import ValidationError from rest_framework.response import Response @@ -53,7 +54,7 @@ def get_object(self): if obj is None: from rest_framework.exceptions import NotFound - raise NotFound("File not found") + raise NotFound(_("File not found")) self.check_object_permissions(self.request, obj) return obj @@ -106,10 +107,10 @@ def create(self, request): except ValueError: logger.warning("Upload initiation rejected", exc_info=True) - raise ValidationError(["Invalid upload parameters."]) from None + raise ValidationError([_("Invalid upload parameters.")]) from None except Exception: logger.exception("Failed to initiate upload") - raise ValidationError(["Failed to initiate upload."]) from None + raise ValidationError([_("Failed to initiate upload.")]) from None @action(detail=True, methods=["post"]) def complete(self, request, pk=None): @@ -130,7 +131,7 @@ def complete(self, request, pk=None): # Check permission (owner or has change_file permission) if not request.user.has_perm(f"{file_app_label}.change_{file_model_name}", file_obj): return Response( - {"error": "You don't have permission to complete this upload"}, + {"error": _("You don't have permission to complete this upload")}, status=status.HTTP_403_FORBIDDEN, ) @@ -151,10 +152,10 @@ def complete(self, request, pk=None): except ValueError: logger.warning("Upload completion rejected", exc_info=True) - raise ValidationError(["Invalid or incomplete upload parts."]) from None + raise ValidationError([_("Invalid or incomplete upload parts.")]) from None except Exception: logger.exception("Failed to complete upload") - raise ValidationError(["Failed to complete upload."]) from None + raise ValidationError([_("Failed to complete upload.")]) from None def destroy(self, request, pk=None): """ @@ -167,7 +168,7 @@ def destroy(self, request, pk=None): # Check permission (owner or has delete_file permission) if not request.user.has_perm(f"{file_app_label}.delete_{file_model_name}", file_obj): return Response( - {"error": "You don't have permission to abort this upload"}, + {"error": _("You don't have permission to abort this upload")}, status=status.HTTP_403_FORBIDDEN, ) @@ -177,4 +178,4 @@ def destroy(self, request, pk=None): except Exception: logger.exception("Failed to abort upload") - raise ValidationError(["Failed to abort upload."]) from None + raise ValidationError([_("Failed to abort upload.")]) from None diff --git a/baseapp/files/rest_framework/utils.py b/baseapp/files/rest_framework/utils.py index c8dc2bae..bd8cc908 100644 --- a/baseapp/files/rest_framework/utils.py +++ b/baseapp/files/rest_framework/utils.py @@ -1,5 +1,6 @@ import swapper from django.contrib.auth.models import AbstractBaseUser, AnonymousUser +from django.utils.translation import gettext_lazy as _ from rest_framework.exceptions import PermissionDenied, ValidationError from baseapp_core.models import DocumentId @@ -24,11 +25,11 @@ def enforce_can_attach_to_parent(user: UserType, parent_document_pk: int) -> Non try: document = DocumentId.objects.get(pk=parent_document_pk) except DocumentId.DoesNotExist: - raise ValidationError(["Invalid parent: target not found."]) from None + raise ValidationError([_("Invalid parent: target not found.")]) from None target = document.content_object if not isinstance(target, FileableModel): - raise ValidationError(["This object does not support file attachments."]) + raise ValidationError([_("This object does not support file attachments.")]) if not user.has_perm(f"{file_app_label}.add_{file_model_name}", target): - raise PermissionDenied("You don't have permission to attach files to this object.") + raise PermissionDenied(_("You don't have permission to attach files to this object.")) diff --git a/baseapp/files/services/upload_service.py b/baseapp/files/services/upload_service.py index ffc385ba..4c6ee531 100644 --- a/baseapp/files/services/upload_service.py +++ b/baseapp/files/services/upload_service.py @@ -19,7 +19,6 @@ class UploadService: def __init__(self): self.handler = get_upload_handler() - @transaction.atomic def initiate_multipart_upload( self, user, @@ -34,6 +33,11 @@ def initiate_multipart_upload( """ Initiate a multipart upload. + The storage handler's ``initiate_upload`` (a remote call for S3) runs + outside any DB transaction: the File row is created in its own + statement, the network call happens without holding a transaction open, + and the row is then updated. On failure the orphan row is removed. + Args: user: The user initiating the upload file_name: Original file name @@ -65,28 +69,32 @@ def initiate_multipart_upload( upload_expires_at=timezone.now() + timedelta(hours=24), ) - # Get presigned URLs from storage handler + # Get presigned URLs from storage handler (network I/O — not in a transaction) try: upload_data = self.handler.initiate_upload(file_obj, num_parts, part_size) - - # Store upload_id - file_obj.upload_id = upload_data["upload_id"] - file_obj.upload_status = File.UploadStatus.UPLOADING - file_obj.save(update_fields=["upload_id", "upload_status"]) - - return file_obj, upload_data - except Exception: # Cleanup file record if upload initiation fails file_obj.delete() raise - @transaction.atomic + # Store upload_id + file_obj.upload_id = upload_data["upload_id"] + file_obj.upload_status = File.UploadStatus.UPLOADING + file_obj.save(update_fields=["upload_id", "upload_status"]) + + return file_obj, upload_data + def complete_multipart_upload(self, file_id: int, parts: list): """ Complete a multipart upload. + + The storage handler's ``complete_upload`` (a remote call for S3) runs + outside the transaction: state is validated first, the remote assembly + happens without holding a row lock, then a short ``select_for_update`` + transaction flips the row to COMPLETED — guarding against a concurrent + complete/abort without pinning the lock across network I/O. """ - file_obj = File.objects.select_for_update().get(id=file_id) + file_obj = File.objects.get(id=file_id) # Validate state if file_obj.upload_status not in [File.UploadStatus.PENDING, File.UploadStatus.UPLOADING]: @@ -98,31 +106,32 @@ def complete_multipart_upload(self, file_id: int, parts: list): # Validate parts self._validate_parts(file_obj, parts) - # Complete with storage handler + # Complete with storage handler (network I/O — outside the transaction) try: file_path = self.handler.complete_upload(file_obj, file_obj.upload_id, parts) - - # Update file record - file_obj.file = file_path - file_obj.upload_status = File.UploadStatus.COMPLETED - file_obj.uploaded_parts = {str(p["part_number"]): p["etag"] for p in parts} - file_obj.upload_id = "" - file_obj.upload_expires_at = None - file_obj.save() - - return file_obj - except Exception: - file_obj.upload_status = File.UploadStatus.FAILED - file_obj.save(update_fields=["upload_status"]) + File.objects.filter(id=file_id).update(upload_status=File.UploadStatus.FAILED) raise - @transaction.atomic + # Persist the result under a short row lock. If a concurrent request + # already finished (or aborted) this upload, don't clobber it. + with transaction.atomic(): + locked = File.objects.select_for_update().get(id=file_id) + if locked.upload_status in (File.UploadStatus.PENDING, File.UploadStatus.UPLOADING): + locked.file = file_path + locked.upload_status = File.UploadStatus.COMPLETED + locked.uploaded_parts = {str(p["part_number"]): p["etag"] for p in parts} + locked.upload_id = "" + locked.upload_expires_at = None + locked.save() + return locked + def abort_multipart_upload(self, file_id: int): """ - Abort a multipart upload and cleanup. + Abort a multipart upload and cleanup. The storage abort (network I/O) + runs outside the transaction; the row is then marked ABORTED. """ - file_obj = File.objects.select_for_update().get(id=file_id) + file_obj = File.objects.get(id=file_id) if file_obj.upload_id: try: @@ -136,11 +145,12 @@ def abort_multipart_upload(self, file_id: int): file_obj.upload_id, ) - file_obj.upload_status = File.UploadStatus.ABORTED - file_obj.upload_id = "" - file_obj.save() - - return file_obj + with transaction.atomic(): + locked = File.objects.select_for_update().get(id=file_id) + locked.upload_status = File.UploadStatus.ABORTED + locked.upload_id = "" + locked.save() + return locked def _validate_file_params(self, file_size, num_parts, part_size): """Validate file upload parameters.""" diff --git a/testproject/files/migrations/0003_alter_file_options_alter_filetarget_options_and_more.py b/testproject/files/migrations/0003_alter_file_options_alter_filetarget_options_and_more.py new file mode 100644 index 00000000..895498cb --- /dev/null +++ b/testproject/files/migrations/0003_alter_file_options_alter_filetarget_options_and_more.py @@ -0,0 +1,204 @@ +# Generated by Django 5.2.14 on 2026-07-12 12:58 + +import baseapp.files.utils +import baseapp_core.models +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("baseapp_core", "0001_initial"), + ("files", "0002_alter_file_description_alter_file_file_content_type_and_more"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AlterModelOptions( + name="file", + options={"verbose_name": "file", "verbose_name_plural": "files"}, + ), + migrations.AlterModelOptions( + name="filetarget", + options={"verbose_name": "file target", "verbose_name_plural": "file targets"}, + ), + migrations.AlterField( + model_name="file", + name="created_by", + field=models.ForeignKey( + help_text="The user who uploaded the file", + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="files_created", + to=settings.AUTH_USER_MODEL, + verbose_name="created by", + ), + ), + migrations.AlterField( + model_name="file", + name="description", + field=models.TextField( + blank=True, + default="", + help_text="Optional description of the file", + verbose_name="description", + ), + ), + migrations.AlterField( + model_name="file", + name="file", + field=models.FileField( + help_text="The stored file", + max_length=512, + upload_to=baseapp_core.models.random_name_in("files"), + verbose_name="file", + ), + ), + migrations.AlterField( + model_name="file", + name="file_content_type", + field=models.CharField( + blank=True, + default="", + help_text="MIME type of the uploaded file", + max_length=150, + verbose_name="content type", + ), + ), + migrations.AlterField( + model_name="file", + name="file_name", + field=models.CharField( + blank=True, + default="", + help_text="Original name of the uploaded file", + max_length=512, + verbose_name="file name", + ), + ), + migrations.AlterField( + model_name="file", + name="file_size", + field=models.PositiveIntegerField( + help_text="File size in bytes", null=True, verbose_name="file size" + ), + ), + migrations.AlterField( + model_name="file", + name="name", + field=models.CharField( + blank=True, + default="", + help_text="Display name for the file", + max_length=512, + verbose_name="name", + ), + ), + migrations.AlterField( + model_name="file", + name="parent", + field=models.ForeignKey( + blank=True, + help_text="The parent document this file belongs to", + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="files", + to="baseapp_core.documentid", + verbose_name="parent", + ), + ), + migrations.AlterField( + model_name="file", + name="total_parts", + field=models.PositiveIntegerField( + blank=True, + help_text="Total number of parts for multipart upload", + null=True, + verbose_name="total parts", + ), + ), + migrations.AlterField( + model_name="file", + name="upload_expires_at", + field=models.DateTimeField( + blank=True, + db_index=True, + help_text="Expiration time for pending uploads", + null=True, + verbose_name="upload expires at", + ), + ), + migrations.AlterField( + model_name="file", + name="upload_id", + field=models.CharField( + blank=True, + default="", + help_text="S3 multipart upload ID for in-progress uploads", + max_length=512, + verbose_name="upload ID", + ), + ), + migrations.AlterField( + model_name="file", + name="upload_status", + field=models.CharField( + choices=[ + ("pending", "Pending Upload"), + ("uploading", "Upload in Progress"), + ("completed", "Upload Completed"), + ("failed", "Upload Failed"), + ("aborted", "Upload Aborted"), + ], + db_index=True, + default="completed", + help_text="Status of the file upload", + max_length=20, + verbose_name="upload status", + ), + ), + migrations.AlterField( + model_name="file", + name="uploaded_parts", + field=models.JSONField( + blank=True, + default=dict, + help_text="Tracking uploaded parts with ETags", + null=True, + verbose_name="uploaded parts", + ), + ), + migrations.AlterField( + model_name="filetarget", + name="files_count", + field=models.JSONField( + default=baseapp.files.utils.default_files_count, + help_text="Per-content-type file counts for the target", + verbose_name="files count", + ), + ), + migrations.AlterField( + model_name="filetarget", + name="is_files_enabled", + field=models.BooleanField( + default=True, + help_text="Whether files can be attached to the target", + verbose_name="is files enabled", + ), + ), + migrations.AlterField( + model_name="filetarget", + name="target", + field=models.OneToOneField( + help_text="The document this file target belongs to", + on_delete=django.db.models.deletion.CASCADE, + primary_key=True, + related_name="file_target", + serialize=False, + to="baseapp_core.documentid", + verbose_name="target", + ), + ), + ] From c5b40552f8d67e37b534febf93887fd46f7da12c Mon Sep 17 00:00:00 2001 From: Alisson Patricio Date: Sun, 12 Jul 2026 16:49:50 +0000 Subject: [PATCH 17/20] files: isort the generated 0003 migration (CI isort --check runs repo-wide) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146oeiGsELS6QVpVUQHDQXd --- ...3_alter_file_options_alter_filetarget_options_and_more.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/testproject/files/migrations/0003_alter_file_options_alter_filetarget_options_and_more.py b/testproject/files/migrations/0003_alter_file_options_alter_filetarget_options_and_more.py index 895498cb..0bfe24c2 100644 --- a/testproject/files/migrations/0003_alter_file_options_alter_filetarget_options_and_more.py +++ b/testproject/files/migrations/0003_alter_file_options_alter_filetarget_options_and_more.py @@ -1,11 +1,12 @@ # Generated by Django 5.2.14 on 2026-07-12 12:58 -import baseapp.files.utils -import baseapp_core.models import django.db.models.deletion from django.conf import settings from django.db import migrations, models +import baseapp.files.utils +import baseapp_core.models + class Migration(migrations.Migration): From 5895207d9d507ca8578a135034b914f852262489 Mon Sep 17 00:00:00 2001 From: Alisson Patricio Date: Sun, 12 Jul 2026 17:28:11 +0000 Subject: [PATCH 18/20] files: add 'from None' to serializer parent_id re-raises (Ruff B904) Suppress implicit exception chaining on the two validate_parent_id DocumentId.DoesNotExist -> ValidationError re-raises. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0146oeiGsELS6QVpVUQHDQXd --- baseapp/files/rest_framework/uploads/serializers.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/baseapp/files/rest_framework/uploads/serializers.py b/baseapp/files/rest_framework/uploads/serializers.py index cc4eb5ca..f323b953 100644 --- a/baseapp/files/rest_framework/uploads/serializers.py +++ b/baseapp/files/rest_framework/uploads/serializers.py @@ -28,7 +28,9 @@ def validate_parent_id(self, value): document_id = DocumentId.objects.get(public_id=value) return document_id.pk except DocumentId.DoesNotExist: - raise serializers.ValidationError([_("Invalid parent_id: DocumentId not found")]) + raise serializers.ValidationError( + [_("Invalid parent_id: DocumentId not found")] + ) from None def validate(self, data): """Validate file size matches parts.""" @@ -99,4 +101,6 @@ def validate_parent_id(self, value): document_id = DocumentId.objects.get(public_id=value) return document_id.pk except DocumentId.DoesNotExist: - raise serializers.ValidationError([_("Invalid parent_id: DocumentId not found")]) + raise serializers.ValidationError( + [_("Invalid parent_id: DocumentId not found")] + ) from None From 3e239eb7fb7fcae7e342a04fc4ee27328ce2cd06 Mon Sep 17 00:00:00 2001 From: Alisson Patricio Date: Fri, 7 Aug 2026 18:50:04 +0000 Subject: [PATCH 19/20] files: register count triggers via pgtrigger_register_default_track Replace the hand-rolled class_prepared receiver with the baseapp_core helper, matching baseapp_chats. Resolves the swap target with init_swapped_models so the triggers attach to the concrete table, and routes registration through pgtrigger.register (via apply_pgtrigger_tracks) instead of mutating _meta.triggers directly. Also add the swappable Meta frag to AbstractFile and AbstractFileTarget, so the concrete models inherit it like every other consumer model in the repo rather than redeclaring it. No migration: the three triggers were already in files/0001_initial and the swappable move is state-neutral (makemigrations --check is clean). Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013LkamMd6XMPeYNHbAV1DZ9 --- baseapp/files/models.py | 53 +++++++++++++------------------------ testproject/files/models.py | 8 ++---- 2 files changed, 21 insertions(+), 40 deletions(-) diff --git a/baseapp/files/models.py b/baseapp/files/models.py index 7074a27a..9e8cfc95 100644 --- a/baseapp/files/models.py +++ b/baseapp/files/models.py @@ -7,8 +7,6 @@ from django.db import models from django.db.models import OuterRef, Subquery, Value from django.db.models.functions import Coalesce -from django.db.models.signals import class_prepared -from django.dispatch import receiver from django.utils.translation import gettext_lazy as _ from model_utils.models import TimeStampedModel @@ -19,6 +17,8 @@ DocumentIdUniqueTargetMixin, random_name_in, ) +from baseapp_core.pghelpers import pgtrigger_register_default_track +from baseapp_core.swapper import init_swapped_models from .utils import default_files_count @@ -188,36 +188,6 @@ def file_target_delete_trigger(): ) -@receiver(class_prepared) -def add_file_target_triggers(sender, **kwargs): - """ - Add the FileTarget count triggers to the model when it is prepared. - This handles swappable models by adding triggers to the concrete model. - """ - # Only models that inherit from AbstractFile - if not issubclass(sender, AbstractFile): - return - - # Skip non-schema models - if sender._meta.abstract or sender._meta.proxy: - return - - # Skip swapped-out models - if sender._meta.swapped: - return - - if not hasattr(sender._meta, "triggers"): - sender._meta.triggers = [] - - existing = [t.name for t in sender._meta.triggers] - if "update_file_target_on_insert" not in existing: - sender._meta.triggers.append(file_target_insert_trigger()) - if "update_file_target_on_update" not in existing: - sender._meta.triggers.append(file_target_update_trigger()) - if "update_file_target_on_delete" not in existing: - sender._meta.triggers.append(file_target_delete_trigger()) - - class AbstractFileTarget(DocumentIdUniqueTargetMixin): # Override only to keep the existing related_name/help_text (avoids a no-op # AlterField migration on the concrete swappable models); get_for_object and @@ -246,6 +216,7 @@ class Meta: abstract = True verbose_name = _("file target") verbose_name_plural = _("file targets") + swappable = swapper.swappable_setting("baseapp_files", "FileTarget") def __str__(self): return f"FileTarget for {self.target}" @@ -434,11 +405,25 @@ class Meta: abstract = True verbose_name = _("file") verbose_name_plural = _("files") - # Note: Triggers are registered via the class_prepared signal in this module - # to properly handle swappable models. + swappable = swapper.swappable_setting("baseapp_files", "File") @classmethod def get_graphql_object_type(cls): from .graphql.object_types import FileObjectType return FileObjectType + + +# Resolve the concrete swap target (e.g. files.File) so the count triggers are +# attached to the real table. Consumers can override these via +# `@pgtrigger_register_track(...)` on their concrete model. +File = init_swapped_models([("baseapp_files", "File")]) + +pgtrigger_register_default_track( + File, + [ + file_target_insert_trigger(), + file_target_update_trigger(), + file_target_delete_trigger(), + ], +) diff --git a/testproject/files/models.py b/testproject/files/models.py index 2fb2a62e..7f28c177 100644 --- a/testproject/files/models.py +++ b/testproject/files/models.py @@ -1,15 +1,11 @@ -import swapper - from baseapp.files.models import AbstractFile, AbstractFileTarget class FileTarget(AbstractFileTarget): class Meta(AbstractFileTarget.Meta): - abstract = False - swappable = swapper.swappable_setting("baseapp_files", "FileTarget") + pass class File(AbstractFile): class Meta(AbstractFile.Meta): - abstract = False - swappable = swapper.swappable_setting("baseapp_files", "File") + pass From 843399335e1e42773e40927d1bef32da7a88be53 Mon Sep 17 00:00:00 2001 From: Alisson Patricio Date: Fri, 7 Aug 2026 19:10:24 +0000 Subject: [PATCH 20/20] files: isolate presigned upload token logic in PresignedUploadToken The signed tokens that authorize part uploads on the local-storage fallback were minted in storage/local.py and verified inline in presigned_views.py, with the payload shape spelled out in both places (plus twice more in tests) and the 1h lifetime hardcoded three times. Move mint/verify into baseapp/files/tokens.py, which cuts ~30 lines of branching out of the view. Two behaviour changes fall out of centralising it: - Tokens are now salted, so a signed payload minted elsewhere in the project can no longer satisfy this endpoint. - Lifetime comes from FILE_UPLOAD_PRESIGNED_URL_EXPIRATION, the setting the S3 handler and README already use, instead of a bare 3600. The local handler's reported `expires_in` now follows it too. Tests mint through the class rather than duplicating the payload, and cover the previously untested upload-session mismatch branch. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013LkamMd6XMPeYNHbAV1DZ9 --- baseapp/files/UPLOAD_FLOW.md | 2 +- .../rest_framework/uploads/presigned_views.py | 48 +++------- baseapp/files/storage/local.py | 16 ++-- baseapp/files/tests/test_presigned_uploads.py | 39 +++++--- baseapp/files/tokens.py | 93 +++++++++++++++++++ 5 files changed, 138 insertions(+), 60 deletions(-) create mode 100644 baseapp/files/tokens.py diff --git a/baseapp/files/UPLOAD_FLOW.md b/baseapp/files/UPLOAD_FLOW.md index c66938ca..16ddbf67 100644 --- a/baseapp/files/UPLOAD_FLOW.md +++ b/baseapp/files/UPLOAD_FLOW.md @@ -75,7 +75,7 @@ Body: ETag: abc123def456... ``` -**Note:** The token is generated using Django signing and is valid for 1 hour. No session authentication or CSRF protection is required for this endpoint. The ETag is returned both in the response body and as an HTTP header for S3 compatibility. +**Note:** The token is minted and verified by `PresignedUploadToken` (`baseapp/files/tokens.py`) using Django signing, and is valid for `FILE_UPLOAD_PRESIGNED_URL_EXPIRATION` seconds (1 hour by default). No session authentication or CSRF protection is required for this endpoint. The ETag is returned both in the response body and as an HTTP header for S3 compatibility. ### 3. Complete Upload **POST** `/v1/files/uploads/{id}/complete/` diff --git a/baseapp/files/rest_framework/uploads/presigned_views.py b/baseapp/files/rest_framework/uploads/presigned_views.py index 2ea66dc0..601f9b6f 100644 --- a/baseapp/files/rest_framework/uploads/presigned_views.py +++ b/baseapp/files/rest_framework/uploads/presigned_views.py @@ -1,7 +1,6 @@ import logging import swapper -from django.core import signing from django.utils.translation import gettext_lazy as _ from django.views.decorators.csrf import csrf_exempt from rest_framework import status, viewsets @@ -11,6 +10,7 @@ from rest_framework.response import Response from ...services.upload_service import UploadService +from ...tokens import PresignedUploadToken, PresignedUploadTokenError logger = logging.getLogger(__name__) @@ -63,37 +63,14 @@ def upload_part(self, request, pk=None, part_number=None): "etag": "abc123..." } """ - # Get and validate the signed token - token = request.query_params.get("token") - if not token: - return Response( - {"error": _("Missing token parameter")}, - status=status.HTTP_401_UNAUTHORIZED, - ) - try: - # Verify the token (max age: 1 hour) - token_data = signing.loads(token, max_age=3600) - - # Validate token matches request - if str(token_data.get("file_id")) != str(pk) or str( - token_data.get("part_number") - ) != str(part_number): - return Response( - {"error": _("Invalid token for this file/part")}, - status=status.HTTP_401_UNAUTHORIZED, - ) - - except signing.SignatureExpired: - return Response( - {"error": _("Token has expired")}, - status=status.HTTP_401_UNAUTHORIZED, - ) - except signing.BadSignature: - return Response( - {"error": _("Invalid token signature")}, - status=status.HTTP_401_UNAUTHORIZED, + token_data = PresignedUploadToken.verify( + request.query_params.get("token"), + file_id=pk, + part_number=part_number, ) + except PresignedUploadTokenError as exc: + return Response({"error": exc.message}, status=status.HTTP_401_UNAUTHORIZED) # Get the file object try: @@ -104,13 +81,10 @@ def upload_part(self, request, pk=None, part_number=None): status=status.HTTP_404_NOT_FOUND, ) - # Bind the token to the current upload session: a token minted for a - # previous initiation (different upload_id) must not upload into a new one. - if str(token_data.get("upload_id")) != str(file_obj.upload_id): - return Response( - {"error": _("Token does not match the current upload session")}, - status=status.HTTP_401_UNAUTHORIZED, - ) + try: + PresignedUploadToken.check_upload_session(token_data, file_obj.upload_id) + except PresignedUploadTokenError as exc: + return Response({"error": exc.message}, status=status.HTTP_401_UNAUTHORIZED) # Validate state if file_obj.upload_status not in [File.UploadStatus.PENDING, File.UploadStatus.UPLOADING]: diff --git a/baseapp/files/storage/local.py b/baseapp/files/storage/local.py index affb5d2c..7d05c51a 100644 --- a/baseapp/files/storage/local.py +++ b/baseapp/files/storage/local.py @@ -4,10 +4,10 @@ from typing import Any, Dict, List, Optional from django.conf import settings -from django.core import signing from baseapp_core.models import random_name_in +from ..tokens import PresignedUploadToken from .base import BaseUploadHandler @@ -41,13 +41,11 @@ def initiate_upload(self, file_obj, num_parts: int, part_size: int) -> Dict[str, # Generate presigned URLs with signed tokens presigned_urls = [] for part_num in range(1, num_parts + 1): - # Create a signed token for this specific file and part - token_data = { - "file_id": file_obj.id, - "part_number": part_num, - "upload_id": upload_id, - } - token = signing.dumps(token_data) + token = PresignedUploadToken.mint( + file_id=file_obj.id, + part_number=part_num, + upload_id=upload_id, + ) presigned_urls.append( { @@ -60,7 +58,7 @@ def initiate_upload(self, file_obj, num_parts: int, part_size: int) -> Dict[str, return { "upload_id": upload_id, "presigned_urls": presigned_urls, - "expires_in": 3600, + "expires_in": PresignedUploadToken.max_age(), } def upload_part(self, upload_id: str, part_number: int, data: bytes) -> str: diff --git a/baseapp/files/tests/test_presigned_uploads.py b/baseapp/files/tests/test_presigned_uploads.py index 56af512c..9642eebe 100644 --- a/baseapp/files/tests/test_presigned_uploads.py +++ b/baseapp/files/tests/test_presigned_uploads.py @@ -14,11 +14,12 @@ import pytest import swapper from django.conf import settings -from django.core import signing from django.utils import timezone from freezegun import freeze_time from rest_framework import status +from baseapp.files.tokens import PresignedUploadToken + File = swapper.load_model("baseapp_files", "File") pytestmark = pytest.mark.django_db @@ -131,12 +132,10 @@ def test_expired_token(self, client, initiated_upload): file_obj = File.get_by_public_id(initiated_upload["id"]) with freeze_time(timezone.now() - timedelta(hours=2)): - token = signing.dumps( - { - "file_id": file_obj.id, - "part_number": 1, - "upload_id": file_obj.upload_id, - } + token = PresignedUploadToken.mint( + file_id=file_obj.id, + part_number=1, + upload_id=file_obj.upload_id, ) response = put_part( @@ -161,12 +160,10 @@ def test_upload_part_invalid_status(self, client, initiated_upload): def test_upload_part_number_out_of_range(self, client, initiated_upload): """A part number above total_parts is rejected even with a valid token.""" file_obj = File.get_by_public_id(initiated_upload["id"]) - token = signing.dumps( - { - "file_id": file_obj.id, - "part_number": 5, - "upload_id": file_obj.upload_id, - } + token = PresignedUploadToken.mint( + file_id=file_obj.id, + part_number=5, + upload_id=file_obj.upload_id, ) response = put_part( @@ -176,6 +173,22 @@ def test_upload_part_number_out_of_range(self, client, initiated_upload): assert response.status_code == status.HTTP_400_BAD_REQUEST assert "Invalid part number" in response.json()["error"] + def test_token_from_previous_upload_session(self, client, initiated_upload): + """A token minted for an earlier initiation cannot upload into a new session.""" + file_obj = File.get_by_public_id(initiated_upload["id"]) + token = PresignedUploadToken.mint( + file_id=file_obj.id, + part_number=1, + upload_id="stale-upload-id", + ) + + response = put_part( + client, f"/v1/files/presigned-uploads/{file_obj.id}/upload-part/1?token={token}" + ) + + assert response.status_code == status.HTTP_401_UNAUTHORIZED + assert "does not match the current upload session" in response.json()["error"] + def test_token_for_other_file(self, client, user_client, temp_media_root, initiated_upload): """A token signed for one file cannot be used on another file's URL.""" other_file = File.objects.create( diff --git a/baseapp/files/tokens.py b/baseapp/files/tokens.py new file mode 100644 index 00000000..7545e5f5 --- /dev/null +++ b/baseapp/files/tokens.py @@ -0,0 +1,93 @@ +from typing import Any, Dict + +from django.conf import settings +from django.core import signing +from django.utils.translation import gettext_lazy as _ + + +class PresignedUploadTokenError(Exception): + """ + Raised when a presigned upload token is missing, expired, or does not match + the request it was presented with. + + `message` is a client-safe, translated string: it describes only which check + failed, never any detail of the signed payload. + """ + + def __init__(self, message: str) -> None: + self.message = message + super().__init__(message) + + +class PresignedUploadToken: + """ + Mints and verifies the signed tokens that authenticate part uploads against + the local-storage presigned fallback. + + `LocalUploadHandler` embeds a token in each presigned URL it hands out, and + `PresignedUploadViewSet` verifies it — that endpoint has no session auth or + CSRF, so the token is the only thing authorizing the write. The S3 handler + issues real presigned URLs and does not use this. + """ + + SALT = "baseapp_files.presigned_upload" + + @classmethod + def max_age(cls) -> int: + """Token lifetime in seconds; shared with the S3 handler's URL expiration.""" + return getattr(settings, "FILE_UPLOAD_PRESIGNED_URL_EXPIRATION", 3600) + + @classmethod + def mint(cls, *, file_id: Any, part_number: int, upload_id: str) -> str: + """Sign a token authorizing one part of one upload session.""" + return signing.dumps( + { + "file_id": file_id, + "part_number": part_number, + "upload_id": upload_id, + }, + salt=cls.SALT, + ) + + @classmethod + def verify(cls, token: str | None, *, file_id: Any, part_number: Any) -> Dict[str, Any]: + """ + Validate `token` against the file and part it is being used for and + return its payload. + + The payload's `upload_id` still has to be checked against the file's + current upload session — see `check_upload_session`, which runs once the + file has been loaded. + + Raises: + PresignedUploadTokenError: token missing, expired, tampered with, or + issued for a different file/part. + """ + if not token: + raise PresignedUploadTokenError(_("Missing token parameter")) + + try: + token_data = signing.loads(token, max_age=cls.max_age(), salt=cls.SALT) + except signing.SignatureExpired: + raise PresignedUploadTokenError(_("Token has expired")) from None + except signing.BadSignature: + raise PresignedUploadTokenError(_("Invalid token signature")) from None + + if str(token_data.get("file_id")) != str(file_id) or str( + token_data.get("part_number") + ) != str(part_number): + raise PresignedUploadTokenError(_("Invalid token for this file/part")) + + return token_data + + @staticmethod + def check_upload_session(token_data: Dict[str, Any], upload_id: str) -> None: + """ + Bind the token to the current upload session: one minted for a previous + initiation (different `upload_id`) must not upload into a new one. + + Raises: + PresignedUploadTokenError: token belongs to a different session. + """ + if str(token_data.get("upload_id")) != str(upload_id): + raise PresignedUploadTokenError(_("Token does not match the current upload session"))