diff --git a/baseapp/activity_log/tests/test_graphql_mutations.py b/baseapp/activity_log/tests/test_graphql_mutations.py index f8e79a4e..e52fa1af 100644 --- a/baseapp/activity_log/tests/test_graphql_mutations.py +++ b/baseapp/activity_log/tests/test_graphql_mutations.py @@ -57,12 +57,8 @@ def test_add_comment_is_public(django_user_client, graphql_user_client, celery_c id name biography - image(width: 100, height: 100) { - url - } - bannerImage(width: 100, height: 100) { - url - } + image(width: 100, height: 100) + bannerImage(width: 100, height: 100) } errors { field diff --git a/baseapp/content_feed/graphql/object_types.py b/baseapp/content_feed/graphql/object_types.py index 1d092834..991d0013 100644 --- a/baseapp/content_feed/graphql/object_types.py +++ b/baseapp/content_feed/graphql/object_types.py @@ -43,7 +43,7 @@ class ContentPostObjectType(DjangoObjectType): class Meta: interfaces = graphql_shared_interfaces.get( - RelayNode, "MentionsInterface", "ReactionsInterface" + RelayNode, "MentionsInterface", "ReactionsInterface", "CommentsInterface" ) model = ContentPost fields = ( @@ -66,4 +66,7 @@ def get_queryset(cls, queryset, info) -> "QuerySet": # (`reactions_count`, `is_reactions_enabled`) don't N+1 per post. if service := shared_services.get("reactable_metadata"): queryset = service.annotate_queryset(queryset) + # Same for CommentsInterface (`comments_count`, `is_comments_enabled`). + if service := shared_services.get("commentable_metadata"): + queryset = service.annotate_queryset(queryset) return queryset diff --git a/baseapp/content_feed/tests/test_graphql_mutations_create.py b/baseapp/content_feed/tests/test_graphql_mutations_create.py index 0f5fccce..f0e410ff 100644 --- a/baseapp/content_feed/tests/test_graphql_mutations_create.py +++ b/baseapp/content_feed/tests/test_graphql_mutations_create.py @@ -28,9 +28,7 @@ node { id pk - image(width:600, height:0){ - url - } + image(width:600, height:0) } } } diff --git a/baseapp/files/README.md b/baseapp/files/README.md new file mode 100644 index 00000000..cd9d4f68 --- /dev/null +++ b/baseapp/files/README.md @@ -0,0 +1,811 @@ +# BaseApp Files - S3 Direct Multipart Upload + +REST API endpoints for S3 direct multipart uploads, allowing clients to upload large files directly to S3 without going through the Django backend. + +## Table of Contents + +- [Overview](#overview) +- [Installation](#installation) +- [Configuration](#configuration) + - [Production (S3)](#production-s3) + - [Development (Local Storage)](#development-local-storage) +- [API Usage](#api-usage) + - [Complete Upload Flow](#complete-upload-flow) + - [API Endpoints](#api-endpoints) +- [Client Implementation](#client-implementation) +- [Background Tasks](#background-tasks) +- [Troubleshooting](#troubleshooting) +- [Advanced Topics](#advanced-topics) + +## Overview + +This module provides S3-compatible multipart upload functionality with: + +- **Direct S3 uploads** - Files upload directly to S3, not through Django +- **Presigned URLs** - Secure, time-limited upload URLs +- **Automatic storage detection** - Works with S3 in production, local storage in dev/test +- **Parent object support** - Attach files to any model (posts, comments, etc.) +- **Automatic cleanup** - Celery task cleans up expired/abandoned uploads + +### Architecture + +``` +Client → Backend (initiate) → S3 Presigned URLs +Client → S3 (upload parts directly) +Client → Backend (complete) → File record created +``` + +## Installation + +### 1. Run Migration + +After deploying the code, run the migration to add upload tracking fields: + +```bash +# Using docker compose +docker compose exec backend python manage.py migrate files + +# Or directly +python manage.py migrate files +``` + +This adds the following fields to the File model: +- `upload_status` - Track upload state +- `upload_id` - S3 multipart upload ID +- `total_parts` - Number of parts +- `uploaded_parts` - JSON tracking ETags +- `upload_expires_at` - Expiration timestamp + +### 2. URLs Already Configured + +The URLs are automatically included in your project via `testproject/urls.py`. + +Available endpoints: +``` +POST /v1/files/uploads - Initiate upload +POST /v1/files/uploads/{id}/complete - Complete upload +DELETE /v1/files/uploads/{id} - Abort upload +GET /v1/files - List files +GET /v1/files/{id} - Retrieve file +PATCH /v1/files/{id} - Update file metadata +DELETE /v1/files/{id} - Delete file +POST /v1/files/{id}/set-parent - Set parent relationship +``` + +## Configuration + +### Production (S3) + +#### 1. Install boto3 (if not already installed) + +```bash +pip install boto3 +``` + +#### 2. Configure Django Settings + +Add to your `settings.py` or environment variables: + +```python +# AWS S3 Configuration +AWS_ACCESS_KEY_ID = env('AWS_ACCESS_KEY_ID') +AWS_SECRET_ACCESS_KEY = env('AWS_SECRET_ACCESS_KEY') +AWS_STORAGE_BUCKET_NAME = env('AWS_STORAGE_BUCKET_NAME') +AWS_S3_REGION_NAME = env('AWS_S3_REGION_NAME', default='us-east-1') + +# File upload settings +MAX_FILE_UPLOAD_SIZE = 5 * 1024 * 1024 * 1024 # 5GB default +FILE_UPLOAD_PRESIGNED_URL_EXPIRATION = 3600 # 1 hour + +# Optional: Use S3 as default storage +STORAGES = { + 'default': { + 'BACKEND': 'storages.backends.s3boto3.S3Boto3Storage', + }, + 'staticfiles': { + 'BACKEND': 'django.contrib.staticfiles.storage.StaticFilesStorage', + }, +} +``` + +#### 3. Environment Variables + +Create or update your `.env` file: + +```bash +AWS_ACCESS_KEY_ID=your-access-key-id +AWS_SECRET_ACCESS_KEY=your-secret-access-key +AWS_STORAGE_BUCKET_NAME=your-bucket-name +AWS_S3_REGION_NAME=us-east-1 +``` + +#### 4. S3 Bucket Configuration + +Ensure your S3 bucket has the correct CORS configuration: + +```json +[ + { + "AllowedHeaders": ["*"], + "AllowedMethods": ["GET", "PUT", "POST", "DELETE"], + "AllowedOrigins": ["https://yourdomain.com"], + "ExposeHeaders": ["ETag"], + "MaxAgeSeconds": 3000 + } +] +``` + +**Important**: The `ExposeHeaders: ["ETag"]` is required so clients can read ETags from S3 responses. + +### Development (Local Storage) + +No additional configuration needed! The system automatically detects local storage and provides a fallback implementation. + +**How it works:** +- Storage factory checks if `default_storage` is `S3Boto3Storage` +- If not S3, uses `LocalUploadHandler` automatically +- Local handler stores parts in `MEDIA_ROOT/temp_uploads/` +- Parts are combined on completion + +**Note**: Local storage doesn't support true direct uploads - it's a simplified fallback for development/testing only. + +## API Usage + +### Complete Upload Flow + +#### 1. Initiate Upload + +**Request:** +```bash +POST /v1/files/uploads +Authorization: Bearer {jwt_token} +Content-Type: application/json + +{ + "file_name": "large-video.mp4", + "file_size": 104857600, + "file_content_type": "video/mp4", + "num_parts": 20, + "part_size": 5242880, + "parent_id": "b3c5…-document-public-id" // optional (DocumentId public_id of the parent) +} +``` + +**Response:** +```json +{ + "id": 456, + "upload_id": "abc123xyz...", + "presigned_urls": [ + { + "part_number": 1, + "url": "https://bucket.s3.amazonaws.com/...?signature=..." + }, + { + "part_number": 2, + "url": "https://bucket.s3.amazonaws.com/...?signature=..." + } + // ... up to 20 parts + ], + "expires_in": 3600, + "upload_status": "uploading" +} +``` + +**Calculating Parts:** +```python +import math + +file_size = 104857600 # 100 MB +part_size = 5242880 # 5 MB (minimum for S3) +num_parts = math.ceil(file_size / part_size) # 20 parts +``` + +#### 2. Upload Parts Directly to S3 + +For each presigned URL, upload the corresponding file chunk using HTTP PUT: + +```javascript +// Example with fetch API +const part_number = 1; +const presigned_url = presigned_urls[0].url; +const chunk = file.slice(start, end); + +const response = await fetch(presigned_url, { + method: 'PUT', + body: chunk, + headers: { + 'Content-Type': file_content_type + } +}); + +// IMPORTANT: Save the ETag from response headers +const etag = response.headers.get('ETag').replace(/"/g, ''); +``` + +**Important Notes:** +- Upload to presigned URLs with HTTP PUT (not POST) +- S3 returns an ETag header for each part - you MUST save these +- ETags are required to complete the upload +- Upload parts in any order (parallel uploads supported) + +#### 3. Complete Upload + +Once all parts are uploaded, send the completion request with ETags: + +**Request:** +```bash +POST /v1/files/uploads/456/complete +Authorization: Bearer {jwt_token} +Content-Type: application/json + +{ + "parts": [ + {"part_number": 1, "etag": "abc123..."}, + {"part_number": 2, "etag": "def456..."}, + {"part_number": 3, "etag": "ghi789..."} + // ... all parts with their ETags + ] +} +``` + +**Response:** +```json +{ + "id": 456, + "file_name": "large-video.mp4", + "file_size": 104857600, + "file_content_type": "video/mp4", + "url": "https://bucket.s3.amazonaws.com/files/uuid.mp4", + "upload_status": "completed", + "created_by": 1, + "created_by_name": "John Doe", + "created": "2025-12-29T10:00:00Z", + "parent_id": "b3c5…-document-public-id" +} +``` + +#### 4. (Optional) Set Parent Later + +If you didn't specify a parent during initiation, you can set it later: + +**Request:** +```bash +POST /v1/files/456/set-parent +Authorization: Bearer {jwt_token} +Content-Type: application/json + +{ + "parent_id": "b3c5…-document-public-id" +} +``` + +### API Endpoints + +#### Initiate Upload +``` +POST /v1/files/uploads +``` + +**Parameters:** +- `file_name` (string, required) - Original filename +- `file_size` (integer, required) - Total file size in bytes +- `file_content_type` (string, required) - MIME type (e.g., "video/mp4") +- `num_parts` (integer, required) - Number of parts (1-10,000) +- `part_size` (integer, required) - Size of each part in bytes (min 5MB for multipart) +- `parent_id` (UUID, optional) - `DocumentId` public_id of the parent object + +**Validation:** +- File size must not exceed `MAX_FILE_UPLOAD_SIZE` (default 5GB) +- Part size must be at least 5MB for multipart uploads (S3 requirement) +- Number of parts must be between 1 and 10,000 +- File size must match parts: `(num_parts - 1) * part_size <= file_size <= num_parts * part_size` + +#### Complete Upload +``` +POST /v1/files/uploads/{id}/complete +``` + +**Parameters:** +- `parts` (array, required) - Array of objects with: + - `part_number` (integer) - Part number (1-indexed) + - `etag` (string) - ETag from S3 response + +**Requirements:** +- All parts must be included +- Part numbers must be sequential (1, 2, 3, ...) +- ETags must match S3-uploaded parts +- Only file owner can complete upload + +#### Abort Upload +``` +DELETE /v1/files/uploads/{id} +``` + +**Notes:** +- Cleans up S3 multipart upload +- Marks file as "aborted" +- Only file owner can abort +- Prevents charges for incomplete S3 uploads + +#### List Files +``` +GET /v1/files +``` + +**Query Parameters:** +- `status` (string, optional) - Filter by upload_status (default: "completed") +- `parent_id` (UUID, optional) - Filter by parent `DocumentId` public_id + +**Notes:** +- Returns only user's own files (unless staff) +- Includes only completed files by default +- Use `?status=uploading` to see in-progress uploads + +#### Retrieve File +``` +GET /v1/files/{id} +``` + +#### Update File Metadata +``` +PATCH /v1/files/{id} +``` + +**Editable fields:** +- `name` - Display name +- `description` - File description + +**Read-only fields:** +- `file_name`, `file_size`, `file_content_type` - Set during upload +- `upload_status` - Managed by system + +#### Delete File +``` +DELETE /v1/files/{id} +``` + +**Notes:** +- Only file owner can delete +- Deletes database record +- Does NOT delete S3 object (by design, for safety) + +#### Set Parent +``` +POST /v1/files/{id}/set-parent +``` + +**Parameters:** +- `parent_id` (UUID, required) - `DocumentId` public_id of the parent object + +## Client Implementation + +### JavaScript Example (Complete Flow) + +```javascript +class MultipartUploader { + constructor(apiUrl, token) { + this.apiUrl = apiUrl; + this.token = token; + } + + async uploadFile(file, parentId = null) { + // 1. Calculate parts + const partSize = 5 * 1024 * 1024; // 5MB + const numParts = Math.ceil(file.size / partSize); + + // 2. Initiate upload + const initResponse = await fetch(`${this.apiUrl}/v1/files/uploads`, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${this.token}`, + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + file_name: file.name, + file_size: file.size, + file_content_type: file.type, + num_parts: numParts, + part_size: partSize, + parent_id: parentId + }) + }); + + const { id, upload_id, presigned_urls } = await initResponse.json(); + + // 3. Upload parts to S3 + const parts = []; + for (let i = 0; i < numParts; i++) { + const start = i * partSize; + const end = Math.min(start + partSize, file.size); + const chunk = file.slice(start, end); + + const partResponse = await fetch(presigned_urls[i].url, { + method: 'PUT', + body: chunk + }); + + // Extract ETag (remove quotes) + const etag = partResponse.headers.get('ETag').replace(/"/g, ''); + + parts.push({ + part_number: i + 1, + etag: etag + }); + + // Optional: Report progress + const progress = ((i + 1) / numParts) * 100; + console.log(`Upload progress: ${progress.toFixed(1)}%`); + } + + // 4. Complete upload + const completeResponse = await fetch( + `${this.apiUrl}/v1/files/uploads/${id}/complete`, + { + method: 'POST', + headers: { + 'Authorization': `Bearer ${this.token}`, + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ parts }) + } + ); + + return await completeResponse.json(); + } + + async abortUpload(uploadId) { + await fetch(`${this.apiUrl}/v1/files/uploads/${uploadId}`, { + method: 'DELETE', + headers: { + 'Authorization': `Bearer ${this.token}` + } + }); + } +} + +// Usage +const uploader = new MultipartUploader('https://api.example.com', userToken); + +try { + const fileInput = document.querySelector('input[type="file"]'); + const file = fileInput.files[0]; + + const result = await uploader.uploadFile(file, 'b3c5…-document-public-id'); + console.log('Upload complete:', result.url); +} catch (error) { + console.error('Upload failed:', error); +} +``` + +### Python Example (for testing) + +```python +import requests +import math + +def upload_file_multipart(api_url, token, file_path, parent_id=None): + """Upload a file using multipart upload.""" + + # 1. Read file info + with open(file_path, 'rb') as f: + file_data = f.read() + + file_size = len(file_data) + file_name = file_path.split('/')[-1] + + # 2. Calculate parts + part_size = 5 * 1024 * 1024 # 5MB + num_parts = math.ceil(file_size / part_size) + + # 3. Initiate upload + headers = {'Authorization': f'Bearer {token}'} + init_response = requests.post( + f'{api_url}/v1/files/uploads', + headers=headers, + json={ + 'file_name': file_name, + 'file_size': file_size, + 'file_content_type': 'application/octet-stream', + 'num_parts': num_parts, + 'part_size': part_size, + 'parent_id': parent_id + } + ) + init_data = init_response.json() + + # 4. Upload parts + parts = [] + for i, presigned_url_data in enumerate(init_data['presigned_urls']): + start = i * part_size + end = min(start + part_size, file_size) + chunk = file_data[start:end] + + # Upload to S3 + part_response = requests.put( + presigned_url_data['url'], + data=chunk + ) + + # Extract ETag + etag = part_response.headers['ETag'].strip('"') + + parts.append({ + 'part_number': i + 1, + 'etag': etag + }) + + # 5. Complete upload + complete_response = requests.post( + f"{api_url}/v1/files/uploads/{init_data['id']}/complete", + headers=headers, + json={'parts': parts} + ) + + return complete_response.json() + +# Usage +result = upload_file_multipart( + 'https://api.example.com', + 'your-jwt-token', + '/path/to/large-file.mp4', + parent_id='b3c5…-document-public-id' +) +print(f"File uploaded: {result['url']}") +``` + +## Background Tasks + +### Automatic Cleanup of Expired Uploads + +Uploads that aren't completed within 24 hours are automatically cleaned up. + +#### Setup Celery Beat + +Add to your `settings.py`: + +```python +from celery.schedules import crontab + +CELERY_BEAT_SCHEDULE = { + 'cleanup-expired-uploads': { + 'task': 'baseapp.files.services.cleanup.cleanup_expired_uploads', + 'schedule': crontab(minute=0), # Run every hour + }, +} +``` + +#### What It Does + +The cleanup task: +1. Finds files with status "pending" or "uploading" where `upload_expires_at < now()` +2. Aborts the S3 multipart upload (prevents storage charges) +3. Marks the file as "aborted" +4. Clears the `upload_id` + +#### Manual Cleanup + +You can also run cleanup manually: + +```python +from baseapp.files.services.cleanup import cleanup_expired_uploads, cleanup_failed_uploads + +# Cleanup expired uploads +cleanup_expired_uploads() + +# Cleanup old failed/aborted uploads (older than 7 days) +cleanup_failed_uploads(days_old=7) +``` + +Or via management command: + +```bash +docker compose exec backend python manage.py shell +>>> from baseapp.files.services.cleanup import cleanup_expired_uploads +>>> cleanup_expired_uploads() +``` + +## Troubleshooting + +### "Part size must be at least 5MB" + +**Problem:** S3 requires multipart uploads to have parts of at least 5MB (except the last part). + +**Solution:** +- Use `part_size >= 5242880` (5MB) for files larger than 5MB +- For files smaller than 5MB, use `num_parts = 1` and set `part_size = file_size` + +### "CORS policy" errors when uploading to S3 + +**Problem:** Browser blocks S3 upload due to CORS restrictions. + +**Solution:** Configure CORS on your S3 bucket (see [Configuration](#4-s3-bucket-configuration) above). + +### ETags don't match / Upload completion fails + +**Problem:** ETags sent to complete endpoint don't match S3's records. + +**Solution:** +- Ensure you're extracting ETags from S3 response headers correctly +- Remove quotes from ETags: `etag.replace(/"/g, '')` +- Upload parts in the correct order (part 1, 2, 3...) +- Don't modify file chunks before uploading + +### Uploads stuck in "uploading" status + +**Problem:** Client uploaded parts but never called complete endpoint. + +**Solution:** +- Ensure your client code calls the complete endpoint +- Check for JavaScript errors during upload +- Abandoned uploads are automatically cleaned up after 24 hours + +### "File size doesn't match parts" + +**Problem:** The validation `(num_parts - 1) * part_size <= file_size <= num_parts * part_size` failed. + +**Solution:** +```javascript +// Correct calculation +const partSize = 5 * 1024 * 1024; // 5MB +const numParts = Math.ceil(fileSize / partSize); + +// Incorrect - don't round down! +const numParts = Math.floor(fileSize / partSize); // ❌ Wrong +``` + +### Local storage fallback not working + +**Problem:** Getting S3 errors in development. + +**Solution:** +- Local fallback only works if NOT using S3Boto3Storage +- Check `default_storage` in Django shell: + ```python + from django.core.files.storage import default_storage + print(type(default_storage)) + # Should NOT be S3Boto3Storage for local fallback + ``` + +## Advanced Topics + +### Custom Storage Backends + +You can add support for other storage backends (GCS, Azure, etc.) by: + +1. Create a new handler in `baseapp/files/storage/`: + +```python +# baseapp/files/storage/gcs.py +from .base import BaseUploadHandler + +class GCSMultipartUploadHandler(BaseUploadHandler): + def supports_multipart(self) -> bool: + return True + + # Implement other methods... +``` + +2. Update the factory in `baseapp/files/storage/__init__.py`: + +```python +def get_upload_handler(): + storage_class = type(default_storage) + + if "S3Boto3Storage" in str(storage_class): + from .s3 import S3MultipartUploadHandler + return S3MultipartUploadHandler() + elif "GoogleCloudStorage" in str(storage_class): + from .gcs import GCSMultipartUploadHandler + return GCSMultipartUploadHandler() + else: + from .local import LocalUploadHandler + return LocalUploadHandler() +``` + +### Resumable Uploads (not planned) + +Resuming interrupted uploads across sessions is intentionally **out of scope** — the +package provides multipart upload for large files only. If an upload is interrupted, +the client aborts (or lets the expiration cleanup task abort) and re-initiates. +Clients may still pause/resume within a session by holding on to the presigned URLs +returned at initiation, uploading only the remaining parts while the URLs are valid. + +### File Size Quotas + +Add per-user file size quotas: + +```python +# In UploadService._validate_file_params +def _validate_file_params(self, file_size, num_parts, part_size, user): + # Existing validation... + + # Check user quota + user_total = File.objects.filter( + created_by=user, + upload_status='completed' + ).aggregate(total=Sum('file_size'))['total'] or 0 + + user_quota = getattr(settings, 'FILE_UPLOAD_QUOTA_PER_USER', 10 * 1024 * 1024 * 1024) # 10GB + + if user_total + file_size > user_quota: + raise ValueError(f"Upload would exceed your quota of {user_quota} bytes") +``` + +### File Type Restrictions + +Restrict allowed file types: + +```python +# In settings.py +ALLOWED_FILE_TYPES = [ + 'image/jpeg', + 'image/png', + 'video/mp4', + 'application/pdf', +] + +# In InitiateUploadSerializer.validate_file_content_type +def validate_file_content_type(self, value): + allowed = getattr(settings, 'ALLOWED_FILE_TYPES', None) + if allowed and value not in allowed: + raise serializers.ValidationError( + f"File type {value} not allowed. Allowed types: {', '.join(allowed)}" + ) + return value +``` + +### Progress Notifications + +Add real-time progress updates via WebSockets: + +```python +# When parts are uploaded (would need additional endpoint) +from channels.layers import get_channel_layer +from asgiref.sync import async_to_sync + +def notify_upload_progress(file_id, uploaded_parts, total_parts): + channel_layer = get_channel_layer() + async_to_sync(channel_layer.group_send)( + f'upload_{file_id}', + { + 'type': 'upload_progress', + 'uploaded': uploaded_parts, + 'total': total_parts, + 'progress': (uploaded_parts / total_parts) * 100 + } + ) +``` + +### Parallel Part Uploads + +The API supports parallel part uploads out of the box. Example: + +```javascript +// Upload all parts in parallel +const uploadPromises = presigned_urls.map(async (urlData, index) => { + const start = index * partSize; + const end = Math.min(start + partSize, file.size); + const chunk = file.slice(start, end); + + const response = await fetch(urlData.url, { + method: 'PUT', + body: chunk + }); + + return { + part_number: index + 1, + etag: response.headers.get('ETag').replace(/"/g, '') + }; +}); + +const parts = await Promise.all(uploadPromises); +``` + +## Support + +For issues or questions: +1. Check the [Troubleshooting](#troubleshooting) section +2. Review API error responses (they include detailed error messages) +3. Check Django logs for server-side errors +4. Verify S3 bucket permissions and CORS configuration diff --git a/baseapp/files/UPLOAD_FLOW.md b/baseapp/files/UPLOAD_FLOW.md new file mode 100644 index 00000000..16ddbf67 --- /dev/null +++ b/baseapp/files/UPLOAD_FLOW.md @@ -0,0 +1,190 @@ +# File Upload Flow + +## Overview + +The file upload system supports both direct S3 uploads (production) and local storage uploads (dev/test) with multipart upload capabilities. + +## Multipart Upload Workflow + +### 1. Initiate Upload +**POST** `/v1/files/uploads/` + +```json +{ + "file_name": "large-video.mp4", + "file_size": 104857600, + "file_content_type": "video/mp4", + "num_parts": 20, + "part_size": 5242880, + "parent_id": "b3c5…-document-public-id" // optional (DocumentId public_id) +} +``` + +**Response:** +```json +{ + "file_obj": { + "id": "7", + "file_name": "large-video.mp4", + "upload_status": "uploading" + }, + "upload_id": "abc-123-def-456", + "presigned_urls": [ + { + "part_number": 1, + "url": "https://s3.amazonaws.com/...", // or local endpoint + "method": "PUT" + }, + ... + ], + "expires_in": 3600 +} +``` + +### 2. Upload Parts + +#### For S3 (Production) +Upload directly to the presigned URLs: +```bash +PUT https://s3.amazonaws.com/bucket/path?uploadId=...&partNumber=1 +Content-Type: application/octet-stream +Body: +``` + +#### For Local Storage (Dev/Test) +**PUT** `/v1/files/presigned-uploads/{file_id}/upload-part/{part_number}/?token=` + +The presigned URLs returned from the initiate endpoint include a signed token for authentication. + +```bash +PUT /v1/files/presigned-uploads/7/upload-part/1/?token= +Content-Type: application/octet-stream +Body: +``` + +**Response:** +```json +{ + "part_number": 1, + "etag": "abc123def456..." +} +``` + +**Response Headers:** +``` +ETag: abc123def456... +``` + +**Note:** The token is minted and verified by `PresignedUploadToken` (`baseapp/files/tokens.py`) using Django signing, and is valid for `FILE_UPLOAD_PRESIGNED_URL_EXPIRATION` seconds (1 hour by default). No session authentication or CSRF protection is required for this endpoint. The ETag is returned both in the response body and as an HTTP header for S3 compatibility. + +### 3. Complete Upload +**POST** `/v1/files/uploads/{id}/complete/` + +```json +{ + "parts": [ + {"part_number": 1, "etag": "abc123..."}, + {"part_number": 2, "etag": "def456..."}, + ... + ] +} +``` + +**Response:** +```json +{ + "id": "7", + "file_name": "large-video.mp4", + "file_size": 104857600, + "file_content_type": "video/mp4", + "upload_status": "completed", + "url": "https://cdn.example.com/files/...", + "created": "2025-01-01T12:00:00Z" +} +``` + +### 4. Abort Upload (Optional) +**DELETE** `/v1/files/uploads/{id}/` + +Cancels the upload and cleans up temporary files. + +## Storage Handlers + +### S3 (Production) +- Generates real presigned URLs pointing to S3 +- Client uploads directly to S3 +- No backend bandwidth used for file data +- Supports true multipart uploads + +### Local (Dev/Test) +- Generates "presigned URLs" with signed tokens pointing to backend endpoint +- Client uploads to backend via `/presigned-uploads/{id}/upload-part/{part_number}/` endpoint +- Token authentication using Django signing (no session auth or CSRF required) +- Backend stores parts in temp directory +- On complete, parts are concatenated into final file + +## Endpoints Summary + +| Method | Endpoint | Description | +|--------|----------|-------------| +| POST | `/v1/files/uploads/` | Initiate multipart upload | +| PUT | `/v1/files/presigned-uploads/{id}/upload-part/{part_number}/?token=...` | Upload a single part (local storage, token-authenticated) | +| POST | `/v1/files/uploads/{id}/complete/` | Complete the upload | +| DELETE | `/v1/files/uploads/{id}/` | Abort the upload | +| GET | `/v1/files/` | List files | +| GET | `/v1/files/{id}/` | Get file details | +| PATCH | `/v1/files/{id}/` | Update file metadata | +| DELETE | `/v1/files/{id}/` | Delete file | + +## Configuration + +### Settings +```python +# Maximum file size (default: 5GB) +MAX_FILE_UPLOAD_SIZE = 5 * 1024 * 1024 * 1024 +``` + +The storage handler is selected automatically from Django's `default_storage` +(`baseapp/files/storage/__init__.py`): an `S3Boto3Storage` backend selects the +S3 multipart handler, anything else falls back to the local handler. There is +no dedicated setting. + +### Local Storage (default_storage is not S3) +- Uses `LocalUploadHandler` +- Stores files in `MEDIA_ROOT/files/` +- Temp parts in `MEDIA_ROOT/temp_uploads/{upload_id}/` + +### S3 Storage (default_storage is S3Boto3Storage) +```python +AWS_ACCESS_KEY_ID = "..." +AWS_SECRET_ACCESS_KEY = "..." +AWS_STORAGE_BUCKET_NAME = "my-bucket" +AWS_S3_REGION_NAME = "us-east-1" +``` +- Uses `S3MultipartUploadHandler` +- Generates presigned URLs for direct S3 upload +- No backend file handling + +## Error Handling + +### Common Errors +- **400 Bad Request**: Invalid parameters, part numbers, or file size +- **403 Forbidden**: User doesn't own the file/upload +- **404 Not Found**: File or upload not found +- **413 Payload Too Large**: File exceeds `MAX_FILE_UPLOAD_SIZE` + +### Upload Status +- `pending`: Upload initiated but not started +- `uploading`: Parts being uploaded +- `completed`: Upload finished successfully +- `failed`: Upload failed (error during completion) +- `aborted`: Upload cancelled by user + +## Best Practices + +1. **Chunk Size**: Use 5MB minimum for parts (S3 requirement) +2. **Part Count**: Keep between 1-10,000 parts +3. **Error Handling**: Always store ETags from part uploads +4. **Cleanup**: Call abort endpoint if upload fails +5. **Expiration**: Complete uploads within 24 hours (tokens expire in 1 hour) +6. **Local Dev**: Use the presigned URLs with tokens returned from the initiate endpoint diff --git a/baseapp/files/__init__.py b/baseapp/files/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/baseapp/files/admin.py b/baseapp/files/admin.py new file mode 100644 index 00000000..dce2dc7d --- /dev/null +++ b/baseapp/files/admin.py @@ -0,0 +1,26 @@ +import swapper +from django.contrib import admin + +from baseapp_core.admin_helpers import ModelAdmin + +File = swapper.load_model("baseapp_files", "File") +FileTarget = swapper.load_model("baseapp_files", "FileTarget") + + +@admin.register(File) +class FileAdmin(ModelAdmin): + list_display = ("pk", "name", "file_content_type", "parent", "created_by", "created") + search_fields = ("name",) + raw_id_fields = ("created_by",) + + +@admin.register(FileTarget) +class FileTargetAdmin(ModelAdmin): + list_display = ( + "pk", + "target", + "is_files_enabled", + "files_count", + ) + list_filter = ("is_files_enabled",) + search_fields = ("target_id",) diff --git a/baseapp/files/apps.py b/baseapp/files/apps.py new file mode 100644 index 00000000..d5f10a75 --- /dev/null +++ b/baseapp/files/apps.py @@ -0,0 +1,19 @@ +from baseapp_core.plugins import BaseAppConfig, GraphQLContributor, ServicesContributor + + +class FilesConfig(BaseAppConfig, ServicesContributor, GraphQLContributor): + default = True + name = "baseapp.files" + label = "baseapp_files" + verbose_name = "BaseApp Files" + default_auto_field = "django.db.models.BigAutoField" + + def register_shared_services(self, registry): + from .services.metadata import FilesMetadataService + + registry.register(FilesMetadataService()) + + def register_graphql_shared_interfaces(self, registry): + from .graphql.interfaces import get_files_interface + + registry.register("FilesInterface", get_files_interface) diff --git a/baseapp/files/base.py b/baseapp/files/base.py new file mode 100644 index 00000000..d973d3f5 --- /dev/null +++ b/baseapp/files/base.py @@ -0,0 +1,31 @@ +import swapper +from django.contrib.contenttypes.models import ContentType +from django.db import models + +from .utils import get_or_create_file_target + + +class FileableModel(models.Model): + class Meta: + abstract = True + + @property + def files(self) -> models.QuerySet: + """Returns files related to this object through the DocumentId join.""" + File = swapper.load_model("baseapp_files", "File") + content_type = ContentType.objects.get_for_model(self) + return File.objects.filter( + parent__content_type=content_type, + parent__object_id=self.pk, + ) + + def get_file_target(self): + return get_or_create_file_target(self) + + @property + def files_count(self) -> dict: + return self.get_file_target().files_count + + @property + def is_files_enabled(self) -> bool: + return self.get_file_target().is_files_enabled diff --git a/baseapp/files/graphql/__init__.py b/baseapp/files/graphql/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/baseapp/files/graphql/interfaces.py b/baseapp/files/graphql/interfaces.py new file mode 100644 index 00000000..5cee98a9 --- /dev/null +++ b/baseapp/files/graphql/interfaces.py @@ -0,0 +1,49 @@ +import swapper +from django.contrib.contenttypes.models import ContentType +from graphene import Boolean, Interface +from graphene.types.generic import GenericScalar +from graphene_django.filter import DjangoFilterConnectionField + +from baseapp_core.plugins import shared_services + +from ..utils import default_files_count + + +class FilesInterface(Interface): + files_count = GenericScalar() + is_files_enabled = Boolean() + files = DjangoFilterConnectionField( + lambda: swapper.load_model("baseapp_files", "File").get_graphql_object_type() + ) + + def resolve_files_count(self, info, **kwargs) -> dict: + if service := shared_services.get("files_metadata"): + return service.get_files_count(self) + return default_files_count() + + def resolve_is_files_enabled(self, info, **kwargs) -> bool: + if service := shared_services.get("files_metadata"): + return service.is_files_enabled(self) + return True + + def resolve_files(self, info, **kwargs): + File = swapper.load_model("baseapp_files", "File") + + service = shared_services.get("files_metadata") + if service is not None and not service.is_files_enabled(self): + return File.objects.none() + + # Filter through the DocumentId join instead of fetching the DocumentId + # row per object — keeps the connection a single query per list. + content_type = ContentType.objects.get_for_model(self) + # Total ordering (-created, -pk): AbstractFile has no Meta.ordering, and + # cursor pagination over a non-total order can duplicate/drop rows when + # timestamps collide. Matches FilesQueries.resolve_my_files. + return File.objects.filter( + parent__content_type=content_type, + parent__object_id=self.pk, + ).order_by("-created", "-pk") + + +def get_files_interface(): + return FilesInterface diff --git a/baseapp/files/graphql/mutations.py b/baseapp/files/graphql/mutations.py new file mode 100644 index 00000000..8039216c --- /dev/null +++ b/baseapp/files/graphql/mutations.py @@ -0,0 +1,181 @@ +import graphene +import swapper +from django.db import transaction +from django.utils.translation import gettext_lazy as _ +from graphql.error import GraphQLError +from graphql_relay.connection.arrayconnection import offset_to_cursor + +from baseapp_core.graphql import RelayMutation, get_obj_from_relay_id, login_required +from baseapp_core.models import DocumentId +from baseapp_core.plugins import shared_services + +from ..base import FileableModel +from .interfaces import FilesInterface + +File = swapper.load_model("baseapp_files", "File") +app_label = File._meta.app_label +file_model_name = File._meta.model_name +FileObjectType = File.get_graphql_object_type() + + +class FileAttachToTarget(RelayMutation): + """ + Attach one or more files to a target object. + + Usage: + mutation { + fileAttachToTarget(input: { + fileRelayIds: ["RmlsZU9iamVjdFR5cGU6MQ==", "RmlsZU9iamVjdFR5cGU6Mg=="] + targetObjectId: "Q29tbWVudE9iamVjdFR5cGU6MQ==" + }) { + attachedFiles { + node { + id + fileName + } + } + target { + ... on CommentsInterface { + filesCount + isFilesEnabled + } + } + } + } + """ + + attached_files = graphene.List( + graphene.NonNull(FileObjectType._meta.connection.Edge), + required=True, + ) + target = graphene.Field(FilesInterface, required=True) + + class Input: + file_relay_ids = graphene.List(graphene.ID, required=True) + target_object_id = graphene.ID(required=True) + + @classmethod + @login_required + @transaction.atomic + def mutate_and_get_payload(cls, root, info, **input): + file_relay_ids = input.get("file_relay_ids", []) + target_object_id = input.get("target_object_id") + + if not file_relay_ids: + raise GraphQLError( + str(_("At least one file must be provided")), + extensions={"code": "invalid_input"}, + ) + + # Get the target object + target = get_obj_from_relay_id(info, target_object_id) + if not target: + raise GraphQLError( + str(_("Target object not found")), + extensions={"code": "not_found"}, + ) + + # Only objects that opt into files (FileableModel) can be targets — this + # also guarantees the payload's `target` field resolves as FilesInterface. + if not isinstance(target, FileableModel): + raise GraphQLError( + str(_("This object does not support file attachments")), + extensions={"code": "invalid_target"}, + ) + + # Target-level authorization: whether this user may attach files to this + # object is delegated to the permission backend so projects can override + # it (default: authenticated + files enabled). Keep the explicit enabled + # check for a precise error code. + if service := shared_services.get("files_metadata"): + if not service.is_files_enabled(target): + raise GraphQLError( + str(_("Files are not enabled for this target")), + extensions={"code": "files_disabled"}, + ) + if not info.context.user.has_perm(f"{app_label}.add_{file_model_name}", target): + raise GraphQLError( + str(_("You don't have permission to attach files to this object")), + extensions={"code": "permission_required"}, + ) + + # Get all files and verify each is a File the user is allowed to attach. + files = [] + for file_relay_id in file_relay_ids: + file_obj = get_obj_from_relay_id(info, file_relay_id) + if not isinstance(file_obj, File): + raise GraphQLError( + str(_("File not found: {file_id}")).format(file_id=file_relay_id), + extensions={"code": "not_found"}, + ) + + # Object-level permission (FilesPermissionsBackend grants the owner) + if not info.context.user.has_perm(f"{app_label}.attach_{file_model_name}", file_obj): + raise GraphQLError( + str(_("You don't have permission to attach this file")), + extensions={"code": "permission_required"}, + ) + + # Check if file is already attached to another parent + if file_obj.parent_id: + raise GraphQLError( + str(_("File {file_name} is already attached to another object")).format( + file_name=file_obj.file_name + ), + extensions={"code": "already_attached"}, + ) + + files.append(file_obj) + + # All validation passed — now materialize the target DocumentId and attach. + target_document_id = DocumentId.get_or_create_for_object(target) + for file_obj in files: + file_obj.parent = target_document_id + file_obj.save(update_fields=["parent"]) + + # Refresh target to get updated files_count + target.refresh_from_db() + + # Create file edges for response + attached_files = [ + FileObjectType._meta.connection.Edge(node=file_obj, cursor=offset_to_cursor(i)) + for i, file_obj in enumerate(files) + ] + + return FileAttachToTarget(attached_files=attached_files, target=target) + + +class FileDelete(RelayMutation): + deleted_id = graphene.ID() + parent = graphene.Field(FilesInterface) + + class Input: + id = graphene.ID(required=True) + + @classmethod + @login_required + def mutate_and_get_payload(cls, root, info, **input): + relay_id = input.get("id") + + error_exception = GraphQLError( + str(_("You don't have permission to perform this action")), + extensions={"code": "permission_required"}, + ) + + obj = get_obj_from_relay_id(info, relay_id) + if not obj: + raise error_exception + + if not info.context.user.has_perm(f"{app_label}.delete_{file_model_name}", obj): + raise error_exception + + parent = obj.parent.content_object if obj.parent else None + + obj.delete() + + return FileDelete(deleted_id=relay_id, parent=parent) + + +class FilesMutations(object): + file_attach_to_target = FileAttachToTarget.Field() + file_delete = FileDelete.Field() diff --git a/baseapp/files/graphql/object_types.py b/baseapp/files/graphql/object_types.py new file mode 100644 index 00000000..dbeece7c --- /dev/null +++ b/baseapp/files/graphql/object_types.py @@ -0,0 +1,50 @@ +import django_filters +import graphene +import swapper + +from baseapp_auth.graphql.permissions import PermissionsInterface +from baseapp_core.graphql import DjangoObjectType, FileInterface +from baseapp_core.graphql import Node as RelayNode +from baseapp_core.graphql import ThumbnailField +from baseapp_core.plugins import graphql_shared_interfaces + +File = swapper.load_model("baseapp_files", "File") + + +class FileFilter(django_filters.FilterSet): + no_parent = django_filters.BooleanFilter(field_name="parent", lookup_expr="isnull") + + class Meta: + model = File + fields = ["no_parent"] + + +class AbstractFileObjectType: + parent = graphene.Field(RelayNode) + thumbnail = ThumbnailField() + + class Meta: + interfaces = graphql_shared_interfaces.get( + RelayNode, + PermissionsInterface, + FileInterface, + "CommentsInterface", + "ReactionsInterface", + ) + model = File + filterset_class = FileFilter + + def resolve_url(self, info, **kwargs): + if not self.file: + if hasattr(self, "url"): + return self.url + return None + return info.context.build_absolute_uri(self.file.url) + + def resolve_thumbnail(self, *args, **kwargs): + return self.file + + +class FileObjectType(AbstractFileObjectType, DjangoObjectType): + class Meta(AbstractFileObjectType.Meta): + pass diff --git a/baseapp/files/graphql/queries.py b/baseapp/files/graphql/queries.py new file mode 100644 index 00000000..36647cae --- /dev/null +++ b/baseapp/files/graphql/queries.py @@ -0,0 +1,23 @@ +import graphene +import swapper +from django.db import models +from graphene_django.filter import DjangoFilterConnectionField + +from baseapp_core.graphql import Node + +File = swapper.load_model("baseapp_files", "File") +FileObjectType = File.get_graphql_object_type() + + +class FilesQueries: + my_files = DjangoFilterConnectionField( + FileObjectType, description="Files created by the current user." + ) + file = Node.Field(FileObjectType) + + def resolve_my_files(self, info: graphene.ResolveInfo, **kwargs) -> models.QuerySet: + if not info.context.user.is_authenticated: + return File.objects.none() + # Explicit total ordering — AbstractFile has no Meta.ordering, and cursor + # pagination over an unordered queryset yields duplicated/missing rows. + return File.objects.filter(created_by=info.context.user).order_by("-created", "-pk") diff --git a/baseapp/files/graphql/utils.py b/baseapp/files/graphql/utils.py new file mode 100644 index 00000000..89231777 --- /dev/null +++ b/baseapp/files/graphql/utils.py @@ -0,0 +1,47 @@ +import swapper +from django.utils.translation import gettext_lazy as _ +from graphql.error import GraphQLError + +from baseapp_core.graphql import get_pk_from_relay_id + +from ..utils import set_files_parent + + +def attach_files_from_relay_ids(parent, file_ids, user): + if file_ids in (None, []): + return + + File = swapper.load_model("baseapp_files", "File") + not_found_error = GraphQLError( + str(_("One or more files could not be found")), + extensions={"code": "not_found"}, + ) + try: + file_pks = [get_pk_from_relay_id(file_id) for file_id in file_ids] + except Exception: + # Unresolvable relay ids (e.g. the DocumentId row was deleted) must + # surface as the same not_found error as a missing File row. + raise not_found_error from None + pk_lookup = [str(pk) for pk in file_pks] + files_qs = File.objects.filter(pk__in=file_pks) + files_map = {str(file.pk): file for file in files_qs} + + missing = [pk for pk in pk_lookup if pk not in files_map] + if missing: + raise not_found_error + + files = [files_map[pk] for pk in pk_lookup] + + if user and not user.is_superuser: + # Only the file's own creator may attach it. A NULL creator is not an + # implicit grant — treating it as one would let any user re-parent + # (steal) system/orphan files, diverging from FileAttachToTarget which + # enforces owner-only via the attach permission. + unauthorized = [file_obj for file_obj in files if file_obj.created_by_id != user.id] + if unauthorized: + raise GraphQLError( + str(_("You don't have permission to perform this action")), + extensions={"code": "permission_required"}, + ) + + set_files_parent(parent, files) diff --git a/baseapp/files/models.py b/baseapp/files/models.py new file mode 100644 index 00000000..9e8cfc95 --- /dev/null +++ b/baseapp/files/models.py @@ -0,0 +1,429 @@ +import pgtrigger +import swapper +from django.apps import apps +from django.conf import settings +from django.contrib.contenttypes.models import ContentType +from django.core.exceptions import ValidationError +from django.db import models +from django.db.models import OuterRef, Subquery, Value +from django.db.models.functions import Coalesce +from django.utils.translation import gettext_lazy as _ +from model_utils.models import TimeStampedModel + +from baseapp_core.graphql import RelayModel +from baseapp_core.models import ( + DocumentId, + DocumentIdMixin, + DocumentIdUniqueTargetMixin, + random_name_in, +) +from baseapp_core.pghelpers import pgtrigger_register_default_track +from baseapp_core.swapper import init_swapped_models + +from .utils import default_files_count + + +class FileTargetCountFunc(pgtrigger.Func): + """ + Reusable pgtrigger function for updating FileTarget counts. + Dynamically resolves table names for swappable models. + """ + + def render( + self, + meta=None, + fields=None, + columns=None, + **kwargs, + ) -> str: + FileTarget = swapper.load_model("baseapp_files", "FileTarget") + return self.func.format( + file_table=meta.db_table, + file_target_table=FileTarget._meta.db_table, + ) + + +def file_target_insert_trigger(): + """Trigger to update FileTarget counts when a file is inserted.""" + return pgtrigger.Trigger( + name="update_file_target_on_insert", + level=pgtrigger.Row, + when=pgtrigger.After, + operation=pgtrigger.Insert, + func=FileTargetCountFunc(""" + DECLARE + new_counts JSONB; + BEGIN + IF NEW.parent_id IS NOT NULL THEN + -- Calculate new counts + SELECT COALESCE( + jsonb_object_agg( + file_content_type, + count + ) || jsonb_build_object('total', SUM(count)), + '{{"total": 0}}'::jsonb + ) INTO new_counts + FROM ( + SELECT + COALESCE(file_content_type, 'unknown') as file_content_type, + COUNT(*)::int as count + FROM {file_table} + WHERE parent_id = NEW.parent_id + GROUP BY file_content_type + ) counts; + + -- Insert or update FileTarget + INSERT INTO {file_target_table} (target_id, files_count, is_files_enabled) + VALUES (NEW.parent_id, new_counts, true) + ON CONFLICT (target_id) + DO UPDATE SET files_count = new_counts; + END IF; + RETURN NEW; + END; + """), + ) + + +def file_target_update_trigger(): + """Trigger to update FileTarget counts when a file's parent is updated.""" + return pgtrigger.Trigger( + name="update_file_target_on_update", + level=pgtrigger.Row, + when=pgtrigger.After, + operation=pgtrigger.Update, + func=FileTargetCountFunc(""" + DECLARE + new_counts JSONB; + old_counts JSONB; + BEGIN + -- Update old parent if it changed + IF OLD.parent_id IS DISTINCT FROM NEW.parent_id + AND OLD.parent_id IS NOT NULL THEN + + SELECT COALESCE( + jsonb_object_agg( + file_content_type, + count + ) || jsonb_build_object('total', SUM(count)), + '{{"total": 0}}'::jsonb + ) INTO old_counts + FROM ( + SELECT + COALESCE(file_content_type, 'unknown') as file_content_type, + COUNT(*)::int as count + FROM {file_table} + WHERE parent_id = OLD.parent_id + GROUP BY file_content_type + ) counts; + + UPDATE {file_target_table} SET files_count = old_counts + WHERE target_id = OLD.parent_id; + END IF; + + -- Update new parent + IF NEW.parent_id IS NOT NULL THEN + SELECT COALESCE( + jsonb_object_agg( + file_content_type, + count + ) || jsonb_build_object('total', SUM(count)), + '{{"total": 0}}'::jsonb + ) INTO new_counts + FROM ( + SELECT + COALESCE(file_content_type, 'unknown') as file_content_type, + COUNT(*)::int as count + FROM {file_table} + WHERE parent_id = NEW.parent_id + GROUP BY file_content_type + ) counts; + + INSERT INTO {file_target_table} (target_id, files_count, is_files_enabled) + VALUES (NEW.parent_id, new_counts, true) + ON CONFLICT (target_id) + DO UPDATE SET files_count = new_counts; + END IF; + + RETURN NEW; + END; + """), + ) + + +def file_target_delete_trigger(): + """Trigger to update FileTarget counts when a file is deleted.""" + return pgtrigger.Trigger( + name="update_file_target_on_delete", + level=pgtrigger.Row, + when=pgtrigger.After, + operation=pgtrigger.Delete, + func=FileTargetCountFunc(""" + DECLARE + new_counts JSONB; + BEGIN + IF OLD.parent_id IS NOT NULL THEN + SELECT COALESCE( + jsonb_object_agg( + file_content_type, + count + ) || jsonb_build_object('total', SUM(count)), + '{{"total": 0}}'::jsonb + ) INTO new_counts + FROM ( + SELECT + COALESCE(file_content_type, 'unknown') as file_content_type, + COUNT(*)::int as count + FROM {file_table} + WHERE parent_id = OLD.parent_id + GROUP BY file_content_type + ) counts; + + UPDATE {file_target_table} SET files_count = new_counts + WHERE target_id = OLD.parent_id; + END IF; + + RETURN OLD; + END; + """), + ) + + +class AbstractFileTarget(DocumentIdUniqueTargetMixin): + # Override only to keep the existing related_name/help_text (avoids a no-op + # AlterField migration on the concrete swappable models); get_for_object and + # get_or_create_for_object are inherited from the mixin. + target = models.OneToOneField( + DocumentId, + on_delete=models.CASCADE, + primary_key=True, + related_name="file_target", + verbose_name=_("target"), + help_text=_("The document this file target belongs to"), + ) + + files_count = models.JSONField( + default=default_files_count, + verbose_name=_("files count"), + help_text=_("Per-content-type file counts for the target"), + ) + is_files_enabled = models.BooleanField( + default=True, + verbose_name=_("is files enabled"), + help_text=_("Whether files can be attached to the target"), + ) + + class Meta: + abstract = True + verbose_name = _("file target") + verbose_name_plural = _("file targets") + swappable = swapper.swappable_setting("baseapp_files", "FileTarget") + + def __str__(self): + return f"FileTarget for {self.target}" + + @classmethod + def annotate_queryset(cls, queryset: models.QuerySet) -> models.QuerySet: + """ + Annotate `queryset` with file-target metadata so the FilesInterface + resolvers (`files_count`, `is_files_enabled`) don't N+1. Reads the + annotated values via `FilesMetadataService`. Mirrors + `ReactableMetadata.annotate_queryset`, minus a flat count expression — + no consumer filters/orders files by count, so that extra subquery is + omitted; add it here (as reactions does with `reactions_count_total`) + if a FileFilter ever needs it. + """ + model_cls = queryset.model + ct_id = ContentType.objects.get_for_model(model_cls).pk + file_target_qs = cls.objects.filter( + target__content_type_id=ct_id, + target__object_id=OuterRef("pk"), + ) + return queryset.annotate( + _file_target_files_count=Subquery(file_target_qs.values("files_count")[:1]), + _file_target_is_files_enabled=Coalesce( + Subquery( + file_target_qs.values("is_files_enabled")[:1], + output_field=models.BooleanField(), + ), + Value(True), + ), + ) + + +file_inheritances = [] + +if apps.is_installed("baseapp_profiles"): + + class ProfileMixin(models.Model): + profile = models.ForeignKey( + swapper.get_model_name("baseapp_profiles", "Profile"), + verbose_name=_("profile"), + related_name="files", + on_delete=models.SET_NULL, + null=True, + blank=True, + ) + + class Meta: + abstract = True + + file_inheritances.append(ProfileMixin) + + +class AbstractFile(*file_inheritances, DocumentIdMixin, RelayModel, TimeStampedModel): + parent = models.ForeignKey( + DocumentId, + null=True, + blank=True, + on_delete=models.CASCADE, + related_name="files", + verbose_name=_("parent"), + help_text=_("The parent document this file belongs to"), + ) + + file_content_type = models.CharField( + max_length=150, + blank=True, + default="", + verbose_name=_("content type"), + help_text=_("MIME type of the uploaded file"), + ) + file_name = models.CharField( + max_length=512, + blank=True, + default="", + verbose_name=_("file name"), + help_text=_("Original name of the uploaded file"), + ) + file_size = models.PositiveIntegerField( + null=True, + verbose_name=_("file size"), + help_text=_("File size in bytes"), + ) + file = models.FileField( + max_length=512, + upload_to=random_name_in("files"), + verbose_name=_("file"), + help_text=_("The stored file"), + ) + + name = models.CharField( + max_length=512, + blank=True, + default="", + verbose_name=_("name"), + help_text=_("Display name for the file"), + ) + description = models.TextField( + blank=True, + default="", + verbose_name=_("description"), + help_text=_("Optional description of the file"), + ) + + created_by = models.ForeignKey( + settings.AUTH_USER_MODEL, + related_name="files_created", + on_delete=models.CASCADE, + null=True, + verbose_name=_("created by"), + help_text=_("The user who uploaded the file"), + ) + + # Upload status tracking + class UploadStatus(models.TextChoices): + PENDING = "pending", _("Pending Upload") + UPLOADING = "uploading", _("Upload in Progress") + COMPLETED = "completed", _("Upload Completed") + FAILED = "failed", _("Upload Failed") + ABORTED = "aborted", _("Upload Aborted") + + upload_status = models.CharField( + max_length=20, + choices=UploadStatus.choices, + default=UploadStatus.COMPLETED, + db_index=True, + verbose_name=_("upload status"), + help_text=_("Status of the file upload"), + ) + + # S3 multipart upload tracking ("" when there is no active multipart upload) + upload_id = models.CharField( + max_length=512, + blank=True, + default="", + verbose_name=_("upload ID"), + help_text=_("S3 multipart upload ID for in-progress uploads"), + ) + + # Metadata for multipart uploads + total_parts = models.PositiveIntegerField( + null=True, + blank=True, + verbose_name=_("total parts"), + help_text=_("Total number of parts for multipart upload"), + ) + + uploaded_parts = models.JSONField( + null=True, + blank=True, + default=dict, + verbose_name=_("uploaded parts"), + help_text=_("Tracking uploaded parts with ETags"), + ) + + # Upload expiration for cleanup + upload_expires_at = models.DateTimeField( + null=True, + blank=True, + db_index=True, + verbose_name=_("upload expires at"), + help_text=_("Expiration time for pending uploads"), + ) + + def save(self, *args, **kwargs): + """ + Validate that files are enabled for the parent object before saving. + """ + if self.parent_id: + FileTarget = swapper.load_model("baseapp_files", "FileTarget") + try: + file_target = FileTarget.objects.get(target_id=self.parent_id) + if not file_target.is_files_enabled: + raise ValidationError( + _("Files are not enabled for this object."), + code="files_disabled", + ) + except FileTarget.DoesNotExist: + # If FileTarget doesn't exist, it will be created by the trigger + # and is_files_enabled defaults to True, so we allow the save + pass + + super().save(*args, **kwargs) + + class Meta: + abstract = True + verbose_name = _("file") + verbose_name_plural = _("files") + swappable = swapper.swappable_setting("baseapp_files", "File") + + @classmethod + def get_graphql_object_type(cls): + from .graphql.object_types import FileObjectType + + return FileObjectType + + +# Resolve the concrete swap target (e.g. files.File) so the count triggers are +# attached to the real table. Consumers can override these via +# `@pgtrigger_register_track(...)` on their concrete model. +File = init_swapped_models([("baseapp_files", "File")]) + +pgtrigger_register_default_track( + File, + [ + file_target_insert_trigger(), + file_target_update_trigger(), + file_target_delete_trigger(), + ], +) diff --git a/baseapp/files/permissions.py b/baseapp/files/permissions.py new file mode 100644 index 00000000..fa7a6b27 --- /dev/null +++ b/baseapp/files/permissions.py @@ -0,0 +1,57 @@ +from typing import Any, Optional + +import swapper +from django.contrib.auth.backends import BaseBackend +from django.contrib.auth.models import AbstractBaseUser, AnonymousUser + +File = swapper.load_model("baseapp_files", "File") +file_app_label = File._meta.app_label +file_model_name = File._meta.model_name.lower() + +UserType = AbstractBaseUser | AnonymousUser + + +class FilesPermissionsBackend(BaseBackend): + def authenticate(self, request, **kwargs) -> None: + return None + + @staticmethod + def _can_add_to_target(user_obj: UserType, obj: Optional[Any]) -> bool: + """Target-level: may this user attach files to `obj`? Default policy + mirrors comments' add_comment — authenticated and files enabled. + Projects can override this backend to enforce target ownership.""" + if not obj: + return False + from baseapp_core.plugins import shared_services + + service = shared_services.get("files_metadata") + enabled = service.is_files_enabled(obj) if service else True + return user_obj.is_authenticated and enabled + + @staticmethod + def _is_owner_or_has_global(user_obj: UserType, perm: str, obj: Optional[Any]) -> bool: + """Owner acts on their own file; otherwise fall back to the global perm + (ModelBackend doesn't grant global perms when an obj is passed).""" + if not obj: + return False + return obj.created_by == user_obj or user_obj.has_perm(perm) + + def has_perm(self, user_obj: UserType, perm: str, obj: Optional[Any] = None) -> bool: + """Resolve object-level file permissions: ``add_`` (attach to a + target), ``attach_`` (owner-only), and view/change/delete + (owner or global perm). Unknown perms return False so other backends + can decide.""" + if perm == f"{file_app_label}.add_{file_model_name}": + return self._can_add_to_target(user_obj, obj) + + if perm == f"{file_app_label}.attach_{file_model_name}": + return bool(obj) and obj.created_by == user_obj + + if perm in ( + f"{file_app_label}.view_{file_model_name}", + f"{file_app_label}.change_{file_model_name}", + f"{file_app_label}.delete_{file_model_name}", + ): + return self._is_owner_or_has_global(user_obj, perm, obj) + + return False diff --git a/baseapp/files/plugin.py b/baseapp/files/plugin.py new file mode 100644 index 00000000..67d2ab4d --- /dev/null +++ b/baseapp/files/plugin.py @@ -0,0 +1,54 @@ +from baseapp_core.plugins.base import BaseAppPlugin, PackageSettings + + +class FilesPlugin(BaseAppPlugin): + """ + The Files plugin provides file uploads, file management, and the ability to attach + files to any object exposing a ``DocumentId`` (via ``FilesInterface``). + """ + + @property + def name(self) -> str: + return "baseapp_files" + + @property + def package_name(self) -> str: + return "baseapp.files" + + def get_settings(self) -> PackageSettings: + return PackageSettings( + INSTALLED_APPS=[], + AUTHENTICATION_BACKENDS={ + "baseapp_files": [ + "baseapp.files.permissions.FilesPermissionsBackend", + ], + }, + graphql_queries=[ + "baseapp.files.graphql.queries.FilesQueries", + ], + graphql_mutations=[ + "baseapp.files.graphql.mutations.FilesMutations", + ], + v1_urlpatterns=self.v1_urlpatterns, + required_packages=[ + {"baseapp_core": "Core shared models (DocumentId) and plugin infrastructure"}, + ], + optional_packages=[ + { + "baseapp_profiles": "If enabled, File exposes the authoring Profile via the `profile` field." + }, + { + "baseapp_comments": "If enabled, files expose comments through the CommentsInterface." + }, + { + "baseapp_reactions": "If enabled, files expose reactions through the ReactionsInterface." + }, + ], + ) + + def v1_urlpatterns(self, include, path, re_path): + from baseapp.files.rest_framework.routers import files_router + + return [ + re_path(r"", include(files_router.urls)), + ] diff --git a/baseapp/files/rest_framework/__init__.py b/baseapp/files/rest_framework/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/baseapp/files/rest_framework/files/__init__.py b/baseapp/files/rest_framework/files/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/baseapp/files/rest_framework/files/serializers.py b/baseapp/files/rest_framework/files/serializers.py new file mode 100644 index 00000000..cde12db2 --- /dev/null +++ b/baseapp/files/rest_framework/files/serializers.py @@ -0,0 +1,72 @@ +import swapper +from rest_framework import serializers + +from baseapp_core.rest_framework.serializers import ModelSerializer + +File = swapper.load_model("baseapp_files", "File") + + +class FileSerializer(ModelSerializer): + """Standard file serializer for CRUD operations.""" + + # Use public_id instead of primary key + id = serializers.SerializerMethodField() + url = serializers.SerializerMethodField() + relay_id = serializers.SerializerMethodField() + created_by_name = serializers.CharField(source="created_by.get_full_name", read_only=True) + + # Parent DocumentId public_id + parent_id = serializers.SerializerMethodField() + + class Meta: + model = File + fields = [ + "id", + "relay_id", + "file_name", + "file_size", + "file_content_type", + "name", + "description", + "url", + "upload_status", + "created_by", + "created_by_name", + "created", + "modified", + "parent_id", + ] + read_only_fields = [ + "id", + "relay_id", + "upload_status", + "created_by", + "created", + "modified", + "file_size", + "parent_id", + ] + + def get_id(self, obj): + """Get public_id instead of primary key.""" + return obj.public_id + + def get_url(self, obj): + """Get file URL if upload completed.""" + if obj.upload_status == "completed" and obj.file: + request = self.context.get("request") + url = obj.file.url + if request: + return request.build_absolute_uri(url) + return url + return None + + def get_relay_id(self, obj): + """Get GraphQL relay ID for the file.""" + return obj.relay_id + + def get_parent_id(self, obj): + """Get parent DocumentId public_id.""" + if obj.parent: + return obj.parent.public_id + return None diff --git a/baseapp/files/rest_framework/files/views.py b/baseapp/files/rest_framework/files/views.py new file mode 100644 index 00000000..aaf23fb8 --- /dev/null +++ b/baseapp/files/rest_framework/files/views.py @@ -0,0 +1,105 @@ +import swapper +from django.utils.translation import gettext_lazy as _ +from rest_framework import mixins, permissions, status, viewsets +from rest_framework.response import Response + +from baseapp_core.models import DocumentId +from baseapp_core.rest_framework.decorators import action +from baseapp_profiles.rest_framework import CurrentProfileMixin + +from ..uploads.permissions import IsOwnerOrReadOnly +from ..uploads.serializers import SetParentSerializer +from ..utils import enforce_can_attach_to_parent +from .serializers import FileSerializer + +File = swapper.load_model("baseapp_files", "File") + + +class FilesViewSet( + CurrentProfileMixin, + mixins.RetrieveModelMixin, + mixins.UpdateModelMixin, + mixins.DestroyModelMixin, + mixins.ListModelMixin, + viewsets.GenericViewSet, +): + """ + ViewSet for file CRUD operations. + + Endpoints: + GET /v1/files/ - List files + GET /v1/files/{public_id} - Retrieve file + PATCH /v1/files/{public_id} - Update file metadata + DELETE /v1/files/{public_id} - Delete file + POST /v1/files/{public_id}/set-parent - Set parent after upload + """ + + serializer_class = FileSerializer + permission_classes = [permissions.IsAuthenticated, IsOwnerOrReadOnly] + lookup_field = "pk" + lookup_url_kwarg = "pk" + + def get_object(self): + """Look up file by public_id instead of primary key.""" + from rest_framework.exceptions import NotFound + + lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field + public_id = self.kwargs[lookup_url_kwarg] + + obj = File.get_by_public_id(public_id) + if obj is None: + raise NotFound(_("File not found")) + + self.check_object_permissions(self.request, obj) + return obj + + def get_queryset(self): + """Filter files based on user and query params.""" + qs = File.objects.all() + + # Only show completed files by default. Use `is not None` so an empty + # `?status=` doesn't fall through and expose files in every state. + status_filter = self.request.query_params.get("status", File.UploadStatus.COMPLETED) + if status_filter is not None and status_filter != "": + qs = qs.filter(upload_status=status_filter) + + # Filter by parent using DocumentId public_id + parent_id = self.request.query_params.get("parent_id") + if parent_id: + try: + document_id = DocumentId.objects.get(public_id=parent_id) + qs = qs.filter(parent=document_id) + except DocumentId.DoesNotExist: + qs = qs.none() + + # Filter by owner (users can see their own files) + if not self.request.user.is_staff: + qs = qs.filter(created_by=self.request.user) + + return qs.select_related("created_by", "parent__content_type").order_by("-created") + + @action(detail=True, methods=["post"], serializer_class=SetParentSerializer) + def set_parent(self, request, pk=None): + """ + Set parent for a standalone file. + + POST /v1/files/{id}/set-parent + { + "parent_id": "uuid-of-document-id" + } + """ + file_obj = self.get_object() + + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + + parent_pk = serializer.validated_data["parent_id"] + # Enforce target-level authorization, mirroring fileAttachToTarget. + enforce_can_attach_to_parent(request.user, parent_pk) + + file_obj.parent_id = parent_pk + file_obj.save() + + return Response( + FileSerializer(file_obj, context={"request": request}).data, status=status.HTTP_200_OK + ) diff --git a/baseapp/files/rest_framework/routers.py b/baseapp/files/rest_framework/routers.py new file mode 100644 index 00000000..32011d90 --- /dev/null +++ b/baseapp/files/rest_framework/routers.py @@ -0,0 +1,13 @@ +from baseapp_core.rest_framework.routers import DefaultRouter + +from .files.views import FilesViewSet +from .uploads.presigned_views import PresignedUploadViewSet +from .uploads.views import FileUploadViewSet + +files_router = DefaultRouter(trailing_slash=False) + +files_router.register( + r"files/presigned-uploads", PresignedUploadViewSet, basename="presigned-uploads" +) +files_router.register(r"files/uploads", FileUploadViewSet, basename="file-uploads") +files_router.register(r"files", FilesViewSet, basename="files") diff --git a/baseapp/files/rest_framework/uploads/__init__.py b/baseapp/files/rest_framework/uploads/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/baseapp/files/rest_framework/uploads/permissions.py b/baseapp/files/rest_framework/uploads/permissions.py new file mode 100644 index 00000000..12e698d6 --- /dev/null +++ b/baseapp/files/rest_framework/uploads/permissions.py @@ -0,0 +1,33 @@ +import swapper +from rest_framework import permissions + +File = swapper.load_model("baseapp_files", "File") +file_app_label = File._meta.app_label +file_model_name = File._meta.model_name.lower() + + +class IsOwner(permissions.BasePermission): + """ + Permission to only allow owners to access their files. + Uses has_perm for easy override at project level. + """ + + def has_object_permission(self, request, view, obj): + return request.user.has_perm(f"{file_app_label}.view_{file_model_name}", obj) + + +class IsOwnerOrReadOnly(permissions.BasePermission): + """ + Permission to only allow owners of a file to modify it. + Read permissions check view_file permission. + Write permissions check change_file permission. + Uses has_perm for easy override at project level. + """ + + def has_object_permission(self, request, view, obj): + # Read permissions - check view permission + if request.method in permissions.SAFE_METHODS: + return request.user.has_perm(f"{file_app_label}.view_{file_model_name}", obj) + + # Write permissions - check change permission + return request.user.has_perm(f"{file_app_label}.change_{file_model_name}", obj) diff --git a/baseapp/files/rest_framework/uploads/presigned_views.py b/baseapp/files/rest_framework/uploads/presigned_views.py new file mode 100644 index 00000000..601f9b6f --- /dev/null +++ b/baseapp/files/rest_framework/uploads/presigned_views.py @@ -0,0 +1,144 @@ +import logging + +import swapper +from django.utils.translation import gettext_lazy as _ +from django.views.decorators.csrf import csrf_exempt +from rest_framework import status, viewsets +from rest_framework.decorators import action +from rest_framework.exceptions import ValidationError +from rest_framework.permissions import AllowAny +from rest_framework.response import Response + +from ...services.upload_service import UploadService +from ...tokens import PresignedUploadToken, PresignedUploadTokenError + +logger = logging.getLogger(__name__) + +File = swapper.load_model("baseapp_files", "File") + + +class PresignedUploadViewSet(viewsets.GenericViewSet): + """ + ViewSet for handling presigned upload URLs (local storage fallback). + + This ViewSet does NOT use session authentication or CSRF protection. + Instead, it validates signed tokens embedded in the URL. + + Endpoints: + PUT /v1/files/presigned-uploads/{file_id}/upload-part/{part_number}/?token=... + """ + + queryset = File.objects.all() + permission_classes = [AllowAny] # Authentication via signed token + authentication_classes = [] # No authentication classes + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.upload_service = UploadService() + + @csrf_exempt + def dispatch(self, request, *args, **kwargs): + """Override dispatch to disable CSRF protection.""" + return super().dispatch(request, *args, **kwargs) + + def perform_authentication(self, request): + """ + Skip DRF's authentication process entirely. + Authentication is handled via signed tokens in the URL. + """ + pass + + @action(detail=True, methods=["put"], url_path="upload-part/(?P[0-9]+)") + def upload_part(self, request, pk=None, part_number=None): + """ + Upload a single part using a presigned URL. + + PUT /v1/files/presigned-uploads/{file_id}/upload-part/{part_number}/?token= + + Body: raw binary data of the part + + Returns: + { + "part_number": 1, + "etag": "abc123..." + } + """ + try: + token_data = PresignedUploadToken.verify( + request.query_params.get("token"), + file_id=pk, + part_number=part_number, + ) + except PresignedUploadTokenError as exc: + return Response({"error": exc.message}, status=status.HTTP_401_UNAUTHORIZED) + + # Get the file object + try: + file_obj = File.objects.get(id=pk) + except File.DoesNotExist: + return Response( + {"error": _("File not found")}, + status=status.HTTP_404_NOT_FOUND, + ) + + try: + PresignedUploadToken.check_upload_session(token_data, file_obj.upload_id) + except PresignedUploadTokenError as exc: + return Response({"error": exc.message}, status=status.HTTP_401_UNAUTHORIZED) + + # Validate state + if file_obj.upload_status not in [File.UploadStatus.PENDING, File.UploadStatus.UPLOADING]: + return Response( + { + "error": _("Cannot upload parts in status: %(status)s") + % {"status": file_obj.upload_status} + }, + status=status.HTTP_400_BAD_REQUEST, + ) + + # Only the local handler stores parts through this endpoint; S3 uploads + # go straight to S3, so its handler has no upload_part. + if not hasattr(self.upload_service.handler, "upload_part"): + return Response( + {"error": _("The active storage backend does not support presigned part uploads")}, + status=status.HTTP_400_BAD_REQUEST, + ) + + try: + part_number = int(part_number) + + # Validate part number + if part_number < 1 or part_number > file_obj.total_parts: + return Response( + { + "error": _("Invalid part number. Must be between 1 and %(total)s") + % {"total": file_obj.total_parts} + }, + status=status.HTTP_400_BAD_REQUEST, + ) + + # Get the raw binary data from request + data = request.body + + # Upload the part using the storage handler + etag = self.upload_service.handler.upload_part( + upload_id=file_obj.upload_id, + part_number=part_number, + data=data, + ) + + # Return response with ETag header (S3-compatible) + response = Response( + {"part_number": part_number, "etag": etag}, + status=status.HTTP_200_OK, + ) + response["Access-Control-Expose-Headers"] = "*" + response["ETag"] = etag + return response + + except ValueError: + logger.warning("Upload part rejected", exc_info=True) + raise ValidationError([_("Invalid upload part request.")]) from None + except Exception: + logger.exception("Failed to upload part") + raise ValidationError([_("Failed to upload part.")]) from None diff --git a/baseapp/files/rest_framework/uploads/serializers.py b/baseapp/files/rest_framework/uploads/serializers.py new file mode 100644 index 00000000..f323b953 --- /dev/null +++ b/baseapp/files/rest_framework/uploads/serializers.py @@ -0,0 +1,106 @@ +import swapper +from django.utils.translation import gettext_lazy as _ +from rest_framework import serializers + +from baseapp_core.models import DocumentId + +File = swapper.load_model("baseapp_files", "File") + + +class InitiateUploadSerializer(serializers.Serializer): + """Serializer for initiating a multipart upload.""" + + file_name = serializers.CharField(max_length=512) + file_size = serializers.IntegerField(min_value=1) + file_content_type = serializers.CharField(max_length=150) + num_parts = serializers.IntegerField(min_value=1, max_value=10000) + part_size = serializers.IntegerField(min_value=1) + + # Optional parent relationship via DocumentId public_id + parent_id = serializers.UUIDField(required=False, allow_null=True) + + def validate_parent_id(self, value): + """Validate parent_id exists as a DocumentId public_id.""" + if not value: + return None + + try: + document_id = DocumentId.objects.get(public_id=value) + return document_id.pk + except DocumentId.DoesNotExist: + raise serializers.ValidationError( + [_("Invalid parent_id: DocumentId not found")] + ) from None + + def validate(self, data): + """Validate file size matches parts.""" + num_parts = data["num_parts"] + part_size = data["part_size"] + file_size = data["file_size"] + + min_size = (num_parts - 1) * part_size + max_size = num_parts * part_size + + if file_size < min_size or file_size > max_size: + raise serializers.ValidationError( + { + "non_field_errors": [ + _("File size %(size)s doesn't match %(parts)s parts of %(part_size)s bytes") + % {"size": file_size, "parts": num_parts, "part_size": part_size} + ] + } + ) + + return data + + +class UploadResponseSerializer(serializers.Serializer): + """Response serializer for initiated upload.""" + + id = serializers.UUIDField(source="file_obj.public_id") + relay_id = serializers.CharField(source="file_obj.relay_id", read_only=True) + upload_id = serializers.CharField() + presigned_urls = serializers.ListField() + expires_in = serializers.IntegerField() + upload_status = serializers.CharField(source="file_obj.upload_status") + + +class CompleteUploadSerializer(serializers.Serializer): + """Serializer for completing a multipart upload.""" + + parts = serializers.ListField( + child=serializers.DictField(), + min_length=1, + ) + + def validate_parts(self, value): + """Validate parts structure.""" + for part in value: + if "part_number" not in part or "etag" not in part: + raise serializers.ValidationError( + [_("Each part must have 'part_number' and 'etag'")] + ) + + if not isinstance(part["part_number"], int) or part["part_number"] < 1: + raise serializers.ValidationError([_("part_number must be a positive integer")]) + + if not isinstance(part["etag"], str) or not part["etag"]: + raise serializers.ValidationError([_("etag must be a non-empty string")]) + + return value + + +class SetParentSerializer(serializers.Serializer): + """Serializer for setting parent after upload.""" + + parent_id = serializers.UUIDField() + + def validate_parent_id(self, value): + """Validate parent_id exists as a DocumentId public_id.""" + try: + document_id = DocumentId.objects.get(public_id=value) + return document_id.pk + except DocumentId.DoesNotExist: + raise serializers.ValidationError( + [_("Invalid parent_id: DocumentId not found")] + ) from None diff --git a/baseapp/files/rest_framework/uploads/views.py b/baseapp/files/rest_framework/uploads/views.py new file mode 100644 index 00000000..45c33a42 --- /dev/null +++ b/baseapp/files/rest_framework/uploads/views.py @@ -0,0 +1,181 @@ +import logging + +import swapper +from django.utils.translation import gettext_lazy as _ +from rest_framework import permissions, status, viewsets +from rest_framework.exceptions import ValidationError +from rest_framework.response import Response + +from baseapp_core.rest_framework.decorators import action +from baseapp_profiles.rest_framework import CurrentProfileMixin + +from ...services.upload_service import UploadService +from ..utils import enforce_can_attach_to_parent +from .serializers import ( + CompleteUploadSerializer, + InitiateUploadSerializer, + UploadResponseSerializer, +) + +logger = logging.getLogger(__name__) + +File = swapper.load_model("baseapp_files", "File") +file_app_label = File._meta.app_label +file_model_name = File._meta.model_name.lower() + + +class FileUploadViewSet(CurrentProfileMixin, viewsets.GenericViewSet): + """ + ViewSet for managing multipart file uploads. + + Endpoints: + POST /v1/files/uploads/ - Initiate upload + POST /v1/files/uploads/{public_id}/complete - Complete upload + DELETE /v1/files/uploads/{public_id} - Abort upload + + Note: Part uploads for local storage use presigned URLs handled by PresignedUploadViewSet + """ + + queryset = File.objects.all() + permission_classes = [permissions.IsAuthenticated] + lookup_field = "pk" + lookup_url_kwarg = "pk" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.upload_service = UploadService() + + def get_object(self): + """Look up file by public_id instead of primary key.""" + lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field + public_id = self.kwargs[lookup_url_kwarg] + + obj = File.get_by_public_id(public_id) + if obj is None: + from rest_framework.exceptions import NotFound + + raise NotFound(_("File not found")) + + self.check_object_permissions(self.request, obj) + return obj + + def get_serializer_class(self): + if self.action == "create": + return InitiateUploadSerializer + elif self.action == "complete": + return CompleteUploadSerializer + return InitiateUploadSerializer + + def create(self, request): + """ + Initiate a multipart upload. + + POST /v1/files/uploads/ + { + "file_name": "large-video.mp4", + "file_size": 104857600, + "file_content_type": "video/mp4", + "num_parts": 20, + "part_size": 5242880, + "parent_id": "ab12" // optional + } + + Returns presigned URLs for each part. + """ + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + + # Authorize the target before creating anything, mirroring the + # fileAttachToTarget mutation so this endpoint can't bypass it. + parent_pk = serializer.validated_data.get("parent_id") + if parent_pk: + enforce_can_attach_to_parent(request.user, parent_pk) + + try: + # Get user's profile if available + profile = request.user.current_profile + + file_obj, upload_data = self.upload_service.initiate_multipart_upload( + user=request.user, profile=profile, **serializer.validated_data + ) + + # Prepare response + response_data = {"file_obj": file_obj, **upload_data} + + response_serializer = UploadResponseSerializer(response_data) + return Response(response_serializer.data, status=status.HTTP_201_CREATED) + + except ValueError: + logger.warning("Upload initiation rejected", exc_info=True) + raise ValidationError([_("Invalid upload parameters.")]) from None + except Exception: + logger.exception("Failed to initiate upload") + raise ValidationError([_("Failed to initiate upload.")]) from None + + @action(detail=True, methods=["post"]) + def complete(self, request, pk=None): + """ + Complete a multipart upload. + + POST /v1/files/uploads/{id}/complete + { + "parts": [ + {"part_number": 1, "etag": "abc123..."}, + {"part_number": 2, "etag": "def456..."}, + ... + ] + } + """ + file_obj = self.get_object() + + # Check permission (owner or has change_file permission) + if not request.user.has_perm(f"{file_app_label}.change_{file_model_name}", file_obj): + return Response( + {"error": _("You don't have permission to complete this upload")}, + status=status.HTTP_403_FORBIDDEN, + ) + + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + + try: + completed_file = self.upload_service.complete_multipart_upload( + file_id=file_obj.id, parts=serializer.validated_data["parts"] + ) + + from ..files.serializers import FileSerializer + + return Response( + FileSerializer(completed_file, context={"request": request}).data, + status=status.HTTP_200_OK, + ) + + except ValueError: + logger.warning("Upload completion rejected", exc_info=True) + raise ValidationError([_("Invalid or incomplete upload parts.")]) from None + except Exception: + logger.exception("Failed to complete upload") + raise ValidationError([_("Failed to complete upload.")]) from None + + def destroy(self, request, pk=None): + """ + Abort a multipart upload. + + DELETE /v1/files/uploads/{id} + """ + file_obj = self.get_object() + + # Check permission (owner or has delete_file permission) + if not request.user.has_perm(f"{file_app_label}.delete_{file_model_name}", file_obj): + return Response( + {"error": _("You don't have permission to abort this upload")}, + status=status.HTTP_403_FORBIDDEN, + ) + + try: + self.upload_service.abort_multipart_upload(file_obj.id) + return Response(status=status.HTTP_204_NO_CONTENT) + + except Exception: + logger.exception("Failed to abort upload") + raise ValidationError([_("Failed to abort upload.")]) from None diff --git a/baseapp/files/rest_framework/utils.py b/baseapp/files/rest_framework/utils.py new file mode 100644 index 00000000..bd8cc908 --- /dev/null +++ b/baseapp/files/rest_framework/utils.py @@ -0,0 +1,35 @@ +import swapper +from django.contrib.auth.models import AbstractBaseUser, AnonymousUser +from django.utils.translation import gettext_lazy as _ +from rest_framework.exceptions import PermissionDenied, ValidationError + +from baseapp_core.models import DocumentId + +from ..base import FileableModel + +File = swapper.load_model("baseapp_files", "File") +file_app_label = File._meta.app_label +file_model_name = File._meta.model_name.lower() + +UserType = AbstractBaseUser | AnonymousUser + + +def enforce_can_attach_to_parent(user: UserType, parent_document_pk: int) -> None: + """ + Authorize attaching a file to the object behind ``parent_document_pk`` (a + ``DocumentId`` pk). Mirrors the ``FileAttachToTarget`` GraphQL mutation so + the REST upload/set-parent paths cannot bypass target-level authorization: + the target must opt into files (``FileableModel``) and the user must hold + the ``add_`` permission on it. + """ + try: + document = DocumentId.objects.get(pk=parent_document_pk) + except DocumentId.DoesNotExist: + raise ValidationError([_("Invalid parent: target not found.")]) from None + + target = document.content_object + if not isinstance(target, FileableModel): + raise ValidationError([_("This object does not support file attachments.")]) + + if not user.has_perm(f"{file_app_label}.add_{file_model_name}", target): + raise PermissionDenied(_("You don't have permission to attach files to this object.")) diff --git a/baseapp/files/services/__init__.py b/baseapp/files/services/__init__.py new file mode 100644 index 00000000..ba060fc4 --- /dev/null +++ b/baseapp/files/services/__init__.py @@ -0,0 +1,10 @@ +from .cleanup import cleanup_expired_uploads, cleanup_failed_uploads +from .metadata import FilesMetadataService +from .upload_service import UploadService + +__all__ = [ + "FilesMetadataService", + "UploadService", + "cleanup_expired_uploads", + "cleanup_failed_uploads", +] diff --git a/baseapp/files/services/cleanup.py b/baseapp/files/services/cleanup.py new file mode 100644 index 00000000..4b85343a --- /dev/null +++ b/baseapp/files/services/cleanup.py @@ -0,0 +1,66 @@ +import logging +from datetime import timedelta + +import swapper +from celery import shared_task +from django.utils import timezone + +from ..storage import get_upload_handler + +logger = logging.getLogger(__name__) + + +@shared_task +def cleanup_expired_uploads(): + """ + Celery task to cleanup expired/abandoned uploads. + + Run this periodically (e.g., hourly via celery beat). + """ + # Loaded lazily (not at import time): this module is imported during Celery + # startup, which can precede Django's app registry being ready. + File = swapper.load_model("baseapp_files", "File") + handler = get_upload_handler() + now = timezone.now() + + # Find expired uploads + expired_files = File.objects.filter( + upload_status__in=[File.UploadStatus.PENDING, File.UploadStatus.UPLOADING], + upload_expires_at__lt=now, + ) + + cleaned_count = 0 + for file_obj in expired_files: + try: + if file_obj.upload_id: + handler.abort_upload(file_obj, file_obj.upload_id) + + file_obj.upload_status = File.UploadStatus.ABORTED + file_obj.upload_id = "" + file_obj.save() + + cleaned_count += 1 + except Exception: + # Log with traceback but continue with the next file. + logger.exception("Failed to cleanup file %s", file_obj.id) + + logger.info(f"Cleaned up {cleaned_count} expired uploads") + return f"Cleaned up {cleaned_count} expired uploads" + + +def cleanup_failed_uploads(days_old=7): + """ + Cleanup failed/aborted uploads older than specified days. + + Can be called manually or via management command. + """ + File = swapper.load_model("baseapp_files", "File") + cutoff = timezone.now() - timedelta(days=days_old) + + deleted_count, _ = File.objects.filter( + upload_status__in=[File.UploadStatus.FAILED, File.UploadStatus.ABORTED], + modified__lt=cutoff, + ).delete() + + logger.info(f"Deleted {deleted_count} old failed/aborted uploads") + return f"Deleted {deleted_count} old failed/aborted uploads" diff --git a/baseapp/files/services/metadata.py b/baseapp/files/services/metadata.py new file mode 100644 index 00000000..4e655c24 --- /dev/null +++ b/baseapp/files/services/metadata.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import swapper +from django.apps import apps +from django.db import models + +from baseapp_core.plugins import SharedServiceProvider + + +class FilesMetadataService(SharedServiceProvider): + """ + Service that provides files metadata (per-content-type counts dict + enabled + flag) for any object that has a `DocumentId`. Registered in `apps.py` + `ready()`. Mirrors `ReactableMetadataService`. + """ + + @property + def service_name(self) -> str: + return "files_metadata" + + def is_available(self) -> bool: + return apps.is_installed("baseapp.files") + + def _get_model(self): + return swapper.load_model("baseapp_files", "FileTarget") + + def _default_counts(self) -> dict: + from ..utils import default_files_count + + return default_files_count() + + def get_file_target(self, obj): + """Return `FileTarget` for `obj`, or `None` if not found.""" + return self._get_model().get_for_object(obj) + + def get_or_create_file_target(self, obj): + """Return or create `FileTarget` for `obj`.""" + from ..utils import get_or_create_file_target + + return get_or_create_file_target(obj) + + def get_files_count(self, obj) -> dict: + """Return per-content-type files count dict for `obj`. Uses annotation if + available; when the annotation is present but NULL (no FileTarget row exists + yet) return the default counts dict immediately — no extra round-trip.""" + if hasattr(obj, "_file_target_files_count"): + val = obj._file_target_files_count + return val if val is not None else self._default_counts() + file_target = self.get_file_target(obj) + if file_target is not None: + return file_target.files_count + return self._default_counts() + + def is_files_enabled(self, obj) -> bool: + """Return whether files are enabled for `obj`. Uses annotation if available.""" + if hasattr(obj, "_file_target_is_files_enabled"): + val = obj._file_target_is_files_enabled + return val if val is not None else True + file_target = self.get_file_target(obj) + return file_target.is_files_enabled if file_target else True + + def set_is_files_enabled(self, obj, value: bool) -> None: + """Write-through setter for the `is_files_enabled` flag, creating the + FileTarget row if needed.""" + file_target = self.get_or_create_file_target(obj) + if file_target is not None: + file_target.is_files_enabled = bool(value) + file_target.save(update_fields=["is_files_enabled"]) + + def annotate_queryset(self, queryset: models.QuerySet) -> models.QuerySet: + return self._get_model().annotate_queryset(queryset) diff --git a/baseapp/files/services/upload_service.py b/baseapp/files/services/upload_service.py new file mode 100644 index 00000000..4c6ee531 --- /dev/null +++ b/baseapp/files/services/upload_service.py @@ -0,0 +1,185 @@ +import logging +from datetime import timedelta + +import swapper +from django.conf import settings +from django.db import transaction +from django.utils import timezone + +from ..storage import get_upload_handler + +logger = logging.getLogger(__name__) + +File = swapper.load_model("baseapp_files", "File") + + +class UploadService: + """Business logic for file uploads.""" + + def __init__(self): + self.handler = get_upload_handler() + + def initiate_multipart_upload( + self, + user, + file_name: str, + file_size: int, + file_content_type: str, + num_parts: int, + part_size: int, + parent_id=None, + profile=None, + ): + """ + Initiate a multipart upload. + + The storage handler's ``initiate_upload`` (a remote call for S3) runs + outside any DB transaction: the File row is created in its own + statement, the network call happens without holding a transaction open, + and the row is then updated. On failure the orphan row is removed. + + Args: + user: The user initiating the upload + file_name: Original file name + file_size: Size in bytes + file_content_type: MIME type + num_parts: Number of parts for multipart upload + part_size: Size of each part in bytes + parent_id: DocumentId primary key (optional) + profile: Profile object (optional) + + Returns: + (file_obj, upload_data) + """ + + # Validate inputs + self._validate_file_params(file_size, num_parts, part_size) + + # Create File record in pending state + file_obj = File.objects.create( + file_name=file_name, + file_size=file_size, + file_content_type=file_content_type, + upload_status=File.UploadStatus.PENDING, + total_parts=num_parts, + created_by=user, + profile=profile, + parent_id=parent_id, + uploaded_parts={}, + upload_expires_at=timezone.now() + timedelta(hours=24), + ) + + # Get presigned URLs from storage handler (network I/O — not in a transaction) + try: + upload_data = self.handler.initiate_upload(file_obj, num_parts, part_size) + except Exception: + # Cleanup file record if upload initiation fails + file_obj.delete() + raise + + # Store upload_id + file_obj.upload_id = upload_data["upload_id"] + file_obj.upload_status = File.UploadStatus.UPLOADING + file_obj.save(update_fields=["upload_id", "upload_status"]) + + return file_obj, upload_data + + def complete_multipart_upload(self, file_id: int, parts: list): + """ + Complete a multipart upload. + + The storage handler's ``complete_upload`` (a remote call for S3) runs + outside the transaction: state is validated first, the remote assembly + happens without holding a row lock, then a short ``select_for_update`` + transaction flips the row to COMPLETED — guarding against a concurrent + complete/abort without pinning the lock across network I/O. + """ + file_obj = File.objects.get(id=file_id) + + # Validate state + if file_obj.upload_status not in [File.UploadStatus.PENDING, File.UploadStatus.UPLOADING]: + raise ValueError(f"Cannot complete upload in status: {file_obj.upload_status}") + + if not file_obj.upload_id: + raise ValueError("No upload_id found") + + # Validate parts + self._validate_parts(file_obj, parts) + + # Complete with storage handler (network I/O — outside the transaction) + try: + file_path = self.handler.complete_upload(file_obj, file_obj.upload_id, parts) + except Exception: + File.objects.filter(id=file_id).update(upload_status=File.UploadStatus.FAILED) + raise + + # Persist the result under a short row lock. If a concurrent request + # already finished (or aborted) this upload, don't clobber it. + with transaction.atomic(): + locked = File.objects.select_for_update().get(id=file_id) + if locked.upload_status in (File.UploadStatus.PENDING, File.UploadStatus.UPLOADING): + locked.file = file_path + locked.upload_status = File.UploadStatus.COMPLETED + locked.uploaded_parts = {str(p["part_number"]): p["etag"] for p in parts} + locked.upload_id = "" + locked.upload_expires_at = None + locked.save() + return locked + + def abort_multipart_upload(self, file_id: int): + """ + Abort a multipart upload and cleanup. The storage abort (network I/O) + runs outside the transaction; the row is then marked ABORTED. + """ + file_obj = File.objects.get(id=file_id) + + if file_obj.upload_id: + try: + self.handler.abort_upload(file_obj, file_obj.upload_id) + except Exception: + # Continue to mark as aborted, but record the storage-side + # failure so orphaned remote uploads are diagnosable. + logger.exception( + "Storage abort failed for file %s (upload_id=%s)", + file_obj.id, + file_obj.upload_id, + ) + + with transaction.atomic(): + locked = File.objects.select_for_update().get(id=file_id) + locked.upload_status = File.UploadStatus.ABORTED + locked.upload_id = "" + locked.save() + return locked + + def _validate_file_params(self, file_size, num_parts, part_size): + """Validate file upload parameters.""" + max_file_size = getattr(settings, "MAX_FILE_UPLOAD_SIZE", 5 * 1024 * 1024 * 1024) # 5GB + + if file_size > max_file_size: + raise ValueError(f"File size exceeds maximum {max_file_size} bytes") + + if num_parts < 1 or num_parts > 10000: + raise ValueError("Number of parts must be between 1 and 10,000") + + if num_parts > 1 and part_size < 5 * 1024 * 1024: + raise ValueError("Part size must be at least 5MB for multipart uploads") + + # file_size must fit within the declared parts: strictly greater than + # the first (num_parts - 1) full parts, and no larger than all of them. + # Mirrors InitiateUploadSerializer so non-REST callers are protected too. + min_size = (num_parts - 1) * part_size + max_size = num_parts * part_size + if file_size < min_size or file_size > max_size: + raise ValueError("Part size and count don't match file size") + + def _validate_parts(self, file_obj, parts): + """Validate uploaded parts.""" + if len(parts) != file_obj.total_parts: + raise ValueError(f"Expected {file_obj.total_parts} parts, received {len(parts)}") + + part_numbers = {p["part_number"] for p in parts} + expected = set(range(1, file_obj.total_parts + 1)) + + if part_numbers != expected: + raise ValueError("Missing or duplicate part numbers") diff --git a/baseapp/files/storage/__init__.py b/baseapp/files/storage/__init__.py new file mode 100644 index 00000000..7185d70a --- /dev/null +++ b/baseapp/files/storage/__init__.py @@ -0,0 +1,21 @@ +from django.core.files.storage import default_storage + + +def get_upload_handler(): + """ + Factory to get appropriate upload handler based on storage backend. + """ + # `default_storage` is a LazyObject: type() returns the wrapper class, so + # use `__class__`, which LazyObject proxies to the wrapped backend, to + # detect the real storage backend. + storage_class_name = default_storage.__class__.__name__ + + # Check if using S3 storage + if "S3Boto3Storage" in storage_class_name: + from .s3 import S3MultipartUploadHandler + + return S3MultipartUploadHandler() + else: + from .local import LocalUploadHandler + + return LocalUploadHandler() diff --git a/baseapp/files/storage/base.py b/baseapp/files/storage/base.py new file mode 100644 index 00000000..9e1fc61e --- /dev/null +++ b/baseapp/files/storage/base.py @@ -0,0 +1,51 @@ +from abc import ABC, abstractmethod +from typing import Any, Dict, List, Optional + + +class BaseUploadHandler(ABC): + """Abstract interface for upload handlers (S3 vs Local).""" + + @abstractmethod + def supports_multipart(self) -> bool: + """Whether this handler supports true multipart uploads.""" + pass + + @abstractmethod + def initiate_upload(self, file_obj, num_parts: int, part_size: int) -> Dict[str, Any]: + """ + Initiate an upload and return presigned URLs. + + Returns: + { + 'upload_id': str, + 'presigned_urls': [ + {'part_number': 1, 'url': 'https://...'}, + {'part_number': 2, 'url': 'https://...'}, + ], + 'expires_in': int (seconds) + } + """ + pass + + @abstractmethod + def complete_upload(self, file_obj, upload_id: str, parts: List[Dict[str, Any]]) -> str: + """ + Complete the upload. + + Args: + parts: [{'part_number': 1, 'etag': 'xxx'}, ...] + + Returns: + final_url: str (file URL) + """ + pass + + @abstractmethod + def abort_upload(self, file_obj, upload_id: str) -> None: + """Abort and cleanup an upload.""" + pass + + @abstractmethod + def get_file_url(self, file_obj) -> Optional[str]: + """Get the URL for accessing the file, or None if not available yet.""" + pass diff --git a/baseapp/files/storage/local.py b/baseapp/files/storage/local.py new file mode 100644 index 00000000..7d05c51a --- /dev/null +++ b/baseapp/files/storage/local.py @@ -0,0 +1,126 @@ +import os +import shutil +import uuid +from typing import Any, Dict, List, Optional + +from django.conf import settings + +from baseapp_core.models import random_name_in + +from ..tokens import PresignedUploadToken +from .base import BaseUploadHandler + + +class LocalUploadHandler(BaseUploadHandler): + """ + Simplified local storage handler for dev/test. + + This doesn't implement true multipart upload - instead it provides + "presigned URLs" that are actually backend endpoints with signed tokens + that accept the file parts and store them locally. + """ + + def supports_multipart(self) -> bool: + return False # Simplified multipart + + def initiate_upload(self, file_obj, num_parts: int, part_size: int) -> Dict[str, Any]: + """ + Generate presigned URLs with signed tokens for secure uploads. + """ + + # Generate a simple upload token + upload_id = str(uuid.uuid4()) + + # Create temp directory for parts if needed + temp_dir = os.path.join(settings.MEDIA_ROOT, "temp_uploads", upload_id) + os.makedirs(temp_dir, exist_ok=True) + + # Get the base URL from settings or use default + base_url = settings.URL + + # Generate presigned URLs with signed tokens + presigned_urls = [] + for part_num in range(1, num_parts + 1): + token = PresignedUploadToken.mint( + file_id=file_obj.id, + part_number=part_num, + upload_id=upload_id, + ) + + presigned_urls.append( + { + "part_number": part_num, + "url": f"{base_url}/v1/files/presigned-uploads/{file_obj.id}/upload-part/{part_num}?token={token}", + "method": "PUT", + } + ) + + return { + "upload_id": upload_id, + "presigned_urls": presigned_urls, + "expires_in": PresignedUploadToken.max_age(), + } + + def upload_part(self, upload_id: str, part_number: int, data: bytes) -> str: + """ + Upload a part to local storage. + This method is called by the backend when handling local uploads. + + Returns: + etag: str (hash of the data) + """ + import hashlib + + temp_dir = os.path.join(settings.MEDIA_ROOT, "temp_uploads", upload_id) + os.makedirs(temp_dir, exist_ok=True) + + part_file = os.path.join(temp_dir, f"part_{part_number}") + + # Write the part + with open(part_file, "wb") as f: + f.write(data) + + # Generate ETag (MD5 hash like S3). Content checksum, not a security + # primitive — usedforsecurity=False documents intent and satisfies linters. + etag = hashlib.md5(data, usedforsecurity=False).hexdigest() + return etag + + def complete_upload(self, file_obj, upload_id: str, parts: List[Dict]) -> str: + """ + Combine uploaded parts into final file. + """ + temp_dir = os.path.join(settings.MEDIA_ROOT, "temp_uploads", upload_id) + + # Generate final file path + final_path = random_name_in("files")(file_obj, file_obj.file_name or "file") + full_path = os.path.join(settings.MEDIA_ROOT, final_path) + + # Ensure directory exists + os.makedirs(os.path.dirname(full_path), exist_ok=True) + + # Combine parts in order. A missing part would silently truncate the + # assembled file, so fail loudly instead. + with open(full_path, "wb") as outfile: + for part in sorted(parts, key=lambda x: x["part_number"]): + part_file = os.path.join(temp_dir, f"part_{part['part_number']}") + if not os.path.exists(part_file): + raise ValueError(f"Missing uploaded part {part['part_number']}") + with open(part_file, "rb") as infile: + outfile.write(infile.read()) + + # Cleanup temp files + shutil.rmtree(temp_dir, ignore_errors=True) + + # Return the relative path (FileField will handle this) + return final_path + + def abort_upload(self, file_obj, upload_id: str): + """Cleanup temp files.""" + temp_dir = os.path.join(settings.MEDIA_ROOT, "temp_uploads", upload_id) + shutil.rmtree(temp_dir, ignore_errors=True) + + def get_file_url(self, file_obj) -> Optional[str]: + """Get the local file URL, or None if the file isn't stored yet.""" + if file_obj.file: + return file_obj.file.url + return None diff --git a/baseapp/files/storage/s3.py b/baseapp/files/storage/s3.py new file mode 100644 index 00000000..4af94420 --- /dev/null +++ b/baseapp/files/storage/s3.py @@ -0,0 +1,130 @@ +from typing import Any, Dict, List, Optional + +import boto3 +from botocore.exceptions import ClientError +from django.conf import settings + +from baseapp_core.models import random_name_in + +from .base import BaseUploadHandler + + +class S3MultipartUploadHandler(BaseUploadHandler): + """Production S3 multipart upload with presigned URLs.""" + + def __init__(self): + self.s3_client = boto3.client( + "s3", + aws_access_key_id=settings.AWS_ACCESS_KEY_ID, + aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, + region_name=getattr(settings, "AWS_S3_REGION_NAME", "us-east-1"), + ) + self.bucket = settings.AWS_STORAGE_BUCKET_NAME + self.url_expiration = getattr(settings, "FILE_UPLOAD_PRESIGNED_URL_EXPIRATION", 3600) + + def supports_multipart(self) -> bool: + return True + + def initiate_upload(self, file_obj, num_parts: int, part_size: int) -> Dict[str, Any]: + """Initiate S3 multipart upload and generate presigned URLs.""" + + # Validate constraints + if num_parts > 10000: + raise ValueError("S3 supports maximum 10,000 parts") + if num_parts > 1 and part_size < 5 * 1024 * 1024: # 5MB minimum + raise ValueError("S3 requires minimum 5MB per part (except last)") + + # Get the S3 key for this file + key = self._get_s3_key(file_obj) + + # Initiate multipart upload + response = self.s3_client.create_multipart_upload( + Bucket=self.bucket, + Key=key, + ContentType=file_obj.file_content_type or "application/octet-stream", + Metadata={ + "original_filename": file_obj.file_name or "", + "created_by": str(file_obj.created_by_id) if file_obj.created_by_id else "", + }, + ) + + upload_id = response["UploadId"] + + # Generate presigned URLs for each part + presigned_urls = [] + for part_num in range(1, num_parts + 1): + url = self.s3_client.generate_presigned_url( + "upload_part", + Params={ + "Bucket": self.bucket, + "Key": key, + "UploadId": upload_id, + "PartNumber": part_num, + }, + ExpiresIn=self.url_expiration, + ) + presigned_urls.append( + { + "part_number": part_num, + "url": url, + } + ) + + return { + "upload_id": upload_id, + "presigned_urls": presigned_urls, + "expires_in": self.url_expiration, + } + + def complete_upload(self, file_obj, upload_id: str, parts: List[Dict]) -> str: + """Complete S3 multipart upload.""" + + key = self._get_s3_key(file_obj) + + # Format parts for S3 API + multipart_upload = { + "Parts": [ + { + "PartNumber": part["part_number"], + "ETag": part["etag"], + } + for part in sorted(parts, key=lambda x: x["part_number"]) + ] + } + + # Complete the upload + self.s3_client.complete_multipart_upload( + Bucket=self.bucket, + Key=key, + UploadId=upload_id, + MultipartUpload=multipart_upload, + ) + + # Return the file key (not full URL, FileField will handle that) + return key + + def abort_upload(self, file_obj, upload_id: str): + """Abort S3 multipart upload and cleanup parts.""" + key = self._get_s3_key(file_obj) + + try: + self.s3_client.abort_multipart_upload( + Bucket=self.bucket, + Key=key, + UploadId=upload_id, + ) + except ClientError as e: + # Log but don't fail if already aborted/completed + if e.response["Error"]["Code"] != "NoSuchUpload": + raise + + def get_file_url(self, file_obj) -> Optional[str]: + """Get the S3 URL for the file, or None if the file isn't stored yet.""" + if file_obj.file: + return file_obj.file.url + return None + + def _get_s3_key(self, file_obj) -> str: + """Generate S3 key from file object.""" + # Use the same upload_to logic from the FileField + return random_name_in("files")(file_obj, file_obj.file_name or "file") diff --git a/baseapp/files/tests/__init__.py b/baseapp/files/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/baseapp/files/tests/conftest.py b/baseapp/files/tests/conftest.py new file mode 100644 index 00000000..d91acda5 --- /dev/null +++ b/baseapp/files/tests/conftest.py @@ -0,0 +1,2 @@ +from baseapp_core.graphql.testing.fixtures import * # noqa: F401,F403 +from baseapp_core.tests.fixtures import * # noqa: F401,F403 diff --git a/baseapp/files/tests/test_cleanup.py b/baseapp/files/tests/test_cleanup.py new file mode 100644 index 00000000..c0fb62cb --- /dev/null +++ b/baseapp/files/tests/test_cleanup.py @@ -0,0 +1,190 @@ +from datetime import timedelta +from typing import Optional +from unittest.mock import MagicMock + +import pytest +import swapper +from django.contrib.auth import get_user_model +from django.utils import timezone + +from baseapp.files.services.cleanup import ( + cleanup_expired_uploads, + cleanup_failed_uploads, +) + +File = swapper.load_model("baseapp_files", "File") +User = get_user_model() + +pytestmark = pytest.mark.django_db + + +@pytest.fixture +def user() -> "User": + return User.objects.create_user(email="cleanup@example.com", password="testpass123") + + +@pytest.fixture +def mock_handler(monkeypatch: pytest.MonkeyPatch) -> MagicMock: + """Replace the storage handler used by the cleanup task.""" + handler = MagicMock() + monkeypatch.setattr("baseapp.files.services.cleanup.get_upload_handler", lambda: handler) + return handler + + +def create_file( + user: "User", + upload_status: str, + expires_delta: Optional[timedelta] = None, + upload_id: Optional[str] = "upload-id-123", +) -> "File": + """Create a File in the given upload status with an optional expiration offset.""" + upload_expires_at = timezone.now() + expires_delta if expires_delta is not None else None + return File.objects.create( + file_name="test.mp4", + file_size=1024, + file_content_type="video/mp4", + upload_status=upload_status, + upload_id=upload_id, + total_parts=1, + created_by=user, + upload_expires_at=upload_expires_at, + ) + + +class TestCleanupExpiredUploads: + """Tests for the cleanup_expired_uploads celery task.""" + + def test_aborts_expired_pending_and_uploading(self, user, mock_handler): + """Expired PENDING and UPLOADING uploads are aborted on storage and in the DB.""" + pending = create_file(user, File.UploadStatus.PENDING, expires_delta=timedelta(hours=-1)) + uploading = create_file( + user, File.UploadStatus.UPLOADING, expires_delta=timedelta(hours=-1) + ) + + result = cleanup_expired_uploads() + + assert result == "Cleaned up 2 expired uploads" + assert mock_handler.abort_upload.call_count == 2 + + for file_obj in (pending, uploading): + file_obj.refresh_from_db() + assert file_obj.upload_status == File.UploadStatus.ABORTED + assert file_obj.upload_id == "" + + def test_leaves_non_expired_uploads_alone(self, user, mock_handler): + """Uploads that have not expired yet are untouched.""" + file_obj = create_file(user, File.UploadStatus.UPLOADING, expires_delta=timedelta(hours=1)) + + result = cleanup_expired_uploads() + + assert result == "Cleaned up 0 expired uploads" + mock_handler.abort_upload.assert_not_called() + + file_obj.refresh_from_db() + assert file_obj.upload_status == File.UploadStatus.UPLOADING + assert file_obj.upload_id == "upload-id-123" + + def test_leaves_completed_files_alone(self, user, mock_handler): + """COMPLETED files are never aborted, even with a past expiration date.""" + file_obj = create_file( + user, File.UploadStatus.COMPLETED, expires_delta=timedelta(hours=-1), upload_id="" + ) + + result = cleanup_expired_uploads() + + assert result == "Cleaned up 0 expired uploads" + mock_handler.abort_upload.assert_not_called() + + file_obj.refresh_from_db() + assert file_obj.upload_status == File.UploadStatus.COMPLETED + + def test_skips_storage_abort_without_upload_id(self, user, mock_handler): + """Expired uploads without an upload_id are aborted in the DB only.""" + file_obj = create_file( + user, File.UploadStatus.PENDING, expires_delta=timedelta(hours=-1), upload_id="" + ) + + result = cleanup_expired_uploads() + + assert result == "Cleaned up 1 expired uploads" + mock_handler.abort_upload.assert_not_called() + + file_obj.refresh_from_db() + assert file_obj.upload_status == File.UploadStatus.ABORTED + + def test_continues_past_handler_exception(self, user, mock_handler): + """A storage failure on one file doesn't stop cleanup of the others.""" + failing = create_file(user, File.UploadStatus.UPLOADING, expires_delta=timedelta(hours=-1)) + succeeding = create_file( + user, File.UploadStatus.UPLOADING, expires_delta=timedelta(hours=-1) + ) + + def abort_upload(file_obj, upload_id): + if file_obj.pk == failing.pk: + raise Exception("S3 is down") + + mock_handler.abort_upload.side_effect = abort_upload + + result = cleanup_expired_uploads() + + assert result == "Cleaned up 1 expired uploads" + assert mock_handler.abort_upload.call_count == 2 + + failing.refresh_from_db() + assert failing.upload_status == File.UploadStatus.UPLOADING + + succeeding.refresh_from_db() + assert succeeding.upload_status == File.UploadStatus.ABORTED + + +class TestCleanupFailedUploads: + """Tests for the cleanup_failed_uploads helper.""" + + def _age_file(self, file_obj: "File", days: int) -> None: + """Backdate the modified timestamp bypassing auto_now.""" + File.objects.filter(pk=file_obj.pk).update(modified=timezone.now() - timedelta(days=days)) + + def test_deletes_old_failed_and_aborted(self, user): + """FAILED/ABORTED files older than the cutoff are deleted.""" + failed = create_file(user, File.UploadStatus.FAILED, upload_id="") + aborted = create_file(user, File.UploadStatus.ABORTED, upload_id="") + self._age_file(failed, days=8) + self._age_file(aborted, days=8) + + result = cleanup_failed_uploads(days_old=7) + + assert result == "Deleted 2 old failed/aborted uploads" + assert not File.objects.filter(pk__in=[failed.pk, aborted.pk]).exists() + + def test_keeps_recent_failed_uploads(self, user): + """FAILED/ABORTED files newer than the cutoff are kept.""" + failed = create_file(user, File.UploadStatus.FAILED, upload_id="") + aborted = create_file(user, File.UploadStatus.ABORTED, upload_id="") + self._age_file(failed, days=3) + + result = cleanup_failed_uploads(days_old=7) + + assert result == "Deleted 0 old failed/aborted uploads" + assert File.objects.filter(pk__in=[failed.pk, aborted.pk]).count() == 2 + + def test_keeps_old_files_in_other_statuses(self, user): + """Old COMPLETED/PENDING/UPLOADING files are never deleted.""" + completed = create_file(user, File.UploadStatus.COMPLETED, upload_id="") + pending = create_file(user, File.UploadStatus.PENDING) + uploading = create_file(user, File.UploadStatus.UPLOADING) + for file_obj in (completed, pending, uploading): + self._age_file(file_obj, days=30) + + result = cleanup_failed_uploads(days_old=7) + + assert result == "Deleted 0 old failed/aborted uploads" + assert File.objects.count() == 3 + + def test_respects_custom_cutoff(self, user): + """The days_old argument moves the deletion cutoff.""" + failed = create_file(user, File.UploadStatus.FAILED, upload_id="") + self._age_file(failed, days=2) + + assert cleanup_failed_uploads(days_old=7) == "Deleted 0 old failed/aborted uploads" + assert cleanup_failed_uploads(days_old=1) == "Deleted 1 old failed/aborted uploads" + assert not File.objects.filter(pk=failed.pk).exists() diff --git a/baseapp/files/tests/test_graphql.py b/baseapp/files/tests/test_graphql.py new file mode 100644 index 00000000..5350a419 --- /dev/null +++ b/baseapp/files/tests/test_graphql.py @@ -0,0 +1,797 @@ +import pytest +import swapper +from django.contrib.contenttypes.models import ContentType + +from baseapp_comments.tests.factories import CommentFactory +from baseapp_core.models import DocumentId + +File = swapper.load_model("baseapp_files", "File") +FileTarget = swapper.load_model("baseapp_files", "FileTarget") + +pytestmark = pytest.mark.django_db + +COMMENT_FILES_QUERY = """ + query CommentFiles($id: ID!) { + node(id: $id) { + ... on FilesInterface { + files { + edges { + node { + id + } + } + } + } + } + } +""" + +COMMENT_FILES_COUNT_QUERY = """ + query CommentFilesCount($id: ID!) { + node(id: $id) { + ... on FilesInterface { + filesCount + } + } + } +""" + +COMMENT_IS_FILES_ENABLED_QUERY = """ + query CommentIsFilesEnabled($id: ID!) { + node(id: $id) { + ... on FilesInterface { + isFilesEnabled + } + } + } +""" + +COMMENT_FILES_FULL_QUERY = """ + query CommentFilesFull($id: ID!) { + node(id: $id) { + ... on FilesInterface { + filesCount + isFilesEnabled + files { + edges { + node { + id + } + } + } + } + } + } +""" + + +def test_comment_files_interface(graphql_client): + comment = CommentFactory() + comment_content_type = ContentType.objects.get_for_model(comment) + comment_document_id, _ = DocumentId.objects.get_or_create( + content_type=comment_content_type, + object_id=comment.pk, + ) + file = File.objects.create( + parent=comment_document_id, + file_name="example.txt", + ) + + response = graphql_client(COMMENT_FILES_QUERY, variables={"id": comment.relay_id}) + content = response.json() + + edges = content["data"]["node"]["files"]["edges"] + assert len(edges) == 1 + assert edges[0]["node"]["id"] == file.relay_id + + +def test_comment_files_count_interface(graphql_client): + comment = CommentFactory() + comment_content_type = ContentType.objects.get_for_model(comment) + comment_document_id, _ = DocumentId.objects.get_or_create( + content_type=comment_content_type, + object_id=comment.pk, + ) + + File.objects.create( + parent=comment_document_id, + file_content_type="image/png", + ) + File.objects.create( + parent=comment_document_id, + file_content_type="image/jpeg", + ) + + response = graphql_client(COMMENT_FILES_COUNT_QUERY, variables={"id": comment.relay_id}) + content = response.json() + + files_count = content["data"]["node"]["filesCount"] + assert files_count["total"] == 2 + assert files_count["image/png"] == 1 + assert files_count["image/jpeg"] == 1 + + +def test_comment_files_count_interface_empty(graphql_client): + comment = CommentFactory() + + response = graphql_client(COMMENT_FILES_COUNT_QUERY, variables={"id": comment.relay_id}) + content = response.json() + + files_count = content["data"]["node"]["filesCount"] + assert files_count["total"] == 0 + + +def test_comment_is_files_enabled_interface(graphql_client): + comment = CommentFactory() + + response = graphql_client(COMMENT_IS_FILES_ENABLED_QUERY, variables={"id": comment.relay_id}) + content = response.json() + + is_files_enabled = content["data"]["node"]["isFilesEnabled"] + assert is_files_enabled is True + + +def test_comment_is_files_enabled_interface_disabled(graphql_client): + comment = CommentFactory() + file_target = comment.get_file_target() + file_target.is_files_enabled = False + file_target.save() + + response = graphql_client(COMMENT_IS_FILES_ENABLED_QUERY, variables={"id": comment.relay_id}) + content = response.json() + + is_files_enabled = content["data"]["node"]["isFilesEnabled"] + assert is_files_enabled is False + + +def test_comment_files_full_interface(graphql_client): + comment = CommentFactory() + comment_content_type = ContentType.objects.get_for_model(comment) + comment_document_id, _ = DocumentId.objects.get_or_create( + content_type=comment_content_type, + object_id=comment.pk, + ) + + file = File.objects.create( + parent=comment_document_id, + file_content_type="image/png", + file_name="example.png", + ) + + response = graphql_client(COMMENT_FILES_FULL_QUERY, variables={"id": comment.relay_id}) + content = response.json() + + data = content["data"]["node"] + assert data["filesCount"]["total"] == 1 + assert data["isFilesEnabled"] is True + assert len(data["files"]["edges"]) == 1 + assert data["files"]["edges"][0]["node"]["id"] == file.relay_id + + +def test_comment_files_interface_multiple_files(graphql_client): + comment = CommentFactory() + comment_content_type = ContentType.objects.get_for_model(comment) + comment_document_id, _ = DocumentId.objects.get_or_create( + content_type=comment_content_type, + object_id=comment.pk, + ) + + for i in range(5): + File.objects.create( + parent=comment_document_id, + file_name=f"file_{i}.txt", + ) + + response = graphql_client(COMMENT_FILES_QUERY, variables={"id": comment.relay_id}) + content = response.json() + + edges = content["data"]["node"]["files"]["edges"] + assert len(edges) == 5 + + +FILE_ATTACH_TO_TARGET_MUTATION = """ + mutation FileAttachToTarget($fileRelayIds: [ID]!, $targetObjectId: ID!) { + fileAttachToTarget(input: { + fileRelayIds: $fileRelayIds + targetObjectId: $targetObjectId + }) { + attachedFiles { + node { + id + fileName + } + } + target { + ... on FilesInterface { + filesCount + isFilesEnabled + } + } + } + } +""" + + +def test_file_attach_to_target_success(graphql_user_client, django_user_client): + user = django_user_client.user + comment = CommentFactory(user=user) + + # Create standalone files (no parent) + file1 = File.objects.create( + file_name="file1.txt", + file_size=1024, + upload_status=File.UploadStatus.COMPLETED, + created_by=user, + ) + file2 = File.objects.create( + file_name="file2.txt", + file_size=2048, + upload_status=File.UploadStatus.COMPLETED, + created_by=user, + ) + + response = graphql_user_client( + FILE_ATTACH_TO_TARGET_MUTATION, + variables={ + "fileRelayIds": [file1.relay_id, file2.relay_id], + "targetObjectId": comment.relay_id, + }, + ) + content = response.json() + + assert "errors" not in content + data = content["data"]["fileAttachToTarget"] + + # Check attached files + assert len(data["attachedFiles"]) == 2 + assert data["attachedFiles"][0]["node"]["id"] == file1.relay_id + assert data["attachedFiles"][0]["node"]["fileName"] == "file1.txt" + assert data["attachedFiles"][1]["node"]["id"] == file2.relay_id + + # Check target + assert data["target"]["filesCount"]["total"] == 2 + assert data["target"]["isFilesEnabled"] is True + + # Verify files are attached in database + file1.refresh_from_db() + file2.refresh_from_db() + comment_ct = ContentType.objects.get_for_model(comment) + comment_doc_id = DocumentId.objects.get(content_type=comment_ct, object_id=comment.pk) + assert file1.parent == comment_doc_id + assert file2.parent == comment_doc_id + + +def test_file_attach_to_target_single_file(graphql_user_client, django_user_client): + user = django_user_client.user + comment = CommentFactory(user=user) + + file = File.objects.create( + file_name="single.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=user, + ) + + response = graphql_user_client( + FILE_ATTACH_TO_TARGET_MUTATION, + variables={ + "fileRelayIds": [file.relay_id], + "targetObjectId": comment.relay_id, + }, + ) + content = response.json() + + assert "errors" not in content + data = content["data"]["fileAttachToTarget"] + assert len(data["attachedFiles"]) == 1 + assert data["target"]["filesCount"]["total"] == 1 + + +def test_file_attach_to_target_rejects_non_file_relay_id(graphql_user_client, django_user_client): + """Passing a non-File relay id (e.g. a Comment) is rejected as not_found + rather than crashing on the permission check / parent assignment.""" + user = django_user_client.user + comment = CommentFactory(user=user) + not_a_file = CommentFactory(user=user) + + response = graphql_user_client( + FILE_ATTACH_TO_TARGET_MUTATION, + variables={ + "fileRelayIds": [not_a_file.relay_id], + "targetObjectId": comment.relay_id, + }, + ) + content = response.json() + + assert "errors" in content + assert content["errors"][0]["extensions"]["code"] == "not_found" + + +def test_file_attach_to_target_rejects_non_fileable_target(graphql_user_client, django_user_client): + """A target that does not opt into files (not a FileableModel) is rejected + with invalid_target instead of committing writes then erroring on the + FilesInterface payload.""" + from baseapp_profiles.tests.factories import ProfileFactory + + user = django_user_client.user + # A Profile is a resolvable node but not a FileableModel, so it cannot be a + # files target. + profile = ProfileFactory() + file = File.objects.create( + file_name="test.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=user, + ) + + response = graphql_user_client( + FILE_ATTACH_TO_TARGET_MUTATION, + variables={ + "fileRelayIds": [file.relay_id], + "targetObjectId": profile.relay_id, + }, + ) + content = response.json() + + assert "errors" in content + assert content["errors"][0]["extensions"]["code"] == "invalid_target" + file.refresh_from_db() + assert file.parent_id is None + + +def test_file_attach_to_target_requires_authentication(graphql_client): + comment = CommentFactory() + file = File.objects.create(file_name="test.txt") + + response = graphql_client( + FILE_ATTACH_TO_TARGET_MUTATION, + variables={ + "fileRelayIds": [file.relay_id], + "targetObjectId": comment.relay_id, + }, + ) + content = response.json() + + assert "errors" in content + assert content["errors"][0]["extensions"]["code"] == "authentication_required" + + +def test_file_attach_to_target_requires_ownership(graphql_user_client, django_user_client): + user = django_user_client.user + comment = CommentFactory(user=user) + + # Create file owned by different user + from django.contrib.auth import get_user_model + + User = get_user_model() + other_user = User.objects.create_user(email="other@test.com", password="pass123") + file = File.objects.create( + file_name="test.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=other_user, + ) + + response = graphql_user_client( + FILE_ATTACH_TO_TARGET_MUTATION, + variables={ + "fileRelayIds": [file.relay_id], + "targetObjectId": comment.relay_id, + }, + ) + content = response.json() + + assert "errors" in content + assert content["errors"][0]["extensions"]["code"] == "permission_required" + + +def test_file_attach_to_target_superuser_can_attach_any_file( + graphql_user_client, django_user_client +): + from django.contrib.auth import get_user_model + + User = get_user_model() + other_user = User.objects.create_user(email="other@test.com", password="pass123") + + django_user_client.user.is_superuser = True + django_user_client.user.save() + + comment = CommentFactory(user=other_user) + file = File.objects.create( + file_name="test.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=other_user, + ) + + response = graphql_user_client( + FILE_ATTACH_TO_TARGET_MUTATION, + variables={ + "fileRelayIds": [file.relay_id], + "targetObjectId": comment.relay_id, + }, + ) + content = response.json() + + assert "errors" not in content + assert len(content["data"]["fileAttachToTarget"]["attachedFiles"]) == 1 + + +def test_file_attach_to_target_files_disabled(graphql_user_client, django_user_client): + user = django_user_client.user + comment = CommentFactory(user=user) + file_target = comment.get_file_target() + file_target.is_files_enabled = False + file_target.save() + + file = File.objects.create( + file_name="test.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=user, + ) + + response = graphql_user_client( + FILE_ATTACH_TO_TARGET_MUTATION, + variables={ + "fileRelayIds": [file.relay_id], + "targetObjectId": comment.relay_id, + }, + ) + content = response.json() + + assert "errors" in content + assert content["errors"][0]["extensions"]["code"] == "files_disabled" + file.refresh_from_db() + assert file.parent_id is None + + +def test_file_attach_to_target_already_attached(graphql_user_client, django_user_client): + user = django_user_client.user + comment1 = CommentFactory(user=user) + comment2 = CommentFactory(user=user) + + comment1_ct = ContentType.objects.get_for_model(comment1) + comment1_doc_id, _ = DocumentId.objects.get_or_create( + content_type=comment1_ct, + object_id=comment1.pk, + ) + + # Create file already attached to comment1 + file = File.objects.create( + file_name="attached.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=user, + parent=comment1_doc_id, + ) + + # Try to attach to comment2 + response = graphql_user_client( + FILE_ATTACH_TO_TARGET_MUTATION, + variables={ + "fileRelayIds": [file.relay_id], + "targetObjectId": comment2.relay_id, + }, + ) + content = response.json() + + assert "errors" in content + assert content["errors"][0]["extensions"]["code"] == "already_attached" + + +def test_file_attach_to_target_empty_list(graphql_user_client, django_user_client): + user = django_user_client.user + comment = CommentFactory(user=user) + + response = graphql_user_client( + FILE_ATTACH_TO_TARGET_MUTATION, + variables={ + "fileRelayIds": [], + "targetObjectId": comment.relay_id, + }, + ) + content = response.json() + + assert "errors" in content + assert content["errors"][0]["extensions"]["code"] == "invalid_input" + + +def test_file_attach_to_target_invalid_file_id(graphql_user_client, django_user_client): + user = django_user_client.user + comment = CommentFactory(user=user) + + response = graphql_user_client( + FILE_ATTACH_TO_TARGET_MUTATION, + variables={ + "fileRelayIds": ["RmlsZU9iamVjdFR5cGU6OTk5OTk="], # Non-existent file ID + "targetObjectId": comment.relay_id, + }, + ) + content = response.json() + + assert "errors" in content + # Error might not have extensions if it's a relay ID parsing error + if "extensions" in content["errors"][0]: + assert content["errors"][0]["extensions"]["code"] == "not_found" + + +FILE_DELETE_MUTATION = """ + mutation FileDeleteMutation($input: FileDeleteInput!) { + fileDelete(input: $input) { + deletedId + parent { + ... on FilesInterface { + filesCount + } + } + errors { + field + messages + } + } + } +""" + + +def test_anon_cant_delete_file(graphql_client, django_user_client): + user = django_user_client.user + file = File.objects.create( + file_name="test.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=user, + ) + + response = graphql_client( + FILE_DELETE_MUTATION, + variables={"input": {"id": file.relay_id}}, + ) + content = response.json() + assert content["errors"][0]["message"] == "authentication required" + assert File.objects.count() == 1 + + +def test_user_cant_delete_any_file(graphql_user_client, django_user_client): + from django.contrib.auth import get_user_model + + User = get_user_model() + other_user = User.objects.create_user(email="other@test.com", password="pass123") + + file = File.objects.create( + file_name="test.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=other_user, + ) + + response = graphql_user_client( + FILE_DELETE_MUTATION, + variables={"input": {"id": file.relay_id}}, + ) + content = response.json() + assert content["errors"][0]["extensions"]["code"] == "permission_required" + assert File.objects.count() == 1 + + +def test_owner_can_delete_file(django_user_client, graphql_user_client): + user = django_user_client.user + file = File.objects.create( + file_name="test.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=user, + ) + + response = graphql_user_client( + FILE_DELETE_MUTATION, + variables={"input": {"id": file.relay_id}}, + ) + content = response.json() + assert content["data"]["fileDelete"]["deletedId"] == file.relay_id + assert File.objects.count() == 0 + + +def test_superuser_can_delete_file(django_user_client, graphql_user_client): + from django.contrib.auth import get_user_model + + User = get_user_model() + other_user = User.objects.create_user(email="other@test.com", password="pass123") + + django_user_client.user.is_superuser = True + django_user_client.user.save() + + file = File.objects.create( + file_name="test.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=other_user, + ) + + response = graphql_user_client( + FILE_DELETE_MUTATION, + variables={"input": {"id": file.relay_id}}, + ) + content = response.json() + assert content["data"]["fileDelete"]["deletedId"] == file.relay_id + assert File.objects.count() == 0 + + +def test_user_with_permission_can_delete_file(django_user_client, graphql_user_client): + from django.contrib.auth import get_user_model + from django.contrib.auth.models import Permission + + User = get_user_model() + other_user = User.objects.create_user(email="other@test.com", password="pass123") + + app_label = File._meta.app_label + perm = Permission.objects.get(content_type__app_label=app_label, codename="delete_file") + django_user_client.user.user_permissions.add(perm) + + file = File.objects.create( + file_name="test.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=other_user, + ) + + response = graphql_user_client( + FILE_DELETE_MUTATION, + variables={"input": {"id": file.relay_id}}, + ) + content = response.json() + assert content["data"]["fileDelete"]["deletedId"] == file.relay_id + assert File.objects.count() == 0 + + +def test_update_files_count_after_delete_file(django_user_client, graphql_user_client): + user = django_user_client.user + comment = CommentFactory(user=user) + comment_content_type = ContentType.objects.get_for_model(comment) + comment_document_id, _ = DocumentId.objects.get_or_create( + content_type=comment_content_type, + object_id=comment.pk, + ) + + file = File.objects.create( + file_name="test.txt", + file_content_type="text/plain", + upload_status=File.UploadStatus.COMPLETED, + created_by=user, + parent=comment_document_id, + ) + + # Refresh to get updated files_count + comment.refresh_from_db() + assert comment.files_count["total"] == 1 + assert comment.files_count["text/plain"] == 1 + + response = graphql_user_client( + FILE_DELETE_MUTATION, + variables={"input": {"id": file.relay_id}}, + ) + content = response.json() + + assert content["data"]["fileDelete"]["parent"]["filesCount"]["total"] == 0 + assert File.objects.count() == 0 + + comment.refresh_from_db() + assert comment.files_count["total"] == 0 + + +MY_FILES_QUERY = """ + query MyFiles($noParent: Boolean) { + myFiles(noParent: $noParent) { + edges { + node { + id + fileName + } + } + } + } +""" + +FILE_NODE_QUERY = """ + query GetFile($id: ID!) { + file(id: $id) { + id + fileName + } + } +""" + + +def test_my_files_includes_own_files(graphql_user_client, django_user_client): + own_file = File.objects.create( + file_name="mine.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=django_user_client.user, + ) + + response = graphql_user_client(MY_FILES_QUERY) + content = response.json() + + assert "errors" not in content + edges = content["data"]["myFiles"]["edges"] + assert len(edges) == 1 + assert edges[0]["node"]["id"] == own_file.relay_id + assert edges[0]["node"]["fileName"] == "mine.txt" + + +def test_my_files_excludes_other_users_files(graphql_user_client, django_user_client): + """myFiles is scoped to the authenticated user — other users' files must + never be returned.""" + from django.contrib.auth import get_user_model + + User = get_user_model() + other_user = User.objects.create_user(email="other@test.com", password="pass123") + + File.objects.create( + file_name="theirs.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=other_user, + ) + own_file = File.objects.create( + file_name="mine.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=django_user_client.user, + ) + + response = graphql_user_client(MY_FILES_QUERY) + content = response.json() + + assert "errors" not in content + edges = content["data"]["myFiles"]["edges"] + assert len(edges) == 1 + assert edges[0]["node"]["id"] == own_file.relay_id + + +def test_my_files_no_parent_filter(graphql_user_client, django_user_client): + user = django_user_client.user + comment = CommentFactory(user=user) + comment_content_type = ContentType.objects.get_for_model(comment) + comment_document_id, _ = DocumentId.objects.get_or_create( + content_type=comment_content_type, + object_id=comment.pk, + ) + + standalone = File.objects.create( + file_name="standalone.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=user, + ) + File.objects.create( + file_name="attached.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=user, + parent=comment_document_id, + ) + + response = graphql_user_client(MY_FILES_QUERY, variables={"noParent": True}) + content = response.json() + + assert "errors" not in content + edges = content["data"]["myFiles"]["edges"] + assert len(edges) == 1 + assert edges[0]["node"]["id"] == standalone.relay_id + + +def test_my_files_empty_for_anonymous(graphql_client, django_user_client): + """Anonymous visitors must not be able to list any files via myFiles.""" + File.objects.create( + file_name="mine.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=django_user_client.user, + ) + + response = graphql_client(MY_FILES_QUERY) + content = response.json() + + assert "errors" not in content + edges = content["data"]["myFiles"]["edges"] + assert len(edges) == 0 + + +def test_file_node_query_returns_file_for_owner(graphql_user_client, django_user_client): + file = File.objects.create( + file_name="mine.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=django_user_client.user, + ) + + response = graphql_user_client(FILE_NODE_QUERY, variables={"id": file.relay_id}) + content = response.json() + + assert "errors" not in content + assert content["data"]["file"]["id"] == file.relay_id + assert content["data"]["file"]["fileName"] == "mine.txt" diff --git a/baseapp/files/tests/test_graphql_queries_object_files.py b/baseapp/files/tests/test_graphql_queries_object_files.py new file mode 100644 index 00000000..17550b81 --- /dev/null +++ b/baseapp/files/tests/test_graphql_queries_object_files.py @@ -0,0 +1,194 @@ +""" +Query-count tests for the `FilesInterface`. + +Mirrors `baseapp_reactions/tests/test_graphql_queries_object_reactions.py`: assert +the GraphQL `FilesInterface` resolves in a flat number of database queries +regardless of how many `File` rows point at the target — locks in the +`FilesMetadataService.annotate_queryset` optimisation. +""" + +import pytest +import swapper +from constance.test import override_config +from django.contrib.contenttypes.models import ContentType + +from baseapp_comments.tests.factories import CommentFactory +from baseapp_core.models import DocumentId + +pytestmark = pytest.mark.django_db + +File = swapper.load_model("baseapp_files", "File") +FileTarget = swapper.load_model("baseapp_files", "FileTarget") + +# Baseline for `filesCount` + `isFilesEnabled` on one Comment node: 1× relay +# `node()` (documentid + content_type JOIN) + 1× `comments_comment` load with +# `FileTarget` inlined as a Subquery via `pre_optimization_hook`. +# Update this number deliberately if you change the resolver / annotation path. +EXPECTED_FILES_INTERFACE_QUERY_COUNT = 2 + +COUNTS_ONLY_QUERY = """ + query GetTarget($id: ID!) { + node(id: $id) { + ... on FilesInterface { + filesCount + isFilesEnabled + } + } + } +""" + +COMMENTS_WITH_FILES_LIST_QUERY = """ + query GetTarget($id: ID!) { + node(id: $id) { + ... on CommentsInterface { + comments(first: 50) { + edges { + node { + id + ... on FilesInterface { + filesCount + isFilesEnabled + } + } + } + } + } + } + } +""" + + +def attach_files(target, amount: int) -> None: + document_id = DocumentId.get_or_create_for_object(target) + for i in range(amount): + File.objects.create( + parent=document_id, + file_name=f"file-{i}.png", + file_content_type="image/png", + upload_status=File.UploadStatus.COMPLETED, + ) + + +@override_config(ENABLE_PUBLIC_ID_LOGIC=True) +def test_files_count_is_flat_regardless_of_file_volume(graphql_client_with_queries): + """`filesCount` should be a flat query path: regardless of how many `File` + rows point at the target, the GraphQL query should make the same number of + DB round-trips.""" + target_small = CommentFactory() + attach_files(target_small, 3) + + ContentType.objects.clear_cache() + response_small, queries_small = graphql_client_with_queries( + COUNTS_ONLY_QUERY, variables={"id": target_small.relay_id} + ) + assert response_small.json()["data"]["node"]["filesCount"]["total"] == 3 + small_count = queries_small.count + + target_big = CommentFactory() + attach_files(target_big, 20) + + ContentType.objects.clear_cache() + response_big, queries_big = graphql_client_with_queries( + COUNTS_ONLY_QUERY, variables={"id": target_big.relay_id} + ) + assert response_big.json()["data"]["node"]["filesCount"]["total"] == 20 + + assert queries_big.count == small_count + # Absolute baseline, catches regressions that bump both sides together. + assert small_count == EXPECTED_FILES_INTERFACE_QUERY_COUNT + + +@override_config(ENABLE_PUBLIC_ID_LOGIC=True) +def test_files_count_zero_when_no_files_does_not_extra_query(graphql_client_with_queries): + """A target with no files should resolve in the same query budget as one + with files — the FileTarget row simply doesn't exist yet and the annotation + falls back to the defaults without an extra round-trip.""" + target_with_files = CommentFactory() + attach_files(target_with_files, 3) + + ContentType.objects.clear_cache() + response_with, queries_with = graphql_client_with_queries( + COUNTS_ONLY_QUERY, variables={"id": target_with_files.relay_id} + ) + assert response_with.json()["data"]["node"]["filesCount"]["total"] == 3 + + target_empty = CommentFactory() + + ContentType.objects.clear_cache() + response_empty, queries_empty = graphql_client_with_queries( + COUNTS_ONLY_QUERY, variables={"id": target_empty.relay_id} + ) + assert response_empty.json()["data"]["node"]["filesCount"]["total"] == 0 + + assert queries_empty.count == queries_with.count + assert queries_empty.count == EXPECTED_FILES_INTERFACE_QUERY_COUNT + + +@override_config(ENABLE_PUBLIC_ID_LOGIC=True) +def test_listing_comments_with_files_is_flat(graphql_client_with_queries): + """Listing comments that each have files must not fan out one FileTarget + query per comment — the files annotations ride along in the comment list + SELECT via `pre_optimization_hook`.""" + target_small = CommentFactory() + for _ in range(2): + reply = CommentFactory(target=target_small) + attach_files(reply, 2) + + ContentType.objects.clear_cache() + response_small, queries_small = graphql_client_with_queries( + COMMENTS_WITH_FILES_LIST_QUERY, variables={"id": target_small.relay_id} + ) + edges_small = response_small.json()["data"]["node"]["comments"]["edges"] + assert len(edges_small) == 2 + assert all(edge["node"]["filesCount"]["total"] == 2 for edge in edges_small) + small_count = queries_small.count + + target_big = CommentFactory() + for _ in range(8): + reply = CommentFactory(target=target_big) + attach_files(reply, 3) + + ContentType.objects.clear_cache() + response_big, queries_big = graphql_client_with_queries( + COMMENTS_WITH_FILES_LIST_QUERY, variables={"id": target_big.relay_id} + ) + edges_big = response_big.json()["data"]["node"]["comments"]["edges"] + assert len(edges_big) == 8 + assert all(edge["node"]["filesCount"]["total"] == 3 for edge in edges_big) + + # The query count must not scale with the number of comments or files. + assert queries_big.count == small_count + + +def test_annotate_queryset_evaluates_in_single_query(django_assert_num_queries): + """`FileTarget.annotate_queryset` inlines the metadata as Subqueries — the + annotated queryset must evaluate in exactly one SELECT.""" + Comment = CommentFactory._meta.model + + comment_with_files = CommentFactory() + attach_files(comment_with_files, 2) + comment_disabled = CommentFactory() + file_target = comment_disabled.get_file_target() + file_target.is_files_enabled = False + file_target.save() + comment_bare = CommentFactory() + + ContentType.objects.get_for_model(Comment) # warm the CT cache + + with django_assert_num_queries(1): + rows = { + row.pk: row + for row in FileTarget.annotate_queryset( + Comment.objects.filter( + pk__in=[comment_with_files.pk, comment_disabled.pk, comment_bare.pk] + ) + ) + } + + assert rows[comment_with_files.pk]._file_target_files_count["total"] == 2 + assert rows[comment_with_files.pk]._file_target_is_files_enabled is True + + assert rows[comment_disabled.pk]._file_target_is_files_enabled is False + + assert rows[comment_bare.pk]._file_target_files_count is None + assert rows[comment_bare.pk]._file_target_is_files_enabled is True diff --git a/baseapp/files/tests/test_graphql_utils.py b/baseapp/files/tests/test_graphql_utils.py new file mode 100644 index 00000000..1b01c153 --- /dev/null +++ b/baseapp/files/tests/test_graphql_utils.py @@ -0,0 +1,166 @@ +"""Unit tests for baseapp.files.graphql.utils.attach_files_from_relay_ids.""" + +import pytest +import swapper +from django.contrib.auth import get_user_model +from graphql.error import GraphQLError + +from baseapp.files.graphql.utils import attach_files_from_relay_ids +from baseapp_comments.tests.factories import CommentFactory +from baseapp_core.models import DocumentId + +File = swapper.load_model("baseapp_files", "File") +FileTarget = swapper.load_model("baseapp_files", "FileTarget") +User = get_user_model() + +pytestmark = pytest.mark.django_db + + +@pytest.fixture +def user() -> "User": + return User.objects.create_user(email="owner@example.com", password="testpass123") + + +def create_file(user: "User", parent: DocumentId = None, file_name: str = "file.txt") -> "File": + """Create a completed file owned by the given user.""" + return File.objects.create( + file_name=file_name, + upload_status=File.UploadStatus.COMPLETED, + created_by=user, + parent=parent, + ) + + +def test_attaches_files_to_parent(user): + """Provided relay ids are attached to the parent's DocumentId.""" + comment = CommentFactory(user=user) + file1 = create_file(user, file_name="a.txt") + file2 = create_file(user, file_name="b.txt") + + attach_files_from_relay_ids(comment, [file1.relay_id, file2.relay_id], user) + + parent_document_id = DocumentId.get_or_create_for_object(comment) + file1.refresh_from_db() + file2.refresh_from_db() + assert file1.parent == parent_document_id + assert file2.parent == parent_document_id + + +def test_noop_for_none_and_empty_list(user): + """None or an empty list of ids is a no-op.""" + comment = CommentFactory(user=user) + + assert attach_files_from_relay_ids(comment, None, user) is None + assert attach_files_from_relay_ids(comment, [], user) is None + + +def test_skips_files_already_attached_to_same_parent(user): + """Files already attached to the parent are left untouched.""" + comment = CommentFactory(user=user) + parent_document_id = DocumentId.get_or_create_for_object(comment) + file_obj = create_file(user, parent=parent_document_id) + + attach_files_from_relay_ids(comment, [file_obj.relay_id], user) + + file_obj.refresh_from_db() + assert file_obj.parent == parent_document_id + assert comment.files.count() == 1 + + +def test_moves_file_from_previous_parent_and_recounts(user): + """A file attached to another parent is moved and the old parent's count updates.""" + old_comment = CommentFactory(user=user) + new_comment = CommentFactory(user=user) + old_document_id = DocumentId.get_or_create_for_object(old_comment) + file_obj = create_file(user, parent=old_document_id) + + attach_files_from_relay_ids(new_comment, [file_obj.relay_id], user) + + file_obj.refresh_from_db() + assert file_obj.parent == DocumentId.get_or_create_for_object(new_comment) + + old_file_target = FileTarget.objects.get(target=old_document_id) + assert old_file_target.files_count["total"] == 0 + + +def test_missing_file_raises_not_found(user): + """A relay id that doesn't resolve to a File raises a not_found error.""" + comment = CommentFactory(user=user) + # The comment's relay id resolves to a pk with no matching File row. + with pytest.raises(GraphQLError) as exc_info: + attach_files_from_relay_ids(comment, [comment.relay_id], user) + + assert exc_info.value.extensions["code"] == "not_found" + + +def test_deleted_file_relay_id_raises_not_found(user): + """A relay id whose file (and DocumentId) was deleted raises the same + not_found GraphQLError as a missing File row.""" + comment = CommentFactory(user=user) + file_obj = create_file(user) + relay_id = file_obj.relay_id + file_obj.delete() + + with pytest.raises(GraphQLError) as exc_info: + attach_files_from_relay_ids(comment, [relay_id], user) + + assert exc_info.value.extensions["code"] == "not_found" + + +def test_non_owner_cannot_attach(user): + """A user cannot attach files created by someone else.""" + other_user = User.objects.create_user(email="other@example.com", password="testpass123") + comment = CommentFactory(user=user) + file_obj = create_file(other_user) + + with pytest.raises(GraphQLError) as exc_info: + attach_files_from_relay_ids(comment, [file_obj.relay_id], user) + + assert exc_info.value.extensions["code"] == "permission_required" + file_obj.refresh_from_db() + assert file_obj.parent is None + + +def test_superuser_can_attach_any_file(user): + """A superuser can attach files owned by other users.""" + superuser = User.objects.create_user( + email="admin@example.com", password="testpass123", is_superuser=True + ) + comment = CommentFactory(user=user) + file_obj = create_file(user) + + attach_files_from_relay_ids(comment, [file_obj.relay_id], superuser) + + file_obj.refresh_from_db() + assert file_obj.parent == DocumentId.get_or_create_for_object(comment) + + +def test_without_user_skips_ownership_check(user): + """System calls without a user bypass the ownership check.""" + comment = CommentFactory(user=user) + file_obj = create_file(user) + + attach_files_from_relay_ids(comment, [file_obj.relay_id], None) + + file_obj.refresh_from_db() + assert file_obj.parent == DocumentId.get_or_create_for_object(comment) + + +def test_non_owner_cannot_attach_null_creator_file(user): + """A NULL creator is not an implicit grant — a non-owner cannot re-parent + (steal) a system/orphan file whose created_by is NULL.""" + comment = CommentFactory(user=user) + other_parent = CommentFactory(user=user) + orphan = File.objects.create( + file_name="orphan.txt", + upload_status=File.UploadStatus.COMPLETED, + created_by=None, + parent=DocumentId.get_or_create_for_object(other_parent), + ) + + with pytest.raises(GraphQLError) as exc_info: + attach_files_from_relay_ids(comment, [orphan.relay_id], user) + + assert exc_info.value.extensions["code"] == "permission_required" + orphan.refresh_from_db() + assert orphan.parent == DocumentId.get_or_create_for_object(other_parent) diff --git a/baseapp/files/tests/test_models.py b/baseapp/files/tests/test_models.py new file mode 100644 index 00000000..6b9b637d --- /dev/null +++ b/baseapp/files/tests/test_models.py @@ -0,0 +1,297 @@ +import swapper +from django.contrib.contenttypes.models import ContentType +from django.core.exceptions import ValidationError +from django.test import TestCase + +from baseapp.files.utils import ( + default_files_count, + get_or_create_file_target, + recalculate_files_count, + set_files_parent, +) +from baseapp_comments.tests.factories import CommentFactory +from baseapp_core.models import DocumentId + +File = swapper.load_model("baseapp_files", "File") +FileTarget = swapper.load_model("baseapp_files", "FileTarget") +Comment = swapper.load_model("baseapp_comments", "Comment") + + +def get_or_create_document_id(obj): + """Helper to get or create a DocumentId for an object.""" + ct = ContentType.objects.get_for_model(obj, for_concrete_model=False) + document_id, _ = DocumentId.objects.get_or_create( + content_type=ct, + object_id=obj.pk, + ) + return document_id + + +class FileModelTest(TestCase): + def test_create_file(self): + File.objects.create() + self.assertEqual(File.objects.count(), 1) + + def test_comment_files_count(self): + comment = CommentFactory() + parent_doc_id = get_or_create_document_id(comment) + + File.objects.create(parent=parent_doc_id) + + self.assertEqual(comment.files.count(), 1) + + +class FileTargetTest(TestCase): + def test_file_target_creation(self): + comment = CommentFactory() + file_target = comment.get_file_target() + + self.assertIsNotNone(file_target) + self.assertEqual(file_target.target.object_id, comment.pk) + self.assertTrue(file_target.is_files_enabled) + self.assertEqual(file_target.files_count, {"total": 0}) + + def test_file_target_str_method(self): + comment = CommentFactory() + file_target = comment.get_file_target() + + expected_str = f"FileTarget for {file_target.target}" + self.assertEqual(str(file_target), expected_str) + + def test_file_target_unique_constraint(self): + comment = CommentFactory() + doc_id = get_or_create_document_id(comment) + + FileTarget.objects.create(target=doc_id) + + file_target2, created = FileTarget.objects.get_or_create(target=doc_id) + + self.assertFalse(created) + self.assertEqual(FileTarget.objects.count(), 1) + + def test_file_target_files_count_update_single_type(self): + comment = CommentFactory() + parent_doc_id = get_or_create_document_id(comment) + + File.objects.create( + parent=parent_doc_id, + file_content_type="image/png", + ) + + file_target = comment.get_file_target() + self.assertEqual(file_target.files_count["total"], 1) + self.assertEqual(file_target.files_count.get("image/png"), 1) + + def test_file_target_files_count_update_multiple_types(self): + comment = CommentFactory() + parent_doc_id = get_or_create_document_id(comment) + + File.objects.create( + parent=parent_doc_id, + file_content_type="image/png", + ) + File.objects.create( + parent=parent_doc_id, + file_content_type="image/jpeg", + ) + File.objects.create( + parent=parent_doc_id, + file_content_type="image/png", + ) + + file_target = comment.get_file_target() + self.assertEqual(file_target.files_count["total"], 3) + self.assertEqual(file_target.files_count.get("image/png"), 2) + self.assertEqual(file_target.files_count.get("image/jpeg"), 1) + + def test_file_target_is_files_enabled_can_be_disabled(self): + comment = CommentFactory() + file_target = comment.get_file_target() + + file_target.is_files_enabled = False + file_target.save() + + self.assertFalse(comment.is_files_enabled) + + def test_cannot_attach_file_when_files_disabled(self): + comment = CommentFactory() + parent_doc_id = get_or_create_document_id(comment) + + # Create FileTarget with is_files_enabled=False + file_target = comment.get_file_target() + file_target.is_files_enabled = False + file_target.save() + + # Try to create a file for this comment + with self.assertRaises(ValidationError) as cm: + File.objects.create( + parent=parent_doc_id, + file_content_type="image/png", + ) + + self.assertEqual(cm.exception.code, "files_disabled") + self.assertIn("Files are not enabled", str(cm.exception)) + + def test_can_attach_file_when_files_enabled(self): + comment = CommentFactory() + parent_doc_id = get_or_create_document_id(comment) + + # Create FileTarget with is_files_enabled=True (default) + file_target = comment.get_file_target() + self.assertTrue(file_target.is_files_enabled) + + # This should work without raising an error + file_obj = File.objects.create( + parent=parent_doc_id, + file_content_type="image/png", + ) + + self.assertIsNotNone(file_obj.pk) + self.assertEqual(file_obj.parent.object_id, comment.pk) + + def test_fileable_model_properties(self): + comment = CommentFactory() + + self.assertTrue(comment.is_files_enabled) + self.assertEqual(comment.files_count, {"total": 0}) + + def test_fileable_model_properties_multiple_accesses(self): + comment = CommentFactory() + + files_count_1 = comment.files_count + files_count_2 = comment.files_count + + self.assertEqual(files_count_1, files_count_2) + self.assertEqual(FileTarget.objects.count(), 1) + + +class FileUtilsTest(TestCase): + def test_default_files_count(self): + result = default_files_count() + self.assertEqual(result, {"total": 0}) + + def test_get_or_create_file_target_creates_new(self): + comment = CommentFactory() + + initial_count = FileTarget.objects.count() + file_target = get_or_create_file_target(comment) + + self.assertEqual(FileTarget.objects.count(), initial_count + 1) + self.assertEqual(file_target.target.object_id, comment.pk) + + def test_get_or_create_file_target_gets_existing(self): + comment = CommentFactory() + + file_target1 = get_or_create_file_target(comment) + file_target2 = get_or_create_file_target(comment) + + self.assertEqual(file_target1.pk, file_target2.pk) + self.assertEqual(FileTarget.objects.count(), 1) + + def test_get_or_create_file_target_returns_none_for_unsaved_object(self): + """An unsaved object has no pk, so the mixin guards against a NULL + primary key instead of raising IntegrityError.""" + Comment = CommentFactory._meta.model + + self.assertIsNone(get_or_create_file_target(Comment())) + self.assertIsNone(get_or_create_file_target(None)) + self.assertEqual(FileTarget.objects.count(), 0) + + def test_recalculate_files_count_with_none(self): + recalculate_files_count(None) + + def test_recalculate_files_count_empty(self): + comment = CommentFactory() + recalculate_files_count(comment) + + file_target = comment.get_file_target() + self.assertEqual(file_target.files_count, {"total": 0}) + + def test_recalculate_files_count_with_files(self): + comment = CommentFactory() + parent_doc_id = get_or_create_document_id(comment) + + File.objects.create( + parent=parent_doc_id, + file_content_type="image/png", + ) + + recalculate_files_count(comment) + + file_target = comment.get_file_target() + self.assertEqual(file_target.files_count["total"], 1) + + def test_set_files_parent_with_empty_list(self): + comment = CommentFactory() + set_files_parent(comment, []) + + def test_set_files_parent_with_none(self): + comment = CommentFactory() + set_files_parent(comment, None) + + def test_set_files_parent_moves_file(self): + comment1 = CommentFactory() + comment2 = CommentFactory() + parent_doc_id1 = get_or_create_document_id(comment1) + + file_obj = File.objects.create( + parent=parent_doc_id1, + file_content_type="image/png", + ) + + set_files_parent(comment2, [file_obj]) + + file_obj.refresh_from_db() + self.assertEqual(file_obj.parent.object_id, comment2.pk) + + file_target1 = comment1.get_file_target() + file_target2 = comment2.get_file_target() + self.assertEqual(file_target1.files_count["total"], 0) + self.assertEqual(file_target2.files_count["total"], 1) + + def test_set_files_parent_same_parent(self): + comment = CommentFactory() + parent_doc_id = get_or_create_document_id(comment) + + file_obj = File.objects.create(parent=parent_doc_id) + + set_files_parent(comment, [file_obj]) + + file_obj.refresh_from_db() + self.assertEqual(file_obj.parent.object_id, comment.pk) + + +class FileTargetSignalsTest(TestCase): + def test_file_creation_updates_file_target(self): + comment = CommentFactory() + comment_content_type = ContentType.objects.get_for_model(Comment) + parent_doc_id = get_or_create_document_id(comment) + + File.objects.create( + parent=parent_doc_id, + file_content_type="image/png", + ) + + file_target = FileTarget.objects.get( + target__content_type=comment_content_type, + target__object_id=comment.pk, + ) + self.assertEqual(file_target.files_count["total"], 1) + + def test_file_deletion_updates_file_target(self): + comment = CommentFactory() + comment_content_type = ContentType.objects.get_for_model(Comment) + parent_doc_id = get_or_create_document_id(comment) + + file_obj = File.objects.create( + parent=parent_doc_id, + file_content_type="image/png", + ) + + file_obj.delete() + + file_target = FileTarget.objects.get( + target__content_type=comment_content_type, + target__object_id=comment.pk, + ) + self.assertEqual(file_target.files_count["total"], 0) diff --git a/baseapp/files/tests/test_presigned_uploads.py b/baseapp/files/tests/test_presigned_uploads.py new file mode 100644 index 00000000..9642eebe --- /dev/null +++ b/baseapp/files/tests/test_presigned_uploads.py @@ -0,0 +1,211 @@ +"""Tests for the presigned upload-part endpoint backed by local storage. + +Exercises the full local upload flow: the initiate REST endpoint (which uses +LocalUploadHandler in tests since default storage is FileSystemStorage) +returns presigned URLs containing signed tokens, and the token-authenticated +PUT endpoint stores the part data. +""" + +from datetime import timedelta +from typing import Any, Dict +from unittest.mock import patch +from urllib.parse import urlparse + +import pytest +import swapper +from django.conf import settings +from django.utils import timezone +from freezegun import freeze_time +from rest_framework import status + +from baseapp.files.tokens import PresignedUploadToken + +File = swapper.load_model("baseapp_files", "File") + +pytestmark = pytest.mark.django_db + +PART_DATA = b"hello world" + + +@pytest.fixture +def temp_media_root(tmp_path): + """Use a temporary directory for media root.""" + with patch.object(settings, "MEDIA_ROOT", str(tmp_path)): + yield tmp_path + + +@pytest.fixture +def initiated_upload(user_client, temp_media_root) -> Dict[str, Any]: + """Initiate a single-part upload through the REST endpoint with local storage.""" + response = user_client.post( + "/v1/files/uploads", + { + "file_name": "doc.txt", + "file_size": len(PART_DATA), + "file_content_type": "text/plain", + "num_parts": 1, + "part_size": len(PART_DATA), + }, + format="json", + ) + assert response.status_code == status.HTTP_201_CREATED + return response.json() + + +def presigned_path(presigned_url: Dict[str, Any]) -> str: + """Extract the relative path (with token query string) from a presigned URL.""" + parsed = urlparse(presigned_url["url"]) + return f"{parsed.path}?{parsed.query}" + + +def put_part(client, path: str, data: bytes = PART_DATA): + """PUT raw binary data to a presigned upload-part path.""" + return client.put(path, data=data, content_type="application/octet-stream") + + +class TestPresignedUploadPart: + """Tests for PUT /v1/files/presigned-uploads/{id}/upload-part/{part}?token=...""" + + def test_upload_part_success(self, client, temp_media_root, initiated_upload): + """A valid token uploads the part and returns the etag in body and header.""" + path = presigned_path(initiated_upload["presigned_urls"][0]) + + response = put_part(client, path) + + assert response.status_code == status.HTTP_200_OK + data = response.json() + assert data["part_number"] == 1 + assert len(data["etag"]) == 32 # MD5 hex digest + assert response["ETag"] == data["etag"] + + part_file = temp_media_root / "temp_uploads" / initiated_upload["upload_id"] / "part_1" + assert part_file.read_bytes() == PART_DATA + + def test_full_upload_flow(self, client, user_client, temp_media_root, initiated_upload): + """Upload a part via presigned URL, then complete the upload.""" + path = presigned_path(initiated_upload["presigned_urls"][0]) + etag = put_part(client, path).json()["etag"] + + response = user_client.post( + f"/v1/files/uploads/{initiated_upload['id']}/complete", + {"parts": [{"part_number": 1, "etag": etag}]}, + format="json", + ) + + assert response.status_code == status.HTTP_200_OK + assert response.json()["upload_status"] == File.UploadStatus.COMPLETED + + file_obj = File.get_by_public_id(initiated_upload["id"]) + assert file_obj.upload_status == File.UploadStatus.COMPLETED + assert (temp_media_root / file_obj.file.name).read_bytes() == PART_DATA + + def test_missing_token(self, client, initiated_upload): + """A request without a token is rejected.""" + path = presigned_path(initiated_upload["presigned_urls"][0]).split("?")[0] + + response = put_part(client, path) + + assert response.status_code == status.HTTP_401_UNAUTHORIZED + assert "Missing token" in response.json()["error"] + + def test_tampered_token(self, client, initiated_upload): + """A token with an invalid signature is rejected.""" + path = presigned_path(initiated_upload["presigned_urls"][0]) + + response = put_part(client, path + "tampered") + + assert response.status_code == status.HTTP_401_UNAUTHORIZED + assert "Invalid token signature" in response.json()["error"] + + def test_token_for_wrong_part(self, client, initiated_upload): + """A valid token cannot be reused for a different part number.""" + parsed = urlparse(initiated_upload["presigned_urls"][0]["url"]) + wrong_part_path = parsed.path.replace("/upload-part/1", "/upload-part/2") + + response = put_part(client, f"{wrong_part_path}?{parsed.query}") + + assert response.status_code == status.HTTP_401_UNAUTHORIZED + assert "Invalid token for this file/part" in response.json()["error"] + + def test_expired_token(self, client, initiated_upload): + """A token older than 1 hour is rejected.""" + file_obj = File.get_by_public_id(initiated_upload["id"]) + + with freeze_time(timezone.now() - timedelta(hours=2)): + token = PresignedUploadToken.mint( + file_id=file_obj.id, + part_number=1, + upload_id=file_obj.upload_id, + ) + + response = put_part( + client, f"/v1/files/presigned-uploads/{file_obj.id}/upload-part/1?token={token}" + ) + + assert response.status_code == status.HTTP_401_UNAUTHORIZED + assert "Token has expired" in response.json()["error"] + + def test_upload_part_invalid_status(self, client, initiated_upload): + """Parts cannot be uploaded once the file is no longer pending/uploading.""" + file_obj = File.get_by_public_id(initiated_upload["id"]) + file_obj.upload_status = File.UploadStatus.COMPLETED + file_obj.save(update_fields=["upload_status"]) + + path = presigned_path(initiated_upload["presigned_urls"][0]) + response = put_part(client, path) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert "Cannot upload parts" in response.json()["error"] + + def test_upload_part_number_out_of_range(self, client, initiated_upload): + """A part number above total_parts is rejected even with a valid token.""" + file_obj = File.get_by_public_id(initiated_upload["id"]) + token = PresignedUploadToken.mint( + file_id=file_obj.id, + part_number=5, + upload_id=file_obj.upload_id, + ) + + response = put_part( + client, f"/v1/files/presigned-uploads/{file_obj.id}/upload-part/5?token={token}" + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert "Invalid part number" in response.json()["error"] + + def test_token_from_previous_upload_session(self, client, initiated_upload): + """A token minted for an earlier initiation cannot upload into a new session.""" + file_obj = File.get_by_public_id(initiated_upload["id"]) + token = PresignedUploadToken.mint( + file_id=file_obj.id, + part_number=1, + upload_id="stale-upload-id", + ) + + response = put_part( + client, f"/v1/files/presigned-uploads/{file_obj.id}/upload-part/1?token={token}" + ) + + assert response.status_code == status.HTTP_401_UNAUTHORIZED + assert "does not match the current upload session" in response.json()["error"] + + def test_token_for_other_file(self, client, user_client, temp_media_root, initiated_upload): + """A token signed for one file cannot be used on another file's URL.""" + other_file = File.objects.create( + file_name="other.txt", + file_size=len(PART_DATA), + file_content_type="text/plain", + upload_status=File.UploadStatus.UPLOADING, + upload_id="other-upload-id", + total_parts=1, + created_by=user_client.user, + ) + + parsed = urlparse(initiated_upload["presigned_urls"][0]["url"]) + response = put_part( + client, + f"/v1/files/presigned-uploads/{other_file.id}/upload-part/1?{parsed.query}", + ) + + assert response.status_code == status.HTTP_401_UNAUTHORIZED + assert "Invalid token for this file/part" in response.json()["error"] diff --git a/baseapp/files/tests/test_rest_api.py b/baseapp/files/tests/test_rest_api.py new file mode 100644 index 00000000..2d1fb6a2 --- /dev/null +++ b/baseapp/files/tests/test_rest_api.py @@ -0,0 +1,618 @@ +from datetime import timedelta +from unittest.mock import MagicMock, patch + +import pytest +import swapper +from django.contrib.auth import get_user_model +from django.contrib.contenttypes.models import ContentType +from django.utils import timezone +from rest_framework import status + +from baseapp_comments.tests.factories import CommentFactory +from baseapp_core.models import DocumentId + +File = swapper.load_model("baseapp_files", "File") +User = get_user_model() + + +@pytest.fixture +def mock_s3_handler(): + """Mock S3 upload handler to avoid actual S3 calls.""" + with patch("baseapp.files.services.upload_service.get_upload_handler") as mock_get_handler: + mock_handler = MagicMock() + mock_handler.supports_multipart.return_value = True + mock_handler.initiate_upload.return_value = { + "upload_id": "test-upload-id-123", + "presigned_urls": [ + {"part_number": 1, "url": "https://s3.amazonaws.com/bucket/part1?signature=abc"}, + {"part_number": 2, "url": "https://s3.amazonaws.com/bucket/part2?signature=def"}, + ], + "expires_in": 3600, + } + mock_handler.complete_upload.return_value = "files/test-file.mp4" + mock_handler.abort_upload.return_value = None + mock_get_handler.return_value = mock_handler + yield mock_handler + + +@pytest.mark.django_db +class TestFileUploadInitiation: + """Tests for POST /v1/files/uploads/ (initiate upload).""" + + def test_initiate_upload_success(self, user_client, mock_s3_handler): + """Test successful upload initiation.""" + response = user_client.post( + "/v1/files/uploads", + { + "file_name": "test-video.mp4", + "file_size": 10485760, # 10MB + "file_content_type": "video/mp4", + "num_parts": 2, + "part_size": 5242880, # 5MB + }, + format="json", + ) + + assert response.status_code == status.HTTP_201_CREATED + data = response.json() + + # Check response structure + assert "id" in data + assert data["upload_id"] == "test-upload-id-123" + assert len(data["presigned_urls"]) == 2 + assert data["presigned_urls"][0]["part_number"] == 1 + assert data["expires_in"] == 3600 + assert data["upload_status"] == File.UploadStatus.UPLOADING + + # Check file was created in database (id is now public_id UUID) + file_obj = File.get_by_public_id(data["id"]) + assert file_obj.file_name == "test-video.mp4" + assert file_obj.file_size == 10485760 + assert file_obj.upload_status == File.UploadStatus.UPLOADING + assert file_obj.total_parts == 2 + assert file_obj.created_by == user_client.user + + # Verify S3 handler was called + mock_s3_handler.initiate_upload.assert_called_once() + + def test_initiate_upload_with_parent(self, user_client, mock_s3_handler): + """Test upload initiation with a files-enabled parent object.""" + comment = CommentFactory(user=user_client.user) + comment_ct = ContentType.objects.get_for_model(comment) + document_id, _ = DocumentId.objects.get_or_create( + content_type=comment_ct, + object_id=comment.id, + ) + + response = user_client.post( + "/v1/files/uploads", + { + "file_name": "avatar.jpg", + "file_size": 1048576, # 1MB + "file_content_type": "image/jpeg", + "num_parts": 1, + "part_size": 1048576, + "parent_id": str(document_id.public_id), + }, + format="json", + ) + + assert response.status_code == status.HTTP_201_CREATED + data = response.json() + + file_obj = File.get_by_public_id(data["id"]) + # Parent is now a ForeignKey to DocumentId + assert file_obj.parent is not None + assert file_obj.parent.content_type_id == comment_ct.id + assert file_obj.parent.object_id == comment.id + + def test_initiate_upload_rejects_non_fileable_parent(self, user_client, mock_s3_handler): + """A parent that does not support files (not a FileableModel) is rejected, + mirroring the fileAttachToTarget mutation.""" + user_ct = ContentType.objects.get_for_model(User) + document_id, _ = DocumentId.objects.get_or_create( + content_type=user_ct, + object_id=user_client.user.id, + ) + + response = user_client.post( + "/v1/files/uploads", + { + "file_name": "avatar.jpg", + "file_size": 1048576, + "file_content_type": "image/jpeg", + "num_parts": 1, + "part_size": 1048576, + "parent_id": str(document_id.public_id), + }, + format="json", + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + def test_initiate_upload_requires_authentication(self, client): + """Test that upload initiation requires authentication.""" + response = client.post( + "/v1/files/uploads", + { + "file_name": "test.mp4", + "file_size": 1048576, + "file_content_type": "video/mp4", + "num_parts": 1, + "part_size": 1048576, + }, + format="json", + ) + + assert response.status_code == status.HTTP_401_UNAUTHORIZED + + def test_initiate_upload_invalid_part_size(self, user_client, mock_s3_handler): + """Test validation for part size (must be at least 5MB for multipart).""" + response = user_client.post( + "/v1/files/uploads", + { + "file_name": "test.mp4", + "file_size": 10485760, + "file_content_type": "video/mp4", + "num_parts": 3, + "part_size": 1048576, # Too small (1MB) + }, + format="json", + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + def test_initiate_upload_file_size_mismatch(self, user_client, mock_s3_handler): + """Test validation for file size matching parts.""" + response = user_client.post( + "/v1/files/uploads", + { + "file_name": "test.mp4", + "file_size": 100000, # 100KB - doesn't match parts + "file_content_type": "video/mp4", + "num_parts": 2, + "part_size": 5242880, # 5MB + }, + format="json", + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + response_data = response.json() + # DRF returns validation errors as dict with field names or non_field_errors + error_message = str(response_data) + assert "doesn't match" in error_message + + def test_initiate_upload_too_many_parts(self, user_client, mock_s3_handler): + """Test validation for maximum parts (10,000).""" + response = user_client.post( + "/v1/files/uploads", + { + "file_name": "test.mp4", + "file_size": 52428800000, # 50GB + "file_content_type": "video/mp4", + "num_parts": 10001, # Too many + "part_size": 5242880, + }, + format="json", + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + def test_initiate_upload_invalid_parent_id(self, user_client, mock_s3_handler): + """Test that invalid parent_id returns error.""" + import uuid + + response = user_client.post( + "/v1/files/uploads", + { + "file_name": "test.mp4", + "file_size": 1048576, + "file_content_type": "video/mp4", + "num_parts": 1, + "part_size": 1048576, + "parent_id": str(uuid.uuid4()), # Non-existent DocumentId + }, + format="json", + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + +@pytest.mark.django_db +class TestFileUploadCompletion: + """Tests for POST /v1/files/uploads/{id}/complete.""" + + @pytest.fixture + def pending_upload(self, user_client): + """Create a pending upload for testing.""" + return File.objects.create( + file_name="test.mp4", + file_size=10485760, + file_content_type="video/mp4", + upload_status=File.UploadStatus.UPLOADING, + upload_id="test-upload-id", + total_parts=2, + created_by=user_client.user, + upload_expires_at=timezone.now() + timedelta(hours=24), + ) + + def test_complete_upload_success(self, user_client, pending_upload, mock_s3_handler): + """Test successful upload completion.""" + response = user_client.post( + f"/v1/files/uploads/{pending_upload.public_id}/complete", + { + "parts": [ + {"part_number": 1, "etag": "abc123"}, + {"part_number": 2, "etag": "def456"}, + ] + }, + format="json", + ) + + assert response.status_code == status.HTTP_200_OK + data = response.json() + + assert data["upload_status"] == File.UploadStatus.COMPLETED + assert "url" in data + + # Check file was updated in database + pending_upload.refresh_from_db() + assert pending_upload.upload_status == File.UploadStatus.COMPLETED + assert pending_upload.upload_id == "" + assert pending_upload.uploaded_parts == {"1": "abc123", "2": "def456"} + + # Verify S3 handler was called + mock_s3_handler.complete_upload.assert_called_once() + + def test_complete_upload_wrong_number_of_parts( + self, user_client, pending_upload, mock_s3_handler + ): + """Test validation for incorrect number of parts.""" + response = user_client.post( + f"/v1/files/uploads/{pending_upload.public_id}/complete", + { + "parts": [ + {"part_number": 1, "etag": "abc123"}, + # Missing part 2 + ] + }, + format="json", + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + # The specific reason is logged server-side; the client gets a generic + # message so internal exception details are not exposed. + assert "Invalid or incomplete upload parts" in str(response.json()) + + def test_complete_upload_missing_part_number( + self, user_client, pending_upload, mock_s3_handler + ): + """Test validation for missing part numbers.""" + response = user_client.post( + f"/v1/files/uploads/{pending_upload.public_id}/complete", + { + "parts": [ + {"part_number": 1, "etag": "abc123"}, + {"part_number": 3, "etag": "ghi789"}, # Skipped part 2 + ] + }, + format="json", + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert "Invalid or incomplete upload parts" in str(response.json()) + + def test_complete_upload_requires_ownership(self, client, pending_upload, mock_s3_handler): + """Test that only the file owner can complete upload.""" + other_user = User.objects.create_user(email="other@example.com", password="pass123") + client.force_authenticate(user=other_user) + + response = client.post( + f"/v1/files/uploads/{pending_upload.public_id}/complete", + { + "parts": [ + {"part_number": 1, "etag": "abc123"}, + {"part_number": 2, "etag": "def456"}, + ] + }, + format="json", + ) + + assert response.status_code == status.HTTP_403_FORBIDDEN + + def test_complete_upload_invalid_status(self, user_client, mock_s3_handler): + """Test that completed uploads cannot be completed again.""" + completed_file = File.objects.create( + file_name="test.mp4", + file_size=10485760, + file_content_type="video/mp4", + upload_status=File.UploadStatus.COMPLETED, + created_by=user_client.user, + ) + + response = user_client.post( + f"/v1/files/uploads/{completed_file.public_id}/complete", + { + "parts": [ + {"part_number": 1, "etag": "abc123"}, + ] + }, + format="json", + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + def test_complete_upload_invalid_part_structure( + self, user_client, pending_upload, mock_s3_handler + ): + """Test validation for part structure.""" + response = user_client.post( + f"/v1/files/uploads/{pending_upload.public_id}/complete", + { + "parts": [ + {"part_number": 1}, # Missing etag + {"etag": "def456"}, # Missing part_number + ] + }, + format="json", + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + +@pytest.mark.django_db +class TestFileUploadAbort: + """Tests for DELETE /v1/files/uploads/{id}.""" + + @pytest.fixture + def uploading_file(self, user_client): + """Create an uploading file for testing.""" + return File.objects.create( + file_name="test.mp4", + file_size=10485760, + file_content_type="video/mp4", + upload_status=File.UploadStatus.UPLOADING, + upload_id="test-upload-id", + total_parts=2, + created_by=user_client.user, + ) + + def test_abort_upload_success(self, user_client, uploading_file, mock_s3_handler): + """Test successful upload abort.""" + response = user_client.delete(f"/v1/files/uploads/{uploading_file.public_id}") + + assert response.status_code == status.HTTP_204_NO_CONTENT + + # Check file was updated + uploading_file.refresh_from_db() + assert uploading_file.upload_status == File.UploadStatus.ABORTED + assert uploading_file.upload_id == "" + + # Verify S3 handler was called + mock_s3_handler.abort_upload.assert_called_once_with(uploading_file, "test-upload-id") + + def test_abort_upload_requires_ownership(self, client, uploading_file, mock_s3_handler): + """Test that only file owner can abort upload.""" + other_user = User.objects.create_user(email="other@example.com", password="pass123") + client.force_authenticate(user=other_user) + + response = client.delete(f"/v1/files/uploads/{uploading_file.public_id}") + + assert response.status_code == status.HTTP_403_FORBIDDEN + + def test_abort_upload_without_upload_id(self, user_client, mock_s3_handler): + """Test aborting upload without upload_id still marks as aborted.""" + file_obj = File.objects.create( + file_name="test.mp4", + file_size=10485760, + file_content_type="video/mp4", + upload_status=File.UploadStatus.PENDING, + upload_id="", # No upload_id + total_parts=2, + created_by=user_client.user, + ) + + response = user_client.delete(f"/v1/files/uploads/{file_obj.public_id}") + + assert response.status_code == status.HTTP_204_NO_CONTENT + + file_obj.refresh_from_db() + assert file_obj.upload_status == File.UploadStatus.ABORTED + + +@pytest.mark.django_db +class TestFileCRUD: + """Tests for file CRUD endpoints.""" + + @pytest.fixture + def completed_file(self, user_client): + """Create a completed file for testing.""" + return File.objects.create( + file_name="test.mp4", + file_size=10485760, + file_content_type="video/mp4", + upload_status=File.UploadStatus.COMPLETED, + created_by=user_client.user, + ) + + def test_list_files(self, user_client, completed_file): + """Test listing files.""" + response = user_client.get("/v1/files") + + assert response.status_code == status.HTTP_200_OK + data = response.json() + + assert "results" in data + assert len(data["results"]) == 1 + assert str(data["results"][0]["id"]) == str(completed_file.public_id) + + def test_list_files_filter_by_status(self, user_client): + """Test filtering files by status.""" + File.objects.create( + file_name="completed.mp4", + file_size=100, + file_content_type="video/mp4", + upload_status=File.UploadStatus.COMPLETED, + created_by=user_client.user, + ) + File.objects.create( + file_name="uploading.mp4", + file_size=100, + file_content_type="video/mp4", + upload_status=File.UploadStatus.UPLOADING, + created_by=user_client.user, + ) + + # Default: only completed + response = user_client.get("/v1/files") + assert len(response.json()["results"]) == 1 + + # Filter by uploading + response = user_client.get(f"/v1/files?status={File.UploadStatus.UPLOADING}") + assert len(response.json()["results"]) == 1 + assert response.json()["results"][0]["upload_status"] == File.UploadStatus.UPLOADING + + def test_list_files_only_own_files(self, user_client): + """Test that users only see their own files.""" + other_user = User.objects.create_user(email="other@example.com", password="pass123") + + # Create file for other user + File.objects.create( + file_name="other.mp4", + file_size=100, + file_content_type="video/mp4", + upload_status=File.UploadStatus.COMPLETED, + created_by=other_user, + ) + + response = user_client.get("/v1/files") + + assert response.status_code == status.HTTP_200_OK + assert len(response.json()["results"]) == 0 + + def test_retrieve_file(self, user_client, completed_file): + """Test retrieving a single file.""" + response = user_client.get(f"/v1/files/{completed_file.public_id}") + + assert response.status_code == status.HTTP_200_OK + data = response.json() + + assert str(data["id"]) == str(completed_file.public_id) + assert data["file_name"] == "test.mp4" + assert data["upload_status"] == File.UploadStatus.COMPLETED + + def test_update_file_metadata(self, user_client, completed_file): + """Test updating file metadata.""" + response = user_client.patch( + f"/v1/files/{completed_file.public_id}", + {"name": "My Video", "description": "Test description"}, + format="json", + ) + + assert response.status_code == status.HTTP_200_OK + + completed_file.refresh_from_db() + assert completed_file.name == "My Video" + assert completed_file.description == "Test description" + + def test_delete_file(self, user_client, completed_file): + """Test deleting a file.""" + response = user_client.delete(f"/v1/files/{completed_file.public_id}") + + assert response.status_code == status.HTTP_204_NO_CONTENT + assert not File.objects.filter(id=completed_file.id).exists() + + +@pytest.mark.django_db +class TestSetParent: + """Tests for POST /v1/files/{id}/set-parent.""" + + @pytest.fixture + def standalone_file(self, user_client): + """Create a standalone file without parent.""" + return File.objects.create( + file_name="test.mp4", + file_size=10485760, + file_content_type="video/mp4", + upload_status=File.UploadStatus.COMPLETED, + created_by=user_client.user, + ) + + def test_set_parent_success(self, user_client, standalone_file): + """Test setting a files-enabled parent on a standalone file.""" + comment = CommentFactory(user=user_client.user) + comment_ct = ContentType.objects.get_for_model(comment) + document_id, _ = DocumentId.objects.get_or_create( + content_type=comment_ct, + object_id=comment.id, + ) + + response = user_client.post( + f"/v1/files/{standalone_file.public_id}/set-parent", + { + "parent_id": str(document_id.public_id), + }, + format="json", + ) + + assert response.status_code == status.HTTP_200_OK + + standalone_file.refresh_from_db() + # Parent is now a ForeignKey to DocumentId + assert standalone_file.parent is not None + assert standalone_file.parent.content_type_id == comment_ct.id + assert standalone_file.parent.object_id == comment.id + + def test_set_parent_rejects_non_fileable_target(self, user_client, standalone_file): + """set-parent enforces the same target authorization as the mutation.""" + user_ct = ContentType.objects.get_for_model(User) + document_id, _ = DocumentId.objects.get_or_create( + content_type=user_ct, + object_id=user_client.user.id, + ) + + response = user_client.post( + f"/v1/files/{standalone_file.public_id}/set-parent", + { + "parent_id": str(document_id.public_id), + }, + format="json", + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + def test_set_parent_invalid_parent_id(self, user_client, standalone_file): + """Test setting invalid parent_id.""" + import uuid + + response = user_client.post( + f"/v1/files/{standalone_file.public_id}/set-parent", + { + "parent_id": str(uuid.uuid4()), # Non-existent DocumentId + }, + format="json", + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + def test_set_parent_requires_ownership(self, client, standalone_file): + """Test that only file owner can set parent.""" + other_user = User.objects.create_user(email="other@example.com", password="pass123") + client.force_authenticate(user=other_user) + + user_ct = ContentType.objects.get_for_model(User) + document_id, _ = DocumentId.objects.get_or_create( + content_type=user_ct, + object_id=other_user.id, + ) + + response = client.post( + f"/v1/files/{standalone_file.public_id}/set-parent", + { + "parent_id": str(document_id.public_id), + }, + format="json", + ) + + # Returns 403 because permission check fails (user doesn't own the file) + assert response.status_code == status.HTTP_403_FORBIDDEN diff --git a/baseapp/files/tests/test_storage.py b/baseapp/files/tests/test_storage.py new file mode 100644 index 00000000..4def9892 --- /dev/null +++ b/baseapp/files/tests/test_storage.py @@ -0,0 +1,280 @@ +from unittest.mock import MagicMock, patch + +import pytest +import swapper +from botocore.exceptions import ClientError +from django.conf import settings +from django.contrib.auth import get_user_model + +from baseapp.files.storage.base import BaseUploadHandler +from baseapp.files.storage.local import LocalUploadHandler +from baseapp.files.storage.s3 import S3MultipartUploadHandler + +File = swapper.load_model("baseapp_files", "File") +User = get_user_model() + + +@pytest.fixture +def user(db): + return User.objects.create_user( + email="test@example.com", + password="testpass123", + ) + + +@pytest.fixture +def file_obj(user): + """Create a file object for testing.""" + return File.objects.create( + file_name="test.mp4", + file_size=10485760, + file_content_type="video/mp4", + upload_status=File.UploadStatus.PENDING, + total_parts=2, + created_by=user, + ) + + +@pytest.mark.django_db +class TestBaseUploadHandler: + """Tests for the base upload handler interface.""" + + def test_base_handler_is_abstract(self): + """Test that base handler cannot be instantiated.""" + with pytest.raises(TypeError): + BaseUploadHandler() + + +@pytest.mark.django_db +@pytest.mark.skipif( + not hasattr(settings, "AWS_ACCESS_KEY_ID"), + reason="AWS settings not configured - skipping S3 tests", +) +class TestS3MultipartUploadHandler: + """Tests for S3 multipart upload handler.""" + + @pytest.fixture + def mock_s3_client(self): + """Mock boto3 S3 client.""" + with patch("baseapp.files.storage.s3.boto3.client") as mock_client: + s3 = MagicMock() + mock_client.return_value = s3 + + # Mock create_multipart_upload + s3.create_multipart_upload.return_value = {"UploadId": "test-upload-id-123"} + + # Mock generate_presigned_url + s3.generate_presigned_url.return_value = ( + "https://bucket.s3.amazonaws.com/test?signature=abc" + ) + + # Mock complete_multipart_upload + s3.complete_multipart_upload.return_value = { + "Location": "https://bucket.s3.amazonaws.com/files/test.mp4" + } + + yield s3 + + @pytest.fixture + def s3_handler(self, mock_s3_client): + """Create S3 handler with mocked client.""" + handler = S3MultipartUploadHandler() + handler.s3_client = mock_s3_client + yield handler + + def test_s3_handler_supports_multipart(self, s3_handler): + """Test that S3 handler supports multipart.""" + assert s3_handler.supports_multipart() is True + + def test_s3_initiate_upload(self, s3_handler, file_obj, mock_s3_client): + """Test S3 multipart upload initiation.""" + result = s3_handler.initiate_upload(file_obj, num_parts=2, part_size=5242880) + + assert result["upload_id"] == "test-upload-id-123" + assert len(result["presigned_urls"]) == 2 + assert result["presigned_urls"][0]["part_number"] == 1 + assert result["presigned_urls"][1]["part_number"] == 2 + assert result["expires_in"] == 3600 + + # Verify S3 calls + mock_s3_client.create_multipart_upload.assert_called_once() + assert mock_s3_client.generate_presigned_url.call_count == 2 + + def test_s3_initiate_upload_validates_part_count(self, s3_handler, file_obj): + """Test that S3 handler validates max part count.""" + with pytest.raises(ValueError, match="maximum 10,000 parts"): + s3_handler.initiate_upload(file_obj, num_parts=10001, part_size=5242880) + + def test_s3_initiate_upload_validates_part_size(self, s3_handler, file_obj): + """Test that S3 handler validates minimum part size.""" + with pytest.raises(ValueError, match="minimum 5MB"): + s3_handler.initiate_upload(file_obj, num_parts=2, part_size=1048576) + + def test_s3_complete_upload(self, s3_handler, file_obj, mock_s3_client): + """Test S3 multipart upload completion.""" + parts = [ + {"part_number": 1, "etag": "abc123"}, + {"part_number": 2, "etag": "def456"}, + ] + + result = s3_handler.complete_upload(file_obj, "test-upload-id", parts) + + assert result # Returns the S3 key + + # Verify S3 call + mock_s3_client.complete_multipart_upload.assert_called_once() + call_args = mock_s3_client.complete_multipart_upload.call_args + assert call_args.kwargs["UploadId"] == "test-upload-id" + assert len(call_args.kwargs["MultipartUpload"]["Parts"]) == 2 + + def test_s3_abort_upload(self, s3_handler, file_obj, mock_s3_client): + """Test S3 multipart upload abort.""" + s3_handler.abort_upload(file_obj, "test-upload-id") + + mock_s3_client.abort_multipart_upload.assert_called_once() + + def test_s3_abort_upload_handles_no_such_upload(self, s3_handler, file_obj, mock_s3_client): + """Test that abort handles NoSuchUpload error gracefully.""" + mock_s3_client.abort_multipart_upload.side_effect = ClientError( + {"Error": {"Code": "NoSuchUpload"}}, "abort_multipart_upload" + ) + + # Should not raise exception + s3_handler.abort_upload(file_obj, "test-upload-id") + + def test_s3_abort_upload_raises_other_errors(self, s3_handler, file_obj, mock_s3_client): + """Test that abort raises other errors.""" + mock_s3_client.abort_multipart_upload.side_effect = ClientError( + {"Error": {"Code": "AccessDenied"}}, "abort_multipart_upload" + ) + + with pytest.raises(ClientError): + s3_handler.abort_upload(file_obj, "test-upload-id") + + +@pytest.mark.django_db +class TestLocalUploadHandler: + """Tests for local upload handler.""" + + @pytest.fixture + def local_handler(self): + """Create local handler.""" + return LocalUploadHandler() + + @pytest.fixture + def temp_media_root(self, tmp_path): + """Use temporary directory for media root.""" + with patch.object(settings, "MEDIA_ROOT", str(tmp_path)): + yield tmp_path + + def test_local_handler_does_not_support_true_multipart(self, local_handler): + """Test that local handler doesn't support true multipart.""" + assert local_handler.supports_multipart() is False + + def test_local_initiate_upload(self, local_handler, file_obj, temp_media_root): + """Test local upload initiation.""" + result = local_handler.initiate_upload(file_obj, num_parts=2, part_size=5242880) + + assert "upload_id" in result + assert len(result["presigned_urls"]) == 2 + assert result["presigned_urls"][0]["part_number"] == 1 + assert "url" in result["presigned_urls"][0] + + # Check temp directory was created + temp_dir = temp_media_root / "temp_uploads" / result["upload_id"] + assert temp_dir.exists() + + def test_local_upload_part(self, local_handler, file_obj, temp_media_root): + """Test uploading a part to local storage.""" + upload_id = "test-upload-123" + data = b"test file content" + + etag = local_handler.upload_part(upload_id, part_number=1, data=data) + + assert etag # Should return MD5 hash + assert len(etag) == 32 # MD5 hash length + + # Check part file was created + part_file = temp_media_root / "temp_uploads" / upload_id / "part_1" + assert part_file.exists() + assert part_file.read_bytes() == data + + def test_local_complete_upload(self, local_handler, file_obj, temp_media_root): + """Test completing local upload.""" + upload_id = "test-upload-123" + + # Create temp parts + temp_dir = temp_media_root / "temp_uploads" / upload_id + temp_dir.mkdir(parents=True) + + (temp_dir / "part_1").write_bytes(b"part 1 content") + (temp_dir / "part_2").write_bytes(b"part 2 content") + + parts = [ + {"part_number": 1, "etag": "abc"}, + {"part_number": 2, "etag": "def"}, + ] + + result = local_handler.complete_upload(file_obj, upload_id, parts) + + assert result # Returns file path + + # Check final file exists + final_file = temp_media_root / result + assert final_file.exists() + assert final_file.read_bytes() == b"part 1 contentpart 2 content" + + # Check temp directory was cleaned up + assert not temp_dir.exists() + + def test_local_abort_upload(self, local_handler, file_obj, temp_media_root): + """Test aborting local upload.""" + upload_id = "test-upload-123" + + # Create temp directory + temp_dir = temp_media_root / "temp_uploads" / upload_id + temp_dir.mkdir(parents=True) + (temp_dir / "part_1").write_bytes(b"test") + + # Abort + local_handler.abort_upload(file_obj, upload_id) + + # Check temp directory was cleaned up + assert not temp_dir.exists() + + +@pytest.mark.django_db +class TestStorageFactory: + """Tests for storage handler factory.""" + + @pytest.mark.skipif( + not hasattr(settings, "AWS_ACCESS_KEY_ID"), + reason="AWS settings not configured - skipping S3 factory test", + ) + def test_factory_returns_s3_handler_for_s3_storage(self): + """Test that factory returns S3 handler when using S3 storage.""" + with patch("baseapp.files.storage.default_storage") as mock_storage: + # Mock S3 storage + mock_storage.__class__.__name__ = "S3Boto3Storage" + + with patch("baseapp.files.storage.s3.S3MultipartUploadHandler") as mock_s3: + from baseapp.files.storage import get_upload_handler + + get_upload_handler() + + # Should instantiate S3 handler + mock_s3.assert_called_once() + + def test_factory_returns_local_handler_for_file_system_storage(self): + """Test that factory returns local handler for non-S3 storage.""" + with patch("baseapp.files.storage.default_storage") as mock_storage: + # Mock file system storage + mock_storage.__class__.__name__ = "FileSystemStorage" + + with patch("baseapp.files.storage.local.LocalUploadHandler") as mock_local: + from baseapp.files.storage import get_upload_handler + + get_upload_handler() + + # Should instantiate local handler + mock_local.assert_called_once() diff --git a/baseapp/files/tokens.py b/baseapp/files/tokens.py new file mode 100644 index 00000000..7545e5f5 --- /dev/null +++ b/baseapp/files/tokens.py @@ -0,0 +1,93 @@ +from typing import Any, Dict + +from django.conf import settings +from django.core import signing +from django.utils.translation import gettext_lazy as _ + + +class PresignedUploadTokenError(Exception): + """ + Raised when a presigned upload token is missing, expired, or does not match + the request it was presented with. + + `message` is a client-safe, translated string: it describes only which check + failed, never any detail of the signed payload. + """ + + def __init__(self, message: str) -> None: + self.message = message + super().__init__(message) + + +class PresignedUploadToken: + """ + Mints and verifies the signed tokens that authenticate part uploads against + the local-storage presigned fallback. + + `LocalUploadHandler` embeds a token in each presigned URL it hands out, and + `PresignedUploadViewSet` verifies it — that endpoint has no session auth or + CSRF, so the token is the only thing authorizing the write. The S3 handler + issues real presigned URLs and does not use this. + """ + + SALT = "baseapp_files.presigned_upload" + + @classmethod + def max_age(cls) -> int: + """Token lifetime in seconds; shared with the S3 handler's URL expiration.""" + return getattr(settings, "FILE_UPLOAD_PRESIGNED_URL_EXPIRATION", 3600) + + @classmethod + def mint(cls, *, file_id: Any, part_number: int, upload_id: str) -> str: + """Sign a token authorizing one part of one upload session.""" + return signing.dumps( + { + "file_id": file_id, + "part_number": part_number, + "upload_id": upload_id, + }, + salt=cls.SALT, + ) + + @classmethod + def verify(cls, token: str | None, *, file_id: Any, part_number: Any) -> Dict[str, Any]: + """ + Validate `token` against the file and part it is being used for and + return its payload. + + The payload's `upload_id` still has to be checked against the file's + current upload session — see `check_upload_session`, which runs once the + file has been loaded. + + Raises: + PresignedUploadTokenError: token missing, expired, tampered with, or + issued for a different file/part. + """ + if not token: + raise PresignedUploadTokenError(_("Missing token parameter")) + + try: + token_data = signing.loads(token, max_age=cls.max_age(), salt=cls.SALT) + except signing.SignatureExpired: + raise PresignedUploadTokenError(_("Token has expired")) from None + except signing.BadSignature: + raise PresignedUploadTokenError(_("Invalid token signature")) from None + + if str(token_data.get("file_id")) != str(file_id) or str( + token_data.get("part_number") + ) != str(part_number): + raise PresignedUploadTokenError(_("Invalid token for this file/part")) + + return token_data + + @staticmethod + def check_upload_session(token_data: Dict[str, Any], upload_id: str) -> None: + """ + Bind the token to the current upload session: one minted for a previous + initiation (different `upload_id`) must not upload into a new one. + + Raises: + PresignedUploadTokenError: token belongs to a different session. + """ + if str(token_data.get("upload_id")) != str(upload_id): + raise PresignedUploadTokenError(_("Token does not match the current upload session")) diff --git a/baseapp/files/utils.py b/baseapp/files/utils.py new file mode 100644 index 00000000..5ec7c081 --- /dev/null +++ b/baseapp/files/utils.py @@ -0,0 +1,68 @@ +from collections.abc import Iterable + +import swapper +from django.contrib.contenttypes.models import ContentType +from django.db import models + +from baseapp_core.models import DocumentId + + +def default_files_count(): + return {"total": 0} + + +def get_or_create_file_target(target_obj): + """Get or create a FileTarget for the given object. + + Returns None for an unsaved/None object (the mixin guards against a NULL + primary key rather than raising IntegrityError).""" + FileTarget = swapper.load_model("baseapp_files", "FileTarget") + return FileTarget.get_or_create_for_object(target_obj) + + +def recalculate_files_count(parent): + if not parent: + return + + file_target = get_or_create_file_target(parent) + + # Count by a non-null column: Count("file_content_type") ignores NULL rows, + # under-counting files with no content type. Bucket NULLs under "unknown" + # to match the pgtrigger that maintains this same JSON on insert/update. + files_count_qs = parent.files.values("file_content_type").annotate(count=models.Count("id")) + file_target.files_count = default_files_count() + for item in files_count_qs: + count = item["count"] + key = item["file_content_type"] or "unknown" + file_target.files_count[key] = count + file_target.files_count["total"] += count + file_target.save(update_fields=["files_count"]) + + +def set_files_parent(parent, files: Iterable): + if not files: + return + + parent_content_type = ContentType.objects.get_for_model(parent) + parent_document_id = DocumentId.get_or_create_for_object(parent) + previous_parents = {} + + for file_obj in files: + previous_parent_doc_id = file_obj.parent + if previous_parent_doc_id and previous_parent_doc_id == parent_document_id: + continue + + if previous_parent_doc_id: + previous_parent = previous_parent_doc_id.content_object + key = (previous_parent_doc_id.content_type_id, previous_parent_doc_id.object_id) + if key not in previous_parents: + previous_parents[key] = previous_parent + + file_obj.parent = parent_document_id + file_obj.save(update_fields=["parent"]) + + for (content_type_id, object_id), previous_parent in previous_parents.items(): + if content_type_id == parent_content_type.pk and object_id == parent.pk: + continue + + recalculate_files_count(previous_parent) diff --git a/baseapp_auth/tests/integration/test_auth_without_baseapp_profiles.py b/baseapp_auth/tests/integration/test_auth_without_baseapp_profiles.py index c90cfd8e..4b0c6485 100644 --- a/baseapp_auth/tests/integration/test_auth_without_baseapp_profiles.py +++ b/baseapp_auth/tests/integration/test_auth_without_baseapp_profiles.py @@ -10,9 +10,7 @@ query { me { fullName - avatar(width: 32, height: 32) { - url - } + avatar(width: 32, height: 32) } } """ diff --git a/baseapp_chats/tests/test_graphql_mutations.py b/baseapp_chats/tests/test_graphql_mutations.py index 5dbcd731..a7cb03f4 100644 --- a/baseapp_chats/tests/test_graphql_mutations.py +++ b/baseapp_chats/tests/test_graphql_mutations.py @@ -60,9 +60,7 @@ } } } - image(width: 100, height: 100) { - url - } + image(width: 100, height: 100) allMessages { edges { node { @@ -101,9 +99,7 @@ } } } - image(width: 100, height: 100) { - url - } + image(width: 100, height: 100) } } removedParticipants { @@ -848,7 +844,7 @@ def test_user_can_create_group(django_user_client, graphql_user_client, image_dj assert content["data"]["chatRoomCreate"]["room"]["node"]["title"] == "group" assert content["data"]["chatRoomCreate"]["room"]["node"]["isGroup"] assert len(content["data"]["chatRoomCreate"]["room"]["node"]["participants"]["edges"]) == 3 - assert content["data"]["chatRoomCreate"]["room"]["node"]["image"]["url"].startswith( + assert content["data"]["chatRoomCreate"]["room"]["node"]["image"].startswith( "http://" # NOSONAR ) @@ -1109,7 +1105,7 @@ def test_admin_user_can_update_group_image( extra={"image": image_djangofile}, ) content = response.json() - assert content["data"]["chatRoomUpdate"]["room"]["node"]["image"]["url"].startswith( + assert content["data"]["chatRoomUpdate"]["room"]["node"]["image"].startswith( "http://" # NOSONAR ) diff --git a/baseapp_chats/tests/test_graphql_queries.py b/baseapp_chats/tests/test_graphql_queries.py index 71a71f0f..8e17da35 100644 --- a/baseapp_chats/tests/test_graphql_queries.py +++ b/baseapp_chats/tests/test_graphql_queries.py @@ -104,9 +104,7 @@ query GetRoom($roomId: ID!) { chatRoom(id: $roomId) { id - image(width: 100, height: 100) { - url - } + image(width: 100, height: 100) } } """ @@ -917,8 +915,8 @@ def test_resolve_image_for_1on1_chat_returns_other_participant_image( content = response.json() assert content["data"]["chatRoom"]["image"] is not None - assert content["data"]["chatRoom"]["image"]["url"] is not None - assert "100x100" in content["data"]["chatRoom"]["image"]["url"] + assert content["data"]["chatRoom"]["image"] is not None + assert "100x100" in content["data"]["chatRoom"]["image"] def test_resolve_image_for_group_chat_returns_group_image(django_client, image_djangofile) -> None: @@ -941,8 +939,8 @@ def test_resolve_image_for_group_chat_returns_group_image(django_client, image_d content = response.json() assert content["data"]["chatRoom"]["image"] is not None - assert content["data"]["chatRoom"]["image"]["url"] is not None - assert "100x100" in content["data"]["chatRoom"]["image"]["url"] + assert content["data"]["chatRoom"]["image"] is not None + assert "100x100" in content["data"]["chatRoom"]["image"] def test_resolve_image_returns_none_when_other_participant_has_no_image(django_client) -> None: diff --git a/baseapp_chats/tests/test_graphql_subscriptions.py b/baseapp_chats/tests/test_graphql_subscriptions.py index 0a97d8a5..fbbfa007 100644 --- a/baseapp_chats/tests/test_graphql_subscriptions.py +++ b/baseapp_chats/tests/test_graphql_subscriptions.py @@ -168,9 +168,7 @@ async def test_build_absolute_uri_on_graphql_subscription( message { node { profile { - image(width: 100, height: 100) { - url - } + image(width: 100, height: 100) } } } @@ -198,9 +196,9 @@ async def test_build_absolute_uri_on_graphql_subscription( # Check that subscription message were sent. resp = await client.receive(assert_id=sub_id, assert_type="next") - assert resp["data"]["chatRoomOnMessage"]["message"]["node"]["profile"]["image"][ - "url" - ].startswith("http") + assert resp["data"]["chatRoomOnMessage"]["message"]["node"]["profile"]["image"].startswith( + "http" + ) # Disconnect and wait the application to finish gracefully. await client.finalize() diff --git a/baseapp_comments/graphql/object_types.py b/baseapp_comments/graphql/object_types.py index 14803447..65052c85 100644 --- a/baseapp_comments/graphql/object_types.py +++ b/baseapp_comments/graphql/object_types.py @@ -121,6 +121,7 @@ class Meta: "ReactionsInterface", "MentionsInterface", "NodeActivityLogInterface", + "FilesInterface", ) model = Comment fields = ( @@ -180,6 +181,10 @@ def pre_optimization_hook(cls, queryset, optimizer) -> models.QuerySet: if service := shared_services.get("reactable_metadata"): queryset = service.annotate_queryset(queryset) + # Annotate files metadata so FilesInterface resolvers don't N+1. + if service := shared_services.get("files_metadata"): + queryset = service.annotate_queryset(queryset) + return queryset @classmethod diff --git a/baseapp_comments/models.py b/baseapp_comments/models.py index 20811def..d1bd6f0c 100644 --- a/baseapp_comments/models.py +++ b/baseapp_comments/models.py @@ -82,6 +82,12 @@ class Meta: comment_inheritances.append(ProfileMixin) +if apps.is_installed("baseapp.files"): + from baseapp.files.base import FileableModel + + comment_inheritances.append(FileableModel) + + class AbstractComment( *comment_inheritances, DocumentIdTargetMixin, diff --git a/baseapp_comments/tests/test_graphql_mutations_update.py b/baseapp_comments/tests/test_graphql_mutations_update.py index 98b4cfc1..8f9606c4 100644 --- a/baseapp_comments/tests/test_graphql_mutations_update.py +++ b/baseapp_comments/tests/test_graphql_mutations_update.py @@ -14,6 +14,13 @@ comment { id body + files { + edges { + node { + pk + } + } + } } errors { field diff --git a/baseapp_core/graphql/__init__.py b/baseapp_core/graphql/__init__.py index 1b5c0950..765a40cf 100644 --- a/baseapp_core/graphql/__init__.py +++ b/baseapp_core/graphql/__init__.py @@ -1,7 +1,7 @@ from .connections import CountedConnection # noqa from .decorators import login_required, user_passes_test # noqa from .errors import Errors, ErrorType # noqa -from .fields import File, ThumbnailField # noqa +from .fields import FileInterface, ThumbnailField, get_file_object_type # noqa from .middlewares import ( # noqa JWTAuthentication, LogExceptionMiddleware, diff --git a/baseapp_core/graphql/fields.py b/baseapp_core/graphql/fields.py index 2d5b0e8a..cd962170 100644 --- a/baseapp_core/graphql/fields.py +++ b/baseapp_core/graphql/fields.py @@ -1,8 +1,10 @@ from collections.abc import Callable import graphene +from django.apps import apps from django.conf import settings from django.core.cache import InvalidCacheBackendError, caches +from easy_thumbnails.engine import NoSourceGenerator from easy_thumbnails.files import get_thumbnailer try: @@ -11,14 +13,37 @@ cache = None -class File(graphene.ObjectType): - url = graphene.String(required=True) - # contentType = graphene.String() - # bytes = graphene.Int() +class FileInterface(graphene.Interface): + url = graphene.String() + + class Meta: + name = "FileInterface" + + +class ThumbnailObjectType(graphene.ObjectType): + class Meta: + name = "Thumbnail" + interfaces = (FileInterface,) + + +def get_file_object_type(): + if apps.is_installed("baseapp.files"): + import swapper + + File = swapper.load_model("baseapp_files", "File") + FileObjectType = File.get_graphql_object_type() + else: + + class FileObjectType(graphene.ObjectType): + class Meta: + interfaces = (FileInterface,) + name = "File" + + return FileObjectType class ThumbnailField(graphene.Field): - def __init__(self, type=File, **kwargs) -> None: + def __init__(self, type=graphene.String, **kwargs) -> None: kwargs.update( { "args": { @@ -32,7 +57,7 @@ def __init__(self, type=File, **kwargs) -> None: def get_resolver(self, parent_resolver) -> Callable: resolver = self.resolver or parent_resolver - def built_thumbnail(instance, info, width, height, **kwargs) -> File | None: + def built_thumbnail(instance, info, width, height, **kwargs) -> str | None: instance = resolver(instance, info, **kwargs) if not instance: @@ -42,16 +67,19 @@ def built_thumbnail(instance, info, width, height, **kwargs) -> File | None: cache_key = self._get_cache_key(instance, width, height) value_from_cache = cache.get(cache_key) if value_from_cache: - return File(url=value_from_cache) + return value_from_cache thumbnailer = get_thumbnailer(instance) - url = thumbnailer.get_thumbnail({"size": (width, height)}).url - absolute_url = info.context.build_absolute_uri(url) + try: + url = thumbnailer.get_thumbnail({"size": (width, height)}).url + absolute_url = info.context.build_absolute_uri(url) + except NoSourceGenerator: + absolute_url = None if cache: cache.set(cache_key, absolute_url, timeout=None) - return File(url=absolute_url) + return absolute_url return built_thumbnail diff --git a/baseapp_core/graphql/views.py b/baseapp_core/graphql/views.py index e08e62c4..43ca77eb 100644 --- a/baseapp_core/graphql/views.py +++ b/baseapp_core/graphql/views.py @@ -1,3 +1,4 @@ +import json import logging from collections.abc import Generator from contextlib import contextmanager @@ -61,3 +62,14 @@ def execute_graphql_request( return super().execute_graphql_request( request, data, query, variables, operation_name, show_graphiql ) + + def parse_body(self, request): + """Handle multipart request spec for multipart/form-data""" + content_type = self.get_content_type(request) + if content_type == "multipart/form-data" and "operations" in request.POST: + operations = json.loads(request.POST.get("operations", "{}")) + files_map = json.loads(request.POST.get("map", "{}")) + from graphene_file_upload.utils import place_files_in_operations + + return place_files_in_operations(operations, files_map, request.FILES) + return super(GraphQLView, self).parse_body(request) diff --git a/baseapp_organizations/tests/test_graphql_mutations_create.py b/baseapp_organizations/tests/test_graphql_mutations_create.py index 285ee5eb..e6d7b044 100644 --- a/baseapp_organizations/tests/test_graphql_mutations_create.py +++ b/baseapp_organizations/tests/test_graphql_mutations_create.py @@ -17,9 +17,7 @@ node { id name - image(width: 100, height: 100) { - url - } + image(width: 100, height: 100) urlPath { path } diff --git a/baseapp_profiles/rest_framework/README.md b/baseapp_profiles/rest_framework/README.md new file mode 100644 index 00000000..9b35541d --- /dev/null +++ b/baseapp_profiles/rest_framework/README.md @@ -0,0 +1,72 @@ +# CurrentProfile REST Framework Integration + +## Overview + +This module provides the `CurrentProfileMixin` to enable the same current profile functionality in REST framework views that exists in Django middleware and GraphQL middleware. + +## How It Works + +The `CurrentProfileMixin` reads the `Current-Profile` header from the request and sets `request.user.current_profile` based on: + +1. **No header provided**: Uses the user's default profile (`request.user.profile`) +2. **Header provided**: Attempts to load the profile with the specified ID (relay ID format) + - Only sets the profile if the user has permission to use it (`use_profile` permission) + - Falls back to `None` if permission check fails + +This replicates the logic from: +- `baseapp_profiles.middleware.CurrentProfileMiddleware` (Django middleware) +- `baseapp_profiles.graphql.middleware.CurrentProfileMiddleware` (GraphQL middleware) + +## Usage + +### In ViewSets + +Add the mixin to your viewset classes: + +```python +from baseapp_profiles.rest_framework import CurrentProfileMixin +from rest_framework import viewsets + +class MyViewSet(CurrentProfileMixin, viewsets.ModelViewSet): + queryset = MyModel.objects.all() + serializer_class = MySerializer +``` + +### Making Requests + +Send the `Current-Profile` header with the relay ID of the profile you want to use: + +```bash +curl -H "Authorization: Bearer " \ + -H "Current-Profile: UHJvZmlsZU9iamVjdFR5cGU6MQ==" \ + https://api.example.com/v1/my-endpoint/ +``` + +### Accessing Current Profile + +In your view methods, access the current profile via: + +```python +def list(self, request): + current_profile = request.user.current_profile + # Use the profile... +``` + +## Example Implementation + +See `baseapp.files.rest_framework.files.views.FilesViewSet` for an example: + +```python +from baseapp_profiles.rest_framework import CurrentProfileMixin +from rest_framework import viewsets + +class FilesViewSet(CurrentProfileMixin, mixins.RetrieveModelMixin, ...): + """ViewSet for file CRUD operations with current profile support.""" + pass +``` + +## Notes + +- The mixin executes in the `initial()` method, which runs before any view logic +- Authentication must happen before this mixin runs (ensure authentication classes are properly configured) +- The `Current-Profile` header must be whitelisted in CORS settings diff --git a/baseapp_profiles/rest_framework/__init__.py b/baseapp_profiles/rest_framework/__init__.py index ee5a3f30..e1e1e5d4 100644 --- a/baseapp_profiles/rest_framework/__init__.py +++ b/baseapp_profiles/rest_framework/__init__.py @@ -1,3 +1,4 @@ +from .mixins import CurrentProfileMixin # noqa from .serializers import JWTProfileSerializer -__all__ = ["JWTProfileSerializer"] +__all__ = ["CurrentProfileMixin", "JWTProfileSerializer"] diff --git a/baseapp_profiles/rest_framework/mixins.py b/baseapp_profiles/rest_framework/mixins.py new file mode 100644 index 00000000..dfd5f12f --- /dev/null +++ b/baseapp_profiles/rest_framework/mixins.py @@ -0,0 +1,50 @@ +import swapper + +from baseapp_core.graphql import get_pk_from_relay_id + +Profile = swapper.load_model("baseapp_profiles", "Profile") +profile_app_label = Profile._meta.app_label + + +class CurrentProfileMixin: + """ + Mixin to set current_profile on request.user for REST framework views. + + This replicates the logic from baseapp_profiles.middleware.CurrentProfileMiddleware + and baseapp_profiles.graphql.middleware.CurrentProfileMiddleware for DRF views. + + Usage: + class MyViewSet(CurrentProfileMixin, viewsets.ModelViewSet): + ... + """ + + def initial(self, request, *args, **kwargs): + """ + Runs anything that needs to occur prior to calling the method handler. + This is where we set the current_profile. + """ + super().initial(request, *args, **kwargs) + self.set_current_profile(request) + + def set_current_profile(self, request): + """ + Set current_profile on request.user based on Current-Profile header. + """ + current_profile_header = request.headers.get("Current-Profile") + request.user.current_profile = None + + if not request.user.is_authenticated: + return + + if not current_profile_header: + # No header provided, use the user's default profile + request.user.current_profile = request.user.profile + else: + # Header provided, try to load the specified profile + pk = get_pk_from_relay_id(current_profile_header) + if pk: + profile = Profile.objects.filter(pk=pk).first() + + # Check if user has permission to use this profile + if profile and request.user.has_perm(f"{profile_app_label}.use_profile", profile): + request.user.current_profile = profile diff --git a/baseapp_profiles/tests/integration/test_profiles_queries_without_baseapp_pages.py b/baseapp_profiles/tests/integration/test_profiles_queries_without_baseapp_pages.py index a9e8dde4..b7554fae 100644 --- a/baseapp_profiles/tests/integration/test_profiles_queries_without_baseapp_pages.py +++ b/baseapp_profiles/tests/integration/test_profiles_queries_without_baseapp_pages.py @@ -12,9 +12,7 @@ name metadata { metaTitle - metaOgImage(width: 100, height: 100) { - url - } + metaOgImage(width: 100, height: 100) } } } diff --git a/baseapp_profiles/tests/test_get_queries.py b/baseapp_profiles/tests/test_get_queries.py index 4078fe36..476fd9fc 100644 --- a/baseapp_profiles/tests/test_get_queries.py +++ b/baseapp_profiles/tests/test_get_queries.py @@ -20,9 +20,7 @@ name metadata { metaTitle - metaOgImage(width: 100, height: 100) { - url - } + metaOgImage(width: 100, height: 100) } } } @@ -49,7 +47,7 @@ def test_profile_metadata(graphql_user_client, image_djangofile) -> None: assert content["data"]["profile"]["id"] == profile.relay_id assert content["data"]["profile"]["name"] == profile.name assert content["data"]["profile"]["metadata"]["metaTitle"] == profile.name - assert content["data"]["profile"]["metadata"]["metaOgImage"]["url"].startswith("http") is True + assert content["data"]["profile"]["metadata"]["metaOgImage"].startswith("http") is True def test_owner_can_change_profile(django_user_client, graphql_user_client) -> None: diff --git a/baseapp_profiles/tests/test_graphql_mutations_update.py b/baseapp_profiles/tests/test_graphql_mutations_update.py index 71e40e95..eadd814a 100644 --- a/baseapp_profiles/tests/test_graphql_mutations_update.py +++ b/baseapp_profiles/tests/test_graphql_mutations_update.py @@ -20,12 +20,8 @@ id name biography - image(width: 100, height: 100) { - url - } - bannerImage(width: 100, height: 100) { - url - } + image(width: 100, height: 100) + bannerImage(width: 100, height: 100) } errors { field @@ -133,7 +129,7 @@ def test_owner_can_update_profile_image( ) content = response.json() - assert content["data"]["profileUpdate"]["profile"]["image"]["url"].startswith("http") + assert content["data"]["profileUpdate"]["profile"]["image"].startswith("http") def test_owner_can_update_profile_banner_image( @@ -148,7 +144,7 @@ def test_owner_can_update_profile_banner_image( ) content = response.json() - assert content["data"]["profileUpdate"]["profile"]["bannerImage"]["url"].startswith( + assert content["data"]["profileUpdate"]["profile"]["bannerImage"].startswith( "http://" # NOSONAR ) @@ -187,7 +183,7 @@ def test_owner_can_update_profile_banner_image_camel_case( ) content = response.json() - assert content["data"]["profileUpdate"]["profile"]["bannerImage"]["url"].startswith( + assert content["data"]["profileUpdate"]["profile"]["bannerImage"].startswith( "http://" # NOSONAR ) diff --git a/pyproject.toml b/pyproject.toml index fe1aac0e..d716b011 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -217,6 +217,7 @@ baseapp_geo = "baseapp.geo.plugin:GeoPlugin" baseapp_reactions = "baseapp_reactions.plugin:ReactionsPlugin" baseapp_blocks = "baseapp_blocks.plugin:BlocksPlugin" baseapp_content_feed = "baseapp.content_feed.plugin:ContentFeedPlugin" +baseapp_files = "baseapp.files.plugin:FilesPlugin" baseapp_chats = "baseapp_chats.plugin:ChatsPlugin" [tool.black] diff --git a/testproject/files/__init__.py b/testproject/files/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testproject/files/apps.py b/testproject/files/apps.py new file mode 100644 index 00000000..d4647971 --- /dev/null +++ b/testproject/files/apps.py @@ -0,0 +1,7 @@ +from django.apps import AppConfig + + +class FilesConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "testproject.files" + label = "files" diff --git a/testproject/files/migrations/0001_initial.py b/testproject/files/migrations/0001_initial.py new file mode 100644 index 00000000..7b6958e9 --- /dev/null +++ b/testproject/files/migrations/0001_initial.py @@ -0,0 +1,244 @@ +# Generated by Django 5.2.14 on 2026-06-26 21:23 + +import django.db.models.deletion +import django.utils.timezone +import model_utils.fields +import pgtrigger.compiler +import pgtrigger.migrations +from django.conf import settings +from django.db import migrations, models + +import baseapp.files.utils +import baseapp_core.models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ("baseapp_core", "0001_initial"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + migrations.swappable_dependency(settings.BASEAPP_PROFILES_PROFILE_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="FileTarget", + fields=[ + ( + "target", + models.OneToOneField( + help_text="The document this file target belongs to", + on_delete=django.db.models.deletion.CASCADE, + primary_key=True, + related_name="file_target", + serialize=False, + to="baseapp_core.documentid", + ), + ), + ("files_count", models.JSONField(default=baseapp.files.utils.default_files_count)), + ( + "is_files_enabled", + models.BooleanField(default=True, verbose_name="is files enabled"), + ), + ], + options={ + "abstract": False, + "swappable": "BASEAPP_FILES_FILETARGET_MODEL", + }, + ), + migrations.CreateModel( + name="File", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ( + "created", + model_utils.fields.AutoCreatedField( + default=django.utils.timezone.now, editable=False, verbose_name="created" + ), + ), + ( + "modified", + model_utils.fields.AutoLastModifiedField( + default=django.utils.timezone.now, editable=False, verbose_name="modified" + ), + ), + ("file_content_type", models.CharField(blank=True, max_length=150, null=True)), + ("file_name", models.CharField(blank=True, max_length=512, null=True)), + ( + "file_size", + models.PositiveIntegerField(help_text="File size in bytes", null=True), + ), + ( + "file", + models.FileField( + max_length=512, upload_to=baseapp_core.models.random_name_in("files") + ), + ), + ("name", models.CharField(blank=True, max_length=512, null=True)), + ("description", models.TextField(blank=True, null=True)), + ( + "upload_status", + models.CharField( + choices=[ + ("pending", "Pending Upload"), + ("uploading", "Upload in Progress"), + ("completed", "Upload Completed"), + ("failed", "Upload Failed"), + ("aborted", "Upload Aborted"), + ], + db_index=True, + default="completed", + help_text="Status of the file upload", + max_length=20, + ), + ), + ( + "upload_id", + models.CharField( + blank=True, + help_text="S3 multipart upload ID for in-progress uploads", + max_length=512, + null=True, + ), + ), + ( + "total_parts", + models.PositiveIntegerField( + blank=True, + help_text="Total number of parts for multipart upload", + null=True, + ), + ), + ( + "uploaded_parts", + models.JSONField( + blank=True, + default=dict, + help_text="Tracking uploaded parts with ETags", + null=True, + ), + ), + ( + "upload_expires_at", + models.DateTimeField( + blank=True, + db_index=True, + help_text="Expiration time for pending uploads", + null=True, + ), + ), + ( + "created_by", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="files_created", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "parent", + models.ForeignKey( + blank=True, + help_text="The parent document this file belongs to", + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="files", + to="baseapp_core.documentid", + ), + ), + ( + "profile", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="files", + to=settings.BASEAPP_PROFILES_PROFILE_MODEL, + verbose_name="profile", + ), + ), + ], + options={ + "abstract": False, + "swappable": "BASEAPP_FILES_FILE_MODEL", + }, + ), + pgtrigger.migrations.AddTrigger( + model_name="file", + trigger=pgtrigger.compiler.Trigger( + name="insert_document_id", + sql=pgtrigger.compiler.UpsertTriggerSql( + func="\n INSERT INTO baseapp_core_documentid (public_id, content_type_id, object_id, created, modified)\n VALUES (\n gen_random_uuid(),\n (SELECT id FROM django_content_type WHERE app_label = 'files' AND model = 'file'),\n NEW.id,\n NOW(),\n NOW()\n )\n ON CONFLICT (content_type_id, object_id) DO NOTHING;\n RETURN NULL;\n ", + hash="6f43d1588d53b00ddfa0d5a563a4cd0dbb9e6f62", + operation="INSERT", + pgid="pgtrigger_insert_document_id_d067b", + table="files_file", + when="AFTER", + ), + ), + ), + pgtrigger.migrations.AddTrigger( + model_name="file", + trigger=pgtrigger.compiler.Trigger( + name="delete_document_id", + sql=pgtrigger.compiler.UpsertTriggerSql( + func="\n DELETE FROM baseapp_core_documentid\n WHERE\n content_type_id = (SELECT id FROM django_content_type WHERE app_label = 'files' AND model = 'file')\n AND object_id = OLD.id;\n RETURN NULL;\n ", + hash="877efe613614ca4ef5d93a264fde512958f34df2", + operation="DELETE", + pgid="pgtrigger_delete_document_id_23af1", + table="files_file", + when="AFTER", + ), + ), + ), + pgtrigger.migrations.AddTrigger( + model_name="file", + trigger=pgtrigger.compiler.Trigger( + name="update_file_target_on_insert", + sql=pgtrigger.compiler.UpsertTriggerSql( + func="\n DECLARE\n new_counts JSONB;\n BEGIN\n IF NEW.parent_id IS NOT NULL THEN\n -- Calculate new counts\n SELECT COALESCE(\n jsonb_object_agg(\n file_content_type,\n count\n ) || jsonb_build_object('total', SUM(count)),\n '{\"total\": 0}'::jsonb\n ) INTO new_counts\n FROM (\n SELECT\n COALESCE(file_content_type, 'unknown') as file_content_type,\n COUNT(*)::int as count\n FROM files_file\n WHERE parent_id = NEW.parent_id\n GROUP BY file_content_type\n ) counts;\n\n -- Insert or update FileTarget\n INSERT INTO files_filetarget (target_id, files_count, is_files_enabled)\n VALUES (NEW.parent_id, new_counts, true)\n ON CONFLICT (target_id)\n DO UPDATE SET files_count = new_counts;\n END IF;\n RETURN NEW;\n END;\n ", + hash="fe7e8e6f14c341e981b6cec3920eb6ba96222706", + operation="INSERT", + pgid="pgtrigger_update_file_target_on_insert_8a1d9", + table="files_file", + when="AFTER", + ), + ), + ), + pgtrigger.migrations.AddTrigger( + model_name="file", + trigger=pgtrigger.compiler.Trigger( + name="update_file_target_on_update", + sql=pgtrigger.compiler.UpsertTriggerSql( + func="\n DECLARE\n new_counts JSONB;\n old_counts JSONB;\n BEGIN\n -- Update old parent if it changed\n IF OLD.parent_id IS DISTINCT FROM NEW.parent_id\n AND OLD.parent_id IS NOT NULL THEN\n\n SELECT COALESCE(\n jsonb_object_agg(\n file_content_type,\n count\n ) || jsonb_build_object('total', SUM(count)),\n '{\"total\": 0}'::jsonb\n ) INTO old_counts\n FROM (\n SELECT\n COALESCE(file_content_type, 'unknown') as file_content_type,\n COUNT(*)::int as count\n FROM files_file\n WHERE parent_id = OLD.parent_id\n GROUP BY file_content_type\n ) counts;\n\n UPDATE files_filetarget SET files_count = old_counts\n WHERE target_id = OLD.parent_id;\n END IF;\n\n -- Update new parent\n IF NEW.parent_id IS NOT NULL THEN\n SELECT COALESCE(\n jsonb_object_agg(\n file_content_type,\n count\n ) || jsonb_build_object('total', SUM(count)),\n '{\"total\": 0}'::jsonb\n ) INTO new_counts\n FROM (\n SELECT\n COALESCE(file_content_type, 'unknown') as file_content_type,\n COUNT(*)::int as count\n FROM files_file\n WHERE parent_id = NEW.parent_id\n GROUP BY file_content_type\n ) counts;\n\n INSERT INTO files_filetarget (target_id, files_count, is_files_enabled)\n VALUES (NEW.parent_id, new_counts, true)\n ON CONFLICT (target_id)\n DO UPDATE SET files_count = new_counts;\n END IF;\n\n RETURN NEW;\n END;\n ", + hash="4e27b58158d9658648a4a79af0d8bfed613196ef", + operation="UPDATE", + pgid="pgtrigger_update_file_target_on_update_89baa", + table="files_file", + when="AFTER", + ), + ), + ), + pgtrigger.migrations.AddTrigger( + model_name="file", + trigger=pgtrigger.compiler.Trigger( + name="update_file_target_on_delete", + sql=pgtrigger.compiler.UpsertTriggerSql( + func="\n DECLARE\n new_counts JSONB;\n BEGIN\n IF OLD.parent_id IS NOT NULL THEN\n SELECT COALESCE(\n jsonb_object_agg(\n file_content_type,\n count\n ) || jsonb_build_object('total', SUM(count)),\n '{\"total\": 0}'::jsonb\n ) INTO new_counts\n FROM (\n SELECT\n COALESCE(file_content_type, 'unknown') as file_content_type,\n COUNT(*)::int as count\n FROM files_file\n WHERE parent_id = OLD.parent_id\n GROUP BY file_content_type\n ) counts;\n\n UPDATE files_filetarget SET files_count = new_counts\n WHERE target_id = OLD.parent_id;\n END IF;\n\n RETURN OLD;\n END;\n ", + hash="3ccd45f3c662961c3959e628d80113b1a4203f1b", + operation="DELETE", + pgid="pgtrigger_update_file_target_on_delete_3c3fa", + table="files_file", + when="AFTER", + ), + ), + ), + ] diff --git a/testproject/files/migrations/0002_alter_file_description_alter_file_file_content_type_and_more.py b/testproject/files/migrations/0002_alter_file_description_alter_file_file_content_type_and_more.py new file mode 100644 index 00000000..c3b43f22 --- /dev/null +++ b/testproject/files/migrations/0002_alter_file_description_alter_file_file_content_type_and_more.py @@ -0,0 +1,55 @@ +# Generated by Django 5.2.14 on 2026-07-12 00:54 + +from django.db import migrations, models + +# These string fields dropped null=True in favour of default="". Convert any +# existing NULLs to "" before the AlterField enforces NOT NULL, so the +# migration is safe on databases populated by the previous schema. +NULLABLE_STRING_FIELDS = ("description", "file_content_type", "file_name", "name", "upload_id") + + +def fill_null_strings(apps, schema_editor): + File = apps.get_model("files", "File") + for field in NULLABLE_STRING_FIELDS: + File.objects.filter(**{f"{field}__isnull": True}).update(**{field: ""}) + + +class Migration(migrations.Migration): + + dependencies = [ + ("files", "0001_initial"), + ] + + operations = [ + migrations.RunPython(fill_null_strings, migrations.RunPython.noop), + migrations.AlterField( + model_name="file", + name="description", + field=models.TextField(blank=True, default=""), + ), + migrations.AlterField( + model_name="file", + name="file_content_type", + field=models.CharField(blank=True, default="", max_length=150), + ), + migrations.AlterField( + model_name="file", + name="file_name", + field=models.CharField(blank=True, default="", max_length=512), + ), + migrations.AlterField( + model_name="file", + name="name", + field=models.CharField(blank=True, default="", max_length=512), + ), + migrations.AlterField( + model_name="file", + name="upload_id", + field=models.CharField( + blank=True, + default="", + help_text="S3 multipart upload ID for in-progress uploads", + max_length=512, + ), + ), + ] diff --git a/testproject/files/migrations/0003_alter_file_options_alter_filetarget_options_and_more.py b/testproject/files/migrations/0003_alter_file_options_alter_filetarget_options_and_more.py new file mode 100644 index 00000000..0bfe24c2 --- /dev/null +++ b/testproject/files/migrations/0003_alter_file_options_alter_filetarget_options_and_more.py @@ -0,0 +1,205 @@ +# Generated by Django 5.2.14 on 2026-07-12 12:58 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + +import baseapp.files.utils +import baseapp_core.models + + +class Migration(migrations.Migration): + + dependencies = [ + ("baseapp_core", "0001_initial"), + ("files", "0002_alter_file_description_alter_file_file_content_type_and_more"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AlterModelOptions( + name="file", + options={"verbose_name": "file", "verbose_name_plural": "files"}, + ), + migrations.AlterModelOptions( + name="filetarget", + options={"verbose_name": "file target", "verbose_name_plural": "file targets"}, + ), + migrations.AlterField( + model_name="file", + name="created_by", + field=models.ForeignKey( + help_text="The user who uploaded the file", + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="files_created", + to=settings.AUTH_USER_MODEL, + verbose_name="created by", + ), + ), + migrations.AlterField( + model_name="file", + name="description", + field=models.TextField( + blank=True, + default="", + help_text="Optional description of the file", + verbose_name="description", + ), + ), + migrations.AlterField( + model_name="file", + name="file", + field=models.FileField( + help_text="The stored file", + max_length=512, + upload_to=baseapp_core.models.random_name_in("files"), + verbose_name="file", + ), + ), + migrations.AlterField( + model_name="file", + name="file_content_type", + field=models.CharField( + blank=True, + default="", + help_text="MIME type of the uploaded file", + max_length=150, + verbose_name="content type", + ), + ), + migrations.AlterField( + model_name="file", + name="file_name", + field=models.CharField( + blank=True, + default="", + help_text="Original name of the uploaded file", + max_length=512, + verbose_name="file name", + ), + ), + migrations.AlterField( + model_name="file", + name="file_size", + field=models.PositiveIntegerField( + help_text="File size in bytes", null=True, verbose_name="file size" + ), + ), + migrations.AlterField( + model_name="file", + name="name", + field=models.CharField( + blank=True, + default="", + help_text="Display name for the file", + max_length=512, + verbose_name="name", + ), + ), + migrations.AlterField( + model_name="file", + name="parent", + field=models.ForeignKey( + blank=True, + help_text="The parent document this file belongs to", + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="files", + to="baseapp_core.documentid", + verbose_name="parent", + ), + ), + migrations.AlterField( + model_name="file", + name="total_parts", + field=models.PositiveIntegerField( + blank=True, + help_text="Total number of parts for multipart upload", + null=True, + verbose_name="total parts", + ), + ), + migrations.AlterField( + model_name="file", + name="upload_expires_at", + field=models.DateTimeField( + blank=True, + db_index=True, + help_text="Expiration time for pending uploads", + null=True, + verbose_name="upload expires at", + ), + ), + migrations.AlterField( + model_name="file", + name="upload_id", + field=models.CharField( + blank=True, + default="", + help_text="S3 multipart upload ID for in-progress uploads", + max_length=512, + verbose_name="upload ID", + ), + ), + migrations.AlterField( + model_name="file", + name="upload_status", + field=models.CharField( + choices=[ + ("pending", "Pending Upload"), + ("uploading", "Upload in Progress"), + ("completed", "Upload Completed"), + ("failed", "Upload Failed"), + ("aborted", "Upload Aborted"), + ], + db_index=True, + default="completed", + help_text="Status of the file upload", + max_length=20, + verbose_name="upload status", + ), + ), + migrations.AlterField( + model_name="file", + name="uploaded_parts", + field=models.JSONField( + blank=True, + default=dict, + help_text="Tracking uploaded parts with ETags", + null=True, + verbose_name="uploaded parts", + ), + ), + migrations.AlterField( + model_name="filetarget", + name="files_count", + field=models.JSONField( + default=baseapp.files.utils.default_files_count, + help_text="Per-content-type file counts for the target", + verbose_name="files count", + ), + ), + migrations.AlterField( + model_name="filetarget", + name="is_files_enabled", + field=models.BooleanField( + default=True, + help_text="Whether files can be attached to the target", + verbose_name="is files enabled", + ), + ), + migrations.AlterField( + model_name="filetarget", + name="target", + field=models.OneToOneField( + help_text="The document this file target belongs to", + on_delete=django.db.models.deletion.CASCADE, + primary_key=True, + related_name="file_target", + serialize=False, + to="baseapp_core.documentid", + verbose_name="target", + ), + ), + ] diff --git a/testproject/files/migrations/__init__.py b/testproject/files/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testproject/files/models.py b/testproject/files/models.py new file mode 100644 index 00000000..7f28c177 --- /dev/null +++ b/testproject/files/models.py @@ -0,0 +1,11 @@ +from baseapp.files.models import AbstractFile, AbstractFileTarget + + +class FileTarget(AbstractFileTarget): + class Meta(AbstractFileTarget.Meta): + pass + + +class File(AbstractFile): + class Meta(AbstractFile.Meta): + pass diff --git a/testproject/settings.py b/testproject/settings.py index 53fcd3f4..0d395a62 100644 --- a/testproject/settings.py +++ b/testproject/settings.py @@ -44,6 +44,7 @@ "baseapp_social_auth", "baseapp_social_auth.cache", "baseapp.content_feed", + "baseapp.files", "baseapp_pdf", "baseapp_api_key", *WAGTAIL_INSTALLED_APPS, @@ -55,6 +56,7 @@ "testproject.profiles", "testproject.reactions", "testproject.content_feed", + "testproject.files", "testproject.follows", "testproject.blocks", "testproject.base", @@ -142,6 +144,7 @@ *plugin_registry.get("AUTHENTICATION_BACKENDS", "baseapp_blocks"), *plugin_registry.get("AUTHENTICATION_BACKENDS", "baseapp_chats"), *plugin_registry.get("AUTHENTICATION_BACKENDS", "baseapp_organizations"), + *plugin_registry.get("AUTHENTICATION_BACKENDS", "baseapp_files"), ] ADMIN_TIME_ZONE = "UTC" @@ -245,6 +248,10 @@ BASEAPP_CHATS_MESSAGE_MODEL = "social_chats.Message" BASEAPP_CHATS_MESSAGESTATUS_MODEL = "social_chats.MessageStatus" +# Files +BASEAPP_FILES_FILE_MODEL = "files.File" +BASEAPP_FILES_FILETARGET_MODEL = "files.FileTarget" + # Notifications NOTIFICATIONS_NOTIFICATION_MODEL = "notifications.Notification" BASEAPP_NOTIFICATIONS_NOTIFICATIONSETTING_MODEL = "notifications.NotificationSetting"