diff --git a/lib/phoenix_kit/migrations/postgres.ex b/lib/phoenix_kit/migrations/postgres.ex index 6ee4ecd31..5208fa954 100644 --- a/lib/phoenix_kit/migrations/postgres.ex +++ b/lib/phoenix_kit/migrations/postgres.ex @@ -158,24 +158,43 @@ defmodule PhoenixKit.Migrations.Postgres do - Configurable strictness: Can log warnings or force re-authentication - Performance indexes for efficient fingerprint verification - ### V24 - File Checksum Unique Index ⚡ LATEST + ### V24 - File Checksum Unique Index - Unique index on phoenix_kit_files.checksum for O(1) duplicate detection - Enables automatic deduplication of uploaded files - Prevents redundant storage of identical files - Improves performance of duplicate file lookups + ### V25 - Aspect Ratio Control for Dimensions + - Adds maintain_aspect_ratio boolean column to phoenix_kit_storage_dimensions + - Allows choosing between aspect ratio preservation (width-only) or fixed dimensions + - Per-dimension control for responsive sizing vs exact crops + - Defaults to maintaining aspect ratio for all dimensions + + ### V26 - Rename Checksum Fields & Per-User Deduplication ⚡ LATEST + - Renames `checksum` to `file_checksum` (clearer naming) + - Removes unique index on file_checksum (allows same file from different users) + - Adds `user_file_checksum` column (SHA256 of user_id + file_checksum) + - Creates unique index on user_file_checksum for per-user duplicate detection + - Same user cannot upload same file twice (enforced by user_file_checksum) + - Different users CAN upload same file (different user_file_checksum values) + - Preserves file_checksum field for popularity analytics across all users + - Clearer naming convention: file_checksum vs user_file_checksum + ## Migration Paths ### Fresh Installation (0 → Current) - Runs all migrations V01 through V24 in sequence. + Runs all migrations V01 through V26 in sequence. ### Incremental Updates - - V01 → V24: Runs V02 through V24 in sequence - - V23 → V24: Runs V24 only (adds file checksum unique index) - - V22 → V24: Runs V23 and V24 in sequence - - V20 → V24: Runs V21, V22, V23, and V24 in sequence + - V01 → V26: Runs V02 through V26 in sequence + - V25 → V26: Runs V26 only (adds per-user file hash) + - V24 → V26: Runs V25 and V26 in sequence + - V23 → V26: Runs V24, V25, and V26 in sequence + - V20 → V26: Runs V21 through V26 in sequence ### Rollback Support + - V26 → V25: Removes user_file_checksum, renames file_checksum back to checksum, restores checksum unique index + - V25 → V24: Removes aspect ratio control from dimensions - V24 → V23: Removes unique index on checksum - V23 → V22: Removes session fingerprinting columns and indexes - V22 → V21: Removes audit logging system, email orphaned events, and email metrics @@ -193,14 +212,14 @@ defmodule PhoenixKit.Migrations.Postgres do ## Usage Examples - # Update to latest version (V24) + # Update to latest version (V26) PhoenixKit.Migrations.Postgres.up(prefix: "myapp") # Update to specific version - PhoenixKit.Migrations.Postgres.up(prefix: "myapp", version: 24) + PhoenixKit.Migrations.Postgres.up(prefix: "myapp", version: 26) # Rollback to specific version - PhoenixKit.Migrations.Postgres.down(prefix: "myapp", version: 23) + PhoenixKit.Migrations.Postgres.down(prefix: "myapp", version: 25) # Complete rollback PhoenixKit.Migrations.Postgres.down(prefix: "myapp", version: 0) @@ -218,7 +237,7 @@ defmodule PhoenixKit.Migrations.Postgres do use Ecto.Migration @initial_version 1 - @current_version 25 + @current_version 26 @default_prefix "public" @doc false diff --git a/lib/phoenix_kit/migrations/postgres/v26.ex b/lib/phoenix_kit/migrations/postgres/v26.ex new file mode 100644 index 000000000..1b22fd631 --- /dev/null +++ b/lib/phoenix_kit/migrations/postgres/v26.ex @@ -0,0 +1,91 @@ +defmodule PhoenixKit.Migrations.Postgres.V26 do + @moduledoc """ + Migration V26: Rename checksum fields and add per-user deduplication. + + This migration renames checksum fields for clarity and adds per-user file deduplication + while preserving the ability to query for popular files across all users. + + ## Changes + - Enables `pgcrypto` extension for cryptographic functions + - Renames `checksum` column to `file_checksum` (for clarity) + - Drops unique index on `file_checksum` (allows same file from different users) + - Adds `user_file_checksum` column (SHA256 of user_id + file_checksum) + - Creates unique index on `user_file_checksum` to enforce per-user uniqueness + - Backfills existing records with calculated user_file_checksum values + + ## Requirements + - PostgreSQL with `pgcrypto` extension support (enabled automatically) + + ## Purpose + - Same user cannot upload the same file twice (duplicate prevention via user_file_checksum) + - Different users CAN upload the same file (no unique constraint on file_checksum) + - Original `file_checksum` field preserved for finding most popular images across all users + - Clearer naming: file_checksum vs user_file_checksum + """ + + use Ecto.Migration + + def up(%{prefix: prefix} = _opts) do + # Enable pgcrypto extension for digest function + execute "CREATE EXTENSION IF NOT EXISTS pgcrypto" + + # Drop the unique index on checksum (from V24) + drop_if_exists unique_index(:phoenix_kit_files, [:checksum], prefix: prefix) + + # Rename checksum to file_checksum for clarity + rename table(:phoenix_kit_files, prefix: prefix), :checksum, to: :file_checksum + + # Add user_file_checksum column + alter table(:phoenix_kit_files, prefix: prefix) do + add :user_file_checksum, :string + end + + # Backfill existing records with user_file_checksum + execute """ + UPDATE #{prefix}.phoenix_kit_files + SET user_file_checksum = encode(digest(CAST(user_id AS text) || file_checksum, 'sha256'), 'hex') + WHERE user_file_checksum IS NULL + """ + + # Make the column NOT NULL after backfill + alter table(:phoenix_kit_files, prefix: prefix) do + modify :user_file_checksum, :string, null: false + end + + # Create unique index on user_file_checksum for fast per-user duplicate detection + create unique_index(:phoenix_kit_files, [:user_file_checksum], + prefix: prefix, + name: "#{prefix}_phoenix_kit_files_user_file_checksum_index" + ) + + # Set version comment on phoenix_kit table for version tracking + execute "COMMENT ON TABLE #{prefix_table_name("phoenix_kit", prefix)} IS '26'" + end + + def down(%{prefix: prefix} = _opts) do + # Drop the user_file_checksum unique index + drop_if_exists unique_index(:phoenix_kit_files, [:user_file_checksum], + prefix: prefix, + name: "#{prefix}_phoenix_kit_files_user_file_checksum_index" + ) + + # Remove user_file_checksum column + alter table(:phoenix_kit_files, prefix: prefix) do + remove :user_file_checksum + end + + # Rename file_checksum back to checksum + rename table(:phoenix_kit_files, prefix: prefix), :file_checksum, to: :checksum + + # Restore the unique index on checksum (from V24) + create_if_not_exists unique_index(:phoenix_kit_files, [:checksum], prefix: prefix) + + # Update version comment on phoenix_kit table to previous version + execute "COMMENT ON TABLE #{prefix_table_name("phoenix_kit", prefix)} IS '25'" + end + + # Helper functions + + defp prefix_table_name(table_name, nil), do: table_name + defp prefix_table_name(table_name, prefix), do: "#{prefix}.#{table_name}" +end diff --git a/lib/phoenix_kit/storage.ex b/lib/phoenix_kit/storage.ex index 34183e5a5..6ecd379df 100644 --- a/lib/phoenix_kit/storage.ex +++ b/lib/phoenix_kit/storage.ex @@ -413,10 +413,41 @@ defmodule PhoenixKit.Storage do def get_file(id), do: repo().get(PhoenixKit.Storage.File, id) @doc """ - Gets a file by its hash. + Calculates user-specific file checksum (salted with user_id). + + This creates a unique checksum per user+file combination for duplicate detection, + while preserving the original file checksum for popularity queries. + + ## Parameters + - user_id: The user ID (integer or string) + - file_checksum: The SHA256 checksum of the file content + + ## Returns + String representing the SHA256 checksum of "user_id + file_checksum" """ - def get_file_by_hash(hash) do - repo().get_by(PhoenixKit.Storage.File, checksum: hash) + def calculate_user_file_checksum(user_id, file_checksum) do + "#{user_id}#{file_checksum}" + |> then(fn data -> :crypto.hash(:sha256, data) end) + |> Base.encode16(case: :lower) + end + + @doc """ + Gets a file by its user-specific checksum. + + This checks for duplicates for a specific user. + """ + def get_file_by_user_checksum(user_file_checksum) do + repo().get_by(PhoenixKit.Storage.File, user_file_checksum: user_file_checksum) + end + + @doc """ + Gets a file by its original content checksum (file_checksum). + + This can find files uploaded by any user with the same content. + Useful for popularity queries. + """ + def get_file_by_checksum(file_checksum) do + repo().get_by(PhoenixKit.Storage.File, file_checksum: file_checksum) end @doc """ @@ -647,20 +678,24 @@ defmodule PhoenixKit.Storage do # Validate required fields if Elixir.File.exists?(source_path) do - # Calculate file hash - file_hash = calculate_file_hash(source_path) + # Calculate file checksum + file_checksum = calculate_file_hash(source_path) - # Check if file already exists - case get_file_by_hash(file_hash) do + # Calculate user-specific checksum for duplicate detection + user_file_checksum = calculate_user_file_checksum(user_id, file_checksum) + + # Check if this user already uploaded this file + case get_file_by_user_checksum(user_file_checksum) do %PhoenixKit.Storage.File{} = existing_file -> - # File already exists, return existing file + # File already exists for this user, return existing file {:ok, existing_file} nil -> - # New file, proceed with storage + # New file for this user, proceed with storage store_new_file( source_path, - file_hash, + file_checksum, + user_file_checksum, filename, content_type, size_bytes, @@ -706,7 +741,7 @@ defmodule PhoenixKit.Storage do Retrieves a file by its hash. """ def retrieve_file_by_hash(hash) do - case get_file_by_hash(hash) do + case get_file_by_checksum(hash) do %PhoenixKit.Storage.File{} = file -> retrieve_file(file.id) @@ -858,15 +893,18 @@ defmodule PhoenixKit.Storage do source_path, file_type, user_id, - file_hash, + file_checksum, ext, original_filename \\ nil ) do - # Check if file already exists by hash - case get_file_by_hash(file_hash) do + # Calculate user-specific hash for duplicate detection + user_file_checksum = calculate_user_file_checksum(user_id, file_checksum) + + # Check if this user already uploaded this file + case get_file_by_user_checksum(user_file_checksum) do %PhoenixKit.Storage.File{} = existing_file -> Logger.info("=== DUPLICATE FILE DETECTED ===") - Logger.info("File ID: #{existing_file.id}, Checksum: #{file_hash}") + Logger.info("File ID: #{existing_file.id}, Checksum: #{file_checksum}") Logger.info("File path: #{existing_file.file_path}") # File already exists, but check if instances and actual files are healthy @@ -892,7 +930,7 @@ defmodule PhoenixKit.Storage do restore_missing_file( existing_file, source_path, - file_hash, + file_checksum, user_id, original_filename ) @@ -910,7 +948,7 @@ defmodule PhoenixKit.Storage do recreate_file_instances( existing_file, source_path, - file_hash, + file_checksum, user_id, original_filename ) @@ -923,7 +961,8 @@ defmodule PhoenixKit.Storage do source_path, file_type, user_id, - file_hash, + file_checksum, + user_file_checksum, ext, original_filename ) @@ -934,7 +973,8 @@ defmodule PhoenixKit.Storage do source_path, file_type, user_id, - file_hash, + file_checksum, + user_file_checksum, ext, original_filename ) do @@ -965,7 +1005,8 @@ defmodule PhoenixKit.Storage do mime_type: determine_mime_type(ext), file_type: file_type, ext: ext, - checksum: file_hash, + file_checksum: file_checksum, + user_file_checksum: user_file_checksum, size: get_file_size(source_path), status: "processing", user_id: user_id @@ -984,7 +1025,7 @@ defmodule PhoenixKit.Storage do file_name: original_path, mime_type: file.mime_type, ext: ext, - checksum: file_hash, + checksum: file_checksum, size: get_file_size(source_path), processing_status: "completed", file_id: file.id @@ -1067,7 +1108,7 @@ defmodule PhoenixKit.Storage do deleted_count end - defp recreate_file_instances(file, source_path, file_hash, user_id, original_filename) do + defp recreate_file_instances(file, source_path, file_checksum, user_id, original_filename) do # File record exists but instances are missing or broken # First store the file in buckets, then recreate the instance record @@ -1103,7 +1144,7 @@ defmodule PhoenixKit.Storage do file_name: original_path, mime_type: file.mime_type, ext: file.ext, - checksum: file_hash, + checksum: file_checksum, size: file_size, processing_status: "completed", file_id: file.id @@ -1260,7 +1301,8 @@ defmodule PhoenixKit.Storage do defp store_new_file( source_path, - file_hash, + file_checksum, + user_file_checksum, filename, content_type, size_bytes, @@ -1275,7 +1317,8 @@ defmodule PhoenixKit.Storage do storage_info, filename, content_type, - file_hash, + file_checksum, + user_file_checksum, size_bytes, metadata, user_id @@ -1284,7 +1327,7 @@ defmodule PhoenixKit.Storage do case create_file(file_attrs) do {:ok, file} -> # Create original instance and variants (non-critical operations) - create_original_instance_and_variants(file, file_hash, size_bytes) + create_original_instance_and_variants(file, file_checksum, size_bytes) {:ok, file} {:error, changeset} -> @@ -1302,7 +1345,8 @@ defmodule PhoenixKit.Storage do storage_info, filename, content_type, - file_hash, + file_checksum, + user_file_checksum, size_bytes, metadata, user_id @@ -1313,7 +1357,8 @@ defmodule PhoenixKit.Storage do mime_type: content_type, file_type: determine_file_type(content_type), ext: Path.extname(filename), - checksum: file_hash, + file_checksum: file_checksum, + user_file_checksum: user_file_checksum, size: size_bytes, # Convert to MB size_mb: size_bytes / (1024 * 1024), @@ -1323,13 +1368,13 @@ defmodule PhoenixKit.Storage do } end - defp create_original_instance_and_variants(file, file_hash, size_bytes) do + defp create_original_instance_and_variants(file, file_checksum, size_bytes) do original_instance_attrs = %{ variant_name: "original", file_name: file.file_name, mime_type: file.mime_type, ext: file.ext, - checksum: file_hash, + checksum: file_checksum, size: size_bytes, # Will be populated if we can detect dimensions width: nil, diff --git a/lib/phoenix_kit/storage/file.ex b/lib/phoenix_kit/storage/file.ex index e0bc88d9b..2c8c451ad 100644 --- a/lib/phoenix_kit/storage/file.ex +++ b/lib/phoenix_kit/storage/file.ex @@ -25,7 +25,8 @@ defmodule PhoenixKit.Storage.File do - `mime_type` - MIME type (image/jpeg, video/mp4, etc.) - `file_type` - High-level type (image, video, document, archive) - `ext` - File extension (jpg, mp4, pdf, etc.) - - `checksum` - MD5 or SHA256 hash for integrity verification + - `file_checksum` - SHA256 hash of file content for integrity verification + - `user_file_checksum` - SHA256 hash of (user_id + file_checksum) for per-user deduplication - `size` - File size in bytes - `width` - Image/video width in pixels (nullable) - `height` - Image/video height in pixels (nullable) @@ -44,7 +45,8 @@ defmodule PhoenixKit.Storage.File do mime_type: "image/jpeg", file_type: "image", ext: "jpg", - checksum: "abc123def456...", + file_checksum: "abc123def456...", + user_file_checksum: "xyz789ghi012...", size: 524_288, # 512 KB width: 2000, height: 2000, @@ -60,6 +62,8 @@ defmodule PhoenixKit.Storage.File do mime_type: "video/mp4", file_type: "video", ext: "mp4", + file_checksum: "def456ghi789...", + user_file_checksum: "mno345pqr678...", size: 10_485_760, # 10 MB width: 1920, height: 1080, @@ -75,6 +79,8 @@ defmodule PhoenixKit.Storage.File do mime_type: "application/pdf", file_type: "document", ext: "pdf", + file_checksum: "ghi789jkl012...", + user_file_checksum: "stu901vwx234...", size: 2_097_152, # 2 MB status: "active" } @@ -93,7 +99,8 @@ defmodule PhoenixKit.Storage.File do mime_type: String.t(), file_type: String.t(), ext: String.t(), - checksum: String.t(), + file_checksum: String.t(), + user_file_checksum: String.t(), size: integer(), width: integer() | nil, height: integer() | nil, @@ -114,7 +121,8 @@ defmodule PhoenixKit.Storage.File do field :mime_type, :string field :file_type, :string field :ext, :string - field :checksum, :string + field :file_checksum, :string + field :user_file_checksum, :string field :size, :integer field :width, :integer field :height, :integer @@ -138,7 +146,8 @@ defmodule PhoenixKit.Storage.File do - `mime_type` - `file_type` (must be: "image", "video", "document", "archive") - `ext` - - `checksum` + - `file_checksum` + - `user_file_checksum` - `size` - `user_id` @@ -159,7 +168,8 @@ defmodule PhoenixKit.Storage.File do :mime_type, :file_type, :ext, - :checksum, + :file_checksum, + :user_file_checksum, :size, :width, :height, @@ -174,7 +184,8 @@ defmodule PhoenixKit.Storage.File do :mime_type, :file_type, :ext, - :checksum, + :file_checksum, + :user_file_checksum, :size, :user_id ]) diff --git a/lib/phoenix_kit_web/controllers/upload_controller.ex b/lib/phoenix_kit_web/controllers/upload_controller.ex index ce5608f9d..41d64a801 100644 --- a/lib/phoenix_kit_web/controllers/upload_controller.ex +++ b/lib/phoenix_kit_web/controllers/upload_controller.ex @@ -150,19 +150,22 @@ defmodule PhoenixKitWeb.UploadController do defp process_upload(upload, user_id) do with {:ok, stat} <- File.stat(upload.path), - {:ok, file_hash} <- safe_calculate_file_hash(upload.path) do + {:ok, file_checksum} <- safe_calculate_file_hash(upload.path) do file_size = stat.size - # Check if file already exists - case Storage.get_file_by_hash(file_hash) do + # Calculate user-specific checksum for per-user duplicate detection + user_file_checksum = Storage.calculate_user_file_checksum(user_id, file_checksum) + + # Check if this user already uploaded this file + case Storage.get_file_by_user_checksum(user_file_checksum) do %StorageFile{} = existing_file -> - # File already exists, delete temp upload and return existing file + # File already exists for this user, delete temp upload and return existing file File.rm(upload.path) {:ok, existing_file.id} nil -> - # New file, proceed with upload - perform_upload(upload, user_id, file_size, file_hash) + # New file for this user, proceed with upload + perform_upload(upload, user_id, file_size, file_checksum) end else {:error, reason} -> {:error, reason} @@ -184,12 +187,12 @@ defmodule PhoenixKitWeb.UploadController do end end - defp perform_upload(upload, user_id, _file_size, file_hash) do + defp perform_upload(upload, user_id, _file_size, file_checksum) do file_type = determine_file_type(upload.content_type) ext = Path.extname(upload.filename) |> String.replace_leading(".", "") # Store in buckets with hierarchical path structure - case Storage.store_file_in_buckets(upload.path, file_type, user_id, file_hash, ext) do + case Storage.store_file_in_buckets(upload.path, file_type, user_id, file_checksum, ext) do {:ok, file} -> # Queue background job for variant generation %{file_id: file.id, user_id: user_id, filename: upload.filename}