diff --git a/app/models/published_database.rb b/app/models/published_database.rb new file mode 100644 index 0000000..7d3863d --- /dev/null +++ b/app/models/published_database.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +# Model to hold the URLs of published SQLite databases of card data. +class PublishedDatabase < ApplicationRecord +end diff --git a/db/migrate/20260509183917_create_published_databases.rb b/db/migrate/20260509183917_create_published_databases.rb new file mode 100644 index 0000000..75a0b32 --- /dev/null +++ b/db/migrate/20260509183917_create_published_databases.rb @@ -0,0 +1,8 @@ +class CreatePublishedDatabases < ActiveRecord::Migration[8.1] + def change + create_table :published_databases do |t| + t.string :url, null: false + t.timestamps + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 3dada31..8869ee7 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.1].define(version: 2026_04_26_160000) do +ActiveRecord::Schema[8.1].define(version: 2026_05_09_183917) do # These are extensions that must be enabled in order to support this database enable_extension "pg_catalog.plpgsql" enable_extension "pgcrypto" @@ -266,6 +266,12 @@ t.index ["printing_id", "card_subtype_id"], name: "index_printings_card_subtypes_on_card_id_and_subtype_id" end + create_table "published_databases", force: :cascade do |t| + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.string "url", null: false + end + create_table "restrictions", id: :string, force: :cascade do |t| t.datetime "created_at", null: false t.text "date_start", null: false diff --git a/lib/tasks/sqlite.rake b/lib/tasks/sqlite.rake index 1d8d90b..def4b09 100644 --- a/lib/tasks/sqlite.rake +++ b/lib/tasks/sqlite.rake @@ -13,7 +13,7 @@ namespace :sqlite do if upload missing_vars = [] %w[DO_SPACES_KEY DO_SPACES_SECRET DO_SPACES_ENDPOINT DO_SPACES_REGION DO_SPACES_BUCKET - DO_SPACES_FOLDER].each do |var| + DO_SPACES_FOLDER DB_URL_PREFIX].each do |var| missing_vars << var if ENV[var].nil? || ENV[var].strip.empty? end @@ -88,6 +88,9 @@ namespace :sqlite do db_file = Rails.root.join('db', 'netrunnerdb.sqlite3') FileUtils.rm_f(db_file) + # Store current Postgres connection config to restore once the SQLite interactions are complete. + pg_config = ActiveRecord::Base.connection_db_config + puts "Creating tables in SQLite db at #{db_file}..." ActiveRecord::Base.establish_connection( adapter: 'sqlite3', @@ -141,9 +144,12 @@ namespace :sqlite do end end - gzip_path = compress_db(db_file) + puts 'Restoring Postgres connection...' + ActiveRecord::Base.establish_connection(pg_config) + + db_details = compress_db(db_file) - upload_to_spaces(gzip_path) if upload + publish_db(db_details) if upload puts 'Done.' end @@ -256,21 +262,23 @@ namespace :sqlite do def compress_db(db_file) puts 'Compressing database...' - timestamp = Time.now.to_i - gzip_path = "#{db_file}.#{timestamp}.gz" + r = { db_file: db_file, time: Time.zone.now } + r[:gzip_path] = "#{db_file}.#{r[:time].to_i}.gz" - Zlib::GzipWriter.open(gzip_path) do |gz| + Zlib::GzipWriter.open(r[:gzip_path]) do |gz| File.open(db_file.to_s, 'rb') do |file| while (chunk = file.read(1024 * 1024)) gz.write(chunk) end end end - puts "Database compressed to #{gzip_path}" - gzip_path + puts "Database compressed to #{r[:gzip_path]}" + + r end - def upload_to_spaces(file_path) + def publish_db(output_details) + file_path = output_details[:gzip_path] puts "Uploading #{file_path} to Digital Ocean Spaces..." s3_client = Aws::S3::Client.new( access_key_id: ENV.fetch('DO_SPACES_KEY'), @@ -279,20 +287,23 @@ namespace :sqlite do region: ENV.fetch('DO_SPACES_REGION') ) - bucket = ENV.fetch('DO_SPACES_BUCKET') - folder = ENV.fetch('DO_SPACES_FOLDER') - filename = File.basename(file_path) - key = folder ? File.join(folder, filename) : filename + file_path = File.join(ENV.fetch('DO_SPACES_FOLDER'), File.basename(file_path)) - File.open(file_path, 'rb') do |file| + File.open(output_details[:db_file], 'rb') do |file| s3_client.put_object( - bucket: bucket, - key: key, + bucket: ENV.fetch('DO_SPACES_BUCKET'), + key: file_path, body: file, acl: 'public-read', content_type: 'application/gzip' ) end - puts "Successfully #{file_path} uploaded to spaces: #{bucket}/#{key}" + + published_db = PublishedDatabase.create!( + url: "#{ENV.fetch('DB_URL_PREFIX').gsub(%r{/$}, '')}/#{file_path}", + created_at: output_details[:time], + updated_at: output_details[:time] + ) + puts "Successfully #{file_path} uploaded to spaces: #{published_db.url}" end end diff --git a/test/fixtures/published_databases.yml b/test/fixtures/published_databases.yml new file mode 100644 index 0000000..c0e7925 --- /dev/null +++ b/test/fixtures/published_databases.yml @@ -0,0 +1,11 @@ +latest_db: + id: 23 + url: https://download/netrunnerdb.sqlite3.1778352894.gz + created_at: <%= Time.utc(2026, 5, 9, 18, 54, 54).to_fs(:db) %> + updated_at: <%= Time.utc(2026, 5, 9, 18, 54, 54).to_fs(:db) %> + +previous_db: + id: 22 + url: https://download/netrunnerdb.sqlite3.1778003694.gz + created_at: <%= Time.utc(2026, 5, 5, 17, 54, 54).to_fs(:db) %> + updated_at: <%= Time.utc(2026, 5, 5, 17, 54, 54).to_fs(:db) %>