Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions app/models/published_database.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# frozen_string_literal: true

# Model to hold the URLs of published SQLite databases of card data.
class PublishedDatabase < ApplicationRecord
end
8 changes: 8 additions & 0 deletions db/migrate/20260509183917_create_published_databases.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
class CreatePublishedDatabases < ActiveRecord::Migration[8.1]
def change
create_table :published_databases do |t|
t.string :url, null: false
t.timestamps
end
end
end
8 changes: 7 additions & 1 deletion db/schema.rb
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.

ActiveRecord::Schema[8.1].define(version: 2026_04_26_160000) do
ActiveRecord::Schema[8.1].define(version: 2026_05_09_183917) do
# These are extensions that must be enabled in order to support this database
enable_extension "pg_catalog.plpgsql"
enable_extension "pgcrypto"
Expand Down Expand Up @@ -266,6 +266,12 @@
t.index ["printing_id", "card_subtype_id"], name: "index_printings_card_subtypes_on_card_id_and_subtype_id"
end

create_table "published_databases", force: :cascade do |t|
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.string "url", null: false
end

create_table "restrictions", id: :string, force: :cascade do |t|
t.datetime "created_at", null: false
t.text "date_start", null: false
Expand Down
45 changes: 28 additions & 17 deletions lib/tasks/sqlite.rake
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ namespace :sqlite do
if upload
missing_vars = []
%w[DO_SPACES_KEY DO_SPACES_SECRET DO_SPACES_ENDPOINT DO_SPACES_REGION DO_SPACES_BUCKET
DO_SPACES_FOLDER].each do |var|
DO_SPACES_FOLDER DB_URL_PREFIX].each do |var|
missing_vars << var if ENV[var].nil? || ENV[var].strip.empty?
end

Expand Down Expand Up @@ -88,6 +88,9 @@ namespace :sqlite do
db_file = Rails.root.join('db', 'netrunnerdb.sqlite3')
FileUtils.rm_f(db_file)

# Store current Postgres connection config to restore once the SQLite interactions are complete.
pg_config = ActiveRecord::Base.connection_db_config

puts "Creating tables in SQLite db at #{db_file}..."
ActiveRecord::Base.establish_connection(
adapter: 'sqlite3',
Expand Down Expand Up @@ -141,9 +144,12 @@ namespace :sqlite do
end
end

gzip_path = compress_db(db_file)
puts 'Restoring Postgres connection...'
ActiveRecord::Base.establish_connection(pg_config)

db_details = compress_db(db_file)

upload_to_spaces(gzip_path) if upload
publish_db(db_details) if upload

puts 'Done.'
end
Expand Down Expand Up @@ -256,21 +262,23 @@ namespace :sqlite do

def compress_db(db_file)
puts 'Compressing database...'
timestamp = Time.now.to_i
gzip_path = "#{db_file}.#{timestamp}.gz"
r = { db_file: db_file, time: Time.zone.now }
r[:gzip_path] = "#{db_file}.#{r[:time].to_i}.gz"

Zlib::GzipWriter.open(gzip_path) do |gz|
Zlib::GzipWriter.open(r[:gzip_path]) do |gz|
File.open(db_file.to_s, 'rb') do |file|
while (chunk = file.read(1024 * 1024))
gz.write(chunk)
end
end
end
puts "Database compressed to #{gzip_path}"
gzip_path
puts "Database compressed to #{r[:gzip_path]}"

r
end

def upload_to_spaces(file_path)
def publish_db(output_details)
file_path = output_details[:gzip_path]
puts "Uploading #{file_path} to Digital Ocean Spaces..."
s3_client = Aws::S3::Client.new(
access_key_id: ENV.fetch('DO_SPACES_KEY'),
Expand All @@ -279,20 +287,23 @@ namespace :sqlite do
region: ENV.fetch('DO_SPACES_REGION')
)

bucket = ENV.fetch('DO_SPACES_BUCKET')
folder = ENV.fetch('DO_SPACES_FOLDER')
filename = File.basename(file_path)
key = folder ? File.join(folder, filename) : filename
file_path = File.join(ENV.fetch('DO_SPACES_FOLDER'), File.basename(file_path))

File.open(file_path, 'rb') do |file|
File.open(output_details[:db_file], 'rb') do |file|
s3_client.put_object(
bucket: bucket,
key: key,
bucket: ENV.fetch('DO_SPACES_BUCKET'),
key: file_path,
body: file,
acl: 'public-read',
content_type: 'application/gzip'
)
end
puts "Successfully #{file_path} uploaded to spaces: #{bucket}/#{key}"

published_db = PublishedDatabase.create!(
url: "#{ENV.fetch('DB_URL_PREFIX').gsub(%r{/$}, '')}/#{file_path}",
created_at: output_details[:time],
updated_at: output_details[:time]
)
puts "Successfully #{file_path} uploaded to spaces: #{published_db.url}"
end
end
11 changes: 11 additions & 0 deletions test/fixtures/published_databases.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
latest_db:
id: 23
url: https://download/netrunnerdb.sqlite3.1778352894.gz
created_at: <%= Time.utc(2026, 5, 9, 18, 54, 54).to_fs(:db) %>
updated_at: <%= Time.utc(2026, 5, 9, 18, 54, 54).to_fs(:db) %>

previous_db:
id: 22
url: https://download/netrunnerdb.sqlite3.1778003694.gz
created_at: <%= Time.utc(2026, 5, 5, 17, 54, 54).to_fs(:db) %>
updated_at: <%= Time.utc(2026, 5, 5, 17, 54, 54).to_fs(:db) %>
Loading