- <%= link_to "Jobs", sidekiq_web_url, class: "text-white underline hover:text-gray-100" %>
+ <%= link_to "Jobs", sidekiq_web_url, class: "text-white underline hover:text-gray-100" %>
diff --git a/app/views/pages/redis_configuration_error.html.erb b/app/views/pages/redis_configuration_error.html.erb
deleted file mode 100644
index 53c43387f4a..00000000000
--- a/app/views/pages/redis_configuration_error.html.erb
+++ /dev/null
@@ -1,59 +0,0 @@
-<% content_for :title, "Redis Configuration Required - Sure" %>
-
-
-
-
-
-
-
- <%= icon "alert-triangle", class: "w-8 h-8 text-red-600" %>
-
-
Redis Configuration Required
-
Your self-hosted Sure installation needs Redis to be properly configured.
-
-
-
-
-
-
- <%= icon "info", class: "w-5 h-5 text-amber-600 mt-0.5 mr-3 flex-shrink-0" %>
-
-
Why is Redis required?
-
Sure uses Redis to power Sidekiq background jobs for tasks like syncing account data, processing imports, and other background operations that keep your financial data up to date.
-
-
-
-
-
-
- <%= render DS::Link.new(
- text: "View Setup Guide",
- href: "https://github.com/we-promise/sure/blob/main/docs/hosting/docker.md",
- variant: "primary",
- size: "lg",
- icon: "external-link",
- full_width: true,
- target: "_blank",
- rel: "noopener noreferrer"
- ) %>
-
Follow our complete Docker setup guide to configure Redis
-
-
-
-
-
-
-
Once you've configured Redis, refresh this page to continue.
- <%= render DS::Button.new(
- text: "Refresh Page",
- variant: "secondary",
- icon: "refresh-cw",
- type: "button",
- full_width: true,
- onclick: "window.location.reload()"
- ) %>
-
-
-
-
-
diff --git a/bin/jobs b/bin/jobs
new file mode 100755
index 00000000000..dcf59f309ae
--- /dev/null
+++ b/bin/jobs
@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require_relative "../config/environment"
+require "solid_queue/cli"
+
+SolidQueue::Cli.start(ARGV)
diff --git a/compose.example.yml b/compose.example.yml
index 01a97368b0d..172461178d1 100644
--- a/compose.example.yml
+++ b/compose.example.yml
@@ -40,7 +40,6 @@ x-rails-env: &rails_env
RAILS_ASSUME_SSL: "false"
DB_HOST: db
DB_PORT: 5432
- REDIS_URL: redis://redis:6379/1
# NOTE: enabling OpenAI will incur costs when you use AI-related features in the app (chat, rules). Make sure you have set appropriate spend limits on your account before adding this.
OPENAI_ACCESS_TOKEN: ${OPENAI_ACCESS_TOKEN}
@@ -57,24 +56,6 @@ services:
depends_on:
db:
condition: service_healthy
- redis:
- condition: service_healthy
- networks:
- - sure_net
-
- worker:
- image: ghcr.io/we-promise/sure:latest
- command: bundle exec sidekiq
- volumes:
- - app-storage:/rails/storage
- restart: unless-stopped
- depends_on:
- db:
- condition: service_healthy
- redis:
- condition: service_healthy
- environment:
- <<: *rails_env
networks:
- sure_net
@@ -93,23 +74,9 @@ services:
networks:
- sure_net
- redis:
- image: redis:latest
- restart: unless-stopped
- volumes:
- - redis-data:/data
- healthcheck:
- test: [ "CMD", "redis-cli", "ping" ]
- interval: 5s
- timeout: 5s
- retries: 5
- networks:
- - sure_net
-
volumes:
app-storage:
postgres-data:
- redis-data:
networks:
sure_net:
diff --git a/config/application.rb b/config/application.rb
index 77d071173e1..9a2206ea717 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -41,5 +41,7 @@ class Application < Rails::Application
# Enable Rack::Attack middleware for API rate limiting
config.middleware.use Rack::Attack
+ MissionControl::Jobs.http_basic_auth_user = "mission_control_jobs"
+ MissionControl::Jobs.http_basic_auth_password = ENV["MISSION_CONTROL_JOBS_PASSWORD"] || "mission_control_jobs"
end
end
diff --git a/config/cable.yml b/config/cable.yml
index fc5a1eb8efa..7ca155ef873 100644
--- a/config/cable.yml
+++ b/config/cable.yml
@@ -1,10 +1,21 @@
+# Async adapter only works within the same process, so for manually triggering cable updates from a console,
+# and seeing results in the browser, you must do so from the web console (running inside the dev process),
+# not a terminal started via bin/rails console! Add "console" to any action or any ERB template view
+# to make the web console appear.
+
+default: &default
+ adapter: solid_cable
+ connects_to:
+ database:
+ writing: cable
+ polling_interval: 0.1.seconds
+ message_retention: 1.day
+
development:
- adapter: async
+ <<: *default
test:
adapter: test
production:
- adapter: redis
- url: <%= ENV.fetch("REDIS_URL") { "redis://localhost:6379/1" } %>
- channel_prefix: sure_production
+ <<: *default
diff --git a/config/cache.yml b/config/cache.yml
new file mode 100644
index 00000000000..fdb4f759653
--- /dev/null
+++ b/config/cache.yml
@@ -0,0 +1,17 @@
+default: &default
+ store_options:
+ # Cap age of oldest cache entry to fulfill retention policies
+ max_age: <%= 60.days.to_i %>
+ max_size: <%= 1.gigabytes %>
+ namespace: <%= Rails.env %>
+
+development: &development
+ <<: *default
+ database: cache
+
+test:
+ <<: *default
+
+production: &production
+ <<: *default
+ database: cache
\ No newline at end of file
diff --git a/config/database.yml b/config/database.yml
index 5b249238a33..9d8081311e2 100644
--- a/config/database.yml
+++ b/config/database.yml
@@ -8,13 +8,39 @@ default: &default
password: <%= ENV.fetch("POSTGRES_PASSWORD") { nil } %>
development:
- <<: *default
- database: <%= ENV.fetch("POSTGRES_DB") { "sure_development" } %>
+ primary:
+ <<: *default
+ database: <%= ENV.fetch("POSTGRES_DB") { "maybe_development" } %>
+ cache:
+ adapter: sqlite3
+ database: storage/development_cache.sqlite3
+ migrations_paths: db/cache_migrate
+ queue:
+ adapter: sqlite3
+ database: storage/development_queue.sqlite3
+ migrations_paths: db/queue_migrate
+ cable:
+ adapter: sqlite3
+ database: storage/development_cable.sqlite3
+ migrations_paths: db/cable_migrate
test:
<<: *default
- database: <%= ENV.fetch("POSTGRES_DB") { "sure_test" } %>
+ database: <%= ENV.fetch("POSTGRES_DB") { "maybe_test" } %>
production:
- <<: *default
- database: <%= ENV.fetch("POSTGRES_DB") { "sure_production" } %>
+ primary:
+ <<: *default
+ database: <%= ENV.fetch("POSTGRES_DB") { "maybe_production" } %>
+ cache:
+ adapter: sqlite3
+ database: storage/production_cache.sqlite3
+ migrations_paths: db/cache_migrate
+ queue:
+ adapter: sqlite3
+ database: storage/production_queue.sqlite3
+ migrations_paths: db/queue_migrate
+ cable:
+ adapter: sqlite3
+ database: storage/production_cable.sqlite3
+ migrations_paths: db/cable_migrate
diff --git a/config/environments/development.rb b/config/environments/development.rb
index 553da47e3e8..f8c5242ca61 100644
--- a/config/environments/development.rb
+++ b/config/environments/development.rb
@@ -67,6 +67,9 @@
config.assets.quiet = true
config.active_record.verbose_query_logs = true
config.active_job.verbose_enqueue_logs = true
+ config.active_job.queue_adapter = :solid_queue
+ config.solid_queue.connects_to = { database: { writing: :queue } }
+ config.solid_queue.logger = ActiveSupport::Logger.new(STDOUT)
# Raises error for missing translations.
# config.i18n.raise_on_missing_translations = true
diff --git a/config/environments/production.rb b/config/environments/production.rb
index 671b5239fe0..31f9a0a59b1 100644
--- a/config/environments/production.rb
+++ b/config/environments/production.rb
@@ -69,9 +69,7 @@
# want to log everything, set the level to "debug".
config.log_level = ENV.fetch("RAILS_LOG_LEVEL", "info")
- if ENV["CACHE_REDIS_URL"].present?
- config.cache_store = :redis_cache_store, { url: ENV["CACHE_REDIS_URL"] }
- end
+ config.cache_store = :solid_cache_store
config.action_mailer.perform_caching = false
config.action_mailer.deliver_later_queue_name = :high_priority
@@ -107,6 +105,6 @@
# Skip DNS rebinding protection for the default health check endpoint.
# config.host_authorization = { exclude: ->(request) { request.path == "/up" } }
- # set REDIS_URL for Sidekiq to use Redis
- config.active_job.queue_adapter = :sidekiq
+ config.active_job.queue_adapter = :solid_queue
+ config.solid_queue.connects_to = { database: { writing: :queue } }
end
diff --git a/config/initializers/rails_db.rb b/config/initializers/rails_db.rb
new file mode 100644
index 00000000000..359d9d7b79f
--- /dev/null
+++ b/config/initializers/rails_db.rb
@@ -0,0 +1,30 @@
+if Object.const_defined?("RailsDb")
+ RailsDb.setup do |config|
+ # # enabled or not
+ config.enabled = true
+
+ # # automatic engine routes mounting
+ config.automatic_routes_mount = true
+
+ # set tables which you want to hide ONLY
+ # config.black_list_tables = [ "users", "accounts" ]
+
+ # set tables which you want to show ONLY
+ # config.white_list_tables = [ "posts", "comments" ]
+
+ # # Enable http basic authentication
+ config.http_basic_authentication_enabled = true
+
+ # # Enable http basic authentication
+ config.http_basic_authentication_user_name = "rails_db"
+
+ # # Enable http basic authentication
+ config.http_basic_authentication_password = ENV.fetch("RAILS_DB_PASSWORD", "rails_db")
+
+ # # Enable verify access proc
+ # config.verify_access_proc = proc { |controller| true }
+
+ # # Sandbox mode (only read-only operations)
+ # config.sandbox = false
+ end
+end
diff --git a/config/initializers/sentry.rb b/config/initializers/sentry.rb
index d964a09b85d..1a6e9143f78 100644
--- a/config/initializers/sentry.rb
+++ b/config/initializers/sentry.rb
@@ -16,5 +16,7 @@
config.profiles_sample_rate = 0.25
config.profiler_class = Sentry::Vernier::Profiler
+ config.enable_logs = true
+ config.enabled_patches = [ :logger, :http, :puma ]
end
end
diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb
deleted file mode 100644
index b80a8fddf23..00000000000
--- a/config/initializers/sidekiq.rb
+++ /dev/null
@@ -1,16 +0,0 @@
-require "sidekiq/web"
-
-if Rails.env.production?
- Sidekiq::Web.use(Rack::Auth::Basic) do |username, password|
- configured_username = ::Digest::SHA256.hexdigest(ENV.fetch("SIDEKIQ_WEB_USERNAME", "sure"))
- configured_password = ::Digest::SHA256.hexdigest(ENV.fetch("SIDEKIQ_WEB_PASSWORD", "sure"))
-
- ActiveSupport::SecurityUtils.secure_compare(::Digest::SHA256.hexdigest(username), configured_username) &&
- ActiveSupport::SecurityUtils.secure_compare(::Digest::SHA256.hexdigest(password), configured_password)
- end
-end
-
-Sidekiq::Cron.configure do |config|
- # 10 min "catch-up" window in case worker process is re-deploying when cron tick occurs
- config.reschedule_grace_period = 600
-end
diff --git a/config/puma.rb b/config/puma.rb
index 47a2362e275..160bbef9187 100644
--- a/config/puma.rb
+++ b/config/puma.rb
@@ -47,6 +47,8 @@
# Allow puma to be restarted by `bin/rails restart` command.
plugin :tmp_restart
+plugin :solid_queue
+
pidfile ENV["PIDFILE"] if ENV["PIDFILE"]
if rails_env == "development"
diff --git a/config/queue.yml b/config/queue.yml
new file mode 100644
index 00000000000..9eace59c41e
--- /dev/null
+++ b/config/queue.yml
@@ -0,0 +1,18 @@
+default: &default
+ dispatchers:
+ - polling_interval: 1
+ batch_size: 500
+ workers:
+ - queues: "*"
+ threads: 3
+ processes: <%= ENV.fetch("JOB_CONCURRENCY", 1) %>
+ polling_interval: 0.1
+
+development:
+ <<: *default
+
+test:
+ <<: *default
+
+production:
+ <<: *default
diff --git a/config/recurring.yml b/config/recurring.yml
new file mode 100644
index 00000000000..1b5043f2ea5
--- /dev/null
+++ b/config/recurring.yml
@@ -0,0 +1,34 @@
+# examples:
+# periodic_cleanup:
+# class: CleanSoftDeletedRecordsJob
+# queue: background
+# args: [ 1000, { batch_size: 500 } ]
+# schedule: every hour
+# periodic_cleanup_with_command:
+# command: "SoftDeletedRecord.due.delete_all"
+# priority: 2
+# schedule: at 5am every day
+
+development:
+ clear_solid_queue_finished_jobs:
+ command: "SolidQueue::Job.clear_finished_in_batches(sleep_between_batches: 0.3)"
+ schedule: every hour at minute 12
+
+ import_market_data:
+ class: "ImportMarketDataJob"
+ queue: "scheduled"
+ args: { mode: "full", clear_cache: false }
+ schedule: "0 22 * * 1-5" # 5:00 PM EST / 6:00 PM EDT (NY time) Monday through Friday
+ description: "Imports market data daily at 5:00 PM EST (1 hour after market close)"
+
+ clean_syncs:
+ schedule: "0 * * * *" # every hour
+ class: "SyncCleanerJob"
+ queue: "scheduled"
+ description: "Cleans up stale syncs"
+
+ run_security_health_checks:
+ schedule: "0 2 * * 1-5" # 2:00 AM EST / 3:00 AM EDT (NY time) Monday through Friday
+ class: "SecurityHealthCheckJob"
+ queue: "scheduled"
+ description: "Runs security health checks to detect issues with security data"
diff --git a/config/routes.rb b/config/routes.rb
index fbdecead056..f5a8547e278 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -1,6 +1,3 @@
-require "sidekiq/web"
-require "sidekiq/cron/web"
-
Rails.application.routes.draw do
use_doorkeeper
# MFA routes
@@ -12,8 +9,7 @@
mount Lookbook::Engine, at: "/design-system"
- # Uses basic auth - see config/initializers/sidekiq.rb
- mount Sidekiq::Web => "/sidekiq"
+ mount MissionControl::Jobs::Engine, at: "/jobs"
# AI chats
resources :chats do
@@ -303,8 +299,6 @@
post "stripe"
end
- get "redis-configuration-error", to: "pages#redis_configuration_error"
-
# Reveal health status on /up that returns 200 if the app boots with no exceptions, otherwise 500.
# Can be used by load balancers and uptime monitors to verify that the app is live.
get "up" => "rails/health#show", as: :rails_health_check
diff --git a/db/cable_schema.rb b/db/cable_schema.rb
new file mode 100644
index 00000000000..de80d1068a8
--- /dev/null
+++ b/db/cable_schema.rb
@@ -0,0 +1,23 @@
+# This file is auto-generated from the current state of the database. Instead
+# of editing this file, please use the migrations feature of Active Record to
+# incrementally modify your database, and then regenerate this schema definition.
+#
+# This file is the source Rails uses to define your schema when running `bin/rails
+# db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to
+# be faster and is potentially less error prone than running all of your
+# migrations from scratch. Old migrations may fail to apply correctly if those
+# migrations use external dependencies or application code.
+#
+# It's strongly recommended that you check this file into your version control system.
+
+ActiveRecord::Schema[7.2].define(version: 1) do
+ create_table "solid_cable_messages", force: :cascade do |t|
+ t.binary "channel", limit: 1024, null: false
+ t.binary "payload", limit: 536870912, null: false
+ t.datetime "created_at", null: false
+ t.integer "channel_hash", limit: 8, null: false
+ t.index ["channel"], name: "index_solid_cable_messages_on_channel"
+ t.index ["channel_hash"], name: "index_solid_cable_messages_on_channel_hash"
+ t.index ["created_at"], name: "index_solid_cable_messages_on_created_at"
+ end
+end
diff --git a/db/cache_schema.rb b/db/cache_schema.rb
new file mode 100644
index 00000000000..fec220f6d64
--- /dev/null
+++ b/db/cache_schema.rb
@@ -0,0 +1,24 @@
+# This file is auto-generated from the current state of the database. Instead
+# of editing this file, please use the migrations feature of Active Record to
+# incrementally modify your database, and then regenerate this schema definition.
+#
+# This file is the source Rails uses to define your schema when running `bin/rails
+# db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to
+# be faster and is potentially less error prone than running all of your
+# migrations from scratch. Old migrations may fail to apply correctly if those
+# migrations use external dependencies or application code.
+#
+# It's strongly recommended that you check this file into your version control system.
+
+ActiveRecord::Schema[7.2].define(version: 1) do
+ create_table "solid_cache_entries", force: :cascade do |t|
+ t.binary "key", limit: 1024, null: false
+ t.binary "value", limit: 536870912, null: false
+ t.datetime "created_at", null: false
+ t.integer "key_hash", limit: 8, null: false
+ t.integer "byte_size", limit: 4, null: false
+ t.index ["byte_size"], name: "index_solid_cache_entries_on_byte_size"
+ t.index ["key_hash", "byte_size"], name: "index_solid_cache_entries_on_key_hash_and_byte_size"
+ t.index ["key_hash"], name: "index_solid_cache_entries_on_key_hash", unique: true
+ end
+end
diff --git a/db/queue_schema.rb b/db/queue_schema.rb
new file mode 100644
index 00000000000..df3358c2874
--- /dev/null
+++ b/db/queue_schema.rb
@@ -0,0 +1,141 @@
+# This file is auto-generated from the current state of the database. Instead
+# of editing this file, please use the migrations feature of Active Record to
+# incrementally modify your database, and then regenerate this schema definition.
+#
+# This file is the source Rails uses to define your schema when running `bin/rails
+# db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to
+# be faster and is potentially less error prone than running all of your
+# migrations from scratch. Old migrations may fail to apply correctly if those
+# migrations use external dependencies or application code.
+#
+# It's strongly recommended that you check this file into your version control system.
+
+ActiveRecord::Schema[7.2].define(version: 1) do
+ create_table "solid_queue_blocked_executions", force: :cascade do |t|
+ t.bigint "job_id", null: false
+ t.string "queue_name", null: false
+ t.integer "priority", default: 0, null: false
+ t.string "concurrency_key", null: false
+ t.datetime "expires_at", null: false
+ t.datetime "created_at", null: false
+ t.index ["concurrency_key", "priority", "job_id"], name: "index_solid_queue_blocked_executions_for_release"
+ t.index ["expires_at", "concurrency_key"], name: "index_solid_queue_blocked_executions_for_maintenance"
+ t.index ["job_id"], name: "index_solid_queue_blocked_executions_on_job_id", unique: true
+ end
+
+ create_table "solid_queue_claimed_executions", force: :cascade do |t|
+ t.bigint "job_id", null: false
+ t.bigint "process_id"
+ t.datetime "created_at", null: false
+ t.index ["job_id"], name: "index_solid_queue_claimed_executions_on_job_id", unique: true
+ t.index ["process_id", "job_id"], name: "index_solid_queue_claimed_executions_on_process_id_and_job_id"
+ end
+
+ create_table "solid_queue_failed_executions", force: :cascade do |t|
+ t.bigint "job_id", null: false
+ t.text "error"
+ t.datetime "created_at", null: false
+ t.index ["job_id"], name: "index_solid_queue_failed_executions_on_job_id", unique: true
+ end
+
+ create_table "solid_queue_jobs", force: :cascade do |t|
+ t.string "queue_name", null: false
+ t.string "class_name", null: false
+ t.text "arguments"
+ t.integer "priority", default: 0, null: false
+ t.string "active_job_id"
+ t.datetime "scheduled_at"
+ t.datetime "finished_at"
+ t.string "concurrency_key"
+ t.datetime "created_at", null: false
+ t.datetime "updated_at", null: false
+ t.index ["active_job_id"], name: "index_solid_queue_jobs_on_active_job_id"
+ t.index ["class_name"], name: "index_solid_queue_jobs_on_class_name"
+ t.index ["finished_at"], name: "index_solid_queue_jobs_on_finished_at"
+ t.index ["queue_name", "finished_at"], name: "index_solid_queue_jobs_for_filtering"
+ t.index ["scheduled_at", "finished_at"], name: "index_solid_queue_jobs_for_alerting"
+ end
+
+ create_table "solid_queue_pauses", force: :cascade do |t|
+ t.string "queue_name", null: false
+ t.datetime "created_at", null: false
+ t.index ["queue_name"], name: "index_solid_queue_pauses_on_queue_name", unique: true
+ end
+
+ create_table "solid_queue_processes", force: :cascade do |t|
+ t.string "kind", null: false
+ t.datetime "last_heartbeat_at", null: false
+ t.bigint "supervisor_id"
+ t.integer "pid", null: false
+ t.string "hostname"
+ t.text "metadata"
+ t.datetime "created_at", null: false
+ t.string "name", null: false
+ t.index ["last_heartbeat_at"], name: "index_solid_queue_processes_on_last_heartbeat_at"
+ t.index ["name", "supervisor_id"], name: "index_solid_queue_processes_on_name_and_supervisor_id", unique: true
+ t.index ["supervisor_id"], name: "index_solid_queue_processes_on_supervisor_id"
+ end
+
+ create_table "solid_queue_ready_executions", force: :cascade do |t|
+ t.bigint "job_id", null: false
+ t.string "queue_name", null: false
+ t.integer "priority", default: 0, null: false
+ t.datetime "created_at", null: false
+ t.index ["job_id"], name: "index_solid_queue_ready_executions_on_job_id", unique: true
+ t.index ["priority", "job_id"], name: "index_solid_queue_poll_all"
+ t.index ["queue_name", "priority", "job_id"], name: "index_solid_queue_poll_by_queue"
+ end
+
+ create_table "solid_queue_recurring_executions", force: :cascade do |t|
+ t.bigint "job_id", null: false
+ t.string "task_key", null: false
+ t.datetime "run_at", null: false
+ t.datetime "created_at", null: false
+ t.index ["job_id"], name: "index_solid_queue_recurring_executions_on_job_id", unique: true
+ t.index ["task_key", "run_at"], name: "index_solid_queue_recurring_executions_on_task_key_and_run_at", unique: true
+ end
+
+ create_table "solid_queue_recurring_tasks", force: :cascade do |t|
+ t.string "key", null: false
+ t.string "schedule", null: false
+ t.string "command", limit: 2048
+ t.string "class_name"
+ t.text "arguments"
+ t.string "queue_name"
+ t.integer "priority", default: 0
+ t.boolean "static", default: true, null: false
+ t.text "description"
+ t.datetime "created_at", null: false
+ t.datetime "updated_at", null: false
+ t.index ["key"], name: "index_solid_queue_recurring_tasks_on_key", unique: true
+ t.index ["static"], name: "index_solid_queue_recurring_tasks_on_static"
+ end
+
+ create_table "solid_queue_scheduled_executions", force: :cascade do |t|
+ t.bigint "job_id", null: false
+ t.string "queue_name", null: false
+ t.integer "priority", default: 0, null: false
+ t.datetime "scheduled_at", null: false
+ t.datetime "created_at", null: false
+ t.index ["job_id"], name: "index_solid_queue_scheduled_executions_on_job_id", unique: true
+ t.index ["scheduled_at", "priority", "job_id"], name: "index_solid_queue_dispatch_all"
+ end
+
+ create_table "solid_queue_semaphores", force: :cascade do |t|
+ t.string "key", null: false
+ t.integer "value", default: 1, null: false
+ t.datetime "expires_at", null: false
+ t.datetime "created_at", null: false
+ t.datetime "updated_at", null: false
+ t.index ["expires_at"], name: "index_solid_queue_semaphores_on_expires_at"
+ t.index ["key", "value"], name: "index_solid_queue_semaphores_on_key_and_value"
+ t.index ["key"], name: "index_solid_queue_semaphores_on_key", unique: true
+ end
+
+ add_foreign_key "solid_queue_blocked_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade
+ add_foreign_key "solid_queue_claimed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade
+ add_foreign_key "solid_queue_failed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade
+ add_foreign_key "solid_queue_ready_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade
+ add_foreign_key "solid_queue_recurring_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade
+ add_foreign_key "solid_queue_scheduled_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade
+end
diff --git a/db/schema.rb b/db/schema.rb
index 955a0cfd343..8724b0deaf3 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -254,8 +254,11 @@
t.index ["account_id", "date"], name: "index_entries_on_account_id_and_date"
t.index ["account_id", "source", "external_id"], name: "index_entries_on_account_source_and_external_id", unique: true, where: "((external_id IS NOT NULL) AND (source IS NOT NULL))"
t.index ["account_id"], name: "index_entries_on_account_id"
+ t.index ["amount"], name: "index_entries_on_amount"
t.index ["date"], name: "index_entries_on_date"
+ t.index ["entryable_id", "entryable_type"], name: "index_entries_on_entryable"
t.index ["entryable_type"], name: "index_entries_on_entryable_type"
+ t.index ["excluded"], name: "index_entries_on_excluded"
t.index ["import_id"], name: "index_entries_on_import_id"
end
@@ -266,6 +269,7 @@
t.date "date", null: false
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
+ t.index ["date", "from_currency", "to_currency"], name: "index_exchange_rates_on_date_and_currencies"
t.index ["from_currency", "to_currency", "date"], name: "index_exchange_rates_on_base_converted_date_unique", unique: true
t.index ["from_currency"], name: "index_exchange_rates_on_from_currency"
t.index ["to_currency"], name: "index_exchange_rates_on_to_currency"
@@ -874,6 +878,7 @@
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.index ["tag_id"], name: "index_taggings_on_tag_id"
+ t.index ["taggable_id", "taggable_type"], name: "index_taggings_on_taggable_id_and_type"
t.index ["taggable_type", "taggable_id"], name: "index_taggings_on_taggable"
end
diff --git a/docker-compose.yaml b/docker-compose.yaml
new file mode 100644
index 00000000000..af2d736dd2e
--- /dev/null
+++ b/docker-compose.yaml
@@ -0,0 +1,29 @@
+x-db-env: &db_env
+ POSTGRES_USER: ${POSTGRES_USER:-maybe_user}
+ POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-maybe_password}
+ POSTGRES_DB: ${POSTGRES_DB:-maybe-development}
+
+services:
+ db:
+ image: postgres:16
+ restart: unless-stopped
+ volumes:
+ - maybe-dev-postgres-data:/var/lib/postgresql/data
+ ports:
+ - 5432:5432
+ environment:
+ <<: *db_env
+ healthcheck:
+ test: [ "CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB" ]
+ interval: 5s
+ timeout: 5s
+ retries: 5
+ networks:
+ - sure_net
+
+volumes:
+ maybe-dev-postgres-data:
+
+networks:
+ sure_net:
+ driver: bridge
diff --git a/docs/api/chats.md b/docs/api/chats.md
index 3bc8ea43668..ff7a3ca8512 100644
--- a/docs/api/chats.md
+++ b/docs/api/chats.md
@@ -221,7 +221,6 @@ Common error codes:
- `forbidden` - Insufficient permissions or AI not enabled
- `not_found` - Resource not found
- `unprocessable_entity` - Invalid request data
-- `rate_limit_exceeded` - Too many requests
## Rate Limits
diff --git a/test/controllers/api/v1/base_controller_test.rb b/test/controllers/api/v1/base_controller_test.rb
index af9e7066dba..036b356c7cf 100644
--- a/test/controllers/api/v1/base_controller_test.rb
+++ b/test/controllers/api/v1/base_controller_test.rb
@@ -22,14 +22,6 @@ class Api::V1::BaseControllerTest < ActionDispatch::IntegrationTest
display_key: @plain_api_key,
scopes: [ "read_write" ]
)
-
- # Clear any existing rate limit data
- Redis.new.del("api_rate_limit:#{@api_key.id}")
- end
-
- teardown do
- # Clean up Redis data after each test
- Redis.new.del("api_rate_limit:#{@api_key.id}")
end
test "should require authentication" do
@@ -325,106 +317,6 @@ class Api::V1::BaseControllerTest < ActionDispatch::IntegrationTest
assert_equal "forbidden", response_body["error"]
end
- test "should include rate limit headers on successful API key requests" do
- get "/api/v1/test", headers: { "X-Api-Key" => @plain_api_key }
-
- assert_response :success
- assert_not_nil response.headers["X-RateLimit-Limit"]
- assert_not_nil response.headers["X-RateLimit-Remaining"]
- assert_not_nil response.headers["X-RateLimit-Reset"]
-
- assert_equal "100", response.headers["X-RateLimit-Limit"]
- assert_equal "99", response.headers["X-RateLimit-Remaining"]
- end
-
- test "should increment rate limit count with each request" do
- # First request
- get "/api/v1/test", headers: { "X-Api-Key" => @plain_api_key }
- assert_response :success
- assert_equal "99", response.headers["X-RateLimit-Remaining"]
-
- # Second request
- get "/api/v1/test", headers: { "X-Api-Key" => @plain_api_key }
- assert_response :success
- assert_equal "98", response.headers["X-RateLimit-Remaining"]
- end
-
- test "should return 429 when rate limit exceeded" do
- # Make 100 requests to exhaust the rate limit
- 100.times do
- get "/api/v1/test", headers: { "X-Api-Key" => @plain_api_key }
- assert_response :success
- end
-
- # 101st request should be rate limited
- get "/api/v1/test", headers: { "X-Api-Key" => @plain_api_key }
- assert_response :too_many_requests
-
- response_body = JSON.parse(response.body)
- assert_equal "rate_limit_exceeded", response_body["error"]
- assert_includes response_body["message"], "Rate limit exceeded"
-
- # Check response headers
- assert_equal "100", response.headers["X-RateLimit-Limit"]
- assert_equal "0", response.headers["X-RateLimit-Remaining"]
- assert_not_nil response.headers["X-RateLimit-Reset"]
- assert_not_nil response.headers["Retry-After"]
- end
-
- test "should not apply rate limiting to OAuth requests" do
- # This would need to be implemented based on your OAuth setup
- # For now, just verify that requests without API keys don't trigger rate limiting
- get "/api/v1/test"
- assert_response :unauthorized
-
- # Should not have rate limit headers for unauthorized requests
- assert_nil response.headers["X-RateLimit-Limit"]
- end
-
- test "should provide detailed rate limit information in 429 response" do
- # Exhaust the rate limit
- 100.times do
- get "/api/v1/test", headers: { "X-Api-Key" => @plain_api_key }
- end
-
- # Make the rate-limited request
- get "/api/v1/test", headers: { "X-Api-Key" => @plain_api_key }
- assert_response :too_many_requests
-
- response_body = JSON.parse(response.body)
- assert_equal "rate_limit_exceeded", response_body["error"]
- assert response_body["details"]["limit"] == 100
- assert response_body["details"]["current"] >= 100
- assert response_body["details"]["reset_in_seconds"] > 0
- end
-
- test "rate limiting should be per API key" do
- # Create a second user for independent API keys
- other_user = users(:family_member)
- other_api_key = ApiKey.create!(
- user: other_user,
- name: "Other Test API Key",
- scopes: [ "read" ],
- display_key: "other_rate_test_#{SecureRandom.hex(8)}"
- )
-
- begin
- # Make 50 requests with first API key
- 50.times do
- get "/api/v1/test", headers: { "X-Api-Key" => @plain_api_key }
- assert_response :success
- end
-
- # Should still be able to make requests with second API key
- get "/api/v1/test", headers: { "X-Api-Key" => other_api_key.display_key }
- assert_response :success
- assert_equal "99", response.headers["X-RateLimit-Remaining"]
- ensure
- Redis.new.del("api_rate_limit:#{other_api_key.id}")
- other_api_key.destroy
- end
- end
-
private
def capture_log(&block)
diff --git a/test/controllers/api/v1/transactions_controller_test.rb b/test/controllers/api/v1/transactions_controller_test.rb
index b028562a632..dbd355e2d11 100644
--- a/test/controllers/api/v1/transactions_controller_test.rb
+++ b/test/controllers/api/v1/transactions_controller_test.rb
@@ -27,10 +27,6 @@ class Api::V1::TransactionsControllerTest < ActionDispatch::IntegrationTest
display_key: "test_ro_#{SecureRandom.hex(8)}",
source: "mobile" # Use different source to allow multiple keys
)
-
- # Clear any existing rate limit data
- Redis.new.del("api_rate_limit:#{@api_key.id}")
- Redis.new.del("api_rate_limit:#{@read_only_api_key.id}")
end
# INDEX action tests
diff --git a/test/controllers/api/v1/usage_controller_test.rb b/test/controllers/api/v1/usage_controller_test.rb
index 27235826139..e169d51b385 100644
--- a/test/controllers/api/v1/usage_controller_test.rb
+++ b/test/controllers/api/v1/usage_controller_test.rb
@@ -12,14 +12,6 @@ class Api::V1::UsageControllerTest < ActionDispatch::IntegrationTest
scopes: [ "read" ],
display_key: "usage_test_#{SecureRandom.hex(8)}"
)
-
- # Clear any existing rate limit data
- Redis.new.del("api_rate_limit:#{@api_key.id}")
- end
-
- teardown do
- # Clean up Redis data after each test
- Redis.new.del("api_rate_limit:#{@api_key.id}")
end
test "should return usage information for API key authentication" do
@@ -40,14 +32,6 @@ class Api::V1::UsageControllerTest < ActionDispatch::IntegrationTest
assert_equal [ "read" ], response_body["api_key"]["scopes"]
assert_not_nil response_body["api_key"]["last_used_at"]
assert_not_nil response_body["api_key"]["created_at"]
-
- # Check rate limit information
- assert_equal "standard", response_body["rate_limit"]["tier"]
- assert_equal 100, response_body["rate_limit"]["limit"]
- assert_equal 4, response_body["rate_limit"]["current_count"] # 3 test requests + 1 usage request
- assert_equal 96, response_body["rate_limit"]["remaining"]
- assert response_body["rate_limit"]["reset_in_seconds"] > 0
- assert_not_nil response_body["rate_limit"]["reset_at"]
end
test "should require read scope for usage endpoint" do
@@ -68,7 +52,6 @@ class Api::V1::UsageControllerTest < ActionDispatch::IntegrationTest
response_body = JSON.parse(response.body)
assert_equal "insufficient_scope", response_body["error"]
ensure
- Redis.new.del("api_rate_limit:#{api_key_no_read.id}")
api_key_no_read.destroy
end
end
@@ -79,58 +62,4 @@ class Api::V1::UsageControllerTest < ActionDispatch::IntegrationTest
get "/api/v1/usage"
assert_response :unauthorized
end
-
- test "should update usage count when accessing usage endpoint" do
- # Check initial state
- get "/api/v1/usage", headers: { "X-Api-Key" => @api_key.display_key }
- assert_response :success
-
- response_body = JSON.parse(response.body)
- first_count = response_body["rate_limit"]["current_count"]
-
- # Make another usage request
- get "/api/v1/usage", headers: { "X-Api-Key" => @api_key.display_key }
- assert_response :success
-
- response_body = JSON.parse(response.body)
- second_count = response_body["rate_limit"]["current_count"]
-
- assert_equal first_count + 1, second_count
- end
-
- test "should include rate limit headers in usage response" do
- get "/api/v1/usage", headers: { "X-Api-Key" => @api_key.display_key }
- assert_response :success
-
- assert_not_nil response.headers["X-RateLimit-Limit"]
- assert_not_nil response.headers["X-RateLimit-Remaining"]
- assert_not_nil response.headers["X-RateLimit-Reset"]
-
- assert_equal "100", response.headers["X-RateLimit-Limit"]
- assert_equal "99", response.headers["X-RateLimit-Remaining"]
- end
-
- test "should work correctly when approaching rate limit" do
- # Make 98 requests to get close to the limit
- 98.times do
- get "/api/v1/test", headers: { "X-Api-Key" => @api_key.display_key }
- assert_response :success
- end
-
- # Check usage - this should be request 99
- get "/api/v1/usage", headers: { "X-Api-Key" => @api_key.display_key }
- assert_response :success
-
- response_body = JSON.parse(response.body)
- assert_equal 99, response_body["rate_limit"]["current_count"]
- assert_equal 1, response_body["rate_limit"]["remaining"]
-
- # One more request should hit the limit
- get "/api/v1/test", headers: { "X-Api-Key" => @api_key.display_key }
- assert_response :success
-
- # Now we should be rate limited
- get "/api/v1/usage", headers: { "X-Api-Key" => @api_key.display_key }
- assert_response :too_many_requests
- end
end
diff --git a/test/services/api_rate_limiter_test.rb b/test/services/api_rate_limiter_test.rb
deleted file mode 100644
index 8afc6bb9a8e..00000000000
--- a/test/services/api_rate_limiter_test.rb
+++ /dev/null
@@ -1,138 +0,0 @@
-require "test_helper"
-
-class ApiRateLimiterTest < ActiveSupport::TestCase
- setup do
- @user = users(:family_admin)
- # Destroy any existing active API keys for this user
- @user.api_keys.active.destroy_all
-
- @api_key = ApiKey.create!(
- user: @user,
- name: "Rate Limiter Test Key",
- scopes: [ "read" ],
- display_key: "rate_limiter_test_#{SecureRandom.hex(8)}"
- )
- @rate_limiter = ApiRateLimiter.new(@api_key)
-
- # Clear any existing rate limit data
- Redis.new.del("api_rate_limit:#{@api_key.id}")
- end
-
- teardown do
- # Clean up Redis data after each test
- Redis.new.del("api_rate_limit:#{@api_key.id}")
- end
-
- test "should have default rate limit" do
- assert_equal 100, @rate_limiter.rate_limit
- end
-
- test "should start with zero request count" do
- assert_equal 0, @rate_limiter.current_count
- end
-
- test "should not be rate limited initially" do
- assert_not @rate_limiter.rate_limit_exceeded?
- end
-
- test "should increment request count" do
- assert_equal 0, @rate_limiter.current_count
-
- @rate_limiter.increment_request_count!
- assert_equal 1, @rate_limiter.current_count
-
- @rate_limiter.increment_request_count!
- assert_equal 2, @rate_limiter.current_count
- end
-
- test "should be rate limited when exceeding limit" do
- # Simulate reaching the rate limit
- 100.times { @rate_limiter.increment_request_count! }
-
- assert_equal 100, @rate_limiter.current_count
- assert @rate_limiter.rate_limit_exceeded?
- end
-
- test "should provide correct usage info" do
- 5.times { @rate_limiter.increment_request_count! }
-
- usage_info = @rate_limiter.usage_info
-
- assert_equal 5, usage_info[:current_count]
- assert_equal 100, usage_info[:rate_limit]
- assert_equal 95, usage_info[:remaining]
- assert_equal :standard, usage_info[:tier]
- assert usage_info[:reset_time] > 0
- assert usage_info[:reset_time] <= 3600
- end
-
- test "should calculate remaining requests correctly" do
- 10.times { @rate_limiter.increment_request_count! }
-
- usage_info = @rate_limiter.usage_info
- assert_equal 90, usage_info[:remaining]
- end
-
- test "should have zero remaining when at limit" do
- 100.times { @rate_limiter.increment_request_count! }
-
- usage_info = @rate_limiter.usage_info
- assert_equal 0, usage_info[:remaining]
- end
-
- test "should have zero remaining when over limit" do
- 105.times { @rate_limiter.increment_request_count! }
-
- usage_info = @rate_limiter.usage_info
- assert_equal 0, usage_info[:remaining]
- end
-
- test "class method usage_for should work without incrementing" do
- 5.times { @rate_limiter.increment_request_count! }
-
- usage_info = ApiRateLimiter.usage_for(@api_key)
- assert_equal 5, usage_info[:current_count]
-
- # Should not increment when just checking usage
- usage_info_again = ApiRateLimiter.usage_for(@api_key)
- assert_equal 5, usage_info_again[:current_count]
- end
-
- test "should handle multiple API keys separately" do
- # Create a different user for the second API key
- other_user = users(:family_member)
- other_api_key = ApiKey.create!(
- user: other_user,
- name: "Other API Key",
- scopes: [ "read_write" ],
- display_key: "rate_limiter_other_#{SecureRandom.hex(8)}"
- )
-
- other_rate_limiter = ApiRateLimiter.new(other_api_key)
-
- @rate_limiter.increment_request_count!
- other_rate_limiter.increment_request_count!
- other_rate_limiter.increment_request_count!
-
- assert_equal 1, @rate_limiter.current_count
- assert_equal 2, other_rate_limiter.current_count
- ensure
- Redis.new.del("api_rate_limit:#{other_api_key.id}")
- other_api_key.destroy
- end
-
- test "should calculate reset time correctly" do
- reset_time = @rate_limiter.reset_time
-
- # Reset time should be within the current hour
- assert reset_time > 0
- assert reset_time <= 3600
-
- # Should be roughly the time until the next hour
- current_time = Time.current.to_i
- next_window = ((current_time / 3600) + 1) * 3600
- expected_reset = next_window - current_time
-
- assert_in_delta expected_reset, reset_time, 1
- end
-end
diff --git a/test/services/noop_api_rate_limiter_test.rb b/test/services/noop_api_rate_limiter_test.rb
deleted file mode 100644
index 9c7105b12c8..00000000000
--- a/test/services/noop_api_rate_limiter_test.rb
+++ /dev/null
@@ -1,58 +0,0 @@
-require "test_helper"
-
-class NoopApiRateLimiterTest < ActiveSupport::TestCase
- setup do
- @user = users(:family_admin)
- # Clean up any existing API keys for this user to ensure tests start fresh
- @user.api_keys.destroy_all
-
- @api_key = ApiKey.create!(
- user: @user,
- name: "Noop Rate Limiter Test Key",
- scopes: [ "read" ],
- display_key: "noop_rate_limiter_test_#{SecureRandom.hex(8)}"
- )
- @rate_limiter = NoopApiRateLimiter.new(@api_key)
- end
-
- test "should never be rate limited" do
- assert_not @rate_limiter.rate_limit_exceeded?
- end
-
- test "should not increment request count" do
- @rate_limiter.increment_request_count!
- assert_equal 0, @rate_limiter.current_count
- end
-
- test "should always have zero request count" do
- assert_equal 0, @rate_limiter.current_count
- end
-
- test "should have infinite rate limit" do
- assert_equal Float::INFINITY, @rate_limiter.rate_limit
- end
-
- test "should have zero reset time" do
- assert_equal 0, @rate_limiter.reset_time
- end
-
- test "should provide correct usage info" do
- usage_info = @rate_limiter.usage_info
-
- assert_equal 0, usage_info[:current_count]
- assert_equal Float::INFINITY, usage_info[:rate_limit]
- assert_equal Float::INFINITY, usage_info[:remaining]
- assert_equal 0, usage_info[:reset_time]
- assert_equal :noop, usage_info[:tier]
- end
-
- test "class method usage_for should work" do
- usage_info = NoopApiRateLimiter.usage_for(@api_key)
-
- assert_equal 0, usage_info[:current_count]
- assert_equal Float::INFINITY, usage_info[:rate_limit]
- assert_equal Float::INFINITY, usage_info[:remaining]
- assert_equal 0, usage_info[:reset_time]
- assert_equal :noop, usage_info[:tier]
- end
-end