diff --git a/Gemfile b/Gemfile index 6820e1d2..1d58c8f9 100644 --- a/Gemfile +++ b/Gemfile @@ -15,8 +15,7 @@ gem "redis", "~> 5.4" gem "puma", "~> 6.6" # Jobs -gem "resque", "~> 2.7.0" -gem "resque-pool", "~> 0.7.1" +gem "solid_queue" # Assets gem "propshaft", github: "rails/propshaft" diff --git a/Gemfile.lock b/Gemfile.lock index 60e9fb2d..8ba3bf79 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -136,7 +136,7 @@ GEM base64 (0.3.0) bcrypt (3.1.20) benchmark (0.5.0) - bigdecimal (3.3.1) + bigdecimal (4.0.1) brakeman (7.1.2) racc builder (3.3.0) @@ -150,45 +150,50 @@ GEM regexp_parser (>= 1.5, < 3.0) xpath (~> 3.2) chunky_png (1.4.0) - concurrent-ruby (1.3.5) + concurrent-ruby (1.3.6) connection_pool (2.5.5) crack (1.0.0) bigdecimal rexml crass (1.0.6) - date (3.5.0) + date (3.5.1) debug (1.11.0) irb (~> 1.10) reline (>= 0.3.8) drb (2.2.3) - erb (6.0.0) + erb (6.0.1) erubi (1.13.1) + et-orbi (1.4.0) + tzinfo faker (3.5.2) i18n (>= 1.8.11, < 2) ffi (1.17.2-aarch64-linux-gnu) ffi (1.17.2-arm64-darwin) ffi (1.17.2-x86_64-darwin) ffi (1.17.2-x86_64-linux-gnu) + fugit (1.12.1) + et-orbi (~> 1.4) + raabro (~> 1.4) geared_pagination (1.2.0) activesupport (>= 5.0) addressable (>= 2.5.0) globalid (1.3.0) activesupport (>= 6.1) hashdiff (1.2.0) - i18n (1.14.7) + i18n (1.14.8) concurrent-ruby (~> 1.0) image_processing (1.14.0) mini_magick (>= 4.9.5, < 6) ruby-vips (>= 2.0.17, < 3) - io-console (0.8.1) - irb (1.15.3) + io-console (0.8.2) + irb (1.16.0) pp (>= 0.6.0) rdoc (>= 4.0.0) reline (>= 0.4.2) jbuilder (2.14.1) actionview (>= 7.0.0) activesupport (>= 7.0.0) - json (2.13.2) + json (2.18.0) jwt (3.1.2) base64 kredis (1.8.0) @@ -198,7 +203,7 @@ GEM language_server-protocol (3.17.0.5) lint_roller (1.1.0) logger (1.7.0) - loofah (2.24.1) + loofah (2.25.0) crass (~> 1.0.2) nokogiri (>= 1.12.0) mail (2.9.0) @@ -212,13 +217,9 @@ GEM mini_magick (5.3.1) logger mini_mime (1.1.5) - minitest (5.26.2) + minitest (5.27.0) mocha (2.7.1) ruby2_keywords (>= 0.0.5) - mono_logger (1.1.2) - multi_json (1.17.0) - mustermann (3.0.4) - ruby2_keywords (~> 0.0.1) net-http-persistent (4.0.6) connection_pool (~> 2.2, >= 2.2.4) net-imap (0.5.12) @@ -231,13 +232,13 @@ GEM net-smtp (0.5.1) net-protocol nio4r (2.7.5) - nokogiri (1.18.10-aarch64-linux-gnu) + nokogiri (1.19.0-aarch64-linux-gnu) racc (~> 1.4) - nokogiri (1.18.10-arm64-darwin) + nokogiri (1.19.0-arm64-darwin) racc (~> 1.4) - nokogiri (1.18.10-x86_64-darwin) + nokogiri (1.19.0-x86_64-darwin) racc (~> 1.4) - nokogiri (1.18.10-x86_64-linux-gnu) + nokogiri (1.19.0-x86_64-linux-gnu) racc (~> 1.4) openssl (3.3.0) ostruct (0.6.3) @@ -252,24 +253,21 @@ GEM prettyprint prettyprint (0.2.0) prism (1.4.0) - psych (5.2.6) + psych (5.3.1) date stringio public_suffix (6.0.2) puma (6.6.1) nio4r (~> 2.0) + raabro (1.4.0) racc (1.8.1) rack (3.2.4) - rack-protection (4.1.1) - base64 (>= 0.1.0) - logger (>= 1.6.0) - rack (>= 3.0.0, < 4) rack-session (2.1.1) base64 (>= 0.1.0) rack (>= 3.0.0) rack-test (2.2.0) rack (>= 1.3) - rackup (2.2.1) + rackup (2.3.1) rack (>= 3) rails-dom-testing (2.3.0) activesupport (>= 5.0.0) @@ -284,7 +282,7 @@ GEM railties (> 3.1) rainbow (3.1.1) rake (13.3.1) - rdoc (6.16.1) + rdoc (7.0.3) erb psych (>= 4.0.0) tsort @@ -292,19 +290,9 @@ GEM redis-client (>= 0.22.0) redis-client (0.25.2) connection_pool - redis-namespace (1.11.0) - redis (>= 4) regexp_parser (2.11.2) reline (0.6.3) io-console (~> 0.5) - resque (2.7.0) - mono_logger (~> 1) - multi_json (~> 1.0) - redis-namespace (~> 1.6) - sinatra (>= 0.9.2) - resque-pool (0.7.1) - rake (>= 10.0, < 14.0) - resque (>= 1.22, < 3) rexml (3.4.1) rqrcode (3.1.0) chunky_png (~> 1.0) @@ -357,27 +345,26 @@ GEM sentry-ruby (5.26.0) bigdecimal concurrent-ruby (~> 1.0, >= 1.0.2) - sinatra (4.1.1) - logger (>= 1.6.0) - mustermann (~> 3.0) - rack (>= 3.0.0, < 4) - rack-protection (= 4.1.1) - rack-session (>= 2.0.0, < 3) - tilt (~> 2.0) + solid_queue (1.3.1) + activejob (>= 7.1) + activerecord (>= 7.1) + concurrent-ruby (>= 1.3.1) + fugit (~> 1.11) + railties (>= 7.1) + thor (>= 1.3.1) sqlite3 (2.7.3-aarch64-linux-gnu) sqlite3 (2.7.3-arm64-darwin) sqlite3 (2.7.3-x86_64-darwin) sqlite3 (2.7.3-x86_64-linux-gnu) stimulus-rails (1.3.4) railties (>= 6.0.0) - stringio (3.1.8) - thor (1.4.0) + stringio (3.2.0) + thor (1.5.0) thruster (0.1.15-aarch64-linux) thruster (0.1.15-arm64-darwin) thruster (0.1.15-x86_64-darwin) thruster (0.1.15-x86_64-linux) - tilt (2.6.1) - timeout (0.4.4) + timeout (0.6.0) tsort (0.2.0) tzinfo (2.0.6) concurrent-ruby (~> 1.0) @@ -400,7 +387,7 @@ GEM websocket-extensions (0.1.5) xpath (3.2.0) nokogiri (~> 1.8) - zeitwerk (2.7.3) + zeitwerk (2.7.4) PLATFORMS aarch64-linux @@ -429,13 +416,12 @@ DEPENDENCIES rails! rails_autolink redis (~> 5.4) - resque (~> 2.7.0) - resque-pool (~> 0.7.1) rqrcode rubocop-rails-omakase selenium-webdriver sentry-rails sentry-ruby + solid_queue sqlite3 stimulus-rails thruster diff --git a/Procfile b/Procfile index 20e14304..f6beda00 100644 --- a/Procfile +++ b/Procfile @@ -1,3 +1,3 @@ web: bundle exec thrust bin/start-app +jobs: bundle exec bin/jobs redis: redis-server config/redis.conf -workers: FORK_PER_JOB=false INTERVAL=0.1 bundle exec resque-pool diff --git a/bin/jobs b/bin/jobs new file mode 100755 index 00000000..dcf59f30 --- /dev/null +++ b/bin/jobs @@ -0,0 +1,6 @@ +#!/usr/bin/env ruby + +require_relative "../config/environment" +require "solid_queue/cli" + +SolidQueue::Cli.start(ARGV) diff --git a/config/database.yml b/config/database.yml index ec220a48..bf5967f4 100644 --- a/config/database.yml +++ b/config/database.yml @@ -14,6 +14,10 @@ development: primary: <<: *default database: storage/db/development.sqlite3 + queue: + <<: *default + database: storage/db/development_queue.sqlite3 + migrations_paths: db/queue_migrate # Warning: The database defined as "test" will be erased and # re-generated from your development database when you run "rake". @@ -22,13 +26,25 @@ test: primary: <<: *default database: storage/db/test.sqlite3 + queue: + <<: *default + database: storage/db/test_queue.sqlite3 + migrations_paths: db/queue_migrate performance: primary: <<: *default database: storage/db/performance.sqlite3 + queue: + <<: *default + database: storage/db/performance_queue.sqlite3 + migrations_paths: db/queue_migrate production: primary: <<: *default database: storage/db/production.sqlite3 + queue: + <<: *default + database: storage/db/production_queue.sqlite3 + migrations_paths: db/queue_migrate diff --git a/config/environments/development.rb b/config/environments/development.rb index 7994eb00..781a60b6 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -64,8 +64,8 @@ # Uncomment if you wish to allow Action Cable access from any origin. # config.action_cable.disable_request_forgery_protection = true - # Uncomment to test with production class job queue - # config.active_job.queue_adapter = :resque + config.active_job.queue_adapter = :solid_queue + config.solid_queue.connects_to = { database: { writing: :queue } } # Highlight code that enqueued background job in logs. config.active_job.verbose_enqueue_logs = true diff --git a/config/environments/production.rb b/config/environments/production.rb index d58644a1..9ead4056 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -92,5 +92,6 @@ # Only use :id for inspections in production. config.active_record.attributes_for_inspect = [ :id ] - config.active_job.queue_adapter = :resque + config.active_job.queue_adapter = :solid_queue + config.solid_queue.connects_to = { database: { writing: :queue } } end diff --git a/config/environments/test.rb b/config/environments/test.rb index 70deabfd..eb68e608 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -57,4 +57,6 @@ # Load test helpers config.autoload_paths += %w[ test/test_helpers ] + + config.solid_queue.connects_to = { database: { writing: :queue } } end diff --git a/config/puma.rb b/config/puma.rb index 960ab6a2..4f908f7b 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -47,6 +47,13 @@ # Allow puma to be restarted by `bin/rails restart` command. plugin :tmp_restart +# Optionally run Solid Queue inside Puma for lower memory usage. +# Set SOLID_QUEUE_IN_PUMA=true and remove the jobs process from Procfile. +if ENV["SOLID_QUEUE_IN_PUMA"] + plugin :solid_queue + solid_queue_mode :async +end + # Reset all membership connections Membership.disconnect_all diff --git a/config/queue.yml b/config/queue.yml new file mode 100644 index 00000000..9eace59c --- /dev/null +++ b/config/queue.yml @@ -0,0 +1,18 @@ +default: &default + dispatchers: + - polling_interval: 1 + batch_size: 500 + workers: + - queues: "*" + threads: 3 + processes: <%= ENV.fetch("JOB_CONCURRENCY", 1) %> + polling_interval: 0.1 + +development: + <<: *default + +test: + <<: *default + +production: + <<: *default diff --git a/config/recurring.yml b/config/recurring.yml new file mode 100644 index 00000000..854f415b --- /dev/null +++ b/config/recurring.yml @@ -0,0 +1,4 @@ +production: + clear_solid_queue_finished_jobs: + command: "SolidQueue::Job.clear_finished_in_batches(sleep_between_batches: 0.3)" + schedule: every hour at minute 12 diff --git a/config/resque-pool.yml b/config/resque-pool.yml deleted file mode 100644 index 64f569bd..00000000 --- a/config/resque-pool.yml +++ /dev/null @@ -1 +0,0 @@ -default: <%= (Concurrent.processor_count * 0.5).ceil %> diff --git a/db/queue_schema.rb b/db/queue_schema.rb new file mode 100644 index 00000000..85194b6a --- /dev/null +++ b/db/queue_schema.rb @@ -0,0 +1,129 @@ +ActiveRecord::Schema[7.1].define(version: 1) do + create_table "solid_queue_blocked_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.string "concurrency_key", null: false + t.datetime "expires_at", null: false + t.datetime "created_at", null: false + t.index [ "concurrency_key", "priority", "job_id" ], name: "index_solid_queue_blocked_executions_for_release" + t.index [ "expires_at", "concurrency_key" ], name: "index_solid_queue_blocked_executions_for_maintenance" + t.index [ "job_id" ], name: "index_solid_queue_blocked_executions_on_job_id", unique: true + end + + create_table "solid_queue_claimed_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.bigint "process_id" + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_claimed_executions_on_job_id", unique: true + t.index [ "process_id", "job_id" ], name: "index_solid_queue_claimed_executions_on_process_id_and_job_id" + end + + create_table "solid_queue_failed_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.text "error" + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_failed_executions_on_job_id", unique: true + end + + create_table "solid_queue_jobs", force: :cascade do |t| + t.string "queue_name", null: false + t.string "class_name", null: false + t.text "arguments" + t.integer "priority", default: 0, null: false + t.string "active_job_id" + t.datetime "scheduled_at" + t.datetime "finished_at" + t.string "concurrency_key" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index [ "active_job_id" ], name: "index_solid_queue_jobs_on_active_job_id" + t.index [ "class_name" ], name: "index_solid_queue_jobs_on_class_name" + t.index [ "finished_at" ], name: "index_solid_queue_jobs_on_finished_at" + t.index [ "queue_name", "finished_at" ], name: "index_solid_queue_jobs_for_filtering" + t.index [ "scheduled_at", "finished_at" ], name: "index_solid_queue_jobs_for_alerting" + end + + create_table "solid_queue_pauses", force: :cascade do |t| + t.string "queue_name", null: false + t.datetime "created_at", null: false + t.index [ "queue_name" ], name: "index_solid_queue_pauses_on_queue_name", unique: true + end + + create_table "solid_queue_processes", force: :cascade do |t| + t.string "kind", null: false + t.datetime "last_heartbeat_at", null: false + t.bigint "supervisor_id" + t.integer "pid", null: false + t.string "hostname" + t.text "metadata" + t.datetime "created_at", null: false + t.string "name", null: false + t.index [ "last_heartbeat_at" ], name: "index_solid_queue_processes_on_last_heartbeat_at" + t.index [ "name", "supervisor_id" ], name: "index_solid_queue_processes_on_name_and_supervisor_id", unique: true + t.index [ "supervisor_id" ], name: "index_solid_queue_processes_on_supervisor_id" + end + + create_table "solid_queue_ready_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_ready_executions_on_job_id", unique: true + t.index [ "priority", "job_id" ], name: "index_solid_queue_poll_all" + t.index [ "queue_name", "priority", "job_id" ], name: "index_solid_queue_poll_by_queue" + end + + create_table "solid_queue_recurring_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "task_key", null: false + t.datetime "run_at", null: false + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_recurring_executions_on_job_id", unique: true + t.index [ "task_key", "run_at" ], name: "index_solid_queue_recurring_executions_on_task_key_and_run_at", unique: true + end + + create_table "solid_queue_recurring_tasks", force: :cascade do |t| + t.string "key", null: false + t.string "schedule", null: false + t.string "command", limit: 2048 + t.string "class_name" + t.text "arguments" + t.string "queue_name" + t.integer "priority", default: 0 + t.boolean "static", default: true, null: false + t.text "description" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index [ "key" ], name: "index_solid_queue_recurring_tasks_on_key", unique: true + t.index [ "static" ], name: "index_solid_queue_recurring_tasks_on_static" + end + + create_table "solid_queue_scheduled_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.datetime "scheduled_at", null: false + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_scheduled_executions_on_job_id", unique: true + t.index [ "scheduled_at", "priority", "job_id" ], name: "index_solid_queue_dispatch_all" + end + + create_table "solid_queue_semaphores", force: :cascade do |t| + t.string "key", null: false + t.integer "value", default: 1, null: false + t.datetime "expires_at", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index [ "expires_at" ], name: "index_solid_queue_semaphores_on_expires_at" + t.index [ "key", "value" ], name: "index_solid_queue_semaphores_on_key_and_value" + t.index [ "key" ], name: "index_solid_queue_semaphores_on_key", unique: true + end + + add_foreign_key "solid_queue_blocked_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_claimed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_failed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_ready_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_recurring_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_scheduled_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade +end diff --git a/lib/tasks/resque.rake b/lib/tasks/resque.rake deleted file mode 100644 index 63ecc9d3..00000000 --- a/lib/tasks/resque.rake +++ /dev/null @@ -1,12 +0,0 @@ -task "resque:setup" do - require_relative "../../config/environment" -end - -task "resque:pool:setup" do - ActiveRecord::Base.connection.disconnect! - - Resque::Pool.after_prefork do |job| - ActiveRecord::Base.establish_connection - Resque.redis.client.close - end -end