Skip to content

Commit

Permalink
improve queue name select performance for postgresql
Browse files Browse the repository at this point in the history
  • Loading branch information
nebel95 committed Oct 29, 2024
1 parent ab89cc1 commit f6ae3be
Showing 1 changed file with 51 additions and 2 deletions.
53 changes: 51 additions & 2 deletions app/models/solid_queue/queue.rb
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,63 @@ class Queue

class << self
def all
Job.select(:queue_name).distinct.collect do |job|
new(job.queue_name)
queue_names.collect do |queue_name|
new(queue_name)
end
end

def find_by_name(name)
new(name)
end

private

def queue_names
# PostgreSQL doesn't perform well with SELECT DISTINCT
# => Use recursive common table expressions if possible for better performance (https://wiki.postgresql.org/wiki/Loose_indexscan)
if SolidQueue::Record.connection.adapter_name.downcase == "postgresql" && SolidQueue::Record.connection.supports_common_table_expressions?
Job.connection.execute(queue_names_recursive_cte_sql).to_a.map { |row| row["queue_name"] }
else
Job.select(:queue_name).distinct.map(&:queue_name)
end
end

def queue_names_recursive_cte_sql
# This relies on the fact that queue_name in solid_queue_jobs is NOT NULL
# The sql looks something like below:
# WITH RECURSIVE t AS (
# (SELECT queue_name FROM solid_queue_jobs ORDER BY queue_name LIMIT 1) -- parentheses required
# UNION ALL
# SELECT (SELECT queue_name FROM solid_queue_jobs WHERE queue_name > t.queue_name ORDER BY queue_name LIMIT 1)
# FROM t
# WHERE t.queue_name IS NOT NULL
# )
# SELECT queue_name FROM t WHERE queue_name IS NOT NULL;

cte_table = Arel::Table.new(:t)
jobs_table = Job.arel_table

cte_base_case = jobs_table.project(jobs_table[:queue_name]).order(jobs_table[:queue_name]).take(1)

subquery = jobs_table
.project(jobs_table[:queue_name])
.where(jobs_table[:queue_name].gt(cte_table[:queue_name]))
.order(jobs_table[:queue_name])
.take(1)
cte_recursive_case = cte_table.project(subquery)
.where(cte_table[:queue_name].not_eq(nil))

cte_definition = Arel::Nodes::Cte.new(
Arel.sql("t"),
Arel::Nodes::UnionAll.new(cte_base_case, cte_recursive_case),
)

cte_table
.project(cte_table[:queue_name])
.where(cte_table[:queue_name].not_eq(nil))
.with(:recursive, cte_definition)
.to_sql
end
end

def initialize(name)
Expand Down

0 comments on commit f6ae3be

Please sign in to comment.