From f722bd2387df9163760014e9555928ec487ae95f Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 4 Apr 2017 00:53:20 +0200
Subject: [PATCH] Separate background jobs into different queues. ATTENTION:
new queue "pull" must be added to the Sidekiq invokation in your systemd file
The pull queue will handle link crawling, thread resolving, and OStatus
processing. Such tasks are more likely to hang for a longer time (due to
network requests) so it is more sensible to not make the "in-house" tasks
wait for them.
---
app/workers/after_remote_follow_request_worker.rb | 2 +-
app/workers/after_remote_follow_worker.rb | 2 +-
app/workers/import_worker.rb | 2 +-
app/workers/link_crawl_worker.rb | 2 +-
app/workers/merge_worker.rb | 2 ++
app/workers/notification_worker.rb | 2 +-
app/workers/processing_worker.rb | 2 +-
app/workers/regeneration_worker.rb | 2 ++
app/workers/salmon_worker.rb | 2 +-
app/workers/thread_resolve_worker.rb | 2 +-
app/workers/unmerge_worker.rb | 2 ++
docker-compose.yml | 2 +-
docs/Running-Mastodon/Production-guide.md | 2 +-
13 files changed, 16 insertions(+), 10 deletions(-)
diff --git a/app/workers/after_remote_follow_request_worker.rb b/app/workers/after_remote_follow_request_worker.rb
index f1d6869cc7..1f2db30615 100644
--- a/app/workers/after_remote_follow_request_worker.rb
+++ b/app/workers/after_remote_follow_request_worker.rb
@@ -3,7 +3,7 @@
class AfterRemoteFollowRequestWorker
include Sidekiq::Worker
- sidekiq_options retry: 5
+ sidekiq_options queue: 'pull', retry: 5
def perform(follow_request_id)
follow_request = FollowRequest.find(follow_request_id)
diff --git a/app/workers/after_remote_follow_worker.rb b/app/workers/after_remote_follow_worker.rb
index 0d04456a94..bdd2c2a91d 100644
--- a/app/workers/after_remote_follow_worker.rb
+++ b/app/workers/after_remote_follow_worker.rb
@@ -3,7 +3,7 @@
class AfterRemoteFollowWorker
include Sidekiq::Worker
- sidekiq_options retry: 5
+ sidekiq_options queue: 'pull', retry: 5
def perform(follow_id)
follow = Follow.find(follow_id)
diff --git a/app/workers/import_worker.rb b/app/workers/import_worker.rb
index a3ae2a85a4..7cf29fb53c 100644
--- a/app/workers/import_worker.rb
+++ b/app/workers/import_worker.rb
@@ -5,7 +5,7 @@ require 'csv'
class ImportWorker
include Sidekiq::Worker
- sidekiq_options retry: false
+ sidekiq_options queue: 'pull', retry: false
def perform(import_id)
import = Import.find(import_id)
diff --git a/app/workers/link_crawl_worker.rb b/app/workers/link_crawl_worker.rb
index af3394b8b3..834b0088bd 100644
--- a/app/workers/link_crawl_worker.rb
+++ b/app/workers/link_crawl_worker.rb
@@ -3,7 +3,7 @@
class LinkCrawlWorker
include Sidekiq::Worker
- sidekiq_options retry: false
+ sidekiq_options queue: 'pull', retry: false
def perform(status_id)
FetchLinkCardService.new.call(Status.find(status_id))
diff --git a/app/workers/merge_worker.rb b/app/workers/merge_worker.rb
index 0f288f43fe..d745cb99c7 100644
--- a/app/workers/merge_worker.rb
+++ b/app/workers/merge_worker.rb
@@ -3,6 +3,8 @@
class MergeWorker
include Sidekiq::Worker
+ sidekiq_options queue: 'pull'
+
def perform(from_account_id, into_account_id)
FeedManager.instance.merge_into_timeline(Account.find(from_account_id), Account.find(into_account_id))
end
diff --git a/app/workers/notification_worker.rb b/app/workers/notification_worker.rb
index 1a2faefd8e..da1d6ab455 100644
--- a/app/workers/notification_worker.rb
+++ b/app/workers/notification_worker.rb
@@ -3,7 +3,7 @@
class NotificationWorker
include Sidekiq::Worker
- sidekiq_options retry: 5
+ sidekiq_options queue: 'push', retry: 5
def perform(xml, source_account_id, target_account_id)
SendInteractionService.new.call(xml, Account.find(source_account_id), Account.find(target_account_id))
diff --git a/app/workers/processing_worker.rb b/app/workers/processing_worker.rb
index 5df404bcc9..4a467d9244 100644
--- a/app/workers/processing_worker.rb
+++ b/app/workers/processing_worker.rb
@@ -3,7 +3,7 @@
class ProcessingWorker
include Sidekiq::Worker
- sidekiq_options backtrace: true
+ sidekiq_options queue: 'pull', backtrace: true
def perform(account_id, body)
ProcessFeedService.new.call(body, Account.find(account_id))
diff --git a/app/workers/regeneration_worker.rb b/app/workers/regeneration_worker.rb
index 3aece0ba28..289b63d84a 100644
--- a/app/workers/regeneration_worker.rb
+++ b/app/workers/regeneration_worker.rb
@@ -3,6 +3,8 @@
class RegenerationWorker
include Sidekiq::Worker
+ sidekiq_options queue: 'pull', backtrace: true
+
def perform(account_id, timeline_type)
PrecomputeFeedService.new.call(timeline_type, Account.find(account_id))
end
diff --git a/app/workers/salmon_worker.rb b/app/workers/salmon_worker.rb
index fc95ce47ff..2888b574b6 100644
--- a/app/workers/salmon_worker.rb
+++ b/app/workers/salmon_worker.rb
@@ -3,7 +3,7 @@
class SalmonWorker
include Sidekiq::Worker
- sidekiq_options backtrace: true
+ sidekiq_options queue: 'pull', backtrace: true
def perform(account_id, body)
ProcessInteractionService.new.call(body, Account.find(account_id))
diff --git a/app/workers/thread_resolve_worker.rb b/app/workers/thread_resolve_worker.rb
index 593edd032f..38287e8e64 100644
--- a/app/workers/thread_resolve_worker.rb
+++ b/app/workers/thread_resolve_worker.rb
@@ -3,7 +3,7 @@
class ThreadResolveWorker
include Sidekiq::Worker
- sidekiq_options retry: false
+ sidekiq_options queue: 'pull', retry: false
def perform(child_status_id, parent_url)
child_status = Status.find(child_status_id)
diff --git a/app/workers/unmerge_worker.rb b/app/workers/unmerge_worker.rb
index dbf7243de7..ea6aacebf6 100644
--- a/app/workers/unmerge_worker.rb
+++ b/app/workers/unmerge_worker.rb
@@ -3,6 +3,8 @@
class UnmergeWorker
include Sidekiq::Worker
+ sidekiq_options queue: 'pull'
+
def perform(from_account_id, into_account_id)
FeedManager.instance.unmerge_from_timeline(Account.find(from_account_id), Account.find(into_account_id))
end
diff --git a/docker-compose.yml b/docker-compose.yml
index 68c8ef960e..d6ba66ddeb 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -33,7 +33,7 @@ services:
restart: always
build: .
env_file: .env.production
- command: bundle exec sidekiq -q default -q mailers -q push
+ command: bundle exec sidekiq -q default -q mailers -q pull -q push
depends_on:
- db
- redis
diff --git a/docs/Running-Mastodon/Production-guide.md b/docs/Running-Mastodon/Production-guide.md
index f0dd7bd2b5..469fefa943 100644
--- a/docs/Running-Mastodon/Production-guide.md
+++ b/docs/Running-Mastodon/Production-guide.md
@@ -180,7 +180,7 @@ User=mastodon
WorkingDirectory=/home/mastodon/live
Environment="RAILS_ENV=production"
Environment="DB_POOL=5"
-ExecStart=/home/mastodon/.rbenv/shims/bundle exec sidekiq -c 5 -q default -q mailers -q push
+ExecStart=/home/mastodon/.rbenv/shims/bundle exec sidekiq -c 5 -q default -q mailers -q pull -q push
TimeoutSec=15
Restart=always