Browse Source

More robust PuSH subscription refreshes (#2799)

* Fix #2473 - Use sidekiq scheduler to refresh PuSH subscriptions instead of cron

Fix an issue where / in domain would raise exception in TagManager#normalize_domain

PuSH subscriptions refresh done in a round-robin way to avoid hammering a single
server's hub in sequence. Correct handling of failures/retries through Sidekiq (see
also #2613). Optimize Account#with_followers scope. Also, since subscriptions
are now delegated to Sidekiq jobs, an uncaught exception will not stop the entire
refreshing operation halfway through

Fix #2702 - Correct user agent header on outgoing http requests

* Add test for SubscribeService

* Extract #expiring_accounts into method

* Make mastodon:push:refresh no-op

* Queues are now defined in sidekiq.yml

* Queues are now in sidekiq.yml
Eugen Rochko 1 year ago
parent
commit
81584779cb

+ 2
- 1
Gemfile View File

@@ -35,7 +35,7 @@ gem 'link_header'
35 35
 gem 'local_time'
36 36
 gem 'nokogiri'
37 37
 gem 'oj'
38
-gem 'ostatus2', '~> 1.1'
38
+gem 'ostatus2', '~> 2.0'
39 39
 gem 'ox'
40 40
 gem 'rabl'
41 41
 gem 'rack-attack'
@@ -48,6 +48,7 @@ gem 'rqrcode'
48 48
 gem 'ruby-oembed', require: 'oembed'
49 49
 gem 'sanitize'
50 50
 gem 'sidekiq'
51
+gem 'sidekiq-scheduler'
51 52
 gem 'sidekiq-unique-jobs'
52 53
 gem 'simple-navigation'
53 54
 gem 'simple_form'

+ 12
- 2
Gemfile.lock View File

@@ -143,6 +143,8 @@ GEM
143 143
       thread_safe
144 144
     encryptor (3.0.0)
145 145
     erubis (2.7.0)
146
+    et-orbi (1.0.3)
147
+      tzinfo
146 148
     execjs (2.7.0)
147 149
     fabrication (2.16.1)
148 150
     faker (1.7.3)
@@ -251,7 +253,7 @@ GEM
251 253
     oj (3.0.5)
252 254
     openssl (2.0.3)
253 255
     orm_adapter (0.5.0)
254
-    ostatus2 (1.1.0)
256
+    ostatus2 (2.0.0)
255 257
       addressable (~> 2.4)
256 258
       http (~> 2.0)
257 259
       nokogiri (~> 1.6)
@@ -386,6 +388,8 @@ GEM
386 388
       unicode-display_width (~> 1.0, >= 1.0.1)
387 389
     ruby-oembed (0.12.0)
388 390
     ruby-progressbar (1.8.1)
391
+    rufus-scheduler (3.4.0)
392
+      et-orbi (~> 1.0)
389 393
     safe_yaml (1.0.4)
390 394
     sanitize (4.4.0)
391 395
       crass (~> 1.0.2)
@@ -396,6 +400,11 @@ GEM
396 400
       connection_pool (~> 2.2, >= 2.2.0)
397 401
       rack-protection (>= 1.5.0)
398 402
       redis (~> 3.3, >= 3.3.3)
403
+    sidekiq-scheduler (2.1.4)
404
+      redis (~> 3)
405
+      rufus-scheduler (~> 3.2)
406
+      sidekiq (>= 3)
407
+      tilt (>= 1.4.0)
399 408
     sidekiq-unique-jobs (5.0.7)
400 409
       sidekiq (>= 4.0, <= 6.0)
401 410
       thor (~> 0)
@@ -499,7 +508,7 @@ DEPENDENCIES
499 508
   microformats2
500 509
   nokogiri
501 510
   oj
502
-  ostatus2 (~> 1.1)
511
+  ostatus2 (~> 2.0)
503 512
   ox
504 513
   paperclip (~> 5.1)
505 514
   paperclip-av-transcoder
@@ -527,6 +536,7 @@ DEPENDENCIES
527 536
   ruby-oembed
528 537
   sanitize
529 538
   sidekiq
539
+  sidekiq-scheduler
530 540
   sidekiq-unique-jobs
531 541
   simple-navigation
532 542
   simple_form

+ 1
- 1
Procfile View File

@@ -1,2 +1,2 @@
1 1
 web: bundle exec puma -C config/puma.rb
2
-worker: bundle exec sidekiq -q default -q push -q pull -q mailers
2
+worker: bundle exec sidekiq

+ 1
- 0
Procfile.dev View File

@@ -1,3 +1,4 @@
1 1
 web: PORT=3000 bundle exec puma -C config/puma.rb
2
+sidekiq: bundle exec sidekiq
2 3
 stream: PORT=4000 yarn run start
3 4
 webpack: ./bin/webpack-dev-server --host 0.0.0.0

+ 7
- 3
app/helpers/http_helper.rb View File

@@ -1,13 +1,17 @@
1 1
 # frozen_string_literal: true
2 2
 
3 3
 module HttpHelper
4
-  USER_AGENT = "#{HTTP::Request::USER_AGENT} (Mastodon/#{Mastodon::Version}; +http://#{Rails.configuration.x.local_domain}/)"
5
-
6 4
   def http_client(options = {})
7 5
     timeout = { write: 10, connect: 10, read: 10 }.merge(options)
8 6
 
9
-    HTTP.headers(user_agent: USER_AGENT)
7
+    HTTP.headers(user_agent: user_agent)
10 8
         .timeout(:per_operation, timeout)
11 9
         .follow
12 10
   end
11
+
12
+  private
13
+
14
+  def user_agent
15
+    @user_agent ||= "#{HTTP::Request::USER_AGENT} (Mastodon/#{Mastodon::Version}; +http://#{Rails.configuration.x.local_domain}/)"
16
+  end
13 17
 end

+ 3
- 1
app/lib/tag_manager.rb View File

@@ -65,8 +65,10 @@ class TagManager
65 65
   end
66 66
 
67 67
   def normalize_domain(domain)
68
+    return if domain.nil?
69
+
68 70
     uri = Addressable::URI.new
69
-    uri.host = domain
71
+    uri.host = domain.gsub(/[\/]/, '')
70 72
     uri.normalize.host
71 73
   end
72 74
 

+ 3
- 2
app/models/account.rb View File

@@ -103,9 +103,10 @@ class Account < ApplicationRecord
103 103
 
104 104
   scope :remote, -> { where.not(domain: nil) }
105 105
   scope :local, -> { where(domain: nil) }
106
-  scope :without_followers, -> { where('(select count(f.id) from follows as f where f.target_account_id = accounts.id) = 0') }
107
-  scope :with_followers, -> { where('(select count(f.id) from follows as f where f.target_account_id = accounts.id) > 0') }
106
+  scope :without_followers, -> { where(followers_count: 0) }
107
+  scope :with_followers, -> { where('followers_count > 0') }
108 108
   scope :expiring, ->(time) { where(subscription_expires_at: nil).or(where('subscription_expires_at < ?', time)).remote.with_followers }
109
+  scope :partitioned, -> { order('row_number() over (partition by domain)') }
109 110
   scope :silenced, -> { where(silenced: true) }
110 111
   scope :suspended, -> { where(suspended: true) }
111 112
   scope :recent, -> { reorder(id: :desc) }

+ 1
- 1
app/services/follow_service.rb View File

@@ -40,7 +40,7 @@ class FollowService < BaseService
40 40
     if target_account.local?
41 41
       NotifyService.new.call(target_account, follow)
42 42
     else
43
-      SubscribeService.new.call(target_account) unless target_account.subscribed?
43
+      Pubsubhubbub::SubscribeWorker.perform_async(target_account.id) unless target_account.subscribed?
44 44
       NotificationWorker.perform_async(build_follow_xml(follow), source_account.id, target_account.id)
45 45
       AfterRemoteFollowWorker.perform_async(follow.id)
46 46
     end

+ 1
- 1
app/services/process_interaction_service.rb View File

@@ -77,7 +77,7 @@ class ProcessInteractionService < BaseService
77 77
   def authorize_follow_request!(account, target_account)
78 78
     follow_request = FollowRequest.find_by(account: target_account, target_account: account)
79 79
     follow_request&.authorize!
80
-    SubscribeService.new.call(account) unless account.subscribed?
80
+    Pubsubhubbub::SubscribeWorker.perform_async(account.id) unless account.subscribed?
81 81
   end
82 82
 
83 83
   def reject_follow_request!(account, target_account)

+ 22
- 6
app/services/subscribe_service.rb View File

@@ -5,15 +5,31 @@ class SubscribeService < BaseService
5 5
     account.secret = SecureRandom.hex
6 6
 
7 7
     subscription = account.subscription(api_subscription_url(account.id))
8
-    response = subscription.subscribe
8
+    response     = subscription.subscribe
9 9
 
10
-    unless response.successful?
10
+    if response_failed_permanently?(response)
11
+      # An error in the 4xx range (except for 429, which is rate limiting)
12
+      # means we're not allowed to subscribe. Fail and move on
11 13
       account.secret = ''
12
-      Rails.logger.debug "PuSH subscription request for #{account.acct} failed: #{response.message}"
14
+      account.save!
15
+    elsif response_successful?(response)
16
+      # Anything in the 2xx range means the subscription will be confirmed
17
+      # asynchronously, we've done what we needed to do
18
+      account.save!
19
+    else
20
+      # What's left is the 5xx range and 429, which means we need to retry
21
+      # at a later time. Fail loudly!
22
+      raise "Subscription attempt failed for #{account.acct} (#{account.hub_url}): HTTP #{response.code}"
13 23
     end
24
+  end
25
+
26
+  private
27
+
28
+  def response_failed_permanently?(response)
29
+    response.code > 299 && response.code < 500 && response.code != 429
30
+  end
14 31
 
15
-    account.save!
16
-  rescue HTTP::Error, OpenSSL::SSL::SSLError
17
-    Rails.logger.debug "PuSH subscription request for #{account.acct} could not be made due to HTTP or SSL error"
32
+  def response_successful?(response)
33
+    response.code > 199 && response.code < 300
18 34
   end
19 35
 end

+ 1
- 1
app/services/update_remote_profile_service.rb View File

@@ -28,7 +28,7 @@ class UpdateRemoteProfileService < BaseService
28 28
 
29 29
     account.save_with_optional_avatar!
30 30
 
31
-    SubscribeService.new.call(account) if resubscribe && (account.hub_url != old_hub_url)
31
+    Pubsubhubbub::SubscribeWorker.perform_async(account.id) if resubscribe && (account.hub_url != old_hub_url)
32 32
   end
33 33
 
34 34
   private

+ 10
- 2
app/workers/pubsubhubbub/delivery_worker.rb View File

@@ -25,8 +25,8 @@ class Pubsubhubbub::DeliveryWorker
25 25
                    .headers(headers)
26 26
                    .post(subscription.callback_url, body: payload)
27 27
 
28
-    return subscription.destroy! if response.code > 299 && response.code < 500 && response.code != 429 # HTTP 4xx means error is not temporary, except for 429 (throttling)
29
-    raise "Delivery failed for #{subscription.callback_url}: HTTP #{response.code}" unless response.code > 199 && response.code < 300
28
+    return subscription.destroy! if response_failed_permanently?(response) # HTTP 4xx means error is not temporary, except for 429 (throttling)
29
+    raise "Delivery failed for #{subscription.callback_url}: HTTP #{response.code}" unless response_successful?(response)
30 30
 
31 31
     subscription.touch(:last_successful_delivery_at)
32 32
   end
@@ -37,4 +37,12 @@ class Pubsubhubbub::DeliveryWorker
37 37
     hmac = OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new('sha1'), secret, payload)
38 38
     "sha1=#{hmac}"
39 39
   end
40
+
41
+  def response_failed_permanently?(response)
42
+    response.code > 299 && response.code < 500 && response.code != 429
43
+  end
44
+
45
+  def response_successful?(response)
46
+    response.code > 199 && response.code < 300
47
+  end
40 48
 end

+ 13
- 0
app/workers/pubsubhubbub/subscribe_worker.rb View File

@@ -0,0 +1,13 @@
1
+# frozen_string_literal: true
2
+
3
+class Pubsubhubbub::SubscribeWorker
4
+  include Sidekiq::Worker
5
+
6
+  sidekiq_options queue: 'push'
7
+
8
+  def perform(account_id)
9
+    account = Account.find(account_id)
10
+    Rails.logger.debug "PuSH re-subscribing to #{account.acct}"
11
+    ::SubscribeService.new.call(account)
12
+  end
13
+end

+ 20
- 0
app/workers/scheduler/subscriptions_scheduler.rb View File

@@ -0,0 +1,20 @@
1
+# frozen_string_literal: true
2
+require 'sidekiq-scheduler'
3
+
4
+class Scheduler::SubscriptionsScheduler
5
+  include Sidekiq::Worker
6
+
7
+  def perform
8
+    Rails.logger.debug 'Queueing PuSH re-subscriptions'
9
+
10
+    expiring_accounts.pluck(:id) do |id|
11
+      Pubsubhubbub::SubscribeWorker.perform_async(id)
12
+    end
13
+  end
14
+
15
+  private
16
+
17
+  def expiring_accounts
18
+    Account.expiring(1.day.from_now).partitioned
19
+  end
20
+end

+ 0
- 3
config/environments/development.rb View File

@@ -69,7 +69,4 @@ Rails.application.configure do
69 69
   end
70 70
 end
71 71
 
72
-require 'sidekiq/testing'
73
-Sidekiq::Testing.inline!
74
-
75 72
 ActiveRecordQueryTrace.enabled = ENV.fetch('QUERY_TRACE_ENABLED') { false }

+ 9
- 0
config/sidekiq.yml View File

@@ -1,2 +1,11 @@
1 1
 ---
2 2
 :concurrency: 5
3
+:queues:
4
+  - default
5
+  - push
6
+  - pull
7
+  - mailers
8
+:schedule:
9
+  subscriptions_scheduler:
10
+    cron: '0 5 * * *'
11
+    class: Scheduler::SubscriptionsScheduler

+ 2
- 4
lib/tasks/mastodon.rake View File

@@ -77,10 +77,8 @@ namespace :mastodon do
77 77
 
78 78
     desc 'Re-subscribes to soon expiring PuSH subscriptions'
79 79
     task refresh: :environment do
80
-      Account.expiring(1.day.from_now).find_each do |a|
81
-        Rails.logger.debug "PuSH re-subscribing to #{a.acct}"
82
-        SubscribeService.new.call(a)
83
-      end
80
+      # No-op
81
+      # This task is now executed via sidekiq-scheduler
84 82
     end
85 83
   end
86 84
 

+ 6
- 1
spec/services/follow_service_spec.rb View File

@@ -53,10 +53,11 @@ RSpec.describe FollowService do
53 53
     end
54 54
 
55 55
     describe 'unlocked account' do
56
-      let(:bob) { Fabricate(:user, email: 'bob@example.com', account: Fabricate(:account, username: 'bob', domain: 'example.com', salmon_url: 'http://salmon.example.com')).account }
56
+      let(:bob) { Fabricate(:user, email: 'bob@example.com', account: Fabricate(:account, username: 'bob', domain: 'example.com', salmon_url: 'http://salmon.example.com', hub_url: 'http://hub.example.com')).account }
57 57
 
58 58
       before do
59 59
         stub_request(:post, "http://salmon.example.com/").to_return(:status => 200, :body => "", :headers => {})
60
+        stub_request(:post, "http://hub.example.com/").to_return(status: 202)
60 61
         subject.call(sender, bob.acct)
61 62
       end
62 63
 
@@ -70,6 +71,10 @@ RSpec.describe FollowService do
70 71
           xml.match(TagManager::VERBS[:follow])
71 72
         }).to have_been_made.once
72 73
       end
74
+
75
+      it 'subscribes to PuSH' do
76
+        expect(a_request(:post, "http://hub.example.com/")).to have_been_made.once
77
+      end
73 78
     end
74 79
   end
75 80
 end

+ 38
- 0
spec/services/subscribe_service_spec.rb View File

@@ -0,0 +1,38 @@
1
+require 'rails_helper'
2
+
3
+RSpec.describe SubscribeService do
4
+  let(:account) { Fabricate(:account, username: 'bob', domain: 'example.com', hub_url: 'http://hub.example.com') }
5
+  subject { SubscribeService.new }
6
+
7
+  it 'sends subscription request to PuSH hub' do
8
+    stub_request(:post, 'http://hub.example.com/').to_return(status: 202)
9
+    subject.call(account)
10
+    expect(a_request(:post, 'http://hub.example.com/')).to have_been_made.once
11
+  end
12
+
13
+  it 'generates and keeps PuSH secret on successful call' do
14
+    stub_request(:post, 'http://hub.example.com/').to_return(status: 202)
15
+    subject.call(account)
16
+    expect(account.secret).to_not be_blank
17
+  end
18
+
19
+  it 'fails silently if PuSH hub forbids subscription' do
20
+    stub_request(:post, 'http://hub.example.com/').to_return(status: 403)
21
+    subject.call(account)
22
+  end
23
+
24
+  it 'fails silently if PuSH hub is not found' do
25
+    stub_request(:post, 'http://hub.example.com/').to_return(status: 404)
26
+    subject.call(account)
27
+  end
28
+
29
+  it 'fails loudly if there is a network error' do
30
+    stub_request(:post, 'http://hub.example.com/').to_raise(HTTP::Error)
31
+    expect { subject.call(account) }.to raise_error HTTP::Error
32
+  end
33
+
34
+  it 'fails loudly if PuSH hub is unavailable' do
35
+    stub_request(:post, 'http://hub.example.com/').to_return(status: 503)
36
+    expect { subject.call(account) }.to raise_error
37
+  end
38
+end