From 1e96ce378e2aa35ed7287a4a88e5165c2ee20101 Mon Sep 17 00:00:00 2001 From: Kurtis Rainbolt-Greene Date: Tue, 4 Apr 2017 20:16:53 -0700 Subject: [PATCH 01/10] By pushing this into a worker we can reduce the amount of time the feed manager using workers eat up a connection --- app/lib/feed_manager.rb | 2 +- app/workers/push_update_worker.rb | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 app/workers/push_update_worker.rb diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index 2cca1cefe..075f86c2d 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -34,7 +34,7 @@ class FeedManager trim(timeline_type, account.id) end - broadcast(account.id, event: 'update', payload: inline_render(account, 'api/v1/statuses/show', status)) + PushUpdateWorker.perform_async(timeline_type, account.id, status.id) end def broadcast(timeline_id, options = {}) diff --git a/app/workers/push_update_worker.rb b/app/workers/push_update_worker.rb new file mode 100644 index 000000000..3d398b5ac --- /dev/null +++ b/app/workers/push_update_worker.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class PushUpdateWorker + include Sidekiq::Worker + + def perform(timeline, account_id, status_id) + account = Account.find(account_id) + status = Status.find(status_id) + message = inline_render(account, 'api/v1/statuses/show', status) + + broadcast(account_id, type: 'update', timeline: timeline, message: message) + end +end From 96ef9338208e09cbc52a49a3d7171d877eab3c43 Mon Sep 17 00:00:00 2001 From: Kurtis Rainbolt-Greene Date: Tue, 4 Apr 2017 20:36:03 -0700 Subject: [PATCH 02/10] Replacing the broadcast method with the one defined in the feed manager --- app/workers/push_update_worker.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/push_update_worker.rb b/app/workers/push_update_worker.rb index 3d398b5ac..5b5e9f68a 100644 --- a/app/workers/push_update_worker.rb +++ b/app/workers/push_update_worker.rb @@ -8,6 +8,6 @@ class PushUpdateWorker status = Status.find(status_id) message = inline_render(account, 'api/v1/statuses/show', status) - broadcast(account_id, type: 'update', timeline: timeline, message: message) + ActionCable.server.broadcast("timeline:#{account_id}", type: 'update', timeline: timeline, message: message) end end From dc5704b0b0c8f5a58ff95d3f3c4055929c6ecfba Mon Sep 17 00:00:00 2001 From: Kurtis Rainbolt-Greene Date: Tue, 4 Apr 2017 20:38:07 -0700 Subject: [PATCH 03/10] This method isn't used anymore --- app/lib/feed_manager.rb | 5 ----- 1 file changed, 5 deletions(-) diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index 075f86c2d..6698c78a5 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -37,11 +37,6 @@ class FeedManager PushUpdateWorker.perform_async(timeline_type, account.id, status.id) end - def broadcast(timeline_id, options = {}) - options[:queued_at] = (Time.now.to_f * 1000.0).to_i - ActionCable.server.broadcast("timeline:#{timeline_id}", options) - end - def trim(type, account_id) return unless redis.zcard(key(type, account_id)) > FeedManager::MAX_ITEMS last = redis.zrevrange(key(type, account_id), FeedManager::MAX_ITEMS - 1, FeedManager::MAX_ITEMS - 1) From 0069c01285d7fc6b97220fd678f6e5b82f301b1a Mon Sep 17 00:00:00 2001 From: Kurtis Rainbolt-Greene Date: Tue, 4 Apr 2017 20:39:14 -0700 Subject: [PATCH 04/10] Moving the queue_at into the worker --- app/workers/push_update_worker.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/push_update_worker.rb b/app/workers/push_update_worker.rb index 5b5e9f68a..6512e13ad 100644 --- a/app/workers/push_update_worker.rb +++ b/app/workers/push_update_worker.rb @@ -7,7 +7,7 @@ class PushUpdateWorker account = Account.find(account_id) status = Status.find(status_id) message = inline_render(account, 'api/v1/statuses/show', status) - + queue_at = (Time.now.to_f * 1000.0).to_i ActionCable.server.broadcast("timeline:#{account_id}", type: 'update', timeline: timeline, message: message) end end From 220051b8b2d09a741f5edadd34e21115c5938bf0 Mon Sep 17 00:00:00 2001 From: Kurtis Rainbolt-Greene Date: Tue, 4 Apr 2017 20:48:22 -0700 Subject: [PATCH 05/10] I don't actually think we need that. --- app/workers/push_update_worker.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/push_update_worker.rb b/app/workers/push_update_worker.rb index 6512e13ad..5b5e9f68a 100644 --- a/app/workers/push_update_worker.rb +++ b/app/workers/push_update_worker.rb @@ -7,7 +7,7 @@ class PushUpdateWorker account = Account.find(account_id) status = Status.find(status_id) message = inline_render(account, 'api/v1/statuses/show', status) - queue_at = (Time.now.to_f * 1000.0).to_i + ActionCable.server.broadcast("timeline:#{account_id}", type: 'update', timeline: timeline, message: message) end end From 9638894233d31368733574217e4d173e4cd5d13c Mon Sep 17 00:00:00 2001 From: Kurtis Rainbolt-Greene Date: Tue, 4 Apr 2017 20:51:18 -0700 Subject: [PATCH 06/10] Moving in the inline render --- app/workers/push_update_worker.rb | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/app/workers/push_update_worker.rb b/app/workers/push_update_worker.rb index 5b5e9f68a..fef75d909 100644 --- a/app/workers/push_update_worker.rb +++ b/app/workers/push_update_worker.rb @@ -6,8 +6,14 @@ class PushUpdateWorker def perform(timeline, account_id, status_id) account = Account.find(account_id) status = Status.find(status_id) - message = inline_render(account, 'api/v1/statuses/show', status) + message = Rabl::Renderer.new( + 'api/v1/statuses/show', + status, + view_path: 'app/views', + format: :json, + scope: InlineRablScope.new(account) + ) - ActionCable.server.broadcast("timeline:#{account_id}", type: 'update', timeline: timeline, message: message) + ActionCable.server.broadcast("timeline:#{account_id}", type: 'update', timeline: timeline, message: message.render) end end From 7bed4e51db18c864c36c6b48eb22c65f11c16b1c Mon Sep 17 00:00:00 2001 From: Kurtis Rainbolt-Greene Date: Tue, 4 Apr 2017 20:51:44 -0700 Subject: [PATCH 07/10] Moved to the worker --- app/lib/feed_manager.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index 6698c78a5..87865bfdc 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -76,10 +76,6 @@ class FeedManager end end - def inline_render(target_account, template, object) - Rabl::Renderer.new(template, object, view_path: 'app/views', format: :json, scope: InlineRablScope.new(target_account)).render - end - private def redis From 1b8c244dff84ae981d89a1672a9db06f08cf405e Mon Sep 17 00:00:00 2001 From: Eugen Date: Wed, 5 Apr 2017 18:48:41 +0200 Subject: [PATCH 08/10] Add proper message to PushUpdateWorker, use redis directly --- app/workers/push_update_worker.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/workers/push_update_worker.rb b/app/workers/push_update_worker.rb index fef75d909..9d16c20bf 100644 --- a/app/workers/push_update_worker.rb +++ b/app/workers/push_update_worker.rb @@ -5,7 +5,8 @@ class PushUpdateWorker def perform(timeline, account_id, status_id) account = Account.find(account_id) - status = Status.find(status_id) + status = Status.find(status_id) + message = Rabl::Renderer.new( 'api/v1/statuses/show', status, @@ -14,6 +15,8 @@ class PushUpdateWorker scope: InlineRablScope.new(account) ) - ActionCable.server.broadcast("timeline:#{account_id}", type: 'update', timeline: timeline, message: message.render) + Redis.current.publish("timeline:#{timeline_id}", Oj.dump({ event: :update, payload: message, queued_at: (Time.now.to_f * 1000.0).to_i })) + rescue ActiveRecord::RecordNotFound + true end end From c9ebd5d19fccaabd1192f5e61537251c2c2d782e Mon Sep 17 00:00:00 2001 From: Eugen Date: Wed, 5 Apr 2017 18:58:32 +0200 Subject: [PATCH 09/10] Fix wrong variable used in publish channel --- app/workers/push_update_worker.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/push_update_worker.rb b/app/workers/push_update_worker.rb index 9d16c20bf..166a9b449 100644 --- a/app/workers/push_update_worker.rb +++ b/app/workers/push_update_worker.rb @@ -15,7 +15,7 @@ class PushUpdateWorker scope: InlineRablScope.new(account) ) - Redis.current.publish("timeline:#{timeline_id}", Oj.dump({ event: :update, payload: message, queued_at: (Time.now.to_f * 1000.0).to_i })) + Redis.current.publish("timeline:#{account.id}", Oj.dump({ event: :update, payload: message, queued_at: (Time.now.to_f * 1000.0).to_i })) rescue ActiveRecord::RecordNotFound true end From 5b95be1c42ba69c9a3a79cfa990c80a5f2debfc6 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 5 Apr 2017 19:45:18 +0200 Subject: [PATCH 10/10] Replace calls to FeedManager#inline_render and #broadcast --- app/lib/feed_manager.rb | 2 +- app/lib/inline_renderer.rb | 13 +++++++++++++ app/services/fan_out_on_write_service.rb | 10 +++++----- app/services/notify_service.rb | 2 +- app/services/remove_status_service.rb | 6 +++--- app/workers/push_update_worker.rb | 13 +++---------- 6 files changed, 26 insertions(+), 20 deletions(-) create mode 100644 app/lib/inline_renderer.rb diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index 87865bfdc..58d9fb1fc 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -34,7 +34,7 @@ class FeedManager trim(timeline_type, account.id) end - PushUpdateWorker.perform_async(timeline_type, account.id, status.id) + PushUpdateWorker.perform_async(account.id, status.id) end def trim(type, account_id) diff --git a/app/lib/inline_renderer.rb b/app/lib/inline_renderer.rb new file mode 100644 index 000000000..8e04ad1d5 --- /dev/null +++ b/app/lib/inline_renderer.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class InlineRenderer + def self.render(status, current_account, template) + Rabl::Renderer.new( + template, + status, + view_path: 'app/views', + format: :json, + scope: InlineRablScope.new(current_account) + ).render + end +end diff --git a/app/services/fan_out_on_write_service.rb b/app/services/fan_out_on_write_service.rb index 106d257ba..c63fcc1fe 100644 --- a/app/services/fan_out_on_write_service.rb +++ b/app/services/fan_out_on_write_service.rb @@ -50,22 +50,22 @@ class FanOutOnWriteService < BaseService end def render_anonymous_payload(status) - @payload = FeedManager.instance.inline_render(nil, 'api/v1/statuses/show', status) + @payload = InlineRenderer.render(status, nil, 'api/v1/statuses/show') end def deliver_to_hashtags(status) Rails.logger.debug "Delivering status #{status.id} to hashtags" status.tags.pluck(:name).each do |hashtag| - FeedManager.instance.broadcast("hashtag:#{hashtag}", event: 'update', payload: @payload) - FeedManager.instance.broadcast("hashtag:#{hashtag}:local", event: 'update', payload: @payload) if status.account.local? + Redis.current.publish("hashtag:#{hashtag}", Oj.dump(event: :update, payload: @payload)) + Redis.current.publish("hashtag:#{hashtag}:local", Oj.dump(event: :update, payload: @payload)) if status.account.local? end end def deliver_to_public(status) Rails.logger.debug "Delivering status #{status.id} to public timeline" - FeedManager.instance.broadcast(:public, event: 'update', payload: @payload) - FeedManager.instance.broadcast('public:local', event: 'update', payload: @payload) if status.account.local? + Redis.current.publish('public', Oj.dump(event: 'update', payload: @payload)) + Redis.current.publish('public:local', Oj.dump(event: 'update', payload: @payload)) if status.account.local? end end diff --git a/app/services/notify_service.rb b/app/services/notify_service.rb index 24486f220..62508a049 100644 --- a/app/services/notify_service.rb +++ b/app/services/notify_service.rb @@ -50,7 +50,7 @@ class NotifyService < BaseService def create_notification @notification.save! return unless @notification.browserable? - FeedManager.instance.broadcast(@recipient.id, event: 'notification', payload: FeedManager.instance.inline_render(@recipient, 'api/v1/notifications/show', @notification)) + Redis.current.publish(@recipient.id, Oj.dump(event: :notification, payload: InlineRenderer.render(@notification, @recipient, 'api/v1/notifications/show'))) end def send_email diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb index cf1f432e4..e19fdd030 100644 --- a/app/services/remove_status_service.rb +++ b/app/services/remove_status_service.rb @@ -65,17 +65,17 @@ class RemoveStatusService < BaseService redis.zremrangebyscore(FeedManager.instance.key(type, receiver.id), status.id, status.id) end - FeedManager.instance.broadcast(receiver.id, event: 'delete', payload: status.id) + Redis.current.publish(receiver.id, Oj.dump(event: :delete, payload: status.id)) end def remove_from_hashtags(status) status.tags.each do |tag| - FeedManager.instance.broadcast("hashtag:#{tag.name}", event: 'delete', payload: status.id) + Redis.current.publish("hashtag:#{tag.name}", Oj.dump(event: :delete, payload: status.id)) end end def remove_from_public(status) - FeedManager.instance.broadcast(:public, event: 'delete', payload: status.id) + Redis.current.publish('public', Oj.dump(event: :delete, payload: status.id)) end def redis diff --git a/app/workers/push_update_worker.rb b/app/workers/push_update_worker.rb index 166a9b449..fbcdcf634 100644 --- a/app/workers/push_update_worker.rb +++ b/app/workers/push_update_worker.rb @@ -3,19 +3,12 @@ class PushUpdateWorker include Sidekiq::Worker - def perform(timeline, account_id, status_id) + def perform(account_id, status_id) account = Account.find(account_id) status = Status.find(status_id) - - message = Rabl::Renderer.new( - 'api/v1/statuses/show', - status, - view_path: 'app/views', - format: :json, - scope: InlineRablScope.new(account) - ) + message = InlineRenderer.render(status, account, 'api/v1/statuses/show') - Redis.current.publish("timeline:#{account.id}", Oj.dump({ event: :update, payload: message, queued_at: (Time.now.to_f * 1000.0).to_i })) + Redis.current.publish("timeline:#{account.id}", Oj.dump(event: :update, payload: message, queued_at: (Time.now.to_f * 1000.0).to_i)) rescue ActiveRecord::RecordNotFound true end