Moving some counter queries out of subqueries in the API

This commit is contained in:
Eugen Rochko 2016-11-22 22:59:54 +01:00
parent 4b5b953d42
commit fc90d38893
9 changed files with 62 additions and 9 deletions

View File

@ -50,6 +50,11 @@ gem 'react-rails'
gem 'browserify-rails' gem 'browserify-rails'
gem 'autoprefixer-rails' gem 'autoprefixer-rails'
gem 'rack-mini-profiler', require: false
gem 'flamegraph'
gem 'stackprof'
gem 'memory_profiler'
group :development, :test do group :development, :test do
gem 'rspec-rails' gem 'rspec-rails'
gem 'pry-rails' gem 'pry-rails'

View File

@ -136,6 +136,7 @@ GEM
execjs (2.7.0) execjs (2.7.0)
fabrication (2.15.2) fabrication (2.15.2)
fast_blank (1.0.0) fast_blank (1.0.0)
flamegraph (0.9.5)
font-awesome-rails (4.6.3.1) font-awesome-rails (4.6.3.1)
railties (>= 3.2, < 5.1) railties (>= 3.2, < 5.1)
fuubar (2.1.1) fuubar (2.1.1)
@ -205,6 +206,7 @@ GEM
nokogiri (>= 1.5.9) nokogiri (>= 1.5.9)
mail (2.6.4) mail (2.6.4)
mime-types (>= 1.16, < 4) mime-types (>= 1.16, < 4)
memory_profiler (0.9.7)
method_source (0.8.2) method_source (0.8.2)
mime-types (3.1) mime-types (3.1)
mime-types-data (~> 3.2015) mime-types-data (~> 3.2015)
@ -262,6 +264,8 @@ GEM
rack-attack (5.0.1) rack-attack (5.0.1)
rack rack
rack-cors (0.4.0) rack-cors (0.4.0)
rack-mini-profiler (0.10.1)
rack (>= 1.2.0)
rack-protection (1.5.3) rack-protection (1.5.3)
rack rack
rack-test (0.6.3) rack-test (0.6.3)
@ -372,6 +376,7 @@ GEM
actionpack (>= 4.0) actionpack (>= 4.0)
activesupport (>= 4.0) activesupport (>= 4.0)
sprockets (>= 3.0.0) sprockets (>= 3.0.0)
stackprof (0.2.10)
temple (0.7.7) temple (0.7.7)
term-ansicolor (1.4.0) term-ansicolor (1.4.0)
tins (~> 1.0) tins (~> 1.0)
@ -420,6 +425,7 @@ DEPENDENCIES
dotenv-rails dotenv-rails
fabrication fabrication
fast_blank fast_blank
flamegraph
font-awesome-rails font-awesome-rails
fuubar fuubar
goldfinger goldfinger
@ -435,6 +441,7 @@ DEPENDENCIES
letter_opener letter_opener
link_header link_header
lograge lograge
memory_profiler
neography neography
nokogiri nokogiri
oj oj
@ -449,6 +456,7 @@ DEPENDENCIES
rabl rabl
rack-attack rack-attack
rack-cors rack-cors
rack-mini-profiler
rails! rails!
rails_12factor rails_12factor
rails_autolink rails_autolink
@ -463,6 +471,7 @@ DEPENDENCIES
sidekiq sidekiq
simple_form simple_form
simplecov simplecov
stackprof
uglifier (>= 1.3.0) uglifier (>= 1.3.0)
webmock webmock
will_paginate will_paginate

View File

@ -1,11 +1,13 @@
import { showAlert } from '../actions/alerts'; import { showAlert } from '../actions/alerts';
const defaultSuccessSuffix = 'SUCCESS';
const defaultFailSuffix = 'FAIL'; const defaultFailSuffix = 'FAIL';
export default function errorsMiddleware() { export default function errorsMiddleware() {
return ({ dispatch }) => next => action => { return ({ dispatch }) => next => action => {
if (action.type) { if (action.type) {
const isFail = new RegExp(`${defaultFailSuffix}$`, 'g'); const isFail = new RegExp(`${defaultFailSuffix}$`, 'g');
const isSuccess = new RegExp(`${defaultSuccessSuffix}$`, 'g');
if (action.type.match(isFail)) { if (action.type.match(isFail)) {
if (action.error.response) { if (action.error.response) {

View File

@ -18,9 +18,11 @@ class Api::V1::AccountsController < ApiController
def following def following
results = Follow.where(account: @account).paginate_by_max_id(DEFAULT_ACCOUNTS_LIMIT, params[:max_id], params[:since_id]) results = Follow.where(account: @account).paginate_by_max_id(DEFAULT_ACCOUNTS_LIMIT, params[:max_id], params[:since_id])
accounts = Account.where(id: results.map(&:target_account_id)).with_counters.map { |a| [a.id, a] }.to_h accounts = Account.where(id: results.map(&:target_account_id)).map { |a| [a.id, a] }.to_h
@accounts = results.map { |f| accounts[f.target_account_id] } @accounts = results.map { |f| accounts[f.target_account_id] }
set_account_counters_maps(@accounts)
next_path = following_api_v1_account_url(max_id: results.last.id) if results.size == DEFAULT_ACCOUNTS_LIMIT next_path = following_api_v1_account_url(max_id: results.last.id) if results.size == DEFAULT_ACCOUNTS_LIMIT
prev_path = following_api_v1_account_url(since_id: results.first.id) unless results.empty? prev_path = following_api_v1_account_url(since_id: results.first.id) unless results.empty?
@ -31,9 +33,11 @@ class Api::V1::AccountsController < ApiController
def followers def followers
results = Follow.where(target_account: @account).paginate_by_max_id(DEFAULT_ACCOUNTS_LIMIT, params[:max_id], params[:since_id]) results = Follow.where(target_account: @account).paginate_by_max_id(DEFAULT_ACCOUNTS_LIMIT, params[:max_id], params[:since_id])
accounts = Account.where(id: results.map(&:account_id)).with_counters.map { |a| [a.id, a] }.to_h accounts = Account.where(id: results.map(&:account_id)).map { |a| [a.id, a] }.to_h
@accounts = results.map { |f| accounts[f.account_id] } @accounts = results.map { |f| accounts[f.account_id] }
set_account_counters_maps(@accounts)
next_path = followers_api_v1_account_url(max_id: results.last.id) if results.size == DEFAULT_ACCOUNTS_LIMIT next_path = followers_api_v1_account_url(max_id: results.last.id) if results.size == DEFAULT_ACCOUNTS_LIMIT
prev_path = followers_api_v1_account_url(since_id: results.first.id) unless results.empty? prev_path = followers_api_v1_account_url(since_id: results.first.id) unless results.empty?
@ -53,9 +57,10 @@ class Api::V1::AccountsController < ApiController
end end
def statuses def statuses
@statuses = @account.statuses.with_includes.with_counters.paginate_by_max_id(DEFAULT_STATUSES_LIMIT, params[:max_id], params[:since_id]).to_a @statuses = @account.statuses.with_includes.paginate_by_max_id(DEFAULT_STATUSES_LIMIT, params[:max_id], params[:since_id]).to_a
set_maps(@statuses) set_maps(@statuses)
set_counters_maps(@statuses)
next_path = statuses_api_v1_account_url(max_id: @statuses.last.id) if @statuses.size == DEFAULT_STATUSES_LIMIT next_path = statuses_api_v1_account_url(max_id: @statuses.last.id) if @statuses.size == DEFAULT_STATUSES_LIMIT
prev_path = statuses_api_v1_account_url(since_id: @statuses.first.id) unless @statuses.empty? prev_path = statuses_api_v1_account_url(since_id: @statuses.first.id) unless @statuses.empty?
@ -98,6 +103,9 @@ class Api::V1::AccountsController < ApiController
def search def search
limit = params[:limit] ? [DEFAULT_ACCOUNTS_LIMIT, params[:limit].to_i].min : DEFAULT_ACCOUNTS_LIMIT limit = params[:limit] ? [DEFAULT_ACCOUNTS_LIMIT, params[:limit].to_i].min : DEFAULT_ACCOUNTS_LIMIT
@accounts = SearchService.new.call(params[:q], limit, params[:resolve] == 'true') @accounts = SearchService.new.call(params[:q], limit, params[:resolve] == 'true')
set_account_counters_maps(@accounts)
render action: :index render action: :index
end end

View File

@ -13,14 +13,19 @@ class Api::V1::StatusesController < ApiController
def context def context
@context = OpenStruct.new(ancestors: @status.ancestors(current_account), descendants: @status.descendants(current_account)) @context = OpenStruct.new(ancestors: @status.ancestors(current_account), descendants: @status.descendants(current_account))
set_maps([@status] + @context[:ancestors] + @context[:descendants]) statuses = [@status] + @context[:ancestors] + @context[:descendants]
set_maps(statuses)
set_counters_maps(statuses)
end end
def reblogged_by def reblogged_by
results = @status.reblogs.paginate_by_max_id(DEFAULT_ACCOUNTS_LIMIT, params[:max_id], params[:since_id]) results = @status.reblogs.paginate_by_max_id(DEFAULT_ACCOUNTS_LIMIT, params[:max_id], params[:since_id])
accounts = Account.where(id: results.map(&:account_id)).with_counters.map { |a| [a.id, a] }.to_h accounts = Account.where(id: results.map(&:account_id)).map { |a| [a.id, a] }.to_h
@accounts = results.map { |r| accounts[r.account_id] } @accounts = results.map { |r| accounts[r.account_id] }
set_account_counters_maps(@accounts)
next_path = reblogged_by_api_v1_status_url(max_id: results.last.id) if results.size == DEFAULT_ACCOUNTS_LIMIT next_path = reblogged_by_api_v1_status_url(max_id: results.last.id) if results.size == DEFAULT_ACCOUNTS_LIMIT
prev_path = reblogged_by_api_v1_status_url(since_id: results.first.id) unless results.empty? prev_path = reblogged_by_api_v1_status_url(since_id: results.first.id) unless results.empty?
@ -31,9 +36,11 @@ class Api::V1::StatusesController < ApiController
def favourited_by def favourited_by
results = @status.favourites.paginate_by_max_id(DEFAULT_ACCOUNTS_LIMIT, params[:max_id], params[:since_id]) results = @status.favourites.paginate_by_max_id(DEFAULT_ACCOUNTS_LIMIT, params[:max_id], params[:since_id])
accounts = Account.where(id: results.map(&:account_id)).with_counters.map { |a| [a.id, a] }.to_h accounts = Account.where(id: results.map(&:account_id)).map { |a| [a.id, a] }.to_h
@accounts = results.map { |f| accounts[f.account_id] } @accounts = results.map { |f| accounts[f.account_id] }
set_account_counters_maps(@accounts)
next_path = favourited_by_api_v1_status_url(max_id: results.last.id) if results.size == DEFAULT_ACCOUNTS_LIMIT next_path = favourited_by_api_v1_status_url(max_id: results.last.id) if results.size == DEFAULT_ACCOUNTS_LIMIT
prev_path = favourited_by_api_v1_status_url(since_id: results.first.id) unless results.empty? prev_path = favourited_by_api_v1_status_url(since_id: results.first.id) unless results.empty?

View File

@ -14,6 +14,7 @@ class ApplicationController < ActionController::Base
before_action :store_current_location, except: :raise_not_found, unless: :devise_controller? before_action :store_current_location, except: :raise_not_found, unless: :devise_controller?
before_action :set_locale before_action :set_locale
before_action :check_rack_mini_profiler
def raise_not_found def raise_not_found
raise ActionController::RoutingError, "No route matches #{params[:unmatched_route]}" raise ActionController::RoutingError, "No route matches #{params[:unmatched_route]}"
@ -31,6 +32,10 @@ class ApplicationController < ActionController::Base
I18n.locale = I18n.default_locale I18n.locale = I18n.default_locale
end end
def check_rack_mini_profiler
Rack::MiniProfiler.authorize_request if current_user && current_user.admin?
end
protected protected
def not_found def not_found

View File

@ -71,7 +71,7 @@ class Status < ApplicationRecord
def ancestors(account = nil) def ancestors(account = nil)
ids = (Status.find_by_sql(['WITH RECURSIVE search_tree(id, in_reply_to_id, path) AS (SELECT id, in_reply_to_id, ARRAY[id] FROM statuses WHERE id = ? UNION ALL SELECT statuses.id, statuses.in_reply_to_id, path || statuses.id FROM search_tree JOIN statuses ON statuses.id = search_tree.in_reply_to_id WHERE NOT statuses.id = ANY(path)) SELECT id FROM search_tree ORDER BY path DESC', id]) - [self]).pluck(:id) ids = (Status.find_by_sql(['WITH RECURSIVE search_tree(id, in_reply_to_id, path) AS (SELECT id, in_reply_to_id, ARRAY[id] FROM statuses WHERE id = ? UNION ALL SELECT statuses.id, statuses.in_reply_to_id, path || statuses.id FROM search_tree JOIN statuses ON statuses.id = search_tree.in_reply_to_id WHERE NOT statuses.id = ANY(path)) SELECT id FROM search_tree ORDER BY path DESC', id]) - [self]).pluck(:id)
statuses = Status.where(id: ids).with_counters.with_includes.group_by(&:id) statuses = Status.where(id: ids).with_includes.group_by(&:id)
results = ids.map { |id| statuses[id].first } results = ids.map { |id| statuses[id].first }
results = results.reject { |status| account.blocking?(status.account) } unless account.nil? results = results.reject { |status| account.blocking?(status.account) } unless account.nil?
@ -80,7 +80,7 @@ class Status < ApplicationRecord
def descendants(account = nil) def descendants(account = nil)
ids = (Status.find_by_sql(['WITH RECURSIVE search_tree(id, path) AS (SELECT id, ARRAY[id] FROM statuses WHERE id = ? UNION ALL SELECT statuses.id, path || statuses.id FROM search_tree JOIN statuses ON statuses.in_reply_to_id = search_tree.id WHERE NOT statuses.id = ANY(path)) SELECT id FROM search_tree ORDER BY path', id]) - [self]).pluck(:id) ids = (Status.find_by_sql(['WITH RECURSIVE search_tree(id, path) AS (SELECT id, ARRAY[id] FROM statuses WHERE id = ? UNION ALL SELECT statuses.id, path || statuses.id FROM search_tree JOIN statuses ON statuses.in_reply_to_id = search_tree.id WHERE NOT statuses.id = ANY(path)) SELECT id FROM search_tree ORDER BY path', id]) - [self]).pluck(:id)
statuses = Status.where(id: ids).with_counters.with_includes.group_by(&:id) statuses = Status.where(id: ids).with_includes.group_by(&:id)
results = ids.map { |id| statuses[id].first } results = ids.map { |id| statuses[id].first }
results = results.reject { |status| account.blocking?(status.account) } unless account.nil? results = results.reject { |status| account.blocking?(status.account) } unless account.nil?

View File

@ -12,7 +12,7 @@ class SearchService < BaseService
Account.search_for("#{username} #{domain}") Account.search_for("#{username} #{domain}")
end end
results = results.limit(limit).with_counters results = results.limit(limit)
if resolve && results.empty? && !domain.nil? if resolve && results.empty? && !domain.nil?
results = [FollowRemoteAccountService.new.call("#{username}@#{domain}")] results = [FollowRemoteAccountService.new.call("#{username}@#{domain}")]

View File

@ -0,0 +1,17 @@
require 'rack-mini-profiler'
Rack::MiniProfilerRails.initialize!(Rails.application)
Rails.application.middleware.swap(Rack::Deflater, Rack::MiniProfiler)
Rails.application.middleware.swap(Rack::MiniProfiler, Rack::Deflater)
Rack::MiniProfiler.config.storage = Rack::MiniProfiler::MemoryStore
if Rails.env.production?
Rack::MiniProfiler.config.storage_options = {
host: ENV.fetch('REDIS_HOST') { 'localhost' },
port: ENV.fetch('REDIS_PORT') { 6379 },
}
Rack::MiniProfiler.config.storage = Rack::MiniProfiler::RedisStore
end