forked from cybrespace/mastodon
Fix issues with tootctl's parallelization and progress reporting (#12093)
This commit is contained in:
parent
b5be067c88
commit
38b6c34e32
|
@ -15,6 +15,11 @@ module Mastodon
|
||||||
end
|
end
|
||||||
|
|
||||||
def parallelize_with_progress(scope)
|
def parallelize_with_progress(scope)
|
||||||
|
if options[:concurrency] < 1
|
||||||
|
say('Cannot run with this concurrency setting, must be at least 1', :red)
|
||||||
|
exit(1)
|
||||||
|
end
|
||||||
|
|
||||||
ActiveRecord::Base.configurations[Rails.env]['pool'] = options[:concurrency]
|
ActiveRecord::Base.configurations[Rails.env]['pool'] = options[:concurrency]
|
||||||
|
|
||||||
progress = create_progress_bar(scope.count)
|
progress = create_progress_bar(scope.count)
|
||||||
|
@ -27,11 +32,21 @@ module Mastodon
|
||||||
|
|
||||||
items.each do |item|
|
items.each do |item|
|
||||||
futures << Concurrent::Future.execute(executor: pool) do
|
futures << Concurrent::Future.execute(executor: pool) do
|
||||||
ActiveRecord::Base.connection_pool.with_connection do
|
|
||||||
begin
|
begin
|
||||||
|
if !progress.total.nil? && progress.progress + 1 > progress.total
|
||||||
|
# The number of items has changed between start and now,
|
||||||
|
# since there is no good way to predict the final count from
|
||||||
|
# here, just change the progress bar to an indeterminate one
|
||||||
|
|
||||||
|
progress.total = nil
|
||||||
|
end
|
||||||
|
|
||||||
progress.log("Processing #{item.id}") if options[:verbose]
|
progress.log("Processing #{item.id}") if options[:verbose]
|
||||||
|
|
||||||
result = yield(item)
|
result = ActiveRecord::Base.connection_pool.with_connection do
|
||||||
|
yield(item)
|
||||||
|
end
|
||||||
|
|
||||||
aggregate.increment(result) if result.is_a?(Integer)
|
aggregate.increment(result) if result.is_a?(Integer)
|
||||||
rescue => e
|
rescue => e
|
||||||
progress.log pastel.red("Error processing #{item.id}: #{e}")
|
progress.log pastel.red("Error processing #{item.id}: #{e}")
|
||||||
|
@ -40,13 +55,12 @@ module Mastodon
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
|
||||||
|
|
||||||
total.increment(items.size)
|
total.increment(items.size)
|
||||||
futures.map(&:value)
|
futures.map(&:value)
|
||||||
end
|
end
|
||||||
|
|
||||||
progress.finish
|
progress.stop
|
||||||
|
|
||||||
[total.value, aggregate.value]
|
[total.value, aggregate.value]
|
||||||
end
|
end
|
||||||
|
|
Loading…
Reference in New Issue