Fix issues with tootctl's parallelization and progress reporting (#12093)
This commit is contained in:
		
							parent
							
								
									b5be067c88
								
							
						
					
					
						commit
						38b6c34e32
					
				
					 1 changed files with 24 additions and 10 deletions
				
			
		| 
						 | 
					@ -15,6 +15,11 @@ module Mastodon
 | 
				
			||||||
    end
 | 
					    end
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    def parallelize_with_progress(scope)
 | 
					    def parallelize_with_progress(scope)
 | 
				
			||||||
 | 
					      if options[:concurrency] < 1
 | 
				
			||||||
 | 
					        say('Cannot run with this concurrency setting, must be at least 1', :red)
 | 
				
			||||||
 | 
					        exit(1)
 | 
				
			||||||
 | 
					      end
 | 
				
			||||||
 | 
					
 | 
				
			||||||
      ActiveRecord::Base.configurations[Rails.env]['pool'] = options[:concurrency]
 | 
					      ActiveRecord::Base.configurations[Rails.env]['pool'] = options[:concurrency]
 | 
				
			||||||
 | 
					
 | 
				
			||||||
      progress  = create_progress_bar(scope.count)
 | 
					      progress  = create_progress_bar(scope.count)
 | 
				
			||||||
| 
						 | 
					@ -27,11 +32,21 @@ module Mastodon
 | 
				
			||||||
 | 
					
 | 
				
			||||||
        items.each do |item|
 | 
					        items.each do |item|
 | 
				
			||||||
          futures << Concurrent::Future.execute(executor: pool) do
 | 
					          futures << Concurrent::Future.execute(executor: pool) do
 | 
				
			||||||
            ActiveRecord::Base.connection_pool.with_connection do
 | 
					 | 
				
			||||||
            begin
 | 
					            begin
 | 
				
			||||||
 | 
					              if !progress.total.nil? && progress.progress + 1 > progress.total
 | 
				
			||||||
 | 
					                # The number of items has changed between start and now,
 | 
				
			||||||
 | 
					                # since there is no good way to predict the final count from
 | 
				
			||||||
 | 
					                # here, just change the progress bar to an indeterminate one
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					                progress.total = nil
 | 
				
			||||||
 | 
					              end
 | 
				
			||||||
 | 
					
 | 
				
			||||||
              progress.log("Processing #{item.id}") if options[:verbose]
 | 
					              progress.log("Processing #{item.id}") if options[:verbose]
 | 
				
			||||||
 | 
					
 | 
				
			||||||
                result = yield(item)
 | 
					              result = ActiveRecord::Base.connection_pool.with_connection do
 | 
				
			||||||
 | 
					                yield(item)
 | 
				
			||||||
 | 
					              end
 | 
				
			||||||
 | 
					
 | 
				
			||||||
              aggregate.increment(result) if result.is_a?(Integer)
 | 
					              aggregate.increment(result) if result.is_a?(Integer)
 | 
				
			||||||
            rescue => e
 | 
					            rescue => e
 | 
				
			||||||
              progress.log pastel.red("Error processing #{item.id}: #{e}")
 | 
					              progress.log pastel.red("Error processing #{item.id}: #{e}")
 | 
				
			||||||
| 
						 | 
					@ -40,13 +55,12 @@ module Mastodon
 | 
				
			||||||
            end
 | 
					            end
 | 
				
			||||||
          end
 | 
					          end
 | 
				
			||||||
        end
 | 
					        end
 | 
				
			||||||
        end
 | 
					 | 
				
			||||||
 | 
					
 | 
				
			||||||
        total.increment(items.size)
 | 
					        total.increment(items.size)
 | 
				
			||||||
        futures.map(&:value)
 | 
					        futures.map(&:value)
 | 
				
			||||||
      end
 | 
					      end
 | 
				
			||||||
 | 
					
 | 
				
			||||||
      progress.finish
 | 
					      progress.stop
 | 
				
			||||||
 | 
					
 | 
				
			||||||
      [total.value, aggregate.value]
 | 
					      [total.value, aggregate.value]
 | 
				
			||||||
    end
 | 
					    end
 | 
				
			||||||
| 
						 | 
					
 | 
				
			||||||
		Loading…
	
	Add table
		
		Reference in a new issue