From 6f78500d4f515c65ec66416e2d78bc9ae247f91c Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 27 Oct 2018 22:56:16 +0200
Subject: [PATCH] Do not remove "dead" domains in tootctl accounts cull (#9108)

Leave `tootctl accounts cull` to simply check removed accounts from
live domains, and skip temporarily unavailable domains, while listing
them in the final output for further action.

Add `tootctl domains purge DOMAIN` to be able to purge a domain from
that list manually
---
 lib/cli.rb                   |  4 ++++
 lib/mastodon/accounts_cli.rb | 45 +++++++++++-------------------------
 lib/mastodon/domains_cli.rb  | 40 ++++++++++++++++++++++++++++++++
 lib/mastodon/emoji_cli.rb    |  1 +
 lib/mastodon/feeds_cli.rb    |  3 ++-
 lib/mastodon/media_cli.rb    |  1 +
 lib/mastodon/settings_cli.rb |  1 +
 7 files changed, 62 insertions(+), 33 deletions(-)
 create mode 100644 lib/mastodon/domains_cli.rb

diff --git a/lib/cli.rb b/lib/cli.rb
index bff6d5809..a810c632a 100644
--- a/lib/cli.rb
+++ b/lib/cli.rb
@@ -6,6 +6,7 @@ require_relative 'mastodon/emoji_cli'
 require_relative 'mastodon/accounts_cli'
 require_relative 'mastodon/feeds_cli'
 require_relative 'mastodon/settings_cli'
+require_relative 'mastodon/domains_cli'
 
 module Mastodon
   class CLI < Thor
@@ -27,5 +28,8 @@ module Mastodon
 
     desc 'settings SUBCOMMAND ...ARGS', 'Manage dynamic settings'
     subcommand 'settings', Mastodon::SettingsCLI
+
+    desc 'domains SUBCOMMAND ...ARGS', 'Manage account domains'
+    subcommand 'domains', Mastodon::DomainsCLI
   end
 end
diff --git a/lib/mastodon/accounts_cli.rb b/lib/mastodon/accounts_cli.rb
index 055333080..142436c19 100644
--- a/lib/mastodon/accounts_cli.rb
+++ b/lib/mastodon/accounts_cli.rb
@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'rubygems/package'
+require 'set'
 require_relative '../../config/boot'
 require_relative '../../config/environment'
 require_relative 'cli_helper'
@@ -10,6 +10,7 @@ module Mastodon
     def self.exit_on_failure?
       true
     end
+
     option :all, type: :boolean
     desc 'rotate [USERNAME]', 'Generate and broadcast new keys'
     long_desc <<-LONG_DESC
@@ -210,33 +211,25 @@ module Mastodon
       Accounts that have had confirmed activity within the last week
       are excluded from the checks.
 
-      If 10 or more accounts from the same domain cannot be queried
-      due to a connection error (such as missing DNS records) then
-      the domain is considered dead, and all other accounts from it
-      are deleted without further querying.
+      Domains that are unreachable are not checked.
 
       With the --dry-run option, no deletes will actually be carried
       out.
     LONG_DESC
     def cull
-      domain_thresholds = Hash.new { |hash, key| hash[key] = 0 }
-      skip_threshold    = 7.days.ago
-      culled            = 0
-      dead_servers      = []
-      dry_run           = options[:dry_run] ? ' (DRY RUN)' : ''
+      skip_threshold = 7.days.ago
+      culled         = 0
+      skip_domains   = Set.new
+      dry_run        = options[:dry_run] ? ' (DRY RUN)' : ''
 
       Account.remote.where(protocol: :activitypub).partitioned.find_each do |account|
         next if account.updated_at >= skip_threshold || (account.last_webfingered_at.present? && account.last_webfingered_at >= skip_threshold)
 
-        unless dead_servers.include?(account.domain)
+        unless skip_domains.include?(account.domain)
           begin
             code = Request.new(:head, account.uri).perform(&:code)
           rescue HTTP::ConnectionError
-            domain_thresholds[account.domain] += 1
-
-            if domain_thresholds[account.domain] >= 10
-              dead_servers << account.domain
-            end
+            skip_domains << account.domain
           rescue StandardError
             next
           end
@@ -255,24 +248,12 @@ module Mastodon
         end
       end
 
-      # Remove dead servers
-      unless dead_servers.empty? || options[:dry_run]
-        dead_servers.each do |domain|
-          Account.where(domain: domain).find_each do |account|
-            SuspendAccountService.new.call(account)
-            account.destroy
-            culled += 1
-            say('.', :green, false)
-          end
-        end
-      end
-
       say
-      say("Removed #{culled} accounts (#{dead_servers.size} dead servers)#{dry_run}", :green)
+      say("Removed #{culled} accounts. #{skip_domains.size} servers skipped#{dry_run}", skip_domains.empty? ? :green : :yellow)
 
-      unless dead_servers.empty?
-        say('R.I.P.:', :yellow)
-        dead_servers.each { |domain| say('    ' + domain) }
+      unless skip_domains.empty?
+        say('The following servers were not available during the check:', :yellow)
+        skip_domains.each { |domain| say('    ' + domain) }
       end
     end
 
diff --git a/lib/mastodon/domains_cli.rb b/lib/mastodon/domains_cli.rb
new file mode 100644
index 000000000..a7a5caa11
--- /dev/null
+++ b/lib/mastodon/domains_cli.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require_relative '../../config/boot'
+require_relative '../../config/environment'
+require_relative 'cli_helper'
+
+module Mastodon
+  class DomainsCLI < Thor
+    def self.exit_on_failure?
+      true
+    end
+
+    option :dry_run, type: :boolean
+    desc 'purge DOMAIN', 'Remove accounts from a DOMAIN without a trace'
+    long_desc <<-LONG_DESC
+      Remove all accounts from a given DOMAIN without leaving behind any
+      records. Unlike a suspension, if the DOMAIN still exists in the wild,
+      it means the accounts could return if they are resolved again.
+    LONG_DESC
+    def purge(domain)
+      removed = 0
+      dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
+
+      Account.where(domain: domain).find_each do |account|
+        unless options[:dry_run]
+          SuspendAccountService.new.call(account)
+          account.destroy
+        end
+
+        removed += 1
+        say('.', :green, false)
+      end
+
+      DomainBlock.where(domain: domain).destroy_all
+
+      say
+      say("Removed #{removed} accounts#{dry_run}", :green)
+    end
+  end
+end
diff --git a/lib/mastodon/emoji_cli.rb b/lib/mastodon/emoji_cli.rb
index 1987c6394..2262040d4 100644
--- a/lib/mastodon/emoji_cli.rb
+++ b/lib/mastodon/emoji_cli.rb
@@ -10,6 +10,7 @@ module Mastodon
     def self.exit_on_failure?
       true
     end
+
     option :prefix
     option :suffix
     option :overwrite, type: :boolean
diff --git a/lib/mastodon/feeds_cli.rb b/lib/mastodon/feeds_cli.rb
index 817ed4e79..fe11c3df4 100644
--- a/lib/mastodon/feeds_cli.rb
+++ b/lib/mastodon/feeds_cli.rb
@@ -9,6 +9,7 @@ module Mastodon
     def self.exit_on_failure?
       true
     end
+
     option :all, type: :boolean, default: false
     option :background, type: :boolean, default: false
     option :dry_run, type: :boolean, default: false
@@ -58,7 +59,7 @@ module Mastodon
         account = Account.find_local(username)
 
         if account.nil?
-          say("Account #{username} is not found", :red)
+          say('No such account', :red)
           exit(1)
         end
 
diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb
index 95d2a8d64..179d1b6b5 100644
--- a/lib/mastodon/media_cli.rb
+++ b/lib/mastodon/media_cli.rb
@@ -9,6 +9,7 @@ module Mastodon
     def self.exit_on_failure?
       true
     end
+
     option :days, type: :numeric, default: 7
     option :background, type: :boolean, default: false
     option :verbose, type: :boolean, default: false
diff --git a/lib/mastodon/settings_cli.rb b/lib/mastodon/settings_cli.rb
index 69485600a..c81cfbe52 100644
--- a/lib/mastodon/settings_cli.rb
+++ b/lib/mastodon/settings_cli.rb
@@ -9,6 +9,7 @@ module Mastodon
     def self.exit_on_failure?
       true
     end
+
     desc 'open', 'Open registrations'
     def open
       Setting.open_registrations = true