Browse Source

Add improved CLI interface for removing remote media (#8411)

./bin/tootctl media remove --days 7 --background

Make the old rake task point to it
Eugen Rochko 1 month ago
parent
commit
793eea2982
No account linked to committer's email address

+ 1
- 1
app/workers/maintenance/destroy_media_worker.rb View File

@@ -6,7 +6,7 @@ class Maintenance::DestroyMediaWorker
6 6
   sidekiq_options queue: 'pull'
7 7
 
8 8
   def perform(media_attachment_id)
9
-    media = MediaAttachment.find(media_attachment_id)
9
+    media = media_attachment_id.is_a?(MediaAttachment) ? media_attachment_id : MediaAttachment.find(media_attachment_id)
10 10
     media.destroy
11 11
   rescue ActiveRecord::RecordNotFound
12 12
     true

+ 1
- 1
app/workers/maintenance/redownload_account_media_worker.rb View File

@@ -6,7 +6,7 @@ class Maintenance::RedownloadAccountMediaWorker
6 6
   sidekiq_options queue: 'pull', retry: false
7 7
 
8 8
   def perform(account_id)
9
-    account = Account.find(account_id)
9
+    account = account_id.is_a?(Account) ? account_id : Account.find(account_id)
10 10
     account.reset_avatar!
11 11
     account.reset_header!
12 12
     account.save

+ 1
- 1
app/workers/maintenance/uncache_media_worker.rb View File

@@ -6,7 +6,7 @@ class Maintenance::UncacheMediaWorker
6 6
   sidekiq_options queue: 'pull'
7 7
 
8 8
   def perform(media_attachment_id)
9
-    media = MediaAttachment.find(media_attachment_id)
9
+    media = media_attachment_id.is_a?(MediaAttachment) ? media_attachment_id : MediaAttachment.find(media_attachment_id)
10 10
 
11 11
     return if media.file.blank?
12 12
 

+ 4
- 0
bin/tootctl View File

@@ -0,0 +1,4 @@
1
+#!/usr/bin/env ruby
2
+APP_PATH = File.expand_path('../config/application', __dir__)
3
+require_relative '../lib/cli'
4
+Mastodon::CLI.start(ARGV)

+ 11
- 0
lib/cli.rb View File

@@ -0,0 +1,11 @@
1
+# frozen_string_literal: true
2
+
3
+require 'thor'
4
+require_relative 'mastodon/media_cli'
5
+
6
+module Mastodon
7
+  class CLI < Thor
8
+    desc 'media SUBCOMMAND ...ARGS', 'manage media files'
9
+    subcommand 'media', Mastodon::MediaCLI
10
+  end
11
+end

+ 47
- 0
lib/mastodon/media_cli.rb View File

@@ -0,0 +1,47 @@
1
+# frozen_string_literal: true
2
+
3
+require_relative '../../config/boot'
4
+require_relative '../../config/environment'
5
+
6
+# rubocop:disable Rails/Output
7
+
8
+module Mastodon
9
+  class MediaCLI < Thor
10
+    option :days, type: :numeric, default: 7
11
+    option :background, type: :boolean, default: false
12
+    desc 'remove', 'remove remote media files'
13
+    long_desc <<-DESC
14
+      Removes locally cached copies of media attachments from other servers.
15
+
16
+      The --days option specifies how old media attachments have to be before
17
+      they are removed. It defaults to 7 days.
18
+
19
+      With the --background option, instead of deleting the files sequentially,
20
+      they will be queued into Sidekiq and the command will exit as soon as
21
+      possible. In Sidekiq they will be processed with higher concurrency, but
22
+      it may impact other operations of the Mastodon server, and it may overload
23
+      the underlying file storage.
24
+    DESC
25
+    def remove
26
+      time_ago = options[:days].days.ago
27
+      queued   = 0
28
+
29
+      MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).select(:id).reorder(nil).find_in_batches do |media_attachments|
30
+        if options[:background]
31
+          queued += media_attachments.size
32
+          Maintenance::UncacheMediaWorker.push_bulk(media_attachments.map(&:id))
33
+        else
34
+          media_attachments.each do |m|
35
+            Maintenance::UncacheMediaWorker.new.perform(m)
36
+            print '.'
37
+          end
38
+        end
39
+      end
40
+
41
+      puts
42
+      puts "Scheduled the deletion of #{queued} media attachments" if options[:background]
43
+    end
44
+  end
45
+end
46
+
47
+# rubocop:enable Rails/Output

+ 3
- 8
lib/tasks/mastodon.rake View File

@@ -512,14 +512,9 @@ namespace :mastodon do
512 512
 
513 513
     desc 'Remove cached remote media attachments that are older than NUM_DAYS. By default 7 (week)'
514 514
     task remove_remote: :environment do
515
-      time_ago = ENV.fetch('NUM_DAYS') { 7 }.to_i.days.ago
516
-      nb_media_attachments = 0
517
-
518
-      MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).select(:id).reorder(nil).find_in_batches do |media_attachments|
519
-        nb_media_attachments += media_attachments.length
520
-        Maintenance::UncacheMediaWorker.push_bulk(media_attachments.map(&:id))
521
-      end
522
-      puts "Scheduled the deletion of #{nb_media_attachments} media attachments"
515
+      require_relative '../mastodon/media_cli'
516
+      cli = Mastodon::MediaCLI.new([], days: ENV['NUM_DAYS'] || 7)
517
+      cli.invoke(:remove)
523 518
     end
524 519
 
525 520
     desc 'Set unknown attachment type for remote-only attachments'