From b0e3f0312c3271a2705f912602fcba70f4ed8b69 Mon Sep 17 00:00:00 2001 From: Takeshi Umeda Date: Thu, 20 Oct 2022 16:15:52 +0900 Subject: [PATCH] Add synchronization of remote featured tags (#19380) * Add LIMIT of featured tag to instance API response * Add featured_tags_collection_url to Account * Add synchronization of remote featured tags * Deliver update activity when updating featured tag * Remove featured_tags_collection_url * Revert "Add featured_tags_collection_url to Account" This reverts commit cff349fc27b104ded2df6bb5665132dc24dab09c. * Add hashtag sync from featured collections * Fix tag name normalize * Add target option to fetch featured collection * Refactor fetch_featured_tags_collection_service * Add LIMIT of featured tag to v1/instance API response --- .../api/v1/featured_tags_controller.rb | 2 + .../settings/featured_tags_controller.rb | 2 + app/models/featured_tag.rb | 4 +- app/serializers/rest/instance_serializer.rb | 4 + .../rest/v1/instance_serializer.rb | 4 + .../fetch_featured_collection_service.rb | 31 +++++++- .../fetch_featured_tags_collection_service.rb | 78 +++++++++++++++++++ .../activitypub/process_account_service.rb | 7 +- .../synchronize_featured_collection_worker.rb | 6 +- ...hronize_featured_tags_collection_worker.rb | 13 ++++ .../activitypub/update_distribution_worker.rb | 2 + 11 files changed, 148 insertions(+), 5 deletions(-) create mode 100644 app/services/activitypub/fetch_featured_tags_collection_service.rb create mode 100644 app/workers/activitypub/synchronize_featured_tags_collection_worker.rb diff --git a/app/controllers/api/v1/featured_tags_controller.rb b/app/controllers/api/v1/featured_tags_controller.rb index c1ead4f54..a67db7040 100644 --- a/app/controllers/api/v1/featured_tags_controller.rb +++ b/app/controllers/api/v1/featured_tags_controller.rb @@ -14,11 +14,13 @@ class Api::V1::FeaturedTagsController < Api::BaseController def create @featured_tag = current_account.featured_tags.create!(featured_tag_params) + ActivityPub::UpdateDistributionWorker.perform_in(3.minutes, current_account.id) render json: @featured_tag, serializer: REST::FeaturedTagSerializer end def destroy @featured_tag.destroy! + ActivityPub::UpdateDistributionWorker.perform_in(3.minutes, current_account.id) render_empty end diff --git a/app/controllers/settings/featured_tags_controller.rb b/app/controllers/settings/featured_tags_controller.rb index aadff7c83..ae714e912 100644 --- a/app/controllers/settings/featured_tags_controller.rb +++ b/app/controllers/settings/featured_tags_controller.rb @@ -13,6 +13,7 @@ class Settings::FeaturedTagsController < Settings::BaseController @featured_tag = current_account.featured_tags.new(featured_tag_params) if @featured_tag.save + ActivityPub::UpdateDistributionWorker.perform_in(3.minutes, current_account.id) redirect_to settings_featured_tags_path else set_featured_tags @@ -24,6 +25,7 @@ class Settings::FeaturedTagsController < Settings::BaseController def destroy @featured_tag.destroy! + ActivityPub::UpdateDistributionWorker.perform_in(3.minutes, current_account.id) redirect_to settings_featured_tags_path end diff --git a/app/models/featured_tag.rb b/app/models/featured_tag.rb index 201ce75f5..b16c79ac7 100644 --- a/app/models/featured_tag.rb +++ b/app/models/featured_tag.rb @@ -26,6 +26,8 @@ class FeaturedTag < ApplicationRecord attr_writer :name + LIMIT = 10 + def name tag_id.present? ? tag.name : @name end @@ -50,7 +52,7 @@ class FeaturedTag < ApplicationRecord end def validate_featured_tags_limit - errors.add(:base, I18n.t('featured_tags.errors.limit')) if account.featured_tags.count >= 10 + errors.add(:base, I18n.t('featured_tags.errors.limit')) if account.featured_tags.count >= LIMIT end def validate_tag_name diff --git a/app/serializers/rest/instance_serializer.rb b/app/serializers/rest/instance_serializer.rb index dfa8ce40a..7d00b20ba 100644 --- a/app/serializers/rest/instance_serializer.rb +++ b/app/serializers/rest/instance_serializer.rb @@ -47,6 +47,10 @@ class REST::InstanceSerializer < ActiveModel::Serializer streaming: Rails.configuration.x.streaming_api_base_url, }, + accounts: { + max_featured_tags: FeaturedTag::LIMIT, + }, + statuses: { max_characters: StatusLengthValidator::MAX_CHARS, max_media_attachments: 4, diff --git a/app/serializers/rest/v1/instance_serializer.rb b/app/serializers/rest/v1/instance_serializer.rb index 872175451..99d1b2bd6 100644 --- a/app/serializers/rest/v1/instance_serializer.rb +++ b/app/serializers/rest/v1/instance_serializer.rb @@ -58,6 +58,10 @@ class REST::V1::InstanceSerializer < ActiveModel::Serializer def configuration { + accounts: { + max_featured_tags: FeaturedTag::LIMIT, + }, + statuses: { max_characters: StatusLengthValidator::MAX_CHARS, max_media_attachments: 4, diff --git a/app/services/activitypub/fetch_featured_collection_service.rb b/app/services/activitypub/fetch_featured_collection_service.rb index 37d05e055..026fe24c5 100644 --- a/app/services/activitypub/fetch_featured_collection_service.rb +++ b/app/services/activitypub/fetch_featured_collection_service.rb @@ -3,10 +3,11 @@ class ActivityPub::FetchFeaturedCollectionService < BaseService include JsonLdHelper - def call(account) + def call(account, **options) return if account.featured_collection_url.blank? || account.suspended? || account.local? @account = account + @options = options @json = fetch_resource(@account.featured_collection_url, true, local_follower) return unless supported_context?(@json) @@ -36,7 +37,15 @@ class ActivityPub::FetchFeaturedCollectionService < BaseService end def process_items(items) + process_note_items(items) if @options[:note] + process_hashtag_items(items) if @options[:hashtag] + end + + def process_note_items(items) status_ids = items.filter_map do |item| + type = item['type'] + next unless type == 'Note' + uri = value_or_id(item) next if ActivityPub::TagManager.instance.local_uri?(uri) @@ -67,6 +76,26 @@ class ActivityPub::FetchFeaturedCollectionService < BaseService end end + def process_hashtag_items(items) + names = items.filter_map { |item| item['type'] == 'Hashtag' && item['name']&.delete_prefix('#') }.map { |name| HashtagNormalizer.new.normalize(name) } + to_remove = [] + to_add = names + + FeaturedTag.where(account: @account).map(&:name).each do |name| + if names.include?(name) + to_add.delete(name) + else + to_remove << name + end + end + + FeaturedTag.includes(:tag).where(account: @account, tags: { name: to_remove }).delete_all unless to_remove.empty? + + to_add.each do |name| + FeaturedTag.create!(account: @account, name: name) + end + end + def local_follower return @local_follower if defined?(@local_follower) diff --git a/app/services/activitypub/fetch_featured_tags_collection_service.rb b/app/services/activitypub/fetch_featured_tags_collection_service.rb new file mode 100644 index 000000000..555919938 --- /dev/null +++ b/app/services/activitypub/fetch_featured_tags_collection_service.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +class ActivityPub::FetchFeaturedTagsCollectionService < BaseService + include JsonLdHelper + + def call(account, url) + return if url.blank? || account.suspended? || account.local? + + @account = account + @json = fetch_resource(url, true, local_follower) + + return unless supported_context?(@json) + + process_items(collection_items(@json)) + end + + private + + def collection_items(collection) + all_items = [] + + collection = fetch_collection(collection['first']) if collection['first'].present? + + while collection.is_a?(Hash) + items = begin + case collection['type'] + when 'Collection', 'CollectionPage' + collection['items'] + when 'OrderedCollection', 'OrderedCollectionPage' + collection['orderedItems'] + end + end + + break if items.blank? + + all_items.concat(items) + + break if all_items.size >= FeaturedTag::LIMIT + + collection = collection['next'].present? ? fetch_collection(collection['next']) : nil + end + + all_items + end + + def fetch_collection(collection_or_uri) + return collection_or_uri if collection_or_uri.is_a?(Hash) + return if invalid_origin?(collection_or_uri) + + fetch_resource_without_id_validation(collection_or_uri, local_follower, true) + end + + def process_items(items) + names = items.filter_map { |item| item['type'] == 'Hashtag' && item['name']&.delete_prefix('#') }.map { |name| HashtagNormalizer.new.normalize(name) } + to_remove = [] + to_add = names + + FeaturedTag.where(account: @account).map(&:name).each do |name| + if names.include?(name) + to_add.delete(name) + else + to_remove << name + end + end + + FeaturedTag.includes(:tag).where(account: @account, tags: { name: to_remove }).delete_all unless to_remove.empty? + + to_add.each do |name| + FeaturedTag.create!(account: @account, name: name) + end + end + + def local_follower + return @local_follower if defined?(@local_follower) + + @local_follower = @account.followers.local.without_suspended.first + end +end diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb index 456b3524b..3834d79cc 100644 --- a/app/services/activitypub/process_account_service.rb +++ b/app/services/activitypub/process_account_service.rb @@ -39,6 +39,7 @@ class ActivityPub::ProcessAccountService < BaseService unless @options[:only_key] || @account.suspended? check_featured_collection! if @account.featured_collection_url.present? + check_featured_tags_collection! if @json['featuredTags'].present? check_links! unless @account.fields.empty? end @@ -149,7 +150,11 @@ class ActivityPub::ProcessAccountService < BaseService end def check_featured_collection! - ActivityPub::SynchronizeFeaturedCollectionWorker.perform_async(@account.id) + ActivityPub::SynchronizeFeaturedCollectionWorker.perform_async(@account.id, { 'hashtag' => @json['featuredTags'].blank? }) + end + + def check_featured_tags_collection! + ActivityPub::SynchronizeFeaturedTagsCollectionWorker.perform_async(@account.id, @json['featuredTags']) end def check_links! diff --git a/app/workers/activitypub/synchronize_featured_collection_worker.rb b/app/workers/activitypub/synchronize_featured_collection_worker.rb index 7a0898e89..f67d693cb 100644 --- a/app/workers/activitypub/synchronize_featured_collection_worker.rb +++ b/app/workers/activitypub/synchronize_featured_collection_worker.rb @@ -5,8 +5,10 @@ class ActivityPub::SynchronizeFeaturedCollectionWorker sidekiq_options queue: 'pull', lock: :until_executed - def perform(account_id) - ActivityPub::FetchFeaturedCollectionService.new.call(Account.find(account_id)) + def perform(account_id, options = {}) + options = { note: true, hashtag: false }.deep_merge(options.deep_symbolize_keys) + + ActivityPub::FetchFeaturedCollectionService.new.call(Account.find(account_id), **options) rescue ActiveRecord::RecordNotFound true end diff --git a/app/workers/activitypub/synchronize_featured_tags_collection_worker.rb b/app/workers/activitypub/synchronize_featured_tags_collection_worker.rb new file mode 100644 index 000000000..14af4f725 --- /dev/null +++ b/app/workers/activitypub/synchronize_featured_tags_collection_worker.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class ActivityPub::SynchronizeFeaturedTagsCollectionWorker + include Sidekiq::Worker + + sidekiq_options queue: 'pull', lock: :until_executed + + def perform(account_id, url) + ActivityPub::FetchFeaturedTagsCollectionService.new.call(Account.find(account_id), url) + rescue ActiveRecord::RecordNotFound + true + end +end diff --git a/app/workers/activitypub/update_distribution_worker.rb b/app/workers/activitypub/update_distribution_worker.rb index 81fde63b8..d0391bb6f 100644 --- a/app/workers/activitypub/update_distribution_worker.rb +++ b/app/workers/activitypub/update_distribution_worker.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class ActivityPub::UpdateDistributionWorker < ActivityPub::RawDistributionWorker + sidekiq_options queue: 'push', lock: :until_executed + # Distribute an profile update to servers that might have a copy # of the account in question def perform(account_id, options = {})