From 4e75f0d88932511ad154773f4c77a485367ed36c Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Mon, 14 Aug 2017 02:29:36 +0200 Subject: [PATCH] Hook up URL-based resource look-up to ActivityPub (#4589) --- app/helpers/jsonld_helper.rb | 8 +- .../fetch_remote_account_service.rb | 4 +- .../fetch_remote_status_service.rb | 4 +- app/services/fetch_atom_service.rb | 78 ++++++++++++------- app/services/fetch_remote_account_service.rb | 15 ++-- app/services/fetch_remote_status_service.rb | 15 ++-- .../api/subscriptions_controller_spec.rb | 26 +++---- spec/services/process_feed_service_spec.rb | 5 +- 8 files changed, 95 insertions(+), 60 deletions(-) diff --git a/app/helpers/jsonld_helper.rb b/app/helpers/jsonld_helper.rb index b0db025bc..c750a7038 100644 --- a/app/helpers/jsonld_helper.rb +++ b/app/helpers/jsonld_helper.rb @@ -16,7 +16,11 @@ module JsonLdHelper def fetch_resource(uri) response = build_request(uri).perform return if response.code != 200 - Oj.load(response.to_s, mode: :strict) + body_to_json(response.to_s) + end + + def body_to_json(body) + body.nil? ? nil : Oj.load(body, mode: :strict) rescue Oj::ParseError nil end @@ -25,7 +29,7 @@ module JsonLdHelper def build_request(uri) request = Request.new(:get, uri) - request.add_headers('Accept' => 'application/activity+json') + request.add_headers('Accept' => 'application/activity+json, application/ld+json') request end end diff --git a/app/services/activitypub/fetch_remote_account_service.rb b/app/services/activitypub/fetch_remote_account_service.rb index e443b9463..3eeca585e 100644 --- a/app/services/activitypub/fetch_remote_account_service.rb +++ b/app/services/activitypub/fetch_remote_account_service.rb @@ -5,8 +5,8 @@ class ActivityPub::FetchRemoteAccountService < BaseService # Should be called when uri has already been checked for locality # Does a WebFinger roundtrip on each call - def call(uri) - @json = fetch_resource(uri) + def call(uri, prefetched_json = nil) + @json = body_to_json(prefetched_json) || fetch_resource(uri) return unless supported_context? && expected_type? diff --git a/app/services/activitypub/fetch_remote_status_service.rb b/app/services/activitypub/fetch_remote_status_service.rb index 80305c53d..993e5389c 100644 --- a/app/services/activitypub/fetch_remote_status_service.rb +++ b/app/services/activitypub/fetch_remote_status_service.rb @@ -4,8 +4,8 @@ class ActivityPub::FetchRemoteStatusService < BaseService include JsonLdHelper # Should be called when uri has already been checked for locality - def call(uri) - @json = fetch_resource(uri) + def call(uri, prefetched_json = nil) + @json = body_to_json(prefetched_json) || fetch_resource(uri) return unless supported_context? && expected_type? diff --git a/app/services/fetch_atom_service.rb b/app/services/fetch_atom_service.rb index 3ac441e3e..c6a4dc2e9 100644 --- a/app/services/fetch_atom_service.rb +++ b/app/services/fetch_atom_service.rb @@ -4,18 +4,10 @@ class FetchAtomService < BaseService def call(url) return if url.blank? - response = Request.new(:head, url).perform + @url = url - Rails.logger.debug "Remote status HEAD request returned code #{response.code}" - - response = Request.new(:get, url).perform if response.code == 405 - - Rails.logger.debug "Remote status GET request returned code #{response.code}" - - return nil if response.code != 200 - return [url, fetch(url)] if response.mime_type == 'application/atom+xml' - return process_headers(url, response) if response['Link'].present? - process_html(fetch(url)) + perform_request + process_response rescue OpenSSL::SSL::SSLError => e Rails.logger.debug "SSL error: #{e}" nil @@ -26,27 +18,57 @@ class FetchAtomService < BaseService private - def process_html(body) - Rails.logger.debug 'Processing HTML' - - page = Nokogiri::HTML(body) - alternate_link = page.xpath('//link[@rel="alternate"]').find { |link| link['type'] == 'application/atom+xml' } - - return nil if alternate_link.nil? - [alternate_link['href'], fetch(alternate_link['href'])] + def perform_request + @response = Request.new(:get, @url) + .add_headers('Accept' => 'application/activity+json, application/ld+json, application/atom+xml, text/html') + .perform end - def process_headers(url, response) - Rails.logger.debug 'Processing link header' + def process_response(terminal = false) + return nil if @response.code != 200 - link_header = LinkHeader.parse(response['Link'].is_a?(Array) ? response['Link'].first : response['Link']) - alternate_link = link_header.find_link(%w(rel alternate), %w(type application/atom+xml)) - - return process_html(fetch(url)) if alternate_link.nil? - [alternate_link.href, fetch(alternate_link.href)] + if @response.mime_type == 'application/atom+xml' + [@url, @response.to_s, :ostatus] + elsif ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(@response.mime_type) + [@url, @response.to_s, :activitypub] + elsif @response['Link'] && !terminal + process_headers + elsif @response.mime_type == 'text/html' && !terminal + process_html + end end - def fetch(url) - Request.new(:get, url).perform.to_s + def process_html + page = Nokogiri::HTML(@response.to_s) + + json_link = page.xpath('//link[@rel="alternate"]').find { |link| ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(link['type']) } + atom_link = page.xpath('//link[@rel="alternate"]').find { |link| link['type'] == 'application/atom+xml' } + + if !json_link.nil? + @url = json_link['href'] + perform_request + process_response(true) + elsif !atom_link.nil? + @url = atom_link['href'] + perform_request + process_response(true) + end + end + + def process_headers + link_header = LinkHeader.parse(@response['Link'].is_a?(Array) ? @response['Link'].first : @response['Link']) + + json_link = link_header.find_link(%w(rel alternate), %w(type application/activity+json)) || link_header.find_link(%w(rel alternate), ['type', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"']) + atom_link = link_header.find_link(%w(rel alternate), %w(type application/atom+xml)) + + if !json_link.nil? + @url = json_link.href + perform_request + process_response(true) + elsif !atom_link.nil? + @url = atom_link.href + perform_request + process_response(true) + end end end diff --git a/app/services/fetch_remote_account_service.rb b/app/services/fetch_remote_account_service.rb index 8eed0d454..41b5374b4 100644 --- a/app/services/fetch_remote_account_service.rb +++ b/app/services/fetch_remote_account_service.rb @@ -5,14 +5,19 @@ class FetchRemoteAccountService < BaseService def call(url, prefetched_body = nil) if prefetched_body.nil? - atom_url, body = FetchAtomService.new.call(url) + resource_url, body, protocol = FetchAtomService.new.call(url) else - atom_url = url - body = prefetched_body + resource_url = url + body = prefetched_body + protocol = :ostatus end - return nil if atom_url.nil? - process_atom(atom_url, body) + case protocol + when :ostatus + process_atom(resource_url, body) + when :activitypub + ActivityPub::FetchRemoteAccountService.new.call(resource_url, body) + end end private diff --git a/app/services/fetch_remote_status_service.rb b/app/services/fetch_remote_status_service.rb index b9f5f97b1..30d8d2538 100644 --- a/app/services/fetch_remote_status_service.rb +++ b/app/services/fetch_remote_status_service.rb @@ -5,14 +5,19 @@ class FetchRemoteStatusService < BaseService def call(url, prefetched_body = nil) if prefetched_body.nil? - atom_url, body = FetchAtomService.new.call(url) + resource_url, body, protocol = FetchAtomService.new.call(url) else - atom_url = url - body = prefetched_body + resource_url = url + body = prefetched_body + protocol = :ostatus end - return nil if atom_url.nil? - process_atom(atom_url, body) + case protocol + when :ostatus + process_atom(resource_url, body) + when :activitypub + ActivityPub::FetchRemoteStatusService.new.call(resource_url, body) + end end private diff --git a/spec/controllers/api/subscriptions_controller_spec.rb b/spec/controllers/api/subscriptions_controller_spec.rb index 76f9740ca..d90da9e32 100644 --- a/spec/controllers/api/subscriptions_controller_spec.rb +++ b/spec/controllers/api/subscriptions_controller_spec.rb @@ -38,19 +38,19 @@ RSpec.describe Api::SubscriptionsController, type: :controller do before do stub_request(:post, "https://quitter.no/main/push/hub").to_return(:status => 200, :body => "", :headers => {}) stub_request(:get, "https://quitter.no/avatar/7477-300-20160211190340.png").to_return(request_fixture('avatar.txt')) - stub_request(:head, "https://quitter.no/notice/1269244").to_return(status: 404) - stub_request(:head, "https://quitter.no/notice/1265331").to_return(status: 404) - stub_request(:head, "https://community.highlandarrow.com/notice/54411").to_return(status: 404) - stub_request(:head, "https://community.highlandarrow.com/notice/53857").to_return(status: 404) - stub_request(:head, "https://community.highlandarrow.com/notice/51852").to_return(status: 404) - stub_request(:head, "https://social.umeahackerspace.se/notice/424348").to_return(status: 404) - stub_request(:head, "https://community.highlandarrow.com/notice/50467").to_return(status: 404) - stub_request(:head, "https://quitter.no/notice/1243309").to_return(status: 404) - stub_request(:head, "https://quitter.no/user/7477").to_return(status: 404) - stub_request(:head, "https://community.highlandarrow.com/user/1").to_return(status: 404) - stub_request(:head, "https://social.umeahackerspace.se/user/2").to_return(status: 404) - stub_request(:head, "https://gs.kawa-kun.com/user/2").to_return(status: 404) - stub_request(:head, "https://mastodon.social/users/Gargron").to_return(status: 404) + stub_request(:get, "https://quitter.no/notice/1269244").to_return(status: 404) + stub_request(:get, "https://quitter.no/notice/1265331").to_return(status: 404) + stub_request(:get, "https://community.highlandarrow.com/notice/54411").to_return(status: 404) + stub_request(:get, "https://community.highlandarrow.com/notice/53857").to_return(status: 404) + stub_request(:get, "https://community.highlandarrow.com/notice/51852").to_return(status: 404) + stub_request(:get, "https://social.umeahackerspace.se/notice/424348").to_return(status: 404) + stub_request(:get, "https://community.highlandarrow.com/notice/50467").to_return(status: 404) + stub_request(:get, "https://quitter.no/notice/1243309").to_return(status: 404) + stub_request(:get, "https://quitter.no/user/7477").to_return(status: 404) + stub_request(:any, "https://community.highlandarrow.com/user/1").to_return(status: 404) + stub_request(:any, "https://social.umeahackerspace.se/user/2").to_return(status: 404) + stub_request(:any, "https://gs.kawa-kun.com/user/2").to_return(status: 404) + stub_request(:any, "https://mastodon.social/users/Gargron").to_return(status: 404) request.env['HTTP_X_HUB_SIGNATURE'] = "sha1=#{OpenSSL::HMAC.hexdigest('sha1', 'abc', feed)}" request.env['RAW_POST_DATA'] = feed diff --git a/spec/services/process_feed_service_spec.rb b/spec/services/process_feed_service_spec.rb index 5e34370ee..aca675dc6 100644 --- a/spec/services/process_feed_service_spec.rb +++ b/spec/services/process_feed_service_spec.rb @@ -124,8 +124,7 @@ RSpec.describe ProcessFeedService do XML - stub_request(:head, 'https://overwatch.com/users/tracer/updates/1').to_return(status: 200, headers: { 'Content-Type' => 'application/atom+xml' }) - stub_request(:get, 'https://overwatch.com/users/tracer/updates/1').to_return(status: 200, body: real_body) + stub_request(:get, 'https://overwatch.com/users/tracer/updates/1').to_return(status: 200, body: real_body, headers: { 'Content-Type' => 'application/atom+xml' }) bad_actor = Fabricate(:account, username: 'sombra', domain: 'talon.xyz') @@ -168,7 +167,7 @@ XML end it 'ignores reblogs if it failed to retreive reblogged statuses' do - stub_request(:head, 'https://overwatch.com/users/tracer/updates/1').to_return(status: 404) + stub_request(:get, 'https://overwatch.com/users/tracer/updates/1').to_return(status: 404) actor = Fabricate(:account, username: 'tracer', domain: 'overwatch.com')