Browse Source

Add E2EE API (#13820)

feature_cybrespace_locale
Eugen Rochko 1 year ago
committed by GitHub
parent
commit
5d8398c8b8
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
  1. 1
      Gemfile
  2. 2
      Gemfile.lock
  3. 21
      app/controllers/activitypub/claims_controller.rb
  4. 48
      app/controllers/activitypub/collections_controller.rb
  5. 30
      app/controllers/api/v1/crypto/deliveries_controller.rb
  6. 59
      app/controllers/api/v1/crypto/encrypted_messages_controller.rb
  7. 25
      app/controllers/api/v1/crypto/keys/claims_controller.rb
  8. 17
      app/controllers/api/v1/crypto/keys/counts_controller.rb
  9. 26
      app/controllers/api/v1/crypto/keys/queries_controller.rb
  10. 29
      app/controllers/api/v1/crypto/keys/uploads_controller.rb
  11. 2
      app/controllers/statuses_controller.rb
  12. 50
      app/lib/activitypub/activity/create.rb
  13. 1
      app/lib/activitypub/adapter.rb
  14. 2
      app/lib/inline_renderer.rb
  15. 1
      app/models/account.rb
  16. 1
      app/models/concerns/account_associations.rb
  17. 35
      app/models/device.rb
  18. 50
      app/models/encrypted_message.rb
  19. 19
      app/models/message_franking.rb
  20. 21
      app/models/one_time_key.rb
  21. 41
      app/models/system_key.rb
  22. 41
      app/presenters/activitypub/activity_presenter.rb
  23. 54
      app/serializers/activitypub/activity_serializer.rb
  24. 9
      app/serializers/activitypub/actor_serializer.rb
  25. 13
      app/serializers/activitypub/collection_serializer.rb
  26. 52
      app/serializers/activitypub/device_serializer.rb
  27. 61
      app/serializers/activitypub/encrypted_message_serializer.rb
  28. 35
      app/serializers/activitypub/one_time_key_serializer.rb
  29. 11
      app/serializers/activitypub/outbox_serializer.rb
  30. 6
      app/serializers/activitypub/undo_announce_serializer.rb
  31. 18
      app/serializers/rest/encrypted_message_serializer.rb
  32. 9
      app/serializers/rest/keys/claim_result_serializer.rb
  33. 6
      app/serializers/rest/keys/device_serializer.rb
  34. 11
      app/serializers/rest/keys/query_result_serializer.rb
  35. 1
      app/services/activitypub/process_account_service.rb
  36. 2
      app/services/backup_service.rb
  37. 78
      app/services/deliver_to_device_service.rb
  38. 77
      app/services/keys/claim_service.rb
  39. 75
      app/services/keys/query_service.rb
  40. 2
      app/services/process_mentions_service.rb
  41. 2
      app/services/reblog_service.rb
  42. 19
      app/validators/ed25519_key_validator.rb
  43. 29
      app/validators/ed25519_signature_validator.rb
  44. 2
      app/workers/activitypub/distribution_worker.rb
  45. 2
      app/workers/activitypub/reply_distribution_worker.rb
  46. 3
      app/workers/push_conversation_worker.rb
  47. 16
      app/workers/push_encrypted_message_worker.rb
  48. 1
      app/workers/scheduler/doorkeeper_cleanup_scheduler.rb
  49. 228
      config/brakeman.ignore
  50. 3
      config/initializers/doorkeeper.rb
  51. 1
      config/initializers/inflections.rb
  52. 4
      config/locales/en.yml
  53. 18
      config/routes.rb
  54. 13
      db/migrate/20170129000348_create_devices.rb
  55. 2
      db/migrate/20170205175257_remove_devices.rb
  56. 14
      db/migrate/20200516180352_create_devices.rb
  57. 12
      db/migrate/20200516183822_create_one_time_keys.rb
  58. 15
      db/migrate/20200518083523_create_encrypted_messages.rb
  59. 13
      db/migrate/20200521180606_encrypted_message_ids_to_timestamp_ids.rb
  60. 5
      db/migrate/20200529214050_add_devices_url_to_accounts.rb
  61. 9
      db/migrate/20200601222558_create_system_keys.rb
  62. 52
      db/schema.rb
  63. 8
      spec/fabricators/device_fabricator.rb
  64. 8
      spec/fabricators/encrypted_message_fabricator.rb
  65. 11
      spec/fabricators/one_time_key_fabricator.rb
  66. 3
      spec/fabricators/system_key_fabricator.rb
  67. 56
      spec/lib/activitypub/activity/create_spec.rb
  68. 5
      spec/models/device_spec.rb
  69. 5
      spec/models/encrypted_message_spec.rb
  70. 5
      spec/models/one_time_key_spec.rb
  71. 5
      spec/models/system_key_spec.rb
  72. 65
      streaming/index.js

1
Gemfile

@ -50,6 +50,7 @@ gem 'omniauth', '~> 1.9'
gem 'discard', '~> 1.2'
gem 'doorkeeper', '~> 5.4'
gem 'ed25519', '~> 1.2'
gem 'fast_blank', '~> 1.0'
gem 'fastimage'
gem 'goldfinger', '~> 2.1'

2
Gemfile.lock

@ -201,6 +201,7 @@ GEM
dotenv (= 2.7.5)
railties (>= 3.2, < 6.1)
e2mmap (0.1.0)
ed25519 (1.2.4)
elasticsearch (7.7.0)
elasticsearch-api (= 7.7.0)
elasticsearch-transport (= 7.7.0)
@ -701,6 +702,7 @@ DEPENDENCIES
doorkeeper (~> 5.4)
dotenv-rails (~> 2.7)
e2mmap (~> 0.1.0)
ed25519 (~> 1.2)
fabrication (~> 2.21)
faker (~> 2.12)
fast_blank (~> 1.0)

21
app/controllers/activitypub/claims_controller.rb

@ -0,0 +1,21 @@
# frozen_string_literal: true
class ActivityPub::ClaimsController < ActivityPub::BaseController
include SignatureVerification
include AccountOwnedConcern
skip_before_action :authenticate_user!
before_action :require_signature!
before_action :set_claim_result
def create
render json: @claim_result, serializer: ActivityPub::OneTimeKeySerializer
end
private
def set_claim_result
@claim_result = ::Keys::ClaimService.new.call(@account.id, params[:id])
end
end

48
app/controllers/activitypub/collections_controller.rb

@ -5,8 +5,9 @@ class ActivityPub::CollectionsController < ActivityPub::BaseController
include AccountOwnedConcern
before_action :require_signature!, if: :authorized_fetch_mode?
before_action :set_items
before_action :set_size
before_action :set_statuses
before_action :set_type
before_action :set_cache_headers
def show
@ -16,40 +17,53 @@ class ActivityPub::CollectionsController < ActivityPub::BaseController
private
def set_statuses
@statuses = scope_for_collection
@statuses = cache_collection(@statuses, Status)
def set_items
case params[:id]
when 'featured'
@items = begin
# Because in public fetch mode we cache the response, there would be no
# benefit from performing the check below, since a blocked account or domain
# would likely be served the cache from the reverse proxy anyway
if authorized_fetch_mode? && !signed_request_account.nil? && (@account.blocking?(signed_request_account) || (!signed_request_account.domain.nil? && @account.domain_blocking?(signed_request_account.domain)))
[]
else
cache_collection(@account.pinned_statuses, Status)
end
end
when 'devices'
@items = @account.devices
else
not_found
end
end
def set_size
case params[:id]
when 'featured'
@size = @account.pinned_statuses.count
when 'featured', 'devices'
@size = @items.size
else
not_found
end
end
def scope_for_collection
def set_type
case params[:id]
when 'featured'
# Because in public fetch mode we cache the response, there would be no
# benefit from performing the check below, since a blocked account or domain
# would likely be served the cache from the reverse proxy anyway
if authorized_fetch_mode? && !signed_request_account.nil? && (@account.blocking?(signed_request_account) || (!signed_request_account.domain.nil? && @account.domain_blocking?(signed_request_account.domain)))
Status.none
else
@account.pinned_statuses
end
@type = :ordered
when 'devices'
@type = :unordered
else
not_found
end
end
def collection_presenter
ActivityPub::CollectionPresenter.new(
id: account_collection_url(@account, params[:id]),
type: :ordered,
type: @type,
size: @size,
items: @statuses
items: @items
)
end
end

30
app/controllers/api/v1/crypto/deliveries_controller.rb

@ -0,0 +1,30 @@
# frozen_string_literal: true
class Api::V1::Crypto::DeliveriesController < Api::BaseController
before_action -> { doorkeeper_authorize! :crypto }
before_action :require_user!
before_action :set_current_device
def create
devices.each do |device_params|
DeliverToDeviceService.new.call(current_account, @current_device, device_params)
end
render_empty
end
private
def set_current_device
@current_device = Device.find_by!(access_token: doorkeeper_token)
end
def resource_params
params.require(:device)
params.permit(device: [:account_id, :device_id, :type, :body, :hmac])
end
def devices
Array(resource_params[:device])
end
end

59
app/controllers/api/v1/crypto/encrypted_messages_controller.rb

@ -0,0 +1,59 @@
# frozen_string_literal: true
class Api::V1::Crypto::EncryptedMessagesController < Api::BaseController
LIMIT = 80
before_action -> { doorkeeper_authorize! :crypto }
before_action :require_user!
before_action :set_current_device
before_action :set_encrypted_messages, only: :index
after_action :insert_pagination_headers, only: :index
def index
render json: @encrypted_messages, each_serializer: REST::EncryptedMessageSerializer
end
def clear
@current_device.encrypted_messages.up_to(params[:up_to_id]).delete_all
render_empty
end
private
def set_current_device
@current_device = Device.find_by!(access_token: doorkeeper_token)
end
def set_encrypted_messages
@encrypted_messages = @current_device.encrypted_messages.paginate_by_id(limit_param(LIMIT), params_slice(:max_id, :since_id, :min_id))
end
def insert_pagination_headers
set_pagination_headers(next_path, prev_path)
end
def next_path
api_v1_encrypted_messages_url pagination_params(max_id: pagination_max_id) if records_continue?
end
def prev_path
api_v1_encrypted_messages_url pagination_params(min_id: pagination_since_id) unless @encrypted_messages.empty?
end
def pagination_max_id
@encrypted_messages.last.id
end
def pagination_since_id
@encrypted_messages.first.id
end
def records_continue?
@encrypted_messages.size == limit_param(LIMIT)
end
def pagination_params(core_params)
params.slice(:limit).permit(:limit).merge(core_params)
end
end

25
app/controllers/api/v1/crypto/keys/claims_controller.rb

@ -0,0 +1,25 @@
# frozen_string_literal: true
class Api::V1::Crypto::Keys::ClaimsController < Api::BaseController
before_action -> { doorkeeper_authorize! :crypto }
before_action :require_user!
before_action :set_claim_results
def create
render json: @claim_results, each_serializer: REST::Keys::ClaimResultSerializer
end
private
def set_claim_results
@claim_results = devices.map { |device_params| ::Keys::ClaimService.new.call(current_account, device_params[:account_id], device_params[:device_id]) }.compact
end
def resource_params
params.permit(device: [:account_id, :device_id])
end
def devices
Array(resource_params[:device])
end
end

17
app/controllers/api/v1/crypto/keys/counts_controller.rb

@ -0,0 +1,17 @@
# frozen_string_literal: true
class Api::V1::Crypto::Keys::CountsController < Api::BaseController
before_action -> { doorkeeper_authorize! :crypto }
before_action :require_user!
before_action :set_current_device
def show
render json: { one_time_keys: @current_device.one_time_keys.count }
end
private
def set_current_device
@current_device = Device.find_by!(access_token: doorkeeper_token)
end
end

26
app/controllers/api/v1/crypto/keys/queries_controller.rb

@ -0,0 +1,26 @@
# frozen_string_literal: true
class Api::V1::Crypto::Keys::QueriesController < Api::BaseController
before_action -> { doorkeeper_authorize! :crypto }
before_action :require_user!
before_action :set_accounts
before_action :set_query_results
def create
render json: @query_results, each_serializer: REST::Keys::QueryResultSerializer
end
private
def set_accounts
@accounts = Account.where(id: account_ids).includes(:devices)
end
def set_query_results
@query_results = @accounts.map { |account| ::Keys::QueryService.new.call(account) }.compact
end
def account_ids
Array(params[:id]).map(&:to_i)
end
end

29
app/controllers/api/v1/crypto/keys/uploads_controller.rb

@ -0,0 +1,29 @@
# frozen_string_literal: true
class Api::V1::Crypto::Keys::UploadsController < Api::BaseController
before_action -> { doorkeeper_authorize! :crypto }
before_action :require_user!
def create
device = Device.find_or_initialize_by(access_token: doorkeeper_token)
device.transaction do
device.account = current_account
device.update!(resource_params[:device])
if resource_params[:one_time_keys].present? && resource_params[:one_time_keys].is_a?(Enumerable)
resource_params[:one_time_keys].each do |one_time_key_params|
device.one_time_keys.create!(one_time_key_params)
end
end
end
render json: device, serializer: REST::Keys::DeviceSerializer
end
private
def resource_params
params.permit(device: [:device_id, :name, :fingerprint_key, :identity_key], one_time_keys: [:key_id, :key, :signature])
end
end

2
app/controllers/statuses_controller.rb

@ -42,7 +42,7 @@ class StatusesController < ApplicationController
def activity
expires_in 3.minutes, public: @status.distributable? && public_fetch_mode?
render_with_cache json: @status, content_type: 'application/activity+json', serializer: ActivityPub::ActivitySerializer, adapter: ActivityPub::Adapter
render_with_cache json: ActivityPub::ActivityPresenter.from_status(@status), content_type: 'application/activity+json', serializer: ActivityPub::ActivitySerializer, adapter: ActivityPub::Adapter
end
def embed

50
app/lib/activitypub/activity/create.rb

@ -2,6 +2,45 @@
class ActivityPub::Activity::Create < ActivityPub::Activity
def perform
case @object['type']
when 'EncryptedMessage'
create_encrypted_message
else
create_status
end
end
private
def create_encrypted_message
return reject_payload! if invalid_origin?(@object['id']) || @options[:delivered_to_account_id].blank?
target_account = Account.find(@options[:delivered_to_account_id])
target_device = target_account.devices.find_by(device_id: @object.dig('to', 'deviceId'))
return if target_device.nil?
target_device.encrypted_messages.create!(
from_account: @account,
from_device_id: @object.dig('attributedTo', 'deviceId'),
type: @object['messageType'],
body: @object['cipherText'],
digest: @object.dig('digest', 'digestValue'),
message_franking: message_franking.to_token
)
end
def message_franking
MessageFranking.new(
hmac: @object.dig('digest', 'digestValue'),
original_franking: @object['messageFranking'],
source_account_id: @account.id,
target_account_id: @options[:delivered_to_account_id],
timestamp: Time.now.utc
)
end
def create_status
return reject_payload! if unsupported_object_type? || invalid_origin?(@object['id']) || Tombstone.exists?(uri: @object['id']) || !related_to_local_activity?
RedisLock.acquire(lock_options) do |lock|
@ -23,8 +62,6 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
@status
end
private
def audience_to
@object['to'] || @json['to']
end
@ -262,6 +299,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
def poll_vote!
poll = replied_to_status.preloadable_poll
already_voted = true
RedisLock.acquire(poll_lock_options) do |lock|
if lock.acquired?
already_voted = poll.votes.where(account: @account).exists?
@ -270,20 +308,24 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
raise Mastodon::RaceConditionError
end
end
increment_voters_count! unless already_voted
ActivityPub::DistributePollUpdateWorker.perform_in(3.minutes, replied_to_status.id) unless replied_to_status.preloadable_poll.hide_totals?
end
def resolve_thread(status)
return unless status.reply? && status.thread.nil? && Request.valid_url?(in_reply_to_uri)
ThreadResolveWorker.perform_async(status.id, in_reply_to_uri)
end
def fetch_replies(status)
collection = @object['replies']
return if collection.nil?
replies = ActivityPub::FetchRepliesService.new.call(status, collection, false)
return unless replies.nil?
uri = value_or_id(collection)
ActivityPub::FetchRepliesWorker.perform_async(status.id, uri) unless uri.nil?
end
@ -291,6 +333,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
def conversation_from_uri(uri)
return nil if uri.nil?
return Conversation.find_by(id: OStatus::TagManager.instance.unique_tag_to_local_id(uri, 'Conversation')) if OStatus::TagManager.instance.local_id?(uri)
begin
Conversation.find_or_create_by!(uri: uri)
rescue ActiveRecord::RecordInvalid, ActiveRecord::RecordNotUnique
@ -404,6 +447,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
def skip_download?
return @skip_download if defined?(@skip_download)
@skip_download ||= DomainBlock.reject_media?(@account.domain)
end
@ -436,11 +480,13 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
def forward_for_reply
return unless @json['signature'].present? && reply_to_local?
ActivityPub::RawDistributionWorker.perform_async(Oj.dump(@json), replied_to_status.account_id, [@account.preferred_inbox_url])
end
def increment_voters_count!
poll = replied_to_status.preloadable_poll
unless poll.voters_count.nil?
poll.voters_count = poll.voters_count + 1
poll.save

1
app/lib/activitypub/adapter.rb

@ -22,6 +22,7 @@ class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base
blurhash: { 'toot' => 'http://joinmastodon.org/ns#', 'blurhash' => 'toot:blurhash' },
discoverable: { 'toot' => 'http://joinmastodon.org/ns#', 'discoverable' => 'toot:discoverable' },
voters_count: { 'toot' => 'http://joinmastodon.org/ns#', 'votersCount' => 'toot:votersCount' },
olm: { 'toot' => 'http://joinmastodon.org/ns#', 'Device' => 'toot:Device', 'Ed25519Signature' => 'toot:Ed25519Signature', 'Ed25519Key' => 'toot:Ed25519Key', 'Curve25519Key' => 'toot:Curve25519Key', 'EncryptedMessage' => 'toot:EncryptedMessage', 'publicKeyBase64' => 'toot:publicKeyBase64', 'deviceId' => 'toot:deviceId', 'claim' => { '@type' => '@id', '@id' => 'toot:claim' }, 'fingerprintKey' => { '@type' => '@id', '@id' => 'toot:fingerprintKey' }, 'identityKey' => { '@type' => '@id', '@id' => 'toot:identityKey' }, 'devices' => { '@type' => '@id', '@id' => 'toot:devices' }, 'messageFranking' => 'toot:messageFranking', 'messageType' => 'toot:messageType', 'cipherText' => 'toot:cipherText' },
}.freeze
def self.default_key_transform

2
app/lib/inline_renderer.rb

@ -19,6 +19,8 @@ class InlineRenderer
serializer = REST::AnnouncementSerializer
when :reaction
serializer = REST::ReactionSerializer
when :encrypted_message
serializer = REST::EncryptedMessageSerializer
else
return
end

1
app/models/account.rb

@ -49,6 +49,7 @@
# hide_collections :boolean
# avatar_storage_schema_version :integer
# header_storage_schema_version :integer
# devices_url :string
#
class Account < ApplicationRecord

1
app/models/concerns/account_associations.rb

@ -9,6 +9,7 @@ module AccountAssociations
# Identity proofs
has_many :identity_proofs, class_name: 'AccountIdentityProof', dependent: :destroy, inverse_of: :account
has_many :devices, dependent: :destroy, inverse_of: :account
# Timelines
has_many :statuses, inverse_of: :account, dependent: :destroy

35
app/models/device.rb

@ -0,0 +1,35 @@
# frozen_string_literal: true
# == Schema Information
#
# Table name: devices
#
# id :bigint(8) not null, primary key
# access_token_id :bigint(8)
# account_id :bigint(8)
# device_id :string default(""), not null
# name :string default(""), not null
# fingerprint_key :text default(""), not null
# identity_key :text default(""), not null
# created_at :datetime not null
# updated_at :datetime not null
#
class Device < ApplicationRecord
belongs_to :access_token, class_name: 'Doorkeeper::AccessToken'
belongs_to :account
has_many :one_time_keys, dependent: :destroy, inverse_of: :device
has_many :encrypted_messages, dependent: :destroy, inverse_of: :device
validates :name, :fingerprint_key, :identity_key, presence: true
validates :fingerprint_key, :identity_key, ed25519_key: true
before_save :invalidate_associations, if: -> { device_id_changed? || fingerprint_key_changed? || identity_key_changed? }
private
def invalidate_associations
one_time_keys.destroy_all
encrypted_messages.destroy_all
end
end

50
app/models/encrypted_message.rb

@ -0,0 +1,50 @@
# frozen_string_literal: true
# == Schema Information
#
# Table name: encrypted_messages
#
# id :bigint(8) not null, primary key
# device_id :bigint(8)
# from_account_id :bigint(8)
# from_device_id :string default(""), not null
# type :integer default(0), not null
# body :text default(""), not null
# digest :text default(""), not null
# message_franking :text default(""), not null
# created_at :datetime not null
# updated_at :datetime not null
#
class EncryptedMessage < ApplicationRecord
self.inheritance_column = nil
include Paginable
scope :up_to, ->(id) { where(arel_table[:id].lteq(id)) }
belongs_to :device
belongs_to :from_account, class_name: 'Account'
around_create Mastodon::Snowflake::Callbacks
after_commit :push_to_streaming_api
private
def push_to_streaming_api
Rails.logger.info(streaming_channel)
Rails.logger.info(subscribed_to_timeline?)
return if destroyed? || !subscribed_to_timeline?
PushEncryptedMessageWorker.perform_async(id)
end
def subscribed_to_timeline?
Redis.current.exists("subscribed:#{streaming_channel}")
end
def streaming_channel
"timeline:#{device.account_id}:#{device.device_id}"
end
end

19
app/models/message_franking.rb

@ -0,0 +1,19 @@
# frozen_string_literal: true
class MessageFranking
attr_reader :hmac, :source_account_id, :target_account_id,
:timestamp, :original_franking
def initialize(attributes = {})
@hmac = attributes[:hmac]
@source_account_id = attributes[:source_account_id]
@target_account_id = attributes[:target_account_id]
@timestamp = attributes[:timestamp]
@original_franking = attributes[:original_franking]
end
def to_token
crypt = ActiveSupport::MessageEncryptor.new(SystemKey.current_key, serializer: Oj)
crypt.encrypt_and_sign(self)
end
end

21
app/models/one_time_key.rb

@ -0,0 +1,21 @@
# frozen_string_literal: true
# == Schema Information
#
# Table name: one_time_keys
#
# id :bigint(8) not null, primary key
# device_id :bigint(8)
# key_id :string default(""), not null
# key :text default(""), not null
# signature :text default(""), not null
# created_at :datetime not null
# updated_at :datetime not null
#
class OneTimeKey < ApplicationRecord
belongs_to :device
validates :key_id, :key, :signature, presence: true
validates :key, ed25519_key: true
validates :signature, ed25519_signature: { message: :key, verify_key: ->(one_time_key) { one_time_key.device.fingerprint_key } }
end

41
app/models/system_key.rb

@ -0,0 +1,41 @@
# frozen_string_literal: true
# == Schema Information
#
# Table name: system_keys
#
# id :bigint(8) not null, primary key
# key :binary
# created_at :datetime not null
# updated_at :datetime not null
#
class SystemKey < ApplicationRecord
ROTATION_PERIOD = 1.week.freeze
before_validation :set_key
scope :expired, ->(now = Time.now.utc) { where(arel_table[:created_at].lt(now - ROTATION_PERIOD * 3)) }
class << self
def current_key
previous_key = order(id: :asc).last
if previous_key && previous_key.created_at >= ROTATION_PERIOD.ago
previous_key.key
else
create.key
end
end
end
private
def set_key
return if key.present?
cipher = OpenSSL::Cipher.new('AES-256-GCM')
cipher.encrypt
self.key = cipher.random_key
end
end

41
app/presenters/activitypub/activity_presenter.rb

@ -0,0 +1,41 @@
# frozen_string_literal: true
class ActivityPub::ActivityPresenter < ActiveModelSerializers::Model
attributes :id, :type, :actor, :published, :to, :cc, :virtual_object
class << self
def from_status(status)
new.tap do |presenter|
presenter.id = ActivityPub::TagManager.instance.activity_uri_for(status)
presenter.type = status.reblog? ? 'Announce' : 'Create'
presenter.actor = ActivityPub::TagManager.instance.uri_for(status.account)
presenter.published = status.created_at
presenter.to = ActivityPub::TagManager.instance.to(status)
presenter.cc = ActivityPub::TagManager.instance.cc(status)
presenter.virtual_object = begin
if status.reblog?
if status.account == status.proper.account && status.proper.private_visibility? && status.local?
status.proper
else
ActivityPub::TagManager.instance.uri_for(status.proper)
end
else
status.proper
end
end
end
end
def from_encrypted_message(encrypted_message)
new.tap do |presenter|
presenter.id = ActivityPub::TagManager.instance.generate_uri_for(nil)
presenter.type = 'Create'
presenter.actor = ActivityPub::TagManager.instance.uri_for(encrypted_message.source_account)
presenter.published = Time.now.utc
presenter.to = ActivityPub::TagManager.instance.uri_for(encrypted_message.target_account)
presenter.virtual_object = encrypted_message
end
end
end
end

54
app/serializers/activitypub/activity_serializer.rb

@ -1,52 +1,22 @@
# frozen_string_literal: true
class ActivityPub::ActivitySerializer < ActivityPub::Serializer
attributes :id, :type, :actor, :published, :to, :cc
has_one :proper, key: :object, serializer: ActivityPub::NoteSerializer, if: :serialize_object?
attribute :proper_uri, key: :object, unless: :serialize_object?
attribute :atom_uri, if: :announce?
def id
ActivityPub::TagManager.instance.activity_uri_for(object)
def self.serializer_for(model, options)
case model.class.name
when 'Status'
ActivityPub::NoteSerializer
when 'DeliverToDeviceService::EncryptedMessage'
ActivityPub::EncryptedMessageSerializer
else
super
end
end
def type
announce? ? 'Announce' : 'Create'
end
attributes :id, :type, :actor, :published, :to, :cc
def actor
ActivityPub::TagManager.instance.uri_for(object.account)
end
has_one :virtual_object, key: :object
def published
object.created_at.iso8601
end
def to
ActivityPub::TagManager.instance.to(object)
end
def cc
ActivityPub::TagManager.instance.cc(object)
end
def proper_uri
ActivityPub::TagManager.instance.uri_for(object.proper)
end
def atom_uri
OStatus::TagManager.instance.uri_for(object)
end
def announce?
object.reblog?
end
def serialize_object?
return true unless announce?
# Serialize private self-boosts of local toots
object.account == object.proper.account && object.proper.private_visibility? && object.local?
object.published.iso8601
end
end

9
app/serializers/activitypub/actor_serializer.rb

@ -7,7 +7,7 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer
context_extensions :manually_approves_followers, :featured, :also_known_as,
:moved_to, :property_value, :identity_proof,
:discoverable
:discoverable, :olm
attributes :id, :type, :following, :followers,
:inbox, :outbox, :featured,
@ -20,6 +20,7 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer
has_many :virtual_tags, key: :tag
has_many :virtual_attachments, key: :attachment
attribute :devices, unless: :instance_actor?
attribute :moved_to, if: :moved?
attribute :also_known_as, if: :also_known_as?
@ -38,7 +39,7 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer
has_one :icon, serializer: ActivityPub::ImageSerializer, if: :avatar_exists?
has_one :image, serializer: ActivityPub::ImageSerializer, if: :header_exists?
delegate :moved?, to: :object
delegate :moved?, :instance_actor?, to: :object
def id
object.instance_actor? ? instance_actor_url : account_url(object)
@ -68,6 +69,10 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer
object.instance_actor? ? instance_actor_inbox_url : account_inbox_url(object)
end
def devices
account_collection_url(object, :devices)
end
def outbox
account_outbox_url(object)
end

13
app/serializers/activitypub/collection_serializer.rb

@ -2,9 +2,16 @@
class ActivityPub::CollectionSerializer < ActivityPub::Serializer
def self.serializer_for(model, options)
return ActivityPub::NoteSerializer if model.class.name == 'Status'
return ActivityPub::CollectionSerializer if model.class.name == 'ActivityPub::CollectionPresenter'
super
case model.class.name
when 'Status'
ActivityPub::NoteSerializer
when 'Device'
ActivityPub::DeviceSerializer
when 'ActivityPub::CollectionPresenter'
ActivityPub::CollectionSerializer
else
super
end
end
attribute :id, if: -> { object.id.present? }

52
app/serializers/activitypub/device_serializer.rb

@ -0,0 +1,52 @@
# frozen_string_literal: true
class ActivityPub::DeviceSerializer < ActivityPub::Serializer
context_extensions :olm
include RoutingHelper
class FingerprintKeySerializer < ActivityPub::Serializer
attributes :type, :public_key_base64
def type
'Ed25519Key'
end
def public_key_base64
object.fingerprint_key
end
end
class IdentityKeySerializer < ActivityPub::Serializer
attributes :type, :public_key_base64
def type
'Curve25519Key'
end
def public_key_base64
object.identity_key
end
end
attributes :device_id, :type, :name, :claim
has_one :fingerprint_key, serializer: FingerprintKeySerializer
has_one :identity_key, serializer: IdentityKeySerializer
def type
'Device'
end
def claim
account_claim_url(object.account, id: object.device_id)
end
def fingerprint_key
object
end
def identity_key
object
end
end

61
app/serializers/activitypub/encrypted_message_serializer.rb

@ -0,0 +1,61 @@
# frozen_string_literal: true
class ActivityPub::EncryptedMessageSerializer < ActivityPub::Serializer
context :security
context_extensions :olm
class DeviceSerializer < ActivityPub::Serializer
attributes :type, :device_id
def type
'Device'
end
def device_id
object
end
end
class DigestSerializer < ActivityPub::Serializer
attributes :type, :digest_algorithm, :digest_value
def type
'Digest'
end
def digest_algorithm
'http://www.w3.org/2000/09/xmldsig#hmac-sha256'
end
def digest_value
object
end
end
attributes :type, :message_type, :cipher_text, :message_franking
has_one :attributed_to, serializer: DeviceSerializer
has_one :to, serializer: DeviceSerializer
has_one :digest, serializer: DigestSerializer
def type
'EncryptedMessage'
end
def attributed_to
object.source_device.device_id
end
def to
object.target_device_id
end
def message_type
object.type
end
def cipher_text
object.body
end
end

35
app/serializers/activitypub/one_time_key_serializer.rb

@ -0,0 +1,35 @@
# frozen_string_literal: true
class ActivityPub::OneTimeKeySerializer < ActivityPub::Serializer
context :security
context_extensions :olm
class SignatureSerializer < ActivityPub::Serializer
attributes :type, :signature_value
def type
'Ed25519Signature'
end
def signature_value
object.signature
end
end
attributes :key_id, :type, :public_key_base64
has_one :signature, serializer: SignatureSerializer
def type
'Curve25519Key'
end
def public_key_base64
object.key
end
def signature
object
end
end

11
app/serializers/activitypub/outbox_serializer.rb

@ -2,7 +2,14 @@
class ActivityPub::OutboxSerializer < ActivityPub::CollectionSerializer
def self.serializer_for(model, options)
return ActivityPub::ActivitySerializer if model.is_a?(Status)
super
if model.class.name == 'ActivityPub::ActivityPresenter'
ActivityPub::ActivitySerializer
else
super
end
end
def items
object.items.map { |status| ActivityPub::ActivityPresenter.from_status(status) }
end
end

6
app/serializers/activitypub/undo_announce_serializer.rb

@ -3,7 +3,7 @@
class ActivityPub::UndoAnnounceSerializer < ActivityPub::Serializer
attributes :id, :type, :actor, :to
has_one :object, serializer: ActivityPub::ActivitySerializer
has_one :virtual_object, key: :object, serializer: ActivityPub::ActivitySerializer
def id
[ActivityPub::TagManager.instance.uri_for(object.account), '#announces/', object.id, '/undo'].join
@ -20,4 +20,8 @@ class ActivityPub::UndoAnnounceSerializer < ActivityPub::Serializer
def to
[ActivityPub::TagManager::COLLECTIONS[:public]]
end
def virtual_object
ActivityPub::ActivityPresenter.from_status(object)
end
end

18
app/serializers/rest/encrypted_message_serializer.rb

@ -0,0 +1,18 @@
# frozen_string_literal: true
class REST::EncryptedMessageSerializer < ActiveModel::Serializer
attributes :id, :account_id, :device_id,
:type, :body, :digest, :message_franking
def id
object.id.to_s
end
def account_id
object.from_account_id.to_s
end
def device_id
object.from_device_id
end
end

9
app/serializers/rest/keys/claim_result_serializer.rb

@ -0,0 +1,9 @@
# frozen_string_literal: true
class REST::Keys::ClaimResultSerializer < ActiveModel::Serializer
attributes :account_id, :device_id, :key_id, :key, :signature
def account_id
object.account.id.to_s
end
end

6
app/serializers/rest/keys/device_serializer.rb

@ -0,0 +1,6 @@
# frozen_string_literal: true
class REST::Keys::DeviceSerializer < ActiveModel::Serializer
attributes :device_id, :name, :identity_key,
:fingerprint_key
end

11
app/serializers/rest/keys/query_result_serializer.rb

@ -0,0 +1,11 @@
# frozen_string_literal: true
class REST::Keys::QueryResultSerializer < ActiveModel::Serializer
attributes :account_id
has_many :devices, serializer: REST::Keys::DeviceSerializer
def account_id
object.account.id.to_s
end
end

1
app/services/activitypub/process_account_service.rb

@ -76,6 +76,7 @@ class ActivityPub::ProcessAccountService < BaseService
@account.shared_inbox_url = (@json['endpoints'].is_a?(Hash) ? @json['endpoints']['sharedInbox'] : @json['sharedInbox']) || ''
@account.followers_url = @json['followers'] || ''
@account.featured_collection_url = @json['featured'] || ''
@account.devices_url = @json['devices'] || ''
@account.url = url || @uri
@account.uri = @uri
@account.display_name = @json['name'] || ''

2
app/services/backup_service.rb

@ -22,7 +22,7 @@ class BackupService < BaseService
account.statuses.with_includes.reorder(nil).find_in_batches do |statuses|
statuses.each do |status|
item = serialize_payload(status, ActivityPub::ActivitySerializer, signer: @account)
item = serialize_payload(ActivityPub::ActivityPresenter.from_status(status), ActivityPub::ActivitySerializer, signer: @account)
item.delete(:'@context')
unless item[:type] == 'Announce' || item[:object][:attachment].blank?

78
app/services/deliver_to_device_service.rb

@ -0,0 +1,78 @@
# frozen_string_literal: true
class DeliverToDeviceService < BaseService
include Payloadable
class EncryptedMessage < ActiveModelSerializers::Model
attributes :source_account, :target_account, :source_device,
:target_device_id, :type, :body, :digest,
:message_franking
end
def call(source_account, source_device, options = {})
@source_account = source_account
@source_device = source_device
@target_account = Account.find(options[:account_id])
@target_device_id = options[:device_id]
@body = options[:body]
@type = options[:type]
@hmac = options[:hmac]
set_message_franking!
if @target_account.local?
deliver_to_local!
else
deliver_to_remote!
end
end
private
def set_message_franking!
@message_franking = message_franking.to_token
end
def deliver_to_local!
target_device = @target_account.devices.find_by!(device_id: @target_device_id)
target_device.encrypted_messages.create!(
from_account: @source_account,
from_device_id: @source_device.device_id,
type: @type,
body: @body,
digest: @hmac,
message_franking: @message_franking
)
end
def deliver_to_remote!
ActivityPub::DeliveryWorker.perform_async(
Oj.dump(serialize_payload(ActivityPub::ActivityPresenter.from_encrypted_message(encrypted_message), ActivityPub::ActivitySerializer)),
@source_account.id,
@target_account.inbox_url
)
end
def message_franking
MessageFranking.new(
source_account_id: @source_account.id,
target_account_id: @target_account.id,
hmac: @hmac,
timestamp: Time.now.utc
)
end
def encrypted_message
EncryptedMessage.new(
source_account: @source_account,
target_account: @target_account,
source_device: @source_device,
target_device_id: @target_device_id,
type: @type,
body: @body,
digest: @hmac,
message_franking: @message_franking
)
end
end

77
app/services/keys/claim_service.rb

@ -0,0 +1,77 @@
# frozen_string_literal: true
class Keys::ClaimService < BaseService
HEADERS = { 'Content-Type' => 'application/activity+json' }.freeze
class Result < ActiveModelSerializers::Model
attributes :account, :device_id, :key_id,
:key, :signature
def initialize(account, device_id, key_attributes = {})
@account = account
@device_id = device_id
@key_id = key_attributes[:key_id]
@key = key_attributes[:key]
@signature = key_attributes[:signature]
end
end
def call(source_account, target_account_id, device_id)
@source_account = source_account
@target_account = Account.find(target_account_id)
@device_id = device_id
if @target_account.local?
claim_local_key!
else
claim_remote_key!
end
rescue ActiveRecord::RecordNotFound
nil
end
private
def claim_local_key!
device = @target_account.devices.find_by(device_id: @device_id)
key = nil
ApplicationRecord.transaction do
key = device.one_time_keys.order(Arel.sql('random()')).first!
key.destroy!
end
@result = Result.new(@target_account, @device_id, key)
end
def claim_remote_key!
query_result = QueryService.new.call(@target_account)
device = query_result.find(@device_id)
return unless device.present? && device.valid_claim_url?
json = fetch_resource_with_post(device.claim_url)
return unless json.present? && json['publicKeyBase64'].present?
@result = Result.new(@target_account, @device_id, key_id: json['id'], key: json['publicKeyBase64'], signature: json.dig('signature', 'signatureValue'))
rescue HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::Error => e
Rails.logger.debug "Claiming one-time key for #{@target_account.acct}:#{@device_id} failed: #{e}"
nil
end
def fetch_resource_with_post(uri)
build_post_request(uri).perform do |response|
raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response)
body_to_json(response.body_with_limit) if response.code == 200
end
end
def build_post_request(uri)
Request.new(:post, uri).tap do |request|
request.on_behalf_of(@source_account, :uri)
request.add_headers(HEADERS)
end
end
end

75
app/services/keys/query_service.rb

@ -0,0 +1,75 @@
# frozen_string_literal: true
class Keys::QueryService < BaseService
include JsonLdHelper
class Result < ActiveModelSerializers::Model
attributes :account, :devices
def initialize(account, devices)
@account = account
@devices = devices || []
end
def find(device_id)
@devices.find { |device| device.device_id == device_id }
end
end
class Device < ActiveModelSerializers::Model
attributes :device_id, :name, :identity_key, :fingerprint_key
def initialize(attributes = {})
@device_id = attributes[:device_id]
@name = attributes[:name]
@identity_key = attributes[:identity_key]
@fingerprint_key = attributes[:fingerprint_key]
@claim_url = attributes[:claim_url]
end
def valid_claim_url?
return false if @claim_url.blank?
begin
parsed_url = Addressable::URI.parse(@claim_url).normalize
rescue Addressable::URI::InvalidURIError
return false
end
%w(http https).include?(parsed_url.scheme) && parsed_url.host.present?
end
end
def call(account)
@account = account
if @account.local?
query_local_devices!
else
query_remote_devices!
end
Result.new(@account, @devices)
end
private
def query_local_devices!
@devices = @account.devices.map { |device| Device.new(device) }
end
def query_remote_devices!
return if @account.devices_url.blank?
json = fetch_resource(@account.devices_url)
return if json['items'].blank?
@devices = json['items'].map do |device|
Device.new(device_id: device['id'], name: device['name'], identity_key: device.dig('identityKey', 'publicKeyBase64'), fingerprint_key: device.dig('fingerprintKey', 'publicKeyBase64'), claim_url: device['claim'])
end
rescue HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::Error => e
Rails.logger.debug "Querying devices for #{@account.acct} failed: #{e}"
nil
end
end

2
app/services/process_mentions_service.rb

@ -65,7 +65,7 @@ class ProcessMentionsService < BaseService
def activitypub_json
return @activitypub_json if defined?(@activitypub_json)
@activitypub_json = Oj.dump(serialize_payload(@status, ActivityPub::ActivitySerializer, signer: @status.account))
@activitypub_json = Oj.dump(serialize_payload(ActivityPub::ActivityPresenter.from_status(@status), ActivityPub::ActivitySerializer, signer: @status.account))
end
def resolve_account_service

2
app/services/reblog_service.rb

@ -60,6 +60,6 @@ class ReblogService < BaseService
end
def build_json(reblog)
Oj.dump(serialize_payload(reblog, ActivityPub::ActivitySerializer, signer: reblog.account))
Oj.dump(serialize_payload(ActivityPub::ActivityPresenter.from_status(reblog), ActivityPub::ActivitySerializer, signer: reblog.account))
end
end

19
app/validators/ed25519_key_validator.rb

@ -0,0 +1,19 @@
# frozen_string_literal: true
class Ed25519KeyValidator < ActiveModel::EachValidator
def validate_each(record, attribute, value)
return if value.blank?
key = Base64.decode64(value)
record.errors[attribute] << I18n.t('crypto.errors.invalid_key') unless verified?(key)
end
private
def verified?(key)
Ed25519.validate_key_bytes(key)
rescue ArgumentError
false
end
end

29
app/validators/ed25519_signature_validator.rb

@ -0,0 +1,29 @@
# frozen_string_literal: true
class Ed25519SignatureValidator < ActiveModel::EachValidator
def validate_each(record, attribute, value)
return if value.blank?
verify_key = Ed25519::VerifyKey.new(Base64.decode64(option_to_value(record, :verify_key)))
signature = Base64.decode64(value)
message = option_to_value(record, :message)
record.errors[attribute] << I18n.t('crypto.errors.invalid_signature') unless verified?(verify_key, signature, message)
end
private
def verified?(verify_key, signature, message)
verify_key.verify(signature, message)
rescue Ed25519::VerifyError, ArgumentError
false
end
def option_to_value(record, key)
if options[key].is_a?(Proc)
options[key].call(record)
else
record.public_send(options[key])
end
end
end

2
app/workers/activitypub/distribution_worker.rb

@ -43,7 +43,7 @@ class ActivityPub::DistributionWorker
end
def payload
@payload ||= Oj.dump(serialize_payload(@status, ActivityPub::ActivitySerializer, signer: @account))
@payload ||= Oj.dump(serialize_payload(ActivityPub::ActivityPresenter.from_status(@status), ActivityPub::ActivitySerializer, signer: @account))
end
def relay!

2
app/workers/activitypub/reply_distribution_worker.rb

@ -29,6 +29,6 @@ class ActivityPub::ReplyDistributionWorker
end
def payload
@payload ||= Oj.dump(serialize_payload(@status, ActivityPub::ActivitySerializer, signer: @status.account))
@payload ||= Oj.dump(serialize_payload(ActivityPub::ActivityPresenter.from_status(@status), ActivityPub::ActivitySerializer, signer: @status.account))
end
end

3
app/workers/push_conversation_worker.rb

@ -2,13 +2,14 @@
class PushConversationWorker
include Sidekiq::Worker
include Redisable
def perform(conversation_account_id)
conversation = AccountConversation.find(conversation_account_id)
message = InlineRenderer.render(conversation, conversation.account, :conversation)
timeline_id = "timeline:direct:#{conversation.account_id}"
Redis.current.publish(timeline_id, Oj.dump(event: :conversation, payload: message, queued_at: (Time.now.to_f * 1000.0).to_i))
redis.publish(timeline_id, Oj.dump(event: :conversation, payload: message, queued_at: (Time.now.to_f * 1000.0).to_i))
rescue ActiveRecord::RecordNotFound
true
end

16
app/workers/push_encrypted_message_worker.rb

@ -0,0 +1,16 @@
# frozen_string_literal: true
class PushEncryptedMessageWorker
include Sidekiq::Worker
include Redisable
def perform(encrypted_message_id)
encrypted_message = EncryptedMessage.find(encrypted_message_id)
message = InlineRenderer.render(encrypted_message, nil, :encrypted_message)
timeline_id = "timeline:#{encrypted_message.device.account_id}:#{encrypted_message.device.device_id}"
redis.publish(timeline_id, Oj.dump(event: :encrypted_message, payload: message, queued_at: (Time.now.to_f * 1000.0).to_i))
rescue ActiveRecord::RecordNotFound
true
end
end

1
app/workers/scheduler/doorkeeper_cleanup_scheduler.rb

@ -8,5 +8,6 @@ class Scheduler::DoorkeeperCleanupScheduler
def perform
Doorkeeper::