From 256e3adc1d9508423aab8fcfb13745c9f85ff948 Mon Sep 17 00:00:00 2001 From: Daniel Hunsaker Date: Tue, 23 May 2017 08:54:44 -0600 Subject: [PATCH] Add Support for Nanobox (#1709) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Nanobox Support - Added support for running Mastodon using Nanobox, both for local development, and for deployment to production - Dev mode tested and is working properly - Deployment is undergoing test as of this writing. If it works, this line will be amended to state success; if not, one or more subsequent commits will provide fixes. * [nanobox] Resolve Deploy Issues Everything seems to work except routing to the streaming API. Will investigate with the Nanobox staff and make fix commits if needed. Changes made: - Also need `NODE_ENV` in production - Node runs on `:4000` - Use `envsubst` to commit `.env.production` values, since `dotEnv` packages don't always support referencing other variables - Can't precompile assets after `transform` hook, but do this locally so it only has to be done once. - Rails won't create `production.log` on its own, so we do this ourselves. - Some `start` commands run from `/data/` for some reason, so use absolute paths in command arguments * [nanobox] Update Ruby version * [nanobox] Fix db.rake Ruby code style issues * [nanobox] Minor Fixes Some minor adjustments to improve functionality: - Fixed routing to `web.stream` instances - Adjust `.env.nanobox` to properly generate a default `SMTP_FROM_ADDRESS` via `envsubst` - Update Nginx configs to properly support the needed HTTP version and headers for proper functionality (the streaming API doesn't work without some of these settings in place) * [nanobox] Move usage info to docs repo * [nanobox] Updates for 1.2.x - Need to leave out `pkg-config` since Nanobox deploys without Ruby's headers - create a gem group to exclude the gem during Nanobox installs, but allow it to remain part of the default set otherwise - Update cron jobs to cover new/updated Rake tasks - Update `.env.nanobox` to include latest defaults and additions * [nanobox] Fix for nokogumbo, added in 1.3.x Apparently, nokogumbo (pulled in by sanitize, added with `OEmbed Support for PreviewCard` (#2337) - 88725d6) tries to install before nokogiri, despite needing nokogiri available to build properly. Instruct it to use the same settings as nokogiri does when building nokogiri directly, instead of via bundler. * [nanobox] Set NODE_ENV during asset compile The switch to WebPack will rely on the local value of the NODE_ENV evar, so set it to production during asset compilation. * [nanobox] Rebase on master; update Nginx configs - `pkg-config` Gem no longer causes issues in Nanobox, so revert the Gemfile change which allowed excluding it - Update Nginx configuration files with latest recommendations from production documentation - Rebase on master to Get This Merged™ Everything should be golden! --- .env.nanobox | 109 +++++++++++++++++++++++ .nanoignore | 20 +++++ boxfile.yml | 158 ++++++++++++++++++++++++++++++++++ lib/tasks/db.rake | 16 ++++ nanobox/nginx-local.conf | 76 ++++++++++++++++ nanobox/nginx-stream.conf.erb | 57 ++++++++++++ nanobox/nginx-web.conf.erb | 65 ++++++++++++++ 7 files changed, 501 insertions(+) create mode 100644 .env.nanobox create mode 100644 .nanoignore create mode 100644 boxfile.yml create mode 100644 lib/tasks/db.rake create mode 100644 nanobox/nginx-local.conf create mode 100644 nanobox/nginx-stream.conf.erb create mode 100644 nanobox/nginx-web.conf.erb diff --git a/.env.nanobox b/.env.nanobox new file mode 100644 index 000000000..4a89e0e41 --- /dev/null +++ b/.env.nanobox @@ -0,0 +1,109 @@ +# Service dependencies +# You may set REDIS_URL instead for more advanced options +REDIS_HOST=$DATA_REDIS_HOST +REDIS_PORT=6379 +# REDIS_DB=0 + +# You may set DATABASE_URL instead for more advanced options +DB_HOST=$DATA_DB_HOST +DB_USER=$DATA_DB_USER +DB_NAME=gonano +DB_PASS=$DATA_DB_PASS +DB_PORT=5432 + +# Federation +# Note: Changing LOCAL_DOMAIN or LOCAL_HTTPS at a later time will cause unwanted side effects. +# LOCAL_DOMAIN should *NOT* contain the protocol part of the domain e.g https://example.com. +LOCAL_DOMAIN=${APP_NAME}.nanoapp.io +LOCAL_HTTPS=false + +# Use this only if you need to run mastodon on a different domain than the one used for federation. +# You can read more about this option on https://github.com/tootsuite/documentation/blob/master/Running-Mastodon/Serving_a_different_domain.md +# DO *NOT* USE THIS UNLESS YOU KNOW *EXACTLY* WHAT YOU ARE DOING. +# WEB_DOMAIN=mastodon.example.com + +# Use this if you want to have several aliases handler@example1.com +# handler@example2.com etc. for the same user. LOCAL_DOMAIN should not +# be added. Comma separated values +# ALTERNATE_DOMAINS=example1.com,example2.com + +# Application secrets +# Generate each with the `rake secret` task (`nanobox run bundle exec rake secret`) +PAPERCLIP_SECRET=$PAPERCLIP_SECRET +SECRET_KEY_BASE=$SECRET_KEY_BASE +OTP_SECRET=$OTP_SECRET + +# Registrations +# Single user mode will disable registrations and redirect frontpage to the first profile +# SINGLE_USER_MODE=true +# Prevent registrations with following e-mail domains +# EMAIL_DOMAIN_BLACKLIST=example1.com|example2.de|etc +# Only allow registrations with the following e-mail domains +# EMAIL_DOMAIN_WHITELIST=example1.com|example2.de|etc + +# Optionally change default language +# DEFAULT_LOCALE=de + +# E-mail configuration +# Note: Mailgun and SparkPost (https://sparkpo.st/smtp) each have good free tiers +# If you want to use an SMTP server without authentication (e.g local Postfix relay) +# then set SMTP_AUTH_METHOD and SMTP_OPENSSL_VERIFY_MODE to 'none' and +# *comment* SMTP_LOGIN and SMTP_PASSWORD (leaving them blank is not enough). +SMTP_SERVER=$SMTP_SERVER +SMTP_PORT=587 +SMTP_LOGIN=$SMTP_LOGIN +SMTP_PASSWORD=$SMTP_PASSWORD +SMTP_FROM_ADDRESS=notifications@${APP_NAME}.nanoapp.io +#SMTP_DOMAIN= # defaults to LOCAL_DOMAIN +#SMTP_DELIVERY_METHOD=smtp # delivery method can also be sendmail +#SMTP_AUTH_METHOD=plain +#SMTP_CA_FILE=/etc/ssl/certs/ca-certificates.crt +#SMTP_OPENSSL_VERIFY_MODE=peer +#SMTP_ENABLE_STARTTLS_AUTO=true + + +# Optional user upload path and URL (images, avatars). Default is :rails_root/public/system. If you set this variable, you are responsible for making your HTTP server (eg. nginx) serve these files. +# PAPERCLIP_ROOT_PATH=/var/lib/mastodon/public-system +# PAPERCLIP_ROOT_URL=/system + +# Optional asset host for multi-server setups +# CDN_HOST=assets.example.com + +# S3 (optional) +# S3_ENABLED=true +# S3_BUCKET= +# AWS_ACCESS_KEY_ID= +# AWS_SECRET_ACCESS_KEY= +# S3_REGION= +# S3_PROTOCOL=http +# S3_HOSTNAME=192.168.1.123:9000 + +# S3 (Minio Config (optional) Please check Minio instance for details) +# S3_ENABLED=true +# S3_BUCKET= +# AWS_ACCESS_KEY_ID= +# AWS_SECRET_ACCESS_KEY= +# S3_REGION= +# S3_PROTOCOL=https +# S3_HOSTNAME= +# S3_ENDPOINT= +# S3_SIGNATURE_VERSION= + +# Optional alias for S3 if you want to use Cloudfront or Cloudflare in front +# S3_CLOUDFRONT_HOST= + +# Streaming API integration +# STREAMING_API_BASE_URL= + +# Advanced settings +# If you need to use pgBouncer, you need to disable prepared statements: +# PREPARED_STATEMENTS=false + +# Cluster number setting for streaming API server. +# If you comment out following line, cluster number will be `numOfCpuCores - 1`. +STREAMING_CLUSTER_NUM=1 + +# Docker mastodon user +# If you use Docker, you may want to assign UID/GID manually. +# UID=1000 +# GID=1000 diff --git a/.nanoignore b/.nanoignore new file mode 100644 index 000000000..f02c0a68a --- /dev/null +++ b/.nanoignore @@ -0,0 +1,20 @@ +.DS_Store +.git/ +.gitignore + +.bundle/ +.cache/ +config/deploy/* +coverage +docs/ +.env +log/*.log +neo4j/ +node_modules/ +public/assets/ +public/system/ +spec/ +storybook/ +tmp/ +.vagrant/ +vendor/bundle/ diff --git a/boxfile.yml b/boxfile.yml new file mode 100644 index 000000000..65f5f6c8c --- /dev/null +++ b/boxfile.yml @@ -0,0 +1,158 @@ +run.config: + engine: ruby + engine.config: + runtime: ruby-2.4.1 + + extra_packages: + # basic servers: + - nginx + - nodejs + + # for images: + - ImageMagick + + # for videos: + - ffmpeg3 + + # to prep the .env file + - gettext-tools + + cache_dirs: + - node_modules + + extra_path_dirs: + - node_modules/.bin + + build_triggers: + - .ruby-version + - Gemfile + - Gemfile.lock + - package.json + - yarn.lock + + extra_steps: + - cp .env.nanobox .env + - gem install bundler + - bundle config build.nokogiri --with-iconv-dir=/data/ --with-zlib-dir=/data/ + - bundle config build.nokogumbo --with-iconv-dir=/data/ --with-zlib-dir=/data/ + - bundle install --clean + - yarn + + fs_watch: true + +deploy.config: + extra_steps: + - NODE_ENV=production bundle exec rake assets:precompile + - "[ -r /app/.env.production ] || sed 's/LOCAL_HTTPS=.*/LOCAL_HTTPS=true/i' /app/.env.nanobox > /app/.env.production" + transform: + - envsubst < /app/.env.production > /tmp/.env.production && mv /tmp/.env.production /app/.env.production + - |- + if [ -z "$LOCAL_DOMAIN" ] + then + . /app/.env.production + export LOCAL_DOMAIN + fi + erb /app/nanobox/nginx-web.conf.erb > /app/nanobox/nginx-web.conf + erb /app/nanobox/nginx-stream.conf.erb > /app/nanobox/nginx-stream.conf + - touch /app/log/production.log + before_live: + web.web: + - bundle exec rake db:migrate:setup + +web.web: + start: + nginx: nginx -c /app/nanobox/nginx-web.conf + rails: bundle exec puma -C /app/config/puma.rb + + routes: + - '/' + + writable_dirs: + - tmp + + log_watch: + rails: 'log/production.log' + + network_dirs: + data.storage: + - public/system + +web.stream: + start: + nginx: nginx -c /app/nanobox/nginx-stream.conf + node: yarn run start + + routes: + - '/api/v1/streaming*' + # Somehow we're getting requests for scheme://domain//api/v1/streaming* - match those, too + - '//api/v1/streaming*' + + writable_dirs: + - tmp + +worker.sidekiq: + start: bundle exec sidekiq -c 5 -q default -q mailers -q pull -q push -L /app/log/sidekiq.log + + writable_dirs: + - tmp + + log_watch: + rails: 'log/production.log' + sidekiq: 'log/sidekiq.log' + + network_dirs: + data.storage: + - public/system + + cron: + - id: generate_static_gifs + schedule: '*/15 * * * *' + command: 'bundle exec rake mastodon:maintenance:add_static_avatars' + + - id: update_counter_caches + schedule: '50 * * * *' + command: 'bundle exec rake mastodon:maintenance:update_counter_caches' + + # runs feeds:clear, media:clear, users:clear, and push:refresh + - id: do_daily_tasks + schedule: '00 00 * * *' + command: 'bundle exec rake mastodon:daily' + + - id: clear_silenced_media + schedule: '10 00 * * *' + command: 'bundle exec rake mastodon:media:remove_silenced' + + - id: clear_remote_media + schedule: '20 00 * * *' + command: 'bundle exec rake mastodon:media:remove_remote' + + - id: clear_unfollowed_subs + schedule: '30 00 * * *' + command: 'bundle exec rake mastodon:push:clear' + + - id: send_digest_emails + schedule: '00 20 * * *' + command: 'bundle exec rake mastodon:emails:digest' + + # The following two tasks can be uncommented to automatically open and close + # registrations on a schedule. The format of 'schedule' is a standard cron + # time expression: minute hour day month day-of-week; search for "cron + # time expressions" for more info on how to set these up. The examples here + # open registration only from 8 am to 4 pm, server time. + # + # - id: open_registrations + # schedule: '00 08 * * *' + # command: 'bundle exec rake mastodon:settings:open_registrations' + # + # - id: close_registrations + # schedule: '00 16 * * *' + # command: 'bundle exec rake mastodon:settings:close_registrations' + +data.db: + image: nanobox/postgresql:9.5 + +data.redis: + image: nanobox/redis:3.0 + +data.storage: + image: nanobox/unfs:0.9 diff --git a/lib/tasks/db.rake b/lib/tasks/db.rake new file mode 100644 index 000000000..a1211f097 --- /dev/null +++ b/lib/tasks/db.rake @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +namespace :db do + namespace :migrate do + desc 'Setup the db or migrate depending on state of db' + task setup: :environment do + begin + ActiveRecord::Base.connection + rescue ActiveRecord::NoDatabaseError + Rake::Task['db:setup'].invoke + else + Rake::Task['db:migrate'].invoke + end + end + end +end diff --git a/nanobox/nginx-local.conf b/nanobox/nginx-local.conf new file mode 100644 index 000000000..023328733 --- /dev/null +++ b/nanobox/nginx-local.conf @@ -0,0 +1,76 @@ +worker_processes 1; +daemon off; + +events { + worker_connections 1024; +} + +http { + include /data/etc/nginx/mime.types; + sendfile on; + + gzip on; + gzip_http_version 1.0; + gzip_proxied any; + gzip_min_length 500; + gzip_disable "MSIE [1-6]\."; + gzip_types text/plain text/xml text/javascript text/css text/comma-separated-values application/xml+rss application/xml application/x-javascript application/json application/javascript application/atom+xml; + + # Proxy upstream to the puma process + upstream rails { + server 127.0.0.1:3000; + } + + # Proxy upstream to the node process + upstream node { + server 127.0.0.1:4000; + } + + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + # Configuration for Nginx + server { + # Listen on port 8080 + listen 8080; + + root /app/public; + + location / { + try_files $uri @rails; + } + + # Proxy connections to rails + location @rails { + proxy_set_header Host $host; + proxy_pass_header Server; + + proxy_pass http://rails; + proxy_buffering off; + proxy_redirect off; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + + tcp_nodelay on; + } + + # Proxy connections to node + location /api/v1/streaming { + proxy_set_header Host $host; + + proxy_pass http://node; + proxy_buffering off; + proxy_redirect off; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + + tcp_nodelay on; + } + } + + error_page 500 501 502 503 504 /500.html; +} diff --git a/nanobox/nginx-stream.conf.erb b/nanobox/nginx-stream.conf.erb new file mode 100644 index 000000000..b39d3ff1d --- /dev/null +++ b/nanobox/nginx-stream.conf.erb @@ -0,0 +1,57 @@ +worker_processes 1; +daemon off; + +events { + worker_connections 1024; +} + +http { + include /data/etc/nginx/mime.types; + sendfile on; + + gzip on; + gzip_http_version 1.1; + gzip_proxied any; + gzip_min_length 500; + gzip_disable "MSIE [1-6]\."; + gzip_types text/plain text/xml text/javascript text/css text/comma-separated-values application/xml+rss application/xml application/x-javascript application/json application/javascript application/atom+xml; + + # Proxy upstream to the node process + upstream node { + server 127.0.0.1:4000; + } + + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + # Configuration for Nginx + server { + # Listen on port 8080 + listen 8080; + + add_header Strict-Transport-Security "max-age=31536000"; + add_header Content-Security-Policy "style-src 'self' 'unsafe-inline'; script-src 'self'; object-src 'self'; img-src data: https:; media-src data: https:; connect-src 'self' wss://<%= ENV["LOCAL_DOMAIN"] %>; upgrade-insecure-requests"; + + root /app/public; + + location / { + try_files $uri @node; + } + + # Proxy connections to node + location @node { + proxy_set_header Host $host; + + proxy_pass http://node; + proxy_buffering off; + proxy_redirect off; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + + tcp_nodelay on; + } + } +} diff --git a/nanobox/nginx-web.conf.erb b/nanobox/nginx-web.conf.erb new file mode 100644 index 000000000..55245bf28 --- /dev/null +++ b/nanobox/nginx-web.conf.erb @@ -0,0 +1,65 @@ +worker_processes 1; +daemon off; + +events { + worker_connections 1024; +} + +http { + include /data/etc/nginx/mime.types; + sendfile on; + + gzip on; + gzip_http_version 1.0; + gzip_proxied any; + gzip_min_length 500; + gzip_disable "MSIE [1-6]\."; + gzip_types text/plain text/xml text/javascript text/css text/comma-separated-values application/xml+rss application/xml application/x-javascript application/json application/javascript application/atom+xml; + + # Proxy upstream to the puma process + upstream rails { + server 127.0.0.1:3000; + } + + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + # Configuration for Nginx + server { + # Listen on port 8080 + listen 8080; + + add_header Strict-Transport-Security "max-age=31536000"; + add_header Content-Security-Policy "style-src 'self' 'unsafe-inline'; script-src 'self'; object-src 'self'; img-src data: https:; media-src data: https:; connect-src 'self' wss://<%= ENV["LOCAL_DOMAIN"] %>; upgrade-insecure-requests"; + + root /app/public; + + location / { + try_files $uri @rails; + } + + location ~ ^/(assets|system/media_attachments/files|system/accounts/avatars) { + add_header Cache-Control "public, max-age=31536000, immutable"; + try_files $uri @rails; + } + + # Proxy connections to rails + location @rails { + proxy_set_header Host $host; + proxy_pass_header Server; + + proxy_pass http://rails; + proxy_buffering off; + proxy_redirect off; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + + tcp_nodelay on; + } + } + + error_page 500 501 502 503 504 /500.html; +}