Merge tag 'v2.6.5' into friends.nico-update-v2.6.2

CHANGELOG.md

@@ -1,4 +1,187 @@
-## 2.5.2
+Changelog
+=========
+
+All notable changes to this project will be documented in this file.
+
+## [2.6.5] - 2018-12-01
+### Changed
+
+- Change lists to display replies to others on the list and list owner (#9324)
+
+### Fixed
+
+- Fix failures caused by commonly-used JSON-LD contexts being unavailable (#9412)
+
+## [2.6.4] - 2018-11-30
+### Fixed
+
+- Fix yarn dependencies not installing due to yanked event-stream package (#9401)
+
+## [2.6.3] - 2018-11-30
+### Added
+
+- Add hyphen to characters allowed in remote usernames (#9345)
+
+### Changed
+
+- Change server user count to exclude suspended accounts (#9380)
+
+### Fixed
+
+- Fix ffmpeg processing sometimes stalling due to overfilled stdout buffer (#9368)
+- Fix missing DNS records raising the wrong kind of exception (#9379)
+- Fix already queued deliveries still trying to reach inboxes marked as unavailable (#9358)
+
+### Security
+
+- Fix TLS handshake timeout not being enforced (#9381)
+
+## [2.6.2] - 2018-11-23
+### Added
+
+- Add Page to whitelisted ActivityPub types (#9188)
+- Add 20px to column width in web UI (#9227)
+- Add amount of freed disk space in `tootctl media remove` (#9229, #9239, #9288)
+- Add "Show thread" link to self-replies (#9228)
+
+### Changed
+
+- Change order of Atom and RSS links so Atom is first (#9302)
+- Change Nginx configuration for Nanobox apps (#9310)
+- Change the follow action to appear instant in web UI (#9220)
+- Change how the ActiveRecord connection is instantiated in on_worker_boot (#9238)
+- Change `tootctl accounts cull` to always touch accounts so they can be skipped (#9293)
+- Change mime type comparison to ignore JSON-LD profile (#9179)
+
+### Fixed
+
+- Fix web UI crash when conversation has no last status (#9207)
+- Fix follow limit validator reporting lower number past threshold (#9230)
+- Fix form validation flash message color and input borders (#9235)
+- Fix invalid twitter:player cards being displayed (#9254)
+- Fix emoji update date being processed incorrectly (#9255)
+- Fix playing embed resetting if status is reloaded in web UI (#9270, #9275)
+- Fix web UI crash when favouriting a deleted status (#9272)
+- Fix intermediary arrays being created for hash maps (#9291)
+- Fix filter ID not being a string in REST API (#9303)
+
+### Security
+
+- Fix multiple remote account deletions being able to deadlock the database (#9292)
+- Fix HTTP connection timeout of 10s not being enforced (#9329)
+
+## [2.6.1] - 2018-10-30
+### Fixed
+
+- Fix resolving resources by URL not working due to a regression in #9132 (#9171)
+- Fix reducer error in web UI when a conversation has no last status (#9173)
+
+## [2.6.0] - 2018-10-30
+### Added
+
+- Add link ownership verification (#8703)
+- Add conversations API (#8832)
+- Add limit for the number of people that can be followed from one account (#8807)
+- Add admin setting to customize mascot (#8766)
+- Add support for more granular ActivityPub audiences from other software, i.e. circles (#8950, #9093, #9150)
+- Add option to block all reports from a domain (#8830)
+- Add user preference to always expand toots marked with content warnings (#8762)
+- Add user preference to always hide all media (#8569)
+- Add `force_login` param to OAuth authorize page (#8655)
+- Add `tootctl accounts backup` (#8642, #8811)
+- Add `tootctl accounts create` (#8642, #8811)
+- Add `tootctl accounts cull` (#8642, #8811)
+- Add `tootctl accounts delete` (#8642, #8811)
+- Add `tootctl accounts modify` (#8642, #8811)
+- Add `tootctl accounts refresh` (#8642, #8811)
+- Add `tootctl feeds build` (#8642, #8811)
+- Add `tootctl feeds clear` (#8642, #8811)
+- Add `tootctl settings registrations open` (#8642, #8811)
+- Add `tootctl settings registrations close` (#8642, #8811)
+- Add `min_id` param to REST API to support backwards pagination (#8736)
+- Add a confirmation dialog when hitting reply and the compose box isn't empty (#8893)
+- Add PostgreSQL disk space growth tracking in PGHero (#8906)
+- Add button for disabling local account to report quick actions bar (#9024)
+- Add Czech language (#8594)
+- Add `same-site` (`lax`) attribute to cookies (#8626)
+- Add support for styled scrollbars in Firefox Nightly (#8653)
+- Add highlight to the active tab in web UI profiles (#8673)
+- Add auto-focus for comment textarea in report modal (#8689)
+- Add auto-focus for emoji picker's search field (#8688)
+- Add nginx and systemd templates to `dist/` directory (#8770)
+- Add support for `/.well-known/change-password` (#8828)
+- Add option to override FFMPEG binary path (#8855)
+- Add `dns-prefetch` tag when using different host for assets or uploads (#8942)
+- Add `description` meta tag (#8941)
+- Add `Content-Security-Policy` header (#8957)
+- Add cache for the instance info API (#8765)
+- Add suggested follows to search screen in mobile layout (#9010)
+- Add CORS header to `/.well-known/*` routes (#9083)
+- Add `card` attribute to statuses returned from REST API (#9120)
+- Add in-stream link preview (#9120)
+- Add support for ActivityPub `Page` objects (#9121)
+
+### Changed
+
+- Change forms design (#8703)
+- Change reports overview to group by target account (#8674)
+- Change web UI to show "read more" link on overly long in-stream statuses (#8205)
+- Change design of direct messages column (#8832, #9022)
+- Change home timelines to exclude DMs (#8940)
+- Change list timelines to exclude all replies (#8683)
+- Change admin accounts UI default sort to most recent (#8813)
+- Change documentation URL in the UI (#8898)
+- Change style of success and failure messages (#8973)
+- Change DM filtering to always allow DMs from staff (#8993)
+- Change recommended Ruby version to 2.5.3 (#9003)
+- Change docker-compose default to persist volumes in current directory (#9055)
+- Change character counters on edit profile page to input length limit (#9100)
+- Change notification filtering to always let through messages from staff (#9152)
+- Change "hide boosts from user" function also hiding notifications about boosts (#9147)
+- Change CSS `detailed-status__wrapper` class actually wrap the detailed status (#8547)
+
+### Deprecated
+
+- `GET /api/v1/timelines/direct` → `GET /api/v1/conversations` (#8832)
+- `POST /api/v1/notifications/dismiss` → `POST /api/v1/notifications/:id/dismiss` (#8905)
+- `GET /api/v1/statuses/:id/card` → `card` attributed included in status (#9120)
+
+### Removed
+
+- Remove "on this device" label in column push settings (#8704)
+- Remove rake tasks in favour of tootctl commands (#8675)
+
+### Fixed
+
+- Fix remote statuses using instance's default locale if no language given (#8861)
+- Fix streaming API not exiting when port or socket is unavailable (#9023)
+- Fix network calls being performed in database transaction in ActivityPub handler (#8951)
+- Fix dropdown arrow position (#8637)
+- Fix first element of dropdowns being focused even if not using keyboard (#8679)
+- Fix tootctl requiring `bundle exec` invocation (#8619)
+- Fix public pages not using animation preference for avatars (#8614)
+- Fix OEmbed/OpenGraph cards not understanding relative URLs (#8669)
+- Fix some dark emojis not having a white outline (#8597)
+- Fix media description not being displayed in various media modals (#8678)
+- Fix generated URLs of desktop notifications missing base URL (#8758)
+- Fix RTL styles (#8764, #8767, #8823, #8897, #9005, #9007, #9018, #9021, #9145, #9146)
+- Fix crash in streaming API when tag param missing (#8955)
+- Fix hotkeys not working when no element is focused (#8998)
+- Fix some hotkeys not working on detailed status view (#9006)
+- Fix og:url on status pages (#9047)
+- Fix upload option buttons only being visible on hover (#9074)
+- Fix tootctl not returning exit code 1 on wrong arguments (#9094)
+- Fix preview cards for appearing for profiles mentioned in toot (#6934, #9158)
+- Fix local accounts sometimes being duplicated as faux-remote (#9109)
+- Fix emoji search when the shortcode has multiple separators (#9124)
+- Fix dropdowns sometimes being partially obscured by other elements (#9126)
+- Fix cache not updating when reply/boost/favourite counters or media sensitivity update (#9119)
+- Fix empty display name precedence over username in web UI (#9163)
+- Fix td instead of th in sessions table header (#9162)
+- Fix handling of content types with profile (#9132)
+
+## [2.5.2] - 2018-10-12
+### Security
 
 - Fix XSS vulnerability (#8959)
 

Gemfile

@@ -91,6 +91,7 @@ gem 'webpush'
 gem 'airbrake', '~> 5.0'
 
 gem 'json-ld', '~> 2.2'
+gem 'json-ld-preloaded', '~> 2.2'
 gem 'rdf-normalize', '~> 0.3'
 
 group :development, :test do

Gemfile.lock

@@ -294,6 +294,10 @@ GEM
     json-ld (2.2.1)
       multi_json (~> 1.12)
       rdf (>= 2.2.8, < 4.0)
+    json-ld-preloaded (2.2.3)
+      json-ld (>= 2.2, < 4.0)
+      multi_json (~> 1.12)
+      rdf (>= 2.2, < 4.0)
     jsonapi-renderer (0.2.0)
     jwt (2.1.0)
     kaminari (1.1.1)
@@ -712,6 +716,7 @@ DEPENDENCIES
   idn-ruby
   iso-639
   json-ld (~> 2.2)
+  json-ld-preloaded (~> 2.2)
   kaminari (~> 1.1)
   letter_opener (~> 1.4)
   letter_opener_web (~> 1.3)

app/lib/feed_manager.rb

@@ -40,7 +40,11 @@ def unpush_from_home(account, status)
   end
 
   def push_to_list(list, status)
-    return false if status.reply? && status.in_reply_to_account_id != status.account_id
+    if status.reply? && status.in_reply_to_account_id != status.account_id
+      should_filter = status.in_reply_to_account_id != list.account_id
+      should_filter &&= !ListAccount.where(list_id: list.id, account_id: status.in_reply_to_account_id).exists?
+      return false if should_filter
+    end
     return false unless add_to_feed(:list, list.id, status)
     trim(:list, list.id)
     PushUpdateWorker.perform_async(list.account_id, status.id, "timeline:list:#{list.id}") if push_update_required?("timeline:list:#{list.id}")

app/lib/request.rb

@@ -4,6 +4,16 @@
 require 'socket'
 require 'resolv'
 
+# Monkey-patch the HTTP.rb timeout class to avoid using a timeout block
+# around the Socket#open method, since we use our own timeout blocks inside
+# that method
+class HTTP::Timeout::PerOperation
+  def connect(socket_class, host, port, nodelay = false)
+    @socket = socket_class.open(host, port)
+    @socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) if nodelay
+  end
+end
+
 class Request
   REQUEST_TARGET = '(request-target)'
 
@@ -95,7 +105,11 @@ def key_id
   end
 
   def timeout
-    { connect: nil, read: 10, write: 10 }
+    # We enforce a 1s timeout on DNS resolving, 10s timeout on socket opening
+    # and 5s timeout on the TLS handshake, meaning the worst case should take
+    # about 16s in total
+
+    { connect: 5, read: 10, write: 10 }
   end
 
   def http_client
@@ -163,7 +177,11 @@ def open(host, *args)
           end
         end
 
-        raise outer_e if outer_e
+        if outer_e
+          raise outer_e
+        else
+          raise SocketError, "No address for #{host}"
+        end
       end
 
       alias new open

app/models/account.rb

@@ -49,7 +49,7 @@
 #
 
 class Account < ApplicationRecord
-  USERNAME_RE = /[a-z0-9_]+([a-z0-9_\.]+[a-z0-9_]+)?/i
+  USERNAME_RE = /[a-z0-9_]+([a-z0-9_\.-]+[a-z0-9_]+)?/i
   MENTION_RE  = /(?<=^|[^\/[:word:]:])@((#{USERNAME_RE})(?:@[a-z0-9\.\-]+[a-z0-9]+)?)/i
 
   include AccountAvatar

app/models/media_attachment.rb

@@ -59,6 +59,7 @@ class MediaAttachment < ApplicationRecord
     format: 'mp4',
     convert_options: {
       output: {
+        'loglevel' => 'fatal',
         'movflags' => 'faststart',
         'pix_fmt'  => 'yuv420p',
         'vf'       => 'scale=\'trunc(iw/2)*2:trunc(ih/2)*2\'',

app/presenters/instance_presenter.rb

@@ -18,7 +18,7 @@ def contact_account
   end
 
   def user_count
-    Rails.cache.fetch('user_count') { User.confirmed.count }
+    Rails.cache.fetch('user_count') { User.confirmed.joins(:account).merge(Account.without_suspended).count }
   end
 
   def status_count

app/workers/activitypub/delivery_worker.rb

@@ -11,6 +11,8 @@ class ActivityPub::DeliveryWorker
   HEADERS = { 'Content-Type' => 'application/activity+json' }.freeze
 
   def perform(json, source_account_id, inbox_url, options = {})
+    return if DeliveryFailureTracker.unavailable?(inbox_url)
+
     @options        = options.with_indifferent_access
     @json           = json
     @source_account = Account.find(source_account_id)

config/initializers/json_ld.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require_relative '../../lib/json_ld/security'

lib/json_ld/security.rb

@@ -0,0 +1,50 @@
+# -*- encoding: utf-8 -*-
+# frozen_string_literal: true
+# This file generated automatically from https://w3id.org/security/v1
+require 'json/ld'
+class JSON::LD::Context
+  add_preloaded("https://w3id.org/security/v1") do
+    new(processingMode: "json-ld-1.0", term_definitions: {
+      "CryptographicKey" => TermDefinition.new("CryptographicKey", id: "https://w3id.org/security#Key", simple: true),
+      "EcdsaKoblitzSignature2016" => TermDefinition.new("EcdsaKoblitzSignature2016", id: "https://w3id.org/security#EcdsaKoblitzSignature2016", simple: true),
+      "EncryptedMessage" => TermDefinition.new("EncryptedMessage", id: "https://w3id.org/security#EncryptedMessage", simple: true),
+      "GraphSignature2012" => TermDefinition.new("GraphSignature2012", id: "https://w3id.org/security#GraphSignature2012", simple: true),
+      "LinkedDataSignature2015" => TermDefinition.new("LinkedDataSignature2015", id: "https://w3id.org/security#LinkedDataSignature2015", simple: true),
+      "LinkedDataSignature2016" => TermDefinition.new("LinkedDataSignature2016", id: "https://w3id.org/security#LinkedDataSignature2016", simple: true),
+      "authenticationTag" => TermDefinition.new("authenticationTag", id: "https://w3id.org/security#authenticationTag", simple: true),
+      "canonicalizationAlgorithm" => TermDefinition.new("canonicalizationAlgorithm", id: "https://w3id.org/security#canonicalizationAlgorithm", simple: true),
+      "cipherAlgorithm" => TermDefinition.new("cipherAlgorithm", id: "https://w3id.org/security#cipherAlgorithm", simple: true),
+      "cipherData" => TermDefinition.new("cipherData", id: "https://w3id.org/security#cipherData", simple: true),
+      "cipherKey" => TermDefinition.new("cipherKey", id: "https://w3id.org/security#cipherKey", simple: true),
+      "created" => TermDefinition.new("created", id: "http://purl.org/dc/terms/created", type_mapping: "http://www.w3.org/2001/XMLSchema#dateTime"),
+      "creator" => TermDefinition.new("creator", id: "http://purl.org/dc/terms/creator", type_mapping: "@id"),
+      "dc" => TermDefinition.new("dc", id: "http://purl.org/dc/terms/", simple: true, prefix: true),
+      "digestAlgorithm" => TermDefinition.new("digestAlgorithm", id: "https://w3id.org/security#digestAlgorithm", simple: true),
+      "digestValue" => TermDefinition.new("digestValue", id: "https://w3id.org/security#digestValue", simple: true),
+      "domain" => TermDefinition.new("domain", id: "https://w3id.org/security#domain", simple: true),
+      "encryptionKey" => TermDefinition.new("encryptionKey", id: "https://w3id.org/security#encryptionKey", simple: true),
+      "expiration" => TermDefinition.new("expiration", id: "https://w3id.org/security#expiration", type_mapping: "http://www.w3.org/2001/XMLSchema#dateTime"),
+      "expires" => TermDefinition.new("expires", id: "https://w3id.org/security#expiration", type_mapping: "http://www.w3.org/2001/XMLSchema#dateTime"),
+      "id" => TermDefinition.new("id", id: "@id", simple: true),
+      "initializationVector" => TermDefinition.new("initializationVector", id: "https://w3id.org/security#initializationVector", simple: true),
+      "iterationCount" => TermDefinition.new("iterationCount", id: "https://w3id.org/security#iterationCount", simple: true),
+      "nonce" => TermDefinition.new("nonce", id: "https://w3id.org/security#nonce", simple: true),
+      "normalizationAlgorithm" => TermDefinition.new("normalizationAlgorithm", id: "https://w3id.org/security#normalizationAlgorithm", simple: true),
+      "owner" => TermDefinition.new("owner", id: "https://w3id.org/security#owner", type_mapping: "@id"),
+      "password" => TermDefinition.new("password", id: "https://w3id.org/security#password", simple: true),
+      "privateKey" => TermDefinition.new("privateKey", id: "https://w3id.org/security#privateKey", type_mapping: "@id"),
+      "privateKeyPem" => TermDefinition.new("privateKeyPem", id: "https://w3id.org/security#privateKeyPem", simple: true),
+      "publicKey" => TermDefinition.new("publicKey", id: "https://w3id.org/security#publicKey", type_mapping: "@id"),
+      "publicKeyPem" => TermDefinition.new("publicKeyPem", id: "https://w3id.org/security#publicKeyPem", simple: true),
+      "publicKeyService" => TermDefinition.new("publicKeyService", id: "https://w3id.org/security#publicKeyService", type_mapping: "@id"),
+      "revoked" => TermDefinition.new("revoked", id: "https://w3id.org/security#revoked", type_mapping: "http://www.w3.org/2001/XMLSchema#dateTime"),
+      "salt" => TermDefinition.new("salt", id: "https://w3id.org/security#salt", simple: true),
+      "sec" => TermDefinition.new("sec", id: "https://w3id.org/security#", simple: true, prefix: true),
+      "signature" => TermDefinition.new("signature", id: "https://w3id.org/security#signature", simple: true),
+      "signatureAlgorithm" => TermDefinition.new("signatureAlgorithm", id: "https://w3id.org/security#signingAlgorithm", simple: true),
+      "signatureValue" => TermDefinition.new("signatureValue", id: "https://w3id.org/security#signatureValue", simple: true),
+      "type" => TermDefinition.new("type", id: "@type", simple: true),
+      "xsd" => TermDefinition.new("xsd", id: "http://www.w3.org/2001/XMLSchema#", simple: true, prefix: true)
+    })
+  end
+end

lib/mastodon/version.rb

@@ -13,7 +13,7 @@ def minor
     end
 
     def patch
-      2
+      5
     end
 
     def pre

package.json

@@ -10,7 +10,7 @@
     "build:production": "cross-env RAILS_ENV=production NODE_ENV=production ./bin/webpack",
     "manage:translations": "node ./config/webpack/translationRunner.js",
     "start": "node ./streaming/index.js",
-    "test": "npm-run-all test:lint test:jest",
+    "test": "npm run test:lint && npm run test:jest",
     "test:lint": "eslint -c .eslintrc.yml --ext=js app/javascript/ config/webpack/ streaming/",
     "test:jest": "cross-env NODE_ENV=test jest --coverage"
   },
@@ -76,7 +76,6 @@
     "mini-css-extract-plugin": "^0.4.2",
     "mkdirp": "^0.5.1",
     "node-sass": "^4.9.2",
-    "npm-run-all": "^4.1.2",
     "npmlog": "^4.1.2",
     "object-assign": "^4.1.1",
     "object-fit-images": "^3.2.3",