Merge remote-tracking branch 'origin/develop' into fork

Signed-off-by: marcin mikołajczak <[email protected]>

.dialyzer_ignore.exs

@@ -2,5 +2,8 @@
 {"lib/cachex.ex", "Unknown type: Spec.cache/0."},
 {"lib/pleroma/web/plugs/rate_limiter.ex", "The pattern can never match the type {:commit, _} | {:ignore, _}."},
 {"lib/pleroma/web/plugs/rate_limiter.ex", "Function get_scale/2 will never be called."},
-{"lib/pleroma/web/plugs/rate_limiter.ex", "Function initialize_buckets!/1 will never be called."}
+{"lib/pleroma/web/plugs/rate_limiter.ex", "Function initialize_buckets!/1 will never be called."},
+{"lib/pleroma/workers/receiver_worker.ex", :call},
+{"lib/pleroma/workers/receiver_worker.ex", :pattern_match},
+{"lib/pleroma/workers/receiver_worker.ex", :pattern_match_cov},
 ]

changelog.d/backups-refactor.change

@@ -0,0 +1 @@
+Refactor the user backups code and improve test coverage

changelog.d/disclose_client.fix

@@ -0,0 +1 @@
+Client application data was always missing from the status

changelog.d/federator_outgoing_increase.change

@@ -0,0 +1 @@
+Increase outgoing federation parallelism

changelog.d/hackney-pool-timeouts.change

@@ -0,0 +1 @@
+Change Hackney connection pool timeouts to align with the values Gun uses

changelog.d/optimistic-inbox-sigs.fix

@@ -0,0 +1 @@
+Fix Optimistic Inbox for failed signatures

changelog.d/publisher.change

@@ -0,0 +1 @@
+Publisher jobs now store the the activity id instead of inserting duplicate JSON data in the Oban queue for each delivery.

config/config.exs

@@ -598,10 +598,8 @@ config :pleroma, Oban,
   queues: [
     activity_expiration: 10,
     federator_incoming: 5,
-    federator_outgoing: 5,
+    federator_outgoing: 25,
     web_push: 50,
-    transmogrifier: 20,
-    notifications: 20,
     background: 20,
     search_indexing: [limit: 10, paused: true],
     slow: 5
@@ -882,19 +880,19 @@ config :pleroma, :pools,
 config :pleroma, :hackney_pools,
   federation: [
     max_connections: 50,
-    timeout: 150_000
+    timeout: 10_000
   ],
   media: [
     max_connections: 50,
-    timeout: 150_000
+    timeout: 15_000
   ],
   rich_media: [
     max_connections: 50,
-    timeout: 150_000
+    timeout: 15_000
   ],
   upload: [
     max_connections: 25,
-    timeout: 300_000
+    timeout: 15_000
   ]
 
 config :pleroma, :majic_pool, size: 2
@@ -933,8 +931,8 @@ config :pleroma, Pleroma.User.Backup,
   purge_after_days: 30,
   limit_days: 7,
   dir: nil,
-  process_wait_time: 30_000,
-  process_chunk_size: 100
+  process_chunk_size: 100,
+  timeout: :timer.minutes(30)
 
 config :pleroma, ConcurrentLimiter, [
   {Pleroma.Search, [max_running: 30, max_waiting: 50]},

config/description.exs

@@ -3386,20 +3386,19 @@ config :pleroma, :config_description, [
         description: "Limit user to export not more often than once per N days",
         suggestions: [7]
       },
-      %{
-        key: :process_wait_time,
-        type: :integer,
-        label: "Process Wait Time",
-        description:
-          "The amount of time to wait for backup to report progress, in milliseconds. If no progress is received from the backup job for that much time, terminate it and deem it failed.",
-        suggestions: [30_000]
-      },
       %{
         key: :process_chunk_size,
         type: :integer,
         label: "Process Chunk Size",
         description: "The number of activities to fetch in the backup job for each chunk.",
         suggestions: [100]
+      },
+      %{
+        key: :timeout,
+        type: :integer,
+        label: "Timeout",
+        description: "The amount of time to wait for backup to complete in seconds.",
+        suggestions: [1_800]
       }
     ]
   },

config/test.exs

@@ -170,8 +170,7 @@ config :pleroma, Pleroma.Uploaders.IPFS, config_impl: Pleroma.UnstubbedConfigMoc
 config :pleroma, Pleroma.Web.Plugs.HTTPSecurityPlug, config_impl: Pleroma.StaticStubbedConfigMock
 config :pleroma, Pleroma.Web.Plugs.HTTPSignaturePlug, config_impl: Pleroma.StaticStubbedConfigMock
 
-config :pleroma, Pleroma.Web.Plugs.HTTPSignaturePlug,
-  http_signatures_impl: Pleroma.StubbedHTTPSignaturesMock
+config :pleroma, Pleroma.Signature, http_signatures_impl: Pleroma.StubbedHTTPSignaturesMock
 
 peer_module =
   if String.to_integer(System.otp_release()) >= 25 do
@@ -201,6 +200,8 @@ config :pleroma, Pleroma.Web.RichMedia.Backfill,
 
 config :pleroma, Pleroma.Web.Plugs.HTTPSecurityPlug, enable: false
 
+config :pleroma, Pleroma.User.Backup, tempdir: "test/tmp"
+
 if File.exists?("./config/test.secret.exs") do
   import_config "test.secret.exs"
 else

docs/configuration/cheatsheet.md

@@ -1171,6 +1171,7 @@ Control favicons for instances.
     3. the directory named by the TMP environment variable
     4. C:\TMP on Windows or /tmp on Unix-like operating systems
     5. as a last resort, the current working directory
+* `:timeout` an integer representing seconds
 
 ## Frontend management
 

lib/pleroma/ecto_enums.ex

@@ -27,11 +27,3 @@ defenum(Pleroma.DataMigration.State,
   failed: 4,
   manual: 5
 )
-
-defenum(Pleroma.User.Backup.State,
-  pending: 1,
-  running: 2,
-  complete: 3,
-  failed: 4,
-  invalid: 5
-)

lib/pleroma/emails/user_email.ex

@@ -345,37 +345,22 @@ defmodule Pleroma.Emails.UserEmail do
     Router.Helpers.subscription_url(Endpoint, :unsubscribe, token)
   end
 
-  def backup_is_ready_email(backup, admin_user_id \\ nil) do
+  def backup_is_ready_email(backup) do
     %{user: user} = Pleroma.Repo.preload(backup, :user)
 
     Gettext.with_locale_or_default user.language do
       download_url = Pleroma.Web.PleromaAPI.BackupView.download_url(backup)
 
       html_body =
-        if is_nil(admin_user_id) do
-          Gettext.dpgettext(
-            "static_pages",
-            "account archive email body - self-requested",
-            """
-            <p>You requested a full backup of your Pleroma account. It's ready for download:</p>
-            <p><a href="%{download_url}">%{download_url}</a></p>
-            """,
-            download_url: download_url
-          )
-        else
-          admin = Pleroma.Repo.get(User, admin_user_id)
-
-          Gettext.dpgettext(
-            "static_pages",
-            "account archive email body - admin requested",
-            """
-            <p>Admin @%{admin_nickname} requested a full backup of your Pleroma account. It's ready for download:</p>
-            <p><a href="%{download_url}">%{download_url}</a></p>
-            """,
-            admin_nickname: admin.nickname,
-            download_url: download_url
-          )
-        end
+        Gettext.dpgettext(
+          "static_pages",
+          "account archive email body",
+          """
+          <p>A full backup of your Pleroma account was requested. It's ready for download:</p>
+          <p><a href="%{download_url}">%{download_url}</a></p>
+          """,
+          download_url: download_url
+        )
 
       new()
       |> to(recipient(user))

lib/pleroma/signature.ex

@@ -10,6 +10,14 @@ defmodule Pleroma.Signature do
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
 
+  import Plug.Conn, only: [put_req_header: 3]
+
+  @http_signatures_impl Application.compile_env(
+                          :pleroma,
+                          [__MODULE__, :http_signatures_impl],
+                          HTTPSignatures
+                        )
+
   @known_suffixes ["/publickey", "/main-key"]
 
   def key_id_to_actor_id(key_id) do
@@ -85,4 +93,48 @@ defmodule Pleroma.Signature do
   def signed_date(%NaiveDateTime{} = date) do
     Timex.format!(date, "{WDshort}, {0D} {Mshort} {YYYY} {h24}:{m}:{s} GMT")
   end
+
+  @spec validate_signature(Plug.Conn.t(), String.t()) :: boolean()
+  def validate_signature(%Plug.Conn{} = conn, request_target) do
+    # Newer drafts for HTTP signatures now use @request-target instead of the
+    # old (request-target). We'll now support both for incoming signatures.
+    conn =
+      conn
+      |> put_req_header("(request-target)", request_target)
+      |> put_req_header("@request-target", request_target)
+
+    @http_signatures_impl.validate_conn(conn)
+  end
+
+  @spec validate_signature(Plug.Conn.t()) :: boolean()
+  def validate_signature(%Plug.Conn{} = conn) do
+    # This (request-target) is non-standard, but many implementations do it
+    # this way due to a misinterpretation of
+    # https://datatracker.ietf.org/doc/html/draft-cavage-http-signatures-06
+    # "path" was interpreted as not having the query, though later examples
+    # show that it must be the absolute path + query. This behavior is kept to
+    # make sure most software (Pleroma itself, Mastodon, and probably others)
+    # do not break.
+    request_target = Enum.join([String.downcase(conn.method), conn.request_path], " ")
+
+    # This is the proper way to build the @request-target, as expected by
+    # many HTTP signature libraries, clarified in the following draft:
+    # https://www.ietf.org/archive/id/draft-ietf-httpbis-message-signatures-11.html#section-2.2.6
+    # It is the same as before, but containing the query part as well.
+    proper_target = Enum.join([request_target, "?", conn.query_string], "")
+
+    cond do
+      # Normal, non-standard behavior but expected by Pleroma and more.
+      validate_signature(conn, request_target) ->
+        true
+
+      # Has query string and the previous one failed: let's try the standard.
+      conn.query_string != "" ->
+        validate_signature(conn, proper_target)
+
+      # If there's no query string and signature fails, it's rotten.
+      true ->
+        false
+    end
+  end
 end