From 52e9bec15655dd3ba75c133b85266a1ea65a8eef Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 28 Jun 2024 11:47:31 -0400 Subject: [PATCH 01/34] Remove WorkerHelper --- changelog.d/workerhelper.skip | 0 config/config.exs | 7 --- lib/mix/tasks/pleroma/database.ex | 10 ++-- lib/pleroma/emails/mailer.ex | 3 +- lib/pleroma/filter.ex | 11 +++-- lib/pleroma/instances/instance.ex | 3 +- lib/pleroma/mfa/token.ex | 13 +++-- lib/pleroma/object.ex | 3 +- lib/pleroma/search.ex | 6 ++- lib/pleroma/user.ex | 19 +++++--- lib/pleroma/user/import.ex | 30 +++++++----- lib/pleroma/web/activity_pub/activity_pub.ex | 18 ++++--- lib/pleroma/web/activity_pub/publisher.ex | 6 +-- lib/pleroma/web/activity_pub/side_effects.ex | 14 ++++-- lib/pleroma/web/common_api.ex | 6 +-- lib/pleroma/web/federator.ex | 27 +++++++---- lib/pleroma/web/o_auth/token.ex | 9 ++-- lib/pleroma/web/push.ex | 2 +- .../workers/attachments_cleanup_worker.ex | 6 +-- lib/pleroma/workers/background_worker.ex | 8 ++-- lib/pleroma/workers/backup_worker.ex | 4 +- .../workers/cron/digest_emails_worker.ex | 4 +- .../workers/cron/new_users_digest_worker.ex | 6 +-- lib/pleroma/workers/delete_worker.ex | 7 ++- lib/pleroma/workers/mailer_worker.ex | 6 +-- lib/pleroma/workers/mute_expire_worker.ex | 6 +-- lib/pleroma/workers/poll_worker.ex | 6 +-- lib/pleroma/workers/publisher_worker.ex | 22 ++++++--- lib/pleroma/workers/purge_expired_activity.ex | 11 ++--- lib/pleroma/workers/purge_expired_filter.ex | 2 +- lib/pleroma/workers/purge_expired_token.ex | 12 +---- lib/pleroma/workers/receiver_worker.ex | 6 +-- lib/pleroma/workers/remote_fetcher_worker.ex | 6 +-- lib/pleroma/workers/rich_media_worker.ex | 4 +- .../workers/scheduled_activity_worker.ex | 6 +-- lib/pleroma/workers/search_indexing_worker.ex | 6 +-- lib/pleroma/workers/user_refresh_worker.ex | 2 +- lib/pleroma/workers/web_pusher_worker.ex | 6 +-- lib/pleroma/workers/worker_helper.ex | 48 ------------------- test/mix/tasks/pleroma/database_test.exs | 10 ++-- .../workers/purge_expired_activity_test.exs | 30 +++++++----- 41 files changed, 200 insertions(+), 211 deletions(-) create mode 100644 changelog.d/workerhelper.skip delete mode 100644 lib/pleroma/workers/worker_helper.ex diff --git a/changelog.d/workerhelper.skip b/changelog.d/workerhelper.skip new file mode 100644 index 0000000000..e69de29bb2 diff --git a/config/config.exs b/config/config.exs index 4780892f7a..b835a7c807 100644 --- a/config/config.exs +++ b/config/config.exs @@ -600,13 +600,6 @@ config :pleroma, Oban, {"0 0 * * *", Pleroma.Workers.Cron.NewUsersDigestWorker} ] -config :pleroma, :workers, - retries: [ - federator_incoming: 5, - federator_outgoing: 5, - search_indexing: 2 - ] - config :pleroma, Pleroma.Formatter, class: false, rel: "ugc", diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index b82d1f079d..e52b5e0a72 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -295,10 +295,12 @@ defmodule Mix.Tasks.Pleroma.Database do |> DateTime.from_naive!("Etc/UTC") |> Timex.shift(days: days) - Pleroma.Workers.PurgeExpiredActivity.enqueue(%{ - activity_id: activity.id, - expires_at: expires_at - }) + Pleroma.Workers.PurgeExpiredActivity.enqueue( + %{ + activity_id: activity.id + }, + scheduled_at: expires_at + ) end) end) |> Stream.run() diff --git a/lib/pleroma/emails/mailer.ex b/lib/pleroma/emails/mailer.ex index 1014421308..2a80f8547c 100644 --- a/lib/pleroma/emails/mailer.ex +++ b/lib/pleroma/emails/mailer.ex @@ -25,7 +25,8 @@ defmodule Pleroma.Emails.Mailer do |> :erlang.term_to_binary() |> Base.encode64() - MailerWorker.enqueue("email", %{"encoded_email" => encoded_email, "config" => config}) + MailerWorker.new(%{"op" => "email", "encoded_email" => encoded_email, "config" => config}) + |> Oban.insert() end @doc "callback to perform send email from queue" diff --git a/lib/pleroma/filter.ex b/lib/pleroma/filter.ex index e827d3cbcb..77ed64d4fc 100644 --- a/lib/pleroma/filter.ex +++ b/lib/pleroma/filter.ex @@ -133,10 +133,13 @@ defmodule Pleroma.Filter do defp maybe_add_expires_at(changeset, _), do: changeset defp maybe_add_expiration_job(%{expires_at: %NaiveDateTime{} = expires_at} = filter) do - Pleroma.Workers.PurgeExpiredFilter.enqueue(%{ - filter_id: filter.id, - expires_at: DateTime.from_naive!(expires_at, "Etc/UTC") - }) + Pleroma.Workers.PurgeExpiredFilter.new( + %{ + filter_id: filter.id + }, + scheduled_at: DateTime.from_naive!(expires_at, "Etc/UTC") + ) + |> Oban.insert() end defp maybe_add_expiration_job(_), do: {:ok, nil} diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index 288555146e..33f1229d02 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -297,7 +297,8 @@ defmodule Pleroma.Instances.Instance do all of those users' activities and notifications. """ def delete_users_and_activities(host) when is_binary(host) do - DeleteWorker.enqueue("delete_instance", %{"host" => host}) + DeleteWorker.new(%{"op" => "delete_instance", "host" => host}) + |> Oban.insert() end def perform(:delete_instance, host) when is_binary(host) do diff --git a/lib/pleroma/mfa/token.ex b/lib/pleroma/mfa/token.ex index 57bc11ed52..b53e1c7d0e 100644 --- a/lib/pleroma/mfa/token.ex +++ b/lib/pleroma/mfa/token.ex @@ -52,11 +52,14 @@ defmodule Pleroma.MFA.Token do @spec create(User.t(), Authorization.t() | nil) :: {:ok, t()} | {:error, Ecto.Changeset.t()} def create(user, authorization \\ nil) do with {:ok, token} <- do_create(user, authorization) do - Pleroma.Workers.PurgeExpiredToken.enqueue(%{ - token_id: token.id, - valid_until: DateTime.from_naive!(token.valid_until, "Etc/UTC"), - mod: __MODULE__ - }) + Pleroma.Workers.PurgeExpiredToken.new( + %{ + token_id: token.id, + mod: __MODULE__ + }, + scheduled_at: DateTime.from_naive!(token.valid_until, "Etc/UTC") + ) + |> Oban.insert() {:ok, token} end diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index eb44b3855b..748f18e6cd 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -255,7 +255,8 @@ defmodule Pleroma.Object do @spec cleanup_attachments(boolean(), Object.t()) :: {:ok, Oban.Job.t() | nil} def cleanup_attachments(true, %Object{} = object) do - AttachmentsCleanupWorker.enqueue("cleanup_attachments", %{"object" => object}) + AttachmentsCleanupWorker.new(%{"op" => "cleanup_attachments", "object" => object}) + |> Oban.insert() end def cleanup_attachments(_, _), do: {:ok, nil} diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex index b9d2a0188e..30b3ba9586 100644 --- a/lib/pleroma/search.ex +++ b/lib/pleroma/search.ex @@ -2,11 +2,13 @@ defmodule Pleroma.Search do alias Pleroma.Workers.SearchIndexingWorker def add_to_index(%Pleroma.Activity{id: activity_id}) do - SearchIndexingWorker.enqueue("add_to_index", %{"activity" => activity_id}) + SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => activity_id}) + |> Oban.insert() end def remove_from_index(%Pleroma.Object{id: object_id}) do - SearchIndexingWorker.enqueue("remove_from_index", %{"object" => object_id}) + SearchIndexingWorker.new(%{"op" => "remove_from_index", "object" => object_id}) + |> Oban.insert() end def search(query, options) do diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index e28d76a7c8..0e9d708313 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -736,7 +736,8 @@ defmodule Pleroma.User do end def force_password_reset_async(user) do - BackgroundWorker.enqueue("force_password_reset", %{"user_id" => user.id}) + BackgroundWorker.new(%{"op" => "force_password_reset", "user_id" => user.id}) + |> Oban.insert() end @spec force_password_reset(User.t()) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()} @@ -1218,7 +1219,8 @@ defmodule Pleroma.User do def update_and_set_cache(changeset) do with {:ok, user} <- Repo.update(changeset, stale_error_field: :id) do if get_change(changeset, :raw_fields) do - BackgroundWorker.enqueue("verify_fields_links", %{"user_id" => user.id}) + BackgroundWorker.new(%{"op" => "verify_fields_links", "user_id" => user.id}) + |> Oban.insert() end set_cache(user) @@ -1589,11 +1591,11 @@ defmodule Pleroma.User do )) || {:ok, nil} do if duration > 0 do - Pleroma.Workers.MuteExpireWorker.enqueue( - "unmute_user", - %{"muter_id" => muter.id, "mutee_id" => mutee.id}, + Pleroma.Workers.MuteExpireWorker.new( + %{"op" => "unmute_user", "muter_id" => muter.id, "mutee_id" => mutee.id}, scheduled_at: expires_at ) + |> Oban.insert() end @cachex.del(:user_cache, "muted_users_ap_ids:#{muter.ap_id}") @@ -1836,7 +1838,8 @@ defmodule Pleroma.User do defp maybe_filter_on_ap_id(query, _ap_ids), do: query def set_activation_async(user, status \\ true) do - BackgroundWorker.enqueue("user_activation", %{"user_id" => user.id, "status" => status}) + BackgroundWorker.new(%{"op" => "user_activation", "user_id" => user.id, "status" => status}) + |> Oban.insert() end @spec set_activation([User.t()], boolean()) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()} @@ -1983,7 +1986,9 @@ defmodule Pleroma.User do def delete(%User{} = user) do # Purge the user immediately purge(user) - DeleteWorker.enqueue("delete_user", %{"user_id" => user.id}) + + DeleteWorker.new(%{"op" => "delete_user", "user_id" => user.id}) + |> Oban.insert() end # *Actually* delete the user from the DB diff --git a/lib/pleroma/user/import.ex b/lib/pleroma/user/import.ex index 53ffd1ab3d..11905237c3 100644 --- a/lib/pleroma/user/import.ex +++ b/lib/pleroma/user/import.ex @@ -63,23 +63,29 @@ defmodule Pleroma.User.Import do end def blocks_import(%User{} = blocker, [_ | _] = identifiers) do - BackgroundWorker.enqueue( - "blocks_import", - %{"user_id" => blocker.id, "identifiers" => identifiers} - ) + BackgroundWorker.new(%{ + "op" => "blocks_import", + "user_id" => blocker.id, + "identifiers" => identifiers + }) + |> Oban.insert() end def follow_import(%User{} = follower, [_ | _] = identifiers) do - BackgroundWorker.enqueue( - "follow_import", - %{"user_id" => follower.id, "identifiers" => identifiers} - ) + BackgroundWorker.new(%{ + "op" => "follow_import", + "user_id" => follower.id, + "identifiers" => identifiers + }) + |> Oban.insert() end def mutes_import(%User{} = user, [_ | _] = identifiers) do - BackgroundWorker.enqueue( - "mutes_import", - %{"user_id" => user.id, "identifiers" => identifiers} - ) + BackgroundWorker.new(%{ + "op" => "mutes_import", + "user_id" => user.id, + "identifiers" => identifiers + }) + |> Oban.insert() end end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index b30b0cabe5..a2a94a0ff4 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -222,10 +222,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do %{data: %{"expires_at" => %DateTime{} = expires_at}} = activity ) do with {:ok, _job} <- - Pleroma.Workers.PurgeExpiredActivity.enqueue(%{ - activity_id: activity.id, - expires_at: expires_at - }) do + Pleroma.Workers.PurgeExpiredActivity.enqueue( + %{ + activity_id: activity.id + }, + scheduled_at: expires_at + ) do {:ok, activity} end end @@ -446,10 +448,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do _ <- notify_and_stream(activity) do maybe_federate(activity) - BackgroundWorker.enqueue("move_following", %{ + BackgroundWorker.new(%{ + "op" => "move_following", "origin_id" => origin.id, "target_id" => target.id }) + |> Oban.insert() {:ok, activity} else @@ -1797,10 +1801,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do # enqueue a task to fetch all pinned objects Enum.each(pins, fn {ap_id, _} -> if is_nil(Object.get_cached_by_ap_id(ap_id)) do - Pleroma.Workers.RemoteFetcherWorker.enqueue("fetch_remote", %{ + Pleroma.Workers.RemoteFetcherWorker.new(%{ + "op" => "fetch_remote", "id" => ap_id, "depth" => 1 }) + |> Oban.insert() end end) end diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index e040753dc6..f71652cb74 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -30,11 +30,11 @@ defmodule Pleroma.Web.ActivityPub.Publisher do """ @spec enqueue_one(map(), Keyword.t()) :: {:ok, %Oban.Job{}} def enqueue_one(%{} = params, worker_args \\ []) do - PublisherWorker.enqueue( - "publish_one", - %{"params" => params}, + PublisherWorker.new( + %{"op" => "publish_one", "params" => params}, worker_args ) + |> Oban.insert() end @doc """ diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index cc1c7a0afb..d6d4036719 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -223,10 +223,12 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do if Pleroma.Web.Federator.allowed_thread_distance?(reply_depth) and object.data["replies"] != nil do for reply_id <- object.data["replies"] do - Pleroma.Workers.RemoteFetcherWorker.enqueue("fetch_remote", %{ + Pleroma.Workers.RemoteFetcherWorker.new(%{ + "op" => "fetch_remote", "id" => reply_id, "depth" => reply_depth }) + |> Oban.insert() end end @@ -410,10 +412,12 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do {:ok, expires_at} = Pleroma.EctoType.ActivityPub.ObjectValidators.DateTime.cast(meta[:expires_at]) - Pleroma.Workers.PurgeExpiredActivity.enqueue(%{ - activity_id: meta[:activity_id], - expires_at: expires_at - }) + Pleroma.Workers.PurgeExpiredActivity.enqueue( + %{ + activity_id: meta[:activity_id] + }, + scheduled_at: expires_at + ) end {:ok, object, meta} diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index b90b6a6d90..1ed905d6cb 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -559,11 +559,11 @@ defmodule Pleroma.Web.CommonAPI do with {:ok, _} <- ThreadMute.add_mute(user.id, activity.data["context"]), _ <- Pleroma.Notification.mark_context_as_read(user, activity.data["context"]) do if expires_in > 0 do - Pleroma.Workers.MuteExpireWorker.enqueue( - "unmute_conversation", - %{"user_id" => user.id, "activity_id" => activity.id}, + Pleroma.Workers.MuteExpireWorker.new( + %{"op" => "unmute_conversation", "user_id" => user.id, "activity_id" => activity.id}, schedule_in: expires_in ) + |> Oban.insert() end {:ok, activity} diff --git a/lib/pleroma/web/federator.ex b/lib/pleroma/web/federator.ex index 3d3101d61e..c740fc85fa 100644 --- a/lib/pleroma/web/federator.ex +++ b/lib/pleroma/web/federator.ex @@ -35,22 +35,30 @@ defmodule Pleroma.Web.Federator do end # Client API - def incoming_ap_doc(%{params: _params, req_headers: _req_headers} = args) do - job_args = Enum.into(args, %{}, fn {k, v} -> {Atom.to_string(k), v} end) - - ReceiverWorker.enqueue( - "incoming_ap_doc", - Map.put(job_args, "timeout", :timer.seconds(20)), + def incoming_ap_doc(%{params: params, req_headers: req_headers}) do + ReceiverWorker.new( + %{ + "op" => "incoming_ap_doc", + "req_headers" => req_headers, + "params" => params, + "timeout" => :timer.seconds(20) + }, priority: 2 ) + |> Oban.insert() end def incoming_ap_doc(%{"type" => "Delete"} = params) do - ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}, priority: 3, queue: :slow) + ReceiverWorker.new(%{"op" => "incoming_ap_doc", "params" => params}, + priority: 3, + queue: :slow + ) + |> Oban.insert() end def incoming_ap_doc(params) do - ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}) + ReceiverWorker.new(%{"op" => "incoming_ap_doc", "params" => params}) + |> Oban.insert() end @impl true @@ -60,9 +68,10 @@ defmodule Pleroma.Web.Federator do @impl true def publish(%Pleroma.Activity{data: %{"type" => type}} = activity) do - PublisherWorker.enqueue("publish", %{"activity_id" => activity.id}, + PublisherWorker.new(%{"op" => "publish", "activity_id" => activity.id}, priority: publish_priority(type) ) + |> Oban.insert() end defp publish_priority("Delete"), do: 3 diff --git a/lib/pleroma/web/o_auth/token.ex b/lib/pleroma/web/o_auth/token.ex index 9b1198b428..d964250944 100644 --- a/lib/pleroma/web/o_auth/token.ex +++ b/lib/pleroma/web/o_auth/token.ex @@ -100,11 +100,10 @@ defmodule Pleroma.Web.OAuth.Token do def create(%App{} = app, %User{} = user, attrs \\ %{}) do with {:ok, token} <- do_create(app, user, attrs) do if Pleroma.Config.get([:oauth2, :clean_expired_tokens]) do - Pleroma.Workers.PurgeExpiredToken.enqueue(%{ - token_id: token.id, - valid_until: DateTime.from_naive!(token.valid_until, "Etc/UTC"), - mod: __MODULE__ - }) + Pleroma.Workers.PurgeExpiredToken.new(%{token_id: token.id, mod: __MODULE__}, + scheduled_at: DateTime.from_naive!(token.valid_until, "Etc/UTC") + ) + |> Oban.insert() end {:ok, token} diff --git a/lib/pleroma/web/push.ex b/lib/pleroma/web/push.ex index d4693f63e4..d783f776a5 100644 --- a/lib/pleroma/web/push.ex +++ b/lib/pleroma/web/push.ex @@ -28,6 +28,6 @@ defmodule Pleroma.Web.Push do @spec send(Pleroma.Notification.t()) :: {:ok, Oban.Job.t()} | {:error, Oban.Job.changeset() | term()} def send(notification) do - WebPusherWorker.enqueue("web_push", %{"notification_id" => notification.id}) + WebPusherWorker.new(%{"op" => "web_push", "notification_id" => notification.id}) end end diff --git a/lib/pleroma/workers/attachments_cleanup_worker.ex b/lib/pleroma/workers/attachments_cleanup_worker.ex index 0b570b70b0..e2f92b1fd0 100644 --- a/lib/pleroma/workers/attachments_cleanup_worker.ex +++ b/lib/pleroma/workers/attachments_cleanup_worker.ex @@ -8,9 +8,9 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do alias Pleroma.Object alias Pleroma.Repo - use Pleroma.Workers.WorkerHelper, queue: "slow" + use Oban.Worker, queue: :slow - @impl Oban.Worker + @impl true def perform(%Job{ args: %{ "op" => "cleanup_attachments", @@ -31,7 +31,7 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do def perform(%Job{args: %{"op" => "cleanup_attachments", "object" => _object}}), do: {:ok, :skip} - @impl Oban.Worker + @impl true def timeout(_job), do: :timer.seconds(900) defp do_clean({object_ids, attachment_urls}) do diff --git a/lib/pleroma/workers/background_worker.ex b/lib/pleroma/workers/background_worker.ex index 870aef3c6a..60da2d5ca7 100644 --- a/lib/pleroma/workers/background_worker.ex +++ b/lib/pleroma/workers/background_worker.ex @@ -5,9 +5,9 @@ defmodule Pleroma.Workers.BackgroundWorker do alias Pleroma.User - use Pleroma.Workers.WorkerHelper, queue: "background" + use Oban.Worker, queue: :background - @impl Oban.Worker + @impl true def perform(%Job{args: %{"op" => "user_activation", "user_id" => user_id, "status" => status}}) do user = User.get_cached_by_id(user_id) @@ -39,6 +39,6 @@ defmodule Pleroma.Workers.BackgroundWorker do User.perform(:verify_fields_links, user) end - @impl Oban.Worker - def timeout(_job), do: :timer.seconds(15) + @impl true + def timeout(_job), do: :timer.seconds(900) end diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex index d1b6fcdadf..6466d8d73e 100644 --- a/lib/pleroma/workers/backup_worker.ex +++ b/lib/pleroma/workers/backup_worker.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Workers.BackupWorker do alias Pleroma.Config.Getting, as: Config alias Pleroma.User.Backup - @impl Oban.Worker + @impl true def perform(%Job{ args: %{"op" => "process", "backup_id" => backup_id} }) do @@ -32,7 +32,7 @@ defmodule Pleroma.Workers.BackupWorker do end end - @impl Oban.Worker + @impl true def timeout(_job), do: Config.get([Backup, :timeout], :timer.minutes(30)) defp has_email?(user) do diff --git a/lib/pleroma/workers/cron/digest_emails_worker.ex b/lib/pleroma/workers/cron/digest_emails_worker.ex index 17e92d10b0..b50b52a7b0 100644 --- a/lib/pleroma/workers/cron/digest_emails_worker.ex +++ b/lib/pleroma/workers/cron/digest_emails_worker.ex @@ -18,7 +18,7 @@ defmodule Pleroma.Workers.Cron.DigestEmailsWorker do require Logger - @impl Oban.Worker + @impl true def perform(_job) do config = Config.get([:email_notifications, :digest]) @@ -59,6 +59,6 @@ defmodule Pleroma.Workers.Cron.DigestEmailsWorker do User.touch_last_digest_emailed_at(user) end - @impl Oban.Worker + @impl true def timeout(_job), do: :timer.seconds(5) end diff --git a/lib/pleroma/workers/cron/new_users_digest_worker.ex b/lib/pleroma/workers/cron/new_users_digest_worker.ex index 1f57aad4a2..7876499831 100644 --- a/lib/pleroma/workers/cron/new_users_digest_worker.ex +++ b/lib/pleroma/workers/cron/new_users_digest_worker.ex @@ -9,9 +9,9 @@ defmodule Pleroma.Workers.Cron.NewUsersDigestWorker do import Ecto.Query - use Pleroma.Workers.WorkerHelper, queue: "background" + use Oban.Worker, queue: :background - @impl Oban.Worker + @impl true def perform(_job) do if Pleroma.Config.get([Pleroma.Emails.NewUsersDigestEmail, :enabled]) do today = NaiveDateTime.utc_now() |> Timex.beginning_of_day() @@ -61,6 +61,6 @@ defmodule Pleroma.Workers.Cron.NewUsersDigestWorker do :ok end - @impl Oban.Worker + @impl true def timeout(_job), do: :timer.seconds(5) end diff --git a/lib/pleroma/workers/delete_worker.ex b/lib/pleroma/workers/delete_worker.ex index 97003fb69e..6a1c7bb383 100644 --- a/lib/pleroma/workers/delete_worker.ex +++ b/lib/pleroma/workers/delete_worker.ex @@ -6,10 +6,9 @@ defmodule Pleroma.Workers.DeleteWorker do alias Pleroma.Instances.Instance alias Pleroma.User - use Pleroma.Workers.WorkerHelper, queue: "slow" - - @impl Oban.Worker + use Oban.Worker, queue: :slow + @impl true def perform(%Job{args: %{"op" => "delete_user", "user_id" => user_id}}) do user = User.get_cached_by_id(user_id) User.perform(:delete, user) @@ -19,6 +18,6 @@ defmodule Pleroma.Workers.DeleteWorker do Instance.perform(:delete_instance, host) end - @impl Oban.Worker + @impl true def timeout(_job), do: :timer.seconds(900) end diff --git a/lib/pleroma/workers/mailer_worker.ex b/lib/pleroma/workers/mailer_worker.ex index 652bf77e01..b0259b1917 100644 --- a/lib/pleroma/workers/mailer_worker.ex +++ b/lib/pleroma/workers/mailer_worker.ex @@ -3,9 +3,9 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.MailerWorker do - use Pleroma.Workers.WorkerHelper, queue: "background" + use Oban.Worker, queue: :background - @impl Oban.Worker + @impl true def perform(%Job{args: %{"op" => "email", "encoded_email" => encoded_email, "config" => config}}) do encoded_email |> Base.decode64!() @@ -13,6 +13,6 @@ defmodule Pleroma.Workers.MailerWorker do |> Pleroma.Emails.Mailer.deliver(config) end - @impl Oban.Worker + @impl true def timeout(_job), do: :timer.seconds(5) end diff --git a/lib/pleroma/workers/mute_expire_worker.ex b/lib/pleroma/workers/mute_expire_worker.ex index a7ab5883af..8356a775d7 100644 --- a/lib/pleroma/workers/mute_expire_worker.ex +++ b/lib/pleroma/workers/mute_expire_worker.ex @@ -3,9 +3,9 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.MuteExpireWorker do - use Pleroma.Workers.WorkerHelper, queue: "background" + use Oban.Worker, queue: :background - @impl Oban.Worker + @impl true def perform(%Job{args: %{"op" => "unmute_user", "muter_id" => muter_id, "mutee_id" => mutee_id}}) do Pleroma.User.unmute(muter_id, mutee_id) :ok @@ -18,6 +18,6 @@ defmodule Pleroma.Workers.MuteExpireWorker do :ok end - @impl Oban.Worker + @impl true def timeout(_job), do: :timer.seconds(5) end diff --git a/lib/pleroma/workers/poll_worker.ex b/lib/pleroma/workers/poll_worker.ex index af8997e702..d263aa1b9e 100644 --- a/lib/pleroma/workers/poll_worker.ex +++ b/lib/pleroma/workers/poll_worker.ex @@ -6,13 +6,13 @@ defmodule Pleroma.Workers.PollWorker do @moduledoc """ Generates notifications when a poll ends. """ - use Pleroma.Workers.WorkerHelper, queue: "background" + use Oban.Worker, queue: :background alias Pleroma.Activity alias Pleroma.Notification alias Pleroma.Object - @impl Oban.Worker + @impl true def perform(%Job{args: %{"op" => "poll_end", "activity_id" => activity_id}}) do with %Activity{} = activity <- find_poll_activity(activity_id), {:ok, notifications} <- Notification.create_poll_notifications(activity) do @@ -23,7 +23,7 @@ defmodule Pleroma.Workers.PollWorker do end end - @impl Oban.Worker + @impl true def timeout(_job), do: :timer.seconds(5) defp find_poll_activity(activity_id) do diff --git a/lib/pleroma/workers/publisher_worker.ex b/lib/pleroma/workers/publisher_worker.ex index 63fcf4ac22..7d9b022de3 100644 --- a/lib/pleroma/workers/publisher_worker.ex +++ b/lib/pleroma/workers/publisher_worker.ex @@ -6,13 +6,9 @@ defmodule Pleroma.Workers.PublisherWorker do alias Pleroma.Activity alias Pleroma.Web.Federator - use Pleroma.Workers.WorkerHelper, queue: "federator_outgoing" + use Oban.Worker, queue: :federator_outgoing, max_attempts: 5 - def backoff(%Job{attempt: attempt}) when is_integer(attempt) do - Pleroma.Workers.WorkerHelper.sidekiq_backoff(attempt, 5) - end - - @impl Oban.Worker + @impl true def perform(%Job{args: %{"op" => "publish", "activity_id" => activity_id}}) do activity = Activity.get_by_id(activity_id) Federator.perform(:publish, activity) @@ -23,6 +19,18 @@ defmodule Pleroma.Workers.PublisherWorker do Federator.perform(:publish_one, params) end - @impl Oban.Worker + @impl true def timeout(_job), do: :timer.seconds(10) + + @base_backoff 15 + @pow 5 + @impl true + def backoff(%Job{attempt: attempt}) when is_integer(attempt) do + backoff = + :math.pow(attempt, @pow) + + @base_backoff + + :rand.uniform(2 * @base_backoff) * attempt + + trunc(backoff) + end end diff --git a/lib/pleroma/workers/purge_expired_activity.ex b/lib/pleroma/workers/purge_expired_activity.ex index f48e340420..f05e75f46a 100644 --- a/lib/pleroma/workers/purge_expired_activity.ex +++ b/lib/pleroma/workers/purge_expired_activity.ex @@ -13,16 +13,13 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do alias Pleroma.Activity - @spec enqueue(map()) :: + @spec enqueue(map(), list()) :: {:ok, Oban.Job.t()} | {:error, :expired_activities_disabled} | {:error, :expiration_too_close} - def enqueue(args) do + def enqueue(params, worker_args) do with true <- enabled?() do - {scheduled_at, args} = Map.pop(args, :expires_at) - - args - |> new(scheduled_at: scheduled_at) + new(params, worker_args) |> Oban.insert() end end @@ -35,7 +32,7 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do end end - @impl Oban.Worker + @impl true def timeout(_job), do: :timer.seconds(5) defp enabled? do diff --git a/lib/pleroma/workers/purge_expired_filter.ex b/lib/pleroma/workers/purge_expired_filter.ex index 1f6931e4c2..0405f66847 100644 --- a/lib/pleroma/workers/purge_expired_filter.ex +++ b/lib/pleroma/workers/purge_expired_filter.ex @@ -31,7 +31,7 @@ defmodule Pleroma.Workers.PurgeExpiredFilter do |> Repo.delete() end - @impl Oban.Worker + @impl true def timeout(_job), do: :timer.seconds(5) @spec get_expiration(pos_integer()) :: Job.t() | nil diff --git a/lib/pleroma/workers/purge_expired_token.ex b/lib/pleroma/workers/purge_expired_token.ex index 1854bf5619..ff962f21b6 100644 --- a/lib/pleroma/workers/purge_expired_token.ex +++ b/lib/pleroma/workers/purge_expired_token.ex @@ -9,16 +9,6 @@ defmodule Pleroma.Workers.PurgeExpiredToken do use Oban.Worker, queue: :background, max_attempts: 1 - @spec enqueue(%{token_id: integer(), valid_until: DateTime.t(), mod: module()}) :: - {:ok, Oban.Job.t()} | {:error, Ecto.Changeset.t()} - def enqueue(args) do - {scheduled_at, args} = Map.pop(args, :valid_until) - - args - |> __MODULE__.new(scheduled_at: scheduled_at) - |> Oban.insert() - end - @impl true def perform(%Oban.Job{args: %{"token_id" => id, "mod" => module}}) do module @@ -27,6 +17,6 @@ defmodule Pleroma.Workers.PurgeExpiredToken do |> Pleroma.Repo.delete() end - @impl Oban.Worker + @impl true def timeout(_job), do: :timer.seconds(5) end diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index fd5c13fca2..d4db97b639 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -7,9 +7,9 @@ defmodule Pleroma.Workers.ReceiverWorker do alias Pleroma.User alias Pleroma.Web.Federator - use Pleroma.Workers.WorkerHelper, queue: "federator_incoming" + use Oban.Worker, queue: :federator_incoming, max_attempts: 5 - @impl Oban.Worker + @impl true def perform(%Job{ args: %{ @@ -51,7 +51,7 @@ defmodule Pleroma.Workers.ReceiverWorker do end end - @impl Oban.Worker + @impl true def timeout(%_{args: %{"timeout" => timeout}}), do: timeout def timeout(_job), do: :timer.seconds(5) diff --git a/lib/pleroma/workers/remote_fetcher_worker.ex b/lib/pleroma/workers/remote_fetcher_worker.ex index 60096e14b5..e43765733e 100644 --- a/lib/pleroma/workers/remote_fetcher_worker.ex +++ b/lib/pleroma/workers/remote_fetcher_worker.ex @@ -5,9 +5,9 @@ defmodule Pleroma.Workers.RemoteFetcherWorker do alias Pleroma.Object.Fetcher - use Pleroma.Workers.WorkerHelper, queue: "background" + use Oban.Worker, queue: :background - @impl Oban.Worker + @impl true def perform(%Job{args: %{"op" => "fetch_remote", "id" => id} = args}) do case Fetcher.fetch_object_from_id(id, depth: args["depth"]) do {:ok, _object} -> @@ -30,6 +30,6 @@ defmodule Pleroma.Workers.RemoteFetcherWorker do end end - @impl Oban.Worker + @impl true def timeout(_job), do: :timer.seconds(15) end diff --git a/lib/pleroma/workers/rich_media_worker.ex b/lib/pleroma/workers/rich_media_worker.ex index 2ebf42d4f2..d5ba7b63ec 100644 --- a/lib/pleroma/workers/rich_media_worker.ex +++ b/lib/pleroma/workers/rich_media_worker.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Workers.RichMediaWorker do use Oban.Worker, queue: :background, max_attempts: 3, unique: [period: 300] - @impl Oban.Worker + @impl true def perform(%Job{args: %{"op" => "expire", "url" => url} = _args}) do Card.delete(url) end @@ -33,7 +33,7 @@ defmodule Pleroma.Workers.RichMediaWorker do # a slow/infinite data stream and insert a negative cache entry for the URL # We pad it by 2 seconds to be certain a slow connection is detected and we # can inject a negative cache entry for the URL - @impl Oban.Worker + @impl true def timeout(_job) do Config.get!([:rich_media, :timeout]) + :timer.seconds(2) end diff --git a/lib/pleroma/workers/scheduled_activity_worker.ex b/lib/pleroma/workers/scheduled_activity_worker.ex index ab62686f42..da386e0c32 100644 --- a/lib/pleroma/workers/scheduled_activity_worker.ex +++ b/lib/pleroma/workers/scheduled_activity_worker.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Workers.ScheduledActivityWorker do The worker to post scheduled activity. """ - use Pleroma.Workers.WorkerHelper, queue: "federator_outgoing" + use Oban.Worker, queue: :federator_outgoing, max_attempts: 5 alias Pleroma.Repo alias Pleroma.ScheduledActivity @@ -15,7 +15,7 @@ defmodule Pleroma.Workers.ScheduledActivityWorker do require Logger - @impl Oban.Worker + @impl true def perform(%Job{args: %{"activity_id" => activity_id}}) do with %ScheduledActivity{} = scheduled_activity <- find_scheduled_activity(activity_id), %User{} = user <- find_user(scheduled_activity.user_id) do @@ -37,7 +37,7 @@ defmodule Pleroma.Workers.ScheduledActivityWorker do end end - @impl Oban.Worker + @impl true def timeout(_job), do: :timer.seconds(5) defp find_scheduled_activity(id) do diff --git a/lib/pleroma/workers/search_indexing_worker.ex b/lib/pleroma/workers/search_indexing_worker.ex index 8969ae3788..001f5254d4 100644 --- a/lib/pleroma/workers/search_indexing_worker.ex +++ b/lib/pleroma/workers/search_indexing_worker.ex @@ -1,7 +1,7 @@ defmodule Pleroma.Workers.SearchIndexingWorker do - use Pleroma.Workers.WorkerHelper, queue: "search_indexing" + use Oban.Worker, queue: :search_indexing, max_attempts: 2 - @impl Oban.Worker + @impl true alias Pleroma.Config.Getting, as: Config @@ -21,6 +21,6 @@ defmodule Pleroma.Workers.SearchIndexingWorker do search_module.remove_from_index(object) end - @impl Oban.Worker + @impl true def timeout(_job), do: :timer.seconds(5) end diff --git a/lib/pleroma/workers/user_refresh_worker.ex b/lib/pleroma/workers/user_refresh_worker.ex index fb90e9c9c0..222a4a8f7e 100644 --- a/lib/pleroma/workers/user_refresh_worker.ex +++ b/lib/pleroma/workers/user_refresh_worker.ex @@ -12,6 +12,6 @@ defmodule Pleroma.Workers.UserRefreshWorker do User.fetch_by_ap_id(ap_id) end - @impl Oban.Worker + @impl true def timeout(_job), do: :timer.seconds(15) end diff --git a/lib/pleroma/workers/web_pusher_worker.ex b/lib/pleroma/workers/web_pusher_worker.ex index c549d3cd65..f4232d02af 100644 --- a/lib/pleroma/workers/web_pusher_worker.ex +++ b/lib/pleroma/workers/web_pusher_worker.ex @@ -7,9 +7,9 @@ defmodule Pleroma.Workers.WebPusherWorker do alias Pleroma.Repo alias Pleroma.Web.Push.Impl - use Pleroma.Workers.WorkerHelper, queue: "web_push" + use Oban.Worker, queue: :web_push - @impl Oban.Worker + @impl true def perform(%Job{args: %{"op" => "web_push", "notification_id" => notification_id}}) do notification = Notification @@ -20,6 +20,6 @@ defmodule Pleroma.Workers.WebPusherWorker do |> Enum.each(&Impl.deliver(&1)) end - @impl Oban.Worker + @impl true def timeout(_job), do: :timer.seconds(5) end diff --git a/lib/pleroma/workers/worker_helper.ex b/lib/pleroma/workers/worker_helper.ex deleted file mode 100644 index 1d20cbd893..0000000000 --- a/lib/pleroma/workers/worker_helper.ex +++ /dev/null @@ -1,48 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Workers.WorkerHelper do - alias Pleroma.Config - alias Pleroma.Workers.WorkerHelper - - def worker_args(queue) do - case Config.get([:workers, :retries, queue]) do - nil -> [] - max_attempts -> [max_attempts: max_attempts] - end - end - - def sidekiq_backoff(attempt, pow \\ 4, base_backoff \\ 15) do - backoff = - :math.pow(attempt, pow) + - base_backoff + - :rand.uniform(2 * base_backoff) * attempt - - trunc(backoff) - end - - defmacro __using__(opts) do - caller_module = __CALLER__.module - queue = Keyword.fetch!(opts, :queue) - - quote do - # Note: `max_attempts` is intended to be overridden in `new/2` call - use Oban.Worker, - queue: unquote(queue), - max_attempts: 1 - - alias Oban.Job - - def enqueue(op, params, worker_args \\ []) do - params = Map.merge(%{"op" => op}, params) - queue_atom = String.to_atom(unquote(queue)) - worker_args = worker_args ++ WorkerHelper.worker_args(queue_atom) - - unquote(caller_module) - |> apply(:new, [params, worker_args]) - |> Oban.insert() - end - end - end -end diff --git a/test/mix/tasks/pleroma/database_test.exs b/test/mix/tasks/pleroma/database_test.exs index a51a3bf3dc..96a9255288 100644 --- a/test/mix/tasks/pleroma/database_test.exs +++ b/test/mix/tasks/pleroma/database_test.exs @@ -623,10 +623,12 @@ defmodule Mix.Tasks.Pleroma.DatabaseTest do expires_at = DateTime.add(DateTime.utc_now(), 60 * 61) - Pleroma.Workers.PurgeExpiredActivity.enqueue(%{ - activity_id: activity_id3, - expires_at: expires_at - }) + Pleroma.Workers.PurgeExpiredActivity.enqueue( + %{ + activity_id: activity_id3 + }, + scheduled_at: expires_at + ) Mix.Tasks.Pleroma.Database.run(["ensure_expiration"]) diff --git a/test/pleroma/workers/purge_expired_activity_test.exs b/test/pleroma/workers/purge_expired_activity_test.exs index 040ff6a51c..ea563d3d31 100644 --- a/test/pleroma/workers/purge_expired_activity_test.exs +++ b/test/pleroma/workers/purge_expired_activity_test.exs @@ -14,10 +14,12 @@ defmodule Pleroma.Workers.PurgeExpiredActivityTest do activity = insert(:note_activity) assert {:ok, _} = - PurgeExpiredActivity.enqueue(%{ - activity_id: activity.id, - expires_at: DateTime.add(DateTime.utc_now(), 3601) - }) + PurgeExpiredActivity.enqueue( + %{ + activity_id: activity.id + }, + scheduled_at: DateTime.add(DateTime.utc_now(), 3601) + ) assert_enqueued( worker: Pleroma.Workers.PurgeExpiredActivity, @@ -34,10 +36,12 @@ defmodule Pleroma.Workers.PurgeExpiredActivityTest do activity = insert(:note_activity) assert {:ok, _} = - PurgeExpiredActivity.enqueue(%{ - activity_id: activity.id, - expires_at: DateTime.add(DateTime.utc_now(), 3601) - }) + PurgeExpiredActivity.enqueue( + %{ + activity_id: activity.id + }, + scheduled_at: DateTime.add(DateTime.utc_now(), 3601) + ) user = Pleroma.User.get_by_ap_id(activity.actor) Pleroma.Repo.delete(user) @@ -48,10 +52,12 @@ defmodule Pleroma.Workers.PurgeExpiredActivityTest do test "error if actiivity was not found" do assert {:ok, _} = - PurgeExpiredActivity.enqueue(%{ - activity_id: "some_id", - expires_at: DateTime.add(DateTime.utc_now(), 3601) - }) + PurgeExpiredActivity.enqueue( + %{ + activity_id: "some_id" + }, + scheduled_at: DateTime.add(DateTime.utc_now(), 3601) + ) assert {:cancel, :activity_not_found} = perform_job(Pleroma.Workers.PurgeExpiredActivity, %{activity_id: "some_if"}) From 1d3a92be1101b04d1b69b8b3fd030d5d06f4dbea Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 28 Jun 2024 12:06:43 -0400 Subject: [PATCH 02/34] Remove :workers config from ConfigDB --- config/description.exs | 17 ----------------- ...240628160536_deprecate_config_db_workers.exs | 7 +++++++ 2 files changed, 7 insertions(+), 17 deletions(-) create mode 100644 priv/repo/migrations/20240628160536_deprecate_config_db_workers.exs diff --git a/config/description.exs b/config/description.exs index 2809e91307..15faecb38c 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2013,23 +2013,6 @@ config :pleroma, :config_description, [ } ] }, - %{ - group: :pleroma, - key: :workers, - type: :group, - description: "Includes custom worker options not interpretable directly by `Oban`", - children: [ - %{ - key: :retries, - type: {:keyword, :integer}, - description: "Max retry attempts for failed jobs, per `Oban` queue", - suggestions: [ - federator_incoming: 5, - federator_outgoing: 5 - ] - } - ] - }, %{ group: :pleroma, key: Pleroma.Web.Metadata, diff --git a/priv/repo/migrations/20240628160536_deprecate_config_db_workers.exs b/priv/repo/migrations/20240628160536_deprecate_config_db_workers.exs new file mode 100644 index 0000000000..549dd22e9e --- /dev/null +++ b/priv/repo/migrations/20240628160536_deprecate_config_db_workers.exs @@ -0,0 +1,7 @@ +defmodule Pleroma.Repo.Migrations.DeprecateConfigDBWorkers do + use Ecto.Migration + + def change do + execute("DELETE FROM config WHERE config.group = ':workers'") + end +end From d56b889cf1f530364ecd10836f8bd11b67eccb47 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 28 Jun 2024 12:21:27 -0400 Subject: [PATCH 03/34] Update changelog --- changelog.d/workerhelper.change | 1 + changelog.d/workerhelper.skip | 0 2 files changed, 1 insertion(+) create mode 100644 changelog.d/workerhelper.change delete mode 100644 changelog.d/workerhelper.skip diff --git a/changelog.d/workerhelper.change b/changelog.d/workerhelper.change new file mode 100644 index 0000000000..7a20c4af82 --- /dev/null +++ b/changelog.d/workerhelper.change @@ -0,0 +1 @@ +Worker configuration is no longer available. This only affects custom max_retries values for Oban queues. diff --git a/changelog.d/workerhelper.skip b/changelog.d/workerhelper.skip deleted file mode 100644 index e69de29bb2..0000000000 From 52f7033f7ac82155fc927f4b0a3f4f9e8ae11114 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sun, 4 Aug 2024 16:02:44 +0200 Subject: [PATCH 04/34] StreamerView: Do not leak follows count if hidden MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- .../stream-follow-relationships-count.fix | 1 + lib/pleroma/web/views/streamer_view.ex | 26 ++++++++++++++----- 2 files changed, 20 insertions(+), 7 deletions(-) create mode 100644 changelog.d/stream-follow-relationships-count.fix diff --git a/changelog.d/stream-follow-relationships-count.fix b/changelog.d/stream-follow-relationships-count.fix new file mode 100644 index 0000000000..68452a88b8 --- /dev/null +++ b/changelog.d/stream-follow-relationships-count.fix @@ -0,0 +1 @@ +StreamerView: Do not leak follows count if hidden \ No newline at end of file diff --git a/lib/pleroma/web/views/streamer_view.ex b/lib/pleroma/web/views/streamer_view.ex index f97570b0a2..6016d821b3 100644 --- a/lib/pleroma/web/views/streamer_view.ex +++ b/lib/pleroma/web/views/streamer_view.ex @@ -109,7 +109,11 @@ defmodule Pleroma.Web.StreamerView do |> Jason.encode!() end - def render("follow_relationships_update.json", item, topic) do + def render( + "follow_relationships_update.json", + %{follower: follower, following: following} = item, + topic + ) do %{ stream: render("stream.json", %{topic: topic}), event: "pleroma:follow_relationships_update", @@ -117,14 +121,22 @@ defmodule Pleroma.Web.StreamerView do %{ state: item.state, follower: %{ - id: item.follower.id, - follower_count: item.follower.follower_count, - following_count: item.follower.following_count + id: follower.id, + follower_count: follower.follower_count, + following_count: follower.following_count }, following: %{ - id: item.following.id, - follower_count: item.following.follower_count, - following_count: item.following.following_count + id: following.id, + follower_count: + if(!following.hide_followers_count or !following.hide_followers, + do: following.follower_count, + else: 0 + ), + following_count: + if(!following.hide_follows_count or !following.hide_follows, + do: following.following_count, + else: 0 + ) } } |> Jason.encode!() From c284c4e3e697dda5c5965588667317090a51bfca Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 7 Aug 2024 09:19:33 -0400 Subject: [PATCH 05/34] Extract the logic from the map --- lib/pleroma/web/views/streamer_view.ex | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/web/views/streamer_view.ex b/lib/pleroma/web/views/streamer_view.ex index 6016d821b3..8e5a9f2a47 100644 --- a/lib/pleroma/web/views/streamer_view.ex +++ b/lib/pleroma/web/views/streamer_view.ex @@ -114,6 +114,20 @@ defmodule Pleroma.Web.StreamerView do %{follower: follower, following: following} = item, topic ) do + follower_count = + if Enum.any?([following.hide_followers_count, following.hide_followers]) do + 0 + else + following.follower_count + end + + following_count = + if Enum.any?([following.hide_follows_count, following.hide_follows]) do + 0 + else + following.following_count + end + %{ stream: render("stream.json", %{topic: topic}), event: "pleroma:follow_relationships_update", @@ -127,16 +141,8 @@ defmodule Pleroma.Web.StreamerView do }, following: %{ id: following.id, - follower_count: - if(!following.hide_followers_count or !following.hide_followers, - do: following.follower_count, - else: 0 - ), - following_count: - if(!following.hide_follows_count or !following.hide_follows, - do: following.following_count, - else: 0 - ) + follower_count: follower_count, + following_count: following_count } } |> Jason.encode!() From 7d33b53908242bf420d9f84550cabb5f86bc4738 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 7 Aug 2024 11:47:43 -0400 Subject: [PATCH 06/34] Improve the variable naming --- lib/pleroma/web/views/streamer_view.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/views/streamer_view.ex b/lib/pleroma/web/views/streamer_view.ex index 8e5a9f2a47..079a373516 100644 --- a/lib/pleroma/web/views/streamer_view.ex +++ b/lib/pleroma/web/views/streamer_view.ex @@ -114,14 +114,14 @@ defmodule Pleroma.Web.StreamerView do %{follower: follower, following: following} = item, topic ) do - follower_count = + following_follower_count = if Enum.any?([following.hide_followers_count, following.hide_followers]) do 0 else following.follower_count end - following_count = + following_following_count = if Enum.any?([following.hide_follows_count, following.hide_follows]) do 0 else @@ -141,8 +141,8 @@ defmodule Pleroma.Web.StreamerView do }, following: %{ id: following.id, - follower_count: follower_count, - following_count: following_count + follower_count: following_follower_count, + following_count: following_following_count } } |> Jason.encode!() From ad7fe4e95de42a91ef46a88a3f2863682c68ec5d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 7 Aug 2024 11:47:59 -0400 Subject: [PATCH 07/34] Tests to confirm wanted behavior --- test/pleroma/web/views/streamer_view_test.ex | 100 +++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 test/pleroma/web/views/streamer_view_test.ex diff --git a/test/pleroma/web/views/streamer_view_test.ex b/test/pleroma/web/views/streamer_view_test.ex new file mode 100644 index 0000000000..43a17a43ea --- /dev/null +++ b/test/pleroma/web/views/streamer_view_test.ex @@ -0,0 +1,100 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.StreamerViewTest do + use Pleroma.Web.ConnCase, async: true + # import ExUnit.CaptureLog + import Pleroma.Factory + + alias Pleroma.Web.CommonAPI + alias Pleroma.Web.StreamerView + + describe "follow_relationships_update.json" do + test "shows follower/following count normally" do + other_user = insert(:user) + %{id: following_id} = following = insert(:user) + follower = insert(:user) + + {:ok, _, _, _} = CommonAPI.follow(other_user, following) + {:ok, follower, following, _activity} = CommonAPI.follow(following, follower) + + result = + StreamerView.render( + "follow_relationships_update.json", + %{follower: follower, following: following, state: :test}, + "user:test" + ) + + {:ok, %{"payload" => payload}} = Jason.decode(result) + + {:ok, decoded_payload} = Jason.decode(payload) + + # check the payload updating the user that was followed + assert match?( + %{"follower_count" => 1, "following_count" => 1, "id" => ^following_id}, + decoded_payload["following"] + ) + end + + test "hides follower count for :hide_followers and :hide_followers_count" do + user_attrs = [%{hide_followers: true}, %{hide_followers_count: true}] + + Enum.each(user_attrs, fn attrs -> + other_user = insert(:user) + %{id: following_id} = following = insert(:user, attrs) + follower = insert(:user) + + {:ok, _, _, _} = CommonAPI.follow(other_user, following) + {:ok, follower, following, _activity} = CommonAPI.follow(following, follower) + + result = + StreamerView.render( + "follow_relationships_update.json", + %{follower: follower, following: following, state: :test}, + "user:test" + ) + + {:ok, %{"payload" => payload}} = Jason.decode(result) + + {:ok, decoded_payload} = Jason.decode(payload) + + # check the payload updating the user that was followed + assert match?( + %{"follower_count" => 0, "following_count" => 1, "id" => ^following_id}, + decoded_payload["following"] + ) + end) + end + + test "hides follows count for :hide_follows and :hide_follows_count" do + user_attrs = [%{hide_follows: true}, %{hide_follows_count: true}] + + Enum.each(user_attrs, fn attrs -> + other_user = insert(:user) + %{id: following_id} = following = insert(:user, attrs) + follower = insert(:user) + + {:ok, _, _, _} = CommonAPI.follow(other_user, following) + {:ok, follower, following, _activity} = CommonAPI.follow(following, follower) + + result = + StreamerView.render( + "follow_relationships_update.json", + %{follower: follower, following: following, state: :test}, + "user:test" + ) + + {:ok, %{"payload" => payload}} = Jason.decode(result) + + {:ok, decoded_payload} = Jason.decode(payload) + + # check the payload updating the user that was followed + assert match?( + %{"follower_count" => 1, "following_count" => 0, "id" => ^following_id}, + decoded_payload["following"] + ) + end) + end + end +end From 06e8ece4cc3956d991d48dbd338604b7940d167c Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 7 Aug 2024 12:37:58 -0400 Subject: [PATCH 08/34] Fix CommonAPI.follow/2 which returned users in the reverse order they were provided to the function --- changelog.d/commonapi.skip | 0 lib/pleroma/web/common_api.ex | 2 +- lib/pleroma/web/mastodon_api/mastodon_api.ex | 12 ++++++------ .../activity_pub_controller_test.exs | 4 ++-- .../web/activity_pub/views/user_view_test.exs | 8 ++++---- test/pleroma/web/common_api_test.exs | 10 +++++----- .../mastodon_api/views/account_view_test.exs | 18 +++++++++--------- .../views/notification_view_test.exs | 2 +- test/pleroma/web/views/streamer_view_test.ex | 6 +++--- 9 files changed, 31 insertions(+), 31 deletions(-) create mode 100644 changelog.d/commonapi.skip diff --git a/changelog.d/commonapi.skip b/changelog.d/commonapi.skip new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index 1ed905d6cb..921e414c34 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -130,7 +130,7 @@ defmodule Pleroma.Web.CommonAPI do if activity.data["state"] == "reject" do {:error, :rejected} else - {:ok, follower, followed, activity} + {:ok, followed, follower, activity} end end end diff --git a/lib/pleroma/web/mastodon_api/mastodon_api.ex b/lib/pleroma/web/mastodon_api/mastodon_api.ex index 6dcbfb097e..c9e045d238 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api.ex @@ -18,10 +18,10 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do if not User.following?(follower, followed) do CommonAPI.follow(followed, follower) else - {:ok, follower, followed, nil} + {:ok, followed, follower, nil} end - with {:ok, follower, _followed, _} <- result do + with {:ok, _followed, follower, _} <- result do options = cast_params(params) set_reblogs_visibility(options[:reblogs], result) set_subscription(options[:notify], result) @@ -29,19 +29,19 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do end end - defp set_reblogs_visibility(false, {:ok, follower, followed, _}) do + defp set_reblogs_visibility(false, {:ok, followed, follower, _}) do CommonAPI.hide_reblogs(followed, follower) end - defp set_reblogs_visibility(_, {:ok, follower, followed, _}) do + defp set_reblogs_visibility(_, {:ok, followed, follower, _}) do CommonAPI.show_reblogs(followed, follower) end - defp set_subscription(true, {:ok, follower, followed, _}) do + defp set_subscription(true, {:ok, followed, follower, _}) do User.subscribe(follower, followed) end - defp set_subscription(false, {:ok, follower, followed, _}) do + defp set_subscription(false, {:ok, followed, follower, _}) do User.unsubscribe(follower, followed) end diff --git a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs index 6aae618355..af1a32fed8 100644 --- a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs @@ -1747,7 +1747,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do %{conn: conn} do user = insert(:user, hide_followers: true) other_user = insert(:user) - {:ok, _other_user, user, _activity} = CommonAPI.follow(user, other_user) + {:ok, user, _other_user, _activity} = CommonAPI.follow(user, other_user) result = conn @@ -1843,7 +1843,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do %{conn: conn} do user = insert(:user, hide_follows: true) other_user = insert(:user) - {:ok, user, _other_user, _activity} = CommonAPI.follow(other_user, user) + {:ok, _other_user, user, _activity} = CommonAPI.follow(other_user, user) result = conn diff --git a/test/pleroma/web/activity_pub/views/user_view_test.exs b/test/pleroma/web/activity_pub/views/user_view_test.exs index c94f8a2bc5..651e535ac4 100644 --- a/test/pleroma/web/activity_pub/views/user_view_test.exs +++ b/test/pleroma/web/activity_pub/views/user_view_test.exs @@ -138,7 +138,7 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do test "sets totalItems to zero when followers are hidden" do user = insert(:user) other_user = insert(:user) - {:ok, _other_user, user, _activity} = CommonAPI.follow(user, other_user) + {:ok, user, _other_user, _activity} = CommonAPI.follow(user, other_user) assert %{"totalItems" => 1} = UserView.render("followers.json", %{user: user}) user = Map.merge(user, %{hide_followers_count: true, hide_followers: true}) refute UserView.render("followers.json", %{user: user}) |> Map.has_key?("totalItems") @@ -147,7 +147,7 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do test "sets correct totalItems when followers are hidden but the follower counter is not" do user = insert(:user) other_user = insert(:user) - {:ok, _other_user, user, _activity} = CommonAPI.follow(user, other_user) + {:ok, user, _other_user, _activity} = CommonAPI.follow(user, other_user) assert %{"totalItems" => 1} = UserView.render("followers.json", %{user: user}) user = Map.merge(user, %{hide_followers_count: false, hide_followers: true}) assert %{"totalItems" => 1} = UserView.render("followers.json", %{user: user}) @@ -158,7 +158,7 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do test "sets totalItems to zero when follows are hidden" do user = insert(:user) other_user = insert(:user) - {:ok, user, _other_user, _activity} = CommonAPI.follow(other_user, user) + {:ok, _other_user, user, _activity} = CommonAPI.follow(other_user, user) assert %{"totalItems" => 1} = UserView.render("following.json", %{user: user}) user = Map.merge(user, %{hide_follows_count: true, hide_follows: true}) assert %{"totalItems" => 0} = UserView.render("following.json", %{user: user}) @@ -167,7 +167,7 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do test "sets correct totalItems when follows are hidden but the follow counter is not" do user = insert(:user) other_user = insert(:user) - {:ok, user, _other_user, _activity} = CommonAPI.follow(other_user, user) + {:ok, _other_user, user, _activity} = CommonAPI.follow(other_user, user) assert %{"totalItems" => 1} = UserView.render("following.json", %{user: user}) user = Map.merge(user, %{hide_follows_count: false, hide_follows: true}) assert %{"totalItems" => 1} = UserView.render("following.json", %{user: user}) diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs index 4cdd3cffaa..73230a58c5 100644 --- a/test/pleroma/web/common_api_test.exs +++ b/test/pleroma/web/common_api_test.exs @@ -1420,7 +1420,7 @@ defmodule Pleroma.Web.CommonAPITest do describe "follow/2" do test "directly follows a non-locked local user" do [follower, followed] = insert_pair(:user) - {:ok, follower, followed, _} = CommonAPI.follow(followed, follower) + {:ok, followed, follower, _} = CommonAPI.follow(followed, follower) assert User.following?(follower, followed) end @@ -1429,7 +1429,7 @@ defmodule Pleroma.Web.CommonAPITest do describe "unfollow/2" do test "also unsubscribes a user" do [follower, followed] = insert_pair(:user) - {:ok, follower, followed, _} = CommonAPI.follow(followed, follower) + {:ok, followed, follower, _} = CommonAPI.follow(followed, follower) {:ok, _subscription} = User.subscribe(follower, followed) assert User.subscribed_to?(follower, followed) @@ -1441,7 +1441,7 @@ defmodule Pleroma.Web.CommonAPITest do test "also unpins a user" do [follower, followed] = insert_pair(:user) - {:ok, follower, followed, _} = CommonAPI.follow(followed, follower) + {:ok, followed, follower, _} = CommonAPI.follow(followed, follower) {:ok, _endorsement} = User.endorse(follower, followed) assert User.endorses?(follower, followed) @@ -1455,7 +1455,7 @@ defmodule Pleroma.Web.CommonAPITest do follower = insert(:user) followed = insert(:user, is_locked: true) - assert {:ok, follower, followed, %{id: activity_id, data: %{"state" => "pending"}}} = + assert {:ok, followed, follower, %{id: activity_id, data: %{"state" => "pending"}}} = CommonAPI.follow(followed, follower) assert User.get_follow_state(follower, followed) == :follow_pending @@ -1477,7 +1477,7 @@ defmodule Pleroma.Web.CommonAPITest do follower = insert(:user) followed = insert(:user, is_locked: true, local: false) - assert {:ok, follower, followed, %{id: activity_id, data: %{"state" => "pending"}}} = + assert {:ok, followed, follower, %{id: activity_id, data: %{"state" => "pending"}}} = CommonAPI.follow(followed, follower) assert User.get_follow_state(follower, followed) == :follow_pending diff --git a/test/pleroma/web/mastodon_api/views/account_view_test.exs b/test/pleroma/web/mastodon_api/views/account_view_test.exs index f0711fa0d6..dca64853d1 100644 --- a/test/pleroma/web/mastodon_api/views/account_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/account_view_test.exs @@ -493,7 +493,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do user = insert(:user) other_user = insert(:user, is_locked: true) - {:ok, user, other_user, _} = CommonAPI.follow(other_user, user) + {:ok, other_user, user, _} = CommonAPI.follow(other_user, user) user = User.get_cached_by_id(user.id) other_user = User.get_cached_by_id(other_user.id) @@ -560,8 +560,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do test "shows when follows/followers are hidden" do user = insert(:user, hide_followers: true, hide_follows: true) other_user = insert(:user) - {:ok, user, other_user, _activity} = CommonAPI.follow(other_user, user) - {:ok, _other_user, user, _activity} = CommonAPI.follow(user, other_user) + {:ok, other_user, user, _activity} = CommonAPI.follow(other_user, user) + {:ok, user, _other_user, _activity} = CommonAPI.follow(user, other_user) assert %{ followers_count: 1, @@ -573,11 +573,11 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do test "shows actual follower/following count to the account owner" do user = insert(:user, hide_followers: true, hide_follows: true) other_user = insert(:user) - {:ok, user, other_user, _activity} = CommonAPI.follow(other_user, user) + {:ok, other_user, user, _activity} = CommonAPI.follow(other_user, user) assert User.following?(user, other_user) assert Pleroma.FollowingRelationship.follower_count(other_user) == 1 - {:ok, _other_user, user, _activity} = CommonAPI.follow(user, other_user) + {:ok, user, _other_user, _activity} = CommonAPI.follow(user, other_user) assert %{ followers_count: 1, @@ -696,7 +696,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do assert %{locked: true} = AccountView.render("show.json", %{user: user, for: user}) other_user = insert(:user) - {:ok, _other_user, user, _activity} = CommonAPI.follow(user, other_user) + {:ok, user, _other_user, _activity} = CommonAPI.follow(user, other_user) assert %{locked: true, follow_requests_count: 1} = AccountView.render("show.json", %{user: user, for: user}) @@ -708,7 +708,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do assert %{locked: true} = AccountView.render("show.json", %{user: user, for: user}) other_user = insert(:user) - {:ok, other_user, user, _activity} = CommonAPI.follow(user, other_user) + {:ok, user, other_user, _activity} = CommonAPI.follow(user, other_user) assert %{locked: true, follow_requests_count: 1} = AccountView.render("show.json", %{user: user, for: user}) @@ -725,7 +725,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do assert %{locked: true} = AccountView.render("show.json", %{user: user, for: user}) other_user = insert(:user) - {:ok, other_user, user, _activity} = CommonAPI.follow(user, other_user) + {:ok, user, other_user, _activity} = CommonAPI.follow(user, other_user) assert %{locked: true, follow_requests_count: 1} = AccountView.render("show.json", %{user: user, for: user}) @@ -742,7 +742,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do assert %{locked: true} = AccountView.render("show.json", %{user: user, for: user}) other_user = insert(:user) - {:ok, _other_user, user, _activity} = CommonAPI.follow(user, other_user) + {:ok, user, _other_user, _activity} = CommonAPI.follow(user, other_user) {:ok, user} = User.update_and_set_cache(user, %{is_locked: false}) diff --git a/test/pleroma/web/mastodon_api/views/notification_view_test.exs b/test/pleroma/web/mastodon_api/views/notification_view_test.exs index fae6728713..75ab375aae 100644 --- a/test/pleroma/web/mastodon_api/views/notification_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/notification_view_test.exs @@ -132,7 +132,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do test "Follow notification" do follower = insert(:user) followed = insert(:user) - {:ok, follower, followed, _activity} = CommonAPI.follow(followed, follower) + {:ok, followed, follower, _activity} = CommonAPI.follow(followed, follower) notification = Notification |> Repo.one() |> Repo.preload(:activity) expected = %{ diff --git a/test/pleroma/web/views/streamer_view_test.ex b/test/pleroma/web/views/streamer_view_test.ex index 43a17a43ea..b1c5f52abe 100644 --- a/test/pleroma/web/views/streamer_view_test.ex +++ b/test/pleroma/web/views/streamer_view_test.ex @@ -17,7 +17,7 @@ defmodule Pleroma.Web.StreamerViewTest do follower = insert(:user) {:ok, _, _, _} = CommonAPI.follow(other_user, following) - {:ok, follower, following, _activity} = CommonAPI.follow(following, follower) + {:ok, following, follower, _activity} = CommonAPI.follow(following, follower) result = StreamerView.render( @@ -46,7 +46,7 @@ defmodule Pleroma.Web.StreamerViewTest do follower = insert(:user) {:ok, _, _, _} = CommonAPI.follow(other_user, following) - {:ok, follower, following, _activity} = CommonAPI.follow(following, follower) + {:ok, following, follower, _activity} = CommonAPI.follow(following, follower) result = StreamerView.render( @@ -76,7 +76,7 @@ defmodule Pleroma.Web.StreamerViewTest do follower = insert(:user) {:ok, _, _, _} = CommonAPI.follow(other_user, following) - {:ok, follower, following, _activity} = CommonAPI.follow(following, follower) + {:ok, following, follower, _activity} = CommonAPI.follow(following, follower) result = StreamerView.render( From 721005b3126e0920ab861fcc83195f160b0ee9a0 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 7 Aug 2024 13:00:31 -0400 Subject: [PATCH 09/34] Fix WebPush notifications not generating jobs Dialyzer pointed this one out. The WorkerHelper removal in !4166 was missing this Oban.insert() and tests were not noticing any problems because we mocked the Push.send function instead of executing it and checking for the Oban job. --- lib/pleroma/web/push.ex | 1 + test/pleroma/web/activity_pub/side_effects_test.exs | 13 +++++-------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/push.ex b/lib/pleroma/web/push.ex index d783f776a5..6d777142e9 100644 --- a/lib/pleroma/web/push.ex +++ b/lib/pleroma/web/push.ex @@ -29,5 +29,6 @@ defmodule Pleroma.Web.Push do {:ok, Oban.Job.t()} | {:error, Oban.Job.changeset() | term()} def send(notification) do WebPusherWorker.new(%{"op" => "web_push", "notification_id" => notification.id}) + |> Oban.insert() end end diff --git a/test/pleroma/web/activity_pub/side_effects_test.exs b/test/pleroma/web/activity_pub/side_effects_test.exs index 68922e536a..4a18cab686 100644 --- a/test/pleroma/web/activity_pub/side_effects_test.exs +++ b/test/pleroma/web/activity_pub/side_effects_test.exs @@ -54,20 +54,17 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do [ stream: fn _, _ -> nil end ] - }, - { - Pleroma.Web.Push, - [], - [ - send: fn _ -> nil end - ] } ]) do SideEffects.handle_after_transaction(meta) assert called(Pleroma.Web.Streamer.stream(["user", "user:notification"], notification)) assert called(Pleroma.Web.Streamer.stream(["user", "user:pleroma_chat"], :_)) - assert called(Pleroma.Web.Push.send(notification)) + + assert_enqueued( + worker: "Pleroma.Workers.WebPusherWorker", + args: %{"notification_id" => notification.id, "op" => "web_push"} + ) end end end From 6900040fd075ed0bdee9aab0fe7bf2796d149634 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 7 Aug 2024 13:03:25 -0400 Subject: [PATCH 10/34] Update changelog --- changelog.d/workerhelper.change | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/workerhelper.change b/changelog.d/workerhelper.change index 7a20c4af82..539c9b54f7 100644 --- a/changelog.d/workerhelper.change +++ b/changelog.d/workerhelper.change @@ -1 +1 @@ -Worker configuration is no longer available. This only affects custom max_retries values for Oban queues. +Worker configuration is no longer available. This only affects custom max_retries values for a couple Oban queues. From 253178538410c63be6bd14016b800b42df59bfb6 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 7 Aug 2024 13:07:54 -0400 Subject: [PATCH 11/34] Do not allow committing tests with a .ex extension --- .gitignore | 3 +++ changelog.d/text-extensions.skip | 0 .../views/{streamer_view_test.ex => streamer_view_test.exs} | 6 +++--- 3 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 changelog.d/text-extensions.skip rename test/pleroma/web/views/{streamer_view_test.ex => streamer_view_test.exs} (94%) diff --git a/.gitignore b/.gitignore index c0d0179514..355cea0696 100644 --- a/.gitignore +++ b/.gitignore @@ -63,3 +63,6 @@ pleroma.iml archive-* .gitlab-ci-local + +# Test files should be named *.exs +test/pleroma/**/*.ex diff --git a/changelog.d/text-extensions.skip b/changelog.d/text-extensions.skip new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/pleroma/web/views/streamer_view_test.ex b/test/pleroma/web/views/streamer_view_test.exs similarity index 94% rename from test/pleroma/web/views/streamer_view_test.ex rename to test/pleroma/web/views/streamer_view_test.exs index 43a17a43ea..b1c5f52abe 100644 --- a/test/pleroma/web/views/streamer_view_test.ex +++ b/test/pleroma/web/views/streamer_view_test.exs @@ -17,7 +17,7 @@ defmodule Pleroma.Web.StreamerViewTest do follower = insert(:user) {:ok, _, _, _} = CommonAPI.follow(other_user, following) - {:ok, follower, following, _activity} = CommonAPI.follow(following, follower) + {:ok, following, follower, _activity} = CommonAPI.follow(following, follower) result = StreamerView.render( @@ -46,7 +46,7 @@ defmodule Pleroma.Web.StreamerViewTest do follower = insert(:user) {:ok, _, _, _} = CommonAPI.follow(other_user, following) - {:ok, follower, following, _activity} = CommonAPI.follow(following, follower) + {:ok, following, follower, _activity} = CommonAPI.follow(following, follower) result = StreamerView.render( @@ -76,7 +76,7 @@ defmodule Pleroma.Web.StreamerViewTest do follower = insert(:user) {:ok, _, _, _} = CommonAPI.follow(other_user, following) - {:ok, follower, following, _activity} = CommonAPI.follow(following, follower) + {:ok, following, follower, _activity} = CommonAPI.follow(following, follower) result = StreamerView.render( From d2d07bfe4b9f1567c205a736e6e81e02d13598c1 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 7 Aug 2024 14:04:20 -0400 Subject: [PATCH 12/34] Add test for Follow objects with a cc --- .../object_validators/follow_validation_test.exs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/pleroma/web/activity_pub/object_validators/follow_validation_test.exs b/test/pleroma/web/activity_pub/object_validators/follow_validation_test.exs index 371368e0e0..e684042a1f 100644 --- a/test/pleroma/web/activity_pub/object_validators/follow_validation_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/follow_validation_test.exs @@ -22,5 +22,15 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.FollowValidationTest do test "validates a basic follow object", %{valid_follow: valid_follow} do assert {:ok, _follow, []} = ObjectValidator.validate(valid_follow, []) end + + test "supports a nil cc", %{valid_follow: valid_follow} do + valid_follow_with_nil_cc = Map.put(valid_follow, "cc", nil) + assert {:ok, _follow, []} = ObjectValidator.validate(valid_follow_with_nil_cc, []) + end + + test "supports an empty cc", %{valid_follow: valid_follow} do + valid_follow_with_nil_cc = Map.put(valid_follow, "cc", []) + assert {:ok, _follow, []} = ObjectValidator.validate(valid_follow_with_nil_cc, []) + end end end From 8f15000c0f4c81b27b3a7077092f1ba1b37da205 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 7 Aug 2024 14:38:29 -0400 Subject: [PATCH 13/34] Do not require a cc field when validating an incoming Follow activity The cc field is not required and the code was keeping the cc if it existed on an activity or replacing it with the default of an empty list when casting. If any Follow activity was received with a cc field, it would attempt to keep it. This was noticed in !4208 where we would craft Follow requests with a cc value of nil. --- .../web/activity_pub/object_validators/follow_validator.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/follow_validator.ex b/lib/pleroma/web/activity_pub/object_validators/follow_validator.ex index b3ca5b6910..e4e97bf72a 100644 --- a/lib/pleroma/web/activity_pub/object_validators/follow_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/follow_validator.ex @@ -29,7 +29,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.FollowValidator do defp validate_data(cng) do cng - |> validate_required([:id, :type, :actor, :to, :cc, :object]) + |> validate_required([:id, :type, :actor, :to, :object]) |> validate_inclusion(:type, ["Follow"]) |> validate_inclusion(:state, ~w{pending reject accept}) |> validate_actor_presence() From b25f67372b0f425ae428a1b6b18156553c995dc2 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 7 Aug 2024 14:42:44 -0400 Subject: [PATCH 14/34] Improve the FollowValidator --- changelog.d/follow-validator.fix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/follow-validator.fix diff --git a/changelog.d/follow-validator.fix b/changelog.d/follow-validator.fix new file mode 100644 index 0000000000..d49932b7b3 --- /dev/null +++ b/changelog.d/follow-validator.fix @@ -0,0 +1 @@ +Improve the FollowValidator to successfully incoming activities with an errant cc field. From fcda1b5e2aeb166dfd84ff17d1840fcfbd6f1490 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 7 Aug 2024 14:46:39 -0400 Subject: [PATCH 15/34] Improve variable name --- .../activity_pub/object_validators/follow_validation_test.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/pleroma/web/activity_pub/object_validators/follow_validation_test.exs b/test/pleroma/web/activity_pub/object_validators/follow_validation_test.exs index e684042a1f..acf6e8d8fe 100644 --- a/test/pleroma/web/activity_pub/object_validators/follow_validation_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/follow_validation_test.exs @@ -29,8 +29,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.FollowValidationTest do end test "supports an empty cc", %{valid_follow: valid_follow} do - valid_follow_with_nil_cc = Map.put(valid_follow, "cc", []) - assert {:ok, _follow, []} = ObjectValidator.validate(valid_follow_with_nil_cc, []) + valid_follow_with_empty_cc = Map.put(valid_follow, "cc", []) + assert {:ok, _follow, []} = ObjectValidator.validate(valid_follow_with_empty_cc, []) end end end From 526a57ff9fe2be729c6e4d95f3eb5991047c41a2 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 7 Aug 2024 21:51:22 -0400 Subject: [PATCH 16/34] Remove validation for cc fields on Follow Accept/Reject --- .../activity_pub/object_validators/accept_reject_validator.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/accept_reject_validator.ex b/lib/pleroma/web/activity_pub/object_validators/accept_reject_validator.ex index d611da0516..03ab83347f 100644 --- a/lib/pleroma/web/activity_pub/object_validators/accept_reject_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/accept_reject_validator.ex @@ -29,7 +29,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AcceptRejectValidator do defp validate_data(cng) do cng - |> validate_required([:id, :type, :actor, :to, :cc, :object]) + |> validate_required([:id, :type, :actor, :to, :object]) |> validate_inclusion(:type, ["Accept", "Reject"]) |> validate_actor_presence() |> validate_object_presence(allowed_types: ["Follow"]) From ca934b744f9c6095315421990975c1e9a7b9cd85 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 7 Aug 2024 21:51:43 -0400 Subject: [PATCH 17/34] Remove validation for cc fields on Blocks --- .../web/activity_pub/object_validators/block_validator.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/block_validator.ex b/lib/pleroma/web/activity_pub/object_validators/block_validator.ex index 0de87a27ef..98340545cc 100644 --- a/lib/pleroma/web/activity_pub/object_validators/block_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/block_validator.ex @@ -29,7 +29,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator do defp validate_data(cng) do cng - |> validate_required([:id, :type, :actor, :to, :cc, :object]) + |> validate_required([:id, :type, :actor, :to, :object]) |> validate_inclusion(:type, ["Block"]) |> CommonValidations.validate_actor_presence() |> CommonValidations.validate_actor_presence(field_name: :object) From a0af6cba09f6c85f524d31f7d1a68948b77d9e53 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 8 Aug 2024 14:12:44 -0400 Subject: [PATCH 18/34] Added MRF.QuietReply which prevents replies to public posts from being published to the timelines --- changelog.d/mrf-quietreply.add | 1 + .../web/activity_pub/mrf/quiet_reply.ex | 55 ++++++++++ .../web/activity_pub/mrf/quiet_reply_test.exs | 103 ++++++++++++++++++ test/support/factory.ex | 16 ++- 4 files changed, 169 insertions(+), 6 deletions(-) create mode 100644 changelog.d/mrf-quietreply.add create mode 100644 lib/pleroma/web/activity_pub/mrf/quiet_reply.ex create mode 100644 test/pleroma/web/activity_pub/mrf/quiet_reply_test.exs diff --git a/changelog.d/mrf-quietreply.add b/changelog.d/mrf-quietreply.add new file mode 100644 index 0000000000..4ed20bce60 --- /dev/null +++ b/changelog.d/mrf-quietreply.add @@ -0,0 +1 @@ +Added MRF.QuietReply which prevents replies to public posts from being published to the timelines diff --git a/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex b/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex new file mode 100644 index 0000000000..8a9b2beb8f --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex @@ -0,0 +1,55 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2023 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.QuietReply do + require Pleroma.Constants + + alias Pleroma.User + + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + + @impl true + def history_awareness, do: :auto + + @impl true + def filter( + %{ + "type" => "Create", + "object" => %{ + "actor" => actor, + "type" => "Note", + "to" => to, + "cc" => cc, + "inReplyTo" => in_reply_to + } + } = object + ) do + with true <- is_binary(in_reply_to), + false <- match?([], cc), + %User{follower_address: followers_collection, local: true} <- + User.get_by_ap_id(actor) do + updated_to = + to + |> Kernel.++([followers_collection]) + |> Kernel.--([Pleroma.Constants.as_public()]) + + updated_cc = [Pleroma.Constants.as_public()] + + updated_object = + object + |> put_in(["object", "to"], updated_to) + |> put_in(["object", "cc"], updated_cc) + + {:ok, updated_object} + else + _ -> {:ok, object} + end + end + + @impl true + def filter(object), do: {:ok, object} + + @impl true + def describe, do: {:ok, %{}} +end diff --git a/test/pleroma/web/activity_pub/mrf/quiet_reply_test.exs b/test/pleroma/web/activity_pub/mrf/quiet_reply_test.exs new file mode 100644 index 0000000000..e2b3ce4391 --- /dev/null +++ b/test/pleroma/web/activity_pub/mrf/quiet_reply_test.exs @@ -0,0 +1,103 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.QuietReplyTest do + use Pleroma.DataCase + import Pleroma.Factory + + require Pleroma.Constants + + alias Pleroma.Object + alias Pleroma.Web.ActivityPub.MRF.QuietReply + alias Pleroma.Web.CommonAPI + + test "replying to public post is forced to be quiet" do + batman = insert(:user, nickname: "batman") + robin = insert(:user, nickname: "robin") + + {:ok, post} = CommonAPI.post(batman, %{status: "To the Batmobile!"}) + + reply = %{ + "type" => "Create", + "actor" => robin.ap_id, + "object" => %{ + "type" => "Note", + "actor" => robin.ap_id, + "content" => "@batman Wait up, I forgot my spandex!", + "to" => [ + batman.ap_id, + Pleroma.Constants.as_public() + ], + "cc" => [robin.follower_address], + "inReplyTo" => Object.normalize(post).data["id"] + } + } + + expected_to = [batman.ap_id, robin.follower_address] + expected_cc = [Pleroma.Constants.as_public()] + + assert {:ok, filtered} = QuietReply.filter(reply) + + assert expected_to == filtered["object"]["to"] + assert expected_cc == filtered["object"]["cc"] + end + + test "replying direct is unmodified" do + batman = insert(:user, nickname: "batman") + robin = insert(:user, nickname: "robin") + + {:ok, post} = CommonAPI.post(batman, %{status: "To the Batmobile!"}) + + reply = %{ + "type" => "Create", + "actor" => robin.ap_id, + "object" => %{ + "type" => "Note", + "actor" => robin.ap_id, + "content" => "@batman Wait up, I forgot my spandex!", + "to" => [batman.ap_id], + "cc" => [], + "inReplyTo" => Object.normalize(post).data["id"] + } + } + + assert {:ok, filtered} = QuietReply.filter(reply) + + assert match?(^filtered, reply) + end + + test "replying followers-only is unmodified" do + batman = insert(:user, nickname: "batman") + robin = insert(:user, nickname: "robin") + + {:ok, post} = CommonAPI.post(batman, %{status: "To the Batmobile!"}) + + reply = %{ + "type" => "Create", + "actor" => robin.ap_id, + "object" => %{ + "type" => "Note", + "actor" => robin.ap_id, + "content" => "@batman Wait up, I forgot my spandex!", + "to" => [batman.ap_id, robin.follower_address], + "cc" => [], + "inReplyTo" => Object.normalize(post).data["id"] + } + } + + assert {:ok, filtered} = QuietReply.filter(reply) + + assert match?(^filtered, reply) + end + + test "non-reply posts are unmodified" do + batman = insert(:user, nickname: "batman") + + {:ok, post} = CommonAPI.post(batman, %{status: "To the Batmobile!"}) + + assert {:ok, filtered} = QuietReply.filter(post) + + assert match?(^filtered, post) + end +end diff --git a/test/support/factory.ex b/test/support/factory.ex index fb26f41626..64669fdf17 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -53,6 +53,15 @@ defmodule Pleroma.Factory do keys: pem } + attrs = Map.delete(attrs, :domain) + + user + |> Map.put(:raw_bio, user.bio) + |> merge_attributes(attrs) + |> user_urls(attrs) + end + + defp user_urls(user, attrs) do urls = if attrs[:local] == false do base_domain = attrs[:domain] || Enum.random(["domain1.com", "domain2.com", "domain3.com"]) @@ -75,12 +84,7 @@ defmodule Pleroma.Factory do } end - attrs = Map.delete(attrs, :domain) - - user - |> Map.put(:raw_bio, user.bio) - |> Map.merge(urls) - |> merge_attributes(attrs) + Map.merge(user, urls) end def user_relationship_factory(attrs \\ %{}) do From d6cc6aff9b617a50679fe99d1533d2b3d118b656 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 8 Aug 2024 14:22:25 -0400 Subject: [PATCH 19/34] Unintended commit --- test/support/factory.ex | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/test/support/factory.ex b/test/support/factory.ex index 64669fdf17..fb26f41626 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -53,15 +53,6 @@ defmodule Pleroma.Factory do keys: pem } - attrs = Map.delete(attrs, :domain) - - user - |> Map.put(:raw_bio, user.bio) - |> merge_attributes(attrs) - |> user_urls(attrs) - end - - defp user_urls(user, attrs) do urls = if attrs[:local] == false do base_domain = attrs[:domain] || Enum.random(["domain1.com", "domain2.com", "domain3.com"]) @@ -84,7 +75,12 @@ defmodule Pleroma.Factory do } end - Map.merge(user, urls) + attrs = Map.delete(attrs, :domain) + + user + |> Map.put(:raw_bio, user.bio) + |> Map.merge(urls) + |> merge_attributes(attrs) end def user_relationship_factory(attrs \\ %{}) do From 5a134a46f7a59eba131c9e484d49e09394e341a8 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 8 Aug 2024 15:23:59 -0400 Subject: [PATCH 20/34] We must change to/cc in the activity and inner object --- lib/pleroma/web/activity_pub/mrf/quiet_reply.ex | 6 ++++-- .../pleroma/web/activity_pub/mrf/quiet_reply_test.exs | 11 +++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex b/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex index 8a9b2beb8f..52d92f2677 100644 --- a/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex +++ b/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex @@ -16,11 +16,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.QuietReply do def filter( %{ "type" => "Create", + "to" => to, + "cc" => cc, "object" => %{ "actor" => actor, "type" => "Note", - "to" => to, - "cc" => cc, "inReplyTo" => in_reply_to } } = object @@ -38,6 +38,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.QuietReply do updated_object = object + |> Map.put("to", updated_to) + |> Map.put("cc", updated_cc) |> put_in(["object", "to"], updated_to) |> put_in(["object", "cc"], updated_cc) diff --git a/test/pleroma/web/activity_pub/mrf/quiet_reply_test.exs b/test/pleroma/web/activity_pub/mrf/quiet_reply_test.exs index e2b3ce4391..35e321fd18 100644 --- a/test/pleroma/web/activity_pub/mrf/quiet_reply_test.exs +++ b/test/pleroma/web/activity_pub/mrf/quiet_reply_test.exs @@ -21,6 +21,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.QuietReplyTest do reply = %{ "type" => "Create", "actor" => robin.ap_id, + "to" => [ + batman.ap_id, + Pleroma.Constants.as_public() + ], + "cc" => [robin.follower_address], "object" => %{ "type" => "Note", "actor" => robin.ap_id, @@ -39,6 +44,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.QuietReplyTest do assert {:ok, filtered} = QuietReply.filter(reply) + assert expected_to == filtered["to"] + assert expected_cc == filtered["cc"] assert expected_to == filtered["object"]["to"] assert expected_cc == filtered["object"]["cc"] end @@ -52,6 +59,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.QuietReplyTest do reply = %{ "type" => "Create", "actor" => robin.ap_id, + "to" => [batman.ap_id], + "cc" => [], "object" => %{ "type" => "Note", "actor" => robin.ap_id, @@ -76,6 +85,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.QuietReplyTest do reply = %{ "type" => "Create", "actor" => robin.ap_id, + "to" => [batman.ap_id, robin.follower_address], + "cc" => [], "object" => %{ "type" => "Note", "actor" => robin.ap_id, From 6e53e94bdab91044ffa24e9d6585fdde1727b0f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Mon, 12 Aug 2024 12:23:38 +0200 Subject: [PATCH 21/34] Remove stub for /api/v1/accounts/:id/identity_proofs (deprecated by Mastodon 3.5.0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- changelog.d/identity-proofs.remove | 1 + .../API/differences_in_mastoapi_responses.md | 6 ------ .../web/api_spec/operations/account_operation.ex | 16 ---------------- .../controllers/account_controller.ex | 5 +---- lib/pleroma/web/router.ex | 1 - .../mastodon_api_controller_test.exs | 9 --------- 6 files changed, 2 insertions(+), 36 deletions(-) create mode 100644 changelog.d/identity-proofs.remove diff --git a/changelog.d/identity-proofs.remove b/changelog.d/identity-proofs.remove new file mode 100644 index 0000000000..efe1c34f57 --- /dev/null +++ b/changelog.d/identity-proofs.remove @@ -0,0 +1 @@ +Remove stub for /api/v1/accounts/:id/identity_proofs (deprecated by Mastodon 3.5.0) \ No newline at end of file diff --git a/docs/development/API/differences_in_mastoapi_responses.md b/docs/development/API/differences_in_mastoapi_responses.md index e3b6a3c779..41464e8021 100644 --- a/docs/development/API/differences_in_mastoapi_responses.md +++ b/docs/development/API/differences_in_mastoapi_responses.md @@ -510,12 +510,6 @@ Pleroma is generally compatible with the Mastodon 2.7.2 API, but some newer feat - `GET /api/v1/trends`: Returns an empty array, `[]` -### Identity proofs - -*Added in Mastodon 2.8.0* - -- `GET /api/v1/identity_proofs`: Returns an empty array, `[]` - ### Featured tags *Added in Mastodon 3.0.0* diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex index 85f02166f0..d9614bc481 100644 --- a/lib/pleroma/web/api_spec/operations/account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/account_operation.ex @@ -498,22 +498,6 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do } end - def identity_proofs_operation do - %Operation{ - tags: ["Retrieve account information"], - summary: "Identity proofs", - operationId: "AccountController.identity_proofs", - # Validators complains about unused path params otherwise - parameters: [ - %Reference{"$ref": "#/components/parameters/accountIdOrNickname"} - ], - description: "Not implemented", - responses: %{ - 200 => empty_array_response() - } - } - end - def familiar_followers_operation do %Operation{ tags: ["Retrieve account information"], diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 80ab95a572..6a6911727e 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -51,7 +51,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do plug( OAuthScopesPlug, %{scopes: ["read:accounts"]} - when action in [:verify_credentials, :endorsements, :identity_proofs] + when action in [:verify_credentials, :endorsements] ) plug( @@ -660,7 +660,4 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do defp get_familiar_followers(user, current_user) do User.get_familiar_followers(user, current_user) end - - @doc "GET /api/v1/identity_proofs" - def identity_proofs(conn, params), do: MastodonAPIController.empty_array(conn, params) end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index fc40a11432..6492e38619 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -648,7 +648,6 @@ defmodule Pleroma.Web.Router do get("/accounts/relationships", AccountController, :relationships) get("/accounts/familiar_followers", AccountController, :familiar_followers) get("/accounts/:id/lists", AccountController, :lists) - get("/accounts/:id/identity_proofs", AccountController, :identity_proofs) get("/endorsements", AccountController, :endorsements) get("/blocks", AccountController, :blocks) get("/mutes", AccountController, :mutes) diff --git a/test/pleroma/web/mastodon_api/mastodon_api_controller_test.exs b/test/pleroma/web/mastodon_api/mastodon_api_controller_test.exs index 622c5e3d5f..8e7fe40137 100644 --- a/test/pleroma/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/pleroma/web/mastodon_api/mastodon_api_controller_test.exs @@ -6,15 +6,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do use Pleroma.Web.ConnCase, async: true describe "empty_array/2 (stubs)" do - test "GET /api/v1/accounts/:id/identity_proofs" do - %{user: user, conn: conn} = oauth_access(["read:accounts"]) - - assert [] == - conn - |> get("/api/v1/accounts/#{user.id}/identity_proofs") - |> json_response(200) - end - test "GET /api/v1/endorsements" do %{conn: conn} = oauth_access(["read:accounts"]) From 29f7ab71169482a68c24729c6cf7607d43d8224d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Mon, 12 Aug 2024 12:25:12 +0200 Subject: [PATCH 22/34] Update test as /api/v1/endorsements is not a stub MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- .../web/mastodon_api/mastodon_api_controller_test.exs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/test/pleroma/web/mastodon_api/mastodon_api_controller_test.exs b/test/pleroma/web/mastodon_api/mastodon_api_controller_test.exs index 8e7fe40137..97ae3d37b1 100644 --- a/test/pleroma/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/pleroma/web/mastodon_api/mastodon_api_controller_test.exs @@ -6,15 +6,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do use Pleroma.Web.ConnCase, async: true describe "empty_array/2 (stubs)" do - test "GET /api/v1/endorsements" do - %{conn: conn} = oauth_access(["read:accounts"]) - - assert [] == - conn - |> get("/api/v1/endorsements") - |> json_response(200) - end - test "GET /api/v1/trends", %{conn: conn} do assert [] == conn From c1c0f0b1443aa486f7867b9fa02cc41dbbc74261 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Mon, 12 Aug 2024 12:33:20 +0200 Subject: [PATCH 23/34] Mark `/api/v1/pleroma/accounts/:id/subscribe`/`unsubscribe` as deprecated MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- changelog.d/deprecate-subscribe.change | 1 + docs/development/API/pleroma_api.md | 6 ++++++ .../web/api_spec/operations/pleroma_account_operation.ex | 8 ++++++-- 3 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 changelog.d/deprecate-subscribe.change diff --git a/changelog.d/deprecate-subscribe.change b/changelog.d/deprecate-subscribe.change new file mode 100644 index 0000000000..bd7e8aec75 --- /dev/null +++ b/changelog.d/deprecate-subscribe.change @@ -0,0 +1 @@ +Deprecate `/api/v1/pleroma/accounts/:id/subscribe`/`unsubscribe` \ No newline at end of file diff --git a/docs/development/API/pleroma_api.md b/docs/development/API/pleroma_api.md index 57d333ffe1..000d7d27d9 100644 --- a/docs/development/API/pleroma_api.md +++ b/docs/development/API/pleroma_api.md @@ -145,6 +145,9 @@ See [Admin-API](admin_api.md) ## `/api/v1/pleroma/accounts/:id/subscribe` ### Subscribe to receive notifications for all statuses posted by a user + +Deprecated. `notify` parameter in `POST /api/v1/accounts/:id/follow` should be used instead. + * Method `POST` * Authentication: required * Params: @@ -171,6 +174,9 @@ See [Admin-API](admin_api.md) ## `/api/v1/pleroma/accounts/:id/unsubscribe` ### Unsubscribe to stop receiving notifications from user statuses + +Deprecated. `notify` parameter in `POST /api/v1/accounts/:id/follow` should be used instead. + * Method `POST` * Authentication: required * Params: diff --git a/lib/pleroma/web/api_spec/operations/pleroma_account_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_account_operation.ex index 7340653fba..b8b37d7cf3 100644 --- a/lib/pleroma/web/api_spec/operations/pleroma_account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/pleroma_account_operation.ex @@ -85,9 +85,11 @@ defmodule Pleroma.Web.ApiSpec.PleromaAccountOperation do def subscribe_operation do %Operation{ + deprecated: true, tags: ["Account actions"], summary: "Subscribe", - description: "Receive notifications for all statuses posted by the account.", + description: + "Receive notifications for all statuses posted by the account. Deprecated, use `notify: true` in follow operation instead.", operationId: "PleromaAPI.AccountController.subscribe", parameters: [id_param()], security: [%{"oAuth" => ["follow", "write:follows"]}], @@ -100,9 +102,11 @@ defmodule Pleroma.Web.ApiSpec.PleromaAccountOperation do def unsubscribe_operation do %Operation{ + deprecated: true, tags: ["Account actions"], summary: "Unsubscribe", - description: "Stop receiving notifications for all statuses posted by the account.", + description: + "Stop receiving notifications for all statuses posted by the account. Deprecated, use `notify: false` in follow operation instead.", operationId: "PleromaAPI.AccountController.unsubscribe", parameters: [id_param()], security: [%{"oAuth" => ["follow", "write:follows"]}], From 34715b8581daca64b474438b590730ee53e86017 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Mon, 12 Aug 2024 13:38:33 +0200 Subject: [PATCH 24/34] remove unused alias MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/mastodon_api/controllers/account_controller.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 6a6911727e..54d46c86b8 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -22,7 +22,6 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do alias Pleroma.Web.CommonAPI alias Pleroma.Web.MastodonAPI.ListView alias Pleroma.Web.MastodonAPI.MastodonAPI - alias Pleroma.Web.MastodonAPI.MastodonAPIController alias Pleroma.Web.MastodonAPI.StatusView alias Pleroma.Web.OAuth.OAuthController alias Pleroma.Web.Plugs.OAuthScopesPlug From c116024bba029438a4835b1158de373042a937ee Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 12 Aug 2024 15:28:33 -0400 Subject: [PATCH 25/34] Fix Swoosh Mailgun support --- changelog.d/mailgun.fix | 1 + mix.exs | 1 + mix.lock | 1 + 3 files changed, 3 insertions(+) create mode 100644 changelog.d/mailgun.fix diff --git a/changelog.d/mailgun.fix b/changelog.d/mailgun.fix new file mode 100644 index 0000000000..8555887526 --- /dev/null +++ b/changelog.d/mailgun.fix @@ -0,0 +1 @@ +The Swoosh email adapter for Mailgun was missing a new dependency on :multipart diff --git a/mix.exs b/mix.exs index 69e52e5264..e3c8559baf 100644 --- a/mix.exs +++ b/mix.exs @@ -202,6 +202,7 @@ defmodule Pleroma.Mixfile do {:bandit, "~> 1.5.2"}, {:websock_adapter, "~> 0.5.6"}, {:oban_live_dashboard, "~> 0.1.1"}, + {:multipart, "~> 0.4.0", optional: true}, ## dev & test {:phoenix_live_reload, "~> 1.3.3", only: :dev}, diff --git a/mix.lock b/mix.lock index 61ede9e5eb..37ac1768b4 100644 --- a/mix.lock +++ b/mix.lock @@ -84,6 +84,7 @@ "mock": {:hex, :mock, "0.3.8", "7046a306b71db2488ef54395eeb74df0a7f335a7caca4a3d3875d1fc81c884dd", [:mix], [{:meck, "~> 0.9.2", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm", "7fa82364c97617d79bb7d15571193fc0c4fe5afd0c932cef09426b3ee6fe2022"}, "mogrify": {:hex, :mogrify, "0.8.0", "3506f3ca3f7b95a155f3b4ef803b5db176f5a0633723e3fe85e0d6399e3b11c8", [:mix], [], "hexpm", "2278d245f07056ea3b586e98801e933695147066fa4cf563f552c1b4f0ff8ad9"}, "mox": {:hex, :mox, "1.1.0", "0f5e399649ce9ab7602f72e718305c0f9cdc351190f72844599545e4996af73c", [:mix], [], "hexpm", "d44474c50be02d5b72131070281a5d3895c0e7a95c780e90bc0cfe712f633a13"}, + "multipart": {:hex, :multipart, "0.4.0", "634880a2148d4555d050963373d0e3bbb44a55b2badd87fa8623166172e9cda0", [:mix], [{:mime, "~> 1.2 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}], "hexpm", "3c5604bc2fb17b3137e5d2abdf5dacc2647e60c5cc6634b102cf1aef75a06f0a"}, "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, "nimble_parsec": {:hex, :nimble_parsec, "0.6.0", "32111b3bf39137144abd7ba1cce0914533b2d16ef35e8abc5ec8be6122944263", [:mix], [], "hexpm", "27eac315a94909d4dc68bc07a4a83e06c8379237c5ea528a9acff4ca1c873c52"}, "nimble_pool": {:hex, :nimble_pool, "0.2.6", "91f2f4c357da4c4a0a548286c84a3a28004f68f05609b4534526871a22053cde", [:mix], [], "hexpm", "1c715055095d3f2705c4e236c18b618420a35490da94149ff8b580a2144f653f"}, From b76dfd8146116320cbbc00d39f29c7232f5dcfab Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 12 Aug 2024 15:36:01 -0400 Subject: [PATCH 26/34] Revert accidental removal of test unrelated to identity proofs --- .../web/mastodon_api/mastodon_api_controller_test.exs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/pleroma/web/mastodon_api/mastodon_api_controller_test.exs b/test/pleroma/web/mastodon_api/mastodon_api_controller_test.exs index 97ae3d37b1..8e7fe40137 100644 --- a/test/pleroma/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/pleroma/web/mastodon_api/mastodon_api_controller_test.exs @@ -6,6 +6,15 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do use Pleroma.Web.ConnCase, async: true describe "empty_array/2 (stubs)" do + test "GET /api/v1/endorsements" do + %{conn: conn} = oauth_access(["read:accounts"]) + + assert [] == + conn + |> get("/api/v1/endorsements") + |> json_response(200) + end + test "GET /api/v1/trends", %{conn: conn} do assert [] == conn From 471f5c81fd279ccbc0cee7196573485b8608786b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 12 Aug 2024 16:07:45 -0400 Subject: [PATCH 27/34] Add module documentation --- lib/pleroma/web/activity_pub/mrf/quiet_reply.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex b/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex index 52d92f2677..ae5e2cdc70 100644 --- a/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex +++ b/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex @@ -3,6 +3,9 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.QuietReply do + @moduledoc """ + QuietReply alters the scope of activities from local users when replying by enforcing them to be "Unlisted" or "Quiet Public". This delivers the activity to all the expected recipients and instances, but it will not be published in the Federated / The Whole Known Network timelines. It will still be published to the Home timelines of the user's followers and visible to anyone who opens the thread. + """ require Pleroma.Constants alias Pleroma.User From 488c4b8b983f99d036e3f3bf67dec782f9959319 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 15 Mar 2024 15:10:45 -0400 Subject: [PATCH 28/34] MRF.FODirectReply Force replies to followers-only posts to always be direct --- changelog.d/mrf-fodirectreply.add | 1 + .../web/activity_pub/mrf/fo_direct_reply.ex | 65 ++++++++++++++ .../activity_pub/mrf/fo_direct_reply_test.exs | 87 +++++++++++++++++++ 3 files changed, 153 insertions(+) create mode 100644 changelog.d/mrf-fodirectreply.add create mode 100644 lib/pleroma/web/activity_pub/mrf/fo_direct_reply.ex create mode 100644 test/pleroma/web/activity_pub/mrf/fo_direct_reply_test.exs diff --git a/changelog.d/mrf-fodirectreply.add b/changelog.d/mrf-fodirectreply.add new file mode 100644 index 0000000000..c144e1bc0b --- /dev/null +++ b/changelog.d/mrf-fodirectreply.add @@ -0,0 +1 @@ +Added MRF.FODirectReply which changes replies to followers-only posts to be direct diff --git a/lib/pleroma/web/activity_pub/mrf/fo_direct_reply.ex b/lib/pleroma/web/activity_pub/mrf/fo_direct_reply.ex new file mode 100644 index 0000000000..4eb97afa84 --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/fo_direct_reply.ex @@ -0,0 +1,65 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2024 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.FODirectReply do + @moduledoc """ + FODirectReply alters the scope of replies to activities which are Followers Only to be Direct. The purpose of this policy is to prevent broken threads for followers of the reply author because their response was to a user that they are not also following. + """ + + alias Pleroma.User + + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + + @impl true + def filter( + %{ + "type" => "Create", + "to" => to, + "object" => %{ + "actor" => actor, + "type" => "Note", + "inReplyTo" => in_reply_to + } + } = activity + ) do + with true <- is_binary(in_reply_to), + %User{follower_address: followers_collection, local: true} <- User.get_by_ap_id(actor), + true <- followers_only?(in_reply_to) do + direct_to = to -- [followers_collection] + + updated_activity = + activity + |> Map.put("cc", []) + |> Map.put("to", direct_to) + |> Map.put("directMessage", true) + |> put_in(["object", "cc"], []) + |> put_in(["object", "to"], direct_to) + + {:ok, updated_activity} + else + _ -> {:ok, activity} + end + end + + @impl true + def filter(activity), do: {:ok, activity} + + @impl true + def describe, do: {:ok, %{}} + + defp followers_only?(parent_ap_id) do + with %Pleroma.Object{} = object <- Pleroma.Object.get_by_ap_id(parent_ap_id), + object_data <- Map.get(object, :data), + %Pleroma.User{} = user <- User.get_cached_by_ap_id(object_data["actor"]) do + if user.follower_address in object_data["to"] do + true + else + false + end + else + _ -> + false + end + end +end diff --git a/test/pleroma/web/activity_pub/mrf/fo_direct_reply_test.exs b/test/pleroma/web/activity_pub/mrf/fo_direct_reply_test.exs new file mode 100644 index 0000000000..7afc83ffca --- /dev/null +++ b/test/pleroma/web/activity_pub/mrf/fo_direct_reply_test.exs @@ -0,0 +1,87 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.FODirectReplyTest do + use Pleroma.DataCase + import Pleroma.Factory + + require Pleroma.Constants + + alias Pleroma.Object + alias Pleroma.Web.ActivityPub.MRF.FODirectReply + alias Pleroma.Web.CommonAPI + + test "replying to followers-only/private is changed to direct" do + batman = insert(:user, nickname: "batman") + robin = insert(:user, nickname: "robin") + + {:ok, post} = + CommonAPI.post(batman, %{ + status: "Has anyone seen Selina Kyle's latest selfies?", + visibility: "private" + }) + + reply = %{ + "type" => "Create", + "actor" => robin.ap_id, + "to" => [batman.ap_id, robin.follower_address], + "cc" => [], + "object" => %{ + "type" => "Note", + "actor" => robin.ap_id, + "content" => "@batman 🤤 ❤️ 🐈‍⬛", + "to" => [batman.ap_id, robin.follower_address], + "cc" => [], + "inReplyTo" => Object.normalize(post).data["id"] + } + } + + expected_to = [batman.ap_id] + expected_cc = [] + + assert {:ok, filtered} = FODirectReply.filter(reply) + + assert expected_to == filtered["to"] + assert expected_cc == filtered["cc"] + assert expected_to == filtered["object"]["to"] + assert expected_cc == filtered["object"]["cc"] + end + + test "replies to public posts are unmodified" do + batman = insert(:user, nickname: "batman") + robin = insert(:user, nickname: "robin") + + {:ok, post} = + CommonAPI.post(batman, %{status: "Has anyone seen Selina Kyle's latest selfies?"}) + + reply = %{ + "type" => "Create", + "actor" => robin.ap_id, + "to" => [batman.ap_id, robin.follower_address], + "cc" => [], + "object" => %{ + "type" => "Note", + "actor" => robin.ap_id, + "content" => "@batman 🤤 ❤️ 🐈<200d>⬛", + "to" => [batman.ap_id, robin.follower_address], + "cc" => [], + "inReplyTo" => Object.normalize(post).data["id"] + } + } + + assert {:ok, filtered} = FODirectReply.filter(reply) + + assert match?(^filtered, reply) + end + + test "non-reply posts are unmodified" do + batman = insert(:user, nickname: "batman") + + {:ok, post} = CommonAPI.post(batman, %{status: "To the Batmobile!"}) + + assert {:ok, filtered} = FODirectReply.filter(post) + + assert match?(^filtered, post) + end +end From c0ca7a4ecab76607227221fa96df1ccb9313131e Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 8 Aug 2024 14:41:28 -0400 Subject: [PATCH 29/34] User Factory: include the nickname in the generated URLs --- changelog.d/user-factory.skip | 0 test/support/factory.ex | 32 +++++++++++++++++--------------- 2 files changed, 17 insertions(+), 15 deletions(-) create mode 100644 changelog.d/user-factory.skip diff --git a/changelog.d/user-factory.skip b/changelog.d/user-factory.skip new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/support/factory.ex b/test/support/factory.ex index fb26f41626..8f1c6faf97 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -53,6 +53,13 @@ defmodule Pleroma.Factory do keys: pem } + user + |> Map.put(:raw_bio, user.bio) + |> merge_attributes(Map.delete(attrs, :domain)) + |> make_user_urls(attrs) + end + + defp make_user_urls(user, attrs) do urls = if attrs[:local] == false do base_domain = attrs[:domain] || Enum.random(["domain1.com", "domain2.com", "domain3.com"]) @@ -60,27 +67,22 @@ defmodule Pleroma.Factory do ap_id = "https://#{base_domain}/users/#{user.nickname}" %{ - ap_id: ap_id, - follower_address: ap_id <> "/followers", - following_address: ap_id <> "/following", - featured_address: ap_id <> "/collections/featured", - inbox: "https://#{base_domain}/inbox" + ap_id: attrs[:ap_id] || ap_id, + follower_address: attrs[:follower_address] || ap_id <> "/followers", + following_address: attrs[:following_address] || ap_id <> "/following", + featured_address: attrs[:featured_address] || ap_id <> "/collections/featured", + inbox: attrs[:inbox] || "https://#{base_domain}/inbox" } else %{ - ap_id: User.ap_id(user), - follower_address: User.ap_followers(user), - following_address: User.ap_following(user), - featured_address: User.ap_featured_collection(user) + ap_id: attrs[:ap_id] || User.ap_id(user), + follower_address: attrs[:follower_address] || User.ap_followers(user), + following_address: attrs[:following_address] || User.ap_following(user), + featured_address: attrs[:featured_address] || User.ap_featured_collection(user) } end - attrs = Map.delete(attrs, :domain) - - user - |> Map.put(:raw_bio, user.bio) - |> Map.merge(urls) - |> merge_attributes(attrs) + Map.merge(user, urls) end def user_relationship_factory(attrs \\ %{}) do From 8c978727c210da8558b15ee515b3b8824ff3a912 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 12 Aug 2024 20:10:09 -0400 Subject: [PATCH 30/34] MRF.QuietReply: add test for replies to unlisted posts --- .../web/activity_pub/mrf/quiet_reply_test.exs | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/test/pleroma/web/activity_pub/mrf/quiet_reply_test.exs b/test/pleroma/web/activity_pub/mrf/quiet_reply_test.exs index 35e321fd18..79e64d650c 100644 --- a/test/pleroma/web/activity_pub/mrf/quiet_reply_test.exs +++ b/test/pleroma/web/activity_pub/mrf/quiet_reply_test.exs @@ -50,6 +50,32 @@ defmodule Pleroma.Web.ActivityPub.MRF.QuietReplyTest do assert expected_cc == filtered["object"]["cc"] end + test "replying to unlisted post is unmodified" do + batman = insert(:user, nickname: "batman") + robin = insert(:user, nickname: "robin") + + {:ok, post} = CommonAPI.post(batman, %{status: "To the Batmobile!", visibility: "private"}) + + reply = %{ + "type" => "Create", + "actor" => robin.ap_id, + "to" => [batman.ap_id], + "cc" => [], + "object" => %{ + "type" => "Note", + "actor" => robin.ap_id, + "content" => "@batman Wait up, I forgot my spandex!", + "to" => [batman.ap_id], + "cc" => [], + "inReplyTo" => Object.normalize(post).data["id"] + } + } + + assert {:ok, filtered} = QuietReply.filter(reply) + + assert match?(^filtered, reply) + end + test "replying direct is unmodified" do batman = insert(:user, nickname: "batman") robin = insert(:user, nickname: "robin") From b0c64945c2cfd622b9f2c68d594bda4fd4c1b9eb Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 12 Aug 2024 20:17:35 -0400 Subject: [PATCH 31/34] MRF.FODirectReply: use Visibility module to verify the scope --- changelog.d/mrf-fodirectreply.add | 2 +- .../web/activity_pub/mrf/fo_direct_reply.ex | 20 +++---------- .../activity_pub/mrf/fo_direct_reply_test.exs | 30 +++++++++++++++++++ 3 files changed, 35 insertions(+), 17 deletions(-) diff --git a/changelog.d/mrf-fodirectreply.add b/changelog.d/mrf-fodirectreply.add index c144e1bc0b..10fd5d16ac 100644 --- a/changelog.d/mrf-fodirectreply.add +++ b/changelog.d/mrf-fodirectreply.add @@ -1 +1 @@ -Added MRF.FODirectReply which changes replies to followers-only posts to be direct +Added MRF.FODirectReply which changes replies to followers-only posts to be direct. diff --git a/lib/pleroma/web/activity_pub/mrf/fo_direct_reply.ex b/lib/pleroma/web/activity_pub/mrf/fo_direct_reply.ex index 4eb97afa84..2cf22745a2 100644 --- a/lib/pleroma/web/activity_pub/mrf/fo_direct_reply.ex +++ b/lib/pleroma/web/activity_pub/mrf/fo_direct_reply.ex @@ -7,7 +7,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.FODirectReply do FODirectReply alters the scope of replies to activities which are Followers Only to be Direct. The purpose of this policy is to prevent broken threads for followers of the reply author because their response was to a user that they are not also following. """ + alias Pleroma.Object alias Pleroma.User + alias Pleroma.Web.ActivityPub.Visibility @behaviour Pleroma.Web.ActivityPub.MRF.Policy @@ -25,7 +27,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.FODirectReply do ) do with true <- is_binary(in_reply_to), %User{follower_address: followers_collection, local: true} <- User.get_by_ap_id(actor), - true <- followers_only?(in_reply_to) do + %Object{} = in_reply_to_object <- Object.get_by_ap_id(in_reply_to), + "private" <- Visibility.get_visibility(in_reply_to_object) do direct_to = to -- [followers_collection] updated_activity = @@ -47,19 +50,4 @@ defmodule Pleroma.Web.ActivityPub.MRF.FODirectReply do @impl true def describe, do: {:ok, %{}} - - defp followers_only?(parent_ap_id) do - with %Pleroma.Object{} = object <- Pleroma.Object.get_by_ap_id(parent_ap_id), - object_data <- Map.get(object, :data), - %Pleroma.User{} = user <- User.get_cached_by_ap_id(object_data["actor"]) do - if user.follower_address in object_data["to"] do - true - else - false - end - else - _ -> - false - end - end end diff --git a/test/pleroma/web/activity_pub/mrf/fo_direct_reply_test.exs b/test/pleroma/web/activity_pub/mrf/fo_direct_reply_test.exs index 7afc83ffca..2d6af3b68e 100644 --- a/test/pleroma/web/activity_pub/mrf/fo_direct_reply_test.exs +++ b/test/pleroma/web/activity_pub/mrf/fo_direct_reply_test.exs @@ -48,6 +48,36 @@ defmodule Pleroma.Web.ActivityPub.MRF.FODirectReplyTest do assert expected_cc == filtered["object"]["cc"] end + test "replies to unlisted posts are unmodified" do + batman = insert(:user, nickname: "batman") + robin = insert(:user, nickname: "robin") + + {:ok, post} = + CommonAPI.post(batman, %{ + status: "Has anyone seen Selina Kyle's latest selfies?", + visibility: "unlisted" + }) + + reply = %{ + "type" => "Create", + "actor" => robin.ap_id, + "to" => [batman.ap_id, robin.follower_address], + "cc" => [], + "object" => %{ + "type" => "Note", + "actor" => robin.ap_id, + "content" => "@batman 🤤 ❤️ 🐈<200d>⬛", + "to" => [batman.ap_id, robin.follower_address], + "cc" => [], + "inReplyTo" => Object.normalize(post).data["id"] + } + } + + assert {:ok, filtered} = FODirectReply.filter(reply) + + assert match?(^filtered, reply) + end + test "replies to public posts are unmodified" do batman = insert(:user, nickname: "batman") robin = insert(:user, nickname: "robin") From 2ba5ad8eb51c9a87da3062328468ea0824d41cc9 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 13 Aug 2024 13:59:07 -0400 Subject: [PATCH 32/34] MRF cleanup MRFs written over time have been inconsistent with the terminology of what is being processed. MRFs work on Activities, but generally we always named the assignments "message" or "object" which is really confusing when trying to debug or write tests. --- changelog.d/mrf-cleanup.skip | 0 .../activity_pub/mrf/anti_followbot_policy.ex | 8 +- .../activity_pub/mrf/anti_link_spam_policy.ex | 10 +- .../mrf/anti_mention_spam_policy.ex | 8 +- .../web/activity_pub/mrf/dnsrbl_policy.ex | 14 +- .../web/activity_pub/mrf/drop_policy.ex | 6 +- .../web/activity_pub/mrf/emoji_policy.ex | 30 ++-- .../activity_pub/mrf/ensure_re_prepended.ex | 14 +- .../web/activity_pub/mrf/follow_bot_policy.ex | 20 +-- .../mrf/force_bot_unlisted_policy.ex | 12 +- .../web/activity_pub/mrf/force_mention.ex | 2 +- .../mrf/force_mentions_in_content.ex | 12 +- .../web/activity_pub/mrf/hashtag_policy.ex | 51 +++--- .../web/activity_pub/mrf/hellthread_policy.ex | 54 +++---- .../activity_pub/mrf/inline_quote_policy.ex | 4 +- .../web/activity_pub/mrf/keyword_policy.ex | 48 +++--- .../mrf/media_proxy_warming_policy.ex | 10 +- .../web/activity_pub/mrf/mention_policy.ex | 12 +- .../web/activity_pub/mrf/no_empty_policy.ex | 14 +- .../web/activity_pub/mrf/no_op_policy.ex | 4 +- .../mrf/no_placeholder_text_policy.ex | 8 +- .../web/activity_pub/mrf/normalize_markup.ex | 10 +- .../web/activity_pub/mrf/nsfw_api_policy.ex | 48 +++--- .../web/activity_pub/mrf/object_age_policy.ex | 58 +++---- lib/pleroma/web/activity_pub/mrf/policy.ex | 2 +- .../web/activity_pub/mrf/quiet_reply.ex | 12 +- .../mrf/quote_to_link_tag_policy.ex | 10 +- .../web/activity_pub/mrf/simple_policy.ex | 150 +++++++++--------- .../activity_pub/mrf/steal_emoji_policy.ex | 6 +- .../web/activity_pub/mrf/subchain_policy.ex | 10 +- .../web/activity_pub/mrf/tag_policy.ex | 62 ++++---- .../mrf/user_allow_list_policy.ex | 12 +- .../web/activity_pub/mrf/vocabulary_policy.ex | 30 ++-- 33 files changed, 376 insertions(+), 375 deletions(-) create mode 100644 changelog.d/mrf-cleanup.skip diff --git a/changelog.d/mrf-cleanup.skip b/changelog.d/mrf-cleanup.skip new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex index df4ba819c8..8ea61aec26 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex @@ -63,20 +63,20 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do end @impl true - def filter(%{"type" => "Follow", "actor" => actor_id} = message) do + def filter(%{"type" => "Follow", "actor" => actor_id} = activity) do %User{} = actor = normalize_by_ap_id(actor_id) score = determine_if_followbot(actor) - if score < 0.8 || bot_allowed?(message, actor) do - {:ok, message} + if score < 0.8 || bot_allowed?(activity, actor) do + {:ok, activity} else {:reject, "[AntiFollowbotPolicy] Scored #{actor_id} as #{score}"} end end @impl true - def filter(message), do: {:ok, message} + def filter(activity), do: {:ok, activity} @impl true def describe, do: {:ok, %{}} diff --git a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex index 3ec9c52ee7..2be6d8df46 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex @@ -29,17 +29,17 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy do defp contains_links?(_), do: false @impl true - def filter(%{"type" => "Create", "actor" => actor, "object" => object} = message) do + def filter(%{"type" => "Create", "actor" => actor, "object" => object} = activity) do with {:ok, %User{local: false} = u} <- User.get_or_fetch_by_ap_id(actor), {:contains_links, true} <- {:contains_links, contains_links?(object)}, {:old_user, true} <- {:old_user, old_user?(u)} do - {:ok, message} + {:ok, activity} else {:ok, %User{local: true}} -> - {:ok, message} + {:ok, activity} {:contains_links, false} -> - {:ok, message} + {:ok, activity} {:old_user, false} -> {:reject, "[AntiLinkSpamPolicy] User has no posts nor followers"} @@ -53,7 +53,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy do end # in all other cases, pass through - def filter(message), do: {:ok, message} + def filter(activity), do: {:ok, activity} @impl true def describe, do: {:ok, %{}} diff --git a/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex index 531e75ce80..1d76a307bf 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex @@ -22,11 +22,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiMentionSpamPolicy do end # copied from HellthreadPolicy - defp get_recipient_count(message) do - recipients = (message["to"] || []) ++ (message["cc"] || []) + defp get_recipient_count(activity) do + recipients = (activity["to"] || []) ++ (activity["cc"] || []) follower_collection = - User.get_cached_by_ap_id(message["actor"] || message["attributedTo"]).follower_address + User.get_cached_by_ap_id(activity["actor"] || activity["attributedTo"]).follower_address if Enum.member?(recipients, Pleroma.Constants.as_public()) do recipients = @@ -80,7 +80,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiMentionSpamPolicy do end # in all other cases, pass through - def filter(message), do: {:ok, message} + def filter(activity), do: {:ok, activity} @impl true def describe, do: {:ok, %{}} diff --git a/lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex b/lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex index 7c6bb888f1..ca41c464c4 100644 --- a/lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex @@ -38,18 +38,18 @@ defmodule Pleroma.Web.ActivityPub.MRF.DNSRBLPolicy do @query_timeout 500 @impl true - def filter(%{"actor" => actor} = object) do + def filter(%{"actor" => actor} = activity) do actor_info = URI.parse(actor) - with {:ok, object} <- check_rbl(actor_info, object) do - {:ok, object} + with {:ok, activity} <- check_rbl(actor_info, activity) do + {:ok, activity} else _ -> {:reject, "[DNSRBLPolicy]"} end end @impl true - def filter(object), do: {:ok, object} + def filter(activity), do: {:ok, activity} @impl true def describe do @@ -90,7 +90,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.DNSRBLPolicy do } end - defp check_rbl(%{host: actor_host}, object) do + defp check_rbl(%{host: actor_host}, activity) do with false <- match?(^actor_host, Pleroma.Web.Endpoint.host()), zone when not is_nil(zone) <- Keyword.get(Config.get([:mrf_dnsrbl]), :zone) do query = @@ -100,7 +100,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.DNSRBLPolicy do rbl_response = rblquery(query) if Enum.empty?(rbl_response) do - {:ok, object} + {:ok, activity} else Task.start(fn -> reason = @@ -117,7 +117,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.DNSRBLPolicy do :error end else - _ -> {:ok, object} + _ -> {:ok, activity} end end diff --git a/lib/pleroma/web/activity_pub/mrf/drop_policy.ex b/lib/pleroma/web/activity_pub/mrf/drop_policy.ex index ad09368397..e4fcc9935b 100644 --- a/lib/pleroma/web/activity_pub/mrf/drop_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/drop_policy.ex @@ -8,9 +8,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.DropPolicy do @behaviour Pleroma.Web.ActivityPub.MRF.Policy @impl true - def filter(object) do - Logger.debug("REJECTING #{inspect(object)}") - {:reject, object} + def filter(activity) do + Logger.debug("REJECTING #{inspect(activity)}") + {:reject, activity} end @impl true diff --git a/lib/pleroma/web/activity_pub/mrf/emoji_policy.ex b/lib/pleroma/web/activity_pub/mrf/emoji_policy.ex index f884962b92..1de5280d9c 100644 --- a/lib/pleroma/web/activity_pub/mrf/emoji_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/emoji_policy.ex @@ -28,11 +28,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do Pleroma.Config.get([:mrf_emoji, :federated_timeline_removal_shortcode], []) end - @impl Pleroma.Web.ActivityPub.MRF.Policy + @impl true def history_awareness, do: :manual - @impl Pleroma.Web.ActivityPub.MRF.Policy - def filter(%{"type" => type, "object" => %{"type" => objtype} = object} = message) + @impl true + def filter(%{"type" => type, "object" => %{"type" => objtype} = object} = activity) when type in ["Create", "Update"] and objtype in Pleroma.Constants.status_object_types() do with {:ok, object} <- Updater.do_with_history(object, fn object -> @@ -42,13 +42,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do Updater.do_with_history(object, fn object -> {:ok, process_remove(object, :shortcode, config_remove_shortcode())} end), - activity <- Map.put(message, "object", object), + activity <- Map.put(activity, "object", object), activity <- maybe_delist(activity) do {:ok, activity} end end - @impl Pleroma.Web.ActivityPub.MRF.Policy + @impl true def filter(%{"type" => type} = object) when type in Pleroma.Constants.actor_types() do with object <- process_remove(object, :url, config_remove_url()), object <- process_remove(object, :shortcode, config_remove_shortcode()) do @@ -56,7 +56,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do end end - @impl Pleroma.Web.ActivityPub.MRF.Policy + @impl true def filter(%{"type" => "EmojiReact"} = object) do with {:ok, _} <- matched_emoji_checker(config_remove_url(), config_remove_shortcode()).(object) do @@ -67,9 +67,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do end end - @impl Pleroma.Web.ActivityPub.MRF.Policy - def filter(message) do - {:ok, message} + @impl true + def filter(activity) do + {:ok, activity} end defp match_string?(string, pattern) when is_binary(pattern) do @@ -214,7 +214,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do ) end - @impl Pleroma.Web.ActivityPub.MRF.Policy + @impl true def describe do mrf_emoji = Pleroma.Config.get(:mrf_emoji, []) @@ -226,7 +226,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do {:ok, %{mrf_emoji: mrf_emoji}} end - @impl Pleroma.Web.ActivityPub.MRF.Policy + @impl true def config_description do %{ key: :mrf_emoji, @@ -239,7 +239,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do key: :remove_url, type: {:list, :string}, description: """ - A list of patterns which result in emoji whose URL matches being removed from the message. This will apply to statuses, emoji reactions, and user profiles. + A list of patterns which result in emoji whose URL matches being removed from the activity. This will apply to statuses, emoji reactions, and user profiles. Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`. """, @@ -249,7 +249,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do key: :remove_shortcode, type: {:list, :string}, description: """ - A list of patterns which result in emoji whose shortcode matches being removed from the message. This will apply to statuses, emoji reactions, and user profiles. + A list of patterns which result in emoji whose shortcode matches being removed from the activity. This will apply to statuses, emoji reactions, and user profiles. Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`. """, @@ -259,7 +259,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do key: :federated_timeline_removal_url, type: {:list, :string}, description: """ - A list of patterns which result in message with emojis whose URLs match being removed from federated timelines (a.k.a unlisted). This will apply only to statuses. + A list of patterns which result in activity with emojis whose URLs match being removed from federated timelines (a.k.a unlisted). This will apply only to statuses. Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`. """, @@ -269,7 +269,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do key: :federated_timeline_removal_shortcode, type: {:list, :string}, description: """ - A list of patterns which result in message with emojis whose shortcodes match being removed from federated timelines (a.k.a unlisted). This will apply only to statuses. + A list of patterns which result in activities with emojis whose shortcodes match being removed from federated timelines (a.k.a unlisted). This will apply only to statuses. Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`. """, diff --git a/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex b/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex index a148cc1e7f..f5983c8a70 100644 --- a/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex +++ b/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex @@ -29,19 +29,19 @@ defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrepended do def filter_by_summary(_in_reply_to, child), do: child - def filter(%{"type" => type, "object" => child_object} = object) - when type in ["Create", "Update"] and is_map(child_object) do + def filter(%{"type" => type, "object" => object} = activity) + when type in ["Create", "Update"] and is_map(object) do child = - child_object["inReplyTo"] + object["inReplyTo"] |> Object.normalize(fetch: false) - |> filter_by_summary(child_object) + |> filter_by_summary(object) - object = Map.put(object, "object", child) + activity = Map.put(activity, "object", child) - {:ok, object} + {:ok, activity} end - def filter(object), do: {:ok, object} + def filter(activity), do: {:ok, activity} def describe, do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex index 55ea2683c3..480a03ef6e 100644 --- a/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex @@ -11,12 +11,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.FollowBotPolicy do require Logger @impl true - def filter(message) do + def filter(activity) do with follower_nickname <- Config.get([:mrf_follow_bot, :follower_nickname]), %User{actor_type: "Service"} = follower <- User.get_cached_by_nickname(follower_nickname), - %{"type" => "Create", "object" => %{"type" => "Note"}} <- message do - try_follow(follower, message) + %{"type" => "Create", "object" => %{"type" => "Note"}} <- activity do + try_follow(follower, activity) else nil -> Logger.warning( @@ -24,17 +24,17 @@ defmodule Pleroma.Web.ActivityPub.MRF.FollowBotPolicy do account does not exist, or the account is not correctly configured as a bot." ) - {:ok, message} + {:ok, activity} _ -> - {:ok, message} + {:ok, activity} end end - defp try_follow(follower, message) do - to = Map.get(message, "to", []) - cc = Map.get(message, "cc", []) - actor = [message["actor"]] + defp try_follow(follower, activity) do + to = Map.get(activity, "to", []) + cc = Map.get(activity, "cc", []) + actor = [activity["actor"]] Enum.concat([to, cc, actor]) |> List.flatten() @@ -53,7 +53,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.FollowBotPolicy do end end) - {:ok, message} + {:ok, activity} end @impl true diff --git a/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex b/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex index 8cec8eabe9..3b3251dc3f 100644 --- a/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex @@ -22,7 +22,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy do "cc" => cc, "actor" => actor, "object" => object - } = message + } = activity ) do user = User.get_cached_by_ap_id(actor) isbot = check_if_bot(user) @@ -36,20 +36,20 @@ defmodule Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy do |> Map.put("to", to) |> Map.put("cc", cc) - message = - message + activity = + activity |> Map.put("to", to) |> Map.put("cc", cc) |> Map.put("object", object) - {:ok, message} + {:ok, activity} else - {:ok, message} + {:ok, activity} end end @impl true - def filter(message), do: {:ok, message} + def filter(activity), do: {:ok, activity} @impl true def describe, do: {:ok, %{}} diff --git a/lib/pleroma/web/activity_pub/mrf/force_mention.ex b/lib/pleroma/web/activity_pub/mrf/force_mention.ex index 3853489fcf..4ea23540da 100644 --- a/lib/pleroma/web/activity_pub/mrf/force_mention.ex +++ b/lib/pleroma/web/activity_pub/mrf/force_mention.ex @@ -52,7 +52,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ForceMention do end @impl true - def filter(object), do: {:ok, object} + def filter(activity), do: {:ok, activity} @impl true def describe, do: {:ok, %{}} diff --git a/lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex b/lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex index 5532093cb7..caae365e5f 100644 --- a/lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex +++ b/lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex @@ -79,18 +79,18 @@ defmodule Pleroma.Web.ActivityPub.MRF.ForceMentionsInContent do %{ "type" => type, "object" => %{"type" => "Note", "to" => to, "inReplyTo" => in_reply_to} - } = object + } = activity ) when type in ["Create", "Update"] and is_list(to) and is_binary(in_reply_to) do # image-only posts from pleroma apparently reach this MRF without the content field - content = object["object"]["content"] || "" + content = activity["object"]["content"] || "" # Get the replied-to user for sorting - replied_to_user = get_replied_to_user(object["object"]) + replied_to_user = get_replied_to_user(activity["object"]) mention_users = to - |> clean_recipients(object) + |> clean_recipients(activity) |> Enum.map(&User.get_cached_by_ap_id/1) |> Enum.reject(&is_nil/1) |> sort_replied_user(replied_to_user) @@ -126,11 +126,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.ForceMentionsInContent do content end - {:ok, put_in(object["object"]["content"], content)} + {:ok, put_in(activity["object"]["content"], content)} end @impl true - def filter(object), do: {:ok, object} + def filter(activity), do: {:ok, activity} @impl true def describe, do: {:ok, %{}} diff --git a/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex b/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex index fdb9a9dba8..72f2274ed2 100644 --- a/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do alias Pleroma.Object @moduledoc """ - Reject, TWKN-remove or Set-Sensitive messages with specific hashtags (without the leading #) + Reject, TWKN-remove or Set-Sensitive activities with specific hashtags (without the leading #) Note: This MRF Policy is always enabled, if you want to disable it you have to set empty lists. """ @@ -19,40 +19,40 @@ defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do @impl true def history_awareness, do: :manual - defp check_reject(message, hashtags) do + defp check_reject(activity, hashtags) do if Enum.any?(Config.get([:mrf_hashtag, :reject]), fn match -> match in hashtags end) do {:reject, "[HashtagPolicy] Matches with rejected keyword"} else - {:ok, message} + {:ok, activity} end end - defp check_ftl_removal(%{"to" => to} = message, hashtags) do + defp check_ftl_removal(%{"to" => to} = activity, hashtags) do if Pleroma.Constants.as_public() in to and Enum.any?(Config.get([:mrf_hashtag, :federated_timeline_removal]), fn match -> match in hashtags end) do to = List.delete(to, Pleroma.Constants.as_public()) - cc = [Pleroma.Constants.as_public() | message["cc"] || []] + cc = [Pleroma.Constants.as_public() | activity["cc"] || []] - message = - message + activity = + activity |> Map.put("to", to) |> Map.put("cc", cc) |> Kernel.put_in(["object", "to"], to) |> Kernel.put_in(["object", "cc"], cc) - {:ok, message} + {:ok, activity} else - {:ok, message} + {:ok, activity} end end - defp check_ftl_removal(message, _hashtags), do: {:ok, message} + defp check_ftl_removal(activity, _hashtags), do: {:ok, activity} - defp check_sensitive(message) do + defp check_sensitive(activity) do {:ok, new_object} = - Object.Updater.do_with_history(message["object"], fn object -> + Object.Updater.do_with_history(activity["object"], fn object -> hashtags = Object.hashtags(%Object{data: object}) if Enum.any?(Config.get([:mrf_hashtag, :sensitive]), fn match -> match in hashtags end) do @@ -62,11 +62,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do end end) - {:ok, Map.put(message, "object", new_object)} + {:ok, Map.put(activity, "object", new_object)} end @impl true - def filter(%{"type" => type, "object" => object} = message) when type in ["Create", "Update"] do + def filter(%{"type" => type, "object" => object} = activity) + when type in ["Create", "Update"] do history_items = with %{"formerRepresentations" => %{"orderedItems" => items}} <- object do items @@ -82,23 +83,23 @@ defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do hashtags = Object.hashtags(%Object{data: object}) ++ historical_hashtags if hashtags != [] do - with {:ok, message} <- check_reject(message, hashtags), - {:ok, message} <- + with {:ok, activity} <- check_reject(activity, hashtags), + {:ok, activity} <- (if type == "Create" do - check_ftl_removal(message, hashtags) + check_ftl_removal(activity, hashtags) else - {:ok, message} + {:ok, activity} end), - {:ok, message} <- check_sensitive(message) do - {:ok, message} + {:ok, activity} <- check_sensitive(activity) do + {:ok, activity} end else - {:ok, message} + {:ok, activity} end end @impl true - def filter(message), do: {:ok, message} + def filter(activity), do: {:ok, activity} @impl true def describe do @@ -120,21 +121,21 @@ defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do %{ key: :reject, type: {:list, :string}, - description: "A list of hashtags which result in message being rejected.", + description: "A list of hashtags which result in the activity being rejected.", suggestions: ["foo"] }, %{ key: :federated_timeline_removal, type: {:list, :string}, description: - "A list of hashtags which result in message being removed from federated timelines (a.k.a unlisted).", + "A list of hashtags which result in the activity being removed from federated timelines (a.k.a unlisted).", suggestions: ["foo"] }, %{ key: :sensitive, type: {:list, :string}, description: - "A list of hashtags which result in message being set as sensitive (a.k.a NSFW/R-18)", + "A list of hashtags which result in the activity being set as sensitive (a.k.a NSFW/R-18)", suggestions: ["nsfw", "r18"] } ] diff --git a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex index 80e235d6ec..3a80d0a699 100644 --- a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex @@ -7,54 +7,54 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do require Pleroma.Constants - @moduledoc "Block messages with too much mentions (configurable)" + @moduledoc "Block activities with too much mentions (configurable)" @behaviour Pleroma.Web.ActivityPub.MRF.Policy - defp delist_message(message, threshold) when threshold > 0 do - follower_collection = User.get_cached_by_ap_id(message["actor"]).follower_address - to = message["to"] || [] - cc = message["cc"] || [] + defp delist_activity(activity, threshold) when threshold > 0 do + follower_collection = User.get_cached_by_ap_id(activity["actor"]).follower_address + to = activity["to"] || [] + cc = activity["cc"] || [] follower_collection? = Enum.member?(to ++ cc, follower_collection) - message = - case get_recipient_count(message) do + activity = + case get_recipient_count(activity) do {:public, recipients} when follower_collection? and recipients > threshold -> - message + activity |> Map.put("to", [follower_collection]) |> Map.put("cc", [Pleroma.Constants.as_public()]) {:public, recipients} when recipients > threshold -> - message + activity |> Map.put("to", []) |> Map.put("cc", [Pleroma.Constants.as_public()]) _ -> - message + activity end - {:ok, message} + {:ok, activity} end - defp delist_message(message, _threshold), do: {:ok, message} + defp delist_activity(activity, _threshold), do: {:ok, activity} - defp reject_message(message, threshold) when threshold > 0 do - with {_, recipients} <- get_recipient_count(message) do + defp reject_activity(activity, threshold) when threshold > 0 do + with {_, recipients} <- get_recipient_count(activity) do if recipients > threshold do {:reject, "[HellthreadPolicy] #{recipients} recipients is over the limit of #{threshold}"} else - {:ok, message} + {:ok, activity} end end end - defp reject_message(message, _threshold), do: {:ok, message} + defp reject_activity(activity, _threshold), do: {:ok, activity} - defp get_recipient_count(message) do - recipients = (message["to"] || []) ++ (message["cc"] || []) - follower_collection = User.get_cached_by_ap_id(message["actor"]).follower_address + defp get_recipient_count(activity) do + recipients = (activity["to"] || []) ++ (activity["cc"] || []) + follower_collection = User.get_cached_by_ap_id(activity["actor"]).follower_address if Enum.member?(recipients, Pleroma.Constants.as_public()) do recipients = @@ -73,7 +73,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do end @impl true - def filter(%{"type" => "Create", "object" => %{"type" => object_type}} = message) + def filter(%{"type" => "Create", "object" => %{"type" => object_type}} = activity) when object_type in ~w{Note Article} do reject_threshold = Pleroma.Config.get( @@ -83,16 +83,16 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do delist_threshold = Pleroma.Config.get([:mrf_hellthread, :delist_threshold]) - with {:ok, message} <- reject_message(message, reject_threshold), - {:ok, message} <- delist_message(message, delist_threshold) do - {:ok, message} + with {:ok, activity} <- reject_activity(activity, reject_threshold), + {:ok, activity} <- delist_activity(activity, delist_threshold) do + {:ok, activity} else e -> e end end @impl true - def filter(message), do: {:ok, message} + def filter(activity), do: {:ok, activity} @impl true def describe, @@ -104,13 +104,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do key: :mrf_hellthread, related_policy: "Pleroma.Web.ActivityPub.MRF.HellthreadPolicy", label: "MRF Hellthread", - description: "Block messages with excessive user mentions", + description: "Block activities with excessive user mentions", children: [ %{ key: :delist_threshold, type: :integer, description: - "Number of mentioned users after which the message gets removed from timelines and" <> + "Number of mentioned users after which the activity gets removed from timelines and" <> "disables notifications. Set to 0 to disable.", suggestions: [10] }, @@ -118,7 +118,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do key: :reject_threshold, type: :integer, description: - "Number of mentioned users after which the messaged gets rejected. Set to 0 to disable.", + "Number of mentioned users after which the activity gets rejected. Set to 0 to disable.", suggestions: [20] } ] diff --git a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex index b7a01c27c2..469d06ef60 100644 --- a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex @@ -48,12 +48,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy do end @impl true - def filter(object), do: {:ok, object} + def filter(activity), do: {:ok, activity} @impl true def describe, do: {:ok, %{}} - @impl Pleroma.Web.ActivityPub.MRF.Policy + @impl true def history_awareness, do: :auto @impl true diff --git a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex index 729da4e9c9..6ba6fd5096 100644 --- a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do alias Pleroma.Web.ActivityPub.MRF.Utils - @moduledoc "Reject or Word-Replace messages with a keyword or regex" + @moduledoc "Reject or Word-Replace activities with a keyword or regex" @behaviour Pleroma.Web.ActivityPub.MRF.Policy @@ -25,7 +25,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do |> Enum.join("\n") end - defp check_reject(%{"object" => %{} = object} = message) do + defp check_reject(%{"object" => %{} = object} = activity) do with {:ok, _new_object} <- Pleroma.Object.Updater.do_with_history(object, fn object -> payload = object_payload(object) @@ -35,16 +35,16 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do end) do {:reject, "[KeywordPolicy] Matches with rejected keyword"} else - {:ok, message} + {:ok, activity} end end) do - {:ok, message} + {:ok, activity} else e -> e end end - defp check_ftl_removal(%{"type" => "Create", "to" => to, "object" => %{} = object} = message) do + defp check_ftl_removal(%{"type" => "Create", "to" => to, "object" => %{} = object} = activity) do check_keyword = fn object -> payload = object_payload(object) @@ -67,24 +67,24 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do if Pleroma.Constants.as_public() in to and should_delist?.(object) do to = List.delete(to, Pleroma.Constants.as_public()) - cc = [Pleroma.Constants.as_public() | message["cc"] || []] + cc = [Pleroma.Constants.as_public() | activity["cc"] || []] - message = - message + activity = + activity |> Map.put("to", to) |> Map.put("cc", cc) - {:ok, message} + {:ok, activity} else - {:ok, message} + {:ok, activity} end end - defp check_ftl_removal(message) do - {:ok, message} + defp check_ftl_removal(activity) do + {:ok, activity} end - defp check_replace(%{"object" => %{} = object} = message) do + defp check_replace(%{"object" => %{} = object} = activity) do replace_kw = fn object -> ["content", "name", "summary"] |> Enum.filter(fn field -> Map.has_key?(object, field) && object[field] end) @@ -103,18 +103,18 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do {:ok, object} = Pleroma.Object.Updater.do_with_history(object, replace_kw) - message = Map.put(message, "object", object) + activity = Map.put(activity, "object", object) - {:ok, message} + {:ok, activity} end @impl true - def filter(%{"type" => type, "object" => %{"content" => _content}} = message) + def filter(%{"type" => type, "object" => %{"content" => _content}} = activity) when type in ["Create", "Update"] do - with {:ok, message} <- check_reject(message), - {:ok, message} <- check_ftl_removal(message), - {:ok, message} <- check_replace(message) do - {:ok, message} + with {:ok, activity} <- check_reject(activity), + {:ok, activity} <- check_ftl_removal(activity), + {:ok, activity} <- check_replace(activity) do + {:ok, activity} else {:reject, nil} -> {:reject, "[KeywordPolicy] "} {:reject, _} = e -> e @@ -123,7 +123,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do end @impl true - def filter(message), do: {:ok, message} + def filter(activity), do: {:ok, activity} @impl true def describe do @@ -154,13 +154,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do related_policy: "Pleroma.Web.ActivityPub.MRF.KeywordPolicy", label: "MRF Keyword", description: - "Reject or Word-Replace messages matching a keyword or [Regex](https://hexdocs.pm/elixir/Regex.html).", + "Reject or Word-Replace activities matching a keyword or [Regex](https://hexdocs.pm/elixir/Regex.html).", children: [ %{ key: :reject, type: {:list, :string}, description: """ - A list of patterns which result in message being rejected. + A list of patterns which result in the activity being rejected. Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`. """, @@ -170,7 +170,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do key: :federated_timeline_removal, type: {:list, :string}, description: """ - A list of patterns which result in message being removed from federated timelines (a.k.a unlisted). + A list of patterns which result in the activity being removed from federated timelines (a.k.a unlisted). Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`. """, diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex index 0c5b53def0..b0d07a6f86 100644 --- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex @@ -31,7 +31,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do HTTP.get(url, [], http_client_opts) end - defp preload(%{"object" => %{"attachment" => attachments}} = _message) do + defp preload(%{"object" => %{"attachment" => attachments}} = _activity) do Enum.each(attachments, fn %{"url" => url} when is_list(url) -> url @@ -49,15 +49,15 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do end @impl true - def filter(%{"type" => type, "object" => %{"attachment" => attachments} = _object} = message) + def filter(%{"type" => type, "object" => %{"attachment" => attachments} = _object} = activity) when type in ["Create", "Update"] and is_list(attachments) and length(attachments) > 0 do - preload(message) + preload(activity) - {:ok, message} + {:ok, activity} end @impl true - def filter(message), do: {:ok, message} + def filter(activity), do: {:ok, activity} @impl true def describe, do: {:ok, %{}} diff --git a/lib/pleroma/web/activity_pub/mrf/mention_policy.ex b/lib/pleroma/web/activity_pub/mrf/mention_policy.ex index 8aa4f347f1..f7bff121fd 100644 --- a/lib/pleroma/web/activity_pub/mrf/mention_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/mention_policy.ex @@ -3,25 +3,25 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.MentionPolicy do - @moduledoc "Block messages which mention a user" + @moduledoc "Block activities which mention a user" @behaviour Pleroma.Web.ActivityPub.MRF.Policy @impl true - def filter(%{"type" => "Create"} = message) do + def filter(%{"type" => "Create"} = activity) do reject_actors = Pleroma.Config.get([:mrf_mention, :actors], []) - recipients = (message["to"] || []) ++ (message["cc"] || []) + recipients = (activity["to"] || []) ++ (activity["cc"] || []) if rejected_mention = Enum.find(recipients, fn recipient -> Enum.member?(reject_actors, recipient) end) do {:reject, "[MentionPolicy] Rejected for mention of #{rejected_mention}"} else - {:ok, message} + {:ok, activity} end end @impl true - def filter(message), do: {:ok, message} + def filter(activity), do: {:ok, activity} @impl true def describe, do: {:ok, %{}} @@ -32,7 +32,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MentionPolicy do key: :mrf_mention, related_policy: "Pleroma.Web.ActivityPub.MRF.MentionPolicy", label: "MRF Mention", - description: "Block messages which mention a specific user", + description: "Block activities which mention a specific user", children: [ %{ key: :actors, diff --git a/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex b/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex index 12bf4ddd27..08dd39878d 100644 --- a/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex @@ -9,20 +9,20 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoEmptyPolicy do alias Pleroma.Web.Endpoint @impl true - def filter(%{"actor" => actor} = object) do + def filter(%{"actor" => actor} = activity) do with true <- local?(actor), - true <- eligible_type?(object), - true <- note?(object), - false <- has_attachment?(object), - true <- only_mentions?(object) do + true <- eligible_type?(activity), + true <- note?(activity), + false <- has_attachment?(activity), + true <- only_mentions?(activity) do {:reject, "[NoEmptyPolicy]"} else _ -> - {:ok, object} + {:ok, activity} end end - def filter(object), do: {:ok, object} + def filter(activity), do: {:ok, activity} defp local?(actor) do if actor |> String.starts_with?("#{Endpoint.url()}") do diff --git a/lib/pleroma/web/activity_pub/mrf/no_op_policy.ex b/lib/pleroma/web/activity_pub/mrf/no_op_policy.ex index 8840c4fac4..64a5872bc1 100644 --- a/lib/pleroma/web/activity_pub/mrf/no_op_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/no_op_policy.ex @@ -7,8 +7,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoOpPolicy do @behaviour Pleroma.Web.ActivityPub.MRF.Policy @impl true - def filter(object) do - {:ok, object} + def filter(activity) do + {:ok, activity} end @impl true diff --git a/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex b/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex index f81e9e52a0..c6f239a5ea 100644 --- a/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex @@ -13,15 +13,15 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicy do def filter( %{ "type" => type, - "object" => %{"content" => content, "attachment" => _} = _child_object - } = object + "object" => %{"content" => content, "attachment" => _} = _object + } = activity ) when type in ["Create", "Update"] and content in [".", "

.

"] do - {:ok, put_in(object, ["object", "content"], "")} + {:ok, put_in(activity, ["object", "content"], "")} end @impl true - def filter(object), do: {:ok, object} + def filter(activity), do: {:ok, activity} @impl true def describe, do: {:ok, %{}} diff --git a/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex b/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex index 2dfc9a901c..91855ef845 100644 --- a/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex +++ b/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex @@ -12,20 +12,20 @@ defmodule Pleroma.Web.ActivityPub.MRF.NormalizeMarkup do def history_awareness, do: :auto @impl true - def filter(%{"type" => type, "object" => child_object} = object) + def filter(%{"type" => type, "object" => object} = activity) when type in ["Create", "Update"] do scrub_policy = Pleroma.Config.get([:mrf_normalize_markup, :scrub_policy]) content = - child_object["content"] + object["content"] |> HTML.filter_tags(scrub_policy) - object = put_in(object, ["object", "content"], content) + activity = put_in(activity, ["object", "content"], content) - {:ok, object} + {:ok, activity} end - def filter(object), do: {:ok, object} + def filter(activity), do: {:ok, activity} @impl true def describe, do: {:ok, %{}} diff --git a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex index 451a212d44..52aaf05aaa 100644 --- a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex @@ -122,52 +122,52 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do end end - def check_object_nsfw(%{"object" => %{} = child_object} = object) do - case check_object_nsfw(child_object) do - {:sfw, _} -> {:sfw, object} - {:nsfw, _} -> {:nsfw, object} + def check_object_nsfw(%{"object" => %{} = object} = activity) do + case check_object_nsfw(object) do + {:sfw, _} -> {:sfw, activity} + {:nsfw, _} -> {:nsfw, activity} end end def check_object_nsfw(object), do: {:sfw, object} @impl true - def filter(object) do - with {:sfw, object} <- check_object_nsfw(object) do - {:ok, object} + def filter(activity) do + with {:sfw, activity} <- check_object_nsfw(activity) do + {:ok, activity} else - {:nsfw, _data} -> handle_nsfw(object) + {:nsfw, _data} -> handle_nsfw(activity) end end - defp handle_nsfw(object) do + defp handle_nsfw(activity) do if Config.get([@policy, :reject]) do - {:reject, object} + {:reject, activity} else {:ok, - object + activity |> maybe_unlist() |> maybe_mark_sensitive()} end end - defp maybe_unlist(object) do + defp maybe_unlist(activity) do if Config.get([@policy, :unlist]) do - unlist(object) + unlist(activity) else - object + activity end end - defp maybe_mark_sensitive(object) do + defp maybe_mark_sensitive(activity) do if Config.get([@policy, :mark_sensitive]) do - mark_sensitive(object) + mark_sensitive(activity) else - object + activity end end - def unlist(%{"to" => to, "cc" => cc, "actor" => actor} = object) do + def unlist(%{"to" => to, "cc" => cc, "actor" => actor} = activity) do with %User{} = user <- User.get_cached_by_ap_id(actor) do to = [user.follower_address | to] @@ -179,7 +179,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do |> List.delete(user.follower_address) |> Enum.uniq() - object + activity |> Map.put("to", to) |> Map.put("cc", cc) else @@ -187,14 +187,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do end end - def mark_sensitive(%{"object" => child_object} = object) when is_map(child_object) do - Map.put(object, "object", mark_sensitive(child_object)) + def mark_sensitive(%{"object" => object} = activity) when is_map(object) do + Map.put(activity, "object", mark_sensitive(object)) end - def mark_sensitive(object) when is_map(object) do - tags = (object["tag"] || []) ++ ["nsfw"] + def mark_sensitive(activity) when is_map(activity) do + tags = (activity["tag"] || []) ++ ["nsfw"] - object + activity |> Map.put("tag", tags) |> Map.put("sensitive", true) end diff --git a/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex b/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex index df1a6dcbbc..34905fc216 100644 --- a/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex @@ -11,12 +11,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do @moduledoc "Filter activities depending on their age" @behaviour Pleroma.Web.ActivityPub.MRF.Policy - defp check_date(%{"object" => %{"published" => published}} = message) do + defp check_date(%{"object" => %{"published" => published}} = activity) do with %DateTime{} = now <- DateTime.utc_now(), {:ok, %DateTime{} = then, _} <- DateTime.from_iso8601(published), max_ttl <- Config.get([:mrf_object_age, :threshold]), {:ttl, false} <- {:ttl, DateTime.diff(now, then) > max_ttl} do - {:ok, message} + {:ok, activity} else {:ttl, true} -> {:reject, nil} @@ -26,73 +26,73 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do end end - defp check_reject(message, actions) do + defp check_reject(activity, actions) do if :reject in actions do {:reject, "[ObjectAgePolicy]"} else - {:ok, message} + {:ok, activity} end end - defp check_delist(message, actions) do + defp check_delist(activity, actions) do if :delist in actions do - with %User{} = user <- User.get_cached_by_ap_id(message["actor"]) do + with %User{} = user <- User.get_cached_by_ap_id(activity["actor"]) do to = - List.delete(message["to"] || [], Pleroma.Constants.as_public()) ++ + List.delete(activity["to"] || [], Pleroma.Constants.as_public()) ++ [user.follower_address] cc = - List.delete(message["cc"] || [], user.follower_address) ++ + List.delete(activity["cc"] || [], user.follower_address) ++ [Pleroma.Constants.as_public()] - message = - message + activity = + activity |> Map.put("to", to) |> Map.put("cc", cc) |> Kernel.put_in(["object", "to"], to) |> Kernel.put_in(["object", "cc"], cc) - {:ok, message} + {:ok, activity} else _e -> {:reject, "[ObjectAgePolicy] Unhandled error"} end else - {:ok, message} + {:ok, activity} end end - defp check_strip_followers(message, actions) do + defp check_strip_followers(activity, actions) do if :strip_followers in actions do - with %User{} = user <- User.get_cached_by_ap_id(message["actor"]) do - to = List.delete(message["to"] || [], user.follower_address) - cc = List.delete(message["cc"] || [], user.follower_address) + with %User{} = user <- User.get_cached_by_ap_id(activity["actor"]) do + to = List.delete(activity["to"] || [], user.follower_address) + cc = List.delete(activity["cc"] || [], user.follower_address) - message = - message + activity = + activity |> Map.put("to", to) |> Map.put("cc", cc) |> Kernel.put_in(["object", "to"], to) |> Kernel.put_in(["object", "cc"], cc) - {:ok, message} + {:ok, activity} else _e -> {:reject, "[ObjectAgePolicy] Unhandled error"} end else - {:ok, message} + {:ok, activity} end end @impl true - def filter(%{"type" => "Create", "object" => %{"published" => _}} = message) do + def filter(%{"type" => "Create", "object" => %{"published" => _}} = activity) do with actions <- Config.get([:mrf_object_age, :actions]), - {:reject, _} <- check_date(message), - {:ok, message} <- check_reject(message, actions), - {:ok, message} <- check_delist(message, actions), - {:ok, message} <- check_strip_followers(message, actions) do - {:ok, message} + {:reject, _} <- check_date(activity), + {:ok, activity} <- check_reject(activity, actions), + {:ok, activity} <- check_delist(activity, actions), + {:ok, activity} <- check_strip_followers(activity, actions) do + {:ok, activity} else # check_date() is allowed to short-circuit the pipeline e -> e @@ -100,7 +100,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do end @impl true - def filter(message), do: {:ok, message} + def filter(activity), do: {:ok, activity} @impl true def describe do @@ -131,8 +131,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do type: {:list, :atom}, description: "A list of actions to apply to the post. `:delist` removes the post from public timelines; " <> - "`:strip_followers` removes followers from the ActivityPub recipient list ensuring they won't be delivered to home timelines, additionally for followers-only it degrades to a direct message; " <> - "`:reject` rejects the message entirely", + "`:strip_followers` removes followers from the ActivityPub recipient list ensuring they won't be delivered to home timelines, additionally for followers-only it degrades to a direct activity; " <> + "`:reject` rejects the activity entirely", suggestions: [:delist, :strip_followers, :reject] } ] diff --git a/lib/pleroma/web/activity_pub/mrf/policy.ex b/lib/pleroma/web/activity_pub/mrf/policy.ex index 1f34883e70..54ca4b7357 100644 --- a/lib/pleroma/web/activity_pub/mrf/policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/policy.ex @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.Policy do - @callback filter(map()) :: {:ok | :reject, map()} + @callback filter(Pleroma.Activity.t()) :: {:ok | :reject, Pleroma.Activity.t()} @callback describe() :: {:ok | :error, map()} @callback config_description() :: %{ optional(:children) => [map()], diff --git a/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex b/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex index ae5e2cdc70..b07dc3b562 100644 --- a/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex +++ b/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex @@ -26,7 +26,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.QuietReply do "type" => "Note", "inReplyTo" => in_reply_to } - } = object + } = activity ) do with true <- is_binary(in_reply_to), false <- match?([], cc), @@ -39,21 +39,21 @@ defmodule Pleroma.Web.ActivityPub.MRF.QuietReply do updated_cc = [Pleroma.Constants.as_public()] - updated_object = - object + updated_activity = + activity |> Map.put("to", updated_to) |> Map.put("cc", updated_cc) |> put_in(["object", "to"], updated_to) |> put_in(["object", "cc"], updated_cc) - {:ok, updated_object} + {:ok, updated_activity} else - _ -> {:ok, object} + _ -> {:ok, activity} end end @impl true - def filter(object), do: {:ok, object} + def filter(activity), do: {:ok, activity} @impl true def describe, do: {:ok, %{}} diff --git a/lib/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy.ex b/lib/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy.ex index ac353f03f9..2a17b67618 100644 --- a/lib/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy.ex @@ -10,18 +10,18 @@ defmodule Pleroma.Web.ActivityPub.MRF.QuoteToLinkTagPolicy do require Pleroma.Constants - @impl Pleroma.Web.ActivityPub.MRF.Policy + @impl true def filter(%{"object" => %{"quoteUrl" => _} = object} = activity) do {:ok, Map.put(activity, "object", filter_object(object))} end - @impl Pleroma.Web.ActivityPub.MRF.Policy - def filter(object), do: {:ok, object} + @impl true + def filter(activity), do: {:ok, activity} - @impl Pleroma.Web.ActivityPub.MRF.Policy + @impl true def describe, do: {:ok, %{}} - @impl Pleroma.Web.ActivityPub.MRF.Policy + @impl true def history_awareness, do: :auto defp filter_object(%{"quoteUrl" => quote_url} = object) do diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index d708c99ebd..ae7f18bfe5 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -13,20 +13,20 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do require Pleroma.Constants - defp check_accept(%{host: actor_host} = _actor_info, object) do + defp check_accept(%{host: actor_host} = _actor_info, activity) do accepts = instance_list(:accept) |> MRF.subdomains_regex() cond do - accepts == [] -> {:ok, object} - actor_host == Config.get([Pleroma.Web.Endpoint, :url, :host]) -> {:ok, object} - MRF.subdomain_match?(accepts, actor_host) -> {:ok, object} + accepts == [] -> {:ok, activity} + actor_host == Config.get([Pleroma.Web.Endpoint, :url, :host]) -> {:ok, activity} + MRF.subdomain_match?(accepts, actor_host) -> {:ok, activity} true -> {:reject, "[SimplePolicy] host not in accept list"} end end - defp check_reject(%{host: actor_host} = _actor_info, object) do + defp check_reject(%{host: actor_host} = _actor_info, activity) do rejects = instance_list(:reject) |> MRF.subdomains_regex() @@ -34,109 +34,109 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do if MRF.subdomain_match?(rejects, actor_host) do {:reject, "[SimplePolicy] host in reject list"} else - {:ok, object} + {:ok, activity} end end defp check_media_removal( %{host: actor_host} = _actor_info, - %{"type" => type, "object" => %{"attachment" => child_attachment}} = object + %{"type" => type, "object" => %{"attachment" => object_attachment}} = activity ) - when length(child_attachment) > 0 and type in ["Create", "Update"] do + when length(object_attachment) > 0 and type in ["Create", "Update"] do media_removal = instance_list(:media_removal) |> MRF.subdomains_regex() - object = + activity = if MRF.subdomain_match?(media_removal, actor_host) do - child_object = Map.delete(object["object"], "attachment") - Map.put(object, "object", child_object) + object = Map.delete(activity["object"], "attachment") + Map.put(activity, "object", object) else - object + activity end - {:ok, object} + {:ok, activity} end - defp check_media_removal(_actor_info, object), do: {:ok, object} + defp check_media_removal(_actor_info, activity), do: {:ok, activity} defp check_media_nsfw( %{host: actor_host} = _actor_info, %{ "type" => type, - "object" => %{} = _child_object - } = object + "object" => %{} = _object + } = activity ) when type in ["Create", "Update"] do media_nsfw = instance_list(:media_nsfw) |> MRF.subdomains_regex() - object = + activity = if MRF.subdomain_match?(media_nsfw, actor_host) do - Kernel.put_in(object, ["object", "sensitive"], true) + Kernel.put_in(activity, ["object", "sensitive"], true) else - object + activity end - {:ok, object} + {:ok, activity} end - defp check_media_nsfw(_actor_info, object), do: {:ok, object} + defp check_media_nsfw(_actor_info, activity), do: {:ok, activity} - defp check_ftl_removal(%{host: actor_host} = _actor_info, object) do + defp check_ftl_removal(%{host: actor_host} = _actor_info, activity) do timeline_removal = instance_list(:federated_timeline_removal) |> MRF.subdomains_regex() - object = + activity = with true <- MRF.subdomain_match?(timeline_removal, actor_host), - user <- User.get_cached_by_ap_id(object["actor"]), - true <- Pleroma.Constants.as_public() in object["to"] do - to = List.delete(object["to"], Pleroma.Constants.as_public()) ++ [user.follower_address] + user <- User.get_cached_by_ap_id(activity["actor"]), + true <- Pleroma.Constants.as_public() in activity["to"] do + to = List.delete(activity["to"], Pleroma.Constants.as_public()) ++ [user.follower_address] - cc = List.delete(object["cc"], user.follower_address) ++ [Pleroma.Constants.as_public()] + cc = List.delete(activity["cc"], user.follower_address) ++ [Pleroma.Constants.as_public()] - object + activity |> Map.put("to", to) |> Map.put("cc", cc) else - _ -> object + _ -> activity end - {:ok, object} + {:ok, activity} end defp intersection(list1, list2) do list1 -- list1 -- list2 end - defp check_followers_only(%{host: actor_host} = _actor_info, object) do + defp check_followers_only(%{host: actor_host} = _actor_info, activity) do followers_only = instance_list(:followers_only) |> MRF.subdomains_regex() - object = + activity = with true <- MRF.subdomain_match?(followers_only, actor_host), - user <- User.get_cached_by_ap_id(object["actor"]) do + user <- User.get_cached_by_ap_id(activity["actor"]) do # Don't use Map.get/3 intentionally, these must not be nil - fixed_to = object["to"] || [] - fixed_cc = object["cc"] || [] + fixed_to = activity["to"] || [] + fixed_cc = activity["cc"] || [] to = FollowingRelationship.followers_ap_ids(user, fixed_to) cc = FollowingRelationship.followers_ap_ids(user, fixed_cc) - object + activity |> Map.put("to", intersection([user.follower_address | to], fixed_to)) |> Map.put("cc", intersection([user.follower_address | cc], fixed_cc)) else - _ -> object + _ -> activity end - {:ok, object} + {:ok, activity} end - defp check_report_removal(%{host: actor_host} = _actor_info, %{"type" => "Flag"} = object) do + defp check_report_removal(%{host: actor_host} = _actor_info, %{"type" => "Flag"} = activity) do report_removal = instance_list(:report_removal) |> MRF.subdomains_regex() @@ -144,39 +144,39 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do if MRF.subdomain_match?(report_removal, actor_host) do {:reject, "[SimplePolicy] host in report_removal list"} else - {:ok, object} + {:ok, activity} end end - defp check_report_removal(_actor_info, object), do: {:ok, object} + defp check_report_removal(_actor_info, activity), do: {:ok, activity} - defp check_avatar_removal(%{host: actor_host} = _actor_info, %{"icon" => _icon} = object) do + defp check_avatar_removal(%{host: actor_host} = _actor_info, %{"icon" => _icon} = activity) do avatar_removal = instance_list(:avatar_removal) |> MRF.subdomains_regex() if MRF.subdomain_match?(avatar_removal, actor_host) do - {:ok, Map.delete(object, "icon")} + {:ok, Map.delete(activity, "icon")} else - {:ok, object} + {:ok, activity} end end - defp check_avatar_removal(_actor_info, object), do: {:ok, object} + defp check_avatar_removal(_actor_info, activity), do: {:ok, activity} - defp check_banner_removal(%{host: actor_host} = _actor_info, %{"image" => _image} = object) do + defp check_banner_removal(%{host: actor_host} = _actor_info, %{"image" => _image} = activity) do banner_removal = instance_list(:banner_removal) |> MRF.subdomains_regex() if MRF.subdomain_match?(banner_removal, actor_host) do - {:ok, Map.delete(object, "image")} + {:ok, Map.delete(activity, "image")} else - {:ok, object} + {:ok, activity} end end - defp check_banner_removal(_actor_info, object), do: {:ok, object} + defp check_banner_removal(_actor_info, activity), do: {:ok, activity} defp check_object(%{"object" => object} = activity) do with {:ok, _object} <- filter(object) do @@ -184,7 +184,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do end end - defp check_object(object), do: {:ok, object} + defp check_object(activity), do: {:ok, activity} defp instance_list(config_key) do Config.get([:mrf_simple, config_key]) @@ -192,7 +192,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do end @impl true - def filter(%{"type" => "Delete", "actor" => actor} = object) do + def filter(%{"type" => "Delete", "actor" => actor} = activity) do %{host: actor_host} = URI.parse(actor) reject_deletes = @@ -202,54 +202,54 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do if MRF.subdomain_match?(reject_deletes, actor_host) do {:reject, "[SimplePolicy] host in reject_deletes list"} else - {:ok, object} + {:ok, activity} end end @impl true - def filter(%{"actor" => actor} = object) do + def filter(%{"actor" => actor} = activity) do actor_info = URI.parse(actor) - with {:ok, object} <- check_accept(actor_info, object), - {:ok, object} <- check_reject(actor_info, object), - {:ok, object} <- check_media_removal(actor_info, object), - {:ok, object} <- check_media_nsfw(actor_info, object), - {:ok, object} <- check_ftl_removal(actor_info, object), - {:ok, object} <- check_followers_only(actor_info, object), - {:ok, object} <- check_report_removal(actor_info, object), - {:ok, object} <- check_object(object) do - {:ok, object} + with {:ok, activity} <- check_accept(actor_info, activity), + {:ok, activity} <- check_reject(actor_info, activity), + {:ok, activity} <- check_media_removal(actor_info, activity), + {:ok, activity} <- check_media_nsfw(actor_info, activity), + {:ok, activity} <- check_ftl_removal(actor_info, activity), + {:ok, activity} <- check_followers_only(actor_info, activity), + {:ok, activity} <- check_report_removal(actor_info, activity), + {:ok, activity} <- check_object(activity) do + {:ok, activity} else {:reject, _} = e -> e end end - def filter(%{"id" => actor, "type" => obj_type} = object) - when obj_type in ["Application", "Group", "Organization", "Person", "Service"] do + def filter(%{"id" => actor, "type" => actor_type} = activity) + when actor_type in ["Application", "Group", "Organization", "Person", "Service"] do actor_info = URI.parse(actor) - with {:ok, object} <- check_accept(actor_info, object), - {:ok, object} <- check_reject(actor_info, object), - {:ok, object} <- check_avatar_removal(actor_info, object), - {:ok, object} <- check_banner_removal(actor_info, object) do - {:ok, object} + with {:ok, activity} <- check_accept(actor_info, activity), + {:ok, activity} <- check_reject(actor_info, activity), + {:ok, activity} <- check_avatar_removal(actor_info, activity), + {:ok, activity} <- check_banner_removal(actor_info, activity) do + {:ok, activity} else {:reject, _} = e -> e end end - def filter(object) when is_binary(object) do - uri = URI.parse(object) + def filter(activity) when is_binary(activity) do + uri = URI.parse(activity) - with {:ok, object} <- check_accept(uri, object), - {:ok, object} <- check_reject(uri, object) do - {:ok, object} + with {:ok, activity} <- check_accept(uri, activity), + {:ok, activity} <- check_reject(uri, activity) do + {:ok, activity} else {:reject, _} = e -> e end end - def filter(object), do: {:ok, object} + def filter(activity), do: {:ok, activity} @impl true def describe do diff --git a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex index fa6b595ea9..6edfb124e4 100644 --- a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex @@ -62,7 +62,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do end @impl true - def filter(%{"object" => %{"emoji" => foreign_emojis, "actor" => actor}} = message) do + def filter(%{"object" => %{"emoji" => foreign_emojis, "actor" => actor}} = activity) do host = URI.parse(actor).host if host != Pleroma.Web.Endpoint.host() and accept_host?(host) do @@ -97,10 +97,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do end end - {:ok, message} + {:ok, activity} end - def filter(message), do: {:ok, message} + def filter(activity), do: {:ok, activity} @impl true @spec config_description :: %{ diff --git a/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex b/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex index fdb9e51765..97acca7e87 100644 --- a/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex @@ -20,20 +20,20 @@ defmodule Pleroma.Web.ActivityPub.MRF.SubchainPolicy do end @impl true - def filter(%{"actor" => actor} = message) do + def filter(%{"actor" => actor} = activity) do with {:ok, match, subchain} <- lookup_subchain(actor) do Logger.debug( "[SubchainPolicy] Matched #{actor} against #{inspect(match)} with subchain #{inspect(subchain)}" ) - MRF.filter(subchain, message) + MRF.filter(subchain, activity) else - _e -> {:ok, message} + _e -> {:ok, activity} end end @impl true - def filter(message), do: {:ok, message} + def filter(activity), do: {:ok, activity} @impl true def describe, do: {:ok, %{}} @@ -45,7 +45,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SubchainPolicy do related_policy: "Pleroma.Web.ActivityPub.MRF.SubchainPolicy", label: "MRF Subchain", description: - "This policy processes messages through an alternate pipeline when a given message matches certain criteria." <> + "This policy processes activities through an alternate pipeline when a given activity matches certain criteria." <> " All criteria are configured as a map of regular expressions to lists of policy modules.", children: [ %{ diff --git a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex index 73760ca8f2..c236a5a99e 100644 --- a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex @@ -28,25 +28,25 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do "mrf_tag:media-force-nsfw", %{ "type" => type, - "object" => %{"attachment" => child_attachment} - } = message + "object" => %{"attachment" => object_attachment} + } = activity ) - when length(child_attachment) > 0 and type in ["Create", "Update"] do - {:ok, Kernel.put_in(message, ["object", "sensitive"], true)} + when length(object_attachment) > 0 and type in ["Create", "Update"] do + {:ok, Kernel.put_in(activity, ["object", "sensitive"], true)} end defp process_tag( "mrf_tag:media-strip", %{ "type" => type, - "object" => %{"attachment" => child_attachment} = object - } = message + "object" => %{"attachment" => object_attachment} = object + } = activity ) - when length(child_attachment) > 0 and type in ["Create", "Update"] do + when length(object_attachment) > 0 and type in ["Create", "Update"] do object = Map.delete(object, "attachment") - message = Map.put(message, "object", object) + activity = Map.put(activity, "object", object) - {:ok, message} + {:ok, activity} end defp process_tag( @@ -57,7 +57,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do "cc" => cc, "actor" => actor, "object" => object - } = message + } = activity ) do user = User.get_cached_by_ap_id(actor) @@ -70,15 +70,15 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do |> Map.put("to", to) |> Map.put("cc", cc) - message = - message + activity = + activity |> Map.put("to", to) |> Map.put("cc", cc) |> Map.put("object", object) - {:ok, message} + {:ok, activity} else - {:ok, message} + {:ok, activity} end end @@ -90,7 +90,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do "cc" => cc, "actor" => actor, "object" => object - } = message + } = activity ) do user = User.get_cached_by_ap_id(actor) @@ -104,26 +104,26 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do |> Map.put("to", to) |> Map.put("cc", cc) - message = - message + activity = + activity |> Map.put("to", to) |> Map.put("cc", cc) |> Map.put("object", object) - {:ok, message} + {:ok, activity} else - {:ok, message} + {:ok, activity} end end defp process_tag( "mrf_tag:disable-remote-subscription", - %{"type" => "Follow", "actor" => actor} = message + %{"type" => "Follow", "actor" => actor} = activity ) do user = User.get_cached_by_ap_id(actor) if user.local == true do - {:ok, message} + {:ok, activity} else {:reject, "[TagPolicy] Follow from #{actor} tagged with mrf_tag:disable-remote-subscription"} @@ -133,14 +133,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do defp process_tag("mrf_tag:disable-any-subscription", %{"type" => "Follow", "actor" => actor}), do: {:reject, "[TagPolicy] Follow from #{actor} tagged with mrf_tag:disable-any-subscription"} - defp process_tag(_, message), do: {:ok, message} + defp process_tag(_, activity), do: {:ok, activity} - def filter_message(actor, message) do + def filter_activity(actor, activity) do User.get_cached_by_ap_id(actor) |> get_tags() - |> Enum.reduce({:ok, message}, fn - tag, {:ok, message} -> - process_tag(tag, message) + |> Enum.reduce({:ok, activity}, fn + tag, {:ok, activity} -> + process_tag(tag, activity) _, error -> error @@ -148,15 +148,15 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do end @impl true - def filter(%{"object" => target_actor, "type" => "Follow"} = message), - do: filter_message(target_actor, message) + def filter(%{"object" => target_actor, "type" => "Follow"} = activity), + do: filter_activity(target_actor, activity) @impl true - def filter(%{"actor" => actor, "type" => type} = message) when type in ["Create", "Update"], - do: filter_message(actor, message) + def filter(%{"actor" => actor, "type" => type} = activity) when type in ["Create", "Update"], + do: filter_activity(actor, activity) @impl true - def filter(message), do: {:ok, message} + def filter(activity), do: {:ok, activity} @impl true def describe, do: {:ok, %{}} diff --git a/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex b/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex index e14047d4e3..10cc0e09d0 100644 --- a/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex @@ -8,18 +8,18 @@ defmodule Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy do @moduledoc "Accept-list of users from specified instances" @behaviour Pleroma.Web.ActivityPub.MRF.Policy - defp filter_by_list(object, []), do: {:ok, object} + defp filter_by_list(activity, []), do: {:ok, activity} - defp filter_by_list(%{"actor" => actor} = object, allow_list) do + defp filter_by_list(%{"actor" => actor} = activity, allow_list) do if actor in allow_list do - {:ok, object} + {:ok, activity} else {:reject, "[UserAllowListPolicy] #{actor} not in the list"} end end @impl true - def filter(%{"actor" => actor} = object) do + def filter(%{"actor" => actor} = activity) do actor_info = URI.parse(actor) allow_list = @@ -28,10 +28,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy do [] ) - filter_by_list(object, allow_list) + filter_by_list(activity, allow_list) end - def filter(object), do: {:ok, object} + def filter(activity), do: {:ok, activity} @impl true def describe do diff --git a/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex index 1c114558e6..5671e4cf32 100644 --- a/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex @@ -3,38 +3,38 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.VocabularyPolicy do - @moduledoc "Filter messages which belong to certain activity vocabularies" + @moduledoc "Filter activities which belong to certain activity vocabularies" @behaviour Pleroma.Web.ActivityPub.MRF.Policy @impl true - def filter(%{"type" => "Undo", "object" => child_message} = message) do - with {:ok, _} <- filter(child_message) do - {:ok, message} + def filter(%{"type" => "Undo", "object" => object} = activity) do + with {:ok, _} <- filter(object) do + {:ok, activity} else {:reject, _} = e -> e end end - def filter(%{"type" => message_type} = message) do + def filter(%{"type" => activity_type} = activity) do with accepted_vocabulary <- Pleroma.Config.get([:mrf_vocabulary, :accept]), rejected_vocabulary <- Pleroma.Config.get([:mrf_vocabulary, :reject]), {_, true} <- {:accepted, - Enum.empty?(accepted_vocabulary) || Enum.member?(accepted_vocabulary, message_type)}, + Enum.empty?(accepted_vocabulary) || Enum.member?(accepted_vocabulary, activity_type)}, {_, false} <- {:rejected, - length(rejected_vocabulary) > 0 && Enum.member?(rejected_vocabulary, message_type)}, - {:ok, _} <- filter(message["object"]) do - {:ok, message} + length(rejected_vocabulary) > 0 && Enum.member?(rejected_vocabulary, activity_type)}, + {:ok, _} <- filter(activity["object"]) do + {:ok, activity} else {:reject, _} = e -> e - {:accepted, _} -> {:reject, "[VocabularyPolicy] #{message_type} not in accept list"} - {:rejected, _} -> {:reject, "[VocabularyPolicy] #{message_type} in reject list"} + {:accepted, _} -> {:reject, "[VocabularyPolicy] #{activity_type} not in accept list"} + {:rejected, _} -> {:reject, "[VocabularyPolicy] #{activity_type} in reject list"} end end - def filter(message), do: {:ok, message} + def filter(activity), do: {:ok, activity} @impl true def describe, @@ -46,20 +46,20 @@ defmodule Pleroma.Web.ActivityPub.MRF.VocabularyPolicy do key: :mrf_vocabulary, related_policy: "Pleroma.Web.ActivityPub.MRF.VocabularyPolicy", label: "MRF Vocabulary", - description: "Filter messages which belong to certain activity vocabularies", + description: "Filter activities which belong to certain activity vocabularies", children: [ %{ key: :accept, type: {:list, :string}, description: - "A list of ActivityStreams terms to accept. If empty, all supported messages are accepted.", + "A list of ActivityStreams terms to accept. If empty, all supported activities are accepted.", suggestions: ["Create", "Follow", "Mention", "Announce", "Like"] }, %{ key: :reject, type: {:list, :string}, description: - "A list of ActivityStreams terms to reject. If empty, no messages are rejected.", + "A list of ActivityStreams terms to reject. If empty, no activities are rejected.", suggestions: ["Create", "Follow", "Mention", "Announce", "Like"] } ] From 648e94b369c4a356feda22c0f8ee8a01f7d798f8 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 13 Aug 2024 15:28:28 -0400 Subject: [PATCH 33/34] Fix the uploads migration mix task test which leaked a change to the configured Uploader --- test/mix/tasks/pleroma/uploads_test.exs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/test/mix/tasks/pleroma/uploads_test.exs b/test/mix/tasks/pleroma/uploads_test.exs index d4ea895014..22b6baf24c 100644 --- a/test/mix/tasks/pleroma/uploads_test.exs +++ b/test/mix/tasks/pleroma/uploads_test.exs @@ -3,12 +3,14 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Mix.Tasks.Pleroma.UploadsTest do + alias Pleroma.Config alias Pleroma.Upload - use Pleroma.DataCase + use Pleroma.DataCase, async: false import Mock setup_all do + prep_uploads() Mix.shell(Mix.Shell.Process) on_exit(fn -> @@ -18,6 +20,8 @@ defmodule Mix.Tasks.Pleroma.UploadsTest do :ok end + setup do: clear_config([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) + describe "running migrate_local" do test "uploads migrated" do with_mock Upload, @@ -53,4 +57,15 @@ defmodule Mix.Tasks.Pleroma.UploadsTest do end end end + + defp prep_uploads() do + upload_dir = Config.get([Pleroma.Uploaders.Local, :uploads]) + + if not File.exists?(upload_dir) || File.ls!(upload_dir) == [] do + File.mkdir_p(upload_dir) + + Path.join([upload_dir, "file.txt"]) + |> File.touch() + end + end end From b281ad06de2de331450a5e319e3ba497071d4197 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 13 Aug 2024 15:34:56 -0400 Subject: [PATCH 34/34] Revert "Custom mix task to retry failed tests once in CI pipeline" This reverts commit ed2976b237b53a524247d564691e0a12d3231d68. --- .gitlab-ci.yml | 2 +- changelog.d/fix-test-failures.skip | 0 lib/mix/tasks/pleroma/test_runner.ex | 25 ------------------------- test/mix/tasks/pleroma/uploads_test.exs | 2 +- 4 files changed, 2 insertions(+), 27 deletions(-) create mode 100644 changelog.d/fix-test-failures.skip delete mode 100644 lib/mix/tasks/pleroma/test_runner.ex diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eaa9d3b25c..0c6631797a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -133,7 +133,7 @@ unit-testing-1.13.4-otp-25: script: &testing_script - mix ecto.create - mix ecto.migrate - - mix pleroma.test_runner --cover --preload-modules + - mix test --cover --preload-modules coverage: '/^Line total: ([^ ]*%)$/' artifacts: reports: diff --git a/changelog.d/fix-test-failures.skip b/changelog.d/fix-test-failures.skip new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lib/mix/tasks/pleroma/test_runner.ex b/lib/mix/tasks/pleroma/test_runner.ex deleted file mode 100644 index 69fefb0014..0000000000 --- a/lib/mix/tasks/pleroma/test_runner.ex +++ /dev/null @@ -1,25 +0,0 @@ -defmodule Mix.Tasks.Pleroma.TestRunner do - @shortdoc "Retries tests once if they fail" - - use Mix.Task - - def run(args \\ []) do - case System.cmd("mix", ["test"] ++ args, into: IO.stream(:stdio, :line)) do - {_, 0} -> - :ok - - _ -> - retry(args) - end - end - - def retry(args) do - case System.cmd("mix", ["test", "--failed"] ++ args, into: IO.stream(:stdio, :line)) do - {_, 0} -> - :ok - - _ -> - exit(1) - end - end -end diff --git a/test/mix/tasks/pleroma/uploads_test.exs b/test/mix/tasks/pleroma/uploads_test.exs index 22b6baf24c..f3d5aa64fd 100644 --- a/test/mix/tasks/pleroma/uploads_test.exs +++ b/test/mix/tasks/pleroma/uploads_test.exs @@ -58,7 +58,7 @@ defmodule Mix.Tasks.Pleroma.UploadsTest do end end - defp prep_uploads() do + defp prep_uploads do upload_dir = Config.get([Pleroma.Uploaders.Local, :uploads]) if not File.exists?(upload_dir) || File.ls!(upload_dir) == [] do