diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index eaa9d3b25c..0c6631797a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -133,7 +133,7 @@ unit-testing-1.13.4-otp-25:
   script: &testing_script
     - mix ecto.create
     - mix ecto.migrate
-    - mix pleroma.test_runner --cover --preload-modules
+    - mix test --cover --preload-modules
   coverage: '/^Line total: ([^ ]*%)$/'
   artifacts:
     reports:
diff --git a/changelog.d/fix-test-failures.skip b/changelog.d/fix-test-failures.skip
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/lib/mix/tasks/pleroma/test_runner.ex b/lib/mix/tasks/pleroma/test_runner.ex
deleted file mode 100644
index 69fefb0014..0000000000
--- a/lib/mix/tasks/pleroma/test_runner.ex
+++ /dev/null
@@ -1,25 +0,0 @@
-defmodule Mix.Tasks.Pleroma.TestRunner do
-  @shortdoc "Retries tests once if they fail"
-
-  use Mix.Task
-
-  def run(args \\ []) do
-    case System.cmd("mix", ["test"] ++ args, into: IO.stream(:stdio, :line)) do
-      {_, 0} ->
-        :ok
-
-      _ ->
-        retry(args)
-    end
-  end
-
-  def retry(args) do
-    case System.cmd("mix", ["test", "--failed"] ++ args, into: IO.stream(:stdio, :line)) do
-      {_, 0} ->
-        :ok
-
-      _ ->
-        exit(1)
-    end
-  end
-end
diff --git a/test/mix/tasks/pleroma/uploads_test.exs b/test/mix/tasks/pleroma/uploads_test.exs
index d4ea895014..f3d5aa64fd 100644
--- a/test/mix/tasks/pleroma/uploads_test.exs
+++ b/test/mix/tasks/pleroma/uploads_test.exs
@@ -3,12 +3,14 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.UploadsTest do
+  alias Pleroma.Config
   alias Pleroma.Upload
-  use Pleroma.DataCase
+  use Pleroma.DataCase, async: false
 
   import Mock
 
   setup_all do
+    prep_uploads()
     Mix.shell(Mix.Shell.Process)
 
     on_exit(fn ->
@@ -18,6 +20,8 @@ defmodule Mix.Tasks.Pleroma.UploadsTest do
     :ok
   end
 
+  setup do: clear_config([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
+
   describe "running migrate_local" do
     test "uploads migrated" do
       with_mock Upload,
@@ -53,4 +57,15 @@ defmodule Mix.Tasks.Pleroma.UploadsTest do
       end
     end
   end
+
+  defp prep_uploads do
+    upload_dir = Config.get([Pleroma.Uploaders.Local, :uploads])
+
+    if not File.exists?(upload_dir) || File.ls!(upload_dir) == [] do
+      File.mkdir_p(upload_dir)
+
+      Path.join([upload_dir, "file.txt"])
+      |> File.touch()
+    end
+  end
 end