diff --git a/.formatter.exs b/.formatter.exs
index d2cda26..447c223 100644
--- a/.formatter.exs
+++ b/.formatter.exs
@@ -1,4 +1,4 @@
 # Used by "mix format"
 [
-  inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"]
+  inputs: ["{mix,.formatter}.exs", "{bin,config,lib,test}/**/*.{ex,exs}"]
 ]
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index bafea23..96b3e20 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -83,6 +83,9 @@ jobs:
       - name: Run tests
         run: mix test
 
+      - name: Run smoke tests
+        run: elixir ./bin/smoke_test.exs
+
       - name: Run Dialyzer
         run: mix dialyzer
         if: ${{ matrix.current_version }}
diff --git a/bin/smoke_test.exs b/bin/smoke_test.exs
new file mode 100755
index 0000000..2784d76
--- /dev/null
+++ b/bin/smoke_test.exs
@@ -0,0 +1,114 @@
+IO.puts("Running smoke tests")
+IO.puts("\n\n")
+
+{diff, 0} = System.cmd("git", ["diff"])
+
+if diff != "" do
+  IO.puts(
+    "There are unstaged changes. Stage them (git add) or remove them for this check to be able to work."
+  )
+
+  System.halt(1)
+end
+
+IO.puts("checking smoke_test_data/project_with_formatted_code")
+
+{_, 0} = System.cmd("mix", ["format"], cd: "smoke_test_data/project_with_formatted_code")
+{diff, 0} = System.cmd("git", ["diff"], cd: "smoke_test_data/project_with_formatted_code")
+
+{"", 0} =
+  System.cmd("git", ["checkout", "--", "."], cd: "smoke_test_data/project_with_formatted_code")
+
+if diff == "" do
+  IO.puts("OK ")
+else
+  IO.puts(
+    "Expected project_with_formatted_code to be unchanged after running `mix format` in it, but there were changes:"
+  )
+
+  IO.puts(diff)
+  System.halt(2)
+end
+
+IO.puts("checking smoke_test_data/project_with_unformatted_code")
+
+{_, 0} = System.cmd("mix", ["format"], cd: "smoke_test_data/project_with_unformatted_code")
+{diff, 0} = System.cmd("git", ["diff"], cd: "smoke_test_data/project_with_unformatted_code")
+
+expected_diff =
+  """
+  diff --git a/smoke_test_data/project_with_unformatted_code/README.md b/smoke_test_data/project_with_unformatted_code/README.md
+  index d066e9b..0ccfa0b 100644
+  --- a/smoke_test_data/project_with_unformatted_code/README.md
+  +++ b/smoke_test_data/project_with_unformatted_code/README.md
+  @@ -9,9 +9,9 @@ by adding `project_with_unformatted_code` to your list of dependencies in `mix.e
+  #{" "}
+   ```elixir
+   def deps do
+  -[
+  -{ :project_with_unformatted_code, "~> 0.1.0" }
+  -]
+  +  [
+  +    {:project_with_unformatted_code, "~> 0.1.0"}
+  +  ]
+   end
+   ```
+  #{" "}
+  diff --git a/smoke_test_data/project_with_unformatted_code/docs/hello.md b/smoke_test_data/project_with_unformatted_code/docs/hello.md
+  index 114bb32..8d4ffe5 100644
+  --- a/smoke_test_data/project_with_unformatted_code/docs/hello.md
+  +++ b/smoke_test_data/project_with_unformatted_code/docs/hello.md
+  @@ -1,7 +1,7 @@
+   # Hello!
+  #{" "}
+   ~~~~elixir
+  -%{ x: 7,   y: 8}
+  +%{x: 7, y: 8}
+   ~~~~
+  #{" "}
+   [//]: # (elixir-formatter-disable-next-block)
+  @@ -11,7 +11,7 @@
+   ~~~~
+  #{" "}
+   ```elixir
+  -%{ x: 7,   y: 8}
+  +%{x: 7, y: 8}
+   ```
+  #{" "}
+   ```markdown
+  diff --git a/smoke_test_data/project_with_unformatted_code/lib/project_with_unformatted_code.ex b/smoke_test_data/project_with_unformatted_code/lib/project_with_unformatted_code.ex
+  index 7af383c..c94df7f 100644
+  --- a/smoke_test_data/project_with_unformatted_code/lib/project_with_unformatted_code.ex
+  +++ b/smoke_test_data/project_with_unformatted_code/lib/project_with_unformatted_code.ex
+  @@ -17,13 +17,14 @@ defmodule ProjectWithUnformattedCode do
+       # Hello, World!
+  #{" "}
+       ```elixir
+  -    def add(a,b), do: a+b
+  +    def add(a, b), do: a + b
+       ```
+  #{" "}
+       ```js
+       1+2+3
+       ```
+       \"""
+  +
+       :world
+     end
+   end
+  """
+
+{"", 0} =
+  System.cmd("git", ["checkout", "--", "."], cd: "smoke_test_data/project_with_unformatted_code")
+
+if diff == expected_diff do
+  IO.puts("OK")
+else
+  IO.puts(
+    "Expected project_with_unformatted_code to have specific changes after running `mix format` in it (see `bin/smoke_test.exs`), the actual changes differ:"
+  )
+
+  IO.inspect(String.myers_difference(diff, expected_diff))
+
+  System.halt(2)
+end
diff --git a/smoke_test_data/project_with_unformatted_code_expected_output/README.md b/smoke_test_data/project_with_unformatted_code_expected_output/README.md
deleted file mode 100644
index 799476c..0000000
--- a/smoke_test_data/project_with_unformatted_code_expected_output/README.md
+++ /dev/null
@@ -1,21 +0,0 @@
-# ProjectWithUnformattedCode
-
-**TODO: Add description**
-
-## Installation
-
-If [available in Hex](https://hex.pm/docs/publish), the package can be installed
-by adding `project_with_unformatted_code` to your list of dependencies in `mix.exs`:
-
-```elixir
-def deps do
-[
-  {:project_with_unformatted_code, "~> 0.1.0"}
-]
-end
-```
-
-Documentation can be generated with [ExDoc](https://github.com/elixir-lang/ex_doc)
-and published on [HexDocs](https://hexdocs.pm). Once published, the docs can
-be found at <https://hexdocs.pm/project_with_unformatted_code>.
-
diff --git a/smoke_test_data/project_with_unformatted_code_expected_output/docs/hello.md b/smoke_test_data/project_with_unformatted_code_expected_output/docs/hello.md
deleted file mode 100644
index 8d4ffe5..0000000
--- a/smoke_test_data/project_with_unformatted_code_expected_output/docs/hello.md
+++ /dev/null
@@ -1,21 +0,0 @@
-# Hello!
-
-~~~~elixir
-%{x: 7, y: 8}
-~~~~
-
-[//]: # (elixir-formatter-disable-next-block)
-
-~~~~elixir
-%{ x: 7,   y: 8}
-~~~~
-
-```elixir
-%{x: 7, y: 8}
-```
-
-```markdown
-~~~~elixir
-%{ x: 7,   y: 8}
-~~~~
-```
diff --git a/smoke_test_data/project_with_unformatted_code_expected_output/lib/project_with_unformatted_code.ex b/smoke_test_data/project_with_unformatted_code_expected_output/lib/project_with_unformatted_code.ex
deleted file mode 100644
index 7ecf9b0..0000000
--- a/smoke_test_data/project_with_unformatted_code_expected_output/lib/project_with_unformatted_code.ex
+++ /dev/null
@@ -1,29 +0,0 @@
-defmodule ProjectWithUnformattedCode do
-  @moduledoc """
-  Documentation for `ProjectWithUnformattedCode`.
-  """
-
-  @doc """
-  Hello world.
-
-  ## Examples
-
-      iex> ProjectWithUnformattedCode.hello()
-      :world
-
-  """
-  def hello do
-    ~M"""
-    # Hello, World!
-
-    ```elixir
-    def add(a, b), do: a + b
-    ```
-
-    ```js
-    1+2+3
-    ```
-    """
-    :world
-  end
-end
diff --git a/test/fixtures/simple_example.md b/test/fixtures/simple_example.md
deleted file mode 100644
index ae1fe65..0000000
--- a/test/fixtures/simple_example.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# Hello, World!
-
-Elixir is super **cool** because it supports piping with the `|>` pipe operator.
-
-```elixir
-"Hello"
-|> String.upcase()
-|> String.split()
-```
-
-Whoa!