diff --git a/src/github/handlers/push-event.ts b/src/github/handlers/push-event.ts
index 5be0c26..2cf8f77 100644
--- a/src/github/handlers/push-event.ts
+++ b/src/github/handlers/push-event.ts
@@ -125,18 +125,8 @@ async function checkPluginConfigurations(context: GitHubContext<"push">, config:
 export default async function handlePushEvent(context: GitHubContext<"push">) {
   const { payload } = context;
   const { repository, commits, after } = payload;
-  const configPaths = [CONFIG_FULL_PATH, DEV_CONFIG_FULL_PATH];
-  const didConfigurationFileChange = commits.some((commit) =>
-    configPaths.some((path) => {
-      if (commit.modified?.includes(path) || commit.added?.includes(path)) {
-        // Keeps only the config that matched the modified elements
-        configPaths.length = 0;
-        configPaths.push(path);
-        return true;
-      }
-      return false;
-    })
-  );
+  const configPath = context.eventHandler.environment === "production" ? CONFIG_FULL_PATH : DEV_CONFIG_FULL_PATH;
+  const didConfigurationFileChange = commits.some((commit) => commit.modified?.includes(configPath) || commit.added?.includes(configPath));
 
   if (!didConfigurationFileChange || !repository.owner) {
     return;
@@ -154,7 +144,7 @@ export default async function handlePushEvent(context: GitHubContext<"push">) {
   try {
     if (errors.length) {
       const body = [];
-      body.push(...constructErrorBody(errors, rawData, repository, after, configPaths[0]));
+      body.push(...constructErrorBody(errors, rawData, repository, after, configPath));
       await createCommitComment(
         context,
         {
diff --git a/tests/events.test.ts b/tests/events.test.ts
index 49f1eb3..d4d831c 100644
--- a/tests/events.test.ts
+++ b/tests/events.test.ts
@@ -5,6 +5,7 @@ import { http, HttpResponse } from "msw";
 import { GitHubContext } from "../src/github/github-context";
 import { GitHubEventHandler } from "../src/github/github-event-handler";
 import issueCommentCreated from "../src/github/handlers/issue-comment-created";
+import handlePushEvent from "../src/github/handlers/push-event";
 import { CONFIG_FULL_PATH } from "../src/github/utils/config";
 import { server } from "./__mocks__/node";
 import "./__mocks__/webhooks";
@@ -17,6 +18,8 @@ jest.mock("@octokit/auth-app", () => ({}));
 
 config({ path: ".dev.vars" });
 
+const name = "ubiquity-os-kernel";
+
 beforeAll(() => {
   server.listen();
 });
@@ -107,7 +110,7 @@ describe("Event related tests", () => {
       payload: {
         repository: {
           owner: { login: "ubiquity" },
-          name: "ubiquity-os-kernel",
+          name,
         },
         issue: { number: 1 },
         comment: {
@@ -124,9 +127,91 @@ describe("Event related tests", () => {
             " all available commands. | `/help` |\n| `/action` | action | `/action` |\n| `/bar` | bar command | `/bar foo` |\n| `/foo` | foo command | `/foo bar` |",
           issue_number: 1,
           owner: "ubiquity",
-          repo: "ubiquity-os-kernel",
+          repo: name,
         },
       ],
     ]);
   });
+  it("should handle push event correctly", async () => {
+    const issues = {
+      createComment(params?: RestEndpointMethodTypes["issues"]["createComment"]["parameters"]) {
+        return params;
+      },
+    };
+    const createCommitComment = jest.fn();
+    const context = {
+      id: "",
+      key: "issue_comment.created",
+      octokit: {
+        rest: {
+          issues,
+          repos: {
+            listCommentsForCommit: jest.fn(() => ({ data: [] })),
+            createCommitComment: createCommitComment,
+            getContent(params?: RestEndpointMethodTypes["repos"]["getContent"]["parameters"]) {
+              if (params?.path === CONFIG_FULL_PATH) {
+                return {
+                  data: `
+                    plugins:
+                      - name: "Run on comment created"
+                        uses:
+                          - id: plugin-A
+                            plugin: https://plugin-a.internal
+                            with:
+                              arg: "true"
+                      - name: "Some Action plugin"
+                        uses:
+                          - id: plugin-B
+                            plugin: ubiquity-os/plugin-b
+                    `,
+                };
+              } else if (params?.path === "manifest.json") {
+                return {
+                  data: {
+                    content: btoa(
+                      JSON.stringify({
+                        name: "plugin",
+                        commands: {
+                          action: {
+                            description: "action",
+                            "ubiquity:example": "/action",
+                          },
+                        },
+                        configuration: {
+                          default: {},
+                          type: "object",
+                          properties: {
+                            arg: {
+                              type: "number",
+                            },
+                          },
+                          required: ["arg"],
+                        },
+                      })
+                    ),
+                  },
+                };
+              } else {
+                throw new Error("Not found");
+              }
+            },
+          },
+        },
+      },
+      eventHandler: eventHandler,
+      payload: {
+        repository: {
+          owner: { login: "ubiquity" },
+          name,
+        },
+        issue: { number: 1 },
+        comment: {
+          body: "/help",
+        },
+        commits: [{ modified: [CONFIG_FULL_PATH] }],
+      } as unknown as GitHubContext<"issue_comment.created">["payload"],
+    } as unknown as GitHubContext;
+    await expect(handlePushEvent(context)).resolves.not.toThrow();
+    expect(createCommitComment).toBeCalledTimes(1);
+  });
 });