Merge pull request #42 from gentlementlegen/feat/workers

feat: plugins can now be spawned as Workers

.github/workflows/build.yml

@@ -1,8 +1,9 @@
-name: Build & Deploy
+name: Build & Deploy to Cloudflare
 
 on:
   push:
-  pull_request:
+    branches:
+      - main
   workflow_dispatch:
 
 permissions:
@@ -15,19 +16,25 @@ jobs:
     steps:
       - name: Check out repository
         uses: actions/checkout@v4
-        # with:
-        # submodules: "recursive" # Ensures submodules are checked out
 
       - name: Set up Node.js
         uses: actions/setup-node@v4
         with:
           node-version: 20.10.0
 
-      - name: Build
-        run: |
-          npm i -g bun
-          bun install
-          bun build src/worker.ts
-        # env: # Set environment variables for the build
-          # SUPABASE_URL: "https://wfzpewmlyiozupulbuur.supabase.co"
-          # SUPABASE_ANON_KEY: "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6IndmenBld21seWlvenVwdWxidXVyIiwicm9sZSI6ImFub24iLCJpYXQiOjE2OTU2NzQzMzksImV4cCI6MjAxMTI1MDMzOX0.SKIL3Q0NOBaMehH0ekFspwgcu3afp3Dl9EDzPqs1nKs"
+      - uses: oven-sh/setup-bun@v1
+
+      - uses: cloudflare/wrangler-action@v3
+        with:
+          wranglerVersion: '3.57.0'
+          apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+          secrets: |
+            WEBHOOK_PROXY_URL
+            WEBHOOK_SECRET
+            APP_ID
+            PRIVATE_KEY
+        env:
+          WEBHOOK_PROXY_URL: ${{ secrets.WEBHOOK_PROXY_URL }}
+          WEBHOOK_SECRET: ${{ secrets.WEBHOOK_SECRET }}
+          APP_ID: ${{ secrets.APP_ID }}
+          PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }}

.github/workflows/bun-testing.yml

@@ -1,7 +1,7 @@
 name: Run Bun testing suite
 on:
   workflow_dispatch:
-  pull_request_target:
+  pull_request:
     types: [ opened, synchronize ]
 
 env:

src/github/handlers/index.ts

@@ -1,9 +1,11 @@
 import { EmitterWebhookEvent } from "@octokit/webhooks";
+import { GitHubContext } from "../github-context";
 import { GitHubEventHandler } from "../github-event-handler";
 import { getConfig } from "../utils/config";
 import { repositoryDispatch } from "./repository-dispatch";
-import { dispatchWorkflow, getDefaultBranch } from "../utils/workflow-dispatch";
+import { dispatchWorker, dispatchWorkflow, getDefaultBranch } from "../utils/workflow-dispatch";
 import { DelegatedComputeInputs } from "../types/plugin";
+import { isGithubPlugin, PluginConfiguration } from "../types/plugin-configuration";
 
 function tryCatchWrapper(fn: (event: EmitterWebhookEvent) => unknown) {
   return async (event: EmitterWebhookEvent) => {
@@ -20,6 +22,24 @@ export function bindHandlers(eventHandler: GitHubEventHandler) {
   eventHandler.onAny(tryCatchWrapper((event) => handleEvent(event, eventHandler))); // onAny should also receive GithubContext but the types in octokit/webhooks are weird
 }
 
+function shouldSkipPlugin(event: EmitterWebhookEvent, context: GitHubContext, pluginChain: PluginConfiguration["plugins"]["*"][0]) {
+  if (pluginChain.skipBotEvents && "sender" in event.payload && event.payload.sender?.type === "Bot") {
+    console.log("Skipping plugin chain because sender is a bot");
+    return true;
+  }
+  if (
+    context.key === "issue_comment.created" &&
+    pluginChain.command &&
+    "comment" in context.payload &&
+    typeof context.payload.comment !== "string" &&
+    !context.payload.comment?.body.startsWith(pluginChain.command)
+  ) {
+    console.log("Skipping plugin chain because command does not match");
+    return true;
+  }
+  return false;
+}
+
 async function handleEvent(event: EmitterWebhookEvent, eventHandler: InstanceType<typeof GitHubEventHandler>) {
   const context = eventHandler.transformEvent(event);
 
@@ -43,22 +63,13 @@ async function handleEvent(event: EmitterWebhookEvent, eventHandler: InstanceTyp
   }
 
   for (const pluginChain of pluginChains) {
-    if (pluginChain.skipBotEvents && "sender" in event.payload && event.payload.sender?.type === "Bot") {
-      console.log("Skipping plugin chain because sender is a bot");
-      continue;
-    }
-    if (
-      context.key === "issue_comment.created" &&
-      pluginChain.command &&
-      "comment" in context.payload &&
-      !context.payload.comment.body.startsWith(pluginChain.command)
-    ) {
-      console.log("Skipping plugin chain because command does not match");
+    if (shouldSkipPlugin(event, context, pluginChain)) {
       continue;
     }
 
     // invoke the first plugin in the chain
     const { plugin, with: settings } = pluginChain.uses[0];
+    const isGithubPluginObject = isGithubPlugin(plugin);
     console.log(`Calling handler for event ${event.name}`);
 
     const stateId = crypto.randomUUID();
@@ -73,19 +84,23 @@ async function handleEvent(event: EmitterWebhookEvent, eventHandler: InstanceTyp
       inputs: new Array(pluginChain.uses.length),
     };
 
-    const ref = plugin.ref ?? (await getDefaultBranch(context, plugin.owner, plugin.repo));
+    const ref = isGithubPluginObject ? plugin.ref ?? (await getDefaultBranch(context, plugin.owner, plugin.repo)) : plugin;
     const token = await eventHandler.getToken(event.payload.installation.id);
     const inputs = new DelegatedComputeInputs(stateId, context.key, event.payload, settings, token, ref);
 
     state.inputs[0] = inputs;
     await eventHandler.pluginChainState.put(stateId, state);
 
-    await dispatchWorkflow(context, {
-      owner: plugin.owner,
-      repository: plugin.repo,
-      workflowId: plugin.workflowId,
-      ref: plugin.ref,
-      inputs: inputs.getInputs(),
-    });
+    if (!isGithubPluginObject) {
+      await dispatchWorker(plugin, inputs.getInputs());
+    } else {
+      await dispatchWorkflow(context, {
+        owner: plugin.owner,
+        repository: plugin.repo,
+        workflowId: plugin.workflowId,
+        ref: plugin.ref,
+        inputs: inputs.getInputs(),
+      });
+    }
   }
 }

src/github/handlers/repository-dispatch.ts

@@ -1,7 +1,8 @@
 import { GitHubContext } from "../github-context";
-import { dispatchWorkflow, getDefaultBranch } from "../utils/workflow-dispatch";
+import { dispatchWorker, dispatchWorkflow, getDefaultBranch } from "../utils/workflow-dispatch";
 import { Value } from "@sinclair/typebox/value";
 import { DelegatedComputeInputs, PluginChainState, expressionRegex, pluginOutputSchema } from "../types/plugin";
+import { isGithubPlugin } from "../types/plugin-configuration";
 
 export async function repositoryDispatch(context: GitHubContext<"repository_dispatch">) {
   console.log("Repository dispatch event received", context.payload.client_payload);
@@ -33,7 +34,10 @@ export async function repositoryDispatch(context: GitHubContext<"repository_disp
   }
 
   const currentPlugin = state.pluginChain[state.currentPlugin];
-  if (currentPlugin.plugin.owner !== context.payload.repository.owner.login || currentPlugin.plugin.repo !== context.payload.repository.name) {
+  if (
+    isGithubPlugin(currentPlugin.plugin) &&
+    (currentPlugin.plugin.owner !== context.payload.repository.owner.login || currentPlugin.plugin.repo !== context.payload.repository.name)
+  ) {
     console.error("Plugin chain state does not match payload");
     return;
   }
@@ -48,23 +52,32 @@ export async function repositoryDispatch(context: GitHubContext<"repository_disp
   }
   console.log("Dispatching next plugin", nextPlugin);
 
-  const defaultBranch = await getDefaultBranch(context, nextPlugin.plugin.owner, nextPlugin.plugin.repo);
   const token = await context.eventHandler.getToken(state.eventPayload.installation.id);
-  const ref = nextPlugin.plugin.ref ?? defaultBranch;
   const settings = findAndReplaceExpressions(nextPlugin.with, state);
+  let ref: string;
+  if (isGithubPlugin(nextPlugin.plugin)) {
+    const defaultBranch = await getDefaultBranch(context, nextPlugin.plugin.owner, nextPlugin.plugin.repo);
+    ref = nextPlugin.plugin.ref ?? defaultBranch;
+  } else {
+    ref = nextPlugin.plugin;
+  }
   const inputs = new DelegatedComputeInputs(pluginOutput.state_id, state.eventName, state.eventPayload, settings, token, ref);
 
   state.currentPlugin++;
   state.inputs[state.currentPlugin] = inputs;
   await context.eventHandler.pluginChainState.put(pluginOutput.state_id, state);
 
-  await dispatchWorkflow(context, {
-    owner: nextPlugin.plugin.owner,
-    repository: nextPlugin.plugin.repo,
-    ref: nextPlugin.plugin.ref,
-    workflowId: nextPlugin.plugin.workflowId,
-    inputs: inputs.getInputs(),
-  });
+  if (isGithubPlugin(nextPlugin.plugin)) {
+    await dispatchWorkflow(context, {
+      owner: nextPlugin.plugin.owner,
+      repository: nextPlugin.plugin.repo,
+      ref: nextPlugin.plugin.ref,
+      workflowId: nextPlugin.plugin.workflowId,
+      inputs: inputs.getInputs(),
+    });
+  } else {
+    await dispatchWorker(nextPlugin.plugin, inputs.getInputs());
+  }
 }
 
 function findAndReplaceExpressions(settings: object, state: PluginChainState): Record<string, unknown> {
@@ -78,17 +91,7 @@ function findAndReplaceExpressions(settings: object, state: PluginChainState): R
         continue;
       }
       const parts = matches[1].split(".");
-      if (parts.length !== 3) {
-        throw new Error(`Invalid expression: ${value}`);
-      }
-      const pluginId = parts[0];
-
-      if (parts[1] === "output") {
-        const outputProperty = parts[2];
-        newSettings[key] = getPluginOutputValue(state, pluginId, outputProperty);
-      } else {
-        throw new Error(`Invalid expression: ${value}`);
-      }
+      newSettings[key] = getPluginInfosFromParts(parts, value, state);
     } else if (typeof value === "object" && value !== null) {
       newSettings[key] = findAndReplaceExpressions(value, state);
     } else {
@@ -99,6 +102,20 @@ function findAndReplaceExpressions(settings: object, state: PluginChainState): R
   return newSettings;
 }
 
+function getPluginInfosFromParts(parts: string[], value: string, state: PluginChainState) {
+  if (parts.length !== 3) {
+    throw new Error(`Invalid expression: ${value}`);
+  }
+  const pluginId = parts[0];
+
+  if (parts[1] === "output") {
+    const outputProperty = parts[2];
+    return getPluginOutputValue(state, pluginId, outputProperty);
+  } else {
+    throw new Error(`Invalid expression: ${value}`);
+  }
+}
+
 function getPluginOutputValue(state: PluginChainState, pluginId: string, outputKey: string): unknown {
   const pluginIdx = state.pluginChain.findIndex((plugin) => plugin.id === pluginId);
   if (pluginIdx === -1) {

src/github/types/plugin-configuration.ts

@@ -11,9 +11,21 @@ type GithubPlugin = {
   ref?: string;
 };
 
+const urlRegex = /^https?:\/\/\S+?$/;
+
+export function isGithubPlugin(plugin: string | GithubPlugin): plugin is GithubPlugin {
+  return typeof plugin !== "string";
+}
+
+/**
+ * Transforms the string into a plugin object if the string is not an url
+ */
 function githubPluginType() {
   return T.Transform(T.String())
     .Decode((value) => {
+      if (urlRegex.test(value)) {
+        return value;
+      }
       const matches = value.match(pluginNameRegex);
       if (!matches) {
         throw new Error(`Invalid plugin name: ${value}`);
@@ -26,6 +38,9 @@ function githubPluginType() {
       } as GithubPlugin;
     })
     .Encode((value) => {
+      if (typeof value === "string") {
+        return value;
+      }
       return `${value.owner}/${value.repo}${value.workflowId ? ":" + value.workflowId : ""}${value.ref ? "@" + value.ref : ""}`;
     });
 }

src/github/utils/workflow-dispatch.ts

@@ -32,6 +32,17 @@ export async function dispatchWorkflow(context: GitHubContext, options: Workflow
   });
 }
 
+export async function dispatchWorker(targetUrl: string, payload: WorkflowDispatchOptions["inputs"]) {
+  const result = await fetch(targetUrl, {
+    body: JSON.stringify(payload),
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+    },
+  });
+  return result.json();
+}
+
 export async function getDefaultBranch(context: GitHubContext, owner: string, repository: string) {
   const octokit = await getInstallationOctokitForOrg(context, owner); // we cannot access other repos with the context's octokit
   const repo = await octokit.repos.get({

tests/main.test.ts

@@ -1,7 +1,13 @@
 /* eslint-disable @typescript-eslint/naming-convention */
-
 // @ts-expect-error package name is correct, TypeScript doesn't recognize it
 import { afterAll, afterEach, beforeAll, describe, expect, it, jest, mock, spyOn } from "bun:test";
+import { config } from "dotenv";
+import { GitHubContext } from "../src/github/github-context";
+import { GitHubEventHandler } from "../src/github/github-event-handler";
+import { getConfig } from "../src/github/utils/config";
+import worker from "../src/worker";
+import { server } from "./__mocks__/node";
+
 mock.module("@octokit/webhooks", () => ({
   Webhooks: WebhooksMocked,
 }));
@@ -22,13 +28,6 @@ class WebhooksMocked {
   receive(_: unknown) {}
 }
 
-import { config } from "dotenv";
-import { GitHubContext } from "../src/github/github-context";
-import { GitHubEventHandler } from "../src/github/github-event-handler";
-import { getConfig } from "../src/github/utils/config";
-import worker from "../src/worker";
-import { server } from "./__mocks__/node";
-
 config({ path: ".dev.vars" });
 
 beforeAll(() => {
@@ -64,9 +63,9 @@ describe("Worker tests", () => {
       },
     });
     const res = await worker.fetch(req, {
-      WEBHOOK_SECRET: process.env.WEBHOOK_SECRET,
-      APP_ID: process.env.APP_ID,
-      PRIVATE_KEY: process.env.PRIVATE_KEY,
+      WEBHOOK_SECRET: "webhook-secret",
+      APP_ID: "app-id",
+      PRIVATE_KEY: "private-key",
       PLUGIN_CHAIN_STATE: {} as KVNamespace,
     });
     expect(res.status).toEqual(200);