diff --git a/.eslintrc b/.eslintrc index 5996f04..338c3bb 100644 --- a/.eslintrc +++ b/.eslintrc @@ -15,7 +15,7 @@ "constructor-super": "error", "no-invalid-this": "off", "@typescript-eslint/no-invalid-this": ["error"], - "no-restricted-syntax": ["error", "ForInStatement"], + "no-restricted-syntax": ["error"], "use-isnan": "error", "@typescript-eslint/no-unused-vars": [ "error", diff --git a/README.md b/README.md index 0d0b4c4..6957131 100644 --- a/README.md +++ b/README.md @@ -6,25 +6,25 @@ The kernel is designed to: - Interface with plugins (GitHub Actions) for longer running processes. - Run on Cloudflare Workers. -## Environment variables +## Environment Variables -- `PRIVATE_KEY` - You need to obtain a private key from your GitHub App settings and convert it to Public-Key Cryptography Standards #8 (PKCS#8) format. You can use the following command to perform this conversion and append the result to your `.dev.vars` file: +- **`PRIVATE_KEY`** + Obtain a private key from your GitHub App settings and convert it to the Public-Key Cryptography Standards #8 (PKCS#8) format. Use the following command to perform this conversion and append the result to your `.dev.vars` file: - ```sh - echo "PRIVATE_KEY=\"$(openssl pkcs8 -topk8 -inform PEM -outform PEM -nocrypt -in YOUR_PRIVATE_KEY.PEM | awk 'BEGIN{ORS="\\n"} 1')\"" >> .dev.vars - ``` + ```sh + echo "PRIVATE_KEY=\"$(openssl pkcs8 -topk8 -inform PEM -outform PEM -nocrypt -in YOUR_PRIVATE_KEY.PEM | awk 'BEGIN{ORS="\\n"} 1')\"" >> .dev.vars + ``` -###### Please replace `YOUR_PRIVATE_KEY.PEM` with the path to your actual PEM file when running the command. + **Note:** Replace `YOUR_PRIVATE_KEY.PEM` with the path to your actual PEM file when running the command. -- `WEBHOOK_SECRET` - This should be set in your GitHub App settings and also here. +- **`WEBHOOK_SECRET`** + Set this value in both your GitHub App settings and here. -- `APP_ID` - You can find this in your GitHub App settings. +- **`APP_ID`** + Retrieve this from your GitHub App settings. -- `WEBHOOK_PROXY_URL` (only for development) - You need to obtain a webhook URL at and set it in the GitHub App settings. +- **`WEBHOOK_PROXY_URL` (only for development)** + Obtain a webhook URL at [smee.io](https://smee.io/) and set it in your GitHub App settings. ### Quick Start @@ -32,10 +32,69 @@ The kernel is designed to: ```bash git clone https://github.com/ubiquity/ubiquibot-kernel cd ubiquibot-kernel -bun +bun install bun dev ``` +### Deploying to Cloudflare Workers + +1. **Install Dependencies:** + - Execute `bun install` to install the required dependencies. + +2. **Create a Github App:** + - Generate a Github App and configure its settings. + - Navigate to app settings and click `Permissions & events`. + - Ensure the app is subscribed to all events with the following permissions: + + Repository permissions: + - Actions: Read & Write + - Contents: Read & Write + - Issues: Read & Write + - Pull Requests: Read & Write + + Organization permissions: + - Members: Read only + +3. **Cloudflare Account Setup:** + - If not done already, create a Cloudflare account. + - Run `npx wrangler login` to log in. + +4. **Create a KV Namespace:** + - Generate a KV namespace using `npx wrangler kv:namespace create PLUGIN_CHAIN_STATE`. + - Copy the generated ID and paste it under `[env.dev]` in `wrangler.toml`. + +5. **Manage Secrets:** + - Add (env) secrets using `npx wrangler secret put --env dev`. + - For the private key, execute the following (replace `YOUR_PRIVATE_KEY.PEM` with the actual PEM file path): + + ```sh + echo $(openssl pkcs8 -topk8 -inform PEM -outform PEM -nocrypt -in YOUR_PRIVATE_KEY.PEM) | npx wrangler secret put PRIVATE_KEY --env dev + ``` + +6. **Deploy the Kernel:** + - Execute `bun run deploy-dev` to deploy the kernel. + +### Plugin Input and Output + +#### Input + +Inputs are received within the workflow, triggered by the `workflow_dispatch` event. The plugin is designed to handle the following inputs: + +- `stateId`: An identifier used to track the state of plugin chain execution in Cloudflare KV. It is crucial to pass this identifier back in the output. +- `eventName`: The complete name of the event (e.g., `issue_comment.created`). +- `eventPayload`: The payload associated with the event. +- `settings`: A string containing JSON with settings specific to your plugin. The plugin itself defines these settings. +- `authToken`: A JWT token for accessing GitHub's API to the repository where the event occurred. +- `ref`: A reference (branch, tag, commit SHA) indicating the version of the plugin to be utilized. + +#### Output + +Data is returned using the `repository_dispatch` event on the plugin's repository, and the output is structured within the `client_payload`. +The `event_type` must be set to `return_data_to_ubiquibot_kernel`. + +- `state_id`: The state ID passed in the inputs must be included here. +- `output`: A string containing JSON with custom output, defined by the plugin itself. + ## Testing ### Jest diff --git a/bun.lockb b/bun.lockb index b970341..b7fcc86 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/package.json b/package.json index 05a0b76..d21c0f2 100644 --- a/package.json +++ b/package.json @@ -1,90 +1,91 @@ { - "name": "ubiquibot-kernel", - "version": "0.0.0", - "private": false, - "description": "The kernel for UbiquiBot.", - "main": "src/worker.ts", - "author": "Ubiquity DAO", - "license": "MIT", - "engines": { - "node": ">=20.10.0" - }, - "scripts": { - "dev": "run-p worker proxy", - "predev": "lsof -i tcp:8787 | grep LISTEN | awk '{print $2}' | xargs kill -9", - "format": "run-s format:lint format:prettier format:cspell", - "format:lint": "eslint --fix .", - "format:prettier": "prettier --write .", - "format:cspell": "cspell **/*", - "prepare": "husky install", - "deploy": "wrangler deploy", - "worker": "wrangler dev --port 8787", - "proxy": "tsx src/proxy.ts", - "knip": "knip --config .github/knip.ts", - "knip-ci": "knip --no-exit-code --reporter json --config .github/knip.ts", - "test": "jest --setupFiles dotenv/config --coverage" - }, - "keywords": [ - "typescript", - "template", - "dao", - "ubiquity", - "open-source" - ], - "dependencies": { - "@octokit/webhooks": "^12.0.10", - "@octokit/webhooks-types": "^7.3.1", - "@sinclair/typebox": "^0.32.5", - "create-cloudflare": "^2.8.3", - "octokit": "^3.1.2", - "smee-client": "^2.0.0", - "universal-github-app-jwt": "^2.0.5", - "dotenv": "^16.4.4" - }, - "devDependencies": { - "@cloudflare/workers-types": "^4.20240117.0", - "@commitlint/cli": "^18.6.1", - "@commitlint/config-conventional": "^18.6.2", - "@cspell/dict-node": "^4.0.3", - "@cspell/dict-software-terms": "^3.3.18", - "@cspell/dict-typescript": "^3.1.2", - "@mswjs/data": "0.16.1", - "@types/jest": "29.5.12", - "@types/node": "^20.11.19", - "@typescript-eslint/eslint-plugin": "^7.0.1", - "@typescript-eslint/parser": "^7.0.1", - "cspell": "^8.4.0", - "esbuild": "^0.20.1", - "eslint": "^8.56.0", - "eslint-config-prettier": "^9.1.0", - "eslint-plugin-prettier": "^5.1.3", - "eslint-plugin-sonarjs": "^0.24.0", - "husky": "^9.0.11", - "jest": "29.7.0", - "jest-junit": "16.0.0", - "knip": "^5.0.1", - "lint-staged": "^15.2.2", - "npm-run-all": "^4.1.5", - "prettier": "^3.2.5", - "ts-jest": "29.1.2", - "ts-node": "10.9.2", - "tsx": "^4.7.1", - "typescript": "^5.3.3", - "wrangler": "^3.23.0" - }, - "lint-staged": { - "*.ts": [ - "prettier --write", - "eslint --fix" - ], - "src/**.*": [ - "cspell" - ] - }, - "commitlint": { - "extends": [ - "@commitlint/config-conventional" - ] - }, - "packageManager": "bun@1.0.23" + "name": "ubiquibot-kernel", + "version": "0.0.0", + "private": false, + "description": "The kernel for UbiquiBot.", + "main": "src/worker.ts", + "author": "Ubiquity DAO", + "license": "MIT", + "engines": { + "node": ">=20.10.0" + }, + "scripts": { + "dev": "run-p worker proxy", + "predev": "lsof -i tcp:8787 | grep LISTEN | awk '{print $2}' | xargs kill -9", + "format": "run-s format:lint format:prettier format:cspell", + "format:lint": "eslint --fix .", + "format:prettier": "prettier --write .", + "format:cspell": "cspell **/*", + "prepare": "husky install", + "deploy-dev": "wrangler deploy --env dev", + "deploy-production": "wrangler deploy --env production", + "worker": "wrangler dev --env dev --port 8787", + "proxy": "tsx src/proxy.ts", + "knip": "knip --config .github/knip.ts", + "knip-ci": "knip --no-exit-code --reporter json --config .github/knip.ts", + "test": "jest --setupFiles dotenv/config --coverage" + }, + "keywords": [ + "typescript", + "template", + "dao", + "ubiquity", + "open-source" + ], + "dependencies": { + "@octokit/webhooks": "^12.0.10", + "@octokit/webhooks-types": "^7.3.1", + "@sinclair/typebox": "^0.32.5", + "create-cloudflare": "^2.8.3", + "octokit": "^3.1.2", + "smee-client": "^2.0.0", + "universal-github-app-jwt": "^2.0.5", + "dotenv": "^16.4.4" + }, + "devDependencies": { + "@cloudflare/workers-types": "^4.20240117.0", + "@commitlint/cli": "^18.6.1", + "@commitlint/config-conventional": "^18.6.2", + "@cspell/dict-node": "^4.0.3", + "@cspell/dict-software-terms": "^3.3.18", + "@cspell/dict-typescript": "^3.1.2", + "@mswjs/data": "0.16.1", + "@types/jest": "29.5.12", + "@types/node": "^20.11.19", + "@typescript-eslint/eslint-plugin": "^7.0.1", + "@typescript-eslint/parser": "^7.0.1", + "cspell": "^8.4.0", + "esbuild": "^0.20.1", + "eslint": "^8.56.0", + "eslint-config-prettier": "^9.1.0", + "eslint-plugin-prettier": "^5.1.3", + "eslint-plugin-sonarjs": "^0.24.0", + "husky": "^9.0.11", + "jest": "29.7.0", + "jest-junit": "16.0.0", + "knip": "^5.0.1", + "lint-staged": "^15.2.2", + "npm-run-all": "^4.1.5", + "prettier": "^3.2.5", + "ts-jest": "29.1.2", + "ts-node": "10.9.2", + "tsx": "^4.7.1", + "typescript": "^5.3.3", + "wrangler": "^3.23.0" + }, + "lint-staged": { + "*.ts": [ + "prettier --write", + "eslint --fix" + ], + "src/**.*": [ + "cspell" + ] + }, + "commitlint": { + "extends": [ + "@commitlint/config-conventional" + ] + }, + "packageManager": "bun@1.0.23" } diff --git a/src/github/github-context.ts b/src/github/github-context.ts index f5157ec..56fa556 100644 --- a/src/github/github-context.ts +++ b/src/github/github-context.ts @@ -1,5 +1,6 @@ import { EmitterWebhookEvent as WebhookEvent, EmitterWebhookEventName as WebhookEventName } from "@octokit/webhooks"; import { customOctokit } from "./github-client"; +import { GitHubEventHandler } from "./github-event-handler"; export class GitHubContext { public key: WebhookEventName; @@ -7,8 +8,10 @@ export class GitHubContext { public id: string; public payload: WebhookEvent["payload"]; public octokit: InstanceType; + public eventHandler: InstanceType; - constructor(event: WebhookEvent, octokit: InstanceType) { + constructor(eventHandler: InstanceType, event: WebhookEvent, octokit: InstanceType) { + this.eventHandler = eventHandler; this.name = event.name; this.id = event.id; this.payload = event.payload; diff --git a/src/github/github-event-handler.ts b/src/github/github-event-handler.ts index fbbb2d5..235aff5 100644 --- a/src/github/github-event-handler.ts +++ b/src/github/github-event-handler.ts @@ -1,11 +1,15 @@ -import { Webhooks } from "@octokit/webhooks"; +import { EmitterWebhookEvent, Webhooks } from "@octokit/webhooks"; import { customOctokit } from "./github-client"; import { GitHubContext, SimplifiedContext } from "./github-context"; +import { createAppAuth } from "@octokit/auth-app"; +import { CloudflareKV } from "./utils/cloudflare-kv"; +import { PluginChainState } from "./types/plugin"; export type Options = { webhookSecret: string; appId: string | number; privateKey: string; + pluginChainState: CloudflareKV; }; export class GitHubEventHandler { @@ -13,6 +17,7 @@ export class GitHubEventHandler { public on: Webhooks["on"]; public onAny: Webhooks["onAny"]; public onError: Webhooks["onError"]; + public pluginChainState: CloudflareKV; private _webhookSecret: string; private _privateKey: string; @@ -22,24 +27,11 @@ export class GitHubEventHandler { this._privateKey = options.privateKey; this._appId = Number(options.appId); this._webhookSecret = options.webhookSecret; + this.pluginChainState = options.pluginChainState; this.webhooks = new Webhooks({ secret: this._webhookSecret, - transform: (event) => { - let installationId: number | undefined = undefined; - if ("installation" in event.payload) { - installationId = event.payload.installation?.id; - } - const octokit = new customOctokit({ - auth: { - appId: this._appId, - privateKey: this._privateKey, - installationId: installationId, - }, - }); - - return new GitHubContext(event, octokit); - }, + transform: (event) => this.transformEvent(event), // it is important to use an arrow function here to keep the context of `this` }); this.on = this.webhooks.on; @@ -53,4 +45,42 @@ export class GitHubEventHandler { console.error(error); }); } + + transformEvent(event: EmitterWebhookEvent) { + if ("installation" in event.payload && event.payload.installation?.id !== undefined) { + const octokit = this.getAuthenticatedOctokit(event.payload.installation.id); + return new GitHubContext(this, event, octokit); + } else { + const octokit = this.getUnauthenticatedOctokit(); + return new GitHubContext(this, event, octokit); + } + } + + getAuthenticatedOctokit(installationId: number) { + return new customOctokit({ + auth: { + appId: this._appId, + privateKey: this._privateKey, + installationId: installationId, + }, + }); + } + + getUnauthenticatedOctokit() { + return new customOctokit({ + auth: { + appId: this._appId, + privateKey: this._privateKey, + }, + }); + } + + async getToken(installationId: number) { + const auth = createAppAuth({ + appId: this._appId, + privateKey: this._privateKey, + }); + const token = await auth({ type: "installation", installationId }); + return token.token; + } } diff --git a/src/github/handlers/index.ts b/src/github/handlers/index.ts index f349638..e84843d 100644 --- a/src/github/handlers/index.ts +++ b/src/github/handlers/index.ts @@ -1,6 +1,82 @@ +import { EmitterWebhookEvent } from "@octokit/webhooks"; import { GitHubEventHandler } from "../github-event-handler"; +import { getConfig } from "../utils/config"; import { issueCommentCreated } from "./issue-comment/created"; +import { repositoryDispatch } from "./repository-dispatch"; +import { dispatchWorkflow, getDefaultBranch } from "../utils/workflow-dispatch"; +import { DelegatedComputeInputs } from "../types/plugin"; -export function bindHandlers(webhooks: GitHubEventHandler) { - webhooks.on("issue_comment.created", issueCommentCreated); +function tryCatchWrapper(fn: (event: EmitterWebhookEvent) => unknown) { + return async (event: EmitterWebhookEvent) => { + try { + await fn(event); + } catch (error) { + console.error("Error in event handler", error); + } + }; +} + +export function bindHandlers(eventHandler: GitHubEventHandler) { + eventHandler.on("issue_comment.created", issueCommentCreated); + eventHandler.on("repository_dispatch", repositoryDispatch); + eventHandler.onAny(tryCatchWrapper((event) => handleEvent(event, eventHandler))); // onAny should also receive GithubContext but the types in octokit/webhooks are weird +} + +async function handleEvent(event: EmitterWebhookEvent, eventHandler: InstanceType) { + const context = eventHandler.transformEvent(event); + + const config = await getConfig(context); + + if (!config) { + console.log("No config found"); + return; + } + + if (!("installation" in event.payload) || event.payload.installation?.id === undefined) { + console.log("No installation found"); + return; + } + + const pluginChains = config.plugins[context.key].concat(config.plugins["*"]); + + if (pluginChains.length === 0) { + console.log(`No handler found for event ${event.name}`); + return; + } + + for (const pluginChain of pluginChains) { + if (pluginChain.skipBotEvents && "sender" in event.payload && event.payload.sender?.type === "Bot") { + continue; + } + // invoke the first plugin in the chain + const { plugin, with: settings } = pluginChain.uses[0]; + console.log(`Calling handler for event ${event.name}`); + + const stateId = crypto.randomUUID(); + + const state = { + eventId: context.id, + eventName: context.key, + eventPayload: event.payload, + currentPlugin: 0, + pluginChain: pluginChain.uses, + outputs: new Array(pluginChain.uses.length), + inputs: new Array(pluginChain.uses.length), + }; + + const ref = plugin.ref ?? (await getDefaultBranch(context, plugin.owner, plugin.repo)); + const token = await eventHandler.getToken(event.payload.installation.id); + const inputs = new DelegatedComputeInputs(stateId, context.key, event.payload, settings, token, ref); + + state.inputs[0] = inputs; + await eventHandler.pluginChainState.put(stateId, state); + + await dispatchWorkflow(context, { + owner: plugin.owner, + repository: plugin.repo, + workflowId: plugin.workflowId, + ref: plugin.ref, + inputs: inputs.getInputs(), + }); + } } diff --git a/src/github/handlers/repository-dispatch.ts b/src/github/handlers/repository-dispatch.ts new file mode 100644 index 0000000..6ca1f3b --- /dev/null +++ b/src/github/handlers/repository-dispatch.ts @@ -0,0 +1,118 @@ +import { GitHubContext } from "../github-context"; +import { dispatchWorkflow, getDefaultBranch } from "../utils/workflow-dispatch"; +import { Value } from "@sinclair/typebox/value"; +import { DelegatedComputeInputs, PluginChainState, expressionRegex, pluginOutputSchema } from "../types/plugin"; +import { PluginChain } from "../types/config"; + +export async function repositoryDispatch(context: GitHubContext<"repository_dispatch">) { + console.log("Repository dispatch event received", context.payload.client_payload); + + if (context.payload.action !== "return_data_to_ubiquibot_kernel") { + console.log("Skipping non-ubiquibot event"); + return; + } + + let pluginOutput; + + try { + pluginOutput = Value.Decode(pluginOutputSchema, context.payload.client_payload); + } catch (error) { + console.error("Cannot decode plugin output", error); + throw error; + } + console.log("Plugin output", pluginOutput); + + const state = await context.eventHandler.pluginChainState.get(pluginOutput.state_id); + if (!state) { + console.error("No state found for plugin chain"); + return; + } + + if (!("installation" in state.eventPayload) || state.eventPayload.installation?.id === undefined) { + console.error("No installation found"); + return; + } + + const currentPlugin = state.pluginChain[state.currentPlugin]; + if (currentPlugin.plugin.owner !== context.payload.repository.owner.login || currentPlugin.plugin.repo !== context.payload.repository.name) { + console.error("Plugin chain state does not match payload"); + return; + } + state.outputs[state.currentPlugin] = pluginOutput; + console.log("State", state); + + const nextPlugin = state.pluginChain[state.currentPlugin + 1]; + if (!nextPlugin) { + console.log("No more plugins to call"); + await context.eventHandler.pluginChainState.put(pluginOutput.state_id, state); + return; + } + console.log("Dispatching next plugin", nextPlugin); + + const defaultBranch = await getDefaultBranch(context, nextPlugin.plugin.owner, nextPlugin.plugin.repo); + const token = await context.eventHandler.getToken(state.eventPayload.installation.id); + const ref = nextPlugin.plugin.ref ?? defaultBranch; + const settings = findAndReplaceExpressions(nextPlugin, state); + const inputs = new DelegatedComputeInputs(pluginOutput.state_id, state.eventName, state.eventPayload, settings, token, ref); + + state.currentPlugin++; + state.inputs[state.currentPlugin] = inputs; + await context.eventHandler.pluginChainState.put(pluginOutput.state_id, state); + + await dispatchWorkflow(context, { + owner: nextPlugin.plugin.owner, + repository: nextPlugin.plugin.repo, + ref: nextPlugin.plugin.ref, + workflowId: nextPlugin.plugin.workflowId, + inputs: inputs.getInputs(), + }); +} + +function findAndReplaceExpressions(plugin: PluginChain[0], state: PluginChainState): Record { + const settings: Record = {}; + + for (const key in plugin.with) { + const value = plugin.with[key]; + + if (typeof value === "string") { + const matches = value.match(expressionRegex); + if (!matches) { + settings[key] = value; + continue; + } + const parts = matches[1].split("."); + if (parts.length !== 3) { + throw new Error(`Invalid expression: ${value}`); + } + const pluginId = parts[0]; + + if (parts[1] === "output") { + const outputProperty = parts[2]; + settings[key] = getPluginOutputValue(state, pluginId, outputProperty); + } else { + throw new Error(`Invalid expression: ${value}`); + } + } else { + settings[key] = value; + } + } + + return settings; +} + +function getPluginOutputValue(state: PluginChainState, pluginId: string, outputKey: string): unknown { + const pluginIdx = state.pluginChain.findIndex((plugin) => plugin.id === pluginId); + if (pluginIdx === -1) { + throw new Error(`Plugin ${pluginId} not found in the chain`); + } + if (pluginIdx > state.currentPlugin) { + throw new Error(`You cannot use output values from plugin ${pluginId} because it's not been called yet`); + } + + const outputValue = state.outputs[pluginIdx].output[outputKey]; + if (outputValue === undefined) { + throw new Error(`Output key '${outputKey}' not found for plugin ${pluginId}`); + } + + return outputValue; +} diff --git a/src/github/types/config.ts b/src/github/types/config.ts new file mode 100644 index 0000000..58e153d --- /dev/null +++ b/src/github/types/config.ts @@ -0,0 +1,61 @@ +import { Type as T } from "@sinclair/typebox"; +import { StaticDecode } from "@sinclair/typebox"; +import { githubWebhookEvents } from "./webhook-events"; + +const pluginNameRegex = new RegExp("^([0-9a-zA-Z-._]+)/([0-9a-zA-Z-._]+)(?::([0-9a-zA-Z-._]+))?(?:@([0-9a-zA-Z-._]+))?$"); + +type GithubPlugin = { + owner: string; + repo: string; + workflowId: string; + ref?: string; +}; + +function githubPluginType() { + return T.Transform(T.String()) + .Decode((value) => { + const matches = value.match(pluginNameRegex); + if (!matches) { + throw new Error(`Invalid plugin name: ${value}`); + } + return { + owner: matches[1], + repo: matches[2], + workflowId: matches[3] || "compute.yml", + ref: matches[4] || undefined, + } as GithubPlugin; + }) + .Encode((value) => { + return `${value.owner}/${value.repo}${value.workflowId ? ":" + value.workflowId : ""}${value.ref ? "@" + value.ref : ""}`; + }); +} + +const pluginChainSchema = T.Array( + T.Object({ + id: T.Optional(T.String()), + plugin: githubPluginType(), + type: T.Union([T.Literal("github")], { default: "github" }), + with: T.Record(T.String(), T.Unknown()), + }), + { minItems: 1 } +); + +export type PluginChain = StaticDecode; + +const handlerSchema = T.Array( + T.Object({ + name: T.Optional(T.String()), + description: T.Optional(T.String()), + command: T.Optional(T.String()), + example: T.Optional(T.String()), + uses: pluginChainSchema, + skipBotEvents: T.Boolean({ default: true }), + }), + { default: [] } +); + +export const configSchema = T.Object({ + plugins: T.Record(T.Enum(githubWebhookEvents), handlerSchema, { default: {} }), +}); + +export type Config = StaticDecode; diff --git a/src/github/types/env.ts b/src/github/types/env.ts index 7548be2..d929f23 100644 --- a/src/github/types/env.ts +++ b/src/github/types/env.ts @@ -1,4 +1,7 @@ import { Type as T, type Static } from "@sinclair/typebox"; export const envSchema = T.Object({ WEBHOOK_SECRET: T.String(), APP_ID: T.String(), PRIVATE_KEY: T.String() }); -export type Env = Static; + +export type Env = Static & { + PLUGIN_CHAIN_STATE: KVNamespace; +}; diff --git a/src/github/types/plugin.ts b/src/github/types/plugin.ts new file mode 100644 index 0000000..1edaa32 --- /dev/null +++ b/src/github/types/plugin.ts @@ -0,0 +1,72 @@ +import { EmitterWebhookEvent, EmitterWebhookEventName } from "@octokit/webhooks"; +import { PluginChain } from "./config"; +import { StaticDecode, Type } from "@sinclair/typebox"; + +export const expressionRegex = /^\s*\${{\s*(\S+)\s*}}\s*$/; + +function jsonString() { + return Type.Transform(Type.String()) + .Decode((value) => JSON.parse(value) as Record) + .Encode((value) => JSON.stringify(value)); +} + +export const pluginOutputSchema = Type.Object({ + state_id: Type.String(), // Github forces snake_case + output: jsonString(), +}); + +export type PluginOutput = StaticDecode; + +export class DelegatedComputeInputs { + public stateId: string; + public eventName: T; + public eventPayload: EmitterWebhookEvent["payload"]; + public settings: unknown; + public authToken: string; + public ref: string; + + constructor(stateId: string, eventName: T, eventPayload: EmitterWebhookEvent["payload"], settings: unknown, authToken: string, ref: string) { + this.stateId = stateId; + this.eventName = eventName; + this.eventPayload = eventPayload; + this.settings = settings; + this.authToken = authToken; + this.ref = ref; + } + + public getInputs() { + return { + stateId: this.stateId, + eventName: this.eventName, + eventPayload: JSON.stringify(this.eventPayload), + settings: JSON.stringify(this.settings), + authToken: this.authToken, + ref: this.ref, + }; + } +} + +export type PluginChainState = { + eventId: string; + eventName: T; + eventPayload: EmitterWebhookEvent["payload"]; + currentPlugin: number; + pluginChain: PluginChain; + inputs: DelegatedComputeInputs[]; + outputs: PluginOutput[]; +}; + +// convert top level properties to string +export function convertToString(obj: Record): Record { + const newObj: Record = {}; + for (let i = 0; i < Object.keys(obj).length; i++) { + const key = Object.keys(obj)[i]; + const val = obj[key]; + if (typeof val === "string") { + newObj[key] = val; + } else { + newObj[key] = JSON.stringify(val); + } + } + return newObj; +} diff --git a/src/github/types/webhook-events.ts b/src/github/types/webhook-events.ts index 51e00fa..797f9cf 100644 --- a/src/github/types/webhook-events.ts +++ b/src/github/types/webhook-events.ts @@ -1,2 +1,28 @@ -import { emitterEventNames } from "@octokit/webhooks/dist-types/generated/webhook-names.js"; -export type GitHubEventClassName = (typeof emitterEventNames)[number]; +import { emitterEventNames, EmitterWebhookEventName as GitHubEventClassName } from "@octokit/webhooks"; + +export type EventName = GitHubEventClassName | "*"; +export const eventNames: EventName[] = [...emitterEventNames, "*"]; + +type Formatted = T extends `${infer Prefix}.${infer Rest}` ? `${Prefix}_${Formatted}` : T; + +type GithubEventWebHookEvents = { + [K in EventName as Formatted>]: K; +}; + +type Prettify = { + [K in keyof T]: T[K]; + // this just spreads the object into a type + + // we need to use {} otherwise it'll type it as an object + // eslint-disable-next-line @typescript-eslint/ban-types +} & {}; + +export const githubWebhookEvents: Prettify = emitterEventNames.reduce( + (acc: GithubEventWebHookEvents, cur) => { + const formatted = cur.replace(/\./g, "_"); + const upper = formatted.toUpperCase() as Formatted>; + acc[upper] = cur as Extract>; + return acc; + }, + { "*": "*" } as GithubEventWebHookEvents +); diff --git a/src/github/utils/cloudflare-kv.ts b/src/github/utils/cloudflare-kv.ts new file mode 100644 index 0000000..f6dc1a2 --- /dev/null +++ b/src/github/utils/cloudflare-kv.ts @@ -0,0 +1,15 @@ +export class CloudflareKV { + private _kv: KVNamespace; + + constructor(kv: KVNamespace) { + this._kv = kv; + } + + get(id: string): Promise { + return this._kv.get(id, "json"); + } + + put(id: string, state: T): Promise { + return this._kv.put(id, JSON.stringify(state)); + } +} diff --git a/src/github/utils/config.ts b/src/github/utils/config.ts new file mode 100644 index 0000000..ea7ad77 --- /dev/null +++ b/src/github/utils/config.ts @@ -0,0 +1,118 @@ +import { Value } from "@sinclair/typebox/value"; +import { GitHubContext } from "../github-context"; +import YAML from "yaml"; +import { Config, configSchema } from "../types/config"; +import { expressionRegex } from "../types/plugin"; +import { eventNames } from "../types/webhook-events"; + +const UBIQUIBOT_CONFIG_FULL_PATH = ".github/ubiquibot-config.yml"; + +export async function getConfig(context: GitHubContext): Promise { + const payload = context.payload; + if (!("repository" in payload) || !payload.repository) throw new Error("Repository is not defined"); + + const _repoConfig = parseYaml( + await download({ + context, + repository: payload.repository.name, + owner: payload.repository.owner.login, + }) + ); + if (!_repoConfig) return null; + + let config: Config; + try { + config = Value.Decode(configSchema, Value.Default(configSchema, _repoConfig)); + } catch (error) { + console.error("Error decoding config", error); + return null; + } + + checkPluginChains(config); + + return config; +} + +function checkPluginChains(config: Config) { + for (const eventName of eventNames) { + const plugins = config.plugins[eventName]; + for (const plugin of plugins) { + const allIds = checkPluginChainUniqueIds(plugin); + checkPluginChainExpressions(plugin, allIds); + } + } +} + +function checkPluginChainUniqueIds(plugin: Config["plugins"]["*"][0]) { + const allIds = new Set(); + for (const use of plugin.uses) { + if (!use.id) continue; + + if (allIds.has(use.id)) { + throw new Error(`Duplicate id ${use.id} in plugin chain`); + } + allIds.add(use.id); + } + return allIds; +} + +function checkPluginChainExpressions(plugin: Config["plugins"]["*"][0], allIds: Set) { + const calledIds = new Set(); + for (const use of plugin.uses) { + if (!use.id) continue; + for (const key in use.with) { + const value = use.with[key]; + if (typeof value !== "string") continue; + checkExpression(value, allIds, calledIds); + } + calledIds.add(use.id); + } +} + +function checkExpression(value: string, allIds: Set, calledIds: Set) { + const matches = value.match(expressionRegex); + if (!matches) { + return; + } + const parts = matches[1].split("."); + if (parts.length !== 3) { + throw new Error(`Invalid expression: ${value}`); + } + const id = parts[0]; + if (!allIds.has(id)) { + throw new Error(`Expression ${value} refers to non-existent id ${id}`); + } + if (!calledIds.has(id)) { + throw new Error(`Expression ${value} refers to plugin id ${id} before it is called`); + } + if (parts[1] !== "output") { + throw new Error(`Invalid expression: ${value}`); + } +} + +async function download({ context, repository, owner }: { context: GitHubContext; repository: string; owner: string }): Promise { + if (!repository || !owner) throw new Error("Repo or owner is not defined"); + try { + const { data } = await context.octokit.rest.repos.getContent({ + owner, + repo: repository, + path: UBIQUIBOT_CONFIG_FULL_PATH, + mediaType: { format: "raw" }, + }); + return data as unknown as string; // this will be a string if media format is raw + } catch (err) { + return null; + } +} + +export function parseYaml(data: null | string) { + try { + if (data) { + const parsedData = YAML.parse(data); + return parsedData ?? null; + } + } catch (error) { + console.error("Error parsing YAML", error); + } + return null; +} diff --git a/src/github/utils/workflow-dispatch.ts b/src/github/utils/workflow-dispatch.ts new file mode 100644 index 0000000..d91d24e --- /dev/null +++ b/src/github/utils/workflow-dispatch.ts @@ -0,0 +1,42 @@ +import { customOctokit } from "../github-client"; +import { GitHubContext } from "../github-context"; + +interface WorkflowDispatchOptions { + owner: string; + repository: string; + workflowId: string; + ref?: string; + inputs?: { [key: string]: string }; +} + +async function getInstallationOctokitForOrg(context: GitHubContext, owner: string): Promise> { + const installations = await context.octokit.apps.listInstallations(); + const installation = installations.data.find((inst) => inst.account?.login === owner); + + if (!installation) { + throw new Error(`No installation found for owner: ${owner}`); + } + + return context.eventHandler.getAuthenticatedOctokit(installation.id); +} + +export async function dispatchWorkflow(context: GitHubContext, options: WorkflowDispatchOptions) { + const authenticatedOctokit = await getInstallationOctokitForOrg(context, options.owner); + + return await authenticatedOctokit.actions.createWorkflowDispatch({ + owner: options.owner, + repo: options.repository, + workflow_id: options.workflowId, + ref: options.ref ?? (await getDefaultBranch(context, options.owner, options.repository)), + inputs: options.inputs, + }); +} + +export async function getDefaultBranch(context: GitHubContext, owner: string, repository: string) { + const octokit = await getInstallationOctokitForOrg(context, owner); // we cannot access other repos with the context's octokit + const repo = await octokit.repos.get({ + owner: owner, + repo: repository, + }); + return repo.data.default_branch; +} diff --git a/src/worker.ts b/src/worker.ts index 4b8cf8b..adf3adc 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -3,6 +3,8 @@ import { Value } from "@sinclair/typebox/value"; import { GitHubEventHandler } from "./github/github-event-handler"; import { bindHandlers } from "./github/handlers"; import { Env, envSchema } from "./github/types/env"; +import { CloudflareKV } from "./github/utils/cloudflare-kv"; + export default { async fetch(request: Request, env: Env): Promise { try { @@ -10,7 +12,12 @@ export default { const eventName = getEventName(request); const signatureSHA256 = getSignature(request); const id = getId(request); - const eventHandler = new GitHubEventHandler({ webhookSecret: env.WEBHOOK_SECRET, appId: env.APP_ID, privateKey: env.PRIVATE_KEY }); + const eventHandler = new GitHubEventHandler({ + webhookSecret: env.WEBHOOK_SECRET, + appId: env.APP_ID, + privateKey: env.PRIVATE_KEY, + pluginChainState: new CloudflareKV(env.PLUGIN_CHAIN_STATE), + }); bindHandlers(eventHandler); await eventHandler.webhooks.verifyAndReceive({ id, name: eventName, payload: await request.text(), signature: signatureSHA256 }); return new Response("ok\n", { status: 200, headers: { "content-type": "text/plain" } }); @@ -19,6 +26,7 @@ export default { } }, }; + function handleUncaughtError(error: unknown) { console.error(error); let status = 500; @@ -38,6 +46,7 @@ function validateEnv(env: Env): void { throw new Error("Invalid environment variables"); } } + function getEventName(request: Request): WebhookEventName { const eventName = request.headers.get("x-github-event"); if (!eventName || !emitterEventNames.includes(eventName as WebhookEventName)) { @@ -45,6 +54,7 @@ function getEventName(request: Request): WebhookEventName { } return eventName as WebhookEventName; } + function getSignature(request: Request): string { const signatureSHA256 = request.headers.get("x-hub-signature-256"); if (!signatureSHA256) { @@ -52,6 +62,7 @@ function getSignature(request: Request): string { } return signatureSHA256; } + function getId(request: Request): string { const id = request.headers.get("x-github-delivery"); if (!id) { diff --git a/wrangler.toml b/wrangler.toml index b30dd2c..2073da2 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -2,6 +2,14 @@ name = "ubiquibot-worker" main = "src/worker.ts" compatibility_date = "2023-12-06" +[env.dev] +kv_namespaces = [ + { binding = "PLUGIN_CHAIN_STATE", id = "4f7aadc56bef41a7ae2cc8c0582320b3" }, +] + +[env.production] +kv_namespaces = [{ binding = "PLUGIN_CHAIN_STATE", id = "TO_BE_FILLED" }] + # Variable bindings. These are arbitrary, plaintext strings (similar to environment variables) # Note: Use secrets to store sensitive data. # Docs: https://developers.cloudflare.com/workers/platform/environment-variables