Merge pull request ubiquity-os#47 from gentlementlegen/feat/org-config

feat: read org config

.cspell.json

@@ -4,7 +4,7 @@
   "ignorePaths": ["**/*.json", "**/*.css", "node_modules", "**/*.log"],
   "useGitignore": true,
   "language": "en",
-  "words": ["dataurl", "devpool", "fkey", "outdir", "servedir", "supabase", "typebox", "ubiquibot", "smee"],
+  "words": ["dataurl", "devpool", "fkey", "outdir", "servedir", "supabase", "typebox", "ubiquibot", "smee", "tomlify"],
   "dictionaries": ["typescript", "node", "software-terms"],
   "import": ["@cspell/dict-typescript/cspell-ext.json", "@cspell/dict-node/cspell-ext.json", "@cspell/dict-software-terms"],
   "ignoreRegExpList": ["[0-9a-fA-F]{6}"]

.github/workflows/build.yml

@@ -24,6 +24,13 @@ jobs:
 
       - uses: oven-sh/setup-bun@v1
 
+      - name: Run setup script
+        run: |
+          bun install
+          bun setup-kv
+        env:
+          CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+
       - uses: cloudflare/wrangler-action@v3
         with:
           wranglerVersion: '3.57.0'

deploy/setup_kv_namespace.ts

@@ -0,0 +1,104 @@
+/**
+ * The purpose of the script is to ensure that the KV for the worker is properly set on deployment.
+ * There is currently a bug that makes the environment reset on each deploy, because of a problem with Wrangler not
+ * parsing the TOML configuration properly. See https://github.com/cloudflare/workers-sdk/issues/5634
+ * It seems to only work when the values are set at the root of the TOML, not withing the environments.
+ * This scripts takes out the Production values for kv_namespaces and rewrites them at the root of the TOML file.
+ */
+
+import { execSync } from "child_process";
+import * as fs from "fs";
+import * as toml from "toml";
+// @ts-expect-error No typings exist for this package
+import * as tomlify from "tomlify-j0.4";
+
+const tomlFilePath = "./wrangler.toml";
+const wranglerToml: WranglerConfiguration = toml.parse(fs.readFileSync(tomlFilePath, "utf-8"));
+
+const NAMESPACE_TITLE = "kv";
+const NAMESPACE_TITLE_WITH_PREFIX = `${wranglerToml.name}-${NAMESPACE_TITLE}`;
+const BINDING_NAME = "PLUGIN_CHAIN_STATE";
+
+interface Namespace {
+  id: string;
+  title: string;
+}
+
+interface WranglerConfiguration {
+  name: string;
+  env: {
+    production: {
+      kv_namespaces?: {
+        id: string;
+        binding: string;
+      }[];
+    };
+    dev: {
+      kv_namespaces?: {
+        id: string;
+        binding: string;
+      }[];
+    };
+  };
+  kv_namespaces: {
+    id: string;
+    binding: string;
+  }[];
+}
+
+function updateWranglerToml(namespaceId: string) {
+  // Ensure kv_namespaces array exists
+  if (!wranglerToml.kv_namespaces) {
+    wranglerToml.kv_namespaces = [];
+  }
+  if (wranglerToml.env.production.kv_namespaces) {
+    wranglerToml.kv_namespaces = wranglerToml.env.production.kv_namespaces;
+    delete wranglerToml.env.production.kv_namespaces;
+  }
+  if (wranglerToml.env.dev.kv_namespaces) {
+    delete wranglerToml.env.dev.kv_namespaces;
+  }
+
+  const existingNamespace = wranglerToml.kv_namespaces.find((o) => o.binding === BINDING_NAME);
+  if (existingNamespace) {
+    existingNamespace.id = namespaceId;
+  } else {
+    wranglerToml.kv_namespaces.push({
+      binding: BINDING_NAME,
+      id: namespaceId,
+    });
+  }
+
+  fs.writeFileSync(tomlFilePath, tomlify.toToml(wranglerToml, { space: 1 }));
+}
+
+async function main() {
+  // Check if the namespace exists or create a new one
+  let namespaceId: string;
+  try {
+    const res = execSync(`wrangler kv:namespace create ${NAMESPACE_TITLE}`).toString();
+    const newId = res.match(/id = \s*"([^"]+)"/)?.[1];
+    if (!newId) {
+      throw new Error(`The new ID could not be found.`);
+    }
+    namespaceId = newId;
+    console.log(`Namespace created with ID: ${namespaceId}`);
+  } catch (error) {
+    const listOutput = JSON.parse(execSync(`wrangler kv:namespace list`).toString()) as Namespace[];
+    const existingNamespace = listOutput.find((o) => o.title === NAMESPACE_TITLE_WITH_PREFIX);
+    if (!existingNamespace) {
+      throw new Error(`Error creating namespace: ${error}`);
+    }
+    namespaceId = existingNamespace.id;
+    console.log(`Namespace ${NAMESPACE_TITLE_WITH_PREFIX} already exists with ID: ${namespaceId}`);
+  }
+
+  updateWranglerToml(namespaceId);
+}
+
+main()
+  .then(() => console.log("Successfully bound namespace."))
+  .catch((e) => {
+    console.error("Error checking or creating namespace:", e);
+    process.exit(1);
+  });

package.json

@@ -23,7 +23,8 @@
     "proxy": "tsx src/proxy.ts",
     "knip": "knip --config .github/knip.ts",
     "knip-ci": "knip --no-exit-code --reporter json --config .github/knip.ts",
-    "test": "bun test"
+    "test": "bun test",
+    "setup-kv": "bun --env-file=.dev.vars deploy/setup_kv_namespace.ts"
   },
   "keywords": [
     "typescript",
@@ -45,6 +46,7 @@
     "@sinclair/typebox": "^0.32.5",
     "dotenv": "^16.4.4",
     "smee-client": "^2.0.0",
+    "typebox-validators": "0.3.5",
     "yaml": "^2.4.1"
   },
   "devDependencies": {
@@ -69,9 +71,11 @@
     "lint-staged": "^15.2.2",
     "npm-run-all": "^4.1.5",
     "prettier": "^3.2.5",
+    "toml": "3.0.0",
+    "tomlify-j0.4": "3.0.0",
     "tsx": "^4.7.1",
     "typescript": "^5.3.3",
-    "wrangler": "^3.58.0"
+    "wrangler": "3.58.0"
   },
   "lint-staged": {
     "*.ts": [

src/github/types/plugin-configuration.ts

@@ -1,5 +1,6 @@
 import { Type as T } from "@sinclair/typebox";
 import { StaticDecode } from "@sinclair/typebox";
+import { StandardValidator } from "typebox-validators";
 import { githubWebhookEvents } from "./webhook-events";
 
 const pluginNameRegex = new RegExp("^([0-9a-zA-Z-._]+)\\/([0-9a-zA-Z-._]+)(?::([0-9a-zA-Z-._]+))?(?:@([0-9a-zA-Z-._]+(?:\\/[0-9a-zA-Z-._]+)?))?$");
@@ -73,4 +74,6 @@ export const configSchema = T.Object({
   plugins: T.Record(T.Enum(githubWebhookEvents), handlerSchema, { default: {} }),
 });
 
+export const configSchemaValidator = new StandardValidator(configSchema);
+
 export type PluginConfiguration = StaticDecode<typeof configSchema>;

src/github/utils/config.ts

@@ -2,10 +2,51 @@ import { Value } from "@sinclair/typebox/value";
 import YAML from "yaml";
 import { GitHubContext } from "../github-context";
 import { expressionRegex } from "../types/plugin";
-import { configSchema, PluginConfiguration } from "../types/plugin-configuration";
+import { configSchema, configSchemaValidator, PluginConfiguration } from "../types/plugin-configuration";
 import { eventNames } from "../types/webhook-events";
 
 const UBIQUIBOT_CONFIG_FULL_PATH = ".github/.ubiquibot-config.yml";
+const UBIQUIBOT_CONFIG_ORG_REPO = "ubiquibot-config";
+
+async function getConfigurationFromRepo(context: GitHubContext, repository: string, owner: string) {
+  const targetRepoConfiguration: PluginConfiguration = parseYaml(
+    await download({
+      context,
+      repository,
+      owner,
+    })
+  );
+  if (targetRepoConfiguration) {
+    try {
+      const configSchemaWithDefaults = Value.Default(configSchema, targetRepoConfiguration) as Readonly<unknown>;
+      const errors = configSchemaValidator.testReturningErrors(configSchemaWithDefaults);
+      if (errors !== null) {
+        for (const error of errors) {
+          console.error(error);
+        }
+      }
+      return Value.Decode(configSchema, configSchemaWithDefaults);
+    } catch (error) {
+      console.error(`Error decoding configuration for ${owner}/${repository}, will ignore.`, error);
+      return null;
+    }
+  }
+  return null;
+}
+
+/**
+ * Merge configurations based on their 'plugins' keys
+ */
+function mergeConfigurations(configuration1: PluginConfiguration, configuration2: PluginConfiguration): PluginConfiguration {
+  const mergedConfiguration = { ...configuration1 };
+  for (const key of Object.keys(configuration2.plugins)) {
+    const pluginKey = key as keyof PluginConfiguration["plugins"];
+    if (configuration2.plugins[pluginKey]?.length) {
+      mergedConfiguration.plugins[pluginKey] = configuration2.plugins[pluginKey];
+    }
+  }
+  return mergedConfiguration;
+}
 
 export async function getConfig(context: GitHubContext): Promise<PluginConfiguration> {
   const payload = context.payload;
@@ -19,26 +60,22 @@ export async function getConfig(context: GitHubContext): Promise<PluginConfigura
     return defaultConfiguration;
   }
 
-  const _repoConfig = parseYaml(
-    await download({
-      context,
-      repository: payload.repository.name,
-      owner: payload.repository.owner.login,
-    })
-  );
-  if (!_repoConfig) return defaultConfiguration;
+  let mergedConfiguration: PluginConfiguration = defaultConfiguration;
 
-  let config: PluginConfiguration;
-  try {
-    config = Value.Decode(configSchema, Value.Default(configSchema, _repoConfig));
-  } catch (error) {
-    console.error("Error decoding config, will use default.", error);
-    return defaultConfiguration;
-  }
+  const configurations = await Promise.all([
+    getConfigurationFromRepo(context, UBIQUIBOT_CONFIG_ORG_REPO, payload.repository.owner.login),
+    getConfigurationFromRepo(context, payload.repository.name, payload.repository.owner.login),
+  ]);
+
+  configurations.forEach((configuration) => {
+    if (configuration) {
+      mergedConfiguration = mergeConfigurations(mergedConfiguration, configuration);
+    }
+  });
 
-  checkPluginChains(config);
+  checkPluginChains(mergedConfiguration);
 
-  return config;
+  return mergedConfiguration;
 }
 
 function checkPluginChains(config: PluginConfiguration) {

tests/main.test.ts

@@ -1,3 +1,4 @@
+import { RestEndpointMethodTypes } from "@octokit/plugin-rest-endpoint-methods";
 /* eslint-disable @typescript-eslint/naming-convention */
 // @ts-expect-error package name is correct, TypeScript doesn't recognize it
 import { afterAll, afterEach, beforeAll, describe, expect, it, jest, mock, spyOn } from "bun:test";
@@ -149,5 +150,118 @@ describe("Worker tests", () => {
       expect(pluginChain[0].uses[0].plugin).toBe("https://plugin-a.internal");
       expect(pluginChain[0].uses[0].with).toEqual({});
     });
+    it("Should merge organization and repository configuration", async () => {
+      const workflowId = "compute.yml";
+      const cfg = await getConfig({
+        key: issueOpened,
+        name: issueOpened,
+        id: "",
+        payload: {
+          repository: {
+            owner: { login: "ubiquity" },
+            name: "conversation-rewards",
+          },
+        } as unknown as GitHubContext<"issues.closed">["payload"],
+        octokit: {
+          rest: {
+            repos: {
+              getContent(args: RestEndpointMethodTypes["repos"]["getContent"]["parameters"]) {
+                if (args.repo !== "ubiquibot-config") {
+                  return {
+                    data: `
+plugins:
+  '*':
+    - uses:
+      - plugin: repo-3/plugin-3
+        type: github
+        with:
+          setting1: false
+    - uses:
+      - plugin: repo-1/plugin-1
+        type: github
+        with:
+          setting2: true`,
+                  };
+                }
+                return {
+                  data: `
+plugins:
+  'issues.assigned':
+    - uses:
+      - plugin: uses-1/plugin-1
+        type: github
+        with:
+          settings1: 'enabled'
+  '*':
+    - uses:
+      - plugin: repo-1/plugin-1
+        type: github
+        with:
+          setting1: false
+    - uses:
+      - plugin: repo-2/plugin-2
+        type: github
+        with:
+          setting2: true`,
+                };
+              },
+            },
+          },
+        },
+        eventHandler: {} as GitHubEventHandler,
+      } as unknown as GitHubContext);
+      expect(cfg.plugins["issues.assigned"]).toEqual([
+        {
+          uses: [
+            {
+              plugin: {
+                owner: "uses-1",
+                repo: "plugin-1",
+                workflowId,
+              },
+              type: "github",
+              with: {
+                settings1: "enabled",
+              },
+            },
+          ],
+          skipBotEvents: true,
+        },
+      ]);
+      expect(cfg.plugins["*"]).toEqual([
+        {
+          uses: [
+            {
+              plugin: {
+                owner: "repo-3",
+                repo: "plugin-3",
+                workflowId,
+              },
+              type: "github",
+              with: {
+                setting1: false,
+              },
+            },
+          ],
+          skipBotEvents: true,
+        },
+        {
+          uses: [
+            {
+              plugin: {
+                owner: "repo-1",
+                repo: "plugin-1",
+                workflowId,
+              },
+              type: "github",
+              with: {
+                setting2: true,
+              },
+            },
+          ],
+          skipBotEvents: true,
+        },
+      ]);
+    });
   });
 });