Merge pull request #33 from ubiquity-os-marketplace/development

ubiquity-os-marketplace · Oct 16, 2024 · e4a8408 · e4a8408
2 parents afc43d3 + ab694b1
commit e4a8408
Show file tree

Hide file tree

Showing 30 changed files with 644 additions and 217 deletions.
diff --git a/.cspell.json b/.cspell.json
@@ -13,7 +13,7 @@
     "Supabase",
     "SUPABASE",
     "typebox",
-    "ubiquibot",
+    "ubiquity-os",
     "Smee",
     "typeguards",
     "mswjs",

diff --git a/.github/workflows/update-configuration.yml b/.github/workflows/update-configuration.yml
@@ -0,0 +1,58 @@
+name: "Update Configuration"
+
+on:
+  workflow_dispatch:
+  push:
+
+jobs:
+  update:
+    name: "Update Configuration in manifest.json"
+    runs-on: ubuntu-latest
+    permissions: write-all
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup node
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20.10.0"
+
+      - name: Install deps and run configuration update
+        run: |
+          yarn install --immutable --immutable-cache --check-cache
+          yarn tsc --noCheck --project tsconfig.json
+
+      - name: Update manifest configuration using GitHub Script
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+            const path = require('path');
+
+            const { pluginSettingsSchema } = require('./src/types');
+
+            const manifestPath = path.resolve("${{ github.workspace }}", './manifest.json');
+            const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
+
+            const configuration = JSON.stringify(pluginSettingsSchema);
+
+            manifest["configuration"] = JSON.parse(configuration);
+
+            const updatedManifest = JSON.stringify(manifest, null, 2)
+            console.log('Updated manifest:', updatedManifest);
+            fs.writeFileSync(manifestPath, updatedManifest);
+
+      - name: Commit and Push generated types
+        run: |
+          git config --global user.name 'ubiquity-os[bot]'
+          git config --global user.email 'ubiquity-os[bot]@users.noreply.github.com'
+          git add ./manifest.json
+          if [ -n "$(git diff-index --cached --name-only HEAD)" ]; then
+            git commit -m "chore: updated generated configuration" || echo "Lint-staged check failed"
+            git push origin HEAD:${{ github.ref_name }}
+          else
+            echo "No changes to commit"
+          fi
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/worker-deploy.yml b/.github/workflows/worker-deploy.yml
@@ -35,6 +35,7 @@ jobs:
         id: wrangler_deploy
         uses: cloudflare/wrangler-action@v3
         with:
+          wranglerVersion: "3.78.12"
           apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
           accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
           secrets: |

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,25 +5,25 @@
 
 ### Features
 
-* modified tests ([97e267f](https://github.com/ubiquibot/issue-comment-embeddings/commit/97e267f801ce4e6bd29bbe967de3df4fc3b1942a))
+* modified tests ([97e267f](https://github.com/ubiquity-os/issue-comment-embeddings/commit/97e267f801ce4e6bd29bbe967de3df4fc3b1942a))
 
 
 ### Bug Fixes
 
-* added config.yml ([c0f784b](https://github.com/ubiquibot/issue-comment-embeddings/commit/c0f784b20e59c2c4714805331c7ae9034fd73f73))
-* added config.yml ([221d34d](https://github.com/ubiquibot/issue-comment-embeddings/commit/221d34d801af6ebd764028be4a5c6200a18b776e))
-* added config.yml ([d12c522](https://github.com/ubiquibot/issue-comment-embeddings/commit/d12c522291db36dcf6aea72e5759e1a055185d8f))
-* cspell fix ([736bea6](https://github.com/ubiquibot/issue-comment-embeddings/commit/736bea6172444fdf783ffff729879d8278ff82f3))
-* fixed tests missing supabase files ([0e870ac](https://github.com/ubiquibot/issue-comment-embeddings/commit/0e870ac50eb68249edf5fc4e46fd509425dd7bbb))
-* github workflow, types package.json, env examples ([16786d7](https://github.com/ubiquibot/issue-comment-embeddings/commit/16786d76ee7a598c885f15af1baeadcf6a471b2c))
-* issue_comments linting added issue_comments:edited, created and deleted ([9c0de23](https://github.com/ubiquibot/issue-comment-embeddings/commit/9c0de237048ce30bf4254960c443bf3938037dce))
-* knip workflow ([f325310](https://github.com/ubiquibot/issue-comment-embeddings/commit/f3253109c290c9fce6d14e6a2e1e328133ac6f81))
-* manifest.json, compute.yml ([21409d5](https://github.com/ubiquibot/issue-comment-embeddings/commit/21409d530c3aad6ff2676fc813314e5b29c1a533))
-* package.json ([806c6c0](https://github.com/ubiquibot/issue-comment-embeddings/commit/806c6c0b393a9b87741a6341fa65bc5b3d22cb15))
-* plugin name ([d91b991](https://github.com/ubiquibot/issue-comment-embeddings/commit/d91b991d717b7fb0b73359ca29ae6de08a1074b9))
-* readme.md ([9c5fbfe](https://github.com/ubiquibot/issue-comment-embeddings/commit/9c5fbfe9ca46eb842779468c85d329b9f941fb82))
-* readme.md ([2fec447](https://github.com/ubiquibot/issue-comment-embeddings/commit/2fec44786526e7c10faaa2c13c4349e1232cf5bd))
-* remove config.yml and wrangler.toml namespace entries ([127cc22](https://github.com/ubiquibot/issue-comment-embeddings/commit/127cc225903c3fe3ca934e8407df4eb9c27e378c))
-* removed config.yml changed name ([744e08c](https://github.com/ubiquibot/issue-comment-embeddings/commit/744e08cebac310ae81c3c102f5f3a9473e6e4b9e))
-* test and linting ([a4ee41e](https://github.com/ubiquibot/issue-comment-embeddings/commit/a4ee41e6fca8723ce2fddc96b1171c89cfe7d5b7))
-* wrangler name ([f890071](https://github.com/ubiquibot/issue-comment-embeddings/commit/f890071c01c5bb1d611a5b7aa07cba84f4546251))
+* added config.yml ([c0f784b](https://github.com/ubiquity-os/issue-comment-embeddings/commit/c0f784b20e59c2c4714805331c7ae9034fd73f73))
+* added config.yml ([221d34d](https://github.com/ubiquity-os/issue-comment-embeddings/commit/221d34d801af6ebd764028be4a5c6200a18b776e))
+* added config.yml ([d12c522](https://github.com/ubiquity-os/issue-comment-embeddings/commit/d12c522291db36dcf6aea72e5759e1a055185d8f))
+* cspell fix ([736bea6](https://github.com/ubiquity-os/issue-comment-embeddings/commit/736bea6172444fdf783ffff729879d8278ff82f3))
+* fixed tests missing supabase files ([0e870ac](https://github.com/ubiquity-os/issue-comment-embeddings/commit/0e870ac50eb68249edf5fc4e46fd509425dd7bbb))
+* github workflow, types package.json, env examples ([16786d7](https://github.com/ubiquity-os/issue-comment-embeddings/commit/16786d76ee7a598c885f15af1baeadcf6a471b2c))
+* issue_comments linting added issue_comments:edited, created and deleted ([9c0de23](https://github.com/ubiquity-os/issue-comment-embeddings/commit/9c0de237048ce30bf4254960c443bf3938037dce))
+* knip workflow ([f325310](https://github.com/ubiquity-os/issue-comment-embeddings/commit/f3253109c290c9fce6d14e6a2e1e328133ac6f81))
+* manifest.json, compute.yml ([21409d5](https://github.com/ubiquity-os/issue-comment-embeddings/commit/21409d530c3aad6ff2676fc813314e5b29c1a533))
+* package.json ([806c6c0](https://github.com/ubiquity-os/issue-comment-embeddings/commit/806c6c0b393a9b87741a6341fa65bc5b3d22cb15))
+* plugin name ([d91b991](https://github.com/ubiquity-os/issue-comment-embeddings/commit/d91b991d717b7fb0b73359ca29ae6de08a1074b9))
+* readme.md ([9c5fbfe](https://github.com/ubiquity-os/issue-comment-embeddings/commit/9c5fbfe9ca46eb842779468c85d329b9f941fb82))
+* readme.md ([2fec447](https://github.com/ubiquity-os/issue-comment-embeddings/commit/2fec44786526e7c10faaa2c13c4349e1232cf5bd))
+* remove config.yml and wrangler.toml namespace entries ([127cc22](https://github.com/ubiquity-os/issue-comment-embeddings/commit/127cc225903c3fe3ca934e8407df4eb9c27e378c))
+* removed config.yml changed name ([744e08c](https://github.com/ubiquity-os/issue-comment-embeddings/commit/744e08cebac310ae81c3c102f5f3a9473e6e4b9e))
+* test and linting ([a4ee41e](https://github.com/ubiquity-os/issue-comment-embeddings/commit/a4ee41e6fca8723ce2fddc96b1171c89cfe7d5b7))
+* wrangler name ([f890071](https://github.com/ubiquity-os/issue-comment-embeddings/commit/f890071c01c5bb1d611a5b7aa07cba84f4546251))
diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
-# `@ubiquibot/issue-comment-embeddings`
+# `@ubiquity-os/issue-comment-embeddings`
 
-This is a plugin for [Ubiquibot](https://github.com/ubiquity/ubiquibot-kernel). It listens for issue comments, and adds them to a vector store. It handles comment edits and deletions as well.
+This is a plugin for [Ubiquibot](https://github.com/ubiquity-os/ubiquity-os-kernel). It listens for issue comments, and adds them to a vector store. It handles comment edits and deletions as well.
 
 ## Configuration
 - Host the plugin on a server that Ubiquibot can access.
@@ -10,9 +10,9 @@ To set up the `.dev.vars` file, you will need to provide the following variables
 - `VOYAGEAI_API_KEY`: The API key for Voyage.
 
 ## Usage
-- Add the following to your `.ubiquibot-config.yml` file with the appropriate URL:
+- Add the following to your `.ubiquity-os.config.yml` file with the appropriate URL:
 ```yaml
-- plugin: ubiquity-os-marketplace/generate-vector-embeddings
+- plugin: https://ubiquity-os-comment-vector-embeddings-main.ubiquity.workers.dev
   with:
     matchThreshold: 0.95
     warningThreshold: 0.75

diff --git a/manifest.json b/manifest.json
@@ -1,5 +1,36 @@
 {
   "name": "Generate vector embeddings",
   "description": "Enables the storage, updating, and deletion of issue comment embeddings.",
-  "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issues.opened", "issues.edited", "issues.deleted", "issues.labeled"]
-}
+  "ubiquity:listeners": [
+    "issue_comment.created",
+    "issue_comment.edited",
+    "issue_comment.deleted",
+    "issues.opened",
+    "issues.edited",
+    "issues.deleted",
+    "issues.labeled"
+  ],
+  "configuration": {
+    "default": {},
+    "type": "object",
+    "properties": {
+      "matchThreshold": {
+        "default": 0.95,
+        "type": "number"
+      },
+      "warningThreshold": {
+        "default": 0.75,
+        "type": "number"
+      },
+      "jobMatchingThreshold": {
+        "default": 0.75,
+        "type": "number"
+      }
+    },
+    "required": [
+      "matchThreshold",
+      "warningThreshold",
+      "jobMatchingThreshold"
+    ]
+  }
+}
diff --git a/package.json b/package.json
@@ -1,5 +1,5 @@
 {
-  "name": "@ubiquibot/issue-comment-embeddings",
+  "name": "@ubiquity-os/issue-comment-embeddings",
   "version": "1.0.0",
   "description": "Generates vector embeddings of GitHub comments and stores them in Supabase.",
   "author": "Ubiquity DAO",
@@ -36,7 +36,7 @@
     "@sinclair/typebox": "0.32.33",
     "@supabase/supabase-js": "^2.45.2",
     "@types/markdown-it": "^14.1.2",
-    "@ubiquity-dao/ubiquibot-logger": "^1.3.0",
+    "@ubiquity-os/ubiquity-os-logger": "^1.3.2",
     "dotenv": "16.4.5",
     "markdown-it": "^14.1.0",
     "markdown-it-plain-text": "^0.3.0",
@@ -69,10 +69,10 @@
     "lint-staged": "15.2.7",
     "npm-run-all": "4.1.5",
     "prettier": "3.3.2",
-    "supabase": "1.191.3",
+    "supabase": "1.200.3",
     "ts-jest": "29.1.5",
     "tsx": "4.15.6",
-    "typescript": "5.4.5",
+    "typescript": "5.6.2",
     "typescript-eslint": "7.13.1",
     "wrangler": "3.78.12"
   },

diff --git a/src/adapters/supabase/helpers/comment.ts b/src/adapters/supabase/helpers/comment.ts
@@ -37,7 +37,7 @@ export class Comment extends SuperSupabase {
     } else {
       //Create the embedding for this comment
       const embedding = await this.context.adapters.voyage.embedding.createEmbedding(markdown);
-      let plaintext: string | null = markdownToPlainText(markdown || "");
+      let plaintext: string | null = markdownToPlainText(markdown);
       if (isPrivate) {
         markdown = null as string | null;
         payload = null as Record<string, unknown> | null;
@@ -54,21 +54,34 @@ export class Comment extends SuperSupabase {
     this.context.logger.info("Comment created successfully");
   }
 
-  async updateComment(markdown: string | null, commentNodeId: string, payload: Record<string, unknown> | null, isPrivate: boolean) {
+  async updateComment(
+    markdown: string | null,
+    commentNodeId: string,
+    authorId: number,
+    payload: Record<string, unknown> | null,
+    isPrivate: boolean,
+    issueId: string
+  ) {
     //Create the embedding for this comment
     const embedding = Array.from(await this.context.adapters.voyage.embedding.createEmbedding(markdown));
-    let plaintext: string | null = markdownToPlainText(markdown || "");
+    let plaintext: string | null = markdownToPlainText(markdown);
     if (isPrivate) {
       markdown = null as string | null;
       payload = null as Record<string, unknown> | null;
       plaintext = null as string | null;
     }
-    const { error } = await this.supabase
-      .from("issue_comments")
-      .update({ markdown, plaintext, embedding: embedding, payload, modified_at: new Date() })
-      .eq("id", commentNodeId);
-    if (error) {
-      this.context.logger.error("Error updating comment", error);
+    const comments = await this.getComment(commentNodeId);
+    if (comments && comments.length == 0) {
+      this.context.logger.info("Comment does not exist, creating a new one");
+      await this.createComment(markdown, commentNodeId, authorId, payload, isPrivate, issueId);
+    } else {
+      const { error } = await this.supabase
+        .from("issue_comments")
+        .update({ markdown, plaintext, embedding: embedding, payload, modified_at: new Date() })
+        .eq("id", commentNodeId);
+      if (error) {
+        this.context.logger.error("Error updating comment", error);
+      }
     }
   }
 

diff --git a/src/adapters/supabase/helpers/issues.ts b/src/adapters/supabase/helpers/issues.ts
@@ -37,7 +37,7 @@ export class Issues extends SuperSupabase {
       return;
     } else {
       const embedding = await this.context.adapters.voyage.embedding.createEmbedding(markdown);
-      let plaintext: string | null = markdownToPlainText(markdown || "");
+      let plaintext: string | null = markdownToPlainText(markdown);
       if (isPrivate) {
         payload = null;
         markdown = null;
@@ -52,21 +52,24 @@ export class Issues extends SuperSupabase {
     this.context.logger.info("Issue created successfully");
   }
 
-  async updateIssue(markdown: string | null, issueNodeId: string, payload: Record<string, unknown> | null, isPrivate: boolean) {
-    //Create the embedding for this comment
+  async updateIssue(markdown: string | null, issueNodeId: string, payload: Record<string, unknown> | null, isPrivate: boolean, authorId: number) {
     const embedding = Array.from(await this.context.adapters.voyage.embedding.createEmbedding(markdown));
-    let plaintext: string | null = markdownToPlainText(markdown || "");
+    let plaintext: string | null = markdownToPlainText(markdown);
     if (isPrivate) {
-      markdown = null as string | null;
-      payload = null as Record<string, unknown> | null;
-      plaintext = null as string | null;
+      markdown = null;
+      payload = null;
+      plaintext = null;
     }
-    const { error } = await this.supabase
-      .from("issues")
-      .update({ markdown, plaintext, embedding: embedding, payload, modified_at: new Date() })
-      .eq("id", issueNodeId);
-    if (error) {
-      this.context.logger.error("Error updating comment", error);
+    const issues = await this.getIssue(issueNodeId);
+    if (issues && issues.length == 0) {
+      this.context.logger.info("Issue does not exist, creating a new one");
+      await this.createIssue(issueNodeId, payload, isPrivate, markdown, authorId);
+    } else {
+      const { error } = await this.supabase.from("issues").update({ markdown, plaintext, embedding, payload, modified_at: new Date() }).eq("id", issueNodeId);
+
+      if (error) {
+        this.context.logger.error("Error updating comment", error);
+      }
     }
   }
 
@@ -96,6 +99,7 @@ export class Issues extends SuperSupabase {
       current_id: currentId,
       query_embedding: embedding,
       threshold: threshold,
+      top_k: 5,
     });
     if (error) {
       this.context.logger.error("Error finding similar issues", error);

diff --git a/src/handlers/add-issue.ts b/src/handlers/add-issue.ts
@@ -1,5 +1,6 @@
 import { Context } from "../types";
 import { IssuePayload } from "../types/payload";
+import { removeFootnotes } from "./issue-deduplication";
 
 export async function addIssue(context: Context) {
   const {
@@ -16,7 +17,8 @@ export async function addIssue(context: Context) {
     if (!markdown) {
       throw new Error("Issue body is empty");
     }
-    await supabase.issue.createIssue(nodeId, payload, isPrivate, markdown, authorId);
+    const cleanedIssue = removeFootnotes(markdown);
+    await supabase.issue.createIssue(nodeId, payload, isPrivate, cleanedIssue, authorId);
   } catch (error) {
     if (error instanceof Error) {
       logger.error(`Error creating issue:`, { error: error, stack: error.stack });