Merge pull request #21 from ubiquity-os-marketplace/development

Merge development into main

.cspell.json

@@ -22,7 +22,12 @@
     "knip",
     "mischeck",
     "commentbody",
-    "issuebody"
+    "issuebody",
+    "voyageai",
+    "vectordump",
+    "payloadobject",
+    "markdownit",
+    "plpgsql"
   ],
   "dictionaries": ["typescript", "node", "software-terms"],
   "import": ["@cspell/dict-typescript/cspell-ext.json", "@cspell/dict-node/cspell-ext.json", "@cspell/dict-software-terms"],

.dev.vars.example

@@ -1,3 +1,3 @@
 SUPABASE_URL=""
 SUPABASE_KEY=""
-OPENAI_API_KEY=""
+VOYAGEAI_API_KEY=""

.env.example

@@ -1,3 +1,3 @@
 SUPABASE_URL=
 SUPABASE_KEY=
-OPENAI_API_KEY=
+VOYAGEAI_API_KEY=

.github/workflows/compute.yml

@@ -24,7 +24,7 @@ jobs:
     env:
       SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
       SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }}
-      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      VOYAGEAI_API_KEY: ${{secrets.VOYAGEAI_API_KEY}}
 
     steps:
       - uses: actions/checkout@v4
@@ -43,4 +43,4 @@ jobs:
         env:
           SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
           SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }}
-          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          VOYAGEAI_API_KEY: ${{secrets.VOYAGEAI_API_KEY}}

.github/workflows/worker-delete.yml

@@ -0,0 +1,44 @@
+name: Delete Deployment
+
+on:
+  delete:
+
+jobs:
+  delete:
+    runs-on: ubuntu-latest
+    name: Delete Deployment
+    steps:
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20.10.0"
+
+      - name: Enable corepack
+        run: corepack enable
+
+      - uses: actions/checkout@v4
+
+      - name: Get Deleted Branch Name
+        id: get_branch
+        run: |
+          branch_name=$(echo '${{ github.event.ref }}' | sed 's#refs/heads/##' | sed 's#[^a-zA-Z0-9]#-#g')
+          echo "branch_name=$branch_name" >> $GITHUB_ENV
+      - name: Retrieve and Construct Full Worker Name
+        id: construct_worker_name
+        run: |
+          base_name=$(grep '^name = ' wrangler.toml | sed 's/^name = "\(.*\)"$/\1/')
+          full_worker_name="${base_name}-${{ env.branch_name }}"
+          # Make sure that it doesnt exceed 63 characters or it will break RFC 1035
+          full_worker_name=$(echo "${full_worker_name}" | cut -c 1-63)
+          echo "full_worker_name=$full_worker_name" >> $GITHUB_ENV
+      - name: Delete Deployment with Wrangler
+        uses: cloudflare/wrangler-action@v3
+        with:
+          apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+          accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+          command: delete --name ${{ env.full_worker_name }}
+
+      - name: Output Deletion Result
+        run: |
+          echo "### Deployment URL" >> $GITHUB_STEP_SUMMARY
+          echo 'Deployment `${{ env.full_worker_name }}` has been deleted.' >> $GITHUB_STEP_SUMMARY

.github/workflows/worker-deploy.yml

@@ -0,0 +1,52 @@
+name: Deploy Worker
+
+on:
+  push:
+  workflow_dispatch:
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    name: Deploy
+    steps:
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20.10.0"
+
+      - name: Enable corepack
+        run: corepack enable
+
+      - uses: actions/checkout@v4
+
+      - name: Update wrangler.toml Name Field
+        run: |
+          branch_name=$(echo '${{ github.event.ref }}' | sed 's#refs/heads/##' | sed 's#[^a-zA-Z0-9]#-#g')
+          # Extract base name from wrangler.toml
+          base_name=$(grep '^name = ' wrangler.toml | sed 's/^name = "\(.*\)"$/\1/')
+          # Concatenate branch name with base name
+          new_name="${base_name}-${branch_name}"
+          # Truncate the new name to 63 characters for RFC 1035
+          new_name=$(echo "$new_name" | cut -c 1-63)
+          # Update the wrangler.toml file
+          sed -i "s/^name = .*/name = \"$new_name\"/" wrangler.toml
+          echo "Updated wrangler.toml name to: $new_name"
+      - name: Deploy with Wrangler
+        id: wrangler_deploy
+        uses: cloudflare/wrangler-action@v3
+        with:
+          apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+          accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+          secrets: |
+            SUPABASE_URL
+            SUPABASE_KEY
+            VOYAGEAI_API_KEY
+        env:
+          SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
+          SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }}
+          VOYAGEAI_API_KEY: ${{ secrets.VOYAGEAI_API_KEY }}
+
+      - name: Write Deployment URL to Summary
+        run: |
+          echo "### Deployment URL" >> $GITHUB_STEP_SUMMARY
+          echo "${{ steps.wrangler_deploy.outputs.deployment-url }}" >> $GITHUB_STEP_SUMMARY

CHANGELOG.md

@@ -0,0 +1,29 @@
+# Changelog
+
+## 1.0.0 (2024-09-01)
+
+
+### Features
+
+* modified tests ([97e267f](https://github.com/ubiquibot/issue-comment-embeddings/commit/97e267f801ce4e6bd29bbe967de3df4fc3b1942a))
+
+
+### Bug Fixes
+
+* added config.yml ([c0f784b](https://github.com/ubiquibot/issue-comment-embeddings/commit/c0f784b20e59c2c4714805331c7ae9034fd73f73))
+* added config.yml ([221d34d](https://github.com/ubiquibot/issue-comment-embeddings/commit/221d34d801af6ebd764028be4a5c6200a18b776e))
+* added config.yml ([d12c522](https://github.com/ubiquibot/issue-comment-embeddings/commit/d12c522291db36dcf6aea72e5759e1a055185d8f))
+* cspell fix ([736bea6](https://github.com/ubiquibot/issue-comment-embeddings/commit/736bea6172444fdf783ffff729879d8278ff82f3))
+* fixed tests missing supabase files ([0e870ac](https://github.com/ubiquibot/issue-comment-embeddings/commit/0e870ac50eb68249edf5fc4e46fd509425dd7bbb))
+* github workflow, types package.json, env examples ([16786d7](https://github.com/ubiquibot/issue-comment-embeddings/commit/16786d76ee7a598c885f15af1baeadcf6a471b2c))
+* issue_comments linting added issue_comments:edited, created and deleted ([9c0de23](https://github.com/ubiquibot/issue-comment-embeddings/commit/9c0de237048ce30bf4254960c443bf3938037dce))
+* knip workflow ([f325310](https://github.com/ubiquibot/issue-comment-embeddings/commit/f3253109c290c9fce6d14e6a2e1e328133ac6f81))
+* manifest.json, compute.yml ([21409d5](https://github.com/ubiquibot/issue-comment-embeddings/commit/21409d530c3aad6ff2676fc813314e5b29c1a533))
+* package.json ([806c6c0](https://github.com/ubiquibot/issue-comment-embeddings/commit/806c6c0b393a9b87741a6341fa65bc5b3d22cb15))
+* plugin name ([d91b991](https://github.com/ubiquibot/issue-comment-embeddings/commit/d91b991d717b7fb0b73359ca29ae6de08a1074b9))
+* readme.md ([9c5fbfe](https://github.com/ubiquibot/issue-comment-embeddings/commit/9c5fbfe9ca46eb842779468c85d329b9f941fb82))
+* readme.md ([2fec447](https://github.com/ubiquibot/issue-comment-embeddings/commit/2fec44786526e7c10faaa2c13c4349e1232cf5bd))
+* remove config.yml and wrangler.toml namespace entries ([127cc22](https://github.com/ubiquibot/issue-comment-embeddings/commit/127cc225903c3fe3ca934e8407df4eb9c27e378c))
+* removed config.yml changed name ([744e08c](https://github.com/ubiquibot/issue-comment-embeddings/commit/744e08cebac310ae81c3c102f5f3a9473e6e4b9e))
+* test and linting ([a4ee41e](https://github.com/ubiquibot/issue-comment-embeddings/commit/a4ee41e6fca8723ce2fddc96b1171c89cfe7d5b7))
+* wrangler name ([f890071](https://github.com/ubiquibot/issue-comment-embeddings/commit/f890071c01c5bb1d611a5b7aa07cba84f4546251))

README.md

@@ -7,15 +7,19 @@ This is a plugin for [Ubiquibot](https://github.com/ubiquity/ubiquibot-kernel).
 To set up the `.dev.vars` file, you will need to provide the following variables:
 - `SUPABASE_URL`: The URL for your Supabase instance.
 - `SUPABASE_KEY`: The key for your Supabase instance.
-- `OPENAI_API_KEY`: The API key for OpenAI.
+- `VOYAGEAI_API_KEY`: The API key for Voyage.
 
 ## Usage
-- Add the following to your `.ubiquibot.config.yml` file with the appropriate URL:
-```javascript
-  -plugin: http://127.0.0.1:4000
-      runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" ]
+- Add the following to your `.ubiquibot-config.yml` file with the appropriate URL:
+```yaml
+- plugin: ubiquity-os-marketplace/generate-vector-embeddings
+  with:
+    matchThreshold: 0.95
+    warningThreshold: 0.75
+    jobMatchingThreshold: 0.75
 ```
 
+
 ## Testing Locally
 - Run `yarn install` to install the dependencies.
 - Run `yarn worker` to start the server.
@@ -52,4 +56,4 @@ To set up the `.dev.vars` file, you will need to provide the following variables
 - Replace the placeholders with the appropriate values.
 
 ## Testing
-- Run `yarn test` to run the tests.
+- Run `yarn test` to run the tests.

eslint.config.mjs

@@ -9,7 +9,7 @@ export default tsEslint.config({
     "@typescript-eslint": tsEslint.plugin,
     "check-file": checkFile,
   },
-  ignores: [".github/knip.ts", "src/types/database.ts"],
+  ignores: [".github/knip.ts", "src/types/database.ts", "src/adapters/utils/markdown-to-plaintext.ts"],
   extends: [eslint.configs.recommended, ...tsEslint.configs.recommended, sonarjs.configs.recommended],
   languageOptions: {
     parser: tsEslint.parser,

manifest.json

@@ -1,5 +1,5 @@
 {
-  "name": "@ubiquity-os/comment-vector-embeddings",
-  "description": "Issue comment plugin for Ubiquibot. It enables the storage, updating, and deletion of issue comment embeddings.",
-  "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted"]
+  "name": "Generate vector embeddings",
+  "description": "Enables the storage, updating, and deletion of issue comment embeddings.",
+  "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issues.opened", "issues.edited", "issues.deleted", "issues.labeled"]
 }

package.json

@@ -17,7 +17,7 @@
     "knip-ci": "knip --no-exit-code --reporter json --config .github/knip.ts",
     "prepare": "husky install",
     "test": "jest --setupFiles dotenv/config --coverage",
-    "worker": "wrangler dev --env dev --port 4000",
+    "worker": "wrangler dev --env dev --port 5000",
     "supabase:generate:local": "supabase gen types typescript --local > src/types/database.ts",
     "supabase:generate:remote": "cross-env-shell \"supabase gen types typescript --project-id $SUPABASE_PROJECT_ID --schema public > src/types/database.ts\""
   },
@@ -35,10 +35,13 @@
     "@octokit/webhooks": "13.2.7",
     "@sinclair/typebox": "0.32.33",
     "@supabase/supabase-js": "^2.45.2",
+    "@types/markdown-it": "^14.1.2",
     "@ubiquity-dao/ubiquibot-logger": "^1.3.0",
     "dotenv": "16.4.5",
-    "openai": "^4.56.0",
-    "typebox-validators": "0.3.5"
+    "markdown-it": "^14.1.0",
+    "markdown-it-plain-text": "^0.3.0",
+    "typebox-validators": "0.3.5",
+    "voyageai": "^0.0.1-5"
   },
   "devDependencies": {
     "@commitlint/cli": "19.3.0",
@@ -71,7 +74,7 @@
     "tsx": "4.15.6",
     "typescript": "5.4.5",
     "typescript-eslint": "7.13.1",
-    "wrangler": "3.62.0"
+    "wrangler": "3.78.12"
   },
   "lint-staged": {
     "*.ts": [

src/adapters/index.ts

@@ -2,19 +2,21 @@ import { SupabaseClient } from "@supabase/supabase-js";
 import { Context } from "../types";
 import { Comment } from "./supabase/helpers/comment";
 import { SuperSupabase } from "./supabase/helpers/supabase";
-import { SuperOpenAi } from "./openai/helpers/openai";
-import OpenAI from "openai";
-import { Embedding } from "./openai/helpers/embedding";
+import { Embedding as VoyageEmbedding } from "./voyage/helpers/embedding";
+import { SuperVoyage } from "./voyage/helpers/voyage";
+import { VoyageAIClient } from "voyageai";
+import { Issues } from "./supabase/helpers/issues";
 
-export function createAdapters(supabaseClient: SupabaseClient, openai: OpenAI, context: Context) {
+export function createAdapters(supabaseClient: SupabaseClient, voyage: VoyageAIClient, context: Context) {
   return {
     supabase: {
       comment: new Comment(supabaseClient, context),
+      issue: new Issues(supabaseClient, context),
       super: new SuperSupabase(supabaseClient, context),
     },
-    openai: {
-      embedding: new Embedding(openai, context),
-      super: new SuperOpenAi(openai, context),
+    voyage: {
+      embedding: new VoyageEmbedding(voyage, context),
+      super: new SuperVoyage(voyage, context),
     },
   };
 }