fix: cspell, removed openai, max length of vectors is 1024'

.cspell.json

@@ -24,8 +24,6 @@
     "commentbody",
     "issuebody",
     "voyageai",
-    "cleancommentobject",
-    "commentobject"
   ],
   "dictionaries": ["typescript", "node", "software-terms"],
   "import": ["@cspell/dict-typescript/cspell-ext.json", "@cspell/dict-node/cspell-ext.json", "@cspell/dict-software-terms"],

.dev.vars.example

@@ -1,3 +1,3 @@
 SUPABASE_URL=""
 SUPABASE_KEY=""
-OPENAI_API_KEY=""
+VOYAGEAI_API_KEY=""

.env.example

@@ -1,3 +1,3 @@
 SUPABASE_URL=
 SUPABASE_KEY=
-OPENAI_API_KEY=
+VOYAGEAI_API_KEY=

.github/workflows/compute.yml

@@ -24,7 +24,6 @@ jobs:
     env:
       SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
       SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }}
-      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
       VOYAGEAI_API_KEY: ${{secrets.VOYAGEAI_API_KEY}}
 
     steps:
@@ -44,4 +43,4 @@ jobs:
         env:
           SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
           SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }}
-          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          VOYAGEAI_API_KEY: ${{secrets.VOYAGEAI_API_KEY}}

README.md

@@ -7,7 +7,6 @@ This is a plugin for [Ubiquibot](https://github.com/ubiquity/ubiquibot-kernel).
 To set up the `.dev.vars` file, you will need to provide the following variables:
 - `SUPABASE_URL`: The URL for your Supabase instance.
 - `SUPABASE_KEY`: The key for your Supabase instance.
-- `OPENAI_API_KEY`: The API key for OpenAI.
 - `VOYAGEAI_API_KEY`: The API key for Voyage.
 
 ## Usage

package.json

@@ -17,7 +17,7 @@
     "knip-ci": "knip --no-exit-code --reporter json --config .github/knip.ts",
     "prepare": "husky install",
     "test": "jest --setupFiles dotenv/config --coverage",
-    "worker": "wrangler dev --env dev --port 5000",
+    "worker": "wrangler dev --env dev --port 4000",
     "supabase:generate:local": "supabase gen types typescript --local > src/types/database.ts",
     "supabase:generate:remote": "cross-env-shell \"supabase gen types typescript --project-id $SUPABASE_PROJECT_ID --schema public > src/types/database.ts\""
   },
@@ -37,7 +37,6 @@
     "@supabase/supabase-js": "^2.45.2",
     "@ubiquity-dao/ubiquibot-logger": "^1.3.0",
     "dotenv": "16.4.5",
-    "openai": "^4.56.0",
     "typebox-validators": "0.3.5",
     "voyageai": "^0.0.1-5"
   },

src/adapters/index.ts

@@ -2,23 +2,16 @@ import { SupabaseClient } from "@supabase/supabase-js";
 import { Context } from "../types";
 import { Comment } from "./supabase/helpers/comment";
 import { SuperSupabase } from "./supabase/helpers/supabase";
-import { SuperOpenAi } from "./openai/helpers/openai";
-import OpenAI from "openai";
-import { Embedding as OpenAiEmbedding } from "./openai/helpers/embedding";
 import { Embedding as VoyageEmbedding } from "./voyage/helpers/embedding";
 import { SuperVoyage } from "./voyage/helpers/voyage";
 import { VoyageAIClient } from "voyageai";
 
-export function createAdapters(supabaseClient: SupabaseClient, openai: OpenAI, voyage: VoyageAIClient, context: Context) {
+export function createAdapters(supabaseClient: SupabaseClient, voyage: VoyageAIClient, context: Context) {
   return {
     supabase: {
       comment: new Comment(supabaseClient, context),
       super: new SuperSupabase(supabaseClient, context),
     },
-    openai: {
-      embedding: new OpenAiEmbedding(openai, context),
-      super: new SuperOpenAi(openai, context),
-    },
     voyage: {
       embedding: new VoyageEmbedding(voyage, context),
       super: new SuperVoyage(voyage, context),

src/adapters/supabase/helpers/comment.ts

@@ -29,10 +29,6 @@ export class Comment extends SuperSupabase {
     } else {
       //Create the embedding for this comment
       const embedding = await this.context.adapters.voyage.embedding.createEmbedding(plaintext);
-      //If embedding is smaller than 3072, pad it with 0s
-      if (embedding.length < 3072) {
-        embedding.push(...new Array(3072 - embedding.length).fill(0));
-      }
       if (isPrivate) {
         plaintext = null as string | null;
         commentobject = null as Record<string, unknown> | null;
@@ -51,9 +47,6 @@ export class Comment extends SuperSupabase {
   async updateComment(plaintext: string | null, commentNodeId: string, commentobject: Record<string, unknown> | null, isPrivate: boolean) {
     //Create the embedding for this comment
     const embedding = Array.from(await this.context.adapters.voyage.embedding.createEmbedding(plaintext));
-    if (embedding.length < 3072) {
-      embedding.push(...new Array(3072 - embedding.length).fill(0));
-    }
     if (isPrivate) {
       plaintext = null as string | null;
       commentobject = null as Record<string, unknown> | null;

src/adapters/voyage/helpers/embedding.ts

@@ -1,7 +1,7 @@
 import { VoyageAIClient } from "voyageai";
 import { Context } from "../../../types";
 import { SuperVoyage } from "./voyage";
-const VECTOR_SIZE = 3072;
+const VECTOR_SIZE = 1024;
 
 export class Embedding extends SuperVoyage {
   protected context: Context;

src/handlers/add-comments.ts

@@ -1,4 +1,4 @@
-import { cleanCommentObject } from "../adapters/utils/cleancommentobject";
+import { cleanCommentObject } from "../adapters/utils/cleanCommentObject";
 import { Context } from "../types";
 
 export async function addComments(context: Context) {
@@ -7,15 +7,15 @@ export async function addComments(context: Context) {
     payload,
     adapters: { supabase },
   } = context;
-  const commentobject = cleanCommentObject(payload);
+  const commentObject = cleanCommentObject(payload);
   const plaintext = payload.comment.body;
   const authorId = payload.comment.user?.id || -1;
   const nodeId = payload.comment.node_id;
   const isPrivate = payload.repository.private;
 
   // Add the comment to the database
   try {
-    await supabase.comment.createComment(plaintext, nodeId, authorId, commentobject, isPrivate);
+    await supabase.comment.createComment(plaintext, nodeId, authorId, commentObject, isPrivate);
   } catch (error) {
     if (error instanceof Error) {
       logger.error(`Error creating comment:`, { error: error, stack: error.stack });

src/handlers/update-comments.ts

@@ -1,4 +1,4 @@
-import { cleanCommentObject } from "../adapters/utils/cleancommentobject";
+import { cleanCommentObject } from "../adapters/utils/cleanCommentObject";
 import { Context } from "../types";
 
 export async function updateComment(context: Context) {
@@ -7,13 +7,13 @@ export async function updateComment(context: Context) {
     payload,
     adapters: { supabase },
   } = context;
-  const commentobject = cleanCommentObject(payload);
+  const commentObject = cleanCommentObject(payload);
   const nodeId = payload.comment.node_id;
   const isPrivate = payload.repository.private;
   const plaintext = payload.comment.body;
   // Fetch the previous comment and update it in the db
   try {
-    await supabase.comment.updateComment(plaintext, nodeId, commentobject, isPrivate);
+    await supabase.comment.updateComment(plaintext, nodeId, commentObject, isPrivate);
   } catch (error) {
     if (error instanceof Error) {
       logger.error(`Error updating comment:`, { error: error, stack: error.stack });

src/plugin.ts

@@ -9,7 +9,6 @@ import { createClient } from "@supabase/supabase-js";
 import { addComments } from "./handlers/add-comments";
 import { updateComment } from "./handlers/update-comments";
 import { deleteComment } from "./handlers/delete-comments";
-import OpenAI from "openai";
 import { VoyageAIClient } from "voyageai";
 
 /**
@@ -38,9 +37,6 @@ export async function runPlugin(context: Context) {
 export async function plugin(inputs: PluginInputs, env: Env) {
   const octokit = new Octokit({ auth: inputs.authToken });
   const supabase = createClient<Database>(env.SUPABASE_URL, env.SUPABASE_KEY);
-  const openaiClient = new OpenAI({
-    apiKey: env.OPENAI_API_KEY,
-  });
   const voyageClient = new VoyageAIClient({
     apiKey: env.VOYAGEAI_API_KEY,
   });
@@ -53,6 +49,6 @@ export async function plugin(inputs: PluginInputs, env: Env) {
     logger: new Logs("info" as LogLevel),
     adapters: {} as ReturnType<typeof createAdapters>,
   };
-  context.adapters = createAdapters(supabase, openaiClient, voyageClient, context);
+  context.adapters = createAdapters(supabase, voyageClient, context);
   return runPlugin(context);
 }

src/types/env.ts

@@ -13,7 +13,6 @@ import { StandardValidator } from "typebox-validators";
 export const envSchema = T.Object({
   SUPABASE_URL: T.String(),
   SUPABASE_KEY: T.String(),
-  OPENAI_API_KEY: T.String(),
   VOYAGEAI_API_KEY: T.String(),
 });
 

supabase/migrations/20240911023641_issue_comments.sql

@@ -1,2 +1,2 @@
 ALTER TABLE issue_comments
-ADD COLUMN commentobject jsonb;
+ADD COLUMN commentObject jsonb;

supabase/migrations/20240911104515_issue_comments.sql

@@ -0,0 +1,5 @@
+ALTER TABLE issue_comments
+DROP COLUMN embedding;
+
+ALTER TABLE issue_comments
+ADD COLUMN embedding Vector(1024) NOT NULL;

tests/__mocks__/adapter.ts

@@ -1,13 +1,12 @@
 import { Context } from "../../src/types";
 import { Comment } from "../../src/adapters/supabase/helpers/comment";
-import { Embedding } from "../../src/adapters/openai/helpers/embedding";
 import { STRINGS } from "./strings";
 
 export interface CommentMock {
   id: string;
   plaintext: string | null;
   author_id: number;
-  commentobject?: Record<string, unknown> | null;
+  commentObject?: Record<string, unknown> | null;
   embedding: number[];
 }
 
@@ -21,7 +20,7 @@ export function createMockAdapters(context: Context) {
             if (commentMap.has(commentNodeId)) {
               throw new Error("Comment already exists");
             }
-            const embedding = await context.adapters.openai.embedding.createEmbedding(plaintext);
+            const embedding = await context.adapters.voyage.embedding.createEmbedding(plaintext);
             if (isPrivate) {
               plaintext = null;
             }
@@ -37,7 +36,7 @@ export function createMockAdapters(context: Context) {
             throw new Error(STRINGS.COMMENT_DOES_NOT_EXIST);
           }
           const { id, author_id } = originalComment;
-          const embedding = await context.adapters.openai.embedding.createEmbedding(plaintext);
+          const embedding = await context.adapters.voyage.embedding.createEmbedding(plaintext);
           if (isPrivate) {
             plaintext = null;
           }
@@ -57,15 +56,15 @@ export function createMockAdapters(context: Context) {
         }),
       } as unknown as Comment,
     },
-    openai: {
+    voyage: {
       embedding: {
         createEmbedding: jest.fn(async (text: string) => {
           if (text && text.length > 0) {
             return new Array(3072).fill(1);
           }
           return new Array(3072).fill(0);
         }),
-      } as unknown as Embedding,
+      } as unknown as number[],
     },
   };
 }

tests/main.test.ts

@@ -18,7 +18,6 @@ import { CommentMock, createMockAdapters } from "./__mocks__/adapter";
 dotenv.config();
 jest.requireActual("@octokit/rest");
 jest.requireActual("@supabase/supabase-js");
-jest.requireActual("openai");
 const octokit = new Octokit();
 
 beforeAll(() => {
@@ -41,7 +40,6 @@ describe("Plugin tests", () => {
     const response = await worker.fetch(new Request("http://localhost/manifest.json"), {
       SUPABASE_KEY: "test",
       SUPABASE_URL: "test",
-      OPENAI_API_KEY: "test",
       VOYAGEAI_API_KEY: "test",
     });
     const content = await response.json();