From 120a68847502882c257c4118a77486786165afc5 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Sun, 8 Sep 2024 19:22:10 -0400 Subject: [PATCH 01/37] fix: config added --- .github/.ubiquibot.config.yml | 2 + package.json | 2 +- src/adapters/supabase/helpers/comment.ts | 37 +++++++++++-------- src/handlers/add-comments.ts | 16 +++----- .../20240908231044_issue_comments.sql | 15 ++++++++ 5 files changed, 45 insertions(+), 27 deletions(-) create mode 100644 .github/.ubiquibot.config.yml create mode 100644 supabase/migrations/20240908231044_issue_comments.sql diff --git a/.github/.ubiquibot.config.yml b/.github/.ubiquibot.config.yml new file mode 100644 index 0000000..1ee14a9 --- /dev/null +++ b/.github/.ubiquibot.config.yml @@ -0,0 +1,2 @@ +-plugin: http://127.0.0.1:5000 + runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" ] \ No newline at end of file diff --git a/package.json b/package.json index bb43997..1716467 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "knip-ci": "knip --no-exit-code --reporter json --config .github/knip.ts", "prepare": "husky install", "test": "jest --setupFiles dotenv/config --coverage", - "worker": "wrangler dev --env dev --port 4000", + "worker": "wrangler dev --env dev --port 5000", "supabase:generate:local": "supabase gen types typescript --local > src/types/database.ts", "supabase:generate:remote": "cross-env-shell \"supabase gen types typescript --project-id $SUPABASE_PROJECT_ID --schema public > src/types/database.ts\"" }, diff --git a/src/adapters/supabase/helpers/comment.ts b/src/adapters/supabase/helpers/comment.ts index 7530c10..43ae42e 100644 --- a/src/adapters/supabase/helpers/comment.ts +++ b/src/adapters/supabase/helpers/comment.ts @@ -3,8 +3,11 @@ import { SuperSupabase } from "./supabase"; import { Context } from "../../../types/context"; export interface CommentType { - id: number; - body: string; + id: string; + plaintext: string; + author_id: number; + created_at: string; + modified_at: string; embedding: number[]; } @@ -13,9 +16,9 @@ export class Comment extends SuperSupabase { super(supabase, context); } - async createComment(commentBody: string, commentId: number, issueBody: string) { + async createComment(plaintext: string, commentNodeId: string, authorId: number, isPrivate: boolean) { //First Check if the comment already exists - const { data, error } = await this.supabase.from("issue_comments").select("*").eq("id", commentId); + const { data, error } = await this.supabase.from("issue_comments").select("*").eq("id", commentNodeId); if (error) { this.context.logger.error("Error creating comment", error); return; @@ -25,10 +28,11 @@ export class Comment extends SuperSupabase { return; } else { //Create the embedding for this comment - const embedding = await this.context.adapters.openai.embedding.createEmbedding(commentBody); - const { error } = await this.supabase - .from("issue_comments") - .insert([{ id: commentId, commentbody: commentBody, issuebody: issueBody, embedding: embedding }]); + const embedding = await this.context.adapters.openai.embedding.createEmbedding(plaintext); + if (isPrivate) { + plaintext = "CENSORED"; + } + const { error } = await this.supabase.from("issue_comments").insert([{ id: commentNodeId, plaintext, author_id: authorId, embedding: embedding }]); if (error) { this.context.logger.error("Error creating comment", error); return; @@ -37,25 +41,28 @@ export class Comment extends SuperSupabase { this.context.logger.info("Comment created successfully"); } - async updateComment(commentBody: string, commentId: number) { + async updateComment(plaintext: string, commentNodeId: string, isPrivate: boolean) { //Create the embedding for this comment - const embedding = Array.from(await this.context.adapters.openai.embedding.createEmbedding(commentBody)); - const { error } = await this.supabase.from("issue_comments").update({ commentbody: commentBody, embedding: embedding }).eq("id", commentId); + const embedding = Array.from(await this.context.adapters.openai.embedding.createEmbedding(plaintext)); + if (isPrivate) { + plaintext = "CENSORED"; + } + const { error } = await this.supabase.from("issue_comments").update({ plaintext, embedding: embedding, modified_at: new Date() }).eq("id", commentNodeId); if (error) { this.context.logger.error("Error updating comment", error); } } - async getComment(commentId: number): Promise { - const { data, error } = await this.supabase.from("issue_comments").select("*").eq("id", commentId); + async getComment(commentNodeId: string): Promise { + const { data, error } = await this.supabase.from("issue_comments").select("*").eq("id", commentNodeId); if (error) { this.context.logger.error("Error getting comment", error); } return data; } - async deleteComment(commentId: number) { - const { error } = await this.supabase.from("issue_comments").delete().eq("id", commentId); + async deleteComment(commentNodeId: string) { + const { error } = await this.supabase.from("issue_comments").delete().eq("id", commentNodeId); if (error) { this.context.logger.error("Error deleting comment", error); } diff --git a/src/handlers/add-comments.ts b/src/handlers/add-comments.ts index 35bd387..5758861 100644 --- a/src/handlers/add-comments.ts +++ b/src/handlers/add-comments.ts @@ -6,20 +6,14 @@ export async function addComments(context: Context) { payload, adapters: { supabase }, } = context; - - const sender = payload.comment.user?.login; - const repo = payload.repository.name; - const issueNumber = payload.issue.number; - const issueBody = payload.issue.body || ""; - const owner = payload.repository.owner.login; - const body = payload.comment.body; - - // Log the payload - logger.info(`Executing addComments:`, { sender, repo, issueNumber, owner }); + const plaintext = payload.comment.body; + const authorId = payload.comment.user?.id || 0; + const nodeId = payload.comment.node_id; + const isPrivate = payload.repository.private; // Add the comment to the database try { - await supabase.comment.createComment(body, payload.comment.id, issueBody); + await supabase.comment.createComment(plaintext, nodeId, authorId, isPrivate); } catch (error) { if (error instanceof Error) { logger.error(`Error creating comment:`, { error: error, stack: error.stack }); diff --git a/supabase/migrations/20240908231044_issue_comments.sql b/supabase/migrations/20240908231044_issue_comments.sql new file mode 100644 index 0000000..51afb87 --- /dev/null +++ b/supabase/migrations/20240908231044_issue_comments.sql @@ -0,0 +1,15 @@ +-- Create the extension if it doesn't exist +CREATE EXTENSION IF NOT EXISTS vector; + +-- Drop the old issue_comments table +DROP TABLE IF EXISTS issue_comments; + +-- Create the issue_comments table +CREATE TABLE IF NOT EXISTS issue_comments ( + id VARCHAR PRIMARY KEY, + created_at TIMESTAMP WITH TIME ZONE DEFAULT current_timestamp NOT NULL, + modified_at TIMESTAMP WITH TIME ZONE NOT NULL, + author_id VARCHAR NOT NULL, + plaintext TEXT NOT NULL, + embedding VECTOR(3072) NOT NULL +); \ No newline at end of file From 381fda1960897ec10c7072f1d07b7adcc3fcd27c Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Sun, 8 Sep 2024 19:25:51 -0400 Subject: [PATCH 02/37] fix: config added --- .github/.ubiquibot.config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/.ubiquibot.config.yml b/.github/.ubiquibot.config.yml index 1ee14a9..1fa8ef5 100644 --- a/.github/.ubiquibot.config.yml +++ b/.github/.ubiquibot.config.yml @@ -1,2 +1,2 @@ --plugin: http://127.0.0.1:5000 +-plugin: https://ubiquity-os-comment-vector-embeddings.sshivaditya.workers.dev/ runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" ] \ No newline at end of file From 95703c74ce20f27e289e5c41e21590412f491e0d Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Sun, 8 Sep 2024 19:41:09 -0400 Subject: [PATCH 03/37] fix: config added --- .github/.ubiquibot.config.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/.ubiquibot.config.yml b/.github/.ubiquibot.config.yml index 1fa8ef5..6782ba6 100644 --- a/.github/.ubiquibot.config.yml +++ b/.github/.ubiquibot.config.yml @@ -1,2 +1,6 @@ --plugin: https://ubiquity-os-comment-vector-embeddings.sshivaditya.workers.dev/ - runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" ] \ No newline at end of file +plugins: + - name: test-app + id: test-app + uses: + - plugin: https://ubiquity-os-comment-vector-embeddings.sshivaditya.workers.dev/ + runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" ] \ No newline at end of file From 32bdae93da9c2a462e284879986e6dd83a6f2787 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Sun, 8 Sep 2024 19:43:29 -0400 Subject: [PATCH 04/37] fix: config added and updated readme --- .github/{.ubiquibot.config.yml => .ubiquibot-config.yml} | 0 README.md | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename .github/{.ubiquibot.config.yml => .ubiquibot-config.yml} (100%) diff --git a/.github/.ubiquibot.config.yml b/.github/.ubiquibot-config.yml similarity index 100% rename from .github/.ubiquibot.config.yml rename to .github/.ubiquibot-config.yml diff --git a/README.md b/README.md index 0ee0bbe..fb293b9 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ To set up the `.dev.vars` file, you will need to provide the following variables - `OPENAI_API_KEY`: The API key for OpenAI. ## Usage -- Add the following to your `.ubiquibot.config.yml` file with the appropriate URL: +- Add the following to your `.ubiquibot-config.yml` file with the appropriate URL: ```javascript -plugin: http://127.0.0.1:4000 runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" ] From ca3cf2d783137d49ce445f144ee996a8844b396e Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Sun, 8 Sep 2024 19:44:30 -0400 Subject: [PATCH 05/37] fix: incorrect url --- .github/.ubiquibot-config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/.ubiquibot-config.yml b/.github/.ubiquibot-config.yml index 6782ba6..3653a9c 100644 --- a/.github/.ubiquibot-config.yml +++ b/.github/.ubiquibot-config.yml @@ -2,5 +2,5 @@ plugins: - name: test-app id: test-app uses: - - plugin: https://ubiquity-os-comment-vector-embeddings.sshivaditya.workers.dev/ + - plugin: https://ubiquity-os-comment-vector-embeddings.sshivaditya.workers.dev runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" ] \ No newline at end of file From 71a65a36d06dd8fb24e345d01efe2e1a842bafa5 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Mon, 9 Sep 2024 01:10:31 -0400 Subject: [PATCH 06/37] fix: tests --- src/handlers/delete-comments.ts | 11 +---- src/handlers/update-comments.ts | 14 ++---- src/types/database.ts | 24 ++++++---- .../20240908235818_issue_comments.sql | 2 + tests/__mocks__/adapter.ts | 44 ++++++++++--------- tests/__mocks__/db.ts | 1 + tests/__mocks__/helpers.ts | 3 +- tests/__mocks__/strings.ts | 1 + tests/main.test.ts | 32 +++++++------- 9 files changed, 66 insertions(+), 66 deletions(-) create mode 100644 supabase/migrations/20240908235818_issue_comments.sql diff --git a/src/handlers/delete-comments.ts b/src/handlers/delete-comments.ts index 7e9842f..7b444ea 100644 --- a/src/handlers/delete-comments.ts +++ b/src/handlers/delete-comments.ts @@ -7,17 +7,10 @@ export async function deleteComment(context: Context) { adapters: { supabase }, } = context; - const sender = payload.comment.user?.login; - const repo = payload.repository.name; - const issueNumber = payload.issue.number; - const owner = payload.repository.owner.login; + const nodeId = payload.comment.node_id; - // Log the payload - logger.debug(`Executing deleteComment:`, { sender, repo, issueNumber, owner }); - - // Add the comment to the database try { - await supabase.comment.deleteComment(payload.comment.id); + await supabase.comment.deleteComment(nodeId); } catch (error) { if (error instanceof Error) { logger.error(`Error deleting comment:`, { error: error, stack: error.stack }); diff --git a/src/handlers/update-comments.ts b/src/handlers/update-comments.ts index b0c3142..4c51f17 100644 --- a/src/handlers/update-comments.ts +++ b/src/handlers/update-comments.ts @@ -7,18 +7,12 @@ export async function updateComment(context: Context) { adapters: { supabase }, } = context; - const sender = payload.comment.user?.login; - const repo = payload.repository.name; - const issueNumber = payload.issue.number; - const owner = payload.repository.owner.login; - const body = payload.comment.body; - - // Log the payload - logger.debug(`Executing updateComment:`, { sender, repo, issueNumber, owner }); - + const nodeId = payload.comment.node_id; + const isPrivate = payload.repository.private; + const plaintext = payload.comment.body; // Fetch the previous comment and update it in the db try { - await supabase.comment.updateComment(body, payload.comment.id); + await supabase.comment.updateComment(plaintext, nodeId, isPrivate); } catch (error) { if (error instanceof Error) { logger.error(`Error updating comment:`, { error: error, stack: error.stack }); diff --git a/src/types/database.ts b/src/types/database.ts index 4f00255..009eae6 100644 --- a/src/types/database.ts +++ b/src/types/database.ts @@ -30,22 +30,28 @@ export type Database = { Tables: { issue_comments: { Row: { - commentbody: string; + author_id: string; + created_at: string; embedding: string; - id: number; - issuebody: string | null; + id: string; + modified_at: string; + plaintext: string; }; Insert: { - commentbody: string; + author_id: string; + created_at?: string; embedding: string; - id: number; - issuebody?: string | null; + id: string; + modified_at?: string; + plaintext: string; }; Update: { - commentbody?: string; + author_id?: string; + created_at?: string; embedding?: string; - id?: number; - issuebody?: string | null; + id?: string; + modified_at?: string; + plaintext?: string; }; Relationships: []; }; diff --git a/supabase/migrations/20240908235818_issue_comments.sql b/supabase/migrations/20240908235818_issue_comments.sql new file mode 100644 index 0000000..11eb77a --- /dev/null +++ b/supabase/migrations/20240908235818_issue_comments.sql @@ -0,0 +1,2 @@ +ALTER TABLE issue_comments +ALTER COLUMN modified_at SET DEFAULT current_timestamp; \ No newline at end of file diff --git a/tests/__mocks__/adapter.ts b/tests/__mocks__/adapter.ts index 4c7a797..c2c02ad 100644 --- a/tests/__mocks__/adapter.ts +++ b/tests/__mocks__/adapter.ts @@ -4,47 +4,51 @@ import { Embedding } from "../../src/adapters/openai/helpers/embedding"; import { STRINGS } from "./strings"; export interface CommentMock { - id: number; - commentbody: string; - issuebody: string; + id: string; + plaintext: string; + author_id: number; embedding: number[]; } export function createMockAdapters(context: Context) { - const commentMap: Map = new Map(); + const commentMap: Map = new Map(); return { supabase: { comment: { - createComment: jest.fn(async (commentBody: string, commentId: number, issueBody: string) => { - if (commentMap.has(commentId)) { + createComment: jest.fn(async (plaintext: string, commentNodeId: string, authorId: number) => { + if (commentMap.has(commentNodeId)) { throw new Error("Comment already exists"); } - const embedding = await context.adapters.openai.embedding.createEmbedding(commentBody); - commentMap.set(commentId, { id: commentId, commentbody: commentBody, issuebody: issueBody, embedding }); + const embedding = await context.adapters.openai.embedding.createEmbedding(plaintext); + commentMap.set(commentNodeId, { id: commentNodeId, plaintext, author_id: authorId, embedding }); }), - updateComment: jest.fn(async (commentBody: string, commentId: number) => { - if (!commentMap.has(commentId)) { + updateComment: jest.fn(async (plaintext: string, commentNodeId: string, isPrivate: boolean) => { + console.log(commentMap); + if (!commentMap.has(commentNodeId)) { throw new Error(STRINGS.COMMENT_DOES_NOT_EXIST); } - const originalComment = commentMap.get(commentId); + const originalComment = commentMap.get(commentNodeId); if (!originalComment) { throw new Error(STRINGS.COMMENT_DOES_NOT_EXIST); } - const { id, issuebody } = originalComment; - const embedding = await context.adapters.openai.embedding.createEmbedding(commentBody); - commentMap.set(commentId, { id, commentbody: commentBody, issuebody, embedding }); + const { id, author_id } = originalComment; + const embedding = await context.adapters.openai.embedding.createEmbedding(plaintext); + if (isPrivate) { + plaintext = STRINGS.CENSORED; + } + commentMap.set(commentNodeId, { id, plaintext, author_id, embedding }); }), - deleteComment: jest.fn(async (commentId: number) => { - if (!commentMap.has(commentId)) { + deleteComment: jest.fn(async (commentNodeId: string) => { + if (!commentMap.has(commentNodeId)) { throw new Error(STRINGS.COMMENT_DOES_NOT_EXIST); } - commentMap.delete(commentId); + commentMap.delete(commentNodeId); }), - getComment: jest.fn(async (commentId: number) => { - if (!commentMap.has(commentId)) { + getComment: jest.fn(async (commentNodeId: string) => { + if (!commentMap.has(commentNodeId)) { throw new Error(STRINGS.COMMENT_DOES_NOT_EXIST); } - return commentMap.get(commentId); + return commentMap.get(commentNodeId); }), } as unknown as Comment, }, diff --git a/tests/__mocks__/db.ts b/tests/__mocks__/db.ts index f7bd2b6..e1883e5 100644 --- a/tests/__mocks__/db.ts +++ b/tests/__mocks__/db.ts @@ -75,6 +75,7 @@ export const db = factory({ body: String, created_at: Date, updated_at: Date, + node_id: String, issue_number: Number, user: { login: String, diff --git a/tests/__mocks__/helpers.ts b/tests/__mocks__/helpers.ts index b4c5b14..eb9e2ae 100644 --- a/tests/__mocks__/helpers.ts +++ b/tests/__mocks__/helpers.ts @@ -38,7 +38,7 @@ export async function setupTests() { }); } -export function createComment(comment: string, commentId: number) { +export function createComment(comment: string, commentId: number, nodeId: string) { const isComment = db.issueComments.findFirst({ where: { id: { @@ -63,6 +63,7 @@ export function createComment(comment: string, commentId: number) { id: commentId, body: comment, issue_number: 1, + node_id: nodeId, user: { login: STRINGS.USER_1, id: 1, diff --git a/tests/__mocks__/strings.ts b/tests/__mocks__/strings.ts index 83d7306..11ac64f 100644 --- a/tests/__mocks__/strings.ts +++ b/tests/__mocks__/strings.ts @@ -15,4 +15,5 @@ export const STRINGS = { CONFIGURABLE_RESPONSE: "Hello, Code Reviewers!", INVALID_COMMAND: "/Goodbye", COMMENT_DOES_NOT_EXIST: "Comment does not exist", + CENSORED: "[CENSORED]", }; diff --git a/tests/main.test.ts b/tests/main.test.ts index c78ec3d..a372de2 100644 --- a/tests/main.test.ts +++ b/tests/main.test.ts @@ -48,44 +48,41 @@ describe("Plugin tests", () => { }); it("When a comment is created it should add it to the database", async () => { - const { context } = createContext(); + const { context } = createContext(STRINGS.HELLO_WORLD, 1, 1, 1, 1, "sasasCreate"); await runPlugin(context); const supabase = context.adapters.supabase; try { - const issueBody = context.payload.issue.body || ""; - await supabase.comment.createComment(STRINGS.HELLO_WORLD, 1, issueBody); + await supabase.comment.createComment(STRINGS.HELLO_WORLD, "sasasCreate", 1, false); throw new Error("Expected method to reject."); } catch (error) { if (error instanceof Error) { expect(error.message).toBe("Comment already exists"); } } - const comment = (await supabase.comment.getComment(1)) as unknown as CommentMock; + const comment = (await supabase.comment.getComment("sasasCreate")) as unknown as CommentMock; expect(comment).toBeDefined(); - expect(comment?.commentbody).toBeDefined(); - expect(comment?.commentbody).toBe(STRINGS.HELLO_WORLD); + expect(comment?.plaintext).toBeDefined(); + expect(comment?.plaintext).toBe(STRINGS.HELLO_WORLD); }); it("When a comment is updated it should update the database", async () => { - const { context } = createContext("Updated Message", 1, 1, 1, 1, "issue_comment.edited"); + const { context } = createContext("Updated Message", 1, 1, 1, 1, "sasasUpdate", "issue_comment.edited"); const supabase = context.adapters.supabase; - const issueBody = context.payload.issue.body || ""; - await supabase.comment.createComment(STRINGS.HELLO_WORLD, 1, issueBody); + await supabase.comment.createComment(STRINGS.HELLO_WORLD, "sasasUpdate", 1, false); await runPlugin(context); - const comment = (await supabase.comment.getComment(1)) as unknown as CommentMock; + const comment = (await supabase.comment.getComment("sasasUpdate")) as unknown as CommentMock; expect(comment).toBeDefined(); - expect(comment?.commentbody).toBeDefined(); - expect(comment?.commentbody).toBe("Updated Message"); + expect(comment?.plaintext).toBeDefined(); + expect(comment?.plaintext).toBe("Updated Message"); }); it("When a comment is deleted it should delete it from the database", async () => { - const { context } = createContext("Text Message", 1, 1, 1, 1, "issue_comment.deleted"); + const { context } = createContext("Text Message", 1, 1, 1, 1, "sasasDelete", "issue_comment.deleted"); const supabase = context.adapters.supabase; - const issueBody = context.payload.issue.body || ""; - await supabase.comment.createComment(STRINGS.HELLO_WORLD, 1, issueBody); + await supabase.comment.createComment(STRINGS.HELLO_WORLD, "sasasDelete", 1, false); await runPlugin(context); try { - await supabase.comment.getComment(1); + await supabase.comment.getComment("sasasDelete"); throw new Error("Expected method to reject."); } catch (error) { if (error instanceof Error) { @@ -109,13 +106,14 @@ function createContext( payloadSenderId: number = 1, commentId: number = 1, issueOne: number = 1, + nodeId: string = "sasas", eventName: Context["eventName"] = "issue_comment.created" ) { const repo = db.repo.findFirst({ where: { id: { equals: repoId } } }) as unknown as Context["payload"]["repository"]; const sender = db.users.findFirst({ where: { id: { equals: payloadSenderId } } }) as unknown as Context["payload"]["sender"]; const issue1 = db.issue.findFirst({ where: { id: { equals: issueOne } } }) as unknown as Context["payload"]["issue"]; - createComment(commentBody, commentId); // create it first then pull it from the DB and feed it to _createContext + createComment(commentBody, commentId, nodeId); // create it first then pull it from the DB and feed it to _createContext const comment = db.issueComments.findFirst({ where: { id: { equals: commentId } } }) as unknown as Context["payload"]["comment"]; const context = createContextInner(repo, sender, issue1, comment, eventName); From 86c56d58f48612c3fcfbb36b4510e853163190f3 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Mon, 9 Sep 2024 01:22:53 -0400 Subject: [PATCH 07/37] fix: remove config.yml --- .github/.ubiquibot-config.yml | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 .github/.ubiquibot-config.yml diff --git a/.github/.ubiquibot-config.yml b/.github/.ubiquibot-config.yml deleted file mode 100644 index 3653a9c..0000000 --- a/.github/.ubiquibot-config.yml +++ /dev/null @@ -1,6 +0,0 @@ -plugins: - - name: test-app - id: test-app - uses: - - plugin: https://ubiquity-os-comment-vector-embeddings.sshivaditya.workers.dev - runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" ] \ No newline at end of file From 88477786d7fe67ce4903e5dd97b7e3bef3c05c68 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Mon, 9 Sep 2024 03:36:36 -0400 Subject: [PATCH 08/37] fix: nullable plaintext --- src/adapters/openai/helpers/embedding.ts | 24 +++++++++++-------- src/adapters/supabase/helpers/comment.ts | 6 ++--- src/handlers/add-comments.ts | 2 +- src/types/database.ts | 6 ++--- .../20240909072603_issue_comments.sql | 2 ++ tests/__mocks__/adapter.ts | 8 +++---- tests/__mocks__/strings.ts | 1 - 7 files changed, 27 insertions(+), 22 deletions(-) create mode 100644 supabase/migrations/20240909072603_issue_comments.sql diff --git a/src/adapters/openai/helpers/embedding.ts b/src/adapters/openai/helpers/embedding.ts index 6b912dd..3dfaeb9 100644 --- a/src/adapters/openai/helpers/embedding.ts +++ b/src/adapters/openai/helpers/embedding.ts @@ -11,15 +11,19 @@ export class Embedding extends SuperOpenAi { this.context = context; } - async createEmbedding(text: string): Promise { - const params: OpenAI.EmbeddingCreateParams = { - model: "text-embedding-3-large", - input: text, - dimensions: VECTOR_SIZE, - }; - const response = await this.client.embeddings.create({ - ...params, - }); - return response.data[0]?.embedding; + async createEmbedding(text: string | null): Promise { + if (text === null) { + return new Array(VECTOR_SIZE).fill(0); + } else { + const params: OpenAI.EmbeddingCreateParams = { + model: "text-embedding-3-large", + input: text, + dimensions: VECTOR_SIZE, + }; + const response = await this.client.embeddings.create({ + ...params, + }); + return response.data[0]?.embedding; + } } } diff --git a/src/adapters/supabase/helpers/comment.ts b/src/adapters/supabase/helpers/comment.ts index 43ae42e..d934f35 100644 --- a/src/adapters/supabase/helpers/comment.ts +++ b/src/adapters/supabase/helpers/comment.ts @@ -4,7 +4,7 @@ import { Context } from "../../../types/context"; export interface CommentType { id: string; - plaintext: string; + plaintext?: string; author_id: number; created_at: string; modified_at: string; @@ -30,7 +30,7 @@ export class Comment extends SuperSupabase { //Create the embedding for this comment const embedding = await this.context.adapters.openai.embedding.createEmbedding(plaintext); if (isPrivate) { - plaintext = "CENSORED"; + plaintext = null; } const { error } = await this.supabase.from("issue_comments").insert([{ id: commentNodeId, plaintext, author_id: authorId, embedding: embedding }]); if (error) { @@ -45,7 +45,7 @@ export class Comment extends SuperSupabase { //Create the embedding for this comment const embedding = Array.from(await this.context.adapters.openai.embedding.createEmbedding(plaintext)); if (isPrivate) { - plaintext = "CENSORED"; + plaintext = null; } const { error } = await this.supabase.from("issue_comments").update({ plaintext, embedding: embedding, modified_at: new Date() }).eq("id", commentNodeId); if (error) { diff --git a/src/handlers/add-comments.ts b/src/handlers/add-comments.ts index 5758861..aa3eb2e 100644 --- a/src/handlers/add-comments.ts +++ b/src/handlers/add-comments.ts @@ -7,7 +7,7 @@ export async function addComments(context: Context) { adapters: { supabase }, } = context; const plaintext = payload.comment.body; - const authorId = payload.comment.user?.id || 0; + const authorId = payload.comment.user?.id || -1; const nodeId = payload.comment.node_id; const isPrivate = payload.repository.private; diff --git a/src/types/database.ts b/src/types/database.ts index 009eae6..3622f52 100644 --- a/src/types/database.ts +++ b/src/types/database.ts @@ -35,7 +35,7 @@ export type Database = { embedding: string; id: string; modified_at: string; - plaintext: string; + plaintext: string | null; }; Insert: { author_id: string; @@ -43,7 +43,7 @@ export type Database = { embedding: string; id: string; modified_at?: string; - plaintext: string; + plaintext?: string | null; }; Update: { author_id?: string; @@ -51,7 +51,7 @@ export type Database = { embedding?: string; id?: string; modified_at?: string; - plaintext?: string; + plaintext?: string | null; }; Relationships: []; }; diff --git a/supabase/migrations/20240909072603_issue_comments.sql b/supabase/migrations/20240909072603_issue_comments.sql new file mode 100644 index 0000000..93de8ff --- /dev/null +++ b/supabase/migrations/20240909072603_issue_comments.sql @@ -0,0 +1,2 @@ +ALTER TABLE issue_comments +ALTER COLUMN plaintext DROP NOT NULL; \ No newline at end of file diff --git a/tests/__mocks__/adapter.ts b/tests/__mocks__/adapter.ts index c2c02ad..bb7f4e2 100644 --- a/tests/__mocks__/adapter.ts +++ b/tests/__mocks__/adapter.ts @@ -5,7 +5,7 @@ import { STRINGS } from "./strings"; export interface CommentMock { id: string; - plaintext: string; + plaintext: string | null; author_id: number; embedding: number[]; } @@ -15,14 +15,14 @@ export function createMockAdapters(context: Context) { return { supabase: { comment: { - createComment: jest.fn(async (plaintext: string, commentNodeId: string, authorId: number) => { + createComment: jest.fn(async (plaintext: string | null, commentNodeId: string, authorId: number) => { if (commentMap.has(commentNodeId)) { throw new Error("Comment already exists"); } const embedding = await context.adapters.openai.embedding.createEmbedding(plaintext); commentMap.set(commentNodeId, { id: commentNodeId, plaintext, author_id: authorId, embedding }); }), - updateComment: jest.fn(async (plaintext: string, commentNodeId: string, isPrivate: boolean) => { + updateComment: jest.fn(async (plaintext: string | null, commentNodeId: string, isPrivate: boolean) => { console.log(commentMap); if (!commentMap.has(commentNodeId)) { throw new Error(STRINGS.COMMENT_DOES_NOT_EXIST); @@ -34,7 +34,7 @@ export function createMockAdapters(context: Context) { const { id, author_id } = originalComment; const embedding = await context.adapters.openai.embedding.createEmbedding(plaintext); if (isPrivate) { - plaintext = STRINGS.CENSORED; + plaintext = null; } commentMap.set(commentNodeId, { id, plaintext, author_id, embedding }); }), diff --git a/tests/__mocks__/strings.ts b/tests/__mocks__/strings.ts index 11ac64f..83d7306 100644 --- a/tests/__mocks__/strings.ts +++ b/tests/__mocks__/strings.ts @@ -15,5 +15,4 @@ export const STRINGS = { CONFIGURABLE_RESPONSE: "Hello, Code Reviewers!", INVALID_COMMAND: "/Goodbye", COMMENT_DOES_NOT_EXIST: "Comment does not exist", - CENSORED: "[CENSORED]", }; From 5dd651e9855f2d83f04084afaf8ab4dd2b97ef27 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Mon, 9 Sep 2024 03:41:52 -0400 Subject: [PATCH 09/37] fix: jest tests --- src/adapters/supabase/helpers/comment.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/adapters/supabase/helpers/comment.ts b/src/adapters/supabase/helpers/comment.ts index d934f35..5baa7f5 100644 --- a/src/adapters/supabase/helpers/comment.ts +++ b/src/adapters/supabase/helpers/comment.ts @@ -16,7 +16,7 @@ export class Comment extends SuperSupabase { super(supabase, context); } - async createComment(plaintext: string, commentNodeId: string, authorId: number, isPrivate: boolean) { + async createComment(plaintext: string | null, commentNodeId: string, authorId: number, isPrivate: boolean) { //First Check if the comment already exists const { data, error } = await this.supabase.from("issue_comments").select("*").eq("id", commentNodeId); if (error) { @@ -30,7 +30,7 @@ export class Comment extends SuperSupabase { //Create the embedding for this comment const embedding = await this.context.adapters.openai.embedding.createEmbedding(plaintext); if (isPrivate) { - plaintext = null; + plaintext = null as string | null; } const { error } = await this.supabase.from("issue_comments").insert([{ id: commentNodeId, plaintext, author_id: authorId, embedding: embedding }]); if (error) { @@ -41,11 +41,11 @@ export class Comment extends SuperSupabase { this.context.logger.info("Comment created successfully"); } - async updateComment(plaintext: string, commentNodeId: string, isPrivate: boolean) { + async updateComment(plaintext: string | null, commentNodeId: string, isPrivate: boolean) { //Create the embedding for this comment const embedding = Array.from(await this.context.adapters.openai.embedding.createEmbedding(plaintext)); if (isPrivate) { - plaintext = null; + plaintext = null as string | null; } const { error } = await this.supabase.from("issue_comments").update({ plaintext, embedding: embedding, modified_at: new Date() }).eq("id", commentNodeId); if (error) { From f847053df0525f0238a0caf0f601a7ef4c6a02c4 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Tue, 10 Sep 2024 13:40:17 -0400 Subject: [PATCH 10/37] feat: added voyage ai support --- .github/workflows/compute.yml | 1 + README.md | 1 + package.json | 3 +- src/adapters/index.ts | 13 +++- src/adapters/supabase/helpers/comment.ts | 11 +++- src/adapters/voyage/helpers/embedding.ts | 25 ++++++++ src/adapters/voyage/helpers/voyage.ts | 12 ++++ src/plugin.ts | 6 +- src/types/env.ts | 1 + tests/main.test.ts | 1 + yarn.lock | 78 +++++++++++++++++++++++- 11 files changed, 143 insertions(+), 9 deletions(-) create mode 100644 src/adapters/voyage/helpers/embedding.ts create mode 100644 src/adapters/voyage/helpers/voyage.ts diff --git a/.github/workflows/compute.yml b/.github/workflows/compute.yml index f83e80c..b435d3d 100644 --- a/.github/workflows/compute.yml +++ b/.github/workflows/compute.yml @@ -25,6 +25,7 @@ jobs: SUPABASE_URL: ${{ secrets.SUPABASE_URL }} SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + VOYAGE_API_KEY: ${{secrets.VOYAGE_API_KEY}} steps: - uses: actions/checkout@v4 diff --git a/README.md b/README.md index fb293b9..e612d3c 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ To set up the `.dev.vars` file, you will need to provide the following variables - `SUPABASE_URL`: The URL for your Supabase instance. - `SUPABASE_KEY`: The key for your Supabase instance. - `OPENAI_API_KEY`: The API key for OpenAI. +- `VOYAGE_API_KEY`: The API key for Voyage. ## Usage - Add the following to your `.ubiquibot-config.yml` file with the appropriate URL: diff --git a/package.json b/package.json index 1716467..27277b0 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,8 @@ "@ubiquity-dao/ubiquibot-logger": "^1.3.0", "dotenv": "16.4.5", "openai": "^4.56.0", - "typebox-validators": "0.3.5" + "typebox-validators": "0.3.5", + "voyageai": "^0.0.1-5" }, "devDependencies": { "@commitlint/cli": "19.3.0", diff --git a/src/adapters/index.ts b/src/adapters/index.ts index e5f0346..0e5986c 100644 --- a/src/adapters/index.ts +++ b/src/adapters/index.ts @@ -4,17 +4,24 @@ import { Comment } from "./supabase/helpers/comment"; import { SuperSupabase } from "./supabase/helpers/supabase"; import { SuperOpenAi } from "./openai/helpers/openai"; import OpenAI from "openai"; -import { Embedding } from "./openai/helpers/embedding"; +import { Embedding as OpenAiEmbedding } from "./openai/helpers/embedding"; +import { Embedding as VoyageEmbedding } from "./voyage/helpers/embedding"; +import { SuperVoyage } from "./voyage/helpers/voyage"; +import { VoyageAIClient } from "voyageai"; -export function createAdapters(supabaseClient: SupabaseClient, openai: OpenAI, context: Context) { +export function createAdapters(supabaseClient: SupabaseClient, openai: OpenAI, voyage: VoyageAIClient, context: Context) { return { supabase: { comment: new Comment(supabaseClient, context), super: new SuperSupabase(supabaseClient, context), }, openai: { - embedding: new Embedding(openai, context), + embedding: new OpenAiEmbedding(openai, context), super: new SuperOpenAi(openai, context), }, + voyage: { + embedding: new VoyageEmbedding(voyage, context), + super: new SuperVoyage(voyage, context), + }, }; } diff --git a/src/adapters/supabase/helpers/comment.ts b/src/adapters/supabase/helpers/comment.ts index 5baa7f5..92bf2f6 100644 --- a/src/adapters/supabase/helpers/comment.ts +++ b/src/adapters/supabase/helpers/comment.ts @@ -28,7 +28,11 @@ export class Comment extends SuperSupabase { return; } else { //Create the embedding for this comment - const embedding = await this.context.adapters.openai.embedding.createEmbedding(plaintext); + const embedding = await this.context.adapters.voyage.embedding.createEmbedding(plaintext); + //If embedding is smaller than 3072, pad it with 0s + if (embedding.length < 3072) { + embedding.push(...new Array(3072 - embedding.length).fill(0)); + } if (isPrivate) { plaintext = null as string | null; } @@ -43,7 +47,10 @@ export class Comment extends SuperSupabase { async updateComment(plaintext: string | null, commentNodeId: string, isPrivate: boolean) { //Create the embedding for this comment - const embedding = Array.from(await this.context.adapters.openai.embedding.createEmbedding(plaintext)); + const embedding = Array.from(await this.context.adapters.voyage.embedding.createEmbedding(plaintext)); + if (embedding.length < 3072) { + embedding.push(...new Array(3072 - embedding.length).fill(0)); + } if (isPrivate) { plaintext = null as string | null; } diff --git a/src/adapters/voyage/helpers/embedding.ts b/src/adapters/voyage/helpers/embedding.ts new file mode 100644 index 0000000..19d29ce --- /dev/null +++ b/src/adapters/voyage/helpers/embedding.ts @@ -0,0 +1,25 @@ +import { VoyageAIClient } from "voyageai"; +import { Context } from "../../../types"; +import { SuperVoyage } from "./voyage"; +const VECTOR_SIZE = 3072; + +export class Embedding extends SuperVoyage { + protected context: Context; + + constructor(client: VoyageAIClient, context: Context) { + super(client, context); + this.context = context; + } + + async createEmbedding(text: string | null): Promise { + if (text === null) { + return new Array(VECTOR_SIZE).fill(0); + } else { + const response = await this.client.embed({ + input: text, + model: "voyage-2", + }); + return (response.data && response.data[0]?.embedding) || []; + } + } +} diff --git a/src/adapters/voyage/helpers/voyage.ts b/src/adapters/voyage/helpers/voyage.ts new file mode 100644 index 0000000..c08c0af --- /dev/null +++ b/src/adapters/voyage/helpers/voyage.ts @@ -0,0 +1,12 @@ +import { VoyageAIClient } from "voyageai"; +import { Context } from "../../../types/context"; + +export class SuperVoyage { + protected client: VoyageAIClient; + protected context: Context; + + constructor(client: VoyageAIClient, context: Context) { + this.client = client; + this.context = context; + } +} diff --git a/src/plugin.ts b/src/plugin.ts index 4197ff6..5f79651 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -10,6 +10,7 @@ import { addComments } from "./handlers/add-comments"; import { updateComment } from "./handlers/update-comments"; import { deleteComment } from "./handlers/delete-comments"; import OpenAI from "openai"; +import { VoyageAIClient } from "voyageai"; /** * The main plugin function. Split for easier testing. @@ -40,6 +41,9 @@ export async function plugin(inputs: PluginInputs, env: Env) { const openaiClient = new OpenAI({ apiKey: env.OPENAI_API_KEY, }); + const voyageClient = new VoyageAIClient({ + apiKey: env.VOYAGE_API_KEY, + }); const context: Context = { eventName: inputs.eventName, payload: inputs.eventPayload, @@ -49,6 +53,6 @@ export async function plugin(inputs: PluginInputs, env: Env) { logger: new Logs("info" as LogLevel), adapters: {} as ReturnType, }; - context.adapters = createAdapters(supabase, openaiClient, context); + context.adapters = createAdapters(supabase, openaiClient, voyageClient, context); return runPlugin(context); } diff --git a/src/types/env.ts b/src/types/env.ts index 6f47748..fbe521e 100644 --- a/src/types/env.ts +++ b/src/types/env.ts @@ -14,6 +14,7 @@ export const envSchema = T.Object({ SUPABASE_URL: T.String(), SUPABASE_KEY: T.String(), OPENAI_API_KEY: T.String(), + VOYAGE_API_KEY: T.String(), }); export const envValidator = new StandardValidator(envSchema); diff --git a/tests/main.test.ts b/tests/main.test.ts index a372de2..4833556 100644 --- a/tests/main.test.ts +++ b/tests/main.test.ts @@ -42,6 +42,7 @@ describe("Plugin tests", () => { SUPABASE_KEY: "test", SUPABASE_URL: "test", OPENAI_API_KEY: "test", + VOYAGE_API_KEY: "test", }); const content = await response.json(); expect(content).toEqual(manifest); diff --git a/yarn.lock b/yarn.lock index 78b6afa..b483a55 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2443,6 +2443,11 @@ balanced-match@^1.0.0: resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== +base64-js@^1.3.1: + version "1.5.1" + resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" + integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== + before-after-hook@^2.2.0: version "2.2.3" resolved "https://registry.yarnpkg.com/before-after-hook/-/before-after-hook-2.2.3.tgz#c51e809c81a4e354084422b9b26bad88249c517c" @@ -2519,6 +2524,14 @@ buffer-from@^1.0.0: resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5" integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ== +buffer@^6.0.3: + version "6.0.3" + resolved "https://registry.yarnpkg.com/buffer/-/buffer-6.0.3.tgz#2ace578459cc8fbe2a70aaa8f52ee63b6a74c6c6" + integrity sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA== + dependencies: + base64-js "^1.3.1" + ieee754 "^1.2.1" + call-bind@^1.0.2, call-bind@^1.0.5, call-bind@^1.0.6, call-bind@^1.0.7: version "1.0.7" resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.7.tgz#06016599c40c56498c18769d2730be242b6fa3b9" @@ -3535,6 +3548,11 @@ eventemitter3@^5.0.1: resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-5.0.1.tgz#53f5ffd0a492ac800721bb42c66b841de96423c4" integrity sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA== +events@^3.3.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/events/-/events-3.3.0.tgz#31a95ad0a924e2d2c419a813aeb2c4e878ea7400" + integrity sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q== + execa@^5.0.0: version "5.1.1" resolved "https://registry.yarnpkg.com/execa/-/execa-5.1.1.tgz#f80ad9cbf4298f7bd1d4c9555c21e93741c411dd" @@ -3743,6 +3761,11 @@ formdata-node@^4.3.2: node-domexception "1.0.0" web-streams-polyfill "4.0.0-beta.3" +formdata-node@^6.0.3: + version "6.0.3" + resolved "https://registry.yarnpkg.com/formdata-node/-/formdata-node-6.0.3.tgz#48f8e2206ae2befded82af621ef015f08168dc6d" + integrity sha512-8e1++BCiTzUno9v5IZ2J6bv4RU+3UKDmqWUQD0MIMVCd9AdhWkO1gw57oo1mNEX1dMq2EGI+FbWz4B92pscSQg== + formdata-polyfill@^4.0.10: version "4.0.10" resolved "https://registry.yarnpkg.com/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz#24807c31c9d402e002ab3d8c720144ceb8848423" @@ -4072,6 +4095,11 @@ identity-function@^1.0.0: resolved "https://registry.yarnpkg.com/identity-function/-/identity-function-1.0.0.tgz#bea1159f0985239be3ca348edf40ce2f0dd2c21d" integrity sha512-kNrgUK0qI+9qLTBidsH85HjDLpZfrrS0ElquKKe/fJFdB3D7VeKdXXEvOPDUHSHOzdZKCAAaQIWWyp0l2yq6pw== +ieee754@^1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" + integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA== + ignore@^5.1.4, ignore@^5.1.8, ignore@^5.2.0, ignore@^5.3.1: version "5.3.1" resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.3.1.tgz#5073e554cd42c5b33b394375f538b8593e34d4ef" @@ -4821,6 +4849,11 @@ jiti@^1.21.0: resolved "https://registry.yarnpkg.com/jiti/-/jiti-1.21.6.tgz#6c7f7398dd4b3142767f9a168af2f317a428d268" integrity sha512-2yTgeWTWzMWkHu6Jp9NKgePDaYHbntiwvYuuJLbbN9vl7DC9DvXKOB2BC3ZZ92D3cvV/aflH0osDfwpHepQ53w== +js-base64@3.7.2: + version "3.7.2" + resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-3.7.2.tgz#816d11d81a8aff241603d19ce5761e13e41d7745" + integrity sha512-NnRs6dsyqUXejqk/yv2aiXlAvOs56sLkX6nUdeaNezI5LFFLlsZjOThmwnrcwh5ZZRwZlCMnVAY3CvhIhoVEKQ== + js-tokens@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499" @@ -5330,7 +5363,7 @@ node-fetch-native@^1.6.4: resolved "https://registry.yarnpkg.com/node-fetch-native/-/node-fetch-native-1.6.4.tgz#679fc8fd8111266d47d7e72c379f1bed9acff06e" integrity sha512-IhOigYzAKHd244OC0JIMIUrjzctirCmPkaIfhDeGcEETWof5zKYUW7e7MYvChGWh/4CJeXEgsRyGzuF334rOOQ== -node-fetch@^2.6.7: +node-fetch@2.7.0, node-fetch@^2.6.7: version "2.7.0" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== @@ -5752,6 +5785,11 @@ printable-characters@^1.0.42: resolved "https://registry.yarnpkg.com/printable-characters/-/printable-characters-1.0.42.tgz#3f18e977a9bd8eb37fcc4ff5659d7be90868b3d8" integrity sha512-dKp+C4iXWK4vVYZmYSd0KBH5F/h1HoZRsbJ82AVKRO3PEo8L4lBS/vLwhVtpwwuYcoIsVY+1JYKR268yn480uQ== +process@^0.11.10: + version "0.11.10" + resolved "https://registry.yarnpkg.com/process/-/process-0.11.10.tgz#7332300e840161bda3e69a1d1d91a7d4bc16f182" + integrity sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A== + prompts@^2.0.1: version "2.4.2" resolved "https://registry.yarnpkg.com/prompts/-/prompts-2.4.2.tgz#7b57e73b3a48029ad10ebd44f74b01722a4cb069" @@ -5770,6 +5808,13 @@ pure-rand@^6.0.0: resolved "https://registry.yarnpkg.com/pure-rand/-/pure-rand-6.1.0.tgz#d173cf23258231976ccbdb05247c9787957604f2" integrity sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA== +qs@6.11.2: + version "6.11.2" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.11.2.tgz#64bea51f12c1f5da1bc01496f48ffcff7c69d7d9" + integrity sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA== + dependencies: + side-channel "^1.0.4" + queue-microtask@^1.2.2: version "1.2.3" resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.3.tgz#4929228bbc724dfac43e0efb058caf7b6cfb6243" @@ -5803,6 +5848,17 @@ readable-stream@^3.4.0: string_decoder "^1.1.1" util-deprecate "^1.0.1" +readable-stream@^4.5.2: + version "4.5.2" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-4.5.2.tgz#9e7fc4c45099baeed934bff6eb97ba6cf2729e09" + integrity sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g== + dependencies: + abort-controller "^3.0.0" + buffer "^6.0.3" + events "^3.3.0" + process "^0.11.10" + string_decoder "^1.3.0" + readdirp@~3.6.0: version "3.6.0" resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.6.0.tgz#74a370bd857116e245b29cc97340cd431a02a6c7" @@ -6288,7 +6344,7 @@ string.prototype.trimstart@^1.0.8: define-properties "^1.2.1" es-object-atoms "^1.0.0" -string_decoder@^1.1.1: +string_decoder@^1.1.1, string_decoder@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e" integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA== @@ -6676,6 +6732,11 @@ uri-js@^4.2.2, uri-js@^4.4.1: dependencies: punycode "^2.1.0" +url-join@4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/url-join/-/url-join-4.0.1.tgz#b642e21a2646808ffa178c4c5fda39844e12cde7" + integrity sha512-jk1+QP6ZJqyOiuEI9AEWQfju/nB2Pw466kbA0LEZljHwKeMgd9WrAEgEGxjPDD2+TNbbb37rTyhEfrCXfuKXnA== + util-deprecate@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" @@ -6708,6 +6769,19 @@ vlq@^0.2.1: resolved "https://registry.yarnpkg.com/vlq/-/vlq-0.2.3.tgz#8f3e4328cf63b1540c0d67e1b2778386f8975b26" integrity sha512-DRibZL6DsNhIgYQ+wNdWDL2SL3bKPlVrRiBqV5yuMm++op8W4kGFtaQfCs4KEJn0wBZcHVHJ3eoywX8983k1ow== +voyageai@^0.0.1-5: + version "0.0.1-5" + resolved "https://registry.yarnpkg.com/voyageai/-/voyageai-0.0.1-5.tgz#e0457d991784900c16e4cdf095654f195d62fdf2" + integrity sha512-IuXSXM3l9J3NIq+MLHXacG/yhswpEgWIu9eBqoFqMRnFiDx00dLL62OWg6WqVSipddZLwFeWH1Kaj56x5eqhOQ== + dependencies: + form-data "^4.0.0" + formdata-node "^6.0.3" + js-base64 "3.7.2" + node-fetch "2.7.0" + qs "6.11.2" + readable-stream "^4.5.2" + url-join "4.0.1" + vscode-languageserver-textdocument@^1.0.11: version "1.0.11" resolved "https://registry.yarnpkg.com/vscode-languageserver-textdocument/-/vscode-languageserver-textdocument-1.0.11.tgz#0822a000e7d4dc083312580d7575fe9e3ba2e2bf" From 1549feb6065e5b82f1ec5289287ca88c49810bc6 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Tue, 10 Sep 2024 13:40:42 -0400 Subject: [PATCH 11/37] fix: cspell issue --- .cspell.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.cspell.json b/.cspell.json index 83a383f..b297a90 100644 --- a/.cspell.json +++ b/.cspell.json @@ -22,7 +22,8 @@ "knip", "mischeck", "commentbody", - "issuebody" + "issuebody", + "voyageai" ], "dictionaries": ["typescript", "node", "software-terms"], "import": ["@cspell/dict-typescript/cspell-ext.json", "@cspell/dict-node/cspell-ext.json", "@cspell/dict-software-terms"], From 71854b4aeae548e34e26856b4da4c33881335150 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Tue, 10 Sep 2024 23:45:37 -0400 Subject: [PATCH 12/37] fix: adds serialized comment object and payload to the schema --- .github/.ubiquibot-config.yml | 6 +++ .github/workflows/compute.yml | 2 +- README.md | 2 +- src/adapters/supabase/helpers/comment.ts | 13 +++++-- src/adapters/utils/cleancommentobject.ts | 37 +++++++++++++++++++ src/handlers/add-comments.ts | 4 +- src/handlers/update-comments.ts | 5 ++- src/plugin.ts | 2 +- src/types/env.ts | 2 +- .../20240911023641_issue_comments.sql | 2 + tests/main.test.ts | 12 ++++-- 11 files changed, 72 insertions(+), 15 deletions(-) create mode 100644 .github/.ubiquibot-config.yml create mode 100644 src/adapters/utils/cleancommentobject.ts create mode 100644 supabase/migrations/20240911023641_issue_comments.sql diff --git a/.github/.ubiquibot-config.yml b/.github/.ubiquibot-config.yml new file mode 100644 index 0000000..8a9597c --- /dev/null +++ b/.github/.ubiquibot-config.yml @@ -0,0 +1,6 @@ +plugins: + - name: test-app + id: test-app + uses: + - plugin: https://ubiquity-os-comment-vector-embeddings.sshivaditya.workers.dev + runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" ] diff --git a/.github/workflows/compute.yml b/.github/workflows/compute.yml index b435d3d..10678ce 100644 --- a/.github/workflows/compute.yml +++ b/.github/workflows/compute.yml @@ -25,7 +25,7 @@ jobs: SUPABASE_URL: ${{ secrets.SUPABASE_URL }} SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - VOYAGE_API_KEY: ${{secrets.VOYAGE_API_KEY}} + VOYAGEAI_API_KEY: ${{secrets.VOYAGEAI_API_KEY}} steps: - uses: actions/checkout@v4 diff --git a/README.md b/README.md index e612d3c..29cc9a0 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ To set up the `.dev.vars` file, you will need to provide the following variables - `SUPABASE_URL`: The URL for your Supabase instance. - `SUPABASE_KEY`: The key for your Supabase instance. - `OPENAI_API_KEY`: The API key for OpenAI. -- `VOYAGE_API_KEY`: The API key for Voyage. +- `VOYAGEAI_API_KEY`: The API key for Voyage. ## Usage - Add the following to your `.ubiquibot-config.yml` file with the appropriate URL: diff --git a/src/adapters/supabase/helpers/comment.ts b/src/adapters/supabase/helpers/comment.ts index 92bf2f6..ed37d48 100644 --- a/src/adapters/supabase/helpers/comment.ts +++ b/src/adapters/supabase/helpers/comment.ts @@ -16,7 +16,7 @@ export class Comment extends SuperSupabase { super(supabase, context); } - async createComment(plaintext: string | null, commentNodeId: string, authorId: number, isPrivate: boolean) { + async createComment(plaintext: string | null, commentNodeId: string, authorId: number, commentobject: JSON | null, isPrivate: boolean) { //First Check if the comment already exists const { data, error } = await this.supabase.from("issue_comments").select("*").eq("id", commentNodeId); if (error) { @@ -36,7 +36,9 @@ export class Comment extends SuperSupabase { if (isPrivate) { plaintext = null as string | null; } - const { error } = await this.supabase.from("issue_comments").insert([{ id: commentNodeId, plaintext, author_id: authorId, embedding: embedding }]); + const { error } = await this.supabase + .from("issue_comments") + .insert([{ id: commentNodeId, plaintext, author_id: authorId, commentobject, embedding: embedding }]); if (error) { this.context.logger.error("Error creating comment", error); return; @@ -45,7 +47,7 @@ export class Comment extends SuperSupabase { this.context.logger.info("Comment created successfully"); } - async updateComment(plaintext: string | null, commentNodeId: string, isPrivate: boolean) { + async updateComment(plaintext: string | null, commentNodeId: string, commentobject: JSON, isPrivate: boolean) { //Create the embedding for this comment const embedding = Array.from(await this.context.adapters.voyage.embedding.createEmbedding(plaintext)); if (embedding.length < 3072) { @@ -54,7 +56,10 @@ export class Comment extends SuperSupabase { if (isPrivate) { plaintext = null as string | null; } - const { error } = await this.supabase.from("issue_comments").update({ plaintext, embedding: embedding, modified_at: new Date() }).eq("id", commentNodeId); + const { error } = await this.supabase + .from("issue_comments") + .update({ plaintext, embedding: embedding, commentobject, modified_at: new Date() }) + .eq("id", commentNodeId); if (error) { this.context.logger.error("Error updating comment", error); } diff --git a/src/adapters/utils/cleancommentobject.ts b/src/adapters/utils/cleancommentobject.ts new file mode 100644 index 0000000..715a3ed --- /dev/null +++ b/src/adapters/utils/cleancommentobject.ts @@ -0,0 +1,37 @@ +import { EmitterWebhookEvent as WebhookEvent } from "@octokit/webhooks"; +import { Type } from "@sinclair/typebox"; + +const commentObjectSchema = Type.Object({ + action: Type.String(), + issue: Type.Object({ + id: Type.Number(), + number: Type.Number(), + title: Type.String(), + body: Type.String(), + user: Type.Object({ + login: Type.String(), + id: Type.Number(), + }), + }), + comment: Type.Object({ + author_association: Type.String(), + id: Type.Number(), + html_url: Type.String(), + issue_url: Type.String(), + user: Type.Object({ + login: Type.String(), + id: Type.Number(), + }), + }), +}); + +/** + * Cleans the comment object. + * + * @param commentObject - The comment object. + * @returns The cleaned comment object. + */ +export const cleanCommentObject = (commentObject: WebhookEvent["payload"]): JSON => { + // Apply the schema to the comment object + return commentObjectSchema.parse(commentObject); +}; diff --git a/src/handlers/add-comments.ts b/src/handlers/add-comments.ts index aa3eb2e..f572567 100644 --- a/src/handlers/add-comments.ts +++ b/src/handlers/add-comments.ts @@ -1,3 +1,4 @@ +import { cleanCommentObject } from "../adapters/utils/cleancommentobject"; import { Context } from "../types"; export async function addComments(context: Context) { @@ -6,6 +7,7 @@ export async function addComments(context: Context) { payload, adapters: { supabase }, } = context; + const commentobject = cleanCommentObject(payload); const plaintext = payload.comment.body; const authorId = payload.comment.user?.id || -1; const nodeId = payload.comment.node_id; @@ -13,7 +15,7 @@ export async function addComments(context: Context) { // Add the comment to the database try { - await supabase.comment.createComment(plaintext, nodeId, authorId, isPrivate); + await supabase.comment.createComment(plaintext, nodeId, authorId, commentobject as JSON, isPrivate); } catch (error) { if (error instanceof Error) { logger.error(`Error creating comment:`, { error: error, stack: error.stack }); diff --git a/src/handlers/update-comments.ts b/src/handlers/update-comments.ts index 4c51f17..cf85a9d 100644 --- a/src/handlers/update-comments.ts +++ b/src/handlers/update-comments.ts @@ -1,3 +1,4 @@ +import { cleanCommentObject } from "../adapters/utils/cleancommentobject"; import { Context } from "../types"; export async function updateComment(context: Context) { @@ -6,13 +7,13 @@ export async function updateComment(context: Context) { payload, adapters: { supabase }, } = context; - + const commentobject = cleanCommentObject(payload); const nodeId = payload.comment.node_id; const isPrivate = payload.repository.private; const plaintext = payload.comment.body; // Fetch the previous comment and update it in the db try { - await supabase.comment.updateComment(plaintext, nodeId, isPrivate); + await supabase.comment.updateComment(plaintext, nodeId, commentobject, isPrivate); } catch (error) { if (error instanceof Error) { logger.error(`Error updating comment:`, { error: error, stack: error.stack }); diff --git a/src/plugin.ts b/src/plugin.ts index 5f79651..0b8de96 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -42,7 +42,7 @@ export async function plugin(inputs: PluginInputs, env: Env) { apiKey: env.OPENAI_API_KEY, }); const voyageClient = new VoyageAIClient({ - apiKey: env.VOYAGE_API_KEY, + apiKey: env.VOYAGEAI_API_KEY, }); const context: Context = { eventName: inputs.eventName, diff --git a/src/types/env.ts b/src/types/env.ts index fbe521e..1fe1588 100644 --- a/src/types/env.ts +++ b/src/types/env.ts @@ -14,7 +14,7 @@ export const envSchema = T.Object({ SUPABASE_URL: T.String(), SUPABASE_KEY: T.String(), OPENAI_API_KEY: T.String(), - VOYAGE_API_KEY: T.String(), + VOYAGEAI_API_KEY: T.String(), }); export const envValidator = new StandardValidator(envSchema); diff --git a/supabase/migrations/20240911023641_issue_comments.sql b/supabase/migrations/20240911023641_issue_comments.sql new file mode 100644 index 0000000..06f28b5 --- /dev/null +++ b/supabase/migrations/20240911023641_issue_comments.sql @@ -0,0 +1,2 @@ +ALTER TABLE issue_comments +ADD COLUMN commentobject jsonb; \ No newline at end of file diff --git a/tests/main.test.ts b/tests/main.test.ts index 4833556..384d95a 100644 --- a/tests/main.test.ts +++ b/tests/main.test.ts @@ -14,6 +14,7 @@ import { Logs } from "@ubiquity-dao/ubiquibot-logger"; import { Env } from "../src/types"; import { runPlugin } from "../src/plugin"; import { CommentMock, createMockAdapters } from "./__mocks__/adapter"; +import { cleanCommentObject } from "../src/adapters/utils/cleancommentobject"; dotenv.config(); jest.requireActual("@octokit/rest"); @@ -42,7 +43,7 @@ describe("Plugin tests", () => { SUPABASE_KEY: "test", SUPABASE_URL: "test", OPENAI_API_KEY: "test", - VOYAGE_API_KEY: "test", + VOYAGEAI_API_KEY: "test", }); const content = await response.json(); expect(content).toEqual(manifest); @@ -52,8 +53,9 @@ describe("Plugin tests", () => { const { context } = createContext(STRINGS.HELLO_WORLD, 1, 1, 1, 1, "sasasCreate"); await runPlugin(context); const supabase = context.adapters.supabase; + const commentObject = cleanCommentObject(context.payload); try { - await supabase.comment.createComment(STRINGS.HELLO_WORLD, "sasasCreate", 1, false); + await supabase.comment.createComment(STRINGS.HELLO_WORLD, "sasasCreate", 1, commentObject, false); throw new Error("Expected method to reject."); } catch (error) { if (error instanceof Error) { @@ -69,7 +71,8 @@ describe("Plugin tests", () => { it("When a comment is updated it should update the database", async () => { const { context } = createContext("Updated Message", 1, 1, 1, 1, "sasasUpdate", "issue_comment.edited"); const supabase = context.adapters.supabase; - await supabase.comment.createComment(STRINGS.HELLO_WORLD, "sasasUpdate", 1, false); + const commentObject = cleanCommentObject(context.payload); + await supabase.comment.createComment(STRINGS.HELLO_WORLD, "sasasUpdate", 1, commentObject, false); await runPlugin(context); const comment = (await supabase.comment.getComment("sasasUpdate")) as unknown as CommentMock; expect(comment).toBeDefined(); @@ -80,7 +83,8 @@ describe("Plugin tests", () => { it("When a comment is deleted it should delete it from the database", async () => { const { context } = createContext("Text Message", 1, 1, 1, 1, "sasasDelete", "issue_comment.deleted"); const supabase = context.adapters.supabase; - await supabase.comment.createComment(STRINGS.HELLO_WORLD, "sasasDelete", 1, false); + const commentObject = cleanCommentObject(context.payload); + await supabase.comment.createComment(STRINGS.HELLO_WORLD, "sasasDelete", 1, commentObject, false); await runPlugin(context); try { await supabase.comment.getComment("sasasDelete"); From e6a47b4e470b1730bb46cc638bb48f8ad6986bc1 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Tue, 10 Sep 2024 23:48:29 -0400 Subject: [PATCH 13/37] fix: updated handling for private repo --- src/adapters/supabase/helpers/comment.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/adapters/supabase/helpers/comment.ts b/src/adapters/supabase/helpers/comment.ts index ed37d48..83f0eb1 100644 --- a/src/adapters/supabase/helpers/comment.ts +++ b/src/adapters/supabase/helpers/comment.ts @@ -35,6 +35,7 @@ export class Comment extends SuperSupabase { } if (isPrivate) { plaintext = null as string | null; + commentobject = null as JSON | null; } const { error } = await this.supabase .from("issue_comments") @@ -47,7 +48,7 @@ export class Comment extends SuperSupabase { this.context.logger.info("Comment created successfully"); } - async updateComment(plaintext: string | null, commentNodeId: string, commentobject: JSON, isPrivate: boolean) { + async updateComment(plaintext: string | null, commentNodeId: string, commentobject: JSON | null, isPrivate: boolean) { //Create the embedding for this comment const embedding = Array.from(await this.context.adapters.voyage.embedding.createEmbedding(plaintext)); if (embedding.length < 3072) { @@ -55,6 +56,7 @@ export class Comment extends SuperSupabase { } if (isPrivate) { plaintext = null as string | null; + commentobject = null as JSON | null; } const { error } = await this.supabase .from("issue_comments") From f8a14213588cf00db44c624ca5cb4365e70a0d1d Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Wed, 11 Sep 2024 01:53:15 -0400 Subject: [PATCH 14/37] fix: test --- src/adapters/supabase/helpers/comment.ts | 8 +- src/adapters/utils/cleancommentobject.ts | 80 +++++++++------ src/handlers/add-comments.ts | 2 +- tests/__mocks__/adapter.ts | 21 ++-- tests/__mocks__/db.ts | 122 +++++++++++++++++------ tests/__mocks__/handlers.ts | 8 -- tests/__mocks__/helpers.ts | 76 ++++++++++++-- tests/main.test.ts | 7 +- 8 files changed, 228 insertions(+), 96 deletions(-) diff --git a/src/adapters/supabase/helpers/comment.ts b/src/adapters/supabase/helpers/comment.ts index 83f0eb1..944f610 100644 --- a/src/adapters/supabase/helpers/comment.ts +++ b/src/adapters/supabase/helpers/comment.ts @@ -16,7 +16,7 @@ export class Comment extends SuperSupabase { super(supabase, context); } - async createComment(plaintext: string | null, commentNodeId: string, authorId: number, commentobject: JSON | null, isPrivate: boolean) { + async createComment(plaintext: string | null, commentNodeId: string, authorId: number, commentobject: Record | null, isPrivate: boolean) { //First Check if the comment already exists const { data, error } = await this.supabase.from("issue_comments").select("*").eq("id", commentNodeId); if (error) { @@ -35,7 +35,7 @@ export class Comment extends SuperSupabase { } if (isPrivate) { plaintext = null as string | null; - commentobject = null as JSON | null; + commentobject = null as Record | null; } const { error } = await this.supabase .from("issue_comments") @@ -48,7 +48,7 @@ export class Comment extends SuperSupabase { this.context.logger.info("Comment created successfully"); } - async updateComment(plaintext: string | null, commentNodeId: string, commentobject: JSON | null, isPrivate: boolean) { + async updateComment(plaintext: string | null, commentNodeId: string, commentobject: Record | null, isPrivate: boolean) { //Create the embedding for this comment const embedding = Array.from(await this.context.adapters.voyage.embedding.createEmbedding(plaintext)); if (embedding.length < 3072) { @@ -56,7 +56,7 @@ export class Comment extends SuperSupabase { } if (isPrivate) { plaintext = null as string | null; - commentobject = null as JSON | null; + commentobject = null as Record | null; } const { error } = await this.supabase .from("issue_comments") diff --git a/src/adapters/utils/cleancommentobject.ts b/src/adapters/utils/cleancommentobject.ts index 715a3ed..7dc0d7e 100644 --- a/src/adapters/utils/cleancommentobject.ts +++ b/src/adapters/utils/cleancommentobject.ts @@ -1,37 +1,53 @@ -import { EmitterWebhookEvent as WebhookEvent } from "@octokit/webhooks"; -import { Type } from "@sinclair/typebox"; - -const commentObjectSchema = Type.Object({ - action: Type.String(), - issue: Type.Object({ - id: Type.Number(), - number: Type.Number(), - title: Type.String(), - body: Type.String(), - user: Type.Object({ - login: Type.String(), - id: Type.Number(), - }), - }), - comment: Type.Object({ - author_association: Type.String(), - id: Type.Number(), - html_url: Type.String(), - issue_url: Type.String(), - user: Type.Object({ - login: Type.String(), - id: Type.Number(), - }), - }), -}); +import { SupportedEvents, SupportedEventsU } from "../../types/context"; /** - * Cleans the comment object. + * Extracts only the properties mentioned in the TypeBox schema from the input object. * * @param commentObject - The comment object. - * @returns The cleaned comment object. + * @returns The object containing only the properties defined in the schema. */ -export const cleanCommentObject = (commentObject: WebhookEvent["payload"]): JSON => { - // Apply the schema to the comment object - return commentObjectSchema.parse(commentObject); -}; +export function cleanCommentObject( + commentObject: TU["payload"] +): Record { + // Manually extract properties + return { + action: commentObject.action as string, + issue: { + id: commentObject.issue.id as number, + number: commentObject.issue.number as number, + title: commentObject.issue.title as string, + body: commentObject.issue.body as string, + user: { + login: commentObject.issue.user.login as string, + id: commentObject.issue.user.id as number, + }, + author_association: commentObject.issue.author_association as string, + }, + comment: { + author_association: commentObject.comment.author_association as string, + id: commentObject.comment.id as number, + html_url: commentObject.comment.html_url as string, + issue_url: commentObject.comment.issue_url as string, + user: { + login: (commentObject.comment.user || { login: "" }).login as string, + id: (commentObject.comment.user || { id: -1 }).id as number, + }, + body: commentObject.comment.body as string, + created_at: commentObject.comment.created_at as string, + updated_at: commentObject.comment.updated_at as string, + }, + + repository: { + id: commentObject.repository.id as number, + node_id: commentObject.repository.node_id as string, + name: commentObject.repository.name as string, + full_name: commentObject.repository.full_name as string, + private: commentObject.repository.private as boolean, + owner: { + login: commentObject.repository.owner.login as string, + id: commentObject.repository.owner.id as number, + avatar_url: commentObject.repository.owner.avatar_url as string, + }, + }, + } as Record; +} diff --git a/src/handlers/add-comments.ts b/src/handlers/add-comments.ts index f572567..f854520 100644 --- a/src/handlers/add-comments.ts +++ b/src/handlers/add-comments.ts @@ -15,7 +15,7 @@ export async function addComments(context: Context) { // Add the comment to the database try { - await supabase.comment.createComment(plaintext, nodeId, authorId, commentobject as JSON, isPrivate); + await supabase.comment.createComment(plaintext, nodeId, authorId, commentobject, isPrivate); } catch (error) { if (error instanceof Error) { logger.error(`Error creating comment:`, { error: error, stack: error.stack }); diff --git a/tests/__mocks__/adapter.ts b/tests/__mocks__/adapter.ts index bb7f4e2..65078ea 100644 --- a/tests/__mocks__/adapter.ts +++ b/tests/__mocks__/adapter.ts @@ -7,6 +7,7 @@ export interface CommentMock { id: string; plaintext: string | null; author_id: number; + commentobject?: Record | null; embedding: number[]; } @@ -15,15 +16,19 @@ export function createMockAdapters(context: Context) { return { supabase: { comment: { - createComment: jest.fn(async (plaintext: string | null, commentNodeId: string, authorId: number) => { - if (commentMap.has(commentNodeId)) { - throw new Error("Comment already exists"); + createComment: jest.fn( + async (plaintext: string | null, commentNodeId: string, authorId: number, commentObject: Record | null, isPrivate: boolean) => { + if (commentMap.has(commentNodeId)) { + throw new Error("Comment already exists"); + } + const embedding = await context.adapters.openai.embedding.createEmbedding(plaintext); + if (isPrivate) { + plaintext = null; + } + commentMap.set(commentNodeId, { id: commentNodeId, plaintext, author_id: authorId, embedding }); } - const embedding = await context.adapters.openai.embedding.createEmbedding(plaintext); - commentMap.set(commentNodeId, { id: commentNodeId, plaintext, author_id: authorId, embedding }); - }), - updateComment: jest.fn(async (plaintext: string | null, commentNodeId: string, isPrivate: boolean) => { - console.log(commentMap); + ), + updateComment: jest.fn(async (plaintext: string | null, commentNodeId: string, commentObject: Record | null, isPrivate: boolean) => { if (!commentMap.has(commentNodeId)) { throw new Error(STRINGS.COMMENT_DOES_NOT_EXIST); } diff --git a/tests/__mocks__/db.ts b/tests/__mocks__/db.ts index e1883e5..1681106 100644 --- a/tests/__mocks__/db.ts +++ b/tests/__mocks__/db.ts @@ -7,36 +7,82 @@ import { factory, nullable, primaryKey } from "@mswjs/data"; export const db = factory({ users: { id: primaryKey(Number), - name: String, login: String, + avatar_url: nullable(String), // Add any additional fields based on the schema }, - issue: { + repo: { id: primaryKey(Number), - assignees: Array, + name: String, + full_name: String, // Assuming full_name is needed for repo + private: Boolean, + owner: { + login: String, + id: Number, + avatar_url: nullable(String), + }, html_url: String, - repository_url: String, - state: String, + description: nullable(String), + fork: Boolean, + url: String, + forks_url: String, + keys_url: String, + collaborators_url: String, + teams_url: String, + hooks_url: String, + issue_events_url: String, + events_url: String, + assignees_url: String, + branches_url: String, + tags_url: String, + blobs_url: String, + git_tags_url: String, + git_refs_url: String, + trees_url: String, + statuses_url: String, + languages_url: String, + stargazers_url: String, + contributors_url: String, + subscribers_url: String, + subscription_url: String, + commits_url: String, + git_commits_url: String, + comments_url: String, + issue_comment_url: String, + contents_url: String, + compare_url: String, + merges_url: String, + archive_url: String, + downloads_url: String, + issues_url: String, + pulls_url: String, + milestones_url: String, + notifications_url: String, + labels_url: String, + releases_url: String, + deployments_url: String, + }, + issue: { + id: primaryKey(Number), + number: Number, + title: String, + body: String, + user: { + login: String, + id: Number, + }, owner: String, repo: String, - labels: Array, author_association: String, - body: nullable(String), - closed_at: nullable(Date), - created_at: nullable(Date), + created_at: Date, + updated_at: Date, comments: Number, - comments_url: String, - events_url: String, - labels_url: String, + labels: Array, + state: String, locked: Boolean, - node_id: String, - title: String, - number: Number, - updated_at: Date, - url: String, - user: nullable(Object), - milestone: nullable(Object), assignee: nullable({ - avatar_url: String, + login: String, + id: Number, + avatar_url: nullable(String), email: nullable(String), events_url: String, followers_url: String, @@ -44,8 +90,6 @@ export const db = factory({ gists_url: String, gravatar_id: nullable(String), html_url: String, - id: Number, - login: String, name: nullable(String), node_id: String, organizations_url: String, @@ -58,17 +102,28 @@ export const db = factory({ type: String, url: String, }), - }, - repo: { - id: primaryKey(Number), - html_url: String, - name: String, - labels: Array, - owner: { - login: String, - id: Number, + milestone: nullable({ + title: String, + description: nullable(String), + due_on: nullable(Date), + number: Number, + state: String, + }), + reactions: { + url: String, + total_count: Number, + "+1": Number, + "-1": Number, + laugh: Number, + hooray: Number, + confused: Number, + heart: Number, + rocket: Number, + eyes: Number, }, - issues: Array, + timeline_url: String, + performed_via_github_app: nullable(Object), + state_reason: nullable(String), }, issueComments: { id: primaryKey(Number), @@ -81,5 +136,8 @@ export const db = factory({ login: String, id: Number, }, + author_association: String, + html_url: String, + issue_url: String, }, }); diff --git a/tests/__mocks__/handlers.ts b/tests/__mocks__/handlers.ts index f1c494a..0019673 100644 --- a/tests/__mocks__/handlers.ts +++ b/tests/__mocks__/handlers.ts @@ -1,6 +1,5 @@ import { http, HttpResponse } from "msw"; import { db } from "./db"; -import issueTemplate from "./issue-template"; /** * Intercepts the routes and returns a custom payload */ @@ -31,13 +30,6 @@ export const handlers = [ } return HttpResponse.json(item); }), - // create issue - http.post("https://api.github.com/repos/:owner/:repo/issues", () => { - const id = db.issue.count() + 1; - const newItem = { ...issueTemplate, id }; - db.issue.create(newItem); - return HttpResponse.json(newItem); - }), // create comment http.post("https://api.github.com/repos/:owner/:repo/issues/:issue_number/comments", async ({ params: { issue_number: issueNumber }, request }) => { const { body } = await getValue(request.body); diff --git a/tests/__mocks__/helpers.ts b/tests/__mocks__/helpers.ts index eb9e2ae..236ac8e 100644 --- a/tests/__mocks__/helpers.ts +++ b/tests/__mocks__/helpers.ts @@ -1,5 +1,4 @@ import { db } from "./db"; -import issueTemplate from "./issue-template"; import { STRINGS } from "./strings"; import usersGet from "./users-get.json"; @@ -12,34 +11,93 @@ import usersGet from "./users-get.json"; * Here is where you create issues, commits, pull requests, etc. */ export async function setupTests() { + // Insert users for (const item of usersGet) { - db.users.create(item); + db.users.create({ + login: item.login, + id: item.id, + }); } + // Insert repository db.repo.create({ id: 1, name: STRINGS.TEST_REPO, + full_name: `${STRINGS.USER_1}/${STRINGS.TEST_REPO}`, + private: false, owner: { login: STRINGS.USER_1, id: 1, + avatar_url: "", }, - issues: [], }); + // Insert issues db.issue.create({ - ...issueTemplate, + id: 1, + number: 1, + title: "First Issue", + body: "This is the body of the first issue.", + user: { + login: STRINGS.USER_1, + id: 1, + }, + author_association: "OWNER", + created_at: new Date().toISOString(), + updated_at: new Date().toISOString(), + comments: 0, + labels: [], + state: "open", + locked: false, + reactions: { + url: "", + total_count: 0, + "+1": 0, + "-1": 0, + laugh: 0, + hooray: 0, + confused: 0, + heart: 0, + rocket: 0, + eyes: 0, + }, + timeline_url: "", }); db.issue.create({ - ...issueTemplate, id: 2, number: 2, + title: "Second Issue", + body: "This is the body of the second issue.", + user: { + login: STRINGS.USER_1, + id: 1, + }, + author_association: "OWNER", + created_at: new Date().toISOString(), + updated_at: new Date().toISOString(), + comments: 0, labels: [], + state: "open", + locked: false, + reactions: { + url: "", + total_count: 0, + "+1": 0, + "-1": 0, + laugh: 0, + hooray: 0, + confused: 0, + heart: 0, + rocket: 0, + eyes: 0, + }, + timeline_url: "", }); } export function createComment(comment: string, commentId: number, nodeId: string) { - const isComment = db.issueComments.findFirst({ + const existingComment = db.issueComments.findFirst({ where: { id: { equals: commentId, @@ -47,7 +105,7 @@ export function createComment(comment: string, commentId: number, nodeId: string }, }); - if (isComment) { + if (existingComment) { db.issueComments.update({ where: { id: { @@ -56,6 +114,7 @@ export function createComment(comment: string, commentId: number, nodeId: string }, data: { body: comment, + updated_at: new Date().toISOString(), }, }); } else { @@ -68,6 +127,9 @@ export function createComment(comment: string, commentId: number, nodeId: string login: STRINGS.USER_1, id: 1, }, + created_at: new Date().toISOString(), + updated_at: new Date().toISOString(), + author_association: "OWNER", }); } } diff --git a/tests/main.test.ts b/tests/main.test.ts index 384d95a..4e47dd9 100644 --- a/tests/main.test.ts +++ b/tests/main.test.ts @@ -14,7 +14,6 @@ import { Logs } from "@ubiquity-dao/ubiquibot-logger"; import { Env } from "../src/types"; import { runPlugin } from "../src/plugin"; import { CommentMock, createMockAdapters } from "./__mocks__/adapter"; -import { cleanCommentObject } from "../src/adapters/utils/cleancommentobject"; dotenv.config(); jest.requireActual("@octokit/rest"); @@ -53,7 +52,7 @@ describe("Plugin tests", () => { const { context } = createContext(STRINGS.HELLO_WORLD, 1, 1, 1, 1, "sasasCreate"); await runPlugin(context); const supabase = context.adapters.supabase; - const commentObject = cleanCommentObject(context.payload); + const commentObject = null; try { await supabase.comment.createComment(STRINGS.HELLO_WORLD, "sasasCreate", 1, commentObject, false); throw new Error("Expected method to reject."); @@ -71,7 +70,7 @@ describe("Plugin tests", () => { it("When a comment is updated it should update the database", async () => { const { context } = createContext("Updated Message", 1, 1, 1, 1, "sasasUpdate", "issue_comment.edited"); const supabase = context.adapters.supabase; - const commentObject = cleanCommentObject(context.payload); + const commentObject = null; await supabase.comment.createComment(STRINGS.HELLO_WORLD, "sasasUpdate", 1, commentObject, false); await runPlugin(context); const comment = (await supabase.comment.getComment("sasasUpdate")) as unknown as CommentMock; @@ -83,7 +82,7 @@ describe("Plugin tests", () => { it("When a comment is deleted it should delete it from the database", async () => { const { context } = createContext("Text Message", 1, 1, 1, 1, "sasasDelete", "issue_comment.deleted"); const supabase = context.adapters.supabase; - const commentObject = cleanCommentObject(context.payload); + const commentObject = null; await supabase.comment.createComment(STRINGS.HELLO_WORLD, "sasasDelete", 1, commentObject, false); await runPlugin(context); try { From bccfe2388fb7270d52db0801bd6726a1349ef550 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Wed, 11 Sep 2024 01:54:29 -0400 Subject: [PATCH 15/37] fix: cspell --- .cspell.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.cspell.json b/.cspell.json index b297a90..2ae22ce 100644 --- a/.cspell.json +++ b/.cspell.json @@ -23,7 +23,9 @@ "mischeck", "commentbody", "issuebody", - "voyageai" + "voyageai", + "cleancommentobject", + "commentobject" ], "dictionaries": ["typescript", "node", "software-terms"], "import": ["@cspell/dict-typescript/cspell-ext.json", "@cspell/dict-node/cspell-ext.json", "@cspell/dict-software-terms"], From b6a8c7cce1ba4978757ecf13b887a7c8e87cc18e Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Wed, 11 Sep 2024 07:06:34 -0400 Subject: [PATCH 16/37] fix: cspell, removed openai, max length of vectors is 1024' --- .cspell.json | 2 - .dev.vars.example | 2 +- .env.example | 2 +- .github/workflows/compute.yml | 3 +- README.md | 1 - package.json | 3 +- src/adapters/index.ts | 9 +-- src/adapters/openai/helpers/embedding.ts | 29 -------- src/adapters/openai/helpers/openai.ts | 12 ---- src/adapters/supabase/helpers/comment.ts | 7 -- src/adapters/voyage/helpers/embedding.ts | 2 +- src/handlers/add-comments.ts | 6 +- src/handlers/update-comments.ts | 6 +- src/plugin.ts | 6 +- src/types/env.ts | 1 - .../20240911023641_issue_comments.sql | 2 +- .../20240911104515_issue_comments.sql | 5 ++ tests/__mocks__/adapter.ts | 11 ++- tests/main.test.ts | 2 - yarn.lock | 69 +------------------ 20 files changed, 26 insertions(+), 154 deletions(-) delete mode 100644 src/adapters/openai/helpers/embedding.ts delete mode 100644 src/adapters/openai/helpers/openai.ts create mode 100644 supabase/migrations/20240911104515_issue_comments.sql diff --git a/.cspell.json b/.cspell.json index 2ae22ce..e64757f 100644 --- a/.cspell.json +++ b/.cspell.json @@ -24,8 +24,6 @@ "commentbody", "issuebody", "voyageai", - "cleancommentobject", - "commentobject" ], "dictionaries": ["typescript", "node", "software-terms"], "import": ["@cspell/dict-typescript/cspell-ext.json", "@cspell/dict-node/cspell-ext.json", "@cspell/dict-software-terms"], diff --git a/.dev.vars.example b/.dev.vars.example index 6ceaeff..2fc8cf8 100644 --- a/.dev.vars.example +++ b/.dev.vars.example @@ -1,3 +1,3 @@ SUPABASE_URL="" SUPABASE_KEY="" -OPENAI_API_KEY="" \ No newline at end of file +VOYAGEAI_API_KEY="" \ No newline at end of file diff --git a/.env.example b/.env.example index 0272714..6e8b18f 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,3 @@ SUPABASE_URL= SUPABASE_KEY= -OPENAI_API_KEY= \ No newline at end of file +VOYAGEAI_API_KEY= \ No newline at end of file diff --git a/.github/workflows/compute.yml b/.github/workflows/compute.yml index 10678ce..8cb2597 100644 --- a/.github/workflows/compute.yml +++ b/.github/workflows/compute.yml @@ -24,7 +24,6 @@ jobs: env: SUPABASE_URL: ${{ secrets.SUPABASE_URL }} SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }} - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} VOYAGEAI_API_KEY: ${{secrets.VOYAGEAI_API_KEY}} steps: @@ -44,4 +43,4 @@ jobs: env: SUPABASE_URL: ${{ secrets.SUPABASE_URL }} SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }} - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + VOYAGEAI_API_KEY: ${{secrets.VOYAGEAI_API_KEY}} diff --git a/README.md b/README.md index 29cc9a0..6680b55 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,6 @@ This is a plugin for [Ubiquibot](https://github.com/ubiquity/ubiquibot-kernel). To set up the `.dev.vars` file, you will need to provide the following variables: - `SUPABASE_URL`: The URL for your Supabase instance. - `SUPABASE_KEY`: The key for your Supabase instance. -- `OPENAI_API_KEY`: The API key for OpenAI. - `VOYAGEAI_API_KEY`: The API key for Voyage. ## Usage diff --git a/package.json b/package.json index 27277b0..80f0dbf 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "knip-ci": "knip --no-exit-code --reporter json --config .github/knip.ts", "prepare": "husky install", "test": "jest --setupFiles dotenv/config --coverage", - "worker": "wrangler dev --env dev --port 5000", + "worker": "wrangler dev --env dev --port 4000", "supabase:generate:local": "supabase gen types typescript --local > src/types/database.ts", "supabase:generate:remote": "cross-env-shell \"supabase gen types typescript --project-id $SUPABASE_PROJECT_ID --schema public > src/types/database.ts\"" }, @@ -37,7 +37,6 @@ "@supabase/supabase-js": "^2.45.2", "@ubiquity-dao/ubiquibot-logger": "^1.3.0", "dotenv": "16.4.5", - "openai": "^4.56.0", "typebox-validators": "0.3.5", "voyageai": "^0.0.1-5" }, diff --git a/src/adapters/index.ts b/src/adapters/index.ts index 0e5986c..29c78a2 100644 --- a/src/adapters/index.ts +++ b/src/adapters/index.ts @@ -2,23 +2,16 @@ import { SupabaseClient } from "@supabase/supabase-js"; import { Context } from "../types"; import { Comment } from "./supabase/helpers/comment"; import { SuperSupabase } from "./supabase/helpers/supabase"; -import { SuperOpenAi } from "./openai/helpers/openai"; -import OpenAI from "openai"; -import { Embedding as OpenAiEmbedding } from "./openai/helpers/embedding"; import { Embedding as VoyageEmbedding } from "./voyage/helpers/embedding"; import { SuperVoyage } from "./voyage/helpers/voyage"; import { VoyageAIClient } from "voyageai"; -export function createAdapters(supabaseClient: SupabaseClient, openai: OpenAI, voyage: VoyageAIClient, context: Context) { +export function createAdapters(supabaseClient: SupabaseClient, voyage: VoyageAIClient, context: Context) { return { supabase: { comment: new Comment(supabaseClient, context), super: new SuperSupabase(supabaseClient, context), }, - openai: { - embedding: new OpenAiEmbedding(openai, context), - super: new SuperOpenAi(openai, context), - }, voyage: { embedding: new VoyageEmbedding(voyage, context), super: new SuperVoyage(voyage, context), diff --git a/src/adapters/openai/helpers/embedding.ts b/src/adapters/openai/helpers/embedding.ts deleted file mode 100644 index 3dfaeb9..0000000 --- a/src/adapters/openai/helpers/embedding.ts +++ /dev/null @@ -1,29 +0,0 @@ -import OpenAI from "openai"; -import { Context } from "../../../types"; -import { SuperOpenAi } from "./openai"; -const VECTOR_SIZE = 3072; - -export class Embedding extends SuperOpenAi { - protected context: Context; - - constructor(client: OpenAI, context: Context) { - super(client, context); - this.context = context; - } - - async createEmbedding(text: string | null): Promise { - if (text === null) { - return new Array(VECTOR_SIZE).fill(0); - } else { - const params: OpenAI.EmbeddingCreateParams = { - model: "text-embedding-3-large", - input: text, - dimensions: VECTOR_SIZE, - }; - const response = await this.client.embeddings.create({ - ...params, - }); - return response.data[0]?.embedding; - } - } -} diff --git a/src/adapters/openai/helpers/openai.ts b/src/adapters/openai/helpers/openai.ts deleted file mode 100644 index 108a3d2..0000000 --- a/src/adapters/openai/helpers/openai.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { OpenAI } from "openai"; -import { Context } from "../../../types/context"; - -export class SuperOpenAi { - protected client: OpenAI; - protected context: Context; - - constructor(client: OpenAI, context: Context) { - this.client = client; - this.context = context; - } -} diff --git a/src/adapters/supabase/helpers/comment.ts b/src/adapters/supabase/helpers/comment.ts index 944f610..291a039 100644 --- a/src/adapters/supabase/helpers/comment.ts +++ b/src/adapters/supabase/helpers/comment.ts @@ -29,10 +29,6 @@ export class Comment extends SuperSupabase { } else { //Create the embedding for this comment const embedding = await this.context.adapters.voyage.embedding.createEmbedding(plaintext); - //If embedding is smaller than 3072, pad it with 0s - if (embedding.length < 3072) { - embedding.push(...new Array(3072 - embedding.length).fill(0)); - } if (isPrivate) { plaintext = null as string | null; commentobject = null as Record | null; @@ -51,9 +47,6 @@ export class Comment extends SuperSupabase { async updateComment(plaintext: string | null, commentNodeId: string, commentobject: Record | null, isPrivate: boolean) { //Create the embedding for this comment const embedding = Array.from(await this.context.adapters.voyage.embedding.createEmbedding(plaintext)); - if (embedding.length < 3072) { - embedding.push(...new Array(3072 - embedding.length).fill(0)); - } if (isPrivate) { plaintext = null as string | null; commentobject = null as Record | null; diff --git a/src/adapters/voyage/helpers/embedding.ts b/src/adapters/voyage/helpers/embedding.ts index 19d29ce..cacc320 100644 --- a/src/adapters/voyage/helpers/embedding.ts +++ b/src/adapters/voyage/helpers/embedding.ts @@ -1,7 +1,7 @@ import { VoyageAIClient } from "voyageai"; import { Context } from "../../../types"; import { SuperVoyage } from "./voyage"; -const VECTOR_SIZE = 3072; +const VECTOR_SIZE = 1024; export class Embedding extends SuperVoyage { protected context: Context; diff --git a/src/handlers/add-comments.ts b/src/handlers/add-comments.ts index f854520..52ed0a5 100644 --- a/src/handlers/add-comments.ts +++ b/src/handlers/add-comments.ts @@ -1,4 +1,4 @@ -import { cleanCommentObject } from "../adapters/utils/cleancommentobject"; +import { cleanCommentObject } from "../adapters/utils/cleanCommentObject"; import { Context } from "../types"; export async function addComments(context: Context) { @@ -7,7 +7,7 @@ export async function addComments(context: Context) { payload, adapters: { supabase }, } = context; - const commentobject = cleanCommentObject(payload); + const commentObject = cleanCommentObject(payload); const plaintext = payload.comment.body; const authorId = payload.comment.user?.id || -1; const nodeId = payload.comment.node_id; @@ -15,7 +15,7 @@ export async function addComments(context: Context) { // Add the comment to the database try { - await supabase.comment.createComment(plaintext, nodeId, authorId, commentobject, isPrivate); + await supabase.comment.createComment(plaintext, nodeId, authorId, commentObject, isPrivate); } catch (error) { if (error instanceof Error) { logger.error(`Error creating comment:`, { error: error, stack: error.stack }); diff --git a/src/handlers/update-comments.ts b/src/handlers/update-comments.ts index cf85a9d..9461497 100644 --- a/src/handlers/update-comments.ts +++ b/src/handlers/update-comments.ts @@ -1,4 +1,4 @@ -import { cleanCommentObject } from "../adapters/utils/cleancommentobject"; +import { cleanCommentObject } from "../adapters/utils/cleanCommentObject"; import { Context } from "../types"; export async function updateComment(context: Context) { @@ -7,13 +7,13 @@ export async function updateComment(context: Context) { payload, adapters: { supabase }, } = context; - const commentobject = cleanCommentObject(payload); + const commentObject = cleanCommentObject(payload); const nodeId = payload.comment.node_id; const isPrivate = payload.repository.private; const plaintext = payload.comment.body; // Fetch the previous comment and update it in the db try { - await supabase.comment.updateComment(plaintext, nodeId, commentobject, isPrivate); + await supabase.comment.updateComment(plaintext, nodeId, commentObject, isPrivate); } catch (error) { if (error instanceof Error) { logger.error(`Error updating comment:`, { error: error, stack: error.stack }); diff --git a/src/plugin.ts b/src/plugin.ts index 0b8de96..9673231 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -9,7 +9,6 @@ import { createClient } from "@supabase/supabase-js"; import { addComments } from "./handlers/add-comments"; import { updateComment } from "./handlers/update-comments"; import { deleteComment } from "./handlers/delete-comments"; -import OpenAI from "openai"; import { VoyageAIClient } from "voyageai"; /** @@ -38,9 +37,6 @@ export async function runPlugin(context: Context) { export async function plugin(inputs: PluginInputs, env: Env) { const octokit = new Octokit({ auth: inputs.authToken }); const supabase = createClient(env.SUPABASE_URL, env.SUPABASE_KEY); - const openaiClient = new OpenAI({ - apiKey: env.OPENAI_API_KEY, - }); const voyageClient = new VoyageAIClient({ apiKey: env.VOYAGEAI_API_KEY, }); @@ -53,6 +49,6 @@ export async function plugin(inputs: PluginInputs, env: Env) { logger: new Logs("info" as LogLevel), adapters: {} as ReturnType, }; - context.adapters = createAdapters(supabase, openaiClient, voyageClient, context); + context.adapters = createAdapters(supabase, voyageClient, context); return runPlugin(context); } diff --git a/src/types/env.ts b/src/types/env.ts index 1fe1588..4c6cc1c 100644 --- a/src/types/env.ts +++ b/src/types/env.ts @@ -13,7 +13,6 @@ import { StandardValidator } from "typebox-validators"; export const envSchema = T.Object({ SUPABASE_URL: T.String(), SUPABASE_KEY: T.String(), - OPENAI_API_KEY: T.String(), VOYAGEAI_API_KEY: T.String(), }); diff --git a/supabase/migrations/20240911023641_issue_comments.sql b/supabase/migrations/20240911023641_issue_comments.sql index 06f28b5..4ef319b 100644 --- a/supabase/migrations/20240911023641_issue_comments.sql +++ b/supabase/migrations/20240911023641_issue_comments.sql @@ -1,2 +1,2 @@ ALTER TABLE issue_comments -ADD COLUMN commentobject jsonb; \ No newline at end of file +ADD COLUMN commentObject jsonb; \ No newline at end of file diff --git a/supabase/migrations/20240911104515_issue_comments.sql b/supabase/migrations/20240911104515_issue_comments.sql new file mode 100644 index 0000000..b0b2904 --- /dev/null +++ b/supabase/migrations/20240911104515_issue_comments.sql @@ -0,0 +1,5 @@ +ALTER TABLE issue_comments +DROP COLUMN embedding; + +ALTER TABLE issue_comments +ADD COLUMN embedding Vector(1024) NOT NULL; \ No newline at end of file diff --git a/tests/__mocks__/adapter.ts b/tests/__mocks__/adapter.ts index 65078ea..ee9956f 100644 --- a/tests/__mocks__/adapter.ts +++ b/tests/__mocks__/adapter.ts @@ -1,13 +1,12 @@ import { Context } from "../../src/types"; import { Comment } from "../../src/adapters/supabase/helpers/comment"; -import { Embedding } from "../../src/adapters/openai/helpers/embedding"; import { STRINGS } from "./strings"; export interface CommentMock { id: string; plaintext: string | null; author_id: number; - commentobject?: Record | null; + commentObject?: Record | null; embedding: number[]; } @@ -21,7 +20,7 @@ export function createMockAdapters(context: Context) { if (commentMap.has(commentNodeId)) { throw new Error("Comment already exists"); } - const embedding = await context.adapters.openai.embedding.createEmbedding(plaintext); + const embedding = await context.adapters.voyage.embedding.createEmbedding(plaintext); if (isPrivate) { plaintext = null; } @@ -37,7 +36,7 @@ export function createMockAdapters(context: Context) { throw new Error(STRINGS.COMMENT_DOES_NOT_EXIST); } const { id, author_id } = originalComment; - const embedding = await context.adapters.openai.embedding.createEmbedding(plaintext); + const embedding = await context.adapters.voyage.embedding.createEmbedding(plaintext); if (isPrivate) { plaintext = null; } @@ -57,7 +56,7 @@ export function createMockAdapters(context: Context) { }), } as unknown as Comment, }, - openai: { + voyage: { embedding: { createEmbedding: jest.fn(async (text: string) => { if (text && text.length > 0) { @@ -65,7 +64,7 @@ export function createMockAdapters(context: Context) { } return new Array(3072).fill(0); }), - } as unknown as Embedding, + } as unknown as number[], }, }; } diff --git a/tests/main.test.ts b/tests/main.test.ts index 4e47dd9..5f4fd84 100644 --- a/tests/main.test.ts +++ b/tests/main.test.ts @@ -18,7 +18,6 @@ import { CommentMock, createMockAdapters } from "./__mocks__/adapter"; dotenv.config(); jest.requireActual("@octokit/rest"); jest.requireActual("@supabase/supabase-js"); -jest.requireActual("openai"); const octokit = new Octokit(); beforeAll(() => { @@ -41,7 +40,6 @@ describe("Plugin tests", () => { const response = await worker.fetch(new Request("http://localhost/manifest.json"), { SUPABASE_KEY: "test", SUPABASE_URL: "test", - OPENAI_API_KEY: "test", VOYAGEAI_API_KEY: "test", }); const content = await response.json(); diff --git a/yarn.lock b/yarn.lock index b483a55..e653515 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1978,14 +1978,6 @@ dependencies: "@types/node" "*" -"@types/node-fetch@^2.6.4": - version "2.6.11" - resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.11.tgz#9b39b78665dae0e82a08f02f4967d62c66f95d24" - integrity sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g== - dependencies: - "@types/node" "*" - form-data "^4.0.0" - "@types/node-forge@^1.3.0": version "1.3.11" resolved "https://registry.yarnpkg.com/@types/node-forge/-/node-forge-1.3.11.tgz#0972ea538ddb0f4d9c2fa0ec5db5724773a604da" @@ -2007,13 +1999,6 @@ dependencies: undici-types "~5.26.4" -"@types/node@^18.11.18": - version "18.19.46" - resolved "https://registry.yarnpkg.com/@types/node/-/node-18.19.46.tgz#51801396c01153e0626e36f43386e83bc768b072" - integrity sha512-vnRgMS7W6cKa1/0G3/DTtQYpVrZ8c0Xm6UkLaVFrb9jtcVC3okokW09Ki1Qdrj9ISokszD69nY4WDLRlvHlhAA== - dependencies: - undici-types "~5.26.4" - "@types/phoenix@^1.5.4": version "1.6.5" resolved "https://registry.yarnpkg.com/@types/phoenix/-/phoenix-1.6.5.tgz#5654e14ec7ad25334a157a20015996b6d7d2075e" @@ -2191,13 +2176,6 @@ agent-base@^7.0.2: dependencies: debug "^4.3.4" -agentkeepalive@^4.2.1: - version "4.5.0" - resolved "https://registry.yarnpkg.com/agentkeepalive/-/agentkeepalive-4.5.0.tgz#2673ad1389b3c418c5a20c5d7364f93ca04be923" - integrity sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew== - dependencies: - humanize-ms "^1.2.1" - aggregate-error@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/aggregate-error/-/aggregate-error-3.1.0.tgz#92670ff50f5359bdb7a3e0d40d0ec30c5737687a" @@ -3739,11 +3717,6 @@ foreground-child@^3.1.0: cross-spawn "^7.0.0" signal-exit "^4.0.1" -form-data-encoder@1.7.2: - version "1.7.2" - resolved "https://registry.yarnpkg.com/form-data-encoder/-/form-data-encoder-1.7.2.tgz#1f1ae3dccf58ed4690b86d87e4f57c654fbab040" - integrity sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A== - form-data@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452" @@ -3753,14 +3726,6 @@ form-data@^4.0.0: combined-stream "^1.0.8" mime-types "^2.1.12" -formdata-node@^4.3.2: - version "4.4.1" - resolved "https://registry.yarnpkg.com/formdata-node/-/formdata-node-4.4.1.tgz#23f6a5cb9cb55315912cbec4ff7b0f59bbd191e2" - integrity sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ== - dependencies: - node-domexception "1.0.0" - web-streams-polyfill "4.0.0-beta.3" - formdata-node@^6.0.3: version "6.0.3" resolved "https://registry.yarnpkg.com/formdata-node/-/formdata-node-6.0.3.tgz#48f8e2206ae2befded82af621ef015f08168dc6d" @@ -4078,13 +4043,6 @@ human-signals@^5.0.0: resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-5.0.0.tgz#42665a284f9ae0dade3ba41ebc37eb4b852f3a28" integrity sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ== -humanize-ms@^1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/humanize-ms/-/humanize-ms-1.2.1.tgz#c46e3159a293f6b896da29316d8b6fe8bb79bbed" - integrity sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ== - dependencies: - ms "^2.0.0" - husky@9.0.11: version "9.0.11" resolved "https://registry.yarnpkg.com/husky/-/husky-9.0.11.tgz#fc91df4c756050de41b3e478b2158b87c1e79af9" @@ -5300,11 +5258,6 @@ ms@2.1.2: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== -ms@^2.0.0: - version "2.1.3" - resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" - integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== - msw@^2.0.8: version "2.3.1" resolved "https://registry.yarnpkg.com/msw/-/msw-2.3.1.tgz#bfc73e256ffc2c74ec4381b604abb258df35f32b" @@ -5353,7 +5306,7 @@ nice-try@^1.0.4: resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366" integrity sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ== -node-domexception@1.0.0, node-domexception@^1.0.0: +node-domexception@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5" integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ== @@ -5363,7 +5316,7 @@ node-fetch-native@^1.6.4: resolved "https://registry.yarnpkg.com/node-fetch-native/-/node-fetch-native-1.6.4.tgz#679fc8fd8111266d47d7e72c379f1bed9acff06e" integrity sha512-IhOigYzAKHd244OC0JIMIUrjzctirCmPkaIfhDeGcEETWof5zKYUW7e7MYvChGWh/4CJeXEgsRyGzuF334rOOQ== -node-fetch@2.7.0, node-fetch@^2.6.7: +node-fetch@2.7.0: version "2.7.0" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== @@ -5494,19 +5447,6 @@ onetime@^6.0.0: dependencies: mimic-fn "^4.0.0" -openai@^4.56.0: - version "4.56.0" - resolved "https://registry.yarnpkg.com/openai/-/openai-4.56.0.tgz#07d3982544cabd5781127288a8dfcceb7319a4cf" - integrity sha512-zcag97+3bG890MNNa0DQD9dGmmTWL8unJdNkulZzWRXrl+QeD+YkBI4H58rJcwErxqGK6a0jVPZ4ReJjhDGcmw== - dependencies: - "@types/node" "^18.11.18" - "@types/node-fetch" "^2.6.4" - abort-controller "^3.0.0" - agentkeepalive "^4.2.1" - form-data-encoder "1.7.2" - formdata-node "^4.3.2" - node-fetch "^2.6.7" - optionator@^0.9.3: version "0.9.4" resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.9.4.tgz#7ea1c1a5d91d764fb282139c88fe11e182a3a734" @@ -6806,11 +6746,6 @@ wcwidth@^1.0.1: dependencies: defaults "^1.0.3" -web-streams-polyfill@4.0.0-beta.3: - version "4.0.0-beta.3" - resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz#2898486b74f5156095e473efe989dcf185047a38" - integrity sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug== - web-streams-polyfill@^3.0.3: version "3.3.3" resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz#2073b91a2fdb1fbfbd401e7de0ac9f8214cecb4b" From 73d0a5a9c00577ba65af8f1bd3704717f6ea4cc5 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Wed, 11 Sep 2024 07:10:57 -0400 Subject: [PATCH 17/37] fix: module import error --- src/handlers/add-comments.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/handlers/add-comments.ts b/src/handlers/add-comments.ts index 52ed0a5..d082ddf 100644 --- a/src/handlers/add-comments.ts +++ b/src/handlers/add-comments.ts @@ -13,7 +13,6 @@ export async function addComments(context: Context) { const nodeId = payload.comment.node_id; const isPrivate = payload.repository.private; - // Add the comment to the database try { await supabase.comment.createComment(plaintext, nodeId, authorId, commentObject, isPrivate); } catch (error) { From 4679fe3822ac3abd43490f94d890b9f2b0488121 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Wed, 11 Sep 2024 07:15:51 -0400 Subject: [PATCH 18/37] fix: test --- .../utils/{cleancommentobject.ts => comment-object-clean.ts} | 0 src/handlers/add-comments.ts | 2 +- src/handlers/update-comments.ts | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename src/adapters/utils/{cleancommentobject.ts => comment-object-clean.ts} (100%) diff --git a/src/adapters/utils/cleancommentobject.ts b/src/adapters/utils/comment-object-clean.ts similarity index 100% rename from src/adapters/utils/cleancommentobject.ts rename to src/adapters/utils/comment-object-clean.ts diff --git a/src/handlers/add-comments.ts b/src/handlers/add-comments.ts index d082ddf..62486d4 100644 --- a/src/handlers/add-comments.ts +++ b/src/handlers/add-comments.ts @@ -1,4 +1,4 @@ -import { cleanCommentObject } from "../adapters/utils/cleanCommentObject"; +import { cleanCommentObject } from "../adapters/utils/comment-object-clean"; import { Context } from "../types"; export async function addComments(context: Context) { diff --git a/src/handlers/update-comments.ts b/src/handlers/update-comments.ts index 9461497..5ae6b46 100644 --- a/src/handlers/update-comments.ts +++ b/src/handlers/update-comments.ts @@ -1,4 +1,4 @@ -import { cleanCommentObject } from "../adapters/utils/cleanCommentObject"; +import { cleanCommentObject } from "../adapters/utils/comment-object-clean"; import { Context } from "../types"; export async function updateComment(context: Context) { From 6d1c52197fb8005e39904dcb1cf029c40cef1152 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Wed, 11 Sep 2024 21:31:46 -0400 Subject: [PATCH 19/37] feat: issue dedup --- .github/.ubiquibot-config.yml | 2 +- README.md | 2 +- manifest.json | 2 +- package.json | 2 +- src/adapters/index.ts | 2 + src/adapters/supabase/helpers/comment.ts | 22 ++-- src/adapters/supabase/helpers/issues.ts | 85 +++++++++++++ src/handlers/add-issue.ts | 29 +++++ src/handlers/delete-issues.ts | 26 ++++ src/handlers/issue-deduplication.ts | 113 ++++++++++++++++++ src/handlers/update-issue.ts | 28 +++++ src/plugin.ts | 10 ++ src/types/context.ts | 2 +- src/types/typeguards.ts | 13 +- .../20240911233824_issue_comments.sql | 8 ++ 15 files changed, 328 insertions(+), 18 deletions(-) create mode 100644 src/adapters/supabase/helpers/issues.ts create mode 100644 src/handlers/add-issue.ts create mode 100644 src/handlers/delete-issues.ts create mode 100644 src/handlers/issue-deduplication.ts create mode 100644 src/handlers/update-issue.ts create mode 100644 supabase/migrations/20240911233824_issue_comments.sql diff --git a/.github/.ubiquibot-config.yml b/.github/.ubiquibot-config.yml index 8a9597c..9213535 100644 --- a/.github/.ubiquibot-config.yml +++ b/.github/.ubiquibot-config.yml @@ -3,4 +3,4 @@ plugins: id: test-app uses: - plugin: https://ubiquity-os-comment-vector-embeddings.sshivaditya.workers.dev - runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" ] + runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issue.created", "issue.edited", "issue.deleted" ] diff --git a/README.md b/README.md index 6680b55..e4ca6bf 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ To set up the `.dev.vars` file, you will need to provide the following variables - Add the following to your `.ubiquibot-config.yml` file with the appropriate URL: ```javascript -plugin: http://127.0.0.1:4000 - runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" ] + runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" , "issue.created", "issue.edited", "issue.deleted"] ``` ## Testing Locally diff --git a/manifest.json b/manifest.json index 0e83faf..4b05cc9 100644 --- a/manifest.json +++ b/manifest.json @@ -1,5 +1,5 @@ { "name": "@ubiquity-os/comment-vector-embeddings", "description": "Issue comment plugin for Ubiquibot. It enables the storage, updating, and deletion of issue comment embeddings.", - "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted"] + "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issue.created", "issue.edited", "issue.deleted"] } diff --git a/package.json b/package.json index 80f0dbf..7718367 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "knip-ci": "knip --no-exit-code --reporter json --config .github/knip.ts", "prepare": "husky install", "test": "jest --setupFiles dotenv/config --coverage", - "worker": "wrangler dev --env dev --port 4000", + "worker": "wrangler dev --env dev --port 5000", "supabase:generate:local": "supabase gen types typescript --local > src/types/database.ts", "supabase:generate:remote": "cross-env-shell \"supabase gen types typescript --project-id $SUPABASE_PROJECT_ID --schema public > src/types/database.ts\"" }, diff --git a/src/adapters/index.ts b/src/adapters/index.ts index 29c78a2..ca31d27 100644 --- a/src/adapters/index.ts +++ b/src/adapters/index.ts @@ -5,11 +5,13 @@ import { SuperSupabase } from "./supabase/helpers/supabase"; import { Embedding as VoyageEmbedding } from "./voyage/helpers/embedding"; import { SuperVoyage } from "./voyage/helpers/voyage"; import { VoyageAIClient } from "voyageai"; +import { Issues } from "./supabase/helpers/issues"; export function createAdapters(supabaseClient: SupabaseClient, voyage: VoyageAIClient, context: Context) { return { supabase: { comment: new Comment(supabaseClient, context), + issue: new Issues(supabaseClient, context), super: new SuperSupabase(supabaseClient, context), }, voyage: { diff --git a/src/adapters/supabase/helpers/comment.ts b/src/adapters/supabase/helpers/comment.ts index 291a039..22f5e46 100644 --- a/src/adapters/supabase/helpers/comment.ts +++ b/src/adapters/supabase/helpers/comment.ts @@ -16,9 +16,9 @@ export class Comment extends SuperSupabase { super(supabase, context); } - async createComment(plaintext: string | null, commentNodeId: string, authorId: number, commentobject: Record | null, isPrivate: boolean) { + async createComment(plaintext: string | null, commentNodeId: string, authorId: number, payloadObject: Record | null, isPrivate: boolean) { //First Check if the comment already exists - const { data, error } = await this.supabase.from("issue_comments").select("*").eq("id", commentNodeId); + const { data, error } = await this.supabase.from("vectordump").select("*").eq("id", commentNodeId); if (error) { this.context.logger.error("Error creating comment", error); return; @@ -31,11 +31,11 @@ export class Comment extends SuperSupabase { const embedding = await this.context.adapters.voyage.embedding.createEmbedding(plaintext); if (isPrivate) { plaintext = null as string | null; - commentobject = null as Record | null; + payloadObject = null as Record | null; } const { error } = await this.supabase - .from("issue_comments") - .insert([{ id: commentNodeId, plaintext, author_id: authorId, commentobject, embedding: embedding }]); + .from("vectordump") + .insert([{ id: commentNodeId, plaintext, author_id: authorId, type: "comment", payloadobject: payloadObject, embedding: embedding }]); if (error) { this.context.logger.error("Error creating comment", error); return; @@ -44,16 +44,16 @@ export class Comment extends SuperSupabase { this.context.logger.info("Comment created successfully"); } - async updateComment(plaintext: string | null, commentNodeId: string, commentobject: Record | null, isPrivate: boolean) { + async updateComment(plaintext: string | null, commentNodeId: string, payloadObject: Record | null, isPrivate: boolean) { //Create the embedding for this comment const embedding = Array.from(await this.context.adapters.voyage.embedding.createEmbedding(plaintext)); if (isPrivate) { plaintext = null as string | null; - commentobject = null as Record | null; + payloadObject = null as Record | null; } const { error } = await this.supabase - .from("issue_comments") - .update({ plaintext, embedding: embedding, commentobject, modified_at: new Date() }) + .from("vectordump") + .update({ plaintext, embedding: embedding, payloadobject: payloadObject, modified_at: new Date() }) .eq("id", commentNodeId); if (error) { this.context.logger.error("Error updating comment", error); @@ -61,7 +61,7 @@ export class Comment extends SuperSupabase { } async getComment(commentNodeId: string): Promise { - const { data, error } = await this.supabase.from("issue_comments").select("*").eq("id", commentNodeId); + const { data, error } = await this.supabase.from("vectordump").select("*").eq("id", commentNodeId); if (error) { this.context.logger.error("Error getting comment", error); } @@ -69,7 +69,7 @@ export class Comment extends SuperSupabase { } async deleteComment(commentNodeId: string) { - const { error } = await this.supabase.from("issue_comments").delete().eq("id", commentNodeId); + const { error } = await this.supabase.from("vectordump").delete().eq("id", commentNodeId); if (error) { this.context.logger.error("Error deleting comment", error); } diff --git a/src/adapters/supabase/helpers/issues.ts b/src/adapters/supabase/helpers/issues.ts new file mode 100644 index 0000000..6048ccc --- /dev/null +++ b/src/adapters/supabase/helpers/issues.ts @@ -0,0 +1,85 @@ +import { SupabaseClient } from "@supabase/supabase-js"; +import { SuperSupabase } from "./supabase"; +import { Context } from "../../../types/context"; + +export interface IssueType { + id: string; + plaintext?: string; + author_id: number; + created_at: string; + modified_at: string; + payloadObject: Record | null; + embedding: number[]; +} + +export class Issues extends SuperSupabase { + constructor(supabase: SupabaseClient, context: Context) { + super(supabase, context); + } + + async createIssue(issueNodeId: string, payloadObject: Record | null, isPrivate: boolean, plaintext: string | null, authorId: number) { + //First Check if the issue already exists + const { data, error } = await this.supabase.from("issues").select("*").eq("id", issueNodeId); + if (error) { + this.context.logger.error("Error creating issue", error); + return; + } + if (data && data.length > 0) { + this.context.logger.info("Issue already exists"); + return; + } else { + const embedding = await this.context.adapters.voyage.embedding.createEmbedding(plaintext); + if (isPrivate) { + payloadObject = null; + plaintext = null; + } + const { error } = await this.supabase + .from("issues") + .insert([{ id: issueNodeId, payloadObject, type: "issue", plaintext, author_id: authorId, embedding }]); + if (error) { + this.context.logger.error("Error creating issue", error); + return; + } + } + this.context.logger.info("Issue created successfully"); + } + + async updateIssue(plaintext: string | null, issueNodeId: string, payloadObject: Record | null, isPrivate: boolean) { + //Create the embedding for this comment + const embedding = Array.from(await this.context.adapters.voyage.embedding.createEmbedding(plaintext)); + if (isPrivate) { + plaintext = null as string | null; + payloadObject = null as Record | null; + } + const { error } = await this.supabase + .from("vectordump") + .update({ plaintext, embedding: embedding, payloadObject, modified_at: new Date() }) + .eq("id", issueNodeId); + if (error) { + this.context.logger.error("Error updating comment", error); + } + } + + async deleteIssue(issueNodeId: string) { + const { error } = await this.supabase.from("vectordump").delete().eq("id", issueNodeId); + if (error) { + this.context.logger.error("Error deleting comment", error); + } + } + + async findSimilarIssues(plaintext: string, threshold: number): Promise { + const embedding = await this.context.adapters.voyage.embedding.createEmbedding(plaintext); + const { data, error } = await this.supabase + .from("issues") + .select("*") + .eq("type", "issue") + .textSearch("embedding", embedding.join(",")) + .order("embedding", { foreignTable: "issues", ascending: false }) + .lte("embedding", threshold); + if (error) { + this.context.logger.error("Error finding similar issues", error); + return []; + } + return data; + } +} diff --git a/src/handlers/add-issue.ts b/src/handlers/add-issue.ts new file mode 100644 index 0000000..ce7c315 --- /dev/null +++ b/src/handlers/add-issue.ts @@ -0,0 +1,29 @@ +import { Context } from "../types"; + +export async function addIssue(context: Context) { + const { + logger, + payload, + adapters: { supabase }, + } = context; + const payloadObject = payload; + const plaintext = payload.issue.body + payload.issue.title || ""; + const authorId = payload.issue.user?.id || -1; + const nodeId = payload.issue.node_id; + const isPrivate = payload.repository.private; + + try { + await supabase.issue.createIssue(nodeId, payloadObject, isPrivate, plaintext, authorId); + } catch (error) { + if (error instanceof Error) { + logger.error(`Error creating issue:`, { error: error, stack: error.stack }); + throw error; + } else { + logger.error(`Error creating issue:`, { err: error, error: new Error() }); + throw error; + } + } + + logger.ok(`Successfully created issue!`); + logger.verbose(`Exiting addIssue`); +} diff --git a/src/handlers/delete-issues.ts b/src/handlers/delete-issues.ts new file mode 100644 index 0000000..0e31a87 --- /dev/null +++ b/src/handlers/delete-issues.ts @@ -0,0 +1,26 @@ +import { Context } from "../types"; + +export async function deleteIssues(context: Context) { + const { + logger, + payload, + adapters: { supabase }, + } = context; + + const nodeId = payload.issue.node_id; + + try { + await supabase.issue.deleteIssue(nodeId); + } catch (error) { + if (error instanceof Error) { + logger.error(`Error deleting issue:`, { error: error, stack: error.stack }); + throw error; + } else { + logger.error(`Error deleting issue:`, { err: error, error: new Error() }); + throw error; + } + } + + logger.ok(`Successfully deleted issue!`); + logger.verbose(`Exiting deleteIssue`); +} diff --git a/src/handlers/issue-deduplication.ts b/src/handlers/issue-deduplication.ts new file mode 100644 index 0000000..c9dd522 --- /dev/null +++ b/src/handlers/issue-deduplication.ts @@ -0,0 +1,113 @@ +import { Context } from "../types"; +const MATCH_THRESHOLD = 0.95; +const WARNING_THRESHOLD = 0.75; + +export interface IssueGraphqlResponse { + id: string; + title: string; + url: string; +} + +/** + * Check if an issue is similar to any existing issues in the database + * @param context + * @returns true if the issue is similar to an existing issue, false otherwise + */ +export async function issueChecker(context: Context): Promise { + const { + logger, + payload, + adapters: { supabase }, + } = context; + + const issue = payload.issue; + + //First Check if an issue with more than MATCH_THRESHOLD similarity exists (Very Similar) + const similarIssue = await supabase.issue.findSimilarIssues(issue.body + issue.title, MATCH_THRESHOLD); + if (similarIssue) { + logger.info(`Similar issue which matches more than ${MATCH_THRESHOLD} already exists`); + //Close the issue as "unplanned" + await context.octokit.issues.update({ + owner: payload.repository.owner.login, + repo: payload.repository.name, + issue_number: issue.number, + state: "closed", + labels: ["unplanned"], + }); + return true; + } + + //Second Check if an issue with more than WARNING_THRESHOLD similarity exists (Warning) + const warningIssue = await supabase.issue.findSimilarIssues(issue.body + issue.title, WARNING_THRESHOLD); + if (warningIssue) { + logger.info(`Similar issue which matches more than ${WARNING_THRESHOLD} already exists`); + //Add a comment immediately next to the issue + //Build a list of similar issues url + const issueList = warningIssue.map(async (issue) => { + //fetch the issue url and title using globaNodeId + const issueUrl: IssueGraphqlResponse = await context.octokit.graphql( + `query($issueNodeId: String!) { + node(id: $issueNodeId) { + ... on Issue { + title + url + } + } + }`, + { + issueNodeId: issue.id, + } + ); + return issueUrl; + }); + //Add a comment to the issue + const resolvedIssueList = await Promise.all(issueList); + // Check if there is already a comment on the issue + const existingComment = await context.octokit.issues.listComments({ + owner: payload.repository.owner.login, + repo: payload.repository.name, + issue_number: issue.number, + }); + + if (existingComment.data.length > 0) { + // Find the comment that lists the similar issues + const commentToUpdate = existingComment.data.find( + (comment) => comment && comment.body && comment.body.includes("This issue seems to be similar to the following issue(s)") + ); + + if (commentToUpdate) { + // Update the comment with the latest list of similar issues + await context.octokit.issues.updateComment({ + owner: payload.repository.owner.login, + repo: payload.repository.name, + comment_id: commentToUpdate.id, + body: `This issue seems to be similar to the following issue(s) ${resolvedIssueList.map((issue) => issue.url).join(", ")}`, + }); + } else { + // Add a new comment to the issue + await createNewComment(context, resolvedIssueList); + } + } else { + // Add a new comment to the issue + await createNewComment(context, resolvedIssueList); + } + return true; + } + + logger.info("No similar issue found"); + return false; +} + +/** + * Create a new comment on the issue with the list of similar issues + * @param context + * @param resolvedIssueList + */ +async function createNewComment(context: Context, resolvedIssueList: IssueGraphqlResponse[]) { + await context.octokit.issues.createComment({ + owner: context.payload.repository.owner.login, + repo: context.payload.repository.name, + issue_number: context.payload.issue.number, + body: `This issue seems to be similar to the following issue(s) ${resolvedIssueList.map((issue) => issue.url).join(", ")}`, + }); +} diff --git a/src/handlers/update-issue.ts b/src/handlers/update-issue.ts new file mode 100644 index 0000000..7ad5f3d --- /dev/null +++ b/src/handlers/update-issue.ts @@ -0,0 +1,28 @@ +import { Context } from "../types"; + +export async function updateIssue(context: Context) { + const { + logger, + payload, + adapters: { supabase }, + } = context; + const payloadObject = payload; + const nodeId = payload.issue.node_id; + const isPrivate = payload.repository.private; + const plaintext = payload.issue.body + payload.issue.title || ""; + // Fetch the previous comment and update it in the db + try { + await supabase.issue.updateIssue(plaintext, nodeId, payloadObject, isPrivate); + } catch (error) { + if (error instanceof Error) { + logger.error(`Error updating issue:`, { error: error, stack: error.stack }); + throw error; + } else { + logger.error(`Error updating issue:`, { err: error, error: new Error() }); + throw error; + } + } + + logger.ok(`Successfully updated issue!`); + logger.verbose(`Exiting updateIssue`); +} diff --git a/src/plugin.ts b/src/plugin.ts index 9673231..c117ba9 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -10,6 +10,10 @@ import { addComments } from "./handlers/add-comments"; import { updateComment } from "./handlers/update-comments"; import { deleteComment } from "./handlers/delete-comments"; import { VoyageAIClient } from "voyageai"; +import { issueChecker } from "./handlers/issue-deduplication"; +import { deleteIssues } from "./handlers/delete-issues"; +import { addIssue } from "./handlers/add-issue"; +import { updateIssue } from "./handlers/update-issue"; /** * The main plugin function. Split for easier testing. @@ -24,6 +28,12 @@ export async function runPlugin(context: Context) { return await deleteComment(context); case "issue_comment.edited": return await updateComment(context); + case "issue.created": + return (await issueChecker(context)) === false ? await addIssue(context) : null; + case "issue.edited": + return (await issueChecker(context)) === false ? await updateIssue(context) : null; + case "issue.deleted": + return await deleteIssues(context); } } else { logger.error(`Unsupported event: ${eventName}`); diff --git a/src/types/context.ts b/src/types/context.ts index d7e3cff..b7fc84b 100644 --- a/src/types/context.ts +++ b/src/types/context.ts @@ -10,7 +10,7 @@ import { createAdapters } from "../adapters"; * * ubiquity:listeners: ["issue_comment.created", ...] */ -export type SupportedEventsU = "issue_comment.created" | "issue_comment.deleted" | "issue_comment.edited"; +export type SupportedEventsU = "issue_comment.created" | "issue_comment.deleted" | "issue_comment.edited" | "issue.created" | "issue.edited" | "issue.deleted"; export type SupportedEvents = { [K in SupportedEventsU]: K extends WebhookEventName ? WebhookEvent : never; diff --git a/src/types/typeguards.ts b/src/types/typeguards.ts index 266c856..1f724c9 100644 --- a/src/types/typeguards.ts +++ b/src/types/typeguards.ts @@ -11,6 +11,15 @@ import { Context } from "./context"; * * @param context The context object. */ -export function isIssueCommentEvent(context: Context): context is Context<"issue_comment.created" | "issue_comment.deleted" | "issue_comment.edited"> { - return context.eventName === "issue_comment.created" || context.eventName === "issue_comment.deleted" || context.eventName === "issue_comment.edited"; +export function isIssueCommentEvent( + context: Context +): context is Context<"issue_comment.created" | "issue_comment.deleted" | "issue_comment.edited" | "issue.created" | "issue.edited" | "issue.deleted"> { + return ( + context.eventName === "issue_comment.created" || + context.eventName === "issue_comment.deleted" || + context.eventName === "issue_comment.edited" || + context.eventName === "issue.created" || + context.eventName === "issue.edited" || + context.eventName === "issue.deleted" + ); } diff --git a/supabase/migrations/20240911233824_issue_comments.sql b/supabase/migrations/20240911233824_issue_comments.sql new file mode 100644 index 0000000..c6ef4eb --- /dev/null +++ b/supabase/migrations/20240911233824_issue_comments.sql @@ -0,0 +1,8 @@ +ALTER TABLE issue_comments +RENAME COLUMN commentObject TO payloadObject; + +ALTER TABLE issue_comments +ADD COLUMN type Text NOT NULL; + +ALTER TABLE issue_comments +RENAME TO vectorDump; From 55ded10670d7ad255ff635261608eacc27f06fc2 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Wed, 11 Sep 2024 22:17:50 -0400 Subject: [PATCH 20/37] fix: removing issue.created hook --- .github/.ubiquibot-config.yml | 2 +- manifest.json | 2 +- src/handlers/issue-deduplication.ts | 2 +- src/types/context.ts | 2 +- src/types/typeguards.ts | 14 +++++--------- 5 files changed, 9 insertions(+), 13 deletions(-) diff --git a/.github/.ubiquibot-config.yml b/.github/.ubiquibot-config.yml index 9213535..8a9597c 100644 --- a/.github/.ubiquibot-config.yml +++ b/.github/.ubiquibot-config.yml @@ -3,4 +3,4 @@ plugins: id: test-app uses: - plugin: https://ubiquity-os-comment-vector-embeddings.sshivaditya.workers.dev - runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issue.created", "issue.edited", "issue.deleted" ] + runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" ] diff --git a/manifest.json b/manifest.json index 4b05cc9..0e83faf 100644 --- a/manifest.json +++ b/manifest.json @@ -1,5 +1,5 @@ { "name": "@ubiquity-os/comment-vector-embeddings", "description": "Issue comment plugin for Ubiquibot. It enables the storage, updating, and deletion of issue comment embeddings.", - "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issue.created", "issue.edited", "issue.deleted"] + "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted"] } diff --git a/src/handlers/issue-deduplication.ts b/src/handlers/issue-deduplication.ts index c9dd522..8aedee0 100644 --- a/src/handlers/issue-deduplication.ts +++ b/src/handlers/issue-deduplication.ts @@ -44,7 +44,7 @@ export async function issueChecker(context: Context): Promise { //Add a comment immediately next to the issue //Build a list of similar issues url const issueList = warningIssue.map(async (issue) => { - //fetch the issue url and title using globaNodeId + //fetch the issue url and title using globalNodeId const issueUrl: IssueGraphqlResponse = await context.octokit.graphql( `query($issueNodeId: String!) { node(id: $issueNodeId) { diff --git a/src/types/context.ts b/src/types/context.ts index b7fc84b..d7e3cff 100644 --- a/src/types/context.ts +++ b/src/types/context.ts @@ -10,7 +10,7 @@ import { createAdapters } from "../adapters"; * * ubiquity:listeners: ["issue_comment.created", ...] */ -export type SupportedEventsU = "issue_comment.created" | "issue_comment.deleted" | "issue_comment.edited" | "issue.created" | "issue.edited" | "issue.deleted"; +export type SupportedEventsU = "issue_comment.created" | "issue_comment.deleted" | "issue_comment.edited"; export type SupportedEvents = { [K in SupportedEventsU]: K extends WebhookEventName ? WebhookEvent : never; diff --git a/src/types/typeguards.ts b/src/types/typeguards.ts index 1f724c9..861cca7 100644 --- a/src/types/typeguards.ts +++ b/src/types/typeguards.ts @@ -11,15 +11,11 @@ import { Context } from "./context"; * * @param context The context object. */ -export function isIssueCommentEvent( - context: Context -): context is Context<"issue_comment.created" | "issue_comment.deleted" | "issue_comment.edited" | "issue.created" | "issue.edited" | "issue.deleted"> { +export function isIssueCommentEvent(context: Context): context is Context<"issue_comment.created" | "issue_comment.deleted" | "issue_comment.edited"> { return ( - context.eventName === "issue_comment.created" || - context.eventName === "issue_comment.deleted" || - context.eventName === "issue_comment.edited" || - context.eventName === "issue.created" || - context.eventName === "issue.edited" || - context.eventName === "issue.deleted" + context.eventName === "issue_comment.created" || context.eventName === "issue_comment.deleted" || context.eventName === "issue_comment.edited" //|| + // context.eventName === "issue.created" || + // context.eventName === "issue.edited" || + // context.eventName === "issue.deleted" ); } From d604e4113dde441c6bfe270a02e06e12e6d78d8f Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Wed, 11 Sep 2024 22:26:13 -0400 Subject: [PATCH 21/37] fix: linting and tests --- src/plugin.ts | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/plugin.ts b/src/plugin.ts index c117ba9..9032cf8 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -10,10 +10,10 @@ import { addComments } from "./handlers/add-comments"; import { updateComment } from "./handlers/update-comments"; import { deleteComment } from "./handlers/delete-comments"; import { VoyageAIClient } from "voyageai"; -import { issueChecker } from "./handlers/issue-deduplication"; -import { deleteIssues } from "./handlers/delete-issues"; -import { addIssue } from "./handlers/add-issue"; -import { updateIssue } from "./handlers/update-issue"; +// import { issueChecker } from "./handlers/issue-deduplication"; +// import { deleteIssues } from "./handlers/delete-issues"; +// import { addIssue } from "./handlers/add-issue"; +// import { updateIssue } from "./handlers/update-issue"; /** * The main plugin function. Split for easier testing. @@ -28,12 +28,12 @@ export async function runPlugin(context: Context) { return await deleteComment(context); case "issue_comment.edited": return await updateComment(context); - case "issue.created": - return (await issueChecker(context)) === false ? await addIssue(context) : null; - case "issue.edited": - return (await issueChecker(context)) === false ? await updateIssue(context) : null; - case "issue.deleted": - return await deleteIssues(context); + // case "issue.created": + // return (await issueChecker(context)) === false ? await addIssue(context) : null; + // case "issue.edited": + // return (await issueChecker(context)) === false ? await updateIssue(context) : null; + // case "issue.deleted": + // return await deleteIssues(context); } } else { logger.error(`Unsupported event: ${eventName}`); From 2b7a65c501b1201e4cfcb5507e42a8ae20606251 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Thu, 12 Sep 2024 09:22:34 -0400 Subject: [PATCH 22/37] fix: rollback issue.created, edited, deleted --- README.md | 2 +- src/handlers/add-issue.ts | 29 ------- src/handlers/delete-issues.ts | 26 ------- src/handlers/issue-deduplication.ts | 113 ---------------------------- src/handlers/update-issue.ts | 28 ------- src/plugin.ts | 10 --- src/types/typeguards.ts | 7 +- 7 files changed, 2 insertions(+), 213 deletions(-) delete mode 100644 src/handlers/add-issue.ts delete mode 100644 src/handlers/delete-issues.ts delete mode 100644 src/handlers/issue-deduplication.ts delete mode 100644 src/handlers/update-issue.ts diff --git a/README.md b/README.md index e4ca6bf..ec224cb 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ To set up the `.dev.vars` file, you will need to provide the following variables - Add the following to your `.ubiquibot-config.yml` file with the appropriate URL: ```javascript -plugin: http://127.0.0.1:4000 - runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" , "issue.created", "issue.edited", "issue.deleted"] + runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted"] ``` ## Testing Locally diff --git a/src/handlers/add-issue.ts b/src/handlers/add-issue.ts deleted file mode 100644 index ce7c315..0000000 --- a/src/handlers/add-issue.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { Context } from "../types"; - -export async function addIssue(context: Context) { - const { - logger, - payload, - adapters: { supabase }, - } = context; - const payloadObject = payload; - const plaintext = payload.issue.body + payload.issue.title || ""; - const authorId = payload.issue.user?.id || -1; - const nodeId = payload.issue.node_id; - const isPrivate = payload.repository.private; - - try { - await supabase.issue.createIssue(nodeId, payloadObject, isPrivate, plaintext, authorId); - } catch (error) { - if (error instanceof Error) { - logger.error(`Error creating issue:`, { error: error, stack: error.stack }); - throw error; - } else { - logger.error(`Error creating issue:`, { err: error, error: new Error() }); - throw error; - } - } - - logger.ok(`Successfully created issue!`); - logger.verbose(`Exiting addIssue`); -} diff --git a/src/handlers/delete-issues.ts b/src/handlers/delete-issues.ts deleted file mode 100644 index 0e31a87..0000000 --- a/src/handlers/delete-issues.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { Context } from "../types"; - -export async function deleteIssues(context: Context) { - const { - logger, - payload, - adapters: { supabase }, - } = context; - - const nodeId = payload.issue.node_id; - - try { - await supabase.issue.deleteIssue(nodeId); - } catch (error) { - if (error instanceof Error) { - logger.error(`Error deleting issue:`, { error: error, stack: error.stack }); - throw error; - } else { - logger.error(`Error deleting issue:`, { err: error, error: new Error() }); - throw error; - } - } - - logger.ok(`Successfully deleted issue!`); - logger.verbose(`Exiting deleteIssue`); -} diff --git a/src/handlers/issue-deduplication.ts b/src/handlers/issue-deduplication.ts deleted file mode 100644 index 8aedee0..0000000 --- a/src/handlers/issue-deduplication.ts +++ /dev/null @@ -1,113 +0,0 @@ -import { Context } from "../types"; -const MATCH_THRESHOLD = 0.95; -const WARNING_THRESHOLD = 0.75; - -export interface IssueGraphqlResponse { - id: string; - title: string; - url: string; -} - -/** - * Check if an issue is similar to any existing issues in the database - * @param context - * @returns true if the issue is similar to an existing issue, false otherwise - */ -export async function issueChecker(context: Context): Promise { - const { - logger, - payload, - adapters: { supabase }, - } = context; - - const issue = payload.issue; - - //First Check if an issue with more than MATCH_THRESHOLD similarity exists (Very Similar) - const similarIssue = await supabase.issue.findSimilarIssues(issue.body + issue.title, MATCH_THRESHOLD); - if (similarIssue) { - logger.info(`Similar issue which matches more than ${MATCH_THRESHOLD} already exists`); - //Close the issue as "unplanned" - await context.octokit.issues.update({ - owner: payload.repository.owner.login, - repo: payload.repository.name, - issue_number: issue.number, - state: "closed", - labels: ["unplanned"], - }); - return true; - } - - //Second Check if an issue with more than WARNING_THRESHOLD similarity exists (Warning) - const warningIssue = await supabase.issue.findSimilarIssues(issue.body + issue.title, WARNING_THRESHOLD); - if (warningIssue) { - logger.info(`Similar issue which matches more than ${WARNING_THRESHOLD} already exists`); - //Add a comment immediately next to the issue - //Build a list of similar issues url - const issueList = warningIssue.map(async (issue) => { - //fetch the issue url and title using globalNodeId - const issueUrl: IssueGraphqlResponse = await context.octokit.graphql( - `query($issueNodeId: String!) { - node(id: $issueNodeId) { - ... on Issue { - title - url - } - } - }`, - { - issueNodeId: issue.id, - } - ); - return issueUrl; - }); - //Add a comment to the issue - const resolvedIssueList = await Promise.all(issueList); - // Check if there is already a comment on the issue - const existingComment = await context.octokit.issues.listComments({ - owner: payload.repository.owner.login, - repo: payload.repository.name, - issue_number: issue.number, - }); - - if (existingComment.data.length > 0) { - // Find the comment that lists the similar issues - const commentToUpdate = existingComment.data.find( - (comment) => comment && comment.body && comment.body.includes("This issue seems to be similar to the following issue(s)") - ); - - if (commentToUpdate) { - // Update the comment with the latest list of similar issues - await context.octokit.issues.updateComment({ - owner: payload.repository.owner.login, - repo: payload.repository.name, - comment_id: commentToUpdate.id, - body: `This issue seems to be similar to the following issue(s) ${resolvedIssueList.map((issue) => issue.url).join(", ")}`, - }); - } else { - // Add a new comment to the issue - await createNewComment(context, resolvedIssueList); - } - } else { - // Add a new comment to the issue - await createNewComment(context, resolvedIssueList); - } - return true; - } - - logger.info("No similar issue found"); - return false; -} - -/** - * Create a new comment on the issue with the list of similar issues - * @param context - * @param resolvedIssueList - */ -async function createNewComment(context: Context, resolvedIssueList: IssueGraphqlResponse[]) { - await context.octokit.issues.createComment({ - owner: context.payload.repository.owner.login, - repo: context.payload.repository.name, - issue_number: context.payload.issue.number, - body: `This issue seems to be similar to the following issue(s) ${resolvedIssueList.map((issue) => issue.url).join(", ")}`, - }); -} diff --git a/src/handlers/update-issue.ts b/src/handlers/update-issue.ts deleted file mode 100644 index 7ad5f3d..0000000 --- a/src/handlers/update-issue.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { Context } from "../types"; - -export async function updateIssue(context: Context) { - const { - logger, - payload, - adapters: { supabase }, - } = context; - const payloadObject = payload; - const nodeId = payload.issue.node_id; - const isPrivate = payload.repository.private; - const plaintext = payload.issue.body + payload.issue.title || ""; - // Fetch the previous comment and update it in the db - try { - await supabase.issue.updateIssue(plaintext, nodeId, payloadObject, isPrivate); - } catch (error) { - if (error instanceof Error) { - logger.error(`Error updating issue:`, { error: error, stack: error.stack }); - throw error; - } else { - logger.error(`Error updating issue:`, { err: error, error: new Error() }); - throw error; - } - } - - logger.ok(`Successfully updated issue!`); - logger.verbose(`Exiting updateIssue`); -} diff --git a/src/plugin.ts b/src/plugin.ts index 9032cf8..9673231 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -10,10 +10,6 @@ import { addComments } from "./handlers/add-comments"; import { updateComment } from "./handlers/update-comments"; import { deleteComment } from "./handlers/delete-comments"; import { VoyageAIClient } from "voyageai"; -// import { issueChecker } from "./handlers/issue-deduplication"; -// import { deleteIssues } from "./handlers/delete-issues"; -// import { addIssue } from "./handlers/add-issue"; -// import { updateIssue } from "./handlers/update-issue"; /** * The main plugin function. Split for easier testing. @@ -28,12 +24,6 @@ export async function runPlugin(context: Context) { return await deleteComment(context); case "issue_comment.edited": return await updateComment(context); - // case "issue.created": - // return (await issueChecker(context)) === false ? await addIssue(context) : null; - // case "issue.edited": - // return (await issueChecker(context)) === false ? await updateIssue(context) : null; - // case "issue.deleted": - // return await deleteIssues(context); } } else { logger.error(`Unsupported event: ${eventName}`); diff --git a/src/types/typeguards.ts b/src/types/typeguards.ts index 861cca7..266c856 100644 --- a/src/types/typeguards.ts +++ b/src/types/typeguards.ts @@ -12,10 +12,5 @@ import { Context } from "./context"; * @param context The context object. */ export function isIssueCommentEvent(context: Context): context is Context<"issue_comment.created" | "issue_comment.deleted" | "issue_comment.edited"> { - return ( - context.eventName === "issue_comment.created" || context.eventName === "issue_comment.deleted" || context.eventName === "issue_comment.edited" //|| - // context.eventName === "issue.created" || - // context.eventName === "issue.edited" || - // context.eventName === "issue.deleted" - ); + return context.eventName === "issue_comment.created" || context.eventName === "issue_comment.deleted" || context.eventName === "issue_comment.edited"; } From 7abb239f2854a8a0241ebd05331a3daa8aa73695 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Thu, 12 Sep 2024 19:37:03 -0400 Subject: [PATCH 23/37] feat: updated schema fix tests --- src/adapters/supabase/helpers/comment.ts | 29 ++++++---- src/adapters/supabase/helpers/issues.ts | 4 +- src/adapters/utils/comment-object-clean.ts | 53 ------------------- src/handlers/add-comments.ts | 5 +- src/handlers/update-comments.ts | 4 +- .../20240912225853_issue_comments.sql | 20 +++++++ tests/__mocks__/adapter.ts | 17 ++++-- tests/main.test.ts | 6 +-- 8 files changed, 59 insertions(+), 79 deletions(-) delete mode 100644 src/adapters/utils/comment-object-clean.ts create mode 100644 supabase/migrations/20240912225853_issue_comments.sql diff --git a/src/adapters/supabase/helpers/comment.ts b/src/adapters/supabase/helpers/comment.ts index 22f5e46..0f59531 100644 --- a/src/adapters/supabase/helpers/comment.ts +++ b/src/adapters/supabase/helpers/comment.ts @@ -16,9 +16,16 @@ export class Comment extends SuperSupabase { super(supabase, context); } - async createComment(plaintext: string | null, commentNodeId: string, authorId: number, payloadObject: Record | null, isPrivate: boolean) { + async createComment( + plaintext: string | null, + commentNodeId: string, + authorId: number, + payload: Record | null, + isPrivate: boolean, + issueId: string + ) { //First Check if the comment already exists - const { data, error } = await this.supabase.from("vectordump").select("*").eq("id", commentNodeId); + const { data, error } = await this.supabase.from("issue_comments").select("*").eq("id", commentNodeId); if (error) { this.context.logger.error("Error creating comment", error); return; @@ -31,11 +38,11 @@ export class Comment extends SuperSupabase { const embedding = await this.context.adapters.voyage.embedding.createEmbedding(plaintext); if (isPrivate) { plaintext = null as string | null; - payloadObject = null as Record | null; + payload = null as Record | null; } const { error } = await this.supabase - .from("vectordump") - .insert([{ id: commentNodeId, plaintext, author_id: authorId, type: "comment", payloadobject: payloadObject, embedding: embedding }]); + .from("issue_comments") + .insert([{ id: commentNodeId, plaintext, author_id: authorId, type: "comment", payload, embedding: embedding, issueId }]); if (error) { this.context.logger.error("Error creating comment", error); return; @@ -44,16 +51,16 @@ export class Comment extends SuperSupabase { this.context.logger.info("Comment created successfully"); } - async updateComment(plaintext: string | null, commentNodeId: string, payloadObject: Record | null, isPrivate: boolean) { + async updateComment(plaintext: string | null, commentNodeId: string, payload: Record | null, isPrivate: boolean) { //Create the embedding for this comment const embedding = Array.from(await this.context.adapters.voyage.embedding.createEmbedding(plaintext)); if (isPrivate) { plaintext = null as string | null; - payloadObject = null as Record | null; + payload = null as Record | null; } const { error } = await this.supabase - .from("vectordump") - .update({ plaintext, embedding: embedding, payloadobject: payloadObject, modified_at: new Date() }) + .from("issue_comments") + .update({ plaintext, embedding: embedding, payload, modified_at: new Date() }) .eq("id", commentNodeId); if (error) { this.context.logger.error("Error updating comment", error); @@ -61,7 +68,7 @@ export class Comment extends SuperSupabase { } async getComment(commentNodeId: string): Promise { - const { data, error } = await this.supabase.from("vectordump").select("*").eq("id", commentNodeId); + const { data, error } = await this.supabase.from("issue_comments").select("*").eq("id", commentNodeId); if (error) { this.context.logger.error("Error getting comment", error); } @@ -69,7 +76,7 @@ export class Comment extends SuperSupabase { } async deleteComment(commentNodeId: string) { - const { error } = await this.supabase.from("vectordump").delete().eq("id", commentNodeId); + const { error } = await this.supabase.from("issue_comments").delete().eq("id", commentNodeId); if (error) { this.context.logger.error("Error deleting comment", error); } diff --git a/src/adapters/supabase/helpers/issues.ts b/src/adapters/supabase/helpers/issues.ts index 6048ccc..e7d7e95 100644 --- a/src/adapters/supabase/helpers/issues.ts +++ b/src/adapters/supabase/helpers/issues.ts @@ -52,7 +52,7 @@ export class Issues extends SuperSupabase { payloadObject = null as Record | null; } const { error } = await this.supabase - .from("vectordump") + .from("issue_comments") .update({ plaintext, embedding: embedding, payloadObject, modified_at: new Date() }) .eq("id", issueNodeId); if (error) { @@ -61,7 +61,7 @@ export class Issues extends SuperSupabase { } async deleteIssue(issueNodeId: string) { - const { error } = await this.supabase.from("vectordump").delete().eq("id", issueNodeId); + const { error } = await this.supabase.from("issue_comments").delete().eq("id", issueNodeId); if (error) { this.context.logger.error("Error deleting comment", error); } diff --git a/src/adapters/utils/comment-object-clean.ts b/src/adapters/utils/comment-object-clean.ts deleted file mode 100644 index 7dc0d7e..0000000 --- a/src/adapters/utils/comment-object-clean.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { SupportedEvents, SupportedEventsU } from "../../types/context"; - -/** - * Extracts only the properties mentioned in the TypeBox schema from the input object. - * - * @param commentObject - The comment object. - * @returns The object containing only the properties defined in the schema. - */ -export function cleanCommentObject( - commentObject: TU["payload"] -): Record { - // Manually extract properties - return { - action: commentObject.action as string, - issue: { - id: commentObject.issue.id as number, - number: commentObject.issue.number as number, - title: commentObject.issue.title as string, - body: commentObject.issue.body as string, - user: { - login: commentObject.issue.user.login as string, - id: commentObject.issue.user.id as number, - }, - author_association: commentObject.issue.author_association as string, - }, - comment: { - author_association: commentObject.comment.author_association as string, - id: commentObject.comment.id as number, - html_url: commentObject.comment.html_url as string, - issue_url: commentObject.comment.issue_url as string, - user: { - login: (commentObject.comment.user || { login: "" }).login as string, - id: (commentObject.comment.user || { id: -1 }).id as number, - }, - body: commentObject.comment.body as string, - created_at: commentObject.comment.created_at as string, - updated_at: commentObject.comment.updated_at as string, - }, - - repository: { - id: commentObject.repository.id as number, - node_id: commentObject.repository.node_id as string, - name: commentObject.repository.name as string, - full_name: commentObject.repository.full_name as string, - private: commentObject.repository.private as boolean, - owner: { - login: commentObject.repository.owner.login as string, - id: commentObject.repository.owner.id as number, - avatar_url: commentObject.repository.owner.avatar_url as string, - }, - }, - } as Record; -} diff --git a/src/handlers/add-comments.ts b/src/handlers/add-comments.ts index 62486d4..bb0215c 100644 --- a/src/handlers/add-comments.ts +++ b/src/handlers/add-comments.ts @@ -1,4 +1,3 @@ -import { cleanCommentObject } from "../adapters/utils/comment-object-clean"; import { Context } from "../types"; export async function addComments(context: Context) { @@ -7,14 +6,14 @@ export async function addComments(context: Context) { payload, adapters: { supabase }, } = context; - const commentObject = cleanCommentObject(payload); const plaintext = payload.comment.body; const authorId = payload.comment.user?.id || -1; const nodeId = payload.comment.node_id; const isPrivate = payload.repository.private; + const issueId = payload.issue.node_id; try { - await supabase.comment.createComment(plaintext, nodeId, authorId, commentObject, isPrivate); + await supabase.comment.createComment(plaintext, nodeId, authorId, payload, isPrivate, issueId); } catch (error) { if (error instanceof Error) { logger.error(`Error creating comment:`, { error: error, stack: error.stack }); diff --git a/src/handlers/update-comments.ts b/src/handlers/update-comments.ts index 5ae6b46..f354d91 100644 --- a/src/handlers/update-comments.ts +++ b/src/handlers/update-comments.ts @@ -1,4 +1,3 @@ -import { cleanCommentObject } from "../adapters/utils/comment-object-clean"; import { Context } from "../types"; export async function updateComment(context: Context) { @@ -7,13 +6,12 @@ export async function updateComment(context: Context) { payload, adapters: { supabase }, } = context; - const commentObject = cleanCommentObject(payload); const nodeId = payload.comment.node_id; const isPrivate = payload.repository.private; const plaintext = payload.comment.body; // Fetch the previous comment and update it in the db try { - await supabase.comment.updateComment(plaintext, nodeId, commentObject, isPrivate); + await supabase.comment.updateComment(plaintext, nodeId, payload, isPrivate); } catch (error) { if (error instanceof Error) { logger.error(`Error updating comment:`, { error: error, stack: error.stack }); diff --git a/supabase/migrations/20240912225853_issue_comments.sql b/supabase/migrations/20240912225853_issue_comments.sql new file mode 100644 index 0000000..3a8764a --- /dev/null +++ b/supabase/migrations/20240912225853_issue_comments.sql @@ -0,0 +1,20 @@ +ALTER TABLE vectordump +RENAME TO issue_comments; + +CREATE TABLE IF NOT EXISTS issues ( + id VARCHAR primary key, + plaintext text, + embedding Vector(1024) not null, + payload jsonb, + type text not null default 'issue', + created_at timestamptz not null default now(), + modified_at timestamptz not null default now() +); + +ALTER TABLE issue_comments +ADD COLUMN issue_id VARCHAR +REFERENCES issues(id) +ON DELETE CASCADE; + +ALTER TABLE issue_comments +RENAME COLUMN payloadobject TO payload; \ No newline at end of file diff --git a/tests/__mocks__/adapter.ts b/tests/__mocks__/adapter.ts index ee9956f..d1f634c 100644 --- a/tests/__mocks__/adapter.ts +++ b/tests/__mocks__/adapter.ts @@ -6,7 +6,9 @@ export interface CommentMock { id: string; plaintext: string | null; author_id: number; - commentObject?: Record | null; + payload?: Record | null; + type?: string; + issue_id?: string; embedding: number[]; } @@ -16,7 +18,14 @@ export function createMockAdapters(context: Context) { supabase: { comment: { createComment: jest.fn( - async (plaintext: string | null, commentNodeId: string, authorId: number, commentObject: Record | null, isPrivate: boolean) => { + async ( + plaintext: string | null, + commentNodeId: string, + authorId: number, + payload: Record | null, + isPrivate: boolean, + issueId: string + ) => { if (commentMap.has(commentNodeId)) { throw new Error("Comment already exists"); } @@ -24,10 +33,10 @@ export function createMockAdapters(context: Context) { if (isPrivate) { plaintext = null; } - commentMap.set(commentNodeId, { id: commentNodeId, plaintext, author_id: authorId, embedding }); + commentMap.set(commentNodeId, { id: commentNodeId, plaintext, author_id: authorId, embedding, issue_id: issueId }); } ), - updateComment: jest.fn(async (plaintext: string | null, commentNodeId: string, commentObject: Record | null, isPrivate: boolean) => { + updateComment: jest.fn(async (plaintext: string | null, commentNodeId: string, payload: Record | null, isPrivate: boolean) => { if (!commentMap.has(commentNodeId)) { throw new Error(STRINGS.COMMENT_DOES_NOT_EXIST); } diff --git a/tests/main.test.ts b/tests/main.test.ts index 5f4fd84..adf1481 100644 --- a/tests/main.test.ts +++ b/tests/main.test.ts @@ -52,7 +52,7 @@ describe("Plugin tests", () => { const supabase = context.adapters.supabase; const commentObject = null; try { - await supabase.comment.createComment(STRINGS.HELLO_WORLD, "sasasCreate", 1, commentObject, false); + await supabase.comment.createComment(STRINGS.HELLO_WORLD, "sasasCreate", 1, commentObject, false, "sasasCreateIssue"); throw new Error("Expected method to reject."); } catch (error) { if (error instanceof Error) { @@ -69,7 +69,7 @@ describe("Plugin tests", () => { const { context } = createContext("Updated Message", 1, 1, 1, 1, "sasasUpdate", "issue_comment.edited"); const supabase = context.adapters.supabase; const commentObject = null; - await supabase.comment.createComment(STRINGS.HELLO_WORLD, "sasasUpdate", 1, commentObject, false); + await supabase.comment.createComment(STRINGS.HELLO_WORLD, "sasasUpdate", 1, commentObject, false, "sasasUpdateIssue"); await runPlugin(context); const comment = (await supabase.comment.getComment("sasasUpdate")) as unknown as CommentMock; expect(comment).toBeDefined(); @@ -81,7 +81,7 @@ describe("Plugin tests", () => { const { context } = createContext("Text Message", 1, 1, 1, 1, "sasasDelete", "issue_comment.deleted"); const supabase = context.adapters.supabase; const commentObject = null; - await supabase.comment.createComment(STRINGS.HELLO_WORLD, "sasasDelete", 1, commentObject, false); + await supabase.comment.createComment(STRINGS.HELLO_WORLD, "sasasDelete", 1, commentObject, false, "sasasDeleteIssue"); await runPlugin(context); try { await supabase.comment.getComment("sasasDelete"); From 8b2a945505487e1bff1d2dc8912e4e4fba2293ef Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Thu, 12 Sep 2024 21:24:10 -0400 Subject: [PATCH 24/37] fix: types of payload --- .cspell.json | 2 ++ .github/.ubiquibot-config.yml | 5 ++--- README.md | 3 ++- manifest.json | 2 +- src/handlers/add-comments.ts | 3 ++- src/handlers/add-issue.ts | 29 +++++++++++++++++++++++++++++ src/handlers/delete-comments.ts | 4 ++-- src/handlers/delete-issue.ts | 26 ++++++++++++++++++++++++++ src/handlers/update-comments.ts | 3 ++- src/handlers/update-issue.ts | 29 +++++++++++++++++++++++++++++ src/plugin.ts | 18 ++++++++++++++++-- src/types/context.ts | 8 +++++++- src/types/payload.ts | 3 +++ src/types/typeguards.ts | 9 +++++++++ tests/main.test.ts | 8 +++++--- 15 files changed, 137 insertions(+), 15 deletions(-) create mode 100644 src/handlers/add-issue.ts create mode 100644 src/handlers/delete-issue.ts create mode 100644 src/handlers/update-issue.ts create mode 100644 src/types/payload.ts diff --git a/.cspell.json b/.cspell.json index e64757f..ecd7f51 100644 --- a/.cspell.json +++ b/.cspell.json @@ -24,6 +24,8 @@ "commentbody", "issuebody", "voyageai", + "vectordump", + "payloadobject" ], "dictionaries": ["typescript", "node", "software-terms"], "import": ["@cspell/dict-typescript/cspell-ext.json", "@cspell/dict-node/cspell-ext.json", "@cspell/dict-software-terms"], diff --git a/.github/.ubiquibot-config.yml b/.github/.ubiquibot-config.yml index 8a9597c..bce9467 100644 --- a/.github/.ubiquibot-config.yml +++ b/.github/.ubiquibot-config.yml @@ -1,6 +1,5 @@ -plugins: - - name: test-app - id: test-app + id: test-app uses: - plugin: https://ubiquity-os-comment-vector-embeddings.sshivaditya.workers.dev runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" ] + runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issues.created", "issues.edited", "issues.deleted" ] \ No newline at end of file diff --git a/README.md b/README.md index ec224cb..0bb10f4 100644 --- a/README.md +++ b/README.md @@ -13,9 +13,10 @@ To set up the `.dev.vars` file, you will need to provide the following variables - Add the following to your `.ubiquibot-config.yml` file with the appropriate URL: ```javascript -plugin: http://127.0.0.1:4000 - runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted"] + runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" , "issue.created", "issue.edited", "issue.deleted"] ``` + ## Testing Locally - Run `yarn install` to install the dependencies. - Run `yarn worker` to start the server. diff --git a/manifest.json b/manifest.json index 0e83faf..4b05cc9 100644 --- a/manifest.json +++ b/manifest.json @@ -1,5 +1,5 @@ { "name": "@ubiquity-os/comment-vector-embeddings", "description": "Issue comment plugin for Ubiquibot. It enables the storage, updating, and deletion of issue comment embeddings.", - "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted"] + "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issue.created", "issue.edited", "issue.deleted"] } diff --git a/src/handlers/add-comments.ts b/src/handlers/add-comments.ts index bb0215c..4d3ad4b 100644 --- a/src/handlers/add-comments.ts +++ b/src/handlers/add-comments.ts @@ -1,11 +1,12 @@ import { Context } from "../types"; +import { CommentPayload } from "../types/payload"; export async function addComments(context: Context) { const { logger, - payload, adapters: { supabase }, } = context; + const { payload } = context as { payload: CommentPayload }; const plaintext = payload.comment.body; const authorId = payload.comment.user?.id || -1; const nodeId = payload.comment.node_id; diff --git a/src/handlers/add-issue.ts b/src/handlers/add-issue.ts new file mode 100644 index 0000000..ce7c315 --- /dev/null +++ b/src/handlers/add-issue.ts @@ -0,0 +1,29 @@ +import { Context } from "../types"; + +export async function addIssue(context: Context) { + const { + logger, + payload, + adapters: { supabase }, + } = context; + const payloadObject = payload; + const plaintext = payload.issue.body + payload.issue.title || ""; + const authorId = payload.issue.user?.id || -1; + const nodeId = payload.issue.node_id; + const isPrivate = payload.repository.private; + + try { + await supabase.issue.createIssue(nodeId, payloadObject, isPrivate, plaintext, authorId); + } catch (error) { + if (error instanceof Error) { + logger.error(`Error creating issue:`, { error: error, stack: error.stack }); + throw error; + } else { + logger.error(`Error creating issue:`, { err: error, error: new Error() }); + throw error; + } + } + + logger.ok(`Successfully created issue!`); + logger.verbose(`Exiting addIssue`); +} diff --git a/src/handlers/delete-comments.ts b/src/handlers/delete-comments.ts index 7b444ea..44fb8f2 100644 --- a/src/handlers/delete-comments.ts +++ b/src/handlers/delete-comments.ts @@ -1,12 +1,12 @@ import { Context } from "../types"; +import { CommentPayload } from "../types/payload"; export async function deleteComment(context: Context) { const { logger, - payload, adapters: { supabase }, } = context; - + const { payload } = context as { payload: CommentPayload }; const nodeId = payload.comment.node_id; try { diff --git a/src/handlers/delete-issue.ts b/src/handlers/delete-issue.ts new file mode 100644 index 0000000..ba16032 --- /dev/null +++ b/src/handlers/delete-issue.ts @@ -0,0 +1,26 @@ +import { Context } from "../types"; +import { IssuePayload } from "../types/payload"; + +export async function deleteIssues(context: Context) { + const { + logger, + adapters: { supabase }, + } = context; + const { payload } = context as { payload: IssuePayload }; + const nodeId = payload.issue.node_id; + + try { + await supabase.issue.deleteIssue(nodeId); + } catch (error) { + if (error instanceof Error) { + logger.error(`Error deleting issue:`, { error: error, stack: error.stack }); + throw error; + } else { + logger.error(`Error deleting issue:`, { err: error, error: new Error() }); + throw error; + } + } + + logger.ok(`Successfully deleted issue!`); + logger.verbose(`Exiting deleteIssue`); +} diff --git a/src/handlers/update-comments.ts b/src/handlers/update-comments.ts index f354d91..6dd2dae 100644 --- a/src/handlers/update-comments.ts +++ b/src/handlers/update-comments.ts @@ -1,11 +1,12 @@ import { Context } from "../types"; +import { CommentPayload } from "../types/payload"; export async function updateComment(context: Context) { const { logger, - payload, adapters: { supabase }, } = context; + const { payload } = context as { payload: CommentPayload }; const nodeId = payload.comment.node_id; const isPrivate = payload.repository.private; const plaintext = payload.comment.body; diff --git a/src/handlers/update-issue.ts b/src/handlers/update-issue.ts new file mode 100644 index 0000000..fad405c --- /dev/null +++ b/src/handlers/update-issue.ts @@ -0,0 +1,29 @@ +import { Context } from "../types"; +import { IssuePayload } from "../types/payload"; + +export async function updateIssue(context: Context) { + const { + logger, + adapters: { supabase }, + } = context; + const { payload } = context as { payload: IssuePayload }; + const payloadObject = payload; + const nodeId = payload.issue.node_id; + const isPrivate = payload.repository.private; + const plaintext = payload.issue.body + payload.issue.title || ""; + // Fetch the previous issue and update it in the db + try { + await supabase.issue.updateIssue(plaintext, nodeId, payloadObject, isPrivate); + } catch (error) { + if (error instanceof Error) { + logger.error(`Error updating issue:`, { error: error, stack: error.stack }); + throw error; + } else { + logger.error(`Error updating issue:`, { err: error, error: new Error() }); + throw error; + } + } + + logger.ok(`Successfully updated issue!`); + logger.verbose(`Exiting updateIssue`); +} diff --git a/src/plugin.ts b/src/plugin.ts index 9673231..aa9e871 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -1,7 +1,7 @@ import { Octokit } from "@octokit/rest"; import { Env, PluginInputs } from "./types"; import { Context } from "./types"; -import { isIssueCommentEvent } from "./types/typeguards"; +import { isIssueCommentEvent, isIssueEvent } from "./types/typeguards"; import { LogLevel, Logs } from "@ubiquity-dao/ubiquibot-logger"; import { Database } from "./types/database"; import { createAdapters } from "./adapters"; @@ -10,6 +10,9 @@ import { addComments } from "./handlers/add-comments"; import { updateComment } from "./handlers/update-comments"; import { deleteComment } from "./handlers/delete-comments"; import { VoyageAIClient } from "voyageai"; +import { deleteIssues } from "./handlers/delete-issue"; +import { addIssue } from "./handlers/add-issue"; +import { updateIssue } from "./handlers/update-issue"; /** * The main plugin function. Split for easier testing. @@ -25,6 +28,15 @@ export async function runPlugin(context: Context) { case "issue_comment.edited": return await updateComment(context); } + } else if (isIssueEvent(context)) { + switch (eventName) { + case "issues.created": + return await addIssue(context); + case "issues.deleted": + return await deleteIssues(context); + case "issues.edited": + return await updateIssue(context); + } } else { logger.error(`Unsupported event: ${eventName}`); } @@ -50,5 +62,7 @@ export async function plugin(inputs: PluginInputs, env: Env) { adapters: {} as ReturnType, }; context.adapters = createAdapters(supabase, voyageClient, context); - return runPlugin(context); + if (isIssueEvent(context)) { + return await runPlugin(context); + } } diff --git a/src/types/context.ts b/src/types/context.ts index d7e3cff..06de614 100644 --- a/src/types/context.ts +++ b/src/types/context.ts @@ -10,7 +10,13 @@ import { createAdapters } from "../adapters"; * * ubiquity:listeners: ["issue_comment.created", ...] */ -export type SupportedEventsU = "issue_comment.created" | "issue_comment.deleted" | "issue_comment.edited"; +export type SupportedEventsU = + | "issue_comment.created" + | "issue_comment.deleted" + | "issue_comment.edited" + | "issues.created" + | "issues.edited" + | "issues.deleted"; export type SupportedEvents = { [K in SupportedEventsU]: K extends WebhookEventName ? WebhookEvent : never; diff --git a/src/types/payload.ts b/src/types/payload.ts new file mode 100644 index 0000000..395fa09 --- /dev/null +++ b/src/types/payload.ts @@ -0,0 +1,3 @@ +import { EmitterWebhookEvent as WebhookEvent } from "@octokit/webhooks"; +export type CommentPayload = WebhookEvent<"issue_comment">["payload"]; +export type IssuePayload = WebhookEvent<"issues">["payload"]; diff --git a/src/types/typeguards.ts b/src/types/typeguards.ts index 266c856..8a7499c 100644 --- a/src/types/typeguards.ts +++ b/src/types/typeguards.ts @@ -14,3 +14,12 @@ import { Context } from "./context"; export function isIssueCommentEvent(context: Context): context is Context<"issue_comment.created" | "issue_comment.deleted" | "issue_comment.edited"> { return context.eventName === "issue_comment.created" || context.eventName === "issue_comment.deleted" || context.eventName === "issue_comment.edited"; } + +/** + * Restricts the scope of `context` to the `issues.created`, `issues.edited`, and `issues.deleted` payloads. + * + * @param context The context object. + */ +export function isIssueEvent(context: Context): context is Context<"issues.created" | "issues.edited" | "issues.deleted"> { + return context.eventName === "issues.created" || context.eventName === "issues.edited" || context.eventName === "issues.deleted"; +} diff --git a/tests/main.test.ts b/tests/main.test.ts index adf1481..40eeb7f 100644 --- a/tests/main.test.ts +++ b/tests/main.test.ts @@ -4,7 +4,7 @@ import { drop } from "@mswjs/data"; import { db } from "./__mocks__/db"; import { server } from "./__mocks__/node"; import { expect, describe, beforeAll, beforeEach, afterAll, afterEach, it } from "@jest/globals"; -import { Context } from "../src/types/context"; +import { Context, SupportedEvents } from "../src/types/context"; import { Octokit } from "@octokit/rest"; import { STRINGS } from "./__mocks__/strings"; import { createComment, setupTests } from "./__mocks__/helpers"; @@ -116,7 +116,9 @@ function createContext( const issue1 = db.issue.findFirst({ where: { id: { equals: issueOne } } }) as unknown as Context["payload"]["issue"]; createComment(commentBody, commentId, nodeId); // create it first then pull it from the DB and feed it to _createContext - const comment = db.issueComments.findFirst({ where: { id: { equals: commentId } } }) as unknown as Context["payload"]["comment"]; + const comment = db.issueComments.findFirst({ + where: { id: { equals: commentId } }, + }) as unknown as unknown as SupportedEvents["issue_comment.created"]["payload"]["comment"]; const context = createContextInner(repo, sender, issue1, comment, eventName); context.adapters = createMockAdapters(context) as unknown as Context["adapters"]; @@ -147,7 +149,7 @@ function createContextInner( repo: Context["payload"]["repository"], sender: Context["payload"]["sender"], issue: Context["payload"]["issue"], - comment: Context["payload"]["comment"], + comment: SupportedEvents["issue_comment.created"]["payload"]["comment"], eventName: Context["eventName"] = "issue_comment.created" ): Context { return { From d745150c141ee4be0d9271db6fc53bb8ce5e07e3 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Thu, 12 Sep 2024 21:29:40 -0400 Subject: [PATCH 25/37] fix: config --- .github/.ubiquibot-config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/.ubiquibot-config.yml b/.github/.ubiquibot-config.yml index bce9467..bc81e1a 100644 --- a/.github/.ubiquibot-config.yml +++ b/.github/.ubiquibot-config.yml @@ -1,5 +1,4 @@ id: test-app uses: - plugin: https://ubiquity-os-comment-vector-embeddings.sshivaditya.workers.dev - runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" ] runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issues.created", "issues.edited", "issues.deleted" ] \ No newline at end of file From 68c481a229f8bb7933a55d2565783f8b294f4731 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Thu, 12 Sep 2024 21:32:44 -0400 Subject: [PATCH 26/37] fix: update config --- .github/.ubiquibot-config.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/.ubiquibot-config.yml b/.github/.ubiquibot-config.yml index bc81e1a..1d024d0 100644 --- a/.github/.ubiquibot-config.yml +++ b/.github/.ubiquibot-config.yml @@ -1,4 +1,10 @@ id: test-app uses: - plugin: https://ubiquity-os-comment-vector-embeddings.sshivaditya.workers.dev - runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issues.created", "issues.edited", "issues.deleted" ] \ No newline at end of file + runsOn: + - issue_comment.created + - issue_comment.edited + - issue_comment.deleted + - issues.created + - issues.edited + - issues.deleted \ No newline at end of file From 8be23794f68abcad16906901e7360ee14b51598a Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Thu, 12 Sep 2024 23:25:38 -0400 Subject: [PATCH 27/37] fix: updated manifest.json --- .github/.ubiquibot-config.yml | 8 +------- manifest.json | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/.ubiquibot-config.yml b/.github/.ubiquibot-config.yml index 1d024d0..b15d7dd 100644 --- a/.github/.ubiquibot-config.yml +++ b/.github/.ubiquibot-config.yml @@ -1,10 +1,4 @@ id: test-app uses: - plugin: https://ubiquity-os-comment-vector-embeddings.sshivaditya.workers.dev - runsOn: - - issue_comment.created - - issue_comment.edited - - issue_comment.deleted - - issues.created - - issues.edited - - issues.deleted \ No newline at end of file + runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted"] \ No newline at end of file diff --git a/manifest.json b/manifest.json index 4b05cc9..0e83faf 100644 --- a/manifest.json +++ b/manifest.json @@ -1,5 +1,5 @@ { "name": "@ubiquity-os/comment-vector-embeddings", "description": "Issue comment plugin for Ubiquibot. It enables the storage, updating, and deletion of issue comment embeddings.", - "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issue.created", "issue.edited", "issue.deleted"] + "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted"] } From 9d9197ad4cd0ea880f3669fc22c4c92dc9973377 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Thu, 12 Sep 2024 23:32:41 -0400 Subject: [PATCH 28/37] fix: updated manifest.json --- manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifest.json b/manifest.json index 0e83faf..4b05cc9 100644 --- a/manifest.json +++ b/manifest.json @@ -1,5 +1,5 @@ { "name": "@ubiquity-os/comment-vector-embeddings", "description": "Issue comment plugin for Ubiquibot. It enables the storage, updating, and deletion of issue comment embeddings.", - "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted"] + "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issue.created", "issue.edited", "issue.deleted"] } From 22679ec2177f78119f245de91a13f67a41bd41e9 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Thu, 12 Sep 2024 23:36:34 -0400 Subject: [PATCH 29/37] fix: updated manifest.json and readme.md --- README.md | 2 +- manifest.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0bb10f4..db55b8a 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ To set up the `.dev.vars` file, you will need to provide the following variables - Add the following to your `.ubiquibot-config.yml` file with the appropriate URL: ```javascript -plugin: http://127.0.0.1:4000 - runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" , "issue.created", "issue.edited", "issue.deleted"] + runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" , "issues.created", "issues.edited", "issues.deleted"] ``` diff --git a/manifest.json b/manifest.json index 4b05cc9..8dc6dbd 100644 --- a/manifest.json +++ b/manifest.json @@ -1,5 +1,5 @@ { "name": "@ubiquity-os/comment-vector-embeddings", "description": "Issue comment plugin for Ubiquibot. It enables the storage, updating, and deletion of issue comment embeddings.", - "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issue.created", "issue.edited", "issue.deleted"] + "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issues.created", "issues.edited", "issues.deleted"] } From 2ea6146544acb2df097d9e61b18d4dfbea03cb65 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Thu, 12 Sep 2024 23:40:25 -0400 Subject: [PATCH 30/37] fix: issue.created to issues.opened --- README.md | 2 +- manifest.json | 2 +- src/plugin.ts | 2 +- src/types/context.ts | 2 +- src/types/typeguards.ts | 6 +++--- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index db55b8a..cac1f27 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ To set up the `.dev.vars` file, you will need to provide the following variables - Add the following to your `.ubiquibot-config.yml` file with the appropriate URL: ```javascript -plugin: http://127.0.0.1:4000 - runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" , "issues.created", "issues.edited", "issues.deleted"] + runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" , "issues.opened", "issues.edited", "issues.deleted"] ``` diff --git a/manifest.json b/manifest.json index 8dc6dbd..328a300 100644 --- a/manifest.json +++ b/manifest.json @@ -1,5 +1,5 @@ { "name": "@ubiquity-os/comment-vector-embeddings", "description": "Issue comment plugin for Ubiquibot. It enables the storage, updating, and deletion of issue comment embeddings.", - "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issues.created", "issues.edited", "issues.deleted"] + "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issues.opened", "issues.edited", "issues.deleted"] } diff --git a/src/plugin.ts b/src/plugin.ts index aa9e871..fa67919 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -30,7 +30,7 @@ export async function runPlugin(context: Context) { } } else if (isIssueEvent(context)) { switch (eventName) { - case "issues.created": + case "issues.opened": return await addIssue(context); case "issues.deleted": return await deleteIssues(context); diff --git a/src/types/context.ts b/src/types/context.ts index 06de614..1227abf 100644 --- a/src/types/context.ts +++ b/src/types/context.ts @@ -14,7 +14,7 @@ export type SupportedEventsU = | "issue_comment.created" | "issue_comment.deleted" | "issue_comment.edited" - | "issues.created" + | "issues.opened" | "issues.edited" | "issues.deleted"; diff --git a/src/types/typeguards.ts b/src/types/typeguards.ts index 8a7499c..01a6c26 100644 --- a/src/types/typeguards.ts +++ b/src/types/typeguards.ts @@ -16,10 +16,10 @@ export function isIssueCommentEvent(context: Context): context is Context<"issue } /** - * Restricts the scope of `context` to the `issues.created`, `issues.edited`, and `issues.deleted` payloads. + * Restricts the scope of `context` to the `issues.opened`, `issues.edited`, and `issues.deleted` payloads. * * @param context The context object. */ -export function isIssueEvent(context: Context): context is Context<"issues.created" | "issues.edited" | "issues.deleted"> { - return context.eventName === "issues.created" || context.eventName === "issues.edited" || context.eventName === "issues.deleted"; +export function isIssueEvent(context: Context): context is Context<"issues.opened" | "issues.edited" | "issues.deleted"> { + return context.eventName === "issues.opened" || context.eventName === "issues.edited" || context.eventName === "issues.deleted"; } From a0da267feb7690317a5ced93c5d2ef92708d3dfc Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Fri, 13 Sep 2024 00:38:30 -0400 Subject: [PATCH 31/37] fix: issue config removed updated schema --- .github/.ubiquibot-config.yml | 4 ---- src/adapters/supabase/helpers/comment.ts | 3 ++- src/adapters/supabase/helpers/issues.ts | 22 +++++++++---------- src/handlers/add-issue.ts | 8 +++---- src/handlers/update-issue.ts | 2 +- src/plugin.ts | 4 +--- .../20240912225853_issue_comments.sql | 1 + 7 files changed, 19 insertions(+), 25 deletions(-) delete mode 100644 .github/.ubiquibot-config.yml diff --git a/.github/.ubiquibot-config.yml b/.github/.ubiquibot-config.yml deleted file mode 100644 index b15d7dd..0000000 --- a/.github/.ubiquibot-config.yml +++ /dev/null @@ -1,4 +0,0 @@ - id: test-app - uses: - - plugin: https://ubiquity-os-comment-vector-embeddings.sshivaditya.workers.dev - runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted"] \ No newline at end of file diff --git a/src/adapters/supabase/helpers/comment.ts b/src/adapters/supabase/helpers/comment.ts index 0f59531..e64402b 100644 --- a/src/adapters/supabase/helpers/comment.ts +++ b/src/adapters/supabase/helpers/comment.ts @@ -42,8 +42,9 @@ export class Comment extends SuperSupabase { } const { error } = await this.supabase .from("issue_comments") - .insert([{ id: commentNodeId, plaintext, author_id: authorId, type: "comment", payload, embedding: embedding, issueId }]); + .insert([{ id: commentNodeId, plaintext, author_id: authorId, type: "comment", payload, embedding: embedding, issue_id: issueId }]); if (error) { + console.log(error.message, error.details, { id: commentNodeId, plaintext, author_id: authorId, type: "comment", payload, embedding }); this.context.logger.error("Error creating comment", error); return; } diff --git a/src/adapters/supabase/helpers/issues.ts b/src/adapters/supabase/helpers/issues.ts index e7d7e95..0040796 100644 --- a/src/adapters/supabase/helpers/issues.ts +++ b/src/adapters/supabase/helpers/issues.ts @@ -17,8 +17,9 @@ export class Issues extends SuperSupabase { super(supabase, context); } - async createIssue(issueNodeId: string, payloadObject: Record | null, isPrivate: boolean, plaintext: string | null, authorId: number) { + async createIssue(issueNodeId: string, payload: Record | null, isPrivate: boolean, plaintext: string | null, authorId: number) { //First Check if the issue already exists + console.log(payload); const { data, error } = await this.supabase.from("issues").select("*").eq("id", issueNodeId); if (error) { this.context.logger.error("Error creating issue", error); @@ -30,13 +31,12 @@ export class Issues extends SuperSupabase { } else { const embedding = await this.context.adapters.voyage.embedding.createEmbedding(plaintext); if (isPrivate) { - payloadObject = null; + payload = null; plaintext = null; } - const { error } = await this.supabase - .from("issues") - .insert([{ id: issueNodeId, payloadObject, type: "issue", plaintext, author_id: authorId, embedding }]); + const { error } = await this.supabase.from("issues").insert([{ id: issueNodeId, payload, type: "issue", plaintext, author_id: authorId, embedding }]); if (error) { + console.log(error.message, error.details, { id: issueNodeId, payload, type: "issue", plaintext, author_id: authorId, embedding }); this.context.logger.error("Error creating issue", error); return; } @@ -44,24 +44,22 @@ export class Issues extends SuperSupabase { this.context.logger.info("Issue created successfully"); } - async updateIssue(plaintext: string | null, issueNodeId: string, payloadObject: Record | null, isPrivate: boolean) { + async updateIssue(plaintext: string | null, issueNodeId: string, payload: Record | null, isPrivate: boolean) { //Create the embedding for this comment const embedding = Array.from(await this.context.adapters.voyage.embedding.createEmbedding(plaintext)); if (isPrivate) { plaintext = null as string | null; - payloadObject = null as Record | null; + payload = null as Record | null; } - const { error } = await this.supabase - .from("issue_comments") - .update({ plaintext, embedding: embedding, payloadObject, modified_at: new Date() }) - .eq("id", issueNodeId); + console.log({ id: issueNodeId, plaintext, payload, embedding }); + const { error } = await this.supabase.from("issues").update({ plaintext, embedding: embedding, payload, modified_at: new Date() }).eq("id", issueNodeId); if (error) { this.context.logger.error("Error updating comment", error); } } async deleteIssue(issueNodeId: string) { - const { error } = await this.supabase.from("issue_comments").delete().eq("id", issueNodeId); + const { error } = await this.supabase.from("issues").delete().eq("id", issueNodeId); if (error) { this.context.logger.error("Error deleting comment", error); } diff --git a/src/handlers/add-issue.ts b/src/handlers/add-issue.ts index ce7c315..4264a44 100644 --- a/src/handlers/add-issue.ts +++ b/src/handlers/add-issue.ts @@ -1,19 +1,19 @@ import { Context } from "../types"; +import { IssuePayload } from "../types/payload"; export async function addIssue(context: Context) { const { logger, - payload, adapters: { supabase }, } = context; - const payloadObject = payload; - const plaintext = payload.issue.body + payload.issue.title || ""; + const { payload } = context as { payload: IssuePayload }; + const plaintext = payload.issue.body + " " + payload.issue.title || ""; const authorId = payload.issue.user?.id || -1; const nodeId = payload.issue.node_id; const isPrivate = payload.repository.private; try { - await supabase.issue.createIssue(nodeId, payloadObject, isPrivate, plaintext, authorId); + await supabase.issue.createIssue(nodeId, payload, isPrivate, plaintext, authorId); } catch (error) { if (error instanceof Error) { logger.error(`Error creating issue:`, { error: error, stack: error.stack }); diff --git a/src/handlers/update-issue.ts b/src/handlers/update-issue.ts index fad405c..64f3548 100644 --- a/src/handlers/update-issue.ts +++ b/src/handlers/update-issue.ts @@ -10,7 +10,7 @@ export async function updateIssue(context: Context) { const payloadObject = payload; const nodeId = payload.issue.node_id; const isPrivate = payload.repository.private; - const plaintext = payload.issue.body + payload.issue.title || ""; + const plaintext = payload.issue.body + " " + payload.issue.title || ""; // Fetch the previous issue and update it in the db try { await supabase.issue.updateIssue(plaintext, nodeId, payloadObject, isPrivate); diff --git a/src/plugin.ts b/src/plugin.ts index fa67919..2f7dad8 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -62,7 +62,5 @@ export async function plugin(inputs: PluginInputs, env: Env) { adapters: {} as ReturnType, }; context.adapters = createAdapters(supabase, voyageClient, context); - if (isIssueEvent(context)) { - return await runPlugin(context); - } + return await runPlugin(context); } diff --git a/supabase/migrations/20240912225853_issue_comments.sql b/supabase/migrations/20240912225853_issue_comments.sql index 3a8764a..a6e24b4 100644 --- a/supabase/migrations/20240912225853_issue_comments.sql +++ b/supabase/migrations/20240912225853_issue_comments.sql @@ -6,6 +6,7 @@ CREATE TABLE IF NOT EXISTS issues ( plaintext text, embedding Vector(1024) not null, payload jsonb, + author_id VARCHAR not null, type text not null default 'issue', created_at timestamptz not null default now(), modified_at timestamptz not null default now() From 8d25ce12fe0c14c8d01864d3ab1d8138a86ec770 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Fri, 13 Sep 2024 01:05:03 -0400 Subject: [PATCH 32/37] fix: remove console.log --- src/adapters/supabase/helpers/comment.ts | 1 - src/adapters/supabase/helpers/issues.ts | 3 --- 2 files changed, 4 deletions(-) diff --git a/src/adapters/supabase/helpers/comment.ts b/src/adapters/supabase/helpers/comment.ts index e64402b..f2ff62a 100644 --- a/src/adapters/supabase/helpers/comment.ts +++ b/src/adapters/supabase/helpers/comment.ts @@ -44,7 +44,6 @@ export class Comment extends SuperSupabase { .from("issue_comments") .insert([{ id: commentNodeId, plaintext, author_id: authorId, type: "comment", payload, embedding: embedding, issue_id: issueId }]); if (error) { - console.log(error.message, error.details, { id: commentNodeId, plaintext, author_id: authorId, type: "comment", payload, embedding }); this.context.logger.error("Error creating comment", error); return; } diff --git a/src/adapters/supabase/helpers/issues.ts b/src/adapters/supabase/helpers/issues.ts index 0040796..7624ff8 100644 --- a/src/adapters/supabase/helpers/issues.ts +++ b/src/adapters/supabase/helpers/issues.ts @@ -19,7 +19,6 @@ export class Issues extends SuperSupabase { async createIssue(issueNodeId: string, payload: Record | null, isPrivate: boolean, plaintext: string | null, authorId: number) { //First Check if the issue already exists - console.log(payload); const { data, error } = await this.supabase.from("issues").select("*").eq("id", issueNodeId); if (error) { this.context.logger.error("Error creating issue", error); @@ -36,7 +35,6 @@ export class Issues extends SuperSupabase { } const { error } = await this.supabase.from("issues").insert([{ id: issueNodeId, payload, type: "issue", plaintext, author_id: authorId, embedding }]); if (error) { - console.log(error.message, error.details, { id: issueNodeId, payload, type: "issue", plaintext, author_id: authorId, embedding }); this.context.logger.error("Error creating issue", error); return; } @@ -51,7 +49,6 @@ export class Issues extends SuperSupabase { plaintext = null as string | null; payload = null as Record | null; } - console.log({ id: issueNodeId, plaintext, payload, embedding }); const { error } = await this.supabase.from("issues").update({ plaintext, embedding: embedding, payload, modified_at: new Date() }).eq("id", issueNodeId); if (error) { this.context.logger.error("Error updating comment", error); From 1a1599717374d76918fe5e4970bd8b6d7f8815be Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Fri, 13 Sep 2024 03:20:27 -0400 Subject: [PATCH 33/37] feat: added cols markdown and plaintext, adds code for conversion from markdown to plaintext --- eslint.config.mjs | 2 +- src/adapters/supabase/helpers/comment.ts | 23 +++++++++------ src/adapters/supabase/helpers/issues.ts | 28 ++++++++++++------- src/adapters/utils/markdown-to-plaintext.ts | 20 +++++++++++++ src/handlers/add-issue.ts | 4 +-- src/handlers/update-comments.ts | 4 +-- src/handlers/update-issue.ts | 4 +-- .../20240912225853_issue_comments.sql | 4 ++- .../20240913070225_issue_comments.sql | 5 ++++ 9 files changed, 67 insertions(+), 27 deletions(-) create mode 100644 src/adapters/utils/markdown-to-plaintext.ts create mode 100644 supabase/migrations/20240913070225_issue_comments.sql diff --git a/eslint.config.mjs b/eslint.config.mjs index f43237a..f594657 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -9,7 +9,7 @@ export default tsEslint.config({ "@typescript-eslint": tsEslint.plugin, "check-file": checkFile, }, - ignores: [".github/knip.ts", "src/types/database.ts"], + ignores: [".github/knip.ts", "src/types/database.ts", "src/adapters/utils/markdown-to-plaintext.ts"], extends: [eslint.configs.recommended, ...tsEslint.configs.recommended, sonarjs.configs.recommended], languageOptions: { parser: tsEslint.parser, diff --git a/src/adapters/supabase/helpers/comment.ts b/src/adapters/supabase/helpers/comment.ts index f2ff62a..3fa08b2 100644 --- a/src/adapters/supabase/helpers/comment.ts +++ b/src/adapters/supabase/helpers/comment.ts @@ -1,10 +1,11 @@ import { SupabaseClient } from "@supabase/supabase-js"; import { SuperSupabase } from "./supabase"; import { Context } from "../../../types/context"; +import { markdownToPlainText } from "../../utils/markdown-to-plaintext"; export interface CommentType { id: string; - plaintext?: string; + markdown?: string; author_id: number; created_at: string; modified_at: string; @@ -17,7 +18,7 @@ export class Comment extends SuperSupabase { } async createComment( - plaintext: string | null, + markdown: string | null, commentNodeId: string, authorId: number, payload: Record | null, @@ -35,14 +36,16 @@ export class Comment extends SuperSupabase { return; } else { //Create the embedding for this comment - const embedding = await this.context.adapters.voyage.embedding.createEmbedding(plaintext); + const embedding = await this.context.adapters.voyage.embedding.createEmbedding(markdown); + let plaintext: string | null = markdownToPlainText(markdown || ""); if (isPrivate) { - plaintext = null as string | null; + markdown = null as string | null; payload = null as Record | null; + plaintext = null as string | null; } const { error } = await this.supabase .from("issue_comments") - .insert([{ id: commentNodeId, plaintext, author_id: authorId, type: "comment", payload, embedding: embedding, issue_id: issueId }]); + .insert([{ id: commentNodeId, markdown, plaintext, author_id: authorId, payload, embedding: embedding, issue_id: issueId }]); if (error) { this.context.logger.error("Error creating comment", error); return; @@ -51,16 +54,18 @@ export class Comment extends SuperSupabase { this.context.logger.info("Comment created successfully"); } - async updateComment(plaintext: string | null, commentNodeId: string, payload: Record | null, isPrivate: boolean) { + async updateComment(markdown: string | null, commentNodeId: string, payload: Record | null, isPrivate: boolean) { //Create the embedding for this comment - const embedding = Array.from(await this.context.adapters.voyage.embedding.createEmbedding(plaintext)); + const embedding = Array.from(await this.context.adapters.voyage.embedding.createEmbedding(markdown)); + let plaintext: string | null = markdownToPlainText(markdown || ""); if (isPrivate) { - plaintext = null as string | null; + markdown = null as string | null; payload = null as Record | null; + plaintext = null as string | null; } const { error } = await this.supabase .from("issue_comments") - .update({ plaintext, embedding: embedding, payload, modified_at: new Date() }) + .update({ markdown, plaintext, embedding: embedding, payload, modified_at: new Date() }) .eq("id", commentNodeId); if (error) { this.context.logger.error("Error updating comment", error); diff --git a/src/adapters/supabase/helpers/issues.ts b/src/adapters/supabase/helpers/issues.ts index 7624ff8..063d8a9 100644 --- a/src/adapters/supabase/helpers/issues.ts +++ b/src/adapters/supabase/helpers/issues.ts @@ -1,10 +1,11 @@ import { SupabaseClient } from "@supabase/supabase-js"; import { SuperSupabase } from "./supabase"; import { Context } from "../../../types/context"; +import { markdownToPlainText } from "../../utils/markdown-to-plaintext"; export interface IssueType { id: string; - plaintext?: string; + markdown?: string; author_id: number; created_at: string; modified_at: string; @@ -17,7 +18,7 @@ export class Issues extends SuperSupabase { super(supabase, context); } - async createIssue(issueNodeId: string, payload: Record | null, isPrivate: boolean, plaintext: string | null, authorId: number) { + async createIssue(issueNodeId: string, payload: Record | null, isPrivate: boolean, markdown: string | null, authorId: number) { //First Check if the issue already exists const { data, error } = await this.supabase.from("issues").select("*").eq("id", issueNodeId); if (error) { @@ -28,12 +29,14 @@ export class Issues extends SuperSupabase { this.context.logger.info("Issue already exists"); return; } else { - const embedding = await this.context.adapters.voyage.embedding.createEmbedding(plaintext); + const embedding = await this.context.adapters.voyage.embedding.createEmbedding(markdown); + let plaintext: string | null = markdownToPlainText(markdown || ""); if (isPrivate) { payload = null; + markdown = null; plaintext = null; } - const { error } = await this.supabase.from("issues").insert([{ id: issueNodeId, payload, type: "issue", plaintext, author_id: authorId, embedding }]); + const { error } = await this.supabase.from("issues").insert([{ id: issueNodeId, payload, markdown, plaintext, author_id: authorId, embedding }]); if (error) { this.context.logger.error("Error creating issue", error); return; @@ -42,14 +45,19 @@ export class Issues extends SuperSupabase { this.context.logger.info("Issue created successfully"); } - async updateIssue(plaintext: string | null, issueNodeId: string, payload: Record | null, isPrivate: boolean) { + async updateIssue(markdown: string | null, issueNodeId: string, payload: Record | null, isPrivate: boolean) { //Create the embedding for this comment - const embedding = Array.from(await this.context.adapters.voyage.embedding.createEmbedding(plaintext)); + const embedding = Array.from(await this.context.adapters.voyage.embedding.createEmbedding(markdown)); + let plaintext: string | null = markdownToPlainText(markdown || ""); if (isPrivate) { - plaintext = null as string | null; + markdown = null as string | null; payload = null as Record | null; + plaintext = null as string | null; } - const { error } = await this.supabase.from("issues").update({ plaintext, embedding: embedding, payload, modified_at: new Date() }).eq("id", issueNodeId); + const { error } = await this.supabase + .from("issues") + .update({ markdown, plaintext, embedding: embedding, payload, modified_at: new Date() }) + .eq("id", issueNodeId); if (error) { this.context.logger.error("Error updating comment", error); } @@ -62,8 +70,8 @@ export class Issues extends SuperSupabase { } } - async findSimilarIssues(plaintext: string, threshold: number): Promise { - const embedding = await this.context.adapters.voyage.embedding.createEmbedding(plaintext); + async findSimilarIssues(markdown: string, threshold: number): Promise { + const embedding = await this.context.adapters.voyage.embedding.createEmbedding(markdown); const { data, error } = await this.supabase .from("issues") .select("*") diff --git a/src/adapters/utils/markdown-to-plaintext.ts b/src/adapters/utils/markdown-to-plaintext.ts new file mode 100644 index 0000000..5edf926 --- /dev/null +++ b/src/adapters/utils/markdown-to-plaintext.ts @@ -0,0 +1,20 @@ +/** + * Converts a Markdown string to plain text. + * @param markdown + * @returns + */ +export function markdownToPlainText(markdown: string): string { + let text = markdown.replace(/^#{1,6}\s+/gm, ""); // Remove headers + text = text.replace(/\[([^\]]+)\]\([^\)]+\)/g, "$1"); // Inline links + text = text.replace(/!\[([^\]]*)\]\([^\)]+\)/g, "$1"); // Inline images + text = text.replace(/```[\s\S]*?```/g, (match) => match.replace(/```/g, "").trim()); // Code blocks + text = text.replace(/`([^`]+)`/g, "$1"); // Inline code + text = text.replace(/(\*\*|__)(.*?)\1/g, "$2"); // Bold + text = text.replace(/(\*|_)(.*?)\1/g, "$2"); // Italic + text = text.replace(/~~(.*?)~~/g, "$1"); // Strikethrough + text = text.replace(/^>\s+/gm, ""); // Block quotes + text = text.replace(/^\s*[-*]{3,}\s*$/gm, ""); // Horizontal rules + text = text.replace(/\n{3,}/g, "\n\n"); // Remove extra newlines + text = text.replace(/\s+/g, " ").trim(); // Remove extra spaces + return text; +} diff --git a/src/handlers/add-issue.ts b/src/handlers/add-issue.ts index 4264a44..2afec8c 100644 --- a/src/handlers/add-issue.ts +++ b/src/handlers/add-issue.ts @@ -7,13 +7,13 @@ export async function addIssue(context: Context) { adapters: { supabase }, } = context; const { payload } = context as { payload: IssuePayload }; - const plaintext = payload.issue.body + " " + payload.issue.title || ""; + const markdown = payload.issue.body + " " + payload.issue.title || ""; const authorId = payload.issue.user?.id || -1; const nodeId = payload.issue.node_id; const isPrivate = payload.repository.private; try { - await supabase.issue.createIssue(nodeId, payload, isPrivate, plaintext, authorId); + await supabase.issue.createIssue(nodeId, payload, isPrivate, markdown, authorId); } catch (error) { if (error instanceof Error) { logger.error(`Error creating issue:`, { error: error, stack: error.stack }); diff --git a/src/handlers/update-comments.ts b/src/handlers/update-comments.ts index 6dd2dae..94e9715 100644 --- a/src/handlers/update-comments.ts +++ b/src/handlers/update-comments.ts @@ -9,10 +9,10 @@ export async function updateComment(context: Context) { const { payload } = context as { payload: CommentPayload }; const nodeId = payload.comment.node_id; const isPrivate = payload.repository.private; - const plaintext = payload.comment.body; + const markdown = payload.comment.body; // Fetch the previous comment and update it in the db try { - await supabase.comment.updateComment(plaintext, nodeId, payload, isPrivate); + await supabase.comment.updateComment(markdown, nodeId, payload, isPrivate); } catch (error) { if (error instanceof Error) { logger.error(`Error updating comment:`, { error: error, stack: error.stack }); diff --git a/src/handlers/update-issue.ts b/src/handlers/update-issue.ts index 64f3548..488aec7 100644 --- a/src/handlers/update-issue.ts +++ b/src/handlers/update-issue.ts @@ -10,10 +10,10 @@ export async function updateIssue(context: Context) { const payloadObject = payload; const nodeId = payload.issue.node_id; const isPrivate = payload.repository.private; - const plaintext = payload.issue.body + " " + payload.issue.title || ""; + const markdown = payload.issue.body + " " + payload.issue.title || ""; // Fetch the previous issue and update it in the db try { - await supabase.issue.updateIssue(plaintext, nodeId, payloadObject, isPrivate); + await supabase.issue.updateIssue(markdown, nodeId, payloadObject, isPrivate); } catch (error) { if (error instanceof Error) { logger.error(`Error updating issue:`, { error: error, stack: error.stack }); diff --git a/supabase/migrations/20240912225853_issue_comments.sql b/supabase/migrations/20240912225853_issue_comments.sql index a6e24b4..0d71cab 100644 --- a/supabase/migrations/20240912225853_issue_comments.sql +++ b/supabase/migrations/20240912225853_issue_comments.sql @@ -7,7 +7,6 @@ CREATE TABLE IF NOT EXISTS issues ( embedding Vector(1024) not null, payload jsonb, author_id VARCHAR not null, - type text not null default 'issue', created_at timestamptz not null default now(), modified_at timestamptz not null default now() ); @@ -17,5 +16,8 @@ ADD COLUMN issue_id VARCHAR REFERENCES issues(id) ON DELETE CASCADE; +ALTER TABLE issue_comments +DROP COLUMN type; + ALTER TABLE issue_comments RENAME COLUMN payloadobject TO payload; \ No newline at end of file diff --git a/supabase/migrations/20240913070225_issue_comments.sql b/supabase/migrations/20240913070225_issue_comments.sql new file mode 100644 index 0000000..09daf65 --- /dev/null +++ b/supabase/migrations/20240913070225_issue_comments.sql @@ -0,0 +1,5 @@ +ALTER TABLE issue_comments +ADD COLUMN markdown TEXT; + +ALTER TABLE issues +ADD COLUMN markdown TEXT; \ No newline at end of file From a4bcc711d361c6e3e60fa3864da474fcddcab0fc Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Fri, 13 Sep 2024 03:42:47 -0400 Subject: [PATCH 34/37] fix: fixed verbose to debug, changed model to voyage-large-2-instruct --- src/adapters/utils/markdown-to-plaintext.ts | 5 ++++- src/adapters/voyage/helpers/embedding.ts | 2 +- src/handlers/add-comments.ts | 6 +++--- src/handlers/add-issue.ts | 4 ++-- src/handlers/delete-comments.ts | 2 +- src/handlers/delete-issue.ts | 2 +- src/handlers/update-comments.ts | 4 ++-- src/handlers/update-issue.ts | 4 ++-- 8 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/adapters/utils/markdown-to-plaintext.ts b/src/adapters/utils/markdown-to-plaintext.ts index 5edf926..448a043 100644 --- a/src/adapters/utils/markdown-to-plaintext.ts +++ b/src/adapters/utils/markdown-to-plaintext.ts @@ -3,7 +3,10 @@ * @param markdown * @returns */ -export function markdownToPlainText(markdown: string): string { +export function markdownToPlainText(markdown: string | null): string | null { + if (!markdown) { + return markdown; + } let text = markdown.replace(/^#{1,6}\s+/gm, ""); // Remove headers text = text.replace(/\[([^\]]+)\]\([^\)]+\)/g, "$1"); // Inline links text = text.replace(/!\[([^\]]*)\]\([^\)]+\)/g, "$1"); // Inline images diff --git a/src/adapters/voyage/helpers/embedding.ts b/src/adapters/voyage/helpers/embedding.ts index cacc320..575543e 100644 --- a/src/adapters/voyage/helpers/embedding.ts +++ b/src/adapters/voyage/helpers/embedding.ts @@ -17,7 +17,7 @@ export class Embedding extends SuperVoyage { } else { const response = await this.client.embed({ input: text, - model: "voyage-2", + model: "voyage-large-2-instruct", }); return (response.data && response.data[0]?.embedding) || []; } diff --git a/src/handlers/add-comments.ts b/src/handlers/add-comments.ts index 4d3ad4b..5a3ef1c 100644 --- a/src/handlers/add-comments.ts +++ b/src/handlers/add-comments.ts @@ -7,14 +7,14 @@ export async function addComments(context: Context) { adapters: { supabase }, } = context; const { payload } = context as { payload: CommentPayload }; - const plaintext = payload.comment.body; + const markdown = payload.comment.body || null; const authorId = payload.comment.user?.id || -1; const nodeId = payload.comment.node_id; const isPrivate = payload.repository.private; const issueId = payload.issue.node_id; try { - await supabase.comment.createComment(plaintext, nodeId, authorId, payload, isPrivate, issueId); + await supabase.comment.createComment(markdown, nodeId, authorId, payload, isPrivate, issueId); } catch (error) { if (error instanceof Error) { logger.error(`Error creating comment:`, { error: error, stack: error.stack }); @@ -26,5 +26,5 @@ export async function addComments(context: Context) { } logger.ok(`Successfully created comment!`); - logger.verbose(`Exiting addComments`); + logger.debug(`Exiting addComments`); } diff --git a/src/handlers/add-issue.ts b/src/handlers/add-issue.ts index 2afec8c..8cf0449 100644 --- a/src/handlers/add-issue.ts +++ b/src/handlers/add-issue.ts @@ -7,7 +7,7 @@ export async function addIssue(context: Context) { adapters: { supabase }, } = context; const { payload } = context as { payload: IssuePayload }; - const markdown = payload.issue.body + " " + payload.issue.title || ""; + const markdown = payload.issue.body + " " + payload.issue.title || null; const authorId = payload.issue.user?.id || -1; const nodeId = payload.issue.node_id; const isPrivate = payload.repository.private; @@ -25,5 +25,5 @@ export async function addIssue(context: Context) { } logger.ok(`Successfully created issue!`); - logger.verbose(`Exiting addIssue`); + logger.debug(`Exiting addIssue`); } diff --git a/src/handlers/delete-comments.ts b/src/handlers/delete-comments.ts index 44fb8f2..8c9e394 100644 --- a/src/handlers/delete-comments.ts +++ b/src/handlers/delete-comments.ts @@ -22,5 +22,5 @@ export async function deleteComment(context: Context) { } logger.ok(`Successfully deleted comment!`); - logger.verbose(`Exiting deleteComments`); + logger.debug(`Exiting deleteComments`); } diff --git a/src/handlers/delete-issue.ts b/src/handlers/delete-issue.ts index ba16032..b392c69 100644 --- a/src/handlers/delete-issue.ts +++ b/src/handlers/delete-issue.ts @@ -22,5 +22,5 @@ export async function deleteIssues(context: Context) { } logger.ok(`Successfully deleted issue!`); - logger.verbose(`Exiting deleteIssue`); + logger.debug(`Exiting deleteIssue`); } diff --git a/src/handlers/update-comments.ts b/src/handlers/update-comments.ts index 94e9715..7d103d9 100644 --- a/src/handlers/update-comments.ts +++ b/src/handlers/update-comments.ts @@ -9,7 +9,7 @@ export async function updateComment(context: Context) { const { payload } = context as { payload: CommentPayload }; const nodeId = payload.comment.node_id; const isPrivate = payload.repository.private; - const markdown = payload.comment.body; + const markdown = payload.comment.body || null; // Fetch the previous comment and update it in the db try { await supabase.comment.updateComment(markdown, nodeId, payload, isPrivate); @@ -24,5 +24,5 @@ export async function updateComment(context: Context) { } logger.ok(`Successfully updated comment!`); - logger.verbose(`Exiting updateComment`); + logger.debug(`Exiting updateComment`); } diff --git a/src/handlers/update-issue.ts b/src/handlers/update-issue.ts index 488aec7..b7b5cd9 100644 --- a/src/handlers/update-issue.ts +++ b/src/handlers/update-issue.ts @@ -10,7 +10,7 @@ export async function updateIssue(context: Context) { const payloadObject = payload; const nodeId = payload.issue.node_id; const isPrivate = payload.repository.private; - const markdown = payload.issue.body + " " + payload.issue.title || ""; + const markdown = payload.issue.body + " " + payload.issue.title || null; // Fetch the previous issue and update it in the db try { await supabase.issue.updateIssue(markdown, nodeId, payloadObject, isPrivate); @@ -25,5 +25,5 @@ export async function updateIssue(context: Context) { } logger.ok(`Successfully updated issue!`); - logger.verbose(`Exiting updateIssue`); + logger.debug(`Exiting updateIssue`); } From 6beefba462b7dbb828f580b2535cdbfb2a210949 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Fri, 13 Sep 2024 03:46:32 -0400 Subject: [PATCH 35/37] fix: throw error for empty body --- src/handlers/add-comments.ts | 5 ++++- src/handlers/add-issue.ts | 3 +++ src/handlers/update-comments.ts | 3 +++ src/handlers/update-issue.ts | 3 +++ 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/handlers/add-comments.ts b/src/handlers/add-comments.ts index 5a3ef1c..54745a1 100644 --- a/src/handlers/add-comments.ts +++ b/src/handlers/add-comments.ts @@ -7,13 +7,16 @@ export async function addComments(context: Context) { adapters: { supabase }, } = context; const { payload } = context as { payload: CommentPayload }; - const markdown = payload.comment.body || null; + const markdown = payload.comment.body; const authorId = payload.comment.user?.id || -1; const nodeId = payload.comment.node_id; const isPrivate = payload.repository.private; const issueId = payload.issue.node_id; try { + if (!markdown) { + throw new Error("Comment body is empty"); + } await supabase.comment.createComment(markdown, nodeId, authorId, payload, isPrivate, issueId); } catch (error) { if (error instanceof Error) { diff --git a/src/handlers/add-issue.ts b/src/handlers/add-issue.ts index 8cf0449..969a5c2 100644 --- a/src/handlers/add-issue.ts +++ b/src/handlers/add-issue.ts @@ -13,6 +13,9 @@ export async function addIssue(context: Context) { const isPrivate = payload.repository.private; try { + if (!markdown) { + throw new Error("Issue body is empty"); + } await supabase.issue.createIssue(nodeId, payload, isPrivate, markdown, authorId); } catch (error) { if (error instanceof Error) { diff --git a/src/handlers/update-comments.ts b/src/handlers/update-comments.ts index 7d103d9..b1b9d18 100644 --- a/src/handlers/update-comments.ts +++ b/src/handlers/update-comments.ts @@ -12,6 +12,9 @@ export async function updateComment(context: Context) { const markdown = payload.comment.body || null; // Fetch the previous comment and update it in the db try { + if (!markdown) { + throw new Error("Comment body is empty"); + } await supabase.comment.updateComment(markdown, nodeId, payload, isPrivate); } catch (error) { if (error instanceof Error) { diff --git a/src/handlers/update-issue.ts b/src/handlers/update-issue.ts index b7b5cd9..763b2ba 100644 --- a/src/handlers/update-issue.ts +++ b/src/handlers/update-issue.ts @@ -13,6 +13,9 @@ export async function updateIssue(context: Context) { const markdown = payload.issue.body + " " + payload.issue.title || null; // Fetch the previous issue and update it in the db try { + if (!markdown) { + throw new Error("Issue body is empty"); + } await supabase.issue.updateIssue(markdown, nodeId, payloadObject, isPrivate); } catch (error) { if (error instanceof Error) { From 0d5d9758ef7c992e0beb58d2df5f007e635320e6 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Fri, 13 Sep 2024 03:55:49 -0400 Subject: [PATCH 36/37] fix: removed custom code for conversion of markdown to plaintext --- package.json | 3 + src/adapters/utils/markdown-to-plaintext.ts | 21 +++---- yarn.lock | 62 +++++++++++++++++++++ 3 files changed, 73 insertions(+), 13 deletions(-) diff --git a/package.json b/package.json index 7718367..6ef92a4 100644 --- a/package.json +++ b/package.json @@ -35,8 +35,11 @@ "@octokit/webhooks": "13.2.7", "@sinclair/typebox": "0.32.33", "@supabase/supabase-js": "^2.45.2", + "@types/markdown-it": "^14.1.2", "@ubiquity-dao/ubiquibot-logger": "^1.3.0", "dotenv": "16.4.5", + "markdown-it": "^14.1.0", + "markdown-it-plain-text": "^0.3.0", "typebox-validators": "0.3.5", "voyageai": "^0.0.1-5" }, diff --git a/src/adapters/utils/markdown-to-plaintext.ts b/src/adapters/utils/markdown-to-plaintext.ts index 448a043..93e5787 100644 --- a/src/adapters/utils/markdown-to-plaintext.ts +++ b/src/adapters/utils/markdown-to-plaintext.ts @@ -1,3 +1,6 @@ +import markdownit from "markdown-it"; +import plainTextPlugin from "markdown-it-plain-text"; + /** * Converts a Markdown string to plain text. * @param markdown @@ -7,17 +10,9 @@ export function markdownToPlainText(markdown: string | null): string | null { if (!markdown) { return markdown; } - let text = markdown.replace(/^#{1,6}\s+/gm, ""); // Remove headers - text = text.replace(/\[([^\]]+)\]\([^\)]+\)/g, "$1"); // Inline links - text = text.replace(/!\[([^\]]*)\]\([^\)]+\)/g, "$1"); // Inline images - text = text.replace(/```[\s\S]*?```/g, (match) => match.replace(/```/g, "").trim()); // Code blocks - text = text.replace(/`([^`]+)`/g, "$1"); // Inline code - text = text.replace(/(\*\*|__)(.*?)\1/g, "$2"); // Bold - text = text.replace(/(\*|_)(.*?)\1/g, "$2"); // Italic - text = text.replace(/~~(.*?)~~/g, "$1"); // Strikethrough - text = text.replace(/^>\s+/gm, ""); // Block quotes - text = text.replace(/^\s*[-*]{3,}\s*$/gm, ""); // Horizontal rules - text = text.replace(/\n{3,}/g, "\n\n"); // Remove extra newlines - text = text.replace(/\s+/g, " ").trim(); // Remove extra spaces - return text; + const md = markdownit(); + md.use(plainTextPlugin); + md.render(markdown); + //Package markdown-it-plain-text does not have types + return (md as any).plainText; } diff --git a/yarn.lock b/yarn.lock index e653515..8439328 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1961,16 +1961,34 @@ expect "^29.0.0" pretty-format "^29.0.0" +"@types/linkify-it@^5": + version "5.0.0" + resolved "https://registry.yarnpkg.com/@types/linkify-it/-/linkify-it-5.0.0.tgz#21413001973106cda1c3a9b91eedd4ccd5469d76" + integrity sha512-sVDA58zAw4eWAffKOaQH5/5j3XeayukzDk+ewSsnv3p4yJEZHCCzMDiZM8e0OUrRvmpGZ85jf4yDHkHsgBNr9Q== + "@types/lodash@^4.14.172": version "4.17.4" resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.17.4.tgz#0303b64958ee070059e3a7184048a55159fe20b7" integrity sha512-wYCP26ZLxaT3R39kiN2+HcJ4kTd3U1waI/cY7ivWYqFP6pW3ZNpvi6Wd6PHZx7T/t8z0vlkXMg3QYLa7DZ/IJQ== +"@types/markdown-it@^14.1.2": + version "14.1.2" + resolved "https://registry.yarnpkg.com/@types/markdown-it/-/markdown-it-14.1.2.tgz#57f2532a0800067d9b934f3521429a2e8bfb4c61" + integrity sha512-promo4eFwuiW+TfGxhi+0x3czqTYJkG8qB17ZUJiVF10Xm7NLVRSLUsfRTU/6h1e24VvRnXCx+hG7li58lkzog== + dependencies: + "@types/linkify-it" "^5" + "@types/mdurl" "^2" + "@types/md5@^2.3.0": version "2.3.5" resolved "https://registry.yarnpkg.com/@types/md5/-/md5-2.3.5.tgz#481cef0a896e3a5dcbfc5a8a8b02c05958af48a5" integrity sha512-/i42wjYNgE6wf0j2bcTX6kuowmdL/6PE4IVitMpm2eYKBUuYCprdcWVK+xEF0gcV6ufMCRhtxmReGfc6hIK7Jw== +"@types/mdurl@^2": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@types/mdurl/-/mdurl-2.0.0.tgz#d43878b5b20222682163ae6f897b20447233bdfd" + integrity sha512-RGdgjQUZba5p6QEFAVx2OGb8rQDL/cPRG7GiedRzMcJ1tYnUANBncjbSB1NRGwbvjcPeikRABz2nshyPk1bhWg== + "@types/mute-stream@^0.0.4": version "0.0.4" resolved "https://registry.yarnpkg.com/@types/mute-stream/-/mute-stream-0.0.4.tgz#77208e56a08767af6c5e1237be8888e2f255c478" @@ -3201,6 +3219,11 @@ emoji-regex@^9.2.2: resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-9.2.2.tgz#840c8803b0d8047f4ff0cf963176b32d4ef3ed72" integrity sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg== +entities@^4.4.0: + version "4.5.0" + resolved "https://registry.yarnpkg.com/entities/-/entities-4.5.0.tgz#5d268ea5e7113ec74c4d033b79ea5a35a488fb48" + integrity sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw== + env-paths@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/env-paths/-/env-paths-2.2.1.tgz#420399d416ce1fbe9bc0a07c62fa68d67fd0f8f2" @@ -4937,6 +4960,13 @@ lines-and-columns@^1.1.6: resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.2.4.tgz#eca284f75d2965079309dc0ad9255abb2ebc1632" integrity sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg== +linkify-it@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/linkify-it/-/linkify-it-5.0.0.tgz#9ef238bfa6dc70bd8e7f9572b52d369af569b421" + integrity sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ== + dependencies: + uc.micro "^2.0.0" + lint-staged@15.2.7: version "15.2.7" resolved "https://registry.yarnpkg.com/lint-staged/-/lint-staged-15.2.7.tgz#97867e29ed632820c0fb90be06cd9ed384025649" @@ -5117,6 +5147,23 @@ map-obj@^2.0.0: resolved "https://registry.yarnpkg.com/map-obj/-/map-obj-2.0.0.tgz#a65cd29087a92598b8791257a523e021222ac1f9" integrity sha512-TzQSV2DiMYgoF5RycneKVUzIa9bQsj/B3tTgsE3dOGqlzHnGIDaC7XBE7grnA+8kZPnfqSGFe95VHc2oc0VFUQ== +markdown-it-plain-text@^0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/markdown-it-plain-text/-/markdown-it-plain-text-0.3.0.tgz#374abfcc1c95ae3d7a5e09365558d43184e1e7db" + integrity sha512-JT5x7oXZaTY6lwxsnxaIAT4hm5Q2Mi8dZoCaCNg3s7JXmDxY84D90OtV0US436UvN4zOUHoac4ExceTogydOLw== + +markdown-it@^14.1.0: + version "14.1.0" + resolved "https://registry.yarnpkg.com/markdown-it/-/markdown-it-14.1.0.tgz#3c3c5992883c633db4714ccb4d7b5935d98b7d45" + integrity sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg== + dependencies: + argparse "^2.0.1" + entities "^4.4.0" + linkify-it "^5.0.0" + mdurl "^2.0.0" + punycode.js "^2.3.1" + uc.micro "^2.1.0" + md5@^2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/md5/-/md5-2.3.0.tgz#c3da9a6aae3a30b46b7b0c349b87b110dc3bda4f" @@ -5126,6 +5173,11 @@ md5@^2.3.0: crypt "0.0.2" is-buffer "~1.1.6" +mdurl@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/mdurl/-/mdurl-2.0.0.tgz#80676ec0433025dd3e17ee983d0fe8de5a2237e0" + integrity sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w== + memorystream@^0.3.1: version "0.3.1" resolved "https://registry.yarnpkg.com/memorystream/-/memorystream-0.3.1.tgz#86d7090b30ce455d63fbae12dda51a47ddcaf9b2" @@ -5738,6 +5790,11 @@ prompts@^2.0.1: kleur "^3.0.3" sisteransi "^1.0.5" +punycode.js@^2.3.1: + version "2.3.1" + resolved "https://registry.yarnpkg.com/punycode.js/-/punycode.js-2.3.1.tgz#6b53e56ad75588234e79f4affa90972c7dd8cdb7" + integrity sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA== + punycode@^2.1.0: version "2.3.1" resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.3.1.tgz#027422e2faec0b25e1549c3e1bd8309b9133b6e5" @@ -6601,6 +6658,11 @@ typescript@5.4.5: resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.4.5.tgz#42ccef2c571fdbd0f6718b1d1f5e6e5ef006f611" integrity sha512-vcI4UpRgg81oIRUFwR0WSIHKt11nJ7SAVlYNIu+QpqeyXP+gpQJy/Z4+F0aGxSE4MqwjyXvW/TzgkLAx2AGHwQ== +uc.micro@^2.0.0, uc.micro@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/uc.micro/-/uc.micro-2.1.0.tgz#f8d3f7d0ec4c3dea35a7e3c8efa4cb8b45c9e7ee" + integrity sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A== + ufo@^1.5.3: version "1.5.3" resolved "https://registry.yarnpkg.com/ufo/-/ufo-1.5.3.tgz#3325bd3c977b6c6cd3160bf4ff52989adc9d3344" From 5d6ac5ad648d5c0ad7e70e4db80089c258b11afc Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Fri, 13 Sep 2024 04:00:26 -0400 Subject: [PATCH 37/37] fix: spell check --- .cspell.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.cspell.json b/.cspell.json index ecd7f51..c78b966 100644 --- a/.cspell.json +++ b/.cspell.json @@ -25,7 +25,8 @@ "issuebody", "voyageai", "vectordump", - "payloadobject" + "payloadobject", + "markdownit" ], "dictionaries": ["typescript", "node", "software-terms"], "import": ["@cspell/dict-typescript/cspell-ext.json", "@cspell/dict-node/cspell-ext.json", "@cspell/dict-software-terms"],