From 5a80df0890dc97fadb11be0a02f3f7eaac65875c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 1 Sep 2024 03:35:51 +0000 Subject: [PATCH 01/55] chore(development): release 1.0.0 --- CHANGELOG.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..bf0b82f --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,29 @@ +# Changelog + +## 1.0.0 (2024-09-01) + + +### Features + +* modified tests ([97e267f](https://github.com/ubiquibot/issue-comment-embeddings/commit/97e267f801ce4e6bd29bbe967de3df4fc3b1942a)) + + +### Bug Fixes + +* added config.yml ([c0f784b](https://github.com/ubiquibot/issue-comment-embeddings/commit/c0f784b20e59c2c4714805331c7ae9034fd73f73)) +* added config.yml ([221d34d](https://github.com/ubiquibot/issue-comment-embeddings/commit/221d34d801af6ebd764028be4a5c6200a18b776e)) +* added config.yml ([d12c522](https://github.com/ubiquibot/issue-comment-embeddings/commit/d12c522291db36dcf6aea72e5759e1a055185d8f)) +* cspell fix ([736bea6](https://github.com/ubiquibot/issue-comment-embeddings/commit/736bea6172444fdf783ffff729879d8278ff82f3)) +* fixed tests missing supabase files ([0e870ac](https://github.com/ubiquibot/issue-comment-embeddings/commit/0e870ac50eb68249edf5fc4e46fd509425dd7bbb)) +* github workflow, types package.json, env examples ([16786d7](https://github.com/ubiquibot/issue-comment-embeddings/commit/16786d76ee7a598c885f15af1baeadcf6a471b2c)) +* issue_comments linting added issue_comments:edited, created and deleted ([9c0de23](https://github.com/ubiquibot/issue-comment-embeddings/commit/9c0de237048ce30bf4254960c443bf3938037dce)) +* knip workflow ([f325310](https://github.com/ubiquibot/issue-comment-embeddings/commit/f3253109c290c9fce6d14e6a2e1e328133ac6f81)) +* manifest.json, compute.yml ([21409d5](https://github.com/ubiquibot/issue-comment-embeddings/commit/21409d530c3aad6ff2676fc813314e5b29c1a533)) +* package.json ([806c6c0](https://github.com/ubiquibot/issue-comment-embeddings/commit/806c6c0b393a9b87741a6341fa65bc5b3d22cb15)) +* plugin name ([d91b991](https://github.com/ubiquibot/issue-comment-embeddings/commit/d91b991d717b7fb0b73359ca29ae6de08a1074b9)) +* readme.md ([9c5fbfe](https://github.com/ubiquibot/issue-comment-embeddings/commit/9c5fbfe9ca46eb842779468c85d329b9f941fb82)) +* readme.md ([2fec447](https://github.com/ubiquibot/issue-comment-embeddings/commit/2fec44786526e7c10faaa2c13c4349e1232cf5bd)) +* remove config.yml and wrangler.toml namespace entries ([127cc22](https://github.com/ubiquibot/issue-comment-embeddings/commit/127cc225903c3fe3ca934e8407df4eb9c27e378c)) +* removed config.yml changed name ([744e08c](https://github.com/ubiquibot/issue-comment-embeddings/commit/744e08cebac310ae81c3c102f5f3a9473e6e4b9e)) +* test and linting ([a4ee41e](https://github.com/ubiquibot/issue-comment-embeddings/commit/a4ee41e6fca8723ce2fddc96b1171c89cfe7d5b7)) +* wrangler name ([f890071](https://github.com/ubiquibot/issue-comment-embeddings/commit/f890071c01c5bb1d611a5b7aa07cba84f4546251)) From 120a68847502882c257c4118a77486786165afc5 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Sun, 8 Sep 2024 19:22:10 -0400 Subject: [PATCH 02/55] fix: config added --- .github/.ubiquibot.config.yml | 2 + package.json | 2 +- src/adapters/supabase/helpers/comment.ts | 37 +++++++++++-------- src/handlers/add-comments.ts | 16 +++----- .../20240908231044_issue_comments.sql | 15 ++++++++ 5 files changed, 45 insertions(+), 27 deletions(-) create mode 100644 .github/.ubiquibot.config.yml create mode 100644 supabase/migrations/20240908231044_issue_comments.sql diff --git a/.github/.ubiquibot.config.yml b/.github/.ubiquibot.config.yml new file mode 100644 index 0000000..1ee14a9 --- /dev/null +++ b/.github/.ubiquibot.config.yml @@ -0,0 +1,2 @@ +-plugin: http://127.0.0.1:5000 + runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" ] \ No newline at end of file diff --git a/package.json b/package.json index bb43997..1716467 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "knip-ci": "knip --no-exit-code --reporter json --config .github/knip.ts", "prepare": "husky install", "test": "jest --setupFiles dotenv/config --coverage", - "worker": "wrangler dev --env dev --port 4000", + "worker": "wrangler dev --env dev --port 5000", "supabase:generate:local": "supabase gen types typescript --local > src/types/database.ts", "supabase:generate:remote": "cross-env-shell \"supabase gen types typescript --project-id $SUPABASE_PROJECT_ID --schema public > src/types/database.ts\"" }, diff --git a/src/adapters/supabase/helpers/comment.ts b/src/adapters/supabase/helpers/comment.ts index 7530c10..43ae42e 100644 --- a/src/adapters/supabase/helpers/comment.ts +++ b/src/adapters/supabase/helpers/comment.ts @@ -3,8 +3,11 @@ import { SuperSupabase } from "./supabase"; import { Context } from "../../../types/context"; export interface CommentType { - id: number; - body: string; + id: string; + plaintext: string; + author_id: number; + created_at: string; + modified_at: string; embedding: number[]; } @@ -13,9 +16,9 @@ export class Comment extends SuperSupabase { super(supabase, context); } - async createComment(commentBody: string, commentId: number, issueBody: string) { + async createComment(plaintext: string, commentNodeId: string, authorId: number, isPrivate: boolean) { //First Check if the comment already exists - const { data, error } = await this.supabase.from("issue_comments").select("*").eq("id", commentId); + const { data, error } = await this.supabase.from("issue_comments").select("*").eq("id", commentNodeId); if (error) { this.context.logger.error("Error creating comment", error); return; @@ -25,10 +28,11 @@ export class Comment extends SuperSupabase { return; } else { //Create the embedding for this comment - const embedding = await this.context.adapters.openai.embedding.createEmbedding(commentBody); - const { error } = await this.supabase - .from("issue_comments") - .insert([{ id: commentId, commentbody: commentBody, issuebody: issueBody, embedding: embedding }]); + const embedding = await this.context.adapters.openai.embedding.createEmbedding(plaintext); + if (isPrivate) { + plaintext = "CENSORED"; + } + const { error } = await this.supabase.from("issue_comments").insert([{ id: commentNodeId, plaintext, author_id: authorId, embedding: embedding }]); if (error) { this.context.logger.error("Error creating comment", error); return; @@ -37,25 +41,28 @@ export class Comment extends SuperSupabase { this.context.logger.info("Comment created successfully"); } - async updateComment(commentBody: string, commentId: number) { + async updateComment(plaintext: string, commentNodeId: string, isPrivate: boolean) { //Create the embedding for this comment - const embedding = Array.from(await this.context.adapters.openai.embedding.createEmbedding(commentBody)); - const { error } = await this.supabase.from("issue_comments").update({ commentbody: commentBody, embedding: embedding }).eq("id", commentId); + const embedding = Array.from(await this.context.adapters.openai.embedding.createEmbedding(plaintext)); + if (isPrivate) { + plaintext = "CENSORED"; + } + const { error } = await this.supabase.from("issue_comments").update({ plaintext, embedding: embedding, modified_at: new Date() }).eq("id", commentNodeId); if (error) { this.context.logger.error("Error updating comment", error); } } - async getComment(commentId: number): Promise { - const { data, error } = await this.supabase.from("issue_comments").select("*").eq("id", commentId); + async getComment(commentNodeId: string): Promise { + const { data, error } = await this.supabase.from("issue_comments").select("*").eq("id", commentNodeId); if (error) { this.context.logger.error("Error getting comment", error); } return data; } - async deleteComment(commentId: number) { - const { error } = await this.supabase.from("issue_comments").delete().eq("id", commentId); + async deleteComment(commentNodeId: string) { + const { error } = await this.supabase.from("issue_comments").delete().eq("id", commentNodeId); if (error) { this.context.logger.error("Error deleting comment", error); } diff --git a/src/handlers/add-comments.ts b/src/handlers/add-comments.ts index 35bd387..5758861 100644 --- a/src/handlers/add-comments.ts +++ b/src/handlers/add-comments.ts @@ -6,20 +6,14 @@ export async function addComments(context: Context) { payload, adapters: { supabase }, } = context; - - const sender = payload.comment.user?.login; - const repo = payload.repository.name; - const issueNumber = payload.issue.number; - const issueBody = payload.issue.body || ""; - const owner = payload.repository.owner.login; - const body = payload.comment.body; - - // Log the payload - logger.info(`Executing addComments:`, { sender, repo, issueNumber, owner }); + const plaintext = payload.comment.body; + const authorId = payload.comment.user?.id || 0; + const nodeId = payload.comment.node_id; + const isPrivate = payload.repository.private; // Add the comment to the database try { - await supabase.comment.createComment(body, payload.comment.id, issueBody); + await supabase.comment.createComment(plaintext, nodeId, authorId, isPrivate); } catch (error) { if (error instanceof Error) { logger.error(`Error creating comment:`, { error: error, stack: error.stack }); diff --git a/supabase/migrations/20240908231044_issue_comments.sql b/supabase/migrations/20240908231044_issue_comments.sql new file mode 100644 index 0000000..51afb87 --- /dev/null +++ b/supabase/migrations/20240908231044_issue_comments.sql @@ -0,0 +1,15 @@ +-- Create the extension if it doesn't exist +CREATE EXTENSION IF NOT EXISTS vector; + +-- Drop the old issue_comments table +DROP TABLE IF EXISTS issue_comments; + +-- Create the issue_comments table +CREATE TABLE IF NOT EXISTS issue_comments ( + id VARCHAR PRIMARY KEY, + created_at TIMESTAMP WITH TIME ZONE DEFAULT current_timestamp NOT NULL, + modified_at TIMESTAMP WITH TIME ZONE NOT NULL, + author_id VARCHAR NOT NULL, + plaintext TEXT NOT NULL, + embedding VECTOR(3072) NOT NULL +); \ No newline at end of file From 381fda1960897ec10c7072f1d07b7adcc3fcd27c Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Sun, 8 Sep 2024 19:25:51 -0400 Subject: [PATCH 03/55] fix: config added --- .github/.ubiquibot.config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/.ubiquibot.config.yml b/.github/.ubiquibot.config.yml index 1ee14a9..1fa8ef5 100644 --- a/.github/.ubiquibot.config.yml +++ b/.github/.ubiquibot.config.yml @@ -1,2 +1,2 @@ --plugin: http://127.0.0.1:5000 +-plugin: https://ubiquity-os-comment-vector-embeddings.sshivaditya.workers.dev/ runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" ] \ No newline at end of file From 95703c74ce20f27e289e5c41e21590412f491e0d Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Sun, 8 Sep 2024 19:41:09 -0400 Subject: [PATCH 04/55] fix: config added --- .github/.ubiquibot.config.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/.ubiquibot.config.yml b/.github/.ubiquibot.config.yml index 1fa8ef5..6782ba6 100644 --- a/.github/.ubiquibot.config.yml +++ b/.github/.ubiquibot.config.yml @@ -1,2 +1,6 @@ --plugin: https://ubiquity-os-comment-vector-embeddings.sshivaditya.workers.dev/ - runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" ] \ No newline at end of file +plugins: + - name: test-app + id: test-app + uses: + - plugin: https://ubiquity-os-comment-vector-embeddings.sshivaditya.workers.dev/ + runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" ] \ No newline at end of file From 32bdae93da9c2a462e284879986e6dd83a6f2787 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Sun, 8 Sep 2024 19:43:29 -0400 Subject: [PATCH 05/55] fix: config added and updated readme --- .github/{.ubiquibot.config.yml => .ubiquibot-config.yml} | 0 README.md | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename .github/{.ubiquibot.config.yml => .ubiquibot-config.yml} (100%) diff --git a/.github/.ubiquibot.config.yml b/.github/.ubiquibot-config.yml similarity index 100% rename from .github/.ubiquibot.config.yml rename to .github/.ubiquibot-config.yml diff --git a/README.md b/README.md index 0ee0bbe..fb293b9 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ To set up the `.dev.vars` file, you will need to provide the following variables - `OPENAI_API_KEY`: The API key for OpenAI. ## Usage -- Add the following to your `.ubiquibot.config.yml` file with the appropriate URL: +- Add the following to your `.ubiquibot-config.yml` file with the appropriate URL: ```javascript -plugin: http://127.0.0.1:4000 runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" ] From ca3cf2d783137d49ce445f144ee996a8844b396e Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Sun, 8 Sep 2024 19:44:30 -0400 Subject: [PATCH 06/55] fix: incorrect url --- .github/.ubiquibot-config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/.ubiquibot-config.yml b/.github/.ubiquibot-config.yml index 6782ba6..3653a9c 100644 --- a/.github/.ubiquibot-config.yml +++ b/.github/.ubiquibot-config.yml @@ -2,5 +2,5 @@ plugins: - name: test-app id: test-app uses: - - plugin: https://ubiquity-os-comment-vector-embeddings.sshivaditya.workers.dev/ + - plugin: https://ubiquity-os-comment-vector-embeddings.sshivaditya.workers.dev runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" ] \ No newline at end of file From 71a65a36d06dd8fb24e345d01efe2e1a842bafa5 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Mon, 9 Sep 2024 01:10:31 -0400 Subject: [PATCH 07/55] fix: tests --- src/handlers/delete-comments.ts | 11 +---- src/handlers/update-comments.ts | 14 ++---- src/types/database.ts | 24 ++++++---- .../20240908235818_issue_comments.sql | 2 + tests/__mocks__/adapter.ts | 44 ++++++++++--------- tests/__mocks__/db.ts | 1 + tests/__mocks__/helpers.ts | 3 +- tests/__mocks__/strings.ts | 1 + tests/main.test.ts | 32 +++++++------- 9 files changed, 66 insertions(+), 66 deletions(-) create mode 100644 supabase/migrations/20240908235818_issue_comments.sql diff --git a/src/handlers/delete-comments.ts b/src/handlers/delete-comments.ts index 7e9842f..7b444ea 100644 --- a/src/handlers/delete-comments.ts +++ b/src/handlers/delete-comments.ts @@ -7,17 +7,10 @@ export async function deleteComment(context: Context) { adapters: { supabase }, } = context; - const sender = payload.comment.user?.login; - const repo = payload.repository.name; - const issueNumber = payload.issue.number; - const owner = payload.repository.owner.login; + const nodeId = payload.comment.node_id; - // Log the payload - logger.debug(`Executing deleteComment:`, { sender, repo, issueNumber, owner }); - - // Add the comment to the database try { - await supabase.comment.deleteComment(payload.comment.id); + await supabase.comment.deleteComment(nodeId); } catch (error) { if (error instanceof Error) { logger.error(`Error deleting comment:`, { error: error, stack: error.stack }); diff --git a/src/handlers/update-comments.ts b/src/handlers/update-comments.ts index b0c3142..4c51f17 100644 --- a/src/handlers/update-comments.ts +++ b/src/handlers/update-comments.ts @@ -7,18 +7,12 @@ export async function updateComment(context: Context) { adapters: { supabase }, } = context; - const sender = payload.comment.user?.login; - const repo = payload.repository.name; - const issueNumber = payload.issue.number; - const owner = payload.repository.owner.login; - const body = payload.comment.body; - - // Log the payload - logger.debug(`Executing updateComment:`, { sender, repo, issueNumber, owner }); - + const nodeId = payload.comment.node_id; + const isPrivate = payload.repository.private; + const plaintext = payload.comment.body; // Fetch the previous comment and update it in the db try { - await supabase.comment.updateComment(body, payload.comment.id); + await supabase.comment.updateComment(plaintext, nodeId, isPrivate); } catch (error) { if (error instanceof Error) { logger.error(`Error updating comment:`, { error: error, stack: error.stack }); diff --git a/src/types/database.ts b/src/types/database.ts index 4f00255..009eae6 100644 --- a/src/types/database.ts +++ b/src/types/database.ts @@ -30,22 +30,28 @@ export type Database = { Tables: { issue_comments: { Row: { - commentbody: string; + author_id: string; + created_at: string; embedding: string; - id: number; - issuebody: string | null; + id: string; + modified_at: string; + plaintext: string; }; Insert: { - commentbody: string; + author_id: string; + created_at?: string; embedding: string; - id: number; - issuebody?: string | null; + id: string; + modified_at?: string; + plaintext: string; }; Update: { - commentbody?: string; + author_id?: string; + created_at?: string; embedding?: string; - id?: number; - issuebody?: string | null; + id?: string; + modified_at?: string; + plaintext?: string; }; Relationships: []; }; diff --git a/supabase/migrations/20240908235818_issue_comments.sql b/supabase/migrations/20240908235818_issue_comments.sql new file mode 100644 index 0000000..11eb77a --- /dev/null +++ b/supabase/migrations/20240908235818_issue_comments.sql @@ -0,0 +1,2 @@ +ALTER TABLE issue_comments +ALTER COLUMN modified_at SET DEFAULT current_timestamp; \ No newline at end of file diff --git a/tests/__mocks__/adapter.ts b/tests/__mocks__/adapter.ts index 4c7a797..c2c02ad 100644 --- a/tests/__mocks__/adapter.ts +++ b/tests/__mocks__/adapter.ts @@ -4,47 +4,51 @@ import { Embedding } from "../../src/adapters/openai/helpers/embedding"; import { STRINGS } from "./strings"; export interface CommentMock { - id: number; - commentbody: string; - issuebody: string; + id: string; + plaintext: string; + author_id: number; embedding: number[]; } export function createMockAdapters(context: Context) { - const commentMap: Map = new Map(); + const commentMap: Map = new Map(); return { supabase: { comment: { - createComment: jest.fn(async (commentBody: string, commentId: number, issueBody: string) => { - if (commentMap.has(commentId)) { + createComment: jest.fn(async (plaintext: string, commentNodeId: string, authorId: number) => { + if (commentMap.has(commentNodeId)) { throw new Error("Comment already exists"); } - const embedding = await context.adapters.openai.embedding.createEmbedding(commentBody); - commentMap.set(commentId, { id: commentId, commentbody: commentBody, issuebody: issueBody, embedding }); + const embedding = await context.adapters.openai.embedding.createEmbedding(plaintext); + commentMap.set(commentNodeId, { id: commentNodeId, plaintext, author_id: authorId, embedding }); }), - updateComment: jest.fn(async (commentBody: string, commentId: number) => { - if (!commentMap.has(commentId)) { + updateComment: jest.fn(async (plaintext: string, commentNodeId: string, isPrivate: boolean) => { + console.log(commentMap); + if (!commentMap.has(commentNodeId)) { throw new Error(STRINGS.COMMENT_DOES_NOT_EXIST); } - const originalComment = commentMap.get(commentId); + const originalComment = commentMap.get(commentNodeId); if (!originalComment) { throw new Error(STRINGS.COMMENT_DOES_NOT_EXIST); } - const { id, issuebody } = originalComment; - const embedding = await context.adapters.openai.embedding.createEmbedding(commentBody); - commentMap.set(commentId, { id, commentbody: commentBody, issuebody, embedding }); + const { id, author_id } = originalComment; + const embedding = await context.adapters.openai.embedding.createEmbedding(plaintext); + if (isPrivate) { + plaintext = STRINGS.CENSORED; + } + commentMap.set(commentNodeId, { id, plaintext, author_id, embedding }); }), - deleteComment: jest.fn(async (commentId: number) => { - if (!commentMap.has(commentId)) { + deleteComment: jest.fn(async (commentNodeId: string) => { + if (!commentMap.has(commentNodeId)) { throw new Error(STRINGS.COMMENT_DOES_NOT_EXIST); } - commentMap.delete(commentId); + commentMap.delete(commentNodeId); }), - getComment: jest.fn(async (commentId: number) => { - if (!commentMap.has(commentId)) { + getComment: jest.fn(async (commentNodeId: string) => { + if (!commentMap.has(commentNodeId)) { throw new Error(STRINGS.COMMENT_DOES_NOT_EXIST); } - return commentMap.get(commentId); + return commentMap.get(commentNodeId); }), } as unknown as Comment, }, diff --git a/tests/__mocks__/db.ts b/tests/__mocks__/db.ts index f7bd2b6..e1883e5 100644 --- a/tests/__mocks__/db.ts +++ b/tests/__mocks__/db.ts @@ -75,6 +75,7 @@ export const db = factory({ body: String, created_at: Date, updated_at: Date, + node_id: String, issue_number: Number, user: { login: String, diff --git a/tests/__mocks__/helpers.ts b/tests/__mocks__/helpers.ts index b4c5b14..eb9e2ae 100644 --- a/tests/__mocks__/helpers.ts +++ b/tests/__mocks__/helpers.ts @@ -38,7 +38,7 @@ export async function setupTests() { }); } -export function createComment(comment: string, commentId: number) { +export function createComment(comment: string, commentId: number, nodeId: string) { const isComment = db.issueComments.findFirst({ where: { id: { @@ -63,6 +63,7 @@ export function createComment(comment: string, commentId: number) { id: commentId, body: comment, issue_number: 1, + node_id: nodeId, user: { login: STRINGS.USER_1, id: 1, diff --git a/tests/__mocks__/strings.ts b/tests/__mocks__/strings.ts index 83d7306..11ac64f 100644 --- a/tests/__mocks__/strings.ts +++ b/tests/__mocks__/strings.ts @@ -15,4 +15,5 @@ export const STRINGS = { CONFIGURABLE_RESPONSE: "Hello, Code Reviewers!", INVALID_COMMAND: "/Goodbye", COMMENT_DOES_NOT_EXIST: "Comment does not exist", + CENSORED: "[CENSORED]", }; diff --git a/tests/main.test.ts b/tests/main.test.ts index c78ec3d..a372de2 100644 --- a/tests/main.test.ts +++ b/tests/main.test.ts @@ -48,44 +48,41 @@ describe("Plugin tests", () => { }); it("When a comment is created it should add it to the database", async () => { - const { context } = createContext(); + const { context } = createContext(STRINGS.HELLO_WORLD, 1, 1, 1, 1, "sasasCreate"); await runPlugin(context); const supabase = context.adapters.supabase; try { - const issueBody = context.payload.issue.body || ""; - await supabase.comment.createComment(STRINGS.HELLO_WORLD, 1, issueBody); + await supabase.comment.createComment(STRINGS.HELLO_WORLD, "sasasCreate", 1, false); throw new Error("Expected method to reject."); } catch (error) { if (error instanceof Error) { expect(error.message).toBe("Comment already exists"); } } - const comment = (await supabase.comment.getComment(1)) as unknown as CommentMock; + const comment = (await supabase.comment.getComment("sasasCreate")) as unknown as CommentMock; expect(comment).toBeDefined(); - expect(comment?.commentbody).toBeDefined(); - expect(comment?.commentbody).toBe(STRINGS.HELLO_WORLD); + expect(comment?.plaintext).toBeDefined(); + expect(comment?.plaintext).toBe(STRINGS.HELLO_WORLD); }); it("When a comment is updated it should update the database", async () => { - const { context } = createContext("Updated Message", 1, 1, 1, 1, "issue_comment.edited"); + const { context } = createContext("Updated Message", 1, 1, 1, 1, "sasasUpdate", "issue_comment.edited"); const supabase = context.adapters.supabase; - const issueBody = context.payload.issue.body || ""; - await supabase.comment.createComment(STRINGS.HELLO_WORLD, 1, issueBody); + await supabase.comment.createComment(STRINGS.HELLO_WORLD, "sasasUpdate", 1, false); await runPlugin(context); - const comment = (await supabase.comment.getComment(1)) as unknown as CommentMock; + const comment = (await supabase.comment.getComment("sasasUpdate")) as unknown as CommentMock; expect(comment).toBeDefined(); - expect(comment?.commentbody).toBeDefined(); - expect(comment?.commentbody).toBe("Updated Message"); + expect(comment?.plaintext).toBeDefined(); + expect(comment?.plaintext).toBe("Updated Message"); }); it("When a comment is deleted it should delete it from the database", async () => { - const { context } = createContext("Text Message", 1, 1, 1, 1, "issue_comment.deleted"); + const { context } = createContext("Text Message", 1, 1, 1, 1, "sasasDelete", "issue_comment.deleted"); const supabase = context.adapters.supabase; - const issueBody = context.payload.issue.body || ""; - await supabase.comment.createComment(STRINGS.HELLO_WORLD, 1, issueBody); + await supabase.comment.createComment(STRINGS.HELLO_WORLD, "sasasDelete", 1, false); await runPlugin(context); try { - await supabase.comment.getComment(1); + await supabase.comment.getComment("sasasDelete"); throw new Error("Expected method to reject."); } catch (error) { if (error instanceof Error) { @@ -109,13 +106,14 @@ function createContext( payloadSenderId: number = 1, commentId: number = 1, issueOne: number = 1, + nodeId: string = "sasas", eventName: Context["eventName"] = "issue_comment.created" ) { const repo = db.repo.findFirst({ where: { id: { equals: repoId } } }) as unknown as Context["payload"]["repository"]; const sender = db.users.findFirst({ where: { id: { equals: payloadSenderId } } }) as unknown as Context["payload"]["sender"]; const issue1 = db.issue.findFirst({ where: { id: { equals: issueOne } } }) as unknown as Context["payload"]["issue"]; - createComment(commentBody, commentId); // create it first then pull it from the DB and feed it to _createContext + createComment(commentBody, commentId, nodeId); // create it first then pull it from the DB and feed it to _createContext const comment = db.issueComments.findFirst({ where: { id: { equals: commentId } } }) as unknown as Context["payload"]["comment"]; const context = createContextInner(repo, sender, issue1, comment, eventName); From 86c56d58f48612c3fcfbb36b4510e853163190f3 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Mon, 9 Sep 2024 01:22:53 -0400 Subject: [PATCH 08/55] fix: remove config.yml --- .github/.ubiquibot-config.yml | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 .github/.ubiquibot-config.yml diff --git a/.github/.ubiquibot-config.yml b/.github/.ubiquibot-config.yml deleted file mode 100644 index 3653a9c..0000000 --- a/.github/.ubiquibot-config.yml +++ /dev/null @@ -1,6 +0,0 @@ -plugins: - - name: test-app - id: test-app - uses: - - plugin: https://ubiquity-os-comment-vector-embeddings.sshivaditya.workers.dev - runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" ] \ No newline at end of file From 88477786d7fe67ce4903e5dd97b7e3bef3c05c68 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Mon, 9 Sep 2024 03:36:36 -0400 Subject: [PATCH 09/55] fix: nullable plaintext --- src/adapters/openai/helpers/embedding.ts | 24 +++++++++++-------- src/adapters/supabase/helpers/comment.ts | 6 ++--- src/handlers/add-comments.ts | 2 +- src/types/database.ts | 6 ++--- .../20240909072603_issue_comments.sql | 2 ++ tests/__mocks__/adapter.ts | 8 +++---- tests/__mocks__/strings.ts | 1 - 7 files changed, 27 insertions(+), 22 deletions(-) create mode 100644 supabase/migrations/20240909072603_issue_comments.sql diff --git a/src/adapters/openai/helpers/embedding.ts b/src/adapters/openai/helpers/embedding.ts index 6b912dd..3dfaeb9 100644 --- a/src/adapters/openai/helpers/embedding.ts +++ b/src/adapters/openai/helpers/embedding.ts @@ -11,15 +11,19 @@ export class Embedding extends SuperOpenAi { this.context = context; } - async createEmbedding(text: string): Promise { - const params: OpenAI.EmbeddingCreateParams = { - model: "text-embedding-3-large", - input: text, - dimensions: VECTOR_SIZE, - }; - const response = await this.client.embeddings.create({ - ...params, - }); - return response.data[0]?.embedding; + async createEmbedding(text: string | null): Promise { + if (text === null) { + return new Array(VECTOR_SIZE).fill(0); + } else { + const params: OpenAI.EmbeddingCreateParams = { + model: "text-embedding-3-large", + input: text, + dimensions: VECTOR_SIZE, + }; + const response = await this.client.embeddings.create({ + ...params, + }); + return response.data[0]?.embedding; + } } } diff --git a/src/adapters/supabase/helpers/comment.ts b/src/adapters/supabase/helpers/comment.ts index 43ae42e..d934f35 100644 --- a/src/adapters/supabase/helpers/comment.ts +++ b/src/adapters/supabase/helpers/comment.ts @@ -4,7 +4,7 @@ import { Context } from "../../../types/context"; export interface CommentType { id: string; - plaintext: string; + plaintext?: string; author_id: number; created_at: string; modified_at: string; @@ -30,7 +30,7 @@ export class Comment extends SuperSupabase { //Create the embedding for this comment const embedding = await this.context.adapters.openai.embedding.createEmbedding(plaintext); if (isPrivate) { - plaintext = "CENSORED"; + plaintext = null; } const { error } = await this.supabase.from("issue_comments").insert([{ id: commentNodeId, plaintext, author_id: authorId, embedding: embedding }]); if (error) { @@ -45,7 +45,7 @@ export class Comment extends SuperSupabase { //Create the embedding for this comment const embedding = Array.from(await this.context.adapters.openai.embedding.createEmbedding(plaintext)); if (isPrivate) { - plaintext = "CENSORED"; + plaintext = null; } const { error } = await this.supabase.from("issue_comments").update({ plaintext, embedding: embedding, modified_at: new Date() }).eq("id", commentNodeId); if (error) { diff --git a/src/handlers/add-comments.ts b/src/handlers/add-comments.ts index 5758861..aa3eb2e 100644 --- a/src/handlers/add-comments.ts +++ b/src/handlers/add-comments.ts @@ -7,7 +7,7 @@ export async function addComments(context: Context) { adapters: { supabase }, } = context; const plaintext = payload.comment.body; - const authorId = payload.comment.user?.id || 0; + const authorId = payload.comment.user?.id || -1; const nodeId = payload.comment.node_id; const isPrivate = payload.repository.private; diff --git a/src/types/database.ts b/src/types/database.ts index 009eae6..3622f52 100644 --- a/src/types/database.ts +++ b/src/types/database.ts @@ -35,7 +35,7 @@ export type Database = { embedding: string; id: string; modified_at: string; - plaintext: string; + plaintext: string | null; }; Insert: { author_id: string; @@ -43,7 +43,7 @@ export type Database = { embedding: string; id: string; modified_at?: string; - plaintext: string; + plaintext?: string | null; }; Update: { author_id?: string; @@ -51,7 +51,7 @@ export type Database = { embedding?: string; id?: string; modified_at?: string; - plaintext?: string; + plaintext?: string | null; }; Relationships: []; }; diff --git a/supabase/migrations/20240909072603_issue_comments.sql b/supabase/migrations/20240909072603_issue_comments.sql new file mode 100644 index 0000000..93de8ff --- /dev/null +++ b/supabase/migrations/20240909072603_issue_comments.sql @@ -0,0 +1,2 @@ +ALTER TABLE issue_comments +ALTER COLUMN plaintext DROP NOT NULL; \ No newline at end of file diff --git a/tests/__mocks__/adapter.ts b/tests/__mocks__/adapter.ts index c2c02ad..bb7f4e2 100644 --- a/tests/__mocks__/adapter.ts +++ b/tests/__mocks__/adapter.ts @@ -5,7 +5,7 @@ import { STRINGS } from "./strings"; export interface CommentMock { id: string; - plaintext: string; + plaintext: string | null; author_id: number; embedding: number[]; } @@ -15,14 +15,14 @@ export function createMockAdapters(context: Context) { return { supabase: { comment: { - createComment: jest.fn(async (plaintext: string, commentNodeId: string, authorId: number) => { + createComment: jest.fn(async (plaintext: string | null, commentNodeId: string, authorId: number) => { if (commentMap.has(commentNodeId)) { throw new Error("Comment already exists"); } const embedding = await context.adapters.openai.embedding.createEmbedding(plaintext); commentMap.set(commentNodeId, { id: commentNodeId, plaintext, author_id: authorId, embedding }); }), - updateComment: jest.fn(async (plaintext: string, commentNodeId: string, isPrivate: boolean) => { + updateComment: jest.fn(async (plaintext: string | null, commentNodeId: string, isPrivate: boolean) => { console.log(commentMap); if (!commentMap.has(commentNodeId)) { throw new Error(STRINGS.COMMENT_DOES_NOT_EXIST); @@ -34,7 +34,7 @@ export function createMockAdapters(context: Context) { const { id, author_id } = originalComment; const embedding = await context.adapters.openai.embedding.createEmbedding(plaintext); if (isPrivate) { - plaintext = STRINGS.CENSORED; + plaintext = null; } commentMap.set(commentNodeId, { id, plaintext, author_id, embedding }); }), diff --git a/tests/__mocks__/strings.ts b/tests/__mocks__/strings.ts index 11ac64f..83d7306 100644 --- a/tests/__mocks__/strings.ts +++ b/tests/__mocks__/strings.ts @@ -15,5 +15,4 @@ export const STRINGS = { CONFIGURABLE_RESPONSE: "Hello, Code Reviewers!", INVALID_COMMAND: "/Goodbye", COMMENT_DOES_NOT_EXIST: "Comment does not exist", - CENSORED: "[CENSORED]", }; From 5dd651e9855f2d83f04084afaf8ab4dd2b97ef27 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Mon, 9 Sep 2024 03:41:52 -0400 Subject: [PATCH 10/55] fix: jest tests --- src/adapters/supabase/helpers/comment.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/adapters/supabase/helpers/comment.ts b/src/adapters/supabase/helpers/comment.ts index d934f35..5baa7f5 100644 --- a/src/adapters/supabase/helpers/comment.ts +++ b/src/adapters/supabase/helpers/comment.ts @@ -16,7 +16,7 @@ export class Comment extends SuperSupabase { super(supabase, context); } - async createComment(plaintext: string, commentNodeId: string, authorId: number, isPrivate: boolean) { + async createComment(plaintext: string | null, commentNodeId: string, authorId: number, isPrivate: boolean) { //First Check if the comment already exists const { data, error } = await this.supabase.from("issue_comments").select("*").eq("id", commentNodeId); if (error) { @@ -30,7 +30,7 @@ export class Comment extends SuperSupabase { //Create the embedding for this comment const embedding = await this.context.adapters.openai.embedding.createEmbedding(plaintext); if (isPrivate) { - plaintext = null; + plaintext = null as string | null; } const { error } = await this.supabase.from("issue_comments").insert([{ id: commentNodeId, plaintext, author_id: authorId, embedding: embedding }]); if (error) { @@ -41,11 +41,11 @@ export class Comment extends SuperSupabase { this.context.logger.info("Comment created successfully"); } - async updateComment(plaintext: string, commentNodeId: string, isPrivate: boolean) { + async updateComment(plaintext: string | null, commentNodeId: string, isPrivate: boolean) { //Create the embedding for this comment const embedding = Array.from(await this.context.adapters.openai.embedding.createEmbedding(plaintext)); if (isPrivate) { - plaintext = null; + plaintext = null as string | null; } const { error } = await this.supabase.from("issue_comments").update({ plaintext, embedding: embedding, modified_at: new Date() }).eq("id", commentNodeId); if (error) { From f847053df0525f0238a0caf0f601a7ef4c6a02c4 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Tue, 10 Sep 2024 13:40:17 -0400 Subject: [PATCH 11/55] feat: added voyage ai support --- .github/workflows/compute.yml | 1 + README.md | 1 + package.json | 3 +- src/adapters/index.ts | 13 +++- src/adapters/supabase/helpers/comment.ts | 11 +++- src/adapters/voyage/helpers/embedding.ts | 25 ++++++++ src/adapters/voyage/helpers/voyage.ts | 12 ++++ src/plugin.ts | 6 +- src/types/env.ts | 1 + tests/main.test.ts | 1 + yarn.lock | 78 +++++++++++++++++++++++- 11 files changed, 143 insertions(+), 9 deletions(-) create mode 100644 src/adapters/voyage/helpers/embedding.ts create mode 100644 src/adapters/voyage/helpers/voyage.ts diff --git a/.github/workflows/compute.yml b/.github/workflows/compute.yml index f83e80c..b435d3d 100644 --- a/.github/workflows/compute.yml +++ b/.github/workflows/compute.yml @@ -25,6 +25,7 @@ jobs: SUPABASE_URL: ${{ secrets.SUPABASE_URL }} SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + VOYAGE_API_KEY: ${{secrets.VOYAGE_API_KEY}} steps: - uses: actions/checkout@v4 diff --git a/README.md b/README.md index fb293b9..e612d3c 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ To set up the `.dev.vars` file, you will need to provide the following variables - `SUPABASE_URL`: The URL for your Supabase instance. - `SUPABASE_KEY`: The key for your Supabase instance. - `OPENAI_API_KEY`: The API key for OpenAI. +- `VOYAGE_API_KEY`: The API key for Voyage. ## Usage - Add the following to your `.ubiquibot-config.yml` file with the appropriate URL: diff --git a/package.json b/package.json index 1716467..27277b0 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,8 @@ "@ubiquity-dao/ubiquibot-logger": "^1.3.0", "dotenv": "16.4.5", "openai": "^4.56.0", - "typebox-validators": "0.3.5" + "typebox-validators": "0.3.5", + "voyageai": "^0.0.1-5" }, "devDependencies": { "@commitlint/cli": "19.3.0", diff --git a/src/adapters/index.ts b/src/adapters/index.ts index e5f0346..0e5986c 100644 --- a/src/adapters/index.ts +++ b/src/adapters/index.ts @@ -4,17 +4,24 @@ import { Comment } from "./supabase/helpers/comment"; import { SuperSupabase } from "./supabase/helpers/supabase"; import { SuperOpenAi } from "./openai/helpers/openai"; import OpenAI from "openai"; -import { Embedding } from "./openai/helpers/embedding"; +import { Embedding as OpenAiEmbedding } from "./openai/helpers/embedding"; +import { Embedding as VoyageEmbedding } from "./voyage/helpers/embedding"; +import { SuperVoyage } from "./voyage/helpers/voyage"; +import { VoyageAIClient } from "voyageai"; -export function createAdapters(supabaseClient: SupabaseClient, openai: OpenAI, context: Context) { +export function createAdapters(supabaseClient: SupabaseClient, openai: OpenAI, voyage: VoyageAIClient, context: Context) { return { supabase: { comment: new Comment(supabaseClient, context), super: new SuperSupabase(supabaseClient, context), }, openai: { - embedding: new Embedding(openai, context), + embedding: new OpenAiEmbedding(openai, context), super: new SuperOpenAi(openai, context), }, + voyage: { + embedding: new VoyageEmbedding(voyage, context), + super: new SuperVoyage(voyage, context), + }, }; } diff --git a/src/adapters/supabase/helpers/comment.ts b/src/adapters/supabase/helpers/comment.ts index 5baa7f5..92bf2f6 100644 --- a/src/adapters/supabase/helpers/comment.ts +++ b/src/adapters/supabase/helpers/comment.ts @@ -28,7 +28,11 @@ export class Comment extends SuperSupabase { return; } else { //Create the embedding for this comment - const embedding = await this.context.adapters.openai.embedding.createEmbedding(plaintext); + const embedding = await this.context.adapters.voyage.embedding.createEmbedding(plaintext); + //If embedding is smaller than 3072, pad it with 0s + if (embedding.length < 3072) { + embedding.push(...new Array(3072 - embedding.length).fill(0)); + } if (isPrivate) { plaintext = null as string | null; } @@ -43,7 +47,10 @@ export class Comment extends SuperSupabase { async updateComment(plaintext: string | null, commentNodeId: string, isPrivate: boolean) { //Create the embedding for this comment - const embedding = Array.from(await this.context.adapters.openai.embedding.createEmbedding(plaintext)); + const embedding = Array.from(await this.context.adapters.voyage.embedding.createEmbedding(plaintext)); + if (embedding.length < 3072) { + embedding.push(...new Array(3072 - embedding.length).fill(0)); + } if (isPrivate) { plaintext = null as string | null; } diff --git a/src/adapters/voyage/helpers/embedding.ts b/src/adapters/voyage/helpers/embedding.ts new file mode 100644 index 0000000..19d29ce --- /dev/null +++ b/src/adapters/voyage/helpers/embedding.ts @@ -0,0 +1,25 @@ +import { VoyageAIClient } from "voyageai"; +import { Context } from "../../../types"; +import { SuperVoyage } from "./voyage"; +const VECTOR_SIZE = 3072; + +export class Embedding extends SuperVoyage { + protected context: Context; + + constructor(client: VoyageAIClient, context: Context) { + super(client, context); + this.context = context; + } + + async createEmbedding(text: string | null): Promise { + if (text === null) { + return new Array(VECTOR_SIZE).fill(0); + } else { + const response = await this.client.embed({ + input: text, + model: "voyage-2", + }); + return (response.data && response.data[0]?.embedding) || []; + } + } +} diff --git a/src/adapters/voyage/helpers/voyage.ts b/src/adapters/voyage/helpers/voyage.ts new file mode 100644 index 0000000..c08c0af --- /dev/null +++ b/src/adapters/voyage/helpers/voyage.ts @@ -0,0 +1,12 @@ +import { VoyageAIClient } from "voyageai"; +import { Context } from "../../../types/context"; + +export class SuperVoyage { + protected client: VoyageAIClient; + protected context: Context; + + constructor(client: VoyageAIClient, context: Context) { + this.client = client; + this.context = context; + } +} diff --git a/src/plugin.ts b/src/plugin.ts index 4197ff6..5f79651 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -10,6 +10,7 @@ import { addComments } from "./handlers/add-comments"; import { updateComment } from "./handlers/update-comments"; import { deleteComment } from "./handlers/delete-comments"; import OpenAI from "openai"; +import { VoyageAIClient } from "voyageai"; /** * The main plugin function. Split for easier testing. @@ -40,6 +41,9 @@ export async function plugin(inputs: PluginInputs, env: Env) { const openaiClient = new OpenAI({ apiKey: env.OPENAI_API_KEY, }); + const voyageClient = new VoyageAIClient({ + apiKey: env.VOYAGE_API_KEY, + }); const context: Context = { eventName: inputs.eventName, payload: inputs.eventPayload, @@ -49,6 +53,6 @@ export async function plugin(inputs: PluginInputs, env: Env) { logger: new Logs("info" as LogLevel), adapters: {} as ReturnType, }; - context.adapters = createAdapters(supabase, openaiClient, context); + context.adapters = createAdapters(supabase, openaiClient, voyageClient, context); return runPlugin(context); } diff --git a/src/types/env.ts b/src/types/env.ts index 6f47748..fbe521e 100644 --- a/src/types/env.ts +++ b/src/types/env.ts @@ -14,6 +14,7 @@ export const envSchema = T.Object({ SUPABASE_URL: T.String(), SUPABASE_KEY: T.String(), OPENAI_API_KEY: T.String(), + VOYAGE_API_KEY: T.String(), }); export const envValidator = new StandardValidator(envSchema); diff --git a/tests/main.test.ts b/tests/main.test.ts index a372de2..4833556 100644 --- a/tests/main.test.ts +++ b/tests/main.test.ts @@ -42,6 +42,7 @@ describe("Plugin tests", () => { SUPABASE_KEY: "test", SUPABASE_URL: "test", OPENAI_API_KEY: "test", + VOYAGE_API_KEY: "test", }); const content = await response.json(); expect(content).toEqual(manifest); diff --git a/yarn.lock b/yarn.lock index 78b6afa..b483a55 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2443,6 +2443,11 @@ balanced-match@^1.0.0: resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== +base64-js@^1.3.1: + version "1.5.1" + resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" + integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== + before-after-hook@^2.2.0: version "2.2.3" resolved "https://registry.yarnpkg.com/before-after-hook/-/before-after-hook-2.2.3.tgz#c51e809c81a4e354084422b9b26bad88249c517c" @@ -2519,6 +2524,14 @@ buffer-from@^1.0.0: resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5" integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ== +buffer@^6.0.3: + version "6.0.3" + resolved "https://registry.yarnpkg.com/buffer/-/buffer-6.0.3.tgz#2ace578459cc8fbe2a70aaa8f52ee63b6a74c6c6" + integrity sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA== + dependencies: + base64-js "^1.3.1" + ieee754 "^1.2.1" + call-bind@^1.0.2, call-bind@^1.0.5, call-bind@^1.0.6, call-bind@^1.0.7: version "1.0.7" resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.7.tgz#06016599c40c56498c18769d2730be242b6fa3b9" @@ -3535,6 +3548,11 @@ eventemitter3@^5.0.1: resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-5.0.1.tgz#53f5ffd0a492ac800721bb42c66b841de96423c4" integrity sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA== +events@^3.3.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/events/-/events-3.3.0.tgz#31a95ad0a924e2d2c419a813aeb2c4e878ea7400" + integrity sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q== + execa@^5.0.0: version "5.1.1" resolved "https://registry.yarnpkg.com/execa/-/execa-5.1.1.tgz#f80ad9cbf4298f7bd1d4c9555c21e93741c411dd" @@ -3743,6 +3761,11 @@ formdata-node@^4.3.2: node-domexception "1.0.0" web-streams-polyfill "4.0.0-beta.3" +formdata-node@^6.0.3: + version "6.0.3" + resolved "https://registry.yarnpkg.com/formdata-node/-/formdata-node-6.0.3.tgz#48f8e2206ae2befded82af621ef015f08168dc6d" + integrity sha512-8e1++BCiTzUno9v5IZ2J6bv4RU+3UKDmqWUQD0MIMVCd9AdhWkO1gw57oo1mNEX1dMq2EGI+FbWz4B92pscSQg== + formdata-polyfill@^4.0.10: version "4.0.10" resolved "https://registry.yarnpkg.com/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz#24807c31c9d402e002ab3d8c720144ceb8848423" @@ -4072,6 +4095,11 @@ identity-function@^1.0.0: resolved "https://registry.yarnpkg.com/identity-function/-/identity-function-1.0.0.tgz#bea1159f0985239be3ca348edf40ce2f0dd2c21d" integrity sha512-kNrgUK0qI+9qLTBidsH85HjDLpZfrrS0ElquKKe/fJFdB3D7VeKdXXEvOPDUHSHOzdZKCAAaQIWWyp0l2yq6pw== +ieee754@^1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" + integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA== + ignore@^5.1.4, ignore@^5.1.8, ignore@^5.2.0, ignore@^5.3.1: version "5.3.1" resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.3.1.tgz#5073e554cd42c5b33b394375f538b8593e34d4ef" @@ -4821,6 +4849,11 @@ jiti@^1.21.0: resolved "https://registry.yarnpkg.com/jiti/-/jiti-1.21.6.tgz#6c7f7398dd4b3142767f9a168af2f317a428d268" integrity sha512-2yTgeWTWzMWkHu6Jp9NKgePDaYHbntiwvYuuJLbbN9vl7DC9DvXKOB2BC3ZZ92D3cvV/aflH0osDfwpHepQ53w== +js-base64@3.7.2: + version "3.7.2" + resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-3.7.2.tgz#816d11d81a8aff241603d19ce5761e13e41d7745" + integrity sha512-NnRs6dsyqUXejqk/yv2aiXlAvOs56sLkX6nUdeaNezI5LFFLlsZjOThmwnrcwh5ZZRwZlCMnVAY3CvhIhoVEKQ== + js-tokens@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499" @@ -5330,7 +5363,7 @@ node-fetch-native@^1.6.4: resolved "https://registry.yarnpkg.com/node-fetch-native/-/node-fetch-native-1.6.4.tgz#679fc8fd8111266d47d7e72c379f1bed9acff06e" integrity sha512-IhOigYzAKHd244OC0JIMIUrjzctirCmPkaIfhDeGcEETWof5zKYUW7e7MYvChGWh/4CJeXEgsRyGzuF334rOOQ== -node-fetch@^2.6.7: +node-fetch@2.7.0, node-fetch@^2.6.7: version "2.7.0" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== @@ -5752,6 +5785,11 @@ printable-characters@^1.0.42: resolved "https://registry.yarnpkg.com/printable-characters/-/printable-characters-1.0.42.tgz#3f18e977a9bd8eb37fcc4ff5659d7be90868b3d8" integrity sha512-dKp+C4iXWK4vVYZmYSd0KBH5F/h1HoZRsbJ82AVKRO3PEo8L4lBS/vLwhVtpwwuYcoIsVY+1JYKR268yn480uQ== +process@^0.11.10: + version "0.11.10" + resolved "https://registry.yarnpkg.com/process/-/process-0.11.10.tgz#7332300e840161bda3e69a1d1d91a7d4bc16f182" + integrity sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A== + prompts@^2.0.1: version "2.4.2" resolved "https://registry.yarnpkg.com/prompts/-/prompts-2.4.2.tgz#7b57e73b3a48029ad10ebd44f74b01722a4cb069" @@ -5770,6 +5808,13 @@ pure-rand@^6.0.0: resolved "https://registry.yarnpkg.com/pure-rand/-/pure-rand-6.1.0.tgz#d173cf23258231976ccbdb05247c9787957604f2" integrity sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA== +qs@6.11.2: + version "6.11.2" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.11.2.tgz#64bea51f12c1f5da1bc01496f48ffcff7c69d7d9" + integrity sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA== + dependencies: + side-channel "^1.0.4" + queue-microtask@^1.2.2: version "1.2.3" resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.3.tgz#4929228bbc724dfac43e0efb058caf7b6cfb6243" @@ -5803,6 +5848,17 @@ readable-stream@^3.4.0: string_decoder "^1.1.1" util-deprecate "^1.0.1" +readable-stream@^4.5.2: + version "4.5.2" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-4.5.2.tgz#9e7fc4c45099baeed934bff6eb97ba6cf2729e09" + integrity sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g== + dependencies: + abort-controller "^3.0.0" + buffer "^6.0.3" + events "^3.3.0" + process "^0.11.10" + string_decoder "^1.3.0" + readdirp@~3.6.0: version "3.6.0" resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.6.0.tgz#74a370bd857116e245b29cc97340cd431a02a6c7" @@ -6288,7 +6344,7 @@ string.prototype.trimstart@^1.0.8: define-properties "^1.2.1" es-object-atoms "^1.0.0" -string_decoder@^1.1.1: +string_decoder@^1.1.1, string_decoder@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e" integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA== @@ -6676,6 +6732,11 @@ uri-js@^4.2.2, uri-js@^4.4.1: dependencies: punycode "^2.1.0" +url-join@4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/url-join/-/url-join-4.0.1.tgz#b642e21a2646808ffa178c4c5fda39844e12cde7" + integrity sha512-jk1+QP6ZJqyOiuEI9AEWQfju/nB2Pw466kbA0LEZljHwKeMgd9WrAEgEGxjPDD2+TNbbb37rTyhEfrCXfuKXnA== + util-deprecate@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" @@ -6708,6 +6769,19 @@ vlq@^0.2.1: resolved "https://registry.yarnpkg.com/vlq/-/vlq-0.2.3.tgz#8f3e4328cf63b1540c0d67e1b2778386f8975b26" integrity sha512-DRibZL6DsNhIgYQ+wNdWDL2SL3bKPlVrRiBqV5yuMm++op8W4kGFtaQfCs4KEJn0wBZcHVHJ3eoywX8983k1ow== +voyageai@^0.0.1-5: + version "0.0.1-5" + resolved "https://registry.yarnpkg.com/voyageai/-/voyageai-0.0.1-5.tgz#e0457d991784900c16e4cdf095654f195d62fdf2" + integrity sha512-IuXSXM3l9J3NIq+MLHXacG/yhswpEgWIu9eBqoFqMRnFiDx00dLL62OWg6WqVSipddZLwFeWH1Kaj56x5eqhOQ== + dependencies: + form-data "^4.0.0" + formdata-node "^6.0.3" + js-base64 "3.7.2" + node-fetch "2.7.0" + qs "6.11.2" + readable-stream "^4.5.2" + url-join "4.0.1" + vscode-languageserver-textdocument@^1.0.11: version "1.0.11" resolved "https://registry.yarnpkg.com/vscode-languageserver-textdocument/-/vscode-languageserver-textdocument-1.0.11.tgz#0822a000e7d4dc083312580d7575fe9e3ba2e2bf" From 1549feb6065e5b82f1ec5289287ca88c49810bc6 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Tue, 10 Sep 2024 13:40:42 -0400 Subject: [PATCH 12/55] fix: cspell issue --- .cspell.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.cspell.json b/.cspell.json index 83a383f..b297a90 100644 --- a/.cspell.json +++ b/.cspell.json @@ -22,7 +22,8 @@ "knip", "mischeck", "commentbody", - "issuebody" + "issuebody", + "voyageai" ], "dictionaries": ["typescript", "node", "software-terms"], "import": ["@cspell/dict-typescript/cspell-ext.json", "@cspell/dict-node/cspell-ext.json", "@cspell/dict-software-terms"], From 71854b4aeae548e34e26856b4da4c33881335150 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Tue, 10 Sep 2024 23:45:37 -0400 Subject: [PATCH 13/55] fix: adds serialized comment object and payload to the schema --- .github/.ubiquibot-config.yml | 6 +++ .github/workflows/compute.yml | 2 +- README.md | 2 +- src/adapters/supabase/helpers/comment.ts | 13 +++++-- src/adapters/utils/cleancommentobject.ts | 37 +++++++++++++++++++ src/handlers/add-comments.ts | 4 +- src/handlers/update-comments.ts | 5 ++- src/plugin.ts | 2 +- src/types/env.ts | 2 +- .../20240911023641_issue_comments.sql | 2 + tests/main.test.ts | 12 ++++-- 11 files changed, 72 insertions(+), 15 deletions(-) create mode 100644 .github/.ubiquibot-config.yml create mode 100644 src/adapters/utils/cleancommentobject.ts create mode 100644 supabase/migrations/20240911023641_issue_comments.sql diff --git a/.github/.ubiquibot-config.yml b/.github/.ubiquibot-config.yml new file mode 100644 index 0000000..8a9597c --- /dev/null +++ b/.github/.ubiquibot-config.yml @@ -0,0 +1,6 @@ +plugins: + - name: test-app + id: test-app + uses: + - plugin: https://ubiquity-os-comment-vector-embeddings.sshivaditya.workers.dev + runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" ] diff --git a/.github/workflows/compute.yml b/.github/workflows/compute.yml index b435d3d..10678ce 100644 --- a/.github/workflows/compute.yml +++ b/.github/workflows/compute.yml @@ -25,7 +25,7 @@ jobs: SUPABASE_URL: ${{ secrets.SUPABASE_URL }} SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - VOYAGE_API_KEY: ${{secrets.VOYAGE_API_KEY}} + VOYAGEAI_API_KEY: ${{secrets.VOYAGEAI_API_KEY}} steps: - uses: actions/checkout@v4 diff --git a/README.md b/README.md index e612d3c..29cc9a0 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ To set up the `.dev.vars` file, you will need to provide the following variables - `SUPABASE_URL`: The URL for your Supabase instance. - `SUPABASE_KEY`: The key for your Supabase instance. - `OPENAI_API_KEY`: The API key for OpenAI. -- `VOYAGE_API_KEY`: The API key for Voyage. +- `VOYAGEAI_API_KEY`: The API key for Voyage. ## Usage - Add the following to your `.ubiquibot-config.yml` file with the appropriate URL: diff --git a/src/adapters/supabase/helpers/comment.ts b/src/adapters/supabase/helpers/comment.ts index 92bf2f6..ed37d48 100644 --- a/src/adapters/supabase/helpers/comment.ts +++ b/src/adapters/supabase/helpers/comment.ts @@ -16,7 +16,7 @@ export class Comment extends SuperSupabase { super(supabase, context); } - async createComment(plaintext: string | null, commentNodeId: string, authorId: number, isPrivate: boolean) { + async createComment(plaintext: string | null, commentNodeId: string, authorId: number, commentobject: JSON | null, isPrivate: boolean) { //First Check if the comment already exists const { data, error } = await this.supabase.from("issue_comments").select("*").eq("id", commentNodeId); if (error) { @@ -36,7 +36,9 @@ export class Comment extends SuperSupabase { if (isPrivate) { plaintext = null as string | null; } - const { error } = await this.supabase.from("issue_comments").insert([{ id: commentNodeId, plaintext, author_id: authorId, embedding: embedding }]); + const { error } = await this.supabase + .from("issue_comments") + .insert([{ id: commentNodeId, plaintext, author_id: authorId, commentobject, embedding: embedding }]); if (error) { this.context.logger.error("Error creating comment", error); return; @@ -45,7 +47,7 @@ export class Comment extends SuperSupabase { this.context.logger.info("Comment created successfully"); } - async updateComment(plaintext: string | null, commentNodeId: string, isPrivate: boolean) { + async updateComment(plaintext: string | null, commentNodeId: string, commentobject: JSON, isPrivate: boolean) { //Create the embedding for this comment const embedding = Array.from(await this.context.adapters.voyage.embedding.createEmbedding(plaintext)); if (embedding.length < 3072) { @@ -54,7 +56,10 @@ export class Comment extends SuperSupabase { if (isPrivate) { plaintext = null as string | null; } - const { error } = await this.supabase.from("issue_comments").update({ plaintext, embedding: embedding, modified_at: new Date() }).eq("id", commentNodeId); + const { error } = await this.supabase + .from("issue_comments") + .update({ plaintext, embedding: embedding, commentobject, modified_at: new Date() }) + .eq("id", commentNodeId); if (error) { this.context.logger.error("Error updating comment", error); } diff --git a/src/adapters/utils/cleancommentobject.ts b/src/adapters/utils/cleancommentobject.ts new file mode 100644 index 0000000..715a3ed --- /dev/null +++ b/src/adapters/utils/cleancommentobject.ts @@ -0,0 +1,37 @@ +import { EmitterWebhookEvent as WebhookEvent } from "@octokit/webhooks"; +import { Type } from "@sinclair/typebox"; + +const commentObjectSchema = Type.Object({ + action: Type.String(), + issue: Type.Object({ + id: Type.Number(), + number: Type.Number(), + title: Type.String(), + body: Type.String(), + user: Type.Object({ + login: Type.String(), + id: Type.Number(), + }), + }), + comment: Type.Object({ + author_association: Type.String(), + id: Type.Number(), + html_url: Type.String(), + issue_url: Type.String(), + user: Type.Object({ + login: Type.String(), + id: Type.Number(), + }), + }), +}); + +/** + * Cleans the comment object. + * + * @param commentObject - The comment object. + * @returns The cleaned comment object. + */ +export const cleanCommentObject = (commentObject: WebhookEvent["payload"]): JSON => { + // Apply the schema to the comment object + return commentObjectSchema.parse(commentObject); +}; diff --git a/src/handlers/add-comments.ts b/src/handlers/add-comments.ts index aa3eb2e..f572567 100644 --- a/src/handlers/add-comments.ts +++ b/src/handlers/add-comments.ts @@ -1,3 +1,4 @@ +import { cleanCommentObject } from "../adapters/utils/cleancommentobject"; import { Context } from "../types"; export async function addComments(context: Context) { @@ -6,6 +7,7 @@ export async function addComments(context: Context) { payload, adapters: { supabase }, } = context; + const commentobject = cleanCommentObject(payload); const plaintext = payload.comment.body; const authorId = payload.comment.user?.id || -1; const nodeId = payload.comment.node_id; @@ -13,7 +15,7 @@ export async function addComments(context: Context) { // Add the comment to the database try { - await supabase.comment.createComment(plaintext, nodeId, authorId, isPrivate); + await supabase.comment.createComment(plaintext, nodeId, authorId, commentobject as JSON, isPrivate); } catch (error) { if (error instanceof Error) { logger.error(`Error creating comment:`, { error: error, stack: error.stack }); diff --git a/src/handlers/update-comments.ts b/src/handlers/update-comments.ts index 4c51f17..cf85a9d 100644 --- a/src/handlers/update-comments.ts +++ b/src/handlers/update-comments.ts @@ -1,3 +1,4 @@ +import { cleanCommentObject } from "../adapters/utils/cleancommentobject"; import { Context } from "../types"; export async function updateComment(context: Context) { @@ -6,13 +7,13 @@ export async function updateComment(context: Context) { payload, adapters: { supabase }, } = context; - + const commentobject = cleanCommentObject(payload); const nodeId = payload.comment.node_id; const isPrivate = payload.repository.private; const plaintext = payload.comment.body; // Fetch the previous comment and update it in the db try { - await supabase.comment.updateComment(plaintext, nodeId, isPrivate); + await supabase.comment.updateComment(plaintext, nodeId, commentobject, isPrivate); } catch (error) { if (error instanceof Error) { logger.error(`Error updating comment:`, { error: error, stack: error.stack }); diff --git a/src/plugin.ts b/src/plugin.ts index 5f79651..0b8de96 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -42,7 +42,7 @@ export async function plugin(inputs: PluginInputs, env: Env) { apiKey: env.OPENAI_API_KEY, }); const voyageClient = new VoyageAIClient({ - apiKey: env.VOYAGE_API_KEY, + apiKey: env.VOYAGEAI_API_KEY, }); const context: Context = { eventName: inputs.eventName, diff --git a/src/types/env.ts b/src/types/env.ts index fbe521e..1fe1588 100644 --- a/src/types/env.ts +++ b/src/types/env.ts @@ -14,7 +14,7 @@ export const envSchema = T.Object({ SUPABASE_URL: T.String(), SUPABASE_KEY: T.String(), OPENAI_API_KEY: T.String(), - VOYAGE_API_KEY: T.String(), + VOYAGEAI_API_KEY: T.String(), }); export const envValidator = new StandardValidator(envSchema); diff --git a/supabase/migrations/20240911023641_issue_comments.sql b/supabase/migrations/20240911023641_issue_comments.sql new file mode 100644 index 0000000..06f28b5 --- /dev/null +++ b/supabase/migrations/20240911023641_issue_comments.sql @@ -0,0 +1,2 @@ +ALTER TABLE issue_comments +ADD COLUMN commentobject jsonb; \ No newline at end of file diff --git a/tests/main.test.ts b/tests/main.test.ts index 4833556..384d95a 100644 --- a/tests/main.test.ts +++ b/tests/main.test.ts @@ -14,6 +14,7 @@ import { Logs } from "@ubiquity-dao/ubiquibot-logger"; import { Env } from "../src/types"; import { runPlugin } from "../src/plugin"; import { CommentMock, createMockAdapters } from "./__mocks__/adapter"; +import { cleanCommentObject } from "../src/adapters/utils/cleancommentobject"; dotenv.config(); jest.requireActual("@octokit/rest"); @@ -42,7 +43,7 @@ describe("Plugin tests", () => { SUPABASE_KEY: "test", SUPABASE_URL: "test", OPENAI_API_KEY: "test", - VOYAGE_API_KEY: "test", + VOYAGEAI_API_KEY: "test", }); const content = await response.json(); expect(content).toEqual(manifest); @@ -52,8 +53,9 @@ describe("Plugin tests", () => { const { context } = createContext(STRINGS.HELLO_WORLD, 1, 1, 1, 1, "sasasCreate"); await runPlugin(context); const supabase = context.adapters.supabase; + const commentObject = cleanCommentObject(context.payload); try { - await supabase.comment.createComment(STRINGS.HELLO_WORLD, "sasasCreate", 1, false); + await supabase.comment.createComment(STRINGS.HELLO_WORLD, "sasasCreate", 1, commentObject, false); throw new Error("Expected method to reject."); } catch (error) { if (error instanceof Error) { @@ -69,7 +71,8 @@ describe("Plugin tests", () => { it("When a comment is updated it should update the database", async () => { const { context } = createContext("Updated Message", 1, 1, 1, 1, "sasasUpdate", "issue_comment.edited"); const supabase = context.adapters.supabase; - await supabase.comment.createComment(STRINGS.HELLO_WORLD, "sasasUpdate", 1, false); + const commentObject = cleanCommentObject(context.payload); + await supabase.comment.createComment(STRINGS.HELLO_WORLD, "sasasUpdate", 1, commentObject, false); await runPlugin(context); const comment = (await supabase.comment.getComment("sasasUpdate")) as unknown as CommentMock; expect(comment).toBeDefined(); @@ -80,7 +83,8 @@ describe("Plugin tests", () => { it("When a comment is deleted it should delete it from the database", async () => { const { context } = createContext("Text Message", 1, 1, 1, 1, "sasasDelete", "issue_comment.deleted"); const supabase = context.adapters.supabase; - await supabase.comment.createComment(STRINGS.HELLO_WORLD, "sasasDelete", 1, false); + const commentObject = cleanCommentObject(context.payload); + await supabase.comment.createComment(STRINGS.HELLO_WORLD, "sasasDelete", 1, commentObject, false); await runPlugin(context); try { await supabase.comment.getComment("sasasDelete"); From e6a47b4e470b1730bb46cc638bb48f8ad6986bc1 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Tue, 10 Sep 2024 23:48:29 -0400 Subject: [PATCH 14/55] fix: updated handling for private repo --- src/adapters/supabase/helpers/comment.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/adapters/supabase/helpers/comment.ts b/src/adapters/supabase/helpers/comment.ts index ed37d48..83f0eb1 100644 --- a/src/adapters/supabase/helpers/comment.ts +++ b/src/adapters/supabase/helpers/comment.ts @@ -35,6 +35,7 @@ export class Comment extends SuperSupabase { } if (isPrivate) { plaintext = null as string | null; + commentobject = null as JSON | null; } const { error } = await this.supabase .from("issue_comments") @@ -47,7 +48,7 @@ export class Comment extends SuperSupabase { this.context.logger.info("Comment created successfully"); } - async updateComment(plaintext: string | null, commentNodeId: string, commentobject: JSON, isPrivate: boolean) { + async updateComment(plaintext: string | null, commentNodeId: string, commentobject: JSON | null, isPrivate: boolean) { //Create the embedding for this comment const embedding = Array.from(await this.context.adapters.voyage.embedding.createEmbedding(plaintext)); if (embedding.length < 3072) { @@ -55,6 +56,7 @@ export class Comment extends SuperSupabase { } if (isPrivate) { plaintext = null as string | null; + commentobject = null as JSON | null; } const { error } = await this.supabase .from("issue_comments") From f8a14213588cf00db44c624ca5cb4365e70a0d1d Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Wed, 11 Sep 2024 01:53:15 -0400 Subject: [PATCH 15/55] fix: test --- src/adapters/supabase/helpers/comment.ts | 8 +- src/adapters/utils/cleancommentobject.ts | 80 +++++++++------ src/handlers/add-comments.ts | 2 +- tests/__mocks__/adapter.ts | 21 ++-- tests/__mocks__/db.ts | 122 +++++++++++++++++------ tests/__mocks__/handlers.ts | 8 -- tests/__mocks__/helpers.ts | 76 ++++++++++++-- tests/main.test.ts | 7 +- 8 files changed, 228 insertions(+), 96 deletions(-) diff --git a/src/adapters/supabase/helpers/comment.ts b/src/adapters/supabase/helpers/comment.ts index 83f0eb1..944f610 100644 --- a/src/adapters/supabase/helpers/comment.ts +++ b/src/adapters/supabase/helpers/comment.ts @@ -16,7 +16,7 @@ export class Comment extends SuperSupabase { super(supabase, context); } - async createComment(plaintext: string | null, commentNodeId: string, authorId: number, commentobject: JSON | null, isPrivate: boolean) { + async createComment(plaintext: string | null, commentNodeId: string, authorId: number, commentobject: Record | null, isPrivate: boolean) { //First Check if the comment already exists const { data, error } = await this.supabase.from("issue_comments").select("*").eq("id", commentNodeId); if (error) { @@ -35,7 +35,7 @@ export class Comment extends SuperSupabase { } if (isPrivate) { plaintext = null as string | null; - commentobject = null as JSON | null; + commentobject = null as Record | null; } const { error } = await this.supabase .from("issue_comments") @@ -48,7 +48,7 @@ export class Comment extends SuperSupabase { this.context.logger.info("Comment created successfully"); } - async updateComment(plaintext: string | null, commentNodeId: string, commentobject: JSON | null, isPrivate: boolean) { + async updateComment(plaintext: string | null, commentNodeId: string, commentobject: Record | null, isPrivate: boolean) { //Create the embedding for this comment const embedding = Array.from(await this.context.adapters.voyage.embedding.createEmbedding(plaintext)); if (embedding.length < 3072) { @@ -56,7 +56,7 @@ export class Comment extends SuperSupabase { } if (isPrivate) { plaintext = null as string | null; - commentobject = null as JSON | null; + commentobject = null as Record | null; } const { error } = await this.supabase .from("issue_comments") diff --git a/src/adapters/utils/cleancommentobject.ts b/src/adapters/utils/cleancommentobject.ts index 715a3ed..7dc0d7e 100644 --- a/src/adapters/utils/cleancommentobject.ts +++ b/src/adapters/utils/cleancommentobject.ts @@ -1,37 +1,53 @@ -import { EmitterWebhookEvent as WebhookEvent } from "@octokit/webhooks"; -import { Type } from "@sinclair/typebox"; - -const commentObjectSchema = Type.Object({ - action: Type.String(), - issue: Type.Object({ - id: Type.Number(), - number: Type.Number(), - title: Type.String(), - body: Type.String(), - user: Type.Object({ - login: Type.String(), - id: Type.Number(), - }), - }), - comment: Type.Object({ - author_association: Type.String(), - id: Type.Number(), - html_url: Type.String(), - issue_url: Type.String(), - user: Type.Object({ - login: Type.String(), - id: Type.Number(), - }), - }), -}); +import { SupportedEvents, SupportedEventsU } from "../../types/context"; /** - * Cleans the comment object. + * Extracts only the properties mentioned in the TypeBox schema from the input object. * * @param commentObject - The comment object. - * @returns The cleaned comment object. + * @returns The object containing only the properties defined in the schema. */ -export const cleanCommentObject = (commentObject: WebhookEvent["payload"]): JSON => { - // Apply the schema to the comment object - return commentObjectSchema.parse(commentObject); -}; +export function cleanCommentObject( + commentObject: TU["payload"] +): Record { + // Manually extract properties + return { + action: commentObject.action as string, + issue: { + id: commentObject.issue.id as number, + number: commentObject.issue.number as number, + title: commentObject.issue.title as string, + body: commentObject.issue.body as string, + user: { + login: commentObject.issue.user.login as string, + id: commentObject.issue.user.id as number, + }, + author_association: commentObject.issue.author_association as string, + }, + comment: { + author_association: commentObject.comment.author_association as string, + id: commentObject.comment.id as number, + html_url: commentObject.comment.html_url as string, + issue_url: commentObject.comment.issue_url as string, + user: { + login: (commentObject.comment.user || { login: "" }).login as string, + id: (commentObject.comment.user || { id: -1 }).id as number, + }, + body: commentObject.comment.body as string, + created_at: commentObject.comment.created_at as string, + updated_at: commentObject.comment.updated_at as string, + }, + + repository: { + id: commentObject.repository.id as number, + node_id: commentObject.repository.node_id as string, + name: commentObject.repository.name as string, + full_name: commentObject.repository.full_name as string, + private: commentObject.repository.private as boolean, + owner: { + login: commentObject.repository.owner.login as string, + id: commentObject.repository.owner.id as number, + avatar_url: commentObject.repository.owner.avatar_url as string, + }, + }, + } as Record; +} diff --git a/src/handlers/add-comments.ts b/src/handlers/add-comments.ts index f572567..f854520 100644 --- a/src/handlers/add-comments.ts +++ b/src/handlers/add-comments.ts @@ -15,7 +15,7 @@ export async function addComments(context: Context) { // Add the comment to the database try { - await supabase.comment.createComment(plaintext, nodeId, authorId, commentobject as JSON, isPrivate); + await supabase.comment.createComment(plaintext, nodeId, authorId, commentobject, isPrivate); } catch (error) { if (error instanceof Error) { logger.error(`Error creating comment:`, { error: error, stack: error.stack }); diff --git a/tests/__mocks__/adapter.ts b/tests/__mocks__/adapter.ts index bb7f4e2..65078ea 100644 --- a/tests/__mocks__/adapter.ts +++ b/tests/__mocks__/adapter.ts @@ -7,6 +7,7 @@ export interface CommentMock { id: string; plaintext: string | null; author_id: number; + commentobject?: Record | null; embedding: number[]; } @@ -15,15 +16,19 @@ export function createMockAdapters(context: Context) { return { supabase: { comment: { - createComment: jest.fn(async (plaintext: string | null, commentNodeId: string, authorId: number) => { - if (commentMap.has(commentNodeId)) { - throw new Error("Comment already exists"); + createComment: jest.fn( + async (plaintext: string | null, commentNodeId: string, authorId: number, commentObject: Record | null, isPrivate: boolean) => { + if (commentMap.has(commentNodeId)) { + throw new Error("Comment already exists"); + } + const embedding = await context.adapters.openai.embedding.createEmbedding(plaintext); + if (isPrivate) { + plaintext = null; + } + commentMap.set(commentNodeId, { id: commentNodeId, plaintext, author_id: authorId, embedding }); } - const embedding = await context.adapters.openai.embedding.createEmbedding(plaintext); - commentMap.set(commentNodeId, { id: commentNodeId, plaintext, author_id: authorId, embedding }); - }), - updateComment: jest.fn(async (plaintext: string | null, commentNodeId: string, isPrivate: boolean) => { - console.log(commentMap); + ), + updateComment: jest.fn(async (plaintext: string | null, commentNodeId: string, commentObject: Record | null, isPrivate: boolean) => { if (!commentMap.has(commentNodeId)) { throw new Error(STRINGS.COMMENT_DOES_NOT_EXIST); } diff --git a/tests/__mocks__/db.ts b/tests/__mocks__/db.ts index e1883e5..1681106 100644 --- a/tests/__mocks__/db.ts +++ b/tests/__mocks__/db.ts @@ -7,36 +7,82 @@ import { factory, nullable, primaryKey } from "@mswjs/data"; export const db = factory({ users: { id: primaryKey(Number), - name: String, login: String, + avatar_url: nullable(String), // Add any additional fields based on the schema }, - issue: { + repo: { id: primaryKey(Number), - assignees: Array, + name: String, + full_name: String, // Assuming full_name is needed for repo + private: Boolean, + owner: { + login: String, + id: Number, + avatar_url: nullable(String), + }, html_url: String, - repository_url: String, - state: String, + description: nullable(String), + fork: Boolean, + url: String, + forks_url: String, + keys_url: String, + collaborators_url: String, + teams_url: String, + hooks_url: String, + issue_events_url: String, + events_url: String, + assignees_url: String, + branches_url: String, + tags_url: String, + blobs_url: String, + git_tags_url: String, + git_refs_url: String, + trees_url: String, + statuses_url: String, + languages_url: String, + stargazers_url: String, + contributors_url: String, + subscribers_url: String, + subscription_url: String, + commits_url: String, + git_commits_url: String, + comments_url: String, + issue_comment_url: String, + contents_url: String, + compare_url: String, + merges_url: String, + archive_url: String, + downloads_url: String, + issues_url: String, + pulls_url: String, + milestones_url: String, + notifications_url: String, + labels_url: String, + releases_url: String, + deployments_url: String, + }, + issue: { + id: primaryKey(Number), + number: Number, + title: String, + body: String, + user: { + login: String, + id: Number, + }, owner: String, repo: String, - labels: Array, author_association: String, - body: nullable(String), - closed_at: nullable(Date), - created_at: nullable(Date), + created_at: Date, + updated_at: Date, comments: Number, - comments_url: String, - events_url: String, - labels_url: String, + labels: Array, + state: String, locked: Boolean, - node_id: String, - title: String, - number: Number, - updated_at: Date, - url: String, - user: nullable(Object), - milestone: nullable(Object), assignee: nullable({ - avatar_url: String, + login: String, + id: Number, + avatar_url: nullable(String), email: nullable(String), events_url: String, followers_url: String, @@ -44,8 +90,6 @@ export const db = factory({ gists_url: String, gravatar_id: nullable(String), html_url: String, - id: Number, - login: String, name: nullable(String), node_id: String, organizations_url: String, @@ -58,17 +102,28 @@ export const db = factory({ type: String, url: String, }), - }, - repo: { - id: primaryKey(Number), - html_url: String, - name: String, - labels: Array, - owner: { - login: String, - id: Number, + milestone: nullable({ + title: String, + description: nullable(String), + due_on: nullable(Date), + number: Number, + state: String, + }), + reactions: { + url: String, + total_count: Number, + "+1": Number, + "-1": Number, + laugh: Number, + hooray: Number, + confused: Number, + heart: Number, + rocket: Number, + eyes: Number, }, - issues: Array, + timeline_url: String, + performed_via_github_app: nullable(Object), + state_reason: nullable(String), }, issueComments: { id: primaryKey(Number), @@ -81,5 +136,8 @@ export const db = factory({ login: String, id: Number, }, + author_association: String, + html_url: String, + issue_url: String, }, }); diff --git a/tests/__mocks__/handlers.ts b/tests/__mocks__/handlers.ts index f1c494a..0019673 100644 --- a/tests/__mocks__/handlers.ts +++ b/tests/__mocks__/handlers.ts @@ -1,6 +1,5 @@ import { http, HttpResponse } from "msw"; import { db } from "./db"; -import issueTemplate from "./issue-template"; /** * Intercepts the routes and returns a custom payload */ @@ -31,13 +30,6 @@ export const handlers = [ } return HttpResponse.json(item); }), - // create issue - http.post("https://api.github.com/repos/:owner/:repo/issues", () => { - const id = db.issue.count() + 1; - const newItem = { ...issueTemplate, id }; - db.issue.create(newItem); - return HttpResponse.json(newItem); - }), // create comment http.post("https://api.github.com/repos/:owner/:repo/issues/:issue_number/comments", async ({ params: { issue_number: issueNumber }, request }) => { const { body } = await getValue(request.body); diff --git a/tests/__mocks__/helpers.ts b/tests/__mocks__/helpers.ts index eb9e2ae..236ac8e 100644 --- a/tests/__mocks__/helpers.ts +++ b/tests/__mocks__/helpers.ts @@ -1,5 +1,4 @@ import { db } from "./db"; -import issueTemplate from "./issue-template"; import { STRINGS } from "./strings"; import usersGet from "./users-get.json"; @@ -12,34 +11,93 @@ import usersGet from "./users-get.json"; * Here is where you create issues, commits, pull requests, etc. */ export async function setupTests() { + // Insert users for (const item of usersGet) { - db.users.create(item); + db.users.create({ + login: item.login, + id: item.id, + }); } + // Insert repository db.repo.create({ id: 1, name: STRINGS.TEST_REPO, + full_name: `${STRINGS.USER_1}/${STRINGS.TEST_REPO}`, + private: false, owner: { login: STRINGS.USER_1, id: 1, + avatar_url: "", }, - issues: [], }); + // Insert issues db.issue.create({ - ...issueTemplate, + id: 1, + number: 1, + title: "First Issue", + body: "This is the body of the first issue.", + user: { + login: STRINGS.USER_1, + id: 1, + }, + author_association: "OWNER", + created_at: new Date().toISOString(), + updated_at: new Date().toISOString(), + comments: 0, + labels: [], + state: "open", + locked: false, + reactions: { + url: "", + total_count: 0, + "+1": 0, + "-1": 0, + laugh: 0, + hooray: 0, + confused: 0, + heart: 0, + rocket: 0, + eyes: 0, + }, + timeline_url: "", }); db.issue.create({ - ...issueTemplate, id: 2, number: 2, + title: "Second Issue", + body: "This is the body of the second issue.", + user: { + login: STRINGS.USER_1, + id: 1, + }, + author_association: "OWNER", + created_at: new Date().toISOString(), + updated_at: new Date().toISOString(), + comments: 0, labels: [], + state: "open", + locked: false, + reactions: { + url: "", + total_count: 0, + "+1": 0, + "-1": 0, + laugh: 0, + hooray: 0, + confused: 0, + heart: 0, + rocket: 0, + eyes: 0, + }, + timeline_url: "", }); } export function createComment(comment: string, commentId: number, nodeId: string) { - const isComment = db.issueComments.findFirst({ + const existingComment = db.issueComments.findFirst({ where: { id: { equals: commentId, @@ -47,7 +105,7 @@ export function createComment(comment: string, commentId: number, nodeId: string }, }); - if (isComment) { + if (existingComment) { db.issueComments.update({ where: { id: { @@ -56,6 +114,7 @@ export function createComment(comment: string, commentId: number, nodeId: string }, data: { body: comment, + updated_at: new Date().toISOString(), }, }); } else { @@ -68,6 +127,9 @@ export function createComment(comment: string, commentId: number, nodeId: string login: STRINGS.USER_1, id: 1, }, + created_at: new Date().toISOString(), + updated_at: new Date().toISOString(), + author_association: "OWNER", }); } } diff --git a/tests/main.test.ts b/tests/main.test.ts index 384d95a..4e47dd9 100644 --- a/tests/main.test.ts +++ b/tests/main.test.ts @@ -14,7 +14,6 @@ import { Logs } from "@ubiquity-dao/ubiquibot-logger"; import { Env } from "../src/types"; import { runPlugin } from "../src/plugin"; import { CommentMock, createMockAdapters } from "./__mocks__/adapter"; -import { cleanCommentObject } from "../src/adapters/utils/cleancommentobject"; dotenv.config(); jest.requireActual("@octokit/rest"); @@ -53,7 +52,7 @@ describe("Plugin tests", () => { const { context } = createContext(STRINGS.HELLO_WORLD, 1, 1, 1, 1, "sasasCreate"); await runPlugin(context); const supabase = context.adapters.supabase; - const commentObject = cleanCommentObject(context.payload); + const commentObject = null; try { await supabase.comment.createComment(STRINGS.HELLO_WORLD, "sasasCreate", 1, commentObject, false); throw new Error("Expected method to reject."); @@ -71,7 +70,7 @@ describe("Plugin tests", () => { it("When a comment is updated it should update the database", async () => { const { context } = createContext("Updated Message", 1, 1, 1, 1, "sasasUpdate", "issue_comment.edited"); const supabase = context.adapters.supabase; - const commentObject = cleanCommentObject(context.payload); + const commentObject = null; await supabase.comment.createComment(STRINGS.HELLO_WORLD, "sasasUpdate", 1, commentObject, false); await runPlugin(context); const comment = (await supabase.comment.getComment("sasasUpdate")) as unknown as CommentMock; @@ -83,7 +82,7 @@ describe("Plugin tests", () => { it("When a comment is deleted it should delete it from the database", async () => { const { context } = createContext("Text Message", 1, 1, 1, 1, "sasasDelete", "issue_comment.deleted"); const supabase = context.adapters.supabase; - const commentObject = cleanCommentObject(context.payload); + const commentObject = null; await supabase.comment.createComment(STRINGS.HELLO_WORLD, "sasasDelete", 1, commentObject, false); await runPlugin(context); try { From bccfe2388fb7270d52db0801bd6726a1349ef550 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Wed, 11 Sep 2024 01:54:29 -0400 Subject: [PATCH 16/55] fix: cspell --- .cspell.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.cspell.json b/.cspell.json index b297a90..2ae22ce 100644 --- a/.cspell.json +++ b/.cspell.json @@ -23,7 +23,9 @@ "mischeck", "commentbody", "issuebody", - "voyageai" + "voyageai", + "cleancommentobject", + "commentobject" ], "dictionaries": ["typescript", "node", "software-terms"], "import": ["@cspell/dict-typescript/cspell-ext.json", "@cspell/dict-node/cspell-ext.json", "@cspell/dict-software-terms"], From b6a8c7cce1ba4978757ecf13b887a7c8e87cc18e Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Wed, 11 Sep 2024 07:06:34 -0400 Subject: [PATCH 17/55] fix: cspell, removed openai, max length of vectors is 1024' --- .cspell.json | 2 - .dev.vars.example | 2 +- .env.example | 2 +- .github/workflows/compute.yml | 3 +- README.md | 1 - package.json | 3 +- src/adapters/index.ts | 9 +-- src/adapters/openai/helpers/embedding.ts | 29 -------- src/adapters/openai/helpers/openai.ts | 12 ---- src/adapters/supabase/helpers/comment.ts | 7 -- src/adapters/voyage/helpers/embedding.ts | 2 +- src/handlers/add-comments.ts | 6 +- src/handlers/update-comments.ts | 6 +- src/plugin.ts | 6 +- src/types/env.ts | 1 - .../20240911023641_issue_comments.sql | 2 +- .../20240911104515_issue_comments.sql | 5 ++ tests/__mocks__/adapter.ts | 11 ++- tests/main.test.ts | 2 - yarn.lock | 69 +------------------ 20 files changed, 26 insertions(+), 154 deletions(-) delete mode 100644 src/adapters/openai/helpers/embedding.ts delete mode 100644 src/adapters/openai/helpers/openai.ts create mode 100644 supabase/migrations/20240911104515_issue_comments.sql diff --git a/.cspell.json b/.cspell.json index 2ae22ce..e64757f 100644 --- a/.cspell.json +++ b/.cspell.json @@ -24,8 +24,6 @@ "commentbody", "issuebody", "voyageai", - "cleancommentobject", - "commentobject" ], "dictionaries": ["typescript", "node", "software-terms"], "import": ["@cspell/dict-typescript/cspell-ext.json", "@cspell/dict-node/cspell-ext.json", "@cspell/dict-software-terms"], diff --git a/.dev.vars.example b/.dev.vars.example index 6ceaeff..2fc8cf8 100644 --- a/.dev.vars.example +++ b/.dev.vars.example @@ -1,3 +1,3 @@ SUPABASE_URL="" SUPABASE_KEY="" -OPENAI_API_KEY="" \ No newline at end of file +VOYAGEAI_API_KEY="" \ No newline at end of file diff --git a/.env.example b/.env.example index 0272714..6e8b18f 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,3 @@ SUPABASE_URL= SUPABASE_KEY= -OPENAI_API_KEY= \ No newline at end of file +VOYAGEAI_API_KEY= \ No newline at end of file diff --git a/.github/workflows/compute.yml b/.github/workflows/compute.yml index 10678ce..8cb2597 100644 --- a/.github/workflows/compute.yml +++ b/.github/workflows/compute.yml @@ -24,7 +24,6 @@ jobs: env: SUPABASE_URL: ${{ secrets.SUPABASE_URL }} SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }} - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} VOYAGEAI_API_KEY: ${{secrets.VOYAGEAI_API_KEY}} steps: @@ -44,4 +43,4 @@ jobs: env: SUPABASE_URL: ${{ secrets.SUPABASE_URL }} SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }} - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + VOYAGEAI_API_KEY: ${{secrets.VOYAGEAI_API_KEY}} diff --git a/README.md b/README.md index 29cc9a0..6680b55 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,6 @@ This is a plugin for [Ubiquibot](https://github.com/ubiquity/ubiquibot-kernel). To set up the `.dev.vars` file, you will need to provide the following variables: - `SUPABASE_URL`: The URL for your Supabase instance. - `SUPABASE_KEY`: The key for your Supabase instance. -- `OPENAI_API_KEY`: The API key for OpenAI. - `VOYAGEAI_API_KEY`: The API key for Voyage. ## Usage diff --git a/package.json b/package.json index 27277b0..80f0dbf 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "knip-ci": "knip --no-exit-code --reporter json --config .github/knip.ts", "prepare": "husky install", "test": "jest --setupFiles dotenv/config --coverage", - "worker": "wrangler dev --env dev --port 5000", + "worker": "wrangler dev --env dev --port 4000", "supabase:generate:local": "supabase gen types typescript --local > src/types/database.ts", "supabase:generate:remote": "cross-env-shell \"supabase gen types typescript --project-id $SUPABASE_PROJECT_ID --schema public > src/types/database.ts\"" }, @@ -37,7 +37,6 @@ "@supabase/supabase-js": "^2.45.2", "@ubiquity-dao/ubiquibot-logger": "^1.3.0", "dotenv": "16.4.5", - "openai": "^4.56.0", "typebox-validators": "0.3.5", "voyageai": "^0.0.1-5" }, diff --git a/src/adapters/index.ts b/src/adapters/index.ts index 0e5986c..29c78a2 100644 --- a/src/adapters/index.ts +++ b/src/adapters/index.ts @@ -2,23 +2,16 @@ import { SupabaseClient } from "@supabase/supabase-js"; import { Context } from "../types"; import { Comment } from "./supabase/helpers/comment"; import { SuperSupabase } from "./supabase/helpers/supabase"; -import { SuperOpenAi } from "./openai/helpers/openai"; -import OpenAI from "openai"; -import { Embedding as OpenAiEmbedding } from "./openai/helpers/embedding"; import { Embedding as VoyageEmbedding } from "./voyage/helpers/embedding"; import { SuperVoyage } from "./voyage/helpers/voyage"; import { VoyageAIClient } from "voyageai"; -export function createAdapters(supabaseClient: SupabaseClient, openai: OpenAI, voyage: VoyageAIClient, context: Context) { +export function createAdapters(supabaseClient: SupabaseClient, voyage: VoyageAIClient, context: Context) { return { supabase: { comment: new Comment(supabaseClient, context), super: new SuperSupabase(supabaseClient, context), }, - openai: { - embedding: new OpenAiEmbedding(openai, context), - super: new SuperOpenAi(openai, context), - }, voyage: { embedding: new VoyageEmbedding(voyage, context), super: new SuperVoyage(voyage, context), diff --git a/src/adapters/openai/helpers/embedding.ts b/src/adapters/openai/helpers/embedding.ts deleted file mode 100644 index 3dfaeb9..0000000 --- a/src/adapters/openai/helpers/embedding.ts +++ /dev/null @@ -1,29 +0,0 @@ -import OpenAI from "openai"; -import { Context } from "../../../types"; -import { SuperOpenAi } from "./openai"; -const VECTOR_SIZE = 3072; - -export class Embedding extends SuperOpenAi { - protected context: Context; - - constructor(client: OpenAI, context: Context) { - super(client, context); - this.context = context; - } - - async createEmbedding(text: string | null): Promise { - if (text === null) { - return new Array(VECTOR_SIZE).fill(0); - } else { - const params: OpenAI.EmbeddingCreateParams = { - model: "text-embedding-3-large", - input: text, - dimensions: VECTOR_SIZE, - }; - const response = await this.client.embeddings.create({ - ...params, - }); - return response.data[0]?.embedding; - } - } -} diff --git a/src/adapters/openai/helpers/openai.ts b/src/adapters/openai/helpers/openai.ts deleted file mode 100644 index 108a3d2..0000000 --- a/src/adapters/openai/helpers/openai.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { OpenAI } from "openai"; -import { Context } from "../../../types/context"; - -export class SuperOpenAi { - protected client: OpenAI; - protected context: Context; - - constructor(client: OpenAI, context: Context) { - this.client = client; - this.context = context; - } -} diff --git a/src/adapters/supabase/helpers/comment.ts b/src/adapters/supabase/helpers/comment.ts index 944f610..291a039 100644 --- a/src/adapters/supabase/helpers/comment.ts +++ b/src/adapters/supabase/helpers/comment.ts @@ -29,10 +29,6 @@ export class Comment extends SuperSupabase { } else { //Create the embedding for this comment const embedding = await this.context.adapters.voyage.embedding.createEmbedding(plaintext); - //If embedding is smaller than 3072, pad it with 0s - if (embedding.length < 3072) { - embedding.push(...new Array(3072 - embedding.length).fill(0)); - } if (isPrivate) { plaintext = null as string | null; commentobject = null as Record | null; @@ -51,9 +47,6 @@ export class Comment extends SuperSupabase { async updateComment(plaintext: string | null, commentNodeId: string, commentobject: Record | null, isPrivate: boolean) { //Create the embedding for this comment const embedding = Array.from(await this.context.adapters.voyage.embedding.createEmbedding(plaintext)); - if (embedding.length < 3072) { - embedding.push(...new Array(3072 - embedding.length).fill(0)); - } if (isPrivate) { plaintext = null as string | null; commentobject = null as Record | null; diff --git a/src/adapters/voyage/helpers/embedding.ts b/src/adapters/voyage/helpers/embedding.ts index 19d29ce..cacc320 100644 --- a/src/adapters/voyage/helpers/embedding.ts +++ b/src/adapters/voyage/helpers/embedding.ts @@ -1,7 +1,7 @@ import { VoyageAIClient } from "voyageai"; import { Context } from "../../../types"; import { SuperVoyage } from "./voyage"; -const VECTOR_SIZE = 3072; +const VECTOR_SIZE = 1024; export class Embedding extends SuperVoyage { protected context: Context; diff --git a/src/handlers/add-comments.ts b/src/handlers/add-comments.ts index f854520..52ed0a5 100644 --- a/src/handlers/add-comments.ts +++ b/src/handlers/add-comments.ts @@ -1,4 +1,4 @@ -import { cleanCommentObject } from "../adapters/utils/cleancommentobject"; +import { cleanCommentObject } from "../adapters/utils/cleanCommentObject"; import { Context } from "../types"; export async function addComments(context: Context) { @@ -7,7 +7,7 @@ export async function addComments(context: Context) { payload, adapters: { supabase }, } = context; - const commentobject = cleanCommentObject(payload); + const commentObject = cleanCommentObject(payload); const plaintext = payload.comment.body; const authorId = payload.comment.user?.id || -1; const nodeId = payload.comment.node_id; @@ -15,7 +15,7 @@ export async function addComments(context: Context) { // Add the comment to the database try { - await supabase.comment.createComment(plaintext, nodeId, authorId, commentobject, isPrivate); + await supabase.comment.createComment(plaintext, nodeId, authorId, commentObject, isPrivate); } catch (error) { if (error instanceof Error) { logger.error(`Error creating comment:`, { error: error, stack: error.stack }); diff --git a/src/handlers/update-comments.ts b/src/handlers/update-comments.ts index cf85a9d..9461497 100644 --- a/src/handlers/update-comments.ts +++ b/src/handlers/update-comments.ts @@ -1,4 +1,4 @@ -import { cleanCommentObject } from "../adapters/utils/cleancommentobject"; +import { cleanCommentObject } from "../adapters/utils/cleanCommentObject"; import { Context } from "../types"; export async function updateComment(context: Context) { @@ -7,13 +7,13 @@ export async function updateComment(context: Context) { payload, adapters: { supabase }, } = context; - const commentobject = cleanCommentObject(payload); + const commentObject = cleanCommentObject(payload); const nodeId = payload.comment.node_id; const isPrivate = payload.repository.private; const plaintext = payload.comment.body; // Fetch the previous comment and update it in the db try { - await supabase.comment.updateComment(plaintext, nodeId, commentobject, isPrivate); + await supabase.comment.updateComment(plaintext, nodeId, commentObject, isPrivate); } catch (error) { if (error instanceof Error) { logger.error(`Error updating comment:`, { error: error, stack: error.stack }); diff --git a/src/plugin.ts b/src/plugin.ts index 0b8de96..9673231 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -9,7 +9,6 @@ import { createClient } from "@supabase/supabase-js"; import { addComments } from "./handlers/add-comments"; import { updateComment } from "./handlers/update-comments"; import { deleteComment } from "./handlers/delete-comments"; -import OpenAI from "openai"; import { VoyageAIClient } from "voyageai"; /** @@ -38,9 +37,6 @@ export async function runPlugin(context: Context) { export async function plugin(inputs: PluginInputs, env: Env) { const octokit = new Octokit({ auth: inputs.authToken }); const supabase = createClient(env.SUPABASE_URL, env.SUPABASE_KEY); - const openaiClient = new OpenAI({ - apiKey: env.OPENAI_API_KEY, - }); const voyageClient = new VoyageAIClient({ apiKey: env.VOYAGEAI_API_KEY, }); @@ -53,6 +49,6 @@ export async function plugin(inputs: PluginInputs, env: Env) { logger: new Logs("info" as LogLevel), adapters: {} as ReturnType, }; - context.adapters = createAdapters(supabase, openaiClient, voyageClient, context); + context.adapters = createAdapters(supabase, voyageClient, context); return runPlugin(context); } diff --git a/src/types/env.ts b/src/types/env.ts index 1fe1588..4c6cc1c 100644 --- a/src/types/env.ts +++ b/src/types/env.ts @@ -13,7 +13,6 @@ import { StandardValidator } from "typebox-validators"; export const envSchema = T.Object({ SUPABASE_URL: T.String(), SUPABASE_KEY: T.String(), - OPENAI_API_KEY: T.String(), VOYAGEAI_API_KEY: T.String(), }); diff --git a/supabase/migrations/20240911023641_issue_comments.sql b/supabase/migrations/20240911023641_issue_comments.sql index 06f28b5..4ef319b 100644 --- a/supabase/migrations/20240911023641_issue_comments.sql +++ b/supabase/migrations/20240911023641_issue_comments.sql @@ -1,2 +1,2 @@ ALTER TABLE issue_comments -ADD COLUMN commentobject jsonb; \ No newline at end of file +ADD COLUMN commentObject jsonb; \ No newline at end of file diff --git a/supabase/migrations/20240911104515_issue_comments.sql b/supabase/migrations/20240911104515_issue_comments.sql new file mode 100644 index 0000000..b0b2904 --- /dev/null +++ b/supabase/migrations/20240911104515_issue_comments.sql @@ -0,0 +1,5 @@ +ALTER TABLE issue_comments +DROP COLUMN embedding; + +ALTER TABLE issue_comments +ADD COLUMN embedding Vector(1024) NOT NULL; \ No newline at end of file diff --git a/tests/__mocks__/adapter.ts b/tests/__mocks__/adapter.ts index 65078ea..ee9956f 100644 --- a/tests/__mocks__/adapter.ts +++ b/tests/__mocks__/adapter.ts @@ -1,13 +1,12 @@ import { Context } from "../../src/types"; import { Comment } from "../../src/adapters/supabase/helpers/comment"; -import { Embedding } from "../../src/adapters/openai/helpers/embedding"; import { STRINGS } from "./strings"; export interface CommentMock { id: string; plaintext: string | null; author_id: number; - commentobject?: Record | null; + commentObject?: Record | null; embedding: number[]; } @@ -21,7 +20,7 @@ export function createMockAdapters(context: Context) { if (commentMap.has(commentNodeId)) { throw new Error("Comment already exists"); } - const embedding = await context.adapters.openai.embedding.createEmbedding(plaintext); + const embedding = await context.adapters.voyage.embedding.createEmbedding(plaintext); if (isPrivate) { plaintext = null; } @@ -37,7 +36,7 @@ export function createMockAdapters(context: Context) { throw new Error(STRINGS.COMMENT_DOES_NOT_EXIST); } const { id, author_id } = originalComment; - const embedding = await context.adapters.openai.embedding.createEmbedding(plaintext); + const embedding = await context.adapters.voyage.embedding.createEmbedding(plaintext); if (isPrivate) { plaintext = null; } @@ -57,7 +56,7 @@ export function createMockAdapters(context: Context) { }), } as unknown as Comment, }, - openai: { + voyage: { embedding: { createEmbedding: jest.fn(async (text: string) => { if (text && text.length > 0) { @@ -65,7 +64,7 @@ export function createMockAdapters(context: Context) { } return new Array(3072).fill(0); }), - } as unknown as Embedding, + } as unknown as number[], }, }; } diff --git a/tests/main.test.ts b/tests/main.test.ts index 4e47dd9..5f4fd84 100644 --- a/tests/main.test.ts +++ b/tests/main.test.ts @@ -18,7 +18,6 @@ import { CommentMock, createMockAdapters } from "./__mocks__/adapter"; dotenv.config(); jest.requireActual("@octokit/rest"); jest.requireActual("@supabase/supabase-js"); -jest.requireActual("openai"); const octokit = new Octokit(); beforeAll(() => { @@ -41,7 +40,6 @@ describe("Plugin tests", () => { const response = await worker.fetch(new Request("http://localhost/manifest.json"), { SUPABASE_KEY: "test", SUPABASE_URL: "test", - OPENAI_API_KEY: "test", VOYAGEAI_API_KEY: "test", }); const content = await response.json(); diff --git a/yarn.lock b/yarn.lock index b483a55..e653515 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1978,14 +1978,6 @@ dependencies: "@types/node" "*" -"@types/node-fetch@^2.6.4": - version "2.6.11" - resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.11.tgz#9b39b78665dae0e82a08f02f4967d62c66f95d24" - integrity sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g== - dependencies: - "@types/node" "*" - form-data "^4.0.0" - "@types/node-forge@^1.3.0": version "1.3.11" resolved "https://registry.yarnpkg.com/@types/node-forge/-/node-forge-1.3.11.tgz#0972ea538ddb0f4d9c2fa0ec5db5724773a604da" @@ -2007,13 +1999,6 @@ dependencies: undici-types "~5.26.4" -"@types/node@^18.11.18": - version "18.19.46" - resolved "https://registry.yarnpkg.com/@types/node/-/node-18.19.46.tgz#51801396c01153e0626e36f43386e83bc768b072" - integrity sha512-vnRgMS7W6cKa1/0G3/DTtQYpVrZ8c0Xm6UkLaVFrb9jtcVC3okokW09Ki1Qdrj9ISokszD69nY4WDLRlvHlhAA== - dependencies: - undici-types "~5.26.4" - "@types/phoenix@^1.5.4": version "1.6.5" resolved "https://registry.yarnpkg.com/@types/phoenix/-/phoenix-1.6.5.tgz#5654e14ec7ad25334a157a20015996b6d7d2075e" @@ -2191,13 +2176,6 @@ agent-base@^7.0.2: dependencies: debug "^4.3.4" -agentkeepalive@^4.2.1: - version "4.5.0" - resolved "https://registry.yarnpkg.com/agentkeepalive/-/agentkeepalive-4.5.0.tgz#2673ad1389b3c418c5a20c5d7364f93ca04be923" - integrity sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew== - dependencies: - humanize-ms "^1.2.1" - aggregate-error@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/aggregate-error/-/aggregate-error-3.1.0.tgz#92670ff50f5359bdb7a3e0d40d0ec30c5737687a" @@ -3739,11 +3717,6 @@ foreground-child@^3.1.0: cross-spawn "^7.0.0" signal-exit "^4.0.1" -form-data-encoder@1.7.2: - version "1.7.2" - resolved "https://registry.yarnpkg.com/form-data-encoder/-/form-data-encoder-1.7.2.tgz#1f1ae3dccf58ed4690b86d87e4f57c654fbab040" - integrity sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A== - form-data@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452" @@ -3753,14 +3726,6 @@ form-data@^4.0.0: combined-stream "^1.0.8" mime-types "^2.1.12" -formdata-node@^4.3.2: - version "4.4.1" - resolved "https://registry.yarnpkg.com/formdata-node/-/formdata-node-4.4.1.tgz#23f6a5cb9cb55315912cbec4ff7b0f59bbd191e2" - integrity sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ== - dependencies: - node-domexception "1.0.0" - web-streams-polyfill "4.0.0-beta.3" - formdata-node@^6.0.3: version "6.0.3" resolved "https://registry.yarnpkg.com/formdata-node/-/formdata-node-6.0.3.tgz#48f8e2206ae2befded82af621ef015f08168dc6d" @@ -4078,13 +4043,6 @@ human-signals@^5.0.0: resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-5.0.0.tgz#42665a284f9ae0dade3ba41ebc37eb4b852f3a28" integrity sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ== -humanize-ms@^1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/humanize-ms/-/humanize-ms-1.2.1.tgz#c46e3159a293f6b896da29316d8b6fe8bb79bbed" - integrity sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ== - dependencies: - ms "^2.0.0" - husky@9.0.11: version "9.0.11" resolved "https://registry.yarnpkg.com/husky/-/husky-9.0.11.tgz#fc91df4c756050de41b3e478b2158b87c1e79af9" @@ -5300,11 +5258,6 @@ ms@2.1.2: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== -ms@^2.0.0: - version "2.1.3" - resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" - integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== - msw@^2.0.8: version "2.3.1" resolved "https://registry.yarnpkg.com/msw/-/msw-2.3.1.tgz#bfc73e256ffc2c74ec4381b604abb258df35f32b" @@ -5353,7 +5306,7 @@ nice-try@^1.0.4: resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366" integrity sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ== -node-domexception@1.0.0, node-domexception@^1.0.0: +node-domexception@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5" integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ== @@ -5363,7 +5316,7 @@ node-fetch-native@^1.6.4: resolved "https://registry.yarnpkg.com/node-fetch-native/-/node-fetch-native-1.6.4.tgz#679fc8fd8111266d47d7e72c379f1bed9acff06e" integrity sha512-IhOigYzAKHd244OC0JIMIUrjzctirCmPkaIfhDeGcEETWof5zKYUW7e7MYvChGWh/4CJeXEgsRyGzuF334rOOQ== -node-fetch@2.7.0, node-fetch@^2.6.7: +node-fetch@2.7.0: version "2.7.0" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== @@ -5494,19 +5447,6 @@ onetime@^6.0.0: dependencies: mimic-fn "^4.0.0" -openai@^4.56.0: - version "4.56.0" - resolved "https://registry.yarnpkg.com/openai/-/openai-4.56.0.tgz#07d3982544cabd5781127288a8dfcceb7319a4cf" - integrity sha512-zcag97+3bG890MNNa0DQD9dGmmTWL8unJdNkulZzWRXrl+QeD+YkBI4H58rJcwErxqGK6a0jVPZ4ReJjhDGcmw== - dependencies: - "@types/node" "^18.11.18" - "@types/node-fetch" "^2.6.4" - abort-controller "^3.0.0" - agentkeepalive "^4.2.1" - form-data-encoder "1.7.2" - formdata-node "^4.3.2" - node-fetch "^2.6.7" - optionator@^0.9.3: version "0.9.4" resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.9.4.tgz#7ea1c1a5d91d764fb282139c88fe11e182a3a734" @@ -6806,11 +6746,6 @@ wcwidth@^1.0.1: dependencies: defaults "^1.0.3" -web-streams-polyfill@4.0.0-beta.3: - version "4.0.0-beta.3" - resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz#2898486b74f5156095e473efe989dcf185047a38" - integrity sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug== - web-streams-polyfill@^3.0.3: version "3.3.3" resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz#2073b91a2fdb1fbfbd401e7de0ac9f8214cecb4b" From 73d0a5a9c00577ba65af8f1bd3704717f6ea4cc5 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Wed, 11 Sep 2024 07:10:57 -0400 Subject: [PATCH 18/55] fix: module import error --- src/handlers/add-comments.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/handlers/add-comments.ts b/src/handlers/add-comments.ts index 52ed0a5..d082ddf 100644 --- a/src/handlers/add-comments.ts +++ b/src/handlers/add-comments.ts @@ -13,7 +13,6 @@ export async function addComments(context: Context) { const nodeId = payload.comment.node_id; const isPrivate = payload.repository.private; - // Add the comment to the database try { await supabase.comment.createComment(plaintext, nodeId, authorId, commentObject, isPrivate); } catch (error) { From 4679fe3822ac3abd43490f94d890b9f2b0488121 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Wed, 11 Sep 2024 07:15:51 -0400 Subject: [PATCH 19/55] fix: test --- .../utils/{cleancommentobject.ts => comment-object-clean.ts} | 0 src/handlers/add-comments.ts | 2 +- src/handlers/update-comments.ts | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename src/adapters/utils/{cleancommentobject.ts => comment-object-clean.ts} (100%) diff --git a/src/adapters/utils/cleancommentobject.ts b/src/adapters/utils/comment-object-clean.ts similarity index 100% rename from src/adapters/utils/cleancommentobject.ts rename to src/adapters/utils/comment-object-clean.ts diff --git a/src/handlers/add-comments.ts b/src/handlers/add-comments.ts index d082ddf..62486d4 100644 --- a/src/handlers/add-comments.ts +++ b/src/handlers/add-comments.ts @@ -1,4 +1,4 @@ -import { cleanCommentObject } from "../adapters/utils/cleanCommentObject"; +import { cleanCommentObject } from "../adapters/utils/comment-object-clean"; import { Context } from "../types"; export async function addComments(context: Context) { diff --git a/src/handlers/update-comments.ts b/src/handlers/update-comments.ts index 9461497..5ae6b46 100644 --- a/src/handlers/update-comments.ts +++ b/src/handlers/update-comments.ts @@ -1,4 +1,4 @@ -import { cleanCommentObject } from "../adapters/utils/cleanCommentObject"; +import { cleanCommentObject } from "../adapters/utils/comment-object-clean"; import { Context } from "../types"; export async function updateComment(context: Context) { From 6d1c52197fb8005e39904dcb1cf029c40cef1152 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Wed, 11 Sep 2024 21:31:46 -0400 Subject: [PATCH 20/55] feat: issue dedup --- .github/.ubiquibot-config.yml | 2 +- README.md | 2 +- manifest.json | 2 +- package.json | 2 +- src/adapters/index.ts | 2 + src/adapters/supabase/helpers/comment.ts | 22 ++-- src/adapters/supabase/helpers/issues.ts | 85 +++++++++++++ src/handlers/add-issue.ts | 29 +++++ src/handlers/delete-issues.ts | 26 ++++ src/handlers/issue-deduplication.ts | 113 ++++++++++++++++++ src/handlers/update-issue.ts | 28 +++++ src/plugin.ts | 10 ++ src/types/context.ts | 2 +- src/types/typeguards.ts | 13 +- .../20240911233824_issue_comments.sql | 8 ++ 15 files changed, 328 insertions(+), 18 deletions(-) create mode 100644 src/adapters/supabase/helpers/issues.ts create mode 100644 src/handlers/add-issue.ts create mode 100644 src/handlers/delete-issues.ts create mode 100644 src/handlers/issue-deduplication.ts create mode 100644 src/handlers/update-issue.ts create mode 100644 supabase/migrations/20240911233824_issue_comments.sql diff --git a/.github/.ubiquibot-config.yml b/.github/.ubiquibot-config.yml index 8a9597c..9213535 100644 --- a/.github/.ubiquibot-config.yml +++ b/.github/.ubiquibot-config.yml @@ -3,4 +3,4 @@ plugins: id: test-app uses: - plugin: https://ubiquity-os-comment-vector-embeddings.sshivaditya.workers.dev - runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" ] + runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issue.created", "issue.edited", "issue.deleted" ] diff --git a/README.md b/README.md index 6680b55..e4ca6bf 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ To set up the `.dev.vars` file, you will need to provide the following variables - Add the following to your `.ubiquibot-config.yml` file with the appropriate URL: ```javascript -plugin: http://127.0.0.1:4000 - runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" ] + runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" , "issue.created", "issue.edited", "issue.deleted"] ``` ## Testing Locally diff --git a/manifest.json b/manifest.json index 0e83faf..4b05cc9 100644 --- a/manifest.json +++ b/manifest.json @@ -1,5 +1,5 @@ { "name": "@ubiquity-os/comment-vector-embeddings", "description": "Issue comment plugin for Ubiquibot. It enables the storage, updating, and deletion of issue comment embeddings.", - "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted"] + "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issue.created", "issue.edited", "issue.deleted"] } diff --git a/package.json b/package.json index 80f0dbf..7718367 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "knip-ci": "knip --no-exit-code --reporter json --config .github/knip.ts", "prepare": "husky install", "test": "jest --setupFiles dotenv/config --coverage", - "worker": "wrangler dev --env dev --port 4000", + "worker": "wrangler dev --env dev --port 5000", "supabase:generate:local": "supabase gen types typescript --local > src/types/database.ts", "supabase:generate:remote": "cross-env-shell \"supabase gen types typescript --project-id $SUPABASE_PROJECT_ID --schema public > src/types/database.ts\"" }, diff --git a/src/adapters/index.ts b/src/adapters/index.ts index 29c78a2..ca31d27 100644 --- a/src/adapters/index.ts +++ b/src/adapters/index.ts @@ -5,11 +5,13 @@ import { SuperSupabase } from "./supabase/helpers/supabase"; import { Embedding as VoyageEmbedding } from "./voyage/helpers/embedding"; import { SuperVoyage } from "./voyage/helpers/voyage"; import { VoyageAIClient } from "voyageai"; +import { Issues } from "./supabase/helpers/issues"; export function createAdapters(supabaseClient: SupabaseClient, voyage: VoyageAIClient, context: Context) { return { supabase: { comment: new Comment(supabaseClient, context), + issue: new Issues(supabaseClient, context), super: new SuperSupabase(supabaseClient, context), }, voyage: { diff --git a/src/adapters/supabase/helpers/comment.ts b/src/adapters/supabase/helpers/comment.ts index 291a039..22f5e46 100644 --- a/src/adapters/supabase/helpers/comment.ts +++ b/src/adapters/supabase/helpers/comment.ts @@ -16,9 +16,9 @@ export class Comment extends SuperSupabase { super(supabase, context); } - async createComment(plaintext: string | null, commentNodeId: string, authorId: number, commentobject: Record | null, isPrivate: boolean) { + async createComment(plaintext: string | null, commentNodeId: string, authorId: number, payloadObject: Record | null, isPrivate: boolean) { //First Check if the comment already exists - const { data, error } = await this.supabase.from("issue_comments").select("*").eq("id", commentNodeId); + const { data, error } = await this.supabase.from("vectordump").select("*").eq("id", commentNodeId); if (error) { this.context.logger.error("Error creating comment", error); return; @@ -31,11 +31,11 @@ export class Comment extends SuperSupabase { const embedding = await this.context.adapters.voyage.embedding.createEmbedding(plaintext); if (isPrivate) { plaintext = null as string | null; - commentobject = null as Record | null; + payloadObject = null as Record | null; } const { error } = await this.supabase - .from("issue_comments") - .insert([{ id: commentNodeId, plaintext, author_id: authorId, commentobject, embedding: embedding }]); + .from("vectordump") + .insert([{ id: commentNodeId, plaintext, author_id: authorId, type: "comment", payloadobject: payloadObject, embedding: embedding }]); if (error) { this.context.logger.error("Error creating comment", error); return; @@ -44,16 +44,16 @@ export class Comment extends SuperSupabase { this.context.logger.info("Comment created successfully"); } - async updateComment(plaintext: string | null, commentNodeId: string, commentobject: Record | null, isPrivate: boolean) { + async updateComment(plaintext: string | null, commentNodeId: string, payloadObject: Record | null, isPrivate: boolean) { //Create the embedding for this comment const embedding = Array.from(await this.context.adapters.voyage.embedding.createEmbedding(plaintext)); if (isPrivate) { plaintext = null as string | null; - commentobject = null as Record | null; + payloadObject = null as Record | null; } const { error } = await this.supabase - .from("issue_comments") - .update({ plaintext, embedding: embedding, commentobject, modified_at: new Date() }) + .from("vectordump") + .update({ plaintext, embedding: embedding, payloadobject: payloadObject, modified_at: new Date() }) .eq("id", commentNodeId); if (error) { this.context.logger.error("Error updating comment", error); @@ -61,7 +61,7 @@ export class Comment extends SuperSupabase { } async getComment(commentNodeId: string): Promise { - const { data, error } = await this.supabase.from("issue_comments").select("*").eq("id", commentNodeId); + const { data, error } = await this.supabase.from("vectordump").select("*").eq("id", commentNodeId); if (error) { this.context.logger.error("Error getting comment", error); } @@ -69,7 +69,7 @@ export class Comment extends SuperSupabase { } async deleteComment(commentNodeId: string) { - const { error } = await this.supabase.from("issue_comments").delete().eq("id", commentNodeId); + const { error } = await this.supabase.from("vectordump").delete().eq("id", commentNodeId); if (error) { this.context.logger.error("Error deleting comment", error); } diff --git a/src/adapters/supabase/helpers/issues.ts b/src/adapters/supabase/helpers/issues.ts new file mode 100644 index 0000000..6048ccc --- /dev/null +++ b/src/adapters/supabase/helpers/issues.ts @@ -0,0 +1,85 @@ +import { SupabaseClient } from "@supabase/supabase-js"; +import { SuperSupabase } from "./supabase"; +import { Context } from "../../../types/context"; + +export interface IssueType { + id: string; + plaintext?: string; + author_id: number; + created_at: string; + modified_at: string; + payloadObject: Record | null; + embedding: number[]; +} + +export class Issues extends SuperSupabase { + constructor(supabase: SupabaseClient, context: Context) { + super(supabase, context); + } + + async createIssue(issueNodeId: string, payloadObject: Record | null, isPrivate: boolean, plaintext: string | null, authorId: number) { + //First Check if the issue already exists + const { data, error } = await this.supabase.from("issues").select("*").eq("id", issueNodeId); + if (error) { + this.context.logger.error("Error creating issue", error); + return; + } + if (data && data.length > 0) { + this.context.logger.info("Issue already exists"); + return; + } else { + const embedding = await this.context.adapters.voyage.embedding.createEmbedding(plaintext); + if (isPrivate) { + payloadObject = null; + plaintext = null; + } + const { error } = await this.supabase + .from("issues") + .insert([{ id: issueNodeId, payloadObject, type: "issue", plaintext, author_id: authorId, embedding }]); + if (error) { + this.context.logger.error("Error creating issue", error); + return; + } + } + this.context.logger.info("Issue created successfully"); + } + + async updateIssue(plaintext: string | null, issueNodeId: string, payloadObject: Record | null, isPrivate: boolean) { + //Create the embedding for this comment + const embedding = Array.from(await this.context.adapters.voyage.embedding.createEmbedding(plaintext)); + if (isPrivate) { + plaintext = null as string | null; + payloadObject = null as Record | null; + } + const { error } = await this.supabase + .from("vectordump") + .update({ plaintext, embedding: embedding, payloadObject, modified_at: new Date() }) + .eq("id", issueNodeId); + if (error) { + this.context.logger.error("Error updating comment", error); + } + } + + async deleteIssue(issueNodeId: string) { + const { error } = await this.supabase.from("vectordump").delete().eq("id", issueNodeId); + if (error) { + this.context.logger.error("Error deleting comment", error); + } + } + + async findSimilarIssues(plaintext: string, threshold: number): Promise { + const embedding = await this.context.adapters.voyage.embedding.createEmbedding(plaintext); + const { data, error } = await this.supabase + .from("issues") + .select("*") + .eq("type", "issue") + .textSearch("embedding", embedding.join(",")) + .order("embedding", { foreignTable: "issues", ascending: false }) + .lte("embedding", threshold); + if (error) { + this.context.logger.error("Error finding similar issues", error); + return []; + } + return data; + } +} diff --git a/src/handlers/add-issue.ts b/src/handlers/add-issue.ts new file mode 100644 index 0000000..ce7c315 --- /dev/null +++ b/src/handlers/add-issue.ts @@ -0,0 +1,29 @@ +import { Context } from "../types"; + +export async function addIssue(context: Context) { + const { + logger, + payload, + adapters: { supabase }, + } = context; + const payloadObject = payload; + const plaintext = payload.issue.body + payload.issue.title || ""; + const authorId = payload.issue.user?.id || -1; + const nodeId = payload.issue.node_id; + const isPrivate = payload.repository.private; + + try { + await supabase.issue.createIssue(nodeId, payloadObject, isPrivate, plaintext, authorId); + } catch (error) { + if (error instanceof Error) { + logger.error(`Error creating issue:`, { error: error, stack: error.stack }); + throw error; + } else { + logger.error(`Error creating issue:`, { err: error, error: new Error() }); + throw error; + } + } + + logger.ok(`Successfully created issue!`); + logger.verbose(`Exiting addIssue`); +} diff --git a/src/handlers/delete-issues.ts b/src/handlers/delete-issues.ts new file mode 100644 index 0000000..0e31a87 --- /dev/null +++ b/src/handlers/delete-issues.ts @@ -0,0 +1,26 @@ +import { Context } from "../types"; + +export async function deleteIssues(context: Context) { + const { + logger, + payload, + adapters: { supabase }, + } = context; + + const nodeId = payload.issue.node_id; + + try { + await supabase.issue.deleteIssue(nodeId); + } catch (error) { + if (error instanceof Error) { + logger.error(`Error deleting issue:`, { error: error, stack: error.stack }); + throw error; + } else { + logger.error(`Error deleting issue:`, { err: error, error: new Error() }); + throw error; + } + } + + logger.ok(`Successfully deleted issue!`); + logger.verbose(`Exiting deleteIssue`); +} diff --git a/src/handlers/issue-deduplication.ts b/src/handlers/issue-deduplication.ts new file mode 100644 index 0000000..c9dd522 --- /dev/null +++ b/src/handlers/issue-deduplication.ts @@ -0,0 +1,113 @@ +import { Context } from "../types"; +const MATCH_THRESHOLD = 0.95; +const WARNING_THRESHOLD = 0.75; + +export interface IssueGraphqlResponse { + id: string; + title: string; + url: string; +} + +/** + * Check if an issue is similar to any existing issues in the database + * @param context + * @returns true if the issue is similar to an existing issue, false otherwise + */ +export async function issueChecker(context: Context): Promise { + const { + logger, + payload, + adapters: { supabase }, + } = context; + + const issue = payload.issue; + + //First Check if an issue with more than MATCH_THRESHOLD similarity exists (Very Similar) + const similarIssue = await supabase.issue.findSimilarIssues(issue.body + issue.title, MATCH_THRESHOLD); + if (similarIssue) { + logger.info(`Similar issue which matches more than ${MATCH_THRESHOLD} already exists`); + //Close the issue as "unplanned" + await context.octokit.issues.update({ + owner: payload.repository.owner.login, + repo: payload.repository.name, + issue_number: issue.number, + state: "closed", + labels: ["unplanned"], + }); + return true; + } + + //Second Check if an issue with more than WARNING_THRESHOLD similarity exists (Warning) + const warningIssue = await supabase.issue.findSimilarIssues(issue.body + issue.title, WARNING_THRESHOLD); + if (warningIssue) { + logger.info(`Similar issue which matches more than ${WARNING_THRESHOLD} already exists`); + //Add a comment immediately next to the issue + //Build a list of similar issues url + const issueList = warningIssue.map(async (issue) => { + //fetch the issue url and title using globaNodeId + const issueUrl: IssueGraphqlResponse = await context.octokit.graphql( + `query($issueNodeId: String!) { + node(id: $issueNodeId) { + ... on Issue { + title + url + } + } + }`, + { + issueNodeId: issue.id, + } + ); + return issueUrl; + }); + //Add a comment to the issue + const resolvedIssueList = await Promise.all(issueList); + // Check if there is already a comment on the issue + const existingComment = await context.octokit.issues.listComments({ + owner: payload.repository.owner.login, + repo: payload.repository.name, + issue_number: issue.number, + }); + + if (existingComment.data.length > 0) { + // Find the comment that lists the similar issues + const commentToUpdate = existingComment.data.find( + (comment) => comment && comment.body && comment.body.includes("This issue seems to be similar to the following issue(s)") + ); + + if (commentToUpdate) { + // Update the comment with the latest list of similar issues + await context.octokit.issues.updateComment({ + owner: payload.repository.owner.login, + repo: payload.repository.name, + comment_id: commentToUpdate.id, + body: `This issue seems to be similar to the following issue(s) ${resolvedIssueList.map((issue) => issue.url).join(", ")}`, + }); + } else { + // Add a new comment to the issue + await createNewComment(context, resolvedIssueList); + } + } else { + // Add a new comment to the issue + await createNewComment(context, resolvedIssueList); + } + return true; + } + + logger.info("No similar issue found"); + return false; +} + +/** + * Create a new comment on the issue with the list of similar issues + * @param context + * @param resolvedIssueList + */ +async function createNewComment(context: Context, resolvedIssueList: IssueGraphqlResponse[]) { + await context.octokit.issues.createComment({ + owner: context.payload.repository.owner.login, + repo: context.payload.repository.name, + issue_number: context.payload.issue.number, + body: `This issue seems to be similar to the following issue(s) ${resolvedIssueList.map((issue) => issue.url).join(", ")}`, + }); +} diff --git a/src/handlers/update-issue.ts b/src/handlers/update-issue.ts new file mode 100644 index 0000000..7ad5f3d --- /dev/null +++ b/src/handlers/update-issue.ts @@ -0,0 +1,28 @@ +import { Context } from "../types"; + +export async function updateIssue(context: Context) { + const { + logger, + payload, + adapters: { supabase }, + } = context; + const payloadObject = payload; + const nodeId = payload.issue.node_id; + const isPrivate = payload.repository.private; + const plaintext = payload.issue.body + payload.issue.title || ""; + // Fetch the previous comment and update it in the db + try { + await supabase.issue.updateIssue(plaintext, nodeId, payloadObject, isPrivate); + } catch (error) { + if (error instanceof Error) { + logger.error(`Error updating issue:`, { error: error, stack: error.stack }); + throw error; + } else { + logger.error(`Error updating issue:`, { err: error, error: new Error() }); + throw error; + } + } + + logger.ok(`Successfully updated issue!`); + logger.verbose(`Exiting updateIssue`); +} diff --git a/src/plugin.ts b/src/plugin.ts index 9673231..c117ba9 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -10,6 +10,10 @@ import { addComments } from "./handlers/add-comments"; import { updateComment } from "./handlers/update-comments"; import { deleteComment } from "./handlers/delete-comments"; import { VoyageAIClient } from "voyageai"; +import { issueChecker } from "./handlers/issue-deduplication"; +import { deleteIssues } from "./handlers/delete-issues"; +import { addIssue } from "./handlers/add-issue"; +import { updateIssue } from "./handlers/update-issue"; /** * The main plugin function. Split for easier testing. @@ -24,6 +28,12 @@ export async function runPlugin(context: Context) { return await deleteComment(context); case "issue_comment.edited": return await updateComment(context); + case "issue.created": + return (await issueChecker(context)) === false ? await addIssue(context) : null; + case "issue.edited": + return (await issueChecker(context)) === false ? await updateIssue(context) : null; + case "issue.deleted": + return await deleteIssues(context); } } else { logger.error(`Unsupported event: ${eventName}`); diff --git a/src/types/context.ts b/src/types/context.ts index d7e3cff..b7fc84b 100644 --- a/src/types/context.ts +++ b/src/types/context.ts @@ -10,7 +10,7 @@ import { createAdapters } from "../adapters"; * * ubiquity:listeners: ["issue_comment.created", ...] */ -export type SupportedEventsU = "issue_comment.created" | "issue_comment.deleted" | "issue_comment.edited"; +export type SupportedEventsU = "issue_comment.created" | "issue_comment.deleted" | "issue_comment.edited" | "issue.created" | "issue.edited" | "issue.deleted"; export type SupportedEvents = { [K in SupportedEventsU]: K extends WebhookEventName ? WebhookEvent : never; diff --git a/src/types/typeguards.ts b/src/types/typeguards.ts index 266c856..1f724c9 100644 --- a/src/types/typeguards.ts +++ b/src/types/typeguards.ts @@ -11,6 +11,15 @@ import { Context } from "./context"; * * @param context The context object. */ -export function isIssueCommentEvent(context: Context): context is Context<"issue_comment.created" | "issue_comment.deleted" | "issue_comment.edited"> { - return context.eventName === "issue_comment.created" || context.eventName === "issue_comment.deleted" || context.eventName === "issue_comment.edited"; +export function isIssueCommentEvent( + context: Context +): context is Context<"issue_comment.created" | "issue_comment.deleted" | "issue_comment.edited" | "issue.created" | "issue.edited" | "issue.deleted"> { + return ( + context.eventName === "issue_comment.created" || + context.eventName === "issue_comment.deleted" || + context.eventName === "issue_comment.edited" || + context.eventName === "issue.created" || + context.eventName === "issue.edited" || + context.eventName === "issue.deleted" + ); } diff --git a/supabase/migrations/20240911233824_issue_comments.sql b/supabase/migrations/20240911233824_issue_comments.sql new file mode 100644 index 0000000..c6ef4eb --- /dev/null +++ b/supabase/migrations/20240911233824_issue_comments.sql @@ -0,0 +1,8 @@ +ALTER TABLE issue_comments +RENAME COLUMN commentObject TO payloadObject; + +ALTER TABLE issue_comments +ADD COLUMN type Text NOT NULL; + +ALTER TABLE issue_comments +RENAME TO vectorDump; From 55ded10670d7ad255ff635261608eacc27f06fc2 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Wed, 11 Sep 2024 22:17:50 -0400 Subject: [PATCH 21/55] fix: removing issue.created hook --- .github/.ubiquibot-config.yml | 2 +- manifest.json | 2 +- src/handlers/issue-deduplication.ts | 2 +- src/types/context.ts | 2 +- src/types/typeguards.ts | 14 +++++--------- 5 files changed, 9 insertions(+), 13 deletions(-) diff --git a/.github/.ubiquibot-config.yml b/.github/.ubiquibot-config.yml index 9213535..8a9597c 100644 --- a/.github/.ubiquibot-config.yml +++ b/.github/.ubiquibot-config.yml @@ -3,4 +3,4 @@ plugins: id: test-app uses: - plugin: https://ubiquity-os-comment-vector-embeddings.sshivaditya.workers.dev - runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issue.created", "issue.edited", "issue.deleted" ] + runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" ] diff --git a/manifest.json b/manifest.json index 4b05cc9..0e83faf 100644 --- a/manifest.json +++ b/manifest.json @@ -1,5 +1,5 @@ { "name": "@ubiquity-os/comment-vector-embeddings", "description": "Issue comment plugin for Ubiquibot. It enables the storage, updating, and deletion of issue comment embeddings.", - "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issue.created", "issue.edited", "issue.deleted"] + "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted"] } diff --git a/src/handlers/issue-deduplication.ts b/src/handlers/issue-deduplication.ts index c9dd522..8aedee0 100644 --- a/src/handlers/issue-deduplication.ts +++ b/src/handlers/issue-deduplication.ts @@ -44,7 +44,7 @@ export async function issueChecker(context: Context): Promise { //Add a comment immediately next to the issue //Build a list of similar issues url const issueList = warningIssue.map(async (issue) => { - //fetch the issue url and title using globaNodeId + //fetch the issue url and title using globalNodeId const issueUrl: IssueGraphqlResponse = await context.octokit.graphql( `query($issueNodeId: String!) { node(id: $issueNodeId) { diff --git a/src/types/context.ts b/src/types/context.ts index b7fc84b..d7e3cff 100644 --- a/src/types/context.ts +++ b/src/types/context.ts @@ -10,7 +10,7 @@ import { createAdapters } from "../adapters"; * * ubiquity:listeners: ["issue_comment.created", ...] */ -export type SupportedEventsU = "issue_comment.created" | "issue_comment.deleted" | "issue_comment.edited" | "issue.created" | "issue.edited" | "issue.deleted"; +export type SupportedEventsU = "issue_comment.created" | "issue_comment.deleted" | "issue_comment.edited"; export type SupportedEvents = { [K in SupportedEventsU]: K extends WebhookEventName ? WebhookEvent : never; diff --git a/src/types/typeguards.ts b/src/types/typeguards.ts index 1f724c9..861cca7 100644 --- a/src/types/typeguards.ts +++ b/src/types/typeguards.ts @@ -11,15 +11,11 @@ import { Context } from "./context"; * * @param context The context object. */ -export function isIssueCommentEvent( - context: Context -): context is Context<"issue_comment.created" | "issue_comment.deleted" | "issue_comment.edited" | "issue.created" | "issue.edited" | "issue.deleted"> { +export function isIssueCommentEvent(context: Context): context is Context<"issue_comment.created" | "issue_comment.deleted" | "issue_comment.edited"> { return ( - context.eventName === "issue_comment.created" || - context.eventName === "issue_comment.deleted" || - context.eventName === "issue_comment.edited" || - context.eventName === "issue.created" || - context.eventName === "issue.edited" || - context.eventName === "issue.deleted" + context.eventName === "issue_comment.created" || context.eventName === "issue_comment.deleted" || context.eventName === "issue_comment.edited" //|| + // context.eventName === "issue.created" || + // context.eventName === "issue.edited" || + // context.eventName === "issue.deleted" ); } From d604e4113dde441c6bfe270a02e06e12e6d78d8f Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Wed, 11 Sep 2024 22:26:13 -0400 Subject: [PATCH 22/55] fix: linting and tests --- src/plugin.ts | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/plugin.ts b/src/plugin.ts index c117ba9..9032cf8 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -10,10 +10,10 @@ import { addComments } from "./handlers/add-comments"; import { updateComment } from "./handlers/update-comments"; import { deleteComment } from "./handlers/delete-comments"; import { VoyageAIClient } from "voyageai"; -import { issueChecker } from "./handlers/issue-deduplication"; -import { deleteIssues } from "./handlers/delete-issues"; -import { addIssue } from "./handlers/add-issue"; -import { updateIssue } from "./handlers/update-issue"; +// import { issueChecker } from "./handlers/issue-deduplication"; +// import { deleteIssues } from "./handlers/delete-issues"; +// import { addIssue } from "./handlers/add-issue"; +// import { updateIssue } from "./handlers/update-issue"; /** * The main plugin function. Split for easier testing. @@ -28,12 +28,12 @@ export async function runPlugin(context: Context) { return await deleteComment(context); case "issue_comment.edited": return await updateComment(context); - case "issue.created": - return (await issueChecker(context)) === false ? await addIssue(context) : null; - case "issue.edited": - return (await issueChecker(context)) === false ? await updateIssue(context) : null; - case "issue.deleted": - return await deleteIssues(context); + // case "issue.created": + // return (await issueChecker(context)) === false ? await addIssue(context) : null; + // case "issue.edited": + // return (await issueChecker(context)) === false ? await updateIssue(context) : null; + // case "issue.deleted": + // return await deleteIssues(context); } } else { logger.error(`Unsupported event: ${eventName}`); From 2b7a65c501b1201e4cfcb5507e42a8ae20606251 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Thu, 12 Sep 2024 09:22:34 -0400 Subject: [PATCH 23/55] fix: rollback issue.created, edited, deleted --- README.md | 2 +- src/handlers/add-issue.ts | 29 ------- src/handlers/delete-issues.ts | 26 ------- src/handlers/issue-deduplication.ts | 113 ---------------------------- src/handlers/update-issue.ts | 28 ------- src/plugin.ts | 10 --- src/types/typeguards.ts | 7 +- 7 files changed, 2 insertions(+), 213 deletions(-) delete mode 100644 src/handlers/add-issue.ts delete mode 100644 src/handlers/delete-issues.ts delete mode 100644 src/handlers/issue-deduplication.ts delete mode 100644 src/handlers/update-issue.ts diff --git a/README.md b/README.md index e4ca6bf..ec224cb 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ To set up the `.dev.vars` file, you will need to provide the following variables - Add the following to your `.ubiquibot-config.yml` file with the appropriate URL: ```javascript -plugin: http://127.0.0.1:4000 - runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" , "issue.created", "issue.edited", "issue.deleted"] + runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted"] ``` ## Testing Locally diff --git a/src/handlers/add-issue.ts b/src/handlers/add-issue.ts deleted file mode 100644 index ce7c315..0000000 --- a/src/handlers/add-issue.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { Context } from "../types"; - -export async function addIssue(context: Context) { - const { - logger, - payload, - adapters: { supabase }, - } = context; - const payloadObject = payload; - const plaintext = payload.issue.body + payload.issue.title || ""; - const authorId = payload.issue.user?.id || -1; - const nodeId = payload.issue.node_id; - const isPrivate = payload.repository.private; - - try { - await supabase.issue.createIssue(nodeId, payloadObject, isPrivate, plaintext, authorId); - } catch (error) { - if (error instanceof Error) { - logger.error(`Error creating issue:`, { error: error, stack: error.stack }); - throw error; - } else { - logger.error(`Error creating issue:`, { err: error, error: new Error() }); - throw error; - } - } - - logger.ok(`Successfully created issue!`); - logger.verbose(`Exiting addIssue`); -} diff --git a/src/handlers/delete-issues.ts b/src/handlers/delete-issues.ts deleted file mode 100644 index 0e31a87..0000000 --- a/src/handlers/delete-issues.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { Context } from "../types"; - -export async function deleteIssues(context: Context) { - const { - logger, - payload, - adapters: { supabase }, - } = context; - - const nodeId = payload.issue.node_id; - - try { - await supabase.issue.deleteIssue(nodeId); - } catch (error) { - if (error instanceof Error) { - logger.error(`Error deleting issue:`, { error: error, stack: error.stack }); - throw error; - } else { - logger.error(`Error deleting issue:`, { err: error, error: new Error() }); - throw error; - } - } - - logger.ok(`Successfully deleted issue!`); - logger.verbose(`Exiting deleteIssue`); -} diff --git a/src/handlers/issue-deduplication.ts b/src/handlers/issue-deduplication.ts deleted file mode 100644 index 8aedee0..0000000 --- a/src/handlers/issue-deduplication.ts +++ /dev/null @@ -1,113 +0,0 @@ -import { Context } from "../types"; -const MATCH_THRESHOLD = 0.95; -const WARNING_THRESHOLD = 0.75; - -export interface IssueGraphqlResponse { - id: string; - title: string; - url: string; -} - -/** - * Check if an issue is similar to any existing issues in the database - * @param context - * @returns true if the issue is similar to an existing issue, false otherwise - */ -export async function issueChecker(context: Context): Promise { - const { - logger, - payload, - adapters: { supabase }, - } = context; - - const issue = payload.issue; - - //First Check if an issue with more than MATCH_THRESHOLD similarity exists (Very Similar) - const similarIssue = await supabase.issue.findSimilarIssues(issue.body + issue.title, MATCH_THRESHOLD); - if (similarIssue) { - logger.info(`Similar issue which matches more than ${MATCH_THRESHOLD} already exists`); - //Close the issue as "unplanned" - await context.octokit.issues.update({ - owner: payload.repository.owner.login, - repo: payload.repository.name, - issue_number: issue.number, - state: "closed", - labels: ["unplanned"], - }); - return true; - } - - //Second Check if an issue with more than WARNING_THRESHOLD similarity exists (Warning) - const warningIssue = await supabase.issue.findSimilarIssues(issue.body + issue.title, WARNING_THRESHOLD); - if (warningIssue) { - logger.info(`Similar issue which matches more than ${WARNING_THRESHOLD} already exists`); - //Add a comment immediately next to the issue - //Build a list of similar issues url - const issueList = warningIssue.map(async (issue) => { - //fetch the issue url and title using globalNodeId - const issueUrl: IssueGraphqlResponse = await context.octokit.graphql( - `query($issueNodeId: String!) { - node(id: $issueNodeId) { - ... on Issue { - title - url - } - } - }`, - { - issueNodeId: issue.id, - } - ); - return issueUrl; - }); - //Add a comment to the issue - const resolvedIssueList = await Promise.all(issueList); - // Check if there is already a comment on the issue - const existingComment = await context.octokit.issues.listComments({ - owner: payload.repository.owner.login, - repo: payload.repository.name, - issue_number: issue.number, - }); - - if (existingComment.data.length > 0) { - // Find the comment that lists the similar issues - const commentToUpdate = existingComment.data.find( - (comment) => comment && comment.body && comment.body.includes("This issue seems to be similar to the following issue(s)") - ); - - if (commentToUpdate) { - // Update the comment with the latest list of similar issues - await context.octokit.issues.updateComment({ - owner: payload.repository.owner.login, - repo: payload.repository.name, - comment_id: commentToUpdate.id, - body: `This issue seems to be similar to the following issue(s) ${resolvedIssueList.map((issue) => issue.url).join(", ")}`, - }); - } else { - // Add a new comment to the issue - await createNewComment(context, resolvedIssueList); - } - } else { - // Add a new comment to the issue - await createNewComment(context, resolvedIssueList); - } - return true; - } - - logger.info("No similar issue found"); - return false; -} - -/** - * Create a new comment on the issue with the list of similar issues - * @param context - * @param resolvedIssueList - */ -async function createNewComment(context: Context, resolvedIssueList: IssueGraphqlResponse[]) { - await context.octokit.issues.createComment({ - owner: context.payload.repository.owner.login, - repo: context.payload.repository.name, - issue_number: context.payload.issue.number, - body: `This issue seems to be similar to the following issue(s) ${resolvedIssueList.map((issue) => issue.url).join(", ")}`, - }); -} diff --git a/src/handlers/update-issue.ts b/src/handlers/update-issue.ts deleted file mode 100644 index 7ad5f3d..0000000 --- a/src/handlers/update-issue.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { Context } from "../types"; - -export async function updateIssue(context: Context) { - const { - logger, - payload, - adapters: { supabase }, - } = context; - const payloadObject = payload; - const nodeId = payload.issue.node_id; - const isPrivate = payload.repository.private; - const plaintext = payload.issue.body + payload.issue.title || ""; - // Fetch the previous comment and update it in the db - try { - await supabase.issue.updateIssue(plaintext, nodeId, payloadObject, isPrivate); - } catch (error) { - if (error instanceof Error) { - logger.error(`Error updating issue:`, { error: error, stack: error.stack }); - throw error; - } else { - logger.error(`Error updating issue:`, { err: error, error: new Error() }); - throw error; - } - } - - logger.ok(`Successfully updated issue!`); - logger.verbose(`Exiting updateIssue`); -} diff --git a/src/plugin.ts b/src/plugin.ts index 9032cf8..9673231 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -10,10 +10,6 @@ import { addComments } from "./handlers/add-comments"; import { updateComment } from "./handlers/update-comments"; import { deleteComment } from "./handlers/delete-comments"; import { VoyageAIClient } from "voyageai"; -// import { issueChecker } from "./handlers/issue-deduplication"; -// import { deleteIssues } from "./handlers/delete-issues"; -// import { addIssue } from "./handlers/add-issue"; -// import { updateIssue } from "./handlers/update-issue"; /** * The main plugin function. Split for easier testing. @@ -28,12 +24,6 @@ export async function runPlugin(context: Context) { return await deleteComment(context); case "issue_comment.edited": return await updateComment(context); - // case "issue.created": - // return (await issueChecker(context)) === false ? await addIssue(context) : null; - // case "issue.edited": - // return (await issueChecker(context)) === false ? await updateIssue(context) : null; - // case "issue.deleted": - // return await deleteIssues(context); } } else { logger.error(`Unsupported event: ${eventName}`); diff --git a/src/types/typeguards.ts b/src/types/typeguards.ts index 861cca7..266c856 100644 --- a/src/types/typeguards.ts +++ b/src/types/typeguards.ts @@ -12,10 +12,5 @@ import { Context } from "./context"; * @param context The context object. */ export function isIssueCommentEvent(context: Context): context is Context<"issue_comment.created" | "issue_comment.deleted" | "issue_comment.edited"> { - return ( - context.eventName === "issue_comment.created" || context.eventName === "issue_comment.deleted" || context.eventName === "issue_comment.edited" //|| - // context.eventName === "issue.created" || - // context.eventName === "issue.edited" || - // context.eventName === "issue.deleted" - ); + return context.eventName === "issue_comment.created" || context.eventName === "issue_comment.deleted" || context.eventName === "issue_comment.edited"; } From 7abb239f2854a8a0241ebd05331a3daa8aa73695 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Thu, 12 Sep 2024 19:37:03 -0400 Subject: [PATCH 24/55] feat: updated schema fix tests --- src/adapters/supabase/helpers/comment.ts | 29 ++++++---- src/adapters/supabase/helpers/issues.ts | 4 +- src/adapters/utils/comment-object-clean.ts | 53 ------------------- src/handlers/add-comments.ts | 5 +- src/handlers/update-comments.ts | 4 +- .../20240912225853_issue_comments.sql | 20 +++++++ tests/__mocks__/adapter.ts | 17 ++++-- tests/main.test.ts | 6 +-- 8 files changed, 59 insertions(+), 79 deletions(-) delete mode 100644 src/adapters/utils/comment-object-clean.ts create mode 100644 supabase/migrations/20240912225853_issue_comments.sql diff --git a/src/adapters/supabase/helpers/comment.ts b/src/adapters/supabase/helpers/comment.ts index 22f5e46..0f59531 100644 --- a/src/adapters/supabase/helpers/comment.ts +++ b/src/adapters/supabase/helpers/comment.ts @@ -16,9 +16,16 @@ export class Comment extends SuperSupabase { super(supabase, context); } - async createComment(plaintext: string | null, commentNodeId: string, authorId: number, payloadObject: Record | null, isPrivate: boolean) { + async createComment( + plaintext: string | null, + commentNodeId: string, + authorId: number, + payload: Record | null, + isPrivate: boolean, + issueId: string + ) { //First Check if the comment already exists - const { data, error } = await this.supabase.from("vectordump").select("*").eq("id", commentNodeId); + const { data, error } = await this.supabase.from("issue_comments").select("*").eq("id", commentNodeId); if (error) { this.context.logger.error("Error creating comment", error); return; @@ -31,11 +38,11 @@ export class Comment extends SuperSupabase { const embedding = await this.context.adapters.voyage.embedding.createEmbedding(plaintext); if (isPrivate) { plaintext = null as string | null; - payloadObject = null as Record | null; + payload = null as Record | null; } const { error } = await this.supabase - .from("vectordump") - .insert([{ id: commentNodeId, plaintext, author_id: authorId, type: "comment", payloadobject: payloadObject, embedding: embedding }]); + .from("issue_comments") + .insert([{ id: commentNodeId, plaintext, author_id: authorId, type: "comment", payload, embedding: embedding, issueId }]); if (error) { this.context.logger.error("Error creating comment", error); return; @@ -44,16 +51,16 @@ export class Comment extends SuperSupabase { this.context.logger.info("Comment created successfully"); } - async updateComment(plaintext: string | null, commentNodeId: string, payloadObject: Record | null, isPrivate: boolean) { + async updateComment(plaintext: string | null, commentNodeId: string, payload: Record | null, isPrivate: boolean) { //Create the embedding for this comment const embedding = Array.from(await this.context.adapters.voyage.embedding.createEmbedding(plaintext)); if (isPrivate) { plaintext = null as string | null; - payloadObject = null as Record | null; + payload = null as Record | null; } const { error } = await this.supabase - .from("vectordump") - .update({ plaintext, embedding: embedding, payloadobject: payloadObject, modified_at: new Date() }) + .from("issue_comments") + .update({ plaintext, embedding: embedding, payload, modified_at: new Date() }) .eq("id", commentNodeId); if (error) { this.context.logger.error("Error updating comment", error); @@ -61,7 +68,7 @@ export class Comment extends SuperSupabase { } async getComment(commentNodeId: string): Promise { - const { data, error } = await this.supabase.from("vectordump").select("*").eq("id", commentNodeId); + const { data, error } = await this.supabase.from("issue_comments").select("*").eq("id", commentNodeId); if (error) { this.context.logger.error("Error getting comment", error); } @@ -69,7 +76,7 @@ export class Comment extends SuperSupabase { } async deleteComment(commentNodeId: string) { - const { error } = await this.supabase.from("vectordump").delete().eq("id", commentNodeId); + const { error } = await this.supabase.from("issue_comments").delete().eq("id", commentNodeId); if (error) { this.context.logger.error("Error deleting comment", error); } diff --git a/src/adapters/supabase/helpers/issues.ts b/src/adapters/supabase/helpers/issues.ts index 6048ccc..e7d7e95 100644 --- a/src/adapters/supabase/helpers/issues.ts +++ b/src/adapters/supabase/helpers/issues.ts @@ -52,7 +52,7 @@ export class Issues extends SuperSupabase { payloadObject = null as Record | null; } const { error } = await this.supabase - .from("vectordump") + .from("issue_comments") .update({ plaintext, embedding: embedding, payloadObject, modified_at: new Date() }) .eq("id", issueNodeId); if (error) { @@ -61,7 +61,7 @@ export class Issues extends SuperSupabase { } async deleteIssue(issueNodeId: string) { - const { error } = await this.supabase.from("vectordump").delete().eq("id", issueNodeId); + const { error } = await this.supabase.from("issue_comments").delete().eq("id", issueNodeId); if (error) { this.context.logger.error("Error deleting comment", error); } diff --git a/src/adapters/utils/comment-object-clean.ts b/src/adapters/utils/comment-object-clean.ts deleted file mode 100644 index 7dc0d7e..0000000 --- a/src/adapters/utils/comment-object-clean.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { SupportedEvents, SupportedEventsU } from "../../types/context"; - -/** - * Extracts only the properties mentioned in the TypeBox schema from the input object. - * - * @param commentObject - The comment object. - * @returns The object containing only the properties defined in the schema. - */ -export function cleanCommentObject( - commentObject: TU["payload"] -): Record { - // Manually extract properties - return { - action: commentObject.action as string, - issue: { - id: commentObject.issue.id as number, - number: commentObject.issue.number as number, - title: commentObject.issue.title as string, - body: commentObject.issue.body as string, - user: { - login: commentObject.issue.user.login as string, - id: commentObject.issue.user.id as number, - }, - author_association: commentObject.issue.author_association as string, - }, - comment: { - author_association: commentObject.comment.author_association as string, - id: commentObject.comment.id as number, - html_url: commentObject.comment.html_url as string, - issue_url: commentObject.comment.issue_url as string, - user: { - login: (commentObject.comment.user || { login: "" }).login as string, - id: (commentObject.comment.user || { id: -1 }).id as number, - }, - body: commentObject.comment.body as string, - created_at: commentObject.comment.created_at as string, - updated_at: commentObject.comment.updated_at as string, - }, - - repository: { - id: commentObject.repository.id as number, - node_id: commentObject.repository.node_id as string, - name: commentObject.repository.name as string, - full_name: commentObject.repository.full_name as string, - private: commentObject.repository.private as boolean, - owner: { - login: commentObject.repository.owner.login as string, - id: commentObject.repository.owner.id as number, - avatar_url: commentObject.repository.owner.avatar_url as string, - }, - }, - } as Record; -} diff --git a/src/handlers/add-comments.ts b/src/handlers/add-comments.ts index 62486d4..bb0215c 100644 --- a/src/handlers/add-comments.ts +++ b/src/handlers/add-comments.ts @@ -1,4 +1,3 @@ -import { cleanCommentObject } from "../adapters/utils/comment-object-clean"; import { Context } from "../types"; export async function addComments(context: Context) { @@ -7,14 +6,14 @@ export async function addComments(context: Context) { payload, adapters: { supabase }, } = context; - const commentObject = cleanCommentObject(payload); const plaintext = payload.comment.body; const authorId = payload.comment.user?.id || -1; const nodeId = payload.comment.node_id; const isPrivate = payload.repository.private; + const issueId = payload.issue.node_id; try { - await supabase.comment.createComment(plaintext, nodeId, authorId, commentObject, isPrivate); + await supabase.comment.createComment(plaintext, nodeId, authorId, payload, isPrivate, issueId); } catch (error) { if (error instanceof Error) { logger.error(`Error creating comment:`, { error: error, stack: error.stack }); diff --git a/src/handlers/update-comments.ts b/src/handlers/update-comments.ts index 5ae6b46..f354d91 100644 --- a/src/handlers/update-comments.ts +++ b/src/handlers/update-comments.ts @@ -1,4 +1,3 @@ -import { cleanCommentObject } from "../adapters/utils/comment-object-clean"; import { Context } from "../types"; export async function updateComment(context: Context) { @@ -7,13 +6,12 @@ export async function updateComment(context: Context) { payload, adapters: { supabase }, } = context; - const commentObject = cleanCommentObject(payload); const nodeId = payload.comment.node_id; const isPrivate = payload.repository.private; const plaintext = payload.comment.body; // Fetch the previous comment and update it in the db try { - await supabase.comment.updateComment(plaintext, nodeId, commentObject, isPrivate); + await supabase.comment.updateComment(plaintext, nodeId, payload, isPrivate); } catch (error) { if (error instanceof Error) { logger.error(`Error updating comment:`, { error: error, stack: error.stack }); diff --git a/supabase/migrations/20240912225853_issue_comments.sql b/supabase/migrations/20240912225853_issue_comments.sql new file mode 100644 index 0000000..3a8764a --- /dev/null +++ b/supabase/migrations/20240912225853_issue_comments.sql @@ -0,0 +1,20 @@ +ALTER TABLE vectordump +RENAME TO issue_comments; + +CREATE TABLE IF NOT EXISTS issues ( + id VARCHAR primary key, + plaintext text, + embedding Vector(1024) not null, + payload jsonb, + type text not null default 'issue', + created_at timestamptz not null default now(), + modified_at timestamptz not null default now() +); + +ALTER TABLE issue_comments +ADD COLUMN issue_id VARCHAR +REFERENCES issues(id) +ON DELETE CASCADE; + +ALTER TABLE issue_comments +RENAME COLUMN payloadobject TO payload; \ No newline at end of file diff --git a/tests/__mocks__/adapter.ts b/tests/__mocks__/adapter.ts index ee9956f..d1f634c 100644 --- a/tests/__mocks__/adapter.ts +++ b/tests/__mocks__/adapter.ts @@ -6,7 +6,9 @@ export interface CommentMock { id: string; plaintext: string | null; author_id: number; - commentObject?: Record | null; + payload?: Record | null; + type?: string; + issue_id?: string; embedding: number[]; } @@ -16,7 +18,14 @@ export function createMockAdapters(context: Context) { supabase: { comment: { createComment: jest.fn( - async (plaintext: string | null, commentNodeId: string, authorId: number, commentObject: Record | null, isPrivate: boolean) => { + async ( + plaintext: string | null, + commentNodeId: string, + authorId: number, + payload: Record | null, + isPrivate: boolean, + issueId: string + ) => { if (commentMap.has(commentNodeId)) { throw new Error("Comment already exists"); } @@ -24,10 +33,10 @@ export function createMockAdapters(context: Context) { if (isPrivate) { plaintext = null; } - commentMap.set(commentNodeId, { id: commentNodeId, plaintext, author_id: authorId, embedding }); + commentMap.set(commentNodeId, { id: commentNodeId, plaintext, author_id: authorId, embedding, issue_id: issueId }); } ), - updateComment: jest.fn(async (plaintext: string | null, commentNodeId: string, commentObject: Record | null, isPrivate: boolean) => { + updateComment: jest.fn(async (plaintext: string | null, commentNodeId: string, payload: Record | null, isPrivate: boolean) => { if (!commentMap.has(commentNodeId)) { throw new Error(STRINGS.COMMENT_DOES_NOT_EXIST); } diff --git a/tests/main.test.ts b/tests/main.test.ts index 5f4fd84..adf1481 100644 --- a/tests/main.test.ts +++ b/tests/main.test.ts @@ -52,7 +52,7 @@ describe("Plugin tests", () => { const supabase = context.adapters.supabase; const commentObject = null; try { - await supabase.comment.createComment(STRINGS.HELLO_WORLD, "sasasCreate", 1, commentObject, false); + await supabase.comment.createComment(STRINGS.HELLO_WORLD, "sasasCreate", 1, commentObject, false, "sasasCreateIssue"); throw new Error("Expected method to reject."); } catch (error) { if (error instanceof Error) { @@ -69,7 +69,7 @@ describe("Plugin tests", () => { const { context } = createContext("Updated Message", 1, 1, 1, 1, "sasasUpdate", "issue_comment.edited"); const supabase = context.adapters.supabase; const commentObject = null; - await supabase.comment.createComment(STRINGS.HELLO_WORLD, "sasasUpdate", 1, commentObject, false); + await supabase.comment.createComment(STRINGS.HELLO_WORLD, "sasasUpdate", 1, commentObject, false, "sasasUpdateIssue"); await runPlugin(context); const comment = (await supabase.comment.getComment("sasasUpdate")) as unknown as CommentMock; expect(comment).toBeDefined(); @@ -81,7 +81,7 @@ describe("Plugin tests", () => { const { context } = createContext("Text Message", 1, 1, 1, 1, "sasasDelete", "issue_comment.deleted"); const supabase = context.adapters.supabase; const commentObject = null; - await supabase.comment.createComment(STRINGS.HELLO_WORLD, "sasasDelete", 1, commentObject, false); + await supabase.comment.createComment(STRINGS.HELLO_WORLD, "sasasDelete", 1, commentObject, false, "sasasDeleteIssue"); await runPlugin(context); try { await supabase.comment.getComment("sasasDelete"); From 8b2a945505487e1bff1d2dc8912e4e4fba2293ef Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Thu, 12 Sep 2024 21:24:10 -0400 Subject: [PATCH 25/55] fix: types of payload --- .cspell.json | 2 ++ .github/.ubiquibot-config.yml | 5 ++--- README.md | 3 ++- manifest.json | 2 +- src/handlers/add-comments.ts | 3 ++- src/handlers/add-issue.ts | 29 +++++++++++++++++++++++++++++ src/handlers/delete-comments.ts | 4 ++-- src/handlers/delete-issue.ts | 26 ++++++++++++++++++++++++++ src/handlers/update-comments.ts | 3 ++- src/handlers/update-issue.ts | 29 +++++++++++++++++++++++++++++ src/plugin.ts | 18 ++++++++++++++++-- src/types/context.ts | 8 +++++++- src/types/payload.ts | 3 +++ src/types/typeguards.ts | 9 +++++++++ tests/main.test.ts | 8 +++++--- 15 files changed, 137 insertions(+), 15 deletions(-) create mode 100644 src/handlers/add-issue.ts create mode 100644 src/handlers/delete-issue.ts create mode 100644 src/handlers/update-issue.ts create mode 100644 src/types/payload.ts diff --git a/.cspell.json b/.cspell.json index e64757f..ecd7f51 100644 --- a/.cspell.json +++ b/.cspell.json @@ -24,6 +24,8 @@ "commentbody", "issuebody", "voyageai", + "vectordump", + "payloadobject" ], "dictionaries": ["typescript", "node", "software-terms"], "import": ["@cspell/dict-typescript/cspell-ext.json", "@cspell/dict-node/cspell-ext.json", "@cspell/dict-software-terms"], diff --git a/.github/.ubiquibot-config.yml b/.github/.ubiquibot-config.yml index 8a9597c..bce9467 100644 --- a/.github/.ubiquibot-config.yml +++ b/.github/.ubiquibot-config.yml @@ -1,6 +1,5 @@ -plugins: - - name: test-app - id: test-app + id: test-app uses: - plugin: https://ubiquity-os-comment-vector-embeddings.sshivaditya.workers.dev runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" ] + runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issues.created", "issues.edited", "issues.deleted" ] \ No newline at end of file diff --git a/README.md b/README.md index ec224cb..0bb10f4 100644 --- a/README.md +++ b/README.md @@ -13,9 +13,10 @@ To set up the `.dev.vars` file, you will need to provide the following variables - Add the following to your `.ubiquibot-config.yml` file with the appropriate URL: ```javascript -plugin: http://127.0.0.1:4000 - runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted"] + runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" , "issue.created", "issue.edited", "issue.deleted"] ``` + ## Testing Locally - Run `yarn install` to install the dependencies. - Run `yarn worker` to start the server. diff --git a/manifest.json b/manifest.json index 0e83faf..4b05cc9 100644 --- a/manifest.json +++ b/manifest.json @@ -1,5 +1,5 @@ { "name": "@ubiquity-os/comment-vector-embeddings", "description": "Issue comment plugin for Ubiquibot. It enables the storage, updating, and deletion of issue comment embeddings.", - "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted"] + "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issue.created", "issue.edited", "issue.deleted"] } diff --git a/src/handlers/add-comments.ts b/src/handlers/add-comments.ts index bb0215c..4d3ad4b 100644 --- a/src/handlers/add-comments.ts +++ b/src/handlers/add-comments.ts @@ -1,11 +1,12 @@ import { Context } from "../types"; +import { CommentPayload } from "../types/payload"; export async function addComments(context: Context) { const { logger, - payload, adapters: { supabase }, } = context; + const { payload } = context as { payload: CommentPayload }; const plaintext = payload.comment.body; const authorId = payload.comment.user?.id || -1; const nodeId = payload.comment.node_id; diff --git a/src/handlers/add-issue.ts b/src/handlers/add-issue.ts new file mode 100644 index 0000000..ce7c315 --- /dev/null +++ b/src/handlers/add-issue.ts @@ -0,0 +1,29 @@ +import { Context } from "../types"; + +export async function addIssue(context: Context) { + const { + logger, + payload, + adapters: { supabase }, + } = context; + const payloadObject = payload; + const plaintext = payload.issue.body + payload.issue.title || ""; + const authorId = payload.issue.user?.id || -1; + const nodeId = payload.issue.node_id; + const isPrivate = payload.repository.private; + + try { + await supabase.issue.createIssue(nodeId, payloadObject, isPrivate, plaintext, authorId); + } catch (error) { + if (error instanceof Error) { + logger.error(`Error creating issue:`, { error: error, stack: error.stack }); + throw error; + } else { + logger.error(`Error creating issue:`, { err: error, error: new Error() }); + throw error; + } + } + + logger.ok(`Successfully created issue!`); + logger.verbose(`Exiting addIssue`); +} diff --git a/src/handlers/delete-comments.ts b/src/handlers/delete-comments.ts index 7b444ea..44fb8f2 100644 --- a/src/handlers/delete-comments.ts +++ b/src/handlers/delete-comments.ts @@ -1,12 +1,12 @@ import { Context } from "../types"; +import { CommentPayload } from "../types/payload"; export async function deleteComment(context: Context) { const { logger, - payload, adapters: { supabase }, } = context; - + const { payload } = context as { payload: CommentPayload }; const nodeId = payload.comment.node_id; try { diff --git a/src/handlers/delete-issue.ts b/src/handlers/delete-issue.ts new file mode 100644 index 0000000..ba16032 --- /dev/null +++ b/src/handlers/delete-issue.ts @@ -0,0 +1,26 @@ +import { Context } from "../types"; +import { IssuePayload } from "../types/payload"; + +export async function deleteIssues(context: Context) { + const { + logger, + adapters: { supabase }, + } = context; + const { payload } = context as { payload: IssuePayload }; + const nodeId = payload.issue.node_id; + + try { + await supabase.issue.deleteIssue(nodeId); + } catch (error) { + if (error instanceof Error) { + logger.error(`Error deleting issue:`, { error: error, stack: error.stack }); + throw error; + } else { + logger.error(`Error deleting issue:`, { err: error, error: new Error() }); + throw error; + } + } + + logger.ok(`Successfully deleted issue!`); + logger.verbose(`Exiting deleteIssue`); +} diff --git a/src/handlers/update-comments.ts b/src/handlers/update-comments.ts index f354d91..6dd2dae 100644 --- a/src/handlers/update-comments.ts +++ b/src/handlers/update-comments.ts @@ -1,11 +1,12 @@ import { Context } from "../types"; +import { CommentPayload } from "../types/payload"; export async function updateComment(context: Context) { const { logger, - payload, adapters: { supabase }, } = context; + const { payload } = context as { payload: CommentPayload }; const nodeId = payload.comment.node_id; const isPrivate = payload.repository.private; const plaintext = payload.comment.body; diff --git a/src/handlers/update-issue.ts b/src/handlers/update-issue.ts new file mode 100644 index 0000000..fad405c --- /dev/null +++ b/src/handlers/update-issue.ts @@ -0,0 +1,29 @@ +import { Context } from "../types"; +import { IssuePayload } from "../types/payload"; + +export async function updateIssue(context: Context) { + const { + logger, + adapters: { supabase }, + } = context; + const { payload } = context as { payload: IssuePayload }; + const payloadObject = payload; + const nodeId = payload.issue.node_id; + const isPrivate = payload.repository.private; + const plaintext = payload.issue.body + payload.issue.title || ""; + // Fetch the previous issue and update it in the db + try { + await supabase.issue.updateIssue(plaintext, nodeId, payloadObject, isPrivate); + } catch (error) { + if (error instanceof Error) { + logger.error(`Error updating issue:`, { error: error, stack: error.stack }); + throw error; + } else { + logger.error(`Error updating issue:`, { err: error, error: new Error() }); + throw error; + } + } + + logger.ok(`Successfully updated issue!`); + logger.verbose(`Exiting updateIssue`); +} diff --git a/src/plugin.ts b/src/plugin.ts index 9673231..aa9e871 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -1,7 +1,7 @@ import { Octokit } from "@octokit/rest"; import { Env, PluginInputs } from "./types"; import { Context } from "./types"; -import { isIssueCommentEvent } from "./types/typeguards"; +import { isIssueCommentEvent, isIssueEvent } from "./types/typeguards"; import { LogLevel, Logs } from "@ubiquity-dao/ubiquibot-logger"; import { Database } from "./types/database"; import { createAdapters } from "./adapters"; @@ -10,6 +10,9 @@ import { addComments } from "./handlers/add-comments"; import { updateComment } from "./handlers/update-comments"; import { deleteComment } from "./handlers/delete-comments"; import { VoyageAIClient } from "voyageai"; +import { deleteIssues } from "./handlers/delete-issue"; +import { addIssue } from "./handlers/add-issue"; +import { updateIssue } from "./handlers/update-issue"; /** * The main plugin function. Split for easier testing. @@ -25,6 +28,15 @@ export async function runPlugin(context: Context) { case "issue_comment.edited": return await updateComment(context); } + } else if (isIssueEvent(context)) { + switch (eventName) { + case "issues.created": + return await addIssue(context); + case "issues.deleted": + return await deleteIssues(context); + case "issues.edited": + return await updateIssue(context); + } } else { logger.error(`Unsupported event: ${eventName}`); } @@ -50,5 +62,7 @@ export async function plugin(inputs: PluginInputs, env: Env) { adapters: {} as ReturnType, }; context.adapters = createAdapters(supabase, voyageClient, context); - return runPlugin(context); + if (isIssueEvent(context)) { + return await runPlugin(context); + } } diff --git a/src/types/context.ts b/src/types/context.ts index d7e3cff..06de614 100644 --- a/src/types/context.ts +++ b/src/types/context.ts @@ -10,7 +10,13 @@ import { createAdapters } from "../adapters"; * * ubiquity:listeners: ["issue_comment.created", ...] */ -export type SupportedEventsU = "issue_comment.created" | "issue_comment.deleted" | "issue_comment.edited"; +export type SupportedEventsU = + | "issue_comment.created" + | "issue_comment.deleted" + | "issue_comment.edited" + | "issues.created" + | "issues.edited" + | "issues.deleted"; export type SupportedEvents = { [K in SupportedEventsU]: K extends WebhookEventName ? WebhookEvent : never; diff --git a/src/types/payload.ts b/src/types/payload.ts new file mode 100644 index 0000000..395fa09 --- /dev/null +++ b/src/types/payload.ts @@ -0,0 +1,3 @@ +import { EmitterWebhookEvent as WebhookEvent } from "@octokit/webhooks"; +export type CommentPayload = WebhookEvent<"issue_comment">["payload"]; +export type IssuePayload = WebhookEvent<"issues">["payload"]; diff --git a/src/types/typeguards.ts b/src/types/typeguards.ts index 266c856..8a7499c 100644 --- a/src/types/typeguards.ts +++ b/src/types/typeguards.ts @@ -14,3 +14,12 @@ import { Context } from "./context"; export function isIssueCommentEvent(context: Context): context is Context<"issue_comment.created" | "issue_comment.deleted" | "issue_comment.edited"> { return context.eventName === "issue_comment.created" || context.eventName === "issue_comment.deleted" || context.eventName === "issue_comment.edited"; } + +/** + * Restricts the scope of `context` to the `issues.created`, `issues.edited`, and `issues.deleted` payloads. + * + * @param context The context object. + */ +export function isIssueEvent(context: Context): context is Context<"issues.created" | "issues.edited" | "issues.deleted"> { + return context.eventName === "issues.created" || context.eventName === "issues.edited" || context.eventName === "issues.deleted"; +} diff --git a/tests/main.test.ts b/tests/main.test.ts index adf1481..40eeb7f 100644 --- a/tests/main.test.ts +++ b/tests/main.test.ts @@ -4,7 +4,7 @@ import { drop } from "@mswjs/data"; import { db } from "./__mocks__/db"; import { server } from "./__mocks__/node"; import { expect, describe, beforeAll, beforeEach, afterAll, afterEach, it } from "@jest/globals"; -import { Context } from "../src/types/context"; +import { Context, SupportedEvents } from "../src/types/context"; import { Octokit } from "@octokit/rest"; import { STRINGS } from "./__mocks__/strings"; import { createComment, setupTests } from "./__mocks__/helpers"; @@ -116,7 +116,9 @@ function createContext( const issue1 = db.issue.findFirst({ where: { id: { equals: issueOne } } }) as unknown as Context["payload"]["issue"]; createComment(commentBody, commentId, nodeId); // create it first then pull it from the DB and feed it to _createContext - const comment = db.issueComments.findFirst({ where: { id: { equals: commentId } } }) as unknown as Context["payload"]["comment"]; + const comment = db.issueComments.findFirst({ + where: { id: { equals: commentId } }, + }) as unknown as unknown as SupportedEvents["issue_comment.created"]["payload"]["comment"]; const context = createContextInner(repo, sender, issue1, comment, eventName); context.adapters = createMockAdapters(context) as unknown as Context["adapters"]; @@ -147,7 +149,7 @@ function createContextInner( repo: Context["payload"]["repository"], sender: Context["payload"]["sender"], issue: Context["payload"]["issue"], - comment: Context["payload"]["comment"], + comment: SupportedEvents["issue_comment.created"]["payload"]["comment"], eventName: Context["eventName"] = "issue_comment.created" ): Context { return { From d745150c141ee4be0d9271db6fc53bb8ce5e07e3 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Thu, 12 Sep 2024 21:29:40 -0400 Subject: [PATCH 26/55] fix: config --- .github/.ubiquibot-config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/.ubiquibot-config.yml b/.github/.ubiquibot-config.yml index bce9467..bc81e1a 100644 --- a/.github/.ubiquibot-config.yml +++ b/.github/.ubiquibot-config.yml @@ -1,5 +1,4 @@ id: test-app uses: - plugin: https://ubiquity-os-comment-vector-embeddings.sshivaditya.workers.dev - runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" ] runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issues.created", "issues.edited", "issues.deleted" ] \ No newline at end of file From 68c481a229f8bb7933a55d2565783f8b294f4731 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Thu, 12 Sep 2024 21:32:44 -0400 Subject: [PATCH 27/55] fix: update config --- .github/.ubiquibot-config.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/.ubiquibot-config.yml b/.github/.ubiquibot-config.yml index bc81e1a..1d024d0 100644 --- a/.github/.ubiquibot-config.yml +++ b/.github/.ubiquibot-config.yml @@ -1,4 +1,10 @@ id: test-app uses: - plugin: https://ubiquity-os-comment-vector-embeddings.sshivaditya.workers.dev - runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issues.created", "issues.edited", "issues.deleted" ] \ No newline at end of file + runsOn: + - issue_comment.created + - issue_comment.edited + - issue_comment.deleted + - issues.created + - issues.edited + - issues.deleted \ No newline at end of file From 8be23794f68abcad16906901e7360ee14b51598a Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Thu, 12 Sep 2024 23:25:38 -0400 Subject: [PATCH 28/55] fix: updated manifest.json --- .github/.ubiquibot-config.yml | 8 +------- manifest.json | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/.ubiquibot-config.yml b/.github/.ubiquibot-config.yml index 1d024d0..b15d7dd 100644 --- a/.github/.ubiquibot-config.yml +++ b/.github/.ubiquibot-config.yml @@ -1,10 +1,4 @@ id: test-app uses: - plugin: https://ubiquity-os-comment-vector-embeddings.sshivaditya.workers.dev - runsOn: - - issue_comment.created - - issue_comment.edited - - issue_comment.deleted - - issues.created - - issues.edited - - issues.deleted \ No newline at end of file + runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted"] \ No newline at end of file diff --git a/manifest.json b/manifest.json index 4b05cc9..0e83faf 100644 --- a/manifest.json +++ b/manifest.json @@ -1,5 +1,5 @@ { "name": "@ubiquity-os/comment-vector-embeddings", "description": "Issue comment plugin for Ubiquibot. It enables the storage, updating, and deletion of issue comment embeddings.", - "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issue.created", "issue.edited", "issue.deleted"] + "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted"] } From 9d9197ad4cd0ea880f3669fc22c4c92dc9973377 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Thu, 12 Sep 2024 23:32:41 -0400 Subject: [PATCH 29/55] fix: updated manifest.json --- manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifest.json b/manifest.json index 0e83faf..4b05cc9 100644 --- a/manifest.json +++ b/manifest.json @@ -1,5 +1,5 @@ { "name": "@ubiquity-os/comment-vector-embeddings", "description": "Issue comment plugin for Ubiquibot. It enables the storage, updating, and deletion of issue comment embeddings.", - "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted"] + "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issue.created", "issue.edited", "issue.deleted"] } From 22679ec2177f78119f245de91a13f67a41bd41e9 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Thu, 12 Sep 2024 23:36:34 -0400 Subject: [PATCH 30/55] fix: updated manifest.json and readme.md --- README.md | 2 +- manifest.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0bb10f4..db55b8a 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ To set up the `.dev.vars` file, you will need to provide the following variables - Add the following to your `.ubiquibot-config.yml` file with the appropriate URL: ```javascript -plugin: http://127.0.0.1:4000 - runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" , "issue.created", "issue.edited", "issue.deleted"] + runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" , "issues.created", "issues.edited", "issues.deleted"] ``` diff --git a/manifest.json b/manifest.json index 4b05cc9..8dc6dbd 100644 --- a/manifest.json +++ b/manifest.json @@ -1,5 +1,5 @@ { "name": "@ubiquity-os/comment-vector-embeddings", "description": "Issue comment plugin for Ubiquibot. It enables the storage, updating, and deletion of issue comment embeddings.", - "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issue.created", "issue.edited", "issue.deleted"] + "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issues.created", "issues.edited", "issues.deleted"] } From 2ea6146544acb2df097d9e61b18d4dfbea03cb65 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Thu, 12 Sep 2024 23:40:25 -0400 Subject: [PATCH 31/55] fix: issue.created to issues.opened --- README.md | 2 +- manifest.json | 2 +- src/plugin.ts | 2 +- src/types/context.ts | 2 +- src/types/typeguards.ts | 6 +++--- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index db55b8a..cac1f27 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ To set up the `.dev.vars` file, you will need to provide the following variables - Add the following to your `.ubiquibot-config.yml` file with the appropriate URL: ```javascript -plugin: http://127.0.0.1:4000 - runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" , "issues.created", "issues.edited", "issues.deleted"] + runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" , "issues.opened", "issues.edited", "issues.deleted"] ``` diff --git a/manifest.json b/manifest.json index 8dc6dbd..328a300 100644 --- a/manifest.json +++ b/manifest.json @@ -1,5 +1,5 @@ { "name": "@ubiquity-os/comment-vector-embeddings", "description": "Issue comment plugin for Ubiquibot. It enables the storage, updating, and deletion of issue comment embeddings.", - "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issues.created", "issues.edited", "issues.deleted"] + "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issues.opened", "issues.edited", "issues.deleted"] } diff --git a/src/plugin.ts b/src/plugin.ts index aa9e871..fa67919 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -30,7 +30,7 @@ export async function runPlugin(context: Context) { } } else if (isIssueEvent(context)) { switch (eventName) { - case "issues.created": + case "issues.opened": return await addIssue(context); case "issues.deleted": return await deleteIssues(context); diff --git a/src/types/context.ts b/src/types/context.ts index 06de614..1227abf 100644 --- a/src/types/context.ts +++ b/src/types/context.ts @@ -14,7 +14,7 @@ export type SupportedEventsU = | "issue_comment.created" | "issue_comment.deleted" | "issue_comment.edited" - | "issues.created" + | "issues.opened" | "issues.edited" | "issues.deleted"; diff --git a/src/types/typeguards.ts b/src/types/typeguards.ts index 8a7499c..01a6c26 100644 --- a/src/types/typeguards.ts +++ b/src/types/typeguards.ts @@ -16,10 +16,10 @@ export function isIssueCommentEvent(context: Context): context is Context<"issue } /** - * Restricts the scope of `context` to the `issues.created`, `issues.edited`, and `issues.deleted` payloads. + * Restricts the scope of `context` to the `issues.opened`, `issues.edited`, and `issues.deleted` payloads. * * @param context The context object. */ -export function isIssueEvent(context: Context): context is Context<"issues.created" | "issues.edited" | "issues.deleted"> { - return context.eventName === "issues.created" || context.eventName === "issues.edited" || context.eventName === "issues.deleted"; +export function isIssueEvent(context: Context): context is Context<"issues.opened" | "issues.edited" | "issues.deleted"> { + return context.eventName === "issues.opened" || context.eventName === "issues.edited" || context.eventName === "issues.deleted"; } From a0da267feb7690317a5ced93c5d2ef92708d3dfc Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Fri, 13 Sep 2024 00:38:30 -0400 Subject: [PATCH 32/55] fix: issue config removed updated schema --- .github/.ubiquibot-config.yml | 4 ---- src/adapters/supabase/helpers/comment.ts | 3 ++- src/adapters/supabase/helpers/issues.ts | 22 +++++++++---------- src/handlers/add-issue.ts | 8 +++---- src/handlers/update-issue.ts | 2 +- src/plugin.ts | 4 +--- .../20240912225853_issue_comments.sql | 1 + 7 files changed, 19 insertions(+), 25 deletions(-) delete mode 100644 .github/.ubiquibot-config.yml diff --git a/.github/.ubiquibot-config.yml b/.github/.ubiquibot-config.yml deleted file mode 100644 index b15d7dd..0000000 --- a/.github/.ubiquibot-config.yml +++ /dev/null @@ -1,4 +0,0 @@ - id: test-app - uses: - - plugin: https://ubiquity-os-comment-vector-embeddings.sshivaditya.workers.dev - runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted"] \ No newline at end of file diff --git a/src/adapters/supabase/helpers/comment.ts b/src/adapters/supabase/helpers/comment.ts index 0f59531..e64402b 100644 --- a/src/adapters/supabase/helpers/comment.ts +++ b/src/adapters/supabase/helpers/comment.ts @@ -42,8 +42,9 @@ export class Comment extends SuperSupabase { } const { error } = await this.supabase .from("issue_comments") - .insert([{ id: commentNodeId, plaintext, author_id: authorId, type: "comment", payload, embedding: embedding, issueId }]); + .insert([{ id: commentNodeId, plaintext, author_id: authorId, type: "comment", payload, embedding: embedding, issue_id: issueId }]); if (error) { + console.log(error.message, error.details, { id: commentNodeId, plaintext, author_id: authorId, type: "comment", payload, embedding }); this.context.logger.error("Error creating comment", error); return; } diff --git a/src/adapters/supabase/helpers/issues.ts b/src/adapters/supabase/helpers/issues.ts index e7d7e95..0040796 100644 --- a/src/adapters/supabase/helpers/issues.ts +++ b/src/adapters/supabase/helpers/issues.ts @@ -17,8 +17,9 @@ export class Issues extends SuperSupabase { super(supabase, context); } - async createIssue(issueNodeId: string, payloadObject: Record | null, isPrivate: boolean, plaintext: string | null, authorId: number) { + async createIssue(issueNodeId: string, payload: Record | null, isPrivate: boolean, plaintext: string | null, authorId: number) { //First Check if the issue already exists + console.log(payload); const { data, error } = await this.supabase.from("issues").select("*").eq("id", issueNodeId); if (error) { this.context.logger.error("Error creating issue", error); @@ -30,13 +31,12 @@ export class Issues extends SuperSupabase { } else { const embedding = await this.context.adapters.voyage.embedding.createEmbedding(plaintext); if (isPrivate) { - payloadObject = null; + payload = null; plaintext = null; } - const { error } = await this.supabase - .from("issues") - .insert([{ id: issueNodeId, payloadObject, type: "issue", plaintext, author_id: authorId, embedding }]); + const { error } = await this.supabase.from("issues").insert([{ id: issueNodeId, payload, type: "issue", plaintext, author_id: authorId, embedding }]); if (error) { + console.log(error.message, error.details, { id: issueNodeId, payload, type: "issue", plaintext, author_id: authorId, embedding }); this.context.logger.error("Error creating issue", error); return; } @@ -44,24 +44,22 @@ export class Issues extends SuperSupabase { this.context.logger.info("Issue created successfully"); } - async updateIssue(plaintext: string | null, issueNodeId: string, payloadObject: Record | null, isPrivate: boolean) { + async updateIssue(plaintext: string | null, issueNodeId: string, payload: Record | null, isPrivate: boolean) { //Create the embedding for this comment const embedding = Array.from(await this.context.adapters.voyage.embedding.createEmbedding(plaintext)); if (isPrivate) { plaintext = null as string | null; - payloadObject = null as Record | null; + payload = null as Record | null; } - const { error } = await this.supabase - .from("issue_comments") - .update({ plaintext, embedding: embedding, payloadObject, modified_at: new Date() }) - .eq("id", issueNodeId); + console.log({ id: issueNodeId, plaintext, payload, embedding }); + const { error } = await this.supabase.from("issues").update({ plaintext, embedding: embedding, payload, modified_at: new Date() }).eq("id", issueNodeId); if (error) { this.context.logger.error("Error updating comment", error); } } async deleteIssue(issueNodeId: string) { - const { error } = await this.supabase.from("issue_comments").delete().eq("id", issueNodeId); + const { error } = await this.supabase.from("issues").delete().eq("id", issueNodeId); if (error) { this.context.logger.error("Error deleting comment", error); } diff --git a/src/handlers/add-issue.ts b/src/handlers/add-issue.ts index ce7c315..4264a44 100644 --- a/src/handlers/add-issue.ts +++ b/src/handlers/add-issue.ts @@ -1,19 +1,19 @@ import { Context } from "../types"; +import { IssuePayload } from "../types/payload"; export async function addIssue(context: Context) { const { logger, - payload, adapters: { supabase }, } = context; - const payloadObject = payload; - const plaintext = payload.issue.body + payload.issue.title || ""; + const { payload } = context as { payload: IssuePayload }; + const plaintext = payload.issue.body + " " + payload.issue.title || ""; const authorId = payload.issue.user?.id || -1; const nodeId = payload.issue.node_id; const isPrivate = payload.repository.private; try { - await supabase.issue.createIssue(nodeId, payloadObject, isPrivate, plaintext, authorId); + await supabase.issue.createIssue(nodeId, payload, isPrivate, plaintext, authorId); } catch (error) { if (error instanceof Error) { logger.error(`Error creating issue:`, { error: error, stack: error.stack }); diff --git a/src/handlers/update-issue.ts b/src/handlers/update-issue.ts index fad405c..64f3548 100644 --- a/src/handlers/update-issue.ts +++ b/src/handlers/update-issue.ts @@ -10,7 +10,7 @@ export async function updateIssue(context: Context) { const payloadObject = payload; const nodeId = payload.issue.node_id; const isPrivate = payload.repository.private; - const plaintext = payload.issue.body + payload.issue.title || ""; + const plaintext = payload.issue.body + " " + payload.issue.title || ""; // Fetch the previous issue and update it in the db try { await supabase.issue.updateIssue(plaintext, nodeId, payloadObject, isPrivate); diff --git a/src/plugin.ts b/src/plugin.ts index fa67919..2f7dad8 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -62,7 +62,5 @@ export async function plugin(inputs: PluginInputs, env: Env) { adapters: {} as ReturnType, }; context.adapters = createAdapters(supabase, voyageClient, context); - if (isIssueEvent(context)) { - return await runPlugin(context); - } + return await runPlugin(context); } diff --git a/supabase/migrations/20240912225853_issue_comments.sql b/supabase/migrations/20240912225853_issue_comments.sql index 3a8764a..a6e24b4 100644 --- a/supabase/migrations/20240912225853_issue_comments.sql +++ b/supabase/migrations/20240912225853_issue_comments.sql @@ -6,6 +6,7 @@ CREATE TABLE IF NOT EXISTS issues ( plaintext text, embedding Vector(1024) not null, payload jsonb, + author_id VARCHAR not null, type text not null default 'issue', created_at timestamptz not null default now(), modified_at timestamptz not null default now() From 8d25ce12fe0c14c8d01864d3ab1d8138a86ec770 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Fri, 13 Sep 2024 01:05:03 -0400 Subject: [PATCH 33/55] fix: remove console.log --- src/adapters/supabase/helpers/comment.ts | 1 - src/adapters/supabase/helpers/issues.ts | 3 --- 2 files changed, 4 deletions(-) diff --git a/src/adapters/supabase/helpers/comment.ts b/src/adapters/supabase/helpers/comment.ts index e64402b..f2ff62a 100644 --- a/src/adapters/supabase/helpers/comment.ts +++ b/src/adapters/supabase/helpers/comment.ts @@ -44,7 +44,6 @@ export class Comment extends SuperSupabase { .from("issue_comments") .insert([{ id: commentNodeId, plaintext, author_id: authorId, type: "comment", payload, embedding: embedding, issue_id: issueId }]); if (error) { - console.log(error.message, error.details, { id: commentNodeId, plaintext, author_id: authorId, type: "comment", payload, embedding }); this.context.logger.error("Error creating comment", error); return; } diff --git a/src/adapters/supabase/helpers/issues.ts b/src/adapters/supabase/helpers/issues.ts index 0040796..7624ff8 100644 --- a/src/adapters/supabase/helpers/issues.ts +++ b/src/adapters/supabase/helpers/issues.ts @@ -19,7 +19,6 @@ export class Issues extends SuperSupabase { async createIssue(issueNodeId: string, payload: Record | null, isPrivate: boolean, plaintext: string | null, authorId: number) { //First Check if the issue already exists - console.log(payload); const { data, error } = await this.supabase.from("issues").select("*").eq("id", issueNodeId); if (error) { this.context.logger.error("Error creating issue", error); @@ -36,7 +35,6 @@ export class Issues extends SuperSupabase { } const { error } = await this.supabase.from("issues").insert([{ id: issueNodeId, payload, type: "issue", plaintext, author_id: authorId, embedding }]); if (error) { - console.log(error.message, error.details, { id: issueNodeId, payload, type: "issue", plaintext, author_id: authorId, embedding }); this.context.logger.error("Error creating issue", error); return; } @@ -51,7 +49,6 @@ export class Issues extends SuperSupabase { plaintext = null as string | null; payload = null as Record | null; } - console.log({ id: issueNodeId, plaintext, payload, embedding }); const { error } = await this.supabase.from("issues").update({ plaintext, embedding: embedding, payload, modified_at: new Date() }).eq("id", issueNodeId); if (error) { this.context.logger.error("Error updating comment", error); From 1a1599717374d76918fe5e4970bd8b6d7f8815be Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Fri, 13 Sep 2024 03:20:27 -0400 Subject: [PATCH 34/55] feat: added cols markdown and plaintext, adds code for conversion from markdown to plaintext --- eslint.config.mjs | 2 +- src/adapters/supabase/helpers/comment.ts | 23 +++++++++------ src/adapters/supabase/helpers/issues.ts | 28 ++++++++++++------- src/adapters/utils/markdown-to-plaintext.ts | 20 +++++++++++++ src/handlers/add-issue.ts | 4 +-- src/handlers/update-comments.ts | 4 +-- src/handlers/update-issue.ts | 4 +-- .../20240912225853_issue_comments.sql | 4 ++- .../20240913070225_issue_comments.sql | 5 ++++ 9 files changed, 67 insertions(+), 27 deletions(-) create mode 100644 src/adapters/utils/markdown-to-plaintext.ts create mode 100644 supabase/migrations/20240913070225_issue_comments.sql diff --git a/eslint.config.mjs b/eslint.config.mjs index f43237a..f594657 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -9,7 +9,7 @@ export default tsEslint.config({ "@typescript-eslint": tsEslint.plugin, "check-file": checkFile, }, - ignores: [".github/knip.ts", "src/types/database.ts"], + ignores: [".github/knip.ts", "src/types/database.ts", "src/adapters/utils/markdown-to-plaintext.ts"], extends: [eslint.configs.recommended, ...tsEslint.configs.recommended, sonarjs.configs.recommended], languageOptions: { parser: tsEslint.parser, diff --git a/src/adapters/supabase/helpers/comment.ts b/src/adapters/supabase/helpers/comment.ts index f2ff62a..3fa08b2 100644 --- a/src/adapters/supabase/helpers/comment.ts +++ b/src/adapters/supabase/helpers/comment.ts @@ -1,10 +1,11 @@ import { SupabaseClient } from "@supabase/supabase-js"; import { SuperSupabase } from "./supabase"; import { Context } from "../../../types/context"; +import { markdownToPlainText } from "../../utils/markdown-to-plaintext"; export interface CommentType { id: string; - plaintext?: string; + markdown?: string; author_id: number; created_at: string; modified_at: string; @@ -17,7 +18,7 @@ export class Comment extends SuperSupabase { } async createComment( - plaintext: string | null, + markdown: string | null, commentNodeId: string, authorId: number, payload: Record | null, @@ -35,14 +36,16 @@ export class Comment extends SuperSupabase { return; } else { //Create the embedding for this comment - const embedding = await this.context.adapters.voyage.embedding.createEmbedding(plaintext); + const embedding = await this.context.adapters.voyage.embedding.createEmbedding(markdown); + let plaintext: string | null = markdownToPlainText(markdown || ""); if (isPrivate) { - plaintext = null as string | null; + markdown = null as string | null; payload = null as Record | null; + plaintext = null as string | null; } const { error } = await this.supabase .from("issue_comments") - .insert([{ id: commentNodeId, plaintext, author_id: authorId, type: "comment", payload, embedding: embedding, issue_id: issueId }]); + .insert([{ id: commentNodeId, markdown, plaintext, author_id: authorId, payload, embedding: embedding, issue_id: issueId }]); if (error) { this.context.logger.error("Error creating comment", error); return; @@ -51,16 +54,18 @@ export class Comment extends SuperSupabase { this.context.logger.info("Comment created successfully"); } - async updateComment(plaintext: string | null, commentNodeId: string, payload: Record | null, isPrivate: boolean) { + async updateComment(markdown: string | null, commentNodeId: string, payload: Record | null, isPrivate: boolean) { //Create the embedding for this comment - const embedding = Array.from(await this.context.adapters.voyage.embedding.createEmbedding(plaintext)); + const embedding = Array.from(await this.context.adapters.voyage.embedding.createEmbedding(markdown)); + let plaintext: string | null = markdownToPlainText(markdown || ""); if (isPrivate) { - plaintext = null as string | null; + markdown = null as string | null; payload = null as Record | null; + plaintext = null as string | null; } const { error } = await this.supabase .from("issue_comments") - .update({ plaintext, embedding: embedding, payload, modified_at: new Date() }) + .update({ markdown, plaintext, embedding: embedding, payload, modified_at: new Date() }) .eq("id", commentNodeId); if (error) { this.context.logger.error("Error updating comment", error); diff --git a/src/adapters/supabase/helpers/issues.ts b/src/adapters/supabase/helpers/issues.ts index 7624ff8..063d8a9 100644 --- a/src/adapters/supabase/helpers/issues.ts +++ b/src/adapters/supabase/helpers/issues.ts @@ -1,10 +1,11 @@ import { SupabaseClient } from "@supabase/supabase-js"; import { SuperSupabase } from "./supabase"; import { Context } from "../../../types/context"; +import { markdownToPlainText } from "../../utils/markdown-to-plaintext"; export interface IssueType { id: string; - plaintext?: string; + markdown?: string; author_id: number; created_at: string; modified_at: string; @@ -17,7 +18,7 @@ export class Issues extends SuperSupabase { super(supabase, context); } - async createIssue(issueNodeId: string, payload: Record | null, isPrivate: boolean, plaintext: string | null, authorId: number) { + async createIssue(issueNodeId: string, payload: Record | null, isPrivate: boolean, markdown: string | null, authorId: number) { //First Check if the issue already exists const { data, error } = await this.supabase.from("issues").select("*").eq("id", issueNodeId); if (error) { @@ -28,12 +29,14 @@ export class Issues extends SuperSupabase { this.context.logger.info("Issue already exists"); return; } else { - const embedding = await this.context.adapters.voyage.embedding.createEmbedding(plaintext); + const embedding = await this.context.adapters.voyage.embedding.createEmbedding(markdown); + let plaintext: string | null = markdownToPlainText(markdown || ""); if (isPrivate) { payload = null; + markdown = null; plaintext = null; } - const { error } = await this.supabase.from("issues").insert([{ id: issueNodeId, payload, type: "issue", plaintext, author_id: authorId, embedding }]); + const { error } = await this.supabase.from("issues").insert([{ id: issueNodeId, payload, markdown, plaintext, author_id: authorId, embedding }]); if (error) { this.context.logger.error("Error creating issue", error); return; @@ -42,14 +45,19 @@ export class Issues extends SuperSupabase { this.context.logger.info("Issue created successfully"); } - async updateIssue(plaintext: string | null, issueNodeId: string, payload: Record | null, isPrivate: boolean) { + async updateIssue(markdown: string | null, issueNodeId: string, payload: Record | null, isPrivate: boolean) { //Create the embedding for this comment - const embedding = Array.from(await this.context.adapters.voyage.embedding.createEmbedding(plaintext)); + const embedding = Array.from(await this.context.adapters.voyage.embedding.createEmbedding(markdown)); + let plaintext: string | null = markdownToPlainText(markdown || ""); if (isPrivate) { - plaintext = null as string | null; + markdown = null as string | null; payload = null as Record | null; + plaintext = null as string | null; } - const { error } = await this.supabase.from("issues").update({ plaintext, embedding: embedding, payload, modified_at: new Date() }).eq("id", issueNodeId); + const { error } = await this.supabase + .from("issues") + .update({ markdown, plaintext, embedding: embedding, payload, modified_at: new Date() }) + .eq("id", issueNodeId); if (error) { this.context.logger.error("Error updating comment", error); } @@ -62,8 +70,8 @@ export class Issues extends SuperSupabase { } } - async findSimilarIssues(plaintext: string, threshold: number): Promise { - const embedding = await this.context.adapters.voyage.embedding.createEmbedding(plaintext); + async findSimilarIssues(markdown: string, threshold: number): Promise { + const embedding = await this.context.adapters.voyage.embedding.createEmbedding(markdown); const { data, error } = await this.supabase .from("issues") .select("*") diff --git a/src/adapters/utils/markdown-to-plaintext.ts b/src/adapters/utils/markdown-to-plaintext.ts new file mode 100644 index 0000000..5edf926 --- /dev/null +++ b/src/adapters/utils/markdown-to-plaintext.ts @@ -0,0 +1,20 @@ +/** + * Converts a Markdown string to plain text. + * @param markdown + * @returns + */ +export function markdownToPlainText(markdown: string): string { + let text = markdown.replace(/^#{1,6}\s+/gm, ""); // Remove headers + text = text.replace(/\[([^\]]+)\]\([^\)]+\)/g, "$1"); // Inline links + text = text.replace(/!\[([^\]]*)\]\([^\)]+\)/g, "$1"); // Inline images + text = text.replace(/```[\s\S]*?```/g, (match) => match.replace(/```/g, "").trim()); // Code blocks + text = text.replace(/`([^`]+)`/g, "$1"); // Inline code + text = text.replace(/(\*\*|__)(.*?)\1/g, "$2"); // Bold + text = text.replace(/(\*|_)(.*?)\1/g, "$2"); // Italic + text = text.replace(/~~(.*?)~~/g, "$1"); // Strikethrough + text = text.replace(/^>\s+/gm, ""); // Block quotes + text = text.replace(/^\s*[-*]{3,}\s*$/gm, ""); // Horizontal rules + text = text.replace(/\n{3,}/g, "\n\n"); // Remove extra newlines + text = text.replace(/\s+/g, " ").trim(); // Remove extra spaces + return text; +} diff --git a/src/handlers/add-issue.ts b/src/handlers/add-issue.ts index 4264a44..2afec8c 100644 --- a/src/handlers/add-issue.ts +++ b/src/handlers/add-issue.ts @@ -7,13 +7,13 @@ export async function addIssue(context: Context) { adapters: { supabase }, } = context; const { payload } = context as { payload: IssuePayload }; - const plaintext = payload.issue.body + " " + payload.issue.title || ""; + const markdown = payload.issue.body + " " + payload.issue.title || ""; const authorId = payload.issue.user?.id || -1; const nodeId = payload.issue.node_id; const isPrivate = payload.repository.private; try { - await supabase.issue.createIssue(nodeId, payload, isPrivate, plaintext, authorId); + await supabase.issue.createIssue(nodeId, payload, isPrivate, markdown, authorId); } catch (error) { if (error instanceof Error) { logger.error(`Error creating issue:`, { error: error, stack: error.stack }); diff --git a/src/handlers/update-comments.ts b/src/handlers/update-comments.ts index 6dd2dae..94e9715 100644 --- a/src/handlers/update-comments.ts +++ b/src/handlers/update-comments.ts @@ -9,10 +9,10 @@ export async function updateComment(context: Context) { const { payload } = context as { payload: CommentPayload }; const nodeId = payload.comment.node_id; const isPrivate = payload.repository.private; - const plaintext = payload.comment.body; + const markdown = payload.comment.body; // Fetch the previous comment and update it in the db try { - await supabase.comment.updateComment(plaintext, nodeId, payload, isPrivate); + await supabase.comment.updateComment(markdown, nodeId, payload, isPrivate); } catch (error) { if (error instanceof Error) { logger.error(`Error updating comment:`, { error: error, stack: error.stack }); diff --git a/src/handlers/update-issue.ts b/src/handlers/update-issue.ts index 64f3548..488aec7 100644 --- a/src/handlers/update-issue.ts +++ b/src/handlers/update-issue.ts @@ -10,10 +10,10 @@ export async function updateIssue(context: Context) { const payloadObject = payload; const nodeId = payload.issue.node_id; const isPrivate = payload.repository.private; - const plaintext = payload.issue.body + " " + payload.issue.title || ""; + const markdown = payload.issue.body + " " + payload.issue.title || ""; // Fetch the previous issue and update it in the db try { - await supabase.issue.updateIssue(plaintext, nodeId, payloadObject, isPrivate); + await supabase.issue.updateIssue(markdown, nodeId, payloadObject, isPrivate); } catch (error) { if (error instanceof Error) { logger.error(`Error updating issue:`, { error: error, stack: error.stack }); diff --git a/supabase/migrations/20240912225853_issue_comments.sql b/supabase/migrations/20240912225853_issue_comments.sql index a6e24b4..0d71cab 100644 --- a/supabase/migrations/20240912225853_issue_comments.sql +++ b/supabase/migrations/20240912225853_issue_comments.sql @@ -7,7 +7,6 @@ CREATE TABLE IF NOT EXISTS issues ( embedding Vector(1024) not null, payload jsonb, author_id VARCHAR not null, - type text not null default 'issue', created_at timestamptz not null default now(), modified_at timestamptz not null default now() ); @@ -17,5 +16,8 @@ ADD COLUMN issue_id VARCHAR REFERENCES issues(id) ON DELETE CASCADE; +ALTER TABLE issue_comments +DROP COLUMN type; + ALTER TABLE issue_comments RENAME COLUMN payloadobject TO payload; \ No newline at end of file diff --git a/supabase/migrations/20240913070225_issue_comments.sql b/supabase/migrations/20240913070225_issue_comments.sql new file mode 100644 index 0000000..09daf65 --- /dev/null +++ b/supabase/migrations/20240913070225_issue_comments.sql @@ -0,0 +1,5 @@ +ALTER TABLE issue_comments +ADD COLUMN markdown TEXT; + +ALTER TABLE issues +ADD COLUMN markdown TEXT; \ No newline at end of file From a4bcc711d361c6e3e60fa3864da474fcddcab0fc Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Fri, 13 Sep 2024 03:42:47 -0400 Subject: [PATCH 35/55] fix: fixed verbose to debug, changed model to voyage-large-2-instruct --- src/adapters/utils/markdown-to-plaintext.ts | 5 ++++- src/adapters/voyage/helpers/embedding.ts | 2 +- src/handlers/add-comments.ts | 6 +++--- src/handlers/add-issue.ts | 4 ++-- src/handlers/delete-comments.ts | 2 +- src/handlers/delete-issue.ts | 2 +- src/handlers/update-comments.ts | 4 ++-- src/handlers/update-issue.ts | 4 ++-- 8 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/adapters/utils/markdown-to-plaintext.ts b/src/adapters/utils/markdown-to-plaintext.ts index 5edf926..448a043 100644 --- a/src/adapters/utils/markdown-to-plaintext.ts +++ b/src/adapters/utils/markdown-to-plaintext.ts @@ -3,7 +3,10 @@ * @param markdown * @returns */ -export function markdownToPlainText(markdown: string): string { +export function markdownToPlainText(markdown: string | null): string | null { + if (!markdown) { + return markdown; + } let text = markdown.replace(/^#{1,6}\s+/gm, ""); // Remove headers text = text.replace(/\[([^\]]+)\]\([^\)]+\)/g, "$1"); // Inline links text = text.replace(/!\[([^\]]*)\]\([^\)]+\)/g, "$1"); // Inline images diff --git a/src/adapters/voyage/helpers/embedding.ts b/src/adapters/voyage/helpers/embedding.ts index cacc320..575543e 100644 --- a/src/adapters/voyage/helpers/embedding.ts +++ b/src/adapters/voyage/helpers/embedding.ts @@ -17,7 +17,7 @@ export class Embedding extends SuperVoyage { } else { const response = await this.client.embed({ input: text, - model: "voyage-2", + model: "voyage-large-2-instruct", }); return (response.data && response.data[0]?.embedding) || []; } diff --git a/src/handlers/add-comments.ts b/src/handlers/add-comments.ts index 4d3ad4b..5a3ef1c 100644 --- a/src/handlers/add-comments.ts +++ b/src/handlers/add-comments.ts @@ -7,14 +7,14 @@ export async function addComments(context: Context) { adapters: { supabase }, } = context; const { payload } = context as { payload: CommentPayload }; - const plaintext = payload.comment.body; + const markdown = payload.comment.body || null; const authorId = payload.comment.user?.id || -1; const nodeId = payload.comment.node_id; const isPrivate = payload.repository.private; const issueId = payload.issue.node_id; try { - await supabase.comment.createComment(plaintext, nodeId, authorId, payload, isPrivate, issueId); + await supabase.comment.createComment(markdown, nodeId, authorId, payload, isPrivate, issueId); } catch (error) { if (error instanceof Error) { logger.error(`Error creating comment:`, { error: error, stack: error.stack }); @@ -26,5 +26,5 @@ export async function addComments(context: Context) { } logger.ok(`Successfully created comment!`); - logger.verbose(`Exiting addComments`); + logger.debug(`Exiting addComments`); } diff --git a/src/handlers/add-issue.ts b/src/handlers/add-issue.ts index 2afec8c..8cf0449 100644 --- a/src/handlers/add-issue.ts +++ b/src/handlers/add-issue.ts @@ -7,7 +7,7 @@ export async function addIssue(context: Context) { adapters: { supabase }, } = context; const { payload } = context as { payload: IssuePayload }; - const markdown = payload.issue.body + " " + payload.issue.title || ""; + const markdown = payload.issue.body + " " + payload.issue.title || null; const authorId = payload.issue.user?.id || -1; const nodeId = payload.issue.node_id; const isPrivate = payload.repository.private; @@ -25,5 +25,5 @@ export async function addIssue(context: Context) { } logger.ok(`Successfully created issue!`); - logger.verbose(`Exiting addIssue`); + logger.debug(`Exiting addIssue`); } diff --git a/src/handlers/delete-comments.ts b/src/handlers/delete-comments.ts index 44fb8f2..8c9e394 100644 --- a/src/handlers/delete-comments.ts +++ b/src/handlers/delete-comments.ts @@ -22,5 +22,5 @@ export async function deleteComment(context: Context) { } logger.ok(`Successfully deleted comment!`); - logger.verbose(`Exiting deleteComments`); + logger.debug(`Exiting deleteComments`); } diff --git a/src/handlers/delete-issue.ts b/src/handlers/delete-issue.ts index ba16032..b392c69 100644 --- a/src/handlers/delete-issue.ts +++ b/src/handlers/delete-issue.ts @@ -22,5 +22,5 @@ export async function deleteIssues(context: Context) { } logger.ok(`Successfully deleted issue!`); - logger.verbose(`Exiting deleteIssue`); + logger.debug(`Exiting deleteIssue`); } diff --git a/src/handlers/update-comments.ts b/src/handlers/update-comments.ts index 94e9715..7d103d9 100644 --- a/src/handlers/update-comments.ts +++ b/src/handlers/update-comments.ts @@ -9,7 +9,7 @@ export async function updateComment(context: Context) { const { payload } = context as { payload: CommentPayload }; const nodeId = payload.comment.node_id; const isPrivate = payload.repository.private; - const markdown = payload.comment.body; + const markdown = payload.comment.body || null; // Fetch the previous comment and update it in the db try { await supabase.comment.updateComment(markdown, nodeId, payload, isPrivate); @@ -24,5 +24,5 @@ export async function updateComment(context: Context) { } logger.ok(`Successfully updated comment!`); - logger.verbose(`Exiting updateComment`); + logger.debug(`Exiting updateComment`); } diff --git a/src/handlers/update-issue.ts b/src/handlers/update-issue.ts index 488aec7..b7b5cd9 100644 --- a/src/handlers/update-issue.ts +++ b/src/handlers/update-issue.ts @@ -10,7 +10,7 @@ export async function updateIssue(context: Context) { const payloadObject = payload; const nodeId = payload.issue.node_id; const isPrivate = payload.repository.private; - const markdown = payload.issue.body + " " + payload.issue.title || ""; + const markdown = payload.issue.body + " " + payload.issue.title || null; // Fetch the previous issue and update it in the db try { await supabase.issue.updateIssue(markdown, nodeId, payloadObject, isPrivate); @@ -25,5 +25,5 @@ export async function updateIssue(context: Context) { } logger.ok(`Successfully updated issue!`); - logger.verbose(`Exiting updateIssue`); + logger.debug(`Exiting updateIssue`); } From 6beefba462b7dbb828f580b2535cdbfb2a210949 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Fri, 13 Sep 2024 03:46:32 -0400 Subject: [PATCH 36/55] fix: throw error for empty body --- src/handlers/add-comments.ts | 5 ++++- src/handlers/add-issue.ts | 3 +++ src/handlers/update-comments.ts | 3 +++ src/handlers/update-issue.ts | 3 +++ 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/handlers/add-comments.ts b/src/handlers/add-comments.ts index 5a3ef1c..54745a1 100644 --- a/src/handlers/add-comments.ts +++ b/src/handlers/add-comments.ts @@ -7,13 +7,16 @@ export async function addComments(context: Context) { adapters: { supabase }, } = context; const { payload } = context as { payload: CommentPayload }; - const markdown = payload.comment.body || null; + const markdown = payload.comment.body; const authorId = payload.comment.user?.id || -1; const nodeId = payload.comment.node_id; const isPrivate = payload.repository.private; const issueId = payload.issue.node_id; try { + if (!markdown) { + throw new Error("Comment body is empty"); + } await supabase.comment.createComment(markdown, nodeId, authorId, payload, isPrivate, issueId); } catch (error) { if (error instanceof Error) { diff --git a/src/handlers/add-issue.ts b/src/handlers/add-issue.ts index 8cf0449..969a5c2 100644 --- a/src/handlers/add-issue.ts +++ b/src/handlers/add-issue.ts @@ -13,6 +13,9 @@ export async function addIssue(context: Context) { const isPrivate = payload.repository.private; try { + if (!markdown) { + throw new Error("Issue body is empty"); + } await supabase.issue.createIssue(nodeId, payload, isPrivate, markdown, authorId); } catch (error) { if (error instanceof Error) { diff --git a/src/handlers/update-comments.ts b/src/handlers/update-comments.ts index 7d103d9..b1b9d18 100644 --- a/src/handlers/update-comments.ts +++ b/src/handlers/update-comments.ts @@ -12,6 +12,9 @@ export async function updateComment(context: Context) { const markdown = payload.comment.body || null; // Fetch the previous comment and update it in the db try { + if (!markdown) { + throw new Error("Comment body is empty"); + } await supabase.comment.updateComment(markdown, nodeId, payload, isPrivate); } catch (error) { if (error instanceof Error) { diff --git a/src/handlers/update-issue.ts b/src/handlers/update-issue.ts index b7b5cd9..763b2ba 100644 --- a/src/handlers/update-issue.ts +++ b/src/handlers/update-issue.ts @@ -13,6 +13,9 @@ export async function updateIssue(context: Context) { const markdown = payload.issue.body + " " + payload.issue.title || null; // Fetch the previous issue and update it in the db try { + if (!markdown) { + throw new Error("Issue body is empty"); + } await supabase.issue.updateIssue(markdown, nodeId, payloadObject, isPrivate); } catch (error) { if (error instanceof Error) { From 0d5d9758ef7c992e0beb58d2df5f007e635320e6 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Fri, 13 Sep 2024 03:55:49 -0400 Subject: [PATCH 37/55] fix: removed custom code for conversion of markdown to plaintext --- package.json | 3 + src/adapters/utils/markdown-to-plaintext.ts | 21 +++---- yarn.lock | 62 +++++++++++++++++++++ 3 files changed, 73 insertions(+), 13 deletions(-) diff --git a/package.json b/package.json index 7718367..6ef92a4 100644 --- a/package.json +++ b/package.json @@ -35,8 +35,11 @@ "@octokit/webhooks": "13.2.7", "@sinclair/typebox": "0.32.33", "@supabase/supabase-js": "^2.45.2", + "@types/markdown-it": "^14.1.2", "@ubiquity-dao/ubiquibot-logger": "^1.3.0", "dotenv": "16.4.5", + "markdown-it": "^14.1.0", + "markdown-it-plain-text": "^0.3.0", "typebox-validators": "0.3.5", "voyageai": "^0.0.1-5" }, diff --git a/src/adapters/utils/markdown-to-plaintext.ts b/src/adapters/utils/markdown-to-plaintext.ts index 448a043..93e5787 100644 --- a/src/adapters/utils/markdown-to-plaintext.ts +++ b/src/adapters/utils/markdown-to-plaintext.ts @@ -1,3 +1,6 @@ +import markdownit from "markdown-it"; +import plainTextPlugin from "markdown-it-plain-text"; + /** * Converts a Markdown string to plain text. * @param markdown @@ -7,17 +10,9 @@ export function markdownToPlainText(markdown: string | null): string | null { if (!markdown) { return markdown; } - let text = markdown.replace(/^#{1,6}\s+/gm, ""); // Remove headers - text = text.replace(/\[([^\]]+)\]\([^\)]+\)/g, "$1"); // Inline links - text = text.replace(/!\[([^\]]*)\]\([^\)]+\)/g, "$1"); // Inline images - text = text.replace(/```[\s\S]*?```/g, (match) => match.replace(/```/g, "").trim()); // Code blocks - text = text.replace(/`([^`]+)`/g, "$1"); // Inline code - text = text.replace(/(\*\*|__)(.*?)\1/g, "$2"); // Bold - text = text.replace(/(\*|_)(.*?)\1/g, "$2"); // Italic - text = text.replace(/~~(.*?)~~/g, "$1"); // Strikethrough - text = text.replace(/^>\s+/gm, ""); // Block quotes - text = text.replace(/^\s*[-*]{3,}\s*$/gm, ""); // Horizontal rules - text = text.replace(/\n{3,}/g, "\n\n"); // Remove extra newlines - text = text.replace(/\s+/g, " ").trim(); // Remove extra spaces - return text; + const md = markdownit(); + md.use(plainTextPlugin); + md.render(markdown); + //Package markdown-it-plain-text does not have types + return (md as any).plainText; } diff --git a/yarn.lock b/yarn.lock index e653515..8439328 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1961,16 +1961,34 @@ expect "^29.0.0" pretty-format "^29.0.0" +"@types/linkify-it@^5": + version "5.0.0" + resolved "https://registry.yarnpkg.com/@types/linkify-it/-/linkify-it-5.0.0.tgz#21413001973106cda1c3a9b91eedd4ccd5469d76" + integrity sha512-sVDA58zAw4eWAffKOaQH5/5j3XeayukzDk+ewSsnv3p4yJEZHCCzMDiZM8e0OUrRvmpGZ85jf4yDHkHsgBNr9Q== + "@types/lodash@^4.14.172": version "4.17.4" resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.17.4.tgz#0303b64958ee070059e3a7184048a55159fe20b7" integrity sha512-wYCP26ZLxaT3R39kiN2+HcJ4kTd3U1waI/cY7ivWYqFP6pW3ZNpvi6Wd6PHZx7T/t8z0vlkXMg3QYLa7DZ/IJQ== +"@types/markdown-it@^14.1.2": + version "14.1.2" + resolved "https://registry.yarnpkg.com/@types/markdown-it/-/markdown-it-14.1.2.tgz#57f2532a0800067d9b934f3521429a2e8bfb4c61" + integrity sha512-promo4eFwuiW+TfGxhi+0x3czqTYJkG8qB17ZUJiVF10Xm7NLVRSLUsfRTU/6h1e24VvRnXCx+hG7li58lkzog== + dependencies: + "@types/linkify-it" "^5" + "@types/mdurl" "^2" + "@types/md5@^2.3.0": version "2.3.5" resolved "https://registry.yarnpkg.com/@types/md5/-/md5-2.3.5.tgz#481cef0a896e3a5dcbfc5a8a8b02c05958af48a5" integrity sha512-/i42wjYNgE6wf0j2bcTX6kuowmdL/6PE4IVitMpm2eYKBUuYCprdcWVK+xEF0gcV6ufMCRhtxmReGfc6hIK7Jw== +"@types/mdurl@^2": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@types/mdurl/-/mdurl-2.0.0.tgz#d43878b5b20222682163ae6f897b20447233bdfd" + integrity sha512-RGdgjQUZba5p6QEFAVx2OGb8rQDL/cPRG7GiedRzMcJ1tYnUANBncjbSB1NRGwbvjcPeikRABz2nshyPk1bhWg== + "@types/mute-stream@^0.0.4": version "0.0.4" resolved "https://registry.yarnpkg.com/@types/mute-stream/-/mute-stream-0.0.4.tgz#77208e56a08767af6c5e1237be8888e2f255c478" @@ -3201,6 +3219,11 @@ emoji-regex@^9.2.2: resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-9.2.2.tgz#840c8803b0d8047f4ff0cf963176b32d4ef3ed72" integrity sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg== +entities@^4.4.0: + version "4.5.0" + resolved "https://registry.yarnpkg.com/entities/-/entities-4.5.0.tgz#5d268ea5e7113ec74c4d033b79ea5a35a488fb48" + integrity sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw== + env-paths@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/env-paths/-/env-paths-2.2.1.tgz#420399d416ce1fbe9bc0a07c62fa68d67fd0f8f2" @@ -4937,6 +4960,13 @@ lines-and-columns@^1.1.6: resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.2.4.tgz#eca284f75d2965079309dc0ad9255abb2ebc1632" integrity sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg== +linkify-it@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/linkify-it/-/linkify-it-5.0.0.tgz#9ef238bfa6dc70bd8e7f9572b52d369af569b421" + integrity sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ== + dependencies: + uc.micro "^2.0.0" + lint-staged@15.2.7: version "15.2.7" resolved "https://registry.yarnpkg.com/lint-staged/-/lint-staged-15.2.7.tgz#97867e29ed632820c0fb90be06cd9ed384025649" @@ -5117,6 +5147,23 @@ map-obj@^2.0.0: resolved "https://registry.yarnpkg.com/map-obj/-/map-obj-2.0.0.tgz#a65cd29087a92598b8791257a523e021222ac1f9" integrity sha512-TzQSV2DiMYgoF5RycneKVUzIa9bQsj/B3tTgsE3dOGqlzHnGIDaC7XBE7grnA+8kZPnfqSGFe95VHc2oc0VFUQ== +markdown-it-plain-text@^0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/markdown-it-plain-text/-/markdown-it-plain-text-0.3.0.tgz#374abfcc1c95ae3d7a5e09365558d43184e1e7db" + integrity sha512-JT5x7oXZaTY6lwxsnxaIAT4hm5Q2Mi8dZoCaCNg3s7JXmDxY84D90OtV0US436UvN4zOUHoac4ExceTogydOLw== + +markdown-it@^14.1.0: + version "14.1.0" + resolved "https://registry.yarnpkg.com/markdown-it/-/markdown-it-14.1.0.tgz#3c3c5992883c633db4714ccb4d7b5935d98b7d45" + integrity sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg== + dependencies: + argparse "^2.0.1" + entities "^4.4.0" + linkify-it "^5.0.0" + mdurl "^2.0.0" + punycode.js "^2.3.1" + uc.micro "^2.1.0" + md5@^2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/md5/-/md5-2.3.0.tgz#c3da9a6aae3a30b46b7b0c349b87b110dc3bda4f" @@ -5126,6 +5173,11 @@ md5@^2.3.0: crypt "0.0.2" is-buffer "~1.1.6" +mdurl@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/mdurl/-/mdurl-2.0.0.tgz#80676ec0433025dd3e17ee983d0fe8de5a2237e0" + integrity sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w== + memorystream@^0.3.1: version "0.3.1" resolved "https://registry.yarnpkg.com/memorystream/-/memorystream-0.3.1.tgz#86d7090b30ce455d63fbae12dda51a47ddcaf9b2" @@ -5738,6 +5790,11 @@ prompts@^2.0.1: kleur "^3.0.3" sisteransi "^1.0.5" +punycode.js@^2.3.1: + version "2.3.1" + resolved "https://registry.yarnpkg.com/punycode.js/-/punycode.js-2.3.1.tgz#6b53e56ad75588234e79f4affa90972c7dd8cdb7" + integrity sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA== + punycode@^2.1.0: version "2.3.1" resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.3.1.tgz#027422e2faec0b25e1549c3e1bd8309b9133b6e5" @@ -6601,6 +6658,11 @@ typescript@5.4.5: resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.4.5.tgz#42ccef2c571fdbd0f6718b1d1f5e6e5ef006f611" integrity sha512-vcI4UpRgg81oIRUFwR0WSIHKt11nJ7SAVlYNIu+QpqeyXP+gpQJy/Z4+F0aGxSE4MqwjyXvW/TzgkLAx2AGHwQ== +uc.micro@^2.0.0, uc.micro@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/uc.micro/-/uc.micro-2.1.0.tgz#f8d3f7d0ec4c3dea35a7e3c8efa4cb8b45c9e7ee" + integrity sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A== + ufo@^1.5.3: version "1.5.3" resolved "https://registry.yarnpkg.com/ufo/-/ufo-1.5.3.tgz#3325bd3c977b6c6cd3160bf4ff52989adc9d3344" From 5d6ac5ad648d5c0ad7e70e4db80089c258b11afc Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Fri, 13 Sep 2024 04:00:26 -0400 Subject: [PATCH 38/55] fix: spell check --- .cspell.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.cspell.json b/.cspell.json index ecd7f51..c78b966 100644 --- a/.cspell.json +++ b/.cspell.json @@ -25,7 +25,8 @@ "issuebody", "voyageai", "vectordump", - "payloadobject" + "payloadobject", + "markdownit" ], "dictionaries": ["typescript", "node", "software-terms"], "import": ["@cspell/dict-typescript/cspell-ext.json", "@cspell/dict-node/cspell-ext.json", "@cspell/dict-software-terms"], From da877f7aff6fbccd1ded91dfe61e9057e21679d6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 1 Sep 2024 03:50:45 +0000 Subject: [PATCH 39/55] chore: updated generated Supabase types --- src/types/database.ts | 340 ------------------------------------------ 1 file changed, 340 deletions(-) diff --git a/src/types/database.ts b/src/types/database.ts index 3622f52..f741570 100644 --- a/src/types/database.ts +++ b/src/types/database.ts @@ -1,31 +1,6 @@ export type Json = string | number | boolean | null | { [key: string]: Json | undefined } | Json[]; export type Database = { - graphql_public: { - Tables: { - [_ in never]: never; - }; - Views: { - [_ in never]: never; - }; - Functions: { - graphql: { - Args: { - operationName?: string; - query?: string; - variables?: Json; - extensions?: Json; - }; - Returns: Json; - }; - }; - Enums: { - [_ in never]: never; - }; - CompositeTypes: { - [_ in never]: never; - }; - }; public: { Tables: { issue_comments: { @@ -240,321 +215,6 @@ export type Database = { [_ in never]: never; }; }; - storage: { - Tables: { - buckets: { - Row: { - allowed_mime_types: string[] | null; - avif_autodetection: boolean | null; - created_at: string | null; - file_size_limit: number | null; - id: string; - name: string; - owner: string | null; - owner_id: string | null; - public: boolean | null; - updated_at: string | null; - }; - Insert: { - allowed_mime_types?: string[] | null; - avif_autodetection?: boolean | null; - created_at?: string | null; - file_size_limit?: number | null; - id: string; - name: string; - owner?: string | null; - owner_id?: string | null; - public?: boolean | null; - updated_at?: string | null; - }; - Update: { - allowed_mime_types?: string[] | null; - avif_autodetection?: boolean | null; - created_at?: string | null; - file_size_limit?: number | null; - id?: string; - name?: string; - owner?: string | null; - owner_id?: string | null; - public?: boolean | null; - updated_at?: string | null; - }; - Relationships: []; - }; - migrations: { - Row: { - executed_at: string | null; - hash: string; - id: number; - name: string; - }; - Insert: { - executed_at?: string | null; - hash: string; - id: number; - name: string; - }; - Update: { - executed_at?: string | null; - hash?: string; - id?: number; - name?: string; - }; - Relationships: []; - }; - objects: { - Row: { - bucket_id: string | null; - created_at: string | null; - id: string; - last_accessed_at: string | null; - metadata: Json | null; - name: string | null; - owner: string | null; - owner_id: string | null; - path_tokens: string[] | null; - updated_at: string | null; - user_metadata: Json | null; - version: string | null; - }; - Insert: { - bucket_id?: string | null; - created_at?: string | null; - id?: string; - last_accessed_at?: string | null; - metadata?: Json | null; - name?: string | null; - owner?: string | null; - owner_id?: string | null; - path_tokens?: string[] | null; - updated_at?: string | null; - user_metadata?: Json | null; - version?: string | null; - }; - Update: { - bucket_id?: string | null; - created_at?: string | null; - id?: string; - last_accessed_at?: string | null; - metadata?: Json | null; - name?: string | null; - owner?: string | null; - owner_id?: string | null; - path_tokens?: string[] | null; - updated_at?: string | null; - user_metadata?: Json | null; - version?: string | null; - }; - Relationships: [ - { - foreignKeyName: "objects_bucketId_fkey"; - columns: ["bucket_id"]; - isOneToOne: false; - referencedRelation: "buckets"; - referencedColumns: ["id"]; - }, - ]; - }; - s3_multipart_uploads: { - Row: { - bucket_id: string; - created_at: string; - id: string; - in_progress_size: number; - key: string; - owner_id: string | null; - upload_signature: string; - user_metadata: Json | null; - version: string; - }; - Insert: { - bucket_id: string; - created_at?: string; - id: string; - in_progress_size?: number; - key: string; - owner_id?: string | null; - upload_signature: string; - user_metadata?: Json | null; - version: string; - }; - Update: { - bucket_id?: string; - created_at?: string; - id?: string; - in_progress_size?: number; - key?: string; - owner_id?: string | null; - upload_signature?: string; - user_metadata?: Json | null; - version?: string; - }; - Relationships: [ - { - foreignKeyName: "s3_multipart_uploads_bucket_id_fkey"; - columns: ["bucket_id"]; - isOneToOne: false; - referencedRelation: "buckets"; - referencedColumns: ["id"]; - }, - ]; - }; - s3_multipart_uploads_parts: { - Row: { - bucket_id: string; - created_at: string; - etag: string; - id: string; - key: string; - owner_id: string | null; - part_number: number; - size: number; - upload_id: string; - version: string; - }; - Insert: { - bucket_id: string; - created_at?: string; - etag: string; - id?: string; - key: string; - owner_id?: string | null; - part_number: number; - size?: number; - upload_id: string; - version: string; - }; - Update: { - bucket_id?: string; - created_at?: string; - etag?: string; - id?: string; - key?: string; - owner_id?: string | null; - part_number?: number; - size?: number; - upload_id?: string; - version?: string; - }; - Relationships: [ - { - foreignKeyName: "s3_multipart_uploads_parts_bucket_id_fkey"; - columns: ["bucket_id"]; - isOneToOne: false; - referencedRelation: "buckets"; - referencedColumns: ["id"]; - }, - { - foreignKeyName: "s3_multipart_uploads_parts_upload_id_fkey"; - columns: ["upload_id"]; - isOneToOne: false; - referencedRelation: "s3_multipart_uploads"; - referencedColumns: ["id"]; - }, - ]; - }; - }; - Views: { - [_ in never]: never; - }; - Functions: { - can_insert_object: { - Args: { - bucketid: string; - name: string; - owner: string; - metadata: Json; - }; - Returns: undefined; - }; - extension: { - Args: { - name: string; - }; - Returns: string; - }; - filename: { - Args: { - name: string; - }; - Returns: string; - }; - foldername: { - Args: { - name: string; - }; - Returns: string[]; - }; - get_size_by_bucket: { - Args: Record; - Returns: { - size: number; - bucket_id: string; - }[]; - }; - list_multipart_uploads_with_delimiter: { - Args: { - bucket_id: string; - prefix_param: string; - delimiter_param: string; - max_keys?: number; - next_key_token?: string; - next_upload_token?: string; - }; - Returns: { - key: string; - id: string; - created_at: string; - }[]; - }; - list_objects_with_delimiter: { - Args: { - bucket_id: string; - prefix_param: string; - delimiter_param: string; - max_keys?: number; - start_after?: string; - next_token?: string; - }; - Returns: { - name: string; - id: string; - metadata: Json; - updated_at: string; - }[]; - }; - operation: { - Args: Record; - Returns: string; - }; - search: { - Args: { - prefix: string; - bucketname: string; - limits?: number; - levels?: number; - offsets?: number; - search?: string; - sortcolumn?: string; - sortorder?: string; - }; - Returns: { - name: string; - id: string; - updated_at: string; - created_at: string; - last_accessed_at: string; - metadata: Json; - }[]; - }; - }; - Enums: { - [_ in never]: never; - }; - CompositeTypes: { - [_ in never]: never; - }; - }; }; type PublicSchema = Database[Extract]; From 2a08b22a98526e65000d09119cbc8fa19737bced Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Fri, 13 Sep 2024 16:37:12 -0400 Subject: [PATCH 40/55] feat: issue dedup --- src/adapters/supabase/helpers/issues.ts | 20 +-- src/handlers/issue-deduplication.ts | 136 ++++++++++++++++++ src/plugin.ts | 8 +- .../20240913145445_issue_comments.sql | 13 ++ 4 files changed, 166 insertions(+), 11 deletions(-) create mode 100644 src/handlers/issue-deduplication.ts create mode 100644 supabase/migrations/20240913145445_issue_comments.sql diff --git a/src/adapters/supabase/helpers/issues.ts b/src/adapters/supabase/helpers/issues.ts index 063d8a9..05fc62d 100644 --- a/src/adapters/supabase/helpers/issues.ts +++ b/src/adapters/supabase/helpers/issues.ts @@ -13,6 +13,12 @@ export interface IssueType { embedding: number[]; } +export interface IssueSimilaritySearchResult { + issue_id: string; + issue_plaintext: string; + similarity: number; +} + export class Issues extends SuperSupabase { constructor(supabase: SupabaseClient, context: Context) { super(supabase, context); @@ -70,15 +76,13 @@ export class Issues extends SuperSupabase { } } - async findSimilarIssues(markdown: string, threshold: number): Promise { + async findSimilarIssues(markdown: string, threshold: number, currentId: string): Promise { const embedding = await this.context.adapters.voyage.embedding.createEmbedding(markdown); - const { data, error } = await this.supabase - .from("issues") - .select("*") - .eq("type", "issue") - .textSearch("embedding", embedding.join(",")) - .order("embedding", { foreignTable: "issues", ascending: false }) - .lte("embedding", threshold); + const { data, error } = await this.supabase.rpc("find_similar_issues", { + current_id: currentId, + query_embedding: embedding, + threshold: threshold, + }); if (error) { this.context.logger.error("Error finding similar issues", error); return []; diff --git a/src/handlers/issue-deduplication.ts b/src/handlers/issue-deduplication.ts new file mode 100644 index 0000000..2ef8eb2 --- /dev/null +++ b/src/handlers/issue-deduplication.ts @@ -0,0 +1,136 @@ +import { IssueSimilaritySearchResult } from "../adapters/supabase/helpers/issues"; +import { Context } from "../types"; +const MATCH_THRESHOLD = 0.95; +const WARNING_THRESHOLD = 0.5; + +export interface IssueGraphqlResponse { + node: { + title: string; + url: string; + }; +} + +/** + * Check if an issue is similar to any existing issues in the database + * @param context + * @returns true if the issue is similar to an existing issue, false otherwise + */ +export async function issueChecker(context: Context): Promise { + const { + logger, + payload, + adapters: { supabase }, + } = context; + + const issue = payload.issue; + + //First Check if an issue with more than MATCH_THRESHOLD similarity exists (Very Similar) + const similarIssue = await supabase.issue.findSimilarIssues(issue.body + issue.title, MATCH_THRESHOLD, issue.node_id); + if (similarIssue && similarIssue?.length > 0) { + logger.info(`Similar issue which matches more than ${MATCH_THRESHOLD} already exists`); + //Close the issue as "unplanned" + await context.octokit.issues.update({ + owner: payload.repository.owner.login, + repo: payload.repository.name, + issue_number: issue.number, + state: "closed", + labels: ["unplanned"], + }); + return true; + } + + //Second Check if an issue with more than WARNING_THRESHOLD similarity exists (Warning) + const warningIssue = await supabase.issue.findSimilarIssues(issue.body + issue.title, WARNING_THRESHOLD, issue.node_id); + if (warningIssue && warningIssue?.length > 0) { + logger.info(`Similar issue which matches more than ${WARNING_THRESHOLD} already exists`); + //Add a comment immediately next to the issue + //Build a list of similar issues url + const issueList: IssueGraphqlResponse[] = await Promise.all( + warningIssue.map(async (issue: IssueSimilaritySearchResult) => { + //fetch the issue url and title using globaNodeId + const issueUrl: IssueGraphqlResponse = await context.octokit.graphql( + `query($issueNodeId: ID!) { + node(id: $issueNodeId) { + ... on Issue { + title + url + } + } + }`, + { + issueNodeId: issue.issue_id, + } + ); + return issueUrl; + }) + ); + + // Reopen the issue + await context.octokit.issues.update({ + owner: payload.repository.owner.login, + repo: payload.repository.name, + issue_number: issue.number, + state: "open", + }); + //Remove the "unplanned" label + await context.octokit.issues.removeLabel({ + owner: payload.repository.owner.login, + repo: payload.repository.name, + issue_number: issue.number, + name: "unplanned", + }); + // Check if there is already a comment on the issue + const existingComment = await context.octokit.issues.listComments({ + owner: payload.repository.owner.login, + repo: payload.repository.name, + issue_number: issue.number, + }); + if (existingComment.data.length > 0) { + // Find the comment that lists the similar issues + const commentToUpdate = existingComment.data.find( + (comment) => comment && comment.body && comment.body.includes("This issue seems to be similar to the following issue(s)") + ); + + if (commentToUpdate) { + // Update the comment with the latest list of similar issues + const body = issueList.map((issue) => `- [${issue.node.title}](${issue.node.url})`).join("\n"); + const updatedBody = `This issue seems to be similar to the following issue(s):\n\n${body}`; + await context.octokit.issues.updateComment({ + owner: payload.repository.owner.login, + repo: payload.repository.name, + comment_id: commentToUpdate.id, + body: updatedBody, + }); + } else { + // Add a new comment to the issue + await createNewComment(context, issueList); + } + } else { + // Add a new comment to the issue + await createNewComment(context, issueList); + } + return true; + } + + logger.info("No similar issue found"); + return false; +} + +/** + * Create a new comment on the issue with the list of similar issues + * @param context + * @param resolvedIssueList + */ +async function createNewComment(context: Context, resolvedIssueList: IssueGraphqlResponse[]) { + let body = "This issue seems to be similar to the following issue(s):\n\n"; + resolvedIssueList.forEach((issue) => { + const issueLine = `- [${issue.node.title}](${issue.node.url})\n`; + body += issueLine; + }); + await context.octokit.issues.createComment({ + owner: context.payload.repository.owner.login, + repo: context.payload.repository.name, + issue_number: context.payload.issue.number, + body: body, + }); +} diff --git a/src/plugin.ts b/src/plugin.ts index 2f7dad8..0af10f0 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -13,6 +13,7 @@ import { VoyageAIClient } from "voyageai"; import { deleteIssues } from "./handlers/delete-issue"; import { addIssue } from "./handlers/add-issue"; import { updateIssue } from "./handlers/update-issue"; +import { issueChecker } from "./handlers/issue-deduplication"; /** * The main plugin function. Split for easier testing. @@ -31,11 +32,12 @@ export async function runPlugin(context: Context) { } else if (isIssueEvent(context)) { switch (eventName) { case "issues.opened": - return await addIssue(context); - case "issues.deleted": - return await deleteIssues(context); + return (await issueChecker(context)) ? null : await addIssue(context); case "issues.edited": + await issueChecker(context); return await updateIssue(context); + case "issues.deleted": + return await deleteIssues(context); } } else { logger.error(`Unsupported event: ${eventName}`); diff --git a/supabase/migrations/20240913145445_issue_comments.sql b/supabase/migrations/20240913145445_issue_comments.sql new file mode 100644 index 0000000..1753ab1 --- /dev/null +++ b/supabase/migrations/20240913145445_issue_comments.sql @@ -0,0 +1,13 @@ +CREATE OR REPLACE FUNCTION find_similar_issues(current_id VARCHAR, query_embedding vector(1024), threshold float8) +RETURNS TABLE(issue_id VARCHAR, issue_plaintext TEXT, similarity float8) AS $$ +BEGIN + RETURN QUERY + SELECT id AS issue_id, + plaintext AS issue_plaintext, + 1 - (embedding <=> query_embedding) AS similarity + FROM issues + WHERE id <> current_id + AND 1 - (embedding <=> query_embedding) >= threshold + ORDER BY similarity DESC; +END; +$$ LANGUAGE plpgsql; From 736a0bf134b1b9cead19d487ff22644abf5ffad1 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Fri, 13 Sep 2024 16:38:46 -0400 Subject: [PATCH 41/55] fix: cspell --- .cspell.json | 3 ++- src/handlers/issue-deduplication.ts | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.cspell.json b/.cspell.json index c78b966..43040b8 100644 --- a/.cspell.json +++ b/.cspell.json @@ -26,7 +26,8 @@ "voyageai", "vectordump", "payloadobject", - "markdownit" + "markdownit", + "plpgsql" ], "dictionaries": ["typescript", "node", "software-terms"], "import": ["@cspell/dict-typescript/cspell-ext.json", "@cspell/dict-node/cspell-ext.json", "@cspell/dict-software-terms"], diff --git a/src/handlers/issue-deduplication.ts b/src/handlers/issue-deduplication.ts index 2ef8eb2..07b1af9 100644 --- a/src/handlers/issue-deduplication.ts +++ b/src/handlers/issue-deduplication.ts @@ -47,7 +47,7 @@ export async function issueChecker(context: Context): Promise { //Build a list of similar issues url const issueList: IssueGraphqlResponse[] = await Promise.all( warningIssue.map(async (issue: IssueSimilaritySearchResult) => { - //fetch the issue url and title using globaNodeId + //fetch the issue url and title using globalNodeId const issueUrl: IssueGraphqlResponse = await context.octokit.graphql( `query($issueNodeId: ID!) { node(id: $issueNodeId) { From fe17b3e90ac2ce11c187d9212ee0697bbab7e25a Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Fri, 13 Sep 2024 19:22:51 -0400 Subject: [PATCH 42/55] fix: knip : --- src/adapters/supabase/helpers/issues.ts | 10 ---------- src/handlers/issue-deduplication.ts | 16 +++++++++------- src/plugin.ts | 3 ++- 3 files changed, 11 insertions(+), 18 deletions(-) diff --git a/src/adapters/supabase/helpers/issues.ts b/src/adapters/supabase/helpers/issues.ts index 05fc62d..5f30821 100644 --- a/src/adapters/supabase/helpers/issues.ts +++ b/src/adapters/supabase/helpers/issues.ts @@ -3,16 +3,6 @@ import { SuperSupabase } from "./supabase"; import { Context } from "../../../types/context"; import { markdownToPlainText } from "../../utils/markdown-to-plaintext"; -export interface IssueType { - id: string; - markdown?: string; - author_id: number; - created_at: string; - modified_at: string; - payloadObject: Record | null; - embedding: number[]; -} - export interface IssueSimilaritySearchResult { issue_id: string; issue_plaintext: string; diff --git a/src/handlers/issue-deduplication.ts b/src/handlers/issue-deduplication.ts index 07b1af9..5c01f9e 100644 --- a/src/handlers/issue-deduplication.ts +++ b/src/handlers/issue-deduplication.ts @@ -72,13 +72,15 @@ export async function issueChecker(context: Context): Promise { issue_number: issue.number, state: "open", }); - //Remove the "unplanned" label - await context.octokit.issues.removeLabel({ - owner: payload.repository.owner.login, - repo: payload.repository.name, - issue_number: issue.number, - name: "unplanned", - }); + //Remove the "unplanned" label if it exists + if (issue.labels && issue.labels.find((label) => label.name === "unplanned")) { + await context.octokit.issues.removeLabel({ + owner: payload.repository.owner.login, + repo: payload.repository.name, + issue_number: issue.number, + name: "unplanned", + }); + } // Check if there is already a comment on the issue const existingComment = await context.octokit.issues.listComments({ owner: payload.repository.owner.login, diff --git a/src/plugin.ts b/src/plugin.ts index 0af10f0..0d0876c 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -32,7 +32,8 @@ export async function runPlugin(context: Context) { } else if (isIssueEvent(context)) { switch (eventName) { case "issues.opened": - return (await issueChecker(context)) ? null : await addIssue(context); + await issueChecker(context); + return await addIssue(context); case "issues.edited": await issueChecker(context); return await updateIssue(context); From 4d6491a2984351bfec798221f744e177fb4454f1 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Sat, 14 Sep 2024 12:25:50 -0400 Subject: [PATCH 43/55] fix: remove label controls and issue reopen controls, added similarity to issues in comment --- src/handlers/issue-deduplication.ts | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/src/handlers/issue-deduplication.ts b/src/handlers/issue-deduplication.ts index 5c01f9e..a5d0afa 100644 --- a/src/handlers/issue-deduplication.ts +++ b/src/handlers/issue-deduplication.ts @@ -8,6 +8,7 @@ export interface IssueGraphqlResponse { title: string; url: string; }; + similarity: number; } /** @@ -34,7 +35,7 @@ export async function issueChecker(context: Context): Promise { repo: payload.repository.name, issue_number: issue.number, state: "closed", - labels: ["unplanned"], + state_reason: "not_planned", }); return true; } @@ -61,26 +62,10 @@ export async function issueChecker(context: Context): Promise { issueNodeId: issue.issue_id, } ); + issueUrl.similarity = issue.similarity; return issueUrl; }) ); - - // Reopen the issue - await context.octokit.issues.update({ - owner: payload.repository.owner.login, - repo: payload.repository.name, - issue_number: issue.number, - state: "open", - }); - //Remove the "unplanned" label if it exists - if (issue.labels && issue.labels.find((label) => label.name === "unplanned")) { - await context.octokit.issues.removeLabel({ - owner: payload.repository.owner.login, - repo: payload.repository.name, - issue_number: issue.number, - name: "unplanned", - }); - } // Check if there is already a comment on the issue const existingComment = await context.octokit.issues.listComments({ owner: payload.repository.owner.login, @@ -95,7 +80,7 @@ export async function issueChecker(context: Context): Promise { if (commentToUpdate) { // Update the comment with the latest list of similar issues - const body = issueList.map((issue) => `- [${issue.node.title}](${issue.node.url})`).join("\n"); + const body = issueList.map((issue) => `- [${issue.node.title}](${issue.node.url}) Similarity: ${issue.similarity}`).join("\n"); const updatedBody = `This issue seems to be similar to the following issue(s):\n\n${body}`; await context.octokit.issues.updateComment({ owner: payload.repository.owner.login, @@ -126,7 +111,7 @@ export async function issueChecker(context: Context): Promise { async function createNewComment(context: Context, resolvedIssueList: IssueGraphqlResponse[]) { let body = "This issue seems to be similar to the following issue(s):\n\n"; resolvedIssueList.forEach((issue) => { - const issueLine = `- [${issue.node.title}](${issue.node.url})\n`; + const issueLine = `- [${issue.node.title}](${issue.node.url}) Similarity: ${issue.similarity}\n`; body += issueLine; }); await context.octokit.issues.createComment({ From 33a5f93dd7cde5b87d3b935e8cb4ad93527617c3 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Sat, 14 Sep 2024 12:28:37 -0400 Subject: [PATCH 44/55] fix: similarity in percents --- src/handlers/issue-deduplication.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/handlers/issue-deduplication.ts b/src/handlers/issue-deduplication.ts index a5d0afa..2f5ff8c 100644 --- a/src/handlers/issue-deduplication.ts +++ b/src/handlers/issue-deduplication.ts @@ -8,7 +8,7 @@ export interface IssueGraphqlResponse { title: string; url: string; }; - similarity: number; + similarity: string; } /** @@ -62,7 +62,7 @@ export async function issueChecker(context: Context): Promise { issueNodeId: issue.issue_id, } ); - issueUrl.similarity = issue.similarity; + issueUrl.similarity = (issue.similarity * 100).toFixed(2); return issueUrl; }) ); From 72d72a34d3300faa1134b35fab9627d758beb74a Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Sat, 14 Sep 2024 15:31:29 -0400 Subject: [PATCH 45/55] fix: issue dedup warning threshold to 75 --- src/handlers/issue-deduplication.ts | 163 +++++++++++++--------------- 1 file changed, 77 insertions(+), 86 deletions(-) diff --git a/src/handlers/issue-deduplication.ts b/src/handlers/issue-deduplication.ts index 2f5ff8c..57b5d3e 100644 --- a/src/handlers/issue-deduplication.ts +++ b/src/handlers/issue-deduplication.ts @@ -1,7 +1,9 @@ import { IssueSimilaritySearchResult } from "../adapters/supabase/helpers/issues"; import { Context } from "../types"; +import { IssuePayload } from "../types/payload"; + const MATCH_THRESHOLD = 0.95; -const WARNING_THRESHOLD = 0.5; +const WARNING_THRESHOLD = 0.75; export interface IssueGraphqlResponse { node: { @@ -19,105 +21,94 @@ export interface IssueGraphqlResponse { export async function issueChecker(context: Context): Promise { const { logger, - payload, adapters: { supabase }, + octokit, } = context; - + const { payload } = context as { payload: IssuePayload }; const issue = payload.issue; + const issueContent = issue.body + issue.title; - //First Check if an issue with more than MATCH_THRESHOLD similarity exists (Very Similar) - const similarIssue = await supabase.issue.findSimilarIssues(issue.body + issue.title, MATCH_THRESHOLD, issue.node_id); - if (similarIssue && similarIssue?.length > 0) { - logger.info(`Similar issue which matches more than ${MATCH_THRESHOLD} already exists`); - //Close the issue as "unplanned" - await context.octokit.issues.update({ - owner: payload.repository.owner.login, - repo: payload.repository.name, - issue_number: issue.number, - state: "closed", - state_reason: "not_planned", - }); - return true; - } + // Fetch all similar issues based on WARNING_THRESHOLD + const similarIssues = await supabase.issue.findSimilarIssues(issueContent, WARNING_THRESHOLD, issue.node_id); + console.log(similarIssues); + if (similarIssues && similarIssues.length > 0) { + const matchIssues = similarIssues.filter((issue) => issue.similarity >= MATCH_THRESHOLD); - //Second Check if an issue with more than WARNING_THRESHOLD similarity exists (Warning) - const warningIssue = await supabase.issue.findSimilarIssues(issue.body + issue.title, WARNING_THRESHOLD, issue.node_id); - if (warningIssue && warningIssue?.length > 0) { - logger.info(`Similar issue which matches more than ${WARNING_THRESHOLD} already exists`); - //Add a comment immediately next to the issue - //Build a list of similar issues url - const issueList: IssueGraphqlResponse[] = await Promise.all( - warningIssue.map(async (issue: IssueSimilaritySearchResult) => { - //fetch the issue url and title using globalNodeId - const issueUrl: IssueGraphqlResponse = await context.octokit.graphql( - `query($issueNodeId: ID!) { - node(id: $issueNodeId) { - ... on Issue { - title - url - } - } - }`, - { - issueNodeId: issue.issue_id, - } - ); - issueUrl.similarity = (issue.similarity * 100).toFixed(2); - return issueUrl; - }) - ); - // Check if there is already a comment on the issue - const existingComment = await context.octokit.issues.listComments({ - owner: payload.repository.owner.login, - repo: payload.repository.name, - issue_number: issue.number, - }); - if (existingComment.data.length > 0) { - // Find the comment that lists the similar issues - const commentToUpdate = existingComment.data.find( - (comment) => comment && comment.body && comment.body.includes("This issue seems to be similar to the following issue(s)") - ); + // Handle issues that match the MATCH_THRESHOLD (Very Similar) + if (matchIssues.length > 0) { + logger.info(`Similar issue which matches more than ${MATCH_THRESHOLD} already exists`); + await octokit.issues.update({ + owner: payload.repository.owner.login, + repo: payload.repository.name, + issue_number: issue.number, + state: "closed", + state_reason: "not_planned", + }); + } - if (commentToUpdate) { - // Update the comment with the latest list of similar issues - const body = issueList.map((issue) => `- [${issue.node.title}](${issue.node.url}) Similarity: ${issue.similarity}`).join("\n"); - const updatedBody = `This issue seems to be similar to the following issue(s):\n\n${body}`; - await context.octokit.issues.updateComment({ - owner: payload.repository.owner.login, - repo: payload.repository.name, - comment_id: commentToUpdate.id, - body: updatedBody, - }); - } else { - // Add a new comment to the issue - await createNewComment(context, issueList); - } - } else { - // Add a new comment to the issue - await createNewComment(context, issueList); + // Handle issues that match the WARNING_THRESHOLD but not the MATCH_THRESHOLD + if (similarIssues.length > 0) { + logger.info(`Similar issue which matches more than ${WARNING_THRESHOLD} already exists`); + await handleSimilarIssuesComment(context, payload, issue.number, similarIssues); + return true; } - return true; } - logger.info("No similar issue found"); return false; } /** - * Create a new comment on the issue with the list of similar issues + * Handle commenting on an issue with similar issues information * @param context - * @param resolvedIssueList + * @param payload + * @param issueNumber + * @param similarIssues */ -async function createNewComment(context: Context, resolvedIssueList: IssueGraphqlResponse[]) { - let body = "This issue seems to be similar to the following issue(s):\n\n"; - resolvedIssueList.forEach((issue) => { - const issueLine = `- [${issue.node.title}](${issue.node.url}) Similarity: ${issue.similarity}\n`; - body += issueLine; - }); - await context.octokit.issues.createComment({ - owner: context.payload.repository.owner.login, - repo: context.payload.repository.name, - issue_number: context.payload.issue.number, - body: body, +async function handleSimilarIssuesComment(context: Context, payload: IssuePayload, issueNumber: number, similarIssues: IssueSimilaritySearchResult[]) { + const issueList: IssueGraphqlResponse[] = await Promise.all( + similarIssues.map(async (issue: IssueSimilaritySearchResult) => { + const issueUrl: IssueGraphqlResponse = await context.octokit.graphql( + `query($issueNodeId: ID!) { + node(id: $issueNodeId) { + ... on Issue { + title + url + } + } + }`, + { issueNodeId: issue.issue_id } + ); + issueUrl.similarity = (issue.similarity * 100).toFixed(2); + return issueUrl; + }) + ); + + const commentBody = issueList.map((issue) => `- [${issue.node.title}](${issue.node.url}) Similarity: ${issue.similarity}`).join("\n"); + const body = `This issue seems to be similar to the following issue(s):\n\n${commentBody}`; + + const existingComments = await context.octokit.issues.listComments({ + owner: payload.repository.owner.login, + repo: payload.repository.name, + issue_number: issueNumber, }); + + const existingComment = existingComments.data.find( + (comment) => comment.body && comment.body.includes("This issue seems to be similar to the following issue(s)") + ); + + if (existingComment) { + await context.octokit.issues.updateComment({ + owner: payload.repository.owner.login, + repo: payload.repository.name, + comment_id: existingComment.id, + body: body, + }); + } else { + await context.octokit.issues.createComment({ + owner: payload.repository.owner.login, + repo: payload.repository.name, + issue_number: issueNumber, + body: body, + }); + } } From 7609943c256dddc9fa1e05fbdb77eadd8e63b19e Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Sat, 14 Sep 2024 17:10:11 -0400 Subject: [PATCH 46/55] feat: pass config through plugin settings --- src/handlers/issue-deduplication.ts | 15 ++++++--------- src/types/plugin-inputs.ts | 8 +++++++- tests/main.test.ts | 5 ++++- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/handlers/issue-deduplication.ts b/src/handlers/issue-deduplication.ts index 57b5d3e..2378a1e 100644 --- a/src/handlers/issue-deduplication.ts +++ b/src/handlers/issue-deduplication.ts @@ -2,9 +2,6 @@ import { IssueSimilaritySearchResult } from "../adapters/supabase/helpers/issues import { Context } from "../types"; import { IssuePayload } from "../types/payload"; -const MATCH_THRESHOLD = 0.95; -const WARNING_THRESHOLD = 0.75; - export interface IssueGraphqlResponse { node: { title: string; @@ -28,15 +25,15 @@ export async function issueChecker(context: Context): Promise { const issue = payload.issue; const issueContent = issue.body + issue.title; - // Fetch all similar issues based on WARNING_THRESHOLD - const similarIssues = await supabase.issue.findSimilarIssues(issueContent, WARNING_THRESHOLD, issue.node_id); + // Fetch all similar issues based on settings.warningThreshold + const similarIssues = await supabase.issue.findSimilarIssues(issueContent, context.config.warningThreshold, issue.node_id); console.log(similarIssues); if (similarIssues && similarIssues.length > 0) { - const matchIssues = similarIssues.filter((issue) => issue.similarity >= MATCH_THRESHOLD); + const matchIssues = similarIssues.filter((issue) => issue.similarity >= context.config.matchThreshold); // Handle issues that match the MATCH_THRESHOLD (Very Similar) if (matchIssues.length > 0) { - logger.info(`Similar issue which matches more than ${MATCH_THRESHOLD} already exists`); + logger.info(`Similar issue which matches more than ${context.config.matchThreshold} already exists`); await octokit.issues.update({ owner: payload.repository.owner.login, repo: payload.repository.name, @@ -46,9 +43,9 @@ export async function issueChecker(context: Context): Promise { }); } - // Handle issues that match the WARNING_THRESHOLD but not the MATCH_THRESHOLD + // Handle issues that match the settings.warningThreshold but not the MATCH_THRESHOLD if (similarIssues.length > 0) { - logger.info(`Similar issue which matches more than ${WARNING_THRESHOLD} already exists`); + logger.info(`Similar issue which matches more than ${context.config.warningThreshold} already exists`); await handleSimilarIssuesComment(context, payload, issue.number, similarIssues); return true; } diff --git a/src/types/plugin-inputs.ts b/src/types/plugin-inputs.ts index 77a40f9..a942db2 100644 --- a/src/types/plugin-inputs.ts +++ b/src/types/plugin-inputs.ts @@ -18,7 +18,13 @@ export interface PluginInputs Date: Mon, 23 Sep 2024 11:51:30 -0400 Subject: [PATCH 47/55] feat: label added hook --- README.md | 2 +- manifest.json | 2 +- src/adapters/supabase/helpers/issues.ts | 25 ++ src/handlers/label-added.ts | 77 +++++ src/plugin.ts | 3 + src/types/context.ts | 3 +- src/types/database.ts | 404 ++++++++++++++++++++++++ src/types/plugin-inputs.ts | 3 +- tests/main.test.ts | 1 + 9 files changed, 516 insertions(+), 4 deletions(-) create mode 100644 src/handlers/label-added.ts diff --git a/README.md b/README.md index cac1f27..fa0099e 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ To set up the `.dev.vars` file, you will need to provide the following variables - Add the following to your `.ubiquibot-config.yml` file with the appropriate URL: ```javascript -plugin: http://127.0.0.1:4000 - runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" , "issues.opened", "issues.edited", "issues.deleted"] + runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" , "issues.opened", "issues.edited", "issues.deleted", "issues.labled"] ``` diff --git a/manifest.json b/manifest.json index 328a300..a474f5b 100644 --- a/manifest.json +++ b/manifest.json @@ -1,5 +1,5 @@ { "name": "@ubiquity-os/comment-vector-embeddings", "description": "Issue comment plugin for Ubiquibot. It enables the storage, updating, and deletion of issue comment embeddings.", - "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issues.opened", "issues.edited", "issues.deleted"] + "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issues.opened", "issues.edited", "issues.deleted", "issues.labled"] } diff --git a/src/adapters/supabase/helpers/issues.ts b/src/adapters/supabase/helpers/issues.ts index 5f30821..c874cdc 100644 --- a/src/adapters/supabase/helpers/issues.ts +++ b/src/adapters/supabase/helpers/issues.ts @@ -9,6 +9,17 @@ export interface IssueSimilaritySearchResult { similarity: number; } +export interface IssueType { + id: string; + markdown?: string; + plaintext?: string; + payload?: Record; + author_id: number; + created_at: string; + modified_at: string; + embedding: number[]; +} + export class Issues extends SuperSupabase { constructor(supabase: SupabaseClient, context: Context) { super(supabase, context); @@ -66,6 +77,20 @@ export class Issues extends SuperSupabase { } } + async getIssue(issueNodeId: string): Promise { + const { data, error } = await this.supabase + .from("issues") // Provide the second type argument + .select("*") + .eq("id", issueNodeId) + .returns(); + + if (error) { + this.context.logger.error("Error getting issue", error); + return null; + } + return data; + } + async findSimilarIssues(markdown: string, threshold: number, currentId: string): Promise { const embedding = await this.context.adapters.voyage.embedding.createEmbedding(markdown); const { data, error } = await this.supabase.rpc("find_similar_issues", { diff --git a/src/handlers/label-added.ts b/src/handlers/label-added.ts new file mode 100644 index 0000000..5852c4f --- /dev/null +++ b/src/handlers/label-added.ts @@ -0,0 +1,77 @@ +import { Context } from "../types"; +import { IssuePayload } from "../types/payload"; + +interface IssueGraphqlResponse { + node: { + title: string; + url: string; + }; +} + +export async function labelAdded(context: Context) { + const { + logger, + adapters: { supabase }, + octokit, + } = context; + const { payload } = context as { payload: IssuePayload }; + const issue = payload.issue; + const issueContent = issue.body + issue.title; + + // On Adding the labels to the issue, the bot should + // create a new comment with users who completed task most similar to the issue + // if the comment already exists, it should update the comment with the new users + const matchResultArray: Array = []; + const similarIssues = await supabase.issue.findSimilarIssues(issueContent, context.config.jobMatchingThreshold, issue.node_id); + if (similarIssues && similarIssues.length > 0) { + // Find the most similar issue and the users who completed the task + similarIssues.sort((a, b) => b.similarity - a.similarity); + similarIssues.forEach(async (issue) => { + const data = await supabase.issue.getIssue(issue.issue_id); + const issuePayload = (data?.payload as IssuePayload) || []; + const users = issuePayload?.issue.assignees; + //Make the string + // ## [User Name](Link to User Profile) + // - [Issue] X% Match + users.forEach(async (user) => { + if (user && user.name && user.url) { + matchResultArray.push(`## [${user.name}](${user.url})\n- [Issue] ${issue.similarity}% Match`); + } + }); + }); + // Fetch if any previous comment exists + const issueResponse: IssueGraphqlResponse = await octokit.graphql( + `query($issueId: ID!) { + node(id: $issueId) { + ... on Issue { + comments(first: 100) { + nodes { + id + body + } + } + } + } + } + `, + { issueId: issue.node_id } + ); + console.log(issueResponse); + + // if(issueResponse.node.comments.nodes.length > 0) { + // const commentId = issueResponse.node.comments.nodes[0].id + // const previousComment = issueResponse.node.comments.nodes[0].body + // const newComment = previousComment + "\n" + matchResultArray.join("\n") + // await octokit.issues.updateComment({ + // owner: payload.repository.owner.login, + // repo: payload.repository.name, + // comment_id: commentId, + // body: newComment + // }) + // } + console.log(matchResultArray); + } + + logger.ok(`Successfully created issue!`); + logger.debug(`Exiting addIssue`); +} diff --git a/src/plugin.ts b/src/plugin.ts index 0d0876c..36728b8 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -14,6 +14,7 @@ import { deleteIssues } from "./handlers/delete-issue"; import { addIssue } from "./handlers/add-issue"; import { updateIssue } from "./handlers/update-issue"; import { issueChecker } from "./handlers/issue-deduplication"; +import { labelAdded } from "./handlers/label-added"; /** * The main plugin function. Split for easier testing. @@ -40,6 +41,8 @@ export async function runPlugin(context: Context) { case "issues.deleted": return await deleteIssues(context); } + } else if (eventName == "issues.labeled") { + await labelAdded(context); } else { logger.error(`Unsupported event: ${eventName}`); } diff --git a/src/types/context.ts b/src/types/context.ts index 1227abf..b11ac2f 100644 --- a/src/types/context.ts +++ b/src/types/context.ts @@ -16,7 +16,8 @@ export type SupportedEventsU = | "issue_comment.edited" | "issues.opened" | "issues.edited" - | "issues.deleted"; + | "issues.deleted" + | "issues.labeled"; export type SupportedEvents = { [K in SupportedEventsU]: K extends WebhookEventName ? WebhookEvent : never; diff --git a/src/types/database.ts b/src/types/database.ts index f741570..df59882 100644 --- a/src/types/database.ts +++ b/src/types/database.ts @@ -1,6 +1,31 @@ export type Json = string | number | boolean | null | { [key: string]: Json | undefined } | Json[]; export type Database = { + graphql_public: { + Tables: { + [_ in never]: never; + }; + Views: { + [_ in never]: never; + }; + Functions: { + graphql: { + Args: { + operationName?: string; + query?: string; + variables?: Json; + extensions?: Json; + }; + Returns: Json; + }; + }; + Enums: { + [_ in never]: never; + }; + CompositeTypes: { + [_ in never]: never; + }; + }; public: { Tables: { issue_comments: { @@ -9,24 +34,77 @@ export type Database = { created_at: string; embedding: string; id: string; + issue_id: string | null; + markdown: string | null; modified_at: string; + payloadobject: Json | null; plaintext: string | null; + type: string; }; Insert: { author_id: string; created_at?: string; embedding: string; id: string; + issue_id?: string | null; + markdown?: string | null; modified_at?: string; + payloadobject?: Json | null; plaintext?: string | null; + type: string; }; Update: { author_id?: string; created_at?: string; embedding?: string; id?: string; + issue_id?: string | null; + markdown?: string | null; + modified_at?: string; + payloadobject?: Json | null; + plaintext?: string | null; + type?: string; + }; + Relationships: [ + { + foreignKeyName: "issue_comments_issue_id_fkey"; + columns: ["issue_id"]; + isOneToOne: false; + referencedRelation: "issues"; + referencedColumns: ["id"]; + }, + ]; + }; + issues: { + Row: { + created_at: string; + embedding: string; + id: string; + markdown: string | null; + modified_at: string; + payload: Json | null; + plaintext: string | null; + type: string; + }; + Insert: { + created_at?: string; + embedding: string; + id: string; + markdown?: string | null; + modified_at?: string; + payload?: Json | null; + plaintext?: string | null; + type?: string; + }; + Update: { + created_at?: string; + embedding?: string; + id?: string; + markdown?: string | null; modified_at?: string; + payload?: Json | null; plaintext?: string | null; + type?: string; }; Relationships: []; }; @@ -48,6 +126,17 @@ export type Database = { }; Returns: unknown; }; + find_similar_issues: { + Args: { + query_embedding: string; + threshold: number; + }; + Returns: { + id: number; + issue: string; + similarity: number; + }[]; + }; halfvec_avg: { Args: { "": number[]; @@ -215,6 +304,321 @@ export type Database = { [_ in never]: never; }; }; + storage: { + Tables: { + buckets: { + Row: { + allowed_mime_types: string[] | null; + avif_autodetection: boolean | null; + created_at: string | null; + file_size_limit: number | null; + id: string; + name: string; + owner: string | null; + owner_id: string | null; + public: boolean | null; + updated_at: string | null; + }; + Insert: { + allowed_mime_types?: string[] | null; + avif_autodetection?: boolean | null; + created_at?: string | null; + file_size_limit?: number | null; + id: string; + name: string; + owner?: string | null; + owner_id?: string | null; + public?: boolean | null; + updated_at?: string | null; + }; + Update: { + allowed_mime_types?: string[] | null; + avif_autodetection?: boolean | null; + created_at?: string | null; + file_size_limit?: number | null; + id?: string; + name?: string; + owner?: string | null; + owner_id?: string | null; + public?: boolean | null; + updated_at?: string | null; + }; + Relationships: []; + }; + migrations: { + Row: { + executed_at: string | null; + hash: string; + id: number; + name: string; + }; + Insert: { + executed_at?: string | null; + hash: string; + id: number; + name: string; + }; + Update: { + executed_at?: string | null; + hash?: string; + id?: number; + name?: string; + }; + Relationships: []; + }; + objects: { + Row: { + bucket_id: string | null; + created_at: string | null; + id: string; + last_accessed_at: string | null; + metadata: Json | null; + name: string | null; + owner: string | null; + owner_id: string | null; + path_tokens: string[] | null; + updated_at: string | null; + user_metadata: Json | null; + version: string | null; + }; + Insert: { + bucket_id?: string | null; + created_at?: string | null; + id?: string; + last_accessed_at?: string | null; + metadata?: Json | null; + name?: string | null; + owner?: string | null; + owner_id?: string | null; + path_tokens?: string[] | null; + updated_at?: string | null; + user_metadata?: Json | null; + version?: string | null; + }; + Update: { + bucket_id?: string | null; + created_at?: string | null; + id?: string; + last_accessed_at?: string | null; + metadata?: Json | null; + name?: string | null; + owner?: string | null; + owner_id?: string | null; + path_tokens?: string[] | null; + updated_at?: string | null; + user_metadata?: Json | null; + version?: string | null; + }; + Relationships: [ + { + foreignKeyName: "objects_bucketId_fkey"; + columns: ["bucket_id"]; + isOneToOne: false; + referencedRelation: "buckets"; + referencedColumns: ["id"]; + }, + ]; + }; + s3_multipart_uploads: { + Row: { + bucket_id: string; + created_at: string; + id: string; + in_progress_size: number; + key: string; + owner_id: string | null; + upload_signature: string; + user_metadata: Json | null; + version: string; + }; + Insert: { + bucket_id: string; + created_at?: string; + id: string; + in_progress_size?: number; + key: string; + owner_id?: string | null; + upload_signature: string; + user_metadata?: Json | null; + version: string; + }; + Update: { + bucket_id?: string; + created_at?: string; + id?: string; + in_progress_size?: number; + key?: string; + owner_id?: string | null; + upload_signature?: string; + user_metadata?: Json | null; + version?: string; + }; + Relationships: [ + { + foreignKeyName: "s3_multipart_uploads_bucket_id_fkey"; + columns: ["bucket_id"]; + isOneToOne: false; + referencedRelation: "buckets"; + referencedColumns: ["id"]; + }, + ]; + }; + s3_multipart_uploads_parts: { + Row: { + bucket_id: string; + created_at: string; + etag: string; + id: string; + key: string; + owner_id: string | null; + part_number: number; + size: number; + upload_id: string; + version: string; + }; + Insert: { + bucket_id: string; + created_at?: string; + etag: string; + id?: string; + key: string; + owner_id?: string | null; + part_number: number; + size?: number; + upload_id: string; + version: string; + }; + Update: { + bucket_id?: string; + created_at?: string; + etag?: string; + id?: string; + key?: string; + owner_id?: string | null; + part_number?: number; + size?: number; + upload_id?: string; + version?: string; + }; + Relationships: [ + { + foreignKeyName: "s3_multipart_uploads_parts_bucket_id_fkey"; + columns: ["bucket_id"]; + isOneToOne: false; + referencedRelation: "buckets"; + referencedColumns: ["id"]; + }, + { + foreignKeyName: "s3_multipart_uploads_parts_upload_id_fkey"; + columns: ["upload_id"]; + isOneToOne: false; + referencedRelation: "s3_multipart_uploads"; + referencedColumns: ["id"]; + }, + ]; + }; + }; + Views: { + [_ in never]: never; + }; + Functions: { + can_insert_object: { + Args: { + bucketid: string; + name: string; + owner: string; + metadata: Json; + }; + Returns: undefined; + }; + extension: { + Args: { + name: string; + }; + Returns: string; + }; + filename: { + Args: { + name: string; + }; + Returns: string; + }; + foldername: { + Args: { + name: string; + }; + Returns: string[]; + }; + get_size_by_bucket: { + Args: Record; + Returns: { + size: number; + bucket_id: string; + }[]; + }; + list_multipart_uploads_with_delimiter: { + Args: { + bucket_id: string; + prefix_param: string; + delimiter_param: string; + max_keys?: number; + next_key_token?: string; + next_upload_token?: string; + }; + Returns: { + key: string; + id: string; + created_at: string; + }[]; + }; + list_objects_with_delimiter: { + Args: { + bucket_id: string; + prefix_param: string; + delimiter_param: string; + max_keys?: number; + start_after?: string; + next_token?: string; + }; + Returns: { + name: string; + id: string; + metadata: Json; + updated_at: string; + }[]; + }; + operation: { + Args: Record; + Returns: string; + }; + search: { + Args: { + prefix: string; + bucketname: string; + limits?: number; + levels?: number; + offsets?: number; + search?: string; + sortcolumn?: string; + sortorder?: string; + }; + Returns: { + name: string; + id: string; + updated_at: string; + created_at: string; + last_accessed_at: string; + metadata: Json; + }[]; + }; + }; + Enums: { + [_ in never]: never; + }; + CompositeTypes: { + [_ in never]: never; + }; + }; }; type PublicSchema = Database[Extract]; diff --git a/src/types/plugin-inputs.ts b/src/types/plugin-inputs.ts index a942db2..b5e9ce3 100644 --- a/src/types/plugin-inputs.ts +++ b/src/types/plugin-inputs.ts @@ -22,8 +22,9 @@ export const pluginSettingsSchema = T.Object( { matchThreshold: T.Number(), warningThreshold: T.Number(), + jobMatchingThreshold: T.Number(), }, - { default: { matchThreshold: 0.95, warningThreshold: 0.75 } } + { default: { matchThreshold: 0.95, warningThreshold: 0.75, jobMatchingThreshold: 0.75 } } ); export const pluginSettingsValidator = new StandardValidator(pluginSettingsSchema); diff --git a/tests/main.test.ts b/tests/main.test.ts index a8c14b8..bc4ed19 100644 --- a/tests/main.test.ts +++ b/tests/main.test.ts @@ -166,6 +166,7 @@ function createContextInner( config: { warningThreshold: 0.75, matchThreshold: 0.95, + jobMatchingThreshold: 0.95, }, adapters: {} as Context["adapters"], logger: new Logs("debug"), From 374a5f2605028fc9bbc6681ebc013d3525ed1b75 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Wed, 25 Sep 2024 02:49:33 -0400 Subject: [PATCH 48/55] feat: added issue-matching on label added, issue created and issue edited --- README.md | 2 +- manifest.json | 2 +- src/adapters/supabase/helpers/issues.ts | 12 +++- src/handlers/issue-assignees-changed.ts | 26 ++++++++ src/handlers/issue-deduplication.ts | 1 - src/handlers/issue-matching.ts | 84 +++++++++++++++++++++++++ src/handlers/label-added.ts | 77 ----------------------- src/plugin.ts | 13 ++-- src/types/context.ts | 4 +- 9 files changed, 133 insertions(+), 88 deletions(-) create mode 100644 src/handlers/issue-assignees-changed.ts create mode 100644 src/handlers/issue-matching.ts delete mode 100644 src/handlers/label-added.ts diff --git a/README.md b/README.md index fa0099e..6b03a0f 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ To set up the `.dev.vars` file, you will need to provide the following variables - Add the following to your `.ubiquibot-config.yml` file with the appropriate URL: ```javascript -plugin: http://127.0.0.1:4000 - runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" , "issues.opened", "issues.edited", "issues.deleted", "issues.labled"] + runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" , "issues.opened", "issues.edited", "issues.deleted", "issues.labeled", "issues.assigned", "issues.unassigned"] ``` diff --git a/manifest.json b/manifest.json index a474f5b..f52ef4e 100644 --- a/manifest.json +++ b/manifest.json @@ -1,5 +1,5 @@ { "name": "@ubiquity-os/comment-vector-embeddings", "description": "Issue comment plugin for Ubiquibot. It enables the storage, updating, and deletion of issue comment embeddings.", - "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issues.opened", "issues.edited", "issues.deleted", "issues.labled"] + "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issues.opened", "issues.edited", "issues.deleted", "issues.labeled", "issues.assigned", "issues.unassigned"] } diff --git a/src/adapters/supabase/helpers/issues.ts b/src/adapters/supabase/helpers/issues.ts index c874cdc..6bfef09 100644 --- a/src/adapters/supabase/helpers/issues.ts +++ b/src/adapters/supabase/helpers/issues.ts @@ -77,13 +77,12 @@ export class Issues extends SuperSupabase { } } - async getIssue(issueNodeId: string): Promise { + async getIssue(issueNodeId: string): Promise { const { data, error } = await this.supabase .from("issues") // Provide the second type argument .select("*") .eq("id", issueNodeId) - .returns(); - + .returns(); if (error) { this.context.logger.error("Error getting issue", error); return null; @@ -104,4 +103,11 @@ export class Issues extends SuperSupabase { } return data; } + + async updatePayload(issueNodeId: string, payload: Record) { + const { error } = await this.supabase.from("issues").update({ payload }).eq("id", issueNodeId); + if (error) { + this.context.logger.error("Error updating issue payload", error); + } + } } diff --git a/src/handlers/issue-assignees-changed.ts b/src/handlers/issue-assignees-changed.ts new file mode 100644 index 0000000..d8d74a8 --- /dev/null +++ b/src/handlers/issue-assignees-changed.ts @@ -0,0 +1,26 @@ +import { Context } from "../types"; +import { IssuePayload } from "../types/payload"; + +export async function updateAssignees(context: Context) { + const { + logger, + adapters: { supabase }, + } = context; + const { payload } = context as { payload: IssuePayload }; + const issue = payload.issue; + + try { + await supabase.issue.updatePayload(issue.node_id, payload); + } catch (error) { + if (error instanceof Error) { + logger.error(`Error updating assignees:`, { error: error, stack: error.stack }); + throw error; + } else { + logger.error(`Error updating assignees:`, { err: error, error: new Error() }); + throw error; + } + } + + logger.ok(`Successfully updated assignees!`); + logger.debug(`Exiting updateAssignees`); +} diff --git a/src/handlers/issue-deduplication.ts b/src/handlers/issue-deduplication.ts index 2378a1e..6174e4f 100644 --- a/src/handlers/issue-deduplication.ts +++ b/src/handlers/issue-deduplication.ts @@ -27,7 +27,6 @@ export async function issueChecker(context: Context): Promise { // Fetch all similar issues based on settings.warningThreshold const similarIssues = await supabase.issue.findSimilarIssues(issueContent, context.config.warningThreshold, issue.node_id); - console.log(similarIssues); if (similarIssues && similarIssues.length > 0) { const matchIssues = similarIssues.filter((issue) => issue.similarity >= context.config.matchThreshold); diff --git a/src/handlers/issue-matching.ts b/src/handlers/issue-matching.ts new file mode 100644 index 0000000..6835b1a --- /dev/null +++ b/src/handlers/issue-matching.ts @@ -0,0 +1,84 @@ +import { Context } from "../types"; +import { IssuePayload } from "../types/payload"; + +export async function issueMatching(context: Context) { + const { + logger, + adapters: { supabase }, + octokit, + } = context; + const { payload } = context as { payload: IssuePayload }; + const issue = payload.issue; + const issueContent = issue.body + issue.title; + const commentStart = "The following users have completed similar tasks to this issue:"; + + // On Adding the labels to the issue, the bot should + // create a new comment with users who completed task most similar to the issue + // if the comment already exists, it should update the comment with the new users + const matchResultArray: Array = []; + const similarIssues = await supabase.issue.findSimilarIssues(issueContent, context.config.jobMatchingThreshold, issue.node_id); + if (similarIssues && similarIssues.length > 0) { + // Find the most similar issue and the users who completed the task + similarIssues.sort((a, b) => b.similarity - a.similarity); + similarIssues.forEach(async (issue) => { + const data = await supabase.issue.getIssue(issue.issue_id); + if (data) { + const issuePayload = (data[0].payload as IssuePayload) || []; + const users = issuePayload?.issue.assignees; + //Make the string + // ## [User Name](Link to User Profile) + // - [Issue] X% Match + users.forEach(async (user) => { + if (user && user.login && user.html_url) { + const similarityPercentage = Math.round(issue.similarity * 100); + const githubUserLink = user.html_url.replace(/https?:\/\//, "https://www."); + const issueLink = issuePayload.issue.html_url.replace(/https?:\/\//, "https://www."); + matchResultArray.push(`## [${user.login}](${githubUserLink})\n- [Issue](${issueLink}) ${similarityPercentage}% Match`); + } + }); + } + }); + + // Fetch if any previous comment exists + const listIssues = await octokit.issues.listComments({ + owner: payload.repository.owner.login, + repo: payload.repository.name, + issue_number: issue.number, + }); + //Check if the comment already exists + const existingComment = listIssues.data.find((comment) => comment.body && comment.body.startsWith(commentStart)); + + //Check if matchResultArray is empty + if (matchResultArray.length === 0) { + if (existingComment) { + // If the comment already exists, delete it + await octokit.issues.deleteComment({ + owner: payload.repository.owner.login, + repo: payload.repository.name, + comment_id: existingComment.id, + }); + } + logger.debug("No similar issues found"); + return; + } + + if (existingComment) { + await context.octokit.issues.updateComment({ + owner: payload.repository.owner.login, + repo: payload.repository.name, + comment_id: existingComment.id, + body: commentStart + "\n\n" + matchResultArray.join("\n"), + }); + } else { + await context.octokit.issues.createComment({ + owner: payload.repository.owner.login, + repo: payload.repository.name, + issue_number: payload.issue.number, + body: commentStart + "\n\n" + matchResultArray.join("\n"), + }); + } + } + + logger.ok(`Successfully created issue!`); + logger.debug(`Exiting addIssue`); +} diff --git a/src/handlers/label-added.ts b/src/handlers/label-added.ts deleted file mode 100644 index 5852c4f..0000000 --- a/src/handlers/label-added.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { Context } from "../types"; -import { IssuePayload } from "../types/payload"; - -interface IssueGraphqlResponse { - node: { - title: string; - url: string; - }; -} - -export async function labelAdded(context: Context) { - const { - logger, - adapters: { supabase }, - octokit, - } = context; - const { payload } = context as { payload: IssuePayload }; - const issue = payload.issue; - const issueContent = issue.body + issue.title; - - // On Adding the labels to the issue, the bot should - // create a new comment with users who completed task most similar to the issue - // if the comment already exists, it should update the comment with the new users - const matchResultArray: Array = []; - const similarIssues = await supabase.issue.findSimilarIssues(issueContent, context.config.jobMatchingThreshold, issue.node_id); - if (similarIssues && similarIssues.length > 0) { - // Find the most similar issue and the users who completed the task - similarIssues.sort((a, b) => b.similarity - a.similarity); - similarIssues.forEach(async (issue) => { - const data = await supabase.issue.getIssue(issue.issue_id); - const issuePayload = (data?.payload as IssuePayload) || []; - const users = issuePayload?.issue.assignees; - //Make the string - // ## [User Name](Link to User Profile) - // - [Issue] X% Match - users.forEach(async (user) => { - if (user && user.name && user.url) { - matchResultArray.push(`## [${user.name}](${user.url})\n- [Issue] ${issue.similarity}% Match`); - } - }); - }); - // Fetch if any previous comment exists - const issueResponse: IssueGraphqlResponse = await octokit.graphql( - `query($issueId: ID!) { - node(id: $issueId) { - ... on Issue { - comments(first: 100) { - nodes { - id - body - } - } - } - } - } - `, - { issueId: issue.node_id } - ); - console.log(issueResponse); - - // if(issueResponse.node.comments.nodes.length > 0) { - // const commentId = issueResponse.node.comments.nodes[0].id - // const previousComment = issueResponse.node.comments.nodes[0].body - // const newComment = previousComment + "\n" + matchResultArray.join("\n") - // await octokit.issues.updateComment({ - // owner: payload.repository.owner.login, - // repo: payload.repository.name, - // comment_id: commentId, - // body: newComment - // }) - // } - console.log(matchResultArray); - } - - logger.ok(`Successfully created issue!`); - logger.debug(`Exiting addIssue`); -} diff --git a/src/plugin.ts b/src/plugin.ts index 36728b8..962e74c 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -14,7 +14,8 @@ import { deleteIssues } from "./handlers/delete-issue"; import { addIssue } from "./handlers/add-issue"; import { updateIssue } from "./handlers/update-issue"; import { issueChecker } from "./handlers/issue-deduplication"; -import { labelAdded } from "./handlers/label-added"; +import { issueMatching } from "./handlers/issue-matching"; +import { updateAssignees } from "./handlers/issue-assignees-changed"; /** * The main plugin function. Split for easier testing. @@ -34,15 +35,19 @@ export async function runPlugin(context: Context) { switch (eventName) { case "issues.opened": await issueChecker(context); - return await addIssue(context); + await addIssue(context); + return await issueMatching(context); case "issues.edited": await issueChecker(context); - return await updateIssue(context); + await updateIssue(context); + return await issueMatching(context); case "issues.deleted": return await deleteIssues(context); } } else if (eventName == "issues.labeled") { - await labelAdded(context); + await issueMatching(context); + } else if (eventName == "issues.assigned" || eventName == "issues.unassigned") { + await updateAssignees(context); } else { logger.error(`Unsupported event: ${eventName}`); } diff --git a/src/types/context.ts b/src/types/context.ts index b11ac2f..3a1cf2c 100644 --- a/src/types/context.ts +++ b/src/types/context.ts @@ -17,7 +17,9 @@ export type SupportedEventsU = | "issues.opened" | "issues.edited" | "issues.deleted" - | "issues.labeled"; + | "issues.labeled" + | "issues.assigned" + | "issues.unassigned"; export type SupportedEvents = { [K in SupportedEventsU]: K extends WebhookEventName ? WebhookEvent : never; From d4ce273a3efb0ae62863fcd5c49997ad82596adf Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Fri, 27 Sep 2024 21:34:46 -0400 Subject: [PATCH 49/55] fix: removed code for assign issues --- README.md | 2 +- manifest.json | 2 +- src/handlers/issue-assignees-changed.ts | 26 ------ src/handlers/issue-matching.ts | 108 +++++++++++++++++++----- src/plugin.ts | 5 +- src/types/context.ts | 4 +- 6 files changed, 93 insertions(+), 54 deletions(-) delete mode 100644 src/handlers/issue-assignees-changed.ts diff --git a/README.md b/README.md index 6b03a0f..75ffd81 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ To set up the `.dev.vars` file, you will need to provide the following variables - Add the following to your `.ubiquibot-config.yml` file with the appropriate URL: ```javascript -plugin: http://127.0.0.1:4000 - runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" , "issues.opened", "issues.edited", "issues.deleted", "issues.labeled", "issues.assigned", "issues.unassigned"] + runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" , "issues.opened", "issues.edited", "issues.deleted", "issues.labeled"] ``` diff --git a/manifest.json b/manifest.json index f52ef4e..dbd801c 100644 --- a/manifest.json +++ b/manifest.json @@ -1,5 +1,5 @@ { "name": "@ubiquity-os/comment-vector-embeddings", "description": "Issue comment plugin for Ubiquibot. It enables the storage, updating, and deletion of issue comment embeddings.", - "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issues.opened", "issues.edited", "issues.deleted", "issues.labeled", "issues.assigned", "issues.unassigned"] + "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issues.opened", "issues.edited", "issues.deleted", "issues.labeled"] } diff --git a/src/handlers/issue-assignees-changed.ts b/src/handlers/issue-assignees-changed.ts deleted file mode 100644 index d8d74a8..0000000 --- a/src/handlers/issue-assignees-changed.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { Context } from "../types"; -import { IssuePayload } from "../types/payload"; - -export async function updateAssignees(context: Context) { - const { - logger, - adapters: { supabase }, - } = context; - const { payload } = context as { payload: IssuePayload }; - const issue = payload.issue; - - try { - await supabase.issue.updatePayload(issue.node_id, payload); - } catch (error) { - if (error instanceof Error) { - logger.error(`Error updating assignees:`, { error: error, stack: error.stack }); - throw error; - } else { - logger.error(`Error updating assignees:`, { err: error, error: new Error() }); - throw error; - } - } - - logger.ok(`Successfully updated assignees!`); - logger.debug(`Exiting updateAssignees`); -} diff --git a/src/handlers/issue-matching.ts b/src/handlers/issue-matching.ts index 6835b1a..8e56f94 100644 --- a/src/handlers/issue-matching.ts +++ b/src/handlers/issue-matching.ts @@ -1,6 +1,29 @@ import { Context } from "../types"; import { IssuePayload } from "../types/payload"; +export interface IssueGraphqlResponse { + node: { + title: string; + url: string; + state: string; + stateReason: string; + closed: boolean; + repository: { + owner: { + login: string; + }; + name: string; + }; + assignees: { + nodes: Array<{ + login: string; + url: string; + }>; + }; + }; + similarity: number; +} + export async function issueMatching(context: Context) { const { logger, @@ -15,30 +38,67 @@ export async function issueMatching(context: Context) { // On Adding the labels to the issue, the bot should // create a new comment with users who completed task most similar to the issue // if the comment already exists, it should update the comment with the new users - const matchResultArray: Array = []; + const matchResultArray: Map> = new Map(); const similarIssues = await supabase.issue.findSimilarIssues(issueContent, context.config.jobMatchingThreshold, issue.node_id); if (similarIssues && similarIssues.length > 0) { // Find the most similar issue and the users who completed the task + console.log(similarIssues); similarIssues.sort((a, b) => b.similarity - a.similarity); - similarIssues.forEach(async (issue) => { - const data = await supabase.issue.getIssue(issue.issue_id); - if (data) { - const issuePayload = (data[0].payload as IssuePayload) || []; - const users = issuePayload?.issue.assignees; - //Make the string - // ## [User Name](Link to User Profile) - // - [Issue] X% Match - users.forEach(async (user) => { - if (user && user.login && user.html_url) { - const similarityPercentage = Math.round(issue.similarity * 100); - const githubUserLink = user.html_url.replace(/https?:\/\//, "https://www."); - const issueLink = issuePayload.issue.html_url.replace(/https?:\/\//, "https://www."); - matchResultArray.push(`## [${user.login}](${githubUserLink})\n- [Issue](${issueLink}) ${similarityPercentage}% Match`); + const fetchPromises = similarIssues.map(async (issue) => { + logger.info("Issue ID: " + issue.issue_id); + logger.info("Before query"); + const issueObject: IssueGraphqlResponse = await context.octokit.graphql( + `query ($issueNodeId: ID!) { + node(id: $issueNodeId) { + ... on Issue { + title + url + state + repository{ + name + owner { + login + } + } + stateReason + closed + assignees(first: 10) { + nodes { + login + url + } + } + } + } + }`, + { issueNodeId: issue.issue_id } + ); + issueObject.similarity = issue.similarity; + return issueObject; + }); + + const issueList = await Promise.all(fetchPromises); + issueList.forEach((issue) => { + if (issue.node.closed && issue.node.stateReason === "COMPLETED" && issue.node.assignees.nodes.length > 0) { + const assignees = issue.node.assignees.nodes; + assignees.forEach((assignee) => { + const similarityPercentage = Math.round(issue.similarity * 100); + const githubUserLink = assignee.url.replace(/https?:\/\/github.com/, "https://www.github.com"); + const issueLink = issue.node.url.replace(/https?:\/\/github.com/, "https://www.github.com"); + if (matchResultArray.has(assignee.login)) { + matchResultArray + .get(assignee.login) + ?.push( + `## [${assignee.login}](${githubUserLink})\n- [${issue.node.repository.owner.login}/${issue.node.repository.name}#${issue.node.url.split("/").pop()}](${issueLink}) ${similarityPercentage}% Match` + ); + } else { + matchResultArray.set(assignee.login, [ + `## [${assignee.login}](${githubUserLink})\n- [${issue.node.repository.owner.login}/${issue.node.repository.name}#${issue.node.url.split("/").pop()}](${issueLink}) ${similarityPercentage}% Match`, + ]); } }); } }); - // Fetch if any previous comment exists const listIssues = await octokit.issues.listComments({ owner: payload.repository.owner.login, @@ -49,7 +109,7 @@ export async function issueMatching(context: Context) { const existingComment = listIssues.data.find((comment) => comment.body && comment.body.startsWith(commentStart)); //Check if matchResultArray is empty - if (matchResultArray.length === 0) { + if (matchResultArray && matchResultArray.size === 0) { if (existingComment) { // If the comment already exists, delete it await octokit.issues.deleteComment({ @@ -67,14 +127,24 @@ export async function issueMatching(context: Context) { owner: payload.repository.owner.login, repo: payload.repository.name, comment_id: existingComment.id, - body: commentStart + "\n\n" + matchResultArray.join("\n"), + body: + commentStart + + "\n\n" + + Array.from(matchResultArray.values()) + .map((arr) => arr.join("\n")) + .join("\n"), }); } else { await context.octokit.issues.createComment({ owner: payload.repository.owner.login, repo: payload.repository.name, issue_number: payload.issue.number, - body: commentStart + "\n\n" + matchResultArray.join("\n"), + body: + commentStart + + "\n\n" + + Array.from(matchResultArray.values()) + .map((arr) => arr.join("\n")) + .join("\n"), }); } } diff --git a/src/plugin.ts b/src/plugin.ts index 962e74c..197948b 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -15,7 +15,6 @@ import { addIssue } from "./handlers/add-issue"; import { updateIssue } from "./handlers/update-issue"; import { issueChecker } from "./handlers/issue-deduplication"; import { issueMatching } from "./handlers/issue-matching"; -import { updateAssignees } from "./handlers/issue-assignees-changed"; /** * The main plugin function. Split for easier testing. @@ -45,9 +44,7 @@ export async function runPlugin(context: Context) { return await deleteIssues(context); } } else if (eventName == "issues.labeled") { - await issueMatching(context); - } else if (eventName == "issues.assigned" || eventName == "issues.unassigned") { - await updateAssignees(context); + return await issueMatching(context); } else { logger.error(`Unsupported event: ${eventName}`); } diff --git a/src/types/context.ts b/src/types/context.ts index 3a1cf2c..b11ac2f 100644 --- a/src/types/context.ts +++ b/src/types/context.ts @@ -17,9 +17,7 @@ export type SupportedEventsU = | "issues.opened" | "issues.edited" | "issues.deleted" - | "issues.labeled" - | "issues.assigned" - | "issues.unassigned"; + | "issues.labeled"; export type SupportedEvents = { [K in SupportedEventsU]: K extends WebhookEventName ? WebhookEvent : never; From ff7c83a7566063ee1f1ea160cd16473cb2a18e8c Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Fri, 27 Sep 2024 22:01:17 -0400 Subject: [PATCH 50/55] fix: removed logger --- src/handlers/issue-matching.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/handlers/issue-matching.ts b/src/handlers/issue-matching.ts index 8e56f94..22c7622 100644 --- a/src/handlers/issue-matching.ts +++ b/src/handlers/issue-matching.ts @@ -42,11 +42,8 @@ export async function issueMatching(context: Context) { const similarIssues = await supabase.issue.findSimilarIssues(issueContent, context.config.jobMatchingThreshold, issue.node_id); if (similarIssues && similarIssues.length > 0) { // Find the most similar issue and the users who completed the task - console.log(similarIssues); similarIssues.sort((a, b) => b.similarity - a.similarity); const fetchPromises = similarIssues.map(async (issue) => { - logger.info("Issue ID: " + issue.issue_id); - logger.info("Before query"); const issueObject: IssueGraphqlResponse = await context.octokit.graphql( `query ($issueNodeId: ID!) { node(id: $issueNodeId) { From cf4b5eee918f8aa722227fe7df4488c9702b6856 Mon Sep 17 00:00:00 2001 From: Shivaditya Shivganesh Date: Sat, 28 Sep 2024 14:51:25 -0400 Subject: [PATCH 51/55] feat: updated the comment structure --- src/handlers/issue-matching.ts | 41 +++++++++++++++++----------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/src/handlers/issue-matching.ts b/src/handlers/issue-matching.ts index 22c7622..cc1d060 100644 --- a/src/handlers/issue-matching.ts +++ b/src/handlers/issue-matching.ts @@ -24,6 +24,17 @@ export interface IssueGraphqlResponse { similarity: number; } +const commentBuilder = (matchResultArray: Map>): string => { + const commentLines: string[] = [">[!NOTE]", ">The following contributors may be suitable for this task:"]; + matchResultArray.forEach((issues, assignee) => { + commentLines.push(`>### [${assignee}](https://www.github.com/${assignee})`); + issues.forEach((issue) => { + commentLines.push(issue); + }); + }); + return commentLines.join("\n"); +}; + export async function issueMatching(context: Context) { const { logger, @@ -33,7 +44,7 @@ export async function issueMatching(context: Context) { const { payload } = context as { payload: IssuePayload }; const issue = payload.issue; const issueContent = issue.body + issue.title; - const commentStart = "The following users have completed similar tasks to this issue:"; + const commentStart = ">The following contributors may be suitable for this task:"; // On Adding the labels to the issue, the bot should // create a new comment with users who completed task most similar to the issue @@ -80,17 +91,16 @@ export async function issueMatching(context: Context) { const assignees = issue.node.assignees.nodes; assignees.forEach((assignee) => { const similarityPercentage = Math.round(issue.similarity * 100); - const githubUserLink = assignee.url.replace(/https?:\/\/github.com/, "https://www.github.com"); const issueLink = issue.node.url.replace(/https?:\/\/github.com/, "https://www.github.com"); if (matchResultArray.has(assignee.login)) { matchResultArray .get(assignee.login) ?.push( - `## [${assignee.login}](${githubUserLink})\n- [${issue.node.repository.owner.login}/${issue.node.repository.name}#${issue.node.url.split("/").pop()}](${issueLink}) ${similarityPercentage}% Match` + `> \`${similarityPercentage}% Match\` [${issue.node.repository.owner.login}/${issue.node.repository.name}#${issue.node.url.split("/").pop()}](${issueLink})` ); } else { matchResultArray.set(assignee.login, [ - `## [${assignee.login}](${githubUserLink})\n- [${issue.node.repository.owner.login}/${issue.node.repository.name}#${issue.node.url.split("/").pop()}](${issueLink}) ${similarityPercentage}% Match`, + `> \`${similarityPercentage}% Match\` [${issue.node.repository.owner.login}/${issue.node.repository.name}#${issue.node.url.split("/").pop()}](${issueLink})`, ]); } }); @@ -103,8 +113,7 @@ export async function issueMatching(context: Context) { issue_number: issue.number, }); //Check if the comment already exists - const existingComment = listIssues.data.find((comment) => comment.body && comment.body.startsWith(commentStart)); - + const existingComment = listIssues.data.find((comment) => comment.body && comment.body.includes(">[!NOTE]" + "\n" + commentStart)); //Check if matchResultArray is empty if (matchResultArray && matchResultArray.size === 0) { if (existingComment) { @@ -118,34 +127,24 @@ export async function issueMatching(context: Context) { logger.debug("No similar issues found"); return; } - + const comment = commentBuilder(matchResultArray); if (existingComment) { await context.octokit.issues.updateComment({ owner: payload.repository.owner.login, repo: payload.repository.name, comment_id: existingComment.id, - body: - commentStart + - "\n\n" + - Array.from(matchResultArray.values()) - .map((arr) => arr.join("\n")) - .join("\n"), + body: comment, }); } else { await context.octokit.issues.createComment({ owner: payload.repository.owner.login, repo: payload.repository.name, issue_number: payload.issue.number, - body: - commentStart + - "\n\n" + - Array.from(matchResultArray.values()) - .map((arr) => arr.join("\n")) - .join("\n"), + body: comment, }); } } - logger.ok(`Successfully created issue!`); - logger.debug(`Exiting addIssue`); + logger.ok(`Successfully created issue comment!`); + logger.debug(`Exiting issueMatching handler`); } From 3f6f5260ba1461a6456dfa2e1e0bf5d9b70b8f0c Mon Sep 17 00:00:00 2001 From: gentlementlegen Date: Sun, 29 Sep 2024 23:12:28 +0900 Subject: [PATCH 52/55] fix: env is taken from process on Actions scripts --- src/main.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.ts b/src/main.ts index 34f3e2f..23a3e74 100644 --- a/src/main.ts +++ b/src/main.ts @@ -10,7 +10,7 @@ import { plugin } from "./plugin"; */ export async function run() { const payload = github.context.payload.inputs; - const env = Value.Decode(envSchema, payload.env); + const env = Value.Decode(envSchema, process.env); const settings = Value.Decode(pluginSettingsSchema, Value.Default(pluginSettingsSchema, JSON.parse(payload.settings))); if (!pluginSettingsValidator.test(settings)) { From a6acbf0cc482a7a7a34c81eae9025d3b1391327a Mon Sep 17 00:00:00 2001 From: gentlementlegen Date: Sun, 29 Sep 2024 23:21:34 +0900 Subject: [PATCH 53/55] fix: defaults are properly set from the payload --- README.md | 11 +++++++---- manifest.json | 4 ++-- src/types/plugin-inputs.ts | 8 ++++---- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 75ffd81..9661b9c 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,12 @@ To set up the `.dev.vars` file, you will need to provide the following variables ## Usage - Add the following to your `.ubiquibot-config.yml` file with the appropriate URL: -```javascript - -plugin: http://127.0.0.1:4000 - runsOn: [ "issue_comment.created", "issue_comment.edited", "issue_comment.deleted" , "issues.opened", "issues.edited", "issues.deleted", "issues.labeled"] +```yaml +- plugin: ubiquity-os-marketplace/generate-vector-embeddings + with: + matchThreshold: 0.95 + warningThreshold: 0.75 + jobMatchingThreshold: 0.75 ``` @@ -53,4 +56,4 @@ To set up the `.dev.vars` file, you will need to provide the following variables - Replace the placeholders with the appropriate values. ## Testing -- Run `yarn test` to run the tests. \ No newline at end of file +- Run `yarn test` to run the tests. diff --git a/manifest.json b/manifest.json index dbd801c..39df377 100644 --- a/manifest.json +++ b/manifest.json @@ -1,5 +1,5 @@ { - "name": "@ubiquity-os/comment-vector-embeddings", - "description": "Issue comment plugin for Ubiquibot. It enables the storage, updating, and deletion of issue comment embeddings.", + "name": "Generate vector embeddings", + "description": "Enables the storage, updating, and deletion of issue comment embeddings.", "ubiquity:listeners": ["issue_comment.created", "issue_comment.edited", "issue_comment.deleted", "issues.opened", "issues.edited", "issues.deleted", "issues.labeled"] } diff --git a/src/types/plugin-inputs.ts b/src/types/plugin-inputs.ts index b5e9ce3..226beee 100644 --- a/src/types/plugin-inputs.ts +++ b/src/types/plugin-inputs.ts @@ -20,11 +20,11 @@ export interface PluginInputs Date: Sun, 29 Sep 2024 23:27:58 +0900 Subject: [PATCH 54/55] feat: worker deployment --- .github/workflows/worker-delete.yml | 44 ++++++++++++++++++++++++ .github/workflows/worker-deploy.yml | 52 +++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 .github/workflows/worker-delete.yml create mode 100644 .github/workflows/worker-deploy.yml diff --git a/.github/workflows/worker-delete.yml b/.github/workflows/worker-delete.yml new file mode 100644 index 0000000..89d3e23 --- /dev/null +++ b/.github/workflows/worker-delete.yml @@ -0,0 +1,44 @@ +name: Delete Deployment + +on: + delete: + +jobs: + delete: + runs-on: ubuntu-latest + name: Delete Deployment + steps: + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: "20.10.0" + + - name: Enable corepack + run: corepack enable + + - uses: actions/checkout@v4 + + - name: Get Deleted Branch Name + id: get_branch + run: | + branch_name=$(echo '${{ github.event.ref }}' | sed 's#refs/heads/##' | sed 's#[^a-zA-Z0-9]#-#g') + echo "branch_name=$branch_name" >> $GITHUB_ENV + - name: Retrieve and Construct Full Worker Name + id: construct_worker_name + run: | + base_name=$(grep '^name = ' wrangler.toml | sed 's/^name = "\(.*\)"$/\1/') + full_worker_name="${base_name}-${{ env.branch_name }}" + # Make sure that it doesnt exceed 63 characters or it will break RFC 1035 + full_worker_name=$(echo "${full_worker_name}" | cut -c 1-63) + echo "full_worker_name=$full_worker_name" >> $GITHUB_ENV + - name: Delete Deployment with Wrangler + uses: cloudflare/wrangler-action@v3 + with: + apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} + accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + command: delete --name ${{ env.full_worker_name }} + + - name: Output Deletion Result + run: | + echo "### Deployment URL" >> $GITHUB_STEP_SUMMARY + echo 'Deployment `${{ env.full_worker_name }}` has been deleted.' >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/worker-deploy.yml b/.github/workflows/worker-deploy.yml new file mode 100644 index 0000000..deafdc0 --- /dev/null +++ b/.github/workflows/worker-deploy.yml @@ -0,0 +1,52 @@ +name: Deploy Worker + +on: + push: + workflow_dispatch: + +jobs: + deploy: + runs-on: ubuntu-latest + name: Deploy + steps: + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: "20.10.0" + + - name: Enable corepack + run: corepack enable + + - uses: actions/checkout@v4 + + - name: Update wrangler.toml Name Field + run: | + branch_name=$(echo '${{ github.event.ref }}' | sed 's#refs/heads/##' | sed 's#[^a-zA-Z0-9]#-#g') + # Extract base name from wrangler.toml + base_name=$(grep '^name = ' wrangler.toml | sed 's/^name = "\(.*\)"$/\1/') + # Concatenate branch name with base name + new_name="${base_name}-${branch_name}" + # Truncate the new name to 63 characters for RFC 1035 + new_name=$(echo "$new_name" | cut -c 1-63) + # Update the wrangler.toml file + sed -i "s/^name = .*/name = \"$new_name\"/" wrangler.toml + echo "Updated wrangler.toml name to: $new_name" + - name: Deploy with Wrangler + id: wrangler_deploy + uses: cloudflare/wrangler-action@v3 + with: + apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} + accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + secrets: | + SUPABASE_URL + SUPABASE_KEY + VOYAGEAI_API_KEY + env: + SUPABASE_URL: ${{ secrets.SUPABASE_URL }} + SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }} + VOYAGEAI_API_KEY: ${{ secrets.VOYAGEAI_API_KEY }} + + - name: Write Deployment URL to Summary + run: | + echo "### Deployment URL" >> $GITHUB_STEP_SUMMARY + echo "${{ steps.wrangler_deploy.outputs.deployment-url }}" >> $GITHUB_STEP_SUMMARY From 7197f4bbd4b4d884dfb0a299ebbe77adab20a631 Mon Sep 17 00:00:00 2001 From: gentlementlegen Date: Sun, 29 Sep 2024 23:31:07 +0900 Subject: [PATCH 55/55] chore: update wrangler --- package.json | 2 +- yarn.lock | 152 ++++++++++++++++++++++++++------------------------- 2 files changed, 78 insertions(+), 76 deletions(-) diff --git a/package.json b/package.json index 6ef92a4..54ad601 100644 --- a/package.json +++ b/package.json @@ -74,7 +74,7 @@ "tsx": "4.15.6", "typescript": "5.4.5", "typescript-eslint": "7.13.1", - "wrangler": "3.62.0" + "wrangler": "3.78.12" }, "lint-staged": { "*.ts": [ diff --git a/yarn.lock b/yarn.lock index 8439328..519f6f7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -351,30 +351,38 @@ dependencies: mime "^3.0.0" -"@cloudflare/workerd-darwin-64@1.20240620.1": - version "1.20240620.1" - resolved "https://registry.yarnpkg.com/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20240620.1.tgz#322bc6350f9649d4a649b62a86165fa12bbdc088" - integrity sha512-YWeS2aE8jAzDefuus/3GmZcFGu3Ef94uCAoxsQuaEXNsiGM9NeAhPpKC1BJAlcv168U/Q1J+6hckcGtipf6ZcQ== - -"@cloudflare/workerd-darwin-arm64@1.20240620.1": - version "1.20240620.1" - resolved "https://registry.yarnpkg.com/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20240620.1.tgz#25cf37caf77837b8cd40e796f44aad095c39f322" - integrity sha512-3rdND+EHpmCrwYX6hvxIBSBJ0f40tRNxond1Vfw7GiR1MJVi3gragiBx75UDFHCxfRw3J0GZ1qVlkRce2/Xbsg== - -"@cloudflare/workerd-linux-64@1.20240620.1": - version "1.20240620.1" - resolved "https://registry.yarnpkg.com/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20240620.1.tgz#4138144341d7c02709c518599ae65cb070d0eb7b" - integrity sha512-tURcTrXGeSbYqeM5ISVcofY20StKbVIcdxjJvNYNZ+qmSV9Fvn+zr7rRE+q64pEloVZfhsEPAlUCnFso5VV4XQ== - -"@cloudflare/workerd-linux-arm64@1.20240620.1": - version "1.20240620.1" - resolved "https://registry.yarnpkg.com/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20240620.1.tgz#35eb047c7146075d41ce4ee1af87550de892b95d" - integrity sha512-TThvkwNxaZFKhHZnNjOGqIYCOk05DDWgO+wYMuXg15ymN/KZPnCicRAkuyqiM+R1Fgc4kwe/pehjP8pbmcf6sg== - -"@cloudflare/workerd-windows-64@1.20240620.1": - version "1.20240620.1" - resolved "https://registry.yarnpkg.com/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20240620.1.tgz#f4c6cb81a474aaf019dcf000bb31e46d3538c3f2" - integrity sha512-Y/BA9Yj0r7Al1HK3nDHcfISgFllw6NR3XMMPChev57vrVT9C9D4erBL3sUBfofHU+2U9L+ShLsl6obBpe3vvUw== +"@cloudflare/workerd-darwin-64@1.20240925.0": + version "1.20240925.0" + resolved "https://registry.yarnpkg.com/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20240925.0.tgz#f78fe394f73540594609d8e05a2da7feb46c76c0" + integrity sha512-KdLnSXuzB65CbqZPm+qYzk+zkQ1tUNPaaRGYVd/jPYAxwwtfTUQdQ+ahDPwVVs2tmQELKy7ZjQjf2apqSWUfjw== + +"@cloudflare/workerd-darwin-arm64@1.20240925.0": + version "1.20240925.0" + resolved "https://registry.yarnpkg.com/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20240925.0.tgz#f03b17177744ad898bb12610d15cc0a9744abfe6" + integrity sha512-MiQ6uUmCXjsXgWNV+Ock2tp2/tYqNJGzjuaH6jFioeRF+//mz7Tv7J7EczOL4zq+TH8QFOh0/PUsLyazIWVGng== + +"@cloudflare/workerd-linux-64@1.20240925.0": + version "1.20240925.0" + resolved "https://registry.yarnpkg.com/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20240925.0.tgz#fe0366b804b957acf5012d889e94163bab806a57" + integrity sha512-Rjix8jsJMfsInmq3Hm3fmiRQ+rwzuWRPV1pg/OWhMSfNP7Qp2RCU+RGkhgeR9Z5eNAje0Sn2BMrFq4RvF9/yRA== + +"@cloudflare/workerd-linux-arm64@1.20240925.0": + version "1.20240925.0" + resolved "https://registry.yarnpkg.com/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20240925.0.tgz#fcf82de06def420972c661a6021c87683cd8fbdc" + integrity sha512-VYIPeMHQRtbwQoIjUwS/zULlywPxyDvo46XkTpIW5MScEChfqHvAYviQ7TzYGx6Q+gmZmN+DUB2KOMx+MEpCxA== + +"@cloudflare/workerd-windows-64@1.20240925.0": + version "1.20240925.0" + resolved "https://registry.yarnpkg.com/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20240925.0.tgz#0a5c82b95b03a94591cc8a1830f28d2e41ff7685" + integrity sha512-C8peGvaU5R51bIySi1VbyfRgwNSSRknqoFSnSbSBI3uTN3THTB3UnmRKy7GXJDmyjgXuT9Pcs1IgaWNubLtNtw== + +"@cloudflare/workers-shared@0.5.4": + version "0.5.4" + resolved "https://registry.yarnpkg.com/@cloudflare/workers-shared/-/workers-shared-0.5.4.tgz#bbf8f03b79a6bc0169ad66a6015ebe579d36753a" + integrity sha512-PNL/0TjKRdUHa1kwgVdqUNJVZ9ez4kacsi8omz+gv859EvJmsVuGiMAClY2YfJnC9LVKhKCcjqmFgKNXG9/IXA== + dependencies: + mime "^3.0.0" + zod "^3.22.3" "@commitlint/cli@19.3.0": version "19.3.0" @@ -2792,11 +2800,6 @@ concat-map@0.0.1: resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" integrity sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg== -consola@^3.2.3: - version "3.2.3" - resolved "https://registry.yarnpkg.com/consola/-/consola-3.2.3.tgz#0741857aa88cfa0d6fd53f1cff0375136e98502f" - integrity sha512-I5qxpzLv+sJhTVEoLYNcTW+bThDCPsit0vLNKShZx6rLtpilNpmmeTPaeqJb9ZE9dV3DGaeby6Vuhrw38WjeyQ== - conventional-changelog-angular@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/conventional-changelog-angular/-/conventional-changelog-angular-7.0.0.tgz#5eec8edbff15aa9b1680a8dcfbd53e2d7eb2ba7a" @@ -3065,11 +3068,6 @@ date-fns@^2.21.1: dependencies: "@babel/runtime" "^7.21.0" -date-fns@^3.6.0: - version "3.6.0" - resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-3.6.0.tgz#f20ca4fe94f8b754951b24240676e8618c0206bf" - integrity sha512-fRHTG8g/Gif+kSh50gaGEdToemgfj74aRX3swtiouboip5JDLAyDE9F11nHMIcvOaXeOC6D7SpNhi7uFyB7Uww== - debug@4: version "4.3.6" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.6.tgz#2ab2c38fbaffebf8aa95fdfe6d88438c7a13c52b" @@ -5238,10 +5236,10 @@ mimic-response@^3.1.0: resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-3.1.0.tgz#2d1d59af9c1b129815accc2c46a022a5ce1fa3c9" integrity sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ== -miniflare@3.20240620.0: - version "3.20240620.0" - resolved "https://registry.yarnpkg.com/miniflare/-/miniflare-3.20240620.0.tgz#18e0b0755e7755d6830c67937646049fdcb8aa1a" - integrity sha512-NBMzqUE2mMlh/hIdt6U5MP+aFhEjKDq3l8CAajXAQa1WkndJdciWvzB2mfLETwoVFhMl/lphaVzyEN2AgwJpbQ== +miniflare@3.20240925.0: + version "3.20240925.0" + resolved "https://registry.yarnpkg.com/miniflare/-/miniflare-3.20240925.0.tgz#a291998dedf90bfb4bcfdad033ba030851ff9171" + integrity sha512-2LmQbKHf0n6ertUKhT+Iltixi53giqDH7P71+wCir3OnGyXIODqYwOECx1mSDNhYThpxM2dav8UdPn6SQiMoXw== dependencies: "@cspotcode/source-map-support" "0.8.1" acorn "^8.8.0" @@ -5251,8 +5249,8 @@ miniflare@3.20240620.0: glob-to-regexp "^0.4.1" stoppable "^1.1.0" undici "^5.28.4" - workerd "1.20240620.1" - ws "^8.14.2" + workerd "1.20240925.0" + ws "^8.17.1" youch "^3.2.2" zod "^3.22.3" @@ -5363,11 +5361,6 @@ node-domexception@^1.0.0: resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5" integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ== -node-fetch-native@^1.6.4: - version "1.6.4" - resolved "https://registry.yarnpkg.com/node-fetch-native/-/node-fetch-native-1.6.4.tgz#679fc8fd8111266d47d7e72c379f1bed9acff06e" - integrity sha512-IhOigYzAKHd244OC0JIMIUrjzctirCmPkaIfhDeGcEETWof5zKYUW7e7MYvChGWh/4CJeXEgsRyGzuF334rOOQ== - node-fetch@2.7.0: version "2.7.0" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" @@ -5478,6 +5471,11 @@ object.assign@^4.1.5: has-symbols "^1.0.3" object-keys "^1.1.1" +ohash@^1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/ohash/-/ohash-1.1.4.tgz#ae8d83014ab81157d2c285abf7792e2995fadd72" + integrity sha512-FlDryZAahJmEF3VR3w1KogSEdWX3WhA5GPakFx4J81kEAiHyLMpdLLElS8n8dfNadMgAne/MywcvmogzscVt4g== + once@^1.3.0, once@^1.3.1, once@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" @@ -5665,6 +5663,11 @@ path-to-regexp@^6.2.0: resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-6.2.2.tgz#324377a83e5049cbecadc5554d6a63a9a4866b36" integrity sha512-GQX3SSMokngb36+whdpRXE+3f9V8UzyAorlYvOGx87ufGHehNTn5lCxrKtLyZ4Yl/wEKnNnr98ZzOwwDZV5ogw== +path-to-regexp@^6.3.0: + version "6.3.0" + resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-6.3.0.tgz#2b6a26a337737a8e1416f9272ed0766b1c0389f4" + integrity sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ== + path-type@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/path-type/-/path-type-3.0.0.tgz#cef31dc8e0a1a3bb0d105c0cd97cf3bf47f4e36f" @@ -6663,10 +6666,10 @@ uc.micro@^2.0.0, uc.micro@^2.1.0: resolved "https://registry.yarnpkg.com/uc.micro/-/uc.micro-2.1.0.tgz#f8d3f7d0ec4c3dea35a7e3c8efa4cb8b45c9e7ee" integrity sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A== -ufo@^1.5.3: - version "1.5.3" - resolved "https://registry.yarnpkg.com/ufo/-/ufo-1.5.3.tgz#3325bd3c977b6c6cd3160bf4ff52989adc9d3344" - integrity sha512-Y7HYmWaFwPUmkoQCUIAYpKqkOf+SbVj/2fJJZ4RJMCfZp0rTGwRbzQD+HghfnhKOjL9E01okqz+ncJskGYfBNw== +ufo@^1.5.4: + version "1.5.4" + resolved "https://registry.yarnpkg.com/ufo/-/ufo-1.5.4.tgz#16d6949674ca0c9e0fbbae1fa20a71d7b1ded754" + integrity sha512-UsUk3byDzKd04EyoZ7U4DOlxQaD14JUKQl6/P7wiX4FNvUfm3XL246n9W5AmqwW5RSFJ27NAuM0iLscAOYUiGQ== unbox-primitive@^1.0.2: version "1.0.2" @@ -6690,17 +6693,15 @@ undici@^5.25.4, undici@^5.28.4: dependencies: "@fastify/busboy" "^2.0.0" -"unenv@npm:unenv-nightly@1.10.0-1717606461.a117952": - version "1.10.0-1717606461.a117952" - resolved "https://registry.yarnpkg.com/unenv-nightly/-/unenv-nightly-1.10.0-1717606461.a117952.tgz#ff0b97e1e159f84be747271e1d55263b4b3eae7e" - integrity sha512-u3TfBX02WzbHTpaEfWEKwDijDSFAHcgXkayUZ+MVDrjhLFvgAJzFGTSTmwlEhwWi2exyRQey23ah9wELMM6etg== +"unenv@npm:unenv-nightly@2.0.0-20240919-125358-9a64854": + version "2.0.0-20240919-125358-9a64854" + resolved "https://registry.yarnpkg.com/unenv-nightly/-/unenv-nightly-2.0.0-20240919-125358-9a64854.tgz#13f6812c7b12b9521ea05c6d49259d136e093acd" + integrity sha512-XjsgUTrTHR7iw+k/SRTNjh6EQgwpC9voygnoCJo5kh4hKqsSDHUW84MhL9EsHTNfLctvVBHaSw8e2k3R2fKXsQ== dependencies: - consola "^3.2.3" defu "^6.1.4" - mime "^3.0.0" - node-fetch-native "^1.6.4" + ohash "^1.1.4" pathe "^1.1.2" - ufo "^1.5.3" + ufo "^1.5.4" unescape-js@^1.0.5: version "1.1.4" @@ -6867,37 +6868,38 @@ word-wrap@^1.2.5: resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.5.tgz#d2c45c6dd4fbce621a66f136cbe328afd0410b34" integrity sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA== -workerd@1.20240620.1: - version "1.20240620.1" - resolved "https://registry.yarnpkg.com/workerd/-/workerd-1.20240620.1.tgz#1898fc94de8e53289b2eb261d5592474c4bc9f03" - integrity sha512-Qoq+RrFNk4pvEO+kpJVn8uJ5TRE9YJx5jX5pC5LjdKlw1XeD8EdXt5k0TbByvWunZ4qgYIcF9lnVxhcDFo203g== +workerd@1.20240925.0: + version "1.20240925.0" + resolved "https://registry.yarnpkg.com/workerd/-/workerd-1.20240925.0.tgz#0a2602eabfa7e1d01d89ff2b600ed359be9b515d" + integrity sha512-/Jj6+yLwfieZGEt3Kx4+5MoufuC3g/8iFaIh4MPBNGJOGYmdSKXvgCqz09m2+tVCYnysRfbq2zcbVxJRBfOCqQ== optionalDependencies: - "@cloudflare/workerd-darwin-64" "1.20240620.1" - "@cloudflare/workerd-darwin-arm64" "1.20240620.1" - "@cloudflare/workerd-linux-64" "1.20240620.1" - "@cloudflare/workerd-linux-arm64" "1.20240620.1" - "@cloudflare/workerd-windows-64" "1.20240620.1" + "@cloudflare/workerd-darwin-64" "1.20240925.0" + "@cloudflare/workerd-darwin-arm64" "1.20240925.0" + "@cloudflare/workerd-linux-64" "1.20240925.0" + "@cloudflare/workerd-linux-arm64" "1.20240925.0" + "@cloudflare/workerd-windows-64" "1.20240925.0" -wrangler@3.62.0: - version "3.62.0" - resolved "https://registry.yarnpkg.com/wrangler/-/wrangler-3.62.0.tgz#2b98e47734af2240e8c3136b1f6eaa90e363e1ed" - integrity sha512-TM1Bd8+GzxFw/JzwsC3i/Oss4LTWvIEWXXo1vZhx+7PHcsxdbnQGBBwPurHNJDSu2Pw22+2pCZiUGKexmgJksw== +wrangler@3.78.12: + version "3.78.12" + resolved "https://registry.yarnpkg.com/wrangler/-/wrangler-3.78.12.tgz#c3d7b605856b904ab7cce54afc223286b8454935" + integrity sha512-a/xk/N04IvOGk9J+BLkiFg42GDyPS+0BiJimbrHsbX+CDr8Iqq3HNMEyQld+6zbmq01u/gmc8S7GKVR9vDx4+g== dependencies: "@cloudflare/kv-asset-handler" "0.3.4" + "@cloudflare/workers-shared" "0.5.4" "@esbuild-plugins/node-globals-polyfill" "^0.2.3" "@esbuild-plugins/node-modules-polyfill" "^0.2.2" blake3-wasm "^2.1.5" chokidar "^3.5.3" - date-fns "^3.6.0" esbuild "0.17.19" - miniflare "3.20240620.0" + miniflare "3.20240925.0" nanoid "^3.3.3" - path-to-regexp "^6.2.0" + path-to-regexp "^6.3.0" resolve "^1.22.8" resolve.exports "^2.0.2" selfsigned "^2.0.1" source-map "^0.6.1" - unenv "npm:unenv-nightly@1.10.0-1717606461.a117952" + unenv "npm:unenv-nightly@2.0.0-20240919-125358-9a64854" + workerd "1.20240925.0" xxhash-wasm "^1.0.1" optionalDependencies: fsevents "~2.3.2" @@ -6968,7 +6970,7 @@ write-file-atomic@^5.0.0: imurmurhash "^0.1.4" signal-exit "^4.0.1" -ws@^8.14.2: +ws@^8.14.2, ws@^8.17.1: version "8.18.0" resolved "https://registry.yarnpkg.com/ws/-/ws-8.18.0.tgz#0d7505a6eafe2b0e712d232b42279f53bc289bbc" integrity sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==