From 9d0257745d53c03700465a5f2d38eca46ef34787 Mon Sep 17 00:00:00 2001 From: Amin Latifi Date: Mon, 13 May 2024 15:58:14 +0330 Subject: [PATCH 1/4] Added project stats materialized view --- ...0600963052-ProjectStstsMaterializedView.js | 100 ++++++++++++++++++ src/controllers/utils/modelHelper.ts | 4 + 2 files changed, 104 insertions(+) create mode 100644 db/migrations/1720600963052-ProjectStstsMaterializedView.js diff --git a/db/migrations/1720600963052-ProjectStstsMaterializedView.js b/db/migrations/1720600963052-ProjectStstsMaterializedView.js new file mode 100644 index 0000000..c01cbde --- /dev/null +++ b/db/migrations/1720600963052-ProjectStstsMaterializedView.js @@ -0,0 +1,100 @@ +module.exports = class ProjectStatsMatiralizeView1720600963052 { + name = "ProjectStatsMatiralizeView1720600963052"; + + async up(db) { + await db.query(` + DROP MATERIALIZED VIEW IF EXISTS project_stats_view; + + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_type WHERE typname = 'name_count_type' + ) THEN + CREATE TYPE name_count_type AS ( + name TEXT, + count INTEGER + ); + END IF; + END $$; + + CREATE MATERIALIZED VIEW project_stats_view AS + WITH + ORG_VOUCH AS ( + SELECT + PR_AT.PROJECT_ID, + OG.NAME, + PR_AT.VOUCH + FROM + PROJECT_ATTESTATION AS PR_AT + INNER JOIN ATTESTOR_ORGANISATION AS AT_OG ON PR_AT.ATTESTOR_ORGANISATION_ID = AT_OG.ID + INNER JOIN ORGANISATION AS OG ON AT_OG.ORGANISATION_ID = OG.ID + WHERE + PR_AT.REVOKED = FALSE + AND AT_OG.REVOKED = FALSE + ), + PR_ORG_V AS ( + SELECT + ORG_VOUCH.PROJECT_ID, + ORG_VOUCH.NAME, + ORG_VOUCH.VOUCH, + COUNT(*) + FROM + ORG_VOUCH + GROUP BY + ORG_VOUCH.PROJECT_ID, + ORG_VOUCH.NAME, + ORG_VOUCH.VOUCH + ), + ORG_FLAG_AGG AS ( + SELECT + PR_ORG_V.PROJECT_ID, + ARRAY_AGG( + ROW (PR_ORG_V.NAME, PR_ORG_V.COUNT)::NAME_COUNT_TYPE + ) AS ORG_FLAGS, + SUM(PR_ORG_V.COUNT) AS PR_TOTAL_FLAGS + FROM + PR_ORG_V + WHERE + PR_ORG_V.VOUCH = FALSE + GROUP BY + PR_ORG_V.PROJECT_ID + ), + ORG_VOUCH_AGG AS ( + SELECT + PR_ORG_V.PROJECT_ID, + ARRAY_AGG( + ROW (PR_ORG_V.NAME, PR_ORG_V.COUNT)::NAME_COUNT_TYPE + ) AS ORG_VOUCHES, + SUM(PR_ORG_V.COUNT) AS PR_TOTAL_VOUCHES + FROM + PR_ORG_V + WHERE + PR_ORG_V.VOUCH = TRUE + GROUP BY + PR_ORG_V.PROJECT_ID + ) + SELECT + ID, + TOTAL_VOUCHES, + TOTAL_FLAGS, + PR_TOTAL_FLAGS, + PR_TOTAL_VOUCHES, + ORG_FLAGS, + ORG_VOUCHES + FROM + PROJECT + LEFT JOIN ORG_FLAG_AGG ON ORG_FLAG_AGG.PROJECT_ID = PROJECT.ID + LEFT JOIN ORG_VOUCH_AGG ON ORG_VOUCH_AGG.PROJECT_ID = PROJECT.ID; + + `); + await db.query( + `CREATE INDEX idx_project_stats_view_id ON project_stats_view (ID);` + ); + } + + async down(db) { + await db.query(`DROP INDEX IF EXISTS idx_project_stats_view_id`); + await db.query(`DROP MATERIALIZED VIEW IF EXISTS project_stats_view`); + await db.query(`DROP TYPE IF EXISTS name_count_type CASCADE`); + } +}; diff --git a/src/controllers/utils/modelHelper.ts b/src/controllers/utils/modelHelper.ts index d0e72b4..79c2794 100644 --- a/src/controllers/utils/modelHelper.ts +++ b/src/controllers/utils/modelHelper.ts @@ -1,6 +1,7 @@ import { DataHandlerContext } from "@subsquid/evm-processor"; import { Store } from "@subsquid/typeorm-store"; import { Attestor, Project, ProjectAttestation } from "../../model"; +import { getEntityManger } from "./databaseHelper"; export const updateProjectAttestationCounts = async ( ctx: DataHandlerContext, @@ -27,6 +28,9 @@ export const updateProjectAttestationCounts = async ( project.totalFlags = flagCount; await ctx.store.upsert(project); + await getEntityManger(ctx).query( + "REFRESH MATERIALIZED VIEW project_stats_view;" + ); }; export const getProject = async ( From b2128e7dacfbc0b2b269e7b02563a5c33e469a8d Mon Sep 17 00:00:00 2001 From: Amin Latifi Date: Mon, 13 May 2024 17:33:11 +0330 Subject: [PATCH 2/4] Updated materialized view --- ...0600963052-ProjectStstsMaterializedView.js | 67 +++++++++++-------- 1 file changed, 40 insertions(+), 27 deletions(-) diff --git a/db/migrations/1720600963052-ProjectStstsMaterializedView.js b/db/migrations/1720600963052-ProjectStstsMaterializedView.js index c01cbde..9ec9da8 100644 --- a/db/migrations/1720600963052-ProjectStstsMaterializedView.js +++ b/db/migrations/1720600963052-ProjectStstsMaterializedView.js @@ -3,23 +3,23 @@ module.exports = class ProjectStatsMatiralizeView1720600963052 { async up(db) { await db.query(` - DROP MATERIALIZED VIEW IF EXISTS project_stats_view; + DROP MATERIALIZED VIEW IF EXISTS PROJECT_STATS_VIEW; DO $$ - BEGIN - IF NOT EXISTS ( - SELECT 1 FROM pg_type WHERE typname = 'name_count_type' - ) THEN - CREATE TYPE name_count_type AS ( - name TEXT, - count INTEGER - ); - END IF; - END $$; + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_type WHERE typname = 'name_count_type' + ) THEN + CREATE TYPE name_count_type AS ( + name TEXT, + count INTEGER + ); + END IF; + END $$; - CREATE MATERIALIZED VIEW project_stats_view AS + CREATE MATERIALIZED VIEW PROJECT_STATS_VIEW AS WITH - ORG_VOUCH AS ( + ORG_ATTESTATIONS AS ( SELECT PR_AT.PROJECT_ID, OG.NAME, @@ -32,18 +32,27 @@ module.exports = class ProjectStatsMatiralizeView1720600963052 { PR_AT.REVOKED = FALSE AND AT_OG.REVOKED = FALSE ), + PR_ORG AS ( + SELECT + PROJECT_ID, + ARRAY_AGG(DISTINCT ORG_ATTESTATIONS.NAME) AS UNIQ_ORGS + FROM + ORG_ATTESTATIONS + GROUP BY + PROJECT_ID + ), PR_ORG_V AS ( SELECT - ORG_VOUCH.PROJECT_ID, - ORG_VOUCH.NAME, - ORG_VOUCH.VOUCH, + ORG_ATTESTATIONS.PROJECT_ID, + ORG_ATTESTATIONS.NAME, + ORG_ATTESTATIONS.VOUCH, COUNT(*) FROM - ORG_VOUCH + ORG_ATTESTATIONS GROUP BY - ORG_VOUCH.PROJECT_ID, - ORG_VOUCH.NAME, - ORG_VOUCH.VOUCH + ORG_ATTESTATIONS.PROJECT_ID, + ORG_ATTESTATIONS.NAME, + ORG_ATTESTATIONS.VOUCH ), ORG_FLAG_AGG AS ( SELECT @@ -59,12 +68,12 @@ module.exports = class ProjectStatsMatiralizeView1720600963052 { GROUP BY PR_ORG_V.PROJECT_ID ), - ORG_VOUCH_AGG AS ( + ORG_ATTESTATIONS_AGG AS ( SELECT PR_ORG_V.PROJECT_ID, ARRAY_AGG( ROW (PR_ORG_V.NAME, PR_ORG_V.COUNT)::NAME_COUNT_TYPE - ) AS ORG_VOUCHES, + ) AS ORG_ATTESTATIONSES, SUM(PR_ORG_V.COUNT) AS PR_TOTAL_VOUCHES FROM PR_ORG_V @@ -75,21 +84,25 @@ module.exports = class ProjectStatsMatiralizeView1720600963052 { ) SELECT ID, - TOTAL_VOUCHES, - TOTAL_FLAGS, PR_TOTAL_FLAGS, PR_TOTAL_VOUCHES, + PR_TOTAL_FLAGS + PR_TOTAL_VOUCHES AS TOTAL_ATTESTATIONS, ORG_FLAGS, - ORG_VOUCHES + ORG_ATTESTATIONSES, + UNIQ_ORGS FROM PROJECT + LEFT JOIN PR_ORG ON PR_ORG.PROJECT_ID = PROJECT.ID LEFT JOIN ORG_FLAG_AGG ON ORG_FLAG_AGG.PROJECT_ID = PROJECT.ID - LEFT JOIN ORG_VOUCH_AGG ON ORG_VOUCH_AGG.PROJECT_ID = PROJECT.ID; - + LEFT JOIN ORG_ATTESTATIONS_AGG ON ORG_ATTESTATIONS_AGG.PROJECT_ID = PROJECT.ID; + `); await db.query( `CREATE INDEX idx_project_stats_view_id ON project_stats_view (ID);` ); + await db.query( + `CREATE INDEX idx_project_stats_view_uniq_orgs ON project_stats_view(UNIQ_ORGS);` + ); } async down(db) { From 733f50c4fbe2f0eeae7d6587d69b1a3ba1f830b5 Mon Sep 17 00:00:00 2001 From: Amin Latifi Date: Tue, 14 May 2024 17:21:44 +0330 Subject: [PATCH 3/4] Added project organisation related fields --- ...54645119-Data.js => 1715693415593-Data.js} | 26 ++- ...0600963052-ProjectStstsMaterializedView.js | 113 ------------ jest.config.js | 7 +- package.json | 8 +- schema.graphql | 15 +- src/controllers/utils/easTypes.ts | 12 -- src/controllers/utils/modelHelper.ts | 162 +++++++++++++++--- .../utils/projectVerificationHelper.ts | 2 +- src/controllers/utils/types.test.ts | 20 +++ src/controllers/utils/types.ts | 55 ++++++ src/model/generated/index.ts | 1 + src/model/generated/organisation.model.ts | 4 + .../generated/organisationProject.model.ts | 30 ++++ src/model/generated/project.model.ts | 10 ++ src/test/utils.ts | 2 +- 15 files changed, 301 insertions(+), 166 deletions(-) rename db/migrations/{1715254645119-Data.js => 1715693415593-Data.js} (72%) delete mode 100644 db/migrations/1720600963052-ProjectStstsMaterializedView.js delete mode 100644 src/controllers/utils/easTypes.ts create mode 100644 src/controllers/utils/types.test.ts create mode 100644 src/controllers/utils/types.ts create mode 100644 src/model/generated/organisationProject.model.ts diff --git a/db/migrations/1715254645119-Data.js b/db/migrations/1715693415593-Data.js similarity index 72% rename from db/migrations/1715254645119-Data.js rename to db/migrations/1715693415593-Data.js index d02c925..64383ef 100644 --- a/db/migrations/1715254645119-Data.js +++ b/db/migrations/1715693415593-Data.js @@ -1,19 +1,24 @@ -module.exports = class Data1715254645119 { - name = 'Data1715254645119' +module.exports = class Data1715693415593 { + name = 'Data1715693415593' async up(db) { await db.query(`CREATE TABLE "attestor" ("id" character varying NOT NULL, CONSTRAINT "PK_2ba0dae296b9deebeb9ecbbf508" PRIMARY KEY ("id"))`) + await db.query(`CREATE TABLE "project" ("id" character varying NOT NULL, "source" text NOT NULL, "project_id" text NOT NULL, "title" text, "description" text, "total_vouches" integer NOT NULL, "total_flags" integer NOT NULL, "total_attests" integer NOT NULL, "last_updated_timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, CONSTRAINT "PK_4d68b1358bb5b766d3e78f32f57" PRIMARY KEY ("id"))`) + await db.query(`CREATE INDEX "IDX_399e8555e92ea7fd5f129fe178" ON "project" ("source") `) + await db.query(`CREATE INDEX "IDX_1a480c5734c5aacb9cef7b1499" ON "project" ("project_id") `) + await db.query(`CREATE TABLE "organisation_project" ("id" character varying NOT NULL, "vouch" boolean NOT NULL, "count" integer NOT NULL, "organisation_id" character varying, "project_id" character varying, CONSTRAINT "PK_4ee2279a4757fecde9a56f003f2" PRIMARY KEY ("id"))`) + await db.query(`CREATE INDEX "IDX_202ff9497fc7d9d0c3e7a74b17" ON "organisation_project" ("organisation_id") `) + await db.query(`CREATE INDEX "IDX_356298298d0613568b73c63a1f" ON "organisation_project" ("project_id") `) await db.query(`CREATE TABLE "organisation" ("id" character varying NOT NULL, "name" text NOT NULL, "issuer" text NOT NULL, "color" text, CONSTRAINT "PK_c725ae234ef1b74cce43d2d00c1" PRIMARY KEY ("id"))`) await db.query(`CREATE UNIQUE INDEX "IDX_d9428f9c8e3052d6617e3aab0e" ON "organisation" ("name") `) await db.query(`CREATE TABLE "attestor_organisation" ("id" character varying NOT NULL, "attest_timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, "revoked" boolean NOT NULL, "attestor_id" character varying, "organisation_id" character varying, CONSTRAINT "PK_ac02a8a577635d60275796a9d03" PRIMARY KEY ("id"))`) await db.query(`CREATE INDEX "IDX_22cd09c4533533cebedb5487f4" ON "attestor_organisation" ("attestor_id") `) await db.query(`CREATE INDEX "IDX_b0d947390c1e10152bb1387fa2" ON "attestor_organisation" ("organisation_id") `) - await db.query(`CREATE TABLE "project" ("id" character varying NOT NULL, "source" text NOT NULL, "project_id" text NOT NULL, "title" text, "description" text, "total_vouches" integer NOT NULL, "total_flags" integer NOT NULL, "last_updated_timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, CONSTRAINT "PK_4d68b1358bb5b766d3e78f32f57" PRIMARY KEY ("id"))`) - await db.query(`CREATE INDEX "IDX_399e8555e92ea7fd5f129fe178" ON "project" ("source") `) - await db.query(`CREATE INDEX "IDX_1a480c5734c5aacb9cef7b1499" ON "project" ("project_id") `) await db.query(`CREATE TABLE "project_attestation" ("id" character varying NOT NULL, "recipient" text NOT NULL, "vouch" boolean NOT NULL, "tx_hash" text NOT NULL, "revoked" boolean NOT NULL, "attest_timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, "comment" text, "attestor_organisation_id" character varying, "project_id" character varying, CONSTRAINT "PK_b54887e7eb9193e705303c2b0a0" PRIMARY KEY ("id"))`) await db.query(`CREATE INDEX "IDX_d482a5af31e29569b8b42d9252" ON "project_attestation" ("attestor_organisation_id") `) await db.query(`CREATE INDEX "IDX_1082147528db937cb5b50fb2a0" ON "project_attestation" ("project_id") `) + await db.query(`ALTER TABLE "organisation_project" ADD CONSTRAINT "FK_202ff9497fc7d9d0c3e7a74b17f" FOREIGN KEY ("organisation_id") REFERENCES "organisation"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`) + await db.query(`ALTER TABLE "organisation_project" ADD CONSTRAINT "FK_356298298d0613568b73c63a1fc" FOREIGN KEY ("project_id") REFERENCES "project"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`) await db.query(`ALTER TABLE "attestor_organisation" ADD CONSTRAINT "FK_22cd09c4533533cebedb5487f44" FOREIGN KEY ("attestor_id") REFERENCES "attestor"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`) await db.query(`ALTER TABLE "attestor_organisation" ADD CONSTRAINT "FK_b0d947390c1e10152bb1387fa23" FOREIGN KEY ("organisation_id") REFERENCES "organisation"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`) await db.query(`ALTER TABLE "project_attestation" ADD CONSTRAINT "FK_d482a5af31e29569b8b42d92525" FOREIGN KEY ("attestor_organisation_id") REFERENCES "attestor_organisation"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`) @@ -22,17 +27,22 @@ module.exports = class Data1715254645119 { async down(db) { await db.query(`DROP TABLE "attestor"`) + await db.query(`DROP TABLE "project"`) + await db.query(`DROP INDEX "public"."IDX_399e8555e92ea7fd5f129fe178"`) + await db.query(`DROP INDEX "public"."IDX_1a480c5734c5aacb9cef7b1499"`) + await db.query(`DROP TABLE "organisation_project"`) + await db.query(`DROP INDEX "public"."IDX_202ff9497fc7d9d0c3e7a74b17"`) + await db.query(`DROP INDEX "public"."IDX_356298298d0613568b73c63a1f"`) await db.query(`DROP TABLE "organisation"`) await db.query(`DROP INDEX "public"."IDX_d9428f9c8e3052d6617e3aab0e"`) await db.query(`DROP TABLE "attestor_organisation"`) await db.query(`DROP INDEX "public"."IDX_22cd09c4533533cebedb5487f4"`) await db.query(`DROP INDEX "public"."IDX_b0d947390c1e10152bb1387fa2"`) - await db.query(`DROP TABLE "project"`) - await db.query(`DROP INDEX "public"."IDX_399e8555e92ea7fd5f129fe178"`) - await db.query(`DROP INDEX "public"."IDX_1a480c5734c5aacb9cef7b1499"`) await db.query(`DROP TABLE "project_attestation"`) await db.query(`DROP INDEX "public"."IDX_d482a5af31e29569b8b42d9252"`) await db.query(`DROP INDEX "public"."IDX_1082147528db937cb5b50fb2a0"`) + await db.query(`ALTER TABLE "organisation_project" DROP CONSTRAINT "FK_202ff9497fc7d9d0c3e7a74b17f"`) + await db.query(`ALTER TABLE "organisation_project" DROP CONSTRAINT "FK_356298298d0613568b73c63a1fc"`) await db.query(`ALTER TABLE "attestor_organisation" DROP CONSTRAINT "FK_22cd09c4533533cebedb5487f44"`) await db.query(`ALTER TABLE "attestor_organisation" DROP CONSTRAINT "FK_b0d947390c1e10152bb1387fa23"`) await db.query(`ALTER TABLE "project_attestation" DROP CONSTRAINT "FK_d482a5af31e29569b8b42d92525"`) diff --git a/db/migrations/1720600963052-ProjectStstsMaterializedView.js b/db/migrations/1720600963052-ProjectStstsMaterializedView.js deleted file mode 100644 index 9ec9da8..0000000 --- a/db/migrations/1720600963052-ProjectStstsMaterializedView.js +++ /dev/null @@ -1,113 +0,0 @@ -module.exports = class ProjectStatsMatiralizeView1720600963052 { - name = "ProjectStatsMatiralizeView1720600963052"; - - async up(db) { - await db.query(` - DROP MATERIALIZED VIEW IF EXISTS PROJECT_STATS_VIEW; - - DO $$ - BEGIN - IF NOT EXISTS ( - SELECT 1 FROM pg_type WHERE typname = 'name_count_type' - ) THEN - CREATE TYPE name_count_type AS ( - name TEXT, - count INTEGER - ); - END IF; - END $$; - - CREATE MATERIALIZED VIEW PROJECT_STATS_VIEW AS - WITH - ORG_ATTESTATIONS AS ( - SELECT - PR_AT.PROJECT_ID, - OG.NAME, - PR_AT.VOUCH - FROM - PROJECT_ATTESTATION AS PR_AT - INNER JOIN ATTESTOR_ORGANISATION AS AT_OG ON PR_AT.ATTESTOR_ORGANISATION_ID = AT_OG.ID - INNER JOIN ORGANISATION AS OG ON AT_OG.ORGANISATION_ID = OG.ID - WHERE - PR_AT.REVOKED = FALSE - AND AT_OG.REVOKED = FALSE - ), - PR_ORG AS ( - SELECT - PROJECT_ID, - ARRAY_AGG(DISTINCT ORG_ATTESTATIONS.NAME) AS UNIQ_ORGS - FROM - ORG_ATTESTATIONS - GROUP BY - PROJECT_ID - ), - PR_ORG_V AS ( - SELECT - ORG_ATTESTATIONS.PROJECT_ID, - ORG_ATTESTATIONS.NAME, - ORG_ATTESTATIONS.VOUCH, - COUNT(*) - FROM - ORG_ATTESTATIONS - GROUP BY - ORG_ATTESTATIONS.PROJECT_ID, - ORG_ATTESTATIONS.NAME, - ORG_ATTESTATIONS.VOUCH - ), - ORG_FLAG_AGG AS ( - SELECT - PR_ORG_V.PROJECT_ID, - ARRAY_AGG( - ROW (PR_ORG_V.NAME, PR_ORG_V.COUNT)::NAME_COUNT_TYPE - ) AS ORG_FLAGS, - SUM(PR_ORG_V.COUNT) AS PR_TOTAL_FLAGS - FROM - PR_ORG_V - WHERE - PR_ORG_V.VOUCH = FALSE - GROUP BY - PR_ORG_V.PROJECT_ID - ), - ORG_ATTESTATIONS_AGG AS ( - SELECT - PR_ORG_V.PROJECT_ID, - ARRAY_AGG( - ROW (PR_ORG_V.NAME, PR_ORG_V.COUNT)::NAME_COUNT_TYPE - ) AS ORG_ATTESTATIONSES, - SUM(PR_ORG_V.COUNT) AS PR_TOTAL_VOUCHES - FROM - PR_ORG_V - WHERE - PR_ORG_V.VOUCH = TRUE - GROUP BY - PR_ORG_V.PROJECT_ID - ) - SELECT - ID, - PR_TOTAL_FLAGS, - PR_TOTAL_VOUCHES, - PR_TOTAL_FLAGS + PR_TOTAL_VOUCHES AS TOTAL_ATTESTATIONS, - ORG_FLAGS, - ORG_ATTESTATIONSES, - UNIQ_ORGS - FROM - PROJECT - LEFT JOIN PR_ORG ON PR_ORG.PROJECT_ID = PROJECT.ID - LEFT JOIN ORG_FLAG_AGG ON ORG_FLAG_AGG.PROJECT_ID = PROJECT.ID - LEFT JOIN ORG_ATTESTATIONS_AGG ON ORG_ATTESTATIONS_AGG.PROJECT_ID = PROJECT.ID; - - `); - await db.query( - `CREATE INDEX idx_project_stats_view_id ON project_stats_view (ID);` - ); - await db.query( - `CREATE INDEX idx_project_stats_view_uniq_orgs ON project_stats_view(UNIQ_ORGS);` - ); - } - - async down(db) { - await db.query(`DROP INDEX IF EXISTS idx_project_stats_view_id`); - await db.query(`DROP MATERIALIZED VIEW IF EXISTS project_stats_view`); - await db.query(`DROP TYPE IF EXISTS name_count_type CASCADE`); - } -}; diff --git a/jest.config.js b/jest.config.js index 7c43702..c8eb546 100644 --- a/jest.config.js +++ b/jest.config.js @@ -3,8 +3,11 @@ module.exports = { // tests pat preset: "ts-jest", testEnvironment: "node", - setupFiles: ["./src/test/bootstrap.ts"], - testMatch: ["/src/test/*.test.ts"], + // setupFiles: ["./src/test/bootstrap.ts"], + testMatch: [ + "/src/test/*.test.ts", + "/src/controllers/utils/*.test.ts", + ], forceExit: true, detectOpenHandles: true, testTimeout: 30000, diff --git a/package.json b/package.json index 7eeba44..bebde06 100644 --- a/package.json +++ b/package.json @@ -4,10 +4,10 @@ "scripts": { "build": "rm -rf lib && tsc", "test:run-migration": "dotenvx run --env-file=.env.test -- squid-typeorm-migration apply", - "test:run-fresh-db": "docker-compose -f docker-compose-test.yml down -v; docker-compose --env-file .env.test -f docker-compose-test.yml up -d", - "test": "npm run test:run-fresh-db; jest", - "clear:generate:migration:run:locally": "sqd migration:clean; sqd build ; docker compose -f docker-compose-potgres.yml down -v;docker compose -f docker-compose-potgres.yml up -d;sqd migration:generate;node ./add-organisation.js; sqd run", - "clear:run:locally": "sqd build ; docker compose -f docker-compose-potgres.yml down -v;docker compose -f docker-compose-potgres.yml up -d; sqd run", + "test:run-fresh-db": "docker-compose -f docker-compose-potgres-test.yml down -v; docker-compose --env-file .env.test -f docker-compose-potgres-test.yml up -d", + "test": "npm run test:run-fresh-db; dotenvx run --env-file=.env.test -- jest --runInBand", + "clear:generate:migration:run:locally": "sqd codegen ; docker compose -f docker-compose-potgres.yml down -v;docker compose -f docker-compose-potgres.yml up -d;sqd migration:generate; sqd run", + "clear:run:locally": "sqd codegen ; docker compose -f docker-compose-potgres.yml down -v;docker compose -f docker-compose-potgres.yml up -d; sqd run", "run:locally": "sqd build; sqd run" }, "dependencies": { diff --git a/schema.graphql b/schema.graphql index e70f13a..e8861ea 100644 --- a/schema.graphql +++ b/schema.graphql @@ -39,6 +39,7 @@ type Organisation @entity { color: String "Organization Attestors" attestors: [AttestorOrganisation!]! @derivedFrom(field: "organisation") + attestedProjects: [OrganisationProject!]! @derivedFrom(field: "organisation") } type Project @entity { @@ -56,8 +57,18 @@ type Project @entity { totalVouches: Int! "Total attests with value False" totalFlags: Int! - # givbackEligibleTrue: Int! - # givbackEligibleFalse: Int! + "Total attests" + totalAttests: Int! lastUpdatedTimestamp: DateTime! attests: [ProjectAttestation!]! @derivedFrom(field: "project") + attestedOrganisations: [OrganisationProject!]! @derivedFrom(field: "project") +} + +type OrganisationProject @entity { + "-" + id: ID! + organisation: Organisation! + project: Project! + vouch: Boolean! + count: Int! } diff --git a/src/controllers/utils/easTypes.ts b/src/controllers/utils/easTypes.ts deleted file mode 100644 index 30f5e07..0000000 --- a/src/controllers/utils/easTypes.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { z } from "zod"; - -export const ProjectVerificationAttestation = z.object({ - vouch: z.boolean(), - projectSource: z.string(), - projectId: z.string(), - comment: z.string().optional(), -}); - -export type ProjectVerificationAttestation = z.infer< - typeof ProjectVerificationAttestation ->; diff --git a/src/controllers/utils/modelHelper.ts b/src/controllers/utils/modelHelper.ts index 79c2794..95c763a 100644 --- a/src/controllers/utils/modelHelper.ts +++ b/src/controllers/utils/modelHelper.ts @@ -1,36 +1,58 @@ import { DataHandlerContext } from "@subsquid/evm-processor"; import { Store } from "@subsquid/typeorm-store"; -import { Attestor, Project, ProjectAttestation } from "../../model"; +import { + Attestor, + Organisation, + OrganisationProject, + Project, + ProjectAttestation, +} from "../../model"; import { getEntityManger } from "./databaseHelper"; +import { ProjectStats } from "./types"; +import { In, Not } from "typeorm"; + +export const upsertOrganisatoinProject = async ( + ctx: DataHandlerContext, + project: Project, + organisationId: string, + vouch: boolean, + count: number +): Promise => { + const organisation = await ctx.store.get(Organisation, organisationId); + const key = `${project.id}-${organisationId}-${vouch ? "vouch" : "flag"}`; + const organisationProject = new OrganisationProject({ + id: key, + project, + organisation, + vouch, + count, + }); + ctx.store.upsert(organisationProject); +}; export const updateProjectAttestationCounts = async ( ctx: DataHandlerContext, project: Project ): Promise => { - const [vouchCount, flagCount] = await Promise.all([ - ctx.store.count(ProjectAttestation, { - where: { - project, - vouch: true, - revoked: false, - }, - }), - ctx.store.count(ProjectAttestation, { - where: { - project, - vouch: false, - revoked: false, - }, - }), - ]); - - project.totalVouches = vouchCount; - project.totalFlags = flagCount; + const em = getEntityManger(ctx); + const projectStats = await getProjectStats(ctx, project); + console.log("projectStats:", projectStats); + // throw new Error("Let's exit"); + project.totalVouches = projectStats.pr_total_vouches; + project.totalFlags = projectStats.pr_total_flags; + project.totalAttests = projectStats.pr_total_attestations; await ctx.store.upsert(project); - await getEntityManger(ctx).query( - "REFRESH MATERIALIZED VIEW project_stats_view;" - ); + + await em.getRepository(OrganisationProject).delete({ project }); + + for (const [orgId, count] of projectStats.org_flags) { + await upsertOrganisatoinProject(ctx, project, orgId, false, +count); + } + + for (const [orgId, count] of projectStats.org_vouches) { + await upsertOrganisatoinProject(ctx, project, orgId, true, +count); + } }; export const getProject = async ( @@ -50,6 +72,7 @@ export const getProject = async ( projectId, totalVouches: 0, totalFlags: 0, + totalAttests: 0, lastUpdatedTimestamp: new Date(), }) ); @@ -71,3 +94,96 @@ export const getAttestor = async ( return attestor as Attestor; }; + +export const getProjectStats = async ( + ctx: DataHandlerContext, + project: Project +): Promise => { + const em = getEntityManger(ctx); + + const result = await em.query( + ` + WITH + ORG_ATTESTATIONS AS ( + SELECT + PR_AT.PROJECT_ID, + OG.id as org_id, + PR_AT.VOUCH + FROM + PROJECT_ATTESTATION AS PR_AT + INNER JOIN ATTESTOR_ORGANISATION AS AT_OG ON PR_AT.ATTESTOR_ORGANISATION_ID = AT_OG.ID + INNER JOIN ORGANISATION AS OG ON AT_OG.ORGANISATION_ID = OG.ID + WHERE + PR_AT.REVOKED = FALSE + AND AT_OG.REVOKED = FALSE + AND PR_AT.PROJECT_ID = $1 + ), + PR_ORG AS ( + SELECT + PROJECT_ID, + ARRAY_AGG(DISTINCT ORG_ATTESTATIONS.org_id) AS UNIQ_ORGS + FROM + ORG_ATTESTATIONS + WHERE + PROJECT_ID = $1 + GROUP BY + PROJECT_ID + ), + PR_ORG_V AS ( + SELECT + ORG_ATTESTATIONS.PROJECT_ID, + ORG_ATTESTATIONS.org_id, + ORG_ATTESTATIONS.VOUCH, + COUNT(*) + FROM + ORG_ATTESTATIONS + GROUP BY + ORG_ATTESTATIONS.PROJECT_ID, + ORG_ATTESTATIONS.org_id, + ORG_ATTESTATIONS.VOUCH + ), + ORG_FLAG_AGG AS ( + SELECT + PR_ORG_V.PROJECT_ID, + ARRAY_AGG(ROW (PR_ORG_V.org_id, PR_ORG_V.COUNT)) AS ORG_FLAGS, + SUM(PR_ORG_V.COUNT) AS PR_TOTAL_FLAGS + FROM + PR_ORG_V + WHERE + PR_ORG_V.VOUCH = FALSE + GROUP BY + PR_ORG_V.PROJECT_ID + ), + ORG_ATTESTATIONS_AGG AS ( + SELECT + PR_ORG_V.PROJECT_ID, + ARRAY_AGG(ROW (PR_ORG_V.org_id, PR_ORG_V.COUNT)) AS ORG_VOUCHES, + SUM(PR_ORG_V.COUNT) AS PR_TOTAL_VOUCHES + FROM + PR_ORG_V + WHERE + PR_ORG_V.VOUCH = TRUE + GROUP BY + PR_ORG_V.PROJECT_ID + ) + SELECT + ID, + PR_TOTAL_FLAGS, + PR_TOTAL_VOUCHES, + PR_TOTAL_FLAGS + PR_TOTAL_VOUCHES AS PR_TOTAL_ATTESTATIONS, + ORG_FLAGS, + ORG_VOUCHES, + UNIQ_ORGS + FROM + PROJECT + LEFT JOIN PR_ORG ON PR_ORG.PROJECT_ID = PROJECT.ID + LEFT JOIN ORG_FLAG_AGG ON ORG_FLAG_AGG.PROJECT_ID = PROJECT.ID + LEFT JOIN ORG_ATTESTATIONS_AGG ON ORG_ATTESTATIONS_AGG.PROJECT_ID = PROJECT.ID + WHERE + PROJECT.ID = $1 + `, + [project.id] + ); + + return ProjectStats.parse(result[0]); +}; diff --git a/src/controllers/utils/projectVerificationHelper.ts b/src/controllers/utils/projectVerificationHelper.ts index 879cea0..727cbbe 100644 --- a/src/controllers/utils/projectVerificationHelper.ts +++ b/src/controllers/utils/projectVerificationHelper.ts @@ -1,6 +1,6 @@ import { DataHandlerContext, Log } from "@subsquid/evm-processor"; import { Store } from "@subsquid/typeorm-store"; -import { ProjectVerificationAttestation } from "./easTypes"; +import { ProjectVerificationAttestation } from "./types"; import { SchemaDecodedItem } from "@ethereum-attestation-service/eas-sdk"; import { SafeParseReturnType } from "zod"; import { Attestor, AttestorOrganisation, Organisation } from "../../model"; diff --git a/src/controllers/utils/types.test.ts b/src/controllers/utils/types.test.ts new file mode 100644 index 0000000..24b8c28 --- /dev/null +++ b/src/controllers/utils/types.test.ts @@ -0,0 +1,20 @@ +import { orgCountTuplesTypes } from "./types"; + +describe.only("parse database query", () => { + it("should parse project stats query", () => { + const raw = `{"(0x2e22df9a11e06c306ed8f64ca45ceae02efcf8a443371395a78371bc4fb6f722,1)","(0xf63f2a7159ee674aa6fce42196a8bb0605eafcf20c19e91a7eafba8d39fa0404,1)"}`; + + const result = orgCountTuplesTypes.parse(raw); + + expect(result).toEqual([ + [ + "0x2e22df9a11e06c306ed8f64ca45ceae02efcf8a443371395a78371bc4fb6f722", + "1", + ], + [ + "0xf63f2a7159ee674aa6fce42196a8bb0605eafcf20c19e91a7eafba8d39fa0404", + "1", + ], + ]); + }); +}); diff --git a/src/controllers/utils/types.ts b/src/controllers/utils/types.ts new file mode 100644 index 0000000..b297666 --- /dev/null +++ b/src/controllers/utils/types.ts @@ -0,0 +1,55 @@ +import { z } from "zod"; + +export const ProjectVerificationAttestation = z.object({ + vouch: z.boolean(), + projectSource: z.string(), + projectId: z.string(), + comment: z.string().optional(), +}); + +export type ProjectVerificationAttestation = z.infer< + typeof ProjectVerificationAttestation +>; + +const nullableIntType = z + .string() + .nullable() + .transform((val) => parseInt(val || "0")); + +export const orgCountTuplesTypes = z + .string() + .optional() + .nullable() + .transform((val) => { + if (!val) return []; + return ( + val + // Split the string using a regex that targets commas only if they are outside the parentheses + .slice(1, -1) + // Split the string using a regex that targets commas only if they are outside the parentheses + .split(/(?<=\)\"),(?=\"\()/) + // Remove leading and trailing quotes + .map((part) => + part + // Remove leading and trailing quotes + .trim() + .replace(/^"/, "") + .replace(/"$/, "") + // Remove '(' from begining and ')' from the end + .slice(1, -1) + .split(",") + ) + ); + }); + +export const ProjectStats = z.object({ + id: z.string(), + pr_total_flags: nullableIntType, + pr_total_vouches: nullableIntType, + pr_total_attestations: nullableIntType, + org_flags: orgCountTuplesTypes, + org_vouches: orgCountTuplesTypes, + uniq_orgs: z.array(z.string()), +}); + +export type ProjectStats = z.infer; diff --git a/src/model/generated/index.ts b/src/model/generated/index.ts index 1c13f2c..cbcf289 100644 --- a/src/model/generated/index.ts +++ b/src/model/generated/index.ts @@ -3,3 +3,4 @@ export * from "./attestorOrganisation.model" export * from "./attestor.model" export * from "./organisation.model" export * from "./project.model" +export * from "./organisationProject.model" diff --git a/src/model/generated/organisation.model.ts b/src/model/generated/organisation.model.ts index 64e1b0b..b2db066 100644 --- a/src/model/generated/organisation.model.ts +++ b/src/model/generated/organisation.model.ts @@ -1,5 +1,6 @@ import {Entity as Entity_, Column as Column_, PrimaryColumn as PrimaryColumn_, Index as Index_, OneToMany as OneToMany_} from "typeorm" import {AttestorOrganisation} from "./attestorOrganisation.model" +import {OrganisationProject} from "./organisationProject.model" @Entity_() export class Organisation { @@ -37,4 +38,7 @@ export class Organisation { */ @OneToMany_(() => AttestorOrganisation, e => e.organisation) attestors!: AttestorOrganisation[] + + @OneToMany_(() => OrganisationProject, e => e.organisation) + attestedProjects!: OrganisationProject[] } diff --git a/src/model/generated/organisationProject.model.ts b/src/model/generated/organisationProject.model.ts new file mode 100644 index 0000000..75be74c --- /dev/null +++ b/src/model/generated/organisationProject.model.ts @@ -0,0 +1,30 @@ +import {Entity as Entity_, Column as Column_, PrimaryColumn as PrimaryColumn_, ManyToOne as ManyToOne_, Index as Index_} from "typeorm" +import {Organisation} from "./organisation.model" +import {Project} from "./project.model" + +@Entity_() +export class OrganisationProject { + constructor(props?: Partial) { + Object.assign(this, props) + } + + /** + * - + */ + @PrimaryColumn_() + id!: string + + @Index_() + @ManyToOne_(() => Organisation, {nullable: true}) + organisation!: Organisation + + @Index_() + @ManyToOne_(() => Project, {nullable: true}) + project!: Project + + @Column_("bool", {nullable: false}) + vouch!: boolean + + @Column_("int4", {nullable: false}) + count!: number +} diff --git a/src/model/generated/project.model.ts b/src/model/generated/project.model.ts index 653b257..37ffca4 100644 --- a/src/model/generated/project.model.ts +++ b/src/model/generated/project.model.ts @@ -1,5 +1,6 @@ import {Entity as Entity_, Column as Column_, PrimaryColumn as PrimaryColumn_, Index as Index_, OneToMany as OneToMany_} from "typeorm" import {ProjectAttestation} from "./projectAttestation.model" +import {OrganisationProject} from "./organisationProject.model" @Entity_() export class Project { @@ -51,9 +52,18 @@ export class Project { @Column_("int4", {nullable: false}) totalFlags!: number + /** + * Total attests + */ + @Column_("int4", {nullable: false}) + totalAttests!: number + @Column_("timestamp with time zone", {nullable: false}) lastUpdatedTimestamp!: Date @OneToMany_(() => ProjectAttestation, e => e.project) attests!: ProjectAttestation[] + + @OneToMany_(() => OrganisationProject, e => e.project) + attestedOrganisations!: OrganisationProject[] } diff --git a/src/test/utils.ts b/src/test/utils.ts index 7c1dd56..96cce4b 100644 --- a/src/test/utils.ts +++ b/src/test/utils.ts @@ -1,6 +1,6 @@ import { Store, TypeormDatabase } from "@subsquid/typeorm-store"; import { createOrmConfig } from "@subsquid/typeorm-config"; -import { DataSource, EntityManager } from "typeorm"; +import { DataSource, DataSourceOptions, EntityManager } from "typeorm"; import { DataHandlerContext } from "@subsquid/evm-processor"; // import dotenv from "dotenv"; From 667b038565f8c0971479bcd1b47c7ec8069bdcfb Mon Sep 17 00:00:00 2001 From: Amin Latifi Date: Wed, 15 May 2024 16:14:48 +0330 Subject: [PATCH 4/4] Fixed issue in computing the project verification attestation --- package.json | 4 ++-- src/controllers/authorizeAttestation.ts | 4 +++- src/controllers/projectVerificationAttestation.ts | 4 ++-- src/controllers/utils/modelHelper.ts | 4 ---- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/package.json b/package.json index bebde06..c6ae277 100644 --- a/package.json +++ b/package.json @@ -7,8 +7,8 @@ "test:run-fresh-db": "docker-compose -f docker-compose-potgres-test.yml down -v; docker-compose --env-file .env.test -f docker-compose-potgres-test.yml up -d", "test": "npm run test:run-fresh-db; dotenvx run --env-file=.env.test -- jest --runInBand", "clear:generate:migration:run:locally": "sqd codegen ; docker compose -f docker-compose-potgres.yml down -v;docker compose -f docker-compose-potgres.yml up -d;sqd migration:generate; sqd run", - "clear:run:locally": "sqd codegen ; docker compose -f docker-compose-potgres.yml down -v;docker compose -f docker-compose-potgres.yml up -d; sqd run", - "run:locally": "sqd build; sqd run" + "clear:run:locally": "sqd build ; docker compose -f docker-compose-potgres.yml down -v;docker compose -f docker-compose-potgres.yml up -d; sqd run", + "run:locally": "docker compose -f docker-compose-potgres.yml up -d; sqd build; sqd run" }, "dependencies": { "@ethereum-attestation-service/eas-sdk": "^1.5.0", diff --git a/src/controllers/authorizeAttestation.ts b/src/controllers/authorizeAttestation.ts index a0ec512..0938ee9 100644 --- a/src/controllers/authorizeAttestation.ts +++ b/src/controllers/authorizeAttestation.ts @@ -46,7 +46,9 @@ export const handleAuthorize = async ( ctx.store.upsert(attestorOrganisation); ctx.log.debug( - `Attestor ${accountAddress} authorized for organisation ${organisation.name}: ${attestorOrganisation}` + `Attestor ${accountAddress} authorized for organisation ${ + organisation.name + }: ${JSON.stringify(attestorOrganisation)}` ); }; diff --git a/src/controllers/projectVerificationAttestation.ts b/src/controllers/projectVerificationAttestation.ts index b46890a..0134b0a 100644 --- a/src/controllers/projectVerificationAttestation.ts +++ b/src/controllers/projectVerificationAttestation.ts @@ -62,7 +62,7 @@ export const handleProjectAttestation = async ( uid: ${uid} schemaUid: ${schemaUid} issuer: ${issuer} - decodedData: ${JSON.stringify(decodedData, null, 2)} + decodedData: ${JSON.stringify(decodedData)} projectVerificationAttestation: ${projectVerificationAttestation} `); throw new Error("Error parsing project verification attestation"); @@ -124,7 +124,7 @@ export const handleProjectAttestationRevoke = async ( attestation.revoked = true; await ctx.store.upsert(attestation); - ctx.log.debug(`Revoked project attestation ${attestation}`); + ctx.log.debug(`Revoked project attestation ${JSON.stringify(attestation)}`); await updateProjectAttestationCounts(ctx, attestation.project); }; diff --git a/src/controllers/utils/modelHelper.ts b/src/controllers/utils/modelHelper.ts index 95c763a..8f6d403 100644 --- a/src/controllers/utils/modelHelper.ts +++ b/src/controllers/utils/modelHelper.ts @@ -5,11 +5,9 @@ import { Organisation, OrganisationProject, Project, - ProjectAttestation, } from "../../model"; import { getEntityManger } from "./databaseHelper"; import { ProjectStats } from "./types"; -import { In, Not } from "typeorm"; export const upsertOrganisatoinProject = async ( ctx: DataHandlerContext, @@ -36,8 +34,6 @@ export const updateProjectAttestationCounts = async ( ): Promise => { const em = getEntityManger(ctx); const projectStats = await getProjectStats(ctx, project); - console.log("projectStats:", projectStats); - // throw new Error("Let's exit"); project.totalVouches = projectStats.pr_total_vouches; project.totalFlags = projectStats.pr_total_flags;