diff --git a/.env.template b/.env.template index b40d2ad..eb10b9e 100644 --- a/.env.template +++ b/.env.template @@ -19,3 +19,6 @@ SCHEMA_CONTRACT_ADDRESS=0x0a7E2Ff54e76B8E6659aedc9103FB21c038050D0 PROJECT_VERIFY_ATTESTATION_SCHEMA=0x3D5854AF182F27966DEA837C446A051B3509224DDC03150E55097B362D111B1B PROJECT_GIVBACK_ELIGIBLE_ATTESTATION_SCHEMA=0x0000000000000000000000000000000000000000000000000000000000000000 + +#APIs +GIVETH_API_URL=Https://somewhere.giveth.io/XXXXX diff --git a/.github/workflows/pipeline-develop.yml b/.github/workflows/pipeline-develop.yml index c706ce0..78ea5f7 100644 --- a/.github/workflows/pipeline-develop.yml +++ b/.github/workflows/pipeline-develop.yml @@ -22,7 +22,7 @@ jobs: git checkout -- . git pull docker compose stop devouch squid-db - docker compose rm -v -f devouch squid-db - docker rmi -f $(docker images | grep 'devouch' | awk '{print $3}') - docker volume rm $(docker volume ls | grep 'db-data' | awk '{print $2}') + # docker compose rm -v -f devouch squid-db + # docker rmi -f $(docker images | grep 'devouch' | awk '{print $3}') + # docker volume rm $(docker volume ls | grep 'db-data' | awk '{print $2}') docker compose up -d --build diff --git a/db/migrations/1716207083370-Data.js b/db/migrations/1716207083370-Data.js new file mode 100644 index 0000000..0fc1c3a --- /dev/null +++ b/db/migrations/1716207083370-Data.js @@ -0,0 +1,13 @@ +module.exports = class Data1716207083370 { + name = 'Data1716207083370' + + async up(db) { + await db.query(`ALTER TABLE "project" ADD "slug" text`) + await db.query(`ALTER TABLE "project" ADD "image" text`) + } + + async down(db) { + await db.query(`ALTER TABLE "project" DROP COLUMN "slug"`) + await db.query(`ALTER TABLE "project" DROP COLUMN "image"`) + } +} diff --git a/package-lock.json b/package-lock.json index ed3f44b..310a5b2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "@subsquid/typeorm-store": "^1.4.0", "dotenv": "^16.4.4", "ethers": "^6.12.1", + "node-cron": "^3.0.3", "pg": "^8.11.5", "type-graphql": "^1.2.0-rc.1", "typeorm": "^0.3.20", @@ -25,6 +26,7 @@ "@subsquid/typeorm-codegen": "^1.3.3", "@types/jest": "^29.5.12", "@types/node": "^20.11.17", + "@types/node-cron": "^3.0.11", "jest": "^29.7.0", "ts-jest": "^29.1.2", "ts-node": "^10.9.2", @@ -4305,6 +4307,12 @@ "undici-types": "~5.26.4" } }, + "node_modules/@types/node-cron": { + "version": "3.0.11", + "resolved": "https://registry.npmjs.org/@types/node-cron/-/node-cron-3.0.11.tgz", + "integrity": "sha512-0ikrnug3/IyneSHqCBeslAhlK2aBfYek1fGo4bP4QnZPmiqSGRK+Oy7ZMisLWkesffJvQ1cqAcBnJC+8+nxIAg==", + "dev": true + }, "node_modules/@types/pbkdf2": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/@types/pbkdf2/-/pbkdf2-3.1.2.tgz", @@ -9988,6 +9996,25 @@ "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-2.0.2.tgz", "integrity": "sha512-Ntyt4AIXyaLIuMHF6IOoTakB3K+RWxwtsHNRxllEoA6vPwP9o4866g6YWDLUdnucilZhmkxiHwHr11gAENw+QA==" }, + "node_modules/node-cron": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/node-cron/-/node-cron-3.0.3.tgz", + "integrity": "sha512-dOal67//nohNgYWb+nWmg5dkFdIwDm8EpeGYMekPMrngV3637lqnX0lbUcCtgibHTz6SEz7DAIjKvKDFYCnO1A==", + "dependencies": { + "uuid": "8.3.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/node-cron/node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/node-domexception": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", diff --git a/package.json b/package.json index c6ae277..6a21468 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "@subsquid/typeorm-store": "^1.4.0", "dotenv": "^16.4.4", "ethers": "^6.12.1", + "node-cron": "^3.0.3", "pg": "^8.11.5", "type-graphql": "^1.2.0-rc.1", "typeorm": "^0.3.20", @@ -30,6 +31,7 @@ "@subsquid/typeorm-codegen": "^1.3.3", "@types/jest": "^29.5.12", "@types/node": "^20.11.17", + "@types/node-cron": "^3.0.11", "jest": "^29.7.0", "ts-jest": "^29.1.2", "ts-node": "^10.9.2", diff --git a/schema.graphql b/schema.graphql index e8861ea..d36c109 100644 --- a/schema.graphql +++ b/schema.graphql @@ -59,6 +59,10 @@ type Project @entity { totalFlags: Int! "Total attests" totalAttests: Int! + "Slug of the project" + slug: String + "Image of the project" + image: String lastUpdatedTimestamp: DateTime! attests: [ProjectAttestation!]! @derivedFrom(field: "project") attestedOrganisations: [OrganisationProject!]! @derivedFrom(field: "project") diff --git a/src/features/import-projects/config.ts b/src/features/import-projects/config.ts new file mode 100644 index 0000000..be5f17d --- /dev/null +++ b/src/features/import-projects/config.ts @@ -0,0 +1 @@ +export const CRON_SCHEDULE = "0 0 * * *"; // UTC diff --git a/src/features/import-projects/giveth/constants.ts b/src/features/import-projects/giveth/constants.ts new file mode 100644 index 0000000..dca2815 --- /dev/null +++ b/src/features/import-projects/giveth/constants.ts @@ -0,0 +1,2 @@ +export const GIVETH_API_URL = + process.env.GIVETH_API_URL || "https://mainnet.serve.giveth.io/graphql"; diff --git a/src/features/import-projects/giveth/helpers.ts b/src/features/import-projects/giveth/helpers.ts new file mode 100644 index 0000000..c4f1def --- /dev/null +++ b/src/features/import-projects/giveth/helpers.ts @@ -0,0 +1,92 @@ +import { DataSource } from "typeorm"; +import { getDataSource } from "../../../helpers/db"; +import { Project } from "../../../model"; +import { fetchGivethProjectsBatch } from "./service"; +import { GivethProjectInfo } from "./type"; + +const updateOrCreateProject = async ( + dataSource: DataSource, + project: GivethProjectInfo +) => { + const existingProject = await dataSource + .getRepository(Project) + .createQueryBuilder("project") + .where("project.id = :id", { id: `giveth-${project.id}` }) + .getOne(); + + if (existingProject) { + const isUpdated = + existingProject.title !== project.title || + existingProject.description !== project.descriptionSummary || + existingProject.slug !== project.slug || + existingProject.image !== project.image; + + if (isUpdated) { + const updatedProject = new Project({ + ...existingProject, + title: project.title, + description: project.descriptionSummary, + image: project.image, + slug: project.slug, + lastUpdatedTimestamp: new Date(), + }); + + await dataSource + .createQueryBuilder() + .update(Project) + .set(updatedProject) + .where("id = :id", { id: `giveth-${project.id}` }) + .execute(); + } + } else { + const newProject = new Project({ + id: `giveth-${project.id}`, + title: project.title, + description: project.descriptionSummary, + image: project.image, + slug: project.slug, + projectId: project.id, + source: "giveth", + totalVouches: 0, + totalFlags: 0, + totalAttests: 0, + lastUpdatedTimestamp: new Date(), + }); + + await dataSource + .createQueryBuilder() + .insert() + .into(Project) + .values([newProject]) + .execute(); + } +}; + +const processProjectsBatch = async (projectsBatch: GivethProjectInfo[]) => { + const dataSource = await getDataSource(); + if (!dataSource) return; + for (const project of projectsBatch) { + console.log("Processing Project: Giveth", project.id); + await updateOrCreateProject(dataSource, project); + } +}; + +export const fetchAndProcessGivethProjects = async () => { + try { + let hasMoreProjects = true; + let skip = 0; + const limit = 10; + + while (hasMoreProjects) { + const projectsBatch = await fetchGivethProjectsBatch(limit, skip); + if (projectsBatch.length > 0) { + await processProjectsBatch(projectsBatch); + skip += limit; + } else { + hasMoreProjects = false; + } + } + } catch (error: any) { + console.log("error on fetching giveth projects", error.message); + } +}; diff --git a/src/features/import-projects/giveth/service.ts b/src/features/import-projects/giveth/service.ts new file mode 100644 index 0000000..4854729 --- /dev/null +++ b/src/features/import-projects/giveth/service.ts @@ -0,0 +1,50 @@ +import { graphQLRequest } from "../../../helpers/request"; +import { GIVETH_API_URL } from "./constants"; +import { GivethProjectInfo } from "./type"; + +export const fetchGivethProjectsBatch = async (limit: number, skip: number) => { + const res = await graphQLRequest( + GIVETH_API_URL, + `query ($limit: Int, $skip: Int, $sortingBy: SortingField) { + allProjects( + limit: $limit + skip: $skip + sortingBy: $sortingBy + ) { + projects { + id + title + image + slug + descriptionSummary + } + } + }`, + { + limit, + skip, + sortingBy: "Newest", + } + ); + + return res.data.allProjects.projects; +}; + +export const fetchGivethProjects = async () => { + let allProjects: GivethProjectInfo[] = []; + let hasMoreProjects = true; + let skip = 0; + const limit = 10; + + while (hasMoreProjects) { + const projectsBatch = await fetchGivethProjectsBatch(limit, skip); + if (projectsBatch.length > 0) { + allProjects = allProjects.concat(projectsBatch); + skip += limit; + } else { + hasMoreProjects = false; + } + } + + return allProjects; +}; diff --git a/src/features/import-projects/giveth/type.ts b/src/features/import-projects/giveth/type.ts new file mode 100644 index 0000000..a7e9588 --- /dev/null +++ b/src/features/import-projects/giveth/type.ts @@ -0,0 +1,7 @@ +export type GivethProjectInfo = { + id: string; + title: string; + descriptionSummary: string; + slug: string; + image: string; +}; diff --git a/src/features/import-projects/index.ts b/src/features/import-projects/index.ts new file mode 100644 index 0000000..4ef3178 --- /dev/null +++ b/src/features/import-projects/index.ts @@ -0,0 +1,20 @@ +import cron from "node-cron"; +import { fetchAndProcessGivethProjects } from "./giveth/helpers"; +import { CRON_SCHEDULE } from "./config"; + +export const task = async () => { + console.log("Importing Projects", new Date()); + await fetchAndProcessGivethProjects(); +}; + +export const importProjects = async () => { + try { + console.log("Importing Projects has been scheduled."); + cron.schedule(CRON_SCHEDULE, task, { + scheduled: true, + timezone: "UTC", + }); + } catch (error) { + console.log("Error", error); + } +}; diff --git a/src/features/import-projects/standalone.ts b/src/features/import-projects/standalone.ts new file mode 100644 index 0000000..1eb1bfe --- /dev/null +++ b/src/features/import-projects/standalone.ts @@ -0,0 +1,3 @@ +import { task } from "."; + +task(); diff --git a/src/helpers/db.ts b/src/helpers/db.ts new file mode 100644 index 0000000..3e5d0f6 --- /dev/null +++ b/src/helpers/db.ts @@ -0,0 +1,25 @@ +import { createOrmConfig } from "@subsquid/typeorm-config"; +import { DataSource } from "typeorm"; + +let dataSource: DataSource | undefined = undefined; + +export const getDataSource = async () => { + try { + if (!dataSource || !dataSource.isInitialized) { + let cfg = createOrmConfig({ projectDir: __dirname + "/../.." }); + (cfg.entities as string[]).push(__dirname + "/../model/generated/*.ts"); + + dataSource = await new DataSource(cfg).initialize(); + } + return dataSource; + } catch (error) { + console.error("Failed to initialize DataSource", error); + return null; + } +}; + +export const createEntityManager = async () => { + const dataSource = await getDataSource(); + if (!dataSource) return null; + return dataSource.createEntityManager(); +}; diff --git a/src/helpers/request.ts b/src/helpers/request.ts new file mode 100644 index 0000000..f7ee8a8 --- /dev/null +++ b/src/helpers/request.ts @@ -0,0 +1,14 @@ +export const graphQLRequest = async ( + url: string, + query: string, + variables: any +) => { + const res = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ query, variables }), + }); + return await res.json(); +}; diff --git a/src/main.ts b/src/main.ts index dcf6af9..95e6ade 100644 --- a/src/main.ts +++ b/src/main.ts @@ -3,6 +3,7 @@ import { processor } from "./processor"; import * as EASContract from "./abi/EAS"; import { processAttest } from "./mappings/attest"; import { processRevokeLog } from "./mappings/revoke"; +import { importProjects } from "./features/import-projects/index"; processor.run(new TypeormDatabase({ supportHotBlocks: false }), async (ctx) => { for (let _block of ctx.blocks) { @@ -18,3 +19,5 @@ processor.run(new TypeormDatabase({ supportHotBlocks: false }), async (ctx) => { } } }); + +importProjects(); diff --git a/src/model/generated/project.model.ts b/src/model/generated/project.model.ts index 37ffca4..bce0129 100644 --- a/src/model/generated/project.model.ts +++ b/src/model/generated/project.model.ts @@ -58,6 +58,18 @@ export class Project { @Column_("int4", {nullable: false}) totalAttests!: number + /** + * Slug of the project + */ + @Column_("text", {nullable: true}) + slug!: string | undefined | null + + /** + * Image of the project + */ + @Column_("text", {nullable: true}) + image!: string | undefined | null + @Column_("timestamp with time zone", {nullable: false}) lastUpdatedTimestamp!: Date