diff --git a/.env.template b/.env.template index eb10b9e..7957e45 100644 --- a/.env.template +++ b/.env.template @@ -21,4 +21,5 @@ PROJECT_VERIFY_ATTESTATION_SCHEMA=0x3D5854AF182F27966DEA837C446A051B3509224DDC03 PROJECT_GIVBACK_ELIGIBLE_ATTESTATION_SCHEMA=0x0000000000000000000000000000000000000000000000000000000000000000 #APIs -GIVETH_API_URL=Https://somewhere.giveth.io/XXXXX +GIVETH_API_URL=https://somewhere.giveth.io/XXXXX +RPGF3_API_URL=https://somewhere.RPGF3.io/XXXXX \ No newline at end of file diff --git a/src/features/import-projects/giveth/constants.ts b/src/features/import-projects/giveth/constants.ts index dca2815..a8dafc1 100644 --- a/src/features/import-projects/giveth/constants.ts +++ b/src/features/import-projects/giveth/constants.ts @@ -1,2 +1,11 @@ export const GIVETH_API_URL = process.env.GIVETH_API_URL || "https://mainnet.serve.giveth.io/graphql"; + +export const givethSourceConfig: SourceConfig = { + source: "giveth", + idField: "id", + titleField: "title", + descriptionField: "descriptionSummary", + slugField: "slug", + imageField: "image", +}; diff --git a/src/features/import-projects/giveth/helpers.ts b/src/features/import-projects/giveth/helpers.ts index c4f1def..4044dce 100644 --- a/src/features/import-projects/giveth/helpers.ts +++ b/src/features/import-projects/giveth/helpers.ts @@ -1,92 +1,12 @@ -import { DataSource } from "typeorm"; import { getDataSource } from "../../../helpers/db"; -import { Project } from "../../../model"; -import { fetchGivethProjectsBatch } from "./service"; -import { GivethProjectInfo } from "./type"; +import { type GivethProjectInfo } from "./type"; +import { updateOrCreateProject } from "../helpers"; +import { givethSourceConfig } from "./constants"; -const updateOrCreateProject = async ( - dataSource: DataSource, - project: GivethProjectInfo +export const processProjectsBatch = async ( + projectsBatch: GivethProjectInfo[] ) => { - const existingProject = await dataSource - .getRepository(Project) - .createQueryBuilder("project") - .where("project.id = :id", { id: `giveth-${project.id}` }) - .getOne(); - - if (existingProject) { - const isUpdated = - existingProject.title !== project.title || - existingProject.description !== project.descriptionSummary || - existingProject.slug !== project.slug || - existingProject.image !== project.image; - - if (isUpdated) { - const updatedProject = new Project({ - ...existingProject, - title: project.title, - description: project.descriptionSummary, - image: project.image, - slug: project.slug, - lastUpdatedTimestamp: new Date(), - }); - - await dataSource - .createQueryBuilder() - .update(Project) - .set(updatedProject) - .where("id = :id", { id: `giveth-${project.id}` }) - .execute(); - } - } else { - const newProject = new Project({ - id: `giveth-${project.id}`, - title: project.title, - description: project.descriptionSummary, - image: project.image, - slug: project.slug, - projectId: project.id, - source: "giveth", - totalVouches: 0, - totalFlags: 0, - totalAttests: 0, - lastUpdatedTimestamp: new Date(), - }); - - await dataSource - .createQueryBuilder() - .insert() - .into(Project) - .values([newProject]) - .execute(); - } -}; - -const processProjectsBatch = async (projectsBatch: GivethProjectInfo[]) => { - const dataSource = await getDataSource(); - if (!dataSource) return; for (const project of projectsBatch) { - console.log("Processing Project: Giveth", project.id); - await updateOrCreateProject(dataSource, project); - } -}; - -export const fetchAndProcessGivethProjects = async () => { - try { - let hasMoreProjects = true; - let skip = 0; - const limit = 10; - - while (hasMoreProjects) { - const projectsBatch = await fetchGivethProjectsBatch(limit, skip); - if (projectsBatch.length > 0) { - await processProjectsBatch(projectsBatch); - skip += limit; - } else { - hasMoreProjects = false; - } - } - } catch (error: any) { - console.log("error on fetching giveth projects", error.message); + await updateOrCreateProject(project, givethSourceConfig); } }; diff --git a/src/features/import-projects/giveth/index.ts b/src/features/import-projects/giveth/index.ts new file mode 100644 index 0000000..5fcecfe --- /dev/null +++ b/src/features/import-projects/giveth/index.ts @@ -0,0 +1,22 @@ +import { processProjectsBatch } from "./helpers"; +import { fetchGivethProjectsBatch } from "./service"; + +export const fetchAndProcessGivethProjects = async () => { + try { + let hasMoreProjects = true; + let skip = 0; + const limit = 10; + + while (hasMoreProjects) { + const projectsBatch = await fetchGivethProjectsBatch(limit, skip); + if (projectsBatch.length > 0) { + await processProjectsBatch(projectsBatch); + skip += limit; + } else { + hasMoreProjects = false; + } + } + } catch (error: any) { + console.log("error on fetchAndProcessGivethProjects", error.message); + } +}; diff --git a/src/features/import-projects/giveth/service.ts b/src/features/import-projects/giveth/service.ts index 4854729..9869b33 100644 --- a/src/features/import-projects/giveth/service.ts +++ b/src/features/import-projects/giveth/service.ts @@ -3,31 +3,36 @@ import { GIVETH_API_URL } from "./constants"; import { GivethProjectInfo } from "./type"; export const fetchGivethProjectsBatch = async (limit: number, skip: number) => { - const res = await graphQLRequest( - GIVETH_API_URL, - `query ($limit: Int, $skip: Int, $sortingBy: SortingField) { - allProjects( - limit: $limit - skip: $skip - sortingBy: $sortingBy - ) { - projects { - id - title - image - slug - descriptionSummary + try { + const res = await graphQLRequest( + GIVETH_API_URL, + `query ($limit: Int, $skip: Int, $sortingBy: SortingField) { + allProjects( + limit: $limit + skip: $skip + sortingBy: $sortingBy + ) { + projects { + id + title + image + slug + descriptionSummary + } } + }`, + { + limit, + skip, + sortingBy: "Newest", } - }`, - { - limit, - skip, - sortingBy: "Newest", - } - ); + ); - return res.data.allProjects.projects; + return res.data.allProjects.projects; + } catch (error: any) { + console.log("error on fetchGivethProjectsBatch", error.message); + return []; + } }; export const fetchGivethProjects = async () => { diff --git a/src/features/import-projects/helpers.ts b/src/features/import-projects/helpers.ts new file mode 100644 index 0000000..d9f1685 --- /dev/null +++ b/src/features/import-projects/helpers.ts @@ -0,0 +1,88 @@ +import { type DataSource } from "typeorm"; +import { Project } from "../../model"; +import { getDataSource } from "../../helpers/db"; + +export const updateOrCreateProject = async ( + project: any, + sourConfig: SourceConfig +) => { + const { + source, + idField, + titleField, + descriptionField, + slugField, + imageField, + } = sourConfig; + + const id = `${source}-${project[idField]}`; + + const dataSource = await getDataSource(); + if (!dataSource) { + console.log( + `[${new Date().toISOString()}] - ERROR: Failed to UPSERT project. Data source not found. Project ID: ${id}` + ); + return; + } + + const existingProject = await dataSource + .getRepository(Project) + .createQueryBuilder("project") + .where("project.id = :id", { id }) + .getOne(); + + if (existingProject) { + const isUpdated = + existingProject.title !== project[titleField] || + existingProject.description !== project[descriptionField] || + existingProject.slug !== project[slugField] || + existingProject.image !== project[imageField]; + + if (isUpdated) { + const updatedProject = new Project({ + ...existingProject, + title: project[titleField], + description: project[descriptionField], + image: project[imageField], + slug: project[slugField], + lastUpdatedTimestamp: new Date(), + }); + + await dataSource + .createQueryBuilder() + .update(Project) + .set(updatedProject) + .where("id = :id", { id }) + .execute(); + + console.log( + `[${new Date().toISOString()}] - INFO: Project Updated. Project ID: ${id}` + ); + } + } else { + const newProject = new Project({ + id, + title: project[titleField], + description: project[descriptionField], + image: project[imageField], + slug: project[slugField], + projectId: project[idField], + source: source, + totalVouches: 0, + totalFlags: 0, + totalAttests: 0, + lastUpdatedTimestamp: new Date(), + }); + + await dataSource + .createQueryBuilder() + .insert() + .into(Project) + .values([newProject]) + .execute(); + + console.log( + `[${new Date().toISOString()}] - INFO: Project Created. Project ID: ${id}` + ); + } +}; diff --git a/src/features/import-projects/index.ts b/src/features/import-projects/index.ts index 3a059c0..ad428b6 100644 --- a/src/features/import-projects/index.ts +++ b/src/features/import-projects/index.ts @@ -1,10 +1,12 @@ import cron from "node-cron"; -import { fetchAndProcessGivethProjects } from "./giveth/helpers"; import { IMPORT_PROJECT_CRON_SCHEDULE } from "../../constants"; +import { fetchAndProcessGivethProjects } from "./giveth/index"; +import { fetchAndProcessRpgf3Projects } from "./rpgf"; export const task = async () => { console.log("Importing Projects", new Date()); - await fetchAndProcessGivethProjects(); + fetchAndProcessGivethProjects(); + fetchAndProcessRpgf3Projects(); }; export const importProjects = async () => { @@ -17,6 +19,6 @@ export const importProjects = async () => { timezone: "UTC", }); } catch (error) { - console.log("Error", error); + console.log("Error on scheduling importing project:", error); } }; diff --git a/src/features/import-projects/rpgf/constants.ts b/src/features/import-projects/rpgf/constants.ts new file mode 100644 index 0000000..e912b1c --- /dev/null +++ b/src/features/import-projects/rpgf/constants.ts @@ -0,0 +1,11 @@ +export const RPGF3_API_URL = + process.env.RPGF3_API_URL || "https://backend.pairwise.vote/mock/projects"; + +export const rpgf3SourceConfig: SourceConfig = { + source: "rpgf3", + idField: "RPGF3Id", + titleField: "name", + descriptionField: "impactDescription", + slugField: "url", + imageField: "image", +}; diff --git a/src/features/import-projects/rpgf/index.ts b/src/features/import-projects/rpgf/index.ts new file mode 100644 index 0000000..4e58cd1 --- /dev/null +++ b/src/features/import-projects/rpgf/index.ts @@ -0,0 +1,17 @@ +import { getDataSource } from "../../../helpers/db"; +import { updateOrCreateProject } from "../helpers"; +import { rpgf3SourceConfig } from "./constants"; +import { fetchRpgf3Projects } from "./service"; + +export const fetchAndProcessRpgf3Projects = async () => { + try { + const data = await fetchRpgf3Projects(); + if (!data) return; + for (const project of data) { + if (project.type.toLowerCase() !== "project") continue; + await updateOrCreateProject(project, rpgf3SourceConfig); + } + } catch (error: any) { + console.log("error on fetchAndProcessRpgf3Projects", error.message); + } +}; diff --git a/src/features/import-projects/rpgf/service.ts b/src/features/import-projects/rpgf/service.ts new file mode 100644 index 0000000..11df8cf --- /dev/null +++ b/src/features/import-projects/rpgf/service.ts @@ -0,0 +1,19 @@ +import { RPGF3_API_URL } from "./constants"; +import { Rpgf3ProjectInfo } from "./type"; + +export const fetchRpgf3Projects = async () => { + try { + const res = await fetch(RPGF3_API_URL, { + method: "GET", + headers: { + "Content-Type": "application/json", + }, + }); + const data: Rpgf3ProjectInfo[] = await res.json(); + + return data; + } catch (error: any) { + console.log("error on fetching fetchRpgf3Projects", error.message); + return null; + } +}; diff --git a/src/features/import-projects/rpgf/type.ts b/src/features/import-projects/rpgf/type.ts new file mode 100644 index 0000000..6944940 --- /dev/null +++ b/src/features/import-projects/rpgf/type.ts @@ -0,0 +1,14 @@ +export type Rpgf3ProjectInfo = { + id: string; + name: string; + poll_id: string; + url: string; + impactDescription: string; + contributionDescription: string; + RPGF3Id: string; + parentId: string; + image: string; + metadataUrl: string; + created_at: string; + type: string; +}; diff --git a/src/features/import-projects/types.ts b/src/features/import-projects/types.ts new file mode 100644 index 0000000..33351d4 --- /dev/null +++ b/src/features/import-projects/types.ts @@ -0,0 +1,8 @@ +interface SourceConfig { + source: string; + idField: string; + titleField: string; + descriptionField: string; + slugField: string; + imageField: string; +}