improvments for GG (#662)

* have fallback IPFS

* feat: store ipfs data in db

* chore: move prices to it's own schema

* Refactor lock acquisition and release logic (#661)

* update lock logic

* forceRelease of schema

* index single chain (#657)

* update schema name

* updates

* fix

* Revert "Refactor lock acquisition and release logic (#661)"

This reverts commit 9a2ffe1.

* chore: avoid duplicate write + cleanup (#664)

* chore: avoid duplicate write

* cleanup

* improve locking (#665)

add forciblyAcquireLockForSchema

* test

* remove unused code

* add unique constraint

* chore: revert acquireWriteLock function

* f

* fix: args parser

---------

Co-authored-by: Kurt <[email protected]>
Co-authored-by: Hussein Martinez <[email protected]>

Author: 3 people

.env.example

@@ -42,7 +42,8 @@ DATABASE_URL=postgres://postgres:postgres@localhost:5432/grants_stack_indexer
 # METIS_ANDROMEDA_RPC_URL
 
 #COINGECKO_API_KEY=
-#IPFS_GATEWAY=
+#IPFS_GATEWAYs=[]
+#WHITELISTED_ADDRESSES=["0x123..","0x456.."]
 
 # optional, enable the Postgraphile Pro plugin: https://www.npmjs.com/package/@graphile/pro
 #GRAPHILE_LICENSE

docs/reindexing.md

@@ -12,9 +12,15 @@ When deploying changes to the indexer, it's important to clarify the results you
     - The indexer will create a new schema in Postgres named `chain_data_${version}`. If this schema does not exist, it will be created, all necessary tables will be set up, and indexing will start from scratch.
     - If the schema already exists, the indexer will resume indexing from the last indexed block unless the `--drop-db` flag is specified via the CLI. This will drop the existing database and start fresh.
 
-### Using `--drop-db` in Development
+### Dropping Schemas in Development
 
-- During development, you can use the `--drop-db` flag to ensure the indexer always deletes the existing schema and migrates from scratch. This can be useful for testing schema changes and event handler modifications without retaining old data.
+- During development, you can use the `--drop-db` flag to ensure the indexer always deletes all existing schema and migrates from scratch. This can be useful for testing schema changes and event handler modifications without retaining old data.
+
+- During development, you can use the `--drop-chain-db` flag to ensure the indexer always deletes chain schema and migrates from scratch. 
+
+- During development, you can use the `--drop-ipfs-db` flag to ensure the indexer always deletes ipfs schema and migrates from scratch. 
+
+- During development, you can use the `--drop-price-db` flag to ensure the indexer always deletes price schema and migrates from scratch. 
 
 ### Important Notes
 

indexer-compose.yml

@@ -20,7 +20,7 @@ services:
       ENABLE_RESOURCE_MONITOR: ${ENABLE_RESOURCE_MONITOR}
       ESTIMATES_LINEARQF_WORKER_POOL_SIZE: ${ESTIMATES_LINEARQF_WORKER_POOL_SIZE}
       PINO_PRETTY: ${PINO_PRETTY}
-      IPFS_GATEWAY: ${IPFS_GATEWAY}
+      IPFS_GATEWAYS: ${IPFS_GATEWAYS}
       COINGECKO_API_KEY: ${COINGECKO_API_KEY}
       GRAPHILE_LICENSE: ${GRAPHILE_LICENSE}
       SEPOLIA_RPC_URL: ${SEPOLIA_RPC_URL}
@@ -62,7 +62,7 @@ services:
       ENABLE_RESOURCE_MONITOR: ${ENABLE_RESOURCE_MONITOR}
       ESTIMATES_LINEARQF_WORKER_POOL_SIZE: ${ESTIMATES_LINEARQF_WORKER_POOL_SIZE}
       PINO_PRETTY: ${PINO_PRETTY}
-      IPFS_GATEWAY: ${IPFS_GATEWAY}
+      IPFS_GATEWAYS: ${IPFS_GATEWAYS}
       COINGECKO_API_KEY: ${COINGECKO_API_KEY}
       GRAPHILE_LICENSE: ${GRAPHILE_LICENSE}
       SEPOLIA_RPC_URL: ${SEPOLIA_RPC_URL}

src/config.ts

@@ -20,7 +20,9 @@ type CoingeckoSupportedChainId =
   | 42220
   | 1088;
 
-const CHAIN_DATA_VERSION = "81";
+const CHAIN_DATA_VERSION = "83";
+const IPFS_DATA_VERSION = "1";
+const PRICE_DATA_VERSION = "1";
 
 export type Token = {
   code: string;
@@ -1818,7 +1820,7 @@ export type Config = {
   httpServerWaitForSync: boolean;
   httpServerEnabled: boolean;
   indexerEnabled: boolean;
-  ipfsGateway: string;
+  ipfsGateways: string[];
   coingeckoApiKey: string | null;
   coingeckoApiUrl: string;
   chains: Chain[];
@@ -1829,11 +1831,18 @@ export type Config = {
   readOnlyDatabaseUrl: string;
   dataVersion: string;
   databaseSchemaName: string;
+  ipfsDataVersion: string;
+  ipfsDatabaseSchemaName: string;
+  priceDataVersion: string;
+  priceDatabaseSchemaName: string;
   hostname: string;
   pinoPretty: boolean;
   deploymentEnvironment: "local" | "development" | "staging" | "production";
   enableResourceMonitor: boolean;
   dropDb: boolean;
+  dropChainDb: boolean;
+  dropIpfsDb: boolean;
+  dropPriceDb: boolean;
   removeCache: boolean;
   estimatesLinearQfWorkerPoolSize: number | null;
 };
@@ -1847,9 +1856,18 @@ export function getConfig(): Config {
       "from-block": {
         type: "string",
       },
+      "drop-chain-db": {
+        type: "boolean",
+      },
+      "drop-ipfs-db": {
+        type: "boolean",
+      },
       "drop-db": {
         type: "boolean",
       },
+      "drop-price-db": {
+        type: "boolean",
+      },
       "rm-cache": {
         type: "boolean",
       },
@@ -1981,10 +1999,11 @@ export function getConfig(): Config {
 
   const runOnce = z.boolean().default(false).parse(args["run-once"]);
 
-  const ipfsGateway = z
+  const ipfsGateways = z
     .string()
-    .default("https://ipfs.io")
-    .parse(process.env.IPFS_GATEWAY);
+    .array()
+    .default(["https://ipfs.io"])
+    .parse(JSON.parse(process.env.IPFS_GATEWAYS!));
 
   const sentryDsn = z
     .union([z.string(), z.null()])
@@ -2001,7 +2020,16 @@ export function getConfig(): Config {
   const dataVersion = CHAIN_DATA_VERSION;
   const databaseSchemaName = `chain_data_${dataVersion}`;
 
+  const ipfsDataVersion = IPFS_DATA_VERSION;
+  const ipfsDatabaseSchemaName = `ipfs_data_${ipfsDataVersion}`;
+
+  const priceDataVersion = PRICE_DATA_VERSION;
+  const priceDatabaseSchemaName = `price_data_${priceDataVersion}`;
+
   const dropDb = z.boolean().default(false).parse(args["drop-db"]);
+  const dropChainDb = z.boolean().default(false).parse(args["drop-chain-db"]);
+  const dropIpfsDb = z.boolean().default(false).parse(args["drop-ipfs-db"]);
+  const dropPriceDb = z.boolean().default(false).parse(args["drop-price-db"]);
 
   const removeCache = z.boolean().default(false).parse(args["rm-cache"]);
 
@@ -2041,7 +2069,7 @@ export function getConfig(): Config {
     cacheDir,
     logLevel,
     runOnce,
-    ipfsGateway,
+    ipfsGateways,
     passportScorerId,
     apiHttpPort,
     pinoPretty,
@@ -2050,9 +2078,16 @@ export function getConfig(): Config {
     databaseUrl,
     readOnlyDatabaseUrl,
     dropDb,
+    dropChainDb,
+    dropIpfsDb,
+    dropPriceDb,
     removeCache,
     dataVersion,
     databaseSchemaName,
+    ipfsDataVersion,
+    ipfsDatabaseSchemaName,
+    priceDataVersion,
+    priceDatabaseSchemaName,
     httpServerWaitForSync,
     httpServerEnabled,
     indexerEnabled,

src/database/changeset.ts

@@ -16,6 +16,7 @@ import {
   NewPrice,
   NewLegacyProject,
   NewApplicationPayout,
+  NewIpfsData,
 } from "./schema.js";
 
 export type DataChange =
@@ -140,4 +141,8 @@ export type DataChange =
   | {
       type: "InsertApplicationPayout";
       payout: NewApplicationPayout;
+    }
+  | {
+      type: "InsertIpfsData";
+      ipfs: NewIpfsData;
     };