diff --git a/.github/workflows/deploy-branch.yml b/.github/workflows/deploy-branch.yml index 752d61b3..10b7bf3c 100644 --- a/.github/workflows/deploy-branch.yml +++ b/.github/workflows/deploy-branch.yml @@ -34,7 +34,7 @@ jobs: - name: Deploy run: | - flyctl -c fly.${{ env.DEPLOYMENT_ENVIRONMENT }}.toml deploy --remote-only --wait-timeout=3600 --env BUILD_TAG=`git rev-parse --short HEAD` + flyctl -c fly.${{ env.DEPLOYMENT_ENVIRONMENT }}.toml deploy --remote-only --wait-timeout=7200 --env BUILD_TAG=`git rev-parse --short HEAD` env: FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} diff --git a/package-lock.json b/package-lock.json index 230f3306..f2572fab 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "license": "ISC", "dependencies": { "@sentry/node": "^7.51.0", - "chainsauce": "github:gitcoinco/chainsauce", + "chainsauce": "github:gitcoinco/chainsauce#main", "cors": "^2.8.5", "csv-parser": "^3.0.0", "csv-writer": "^1.6.0", @@ -1344,6 +1344,97 @@ "node": ">= 8" } }, + "node_modules/@npmcli/agent": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@npmcli/agent/-/agent-2.1.1.tgz", + "integrity": "sha512-6RlbiOAi6L6uUYF4/CDEkDZQnKw0XDsFJVrEpnib8rAx2WRMOsUyAdgnvDpX/fdkDWxtqE+NHwF465llI2wR0g==", + "dependencies": { + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.1", + "lru-cache": "^10.0.1", + "socks-proxy-agent": "^8.0.1" + }, + "engines": { + "node": "^16.14.0 || >=18.0.0" + } + }, + "node_modules/@npmcli/agent/node_modules/agent-base": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.0.tgz", + "integrity": "sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==", + "dependencies": { + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@npmcli/agent/node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@npmcli/agent/node_modules/http-proxy-agent": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.0.tgz", + "integrity": "sha512-+ZT+iBxVUQ1asugqnD6oWoRiS25AkjNfG085dKJGtGxkdwLQrMKU5wJr2bOOFAXzKcTuqq+7fZlTMgG3SRfIYQ==", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@npmcli/agent/node_modules/https-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.2.tgz", + "integrity": "sha512-NmLNjm6ucYwtcUmL7JQC1ZQ57LmHP4lT15FQ8D61nak1rO6DH+fz5qNK2Ap5UN4ZapYICE3/0KodcLYSPsPbaA==", + "dependencies": { + "agent-base": "^7.0.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@npmcli/agent/node_modules/lru-cache": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.0.1.tgz", + "integrity": "sha512-IJ4uwUTi2qCccrioU6g9g/5rvvVl13bsdczUUcqbciD9iLr095yj8DQKdObriEvuNSx325N1rV1O0sJFszx75g==", + "engines": { + "node": "14 || >=16.14" + } + }, + "node_modules/@npmcli/agent/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "node_modules/@npmcli/agent/node_modules/socks-proxy-agent": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.2.tgz", + "integrity": "sha512-8zuqoLv1aP/66PHF5TqwJ7Czm3Yv32urJQHrVyhD7mmA6d61Zv8cIXQYPTWwmg6qlupnPvs/QKDmfa4P/qct2g==", + "dependencies": { + "agent-base": "^7.0.2", + "debug": "^4.3.4", + "socks": "^2.7.1" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/@npmcli/fs": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/@npmcli/fs/-/fs-3.1.0.tgz", @@ -2699,16 +2790,119 @@ }, "node_modules/chainsauce": { "version": "1.0.19", - "resolved": "git+ssh://git@github.com/gitcoinco/chainsauce.git#a50b20afcab0e14f8508f39e35f15852c4406900", + "resolved": "git+ssh://git@github.com/gitcoinco/chainsauce.git#d64c96523160cf6341a13388299d2fa20dc3232f", "license": "MIT", "dependencies": { "better-sqlite3": "^8.2.0", "ethers": "^5.7.2", "express": "^4.18.2", - "node-fetch": "^3.3.1", + "fetch-retry": "^5.0.6", + "make-fetch-happen": "^13.0.0", "pino": "^8.14.1" } }, + "node_modules/chainsauce/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/chainsauce/node_modules/cacache": { + "version": "18.0.0", + "resolved": "https://registry.npmjs.org/cacache/-/cacache-18.0.0.tgz", + "integrity": "sha512-I7mVOPl3PUCeRub1U8YoGz2Lqv9WOBpobZ8RyWFXmReuILz+3OAyTa5oH3QPdtKZD7N0Yk00aLfzn0qvp8dZ1w==", + "dependencies": { + "@npmcli/fs": "^3.1.0", + "fs-minipass": "^3.0.0", + "glob": "^10.2.2", + "lru-cache": "^10.0.1", + "minipass": "^7.0.3", + "minipass-collect": "^1.0.2", + "minipass-flush": "^1.0.5", + "minipass-pipeline": "^1.2.4", + "p-map": "^4.0.0", + "ssri": "^10.0.0", + "tar": "^6.1.11", + "unique-filename": "^3.0.0" + }, + "engines": { + "node": "^16.14.0 || >=18.0.0" + } + }, + "node_modules/chainsauce/node_modules/glob": { + "version": "10.3.4", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.4.tgz", + "integrity": "sha512-6LFElP3A+i/Q8XQKEvZjkEWEOTgAIALR9AO2rwT8bgPhDd1anmqDJDZ6lLddI4ehxxxR1S5RIqKe1uapMQfYaQ==", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^2.0.3", + "minimatch": "^9.0.1", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0", + "path-scurry": "^1.10.1" + }, + "bin": { + "glob": "dist/cjs/src/bin.js" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/chainsauce/node_modules/lru-cache": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.0.1.tgz", + "integrity": "sha512-IJ4uwUTi2qCccrioU6g9g/5rvvVl13bsdczUUcqbciD9iLr095yj8DQKdObriEvuNSx325N1rV1O0sJFszx75g==", + "engines": { + "node": "14 || >=16.14" + } + }, + "node_modules/chainsauce/node_modules/make-fetch-happen": { + "version": "13.0.0", + "resolved": "https://registry.npmjs.org/make-fetch-happen/-/make-fetch-happen-13.0.0.tgz", + "integrity": "sha512-7ThobcL8brtGo9CavByQrQi+23aIfgYU++wg4B87AIS8Rb2ZBt/MEaDqzA00Xwv/jUjAjYkLHjVolYuTLKda2A==", + "dependencies": { + "@npmcli/agent": "^2.0.0", + "cacache": "^18.0.0", + "http-cache-semantics": "^4.1.1", + "is-lambda": "^1.0.1", + "minipass": "^7.0.2", + "minipass-fetch": "^3.0.0", + "minipass-flush": "^1.0.5", + "minipass-pipeline": "^1.2.4", + "negotiator": "^0.6.3", + "promise-retry": "^2.0.1", + "ssri": "^10.0.0" + }, + "engines": { + "node": "^16.14.0 || >=18.0.0" + } + }, + "node_modules/chainsauce/node_modules/minimatch": { + "version": "9.0.3", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.3.tgz", + "integrity": "sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/chainsauce/node_modules/minipass": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.3.tgz", + "integrity": "sha512-LhbbwCfz3vsb12j/WkWQPZfKTsgqIe1Nf/ti1pKjYESGLHIVjWU96G9/ljLH4F9mWNVhlQOm0VySdAWzf05dpg==", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, "node_modules/check-error": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.2.tgz", @@ -3040,13 +3234,6 @@ "resolved": "https://registry.npmjs.org/csv-writer/-/csv-writer-1.6.0.tgz", "integrity": "sha512-NOx7YDFWEsM/fTRAJjRpPp8t+MKRVvniAg9wQlUKx20MFrPs73WLJhFf5iteqrxNYnsy924K3Iroh3yNHeYd2g==" }, - "node_modules/data-uri-to-buffer": { - "version": "4.0.1", - "license": "MIT", - "engines": { - "node": ">= 12" - } - }, "node_modules/date-fns": { "version": "2.30.0", "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.30.0.tgz", @@ -3756,27 +3943,6 @@ "reusify": "^1.0.4" } }, - "node_modules/fetch-blob": { - "version": "3.2.0", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "paypal", - "url": "https://paypal.me/jimmywarting" - } - ], - "license": "MIT", - "dependencies": { - "node-domexception": "^1.0.0", - "web-streams-polyfill": "^3.0.3" - }, - "engines": { - "node": "^12.20 || >= 14.13" - } - }, "node_modules/fetch-retry": { "version": "5.0.6", "resolved": "https://registry.npmjs.org/fetch-retry/-/fetch-retry-5.0.6.tgz", @@ -3882,16 +4048,6 @@ "node": ">= 6" } }, - "node_modules/formdata-polyfill": { - "version": "4.0.10", - "license": "MIT", - "dependencies": { - "fetch-blob": "^3.1.2" - }, - "engines": { - "node": ">=12.20.0" - } - }, "node_modules/formidable": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/formidable/-/formidable-2.1.2.tgz", @@ -5173,39 +5329,6 @@ "node": ">=10" } }, - "node_modules/node-domexception": { - "version": "1.0.0", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "github", - "url": "https://paypal.me/jimmywarting" - } - ], - "license": "MIT", - "engines": { - "node": ">=10.5.0" - } - }, - "node_modules/node-fetch": { - "version": "3.3.1", - "license": "MIT", - "dependencies": { - "data-uri-to-buffer": "^4.0.0", - "fetch-blob": "^3.1.4", - "formdata-polyfill": "^4.0.10" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/node-fetch" - } - }, "node_modules/node-gyp-build": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.6.0.tgz", @@ -7046,13 +7169,6 @@ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", "dev": true }, - "node_modules/web-streams-polyfill": { - "version": "3.2.1", - "license": "MIT", - "engines": { - "node": ">= 8" - } - }, "node_modules/well-known-symbols": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/well-known-symbols/-/well-known-symbols-2.0.0.tgz", @@ -7896,6 +8012,73 @@ "fastq": "^1.6.0" } }, + "@npmcli/agent": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@npmcli/agent/-/agent-2.1.1.tgz", + "integrity": "sha512-6RlbiOAi6L6uUYF4/CDEkDZQnKw0XDsFJVrEpnib8rAx2WRMOsUyAdgnvDpX/fdkDWxtqE+NHwF465llI2wR0g==", + "requires": { + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.1", + "lru-cache": "^10.0.1", + "socks-proxy-agent": "^8.0.1" + }, + "dependencies": { + "agent-base": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.0.tgz", + "integrity": "sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==", + "requires": { + "debug": "^4.3.4" + } + }, + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "requires": { + "ms": "2.1.2" + } + }, + "http-proxy-agent": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.0.tgz", + "integrity": "sha512-+ZT+iBxVUQ1asugqnD6oWoRiS25AkjNfG085dKJGtGxkdwLQrMKU5wJr2bOOFAXzKcTuqq+7fZlTMgG3SRfIYQ==", + "requires": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + } + }, + "https-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.2.tgz", + "integrity": "sha512-NmLNjm6ucYwtcUmL7JQC1ZQ57LmHP4lT15FQ8D61nak1rO6DH+fz5qNK2Ap5UN4ZapYICE3/0KodcLYSPsPbaA==", + "requires": { + "agent-base": "^7.0.2", + "debug": "4" + } + }, + "lru-cache": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.0.1.tgz", + "integrity": "sha512-IJ4uwUTi2qCccrioU6g9g/5rvvVl13bsdczUUcqbciD9iLr095yj8DQKdObriEvuNSx325N1rV1O0sJFszx75g==" + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "socks-proxy-agent": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.2.tgz", + "integrity": "sha512-8zuqoLv1aP/66PHF5TqwJ7Czm3Yv32urJQHrVyhD7mmA6d61Zv8cIXQYPTWwmg6qlupnPvs/QKDmfa4P/qct2g==", + "requires": { + "agent-base": "^7.0.2", + "debug": "^4.3.4", + "socks": "^2.7.1" + } + } + } + }, "@npmcli/fs": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/@npmcli/fs/-/fs-3.1.0.tgz", @@ -8885,14 +9068,92 @@ } }, "chainsauce": { - "version": "git+ssh://git@github.com/gitcoinco/chainsauce.git#a50b20afcab0e14f8508f39e35f15852c4406900", - "from": "chainsauce@github:gitcoinco/chainsauce", + "version": "git+ssh://git@github.com/gitcoinco/chainsauce.git#d64c96523160cf6341a13388299d2fa20dc3232f", + "from": "chainsauce@github:gitcoinco/chainsauce#main", "requires": { "better-sqlite3": "^8.2.0", "ethers": "^5.7.2", "express": "^4.18.2", - "node-fetch": "^3.3.1", + "fetch-retry": "^5.0.6", + "make-fetch-happen": "^13.0.0", "pino": "^8.14.1" + }, + "dependencies": { + "brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "requires": { + "balanced-match": "^1.0.0" + } + }, + "cacache": { + "version": "18.0.0", + "resolved": "https://registry.npmjs.org/cacache/-/cacache-18.0.0.tgz", + "integrity": "sha512-I7mVOPl3PUCeRub1U8YoGz2Lqv9WOBpobZ8RyWFXmReuILz+3OAyTa5oH3QPdtKZD7N0Yk00aLfzn0qvp8dZ1w==", + "requires": { + "@npmcli/fs": "^3.1.0", + "fs-minipass": "^3.0.0", + "glob": "^10.2.2", + "lru-cache": "^10.0.1", + "minipass": "^7.0.3", + "minipass-collect": "^1.0.2", + "minipass-flush": "^1.0.5", + "minipass-pipeline": "^1.2.4", + "p-map": "^4.0.0", + "ssri": "^10.0.0", + "tar": "^6.1.11", + "unique-filename": "^3.0.0" + } + }, + "glob": { + "version": "10.3.4", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.4.tgz", + "integrity": "sha512-6LFElP3A+i/Q8XQKEvZjkEWEOTgAIALR9AO2rwT8bgPhDd1anmqDJDZ6lLddI4ehxxxR1S5RIqKe1uapMQfYaQ==", + "requires": { + "foreground-child": "^3.1.0", + "jackspeak": "^2.0.3", + "minimatch": "^9.0.1", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0", + "path-scurry": "^1.10.1" + } + }, + "lru-cache": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.0.1.tgz", + "integrity": "sha512-IJ4uwUTi2qCccrioU6g9g/5rvvVl13bsdczUUcqbciD9iLr095yj8DQKdObriEvuNSx325N1rV1O0sJFszx75g==" + }, + "make-fetch-happen": { + "version": "13.0.0", + "resolved": "https://registry.npmjs.org/make-fetch-happen/-/make-fetch-happen-13.0.0.tgz", + "integrity": "sha512-7ThobcL8brtGo9CavByQrQi+23aIfgYU++wg4B87AIS8Rb2ZBt/MEaDqzA00Xwv/jUjAjYkLHjVolYuTLKda2A==", + "requires": { + "@npmcli/agent": "^2.0.0", + "cacache": "^18.0.0", + "http-cache-semantics": "^4.1.1", + "is-lambda": "^1.0.1", + "minipass": "^7.0.2", + "minipass-fetch": "^3.0.0", + "minipass-flush": "^1.0.5", + "minipass-pipeline": "^1.2.4", + "negotiator": "^0.6.3", + "promise-retry": "^2.0.1", + "ssri": "^10.0.0" + } + }, + "minimatch": { + "version": "9.0.3", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.3.tgz", + "integrity": "sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==", + "requires": { + "brace-expansion": "^2.0.1" + } + }, + "minipass": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.3.tgz", + "integrity": "sha512-LhbbwCfz3vsb12j/WkWQPZfKTsgqIe1Nf/ti1pKjYESGLHIVjWU96G9/ljLH4F9mWNVhlQOm0VySdAWzf05dpg==" + } } }, "check-error": { @@ -9148,9 +9409,6 @@ "resolved": "https://registry.npmjs.org/csv-writer/-/csv-writer-1.6.0.tgz", "integrity": "sha512-NOx7YDFWEsM/fTRAJjRpPp8t+MKRVvniAg9wQlUKx20MFrPs73WLJhFf5iteqrxNYnsy924K3Iroh3yNHeYd2g==" }, - "data-uri-to-buffer": { - "version": "4.0.1" - }, "date-fns": { "version": "2.30.0", "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.30.0.tgz", @@ -9662,13 +9920,6 @@ "reusify": "^1.0.4" } }, - "fetch-blob": { - "version": "3.2.0", - "requires": { - "node-domexception": "^1.0.0", - "web-streams-polyfill": "^3.0.3" - } - }, "fetch-retry": { "version": "5.0.6", "resolved": "https://registry.npmjs.org/fetch-retry/-/fetch-retry-5.0.6.tgz", @@ -9742,12 +9993,6 @@ "mime-types": "^2.1.12" } }, - "formdata-polyfill": { - "version": "4.0.10", - "requires": { - "fetch-blob": "^3.1.2" - } - }, "formidable": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/formidable/-/formidable-2.1.2.tgz", @@ -10607,17 +10852,6 @@ } } }, - "node-domexception": { - "version": "1.0.0" - }, - "node-fetch": { - "version": "3.3.1", - "requires": { - "data-uri-to-buffer": "^4.0.0", - "fetch-blob": "^3.1.4", - "formdata-polyfill": "^4.0.10" - } - }, "node-gyp-build": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.6.0.tgz", @@ -11812,9 +12046,6 @@ } } }, - "web-streams-polyfill": { - "version": "3.2.1" - }, "well-known-symbols": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/well-known-symbols/-/well-known-symbols-2.0.0.tgz", diff --git a/package.json b/package.json index 70b69f61..5692c500 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,10 @@ "scripts": { "start": "node dist/src/index.js", "dev": "tsx watch src/index.ts | pino-pretty", + "test:snapshot:generate-before": "NODE_ENV=snapshot-testing INDEXED_CHAINS=mainnet STORAGE_DIR=test/snapshot.BEFORE/data CACHE_DIR=test/snapshot.BEFORE/cache tsx src/index.ts --to-block 17199975 --run-once | pino-pretty", + "test:snapshot:generate-after": "NODE_ENV=snapshot-testing INDEXED_CHAINS=mainnet STORAGE_DIR=test/snapshot.AFTER/data CACHE_DIR=test/snapshot.AFTER/cache tsx src/index.ts --to-block 17199975 --run-once | pino-pretty", + "test:snapshot:compare": "diff -r --brief test/snapshot.BEFORE/data test/snapshot.AFTER/data", + "test:snapshot": "test -d test/snapshot.BEFORE/data || (echo 'Missing BEFORE snapshot.'; exit 1); npm run test:snapshot:generate-after && npm run test:snapshot:compare && echo 'Snapshots match!'", "build": "tsc", "lint": "eslint src", "test": "vitest run --reporter verbose", @@ -25,7 +29,7 @@ "license": "ISC", "dependencies": { "@sentry/node": "^7.51.0", - "chainsauce": "github:gitcoinco/chainsauce", + "chainsauce": "github:gitcoinco/chainsauce#main", "cors": "^2.8.5", "csv-parser": "^3.0.0", "csv-writer": "^1.6.0", diff --git a/src/index.ts b/src/index.ts index 80119996..2b886235 100644 --- a/src/index.ts +++ b/src/index.ts @@ -22,6 +22,7 @@ import { createPriceProvider } from "./prices/provider.js"; import { createHttpApi } from "./http/app.js"; import { FileSystemDataProvider } from "./calculator/index.js"; import { AsyncSentinel } from "./utils/asyncSentinel.js"; +import { ethers } from "ethers"; // If, during reindexing, a chain has these many blocks left to index, consider // it caught up and start serving @@ -30,6 +31,9 @@ const MINIMUM_BLOCKS_LEFT_BEFORE_STARTING = 500; async function main(): Promise { const config = getConfig(); + // https://github.com/gitcoinco/allo-indexer/issues/215#issuecomment-1711380810 + ethers.utils.Logger.setLogLevel(ethers.utils.Logger.levels.ERROR); + if (config.sentryDsn !== null) { Sentry.init({ environment: config.deploymentEnvironment, @@ -65,24 +69,33 @@ async function main(): Promise { ), }); - // Promise will be resolved once the catchup is done. Afterwards, services - // will still be in listen-and-update mode - const [passportProvider, _] = await Promise.all([ - catchupAndWatchPassport({ - ...config, - baseLogger, - runOnce: config.runOnce, - }), - ...config.chains.map(async (chain) => - catchupAndWatchChain({ - chain, - baseLogger, + if (config.runOnce) { + await Promise.all( + config.chains.map(async (chain) => + catchupAndWatchChain({ chain, baseLogger, ...config }) + ) + ); + // Workaround for active handles preventing process to terminate + // (to investigate: console.log(process._getActiveHandles())) + // Note: the delay is necessary to allow completing writes. + baseLogger.info("exiting"); + await new Promise((resolve) => setTimeout(resolve, 1000)); + process.exit(0); + } else { + // Promises will be resolved once the initial catchup is done. Afterwards, services + // will still be in listen-and-update mode. + + const [passportProvider] = await Promise.all([ + catchupAndWatchPassport({ ...config, - }) - ), - ]); + baseLogger, + runOnce: config.runOnce, + }), + ...config.chains.map(async (chain) => + catchupAndWatchChain({ chain, baseLogger, ...config }) + ), + ]); - if (!config.runOnce) { const httpApi = createHttpApi({ storageDir: config.storageDir, priceProvider: createPriceProvider({ @@ -177,9 +190,20 @@ async function catchupAndWatchChain( return undefined; } - const res = await fetch(`${config.ipfsGateway}/ipfs/${cid}`, { + const url = `${config.ipfsGateway}/ipfs/${cid}`; + + chainLogger.trace(`Fetching ${url}`); + + const res = await fetch(url, { timeout: 2000, - retry: { retries: 10, maxTimeout: 60 * 1000 }, + onRetry(cause) { + chainLogger.debug({ + msg: "Retrying IPFS request", + url: url, + err: cause, + }); + }, + retry: { retries: 3, minTimeout: 2000, maxTimeout: 60 * 10000 }, // IPFS data is immutable, we can rely entirely on the cache when present cache: "force-cache", cachePath: @@ -219,9 +243,9 @@ async function catchupAndWatchChain( // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call const throttledLogProgress = throttle( 5000, - (currentBlock: number, lastBlock: number, blocksLeft: number) => { + (currentBlock: number, lastBlock: number, pendingEventsCount: number) => { indexerLogger.info( - `indexed to block ${currentBlock}; last block on chain: ${lastBlock}; left: ${blocksLeft}` + `pending events: ${pendingEventsCount} (indexing blocks ${currentBlock}-${lastBlock})` ); } ); @@ -253,23 +277,19 @@ async function catchupAndWatchChain( eventCacheDirectory: config.cacheDir ? path.join(config.cacheDir, "events") : null, - onProgress: ({ currentBlock, lastBlock }) => { - throttledLogProgress( - currentBlock, - lastBlock, - lastBlock - currentBlock - ); - - indexerLogger.trace( - `indexed to block ${currentBlock}; last block on chain: ${lastBlock}; left: ${ - lastBlock - currentBlock - }` - ); + requireExplicitStart: true, + onProgress: ({ currentBlock, lastBlock, pendingEventsCount }) => { + throttledLogProgress(currentBlock, lastBlock, pendingEventsCount); + if ( lastBlock - currentBlock < MINIMUM_BLOCKS_LEFT_BEFORE_STARTING && !catchupSentinel.isDone() ) { - indexerLogger.info("caught up with blockchain events"); + indexerLogger.info({ + msg: "caught up with blockchain events", + lastBlock, + currentBlock, + }); catchupSentinel.declareDone(); } }, @@ -277,6 +297,7 @@ async function catchupAndWatchChain( ); for (const subscription of config.chain.subscriptions) { + chainLogger.info(`subscribing to ${subscription.address}`); indexer.subscribe( subscription.address, subscription.abi, @@ -284,9 +305,12 @@ async function catchupAndWatchChain( ); } + indexer.start(); + await catchupSentinel.untilDone(); if (config.runOnce) { + priceUpdater.stop(); indexer.stop(); } else { chainLogger.info("listening to new blockchain events"); diff --git a/src/indexer/handleEvent.ts b/src/indexer/handleEvent.ts index b942b8f0..6edefa31 100644 --- a/src/indexer/handleEvent.ts +++ b/src/indexer/handleEvent.ts @@ -661,7 +661,7 @@ async function handleEvent( .insert({ ...vote, roundName: round.metadata?.name, - projectTitle: application.metadata?.application.project.title, + projectTitle: application.metadata?.application?.project?.title, roundStartTime: round.roundStartTime, roundEndTime: round.roundEndTime, }), diff --git a/src/prices/updater.ts b/src/prices/updater.ts index da3bb5c3..99b425ad 100644 --- a/src/prices/updater.ts +++ b/src/prices/updater.ts @@ -20,6 +20,7 @@ const POLL_INTERVAL_MS = 60 * 1000; export interface PriceUpdaterService { start: (opts?: { watch: boolean; toBlock: ToBlock }) => Promise; + stop: () => void; } interface PriceUpdaterConfig { @@ -37,6 +38,7 @@ export function createPriceUpdater( ): PriceUpdaterService { const { logger } = config; const withCacheMaybe = config.withCacheFn ?? ((_cacheKey, fn) => fn()); + let pollTimeoutId: NodeJS.Timeout | null = null; // PUBLIC @@ -51,7 +53,13 @@ export function createPriceUpdater( if (opts.watch) { logger.info("begin polling for new prices"); - setTimeout(poll, POLL_INTERVAL_MS); + pollTimeoutId = setTimeout(poll, POLL_INTERVAL_MS); + } + } + + function stop() { + if (pollTimeoutId) { + clearTimeout(pollTimeoutId); } } @@ -59,7 +67,7 @@ export function createPriceUpdater( const poll = async (): Promise => { await update("latest"); - setTimeout(poll, POLL_INTERVAL_MS); + pollTimeoutId = setTimeout(poll, POLL_INTERVAL_MS); }; async function update(toBlock: ToBlock) { @@ -190,7 +198,7 @@ export function createPriceUpdater( // API - return { start }; + return { start, stop }; } // UTILITIES