From cb263b801270147257243a7fa079f0c84115bc8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20C=C3=A1rdenas?= Date: Wed, 1 Nov 2023 16:28:05 -0600 Subject: [PATCH 01/14] feat: add iterator and enum value handling helpers (#16) --- src/helpers/index.ts | 1 + src/helpers/iterators.ts | 78 ++++++++++++++++++++++++++++++++++++++++ src/helpers/values.ts | 12 +++++++ 3 files changed, 91 insertions(+) create mode 100644 src/helpers/iterators.ts diff --git a/src/helpers/index.ts b/src/helpers/index.ts index 430f360..4fa4ee7 100644 --- a/src/helpers/index.ts +++ b/src/helpers/index.ts @@ -1,2 +1,3 @@ +export * from './iterators'; export * from './time'; export * from './values'; diff --git a/src/helpers/iterators.ts b/src/helpers/iterators.ts new file mode 100644 index 0000000..0bbc61d --- /dev/null +++ b/src/helpers/iterators.ts @@ -0,0 +1,78 @@ +import { logger } from '../logger'; +import { isDevEnv } from './values'; + +/** + * Iterate over an array, yielding multiple items at a time. If the size of the given array + * is not divisible by the given batch size, then the length of the last items returned will + * be smaller than the given batch size, i.e.: + * ```typescript + * items.length % batchSize + * ``` + * @param items - The array to iterate over. + * @param batchSize - Maximum number of items to return at a time. + * @param printBenchmark - If we should print benchmark of items per second + */ +export function* batchIterate( + items: T[], + batchSize: number, + printBenchmark = isDevEnv +): Generator { + if (items.length === 0) return; + const startTime = Date.now(); + for (let i = 0; i < items.length; ) { + const itemsRemaining = items.length - i; + const sliceSize = Math.min(batchSize, itemsRemaining); + yield items.slice(i, i + sliceSize); + i += sliceSize; + } + if (printBenchmark) { + const itemsPerSecond = Math.round((items.length / (Date.now() - startTime)) * 1000); + const caller = new Error().stack?.split('at ')[3].trim(); + logger.debug(`Iterated ${itemsPerSecond} items/second at ${caller}`); + } +} + +/** + * Iterate over an `AsyncIterable`, yielding multiple items at a time. If the size of the given + * array is not divisible by the given batch size, then the length of the last items returned will + * be smaller than the given batch size. + * + * @param items - AsyncIterable + * @param batchSize - Batch size + * @param printBenchmark - If we should print benchmark of items per second + */ +export async function* asyncBatchIterate( + items: AsyncIterable, + batchSize: number, + printBenchmark = isDevEnv +): AsyncGenerator { + const startTime = Date.now(); + let itemCount = 0; + let itemBatch: T[] = []; + for await (const item of items) { + itemBatch.push(item); + itemCount++; + if (itemBatch.length >= batchSize) { + yield itemBatch; + itemBatch = []; + if (printBenchmark) { + const itemsPerSecond = Math.round((itemCount / (Date.now() - startTime)) * 1000); + const caller = new Error().stack?.split('at ')[3].trim(); + logger.debug(`Iterated ${itemsPerSecond} items/second at ${caller}`); + } + } + } + if (itemBatch.length > 0) { + yield itemBatch; + } +} + +/** + * Convert an `AsyncIterable` to a generator + * @param iter - AsyncIterable + */ +export async function* asyncIterableToGenerator(iter: AsyncIterable) { + for await (const entry of iter) { + yield entry; + } +} diff --git a/src/helpers/values.ts b/src/helpers/values.ts index 6f7dd98..cc1a34e 100644 --- a/src/helpers/values.ts +++ b/src/helpers/values.ts @@ -130,3 +130,15 @@ export function numberToHex(number: number, paddingBytes: number = 4): string { * @returns Boolean */ export const has0xPrefix = (val: string) => val.substring(0, 2).toLowerCase() === '0x'; + +/** + * Converts a string to an enum value. + * @param enumType - The enum type + * @param value - The string value to convert + * @returns Enum item or undefined + */ +export function toEnumValue(enm: { [s: string]: T }, value: string): T | undefined { + return (Object.values(enm) as unknown as string[]).includes(value) + ? (value as unknown as T) + : undefined; +} From c170280c6433c542312ca89adce00d3bb6200ada Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 1 Nov 2023 22:29:45 +0000 Subject: [PATCH 02/14] chore(release): 1.3.0 [skip ci] ## [1.3.0](https://github.com/hirosystems/api-toolkit/compare/v1.2.2...v1.3.0) (2023-11-01) ### Features * add iterator and enum value handling helpers ([#16](https://github.com/hirosystems/api-toolkit/issues/16)) ([cb263b8](https://github.com/hirosystems/api-toolkit/commit/cb263b801270147257243a7fa079f0c84115bc8d)) --- CHANGELOG.md | 7 +++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d83a9c3..d7e4bdf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [1.3.0](https://github.com/hirosystems/api-toolkit/compare/v1.2.2...v1.3.0) (2023-11-01) + + +### Features + +* add iterator and enum value handling helpers ([#16](https://github.com/hirosystems/api-toolkit/issues/16)) ([cb263b8](https://github.com/hirosystems/api-toolkit/commit/cb263b801270147257243a7fa079f0c84115bc8d)) + ## [1.2.2](https://github.com/hirosystems/api-toolkit/compare/v1.2.1...v1.2.2) (2023-10-27) diff --git a/package-lock.json b/package-lock.json index a80b738..ec19004 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@hirosystems/api-toolkit", - "version": "1.2.2", + "version": "1.3.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@hirosystems/api-toolkit", - "version": "1.2.2", + "version": "1.3.0", "license": "Apache 2.0", "dependencies": { "@fastify/cors": "^8.0.0", diff --git a/package.json b/package.json index 587d39d..f2c28e3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@hirosystems/api-toolkit", - "version": "1.2.2", + "version": "1.3.0", "description": "API development toolkit", "main": "./dist/index.js", "typings": "./dist/index.d.ts", From 9c56c19377e9b1f85d0f41eb6992083cabf8e9b2 Mon Sep 17 00:00:00 2001 From: Rafael Cardenas Date: Thu, 16 Nov 2023 17:28:21 -0600 Subject: [PATCH 03/14] fix: allow logging migrations --- src/postgres/migrations.ts | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/postgres/migrations.ts b/src/postgres/migrations.ts index 9e7a213..abea5a9 100644 --- a/src/postgres/migrations.ts +++ b/src/postgres/migrations.ts @@ -4,20 +4,24 @@ import { logger } from '../logger'; import { PgConnectionArgs, connectPostgres, standardizedConnectionArgs } from './connection'; import { isDevEnv, isTestEnv } from '../helpers/values'; +export interface MigrationOptions { + // Bypass the NODE_ENV check when performing a "down" migration which irreversibly drops data. + dangerousAllowDataLoss?: boolean; + logMigrations?: boolean; +} + /** * Run migrations in one direction. * @param dir - Migrations directory * @param direction - Migration direction (`'down'` or `'up'`) * @param connectionArgs - Postgres connection args + * @param opts - Migration options */ export async function runMigrations( dir: string, direction: MigrationDirection, connectionArgs?: PgConnectionArgs, - opts?: { - // Bypass the NODE_ENV check when performing a "down" migration which irreversibly drops data. - dangerousAllowDataLoss?: boolean; - } + opts?: MigrationOptions ) { if (!opts?.dangerousAllowDataLoss && direction !== 'up' && !isTestEnv && !isDevEnv) { throw new Error( @@ -43,7 +47,7 @@ export async function runMigrations( }, migrationsTable: 'pgmigrations', logger: { - info: _msg => {}, + info: msg => (opts?.logMigrations === true ? logger.info(msg) : {}), warn: msg => logger.warn(msg), error: msg => logger.error(msg), }, @@ -54,24 +58,23 @@ export async function runMigrations( * Cycle migrations down and up. * @param dir - Migrations directory * @param connectionArgs - Postgres connection args + * @param opts - Migration options */ export async function cycleMigrations( dir: string, connectionArgs?: PgConnectionArgs, - opts?: { - // Bypass the NODE_ENV check when performing a "down" migration which irreversibly drops data. - dangerousAllowDataLoss?: boolean; + opts?: MigrationOptions & { checkForEmptyData?: boolean; } ) { - await runMigrations(dir, 'down', connectionArgs); + await runMigrations(dir, 'down', connectionArgs, opts); if ( opts?.checkForEmptyData && (await databaseHasData(connectionArgs, { ignoreMigrationTables: true })) ) { throw new Error('Migration down process did not completely remove DB tables'); } - await runMigrations(dir, 'up', connectionArgs); + await runMigrations(dir, 'up', connectionArgs, opts); } /** From 31d0c77dd47ce560d07770bf3a8adf4ce922d536 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 16 Nov 2023 23:29:45 +0000 Subject: [PATCH 04/14] chore(release): 1.3.1 [skip ci] ## [1.3.1](https://github.com/hirosystems/api-toolkit/compare/v1.3.0...v1.3.1) (2023-11-16) ### Bug Fixes * allow logging migrations ([9c56c19](https://github.com/hirosystems/api-toolkit/commit/9c56c19377e9b1f85d0f41eb6992083cabf8e9b2)) --- CHANGELOG.md | 7 +++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7e4bdf..de9a222 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [1.3.1](https://github.com/hirosystems/api-toolkit/compare/v1.3.0...v1.3.1) (2023-11-16) + + +### Bug Fixes + +* allow logging migrations ([9c56c19](https://github.com/hirosystems/api-toolkit/commit/9c56c19377e9b1f85d0f41eb6992083cabf8e9b2)) + ## [1.3.0](https://github.com/hirosystems/api-toolkit/compare/v1.2.2...v1.3.0) (2023-11-01) diff --git a/package-lock.json b/package-lock.json index ec19004..e0482b5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@hirosystems/api-toolkit", - "version": "1.3.0", + "version": "1.3.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@hirosystems/api-toolkit", - "version": "1.3.0", + "version": "1.3.1", "license": "Apache 2.0", "dependencies": { "@fastify/cors": "^8.0.0", diff --git a/package.json b/package.json index f2c28e3..bd73baf 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@hirosystems/api-toolkit", - "version": "1.3.0", + "version": "1.3.1", "description": "API development toolkit", "main": "./dist/index.js", "typings": "./dist/index.d.ts", From aa9c0dd91acfbbf57ba671df0c0ecaacd0a8ea5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20C=C3=A1rdenas?= Date: Fri, 17 Nov 2023 12:35:42 -0600 Subject: [PATCH 05/14] fix: allow schema to be specified for migrations table (#17) --- package.json | 2 +- src/postgres/migrations.ts | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index bd73baf..d4e2f09 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "typings": "./dist/index.d.ts", "scripts": { "build": "rimraf ./dist && tsc --project tsconfig.build.json && copyfiles -u 1 ./src/server-version/*.mjs ./dist", - "test": "jest", + "test": "jest --runInBand", "lint:eslint": "eslint . --ext .js,.jsx,.ts,.tsx -f unix", "lint:prettier": "prettier --check src/**/*.ts", "testenv:run": "docker-compose -f docker/docker-compose.dev.postgres.yml up", diff --git a/src/postgres/migrations.ts b/src/postgres/migrations.ts index abea5a9..56eb599 100644 --- a/src/postgres/migrations.ts +++ b/src/postgres/migrations.ts @@ -5,9 +5,12 @@ import { PgConnectionArgs, connectPostgres, standardizedConnectionArgs } from '. import { isDevEnv, isTestEnv } from '../helpers/values'; export interface MigrationOptions { - // Bypass the NODE_ENV check when performing a "down" migration which irreversibly drops data. + /** Bypass the NODE_ENV check when performing a "down" migration which irreversibly drops data. */ dangerousAllowDataLoss?: boolean; + /** Log all applied migrations */ logMigrations?: boolean; + /** Name of the table used for migrations. Defaults to `pgmigrations`. */ + migrationsTable?: string; } /** @@ -45,7 +48,8 @@ export async function runMigrations( password: args.password, database: args.database, }, - migrationsTable: 'pgmigrations', + migrationsTable: opts?.migrationsTable ?? 'pgmigrations', + schema: typeof args === 'string' ? 'public' : args.schema, logger: { info: msg => (opts?.logMigrations === true ? logger.info(msg) : {}), warn: msg => logger.warn(msg), @@ -64,13 +68,17 @@ export async function cycleMigrations( dir: string, connectionArgs?: PgConnectionArgs, opts?: MigrationOptions & { + /** Validates if the database was cleared completely after all `down` migrations are done */ checkForEmptyData?: boolean; } ) { await runMigrations(dir, 'down', connectionArgs, opts); if ( opts?.checkForEmptyData && - (await databaseHasData(connectionArgs, { ignoreMigrationTables: true })) + (await databaseHasData(connectionArgs, { + ignoreMigrationTables: true, + migrationsTable: opts.migrationsTable, + })) ) { throw new Error('Migration down process did not completely remove DB tables'); } @@ -88,6 +96,7 @@ export async function databaseHasData( connectionArgs?: PgConnectionArgs, opts?: { ignoreMigrationTables?: boolean; + migrationsTable?: string; } ): Promise { const sql = await connectPostgres({ @@ -96,12 +105,13 @@ export async function databaseHasData( }); try { const ignoreMigrationTables = opts?.ignoreMigrationTables ?? false; + const tableName = opts?.migrationsTable ?? 'pgmigrations'; const result = await sql<{ count: number }[]>` SELECT COUNT(*) FROM pg_class c JOIN pg_namespace s ON s.oid = c.relnamespace WHERE s.nspname = ${sql.options.connection.search_path} - ${ignoreMigrationTables ? sql`AND c.relname NOT LIKE 'pgmigrations%'` : sql``} + ${ignoreMigrationTables ? sql`AND c.relname NOT LIKE '${tableName}%'` : sql``} `; return result.count > 0 && result[0].count > 0; } catch (error: any) { From 0c51e416d0633525465609ece83853acaa9909ab Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 17 Nov 2023 18:37:05 +0000 Subject: [PATCH 06/14] chore(release): 1.3.2 [skip ci] ## [1.3.2](https://github.com/hirosystems/api-toolkit/compare/v1.3.1...v1.3.2) (2023-11-17) ### Bug Fixes * allow schema to be specified for migrations table ([#17](https://github.com/hirosystems/api-toolkit/issues/17)) ([aa9c0dd](https://github.com/hirosystems/api-toolkit/commit/aa9c0dd91acfbbf57ba671df0c0ecaacd0a8ea5e)) --- CHANGELOG.md | 7 +++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de9a222..47b03dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [1.3.2](https://github.com/hirosystems/api-toolkit/compare/v1.3.1...v1.3.2) (2023-11-17) + + +### Bug Fixes + +* allow schema to be specified for migrations table ([#17](https://github.com/hirosystems/api-toolkit/issues/17)) ([aa9c0dd](https://github.com/hirosystems/api-toolkit/commit/aa9c0dd91acfbbf57ba671df0c0ecaacd0a8ea5e)) + ## [1.3.1](https://github.com/hirosystems/api-toolkit/compare/v1.3.0...v1.3.1) (2023-11-16) diff --git a/package-lock.json b/package-lock.json index e0482b5..ec80c89 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@hirosystems/api-toolkit", - "version": "1.3.1", + "version": "1.3.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@hirosystems/api-toolkit", - "version": "1.3.1", + "version": "1.3.2", "license": "Apache 2.0", "dependencies": { "@fastify/cors": "^8.0.0", diff --git a/package.json b/package.json index d4e2f09..b7df4de 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@hirosystems/api-toolkit", - "version": "1.3.1", + "version": "1.3.2", "description": "API development toolkit", "main": "./dist/index.js", "typings": "./dist/index.d.ts", From 2296ff373ca0fc8e04b4d3e37a0a59307097b10e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20C=C3=A1rdenas?= Date: Wed, 6 Dec 2023 16:35:08 -0600 Subject: [PATCH 07/14] fix: cast table name explicitly when checking for DB data (#18) --- src/postgres/migrations.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/postgres/migrations.ts b/src/postgres/migrations.ts index 56eb599..75d04d1 100644 --- a/src/postgres/migrations.ts +++ b/src/postgres/migrations.ts @@ -111,7 +111,7 @@ export async function databaseHasData( FROM pg_class c JOIN pg_namespace s ON s.oid = c.relnamespace WHERE s.nspname = ${sql.options.connection.search_path} - ${ignoreMigrationTables ? sql`AND c.relname NOT LIKE '${tableName}%'` : sql``} + ${ignoreMigrationTables ? sql`AND c.relname NOT LIKE ${tableName}::text || '%'` : sql``} `; return result.count > 0 && result[0].count > 0; } catch (error: any) { From bf2a9030307591a733ec0615a0143183aa5f56cd Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 6 Dec 2023 22:36:32 +0000 Subject: [PATCH 08/14] chore(release): 1.3.3 [skip ci] ## [1.3.3](https://github.com/hirosystems/api-toolkit/compare/v1.3.2...v1.3.3) (2023-12-06) ### Bug Fixes * cast table name explicitly when checking for DB data ([#18](https://github.com/hirosystems/api-toolkit/issues/18)) ([2296ff3](https://github.com/hirosystems/api-toolkit/commit/2296ff373ca0fc8e04b4d3e37a0a59307097b10e)) --- CHANGELOG.md | 7 +++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47b03dc..e8acbd6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [1.3.3](https://github.com/hirosystems/api-toolkit/compare/v1.3.2...v1.3.3) (2023-12-06) + + +### Bug Fixes + +* cast table name explicitly when checking for DB data ([#18](https://github.com/hirosystems/api-toolkit/issues/18)) ([2296ff3](https://github.com/hirosystems/api-toolkit/commit/2296ff373ca0fc8e04b4d3e37a0a59307097b10e)) + ## [1.3.2](https://github.com/hirosystems/api-toolkit/compare/v1.3.1...v1.3.2) (2023-11-17) diff --git a/package-lock.json b/package-lock.json index ec80c89..5d9807c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@hirosystems/api-toolkit", - "version": "1.3.2", + "version": "1.3.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@hirosystems/api-toolkit", - "version": "1.3.2", + "version": "1.3.3", "license": "Apache 2.0", "dependencies": { "@fastify/cors": "^8.0.0", diff --git a/package.json b/package.json index b7df4de..494cb4c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@hirosystems/api-toolkit", - "version": "1.3.2", + "version": "1.3.3", "description": "API development toolkit", "main": "./dist/index.js", "typings": "./dist/index.d.ts", From 4730205c22e2d747c4cc24ffdcbe5bf7889f223a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20C=C3=A1rdenas?= Date: Wed, 21 Feb 2024 14:32:43 -0600 Subject: [PATCH 09/14] feat: expose graceful shutdown method (#19) --- src/shutdown-handler/index.ts | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/shutdown-handler/index.ts b/src/shutdown-handler/index.ts index d7757a9..589e8c0 100644 --- a/src/shutdown-handler/index.ts +++ b/src/shutdown-handler/index.ts @@ -15,7 +15,10 @@ const shutdownConfigs: ShutdownConfig[] = []; let isShuttingDown = false; -async function startShutdown() { +/** + * Start a graceful API shutdown. + */ +export async function shutdown(): Promise { if (isShuttingDown) { return; } @@ -67,26 +70,30 @@ function registerShutdownSignals() { SHUTDOWN_SIGNALS.forEach(sig => { process.once(sig, () => { logger.info(`Shutting down... received signal: ${sig}`); - void startShutdown(); + void shutdown(); }); }); process.once('unhandledRejection', error => { logger.error(error, 'unhandledRejection'); logger.error('Shutting down... received unhandledRejection.'); - void startShutdown(); + void shutdown(); }); process.once('uncaughtException', error => { logger.error(error, 'uncaughtException'); logger.error('Shutting down... received uncaughtException.'); - void startShutdown(); + void shutdown(); }); process.once('beforeExit', () => { logger.error('Shutting down... received beforeExit.'); - void startShutdown(); + void shutdown(); }); } -export function registerShutdownConfig(...configs: ShutdownConfig[]) { +/** + * Register shutdown handlers for different API components. + * @param configs - Array of shutdown configurations + */ +export function registerShutdownConfig(...configs: ShutdownConfig[]): void { registerShutdownSignals(); shutdownConfigs.push(...configs); } From 164184dea26d8688ee57a0434da40660b316d1e4 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 21 Feb 2024 20:34:09 +0000 Subject: [PATCH 10/14] chore(release): 1.4.0 [skip ci] ## [1.4.0](https://github.com/hirosystems/api-toolkit/compare/v1.3.3...v1.4.0) (2024-02-21) ### Features * expose graceful shutdown method ([#19](https://github.com/hirosystems/api-toolkit/issues/19)) ([4730205](https://github.com/hirosystems/api-toolkit/commit/4730205c22e2d747c4cc24ffdcbe5bf7889f223a)) --- CHANGELOG.md | 7 +++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e8acbd6..87bcef1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [1.4.0](https://github.com/hirosystems/api-toolkit/compare/v1.3.3...v1.4.0) (2024-02-21) + + +### Features + +* expose graceful shutdown method ([#19](https://github.com/hirosystems/api-toolkit/issues/19)) ([4730205](https://github.com/hirosystems/api-toolkit/commit/4730205c22e2d747c4cc24ffdcbe5bf7889f223a)) + ## [1.3.3](https://github.com/hirosystems/api-toolkit/compare/v1.3.2...v1.3.3) (2023-12-06) diff --git a/package-lock.json b/package-lock.json index 5d9807c..f652414 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@hirosystems/api-toolkit", - "version": "1.3.3", + "version": "1.4.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@hirosystems/api-toolkit", - "version": "1.3.3", + "version": "1.4.0", "license": "Apache 2.0", "dependencies": { "@fastify/cors": "^8.0.0", diff --git a/package.json b/package.json index 494cb4c..edcb695 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@hirosystems/api-toolkit", - "version": "1.3.3", + "version": "1.4.0", "description": "API development toolkit", "main": "./dist/index.js", "typings": "./dist/index.d.ts", From e99bfbb316f7d3d097bd922013f2e14fef9739f3 Mon Sep 17 00:00:00 2001 From: Matthew Little Date: Fri, 12 Apr 2024 18:51:26 +0200 Subject: [PATCH 11/14] feat: option to configure pg migration logging (#20) --- src/postgres/migrations.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/postgres/migrations.ts b/src/postgres/migrations.ts index 75d04d1..513330f 100644 --- a/src/postgres/migrations.ts +++ b/src/postgres/migrations.ts @@ -1,5 +1,5 @@ import PgMigrate from 'node-pg-migrate'; -import { MigrationDirection } from 'node-pg-migrate/dist/types'; +import { Logger as PgMigrateLogger, MigrationDirection } from 'node-pg-migrate/dist/types'; import { logger } from '../logger'; import { PgConnectionArgs, connectPostgres, standardizedConnectionArgs } from './connection'; import { isDevEnv, isTestEnv } from '../helpers/values'; @@ -11,6 +11,8 @@ export interface MigrationOptions { logMigrations?: boolean; /** Name of the table used for migrations. Defaults to `pgmigrations`. */ migrationsTable?: string; + /** Custom logging configuration */ + logger?: PgMigrateLogger; } /** @@ -50,7 +52,7 @@ export async function runMigrations( }, migrationsTable: opts?.migrationsTable ?? 'pgmigrations', schema: typeof args === 'string' ? 'public' : args.schema, - logger: { + logger: opts?.logger ?? { info: msg => (opts?.logMigrations === true ? logger.info(msg) : {}), warn: msg => logger.warn(msg), error: msg => logger.error(msg), From a40e83e4a9fb4dd9f1c0c42fd8fc921b8fd2255d Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 12 Apr 2024 16:52:56 +0000 Subject: [PATCH 12/14] chore(release): 1.5.0 [skip ci] ## [1.5.0](https://github.com/hirosystems/api-toolkit/compare/v1.4.0...v1.5.0) (2024-04-12) ### Features * option to configure pg migration logging ([#20](https://github.com/hirosystems/api-toolkit/issues/20)) ([e99bfbb](https://github.com/hirosystems/api-toolkit/commit/e99bfbb316f7d3d097bd922013f2e14fef9739f3)) --- CHANGELOG.md | 7 +++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87bcef1..11ccc00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [1.5.0](https://github.com/hirosystems/api-toolkit/compare/v1.4.0...v1.5.0) (2024-04-12) + + +### Features + +* option to configure pg migration logging ([#20](https://github.com/hirosystems/api-toolkit/issues/20)) ([e99bfbb](https://github.com/hirosystems/api-toolkit/commit/e99bfbb316f7d3d097bd922013f2e14fef9739f3)) + ## [1.4.0](https://github.com/hirosystems/api-toolkit/compare/v1.3.3...v1.4.0) (2024-02-21) diff --git a/package-lock.json b/package-lock.json index f652414..c8bb4fb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@hirosystems/api-toolkit", - "version": "1.4.0", + "version": "1.5.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@hirosystems/api-toolkit", - "version": "1.4.0", + "version": "1.5.0", "license": "Apache 2.0", "dependencies": { "@fastify/cors": "^8.0.0", diff --git a/package.json b/package.json index edcb695..a05d2ad 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@hirosystems/api-toolkit", - "version": "1.4.0", + "version": "1.5.0", "description": "API development toolkit", "main": "./dist/index.js", "typings": "./dist/index.d.ts", From 49aa9e69c595490cc88a4044245bf0a00313734a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20C=C3=A1rdenas?= Date: Fri, 28 Jun 2024 12:18:00 -0600 Subject: [PATCH 13/14] feat: add fastify cache and schema helpers (#22) --- package-lock.json | 7 +++-- package.json | 1 + src/fastify/cache.ts | 61 ++++++++++++++++++++++++++++++++++++++++++ src/fastify/index.ts | 2 ++ src/fastify/schemas.ts | 15 +++++++++++ 5 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 src/fastify/cache.ts create mode 100644 src/fastify/schemas.ts diff --git a/package-lock.json b/package-lock.json index c8bb4fb..493d960 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "@fastify/cors": "^8.0.0", "@fastify/swagger": "^8.3.1", "@fastify/type-provider-typebox": "^3.2.0", + "@sinclair/typebox": "^0.28.20", "fastify": "^4.3.0", "fastify-metrics": "^10.2.0", "node-pg-migrate": "^6.2.2", @@ -40,6 +41,9 @@ "ts-jest": "^29.0.5", "ts-node": "^10.9.1", "typescript": "^5.0.2" + }, + "engines": { + "node": ">=18" } }, "node_modules/@aashutoshrathi/word-wrap": { @@ -2178,8 +2182,7 @@ "node_modules/@sinclair/typebox": { "version": "0.28.20", "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.28.20.tgz", - "integrity": "sha512-QCF3BGfacwD+3CKhGsMeixnwOmX4AWgm61nKkNdRStyLVu0mpVFYlDSY8gVBOOED1oSwzbJauIWl/+REj8K5+w==", - "peer": true + "integrity": "sha512-QCF3BGfacwD+3CKhGsMeixnwOmX4AWgm61nKkNdRStyLVu0mpVFYlDSY8gVBOOED1oSwzbJauIWl/+REj8K5+w==" }, "node_modules/@sinonjs/commons": { "version": "3.0.0", diff --git a/package.json b/package.json index a05d2ad..c33435c 100644 --- a/package.json +++ b/package.json @@ -62,6 +62,7 @@ "@fastify/cors": "^8.0.0", "@fastify/swagger": "^8.3.1", "@fastify/type-provider-typebox": "^3.2.0", + "@sinclair/typebox": "^0.28.20", "fastify": "^4.3.0", "fastify-metrics": "^10.2.0", "node-pg-migrate": "^6.2.2", diff --git a/src/fastify/cache.ts b/src/fastify/cache.ts new file mode 100644 index 0000000..f201805 --- /dev/null +++ b/src/fastify/cache.ts @@ -0,0 +1,61 @@ +import { FastifyReply } from 'fastify'; +import { logger } from '../logger'; + +/** + * A `Cache-Control` header used for re-validation based caching. + * * `public` == allow proxies/CDNs to cache as opposed to only local browsers. + * * `no-cache` == clients can cache a resource but should revalidate each time before using it. + * * `must-revalidate` == somewhat redundant directive to assert that cache must be revalidated, required by some CDNs + */ +export const CACHE_CONTROL_MUST_REVALIDATE = 'public, no-cache, must-revalidate'; + +export async function setResponseNonCacheable(reply: FastifyReply) { + await reply.removeHeader('Cache-Control'); + await reply.removeHeader('ETag'); +} + +/** + * Parses the etag values from a raw `If-None-Match` request header value. + * The wrapping double quotes (if any) and validation prefix (if any) are stripped. + * The parsing is permissive to account for commonly non-spec-compliant clients, proxies, CDNs, etc. + * E.g. the value: + * ```js + * `"a", W/"b", c,d, "e", "f"` + * ``` + * Would be parsed and returned as: + * ```js + * ['a', 'b', 'c', 'd', 'e', 'f'] + * ``` + * @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-None-Match#syntax + * ``` + * If-None-Match: "etag_value" + * If-None-Match: "etag_value", "etag_value", ... + * If-None-Match: * + * ``` + * @param ifNoneMatchHeaderValue - raw header value + * @returns an array of etag values + */ +export function parseIfNoneMatchHeader( + ifNoneMatchHeaderValue: string | undefined +): string[] | undefined { + if (!ifNoneMatchHeaderValue) { + return undefined; + } + // Strip wrapping double quotes like `"hello"` and the ETag validation-prefix like `W/"hello"`. + // The API returns compliant, strong-validation ETags (double quoted ASCII), but can't control what + // clients, proxies, CDNs, etc may provide. + const normalized = /^(?:"|W\/")?(.*?)"?$/gi.exec(ifNoneMatchHeaderValue.trim())?.[1]; + if (!normalized) { + // This should never happen unless handling a buggy request with something like `If-None-Match: ""`, + // or if there's a flaw in the above code. Log warning for now. + logger.warn(`Normalized If-None-Match header is falsy: ${ifNoneMatchHeaderValue}`); + return undefined; + } else if (normalized.includes(',')) { + // Multiple etag values provided, likely irrelevant extra values added by a proxy/CDN. + // Split on comma, also stripping quotes, weak-validation prefixes, and extra whitespace. + return normalized.split(/(?:W\/"|")?(?:\s*),(?:\s*)(?:W\/"|")?/gi); + } else { + // Single value provided (the typical case) + return [normalized]; + } +} diff --git a/src/fastify/index.ts b/src/fastify/index.ts index 869246e..1bc83bd 100644 --- a/src/fastify/index.ts +++ b/src/fastify/index.ts @@ -1,2 +1,4 @@ +export * from './cache'; export * from './fastify'; export * from './openapi'; +export * from './schemas'; diff --git a/src/fastify/schemas.ts b/src/fastify/schemas.ts new file mode 100644 index 0000000..62400f0 --- /dev/null +++ b/src/fastify/schemas.ts @@ -0,0 +1,15 @@ +import { TSchema, Type } from '@sinclair/typebox'; + +export const Nullable = (type: T) => Type.Union([type, Type.Null()]); +export const Optional = (type: T) => Type.Optional(type); + +export const PaginatedResponse = (type: T, title: string) => + Type.Object( + { + limit: Type.Integer({ examples: [20] }), + offset: Type.Integer({ examples: [0] }), + total: Type.Integer({ examples: [1] }), + results: Type.Array(type), + }, + { title } + ); From 9e105a0063198cd610d8c073ac7a2df57c360327 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 28 Jun 2024 18:19:21 +0000 Subject: [PATCH 14/14] chore(release): 1.6.0 [skip ci] ## [1.6.0](https://github.com/hirosystems/api-toolkit/compare/v1.5.0...v1.6.0) (2024-06-28) ### Features * add fastify cache and schema helpers ([#22](https://github.com/hirosystems/api-toolkit/issues/22)) ([49aa9e6](https://github.com/hirosystems/api-toolkit/commit/49aa9e69c595490cc88a4044245bf0a00313734a)) --- CHANGELOG.md | 7 +++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 11ccc00..8f9fe2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [1.6.0](https://github.com/hirosystems/api-toolkit/compare/v1.5.0...v1.6.0) (2024-06-28) + + +### Features + +* add fastify cache and schema helpers ([#22](https://github.com/hirosystems/api-toolkit/issues/22)) ([49aa9e6](https://github.com/hirosystems/api-toolkit/commit/49aa9e69c595490cc88a4044245bf0a00313734a)) + ## [1.5.0](https://github.com/hirosystems/api-toolkit/compare/v1.4.0...v1.5.0) (2024-04-12) diff --git a/package-lock.json b/package-lock.json index 493d960..d6d13af 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@hirosystems/api-toolkit", - "version": "1.5.0", + "version": "1.6.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@hirosystems/api-toolkit", - "version": "1.5.0", + "version": "1.6.0", "license": "Apache 2.0", "dependencies": { "@fastify/cors": "^8.0.0", diff --git a/package.json b/package.json index c33435c..49ca98a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@hirosystems/api-toolkit", - "version": "1.5.0", + "version": "1.6.0", "description": "API development toolkit", "main": "./dist/index.js", "typings": "./dist/index.d.ts",