Merge pull request #21 from hirosystems/master

merge master into develop

CHANGELOG.md

@@ -1,3 +1,52 @@
+## [1.6.0](https://github.com/hirosystems/api-toolkit/compare/v1.5.0...v1.6.0) (2024-06-28)
+
+
+### Features
+
+* add fastify cache and schema helpers ([#22](https://github.com/hirosystems/api-toolkit/issues/22)) ([49aa9e6](https://github.com/hirosystems/api-toolkit/commit/49aa9e69c595490cc88a4044245bf0a00313734a))
+
+## [1.5.0](https://github.com/hirosystems/api-toolkit/compare/v1.4.0...v1.5.0) (2024-04-12)
+
+
+### Features
+
+* option to configure pg migration logging ([#20](https://github.com/hirosystems/api-toolkit/issues/20)) ([e99bfbb](https://github.com/hirosystems/api-toolkit/commit/e99bfbb316f7d3d097bd922013f2e14fef9739f3))
+
+## [1.4.0](https://github.com/hirosystems/api-toolkit/compare/v1.3.3...v1.4.0) (2024-02-21)
+
+
+### Features
+
+* expose graceful shutdown method ([#19](https://github.com/hirosystems/api-toolkit/issues/19)) ([4730205](https://github.com/hirosystems/api-toolkit/commit/4730205c22e2d747c4cc24ffdcbe5bf7889f223a))
+
+## [1.3.3](https://github.com/hirosystems/api-toolkit/compare/v1.3.2...v1.3.3) (2023-12-06)
+
+
+### Bug Fixes
+
+* cast table name explicitly when checking for DB data ([#18](https://github.com/hirosystems/api-toolkit/issues/18)) ([2296ff3](https://github.com/hirosystems/api-toolkit/commit/2296ff373ca0fc8e04b4d3e37a0a59307097b10e))
+
+## [1.3.2](https://github.com/hirosystems/api-toolkit/compare/v1.3.1...v1.3.2) (2023-11-17)
+
+
+### Bug Fixes
+
+* allow schema to be specified for migrations table ([#17](https://github.com/hirosystems/api-toolkit/issues/17)) ([aa9c0dd](https://github.com/hirosystems/api-toolkit/commit/aa9c0dd91acfbbf57ba671df0c0ecaacd0a8ea5e))
+
+## [1.3.1](https://github.com/hirosystems/api-toolkit/compare/v1.3.0...v1.3.1) (2023-11-16)
+
+
+### Bug Fixes
+
+* allow logging migrations ([9c56c19](https://github.com/hirosystems/api-toolkit/commit/9c56c19377e9b1f85d0f41eb6992083cabf8e9b2))
+
+## [1.3.0](https://github.com/hirosystems/api-toolkit/compare/v1.2.2...v1.3.0) (2023-11-01)
+
+
+### Features
+
+* add iterator and enum value handling helpers ([#16](https://github.com/hirosystems/api-toolkit/issues/16)) ([cb263b8](https://github.com/hirosystems/api-toolkit/commit/cb263b801270147257243a7fa079f0c84115bc8d))
+
 ## [1.2.2](https://github.com/hirosystems/api-toolkit/compare/v1.2.1...v1.2.2) (2023-10-27)
 
 

package.json

@@ -1,12 +1,12 @@
 {
   "name": "@hirosystems/api-toolkit",
-  "version": "1.2.2",
+  "version": "1.6.0",
   "description": "API development toolkit",
   "main": "./dist/index.js",
   "typings": "./dist/index.d.ts",
   "scripts": {
     "build": "rimraf ./dist && tsc --project tsconfig.build.json && copyfiles -u 1 ./src/server-version/*.mjs ./dist",
-    "test": "jest",
+    "test": "jest --runInBand",
     "lint:eslint": "eslint . --ext .js,.jsx,.ts,.tsx -f unix",
     "lint:prettier": "prettier --check src/**/*.ts",
     "testenv:run": "docker-compose -f docker/docker-compose.dev.postgres.yml up",
@@ -62,6 +62,7 @@
     "@fastify/cors": "^8.0.0",
     "@fastify/swagger": "^8.3.1",
     "@fastify/type-provider-typebox": "^3.2.0",
+    "@sinclair/typebox": "^0.28.20",
     "fastify": "^4.3.0",
     "fastify-metrics": "^10.2.0",
     "node-pg-migrate": "^6.2.2",

src/fastify/cache.ts

@@ -0,0 +1,61 @@
+import { FastifyReply } from 'fastify';
+import { logger } from '../logger';
+
+/**
+ * A `Cache-Control` header used for re-validation based caching.
+ * * `public` == allow proxies/CDNs to cache as opposed to only local browsers.
+ * * `no-cache` == clients can cache a resource but should revalidate each time before using it.
+ * * `must-revalidate` == somewhat redundant directive to assert that cache must be revalidated, required by some CDNs
+ */
+export const CACHE_CONTROL_MUST_REVALIDATE = 'public, no-cache, must-revalidate';
+
+export async function setResponseNonCacheable(reply: FastifyReply) {
+  await reply.removeHeader('Cache-Control');
+  await reply.removeHeader('ETag');
+}
+
+/**
+ * Parses the etag values from a raw `If-None-Match` request header value.
+ * The wrapping double quotes (if any) and validation prefix (if any) are stripped.
+ * The parsing is permissive to account for commonly non-spec-compliant clients, proxies, CDNs, etc.
+ * E.g. the value:
+ * ```js
+ * `"a", W/"b", c,d,   "e", "f"`
+ * ```
+ * Would be parsed and returned as:
+ * ```js
+ * ['a', 'b', 'c', 'd', 'e', 'f']
+ * ```
+ * @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-None-Match#syntax
+ * ```
+ * If-None-Match: "etag_value"
+ * If-None-Match: "etag_value", "etag_value", ...
+ * If-None-Match: *
+ * ```
+ * @param ifNoneMatchHeaderValue - raw header value
+ * @returns an array of etag values
+ */
+export function parseIfNoneMatchHeader(
+  ifNoneMatchHeaderValue: string | undefined
+): string[] | undefined {
+  if (!ifNoneMatchHeaderValue) {
+    return undefined;
+  }
+  // Strip wrapping double quotes like `"hello"` and the ETag validation-prefix like `W/"hello"`.
+  // The API returns compliant, strong-validation ETags (double quoted ASCII), but can't control what
+  // clients, proxies, CDNs, etc may provide.
+  const normalized = /^(?:"|W\/")?(.*?)"?$/gi.exec(ifNoneMatchHeaderValue.trim())?.[1];
+  if (!normalized) {
+    // This should never happen unless handling a buggy request with something like `If-None-Match: ""`,
+    // or if there's a flaw in the above code. Log warning for now.
+    logger.warn(`Normalized If-None-Match header is falsy: ${ifNoneMatchHeaderValue}`);
+    return undefined;
+  } else if (normalized.includes(',')) {
+    // Multiple etag values provided, likely irrelevant extra values added by a proxy/CDN.
+    // Split on comma, also stripping quotes, weak-validation prefixes, and extra whitespace.
+    return normalized.split(/(?:W\/"|")?(?:\s*),(?:\s*)(?:W\/"|")?/gi);
+  } else {
+    // Single value provided (the typical case)
+    return [normalized];
+  }
+}

src/fastify/index.ts

@@ -1,2 +1,4 @@
+export * from './cache';
 export * from './fastify';
 export * from './openapi';
+export * from './schemas';

src/fastify/schemas.ts

@@ -0,0 +1,15 @@
+import { TSchema, Type } from '@sinclair/typebox';
+
+export const Nullable = <T extends TSchema>(type: T) => Type.Union([type, Type.Null()]);
+export const Optional = <T extends TSchema>(type: T) => Type.Optional(type);
+
+export const PaginatedResponse = <T extends TSchema>(type: T, title: string) =>
+  Type.Object(
+    {
+      limit: Type.Integer({ examples: [20] }),
+      offset: Type.Integer({ examples: [0] }),
+      total: Type.Integer({ examples: [1] }),
+      results: Type.Array(type),
+    },
+    { title }
+  );

src/helpers/index.ts

@@ -1,2 +1,3 @@
+export * from './iterators';
 export * from './time';
 export * from './values';

src/helpers/iterators.ts

@@ -0,0 +1,78 @@
+import { logger } from '../logger';
+import { isDevEnv } from './values';
+
+/**
+ * Iterate over an array, yielding multiple items at a time. If the size of the given array
+ * is not divisible by the given batch size, then the length of the last items returned will
+ * be smaller than the given batch size, i.e.:
+ * ```typescript
+ * items.length % batchSize
+ * ```
+ * @param items - The array to iterate over.
+ * @param batchSize - Maximum number of items to return at a time.
+ * @param printBenchmark - If we should print benchmark of items per second
+ */
+export function* batchIterate<T>(
+  items: T[],
+  batchSize: number,
+  printBenchmark = isDevEnv
+): Generator<T[]> {
+  if (items.length === 0) return;
+  const startTime = Date.now();
+  for (let i = 0; i < items.length; ) {
+    const itemsRemaining = items.length - i;
+    const sliceSize = Math.min(batchSize, itemsRemaining);
+    yield items.slice(i, i + sliceSize);
+    i += sliceSize;
+  }
+  if (printBenchmark) {
+    const itemsPerSecond = Math.round((items.length / (Date.now() - startTime)) * 1000);
+    const caller = new Error().stack?.split('at ')[3].trim();
+    logger.debug(`Iterated ${itemsPerSecond} items/second at ${caller}`);
+  }
+}
+
+/**
+ * Iterate over an `AsyncIterable`, yielding multiple items at a time. If the size of the given
+ * array is not divisible by the given batch size, then the length of the last items returned will
+ * be smaller than the given batch size.
+ *
+ * @param items - AsyncIterable
+ * @param batchSize - Batch size
+ * @param printBenchmark - If we should print benchmark of items per second
+ */
+export async function* asyncBatchIterate<T>(
+  items: AsyncIterable<T>,
+  batchSize: number,
+  printBenchmark = isDevEnv
+): AsyncGenerator<T[], void, unknown> {
+  const startTime = Date.now();
+  let itemCount = 0;
+  let itemBatch: T[] = [];
+  for await (const item of items) {
+    itemBatch.push(item);
+    itemCount++;
+    if (itemBatch.length >= batchSize) {
+      yield itemBatch;
+      itemBatch = [];
+      if (printBenchmark) {
+        const itemsPerSecond = Math.round((itemCount / (Date.now() - startTime)) * 1000);
+        const caller = new Error().stack?.split('at ')[3].trim();
+        logger.debug(`Iterated ${itemsPerSecond} items/second at ${caller}`);
+      }
+    }
+  }
+  if (itemBatch.length > 0) {
+    yield itemBatch;
+  }
+}
+
+/**
+ * Convert an `AsyncIterable` to a generator
+ * @param iter - AsyncIterable
+ */
+export async function* asyncIterableToGenerator<T>(iter: AsyncIterable<T>) {
+  for await (const entry of iter) {
+    yield entry;
+  }
+}

src/helpers/values.ts

@@ -130,3 +130,15 @@ export function numberToHex(number: number, paddingBytes: number = 4): string {
  * @returns Boolean
  */
 export const has0xPrefix = (val: string) => val.substring(0, 2).toLowerCase() === '0x';
+
+/**
+ * Converts a string to an enum value.
+ * @param enumType - The enum type
+ * @param value - The string value to convert
+ * @returns Enum item or undefined
+ */
+export function toEnumValue<T>(enm: { [s: string]: T }, value: string): T | undefined {
+  return (Object.values(enm) as unknown as string[]).includes(value)
+    ? (value as unknown as T)
+    : undefined;
+}