From df064afe05a53600f74b630ee018d17a2712ddd3 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Tue, 14 Nov 2023 17:41:11 +0000 Subject: [PATCH] feat: add `netlify-blobs` driver (#337) Co-authored-by: Pooya Parsa --- .gitignore | 1 + docs/content/6.drivers/netlify-blobs.md | 55 ++++++++++++ package.json | 5 ++ pnpm-lock.yaml | 8 ++ src/drivers/netlify-blobs.ts | 114 ++++++++++++++++++++++++ src/index.ts | 4 + test/drivers/netlify-blobs.test.ts | 48 ++++++++++ test/drivers/utils.ts | 7 +- 8 files changed, 239 insertions(+), 3 deletions(-) create mode 100644 docs/content/6.drivers/netlify-blobs.md create mode 100644 src/drivers/netlify-blobs.ts create mode 100644 test/drivers/netlify-blobs.test.ts diff --git a/.gitignore b/.gitignore index fcfd145b..57313242 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ tmp /test.* __* .vercel +.netlify \ No newline at end of file diff --git a/docs/content/6.drivers/netlify-blobs.md b/docs/content/6.drivers/netlify-blobs.md new file mode 100644 index 00000000..becbdcfa --- /dev/null +++ b/docs/content/6.drivers/netlify-blobs.md @@ -0,0 +1,55 @@ +# Netlify Blobs + +Store data in a [Netlify Blobs](https://docs.netlify.com/blobs/overview/) store. This is supported in both edge and Node.js runtimes, as well at during builds. + +::alert{type="warning"} +Netlify Blobs are in beta. +:: + +```js +import { createStorage } from "unstorage"; +import netlifyBlobsDriver from "unstorage/drivers/netlify-blobs"; + +const storage = createStorage({ + driver: netlifyBlobsDriver({ + name: "blob-store-name", + }), +}); +``` + +You can create a deploy-scoped store by settings `deployScoped` option to `true`. This will mean that the deploy only has access to its own store. The store is managed alongside the deploy, with the same deploy previews, deletes, and rollbacks. + +```js +import { createStorage } from "unstorage"; +import netlifyBlobsDriver from "unstorage/drivers/netlify-blobs"; + +const storage = createStorage({ + driver: netlifyBlobsDriver({ + deployScoped: true, + }), +}); +``` + +To use, you will need to install `@netlify/blobs` as dependency or devDependency in your project: + +```json +{ + "devDependencies": { + "@netlify/blobs": "*" + } +} +``` + +**Options:** + +- `name` - The name of the store to use. It is created if needed. This is required except for deploy-scoped stores. +- `deployScoped` - If set to `true`, the store is scoped to the deploy. This means that it is only available from that deploy, and will be deleted or rolled-back alongside it. +- `siteID` - Required during builds, where it is available as `constants.SITE_ID`. At runtime this is set automatically. +- `token` - Required during builds, where it is available as `constants.NETLIFY_API_TOKEN`. At runtime this is set automatically. + +**Advanced options:** + +These are not normally needed, but are available for advanced use cases or for use in unit tests. + +- `apiURL` +- `edgeURL` diff --git a/package.json b/package.json index 734096dd..227f1187 100644 --- a/package.json +++ b/package.json @@ -64,6 +64,7 @@ "@azure/storage-blob": "^12.17.0", "@capacitor/preferences": "^5.0.6", "@cloudflare/workers-types": "^4.20231025.0", + "@netlify/blobs": "^6.2.0", "@planetscale/database": "^1.11.0", "@types/ioredis-mock": "^8.2.5", "@types/jsdom": "^21.1.5", @@ -103,6 +104,7 @@ "@azure/keyvault-secrets": "^4.7.0", "@azure/storage-blob": "^12.16.0", "@capacitor/preferences": "^5.0.6", + "@netlify/blobs": "^6.2.0", "@planetscale/database": "^1.11.0", "@upstash/redis": "^1.23.4", "@vercel/kv": "^0.2.3", @@ -130,6 +132,9 @@ "@capacitor/preferences": { "optional": true }, + "@netlify/blobs": { + "optional": true + }, "@planetscale/database": { "optional": true }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d6e61a3c..2f784acf 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -64,6 +64,9 @@ devDependencies: '@cloudflare/workers-types': specifier: ^4.20231025.0 version: 4.20231025.0 + '@netlify/blobs': + specifier: ^6.2.0 + version: 6.2.0 '@planetscale/database': specifier: ^1.11.0 version: 1.11.0 @@ -1307,6 +1310,11 @@ packages: - supports-color dev: true + /@netlify/blobs@6.2.0: + resolution: {integrity: sha512-eeOA5et1mdyZu/DlbyocDULH7HUjzrLQfznBnkta8El1Ls8hcjADs7y7H+mkx1nAW8ZR9KbBDt92QloZ6J2uWw==} + engines: {node: ^14.16.0 || >=16.0.0} + dev: true + /@nodelib/fs.scandir@2.1.5: resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} engines: {node: '>= 8'} diff --git a/src/drivers/netlify-blobs.ts b/src/drivers/netlify-blobs.ts new file mode 100644 index 00000000..dbf814b7 --- /dev/null +++ b/src/drivers/netlify-blobs.ts @@ -0,0 +1,114 @@ +import { createError, createRequiredError, defineDriver } from "./utils"; +import { getStore, getDeployStore } from "@netlify/blobs"; +import type { + Store, + BlobResponseType, + SetOptions, + ListOptions, +} from "@netlify/blobs"; +import { fetch } from "ofetch"; + +const DRIVER_NAME = "netlify-blobs"; + +type GetOptions = { type?: BlobResponseType }; + +export interface NetlifyBaseStoreOptions { + /** The name of the store to use. It is created if needed. This is required except for deploy-scoped stores. */ + name?: string; + /** If set to `true`, the store is scoped to the deploy. This means that it is only available from that deploy, and will be deleted or rolled-back alongside it. */ + deployScoped?: boolean; + /** Required during builds, where it is available as `constants.SITE_ID`. At runtime this is set automatically. */ + siteID?: string; + /** Required during builds, where it is available as `constants.NETLIFY_API_TOKEN`. At runtime this is set automatically. */ + token?: string; + /** Used for advanced use cases and unit tests */ + apiURL?: string; + /** Used for advanced use cases and unit tests */ + edgeURL?: string; +} + +export interface NetlifyDeployStoreOptions extends NetlifyBaseStoreOptions { + name?: never; + deployScoped: true; + deployID?: string; +} + +export interface NetlifyNamedStoreOptions extends NetlifyBaseStoreOptions { + name: string; + deployScoped?: false; +} + +export type NetlifyStoreOptions = + | NetlifyDeployStoreOptions + | NetlifyNamedStoreOptions; + +export default defineDriver( + ({ deployScoped, name, ...opts }: NetlifyStoreOptions) => { + let store: Store; + + const getClient = () => { + if (!store) { + if (deployScoped) { + if (name) { + throw createError( + DRIVER_NAME, + "deploy-scoped stores cannot have a name" + ); + } + store = getDeployStore({ fetch, ...opts }); + } else { + if (!name) { + throw createRequiredError(DRIVER_NAME, "name"); + } + // Ensures that reserved characters are encoded + store = getStore({ name: encodeURIComponent(name), fetch, ...opts }); + } + } + return store; + }; + + return { + name: DRIVER_NAME, + options: {}, + async hasItem(key) { + return getClient().getMetadata(key).then(Boolean); + }, + getItem: (key, tops?: GetOptions) => { + // @ts-expect-error has trouble with the overloaded types + return getClient().get(key, tops); + }, + getMeta(key) { + return getClient().getMetadata(key); + }, + getItemRaw(key, topts?: GetOptions) { + // @ts-expect-error has trouble with the overloaded types + return getClient().get(key, { type: topts?.type ?? "arrayBuffer" }); + }, + setItem(key, value, topts?: SetOptions) { + return getClient().set(key, value, topts); + }, + setItemRaw(key, value: string | ArrayBuffer | Blob, topts?: SetOptions) { + return getClient().set(key, value, topts); + }, + removeItem(key) { + return getClient().delete(key); + }, + async getKeys( + base?: string, + tops?: Omit + ) { + return (await getClient().list({ ...tops, prefix: base })).blobs.map( + (item) => item.key + ); + }, + async clear(base?: string) { + const client = getClient(); + return Promise.allSettled( + (await client.list({ prefix: base })).blobs.map((item) => + client.delete(item.key) + ) + ).then(() => {}); + }, + }; + } +); diff --git a/src/index.ts b/src/index.ts index d1eef423..26f4875a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -21,6 +21,7 @@ export const builtinDrivers = { lruCache: "unstorage/drivers/lru-cache", memory: "unstorage/drivers/memory", mongodb: "unstorage/drivers/mongodb", + netlifyBlobs: "unstorage/drivers/netlify-blobs", overlay: "unstorage/drivers/overlay", planetscale: "unstorage/drivers/planetscale", redis: "unstorage/drivers/redis", @@ -71,6 +72,9 @@ export type BuiltinDriverOptions = { lruCache: ExtractOpts<(typeof import("./drivers/lru-cache"))["default"]>; memory: ExtractOpts<(typeof import("./drivers/memory"))["default"]>; mongodb: ExtractOpts<(typeof import("./drivers/mongodb"))["default"]>; + netlifyBlobs: ExtractOpts< + (typeof import("./drivers/netlify-blobs"))["default"] + >; overlay: ExtractOpts<(typeof import("./drivers/overlay"))["default"]>; planetscale: ExtractOpts<(typeof import("./drivers/planetscale"))["default"]>; redis: ExtractOpts<(typeof import("./drivers/redis"))["default"]>; diff --git a/test/drivers/netlify-blobs.test.ts b/test/drivers/netlify-blobs.test.ts new file mode 100644 index 00000000..84007373 --- /dev/null +++ b/test/drivers/netlify-blobs.test.ts @@ -0,0 +1,48 @@ +import { afterAll, beforeAll, describe } from "vitest"; +import driver from "../../src/drivers/netlify-blobs"; +import { testDriver } from "./utils"; +import { BlobsServer } from "@netlify/blobs"; +import { resolve } from "path"; +import { rm, mkdir } from "node:fs/promises"; + +describe("drivers: netlify-blobs", async () => { + const dataDir = resolve(__dirname, "tmp/netlify-blobs"); + await rm(dataDir, { recursive: true, force: true }).catch(() => {}); + await mkdir(dataDir, { recursive: true }); + + let server: BlobsServer; + const token = "mock"; + const siteID = "1"; + beforeAll(async () => { + server = new BlobsServer({ + directory: dataDir, + debug: !true, + token, + port: 8971, + }); + await server.start(); + }); + + testDriver({ + driver: driver({ + name: "test", + edgeURL: `http://localhost:8971`, + token, + siteID, + }), + }); + + testDriver({ + driver: driver({ + deployScoped: true, + edgeURL: `http://localhost:8971`, + token, + siteID, + deployID: "test", + }), + }); + + afterAll(async () => { + await server.stop(); + }); +}); diff --git a/test/drivers/utils.ts b/test/drivers/utils.ts index bb8ea5c6..e4dafa2d 100644 --- a/test/drivers/utils.ts +++ b/test/drivers/utils.ts @@ -91,10 +91,11 @@ export function testDriver(opts: TestOptions) { const value = new Uint8Array([1, 2, 3]); await ctx.storage.setItemRaw("/data/raw.bin", value); const rValue = await ctx.storage.getItemRaw("/data/raw.bin"); - if (rValue?.length !== value.length) { - console.log(rValue); + const rValueLen = rValue?.length || rValue?.byteLength; + if (rValueLen !== value.length) { + console.log("Invalid raw value length:", rValue, "Length:", rValueLen); } - expect(rValue?.length).toBe(value.length); + expect(rValueLen).toBe(value.length); expect(Buffer.from(rValue).toString("base64")).toBe( Buffer.from(value).toString("base64") );