diff --git a/exports/engine/s3.js b/exports/engine/s3.js new file mode 100644 index 0000000..46b8ccc --- /dev/null +++ b/exports/engine/s3.js @@ -0,0 +1,3 @@ +import S3Engine from '../../src/engine/S3Engine.js'; + +export default S3Engine; diff --git a/package-lock.json b/package-lock.json index 5feaa26..e18e30a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,6 +23,7 @@ "eslint": "^9.6.0", "globals": "^15.8.0", "husky": "^9.1.1", + "lodash": "^4.17.21", "monocart-coverage-reports": "^2.9.2", "semantic-release": "^24.0.0", "sinon": "^18.0.0" diff --git a/package.json b/package.json index 62eb058..c6557e1 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "eslint": "^9.6.0", "globals": "^15.8.0", "husky": "^9.1.1", + "lodash": "^4.17.21", "monocart-coverage-reports": "^2.9.2", "semantic-release": "^24.0.0", "sinon": "^18.0.0" diff --git a/src/engine/Engine.js b/src/engine/Engine.js index a28d5bd..f062612 100644 --- a/src/engine/Engine.js +++ b/src/engine/Engine.js @@ -141,7 +141,7 @@ export default class Engine { } static toString() { - return 'Engine'; + return this.name; } }; diff --git a/src/engine/FileEngine.js b/src/engine/FileEngine.js index bb15e51..a572974 100644 --- a/src/engine/FileEngine.js +++ b/src/engine/FileEngine.js @@ -58,8 +58,4 @@ export default class FileEngine extends Engine { await processIndex('', Object.values(index).flat()); } - - static toString() { - return 'FileEngine'; - } } diff --git a/src/engine/FileEngine.test.js b/src/engine/FileEngine.test.js index a7223c1..8d80ec0 100644 --- a/src/engine/FileEngine.test.js +++ b/src/engine/FileEngine.test.js @@ -35,8 +35,7 @@ test('FileEngine.get(MainModel, id) when id exists', async t => { }); test('FileEngine.get(MainModel, id) when id does not exist', async t => { - const filesystem = stubFs(); - filesystem.readFile.rejects(new Error); + const filesystem = stubFs({}); await t.throwsAsync( () => FileEngine.configure({ @@ -51,7 +50,7 @@ test('FileEngine.get(MainModel, id) when id does not exist', async t => { }); test('FileEngine.put(model)', async t => { - const filesystem = stubFs(); + const filesystem = stubFs({}); const model = getTestModelInstance(valid); await t.notThrowsAsync(() => FileEngine.configure({ diff --git a/src/engine/S3Engine.js b/src/engine/S3Engine.js new file mode 100644 index 0000000..7eaa35e --- /dev/null +++ b/src/engine/S3Engine.js @@ -0,0 +1,77 @@ +import {GetObjectCommand, PutObjectCommand} from '@aws-sdk/client-s3'; + +import Engine from './Engine.js'; + +export default class S3Engine extends Engine { + static async getById(id) { + const objectPath = [this._configuration.prefix, `${id}.json`].join('/'); + + try { + const data = await this._configuration.client.send(new GetObjectCommand({ + Bucket: this._configuration.bucket, + Key: objectPath, + })); + return JSON.parse(await data.Body.transformToString()); + } catch (_error) { + return undefined; + } + } + + static async findByValue(model, parameters) { + const index = await this.getIndex(model.name); + return Object.values(index) + .filter((model) => + Object.entries(parameters) + .some(([name, value]) => model[name] === value), + ); + } + + static async putModel(model) { + const Key = [this._configuration.prefix, `${model.id}.json`].join('/'); + + await this._configuration.client.send(new PutObjectCommand({ + Key, + Body: JSON.stringify(model.toData()), + Bucket: this._configuration.bucket, + ContentType: 'application/json', + })); + } + + static async getIndex(location) { + try { + const data = await this._configuration.client.send(new GetObjectCommand({ + Key: [this._configuration.prefix].concat([location]).concat(['_index.json']).filter(e => !!e).join('/'), + Bucket: this._configuration.bucket, + })); + + return JSON.parse(await data.Body.transformToString()); + } catch (_error) { + return {}; + } + } + + static async putIndex(index) { + const processIndex = async (location, models) => { + const modelIndex = Object.fromEntries(models.map(m => [m.id, m.toIndexData()])); + const Key = [this._configuration.prefix].concat([location]).concat(['_index.json']).filter(e => !!e).join('/'); + + const currentIndex = await this.getIndex(location); + + await this._configuration.client.send(new PutObjectCommand({ + Key, + Bucket: this._configuration.bucket, + ContentType: 'application/json', + Body: JSON.stringify({ + ...currentIndex, + ...modelIndex, + }), + })); + }; + + for (const [location, models] of Object.entries(index)) { + await processIndex(location, models); + } + + await processIndex(null, Object.values(index).flat()); + } +} diff --git a/src/engine/S3Engine.test.js b/src/engine/S3Engine.test.js new file mode 100644 index 0000000..2a88a22 --- /dev/null +++ b/src/engine/S3Engine.test.js @@ -0,0 +1,404 @@ +import {GetObjectCommand, PutObjectCommand} from '@aws-sdk/client-s3'; +import {MainModel, getTestModelInstance, valid} from '../../test/fixtures/TestModel.js'; +import {NotFoundEngineError} from './Engine.js'; +import S3Engine from './S3Engine.js'; +import assertions from '../../test/assertions.js'; +import stubFs from '../../test/mocks/fs.js'; +import stubS3Client from '../../test/mocks/s3.js'; +import test from 'ava'; + +test('S3Engine.configure returns a new engine without altering the exising one', t => { + const originalStore = S3Engine; + const configuredStore = originalStore.configure({ + bucket: 'test-bucket', + prefix: 'test', + client: stubS3Client(), + }); + + t.is(originalStore._configuration, undefined); + t.like(configuredStore._configuration, { + bucket: 'test-bucket', + prefix: 'test', + }); +}); + +test('S3Engine.get(MainModel, id) when id exists', async t => { + const client = stubS3Client({ + 'test-bucket': { + 'test/MainModel/000000000000.json': getTestModelInstance(valid).toData(), + }, + }); + + const model = await S3Engine.configure({ + bucket: 'test-bucket', + prefix: 'test', + client, + }).get(MainModel, 'MainModel/000000000000'); + + assertions.calledWith(t, client.send, new GetObjectCommand({ + Bucket: 'test-bucket', + Key: 'test/MainModel/000000000000.json', + })); + t.true(model instanceof MainModel); + t.true(model.validate()); + t.like(model.toData(), { + ...valid, + stringSlug: 'string', + requiredStringSlug: 'required-string', + }); +}); + +test('S3Engine.get(MainModel, id) when id does not exist', async t => { + const client = stubS3Client(); + + await t.throwsAsync( + () => S3Engine.configure({ + bucket: 'test-bucket', + prefix: 'test', + client, + }).get(MainModel, 'MainModel/000000000000'), + { + instanceOf: NotFoundEngineError, + message: 'S3Engine.get(MainModel/000000000000) model not found', + }, + ); +}); + +test('S3Engine.put(model)', async t => { + const client = stubS3Client(); + + const model = getTestModelInstance(valid); + await t.notThrowsAsync(() => S3Engine.configure({ + bucket: 'test-bucket', + prefix: 'test', + client, + }).put(model)); + + assertions.calledWith(t, client.send, new PutObjectCommand({ + Key: 'test/MainModel/000000000000.json', + Body: JSON.stringify(model.toData()), + Bucket: 'test-bucket', + ContentType: 'application/json', + })); + + assertions.calledWith(t, client.send, new GetObjectCommand({ + Key: 'test/MainModel/_index.json', + Bucket: 'test-bucket', + })); + + assertions.calledWith(t, client.send, new PutObjectCommand({ + Key: 'test/MainModel/_index.json', + Bucket: 'test-bucket', + ContentType: 'application/json', + Body: JSON.stringify({ + 'MainModel/000000000000': { + id: 'MainModel/000000000000', + string: 'String', + stringSlug: 'string', + }, + }), + })); + + assertions.calledWith(t, client.send, new PutObjectCommand({ + Key: 'test/LinkedModel/000000000000.json', + Body: JSON.stringify(model.linked.toData()), + Bucket: 'test-bucket', + ContentType: 'application/json', + })); + + assertions.calledWith(t, client.send, new PutObjectCommand({ + Key: 'test/LinkedModel/111111111111.json', + Body: JSON.stringify(model.requiredLinked.toData()), + Bucket: 'test-bucket', + ContentType: 'application/json', + })); + + assertions.calledWith(t, client.send, new GetObjectCommand({ + Key: 'test/LinkedModel/_index.json', + Bucket: 'test-bucket', + })); + + assertions.calledWith(t, client.send, new PutObjectCommand({ + Key: 'test/LinkedModel/_index.json', + Bucket: 'test-bucket', + ContentType: 'application/json', + Body: JSON.stringify({ + 'LinkedModel/000000000000': {id: 'LinkedModel/000000000000'}, + 'LinkedModel/111111111111': {id: 'LinkedModel/111111111111'}, + }), + })); + + assertions.calledWith(t, client.send, new PutObjectCommand({ + Key: 'test/LinkedManyModel/000000000000.json', + Body: JSON.stringify(model.linkedMany[0].toData()), + Bucket: 'test-bucket', + ContentType: 'application/json', + })); + + assertions.calledWith(t, client.send, new GetObjectCommand({ + Key: 'test/LinkedManyModel/_index.json', + Bucket: 'test-bucket', + })); + + assertions.calledWith(t, client.send, new PutObjectCommand({ + Key: 'test/LinkedManyModel/_index.json', + Bucket: 'test-bucket', + ContentType: 'application/json', + Body: JSON.stringify({ + 'LinkedManyModel/000000000000': {id: 'LinkedManyModel/000000000000'}, + }), + })); + + assertions.calledWith(t, client.send, new PutObjectCommand({ + Key: 'test/CircularModel/000000000000.json', + Body: JSON.stringify(model.circular.toData()), + Bucket: 'test-bucket', + ContentType: 'application/json', + })); + + assertions.calledWith(t, client.send, new GetObjectCommand({ + Key: 'test/CircularModel/_index.json', + Bucket: 'test-bucket', + })); + + assertions.calledWith(t, client.send, new PutObjectCommand({ + Key: 'test/CircularModel/_index.json', + Bucket: 'test-bucket', + ContentType: 'application/json', + Body: JSON.stringify({ + 'CircularModel/000000000000': {id: 'CircularModel/000000000000'}, + }), + })); + + assertions.calledWith(t, client.send, new PutObjectCommand({ + Key: 'test/CircularManyModel/000000000000.json', + Body: JSON.stringify(model.circularMany[0].toData()), + Bucket: 'test-bucket', + ContentType: 'application/json', + })); + + assertions.calledWith(t, client.send, new GetObjectCommand({ + Key: 'test/CircularManyModel/_index.json', + Bucket: 'test-bucket', + })); + + assertions.calledWith(t, client.send, new PutObjectCommand({ + Key: 'test/CircularManyModel/_index.json', + Bucket: 'test-bucket', + ContentType: 'application/json', + Body: JSON.stringify({ + 'CircularManyModel/000000000000': {id: 'CircularManyModel/000000000000'}, + }), + })); + + assertions.calledWith(t, client.send, new GetObjectCommand({ + Key: 'test/_index.json', + Bucket: 'test-bucket', + })); + + assertions.calledWith(t, client.send, new PutObjectCommand({ + Key: 'test/_index.json', + Bucket: 'test-bucket', + ContentType: 'application/json', + Body: JSON.stringify({ + 'MainModel/000000000000': { + id: 'MainModel/000000000000', + string: 'String', + stringSlug: 'string', + }, + 'CircularModel/000000000000': {id: 'CircularModel/000000000000'}, + 'LinkedModel/000000000000': {id: 'LinkedModel/000000000000'}, + 'LinkedModel/111111111111': {id: 'LinkedModel/111111111111'}, + 'CircularManyModel/000000000000': {id: 'CircularManyModel/000000000000'}, + 'LinkedManyModel/000000000000': {id: 'LinkedManyModel/000000000000'}, + }), + })); +}); + +test('S3Engine.put(model) updates existing indexes', async t => { + const client = stubS3Client({ + 'test-bucket': { + 'MainModel/_index.json': { + 'MainModel/111111111111': { + id: 'MainModel/111111111111', + string: 'String', + }, + }, + }, + }); + + const model = getTestModelInstance(valid); + + await t.notThrowsAsync(() => S3Engine.configure({ + bucket: 'test-bucket', + prefix: 'test', + client, + }).put(model)); + + assertions.calledWith(t, client.send, new GetObjectCommand({ + Key: 'test/MainModel/_index.json', + Bucket: 'test-bucket', + })); + + assertions.calledWith(t, client.send, new PutObjectCommand({ + Key: 'test/MainModel/_index.json', + Bucket: 'test-bucket', + ContentType: 'application/json', + Body: JSON.stringify({ + 'MainModel/111111111111': { + id: 'MainModel/111111111111', + string: 'String', + }, + 'MainModel/000000000000': { + id: 'MainModel/000000000000', + string: 'String', + stringSlug: 'string', + }, + }), + })); + + assertions.calledWith(t, client.send, new GetObjectCommand({ + Key: 'test/LinkedModel/_index.json', + Bucket: 'test-bucket', + })); + + assertions.calledWith(t, client.send, new PutObjectCommand({ + Key: 'test/LinkedModel/_index.json', + Bucket: 'test-bucket', + ContentType: 'application/json', + Body: JSON.stringify({ + 'LinkedModel/000000000000': {id: 'LinkedModel/000000000000'}, + 'LinkedModel/111111111111': {id: 'LinkedModel/111111111111'}, + }), + })); + + assertions.calledWith(t, client.send, new GetObjectCommand({ + Key: 'test/LinkedManyModel/_index.json', + Bucket: 'test-bucket', + })); + + assertions.calledWith(t, client.send, new PutObjectCommand({ + Key: 'test/LinkedManyModel/_index.json', + Bucket: 'test-bucket', + ContentType: 'application/json', + Body: JSON.stringify({ + 'LinkedManyModel/000000000000': {id: 'LinkedManyModel/000000000000'}, + }), + })); + + assertions.calledWith(t, client.send, new GetObjectCommand({ + Key: 'test/CircularModel/_index.json', + Bucket: 'test-bucket', + })); + + assertions.calledWith(t, client.send, new PutObjectCommand({ + Key: 'test/CircularModel/_index.json', + Bucket: 'test-bucket', + ContentType: 'application/json', + Body: JSON.stringify({ + 'CircularModel/000000000000': {id: 'CircularModel/000000000000'}, + }), + })); + + assertions.calledWith(t, client.send, new GetObjectCommand({ + Key: 'test/CircularManyModel/_index.json', + Bucket: 'test-bucket', + })); + + assertions.calledWith(t, client.send, new PutObjectCommand({ + Key: 'test/CircularManyModel/_index.json', + Bucket: 'test-bucket', + ContentType: 'application/json', + Body: JSON.stringify({ + 'CircularManyModel/000000000000': {id: 'CircularManyModel/000000000000'}, + }), + })); + + assertions.calledWith(t, client.send, new GetObjectCommand({ + Key: 'test/_index.json', + Bucket: 'test-bucket', + })); + + assertions.calledWith(t, client.send, new PutObjectCommand({ + Key: 'test/_index.json', + Bucket: 'test-bucket', + ContentType: 'application/json', + Body: JSON.stringify({ + 'MainModel/000000000000': { + id: 'MainModel/000000000000', + string: 'String', + stringSlug: 'string', + }, + 'CircularModel/000000000000': {id: 'CircularModel/000000000000'}, + 'LinkedModel/000000000000': {id: 'LinkedModel/000000000000'}, + 'LinkedModel/111111111111': {id: 'LinkedModel/111111111111'}, + 'CircularManyModel/000000000000': {id: 'CircularManyModel/000000000000'}, + 'LinkedManyModel/000000000000': {id: 'LinkedManyModel/000000000000'}, + }), + })); +}); + +test('S3Engine.find(MainModel, {string: "test"}) when a matching model exists', async t => { + const client = stubS3Client({}, { + 'test-bucket': [ + getTestModelInstance(valid), + getTestModelInstance({ + id: 'MainModel/1111111111111', + string: 'another string', + }), + ], + }); + + const models = await S3Engine.configure({ + bucket: 'test-bucket', + prefix: 'test', + client, + }).find(MainModel, {string: 'String'}); + + t.like(models, [{id: 'MainModel/000000000000', string: 'String'}]); +}); + +test('S3Engine.find(MainModel, {string: "test"}) when a matching model does not exist', async t => { + const client = stubS3Client({'test-bucket': {'MainModel/_index.json': {}}}); + + const models = await S3Engine.configure({ + bucket: 'test-bucket', + prefix: 'test', + client, + }).find(MainModel, {string: 'String'}); + + assertions.calledWith(t, client.send, new GetObjectCommand({ + Key: 'test/MainModel/_index.json', + Bucket: 'test-bucket', + })); + + t.deepEqual(models, []); +}); + +test('S3Engine.find(MainModel, {string: "test"}) when no index exists', async t => { + const filesystem = stubFs(); + + const models = await S3Engine.configure({ + path: '/tmp/fileEngine', + filesystem, + }).find(MainModel, {string: 'String'}); + + t.deepEqual(models, []); +}); + +test('S3Engine.hydrate(model)', async t => { + const model = getTestModelInstance(valid); + + const dryModel = new MainModel(); + dryModel.id = 'MainModel/000000000000'; + + const client = stubS3Client({}, {'test-bucket': [model]}); + + const hydratedModel = await S3Engine.configure({ + bucket: 'test-bucket', + prefix: 'test', + client, + }).hydrate(dryModel); + + t.deepEqual(hydratedModel, model); +}); diff --git a/test/acceptance/engines/S3Engine.test.js b/test/acceptance/engines/S3Engine.test.js new file mode 100644 index 0000000..b2c2b66 --- /dev/null +++ b/test/acceptance/engines/S3Engine.test.js @@ -0,0 +1,23 @@ +import Persist from '@acodeninja/persist'; +import S3Engine from '@acodeninja/persist/engine/s3'; +import test from 'ava'; + +test('Persist allows adding the S3Engine', t => { + Persist.addEngine('files', S3Engine, { + path: '/tmp/fileEngine', + }); + + t.like(Persist._engine.files.S3Engine._configuration, { + path: '/tmp/fileEngine', + }); +}); + +test('Persist allows retrieving a S3Engine', t => { + Persist.addEngine('files', S3Engine, { + path: '/tmp/fileEngine', + }); + + t.like(Persist.getEngine('files', S3Engine)._configuration, { + path: '/tmp/fileEngine', + }); +}); diff --git a/test/assertions.js b/test/assertions.js index 4f52b81..eaa8e63 100644 --- a/test/assertions.js +++ b/test/assertions.js @@ -2,7 +2,7 @@ import {inspect} from 'node:util'; function parseArgument(arg) { try { - return JSON.parse(arg); + return JSON.parse(JSON.stringify(arg)); } catch (_) { return arg; } @@ -17,8 +17,9 @@ export function calledWith(t, spy, ...args) { const calledArguments = call.args.map(parseArgument); const expectedArguments = args.map(parseArgument); - if (calledArguments[0] === expectedArguments[0]) { + if (JSON.stringify(expectedArguments) === JSON.stringify(calledArguments)) { t.like(calledArguments, expectedArguments); + return; } } diff --git a/test/mocks/s3.js b/test/mocks/s3.js new file mode 100644 index 0000000..535b758 --- /dev/null +++ b/test/mocks/s3.js @@ -0,0 +1,76 @@ +import Model from '../../src/type/Model.js'; +import sinon from 'sinon'; + +function S3ObjectWrapper(data) { + return { + Body: { + transformToString: async () => { + return data.toString(); + }, + }, + }; +} + +function stubS3Client(filesystem = {}, models = {}) { + const modelsAddedToFilesystem = []; + + function bucketFilesFromModels(initialFilesystem = {}, ...models) { + for (const model of models) { + const modelIndexPath = model.id.replace(/[A-Z0-9]+$/, '_index.json'); + const modelIndex = initialFilesystem[modelIndexPath]; + initialFilesystem[model.id + '.json'] = model.toData(); + initialFilesystem[modelIndexPath] = { + ...modelIndex, + [model.id]: model.toIndexData(), + }; + modelsAddedToFilesystem.push(model.id); + + for (const [_, value] of Object.entries(model)) { + if (Model.isModel(value) && !modelsAddedToFilesystem.includes(value.id)) { + initialFilesystem = bucketFilesFromModels(initialFilesystem, value); + } + + if (Array.isArray(value)) { + for (const [_, subModel] of Object.entries(value)) { + if (Model.isModel(subModel) && !modelsAddedToFilesystem.includes(subModel.id)) { + initialFilesystem = bucketFilesFromModels(initialFilesystem, subModel); + } + } + } + } + } + return initialFilesystem; + } + + const resolvedBuckets = filesystem; + + for (const [bucket, modelList] of Object.entries(models)) { + resolvedBuckets[bucket] = { + ...(resolvedBuckets[bucket] || {}), + ...bucketFilesFromModels(resolvedBuckets[bucket], ...modelList), + } + } + + const send = sinon.stub().callsFake(async (command) => { + switch (command.constructor.name) { + case 'GetObjectCommand': + if (resolvedBuckets[command.input.Bucket]) { + for (const [filename, value] of Object.entries(resolvedBuckets[command.input.Bucket])) { + if (command.input.Key.endsWith(filename)) { + if (typeof value === 'string') { + return S3ObjectWrapper(Buffer.from(value)); + } + return S3ObjectWrapper(Buffer.from(JSON.stringify(value))); + } + } + } + break; + case 'PutObjectCommand': + break; + } + }); + + return {send}; +} + +export default stubS3Client;