feat: complex index querying (#11)

* refactor: hoist find logic into Engine class * refactor: move query execution to Query class * feat(query): add $is to query input * feat(query): add $contains for string and array properties * feat: deeply nested queries * feat: index nested properties * feat: nested array index properties * feat: use [*] to index nested data in an array
acodeninja · Sep 14, 2024 · fa51c34 · fa51c34
1 parent a22f397
commit fa51c34
Show file tree

Hide file tree

Showing 14 changed files with 283 additions and 41 deletions.
diff --git a/src/Query.js b/src/Query.js
@@ -0,0 +1,62 @@
+/**
+ * persist query language features:
+ * - value match {title: 'test'} or {title: {$is: 'test'}}
+ * - contains match {list: {$contains: 'test'}} or {string: {$contains: 'es'}}
+ * - nested query {list: {$contains: {slug: 'test'}}}
+ * - deep nesting queries {list: {$contains: {string: {$contains: 'test'}}}}
+ */
+
+/**
+ * @class Query
+ */
+class Query {
+    query;
+
+    /**
+     *
+     * @param {object} query
+     */
+    constructor(query) {
+        this.query = query;
+    }
+
+    /**
+     * Using the input query, find records in an index that match
+     *
+     * @param {typeof Model} model
+     * @param {object} index
+     */
+    execute(model, index) {
+        const matchIs = (query) => !!query?.$is;
+        const matchPrimitive = (query) => ['string', 'number', 'boolean'].includes(typeof query);
+        const matchContains = (query) => !!query?.$contains;
+
+        const matchesQuery = (subject, inputQuery = this.query) => {
+            if (!subject || !inputQuery) return false;
+
+            if (matchPrimitive(inputQuery)) return subject === inputQuery;
+
+            if (matchIs(inputQuery))
+                if (subject === inputQuery.$is) return true;
+
+            if (matchContains(inputQuery)) {
+                if (subject.includes?.(inputQuery.$contains)) return true;
+
+                for (const value of subject) {
+                    if (matchesQuery(value, inputQuery.$contains)) return true;
+                }
+            }
+
+            for (const key of Object.keys(inputQuery)) {
+                if (!['$is', '$contains'].includes(key))
+                    if (matchesQuery(subject[key], inputQuery[key])) return true;
+            }
+        };
+
+        return Object.values(index)
+            .filter(m => matchesQuery(m))
+            .map(m => model.fromData(m));
+    }
+}
+
+export default Query;
diff --git a/src/Query.test.js b/src/Query.test.js
@@ -0,0 +1,130 @@
+import {MainModel} from '../test/fixtures/TestModel.js';
+import Query from './Query.js';
+import {TestIndex} from '../test/fixtures/TestIndex.js';
+import test from 'ava';
+
+test('new Query(query) stores the query', t => {
+    const query = new Query({string: 'test'});
+
+    t.deepEqual(query.query, {string: 'test'});
+});
+
+test('Query.execute(index) finds exact matches with primitive types', t => {
+    const query = new Query({string: 'test'});
+    const results = query.execute(MainModel, TestIndex);
+
+    t.deepEqual(results, [
+        MainModel.fromData({
+            id: 'MainModel/000000000000',
+            string: 'test',
+            arrayOfString: ['test'],
+            linkedMany: [{
+                id: 'LinkedManyModel/000000000000000',
+                string: 'test',
+            }],
+        }),
+    ]);
+});
+
+test('Query.execute(index) finds exact matches with $is', t => {
+    const query = new Query({string: {$is: 'test'}});
+    const results = query.execute(MainModel, TestIndex);
+
+    t.deepEqual(results, [
+        MainModel.fromData({
+            id: 'MainModel/000000000000',
+            string: 'test',
+            arrayOfString: ['test'],
+            linkedMany: [{
+                id: 'LinkedManyModel/000000000000000',
+                string: 'test',
+            }],
+        }),
+    ]);
+});
+
+test('Query.execute(index) finds matches containing for strings', t => {
+    const query = new Query({string: {$contains: 'test'}});
+    const results = query.execute(MainModel, TestIndex);
+
+    t.deepEqual(results, [
+        MainModel.fromData({
+            id: 'MainModel/000000000000',
+            string: 'test',
+            arrayOfString: ['test'],
+            linkedMany: [{
+                id: 'LinkedManyModel/000000000000000',
+                string: 'test',
+            }],
+        }),
+        MainModel.fromData({
+            id: 'MainModel/111111111111',
+            string: 'testing',
+            arrayOfString: ['testing'],
+            linkedMany: [{
+                id: 'LinkedManyModel/111111111111',
+                string: 'testing',
+            }],
+        }),
+    ]);
+});
+
+test('Query.execute(index) finds matches containing for arrays', t => {
+    const query = new Query({arrayOfString: {$contains: 'test'}});
+    const results = query.execute(MainModel, TestIndex);
+
+    t.deepEqual(results, [
+        MainModel.fromData({
+            id: 'MainModel/000000000000',
+            string: 'test',
+            arrayOfString: ['test'],
+            linkedMany: [{
+                id: 'LinkedManyModel/000000000000000',
+                string: 'test',
+            }],
+        }),
+    ]);
+});
+
+test('Query.execute(index) finds exact matches for elements in arrays', t => {
+    const query = new Query({linkedMany: {$contains: {string: 'test'}}});
+    const results = query.execute(MainModel, TestIndex);
+
+    t.deepEqual(results, [
+        MainModel.fromData({
+            id: 'MainModel/000000000000',
+            string: 'test',
+            arrayOfString: ['test'],
+            linkedMany: [{
+                id: 'LinkedManyModel/000000000000000',
+                string: 'test',
+            }],
+        }),
+    ]);
+});
+
+test('Query.execute(index) finds partial matches for elements in arrays', t => {
+    const query = new Query({linkedMany: {$contains: {string: {$contains: 'test'}}}});
+    const results = query.execute(MainModel, TestIndex);
+
+    t.deepEqual(results, [
+        MainModel.fromData({
+            id: 'MainModel/000000000000',
+            string: 'test',
+            arrayOfString: ['test'],
+            linkedMany: [{
+                id: 'LinkedManyModel/000000000000000',
+                string: 'test',
+            }],
+        }),
+        MainModel.fromData({
+            id: 'MainModel/111111111111',
+            string: 'testing',
+            arrayOfString: ['testing'],
+            linkedMany: [{
+                id: 'LinkedManyModel/111111111111',
+                string: 'testing',
+            }],
+        }),
+    ]);
+});
diff --git a/src/engine/Engine.js b/src/engine/Engine.js
@@ -1,3 +1,4 @@
+import Query from '../Query.js';
 import Type from '../type/index.js';
 import lunr from 'lunr';
 
@@ -15,6 +16,10 @@ export default class Engine {
         throw new NotImplementedError(`${this.name} must implement .putModel()`);
     }
 
+    static async getIndex(_model) {
+        throw new NotImplementedError(`${this.name} does not implement .getIndex()`);
+    }
+
     static async putIndex(_index) {
         throw new NotImplementedError(`${this.name} does not implement .putIndex()`);
     }
@@ -35,10 +40,6 @@ export default class Engine {
         throw new NotImplementedError(`${this.name} does not implement .putSearchIndexRaw()`);
     }
 
-    static async findByValue(_model, _parameters) {
-        throw new NotImplementedError(`${this.name} does not implement .findByValue()`);
-    }
-
     static async search(model, query) {
         this.checkConfiguration();
 
@@ -61,10 +62,11 @@ export default class Engine {
         return output;
     }
 
-    static async find(model, parameters) {
+    static async find(model, query) {
         this.checkConfiguration();
-        const response = await this.findByValue(model, parameters);
-        return response.map(m => model.fromData(m));
+        const index = await this.getIndex(model);
+
+        return new Query(query).execute(model, index);
     }
 
     static async put(model) {

diff --git a/src/engine/Engine.test.js b/src/engine/Engine.test.js
@@ -39,12 +39,12 @@ test('UnimplementedEngine.putIndex(model) raises a putIndex not implemented erro
     t.is(error.message, 'UnimplementedEngine does not implement .putIndex()');
 });
 
-test('UnimplementedEngine.find(Model, {param: value}) raises a findByValue not implemented error', async t => {
+test('UnimplementedEngine.find(Model, {param: value}) raises a getIndex not implemented error', async t => {
     const error = await t.throwsAsync(() =>
             UnimplementedEngine.find(Type.Model, {param: 'value'}),
         {instanceOf: NotImplementedError},
     );
-    t.is(error.message, 'UnimplementedEngine does not implement .findByValue()');
+    t.is(error.message, 'UnimplementedEngine does not implement .getIndex()');
 });
 
 test('UnimplementedEngine.getSearchIndexCompiled(Model, {param: value}) raises a getSearchIndexCompiled not implemented error', async t => {

diff --git a/src/engine/FileEngine.js b/src/engine/FileEngine.js
@@ -31,14 +31,8 @@ export default class FileEngine extends Engine {
         return JSON.parse(await this._configuration.filesystem.readFile(filePath).then(f => f.toString()));
     }
 
-    static async findByValue(model, parameters) {
-        const index = JSON.parse((await this._configuration.filesystem.readFile(join(this._configuration.path, model.name, '_index.json')).catch(() => '{}')).toString());
-
-        return Object.values(index)
-            .filter((model) =>
-                Object.entries(parameters)
-                    .some(([name, value]) => model[name] === value),
-            );
+    static async getIndex(model) {
+        return JSON.parse((await this._configuration.filesystem.readFile(join(this._configuration.path, model.name, '_index.json')).catch(() => '{}')).toString());
     }
 
     static async putModel(model) {

diff --git a/src/engine/FileEngine.test.js b/src/engine/FileEngine.test.js
@@ -75,6 +75,8 @@ test('FileEngine.put(model)', async t => {
             id: 'MainModel/000000000000',
             string: 'String',
             stringSlug: 'string',
+            linked: {string: 'test'},
+            linkedMany: [{string: 'many'}],
         },
     }));
 
@@ -120,6 +122,8 @@ test('FileEngine.put(model)', async t => {
             id: 'MainModel/000000000000',
             string: 'String',
             stringSlug: 'string',
+            linked: {string: 'test'},
+            linkedMany: [{string: 'many'}],
         },
         'CircularModel/000000000000': {id: 'CircularModel/000000000000'},
         'LinkedModel/000000000000': {id: 'LinkedModel/000000000000'},
@@ -194,6 +198,8 @@ test('FileEngine.put(model) updates existing indexes', async t => {
             id: 'MainModel/000000000000',
             string: 'String',
             stringSlug: 'string',
+            linked: {string: 'test'},
+            linkedMany: [{string: 'many'}],
         },
     }));
 
@@ -219,6 +225,8 @@ test('FileEngine.put(model) updates existing indexes', async t => {
             id: 'MainModel/000000000000',
             string: 'String',
             stringSlug: 'string',
+            linked: {string: 'test'},
+            linkedMany: [{string: 'many'}],
         },
         'CircularModel/000000000000': {id: 'CircularModel/000000000000'},
         'LinkedModel/000000000000': {id: 'LinkedModel/000000000000'},
@@ -266,6 +274,8 @@ test('FileEngine.put(model) when putting an index fails', async t => {
                 id: 'MainModel/000000000000',
                 string: 'String',
                 stringSlug: 'string',
+                linked: {string: 'test'},
+                linkedMany: [{string: 'many'}],
             },
         },
     ));

diff --git a/src/engine/HTTPEngine.js b/src/engine/HTTPEngine.js
@@ -75,15 +75,6 @@ export default class HTTPEngine extends Engine {
         return await this._processFetch(url, this._getReadOptions());
     }
 
-    static async findByValue(model, parameters) {
-        const index = await this.getIndex(model.name);
-        return Object.values(index)
-            .filter((model) =>
-                Object.entries(parameters)
-                    .some(([name, value]) => model[name] === value),
-            );
-    }
-
     static async putModel(model) {
         const url = new URL([
             this._configuration.host,

diff --git a/src/engine/HTTPEngine.test.js b/src/engine/HTTPEngine.test.js
@@ -133,6 +133,8 @@ test('HTTPEngine.put(model)', async t => {
                 id: 'MainModel/000000000000',
                 string: 'String',
                 stringSlug: 'string',
+                linked: {string: 'test'},
+                linkedMany: [{string: 'many'}],
             },
         }),
     });
@@ -279,6 +281,8 @@ test('HTTPEngine.put(model)', async t => {
                 id: 'MainModel/000000000000',
                 string: 'String',
                 stringSlug: 'string',
+                linked: {string: 'test'},
+                linkedMany: [{string: 'many'}],
             },
             'CircularModel/000000000000': {id: 'CircularModel/000000000000'},
             'LinkedModel/000000000000': {id: 'LinkedModel/000000000000'},
@@ -399,6 +403,8 @@ test('HTTPEngine.put(model) when the engine fails to put a raw search index', as
                 id: 'MainModel/000000000000',
                 string: 'String',
                 stringSlug: 'string',
+                linked: {string: 'test'},
+                linkedMany: [{string: 'many'}],
             },
         }),
     });
@@ -454,6 +460,8 @@ test('HTTPEngine.put(model) when putting an index fails', async t => {
                 id: 'MainModel/000000000000',
                 string: 'String',
                 stringSlug: 'string',
+                linked: {string: 'test'},
+                linkedMany: [{string: 'many'}],
             },
         }),
     });
@@ -687,6 +695,8 @@ test('HTTPEngine.put(model) updates existing indexes', async t => {
                 id: 'MainModel/000000000000',
                 string: 'String',
                 stringSlug: 'string',
+                linked: {string: 'test'},
+                linkedMany: [{string: 'many'}],
             },
         }),
     });
@@ -757,6 +767,8 @@ test('HTTPEngine.put(model) updates existing indexes', async t => {
                 id: 'MainModel/000000000000',
                 string: 'String',
                 stringSlug: 'string',
+                linked: {string: 'test'},
+                linkedMany: [{string: 'many'}],
             },
             'CircularModel/000000000000': {id: 'CircularModel/000000000000'},
             'LinkedModel/000000000000': {id: 'LinkedModel/000000000000'},

diff --git a/src/engine/S3Engine.js b/src/engine/S3Engine.js
@@ -24,16 +24,6 @@ export default class S3Engine extends Engine {
         return JSON.parse(await data.Body.transformToString());
     }
 
-    static async findByValue(model, parameters) {
-        const index = await this.getIndex(model.name);
-
-        return Object.values(index)
-            .filter((model) =>
-                Object.entries(parameters)
-                    .some(([name, value]) => model[name] === value),
-            );
-    }
-
     static async putModel(model) {
         const Key = [this._configuration.prefix, `${model.id}.json`].join('/');