Skip to content

Commit 7976276

Browse files
acodeninjaacodeninja
authored
feat: search (#3)
* feat(search): add search data method to model * feat(search): implement searching with File and S3 engines * docs: document the search and index engine methods
1 parent 5295b22 commit 7976276

15 files changed

+582
-16
lines changed

‎README.md

+27
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,34 @@ export class ModelB extends Persist.Type.Model {
122122
```
123123
</details>
124124

125+
## Find and Search
125126

127+
Models may expose a `searchProperties()` and `indexProperties()` static method to indicate which
128+
fields should be indexed for storage engine `find()` and `search()` methods.
129+
130+
Use `find()` for a low usage exact string match on any indexed attribute of a model.
131+
132+
Use `search()` for a medium usage fuzzy string match on any search indexed attribute of a model.
133+
134+
```javascript
135+
import Persist from "@acodeninja/persist";
136+
import FileEngine from "@acodeninja/persist/engine/file";
137+
138+
export class Tag extends Persist.Type.Model {
139+
static tag = Persist.Type.String.required;
140+
static description = Persist.Type.String;
141+
static searchProperties = () => ['tag', 'description'];
142+
static indexProperties = () => ['tag'];
143+
}
144+
145+
const tag = new Tag({tag: 'documentation', description: 'How to use the persist library'});
146+
147+
FileEngine.find(Tag, {tag: 'documentation'});
148+
// [Tag {tag: 'documentation', description: 'How to use the persist library'}]
149+
150+
FileEngine.search(Tag, 'how to');
151+
// [Tag {tag: 'documentation', description: 'How to use the persist library'}]
152+
```
126153

127154
## Storage
128155

‎package-lock.json

+6
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json

+1
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
"dependencies": {
1919
"ajv": "^8.16.0",
2020
"ajv-errors": "^3.0.0",
21+
"lunr": "^2.3.9",
2122
"slugify": "^1.6.6",
2223
"ulid": "^2.3.0"
2324
},

‎src/SchemaCompiler.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ export default class SchemaCompiler {
2929
}
3030

3131
for (const [name, type] of Object.entries(rawSchema)) {
32-
if (['indexedProperties'].includes(name)) continue;
32+
if (['indexedProperties', 'searchProperties'].includes(name)) continue;
3333

3434
const property = type instanceof Function && !type.prototype ? type() : type;
3535

‎src/engine/Engine.js

+60
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import Type from '../type/index.js';
2+
import lunr from 'lunr';
23

34
/**
45
* @class Engine
@@ -18,10 +19,47 @@ export default class Engine {
1819
throw new NotImplementedError(`${this.name} does not implement .putIndex()`);
1920
}
2021

22+
static async getSearchIndexCompiled(_model) {
23+
throw new NotImplementedError(`${this.name} does not implement .getSearchIndexCompiled()`);
24+
}
25+
26+
static async getSearchIndexRaw(_model) {
27+
throw new NotImplementedError(`${this.name} does not implement .getSearchIndexRaw()`);
28+
}
29+
30+
static async putSearchIndexCompiled(_model, _compiledIndex) {
31+
throw new NotImplementedError(`${this.name} does not implement .putSearchIndexCompiled()`);
32+
}
33+
34+
static async putSearchIndexRaw(_model, _rawIndex) {
35+
throw new NotImplementedError(`${this.name} does not implement .putSearchIndexRaw()`);
36+
}
37+
2138
static async findByValue(_model, _parameters) {
2239
throw new NotImplementedError(`${this.name} does not implement .findByValue()`);
2340
}
2441

42+
static async search(model, query) {
43+
const index = await this.getSearchIndexCompiled(model);
44+
45+
try {
46+
const searchIndex = lunr.Index.load(index);
47+
48+
const results = searchIndex.search(query);
49+
const output = [];
50+
for (const result of results) {
51+
output.push({
52+
...result,
53+
model: await this.get(model, result.ref),
54+
});
55+
}
56+
57+
return output;
58+
} catch (_) {
59+
throw new NotImplementedError(`The model ${model.name} does not have a search index available.`);
60+
}
61+
}
62+
2563
static async find(model, parameters) {
2664
const response = await this.findByValue(model, parameters);
2765

@@ -41,6 +79,28 @@ export default class Engine {
4179
await this.putModel(model);
4280
indexUpdates[model.constructor.name] = (indexUpdates[model.constructor.name] ?? []).concat([model]);
4381

82+
if (model.constructor.searchProperties().length > 0) {
83+
const rawSearchIndex = {
84+
...await this.getSearchIndexRaw(model.constructor),
85+
[model.id]: model.toSearchData(),
86+
};
87+
await this.putSearchIndexRaw(model.constructor, rawSearchIndex);
88+
89+
const compiledIndex = lunr(function () {
90+
this.ref('id')
91+
92+
for (const field of model.constructor.searchProperties()) {
93+
this.field(field);
94+
}
95+
96+
Object.values(rawSearchIndex).forEach(function (doc) {
97+
this.add(doc);
98+
}, this)
99+
});
100+
101+
await this.putSearchIndexCompiled(model.constructor, compiledIndex);
102+
}
103+
44104
for (const [_, property] of Object.entries(model)) {
45105
if (Type.Model.isModel(property)) {
46106
await processModel(property);

‎src/engine/Engine.test.js

+32-4
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import Engine, {NotFoundEngineError, NotImplementedError} from './Engine.js';
22
import {MainModel} from '../../test/fixtures/TestModel.js';
33
import Type from '../type/index.js';
4-
import stubFs from '../../test/mocks/fs.js';
54
import test from 'ava';
65

76
class UnimplementedEngine extends Engine {
@@ -48,16 +47,45 @@ test('UnimplementedEngine.find(Model, {param: value}) raises a findByValue not i
4847
t.is(error.message, 'UnimplementedEngine does not implement .findByValue()');
4948
});
5049

50+
test('UnimplementedEngine.getSearchIndexCompiled(Model, {param: value}) raises a getSearchIndexCompiled not implemented error', async t => {
51+
const error = await t.throwsAsync(() =>
52+
UnimplementedEngine.getSearchIndexCompiled(Type.Model, {param: 'value'}),
53+
{instanceOf: NotImplementedError},
54+
);
55+
t.is(error.message, 'UnimplementedEngine does not implement .getSearchIndexCompiled()');
56+
});
57+
58+
test('UnimplementedEngine.getSearchIndexRaw(Model, {param: value}) raises a getSearchIndexRaw not implemented error', async t => {
59+
const error = await t.throwsAsync(() =>
60+
UnimplementedEngine.getSearchIndexRaw(Type.Model, {param: 'value'}),
61+
{instanceOf: NotImplementedError},
62+
);
63+
t.is(error.message, 'UnimplementedEngine does not implement .getSearchIndexRaw()');
64+
});
65+
66+
test('UnimplementedEngine.putSearchIndexCompiled(Model, {param: value}) raises a putSearchIndexCompiled not implemented error', async t => {
67+
const error = await t.throwsAsync(() =>
68+
UnimplementedEngine.putSearchIndexCompiled(Type.Model, {param: 'value'}),
69+
{instanceOf: NotImplementedError},
70+
);
71+
t.is(error.message, 'UnimplementedEngine does not implement .putSearchIndexCompiled()');
72+
});
73+
74+
test('UnimplementedEngine.putSearchIndexRaw(Model, {param: value}) raises a putSearchIndexRaw not implemented error', async t => {
75+
const error = await t.throwsAsync(() =>
76+
UnimplementedEngine.putSearchIndexRaw(Type.Model, {param: 'value'}),
77+
{instanceOf: NotImplementedError},
78+
);
79+
t.is(error.message, 'UnimplementedEngine does not implement .putSearchIndexRaw()');
80+
});
81+
5182
class ImplementedEngine extends Engine {
5283
static getById(_model, _id) {
5384
return null;
5485
}
5586
}
5687

5788
test('ImplementedEngine.get(MainModel, id) when id does not exist', async t => {
58-
const filesystem = stubFs();
59-
filesystem.readFile.rejects(new Error);
60-
6189
await t.throwsAsync(
6290
() => ImplementedEngine.get(MainModel, 'MainModel/000000000000'),
6391
{

‎src/engine/FileEngine.js

+18
Original file line numberDiff line numberDiff line change
@@ -58,4 +58,22 @@ export default class FileEngine extends Engine {
5858

5959
await processIndex('', Object.values(index).flat());
6060
}
61+
62+
static async getSearchIndexCompiled(model) {
63+
return JSON.parse((await this._configuration.filesystem.readFile(join(this._configuration.path, model.name, '_search_index.json')).catch(() => '{}')).toString());
64+
}
65+
66+
static async getSearchIndexRaw(model) {
67+
return JSON.parse((await this._configuration.filesystem.readFile(join(this._configuration.path, model.name, '_search_index_raw.json')).catch(() => '{}')).toString());
68+
}
69+
70+
static async putSearchIndexCompiled(model, compiledIndex) {
71+
const filePath = join(this._configuration.path, model.name, '_search_index.json');
72+
await this._configuration.filesystem.writeFile(filePath, JSON.stringify(compiledIndex));
73+
}
74+
75+
static async putSearchIndexRaw(model, rawIndex) {
76+
const filePath = join(this._configuration.path, model.name, '_search_index_raw.json');
77+
await this._configuration.filesystem.writeFile(filePath, JSON.stringify(rawIndex));
78+
}
6179
}

‎src/engine/FileEngine.test.js

+120-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import {MainModel, getTestModelInstance, valid} from '../../test/fixtures/TestModel.js';
2+
import {NotFoundEngineError, NotImplementedError} from './Engine.js';
23
import FileEngine from './FileEngine.js';
3-
import {NotFoundEngineError} from './Engine.js';
44
import assertions from '../../test/assertions.js';
55
import fs from 'node:fs/promises';
66
import stubFs from '../../test/mocks/fs.js';
@@ -67,6 +67,21 @@ test('FileEngine.put(model)', async t => {
6767
},
6868
}));
6969

70+
assertions.calledWith(t, filesystem.readFile, '/tmp/fileEngine/MainModel/_search_index_raw.json');
71+
assertions.calledWith(t, filesystem.writeFile, '/tmp/fileEngine/MainModel/_search_index_raw.json', JSON.stringify({
72+
'MainModel/000000000000': {
73+
id: 'MainModel/000000000000',
74+
string: 'String',
75+
},
76+
}));
77+
assertions.calledWith(t, filesystem.writeFile, '/tmp/fileEngine/MainModel/_search_index.json', JSON.stringify({
78+
version: '2.3.9',
79+
fields: ['string'],
80+
fieldVectors: [['string/MainModel/000000000000', [0, 0.288]]],
81+
invertedIndex: [['string', {_index: 0, string: {'MainModel/000000000000': {}}}]],
82+
pipeline: ['stemmer'],
83+
}));
84+
7085
assertions.calledWith(t, filesystem.writeFile, '/tmp/fileEngine/LinkedModel/000000000000.json', JSON.stringify(model.linked.toData()));
7186
assertions.calledWith(t, filesystem.writeFile, '/tmp/fileEngine/LinkedModel/111111111111.json', JSON.stringify(model.requiredLinked.toData()));
7287
assertions.calledWith(t, filesystem.writeFile, '/tmp/fileEngine/LinkedModel/_index.json', JSON.stringify({
@@ -103,6 +118,45 @@ test('FileEngine.put(model)', async t => {
103118
}));
104119
});
105120

121+
test('FileEngine.put(model) updates existing search indexes', async t => {
122+
const filesystem = stubFs({
123+
'MainModel/_search_index_raw.json': {
124+
'MainModel/111111111111': {
125+
id: 'MainModel/111111111111',
126+
string: 'String',
127+
},
128+
},
129+
});
130+
131+
const model = getTestModelInstance(valid);
132+
await t.notThrowsAsync(() => FileEngine.configure({
133+
path: '/tmp/fileEngine',
134+
filesystem,
135+
}).put(model));
136+
137+
assertions.calledWith(t, filesystem.readFile, '/tmp/fileEngine/MainModel/_search_index_raw.json');
138+
assertions.calledWith(t, filesystem.writeFile, '/tmp/fileEngine/MainModel/_search_index_raw.json', JSON.stringify({
139+
'MainModel/111111111111': {
140+
id: 'MainModel/111111111111',
141+
string: 'String',
142+
},
143+
'MainModel/000000000000': {
144+
id: 'MainModel/000000000000',
145+
string: 'String',
146+
},
147+
}));
148+
assertions.calledWith(t, filesystem.writeFile, '/tmp/fileEngine/MainModel/_search_index.json', JSON.stringify({
149+
version: '2.3.9',
150+
fields: ['string'],
151+
fieldVectors: [['string/MainModel/111111111111', [0, 0.182]], ['string/MainModel/000000000000', [0, 0.182]]],
152+
invertedIndex: [['string', {
153+
_index: 0,
154+
string: {'MainModel/111111111111': {}, 'MainModel/000000000000': {}},
155+
}]],
156+
pipeline: ['stemmer'],
157+
}));
158+
});
159+
106160
test('FileEngine.put(model) updates existing indexes', async t => {
107161
const filesystem = stubFs({
108162
'MainModel/_index.json': {
@@ -194,7 +248,7 @@ test('FileEngine.find(MainModel, {string: "test"}) when a matching model does no
194248
});
195249

196250
test('FileEngine.find(MainModel, {string: "test"}) when no index exists', async t => {
197-
const filesystem = stubFs();
251+
const filesystem = stubFs({}, []);
198252

199253
const models = await FileEngine.configure({
200254
path: '/tmp/fileEngine',
@@ -204,6 +258,70 @@ test('FileEngine.find(MainModel, {string: "test"}) when no index exists', async
204258
t.deepEqual(models, []);
205259
});
206260

261+
test('FileEngine.search(MainModel, "String") when a matching model exists', async t => {
262+
const filesystem = stubFs({}, [
263+
getTestModelInstance(valid),
264+
getTestModelInstance({
265+
id: 'MainModel/1111111111111',
266+
string: 'another string',
267+
}),
268+
]);
269+
270+
const configuration = {
271+
path: '/tmp/fileEngine',
272+
filesystem,
273+
};
274+
275+
const model0 = await FileEngine.configure(configuration).get(MainModel, 'MainModel/000000000000');
276+
277+
const model1 = await FileEngine.configure(configuration).get(MainModel, 'MainModel/1111111111111');
278+
279+
const models = await FileEngine.configure(configuration).search(MainModel, 'String');
280+
281+
t.like(models, [{
282+
ref: 'MainModel/000000000000',
283+
score: 0.211,
284+
model: model0,
285+
}, {
286+
ref: 'MainModel/1111111111111',
287+
score: 0.16,
288+
model: model1,
289+
}]);
290+
});
291+
292+
test('FileEngine.search(MainModel, "not-even-close-to-a-match") when a matching model exists', async t => {
293+
const filesystem = stubFs({}, [
294+
getTestModelInstance(valid),
295+
getTestModelInstance({
296+
id: 'MainModel/1111111111111',
297+
string: 'another string',
298+
}),
299+
]);
300+
301+
const configuration = {
302+
path: '/tmp/fileEngine',
303+
filesystem,
304+
};
305+
306+
const models = await FileEngine.configure(configuration).search(MainModel, 'not-even-close-to-a-match');
307+
308+
t.deepEqual(models, []);
309+
});
310+
311+
test('FileEngine.search(MainModel, "String") when no index exists for the model', async t => {
312+
const filesystem = stubFs({}, []);
313+
314+
const configuration = {
315+
path: '/tmp/fileEngine',
316+
filesystem,
317+
};
318+
319+
await t.throwsAsync(async () => await FileEngine.configure(configuration).search(MainModel, 'String'), {
320+
instanceOf: NotImplementedError,
321+
message: 'The model MainModel does not have a search index available.',
322+
});
323+
});
324+
207325
test('FileEngine.hydrate(model)', async t => {
208326
const model = getTestModelInstance(valid);
209327

0 commit comments

Comments
 (0)