Vector Similarity Index

.github/workflows/ci.yml

@@ -26,17 +26,13 @@ jobs:
           --health-retries 5
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v2.3.0
+        uses: actions/setup-node@v3
         with:
           node-version: ${{ matrix.node-version }}
-      - name: Cache dependencies
-        uses: c-hive/gha-npm-cache@v1
-
-      - name: Update npm
-        run: npm install --global npm
+          cache: 'npm'
 
       - name: Install packages
         run: npm ci

README.md

@@ -222,7 +222,7 @@ const studioSchema = new Schema(Studio, {
 })
 ```
 
-When you create a `Schema`, it modifies the entity you handed it, adding getters and setters for the properties you define. The type those getters and setters accept and return are defined with the type parameter above. Valid values are: `string`, `number`, `boolean`, `string[]`, `date`, `point`, or `text`.
+When you create a `Schema`, it modifies the entity you handed it, adding getters and setters for the properties you define. The type those getters and setters accept and return are defined with the type parameter above. Valid values are: `string`, `number`, `boolean`, `string[]`, `date`, `point`, `text`, or `binary`.
 
 The first three do exactly what you think—they define a property that is a [String](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String), a [Number](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number), or a [Boolean](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Boolean). `string[]` does what you'd think as well, specifically defining an [Array](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array) of Strings.
 
@@ -236,17 +236,20 @@ const point = { longitude: 12.34, latitude: 56.78 }
 
 A `text` field is a lot like a `string`. If you're just reading and writing objects, they are identical. But if you want to *search* on them, they are very, very different. I'll cover that in detail when I talk about [using RediSearch](#-using-redisearch) but the tl;dr is that `string` fields can only be matched on their whole value—no partial matches—and are best for keys while `text` fields have full-text search enabled on them and are optimized for human-readable text.
 
-Additional field options can be set depending on the field type. These correspond to the [Field Options](https://redis.io/commands/ft.create/#field-options) available when creating a RediSearch full-text index. Other than the `separator` option, these only affect how content is indexed and searched.
+A `binary` field is a binary blob of data using a `Buffer` object. For Hash data structures it will be stored as a binary field in Redis, for JSON data structures it will be serialized to a numeric array. The `binary` field can be indexed as a [Vector Similarity](https://redis.io/docs/stack/search/reference/vectors/) field.
 
-|  schema type   | RediSearch type | `indexed` | `sortable` | `normalized` | `stemming` | `phonetic` | `weight` | `separator` | `caseSensitive` |
-| -------------- | :-------------: | :-------: | :--------: | :----------: | :--------: | :--------: | :------: | :---------: | :-------------: |
-| `string`       |       TAG       |    yes    |  HASH Only |   HASH Only  |      -     |      -     |     -    |     yes     |        yes      |
-| `number`       |     NUMERIC     |    yes    |    yes     |       -      |      -     |      -     |     -    |      -      |         -       |
-| `boolean`      |       TAG       |    yes    |  HASH Only |       -      |      -     |      -     |     -    |      -      |         -       |
-| `string[]`     |       TAG       |    yes    |  HASH Only |   HASH Only  |      -     |      -     |     -    |     yes     |        yes      |
-| `date`         |     NUMERIC     |    yes    |    yes     |       -      |            |      -     |     -    |      -      |         -       |
-| `point`        |       GEO       |    yes    |     -      |       -      |            |      -     |     -    |      -      |         -       |
-| `text`         |       TEXT      |    yes    |    yes     |      yes     |     yes    |     yes    |    yes   |      -      |         -       |
+Additional field options can be set depending on the field type. These correspond to the [Field Options](https://redis.io/commands/ft.create/#field-options) avialable when creating a RediSearch full-text index. Other than the `separator` option, these only affect how content is indexed and searched.
+
+|  schema type   | RediSearch type | `indexed` | `sortable` | `normalized` | `stemming` | `phonetic` | `weight` | `separator` | `caseSensitive` | `vector` |
+| -------------- | :-------------: | :-------: | :--------: | :----------: | :--------: | :--------: | :------: | :---------: | :-------------: | :------: |
+| `string`       |       TAG       |    yes    |  HASH Only |   HASH Only  |      -     |      -     |     -    |     yes     |        yes      |     -    |
+| `number`       |     NUMERIC     |    yes    |    yes     |       -      |      -     |      -     |     -    |      -      |         -       |     -    |
+| `boolean`      |       TAG       |    yes    |  HASH Only |       -      |      -     |      -     |     -    |      -      |         -       |     -    |
+| `string[]`     |       TAG       |    yes    |  HASH Only |   HASH Only  |      -     |      -     |     -    |     yes     |        yes      |     -    |
+| `date`         |     NUMERIC     |    yes    |    yes     |       -      |      -     |      -     |     -    |      -      |         -       |     -    |
+| `point`        |       GEO       |    yes    |     -      |       -      |      -     |      -     |     -    |      -      |         -       |     -    |
+| `text`         |       TEXT      |    yes    |    yes     |      yes     |     yes    |     yes    |    yes   |      -      |         -       |     -    |
+| `binary`       |      VECTOR     |    yes    |     -      |       -      |      -     |      -     |     -    |      -      |         -       |    yes   |
 
 * `indexed`: true | false, whether this field is indexed by RediSearch (default true)
 * `sortable`: true | false, whether to create an additional index to optmize sorting (default false)
@@ -256,6 +259,7 @@ Additional field options can be set depending on the field type. These correspon
 * `weight`: number, the importance weighting to use when ranking results (default 1)
 * `separator`: string, the character to delimit multiple tags (default '|')
 * `caseSensitive`: true | false, whether original letter casing is kept for search (default false)
+* `vector`: object containing [Vector Similarity](https://redis.io/docs/stack/search/reference/vectors/) configuration
 
 Example showing additional options:
 
@@ -269,6 +273,7 @@ const commentSchema = new Schema(Comment, {
   approved: { type: 'boolean', indexed: false },
   iphash: { type: 'string', caseSensitive: true },
   notes: { type: 'string', indexed: false },
+  image: { type: 'binary', vector: { algorithm: 'FLAT', dim: 512, distance_metric: 'COSINE' } },
 })
 ```
 

lib/client.ts

@@ -1,4 +1,4 @@
-import { createClient } from 'redis';
+import { createClient, commandOptions } from 'redis';
 import { Repository } from './repository';
 import { JsonRepository, HashRepository } from './repository';
 import { Entity } from './entity/entity';
@@ -11,7 +11,7 @@ export type RedisConnection = ReturnType<typeof createClient>;
  * Alias for a JavaScript object used by HSET.
  * @internal
  */
-export type RedisHashData = { [key: string]: string };
+export type RedisHashData = { [key: string]: string | Buffer };
 
 /**
  * Alias for any old JavaScript object used by JSON.SET.
@@ -174,7 +174,7 @@ export class Client {
 
     if (keysOnly) command.push('RETURN', '0');
 
-    return this.redis.sendCommand<any[]>(command);
+    return this.redis.sendCommand<any[]>(command, commandOptions({ returnBuffers: true }));
   }
 
   /** @internal */
@@ -204,7 +204,7 @@ export class Client {
   /** @internal */
   async hgetall(key: string): Promise<RedisHashData> {
     this.validateRedisOpen();
-    return this.redis.hGetAll(key);
+    return this.redis.hGetAll(commandOptions({ returnBuffers: true }), key);
   }
 
   /** @internal */

lib/entity/entity-value.ts

@@ -3,4 +3,4 @@ import { Point } from "./point";
 /**
  * Valid types for properties on an {@link Entity}.
  */
-export type EntityValue = string | number | boolean | Point | Date | any[] | null;
+export type EntityValue = string | number | boolean | Point | Date | any[] | Buffer | null;

lib/entity/entity.ts

@@ -8,6 +8,7 @@ import {
   EntityStringArrayField,
   EntityStringField,
   EntityTextField,
+  EntityBinaryField,
   EntityFieldConstructor,
 } from "./fields";
 import { Schema } from "../schema/schema";
@@ -21,7 +22,8 @@ const ENTITY_FIELD_CONSTRUCTORS: Record<SchemaFieldType, EntityFieldConstructor>
   'text': EntityTextField,
   'date': EntityDateField,
   'point': EntityPointField,
-  'string[]': EntityStringArrayField
+  'string[]': EntityStringArrayField,
+  'binary': EntityBinaryField,
 }
 
 /**

lib/entity/fields/entity-binary-field.ts

@@ -0,0 +1,45 @@
+import { EntityField } from "./entity-field";
+import { RedisHashData, RedisJsonData } from "../../client";
+import { EntityValue } from "../entity-value";
+
+export class EntityBinaryField extends EntityField {
+  toRedisJson(): RedisJsonData {
+    const data: RedisJsonData = {};
+    if (this.value !== null) {
+      const bytes = this.valueAsBuffer
+      const arr = new Float32Array(bytes.buffer, bytes.byteOffset, bytes.length / Float32Array.BYTES_PER_ELEMENT)
+      data[this.name] = [...arr]
+    }
+    return data;
+  }
+
+  fromRedisJson(value: any) {
+    if (!this.isBuffer(value)) {
+      throw Error(`Non-binary value of '${value}' read from Redis for binary field.`)
+    }
+    this.value = value
+  }
+
+  toRedisHash(): RedisHashData {
+    const data: RedisHashData = {};
+    if (this.value !== null) data[this.name] = this.valueAsBuffer
+    return data;
+  }
+
+  fromRedisHash(value: string | Buffer) {
+    if (!this.isBuffer(value)) {
+      throw Error(`Non-binary value of '${value}' read from Redis for binary field.`)
+    }
+    this.value = value as Buffer
+  }
+
+  protected validateValue(value: EntityValue) {
+    super.validateValue(value);
+    if (value !== null && !this.isBuffer(value))
+      throw Error(`Expected value with type of 'binary' but received '${value}'.`);
+  }
+
+  private get valueAsBuffer(): Buffer {
+    return this.value as Buffer
+  }
+}

lib/entity/fields/entity-boolean-field.ts

@@ -9,10 +9,11 @@ export class EntityBooleanField extends EntityField {
     return data;
   };
 
-  fromRedisHash(value: string) {
-    if (value === '0') {
+  fromRedisHash(value: string | Buffer) {
+    const str = value.toString()
+    if (str === '0') {
       this.value = false;
-    } else if (value === '1') {
+    } else if (str === '1') {
       this.value = true;
     } else {
       throw Error(`Non-boolean value of '${value}' read from Redis for boolean field.`);

lib/entity/fields/entity-date-field.ts

@@ -20,8 +20,8 @@ export class EntityDateField extends EntityField {
     return data;
   }
 
-  fromRedisHash(value: string) {
-    const parsed = Number.parseFloat(value);
+  fromRedisHash(value: string | Buffer) {
+    const parsed = Number.parseFloat(value.toString());
     if (Number.isNaN(parsed)) throw Error(`Non-numeric value of '${value}' read from Redis for date field.`);
     const date = new Date();
     date.setTime(parsed * 1000);

lib/entity/fields/entity-field.ts

@@ -44,8 +44,8 @@ export abstract class EntityField {
     return data;
   }
 
-  fromRedisHash(value: string) {
-    this.value = value;
+  fromRedisHash(value: string | Buffer) {
+    this.value = value.toString();
   }
 
   protected validateValue(value: EntityValue) {
@@ -67,4 +67,8 @@ export abstract class EntityField {
   protected isBoolean(value: EntityValue) {
     return typeof value === 'boolean';
   }
+
+  protected isBuffer(value: EntityValue) {
+    return value instanceof Buffer;
+  }
 }

lib/entity/fields/entity-number-field.ts

@@ -2,8 +2,8 @@ import { EntityField } from "./entity-field";
 import { EntityValue } from "../entity-value";
 
 export class EntityNumberField extends EntityField {
-  fromRedisHash(value: string) {
-    const number = Number.parseFloat(value);
+  fromRedisHash(value: string | Buffer) {
+    const number = Number.parseFloat(value.toString());
     if (Number.isNaN(number)) throw Error(`Non-numeric value of '${value}' read from Redis for number field.`);
     this.value = number;
   }

lib/entity/fields/entity-point-field.ts

@@ -29,9 +29,10 @@ export class EntityPointField extends EntityField {
     return data;
   };
 
-  fromRedisHash(value: string) {
-    if (value.match(IS_COORD_PAIR)) {
-      const [longitude, latitude] = value.split(',').map(Number.parseFloat);
+  fromRedisHash(value: string | Buffer) {
+    const str = value.toString()
+    if (str.match(IS_COORD_PAIR)) {
+      const [longitude, latitude] = str.split(',').map(Number.parseFloat);
       this.value = { longitude, latitude };
     } else {
       throw Error(`Non-point value of '${value}' read from Redis for point field.`);

lib/entity/fields/entity-string-array-field.ts

@@ -10,8 +10,8 @@ export class EntityStringArrayField extends EntityField {
     return data;
   }
 
-  fromRedisHash(value: string) {
-    this.value = value.split(this.separator);
+  fromRedisHash(value: string | Buffer) {
+    this.value = value.toString().split(this.separator);
   }
 
   protected validateValue(value: EntityValue) {

lib/entity/fields/index.ts

@@ -1,3 +1,4 @@
+export * from './entity-binary-field'
 export * from './entity-boolean-field'
 export * from './entity-date-field'
 export * from './entity-field-constructor'

lib/schema/builders/hash-schema-builder.ts

@@ -52,6 +52,11 @@ export class HashSchemaBuilder<TEntity extends Entity> extends SchemaBuilder<TEn
           ...this.buildWeight(fieldDef),
           ...this.buildIndexed(fieldDef),
         ]
+      case 'binary':
+        return [
+          fieldAlias,
+          ...this.buildVector(fieldDef),
+        ]
     };
   }
 }

lib/schema/builders/json-schema-builder.ts

@@ -56,6 +56,11 @@ export class JsonSchemaBuilder<TEntity extends Entity> extends SchemaBuilder<TEn
           ...this.buildWeight(fieldDef),
           ...this.buildIndexed(fieldDef),
         ]
+      case 'binary':
+        return [
+          ...fieldInfo,
+          ...this.buildVector(fieldDef),
+        ]
     };
   }
 }

lib/schema/builders/schema-builder.ts

@@ -8,6 +8,7 @@ import {
   SortableFieldDefinition,
   NormalizedFieldDefinition,
   WeightFieldDefinition,
+  BinaryFieldDefinition,
 } from "../definition";
 import { Schema } from "../schema";
 
@@ -60,4 +61,48 @@ export abstract class SchemaBuilder<TEntity extends Entity> {
   protected buildWeight(field: WeightFieldDefinition) {
     return field.weight ? ['WEIGHT', field.weight.toString()] : []
   }
+
+  protected buildVector(field: BinaryFieldDefinition) {
+    // assume that indexed: false takes precedence
+    if (!(field.indexed ?? this.schema.indexedDefault) || !field.vector) {
+      return ['NOINDEX']
+    }
+
+    const results = [
+      'TYPE', field.vector.vector_type ?? 'FLOAT32',
+      'DIM', field.vector.dim.toString(),
+      'DISTANCE_METRIC', field.vector.distance_metric,
+    ]
+
+    if (field.vector.initial_cap) {
+      results.push('INITIAL_CAP', field.vector.initial_cap.toString())
+    }
+
+    switch (field.vector.algorithm) {
+      case 'FLAT':
+        if (field.vector.block_size) {
+          results.push('BLOCK_SIZE', field.vector.block_size.toString())
+        }
+        break
+
+      case 'HNSW':
+        if (field.vector.m) {
+          results.push('M', field.vector.m.toString())
+        }
+        if (field.vector.ef_construction) {
+          results.push('EF_CONSTRUCTION', field.vector.ef_construction.toString())
+        }
+        if (field.vector.ef_runtime) {
+          results.push('EF_RUNTIME', field.vector.ef_runtime.toString())
+        }
+        break
+    }
+
+    return [
+      'VECTOR',
+      field.vector.algorithm,
+      results.length.toString(),
+      ...results,
+    ]
+  }
 }