feat: Add support for VoyageAI embeddings API

.env.example

@@ -136,6 +136,12 @@ SMALL_ANTHROPIC_MODEL=          # Default: claude-3-haiku-20240307
 MEDIUM_ANTHROPIC_MODEL=         # Default: claude-3-5-sonnet-20241022
 LARGE_ANTHROPIC_MODEL=          # Default: claude-3-5-sonnet-20241022
 
+# VoyageAI Configuration
+VOYAGEAI_API_KEY=
+USE_VOYAGEAI_EMBEDDING=         # Set to TRUE for VoyageAI, leave blank for local
+VOYAGEAI_EMBEDDING_MODEL=       # Default: voyage-3-lite
+VOYAGEAI_EMBEDDING_DIMENSIONS=  # Default: 512
+
 # Heurist Configuration
 HEURIST_API_KEY=                # Get from https://heurist.ai/dev-access
 SMALL_HEURIST_MODEL=            # Default: meta-llama/llama-3-70b-instruct

packages/core/src/embedding.ts

@@ -1,24 +1,16 @@
 import path from "node:path";
 import { models } from "./models.ts";
-import { IAgentRuntime, ModelProviderName } from "./types.ts";
+import { IAgentRuntime } from "./types.ts";
 import settings from "./settings.ts";
 import elizaLogger from "./logger.ts";
-
-interface EmbeddingOptions {
-    model: string;
-    endpoint: string;
-    apiKey?: string;
-    length?: number;
-    isOllama?: boolean;
-    dimensions?: number;
-    provider?: string;
-}
+import { getVoyageAIEmbeddingConfig } from "./voyageai.ts";
 
 export const EmbeddingProvider = {
     OpenAI: "OpenAI",
     Ollama: "Ollama",
     GaiaNet: "GaiaNet",
     BGE: "BGE",
+    VoyageAI: "VoyageAI",
 } as const;
 
 export type EmbeddingProvider =
@@ -29,52 +21,82 @@ export namespace EmbeddingProvider {
     export type Ollama = typeof EmbeddingProvider.Ollama;
     export type GaiaNet = typeof EmbeddingProvider.GaiaNet;
     export type BGE = typeof EmbeddingProvider.BGE;
+    export type VoyageAI = typeof EmbeddingProvider.VoyageAI;
 }
 
 export type EmbeddingConfig = {
     readonly dimensions: number;
     readonly model: string;
     readonly provider: EmbeddingProvider;
+    readonly endpoint?: string;
+    readonly apiKey?: string;
+    readonly maxInputTokens?: number;
 };
 
-export const getEmbeddingConfig = (): EmbeddingConfig => ({
-    dimensions:
-        settings.USE_OPENAI_EMBEDDING?.toLowerCase() === "true"
-            ? 1536 // OpenAI
-            : settings.USE_OLLAMA_EMBEDDING?.toLowerCase() === "true"
-              ? 1024 // Ollama mxbai-embed-large
-              : settings.USE_GAIANET_EMBEDDING?.toLowerCase() === "true"
-                ? 768 // GaiaNet
-                : 384, // BGE
-    model:
-        settings.USE_OPENAI_EMBEDDING?.toLowerCase() === "true"
-            ? "text-embedding-3-small"
-            : settings.USE_OLLAMA_EMBEDDING?.toLowerCase() === "true"
-              ? settings.OLLAMA_EMBEDDING_MODEL || "mxbai-embed-large"
-              : settings.USE_GAIANET_EMBEDDING?.toLowerCase() === "true"
-                ? settings.GAIANET_EMBEDDING_MODEL || "nomic-embed"
-                : "BGE-small-en-v1.5",
-    provider:
-        settings.USE_OPENAI_EMBEDDING?.toLowerCase() === "true"
-            ? "OpenAI"
-            : settings.USE_OLLAMA_EMBEDDING?.toLowerCase() === "true"
-              ? "Ollama"
-              : settings.USE_GAIANET_EMBEDDING?.toLowerCase() === "true"
-                ? "GaiaNet"
-                : "BGE",
-});
+// Get embedding config based on settings
+export function getEmbeddingConfig(): EmbeddingConfig {
+    if (settings.USE_OPENAI_EMBEDDING?.toLowerCase() === "true") {
+        return {
+            dimensions: 1536,
+            model: "text-embedding-3-small",
+            provider: "OpenAI",
+            endpoint: "https://api.openai.com/v1",
+            apiKey: settings.OPENAI_API_KEY,
+            maxInputTokens: 1000000,
+        };
+    }
+    if (settings.USE_OLLAMA_EMBEDDING?.toLowerCase() === "true") {
+        return {
+            dimensions: 1024,
+            model: settings.OLLAMA_EMBEDDING_MODEL || "mxbai-embed-large",
+            provider: "Ollama",
+            endpoint: "https://ollama.eliza.ai/",
+            apiKey: settings.OLLAMA_API_KEY,
+            maxInputTokens: 1000000,
+        };
+    }
+    if (settings.USE_GAIANET_EMBEDDING?.toLowerCase() === "true") {
+        return {
+            dimensions: 768,
+            model: settings.GAIANET_EMBEDDING_MODEL || "nomic-embed",
+            provider: "GaiaNet",
+            endpoint: settings.SMALL_GAIANET_SERVER_URL || settings.MEDIUM_GAIANET_SERVER_URL || settings.LARGE_GAIANET_SERVER_URL,
+            apiKey: settings.GAIANET_API_KEY,
+            maxInputTokens: 1000000,
+        };
+    }
+    if (settings.USE_VOYAGEAI_EMBEDDING?.toLowerCase() === "true") {
+        return getVoyageAIEmbeddingConfig();
+    }
+
+    // Fallback to local BGE
+    return {
+        dimensions: 384,
+        model: "BGE-small-en-v1.5",
+        provider: "BGE",
+        maxInputTokens: 1000000,
+    };
+};
 
 async function getRemoteEmbedding(
     input: string,
-    options: EmbeddingOptions
+    options: EmbeddingConfig
 ): Promise<number[]> {
-    // Ensure endpoint ends with /v1 for OpenAI
-    const baseEndpoint = options.endpoint.endsWith("/v1")
-        ? options.endpoint
-        : `${options.endpoint}${options.isOllama ? "/v1" : ""}`;
+    elizaLogger.debug("Getting remote embedding using provider:", options.provider);
 
     // Construct full URL
-    const fullUrl = `${baseEndpoint}/embeddings`;
+    const fullUrl = `${options.endpoint}/embeddings`;
+
+    // jank. voyageai is the only one that doesn't use "dimensions".
+    const body = options.provider === "VoyageAI" ? {
+        input,
+        model: options.model,
+        output_dimension: options.dimensions,
+    } : {
+        input,
+        model: options.model,
+        dimensions: options.dimensions,
+    };
 
     const requestOptions = {
         method: "POST",
@@ -86,14 +108,7 @@ async function getRemoteEmbedding(
                   }
                 : {}),
         },
-        body: JSON.stringify({
-            input,
-            model: options.model,
-            dimensions:
-                options.dimensions ||
-                options.length ||
-                getEmbeddingConfig().dimensions, // Prefer dimensions, fallback to length
-        }),
+        body: JSON.stringify(body),
     };
 
     try {
@@ -118,52 +133,19 @@ async function getRemoteEmbedding(
     }
 }
 
-export function getEmbeddingType(runtime: IAgentRuntime): "local" | "remote" {
-    const isNode =
-        typeof process !== "undefined" &&
-        process.versions != null &&
-        process.versions.node != null;
-
-    // Use local embedding if:
-    // - Running in Node.js
-    // - Not using OpenAI provider
-    // - Not forcing OpenAI embeddings
-    const isLocal =
-        isNode &&
-        runtime.character.modelProvider !== ModelProviderName.OPENAI &&
-        runtime.character.modelProvider !== ModelProviderName.GAIANET &&
-        !settings.USE_OPENAI_EMBEDDING;
-
-    return isLocal ? "local" : "remote";
-}
-
 export function getEmbeddingZeroVector(): number[] {
-    let embeddingDimension = 384; // Default BGE dimension
-
-    if (settings.USE_OPENAI_EMBEDDING?.toLowerCase() === "true") {
-        embeddingDimension = 1536; // OpenAI dimension
-    } else if (settings.USE_OLLAMA_EMBEDDING?.toLowerCase() === "true") {
-        embeddingDimension = 1024; // Ollama mxbai-embed-large dimension
-    }
-
-    return Array(embeddingDimension).fill(0);
+    // Default BGE dimension is 384
+    return Array(getEmbeddingConfig().dimensions).fill(0);
 }
 
 /**
  * Gets embeddings from a remote API endpoint.  Falls back to local BGE/384
  *
  * @param {string} input - The text to generate embeddings for
- * @param {EmbeddingOptions} options - Configuration options including:
- *   - model: The model name to use
- *   - endpoint: Base API endpoint URL
- *   - apiKey: Optional API key for authentication
- *   - isOllama: Whether this is an Ollama endpoint
- *   - dimensions: Desired embedding dimensions
  * @param {IAgentRuntime} runtime - The agent runtime context
  * @returns {Promise<number[]>} Array of embedding values
- * @throws {Error} If the API request fails
+ * @throws {Error} If the API request fails or configuration is invalid
  */
-
 export async function embed(runtime: IAgentRuntime, input: string) {
     elizaLogger.debug("Embedding request:", {
         modelProvider: runtime.character.modelProvider,
@@ -192,39 +174,9 @@ export async function embed(runtime: IAgentRuntime, input: string) {
     const config = getEmbeddingConfig();
     const isNode = typeof process !== "undefined" && process.versions?.node;
 
-    // Determine which embedding path to use
-    if (config.provider === EmbeddingProvider.OpenAI) {
-        return await getRemoteEmbedding(input, {
-            model: config.model,
-            endpoint: "https://api.openai.com/v1",
-            apiKey: settings.OPENAI_API_KEY,
-            dimensions: config.dimensions,
-        });
-    }
-
-    if (config.provider === EmbeddingProvider.Ollama) {
-        return await getRemoteEmbedding(input, {
-            model: config.model,
-            endpoint:
-                runtime.character.modelEndpointOverride ||
-                models[ModelProviderName.OLLAMA].endpoint,
-            isOllama: true,
-            dimensions: config.dimensions,
-        });
-    }
-
-    if (config.provider == EmbeddingProvider.GaiaNet) {
-        return await getRemoteEmbedding(input, {
-            model: config.model,
-            endpoint:
-                runtime.character.modelEndpointOverride ||
-                models[ModelProviderName.GAIANET].endpoint ||
-                settings.SMALL_GAIANET_SERVER_URL ||
-                settings.MEDIUM_GAIANET_SERVER_URL ||
-                settings.LARGE_GAIANET_SERVER_URL,
-            apiKey: settings.GAIANET_API_KEY || runtime.token,
-            dimensions: config.dimensions,
-        });
+    // Attempt remote embedding if it is configured.
+    if (config.provider !== EmbeddingProvider.BGE) {
+        return await getRemoteEmbedding(input, config);
     }
 
     // BGE - try local first if in Node
@@ -247,6 +199,7 @@ export async function embed(runtime: IAgentRuntime, input: string) {
             models[runtime.character.modelProvider].endpoint,
         apiKey: runtime.token,
         dimensions: config.dimensions,
+        provider: config.provider,
     });
 
     async function getLocalEmbedding(input: string): Promise<number[]> {

packages/core/src/tests/embeddings.test.ts

@@ -0,0 +1,102 @@
+
+import { describe, expect, vi } from "vitest";
+import { getEmbeddingConfig } from '../embedding';
+import settings from '../settings';
+
+vi.mock("../settings");
+const mockedSettings = vi.mocked(settings);
+
+describe('getEmbeddingConfig', () => {
+  beforeEach(() => {
+    // Clear the specific mock
+    Object.keys(mockedSettings).forEach(key => {
+    delete mockedSettings[key];
+    });
+
+    vi.clearAllMocks();
+  });
+
+  afterEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('should return BGE config by default', () => {
+
+    mockedSettings.USE_OPENAI_EMBEDDING = 'false';
+    mockedSettings.USE_OLLAMA_EMBEDDING = 'false';
+    mockedSettings.USE_GAIANET_EMBEDDING = 'false';
+    mockedSettings.USE_VOYAGEAI_EMBEDDING = 'false';
+
+    const config = getEmbeddingConfig();
+    expect(config).toEqual({
+      dimensions: 384,
+      model: 'BGE-small-en-v1.5',
+      provider: 'BGE',
+      maxInputTokens: 1000000,
+    });
+  });
+
+  it('should return GaiaNet config when USE_GAIANET_EMBEDDING is true', () => {
+    mockedSettings.USE_GAIANET_EMBEDDING = 'true';
+    mockedSettings.GAIANET_EMBEDDING_MODEL = 'test-model';
+    mockedSettings.GAIANET_API_KEY = 'test-key';
+    mockedSettings.SMALL_GAIANET_SERVER_URL = 'https://test.gaianet.ai';
+
+    const config = getEmbeddingConfig();
+    expect(config).toEqual({
+      dimensions: 768,
+      model: 'test-model',
+      provider: 'GaiaNet',
+      endpoint: 'https://test.gaianet.ai',
+      apiKey: 'test-key',
+      maxInputTokens: 1000000,
+    });
+  });
+
+
+  it('should return VoyageAI config when USE_VOYAGEAI_EMBEDDING is true', () => {
+    mockedSettings.USE_VOYAGEAI_EMBEDDING = 'true';
+    mockedSettings.VOYAGEAI_API_KEY = 'test-key';
+
+    const config = getEmbeddingConfig();
+    expect(config).toEqual({
+      dimensions: 512,
+      model: 'voyage-3-lite',
+      provider: 'VoyageAI',
+      endpoint: 'https://api.voyageai.com/v1',
+      apiKey: 'test-key',
+      maxInputTokens: 1000000,
+    });
+  });
+
+  it('should return OpenAI config when USE_OPENAI_EMBEDDING is true', () => {
+    mockedSettings.USE_OPENAI_EMBEDDING = 'true';
+    mockedSettings.OPENAI_API_KEY = 'test-key';
+
+    const config = getEmbeddingConfig();
+    expect(config).toEqual({
+      dimensions: 1536,
+      model: 'text-embedding-3-small',
+      provider: 'OpenAI',
+      endpoint: 'https://api.openai.com/v1',
+      apiKey: 'test-key',
+      maxInputTokens: 1000000,
+    });
+  });
+
+  it('should return Ollama config when USE_OLLAMA_EMBEDDING is true', () => {
+    mockedSettings.USE_OLLAMA_EMBEDDING = 'true';
+    mockedSettings.OLLAMA_EMBEDDING_MODEL = 'test-model';
+    mockedSettings.OLLAMA_API_KEY = 'test-key';
+
+    const config = getEmbeddingConfig();
+    expect(config).toEqual({
+      dimensions: 1024,
+      model: 'test-model',
+      provider: 'Ollama',
+      endpoint: 'https://ollama.eliza.ai/v1',
+      apiKey: 'test-key',
+      maxInputTokens: 1000000,
+    });
+  });
+});