Add support for VoyageAI embeddings API

Added support for environment variables:
- USE_VOYAGEAI_EMBEDDING
- VOYAGEAI_API_KEY
- VOYAGEAI_EMBEDDING_DIMENSIONS
- VOYAGEAI_EMBEDDING_MODEL
Configuration follows existing patterns. Values for dimensions and model
can be found in the VoyageAI API documentation.

Some minor cleanup of the embedding.ts file.

Added unit tests around embedding configuration.

.env.example

@@ -136,6 +136,12 @@ SMALL_ANTHROPIC_MODEL=          # Default: claude-3-haiku-20240307
 MEDIUM_ANTHROPIC_MODEL=         # Default: claude-3-5-sonnet-20241022
 LARGE_ANTHROPIC_MODEL=          # Default: claude-3-5-sonnet-20241022
 
+# VoyageAI Configuration
+VOYAGEAI_API_KEY=
+USE_VOYAGEAI_EMBEDDING=         # Set to TRUE for VoyageAI, leave blank for local
+VOYAGEAI_EMBEDDING_MODEL=       # Default: voyage-3-lite
+VOYAGEAI_EMBEDDING_DIMENSIONS=  # Default: 512
+
 # Heurist Configuration
 HEURIST_API_KEY=                # Get from https://heurist.ai/dev-access
 SMALL_HEURIST_MODEL=            # Default: meta-llama/llama-3-70b-instruct

packages/core/src/embedding.ts

@@ -1,51 +1,61 @@
 import path from "node:path";
 import { models } from "./models.ts";
-import { IAgentRuntime, ModelProviderName } from "./types.ts";
+import { EmbeddingOptions, IAgentRuntime } from "./types.ts";
 import settings from "./settings.ts";
 import elizaLogger from "./logger.ts";
+import { getVoyageAIEmbeddingConfig } from "./voyageai.ts";
 
-interface EmbeddingOptions {
-    model: string;
-    endpoint: string;
-    apiKey?: string;
-    length?: number;
-    isOllama?: boolean;
-    dimensions?: number;
-    provider?: string;
-}
+// Get embedding config based on settings
+export function getEmbeddingConfig(): EmbeddingOptions {
+    if (settings.USE_OPENAI_EMBEDDING?.toLowerCase() === "true") {
+        return {
+            dimensions: 1536,
+            model: "text-embedding-3-small",
+            provider: "OpenAI",
+            endpoint: "https://api.openai.com/v1",
+            apiKey: settings.OPENAI_API_KEY,
+            maxInputTokens: 1000000,
+        };
+    }
+    if (settings.USE_OLLAMA_EMBEDDING?.toLowerCase() === "true") {
+        return {
+            dimensions: 1024,
+            model: settings.OLLAMA_EMBEDDING_MODEL || "mxbai-embed-large",
+            provider: "Ollama",
+            endpoint: "https://ollama.eliza.ai/v1",
+            apiKey: settings.OLLAMA_API_KEY,
+            maxInputTokens: 1000000,
+        };
+    }
+    if (settings.USE_GAIANET_EMBEDDING?.toLowerCase() === "true") {
+        return {
+            dimensions: 768,
+            model: settings.GAIANET_EMBEDDING_MODEL || "nomic-embed",
+            provider: "GaiaNet",
+            endpoint: settings.SMALL_GAIANET_SERVER_URL || settings.MEDIUM_GAIANET_SERVER_URL || settings.LARGE_GAIANET_SERVER_URL,
+            apiKey: settings.GAIANET_API_KEY,
+            maxInputTokens: 1000000,
+        };
+    }
+    if (settings.USE_VOYAGEAI_EMBEDDING?.toLowerCase() === "true") {
+        return getVoyageAIEmbeddingConfig();
+    }
 
-// Add the embedding configuration
-export const getEmbeddingConfig = () => ({
-    dimensions:
-        settings.USE_OPENAI_EMBEDDING?.toLowerCase() === "true"
-            ? 1536 // OpenAI
-            : settings.USE_OLLAMA_EMBEDDING?.toLowerCase() === "true"
-              ? 1024 // Ollama mxbai-embed-large
-              :settings.USE_GAIANET_EMBEDDING?.toLowerCase() === "true"
-                ? 768 // GaiaNet
-                : 384, // BGE
-    model:
-        settings.USE_OPENAI_EMBEDDING?.toLowerCase() === "true"
-            ? "text-embedding-3-small"
-            : settings.USE_OLLAMA_EMBEDDING?.toLowerCase() === "true"
-              ? settings.OLLAMA_EMBEDDING_MODEL || "mxbai-embed-large"
-              : settings.USE_GAIANET_EMBEDDING?.toLowerCase() === "true"
-                ? settings.GAIANET_EMBEDDING_MODEL || "nomic-embed"
-                : "BGE-small-en-v1.5",
-    provider:
-        settings.USE_OPENAI_EMBEDDING?.toLowerCase() === "true"
-            ? "OpenAI"
-            : settings.USE_OLLAMA_EMBEDDING?.toLowerCase() === "true"
-              ? "Ollama"
-              : settings.USE_GAIANET_EMBEDDING?.toLowerCase() === "true"
-                ? "GaiaNet"
-                : "BGE",
-});
+    // Fallback to local BGE
+    return {
+        dimensions: 384,
+        model: "BGE-small-en-v1.5",
+        provider: "BGE",
+        maxInputTokens: 1000000,
+    };
+};
 
 async function getRemoteEmbedding(
     input: string,
     options: EmbeddingOptions
 ): Promise<number[]> {
+    elizaLogger.debug("Getting remote embedding using provider:", options.provider);
+
     // Ensure endpoint ends with /v1 for OpenAI
     const baseEndpoint = options.endpoint.endsWith("/v1")
         ? options.endpoint
@@ -54,6 +64,17 @@ async function getRemoteEmbedding(
     // Construct full URL
     const fullUrl = `${baseEndpoint}/embeddings`;
 
+    // jank. voyageai is the only one that doesn't use "dimensions".
+    const body = options.provider === "VoyageAI" ? {
+        input,
+        model: options.model,
+        output_dimension: options.dimensions,
+    } : {
+        input,
+        model: options.model,
+        dimensions: options.dimensions || options.length,
+    };
+
     const requestOptions = {
         method: "POST",
         headers: {
@@ -64,14 +85,7 @@ async function getRemoteEmbedding(
                   }
                 : {}),
         },
-        body: JSON.stringify({
-            input,
-            model: options.model,
-            dimensions:
-                options.dimensions ||
-                options.length ||
-                getEmbeddingConfig().dimensions, // Prefer dimensions, fallback to length
-        }),
+        body: JSON.stringify(body),
     };
 
     try {
@@ -96,52 +110,19 @@ async function getRemoteEmbedding(
     }
 }
 
-export function getEmbeddingType(runtime: IAgentRuntime): "local" | "remote" {
-    const isNode =
-        typeof process !== "undefined" &&
-        process.versions != null &&
-        process.versions.node != null;
-
-    // Use local embedding if:
-    // - Running in Node.js
-    // - Not using OpenAI provider
-    // - Not forcing OpenAI embeddings
-    const isLocal =
-        isNode &&
-        runtime.character.modelProvider !== ModelProviderName.OPENAI &&
-        runtime.character.modelProvider !== ModelProviderName.GAIANET &&
-        !settings.USE_OPENAI_EMBEDDING;
-
-    return isLocal ? "local" : "remote";
-}
-
 export function getEmbeddingZeroVector(): number[] {
-    let embeddingDimension = 384; // Default BGE dimension
-
-    if (settings.USE_OPENAI_EMBEDDING?.toLowerCase() === "true") {
-        embeddingDimension = 1536; // OpenAI dimension
-    } else if (settings.USE_OLLAMA_EMBEDDING?.toLowerCase() === "true") {
-        embeddingDimension = 1024; // Ollama mxbai-embed-large dimension
-    }
-
-    return Array(embeddingDimension).fill(0);
+    // Default BGE dimension is 384
+    return Array(getEmbeddingConfig().dimensions).fill(0);
 }
 
 /**
  * Gets embeddings from a remote API endpoint.  Falls back to local BGE/384
  *
  * @param {string} input - The text to generate embeddings for
- * @param {EmbeddingOptions} options - Configuration options including:
- *   - model: The model name to use
- *   - endpoint: Base API endpoint URL
- *   - apiKey: Optional API key for authentication
- *   - isOllama: Whether this is an Ollama endpoint
- *   - dimensions: Desired embedding dimensions
  * @param {IAgentRuntime} runtime - The agent runtime context
  * @returns {Promise<number[]>} Array of embedding values
- * @throws {Error} If the API request fails
+ * @throws {Error} If the API request fails or configuration is invalid
  */
-
 export async function embed(runtime: IAgentRuntime, input: string) {
     elizaLogger.debug("Embedding request:", {
         modelProvider: runtime.character.modelProvider,
@@ -170,39 +151,9 @@ export async function embed(runtime: IAgentRuntime, input: string) {
     const config = getEmbeddingConfig();
     const isNode = typeof process !== "undefined" && process.versions?.node;
 
-    // Determine which embedding path to use
-    if (config.provider === "OpenAI") {
-        return await getRemoteEmbedding(input, {
-            model: config.model,
-            endpoint: "https://api.openai.com/v1",
-            apiKey: settings.OPENAI_API_KEY,
-            dimensions: config.dimensions,
-        });
-    }
-
-    if (config.provider === "Ollama") {
-        return await getRemoteEmbedding(input, {
-            model: config.model,
-            endpoint:
-                runtime.character.modelEndpointOverride ||
-                models[ModelProviderName.OLLAMA].endpoint,
-            isOllama: true,
-            dimensions: config.dimensions,
-        });
-    }
-
-    if (config.provider=="GaiaNet") {
-        return await getRemoteEmbedding(input, {
-            model: config.model,
-            endpoint:
-                runtime.character.modelEndpointOverride ||
-                models[ModelProviderName.GAIANET].endpoint ||
-                settings.SMALL_GAIANET_SERVER_URL ||
-                settings.MEDIUM_GAIANET_SERVER_URL ||
-                settings.LARGE_GAIANET_SERVER_URL,
-            apiKey: settings.GAIANET_API_KEY || runtime.token,
-            dimensions: config.dimensions,
-        });
+    // Attempt remote embedding if it is configured.
+    if (config.provider !== "BGE") {
+        return await getRemoteEmbedding(input, config);
     }
 
     // BGE - try local first if in Node
@@ -225,6 +176,7 @@ export async function embed(runtime: IAgentRuntime, input: string) {
             models[runtime.character.modelProvider].endpoint,
         apiKey: runtime.token,
         dimensions: config.dimensions,
+        provider: config.provider,
     });
 
     async function getLocalEmbedding(input: string): Promise<number[]> {

packages/core/src/tests/embeddings.test.ts

@@ -0,0 +1,102 @@
+
+import { describe, expect, vi } from "vitest";
+import { getEmbeddingConfig } from '../embedding';
+import settings from '../settings';
+
+vi.mock("../settings");
+const mockedSettings = vi.mocked(settings);
+
+describe('getEmbeddingConfig', () => {
+  beforeEach(() => {
+    // Clear the specific mock
+    Object.keys(mockedSettings).forEach(key => {
+    delete mockedSettings[key];
+    });
+
+    vi.clearAllMocks();
+  });
+
+  afterEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('should return BGE config by default', () => {
+
+    mockedSettings.USE_OPENAI_EMBEDDING = 'false';
+    mockedSettings.USE_OLLAMA_EMBEDDING = 'false';
+    mockedSettings.USE_GAIANET_EMBEDDING = 'false';
+    mockedSettings.USE_VOYAGEAI_EMBEDDING = 'false';
+
+    const config = getEmbeddingConfig();
+    expect(config).toEqual({
+      dimensions: 384,
+      model: 'BGE-small-en-v1.5',
+      provider: 'BGE',
+      maxInputTokens: 1000000,
+    });
+  });
+
+  it('should return GaiaNet config when USE_GAIANET_EMBEDDING is true', () => {
+    mockedSettings.USE_GAIANET_EMBEDDING = 'true';
+    mockedSettings.GAIANET_EMBEDDING_MODEL = 'test-model';
+    mockedSettings.GAIANET_API_KEY = 'test-key';
+    mockedSettings.SMALL_GAIANET_SERVER_URL = 'https://test.gaianet.ai';
+
+    const config = getEmbeddingConfig();
+    expect(config).toEqual({
+      dimensions: 768,
+      model: 'test-model',
+      provider: 'GaiaNet',
+      endpoint: 'https://test.gaianet.ai',
+      apiKey: 'test-key',
+      maxInputTokens: 1000000,
+    });
+  });
+
+
+  it('should return VoyageAI config when USE_VOYAGEAI_EMBEDDING is true', () => {
+    mockedSettings.USE_VOYAGEAI_EMBEDDING = 'true';
+    mockedSettings.VOYAGEAI_API_KEY = 'test-key';
+
+    const config = getEmbeddingConfig();
+    expect(config).toEqual({
+      dimensions: 512,
+      model: 'voyage-3-lite',
+      provider: 'VoyageAI',
+      endpoint: 'https://api.voyageai.com/v1',
+      apiKey: 'test-key',
+      maxInputTokens: 1000000,
+    });
+  });
+
+  it('should return OpenAI config when USE_OPENAI_EMBEDDING is true', () => {
+    mockedSettings.USE_OPENAI_EMBEDDING = 'true';
+    mockedSettings.OPENAI_API_KEY = 'test-key';
+
+    const config = getEmbeddingConfig();
+    expect(config).toEqual({
+      dimensions: 1536,
+      model: 'text-embedding-3-small',
+      provider: 'OpenAI',
+      endpoint: 'https://api.openai.com/v1',
+      apiKey: 'test-key',
+      maxInputTokens: 1000000,
+    });
+  });
+
+  it('should return Ollama config when USE_OLLAMA_EMBEDDING is true', () => {
+    mockedSettings.USE_OLLAMA_EMBEDDING = 'true';
+    mockedSettings.OLLAMA_EMBEDDING_MODEL = 'test-model';
+    mockedSettings.OLLAMA_API_KEY = 'test-key';
+
+    const config = getEmbeddingConfig();
+    expect(config).toEqual({
+      dimensions: 1024,
+      model: 'test-model',
+      provider: 'Ollama',
+      endpoint: 'https://ollama.eliza.ai/v1',
+      apiKey: 'test-key',
+      maxInputTokens: 1000000,
+    });
+  });
+});

packages/core/src/types.ts

@@ -240,6 +240,20 @@ export enum ModelProviderName {
     AKASH_CHAT_API = "akash_chat_api",
 }
 
+/**
+ * Configuration for remote embedding providers.
+ */
+export interface EmbeddingOptions {
+    model: string;
+    endpoint?: string;
+    apiKey?: string;
+    length?: number;
+    isOllama?: boolean;
+    dimensions: number;
+    provider: string;
+    maxInputTokens?: number;
+}
+
 /**
  * Represents the current state/context of a conversation
  */