Allow specifying a custom tokenizer in RunGraphOptions

Ironclad · Jan 2, 2025 · 6d44f10 · 6d44f10
1 parent 393170e
commit 6d44f10
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 0 deletions.
diff --git a/packages/core/src/api/createProcessor.ts b/packages/core/src/api/createProcessor.ts
@@ -19,6 +19,7 @@ import { getProcessorEvents, getProcessorSSEStream, getSingleNodeStream } from '
 import { GraphProcessor } from '../model/GraphProcessor.js';
 import { deserializeProject } from '../utils/serialization/serialization.js';
 import { DEFAULT_CHAT_NODE_TIMEOUT } from '../utils/defaults.js';
+import type { Tokenizer } from '../integrations/Tokenizer.js';
 
 export type LooseDataValue = DataValue | string | number | boolean;
 
@@ -38,6 +39,7 @@ export type RunGraphOptions = {
   abortSignal?: AbortSignal;
   registry?: NodeRegistration;
   getChatNodeEndpoint?: ProcessContext['getChatNodeEndpoint'];
+  tokenizer?: Tokenizer;
 } & {
   [P in keyof ProcessEvents as `on${PascalCase<P>}`]?: (params: ProcessEvents[P]) => void;
 } & Settings;

diff --git a/packages/node/src/api.ts b/packages/node/src/api.ts
@@ -66,6 +66,7 @@ export function createProcessor(
           nativeApi: options.nativeApi ?? new NodeNativeApi(),
           datasetProvider: options.datasetProvider,
           audioProvider: options.audioProvider,
+          tokenizer: options.tokenizer,
           settings: {
             openAiKey: options.openAiKey ?? process.env.OPENAI_API_KEY ?? '',
             openAiOrganization: options.openAiOrganization ?? process.env.OPENAI_ORG_ID ?? '',