From 6d44f109137539a00524b137e61fe738f5254af3 Mon Sep 17 00:00:00 2001 From: Andy Brenneke Date: Thu, 2 Jan 2025 11:34:01 -0800 Subject: [PATCH] Allow specifying a custom tokenizer in RunGraphOptions --- packages/core/src/api/createProcessor.ts | 2 ++ packages/node/src/api.ts | 1 + 2 files changed, 3 insertions(+) diff --git a/packages/core/src/api/createProcessor.ts b/packages/core/src/api/createProcessor.ts index b44ce2d18..b20335438 100644 --- a/packages/core/src/api/createProcessor.ts +++ b/packages/core/src/api/createProcessor.ts @@ -19,6 +19,7 @@ import { getProcessorEvents, getProcessorSSEStream, getSingleNodeStream } from ' import { GraphProcessor } from '../model/GraphProcessor.js'; import { deserializeProject } from '../utils/serialization/serialization.js'; import { DEFAULT_CHAT_NODE_TIMEOUT } from '../utils/defaults.js'; +import type { Tokenizer } from '../integrations/Tokenizer.js'; export type LooseDataValue = DataValue | string | number | boolean; @@ -38,6 +39,7 @@ export type RunGraphOptions = { abortSignal?: AbortSignal; registry?: NodeRegistration; getChatNodeEndpoint?: ProcessContext['getChatNodeEndpoint']; + tokenizer?: Tokenizer; } & { [P in keyof ProcessEvents as `on${PascalCase

}`]?: (params: ProcessEvents[P]) => void; } & Settings; diff --git a/packages/node/src/api.ts b/packages/node/src/api.ts index 63e090ea6..32277236d 100644 --- a/packages/node/src/api.ts +++ b/packages/node/src/api.ts @@ -66,6 +66,7 @@ export function createProcessor( nativeApi: options.nativeApi ?? new NodeNativeApi(), datasetProvider: options.datasetProvider, audioProvider: options.audioProvider, + tokenizer: options.tokenizer, settings: { openAiKey: options.openAiKey ?? process.env.OPENAI_API_KEY ?? '', openAiOrganization: options.openAiOrganization ?? process.env.OPENAI_ORG_ID ?? '',