Tool call support interactive node (labring#2903)

* feat: tool call support interactive node

* feat: interactive node tool response

* fix: tool call concat

* fix: llm history concat

docSite/content/zh-cn/docs/development/upgrading/4812.md

@@ -11,4 +11,6 @@ weight: 812
 
 1. 新增 - 全局变量支持更多数据类型
 2. 新增 - FE_DOMAIN 环境变量，配置该环境变量后，上传文件/图片会补全后缀后得到完整地址。（可解决 docx 文件图片链接，有时会无法被模型识别问题）
-3. 修复 - 文件后缀判断，去除 query 影响。
+3. 新增 - 工具调用支持交互模式
+4. 修复 - 文件后缀判断，去除 query 影响。
+5. 修复 - AI 响应为空时，会造成 LLM 历史记录合并。

packages/global/common/string/markdown.ts

@@ -70,10 +70,10 @@ export const uploadMarkdownBase64 = async ({
  }
 
  // Remove white space on both sides of the picture
- const trimReg = /(!\[.*\]\(.*\))\s*/g;
- if (trimReg.test(rawText)) {
- rawText = rawText.replace(trimReg, '$1');
- }
+ // const trimReg = /(!\[.*\]\(.*\))\s*/g;
+ // if (trimReg.test(rawText)) {
+ //  rawText = rawText.replace(trimReg, '$1');
+ // }
 
  return rawText;
 };

packages/global/core/ai/type.d.ts

@@ -4,12 +4,14 @@ import type {
  ChatCompletionChunk,
  ChatCompletionMessageParam as SdkChatCompletionMessageParam,
  ChatCompletionToolMessageParam,
- ChatCompletionAssistantMessageParam,
  ChatCompletionContentPart as SdkChatCompletionContentPart,
- ChatCompletionUserMessageParam as SdkChatCompletionUserMessageParam
+ ChatCompletionUserMessageParam as SdkChatCompletionUserMessageParam,
+ ChatCompletionToolMessageParam as SdkChatCompletionToolMessageParam,
+ ChatCompletionAssistantMessageParam as SdkChatCompletionAssistantMessageParam,
+ ChatCompletionContentPartText
 } from 'openai/resources';
 import { ChatMessageTypeEnum } from './constants';
-import { InteractiveNodeResponseItemType } from '../workflow/template/system/interactive/type';
+import { WorkflowInteractiveResponseType } from '../workflow/template/system/interactive/type';
 export * from 'openai/resources';
 
 // Extension of ChatCompletionMessageParam, Add file url type
@@ -22,18 +24,31 @@ export type ChatCompletionContentPartFile = {
 export type ChatCompletionContentPart =
  | SdkChatCompletionContentPart
  | ChatCompletionContentPartFile;
-type CustomChatCompletionUserMessageParam = {
- content: string | Array<ChatCompletionContentPart>;
+type CustomChatCompletionUserMessageParam = Omit<ChatCompletionUserMessageParam, 'content'> & {
  role: 'user';
+ content: string | Array<ChatCompletionContentPart>;
+};
+type CustomChatCompletionToolMessageParam = SdkChatCompletionToolMessageParam & {
+ role: 'tool';
  name?: string;
 };
+type CustomChatCompletionAssistantMessageParam = SdkChatCompletionAssistantMessageParam & {
+ role: 'assistant';
+ interactive?: WorkflowInteractiveResponseType;
+};
 
 export type ChatCompletionMessageParam = (
- | Exclude<SdkChatCompletionMessageParam, SdkChatCompletionUserMessageParam>
+ | Exclude<
+ SdkChatCompletionMessageParam,
+ | SdkChatCompletionUserMessageParam
+ | SdkChatCompletionToolMessageParam
+ | SdkChatCompletionAssistantMessageParam
+ >
  | CustomChatCompletionUserMessageParam
+ | CustomChatCompletionToolMessageParam
+ | CustomChatCompletionAssistantMessageParam
 ) & {
  dataId?: string;
- interactive?: InteractiveNodeResponseItemType;
 };
 export type SdkChatCompletionMessageParam = SdkChatCompletionMessageParam;
 
@@ -47,11 +62,12 @@ export type ChatCompletionMessageToolCall = ChatCompletionMessageToolCall & {
  toolName?: string;
  toolAvatar?: string;
 };
-export type ChatCompletionMessageFunctionCall = ChatCompletionAssistantMessageParam.FunctionCall & {
- id?: string;
- toolName?: string;
- toolAvatar?: string;
-};
+export type ChatCompletionMessageFunctionCall =
+ SdkChatCompletionAssistantMessageParam.FunctionCall & {
+ id?: string;
+ toolName?: string;
+ toolAvatar?: string;
+ };
 
 // Stream response
 export type StreamChatType = Stream<ChatCompletionChunk>;

packages/global/core/chat/adapt.ts

@@ -90,8 +90,9 @@ export const chats2GPTMessages = ({
  });
  }
  } else {
+ const aiResults: ChatCompletionMessageParam[] = [];
  //AI
- item.value.forEach((value) => {
+ item.value.forEach((value, i) => {
  if (value.type === ChatItemValueTypeEnum.tool && value.tools && reserveTool) {
  const tool_calls: ChatCompletionMessageToolCall[] = [];
  const toolResponse: ChatCompletionToolMessageParam[] = [];
@@ -111,28 +112,53 @@ export const chats2GPTMessages = ({
  content: tool.response
  });
  });
- results = results
- .concat({
- dataId,
- role: ChatCompletionRequestMessageRoleEnum.Assistant,
- tool_calls
- })
- .concat(toolResponse);
- } else if (value.text?.content) {
- results.push({
+ aiResults.push({
  dataId,
  role: ChatCompletionRequestMessageRoleEnum.Assistant,
- content: value.text.content
+ tool_calls
  });
+ aiResults.push(...toolResponse);
+ } else if (
+ value.type === ChatItemValueTypeEnum.text &&
+ typeof value.text?.content === 'string'
+ ) {
+ // Concat text
+ const lastValue = item.value[i - 1];
+ const lastResult = aiResults[aiResults.length - 1];
+ if (
+ lastValue &&
+ lastValue.type === ChatItemValueTypeEnum.text &&
+ typeof lastResult.content === 'string'
+ ) {
+ lastResult.content += value.text.content;
+ } else {
+ aiResults.push({
+ dataId,
+ role: ChatCompletionRequestMessageRoleEnum.Assistant,
+ content: value.text.content
+ });
+ }
  } else if (value.type === ChatItemValueTypeEnum.interactive) {
- results = results.concat({
+ aiResults.push({
  dataId,
  role: ChatCompletionRequestMessageRoleEnum.Assistant,
- interactive: value.interactive,
- content: ''
+ interactive: value.interactive
  });
  }
  });
+
+ // Auto add empty assistant message
+ results = results.concat(
+ aiResults.length > 0
+ ? aiResults
+ : [
+ {
+ dataId,
+ role: ChatCompletionRequestMessageRoleEnum.Assistant,
+ content: ''
+ }
+ ]
+ );
  }
  });
 
@@ -215,14 +241,7 @@ export const GPTMessages2Chats = (
  obj === ChatRoleEnum.AI &&
  item.role === ChatCompletionRequestMessageRoleEnum.Assistant
  ) {
- if (item.content && typeof item.content === 'string') {
- value.push({
- type: ChatItemValueTypeEnum.text,
- text: {
- content: item.content
- }
- });
- } else if (item.tool_calls && reserveTool) {
+ if (item.tool_calls && reserveTool) {
  // save tool calls
  const toolCalls = item.tool_calls as ChatCompletionMessageToolCall[];
  value.push({
@@ -278,6 +297,18 @@ export const GPTMessages2Chats = (
  type: ChatItemValueTypeEnum.interactive,
  interactive: item.interactive
  });
+ } else if (typeof item.content === 'string') {
+ const lastValue = value[value.length - 1];
+ if (lastValue && lastValue.type === ChatItemValueTypeEnum.text && lastValue.text) {
+ lastValue.text.content += item.content;
+ } else {
+ value.push({
+ type: ChatItemValueTypeEnum.text,
+ text: {
+ content: item.content
+ }
+ });
+ }
  }
  }
 

packages/global/core/chat/type.d.ts

@@ -15,7 +15,7 @@ import type { AppSchema as AppType } from '@fastgpt/global/core/app/type.d';
 import { DatasetSearchModeEnum } from '../dataset/constants';
 import { DispatchNodeResponseType } from '../workflow/runtime/type.d';
 import { ChatBoxInputType } from '../../../../projects/app/src/components/core/chat/ChatContainer/ChatBox/type';
-import { InteractiveNodeResponseItemType } from '../workflow/template/system/interactive/type';
+import { WorkflowInteractiveResponseType } from '../workflow/template/system/interactive/type';
 
 export type ChatSchema = {
  _id: string;
@@ -73,7 +73,7 @@ export type AIChatItemValueItemType = {
  content: string;
  };
  tools?: ToolModuleResponseItemType[];
- interactive?: InteractiveNodeResponseItemType;
+ interactive?: WorkflowInteractiveResponseType;
 };
 export type AIChatItemType = {
  obj: ChatRoleEnum.AI;

packages/global/core/chat/utils.ts

@@ -143,3 +143,29 @@ export const getChatSourceByPublishChannel = (publishChannel: PublishChannelEnum
  return ChatSourceEnum.online;
  }
 };
+
+/* 
+ Merge chat responseData
+ 1. Same tool mergeSignId (Interactive tool node)
+*/
+export const mergeChatResponseData = (responseDataList: ChatHistoryItemResType[]) => {
+ let lastResponse: ChatHistoryItemResType | undefined = undefined;
+
+ return responseDataList.reduce<ChatHistoryItemResType[]>((acc, curr) => {
+ if (lastResponse && lastResponse.mergeSignId && curr.mergeSignId === lastResponse.mergeSignId) {
+ // 替换 lastResponse
+ const concatResponse: ChatHistoryItemResType = {
+ ...curr,
+ runningTime: +((lastResponse.runningTime || 0) + (curr.runningTime || 0)).toFixed(2),
+ totalPoints: (lastResponse.totalPoints || 0) + (curr.totalPoints || 0),
+ childTotalPoints: (lastResponse.childTotalPoints || 0) + (curr.childTotalPoints || 0),
+ toolCallTokens: (lastResponse.toolCallTokens || 0) + (curr.toolCallTokens || 0),
+ toolDetail: [...(lastResponse.toolDetail || []), ...(curr.toolDetail || [])]
+ };
+ return [...acc.slice(0, -1), concatResponse];
+ } else {
+ lastResponse = curr;
+ return [...acc, curr];
+ }
+ }, []);
+};

packages/global/core/workflow/runtime/type.d.ts

@@ -73,7 +73,7 @@ export type RuntimeNodeItemType = {
  intro?: StoreNodeItemType['intro'];
  flowNodeType: StoreNodeItemType['flowNodeType'];
  showStatus?: StoreNodeItemType['showStatus'];
- isEntry?: StoreNodeItemType['isEntry'];
+ isEntry?: boolean;
 
  inputs: FlowNodeInputItemType[];
  outputs: FlowNodeOutputItemType[];
@@ -108,12 +108,14 @@ export type DispatchNodeResponseType = {
  customOutputs?: Record<string, any>;
  nodeInputs?: Record<string, any>;
  nodeOutputs?: Record<string, any>;
+ mergeSignId?: string;
 
  // bill
  tokens?: number;
  model?: string;
  contextTotalLen?: number;
  totalPoints?: number;
+ childTotalPoints?: number;
 
  // chat
  temperature?: number;

packages/global/core/workflow/runtime/utils.ts

@@ -69,7 +69,7 @@ export const initWorkflowEdgeStatus = (
  histories?: ChatItemType[]
 ): RuntimeEdgeItemType[] => {
  // If there is a history, use the last interactive value
- if (!!histories) {
+ if (histories && histories.length > 0) {
  const memoryEdges = getLastInteractiveValue(histories)?.memoryEdges;
 
  if (memoryEdges && memoryEdges.length > 0) {
@@ -90,7 +90,7 @@ export const getWorkflowEntryNodeIds = (
  histories?: ChatItemType[]
 ) => {
  // If there is a history, use the last interactive entry node
- if (!!histories) {
+ if (histories && histories.length > 0) {
  const entryNodeIds = getLastInteractiveValue(histories)?.entryNodeIds;
 
  if (Array.isArray(entryNodeIds) && entryNodeIds.length > 0) {

packages/global/core/workflow/template/system/interactive/formInput.ts

@@ -22,7 +22,7 @@ export const FormInputNode: FlowNodeTemplateType = {
  avatar: 'core/workflow/template/formInput',
  name: i18nT('app:workflow.form_input'),
  intro: i18nT(`app:workflow.form_input_tip`),
- showStatus: true,
+ isTool: true,
  version: '4811',
  inputs: [
  {

packages/global/core/workflow/template/system/interactive/type.d.ts

@@ -1,8 +1,9 @@
-import { NodeOutputItemType } from '../../../../chat/type';
-import { FlowNodeOutputItemType } from '../../../type/io';
-import { RuntimeEdgeItemType } from '../../../runtime/type';
+import type { NodeOutputItemType } from '../../../../chat/type';
+import type { FlowNodeOutputItemType } from '../../../type/io';
+import type { RuntimeEdgeItemType } from '../../../runtime/type';
 import { FlowNodeInputTypeEnum } from 'core/workflow/node/constant';
 import { WorkflowIOValueTypeEnum } from 'core/workflow/constants';
+import type { ChatCompletionMessageParam } from '../../../../ai/type';
 
 export type UserSelectOptionItemType = {
  key: string;
@@ -32,6 +33,12 @@ type InteractiveBasicType = {
  entryNodeIds: string[];
  memoryEdges: RuntimeEdgeItemType[];
  nodeOutputs: NodeOutputItemType[];
+
+ toolParams?: {
+ entryNodeIds: string[]; // 记录工具中，交互节点的 Id，而不是起始工作流的入口
+ memoryMessages: ChatCompletionMessageParam[]; // 这轮工具中，产生的新的 messages
+ toolCallId: string; // 记录对应 tool 的id，用于后续交互节点可以替换掉 tool 的 response
+ };
 };
 
 type UserSelectInteractive = {
@@ -52,5 +59,5 @@ type UserInputInteractive = {
  };
 };
 
-export type InteractiveNodeResponseItemType = InteractiveBasicType &
- (UserSelectInteractive | UserInputInteractive);
+export type InteractiveNodeResponseType = UserSelectInteractive | UserInputInteractive;
+export type WorkflowInteractiveResponseType = InteractiveBasicType & InteractiveNodeResponseType;

packages/global/core/workflow/template/system/interactive/userSelect.ts

@@ -23,7 +23,7 @@ export const UserSelectNode: FlowNodeTemplateType = {
  diagram: '/imgs/app/userSelect.svg',
  name: i18nT('app:workflow.user_select'),
  intro: i18nT(`app:workflow.user_select_tip`),
- showStatus: true,
+ isTool: true,
  version: '489',
  inputs: [
  {