Add laf module and timer lock (#71)

* timer lock * update iframe.js * fix: timer lock and http histories * fix: tool module connection type * file icon * doc * fix: index decod * update laf module * doc * laf env config
c121914yu · Apr 3, 2024 · 44d3bb9 · 44d3bb9
1 parent 03f12ae
commit 44d3bb9
Show file tree

Hide file tree

Showing 52 changed files with 2,531 additions and 2,523 deletions.
diff --git a/README.md b/README.md
@@ -113,7 +113,7 @@ fastgpt.run 域名会弃用。
 * [多模型配置](https://doc.fastgpt.in/docs/development/one-api/)
 * [版本更新/升级介绍](https://doc.fastgpt.in/docs/development/upgrading)
 * [OpenAPI API 文档](https://doc.fastgpt.in/docs/development/openapi/)
-* [知识库结构详解](https://doc.fastgpt.in/docs/course/datasetengine/)
+* [知识库结构详解](https://doc.fastgpt.in/docs/course/dataset_engine/)
 
 <a href="#readme">
  <img src="https://img.shields.io/badge/-返回顶部-7d09f1.svg" alt="#" align="right">

diff --git a/docSite/content/docs/course/ai_settings.md b/docSite/content/docs/course/ai_settings.md
@@ -64,7 +64,7 @@ Tips: 可以通过点击上下文按键查看完整的上下文组成，便于
 
 FastGPT 知识库采用 QA 对(不一定都是问答格式，仅代表两个变量)的格式存储，在转义成字符串时候会根据**引用模板**来进行格式化。知识库包含多个可用变量： q, a, sourceId（数据的ID）, index(第n个数据), source(数据的集合名、文件名)，score(距离得分，0-1) 可以通过 {{q}} {{a}} {{sourceId}} {{index}} {{source}} {{score}} 按需引入。下面一个模板例子：
 
-可以通过 [知识库结构讲解](/docs/course/datasetEngine/) 了解详细的知识库的结构。
+可以通过 [知识库结构讲解](/docs/course/dataset_engine/) 了解详细的知识库的结构。
 
 #### 引用模板
 

diff --git a/docSite/content/docs/course/datasetEngine.md → ...ite/content/docs/course/dataset_engine.md b/docSite/content/docs/course/datasetEngine.md → ...ite/content/docs/course/dataset_engine.md
diff --git a/docSite/content/docs/development/configuration.md b/docSite/content/docs/development/configuration.md
@@ -19,6 +19,9 @@ llm模型全部合并
 
 ```json
 {
+ "feConfigs": {
+ "lafEnv": "https://laf.dev" // laf环境
+ },
  "systemEnv": {
  "vectorMaxProcess": 15,
  "qaMaxProcess": 15,
@@ -164,7 +167,7 @@ llm模型全部合并
  "model": "bge-reranker-base", // 随意
  "name": "检索重排-base", // 随意
  "charsPointsPrice": 0,
- "requestUrl": "{{host}}/api/v1/rerank",
+ "requestUrl": "{{host}}/v1/rerank",
  "requestAuth": "安全凭证，已自动补 Bearer"
  }
  ]

diff --git a/docSite/content/docs/development/custom-models/reranker.md b/docSite/content/docs/development/custom-models/reranker.md
@@ -44,7 +44,7 @@ weight: 910
 
 ### docker 部署
 
-+ 镜像名: `luanshaotong/reranker:v0.2` 
++ 镜像名: `registry.cn-hangzhou.aliyuncs.com/fastgpt/rerank:v0.2` 
 + 端口号: 6006
 + 大小：约8GB
 
@@ -56,20 +56,20 @@ ACCESS_TOKEN=mytoken
 **运行命令示例**
 - 无需GPU环境，使用CPU运行
 ```sh
-docker run -d --name reranker -p 6006:6006 -e ACCESS_TOKEN=mytoken luanshaotong/reranker:v0.2
+docker run -d --name reranker -p 6006:6006 -e ACCESS_TOKEN=mytoken registry.cn-hangzhou.aliyuncs.com/fastgpt/rerank:v0.2
 ```
 
 - 需要CUDA 11.7环境
 ```sh
-docker run -d --gpus all --name reranker -p 6006:6006 -e ACCESS_TOKEN=mytoken luanshaotong/reranker:v0.2
+docker run -d --gpus all --name reranker -p 6006:6006 -e ACCESS_TOKEN=mytoken registry.cn-hangzhou.aliyuncs.com/fastgpt/rerank:v0.2
 ```
 
 **docker-compose.yml示例**
 ```
 version: "3"
 services:
  reranker:
- image: luanshaotong/reranker:v0.2
+ image: registry.cn-hangzhou.aliyuncs.com/fastgpt/rerank:v0.2
  container_name: reranker
  # GPU运行环境，如果宿主机未安装，将deploy配置隐藏即可
  deploy:

diff --git a/docSite/content/docs/development/upgrading/47.md b/docSite/content/docs/development/upgrading/47.md
@@ -26,7 +26,7 @@ curl --location --request POST 'https://{{host}}/api/admin/initv47' \
 
 ## 3. 升级 ReRank 模型
 
-4.7对ReRank模型进行了格式变动，兼容 cohere 的格式，可以直接使用 cohere 提供的 API。如果是本地的 ReRank 模型，需要修改镜像为：`luanshaotong/reranker:v0.2` 。
+4.7对ReRank模型进行了格式变动，兼容 cohere 的格式，可以直接使用 cohere 提供的 API。如果是本地的 ReRank 模型，需要修改镜像为：`registry.cn-hangzhou.aliyuncs.com/fastgpt/rerank:v0.2` 。
 
 cohere的重排模型对中文不是很好，感觉不如 bge 的好用，接入教程如下：
 

diff --git a/docSite/content/docs/development/upgrading/471.md b/docSite/content/docs/development/upgrading/471.md
@@ -4,10 +4,25 @@ description: 'FastGPT V4.7.1 更新说明'
 icon: 'upgrade'
 draft: false
 toc: true
-weight: 826
+weight: 825
 ---
 
-## V4.7 更新说明
+## 初始化脚本
+
+从任意终端，发起 1 个 HTTP 请求。其中 {{rootkey}} 替换成环境变量里的 `rootkey`；{{host}} 替换成FastGPT的域名。
+
+```bash
+curl --location --request POST 'https://{{host}}/api/admin/clearInvalidData' \
+--header 'rootkey: {{rootkey}}' \
+--header 'Content-Type: application/json'
+```
+
+该请求会执行脏数据清理（清理无效的文件、清理无效的图片、清理无效的知识库集合、清理无效的向量）
+
+## V4.7.1 更新说明
 
 1. 新增 - Pptx 和 xlsx 文件读取。但所有文件读取都放服务端，会消耗更多的服务器资源，以及无法在上传时预览更多内容。
-2. 修改 - csv导入模板，取消 header 校验，自动获取前两列。
+2. 新增 - 集成 Laf 云函数，可以读取 Laf 账号中的云函数作为 HTTP 模块。
+3. 新增 - 定时器，清理垃圾数据。（采用小范围清理，会清理最近n个小时的，所以请保证服务持续运行，长时间不允许，可以继续执行 clearInvalidData 的接口进行全量清理。）
+4. 修改 - csv导入模板，取消 header 校验，自动获取前两列。
+5. 修复 - 工具调用模块连线数据类型校验错误。
diff --git a/packages/global/common/file/icon.ts b/packages/global/common/file/icon.ts
@@ -1,5 +1,7 @@
 export const fileImgs = [
  { suffix: 'pdf', src: 'file/fill/pdf' },
+ { suffix: 'ppt', src: 'file/fill/ppt' },
+ { suffix: 'xlsx', src: 'file/fill/xlsx' },
  { suffix: 'csv', src: 'file/fill/csv' },
  { suffix: '(doc|docs)', src: 'file/fill/doc' },
  { suffix: 'txt', src: 'file/fill/txt' },

diff --git a/packages/global/core/module/template/system/laf.ts b/packages/global/core/module/template/system/laf.ts
@@ -15,29 +15,19 @@ import {
  Input_Template_Switch,
  Input_Template_AddInputParam
 } from '../input';
-import { Output_Template_Finish } from '../output';
+import { Output_Template_Finish, Output_Template_AddOutput } from '../output';
 
 export const lafModule: FlowNodeTemplateType = {
  id: FlowNodeTypeEnum.lafModule,
  templateType: FlowNodeTemplateTypeEnum.externalCall,
  flowType: FlowNodeTypeEnum.lafModule,
  avatar: '/imgs/module/laf.png',
- name: 'laf 函数调用',
- intro: '可以请求 laf 函数，实现更为复杂的操作（联网搜索、数据库查询等）',
+ name: 'Laf 函数调用（测试）',
+ intro: '可以调用Laf账号下的云函数。',
  showStatus: true,
  isTool: true,
  inputs: [
  Input_Template_Switch,
- {
- key: ModuleInputKeyEnum.httpMethod,
- type: FlowNodeInputTypeEnum.custom,
- valueType: ModuleIOValueTypeEnum.string,
- label: '',
- value: 'POST',
- required: true,
- showTargetInApp: false,
- showTargetInPlugin: false
- },
  {
  key: ModuleInputKeyEnum.httpReqUrl,
  type: FlowNodeInputTypeEnum.hidden,
@@ -49,38 +39,6 @@ export const lafModule: FlowNodeTemplateType = {
  showTargetInApp: false,
  showTargetInPlugin: false
  },
- {
- key: ModuleInputKeyEnum.httpHeaders,
- type: FlowNodeInputTypeEnum.custom,
- valueType: ModuleIOValueTypeEnum.any,
- value: [],
- label: '',
- description: 'core.module.input.description.Http Request Header',
- placeholder: 'core.module.input.description.Http Request Header',
- required: false,
- showTargetInApp: false,
- showTargetInPlugin: false
- },
- {
- key: ModuleInputKeyEnum.httpParams,
- type: FlowNodeInputTypeEnum.hidden,
- valueType: ModuleIOValueTypeEnum.any,
- value: [],
- label: '',
- required: false,
- showTargetInApp: false,
- showTargetInPlugin: false
- },
- {
- key: ModuleInputKeyEnum.httpJsonBody,
- type: FlowNodeInputTypeEnum.hidden,
- valueType: ModuleIOValueTypeEnum.any,
- value: '',
- label: '',
- required: false,
- showTargetInApp: false,
- showTargetInPlugin: false
- },
  Input_Template_DynamicInput,
  {
  ...Input_Template_AddInputParam,
@@ -101,12 +59,28 @@ export const lafModule: FlowNodeTemplateType = {
  outputs: [
  {
  key: ModuleOutputKeyEnum.httpRawResponse,
- label: 'result',
- description: 'laf 函数请求的原始响应。只能接受字符串或JSON类型响应数据。',
+ label: '原始响应',
+ description: 'HTTP请求的原始响应。只能接受字符串或JSON类型响应数据。',
  valueType: ModuleIOValueTypeEnum.any,
  type: FlowNodeOutputTypeEnum.source,
  targets: []
  },
+ {
+ ...Output_Template_AddOutput,
+ editField: {
+ key: true,
+ description: true,
+ dataType: true,
+ defaultValue: true
+ },
+ defaultEditField: {
+ label: '',
+ key: '',
+ description: '',
+ outputType: FlowNodeOutputTypeEnum.source,
+ valueType: ModuleIOValueTypeEnum.string
+ }
+ },
  Output_Template_Finish
  ]
 };
diff --git a/packages/global/core/module/type.d.ts b/packages/global/core/module/type.d.ts
@@ -9,6 +9,7 @@ import { DispatchNodeResponseKeyEnum } from './runtime/constants';
 import { FlowNodeInputItemType, FlowNodeOutputItemType } from './node/type';
 import { UserModelSchema } from 'support/user/type';
 import {
+ ChatItemType,
  ChatItemValueItemType,
  ToolRunResponseItemType,
  UserChatItemValueItemType

diff --git a/packages/global/support/user/team/controller.d.ts b/packages/global/support/user/team/controller.d.ts
@@ -1,5 +1,5 @@
 import { TeamMemberRoleEnum } from './constant';
-import { TeamMemberSchema } from './type';
+import { LafAccountType, TeamMemberSchema } from './type';
 
 export type AuthTeamRoleProps = {
  teamId: string;
@@ -10,18 +10,14 @@ export type CreateTeamProps = {
  name: string;
  avatar?: string;
  defaultTeam?: boolean;
- lafAccount?: lafAccount;
+ lafAccount?: LafAccountType;
 };
 export type UpdateTeamProps = {
  teamId: string;
  name?: string;
  avatar?: string;
  teamDomain?: string;
- lafAccount?: lafAccount;
-};
-export type lafAccount = {
- token: string;
- appid: string;
+ lafAccount?: null | LafAccountType;
 };
 
 /* ------------- member ----------- */

diff --git a/packages/global/support/user/team/type.d.ts b/packages/global/support/user/team/type.d.ts
@@ -1,6 +1,6 @@
 import type { UserModelSchema } from '../type';
 import type { TeamMemberRoleEnum, TeamMemberStatusEnum } from './constant';
-import { lafAccount } from './controller';
+import { LafAccountType } from './type';
 
 export type TeamSchema = {
  _id: string;
@@ -14,7 +14,7 @@ export type TeamSchema = {
  lastExportDatasetTime: Date;
  lastWebsiteSyncTime: Date;
  };
- lafAccount: lafAccount;
+ lafAccount: LafAccountType;
 };
 export type tagsType = {
  label: string;
@@ -60,7 +60,7 @@ export type TeamItemType = {
  role: `${TeamMemberRoleEnum}`;
  status: `${TeamMemberStatusEnum}`;
  canWrite: boolean;
- lafAccount: lafAccount;
+ lafAccount?: LafAccountType;
 };
 
 export type TeamMemberItemType = {
@@ -77,3 +77,8 @@ export type TeamTagItemType = {
  label: string;
  key: string;
 };
+
+export type LafAccountType = {
+ token: string;
+ appid: string;
+};
diff --git a/packages/service/common/file/gridfs/controller.ts b/packages/service/common/file/gridfs/controller.ts
@@ -205,7 +205,7 @@ export const readFileContentFromMongo = async ({
  });
 
  if (rawText.trim()) {
- await MongoRwaTextBuffer.create({
+ MongoRwaTextBuffer.create({
  sourceId: fileId,
  rawText,
  metadata: {

diff --git a/packages/service/common/file/utils.ts b/packages/service/common/file/utils.ts
@@ -55,8 +55,8 @@ export const clearTmpUploadFiles = () => {
  fs.stat(filePath, (err, stats) => {
  if (err) return;
 
- // 如果文件是在1小时前上传的，则认为是临时文件并删除它
- if (Date.now() - stats.mtime.getTime() > 1 * 60 * 60 * 1000) {
+ // 如果文件是在2小时前上传的，则认为是临时文件并删除它
+ if (Date.now() - stats.mtime.getTime() > 2 * 60 * 60 * 1000) {
  fs.unlink(filePath, (err) => {
  if (err) return;
  console.log(`Deleted temp file: ${filePath}`);

diff --git a/packages/service/common/system/timerLock/constants.ts b/packages/service/common/system/timerLock/constants.ts
@@ -0,0 +1,15 @@
+export enum TimerIdEnum {
+ checkInValidDatasetFiles = 'checkInValidDatasetFiles',
+ checkInvalidDatasetData = 'checkInvalidDatasetData',
+ checkInvalidVector = 'checkInvalidVector',
+ clearExpiredSubPlan = 'clearExpiredSubPlan',
+ updateStandardPlan = 'updateStandardPlan'
+}
+
+export const timerIdMap = {
+ [TimerIdEnum.checkInValidDatasetFiles]: 'checkInValidDatasetFiles',
+ [TimerIdEnum.checkInvalidDatasetData]: 'checkInvalidDatasetData',
+ [TimerIdEnum.checkInvalidVector]: 'checkInvalidVector',
+ [TimerIdEnum.clearExpiredSubPlan]: 'clearExpiredSubPlan',
+ [TimerIdEnum.updateStandardPlan]: 'updateStandardPlan'
+};
diff --git a/packages/service/common/system/timerLock/schema.ts b/packages/service/common/system/timerLock/schema.ts
@@ -0,0 +1,29 @@
+import { connectionMongo, type Model } from '../../mongo';
+import { timerIdMap } from './constants';
+const { Schema, model, models } = connectionMongo;
+import { TimerLockSchemaType } from './type.d';
+
+export const collectionName = 'systemtimerlocks';
+
+const TimerLockSchema = new Schema({
+ timerId: {
+ type: String,
+ required: true,
+ unique: true,
+ enum: Object.keys(timerIdMap)
+ },
+ expiredTime: {
+ type: Date,
+ required: true
+ }
+});
+
+try {
+ TimerLockSchema.index({ expiredTime: 1 }, { expireAfterSeconds: 5 });
+} catch (error) {
+ console.log(error);
+}
+
+export const MongoTimerLock: Model<TimerLockSchemaType> =
+ models[collectionName] || model(collectionName, TimerLockSchema);
+MongoTimerLock.syncIndexes();
diff --git a/packages/service/common/system/timerLock/type.d.ts b/packages/service/common/system/timerLock/type.d.ts
@@ -0,0 +1,5 @@
+export type TimerLockSchemaType = {
+ _id: string;
+ timerId: string;
+ expiredTime: Date;
+};
diff --git a/packages/service/common/system/timerLock/utils.ts b/packages/service/common/system/timerLock/utils.ts
@@ -0,0 +1,25 @@
+import { TimerIdEnum } from './constants';
+import { MongoTimerLock } from './schema';
+import { addMinutes } from 'date-fns';
+
+/* 
+ 利用唯一健，使得同一时间只有一个任务在执行，后创建的锁，会因唯一健创建失败，从而无法继续执行任务
+*/
+export const checkTimerLock = async ({
+ timerId,
+ lockMinuted
+}: {
+ timerId: `${TimerIdEnum}`;
+ lockMinuted: number;
+}) => {
+ try {
+ await MongoTimerLock.create({
+ timerId,
+ expiredTime: addMinutes(new Date(), lockMinuted)
+ });
+
+ return true;
+ } catch (error) {
+ return false;
+ }
+};
diff --git a/packages/service/core/workflow/dispatch/tools/http468.ts b/packages/service/core/workflow/dispatch/tools/http468.ts
@@ -61,7 +61,7 @@ export const dispatchHttp468Request = async (props: HttpRequestProps): Promise<H
  chatId,
  responseChatItemId,
  ...variables,
- histories: histories.slice(0, 10),
+ histories: histories.slice(-10),
  ...body
  };