Merge branch 'main-stream-1204' into dev

# Conflicts: # api/.env.example
kenneth-bro · Dec 4, 2024 · 477f73b · 477f73b
2 parents d14a442 + 1cb5a12
commit 477f73b
Show file tree

Hide file tree

Showing 107 changed files with 2,168 additions and 397 deletions.
diff --git a/api/.env.example b/api/.env.example
@@ -413,8 +413,6 @@ RESET_PASSWORD_TOKEN_EXPIRY_MINUTES=5
 
 CREATE_TIDB_SERVICE_JOB_ENABLED=false
 
-RETRIEVAL_TOP_N=0
-
 
 # Agent最大迭代次数
 AGENT_MAX_ITERATION=100
diff --git a/api/.ruff.toml b/api/.ruff.toml
@@ -20,6 +20,8 @@ select = [
     "PLC0208", # iteration-over-set
     "PLC2801", # unnecessary-dunder-call
     "PLC0414", # useless-import-alias
+    "PLE0604", # invalid-all-object
+    "PLE0605", # invalid-all-format
     "PLR0402", # manual-from-import
     "PLR1711", # useless-return
     "PLR1714", # repeated-equality-comparison
@@ -28,6 +30,7 @@ select = [
     "RUF100", # unused-noqa
     "RUF101", # redirected-noqa
     "RUF200", # invalid-pyproject-toml
+    "RUF022", # unsorted-dunder-all
     "S506", # unsafe-yaml-load
     "SIM", # flake8-simplify rules
     "TRY400", # error-instead-of-exception

diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py
@@ -626,8 +626,6 @@ class DataSetConfig(BaseSettings):
         default=30,
     )
 
-    RETRIEVAL_TOP_N: int = Field(description="number of retrieval top_n", default=0)
-
 
 class WorkspaceConfig(BaseSettings):
     """

diff --git a/api/configs/packaging/__init__.py b/api/configs/packaging/__init__.py
@@ -9,7 +9,7 @@ class PackagingInfo(BaseSettings):
 
     CURRENT_VERSION: str = Field(
         description="Dify version",
-        default="0.12.1",
+        default="0.13.0",
     )
 
     COMMIT_SHA: str = Field(

diff --git a/api/controllers/console/app/workflow.py b/api/controllers/console/app/workflow.py
@@ -100,11 +100,11 @@ def post(self, app_model: App):
         try:
             environment_variables_list = args.get("environment_variables") or []
             environment_variables = [
-                variable_factory.build_variable_from_mapping(obj) for obj in environment_variables_list
+                variable_factory.build_environment_variable_from_mapping(obj) for obj in environment_variables_list
             ]
             conversation_variables_list = args.get("conversation_variables") or []
             conversation_variables = [
-                variable_factory.build_variable_from_mapping(obj) for obj in conversation_variables_list
+                variable_factory.build_conversation_variable_from_mapping(obj) for obj in conversation_variables_list
             ]
             workflow = workflow_service.sync_draft_workflow(
                 app_model=app_model,
@@ -383,7 +383,7 @@ def get(self, app_model: App, block_type: str):
         filters = None
         if args.get("q"):
             try:
-                filters = json.loads(args.get("q"))
+                filters = json.loads(args.get("q", ""))
             except json.JSONDecodeError:
                 raise ValueError("Invalid filters")
 

diff --git a/api/core/app/apps/workflow_app_runner.py b/api/core/app/apps/workflow_app_runner.py
@@ -43,7 +43,7 @@
 )
 from core.workflow.graph_engine.entities.graph import Graph
 from core.workflow.nodes import NodeType
-from core.workflow.nodes.node_mapping import node_type_classes_mapping
+from core.workflow.nodes.node_mapping import NODE_TYPE_CLASSES_MAPPING
 from core.workflow.workflow_entry import WorkflowEntry
 from extensions.ext_database import db
 from models.model import App
@@ -138,7 +138,8 @@ def _get_graph_and_variable_pool_of_single_iteration(
 
         # Get node class
         node_type = NodeType(iteration_node_config.get("data", {}).get("type"))
-        node_cls = node_type_classes_mapping[node_type]
+        node_version = iteration_node_config.get("data", {}).get("version", "1")
+        node_cls = NODE_TYPE_CLASSES_MAPPING[node_type][node_version]
 
         # init variable pool
         variable_pool = VariablePool(

diff --git a/api/core/file/__init__.py b/api/core/file/__init__.py
@@ -7,13 +7,13 @@
 )
 
 __all__ = [
+    "FILE_MODEL_IDENTITY",
+    "ArrayFileAttribute",
+    "File",
+    "FileAttribute",
+    "FileBelongsTo",
+    "FileTransferMethod",
     "FileType",
     "FileUploadConfig",
-    "FileTransferMethod",
-    "FileBelongsTo",
-    "File",
     "ImageConfig",
-    "FileAttribute",
-    "ArrayFileAttribute",
-    "FILE_MODEL_IDENTITY",
 ]
diff --git a/api/core/model_runtime/entities/__init__.py b/api/core/model_runtime/entities/__init__.py
@@ -18,25 +18,25 @@
 from .model_entities import ModelPropertyKey
 
 __all__ = [
+    "AssistantPromptMessage",
+    "AudioPromptMessageContent",
+    "DocumentPromptMessageContent",
     "ImagePromptMessageContent",
-    "VideoPromptMessageContent",
-    "PromptMessage",
-    "PromptMessageRole",
+    "LLMResult",
+    "LLMResultChunk",
+    "LLMResultChunkDelta",
     "LLMUsage",
     "ModelPropertyKey",
-    "AssistantPromptMessage",
+    "PromptMessage",
     "PromptMessage",
     "PromptMessageContent",
+    "PromptMessageContentType",
     "PromptMessageRole",
+    "PromptMessageRole",
+    "PromptMessageTool",
     "SystemPromptMessage",
     "TextPromptMessageContent",
-    "UserPromptMessage",
-    "PromptMessageTool",
     "ToolPromptMessage",
-    "PromptMessageContentType",
-    "LLMResult",
-    "LLMResultChunk",
-    "LLMResultChunkDelta",
-    "AudioPromptMessageContent",
-    "DocumentPromptMessageContent",
+    "UserPromptMessage",
+    "VideoPromptMessageContent",
 ]
diff --git a/api/core/model_runtime/model_providers/volcengine_maas/legacy/volc_sdk/__init__.py b/api/core/model_runtime/model_providers/volcengine_maas/legacy/volc_sdk/__init__.py
@@ -1,4 +1,4 @@
 from .common import ChatRole
 from .maas import MaasError, MaasService
 
-__all__ = ["MaasService", "ChatRole", "MaasError"]
+__all__ = ["ChatRole", "MaasError", "MaasService"]
diff --git a/api/core/model_runtime/model_providers/wenxin/rerank/rerank.py b/api/core/model_runtime/model_providers/wenxin/rerank/rerank.py
@@ -17,15 +17,25 @@ class WenxinRerank(_CommonWenxin):
     def rerank(self, model: str, query: str, docs: list[str], top_n: Optional[int] = None):
         access_token = self._get_access_token()
         url = f"{self.api_bases[model]}?access_token={access_token}"
-
+        # For issue #11252
+        # for wenxin Rerank model top_n length should be equal or less than docs length
+        if top_n is not None and top_n > len(docs):
+            top_n = len(docs)
+        # for wenxin Rerank model, query should not be an empty string
+        if query == "":
+            query = " "  # FIXME: this is a workaround for wenxin rerank model for better user experience.
         try:
             response = httpx.post(
                 url,
                 json={"model": model, "query": query, "documents": docs, "top_n": top_n},
                 headers={"Content-Type": "application/json"},
             )
             response.raise_for_status()
-            return response.json()
+            data = response.json()
+            # wenxin error handling
+            if "error_code" in data:
+                raise InternalServerError(data["error_msg"])
+            return data
         except httpx.HTTPStatusError as e:
             raise InternalServerError(str(e))
 
@@ -69,6 +79,9 @@ def _invoke(
             results = wenxin_rerank.rerank(model, query, docs, top_n)
 
             rerank_documents = []
+            if "results" not in results:
+                raise ValueError("results key not found in response")
+
             for result in results["results"]:
                 index = result["index"]
                 if "document" in result:

diff --git a/api/core/rag/datasource/retrieval_service.py b/api/core/rag/datasource/retrieval_service.py
@@ -3,7 +3,6 @@
 
 from flask import Flask, current_app
 
-from configs import DifyConfig
 from core.rag.data_post_processor.data_post_processor import DataPostProcessor
 from core.rag.datasource.keyword.keyword_factory import Keyword
 from core.rag.datasource.vdb.vector_factory import Vector
@@ -114,7 +113,7 @@ def retrieve(
                 query=query,
                 documents=all_documents,
                 score_threshold=score_threshold,
-                top_n=DifyConfig.RETRIEVAL_TOP_N or top_k,
+                top_n=top_k,
             )
 
         return all_documents
@@ -186,7 +185,7 @@ def embedding_search(
                                 query=query,
                                 documents=documents,
                                 score_threshold=score_threshold,
-                                top_n=DifyConfig.RETRIEVAL_TOP_N or len(documents),
+                                top_n=len(documents),
                             )
                         )
                     else:
@@ -231,7 +230,7 @@ def full_text_index_search(
                                 query=query,
                                 documents=documents,
                                 score_threshold=score_threshold,
-                                top_n=DifyConfig.RETRIEVAL_TOP_N or len(documents),
+                                top_n=len(documents),
                             )
                         )
                     else:

diff --git a/api/core/variables/__init__.py b/api/core/variables/__init__.py
@@ -32,32 +32,32 @@
 )
 
 __all__ = [
-    "IntegerVariable",
-    "FloatVariable",
-    "ObjectVariable",
-    "SecretVariable",
-    "StringVariable",
-    "ArrayAnyVariable",
-    "Variable",
-    "SegmentType",
-    "SegmentGroup",
-    "Segment",
-    "NoneSegment",
-    "NoneVariable",
-    "IntegerSegment",
-    "FloatSegment",
-    "ObjectSegment",
     "ArrayAnySegment",
-    "StringSegment",
-    "ArrayStringVariable",
-    "ArrayNumberVariable",
-    "ArrayObjectVariable",
-    "ArraySegment",
+    "ArrayAnyVariable",
     "ArrayFileSegment",
+    "ArrayFileVariable",
     "ArrayNumberSegment",
+    "ArrayNumberVariable",
     "ArrayObjectSegment",
+    "ArrayObjectVariable",
+    "ArraySegment",
     "ArrayStringSegment",
+    "ArrayStringVariable",
     "FileSegment",
     "FileVariable",
-    "ArrayFileVariable",
+    "FloatSegment",
+    "FloatVariable",
+    "IntegerSegment",
+    "IntegerVariable",
+    "NoneSegment",
+    "NoneVariable",
+    "ObjectSegment",
+    "ObjectVariable",
+    "SecretVariable",
+    "Segment",
+    "SegmentGroup",
+    "SegmentType",
+    "StringSegment",
+    "StringVariable",
+    "Variable",
 ]
diff --git a/api/core/variables/types.py b/api/core/variables/types.py
@@ -2,16 +2,19 @@
 
 
 class SegmentType(StrEnum):
-    NONE = "none"
     NUMBER = "number"
     STRING = "string"
+    OBJECT = "object"
     SECRET = "secret"
+
+    FILE = "file"
+
     ARRAY_ANY = "array[any]"
     ARRAY_STRING = "array[string]"
     ARRAY_NUMBER = "array[number]"
     ARRAY_OBJECT = "array[object]"
-    OBJECT = "object"
-    FILE = "file"
     ARRAY_FILE = "array[file]"
 
+    NONE = "none"
+
     GROUP = "group"
diff --git a/api/core/workflow/callbacks/__init__.py b/api/core/workflow/callbacks/__init__.py
@@ -2,6 +2,6 @@
 from .workflow_logging_callback import WorkflowLoggingCallback
 
 __all__ = [
-    "WorkflowLoggingCallback",
     "WorkflowCallback",
+    "WorkflowLoggingCallback",
 ]
diff --git a/api/core/workflow/graph_engine/graph_engine.py b/api/core/workflow/graph_engine/graph_engine.py
@@ -38,7 +38,7 @@
 from core.workflow.nodes.base import BaseNode
 from core.workflow.nodes.end.end_stream_processor import EndStreamProcessor
 from core.workflow.nodes.event import RunCompletedEvent, RunRetrieverResourceEvent, RunStreamChunkEvent
-from core.workflow.nodes.node_mapping import node_type_classes_mapping
+from core.workflow.nodes.node_mapping import NODE_TYPE_CLASSES_MAPPING
 from extensions.ext_database import db
 from models.enums import UserFrom
 from models.workflow import WorkflowNodeExecutionStatus, WorkflowType
@@ -227,7 +227,8 @@ def _run(
 
             # convert to specific node
             node_type = NodeType(node_config.get("data", {}).get("type"))
-            node_cls = node_type_classes_mapping[node_type]
+            node_version = node_config.get("data", {}).get("version", "1")
+            node_cls = NODE_TYPE_CLASSES_MAPPING[node_type][node_version]
 
             previous_node_id = previous_route_node_state.node_id if previous_route_node_state else None
 

diff --git a/api/core/workflow/nodes/answer/__init__.py b/api/core/workflow/nodes/answer/__init__.py
@@ -1,4 +1,4 @@
 from .answer_node import AnswerNode
 from .entities import AnswerStreamGenerateRoute
 
-__all__ = ["AnswerStreamGenerateRoute", "AnswerNode"]
+__all__ = ["AnswerNode", "AnswerStreamGenerateRoute"]
diff --git a/api/core/workflow/nodes/answer/answer_stream_generate_router.py b/api/core/workflow/nodes/answer/answer_stream_generate_router.py
@@ -153,7 +153,7 @@ def _recursive_fetch_answer_dependencies(
                 NodeType.IF_ELSE,
                 NodeType.QUESTION_CLASSIFIER,
                 NodeType.ITERATION,
-                NodeType.CONVERSATION_VARIABLE_ASSIGNER,
+                NodeType.VARIABLE_ASSIGNER,
             }:
                 answer_dependencies[answer_node_id].append(source_node_id)
             else:

diff --git a/api/core/workflow/nodes/base/__init__.py b/api/core/workflow/nodes/base/__init__.py
@@ -1,4 +1,4 @@
 from .entities import BaseIterationNodeData, BaseIterationState, BaseNodeData
 from .node import BaseNode
 
-__all__ = ["BaseNode", "BaseNodeData", "BaseIterationNodeData", "BaseIterationState"]
+__all__ = ["BaseIterationNodeData", "BaseIterationState", "BaseNode", "BaseNodeData"]
diff --git a/api/core/workflow/nodes/base/entities.py b/api/core/workflow/nodes/base/entities.py
@@ -7,6 +7,7 @@
 class BaseNodeData(ABC, BaseModel):
     title: str
     desc: Optional[str] = None
+    version: str = "1"
 
 
 class BaseIterationNodeData(BaseNodeData):

diff --git a/api/core/workflow/nodes/base/node.py b/api/core/workflow/nodes/base/node.py
@@ -55,7 +55,9 @@ def __init__(
             raise ValueError("Node ID is required.")
 
         self.node_id = node_id
-        self.node_data: GenericNodeData = cast(GenericNodeData, self._node_data_cls(**config.get("data", {})))
+
+        node_data = self._node_data_cls.model_validate(config.get("data", {}))
+        self.node_data = cast(GenericNodeData, node_data)
 
     @abstractmethod
     def _run(self) -> NodeRunResult | Generator[Union[NodeEvent, "InNodeEvent"], None, None]:

diff --git a/api/core/workflow/nodes/end/__init__.py b/api/core/workflow/nodes/end/__init__.py
@@ -1,4 +1,4 @@
 from .end_node import EndNode
 from .entities import EndStreamParam
 
-__all__ = ["EndStreamParam", "EndNode"]
+__all__ = ["EndNode", "EndStreamParam"]
diff --git a/api/core/workflow/nodes/enums.py b/api/core/workflow/nodes/enums.py
@@ -14,11 +14,11 @@ class NodeType(StrEnum):
     HTTP_REQUEST = "http-request"
     TOOL = "tool"
     VARIABLE_AGGREGATOR = "variable-aggregator"
-    VARIABLE_ASSIGNER = "variable-assigner"  # TODO: Merge this into VARIABLE_AGGREGATOR in the database.
+    LEGACY_VARIABLE_AGGREGATOR = "variable-assigner"  # TODO: Merge this into VARIABLE_AGGREGATOR in the database.
     LOOP = "loop"
     ITERATION = "iteration"
     ITERATION_START = "iteration-start"  # Fake start node for iteration.
     PARAMETER_EXTRACTOR = "parameter-extractor"
-    CONVERSATION_VARIABLE_ASSIGNER = "assigner"
+    VARIABLE_ASSIGNER = "assigner"
     DOCUMENT_EXTRACTOR = "document-extractor"
     LIST_OPERATOR = "list-operator"
diff --git a/api/core/workflow/nodes/event/__init__.py b/api/core/workflow/nodes/event/__init__.py
@@ -2,9 +2,9 @@
 from .types import NodeEvent
 
 __all__ = [
+    "ModelInvokeCompletedEvent",
+    "NodeEvent",
     "RunCompletedEvent",
     "RunRetrieverResourceEvent",
     "RunStreamChunkEvent",
-    "NodeEvent",
-    "ModelInvokeCompletedEvent",
 ]
-Original file line number
+Diff line change
@@ Expand Up / @@ -626,8 +626,6 @@ class DataSetConfig(BaseSettings): @@
             default=30,
         )
-        RETRIEVAL_TOP_N: int = Field(description="number of retrieval top_n", default=0)
     class WorkspaceConfig(BaseSettings):
         """
@@ Expand Down @@