refactor(core): implement centralized dynamic lazy import system for components (langflow-ai#8932)

* feat: add import utilities for LangFlow components

- Introduced a new module `_importing.py` containing the `import_mod` function.
- This function dynamically imports attributes from specified modules, enhancing modularity and flexibility in component initialization.
- Comprehensive docstring added for clarity on usage and parameters.

* feat: implement dynamic imports for LangFlow components

- Added dynamic import functionality to various LangFlow components, allowing for lazy loading of attributes on access.
- Introduced  mapping in each component's  to manage imports efficiently.
- Enhanced error handling for import failures, providing clearer messages for missing attributes.
- Updated  method to reflect available attributes for better introspection and tab-completion support.
- Comprehensive docstrings added to improve documentation and usability.

* test: add comprehensive tests for dynamic imports and component accessibility

- Introduced integration tests for dynamic import functionality, ensuring components are discoverable and instantiable post-refactor.
- Added unit tests for the `_import_utils` module, validating the `import_mod` function's behavior and error handling.
- Implemented tests to confirm all component modules are importable and maintain backward compatibility with existing import patterns.
- Enhanced performance tests to measure lazy loading efficiency and memory usage during component access.
- Ensured that all components have the required attributes for dynamic loading and that circular imports are prevented.

* chore: update ruff pre-commit hook to version 0.12.2 in configuration file

* refactor: update warning handling for dynamic imports

- Moved the warning suppression for LangChainDeprecationWarning into the dynamic import context to ensure it only applies during the import process.
- This change enhances clarity and maintains the original functionality while improving the robustness of the import mechanism.

* test: enhance dynamic import integration tests for component attributes

- Removed unnecessary import of AgentComponent and added assertions to verify essential attributes of OpenAIModelComponent, including display_name, description, icon, and inputs.
- Ensured that each input field has the required attributes for better validation of component integrity during dynamic imports.

* refactor: update import paths for Message class in conversation utilities

- Changed the import of the Message class from langflow.field_typing to langflow.schema.message across multiple utility files, ensuring consistency and alignment with the updated module structure.
- This refactor enhances code clarity and maintains compatibility with the latest schema definitions.

* refactor: remove Vectara components from LangFlow

- Deleted the Vectara components module from the codebase, streamlining the component structure.
- This change helps to reduce complexity and maintain focus on core functionalities.

* refactor: remove Vectara references from LangFlow component imports

- Eliminated Vectara from both the import statements and dynamic imports mapping, streamlining the component structure.
- This change contributes to reducing complexity and maintaining focus on core functionalities within the LangFlow framework.

* [autofix.ci] apply automated fixes

* fix: remove 'vectara' from __all__ in components module

* refactor: improve error handling tests for dynamic imports

* test: add tests for ModuleNotFoundError handling with None and special module names

---------

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Edwin Jose <[email protected]>

Author: 3 people

src/backend/base/langflow/components/__init__.py

@@ -0,0 +1,273 @@
+"""LangFlow Components module."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from langflow.components._importing import import_mod
+
+if TYPE_CHECKING:
+    from langflow.components import (
+        Notion,
+        agentql,
+        agents,
+        aiml,
+        amazon,
+        anthropic,
+        apify,
+        arxiv,
+        assemblyai,
+        azure,
+        baidu,
+        bing,
+        cleanlab,
+        cloudflare,
+        cohere,
+        composio,
+        confluence,
+        crewai,
+        custom_component,
+        data,
+        datastax,
+        deepseek,
+        docling,
+        duckduckgo,
+        embeddings,
+        exa,
+        firecrawl,
+        git,
+        glean,
+        google,
+        groq,
+        helpers,
+        homeassistant,
+        huggingface,
+        ibm,
+        icosacomputing,
+        input_output,
+        langchain_utilities,
+        langwatch,
+        lmstudio,
+        logic,
+        maritalk,
+        mem0,
+        mistral,
+        models,
+        needle,
+        notdiamond,
+        novita,
+        nvidia,
+        olivya,
+        ollama,
+        openai,
+        openrouter,
+        perplexity,
+        processing,
+        prototypes,
+        redis,
+        sambanova,
+        scrapegraph,
+        searchapi,
+        serpapi,
+        tavily,
+        tools,
+        twelvelabs,
+        unstructured,
+        vectorstores,
+        vertexai,
+        wikipedia,
+        wolframalpha,
+        xai,
+        yahoosearch,
+        youtube,
+        zep,
+    )
+
+_dynamic_imports = {
+    "agents": "langflow.components.agents",
+    "data": "langflow.components.data",
+    "processing": "langflow.components.processing",
+    "vectorstores": "langflow.components.vectorstores",
+    "tools": "langflow.components.tools",
+    "models": "langflow.components.models",
+    "embeddings": "langflow.components.embeddings",
+    "helpers": "langflow.components.helpers",
+    "input_output": "langflow.components.input_output",
+    "logic": "langflow.components.logic",
+    "custom_component": "langflow.components.custom_component",
+    "prototypes": "langflow.components.prototypes",
+    "openai": "langflow.components.openai",
+    "anthropic": "langflow.components.anthropic",
+    "google": "langflow.components.google",
+    "azure": "langflow.components.azure",
+    "huggingface": "langflow.components.huggingface",
+    "ollama": "langflow.components.ollama",
+    "groq": "langflow.components.groq",
+    "cohere": "langflow.components.cohere",
+    "mistral": "langflow.components.mistral",
+    "deepseek": "langflow.components.deepseek",
+    "nvidia": "langflow.components.nvidia",
+    "amazon": "langflow.components.amazon",
+    "vertexai": "langflow.components.vertexai",
+    "xai": "langflow.components.xai",
+    "perplexity": "langflow.components.perplexity",
+    "openrouter": "langflow.components.openrouter",
+    "lmstudio": "langflow.components.lmstudio",
+    "sambanova": "langflow.components.sambanova",
+    "maritalk": "langflow.components.maritalk",
+    "novita": "langflow.components.novita",
+    "olivya": "langflow.components.olivya",
+    "notdiamond": "langflow.components.notdiamond",
+    "needle": "langflow.components.needle",
+    "cloudflare": "langflow.components.cloudflare",
+    "baidu": "langflow.components.baidu",
+    "aiml": "langflow.components.aiml",
+    "ibm": "langflow.components.ibm",
+    "langchain_utilities": "langflow.components.langchain_utilities",
+    "crewai": "langflow.components.crewai",
+    "composio": "langflow.components.composio",
+    "mem0": "langflow.components.mem0",
+    "datastax": "langflow.components.datastax",
+    "cleanlab": "langflow.components.cleanlab",
+    "langwatch": "langflow.components.langwatch",
+    "icosacomputing": "langflow.components.icosacomputing",
+    "homeassistant": "langflow.components.homeassistant",
+    "agentql": "langflow.components.agentql",
+    "assemblyai": "langflow.components.assemblyai",
+    "twelvelabs": "langflow.components.twelvelabs",
+    "docling": "langflow.components.docling",
+    "unstructured": "langflow.components.unstructured",
+    "redis": "langflow.components.redis",
+    "zep": "langflow.components.zep",
+    "bing": "langflow.components.bing",
+    "duckduckgo": "langflow.components.duckduckgo",
+    "serpapi": "langflow.components.serpapi",
+    "searchapi": "langflow.components.searchapi",
+    "tavily": "langflow.components.tavily",
+    "exa": "langflow.components.exa",
+    "glean": "langflow.components.glean",
+    "yahoosearch": "langflow.components.yahoosearch",
+    "apify": "langflow.components.apify",
+    "arxiv": "langflow.components.arxiv",
+    "confluence": "langflow.components.confluence",
+    "firecrawl": "langflow.components.firecrawl",
+    "git": "langflow.components.git",
+    "wikipedia": "langflow.components.wikipedia",
+    "youtube": "langflow.components.youtube",
+    "scrapegraph": "langflow.components.scrapegraph",
+    "Notion": "langflow.components.Notion",
+    "wolframalpha": "langflow.components.wolframalpha",
+}
+
+__all__: list[str] = [
+    "Notion",
+    "agentql",
+    "agents",
+    "aiml",
+    "amazon",
+    "anthropic",
+    "apify",
+    "arxiv",
+    "assemblyai",
+    "azure",
+    "baidu",
+    "bing",
+    "cleanlab",
+    "cloudflare",
+    "cohere",
+    "composio",
+    "confluence",
+    "crewai",
+    "custom_component",
+    "data",
+    "datastax",
+    "deepseek",
+    "docling",
+    "duckduckgo",
+    "embeddings",
+    "exa",
+    "firecrawl",
+    "git",
+    "glean",
+    "google",
+    "groq",
+    "helpers",
+    "homeassistant",
+    "huggingface",
+    "ibm",
+    "icosacomputing",
+    "input_output",
+    "langchain_utilities",
+    "langwatch",
+    "lmstudio",
+    "logic",
+    "maritalk",
+    "mem0",
+    "mistral",
+    "models",
+    "needle",
+    "notdiamond",
+    "novita",
+    "nvidia",
+    "olivya",
+    "ollama",
+    "openai",
+    "openrouter",
+    "perplexity",
+    "processing",
+    "prototypes",
+    "redis",
+    "sambanova",
+    "scrapegraph",
+    "searchapi",
+    "serpapi",
+    "tavily",
+    "tools",
+    "twelvelabs",
+    "unstructured",
+    "vectorstores",
+    "vertexai",
+    "wikipedia",
+    "wolframalpha",
+    "xai",
+    "yahoosearch",
+    "youtube",
+    "zep",
+]
+
+
+def __getattr__(attr_name: str) -> Any:
+    """Lazily import component modules on attribute access.
+
+    Args:
+        attr_name (str): The attribute/module name to import.
+
+    Returns:
+        Any: The imported module or attribute.
+
+    Raises:
+        AttributeError: If the attribute is not a known component or cannot be imported.
+    """
+    if attr_name not in _dynamic_imports:
+        msg = f"module '{__name__}' has no attribute '{attr_name}'"
+        raise AttributeError(msg)
+    try:
+        # Use import_mod as in LangChain, passing the module name and package
+        result = import_mod(attr_name, "__module__", __spec__.parent)
+    except (ModuleNotFoundError, ImportError, AttributeError) as e:
+        msg = f"Could not import '{attr_name}' from '{__name__}': {e}"
+        raise AttributeError(msg) from e
+    globals()[attr_name] = result  # Cache for future access
+    return result
+
+
+def __dir__() -> list[str]:
+    """Return list of available attributes for tab-completion and dir()."""
+    return list(__all__)
+
+
+# Optional: Consistency check (can be removed in production)
+_missing = set(__all__) - set(_dynamic_imports)
+if _missing:
+    msg = f"Missing dynamic import mapping for: {', '.join(_missing)}"
+    raise ImportError(msg)

src/backend/base/langflow/components/_importing.py

@@ -0,0 +1,37 @@
+"""Import utilities for LangFlow components."""
+
+from __future__ import annotations
+
+from importlib import import_module
+
+
+def import_mod(
+    attr_name: str,
+    module_name: str | None,
+    package: str | None,
+) -> object:
+    """Import an attribute from a module located in a package.
+
+    This utility function is used in custom __getattr__ methods within __init__.py
+    files to dynamically import attributes.
+
+    Args:
+        attr_name: The name of the attribute to import.
+        module_name: The name of the module to import from. If None, the attribute
+            is imported from the package itself.
+        package: The name of the package where the module is located.
+    """
+    if module_name == "__module__" or module_name is None:
+        try:
+            result = import_module(f".{attr_name}", package=package)
+        except ModuleNotFoundError:
+            msg = f"module '{package!r}' has no attribute {attr_name!r}"
+            raise AttributeError(msg) from None
+    else:
+        try:
+            module = import_module(f".{module_name}", package=package)
+        except ModuleNotFoundError:
+            msg = f"module '{package!r}.{module_name!r}' not found"
+            raise ImportError(msg) from None
+        result = getattr(module, attr_name)
+    return result

src/backend/base/langflow/components/aiml/__init__.py

@@ -1,7 +1,37 @@
-from .aiml import AIMLModelComponent
-from .aiml_embeddings import AIMLEmbeddingsComponent
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from langflow.components._importing import import_mod
+
+if TYPE_CHECKING:
+    from langflow.components.aiml.aiml import AIMLModelComponent
+    from langflow.components.aiml.aiml_embeddings import AIMLEmbeddingsComponent
+
+_dynamic_imports = {
+    "AIMLModelComponent": "aiml",
+    "AIMLEmbeddingsComponent": "aiml_embeddings",
+}
 
 __all__ = [
     "AIMLEmbeddingsComponent",
     "AIMLModelComponent",
 ]
+
+
+def __getattr__(attr_name: str) -> Any:
+    """Lazily import aiml components on attribute access."""
+    if attr_name not in _dynamic_imports:
+        msg = f"module '{__name__}' has no attribute '{attr_name}'"
+        raise AttributeError(msg)
+    try:
+        result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent)
+    except (ModuleNotFoundError, ImportError, AttributeError) as e:
+        msg = f"Could not import '{attr_name}' from '{__name__}': {e}"
+        raise AttributeError(msg) from e
+    globals()[attr_name] = result
+    return result
+
+
+def __dir__() -> list[str]:
+    return list(__all__)

src/backend/base/langflow/components/amazon/__init__.py

@@ -1,5 +1,36 @@
-from .amazon_bedrock_embedding import AmazonBedrockEmbeddingsComponent
-from .amazon_bedrock_model import AmazonBedrockComponent
-from .s3_bucket_uploader import S3BucketUploaderComponent
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from langflow.components._importing import import_mod
+
+if TYPE_CHECKING:
+    from langflow.components.amazon.amazon_bedrock_embedding import AmazonBedrockEmbeddingsComponent
+    from langflow.components.amazon.amazon_bedrock_model import AmazonBedrockComponent
+    from langflow.components.amazon.s3_bucket_uploader import S3BucketUploaderComponent
+
+_dynamic_imports = {
+    "AmazonBedrockEmbeddingsComponent": "amazon_bedrock_embedding",
+    "AmazonBedrockComponent": "amazon_bedrock_model",
+    "S3BucketUploaderComponent": "s3_bucket_uploader",
+}
 
 __all__ = ["AmazonBedrockComponent", "AmazonBedrockEmbeddingsComponent", "S3BucketUploaderComponent"]
+
+
+def __getattr__(attr_name: str) -> Any:
+    """Lazily import amazon components on attribute access."""
+    if attr_name not in _dynamic_imports:
+        msg = f"module '{__name__}' has no attribute '{attr_name}'"
+        raise AttributeError(msg)
+    try:
+        result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent)
+    except (ModuleNotFoundError, ImportError, AttributeError) as e:
+        msg = f"Could not import '{attr_name}' from '{__name__}': {e}"
+        raise AttributeError(msg) from e
+    globals()[attr_name] = result
+    return result
+
+
+def __dir__() -> list[str]:
+    return list(__all__)