[ON HOLD] Django settings cleanup, logging improvements and critical fixes

.gitignore

@@ -639,7 +639,8 @@ tools/*/data_dir/
 backend/backend/settings/*
 !backend/backend/settings/__init__.py
 !backend/backend/settings/base.py
-!backend/backend/settings/dev.py
+!backend/backend/settings/platform.py
+!backend/backend/settings/celery.py
 !backend/backend/settings/test.py
 
 # Local Dependencies for docker testing

backend/.celery.env

@@ -0,0 +1,30 @@
+DJANGO_SETTINGS_MODULE='backend.settings.celery'
+
+# Default log level
+DEFAULT_LOG_LEVEL="INFO"
+
+# Postgres DB envs
+DB_HOST='unstract-db'
+DB_USER='unstract_dev'
+DB_PASSWORD='unstract_pass'
+DB_NAME='unstract_db'
+DB_PORT=5432
+
+# Redis
+REDIS_HOST="unstract-redis"
+REDIS_PORT=6379
+REDIS_PASSWORD=""
+REDIS_USER=default
+
+# Protocol buffers generated code.
+PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+
+# Enable logging of workflow history.
+ENABLE_LOG_HISTORY=True
+# Interval in seconds for periodic consumer operations.
+LOG_HISTORY_CONSUMER_INTERVAL=30
+# Maximum number of logs to insert in a single batch.
+LOGS_BATCH_LIMIT=30
+
+# Celery Configuration
+CELERY_BROKER_URL="redis://unstract-redis:6379"

backend/README.md

@@ -18,7 +18,7 @@ Contains the backend services for Unstract written with Django and DRF.
 - Copy `sample.env` into `.env` and update the necessary variables. For eg:
 
 ```
-DJANGO_SETTINGS_MODULE='backend.settings.dev'
+DJANGO_SETTINGS_MODULE='backend.settings.platform'
 DB_HOST='localhost'
 DB_USER='unstract_dev'
 DB_PASSWORD='unstract_pass'

backend/account/authentication_controller.py

@@ -113,7 +113,8 @@ def user_organizations(self, request: Request) -> Any:
         try:
             organizations = self.auth_service.user_organizations(request)
         except Exception as ex:
-            #
+            Logger.error(f"Failed to get user orgs: {ex}")
+
             self.user_logout(request)
 
             response = Response(

backend/account/authentication_plugin_registry.py

@@ -43,14 +43,11 @@ def _load_plugins() -> dict[str, dict[str, Any]]:
                     PluginConfig.AUTH_METADATA: module.metadata,
                 }
                 Logger.info(
-                    "Loaded auth plugin: %s, is_active: %s",
-                    module.metadata["name"],
-                    module.metadata["is_active"],
+                    "Loaded active authentication plugin: %s", module.metadata["name"]
                 )
             else:
-                Logger.warning(
-                    "Metadata is not active for %s authentication module.",
-                    auth_module_name,
+                Logger.info(
+                    "Skipping inactive authentication plugin: %s", auth_module_name
                 )
         except ModuleNotFoundError as exception:
             Logger.error(

backend/backend/__init__.py

@@ -1,3 +0,0 @@
-from .celery_service import app as celery_app
-
-__all__ = ["celery_app"]

backend/backend/asgi.py

@@ -8,13 +8,17 @@
 
 import os
 
-from django.core.asgi import get_asgi_application
-from dotenv import load_dotenv
+from dotenv import find_dotenv, load_dotenv
+
+load_dotenv(find_dotenv() or "")
 
 load_dotenv()
+
 os.environ.setdefault(
     "DJANGO_SETTINGS_MODULE",
-    os.environ.get("DJANGO_SETTINGS_MODULE", "backend.settings.dev"),
+    os.environ.get("DJANGO_SETTINGS_MODULE", "backend.settings.platform"),
 )
 
+from django.core.asgi import get_asgi_application  # noqa: E402
+
 application = get_asgi_application()

backend/backend/celery_service.py → backend/backend/celery.py

@@ -2,20 +2,25 @@
 
 import os
 
-from celery import Celery
-from django.conf import settings
-from utils.constants import ExecutionLogConstants
+from dotenv import find_dotenv, load_dotenv
 
-from backend.celery_task import TaskRegistry
-
-# Set the default Django settings module for the 'celery' program.
+# NOTE:
+# Do this before loading any environment variables.
 os.environ.setdefault(
     "DJANGO_SETTINGS_MODULE",
-    os.environ.get("DJANGO_SETTINGS_MODULE", "backend.settings.dev"),
+    os.environ.get("DJANGO_SETTINGS_MODULE", "backend.settings.celery"),
 )
 
+# Load environment variables.
+load_dotenv(find_dotenv() or "")
+
+from celery import Celery  # noqa: E402
+from django.conf import settings  # noqa: E402
+from utils.celery.constants import ExecutionLogConstants  # noqa: E402
+from utils.celery.task_registry import TaskRegistry  # noqa: E402
+
 # Create a Celery instance. Default time zone is UTC.
-app = Celery("backend")
+app = Celery("celery")
 
 # To register the custom tasks.
 TaskRegistry()
@@ -24,7 +29,6 @@
 app.conf.broker_url = settings.CELERY_BROKER_URL
 app.conf.result_backend = settings.CELERY_RESULT_BACKEND
 
-
 # Load task modules from all registered Django app configs.
 app.config_from_object("django.conf:settings", namespace="CELERY")
 

backend/backend/settings/base.py

@@ -9,15 +9,20 @@
 https://docs.djangoproject.com/en/4.2/ref/settings/
 """
 
-import os
-from pathlib import Path
-from typing import Optional
-from urllib.parse import quote_plus
-
 from dotenv import find_dotenv, load_dotenv
 
-from backend.constants import FeatureFlag
-from unstract.flags.feature_flag import check_feature_flag_status
+# Load environment variables.
+load_dotenv(find_dotenv() or "")
+
+import os  # noqa: E402
+from pathlib import Path  # noqa: E402
+from typing import Optional  # noqa: E402
+
+from backend.constants import FeatureFlag  # noqa: E402
+
+# Requires PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python env setting
+# to be loaded prior.
+from unstract.flags.feature_flag import check_feature_flag_status  # noqa: E402
 
 missing_settings = []
 
@@ -85,13 +90,6 @@ def get_required_setting(
     },
 }
 
-
-ENV_FILE = find_dotenv()
-if ENV_FILE:
-    load_dotenv(ENV_FILE)
-
-# Loading environment variables
-
 WORKFLOW_ACTION_EXPIRATION_TIME_IN_SECOND = os.environ.get(
     "WORKFLOW_ACTION_EXPIRATION_TIME_IN_SECOND", 10800
 )
@@ -163,10 +161,6 @@ def get_required_setting(
 LOG_HISTORY_CONSUMER_INTERVAL = int(
     get_required_setting("LOG_HISTORY_CONSUMER_INTERVAL", "60")
 )
-LOGS_BATCH_LIMIT = int(get_required_setting("LOGS_BATCH_LIMIT", "30"))
-CELERY_BROKER_URL = get_required_setting(
-    "CELERY_BROKER_URL", f"redis://{REDIS_HOST}:{REDIS_PORT}"
-)
 
 INDEXING_FLAG_TTL = int(get_required_setting("INDEXING_FLAG_TTL"))
 NOTIFICATION_TIMEOUT = int(get_required_setting("NOTIFICATION_TIMEOUT", "5"))
@@ -401,22 +395,10 @@ def get_required_setting(
 }
 
 
-# CELERY_RESULT_BACKEND = f"redis://{REDIS_HOST}:{REDIS_PORT}/1"
-# Postgres as result backend
-CELERY_RESULT_BACKEND = (
-    f"db+postgresql://{DB_USER}:{quote_plus(DB_PASSWORD)}"
-    f"@{DB_HOST}:{DB_PORT}/{DB_NAME}"
-)
-CELERY_ACCEPT_CONTENT = ["json"]
-CELERY_TASK_SERIALIZER = "json"
-CELERY_RESULT_SERIALIZER = "json"
-CELERY_TIMEZONE = "UTC"
-CELERY_TASK_MAX_RETRIES = 3
-CELERY_TASK_RETRY_BACKOFF = 60  # Time in seconds before retrying the task
-
 # Feature Flag
 FEATURE_FLAG_SERVICE_URL = {"evaluate": f"{FLIPT_BASE_URL}/api/v1/flags/evaluate/"}
 
+
 # Password validation
 # https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validators
 

backend/backend/settings/celery.py

@@ -0,0 +1,100 @@
+import os
+from typing import Any, Optional
+from urllib.parse import quote_plus
+
+from backend.constants import FeatureFlag
+
+# Requires PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python env setting
+# to be loaded prior.
+from unstract.flags.feature_flag import check_feature_flag_status
+
+missing_settings = []
+
+
+def get_required_setting(
+    setting_key: str, default: Optional[Any] = None
+) -> Optional[str]:
+    """Get the value of an environment variable specified by the given key. Add
+    missing keys to `missing_settings` so that exception can be raised at the
+    end.
+
+    Args:
+        key (str): The key of the environment variable
+        default (Optional[str], optional): Default value to return incase of
+                                           env not found. Defaults to None.
+
+    Returns:
+        Optional[str]: The value of the environment variable if found,
+                       otherwise the default value.
+    """
+    data = os.environ.get(setting_key, default)
+    if not data:
+        missing_settings.append(setting_key)
+    return data
+
+
+DB_NAME = os.environ.get("DB_NAME", "unstract_db")
+DB_USER = os.environ.get("DB_USER", "unstract_dev")
+DB_HOST = os.environ.get("DB_HOST", "backend-db-1")
+DB_PASSWORD = os.environ.get("DB_PASSWORD", "unstract_pass")
+DB_PORT = os.environ.get("DB_PORT", 5432)
+if check_feature_flag_status(FeatureFlag.MULTI_TENANCY_V2):
+    DB_ENGINE = "django.db.backends.postgresql"
+else:
+    DB_ENGINE = "django_tenants.postgresql_backend"
+
+
+REDIS_USER = os.environ.get("REDIS_USER", "default")
+REDIS_PASSWORD = os.environ.get("REDIS_PASSWORD", "")
+REDIS_HOST = os.environ.get("REDIS_HOST", "unstract-redis")
+REDIS_PORT = os.environ.get("REDIS_PORT", "6379")
+REDIS_DB = os.environ.get("REDIS_DB", "")
+
+ENABLE_LOG_HISTORY = get_required_setting("ENABLE_LOG_HISTORY")
+LOG_HISTORY_CONSUMER_INTERVAL = int(
+    get_required_setting("LOG_HISTORY_CONSUMER_INTERVAL", "60")
+)
+LOGS_BATCH_LIMIT = int(get_required_setting("LOGS_BATCH_LIMIT", "30"))
+
+CORS_ALLOWED_ORIGINS = [
+    "http://localhost:3000",
+    "http://127.0.0.1:3000",
+    "http://frontend.unstract.localhost",
+    # Other allowed origins if needed
+]
+
+# Database
+# https://docs.djangoproject.com/en/4.2/ref/settings/#databases
+DATABASES = {
+    "default": {
+        "ENGINE": DB_ENGINE,
+        "NAME": f"{DB_NAME}",
+        "USER": f"{DB_USER}",
+        "HOST": f"{DB_HOST}",
+        "PASSWORD": f"{DB_PASSWORD}",
+        "PORT": f"{DB_PORT}",
+        "ATOMIC_REQUESTS": True,
+    }
+}
+
+# SocketIO connection manager
+SOCKET_IO_MANAGER_URL = f"redis://{REDIS_HOST}:{REDIS_PORT}"
+
+CELERY_BROKER_URL = get_required_setting(
+    "CELERY_BROKER_URL", f"redis://{REDIS_HOST}:{REDIS_PORT}"
+)
+# CELERY_RESULT_BACKEND = f"redis://{REDIS_HOST}:{REDIS_PORT}/1"
+# Postgres as result backend
+CELERY_RESULT_BACKEND = (
+    f"db+postgresql://{DB_USER}:{quote_plus(DB_PASSWORD)}"
+    f"@{DB_HOST}:{DB_PORT}/{DB_NAME}"
+)
+CELERY_ACCEPT_CONTENT = ["json"]
+CELERY_TASK_SERIALIZER = "json"
+CELERY_RESULT_SERIALIZER = "json"
+CELERY_TIMEZONE = "UTC"
+CELERY_TASK_MAX_RETRIES = 3
+CELERY_TASK_RETRY_BACKOFF = 60  # Time in seconds before retrying the task
+
+
+INSTALLED_APPS = ["django.contrib.contenttypes", "django_celery_beat"]

backend/backend/settings/dev.py → backend/backend/settings/platform.py

@@ -1,6 +1,6 @@
 from backend.settings.base import *  # noqa: F401, F403
 
-DEBUG = True
+DEBUG = False
 
 X_FRAME_OPTIONS = "http://localhost:3000"
 X_FRAME_OPTIONS = "ALLOW-FROM http://localhost:3000"

backend/backend/wsgi.py

@@ -8,18 +8,19 @@
 
 import os
 
-from django.conf import settings
-from django.core.wsgi import get_wsgi_application
-from dotenv import load_dotenv
-from utils.log_events import start_server
+from dotenv import find_dotenv, load_dotenv
 
-load_dotenv()
+load_dotenv(find_dotenv() or "")
 
 os.environ.setdefault(
     "DJANGO_SETTINGS_MODULE",
-    os.environ.get("DJANGO_SETTINGS_MODULE", "backend.settings.dev"),
+    os.environ.get("DJANGO_SETTINGS_MODULE", "backend.settings.platform"),
 )
 
+from django.conf import settings  # noqa: E402
+from django.core.wsgi import get_wsgi_application  # noqa: E402
+from utils.log_events import start_server  # noqa: E402
+
 django_app = get_wsgi_application()
 
 application = start_server(django_app, f"{settings.PATH_PREFIX}/socket")

backend/manage.py

@@ -12,7 +12,7 @@ def main() -> None:
     load_dotenv()
     os.environ.setdefault(
         "DJANGO_SETTINGS_MODULE",
-        os.environ.get("DJANGO_SETTINGS_MODULE", "backend.settings.dev"),
+        os.environ.get("DJANGO_SETTINGS_MODULE", "backend.settings.platform"),
     )
     try:
         from django.core.management import execute_from_command_line