feat: add two filters to customise celery workers command

changelog.d/20241119_100430_crisgarta8_celery_queues.md

@@ -0,0 +1 @@
+- 💥 [Feature] Add a filter to define the celery workers startup command. (by @Ian2012)

tutor/hooks/catalog.py

@@ -514,6 +514,16 @@ def your_filter_callback(some_data):
     #: :param str file_path: The path to the file being checked.
     IS_FILE_RENDERED: Filter[bool, [str]] = Filter()
 
+    #: List of parameters to use when starting the LMS Celery worker ``celery worker ...``.
+    #:
+    #: :param list[str] command: the list of paramaters to use as the celery command.
+    LMS_WORKER_COMMAND: Filter[list[str], []] = Filter()
+
+    #: List of parameters to use when starting the CMS Celery worker ``celery worker ...``.
+    #:
+    #: :param list[str] command: the list of paramaters to use as the celery command.
+    CMS_WORKER_COMMAND: Filter[list[str], []] = Filter()
+
 
 class Contexts:
     """

tutor/plugins/openedx.py

@@ -142,3 +142,45 @@ def is_directory_mounted(image_name: str, dirname: str) -> bool:
 hooks.Filters.ENV_TEMPLATE_VARIABLES.add_item(
     ("iter_mounted_directories", iter_mounted_directories)
 )
+
+
+hooks.Filters.LMS_WORKER_COMMAND.add_items(
+    [
+        "celery",
+        "--app=lms.celery",
+        "worker",
+        "--loglevel=info",
+        "--hostname=edx.lms.core.default.%h",
+        "--queues=edx.lms.core.default,edx.lms.core.high,edx.lms.core.high_mem",
+        "--max-tasks-per-child=100",
+    ]
+)
+
+
+hooks.Filters.CMS_WORKER_COMMAND.add_items(
+    [
+        "celery",
+        "--app=cms.celery",
+        "worker",
+        "--loglevel=info",
+        "--hostname=edx.cms.core.default.%h",
+        "--queues=edx.cms.core.default,edx.cms.core.high,edx.cms.core.low",
+        "--max-tasks-per-child=100",
+    ]
+)
+
+
+def iter_cms_celery_parameters() -> t.Iterator[str]:
+    yield from hooks.Filters.CMS_WORKER_COMMAND.iterate()
+
+
+def iter_lms_celery_parameters() -> t.Iterator[str]:
+    yield from hooks.Filters.LMS_WORKER_COMMAND.iterate()
+
+
+hooks.Filters.ENV_TEMPLATE_VARIABLES.add_items(
+    [
+        ("iter_cms_celery_parameters", iter_cms_celery_parameters),
+        ("iter_lms_celery_parameters", iter_lms_celery_parameters),
+    ]
+)

tutor/templates/k8s/deployments.yml

@@ -141,7 +141,9 @@ spec:
       containers:
         - name: cms-worker
           image: {{ DOCKER_IMAGE_OPENEDX }}
-          args: ["celery", "--app=cms.celery", "worker", "--loglevel=info", "--hostname=edx.cms.core.default.%%h", "--max-tasks-per-child", "100", "--exclude-queues=edx.lms.core.default"]
+          args:
+          {% for value in iter_cms_celery_parameters() %}
+          - "{{value}}"{% endfor %}
           env:
           - name: SERVICE_VARIANT
             value: cms
@@ -250,7 +252,8 @@ spec:
       containers:
         - name: lms-worker
           image: {{ DOCKER_IMAGE_OPENEDX }}
-          args: ["celery", "--app=lms.celery", "worker", "--loglevel=info", "--hostname=edx.lms.core.default.%%h", "--max-tasks-per-child=100", "--exclude-queues=edx.cms.core.default"]
+          args: {% for value in iter_lms_celery_parameters() %}
+          - "{{value}}"{% endfor %}
           env:
           - name: SERVICE_VARIANT
             value: lms

tutor/templates/local/docker-compose.prod.yml

@@ -31,7 +31,8 @@ services:
     environment:
       SERVICE_VARIANT: lms
       DJANGO_SETTINGS_MODULE: lms.envs.tutor.production
-    command: celery --app=lms.celery worker --loglevel=info --hostname=edx.lms.core.default.%%h --max-tasks-per-child=100 --exclude-queues=edx.cms.core.default
+    command: {% for value in iter_lms_celery_parameters() %}
+      - "{{value}}"{% endfor %}
     restart: unless-stopped
     volumes:
       - ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro
@@ -50,7 +51,8 @@ services:
     environment:
       SERVICE_VARIANT: cms
       DJANGO_SETTINGS_MODULE: cms.envs.tutor.production
-    command: celery --app=cms.celery worker --loglevel=info --hostname=edx.cms.core.default.%%h --max-tasks-per-child 100 --exclude-queues=edx.lms.core.default
+    command: {% for value in iter_lms_celery_parameters() %}
+      - "{{value}}"{% endfor %}
     restart: unless-stopped
     volumes:
       - ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro