diff --git a/changelog.d/20241119_100430_crisgarta8_celery_queues.md b/changelog.d/20241119_100430_crisgarta8_celery_queues.md new file mode 100644 index 0000000000..d0ec11de84 --- /dev/null +++ b/changelog.d/20241119_100430_crisgarta8_celery_queues.md @@ -0,0 +1 @@ +- 💥 [Feature] Add a filter to define the celery workers startup command. (by @Ian2012) diff --git a/tutor/hooks/catalog.py b/tutor/hooks/catalog.py index b1bfcdbf7c..acbb81e59f 100644 --- a/tutor/hooks/catalog.py +++ b/tutor/hooks/catalog.py @@ -514,6 +514,16 @@ def your_filter_callback(some_data): #: :param str file_path: The path to the file being checked. IS_FILE_RENDERED: Filter[bool, [str]] = Filter() + #: List of parameters to use when starting the LMS Celery worker ``celery worker ...``. + #: + #: :param list[str] command: the list of paramaters to use as the celery command. + LMS_WORKER_COMMAND: Filter[list[str], []] = Filter() + + #: List of parameters to use when starting the CMS Celery worker ``celery worker ...``. + #: + #: :param list[str] command: the list of paramaters to use as the celery command. + CMS_WORKER_COMMAND: Filter[list[str], []] = Filter() + class Contexts: """ diff --git a/tutor/plugins/openedx.py b/tutor/plugins/openedx.py index 2984d60792..f01288173e 100644 --- a/tutor/plugins/openedx.py +++ b/tutor/plugins/openedx.py @@ -142,3 +142,45 @@ def is_directory_mounted(image_name: str, dirname: str) -> bool: hooks.Filters.ENV_TEMPLATE_VARIABLES.add_item( ("iter_mounted_directories", iter_mounted_directories) ) + + +hooks.Filters.LMS_WORKER_COMMAND.add_items( + [ + "celery", + "--app=lms.celery", + "worker", + "--loglevel=info", + "--hostname=edx.lms.core.default.%h", + "--queues=edx.lms.core.default,edx.lms.core.high,edx.lms.core.high_mem", + "--max-tasks-per-child=100", + ] +) + + +hooks.Filters.CMS_WORKER_COMMAND.add_items( + [ + "celery", + "--app=cms.celery", + "worker", + "--loglevel=info", + "--hostname=edx.cms.core.default.%h", + "--queues=edx.cms.core.default,edx.cms.core.high,edx.cms.core.low", + "--max-tasks-per-child=100", + ] +) + + +def iter_cms_celery_parameters() -> t.Iterator[str]: + yield from hooks.Filters.CMS_WORKER_COMMAND.iterate() + + +def iter_lms_celery_parameters() -> t.Iterator[str]: + yield from hooks.Filters.LMS_WORKER_COMMAND.iterate() + + +hooks.Filters.ENV_TEMPLATE_VARIABLES.add_items( + [ + ("iter_cms_celery_parameters", iter_cms_celery_parameters), + ("iter_lms_celery_parameters", iter_lms_celery_parameters), + ] +) diff --git a/tutor/templates/k8s/deployments.yml b/tutor/templates/k8s/deployments.yml index bf50b48eb7..f4ba005b53 100644 --- a/tutor/templates/k8s/deployments.yml +++ b/tutor/templates/k8s/deployments.yml @@ -141,7 +141,9 @@ spec: containers: - name: cms-worker image: {{ DOCKER_IMAGE_OPENEDX }} - args: ["celery", "--app=cms.celery", "worker", "--loglevel=info", "--hostname=edx.cms.core.default.%%h", "--max-tasks-per-child", "100", "--exclude-queues=edx.lms.core.default"] + args: + {% for value in iter_cms_celery_parameters() %} + - "{{value}}"{% endfor %} env: - name: SERVICE_VARIANT value: cms @@ -250,7 +252,8 @@ spec: containers: - name: lms-worker image: {{ DOCKER_IMAGE_OPENEDX }} - args: ["celery", "--app=lms.celery", "worker", "--loglevel=info", "--hostname=edx.lms.core.default.%%h", "--max-tasks-per-child=100", "--exclude-queues=edx.cms.core.default"] + args: {% for value in iter_lms_celery_parameters() %} + - "{{value}}"{% endfor %} env: - name: SERVICE_VARIANT value: lms diff --git a/tutor/templates/local/docker-compose.prod.yml b/tutor/templates/local/docker-compose.prod.yml index aa08f24121..ed72ab6b04 100644 --- a/tutor/templates/local/docker-compose.prod.yml +++ b/tutor/templates/local/docker-compose.prod.yml @@ -31,7 +31,8 @@ services: environment: SERVICE_VARIANT: lms DJANGO_SETTINGS_MODULE: lms.envs.tutor.production - command: celery --app=lms.celery worker --loglevel=info --hostname=edx.lms.core.default.%%h --max-tasks-per-child=100 --exclude-queues=edx.cms.core.default + command: {% for value in iter_lms_celery_parameters() %} + - "{{value}}"{% endfor %} restart: unless-stopped volumes: - ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro @@ -50,7 +51,8 @@ services: environment: SERVICE_VARIANT: cms DJANGO_SETTINGS_MODULE: cms.envs.tutor.production - command: celery --app=cms.celery worker --loglevel=info --hostname=edx.cms.core.default.%%h --max-tasks-per-child 100 --exclude-queues=edx.lms.core.default + command: {% for value in iter_lms_celery_parameters() %} + - "{{value}}"{% endfor %} restart: unless-stopped volumes: - ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro