From 28d1806d8660c0ac0871010b25bcd2867d5d9a70 Mon Sep 17 00:00:00 2001 From: Sachin Arora Date: Wed, 18 Dec 2024 21:05:39 +0530 Subject: [PATCH] #45037: Support for additional celery config directly from airflow.cfg file --- chart/values.yaml | 1 + .../src/airflow/providers/celery/executors/default_celery.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/chart/values.yaml b/chart/values.yaml index adf68c3a194d3..34ba7ae8d855f 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -2607,6 +2607,7 @@ config: celery: flower_url_prefix: '{{ ternary "" .Values.ingress.flower.path (eq .Values.ingress.flower.path "/") }}' worker_concurrency: 16 + extra_celery_config: '{}' scheduler: standalone_dag_processor: '{{ ternary "True" "False" .Values.dagProcessor.enabled }}' # statsd params included for Airflow 1.10 backward compatibility; moved to [metrics] in 2.0 diff --git a/providers/src/airflow/providers/celery/executors/default_celery.py b/providers/src/airflow/providers/celery/executors/default_celery.py index 75f8cc2bfdf43..c99eb0178c46e 100644 --- a/providers/src/airflow/providers/celery/executors/default_celery.py +++ b/providers/src/airflow/providers/celery/executors/default_celery.py @@ -69,6 +69,8 @@ def _broker_supports_visibility_timeout(url): log.debug("Value for celery result_backend not found. Using sql_alchemy_conn with db+ prefix.") result_backend = f'db+{conf.get("database", "SQL_ALCHEMY_CONN")}' +extra_celery_config: dict = conf.getjson("celery", "extra_celery_config", fallback={}) if conf.has_option("celery", "extra_celery_config") else {} + DEFAULT_CELERY_CONFIG = { "accept_content": ["json"], "event_serializer": "json", @@ -85,6 +87,7 @@ def _broker_supports_visibility_timeout(url): ), "worker_concurrency": conf.getint("celery", "WORKER_CONCURRENCY", fallback=16), "worker_enable_remote_control": conf.getboolean("celery", "worker_enable_remote_control", fallback=True), + **extra_celery_config }