Merge branch 'main' into update-contributors-quick-start-guide

.github/CODEOWNERS

@@ -112,3 +112,7 @@ ISSUE_TRIAGE_PROCESS.rst @eladkal
 /providers/src/airflow/providers/**/fs/ @bolkedebruin
 /providers/src/airflow/providers/common/io/ @bolkedebruin
 /docs/apache-airflow/core-concepts/objectstorage.rst @bolkedebruin
+
+# Migrations
+/airflow/migrations/ @ephraimbuddy
+/providers/src/airflow/providers/fab/migrations/ @ephraimbuddy

.github/ISSUE_TEMPLATE/airflow_bug_report.yml

@@ -25,7 +25,7 @@ body:
         the latest release or main to see if the issue is fixed before reporting it.
       multiple: false
       options:
-        - "2.10.3"
+        - "2.10.4"
         - "main (development)"
         - "Other Airflow 2 version (please specify below)"
     validations:

.github/actions/migration_tests/action.yml

@@ -45,6 +45,20 @@ runs:
         COMPOSE_PROJECT_NAME: "docker-compose"
       env:
         COMPOSE_PROJECT_NAME: "docker-compose"
+    - name: "Bring compose down again"
+      shell: bash
+      run: breeze down
+      env:
+        COMPOSE_PROJECT_NAME: "docker-compose"
+    - name: "Test offline migration ${{env.BACKEND}}"
+      shell: bash
+      run: >
+        breeze shell "airflow db reset -y &&
+        airflow db downgrade -n 2.7.0 -y &&
+        airflow db migrate -s"
+      env:
+        COMPOSE_PROJECT_NAME: "docker-compose"
+      if: env.BACKEND != 'sqlite'
     - name: "Bring any containers left down"
       shell: bash
       run: breeze down

.github/boring-cyborg.yml

@@ -155,8 +155,8 @@ labelPRBasedOnFilePath:
     - providers/tests/cloudant/**/*
 
   provider:cncf-kubernetes:
-    - airflow/**/kubernetes_*.py
     - airflow/example_dags/example_kubernetes_executor.py
+    - airflow/example_dags/example_local_kubernetes_executor.py
     - providers/src/airflow/providers/cncf/kubernetes/**/*
     - providers/src/airflow/providers/celery/executors/celery_kubernetes_executor.py
     - docs/apache-airflow-providers-cncf-kubernetes/**/*

tests/cli/commands/hybrid_commands/__init__.py → .github/dependabot.yml

@@ -13,4 +13,16 @@
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
-# under the License.
+# under the License
+---
+version: 2
+updates:
+  - package-ecosystem: pip
+    directories:
+      - /clients/python
+      - /dev/breeze
+      - /docker_tests
+      - /task_sdk
+      - /
+    schedule:
+      interval: daily

.github/workflows/basic-tests.yml

@@ -293,6 +293,7 @@ jobs:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       GITHUB_USERNAME: ${{ github.actor }}
       VERBOSE: "true"
+    if: inputs.canary-run == 'true'
     steps:
       - name: "Cleanup repo"
         shell: bash

.github/workflows/ci.yml

@@ -160,12 +160,15 @@ jobs:
       - name: "Get information about the Workflow"
         id: source-run-info
         run: breeze ci get-workflow-info 2>> ${GITHUB_OUTPUT}
+        env:
+          SKIP_BREEZE_SELF_UPGRADE_CHECK: "true"
       - name: Selective checks
         id: selective-checks
         env:
           PR_LABELS: "${{ steps.source-run-info.outputs.pr-labels }}"
           COMMIT_REF: "${{ github.sha }}"
           VERBOSE: "false"
+
         run: breeze ci selective-check 2>> ${GITHUB_OUTPUT}
       - name: env
         run: printenv
@@ -493,6 +496,7 @@ jobs:
        needs.build-info.outputs.full-tests-needed == 'true')
     with:
       test-groups: ${{ needs.build-info.outputs.test-groups }}
+      default-branch: ${{ needs.build-info.outputs.default-branch }}
       runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }}
       image-tag: ${{ needs.build-info.outputs.image-tag }}
       core-test-types-list-as-string: ${{ needs.build-info.outputs.core-test-types-list-as-string }}
@@ -739,28 +743,22 @@ jobs:
       - tests-kubernetes
       - tests-task-sdk
       - finalize-tests
-    if: github.event_name == 'schedule' && failure()
+    if: github.event_name == 'schedule' && failure() && github.run_attempt == 1
     runs-on: ["ubuntu-22.04"]
     steps:
       - name: Notify Slack
         id: slack
-        uses: slackapi/slack-github-action@v1.27.0
+        uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d  # v2.0.0
         with:
-          channel-id: 'internal-airflow-ci-cd'
+          method: chat.postMessage
+          token: ${{ env.SLACK_BOT_TOKEN }}
           # yamllint disable rule:line-length
           payload: |
-            {
-              "text": "🚨🕒 Scheduled CI Failure Alert 🕒🚨\n\n*Details:* <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|View the failure log>",
-              "blocks": [
-                {
-                  "type": "section",
-                  "text": {
-                    "type": "mrkdwn",
-                    "text": "🚨🕒 Scheduled CI Failure Alert 🕒🚨\n\n*Details:* <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|View the failure log>"
-                  }
-                }
-              ]
-            }
+            channel: "internal-airflow-ci-cd"
+            text: "🚨🕒 Scheduled CI Failure Alert 🕒🚨\n\n*Details:* <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|View the failure log>"
+            blocks:
+              - type: "section"
+                text:
+                  type: "mrkdwn"
+                  text: "🚨🕒 Scheduled CI Failure Alert 🕒🚨\n\n*Details:* <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|View the failure log>"
           # yamllint enable rule:line-length
-        env:
-          SLACK_BOT_TOKEN: ${{ env.SLACK_BOT_TOKEN }}

.github/workflows/codeql-analysis.yml

@@ -52,6 +52,8 @@ jobs:
       - name: "Get information about the Workflow"
         id: source-run-info
         run: breeze ci get-workflow-info 2>> ${GITHUB_OUTPUT}
+        env:
+          SKIP_BREEZE_SELF_UPGRADE_CHECK: "true"
       - name: Selective checks
         id: selective-checks
         env:

.github/workflows/run-unit-tests.yml

@@ -183,4 +183,4 @@ jobs:
         if: success()
       - name: "Post Tests failure"
         uses: ./.github/actions/post_tests_failure
-        if: failure()
+        if: failure() || cancelled()

.github/workflows/special-tests.yml

@@ -24,6 +24,10 @@ on:  # yamllint disable-line rule:truthy
         description: "The array of labels (in json form) determining default runner used for the build."
         required: true
         type: string
+      default-branch:
+        description: "The default branch for the repository"
+        required: true
+        type: string
       test-groups:
         description: "The json representing list of test test groups to run"
         required: true
@@ -199,6 +203,7 @@ jobs:
       include-success-outputs: ${{ inputs.include-success-outputs }}
       run-coverage: ${{ inputs.run-coverage }}
       debug-resources: ${{ inputs.debug-resources }}
+    if: ${{ inputs.default-branch == 'main' }}
 
   tests-system:
     name: "System test: ${{ matrix.test-group }}"

.pre-commit-config.yaml

@@ -174,6 +174,13 @@ repos:
         language: python
         additional_dependencies: ['rich>=12.4.4']
         require_serial: true
+      - id: check-imports-in-providers
+        name: Check imports in providers
+        entry: ./scripts/ci/pre_commit/check_imports_in_providers.py
+        language: python
+        additional_dependencies: ['rich>=12.4.4', "ruff==0.8.1"]
+        files: ^providers/src/airflow/providers/.*\.py$
+        require_serial: true
       - id: update-common-sql-api-stubs
         name: Check and update common.sql API stubs
         entry: ./scripts/ci/pre_commit/update_common_sql_api_stubs.py
@@ -484,21 +491,21 @@ repos:
         files: ^docs/apache-airflow/extra-packages-ref\.rst$|^hatch_build.py
         pass_filenames: false
         entry: ./scripts/ci/pre_commit/check_extra_packages_ref.py
-        additional_dependencies: ['rich>=12.4.4', 'hatchling==1.26.3', 'tabulate']
+        additional_dependencies: ['rich>=12.4.4', 'hatchling==1.27.0', 'tabulate']
       - id: check-hatch-build-order
         name: Check order of dependencies in hatch_build.py
         language: python
         files: ^hatch_build.py$
         pass_filenames: false
         entry: ./scripts/ci/pre_commit/check_order_hatch_build.py
-        additional_dependencies: ['rich>=12.4.4', 'hatchling==1.26.3']
+        additional_dependencies: ['rich>=12.4.4', 'hatchling==1.27.0']
       - id: update-extras
         name: Update extras in documentation
         entry: ./scripts/ci/pre_commit/insert_extras.py
         language: python
         files: ^contributing-docs/12_airflow_dependencies_and_extras.rst$|^INSTALL$|^providers/src/airflow/providers/.*/provider\.yaml$|^Dockerfile.*
         pass_filenames: false
-        additional_dependencies: ['rich>=12.4.4', 'hatchling==1.26.3']
+        additional_dependencies: ['rich>=12.4.4', 'hatchling==1.27.0']
       - id: check-extras-order
         name: Check order of extras in Dockerfile
         entry: ./scripts/ci/pre_commit/check_order_dockerfile_extras.py

Dockerfile

@@ -45,7 +45,7 @@ ARG AIRFLOW_UID="50000"
 ARG AIRFLOW_USER_HOME_DIR=/home/airflow
 
 # latest released version here
-ARG AIRFLOW_VERSION="2.10.3"
+ARG AIRFLOW_VERSION="2.10.4"
 
 ARG PYTHON_BASE_IMAGE="python:3.9-slim-bookworm"
 
@@ -55,7 +55,7 @@ ARG PYTHON_BASE_IMAGE="python:3.9-slim-bookworm"
 # Also use `force pip` label on your PR to swap all places we use `uv` to `pip`
 ARG AIRFLOW_PIP_VERSION=24.3.1
 # ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main"
-ARG AIRFLOW_UV_VERSION=0.5.6
+ARG AIRFLOW_UV_VERSION=0.5.11
 ARG AIRFLOW_USE_UV="false"
 ARG UV_HTTP_TIMEOUT="300"
 ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow"

Dockerfile.ci

@@ -993,22 +993,6 @@ function determine_airflow_to_use() {
            --constraint https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt
         # Some packages might leave legacy typing module which causes test issues
         pip uninstall -y typing || true
-        # We need to install `eval-type-backport` to avoid problems with Pydantic 2.10.+ released in
-        # November 2024 for python 3.8 and 3.9. While Pydantic 2.10.0/2.10.1 completely broke Airflow 2
-        # installation and Pydantic 2.10.2 fixed the issue for past versions of Airflow, there are still
-        # Some Typing constructs that are not handled well by Pydantic and in case Pydantic fails with
-        # those errors, it will STILL fall back to `eval-type-backport` to handle those cases (if
-        # if `eval-type-backport` is installed. Therefore - until we have Airflow 2.10.3 for backwards
-        # compatibility tests and we attempt to install "edge" provider that might use such breaking
-        # constructs, we need to install `eval-type-backport` to avoid problems with Pydantic 2.10.2+
-        # as well. As soon as we move to Airflow 2.10.4, we can remove this workaround because Airflow
-        # 2.10.4 adds "eval-type-backport" as a dependency and it will be installed automatically.
-        if [[ ${PYTHON_MAJOR_MINOR_VERSION} == "3.8" || ${PYTHON_MAJOR_MINOR_VERSION} == "3.9" ]]; then
-            echo
-            echo "${COLOR_BLUE}Installing eval-type-backport for Python ${PYTHON_MAJOR_MINOR_VERSION} to workaround Pydantic 2.10.0/2.10.1 issue with new typing style.${COLOR_RESET}"
-            echo
-            pip install eval-type-backport>=0.2.0
-        fi
         if [[ ${LINK_PROVIDERS_TO_AIRFLOW_PACKAGE=} == "true" ]]; then
             echo
             echo "${COLOR_BLUE}Linking providers to airflow package as we are using them from mounted sources.${COLOR_RESET}"
@@ -1381,7 +1365,7 @@ RUN bash /scripts/docker/install_packaging_tools.sh; \
 # Also use `force pip` label on your PR to swap all places we use `uv` to `pip`
 ARG AIRFLOW_PIP_VERSION=24.3.1
 # ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main"
-ARG AIRFLOW_UV_VERSION=0.5.6
+ARG AIRFLOW_UV_VERSION=0.5.11
 # TODO(potiuk): automate with upgrade check (possibly)
 ARG AIRFLOW_PRE_COMMIT_VERSION="4.0.1"
 ARG AIRFLOW_PRE_COMMIT_UV_VERSION="4.1.4"

INTHEWILD.md

@@ -101,7 +101,7 @@ Currently, **officially** using Airflow:
 1. [Bombora Inc](https://bombora.com/) [[@jeffkpayne](https://github.com/jeffkpayne), [@pakelley](https://github.com/pakelley), [@dNavalta](https://github.com/dNavalta), [@austynh](https://github.com/austynh), [@TheOriginalAlex](https://github.com/TheOriginalAlex)]
 1. [Bonial International GmbH](https://www.bonial.com/)
 1. [Bonnier Broadcasting](http://www.bonnierbroadcasting.com) [[@wileeam](https://github.com/wileeam)]
-1. [Bosch (Robert Bosch GmbH)](https://www.bosch.com/stories/topics/automated-driving/) [[@jscheffl](https://github.com/jscheffl), [@clellmann](https://github.com/clellmann), [@wolfdn](https://github.com/wolfdn), [@AutomationDev85](https://github.com/AutomationDev85), [@majorosdonat](https://github.com/majorosdonat)]
+1. [Bosch (Robert Bosch GmbH)](https://www.bosch.com/stories/topics/automated-driving/) [[@jscheffl](https://github.com/jscheffl), [@clellmann](https://github.com/clellmann), [@wolfdn](https://github.com/wolfdn), [@AutomationDev85](https://github.com/AutomationDev85), [@majorosdonat](https://github.com/majorosdonat), [@OliverWannenwetsch](https://github.com/OliverWannenwetsch)]
 1. [BounceX](http://www.bouncex.com) [[@JoshFerge](https://github.com/JoshFerge), [@hudsonrio](https://github.com/hudsonrio), [@ronniekritou](https://github.com/ronniekritou)]
 1. [Braintree](https://www.braintreepayments.com) [[@coopergillan](https://github.com/coopergillan), [@curiousjazz77](https://github.com/curiousjazz77), [@raymondberg](https://github.com/raymondberg)]
 1. [Branch](https://branch.io) [[@sdebarshi](https://github.com/sdebarshi), [@dmitrig01](https://github.com/dmitrig01)]

PROVIDERS.rst

@@ -153,8 +153,8 @@ Airflow version to the next MINOR release, when 12 months passed since the first
 MINOR version of Airflow.
 
 For example this means that by default we upgrade the minimum version of Airflow supported by providers
-to 2.8.0 in the first Provider's release after 18th of December 2024. 18th of December 2023 is the date when the
-first ``PATCHLEVEL`` of 2.8 (2.8.0) has been released.
+to 2.9.0 in the first Provider's release after 8th of April 2025. 8th of April 2024 is the date when the
+first ``PATCHLEVEL`` of 2.9 (2.9.0) has been released.
 
 When we increase the minimum Airflow version, this is not a reason to bump ``MAJOR`` version of the providers
 (unless there are other breaking changes in the provider). The reason for that is that people who use

README.md

@@ -97,7 +97,7 @@ Airflow is not a streaming solution, but it is often used to process real-time d
 
 Apache Airflow is tested with:
 
-|            | Main version (dev)         | Stable version (2.10.3)    |
+|            | Main version (dev)         | Stable version (2.10.4)    |
 |------------|----------------------------|----------------------------|
 | Python     | 3.9, 3.10, 3.11, 3.12      | 3.8, 3.9, 3.10, 3.11, 3.12 |
 | Platform   | AMD64/ARM64(\*)            | AMD64/ARM64(\*)            |
@@ -175,15 +175,15 @@ them to the appropriate format and workflow that your tool requires.
 
 
 ```bash
-pip install 'apache-airflow==2.10.3' \
- --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.10.3/constraints-3.9.txt"
+pip install 'apache-airflow==2.10.4' \
+ --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.10.4/constraints-3.9.txt"
 ```
 
 2. Installing with extras (i.e., postgres, google)
 
 ```bash
-pip install 'apache-airflow[postgres,google]==2.10.3' \
- --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.10.3/constraints-3.9.txt"
+pip install 'apache-airflow[postgres,google]==2.10.4' \
+ --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.10.4/constraints-3.9.txt"
 ```
 
 For information on installing provider packages, check
@@ -288,7 +288,7 @@ Apache Airflow version life cycle:
 
 | Version   | Current Patch/Minor   | State     | First Release   | Limited Support   | EOL/Terminated   |
 |-----------|-----------------------|-----------|-----------------|-------------------|------------------|
-| 2         | 2.10.3                | Supported | Dec 17, 2020    | TBD               | TBD              |
+| 2         | 2.10.4                | Supported | Dec 17, 2020    | TBD               | TBD              |
 | 1.10      | 1.10.15               | EOL       | Aug 27, 2018    | Dec 17, 2020      | June 17, 2021    |
 | 1.9       | 1.9.0                 | EOL       | Jan 03, 2018    | Aug 27, 2018      | Aug 27, 2018     |
 | 1.8       | 1.8.2                 | EOL       | Mar 19, 2017    | Jan 03, 2018      | Jan 03, 2018     |

RELEASE_NOTES.rst

@@ -21,6 +21,52 @@
 
 .. towncrier release notes start
 
+Airflow 2.10.4 (2024-12-16)
+---------------------------
+
+Significant Changes
+^^^^^^^^^^^^^^^^^^^
+
+TaskInstance ``priority_weight`` is capped in 32-bit signed integer ranges (#43611)
+"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
+
+Some database engines are limited to 32-bit integer values. As some users reported errors in
+weight rolled-over to negative values, we decided to cap the value to the 32-bit integer. Even
+if internally in python smaller or larger values to 64 bit are supported, ``priority_weight`` is
+capped and only storing values from -2147483648 to 2147483647.
+
+Bug Fixes
+^^^^^^^^^
+
+- Fix stats of dynamic mapped tasks after automatic retries of failed tasks (#44300)
+- Fix wrong display of multi-line messages in the log after filtering (#44457)
+- Allow "/" in metrics validator (#42934) (#44515)
+- Fix gantt flickering (#44488) (#44517)
+- Fix problem with inability to remove fields from Connection form (#40421) (#44442)
+- Check pool_slots on partial task import instead of execution (#39724) (#42693)
+- Avoid grouping task instance stats by try_number for dynamic mapped tasks (#44300) (#44319)
+- Re-queue task when they are stuck in queued (#43520) (#44158)
+- Suppress the warnings where we check for sensitive values (#44148) (#44167)
+- Fix get_task_instance_try_details to return appropriate schema (#43830) (#44133)
+- Log message source details are grouped (#43681) (#44070)
+- Fix duplication of Task tries in the UI (#43891) (#43950)
+- Add correct mime-type in OpenAPI spec (#43879) (#43901)
+- Disable extra links button if link is null or empty (#43844) (#43851)
+- Disable XCom list ordering by execution_date (#43680) (#43696)
+- Fix venv numpy example which needs to be 1.26 at least to be working in Python 3.12 (#43659)
+- Fix Try Selector in Mapped Tasks also on Index 0 (#43590) (#43591)
+- Prevent using ``trigger_rule="always"`` in a dynamic mapped task (#43810)
+- Prevent using ``trigger_rule=TriggerRule.ALWAYS`` in a task-generated mapping within bare tasks (#44751)
+
+Doc Only Changes
+""""""""""""""""
+- Update XCom docs around containers/helm (#44570) (#44573)
+
+Miscellaneous
+"""""""""""""
+- Raise deprecation warning when accessing inlet or outlet events through str (#43922)
+
+
 Airflow 2.10.3 (2024-11-05)
 ---------------------------
 

airflow/api_connexion/endpoints/asset_endpoint.py

@@ -43,14 +43,14 @@
     queued_event_collection_schema,
     queued_event_schema,
 )
+from airflow.api_fastapi.app import get_auth_manager
 from airflow.assets.manager import asset_manager
 from airflow.models.asset import AssetDagRunQueue, AssetEvent, AssetModel
 from airflow.utils import timezone
 from airflow.utils.api_migration import mark_fastapi_migration_done
 from airflow.utils.db import get_query_count
 from airflow.utils.session import NEW_SESSION, provide_session
 from airflow.www.decorators import action_logging
-from airflow.www.extensions.init_auth_manager import get_auth_manager
 
 if TYPE_CHECKING:
     from sqlalchemy.orm import Session