Merge branch 'airbytehq:master' into master

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.63.14
+current_version = 0.64.3
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

.github/CODEOWNERS

@@ -22,6 +22,13 @@
 # Protocol related items
 /docs/understanding-airbyte/airbyte-protocol.md @airbytehq/protocol-reviewers
 
+# Bulk CDK
+/airbyte-cdk/bulk @airbytehq/dbsources @airbytehq/destinations
+/airbyte-cdk/bulk/core/extract/ @airbytehq/dbsources
+/airbyte-cdk/bulk/core/load/ @airbytehq/destinations
+/airbyte-cdk/bulk/toolkits/extract-*/ @airbytehq/dbsources
+/airbyte-cdk/bulk/toolkits/load-*/ @airbytehq/destinations
+
 # Java CDK
 /airbyte-cdk/java/airbyte-cdk @airbytehq/dbsources @airbytehq/destinations
 /airbyte-cdk/java/airbyte-cdk/*-sources/ @airbytehq/dbsources

.github/actions/run-airbyte-ci/action.yml

@@ -34,6 +34,10 @@ inputs:
     description: "GCP credentials for GCP Secret Manager"
     required: false
     default: ""
+  gcp_integration_tester_credentials:
+    description: "GCP credentials for integration tests"
+    required: false
+    default: ""
   git_repo_url:
     description: "Git repository URL"
     default: https://github.com/airbytehq/airbyte.git

.github/actions/send-pager-duty-event/action.yml

@@ -0,0 +1,36 @@
+name: "Send PagerDuty event"
+description: "Use Event API V2 to send a PagerDuty event."
+inputs:
+  integration_key:
+    description: "The integration key for the PagerDuty service."
+    required: true
+  summary:
+    description: "A brief text summary of the event."
+    required: true
+  severity:
+    description: "The severity of the event. (info, warning, error, critical)"
+    required: true
+  source:
+    description: "Specific human-readable unique identifier, such as a hostname, for the system having the problem."
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Send PagerDuty event
+      id: send-pager-duty-event
+      shell: bash
+      run: |
+        curl --request 'POST' \
+          --fail \
+          --url "https://events.pagerduty.com/v2/enqueue" \
+          --header 'Content-Type: application/json' \
+          --data '{
+            "payload": {
+              "summary": "${{ inputs.summary }}",
+              "severity": "${{ inputs.severity }}",
+              "source": "${{ inputs.source }}"
+            },
+            "event_action": "trigger",
+            "routing_key": "${{ inputs.integration_key }}"
+          }'

.github/workflows/airbyte-ci-tests.yml

@@ -39,6 +39,7 @@ jobs:
             - airbyte-ci/connectors/connector_ops/**
             - airbyte-ci/connectors/connectors_qa/**
             - airbyte-ci/connectors/ci_credentials/**
+            - airbyte-ci/connectors/erd/**
             - airbyte-ci/connectors/metadata_service/lib/**
             - airbyte-ci/connectors/metadata_service/orchestrator/**
             - airbyte-cdk/python/**

.github/workflows/auto_merge.yml

@@ -25,7 +25,7 @@ jobs:
           # We need a custom Github Token as some API endpoints
           # are not available from GHA auto generated tokens
           # like the one to list branch protection rules...
-          GITHUB_TOKEN: ${{ secrets.AUTO_MERGE_GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GH_PAT_MAINTENANCE_OSS }}
           AUTO_MERGE_PRODUCTION: ${{ vars.ENABLE_CONNECTOR_AUTO_MERGE }}
         run: |
           poetry install

.github/workflows/cdk-codeflash.yml

@@ -0,0 +1,44 @@
+name: Run Codeflash on CDK PR
+
+on:
+  pull_request:
+    paths:
+      - airbyte-cdk/python/airbyte_cdk/**
+  workflow_dispatch:
+
+defaults:
+  run:
+    working-directory: ./airbyte-cdk/python/airbyte_cdk
+
+jobs:
+  optimize:
+    name:
+      Optimize new code in this PR
+      # Only run codeflash if
+      # 1. The contributor is not codeflash. To avoid infinite loops
+      # 2. The pull request is not from a forked repository
+    if: ${{ github.actor != 'codeflash-ai[bot]' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork != true}}
+    runs-on: ubuntu-latest
+    env:
+      CODEFLASH_API_KEY: ${{ secrets.CODEFLASH_API_KEY }}
+      CODEFLASH_PR_NUMBER: ${{ github.event.number }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - name: Install Airbyte CDK
+        id: install-airbyte-cdk
+        run: |
+          pip install --upgrade pip
+          pip install poetry
+          poetry install --all-extras --with dev
+      - name: Run Codeflash to optimize code
+        id: optimize_code
+        run: |
+          poetry env use python
+          poetry run codeflash

.github/workflows/cdk_connectors_tests.yml

@@ -67,8 +67,7 @@ jobs:
       - name: Check PAT rate limits
         run: |
           ./tools/bin/find_non_rate_limited_PAT \
-            ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \
-            ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }}
+            ${{ secrets.GH_PAT_MAINTENANCE_OSS }}
       - name: Fetch last commit id from remote branch [PULL REQUESTS]
         if: github.event_name == 'pull_request'
         id: fetch_last_commit_id_pr
@@ -89,6 +88,7 @@ jobs:
           git_branch: ${{ github.head_ref }}
           git_revision: ${{ github.event_name == 'pull_request' && steps.fetch_last_commit_id_pr.outputs.commit_id || steps.fetch_last_commit_id_wd.outputs.commit_id }}
           github_token: ${{ env.PAT }}
+          gcp_integration_tester_credentials: ${{ secrets.GCLOUD_INTEGRATION_TESTER }}
           s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
           # A connector test can't take more than 5 hours to run (5 * 60 * 60 = 18000 seconds)

.github/workflows/community_ci.yml

@@ -176,11 +176,12 @@ jobs:
           docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
           gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
+          gcp_integration_tester_credentials: ${{ secrets.GCLOUD_INTEGRATION_TESTER }}
           sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
           git_repo_url: ${{ github.event.pull_request.head.repo.clone_url }}
           git_branch: ${{ github.head_ref }}
           git_revision: ${{ github.event.pull_request.head.sha }}
-          github_token: ${{ github.token }}
+          github_token: ${{ secrets.GH_PAT_MAINTENANCE_OSS }}
           s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
           subcommand: "connectors --modified test"

.github/workflows/connectors_up_to_date.yml

@@ -35,8 +35,8 @@ jobs:
           docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
           gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
           gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
-          github_token: ${{ secrets.AUTO_MERGE_GITHUB_TOKEN }}
+          github_token: ${{ secrets.GH_PAT_MAINTENANCE_OSS }}
           sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
           s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
-          subcommand: "connectors  ${{ github.event.inputs.connectors-options || '--concurrency=10 --language=python --language=low-code' }} up-to-date --create-prs ${{ github.event.inputs.auto-merge == 'false' && '' || '--auto-merge' }}"
+          subcommand: "connectors  ${{ github.event.inputs.connectors-options || '--concurrency=10 --language=python --language=low-code --support-level=community --support-level=certified' }} up-to-date --ignore-connector=source-declarative-manifest --create-prs ${{ github.event.inputs.auto-merge == 'false' && '' || '--auto-merge' }}"

.github/workflows/live_tests.yml

@@ -25,6 +25,10 @@ on:
         required: true
       streams:
         description: Streams to include in tests
+      should_read_with_state:
+        description: Whether to run tests against the read command with state
+        default: "true"
+        type: boolean
       use_local_cdk:
         description: Use the local CDK when building the target connector
         default: "false"
@@ -93,6 +97,15 @@ jobs:
             echo "USE_LOCAL_CDK_FLAG=" >> $GITHUB_ENV
           fi
 
+      - name: Setup State Flag
+        if: github.event_name == 'workflow_dispatch'
+        run: |
+          if ${{ github.event.inputs.should_read_with_state }}; then
+            echo "READ_WITH_STATE_FLAG=--connector_live_tests.should-read-with-state" >> $GITHUB_ENV
+          else
+            echo "READ_WITH_STATE_FLAG=" >> $GITHUB_ENV
+          fi
+
       - name: Setup Connection Subset Option
         if: github.event_name == 'workflow_dispatch'
         run: |
@@ -114,4 +127,4 @@ jobs:
           github_token: ${{ secrets.GH_PAT_MAINTENANCE_OSS }}
           s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
-          subcommand: connectors ${{ env.USE_LOCAL_CDK_FLAG }} --name ${{ github.event.inputs.connector_name }} test --only-step connector_live_tests --connector_live_tests.test-suite=all --connector_live_tests.connection-id=${{ github.event.inputs.connection_id }} --connector_live_tests.pr-url=${{ github.event.inputs.pr_url }} ${{ env.STREAM_PARAMS }} ${{ env.CONNECTION_SUBSET }}
+          subcommand: connectors ${{ env.USE_LOCAL_CDK_FLAG }} --name ${{ github.event.inputs.connector_name }} test --only-step connector_live_tests --connector_live_tests.test-suite=live --connector_live_tests.connection-id=${{ github.event.inputs.connection_id }} --connector_live_tests.pr-url=${{ github.event.inputs.pr_url }} ${{ env.READ_WITH_STATE_FLAG }} ${{ env.STREAM_PARAMS }} ${{ env.CONNECTION_SUBSET }}

.github/workflows/publish-bulk-cdk.yml

@@ -95,7 +95,7 @@ jobs:
         uses: slackapi/[email protected]
         continue-on-error: true
         with:
-          channel-id: C04J1M66D8B
+          channel-id: C07K1P3UL6Q # The `#dev-java-cdk-releases` channel
           payload: |
             {
                 "text": "Error while publishing Bulk CDK!",
@@ -124,7 +124,7 @@ jobs:
         uses: slackapi/[email protected]
         continue-on-error: true
         with:
-          channel-id: C04J1M66D8B
+          channel-id: C07K1P3UL6Q # The `#dev-java-cdk-releases` channel
           payload: |
             {
                 "text": "Bulk CDK version 0.${{ env.BUILD_NUMBER }} published successfully!",

.github/workflows/publish-cdk-command-manually.yml

@@ -239,7 +239,7 @@ jobs:
           s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
           # There is no pull request number as we do this manually, so will just reference when we started doing it manually for now
-          subcommand: "connectors --concurrency=1 --execute-timeout=3600 --name=source-declarative-manifest bump-version ${{ github.event.inputs.release-type }} 'Bump CDK version to ${{needs.bump-version.outputs.new_cdk_version}}' --pr-number=36501"
+          subcommand: "connectors --concurrency=1 --execute-timeout=3600 --name=source-declarative-manifest bump-version version:${{needs.bump-version.outputs.new_cdk_version}} 'Bump CDK version to ${{needs.bump-version.outputs.new_cdk_version}}' --pr-number=36501"
           python_registry_token: ${{ secrets.PYPI_TOKEN }}
       - name: Commit and Push Changes
         uses: stefanzweifel/git-auto-commit-action@v4

.github/workflows/publish-java-cdk-command.yml

@@ -165,7 +165,7 @@ jobs:
         uses: slackapi/[email protected]
         continue-on-error: true
         with:
-          channel-id: C04J1M66D8B
+          channel-id: C07K1P3UL6Q # The `#dev-java-cdk-releases` channel
           payload: |
             {
                 "text": "Error during `publish-cdk` while publishing Java CDK!",
@@ -194,7 +194,7 @@ jobs:
         uses: slackapi/[email protected]
         continue-on-error: true
         with:
-          channel-id: C04J1M66D8B
+          channel-id: C07K1P3UL6Q # The `#dev-java-cdk-releases` channel
           payload: |
             {
                 "text": "New `${{ env.CDK_VERSION }}` version of Java CDK was successfully published!",

.github/workflows/publish_connectors.yml

@@ -45,6 +45,7 @@ jobs:
           subcommand: "connectors --concurrency=1 --execute-timeout=3600 --metadata-changes-only publish --main-release"
           python_registry_token: ${{ secrets.PYPI_TOKEN }}
           max_attempts: 2
+          retry_wait_seconds: 600 # 10 minutes
 
       - name: Publish connectors [manual]
         id: publish-connectors
@@ -69,19 +70,6 @@ jobs:
           airbyte_ci_binary_url: ${{ github.event.inputs.airbyte_ci_binary_url }}
           max_attempts: 2
 
-  set-instatus-incident-on-failure:
-    name: Create Instatus Incident on Failure
-    runs-on: ubuntu-latest
-    needs:
-      - publish_connectors
-    if: ${{ failure() && github.ref == 'refs/heads/master' }}
-    steps:
-      - name: Call Instatus Webhook
-        uses: joelwmale/webhook-action@master
-        with:
-          url: ${{ secrets.INSTATUS_CONNECTOR_PUBLISH_PIPELINE_WEBHOOK_URL }}
-          body: '{ "trigger": "down", "status": "HASISSUES" }'
-
   notify-failure-slack-channel:
     name: "Notify Slack Channel on Publish Failures"
     runs-on: ubuntu-latest
@@ -106,7 +94,26 @@ jobs:
             {
               "channel": "#connector-publish-failures",
               "username": "Connectors CI/CD Bot",
-              "text": "🚨 Publish workflow failed:\n ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+              "text": "🚨 Publish workflow failed:\n ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} \n merged by ${{ github.actor }} (<@${{ steps.match-github-to-slack-user.outputs.slack_user_ids }}>). "
             }
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.PUBLISH_ON_MERGE_SLACK_WEBHOOK }}
+
+  notify-failure-pager-duty:
+    name: "Notify PagerDuty on Publish Failures"
+    runs-on: ubuntu-latest
+    needs:
+      - publish_connectors
+    if: ${{ always() && contains(needs.*.result, 'failure') && github.ref == 'refs/heads/master' }}
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v4
+      - name: Notify PagerDuty
+        id: pager-duty
+        uses: ./.github/actions/send-pager-duty-event
+        with:
+          # Integration URL: https://airbyte.pagerduty.com/services/P5GNI5T/integrations/PGKH9JV
+          integration_key: ${{ secrets.PAGER_DUTY_PUBLISH_FAILURES_INTEGRATION_KEY }}
+          summary: "Publish workflow failed: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} merged by ${{ github.actor }}"
+          severity: "critical"
+          source: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"

.github/workflows/regression_tests.yml

@@ -25,6 +25,10 @@ on:
         required: true
       streams:
         description: Streams to include in regression tests
+      should_read_with_state:
+        description: Whether to run tests against the read command with state
+        default: "true"
+        type: boolean
       use_local_cdk:
         description: Use the local CDK when building the target connector
         default: "false"
@@ -93,6 +97,15 @@ jobs:
             echo "USE_LOCAL_CDK_FLAG=" >> $GITHUB_ENV
           fi
 
+      - name: Setup State Flag
+        if: github.event_name == 'workflow_dispatch'
+        run: |
+          if ${{ github.event.inputs.should_read_with_state }}; then
+            echo "READ_WITH_STATE_FLAG=--connector_live_tests.should-read-with-state" >> $GITHUB_ENV
+          else
+            echo "READ_WITH_STATE_FLAG=" >> $GITHUB_ENV
+          fi
+
       - name: Setup Connection Subset Option
         if: github.event_name == 'workflow_dispatch'
         run: |
@@ -114,4 +127,4 @@ jobs:
           github_token: ${{ secrets.GH_PAT_MAINTENANCE_OSS }}
           s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
-          subcommand: connectors ${{ env.USE_LOCAL_CDK_FLAG }} --name ${{ github.event.inputs.connector_name }} test --only-step connector_live_tests --connector_live_tests.test-suite=regression --connector_live_tests.connection-id=${{ github.event.inputs.connection_id }} --connector_live_tests.pr-url=${{ github.event.inputs.pr_url }} ${{ env.STREAM_PARAMS }} ${{ env.CONNECTION_SUBSET }} --global-status-check-context="Regression Tests" --global-status-check-description='Running regression tests'
+          subcommand: connectors ${{ env.USE_LOCAL_CDK_FLAG }} --name ${{ github.event.inputs.connector_name }} test --only-step connector_live_tests --connector_live_tests.test-suite=regression --connector_live_tests.connection-id=${{ github.event.inputs.connection_id }} --connector_live_tests.pr-url=${{ github.event.inputs.pr_url }} ${{ env.READ_WITH_STATE_FLAG }} ${{ env.STREAM_PARAMS }} ${{ env.CONNECTION_SUBSET }} --global-status-check-context="Regression Tests" --global-status-check-description='Running regression tests'