changed the service account and updated airflow script to use gcloud storage

Author: Shruti

.github/workflows/push-to-airflow.yml

@@ -7,35 +7,23 @@ jobs:
   push_to_airflow:
     name: Run push_to_airflow.sh
     runs-on: ubuntu-latest
-
     permissions:
-        contents: 'read' # lets the workflow read our code
-        id-token: 'write' # lets the workflow ask GitHub for a temporary token to log in to Google Cloud
-
+        contents: read # lets the workflow read this code
+        id-token: write # lets the workflow ask GitHub for an OIDC token
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
 
+        # Requests a GH token and exchanges it with GCP. GCP sends back a temp access token for the eto-cd-github service account
       - name: Authenticate to Google Cloud
-        id: auth
-        uses: google-github-actions/auth@v2 # takes the Github token and gets a Google token
+        id: 'auth'
+        uses: google-github-actions/auth@v2
         with:
           workload_identity_provider: 'projects/855475113448/locations/global/workloadIdentityPools/eto-github/providers/eto-github'
-          service_account: 'eto-artifact-registry[email protected]'
+          service_account: 'eto-cd[email protected]'
           token_format: 'access_token'
 
-      - name: Testing Authentication (temporary will delete later)
-        env:
-            GOOGLE_OAUTH_ACCESS_TOKEN: ${{ steps.auth.outputs.access_token }} # gsutil will use the Google token to authenticate
-        run: |
-          echo "Testing auth"
-          gcloud auth list
-          echo "Dags folder:"
-          gsutil ls gs://us-east1-production-cc2-202-b42a7a54-bucket
-
       - name: Run push_to_airflow.sh
-        env:
-            GOOGLE_OAUTH_ACCESS_TOKEN: ${{ steps.auth.outputs.access_token }} # gsutil will use the Google token to authenticate
         run: |
           chmod +x ./push_to_airflow.sh
           ./push_to_airflow.sh

push_to_airflow.sh

@@ -1,14 +1,14 @@
-gsutil rm -r gs://us-east1-production-cc2-202-b42a7a54-bucket/dags/airtable_scripts
-gsutil -m cp -r airtable_scripts gs://us-east1-production-cc2-202-b42a7a54-bucket/dags/
-gsutil cp bq_to_airtable.py gs://us-east1-production-cc2-202-b42a7a54-bucket/dags/
-gsutil cp airtable_to_bq.py gs://us-east1-production-cc2-202-b42a7a54-bucket/dags/
-gsutil cp -r examples/multi_airtable_to_bq_test gs://us-east1-production-cc2-202-b42a7a54-bucket/dags/airtable_to_bq_config/
-gsutil cp examples/single_airtable_to_bq_test.json gs://us-east1-production-cc2-202-b42a7a54-bucket/dags/airtable_to_bq_config/
-gsutil cp examples/single_bq_to_airtable_test.json gs://us-east1-production-cc2-202-b42a7a54-bucket/dags/bq_to_airtable_config/
-gsutil cp -r examples/multi_bq_to_airtable_test gs://us-east1-production-cc2-202-b42a7a54-bucket/dags/bq_to_airtable_config/
-gsutil cp examples/sql/* gs://us-east1-production-cc2-202-b42a7a54-bucket/dags/sql/airtable_to_bq/multi_airtable_to_bq_test/
-gsutil cp examples/sql/* gs://us-east1-production-cc2-202-b42a7a54-bucket/dags/sql/bq_to_airtable/multi_bq_to_airtable_test/
-gsutil cp examples/sql/base2_table1_input.sql gs://us-east1-production-cc2-202-b42a7a54-bucket/dags/sql/bq_to_airtable/single_bq_to_airtable_test/
-gsutil cp examples/sql/default_merge.sql gs://us-east1-production-cc2-202-b42a7a54-bucket/dags/sql/airtable_to_bq/single_airtable_to_bq_test/
-gsutil cp examples/schemas/* gs://airflow-data-exchange-development/schemas/airtable_to_bq/multi_airtable_to_bq_test/
-gsutil cp examples/schemas/default.json gs://airflow-data-exchange-development/schemas/airtable_to_bq/single_airtable_to_bq_test/
+gcloud storage rm -r gs://us-east1-production-cc2-202-b42a7a54-bucket/dags/airtable_scripts
+gcloud storage cp -r airtable_scripts gs://us-east1-production-cc2-202-b42a7a54-bucket/dags/
+gcloud storage cp bq_to_airtable.py gs://us-east1-production-cc2-202-b42a7a54-bucket/dags/
+gcloud storage cp airtable_to_bq.py gs://us-east1-production-cc2-202-b42a7a54-bucket/dags/
+gcloud storage cp -r examples/multi_airtable_to_bq_test gs://us-east1-production-cc2-202-b42a7a54-bucket/dags/airtable_to_bq_config/
+gcloud storage cp examples/single_airtable_to_bq_test.json gs://us-east1-production-cc2-202-b42a7a54-bucket/dags/airtable_to_bq_config/
+gcloud storage cp examples/single_bq_to_airtable_test.json gs://us-east1-production-cc2-202-b42a7a54-bucket/dags/bq_to_airtable_config/
+gcloud storage cp -r examples/multi_bq_to_airtable_test gs://us-east1-production-cc2-202-b42a7a54-bucket/dags/bq_to_airtable_config/
+gcloud storage cp examples/sql/* gs://us-east1-production-cc2-202-b42a7a54-bucket/dags/sql/airtable_to_bq/multi_airtable_to_bq_test/
+gcloud storage cp examples/sql/* gs://us-east1-production-cc2-202-b42a7a54-bucket/dags/sql/bq_to_airtable/multi_bq_to_airtable_test/
+gcloud storage cp examples/sql/base2_table1_input.sql gs://us-east1-production-cc2-202-b42a7a54-bucket/dags/sql/bq_to_airtable/single_bq_to_airtable_test/
+gcloud storage cp examples/sql/default_merge.sql gs://us-east1-production-cc2-202-b42a7a54-bucket/dags/sql/airtable_to_bq/single_airtable_to_bq_test/
+gcloud storage cp examples/schemas/* gs://airflow-data-exchange-development/schemas/airtable_to_bq/multi_airtable_to_bq_test/
+gcloud storage cp examples/schemas/default.json gs://airflow-data-exchange-development/schemas/airtable_to_bq/single_airtable_to_bq_test/