Merge pull request #239 from pangeo-forge/psql-backup

Psql backup

.github/workflows/build-review-app.yaml

@@ -1,9 +1,9 @@
 name: Build Review App
 
 on:
-  pull_request:
-    branches: ['main']
-    types: [opened, reopened, synchronize, labeled]
+  # pull_request:
+  #   branches: ['main']
+  #   types: [opened, reopened, synchronize, labeled]
 
 env:
   PIPELINE: '17cc0239-494f-4a68-aa75-3da7c466709c'

.github/workflows/delete-review-app.yaml

@@ -1,9 +1,9 @@
 name: Delete Review App
 
 on:
-  pull_request:
-    branches: ['main']
-    types: [unlabeled]
+  # pull_request:
+  #   branches: ['main']
+  #   types: [unlabeled]
 
 env:
   PIPELINE: '17cc0239-494f-4a68-aa75-3da7c466709c'

.github/workflows/main.yaml

@@ -1,18 +1,18 @@
 name: CI
 
 on:
-  push:
-    branches:
-      - main
-      - prod
-    paths-ignore:
-      - 'docs/**'
-  pull_request:
-    branches:
-      - main
-      - prod
-    paths-ignore:
-      - 'docs/**'
+  # push:
+  #   branches:
+  #     - main
+  #     - prod
+  #   paths-ignore:
+  #     - 'docs/**'
+  # pull_request:
+  #   branches:
+  #     - main
+  #     - prod
+  #   paths-ignore:
+  #     - 'docs/**'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

.github/workflows/push-dataflow-image.yaml

@@ -1,18 +1,18 @@
 name: Push Dataflow Container Image to GCR
 
 on:
-  push:
-    branches:
-      - main
-      - prod
-    paths:
-      - 'dataflow-container-image.txt'
-  pull_request:
-    branches:
-      - main
-      - prod
-    paths:
-      - 'dataflow-container-image.txt'
+  # push:
+  #   branches:
+  #     - main
+  #     - prod
+  #   paths:
+  #     - 'dataflow-container-image.txt'
+  # pull_request:
+  #   branches:
+  #     - main
+  #     - prod
+  #   paths:
+  #     - 'dataflow-container-image.txt'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

.github/workflows/test-dataflow-integration.yaml

@@ -1,11 +1,11 @@
 name: Test Dataflow Integration
 
 on:
-  deployment_status:
-  # TODO: add on 'schedule' against staging deployment?
-  pull_request:
-    branches: ['main']
-    types: [labeled]
+  # deployment_status:
+  # # TODO: add on 'schedule' against staging deployment?
+  # pull_request:
+  #   branches: ['main']
+  #   types: [labeled]
 
 jobs:
   matrix-generate-prs:

.github/workflows/test-db-backup.yaml

@@ -0,0 +1,69 @@
+name: Test database backup
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+env:
+  PYTEST_ADDOPTS: '--color=yes'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    # https://docs.github.com/en/actions/using-containerized-services/creating-postgresql-service-containers
+    # See also https://remarkablemark.org/blog/2021/03/14/setup-postgresql-in-github-actions/
+    # and https://github.com/actions/example-services/blob/master/.github/workflows/postgres-service.yml
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_DB: postgres_db
+          POSTGRES_PASSWORD: postgres_password
+          POSTGRES_PORT: 5432
+          POSTGRES_USER: postgres_user
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install python deps
+        run: |
+          python -m pip install pytest sqlmodel psycopg2-binary
+          python -m pip install . --no-deps
+
+      - name: Install PostgreSQL client
+        run: |
+          sudo apt-get update
+          sudo apt-get install --yes postgresql-client
+
+      # https://devcenter.heroku.com/articles/heroku-postgres-import-export#restore-to-local-database
+      - name: Restore database from file
+        # exit code is 1, but it does actually work, so just continue
+        continue-on-error: true
+        run: |
+          pg_restore --verbose --clean --no-acl --no-owner \
+          -h localhost -U postgres_user -d postgres_db \
+          db-backup/810fa5e1-9b78-4f3c-97e0-a1847b3a65e6
+        env:
+          PGPASSWORD: postgres_password
+
+      - name: Test restored database
+        run: |
+          pytest -vvxs db-backup/test_db_backup.py
+        env:
+          DATABASE_URL: postgresql://postgres_user:postgres_password@localhost:5432/postgres_db

.github/workflows/test-docker.yaml

@@ -2,18 +2,18 @@
 
 name: Tests
 on:
-  push:
-    branches:
-      - main
-      - prod
-    paths-ignore:
-      - 'docs/**'
-  pull_request:
-    branches:
-      - main
-      - prod
-    paths-ignore:
-      - 'docs/**'
+  # push:
+  #   branches:
+  #     - main
+  #     - prod
+  #   paths-ignore:
+  #     - 'docs/**'
+  # pull_request:
+  #   branches:
+  #     - main
+  #     - prod
+  #   paths-ignore:
+  #     - 'docs/**'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

db-backup/test_db_backup.py

@@ -0,0 +1,80 @@
+import os
+
+from sqlmodel import Session, create_engine, select
+
+from pangeo_forge_orchestrator.models import MODELS
+
+
+def test_db_backup():
+    database_url = os.environ["DATABASE_URL"]
+    connect_args = dict(options="-c timezone=utc")
+    engine = create_engine(database_url, echo=False, connect_args=connect_args)
+    with Session(engine) as session:
+        select_feedstocks = select(MODELS["feedstock"].table)
+        feedstocks = session.exec(select_feedstocks).all()
+
+        production_datasets = []
+        for f in feedstocks:
+            reciperun = MODELS["recipe_run"]
+            select_datasets = select(reciperun.table).where(
+                reciperun.table.feedstock_id == f.id,
+                reciperun.table.dataset_public_url.isnot(None),
+                reciperun.table.status == "completed",
+                reciperun.table.conclusion == "success",
+                reciperun.table.is_test.is_(False),
+                reciperun.table.dataset_public_url.isnot(None),
+            )
+            dss = session.exec(select_datasets).all()
+            production_datasets.append(dss)
+
+    # some of these are empty lists, so drop those
+    production_datasets = [dss for dss in production_datasets if dss]
+
+    # get the list of dataset_public_urls
+    dataset_public_urls = [ds.dataset_public_url for dss in production_datasets for ds in dss]
+
+    # make sure it's the expected list of urls
+    assert dataset_public_urls == [
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/noaa-coastwatch-geopolar-sst-feedstock/noaa-coastwatch-geopolar-sst.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/WOA_1degree_monthly-feedstock/woa18-1deg-monthly.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/cmip6-feedstock/test_surface.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/cmip6-feedstock/test_full_depth.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/cmip6-feedstock/CMIP6.CMIP.CCCma.CanESM5.historical.r1i1p1f1.Omon.zos.gn.v20190429.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/cmip6-feedstock/CMIP6.CMIP.CCCma.CanESM5.historical.r1i1p1f1.Omon.zos.gn.v20190429.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/cmip6-feedstock/CMIP6.DAMIP.NOAA-GFDL.GFDL-ESM4.hist-aer.r1i1p1f1.Amon.pr.gr1.v20180701.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/cmip6-feedstock/CMIP6.DAMIP.BCC.BCC-CSM2-MR.hist-aer.r1i1p1f1.Amon.pr.gn.v20190507.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/cmip6-feedstock/CMIP6.DAMIP.CCCma.CanESM5.hist-aer.r13i1p2f1.Amon.pr.gn.v20190429.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/cmip6-feedstock/CMIP6.DAMIP.CCCma.CanESM5.hist-aer.r10i1p1f1.Amon.pr.gn.v20190429.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/cmip6-feedstock/CMIP6.DAMIP.CCCma.CanESM5.hist-aer.r3i1p1f1.Amon.pr.gn.v20190429.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/cmip6-feedstock/CMIP6.PMIP.MPI-M.MPI-ESM1-2-LR.past2k.r1i1p1f1.Amon.tas.gn.v20210714.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/cmip6-feedstock/CMIP6.PMIP.MPI-M.MPI-ESM1-2-LR.past2k.r1i1p1f1.Amon.tas.gn.v20210714.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/CMIP6_static_grids-feedstock/NorESM2-LM.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/CMIP6_static_grids-feedstock/GFDL-ESM4.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/CMIP6_static_grids-feedstock/NorESM2-MM.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/CMIP6_static_grids-feedstock/MPI-ESM1-2-HR.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/CMIP6_static_grids-feedstock/MPI-ESM1-2-LR.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/CMIP6_static_grids-feedstock/NorESM2-MM.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/CMIP6_static_grids-feedstock/GFDL-ESM4.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/CMIP6_static_grids-feedstock/NorESM2-LM.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/CMIP6_static_grids-feedstock/MPI-ESM1-2-LR.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/CMIP6_static_grids-feedstock/MPI-ESM1-2-HR.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/CMIP6_static_grids-feedstock/MPI-ESM1-2-HR.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/gpcp-feedstock/gpcp.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/HadISST-feedstock/hadisst.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/pangeo-forge/AGDC-feedstock/AGCD.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/pangeo-forge/CMIP6-PMIP-feedstock/CMIP6.PMIP.MPI-M.MPI-ESM1-2-LR.past2k.r1i1p1f1.Amon.tas.gn.v20210714.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/pangeo-forge/CMIP6-PMIP-feedstock/CMIP6.PMIP.MRI.MRI-ESM2-0.past1000.r1i1p1f1.Amon.tas.gn.v20200120.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/pangeo-forge/CMIP6-PMIP-feedstock/CMIP6.PMIP.MIROC.MIROC-ES2L.past1000.r1i1p1f2.Amon.tas.gn.v20200318.zarr",
+        "s3://yuvipanda-test1/cmr/gpm3imergdl.zarr/",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/pangeo-forge/eVolv2k-feedstock/eVolv2k.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/pangeo-forge/eVolv2k-feedstock/eVolv2k.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/pangeo-forge/LMRv2p1_MCruns_ensemble_gridded-feedstock/LMRv2p1_MCruns_ensemble_gridded.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/pangeo-forge/EOBS-feedstock/eobs-wind-speed.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/pangeo-forge/EOBS-feedstock/eobs-surface-downwelling.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/pangeo-forge/EOBS-feedstock/eobs-tg-tn-tx-rr-hu-pp.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/pangeo-forge/EOBS-feedstock/eobs-wind-speed.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/pangeo-forge/EOBS-feedstock/eobs-surface-downwelling.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/pangeo-forge/EOBS-feedstock/eobs-tg-tn-tx-rr-hu-pp.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/pangeo-forge/liveocean-feedstock/liveocean.zarr",
+        "https://ncsa.osn.xsede.org/Pangeo/pangeo-forge/pangeo-forge/aws-noaa-oisst-feedstock/aws-noaa-oisst-avhrr-only.zarr",
+    ]