Merge pull request #1 from CIAT-DAPA/develop

Develop

.github/workflows/main.yml

@@ -0,0 +1,136 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Devops WRF Postprocessing
+
+on:
+  push:
+    branches: [ "stage" ]
+    tags:
+    - 'v*'
+
+
+permissions:
+  contents: read
+
+jobs:
+
+# ------- START Scripts PROCCESS -------- #
+
+  TestScripts:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.9"
+    - name: Create environment
+      run: |
+        python -m venv venv
+    - name: Active environment
+      run: |
+        source venv/bin/activate
+    - name: Install dependencies
+      run: |
+        pip install -r ./requirements.txt
+    # - name: Run Tests
+    #   run: |
+    #     python -m unittest discover -s ./test/ 
+
+
+# ------- END Scripts PROCCESS -------- #    
+
+# ------- START MERGE PROCCESS -------- #
+
+  MergeMainScripts:
+    needs: TestScripts
+    name: Merge Stage with main
+    permissions: write-all
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+
+      - name: Merge stage -> main
+        uses: devmasx/merge-branch@master
+        with:
+          type: now
+          head_to_merge: ${{ github.ref }}
+          target_branch: main
+          github_token: ${{ github.token }}
+
+# ------- END MERGE PROCCESS -------- #
+
+# ------- START RELEASE PROCCESS -------- #
+
+  PostRelease:
+    needs: MergeMainScripts
+    name: Create Release
+    runs-on: ubuntu-latest
+    permissions: write-all
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: '0'
+    # Scripts Zip
+    - name: Zip artifact for deployment
+      run: zip releaseScripts.zip ./src/* -r
+    # Upload Artifacts
+    - name: Upload Scripts artifact for deployment job
+      uses: actions/upload-artifact@v3
+      with:
+        name: Scripts
+        path: releaseScripts.zip
+    # Generate Tagname
+    - name: Generate Tagname for release
+      id: taggerDryRun
+      uses: anothrNick/[email protected]
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        WITH_V: true
+        DRY_RUN: true
+        DEFAULT_BUMP: patch
+        RELEASE_BRANCHES : stage,main
+        BRANCH_HISTORY: last
+    # Create release
+    - name: Create Release
+      id: create_release
+      uses: actions/create-release@v1
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      with:
+        tag_name: ${{ steps.taggerDryRun.outputs.new_tag }}
+        release_name: Release ${{ steps.taggerDryRun.outputs.new_tag }}
+        #body_path: ./body.md
+        body: ${{ github.event.head_commit.message }}
+        draft: false
+        prerelease: false
+    # Upload Assets to release
+    - name: Upload Release Asset Scripts
+      id: upload-scripts-release-asset 
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps 
+        asset_path: ./releaseScripts.zip
+        asset_name: releaseScripts.zip
+        asset_content_type: application/zip
+    # update version setup.py
+    - name: Checkout code
+      uses: actions/checkout@v3
+      with:
+        ref: main
+    - name: Update version
+      run: |
+        sed -i "s/version='.*'/version='${{ steps.taggerDryRun.outputs.new_tag }}'/" setup.py
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    - name: Commit changes
+      uses: stefanzweifel/git-auto-commit-action@v4
+      with:
+        commit_message: "Update version to ${{ steps.taggerDryRun.outputs.new_tag }}"
+
+# ------- END RELEASE PROCCESS -------- #

setup.py

@@ -0,0 +1,53 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="agrilac_wrf_postprocessing",
+    version='v0.0.1',
+    author="stevensotelo",
+    author_email="[email protected]",
+    description="Postprocessing wrf outputs",
+    url="https://github.com/CIAT-DAPA/agrilac_wrf_postprocessing",
+    download_url="https://github.com/CIAT-DAPA/agrilac_wrf_postprocessing",
+    packages=find_packages('src'),
+    package_dir={'': 'src'},
+    keywords='postprocessing wrf rasters',
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "Operating System :: OS Independent",
+    ],
+    python_requires=">=3.9",
+    entry_points={
+        'console_scripts': [
+            'wrf_postprocessing=postprocessing.main:main',
+        ],
+    },
+    install_requires=[
+        "certifi==2024.6.2",
+        "cftime==1.6.4",
+        "click==8.1.7",
+        "click-plugins==1.1.1",
+        "cligj==0.7.2",
+        "colorama==0.4.6",
+        "contourpy==1.2.1",
+        "cycler==0.12.1",
+        "fiona==1.9.6",
+        "fonttools==4.53.0",
+        "geopandas==0.14.4",
+        "kiwisolver==1.4.5",
+        "matplotlib==3.9.0",
+        "netCDF4==1.7.1",
+        "numpy==2.0.0",
+        "packaging==24.1",
+        "pandas==2.2.2",
+        "pillow==10.3.0",
+        "pyparsing==3.1.2",
+        "pyproj==3.6.1",
+        "python-dateutil==2.9.0.post0",
+        "pytz==2024.1",
+        "rasterio==1.3.10",
+        "shapely==2.0.4",
+        "six==1.16.0",
+        "snuggs==1.4.7",
+        "tzdata==2024.1"
+    ]
+)

export_average.py → src/postprocessing/export_average.py

@@ -8,13 +8,13 @@
 from datetime import datetime, timedelta
 
 
-def export_raster(dataset, file_name, specific_variable, is4Dim=False):
+def export_raster(dataset, file_name, specific_variable, output_path, is4Dim=False):
 
     # Get the current script directory
     current_path = os.path.dirname(os.path.abspath(__file__))
-    output_path = os.path.join(current_path, 'outputs')
     output_path_folder = os.path.join(output_path, file_name)
-    shp_path = os.path.join(current_path, 'shp', "limite_nacional_2011", "limite_nacional_2011.shp")
+    parent_dir = os.path.dirname(os.path.dirname(current_path))
+    shp_path = os.path.join(parent_dir, 'shp', "limite_nacional_2011", "limite_nacional_2011.shp")
 
     if not os.path.exists(output_path):
         os.makedirs(output_path)

extract_data.py → src/postprocessing/extract_data.py

@@ -5,13 +5,12 @@
 from rasterio.transform import from_origin
 from export_average import export_raster
 import rasterio
-from datetime import datetime, timedelta
 
 
-def extract_data(path):
+def extract_data(inputs_path, outputs_path):
 
     # Filtrar solo los archivos
-    nc_files = [os.path.join(path, file) for file in os.listdir(path) if file.endswith('.nc')]
+    nc_files = [os.path.join(inputs_path, file) for file in os.listdir(inputs_path) if file.endswith('.nc')]
 
     for file in nc_files:
 
@@ -20,26 +19,23 @@ def extract_data(path):
 
         dataset = nc.Dataset(file)
 
-        xtime = dataset.variables['XTIME'][:]
-        start_date_str = dataset.START_DATE
+        T2 = export_raster(dataset, file_name, "T2", outputs_path)
 
-        T2 = export_raster(dataset, file_name, "T2")
-
-        RAINNC = export_raster(dataset, file_name, "RAINNC")
+        RAINNC = export_raster(dataset, file_name, "RAINNC", outputs_path)
 
-        HGT = export_raster(dataset, file_name, "HGT")
+        HGT = export_raster(dataset, file_name, "HGT", outputs_path)
 
-        SWDOWN = export_raster(dataset, file_name, "SWDOWN")
+        SWDOWN = export_raster(dataset, file_name, "SWDOWN", outputs_path)
 
-        U10 = export_raster(dataset, file_name, "U10")
+        U10 = export_raster(dataset, file_name, "U10", outputs_path)
 
-        V10 = export_raster(dataset, file_name, "V10")
+        V10 = export_raster(dataset, file_name, "V10", outputs_path)
 
-        P = export_raster(dataset, file_name, "P", True)
+        P = export_raster(dataset, file_name, "P", outputs_path, True)
 
-        PB = export_raster(dataset, file_name, "PB", True)
+        PB = export_raster(dataset, file_name, "PB", outputs_path, True)
 
-        QVAPOR = export_raster(dataset, file_name, "QVAPOR", True)
+        QVAPOR = export_raster(dataset, file_name, "QVAPOR", outputs_path, True)
 
         WS10m = calcWS10m(U10, V10)
 
@@ -49,6 +45,8 @@ def extract_data(path):
 
         dataset.close()
 
+        return True
+
 
 
 def calcWS10m(U10, V10):

src/postprocessing/main.py

@@ -0,0 +1,26 @@
+import argparse
+
+from extract_data import extract_data
+
+def main():
+
+    parser = argparse.ArgumentParser(description="Resampling script")
+
+    parser.add_argument("-i", "--inputs", help="Inputs path", required=True)
+    parser.add_argument("-o", "--outputs", help="Outputs path", required=True)
+
+
+    args = parser.parse_args()
+
+    print("Reading inputs")
+    print(args)
+
+    input_path = args.inputs
+    output_path = args.outputs
+
+    postp = extract_data(input_path, output_path)
+
+if __name__ == "__main__":
+    main()
+
+#python resampling.py "ETHIOPIA" "D:\\CIAT\\Code\\USAID\\aclimate_resampling\\data\\" "-1" 2 2023