From 7d596cc3e7ac1dd76840b124bb2430e92e775a24 Mon Sep 17 00:00:00 2001 From: Yuan-Kai Liu <55262505+yuankailiu@users.noreply.github.com> Date: Wed, 18 Sep 2024 08:11:28 -0700 Subject: [PATCH] update to the new CDS-beta (#37) + `autoget.pyp`: update url to the new CDS-beta url and API token + README: update instructions on 1) how to setup the API with the new personal access token, 2) where to find my token, and 3) where to accept the terms of use + dependency: add cdsapi>=0.7.0 version constraint on pyproject.toml and requirements.txt files + tests/test_dload.py: update print out website link --------- Co-authored-by: Zhang Yunjun --- README.md | 18 +++++++++--------- pyproject.toml | 2 +- requirements.txt | 4 ++-- src/pyaps3/autoget.py | 12 ++++++------ src/pyaps3/model.cfg | 8 ++++---- tests/test_dload.py | 4 ++-- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 13f14b8..8683bac 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ ## PyAPS - Python based Atmospheric Phase Screen estimation -This python 3 module estimates differential phase delay maps due to the stratified atmosphere for correcting radar interferograms. It is rewritten in Python 3 language from PYAPS source code and adapted for ECMWF's ERA-5 corrections. +This Python 3 module estimates differential phase delay maps due to the stratified atmosphere for correcting radar interferograms. It is rewritten in Python 3 language from PyAPS source code and adapted for ECMWF's ERA-5 corrections. WARNING: The current version does not work with NARR and MERRA datasets. Contributions are welcomed. @@ -57,21 +57,21 @@ Test the installation by running: python PyAPS/tests/test_calc.py ``` -### 2. Account setup for [ERA5](https://retostauffer.org/code/Download-ERA5/) +### 2. Account setup for [ERA5](https://www.ecmwf.int/en/forecasts/dataset/ecmwf-reanalysis-v5) -ERA5 data set is redistributed over the Copernicus Climate Data Store (CDS). Registration is required for the data access and downloading. +ERA5 data set is redistributed over the Copernicus Climate Data Store (CDS)-beta ([migration guide](https://confluence.ecmwf.int/display/CKB/Please+read%3A+CDS+and+ADS+migrating+to+new+infrastructure%3A+Common+Data+Store+%28CDS%29+Engine)). Registration is required for the data access and downloading. -+ [Create a new account](https://cds.climate.copernicus.eu/user/register) on the CDS website if you don't own a user account yet. -+ Create the local file `$HOME/.cdsapirc` and add the following two lines: ++ [Create a new account](https://cds-beta.climate.copernicus.eu/) on the CDS-beta website if you don't own a user account yet. Note: the old CDS account won't work. ++ [CDS API setup](https://cds-beta.climate.copernicus.eu/how-to-api#install-the-cds-api-client): Create the local file `$HOME/.cdsapirc` (in your Unix/Linux environment) and add the following two lines: ```shell -url: https://cds.climate.copernicus.eu/api/v2 -key: 12345:abcdefghij-134-abcdefgadf-82391b9d3f +url: https://cds-beta.climate.copernicus.eu/api +key: your-personal-access-token ``` -where 12345 is your personal user ID (UID), the part behind the colon is your personal API key. More details can be found [here](https://cds.climate.copernicus.eu/api-how-to). Alternatively, you could edit the `model.cfg` file in the package directory, `site-packages/pyaps3` if installed via conda, and fill in the `[CDS]` section. +Your Personal Access Token can be found under [Your profile > Personal Access Token](https://cds-beta.climate.copernicus.eu/profile) section or on the [setup guide](https://cds-beta.climate.copernicus.eu/how-to-api#install-the-cds-api-client) page. Alternatively, you could add the token to the `[CDS]` section in `model.cfg` file in the package directory, `site-packages/pyaps3` if installed via conda. Note: using your [old CDS API key](https://cds.climate.copernicus.eu/) will lead to a 401 Client Error and Authentication failed. -+ **Make sure** that you accept the data license in the Terms of use on ECMWF website. ++ **Make sure** that you accept the data license in the Terms of use on ECMWF website: Login, under [Datasets > ERA5 hourly data on pressure levels from 1940 to present > Download > Terms of use](https://cds-beta.climate.copernicus.eu/datasets/reanalysis-era5-pressure-levels?tab=download), click **Accept** to accespt the license to use Copernicus Products. + Test the account setup by running: diff --git a/pyproject.toml b/pyproject.toml index de4e022..865f914 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ classifiers=[ "Programming Language :: Python :: 3", ] dependencies = [ - "cdsapi", + "cdsapi>=0.7.0", "matplotlib", "numpy", "pygrib", diff --git a/requirements.txt b/requirements.txt index 79dbc5f..438814a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # for running -cdsapi +cdsapi>=0.7.0 matplotlib numpy pygrib @@ -11,4 +11,4 @@ urllib3 pip setuptools setuptools_scm -wheel \ No newline at end of file +wheel diff --git a/src/pyaps3/autoget.py b/src/pyaps3/autoget.py index 76349b7..aeb3aeb 100644 --- a/src/pyaps3/autoget.py +++ b/src/pyaps3/autoget.py @@ -48,8 +48,8 @@ def ECMWFdload(bdate,hr,filedir,model='ERA5',datatype='fc',humidity='Q',snwe=Non print('WARNING: you are downloading from the old ECMWF platform. ' 'ERA-Interim is deprecated, use ERA-5 instead.') if model in 'ERA5': - print('INFO: You are using the latest ECMWF platform for downloading datasets: ' - 'https://cds.climate.copernicus.eu/api/v2') + cds_url = 'https://cds-beta.climate.copernicus.eu/api' + print('INFO: You are using the latest ECMWF platform for downloading datasets: ', cds_url) #------------------------------------------- # Define parameters @@ -91,14 +91,14 @@ def ECMWFdload(bdate,hr,filedir,model='ERA5',datatype='fc',humidity='Q',snwe=Non fname = flist[i] #------------------------------------------- - # CASE 1: request for CDS API client (new ECMWF platform, for ERA5) + # CASE 1: request for CDS API client (new ECMWF platform, for ERA5) if model in 'ERA5': # Contact the server rc_file = os.path.expanduser('~/.cdsapirc') if os.path.isfile(rc_file): c = cdsapi.Client() else: - url = 'https://cds.climate.copernicus.eu/api/v2' + url = cds_url key = config.get('CDS', 'key') c = cdsapi.Client(url=url, key=key) @@ -206,7 +206,7 @@ def MERRAdload(bdate,hr,filedir, hdf=False): else: weburl = '%s%s%s%s%s%s%s%s%s%s%s%s%s' %(url1,yr,url2,mon,url3n,date,url4,hr,url5,hr,url6n,date,url7) dir = '%s' %(filename) - + if not os.path.exists(dir): #urllib3.urlretrieve(weburl,dir) dloadCmd = 'wget "{}" --user {} --password {} -O {}'.format(weburl, user, pw, filename) @@ -222,7 +222,7 @@ def NARRdload(bdate,hr,filedir): os.makedirs(filedir) print('create foler: {}'.format(filedir)) - flist = [] + flist = [] for i, day in enumerate(bdate): webdir = day[0:6] fname = 'narr-a_221_%s_%s00_000.grb'%(day,hr) diff --git a/src/pyaps3/model.cfg b/src/pyaps3/model.cfg index 9b05046..6414d8e 100644 --- a/src/pyaps3/model.cfg +++ b/src/pyaps3/model.cfg @@ -3,19 +3,19 @@ #Get it from https://retostauffer.org/code/Download-ERA5/ #for ERA5 [CDS] -key = your-uid:your-api-key +key = your-personal-access-token #####The key to the new server for ECMWF -#Get it from https://software.ecmwf.int/wiki/display/WEBAPI/Accessing+ECMWF+data+servers+in+batch +#Get it from https://software.ecmwf.int/wiki/display/WEBAPI/Accessing+ECMWF+data+servers+in+batch #for ERA-Interim [outdated] [ECMWF] email = the-email-address-used-as-login@ecmwf-website.org -key = +key = #####Passwd and key for download from ucar [ERA] email = the-email-adress-used-as-login@ucar-website.org -key = +key = #####No username/password is required for NARR [NARR] diff --git a/tests/test_dload.py b/tests/test_dload.py index f9f4d3c..653e333 100644 --- a/tests/test_dload.py +++ b/tests/test_dload.py @@ -8,8 +8,8 @@ print('import pyaps3 from {}'.format(pa.__file__)) print('------------------------------------------------') print('test ERA5 data download') -print('NOTE: Account setup is required on the Copernicus Climate Data Store (CDS).') -print(' More detailed info can be found on: https://retostauffer.org/code/Download-ERA5/') +print('NOTE: Account setup is required on the Copernicus Climate Data Store (CDS)-beta.') +print(' More detailed info can be found on: https://cds-beta.climate.copernicus.eu/how-to-api') print(' Add your account info to ~/.cdsapirc file.') filedir = os.path.join(os.path.dirname(__file__), 'data', 'ERA5') pa.ECMWFdload(['20200601','20200901'], hr='14', filedir=filedir, model='ERA5', snwe=(30,40,120,140))