Merge pull request #982 from cmu-delphi/release/delphi-epidata-0.4.0

Release Delphi Epidata 0.4.0

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.21
+current_version = 0.4.0
 commit = False
 tag = False
 

.github/workflows/ci.yaml

@@ -62,7 +62,7 @@ jobs:
       - name: Start services
         run: |
           docker network create --driver bridge delphi-net
-          docker run --rm -d -p 13306:3306 --network delphi-net --name delphi_database_epidata delphi_database_epidata
+          docker run --rm -d -p 13306:3306 --network delphi-net --name delphi_database_epidata --cap-add=sys_nice delphi_database_epidata
           docker run --rm -d -p 10080:80 --env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" --env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" --network delphi-net --name delphi_web_epidata delphi_web_epidata
           docker ps
 
@@ -109,6 +109,7 @@ jobs:
     needs: build
     # only on main and dev branch
     if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev'
+
     runs-on: ubuntu-latest
     steps:
       - name: Check out code

.gitignore

@@ -1,4 +1,7 @@
 __pycache__/
+*.pyc
+*~
+\#*#
 .DS_Store
 /.vscode
 /delphi-epidata

dev/docker/database/epidata/Dockerfile

@@ -1,5 +1,14 @@
-# start with the `delphi_database` image
-FROM delphi_database
+# start with a standard percona mysql image
+FROM percona:ps-8
+
+# percona exits with the mysql user but we need root for additional setup
+USER root
+
+# use delphi's timezome
+RUN ln -s -f /usr/share/zoneinfo/America/New_York /etc/localtime
+
+# specify a development-only password for the database user "root"
+ENV MYSQL_ROOT_PASSWORD pass
 
 # create the `epidata` database
 ENV MYSQL_DATABASE epidata
@@ -8,8 +17,17 @@ ENV MYSQL_DATABASE epidata
 ENV MYSQL_USER user
 ENV MYSQL_PASSWORD pass
 
+# provide DDL which will configure dev environment at container startup
+COPY repos/delphi/delphi-epidata/dev/docker/database/epidata/_init.sql /docker-entrypoint-initdb.d/
+
 # provide DDL which will create empty tables at container startup
 COPY repos/delphi/delphi-epidata/src/ddl/*.sql /docker-entrypoint-initdb.d/
 
+# provide additional configuration needed for percona
+COPY repos/delphi/delphi-epidata/dev/docker/database/mysql.d/*.cnf /etc/my.cnf.d/
+
 # grant access to SQL scripts
 RUN chmod o+r /docker-entrypoint-initdb.d/*.sql
+
+# restore mysql user for percona
+USER mysql

dev/docker/database/epidata/_init.sql

@@ -0,0 +1,2 @@
+CREATE DATABASE covid;
+GRANT ALL ON covid.* TO 'user';

dev/docker/database/mysql.d/my.cnf

@@ -0,0 +1,2 @@
+[mysqld]
+default_authentication_plugin=mysql_native_password

dev/local/Makefile

@@ -10,19 +10,19 @@
 # Creates all prereq images (delphi_database, delphi_python) only if they don't
 # exist. If you need to rebuild a prereq, you're probably doing something
 # complicated, and can figure out the rebuild command on your own.
-# 
-# 
+#
+#
 # Commands:
-#            
+#
 #   web:     Stops currently-running delphi_web_epidata instances, if any.
 #            Rebuilds delphi_web_epidata image.
 #            Runs image in the background and pipes stdout to a log file.
-# 
+#
 #   db:      Stops currently-running delphi_database_epidata instances, if any.
 #            Rebuilds delphi_database_epidata image.
 #            Runs image in the background and pipes stdout to a log file.
 #            Blocks until database is ready to receive connections.
-# 
+#
 #   python:  Rebuilds delphi_web_python image. You shouldn't need to do this
 #            often; only if you are installing a new environment, or have
 #            made changes to delphi-epidata/dev/docker/python/Dockerfile.
@@ -35,7 +35,7 @@
 #
 #   clean:   Cleans up dangling Docker images.
 #
-# 
+#
 # Optional arguments:
 #   pdb=1         Drops you into debug mode upon test failure, if running tests.
 #   test=         Only runs tests in the directories provided here, e.g.
@@ -94,6 +94,9 @@ db:
 		docker stop $(DATABASE_CONTAINER_ID);\
 	fi
 
+	@# Setup virtual network if it doesn't exist
+	@docker network ls | grep delphi-net || docker network create --driver bridge delphi-net
+
 	@# Only build prereqs if we need them
 	@docker images delphi_database | grep delphi || \
 		docker build -t delphi_database -f repos/delphi/operations/dev/docker/database/Dockerfile .
@@ -105,11 +108,12 @@ db:
 	@# Run the database
 	@docker run --rm -p 127.0.0.1:13306:3306 \
 		--network delphi-net --name delphi_database_epidata \
+                --cap-add=sys_nice \
 		delphi_database_epidata >$(LOG_DB) 2>&1 &
 
 	@# Block until DB is ready
 	@while true; do \
-		sed -n '/Temporary server stopped/,/mysqld: ready for connections/p' $(LOG_DB) | grep "ready for connections" && break; \
+		sed -n '/mysqld: ready for connections/p' $(LOG_DB) | grep "ready for connections" && break; \
 		tail -1 $(LOG_DB); \
 		sleep 1; \
 	done
@@ -124,10 +128,10 @@ py:
 		-f repos/delphi/delphi-epidata/dev/docker/python/Dockerfile .
 
 .PHONY=all
-all: web db py
+all: db web py
 
 .PHONY=test
-test: 
+test:
 	@docker run -i --rm --network delphi-net \
 		--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata,target=/usr/src/app/repos/delphi/delphi-epidata,readonly \
 		--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata/src,target=/usr/src/app/delphi/epidata,readonly \

devops/Dockerfile

@@ -3,13 +3,16 @@ WORKDIR /src
 COPY . /src
 RUN npm ci && npm run build
 
-FROM tiangolo/meinheld-gunicorn:python3.7
+FROM tiangolo/meinheld-gunicorn:python3.8
 LABEL org.opencontainers.image.source=https://github.com/cmu-delphi/delphi-epidata
 # use delphi's timezome
 RUN ln -s -f /usr/share/zoneinfo/America/New_York /etc/localtime
 
-COPY requirements.txt /app
-RUN pip install --no-cache-dir -r requirements.txt
+COPY requirements.txt /app/requirements_also.txt
+RUN pip install --no-cache-dir -r /tmp/requirements.txt -r requirements_also.txt
+# the file /tmp/requirements.txt is created in the parent docker definition.  (see:
+#   https://github.com/tiangolo/meinheld-gunicorn-docker/blob/master/docker-images/python3.8.dockerfile#L5 )
+# this combined requirements installation ensures all version constrants are accounted for.
 
 # disable python stdout buffering
 ENV PYTHONUNBUFFERED 1

docs/api/README.md

@@ -1,6 +1,6 @@
 ---
-title: Epidata API (Other Diseases)
-nav_order: 3
+title: Other Endpoints (COVID-19 and Other Diseases)
+nav_order: 2
 has_children: true
 ---
 

docs/api/afhsb.md

@@ -1,6 +1,6 @@
 ---
 title: AFHSB
-parent: Epidata API (Other Diseases)
+parent: Other Endpoints (COVID-19 and Other Diseases)
 ---
 
 # AFHSB
@@ -36,12 +36,12 @@ See [this documentation](README.md) for details on specifying epiweeks, dates, a
 
 ## Response
 
-| Field | Description | Type |
-| --- | --- | --- |
-| `result` | result code: 1 = success, 2 = too many results, -2 = no results | integer |
-| `epidata` | list of results | array of objects |
-| ... | ... | ... | <!-- TODO -->
-| `message` | `success` or error message | string |
+| Field     | Description                                                     | Type             |
+|-----------|-----------------------------------------------------------------|------------------|
+| `result`  | result code: 1 = success, 2 = too many results, -2 = no results | integer          |
+| `epidata` | list of results                                                 | array of objects |
+| ...       | ...                                                             | ...              | <!-- TODO -->
+| `message` | `success` or error message                                      | string           |
 
 # Example URLs
 

docs/api/cdc.md

@@ -1,6 +1,6 @@
 ---
 title: CDC
-parent: Epidata API (Other Diseases)
+parent: Other Endpoints (COVID-19 and Other Diseases)
 ---
 
 # CDC
@@ -35,12 +35,12 @@ See [this documentation](README.md) for details on specifying epiweeks, dates, a
 
 ## Response
 
-| Field | Description | Type |
-| --- | --- | --- |
-| `result` | result code: 1 = success, 2 = too many results, -2 = no results | integer |
-| `epidata` | list of results | array of objects |
-| ... | ... | ... | <!-- TODO -->
-| `message` | `success` or error message | string |
+| Field     | Description                                                     | Type             |
+|-----------|-----------------------------------------------------------------|------------------|
+| `result`  | result code: 1 = success, 2 = too many results, -2 = no results | integer          |
+| `epidata` | list of results                                                 | array of objects |
+| ...       | ...                                                             | ...              | <!-- TODO -->
+| `message` | `success` or error message                                      | string           |
 
 # Example URLs
 

docs/api/client_libraries.md

@@ -1,6 +1,6 @@
 ---
 title: Epidata API Client Libraries
-parent: Epidata API (Other Diseases)
+parent: Other Endpoints (COVID-19 and Other Diseases)
 nav_order: 1
 ---
 

docs/api/covid_hosp.md

@@ -1,6 +1,6 @@
 ---
 title: COVID-19 Reported Patient Impact and Hospital Capacity by State Timeseries
-parent: Epidata API (Other Diseases)
+parent: Other Endpoints (COVID-19 and Other Diseases)
 ---
 
 # COVID-19 Hospitalization by State
@@ -43,16 +43,16 @@ See [this documentation](README.md) for details on specifying locations and date
 
 ### Required
 
-| Parameter | Description | Type |
-| --- | --- | --- |
-| `states` | two-letter state abbreviations | `list` of states |
-| `dates` | dates | `list` of dates or date ranges |
+| Parameter | Description                    | Type                           |
+|-----------|--------------------------------|--------------------------------|
+| `states`  | two-letter state abbreviations | `list` of states               |
+| `dates`   | dates                          | `list` of dates or date ranges |
 
 ### Optional
 
-| Parameter | Description | Type |
-| --- | --- | --- |
-| `issues` | issues | `list` of "issue" dates or date ranges |
+| Parameter | Description | Type                                   |
+|-----------|-------------|----------------------------------------|
+| `issues`  | issues      | `list` of "issue" dates or date ranges |
 
 If `issues` is not specified, then the most recent issue is used by default.
 

docs/api/covid_hosp_facility.md

@@ -1,6 +1,6 @@
 ---
 title: COVID-19 Reported Patient Impact and Hospital Capacity by Facility
-parent: Epidata API (Other Diseases)
+parent: Other Endpoints (COVID-19 and Other Diseases)
 ---
 
 # COVID-19 Hospitalization by Facility
@@ -38,10 +38,10 @@ See [this documentation](README.md) for details on specifying locations and date
 
 ### Required
 
-| Parameter | Description | Type |
-| --- | --- | --- |
-| `hospital_pks` | facility unique identifiers | `list` of identifiers |
-| `collection_weeks` | dates | `list` of dates or date ranges |
+| Parameter          | Description                 | Type                           |
+|--------------------|-----------------------------|--------------------------------|
+| `hospital_pks`     | facility unique identifiers | `list` of identifiers          |
+| `collection_weeks` | dates                       | `list` of dates or date ranges |
 
 NOTE: The companion
 [`covid_hosp_facility_lookup` endpoint](covid_hosp_facility_lookup.md) can be
@@ -54,8 +54,8 @@ dates) rather than an "epiweek" as in other endpoints.
 
 ### Optional
 
-| Parameter | Description | Type |
-| --- | --- | --- |
+| Parameter           | Description         | Type                                   |
+|---------------------|---------------------|----------------------------------------|
 | `publication_dates` | date of publication | `list` of "issue" dates or date ranges |
 
 If `publication_dates` is not specified, then the most recent issue is used by

docs/api/covid_hosp_facility_lookup.md

@@ -1,6 +1,6 @@
 ---
 title: COVID-19 Reported Patient Impact and Hospital Capacity - Facility lookup
-parent: Epidata API (Other Diseases)
+parent: Other Endpoints (COVID-19 and Other Diseases)
 ---
 
 # COVID-19 Hospitalization: Facility Lookup
@@ -36,13 +36,13 @@ See [this documentation](README.md) for details on specifying locations and date
 
 ## Parameters
 
-| Parameter | Description | Type |
-| --- | --- | --- |
-| `state` | two-letter state abbreviation | string |
-| `ccn` | facility CMS Certification Number | string |
-| `city` | city name | string |
-| `zip` | 5-digit ZIP code | string |
-| `fips_code` | 5-digit FIPS county code | string |
+| Parameter   | Description                       | Type   |
+|-------------|-----------------------------------|--------|
+| `state`     | two-letter state abbreviation     | string |
+| `ccn`       | facility CMS Certification Number | string |
+| `city`      | city name                         | string |
+| `zip`       | 5-digit ZIP code                  | string |
+| `fips_code` | 5-digit FIPS county code          | string |
 
 NOTE: Exactly one of the above parameters must be present in requests.
 Combinations of parameters (e.g. specifying both `city` and `state`) are not

docs/api/covidcast-signals/_source-template.md

@@ -1,7 +1,7 @@
 ---
 title: SOURCE NAME
 parent: Data Sources and Signals
-grand_parent: COVIDcast Epidata API
+grand_parent: COVIDcast Main Endpoint
 ---
 
 # SOURCE NAME
@@ -17,8 +17,8 @@ grand_parent: COVIDcast Epidata API
 
 A brief description of what this source measures.
 
-| Signal | Description |
-| --- | --- |
+| Signal        | Description                                                                                              |
+|---------------|----------------------------------------------------------------------------------------------------------|
 | `signal_name` | Brief description of the signal, including the units it is measured in and any smoothing that is applied |
 
 ## Table of contents

docs/api/covidcast-signals/chng.md

@@ -1,7 +1,7 @@
 ---
 title: Change Healthcare
 parent: Data Sources and Signals
-grand_parent: COVIDcast Epidata API
+grand_parent: COVIDcast Main Endpoint
 ---
 
 # Change Healthcare

docs/api/covidcast-signals/covid-act-now.md

@@ -1,7 +1,7 @@
 ---
 title: COVID Act Now
 parent: Data Sources and Signals
-grand_parent: COVIDcast Epidata API
+grand_parent: COVIDcast Main Endpoint
 ---
 
 # COVID Act Now (CAN)
@@ -21,10 +21,10 @@ While CAN provides this data potentially from multiple sources, we only use data
 [CDC's COVID-19 Integrated County View](https://covid.cdc.gov/covid-data-tracker/#county-view).
 
 
-| Signal | Description |
-| --- | --- |
+| Signal                         | Description                                                    |
+|--------------------------------|----------------------------------------------------------------|
 | `pcr_specimen_positivity_rate` | Proportion of PCR specimens tested that have a positive result |
-| `pcr_specimen_total_tests` | Total number of PCR specimens tested |
+| `pcr_specimen_total_tests`     | Total number of PCR specimens tested                           |
 
 ## Table of contents
 {: .no_toc .text-delta}

docs/api/covidcast-signals/doctor-visits.md

@@ -1,7 +1,7 @@
 ---
 title: Doctor Visits
 parent: Data Sources and Signals
-grand_parent: COVIDcast Epidata API
+grand_parent: COVIDcast Main Endpoint
 ---
 
 # Doctor Visits