From c638cc4c09260d95c29c492bad96b78179e45ef4 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
<41898282+github-actions[bot]@users.noreply.github.com>
Date: Mon, 26 Sep 2022 16:49:21 -0400
Subject: [PATCH 01/10] chore: release delphi-epidata 0.4.0 (#984)
Co-authored-by: melange396 <melange396@users.noreply.github.com>
Co-authored-by: melange396 <george.haff@gmail.com>
---
.bumpversion.cfg | 2 +-
src/client/delphi_epidata.R | 2 +-
src/client/delphi_epidata.js | 2 +-
src/client/packaging/npm/package.json | 2 +-
src/client/packaging/pypi/delphi_epidata/__init__.py | 2 +-
src/client/packaging/pypi/setup.py | 2 +-
src/server/_config.py | 2 +-
7 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index c13343850..3fa399e7a 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
[bumpversion]
-current_version = 0.3.21
+current_version = 0.4.0
commit = False
tag = False
diff --git a/src/client/delphi_epidata.R b/src/client/delphi_epidata.R
index 09fd0b95e..60eef54b1 100644
--- a/src/client/delphi_epidata.R
+++ b/src/client/delphi_epidata.R
@@ -15,7 +15,7 @@ Epidata <- (function() {
# API base url
BASE_URL <- 'https://delphi.cmu.edu/epidata/api.php'
- client_version <- '0.3.21'
+ client_version <- '0.4.0'
# Helper function to cast values and/or ranges to strings
.listitem <- function(value) {
diff --git a/src/client/delphi_epidata.js b/src/client/delphi_epidata.js
index 84e6036de..991d19af3 100644
--- a/src/client/delphi_epidata.js
+++ b/src/client/delphi_epidata.js
@@ -22,7 +22,7 @@
}
})(this, function (exports, fetchImpl, jQuery) {
const BASE_URL = "https://delphi.cmu.edu/epidata/";
- const client_version = "0.3.21";
+ const client_version = "0.4.0";
// Helper function to cast values and/or ranges to strings
function _listitem(value) {
diff --git a/src/client/packaging/npm/package.json b/src/client/packaging/npm/package.json
index f865d2015..5e4fbf02b 100644
--- a/src/client/packaging/npm/package.json
+++ b/src/client/packaging/npm/package.json
@@ -2,7 +2,7 @@
"name": "delphi_epidata",
"description": "Delphi Epidata API Client",
"authors": "Delphi Group",
- "version": "0.3.21",
+ "version": "0.4.0",
"license": "MIT",
"homepage": "https://github.com/cmu-delphi/delphi-epidata",
"bugs": {
diff --git a/src/client/packaging/pypi/delphi_epidata/__init__.py b/src/client/packaging/pypi/delphi_epidata/__init__.py
index 7f10b419a..07cbe2b15 100644
--- a/src/client/packaging/pypi/delphi_epidata/__init__.py
+++ b/src/client/packaging/pypi/delphi_epidata/__init__.py
@@ -1,4 +1,4 @@
from .delphi_epidata import Epidata
name = 'delphi_epidata'
-__version__ = '0.3.21'
+__version__ = '0.4.0'
diff --git a/src/client/packaging/pypi/setup.py b/src/client/packaging/pypi/setup.py
index b7ca8d64a..f4873b4b0 100644
--- a/src/client/packaging/pypi/setup.py
+++ b/src/client/packaging/pypi/setup.py
@@ -5,7 +5,7 @@
setuptools.setup(
name="delphi_epidata",
- version="0.3.21",
+ version="0.4.0",
author="David Farrow",
author_email="dfarrow0@gmail.com",
description="A programmatic interface to Delphi's Epidata API.",
diff --git a/src/server/_config.py b/src/server/_config.py
index 9d64c7682..47688a8ef 100644
--- a/src/server/_config.py
+++ b/src/server/_config.py
@@ -5,7 +5,7 @@
load_dotenv()
-VERSION = "0.3.21"
+VERSION = "0.4.0"
MAX_RESULTS = int(10e6)
MAX_COMPATIBILITY_RESULTS = int(3650)
From bef5839744086eec21081cc86ec2b056fe00658f Mon Sep 17 00:00:00 2001
From: Rostyslav Zatserkovnyi <zatserkovnyi.rostyslav@gmail.com>
Date: Wed, 5 Oct 2022 17:20:31 +0300
Subject: [PATCH 02/10] Tweaks to datetime optimization (#988)
* Tweaks to datetime optimization
* explicit days/weeks for time pairs
* and more validation
* logging
---
src/server/_query.py | 6 +-
src/server/utils/__init__.py | 2 +-
src/server/utils/dates.py | 130 ++++++++++++-------------------
tests/server/test_query.py | 21 +++++
tests/server/utils/test_dates.py | 29 ++++---
5 files changed, 93 insertions(+), 95 deletions(-)
diff --git a/src/server/_query.py b/src/server/_query.py
index 599baedff..1029c5e2c 100644
--- a/src/server/_query.py
+++ b/src/server/_query.py
@@ -22,7 +22,7 @@
from ._exceptions import DatabaseErrorException
from ._validate import DateRange, extract_strings
from ._params import GeoPair, SourceSignalPair, TimePair
-from .utils import dates_to_ranges
+from .utils import time_values_to_ranges, days_to_ranges, weeks_to_ranges
def date_string(value: int) -> str:
@@ -90,7 +90,7 @@ def filter_dates(
param_key: str,
params: Dict[str, Any],
):
- ranges = dates_to_ranges(values)
+ ranges = time_values_to_ranges(values)
return filter_values(field, ranges, param_key, params, date_string)
@@ -187,7 +187,7 @@ def filter_pair(pair: TimePair, i) -> str:
params[type_param] = pair.time_type
if isinstance(pair.time_values, bool) and pair.time_values:
return f"{type_field} = :{type_param}"
- ranges = dates_to_ranges(pair.time_values)
+ ranges = weeks_to_ranges(pair.time_values) if pair.is_week else days_to_ranges(pair.time_values)
return f"({type_field} = :{type_param} AND {filter_integers(time_field, cast(Sequence[Union[int, Tuple[int,int]]], ranges), type_param, params)})"
parts = [filter_pair(p, i) for i, p in enumerate(values)]
diff --git a/src/server/utils/__init__.py b/src/server/utils/__init__.py
index bdb85571d..3198779d0 100644
--- a/src/server/utils/__init__.py
+++ b/src/server/utils/__init__.py
@@ -1 +1 @@
-from .dates import shift_time_value, date_to_time_value, time_value_to_iso, time_value_to_date, days_in_range, weeks_in_range, shift_week_value, week_to_time_value, week_value_to_week, guess_time_value_is_day, dates_to_ranges
+from .dates import shift_time_value, date_to_time_value, time_value_to_iso, time_value_to_date, days_in_range, weeks_in_range, shift_week_value, week_to_time_value, week_value_to_week, guess_time_value_is_day, time_values_to_ranges, days_to_ranges, weeks_to_ranges
diff --git a/src/server/utils/dates.py b/src/server/utils/dates.py
index 6f54a5059..ef34a50b9 100644
--- a/src/server/utils/dates.py
+++ b/src/server/utils/dates.py
@@ -1,4 +1,5 @@
from typing import (
+ Callable,
Optional,
Sequence,
Tuple,
@@ -6,7 +7,7 @@
)
from datetime import date, timedelta
from epiweeks import Week, Year
-
+import logging
def time_value_to_date(value: int) -> date:
year, month, day = value // 10000, (value % 10000) // 100, value % 100
@@ -26,7 +27,7 @@ def week_value_to_week(value: int) -> Week:
def guess_time_value_is_day(value: int) -> bool:
# YYYYMMDD type and not YYYYMM
- return len(str(value)) > 6
+ return len(str(value)) == 8
def guess_time_value_is_week(value: int) -> bool:
# YYYYWW type and not YYYYMMDD
@@ -77,7 +78,7 @@ def weeks_in_range(week_range: Tuple[int, int]) -> int:
acc += year.totalweeks()
return acc + 1 # same week should lead to 1 week that will be queried
-def dates_to_ranges(values: Optional[Sequence[Union[Tuple[int, int], int]]]) -> Optional[Sequence[Union[Tuple[int, int], int]]]:
+def time_values_to_ranges(values: Optional[Sequence[Union[Tuple[int, int], int]]]) -> Optional[Sequence[Union[Tuple[int, int], int]]]:
"""
Converts a mixed list of dates and date ranges to an optimized list where dates are merged into ranges where possible.
e.g. [20200101, 20200102, (20200101, 20200104), 20200106] -> [(20200101, 20200104), 20200106]
@@ -87,84 +88,55 @@ def dates_to_ranges(values: Optional[Sequence[Union[Tuple[int, int], int]]]) ->
return values
# determine whether the list is of days (YYYYMMDD) or weeks (YYYYWW) based on first element
- try:
- if (isinstance(values[0], tuple) and guess_time_value_is_day(values[0][0]))\
- or (isinstance(values[0], int) and guess_time_value_is_day(values[0])):
- return days_to_ranges(values)
- elif (isinstance(values[0], tuple) and guess_time_value_is_week(values[0][0]))\
- or (isinstance(values[0], int) and guess_time_value_is_week(values[0])):
- return weeks_to_ranges(values)
- else:
- return values
- except:
+ first_element = values[0][0] if isinstance(values[0], tuple) else values[0]
+ if guess_time_value_is_day(first_element):
+ return days_to_ranges(values)
+ elif guess_time_value_is_week(first_element):
+ return weeks_to_ranges(values)
+ else:
return values
def days_to_ranges(values: Sequence[Union[Tuple[int, int], int]]) -> Sequence[Union[Tuple[int, int], int]]:
- intervals = []
-
- # populate list of intervals based on original values
- for v in values:
- if isinstance(v, int):
- # 20200101 -> [20200101, 20200101]
- intervals.append([time_value_to_date(v), time_value_to_date(v)])
- else: # tuple
- # (20200101, 20200102) -> [20200101, 20200102]
- intervals.append([time_value_to_date(v[0]), time_value_to_date(v[1])])
-
- intervals.sort(key=lambda x: x[0])
-
- # merge overlapping intervals https://leetcode.com/problems/merge-intervals/
- merged = []
- for interval in intervals:
- # no overlap, append the interval
- # caveat: we subtract 1 from interval[0] so that contiguous intervals are considered overlapping. i.e. [1, 1], [2, 2] -> [1, 2]
- if not merged or merged[-1][1] < interval[0] - timedelta(days=1):
- merged.append(interval)
- # overlap, merge the current and previous intervals
- else:
- merged[-1][1] = max(merged[-1][1], interval[1])
-
- # convert intervals from dates back to integers
- ranges = []
- for m in merged:
- if m[0] == m[1]:
- ranges.append(date_to_time_value(m[0]))
- else:
- ranges.append((date_to_time_value(m[0]), date_to_time_value(m[1])))
-
- return ranges
+ return _to_ranges(values, time_value_to_date, date_to_time_value, timedelta(days=1))
def weeks_to_ranges(values: Sequence[Union[Tuple[int, int], int]]) -> Sequence[Union[Tuple[int, int], int]]:
- intervals = []
-
- # populate list of intervals based on original values
- for v in values:
- if isinstance(v, int):
- # 202001 -> [202001, 202001]
- intervals.append([week_value_to_week(v), week_value_to_week(v)])
- else: # tuple
- # (202001, 202002) -> [202001, 202002]
- intervals.append([week_value_to_week(v[0]), week_value_to_week(v[1])])
-
- intervals.sort(key=lambda x: x[0])
-
- # merge overlapping intervals https://leetcode.com/problems/merge-intervals/
- merged = []
- for interval in intervals:
- # no overlap, append the interval
- # caveat: we subtract 1 from interval[0] so that contiguous intervals are considered overlapping. i.e. [1, 1], [2, 2] -> [1, 2]
- if not merged or merged[-1][1] < interval[0] - 1:
- merged.append(interval)
- # overlap, merge the current and previous intervals
- else:
- merged[-1][1] = max(merged[-1][1], interval[1])
-
- # convert intervals from weeks back to integers
- ranges = []
- for m in merged:
- if m[0] == m[1]:
- ranges.append(week_to_time_value(m[0]))
- else:
- ranges.append((week_to_time_value(m[0]), week_to_time_value(m[1])))
-
- return ranges
+ return _to_ranges(values, week_value_to_week, week_to_time_value, 1)
+
+def _to_ranges(values: Sequence[Union[Tuple[int, int], int]], value_to_date: Callable, date_to_value: Callable, time_unit: Union[int, timedelta]) -> Sequence[Union[Tuple[int, int], int]]:
+ try:
+ intervals = []
+
+ # populate list of intervals based on original date/week values
+ for v in values:
+ if isinstance(v, int):
+ # 20200101 -> [20200101, 20200101]
+ intervals.append([value_to_date(v), value_to_date(v)])
+ else: # tuple
+ # (20200101, 20200102) -> [20200101, 20200102]
+ intervals.append([value_to_date(v[0]), value_to_date(v[1])])
+
+ intervals.sort()
+
+ # merge overlapping intervals https://leetcode.com/problems/merge-intervals/
+ merged = []
+ for interval in intervals:
+ # no overlap, append the interval
+ # caveat: we subtract 1 from interval[0] so that contiguous intervals are considered overlapping. i.e. [1, 1], [2, 2] -> [1, 2]
+ if not merged or merged[-1][1] < interval[0] - time_unit:
+ merged.append(interval)
+ # overlap, merge the current and previous intervals
+ else:
+ merged[-1][1] = max(merged[-1][1], interval[1])
+
+ # convert intervals from dates/weeks back to integers
+ ranges = []
+ for m in merged:
+ if m[0] == m[1]:
+ ranges.append(date_to_value(m[0]))
+ else:
+ ranges.append((date_to_value(m[0]), date_to_value(m[1])))
+
+ return ranges
+ except Exception as e:
+ logging.info('bad input to date ranges', input=values, exception=e)
+ return values
diff --git a/tests/server/test_query.py b/tests/server/test_query.py
index 50d06f071..a59030b75 100644
--- a/tests/server/test_query.py
+++ b/tests/server/test_query.py
@@ -278,3 +278,24 @@ def test_filter_time_pairs(self):
"((t = :p_0t AND (v BETWEEN :p_0t_0 AND :p_0t_0_2)))",
)
self.assertEqual(params, {"p_0t": "day", "p_0t_0": 20201201, "p_0t_0_2": 20201203})
+ with self.subTest("dedupe"):
+ params = {}
+ self.assertEqual(
+ filter_time_pairs("t", "v", [TimePair("day", [20200101, 20200101, (20200101, 20200101), 20200101])], "p", params),
+ "((t = :p_0t AND (v = :p_0t_0)))",
+ )
+ self.assertEqual(params, {"p_0t": "day", "p_0t_0": 20200101})
+ with self.subTest("merge single range"):
+ params = {}
+ self.assertEqual(
+ filter_time_pairs("t", "v", [TimePair("day", [20200101, 20200102, (20200101, 20200104)])], "p", params),
+ "((t = :p_0t AND (v BETWEEN :p_0t_0 AND :p_0t_0_2)))",
+ )
+ self.assertEqual(params, {"p_0t": "day", "p_0t_0": 20200101, "p_0t_0_2": 20200104})
+ with self.subTest("merge ranges and singles"):
+ params = {}
+ self.assertEqual(
+ filter_time_pairs("t", "v", [TimePair("day", [20200101, 20200103, (20200105, 20200107)])], "p", params),
+ "((t = :p_0t AND (v = :p_0t_0 OR v = :p_0t_1 OR v BETWEEN :p_0t_2 AND :p_0t_2_2)))",
+ )
+ self.assertEqual(params, {"p_0t": "day", "p_0t_0": 20200101, "p_0t_1": 20200103, 'p_0t_2': 20200105, 'p_0t_2_2': 20200107})
diff --git a/tests/server/utils/test_dates.py b/tests/server/utils/test_dates.py
index d2331b224..e825bbd9b 100644
--- a/tests/server/utils/test_dates.py
+++ b/tests/server/utils/test_dates.py
@@ -2,7 +2,7 @@
from datetime import date
from epiweeks import Week
-from delphi.epidata.server.utils.dates import time_value_to_date, date_to_time_value, shift_time_value, time_value_to_iso, days_in_range, weeks_in_range, week_to_time_value, week_value_to_week, dates_to_ranges
+from delphi.epidata.server.utils.dates import time_value_to_date, date_to_time_value, shift_time_value, time_value_to_iso, days_in_range, weeks_in_range, week_to_time_value, week_value_to_week, time_values_to_ranges
class UnitTests(unittest.TestCase):
@@ -41,16 +41,21 @@ def test_week_to_time_value(self):
self.assertEqual(week_to_time_value(Week(2021, 1)), 202101)
self.assertEqual(week_to_time_value(Week(2020, 42)), 202042)
- def test_dates_to_ranges(self):
- self.assertEqual(dates_to_ranges(None), None)
- self.assertEqual(dates_to_ranges([]), [])
+ def test_time_values_to_ranges(self):
+ self.assertEqual(time_values_to_ranges(None), None)
+ self.assertEqual(time_values_to_ranges([]), [])
# days
- self.assertEqual(dates_to_ranges([20200101]), [20200101])
- self.assertEqual(dates_to_ranges([(20200101, 20200105)]), [(20200101, 20200105)])
- self.assertEqual(dates_to_ranges([20211231, (20211230, 20220102), 20220102]), [(20211230, 20220102)])
- self.assertEqual(dates_to_ranges([20200101, 20200102, (20200101, 20200104), 20200106]), [(20200101, 20200104), 20200106])
+ self.assertEqual(time_values_to_ranges([20200101]), [20200101])
+ self.assertEqual(time_values_to_ranges([(20200101, 20200105)]), [(20200101, 20200105)])
+ self.assertEqual(time_values_to_ranges([20211231, (20211230, 20220102), 20220102]), [(20211230, 20220102)])
+ self.assertEqual(time_values_to_ranges([20200101, 20200102, (20200101, 20200104), 20200106]), [(20200101, 20200104), 20200106])
# weeks
- self.assertEqual(dates_to_ranges([202001]), [202001])
- self.assertEqual(dates_to_ranges([(202001, 202005)]), [(202001, 202005)])
- self.assertEqual(dates_to_ranges([202051, (202050, 202102), 202101]), [(202050, 202102)])
- self.assertEqual(dates_to_ranges([202050, 202051, (202050, 202101), 202103]), [(202050, 202101), 202103])
+ self.assertEqual(time_values_to_ranges([202001]), [202001])
+ self.assertEqual(time_values_to_ranges([(202001, 202005)]), [(202001, 202005)])
+ self.assertEqual(time_values_to_ranges([202051, (202050, 202102), 202101]), [(202050, 202102)])
+ self.assertEqual(time_values_to_ranges([202050, 202051, (202050, 202101), 202103]), [(202050, 202101), 202103])
+ # non-contiguous integers that represent actually contiguous time objects should join to become a range:
+ self.assertEqual(time_values_to_ranges([20200228, 20200301]), [20200228, 20200301]) # this is NOT a range because 2020 was a leap year
+ self.assertEqual(time_values_to_ranges([20210228, 20210301]), [(20210228, 20210301)]) # this becomes a range because these dates are indeed consecutive
+ # individual weeks become a range (2020 is a rare year with 53 weeks)
+ self.assertEqual(time_values_to_ranges([202051, 202052, 202053, 202101, 202102]), [(202051, 202102)])
From 4ec482c84f99da094869a543097b2dd0a778e60c Mon Sep 17 00:00:00 2001
From: Rostyslav Zatserkovnyi <zatserkovnyi.rostyslav@gmail.com>
Date: Fri, 14 Oct 2022 22:15:46 +0300
Subject: [PATCH 03/10] Add IDE bindings for epidata development (#1004)
---
README.md | 12 +++++++
dev/local/install.sh | 9 ++++-
dev/local/pyproject.toml | 3 ++
dev/local/setup.cfg | 71 ++++++++++++++++++++++++++++++++++++++++
4 files changed, 94 insertions(+), 1 deletion(-)
create mode 100644 dev/local/pyproject.toml
create mode 100644 dev/local/setup.cfg
diff --git a/README.md b/README.md
index 70f1ccc21..170c9a4ef 100644
--- a/README.md
+++ b/README.md
@@ -26,6 +26,8 @@ You should now have the following directory structure:
│ ├── .dockerignore -> repos/delphi/delphi-epidata/dev/local/.dockerignore
│ ├── Makefile -> repos/delphi/delphi-epidata/dev/local/Makefile
│ ├── repos
+│ │ ├── pyproject.toml -> delphi/delphi-epidata/dev/local/pyproject.toml
+│ │ ├── setup.cfg -> delphi/delphi-epidata/dev/local/setup.cfg
│ │ └── delphi
│ │ ├── delphi-epidata
│ │ ├── flu-contest
@@ -52,6 +54,16 @@ $ [sudo] make test pdb=1
$ [sudo] make test test=repos/delphi/delphi-epidata/integrations/acquisition
```
+Enabling features like code autocompletion and linting in your editor
+requires one extra step (prerequisites: up-to-date pip and setuptools v64+):
+
+```sh
+$ cd repos
+
+# Installs the working directory as an "editable package"
+$ pip install -e . --config-settings editable_mode=strict
+```
+
# COVIDcast
At the present, our primary focus is developing and expanding the
diff --git a/dev/local/install.sh b/dev/local/install.sh
index bfe097e94..9fd262e2e 100644
--- a/dev/local/install.sh
+++ b/dev/local/install.sh
@@ -2,7 +2,8 @@
# Bootstrap delphi-epidata development
#
# Downloads the repos needed for local delphi-epidata development into current dir
-# and provides a Makefile with Docker control commands.
+# and provides a Makefile with Docker control commands
+# as well as pyproject/setup.cfg files for IDE mappings.
#
# Creates the directory structure:
#
@@ -10,6 +11,8 @@
# .dockerignore
# Makefile
# repos/
+# pyproject.toml
+# setup.cfg
# delphi/
# operations/
# delphi-epidata/
@@ -43,3 +46,7 @@ cd ../../
ln -s repos/delphi/delphi-epidata/dev/local/Makefile
ln -s repos/delphi/delphi-epidata/dev/local/.dockerignore
+cd repos
+ln -s delphi/delphi-epidata/dev/local/pyproject.toml
+ln -s delphi/delphi-epidata/dev/local/setup.cfg
+cd -
diff --git a/dev/local/pyproject.toml b/dev/local/pyproject.toml
new file mode 100644
index 000000000..5773cb6f7
--- /dev/null
+++ b/dev/local/pyproject.toml
@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools>=65", "wheel"]
+build-backend = "setuptools.build_meta"
diff --git a/dev/local/setup.cfg b/dev/local/setup.cfg
new file mode 100644
index 000000000..84665f923
--- /dev/null
+++ b/dev/local/setup.cfg
@@ -0,0 +1,71 @@
+[metadata]
+name = Delphi Development
+version = 0.4.0
+
+[options]
+packages =
+ delphi.epidata
+ delphi.epidata.acquisition
+ delphi.epidata.acquisition.afhsb
+ delphi.epidata.acquisition.cdcp
+ delphi.epidata.acquisition.covid_hosp
+ delphi.epidata.acquisition.covid_hosp.common
+ delphi.epidata.acquisition.covid_hosp.facility
+ delphi.epidata.acquisition.covid_hosp.state_daily
+ delphi.epidata.acquisition.covid_hosp.state_timeseries
+ delphi.epidata.acquisition.covidcast
+ delphi.epidata.acquisition.covidcast_nowcast
+ delphi.epidata.acquisition.ecdc
+ delphi.epidata.acquisition.flusurv
+ delphi.epidata.acquisition.fluview
+ delphi.epidata.acquisition.ght
+ delphi.epidata.acquisition.kcdc
+ delphi.epidata.acquisition.nidss
+ delphi.epidata.acquisition.norostat
+ delphi.epidata.acquisition.paho
+ delphi.epidata.acquisition.quidel
+ delphi.epidata.acquisition.twtr
+ delphi.epidata.acquisition.wiki
+ delphi.epidata.client
+ delphi.epidata.server
+ delphi.epidata.server.covidcast_issues_migration
+ delphi.epidata.server.endpoints
+ delphi.epidata.server.endpoints.covidcast_utils
+ delphi.epidata.server.utils
+ delphi.flu_contest
+ delphi.flu_contest
+ delphi.flu_contest.archefilter
+ delphi.flu_contest.covid
+ delphi.flu_contest.epicast
+ delphi.flu_contest.forecasters
+ delphi.flu_contest.hosp
+ delphi.flu_contest.main
+ delphi.flu_contest.uploads
+ delphi.flu_contest.utils
+ delphi.github_deploy_repo
+ delphi.github_deploy_repo.actions
+ delphi.nowcast
+ delphi.nowcast.experiments
+ delphi.nowcast.fusion
+ delphi.nowcast.obsolete
+ delphi.nowcast.sensors
+ delphi.nowcast.util
+ delphi.operations
+ delphi.operations.database_metrics
+ delphi.operations.screenshots
+ delphi.operations.screenshots.covidcast
+ delphi.utils
+ delphi.utils.geo
+ delphi.utils.obsolete
+ undefx.py3tester
+ undefx.undef_analysis
+
+package_dir =
+ delphi.epidata = delphi/delphi-epidata/src
+ delphi.flu_contest = delphi/flu-contest/src
+ delphi.github_deploy_repo = delphi/github-deploy-repo/src
+ delphi.nowcast = delphi/nowcast/src
+ delphi.operations = delphi/operations/src
+ delphi.utils = delphi/utils/src
+ undefx.py3tester = undefx/py3tester/src
+ undefx.undef_analysis = undefx/undef-analysis
From 4fff03c83f1c4a60876043ededa314b3f81285c3 Mon Sep 17 00:00:00 2001
From: Kathryn Mazaitis <krivard@cs.cmu.edu>
Date: Thu, 20 Oct 2022 17:05:34 -0400
Subject: [PATCH 04/10] Update covid_hosp endpoint documentation
* describe forward-fill and annual reporting limitations
* correct output documentation for facility lookup
* link to HHS FAQ, which has better information on all fields
---
docs/api/covid_hosp.md | 26 ++++++++++++++++++++------
docs/api/covid_hosp_facility.md | 9 ++++++++-
docs/api/covid_hosp_facility_lookup.md | 12 ++++++++++--
3 files changed, 38 insertions(+), 9 deletions(-)
diff --git a/docs/api/covid_hosp.md b/docs/api/covid_hosp.md
index 0bbb3be85..078c8c1d2 100644
--- a/docs/api/covid_hosp.md
+++ b/docs/api/covid_hosp.md
@@ -11,11 +11,19 @@ Hospital Capacity by State" datasets provided by the US Department of
Health & Human Services via healthdata.gov. The latter provides more frequent updates,
so it is combined with the former to create a single dataset which is as recent as possible.
+HHS performs up to four days of forward-fill for missing values. One day of forward-fill
+is common, affecting 28 states in each issue on average. More than one day of forward-fill
+is rare, affecting one state for every five issues on average.
+
+Starting October 1, 2022, some facilities are only required to report annually.
+
For more information, see the
[official description and data dictionary at healthdata.gov](https://healthdata.gov/Hospital/COVID-19-Reported-Patient-Impact-and-Hospital-Capa/g62h-syeh)
for "COVID-19 Reported Patient Impact and Hospital Capacity by State Timeseries,"
as well as the [official description](https://healthdata.gov/dataset/COVID-19-Reported-Patient-Impact-and-Hospital-Capa/6xf2-c3ie)
-for "COVID-19 Reported Patient Impact and Hospital Capacity by State."
+for "COVID-19 Reported Patient Impact and Hospital Capacity by State." The data elements,
+cadence, and how the data are being used in the federal response are documented in
+[a FAQ published by Health & Human Services](https://www.hhs.gov/sites/default/files/covid-19-faqs-hospitals-hospital-laboratory-acute-care-facility-data-reporting.pdf).
General topics not specific to any particular data source are discussed in the
[API overview](README.md). Such topics include:
@@ -65,7 +73,7 @@ If `issues` is not specified, then the most recent issue is used by default.
| `epidata[].state` | state pertaining to this row | string |
| `epidata[].date` | date pertaining to this row | integer |
| `epidata[].issue` | the date on which the dataset containing this row was published | integer |
-| `epidata[].*` | see the [data dictionary](https://healthdata.gov/covid-19-reported-patient-impact-and-hospital-capacity-state-data-dictionary) | |
+| `epidata[].*` | see the [data dictionary](https://healthdata.gov/Hospital/COVID-19-Reported-Patient-Impact-and-Hospital-Capa/g62h-syeh). Last synced: 2021-10-21 | |
| `message` | `success` or error message | string |
# Example URLs
@@ -149,25 +157,31 @@ The following sample shows how to import the library and fetch MA on 2020-05-10
### Python
Optionally install the package using pip(env):
-````bash
+```bash
pip install delphi-epidata
-````
+```
Otherwise, place `delphi_epidata.py` from this repo next to your python script.
-````python
+```python
# Import
from delphi_epidata import Epidata
# Fetch data
res = Epidata.covid_hosp('MA', 20200510)
print(res['result'], res['message'], len(res['epidata']))
-````
+```
# Repair Log
If we ever need to repair the data record due to a bug in our code (not at the
source), we will update the list below.
+## October 21, 2021
+
+All issues between 20210430 and 20211021 were re-uploaded to include new columns added by
+HHS. If you pulled these issues before October 21, the data you received was correct, but
+was missing the added columns.
+
## January 22, 2021
The following issues were repaired:
diff --git a/docs/api/covid_hosp_facility.md b/docs/api/covid_hosp_facility.md
index b8cd1ec25..70c386cad 100644
--- a/docs/api/covid_hosp_facility.md
+++ b/docs/api/covid_hosp_facility.md
@@ -9,9 +9,16 @@ This data source is a mirror of the "COVID-19 Reported Patient Impact and
Hospital Capacity by Facility" dataset provided by the US Department of Health
& Human Services via healthdata.gov.
+HHS performs up to four days of forward-fill for missing values.
+
+Starting October 1, 2022, some facilities are only required to report annually.
+
See the
[official description and data dictionary at healthdata.gov](https://healthdata.gov/Hospital/COVID-19-Reported-Patient-Impact-and-Hospital-Capa/anag-cw7u)
-for more information.
+for more information. The data elements, cadence, and how the data are being used in the
+federal response are documented in
+[a FAQ published by Health & Human Services](https://www.hhs.gov/sites/default/files/covid-19-faqs-hospitals-hospital-laboratory-acute-care-facility-data-reporting.pdf).
+
General topics not specific to any particular data source are discussed in the
[API overview](README.md). Such topics include:
diff --git a/docs/api/covid_hosp_facility_lookup.md b/docs/api/covid_hosp_facility_lookup.md
index 39424f228..b8c68d8ce 100644
--- a/docs/api/covid_hosp_facility_lookup.md
+++ b/docs/api/covid_hosp_facility_lookup.md
@@ -54,8 +54,16 @@ supported.
| --- | --- | --- |
| `result` | result code: 1 = success, 2 = too many results, -2 = no results | integer |
| `epidata` | list of results | array of objects |
-| `epidata[].hospital_pk` | facility identified by this row | string |
-| `epidata[].*` | see the [data dictionary](https://healthdata.gov/covid-19-reported-patient-impact-and-hospital-capacity-facility-data-dictionary) | |
+| `epidata[].hospital_pk` | unique identifier for this facility (will match CCN if CCN exists) | string |
+| `epidata[].state` | two-letter state code | string |
+| `epidata[].ccn` | CMS Certification Number for this facility | string |
+| `epidata[].hospital_name` | | string |
+| `epidata[].address` | | string |
+| `epidata[].city` | | string |
+| `epidata[].zip` | 5-digit ZIP code | string |
+| `epidata[].hospital_subtype` | one of: Childrens Hospitals, Critical Access Hospitals, Long Term, Psychiatric, Rehabilitation, Short Term | string |
+| `epidata[].fips_code` | 5-digit FIPS county code | string |
+| `epidata[].is_metro_micro` | 1 if this facility serves a metropolitan or micropolitan area, 0 otherwise | integer |
| `message` | `success` or error message | string |
# Example URLs
From e88ecb8498e7963f227b5aca3435e03402429962 Mon Sep 17 00:00:00 2001
From: Kathryn Mazaitis <krivard@cs.cmu.edu>
Date: Thu, 27 Oct 2022 17:10:19 -0400
Subject: [PATCH 05/10] Add suggestions from code review
---
docs/api/covid_hosp.md | 11 +++++++----
docs/api/covid_hosp_facility_lookup.md | 9 ++++++---
2 files changed, 13 insertions(+), 7 deletions(-)
diff --git a/docs/api/covid_hosp.md b/docs/api/covid_hosp.md
index 078c8c1d2..0ed67b2bf 100644
--- a/docs/api/covid_hosp.md
+++ b/docs/api/covid_hosp.md
@@ -11,9 +11,12 @@ Hospital Capacity by State" datasets provided by the US Department of
Health & Human Services via healthdata.gov. The latter provides more frequent updates,
so it is combined with the former to create a single dataset which is as recent as possible.
-HHS performs up to four days of forward-fill for missing values. One day of forward-fill
-is common, affecting 28 states in each issue on average. More than one day of forward-fill
-is rare, affecting one state for every five issues on average.
+HHS performs up to four days of forward-fill for missing values in the
+[facility-level data](covid_hosp_facility.md) which are aggregated to make this
+state-level dataset. This sometimes results in repeated values in the state-level data.
+A sequence of two repeated values is extremely common, and longer sequences are rare.
+Repeated values added in this way are sometimes updated if the underlying missing data can
+be completed at a later date.
Starting October 1, 2022, some facilities are only required to report annually.
@@ -73,7 +76,7 @@ If `issues` is not specified, then the most recent issue is used by default.
| `epidata[].state` | state pertaining to this row | string |
| `epidata[].date` | date pertaining to this row | integer |
| `epidata[].issue` | the date on which the dataset containing this row was published | integer |
-| `epidata[].*` | see the [data dictionary](https://healthdata.gov/Hospital/COVID-19-Reported-Patient-Impact-and-Hospital-Capa/g62h-syeh). Last synced: 2021-10-21 | |
+| `epidata[].*` | see the [data dictionary](https://healthdata.gov/Hospital/COVID-19-Reported-Patient-Impact-and-Hospital-Capa/g62h-syeh). Last synced: 2022-10-21 | |
| `message` | `success` or error message | string |
# Example URLs
diff --git a/docs/api/covid_hosp_facility_lookup.md b/docs/api/covid_hosp_facility_lookup.md
index b8c68d8ce..3a40dfd55 100644
--- a/docs/api/covid_hosp_facility_lookup.md
+++ b/docs/api/covid_hosp_facility_lookup.md
@@ -57,15 +57,18 @@ supported.
| `epidata[].hospital_pk` | unique identifier for this facility (will match CCN if CCN exists) | string |
| `epidata[].state` | two-letter state code | string |
| `epidata[].ccn` | CMS Certification Number for this facility | string |
-| `epidata[].hospital_name` | | string |
-| `epidata[].address` | | string |
-| `epidata[].city` | | string |
+| `epidata[].hospital_name` | facility name | string |
+| `epidata[].address` | facility address | string |
+| `epidata[].city` | facility city | string |
| `epidata[].zip` | 5-digit ZIP code | string |
| `epidata[].hospital_subtype` | one of: Childrens Hospitals, Critical Access Hospitals, Long Term, Psychiatric, Rehabilitation, Short Term | string |
| `epidata[].fips_code` | 5-digit FIPS county code | string |
| `epidata[].is_metro_micro` | 1 if this facility serves a metropolitan or micropolitan area, 0 otherwise | integer |
| `message` | `success` or error message | string |
+Use the `hospital_pk` value when querying
+[the COVID-19 Reported Patient Impact and Hospital Capacity by Facility endpoint](covid_hosp_facility.md).
+
# Example URLs
### Lookup facilities in the city of Southlake (TX)
From 40b073f7bdf84356fb35b514c892e9637238fc6b Mon Sep 17 00:00:00 2001
From: Kathryn Mazaitis <krivard@cs.cmu.edu>
Date: Thu, 27 Oct 2022 17:17:20 -0400
Subject: [PATCH 06/10] Update facility-wise data dictionary link
---
docs/api/covid_hosp_facility.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/api/covid_hosp_facility.md b/docs/api/covid_hosp_facility.md
index 70c386cad..bad1e1690 100644
--- a/docs/api/covid_hosp_facility.md
+++ b/docs/api/covid_hosp_facility.md
@@ -81,7 +81,7 @@ has been renamed here for clarity.
| `epidata[].hospital_pk` | facility identified by this row | string |
| `epidata[].collection_week` | Friday's date in the week pertaining to this row | integer |
| `epidata[].publication_date` | the date on which the dataset containing this row was published | integer |
-| `epidata[].*` | see the [data dictionary](https://healthdata.gov/covid-19-reported-patient-impact-and-hospital-capacity-facility-data-dictionary) | |
+| `epidata[].*` | see the [data dictionary](https://healthdata.gov/Hospital/COVID-19-Reported-Patient-Impact-and-Hospital-Capa/anag-cw7u) | |
| `message` | `success` or error message | string |
# Example URLs
From 9d97492c671979a25931d256b9dc0de975129d1f Mon Sep 17 00:00:00 2001
From: Kathryn Mazaitis <krivard@cs.cmu.edu>
Date: Thu, 27 Oct 2022 17:20:30 -0400
Subject: [PATCH 07/10] Correct data dictionary sync date.
(it really was october 2021; the next sync will add the ~20 columns we've been dropping on the floor since January/April 2022)
---
docs/api/covid_hosp.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/api/covid_hosp.md b/docs/api/covid_hosp.md
index 0ed67b2bf..0509cd283 100644
--- a/docs/api/covid_hosp.md
+++ b/docs/api/covid_hosp.md
@@ -76,7 +76,7 @@ If `issues` is not specified, then the most recent issue is used by default.
| `epidata[].state` | state pertaining to this row | string |
| `epidata[].date` | date pertaining to this row | integer |
| `epidata[].issue` | the date on which the dataset containing this row was published | integer |
-| `epidata[].*` | see the [data dictionary](https://healthdata.gov/Hospital/COVID-19-Reported-Patient-Impact-and-Hospital-Capa/g62h-syeh). Last synced: 2022-10-21 | |
+| `epidata[].*` | see the [data dictionary](https://healthdata.gov/Hospital/COVID-19-Reported-Patient-Impact-and-Hospital-Capa/g62h-syeh). Last synced: 2021-10-21 | |
| `message` | `success` or error message | string |
# Example URLs
From cf63a583db48c82b78a68d0910c06cf1f69fe4ca Mon Sep 17 00:00:00 2001
From: Brian Clark <clark.bg@gmail.com>
Date: Mon, 31 Oct 2022 12:41:38 -0400
Subject: [PATCH 08/10] Increase Gunicorn timeout
---
devops/gunicorn_conf.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/devops/gunicorn_conf.py b/devops/gunicorn_conf.py
index 249d793c9..e41e1c986 100644
--- a/devops/gunicorn_conf.py
+++ b/devops/gunicorn_conf.py
@@ -29,7 +29,7 @@
workers = web_concurrency
bind = use_bind
keepalive = 120
-timeout = 300
+timeout = 900
errorlog = "-"
accesslog = "-"
From 6aae869431745b40c053f190e940bb3502e43e67 Mon Sep 17 00:00:00 2001
From: Brian Clark <clark.bg@gmail.com>
Date: Tue, 1 Nov 2022 10:35:39 -0400
Subject: [PATCH 09/10] Revert "Increase Gunicorn's idle timeout"
---
devops/gunicorn_conf.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/devops/gunicorn_conf.py b/devops/gunicorn_conf.py
index e41e1c986..249d793c9 100644
--- a/devops/gunicorn_conf.py
+++ b/devops/gunicorn_conf.py
@@ -29,7 +29,7 @@
workers = web_concurrency
bind = use_bind
keepalive = 120
-timeout = 900
+timeout = 300
errorlog = "-"
accesslog = "-"
From 800b7f60627f4c57a18d5682b7b0194188c5c055 Mon Sep 17 00:00:00 2001
From: krivard <krivard@users.noreply.github.com>
Date: Thu, 3 Nov 2022 15:00:37 +0000
Subject: [PATCH 10/10] chore: release delphi-epidata 0.4.1
---
.bumpversion.cfg | 2 +-
src/client/delphi_epidata.R | 2 +-
src/client/delphi_epidata.js | 2 +-
src/client/packaging/npm/package.json | 2 +-
src/client/packaging/pypi/delphi_epidata/__init__.py | 2 +-
src/client/packaging/pypi/setup.py | 2 +-
src/server/_config.py | 2 +-
7 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 3fa399e7a..5d4ae4596 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
[bumpversion]
-current_version = 0.4.0
+current_version = 0.4.1
commit = False
tag = False
diff --git a/src/client/delphi_epidata.R b/src/client/delphi_epidata.R
index 60eef54b1..de8f78439 100644
--- a/src/client/delphi_epidata.R
+++ b/src/client/delphi_epidata.R
@@ -15,7 +15,7 @@ Epidata <- (function() {
# API base url
BASE_URL <- 'https://delphi.cmu.edu/epidata/api.php'
- client_version <- '0.4.0'
+ client_version <- '0.4.1'
# Helper function to cast values and/or ranges to strings
.listitem <- function(value) {
diff --git a/src/client/delphi_epidata.js b/src/client/delphi_epidata.js
index 991d19af3..6acd78d25 100644
--- a/src/client/delphi_epidata.js
+++ b/src/client/delphi_epidata.js
@@ -22,7 +22,7 @@
}
})(this, function (exports, fetchImpl, jQuery) {
const BASE_URL = "https://delphi.cmu.edu/epidata/";
- const client_version = "0.4.0";
+ const client_version = "0.4.1";
// Helper function to cast values and/or ranges to strings
function _listitem(value) {
diff --git a/src/client/packaging/npm/package.json b/src/client/packaging/npm/package.json
index 5e4fbf02b..71d01d05e 100644
--- a/src/client/packaging/npm/package.json
+++ b/src/client/packaging/npm/package.json
@@ -2,7 +2,7 @@
"name": "delphi_epidata",
"description": "Delphi Epidata API Client",
"authors": "Delphi Group",
- "version": "0.4.0",
+ "version": "0.4.1",
"license": "MIT",
"homepage": "https://github.com/cmu-delphi/delphi-epidata",
"bugs": {
diff --git a/src/client/packaging/pypi/delphi_epidata/__init__.py b/src/client/packaging/pypi/delphi_epidata/__init__.py
index 07cbe2b15..0c56a4242 100644
--- a/src/client/packaging/pypi/delphi_epidata/__init__.py
+++ b/src/client/packaging/pypi/delphi_epidata/__init__.py
@@ -1,4 +1,4 @@
from .delphi_epidata import Epidata
name = 'delphi_epidata'
-__version__ = '0.4.0'
+__version__ = '0.4.1'
diff --git a/src/client/packaging/pypi/setup.py b/src/client/packaging/pypi/setup.py
index f4873b4b0..68b452eea 100644
--- a/src/client/packaging/pypi/setup.py
+++ b/src/client/packaging/pypi/setup.py
@@ -5,7 +5,7 @@
setuptools.setup(
name="delphi_epidata",
- version="0.4.0",
+ version="0.4.1",
author="David Farrow",
author_email="dfarrow0@gmail.com",
description="A programmatic interface to Delphi's Epidata API.",
diff --git a/src/server/_config.py b/src/server/_config.py
index 47688a8ef..05eae46d4 100644
--- a/src/server/_config.py
+++ b/src/server/_config.py
@@ -5,7 +5,7 @@
load_dotenv()
-VERSION = "0.4.0"
+VERSION = "0.4.1"
MAX_RESULTS = int(10e6)
MAX_COMPATIBILITY_RESULTS = int(3650)