Merge pull request #1197 from cmu-delphi/release/delphi-epidata-4.1.3

Release Delphi Epidata 4.1.3

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 4.1.2
+current_version = 4.1.3
 commit = False
 tag = False
 

dev/local/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = Delphi Development
-version = 4.1.2
+version = 4.1.3
 
 [options]
 packages =

docs/api/api_keys.md

@@ -40,6 +40,14 @@ If you choose to
 [register for an API key](https://api.delphi.cmu.edu/epidata/admin/registration_form),
 there are several ways to use your key to authenticate your requests:
 
+### Using a client
+
+* covidcast
+  * [R client](https://cmu-delphi.github.io/covidcast/covidcastR/reference/covidcast_signal.html#api-keys-1)
+  * [Python client](https://cmu-delphi.github.io/covidcast/covidcast-py/html/signals.html#covidcast.use_api_key)
+* [epidatr](https://github.com/cmu-delphi/epidatr#api-keys)
+* [delphi-epidata](https://cmu-delphi.github.io/delphi-epidata/api/client_libraries.html)
+
 ### Via request parameter
 
 The request parameter “api_key” can be used to pass the API key to the server.

docs/api/client_libraries.md

@@ -4,9 +4,17 @@ parent: Other Endpoints (COVID-19 and Other Diseases)
 nav_order: 1
 ---
 
-# Epidata API Client Libraries.
+# Epidata API Client Libraries
 
-Epidata clients are available for
+For anyone looking for COVIDCast data, please visit our [COVIDCast Libraries](covidcast_clients.md).
+
+We are currently working on fully-featured Epidata clients for R and Python. They are not ready
+for release yet, but you can track our development progress and help us test them out at:
+
+* [epidatr](https://github.com/cmu-delphi/epidatr)
+* [epidatpy](https://github.com/cmu-delphi/epidatpy)
+
+In the meantime, minimalist Epidata clients remain available for
 [JavaScript](https://github.com/cmu-delphi/delphi-epidata/blob/master/src/client/delphi_epidata.js),
 [Python](https://github.com/cmu-delphi/delphi-epidata/blob/master/src/client/delphi_epidata.py),
 and
@@ -15,10 +23,10 @@ The following samples show how to import the library and fetch Delphi's COVID-19
 Surveillance Streams from Facebook Survey CLI for county 06001 and days
 `20200401` and `20200405-20200414` (11 days total).
 
-For anyone looking for COVIDCast data, please visit our [COVIDCast Libraries](covidcast_clients.md).
-
 ### JavaScript (in a web browser)
 
+The minimalist JavaScript client does not currently support API keys. If you need API key support in JavaScript, contact [email protected].
+
 ````html
 <!-- Imports -->
 <script src="delphi_epidata.js"></script>
@@ -45,6 +53,8 @@ in the same directory as your Python script.
 ````python
 # Import
 from delphi_epidata import Epidata
+# [Optional] configure your API key, if desired
+#Epidata.auth = ('epidata', <your API key>)
 # Fetch data
 res = Epidata.covidcast('fb-survey', 'smoothed_cli', 'day', 'county', [20200401, Epidata.range(20200405, 20200414)], '06001')
 print(res['result'], res['message'], len(res['epidata']))
@@ -54,6 +64,8 @@ print(res['result'], res['message'], len(res['epidata']))
 
 
 ````R
+# [Optional] configure your API key, if desired
+#option('epidata.auth', <your API key>)
 # Import
 source('delphi_epidata.R')
 # Fetch data

docs/symptom-survey/publications.md

@@ -26,15 +26,20 @@ Pandemic"](https://www.pnas.org/topic/548) in *PNAS*:
 
 Research publications using the survey data include:
 
+- GK Charles, SP Braunstein, JL Barker, et al (2023). [How do psychobehavioural
+  variables shed light on heterogeneity in COVID-19 vaccine acceptance? Evidence
+  from United States general population surveys on a probability panel and
+  social media](https://doi.org/10.1136/bmjopen-2022-066897). *BMJ Open*
+  13:e066897.
 - S. Soorapanth, R. Cheung, X. Zhang, A. H. Mokdad, G. A. Mensah (2023).
   [Rural–Urban Differences in Vaccination and Hesitancy Rates and Trust: US
   COVID-19 Trends and Impact Survey on a Social Media Platform, May 2021–April
   2022](https://doi.org/10.2105/AJPH.2023.307274). *American Journal of Public
-  Health*.
+  Health* 113 (6), 680-688.
   - See also the associated editorial: T. Callaghan (2023). [Vaccine Uptake and
     Hesitancy in Rural America in the Wake of the COVID-19
     Pandemic](https://doi.org/10.2105/AJPH.2023.307305). *American Journal of
-    Public Health*.
+    Public Health* 113 (6), 615-617.
 - M. Rubinstein, A. Haviland, and J. Breslau (2023). [The effect of COVID-19
   vaccinations on self-reported depression and anxiety during February
   2021](https://doi.org/10.1080/2330443X.2023.2190008). *Statistics and Public

src/client/delphi_epidata.R

@@ -15,7 +15,7 @@ Epidata <- (function() {
   # API base url
   BASE_URL <- getOption('epidata.url', default = 'https://api.delphi.cmu.edu/epidata/')
 
-  client_version <- '4.1.2'
+  client_version <- '4.1.3'
 
   auth <- getOption("epidata.auth", default = NA)
 

src/client/delphi_epidata.js

@@ -22,7 +22,7 @@
   }
 })(this, function (exports, fetchImpl, jQuery) {
   const BASE_URL = "https://api.delphi.cmu.edu/epidata/";
-  const client_version = "4.1.2";
+  const client_version = "4.1.3";
 
   // Helper function to cast values and/or ranges to strings
   function _listitem(value) {

src/client/packaging/npm/package.json

@@ -2,7 +2,7 @@
   "name": "delphi_epidata",
   "description": "Delphi Epidata API Client",
   "authors": "Delphi Group",
-  "version": "4.1.2",
+  "version": "4.1.3",
   "license": "MIT",
   "homepage": "https://github.com/cmu-delphi/delphi-epidata",
   "bugs": {

src/client/packaging/pypi/delphi_epidata/__init__.py

@@ -1,4 +1,4 @@
 from .delphi_epidata import Epidata
 
 name = 'delphi_epidata'
-__version__ = '4.1.2'
+__version__ = '4.1.3'

src/client/packaging/pypi/setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="delphi_epidata",
-    version="4.1.2",
+    version="4.1.3",
     author="David Farrow",
     author_email="[email protected]",
     description="A programmatic interface to Delphi's Epidata API.",

src/server/_common.py

@@ -8,7 +8,7 @@
 from werkzeug.local import LocalProxy
 
 from delphi.epidata.common.logger import get_structured_logger
-from ._config import SECRET, REVERSE_PROXIED
+from ._config import SECRET, REVERSE_PROXY_DEPTH
 from ._db import engine
 from ._exceptions import DatabaseErrorException, EpiDataException
 from ._security import current_user, _is_public_route, resolve_auth_token, show_no_api_key_warning, update_key_last_time_used, ERROR_MSG_INVALID_KEY
@@ -31,14 +31,29 @@ def _get_db() -> Connection:
 
 
 def get_real_ip_addr(req):  # `req` should be a Flask.request object
-    if REVERSE_PROXIED:
-        # NOTE: ONLY trust these headers if reverse proxied!!!
+    if REVERSE_PROXY_DEPTH:
+        # we only expect/trust (up to) "REVERSE_PROXY_DEPTH" number of proxies between this server and the outside world.
+        # a REVERSE_PROXY_DEPTH of 0 means not proxied, i.e. server is globally directly reachable.
+        # a negative proxy depth is a special case to trust the whole chain -- not generally recommended unless the
+        # most-external proxy is configured to disregard "X-Forwarded-For" from outside.
+        # really, ONLY trust the following headers if reverse proxied!!!
         if "X-Forwarded-For" in req.headers:
-            return req.headers["X-Forwarded-For"].split(",")[
-                0
-            ]  # take the first (or only) address from the comma-sep list
+            full_proxy_chain = req.headers["X-Forwarded-For"].split(",")
+            # eliminate any extra addresses at the front of this list, as they could be spoofed.
+            if REVERSE_PROXY_DEPTH > 0:
+                depth = REVERSE_PROXY_DEPTH
+            else:
+                # special case for -1/negative: setting `depth` to 0 will not strip any items from the chain
+                depth = 0
+            trusted_proxy_chain = full_proxy_chain[-depth:]
+            # accept the first (or only) address in the remaining trusted part of the chain as the actual remote address
+            return trusted_proxy_chain[0].strip()
+
+        # fall back to "X-Real-Ip" if "X-Forwarded-For" isnt present
         if "X-Real-Ip" in req.headers:
             return req.headers["X-Real-Ip"]
+
+    # if we are not proxied (or we are proxied but the headers werent present and we fell through to here), just use the remote ip addr as the true client address
     return req.remote_addr
 
 

src/server/_config.py

@@ -7,7 +7,7 @@
 
 load_dotenv()
 
-VERSION = "4.1.2"
+VERSION = "4.1.3"
 
 MAX_RESULTS = int(10e6)
 MAX_COMPATIBILITY_RESULTS = int(3650)
@@ -26,14 +26,41 @@
 SECRET = os.environ.get("FLASK_SECRET", "secret")
 URL_PREFIX = os.environ.get("FLASK_PREFIX", "") # if not empty, this value should begin but not end in a slash ('/')
 
-# REVERSE_PROXIED is a boolean value that indicates whether or not this server instance
-# is running behind a reverse proxy (like nginx).
-# in dev and testing, it is fine (or even preferable) for this variable to be set to 'TRUE'
-# even if it is not actually the case.  in prod, it is very important that this is set accurately --
-# it should _only_ be set to 'TRUE' if it really is behind a reverse proxy, as remote addresses can be "spoofed"
-# which can carry security/identity implications.  conversely, if this is set to 'FALSE' when in fact
-# running behind a reverse proxy, it can hinder logging accuracy.  it defaults to 'FALSE' for safety.
-REVERSE_PROXIED = os.environ.get("REVERSE_PROXIED", "FALSE").upper() == "TRUE"
+
+"""
+REVERSE_PROXY_DEPTH is an integer value that indicates how many "chained" and trusted reverse proxies (like nginx) this
+ server instance is running behind.  "chained" refers to proxies forwarding to other proxies, and then ultimately
+ forwarding to the app server itself.  each of these proxies appends the remote address of the request to the
+ "X-Forwarded-For" header.  in many situations, the most externally facing proxy (the first in the chain, the one that
+ faces the "open internet") can and should be set to write its own "X-Forwarded-For" header, ignoring and replacing
+ (or creating anew, if it didnt exist) such a header from the client request -- thus preserving the chain of trusted
+ proxies under our control.
+
+however, in our typical production environment, the most externally facing "proxy" is the AWS application load balancer,
+ which seemingly cannot be configured to provide this trust boundary without losing the referring client address
+ (see: https://docs.aws.amazon.com/elasticloadbalancing/latest/application/x-forwarded-headers.html ).  accordingly, in
+ our current typical production environment, REVERSE_PROXY_DEPTH should be set to "2": one for the AWS application load
+ balancer, and one for our own nginx/haproxy instance.  thus "2" is our default value.
+
+it is important that REVERSE_PROXY_DEPTH is set accurately for two reasons...
+
+setting it too high (or to -1) will respect more of the entries in the "X-Forwarded-For" header than are appropriate.
+ this can allow remote addresses to be "spoofed" when a client fakes this header, carrying security/identity
+ implications.  in dev and testing, it is not particularly dangerous for this variable to be set to -1 (special case
+ for an "infinite" depth, where any and all proxy hops will be trusted).
+
+setting it too low can hinder logging accuracy -- that can cause an intermediate proxy IP address to be used as the
+ "real" client IP address, which could cause requests to be rate-limited inappropriately.
+
+setting REVERSE_PROXY_DEPTH to "0" essentially indicates there are no proxies between this server and the outside
+ world.  in this case, the "X-Forwarded-For" header is ignored.
+"""
+REVERSE_PROXY_DEPTH = int(os.environ.get("PROXY_DEPTH", 4))
+# TODO: ^ this value should be "4" for the prod CC API server processes, and is currently unclear
+#       for prod AWS API server processes (but should be the same or lower)...  when thats properly
+#       determined, set the default to the minimum of the two environments and special case the
+#       other in conf file(s).
+
 
 REGION_TO_STATE = {
     "hhs1": ["VT", "CT", "ME", "MA", "NH", "RI"],

src/server/admin/models.py

@@ -19,15 +19,17 @@
     Column("role_id", ForeignKey("user_role.id")),
 )
 
+def _default_date_now():
+    return dtime.strftime(dtime.now(), "%Y-%m-%d")
 
 class User(Base):
     __tablename__ = "api_user"
     id = Column(Integer, primary_key=True, autoincrement=True)
     roles = relationship("UserRole", secondary=association_table)
     api_key = Column(String(50), unique=True, nullable=False)
     email = Column(String(320), unique=True, nullable=False)
-    created = Column(Date, default=dtime.strftime(dtime.now(), "%Y-%m-%d"))
-    last_time_used = Column(Date, default=dtime.strftime(dtime.now(), "%Y-%m-%d"))
+    created = Column(Date, default=_default_date_now)
+    last_time_used = Column(Date, default=_default_date_now)
 
     def __init__(self, api_key: str, email: str = None) -> None:
         self.api_key = api_key

src/server/endpoints/admin.py

@@ -5,6 +5,7 @@
 from werkzeug.exceptions import NotFound, Unauthorized
 from werkzeug.utils import redirect
 
+from .._common import log_info_with_request
 from .._config import ADMIN_PASSWORD, API_KEY_REGISTRATION_FORM_LINK, API_KEY_REMOVAL_REQUEST_LINK, REGISTER_WEBHOOK_TOKEN
 from .._security import resolve_auth_token
 from ..admin.models import User, UserRole
@@ -44,6 +45,24 @@ def user_exists(user_email: str = None, api_key: str = None):
     return True if user else False
 
 
+# ~~~~ PUBLIC ROUTES ~~~~
+
+
+@bp.route("/registration_form", methods=["GET"])
+def registration_form_redirect():
+    # TODO: replace this with our own hosted registration form instead of external
+    return redirect(API_KEY_REGISTRATION_FORM_LINK, code=302)
+
+
+@bp.route("/removal_request", methods=["GET"])
+def removal_request_redirect():
+    # TODO: replace this with our own hosted form instead of external
+    return redirect(API_KEY_REMOVAL_REQUEST_LINK, code=302)
+
+
+# ~~~~ PRIVLEGED ROUTES ~~~~
+
+
 @bp.route("/", methods=["GET", "POST"])
 def _index():
     token = _require_admin()
@@ -88,21 +107,6 @@ def _detail(user_id: int):
     return _render("detail", token, flags, user=user.as_dict)
 
 
-def register_new_key(api_key: str, email: str) -> str:
-    User.create_user(api_key=api_key, email=email)
-    return api_key
-
-
-@bp.route("/registration_form", methods=["GET"])
-def registration_form_redirect():
-    # TODO: replace this with our own hosted registration form instead of external
-    return redirect(API_KEY_REGISTRATION_FORM_LINK, code=302)
-
-@bp.route("/removal_request", methods=["GET"])
-def removal_request_redirect():
-    # TODO: replace this with our own hosted form instead of external
-    return redirect(API_KEY_REMOVAL_REQUEST_LINK, code=302)
-
 @bp.route("/register", methods=["POST"])
 def _register():
     body = request.get_json()
@@ -117,5 +121,16 @@ def _register():
             "User with email and/or API Key already exists, use different parameters or contact us for help",
             409,
         )
-    api_key = register_new_key(user_api_key, user_email)
-    return make_response(f"Successfully registered API key '{api_key}'", 200)
+    User.create_user(api_key=user_api_key, email=user_email)
+    return make_response(f"Successfully registered API key '{user_api_key}'", 200)
+
+
+@bp.route("/diagnostics", methods=["GET", "PUT", "POST", "DELETE"])
+def diags():
+    # allows us to get useful diagnostic information written into server logs,
+    # such as a full current "X-Forwarded-For" path as inserted into headers by intermediate proxies...
+    # (but only when initiated purposefully by us to keep junk out of the logs)
+    _require_admin()
+    log_info_with_request("diagnostics", headers=request.headers)
+    response_text = f"request path: {request.headers.get('X-Forwarded-For', 'idk')}"
+    return make_response(response_text, 200, {'content-type': 'text/plain'})