Merge pull request #424 from openaps/dev

PR for dev to master in prep for 0.5.0 release


    Improved oref0-setup.sh experience (colors, etc.)
        AMA is always on by default - no longer prompts in the script
        Autotune and autosens moved out of advanced features menu
        max_IOB will be prompted for
    oref1 features (SMB, UAM) included, and can be set up via oref0-setup.sh under advanced features
        oref1-related pushover is also under advanced features
        oref1 also includes insulinReq and carbsReq, which will show in the pill even if Pushover not enabled
        SMB requires autotune folder to be present on the rig
    oref0-setup installs bluez 5.44 if needed unless version 5.37 or newer is present
    oref0-online/wifi related improvements for connectivity
    oref0-reset-git and logrotate improvements to help with full disks
    oref0-set-device-clocks if pump clock is >1m off
    oref0-set-system-clock if determine-basal complains it's set incorrectly
    Bolus wizard carb entries without a bolus will now be ignored
    Fix for zombie (reappearing) carbs
    Assume remainingCarbs (carbs we’re not yet seeing deviations for) will absorb over 4h
    Various improvements for WW pumps
    Other feature additions:
        Rewind_resets_autosens resets autosensitivity to neutral for a while after each pump rewind
        Autosens_adjust_targets: when autosens detects sensitivity/resistance, also adjust BG target accordingly
        adv_target_adjustments: lowers target automatically when BG and eventualBG are high
        unsuspend_if_no_temp: if enabled, pump will un-suspend after a zero temp finishes
        autotune_isf_adjustmentFraction: keeps autotune ISF closer to pump ISF via a weighted average of fullNewISF and pumpISF.
        allow non-SMB code paths to set temps up to 120 minutes when needed
        Nightscout token auth

.gitignore

@@ -20,3 +20,4 @@ static/bower_components/
 coverage/
 
 npm-debug.log
+bin/__pycache__

Makefile

@@ -8,7 +8,7 @@ all: test
 report:
  # report results to community
 test:
- ./node_modules/.bin/mocha ${TESTS}
+ ./node_modules/.bin/mocha -c ${TESTS}
 
 travis:
  ${ISTANBUL} cover ${MOCHA} --include-all-sources true --report lcovonly -- -R tap ${TESTS}

bin/mm-format-ns-pump-history.sh

@@ -24,11 +24,10 @@ OUTPUT=${2-/dev/fd/1}
 #TZ=${3-$(date +%z)}
 
 cat $HISTORY | \
- json -e "this.medtronic = this._type;" | \
- #json -e "this.dateString = this.timestamp + '$(TZ=TZ date +%z)'" | \
- json -e "this.dateString = this.timestamp + '$(date +%z)'" | \
- json -e "this.type = 'medtronic'" | \
- json -e "this.date = this.date ? this.date : new Date(Date.parse(this.dateString)).getTime( )" \
+ json -e "this.medtronic = this._type;" \
+ -e "this.dateString = this.timestamp + '$(date +%z)'" \
+ -e "this.type = 'medtronic'" \
+ -e "this.date = this.date ? this.date : new Date(Date.parse(this.dateString)).getTime( )" \
  > $OUTPUT
 
 

bin/mm-format-ns-treatments.sh

@@ -27,12 +27,11 @@ model=$(json -f $MODEL)
 
 oref0-normalize-temps $HISTORY \
  | json -e "this.medtronic = 'mm://openaps/$self/' + (this._type || this.eventType);" \
- | json -e "this.created_at = this.created_at ? this.created_at : this.timestamp" \
- | json -e "this.enteredBy = 'openaps://medtronic/$model'" \
- | json -e "if (this.glucose && !this.glucoseType && this.glucose > 0) { this.glucoseType = this.enteredBy }" \
- | json -e "this.eventType = (this.eventType ? this.eventType : 'Note')" \
- | json -e "if (this._type == 'AlarmSensor' && this.alarm_description) {this.notes = this.alarm_description}" \
- | json -e "if (this.eventType == 'Note' && !this.alarm_description) { this.notes = this._type + ' $model ' + (this.notes ? this.notes : '')}" \
- | json > $OUTPUT
-
+ -e "this.created_at = this.created_at ? this.created_at : this.timestamp" \
+ -e "this.enteredBy = 'openaps://medtronic/$model'" \
+ -e "if (this.glucose && !this.glucoseType && this.glucose > 0) { this.glucoseType = this.enteredBy }" \
+ -e "this.eventType = (this.eventType ? this.eventType : 'Note')" \
+ -e "if (this._type == 'AlarmSensor' && this.alarm_description) {this.notes = this.alarm_description}" \
+ -e "if (this.eventType == 'Note' && !this.alarm_description) { this.notes = this._type + ' $model ' + (this.notes ? this.notes : '')}" \
+ > $OUTPUT
 

bin/nightscout.sh

@@ -319,7 +319,11 @@ hash-api-secret)
 autoconfigure-device-crud)
  NIGHTSCOUT_HOST=$1
  PLAIN_NS_SECRET=$2
- API_SECRET=$($self hash-api-secret $2)
+ if [[ "${PLAIN_NS_SECRET,,}" =~ "token=" ]]; then
+ API_SECRET=$2 # store token as API_SECRET
+ else
+ API_SECRET=$($self hash-api-secret $2)
+ fi
  case $1 in
  help|-h|--help)
  setup_help
@@ -331,8 +335,8 @@ autoconfigure-device-crud)
  test -z "$API_SECRET" && setup_help && exit 1;
  openaps device add ns process --require "oper" nightscout ns "NIGHTSCOUT_HOST" "API_SECRET"
  openaps device show ns --json | json \
- | json -e "this.extra.args = this.extra.args.replace(' NIGHTSCOUT_HOST ', ' $NIGHTSCOUT_HOST ')" \
- | json -e "this.extra.args = this.extra.args.replace(' API_SECRET', ' $API_SECRET')" \
+ -e "this.extra.args = this.extra.args.replace(' NIGHTSCOUT_HOST ', ' $NIGHTSCOUT_HOST ')" \
+ -e "this.extra.args = this.extra.args.replace(' API_SECRET', ' $API_SECRET')" \
  | openaps import
  ;;
 cull-latest-openaps-treatments)
@@ -348,4 +352,4 @@ help|--help|-h)
 *)
  test -n "$COMMAND" && exec $COMMAND $*
  ;;
-esac
+esac

bin/ns-dedupe-treatments.sh

@@ -11,7 +11,7 @@ EOF
 }
 
 function fetch ( ) {
- curl -s -g $ENDPOINT.json
+ curl --compressed -s -g $ENDPOINT.json
 }
 
 function flatten ( ) {
@@ -22,7 +22,7 @@ function flatten ( ) {
 function find_dupes_on ( ) {
  count=$1
  date=$2
- test $count -gt 1 && curl -g -s ${ENDPOINT}.json"?count=$(($count-1))&find[created_at]=$date"
+ test $count -gt 1 && curl --compressed -g -s ${ENDPOINT}.json"?count=$(($count-1))&find[created_at]=$date"
 }
 function debug_cmd ( ) {
 tid=$1
@@ -45,8 +45,8 @@ export NIGHTSCOUT_HOST ENDPOINT
 fetch | flatten | while read count date; do
  test $count -gt 1 && echo "{}" \
  | json -e "this.count = $count" \
- | json -e "this.date = '$date'" \
- | json -e "this.created_at = '$date'"
+ -e "this.date = '$date'" \
+ -e "this.created_at = '$date'"
 done | json -g
 }
 

bin/ns-get.sh

@@ -8,7 +8,7 @@ NIGHTSCOUT_HOST=${NIGHTSCOUT_HOST-${2-localhost:1337}}
 QUERY=${3}
 OUTPUT=${4-/dev/fd/1}
 
-CURL_FLAGS="-g -s"
+CURL_FLAGS="--compressed -g -s"
 NIGHTSCOUT_FORMAT=${NIGHTSCOUT_FORMAT-json}
 test "$NIGHTSCOUT_DEBUG" = "1" && CURL_FLAGS="${CURL_FLAGS} -iv"
 test "$NIGHTSCOUT_DEBUG" = "1" && set -x
@@ -20,42 +20,47 @@ Usage: $self <entries.json> [NIGHTSCOUT_HOST|localhost:1337] [QUERY] [stdout|-]
 $self type <entries.json> <NIGHTSCOUT_HOST|localhost:1337] [QUERY] [stdout|-]
 $self host <NIGHTSCOUT_HOST|localhost:1337> <entries.json> [QUERY] [stdout|-]
 
-$self --config <device> <entries.json> <NIGHTSCOUT_HOST> <monitor/entries.json>
 EOF
 }
 
-REPORT_ENDPOINT=$NIGHTSCOUT_HOST/api/v1/${REPORT}'?'${QUERY}
+# use token authentication if the user has a token set in their API_SECRET environment variable
+if [[ "${API_SECRET,,}" =~ "token=" ]]; then
+ if [[ -z ${QUERY} ]]; then
+ REPORT_ENDPOINT=$NIGHTSCOUT_HOST/api/v1/${REPORT}'?'${API_SECRET}
+ else
+ REPORT_ENDPOINT=$NIGHTSCOUT_HOST/api/v1/${REPORT}'?'${API_SECRET}'&'${QUERY}
+ fi
+else
+ REPORT_ENDPOINT=$NIGHTSCOUT_HOST/api/v1/${REPORT}'?'${QUERY}
+fi
+
 case $1 in
- --config)
- test -z $2 && echo "Device name missing." && usage && exit 1;
- devicename=${2-${self}}
- test -z $3 && echo "Type is missing" && usage && exit 1;
- # echo openaps device add $2 process $self $3
- cat <<EOF
-openaps device add $devicename process $self $3 $self-NIGHTSCOUT_HOST
-sed -i -e "s/$self-NIGHTSCOUT_HOST/$4/g" $devicename.ini
-openaps report add $5 text $devicename shell 
-EOF
- exit 0;
- ;;
  host)
  # $self
  NIGHTSCOUT_HOST=${NIGHTSCOUT_HOST-${2-localhost:1337}}
  REPORT=${3-entries.json}
  QUERY=${4}
  OUTPUT=${5-/dev/fd/1}
- REPORT_ENDPOINT=$NIGHTSCOUT_HOST/api/v1/${REPORT}'?'${QUERY}
+
+ # use token authentication if the user has a token set in their API_SECRET environment variable
+ if [[ "${API_SECRET,,}" =~ "token=" ]]; then
+ if [[ -z ${QUERY} ]]; then
+ REPORT_ENDPOINT=$NIGHTSCOUT_HOST/api/v1/${REPORT}'?'${API_SECRET}
+ else
+ REPORT_ENDPOINT=$NIGHTSCOUT_HOST/api/v1/${REPORT}'?'${API_SECRET}'&'${QUERY}
+ fi
+ else
+ REPORT_ENDPOINT=$NIGHTSCOUT_HOST/api/v1/${REPORT}'?'${QUERY}
+ fi
  test -z "$NIGHTSCOUT_HOST" && usage && exit 1;
+
  curl ${CURL_FLAGS} $REPORT_ENDPOINT | $NIGHTSCOUT_FORMAT
 
  ;;
  type)
  shift
  exec $self $*
  ;;
- --noop)
- echo "curl -s $REPORT_ENDPOINT | json"
- ;;
  help|--help|-h)
  usage
  ;;

bin/ns-upload-entries.sh

@@ -6,7 +6,6 @@
 self=$(basename $0)
 ENTRIES=${1-entries.json}
 NIGHTSCOUT_HOST=${NIGHTSCOUT_HOST-localhost:1337}
-#TZ=${3-$(date +%z)}
 OUTPUT=${2}
 
 function usage ( ) {
@@ -24,11 +23,21 @@ case "$1" in
 esac
 
 export ENTRIES API_SECRET NIGHTSCOUT_HOST
+
+# use token authentication if the user has a token set in their API_SECRET environment variable
+if [[ "${API_SECRET,,}" =~ "token=" ]]; then
+ API_SECRET_HEADER=""
+ REST_ENDPOINT="${NIGHTSCOUT_HOST}/api/v1/entries.json?${API_SECRET}"
+else
+ REST_ENDPOINT="${NIGHTSCOUT_HOST}/api/v1/entries.json"
+ API_SECRET_HEADER='-H "API-SECRET: ${API_SECRET}"'
+fi
+
+
 # requires API_SECRET and NIGHTSCOUT_HOST to be set in calling environment (i.e. in crontab)
 (
-curl -m 30 -s -X POST --data-binary @$ENTRIES \
- -H "API-SECRET: $API_SECRET" \
- -H "content-type: application/json" \
- $NIGHTSCOUT_HOST/api/v1/entries.json
-) && ( test -n "$OUTPUT" && touch $OUTPUT ; logger "Uploaded $ENTRIES to $NIGHTSCOUT_HOST" ) || logger "Unable to upload to $NIGHTSCOUT_HOST"
+curl -m 30 -s -X POST --data-binary @${ENTRIES} \
+ ${API_SECRET_HEADER} -H "content-type: application/json" \
+ ${REST_ENDPOINT}
+) && ( test -n "${OUTPUT}" && touch ${OUTPUT} ; logger "Uploaded ${ENTRIES} to ${NIGHTSCOUT_HOST}" ) || logger "Unable to upload to ${NIGHTSCOUT_HOST}"
 

bin/ns-upload.sh

@@ -7,7 +7,6 @@ NIGHTSCOUT_HOST=${NIGHTSCOUT_HOST-${1-localhost:1337}}
 API_SECRET=${2-${API_SECRET}}
 TYPE=${3-entries.json}
 ENTRIES=${4-entries.json}
-#TZ=${3-$(date +%z)}
 OUTPUT=${5}
 
 REST_ENDPOINT="${NIGHTSCOUT_HOST}/api/v1/${TYPE}"
@@ -16,32 +15,19 @@ function usage ( ) {
 cat <<EOF
 Usage: $self <NIGHTSCOUT_HOST|localhost:1337> <API_SECRET> [API-TYPE|entries.json] <monitor/entries-to-upload.json> [stdout|-]
 
-$self --config <NIGHTSCOUT_HOST> <PLAIN_API_SECRET> <API-TYPE|entries.json> <monitor/entries-to-upload.json> output-report.json
-
 $self help - This message.
 EOF
 }
 
 case $1 in
- --config)
- test -z $3 && usage && exit 1;
- # echo openaps device add $2 process $self $3
-API_SECRET=$(echo -n $3 | sha1sum | cut -d ' ' -f 1 | tr -d "\n")
- cat <<EOF
-openaps device add $self process --require "type report" $self "$self-NIGHTSCOUT" "$self-APIKEY"
-sed -i -e "s/$self-NIGHTSCOUT/$2/g" $self.ini
-sed -i -e "s/$self-APIKEY/$API_SECRET/g" $self.ini
-openaps report add $6 text $self shell "$4" "$5"
-EOF
- exit 0;
- ;;
  help)
  usage
  ;;
  *)
- # curl -s $REPORT_ENDPOINT | json
+ # curl -s ${REPORT_ENDPOINT} | json
  ;;
 esac
+
 export ENTRIES API_SECRET NIGHTSCOUT_HOST REST_ENDPOINT
 if [[ -z $API_SECRET ]] ; then
  echo "$self: missing API_SECRET"
@@ -60,10 +46,20 @@ if [[ "$ENTRIES" != "-" ]] ; then
  exit 1;
  fi
 fi
-(test "$ENTRIES" != "-" && cat $ENTRIES || cat )| (
-curl -m 30 -s -X POST --data-binary @- \
- -H "API-SECRET: $API_SECRET" \
- -H "content-type: application/json" \
- $REST_ENDPOINT
-) && ( test -n "$OUTPUT" && touch $OUTPUT ; logger "Uploaded $ENTRIES to $NIGHTSCOUT_HOST" ) || logger "Unable to upload to $NIGHTSCOUT_HOST"
 
+# use token authentication if the user has a token set in their API_SECRET environment variable
+if [[ "${API_SECRET,,}" =~ "token=" ]]; then
+ REST_ENDPOINT="${REST_ENDPOINT}?${API_SECRET}"
+ (test "$ENTRIES" != "-" && cat $ENTRIES || cat )| (
+ curl -m 30 -s -X POST --data-binary @- \
+ -H "content-type: application/json" \
+ $REST_ENDPOINT
+ ) && ( test -n "$OUTPUT" && touch $OUTPUT ; logger "Uploaded $ENTRIES to $NIGHTSCOUT_HOST" ) || logger "Unable to upload to $NIGHTSCOUT_HOST"
+else
+ (test "$ENTRIES" != "-" && cat $ENTRIES || cat )| (
+ curl -m 30 -s -X POST --data-binary @- \
+ -H "API-SECRET: $API_SECRET" \
+ -H "content-type: application/json" \
+ $REST_ENDPOINT
+ ) && ( test -n "$OUTPUT" && touch $OUTPUT ; logger "Uploaded $ENTRIES to $NIGHTSCOUT_HOST" ) || logger "Unable to upload to $NIGHTSCOUT_HOST"
+fi

bin/oref0-autotune.sh

@@ -34,6 +34,7 @@ die() {
 }
 
 # defaults
+CURL_FLAGS="--compressed"
 DIR=""
 NIGHTSCOUT_HOST=""
 START_DATE=""
@@ -44,6 +45,10 @@ TERMINAL_LOGGING=true
 RECOMMENDS_REPORT=true
 UNKNOWN_OPTION=""
 
+if [ -n "${API_SECRET_READ}" ]; then
+ HASHED_API_SECRET_READ=`echo -n ${API_SECRET_READ}|sha1sum|cut -f1 -d '-'|cut -f1 -d ' '`
+fi
+
 # If we are running OS X, we need to use a different version
 # of the 'date' command; the built-in 'date' is BSD, which
 # has fewer options than the linux version. So the user
@@ -153,7 +158,11 @@ echo "Grabbing NIGHTSCOUT treatments.json for date range..."
 # Get Nightscout carb and insulin Treatments
 url="$NIGHTSCOUT_HOST/api/v1/treatments.json?find\[created_at\]\[\$gte\]=`date --date="$START_DATE -4 hours" -Iminutes`&find\[created_at\]\[\$lte\]=`date --date="$END_DATE +1 days" -Iminutes`"
 echo $url
-curl -s $url > ns-treatments.json || die "Couldn't download ns-treatments.json"
+if [ -n "${HASHED_API_SECRET_READ}" ]; then 
+ curl ${CURL_FLAGS} -H "api-secret: ${HASHED_API_SECRET_READ}" -s $url > ns-treatments.json || die "Couldn't download ns-treatments.json"
+else
+ curl ${CURL_FLAGS} -s $url > ns-treatments.json || die "Couldn't download ns-treatments.json"
+fi
 ls -la ns-treatments.json || die "No ns-treatments.json downloaded"
 
 # Build date list for autotune iteration
@@ -176,7 +185,12 @@ for i in "${date_list[@]}"
 do 
  url="$NIGHTSCOUT_HOST/api/v1/entries/sgv.json?find\[date\]\[\$gte\]=`(date -d $i +%s | tr -d '\n'; echo 000)`&find\[date\]\[\$lte\]=`(date --date="$i +1 days" +%s | tr -d '\n'; echo 000)`&count=1000"
  echo $url
- curl -s $url > ns-entries.$i.json || die "Couldn't download ns-entries.$i.json"
+ if [ -n "${HASHED_API_SECRET_READ}" ]; then 
+ curl ${CURL_FLAGS} -H "api-secret: ${HASHED_API_SECRET_READ}" -s $url > ns-entries.$i.json || die "Couldn't download ns-entries.$i.json"
+ else
+ curl ${CURL_FLAGS} -s $url > ns-entries.$i.json || die "Couldn't download ns-entries.$i.json"
+ fi
+
  ls -la ns-entries.$i.json || die "No ns-entries.$i.json downloaded"
 done
 
@@ -201,10 +215,18 @@ do
  # Autotune (required args, <autotune/glucose.json> <autotune/autotune.json> <settings/profile.json>), 
  # output autotuned profile or what will be used as <autotune/autotune.json> in the next iteration
  echo "oref0-autotune-core autotune.$run_number.$i.json profile.json profile.pump.json > newprofile.$run_number.$i.json"
- oref0-autotune-core autotune.$run_number.$i.json profile.json profile.pump.json > newprofile.$run_number.$i.json || die "Could not run oref0-autotune-core autotune.$run_number.$i.json profile.json profile.pump.json"
-
- # Copy tuned profile produced by autotune to profile.json for use with next day of data
- cp newprofile.$run_number.$i.json profile.json
+ if ! oref0-autotune-core autotune.$run_number.$i.json profile.json profile.pump.json > newprofile.$run_number.$i.json; then
+ if cat profile.json | jq --exit-status .carb_ratio==null; then
+ echo "ERROR: profile.json contains null carb_ratio: using profile.pump.json"
+ cp profile.pump.json profile.json
+ exit
+ else
+ die "Could not run oref0-autotune-core autotune.$run_number.$i.json profile.json profile.pump.json"
+ fi
+ else
+ # Copy tuned profile produced by autotune to profile.json for use with next day of data
+ cp newprofile.$run_number.$i.json profile.json
+ fi
 
  done # End Date Range Iteration
 done # End Number of Runs Loop

bin/oref0-detect-sensitivity.js

@@ -37,6 +37,7 @@ if (!module.parent) {
  try {
  var cwd = process.cwd();
  var glucose_data = require(cwd + '/' + glucose_input);
+ // require 6 hours of data to run autosens
  if (glucose_data.length < 72) {
  console.error("Optional feature autosens disabled: not enough glucose data to calculate sensitivity");
  return console.log('{ "ratio": 1, "reason": "not enough glucose data to calculate autosens" }');