y-assert

#!/usr/bin/env bash
[ -z "$DEBUG" ] || set -x
set -eo pipefail

function log {
  date -u +"%Y-%m-%dT%H:%M:%SZ"
}

[ "$1" = "help" ] && echo '
A convention for running https://github.com/Yolean/kubernetes-assert
using ephemeral namespaces.
' && exit 0

[ -z "$DIR" ] && DIR="$PWD" && echo "$(log) Module dir: $DIR"

[ ! -f $DIR/skaffold.yaml ] && echo "Missing a skaffold.yaml in project dir $DIR" && exit 1

[ -z "$PROJECT" ] && PROJECT=$(echo $DIR | awk -F/ '{print $NF}' | tr -cd [:alnum:]- | tr '[:upper:]' '[:lower:]')

[ -z "$ID" ] && ID=assert-$(date -u +"%Y%m%dt%H%M%Sz")-${PROJECT:0:10} && echo "$(log) Generated test ID: $ID"

# What cleanup solutions are there? Let's start with https://github.com/hjacobs/kube-janitor
TTL="janitor/ttl: 23h"

[ -z "$MONITORING_HOST" ] && MONITORING_HOST=http://monitoring.ystack.svc.cluster.local

curl_opts="-s --connect-timeout 3 --retry 5 --retry-connrefused"
! curl -f $curl_opts $MONITORING_HOST:9090/ >/dev/null && echo "Failed to access the Prometheus endpoint" && exit 1
! curl -f $curl_opts $MONITORING_HOST:9093/ >/dev/null && echo "Failed to access the Alertmanager endpoint" && exit 1

ctx=$1
case $ctx in
  "--context="*) shift 1 ;;
  *)
    [ "$CI" != "true" ] && echo "First arg should be --context" && exit 1
    ctx="--context=$(kubectl config current-context)" && echo "$(log) Current ctx: $ctx"
    ;;
esac
skaffoldflags=$(echo $ctx | sed 's|^--|--kube-|')

[ -z "$PROFILES" ] && PROFILES="y-assert"
skaffoldflags="$skaffoldflags -p $PROFILES"

CIRUN=$1
[ -z "$CIRUN" ] && CIRUN="run"

# --- Up to this point we didn't actually do anything in any cluster, from here on we will ---

if [ "$CI" = true ]; then
  [ -f /run/secrets/kubernetes.io/serviceaccount/namespace ] && namespace=$(cat /run/secrets/kubernetes.io/serviceaccount/namespace)
  [ -z "$namespace" ] && namespace=$(kubectl $ctx config view --minify --output 'jsonpath={..namespace}')
  [ -z "$namespace" ] && echo "$(log) Failed to find a current namespace for CI=true mode" && exit 1
else
  namespace=$ID
  echo "$(log) Creating (in $ctx) temp namespace: $namespace"
  cat <<EOF | kubectl $ctx apply -f -
apiVersion: v1
kind: Namespace
metadata:
  name: $namespace
  labels:
    $TTL
EOF
fi
skaffoldflags="$skaffoldflags -n $namespace"

# At this point we should know how to identify relevant assert metrics
[ -z "$assertlabels" ] && assertlabels="namespace=\"$namespace\"";

y-skaffold $skaffoldflags --cache-artifacts=false $CIRUN

echo "$(log) Prometheus queries for test results will use labels: $assertlabels"
echo "$(log) cleanup-ish: kubectl $ctx get namespace -l $(echo $TTL | sed 's/: */=/') -o name | xargs kubectl $ctx delete"

query=assertions_failed{$assertlabels}

[ -z "$INTERVAL" ] && INTERVAL=5
# This loop will continuously list errors and must fail on http errors or absence of the metric
until curl -s --data "query=$query" $MONITORING_HOST:9090/api/v1/query \
  | jq -r '.data.result | if length > 0 then . else error("No results") end | .[] | . as {$metric, $values} | .value[1] | . as $vs | tonumber as $v | if $v > 0 then error($metric.namespace+" "+$metric.pod+" "+$vs) else empty end'
do sleep $INTERVAL
done

echo "Zero failures now with: curl -s --data query=$query $MONITORING_HOST:9090/api/v1/query"