diff --git a/.gitmodules b/.gitmodules index 4ccda555..7e090732 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,5 +5,5 @@ path = contracts/lib/openzeppelin-contracts url = https://github.com/openzeppelin/openzeppelin-contracts [submodule "constellation"] - path = constellation + path = deploy/constellation/constellation-src url = https://github.com/comrade-coop/constellation diff --git a/Tiltfile b/Tiltfile index 0c847db9..d6252e89 100644 --- a/Tiltfile +++ b/Tiltfile @@ -1,16 +1,30 @@ # -*- mode: Python -*- # SPDX-License-Identifier: GPL-3.0 -load( - "./deploy/Tiltfile", "apocryph_resource", "apocryph_build_with_builder", "deploy_apocryph_stack" -) config.define_string_list("include") +config.define_string("allow-context") +config.define_bool("deploy-stack") cfg = config.parse() -apocryph_build_with_builder() -deploy_apocryph_stack() +if "allow-context" in cfg: + allow_k8s_contexts(cfg["allow-context"]) + +load( + "./deploy/Tiltfile", + "apocryph_resource", + "apocryph_build_with_builder", + "deploy_apocryph_stack", + "deploy_apocryph_local", +) + +if cfg.get("deploy-stack", True): + apocryph_build_with_builder() + deploy_apocryph_stack() + deploy_apocryph_local() +else: + apocryph_build_with_builder(skip_images=True) + deploy_apocryph_local(resource_deps=[]) -if "include" in cfg: - for f in cfg["include"]: - load_dynamic(f) +for f in cfg.get("include", []): + load_dynamic(f) diff --git a/deploy/Tiltfile b/deploy/Tiltfile index cea21c3a..810ebc6a 100644 --- a/deploy/Tiltfile +++ b/deploy/Tiltfile @@ -10,7 +10,7 @@ local( "which jq forge cast helm kubectl docker cosign >/dev/null", echo_off=True ) # Check dependencies cluster_ip = local( - "kubectl get no -o jsonpath --template '{$.items[*].status.addresses[?(.type==\"InternalIP\")].address}'" + "kubectl get no -o jsonpath --template '{$.items[0].status.addresses[?(.type==\"InternalIP\")].address}'" ) deploy_dir = os.getcwd() @@ -73,7 +73,7 @@ def apocryph_resource( manifest_file, builder="apocryph-go-builder", docker_ipfs="ipfs-local", - ethereum_resource="anvil", + ethereum_resource="anvil-deploy-contracts", ethereum_namespace="eth", private_key="0x5de4111afa1a4b94908f83103eb1f1706367c2e68ca870fc3fb9a804cdab365a", pod_id="0x00", @@ -220,12 +220,12 @@ def docker_ipfs_resource( dc_resource(name, *args, **(kwargs | kwargs_dc)) # Ideally there would be a better way to do this (other than hardcoding the ipfs privkey) remote_peerid = ( - "$(kubectl exec -n ipfs $(kubectl get po -n ipfs -o name) -- ipfs config Identity.PeerID)" + "$(kubectl exec -n ipfs $(kubectl get po -n %s -o name) -- ipfs config Identity.PeerID)" % (remote_ipfs_namespace,) ) remote_port = ( - "$(kubectl get svc -n ipfs ipfs-swarm -o jsonpath --template '{$.spec.ports[0].nodePort}')" + "$(kubectl get svc -n %s ipfs-swarm -o jsonpath --template '{$.spec.ports[0].nodePort}')" % (remote_ipfs_namespace,) ) - config_cmd = "ipfs swarm connect /ip4/%s/udp/%s/quic-v1/webtransport/p2p/%s" % ( + config_cmd = "ipfs swarm peering add /ip4/%s/udp/%s/quic-v1/webtransport/p2p/%s" % ( cluster_ip, remote_port, remote_peerid, @@ -297,7 +297,10 @@ def cmdline_in_builder(cmd, builder, *, interactive=False): def apocryph_build_with_builder( - root_dir=deploy_dir + "/..", cosign_key=None, cosign_key_path="deploy/keys/" + root_dir=deploy_dir + "/..", + cosign_key=None, + cosign_key_path="deploy/keys/", + skip_images=False, ): if cosign_key == None: cosign_key = "cosign-key" @@ -326,52 +329,53 @@ def apocryph_build_with_builder( deps=[root_dir + "/cmd", root_dir + "/pkg"], allow_parallel=True, ) + + if not skip_images: + docker_build_with_restart( + "comradecoop/apocryph/server", + root_dir, + dockerfile="./Dockerfile", + target="server-copy-local", + entrypoint=["/usr/local/bin/tpodserver"], + only=[root_dir + "/bin"], + live_update=[ + sync(root_dir + "/bin", "/usr/local/bin/"), + ], + ) + docker_build_with_restart( + "comradecoop/apocryph/p2p-helper", + root_dir, + dockerfile="./Dockerfile", + target="p2p-helper-copy-local", + entrypoint=["/usr/local/bin/ipfs-p2p-helper"], + only=[root_dir + "/bin"], + live_update=[ + sync(root_dir + "/bin", "/usr/local/bin/"), + ], + ) - docker_build_with_restart( - "comradecoop/apocryph/server", - root_dir, - dockerfile="./Dockerfile", - target="server-copy-local", - entrypoint=["/usr/local/bin/tpodserver"], - only=[root_dir + "/bin"], - live_update=[ - sync(root_dir + "/bin", "/usr/local/bin/"), - ], - ) - docker_build_with_restart( - "comradecoop/apocryph/p2p-helper", - root_dir, - dockerfile="./Dockerfile", - target="p2p-helper-copy-local", - entrypoint=["/usr/local/bin/ipfs-p2p-helper"], - only=[root_dir + "/bin"], - live_update=[ - sync(root_dir + "/bin", "/usr/local/bin/"), - ], - ) - - # https://stackoverflow.com/a/33511811 for $(docker inspect --format ...) - docker_build_with_restart( - "comradecoop/apocryph/tpod-proxy-unsigned", - root_dir, - dockerfile="./Dockerfile", - target="tpod-proxy-copy-local", - entrypoint=["/usr/local/bin/tpod-proxy"], - only=[root_dir + "/bin"], - live_update=[ - sync(root_dir + "/bin", "/usr/local/bin/"), - ], - ) + # https://stackoverflow.com/a/33511811 for $(docker inspect --format ...) + docker_build_with_restart( + "comradecoop/apocryph/tpod-proxy-unsigned", + root_dir, + dockerfile="./Dockerfile", + target="tpod-proxy-copy-local", + entrypoint=["/usr/local/bin/tpod-proxy"], + only=[root_dir + "/bin"], + live_update=[ + sync(root_dir + "/bin", "/usr/local/bin/"), + ], + ) - cosign_sign_image_key( - "comradecoop/apocryph/tpod-proxy", - "comradecoop/apocryph/tpod-proxy-unsigned", - cosign_key=cosign_key, - cosign_key_path=cosign_key_path, - live_update=[ - sync(root_dir + "/bin", "/usr/local/bin/"), - ], - ) + cosign_sign_image_key( + "comradecoop/apocryph/tpod-proxy", + "comradecoop/apocryph/tpod-proxy-unsigned", + cosign_key=cosign_key, + cosign_key_path=cosign_key_path, + live_update=[ + sync(root_dir + "/bin", "/usr/local/bin/"), + ], + ) """ # TODO: Need to also build a trustedpods image for use with apocryph_resource... @@ -403,6 +407,7 @@ def deploy_apocryph_stack( update_settings(k8s_upsert_timeout_secs=160) + # NOTE: Code below duplicates ./constellation/helmfile.yaml helm_repo("kedacore", "https://kedacore.github.io/charts") helm_repo("ingress-nginx-chart", "https://kubernetes.github.io/ingress-nginx") helm_repo("prometheus-community", "https://prometheus-community.github.io/helm-charts") @@ -433,7 +438,7 @@ def deploy_apocryph_stack( labels=["apocryph-deps", "flaky"], flags=["--create-namespace"], ) - k8s_yaml(root_dir + "/deploy/keda/ingress.yml") + k8s_yaml(root_dir + "/deploy/charts/keda/ingress.yml") k8s_resource( objects=["keda-ingress:ingress"], new_name="keda-ingress", @@ -470,37 +475,33 @@ def deploy_apocryph_stack( "loki", "grafana/loki-stack", namespace="loki", - deps=[root_dir + "/deploy/loki/values.yml"], + deps=[root_dir + "/deploy/charts/loki/values.yml"], resource_deps=["grafana"], labels=["apocryph-deps"], - flags=["-f", root_dir + "/deploy/loki/values.yml", "--create-namespace"], + flags=["-f", root_dir + "/deploy/charts/loki/values.yml", "--create-namespace"], ) namespace_create("eth") # TODO: Recreate anvil when we have new contracts code - k8s_yaml(listdir(root_dir + "/deploy/eth/")) + k8s_yaml(listdir(root_dir + "/deploy/charts/eth/")) k8s_resource("anvil", labels=["apocryph-dev"]) helm_resource( "ipfs", - root_dir + "/deploy/ipfs/", + root_dir + "/deploy/charts/ipfs/", namespace="ipfs", - deps=[root_dir + "/deploy/ipfs/"], + deps=[root_dir + "/charts/deploy/ipfs/"], labels=["apocryph"], flags=["--set=swarm.announceIp=%s" % cluster_ip, "--create-namespace"], image_keys=["p2phelper.image"], image_deps=["comradecoop/apocryph/p2p-helper"], ) - docker_ipfs_resource( - "ipfs-local", "docker.io/ipfs/kubo:v0.23.0", "ipfs", labels=["apocryph-dev", "flaky"], resource_deps=['ipfs'] - ) - helm_resource( "trustedpods", - root_dir + "/deploy/trustedpods/", + root_dir + "/deploy/charts/trustedpods/", namespace="trustedpods", - deps=[root_dir + "/deploy/trustedpods/", cosign_key_path], + deps=[root_dir + "/deploy/charts/trustedpods/", cosign_key_path], resource_deps=["anvil", "ipfs", "loki", "anvil-deploy-contracts", "policy-controller"], labels=["apocryph"], image_keys=["image", "policy.image"], @@ -516,6 +517,18 @@ def deploy_apocryph_stack( "--create-namespace", ], ) + # NOTE: Code above duplicates ./constellation/helmfile.yaml + +def deploy_apocryph_local( + root_dir=deploy_dir + "/..", + deployer_key="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80", + cosign_key="cosign-key", + cosign_key_path="deploy/keys/", + resource_deps=["ipfs", "anvil"] +): + docker_ipfs_resource( + "ipfs-local", "docker.io/ipfs/kubo:v0.23.0", "ipfs", labels=["apocryph-dev", "flaky"], resource_deps=resource_deps + ) local_resource( # TODO: Move to container! "anvil-deploy-contracts", @@ -524,6 +537,6 @@ def deploy_apocryph_stack( # [ -f ./broadcast/Deploy.s.sol/31337/run-latest.json ] || cmd="forge script script/Deploy.s.sol --rpc-url http://%s:$(kubectl get svc -n eth eth-rpc -o jsonpath --template '{$.spec.ports[0].nodePort}') --private-key %s --broadcast" % (cluster_ip, deployer_key), - resource_deps=["anvil"], + resource_deps=resource_deps, deps=["./contracts/src", "./contracts/script", "./contracts/lib"], ) diff --git a/deploy/eth/anvil-proxy.yml b/deploy/charts/eth/anvil-proxy.yml similarity index 100% rename from deploy/eth/anvil-proxy.yml rename to deploy/charts/eth/anvil-proxy.yml diff --git a/deploy/ipfs/Chart.yaml b/deploy/charts/ipfs/Chart.yaml similarity index 100% rename from deploy/ipfs/Chart.yaml rename to deploy/charts/ipfs/Chart.yaml diff --git a/deploy/ipfs/templates/NOTES.txt b/deploy/charts/ipfs/templates/NOTES.txt similarity index 66% rename from deploy/ipfs/templates/NOTES.txt rename to deploy/charts/ipfs/templates/NOTES.txt index 7e4beb7e..5e95a349 100644 --- a/deploy/ipfs/templates/NOTES.txt +++ b/deploy/charts/ipfs/templates/NOTES.txt @@ -1,3 +1,3 @@ - _ _ ___ _ _ __ _ _ _ +__ _ ___ _ _ __ _ _ _ | \|_\ / | |_)|_(_ |\ |/ \| \|_ |_/|_ \/ _|_| | __) | \|\_/|_/|_ diff --git a/deploy/ipfs/templates/ipfs.yml b/deploy/charts/ipfs/templates/ipfs.yml similarity index 100% rename from deploy/ipfs/templates/ipfs.yml rename to deploy/charts/ipfs/templates/ipfs.yml diff --git a/deploy/ipfs/templates/serviceaccount.yml b/deploy/charts/ipfs/templates/serviceaccount.yml similarity index 100% rename from deploy/ipfs/templates/serviceaccount.yml rename to deploy/charts/ipfs/templates/serviceaccount.yml diff --git a/deploy/ipfs/values.yaml b/deploy/charts/ipfs/values.yaml similarity index 100% rename from deploy/ipfs/values.yaml rename to deploy/charts/ipfs/values.yaml diff --git a/deploy/keda/ingress.yml b/deploy/charts/keda/ingress.yml similarity index 100% rename from deploy/keda/ingress.yml rename to deploy/charts/keda/ingress.yml diff --git a/deploy/loki/values.yml b/deploy/charts/loki/values.yml similarity index 100% rename from deploy/loki/values.yml rename to deploy/charts/loki/values.yml diff --git a/deploy/trustedpods/.helmignore b/deploy/charts/trustedpods/.helmignore similarity index 100% rename from deploy/trustedpods/.helmignore rename to deploy/charts/trustedpods/.helmignore diff --git a/deploy/trustedpods/Chart.yaml b/deploy/charts/trustedpods/Chart.yaml similarity index 100% rename from deploy/trustedpods/Chart.yaml rename to deploy/charts/trustedpods/Chart.yaml diff --git a/deploy/trustedpods/templates/NOTES.txt b/deploy/charts/trustedpods/templates/NOTES.txt similarity index 100% rename from deploy/trustedpods/templates/NOTES.txt rename to deploy/charts/trustedpods/templates/NOTES.txt diff --git a/deploy/trustedpods/templates/proxypolicy.yml b/deploy/charts/trustedpods/templates/proxypolicy.yml similarity index 91% rename from deploy/trustedpods/templates/proxypolicy.yml rename to deploy/charts/trustedpods/templates/proxypolicy.yml index 61f82329..01f1d852 100644 --- a/deploy/trustedpods/templates/proxypolicy.yml +++ b/deploy/charts/trustedpods/templates/proxypolicy.yml @@ -1,3 +1,4 @@ +{{ if .Values.policy.enable }} apiVersion: policy.sigstore.dev/v1beta1 kind: ClusterImagePolicy metadata: @@ -16,4 +17,5 @@ spec: - key: data: |{{ .Values.policy.key | nindent 10 }} {{ end }} - +{{ end }} +--- diff --git a/deploy/trustedpods/templates/serviceaccount.yml b/deploy/charts/trustedpods/templates/serviceaccount.yml similarity index 100% rename from deploy/trustedpods/templates/serviceaccount.yml rename to deploy/charts/trustedpods/templates/serviceaccount.yml diff --git a/deploy/trustedpods/templates/tpodserver.yml b/deploy/charts/trustedpods/templates/tpodserver.yml similarity index 100% rename from deploy/trustedpods/templates/tpodserver.yml rename to deploy/charts/trustedpods/templates/tpodserver.yml diff --git a/deploy/trustedpods/values.yaml b/deploy/charts/trustedpods/values.yaml similarity index 88% rename from deploy/trustedpods/values.yaml rename to deploy/charts/trustedpods/values.yaml index be34863f..8979ef9c 100644 --- a/deploy/trustedpods/values.yaml +++ b/deploy/charts/trustedpods/values.yaml @@ -5,7 +5,8 @@ withdraw: tokenContract: "0x5FbDB2315678afecb367f032d93F642f64180aa3" registryContract: "0x9fE46736679d2D9a65F0992F2272dE9f3c7fa6e0" policy: + enable: false issuer: https://github.com/login/oauth subject: comrade@email.example image: comradecoop/apocryph/tpod-proxy:latest - imageGlob: "**apocryph_tpod-proxy**" + imageGlob: "**apocryph/tpod-proxy**" diff --git a/deploy/configure-ipfs.sh b/deploy/configure-ipfs.sh index ff1cdca3..6ac41fb6 100755 --- a/deploy/configure-ipfs.sh +++ b/deploy/configure-ipfs.sh @@ -1,3 +1,4 @@ #!/bin/sh ipfs config --json Experimental.Libp2pStreamMounting true +ipfs config Addresses.Gateway /ip4/127.0.0.1/tcp/8082 diff --git a/test/e2e/constellation/qemu/build.sh b/deploy/constellation/build-qemu.sh similarity index 69% rename from test/e2e/constellation/qemu/build.sh rename to deploy/constellation/build-qemu.sh index 9137326c..8308f7a2 100755 --- a/test/e2e/constellation/qemu/build.sh +++ b/deploy/constellation/build-qemu.sh @@ -1,52 +1,48 @@ #!/bin/sh set -e + +which helmfile >/dev/null +which basel >/dev/null || { echo "Install Bazel, ideally through Bazelisk, https://bazel.build/install/bazelisk"; exit 1; } +which nix >/dev/null || { echo "Install Nix, https://nixos.org/download/"; exit 1; } + set -v -CHART_PATH="$1" SUFFIX=$RANDOM -CONSTELLATION_PATH="../../../../constellation" WORKSPACE_PATH="$HOME/.apocryph/constellation-$SUFFIX" echo "WORKSPACE_PATH:: $WORKSPACE_PATH" -CURRENT_DIR=$(pwd) -if [ -n "$2" ]; then - STEP=${2:-1} +SCRIPT_DIR=$(realpath $(dirname "$0")) +HELMFILE_PATH="$SCRIPT_DIR/helmfile.yaml" +CONSTELLATION_SRC="$SCRIPT_DIR/constellation-src" + +if [ -n "$1" ]; then + STEP=${1:-1} eval "set -v; $(sed -n "/## $STEP: /{:a;n;p;ba};" $0)" exit fi -# Check the number of arguments -if [ "$#" -lt 1 ]; then - echo "Usage: $0 " - exit 1 -fi - if [ "$1" = "teardown" ]; then - ( cd $WORKSPACE_PATH; constellation terminate ) + sudo rm -r "$WORKSPACE_PATH" exit 0 fi -sudo chmod o+rw /run/containerd/containerd.sock - ## 0: Generate helm template and inject it into constellation base image -helmfile template -f "$CHART_PATH" > "$CONSTELLATION_PATH/image/base/mkosi.skeleton/usr/lib/helmfile-template" +helmfile template -f "$HELMFILE_PATH" --kube-version 1.30.0 > "$CONSTELLATION_SRC/image/base/mkosi.skeleton/usr/lib/helmfile-template" ## 1: Build modified image -cd $CURRENT_DIR -cd "$CONSTELLATION_PATH" +pushd "$CONSTELLATION_SRC" bazel run //:tidy +popd ## 1.1: Build the image -cd $CURRENT_DIR -cd "$CONSTELLATION_PATH" +pushd "$CONSTELLATION_SRC" bazel build //image/system:qemu_stable - +popd ## 2: create & configure constellation workspace set -e set -v -cd "$CURRENT_DIR" -cd "$CONSTELLATION_PATH" +pushd "$CONSTELLATION_SRC" # Get the new image measurements link=$(readlink -f bazel-out/k8-opt/bin/image/system/qemu_qemu-vtpm_stable) output=$(bazel run --run_under="sudo -E" //image/measured-boot/cmd $link/constellation.raw "/tmp/custom-measurements.json" 2>&1) # second arg needed @@ -60,13 +56,10 @@ echo "PCR4: $PCR4" echo "PCR9: $PCR9" echo "PCR11: $PCR11" -if [ -d "$WORKSPACE_PATH" ]; then - cd "$WORKSPACE_PATH" && constellation terminate 2>/dev/null - sudo rm -r "$WORKSPACE_PATH" -fi +popd mkdir -p "$WORKSPACE_PATH" -cd "$WORKSPACE_PATH" +pushd "$WORKSPACE_PATH" constellation config generate qemu @@ -84,8 +77,9 @@ version=$(echo "$output" | grep -oP 'Version:\s+\K\S+' | head -n 1) # Copy the image & rename it to the current constellation version to bypass downloading upstream image cp $link/constellation.raw "$version.raw" +popd -export KUBECONFIG="$HOME/.apocryph/constellation-$SUFFIX/constellation-admin.conf" +export KUBECONFIG="$WORKSPACE_PATH/constellation-admin.conf" diff --git a/constellation b/deploy/constellation/constellation-src similarity index 100% rename from constellation rename to deploy/constellation/constellation-src diff --git a/deploy/constellation/deploy-miniconstellation.sh b/deploy/constellation/deploy-miniconstellation.sh new file mode 100755 index 00000000..c29b0d37 --- /dev/null +++ b/deploy/constellation/deploy-miniconstellation.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: GPL-3.0 + +set -e + +which sed >/dev/null +which helmfile >/dev/null; which helm >/dev/null; which kubectl >/dev/null +which constellation >/dev/null || { echo "Install Constellation, https://docs.edgeless.systems/constellation/getting-started/first-steps-local#software-installation-on-ubuntu"; exit 1; } + +WORKSPACE_PATH="$HOME/.apocryph/constellation-mini" +SCRIPT_DIR=$(realpath $(dirname "$0")) +REPO_DIR="$SCRIPT_DIR/../../" + +# based on https://stackoverflow.com/a/31269848 / https://bobcopeland.com/blog/2012/10/goto-in-bash/ +if [ -n "$1" ]; then + STEP=${1:-1} + eval "set -v; $(sed -n "/## $STEP: /{:a;n;p;ba};" $0)" + exit +fi + +echo -e "\e[1;32m---" +echo "Note: To skip steps, use '$0 '" +echo " e.g. to skip ahead to configuring IPFS, run '$0 1.2'" +echo -e "---\e[0m" + +set -v + +## 1: Start Miniconstellation ## + +echo 'CONSTELLATION_PATH='$CONSTELLATION_PATH + +mkdir -p $WORKSPACE_PATH +pushd "$WORKSPACE_PATH" + +constellation mini up || true + +kubectl patch -n kube-system configmap ip-masq-agent --type merge -p '{"data":{"config": "{\"masqLinkLocal\":true,\"nonMasqueradeCIDRs\":[]}"}}' +kubectl rollout restart -n kube-system daemonset cilium +kubectl delete pod -l k8s-app=join-service -n kube-system + +popd + +## 2: Apply the Helm configuration ## + +pushd "$SCRIPT_DIR" + +# IMAGE_PREFIX=$(uuidgen) +# docker build -t ttl.sh/$IMAGE_PREFIX-apocryph-server:1h . --target server-copy-local +# docker push ttl.sh/$IMAGE_PREFIX-apocryph-server:1h +## replace ghcr.io/comrade-coop/apocryph/server:master with ttl.sh/$IMAGE_PREFIX-apocryph-server:1h in helmfile.yaml + +helmfile sync || kubectl wait --namespace keda --for=condition=available deployment/ingress-nginx-controller && helmfile sync + +popd + +echo "Run \`$0 example\` to also run a tilt example" +echo "Run \`$0 teardown\` to stop everything" +exit 0 + +## example: Start a tilt example ## + +pushd "$REPO_DIR" + +tilt up -- --deploy-stack=False --include ./test/e2e/nginx/Tiltfile --allow-context 'mini-qemu-admin@mini-qemu' + +popd + +exit 0 +## teardown: Stop Miniconstellation ## + +pushd "$REPO_DIR" + +constellation mini down + +popd + diff --git a/deploy/constellation/deploy-qemu.sh b/deploy/constellation/deploy-qemu.sh new file mode 100755 index 00000000..e385f64d --- /dev/null +++ b/deploy/constellation/deploy-qemu.sh @@ -0,0 +1,64 @@ +#!/bin/bash +set -e +set -v + +which constellation >/dev/null; which kubectl >/dev/null; which tilt >/dev/null + +WORKSPACE_PATH=$(echo "$HOME/.apocryph/constellation-"*) +SCRIPT_DIR=$(realpath $(dirname "$0")) +REPO_DIR="$SCRIPT_DIR/../../" + +# based on https://stackoverflow.com/a/31269848 / https://bobcopeland.com/blog/2012/10/goto-in-bash/ +if [ -n "$1" ]; then + STEP=${1:-1} + eval "set -v; $(sed -n "/## $STEP: /{:a;n;p;ba};" $0)" + exit +fi + +echo -e "\e[1;32m---" +echo "Note: To skip steps, use '$0 '" +echo -e "---\e[0m" + +## 0: Build custom OS image & run the cluster ## + +"$SCRIPT_DIR/build-qemu.sh" + +## 1: Start the Constellation cluster ## + +pushd "$WORKSPACE_PATH" +constellation terminate || true +constellation apply -y +popd + +## 1.1: Wait for the setup ## + +export KUBECONFIG="$WORKSPACE_PATH/constellation-admin.conf" + +kubectl wait --namespace keda --for=condition=available deployment/ingress-nginx-controller +kubectl wait --namespace prometheus --for=condition=available deployment/prometheus-kube-state-metrics +kubectl wait --namespace prometheus --for=condition=available deployment/prometheus-prometheus-pushgateway +kubectl wait --namespace prometheus --for=condition=available deployment/prometheus-server +kubectl wait --namespace ipfs --for=condition=available StatefulSet/ipfs +kubectl wait --namespace trustedpods --for=condition=available deployment/tpodserver + +echo "Run \`$0 example\` to also run a tilt example" +echo "Run \`$0 teardown\` to stop everything" + +exit 0 +## example: start tilt + +pushd "$REPO_DIR" + +tilt up -- --deploy-stack=False --include ./test/e2e/nginx/Tiltfile + +popd + +exit 0 +## teardown: Stop the Constellation cluster ## + +pushd "$REPO_DIR" + +constellation terminate + +popd + diff --git a/deploy/constellation/helmfile.yaml b/deploy/constellation/helmfile.yaml new file mode 100644 index 00000000..952f8e69 --- /dev/null +++ b/deploy/constellation/helmfile.yaml @@ -0,0 +1,126 @@ +# NOTE: Code below duplicates deploy_apocryph_stack in ../Tiltfile +repositories: + - name: kedacore + url: https://kedacore.github.io/charts + - name: ingress-nginx + url: https://kubernetes.github.io/ingress-nginx + - name: prometheus-community + url: https://prometheus-community.github.io/helm-charts + - name: grafana + url: https://grafana.github.io/helm-charts + - name: sigstore + url: https://sigstore.github.io/helm-charts + - name: local-path-provisioner # Helmfile exclusive, constellation qemu doesn't have PV-s yet + # For url, see also https://github.com/rancher/local-path-provisioner/issues/89 + url: git+https://github.com/rancher/local-path-provisioner@deploy/chart?ref=master + +releases: + - name: namespaces # Helmfile exclusive, see https://github.com/helm/helm/issues/9813 + namespace: default + chart: ./namespaces + + - name: local-path-storage # Helmfile exclusive, constellation qemu/mini doesn't have PV-s yet + namespace: local-path-storage + chart: local-path-provisioner/local-path-provisioner + set: + - name: storageClass.defaultClass + value: true + needs: + - default/namespaces + + - name: policy-controller + namespace: policy-controller + chart: sigstore/policy-controller + needs: + - default/namespaces + + - name: keda + namespace: keda + chart: kedacore/keda + needs: + - default/namespaces + + - name: ingress-nginx + namespace: keda + chart: ingress-nginx/ingress-nginx + set: # Helmfile exclusive, we don't have LoadBalancer in miniconstellation (TODO) + - name: controller.service.type + value: NodePort + - name: controller.service.nodePorts.http + value: 32080 + - name: controller.service.nodePorts.https + value: 32443 + needs: + - default/namespaces + + - name: keda-ingress + chart: ../charts/keda + namespace: keda + needs: + - keda-http-addon + - ingress-nginx # NOTE: nginx admission controllers typically fails to wait long enough for nginx to start at this step + - default/namespaces + + - name: keda-http-addon + namespace: keda + chart: kedacore/keda-add-ons-http + set: + - name: interceptor.replicas.min + value: 1 + - name: scaler.replicas + value: 1 + - name: interceptor.waitTimeout + value: 40s + needs: + - keda + - default/namespaces + + - name: prometheus + chart: prometheus-community/prometheus + namespace: prometheus + set: + - name: alertmanager.enabled + value: false + - name: prometheus-node-exporter.enabled + value: false + - name: server.persistentVolume.size # Helmfile exclusive, reduce prometheus PV + value: 1Gi + needs: + - default/namespaces + + - name: loki + chart: grafana/loki-stack + namespace: loki + values: + - ../charts/loki/values.yml + needs: + - default/namespaces + + - name: ipfs + chart: ../charts/ipfs + namespace: ipfs + set: + - name: swarm.announceIp # Helmfile exclusive, assume the IP of miniconstellation + value: 10.42.1.100 + - name: p2phelper.image # Helmfile exclusive, use ghcr images + value: ghcr.io/comrade-coop/apocryph/p2p-helper:master + needs: + - default/namespaces + + - name: eth + chart: ../charts/eth + namespace: eth + needs: + - default/namespaces + + - name: trustedpods + chart: ../charts/trustedpods + namespace: trustedpods + set: + - name: policy.enable # Helmfile exclusive, disable policy + value: false + - name: image # Helmfile exclusive, use ghcr images + value: ghcr.io/comrade-coop/apocryph/server:master + needs: + - default/namespaces +# NOTE: Code above duplicates deploy_apocryph_stack in ../Tiltfile diff --git a/test/e2e/constellation/miniconstellation/namespaces/namespaces.yml b/deploy/constellation/namespaces/namespaces.yml similarity index 71% rename from test/e2e/constellation/miniconstellation/namespaces/namespaces.yml rename to deploy/constellation/namespaces/namespaces.yml index e45a671d..fb8a53df 100644 --- a/test/e2e/constellation/miniconstellation/namespaces/namespaces.yml +++ b/deploy/constellation/namespaces/namespaces.yml @@ -5,6 +5,11 @@ metadata: --- apiVersion: v1 kind: Namespace +metadata: + name: policy-controller +--- +apiVersion: v1 +kind: Namespace metadata: name: trustedpods --- @@ -27,3 +32,8 @@ apiVersion: v1 kind: Namespace metadata: name: prometheus +--- +apiVersion: v1 +kind: Namespace +metadata: + name: local-path-storage diff --git a/deploy/run-attestation-test.sh b/deploy/run-attestation-test.sh deleted file mode 100755 index d72fbe8f..00000000 --- a/deploy/run-attestation-test.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh - -set -e -echo "USAGE: $0 CERTIFICATE_IDENTITY CERTIFICATE_OIDC_ISSUER STEP" -echo "EXAMPLE: $0 example@email.com https://github.com/login/oauth" -set -v -# NOTE: The oidc-issuer for Google is https://accounts.google.com, Microsoft is https://login.microsoftonline.com, GitHub is https://github.com/login/oauth, and GitLab is https://gitlab.com." - -CERTIFICATE_IDENTITY=$1 -CERTIFICATE_OIDC_ISSUER=$2 - -# based on https://stackoverflow.com/a/31269848 / https://bobcopeland.com/blog/2012/10/goto-in-bash/ -if [ -n "$3" ]; then - STEP=${3:-1} - eval "set -v; $(sed -n "/## $STEP: /{:a;n;p;ba};" $0 )" - exit -fi - - -## 1: configure the cluster to support image validation -../test/e2e/common/scripts/redeploy-images.sh --set policy.issuer=$CERTIFICATE_OIDC_ISSUER --set policy.subject=$CERTIFICATE_IDENTITY - -## 2: deploy app -docker pull nginxdemos/nginx-hello@sha256:2ab1f0bef4461020a1aabee4260a1fe93b03ed69d7f72908acca3a7ec33cb1c0 -docker tag docker.io/nginxdemos/nginx-hello:latest ttl.sh/nginx-hello:1h -docker push ttl.sh/nginx-hello:1h - -./deploy-pod.sh ../test/e2e/common/manifests/manifest-attestation-nginx.yaml --certificate-identity $CERTIFICATE_IDENTITY --certificate-oidc-issuer $CERTIFICATE_OIDC_ISSUER - -## 3: Get Application info -INGRESS_URL=$(minikube service -n keda ingress-nginx-controller --url=true -p c1 | head -n 1); echo $INGRESS_URL -MANIFEST_HOST=example.tpodinfo.local # From manifest-nginx.yaml - -while ! curl --connect-timeout 40 -H "Host: $MANIFEST_HOST" $INGRESS_URL --fail-with-body; do sleep 10; done -echo - -go run ../cmd/trustedpods verify $INGRESS_URL --host-header $MANIFEST_HOST diff --git a/test/e2e/constellation/miniconstellation/eth/anvil.yml b/test/e2e/constellation/miniconstellation/eth/anvil.yml deleted file mode 100644 index 4b6adbb1..00000000 --- a/test/e2e/constellation/miniconstellation/eth/anvil.yml +++ /dev/null @@ -1,41 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: eth-rpc - namespace: eth - labels: - app: eth -spec: - ports: - - name: jsonrpc - port: 8545 - targetPort: jsonrpc - selector: - provide: eth-rpc ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: anvil - namespace: eth -spec: - replicas: 1 - selector: - matchLabels: - app: anvil - template: - metadata: - labels: - app: anvil - provide: eth-rpc - spec: - containers: - - name: anvil - image: ghcr.io/foundry-rs/foundry:nightly-619f3c56302b5a665164002cb98263cd9812e4d5 - command: ["anvil", "--state", "/data/anvil-state"] - ports: - - name: jsonrpc - containerPort: 8545 - env: - - name: ANVIL_IP_ADDR - value: 0.0.0.0 diff --git a/test/e2e/constellation/miniconstellation/helmfile.yaml b/test/e2e/constellation/miniconstellation/helmfile.yaml deleted file mode 100644 index 88b18727..00000000 --- a/test/e2e/constellation/miniconstellation/helmfile.yaml +++ /dev/null @@ -1,77 +0,0 @@ -repositories: - - name: kedacore - url: https://kedacore.github.io/charts - - name: ingress-nginx - url: https://kubernetes.github.io/ingress-nginx - - name: prometheus-community - url: https://prometheus-community.github.io/helm-charts - - name: grafana - url: https://grafana.github.io/helm-charts - -releases: - - name: namespaces - chart: ./namespaces - - name: keda - namespace: keda - chart: kedacore/keda - - name: ingress-nginx - namespace: keda - chart: ingress-nginx/ingress-nginx - hooks: - - events: ['postsync'] - showlogs: true - command: 'kubectl' - args: - - 'wait' - - '--namespace' - - 'keda' - - '--for=condition=available' - - 'deployment/ingress-nginx-controller' - - '--timeout=800s' - set: - - name: controller.service.type - value: NodePort - - name: controller.service.nodePorts.http - value: 32080 - - name: controller.service.nodePorts.https - value: 32443 - - name: keda-http-addon - namespace: keda - chart: kedacore/keda-add-ons-http - set: - - name: interceptor.replicas.min - value: 1 - - name: interceptor.waitTimeout - value: 40s - needs: - - keda - - name: prometheus - chart: prometheus-community/prometheus - namespace: prometheus - set: - - name: alertmanager.enabled - value: false - - name: prometheus-node-exporter.enabled - value: false - - name: server.persistentVolume.enabled - value: false - - name: keda-fixes - chart: ./keda - namespace: keda - needs: - - keda-http-addon - - ingress-nginx # NOTE: nginx admission controllers typically fails to wait long enough for nginx to start at this step - - name: ipfs - chart: ./ipfs - namespace: ipfs - - name: loki - chart: grafana/loki-stack - namespace: loki - values: - - ./loki/values.yml - - name: trustedpods - chart: ./trustedpods - namespace: trustedpods - - name: eth - chart: ./eth - namespace: eth diff --git a/test/e2e/constellation/miniconstellation/ipfs/ipfs.yml b/test/e2e/constellation/miniconstellation/ipfs/ipfs.yml deleted file mode 100644 index 1ccab5be..00000000 --- a/test/e2e/constellation/miniconstellation/ipfs/ipfs.yml +++ /dev/null @@ -1,121 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: ipfs-rpc - namespace: ipfs - labels: - app: ipfs -spec: - ports: - - name: rpc - port: 5001 - targetPort: rpc - clusterIP: None - selector: - app: ipfs ---- -apiVersion: v1 -kind: Service -metadata: - name: ipfs-swarm - namespace: ipfs - labels: - app: ipfs -spec: - ports: - - name: swarm-tcp - port: 4001 - protocol: TCP - targetPort: swarm-tcp - nodePort: 32194 - - name: swarm-udp - port: 4001 - protocol: UDP - targetPort: swarm-udp - nodePort: 32194 - type: NodePort - selector: - app: ipfs ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: ipfs-config - namespace: ipfs -data: - configure.sh: | - #!/bin/sh - set -ex - ipfs config --json Experimental.Libp2pStreamMounting true - ipfs config Addresses.AppendAnnounce --json '["/ip4/10.42.1.100/tcp/32194", "/ip4/10.42.1.100/udp/32194/quic", "/ip4/10.42.1.100/udp/32194/quic-v1", "/ip4/10.42.1.100/udp/32194/quic-v1/webtransport"]' ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ipfs - namespace: ipfs -spec: - replicas: 1 - selector: - matchLabels: - app: ipfs - template: - metadata: - labels: - app: ipfs - spec: - containers: - - name: ipfs - image: docker.io/ipfs/kubo:v0.23.0 - ports: - - name: rpc - containerPort: 5001 - - name: swarm-tcp - containerPort: 4001 - protocol: TCP - - name: swarm-udp - containerPort: 4001 - protocol: UDP - - name: gateway - containerPort: 8080 - volumeMounts: - - name: init-scripts - mountPath: /container-init.d/050-configure.sh - readOnly: true - subPath: 050-configure.sh - env: - - name: IPFS_PROFILE - value: - - name: p2p-helper - image: ghcr.io/comrade-coop/apocryph/p2p-helper:master - command: ["ipfs-p2p-helper", "run", "--ipfs", "/ip4/127.0.0.1/tcp/5001"] - volumes: - - name: init-scripts - configMap: - name: ipfs-config - items: - - key: configure.sh - path: 050-configure.sh - mode: 0555 - serviceAccountName: ipfs-p2p-serviceaccount -# --- -# apiVersion: apps/v1 -# kind: Deployment -# metadata: -# name: ipfs-p2p-helper -# namespace: ipfs -# spec: -# replicas: 1 -# selector: -# matchLabels: -# app: ipfs-p2p-helper -# template: -# metadata: -# labels: -# app: ipfs-p2p-helper -# spec: -# containers: -# - name: p2p-helper -# image: host.minikube.internal:5000/comradecoop/apocryph/p2p-helper -# command: ["ipfs-p2p-helper", "run", "--ipfs", "/dns4/ipfs-rpc.ipfs.svc.cluster.local/tcp/5001"] -# serviceAccountName: ipfs-p2p-serviceaccount diff --git a/test/e2e/constellation/miniconstellation/ipfs/serviceaccount.yml b/test/e2e/constellation/miniconstellation/ipfs/serviceaccount.yml deleted file mode 100644 index b0961dbc..00000000 --- a/test/e2e/constellation/miniconstellation/ipfs/serviceaccount.yml +++ /dev/null @@ -1,29 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - app: ipfs - name: ipfs-p2p-serviceaccount - namespace: ipfs -# --- -# apiVersion: rbac.authorization.k8s.io/v1 -# kind: ClusterRole -# metadata: -# name: service-lister -# rules: -# - apiGroups: ["core"] -# resources: ["service"] -# verbs: ["get", "list", "watch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: ipfs-p2p-cluster-role-binding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: view # TODO! For some reason with the service-lister role, we get random "unknown" errors -subjects: -- kind: ServiceAccount - name: ipfs-p2p-serviceaccount - namespace: ipfs diff --git a/test/e2e/constellation/miniconstellation/keda/ingress.yml b/test/e2e/constellation/miniconstellation/keda/ingress.yml deleted file mode 100644 index f7361082..00000000 --- a/test/e2e/constellation/miniconstellation/keda/ingress.yml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: keda-ingress - namespace: keda -spec: - ingressClassName: nginx - defaultBackend: - service: - name: keda-add-ons-http-interceptor-proxy - port: - number: 8080 diff --git a/test/e2e/constellation/miniconstellation/loki/values.yml b/test/e2e/constellation/miniconstellation/loki/values.yml deleted file mode 100644 index 81b625bb..00000000 --- a/test/e2e/constellation/miniconstellation/loki/values.yml +++ /dev/null @@ -1,39 +0,0 @@ -loki: - enabled: true - isDefault: true - url: http://{{(include "loki.serviceName" .)}}:{{ .Values.loki.service.port }} - readinessProbe: - httpGet: - path: /ready - port: http-metrics - initialDelaySeconds: 45 - livenessProbe: - httpGet: - path: /ready - port: http-metrics - initialDelaySeconds: 45 - datasource: - jsonData: "{}" - uid: "" - -promtail: - enabled: true - config: - logLevel: info - serverPort: 3101 - clients: - - url: http://{{ .Release.Name }}:3100/loki/api/v1/push - -fluent-bit: - enabled: false - -grafana: - enabled: true - sidecar: - datasources: - label: "" - labelValue: "" - enabled: true - maxLines: 1000 - image: - tag: 8.3.5 diff --git a/test/e2e/constellation/miniconstellation/run-test.sh b/test/e2e/constellation/miniconstellation/run-test.sh deleted file mode 100755 index 982176a9..00000000 --- a/test/e2e/constellation/miniconstellation/run-test.sh +++ /dev/null @@ -1,174 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: GPL-3.0 - -set -e - -trap 'pkill -f "kubectl port-forward" && kill $(jobs -p) &>/dev/null' EXIT - -cd "$(dirname "$0")" - -which curl >/dev/null; which jq >/dev/null; which xargs >/dev/null; which sed >/dev/null -which go >/dev/null -which ipfs >/dev/null -which forge &>/dev/null || export PATH=$PATH:~/.bin/foundry -which forge >/dev/null; which cast >/dev/null -which helmfile >/dev/null; which helm >/dev/null; which kubectl >/dev/null -which constellation >/dev/null || { echo "Install Constellation, https://docs.edgeless.systems/constellation/getting-started/first-steps-local#software-installation-on-ubuntu"; exit 1; } - -CONSTELLATION_PATH=${CONSTELLATION_PATH:-~/.constellation-root} -mkdir -p $CONSTELLATION_PATH - -if [ "$1" = "teardown" ]; then - ( cd $CONSTELLATION_PATH; constellation mini down ) - exit 0 -fi - -# based on https://stackoverflow.com/a/31269848 / https://bobcopeland.com/blog/2012/10/goto-in-bash/ -if [ -n "$1" ]; then - STEP=${1:-1} - eval "set -v; $(sed -n "/## $STEP: /{:a;n;p;ba};" $0)" - exit -fi - -echo -e "\e[1;32m---" -echo "Note: To skip steps, use '$0 '" -echo " e.g. to skip ahead to configuring IPFS, run '$0 1.2'" -echo -e "---\e[0m" - -set -v - -## 1: Set up the Kubernetes environment ## - -echo 'CONSTELLATION_PATH='$CONSTELLATION_PATH - -( cd $CONSTELLATION_PATH; constellation mini up || true ) - -kubectl patch -n kube-system configmap ip-masq-agent --type merge -p '{"data":{"config": "{\"masqLinkLocal\":true,\"nonMasqueradeCIDRs\":[]}"}}' -kubectl rollout restart -n kube-system daemonset cilium -kubectl delete pod -l k8s-app=join-service -n kube-system - -## 1.0: Deploy contracts to anvil ## - -helmfile apply -l name=eth - -{ while ! kubectl get -n eth endpoints eth-rpc -o json | jq '.subsets[].addresses[].ip' &>/dev/null; do sleep 1; done; } - -[ "$PORT_8545" == "" ] && { PORT_8545="yes" ; kubectl port-forward --namespace eth svc/eth-rpc 8545:8545 & } - -DEPLOYER_KEY=0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 #TODO= anvil.accounts[0] - -( cd ../../../contracts; forge script script/Deploy.s.sol --private-key "$DEPLOYER_KEY" --rpc-url http://localhost:8545 --broadcast) - -## 1.1: Apply the rest of the Helm configuration ## - -helmfile apply - -## 1.2: Configure provider/in-cluster IPFS and publisher IPFS ## - -{ while ! kubectl get -n ipfs endpoints ipfs-rpc -o json | jq '.subsets[].addresses[].ip' &>/dev/null; do sleep 1; done; } - -[ "$PORT_5004" == "" ] && { PORT_5004="yes" ; kubectl port-forward --namespace ipfs svc/ipfs-rpc 5004:5001 & sleep 0.5; } - -NODE_ADDRESS=$(kubectl get no -o json | jq -r '.items[].status.addresses[] | select(.type == "InternalIP") | .address' | head -n 1) -SWARM_PORT=$(kubectl get svc -n ipfs ipfs-swarm -o json | jq -r '.spec.ports[].nodePort' | head -n 1) - -SWARM_ADDRESSES="[\"/ip4/$NODE_ADDRESS/tcp/$SWARM_PORT\", \"/ip4/$NODE_ADDRESS/udp/$SWARM_PORT/quic\", \"/ip4/$NODE_ADDRESS/udp/$SWARM_PORT/quic-v1\", \"/ip4/$NODE_ADDRESS/udp/$SWARM_PORT/quic-v1/webtransport\"]" - -PROVIDER_IPFS=$(curl -X POST "http://127.0.0.1:5004/api/v0/id" | jq '.ID' -r); echo $PROVIDER_IPFS - -# Unfortunately, we can't restart the ipfs daemon since we don't have persistent storage in miniconstellation. Swarm addresses have been hardcoded. -#O_IPFS_PATH=$IPFS_PATH -#export IPFS_PATH=$(mktemp ipfs.XXXX --tmpdir -d) -#echo /ip4/127.0.0.1/tcp/5004 > $IPFS_PATH/api - -#CONFIG_BEFORE=$(ipfs config Addresses.AppendAnnounce) -#ipfs config Addresses.AppendAnnounce --json "$SWARM_ADDRESSES" -#CONFIG_AFTER=$(ipfs config Addresses.AppendAnnounce) - -#[ "$CONFIG_BEFORE" = "$CONFIG_AFTER" ] || false # Restart ipfs daemon -#export IPFS_PATH=$O_IPFS_PATH - -ipfs id &>/dev/null || ipfs init - -ipfs config --json Experimental.Libp2pStreamMounting true - -[ -n "$IPFS_DAEMON" ] || { IPFS_DAEMON=yes; ipfs daemon & { while ! [ -f ${IPFS_PATH:-~/.ipfs}/api ]; do sleep 0.1; done; } 2>/dev/null; } - -echo "$SWARM_ADDRESSES" | jq -r '.[] + "/p2p/'"$PROVIDER_IPFS"'"' | xargs -n 1 ipfs swarm connect || true - -sleep 1 - -## 1.3: Register the provider - -go run ../../../cmd/tpodserver/ registry register \ - --config ../../common/configs/config.yaml \ - --ipfs /ip4/127.0.0.1/tcp/5001 \ - --ethereum-rpc http://127.0.0.1:8545 \ - --ethereum-key 0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d \ - --token-contract 0x5FbDB2315678afecb367f032d93F642f64180aa3 \ - --registry-contract 0x9fe46736679d2d9a65f0992f2272de9f3c7fa6e0 \ - -## 2: Deploy example manifest to cluster ## - -DEPLOYER_ETH=0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266 #TODO= anvil.accounts[0] -PROVIDER_ETH=0x70997970C51812dc3A010C7d01b50e0d17dc79C8 #TODO= anvil.accounts[1] -PUBLISHER_KEY=0x5de4111afa1a4b94908f83103eb1f1706367c2e68ca870fc3fb9a804cdab365a #TODO= anvil.accounts[2] -PAYMENT_CONTRACT=$(cat ../../../contracts/broadcast/Deploy.s.sol/31337/run-latest.json | jq -r '.returns.payment.value') -REGISTRY_CONTRACT=$(cat ../../../contracts/broadcast/Deploy.s.sol/31337/run-latest.json | jq -r '.returns.registry.value') -FUNDS=10000000000000000000000 - -[ "$PORT_8545" == "" ] && { PORT_8545="yes" ; kubectl port-forward --namespace eth svc/eth-rpc 8545:8545 & } -[ "$PORT_5004" == "" ] && { PORT_5004="yes" ; kubectl port-forward --namespace ipfs svc/ipfs-rpc 5004:5001 & sleep 0.5; } -[ -n "$PROVIDER_IPFS" ] || { PROVIDER_IPFS=$(curl -X POST "http://127.0.0.1:5004/api/v0/id" -s | jq '.ID' -r); echo $PROVIDER_IPFS; } -[ -n "$IPFS_DAEMON" ] || { IPFS_DAEMON=yes; ipfs daemon & { while ! [ -f ${IPFS_PATH:-~/.ipfs}/api ]; do sleep 0.1; done; } 2>/dev/null; } - -set +v -set -x - -go run ../../../cmd/trustedpods/ pod deploy ../common/manifest-guestbook-nostorage.yaml \ - --ethereum-key "$PUBLISHER_KEY" \ - --payment-contract "$PAYMENT_CONTRACT" \ - --registry-contract "$REGISTRY_CONTRACT" \ - --funds "$FUNDS" \ - --upload-images=false \ - --mint-funds - -set +x -set -v - -## 3: Connect and measure balances ## - -DEPLOYER_ETH=0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266 #TODO= anvil.accounts[0] -WITHDRAW_ETH=0x90F79bf6EB2c4f870365E785982E1f101E93b906 # From trustedpods/tpodserver.yml -TOKEN_CONTRACT=$(cat ../../../contracts/broadcast/Deploy.s.sol/31337/run-latest.json | jq -r '.returns.token.value') -NODE_ADDRESS=$(kubectl get no -o json | jq -r '.items[].status.addresses[] | select(.type == "InternalIP") | .address' | head -n 1) -INGRESS_PORT=$(kubectl get svc -n keda ingress-nginx-controller -o json | jq -r '.spec.ports[] | select(.name == "http") | .nodePort' | head -n 1) -INGRESS_URL="http://$NODE_ADDRESS:$INGRESS_PORT"; echo $INGRESS_URL -MANIFEST_HOST=guestbook.localhost # From manifest-guestbook.yaml - -[ "$PORT_8545" == "" ] && { PORT_8545="yes" ; kubectl port-forward --namespace eth svc/eth-rpc 8545:8545 & } - -echo "Provider balance before:" $(cast call "$TOKEN_CONTRACT" "balanceOf(address)" "$WITHDRAW_ETH" | cast to-fixed-point 18) - -set -x - -while ! curl --connect-timeout 40 -H "Host: $MANIFEST_HOST" $INGRESS_URL --fail-with-body; do sleep 10; done -curl -H "Host: $MANIFEST_HOST" $INGRESS_URL/test.html --fail-with-body - -set +x - -sleep 45 - -echo "Provider balance after:" $(cast call "$TOKEN_CONTRACT" "balanceOf(address)" "$WITHDRAW_ETH" | cast to-fixed-point 18) - -## 4: In conclusion.. ## - -set +v - -echo -e "\e[1;32m---" -echo "Note: To interact with the deployed guestbook, run the following" -echo " kubectl port-forward --namespace keda svc/ingress-nginx-controller 1234:80 &" -echo " xdg-open http://guestbook.localhost:1234/" -echo "Note: To stop the minikube cluster/provider, use '$0 teardown'" -echo " and to clean-up everything the script does, use '$0 teardown full'" -echo -e "---\e[0m" diff --git a/test/e2e/constellation/miniconstellation/trustedpods/serviceaccount.yml b/test/e2e/constellation/miniconstellation/trustedpods/serviceaccount.yml deleted file mode 100644 index b6e400b6..00000000 --- a/test/e2e/constellation/miniconstellation/trustedpods/serviceaccount.yml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - app: tpodserver - name: tpodserver-serviceaccount - namespace: trustedpods -# --- -# apiVersion: rbac.authorization.k8s.io/v1 -# kind: ClusterRole -# metadata: -# name: my-cluster-role -# namespace: trustedpods -# rules: -# - apiGroups: [""] -# resources: ["pods"] -# verbs: ["get", "list"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: tpodserver-cluster-role-binding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin # TODO -subjects: -- kind: ServiceAccount - name: tpodserver-serviceaccount - namespace: trustedpods diff --git a/test/e2e/constellation/miniconstellation/trustedpods/tpodserver.yml b/test/e2e/constellation/miniconstellation/trustedpods/tpodserver.yml deleted file mode 100644 index de212781..00000000 --- a/test/e2e/constellation/miniconstellation/trustedpods/tpodserver.yml +++ /dev/null @@ -1,122 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: tpodserver - namespace: trustedpods - labels: - app: tpodserver - coop.comrade/apocryph-p2p-helper: "true" - annotations: - coop.comrade/apocryph-p2p-helper: "/x/apocryph/provision-pod/0.0.1" - -spec: - ports: - - port: 8080 - type: ClusterIP - selector: - app: tpodserver ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: trustedpods-configs - namespace: trustedpods -data: - config.yaml: | - pricing: - table: - format: yaml - # filename: pricing.yaml - contents: | - tables: - - paymentContractAddress: "5/FyXnc0ziiPg2fhuxQ+kLs/BRI=" - resources: - - resource: "cpu" - priceForReservation: 200000000000 - - resource: "ram" - priceForReservation: 500 - withdraw: - address: "0x90F79bf6EB2c4f870365E785982E1f101E93b906" - time: 15 - cpu_model: "Intel Xeon Platinum 8452Y Processor" - tee_type: "Secure Enclaves" - info: - format: yaml - contents: | - regions: - - name: "bul" - zone: "east" - num: 1 - - name: "alg" - zone: "west" - num: 2 - multiaddrs: - - "/dns4/kubo.business" - # tokenAddress=$(echo e7f1725E7734CE288F8367e1Bb143E90bb3F0512 | xxd -p -r | base64) ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: tpodserver - namespace: trustedpods -spec: - replicas: 1 - selector: - matchLabels: - app: tpodserver - template: - metadata: - labels: - app: tpodserver - spec: - containers: - - name: tpodserver - image: ghcr.io/comrade-coop/apocryph/server:master - command: [ - "tpodserver", "listen", - "--address", "0.0.0.0:8080", - "--config", "config.yaml", - "--ipfs", "/dns4/ipfs-rpc.ipfs.svc.cluster.local/tcp/5001", - "--oci-registry", "host.minikube.internal:5000", - "--ethereum-rpc", "http://eth-rpc.eth.svc.cluster.local:8545", - "--ethereum-key", "0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d", # TODO= anvil.accounts[1] prvkey - ] - ports: - - containerPort: 8080 - securityContext: - allowPrivilegeEscalation: false - runAsUser: 0 - volumeMounts: - - name: configs - mountPath: /config.yaml - subPath: config.yaml - readOnly: true - - name: containerd-socket - mountPath: /run/containerd/containerd.sock - - name: tpodmonitor - image: ghcr.io/comrade-coop/apocryph/server:master - command: [ - "tpodserver", "monitor", - "--config", "config.yaml", - "--prometheus", "http://prometheus-server.prometheus.svc.cluster.local:80/", - "--ethereum-rpc", "http://eth-rpc.eth.svc.cluster.local:8545", - "--ethereum-key", "0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d", # TODO= anvil.accounts[1] - ] - ports: - - containerPort: 8080 - volumeMounts: - - name: configs - mountPath: /config.yaml - subPath: config.yaml - readOnly: true - volumes: - - name: configs - configMap: - name: trustedpods-configs - items: - - key: config.yaml - path: config.yaml - - name: containerd-socket - hostPath: - path: /run/containerd/containerd.sock - serviceAccountName: tpodserver-serviceaccount diff --git a/test/e2e/constellation/qemu/run-test.sh b/test/e2e/constellation/qemu/run-test.sh deleted file mode 100755 index ed43cd0e..00000000 --- a/test/e2e/constellation/qemu/run-test.sh +++ /dev/null @@ -1,159 +0,0 @@ -#!/bin/bash -set -e -set -v - -WORKSPACE_PATH="$HOME/.apocryph/constellation-" -CURRENT_DIR=$(pwd) - -trap 'pkill -f "kubectl port-forward" && kill $(jobs -p) &>/dev/null' EXIT - -# based on https://stackoverflow.com/a/31269848 / https://bobcopeland.com/blog/2012/10/goto-in-bash/ -if [ -n "$1" ]; then - STEP=${1:-1} - eval "set -v; $(sed -n "/## $STEP: /{:a;n;p;ba};" $0)" - exit -fi - -echo -e "\e[1;32m---" -echo "Note: To skip steps, use '$0 '" -echo -e "---\e[0m" - -## 0: Build build custom OS image & run the cluster - -# use the miniconstellation chart -. ./build.sh ../miniconstellation - -## 1: Start the constellation cluster -cd "$WORKSPACE_PATH"* -constellation terminate -constellation apply -y - -## 1.1: wait for the setup -cd $WORKSPACE_PATH* -CONF_DIR=$(pwd) -export KUBECONFIG="$CONF_DIR/constellation-admin.conf" -cd $CURRENT_DIR -kubectl wait --namespace keda --for=condition=available deployment/ingress-nginx-controller -kubectl wait --namespace prometheus --for=condition=available deployment/prometheus-kube-state-metrics -kubectl wait --namespace prometheus --for=condition=available deployment/prometheus-prometheus-pushgateway -kubectl wait --namespace prometheus --for=condition=available deployment/prometheus-server -kubectl wait --namespace ipfs --for=condition=available StatefulSet/ipfs -kubectl wait --namespace trustedpods --for=condition=available deployment/tpodserver - -## 2: Deploy sample App - -[ "$PORT_8545" == "" ] && { PORT_8545="yes" ; kubectl port-forward --namespace eth svc/eth-rpc 8545:8545 & } - -sleep 2 - -DEPLOYER_KEY=0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 #TODO= anvil.accounts[0] - -TOKEN_CONTRACT=0x5fbdb2315678afecb367f032d93f642f64180aa3 # TODO= result of forge create - -( cd ../../../../contracts; forge script script/Deploy.s.sol --private-key "$DEPLOYER_KEY" --rpc-url http://127.0.0.1:8545 --broadcast) - -## 2.1: Register the provider -[ "$PORT_5004" == "" ] && { PORT_5004="yes" ; kubectl port-forward --namespace ipfs svc/ipfs-rpc 5004:5001 & sleep 0.5; } - -go run ../../../../cmd/tpodserver registry register \ - --config ../../common/configs/config.yaml \ - --ipfs /ip4/127.0.0.1/tcp/5004 \ - --ethereum-rpc http://127.0.0.1:8545 \ - --ethereum-key 0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d \ - --token-contract 0x5FbDB2315678afecb367f032d93F642f64180aa3 \ - --registry-contract 0x9fe46736679d2d9a65f0992f2272de9f3c7fa6e0 \ - -## 3: Configure provider/in-cluster IPFS and publisher IPFS ## - -{ while ! kubectl get -n ipfs endpoints ipfs-rpc -o json | jq '.subsets[].addresses[].ip' &>/dev/null; do sleep 1; done; } - -[ "$PORT_5004" == "" ] && { PORT_5004="yes" ; kubectl port-forward --namespace ipfs svc/ipfs-rpc 5004:5001 & sleep 0.5; } - -NODE_ADDRESS=$(kubectl get no -o json | jq -r '.items[].status.addresses[] | select(.type == "InternalIP") | .address' | head -n 1) -SWARM_PORT=$(kubectl get svc -n ipfs ipfs-swarm -o json | jq -r '.spec.ports[].nodePort' | head -n 1) - -SWARM_ADDRESSES="[\"/ip4/$NODE_ADDRESS/tcp/$SWARM_PORT\", \"/ip4/$NODE_ADDRESS/udp/$SWARM_PORT/quic\", \"/ip4/$NODE_ADDRESS/udp/$SWARM_PORT/quic-v1\", \"/ip4/$NODE_ADDRESS/udp/$SWARM_PORT/quic-v1/webtransport\"]" - -PROVIDER_IPFS=$(curl -X POST "http://127.0.0.1:5004/api/v0/id" | jq '.ID' -r); echo $PROVIDER_IPFS - -ipfs id &>/dev/null || ipfs init - -ipfs config --json Experimental.Libp2pStreamMounting true - -IPFS_CONFIG="$HOME/.ipfs/config" -# default gateway is already used by libvirt container -NEW_GATEWAY="/ip4/127.0.0.1/tcp/8090" -jq --arg new_gateway "$NEW_GATEWAY" '.Addresses.Gateway = $new_gateway' "$IPFS_CONFIG" > tmp.json && mv tmp.json "$IPFS_CONFIG" - -[ -n "$IPFS_DAEMON" ] || { IPFS_DAEMON=yes; ipfs daemon & { while ! [ -f ${IPFS_PATH:-~/.ipfs}/api ]; do sleep 0.1; done; } 2>/dev/null; } - -echo "$SWARM_ADDRESSES" | jq -r '.[] + "/p2p/'"$PROVIDER_IPFS"'"' | xargs -n 1 ipfs swarm connect || true - -sleep 1 - -# Deploy example manifest to cluster ## - -DEPLOYER_ETH=0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266 #TODO= anvil.accounts[0] -PROVIDER_ETH=0x70997970C51812dc3A010C7d01b50e0d17dc79C8 #TODO= anvil.accounts[1] -PUBLISHER_KEY=0x5de4111afa1a4b94908f83103eb1f1706367c2e68ca870fc3fb9a804cdab365a #TODO= anvil.accounts[2] - -TOKEN_CONTRACT=0x5fbdb2315678afecb367f032d93f642f64180aa3 # TODO= result of forge create -PAYMENT_CONTRACT=0xe7f1725e7734ce288f8367e1bb143e90bb3f0512 # TODO= result of forge create -REGISTRY_CONTRACT=0x9fe46736679d2d9a65f0992f2272de9f3c7fa6e0 # TODO= result of forge create - - -# PAYMENT_CONTRACT=$(cat ../../../../contracts/broadcast/Deploy.s.sol/31337/run-latest.json | jq -r '.returns.payment.value') -# REGISTRY_CONTRACT=$(cat ../../../../contracts/broadcast/Deploy.s.sol/31337/run-latest.json | jq -r '.returns.registry.value') -FUNDS=10000000000000000000000 - - -[ "$PORT_8545" == "" ] && { PORT_8545="yes" ; kubectl port-forward --namespace eth svc/eth-rpc 8545:8545 & } -[ -n "$IPFS_DAEMON" ] || { IPFS_DAEMON=yes; ipfs daemon & { while ! [ -f ${IPFS_PATH:-~/.ipfs}/api ]; do sleep 0.1; done; } 2>/dev/null; } - -set +v -set -x - -go run ../../../../cmd/trustedpods/ pod deploy ../../common/manifest-nginx.yaml \ - --ethereum-key "$PUBLISHER_KEY" \ - --payment-contract "$PAYMENT_CONTRACT" \ - --registry-contract "$REGISTRY_CONTRACT" \ - --funds "$FUNDS" \ - --upload-images=true \ - --mint-funds - -## 4: Connect and measure balances ## - -DEPLOYER_ETH=0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266 #TODO= anvil.accounts[0] -WITHDRAW_ETH=0x90F79bf6EB2c4f870365E785982E1f101E93b906 # From trustedpods/tpodserver.yml -TOKEN_CONTRACT=$(cat ../../../contracts/broadcast/Deploy.s.sol/31337/run-latest.json | jq -r '.returns.token.value') -NODE_ADDRESS=$(kubectl get no -o json | jq -r '.items[].status.addresses[] | select(.type == "InternalIP") | .address' | head -n 1) -INGRESS_PORT=$(kubectl get svc -n keda ingress-nginx-controller -o json | jq -r '.spec.ports[] | select(.name == "http") | .nodePort' | head -n 1) -INGRESS_URL="http://$NODE_ADDRESS:$INGRESS_PORT"; echo $INGRESS_URL -MANIFEST_HOST=example.local # From manifest-guestbook.yaml - -[ "$PORT_8545" == "" ] && { PORT_8545="yes" ; kubectl port-forward --namespace eth svc/eth-rpc 8545:8545 & } - -echo "Provider balance before:" $(cast call "$TOKEN_CONTRACT" "balanceOf(address)" "$WITHDRAW_ETH" | cast to-fixed-point 18) - -set -x - -while ! curl --connect-timeout 40 -H "Host: $MANIFEST_HOST" $INGRESS_URL --fail-with-body; do sleep 10; done -curl -H "Host: $MANIFEST_HOST" $INGRESS_URL --fail-with-body - -set +x - -sleep 45 - -echo "Provider balance after:" $(cast call "$TOKEN_CONTRACT" "balanceOf(address)" "$WITHDRAW_ETH" | cast to-fixed-point 18) - -## 4: In conclusion.. ## - -set +v - -echo -e "\e[1;32m---" -echo "Note: To interact with the deployed guestbook, run the following" -echo " kubectl port-forward --namespace keda svc/ingress-nginx-controller 1234:80 &" -echo " xdg-open http://guestbook.localhost:1234/" -echo "Note: To stop the minikube cluster/provider, use '$0 teardown'" -echo " and to clean-up everything the script does, use '$0 teardown full'" -echo -e "---\e[0m" diff --git a/test/e2e/nginx/Tiltfile b/test/e2e/nginx/Tiltfile index 50341afe..104e29f7 100644 --- a/test/e2e/nginx/Tiltfile +++ b/test/e2e/nginx/Tiltfile @@ -32,7 +32,9 @@ provider_balance_cmd = 'cast call "%s" "balanceOf(address)" "%s" -r %s | cast to ) curl_cmd = 'curl --connect-timeout 40 -H "Host: %s" localhost:8004 --fail-with-body' % manifest_host -k8s_resource(workload="ingress-nginx", port_forwards=["8004:80"]) +local_resource('ingress-nginx-portforward', serve_cmd='kubectl port-forward -n keda svc/ingress-nginx-controller 8004:80') +# k8s_resource(workload="ingress-nginx", port_forwards=["8004:80"]) + local_resource( "nginx-example-measure-and-query", [