Integration tests #285
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Integration tests | |
| on: | |
| workflow_call: | |
| inputs: | |
| resource-group: | |
| description: Resource group for cluster creation | |
| type: string | |
| required: true | |
| default: ops-cli-int-test-rg | |
| runtime-init-args: | |
| description: Additional INIT arguments (beyond cluster name, resource group, schema registry). | |
| type: string | |
| required: false | |
| default: '' | |
| runtime-create-args: | |
| description: Additional CREATE arguments (beyond cluster name, resource group, instance name). | |
| type: string | |
| required: false | |
| default: '' | |
| custom-locations-oid: | |
| description: Custom Locations OID | |
| type: string | |
| required: false | |
| default: '51dfe1e8-70c6-4de5-a08e-e18aff23d815' | |
| init-continue-on-error: | |
| description: Continue on error for init integration tests | |
| type: boolean | |
| required: false | |
| default: true | |
| use-container: | |
| description: Build container image for tests | |
| type: boolean | |
| required: false | |
| default: false | |
| secrets: | |
| # required for az login | |
| AZURE_CLIENT_ID: | |
| required: true | |
| AZURE_TENANT_ID: | |
| required: true | |
| AZURE_SUBSCRIPTION_ID: | |
| required: true | |
| # optional --sp-* init params | |
| AIO_SP_APP_ID: | |
| AIO_SP_OBJECT_ID: | |
| AIO_SP_SECRET: | |
| workflow_dispatch: | |
| inputs: | |
| resource-group: | |
| description: Resource group for cluster creation | |
| type: string | |
| required: true | |
| default: ops-cli-int-test-rg | |
| custom-locations-oid: | |
| description: Object ID of Custom Locations RP | |
| type: string | |
| required: false | |
| default: '51dfe1e8-70c6-4de5-a08e-e18aff23d815' | |
| runtime-init-args: | |
| description: Additional INIT arguments (beyond cluster name, resource group, schema registry). | |
| type: string | |
| required: false | |
| default: '' | |
| runtime-create-args: | |
| description: Additional CREATE arguments (beyond cluster name, resource group, instance name). | |
| type: string | |
| required: false | |
| default: '' | |
| init-continue-on-error: | |
| description: Continue on error for init integration tests | |
| type: boolean | |
| required: false | |
| default: true | |
| use-container: | |
| description: Build container image for tests | |
| type: boolean | |
| required: false | |
| default: false | |
| permissions: | |
| # required for OpenID federation | |
| contents: 'read' | |
| id-token: 'write' | |
| env: | |
| RESOURCE_GROUP: "${{ inputs.resource-group }}" | |
| jobs: | |
| test: | |
| env: | |
| CLUSTER_NAME: "opt${{ github.run_number }}${{ matrix.feature }}" | |
| INSTANCE_NAME: "inst${{ github.run_number }}${{ matrix.feature }}" | |
| CUSTOM_LOCATIONS_OID: ${{ inputs.custom-locations-oid }} | |
| EXTENSION_SOURCE_DIRECTORY: "./azure-iot-ops-cli-extension" | |
| K3S_VERSION: "v1.28.5+k3s1" | |
| CA_FILE: "test-ca.pem" | |
| CA_KEY_FILE: "test-ca-key.pem" | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| feature: [custom-input, default, insecure-listener, syncrules, redeploy, trustbundle] | |
| runtime-args: | |
| - ${{ inputs.runtime-init-args != '' || inputs.runtime-create-args != '' }} | |
| exclude: | |
| - feature: custom-input | |
| runtime-args: false | |
| - feature: default | |
| runtime-args: true | |
| - feature: insecure-listener | |
| runtime-args: true | |
| - feature: syncrules | |
| runtime-args: true | |
| - feature: redeploy | |
| runtime-args: true | |
| - feature: trustbundle | |
| runtime-args: true | |
| name: "Run cluster tests" | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: "Determine Init Args" | |
| id: "init" | |
| run: | | |
| echo "NO_PREFLIGHT=false" >> $GITHUB_OUTPUT | |
| if [[ ${{ matrix.feature }} == "insecure-listener" ]]; then | |
| echo "CREATE_ARG=--add-insecure-listener --broker-listener-type NodePort" >> $GITHUB_OUTPUT | |
| echo "NO_PREFLIGHT=true" >> $GITHUB_OUTPUT | |
| elif [[ ${{ matrix.feature }} == "syncrules" ]]; then | |
| echo "CREATE_ARG=--enable-rsync" >> $GITHUB_OUTPUT | |
| elif [[ ${{ matrix.feature }} == "redeploy" ]]; then | |
| echo "REDEPLOY=True" >> $GITHUB_OUTPUT | |
| elif [[ ${{ matrix.feature }} == "redeploy" ]]; then | |
| echo "CREATE_ARG=${{ inputs.runtime-create-args }}" >> $GITHUB_OUTPUT | |
| echo "INIT_ARG=${{ inputs.runtime-init-args }}" >> $GITHUB_OUTPUT | |
| elif [[ ${{ matrix.feature }} == "trustbundle" ]]; then | |
| echo "CREATE_ARG= --trust-settings configMapName=example-bundle configMapKey=trust-bundle.pem issuerKind=Issuer issuerName=selfsigned-issuer" >> $GITHUB_OUTPUT | |
| echo "INIT_ARG=--user-trust true" >> $GITHUB_OUTPUT | |
| fi | |
| - name: "Output variables for future steps" | |
| id: "env_out" | |
| run: | | |
| echo "RESOURCE_GROUP=${{ env.RESOURCE_GROUP }}" >> $GITHUB_OUTPUT | |
| echo "CLUSTER_NAME=${{ env.CLUSTER_NAME }}" >> $GITHUB_OUTPUT | |
| echo "INSTANCE_NAME=${{ env.INSTANCE_NAME }}" >> $GITHUB_OUTPUT | |
| - name: "Setup python" | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: "Create k3s cluster" | |
| run: | | |
| # vars | |
| release_url="https://github.com/k3s-io/k3s/releases/download" | |
| install_dir="/usr/local/bin/k3s" | |
| # install | |
| sudo apt install nfs-common | |
| curl -Lo $install_dir "$release_url/${{ env.K3S_VERSION }}/k3s" | |
| chmod a+x $install_dir | |
| # config | |
| K3S_KUBECONFIG_MODE="644" | |
| # start cluster in background | |
| sudo k3s server --cluster-init & | |
| - name: "Copy config" | |
| run: | | |
| mkdir ~/.kube 2> /dev/null | |
| sudo k3s kubectl config view --raw > ~/.kube/config | |
| chmod 600 ~/.kube/config | |
| export KUBECONFIG=~/.kube/config | |
| - name: "Apply custom cluster issuer" | |
| if: ${{ matrix.feature == 'trustbundle' && !inputs.use-container }} | |
| run: | | |
| set -x | |
| helm repo add jetstack https://charts.jetstack.io --force-update | |
| helm install \ | |
| cert-manager jetstack/cert-manager \ | |
| --namespace cert-manager \ | |
| --create-namespace \ | |
| --version v1.13.6 \ | |
| --set installCRDs=true \ | |
| --set startupapicheck.timeout=5m | |
| kubectl create namespace azure-iot-operations | |
| helm upgrade trust-manager jetstack/trust-manager \ | |
| --install \ | |
| --namespace cert-manager \ | |
| --set app.trust.namespace=azure-iot-operations \ | |
| --wait | |
| kubectl apply -f - <<EOF | |
| apiVersion: cert-manager.io/v1 | |
| kind: Issuer | |
| metadata: | |
| name: selfsigned-issuer | |
| namespace: azure-iot-operations | |
| spec: | |
| selfSigned: {} | |
| --- | |
| apiVersion: cert-manager.io/v1 | |
| kind: Certificate | |
| metadata: | |
| name: trust-manager-example-ca | |
| namespace: azure-iot-operations | |
| spec: | |
| isCA: true | |
| commonName: trust-manager-example-ca | |
| secretName: trust-manager-example-ca-secret | |
| privateKey: | |
| algorithm: ECDSA | |
| size: 256 | |
| issuerRef: | |
| name: selfsigned-issuer | |
| kind: Issuer | |
| group: cert-manager.io | |
| EOF | |
| kubectl apply -f - <<EOF | |
| apiVersion: trust.cert-manager.io/v1alpha1 | |
| kind: Bundle | |
| metadata: | |
| name: example-bundle | |
| namespace: azure-iot-operations | |
| spec: | |
| sources: | |
| - useDefaultCAs: true | |
| - secret: | |
| name: "trust-manager-example-ca-secret" | |
| key: "tls.crt" | |
| target: | |
| configMap: | |
| key: "trust-bundle.pem" | |
| namespaceSelector: | |
| matchLabels: | |
| kubernetes.io/metadata.name: azure-iot-operations | |
| EOF | |
| - name: "Checkout extension source for build" | |
| uses: actions/checkout@v4 | |
| with: | |
| # ensure source checkout uses our repo instead of calling workflow | |
| repository: azure/azure-iot-ops-cli-extension | |
| path: ${{ env.EXTENSION_SOURCE_DIRECTORY }} | |
| - name: "Build and install local IoT Ops extension from source" | |
| run: | | |
| cd ${{ env.EXTENSION_SOURCE_DIRECTORY }} | |
| pip install -r dev_requirements.txt | |
| python -m setup bdist_wheel -d dist | |
| wheel=$(find ./dist/*.whl) | |
| az extension add --source $wheel -y | |
| - name: "Az CLI login" | |
| uses: azure/login@v2 | |
| with: | |
| client-id: ${{ secrets.AZURE_CLIENT_ID }} | |
| tenant-id: ${{ secrets.AZURE_TENANT_ID }} | |
| subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| - name: "OIDC Token exchange service" | |
| run: | | |
| while true; do | |
| token_request=$ACTIONS_ID_TOKEN_REQUEST_TOKEN | |
| token_uri=$ACTIONS_ID_TOKEN_REQUEST_URL | |
| token=$(curl -H "Authorization: bearer $token_request" "${token_uri}&audience=api://AzureADTokenExchange" | jq .value -r) | |
| az login --service-principal -u ${{ secrets.AZURE_CLIENT_ID }} -t ${{ secrets.AZURE_TENANT_ID }} --federated-token $token --output none | |
| # Sleep for 4 minutes | |
| sleep 240 | |
| done & | |
| - name: "ARC connect cluster" | |
| uses: azure/azure-iot-ops-cli-extension/.github/actions/connect-arc@dev | |
| with: | |
| cluster-name: ${{ env.CLUSTER_NAME }} | |
| resource-group: ${{ env.RESOURCE_GROUP }} | |
| custom-locations-oid: ${{ env.CUSTOM_LOCATIONS_OID }} | |
| - name: "Tox test environment setup for init" | |
| run: | | |
| cd ${{ env.EXTENSION_SOURCE_DIRECTORY }} | |
| python -m pip install tox | |
| tox r -vv -e python-init-int --notest | |
| - name: "Tox test environment setup for integration tests" | |
| if: ${{ matrix.feature == 'default' && !inputs.use-container }} | |
| run: | | |
| cd ${{ env.EXTENSION_SOURCE_DIRECTORY }} | |
| tox r -vv -e python-int --notest | |
| - name: "Tox INIT Integration Tests" | |
| env: | |
| AIO_CLI_INIT_PREFLIGHT_DISABLED: ${{ steps.init.outputs.NO_PREFLIGHT }} | |
| azext_edge_init_continue_on_error: ${{ inputs.init-continue-on-error || '' }} | |
| azext_edge_rg: ${{ steps.env_out.outputs.RESOURCE_GROUP }} | |
| azext_edge_cluster: ${{ steps.env_out.outputs.CLUSTER_NAME }} | |
| azext_edge_instance: ${{ steps.env_out.outputs.INSTANCE_NAME }} | |
| azext_edge_init_args: ${{ steps.init.outputs.INIT_ARG }} | |
| azext_edge_create_args: ${{ steps.init.outputs.CREATE_ARG }} | |
| azext_edge_init_redeployment: ${{ steps.init.outputs.REDEPLOY }} | |
| run: | | |
| cd ${{ env.EXTENSION_SOURCE_DIRECTORY }} | |
| tox r -e python-init-int --skip-pkg-install -- --durations=0 | |
| - name: "Tox Integration Tests" | |
| if: ${{ matrix.feature == 'default' && !inputs.use-container }} | |
| env: | |
| azext_edge_rg: ${{ steps.env_out.outputs.RESOURCE_GROUP }} | |
| azext_edge_cluster: ${{ steps.env_out.outputs.CLUSTER_NAME }} | |
| azext_edge_instance: ${{ steps.env_out.outputs.INSTANCE_NAME }} | |
| run: | | |
| cd ${{ env.EXTENSION_SOURCE_DIRECTORY }} | |
| tox r -e python-int --skip-pkg-install -- --durations=0 | |
| coverage=$(jq .totals.percent_covered coverage.json | cut -c1-4) | |
| echo "Code coverage: $coverage%" >> $GITHUB_STEP_SUMMARY | |
| - name: "Containerized tests" | |
| if: ${{ matrix.feature == 'default' && inputs.use-container }} | |
| env: | |
| azext_edge_skip_init: true # skip init tests in container | |
| azext_edge_init_redeployment: false # ensure no redeployment in container | |
| AIO_CLI_INIT_PREFLIGHT_DISABLED: ${{ steps.init.outputs.NO_PREFLIGHT }} | |
| azext_edge_rg: ${{ steps.env_out.outputs.RESOURCE_GROUP }} | |
| azext_edge_cluster: ${{ steps.env_out.outputs.CLUSTER_NAME }} | |
| azext_edge_instance: ${{ steps.env_out.outputs.INSTANCE_NAME }} | |
| run: | | |
| # volume mounts | |
| azure_dir=$(realpath ~/.azure) | |
| kubeconfig=$(realpath ~/.kube/config) | |
| kubeconfig_mount=/root/.kube/config | |
| tempLog=$(mktemp -d) | |
| # env vars | |
| envVars=() | |
| envVars+=("-e" "azext_edge_skip_init=$azext_edge_skip_init") | |
| envVars+=("-e" "azext_edge_init_redeployment=$azext_edge_init_redeployment") | |
| envVars+=("-e" "AIO_CLI_INIT_PREFLIGHT_DISABLED=$AIO_CLI_INIT_PREFLIGHT_DISABLED") | |
| envVars+=("-e" "azext_edge_rg=$azext_edge_rg") | |
| envVars+=("-e" "azext_edge_cluster=$azext_edge_cluster") | |
| envVars+=("-e" "azext_edge_instance=$azext_edge_instance") | |
| envVars+=("-e" "KUBECONFIG=$kubeconfig_mount") | |
| # Run tests | |
| set +e | |
| docker run \ | |
| --rm \ | |
| "${envVars[@]}" \ | |
| -v "$kubeconfig:$kubeconfig_mount:ro" \ | |
| -v "${azure_dir}:/root/.azure" \ | |
| -v "${tempLog}:/usr/src/azure-iot-ops/junit" \ | |
| --network host \ | |
| $(docker build ${{ env.EXTENSION_SOURCE_DIRECTORY }} -q) | |
| - name: "Run smoke tests" | |
| run: | | |
| az iot ops support create-bundle | |
| az iot ops support create-bundle --svc broker --broker-traces | |
| az iot ops support create-bundle --ops-service acs secretstore | |
| az iot ops check | |
| az iot ops check --pre | |
| az iot ops check --post | |
| az iot ops check --as-object | |
| az iot ops check --svc broker --resources broker brokerlistener | |
| az iot ops asset query -g ${{ env.RESOURCE_GROUP }} --location westus -o table | |
| - name: "Delete Cluster for redeployment" | |
| if: ${{ matrix.feature == 'redeploy' }} | |
| run: | | |
| az iot ops delete --cluster ${{ env.CLUSTER_NAME }} -g ${{ env.RESOURCE_GROUP }} -y | |
| - name: "Redeploy cluster via tox" | |
| if: ${{ matrix.feature == 'redeploy' }} | |
| env: | |
| azext_edge_rg: ${{ steps.env_out.outputs.RESOURCE_GROUP }} | |
| azext_edge_cluster: ${{ steps.env_out.outputs.CLUSTER_NAME }} | |
| azext_edge_instance: ${{ steps.env_out.outputs.INSTANCE_NAME }} | |
| azext_edge_init_args: ${{ steps.init.outputs.INIT_ARG }} | |
| azext_edge_create_args: ${{ steps.init.outputs.CREATE_ARG }} | |
| run: | | |
| cd ${{ env.EXTENSION_SOURCE_DIRECTORY }} | |
| tox r -e python-init-int --skip-pkg-install -- --durations=0 | |
| - name: "Delete AIO resources" | |
| if: ${{ always() }} | |
| run: | | |
| az iot ops delete --cluster ${{ env.CLUSTER_NAME }} -g ${{ env.RESOURCE_GROUP }} -y | |
| - name: "Delete connected cluster" | |
| if: ${{ always() }} | |
| run: | | |
| az resource delete -v --name ${{ env.CLUSTER_NAME }} -g ${{ env.RESOURCE_GROUP }} --resource-type Microsoft.Kubernetes/connectedClusters --verbose |