diff --git a/.cloudtest.yaml b/.cloudtest.yaml deleted file mode 100644 index 247ea4a..0000000 --- a/.cloudtest.yaml +++ /dev/null @@ -1,16 +0,0 @@ ---- -version: 1.0 -root: "./.tests/cloud_test/" -timeout: 10000 -shuffle-enabled: true -statistics: - enabled: true - interval: 60 -import: - - gke/.* - - aks/.* - - aws/.* - - packet/.* - - tests.yaml -reporting: - junit-report: "results/junit.xml" diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8e8ad2e..8db361c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -21,11 +21,15 @@ jobs: checkgomod: uses: networkservicemesh/.github/.github/workflows/checkgomod.yaml@main - cloudtest: + interdomain: runs-on: ubuntu-latest env: KUBERNETES_VERSION: ${{ secrets.NSM_KUBERNETES_VERSION }} steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.9.0 + with: + access_token: ${{ github.token }} - name: Checkout files uses: actions/checkout@v2 with: @@ -55,63 +59,108 @@ jobs: echo GOPATH=$GITHUB_WORKSPACE >> $GITHUB_ENV echo GO111MODULE=on >> $GITHUB_ENV echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH - - name: Get cloudtest - run: go get github.com/networkservicemesh/cloudtest@master - - name: Get kind - run: go get sigs.k8s.io/kind@v0.11.1 - name: Check out code uses: actions/checkout@v2 with: path: ${{ github.workspace }}/src/github.com/${{ github.repository }} + + - name: Setup gke + run: | + scripts/gke/gke-start.sh + env: + GCLOUD_SERVICE_KEY: ${{ secrets.GCLOUD_SERVICE_KEY }} + GCLOUD_PROJECT_ID: ${{ secrets.GOOGLE_PROJECT_ID }} + GKE_PROJECT_ID: ${{ secrets.GOOGLE_PROJECT_ID }} + GKE_CLUSTER_NAME: gke-${{ github.run_id }}-${{ github.run_number }} + GKE_CLUSTER_ZONE: us-central1-a + GKE_CLUSTER_TYPE: n1-standard-2 + GKE_CLUSTER_NUM_NODES: 1 + KUBECONFIG: /tmp/config1 + USE_GKE_GCLOUD_AUTH_PLUGIN: true + working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} + + - name: Setup aws + run: | + scripts/aws/aws-start.sh + env: + KUBECONFIG: /tmp/config2 + AWS_REGION: us-east-2 + AWS_CLUSTER_NAME: aws-${{ github.run_id }}-${{ github.run_number }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} + + - name: Setup aks + run: | + az login --service-principal --username ${AZURE_SERVICE_PRINCIPAL} --password ${AZURE_SERVICE_PRINCIPAL_SECRET} --tenant ${AZURE_TENANT} + scripts/aks/aks-start.sh ${AZURE_RESOURCE_GROUP} ${AZURE_CLUSTER_NAME} ${AZURE_CREDENTIALS_PATH} ${KUBECONFIG} + env: + KUBECONFIG: /tmp/config3 + AZURE_RESOURCE_GROUP: nsm-ci + AZURE_CLUSTER_NAME: aks-${{ github.run_id }}-${{ github.run_number }} + AZURE_SERVICE_PRINCIPAL: ${{ secrets.AZURE_SERVICE_PRINCIPAL }} + AZURE_SERVICE_PRINCIPAL_SECRET: ${{ secrets.AZURE_SERVICE_PRINCIPAL_SECRET }} + AZURE_TENANT: ${{ secrets.AZURE_TENANT }} + working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} + - name: Run interdomain testing run: | - cloudtest + go test -count 1 -timeout 2h -race -v ./... env: + KUBECONFIG1: /tmp/config1 + KUBECONFIG2: /tmp/config2 + KUBECONFIG3: /tmp/config3 AZURE_SERVICE_PRINCIPAL: ${{ secrets.AZURE_SERVICE_PRINCIPAL }} AZURE_SERVICE_PRINCIPAL_SECRET: ${{ secrets.AZURE_SERVICE_PRINCIPAL_SECRET }} AZURE_TENANT: ${{ secrets.AZURE_TENANT }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: "us-west-2" + USE_GKE_GCLOUD_AUTH_PLUGIN: true PACKET_AUTH_TOKEN: ${{ secrets.PACKET_AUTH_TOKEN }} PACKET_PROJECT_ID: ${{ secrets.PACKET_PROJECT_ID }} KUBERNETES_VERSION: ${{ secrets.NSM_KUBERNETES_VERSION }} GCLOUD_SERVICE_KEY: ${{ secrets.GCLOUD_SERVICE_KEY }} GCLOUD_PROJECT_ID: ${{ secrets.GOOGLE_PROJECT_ID }} working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} - - name: Publish test report - uses: mikepenz/action-junit-report@v2.1.0 + + - name: Cleanup GKE if: ${{ always() }} - with: - report_paths: "**/cloud_test/results/junit.xml" - suite_regex: "Test*" - github_token: ${{ secrets.GITHUB_TOKEN }} + working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} + run: | + gcloud container clusters delete "${GKE_CLUSTER_NAME}" --project="${GKE_PROJECT_ID}" --zone="${GKE_CLUSTER_ZONE}" -q + env: + GKE_PROJECT_ID: ${{ secrets.GOOGLE_PROJECT_ID }} + GKE_CLUSTER_NAME: gke-${{ github.run_id }}-${{ github.run_number }} + GKE_CLUSTER_ZONE: us-central1-a + + + - name: Cleanup AWS + if: ${{ always() }} + working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} + run: | + scripts/aws/aws-destroy.sh + env: + AWS_REGION: us-east-2 + AWS_CLUSTER_NAME: aws-${{ github.run_id }}-${{ github.run_number }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + + - name: Cleanup AKS + if: ${{ always() }} + working-directory: ${{ github.workspace }}/src/github.com/${{ github.repository }} + run: | + scripts/aks/aks-destroy.sh ${AZURE_RESOURCE_GROUP} ${AZURE_CLUSTER_NAME} + env: + AZURE_CLUSTER_NAME: aks-${{ github.run_id }}-${{ github.run_number }} + AZURE_SERVICE_PRINCIPAL: ${{ secrets.AZURE_SERVICE_PRINCIPAL }} + AZURE_SERVICE_PRINCIPAL_SECRET: ${{ secrets.AZURE_SERVICE_PRINCIPAL_SECRET }} + AZURE_TENANT: ${{ secrets.AZURE_TENANT }} + AZURE_RESOURCE_GROUP: nsm-ci + - name: Upload logs uses: actions/upload-artifact@v2 if: ${{ always() }} with: name: logs-${{ github.run_number }} path: ${{ github.workspace }}/src/github.com/${{ github.repository }}/.tests/cloud_test/ - - packet-cleanup: - name: packet cleanup - runs-on: ubuntu-latest - needs: - - cloudtest - if: ${{ always() }} - steps: - - name: Set up /bin permissions - run: | - sudo chmod -R 777 /bin - - uses: actions/setup-go@v1 - with: - go-version: 1.16 - - name: Clean up packet - run: | - go get github.com/networkservicemesh/cloudtest/pkg/providers/packet/packet_cleanup@master - packet_cleanup -k y -c y --cluster-prefix "interdomain" - env: - GO111MODULE: on - GOBIN: /bin - PACKET_AUTH_TOKEN: ${{ secrets.PACKET_AUTH_TOKEN }} - PACKET_PROJECT_ID: ${{ secrets.PACKET_PROJECT_ID }} diff --git a/aks/azure.yaml b/aks/azure.yaml deleted file mode 100644 index cbbde81..0000000 --- a/aks/azure.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -version: 1.0 -providers: - - name: "aks" - kind: "shell" - instances: 1 - retry: 1 - node-count: 1 - enabled: true - timeout: 3600 - env: - - AZURE_CLUSTER_NAME=interdomain-$(cluster-name)-$(date)-${GITHUB_RUN_NUMBER} - - AZURE_RESOURCE_GROUP=nsm-ci - - KUBECONFIG=$(tempdir)/config - - AZURE_CREDENTIALS_PATH=$(tempdir)/config - env-check: - - AZURE_SERVICE_PRINCIPAL - - AZURE_SERVICE_PRINCIPAL_SECRET - - AZURE_TENANT - - GITHUB_RUN_NUMBER - scripts: - install: az login --service-principal --username ${AZURE_SERVICE_PRINCIPAL} --password ${AZURE_SERVICE_PRINCIPAL_SECRET} --tenant ${AZURE_TENANT} - start: | - scripts/aks/aks-start.sh ${AZURE_RESOURCE_GROUP} ${AZURE_CLUSTER_NAME} ${AZURE_CREDENTIALS_PATH} ${AZURE_SERVICE_PRINCIPAL} ${AZURE_SERVICE_PRINCIPAL_SECRET} - stop: | - scripts/aks/aks-destroy.sh ${AZURE_RESOURCE_GROUP} ${AZURE_CLUSTER_NAME} - cleanup: | - scripts/aks/destroy-old-clusters.sh 4 "^interdomain-aks" diff --git a/aws/aws.yaml b/aws/aws.yaml deleted file mode 100644 index ccc9840..0000000 --- a/aws/aws.yaml +++ /dev/null @@ -1,26 +0,0 @@ ---- -version: 1.0 -providers: - - name: "aws" - kind: "shell" - instances: 1 - retry: 1 - node-count: 1 - enabled: true - timeout: 3600 - env: - - AWS_CLUSTER_NAME=interdomain-$(cluster-name)-$(date)-${GITHUB_RUN_NUMBER} - - KUBECONFIG=$(tempdir)/config - - NSM_AWS_SERVICE_SUFFIX=${AWS_CLUSTER_NAME} - env-check: - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - GITHUB_RUN_NUMBER - - AWS_REGION - scripts: - start: | - scripts/aws/aws-start.sh - stop: | - scripts/aws/aws-destroy.sh - cleanup: | - scripts/aws/destroy-old-clusters.sh "^interdomain-aws" diff --git a/cluster-config.yaml b/cluster-config.yaml deleted file mode 100644 index f57c67e..0000000 --- a/cluster-config.yaml +++ /dev/null @@ -1,6 +0,0 @@ ---- -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -nodes: - - role: control-plane - - role: worker diff --git a/entry_point_test.go b/entry_point_test.go index a8105d7..3378575 100644 --- a/entry_point_test.go +++ b/entry_point_test.go @@ -1,6 +1,6 @@ // Copyright (c) 2021 Doc.ai and/or its affiliates. // -// Copyright (c) 2022 Cisco and/or its affiliates. +// Copyright (c) 2022-2023 Cisco and/or its affiliates. // // SPDX-License-Identifier: Apache-2.0 // @@ -25,7 +25,6 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" // This is required for GKE authentication "github.com/networkservicemesh/integration-tests/suites/multicluster" ) diff --git a/gke/gke.yaml b/gke/gke.yaml deleted file mode 100644 index 5619b30..0000000 --- a/gke/gke.yaml +++ /dev/null @@ -1,26 +0,0 @@ ---- -version: 1.0 -providers: - - name: "gke" - kind: "shell" - instances: 1 - retry: 3 - node-count: 1 - enabled: true - timeout: 3600 - env: - - GKE_PROJECT_ID=${GCLOUD_PROJECT_ID} - - CLUSTER_RULES_PREFIX=gke - - GKE_CLUSTER_NAME=interdomain-$(cluster-name)-$(date)-${GITHUB_RUN_NUMBER} - - KUBECONFIG=$(tempdir)/config - - GKE_CLUSTER_ZONE=us-central1-a - - GKE_CLUSTER_TYPE=n1-standard-2 - - GKE_CLUSTER_NUM_NODES=2 - env-check: - - GCLOUD_SERVICE_KEY - - GCLOUD_PROJECT_ID - - GITHUB_RUN_NUMBER - scripts: - start: scripts/gke/gke-start.sh - stop: scripts/gke/gke-destroy.sh - cleanup: scripts/gke/destroy-old-clusters.sh 4 "^interdomain-gke" diff --git a/go.mod b/go.mod index 2e159b8..c2ed0ab 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,11 @@ module github.com/networkservicemesh/integration-interdomain-k8s go 1.18 require ( - github.com/networkservicemesh/integration-tests v0.0.0-20221219214735-370246c106f2 + github.com/networkservicemesh/integration-tests v0.0.0-20230503164829-413ea99b39a9 github.com/stretchr/testify v1.7.0 - k8s.io/client-go v0.20.5 ) require ( - cloud.google.com/go v0.54.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-logr/logr v0.2.0 // indirect github.com/gogo/protobuf v1.3.1 // indirect @@ -39,6 +37,7 @@ require ( gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect k8s.io/api v0.20.5 // indirect k8s.io/apimachinery v0.20.5 // indirect + k8s.io/client-go v0.20.5 // indirect k8s.io/klog/v2 v2.4.0 // indirect k8s.io/utils v0.0.0-20201110183641-67b214c5f920 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.0.2 // indirect diff --git a/go.sum b/go.sum index 294d83f..eb5628f 100644 --- a/go.sum +++ b/go.sum @@ -8,7 +8,6 @@ cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0 h1:3ithwDMr7/3vpAMXiH+ZQnYbuIsh+OPhUPMFC9enmn0= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= @@ -159,8 +158,8 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/networkservicemesh/gotestmd v0.0.0-20220628095933-eabbdc09e0dc h1:1L/OisEFsOyhwaqeJpYmM1nlJ2dBusUMiszPDBlUip0= github.com/networkservicemesh/gotestmd v0.0.0-20220628095933-eabbdc09e0dc/go.mod h1:8EWnekTRNX+NxBdTFE24WqUoM7SgJHbiafDBrIIdOmQ= -github.com/networkservicemesh/integration-tests v0.0.0-20221219214735-370246c106f2 h1:mUTx26/F/2z4+9I1UPCh4OgeoUnTWnC6TKNeglhz6T0= -github.com/networkservicemesh/integration-tests v0.0.0-20221219214735-370246c106f2/go.mod h1:qkqJgckg5vY5STByxzKhzhDB+TKCAU0qasuwfJ0cG+s= +github.com/networkservicemesh/integration-tests v0.0.0-20230503164829-413ea99b39a9 h1:qHPXAA1N2VWRpzq2WnBRQq3YIlf4mSpj7f58ieRxpik= +github.com/networkservicemesh/integration-tests v0.0.0-20230503164829-413ea99b39a9/go.mod h1:qkqJgckg5vY5STByxzKhzhDB+TKCAU0qasuwfJ0cG+s= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= diff --git a/packet/packet.yaml b/packet/packet.yaml deleted file mode 100644 index 4911a4b..0000000 --- a/packet/packet.yaml +++ /dev/null @@ -1,45 +0,0 @@ ---- -version: 1.0 -providers: - - name: "packet" - kind: "packet" - instances: 1 - retry: 10 - node-count: 2 - enabled: true - timeout: 3600 - env: - - CLUSTER_RULES_PREFIX=null - - KUBECONFIG=$(tempdir)/config - - CLUSTER_NAME=interdomain-$(cluster-name)-$(date)-${GITHUB_RUN_NUMBER} - env-check: - - PACKET_AUTH_TOKEN - - PACKET_PROJECT_ID - - GITHUB_RUN_NUMBER - - KUBERNETES_VERSION - packet: - ssh-key: sshkey.pub - devices: - - name: "interdomain-Master" - host-name: "interdomain-Master-${CLUSTER_NAME}" - os: "ubuntu_20_04" - billing-cycle: "hourly" - plan: "n3.xlarge.x86" - - name: "interdomain-Worker" - host-name: "interdomain-Worker-${CLUSTER_NAME}" - os: "ubuntu_20_04" - billing-cycle: "hourly" - plan: "n3.xlarge.x86" - facilities: - - baremetal - - backend_transfer - - layer_2 - - global_ipv4 - - ibx - preferred-facility: da11 - scripts: - install: "echo all deps installed" - setup: | - ssh-keygen -t rsa -f $(tempdir)/sshkey -q -N "" - /bin/bash -c "eval `ssh-agent -s`; ssh-add $(tempdir)/sshkey" - start: scripts/packet/create-kubernetes-cluster.sh $(device.interdomain-Master.pub.ip.4) $(device.interdomain-Worker.pub.ip.4) "$(tempdir)/sshkey" diff --git a/scripts/gke/gke-start.sh b/scripts/gke/gke-start.sh index eb6913c..ab5afd2 100755 --- a/scripts/gke/gke-start.sh +++ b/scripts/gke/gke-start.sh @@ -1,15 +1,13 @@ #!/bin/bash -if ! gcloud container clusters list --project="${GKE_PROJECT_ID}" | grep -q ^"${GKE_CLUSTER_NAME}"; then \ - time gcloud container clusters create "${GKE_CLUSTER_NAME}" --project="${GKE_PROJECT_ID}" --machine-type="${GKE_CLUSTER_TYPE}" --num-nodes=1 --zone="${GKE_CLUSTER_ZONE}" -q; \ - echo "Writing config to ${KUBECONFIG}"; \ - gcloud container clusters get-credentials "${GKE_CLUSTER_NAME}" --project="${GKE_PROJECT_ID}" --zone="${GKE_CLUSTER_ZONE}" ; \ - kubectl create clusterrolebinding cluster-admin-binding \ - --clusterrole cluster-admin \ - --user "$(gcloud config get-value account)"; \ - git clone https://github.com/coredns/deployment.git; \ - ./deployment/kubernetes/deploy.sh | kubectl apply -f -; \ - kubectl scale --replicas=0 deployment/kube-dns-autoscaler --namespace=kube-system; \ - kubectl scale --replicas=0 deployment/kube-dns --namespace=kube-system; \ - rm -rf deployment; \ -fi +gcloud components install gke-gcloud-auth-plugin +gcloud components update +time gcloud container clusters create "${GKE_CLUSTER_NAME}" --project="${GKE_PROJECT_ID}" --machine-type="${GKE_CLUSTER_TYPE}" --num-nodes=1 --zone="${GKE_CLUSTER_ZONE}" -q +echo "Writing config to ${KUBECONFIG}" +gcloud container clusters get-credentials "${GKE_CLUSTER_NAME}" --project="${GKE_PROJECT_ID}" --zone="${GKE_CLUSTER_ZONE}" +kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user "$(gcloud config get-value account)" +git clone https://github.com/coredns/deployment.git +./deployment/kubernetes/deploy.sh | kubectl apply -f - +kubectl scale --replicas=0 deployment/kube-dns-autoscaler --namespace=kube-system +kubectl scale --replicas=0 deployment/kube-dns --namespace=kube-system +rm -rf deployment \ No newline at end of file