diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b97e540..66b0610 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -83,22 +83,11 @@ jobs: - name: Set up azure permissions run: | sudo chmod -R 777 ~/.azure - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-2 - name: Install gcloud-sdk uses: google-github-actions/setup-gcloud@v0 with: service_account_key: ${{ secrets.GCLOUD_SERVICE_KEY }} project_id: ${{ secrets.GOOGLE_PROJECT_ID }} - - name: aws-iam-authenticator install - working-directory: ${{ github.repository }} - run: | - ./scripts/aws/aws-init.sh - echo "$HOME/bin" >> $GITHUB_PATH - name: Set up /bin permissions run: | sudo chmod -R 777 /bin diff --git a/aws/aws.yaml b/aws/aws.yaml index c07f850..2e98de8 100644 --- a/aws/aws.yaml +++ b/aws/aws.yaml @@ -22,4 +22,4 @@ providers: stop: | scripts/aws/aws-destroy.sh cleanup: | - scripts/aws/destroy-old-clusters.sh 4 "^interdomain-aws" + scripts/aws/destroy-old-clusters.sh "^interdomain-aws" diff --git a/go.mod b/go.mod index aaa1478..0a3a65d 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.16 require ( github.com/googleapis/gnostic v0.5.1 // indirect - github.com/networkservicemesh/integration-tests v0.0.0-20220430002018-cad775c2971b + github.com/networkservicemesh/integration-tests v0.0.0-20220606110259-f13859021323 github.com/stretchr/testify v1.7.0 gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/client-go v0.20.5 diff --git a/go.sum b/go.sum index 1c07991..9afe381 100644 --- a/go.sum +++ b/go.sum @@ -159,8 +159,8 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/networkservicemesh/gotestmd v0.0.0-20211116145945-871d2aaf07ab h1:/dIr8Nky77grI3s9Rc78eFH9M1Svobyj2XJBaKm27ts= github.com/networkservicemesh/gotestmd v0.0.0-20211116145945-871d2aaf07ab/go.mod h1:8EWnekTRNX+NxBdTFE24WqUoM7SgJHbiafDBrIIdOmQ= -github.com/networkservicemesh/integration-tests v0.0.0-20220430002018-cad775c2971b h1:0yBfhNP0URNcnLLXkPfXA1seJFvARMbg5zueumIjTv8= -github.com/networkservicemesh/integration-tests v0.0.0-20220430002018-cad775c2971b/go.mod h1:0o7WrzxlHEwnDSuZPEM1BnKd4hr7+akKgymoAoTTbv8= +github.com/networkservicemesh/integration-tests v0.0.0-20220606110259-f13859021323 h1:YMpIX7akgx52p/pdSaAFNEREWhLR4F9HA4owh2G4AY8= +github.com/networkservicemesh/integration-tests v0.0.0-20220606110259-f13859021323/go.mod h1:0o7WrzxlHEwnDSuZPEM1BnKd4hr7+akKgymoAoTTbv8= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= diff --git a/scripts/aws/amazon-eks-nodegroup.yaml b/scripts/aws/amazon-eks-nodegroup.yaml deleted file mode 100644 index 876e8e5..0000000 --- a/scripts/aws/amazon-eks-nodegroup.yaml +++ /dev/null @@ -1,326 +0,0 @@ ---- -AWSTemplateFormatVersion: 2010-09-09 -Description: Amazon EKS - Node Group - -Parameters: - KeyName: - Description: The EC2 Key Pair to allow SSH access to the instances - Type: AWS::EC2::KeyPair::KeyName - - NodeImageId: - Description: AMI id for the node instances. - Type: AWS::EC2::Image::Id - - NodeInstanceType: - Description: EC2 instance type for the node instances - Type: String - Default: t2.large - ConstraintDescription: Must be a valid EC2 instance type - AllowedValues: - - t2.small - - t2.medium - - t2.large - - t2.xlarge - - t2.2xlarge - - t3.nano - - t3.micro - - t3.small - - t3.medium - - t3.large - - t3.xlarge - - t3.2xlarge - - m3.medium - - m3.large - - m3.xlarge - - m3.2xlarge - - m4.large - - m4.xlarge - - m4.2xlarge - - m4.4xlarge - - m4.10xlarge - - m5.large - - m5.xlarge - - m5.2xlarge - - m5.4xlarge - - m5.12xlarge - - m5.24xlarge - - c4.large - - c4.xlarge - - c4.2xlarge - - c4.4xlarge - - c4.8xlarge - - c5.large - - c5.xlarge - - c5.2xlarge - - c5.4xlarge - - c5.9xlarge - - c5.18xlarge - - i3.large - - i3.xlarge - - i3.2xlarge - - i3.4xlarge - - i3.8xlarge - - i3.16xlarge - - r3.xlarge - - r3.2xlarge - - r3.4xlarge - - r3.8xlarge - - r4.large - - r4.xlarge - - r4.2xlarge - - r4.4xlarge - - r4.8xlarge - - r4.16xlarge - - x1.16xlarge - - x1.32xlarge - - p2.xlarge - - p2.8xlarge - - p2.16xlarge - - p3.2xlarge - - p3.8xlarge - - p3.16xlarge - - p3dn.24xlarge - - r5.large - - r5.xlarge - - r5.2xlarge - - r5.4xlarge - - r5.12xlarge - - r5.24xlarge - - r5d.large - - r5d.xlarge - - r5d.2xlarge - - r5d.4xlarge - - r5d.12xlarge - - r5d.24xlarge - - z1d.large - - z1d.xlarge - - z1d.2xlarge - - z1d.3xlarge - - z1d.6xlarge - - z1d.12xlarge - - NodeAutoScalingGroupMinSize: - Description: Minimum size of Node Group ASG. - Type: Number - Default: 1 - - NodeAutoScalingGroupMaxSize: - Description: Maximum size of Node Group ASG. Set to at least 1 greater than NodeAutoScalingGroupDesiredCapacity. - Type: Number - Default: 3 - - NodeAutoScalingGroupDesiredCapacity: - Description: Desired capacity of Node Group ASG. - Type: Number - Default: 1 - - NodeVolumeSize: - Description: Node volume size - Type: Number - Default: 20 - - ClusterName: - Description: The cluster name provided when the cluster was created. If it is incorrect, nodes will not be able to join the cluster. - Type: String - - BootstrapArguments: - Description: Arguments to pass to the bootstrap script. See files/bootstrap.sh in https://github.com/awslabs/amazon-eks-ami - Type: String - Default: "" - - NodeGroupName: - Description: Unique identifier for the Node Group. - Type: String - - ClusterControlPlaneSecurityGroup: - Description: The security group of the cluster control plane. - Type: AWS::EC2::SecurityGroup::Id - - VpcId: - Description: The VPC of the worker instances - Type: AWS::EC2::VPC::Id - - Subnets: - Description: The subnets where workers can be created. - Type: List - -Metadata: - AWS::CloudFormation::Interface: - ParameterGroups: - - Label: - default: EKS Cluster - Parameters: - - ClusterName - - ClusterControlPlaneSecurityGroup - - Label: - default: Worker Node Configuration - Parameters: - - NodeGroupName - - NodeAutoScalingGroupMinSize - - NodeAutoScalingGroupDesiredCapacity - - NodeAutoScalingGroupMaxSize - - NodeInstanceType - - NodeImageId - - NodeVolumeSize - - KeyName - - BootstrapArguments - - Label: - default: Worker Network Configuration - Parameters: - - VpcId - - Subnets - -Resources: - NodeInstanceProfile: - Type: AWS::IAM::InstanceProfile - Properties: - Path: "/" - Roles: - - !Ref NodeInstanceRole - - NodeInstanceRole: - Type: AWS::IAM::Role - Properties: - AssumeRolePolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Principal: - Service: ec2.amazonaws.com - Action: sts:AssumeRole - Path: "/" - ManagedPolicyArns: - - arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy - - arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy - - arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly - - NodeSecurityGroup: - Type: AWS::EC2::SecurityGroup - Properties: - GroupDescription: Security group for all nodes in the cluster - VpcId: !Ref VpcId - Tags: - - Key: !Sub kubernetes.io/cluster/${ClusterName} - Value: owned - - NodeSecurityGroupIngress: - Type: AWS::EC2::SecurityGroupIngress - DependsOn: NodeSecurityGroup - Properties: - Description: Allow node to communicate with each other - GroupId: !Ref NodeSecurityGroup - SourceSecurityGroupId: !Ref NodeSecurityGroup - IpProtocol: -1 - FromPort: 0 - ToPort: 65535 - - NodeSecurityGroupFromControlPlaneIngress: - Type: AWS::EC2::SecurityGroupIngress - DependsOn: NodeSecurityGroup - Properties: - Description: Allow worker Kubelets and pods to receive communication from the cluster control plane - GroupId: !Ref NodeSecurityGroup - SourceSecurityGroupId: !Ref ClusterControlPlaneSecurityGroup - IpProtocol: tcp - FromPort: 1025 - ToPort: 65535 - - ControlPlaneEgressToNodeSecurityGroup: - Type: AWS::EC2::SecurityGroupEgress - DependsOn: NodeSecurityGroup - Properties: - Description: Allow the cluster control plane to communicate with worker Kubelet and pods - GroupId: !Ref ClusterControlPlaneSecurityGroup - DestinationSecurityGroupId: !Ref NodeSecurityGroup - IpProtocol: tcp - FromPort: 1025 - ToPort: 65535 - - NodeSecurityGroupFromControlPlaneOn443Ingress: - Type: AWS::EC2::SecurityGroupIngress - DependsOn: NodeSecurityGroup - Properties: - Description: Allow pods running extension API servers on port 443 to receive communication from cluster control plane - GroupId: !Ref NodeSecurityGroup - SourceSecurityGroupId: !Ref ClusterControlPlaneSecurityGroup - IpProtocol: tcp - FromPort: 443 - ToPort: 443 - - ControlPlaneEgressToNodeSecurityGroupOn443: - Type: AWS::EC2::SecurityGroupEgress - DependsOn: NodeSecurityGroup - Properties: - Description: Allow the cluster control plane to communicate with pods running extension API servers on port 443 - GroupId: !Ref ClusterControlPlaneSecurityGroup - DestinationSecurityGroupId: !Ref NodeSecurityGroup - IpProtocol: tcp - FromPort: 443 - ToPort: 443 - - ClusterControlPlaneSecurityGroupIngress: - Type: AWS::EC2::SecurityGroupIngress - DependsOn: NodeSecurityGroup - Properties: - Description: Allow pods to communicate with the cluster API Server - GroupId: !Ref ClusterControlPlaneSecurityGroup - SourceSecurityGroupId: !Ref NodeSecurityGroup - IpProtocol: tcp - ToPort: 443 - FromPort: 443 - - NodeGroup: - Type: AWS::AutoScaling::AutoScalingGroup - Properties: - DesiredCapacity: !Ref NodeAutoScalingGroupDesiredCapacity - LaunchConfigurationName: !Ref NodeLaunchConfig - MinSize: !Ref NodeAutoScalingGroupMinSize - MaxSize: !Ref NodeAutoScalingGroupMaxSize - VPCZoneIdentifier: !Ref Subnets - Tags: - - Key: Name - Value: !Sub ${ClusterName}-${NodeGroupName}-Node - PropagateAtLaunch: true - - Key: !Sub kubernetes.io/cluster/${ClusterName} - Value: owned - PropagateAtLaunch: true - UpdatePolicy: - AutoScalingRollingUpdate: - MaxBatchSize: 1 - MinInstancesInService: !Ref NodeAutoScalingGroupDesiredCapacity - PauseTime: PT5M - - NodeLaunchConfig: - Type: AWS::AutoScaling::LaunchConfiguration - Properties: - AssociatePublicIpAddress: true - IamInstanceProfile: !Ref NodeInstanceProfile - ImageId: !Ref NodeImageId - InstanceType: !Ref NodeInstanceType - KeyName: !Ref KeyName - SecurityGroups: - - !Ref NodeSecurityGroup - BlockDeviceMappings: - - DeviceName: /dev/xvda - Ebs: - VolumeSize: !Ref NodeVolumeSize - VolumeType: gp2 - DeleteOnTermination: true - UserData: - Fn::Base64: !Sub | - #!/bin/bash - set -o xtrace - /etc/eks/bootstrap.sh ${ClusterName} ${BootstrapArguments} - /opt/aws/bin/cfn-signal --exit-code $? \ - --stack ${AWS::StackName} \ - --resource NodeGroup \ - --region ${AWS::Region} - -Outputs: - NodeInstanceRole: - Description: The node instance role - Value: !GetAtt NodeInstanceRole.Arn - - NodeSecurityGroup: - Description: The security group for the node group - Value: !Ref NodeSecurityGroup diff --git a/scripts/aws/amazon-eks-role-policy.json b/scripts/aws/amazon-eks-role-policy.json deleted file mode 100644 index 7180e5d..0000000 --- a/scripts/aws/amazon-eks-role-policy.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Service": "eks.amazonaws.com" - }, - "Action": "sts:AssumeRole" - } - ] -} \ No newline at end of file diff --git a/scripts/aws/amazon-eks-vpc.yaml b/scripts/aws/amazon-eks-vpc.yaml deleted file mode 100644 index bb8f149..0000000 --- a/scripts/aws/amazon-eks-vpc.yaml +++ /dev/null @@ -1,207 +0,0 @@ -# yamllint disable ---- -AWSTemplateFormatVersion: "2010-09-09" -Description: "Amazon EKS Sample VPC" - -Parameters: - VpcBlock: - Type: String - Default: 192.168.0.0/16 - Description: The CIDR range for the VPC. This should be a valid private (RFC 1918) CIDR range. - - Subnet01Block: - Type: String - Default: 192.168.64.0/18 - Description: CidrBlock for subnet 01 within the VPC - - Subnet02Block: - Type: String - Default: 192.168.128.0/18 - Description: CidrBlock for subnet 02 within the VPC - - Subnet03Block: - Type: String - Default: 192.168.192.0/18 - Description: CidrBlock for subnet 03 within the VPC. This is used only if the region has more than 2 AZs. - -Metadata: - AWS::CloudFormation::Interface: - ParameterGroups: - - Label: - default: "Worker Network Configuration" - Parameters: - - VpcBlock - - Subnet01Block - - Subnet02Block - - Subnet03Block - -Conditions: - Has2Azs: - Fn::Or: - - Fn::Equals: - - { Ref: "AWS::Region" } - - ap-south-1 - - Fn::Equals: - - { Ref: "AWS::Region" } - - ap-northeast-2 - - Fn::Equals: - - { Ref: "AWS::Region" } - - ca-central-1 - - Fn::Equals: - - { Ref: "AWS::Region" } - - cn-north-1 - - HasMoreThan2Azs: - Fn::Not: - - Condition: Has2Azs - -Resources: - VPC: - Type: AWS::EC2::VPC - Properties: - CidrBlock: !Ref VpcBlock - EnableDnsSupport: true - EnableDnsHostnames: true - Tags: - - Key: Name - Value: !Sub "${AWS::StackName}-VPC" - - Ipv6VPCCidrBlock: - Type: AWS::EC2::VPCCidrBlock - Properties: - AmazonProvidedIpv6CidrBlock: true - VpcId: !Ref VPC - - InternetGateway: - Type: "AWS::EC2::InternetGateway" - - VPCGatewayAttachment: - Type: "AWS::EC2::VPCGatewayAttachment" - Properties: - InternetGatewayId: !Ref InternetGateway - VpcId: !Ref VPC - - RouteTable: - Type: AWS::EC2::RouteTable - Properties: - VpcId: !Ref VPC - Tags: - - Key: Name - Value: Public Subnets - - Key: Network - Value: Public - - Route: - DependsOn: VPCGatewayAttachment - Type: AWS::EC2::Route - Properties: - RouteTableId: !Ref RouteTable - DestinationCidrBlock: 0.0.0.0/0 - GatewayId: !Ref InternetGateway - - Subnet01: - Type: AWS::EC2::Subnet - Metadata: - Comment: Subnet 01 - Properties: - AvailabilityZone: - Fn::Select: - - "0" - - Fn::GetAZs: - Ref: AWS::Region - AssignIpv6AddressOnCreation: true - CidrBlock: - Ref: Subnet01Block - Ipv6CidrBlock: - !Select [0, !Cidr [!Select [0, !GetAtt VPC.Ipv6CidrBlocks], 256, 64]] - VpcId: - Ref: VPC - Tags: - - Key: Name - Value: !Sub "${AWS::StackName}-Subnet01" - DependsOn: Ipv6VPCCidrBlock - - Subnet02: - Type: AWS::EC2::Subnet - Metadata: - Comment: Subnet 02 - Properties: - AvailabilityZone: - Fn::Select: - - "1" - - Fn::GetAZs: - Ref: AWS::Region - CidrBlock: - Ref: Subnet02Block - Ipv6CidrBlock: - !Select [1, !Cidr [!Select [0, !GetAtt VPC.Ipv6CidrBlocks], 256, 64]] - VpcId: - Ref: VPC - Tags: - - Key: Name - Value: !Sub "${AWS::StackName}-Subnet02" - DependsOn: Ipv6VPCCidrBlock - - Subnet03: - Condition: HasMoreThan2Azs - Type: AWS::EC2::Subnet - Metadata: - Comment: Subnet 03 - Properties: - AvailabilityZone: - Fn::Select: - - "2" - - Fn::GetAZs: - Ref: AWS::Region - CidrBlock: - Ref: Subnet03Block - Ipv6CidrBlock: - !Select [2, !Cidr [!Select [0, !GetAtt VPC.Ipv6CidrBlocks], 256, 64]] - VpcId: - Ref: VPC - Tags: - - Key: Name - Value: !Sub "${AWS::StackName}-Subnet03" - DependsOn: Ipv6VPCCidrBlock - - Subnet01RouteTableAssociation: - Type: AWS::EC2::SubnetRouteTableAssociation - Properties: - SubnetId: !Ref Subnet01 - RouteTableId: !Ref RouteTable - - Subnet02RouteTableAssociation: - Type: AWS::EC2::SubnetRouteTableAssociation - Properties: - SubnetId: !Ref Subnet02 - RouteTableId: !Ref RouteTable - - Subnet03RouteTableAssociation: - Condition: HasMoreThan2Azs - Type: AWS::EC2::SubnetRouteTableAssociation - Properties: - SubnetId: !Ref Subnet03 - RouteTableId: !Ref RouteTable - - ControlPlaneSecurityGroup: - Type: AWS::EC2::SecurityGroup - Properties: - GroupDescription: Cluster communication with worker nodes - VpcId: !Ref VPC - -Outputs: - SubnetIds: - Description: All subnets in the VPC - Value: - Fn::If: - - HasMoreThan2Azs - - !Join [",", [!Ref Subnet01, !Ref Subnet02, !Ref Subnet03]] - - !Join [",", [!Ref Subnet01, !Ref Subnet02]] - - SecurityGroups: - Description: Security group for the cluster control plane communication with worker nodes - Value: !Join [",", [!Ref ControlPlaneSecurityGroup]] - - VpcId: - Description: The VPC Id - Value: !Ref VPC diff --git a/scripts/aws/aws-auth-cm-temp.yaml b/scripts/aws/aws-auth-cm-temp.yaml deleted file mode 100644 index c79b57d..0000000 --- a/scripts/aws/aws-auth-cm-temp.yaml +++ /dev/null @@ -1,13 +0,0 @@ ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: aws-auth - namespace: kube-system -data: - mapRoles: | - - rolearn: - username: system:node:{{EC2PrivateDNSName}} - groups: - - system:bootstrappers - - system:nodes diff --git a/scripts/aws/aws-destroy.sh b/scripts/aws/aws-destroy.sh index ba7b0a5..8fdb660 100755 --- a/scripts/aws/aws-destroy.sh +++ b/scripts/aws/aws-destroy.sh @@ -1,5 +1,3 @@ #!/bin/bash -pushd scripts/aws && \ -AWS_REGION=us-east-2 go run ./... Delete && \ -popd || exit 0 \ No newline at end of file +eksctl delete cluster --name "${AWS_CLUSTER_NAME}" \ No newline at end of file diff --git a/scripts/aws/aws-init.sh b/scripts/aws/aws-init.sh deleted file mode 100755 index cd862d0..0000000 --- a/scripts/aws/aws-init.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -mkdir -p "$HOME/bin" - -if [ ! -f "$HOME/bin/aws-iam-authenticator" ]; then - unameOut="$(uname -s)" - case "${unameOut}" in - Linux*) curl -o "$HOME/bin/aws-iam-authenticator" https://amazon-eks.s3-us-west-2.amazonaws.com/1.12.7/2019-03-27/bin/linux/amd64/aws-iam-authenticator;; - Darwin*) curl -o "$HOME/bin/aws-iam-authenticator" https://amazon-eks.s3-us-west-2.amazonaws.com/1.12.7/2019-03-27/bin/darwin/amd64/aws-iam-authenticator;; - *) echo "Unknown uname $unameOut" && exit 1 - esac -fi -chmod +x "$HOME/bin/aws-iam-authenticator" diff --git a/scripts/aws/aws-k8s-cni.yaml b/scripts/aws/aws-k8s-cni.yaml deleted file mode 100644 index 601d7df..0000000 --- a/scripts/aws/aws-k8s-cni.yaml +++ /dev/null @@ -1,140 +0,0 @@ ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: aws-node -rules: - - apiGroups: - - crd.k8s.amazonaws.com - resources: - - "*" - - namespaces - verbs: - - "*" - - apiGroups: [""] - resources: - - pods - - nodes - - namespaces - verbs: ["list", "watch", "get"] - - apiGroups: ["extensions"] - resources: - - daemonsets - verbs: ["list", "watch"] - ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: aws-node - namespace: kube-system - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: aws-node -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: aws-node -subjects: - - kind: ServiceAccount - name: aws-node - namespace: kube-system - ---- -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: aws-node - namespace: kube-system - labels: - k8s-app: aws-node -spec: - updateStrategy: - type: RollingUpdate - selector: - matchLabels: - k8s-app: aws-node - template: - metadata: - labels: - k8s-app: aws-node - spec: - priorityClassName: system-node-critical - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: "beta.kubernetes.io/os" - operator: In - values: - - linux - - key: "beta.kubernetes.io/arch" - operator: In - values: - - amd64 - serviceAccountName: aws-node - hostNetwork: true - tolerations: - - operator: Exists - containers: - - image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:v1.5.5 - imagePullPolicy: Always - ports: - - containerPort: 61678 - name: metrics - name: aws-node - env: - - name: AWS_VPC_K8S_CNI_LOGLEVEL - value: DEBUG - - name: MY_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - resources: - requests: - cpu: 10m - securityContext: - privileged: true - volumeMounts: - - mountPath: /host/opt/cni/bin - name: cni-bin-dir - - mountPath: /host/etc/cni/net.d - name: cni-net-dir - - mountPath: /host/var/log - name: log-dir - - mountPath: /var/run/docker.sock - name: dockersock - volumes: - - name: cni-bin-dir - hostPath: - path: /opt/cni/bin - - name: cni-net-dir - hostPath: - path: /etc/cni/net.d - - name: log-dir - hostPath: - path: /var/log - - name: dockersock - hostPath: - path: /var/run/docker.sock - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: eniconfigs.crd.k8s.amazonaws.com -spec: - scope: Cluster - group: crd.k8s.amazonaws.com - versions: - - name: v1alpha1 - served: true - storage: true - names: - plural: eniconfigs - singular: eniconfig - kind: ENIConfig diff --git a/scripts/aws/aws-start.sh b/scripts/aws/aws-start.sh index 664e68e..a6500c2 100755 --- a/scripts/aws/aws-start.sh +++ b/scripts/aws/aws-start.sh @@ -1,9 +1,26 @@ #!/bin/bash -pushd scripts/aws || exit 0 -AWS_REGION=us-east-2 go run ./... Create -kubectl describe nodes -kubectl get pods --all-namespaces -o wide -kubectl delete deployment -n kube-system coredns -kubectl apply -f coredns.yaml -popd || exit 0 +apt-get update && apt-get -y install curl dnsutils + +curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.23.6/bin/linux/amd64/kubectl +chmod +x kubectl +mkdir -p ~/.local/bin +mv ./kubectl ~/.local/bin/kubectl + + +curl --silent --location "https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp; \ + mv /tmp/eksctl /usr/local/bin; \ + eksctl version + +curl -o aws-iam-authenticator https://s3.us-west-2.amazonaws.com/amazon-eks/1.21.2/2021-07-05/bin/linux/amd64/aws-iam-authenticator; \ + chmod 755 aws-iam-authenticator; \ + mv ./aws-iam-authenticator /usr/local/bin + +eksctl create cluster \ + --name "${AWS_CLUSTER_NAME}" \ + --version 1.22 \ + --nodegroup-name "${AWS_CLUSTER_NAME}-workers" \ + --node-type t3.xlarge \ + --nodes 1 + +kubectl version --client \ No newline at end of file diff --git a/scripts/aws/coredns.yaml b/scripts/aws/coredns.yaml deleted file mode 100644 index 4a335e3..0000000 --- a/scripts/aws/coredns.yaml +++ /dev/null @@ -1,199 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: coredns - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - kubernetes.io/bootstrapping: rbac-defaults - name: system:coredns -rules: - - apiGroups: - - "" - resources: - - endpoints - - services - - pods - - namespaces - verbs: - - list - - watch - - apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - labels: - kubernetes.io/bootstrapping: rbac-defaults - name: system:coredns -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:coredns -subjects: - - kind: ServiceAccount - name: coredns - namespace: kube-system ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: coredns - namespace: kube-system -data: - Corefile: | - .:53 { - errors - health { - lameduck 5s - } - ready - kubernetes cluster.local in-addr.arpa ip6.arpa { - fallthrough in-addr.arpa ip6.arpa - } - prometheus :9153 - forward . /etc/resolv.conf { - max_concurrent 1000 - } - k8s_external my.cluster - loop - reload - loadbalance - } ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: coredns - namespace: kube-system - labels: - k8s-app: kube-dns - kubernetes.io/name: "CoreDNS" -spec: - # replicas: not specified here: - # 1. Default is 1. - # 2. Will be tuned in real time if DNS horizontal auto-scaling is turned on. - strategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - selector: - matchLabels: - k8s-app: kube-dns - template: - metadata: - labels: - k8s-app: kube-dns - spec: - priorityClassName: system-cluster-critical - serviceAccountName: coredns - tolerations: - - key: "CriticalAddonsOnly" - operator: "Exists" - nodeSelector: - kubernetes.io/os: linux - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: k8s-app - operator: In - values: ["kube-dns"] - topologyKey: kubernetes.io/hostname - containers: - - name: coredns - image: coredns/coredns:1.9.1 - imagePullPolicy: IfNotPresent - resources: - limits: - memory: 170Mi - requests: - cpu: 100m - memory: 70Mi - args: ["-conf", "/etc/coredns/Corefile"] - volumeMounts: - - name: config-volume - mountPath: /etc/coredns - readOnly: true - ports: - - containerPort: 53 - name: dns - protocol: UDP - - containerPort: 53 - name: dns-tcp - protocol: TCP - - containerPort: 9153 - name: metrics - protocol: TCP - securityContext: - allowPrivilegeEscalation: false - capabilities: - add: - - NET_BIND_SERVICE - drop: - - all - readOnlyRootFilesystem: true - livenessProbe: - httpGet: - path: /health - port: 8080 - scheme: HTTP - initialDelaySeconds: 60 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 5 - readinessProbe: - httpGet: - path: /ready - port: 8181 - scheme: HTTP - dnsPolicy: Default - volumes: - - name: config-volume - configMap: - name: coredns - items: - - key: Corefile - path: Corefile ---- -apiVersion: v1 -kind: Service -metadata: - name: kube-dns - namespace: kube-system - annotations: - prometheus.io/port: "9153" - prometheus.io/scrape: "true" - labels: - k8s-app: kube-dns - kubernetes.io/cluster-service: "true" - kubernetes.io/name: "CoreDNS" -spec: - selector: - k8s-app: kube-dns - clusterIP: 10.100.0.10 - ports: - - name: dns - port: 53 - protocol: UDP - - name: dns-tcp - port: 53 - protocol: TCP - - name: metrics - port: 9153 - protocol: TCP diff --git a/scripts/aws/create-kubernetes-cluster.go b/scripts/aws/create-kubernetes-cluster.go deleted file mode 100644 index d1d5b8d..0000000 --- a/scripts/aws/create-kubernetes-cluster.go +++ /dev/null @@ -1,502 +0,0 @@ -package main - -import ( - "bytes" - "io/ioutil" - "log" - "os" - "os/exec" - "path" - "strings" - "time" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/cloudformation" - "github.com/aws/aws-sdk-go/service/ec2" - "github.com/aws/aws-sdk-go/service/eks" - "github.com/aws/aws-sdk-go/service/iam" -) - -type OutputsMap struct { - SecurityGroups *string - VpcId *string - SubnetIds *string -} - -func (ac *AWSCluster) checkError(err error) { - if err != nil { - if aerr, ok := err.(awserr.Error); ok { - switch aerr.Code() { - case "EntityAlreadyExists": - case "AlreadyExistsException": - case "ResourceInUseException": - case "InvalidKeyPair.Duplicate": - case "InvalidPermission.Duplicate": - default: - panic(aerr) - } - log.Printf("Warning (%s): %s\n", aerr.Code(), aerr.Message()) - } else { - panic(err) - } - } -} - -func outputsToMap(outputs []*cloudformation.Output) *OutputsMap { - res := &OutputsMap{} - for _, v := range outputs { - switch *v.OutputKey { - case "SecurityGroups": - res.SecurityGroups = v.OutputValue - case "VpcId": - res.VpcId = v.OutputValue - case "SubnetIds": - res.SubnetIds = v.OutputValue - } - } - return res -} - -func (ac *AWSCluster) createEksRole(iamClient *iam.IAM, eksRoleName *string) *string { - log.Printf("Creating EKS service role \"%s\"...\n", *eksRoleName) - roleDescription := "Allows EKS to manage clusters on your behalf." - - rpf, err := ioutil.ReadFile(path.Join(ac.configPath, "amazon-eks-role-policy.json")) - ac.checkError(err) - - rps := string(rpf) - _, err = iamClient.CreateRole(&iam.CreateRoleInput{ - RoleName: eksRoleName, - Description: &roleDescription, - AssumeRolePolicyDocument: &rps, - }) - ac.checkError(err) - - policyArn := "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy" - _, err = iamClient.AttachRolePolicy(&iam.AttachRolePolicyInput{ - RoleName: eksRoleName, - PolicyArn: &policyArn, - }) - ac.checkError(err) - - policyArn = "arn:aws:iam::aws:policy/AmazonEKSServicePolicy" - _, err = iamClient.AttachRolePolicy(&iam.AttachRolePolicyInput{ - RoleName: eksRoleName, - PolicyArn: &policyArn, - }) - ac.checkError(err) - - result, err := iamClient.GetRole(&iam.GetRoleInput{ - RoleName: eksRoleName, - }) - ac.checkError(err) - - log.Printf("Role \"%s\"(%s) successfully created!\n", *eksRoleName, *result.Role.Arn) - - return result.Role.Arn -} - -func (ac *AWSCluster) createEksClusterVpc(cfClient *cloudformation.CloudFormation, clusterStackName *string) *OutputsMap { - log.Printf("Creating Amazon EKS Cluster VPC \"%s\"...\n", *clusterStackName) - - sf, err := ioutil.ReadFile(path.Join(ac.configPath, "amazon-eks-vpc.yaml")) - ac.checkError(err) - - s := string(sf) - _, err = cfClient.CreateStack(&cloudformation.CreateStackInput{ - StackName: clusterStackName, - TemplateBody: &s, - DisableRollback: aws.Bool(true), - }) - ac.checkError(err) - - for { - resp, err := cfClient.DescribeStacks(&cloudformation.DescribeStacksInput{ - StackName: clusterStackName, - }) - ac.checkError(err) - - switch *resp.Stacks[0].StackStatus { - case "CREATE_COMPLETE": - log.Printf("Cluster VPC \"%s\" successfully created!\n", *clusterStackName) - return outputsToMap(resp.Stacks[0].Outputs) - case "CREATE_IN_PROGRESS": - time.Sleep(requestInterval) - default: - log.Fatalf("Error: Unexpected stack status: %s\n", *resp.Stacks[0].StackStatus) - } - } -} - -func (ac *AWSCluster) createEksCluster(eksClient *eks.EKS, clusterName, eksRoleArn *string, clusterStackOutputs *OutputsMap) *eks.Cluster { - log.Printf("Creating Amazon EKS Cluster \"%s\"...\n", *clusterName) - subnetIdsTemp := strings.Split(*clusterStackOutputs.SubnetIds, ",") - var subnetIds []*string - for i := range subnetIdsTemp { - subnetIds = append(subnetIds, &subnetIdsTemp[i]) - } - - _, err := eksClient.CreateCluster(&eks.CreateClusterInput{ - Name: clusterName, - RoleArn: eksRoleArn, - ResourcesVpcConfig: &eks.VpcConfigRequest{ - SubnetIds: subnetIds, - SecurityGroupIds: []*string{ - clusterStackOutputs.SecurityGroups, - }, - EndpointPrivateAccess: aws.Bool(true), - EndpointPublicAccess: aws.Bool(true), - }, - Version: aws.String("1.21"), - }) - ac.checkError(err) - - for { - resp, err := eksClient.DescribeCluster(&eks.DescribeClusterInput{ - Name: clusterName, - }) - ac.checkError(err) - - switch *resp.Cluster.Status { - case "ACTIVE": - log.Printf("EKS Cluster \"%s\" successfully created!\n", *clusterName) - return resp.Cluster - case "CREATING": - time.Sleep(requestInterval) - default: - log.Fatalf("Error: Unexpected cluster status: %s\n", *resp.Cluster.Status) - } - } -} - -func (ac *AWSCluster) createKubeConfigFile(cluster *eks.Cluster) { - kubeconfigFile := os.Getenv("KUBECONFIG") - if len(kubeconfigFile) == 0 { - kubeconfigFile = path.Join(os.Getenv("HOME"), ".kube/config") - } - kc, err := ioutil.ReadFile(path.Join(ac.configPath, "kube-config-template")) - ac.checkError(err) - kubeconfig := string(kc) - - kubeconfig = strings.Replace(kubeconfig, "", *cluster.CertificateAuthority.Data, -1) - kubeconfig = strings.Replace(kubeconfig, "", *cluster.Endpoint, -1) - kubeconfig = strings.Replace(kubeconfig, "", *cluster.Arn, -1) - kubeconfig = strings.Replace(kubeconfig, "", *cluster.Name, -1) - - err = os.Mkdir(path.Dir(kubeconfigFile), 0775) - err = ioutil.WriteFile(kubeconfigFile, []byte(kubeconfig), 0644) - ac.checkError(err) - - log.Printf("Updated context %s in %s\n", *cluster.Arn, kubeconfigFile) -} - -func (ac *AWSCluster) createEksEc2KeyPair(ec2Client *ec2.EC2, keyPairName *string) { - log.Printf("Creating Amazon EC2 key pair \"%s\"...\n", *keyPairName) - var resp *ec2.CreateKeyPairOutput - resp, err := ec2Client.CreateKeyPair(&ec2.CreateKeyPairInput{ - KeyName: keyPairName, - }) - if err != nil && err.(awserr.Error).Code() == "InvalidKeyPair.Duplicate" { - return - } - ac.checkError(err) - - keyFile := "nsm-key-pair" + os.Getenv("NSM_AWS_SERVICE_SUFFIX") - _ = os.Remove(keyFile) - - err = ioutil.WriteFile(keyFile, []byte(*resp.KeyMaterial), 0400) - ac.checkError(err) - - log.Printf("Amazon EC2 key pair \"%s\" successfully created!\n", *keyPairName) -} - -//nolint:funlen -func (ac *AWSCluster) createEksWorkerNodes(cfClient *cloudformation.CloudFormation, nodesStackName, nodeGroupName, clusterName, keyPairName *string, clusterStackOutputs *OutputsMap) (*string, *string) { - log.Printf("Creating Amazon EKS Worker Nodes on cluster \"%s\"...\n", *clusterName) - - sf, err := ioutil.ReadFile(path.Join(ac.configPath, "amazon-eks-nodegroup.yaml")) - ac.checkError(err) - - s := string(sf) - - // Base image for Amazon EKS worker nodes - // with Kubernetes version 1.14.7 - // for region us-east-2. - // Amazon EKS-Optimized AMI list: https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html - eksAmi := aws.String("ami-053250833d1030033") - - _, err = cfClient.CreateStack(&cloudformation.CreateStackInput{ - StackName: nodesStackName, - TemplateBody: &s, - DisableRollback: aws.Bool(true), - Capabilities: []*string{aws.String("CAPABILITY_IAM")}, - Parameters: []*cloudformation.Parameter{ - { - ParameterKey: aws.String("KeyName"), - ParameterValue: keyPairName, - }, - { - ParameterKey: aws.String("NodeImageId"), - ParameterValue: eksAmi, - }, - { - ParameterKey: aws.String("ClusterName"), - ParameterValue: clusterName, - }, - { - ParameterKey: aws.String("NodeGroupName"), - ParameterValue: nodeGroupName, - }, - { - ParameterKey: aws.String("ClusterControlPlaneSecurityGroup"), - ParameterValue: clusterStackOutputs.SecurityGroups, - }, - { - ParameterKey: aws.String("VpcId"), - ParameterValue: clusterStackOutputs.VpcId, - }, - { - ParameterKey: aws.String("Subnets"), - ParameterValue: &strings.Split(*clusterStackOutputs.SubnetIds, ",")[0], - }, - }, - }) - ac.checkError(err) - - for { - resp, err := cfClient.DescribeStacks(&cloudformation.DescribeStacksInput{ - StackName: nodesStackName, - }) - ac.checkError(err) - - switch *resp.Stacks[0].StackStatus { - case "CREATE_COMPLETE": - log.Printf("EKS Worker Nodes \"%s\" successfully created!\n", *nodesStackName) - - var sequrityGroup *string - var instanceRole *string - - for _, output := range resp.Stacks[0].Outputs { - if *output.OutputKey == "NodeSecurityGroup" { - sequrityGroup = output.OutputValue - break - } - } - - for _, output := range resp.Stacks[0].Outputs { - if *output.OutputKey == "NodeInstanceRole" { - instanceRole = output.OutputValue - } - } - return sequrityGroup, instanceRole - case "CREATE_IN_PROGRESS": - time.Sleep(requestInterval) - default: - log.Fatalf("Error: Unexpected stack status: %s\n", *resp.Stacks[0].StackStatus) - } - } -} - -//nolint:funlen -func (ac *AWSCluster) authorizeSecurityGroupIngress(ec2client *ec2.EC2, groupID *string) { - _, err := ec2client.AuthorizeSecurityGroupIngress(&ec2.AuthorizeSecurityGroupIngressInput{ - GroupId: groupID, - IpPermissions: []*ec2.IpPermission{ - { - IpProtocol: aws.String("tcp"), - ToPort: aws.Int64(22), - FromPort: aws.Int64(22), - IpRanges: []*ec2.IpRange{ - { - CidrIp: aws.String("0.0.0.0/0"), - Description: aws.String("SSH access"), - }, - }, - }, - { - IpProtocol: aws.String("tcp"), - ToPort: aws.Int64(80), - FromPort: aws.Int64(80), - IpRanges: []*ec2.IpRange{ - { - CidrIp: aws.String("0.0.0.0/0"), - Description: aws.String("Remote ip4 access"), - }, - }, - Ipv6Ranges: []*ec2.Ipv6Range{ - { - CidrIpv6: aws.String("::/0"), - Description: aws.String("Remote ip6 access"), - }, - }, - }, - { - IpProtocol: aws.String("tcp"), - ToPort: aws.Int64(5100), - FromPort: aws.Int64(5000), - IpRanges: []*ec2.IpRange{ - { - CidrIp: aws.String("0.0.0.0/0"), - Description: aws.String("Remote ip4 access"), - }, - }, - Ipv6Ranges: []*ec2.Ipv6Range{ - { - CidrIpv6: aws.String("::/0"), - Description: aws.String("Remote ip6 access"), - }, - }, - }, - { - IpProtocol: aws.String("udp"), - ToPort: aws.Int64(4789), - FromPort: aws.Int64(4789), - IpRanges: []*ec2.IpRange{ - { - CidrIp: aws.String("0.0.0.0/0"), - Description: aws.String("vpp vxlan ip4 access"), - }, - }, - Ipv6Ranges: []*ec2.Ipv6Range{ - { - CidrIpv6: aws.String("::/0"), - Description: aws.String("vpp vxlan ip6 access"), - }, - }, - }, - { - IpProtocol: aws.String("udp"), - ToPort: aws.Int64(8472), - FromPort: aws.Int64(8472), - IpRanges: []*ec2.IpRange{ - { - CidrIp: aws.String("0.0.0.0/0"), - Description: aws.String("kernel vxlan ip4 access"), - }, - }, - Ipv6Ranges: []*ec2.Ipv6Range{ - { - CidrIpv6: aws.String("::/0"), - Description: aws.String("kernel vxlan ip6 access"), - }, - }, - }, - { - IpProtocol: aws.String("udp"), - ToPort: aws.Int64(52000), - FromPort: aws.Int64(51820), - IpRanges: []*ec2.IpRange{ - { - CidrIp: aws.String("0.0.0.0/0"), - Description: aws.String("Wireguard ip4 access"), - }, - }, - Ipv6Ranges: []*ec2.Ipv6Range{ - { - CidrIpv6: aws.String("::/0"), - Description: aws.String("Wireguard ip6 access"), - }, - }, - }, - }, - }) - ac.checkError(err) -} - -func (ac *AWSCluster) createSSHConfig(ec2client *ec2.EC2, vpcID *string, scpConfigFileName string) { - res, err := ec2client.DescribeInstances(&ec2.DescribeInstancesInput{ - Filters: []*ec2.Filter{ - { - Name: aws.String("vpc-id"), - Values: []*string{ - vpcID, - }, - }, - }, - }) - ac.checkError(err) - - scpConfigBytes, err := ioutil.ReadFile(path.Join(ac.configPath, "scp-config-template")) - ac.checkError(err) - scpConfig := string(scpConfigBytes) - - i := 0 - hostNames := [2]string{"", ""} - for _, reserv := range res.Reservations { - for _, v := range reserv.Instances { - if len(hostNames) > i { - scpConfig = strings.Replace(scpConfig, hostNames[i], *v.PublicIpAddress, 1) - i++ - } - } - } - - err = ioutil.WriteFile(path.Join(ac.configPath, scpConfigFileName), []byte(scpConfig), 0666) - ac.checkError(err) -} - -// CreateAWSKubernetesCluster - creating AWS kubernetes cluster instances -func (ac *AWSCluster) CreateAWSKubernetesCluster() { - sess := session.Must(session.NewSession()) - iamClient := iam.New(sess) - eksClient := eks.New(sess) - cfClient := cloudformation.New(sess) - ec2Client := ec2.New(sess) - - // Creating Amazon EKS Role - eksRoleName := awsRolePrefix + ac.serviceSuffix - eksRoleArn := ac.createEksRole(iamClient, &eksRoleName) - - // Creating Amazon EKS Cluster VPC - clusterStackName := awsClusterStackPrefix + ac.serviceSuffix - clusterStackOutputs := ac.createEksClusterVpc(cfClient, &clusterStackName) - - // Creating Amazon EKS Cluster - clusterName := awsClusterPrefix + ac.serviceSuffix - cluster := ac.createEksCluster(eksClient, &clusterName, eksRoleArn, clusterStackOutputs) - - // Creating kubeconfig file - ac.createKubeConfigFile(cluster) - - // Creating Amazon EKS Worker Nodes - keyPairName := awsKeyPairPrefix + ac.serviceSuffix - ac.createEksEc2KeyPair(ec2Client, &keyPairName) - - nodesStackName := awsNodesStackPrefix + ac.serviceSuffix - nodeGroupName := awsNodeGroupPrefix + ac.serviceSuffix - nodeSequrityGroup, nodeInstanceRole := ac.createEksWorkerNodes(cfClient, &nodesStackName, &nodeGroupName, &clusterName, &keyPairName, clusterStackOutputs) - - ac.authorizeSecurityGroupIngress(ec2Client, nodeSequrityGroup) - - // Enable worker nodes to join the cluster - sf, err := ioutil.ReadFile(path.Join(ac.configPath, "aws-auth-cm-temp.yaml")) - ac.checkError(err) - - f, err := ioutil.TempFile(os.TempDir(), "aws-auth-cm-temp-*.yaml") - ac.checkError(err) - - s := string(sf) - _, err = f.Write([]byte(strings.Replace(s, "", *nodeInstanceRole, -1))) - ac.checkError(err) - - _ = f.Close() - - ac.execCommand("kubectl", "apply", "-f", f.Name()) - _ = os.Remove(f.Name()) - - ac.createSSHConfig(ec2Client, clusterStackOutputs.VpcId, "scp-config"+ac.serviceSuffix) - - ac.execCommand("kubectl", "apply", "-f", "aws-k8s-cni.yaml") -} - -func (ac *AWSCluster) execCommand(name string, arg ...string) { - log.Printf("> %s %s", name, strings.Join(arg, " ")) - cmd := exec.Command(name, arg...) // #nosec - var out bytes.Buffer - cmd.Stdout = &out - err := cmd.Run() - ac.checkError(err) - log.Printf(out.String()) -} diff --git a/scripts/aws/destroy-kubernetes-cluster.go b/scripts/aws/destroy-kubernetes-cluster.go deleted file mode 100644 index aebed4f..0000000 --- a/scripts/aws/destroy-kubernetes-cluster.go +++ /dev/null @@ -1,358 +0,0 @@ -package main - -import ( - "log" - "regexp" - "sync" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/cloudformation" - "github.com/aws/aws-sdk-go/service/ec2" - "github.com/aws/aws-sdk-go/service/eks" - "github.com/aws/aws-sdk-go/service/iam" -) - -//nolint:gochecknoglobals -var deferError error - -func (ac *AWSCluster) checkDeferError(err error) bool { - if err != nil { - if aerr, ok := err.(awserr.Error); ok { - switch aerr.Code() { - case "NoSuchEntity": - case "ResourceNotFoundException": - case "ValidationError": - case "InvalidParameterValue": - default: - log.Printf("Error (%s): %s\n", aerr.Code(), aerr.Message()) - deferError = aerr - return true - } - log.Printf("Warning (%s): %s\n", aerr.Code(), aerr.Message()) - } else { - log.Printf("Error: %s\n", err.Error()) - deferError = aerr - } - return true - } - return false -} - -func (ac *AWSCluster) deleteEksRole(iamClient *iam.IAM, eksRoleName *string) { - log.Printf("Deleting EKS service role \"%s\"...\n", *eksRoleName) - - policyArn := "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy" - _, err := iamClient.DetachRolePolicy(&iam.DetachRolePolicyInput{ - RoleName: eksRoleName, - PolicyArn: &policyArn, - }) - ac.checkDeferError(err) - - policyArn = "arn:aws:iam::aws:policy/AmazonEKSServicePolicy" - _, err = iamClient.DetachRolePolicy(&iam.DetachRolePolicyInput{ - RoleName: eksRoleName, - PolicyArn: &policyArn, - }) - ac.checkDeferError(err) - - _, err = iamClient.DeleteRole(&iam.DeleteRoleInput{ - RoleName: eksRoleName, - }) - if ac.checkDeferError(err) { - return - } - - log.Printf("Role \"%s\" successfully deleted!\n", *eksRoleName) -} - -func (ac *AWSCluster) deleteEC2NetworkInterfaces(ec2Client *ec2.EC2, cfClient *cloudformation.CloudFormation, nodesStackName *string) { - cfResp, err := cfClient.DescribeStacks(&cloudformation.DescribeStacksInput{ - StackName: nodesStackName, - }) - if ac.checkDeferError(err) { - return - } - - var sequrityGroup *string - for _, output := range cfResp.Stacks[0].Outputs { - if *output.OutputKey == "NodeSecurityGroup" { - sequrityGroup = output.OutputValue - break - } - } - - ec2Resp, err := ec2Client.DescribeNetworkInterfaces(&ec2.DescribeNetworkInterfacesInput{ - Filters: []*ec2.Filter{ - { - Name: aws.String("group-id"), - Values: []*string{sequrityGroup}, - }, - }, - }) - - if ac.checkDeferError(err) { - return - } - - for _, networkInterface := range ec2Resp.NetworkInterfaces { - _, err := ec2Client.DeleteNetworkInterface(&ec2.DeleteNetworkInterfaceInput{ - NetworkInterfaceId: networkInterface.NetworkInterfaceId, - }) - - ac.checkDeferError(err) - } -} - -//nolint:funlen -func (ac *AWSCluster) deleteEksClusterVpc(cfClient *cloudformation.CloudFormation, ec2Client *ec2.EC2, clusterStackName *string) { - log.Printf("Deleting Amazon EKS Cluster VPC \"%s\"...\n", *clusterStackName) - - resp, err := cfClient.DescribeStacks(&cloudformation.DescribeStacksInput{ - StackName: clusterStackName, - }) - if ac.checkDeferError(err) { - return - } - outputsMap := outputsToMap(resp.Stacks[0].Outputs) - - output, err := ec2Client.DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{ - Filters: []*ec2.Filter{ - { - Name: aws.String("vpc-id"), - Values: []*string{outputsMap.VpcId}, - }, - }, - }) - if !ac.checkDeferError(err) { - for _, sg := range output.SecurityGroups { - if *sg.GroupName != "default" { - log.Printf("Deleting EC2 Security Group %s...", *sg.GroupId) - _, err = ec2Client.DeleteSecurityGroup(&ec2.DeleteSecurityGroupInput{GroupId: sg.GroupId}) - ac.checkDeferError(err) - } - } - } - - _, err = ec2Client.DeleteVpc(&ec2.DeleteVpcInput{VpcId: outputsMap.VpcId}) - ac.checkDeferError(err) - - stackId := resp.Stacks[0].StackId - - _, err = cfClient.DeleteStack(&cloudformation.DeleteStackInput{ - StackName: clusterStackName, - }) - if err != nil { - _, err = cfClient.DeleteStack(&cloudformation.DeleteStackInput{ - StackName: clusterStackName, - RetainResources: []*string{aws.String("Ipv6VPCCidrBlock")}, - }) - if ac.checkDeferError(err) { - return - } - } - - for { - resp, err := cfClient.DescribeStacks(&cloudformation.DescribeStacksInput{ - StackName: stackId, - }) - if ac.checkDeferError(err) { - if err.(awserr.Error).Code() == "ValidationError" { - log.Printf("Cluster VPC \"%s\" successfully deleted!\n", *clusterStackName) - } - return - } - - switch *resp.Stacks[0].StackStatus { - case "DELETE_COMPLETE": - log.Printf("Cluster VPC \"%s\" successfully deleted!\n", *clusterStackName) - return - case "DELETE_IN_PROGRESS": - time.Sleep(requestInterval) - default: - log.Printf("Error: Unexpected stack status: %s\n", *resp.Stacks[0].StackStatus) - deferError = errors.Errorf("unexpected stack status: %s", *resp.Stacks[0].StackStatus) - return - } - } -} - -func (ac *AWSCluster) deleteEksCluster(eksClient *eks.EKS, clusterName *string) { - log.Printf("Deleting Amazon EKS Cluster \"%s\"...\n", *clusterName) - - _, err := eksClient.DeleteCluster(&eks.DeleteClusterInput{ - Name: clusterName, - }) - if ac.checkDeferError(err) { - return - } - - for { - resp, err := eksClient.DescribeCluster(&eks.DescribeClusterInput{ - Name: clusterName, - }) - - if err != nil && err.(awserr.Error).Code() == "ResourceNotFoundException" { - log.Printf("EKS Cluster \"%s\" successfully Deleted!\n", *clusterName) - return - } - - if ac.checkDeferError(err) { - return - } - - switch *resp.Cluster.Status { - case "DELETING": - time.Sleep(requestInterval) - default: - log.Printf("Error: Unexpected cluster status: %s\n", *resp.Cluster.Status) - deferError = errors.Errorf("unexpected cluster status: %s", *resp.Cluster.Status) - return - } - } -} - -func (ac *AWSCluster) deleteEksEc2KeyPair(ec2Client *ec2.EC2, keyPairName *string) { - log.Printf("Deleting Amazon EC2 key pair \"%s\"...\n", *keyPairName) - _, err := ec2Client.DeleteKeyPair(&ec2.DeleteKeyPairInput{ - KeyName: keyPairName, - }) - if ac.checkDeferError(err) { - return - } - - log.Printf("Amazon EC2 key pair \"%s\" successfully Deleted!\n", *keyPairName) -} - -func (ac *AWSCluster) deleteEksWorkerNodes(cfClient *cloudformation.CloudFormation, nodesStackName *string) { - log.Printf("Deleting Amazon EKS Worker Nodes...\n") - - resp, err := cfClient.DescribeStacks(&cloudformation.DescribeStacksInput{ - StackName: nodesStackName, - }) - if ac.checkDeferError(err) { - return - } - - stackId := resp.Stacks[0].StackId - - _, err = cfClient.DeleteStack(&cloudformation.DeleteStackInput{ - StackName: nodesStackName, - }) - if ac.checkDeferError(err) { - return - } - - for { - resp, err := cfClient.DescribeStacks(&cloudformation.DescribeStacksInput{ - StackName: stackId, - }) - if ac.checkDeferError(err) { - return - } - - switch *resp.Stacks[0].StackStatus { - case "DELETE_COMPLETE": - log.Printf("EKS Worker Nodes \"%s\" successfully deleted!\n", *nodesStackName) - return - case "DELETE_IN_PROGRESS": - time.Sleep(requestInterval) - default: - log.Printf("Error: Unexpected stack status: %s\n", *resp.Stacks[0].StackStatus) - deferError = errors.Errorf("unexpected stack status: %s", *resp.Stacks[0].StackStatus) - return - } - } -} - -// DeleteAWSKubernetesCluster - deleting AWS cluster instances -func (ac *AWSCluster) DeleteAWSKubernetesCluster() error { - sess := session.Must(session.NewSession()) - iamClient := iam.New(sess) - eksClient := eks.New(sess) - cfClient := cloudformation.New(sess) - ec2Client := ec2.New(sess) - - // Deleting Amazon EKS Cluster - clusterName := awsClusterPrefix + ac.serviceSuffix - ac.deleteEksCluster(eksClient, &clusterName) - - nodesStackName := awsNodesStackPrefix + ac.serviceSuffix - // Deleting EC2 Network Interfaces to allow instance to be properly released - ac.deleteEC2NetworkInterfaces(ec2Client, cfClient, &nodesStackName) - - // Deleting Amazon EKS Worker Nodes - ac.deleteEksWorkerNodes(cfClient, &nodesStackName) - - // Deleting Amazon EKS Cluster VPC - clusterStackName := awsClusterStackPrefix + ac.serviceSuffix - ac.deleteEksClusterVpc(cfClient, ec2Client, &clusterStackName) - - // Deleting Amazon Roles and Keys - eksRoleName := awsRolePrefix + ac.serviceSuffix - ac.deleteEksRole(iamClient, &eksRoleName) - keyPairName := awsKeyPairPrefix + ac.serviceSuffix - ac.deleteEksEc2KeyPair(ec2Client, &keyPairName) - - return deferError -} - -// DeleteAllKubernetesClusters - removes all aws clusters older than saveDuration and filtered by name -func DeleteAllKubernetesClusters(saveDuration time.Duration, namePattern string) { - sess := session.Must(session.NewSession()) - cfClient := cloudformation.New(sess) - - var filter = []*string{aws.String("CREATE_IN_PROGRESS"), aws.String("CREATE_FAILED"), aws.String("CREATE_COMPLETE"), aws.String("ROLLBACK_IN_PROGRESS"), aws.String("ROLLBACK_FAILED"), aws.String("ROLLBACK_COMPLETE"), aws.String("DELETE_IN_PROGRESS"), aws.String("DELETE_FAILED"), aws.String("UPDATE_IN_PROGRESS"), aws.String("UPDATE_COMPLETE_CLEANUP_IN_PROGRESS"), aws.String("UPDATE_COMPLETE"), aws.String("UPDATE_ROLLBACK_IN_PROGRESS"), aws.String("UPDATE_ROLLBACK_FAILED"), aws.String("UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS"), aws.String("UPDATE_ROLLBACK_COMPLETE"), aws.String("REVIEW_IN_PROGRESS")} - stacks, err := cfClient.ListStacks(&cloudformation.ListStacksInput{StackStatusFilter: filter}) - if err != nil { - logrus.Infof("AWS EKS Error: %v", err) - return - } - - var wg sync.WaitGroup - i := 0 - for _, stack := range stacks.StackSummaries { - stackName := aws.StringValue(stack.StackName) - if stackName[:len(awsClusterStackPrefix)] != "nsm-srv" { - continue - } - - if aws.StringValue(stack.StackStatus) == "DELETE_COMPLETE" { - continue - } - - if matched, err := regexp.MatchString(namePattern, stackName[len(awsClusterStackPrefix):]); err != nil || !matched { - logrus.Infof("Skip cluster: %s (matched: %v; err: %v)", stackName[len(awsClusterStackPrefix):], matched, err) - continue - } - - if stack.CreationTime.After(time.Now().Add(-1 * saveDuration)) { - logrus.Infof("Skip cluster: %s (created: %v)", stackName[len(awsClusterStackPrefix):], stack.CreationTime) - continue - } - - wg.Add(1) - - go func() { - defer wg.Done() - for att := 0; att < 3; att++ { - logrus.Infof("Deleting %s (created %v), attempt %d", stackName[len(awsClusterStackPrefix):], stack.CreationTime, att+1) - err := NewAWSCluster(stackName[len(awsClusterStackPrefix):]).DeleteAWSKubernetesCluster() - if err == nil { - break - } - } - }() - - i++ - if i%5 == 0 { - wg.Wait() // Guard from AWS Throttling error - } - } - wg.Wait() -} diff --git a/scripts/aws/destroy-old-clusters.sh b/scripts/aws/destroy-old-clusters.sh index 518176b..5e6f0e4 100755 --- a/scripts/aws/destroy-old-clusters.sh +++ b/scripts/aws/destroy-old-clusters.sh @@ -1,5 +1,12 @@ #!/bin/bash -pushd "$(dirname "${BASH_SOURCE[0]}")" || exit 1 -AWS_REGION=us-east-2 go run ./... DeleteAll "$1" "$2" -popd || exit 0 \ No newline at end of file +out=$(eksctl get clusters) + + +for clusterName in $out +do + if [[ $clusterName =~ $1 ]] + then + eksctl delete cluster --name "$clusterName" + fi +done \ No newline at end of file diff --git a/scripts/aws/go.mod b/scripts/aws/go.mod deleted file mode 100644 index a54e403..0000000 --- a/scripts/aws/go.mod +++ /dev/null @@ -1,13 +0,0 @@ -module github.com/networkservicemesh/networkservicemesh/scripts/aws - -go 1.13 - -require ( - github.com/aws/aws-sdk-go v1.23.19 - github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect - github.com/pkg/errors v0.8.1 - github.com/sirupsen/logrus v1.4.2 - golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa // indirect - golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9 // indirect - golang.org/x/text v0.3.2 // indirect -) diff --git a/scripts/aws/go.sum b/scripts/aws/go.sum deleted file mode 100644 index 6c5279a..0000000 --- a/scripts/aws/go.sum +++ /dev/null @@ -1,29 +0,0 @@ -github.com/aws/aws-sdk-go v1.23.19 h1:QiEkjRHkDXAThgnHKSEC63JwsSjL/jfYUOA2QYFmbSw= -github.com/aws/aws-sdk-go v1.23.19/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa h1:F+8P+gmewFQYRk6JoLQLwjBCTu3mcIURZfNkVweuRKA= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9 h1:1/DFK4b7JH8DmkqhUk48onnSfrPzImPoVxuomtbT2nk= -golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/scripts/aws/kube-config-template b/scripts/aws/kube-config-template deleted file mode 100644 index 517787f..0000000 --- a/scripts/aws/kube-config-template +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: v1 -clusters: -- cluster: - certificate-authority-data: - server: - name: -contexts: -- context: - cluster: - user: - name: -current-context: -kind: Config -preferences: {} -users: -- name: - user: - exec: - apiVersion: client.authentication.k8s.io/v1alpha1 - args: - - token - - -i - - - command: aws-iam-authenticator diff --git a/scripts/aws/kubernetes-cluster.go b/scripts/aws/kubernetes-cluster.go deleted file mode 100644 index ebf3a4c..0000000 --- a/scripts/aws/kubernetes-cluster.go +++ /dev/null @@ -1,120 +0,0 @@ -package main - -import ( - "fmt" - "log" - "os" - "path" - "runtime" - "strconv" - "time" - - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/sirupsen/logrus" -) - -const requestInterval = 5 * time.Second - -const ( - awsClusterPrefix = "nsm" - awsRolePrefix = "nsm-role" - awsClusterStackPrefix = "nsm-srv" - awsKeyPairPrefix = "nsm-key-pair" - awsNodesStackPrefix = "nsm-nodes" - awsNodeGroupPrefix = "nsm-node-group" -) - -// AWSCluster - controlling aws clusters -type AWSCluster struct { - configPath string - deferError error - serviceSuffix string -} - -// NewAWSCluster - Creates new instance of AWS cluster for creation and deletion -func NewAWSCluster(serviceSuffix string) *AWSCluster { - _, currentFilePath, _, ok := runtime.Caller(0) - if !ok { - currentFilePath = "." - } - return &AWSCluster{ - configPath: path.Dir(currentFilePath), - serviceSuffix: serviceSuffix, - deferError: nil, - } -} - -func printUsage() { - fmt.Printf("Usage: go run ./... \n" + - "AWS support commands:\n" + - " Create Create EKS cluster and configure kubernetes\n" + - " Delete Destroy EKS cluster\n" + - " DeleteAll N Destroy All EKS clusters older than N hours (Example: DeleteAll 24) \n") -} - -func makeCreateClusterAttempt(cluster *AWSCluster) (resultErr error) { - defer func() { - rErr := recover() - if err, ok := rErr.(error); ok { - resultErr = err - } - if err, ok := rErr.(awserr.Error); ok { - resultErr = err - } - }() - - cluster.CreateAWSKubernetesCluster() - return nil -} - -func main() { - if len(os.Args) < 2 { - printUsage() - return - } - - switch os.Args[1] { - case "Create": - cluster := NewAWSCluster(os.Getenv("NSM_AWS_SERVICE_SUFFIX")) - for { - err := makeCreateClusterAttempt(cluster) - if aerr, ok := err.(awserr.Error); ok { - if aerr.Code() == "Throttling" { - log.Printf("Warning (%s): %s\n", aerr.Code(), aerr.Message()) - log.Printf("Restarting AWS kubernetes cluster creation...") - continue - } - log.Fatalf("Error (%s): %s\n", aerr.Code(), aerr.Message()) - } else if err != nil { - log.Fatalf("Error: %s\n", err.Error()) - } - break - } - case "Delete": - err := NewAWSCluster(os.Getenv("NSM_AWS_SERVICE_SUFFIX")).DeleteAWSKubernetesCluster() - if err != nil { - os.Exit(1) - } - case "DeleteAll": - var durationHours int64 - var namePattern string - var err error - if len(os.Args) < 4 { - namePattern = ".*" - } else { - namePattern = os.Args[3] - } - if len(os.Args) < 3 { - durationHours = 0 - } else { - durationHours, err = strconv.ParseInt(os.Args[2], 10, 64) - } - if err != nil { - logrus.Errorf("Cannot parse: %v", err) - return - } - DeleteAllKubernetesClusters(time.Duration(durationHours)*time.Hour, namePattern) - default: - printUsage() - } -} diff --git a/scripts/aws/scp-config-template b/scripts/aws/scp-config-template deleted file mode 100644 index 04164d8..0000000 --- a/scripts/aws/scp-config-template +++ /dev/null @@ -1,7 +0,0 @@ -Host aws-master - HostName - User ec2-user - -Host aws-worker - HostName - User ec2-user \ No newline at end of file