diff --git a/.github/workflows/e2e_libvirt.yaml b/.github/workflows/e2e_libvirt.yaml index 8431aaa94..1ef46130b 100644 --- a/.github/workflows/e2e_libvirt.yaml +++ b/.github/workflows/e2e_libvirt.yaml @@ -23,7 +23,11 @@ on: description: Git ref to checkout the cloud-api-adaptor repository. Defaults to main. required: false type: string - + e2e: + default: 'none' + description: SecureComms configuration. Defaults to none. + required: false + type: string env: CLOUD_PROVIDER: libvirt DEBIAN_FRONTEND: noninteractive @@ -36,6 +40,11 @@ jobs: test: runs-on: az-ubuntu-2204 steps: + - name: Test The Test + run: | + echo "This is a test - remove it!" + echo "secure_comms=\"${{ inputs.secure_comms }}\"" + - name: Checkout Code uses: actions/checkout@v4 with: @@ -84,7 +93,8 @@ jobs: - name: Config Libvirt run: | - ./libvirt/config_libvirt.sh + echo "secure_comms=\"${{ inputs.secure_comms }}\"" + ./libvirt/config_libvirt.sh ${{ inputs.secure_comms }} echo "CAA_IMAGE=\"${{ inputs.caa_image }}\"" >> libvirt.properties # For debugging cat libvirt.properties diff --git a/.github/workflows/e2e_run_all.yaml b/.github/workflows/e2e_run_all.yaml index 35ae34698..2ccfad4c1 100644 --- a/.github/workflows/e2e_run_all.yaml +++ b/.github/workflows/e2e_run_all.yaml @@ -165,10 +165,12 @@ jobs: - generic arch: - amd64 + secure_comms: [none, withoutKbs] uses: ./.github/workflows/e2e_libvirt.yaml with: caa_image: ${{ inputs.registry }}/cloud-api-adaptor:${{ inputs.caa_image_tag }}-dev podvm_image: ${{ inputs.registry }}/podvm-${{ matrix.provider }}-${{ matrix.os }}-${{ matrix.arch }}:${{ inputs.podvm_image_tag }} install_directory_artifact: install_directory git_ref: ${{ inputs.git_ref }} + secure_comms: ${{ matrix.provider }} secrets: inherit diff --git a/src/cloud-api-adaptor/cmd/agent-protocol-forwarder/main.go b/src/cloud-api-adaptor/cmd/agent-protocol-forwarder/main.go index 47178c7b8..0d218566c 100644 --- a/src/cloud-api-adaptor/cmd/agent-protocol-forwarder/main.go +++ b/src/cloud-api-adaptor/cmd/agent-protocol-forwarder/main.go @@ -93,7 +93,9 @@ func (cfg *Config) Setup() (cmd.Starter, error) { return nil, err } - if secureComms { + if secureComms || cfg.daemonConfig.SecureComms { + var inbounds, outbounds []string + ppssh.Singleton() host, port, err := net.SplitHostPort(cfg.listenAddr) if err != nil { @@ -103,15 +105,32 @@ func (cfg *Config) Setup() (cmd.Starter, error) { logger.Printf("Address %s is changed to 127.0.0.1:%s since secure-comms is enabled.", cfg.listenAddr, port) cfg.listenAddr = "127.0.0.1:" + port } - inbounds := append([]string{"BOTH_PHASES:KBS:8080"}, strings.Split(secureCommsInbounds, ",")...) - outbounds := append([]string{"KUBERNETES_PHASE:KATAAGENT:" + port}, strings.Split(secureCommsOutbounds, ",")...) // Create a Client that will approach the api-server-rest service at the podns // To obtain secrets from KBS, we approach the api-server-rest service which then approaches the CDH asking for a secret resource // the CDH than contact the KBS (possibly after approaching Attestation Agent for a token) and the KBS serves the requested key // The communication between the CDH (and Attestation Agent) and the KBS is performed via an SSH tunnel named "KBS" apic := apic.NewApiClient(API_SERVER_REST_PORT, cfg.podNamespace) - services = append(services, ppssh.NewSshServer(inbounds, outbounds, ppssh.GetSecret(apic.GetKey), sshutil.SSHPORT)) + + ppSecrets := ppssh.NewPpSecrets(ppssh.GetSecret(apic.GetKey)) + + if secureComms { + // CoCo in production + ppSecrets.AddKey(ppssh.WN_PUBLIC_KEY) + ppSecrets.AddKey(ppssh.PP_PRIVATE_KEY) + inbounds = append([]string{"BOTH_PHASES:KBS:8080"}, strings.Split(secureCommsInbounds, ",")...) + outbounds = append([]string{"KUBERNETES_PHASE:KATAAGENT:" + port}, strings.Split(secureCommsOutbounds, ",")...) + + } else { + // Never here under CoCo in production + // Set secureComms using daemonConfig for testing + ppSecrets.SetKey(ppssh.WN_PUBLIC_KEY, cfg.daemonConfig.WnPublicKey) + ppSecrets.SetKey(ppssh.PP_PRIVATE_KEY, cfg.daemonConfig.PpPrivateKey) + inbounds = append([]string{"BOTH_PHASES:KBS:8080"}, strings.Split(cfg.daemonConfig.SecureCommsInbounds, ",")...) + outbounds = append([]string{"KUBERNETES_PHASE:KATAAGENT:" + port}, strings.Split(cfg.daemonConfig.SecureCommsOutbounds, ",")...) + } + + services = append(services, ppssh.NewSshServer(inbounds, outbounds, ppSecrets, sshutil.SSHPORT)) } else { if !disableTLS { cfg.tlsConfig = &tlsConfig diff --git a/src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go b/src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go index 6bdbadd63..95f4a3721 100644 --- a/src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go +++ b/src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go @@ -9,9 +9,11 @@ import ( "fmt" "io" "os" + "strings" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/cmd" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor" + "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/cloud" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/proxy" daemon "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/forwarder" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/podnetwork/tunneler/vxlan" @@ -28,7 +30,7 @@ const ( ) type daemonConfig struct { - serverConfig adaptor.ServerConfig + serverConfig cloud.ServerConfig networkConfig } @@ -86,12 +88,14 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) { } var ( - disableTLS bool - tlsConfig tlsutil.TLSConfig - secureComms bool - secureCommsInbounds string - secureCommsOutbounds string - secureCommsKbsAddr string + disableTLS bool + tlsConfig tlsutil.TLSConfig + secureComms bool + secureCommsInbounds string + secureCommsOutbounds string + secureCommsPpInbounds string + secureCommsPpOutbounds string + secureCommsKbsAddr string ) cmd.Parse(programName, os.Args[1:], func(flags *flag.FlagSet) { @@ -112,8 +116,10 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) { flags.BoolVar(&tlsConfig.SkipVerify, "tls-skip-verify", false, "Skip TLS certificate verification - use it only for testing") flags.BoolVar(&disableTLS, "disable-tls", false, "Disable TLS encryption - use it only for testing") flags.BoolVar(&secureComms, "secure-comms", false, "Use SSH to secure communication between cluster and peer pods") - flags.StringVar(&secureCommsInbounds, "secure-comms-inbounds", "", "Inbound tags for secure communication tunnels") - flags.StringVar(&secureCommsOutbounds, "secure-comms-outbounds", "", "Outbound tags for secure communication tunnels") + flags.StringVar(&secureCommsInbounds, "secure-comms-inbounds", "", "WN Inbound tags for secure communication tunnels") + flags.StringVar(&secureCommsOutbounds, "secure-comms-outbounds", "", "WN Outbound tags for secure communication tunnels") + flags.StringVar(&secureCommsPpInbounds, "secure-comms-pp-inbounds", "", "PP Inbound tags for secure communication tunnels") + flags.StringVar(&secureCommsPpOutbounds, "secure-comms-pp-outbounds", "", "PP Outbound tags for secure communication tunnels") flags.StringVar(&secureCommsKbsAddr, "secure-comms-kbs", "kbs-service.trustee-operator-system:8080", "Address of a Trustee Service for Secure-Comms") flags.DurationVar(&cfg.serverConfig.ProxyTimeout, "proxy-timeout", proxy.DefaultProxyTimeout, "Maximum timeout in minutes for establishing agent proxy connection") @@ -137,10 +143,16 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) { if err != nil { return nil, fmt.Errorf("secure comms failed to initialize KubeMgr: %w", err) } - + if strings.EqualFold(secureCommsKbsAddr, "false") { + secureCommsKbsAddr = "" + fmt.Printf("secureCommsKbsAddr was false\n") + } + fmt.Printf("secureCommsKbsAddr is %s\n", secureCommsKbsAddr) cfg.serverConfig.SecureComms = true cfg.serverConfig.SecureCommsInbounds = secureCommsInbounds cfg.serverConfig.SecureCommsOutbounds = secureCommsOutbounds + cfg.serverConfig.SecureCommsPpInbounds = secureCommsPpInbounds + cfg.serverConfig.SecureCommsPpOutbounds = secureCommsPpOutbounds cfg.serverConfig.SecureCommsKbsAddress = secureCommsKbsAddr } else { if !disableTLS { diff --git a/src/cloud-api-adaptor/docs/SecureComms.md b/src/cloud-api-adaptor/docs/SecureComms.md index e9737dd8d..6821dd8c8 100644 --- a/src/cloud-api-adaptor/docs/SecureComms.md +++ b/src/cloud-api-adaptor/docs/SecureComms.md @@ -27,7 +27,35 @@ Once the "Kubernetes Phase" SSH channel is established, Secure Comms connects th See [Secure Comms Architecture Slides](./SecureComms.pdf) for more details. -## Setup +## Setup for for testing without Trustee (and for non-CoCo peerpods) + +### Deploy CAA +Use any of the option for installing CAA depending on the cloud driver used. + + +### Activate Secure-Comms feature from CAA side +Activate Secure-Comms from CAA side by changing the `SECURE_COMMS` parameter of the `peer-pods-cm` configMap in the `confidential-containers-system` namespace to `"true"`. + +```sh +kubectl -n confidential-containers-system get cm peer-pods-cm -o yaml | sed "s/SECURE_COMMS: \"false\"/SECURE_COMMS: \"true\"/"|kubectl apply -f - +``` + +You may also include additional Inbounds and Outbounds configurations to the Adaptor side using the `SECURE_COMMS_INBOUNDS` and `SECURE_COMMS_OUTBOUNDS` config points. +You may also add Inbounds and Outbounds configurations to the Forwarder side using the `SECURE_COMMS_PP_INBOUNDS` and `SECURE_COMMS_PP_OUTBOUNDS` config points. [See more details regarding Inbounds and Outbounds below.](#adding-named-tunnels-to-the-ssh-channel) + +Use `kubectl edit cm peer-pods-cm -n confidential-containers-system` to make such changes in the configMap, for example: +```sh +apiVersion: v1 +data: + ... + SECURE_COMMS: "true" + SECURE_COMMS_OUTBOUNDS: "KUBERNETES_PHASE:mytunnel:149.81.64.62:7777" + SECURE_COMMS_PP_INBOUNDS: "KUBERNETES_PHASE:mytunnel:6666" + SECURE_COMMS_KBS_ADDR: "false" + ... +``` + +## Setup for CoCo with Trustee ### Deploy CAA Use any of the option for installing CAA depending on the cloud driver used. @@ -53,8 +81,7 @@ kubectl get secret kbs-client -n trustee-operator-system -o json|jq --arg ns "co For a testing environment, you may need to change the policy of the KBS and AS using the KBS Client to allow all or fit your own policy. One way to do that is: ```sh -kubectl -n trustee-operator-system exec deployment/trustee-deployment --container as -it -- /bin/bash - sed -i.bak 's/^default allow = false/default allow = true/' /opt/confidential-containers/attestation-service/opa/default.rego +kubectl -n trustee-operator-system exec deployment/trustee-deployment --container as -it -- sed -i.bak 's/^default allow = false/default allow = true/' /opt/confidential-containers/attestation-service/opa/default.rego kubectl -n trustee-operator-system get cm resource-policy -o yaml | sed "s/default allow = false/default allow = true/"|kubectl apply -f - ``` @@ -66,18 +93,61 @@ Change the `src/cloud-api-adaptor/podvm/files/etc/systemd/system/agent-protocol- ExecStart=/usr/local/bin/agent-protocol-forwarder -pod-namespace /run/netns/podns -secure-comms -kata-agent-socket /run/kata-containers/agent.sock $TLS_OPTIONS $OPTIONS ``` -You may also include additional Inbounds and Outbounds configurations to the Forwarder using the `-secure-comms-inbounds` and `-secure-comms-outbounds` flags. See more details regarding Inbounds and Outbounds below. +You may also include additional Inbounds and Outbounds configurations to the Forwarder using the `-secure-comms-inbounds` and `-secure-comms-outbounds` flags. [See more details regarding Inbounds and Outbounds below.](#adding-named-tunnels-to-the-ssh-channel) + +For example: +```sh +ExecStart=/usr/local/bin/agent-protocol-forwarder -kata-agent-namespace /run/netns/podns -secure-comms -secure-comms-inbounds KUBERNETES_PHASE:mytunnel:6666 -kata-agent-socket /run/kata-containers/agent.sock $TLS_OPTIONS $OPTIONS +``` Once you changed `podvm/files/etc/systemd/system/agent-protocol-forwarder.service`, you will need to [rebuild the podvm](./../podvm/README.md). ### Activate CAA Secure-Comms feature -Use `kubectl edit cm peer-pods-cm -n confidential-containers-system` to add to the `peer-pods-cm` config map at the `confidential-containers-system` namespace: +Activate Secure-Comms of CAA by changing the `SECURE_COMMS` parameter of the `peer-pods-cm` configMap in the `confidential-containers-system` namespace to `"true"`. + +```sh +kubectl -n confidential-containers-system get cm peer-pods-cm -o yaml | sed "s/SECURE_COMMS: \"false\"/SECURE_COMMS: \"true\"/"|kubectl apply -f - +``` + +Set InitData to point KBC services to IP address 127.0.0.1 +```sh +cat < /tmp/initdata.txt +algorithm = "sha384" +version = "0.1.0" + +[data] +"aa.toml" = ''' +[token_configs] +[token_configs.coco_as] +url = 'http://127.0.0.1:8080' + +[token_configs.kbs] +url = 'http://127.0.0.1:8080' +''' +"cdh.toml" = ''' +socket = 'unix:///run/confidential-containers/cdh.sock' +credentials = [] + +[kbc] +name = 'cc_kbc' +url = 'http://127.0.0.1:8080' +''' +EOF +export INITDATA=`base64 -w 0 /tmp/initdata.txt` +kubectl -n confidential-containers-system get cm peer-pods-cm -o yaml | sed 's/^INITDATA: .*/INITDATA: '$INITDATA'/'|kubectl apply -f - + +``` + +You may also include additional Inbounds and Outbounds configurations to the Adaptor using the `SECURE_COMMS_INBOUNDS` and `SECURE_COMMS_OUTBOUNDS` config points. [See more details regarding Inbounds and Outbounds below.](#adding-named-tunnels-to-the-ssh-channel) + +Use `kubectl edit cm peer-pods-cm -n confidential-containers-system` to make such changes in the configMap, for example: ```sh apiVersion: v1 data: ... SECURE_COMMS: "true" + SECURE_COMMS_OUTBOUNDS: "KUBERNETES_PHASE:mytunnel:149.81.64.62:7777" ... ``` @@ -120,7 +190,7 @@ You may also set the KBS address using the `SECURE_COMMS_KBS_ADDR` config point. > -### Adding named tunnels to the SSH channel +## Adding named tunnels to the SSH channel Named tunnels can be added to the SSH channel. Adding a named tunnel requires adding an Inbound at one of the SSH channel peers and an Outbound at the other SSH channel peer. The Inbound and Outbound both carry the name of the tunnel being created. |---------Tunnel----------| @@ -158,5 +228,4 @@ Alternatively, the client and server can be separately executed in independent t - Add DeleteResource() support in KBS, KBC, api-server-rest, than cleanup resources added by Secure Comms to KBS whenever a Peer Pod fail to be created or when a Peer Pod is terminated. - Add support for running the vxlan tunnel traffic via a Secure Comms SSH tunnel -- Add support for non-confidential Peer Pods which do not go via an Attestation Phase. - Add support for KBS identities allowing a Peer Pod to register its own identity in KBS and replace the current Secure Comms mechanism which delivers a private key to the Peer Pod via the KBS diff --git a/src/cloud-api-adaptor/entrypoint.sh b/src/cloud-api-adaptor/entrypoint.sh index 3810e6bb9..eac62036e 100755 --- a/src/cloud-api-adaptor/entrypoint.sh +++ b/src/cloud-api-adaptor/entrypoint.sh @@ -24,6 +24,8 @@ optionals+="" [[ "${SECURE_COMMS}" == "true" ]] && optionals+="-secure-comms " [[ "${SECURE_COMMS_INBOUNDS}" ]] && optionals+="-secure-comms-inbounds ${SECURE_COMMS_INBOUNDS} " [[ "${SECURE_COMMS_OUTBOUNDS}" ]] && optionals+="-secure-comms-outbounds ${SECURE_COMMS_OUTBOUNDS} " +[[ "${SECURE_COMMS_PP_INBOUNDS}" ]] && optionals+="-secure-comms-pp-inbounds ${SECURE_COMMS_PP_INBOUNDS} " +[[ "${SECURE_COMMS_PP_OUTBOUNDS}" ]] && optionals+="-secure-comms-pp-outbounds ${SECURE_COMMS_PP_OUTBOUNDS} " [[ "${SECURE_COMMS_KBS_ADDR}" ]] && optionals+="-secure-comms-kbs ${SECURE_COMMS_KBS_ADDR} " [[ "${PEERPODS_LIMIT_PER_NODE}" ]] && optionals+="-peerpods-limit-per-node ${PEERPODS_LIMIT_PER_NODE} " diff --git a/src/cloud-api-adaptor/libvirt/README.md b/src/cloud-api-adaptor/libvirt/README.md index f7e1c310a..e3e9a665e 100644 --- a/src/cloud-api-adaptor/libvirt/README.md +++ b/src/cloud-api-adaptor/libvirt/README.md @@ -217,6 +217,7 @@ make TEST_PROVISION=no TEST_TEARDOWN=no TEST_PODVM_IMAGE=$PWD/podvm/podvm.qcow2 * ``TEST_E2E_TIMEOUT`` - test timeout * ``DEPLOY_KBS`` - whether to deploy the key-broker-service, which is used to test the attestation flow * ``TEST_PROVISION_FILE`` - file specifying the libvirt connection and the ssh key file (created earlier by [config_libvirt.sh](config_libvirt.sh)) +* ``TEST_CAA_LOG`` - whether to log the CAA at teh end of the tests run # Delete Confidential Containers and cloud-api-adaptor from the cluster diff --git a/src/cloud-api-adaptor/libvirt/config_libvirt.sh b/src/cloud-api-adaptor/libvirt/config_libvirt.sh index a3d33aacb..54ddcd382 100755 --- a/src/cloud-api-adaptor/libvirt/config_libvirt.sh +++ b/src/cloud-api-adaptor/libvirt/config_libvirt.sh @@ -92,6 +92,15 @@ installK8sclis() { fi } +if [ $# -lt 1 ] +then + SECURE_COMMS="none" +else + SECURE_COMMS=$1 +fi + +echo "SECURE_COMMS is ${SECURE_COMMS}" + echo "Installing Go..." installGolang echo "Installing Libvirt..." @@ -118,3 +127,21 @@ rm -f libvirt.properties echo "libvirt_uri=\"qemu+ssh://${USER}@${IP}/system?no_verify=1\"" >> libvirt.properties echo "libvirt_ssh_key_file=\"id_rsa\"" >> libvirt.properties echo "CLUSTER_NAME=\"peer-pods\"" >> libvirt.properties +KBS_IMAGE=$(./hack/yq-shim.sh '.oci.kbs.registry' ./versions.yaml) +KBS_IMAGE_TAG=$(./hack/yq-shim.sh '.oci.kbs.tag' ./versions.yaml) +[ -z ${KBS_IMAGE} ] || echo "KBS_IMAGE=\"${KBS_IMAGE}\"" >> libvirt.properties +[ -z ${KBS_IMAGE_TAG} ] || echo "KBS_IMAGE_TAG=\"${KBS_IMAGE_TAG}\"" >> libvirt.properties + +case $SECURE_COMMS in + + "withoutKbs") + echo "processing withoutKbs" + echo "SECURE_COMMS=\"true\"" >> libvirt.properties + echo "SECURE_COMMS_KBS_ADDR=\"false\"" >> libvirt.properties + echo "INITDATA=\"\"" >> libvirt.properties + ;; + + *) + echo "processing none" + ;; +esac diff --git a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go index 7afadd439..f040eb744 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go +++ b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go @@ -16,6 +16,7 @@ import ( "path/filepath" "strings" "sync" + "time" "github.com/containerd/containerd/pkg/cri/annotations" pb "github.com/kata-containers/kata-containers/src/runtime/protocols/hypervisor" @@ -27,6 +28,7 @@ import ( "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/podnetwork" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/securecomms/wnssh" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/util" + "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/util/tlsutil" provider "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers" putil "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers/util" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers/util/cloudinit" @@ -48,6 +50,24 @@ type InitData struct { Data map[string]string `toml:"data,omitempty"` } +type ServerConfig struct { + TLSConfig *tlsutil.TLSConfig + SocketPath string + PauseImage string + PodsDir string + ForwarderPort string + ProxyTimeout time.Duration + Initdata string + EnableCloudConfigVerify bool + SecureComms bool + SecureCommsInbounds string + SecureCommsOutbounds string + SecureCommsPpInbounds string + SecureCommsPpOutbounds string + SecureCommsKbsAddress string + PeerPodsLimitPerNode int +} + var logger = log.New(log.Writer(), "[adaptor/cloud] ", log.LstdFlags|log.Lmsgprefix) func (s *cloudService) addSandbox(sid sandboxID, sandbox *sandbox) error { @@ -87,15 +107,20 @@ func (s *cloudService) removeSandbox(id sandboxID) error { } func NewService(provider provider.Provider, proxyFactory proxy.Factory, workerNode podnetwork.WorkerNode, - secureComms bool, secureCommsInbounds, secureCommsOutbounds, kbsAddress, podsDir, - daemonPort, initdata, sshport string) Service { + serverConfig *ServerConfig, sshport string) Service { var err error var sshClient *wnssh.SshClient - if secureComms { - inbounds := append([]string{"KUBERNETES_PHASE:KATAAGENT:0"}, strings.Split(secureCommsInbounds, ",")...) - outbounds := append([]string{"BOTH_PHASES:KBS:" + kbsAddress}, strings.Split(secureCommsOutbounds, ",")...) - sshClient, err = wnssh.InitSshClient(inbounds, outbounds, kbsAddress, sshport) + if serverConfig.SecureComms { + inbounds := append([]string{"KUBERNETES_PHASE:KATAAGENT:0"}, strings.Split(serverConfig.SecureCommsInbounds, ",")...) + + var outbounds []string + outbounds = append(outbounds, strings.Split(serverConfig.SecureCommsOutbounds, ",")...) + if len(serverConfig.SecureCommsKbsAddress) > 0 { + outbounds = append(outbounds, "BOTH_PHASES:KBS:"+serverConfig.SecureCommsKbsAddress) + } + + sshClient, err = wnssh.InitSshClient(inbounds, outbounds, serverConfig.SecureCommsKbsAddress, sshport) if err != nil { log.Fatalf("InitSshClient %v", err) } @@ -105,9 +130,7 @@ func NewService(provider provider.Provider, proxyFactory proxy.Factory, workerNo provider: provider, proxyFactory: proxyFactory, sandboxes: map[sandboxID]*sandbox{}, - podsDir: podsDir, - daemonPort: daemonPort, - initdata: initdata, + serverConfig: serverConfig, workerNode: workerNode, sshClient: sshClient, } @@ -220,7 +243,7 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r return nil, fmt.Errorf("failed to inspect netns %s: %w", netNSPath, err) } - podDir := filepath.Join(s.podsDir, string(sid)) + podDir := filepath.Join(s.serverConfig.PodsDir, string(sid)) if err := os.MkdirAll(podDir, os.ModePerm); err != nil { return nil, fmt.Errorf("creating a pod directory: %s, %w", podDir, err) } @@ -264,6 +287,22 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r daemonConfig.TLSServerKey = string(keyPEM) } + var ci *wnssh.SshClientInstance + + if s.sshClient != nil { + var ppPrivateKey []byte + ci, ppPrivateKey = s.sshClient.InitPP(context.Background(), string(sid)) + if ci == nil { + return nil, fmt.Errorf("failed sshClient.InitPP") + } + + daemonConfig.WnPublicKey = s.sshClient.GetWnPublicKey() + daemonConfig.PpPrivateKey = ppPrivateKey + daemonConfig.SecureCommsOutbounds = s.serverConfig.SecureCommsPpOutbounds + daemonConfig.SecureCommsInbounds = s.serverConfig.SecureCommsPpInbounds + daemonConfig.SecureComms = true + } + daemonJSON, err := json.MarshalIndent(daemonConfig, "", " ") if err != nil { return nil, fmt.Errorf("generating JSON data: %w", err) @@ -304,19 +343,19 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r logger.Printf("initdata in Pod annotation: %s", initdataStr) if initdataStr == "" { - logger.Printf("initdata in pod annotation is empty, use global initdata: %s", s.initdata) - initdataStr = s.initdata + logger.Printf("initdata in pod annotation is empty, use global initdata: %s", s.serverConfig.Initdata) + initdataStr = s.serverConfig.Initdata } if initdataStr != "" { decodedBytes, err := base64.StdEncoding.DecodeString(initdataStr) if err != nil { - return nil, fmt.Errorf("Error base64 decode initdata: %w", err) + return nil, fmt.Errorf("error base64 decode initdata: %w", err) } initdata := InitData{} err = toml.Unmarshal(decodedBytes, &initdata) if err != nil { - return nil, fmt.Errorf("Error unmarshalling initdata: %w", err) + return nil, fmt.Errorf("error unmarshalling initdata: %w", err) } cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, cloudinit.WriteFile{ @@ -326,14 +365,15 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r } sandbox := &sandbox{ - id: sid, - podName: pod, - podNamespace: namespace, - netNSPath: netNSPath, - agentProxy: agentProxy, - podNetwork: podNetworkConfig, - cloudConfig: cloudConfig, - spec: vmSpec, + id: sid, + podName: pod, + podNamespace: namespace, + netNSPath: netNSPath, + agentProxy: agentProxy, + podNetwork: podNetworkConfig, + cloudConfig: cloudConfig, + spec: vmSpec, + sshClientInst: ci, } if err := s.addSandbox(sid, sandbox); err != nil { @@ -377,24 +417,16 @@ func (s *cloudService) StartVM(ctx context.Context, req *pb.StartVMRequest) (res logger.Printf("created an instance %s for sandbox %s", instance.Name, sid) instanceIP := instance.IPs[0].String() - forwarderPort := s.daemonPort + forwarderPort := s.serverConfig.ForwarderPort if s.sshClient != nil { - ci := s.sshClient.InitPP(context.Background(), string(sid), instance.IPs) - if ci == nil { - return nil, fmt.Errorf("failed sshClient.InitPP") - } - - if err := ci.Start(); err != nil { + if err := sandbox.sshClientInst.Start(instance.IPs); err != nil { return nil, fmt.Errorf("failed SshClientInstance.Start: %w", err) } // Set agentProxy instanceIP = "127.0.0.1" - forwarderPort = ci.GetPort("KATAAGENT") - - // Set ci in sandbox - sandbox.sshClientInst = ci + forwarderPort = sandbox.sshClientInst.GetPort("KATAAGENT") } if err := s.workerNode.Setup(sandbox.netNSPath, instance.IPs, sandbox.podNetwork); err != nil { diff --git a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud_test.go b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud_test.go index a8030ef72..0b829a450 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud_test.go +++ b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud_test.go @@ -117,7 +117,14 @@ func TestCloudService(t *testing.T) { podsDir: dir, } - s := NewService(&mockProvider{}, proxyFactory, &mockWorkerNode{}, false, "", "", "", dir, forwarder.DefaultListenPort, "", "") + cfg := &ServerConfig{ + SecureComms: false, + PodsDir: dir, + ForwarderPort: forwarder.DefaultListenPort, + } + + // false, "", "", "", "", "", dir, forwarder.DefaultListenPort, "" + s := NewService(&mockProvider{}, proxyFactory, &mockWorkerNode{}, cfg, "") assert.NotNil(t, s) @@ -160,7 +167,12 @@ func TestCloudServiceWithSecureComms(t *testing.T) { // create a podvm gkc := test.NewGetKeyClient("9019") ctx2, cancel := context.WithCancel(context.Background()) - sshServer := ppssh.NewSshServer([]string{"BOTH_PHASES:KBS:9019"}, []string{"KUBERNETES_PHASE:KATAAGENT:127.0.0.1:7111"}, ppssh.GetSecret(gkc.GetKey), sshport) + + ppSecrets := ppssh.NewPpSecrets(ppssh.GetSecret(gkc.GetKey)) + ppSecrets.AddKey(ppssh.WN_PUBLIC_KEY) + ppSecrets.AddKey(ppssh.PP_PRIVATE_KEY) + + sshServer := ppssh.NewSshServer([]string{"BOTH_PHASES:KBS:9019"}, []string{"KUBERNETES_PHASE:KATAAGENT:127.0.0.1:7111"}, ppSecrets, sshport) _ = sshServer.Start(ctx2) defer func() { cancel() @@ -172,7 +184,14 @@ func TestCloudServiceWithSecureComms(t *testing.T) { podsDir: dir, } - s := NewService(&mockProvider{}, proxyFactory, &mockWorkerNode{}, true, "", "", "127.0.0.1:9009", dir, forwarder.DefaultListenPort, "", sshport) + cfg := &ServerConfig{ + SecureComms: true, + PodsDir: dir, + ForwarderPort: forwarder.DefaultListenPort, + SecureCommsKbsAddress: "127.0.0.1:9009", + } + + s := NewService(&mockProvider{}, proxyFactory, &mockWorkerNode{}, cfg, sshport) assert.NotNil(t, s) diff --git a/src/cloud-api-adaptor/pkg/adaptor/cloud/types.go b/src/cloud-api-adaptor/pkg/adaptor/cloud/types.go index 6d71b6933..e91fff077 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/cloud/types.go +++ b/src/cloud-api-adaptor/pkg/adaptor/cloud/types.go @@ -31,12 +31,10 @@ type cloudService struct { workerNode podnetwork.WorkerNode sandboxes map[sandboxID]*sandbox cond *sync.Cond - podsDir string - daemonPort string mutex sync.Mutex ppService *k8sops.PeerPodService - initdata string sshClient *wnssh.SshClient + serverConfig *ServerConfig } type sandboxID string diff --git a/src/cloud-api-adaptor/pkg/adaptor/server.go b/src/cloud-api-adaptor/pkg/adaptor/server.go index 5539777a5..0eae22fef 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/server.go +++ b/src/cloud-api-adaptor/pkg/adaptor/server.go @@ -10,7 +10,6 @@ import ( "os" "path/filepath" "sync" - "time" "github.com/containerd/ttrpc" pbHypervisor "github.com/kata-containers/kata-containers/src/runtime/protocols/hypervisor" @@ -21,7 +20,6 @@ import ( "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/vminfo" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/podnetwork" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/securecomms/sshutil" - "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/util/tlsutil" pbPodVMInfo "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/proto/podvminfo" provider "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers" ) @@ -33,22 +31,6 @@ const ( DefaultPodsDir = "/run/peerpod/pods" ) -type ServerConfig struct { - TLSConfig *tlsutil.TLSConfig - SocketPath string - PauseImage string - PodsDir string - ForwarderPort string - ProxyTimeout time.Duration - Initdata string - EnableCloudConfigVerify bool - SecureComms bool - SecureCommsInbounds string - SecureCommsOutbounds string - SecureCommsKbsAddress string - PeerPodsLimitPerNode int -} - type Server interface { Start(ctx context.Context) error Shutdown() error @@ -68,14 +50,12 @@ type server struct { PeerPodsLimitPerNode int } -func NewServer(provider provider.Provider, cfg *ServerConfig, workerNode podnetwork.WorkerNode) Server { +func NewServer(provider provider.Provider, cfg *cloud.ServerConfig, workerNode podnetwork.WorkerNode) Server { logger.Printf("server config: %#v", cfg) agentFactory := proxy.NewFactory(cfg.PauseImage, cfg.TLSConfig, cfg.ProxyTimeout) - cloudService := cloud.NewService(provider, agentFactory, workerNode, - cfg.SecureComms, cfg.SecureCommsInbounds, cfg.SecureCommsOutbounds, - cfg.SecureCommsKbsAddress, cfg.PodsDir, cfg.ForwarderPort, cfg.Initdata, sshutil.SSHPORT) + cloudService := cloud.NewService(provider, agentFactory, workerNode, cfg, sshutil.SSHPORT) vmInfoService := vminfo.NewService(cloudService) return &server{ diff --git a/src/cloud-api-adaptor/pkg/adaptor/server_test.go b/src/cloud-api-adaptor/pkg/adaptor/server_test.go index baa9fc095..da51ae12b 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/server_test.go +++ b/src/cloud-api-adaptor/pkg/adaptor/server_test.go @@ -13,6 +13,7 @@ import ( "testing" "time" + "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/cloud" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/proxy" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/podnetwork/tunneler" provider "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers" @@ -150,7 +151,7 @@ func newServer(t *testing.T, socketPath, podsDir string) Server { port := startAgentServer(t) provider := &mockProvider{} - serverConfig := &ServerConfig{ + serverConfig := &cloud.ServerConfig{ SocketPath: socketPath, PodsDir: podsDir, ForwarderPort: port, diff --git a/src/cloud-api-adaptor/pkg/adaptor/shim_test.go b/src/cloud-api-adaptor/pkg/adaptor/shim_test.go index 688973cc4..264ee587c 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/shim_test.go +++ b/src/cloud-api-adaptor/pkg/adaptor/shim_test.go @@ -17,6 +17,7 @@ import ( "testing" "time" + "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/cloud" daemon "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/forwarder" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/forwarder/interceptor" "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/podnetwork" @@ -86,7 +87,7 @@ func TestShim(t *testing.T) { workerNode = podnetwork.NewWorkerNode(t, "", 0, 0) } - serverConfig := &ServerConfig{ + serverConfig := &cloud.ServerConfig{ SocketPath: helperSocketPath, PodsDir: podsDir, ForwarderPort: port, diff --git a/src/cloud-api-adaptor/pkg/forwarder/forwarder.go b/src/cloud-api-adaptor/pkg/forwarder/forwarder.go index fd8a694d0..d175cea85 100644 --- a/src/cloud-api-adaptor/pkg/forwarder/forwarder.go +++ b/src/cloud-api-adaptor/pkg/forwarder/forwarder.go @@ -42,6 +42,12 @@ type Config struct { TLSServerKey string `json:"tls-server-key,omitempty"` TLSServerCert string `json:"tls-server-cert,omitempty"` TLSClientCA string `json:"tls-client-ca,omitempty"` + + PpPrivateKey []byte `json:"sc-pp-prv,omitempty"` + WnPublicKey []byte `json:"sc-wn-pub,omitempty"` + SecureCommsInbounds string `json:"sc-inbounds,omitempty"` + SecureCommsOutbounds string `json:"sc-outbounds,omitempty"` + SecureComms bool `json:"sc,omitempty"` } type Daemon interface { diff --git a/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppsecrets.go b/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppsecrets.go index 1707f6473..48bc6221e 100644 --- a/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppsecrets.go +++ b/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppsecrets.go @@ -5,7 +5,6 @@ import ( ) type PpSecrets struct { - keys []string secrets map[string][]byte getSecret GetSecret } @@ -14,44 +13,54 @@ type GetSecret func(name string) ([]byte, error) func NewPpSecrets(getSecret GetSecret) *PpSecrets { return &PpSecrets{ - keys: []string{}, secrets: make(map[string][]byte), getSecret: getSecret, } } -func (fs *PpSecrets) AddKey(key string) { - fs.keys = append(fs.keys, key) +func (sec *PpSecrets) AddKey(key string) { + if _, ok := sec.secrets[key]; ok { + return + } + sec.secrets[key] = nil } -func (fs *PpSecrets) GetKey(key string) []byte { - return fs.secrets[key] +func (sec *PpSecrets) GetKey(key string) []byte { + return sec.secrets[key] } -func (fs *PpSecrets) Go() { - sleeptime := time.Duration(1) +func (sec *PpSecrets) SetKey(key string, keydata []byte) { + sec.secrets[key] = keydata +} - for len(fs.keys) > 0 { - key := fs.keys[0] - logger.Printf("PpSecrets obtaining key %s", key) +func (sec *PpSecrets) Go() { + sleeptime := time.Duration(1) - data, err := fs.getSecret(key) - if err == nil && len(data) > 0 { - logger.Printf("PpSecrets %s success", key) - fs.secrets[key] = data - fs.keys = fs.keys[1:] + for key, keydata := range sec.secrets { + if keydata != nil { continue } - if err != nil { - logger.Printf("PpSecrets %s getSecret err: %v", key, err) - } else { - logger.Printf("PpSecrets %s getSecret returned an empty secret", key) - } + logger.Printf("PpSecrets obtaining key %s", key) + + // loop until we get a valid key + for { + keydata, err := sec.getSecret(key) + if err == nil && len(keydata) > 0 { + logger.Printf("PpSecrets %s success", key) + sec.secrets[key] = keydata + break + } + if err != nil { + logger.Printf("PpSecrets %s getSecret err: %v", key, err) + } else { + logger.Printf("PpSecrets %s getSecret returned an empty secret", key) + } - time.Sleep(sleeptime * time.Second) - sleeptime *= 2 - if sleeptime > 30 { - sleeptime = 30 + time.Sleep(sleeptime * time.Second) + sleeptime *= 2 + if sleeptime > 30 { + sleeptime = 30 + } } } } diff --git a/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppssh.go b/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppssh.go index d13a24383..08feb291d 100644 --- a/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppssh.go +++ b/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppssh.go @@ -30,7 +30,7 @@ type SshServer struct { outbounds sshproxy.Outbounds wg sync.WaitGroup readyCh chan struct{} - getSecret GetSecret + ppSecrets *PpSecrets sshport string listener net.Listener ctx context.Context @@ -42,9 +42,9 @@ type SshServer struct { // Structure of an inbound tag: "::" // Structure of an outbound tag: ":::" // Phase may be "A" (Attestation), "K" (Kubernetes), or "B" (Both) -func NewSshServer(inbound_strings, outbounds_strings []string, getSecret GetSecret, sshport string) *SshServer { +func NewSshServer(inbound_strings, outbounds_strings []string, ppSecrets *PpSecrets, sshport string) *SshServer { s := &SshServer{ - getSecret: getSecret, + ppSecrets: ppSecrets, sshport: sshport, readyCh: make(chan struct{}), } @@ -119,11 +119,8 @@ func (s *SshServer) attestationPhase() *ssh.ServerConfig { for ctx.Err() == nil { logger.Printf("Attestation phase: getting keys from KBS\n") - ppSecrets := NewPpSecrets(s.getSecret) - ppSecrets.AddKey(WN_PUBLIC_KEY) - ppSecrets.AddKey(PP_PRIVATE_KEY) - ppSecrets.Go() // wait for the keys - config, err := initKubernetesPhaseSshConfig(ppSecrets) + s.ppSecrets.Go() // wait for the keys + config, err := initKubernetesPhaseSshConfig(s.ppSecrets) if err == nil { logger.Printf("Attestation phase: InitKubernetesPhaseSshConfig is ready\n") peer.Upgrade() diff --git a/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppssh_test.go b/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppssh_test.go index c6746a8e3..bee6f6b6d 100644 --- a/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppssh_test.go +++ b/src/cloud-api-adaptor/pkg/securecomms/ppssh/ppssh_test.go @@ -183,7 +183,12 @@ func TestPpssh(t *testing.T) { // Forwarder Initialization ctx := context.Background() ctx, cancel := context.WithCancel(ctx) - sshServer := NewSshServer([]string{"BOTH_PHASES:KBS:9002"}, []string{"KUBERNETES_PHASE:ABC:127.0.0.1:7105"}, GetSecret(getKey), sshport) + + ppSecrets := NewPpSecrets(GetSecret(getKey)) + ppSecrets.AddKey(WN_PUBLIC_KEY) + ppSecrets.AddKey(PP_PRIVATE_KEY) + + sshServer := NewSshServer([]string{"BOTH_PHASES:KBS:9002"}, []string{"KUBERNETES_PHASE:ABC:127.0.0.1:7105"}, ppSecrets, sshport) _ = sshServer.Start(ctx) clientSshPeer, conn := getAttestationClient(t, sshport) clientSshPeer.AddTags(inbounds, outbounds) diff --git a/src/cloud-api-adaptor/pkg/securecomms/wnssh/wnssh.go b/src/cloud-api-adaptor/pkg/securecomms/wnssh/wnssh.go index d252d609f..87a962805 100644 --- a/src/cloud-api-adaptor/pkg/securecomms/wnssh/wnssh.go +++ b/src/cloud-api-adaptor/pkg/securecomms/wnssh/wnssh.go @@ -25,6 +25,7 @@ type SshClient struct { inboundStrings []string outboundStrings []string sshport string + wnPublicKey []byte } type SshClientInstance struct { @@ -72,22 +73,25 @@ func InitSshClient(inbound_strings, outbound_strings []string, kbsAddress string return nil, fmt.Errorf("unable to parse private key: %v", err) } - kbscPrivateKey, _, err := kubemgr.KubeMgr.ReadSecret(sshutil.KBS_CLIENT_SECRET) - if err != nil { - return nil, fmt.Errorf("failed to read KBS client secret: %w", err) - } + var kc *KbsClient + if len(kbsAddress) > 0 { + kbscPrivateKey, _, err := kubemgr.KubeMgr.ReadSecret(sshutil.KBS_CLIENT_SECRET) + if err != nil { + return nil, fmt.Errorf("failed to read KBS client secret: %w", err) + } - kc := InitKbsClient(kbsAddress) - err = kc.SetPemSecret(kbscPrivateKey) - if err != nil { - return nil, fmt.Errorf("KbsClient - %v", err) - } + kc = InitKbsClient(kbsAddress) + err = kc.SetPemSecret(kbscPrivateKey) + if err != nil { + return nil, fmt.Errorf("KbsClient - %v", err) + } - wnSecretPath := "default/sshclient/publicKey" - logger.Printf("Updating KBS with secret for: %s", wnSecretPath) - err = kc.PostResource(wnSecretPath, wnPublicKey) - if err != nil { - return nil, fmt.Errorf("failed to PostResource WN Secret: %v", err) + wnSecretPath := "default/sshclient/publicKey" + logger.Printf("Updating KBS with secret for: %s", wnSecretPath) + err = kc.PostResource(wnSecretPath, wnPublicKey) + if err != nil { + return nil, fmt.Errorf("failed to PostResource WN Secret: %v", err) + } } sshClient := &SshClient{ @@ -96,6 +100,7 @@ func InitSshClient(inbound_strings, outbound_strings []string, kbsAddress string inboundStrings: inbound_strings, outboundStrings: outbound_strings, sshport: sshport, + wnPublicKey: wnPublicKey, } return sshClient, nil @@ -123,9 +128,13 @@ func (ci *SshClientInstance) DisconnectPP(sid string) { kubemgr.KubeMgr.DeleteSecret(PpSecretName(sid)) } -func (c *SshClient) InitPP(ctx context.Context, sid string, ipAddr []netip.Addr) *SshClientInstance { +func (c *SshClient) GetWnPublicKey() []byte { + return c.wnPublicKey +} + +func (c *SshClient) InitPP(ctx context.Context, sid string) (ci *SshClientInstance, ppPrivateKey []byte) { // Create peerPod Secret named peerPodId - var ppPublicKey, ppPrivateKey []byte + var ppPublicKey []byte var err error var kubernetesPhase bool @@ -136,43 +145,39 @@ func (c *SshClient) InitPP(ctx context.Context, sid string, ipAddr []netip.Addr) ppPrivateKey, ppPublicKey, err = kubemgr.KubeMgr.CreateSecret(PpSecretName(sid)) if err != nil { logger.Printf("Failed to create PP secret: %v", err) - return nil + return } } else { // we already have a store secret for this PP kubernetesPhase = true } - // >>> Update the KBS about the SID's Secret !!! <<< - sidSecretPath := fmt.Sprintf("default/pp-%s/privateKey", sid) - logger.Printf("Updating KBS with secret for: %s", sidSecretPath) - err = c.kc.PostResource(sidSecretPath, ppPrivateKey) - if err != nil { - logger.Printf("Failed to PostResource PP Secret: %v", err) - return nil - } + if c.kc != nil { + // >>> Update the KBS about the SID's Secret !!! <<< + sidSecretPath := fmt.Sprintf("default/pp-%s/privateKey", sid) + logger.Printf("Updating KBS with secret for: %s", sidSecretPath) + err = c.kc.PostResource(sidSecretPath, ppPrivateKey) + if err != nil { + logger.Printf("Failed to PostResource PP Secret: %v", err) + return + } + } var ppSshPublicKeyBytes []byte if len(ppPublicKey) > 0 { ppSshPublicKey, _, _, _, err := ssh.ParseAuthorizedKey(ppPublicKey) if err != nil { logger.Printf("Unable to ParseAuthorizedKey serverPublicKey: %v", err) - return nil + return } ppSshPublicKeyBytes = ppSshPublicKey.Marshal() } - ppAddr := make([]string, len(ipAddr)) - for i, ip := range ipAddr { - ppAddr[i] = ip.String() + ":" + c.sshport - } - ctx, cancel := context.WithCancel(ctx) - ci := &SshClientInstance{ + ci = &SshClientInstance{ sid: sid, ppPublicKey: ppSshPublicKeyBytes, - ppAddr: ppAddr, sshClient: c, ctx: ctx, cancel: cancel, @@ -188,10 +193,17 @@ func (c *SshClient) InitPP(ctx context.Context, sid string, ipAddr []netip.Addr) logger.Fatalf("Failed to parse outbound tag %v: %v", c.outboundStrings, err) } - return ci + return } -func (ci *SshClientInstance) Start() error { +func (ci *SshClientInstance) Start(ipAddr []netip.Addr) error { + ppAddr := make([]string, len(ipAddr)) + for i, ip := range ipAddr { + ppAddr[i] = ip.String() + ":" + ci.sshClient.sshport + } + + ci.ppAddr = ppAddr + if !ci.kubernetesPhase { // Attestation phase logger.Println("Attestation phase: starting") diff --git a/src/cloud-api-adaptor/pkg/securecomms/wnssh/wnssh_test.go b/src/cloud-api-adaptor/pkg/securecomms/wnssh/wnssh_test.go index 64b6d3304..ecfb3f440 100644 --- a/src/cloud-api-adaptor/pkg/securecomms/wnssh/wnssh_test.go +++ b/src/cloud-api-adaptor/pkg/securecomms/wnssh/wnssh_test.go @@ -32,7 +32,7 @@ func TestSshProxyReverseKBS(t *testing.T) { ////////// CAA StartVM ipAddr, _ := netip.ParseAddr("127.0.0.1") // ipAddr of the VM ipAddrs := []netip.Addr{ipAddr} - ci := sshClient.InitPP(context.Background(), "sid", ipAddrs) + ci, _ := sshClient.InitPP(context.Background(), "sid") if ci == nil { log.Fatalf("failed InitiatePeerPodTunnel") } @@ -46,12 +46,17 @@ func TestSshProxyReverseKBS(t *testing.T) { // create a podvm gkc := test.NewGetKeyClient("7030") ctx2, cancel2 := context.WithCancel(context.Background()) - sshServer := ppssh.NewSshServer([]string{"BOTH_PHASES:KBS:7030", "KUBERNETES_PHASE:KUBEAPI:16443", "KUBERNETES_PHASE:DNS:9053"}, []string{"KUBERNETES_PHASE:KATAAGENT:127.0.0.1:7121"}, ppssh.GetSecret(gkc.GetKey), sshport) + + ppSecrets := ppssh.NewPpSecrets(ppssh.GetSecret(gkc.GetKey)) + ppSecrets.AddKey(ppssh.WN_PUBLIC_KEY) + ppSecrets.AddKey(ppssh.PP_PRIVATE_KEY) + + sshServer := ppssh.NewSshServer([]string{"BOTH_PHASES:KBS:7030", "KUBERNETES_PHASE:KUBEAPI:16443", "KUBERNETES_PHASE:DNS:9053"}, []string{"KUBERNETES_PHASE:KATAAGENT:127.0.0.1:7121"}, ppSecrets, sshport) _ = sshServer.Start(ctx2) // Forwarder Initialization - if err := ci.Start(); err != nil { + if err := ci.Start(ipAddrs); err != nil { log.Fatalf("failed ci.Start: %s", err) } diff --git a/src/cloud-api-adaptor/test/e2e/main_test.go b/src/cloud-api-adaptor/test/e2e/main_test.go index 330b5d591..4a23bce79 100644 --- a/src/cloud-api-adaptor/test/e2e/main_test.go +++ b/src/cloud-api-adaptor/test/e2e/main_test.go @@ -5,7 +5,9 @@ package e2e import ( "context" + "fmt" "os" + "os/exec" "testing" pv "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/test/provisioner" @@ -58,6 +60,12 @@ func TestMain(m *testing.M) { // unless it is running with an in-cluster configuration. testEnv = env.New() + // TEST_CAA_LOG is an option variable which specifies whether the CAA Log should be + // presented at the end of the test or not + shouldCaaLog := true + if os.Getenv("TEST_CAA_LOG") == "no" { + shouldCaaLog = false + } // TEST_TEARDOWN is an option variable which specifies whether the teardown code path // should run or not. shouldTeardown := true @@ -192,6 +200,18 @@ func TestMain(m *testing.M) { // Run *once* after the tests. testEnv.Finish(func(ctx context.Context, cfg *envconf.Config) (context.Context, error) { + + if shouldCaaLog { + fmt.Printf("CAA LOG: STARTING\n") + caaLogTailCmd := exec.Command("kubectl", "logs", "daemonset/cloud-api-adaptor-daemonset", "-n", "confidential-containers-system") + caaLogTailCmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG="+cfg.KubeconfigFile())) + caaLogTailCmd.Stdout = os.Stdout + caaLogTailCmd.Stderr = os.Stderr + if err := caaLogTailCmd.Run(); err != nil { + return ctx, err + } + } + if !shouldTeardown { return ctx, nil } diff --git a/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go b/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go index 5badefa38..6e8a150a5 100644 --- a/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go +++ b/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go @@ -21,14 +21,17 @@ import ( // LibvirtProvisioner implements the CloudProvisioner interface for Libvirt. type LibvirtProvisioner struct { - conn *libvirt.Connect // Libvirt connection - network string // Network name - ssh_key_file string // SSH key file used to connect to Libvirt - storage string // Storage pool name - uri string // Libvirt URI - wd string // libvirt's directory path on this repository - volumeName string // Podvm volume name - clusterName string // Cluster name + conn *libvirt.Connect // Libvirt connection + network string // Network name + ssh_key_file string // SSH key file used to connect to Libvirt + storage string // Storage pool name + uri string // Libvirt URI + wd string // libvirt's directory path on this repository + volumeName string // Podvm volume name + clusterName string // Cluster name + secure_comms string // Activate CAA SECURE_COMMS + secure_comms_kbs_addr string // KBS URL or "false" + initdata string // InitData } // LibvirtInstallOverlay implements the InstallOverlay interface @@ -80,16 +83,34 @@ func NewLibvirtProvisioner(properties map[string]string) (pv.CloudProvisioner, e clusterName = properties["cluster_name"] } + secure_comms := "false" + if properties["SECURE_COMMS"] != "" { + secure_comms = properties["SECURE_COMMS"] + } + + secure_comms_kbs_addr := "" + if properties["SECURE_COMMS_KBS_ADDR"] != "" { + secure_comms_kbs_addr = properties["SECURE_COMMS_KBS_ADDR"] + } + + initdata := "" + if properties["INITDATA"] != "" { + initdata = properties["INITDATA"] + } + // TODO: Check network and storage are not nil? return &LibvirtProvisioner{ - conn: conn, - network: network, - ssh_key_file: ssh_key_file, - storage: storage, - uri: uri, - wd: wd, - volumeName: vol_name, - clusterName: clusterName, + conn: conn, + network: network, + ssh_key_file: ssh_key_file, + storage: storage, + uri: uri, + wd: wd, + volumeName: vol_name, + clusterName: clusterName, + secure_comms: secure_comms, + secure_comms_kbs_addr: secure_comms_kbs_addr, + initdata: initdata, }, nil } @@ -189,11 +210,14 @@ func (l *LibvirtProvisioner) DeleteVPC(ctx context.Context, cfg *envconf.Config) func (l *LibvirtProvisioner) GetProperties(ctx context.Context, cfg *envconf.Config) map[string]string { return map[string]string{ - "network": l.network, - "podvm_volume": l.volumeName, - "ssh_key_file": l.ssh_key_file, - "storage": l.storage, - "uri": l.uri, + "network": l.network, + "podvm_volume": l.volumeName, + "ssh_key_file": l.ssh_key_file, + "storage": l.storage, + "uri": l.uri, + "SECURE_COMMS": l.secure_comms, + "SECURE_COMMS_KBS_ADDR": l.secure_comms_kbs_addr, + "INITDATA": l.initdata, } } @@ -296,17 +320,20 @@ func (lio *LibvirtInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config, // Mapping the internal properties to ConfigMapGenerator properties and their default values. mapProps := map[string][2]string{ - "network": {"default", "LIBVIRT_NET"}, - "storage": {"default", "LIBVIRT_POOL"}, - "pause_image": {"", "PAUSE_IMAGE"}, - "podvm_volume": {"", "LIBVIRT_VOL_NAME"}, - "uri": {"qemu+ssh://root@192.168.122.1/system?no_verify=1", "LIBVIRT_URI"}, - "vxlan_port": {"", "VXLAN_PORT"}, - "INITDATA": {"", "INITDATA"}, + "network": {"default", "LIBVIRT_NET"}, + "storage": {"default", "LIBVIRT_POOL"}, + "pause_image": {"", "PAUSE_IMAGE"}, + "podvm_volume": {"", "LIBVIRT_VOL_NAME"}, + "uri": {"qemu+ssh://root@192.168.122.1/system?no_verify=1", "LIBVIRT_URI"}, + "vxlan_port": {"", "VXLAN_PORT"}, + "INITDATA": {"", "INITDATA"}, + "SECURE_COMMS": {"", "SECURE_COMMS"}, + "SECURE_COMMS_KBS_ADDR": {"", "SECURE_COMMS_KBS_ADDR"}, } for k, v := range mapProps { if properties[k] != v[0] { + fmt.Printf("TESTONLY setting %s: %s\n", v[1], properties[k]) if err = lio.Overlay.SetKustomizeConfigMapGeneratorLiteral("peer-pods-cm", v[1], properties[k]); err != nil { return err diff --git a/src/cloud-api-adaptor/test/provisioner/provision.go b/src/cloud-api-adaptor/test/provisioner/provision.go index 7001c2b4b..4e098cae7 100644 --- a/src/cloud-api-adaptor/test/provisioner/provision.go +++ b/src/cloud-api-adaptor/test/provisioner/provision.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "time" "github.com/BurntSushi/toml" @@ -220,7 +221,6 @@ func (p *CloudAPIAdaptor) Delete(ctx context.Context, cfg *envconf.Config) error wait.WithTimeout(time.Minute*1)); err != nil { return err } - return nil } @@ -295,6 +295,16 @@ func (p *CloudAPIAdaptor) Deploy(ctx context.Context, cfg *envconf.Config, props } } + fmt.Printf("CAA ConfigMap:\n") + caaConfigMap := exec.Command("kubectl", "get", "cm", "peer-pods-cm", "-n", "confidential-containers-system", "-o", "yaml") + caaConfigMap.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG="+cfg.KubeconfigFile())) + caaConfigMapOut := new(strings.Builder) + caaConfigMap.Stdout = caaConfigMapOut + if err = caaConfigMap.Run(); err != nil { + return err + } + fmt.Printf("%v, CAA ConfigMap: \n%s", caaConfigMap, caaConfigMapOut.String()) + fmt.Printf("Wait for the %s runtimeclass be created\n", p.runtimeClass.GetName()) if err = wait.For(conditions.New(resources).ResourcesFound(&nodev1.RuntimeClassList{Items: []nodev1.RuntimeClass{*p.runtimeClass}}), wait.WithTimeout(time.Second*60)); err != nil { diff --git a/src/cloud-api-adaptor/test/securecomms/ppssh.go b/src/cloud-api-adaptor/test/securecomms/ppssh.go index b9a15d82d..337f3f39e 100644 --- a/src/cloud-api-adaptor/test/securecomms/ppssh.go +++ b/src/cloud-api-adaptor/test/securecomms/ppssh.go @@ -20,7 +20,12 @@ func PP() { // Forwarder Initialization ctx := context.Background() ctx, cancel := context.WithCancel(ctx) - sshServer := ppssh.NewSshServer([]string{"BOTH_PHASES:KBS:7000", "KUBERNETES_PHASE:KUBEAPI:16443", "KUBERNETES_PHASE:DNS:9053"}, []string{"KUBERNETES_PHASE:KATAAGENT:127.0.0.1:7131"}, ppssh.GetSecret(getKey), sshutil.SSHPORT) + + ppSecrets := ppssh.NewPpSecrets(ppssh.GetSecret(getKey)) + ppSecrets.AddKey(ppssh.WN_PUBLIC_KEY) + ppSecrets.AddKey(ppssh.PP_PRIVATE_KEY) + + sshServer := ppssh.NewSshServer([]string{"BOTH_PHASES:KBS:7000", "KUBERNETES_PHASE:KUBEAPI:16443", "KUBERNETES_PHASE:DNS:9053"}, []string{"KUBERNETES_PHASE:KATAAGENT:127.0.0.1:7131"}, ppSecrets, sshutil.SSHPORT) _ = sshServer.Start(ctx) time.Sleep(1 * time.Minute) cancel() diff --git a/src/cloud-api-adaptor/test/securecomms/wnssh.go b/src/cloud-api-adaptor/test/securecomms/wnssh.go index 026125e03..406d78c48 100644 --- a/src/cloud-api-adaptor/test/securecomms/wnssh.go +++ b/src/cloud-api-adaptor/test/securecomms/wnssh.go @@ -30,7 +30,7 @@ func WN() bool { ipAddr, _ := netip.ParseAddr("127.0.0.1") // ipAddr of the VM ipAddrs := []netip.Addr{ipAddr} ctx := context.Background() - ci := sshClient.InitPP(ctx, "sid", ipAddrs) + ci, _ := sshClient.InitPP(ctx, "sid") if ci == nil { log.Fatalf("failed InitiatePeerPodTunnel") } @@ -41,7 +41,7 @@ func WN() bool { log.Fatalf("failed find port") } - if err := ci.Start(); err != nil { + if err := ci.Start(ipAddrs); err != nil { log.Fatalf("failed ci.Start: %s", err) }