diff --git a/config/e2e/kustomization.yaml b/config/e2e/kustomization.yaml index 8ed4cd78173..12a7cef8398 100644 --- a/config/e2e/kustomization.yaml +++ b/config/e2e/kustomization.yaml @@ -9,9 +9,15 @@ resources: apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization patches: -- path: patch_operator.yml +- path: patch_operator.yaml target: group: apps kind: Deployment name: keda-operator version: v1 +- path: patch_rbac.yaml + target: + group: rbac.authorization.k8s.io + kind: ClusterRole + name: keda-operator + version: v1 diff --git a/config/e2e/patch_operator.yml b/config/e2e/patch_operator.yaml similarity index 100% rename from config/e2e/patch_operator.yml rename to config/e2e/patch_operator.yaml diff --git a/config/e2e/patch_rbac.yaml b/config/e2e/patch_rbac.yaml new file mode 100644 index 00000000000..3ea11919d92 --- /dev/null +++ b/config/e2e/patch_rbac.yaml @@ -0,0 +1,9 @@ + - op: add + path: /rules/- + value: + apiGroups: + - "" + resources: + - serviceaccounts/token + verbs: + - create diff --git a/pkg/scaling/resolver/hashicorpvault_handler.go b/pkg/scaling/resolver/hashicorpvault_handler.go index f513909d35f..f109c75ca9a 100644 --- a/pkg/scaling/resolver/hashicorpvault_handler.go +++ b/pkg/scaling/resolver/hashicorpvault_handler.go @@ -19,22 +19,19 @@ package resolver import ( "context" "encoding/json" - "errors" "fmt" "os" "strings" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "github.com/go-logr/logr" vaultapi "github.com/hashicorp/vault/api" - "k8s.io/apimachinery/pkg/types" - - kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" + "github.com/pkg/errors" authenticationv1 "k8s.io/api/authentication/v1" - corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + + kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" ) // HashicorpVaultHandler is specification of Hashi Corp Vault @@ -141,19 +138,15 @@ func (vh *HashicorpVaultHandler) token(client *vaultapi.Client) (string, error) secret := &corev1.Secret{} if err = vh.k8sClient.Get(context.Background(), saName, sa); err != nil { - if apierrors.IsNotFound(err) { - return token, errors.New(fmt.Sprintf("Failed to retreive service account name: %s namespace: %s", saName.Name, saName.Namespace)) - } + return token, errors.Wrap(err, fmt.Sprintf("Failed to retrieve service account name: %s namespace: %s", saName.Name, saName.Namespace)) } if len(sa.Secrets) > 0 { - //using legacy service account secrets + // using legacy service account secrets secretName := types.NamespacedName{Name: sa.Secrets[0].Name, Namespace: vh.namespace} if err = vh.k8sClient.Get(context.Background(), secretName, secret); err != nil { - if apierrors.IsNotFound(err) { - return token, errors.New(fmt.Sprintf("Failed to retreive secret for service account name: %s namespace: %s", secretName.Name, secretName.Namespace)) - } + return token, errors.Wrap(err, fmt.Sprintf("Failed to retrieve secret for service account name: %s namespace: %s", secretName.Name, secretName.Namespace)) } jwt = secret.Data["token"] @@ -171,12 +164,11 @@ func (vh *HashicorpVaultHandler) token(client *vaultapi.Client) (string, error) } if err := vh.k8sClient.SubResource("token").Create(context.Background(), sa, tokenRequest); err != nil { - return token, errors.New(fmt.Sprintf("Failed to create token for service account name: %s namespace: %s", saName.Name, saName.Namespace)) + return token, errors.Wrap(err, fmt.Sprintf("Failed to create token for service account name: %s namespace: %s", saName.Name, saName.Namespace)) } jwt = []byte(tokenRequest.Status.Token) } - } else if len(vh.vault.Credential.ServiceAccount) != 0 { // Get the JWT from POD jwt, err = os.ReadFile(vh.vault.Credential.ServiceAccount) diff --git a/pkg/scaling/resolver/hashicorpvault_handler_test.go b/pkg/scaling/resolver/hashicorpvault_handler_test.go index 0e672df2459..e5c7c930c2f 100644 --- a/pkg/scaling/resolver/hashicorpvault_handler_test.go +++ b/pkg/scaling/resolver/hashicorpvault_handler_test.go @@ -29,14 +29,14 @@ import ( vaultapi "github.com/hashicorp/vault/api" "github.com/stretchr/testify/assert" "go.uber.org/mock/gomock" + authenticationv1 "k8s.io/api/authentication/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" "github.com/kedacore/keda/v2/pkg/mock/mock_client" - authenticationv1 "k8s.io/api/authentication/v1" - "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -78,11 +78,6 @@ var ( "test": kedaSecretValue, "array": []string{kedaSecretValue}, } - kubernetesAuthDataKeda = map[string]interface{}{ - "auth": map[string]interface{}{ - "client_token": vaultTestToken, - }, - } ) type pkiRequestTestData struct { diff --git a/tests/secret-providers/hashicorp_vault/hashicorp_vault_test.go b/tests/secret-providers/hashicorp_vault/hashicorp_vault_test.go index cd135e50d4a..a81fe987eb3 100644 --- a/tests/secret-providers/hashicorp_vault/hashicorp_vault_test.go +++ b/tests/secret-providers/hashicorp_vault/hashicorp_vault_test.go @@ -70,6 +70,8 @@ type templateData struct { MonitoredAppName string PrometheusServerName string VaultPkiCommonName string + VaultRole string + VaultServiceAccountName string } const ( @@ -128,9 +130,12 @@ metadata: spec: hashiCorpVault: address: http://vault.{{.VaultNamespace}}:8200 - authentication: token + authentication: {{.HashiCorpAuthentication}} + role: {{.VaultRole}} + mount: kubernetes credential: token: {{.HashiCorpToken}} + serviceAccountName: {{.VaultServiceAccountName}} secrets: - parameter: connection key: connectionString @@ -413,6 +418,13 @@ spec: pkiPolicyTemplate = `path "pki*" { capabilities = [ "create", "read", "update", "delete", "list", "sudo" ] }` + + secretReadPolicyTemplate = `path "secret/data/keda" { + capabilities = ["read"] +} +path "secret/metadata/keda" { + capabilities = ["read", "list"] +}` ) func TestPkiSecretsEngine(t *testing.T) { @@ -432,7 +444,7 @@ func TestPkiSecretsEngine(t *testing.T) { // Create kubernetes resources kc := GetKubernetesClient(t) useKubernetesAuth := test.authentication == "kubernetes" - hashiCorpToken, promPkiData := setupHashiCorpVault(t, kc, 2, useKubernetesAuth, true) + hashiCorpToken, promPkiData := setupHashiCorpVault(t, kc, 2, useKubernetesAuth, true, false) prometheus.Install(t, kc, prometheusServerName, testNamespace, promPkiData) // Create kubernetes resources for testing @@ -460,16 +472,29 @@ func TestSecretsEngine(t *testing.T) { name string vaultEngineVersion uint vaultSecretPath string + useKubernetesAuth bool + useDelegatesSAAuth bool }{ { name: "vault kv engine v1", vaultEngineVersion: 1, vaultSecretPath: "secret/keda", + useKubernetesAuth: false, + useDelegatesSAAuth: false, + }, + { + name: "vault kv engine v2", + vaultEngineVersion: 2, + vaultSecretPath: "secret/data/keda", + useKubernetesAuth: false, + useDelegatesSAAuth: false, }, { name: "vault kv engine v2", vaultEngineVersion: 2, vaultSecretPath: "secret/data/keda", + useKubernetesAuth: true, + useDelegatesSAAuth: true, }, } @@ -480,7 +505,7 @@ func TestSecretsEngine(t *testing.T) { data, postgreSQLtemplates := getPostgreSQLTemplateData() CreateKubernetesResources(t, kc, testNamespace, data, postgreSQLtemplates) - hashiCorpToken, _ := setupHashiCorpVault(t, kc, test.vaultEngineVersion, false, false) + hashiCorpToken, _ := setupHashiCorpVault(t, kc, test.vaultEngineVersion, test.useKubernetesAuth, false, test.useDelegatesSAAuth) assert.True(t, WaitForStatefulsetReplicaReadyCount(t, kc, postgreSQLStatefulSetName, testNamespace, 1, 60, 3), "replica count should be %d after 3 minutes", 1) @@ -493,8 +518,19 @@ func TestSecretsEngine(t *testing.T) { // Create kubernetes resources for testing data, templates := getTemplateData() - data.HashiCorpToken = RemoveANSI(hashiCorpToken) data.VaultSecretPath = test.vaultSecretPath + data.VaultRole = "keda" + if test.useKubernetesAuth { + data.HashiCorpAuthentication = "kubernetes" + } else { + data.HashiCorpAuthentication = "token" + data.HashiCorpToken = RemoveANSI(hashiCorpToken) + } + + if test.useDelegatesSAAuth { + data.VaultRole = "vault-delegated-sa" + data.VaultServiceAccountName = "default" + } KubectlApplyMultipleWithTemplate(t, data, templates) assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicaCount, 60, 3), @@ -548,7 +584,7 @@ func setupHashiCorpVaultPki(t *testing.T, podName string, nameSpace string) *pro return &pkiData } -func setupHashiCorpVault(t *testing.T, kc *kubernetes.Clientset, kvVersion uint, useKubernetesAuth, pki bool) (string, *prometheus.VaultPkiData) { +func setupHashiCorpVault(t *testing.T, kc *kubernetes.Clientset, kvVersion uint, useKubernetesAuth, pki, delegatedAuth bool) (string, *prometheus.VaultPkiData) { CreateNamespace(t, kc, vaultNamespace) _, err := ExecuteCommand("helm repo add hashicorp https://helm.releases.hashicorp.com") @@ -572,7 +608,7 @@ func setupHashiCorpVault(t *testing.T, kc *kubernetes.Clientset, kvVersion uint, // Enable Kubernetes auth if useKubernetesAuth { if pki { - remoteFile := "/tmp/policy.hcl" + remoteFile := "/tmp/pki_policy.hcl" KubectlCopyToPod(t, pkiPolicyTemplate, remoteFile, podName, vaultNamespace) assert.NoErrorf(t, err, "cannot create policy file in hashicorp vault - %s", err) _, _, err = ExecCommandOnSpecificPod(t, podName, vaultNamespace, fmt.Sprintf("vault policy write pkiPolicy %s", remoteFile)) @@ -584,7 +620,18 @@ func setupHashiCorpVault(t *testing.T, kc *kubernetes.Clientset, kvVersion uint, assert.NoErrorf(t, err, "cannot set kubernetes host in hashicorp vault - %s", err) _, _, err = ExecCommandOnSpecificPod(t, podName, vaultNamespace, "vault write auth/kubernetes/role/keda bound_service_account_names=keda-operator bound_service_account_namespaces=keda policies=pkiPolicy ttl=1h") assert.NoErrorf(t, err, "cannot cerate keda role in hashicorp vault - %s", err) + if delegatedAuth { + remoteFile := "/tmp/secret_read_policy.hcl" + KubectlCopyToPod(t, secretReadPolicyTemplate, remoteFile, podName, vaultNamespace) + assert.NoErrorf(t, err, "cannot create policy file in hashicorp vault - %s", err) + _, _, err = ExecCommandOnSpecificPod(t, podName, vaultNamespace, fmt.Sprintf("vault policy write secretReadPolicy %s", remoteFile)) + assert.NoErrorf(t, err, "cannot create policy in hashicorp vault - %s", err) + + _, _, err = ExecCommandOnSpecificPod(t, podName, vaultNamespace, fmt.Sprintf("vault write auth/kubernetes/role/vault-delegated-sa bound_service_account_names=default bound_service_account_namespaces=%s policies=secretReadPolicy ttl=1h", testNamespace)) + assert.NoErrorf(t, err, "cannot cerate keda role in hashicorp vault - %s", err) + } } + // Create kv secret if !pki { _, _, err = ExecCommandOnSpecificPod(t, podName, vaultNamespace, fmt.Sprintf("vault kv put secret/keda connectionString=%s", postgreSQLConnectionString))