diff --git a/api/v1alpha1/zz_generated.conversion.go b/api/v1alpha1/zz_generated.conversion.go index 89a65d03..f78d6d94 100644 --- a/api/v1alpha1/zz_generated.conversion.go +++ b/api/v1alpha1/zz_generated.conversion.go @@ -877,8 +877,8 @@ func Convert_v1alpha1_FetchConfiguration_To_v1alpha2_FetchConfiguration(in *Fetc } func autoConvert_v1alpha2_FetchConfiguration_To_v1alpha1_FetchConfiguration(in *v1alpha2.FetchConfiguration, out *FetchConfiguration, s conversion.Scope) error { + // WARNING: in.OCIConfiguration requires manual conversion: does not exist in peer-type out.URL = in.URL - // WARNING: in.OCI requires manual conversion: does not exist in peer-type out.Selector = (*metav1.LabelSelector)(unsafe.Pointer(in.Selector)) return nil } diff --git a/api/v1alpha2/provider_types.go b/api/v1alpha2/provider_types.go index 59ff6386..c4a9955b 100644 --- a/api/v1alpha2/provider_types.go +++ b/api/v1alpha2/provider_types.go @@ -210,7 +210,11 @@ type ContainerSpec struct { } // FetchConfiguration determines the way to fetch the components and metadata for the provider. +// +kubebuilder:validation:XValidation:rule="[has(self.oci), has(self.url), has(self.selector)].exists_one(x,x)", message="Must specify one and only one of {oci, url, selector}" type FetchConfiguration struct { + // OCI configurations to be used for fetching the provider’s components and metadata from an OCI artifact. + OCIConfiguration `json:",inline"` + // URL to be used for fetching the provider’s components and metadata from a remote Github repository. // For example, https://github.com/{owner}/{repository}/releases // You must set `providerSpec.Version` field for operator to pick up @@ -218,12 +222,6 @@ type FetchConfiguration struct { // +optional URL string `json:"url,omitempty"` - // OCI to be used for fetching the provider’s components and metadata from an OCI artifact. - // You must set `providerSpec.Version` field for operator to pick up desired version of the release from GitHub. - // If the providerSpec.Version is missing, latest provider version from clusterctl defaults is used. - // +optional - OCI string `json:"oci,omitempty"` - // Selector to be used for fetching provider’s components and metadata from // ConfigMaps stored inside the cluster. Each ConfigMap is expected to contain // components and metadata for a specific version only. @@ -233,6 +231,14 @@ type FetchConfiguration struct { Selector *metav1.LabelSelector `json:"selector,omitempty"` } +type OCIConfiguration struct { + // OCI to be used for fetching the provider’s components and metadata from an OCI artifact. + // You must set `providerSpec.Version` field for operator to pick up desired version of the release from GitHub. + // If the providerSpec.Version is missing, latest provider version from clusterctl defaults is used. + // +optional + OCI string `json:"oci,omitempty"` +} + // ProviderStatus defines the observed state of the Provider. type ProviderStatus struct { // Contract will contain the core provider contract that the provider is diff --git a/api/v1alpha2/zz_generated.deepcopy.go b/api/v1alpha2/zz_generated.deepcopy.go index 72695aeb..4c924acd 100644 --- a/api/v1alpha2/zz_generated.deepcopy.go +++ b/api/v1alpha2/zz_generated.deepcopy.go @@ -651,6 +651,7 @@ func (in *DeploymentSpec) DeepCopy() *DeploymentSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *FetchConfiguration) DeepCopyInto(out *FetchConfiguration) { *out = *in + out.OCIConfiguration = in.OCIConfiguration if in.Selector != nil { in, out := &in.Selector, &out.Selector *out = new(v1.LabelSelector) @@ -873,6 +874,21 @@ func (in *ManagerSpec) DeepCopy() *ManagerSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OCIConfiguration) DeepCopyInto(out *OCIConfiguration) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OCIConfiguration. +func (in *OCIConfiguration) DeepCopy() *OCIConfiguration { + if in == nil { + return nil + } + out := new(OCIConfiguration) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ProviderSpec) DeepCopyInto(out *ProviderSpec) { *out = *in diff --git a/cmd/plugin/cmd/init.go b/cmd/plugin/cmd/init.go index 12c7a1af..d5d643f5 100644 --- a/cmd/plugin/cmd/init.go +++ b/cmd/plugin/cmd/init.go @@ -453,6 +453,7 @@ func deployCAPIOperator(ctx context.Context, opts *initOptions) error { return nil } +// templateGenericProvider prepares the provider manifest based on provided provider string. func templateGenericProvider(providerType clusterctlv1.ProviderType, providerInput, defaultNamespace, configSecretName, configSecretNamespace string) (operatorv1.GenericProvider, error) { // Parse the provider string // Format is :: diff --git a/cmd/plugin/cmd/init_test.go b/cmd/plugin/cmd/init_test.go index 912c22df..d584e12f 100644 --- a/cmd/plugin/cmd/init_test.go +++ b/cmd/plugin/cmd/init_test.go @@ -265,7 +265,6 @@ func TestInitProviders(t *testing.T) { opts: &initOptions{ coreProvider: "cluster-api:capi-system:v1.8.0", infrastructureProviders: []string{ - "cluster-api:capi-system:v1.8.0", "aws:capa-operator-system", "docker:capd-operator-system", }, diff --git a/cmd/plugin/cmd/preload.go b/cmd/plugin/cmd/preload.go index 8acb6635..688858ff 100644 --- a/cmd/plugin/cmd/preload.go +++ b/cmd/plugin/cmd/preload.go @@ -24,8 +24,6 @@ import ( "github.com/spf13/cobra" corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/meta" kerrors "k8s.io/apimachinery/pkg/util/errors" "oras.land/oras-go/v2/registry/remote/auth" operatorv1 "sigs.k8s.io/cluster-api-operator/api/v1alpha2" @@ -207,30 +205,18 @@ func runPreLoad() error { configMaps = append(configMaps, configMap) } - errors := []error{} - - if !loadOpts.existing { - for _, cm := range configMaps { - out, err := yaml.Marshal(cm) - if err != nil { - return fmt.Errorf("cannot serialize provider config map: %w", err) - } - - fmt.Printf("---\n%s", string(out)) + if loadOpts.existing { + client, err := CreateKubeClient(loadOpts.kubeconfig, "") + if err != nil { + return fmt.Errorf("cannot create a client: %w", err) } - return nil - } - - client, err := CreateKubeClient(loadOpts.kubeconfig, "") - if err != nil { - return fmt.Errorf("cannot create a client: %w", err) - } + existing, err := preloadExisting(ctx, client) + if err != nil { + return err + } - for _, list := range operatorv1.ProviderLists { - maps, err := fetchProviders(ctx, client, list.(genericProviderList)) - configMaps = append(configMaps, maps...) - errors = append(errors, err) + configMaps = append(configMaps, existing...) } for _, cm := range configMaps { @@ -242,15 +228,41 @@ func runPreLoad() error { fmt.Printf("---\n%s", string(out)) } - return kerrors.NewAggregate(errors) + return nil +} + +// preloadExisting uses existing cluster kubeconfig to list providers and create configmaps with components for each provider. +func preloadExisting(ctx context.Context, cl client.Client) ([]*corev1.ConfigMap, error) { + errors := []error{} + configMaps := []*corev1.ConfigMap{} + + for _, list := range operatorv1.ProviderLists { + list, ok := list.(genericProviderList) + if !ok { + log.V(5).Info("Expected to get GenericProviderList") + continue + } + + list, ok = list.DeepCopyObject().(genericProviderList) + if !ok { + log.V(5).Info("Expected to get GenericProviderList") + continue + } + + maps, err := fetchProviders(ctx, cl, list) + configMaps = append(configMaps, maps...) + errors = append(errors, err) + } + + return configMaps, kerrors.NewAggregate(errors) } func fetchProviders(ctx context.Context, cl client.Client, providerList genericProviderList) ([]*corev1.ConfigMap, error) { configMaps := []*corev1.ConfigMap{} - if err := cl.List(ctx, providerList, client.InNamespace("")); meta.IsNoMatchError(err) || apierrors.IsNotFound(err) { - return configMaps, nil - } else if err != nil { + if err := retryWithExponentialBackoff(ctx, newReadBackoff(), func(ctx context.Context) error { + return cl.List(ctx, providerList, client.InNamespace("")) + }); err != nil { log.Error(err, fmt.Sprintf("Unable to list providers, %#v", err)) return configMaps, err @@ -285,9 +297,10 @@ func templateConfigMap(ctx context.Context, providerType clusterctlv1.ProviderTy spec := provider.GetSpec() spec.FetchConfig = &operatorv1.FetchConfiguration{ - OCI: url, + OCIConfiguration: operatorv1.OCIConfiguration{ + OCI: url, + }, } - provider.SetSpec(spec) if spec.Version != "" { diff --git a/cmd/plugin/cmd/preload_test.go b/cmd/plugin/cmd/preload_test.go new file mode 100644 index 00000000..c7c6ec8d --- /dev/null +++ b/cmd/plugin/cmd/preload_test.go @@ -0,0 +1,252 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "cmp" + "os" + "path" + "testing" + + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "sigs.k8s.io/cluster-api-operator/api/v1alpha2" + "sigs.k8s.io/cluster-api-operator/internal/controller/genericprovider" +) + +type publishProvider struct { + configMapName string + provider genericprovider.GenericProvider + metadataKey string + componentsKey string + metadataData []byte + componentsData []byte +} + +type publishOptions struct { + ociUrl string + providers []publishProvider +} + +func TestPreloadCommand(t *testing.T) { + tests := []struct { + name string + customURL string + publishOpts *publishOptions + existingProviders []genericprovider.GenericProvider + expectedConfigMaps int + wantErr bool + }{ + { + name: "no providers", + wantErr: false, + }, + { + name: "builtin core provider with OCI override", + publishOpts: &publishOptions{ + ociUrl: "ttl.sh/cluster-api-operator-manifests:1m", + providers: []publishProvider{{ + configMapName: "core-cluster-api-v1.9.3", + provider: generateGenericProvider(clusterctlv1.CoreProviderType, "cluster-api", "default", "v1.9.3", "", ""), + metadataKey: "metadata.yaml", + metadataData: []byte("metadata"), + componentsKey: "components.yaml", + componentsData: []byte("components"), + }}, + }, + expectedConfigMaps: 1, + }, + { + name: "multiple providers with OCI override", + publishOpts: &publishOptions{ + ociUrl: "ttl.sh/cluster-api-operator-manifests:1m", + providers: []publishProvider{{ + configMapName: "core-cluster-api-v1.9.3", + provider: generateGenericProvider(clusterctlv1.CoreProviderType, "cluster-api", "default", "v1.9.3", "", ""), + metadataKey: "core-cluster-api-v1.9.3-metadata.yaml", + metadataData: []byte("metadata"), + componentsKey: "core-cluster-api-v1.9.3-components.yaml", + componentsData: []byte("components"), + }, { + configMapName: "infrastructure-docker-v1.9.3", + provider: generateGenericProvider(clusterctlv1.InfrastructureProviderType, "docker", "default", "v1.9.3", "", ""), + metadataKey: "infrastructure-docker-v1.9.3-metadata.yaml", + metadataData: []byte("metadata"), + componentsKey: "infrastructure-docker-v1.9.3-components.yaml", + componentsData: []byte("components"), + }}, + }, + expectedConfigMaps: 2, + }, + { + name: "custom url infra provider", + existingProviders: []genericprovider.GenericProvider{ + func() genericprovider.GenericProvider { + p := generateGenericProvider(clusterctlv1.InfrastructureProviderType, "docker", "default", "v1.9.3", "", "") + spec := p.GetSpec() + spec.FetchConfig = &operatorv1.FetchConfiguration{ + URL: "https://github.com/kubernetes-sigs/cluster-api/releases/latest/core-components.yaml", + } + p.SetSpec(spec) + + return p + }(), + }, + expectedConfigMaps: 1, + }, + { + name: "regular core and infra provider", + existingProviders: []genericprovider.GenericProvider{ + generateGenericProvider(clusterctlv1.CoreProviderType, "cluster-api", "default", "v1.9.3", "", ""), + generateGenericProvider(clusterctlv1.InfrastructureProviderType, "docker", "default", "v1.9.3", "", ""), + }, + expectedConfigMaps: 2, + }, + { + name: "OCI override with incorrect metadata key", + publishOpts: &publishOptions{ + ociUrl: "ttl.sh/cluster-api-operator-manifests:1m", + providers: []publishProvider{{ + configMapName: "core-cluster-api-v1.9.3", + provider: generateGenericProvider(clusterctlv1.InfrastructureProviderType, "metadata-missing", "default", "v1.9.3", "", ""), + metadataKey: "incorrect-metadata.yaml", + metadataData: []byte("test"), + componentsKey: "components.yaml", + componentsData: []byte("test"), + }}, + }, + wantErr: true, + }, + { + name: "OCI override with incorrect components key", + publishOpts: &publishOptions{ + ociUrl: "ttl.sh/cluster-api-operator-manifests:1m", + providers: []publishProvider{{ + configMapName: "core-cluster-api-v1.9.3", + provider: generateGenericProvider(clusterctlv1.InfrastructureProviderType, "components-missing", "default", "v1.9.3", "", ""), + metadataKey: "metadata.yaml", + metadataData: []byte("test"), + componentsKey: "incorrect-components.yaml", + componentsData: []byte("test"), + }}, + }, + wantErr: true, + }, + { + name: "OCI override with missing image", + existingProviders: []genericprovider.GenericProvider{ + func() genericprovider.GenericProvider { + p := generateGenericProvider(clusterctlv1.InfrastructureProviderType, "docker", "default", "v1.9.3", "", "") + spec := p.GetSpec() + spec.FetchConfig = &operatorv1.FetchConfiguration{ + OCIConfiguration: operatorv1.OCIConfiguration{ + OCI: "ttl.sh/cluster-api-operator-manifests-missing:1m", + }, + } + p.SetSpec(spec) + + return p + }(), + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + dir, err := os.MkdirTemp("", "manifests") + defer func() { + g.Expect(os.RemoveAll(dir)).To(Succeed()) + }() + g.Expect(err).To(Succeed()) + + opts := cmp.Or(tt.publishOpts, &publishOptions{}) + if tt.publishOpts != nil && opts.ociUrl != "" { + for _, provider := range opts.providers { + err = os.WriteFile(path.Join(dir, provider.metadataKey), provider.metadataData, 0o777) + g.Expect(err).To(Succeed()) + err = os.WriteFile(path.Join(dir, provider.componentsKey), provider.componentsData, 0o777) + g.Expect(err).To(Succeed()) + } + + g.Expect(publish(ctx, dir, opts.ociUrl)).To(Succeed()) + + for _, data := range opts.providers { + spec := data.provider.GetSpec() + spec.FetchConfig = &operatorv1.FetchConfiguration{ + OCIConfiguration: operatorv1.OCIConfiguration{ + OCI: opts.ociUrl, + }, + } + data.provider.SetSpec(spec) + g.Expect(env.Client.Create(ctx, data.provider)).To(Succeed()) + } + } + + resources := []ctrlclient.Object{} + for _, provider := range tt.existingProviders { + resources = append(resources, provider) + } + + for _, data := range opts.providers { + resources = append(resources, data.provider) + } + + defer func() { + g.Expect(env.CleanupAndWait(ctx, resources...)).To(Succeed()) + }() + + for _, genericProvider := range tt.existingProviders { + g.Expect(env.Client.Create(ctx, genericProvider)).To(Succeed()) + } + + configMaps := []*corev1.ConfigMap{} + + g.Eventually(func(g Gomega) { + configMaps, err = preloadExisting(ctx, env) + g.Expect(tt.expectedConfigMaps).To(Equal(len(configMaps))) + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + } else { + g.Expect(err).NotTo(HaveOccurred()) + + maps := map[types.NamespacedName]*corev1.ConfigMap{} + for _, cm := range configMaps { + maps[ctrlclient.ObjectKeyFromObject(cm)] = cm + } + + for _, data := range opts.providers { + cm, ok := maps[types.NamespacedName{ + Namespace: data.provider.GetNamespace(), + Name: data.configMapName, + }] + + g.Expect(ok).To(BeTrue()) + + g.Expect(cm.Data["metadata"]).To(Equal(string(data.metadataData))) + g.Expect(cm.Data["components"]).To(Equal(string(data.componentsData))) + } + } + }, "15s", "1s").Should(Succeed()) + }) + } +} diff --git a/cmd/plugin/cmd/publish.go b/cmd/plugin/cmd/publish.go index 0ef32ccd..82a42b85 100644 --- a/cmd/plugin/cmd/publish.go +++ b/cmd/plugin/cmd/publish.go @@ -42,7 +42,7 @@ var publishOpts = &publishManifestsOptions{} var publishCmd = &cobra.Command{ Use: "publish", GroupID: groupManagement, - Short: "publish provider manifests to OCI registry", + Short: "publish provider manifests to an OCI registry", Long: LongDesc(` Publishes provider manifests to an OCI registry. `), @@ -69,27 +69,32 @@ func init() { } func runPublish() (err error) { + ctx := context.Background() + + return publish(ctx, publishOpts.dir, publishOpts.ociUrl, publishOpts.files...) +} + +func publish(ctx context.Context, dir, ociUrl string, files ...string) error { // 0. Create a file store - fs, err := file.New(publishOpts.dir) + fs, err := file.New(dir) if err != nil { return err } + defer func() { err = fs.Close() }() - ctx := context.Background() - // 1. Add files to the file store mediaType := "application/vnd.test.file" fileDescriptors := []v1.Descriptor{} - files, err := os.ReadDir(publishOpts.dir) + manifestFiles, err := os.ReadDir(dir) if err != nil { return err } - for _, file := range files { + for _, file := range manifestFiles { if !file.Type().IsRegular() { continue } @@ -104,7 +109,7 @@ func runPublish() (err error) { fmt.Printf("Added file: %s\n", file.Name()) } - for _, file := range publishOpts.files { + for _, file := range files { fileDescriptor, err := fs.Add(ctx, file, mediaType, "") if err != nil { return err @@ -128,21 +133,32 @@ func runPublish() (err error) { fmt.Println("Packaged manifests") - parts := strings.Split(publishOpts.ociUrl, ":") + ociUrl, plainHTTP := strings.CutPrefix(ociUrl, "http://") + + version := "" - tag := parts[len(parts)-1] - if err = fs.Tag(ctx, manifestDescriptor, tag); err != nil { + if parts := strings.SplitN(ociUrl, ":", 3); len(parts) == 2 { + ociUrl = parts[0] + version = parts[1] + } else if len(parts) == 3 { + version = parts[2] + ociUrl, _ = strings.CutSuffix(ociUrl, version) + } + + if err = fs.Tag(ctx, manifestDescriptor, version); err != nil { return err } // 3. Connect to a remote repository - reg := strings.Split(publishOpts.ociUrl, "/")[0] + reg := strings.Split(ociUrl, "/")[0] - repo, err := remote.NewRepository(publishOpts.ociUrl) + repo, err := remote.NewRepository(ociUrl) if err != nil { return err } + repo.PlainHTTP = plainHTTP + if creds := ociAuthentication(); creds != nil { repo.Client = &auth.Client{ Client: retry.DefaultClient, @@ -152,7 +168,7 @@ func runPublish() (err error) { } // 4. Copy from the file store to the remote repository - _, err = oras.Copy(ctx, fs, tag, repo, tag, oras.DefaultCopyOptions) + _, err = oras.Copy(ctx, fs, version, repo, version, oras.DefaultCopyOptions) if err != nil { return err } diff --git a/cmd/plugin/cmd/utils.go b/cmd/plugin/cmd/utils.go index 936f3425..5d2c1d88 100644 --- a/cmd/plugin/cmd/utils.go +++ b/cmd/plugin/cmd/utils.go @@ -22,6 +22,7 @@ import ( "fmt" "os" "sort" + "time" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -30,6 +31,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/version" + "k8s.io/apimachinery/pkg/util/wait" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/clientcmd" clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3" @@ -81,8 +83,13 @@ var errNotFound = errors.New("404 Not Found") // CreateKubeClient creates a kubernetes client from provided kubeconfig and kubecontext. func CreateKubeClient(kubeconfigPath, kubeconfigContext string) (ctrlclient.Client, error) { // Use specified kubeconfig path and context + loader := &clientcmd.ClientConfigLoadingRules{} + if kubeconfigPath != "" { + loader.ExplicitPath = kubeconfigPath + } + config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( - &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfigPath}, + loader, &clientcmd.ConfigOverrides{ CurrentContext: kubeconfigContext, }).ClientConfig() @@ -267,3 +274,38 @@ func GetLatestRelease(ctx context.Context, repo repository.Repository) (string, // If we reached this point, it means we didn't find any release. return "", errors.New("failed to find releases tagged with a valid semantic version number") } + +// retryWithExponentialBackoff repeats an operation until it passes or the exponential backoff times out. +func retryWithExponentialBackoff(ctx context.Context, opts wait.Backoff, operation func(ctx context.Context) error) error { + i := 0 + if err := wait.ExponentialBackoffWithContext(ctx, opts, func(ctx context.Context) (bool, error) { + i++ + if err := operation(ctx); err != nil { + if i < opts.Steps { + log.V(5).Info("Retrying with backoff", "cause", err.Error()) + return false, nil + } + + return false, err + } + + return true, nil + }); err != nil { + return fmt.Errorf("action failed after %d attempts: %w", i, err) + } + + return nil +} + +// newReadBackoff creates a new API Machinery backoff parameter set suitable for use with CLI cluster operations. +func newReadBackoff() wait.Backoff { + // Return a exponential backoff configuration which returns durations for a total time of ~15s. + // Example: 0, .25s, .6s, 1.2, 2.1s, 3.4s, 5.5s, 8s, 12s + // Jitter is added as a random fraction of the duration multiplied by the jitter factor. + return wait.Backoff{ + Duration: 250 * time.Millisecond, + Factor: 1.5, + Steps: 9, + Jitter: 0.1, + } +} diff --git a/config/crd/bases/operator.cluster.x-k8s.io_addonproviders.yaml b/config/crd/bases/operator.cluster.x-k8s.io_addonproviders.yaml index b094c1af..665446ab 100644 --- a/config/crd/bases/operator.cluster.x-k8s.io_addonproviders.yaml +++ b/config/crd/bases/operator.cluster.x-k8s.io_addonproviders.yaml @@ -2815,6 +2815,9 @@ spec: desired version of the release from GitHub. type: string type: object + x-kubernetes-validations: + - message: Must specify one and only one of {oci, url, selector} + rule: '[has(self.oci), has(self.url), has(self.selector)].exists_one(x,x)' manager: description: Manager defines the properties that can be enabled on the controller manager for the provider. diff --git a/config/crd/bases/operator.cluster.x-k8s.io_bootstrapproviders.yaml b/config/crd/bases/operator.cluster.x-k8s.io_bootstrapproviders.yaml index 18774368..8f634959 100644 --- a/config/crd/bases/operator.cluster.x-k8s.io_bootstrapproviders.yaml +++ b/config/crd/bases/operator.cluster.x-k8s.io_bootstrapproviders.yaml @@ -4445,6 +4445,9 @@ spec: desired version of the release from GitHub. type: string type: object + x-kubernetes-validations: + - message: Must specify one and only one of {oci, url, selector} + rule: '[has(self.oci), has(self.url), has(self.selector)].exists_one(x,x)' manager: description: Manager defines the properties that can be enabled on the controller manager for the provider. diff --git a/config/crd/bases/operator.cluster.x-k8s.io_controlplaneproviders.yaml b/config/crd/bases/operator.cluster.x-k8s.io_controlplaneproviders.yaml index 12d5f58b..0e19a0de 100644 --- a/config/crd/bases/operator.cluster.x-k8s.io_controlplaneproviders.yaml +++ b/config/crd/bases/operator.cluster.x-k8s.io_controlplaneproviders.yaml @@ -4447,6 +4447,9 @@ spec: desired version of the release from GitHub. type: string type: object + x-kubernetes-validations: + - message: Must specify one and only one of {oci, url, selector} + rule: '[has(self.oci), has(self.url), has(self.selector)].exists_one(x,x)' manager: description: Manager defines the properties that can be enabled on the controller manager for the provider. diff --git a/config/crd/bases/operator.cluster.x-k8s.io_coreproviders.yaml b/config/crd/bases/operator.cluster.x-k8s.io_coreproviders.yaml index 4936c6a5..ea97e867 100644 --- a/config/crd/bases/operator.cluster.x-k8s.io_coreproviders.yaml +++ b/config/crd/bases/operator.cluster.x-k8s.io_coreproviders.yaml @@ -4445,6 +4445,9 @@ spec: desired version of the release from GitHub. type: string type: object + x-kubernetes-validations: + - message: Must specify one and only one of {oci, url, selector} + rule: '[has(self.oci), has(self.url), has(self.selector)].exists_one(x,x)' manager: description: Manager defines the properties that can be enabled on the controller manager for the provider. diff --git a/config/crd/bases/operator.cluster.x-k8s.io_infrastructureproviders.yaml b/config/crd/bases/operator.cluster.x-k8s.io_infrastructureproviders.yaml index e7696d85..eb208462 100644 --- a/config/crd/bases/operator.cluster.x-k8s.io_infrastructureproviders.yaml +++ b/config/crd/bases/operator.cluster.x-k8s.io_infrastructureproviders.yaml @@ -4447,6 +4447,9 @@ spec: desired version of the release from GitHub. type: string type: object + x-kubernetes-validations: + - message: Must specify one and only one of {oci, url, selector} + rule: '[has(self.oci), has(self.url), has(self.selector)].exists_one(x,x)' manager: description: Manager defines the properties that can be enabled on the controller manager for the provider. diff --git a/config/crd/bases/operator.cluster.x-k8s.io_ipamproviders.yaml b/config/crd/bases/operator.cluster.x-k8s.io_ipamproviders.yaml index c2086cca..dae9d8b7 100644 --- a/config/crd/bases/operator.cluster.x-k8s.io_ipamproviders.yaml +++ b/config/crd/bases/operator.cluster.x-k8s.io_ipamproviders.yaml @@ -2815,6 +2815,9 @@ spec: desired version of the release from GitHub. type: string type: object + x-kubernetes-validations: + - message: Must specify one and only one of {oci, url, selector} + rule: '[has(self.oci), has(self.url), has(self.selector)].exists_one(x,x)' manager: description: Manager defines the properties that can be enabled on the controller manager for the provider. diff --git a/config/crd/bases/operator.cluster.x-k8s.io_runtimeextensionproviders.yaml b/config/crd/bases/operator.cluster.x-k8s.io_runtimeextensionproviders.yaml index f7cf25b4..f2dfc7d8 100644 --- a/config/crd/bases/operator.cluster.x-k8s.io_runtimeextensionproviders.yaml +++ b/config/crd/bases/operator.cluster.x-k8s.io_runtimeextensionproviders.yaml @@ -2817,6 +2817,9 @@ spec: desired version of the release from GitHub. type: string type: object + x-kubernetes-validations: + - message: Must specify one and only one of {oci, url, selector} + rule: '[has(self.oci), has(self.url), has(self.selector)].exists_one(x,x)' manager: description: Manager defines the properties that can be enabled on the controller manager for the provider. diff --git a/internal/controller/manifests_downloader.go b/internal/controller/manifests_downloader.go index 8f8009ce..41790a86 100644 --- a/internal/controller/manifests_downloader.go +++ b/internal/controller/manifests_downloader.go @@ -84,18 +84,20 @@ func (p *phaseReconciler) downloadManifests(ctx context.Context) (reconcile.Resu log.Info("Downloading provider manifests") - repo, err := util.RepositoryFactory(ctx, p.providerConfig, p.configClient.Variables()) - if err != nil { - err = fmt.Errorf("failed to create repo from provider url for provider %q: %w", p.provider.GetName(), err) + if p.providerConfig.URL() != fakeURL { + p.repo, err = util.RepositoryFactory(ctx, p.providerConfig, p.configClient.Variables()) + if err != nil { + err = fmt.Errorf("failed to create repo from provider url for provider %q: %w", p.provider.GetName(), err) - return reconcile.Result{}, wrapPhaseError(err, operatorv1.ComponentsFetchErrorReason, operatorv1.ProviderInstalledCondition) + return reconcile.Result{}, wrapPhaseError(err, operatorv1.ComponentsFetchErrorReason, operatorv1.ProviderInstalledCondition) + } } spec := p.provider.GetSpec() - if spec.Version == "" { + if spec.Version == "" && p.repo != nil { // User didn't set the version, try to get repository default. - spec.Version = repo.DefaultVersion() + spec.Version = p.repo.DefaultVersion() // Add version to the provider spec. p.provider.SetSpec(spec) @@ -110,7 +112,7 @@ func (p *phaseReconciler) downloadManifests(ctx context.Context) (reconcile.Resu return reconcile.Result{}, wrapPhaseError(err, operatorv1.ComponentsFetchErrorReason, operatorv1.ProviderInstalledCondition) } } else { - configMap, err = RepositoryConfigMap(ctx, p.provider, repo) + configMap, err = RepositoryConfigMap(ctx, p.provider, p.repo) if err != nil { err = fmt.Errorf("failed to create config map for provider %q: %w", p.provider.GetName(), err) @@ -120,18 +122,6 @@ func (p *phaseReconciler) downloadManifests(ctx context.Context) (reconcile.Resu if err := p.ctrlClient.Create(ctx, configMap); client.IgnoreAlreadyExists(err) != nil { return reconcile.Result{}, wrapPhaseError(err, operatorv1.ComponentsFetchErrorReason, operatorv1.ProviderInstalledCondition) - } else if err != nil { - cm := &corev1.ConfigMap{} - if err := p.ctrlClient.Get(ctx, client.ObjectKeyFromObject(configMap), cm); err != nil { - return reconcile.Result{}, wrapPhaseError(err, operatorv1.ComponentsFetchErrorReason, operatorv1.ProviderInstalledCondition) - } - - patchBase := client.MergeFrom(cm) - cm.OwnerReferences = configMap.OwnerReferences - - if err := p.ctrlClient.Patch(ctx, cm, patchBase); err != nil { - return reconcile.Result{}, wrapPhaseError(err, operatorv1.ComponentsFetchErrorReason, operatorv1.ProviderInstalledCondition) - } } return reconcile.Result{}, nil @@ -227,7 +217,7 @@ func TemplateManifestsConfigMap(provider operatorv1.GenericProvider, labels map[ // OCIConfigMap templates config from the OCI source. func OCIConfigMap(ctx context.Context, provider operatorv1.GenericProvider, auth *auth.Credential) (*corev1.ConfigMap, error) { - store, err := FetchOCI(ctx, provider, nil) + store, err := FetchOCI(ctx, provider, auth) if err != nil { return nil, err } @@ -309,7 +299,9 @@ func ProviderLabels(provider operatorv1.GenericProvider) map[string]string { } if provider.GetSpec().FetchConfig != nil && provider.GetSpec().FetchConfig.OCI != "" { - labels[configMapSourceLabel] = strings.ReplaceAll(provider.GetSpec().FetchConfig.OCI, "/", "_") + image := strings.ReplaceAll(provider.GetSpec().FetchConfig.OCI, "/", "_") + image = strings.ReplaceAll(image, ":", "_") + labels[configMapSourceLabel] = image } return labels diff --git a/internal/controller/oci_source.go b/internal/controller/oci_source.go index 14a36fb9..0dfaecb0 100644 --- a/internal/controller/oci_source.go +++ b/internal/controller/oci_source.go @@ -24,6 +24,7 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2" + "oras.land/oras-go/v2/content" "oras.land/oras-go/v2/registry/remote" "oras.land/oras-go/v2/registry/remote/auth" "oras.land/oras-go/v2/registry/remote/retry" @@ -47,16 +48,21 @@ const ( ) // mapStore is a pre-initialized map with expected file names to copy from OCI artifact. -type mapStore map[string][]byte +type mapStore struct { + data map[string][]byte + source oras.Target +} // NewMapStore initializes mapStore for the provider resource. func NewMapStore(p operatorv1.GenericProvider) mapStore { return mapStore{ - metadataFile: nil, - componentsFile: nil, - fmt.Sprintf(typedComponentsFile, p.GetType()): nil, - fmt.Sprintf(fullMetadataFile, p.GetType(), p.GetName(), p.GetSpec().Version): nil, - fmt.Sprintf(fullComponentsFile, p.GetType(), p.GetName(), p.GetSpec().Version): nil, + data: map[string][]byte{ + metadataFile: nil, + componentsFile: nil, + fmt.Sprintf(typedComponentsFile, p.GetType()): nil, + fmt.Sprintf(fullMetadataFile, p.GetType(), p.GetName(), p.GetSpec().Version): nil, + fmt.Sprintf(fullComponentsFile, p.GetType(), p.GetName(), p.GetSpec().Version): nil, + }, } } @@ -64,12 +70,12 @@ func NewMapStore(p operatorv1.GenericProvider) mapStore { func (m mapStore) GetMetadata(p operatorv1.GenericProvider) ([]byte, error) { fullMetadataKey := fmt.Sprintf(fullMetadataFile, p.GetType(), p.GetName(), p.GetSpec().Version) - data := m[fullMetadataKey] + data := m.data[fullMetadataKey] if len(data) != 0 { return data, nil } - data = m[metadataFile] + data = m.data[metadataFile] if len(data) != 0 { return data, nil } @@ -81,19 +87,19 @@ func (m mapStore) GetMetadata(p operatorv1.GenericProvider) ([]byte, error) { func (m mapStore) GetComponents(p operatorv1.GenericProvider) ([]byte, error) { fullComponentsKey := fmt.Sprintf(fullComponentsFile, p.GetType(), p.GetName(), p.GetSpec().Version) - data := m[fullComponentsKey] + data := m.data[fullComponentsKey] if len(data) != 0 { return data, nil } typedComponentsKey := fmt.Sprintf(typedComponentsFile, p.GetType()) - data = m[typedComponentsKey] + data = m.data[typedComponentsKey] if len(data) != 0 { return data, nil } - data = m[componentsFile] + data = m.data[componentsFile] if len(data) != 0 { return data, nil } @@ -102,9 +108,10 @@ func (m mapStore) GetComponents(p operatorv1.GenericProvider) ([]byte, error) { } // selector is a PreCopy implementation for the oras.Target which fetches only expected files. +// This helps to reduce the load on the source registry in case required item was added via restoreDuplicates. func (m mapStore) selector(_ context.Context, desc ocispec.Descriptor) error { file := desc.Annotations[ocispec.AnnotationTitle] - if _, expected := m[file]; expected { + if data := m.data[file]; len(data) == 0 { return nil } @@ -125,13 +132,57 @@ func (m mapStore) Fetch(ctx context.Context, target ocispec.Descriptor) (io.Read func (m mapStore) Push(ctx context.Context, expected ocispec.Descriptor, content io.Reader) (err error) { // Verify we only store expected artifact names file := expected.Annotations[ocispec.AnnotationTitle] - if _, expected := m[file]; expected { - m[file], err = io.ReadAll(content) + if _, expected := m.data[file]; expected { + m.data[file], err = io.ReadAll(content) + } + + if err := m.restoreDuplicates(ctx, expected); err != nil { + return fmt.Errorf("failed to restore duplicated file: %w", err) } return err } +func (m mapStore) restoreDuplicates(ctx context.Context, desc ocispec.Descriptor) (err error) { + successors, err := content.Successors(ctx, m.source, desc) + if err != nil { + return err + } + + for _, successor := range successors { + file := successor.Annotations[ocispec.AnnotationTitle] + if _, expected := m.data[file]; !expected { + continue + } + + if err := func() error { + desc := ocispec.Descriptor{ + MediaType: successor.MediaType, + Digest: successor.Digest, + Size: successor.Size, + } + rc, err := m.source.Fetch(ctx, desc) + if err != nil { + return fmt.Errorf("%q: %s: %w", file, desc.MediaType, err) + } + + defer func() { + err = rc.Close() + }() + + if err := m.Push(ctx, successor, rc); err != nil { + return fmt.Errorf("%q: %s: %w", file, desc.MediaType, err) + } + + return nil + }(); err != nil { + return err + } + } + + return nil +} + // Resolve implements oras.Target. func (m mapStore) Resolve(ctx context.Context, reference string) (ocispec.Descriptor, error) { return ocispec.Descriptor{}, nil @@ -148,9 +199,14 @@ var _ oras.Target = &mapStore{} func CopyOCIStore(ctx context.Context, url string, version string, store *mapStore, credential *auth.Credential) error { log := log.FromContext(ctx) - if parts := strings.SplitN(url, ":", 2); len(parts) == 2 { + url, plainHTTP := strings.CutPrefix(url, "http://") + + if parts := strings.SplitN(url, ":", 3); len(parts) == 2 { url = parts[0] version = parts[1] + } else if len(parts) == 3 { + version = parts[2] + url, _ = strings.CutSuffix(url, version) } repo, err := remote.NewRepository(url) @@ -168,6 +224,11 @@ func CopyOCIStore(ctx context.Context, url string, version string, store *mapSto } } + repo.PlainHTTP = plainHTTP + + // Set the source repository for restoring duplicated content inside the artifact + store.source = repo + _, err = oras.Copy(ctx, repo, version, store, version, oras.CopyOptions{ CopyGraphOptions: oras.CopyGraphOptions{ PreCopy: store.selector, @@ -202,7 +263,7 @@ func OCIAuthentication(c configclient.VariablesClient) *auth.Credential { } // FetchOCI copies the content of OCI. -func FetchOCI(ctx context.Context, provider operatorv1.GenericProvider, cred *auth.Credential) (mapStore, error) { +func FetchOCI(ctx context.Context, provider operatorv1.GenericProvider, cred *auth.Credential) (*mapStore, error) { log := log.FromContext(ctx) log.Info("Custom fetch configuration OCI url was provided") @@ -217,5 +278,5 @@ func FetchOCI(ctx context.Context, provider operatorv1.GenericProvider, cred *au return nil, err } - return store, nil + return &store, nil } diff --git a/internal/controller/phases.go b/internal/controller/phases.go index 2043318b..e8f5800b 100644 --- a/internal/controller/phases.go +++ b/internal/controller/phases.go @@ -48,6 +48,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" ) +// fakeURL is the stub url for custom providers, missing from clusterctl repository. +const fakeURL = "https://example.com/my-provider" + // phaseReconciler holds all required information for interacting with clusterctl code and // helps to iterate through provider reconciliation phases. type phaseReconciler struct { @@ -246,10 +249,16 @@ func (p *phaseReconciler) secretReader(ctx context.Context, providers ...configc log.Info("No configuration secret was specified") } + isCustom := true + for _, provider := range providers { if _, err := mr.AddProvider(provider.Name(), provider.Type(), provider.URL()); err != nil { return nil, err } + + if provider.Type() == clusterctlv1.ProviderType(p.provider.GetType()) && provider.Name() == p.provider.GetName() { + isCustom = false + } } // If provided store fetch config url in memory reader. @@ -265,9 +274,10 @@ func (p *phaseReconciler) secretReader(ctx context.Context, providers ...configc // To register a new provider from the config map, we need to specify a URL with a valid // format. However, since we're using data from a local config map, URLs are not needed. // As a workaround, we add a fake but well-formatted URL. + return mr.AddProvider(p.provider.GetName(), util.ClusterctlProviderType(p.provider), fakeURL) + } - fakeURL := "https://example.com/my-provider" - + if isCustom && p.provider.GetSpec().FetchConfig.OCI != "" { return mr.AddProvider(p.provider.GetName(), util.ClusterctlProviderType(p.provider), fakeURL) } } diff --git a/internal/controller/preflight_checks.go b/internal/controller/preflight_checks.go index 2199e22e..ba345647 100644 --- a/internal/controller/preflight_checks.go +++ b/internal/controller/preflight_checks.go @@ -87,15 +87,15 @@ func preflightChecks(ctx context.Context, c client.Client, provider genericprovi } if !isPredefinedProvider { - if spec.FetchConfig == nil || spec.FetchConfig.Selector == nil && spec.FetchConfig.URL == "" { + if spec.FetchConfig == nil || spec.FetchConfig.Selector == nil && spec.FetchConfig.URL == "" && spec.FetchConfig.OCI == "" { conditions.Set(provider, conditions.FalseCondition( operatorv1.PreflightCheckCondition, operatorv1.FetchConfigValidationErrorReason, clusterv1.ConditionSeverityError, - "Either Selector or URL must be provided for a not predefined provider", + "Either Selector, OCI URL or provider URL must be provided for a not predefined provider", )) - return ctrl.Result{}, fmt.Errorf("either selector or URL must be provided for a not predefined provider %s", provider.GetName()) + return ctrl.Result{}, fmt.Errorf("either selector, OCI URL or provider URL must be provided for a not predefined provider %s", provider.GetName()) } } diff --git a/internal/controller/preflight_checks_test.go b/internal/controller/preflight_checks_test.go index 072e065a..14a1c72f 100644 --- a/internal/controller/preflight_checks_test.go +++ b/internal/controller/preflight_checks_test.go @@ -557,7 +557,7 @@ func TestPreflightChecks(t *testing.T) { Type: operatorv1.PreflightCheckCondition, Reason: operatorv1.FetchConfigValidationErrorReason, Severity: clusterv1.ConditionSeverityError, - Message: "Either Selector or URL must be provided for a not predefined provider", + Message: "Either Selector, OCI URL or provider URL must be provided for a not predefined provider", Status: corev1.ConditionFalse, }, providerList: &operatorv1.CoreProviderList{}, @@ -590,7 +590,7 @@ func TestPreflightChecks(t *testing.T) { Type: operatorv1.PreflightCheckCondition, Reason: operatorv1.FetchConfigValidationErrorReason, Severity: clusterv1.ConditionSeverityError, - Message: "Either Selector or URL must be provided for a not predefined provider", + Message: "Either Selector, OCI URL or provider URL must be provided for a not predefined provider", Status: corev1.ConditionFalse, }, providerList: &operatorv1.CoreProviderList{}, diff --git a/test/e2e/helpers_test.go b/test/e2e/helpers_test.go index df16ea9d..e3e742d1 100644 --- a/test/e2e/helpers_test.go +++ b/test/e2e/helpers_test.go @@ -50,7 +50,7 @@ const ( ipamProviderName = "in-cluster" ipamProviderURL = "https://github.com/kubernetes-sigs/cluster-api-ipam-provider-in-cluster/releases/latest/ipam-components.yaml" - ipamProviderDeploymentName = "caip-in-cluster-controller-manager" + ipamProviderDeploymentName = "capi-ipam-in-cluster-controller-manager" customManifestsFolder = "resources/" ) diff --git a/test/e2e/minimal_configuration_test.go b/test/e2e/minimal_configuration_test.go index 4b518e6e..2e208566 100644 --- a/test/e2e/minimal_configuration_test.go +++ b/test/e2e/minimal_configuration_test.go @@ -22,10 +22,15 @@ package e2e import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + v1 "github.com/opencontainers/image-spec/specs-go/v1" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" + "oras.land/oras-go/v2" + "oras.land/oras-go/v2/content/file" + "oras.land/oras-go/v2/registry/remote" operatorv1 "sigs.k8s.io/cluster-api-operator/api/v1alpha2" "sigs.k8s.io/cluster-api/test/framework" @@ -267,7 +272,7 @@ metadata: e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...) }) - PIt("should successfully create and delete an IPAMProvider", func() { + It("should successfully create and delete an IPAMProvider", func() { bootstrapCluster := bootstrapClusterProxy.GetClient() ipamProvider := &operatorv1.IPAMProvider{ ObjectMeta: metav1.ObjectMeta{ @@ -340,6 +345,287 @@ metadata: }), e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...) }) + It("should successfully create and delete custom provider with OCI override", func() { + fs, err := file.New(customManifestsFolder) + Expect(err).ToNot(HaveOccurred()) + + defer func() { + Expect(fs.Close()).To(Succeed()) + }() + + mediaType := "application/vnd.test.file" + fds := []v1.Descriptor{} + + fileDescriptor, err := fs.Add(ctx, "infrastructure-custom-v0.0.1-metadata.yaml", mediaType, "") + Expect(err).ToNot(HaveOccurred()) + fds = append(fds, fileDescriptor) + + fileDescriptor, err = fs.Add(ctx, "infrastructure-custom-v0.0.1-components.yaml", mediaType, "") + Expect(err).ToNot(HaveOccurred()) + fds = append(fds, fileDescriptor) + + fileDescriptor, err = fs.Add(ctx, "infrastructure-docker-v0.0.1-metadata.yaml", mediaType, "") + Expect(err).ToNot(HaveOccurred()) + fds = append(fds, fileDescriptor) + + fileDescriptor, err = fs.Add(ctx, "infrastructure-docker-v0.0.1-components.yaml", mediaType, "") + Expect(err).ToNot(HaveOccurred()) + fds = append(fds, fileDescriptor) + + artifactType := "application/vnd.acme.config" + opts := oras.PackManifestOptions{ + Layers: fds, + } + + manifestDescriptor, err := oras.PackManifest(ctx, fs, oras.PackManifestVersion1_1, artifactType, opts) + Expect(err).ToNot(HaveOccurred()) + + Expect(fs.Tag(ctx, manifestDescriptor, "v0.0.1")).ToNot(HaveOccurred()) + + repo, err := remote.NewRepository("ttl.sh/cluster-api-operator-custom") + Expect(err).ToNot(HaveOccurred()) + + _, err = oras.Copy(ctx, fs, "v0.0.1", repo, "5m", oras.DefaultCopyOptions) + Expect(err).ToNot(HaveOccurred()) + + bootstrapCluster := bootstrapClusterProxy.GetClient() + provider := &operatorv1.InfrastructureProvider{ + ObjectMeta: metav1.ObjectMeta{ + Name: "custom", + Namespace: operatorNamespace, + }, + Spec: operatorv1.InfrastructureProviderSpec{ + ProviderSpec: operatorv1.ProviderSpec{ + Version: "v0.0.1", + FetchConfig: &operatorv1.FetchConfiguration{ + OCIConfiguration: operatorv1.OCIConfiguration{ + OCI: "ttl.sh/cluster-api-operator-custom:5m", + }, + }, + }, + }, + } + deployment := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{ + Name: "busybox", + Namespace: operatorNamespace, + }} + Expect(bootstrapCluster.Create(ctx, provider)).To(Succeed()) + + By("Waiting for the custom provider deployment to be ready") + framework.WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{ + Getter: bootstrapCluster, + Deployment: deployment, + }, e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...) + + By("Waiting for the custom provider to be ready") + WaitFor(ctx, For(provider).In(bootstrapCluster).ToSatisfy( + HaveStatusCondition(&provider.Status.Conditions, operatorv1.ProviderInstalledCondition)), + e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...) + + By("Waiting for status.IntalledVersion to be set") + WaitFor(ctx, For(provider).In(bootstrapCluster).ToSatisfy(func() bool { + return ptr.Equal(provider.Status.InstalledVersion, &provider.Spec.Version) + }), e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...) + + Expect(bootstrapCluster.Delete(ctx, provider)).To(Succeed()) + + By("Waiting for the custom provider deployment to be deleted") + WaitForDelete(ctx, For(deployment).In(bootstrapCluster), + e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...) + }) + + It("should successfully create and delete docker provider with OCI override", func() { + fs, err := file.New(customManifestsFolder) + Expect(err).ToNot(HaveOccurred()) + + defer func() { + Expect(fs.Close()).To(Succeed()) + }() + + mediaType := "application/vnd.test.file" + fds := []v1.Descriptor{} + + fileDescriptor, err := fs.Add(ctx, "infrastructure-custom-v0.0.1-metadata.yaml", mediaType, "") + Expect(err).ToNot(HaveOccurred()) + fds = append(fds, fileDescriptor) + + fileDescriptor, err = fs.Add(ctx, "infrastructure-custom-v0.0.1-components.yaml", mediaType, "") + Expect(err).ToNot(HaveOccurred()) + fds = append(fds, fileDescriptor) + + fileDescriptor, err = fs.Add(ctx, "infrastructure-docker-v0.0.1-metadata.yaml", mediaType, "") + Expect(err).ToNot(HaveOccurred()) + fds = append(fds, fileDescriptor) + + fileDescriptor, err = fs.Add(ctx, "infrastructure-docker-v0.0.1-components.yaml", mediaType, "") + Expect(err).ToNot(HaveOccurred()) + fds = append(fds, fileDescriptor) + + artifactType := "application/vnd.acme.config" + opts := oras.PackManifestOptions{ + Layers: fds, + } + + manifestDescriptor, err := oras.PackManifest(ctx, fs, oras.PackManifestVersion1_1, artifactType, opts) + Expect(err).ToNot(HaveOccurred()) + + Expect(fs.Tag(ctx, manifestDescriptor, "v0.0.1")).ToNot(HaveOccurred()) + + repo, err := remote.NewRepository("ttl.sh/cluster-api-operator-custom") + Expect(err).ToNot(HaveOccurred()) + + _, err = oras.Copy(ctx, fs, "v0.0.1", repo, "5m", oras.DefaultCopyOptions) + Expect(err).ToNot(HaveOccurred()) + + bootstrapCluster := bootstrapClusterProxy.GetClient() + provider := &operatorv1.InfrastructureProvider{ + ObjectMeta: metav1.ObjectMeta{ + Name: "docker", + Namespace: operatorNamespace, + }, + Spec: operatorv1.InfrastructureProviderSpec{ + ProviderSpec: operatorv1.ProviderSpec{ + Version: "v0.0.1", + FetchConfig: &operatorv1.FetchConfiguration{ + OCIConfiguration: operatorv1.OCIConfiguration{ + OCI: "ttl.sh/cluster-api-operator-custom:5m", + }, + }, + }, + }, + } + deployment := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{ + Name: "busybox", + Namespace: operatorNamespace, + }} + Expect(bootstrapCluster.Create(ctx, provider)).To(Succeed()) + + By("Waiting for the docker provider deployment to be ready") + framework.WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{ + Getter: bootstrapCluster, + Deployment: deployment, + }, e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...) + + By("Waiting for the docker provider to be ready") + WaitFor(ctx, For(provider).In(bootstrapCluster).ToSatisfy( + HaveStatusCondition(&provider.Status.Conditions, operatorv1.ProviderInstalledCondition)), + e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...) + + By("Waiting for status.IntalledVersion to be set") + WaitFor(ctx, For(provider).In(bootstrapCluster).ToSatisfy(func() bool { + return ptr.Equal(provider.Status.InstalledVersion, &provider.Spec.Version) + }), e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...) + + Expect(bootstrapCluster.Delete(ctx, provider)).To(Succeed()) + + By("Waiting for the custom docker provider deployment to be deleted") + WaitForDelete(ctx, For(deployment).In(bootstrapCluster), + e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...) + }) + + It("should successfully upgrade docker provider with OCI override", func() { + fs, err := file.New(customManifestsFolder) + Expect(err).ToNot(HaveOccurred()) + + defer func() { + Expect(fs.Close()).To(Succeed()) + }() + + mediaType := "application/vnd.test.file" + fds := []v1.Descriptor{} + + fileDescriptor, err := fs.Add(ctx, "infrastructure-docker-v0.0.1-metadata.yaml", mediaType, "") + Expect(err).ToNot(HaveOccurred()) + fds = append(fds, fileDescriptor) + + fileDescriptor, err = fs.Add(ctx, "infrastructure-docker-v0.0.1-components.yaml", mediaType, "") + Expect(err).ToNot(HaveOccurred()) + fds = append(fds, fileDescriptor) + + fileDescriptor, err = fs.Add(ctx, "infrastructure-docker-v0.0.2-metadata.yaml", mediaType, "") + Expect(err).ToNot(HaveOccurred()) + fds = append(fds, fileDescriptor) + + fileDescriptor, err = fs.Add(ctx, "infrastructure-docker-v0.0.2-components.yaml", mediaType, "") + Expect(err).ToNot(HaveOccurred()) + fds = append(fds, fileDescriptor) + + artifactType := "application/vnd.acme.config" + opts := oras.PackManifestOptions{ + Layers: fds, + } + + manifestDescriptor, err := oras.PackManifest(ctx, fs, oras.PackManifestVersion1_1, artifactType, opts) + Expect(err).ToNot(HaveOccurred()) + + Expect(fs.Tag(ctx, manifestDescriptor, "5m")).ToNot(HaveOccurred()) + + repo, err := remote.NewRepository("ttl.sh/cluster-api-operator-upgrade") + Expect(err).ToNot(HaveOccurred()) + + _, err = oras.Copy(ctx, fs, "5m", repo, "5m", oras.DefaultCopyOptions) + Expect(err).ToNot(HaveOccurred()) + + bootstrapCluster := bootstrapClusterProxy.GetClient() + provider := &operatorv1.InfrastructureProvider{ + ObjectMeta: metav1.ObjectMeta{ + Name: "docker", + Namespace: operatorNamespace, + }, + Spec: operatorv1.InfrastructureProviderSpec{ + ProviderSpec: operatorv1.ProviderSpec{ + Version: "v0.0.1", + FetchConfig: &operatorv1.FetchConfiguration{ + OCIConfiguration: operatorv1.OCIConfiguration{ + OCI: "ttl.sh/cluster-api-operator-upgrade:5m", + }, + }, + }, + }, + } + deployment := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{ + Name: "busybox", + Namespace: operatorNamespace, + }} + Expect(bootstrapCluster.Create(ctx, provider)).To(Succeed()) + + By("Waiting for the docker provider deployment to be ready") + framework.WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{ + Getter: bootstrapCluster, + Deployment: deployment, + }, e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...) + + By("Waiting for the docker provider to be ready") + WaitFor(ctx, For(provider).In(bootstrapCluster).ToSatisfy( + HaveStatusCondition(&provider.Status.Conditions, operatorv1.ProviderInstalledCondition)), + e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...) + + By("Waiting for status.IntalledVersion to be set") + WaitFor(ctx, For(provider).In(bootstrapCluster).ToSatisfy(func() bool { + return ptr.Equal(provider.Status.InstalledVersion, &provider.Spec.Version) + }), e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...) + + By("Updating verion to v0.0.2 to initiate upgrade") + provider.Spec.Version = "v0.0.2" + Expect(bootstrapCluster.Update(ctx, provider)).To(Succeed()) + + By("Waiting for status.IntalledVersion to be set") + WaitFor(ctx, For(provider).In(bootstrapCluster).ToSatisfy(func() bool { + return ptr.Equal(provider.Status.InstalledVersion, &provider.Spec.Version) + }), e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...) + + By("Waiting for the docker provider to be ready") + WaitFor(ctx, For(provider).In(bootstrapCluster).ToSatisfy( + HaveStatusCondition(&provider.Status.Conditions, operatorv1.ProviderInstalledCondition)), + e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...) + + Expect(bootstrapCluster.Delete(ctx, provider)).To(Succeed()) + + By("Waiting for the custom docker provider deployment to be deleted") + WaitForDelete(ctx, For(deployment).In(bootstrapCluster), + e2eConfig.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...) + }) + It("should successfully delete a CoreProvider", func() { bootstrapCluster := bootstrapClusterProxy.GetClient() coreProvider := &operatorv1.CoreProvider{ObjectMeta: metav1.ObjectMeta{ diff --git a/test/e2e/resources/full-chart-install.yaml b/test/e2e/resources/full-chart-install.yaml index c544109b..db0b8a37 100644 --- a/test/e2e/resources/full-chart-install.yaml +++ b/test/e2e/resources/full-chart-install.yaml @@ -2840,6 +2840,9 @@ spec: desired version of the release from GitHub. type: string type: object + x-kubernetes-validations: + - message: Must specify one and only one of {oci, url, selector} + rule: '[has(self.oci), has(self.url), has(self.selector)].exists_one(x,x)' manager: description: Manager defines the properties that can be enabled on the controller manager for the provider. @@ -7576,6 +7579,9 @@ spec: desired version of the release from GitHub. type: string type: object + x-kubernetes-validations: + - message: Must specify one and only one of {oci, url, selector} + rule: '[has(self.oci), has(self.url), has(self.selector)].exists_one(x,x)' manager: description: Manager defines the properties that can be enabled on the controller manager for the provider. @@ -12314,6 +12320,9 @@ spec: desired version of the release from GitHub. type: string type: object + x-kubernetes-validations: + - message: Must specify one and only one of {oci, url, selector} + rule: '[has(self.oci), has(self.url), has(self.selector)].exists_one(x,x)' manager: description: Manager defines the properties that can be enabled on the controller manager for the provider. @@ -17051,6 +17060,9 @@ spec: desired version of the release from GitHub. type: string type: object + x-kubernetes-validations: + - message: Must specify one and only one of {oci, url, selector} + rule: '[has(self.oci), has(self.url), has(self.selector)].exists_one(x,x)' manager: description: Manager defines the properties that can be enabled on the controller manager for the provider. @@ -21789,6 +21801,9 @@ spec: desired version of the release from GitHub. type: string type: object + x-kubernetes-validations: + - message: Must specify one and only one of {oci, url, selector} + rule: '[has(self.oci), has(self.url), has(self.selector)].exists_one(x,x)' manager: description: Manager defines the properties that can be enabled on the controller manager for the provider. @@ -24896,6 +24911,9 @@ spec: desired version of the release from GitHub. type: string type: object + x-kubernetes-validations: + - message: Must specify one and only one of {oci, url, selector} + rule: '[has(self.oci), has(self.url), has(self.selector)].exists_one(x,x)' manager: description: Manager defines the properties that can be enabled on the controller manager for the provider. @@ -28004,6 +28022,9 @@ spec: desired version of the release from GitHub. type: string type: object + x-kubernetes-validations: + - message: Must specify one and only one of {oci, url, selector} + rule: '[has(self.oci), has(self.url), has(self.selector)].exists_one(x,x)' manager: description: Manager defines the properties that can be enabled on the controller manager for the provider. diff --git a/test/e2e/resources/infrastructure-custom-v0.0.1-components.yaml b/test/e2e/resources/infrastructure-custom-v0.0.1-components.yaml new file mode 100644 index 00000000..d89cb16d --- /dev/null +++ b/test/e2e/resources/infrastructure-custom-v0.0.1-components.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + cluster.x-k8s.io/provider: infrastructure-custom + control-plane: controller-manager + name: custom +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: busybox + namespace: custom +spec: + replicas: 1 + selector: + matchLabels: + busybox: busybox + template: + metadata: + labels: + busybox: busybox + spec: + containers: + - image: gcr.io/google-samples/hello-app:1.0 + imagePullPolicy: IfNotPresent + name: manager + restartPolicy: Always \ No newline at end of file diff --git a/test/e2e/resources/infrastructure-custom-v0.0.1-metadata.yaml b/test/e2e/resources/infrastructure-custom-v0.0.1-metadata.yaml new file mode 100644 index 00000000..9c70ab07 --- /dev/null +++ b/test/e2e/resources/infrastructure-custom-v0.0.1-metadata.yaml @@ -0,0 +1,6 @@ +apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3 +kind: Metadata +releaseSeries: + - major: 0 + minor: 0 + contract: v1beta1 diff --git a/test/e2e/resources/infrastructure-docker-v0.0.1-components.yaml b/test/e2e/resources/infrastructure-docker-v0.0.1-components.yaml new file mode 100644 index 00000000..d89cb16d --- /dev/null +++ b/test/e2e/resources/infrastructure-docker-v0.0.1-components.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + cluster.x-k8s.io/provider: infrastructure-custom + control-plane: controller-manager + name: custom +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: busybox + namespace: custom +spec: + replicas: 1 + selector: + matchLabels: + busybox: busybox + template: + metadata: + labels: + busybox: busybox + spec: + containers: + - image: gcr.io/google-samples/hello-app:1.0 + imagePullPolicy: IfNotPresent + name: manager + restartPolicy: Always \ No newline at end of file diff --git a/test/e2e/resources/infrastructure-docker-v0.0.1-metadata.yaml b/test/e2e/resources/infrastructure-docker-v0.0.1-metadata.yaml new file mode 100644 index 00000000..9c70ab07 --- /dev/null +++ b/test/e2e/resources/infrastructure-docker-v0.0.1-metadata.yaml @@ -0,0 +1,6 @@ +apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3 +kind: Metadata +releaseSeries: + - major: 0 + minor: 0 + contract: v1beta1 diff --git a/test/e2e/resources/infrastructure-docker-v0.0.2-components.yaml b/test/e2e/resources/infrastructure-docker-v0.0.2-components.yaml new file mode 100644 index 00000000..d89cb16d --- /dev/null +++ b/test/e2e/resources/infrastructure-docker-v0.0.2-components.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + cluster.x-k8s.io/provider: infrastructure-custom + control-plane: controller-manager + name: custom +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: busybox + namespace: custom +spec: + replicas: 1 + selector: + matchLabels: + busybox: busybox + template: + metadata: + labels: + busybox: busybox + spec: + containers: + - image: gcr.io/google-samples/hello-app:1.0 + imagePullPolicy: IfNotPresent + name: manager + restartPolicy: Always \ No newline at end of file diff --git a/test/e2e/resources/infrastructure-docker-v0.0.2-metadata.yaml b/test/e2e/resources/infrastructure-docker-v0.0.2-metadata.yaml new file mode 100644 index 00000000..9c70ab07 --- /dev/null +++ b/test/e2e/resources/infrastructure-docker-v0.0.2-metadata.yaml @@ -0,0 +1,6 @@ +apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3 +kind: Metadata +releaseSeries: + - major: 0 + minor: 0 + contract: v1beta1 diff --git a/test/go.mod b/test/go.mod index 9a3e2087..df690e91 100644 --- a/test/go.mod +++ b/test/go.mod @@ -9,12 +9,14 @@ replace sigs.k8s.io/cluster-api-operator => ../ require ( github.com/onsi/ginkgo/v2 v2.22.2 github.com/onsi/gomega v1.36.2 + github.com/opencontainers/image-spec v1.1.0 golang.org/x/tools v0.29.0 k8s.io/api v0.31.4 k8s.io/apiextensions-apiserver v0.31.4 k8s.io/apimachinery v0.31.4 k8s.io/klog/v2 v2.130.1 k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 + oras.land/oras-go/v2 v2.5.0 sigs.k8s.io/cluster-api v1.9.4 sigs.k8s.io/cluster-api-operator v0.0.0-00010101000000-000000000000 sigs.k8s.io/cluster-api/test v1.8.5 @@ -88,7 +90,6 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pkg/errors v0.9.1 // indirect diff --git a/test/go.sum b/test/go.sum index 475ab51b..9621fcc2 100644 --- a/test/go.sum +++ b/test/go.sum @@ -407,6 +407,8 @@ k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7F k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +oras.land/oras-go/v2 v2.5.0 h1:o8Me9kLY74Vp5uw07QXPiitjsw7qNXi8Twd+19Zf02c= +oras.land/oras-go/v2 v2.5.0/go.mod h1:z4eisnLP530vwIOUOJeBIj0aGI0L1C3d53atvCBqZHg= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 h1:2770sDpzrjjsAtVhSeUFseziht227YAWYHLGNM8QPwY= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/cluster-api v1.9.4 h1:pa2Ho50F9Js/Vv/Jy11TcpmGiqY2ukXCoDj/dY25Y7M=