New rhc osdk utils (#710)

* Added new cache version to fix bug * Hash collision update * Added service monitor properly * Fixed issue with secret * Fixed cyndi
RedHatInsights · Oct 11, 2022 · 3602c90 · 3602c90
1 parent 4e6e17f
commit 3602c90
Show file tree

Hide file tree

Showing 38 changed files with 693 additions and 605 deletions.
diff --git a/Makefile b/Makefile
@@ -4,7 +4,7 @@
 # For example, running 'make bundle-build bundle-push catalog-build catalog-push' will build and push both
 # cloud.redhat.com/clowder-bundle:$VERSION and cloud.redhat.com/clowder-catalog:$VERSION.
 IMAGE_TAG_BASE ?= quay.io/cloudservices/clowder
-CLOWDER_BUILD_TAG ?= $(shell git rev-parse --short=7 HEAD)
+CLOWDER_BUILD_TAG ?= $(shell git rev-parse --short=8 HEAD)
 
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
 ENVTEST_K8S_VERSION = 1.23

diff --git a/build/e2e-test.sh b/build/e2e-test.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 set -e
 
-export IMAGE_TAG=`git rev-parse --short HEAD`
+export IMAGE_TAG=`git rev-parse --short=8 HEAD`
 
 bash build/kube_setup.sh
 

diff --git a/build/pr_check_inner.sh b/build/pr_check_inner.sh
@@ -72,7 +72,7 @@ $KUBECTL_CMD get pods --all-namespaces=true
 
 source build/kube_setup.sh
 
-export IMAGE_TAG=`git rev-parse --short HEAD`
+export IMAGE_TAG=`git rev-parse --short=8 HEAD`
 
 $KUBECTL_CMD create namespace clowder-system
 

diff --git a/build_deploy.sh b/build_deploy.sh
@@ -3,7 +3,7 @@
 set -exv
 
 IMAGE="quay.io/cloudservices/clowder"
-IMAGE_TAG=$(git rev-parse --short=7 HEAD)
+IMAGE_TAG=$(git rev-parse --short=8 HEAD)
 
 if [[ -z "$QUAY_USER" || -z "$QUAY_TOKEN" ]]; then
     echo "QUAY_USER and QUAY_TOKEN must be set"
@@ -15,7 +15,7 @@ if [[ -z "$RH_REGISTRY_USER" || -z "$RH_REGISTRY_TOKEN" ]]; then
     exit 1
 fi
 
-BASE_TAG=`cat go.mod go.sum Dockerfile.base | sha256sum  | head -c 7`
+BASE_TAG=`cat go.mod go.sum Dockerfile.base | sha256sum  | head -c 8`
 BASE_IMG=quay.io/cloudservices/clowder-base:$BASE_TAG
 
 DOCKER_CONF="$PWD/.docker"

diff --git a/controllers/cloud.redhat.com/clowdapp_reconciliation.go b/controllers/cloud.redhat.com/clowdapp_reconciliation.go
@@ -246,8 +246,8 @@ func (r *ClowdAppReconciliation) isEnvReady() (ctrl.Result, error) {
 }
 
 func (r *ClowdAppReconciliation) createCache() (ctrl.Result, error) {
-	cacheConfig := rc.NewCacheConfig(Scheme, errors.ClowdKey("log"), ProtectedGVKs, DebugOptions)
-	cache := rc.NewObjectCache(r.ctx, r.client, cacheConfig)
+	cacheConfig := rc.NewCacheConfig(Scheme, nil, ProtectedGVKs, rc.Options{StrictGVK: true, DebugOptions: DebugOptions})
+	cache := rc.NewObjectCache(r.ctx, r.client, r.log, cacheConfig)
 	r.cache = &cache
 	return ctrl.Result{}, nil
 }

diff --git a/controllers/cloud.redhat.com/clowdenvironment_controller.go b/controllers/cloud.redhat.com/clowdenvironment_controller.go
@@ -145,8 +145,8 @@ func (r *ClowdEnvironmentReconciler) Reconcile(ctx context.Context, req ctrl.Req
 	}
 
 	ctx = context.WithValue(ctx, errors.ClowdKey("obj"), &env)
-	cacheConfig := rc.NewCacheConfig(Scheme, errors.ClowdKey("log"), ProtectedGVKs, DebugOptions)
-	cache := rc.NewObjectCache(ctx, r.Client, cacheConfig)
+	cacheConfig := rc.NewCacheConfig(Scheme, nil, ProtectedGVKs, rc.Options{StrictGVK: true, DebugOptions: DebugOptions})
+	cache := rc.NewObjectCache(ctx, r.Client, &log, cacheConfig)
 
 	r.initMetrics(env)
 

diff --git a/controllers/cloud.redhat.com/clowdjobinvocation_controller.go b/controllers/cloud.redhat.com/clowdjobinvocation_controller.go
@@ -25,7 +25,6 @@ import (
 	rc "github.com/RedHatInsights/rhc-osdk-utils/resource_cache"
 	"github.com/go-logr/logr"
 	batchv1 "k8s.io/api/batch/v1"
-	core "k8s.io/api/core/v1"
 	k8serr "k8s.io/apimachinery/pkg/api/errors"
 
 	"k8s.io/apimachinery/pkg/runtime"
@@ -52,10 +51,6 @@ type ClowdJobInvocationReconciler struct {
 	Recorder record.EventRecorder
 }
 
-var IqeClowdJob = rc.NewSingleResourceIdent("cji", "iqe_clowdjob", &batchv1.Job{})
-var ClowdJob = rc.NewMultiResourceIdent("cji", "clowdjob", &batchv1.Job{})
-var IqeSecret = rc.NewSingleResourceIdent("cji", "iqe_secret", &core.Secret{})
-
 // +kubebuilder:rbac:groups=cloud.redhat.com,resources=clowdjobinvocations,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=cloud.redhat.com,resources=clowdjobinvocations/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=cloud.redhat.com,resources=clowdapps,verbs=get;list;watch
@@ -81,9 +76,14 @@ func (r *ClowdJobInvocationReconciler) Reconcile(ctx context.Context, req ctrl.R
 		return ctrl.Result{}, err
 	}
 
-	cacheConfig := rc.NewCacheConfig(Scheme, errors.ClowdKey("log"), ProtectedGVKs, DebugOptions)
-
-	cache := rc.NewObjectCache(ctx, r.Client, cacheConfig)
+	cacheConfig := rc.NewCacheConfig(Scheme, nil, ProtectedGVKs, rc.Options{StrictGVK: true, DebugOptions: DebugOptions})
+	cache := rc.NewObjectCache(ctx, r.Client, &log, cacheConfig)
+	cache.AddPossibleGVKFromIdent(
+		iqe.IqeSecret,
+		iqe.VaultSecret,
+		iqe.ClowdJob,
+		iqe.IqeClowdJob,
+	)
 
 	// Deprecated, used to handle any lagging CJIs that would otherwise throw errors
 	if cji.Status.Jobs != nil {
@@ -207,7 +207,7 @@ func (r *ClowdJobInvocationReconciler) Reconcile(ctx context.Context, req ctrl.R
 		}
 
 		j := batchv1.Job{}
-		if err := cache.Create(IqeClowdJob, nn, &j); err != nil {
+		if err := cache.Create(iqe.IqeClowdJob, nn, &j); err != nil {
 			r.Log.Error(err, "Iqe Job could not be created via cache", "jobinvocation", nn.Name)
 			if condErr := SetClowdJobInvocationConditions(ctx, r.Client, &cji, crd.ReconciliationFailed, err); condErr != nil {
 				return ctrl.Result{}, condErr
@@ -224,7 +224,7 @@ func (r *ClowdJobInvocationReconciler) Reconcile(ctx context.Context, req ctrl.R
 			return ctrl.Result{}, err
 		}
 
-		if err := cache.Update(IqeClowdJob, &j); err != nil {
+		if err := cache.Update(iqe.IqeClowdJob, &j); err != nil {
 			r.Log.Error(err, "Iqe Job could not update via cache", "jobinvocation", nn.Name)
 			if condErr := SetClowdJobInvocationConditions(ctx, r.Client, &cji, crd.ReconciliationFailed, err); condErr != nil {
 				return ctrl.Result{}, condErr
@@ -278,15 +278,15 @@ func (r *ClowdJobInvocationReconciler) InvokeJob(ctx context.Context, cache *rc.
 	}
 
 	j := batchv1.Job{}
-	if err := cache.Create(ClowdJob, nn, &j); err != nil {
+	if err := cache.Create(iqe.ClowdJob, nn, &j); err != nil {
 		return err
 	}
 
 	if err := jobProvider.CreateJobResource(cji, env, app, nn, job, &j); err != nil {
 		return err
 	}
 
-	if err := cache.Update(ClowdJob, &j); err != nil {
+	if err := cache.Update(iqe.ClowdJob, &j); err != nil {
 		return err
 	}
 

diff --git a/controllers/cloud.redhat.com/providers/autoscaler/keda.go b/controllers/cloud.redhat.com/providers/autoscaler/keda.go
@@ -7,15 +7,11 @@ import (
 	"github.com/RedHatInsights/clowder/controllers/cloud.redhat.com/config"
 	"github.com/RedHatInsights/clowder/controllers/cloud.redhat.com/providers"
 	deployProvider "github.com/RedHatInsights/clowder/controllers/cloud.redhat.com/providers/deployment"
-	rc "github.com/RedHatInsights/rhc-osdk-utils/resource_cache"
 	keda "github.com/kedacore/keda/v2/apis/keda/v1alpha1"
 	apps "k8s.io/api/apps/v1"
 	"k8s.io/apimachinery/pkg/types"
 )
 
-// CoreAutoScaler is the config that is presented as the cdappconfig.json file.
-var CoreAutoScaler = rc.NewMultiResourceIdent(ProvName, "core_autoscaler", &keda.ScaledObject{})
-
 func makeAutoScalers(deployment *crd.Deployment, app *crd.ClowdApp, c *config.AppConfig, asp *providers.Provider) error {
 	s := &keda.ScaledObject{}
 	nn := app.GetDeploymentNamespacedName(deployment)

diff --git a/controllers/cloud.redhat.com/providers/autoscaler/provider.go b/controllers/cloud.redhat.com/providers/autoscaler/provider.go
@@ -2,6 +2,9 @@ package autoscaler
 
 import (
 	p "github.com/RedHatInsights/clowder/controllers/cloud.redhat.com/providers"
+	rc "github.com/RedHatInsights/rhc-osdk-utils/resource_cache"
+	keda "github.com/kedacore/keda/v2/apis/keda/v1alpha1"
+	v2 "k8s.io/api/autoscaling/v2"
 )
 
 // ProvName sets the provider name identifier
@@ -10,6 +13,10 @@ var ProvName = "autoscaler"
 const ENABLED = "enabled"
 const KEDA = "keda"
 
+// CoreAutoScaler is the config that is presented as the cdappconfig.json file.
+var CoreAutoScaler = rc.NewMultiResourceIdent(ProvName, "core_autoscaler", &keda.ScaledObject{})
+var SimpleAutoScaler = rc.NewMultiResourceIdent(ProvName, "simple_hpa", &v2.HorizontalPodAutoscaler{})
+
 // GetAutoscaler returns the correct end provider.
 func GetAutoScaler(c *p.Provider) (p.ClowderProvider, error) {
 	mode := c.Env.Spec.Providers.AutoScaler.Mode

diff --git a/controllers/cloud.redhat.com/providers/autoscaler/router.go b/controllers/cloud.redhat.com/providers/autoscaler/router.go
@@ -11,6 +11,10 @@ type autoScaleProviderRouter struct {
 }
 
 func NewAutoScaleProviderRouter(p *providers.Provider) (providers.ClowderProvider, error) {
+	p.Cache.AddPossibleGVKFromIdent(
+		SimpleAutoScaler,
+		CoreAutoScaler,
+	)
 	return &autoScaleProviderRouter{Provider: *p}, nil
 }
 

diff --git a/controllers/cloud.redhat.com/providers/autoscaler/simple.go b/controllers/cloud.redhat.com/providers/autoscaler/simple.go
@@ -10,15 +10,13 @@ import (
 	"github.com/RedHatInsights/clowder/controllers/cloud.redhat.com/errors"
 	"github.com/RedHatInsights/clowder/controllers/cloud.redhat.com/providers"
 	deployProvider "github.com/RedHatInsights/clowder/controllers/cloud.redhat.com/providers/deployment"
-	rc "github.com/RedHatInsights/rhc-osdk-utils/resource_cache"
 	apps "k8s.io/api/apps/v1"
 	v2 "k8s.io/api/autoscaling/v2"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 const (
-	SIMPLE_HPA             = "simple_hpa"
 	CLOWD_API_VERSION      = "clowd.redhat.com/v1alpha1"
 	CLOWD_KIND             = "ClowdApp"
 	DEPLOYMENT_API_VERSION = "apps/v1"
@@ -44,9 +42,8 @@ func ProvideSimpleAutoScaler(app *crd.ClowdApp, appConfig *config.AppConfig, sp
 
 //Adds the HPA to the resource cache
 func cacheAutoscaler(app *crd.ClowdApp, sp *providers.Provider, deployment crd.Deployment, hpaResource v2.HorizontalPodAutoscaler) error {
-	simpleAutoScaler := rc.NewMultiResourceIdent(ProvName, SIMPLE_HPA, &v2.HorizontalPodAutoscaler{})
 	nn := app.GetDeploymentNamespacedName(&deployment)
-	if err := sp.Cache.Create(simpleAutoScaler, nn, &hpaResource); err != nil {
+	if err := sp.Cache.Create(SimpleAutoScaler, nn, &hpaResource); err != nil {
 		return err
 	}
 	return nil

diff --git a/controllers/cloud.redhat.com/providers/confighash/default.go b/controllers/cloud.redhat.com/providers/confighash/default.go
@@ -22,6 +22,7 @@ var CoreConfigSecret = rc.NewSingleResourceIdent(ProvName, "core_config_secret",
 
 // NewConfigHashProvider returns a new End provider run at the end of the provider set.
 func NewConfigHashProvider(p *p.Provider) (p.ClowderProvider, error) {
+	p.Cache.AddPossibleGVKFromIdent(CoreConfigSecret)
 	return &confighashProvider{Provider: *p}, nil
 }
 

diff --git a/controllers/cloud.redhat.com/providers/cronjob/default.go b/controllers/cloud.redhat.com/providers/cronjob/default.go
@@ -3,13 +3,19 @@ package cronjob
 import (
 	crd "github.com/RedHatInsights/clowder/apis/cloud.redhat.com/v1alpha1"
 	p "github.com/RedHatInsights/clowder/controllers/cloud.redhat.com/providers"
+	rc "github.com/RedHatInsights/rhc-osdk-utils/resource_cache"
+	batch "k8s.io/api/batch/v1"
 )
 
 type cronjobProvider struct {
 	p.Provider
 }
 
+// CoreCronJob is the cronjob for the apps cronjobs.
+var CoreCronJob = rc.NewMultiResourceIdent(ProvName, "core_cronjob", &batch.CronJob{})
+
 func NewCronJobProvider(p *p.Provider) (p.ClowderProvider, error) {
+	p.Cache.AddPossibleGVKFromIdent(CoreCronJob)
 	return &cronjobProvider{Provider: *p}, nil
 }
 

diff --git a/controllers/cloud.redhat.com/providers/cronjob/provider.go b/controllers/cloud.redhat.com/providers/cronjob/provider.go
@@ -2,16 +2,11 @@ package cronjob
 
 import (
 	p "github.com/RedHatInsights/clowder/controllers/cloud.redhat.com/providers"
-	rc "github.com/RedHatInsights/rhc-osdk-utils/resource_cache"
-	batch "k8s.io/api/batch/v1"
 )
 
 // ProvName sets the provider name identifier
 var ProvName = "cronjob"
 
-// CoreCronJob is the cronjob for the apps cronjobs.
-var CoreCronJob = rc.NewMultiResourceIdent(ProvName, "core_cronjob", &batch.CronJob{})
-
 // GetCronJob returns the correct cronjob provider.
 func GetCronJob(c *p.Provider) (p.ClowderProvider, error) {
 	return NewCronJobProvider(c)

diff --git a/controllers/cloud.redhat.com/providers/database/localdb.go b/controllers/cloud.redhat.com/providers/database/localdb.go
@@ -37,6 +37,12 @@ type localDbProvider struct {
 
 // NewLocalDBProvider returns a new local DB provider object.
 func NewLocalDBProvider(p *providers.Provider) (providers.ClowderProvider, error) {
+	p.Cache.AddPossibleGVKFromIdent(
+		LocalDBDeployment,
+		LocalDBService,
+		LocalDBPVC,
+		LocalDBSecret,
+	)
 	return &localDbProvider{Provider: *p}, nil
 }
 

diff --git a/controllers/cloud.redhat.com/providers/database/shareddb.go b/controllers/cloud.redhat.com/providers/database/shareddb.go
@@ -48,6 +48,13 @@ type sharedDbProvider struct {
 
 // NewSharedDBProvider returns a new local DB provider object.
 func NewSharedDBProvider(p *providers.Provider) (providers.ClowderProvider, error) {
+	p.Cache.AddPossibleGVKFromIdent(
+		SharedDBDeployment,
+		SharedDBService,
+		SharedDBPVC,
+		SharedDBSecret,
+		SharedDBAppSecret,
+	)
 	return &sharedDbProvider{Provider: *p}, nil
 }
 

diff --git a/controllers/cloud.redhat.com/providers/deployment/default.go b/controllers/cloud.redhat.com/providers/deployment/default.go
@@ -3,13 +3,19 @@ package deployment
 import (
 	crd "github.com/RedHatInsights/clowder/apis/cloud.redhat.com/v1alpha1"
 	"github.com/RedHatInsights/clowder/controllers/cloud.redhat.com/providers"
+	rc "github.com/RedHatInsights/rhc-osdk-utils/resource_cache"
+	apps "k8s.io/api/apps/v1"
 )
 
 type deploymentProvider struct {
 	providers.Provider
 }
 
+// CoreDeployment is the deployment for the apps deployments.
+var CoreDeployment = rc.NewMultiResourceIdent(ProvName, "core_deployment", &apps.Deployment{})
+
 func NewDeploymentProvider(p *providers.Provider) (providers.ClowderProvider, error) {
+	p.Cache.AddPossibleGVKFromIdent(CoreDeployment)
 	return &deploymentProvider{Provider: *p}, nil
 }
 

diff --git a/controllers/cloud.redhat.com/providers/deployment/provider.go b/controllers/cloud.redhat.com/providers/deployment/provider.go
@@ -2,17 +2,11 @@ package deployment
 
 import (
 	"github.com/RedHatInsights/clowder/controllers/cloud.redhat.com/providers"
-	apps "k8s.io/api/apps/v1"
-
-	rc "github.com/RedHatInsights/rhc-osdk-utils/resource_cache"
 )
 
 // ProvName sets the provider name identifier
 var ProvName = "deployment"
 
-// CoreDeployment is the deployment for the apps deployments.
-var CoreDeployment = rc.NewMultiResourceIdent(ProvName, "core_deployment", &apps.Deployment{})
-
 // GetEnd returns the correct end provider.
 func GetDeployment(c *providers.Provider) (providers.ClowderProvider, error) {
 	return NewDeploymentProvider(c)

diff --git a/controllers/cloud.redhat.com/providers/featureflags/localfeatureflags.go b/controllers/cloud.redhat.com/providers/featureflags/localfeatureflags.go
@@ -47,6 +47,14 @@ type localFeatureFlagsProvider struct {
 
 // NewLocalFeatureFlagsProvider returns a new local featureflags provider object.
 func NewLocalFeatureFlagsProvider(p *providers.Provider) (providers.ClowderProvider, error) {
+	p.Cache.AddPossibleGVKFromIdent(
+		LocalFFDBDeployment,
+		LocalFFService,
+		LocalFFDBDeployment,
+		LocalFFDBService,
+		LocalFFDBPVC,
+		LocalFFDBSecret,
+	)
 	return &localFeatureFlagsProvider{Provider: *p}, nil
 }
 

diff --git a/controllers/cloud.redhat.com/providers/inmemorydb/redis.go b/controllers/cloud.redhat.com/providers/inmemorydb/redis.go
@@ -30,6 +30,16 @@ type localRedis struct {
 	providers.Provider
 }
 
+// NewLocalRedis returns a new local redis provider object.
+func NewLocalRedis(p *providers.Provider) (providers.ClowderProvider, error) {
+	p.Cache.AddPossibleGVKFromIdent(
+		RedisDeployment,
+		RedisService,
+		RedisConfigMap,
+	)
+	return &localRedis{Provider: *p}, nil
+}
+
 func (r *localRedis) EnvProvide() error {
 	return nil
 }
@@ -74,11 +84,6 @@ func (r *localRedis) Provide(app *crd.ClowdApp) error {
 	return providers.CachedMakeComponent(r.Provider.Cache, objList, app, "redis", makeLocalRedis, false, r.Env.IsNodePort())
 }
 
-// NewLocalRedis returns a new local redis provider object.
-func NewLocalRedis(p *providers.Provider) (providers.ClowderProvider, error) {
-	return &localRedis{Provider: *p}, nil
-}
-
 func makeLocalRedis(o obj.ClowdObject, objMap providers.ObjectMap, usePVC bool, nodePort bool) {
 	nn := providers.GetNamespacedName(o, "redis")
 

diff --git a/controllers/cloud.redhat.com/providers/iqe/impl.go b/controllers/cloud.redhat.com/providers/iqe/impl.go
@@ -27,6 +27,8 @@ var IMAGE_IQE_SELENIUM = "quay.io/redhatqe/selenium-standalone"
 
 var IqeSecret = rc.NewSingleResourceIdent("cji", "iqe_secret", &core.Secret{})
 var VaultSecret = rc.NewSingleResourceIdent("cji", "vault_secret", &core.Secret{})
+var IqeClowdJob = rc.NewSingleResourceIdent("cji", "iqe_clowdjob", &batchv1.Job{})
+var ClowdJob = rc.NewMultiResourceIdent("cji", "clowdjob", &batchv1.Job{})
 
 func joinNullableSlice(s *[]string) string {
 	if s != nil {