Add recreate + orphan behavior for DDAI (#2509)

Author: khewonc

internal/controller/datadogagentinternal/controller_reconcile_v2_common.go

@@ -8,6 +8,7 @@ package datadogagentinternal
 import (
 	"context"
 	"fmt"
+	"maps"
 	"time"
 
 	edsv1alpha1 "github.com/DataDog/extendeddaemonset/api/v1alpha1"
@@ -21,9 +22,11 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
+	apicommon "github.com/DataDog/datadog-operator/api/datadoghq/common"
 	"github.com/DataDog/datadog-operator/api/datadoghq/v1alpha1"
 	"github.com/DataDog/datadog-operator/pkg/agentprofile"
 	"github.com/DataDog/datadog-operator/pkg/condition"
+	"github.com/DataDog/datadog-operator/pkg/constants"
 	"github.com/DataDog/datadog-operator/pkg/controller/utils/comparison"
 	"github.com/DataDog/datadog-operator/pkg/controller/utils/datadog"
 	"github.com/DataDog/datadog-operator/pkg/kubernetes"
@@ -96,6 +99,14 @@ func (r *Reconciler) createOrUpdateDeployment(parentLogger logr.Logger, ddai *v1
 			}
 			logger.Info("Deployment owner reference patched")
 		}
+
+		if restartDeployment(deployment, currentDeployment) {
+			if err = deleteObjectAndOrphanDependents(context.TODO(), logger, r.client, deployment, deployment.GetLabels()[apicommon.AgentDeploymentComponentLabelKey]); err != nil {
+				return result, err
+			}
+			return result, nil
+		}
+
 		// check if same hash
 		needUpdate := !comparison.IsSameSpecMD5Hash(hash, currentDeployment.GetAnnotations())
 		if !needUpdate {
@@ -195,6 +206,14 @@ func (r *Reconciler) createOrUpdateDaemonset(parentLogger logr.Logger, ddai *v1a
 			}
 			logger.Info("Daemonset owner reference patched")
 		}
+
+		if restartDaemonset(daemonset, currentDaemonset) {
+			if err = deleteObjectAndOrphanDependents(context.TODO(), logger, r.client, daemonset, constants.DefaultAgentResourceSuffix); err != nil {
+				return result, err
+			}
+			return result, nil
+		}
+
 		now := metav1.Now()
 
 		// When overriding node labels in <1.7.0, the hash could be updated
@@ -443,3 +462,31 @@ func IsEqualStatus(current *v1alpha1.DatadogAgentInternalStatus, newStatus *v1al
 
 	return condition.IsEqualConditions(current.Conditions, newStatus.Conditions)
 }
+
+func restartDaemonset(daemonset, currentDaemonset *appsv1.DaemonSet) bool {
+	// name change
+	if daemonset.Name != currentDaemonset.Name {
+		return true
+	}
+
+	// selectors are immutable
+	if !maps.Equal(daemonset.Spec.Selector.MatchLabels, currentDaemonset.Spec.Selector.MatchLabels) {
+		return true
+	}
+
+	return false
+}
+
+func restartDeployment(deployment, currentDeployment *appsv1.Deployment) bool {
+	// name change
+	if deployment.Name != currentDeployment.Name {
+		return true
+	}
+
+	// selectors are immutable
+	if !maps.Equal(deployment.Spec.Selector.MatchLabels, currentDeployment.Spec.Selector.MatchLabels) {
+		return true
+	}
+
+	return false
+}

internal/controller/datadogagentinternal/utils.go

@@ -6,19 +6,25 @@
 package datadogagentinternal
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"maps"
 	"strings"
 
 	"github.com/go-logr/logr"
 	"github.com/gobwas/glob"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
 
+	apicommon "github.com/DataDog/datadog-operator/api/datadoghq/common"
 	apiutils "github.com/DataDog/datadog-operator/api/utils"
+	"github.com/DataDog/datadog-operator/pkg/kubernetes"
 )
 
 func mergeAnnotationsLabels(logger logr.Logger, previousVal map[string]string, newVal map[string]string, filter string) map[string]string {
@@ -161,3 +167,21 @@ func getReplicas(currentReplicas, newReplicas *int32) *int32 {
 
 	return apiutils.NewInt32Pointer(*newReplicas)
 }
+
+// delete ALL workloads for a given DDA/DDAI and orphan pods
+func deleteObjectAndOrphanDependents(ctx context.Context, logger logr.Logger, c client.Client, obj client.Object, component string) error {
+	propagationPolicy := metav1.DeletePropagationOrphan
+	selector := labels.SelectorFromSet(labels.Set{
+		kubernetes.AppKubernetesPartOfLabelKey:     obj.GetLabels()[kubernetes.AppKubernetesPartOfLabelKey],
+		apicommon.AgentDeploymentComponentLabelKey: component,
+	})
+	logger.Info("deleting all workloads for matching DDAI", "labels", selector.String())
+	if err := c.DeleteAllOf(ctx, obj, &client.DeleteAllOfOptions{ListOptions: client.ListOptions{LabelSelector: selector, Namespace: obj.GetNamespace()}, DeleteOptions: client.DeleteOptions{PropagationPolicy: &propagationPolicy}}); err != nil {
+		if apierrors.IsNotFound(err) {
+			logger.Info("object not found, skipping deletion", "object", obj.GetName(), "namespace", obj.GetNamespace())
+			return nil
+		}
+		return err
+	}
+	return nil
+}