feat(syncwaves): add DAG ordering for syncwaves

Signed-off-by: SebastienFelix <[email protected]>

Author: SebastienFelix

controller/sync.go

@@ -571,7 +571,7 @@ func hasSharedResourceCondition(app *v1alpha1.Application) (bool, string) {
 // Note, this is not foolproof, since a proper fix would require the CRD record
 // status.observedGeneration coupled with a health.lua that verifies
 // status.observedGeneration == metadata.generation
-func delayBetweenSyncWaves(_ common.SyncPhase, _ int, finalWave bool) error {
+func delayBetweenSyncWaves(_ []common.SyncIdentity, finalWave bool) error {
 	if !finalWave {
 		delaySec := 2
 		if delaySecStr := os.Getenv(EnvVarSyncWaveDelay); delaySecStr != "" {

docs/user-guide/sync-waves.md

@@ -80,6 +80,14 @@ It repeats this process until all phases and waves are in-sync and healthy.
 
 Because an application can have resources that are unhealthy in the first wave, it may be that the app can never get to healthy.
 
+## How Sync Waves Groups work?
+
+On top of Sync waves, Argo CD offers a way to group resources belonging to a same component (examples : Kafka, UI, Database, MyCusonComponent, ...). These are sync wave groups. They are defined by the argocd.argoproj.io/sync-wave-group annotation. The value is an integer that defines the component of which the resource belongs. Sync Wave groups behave like apps within the main app. Resources within a same Sync Wave group will be synced according to their Sync wave's values.
+
+It is possible to define dependencies between Sync Wave groups. These are sync wave group dependencies. They are defined at resource level by the argocd.argoproj.io/sync-wave-group-dependencies annotation. The value is a list of integers, separated by commas. These integers define the Sync Wave groups that need to be synced before the resource in which this annotation is defined.
+
+Note that in order to avoid circular dependencies, values defined in argocd.argoproj.io/sync-wave-group-dependencies will only be taken into account if they are strictly less than the Sync Wave group value.
+
 ## How Do I Configure Phases?
 
 Pre-sync and post-sync can only contain hooks. Apply the hook annotation:
@@ -100,9 +108,11 @@ Specify the wave using the following annotation:
 metadata:
   annotations:
     argocd.argoproj.io/sync-wave: "5"
+    argocd.argoproj.io/sync-wave-group: "2"
+    argocd.argoproj.io/sync-wave-group-dependencies: "0,1"
 ```
 
-Hooks and resources are assigned to wave zero by default. The wave can be negative, so you can create a wave that runs before all other resources.
+Hooks and resources are assigned to wave zero and wave goup zero by default. The wave can be negative, so you can create a wave that runs before all other resources.
 
 ## Examples
 

gitops-engine/pkg/sync/common/types.go

@@ -12,6 +12,10 @@ const (
 	AnnotationSyncOptions = "argocd.argoproj.io/sync-options"
 	// AnnotationSyncWave indicates which wave of the sync the resource or hook should be in
 	AnnotationSyncWave = "argocd.argoproj.io/sync-wave"
+	// AnnotationSyncWaveGroup indicates which wave of the sync the resource or hook should be in
+	AnnotationSyncWaveGroup = "argocd.argoproj.io/sync-wave-group"
+	// AnnotationSyncWaveGroupDependencies indicates which wave of the sync the resource or hook should be in
+	AnnotationSyncWaveGroupDependencies = "argocd.argoproj.io/sync-wave-group-dependencies"
 	// AnnotationKeyHook contains the hook type of a resource
 	AnnotationKeyHook = "argocd.argoproj.io/hook"
 	// AnnotationKeyHookDeletePolicy is the policy of deleting a hook
@@ -59,10 +63,33 @@ type PermissionValidator func(un *unstructured.Unstructured, res *metav1.APIReso
 
 type SyncPhase string
 
+type SyncIdentity struct {
+	Phase     SyncPhase
+	Wave      int
+	WaveGroup int
+}
+
+// Will be used when using Dependency graph
+type GroupIdentity struct {
+	Phase     SyncPhase
+	WaveGroup int
+}
+
+// Will be used when using Dependency graph
+type WaveDependency struct {
+	Origin      GroupIdentity
+	Destination GroupIdentity
+}
+
+// Will be used when using Dependency graph
+type WaveDependencyGraph struct {
+	Dependencies []WaveDependency
+}
+
 // SyncWaveHook is a callback function which will be invoked after each sync wave is successfully
 // applied during a sync operation. The callback indicates which phase and wave it had just
 // executed, and whether or not that wave was the final one.
-type SyncWaveHook func(phase SyncPhase, wave int, final bool) error
+type SyncWaveHook func(t []SyncIdentity, final bool) error
 
 const (
 	SyncPhasePreSync  = "PreSync"

gitops-engine/pkg/sync/doc.go

@@ -75,6 +75,30 @@ that runs before all other resources. The `argocd.argoproj.io/sync-wave` annotat
 	  annotations:
 	    argocd.argoproj.io/sync-wave: "5"
 
+# Sync Waves Groups & Dependencies
+
+The wave groups allow to group resources belonging to a same component (example: Kafka, MongoDB, FrontEnd, ...). Hooks and
+resources are assigned to wave group "Default" by default. The `argocd.argoproj.io/sync-wave-group` annotation assign resource to a
+wave group:
+
+	metadata:
+	  annotations:
+	    argocd.argoproj.io/sync-wave: "5"
+	    argocd.argoproj.io/sync-wave-group: "2"
+
+It is also possible to define dependencies between waves groups, if you want a group of resources to be synced before another group.
+By default, hooks and resources have no dependencies. The `argocd.argoproj.io/sync-wave-group-dependencies` annotation defines the
+wave groups that need to be synced before a resource can:
+
+	metadata:
+	  annotations:
+	    argocd.argoproj.io/sync-wave: "5"
+	    argocd.argoproj.io/sync-wave-group: "2"
+	    argocd.argoproj.io/sync-wave-group-dependencies "0,1"
+
+In the previous example, a resource belonging to wave group "MyCustomComponent" is defined. This resource will be synced only when
+all resources from wave groups "Kafka" and "FrontEnd" has been synced.
+
 # Sync Options
 
 The sync options allows customizing the synchronization of selected resources. The options are specified using the
@@ -89,6 +113,7 @@ How Does It Work Together?
 Syncing process orders the resources in the following precedence:
 
 - The phase
+- The group with respect to group dependencies
 - The wave they are in (lower values first)
 - By kind (e.g. namespaces first)
 - By name

gitops-engine/pkg/sync/sync_context.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"slices"
 	"sort"
 	"strings"
 	"sync"
@@ -416,6 +417,23 @@ func (sc *syncContext) setRunningPhase(tasks []*syncTask, isPendingDeletion bool
 func (sc *syncContext) Sync() {
 	sc.log.WithValues("skipHooks", sc.skipHooks, "started", sc.started()).Info("Syncing")
 	tasks, ok := sc.getSyncTasks()
+
+	// dependencyGraph will be used to detect circular dependencies and allow for direct dependencies definition
+	// in argocd.argoproj.io/sync-wave-group-dependencies
+	dependencyGraph := common.WaveDependencyGraph{Dependencies: make([]common.WaveDependency, 0)}
+	for _, task := range tasks {
+		if task.targetObj != nil {
+			origin := common.GroupIdentity{Phase: task.phase, WaveGroup: task.wave()}
+			for dependency := range task.waveGroupDependencies() {
+				destination := common.GroupIdentity{Phase: task.phase, WaveGroup: dependency}
+				waveDependency := &common.WaveDependency{Origin: origin, Destination: destination}
+				if !slices.Contains(dependencyGraph.Dependencies, *waveDependency) {
+					dependencyGraph.Dependencies = append(dependencyGraph.Dependencies, *waveDependency)
+				}
+			}
+		}
+	}
+
 	if !ok {
 		sc.setOperationPhase(common.OperationFailed, "one or more synchronization tasks are not valid")
 		return
@@ -560,26 +578,29 @@ func (sc *syncContext) Sync() {
 		return
 	}
 
-	// remove any tasks not in this wave
+	// remove any tasks which have unsynced dependencies
 	phase := tasks.phase()
-	wave := tasks.wave()
-	finalWave := phase == tasks.lastPhase() && wave == tasks.lastWave()
+	independantSyncIdentities := tasks.independantSyncIdentities()
+	allSyncIdentities := tasks.syncIdentities()
 
 	// if it is the last phase/wave and the only remaining tasks are non-hooks, the we are successful
 	// EVEN if those objects subsequently degraded
 	// This handles the common case where neither hooks or waves are used and a sync equates to simply an (asynchronous) kubectl apply of manifests, which succeeds immediately.
-	remainingTasks := tasks.Filter(func(t *syncTask) bool { return t.phase != phase || wave != t.wave() || t.isHook() })
+	remainingTasks := tasks.Filter(func(t *syncTask) bool {
+		return !slices.Contains(independantSyncIdentities, t.identity()) || t.isHook()
+	})
 
-	sc.log.WithValues("phase", phase, "wave", wave, "tasks", tasks, "syncFailTasks", syncFailTasks).V(1).Info("Filtering tasks in correct phase and wave")
-	tasks = tasks.Filter(func(t *syncTask) bool { return t.phase == phase && t.wave() == wave })
+	sc.log.WithValues("phase", phase, "independantSyncIdentities", independantSyncIdentities, "tasks", tasks, "syncFailTasks", syncFailTasks).V(1).Info("Filtering tasks in correct phase and wave")
+	tasks = tasks.Filter(func(t *syncTask) bool { return slices.Contains(independantSyncIdentities, t.identity()) })
 
 	sc.setOperationPhase(common.OperationRunning, "one or more tasks are running")
 
 	sc.log.WithValues("tasks", tasks).V(1).Info("Wet-run")
 	runState := sc.runTasks(tasks, false)
 
 	if sc.syncWaveHook != nil && runState != failed {
-		err := sc.syncWaveHook(phase, wave, finalWave)
+		finalWave := phase == tasks.lastPhase() && len(independantSyncIdentities) == len(allSyncIdentities)
+		err := sc.syncWaveHook(independantSyncIdentities, finalWave)
 		if err != nil {
 			sc.deleteHooks(hooksPendingDeletionFailed)
 			sc.setOperationPhase(common.OperationFailed, fmt.Sprintf("SyncWaveHook failed: %v", err))
@@ -909,52 +930,61 @@ func (sc *syncContext) getSyncTasks() (_ syncTasks, successful bool) {
 	}
 
 	// for prune tasks, modify the waves for proper cleanup i.e reverse of sync wave (creation order)
-	pruneTasks := make(map[int][]*syncTask)
+
+	tasksByWaveGroup := make(map[int][]*syncTask)
 	for _, task := range tasks {
-		if task.isPrune() {
-			pruneTasks[task.wave()] = append(pruneTasks[task.wave()], task)
-		}
+		tasksByWaveGroup[task.waveGroup()] = append(tasksByWaveGroup[task.waveGroup()], task)
 	}
+	for waveGroup := range tasksByWaveGroup {
+		pruneTasks := make(map[int][]*syncTask)
+		for _, task := range tasksByWaveGroup[waveGroup] {
+			if task.isPrune() {
+				pruneTasks[task.wave()] = append(pruneTasks[task.wave()], task)
+			}
+		}
 
-	var uniquePruneWaves []int
-	for k := range pruneTasks {
-		uniquePruneWaves = append(uniquePruneWaves, k)
-	}
-	sort.Ints(uniquePruneWaves)
+		var uniquePruneWaves []int
+		for k := range pruneTasks {
+			uniquePruneWaves = append(uniquePruneWaves, k)
+		}
+		sort.Ints(uniquePruneWaves)
 
-	// reorder waves for pruning tasks using symmetric swap on prune waves
-	n := len(uniquePruneWaves)
-	for i := 0; i < n/2; i++ {
-		// waves to swap
-		startWave := uniquePruneWaves[i]
-		endWave := uniquePruneWaves[n-1-i]
+		// reorder waves for pruning tasks using symmetric swap on prune waves
+		n := len(uniquePruneWaves)
+		for j := 0; j < n/2; j++ {
+			// waves to swap
+			startWave := uniquePruneWaves[j]
+			endWave := uniquePruneWaves[n-1-j]
 
-		for _, task := range pruneTasks[startWave] {
-			task.waveOverride = &endWave
-		}
+			for _, task := range pruneTasks[startWave] {
+				task.waveOverride = &endWave
+			}
 
-		for _, task := range pruneTasks[endWave] {
-			task.waveOverride = &startWave
+			for _, task := range pruneTasks[endWave] {
+				task.waveOverride = &startWave
+			}
 		}
-	}
 
-	// for pruneLast tasks, modify the wave to sync phase last wave of tasks + 1
-	// to ensure proper cleanup, syncPhaseLastWave should also consider prune tasks to determine last wave
-	syncPhaseLastWave := 0
-	for _, task := range tasks {
-		if task.phase == common.SyncPhaseSync {
-			if task.wave() > syncPhaseLastWave {
-				syncPhaseLastWave = task.wave()
+		// for pruneLast tasks, modify the wave to sync phase last wave of tasks + 1
+		// to ensure proper cleanup, syncPhaseLastWave should also consider prune tasks to determine last wave
+
+		syncPhaseLastWave := 0
+		for _, task := range tasksByWaveGroup[waveGroup] {
+			if task.phase == common.SyncPhaseSync {
+				if task.wave() > syncPhaseLastWave {
+					syncPhaseLastWave = task.wave()
+				}
 			}
 		}
-	}
-	syncPhaseLastWave = syncPhaseLastWave + 1
+		syncPhaseLastWave = syncPhaseLastWave + 1
 
-	for _, task := range tasks {
-		if task.isPrune() &&
-			(sc.pruneLast || resourceutil.HasAnnotationOption(task.liveObj, common.AnnotationSyncOptions, common.SyncOptionPruneLast)) {
-			task.waveOverride = &syncPhaseLastWave
+		for _, task := range tasksByWaveGroup[waveGroup] {
+			if task.isPrune() &&
+				(sc.pruneLast || resourceutil.HasAnnotationOption(task.liveObj, common.AnnotationSyncOptions, common.SyncOptionPruneLast)) {
+				task.waveOverride = &syncPhaseLastWave
+			}
 		}
+
 	}
 
 	tasks.Sort()