fix: race bug running more then one pod at a time

lockwobr · lockwobr · commit e460a6565b7b · 2025-03-26T10:16:17.000-07:00
diff --git a/chart/templates/skyhook-crd.yaml b/chart/templates/skyhook-crd.yaml
@@ -431,8 +431,8 @@ spec:
                   concert without needing to delete to ad in discovery of the issue.
                 type: boolean
               podNonInterruptLabels:
-                description: PodNonInteruptLabels are a set of labels we want to monitor
-                  pods for whether they Interruptible
+                description: PodNonInterruptLabels are a set of labels we want to
+                  monitor pods for whether they Interruptible
                 properties:
                   matchExpressions:
                     description: matchExpressions is a list of label selector requirements.
diff --git a/k8s-tests/chainsaw/skyhook/interrupt-grouping/assert.yaml b/k8s-tests/chainsaw/skyhook/interrupt-grouping/assert.yaml
@@ -18,16 +18,9 @@
 # LICENSE END
 # 
 
-
-
-
-
-
-
 kind: Pod
 apiVersion: v1
 metadata:
-  generateName: interrupt-grouping-dax-1.2.3-apply-
   namespace: skyhook
   labels:
     skyhook.nvidia.com/name: interrupt-grouping
@@ -61,7 +54,6 @@ spec:
 kind: Pod
 apiVersion: v1
 metadata:
-  generateName: interrupt-grouping-dax-1.2.3-config-
   namespace: skyhook
   labels:
     skyhook.nvidia.com/name: interrupt-grouping
@@ -92,7 +84,6 @@ spec:
 kind: Pod
 apiVersion: v1
 metadata:
-  generateName: interrupt-grouping-interrupt-service-
   namespace: skyhook
   labels:
     skyhook.nvidia.com/name: interrupt-grouping
@@ -132,7 +123,6 @@ spec:
 kind: Pod
 apiVersion: v1
 metadata:
-  generateName: interrupt-grouping-dax-1.2.3-post-interrupt-
   namespace: skyhook
   labels:
     skyhook.nvidia.com/name: interrupt-grouping
diff --git a/k8s-tests/chainsaw/skyhook/interrupt/assert.yaml b/k8s-tests/chainsaw/skyhook/interrupt/assert.yaml
@@ -18,17 +18,10 @@
 # LICENSE END
 # 
 
-
-
-
-
-
-
 ---
 kind: Pod
 apiVersion: v1
 metadata:
-  generateName: interrupt-invalid-1.2.3-uninstall-
   namespace: skyhook
   labels:
     skyhook.nvidia.com/name: interrupt
@@ -63,7 +56,6 @@ spec:
 kind: Pod
 apiVersion: v1
 metadata:
-  generateName: interrupt-jason-1.3.2-apply-
   namespace: skyhook
   labels:
     skyhook.nvidia.com/name: interrupt
@@ -126,7 +118,6 @@ data:
 kind: Pod
 apiVersion: v1
 metadata:
-  generateName: interrupt-interrupt-service-
   namespace: skyhook
   labels:
     skyhook.nvidia.com/name: interrupt
diff --git a/k8s-tests/chainsaw/skyhook/interrupt/pod.yaml b/k8s-tests/chainsaw/skyhook/interrupt/pod.yaml
@@ -18,12 +18,6 @@
 # LICENSE END
 # 
 
-
-
-
-
-
-
 ---
 apiVersion: v1
 kind: Pod
@@ -64,8 +58,8 @@ metadata:
   name: invalid-package
   namespace: skyhook
   labels:
-      skyhook.nvidia.com/package: invalid-1.2.3
-      skyhook.nvidia.com/name: interrupt
+    skyhook.nvidia.com/package: invalid-1.2.3
+    skyhook.nvidia.com/name: interrupt
   annotations:
     skyhook.nvidia.com/package: '{"name":"invalid","version":"1.2.3","stage":"apply","skyhook":"interrupt","image":"ghcr.io/nvidia/skyhook/agentless"}'
 spec:
diff --git a/k8s-tests/chainsaw/skyhook/interrupt/skyhook.yaml b/k8s-tests/chainsaw/skyhook/interrupt/skyhook.yaml
@@ -18,12 +18,6 @@
 # LICENSE END
 # 
 
-
-
-
-
-
-
 ---
 apiVersion: skyhook.nvidia.com/v1alpha1
 kind: Skyhook
diff --git a/k8s-tests/chainsaw/skyhook/simple-skyhook/assert.yaml b/k8s-tests/chainsaw/skyhook/simple-skyhook/assert.yaml
@@ -18,17 +18,10 @@
 # LICENSE END
 # 
 
-
-
-
-
-
-
 ---
 kind: Pod
 apiVersion: v1
 metadata:
-  generateName: simple-skyhook-spencer-3.2.3-apply-
   namespace: skyhook
   labels:
     skyhook.nvidia.com/name: simple-skyhook
diff --git a/k8s-tests/chainsaw/skyhook/uninstall-upgrade-skyhook/assert-update-no-packages.yaml b/k8s-tests/chainsaw/skyhook/uninstall-upgrade-skyhook/assert-update-no-packages.yaml
@@ -18,17 +18,10 @@
 # LICENSE END
 # 
 
-
-
-
-
-
-
 ---
 kind: Pod
 apiVersion: v1
 metadata:
-  generateName: uninstall-upgrade-skyhook-dogs-1.2.5-uninstall-
   namespace: skyhook
   labels:
     skyhook.nvidia.com/name: uninstall-upgrade-skyhook
@@ -63,7 +56,6 @@ spec:
 kind: Pod
 apiVersion: v1
 metadata:
-  generateName: uninstall-upgrade-skyhook-nullptr-2.0.1-uninstall-
   namespace: skyhook
   labels:
     skyhook.nvidia.com/name: uninstall-upgrade-skyhook
diff --git a/k8s-tests/chainsaw/skyhook/uninstall-upgrade-skyhook/assert-update.yaml b/k8s-tests/chainsaw/skyhook/uninstall-upgrade-skyhook/assert-update.yaml
@@ -18,17 +18,10 @@
 # LICENSE END
 # 
 
-
-
-
-
-
-
 ---
 kind: Pod
 apiVersion: v1
 metadata:
-  generateName: uninstall-upgrade-skyhook-cats-6.2.0-uninstall-
   namespace: skyhook
   labels:
     skyhook.nvidia.com/name: uninstall-upgrade-skyhook
@@ -77,7 +70,6 @@ spec:
 kind: Pod
 apiVersion: v1
 metadata:
-  generateName: uninstall-upgrade-skyhook-dogs-1.2.6-uninstall-
   namespace: skyhook
   labels:
     skyhook.nvidia.com/name: uninstall-upgrade-skyhook
@@ -112,7 +104,6 @@ spec:
 kind: Pod
 apiVersion: v1
 metadata:
-  generateName: uninstall-upgrade-skyhook-nullptr-2.0.1-upgrade-
   namespace: skyhook
   labels:
     skyhook.nvidia.com/name: uninstall-upgrade-skyhook
diff --git a/k8s-tests/chainsaw/skyhook/validate-packages/assert-update.yaml b/k8s-tests/chainsaw/skyhook/validate-packages/assert-update.yaml
@@ -18,12 +18,6 @@
 # LICENSE END
 # 
 
-
-
-
-
-
-
 apiVersion: v1
 kind: Node
 metadata:
@@ -73,24 +67,24 @@ status:
   observedGeneration: 3
   nodeState:
     (values(@)):
-    - invalid-image|3.2.3:
-        name: invalid-image
-        state: complete
-        version: '3.2.3'
+    - invalid-env|5.4.3:
         image: ghcr.io/nvidia/skyhook/agentless
+        name: invalid-env
         stage: config
-      invalid-resources|1.2.3:
-        name: invalid-resources
         state: complete
-        version: '1.2.3'
+        version: 5.4.3
+      invalid-image|3.2.3:
         image: ghcr.io/nvidia/skyhook/agentless
+        name: invalid-image
         stage: config
-      invalid-env|5.4.3:
-        name: invalid-env
         state: complete
-        version: '5.4.3'
+        version: 3.2.3
+      invalid-resources|1.2.3:
         image: ghcr.io/nvidia/skyhook/agentless
+        name: invalid-resources
         stage: config
+        state: complete
+        version: 1.2.3
   nodeStatus:
    # grab values should be one and is complete
     (values(@)):
diff --git a/operator/Makefile b/operator/Makefile
@@ -140,7 +140,7 @@ $(REPORTING):
 	mkdir -p $@
 
 .PHONY: test
-test:: reporting manifests generate fmt vet lint helm-tests e2e-tests unit-tests ## Run all tests.
+test:: reporting manifests generate fmt vet lint unit-tests e2e-tests helm-tests ## Run all tests.
 
 ifndef CI
 ## we double define test so we can do thing different if in ci vs local
@@ -166,7 +166,7 @@ unit-tests: reporting manifests generate envtest ginkgo kill ## Run unit tests.
 ## https://kyverno.github.io/chainsaw/latest/configuration/file/
 CHAINSAW_ARGS:=--exec-timeout 30s --parallel 2
 
-e2e-tests: chainsaw run ## Run end to end tests.
+e2e-tests: chainsaw install run ## Run end to end tests.
 	## requires a cluster to be running with access
 	## locally use kind to create clusters
 	## in ci, the plan current is to have a real cluster, and create a node pool for testing
diff --git a/operator/api/v1alpha1/skyhook_types.go b/operator/api/v1alpha1/skyhook_types.go
@@ -52,8 +52,8 @@ type SkyhookSpec struct {
 	//+kubebuilder:default=false
 	Pause bool `json:"pause,omitempty"`
 
-	// PodNonInteruptLabels are a set of labels we want to monitor pods for whether they Interruptible
-	PodNonInteruptLabels metav1.LabelSelector `json:"podNonInterruptLabels,omitempty"`
+	// PodNonInterruptLabels are a set of labels we want to monitor pods for whether they Interruptible
+	PodNonInterruptLabels metav1.LabelSelector `json:"podNonInterruptLabels,omitempty"`
 
 	// NodeSelector are a set of labels we want to monitor nodes for applying packages too
 	NodeSelector metav1.LabelSelector `json:"nodeSelectors,omitempty"`
@@ -466,6 +466,7 @@ func (left *NodeState) Equal(right *NodeState) bool {
 	return reflect.DeepEqual(left, right)
 }
 
+// GetComplete returns a list of packages that are complete
 func (ns *NodeState) GetComplete(packages Packages, interrupt map[string][]*Interrupt, config map[string][]string) []string {
 
 	ret := make([]string, 0)
@@ -488,6 +489,7 @@ func (ns *NodeState) GetComplete(packages Packages, interrupt map[string][]*Inte
 	return ret
 }
 
+// IsPackageComplete checks if a package is complete
 func (ns *NodeState) IsPackageComplete(_package Package, interrupt map[string][]*Interrupt, config map[string][]string) bool {
 
 	packageStatus, found := (*ns)[_package.GetUniqueName()]
@@ -504,6 +506,7 @@ func (ns *NodeState) IsPackageComplete(_package Package, interrupt map[string][]
 	return false
 }
 
+// ProgressSkipped checks if a package is skipped and should be progressed to complete
 func (ns *NodeState) ProgressSkipped(packages Packages, interrupt map[string][]*Interrupt, config map[string][]string) bool {
 	ret := false
 	for _, s := range *ns {
@@ -574,8 +577,8 @@ type State string
 const (
 	METADATA_PREFIX string = "skyhook.nvidia.com"
 	StateComplete   State  = "complete"
-	StateInProgress State  = "in_progress"
-	StateSkipped    State  = "skipped"
+	StateInProgress State  = "in_progress" // this means its actually running, pod started
+	StateSkipped    State  = "skipped"     // this means this package, stage are skipped mostly for some parts of the lifecycle
 	StateErroring   State  = "erroring"
 	StateUnknown    State  = "unknown"
 )
diff --git a/operator/api/v1alpha1/zz_generated.deepcopy.go b/operator/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/operator/config/crd/bases/skyhook.nvidia.com_skyhooks.yaml b/operator/config/crd/bases/skyhook.nvidia.com_skyhooks.yaml
@@ -437,8 +437,8 @@ spec:
                   concert without needing to delete to ad in discovery of the issue.
                 type: boolean
               podNonInterruptLabels:
-                description: PodNonInteruptLabels are a set of labels we want to monitor
-                  pods for whether they Interruptible
+                description: PodNonInterruptLabels are a set of labels we want to
+                  monitor pods for whether they Interruptible
                 properties:
                   matchExpressions:
                     description: matchExpressions is a list of label selector requirements.
diff --git a/operator/internal/controller/skyhook_controller.go b/operator/internal/controller/skyhook_controller.go
diff --git a/operator/internal/controller/skyhook_controller_test.go b/operator/internal/controller/skyhook_controller_test.go
