Adding support for subpath for model caching

api/apps/v1alpha1/nimcache_types.go

@@ -128,6 +128,7 @@ type PersistentVolumeClaim struct {
 	Size string `json:"size,omitempty"`
 	// VolumeAccessMode is the volume access mode of the PVC
 	VolumeAccessMode corev1.PersistentVolumeAccessMode `json:"volumeAccessMode,omitempty"`
+	SubPath          string                            `json:"subPath,omitempty"`
 }
 
 // NIMCacheStatus defines the observed state of NIMCache

api/apps/v1alpha1/nimservice_types.go

@@ -376,7 +376,7 @@ func (n *NIMService) GetVolumesMounts() []corev1.Volume {
 }
 
 // GetVolumes returns volumes for the NIMService container
-func (n *NIMService) GetVolumes(modelPVC string) []corev1.Volume {
+func (n *NIMService) GetVolumes(modelPVC PersistentVolumeClaim) []corev1.Volume {
 	// TODO: Fetch actual PVC name from associated NIMCache obj
 	volumes := []corev1.Volume{
 		{
@@ -391,7 +391,7 @@ func (n *NIMService) GetVolumes(modelPVC string) []corev1.Volume {
 			Name: "model-store",
 			VolumeSource: corev1.VolumeSource{
 				PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
-					ClaimName: modelPVC,
+					ClaimName: *modelPVC.Name,
 				},
 			},
 		},
@@ -401,11 +401,12 @@ func (n *NIMService) GetVolumes(modelPVC string) []corev1.Volume {
 }
 
 // GetVolumeMounts returns volumes for the NIMService container
-func (n *NIMService) GetVolumeMounts() []corev1.VolumeMount {
+func (n *NIMService) GetVolumeMounts(modelPVC PersistentVolumeClaim) []corev1.VolumeMount {
 	volumeMounts := []corev1.VolumeMount{
 		{
 			Name:      "model-store",
 			MountPath: "/model-store",
+			SubPath:   modelPVC.SubPath,
 		},
 		{
 			Name:      "dshm",
@@ -432,8 +433,8 @@ func (n *NIMService) GetNIMCacheProfile() string {
 }
 
 // GetExternalPVC returns the external PVC name to use for the NIMService deployment
-func (n *NIMService) GetExternalPVC() *string {
-	return n.Spec.Storage.PVC.Name
+func (n *NIMService) GetExternalPVC() PersistentVolumeClaim {
+	return n.Spec.Storage.PVC
 }
 
 // GetHPASpec returns the HPA spec for the NIMService deployment

bundle/manifests/apps.nvidia.com_nimcaches.yaml

@@ -220,6 +220,8 @@ spec:
                         description: StorageClass to be used for PVC creation. Leave
                           it as empty if the PVC is already created.
                         type: string
+                      subPath:
+                        type: string
                       volumeAccessMode:
                         description: VolumeAccessMode is the volume access mode of
                           the PVC

bundle/manifests/apps.nvidia.com_nimpipelines.yaml

@@ -2138,6 +2138,8 @@ spec:
                                   description: StorageClass to be used for PVC creation.
                                     Leave it as empty if the PVC is already created.
                                   type: string
+                                subPath:
+                                  type: string
                                 volumeAccessMode:
                                   description: VolumeAccessMode is the volume access
                                     mode of the PVC

bundle/manifests/apps.nvidia.com_nimservices.yaml

@@ -2084,6 +2084,8 @@ spec:
                         description: StorageClass to be used for PVC creation. Leave
                           it as empty if the PVC is already created.
                         type: string
+                      subPath:
+                        type: string
                       volumeAccessMode:
                         description: VolumeAccessMode is the volume access mode of
                           the PVC

config/crd/bases/apps.nvidia.com_nimcaches.yaml

@@ -220,6 +220,8 @@ spec:
                         description: StorageClass to be used for PVC creation. Leave
                           it as empty if the PVC is already created.
                         type: string
+                      subPath:
+                        type: string
                       volumeAccessMode:
                         description: VolumeAccessMode is the volume access mode of
                           the PVC

config/crd/bases/apps.nvidia.com_nimpipelines.yaml

@@ -2138,6 +2138,8 @@ spec:
                                   description: StorageClass to be used for PVC creation.
                                     Leave it as empty if the PVC is already created.
                                   type: string
+                                subPath:
+                                  type: string
                                 volumeAccessMode:
                                   description: VolumeAccessMode is the volume access
                                     mode of the PVC

config/crd/bases/apps.nvidia.com_nimservices.yaml

@@ -2084,6 +2084,8 @@ spec:
                         description: StorageClass to be used for PVC creation. Leave
                           it as empty if the PVC is already created.
                         type: string
+                      subPath:
+                        type: string
                       volumeAccessMode:
                         description: VolumeAccessMode is the volume access mode of
                           the PVC

deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nimcaches.yaml

@@ -220,6 +220,8 @@ spec:
                         description: StorageClass to be used for PVC creation. Leave
                           it as empty if the PVC is already created.
                         type: string
+                      subPath:
+                        type: string
                       volumeAccessMode:
                         description: VolumeAccessMode is the volume access mode of
                           the PVC
@@ -266,7 +268,6 @@ spec:
                   type: object
                 type: array
             required:
-            - resources
             - source
             - storage
             type: object

deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nimpipelines.yaml

@@ -2138,6 +2138,8 @@ spec:
                                   description: StorageClass to be used for PVC creation.
                                     Leave it as empty if the PVC is already created.
                                   type: string
+                                subPath:
+                                  type: string
                                 volumeAccessMode:
                                   description: VolumeAccessMode is the volume access
                                     mode of the PVC

deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nimservices.yaml

@@ -2084,6 +2084,8 @@ spec:
                         description: StorageClass to be used for PVC creation. Leave
                           it as empty if the PVC is already created.
                         type: string
+                      subPath:
+                        type: string
                       volumeAccessMode:
                         description: VolumeAccessMode is the volume access mode of
                           the PVC

internal/controller/nimcache_controller.go

@@ -735,6 +735,7 @@ func constructJob(nimCache *appsv1alpha1.NIMCache) (*batchv1.Job, error) {
 					{
 						Name:      "nim-cache-volume",
 						MountPath: "/output",
+						SubPath:   nimCache.Spec.Storage.PVC.SubPath,
 					},
 				},
 				SecurityContext: &corev1.SecurityContext{
@@ -779,6 +780,7 @@ func constructJob(nimCache *appsv1alpha1.NIMCache) (*batchv1.Job, error) {
 					{
 						Name:      "nim-cache-volume",
 						MountPath: "/model-store",
+						SubPath:   nimCache.Spec.Storage.PVC.SubPath,
 					},
 				},
 				Resources: corev1.ResourceRequirements{

internal/controller/platform/standalone/nimservice.go

@@ -133,7 +133,7 @@ func (r *NIMServiceReconciler) reconcileNIMService(ctx context.Context, nimServi
 	}
 
 	deploymentParams := nimService.GetDeploymentParams()
-	modelPVC := ""
+	var modelPVC appsv1alpha1.PersistentVolumeClaim
 	modelProfile := ""
 
 	// If external PVC is provided, use that as model-store
@@ -151,8 +151,8 @@ func (r *NIMServiceReconciler) reconcileNIMService(ctx context.Context, nimServi
 			logger.Info("overriding model profile", "profile", profile)
 			modelProfile = profile
 		}
-	} else if externalPVC := nimService.GetExternalPVC(); externalPVC != nil {
-		modelPVC = *externalPVC
+	} else if externalPVC := nimService.GetExternalPVC(); externalPVC != (appsv1alpha1.PersistentVolumeClaim{}) {
+		modelPVC = externalPVC
 	} else if nimService.Spec.Storage.PVC.Create != nil && *nimService.Spec.Storage.PVC.Create {
 		modelPVC, err = r.reconcilePVC(ctx, nimService)
 		if err != nil {
@@ -164,10 +164,9 @@ func (r *NIMServiceReconciler) reconcileNIMService(ctx context.Context, nimServi
 		logger.Error(err, "failed to determine PVC for model-store")
 		return ctrl.Result{}, err
 	}
-
 	// Setup volume mounts with model store
 	deploymentParams.Volumes = nimService.GetVolumes(modelPVC)
-	deploymentParams.VolumeMounts = nimService.GetVolumeMounts()
+	deploymentParams.VolumeMounts = nimService.GetVolumeMounts(modelPVC)
 
 	// Setup env for explicit override profile is specified
 	if modelProfile != "" {
@@ -313,42 +312,43 @@ func (r *NIMServiceReconciler) syncResource(ctx context.Context, obj client.Obje
 }
 
 // getNIMCachePVC returns PVC backing the NIM cache instance
-func (r *NIMServiceReconciler) getNIMCachePVC(ctx context.Context, nimService *appsv1alpha1.NIMService) (string, error) {
+func (r *NIMServiceReconciler) getNIMCachePVC(ctx context.Context, nimService *appsv1alpha1.NIMService) (appsv1alpha1.PersistentVolumeClaim, error) {
 	logger := log.FromContext(ctx)
 
 	if nimService.GetNIMCacheName() == "" {
 		// NIM cache PVC is not used
-		return "", nil
+		return appsv1alpha1.PersistentVolumeClaim{}, nil
 	}
-
 	// Lookup NIMCache instance in the same namespace as the NIMService instance
 	nimCache := &appsv1alpha1.NIMCache{}
 	if err := r.Get(ctx, types.NamespacedName{Name: nimService.GetNIMCacheName(), Namespace: nimService.Namespace}, nimCache); err != nil {
 		logger.Error(err, "unable to fetch nimcache", "nimcache", nimService.GetNIMCacheName(), "nimservice", nimService.Name)
-		return "", err
+		return appsv1alpha1.PersistentVolumeClaim{}, err
 	}
-
 	// Get the status of NIMCache
 	if nimCache.Status.State != appsv1alpha1.NimCacheStatusReady {
-		return "", fmt.Errorf("nimcache %s is not ready, nimservice %s", nimCache.GetName(), nimService.GetName())
+		return appsv1alpha1.PersistentVolumeClaim{}, fmt.Errorf("nimcache %s is not ready, nimservice %s", nimCache.GetName(), nimService.GetName())
 	}
 
 	if nimCache.Status.PVC == "" {
-		return "", fmt.Errorf("missing PVC for the nimcache instance %s, nimservice %s", nimCache.GetName(), nimService.GetName())
+		return appsv1alpha1.PersistentVolumeClaim{}, fmt.Errorf("missing PVC for the nimcache instance %s, nimservice %s", nimCache.GetName(), nimService.GetName())
 	}
 
+	if nimCache.Spec.Storage.PVC.Name == nil {
+		nimCache.Spec.Storage.PVC.Name = &nimCache.Status.PVC
+	}
 	// Get the underlying PVC for the NIMCache instance
-	return nimCache.Status.PVC, nil
+	return nimCache.Spec.Storage.PVC, nil
 }
 
-func (r *NIMServiceReconciler) reconcilePVC(ctx context.Context, nimService *appsv1alpha1.NIMService) (string, error) {
+func (r *NIMServiceReconciler) reconcilePVC(ctx context.Context, nimService *appsv1alpha1.NIMService) (appsv1alpha1.PersistentVolumeClaim, error) {
 	logger := r.GetLogger()
 	pvcName := nimService.GetPVCName(nimService.Spec.Storage.PVC)
 	pvcNamespacedName := types.NamespacedName{Name: pvcName, Namespace: nimService.GetNamespace()}
 	pvc := &corev1.PersistentVolumeClaim{}
 	err := r.Get(ctx, pvcNamespacedName, pvc)
 	if err != nil && client.IgnoreNotFound(err) != nil {
-		return "", err
+		return appsv1alpha1.PersistentVolumeClaim{}, err
 	}
 
 	// If PVC does not exist, create a new one if creation flag is enabled
@@ -357,28 +357,28 @@ func (r *NIMServiceReconciler) reconcilePVC(ctx context.Context, nimService *app
 			pvc, err = shared.ConstructPVC(nimService.Spec.Storage.PVC, metav1.ObjectMeta{Name: pvcName, Namespace: nimService.GetNamespace()})
 			if err != nil {
 				logger.Error(err, "Failed to construct pvc", "name", pvc.Name)
-				return "", err
+				return appsv1alpha1.PersistentVolumeClaim{}, err
 			}
 			if err := controllerutil.SetControllerReference(nimService, pvc, r.GetScheme()); err != nil {
-				return "", err
+				return appsv1alpha1.PersistentVolumeClaim{}, err
 			}
 			err = r.Create(ctx, pvc)
 			if err != nil {
 				logger.Error(err, "Failed to create pvc", "name", pvc.Name)
-				return "", err
+				return appsv1alpha1.PersistentVolumeClaim{}, err
 			}
 			logger.Info("Created PVC for NIM Service", "pvc", pvcName)
 
 			conditions.UpdateCondition(&nimService.Status.Conditions, appsv1alpha1.NimCacheConditionPVCCreated, metav1.ConditionTrue, "PVCCreated", "The PVC has been created for storing NIM")
 			nimService.Status.State = appsv1alpha1.NimCacheStatusPVCCreated
 			if err := r.Status().Update(ctx, nimService); err != nil {
 				logger.Error(err, "Failed to update status", "NIMService", nimService.Name)
-				return "", err
+				return appsv1alpha1.PersistentVolumeClaim{}, err
 			}
 		} else {
 			logger.Error(err, "PVC doesn't exist and auto-creation is not enabled", "name", pvcNamespacedName)
-			return "", err
+			return appsv1alpha1.PersistentVolumeClaim{}, err
 		}
 	}
-	return pvcName, nil
+	return nimService.Spec.Storage.PVC, nil
 }

manifests/deployment.yaml

@@ -36,6 +36,7 @@ spec:
         {{- range .VolumeMounts }}
         - name: {{ .Name }}
           mountPath: {{ .MountPath }}
+          subPath: {{ .SubPath }}
         {{- end }}
         env:
         {{- range .Env }}