diff --git a/crds/embedded/virtualmachineinstances.yaml b/crds/embedded/virtualmachineinstances.yaml index c6624ebef9..196797aa2f 100644 --- a/crds/embedded/virtualmachineinstances.yaml +++ b/crds/embedded/virtualmachineinstances.yaml @@ -3076,6 +3076,8 @@ spec: path: description: Path defines the path to disk file in the container type: string + hotpluggable: + type: boolean required: - image type: object diff --git a/crds/embedded/virtualmachines.yaml b/crds/embedded/virtualmachines.yaml index d4774864c0..b99399f642 100644 --- a/crds/embedded/virtualmachines.yaml +++ b/crds/embedded/virtualmachines.yaml @@ -3868,6 +3868,8 @@ spec: description: Path defines the path to disk file in the container type: string + hotpluggable: + type: boolean required: - image type: object @@ -4491,6 +4493,35 @@ spec: description: VolumeSource represents the source of the volume to map to the disk. properties: + containerDisk: + type: object + description: |- + ContainerDisk references a docker image, embedding a qcow or raw disk. + More info: https://kubevirt.gitbooks.io/user-guide/registry-disk.html + properties: + image: + description: Image is the name of the image with + the embedded disk. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + imagePullSecret: + description: ImagePullSecret is the name of the + Docker registry secret required to pull the image. + The secret must already exist. + type: string + path: + description: Path defines the path to disk file + in the container + type: string + hotpluggable: + type: boolean dataVolume: description: |- DataVolume represents the dynamic creation a PVC for this volume as well as diff --git a/images/virt-artifact/patches/028-hotplug-container-disk.patch b/images/virt-artifact/patches/028-hotplug-container-disk.patch new file mode 100644 index 0000000000..852b8d687b --- /dev/null +++ b/images/virt-artifact/patches/028-hotplug-container-disk.patch @@ -0,0 +1,1867 @@ +diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json +index c4822a0448..d6bb534249 100644 +--- a/api/openapi-spec/swagger.json ++++ b/api/openapi-spec/swagger.json +@@ -12951,6 +12951,9 @@ + "image" + ], + "properties": { ++ "hotpluggable": { ++ "type": "boolean" ++ }, + "image": { + "description": "Image is the name of the image with the embedded disk.", + "type": "string", +@@ -13973,6 +13976,9 @@ + "description": "HotplugVolumeSource Represents the source of a volume to mount which are capable of being hotplugged on a live running VMI. Only one of its members may be specified.", + "type": "object", + "properties": { ++ "containerDisk": { ++ "$ref": "#/definitions/v1.ContainerDiskSource" ++ }, + "dataVolume": { + "description": "DataVolume represents the dynamic creation a PVC for this volume as well as the process of populating that PVC with a disk image.", + "$ref": "#/definitions/v1.DataVolumeSource" +diff --git a/cmd/hp-container-disk/main.go b/cmd/hp-container-disk/main.go +new file mode 100644 +index 0000000000..e4f734a516 +--- /dev/null ++++ b/cmd/hp-container-disk/main.go +@@ -0,0 +1,16 @@ ++package main ++ ++import klog "kubevirt.io/client-go/log" ++ ++func main() { ++ klog.InitializeLogging("virt-hp-container-disk") ++} ++ ++type Config struct { ++ DstDir string `json:"dstDir"` ++ Images []Image `json:"images"` ++} ++ ++type Image struct { ++ Name string `json:"name"` ++} +diff --git a/cmd/virt-chroot/main.go b/cmd/virt-chroot/main.go +index e28daa07c7..7a69b7451b 100644 +--- a/cmd/virt-chroot/main.go ++++ b/cmd/virt-chroot/main.go +@@ -20,6 +20,7 @@ var ( + cpuTime uint64 + memoryBytes uint64 + targetUser string ++ targetUserID int + ) + + func init() { +@@ -51,7 +52,12 @@ func main() { + + // Looking up users needs resources, let's do it before we set rlimits. + var u *user.User +- if targetUser != "" { ++ if targetUserID >= 0 { ++ _, _, errno := syscall.Syscall(syscall.SYS_SETUID, uintptr(targetUserID), 0, 0) ++ if errno != 0 { ++ return fmt.Errorf("failed to switch to user: %d. errno: %d", targetUserID, errno) ++ } ++ } else if targetUser != "" { + var err error + u, err = user.Lookup(targetUser) + if err != nil { +@@ -116,6 +122,7 @@ func main() { + rootCmd.PersistentFlags().Uint64Var(&memoryBytes, "memory", 0, "memory in bytes for the process") + rootCmd.PersistentFlags().StringVar(&mntNamespace, "mount", "", "mount namespace to use") + rootCmd.PersistentFlags().StringVar(&targetUser, "user", "", "switch to this targetUser to e.g. drop privileges") ++ rootCmd.PersistentFlags().IntVar(&targetUserID, "userid", -1, "switch to this targetUser to e.g. drop privileges") + + execCmd := &cobra.Command{ + Use: "exec", +@@ -136,16 +143,39 @@ func main() { + Args: cobra.MinimumNArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + var mntOpts uint = 0 ++ var dataOpts []string + + fsType := cmd.Flag("type").Value.String() + mntOptions := cmd.Flag("options").Value.String() ++ var ( ++ uid = -1 ++ gid = -1 ++ ) + for _, opt := range strings.Split(mntOptions, ",") { + opt = strings.TrimSpace(opt) +- switch opt { +- case "ro": ++ switch { ++ case opt == "ro": + mntOpts = mntOpts | syscall.MS_RDONLY +- case "bind": ++ case opt == "bind": + mntOpts = mntOpts | syscall.MS_BIND ++ case opt == "remount": ++ mntOpts = mntOpts | syscall.MS_REMOUNT ++ case strings.HasPrefix(opt, "uid="): ++ uidS := strings.TrimPrefix(opt, "uid=") ++ uidI, err := strconv.Atoi(uidS) ++ if err != nil { ++ return fmt.Errorf("failed to parse uid: %w", err) ++ } ++ uid = uidI ++ dataOpts = append(dataOpts, opt) ++ case strings.HasPrefix(opt, "gid="): ++ gidS := strings.TrimPrefix(opt, "gid=") ++ gidI, err := strconv.Atoi(gidS) ++ if err != nil { ++ return fmt.Errorf("failed to parse gid: %w", err) ++ } ++ gid = gidI ++ dataOpts = append(dataOpts, opt) + default: + return fmt.Errorf("mount option %s is not supported", opt) + } +@@ -168,8 +198,17 @@ func main() { + return fmt.Errorf("mount target invalid: %v", err) + } + defer targetFile.Close() +- +- return syscall.Mount(sourceFile.SafePath(), targetFile.SafePath(), fsType, uintptr(mntOpts), "") ++ if uid >= 0 && gid >= 0 { ++ err = os.Chown(targetFile.SafePath(), uid, gid) ++ if err != nil { ++ return fmt.Errorf("chown target failed: %w", err) ++ } ++ } ++ var data string ++ if len(dataOpts) > 0 { ++ data = strings.Join(dataOpts, ",") ++ } ++ return syscall.Mount(sourceFile.SafePath(), targetFile.SafePath(), fsType, uintptr(mntOpts), data) + }, + } + mntCmd.Flags().StringP("options", "o", "", "comma separated list of mount options") +diff --git a/manifests/generated/kv-resource.yaml b/manifests/generated/kv-resource.yaml +index 66d1b01dbf..43e36b7195 100644 +--- a/manifests/generated/kv-resource.yaml ++++ b/manifests/generated/kv-resource.yaml +@@ -3307,9 +3307,6 @@ spec: + - jsonPath: .status.phase + name: Phase + type: string +- deprecated: true +- deprecationWarning: kubevirt.io/v1alpha3 is now deprecated and will be removed +- in a future release. + name: v1alpha3 + schema: + openAPIV3Schema: +diff --git a/manifests/generated/operator-csv.yaml.in b/manifests/generated/operator-csv.yaml.in +index 400d118024..05ee099c67 100644 +--- a/manifests/generated/operator-csv.yaml.in ++++ b/manifests/generated/operator-csv.yaml.in +@@ -605,6 +605,13 @@ spec: + - '*' + verbs: + - '*' ++ - apiGroups: ++ - subresources.virtualization.deckhouse.io ++ resources: ++ - virtualmachines/addvolume ++ - virtualmachines/removevolume ++ verbs: ++ - update + - apiGroups: + - subresources.kubevirt.io + resources: +diff --git a/manifests/generated/rbac-operator.authorization.k8s.yaml.in b/manifests/generated/rbac-operator.authorization.k8s.yaml.in +index 10dbb92269..1ccc9e9fa7 100644 +--- a/manifests/generated/rbac-operator.authorization.k8s.yaml.in ++++ b/manifests/generated/rbac-operator.authorization.k8s.yaml.in +@@ -143,7 +143,7 @@ kind: RoleBinding + metadata: + labels: + kubevirt.io: "" +- name: kubevirt-operator-rolebinding ++ name: kubevirt-operator + namespace: {{.Namespace}} + roleRef: + apiGroup: rbac.authorization.k8s.io +@@ -607,6 +607,13 @@ rules: + - '*' + verbs: + - '*' ++- apiGroups: ++ - subresources.virtualization.deckhouse.io ++ resources: ++ - virtualmachines/addvolume ++ - virtualmachines/removevolume ++ verbs: ++ - update + - apiGroups: + - subresources.kubevirt.io + resources: +diff --git a/pkg/container-disk/container-disk.go b/pkg/container-disk/container-disk.go +index 3251d04787..69454ed499 100644 +--- a/pkg/container-disk/container-disk.go ++++ b/pkg/container-disk/container-disk.go +@@ -47,8 +47,10 @@ var containerDiskOwner = "qemu" + var podsBaseDir = util.KubeletPodsDir + + var mountBaseDir = filepath.Join(util.VirtShareDir, "/container-disks") ++var hotplugBaseDir = filepath.Join(util.VirtShareDir, "/hotplug-disks") + + type SocketPathGetter func(vmi *v1.VirtualMachineInstance, volumeIndex int) (string, error) ++type HotplugSocketPathGetter func(vmi *v1.VirtualMachineInstance, volumeName string) (string, error) + type KernelBootSocketPathGetter func(vmi *v1.VirtualMachineInstance) (string, error) + + const KernelBootName = "kernel-boot" +@@ -107,6 +109,10 @@ func GetDiskTargetPathFromLauncherView(volumeIndex int) string { + return filepath.Join(mountBaseDir, GetDiskTargetName(volumeIndex)) + } + ++func GetHotplugContainerDiskTargetPathFromLauncherView(volumeName string) string { ++ return filepath.Join(hotplugBaseDir, fmt.Sprintf("%s.img", volumeName)) ++} ++ + func GetKernelBootArtifactPathFromLauncherView(artifact string) string { + artifactBase := filepath.Base(artifact) + return filepath.Join(mountBaseDir, KernelBootName, artifactBase) +@@ -170,6 +176,23 @@ func NewSocketPathGetter(baseDir string) SocketPathGetter { + } + } + ++func NewHotplugSocketPathGetter(baseDir string) HotplugSocketPathGetter { ++ return func(vmi *v1.VirtualMachineInstance, volumeName string) (string, error) { ++ for _, v := range vmi.Status.VolumeStatus { ++ if v.Name == volumeName && v.HotplugVolume != nil && v.ContainerDiskVolume != nil { ++ basePath := getHotplugContainerDiskSocketBasePath(baseDir, string(v.HotplugVolume.AttachPodUID)) ++ socketPath := filepath.Join(basePath, fmt.Sprintf("hotplug-container-disk-%s.sock", volumeName)) ++ exists, _ := diskutils.FileExists(socketPath) ++ if exists { ++ return socketPath, nil ++ } ++ } ++ } ++ ++ return "", fmt.Errorf("container disk socket path not found for vmi \"%s\"", vmi.Name) ++ } ++} ++ + // NewKernelBootSocketPathGetter get the socket pat of the kernel-boot containerDisk. For testing a baseDir + // can be provided which can for instance point to /tmp. + func NewKernelBootSocketPathGetter(baseDir string) KernelBootSocketPathGetter { +@@ -398,6 +421,10 @@ func getContainerDiskSocketBasePath(baseDir, podUID string) string { + return fmt.Sprintf("%s/pods/%s/volumes/kubernetes.io~empty-dir/container-disks", baseDir, podUID) + } + ++func getHotplugContainerDiskSocketBasePath(baseDir, podUID string) string { ++ return fmt.Sprintf("%s/pods/%s/volumes/kubernetes.io~empty-dir/hotplug-container-disks", baseDir, podUID) ++} ++ + // ExtractImageIDsFromSourcePod takes the VMI and its source pod to determine the exact image used by containerdisks and boot container images, + // which is recorded in the status section of a started pod; if the status section does not contain this info the tag is used. + // It returns a map where the key is the vlume name and the value is the imageID +diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go +index 490cc445ef..4b7dbc12fe 100644 +--- a/pkg/controller/controller.go ++++ b/pkg/controller/controller.go +@@ -278,6 +278,10 @@ func ApplyVolumeRequestOnVMISpec(vmiSpec *v1.VirtualMachineInstanceSpec, request + dvSource := request.AddVolumeOptions.VolumeSource.DataVolume.DeepCopy() + dvSource.Hotpluggable = true + newVolume.VolumeSource.DataVolume = dvSource ++ } else if request.AddVolumeOptions.VolumeSource.ContainerDisk != nil { ++ containerDiskSource := request.AddVolumeOptions.VolumeSource.ContainerDisk.DeepCopy() ++ containerDiskSource.Hotpluggable = true ++ newVolume.VolumeSource.ContainerDisk = containerDiskSource + } + + vmiSpec.Volumes = append(vmiSpec.Volumes, newVolume) +@@ -444,6 +448,9 @@ func VMIHasHotplugVolumes(vmi *v1.VirtualMachineInstance) bool { + if volume.PersistentVolumeClaim != nil && volume.PersistentVolumeClaim.Hotpluggable { + return true + } ++ if volume.ContainerDisk != nil && volume.ContainerDisk.Hotpluggable { ++ return true ++ } + } + return false + } +@@ -557,7 +564,7 @@ func GetHotplugVolumes(vmi *v1.VirtualMachineInstance, virtlauncherPod *k8sv1.Po + podVolumeMap[podVolume.Name] = podVolume + } + for _, vmiVolume := range vmiVolumes { +- if _, ok := podVolumeMap[vmiVolume.Name]; !ok && (vmiVolume.DataVolume != nil || vmiVolume.PersistentVolumeClaim != nil || vmiVolume.MemoryDump != nil) { ++ if _, ok := podVolumeMap[vmiVolume.Name]; !ok && (vmiVolume.DataVolume != nil || vmiVolume.PersistentVolumeClaim != nil || vmiVolume.MemoryDump != nil || vmiVolume.ContainerDisk != nil) { + hotplugVolumes = append(hotplugVolumes, vmiVolume.DeepCopy()) + } + } +diff --git a/pkg/virt-api/rest/subresource.go b/pkg/virt-api/rest/subresource.go +index b5d62f5af5..bf561f00ae 100644 +--- a/pkg/virt-api/rest/subresource.go ++++ b/pkg/virt-api/rest/subresource.go +@@ -1023,7 +1023,8 @@ func volumeSourceName(volumeSource *v1.HotplugVolumeSource) string { + + func volumeSourceExists(volume v1.Volume, volumeName string) bool { + return (volume.DataVolume != nil && volume.DataVolume.Name == volumeName) || +- (volume.PersistentVolumeClaim != nil && volume.PersistentVolumeClaim.ClaimName == volumeName) ++ (volume.PersistentVolumeClaim != nil && volume.PersistentVolumeClaim.ClaimName == volumeName) || ++ (volume.ContainerDisk != nil && volume.ContainerDisk.Image != "") + } + + func volumeExists(volume v1.Volume, volumeName string) bool { +@@ -1125,6 +1126,8 @@ func (app *SubresourceAPIApp) addVolumeRequestHandler(request *restful.Request, + opts.VolumeSource.DataVolume.Hotpluggable = true + } else if opts.VolumeSource.PersistentVolumeClaim != nil { + opts.VolumeSource.PersistentVolumeClaim.Hotpluggable = true ++ } else if opts.VolumeSource.ContainerDisk != nil { ++ opts.VolumeSource.ContainerDisk.Hotpluggable = true + } + + // inject into VMI if ephemeral, else set as a request on the VM to both make permanent and hotplug. +diff --git a/pkg/virt-api/webhooks/validating-webhook/admitters/vmi-update-admitter.go b/pkg/virt-api/webhooks/validating-webhook/admitters/vmi-update-admitter.go +index 0af25f8074..803c0ed4cd 100644 +--- a/pkg/virt-api/webhooks/validating-webhook/admitters/vmi-update-admitter.go ++++ b/pkg/virt-api/webhooks/validating-webhook/admitters/vmi-update-admitter.go +@@ -200,11 +200,11 @@ func verifyHotplugVolumes(newHotplugVolumeMap, oldHotplugVolumeMap map[string]v1 + } + } else { + // This is a new volume, ensure that the volume is either DV, PVC or memoryDumpVolume +- if v.DataVolume == nil && v.PersistentVolumeClaim == nil && v.MemoryDump == nil { ++ if v.DataVolume == nil && v.PersistentVolumeClaim == nil && v.MemoryDump == nil && v.ContainerDisk == nil { + return webhookutils.ToAdmissionResponse([]metav1.StatusCause{ + { + Type: metav1.CauseTypeFieldValueInvalid, +- Message: fmt.Sprintf("volume %s is not a PVC or DataVolume", k), ++ Message: fmt.Sprintf("volume %s is not a PVC,DataVolume,MemoryDumpVolume or ContainerDisk", k), + }, + }) + } +diff --git a/pkg/virt-controller/services/template.go b/pkg/virt-controller/services/template.go +index 76ed7307ec..f607c24786 100644 +--- a/pkg/virt-controller/services/template.go ++++ b/pkg/virt-controller/services/template.go +@@ -64,13 +64,15 @@ import ( + ) + + const ( +- containerDisks = "container-disks" +- hotplugDisks = "hotplug-disks" +- hookSidecarSocks = "hook-sidecar-sockets" +- varRun = "/var/run" +- virtBinDir = "virt-bin-share-dir" +- hotplugDisk = "hotplug-disk" +- virtExporter = "virt-exporter" ++ containerDisks = "container-disks" ++ hotplugDisks = "hotplug-disks" ++ hookSidecarSocks = "hook-sidecar-sockets" ++ varRun = "/var/run" ++ virtBinDir = "virt-bin-share-dir" ++ hotplugDisk = "hotplug-disk" ++ virtExporter = "virt-exporter" ++ hotplugContainerDisks = "hotplug-container-disks" ++ HotplugContainerDisk = "hotplug-container-disk-" + ) + + const KvmDevice = "devices.virtualization.deckhouse.io/kvm" +@@ -846,6 +848,49 @@ func sidecarContainerName(i int) string { + return fmt.Sprintf("hook-sidecar-%d", i) + } + ++func sidecarContainerHotplugContainerdDiskName(name string) string { ++ return fmt.Sprintf("%s%s", HotplugContainerDisk, name) ++} ++ ++func (t *templateService) containerForHotplugContainerDisk(name string, cd *v1.ContainerDiskSource, vmi *v1.VirtualMachineInstance) k8sv1.Container { ++ runUser := int64(util.NonRootUID) ++ sharedMount := k8sv1.MountPropagationHostToContainer ++ path := fmt.Sprintf("/path/%s", name) ++ command := []string{"/init/usr/bin/container-disk"} ++ args := []string{"--copy-path", path} ++ ++ return k8sv1.Container{ ++ Name: name, ++ Image: cd.Image, ++ Command: command, ++ Args: args, ++ Resources: hotplugContainerResourceRequirementsForVMI(vmi, t.clusterConfig), ++ SecurityContext: &k8sv1.SecurityContext{ ++ AllowPrivilegeEscalation: pointer.Bool(false), ++ RunAsNonRoot: pointer.Bool(true), ++ RunAsUser: &runUser, ++ SeccompProfile: &k8sv1.SeccompProfile{ ++ Type: k8sv1.SeccompProfileTypeRuntimeDefault, ++ }, ++ Capabilities: &k8sv1.Capabilities{ ++ Drop: []k8sv1.Capability{"ALL"}, ++ }, ++ SELinuxOptions: &k8sv1.SELinuxOptions{ ++ Type: t.clusterConfig.GetSELinuxLauncherType(), ++ Level: "s0", ++ }, ++ }, ++ VolumeMounts: []k8sv1.VolumeMount{ ++ initContainerVolumeMount(), ++ { ++ Name: hotplugContainerDisks, ++ MountPath: "/path", ++ MountPropagation: &sharedMount, ++ }, ++ }, ++ } ++} ++ + func (t *templateService) RenderHotplugAttachmentPodTemplate(volumes []*v1.Volume, ownerPod *k8sv1.Pod, vmi *v1.VirtualMachineInstance, claimMap map[string]*k8sv1.PersistentVolumeClaim) (*k8sv1.Pod, error) { + zero := int64(0) + runUser := int64(util.NonRootUID) +@@ -924,6 +969,30 @@ func (t *templateService) RenderHotplugAttachmentPodTemplate(volumes []*v1.Volum + TerminationGracePeriodSeconds: &zero, + }, + } ++ first := true ++ for _, vol := range vmi.Spec.Volumes { ++ if vol.ContainerDisk == nil || !vol.ContainerDisk.Hotpluggable { ++ continue ++ } ++ name := sidecarContainerHotplugContainerdDiskName(vol.Name) ++ pod.Spec.Containers = append(pod.Spec.Containers, t.containerForHotplugContainerDisk(name, vol.ContainerDisk, vmi)) ++ if first { ++ first = false ++ userId := int64(util.NonRootUID) ++ initContainerCommand := []string{"/usr/bin/cp", ++ "/usr/bin/container-disk", ++ "/init/usr/bin/container-disk", ++ } ++ pod.Spec.InitContainers = append( ++ pod.Spec.InitContainers, ++ t.newInitContainerRenderer(vmi, ++ initContainerVolumeMount(), ++ initContainerResourceRequirementsForVMI(vmi, v1.ContainerDisk, t.clusterConfig), ++ userId).Render(initContainerCommand)) ++ pod.Spec.Volumes = append(pod.Spec.Volumes, emptyDirVolume(hotplugContainerDisks)) ++ pod.Spec.Volumes = append(pod.Spec.Volumes, emptyDirVolume(virtBinDir)) ++ } ++ } + + err := matchSELinuxLevelOfVMI(pod, vmi) + if err != nil { +diff --git a/pkg/virt-controller/watch/BUILD.bazel b/pkg/virt-controller/watch/BUILD.bazel +index 4fd325ba86..82fcaee0a3 100644 +--- a/pkg/virt-controller/watch/BUILD.bazel ++++ b/pkg/virt-controller/watch/BUILD.bazel +@@ -101,6 +101,7 @@ go_library( + "//vendor/k8s.io/client-go/util/flowcontrol:go_default_library", + "//vendor/k8s.io/client-go/util/workqueue:go_default_library", + "//vendor/k8s.io/utils/pointer:go_default_library", ++ "//vendor/k8s.io/utils/ptr:go_default_library", + "//vendor/k8s.io/utils/trace:go_default_library", + "//vendor/kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1:go_default_library", + ], +diff --git a/pkg/virt-controller/watch/vmi.go b/pkg/virt-controller/watch/vmi.go +index fa4e86ee17..ebe718f90d 100644 +--- a/pkg/virt-controller/watch/vmi.go ++++ b/pkg/virt-controller/watch/vmi.go +@@ -1836,6 +1836,10 @@ func (c *VMIController) handleHotplugVolumes(hotplugVolumes []*virtv1.Volume, ho + readyHotplugVolumes := make([]*virtv1.Volume, 0) + // Find all ready volumes + for _, volume := range hotplugVolumes { ++ if volume.ContainerDisk != nil { ++ readyHotplugVolumes = append(readyHotplugVolumes, volume) ++ continue ++ } + var err error + ready, wffc, err := storagetypes.VolumeReadyToAttachToNode(vmi.Namespace, *volume, dataVolumes, c.dataVolumeIndexer, c.pvcIndexer) + if err != nil { +@@ -1884,7 +1888,15 @@ func (c *VMIController) handleHotplugVolumes(hotplugVolumes []*virtv1.Volume, ho + + func (c *VMIController) podVolumesMatchesReadyVolumes(attachmentPod *k8sv1.Pod, volumes []*virtv1.Volume) bool { + // -2 for empty dir and token +- if len(attachmentPod.Spec.Volumes)-2 != len(volumes) { ++ // -3 if exist container-disk ++ magicNum := len(attachmentPod.Spec.Volumes) - 2 ++ for _, volume := range volumes { ++ if volume.ContainerDisk != nil { ++ magicNum -= 1 ++ break ++ } ++ } ++ if magicNum != len(volumes) { + return false + } + podVolumeMap := make(map[string]k8sv1.Volume) +@@ -1893,10 +1905,20 @@ func (c *VMIController) podVolumesMatchesReadyVolumes(attachmentPod *k8sv1.Pod, + podVolumeMap[volume.Name] = volume + } + } ++ containerDisksNames := make(map[string]struct{}) ++ for _, ctr := range attachmentPod.Spec.Containers { ++ if strings.HasPrefix(ctr.Name, services.HotplugContainerDisk) { ++ containerDisksNames[strings.TrimPrefix(ctr.Name, services.HotplugContainerDisk)] = struct{}{} ++ } ++ } + for _, volume := range volumes { ++ if volume.ContainerDisk != nil { ++ delete(containerDisksNames, volume.Name) ++ continue ++ } + delete(podVolumeMap, volume.Name) + } +- return len(podVolumeMap) == 0 ++ return len(podVolumeMap) == 0 && len(containerDisksNames) == 0 + } + + func (c *VMIController) createAttachmentPod(vmi *virtv1.VirtualMachineInstance, virtLauncherPod *k8sv1.Pod, volumes []*virtv1.Volume) (*k8sv1.Pod, syncError) { +@@ -2007,7 +2029,17 @@ func (c *VMIController) createAttachmentPodTemplate(vmi *virtv1.VirtualMachineIn + var pod *k8sv1.Pod + var err error + +- volumeNamesPVCMap, err := storagetypes.VirtVolumesToPVCMap(volumes, c.pvcIndexer, virtlauncherPod.Namespace) ++ var hasContainerDisk bool ++ var newVolumes []*virtv1.Volume ++ for _, volume := range volumes { ++ if volume.VolumeSource.ContainerDisk != nil { ++ hasContainerDisk = true ++ continue ++ } ++ newVolumes = append(newVolumes, volume) ++ } ++ ++ volumeNamesPVCMap, err := storagetypes.VirtVolumesToPVCMap(newVolumes, c.pvcIndexer, virtlauncherPod.Namespace) + if err != nil { + return nil, fmt.Errorf("failed to get PVC map: %v", err) + } +@@ -2029,7 +2061,7 @@ func (c *VMIController) createAttachmentPodTemplate(vmi *virtv1.VirtualMachineIn + } + } + +- if len(volumeNamesPVCMap) > 0 { ++ if len(volumeNamesPVCMap) > 0 || hasContainerDisk { + pod, err = c.templateService.RenderHotplugAttachmentPodTemplate(volumes, virtlauncherPod, vmi, volumeNamesPVCMap) + } + return pod, err +@@ -2151,23 +2183,39 @@ func (c *VMIController) updateVolumeStatus(vmi *virtv1.VirtualMachineInstance, v + ClaimName: volume.Name, + } + } ++ if volume.ContainerDisk != nil && status.ContainerDiskVolume == nil { ++ status.ContainerDiskVolume = &virtv1.ContainerDiskInfo{} ++ } + if attachmentPod == nil { +- if !c.volumeReady(status.Phase) { +- status.HotplugVolume.AttachPodUID = "" +- // Volume is not hotplugged in VM and Pod is gone, or hasn't been created yet, check for the PVC associated with the volume to set phase and message +- phase, reason, message := c.getVolumePhaseMessageReason(&vmi.Spec.Volumes[i], vmi.Namespace) +- status.Phase = phase +- status.Message = message +- status.Reason = reason ++ if volume.ContainerDisk != nil { ++ if !c.volumeReady(status.Phase) { ++ status.HotplugVolume.AttachPodUID = "" ++ // Volume is not hotplugged in VM and Pod is gone, or hasn't been created yet, check for the PVC associated with the volume to set phase and message ++ phase, reason, message := c.getVolumePhaseMessageReason(&vmi.Spec.Volumes[i], vmi.Namespace) ++ status.Phase = phase ++ status.Message = message ++ status.Reason = reason ++ } + } + } else { + status.HotplugVolume.AttachPodName = attachmentPod.Name +- if len(attachmentPod.Status.ContainerStatuses) == 1 && attachmentPod.Status.ContainerStatuses[0].Ready { ++ if volume.ContainerDisk != nil { ++ uid := types.UID("") ++ for _, cs := range attachmentPod.Status.ContainerStatuses { ++ name := strings.TrimPrefix(cs.Name, "hotplug-container-disk-") ++ if volume.Name == name && cs.Ready { ++ uid = attachmentPod.UID ++ break ++ } ++ } ++ status.HotplugVolume.AttachPodUID = uid ++ } else if len(attachmentPod.Status.ContainerStatuses) == 1 && attachmentPod.Status.ContainerStatuses[0].Ready { + status.HotplugVolume.AttachPodUID = attachmentPod.UID + } else { + // Remove UID of old pod if a new one is available, but not yet ready + status.HotplugVolume.AttachPodUID = "" + } ++ + if c.canMoveToAttachedPhase(status.Phase) { + status.Phase = virtv1.HotplugVolumeAttachedToNode + status.Message = fmt.Sprintf("Created hotplug attachment pod %s, for volume %s", attachmentPod.Name, volume.Name) +@@ -2176,7 +2224,6 @@ func (c *VMIController) updateVolumeStatus(vmi *virtv1.VirtualMachineInstance, v + } + } + } +- + if volume.VolumeSource.PersistentVolumeClaim != nil || volume.VolumeSource.DataVolume != nil || volume.VolumeSource.MemoryDump != nil { + + pvcName := storagetypes.PVCNameFromVirtVolume(&volume) +diff --git a/pkg/virt-handler/container-disk/hotplug.go b/pkg/virt-handler/container-disk/hotplug.go +new file mode 100644 +index 0000000000..eb0bb831fe +--- /dev/null ++++ b/pkg/virt-handler/container-disk/hotplug.go +@@ -0,0 +1,487 @@ ++package container_disk ++ ++import ( ++ "encoding/json" ++ "errors" ++ "fmt" ++ "os" ++ "path/filepath" ++ "strings" ++ "sync" ++ "time" ++ ++ hotplugdisk "kubevirt.io/kubevirt/pkg/hotplug-disk" ++ "kubevirt.io/kubevirt/pkg/unsafepath" ++ ++ "kubevirt.io/kubevirt/pkg/safepath" ++ virtconfig "kubevirt.io/kubevirt/pkg/virt-config" ++ virt_chroot "kubevirt.io/kubevirt/pkg/virt-handler/virt-chroot" ++ ++ "kubevirt.io/client-go/log" ++ ++ containerdisk "kubevirt.io/kubevirt/pkg/container-disk" ++ diskutils "kubevirt.io/kubevirt/pkg/ephemeral-disk-utils" ++ "kubevirt.io/kubevirt/pkg/virt-handler/isolation" ++ ++ "k8s.io/apimachinery/pkg/api/equality" ++ "k8s.io/apimachinery/pkg/types" ++ ++ v1 "kubevirt.io/api/core/v1" ++) ++ ++type HotplugMounter interface { ++ ContainerDisksReady(vmi *v1.VirtualMachineInstance, notInitializedSince time.Time) (bool, error) ++ MountAndVerify(vmi *v1.VirtualMachineInstance) (map[string]*containerdisk.DiskInfo, error) ++ IsMounted(vmi *v1.VirtualMachineInstance, volumeName string) (bool, error) ++ Umount(vmi *v1.VirtualMachineInstance) error ++ UmountAll(vmi *v1.VirtualMachineInstance) error ++ ComputeChecksums(vmi *v1.VirtualMachineInstance) (*DiskChecksums, error) ++} ++ ++type hotplugMounter struct { ++ podIsolationDetector isolation.PodIsolationDetector ++ mountStateDir string ++ mountRecords map[types.UID]*vmiMountTargetRecord ++ mountRecordsLock sync.Mutex ++ suppressWarningTimeout time.Duration ++ clusterConfig *virtconfig.ClusterConfig ++ nodeIsolationResult isolation.IsolationResult ++ ++ hotplugPathGetter containerdisk.HotplugSocketPathGetter ++ hotplugManager hotplugdisk.HotplugDiskManagerInterface ++} ++ ++func (m *hotplugMounter) IsMounted(vmi *v1.VirtualMachineInstance, volumeName string) (bool, error) { ++ virtLauncherUID := m.findVirtlauncherUID(vmi) ++ if virtLauncherUID == "" { ++ return false, nil ++ } ++ target, err := m.hotplugManager.GetFileSystemDiskTargetPathFromHostView(virtLauncherUID, volumeName, false) ++ if err != nil { ++ return false, err ++ } ++ return isolation.IsMounted(target) ++} ++ ++func NewHotplugMounter(isoDetector isolation.PodIsolationDetector, ++ mountStateDir string, ++ clusterConfig *virtconfig.ClusterConfig, ++ hotplugManager hotplugdisk.HotplugDiskManagerInterface, ++) HotplugMounter { ++ return &hotplugMounter{ ++ mountRecords: make(map[types.UID]*vmiMountTargetRecord), ++ podIsolationDetector: isoDetector, ++ mountStateDir: mountStateDir, ++ suppressWarningTimeout: 1 * time.Minute, ++ clusterConfig: clusterConfig, ++ nodeIsolationResult: isolation.NodeIsolationResult(), ++ ++ hotplugPathGetter: containerdisk.NewHotplugSocketPathGetter(""), ++ hotplugManager: hotplugManager, ++ } ++} ++ ++func (m *hotplugMounter) deleteMountTargetRecord(vmi *v1.VirtualMachineInstance) error { ++ if string(vmi.UID) == "" { ++ return fmt.Errorf("unable to find container disk mounted directories for vmi without uid") ++ } ++ ++ recordFile := filepath.Join(m.mountStateDir, string(vmi.UID)) ++ ++ exists, err := diskutils.FileExists(recordFile) ++ if err != nil { ++ return err ++ } ++ ++ if exists { ++ record, err := m.getMountTargetRecord(vmi) ++ if err != nil { ++ return err ++ } ++ ++ for _, target := range record.MountTargetEntries { ++ os.Remove(target.TargetFile) ++ os.Remove(target.SocketFile) ++ } ++ ++ os.Remove(recordFile) ++ } ++ ++ m.mountRecordsLock.Lock() ++ defer m.mountRecordsLock.Unlock() ++ delete(m.mountRecords, vmi.UID) ++ ++ return nil ++} ++ ++func (m *hotplugMounter) getMountTargetRecord(vmi *v1.VirtualMachineInstance) (*vmiMountTargetRecord, error) { ++ var ok bool ++ var existingRecord *vmiMountTargetRecord ++ ++ if string(vmi.UID) == "" { ++ return nil, fmt.Errorf("unable to find container disk mounted directories for vmi without uid") ++ } ++ ++ m.mountRecordsLock.Lock() ++ defer m.mountRecordsLock.Unlock() ++ existingRecord, ok = m.mountRecords[vmi.UID] ++ ++ // first check memory cache ++ if ok { ++ return existingRecord, nil ++ } ++ ++ // if not there, see if record is on disk, this can happen if virt-handler restarts ++ recordFile := filepath.Join(m.mountStateDir, filepath.Clean(string(vmi.UID))) ++ ++ exists, err := diskutils.FileExists(recordFile) ++ if err != nil { ++ return nil, err ++ } ++ ++ if exists { ++ record := vmiMountTargetRecord{} ++ // #nosec No risk for path injection. Using static base and cleaned filename ++ bytes, err := os.ReadFile(recordFile) ++ if err != nil { ++ return nil, err ++ } ++ err = json.Unmarshal(bytes, &record) ++ if err != nil { ++ return nil, err ++ } ++ ++ // XXX: backward compatibility for old unresolved paths, can be removed in July 2023 ++ // After a one-time convert and persist, old records are safe too. ++ if !record.UsesSafePaths { ++ record.UsesSafePaths = true ++ for i, entry := range record.MountTargetEntries { ++ safePath, err := safepath.JoinAndResolveWithRelativeRoot("/", entry.TargetFile) ++ if err != nil { ++ return nil, fmt.Errorf("failed converting legacy path to safepath: %v", err) ++ } ++ record.MountTargetEntries[i].TargetFile = unsafepath.UnsafeAbsolute(safePath.Raw()) ++ } ++ } ++ ++ m.mountRecords[vmi.UID] = &record ++ return &record, nil ++ } ++ ++ // not found ++ return nil, nil ++} ++ ++func (m *hotplugMounter) addMountTargetRecord(vmi *v1.VirtualMachineInstance, record *vmiMountTargetRecord) error { ++ return m.setAddMountTargetRecordHelper(vmi, record, true) ++} ++ ++func (m *hotplugMounter) setMountTargetRecord(vmi *v1.VirtualMachineInstance, record *vmiMountTargetRecord) error { ++ return m.setAddMountTargetRecordHelper(vmi, record, false) ++} ++ ++func (m *hotplugMounter) setAddMountTargetRecordHelper(vmi *v1.VirtualMachineInstance, record *vmiMountTargetRecord, addPreviousRules bool) error { ++ if string(vmi.UID) == "" { ++ return fmt.Errorf("unable to set container disk mounted directories for vmi without uid") ++ } ++ ++ record.UsesSafePaths = true ++ ++ recordFile := filepath.Join(m.mountStateDir, string(vmi.UID)) ++ fileExists, err := diskutils.FileExists(recordFile) ++ if err != nil { ++ return err ++ } ++ ++ m.mountRecordsLock.Lock() ++ defer m.mountRecordsLock.Unlock() ++ ++ existingRecord, ok := m.mountRecords[vmi.UID] ++ if ok && fileExists && equality.Semantic.DeepEqual(existingRecord, record) { ++ // already done ++ return nil ++ } ++ ++ if addPreviousRules && existingRecord != nil && len(existingRecord.MountTargetEntries) > 0 { ++ record.MountTargetEntries = append(record.MountTargetEntries, existingRecord.MountTargetEntries...) ++ } ++ ++ bytes, err := json.Marshal(record) ++ if err != nil { ++ return err ++ } ++ ++ err = os.MkdirAll(filepath.Dir(recordFile), 0750) ++ if err != nil { ++ return err ++ } ++ ++ err = os.WriteFile(recordFile, bytes, 0600) ++ if err != nil { ++ return err ++ } ++ ++ m.mountRecords[vmi.UID] = record ++ ++ return nil ++} ++ ++func (m *hotplugMounter) MountAndVerify(vmi *v1.VirtualMachineInstance) (map[string]*containerdisk.DiskInfo, error) { ++ virtLauncherUID := m.findVirtlauncherUID(vmi) ++ if virtLauncherUID == "" { ++ return nil, nil ++ } ++ ++ record := vmiMountTargetRecord{} ++ disksInfo := map[string]*containerdisk.DiskInfo{} ++ ++ for _, volume := range vmi.Spec.Volumes { ++ if volume.ContainerDisk != nil && volume.ContainerDisk.Hotpluggable { ++ target, err := m.hotplugManager.GetFileSystemDiskTargetPathFromHostView(virtLauncherUID, volume.Name, true) ++ if err != nil { ++ return nil, err ++ } ++ ++ sock, err := m.hotplugPathGetter(vmi, volume.Name) ++ if err != nil { ++ return nil, err ++ } ++ ++ record.MountTargetEntries = append(record.MountTargetEntries, vmiMountTargetEntry{ ++ TargetFile: unsafepath.UnsafeAbsolute(target.Raw()), ++ SocketFile: sock, ++ }) ++ } ++ } ++ ++ if len(record.MountTargetEntries) > 0 { ++ err := m.setMountTargetRecord(vmi, &record) ++ if err != nil { ++ return nil, err ++ } ++ } ++ ++ vmiRes, err := m.podIsolationDetector.Detect(vmi) ++ if err != nil { ++ return nil, fmt.Errorf("failed to detect VMI pod: %v", err) ++ } ++ ++ for _, volume := range vmi.Spec.Volumes { ++ if volume.ContainerDisk != nil && volume.ContainerDisk.Hotpluggable { ++ target, err := m.hotplugManager.GetFileSystemDiskTargetPathFromHostView(virtLauncherUID, volume.Name, false) ++ ++ if isMounted, err := isolation.IsMounted(target); err != nil { ++ return nil, fmt.Errorf("failed to determine if %s is already mounted: %v", target, err) ++ } else if !isMounted { ++ ++ sourceFile, err := m.getContainerDiskPath(vmi, &volume, volume.Name) ++ if err != nil { ++ return nil, fmt.Errorf("failed to find a sourceFile in containerDisk %v: %v", volume.Name, err) ++ } ++ ++ log.DefaultLogger().Object(vmi).Infof("Bind mounting container disk at %s to %s", sourceFile, target) ++ opts := []string{ ++ "bind", "ro", "uid=107", "gid=107", ++ } ++ err = virt_chroot.MountChrootWithOptions(sourceFile, target, opts...) ++ if err != nil { ++ return nil, fmt.Errorf("failed to bindmount containerDisk %v. err: %w", volume.Name, err) ++ } ++ } ++ // qemu-img: Could not open '/var/run/kubevirt/hotplug-disks/alpine.img': Could not open '/var/run/kubevirt/hotplug-disks/alpine.img': Permission denied ++ // containerdisk.GetHotplugContainerDiskTargetPathFromLauncherView(volume.Name), ++ // qemu-img: Could not open 'root: /var/lib/kubelet/pods/06b70b2f-7041-45e6-a333-9b9f009e72ef/volumes/kubernetes.io~empty-dir/hotplug-disks, relative: /alpine.im ++ // qemu-img: Could not open '/var/lib/kubelet/pods/06b70b2f-7041-45e6-a333-9b9f009e72ef/volumes/kubernetes.io~empty-dir/hotplug-disks/alpine.img': Could not open '/var/lib/kubelet/pods/06b70b2f-7041-45e6-a333-9b9f009e72ef/volumes/kubernetes.io~empty-dir/hotplug-disks/alpine.img': No such file or directory ++ // unsafepath.UnsafeAbsolute(target.Raw( ++ imageInfo, err := isolation.GetImageInfo( ++ containerdisk.GetHotplugContainerDiskTargetPathFromLauncherView(volume.Name), ++ vmiRes, ++ m.clusterConfig.GetDiskVerification(), ++ ) ++ if err != nil { ++ return nil, fmt.Errorf("failed to get image info: %v", err) ++ } ++ if err := containerdisk.VerifyImage(imageInfo); err != nil { ++ return nil, fmt.Errorf("invalid image in containerDisk %v: %v", volume.Name, err) ++ } ++ disksInfo[volume.Name] = imageInfo ++ } ++ } ++ ++ return disksInfo, nil ++} ++ ++func (m *hotplugMounter) Umount(vmi *v1.VirtualMachineInstance) error { ++ record, err := m.getMountTargetRecord(vmi) ++ if err != nil { ++ return err ++ } else if record == nil { ++ // no entries to unmount ++ ++ log.DefaultLogger().Object(vmi).Infof("No container disk mount entries found to unmount") ++ return nil ++ } ++ for _, r := range record.MountTargetEntries { ++ name, err := extractNameFromSocket(r.SocketFile) ++ if err != nil { ++ return err ++ } ++ needUmount := true ++ for _, v := range vmi.Status.VolumeStatus { ++ if v.Name == name { ++ needUmount = false ++ } ++ } ++ if needUmount { ++ file, err := safepath.NewFileNoFollow(r.TargetFile) ++ if err != nil { ++ if errors.Is(err, os.ErrNotExist) { ++ continue ++ } ++ return fmt.Errorf(failedCheckMountPointFmt, r.TargetFile, err) ++ } ++ _ = file.Close() ++ // #nosec No risk for attacket injection. Parameters are predefined strings ++ out, err := virt_chroot.UmountChroot(file.Path()).CombinedOutput() ++ if err != nil { ++ return fmt.Errorf(failedUnmountFmt, file, string(out), err) ++ } ++ } ++ } ++ return nil ++} ++ ++func extractNameFromSocket(socketFile string) (string, error) { ++ base := filepath.Base(socketFile) ++ if strings.HasPrefix(base, "hotplug-container-disk-") && strings.HasSuffix(base, ".sock") { ++ name := strings.TrimPrefix(base, "hotplug-container-disk-") ++ name = strings.TrimSuffix(name, ".sock") ++ return name, nil ++ } ++ return "", fmt.Errorf("name not found in path") ++} ++ ++func (m *hotplugMounter) UmountAll(vmi *v1.VirtualMachineInstance) error { ++ if vmi.UID == "" { ++ return nil ++ } ++ ++ record, err := m.getMountTargetRecord(vmi) ++ if err != nil { ++ return err ++ } else if record == nil { ++ // no entries to unmount ++ ++ log.DefaultLogger().Object(vmi).Infof("No container disk mount entries found to unmount") ++ return nil ++ } ++ ++ log.DefaultLogger().Object(vmi).Infof("Found container disk mount entries") ++ for _, entry := range record.MountTargetEntries { ++ log.DefaultLogger().Object(vmi).Infof("Looking to see if containerdisk is mounted at path %s", entry.TargetFile) ++ file, err := safepath.NewFileNoFollow(entry.TargetFile) ++ if err != nil { ++ if errors.Is(err, os.ErrNotExist) { ++ continue ++ } ++ return fmt.Errorf(failedCheckMountPointFmt, entry.TargetFile, err) ++ } ++ _ = file.Close() ++ if mounted, err := isolation.IsMounted(file.Path()); err != nil { ++ return fmt.Errorf(failedCheckMountPointFmt, file, err) ++ } else if mounted { ++ log.DefaultLogger().Object(vmi).Infof("unmounting container disk at path %s", file) ++ // #nosec No risk for attacket injection. Parameters are predefined strings ++ out, err := virt_chroot.UmountChroot(file.Path()).CombinedOutput() ++ if err != nil { ++ return fmt.Errorf(failedUnmountFmt, file, string(out), err) ++ } ++ } ++ } ++ err = m.deleteMountTargetRecord(vmi) ++ if err != nil { ++ return err ++ } ++ ++ return nil ++} ++ ++func (m *hotplugMounter) ContainerDisksReady(vmi *v1.VirtualMachineInstance, notInitializedSince time.Time) (bool, error) { ++ for _, volume := range vmi.Spec.Volumes { ++ if volume.ContainerDisk != nil && volume.ContainerDisk.Hotpluggable { ++ _, err := m.hotplugPathGetter(vmi, volume.Name) ++ if err != nil { ++ log.DefaultLogger().Object(vmi).Reason(err).Infof("containerdisk %s not yet ready", volume.Name) ++ if time.Now().After(notInitializedSince.Add(m.suppressWarningTimeout)) { ++ return false, fmt.Errorf("containerdisk %s still not ready after one minute", volume.Name) ++ } ++ return false, nil ++ } ++ } ++ } ++ ++ log.DefaultLogger().Object(vmi).V(4).Info("all containerdisks are ready") ++ return true, nil ++} ++ ++func (m *hotplugMounter) getContainerDiskPath(vmi *v1.VirtualMachineInstance, volume *v1.Volume, volumeName string) (*safepath.Path, error) { ++ sock, err := m.hotplugPathGetter(vmi, volumeName) ++ if err != nil { ++ return nil, ErrDiskContainerGone ++ } ++ ++ res, err := m.podIsolationDetector.DetectForSocket(vmi, sock) ++ if err != nil { ++ return nil, fmt.Errorf("failed to detect socket for containerDisk %v: %v", volume.Name, err) ++ } ++ ++ mountPoint, err := isolation.ParentPathForRootMount(m.nodeIsolationResult, res) ++ if err != nil { ++ return nil, fmt.Errorf("failed to detect root mount point of containerDisk %v on the node: %v", volume.Name, err) ++ } ++ ++ return containerdisk.GetImage(mountPoint, volume.ContainerDisk.Path) ++} ++ ++func (m *hotplugMounter) ComputeChecksums(vmi *v1.VirtualMachineInstance) (*DiskChecksums, error) { ++ ++ diskChecksums := &DiskChecksums{ ++ ContainerDiskChecksums: map[string]uint32{}, ++ } ++ ++ for _, volume := range vmi.Spec.Volumes { ++ if volume.VolumeSource.ContainerDisk == nil || !volume.VolumeSource.ContainerDisk.Hotpluggable { ++ continue ++ } ++ ++ path, err := m.getContainerDiskPath(vmi, &volume, volume.Name) ++ if err != nil { ++ return nil, err ++ } ++ ++ checksum, err := getDigest(path) ++ if err != nil { ++ return nil, err ++ } ++ ++ diskChecksums.ContainerDiskChecksums[volume.Name] = checksum ++ } ++ ++ return diskChecksums, nil ++} ++ ++func (m *hotplugMounter) findVirtlauncherUID(vmi *v1.VirtualMachineInstance) (uid types.UID) { ++ cnt := 0 ++ for podUID := range vmi.Status.ActivePods { ++ _, err := m.hotplugManager.GetHotplugTargetPodPathOnHost(podUID) ++ if err == nil { ++ uid = podUID ++ cnt++ ++ } ++ } ++ if cnt == 1 { ++ return ++ } ++ // Either no pods, or multiple pods, skip. ++ return types.UID("") ++} +diff --git a/pkg/virt-handler/container-disk/mount.go b/pkg/virt-handler/container-disk/mount.go +index 953c20f3af..d99bec3a43 100644 +--- a/pkg/virt-handler/container-disk/mount.go ++++ b/pkg/virt-handler/container-disk/mount.go +@@ -54,6 +54,8 @@ type mounter struct { + kernelBootSocketPathGetter containerdisk.KernelBootSocketPathGetter + clusterConfig *virtconfig.ClusterConfig + nodeIsolationResult isolation.IsolationResult ++ ++ hotplugPathGetter containerdisk.HotplugSocketPathGetter + } + + type Mounter interface { +@@ -98,6 +100,8 @@ func NewMounter(isoDetector isolation.PodIsolationDetector, mountStateDir string + kernelBootSocketPathGetter: containerdisk.NewKernelBootSocketPathGetter(""), + clusterConfig: clusterConfig, + nodeIsolationResult: isolation.NodeIsolationResult(), ++ ++ hotplugPathGetter: containerdisk.NewHotplugSocketPathGetter(""), + } + } + +@@ -254,7 +258,7 @@ func (m *mounter) MountAndVerify(vmi *v1.VirtualMachineInstance) (map[string]*co + disksInfo := map[string]*containerdisk.DiskInfo{} + + for i, volume := range vmi.Spec.Volumes { +- if volume.ContainerDisk != nil { ++ if volume.ContainerDisk != nil && !volume.ContainerDisk.Hotpluggable { + diskTargetDir, err := containerdisk.GetDiskTargetDirFromHostView(vmi) + if err != nil { + return nil, err +@@ -296,7 +300,7 @@ func (m *mounter) MountAndVerify(vmi *v1.VirtualMachineInstance) (map[string]*co + } + + for i, volume := range vmi.Spec.Volumes { +- if volume.ContainerDisk != nil { ++ if volume.ContainerDisk != nil && !volume.ContainerDisk.Hotpluggable { + diskTargetDir, err := containerdisk.GetDiskTargetDirFromHostView(vmi) + if err != nil { + return nil, err +@@ -394,7 +398,7 @@ func (m *mounter) Unmount(vmi *v1.VirtualMachineInstance) error { + + func (m *mounter) ContainerDisksReady(vmi *v1.VirtualMachineInstance, notInitializedSince time.Time) (bool, error) { + for i, volume := range vmi.Spec.Volumes { +- if volume.ContainerDisk != nil { ++ if volume.ContainerDisk != nil && !volume.ContainerDisk.Hotpluggable { + _, err := m.socketPathGetter(vmi, i) + if err != nil { + log.DefaultLogger().Object(vmi).Reason(err).Infof("containerdisk %s not yet ready", volume.Name) +@@ -706,7 +710,7 @@ func (m *mounter) ComputeChecksums(vmi *v1.VirtualMachineInstance) (*DiskChecksu + + // compute for containerdisks + for i, volume := range vmi.Spec.Volumes { +- if volume.VolumeSource.ContainerDisk == nil { ++ if volume.VolumeSource.ContainerDisk == nil || volume.VolumeSource.ContainerDisk.Hotpluggable { + continue + } + +diff --git a/pkg/virt-handler/hotplug-disk/mount.go b/pkg/virt-handler/hotplug-disk/mount.go +index 971c8d55fc..03fcec8c92 100644 +--- a/pkg/virt-handler/hotplug-disk/mount.go ++++ b/pkg/virt-handler/hotplug-disk/mount.go +@@ -310,14 +310,17 @@ func (m *volumeMounter) mountHotplugVolume( + logger := log.DefaultLogger() + logger.V(4).Infof("Hotplug check volume name: %s", volumeName) + if sourceUID != types.UID("") { +- if m.isBlockVolume(&vmi.Status, volumeName) { ++ switch { ++ case m.isContainerDisk(&vmi.Status, volumeName): ++ // skip ++ case m.isBlockVolume(&vmi.Status, volumeName): + logger.V(4).Infof("Mounting block volume: %s", volumeName) + if err := m.mountBlockHotplugVolume(vmi, volumeName, sourceUID, record, cgroupManager); err != nil { + if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("failed to mount block hotplug volume %s: %v", volumeName, err) + } + } +- } else { ++ default: + logger.V(4).Infof("Mounting file system volume: %s", volumeName) + if err := m.mountFileSystemHotplugVolume(vmi, volumeName, sourceUID, record, mountDirectory); err != nil { + if !errors.Is(err, os.ErrNotExist) { +@@ -382,6 +385,15 @@ func (m *volumeMounter) isBlockVolume(vmiStatus *v1.VirtualMachineInstanceStatus + return false + } + ++func (m *volumeMounter) isContainerDisk(vmiStatus *v1.VirtualMachineInstanceStatus, volumeName string) bool { ++ for _, status := range vmiStatus.VolumeStatus { ++ if status.Name == volumeName { ++ return status.ContainerDiskVolume != nil ++ } ++ } ++ return false ++} ++ + func (m *volumeMounter) mountBlockHotplugVolume( + vmi *v1.VirtualMachineInstance, + volume string, +diff --git a/pkg/virt-handler/isolation/detector.go b/pkg/virt-handler/isolation/detector.go +index f83f96ead4..5e38c6cedd 100644 +--- a/pkg/virt-handler/isolation/detector.go ++++ b/pkg/virt-handler/isolation/detector.go +@@ -24,6 +24,8 @@ package isolation + import ( + "fmt" + "net" ++ "os" ++ "path" + "runtime" + "syscall" + "time" +@@ -207,12 +209,45 @@ func setProcessMemoryLockRLimit(pid int, size int64) error { + return nil + } + ++type deferFunc func() ++ ++func (s *socketBasedIsolationDetector) socketHack(socket string) (sock net.Conn, deferFunc deferFunc, err error) { ++ fn := func() {} ++ if len([]rune(socket)) <= 108 { ++ sock, err = net.DialTimeout("unix", socket, time.Duration(isolationDialTimeout)*time.Second) ++ fn = func() { ++ if err == nil { ++ sock.Close() ++ } ++ } ++ return sock, fn, err ++ } ++ base := path.Base(socket) ++ newPath := fmt.Sprintf("/tmp/%s", base) ++ if err = os.Symlink(socket, newPath); err != nil { ++ return nil, fn, err ++ } ++ sock, err = net.DialTimeout("unix", newPath, time.Duration(isolationDialTimeout)*time.Second) ++ fn = func() { ++ if err == nil { ++ sock.Close() ++ } ++ os.Remove(newPath) ++ } ++ return sock, fn, err ++} ++ + func (s *socketBasedIsolationDetector) getPid(socket string) (int, error) { +- sock, err := net.DialTimeout("unix", socket, time.Duration(isolationDialTimeout)*time.Second) ++ sock, defFn, err := s.socketHack(socket) ++ defer defFn() + if err != nil { + return -1, err + } +- defer sock.Close() ++ //sock, err := net.DialTimeout("unix", socket, time.Duration(isolationDialTimeout)*time.Second) ++ //if err != nil { ++ // return -1, err ++ //} ++ //defer sock.Close() + + ufile, err := sock.(*net.UnixConn).File() + if err != nil { +diff --git a/pkg/virt-handler/virt-chroot/virt-chroot.go b/pkg/virt-handler/virt-chroot/virt-chroot.go +index 4160212b7b..580b788acc 100644 +--- a/pkg/virt-handler/virt-chroot/virt-chroot.go ++++ b/pkg/virt-handler/virt-chroot/virt-chroot.go +@@ -20,7 +20,10 @@ + package virt_chroot + + import ( ++ "bytes" ++ "fmt" + "os/exec" ++ "slices" + "strings" + + "kubevirt.io/kubevirt/pkg/safepath" +@@ -48,6 +51,49 @@ func MountChroot(sourcePath, targetPath *safepath.Path, ro bool) *exec.Cmd { + return UnsafeMountChroot(trimProcPrefix(sourcePath), trimProcPrefix(targetPath), ro) + } + ++func MountChrootWithOptions(sourcePath, targetPath *safepath.Path, mountOptions ...string) error { ++ args := append(getBaseArgs(), "mount") ++ remountArgs := slices.Clone(args) ++ ++ mountOptions = slices.DeleteFunc(mountOptions, func(s string) bool { ++ return s == "remount" ++ }) ++ if len(mountOptions) > 0 { ++ opts := strings.Join(mountOptions, ",") ++ remountOpts := "remount," + opts ++ args = append(args, "-o", opts) ++ remountArgs = append(remountArgs, "-o", remountOpts) ++ } ++ ++ sp := trimProcPrefix(sourcePath) ++ tp := trimProcPrefix(targetPath) ++ args = append(args, sp, tp) ++ remountArgs = append(remountArgs, sp, tp) ++ ++ stdout := new(bytes.Buffer) ++ stderr := new(bytes.Buffer) ++ ++ cmd := exec.Command(binaryPath, args...) ++ cmd.Stdout = stdout ++ cmd.Stderr = stderr ++ err := cmd.Run() ++ if err != nil { ++ return fmt.Errorf("mount failed: %w, stdout: %s, stderr: %s", err, stdout.String(), stderr.String()) ++ } ++ ++ stdout = new(bytes.Buffer) ++ stderr = new(bytes.Buffer) ++ ++ remountCmd := exec.Command(binaryPath, remountArgs...) ++ cmd.Stdout = stdout ++ cmd.Stderr = stderr ++ err = remountCmd.Run() ++ if err != nil { ++ return fmt.Errorf("mount failed: %w, stdout: %s, stderr: %s", err, stdout.String(), stderr.String()) ++ } ++ return nil ++} ++ + // Deprecated: UnsafeMountChroot is used to connect to code which needs to be refactored + // to handle mounts securely. + func UnsafeMountChroot(sourcePath, targetPath string, ro bool) *exec.Cmd { +diff --git a/pkg/virt-handler/vm.go b/pkg/virt-handler/vm.go +index 24352cf6e9..f1de8d1149 100644 +--- a/pkg/virt-handler/vm.go ++++ b/pkg/virt-handler/vm.go +@@ -25,6 +25,7 @@ import ( + goerror "errors" + "fmt" + "io" ++ "maps" + "net" + "os" + "path/filepath" +@@ -247,6 +248,13 @@ func NewController( + vmiExpectations: controller.NewUIDTrackingControllerExpectations(controller.NewControllerExpectations()), + sriovHotplugExecutorPool: executor.NewRateLimitedExecutorPool(executor.NewExponentialLimitedBackoffCreator()), + ioErrorRetryManager: NewFailRetryManager("io-error-retry", 10*time.Second, 3*time.Minute, 30*time.Second), ++ ++ hotplugContainerDiskMounter: container_disk.NewHotplugMounter( ++ podIsolationDetector, ++ filepath.Join(virtPrivateDir, "hotplug-container-disk-mount-state"), ++ clusterConfig, ++ hotplugdisk.NewHotplugDiskManager(kubeletPodsDir), ++ ), + } + + _, err := vmiSourceInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ +@@ -342,6 +350,8 @@ type VirtualMachineController struct { + hostCpuModel string + vmiExpectations *controller.UIDTrackingControllerExpectations + ioErrorRetryManager *FailRetryManager ++ ++ hotplugContainerDiskMounter container_disk.HotplugMounter + } + + type virtLauncherCriticalSecurebootError struct { +@@ -876,7 +886,15 @@ func (d *VirtualMachineController) updateHotplugVolumeStatus(vmi *v1.VirtualMach + needsRefresh := false + if volumeStatus.Target == "" { + needsRefresh = true +- mounted, err := d.hotplugVolumeMounter.IsMounted(vmi, volumeStatus.Name, volumeStatus.HotplugVolume.AttachPodUID) ++ var ( ++ mounted bool ++ err error ++ ) ++ if volumeStatus.ContainerDiskVolume != nil { ++ mounted, err = d.hotplugContainerDiskMounter.IsMounted(vmi, volumeStatus.Name) ++ } else { ++ mounted, err = d.hotplugVolumeMounter.IsMounted(vmi, volumeStatus.Name, volumeStatus.HotplugVolume.AttachPodUID) ++ } + if err != nil { + log.Log.Object(vmi).Errorf("error occurred while checking if volume is mounted: %v", err) + } +@@ -898,6 +916,7 @@ func (d *VirtualMachineController) updateHotplugVolumeStatus(vmi *v1.VirtualMach + volumeStatus.Reason = VolumeUnMountedFromPodReason + } + } ++ + } else { + // Successfully attached to VM. + volumeStatus.Phase = v1.VolumeReady +@@ -2178,6 +2197,11 @@ func (d *VirtualMachineController) processVmCleanup(vmi *v1.VirtualMachineInstan + return err + } + ++ err := d.hotplugContainerDiskMounter.UmountAll(vmi) ++ if err != nil { ++ return err ++ } ++ + // UnmountAll does the cleanup on the "best effort" basis: it is + // safe to pass a nil cgroupManager. + cgroupManager, _ := getCgroupManager(vmi) +@@ -2829,6 +2853,12 @@ func (d *VirtualMachineController) vmUpdateHelperMigrationTarget(origVMI *v1.Vir + return err + } + ++ hotplugDiskInfo, err := d.hotplugContainerDiskMounter.MountAndVerify(vmi) ++ if err != nil { ++ return err ++ } ++ maps.Copy(disksInfo, hotplugDiskInfo) ++ + // Mount hotplug disks + if attachmentPodUID := vmi.Status.MigrationState.TargetAttachmentPodUID; attachmentPodUID != types.UID("") { + cgroupManager, err := getCgroupManager(vmi) +@@ -3051,6 +3081,11 @@ func (d *VirtualMachineController) vmUpdateHelperDefault(origVMI *v1.VirtualMach + if err != nil { + return err + } ++ hotplugDiskInfo, err := d.hotplugContainerDiskMounter.MountAndVerify(vmi) ++ if err != nil { ++ return err ++ } ++ maps.Copy(disksInfo, hotplugDiskInfo) + + // Try to mount hotplug volume if there is any during startup. + if err := d.hotplugVolumeMounter.Mount(vmi, cgroupManager); err != nil { +@@ -3138,6 +3173,11 @@ func (d *VirtualMachineController) vmUpdateHelperDefault(origVMI *v1.VirtualMach + log.Log.Object(vmi).Error(err.Error()) + } + ++ hotplugDiskInfo, err := d.hotplugContainerDiskMounter.MountAndVerify(vmi) ++ if err != nil { ++ return err ++ } ++ maps.Copy(disksInfo, hotplugDiskInfo) + if err := d.hotplugVolumeMounter.Mount(vmi, cgroupManager); err != nil { + return err + } +@@ -3215,6 +3255,9 @@ func (d *VirtualMachineController) vmUpdateHelperDefault(origVMI *v1.VirtualMach + + if vmi.IsRunning() { + // Umount any disks no longer mounted ++ if err := d.hotplugContainerDiskMounter.Umount(vmi); err != nil { ++ return err ++ } + if err := d.hotplugVolumeMounter.Unmount(vmi, cgroupManager); err != nil { + return err + } +diff --git a/pkg/virt-launcher/virtwrap/converter/converter.go b/pkg/virt-launcher/virtwrap/converter/converter.go +index 3318c1c466..1286ef4a06 100644 +--- a/pkg/virt-launcher/virtwrap/converter/converter.go ++++ b/pkg/virt-launcher/virtwrap/converter/converter.go +@@ -649,6 +649,9 @@ func Convert_v1_Hotplug_Volume_To_api_Disk(source *v1.Volume, disk *api.Disk, c + if source.DataVolume != nil { + return Convert_v1_Hotplug_DataVolume_To_api_Disk(source.Name, disk, c) + } ++ if source.ContainerDisk != nil { ++ return Convert_v1_Hotplug_ContainerDisk_To_api_Disk(source.Name, disk, c) ++ } + return fmt.Errorf("hotplug disk %s references an unsupported source", disk.Alias.GetName()) + } + +@@ -690,6 +693,10 @@ func GetHotplugBlockDeviceVolumePath(volumeName string) string { + return filepath.Join(string(filepath.Separator), "var", "run", "kubevirt", "hotplug-disks", volumeName) + } + ++func GetHotplugContainerDiskPath(volumeName string) string { ++ return filepath.Join(string(filepath.Separator), "var", "run", "kubevirt", "hotplug-disks", volumeName) ++} ++ + func Convert_v1_PersistentVolumeClaim_To_api_Disk(name string, disk *api.Disk, c *ConverterContext) error { + if c.IsBlockPVC[name] { + return Convert_v1_BlockVolumeSource_To_api_Disk(name, disk, c.VolumesDiscardIgnore) +@@ -768,6 +775,34 @@ func Convert_v1_Hotplug_BlockVolumeSource_To_api_Disk(volumeName string, disk *a + return nil + } + ++func Convert_v1_Hotplug_ContainerDisk_To_api_Disk(volumeName string, disk *api.Disk, c *ConverterContext) error { ++ if disk.Type == "lun" { ++ return fmt.Errorf(deviceTypeNotCompatibleFmt, disk.Alias.GetName()) ++ } ++ disk.Type = "file" ++ disk.Driver.Type = "qcow2" ++ disk.Driver.ErrorPolicy = v1.DiskErrorPolicyStop ++ disk.ReadOnly = &api.ReadOnly{} ++ if !contains(c.VolumesDiscardIgnore, volumeName) { ++ disk.Driver.Discard = "unmap" ++ } ++ disk.Source.File = GetHotplugContainerDiskPath(volumeName) ++ disk.BackingStore = &api.BackingStore{ ++ Format: &api.BackingStoreFormat{}, ++ Source: &api.DiskSource{}, ++ } ++ ++ if info := c.DisksInfo[volumeName]; info != nil { ++ disk.BackingStore.Format.Type = info.Format ++ disk.BackingStore.Source.File = info.BackingFile ++ } else { ++ return fmt.Errorf("no disk info provided for volume %s", volumeName) ++ } ++ disk.BackingStore.Type = "file" ++ ++ return nil ++} ++ + func Convert_v1_HostDisk_To_api_Disk(volumeName string, path string, disk *api.Disk) error { + disk.Type = "file" + disk.Driver.Type = "raw" +diff --git a/pkg/virt-operator/resource/apply/BUILD.bazel b/pkg/virt-operator/resource/apply/BUILD.bazel +index f6bd9bd4f1..fe6ab54f8c 100644 +--- a/pkg/virt-operator/resource/apply/BUILD.bazel ++++ b/pkg/virt-operator/resource/apply/BUILD.bazel +@@ -4,7 +4,6 @@ go_library( + name = "go_default_library", + srcs = [ + "admissionregistration.go", +- "apiservices.go", + "apps.go", + "certificates.go", + "core.go", +@@ -65,7 +64,6 @@ go_library( + "//vendor/k8s.io/client-go/tools/cache:go_default_library", + "//vendor/k8s.io/client-go/tools/record:go_default_library", + "//vendor/k8s.io/client-go/util/workqueue:go_default_library", +- "//vendor/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1:go_default_library", + "//vendor/k8s.io/utils/pointer:go_default_library", + ], + ) +diff --git a/pkg/virt-operator/resource/generate/components/BUILD.bazel b/pkg/virt-operator/resource/generate/components/BUILD.bazel +index 70d2da0897..affcd3fecd 100644 +--- a/pkg/virt-operator/resource/generate/components/BUILD.bazel ++++ b/pkg/virt-operator/resource/generate/components/BUILD.bazel +@@ -3,7 +3,6 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + go_library( + name = "go_default_library", + srcs = [ +- "apiservices.go", + "crds.go", + "daemonsets.go", + "deployments.go", +@@ -62,7 +61,6 @@ go_library( + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/yaml:go_default_library", +- "//vendor/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1:go_default_library", + "//vendor/k8s.io/utils/pointer:go_default_library", + ], + ) +@@ -70,7 +68,6 @@ go_library( + go_test( + name = "go_default_test", + srcs = [ +- "apiservices_test.go", + "components_suite_test.go", + "crds_test.go", + "deployments_test.go", +@@ -85,7 +82,6 @@ go_test( + deps = [ + "//pkg/certificates/bootstrap:go_default_library", + "//pkg/certificates/triple/cert:go_default_library", +- "//staging/src/kubevirt.io/api/core/v1:go_default_library", + "//staging/src/kubevirt.io/client-go/testutils:go_default_library", + "//vendor/github.com/onsi/ginkgo/v2:go_default_library", + "//vendor/github.com/onsi/gomega:go_default_library", +diff --git a/pkg/virt-operator/resource/generate/components/validations_generated.go b/pkg/virt-operator/resource/generate/components/validations_generated.go +index 4913dbead0..42225780ba 100644 +--- a/pkg/virt-operator/resource/generate/components/validations_generated.go ++++ b/pkg/virt-operator/resource/generate/components/validations_generated.go +@@ -7723,6 +7723,8 @@ var CRDsValidation map[string]string = map[string]string{ + ContainerDisk references a docker image, embedding a qcow or raw disk. + More info: https://kubevirt.gitbooks.io/user-guide/registry-disk.html + properties: ++ hotpluggable: ++ type: boolean + image: + description: Image is the name of the image with the embedded + disk. +@@ -8355,6 +8357,35 @@ var CRDsValidation map[string]string = map[string]string{ + description: VolumeSource represents the source of the volume + to map to the disk. + properties: ++ containerDisk: ++ description: Represents a docker image with an embedded disk. ++ properties: ++ hotpluggable: ++ type: boolean ++ image: ++ description: Image is the name of the image with the embedded ++ disk. ++ type: string ++ imagePullPolicy: ++ description: |- ++ Image pull policy. ++ One of Always, Never, IfNotPresent. ++ Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. ++ Cannot be updated. ++ More info: https://kubernetes.io/docs/concepts/containers/images#updating-images ++ type: string ++ imagePullSecret: ++ description: ImagePullSecret is the name of the Docker ++ registry secret required to pull the image. The secret ++ must already exist. ++ type: string ++ path: ++ description: Path defines the path to disk file in the ++ container ++ type: string ++ required: ++ - image ++ type: object + dataVolume: + description: |- + DataVolume represents the dynamic creation a PVC for this volume as well as +@@ -12768,6 +12799,8 @@ var CRDsValidation map[string]string = map[string]string{ + ContainerDisk references a docker image, embedding a qcow or raw disk. + More info: https://kubevirt.gitbooks.io/user-guide/registry-disk.html + properties: ++ hotpluggable: ++ type: boolean + image: + description: Image is the name of the image with the embedded + disk. +@@ -18328,6 +18361,8 @@ var CRDsValidation map[string]string = map[string]string{ + ContainerDisk references a docker image, embedding a qcow or raw disk. + More info: https://kubevirt.gitbooks.io/user-guide/registry-disk.html + properties: ++ hotpluggable: ++ type: boolean + image: + description: Image is the name of the image with the embedded + disk. +@@ -22835,6 +22870,8 @@ var CRDsValidation map[string]string = map[string]string{ + ContainerDisk references a docker image, embedding a qcow or raw disk. + More info: https://kubevirt.gitbooks.io/user-guide/registry-disk.html + properties: ++ hotpluggable: ++ type: boolean + image: + description: Image is the name of the image with + the embedded disk. +@@ -28015,6 +28052,8 @@ var CRDsValidation map[string]string = map[string]string{ + ContainerDisk references a docker image, embedding a qcow or raw disk. + More info: https://kubevirt.gitbooks.io/user-guide/registry-disk.html + properties: ++ hotpluggable: ++ type: boolean + image: + description: Image is the name of the image + with the embedded disk. +@@ -28673,6 +28712,36 @@ var CRDsValidation map[string]string = map[string]string{ + description: VolumeSource represents the source of + the volume to map to the disk. + properties: ++ containerDisk: ++ description: Represents a docker image with an ++ embedded disk. ++ properties: ++ hotpluggable: ++ type: boolean ++ image: ++ description: Image is the name of the image ++ with the embedded disk. ++ type: string ++ imagePullPolicy: ++ description: |- ++ Image pull policy. ++ One of Always, Never, IfNotPresent. ++ Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. ++ Cannot be updated. ++ More info: https://kubernetes.io/docs/concepts/containers/images#updating-images ++ type: string ++ imagePullSecret: ++ description: ImagePullSecret is the name of ++ the Docker registry secret required to pull ++ the image. The secret must already exist. ++ type: string ++ path: ++ description: Path defines the path to disk ++ file in the container ++ type: string ++ required: ++ - image ++ type: object + dataVolume: + description: |- + DataVolume represents the dynamic creation a PVC for this volume as well as +diff --git a/pkg/virt-operator/resource/generate/install/generated_mock_strategy.go b/pkg/virt-operator/resource/generate/install/generated_mock_strategy.go +index 5f1e9a3121..1fa1416af0 100644 +--- a/pkg/virt-operator/resource/generate/install/generated_mock_strategy.go ++++ b/pkg/virt-operator/resource/generate/install/generated_mock_strategy.go +@@ -241,16 +241,6 @@ func (_mr *_MockStrategyInterfaceRecorder) MutatingWebhookConfigurations() *gomo + return _mr.mock.ctrl.RecordCall(_mr.mock, "MutatingWebhookConfigurations") + } + +-func (_m *MockStrategyInterface) APIServices() []*v18.APIService { +- ret := _m.ctrl.Call(_m, "APIServices") +- ret0, _ := ret[0].([]*v18.APIService) +- return ret0 +-} +- +-func (_mr *_MockStrategyInterfaceRecorder) APIServices() *gomock.Call { +- return _mr.mock.ctrl.RecordCall(_mr.mock, "APIServices") +-} +- + func (_m *MockStrategyInterface) CertificateSecrets() []*v14.Secret { + ret := _m.ctrl.Call(_m, "CertificateSecrets") + ret0, _ := ret[0].([]*v14.Secret) +diff --git a/pkg/virt-operator/resource/generate/rbac/exportproxy.go b/pkg/virt-operator/resource/generate/rbac/exportproxy.go +index ebc9f2adbd..a0dc0586b4 100644 +--- a/pkg/virt-operator/resource/generate/rbac/exportproxy.go ++++ b/pkg/virt-operator/resource/generate/rbac/exportproxy.go +@@ -23,6 +23,7 @@ import ( + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" ++ + "kubevirt.io/kubevirt/pkg/virt-operator/resource/generate/components" + + virtv1 "kubevirt.io/api/core/v1" +diff --git a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.json b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.json +index b651173636..3453dfb0da 100644 +--- a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.json ++++ b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.json +@@ -754,7 +754,8 @@ + "image": "imageValue", + "imagePullSecret": "imagePullSecretValue", + "path": "pathValue", +- "imagePullPolicy": "imagePullPolicyValue" ++ "imagePullPolicy": "imagePullPolicyValue", ++ "hotpluggable": true + }, + "ephemeral": { + "persistentVolumeClaim": { +@@ -1209,6 +1210,13 @@ + "dataVolume": { + "name": "nameValue", + "hotpluggable": true ++ }, ++ "containerDisk": { ++ "image": "imageValue", ++ "imagePullSecret": "imagePullSecretValue", ++ "path": "pathValue", ++ "imagePullPolicy": "imagePullPolicyValue", ++ "hotpluggable": true + } + }, + "dryRun": [ +diff --git a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.yaml b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.yaml +index 53dfdacc3b..8b23193158 100644 +--- a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.yaml ++++ b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachine.yaml +@@ -719,6 +719,7 @@ spec: + optional: true + volumeLabel: volumeLabelValue + containerDisk: ++ hotpluggable: true + image: imageValue + imagePullPolicy: imagePullPolicyValue + imagePullSecret: imagePullSecretValue +@@ -838,6 +839,12 @@ status: + - dryRunValue + name: nameValue + volumeSource: ++ containerDisk: ++ hotpluggable: true ++ image: imageValue ++ imagePullPolicy: imagePullPolicyValue ++ imagePullSecret: imagePullSecretValue ++ path: pathValue + dataVolume: + hotpluggable: true + name: nameValue +diff --git a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.json b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.json +index 3be904512c..f595798e89 100644 +--- a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.json ++++ b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.json +@@ -694,7 +694,8 @@ + "image": "imageValue", + "imagePullSecret": "imagePullSecretValue", + "path": "pathValue", +- "imagePullPolicy": "imagePullPolicyValue" ++ "imagePullPolicy": "imagePullPolicyValue", ++ "hotpluggable": true + }, + "ephemeral": { + "persistentVolumeClaim": { +diff --git a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.yaml b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.yaml +index 6fd2ab6523..b6457ec94d 100644 +--- a/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.yaml ++++ b/staging/src/kubevirt.io/api/apitesting/testdata/HEAD/kubevirt.io.v1.VirtualMachineInstance.yaml +@@ -524,6 +524,7 @@ spec: + optional: true + volumeLabel: volumeLabelValue + containerDisk: ++ hotpluggable: true + image: imageValue + imagePullPolicy: imagePullPolicyValue + imagePullSecret: imagePullSecretValue +diff --git a/staging/src/kubevirt.io/api/core/v1/BUILD.bazel b/staging/src/kubevirt.io/api/core/v1/BUILD.bazel +index f8615293a3..0c6c166985 100644 +--- a/staging/src/kubevirt.io/api/core/v1/BUILD.bazel ++++ b/staging/src/kubevirt.io/api/core/v1/BUILD.bazel +@@ -28,7 +28,6 @@ go_library( + "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", + "//vendor/k8s.io/utils/net:go_default_library", +- "//vendor/k8s.io/utils/pointer:go_default_library", + "//vendor/kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1:go_default_library", + ], + ) +diff --git a/staging/src/kubevirt.io/api/core/v1/deepcopy_generated.go b/staging/src/kubevirt.io/api/core/v1/deepcopy_generated.go +index abd5a495d6..7372b22a9a 100644 +--- a/staging/src/kubevirt.io/api/core/v1/deepcopy_generated.go ++++ b/staging/src/kubevirt.io/api/core/v1/deepcopy_generated.go +@@ -1948,6 +1948,11 @@ func (in *HotplugVolumeSource) DeepCopyInto(out *HotplugVolumeSource) { + *out = new(DataVolumeSource) + **out = **in + } ++ if in.ContainerDisk != nil { ++ in, out := &in.ContainerDisk, &out.ContainerDisk ++ *out = new(ContainerDiskSource) ++ **out = **in ++ } + return + } + +diff --git a/staging/src/kubevirt.io/api/core/v1/schema.go b/staging/src/kubevirt.io/api/core/v1/schema.go +index 29aa3932d3..302ed9ffde 100644 +--- a/staging/src/kubevirt.io/api/core/v1/schema.go ++++ b/staging/src/kubevirt.io/api/core/v1/schema.go +@@ -854,6 +854,8 @@ type HotplugVolumeSource struct { + // the process of populating that PVC with a disk image. + // +optional + DataVolume *DataVolumeSource `json:"dataVolume,omitempty"` ++ ++ ContainerDisk *ContainerDiskSource `json:"containerDisk,omitempty"` + } + + type DataVolumeSource struct { +@@ -911,6 +913,8 @@ type ContainerDiskSource struct { + // More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + // +optional + ImagePullPolicy v1.PullPolicy `json:"imagePullPolicy,omitempty"` ++ ++ Hotpluggable bool `json:"hotpluggable,omitempty"` + } + + // Exactly one of its members must be set. +diff --git a/staging/src/kubevirt.io/client-go/api/openapi_generated.go b/staging/src/kubevirt.io/client-go/api/openapi_generated.go +index cc2d743492..b982b1620c 100644 +--- a/staging/src/kubevirt.io/client-go/api/openapi_generated.go ++++ b/staging/src/kubevirt.io/client-go/api/openapi_generated.go +@@ -17772,6 +17772,12 @@ func schema_kubevirtio_api_core_v1_ContainerDiskSource(ref common.ReferenceCallb + Enum: []interface{}{"Always", "IfNotPresent", "Never"}, + }, + }, ++ "hotpluggable": { ++ SchemaProps: spec.SchemaProps{ ++ Type: []string{"boolean"}, ++ Format: "", ++ }, ++ }, + }, + Required: []string{"image"}, + }, +@@ -19645,11 +19651,16 @@ func schema_kubevirtio_api_core_v1_HotplugVolumeSource(ref common.ReferenceCallb + Ref: ref("kubevirt.io/api/core/v1.DataVolumeSource"), + }, + }, ++ "containerDisk": { ++ SchemaProps: spec.SchemaProps{ ++ Ref: ref("kubevirt.io/api/core/v1.ContainerDiskSource"), ++ }, ++ }, + }, + }, + }, + Dependencies: []string{ +- "kubevirt.io/api/core/v1.DataVolumeSource", "kubevirt.io/api/core/v1.PersistentVolumeClaimVolumeSource"}, ++ "kubevirt.io/api/core/v1.ContainerDiskSource", "kubevirt.io/api/core/v1.DataVolumeSource", "kubevirt.io/api/core/v1.PersistentVolumeClaimVolumeSource"}, + } + } + diff --git a/images/virt-artifact/patches/README.md b/images/virt-artifact/patches/README.md index 862b203507..cad007da6d 100644 --- a/images/virt-artifact/patches/README.md +++ b/images/virt-artifact/patches/README.md @@ -110,3 +110,7 @@ How does it work? 2. When changing the affinity or nodeSelector in vmi, the vmi controller will set the `NodePlacementNotMatched` condition to True in vmi. 3. The workload-updater controller monitors the vmi and starts migration when there is a `NodePlacementNotMatched` conditions on the vmi. 4. When the migration is completed, virt-handler will remove the condition `NodePlacementNotMatched` from the vmi + +#### `028-hotplug-container-disk.patch` + +Hotplug container-disk volumes.