From 4aa82a4cbea3af53c5884000e07c824e7a48ee60 Mon Sep 17 00:00:00 2001 From: Aleksandr Zimin Date: Tue, 4 Jun 2024 18:32:02 +0300 Subject: [PATCH] [csi] Fix snapshots (#9) Signed-off-by: Aleksandr Zimin Signed-off-by: Alexandr Ohrimenko Co-authored-by: Alexandr Ohrimenko --- .werf/bundle.yaml | 2 +- images/controller/Dockerfile | 4 +- ...add-functionality-of-the-tar-utility.patch | 208 ++++++++++++++++++ images/csi-nfs/werf.inc.yaml | 18 +- images/webhooks/src/main.go | 1 - images/webhooks/werf.inc.yaml | 24 +- templates/csi/controller.yaml | 1 + 7 files changed, 249 insertions(+), 9 deletions(-) create mode 100644 images/csi-nfs/patches/0001-add-functionality-of-the-tar-utility.patch diff --git a/.werf/bundle.yaml b/.werf/bundle.yaml index a2242b0..dbeaadf 100644 --- a/.werf/bundle.yaml +++ b/.werf/bundle.yaml @@ -2,7 +2,7 @@ --- image: bundle from: registry.deckhouse.io/base_images/scratch@sha256:b054705fcc9f2205777d80a558d920c0b4209efdc3163c22b5bfcb5dda1db5fc -fromCacheVersion: "2024-05-12.1" +fromCacheVersion: "2024-05-14.1" import: # Rendering .werf/images-digests.yaml is required! - image: images-digests diff --git a/images/controller/Dockerfile b/images/controller/Dockerfile index f98b4ad..4234b88 100644 --- a/images/controller/Dockerfile +++ b/images/controller/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_ALPINE=registry.deckhouse.io/base_images/alpine:3.16.3@sha256:5548e9172c24a1b0ca9afdd2bf534e265c94b12b36b3e0c0302f5853eaf00abb +ARG BASE_SCRATCH=registry.deckhouse.io/base_images/scratch@sha256:b054705fcc9f2205777d80a558d920c0b4209efdc3163c22b5bfcb5dda1db5fc ARG BASE_GOLANG_21_ALPINE_BUILDER=registry.deckhouse.io/base_images/golang:1.21.4-alpine3.18@sha256:cf84f3d6882c49ea04b6478ac514a2582c8922d7e5848b43d2918fff8329f6e6 FROM $BASE_GOLANG_21_ALPINE_BUILDER as builder @@ -11,7 +11,7 @@ COPY . . WORKDIR /go/src/cmd RUN GOOS=linux GOARCH=amd64 go build -o controller -FROM --platform=linux/amd64 $BASE_ALPINE +FROM --platform=linux/amd64 $BASE_SCRATCH COPY --from=builder /go/src/cmd/controller /go/src/cmd/controller ENTRYPOINT ["/go/src/cmd/controller"] diff --git a/images/csi-nfs/patches/0001-add-functionality-of-the-tar-utility.patch b/images/csi-nfs/patches/0001-add-functionality-of-the-tar-utility.patch new file mode 100644 index 0000000..e47ca43 --- /dev/null +++ b/images/csi-nfs/patches/0001-add-functionality-of-the-tar-utility.patch @@ -0,0 +1,208 @@ +From f514b56bbe619c7d85db12483182620fb6349819 Mon Sep 17 00:00:00 2001 +From: Alexandr Ohrimenko +Date: Mon, 20 May 2024 12:25:31 +0300 +Subject: [PATCH] add functionality of the tar utility + +Signed-off-by: Alexandr Ohrimenko +--- + pkg/nfs/controllerserver.go | 10 ++- + pkg/nfs/func_tar.go | 157 ++++++++++++++++++++++++++++++++++++ + 2 files changed, 163 insertions(+), 4 deletions(-) + create mode 100644 pkg/nfs/func_tar.go + +diff --git a/pkg/nfs/controllerserver.go b/pkg/nfs/controllerserver.go +index 7fdad5dd..726df875 100644 +--- a/pkg/nfs/controllerserver.go ++++ b/pkg/nfs/controllerserver.go +@@ -360,9 +360,10 @@ func (cs *ControllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateS + srcPath := getInternalVolumePath(cs.Driver.workingMountDir, srcVol) + dstPath := filepath.Join(snapInternalVolPath, snapshot.archiveName()) + klog.V(2).Infof("archiving %v -> %v", srcPath, dstPath) +- out, err := exec.Command("tar", "-C", srcPath, "-czvf", dstPath, ".").CombinedOutput() ++ //out, err := exec.Command("tar", "-C", srcPath, "-czvf", dstPath, ".").CombinedOutput() ++ err = tarPack(dstPath, srcPath, true) + if err != nil { +- return nil, status.Errorf(codes.Internal, "failed to create archive for snapshot: %v: %v", err, string(out)) ++ return nil, status.Errorf(codes.Internal, "failed to create archive for snapshot: %v", err) + } + klog.V(2).Infof("archived %s -> %s", srcPath, dstPath) + +@@ -515,9 +516,10 @@ func (cs *ControllerServer) copyFromSnapshot(ctx context.Context, req *csi.Creat + snapPath := filepath.Join(getInternalVolumePath(cs.Driver.workingMountDir, snapVol), snap.archiveName()) + dstPath := getInternalVolumePath(cs.Driver.workingMountDir, dstVol) + klog.V(2).Infof("copy volume from snapshot %v -> %v", snapPath, dstPath) +- out, err := exec.Command("tar", "-xzvf", snapPath, "-C", dstPath).CombinedOutput() ++ //out, err := exec.Command("tar", "-xzvf", snapPath, "-C", dstPath).CombinedOutput() ++ err = tarUnPack(snapPath, dstPath, true) + if err != nil { +- return status.Errorf(codes.Internal, "failed to copy volume for snapshot: %v: %v", err, string(out)) ++ return status.Errorf(codes.Internal, "failed to copy volume for snapshot: %v", err) + } + klog.V(2).Infof("volume copied from snapshot %v -> %v", snapPath, dstPath) + return nil +diff --git a/pkg/nfs/func_tar.go b/pkg/nfs/func_tar.go +new file mode 100644 +index 00000000..0131b564 +--- /dev/null ++++ b/pkg/nfs/func_tar.go +@@ -0,0 +1,157 @@ ++package nfs ++ ++import ( ++ "archive/tar" ++ "compress/gzip" ++ "errors" ++ "fmt" ++ "io" ++ "os" ++ "path/filepath" ++) ++ ++func tarPack(dstPathAndFileName, srcPath string, enableCompression bool) error { ++ tarFile, err := os.Create(dstPathAndFileName) ++ if err != nil { ++ return err ++ } ++ defer func() { ++ err = tarFile.Close() ++ }() ++ ++ absDstPathAndFileName, err := filepath.Abs(dstPathAndFileName) ++ if err != nil { ++ return err ++ } ++ ++ tarWriter := tar.NewWriter(tarFile) ++ if enableCompression { ++ gzipWriter := gzip.NewWriter(tarFile) ++ defer gzipWriter.Close() ++ tarWriter = tar.NewWriter(gzipWriter) ++ } ++ defer tarWriter.Close() ++ ++ srcPath = filepath.Clean(srcPath) ++ absSrcPath, err := filepath.Abs(srcPath) ++ if err != nil { ++ return err ++ } ++ if absSrcPath == absDstPathAndFileName { ++ return errors.New(fmt.Sprintf("tar file %s cannot be the source\n", dstPathAndFileName)) ++ } ++ if absSrcPath == filepath.Dir(absDstPathAndFileName) { ++ return errors.New(fmt.Sprintf("tar file %s cannot be in source directory %s\n", dstPathAndFileName, absSrcPath)) ++ } ++ ++ walker := func(file string, fileInfo os.FileInfo, err error) error { ++ if err != nil { ++ return err ++ } ++ ++ tarHeader, err := tar.FileInfoHeader(fileInfo, fileInfo.Name()) ++ if err != nil { ++ return err ++ } ++ ++ relFilePath := file ++ if filepath.IsAbs(srcPath) { ++ relFilePath, err = filepath.Rel(srcPath, file) ++ if err != nil { ++ return err ++ } ++ } ++ tarHeader.Name = relFilePath ++ if err = tarWriter.WriteHeader(tarHeader); err != nil { ++ return err ++ } ++ ++ if fileInfo.Mode().IsDir() { ++ return nil ++ } ++ ++ srcFile, err := os.Open(file) ++ if err != nil { ++ return err ++ } ++ defer srcFile.Close() ++ _, err = io.Copy(tarWriter, srcFile) ++ if err != nil { ++ return err ++ } ++ return nil ++ } ++ ++ if err = filepath.Walk(srcPath, walker); err != nil { ++ return err ++ } ++ return nil ++} ++ ++func tarUnPack(sourcePathAndFileName, desPath string, enableCompression bool) error { ++ tarFile, err := os.Open(sourcePathAndFileName) ++ if err != nil { ++ return err ++ } ++ defer func() { ++ err = tarFile.Close() ++ }() ++ ++ absPath, err := filepath.Abs(desPath) ++ if err != nil { ++ return err ++ } ++ ++ tarReader := tar.NewReader(tarFile) ++ if enableCompression { ++ gzipReader, err := gzip.NewReader(tarFile) ++ if err != nil { ++ return err ++ } ++ defer gzipReader.Close() ++ tarReader = tar.NewReader(gzipReader) ++ } ++ ++ for { ++ tarHeader, err := tarReader.Next() ++ if err == io.EOF { ++ break ++ } ++ if err != nil { ++ return err ++ } ++ ++ fileInfo := tarHeader.FileInfo() ++ fileName := tarHeader.Name ++ if filepath.IsAbs(fileName) { ++ fileName, err = filepath.Rel("/", fileName) ++ if err != nil { ++ return err ++ } ++ } ++ absFileName := filepath.Join(absPath, fileName) ++ ++ if fileInfo.Mode().IsDir() { ++ if err := os.MkdirAll(absFileName, 0755); err != nil { ++ return err ++ } ++ continue ++ } ++ ++ file, err := os.OpenFile(absFileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, fileInfo.Mode().Perm()) ++ if err != nil { ++ return err ++ } ++ n, cpErr := io.Copy(file, tarReader) ++ if closeErr := file.Close(); closeErr != nil { ++ return err ++ } ++ if cpErr != nil { ++ return cpErr ++ } ++ if n != fileInfo.Size() { ++ return fmt.Errorf("unexpected bytes written: wrote %d, want %d", n, fileInfo.Size()) ++ } ++ } ++ return nil ++} +-- +2.39.3 (Apple Git-146) + diff --git a/images/csi-nfs/werf.inc.yaml b/images/csi-nfs/werf.inc.yaml index 258530c..b551851 100644 --- a/images/csi-nfs/werf.inc.yaml +++ b/images/csi-nfs/werf.inc.yaml @@ -1,3 +1,5 @@ +{{- $version := "4.7.0" }} + {{- $_ := set . "BASE_GOLANG_22_ALPINE" "registry.deckhouse.io/base_images/golang:1.22.1-alpine@sha256:0de6cf7cceab6ecbf0718bdfb675b08b78113c3709c5e4b99456cdb2ae8c2495" }} {{- $_ := set . "BASE_SCRATCH" "registry.deckhouse.io/base_images/scratch@sha256:b054705fcc9f2205777d80a558d920c0b4209efdc3163c22b5bfcb5dda1db5fc" }} {{- $_ := set . "BASE_ALPINE_DEV" "registry.deckhouse.io/base_images/dev-alpine:3.16.3@sha256:c706fa83cc129079e430480369a3f062b8178cac9ec89266ebab753a574aca8e" }} @@ -6,6 +8,15 @@ --- image: {{ $.ImageName }}-golang-artifact from: {{ $.BASE_GOLANG_22_ALPINE }} +fromCacheVersion: "2024-05-14.1" +git: + - add: /images/{{ $.ImageName }} + to: / + stageDependencies: + setup: + - "**/*" + includePaths: + - patches mount: - fromPath: ~/go-pkg-cache @@ -16,19 +27,22 @@ shell: install: - export GO_VERSION={{ env "GOLANG_VERSION" }} - export GOPROXY={{ env "GOPROXY" }} - - git clone --depth 1 --branch v4.7.0 {{ env "SOURCE_REPO" }}/kubernetes-csi/csi-driver-nfs.git /csi-driver-nfs + - git clone --depth 1 --branch v{{ $version }} {{ env "SOURCE_REPO" }}/kubernetes-csi/csi-driver-nfs.git /csi-driver-nfs + - cd /csi-driver-nfs + - for patchfile in /patches/*.patch ; do echo -n "Apply ${patchfile} ... "; git apply ${patchfile}; done - cd /csi-driver-nfs/cmd/nfsplugin - GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags="-s -w" -o /nfsplugin - chmod +x /nfsplugin --- -{{ $csiBinaries := "/bin/mount /bin/umount /sbin/mount.nfs /sbin/mount.nfs4" }} +{{ $csiBinaries := "/bin/mount /bin/umount /sbin/mount.nfs /sbin/mount.nfs4 /sbin/umount.nfs /sbin/umount.nfs4 /bin/tar /bin/gzip /bin/cp" }} --- image: {{ $.ImageName }}-binaries-artifact from: {{ $.BASE_ALT_DEV }} shell: install: + - apt update && apt install nfs-common - /binary_replace.sh -i "{{ $csiBinaries }}" -o /relocate --- diff --git a/images/webhooks/src/main.go b/images/webhooks/src/main.go index 6555aab..f8953d0 100644 --- a/images/webhooks/src/main.go +++ b/images/webhooks/src/main.go @@ -33,7 +33,6 @@ type config struct { keyFile string } -//goland:noinspection SpellCheckingInspection func httpHandlerHealthz(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, "Ok.") } diff --git a/images/webhooks/werf.inc.yaml b/images/webhooks/werf.inc.yaml index 50aafaf..f3ef07a 100644 --- a/images/webhooks/werf.inc.yaml +++ b/images/webhooks/werf.inc.yaml @@ -1,6 +1,11 @@ +{{- $_ := set . "BASE_GOLANG_22_ALPINE" "registry.deckhouse.io/base_images/golang:1.22.1-alpine@sha256:0de6cf7cceab6ecbf0718bdfb675b08b78113c3709c5e4b99456cdb2ae8c2495" }} +{{- $_ := set . "BASE_SCRATCH" "registry.deckhouse.io/base_images/scratch@sha256:b054705fcc9f2205777d80a558d920c0b4209efdc3163c22b5bfcb5dda1db5fc" }} + + + --- -image: webhooks -from: "registry.deckhouse.io/base_images/golang:1.22.1-alpine@sha256:0de6cf7cceab6ecbf0718bdfb675b08b78113c3709c5e4b99456cdb2ae8c2495" +image: {{ $.ImageName }}-golang-artifact +from: {{ $.BASE_GOLANG_22_ALPINE }} git: - add: /images/webhooks/src @@ -8,9 +13,22 @@ git: stageDependencies: setup: - "**/*" - +mount: + - fromPath: ~/go-pkg-cache + to: /go/pkg shell: setup: - cd /src - GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags="-s -w" -o webhooks - mv webhooks /webhooks +--- +image: {{ $.ImageName }} +from: {{ $.BASE_SCRATCH }} + +import: + - image: {{ $.ImageName }}-golang-artifact + add: /webhooks + to: /webhooks + before: setup +docker: + ENTRYPOINT: ["/webhooks"] diff --git a/templates/csi/controller.yaml b/templates/csi/controller.yaml index 39a0422..6bcae47 100644 --- a/templates/csi/controller.yaml +++ b/templates/csi/controller.yaml @@ -46,6 +46,7 @@ {{- $_ := set $csiControllerConfig "controllerImage" $csiControllerImage }} {{- $_ := set $csiControllerConfig "snapshotterEnabled" true }} {{- $_ := set $csiControllerConfig "resizerEnabled" false }} +{{- $_ := set $csiControllerConfig "provisionerTimeout" "1200s" }} {{- $_ := set $csiControllerConfig "snapshotterTimeout" "1200s" }} {{- $_ := set $csiControllerConfig "extraCreateMetadataEnabled" true }} {{- $_ := set $csiControllerConfig "livenessProbePort" 29652 }}