Merge branch 'feat/volume-cloning' into main

example/snapshot.yaml

@@ -0,0 +1,47 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: claim
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: my-marvelous-storage
+  resources:
+    requests:
+      storage: 5Gi
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod
+spec:
+  containers:
+  - image: alpine
+    command: ["/bin/sh", "-c", "while sleep 1; do echo hello > /vol/test && ls -l /vol && cat /vol/test; done"]
+    name: container
+    volumeMounts:
+    - mountPath: /vol
+      name: volume
+  volumes:
+  - name: volume
+    persistentVolumeClaim:
+      claimName: claim
+---
+apiVersion: snapshot.storage.k8s.io/v1beta1
+kind: VolumeSnapshotClass
+metadata:
+  name: snapshot-class-dothill
+driver: dothill.csi.enix.io
+deletionPolicy: Delete
+parameters:
+  csi.storage.k8s.io/snapshotter-secret-name: dothill-api
+  csi.storage.k8s.io/snapshotter-secret-namespace: dothill-system
+---
+apiVersion: snapshot.storage.k8s.io/v1
+kind: VolumeSnapshot
+metadata:
+  name: test-snapshot
+spec:
+  volumeSnapshotClassName: snapshot-class-dothill
+  source:
+    persistentVolumeClaimName: claim

go.mod

@@ -4,7 +4,8 @@ go 1.16
 
 require (
 	github.com/container-storage-interface/spec v1.3.0
-	github.com/enix/dothill-api-go v1.6.0
+	github.com/enix/dothill-api-go v1.7.0
+	github.com/golang/protobuf v1.4.3
 	github.com/grpc-ecosystem/go-grpc-middleware v1.2.2
 	github.com/kubernetes-csi/csi-lib-iscsi v0.0.0-20200118015005-959f12c91ca8
 	github.com/kubernetes-csi/csi-test v0.0.0-20191016154743-6931aedb3df0

go.sum

@@ -55,10 +55,10 @@ github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5m
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
-github.com/enix/csi-lib-iscsi v0.0.0-dothill-3-0-2-1 h1:W2US8dUbGdatD+Prb4gZTQFGSHTrqhcBzUtJAmVGlnI=
-github.com/enix/csi-lib-iscsi v0.0.0-dothill-3-0-2-1/go.mod h1:c/keGS6bErOzLrFyNgafdDWT6h72v2XQiA/p2R7yghU=
-github.com/enix/dothill-api-go v1.6.0 h1:lMF6SWwTyyFvGMktj698TQ3E1bGOBHkJbKu3t63AStU=
-github.com/enix/dothill-api-go v1.6.0/go.mod h1:OuhSm5SRGxzXFy3kmRAGCCDpbp6q/2JFmTdES2jP5Jw=
+github.com/enix/csi-lib-iscsi v0.0.0-dothill-3-0-3 h1:PFOkMy2rmW0xviq9stiEKZ1XQu+TtSA6d0tEZyLfwTo=
+github.com/enix/csi-lib-iscsi v0.0.0-dothill-3-0-3/go.mod h1:c/keGS6bErOzLrFyNgafdDWT6h72v2XQiA/p2R7yghU=
+github.com/enix/dothill-api-go v1.7.0 h1:jbziGAb4ecXRgds47zcfXqEeNPJJBOgiDPV4FH0FQgA=
+github.com/enix/dothill-api-go v1.7.0/go.mod h1:OuhSm5SRGxzXFy3kmRAGCCDpbp6q/2JFmTdES2jP5Jw=
 github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=

helm/dothill-csi/templates/deployment.yaml

@@ -62,6 +62,16 @@ spec:
           image: {{ .Values.csiResizer.image.repository }}:{{ .Values.csiResizer.image.tag }}
           args:
             - --csi-address=/csi/csi.sock
+{{- include "dothill.extraArgs" .Values.csiResizer | indent 10 }}
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+        - name: csi-snapshotter
+          image: {{ .Values.csiSnapshotter.image.repository }}:{{ .Values.csiSnapshotter.image.tag }}
+          args:
+            - --csi-address=/csi/csi.sock
+{{- include "dothill.extraArgs" .Values.csiSnapshotter | indent 10 }}
           imagePullPolicy: IfNotPresent
           volumeMounts:
             - name: socket-dir

helm/dothill-csi/templates/rbac.yaml

@@ -23,8 +23,6 @@ metadata:
   labels:
 {{ include "dothill.labels" . | indent 4 }}
 rules:
-  # The following rule should be uncommented for plugins that require secrets
-  # for provisioning.
   - apiGroups: [""]
     resources: ["secrets"]
     verbs: ["get", "list"]
@@ -46,9 +44,15 @@ rules:
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshots"]
     verbs: ["get", "list"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses"]
+    verbs: ["get", "list", "watch"]
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshotcontents"]
-    verbs: ["get", "list"]
+    verbs: ["create", "get", "list", "watch", "update", "delete"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents/status"]
+    verbs: ["update"]
   - apiGroups: ["storage.k8s.io"]
     resources: ["csinodes"]
     verbs: ["get", "list", "watch"]
@@ -119,7 +123,6 @@ roleRef:
   name: external-provisioner-cfg-dothill
   apiGroup: rbac.authorization.k8s.io
 
-
 {{ if .Values.pspAdmissionControllerEnabled }}
 ---
 apiVersion: v1

helm/dothill-csi/values.yaml

@@ -42,6 +42,14 @@ csiResizer:
   # -- Extra arguments for csi-resizer controller sidecar
   extraArgs: []
 
+# -- Controller sidecar for snapshots handling
+csiSnapshotter:
+  image:
+    repository: k8s.gcr.io/sig-storage/csi-snapshotter
+    tag: v4.0.0
+  # -- Extra arguments for csi-snapshotter controller sidecar
+  extraArgs: []
+
 # -- Node sidecar for plugin registration
 csiNodeRegistrar:
   image:

pkg/controller/controller.go

@@ -18,6 +18,9 @@ const (
 	hostMapDoesNotExistsErrorCode = -10074
 	unmapFailedErrorCode          = -10509
 	volumeNotFoundErrorCode       = -10075
+	volumeHasSnapshot             = -10183
+	snapshotNotFoundErrorCode     = -10050
+	snapshotAlreadyExists         = -10186
 )
 
 var volumeCapabilities = []*csi.VolumeCapability{
@@ -106,9 +109,9 @@ func (controller *Controller) ControllerGetCapabilities(ctx context.Context, req
 	cl := []csi.ControllerServiceCapability_RPC_Type{
 		csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME,
 		csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME,
-		// csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT,
-		// csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS,
-		// csi.ControllerServiceCapability_RPC_CLONE_VOLUME,
+		csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT,
+		csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS,
+		csi.ControllerServiceCapability_RPC_CLONE_VOLUME,
 		csi.ControllerServiceCapability_RPC_EXPAND_VOLUME,
 	}
 

pkg/controller/provisioner.go

@@ -60,7 +60,21 @@ func (controller *Controller) CreateVolume(ctx context.Context, req *csi.CreateV
 	}
 
 	if !volumeExists {
-		_, _, err = controller.dothillClient.CreateVolume(volumeID, sizeStr, parameters[common.PoolConfigKey])
+		var sourceID string
+
+		if volume := req.VolumeContentSource.GetVolume(); volume != nil {
+			sourceID = volume.VolumeId
+		}
+
+		if snapshot := req.VolumeContentSource.GetSnapshot(); sourceID == "" && snapshot != nil {
+			sourceID = snapshot.SnapshotId
+		}
+
+		if sourceID != "" {
+			_, _, err = controller.dothillClient.CopyVolume(sourceID, volumeID, parameters[common.PoolConfigKey])
+		} else {
+			_, _, err = controller.dothillClient.CreateVolume(volumeID, sizeStr, parameters[common.PoolConfigKey])
+		}
 		if err != nil {
 			return nil, err
 		}
@@ -87,11 +101,15 @@ func (controller *Controller) DeleteVolume(ctx context.Context, req *csi.DeleteV
 	}
 
 	klog.Infof("deleting volume %s", req.GetVolumeId())
-	_, status, err := controller.dothillClient.DeleteVolume(req.GetVolumeId())
+	_, respStatus, err := controller.dothillClient.DeleteVolume(req.GetVolumeId())
 	if err != nil {
-		if status != nil && status.ReturnCode == volumeNotFoundErrorCode {
-			klog.Infof("volume %s does not exist, assuming it has already been deleted", req.GetVolumeId())
-			return &csi.DeleteVolumeResponse{}, nil
+		if respStatus != nil {
+			if respStatus.ReturnCode == volumeNotFoundErrorCode {
+				klog.Infof("volume %s does not exist, assuming it has already been deleted", req.GetVolumeId())
+				return &csi.DeleteVolumeResponse{}, nil
+			} else if respStatus.ReturnCode == volumeHasSnapshot {
+				return nil, status.Error(codes.FailedPrecondition, fmt.Sprintf("volume %s cannot be deleted since it has snapshots", req.GetVolumeId()))
+			}
 		}
 		return nil, err
 	}

pkg/controller/snapshotter.go

@@ -2,27 +2,124 @@ package controller
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"strconv"
+	"strings"
+	"time"
 
 	"github.com/container-storage-interface/spec/lib/go/csi"
+	"github.com/enix/dothill-api-go"
+	"github.com/golang/protobuf/ptypes"
+	"github.com/golang/protobuf/ptypes/timestamp"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
+	"k8s.io/klog"
 )
 
 // CreateSnapshot creates a snapshot of the given volume
 func (controller *Controller) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) {
-	fmt.Println("CreateSnapshot call")
-	return nil, status.Error(codes.Unimplemented, "CreateSnapshot unimplemented yet")
+	name := strings.Replace(req.Name[9:], "-", "", -1)
+
+	_, respStatus, err := controller.dothillClient.CreateSnapshot(req.SourceVolumeId, name)
+	if err != nil && respStatus.ReturnCode != snapshotAlreadyExists {
+		return nil, err
+	}
+
+	response, _, err := controller.dothillClient.ShowSnapshots(name)
+	if err != nil {
+		return nil, err
+	}
+
+	var snapshot *csi.Snapshot
+	for _, object := range response.Objects {
+		if object.Typ != "snapshots" {
+			continue
+		}
+
+		snapshot, err = newSnapshotFromResponse(&object)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if snapshot == nil {
+		return nil, errors.New("snapshot not found")
+	}
+
+	if snapshot.SourceVolumeId != req.SourceVolumeId {
+		return nil, status.Error(codes.AlreadyExists, "cannot validate volume with empty ID")
+	}
+
+	return &csi.CreateSnapshotResponse{Snapshot: snapshot}, nil
 }
 
 // DeleteSnapshot deletes a snapshot of the given volume
 func (controller *Controller) DeleteSnapshot(ctx context.Context, req *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) {
-	fmt.Println("DeleteSnapshot call")
-	return nil, status.Error(codes.Unimplemented, "DeleteSnapshot unimplemented yet")
+	_, status, err := controller.dothillClient.DeleteSnapshot(req.SnapshotId)
+	if err != nil {
+		if status != nil && status.ReturnCode == snapshotNotFoundErrorCode {
+			klog.Infof("snapshot %s does not exist, assuming it has already been deleted", req.SnapshotId)
+			return &csi.DeleteSnapshotResponse{}, nil
+		}
+		return nil, err
+	}
+	return &csi.DeleteSnapshotResponse{}, nil
 }
 
 // ListSnapshots list existing snapshots
 func (controller *Controller) ListSnapshots(ctx context.Context, req *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) {
-	fmt.Println("ListSnapshots call")
-	return nil, status.Error(codes.Unimplemented, "ListSnapshots unimplemented yet")
+	response, _, err := controller.dothillClient.ShowSnapshots()
+	if err != nil {
+		return nil, err
+	}
+
+	snapshots := []*csi.ListSnapshotsResponse_Entry{}
+	for _, object := range response.Objects {
+		if object.Typ != "snapshots" {
+			continue
+		}
+
+		snapshot, err := newSnapshotFromResponse(&object)
+		if err != nil {
+			return nil, err
+		}
+
+		snapshots = append(snapshots, &csi.ListSnapshotsResponse_Entry{
+			Snapshot: snapshot,
+		})
+	}
+
+	return &csi.ListSnapshotsResponse{
+		Entries: snapshots,
+	}, nil
+}
+
+func newSnapshotFromResponse(object *dothill.Object) (*csi.Snapshot, error) {
+	properties, err := object.GetProperties("total-size-numeric", "name", "master-volume-name", "creation-date-time-numeric")
+	if err != nil {
+		return nil, fmt.Errorf("could not read snapshot %v", err)
+	}
+
+	sizeBytes, err := strconv.ParseInt(properties[0].Data, 10, 64)
+	snapshotId := properties[1].Data
+	sourceVolumeId := properties[2].Data
+	creationTime, err := creationTimeFromString(properties[3].Data)
+
+	return &csi.Snapshot{
+		SizeBytes:      sizeBytes,
+		SnapshotId:     snapshotId,
+		SourceVolumeId: sourceVolumeId,
+		CreationTime:   creationTime,
+		ReadyToUse:     true,
+	}, nil
+}
+
+func creationTimeFromString(creationTime string) (*timestamp.Timestamp, error) {
+	creationTimestamp, err := strconv.ParseInt(creationTime, 10, 64)
+	if err != nil {
+		return nil, err
+	}
+
+	return ptypes.TimestampProto(time.Unix(creationTimestamp, 0))
 }