Skip to content

Commit e4c5d45

Browse files
authored
Merge pull request kubernetes#68761 from fanzhangio/vsphere-volume
Feature: Implements Raw Block Volume Support for in-tree vSphere plugin
2 parents 224448b + dcb3a5f commit e4c5d45

File tree

4 files changed

+326
-3
lines changed

4 files changed

+326
-3
lines changed

pkg/volume/vsphere_volume/BUILD

+6
Original file line numberDiff line numberDiff line change
@@ -11,21 +11,25 @@ go_library(
1111
srcs = [
1212
"attacher.go",
1313
"vsphere_volume.go",
14+
"vsphere_volume_block.go",
1415
"vsphere_volume_util.go",
1516
],
1617
importpath = "k8s.io/kubernetes/pkg/volume/vsphere_volume",
1718
deps = [
1819
"//pkg/cloudprovider/providers/vsphere:go_default_library",
1920
"//pkg/cloudprovider/providers/vsphere/vclib:go_default_library",
21+
"//pkg/features:go_default_library",
2022
"//pkg/util/keymutex:go_default_library",
2123
"//pkg/util/mount:go_default_library",
2224
"//pkg/util/strings:go_default_library",
2325
"//pkg/volume:go_default_library",
2426
"//pkg/volume/util:go_default_library",
27+
"//pkg/volume/util/volumepathhandler:go_default_library",
2528
"//staging/src/k8s.io/api/core/v1:go_default_library",
2629
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
2730
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
2831
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
32+
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
2933
"//staging/src/k8s.io/cloud-provider:go_default_library",
3034
"//vendor/github.com/golang/glog:go_default_library",
3135
],
@@ -35,6 +39,7 @@ go_test(
3539
name = "go_default_test",
3640
srcs = [
3741
"attacher_test.go",
42+
"vsphere_volume_block_test.go",
3843
"vsphere_volume_test.go",
3944
],
4045
embed = [":go_default_library"],
@@ -46,6 +51,7 @@ go_test(
4651
"//pkg/volume:go_default_library",
4752
"//pkg/volume/testing:go_default_library",
4853
"//staging/src/k8s.io/api/core/v1:go_default_library",
54+
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
4955
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
5056
"//staging/src/k8s.io/client-go/util/testing:go_default_library",
5157
"//staging/src/k8s.io/cloud-provider:go_default_library",

pkg/volume/vsphere_volume/vsphere_volume.go

+12-3
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ import (
2727
"k8s.io/apimachinery/pkg/api/resource"
2828
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2929
"k8s.io/apimachinery/pkg/types"
30+
utilfeature "k8s.io/apiserver/pkg/util/feature"
31+
"k8s.io/kubernetes/pkg/features"
3032
"k8s.io/kubernetes/pkg/util/mount"
3133
utilstrings "k8s.io/kubernetes/pkg/util/strings"
3234
"k8s.io/kubernetes/pkg/volume"
@@ -356,9 +358,6 @@ func (v *vsphereVolumeProvisioner) Provision(selectedNode *v1.Node, allowedTopol
356358
if !util.AccessModesContainedInAll(v.plugin.GetAccessModes(), v.options.PVC.Spec.AccessModes) {
357359
return nil, fmt.Errorf("invalid AccessModes %v: only AccessModes %v are supported", v.options.PVC.Spec.AccessModes, v.plugin.GetAccessModes())
358360
}
359-
if util.CheckPersistentVolumeClaimModeBlock(v.options.PVC) {
360-
return nil, fmt.Errorf("%s does not support block volume provisioning", v.plugin.GetPluginName())
361-
}
362361

363362
volSpec, err := v.manager.CreateVolume(v)
364363
if err != nil {
@@ -369,6 +368,15 @@ func (v *vsphereVolumeProvisioner) Provision(selectedNode *v1.Node, allowedTopol
369368
volSpec.Fstype = "ext4"
370369
}
371370

371+
var volumeMode *v1.PersistentVolumeMode
372+
if utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) {
373+
volumeMode = v.options.PVC.Spec.VolumeMode
374+
if volumeMode != nil && *volumeMode == v1.PersistentVolumeBlock {
375+
glog.V(5).Infof("vSphere block volume should not have any FSType")
376+
volSpec.Fstype = ""
377+
}
378+
}
379+
372380
pv := &v1.PersistentVolume{
373381
ObjectMeta: metav1.ObjectMeta{
374382
Name: v.options.PVName,
@@ -383,6 +391,7 @@ func (v *vsphereVolumeProvisioner) Provision(selectedNode *v1.Node, allowedTopol
383391
Capacity: v1.ResourceList{
384392
v1.ResourceName(v1.ResourceStorage): resource.MustParse(fmt.Sprintf("%dKi", volSpec.Size)),
385393
},
394+
VolumeMode: volumeMode,
386395
PersistentVolumeSource: v1.PersistentVolumeSource{
387396
VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{
388397
VolumePath: volSpec.Path,
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
/*
2+
Copyright 2018 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package vsphere_volume
18+
19+
import (
20+
"fmt"
21+
"path"
22+
"path/filepath"
23+
"strings"
24+
25+
"github.com/golang/glog"
26+
"k8s.io/api/core/v1"
27+
"k8s.io/apimachinery/pkg/types"
28+
"k8s.io/kubernetes/pkg/util/mount"
29+
kstrings "k8s.io/kubernetes/pkg/util/strings"
30+
"k8s.io/kubernetes/pkg/volume"
31+
"k8s.io/kubernetes/pkg/volume/util"
32+
"k8s.io/kubernetes/pkg/volume/util/volumepathhandler"
33+
)
34+
35+
var _ volume.BlockVolumePlugin = &vsphereVolumePlugin{}
36+
37+
func (plugin *vsphereVolumePlugin) ConstructBlockVolumeSpec(podUID types.UID, volumeName, mapPath string) (*volume.Spec, error) {
38+
39+
pluginDir := plugin.host.GetPluginDir(plugin.GetPluginName())
40+
blkUtil := volumepathhandler.NewBlockVolumePathHandler()
41+
globalMapPathUUID, err := blkUtil.FindGlobalMapPathUUIDFromPod(pluginDir, mapPath, podUID)
42+
if err != nil {
43+
glog.Errorf("Failed to find GlobalMapPathUUID from Pod: %s with error: %+v", podUID, err)
44+
return nil, err
45+
}
46+
glog.V(5).Infof("globalMapPathUUID: %v", globalMapPathUUID)
47+
globalMapPath := filepath.Dir(globalMapPathUUID)
48+
if len(globalMapPath) <= 1 {
49+
return nil, fmt.Errorf("failed to get volume plugin information from globalMapPathUUID: %v", globalMapPathUUID)
50+
}
51+
return getVolumeSpecFromGlobalMapPath(globalMapPath)
52+
}
53+
54+
func getVolumeSpecFromGlobalMapPath(globalMapPath string) (*volume.Spec, error) {
55+
// Construct volume spec from globalMapPath
56+
// globalMapPath example:
57+
// plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumeID}
58+
// plugins/kubernetes.io/vsphere-volume/volumeDevices/[datastore1]\\040volumes/myDisk
59+
volPath := filepath.Base(globalMapPath)
60+
volPath = strings.Replace(volPath, "\\040", "", -1)
61+
if len(volPath) <= 1 {
62+
return nil, fmt.Errorf("failed to get volume path from global path=%s", globalMapPath)
63+
}
64+
block := v1.PersistentVolumeBlock
65+
vsphereVolume := &v1.PersistentVolume{
66+
Spec: v1.PersistentVolumeSpec{
67+
PersistentVolumeSource: v1.PersistentVolumeSource{
68+
VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{
69+
VolumePath: volPath,
70+
},
71+
},
72+
VolumeMode: &block,
73+
},
74+
}
75+
return volume.NewSpecFromPersistentVolume(vsphereVolume, true), nil
76+
}
77+
78+
func (plugin *vsphereVolumePlugin) NewBlockVolumeMapper(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.BlockVolumeMapper, error) {
79+
// If this called via GenerateUnmapDeviceFunc(), pod is nil.
80+
// Pass empty string as dummy uid since uid isn't used in the case.
81+
var uid types.UID
82+
if pod != nil {
83+
uid = pod.UID
84+
}
85+
return plugin.newBlockVolumeMapperInternal(spec, uid, &VsphereDiskUtil{}, plugin.host.GetMounter(plugin.GetPluginName()))
86+
}
87+
88+
func (plugin *vsphereVolumePlugin) newBlockVolumeMapperInternal(spec *volume.Spec, podUID types.UID, manager vdManager, mounter mount.Interface) (volume.BlockVolumeMapper, error) {
89+
volumeSource, _, err := getVolumeSource(spec)
90+
if err != nil {
91+
glog.Errorf("Failed to get Volume source from volume Spec: %+v with error: %+v", *spec, err)
92+
return nil, err
93+
}
94+
volPath := volumeSource.VolumePath
95+
return &vsphereBlockVolumeMapper{
96+
vsphereVolume: &vsphereVolume{
97+
volName: spec.Name(),
98+
podUID: podUID,
99+
volPath: volPath,
100+
manager: manager,
101+
mounter: mounter,
102+
plugin: plugin,
103+
MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, spec.Name(), plugin.host)),
104+
},
105+
}, nil
106+
107+
}
108+
109+
func (plugin *vsphereVolumePlugin) NewBlockVolumeUnmapper(volName string, podUID types.UID) (volume.BlockVolumeUnmapper, error) {
110+
return plugin.newUnmapperInternal(volName, podUID, &VsphereDiskUtil{})
111+
}
112+
113+
func (plugin *vsphereVolumePlugin) newUnmapperInternal(volName string, podUID types.UID, manager vdManager) (volume.BlockVolumeUnmapper, error) {
114+
return &vsphereBlockVolumeUnmapper{
115+
vsphereVolume: &vsphereVolume{
116+
volName: volName,
117+
podUID: podUID,
118+
volPath: volName,
119+
manager: manager,
120+
plugin: plugin,
121+
},
122+
}, nil
123+
}
124+
125+
var _ volume.BlockVolumeMapper = &vsphereBlockVolumeMapper{}
126+
127+
type vsphereBlockVolumeMapper struct {
128+
*vsphereVolume
129+
}
130+
131+
func (v vsphereBlockVolumeMapper) SetUpDevice() (string, error) {
132+
return "", nil
133+
}
134+
135+
func (v vsphereBlockVolumeMapper) MapDevice(devicePath, globalMapPath, volumeMapPath, volumeMapName string, podUID types.UID) error {
136+
return util.MapBlockVolume(devicePath, globalMapPath, volumeMapPath, volumeMapName, podUID)
137+
}
138+
139+
var _ volume.BlockVolumeUnmapper = &vsphereBlockVolumeUnmapper{}
140+
141+
type vsphereBlockVolumeUnmapper struct {
142+
*vsphereVolume
143+
}
144+
145+
func (v *vsphereBlockVolumeUnmapper) TearDownDevice(mapPath, devicePath string) error {
146+
return nil
147+
}
148+
149+
// GetGlobalMapPath returns global map path and error
150+
// path: plugins/kubernetes.io/{PluginName}/volumeDevices/volumePath
151+
func (v *vsphereVolume) GetGlobalMapPath(spec *volume.Spec) (string, error) {
152+
volumeSource, _, err := getVolumeSource(spec)
153+
if err != nil {
154+
return "", err
155+
}
156+
return path.Join(v.plugin.host.GetVolumeDevicePluginDir(vsphereVolumePluginName), string(volumeSource.VolumePath)), nil
157+
}
158+
159+
func (v *vsphereVolume) GetPodDeviceMapPath() (string, string) {
160+
return v.plugin.host.GetPodVolumeDeviceDir(v.podUID, kstrings.EscapeQualifiedNameForDisk(vsphereVolumePluginName)), v.volName
161+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
/*
2+
Copyright 2018 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package vsphere_volume
18+
19+
import (
20+
"os"
21+
"path"
22+
"testing"
23+
24+
"k8s.io/api/core/v1"
25+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
26+
"k8s.io/apimachinery/pkg/types"
27+
utiltesting "k8s.io/client-go/util/testing"
28+
"k8s.io/kubernetes/pkg/volume"
29+
volumetest "k8s.io/kubernetes/pkg/volume/testing"
30+
)
31+
32+
var (
33+
testVolumePath = "volPath1"
34+
testGlobalPath = "plugins/kubernetes.io/vsphere-volume/volumeDevices/volPath1"
35+
testPodPath = "pods/poduid/volumeDevices/kubernetes.io~vsphere-volume"
36+
)
37+
38+
func TestGetVolumeSpecFromGlobalMapPath(t *testing.T) {
39+
// make our test path for fake GlobalMapPath
40+
// /tmp symbolized our pluginDir
41+
// /tmp/testGlobalPathXXXXX/plugins/kubernetes.io/vsphere-volume/volumeDevices/
42+
tmpVDir, err := utiltesting.MkTmpdir("vsphereBlockVolume")
43+
if err != nil {
44+
t.Fatalf("cant' make a temp dir: %s", err)
45+
}
46+
// deferred clean up
47+
defer os.RemoveAll(tmpVDir)
48+
49+
expectedGlobalPath := path.Join(tmpVDir, testGlobalPath)
50+
51+
// Bad Path
52+
badspec, err := getVolumeSpecFromGlobalMapPath("")
53+
if badspec != nil || err == nil {
54+
t.Errorf("Expected not to get spec from GlobalMapPath but did")
55+
}
56+
57+
// Good Path
58+
spec, err := getVolumeSpecFromGlobalMapPath(expectedGlobalPath)
59+
if spec == nil || err != nil {
60+
t.Fatalf("Failed to get spec from GlobalMapPath: %s", err)
61+
}
62+
if spec.PersistentVolume.Spec.VsphereVolume.VolumePath != testVolumePath {
63+
t.Fatalf("Invalid volumePath from GlobalMapPath spec: %s", spec.PersistentVolume.Spec.VsphereVolume.VolumePath)
64+
}
65+
block := v1.PersistentVolumeBlock
66+
specMode := spec.PersistentVolume.Spec.VolumeMode
67+
if &specMode == nil {
68+
t.Errorf("Invalid volumeMode from GlobalMapPath spec: %v expected: %v", &specMode, block)
69+
}
70+
if *specMode != block {
71+
t.Errorf("Invalid volumeMode from GlobalMapPath spec: %v expected: %v", *specMode, block)
72+
}
73+
}
74+
75+
func TestGetPodAndPluginMapPaths(t *testing.T) {
76+
tmpVDir, err := utiltesting.MkTmpdir("vsphereBlockVolume")
77+
if err != nil {
78+
t.Fatalf("cant' make a temp dir: %s", err)
79+
}
80+
// deferred clean up
81+
defer os.RemoveAll(tmpVDir)
82+
83+
expectedGlobalPath := path.Join(tmpVDir, testGlobalPath)
84+
expectedPodPath := path.Join(tmpVDir, testPodPath)
85+
86+
spec := getTestVolume(true) // block volume
87+
pluginMgr := volume.VolumePluginMgr{}
88+
pluginMgr.InitPlugins(ProbeVolumePlugins(), nil, volumetest.NewFakeVolumeHost(tmpVDir, nil, nil))
89+
plugin, err := pluginMgr.FindMapperPluginByName(vsphereVolumePluginName)
90+
if err != nil {
91+
os.RemoveAll(tmpVDir)
92+
t.Fatalf("Can't find the plugin by name: %q", vsphereVolumePluginName)
93+
}
94+
if plugin.GetPluginName() != vsphereVolumePluginName {
95+
t.Fatalf("Wrong name: %s", plugin.GetPluginName())
96+
}
97+
pod := &v1.Pod{
98+
ObjectMeta: metav1.ObjectMeta{
99+
UID: types.UID("poduid"),
100+
},
101+
}
102+
mapper, err := plugin.NewBlockVolumeMapper(spec, pod, volume.VolumeOptions{})
103+
if err != nil {
104+
t.Fatalf("Failed to make a new Mounter: %v", err)
105+
}
106+
if mapper == nil {
107+
t.Fatalf("Got a nil Mounter")
108+
}
109+
110+
// GetGlobalMapPath
111+
globalMapPath, err := mapper.GetGlobalMapPath(spec)
112+
if err != nil || len(globalMapPath) == 0 {
113+
t.Fatalf("Invalid GlobalMapPath from spec: %s", spec.PersistentVolume.Spec.VsphereVolume.VolumePath)
114+
}
115+
if globalMapPath != expectedGlobalPath {
116+
t.Errorf("Failed to get GlobalMapPath: %s %s", globalMapPath, expectedGlobalPath)
117+
}
118+
119+
// GetPodDeviceMapPath
120+
devicePath, volumeName := mapper.GetPodDeviceMapPath()
121+
if devicePath != expectedPodPath {
122+
t.Errorf("Got unexpected pod path: %s, expected %s", devicePath, expectedPodPath)
123+
}
124+
if volumeName != testVolumePath {
125+
t.Errorf("Got unexpected volNamne: %s, expected %s", volumeName, testVolumePath)
126+
}
127+
}
128+
129+
func getTestVolume(isBlock bool) *volume.Spec {
130+
pv := &v1.PersistentVolume{
131+
ObjectMeta: metav1.ObjectMeta{
132+
Name: testVolumePath,
133+
},
134+
Spec: v1.PersistentVolumeSpec{
135+
PersistentVolumeSource: v1.PersistentVolumeSource{
136+
VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{
137+
VolumePath: testVolumePath,
138+
},
139+
},
140+
},
141+
}
142+
if isBlock {
143+
blockMode := v1.PersistentVolumeBlock
144+
pv.Spec.VolumeMode = &blockMode
145+
}
146+
return volume.NewSpecFromPersistentVolume(pv, true)
147+
}

0 commit comments

Comments
 (0)