Skip to content

Commit f60f7c3

Browse files
YiteGuYiteGu
committed
rbd: support QoS based on capacity for rbd volume
1. QoS provides settings for rbd volume read/write iops and read/write bandwidth. 2. All QoS parameters are placed in the SC, send QoS parameters from SC to Cephcsi through PVC create request. 3. We need provide QoS parameters in the SC as below: - BaseReadIOPS - BaseWriteIOPS - BaseReadBytesPerSecond - BaseWriteBytesPerSecond - ReadIopsPerGB - WriteIopsPerGB - ReadBpsPerGB - WriteBpsPerGB - BaseVolSizeBytes There are 4 base qos parameters among them, when users apply for a volume capacity equal to or less than BaseVolSizebytes, use base qos limit. For the portion of capacity exceeding BaseVolSizebytes, QoS will be increased in steps set per GB. If the step size parameter per GB is not provided, only base QoS limit will be used and not associated with capacity size. 4. If PVC has resize request, adjust the QoS limit according to the QoS parameters after resizing. Signed-off-by: Yite Gu <[email protected]>
1 parent 3c63fea commit f60f7c3

File tree

2 files changed

+275
-0
lines changed

2 files changed

+275
-0
lines changed

‎internal/rbd/controllerserver.go

+87
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,22 @@ func (cs *ControllerServer) parseVolCreateRequest(
231231
return nil, status.Error(codes.InvalidArgument, err.Error())
232232
}
233233

234+
// Get QosParameters from SC if qos configuration existing in SC
235+
baseQosVolSize := ""
236+
params := req.GetParameters()
237+
if v, ok := params[baseVolSizeBytes]; ok && v != "" {
238+
baseQosVolSize = v
239+
}
240+
rbdQosParameters := parseQosParams(params)
241+
for _, param := range rbdQosParameters {
242+
if param.provide {
243+
err = calcQosBasedOnCapacity(ctx, rbdVol, *param, baseQosVolSize)
244+
if err != nil {
245+
return nil, status.Error(codes.InvalidArgument, err.Error())
246+
}
247+
}
248+
}
249+
234250
err = rbdVol.Connect(cr)
235251
if err != nil {
236252
log.ErrorLog(ctx, "failed to connect to volume %v: %v", rbdVol.RbdImageName, err)
@@ -243,6 +259,32 @@ func (cs *ControllerServer) parseVolCreateRequest(
243259
return rbdVol, nil
244260
}
245261

262+
func parseQosParams(
263+
params map[string]string,
264+
) map[string]*rbdQos {
265+
rbdQosParameters := map[string]*rbdQos{
266+
baseReadIOPS: {rbdQosReadIopsLimit, "", readIopsPerGB, "", false},
267+
baseWriteIOPS: {rbdQosWriteIopsLimit, "", writeIopsPerGB, "", false},
268+
baseReadBytesPerSecond: {rbdQosReadBpsLimit, "", readBpsPerGB, "", false},
269+
baseWriteBytesPerSecond: {rbdQosWriteBpsLimit, "", writeBpsPerGB, "", false},
270+
}
271+
for k, v := range params {
272+
if param, ok := rbdQosParameters[k]; ok && v != "" {
273+
param.rbdBaseQosLimit = v
274+
param.provide = true
275+
for p, q := range params {
276+
if p == param.rbdQosPerGBType {
277+
if q != "" {
278+
param.rbdQosPerGB = q
279+
}
280+
}
281+
}
282+
}
283+
}
284+
285+
return rbdQosParameters
286+
}
287+
246288
func (rbdVol *rbdVolume) ToCSI(ctx context.Context) (*csi.Volume, error) {
247289
vol := &csi.Volume{
248290
VolumeId: rbdVol.VolID,
@@ -424,6 +466,19 @@ func (cs *ControllerServer) CreateVolume(
424466
return nil, err
425467
}
426468

469+
// Set rbd qos for image
470+
if len(rbdVol.QosParameters) != 0 && rbdVol.Mounter == rbdNbdMounter {
471+
err = setRbdImageQos(ctx, rbdVol)
472+
if err != nil {
473+
return nil, status.Error(codes.Internal, err.Error())
474+
}
475+
}
476+
// Save qos parameters from SC in image medatate, we will use it while resize volume
477+
err = saveQosToRbdImage(ctx, rbdVol, req.GetParameters())
478+
if err != nil {
479+
return nil, err
480+
}
481+
427482
// Set Metadata on PV Create
428483
metadata := k8s.GetVolumeMetadata(req.GetParameters())
429484
err = rbdVol.setAllMetadata(metadata)
@@ -1607,6 +1662,11 @@ func (cs *ControllerServer) ControllerExpandVolume(
16071662

16081663
return nil, status.Error(codes.Internal, err.Error())
16091664
}
1665+
// we need adjust rbd qos after resize volume
1666+
err = cs.adjustRbdImageQos(ctx, rbdVol)
1667+
if err != nil {
1668+
return nil, status.Error(codes.Internal, err.Error())
1669+
}
16101670
}
16111671

16121672
return &csi.ControllerExpandVolumeResponse{
@@ -1615,6 +1675,33 @@ func (cs *ControllerServer) ControllerExpandVolume(
16151675
}, nil
16161676
}
16171677

1678+
func (cs *ControllerServer) adjustRbdImageQos(
1679+
ctx context.Context,
1680+
rbdVol *rbdVolume,
1681+
) error {
1682+
rbdQosParameters, baseQosVolSize, err := getRbdImageQos(ctx, rbdVol)
1683+
if err != nil {
1684+
log.ErrorLog(ctx, "get image metadata failed: %v", err)
1685+
1686+
return err
1687+
}
1688+
for _, param := range rbdQosParameters {
1689+
err = calcQosBasedOnCapacity(ctx, rbdVol, param, baseQosVolSize)
1690+
if err != nil {
1691+
return err
1692+
}
1693+
}
1694+
err = setRbdImageQos(ctx, rbdVol)
1695+
if err != nil {
1696+
log.ErrorLog(ctx, "set image metadata failed: %v", err)
1697+
1698+
return err
1699+
}
1700+
log.DebugLog(ctx, "adjust rbd image qos successfully")
1701+
1702+
return nil
1703+
}
1704+
16181705
// ControllerPublishVolume is a dummy publish implementation to mimic a successful attach operation being a NOOP.
16191706
func (cs *ControllerServer) ControllerPublishVolume(
16201707
ctx context.Context,

‎internal/rbd/rbd_util.go

+188
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,35 @@ const (
8181

8282
// clusterNameKey cluster Key, set on RBD image.
8383
clusterNameKey = "csi.ceph.com/cluster/name"
84+
85+
// Qos parameters name of StorageClass.
86+
baseReadIOPS = "BaseReadIOPS"
87+
baseWriteIOPS = "BaseWriteIOPS"
88+
baseReadBytesPerSecond = "BaseReadBytesPerSecond"
89+
baseWriteBytesPerSecond = "BaseWriteBytesPerSecond"
90+
readIopsPerGB = "ReadIopsPerGB"
91+
writeIopsPerGB = "WriteIopsPerGB"
92+
readBpsPerGB = "ReadBpsPerGB"
93+
writeBpsPerGB = "WriteBpsPerGB"
94+
baseVolSizeBytes = "BaseVolSizeBytes"
95+
96+
// Qos type name of rbd image.
97+
rbdQosReadIopsLimit = "rbd_qos_read_iops_limit"
98+
rbdQosWriteIopsLimit = "rbd_qos_write_iops_limit"
99+
rbdQosReadBpsLimit = "rbd_qos_read_bps_limit"
100+
rbdQosWriteBpsLimit = "rbd_qos_write_bps_limit"
101+
metadataConfPrefix = "conf_"
102+
103+
// The params use to calc qos based on capacity.
104+
rbdBaseQosReadIopsLimit = "rbd_base_qos_read_iops_limit"
105+
rbdBaseQosWriteIopsLimit = "rbd_base_qos_write_iops_limit"
106+
rbdBaseQosReadBpsLimit = "rbd_base_qos_read_bps_limit"
107+
rbdBaseQosWriteBpsLimit = "rbd_base_qos_write_bps_limit"
108+
rbdReadIopsPerGB = "rbd_read_iops_per_gb"
109+
rbdWriteIopsPerGB = "rbd_write_iops_per_gb"
110+
rbdReadBpsPerGB = "rbd_read_bps_per_gb"
111+
rbdWriteBpsPerGB = "rbd_write_bps_per_gb"
112+
rbdBaseQosVolSize = "rbd_base_qos_vol_size"
84113
)
85114

86115
// rbdImage contains common attributes and methods for the rbdVolume and
@@ -148,6 +177,9 @@ type rbdImage struct {
148177
EnableMetadata bool
149178
// ParentInTrash indicates the parent image is in trash.
150179
ParentInTrash bool
180+
181+
// RBD QoS configuration
182+
QosParameters map[string]string
151183
}
152184

153185
// check that rbdVolume implements the types.Volume interface.
@@ -212,6 +244,14 @@ type migrationVolID struct {
212244
clusterID string
213245
}
214246

247+
type rbdQos struct {
248+
rbdQosType string
249+
rbdBaseQosLimit string
250+
rbdQosPerGBType string
251+
rbdQosPerGB string
252+
provide bool
253+
}
254+
215255
var (
216256
supportedFeatures = map[string]imageFeature{
217257
librbd.FeatureNameLayering: {
@@ -2273,3 +2313,151 @@ func (ri *rbdImage) GetClusterID(ctx context.Context) (string, error) {
22732313

22742314
return ri.ClusterID, nil
22752315
}
2316+
2317+
func calcQosBasedOnCapacity(
2318+
ctx context.Context,
2319+
rbdVol *rbdVolume,
2320+
qosParam rbdQos,
2321+
baseQosVolSize string,
2322+
) error {
2323+
if rbdVol.QosParameters == nil {
2324+
rbdVol.QosParameters = make(map[string]string)
2325+
}
2326+
2327+
// Don't set qos if base limit empty
2328+
if qosParam.rbdBaseQosLimit == "" {
2329+
return nil
2330+
}
2331+
baseQosLimit, err := strconv.ParseInt(qosParam.rbdBaseQosLimit, 10, 64)
2332+
if err != nil {
2333+
log.ErrorLog(ctx, "%v", err)
2334+
2335+
return err
2336+
}
2337+
2338+
// if provide qosPerGB and baseQosVolSize, we will set qos based on capacity,
2339+
// otherwise, we only set base qos limit.
2340+
if qosParam.rbdQosPerGB != "" && baseQosVolSize != "" {
2341+
qosPerGB, err := strconv.ParseInt(qosParam.rbdQosPerGB, 10, 64)
2342+
if err != nil {
2343+
log.ErrorLog(ctx, "%v", err)
2344+
2345+
return err
2346+
}
2347+
2348+
baseQosVolSizeInt, err := strconv.ParseInt(baseQosVolSize, 10, 64)
2349+
if err != nil {
2350+
log.ErrorLog(ctx, "%v", err)
2351+
2352+
return err
2353+
}
2354+
2355+
if rbdVol.VolSize <= baseQosVolSizeInt {
2356+
rbdVol.QosParameters[qosParam.rbdQosType] = qosParam.rbdBaseQosLimit
2357+
} else {
2358+
capacityQos := (rbdVol.VolSize - baseQosVolSizeInt) / int64(oneGB) * qosPerGB
2359+
finalQosLimit := baseQosLimit + capacityQos
2360+
rbdVol.QosParameters[qosParam.rbdQosType] = strconv.FormatInt(finalQosLimit, 10)
2361+
}
2362+
} else {
2363+
rbdVol.QosParameters[qosParam.rbdQosType] = qosParam.rbdBaseQosLimit
2364+
}
2365+
2366+
return nil
2367+
}
2368+
2369+
func setRbdImageQos(
2370+
ctx context.Context,
2371+
rbdVol *rbdVolume,
2372+
) error {
2373+
for k, v := range rbdVol.QosParameters {
2374+
err := rbdVol.SetMetadata(metadataConfPrefix+k, v)
2375+
if err != nil {
2376+
log.ErrorLog(ctx, "failed to set rbd qos, %s: %s, %v", k, v, err)
2377+
2378+
return err
2379+
}
2380+
}
2381+
2382+
return nil
2383+
}
2384+
2385+
func getRbdImageQos(
2386+
ctx context.Context,
2387+
rbdVol *rbdVolume,
2388+
) (map[string]rbdQos, string, error) {
2389+
QosParams := map[string]struct {
2390+
rbdQosType string
2391+
rbdQosPerGBType string
2392+
}{
2393+
rbdBaseQosReadIopsLimit: {rbdQosReadIopsLimit, rbdReadIopsPerGB},
2394+
rbdBaseQosWriteIopsLimit: {rbdQosWriteIopsLimit, rbdWriteIopsPerGB},
2395+
rbdBaseQosReadBpsLimit: {rbdQosReadBpsLimit, rbdReadBpsPerGB},
2396+
rbdBaseQosWriteBpsLimit: {rbdQosWriteBpsLimit, rbdWriteBpsPerGB},
2397+
}
2398+
rbdQosParameters := make(map[string]rbdQos)
2399+
for k, param := range QosParams {
2400+
baseLimit, err := rbdVol.GetMetadata(k)
2401+
if errors.Is(err, librbd.ErrNotFound) {
2402+
// if base qos dose not exist, skipping.
2403+
continue
2404+
} else if err != nil {
2405+
log.ErrorLog(ctx, "failed to get metadata: %v", err)
2406+
2407+
return nil, "", err
2408+
}
2409+
perGBLimit, err := rbdVol.GetMetadata(param.rbdQosPerGBType)
2410+
if errors.Is(err, librbd.ErrNotFound) {
2411+
// rbdQosPerGBType does not exist, set it empty.
2412+
perGBLimit = ""
2413+
} else if err != nil {
2414+
log.ErrorLog(ctx, "failed to get metadata: %v", err)
2415+
2416+
return nil, "", err
2417+
}
2418+
rbdQosParameters[k] = rbdQos{param.rbdQosType, baseLimit, param.rbdQosPerGBType, perGBLimit, true}
2419+
}
2420+
baseQosVolSize, err := rbdVol.GetMetadata(rbdBaseQosVolSize)
2421+
if errors.Is(err, librbd.ErrNotFound) {
2422+
// rbdBaseQosVolSize does not exist, set it empty.
2423+
baseQosVolSize = ""
2424+
} else if err != nil {
2425+
log.ErrorLog(ctx, "failed to get metadata: %v", err)
2426+
2427+
return nil, "", err
2428+
}
2429+
2430+
return rbdQosParameters, baseQosVolSize, nil
2431+
}
2432+
2433+
func saveQosToRbdImage(
2434+
ctx context.Context,
2435+
rbdVol *rbdVolume,
2436+
params map[string]string,
2437+
) error {
2438+
needSaveQosParameters := map[string]string{
2439+
baseReadIOPS: rbdBaseQosReadIopsLimit,
2440+
baseWriteIOPS: rbdBaseQosWriteIopsLimit,
2441+
baseReadBytesPerSecond: rbdBaseQosReadBpsLimit,
2442+
baseWriteBytesPerSecond: rbdBaseQosWriteBpsLimit,
2443+
readIopsPerGB: rbdReadIopsPerGB,
2444+
writeIopsPerGB: rbdWriteIopsPerGB,
2445+
readBpsPerGB: rbdReadBpsPerGB,
2446+
writeBpsPerGB: rbdWriteBpsPerGB,
2447+
baseVolSizeBytes: rbdBaseQosVolSize,
2448+
}
2449+
for k, v := range params {
2450+
if param, ok := needSaveQosParameters[k]; ok {
2451+
if v != "" {
2452+
err := rbdVol.SetMetadata(param, v)
2453+
if err != nil {
2454+
log.ErrorLog(ctx, "failed to save metadata, %s: %s, %v", k, v, err)
2455+
2456+
return err
2457+
}
2458+
}
2459+
}
2460+
}
2461+
2462+
return nil
2463+
}

0 commit comments

Comments
 (0)