Skip to content

Commit 0490294

Browse files
intelligentfu8intelligentfu8
committed
add be antiaffinity with fe
add defaultInitImage for initinal container ddd initialDelay
1 parent 490a4cc commit 0490294

File tree

7 files changed

+154
-43
lines changed

7 files changed

+154
-43
lines changed

‎api/doris/v1/types.go

+3
Original file line numberDiff line numberDiff line change
@@ -240,6 +240,9 @@ type ExportService struct {
240240
//ServicePort config service for NodePort access mode.
241241
ServicePorts []DorisServicePort `json:"servicePorts,omitempty"`
242242

243+
//Annotations for using function on different cloud platform.
244+
Annotations map[string]string `json:"annotations,omitempty"`
245+
243246
// Only applies to Service Type: LoadBalancer.
244247
// This feature depends on whether the underlying cloud-provider supports specifying
245248
// the loadBalancerIP when a load balancer is created.

‎api/doris/v1/zz_generated.deepcopy.go

+7
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎config/crd/bases/doris.selectdb.com_dorisclusters.yaml

+24
Original file line numberDiff line numberDiff line change
@@ -1426,6 +1426,12 @@ spec:
14261426
service:
14271427
description: expose the be listen ports
14281428
properties:
1429+
annotations:
1430+
additionalProperties:
1431+
type: string
1432+
description: Annotations for using function on different cloud
1433+
platform.
1434+
type: object
14291435
loadBalancerIP:
14301436
description: 'Only applies to Service Type: LoadBalancer.
14311437
This feature depends on whether the underlying cloud-provider
@@ -2912,6 +2918,12 @@ spec:
29122918
service:
29132919
description: expose the be listen ports
29142920
properties:
2921+
annotations:
2922+
additionalProperties:
2923+
type: string
2924+
description: Annotations for using function on different cloud
2925+
platform.
2926+
type: object
29152927
loadBalancerIP:
29162928
description: 'Only applies to Service Type: LoadBalancer.
29172929
This feature depends on whether the underlying cloud-provider
@@ -5071,6 +5083,12 @@ spec:
50715083
service:
50725084
description: expose the be listen ports
50735085
properties:
5086+
annotations:
5087+
additionalProperties:
5088+
type: string
5089+
description: Annotations for using function on different cloud
5090+
platform.
5091+
type: object
50745092
loadBalancerIP:
50755093
description: 'Only applies to Service Type: LoadBalancer.
50765094
This feature depends on whether the underlying cloud-provider
@@ -6557,6 +6575,12 @@ spec:
65576575
service:
65586576
description: expose the be listen ports
65596577
properties:
6578+
annotations:
6579+
additionalProperties:
6580+
type: string
6581+
description: Annotations for using function on different cloud
6582+
platform.
6583+
type: object
65606584
loadBalancerIP:
65616585
description: 'Only applies to Service Type: LoadBalancer.
65626586
This feature depends on whether the underlying cloud-provider

‎doc/api.md

+12-1
Original file line numberDiff line numberDiff line change
@@ -1353,6 +1353,17 @@ More info: <a href="https://kubernetes.io/docs/concepts/services-networking/serv
13531353
</tr>
13541354
<tr>
13551355
<td>
1356+
<code>annotations</code><br/>
1357+
<em>
1358+
map[string]string
1359+
</em>
1360+
</td>
1361+
<td>
1362+
<p>Annotations for using function on different cloud platform.</p>
1363+
</td>
1364+
</tr>
1365+
<tr>
1366+
<td>
13561367
<code>loadBalancerIP</code><br/>
13571368
<em>
13581369
string
@@ -2450,5 +2461,5 @@ string
24502461
<hr/>
24512462
<p><em>
24522463
Generated with <code>gen-crd-api-reference-docs</code>
2453-
on git commit <code>002533a</code>.
2464+
on git commit <code>490a4cc</code>.
24542465
</em></p>

‎pkg/common/utils/resource/pod.go

+30-12
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,10 @@ const (
3535
DEFAULT_ROOT_PATH = "/opt/apache-doris"
3636
POD_INFO_PATH = "/etc/podinfo"
3737
POD_INFO_VOLUME_NAME = "podinfo"
38+
39+
NODE_TOPOLOGYKEY = "kubernetes.io/hostname"
40+
41+
DEFAULT_INIT_IMAGE = "selectdb/alpine:latest"
3842
)
3943

4044
func NewPodTemplateSpec(dcr *v1.DorisCluster, componentType v1.ComponentType) corev1.PodTemplateSpec {
@@ -52,11 +56,9 @@ func NewPodTemplateSpec(dcr *v1.DorisCluster, componentType v1.ComponentType) co
5256
volumes = newVolumesFromBaseSpec(dcr.Spec.BeSpec.BaseSpec)
5357
si = dcr.Spec.BeSpec.BaseSpec.SystemInitialization
5458
dcrAffinity = dcr.Spec.BeSpec.BaseSpec.Affinity
55-
defaultInitContainers = append(defaultInitContainers, constructBeDefaultInitContainer())
5659
case v1.Component_CN:
5760
si = dcr.Spec.CnSpec.BaseSpec.SystemInitialization
5861
dcrAffinity = dcr.Spec.CnSpec.BaseSpec.Affinity
59-
defaultInitContainers = append(defaultInitContainers, constructBeDefaultInitContainer())
6062
case v1.Component_Broker:
6163
si = dcr.Spec.BrokerSpec.BaseSpec.SystemInitialization
6264
dcrAffinity = dcr.Spec.BrokerSpec.BaseSpec.Affinity
@@ -92,14 +94,26 @@ func NewPodTemplateSpec(dcr *v1.DorisCluster, componentType v1.ComponentType) co
9294
},
9395
}
9496

97+
constructInitContainers(componentType, &pts.Spec, si)
98+
pts.Spec.Affinity = constructAffinity(dcrAffinity, componentType)
99+
100+
return pts
101+
}
102+
103+
func constructInitContainers(componentType v1.ComponentType, podSpec *corev1.PodSpec, si *v1.SystemInitialization) {
104+
defaultImage := ""
105+
var defaultInitContains []corev1.Container
95106
if si != nil {
96107
initContainer := newBaseInitContainer("init", si)
97-
pts.Spec.InitContainers = append(pts.Spec.InitContainers, initContainer)
108+
defaultImage = si.InitImage
109+
defaultInitContains = append(defaultInitContains, initContainer)
98110
}
99111

100-
pts.Spec.Affinity = constructAffinity(dcrAffinity, componentType)
101-
102-
return pts
112+
// the init containers have sequence，should confirm use initial is always in the first priority.
113+
if componentType == v1.Component_BE || componentType == v1.Component_CN {
114+
podSpec.InitContainers = append(podSpec.InitContainers, constructBeDefaultInitContainer(defaultImage))
115+
}
116+
podSpec.InitContainers = append(podSpec.InitContainers, defaultInitContains...)
103117
}
104118

105119
// newVolumesFromBaseSpec return corev1.Volume build from baseSpec.
@@ -168,7 +182,7 @@ func newBaseInitContainer(name string, si *v1.SystemInitialization) corev1.Conta
168182
enablePrivileged := true
169183
initImage := si.InitImage
170184
if initImage == "" {
171-
initImage = "selectdb/alpine:latest"
185+
initImage = DEFAULT_INIT_IMAGE
172186
}
173187
c := corev1.Container{
174188
Image: initImage,
@@ -483,7 +497,10 @@ func livenessProbe(port int32, path string) *corev1.Probe {
483497
return &corev1.Probe{
484498
PeriodSeconds: 5,
485499
FailureThreshold: 3,
486-
ProbeHandler: getProbe(port, path),
500+
// for pulling image and start doris
501+
InitialDelaySeconds: 80,
502+
TimeoutSeconds: 180,
503+
ProbeHandler: getProbe(port, path),
487504
}
488505
}
489506

@@ -542,7 +559,7 @@ func getDefaultAffinity(componentType v1.ComponentType) *corev1.Affinity {
542559
{Key: v1.ComponentLabelKey, Operator: metav1.LabelSelectorOpIn, Values: []string{string(componentType)}},
543560
},
544561
},
545-
TopologyKey: "kubernetes.io/hostname",
562+
TopologyKey: NODE_TOPOLOGYKEY,
546563
},
547564
}
548565
return &corev1.Affinity{
@@ -571,12 +588,13 @@ func constructAffinity(dcrAffinity *corev1.Affinity, componentType v1.ComponentT
571588
return affinity
572589
}
573590

574-
func constructBeDefaultInitContainer() corev1.Container {
591+
func constructBeDefaultInitContainer(defaultImage string) corev1.Container {
575592
return newBaseInitContainer(
576593
"default-init",
577594
&v1.SystemInitialization{
578-
Command: []string{"/bin/sh"},
579-
Args: []string{"-c", "sysctl -w vm.max_map_count=2000000 && swapoff -a"},
595+
Command: []string{"/bin/sh"},
596+
InitImage: defaultImage,
597+
Args: []string{"-c", "sysctl -w vm.max_map_count=2000000 && swapoff -a"},
580598
},
581599
)
582600
}

‎pkg/common/utils/resource/service.go

+50-30
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@ type hashService struct {
1818
selector map[string]string
1919
//deal with external access load balancer.
2020
//serviceType corev1.ServiceType
21-
labels map[string]string
21+
labels map[string]string
22+
annotations map[string]string
2223
}
2324

2425
func BuildInternalService(dcr *v1.DorisCluster, componentType v1.ComponentType, config map[string]interface{}) corev1.Service {
@@ -79,13 +80,6 @@ func BuildExternalService(dcr *v1.DorisCluster, componentType v1.ComponentType,
7980
//the k8s service type.
8081
var ports []corev1.ServicePort
8182
var exportService *v1.ExportService
82-
svc := corev1.Service{
83-
ObjectMeta: metav1.ObjectMeta{
84-
Name: v1.GenerateExternalServiceName(dcr, componentType),
85-
Namespace: dcr.Namespace,
86-
Labels: labels,
87-
},
88-
}
8983

9084
switch componentType {
9185
case v1.Component_FE:
@@ -102,12 +96,25 @@ func BuildExternalService(dcr *v1.DorisCluster, componentType v1.ComponentType,
10296
klog.Infof("BuildExternalService componentType %s not supported.")
10397
}
10498

99+
svc := corev1.Service{
100+
ObjectMeta: metav1.ObjectMeta{
101+
Name: v1.GenerateExternalServiceName(dcr, componentType),
102+
Namespace: dcr.Namespace,
103+
Labels: labels,
104+
},
105+
}
106+
105107
constructServiceSpec(exportService, &svc, selector, ports)
108+
if exportService != nil {
109+
svc.Annotations = exportService.Annotations
110+
}
111+
106112
svc.OwnerReferences = []metav1.OwnerReference{getOwnerReference(dcr)}
107-
hso := serviceHashObject(&svc)
108-
anno := map[string]string{}
109-
anno[v1.ComponentResourceHash] = hash.HashObject(hso)
110-
svc.Annotations = anno
113+
// the code is invalid, duplicate with ServiceDeepEqual
114+
//hso := serviceHashObject(&svc)
115+
//anno := map[string]string{}
116+
//anno[v1.ComponentResourceHash] = hash.HashObject(hso)
117+
//svc.Annotations = anno
111118
return svc
112119
}
113120

@@ -277,33 +284,46 @@ func setServiceType(svc *v1.ExportService, service *corev1.Service) {
277284
service.Spec.LoadBalancerIP = svc.LoadBalancerIP
278285
}
279286
}
280-
281-
func ServiceDeepEqual(nsvc, oldsvc *corev1.Service) bool {
282-
var nhsvcValue, ohsvcValue string
283-
if _, ok := nsvc.Annotations[v1.ComponentResourceHash]; ok {
284-
nhsvcValue = nsvc.Annotations[v1.ComponentResourceHash]
287+
func ServiceDeepEqual(newSvc, oldSvc *corev1.Service) bool {
288+
var newHashValue, oldHashValue string
289+
if _, ok := newSvc.Annotations[v1.ComponentResourceHash]; ok {
290+
newHashValue = newSvc.Annotations[v1.ComponentResourceHash]
285291
} else {
286-
nhsvc := serviceHashObject(nsvc)
287-
nhsvcValue = hash.HashObject(nhsvc)
292+
newHashService := serviceHashObject(newSvc)
293+
newHashValue = hash.HashObject(newHashService)
288294
}
289295

290-
if _, ok := oldsvc.Annotations[v1.ComponentResourceHash]; ok {
291-
ohsvcValue = oldsvc.Annotations[v1.ComponentResourceHash]
296+
if _, ok := oldSvc.Annotations[v1.ComponentResourceHash]; ok {
297+
oldHashValue = oldSvc.Annotations[v1.ComponentResourceHash]
292298
} else {
293-
ohsvc := serviceHashObject(oldsvc)
294-
ohsvcValue = hash.HashObject(ohsvc)
299+
oldHashService := serviceHashObject(oldSvc)
300+
oldHashValue = hash.HashObject(oldHashService)
295301
}
296302

297-
return nhsvcValue == ohsvcValue &&
298-
nsvc.Namespace == oldsvc.Namespace /*&& oldGeneration == oldsvc.Generation*/
303+
// set hash value in annotation for avoiding deep equal.
304+
newSvc.Annotations = mergeMaps(newSvc.Annotations, map[string]string{v1.ComponentResourceHash: newHashValue})
305+
return newHashValue == oldHashValue &&
306+
newSvc.Namespace == oldSvc.Namespace
299307
}
300308

309+
// hash service for diff new generate service and old service in kubernetes.
301310
func serviceHashObject(svc *corev1.Service) hashService {
311+
annos := make(map[string]string, len(svc.Annotations))
312+
//for support service annotations, avoid hash value in annotations interfere equal comparison.
313+
for key, value := range svc.Annotations {
314+
if key == v1.ComponentResourceHash {
315+
continue
316+
}
317+
318+
annos[key] = value
319+
}
320+
302321
return hashService{
303-
name: svc.Name,
304-
namespace: svc.Namespace,
305-
ports: svc.Spec.Ports,
306-
selector: svc.Spec.Selector,
307-
labels: svc.Labels,
322+
name: svc.Name,
323+
namespace: svc.Namespace,
324+
ports: svc.Spec.Ports,
325+
selector: svc.Spec.Selector,
326+
labels: svc.Labels,
327+
annotations: annos,
308328
}
309329
}

‎pkg/controller/sub_controller/be/pod.go

+28
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package be
22

33
import (
44
"context"
5+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
56
"strconv"
67

78
v1 "github.com/selectdb/doris-operator/api/doris/v1"
@@ -11,6 +12,8 @@ import (
1112

1213
func (be *Controller) buildBEPodTemplateSpec(dcr *v1.DorisCluster) corev1.PodTemplateSpec {
1314
podTemplateSpec := resource.NewPodTemplateSpec(dcr, v1.Component_BE)
15+
be.addFeAntiAffinity(&podTemplateSpec)
16+
1417
var containers []corev1.Container
1518
containers = append(containers, podTemplateSpec.Spec.Containers...)
1619
beContainer := be.beContainer(dcr)
@@ -19,6 +22,31 @@ func (be *Controller) buildBEPodTemplateSpec(dcr *v1.DorisCluster) corev1.PodTem
1922
return podTemplateSpec
2023
}
2124

25+
// be pods add fe anti affinity for prefer deploy fe and be on different nodes.
26+
func (be *Controller) addFeAntiAffinity(tplSpec *corev1.PodTemplateSpec) {
27+
preferedScheduleTerm := corev1.WeightedPodAffinityTerm{
28+
Weight: 80,
29+
PodAffinityTerm: corev1.PodAffinityTerm{
30+
TopologyKey: resource.NODE_TOPOLOGYKEY,
31+
LabelSelector: &metav1.LabelSelector{
32+
MatchLabels: map[string]string{
33+
resource.NODE_TOPOLOGYKEY: string(v1.Component_FE),
34+
},
35+
},
36+
},
37+
}
38+
39+
if tplSpec.Spec.Affinity == nil {
40+
tplSpec.Spec.Affinity = &corev1.Affinity{}
41+
}
42+
if tplSpec.Spec.Affinity.PodAntiAffinity == nil {
43+
tplSpec.Spec.Affinity.PodAntiAffinity = &corev1.PodAntiAffinity{}
44+
}
45+
46+
tplSpec.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution = append(tplSpec.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution,
47+
preferedScheduleTerm)
48+
}
49+
2250
func (be *Controller) beContainer(dcr *v1.DorisCluster) corev1.Container {
2351
config, _ := be.GetConfig(context.Background(), &dcr.Spec.BeSpec.ConfigMapInfo, dcr.Namespace)
2452
c := resource.NewBaseMainContainer(dcr, config, v1.Component_BE)

0 commit comments

Comments
 (0)