first pass at zone awareness

Signed-off-by: Edward Welch <[email protected]>
grafana · Mar 3, 2024 · 5e27c1f · 5e27c1f
1 parent c4d44ea
commit 5e27c1f
Show file tree

Hide file tree

Showing 15 changed files with 836 additions and 10 deletions.
diff --git a/production/helm/loki/ci/distributed-values.yaml b/production/helm/loki/ci/distributed-values.yaml
@@ -12,7 +12,7 @@ read:
 write:
   replicas: 0
 ingester:
-  replicas: 1
+  replicas: 2
 querier:
   replicas: 1
 queryFrontend:

diff --git a/production/helm/loki/templates/distributor/deployment-distributor.yaml b/production/helm/loki/templates/distributor/deployment-distributor.yaml
@@ -68,6 +68,13 @@ spec:
           args:
             - -config.file=/etc/loki/config/config.yaml
             - -target=distributor
+            {{- if .Values.ingester.zoneAwareReplication.enabled }}
+            {{- if and (.Values.ingester.zoneAwareReplication.migration.enabled) (not .Values.ingester.zoneAwareReplication.migration.writePath) }}
+            - -distributor.zone-awareness-enabled=false
+            {{- else }}
+            - -distributor.zone-awareness-enabled=true
+            {{- end }}
+            {{- end }}
             {{- with .Values.distributor.extraArgs }}
             {{- toYaml . | nindent 12 }}
             {{- end }}

diff --git a/production/helm/loki/templates/ingester/poddisruptionbudget-ingester-rollout.yaml b/production/helm/loki/templates/ingester/poddisruptionbudget-ingester-rollout.yaml
@@ -0,0 +1,21 @@
+{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}}
+{{- if and $isDistributed (gt (int .Values.ingester.replicas) 1) (.Values.ingester.zoneAwareReplication.enabled) }}
+{{- if kindIs "invalid" .Values.ingester.maxUnavailable }}
+{{- fail "`.Values.ingester.maxUnavailable` must be set when `.Values.ingester.replicas` is greater than 1." }}
+{{- else }}
+apiVersion: {{ include "loki.pdb.apiVersion" . }}
+kind: PodDisruptionBudget
+metadata:
+  name: {{ include "loki.ingesterFullname" . }}-rollout
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "loki.ingesterLabels" . | nindent 4 }}
+spec:
+  selector:
+    matchLabels:
+      rollout-group: ingester
+  {{- with .Values.ingester.maxUnavailable }}
+  maxUnavailable: {{ . }}
+  {{- end }}
+{{- end }}
+{{- end }}
diff --git a/production/helm/loki/templates/ingester/poddisruptionbudget-ingester.yaml b/production/helm/loki/templates/ingester/poddisruptionbudget-ingester.yaml
@@ -1,5 +1,5 @@
 {{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}}
-{{- if and $isDistributed (gt (int .Values.ingester.replicas) 1) }}
+{{- if and $isDistributed (gt (int .Values.ingester.replicas) 1) (or (not .Values.ingester.zoneAwareReplication.enabled) .Values.ingester.zoneAwareReplication.migration.enabled) }}
 {{- if kindIs "invalid" .Values.ingester.maxUnavailable }}
 {{- fail "`.Values.ingester.maxUnavailable` must be set when `.Values.ingester.replicas` is greater than 1." }}
 {{- else }}
@@ -14,6 +14,12 @@ spec:
   selector:
     matchLabels:
       {{- include "loki.ingesterSelectorLabels" . | nindent 6 }}
+    {{/* zone aware ingesters get their own pod disruption budget, ignore them here */}}
+    matchExpressions:
+      - key: rollout-group
+        operator: NotIn
+        values:
+          - "ingester"
   {{- with .Values.ingester.maxUnavailable }}
   maxUnavailable: {{ . }}
   {{- end }}

diff --git a/production/helm/loki/templates/ingester/service-ingester-headless.yaml b/production/helm/loki/templates/ingester/service-ingester-headless.yaml
@@ -1,5 +1,5 @@
 {{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}}
-{{- if $isDistributed -}}
+{{- if and $isDistributed (or (not .Values.ingester.zoneAwareReplication.enabled) .Values.ingester.zoneAwareReplication.migration.enabled) }}
 apiVersion: v1
 kind: Service
 metadata:

diff --git a/production/helm/loki/templates/ingester/service-ingester-zone-a-headless.yaml b/production/helm/loki/templates/ingester/service-ingester-zone-a-headless.yaml
@@ -0,0 +1,36 @@
+{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}}
+{{- if and $isDistributed .Values.ingester.zoneAwareReplication.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "loki.ingesterFullname" . }}-zone-a
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "loki.ingesterLabels" . | nindent 4 }}
+    name: ingester-zone-a
+    {{- with .Values.ingester.serviceLabels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+  {{- with .Values.loki.serviceAnnotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  clusterIP: None
+  ports:
+    - name: http-metrics
+      port: 3100
+      targetPort: http-metrics
+      protocol: TCP
+    - name: grpc
+      port: 9095
+      targetPort: grpc
+      protocol: TCP
+      {{- if .Values.ingester.appProtocol.grpc }}
+      appProtocol: {{ .Values.ingester.appProtocol.grpc }}
+      {{- end }}
+  selector:
+    {{- include "loki.ingesterSelectorLabels" . | nindent 4 }}
+    name: ingester-zone-a
+    rollout-group: ingester
+{{- end -}}
diff --git a/production/helm/loki/templates/ingester/service-ingester-zone-b-headless.yaml b/production/helm/loki/templates/ingester/service-ingester-zone-b-headless.yaml
@@ -0,0 +1,36 @@
+{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}}
+{{- if and $isDistributed .Values.ingester.zoneAwareReplication.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "loki.ingesterFullname" . }}-zone-b
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "loki.ingesterLabels" . | nindent 4 }}
+    name: ingester-zone-b
+    {{- with .Values.ingester.serviceLabels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+  {{- with .Values.loki.serviceAnnotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  clusterIP: None
+  ports:
+    - name: http-metrics
+      port: 3100
+      targetPort: http-metrics
+      protocol: TCP
+    - name: grpc
+      port: 9095
+      targetPort: grpc
+      protocol: TCP
+      {{- if .Values.ingester.appProtocol.grpc }}
+      appProtocol: {{ .Values.ingester.appProtocol.grpc }}
+      {{- end }}
+  selector:
+    {{- include "loki.ingesterSelectorLabels" . | nindent 4 }}
+    name: ingester-zone-b
+    rollout-group: ingester
+{{- end -}}
diff --git a/production/helm/loki/templates/ingester/service-ingester-zone-c-headless.yaml b/production/helm/loki/templates/ingester/service-ingester-zone-c-headless.yaml
@@ -0,0 +1,36 @@
+{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}}
+{{- if and $isDistributed .Values.ingester.zoneAwareReplication.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "loki.ingesterFullname" . }}-zone-c
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "loki.ingesterLabels" . | nindent 4 }}
+    name: ingester-zone-c
+    {{- with .Values.ingester.serviceLabels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+  {{- with .Values.loki.serviceAnnotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  clusterIP: None
+  ports:
+    - name: http-metrics
+      port: 3100
+      targetPort: http-metrics
+      protocol: TCP
+    - name: grpc
+      port: 9095
+      targetPort: grpc
+      protocol: TCP
+      {{- if .Values.ingester.appProtocol.grpc }}
+      appProtocol: {{ .Values.ingester.appProtocol.grpc }}
+      {{- end }}
+  selector:
+    {{- include "loki.ingesterSelectorLabels" . | nindent 4 }}
+    name: ingester-zone-c
+    rollout-group: ingester
+{{- end -}}
diff --git a/production/helm/loki/templates/ingester/service-ingester.yaml b/production/helm/loki/templates/ingester/service-ingester.yaml
@@ -1,5 +1,5 @@
 {{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}}
-{{- if $isDistributed -}}
+{{- if and $isDistributed (or (not .Values.ingester.zoneAwareReplication.enabled) .Values.ingester.zoneAwareReplication.migration.enabled) }}
 apiVersion: v1
 kind: Service
 metadata: