Ops-warn on any clickhouse error (will see how noisy this becomes)

PRX · Aug 28, 2024 · 120c18a · 120c18a
1 parent 6f4f197
commit 120c18a
Showing 1 changed file with 34 additions and 0 deletions.
diff --git a/spire/templates/shared-clickhouse/instance.yml b/spire/templates/shared-clickhouse/instance.yml
@@ -358,6 +358,40 @@ Resources:
         - { Key: prx:ops:environment, Value: !Ref EnvironmentType }
         - { Key: prx:dev:family, Value: Dovetail }
         - { Key: prx:dev:application, Value: Clickhouse }
+  ClickhouseErrorsMetricFilter:
+    Type: AWS::Logs::MetricFilter
+    Properties:
+      FilterPattern: '{ $.level = "error" }'
+      LogGroupName: !Ref ClickhouseErrorLogGroup
+      MetricTransformations:
+        - MetricName: !Sub clickhouse_errors_${EnvironmentTypeAbbreviation}
+          MetricNamespace: PRX/Clickhouse
+          MetricValue: "1"
+  ClickhouseErrorsAlarm:
+    Type: AWS::CloudWatch::Alarm
+    Condition: IsProduction
+    Properties:
+      AlarmName: !Sub WARN [Shared] Clickhouse <${EnvironmentTypeAbbreviation}> LOGGING ERRORS
+      AlarmDescription: !Sub >-
+        Clickhouse ${EnvironmentType} logged some errors. Go take a look!
+      ComparisonOperator: GreaterThanThreshold
+      EvaluationPeriods: 1
+      MetricName: !Sub clickhouse_errors_${EnvironmentTypeAbbreviation}
+      Namespace: PRX/Clickhouse
+      Period: 60
+      Statistic: Sum
+      Tags:
+        - { Key: prx:meta:tagging-version, Value: "2021-04-07" }
+        - { Key: prx:cloudformation:stack-name, Value: !Ref AWS::StackName }
+        - { Key: prx:cloudformation:stack-id, Value: !Ref AWS::StackId }
+        - { Key: prx:cloudformation:root-stack-name, Value: !Ref RootStackName }
+        - { Key: prx:cloudformation:root-stack-id, Value: !Ref RootStackId }
+        - { Key: prx:ops:environment, Value: !Ref EnvironmentType }
+        - { Key: prx:ops:cloudwatch-log-group-name, Value: !Ref ClickhouseErrorLogGroup }
+        - { Key: prx:dev:family, Value: Dovetail }
+        - { Key: prx:dev:application, Value: Clickhouse }
+      Threshold: 0
+      TreatMissingData: notBreaching
   ClickhouseMemoryLimitExceededMetricFilter:
     Type: AWS::Logs::MetricFilter
     Properties: