Explicitly create log groups; add filter+alarm

PRX · Aug 28, 2024 · 7cd75d0 · 7cd75d0
1 parent 65a040c
commit 7cd75d0
Showing 1 changed file with 74 additions and 3 deletions.
diff --git a/spire/templates/shared-clickhouse/instance.yml b/spire/templates/shared-clickhouse/instance.yml
@@ -67,7 +67,9 @@ Resources:
                   - logs:PutLogEvents
                   - logs:PutRetentionPolicy
                 Effect: Allow
-                Resource: !Sub arn:aws:logs:*:*:log-group:/prx/clickhouse/${EnvironmentTypeAbbreviation}/*
+                Resource:
+                  - !Ref ClickhouseLogGroup
+                  - !Ref ClickhouseErrorLogGroup
             Version: "2012-10-17"
           PolicyName: CloudWatchAgentPermissions
       Tags:
@@ -186,14 +188,14 @@ Resources:
                           {
                             "auto_removal": true,
                             "file_path": "/var/log/clickhouse-server/clickhouse-server.log",
-                            "log_group_name": "/prx/clickhouse/${EnvironmentTypeAbbreviation}/server.log",
+                            "log_group_name": "${ClickhouseLogGroup}",
                             "log_stream_name": "{instance_id}",
                             "retention_in_days": 30
                           },
                           {
                             "auto_removal": true,
                             "file_path": "/var/log/clickhouse-server/clickhouse-server.err.log",
-                            "log_group_name": "/prx/clickhouse/${EnvironmentTypeAbbreviation}/server.err.log",
+                            "log_group_name": "${ClickhouseErrorLogGroup}",
                             "log_stream_name": "{instance_id}",
                             "retention_in_days": 30
                           }
@@ -324,6 +326,75 @@ Resources:
             # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-init.html
             /opt/aws/bin/cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource ClickhouseServer5
 
+  ClickhouseLogGroup:
+    Type: AWS::Logs::LogGroup
+    DeletionPolicy: Delete
+    UpdateReplacePolicy: Delete
+    Properties:
+      LogGroupName: !Sub /prx/clickhouse/${EnvironmentTypeAbbreviation}/clickhouse-server.log
+      RetentionInDays: 14
+      Tags:
+        - { Key: prx:meta:tagging-version, Value: "2021-04-07" }
+        - { Key: prx:cloudformation:stack-name, Value: !Ref AWS::StackName }
+        - { Key: prx:cloudformation:stack-id, Value: !Ref AWS::StackId }
+        - { Key: prx:cloudformation:root-stack-name, Value: !Ref RootStackName }
+        - { Key: prx:cloudformation:root-stack-id, Value: !Ref RootStackId }
+        - { Key: prx:ops:environment, Value: !Ref EnvironmentType }
+        - { Key: prx:dev:family, Value: Dovetail }
+        - { Key: prx:dev:application, Value: Clickhouse }
+  ClickhouseErrorLogGroup:
+    Type: AWS::Logs::LogGroup
+    DeletionPolicy: Delete
+    UpdateReplacePolicy: Delete
+    Properties:
+      LogGroupName: !Sub /prx/clickhouse/${EnvironmentTypeAbbreviation}/clickhouse-server.err.log
+      RetentionInDays: 14
+      Tags:
+        - { Key: prx:meta:tagging-version, Value: "2021-04-07" }
+        - { Key: prx:cloudformation:stack-name, Value: !Ref AWS::StackName }
+        - { Key: prx:cloudformation:stack-id, Value: !Ref AWS::StackId }
+        - { Key: prx:cloudformation:root-stack-name, Value: !Ref RootStackName }
+        - { Key: prx:cloudformation:root-stack-id, Value: !Ref RootStackId }
+        - { Key: prx:ops:environment, Value: !Ref EnvironmentType }
+        - { Key: prx:dev:family, Value: Dovetail }
+        - { Key: prx:dev:application, Value: Clickhouse }
+  ClickhouseMemoryLimitExceededMetricFilter:
+    Type: AWS::Logs::MetricFilter
+    Properties:
+      FilterPattern: "{ $.message = %MEMORY_LIMIT_EXCEEDED% }"
+      LogGroupName: !Ref ClickhouseErrorLogGroup
+      MetricTransformations:
+        - MetricName: !Sub clickhouse_memory_limit_exceeded_${EnvironmentTypeAbbreviation}
+          MetricNamespace: PRX/Clickhouse
+          MetricValue: "1"
+  ClickhouseMemoryLimitExceededAlarm:
+    Type: AWS::CloudWatch::Alarm
+    Condition: IsProduction
+    Properties:
+      AlarmName: !Sub ERROR [Shared] Clickhouse <${EnvironmentTypeAbbreviation}> MEMORY LIMIT EXCEEDED
+      AlarmDescription: !Sub >-
+        Clickhouse ${EnvironmentType} failed to perform a background-merge
+        operation or query that would have exceeded its memory. These are
+        normally not a huge deal, but good to keep an eye on.
+      ComparisonOperator: GreaterThanThreshold
+      EvaluationPeriods: 1
+      MetricName: !Sub clickhouse_memory_limit_exceeded_${EnvironmentTypeAbbreviation}
+      Namespace: PRX/Clickhouse
+      Period: 60
+      Statistic: Sum
+      Tags:
+        - { Key: prx:meta:tagging-version, Value: "2021-04-07" }
+        - { Key: prx:cloudformation:stack-name, Value: !Ref AWS::StackName }
+        - { Key: prx:cloudformation:stack-id, Value: !Ref AWS::StackId }
+        - { Key: prx:cloudformation:root-stack-name, Value: !Ref RootStackName }
+        - { Key: prx:cloudformation:root-stack-id, Value: !Ref RootStackId }
+        - { Key: prx:ops:environment, Value: !Ref EnvironmentType }
+        - { Key: prx:ops:cloudwatch-log-group-name, Value: !Ref ClickhouseErrorLogGroup }
+        - { Key: prx:dev:family, Value: Dovetail }
+        - { Key: prx:dev:application, Value: Clickhouse }
+      Threshold: 0
+      TreatMissingData: notBreaching
+
 Outputs:
   HttpPort:
     Description: Clickhouse HTTP Port