Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(aws-backup): avoid duplicated and ill-routed alerts #508

Merged
merged 3 commits into from
Mar 4, 2024
+17 −21
Merged

fix(aws-backup): avoid duplicated and ill-routed alerts #508

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
62fafbf
fix(aws-backup): avoid duplicated and ill-routed alerts
Oct 19, 2023
19d1aaa
fix(aws-backup): remove duplicated filtering
Mar 1, 2024
a3918fe
fix(aws-backup): remove aws_account_id variable
Mar 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions modules/integration_aws-backup/common-filters.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,11 @@
# Common filters at ../../common/module/filters-integration-aws.tf are not used
# with AWS Backup because recovery points (snapshots) and jobs are unlikely to
# be tagged with `env` and `sfx_monitored`.
#
# We are also assuming that we want to be notified for every backup vault and
# resource types. `match_missing=False` is required to avoid duplicated signal
# and alerts, since CloudWatch returns aggregated metric, and per vault and
# resource type.
locals {
filters = "filter('ResourceType', '*')"
}
filters = "filter('namespace', 'AWS/Backup') and filter('stat', 'sum') and filter('BackupVaultName', '*', match_missing=False) and filter('ResourceType', '*', match_missing=False)"
}
1 change: 0 additions & 1 deletion modules/integration_aws-backup/conf/00-aws-backup-failed.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ id: backup_failed

transformation: ".max(over='1d').fill(0)"
aggregation: true
filtering: "filter('namespace', 'AWS/Backup') and filter('stat', 'sum')"

signals:
signal:
Expand Down
1 change: 0 additions & 1 deletion modules/integration_aws-backup/conf/01-aws-backup-job-expired.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ id: backup_job_expired

transformation: ".max(over='1d').fill(0)"
aggregation: true
filtering: "filter('namespace', 'AWS/Backup') and filter('stat', 'sum')"

signals:
signal:
Expand Down
1 change: 0 additions & 1 deletion modules/integration_aws-backup/conf/02-aws-backup-copyjob-failed.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ id: backup_copy_jobs_failed

transformation: ".max(over='1d').fill(0)"
aggregation: true
filtering: "filter('namespace', 'AWS/Backup') and filter('stat', 'sum')"

signals:
signal:
Expand Down
1 change: 0 additions & 1 deletion modules/integration_aws-backup/conf/03-aws-backup-check.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ id: backup_successful

transformation: ".min(over='23h')"
aggregation: true
filtering: "filter('namespace', 'AWS/Backup') and filter('stat', 'sum')"

signals:
created:
Expand Down
1 change: 0 additions & 1 deletion modules/integration_aws-backup/conf/04-aws-backup-rp-partial.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ id: backup_rp_partial

transformation: ".max(over='1d').fill(0)"
aggregation: true
filtering: "filter('namespace', 'AWS/Backup') and filter('stat', 'sum')"

signals:
signal:
Expand Down
1 change: 0 additions & 1 deletion modules/integration_aws-backup/conf/05-aws-backup-rp-expired.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ id: backup_rp_expired

transformation: ".max(over='1d').fill(0)"
aggregation: true
filtering: "filter('namespace', 'AWS/Backup') and filter('stat', 'sum')"

signals:
signal:
Expand Down
20 changes: 7 additions & 13 deletions modules/integration_aws-backup/detectors-gen.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,7 @@ resource "signalfx_detector" "backup_failed" {
tags = compact(concat(local.common_tags, local.tags, var.extra_tags))

program_text = <<-EOF
base_filtering = filter('namespace', 'AWS/Backup') and filter('stat', 'sum')
signal = data('NumberOfBackupJobsFailed', filter=base_filtering and ${module.filtering.signalflow})${var.backup_failed_aggregation_function}${var.backup_failed_transformation_function}.publish('signal')
signal = data('NumberOfBackupJobsFailed', filter=${module.filtering.signalflow})${var.backup_failed_aggregation_function}${var.backup_failed_transformation_function}.publish('signal')
detect(when(signal > ${var.backup_failed_threshold_critical}, lasting=%{if var.backup_failed_lasting_duration_critical == null}None%{else}'${var.backup_failed_lasting_duration_critical}'%{endif}, at_least=${var.backup_failed_at_least_percentage_critical})).publish('CRIT')
EOF

Expand All @@ -34,8 +33,7 @@ resource "signalfx_detector" "backup_job_expired" {
tags = compact(concat(local.common_tags, local.tags, var.extra_tags))

program_text = <<-EOF
base_filtering = filter('namespace', 'AWS/Backup') and filter('stat', 'sum')
signal = data('NumberOfBackupJobsExpired', filter=base_filtering and ${module.filtering.signalflow}, extrapolation='zero')${var.backup_job_expired_aggregation_function}${var.backup_job_expired_transformation_function}.publish('signal')
signal = data('NumberOfBackupJobsExpired', filter=${module.filtering.signalflow}, extrapolation='zero')${var.backup_job_expired_aggregation_function}${var.backup_job_expired_transformation_function}.publish('signal')
detect(when(signal > ${var.backup_job_expired_threshold_critical}, lasting=%{if var.backup_job_expired_lasting_duration_critical == null}None%{else}'${var.backup_job_expired_lasting_duration_critical}'%{endif}, at_least=${var.backup_job_expired_at_least_percentage_critical})).publish('CRIT')
EOF

Expand All @@ -62,8 +60,7 @@ resource "signalfx_detector" "backup_copy_jobs_failed" {
tags = compact(concat(local.common_tags, local.tags, var.extra_tags))

program_text = <<-EOF
base_filtering = filter('namespace', 'AWS/Backup') and filter('stat', 'sum')
signal = data('NumberOfCopyJobsFailed', filter=base_filtering and ${module.filtering.signalflow})${var.backup_copy_jobs_failed_aggregation_function}${var.backup_copy_jobs_failed_transformation_function}.publish('signal')
signal = data('NumberOfCopyJobsFailed', filter=${module.filtering.signalflow})${var.backup_copy_jobs_failed_aggregation_function}${var.backup_copy_jobs_failed_transformation_function}.publish('signal')
detect(when(signal > ${var.backup_copy_jobs_failed_threshold_critical}, lasting=%{if var.backup_copy_jobs_failed_lasting_duration_critical == null}None%{else}'${var.backup_copy_jobs_failed_lasting_duration_critical}'%{endif}, at_least=${var.backup_copy_jobs_failed_at_least_percentage_critical})).publish('CRIT')
EOF

Expand All @@ -90,9 +87,8 @@ resource "signalfx_detector" "backup_successful" {
tags = compact(concat(local.common_tags, local.tags, var.extra_tags))

program_text = <<-EOF
base_filtering = filter('namespace', 'AWS/Backup') and filter('stat', 'sum')
created = data('NumberOfBackupJobsCreated', filter=base_filtering and ${module.filtering.signalflow}, extrapolation='zero')${var.backup_successful_aggregation_function}${var.backup_successful_transformation_function}
completed = data('NumberOfBackupJobsCompleted', filter=base_filtering and ${module.filtering.signalflow}, extrapolation='zero')${var.backup_successful_aggregation_function}${var.backup_successful_transformation_function}
created = data('NumberOfBackupJobsCreated', filter=${module.filtering.signalflow}, extrapolation='zero')${var.backup_successful_aggregation_function}${var.backup_successful_transformation_function}
completed = data('NumberOfBackupJobsCompleted', filter=${module.filtering.signalflow}, extrapolation='zero')${var.backup_successful_aggregation_function}${var.backup_successful_transformation_function}
signal = (created-completed).publish('signal')
detect(when(signal > ${var.backup_successful_threshold_critical}, lasting=%{if var.backup_successful_lasting_duration_critical == null}None%{else}'${var.backup_successful_lasting_duration_critical}'%{endif}, at_least=${var.backup_successful_at_least_percentage_critical})).publish('CRIT')
EOF
Expand Down Expand Up @@ -120,8 +116,7 @@ resource "signalfx_detector" "backup_rp_partial" {
tags = compact(concat(local.common_tags, local.tags, var.extra_tags))

program_text = <<-EOF
base_filtering = filter('namespace', 'AWS/Backup') and filter('stat', 'sum')
signal = data('NumberOfRecoveryPointsPartial', filter=base_filtering and ${module.filtering.signalflow})${var.backup_rp_partial_aggregation_function}${var.backup_rp_partial_transformation_function}.publish('signal')
signal = data('NumberOfRecoveryPointsPartial', filter=${module.filtering.signalflow})${var.backup_rp_partial_aggregation_function}${var.backup_rp_partial_transformation_function}.publish('signal')
detect(when(signal > ${var.backup_rp_partial_threshold_minor}, lasting=%{if var.backup_rp_partial_lasting_duration_minor == null}None%{else}'${var.backup_rp_partial_lasting_duration_minor}'%{endif}, at_least=${var.backup_rp_partial_at_least_percentage_minor})).publish('MINOR')
EOF

Expand All @@ -148,8 +143,7 @@ resource "signalfx_detector" "backup_rp_expired" {
tags = compact(concat(local.common_tags, local.tags, var.extra_tags))

program_text = <<-EOF
base_filtering = filter('namespace', 'AWS/Backup') and filter('stat', 'sum')
signal = data('NumberOfRecoveryPointsExpired', filter=base_filtering and ${module.filtering.signalflow})${var.backup_rp_expired_aggregation_function}${var.backup_rp_expired_transformation_function}.publish('signal')
signal = data('NumberOfRecoveryPointsExpired', filter=${module.filtering.signalflow})${var.backup_rp_expired_aggregation_function}${var.backup_rp_expired_transformation_function}.publish('signal')
detect(when(signal > ${var.backup_rp_expired_threshold_major}, lasting=%{if var.backup_rp_expired_lasting_duration_major == null}None%{else}'${var.backup_rp_expired_lasting_duration_major}'%{endif}, at_least=${var.backup_rp_expired_at_least_percentage_major})).publish('MAJOR')
EOF

Expand Down
Loading