diff --git a/spire/templates/apps-300A.yml b/spire/templates/apps-300A.yml index 8341b4cd..b072e577 100644 --- a/spire/templates/apps-300A.yml +++ b/spire/templates/apps-300A.yml @@ -151,6 +151,8 @@ Resources: DovetailCountedKinesisStreamName: !Ref DovetailCountedKinesisStreamName DovetailRouterHosts: !Sub /prx/${EnvironmentTypeAbbreviation}/dovetail-analytics/DOVETAIL_ROUTER_HOSTS DovetailRouterApiTokens: !Sub /prx/${EnvironmentTypeAbbreviation}/dovetail-analytics/DOVETAIL_ROUTER_API_TOKENS + FrequencyDynamodbTableName: !Sub /prx/${EnvironmentTypeAbbreviation}/Spire/Dovetail-Analytics/FREQUENCY_DDB_TABLE + FrequencyDynamodbAccessRoleArn: !Sub /prx/${EnvironmentTypeAbbreviation}/Spire/Dovetail-Analytics/FREQUENCY_DDB_ACCESS_ROLE Tags: - { Key: prx:meta:tagging-version, Value: "2021-04-07" } - { Key: prx:cloudformation:stack-name, Value: !Ref AWS::StackName } @@ -195,6 +197,8 @@ Resources: DovetailCdnHostname: !Ref DovetailCdnHostname DovetailRouterHostname: !Ref DovetailRouterHostname DovetailCdnRedirectPrefix: !Sub /prx/${EnvironmentTypeAbbreviation}/Spire/Dovetail-Router/${AWS::Region}/redirect-prefix + FrequencyDynamodbTableName: !Sub /prx/${EnvironmentTypeAbbreviation}/Spire/Dovetail-Analytics/FREQUENCY_DDB_TABLE + FrequencyDynamodbAccessRoleArn: !Sub /prx/${EnvironmentTypeAbbreviation}/Spire/Dovetail-Analytics/FREQUENCY_DDB_ACCESS_ROLE Tags: - { Key: prx:meta:tagging-version, Value: "2021-04-07" } - { Key: prx:cloudformation:stack-name, Value: !Ref AWS::StackName } diff --git a/spire/templates/apps/dovetail-analytics.yml b/spire/templates/apps/dovetail-analytics.yml index 00236819..6ee96f07 100644 --- a/spire/templates/apps/dovetail-analytics.yml +++ b/spire/templates/apps/dovetail-analytics.yml @@ -6,7 +6,8 @@ Transform: AWS::Serverless-2016-10-31 Description: >- Creates a number of Lambda functions that collect Dovetail metrics data from Kinesis streams, and process and forward that data to various - destinations, like BigQuery and third-parties as pingback. + destinations, like BigQuery, 3rd-party pingbacks, and DynamoDB for + campaign impressions and listener frequency. Parameters: kMetricFilterNamespace: @@ -29,6 +30,8 @@ Parameters: DovetailCountedKinesisStreamName: { Type: String } DovetailRouterHosts: { Type: AWS::SSM::Parameter::Value } DovetailRouterApiTokens: { Type: AWS::SSM::Parameter::Value } + FrequencyDynamodbTableName: { Type: AWS::SSM::Parameter::Value } + FrequencyDynamodbAccessRoleArn: { Type: AWS::SSM::Parameter::Value } Conditions: IsProduction: !Equals [!Ref EnvironmentType, Production] @@ -722,6 +725,278 @@ Resources: MetricNamespace: !Ref kMetricFilterNamespace MetricValue: "1" + # Frequency + AnalyticsFrequencyFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: + Bucket: !Ref CodeS3Bucket + Key: !Ref CodeS3ObjectKey + Description: !Sub >- + ${EnvironmentType} Dovetail Analytics saving Frequency data to DynamoDB + Environment: + Variables: + AWS_NODEJS_CONNECTION_REUSE_ENABLED: "1" + FREQUENCY: "true" # set function mode = frequency + DDB_FREQUENCY_TABLE: !Ref FrequencyDynamodbTableName + DDB_ROLE: !Ref FrequencyDynamodbAccessRoleArn + Events: + FrequencyKinesisTrigger: + Properties: + BatchSize: 50 + BisectBatchOnFunctionError: true + Enabled: true + StartingPosition: LATEST + Stream: !Ref DovetailCountedKinesisStreamArn + Type: Kinesis + Handler: index.handler + MemorySize: 512 + Runtime: nodejs16.x + Policies: + - !Ref ParameterStoreReadPolicy + - arn:aws:iam::aws:policy/service-role/AWSLambdaKinesisExecutionRole + - Statement: + - Action: + - dynamodb:BatchGetItem + - dynamodb:BatchWriteItem + - dynamodb:ConditionCheck + - dynamodb:DeleteItem + - dynamodb:DescribeTable + - dynamodb:DescribeTimeToLive + - dynamodb:GetItem + - dynamodb:PutItem + - dynamodb:Query + - dynamodb:UpdateItem + Effect: Allow + # TODO: can this be done with an AWS::Partition Sub? + Resource: + - !Sub "arn:aws:dynamodb:*:*:table/${FrequencyDynamodbTableName}" + Version: "2012-10-17" + - Statement: + - Action: sts:AssumeRole + Effect: Allow + Resource: !Ref FrequencyDynamodbAccessRoleArn + Version: "2012-10-17" + Tags: + prx:meta:tagging-version: "2021-04-07" + prx:cloudformation:stack-name: !Ref AWS::StackName + prx:cloudformation:stack-id: !Ref AWS::StackId + prx:cloudformation:root-stack-name: !Ref RootStackName + prx:cloudformation:root-stack-id: !Ref RootStackId + prx:ops:environment: !Ref EnvironmentType + prx:dev:family: Dovetail + prx:dev:application: Analytics + Timeout: 30 + AnalyticsFrequencyFunctionLogGroup: + Type: AWS::Logs::LogGroup + DeletionPolicy: Delete + UpdateReplacePolicy: Delete + Properties: + LogGroupName: !Sub /aws/lambda/${AnalyticsFrequencyFunction} + RetentionInDays: 14 + Tags: + - { Key: prx:meta:tagging-version, Value: "2021-04-07" } + - { Key: prx:cloudformation:stack-name, Value: !Ref AWS::StackName } + - { Key: prx:cloudformation:stack-id, Value: !Ref AWS::StackId } + - { Key: prx:cloudformation:root-stack-name, Value: !Ref RootStackName } + - { Key: prx:cloudformation:root-stack-id, Value: !Ref RootStackId } + - { Key: prx:ops:environment, Value: !Ref EnvironmentType } + - { Key: prx:dev:family, Value: Dovetail } + - { Key: prx:dev:application, Value: Analytics } + AnalyticsFrequencyFunctionElevatedErrorAlarm: + Type: AWS::CloudWatch::Alarm + Condition: IsProduction + Properties: + AlarmName: !Sub WARN [Dovetail-Analytics] Frequency Lambda function <${EnvironmentTypeAbbreviation}> INVOCATIONS ERRORS (${RootStackName}) + AlarmDescription: !Sub >- + ${EnvironmentType} Dovetail Analytics Frequency Lambda function is failing. + ComparisonOperator: GreaterThanThreshold + Dimensions: + - Name: FunctionName + Value: !Ref AnalyticsFrequencyFunction + EvaluationPeriods: 5 + MetricName: Errors + Namespace: AWS/Lambda + Period: 60 + Statistic: Sum + Tags: + - { Key: prx:meta:tagging-version, Value: "2021-04-07" } + - { Key: prx:cloudformation:stack-name, Value: !Ref AWS::StackName } + - { Key: prx:cloudformation:stack-id, Value: !Ref AWS::StackId } + - { Key: prx:cloudformation:root-stack-name, Value: !Ref RootStackName } + - { Key: prx:cloudformation:root-stack-id, Value: !Ref RootStackId } + - { Key: prx:ops:environment, Value: !Ref EnvironmentType } + - { Key: prx:dev:family, Value: Dovetail } + - { Key: prx:dev:application, Value: Analytics } + Threshold: 0 + TreatMissingData: notBreaching + AnalyticsFrequencyFunctionLogGroupToKinesisSubscriptionFilterRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Statement: + - Action: sts:AssumeRole + Effect: Allow + Principal: + Service: !Sub logs.${AWS::Region}.amazonaws.com + Version: "2012-10-17" + Policies: + - PolicyName: AnalyticsFrequencySubscriptionKinesisPolicy + PolicyDocument: + Statement: + - Effect: Allow + Action: + - kinesis:DescribeStream + - kinesis:PutRecord + - kinesis:PutRecords + Resource: !Ref DovetailVerifiedMetricsKinesisStreamArn + Version: "2012-10-17" + Tags: + - { Key: prx:meta:tagging-version, Value: "2021-04-07" } + - { Key: prx:cloudformation:stack-name, Value: !Ref AWS::StackName } + - { Key: prx:cloudformation:stack-id, Value: !Ref AWS::StackId } + - { Key: prx:cloudformation:root-stack-name, Value: !Ref RootStackName } + - { Key: prx:cloudformation:root-stack-id, Value: !Ref RootStackId } + - { Key: prx:ops:environment, Value: !Ref EnvironmentType } + - { Key: prx:dev:family, Value: Dovetail } + - { Key: prx:dev:application, Value: Analytics } + AnalyticsFrequencyFunctionLogGroupImpressionsToKinesisSubscriptionFilter: + # Send impression data from Frequency Lambda function's logs to Kinesis + Type: AWS::Logs::SubscriptionFilter + Properties: + DestinationArn: !Ref DovetailVerifiedMetricsKinesisStreamArn + FilterPattern: "{$.msg = impression}" + LogGroupName: !Ref AnalyticsFrequencyFunctionLogGroup + RoleArn: !GetAtt AnalyticsFrequencyFunctionLogGroupToKinesisSubscriptionFilterRole.Arn + + AnalyticsFrequencyFunctionKinesisIteratorBehindAlarm: + Type: AWS::CloudWatch::Alarm + Properties: + AlarmName: !Sub WARN [Dovetail-Analytics] Frequency Lambda function <${EnvironmentTypeAbbreviation}> KINESIS ITERATOR FALLING BEHIND (${RootStackName}) + AlarmDescription: !Sub >- + ${EnvironmentType} Dovetail Analytics Frequency Lambda function's + Kinesis iterator age is higher than normal. + ComparisonOperator: GreaterThanThreshold + Dimensions: + - Name: FunctionName + Value: !Ref AnalyticsFrequencyFunction + EvaluationPeriods: 1 + MetricName: IteratorAge + Namespace: AWS/Lambda + Period: 60 + Statistic: Maximum + Tags: + - { Key: prx:meta:tagging-version, Value: "2021-04-07" } + - { Key: prx:cloudformation:stack-name, Value: !Ref AWS::StackName } + - { Key: prx:cloudformation:stack-id, Value: !Ref AWS::StackId } + - { Key: prx:cloudformation:root-stack-name, Value: !Ref RootStackName } + - { Key: prx:cloudformation:root-stack-id, Value: !Ref RootStackId } + - { Key: prx:ops:environment, Value: !Ref EnvironmentType } + - { Key: prx:dev:family, Value: Dovetail } + - { Key: prx:dev:application, Value: Analytics } + Threshold: 900000 # milliseconds + TreatMissingData: missing + Unit: Milliseconds + AnalyticsFrequencyFunctionKinesisIteratorStalledAlarm: + Type: AWS::CloudWatch::Alarm + Properties: + AlarmName: !Sub FATAL [Dovetail-Analytics] Frequency Lambda function <${EnvironmentTypeAbbreviation}> KINESIS ITERATOR STALLED (${RootStackName}) + AlarmDescription: !Sub >- + ${EnvironmentType} Dovetail Analytics Frequency Lambda function's + Kinesis iterator is significantly delayed, and is likely to continue to + fall behind without intervention. + ComparisonOperator: GreaterThanThreshold + Dimensions: + - Name: FunctionName + Value: !Ref AnalyticsFrequencyFunction + EvaluationPeriods: 1 + MetricName: IteratorAge + Namespace: AWS/Lambda + Period: 60 + Statistic: Maximum + Tags: + - { Key: prx:meta:tagging-version, Value: "2021-04-07" } + - { Key: prx:cloudformation:stack-name, Value: !Ref AWS::StackName } + - { Key: prx:cloudformation:stack-id, Value: !Ref AWS::StackId } + - { Key: prx:cloudformation:root-stack-name, Value: !Ref RootStackName } + - { Key: prx:cloudformation:root-stack-id, Value: !Ref RootStackId } + - { Key: prx:ops:environment, Value: !Ref EnvironmentType } + - { Key: prx:dev:family, Value: Dovetail } + - { Key: prx:dev:application, Value: Analytics } + Threshold: 3600000 # milliseconds + TreatMissingData: missing + Unit: Milliseconds + + AnalyticsFrequencyFunctionErrorLevelLogMetricFilter: + # Counts the number of logged errors + Type: AWS::Logs::MetricFilter + Properties: + FilterPattern: '{ $._logLevel = "error" }' + LogGroupName: !Ref AnalyticsFrequencyFunctionLogGroup + MetricTransformations: + - MetricName: !Sub frequency_errors_${AnalyticsFrequencyFunction} + MetricNamespace: !Ref kMetricFilterNamespace + MetricValue: "1" + AnalyticsFrequencyFunctionLoggedErrorsAlarm: + Type: AWS::CloudWatch::Alarm + Properties: + AlarmName: !Sub ERROR [Dovetail-Analytics] Frequency Lambda function <${EnvironmentTypeAbbreviation}> LOGGED ERRORS (${RootStackName}) + AlarmDescription: !Sub >- + ${EnvironmentType} Dovetail Analytics Frequency Lambda function has + logged some errors during execution. + ComparisonOperator: GreaterThanThreshold + EvaluationPeriods: 2 + MetricName: !Sub frequency_errors_${AnalyticsFrequencyFunction} + Namespace: !Ref kMetricFilterNamespace + Period: 60 + Statistic: Sum + Tags: + - { Key: prx:meta:tagging-version, Value: "2021-04-07" } + - { Key: prx:cloudformation:stack-name, Value: !Ref AWS::StackName } + - { Key: prx:cloudformation:stack-id, Value: !Ref AWS::StackId } + - { Key: prx:cloudformation:root-stack-name, Value: !Ref RootStackName } + - { Key: prx:cloudformation:root-stack-id, Value: !Ref RootStackId } + - { Key: prx:ops:environment, Value: !Ref EnvironmentType } + - { Key: prx:ops:cloudwatch-log-group-name, Value: !Ref AnalyticsFrequencyFunctionLogGroup } + - { Key: prx:dev:family, Value: Dovetail } + - { Key: prx:dev:application, Value: Analytics } + Threshold: 0 + TreatMissingData: notBreaching + + AnalyticsFrequencyFunctionInsertsMetricFilter: + # Counts the number of rows inserted to DynamoDB + Type: AWS::Logs::MetricFilter + Properties: + FilterPattern: '{ $.dest = "dynamodb" }' + LogGroupName: !Ref AnalyticsFrequencyFunctionLogGroup + MetricTransformations: + - MetricName: !Sub frequency_inserts_${AnalyticsFrequencyFunction} + MetricNamespace: !Ref kMetricFilterNamespace + MetricValue: $.rows + + AnalyticsFrequencyFunctionLookupsMetricFilter: + # Count the number of redirect-datas we've looked up and shuffeld along to + # the metrics-kinesis stream + Type: AWS::Logs::MetricFilter + Properties: + FilterPattern: '{ $.dest = "kinesis*" }' + LogGroupName: !Ref AnalyticsFrequencyFunctionLogGroup + MetricTransformations: + - MetricName: !Sub frequency_lookups_${AnalyticsFrequencyFunction} + MetricNamespace: !Ref kMetricFilterNamespace + MetricValue: $.rows + + AnalyticsFrequencyFunctionRetriesMetricFilter: + # Counts the number of retried DynamoDB operations + Type: AWS::Logs::MetricFilter + Properties: + FilterPattern: '{ $.ddb = "retrying" }' + LogGroupName: !Ref AnalyticsFrequencyFunctionLogGroup + MetricTransformations: + - MetricName: !Sub frequency_retries_${AnalyticsFrequencyFunction} + MetricNamespace: !Ref kMetricFilterNamespace + MetricValue: "1" + # Pingbacks AnalyticsPingbacksFunction: Type: AWS::Serverless::Function @@ -942,6 +1217,7 @@ Resources: [ "${kMetricFilterNamespace}", "bigquery_downloads_${AnalyticsBigqueryFunction}", { "label": "BigQuery Download Records", "color": "#1f77b4" } ], [ "${kMetricFilterNamespace}", "bigquery_impressions_${AnalyticsBigqueryFunction}", { "label": "BigQuery Impression Records", "color": "#ff7f0e" } ], [ "${kMetricFilterNamespace}", "pingbacks_other_${AnalyticsPingbacksFunction}", { "label": "Pingbacks", "color": "#d62728" } ] + [ "${kMetricFilterNamespace}", "frequency_inserts_${AnalyticsFrequencyFunction}", { "label": "Frequency", "color": "#ff9896" } ] ], "view": "timeSeries", "stacked": false, @@ -967,6 +1243,7 @@ Resources: [ "AWS/Lambda", "IteratorAge", "FunctionName", "${AnalyticsBigqueryFunction}", { "label": "BigQuery Lambda" } ], [ "AWS/Lambda", "IteratorAge", "FunctionName", "${AnalyticsDynamoDbFunction}", { "label": "DynamoDB Lambda" } ], [ "AWS/Lambda", "IteratorAge", "FunctionName", "${AnalyticsPingbacksFunction}", { "label": "Pingbacks Lambda" } ] + [ "AWS/Lambda", "IteratorAge", "FunctionName", "${AnalyticsFrequencyFunction}", { "label": "Frequency Lambda" } ] ], "view": "timeSeries", "stacked": false, @@ -1084,6 +1361,46 @@ Resources: } }, + { + "height": 4, + "width": 6, + "y": 4, + "x": 12, + "type": "metric", + "properties": { + "metrics": [ + [ "AWS/Lambda", "Invocations", "FunctionName", "${AnalyticsFrequencyFunction}", { "label": "Invocations" } ], + [ "AWS/Lambda", "Errors", "FunctionName", "${AnalyticsFrequencyFunction}", { "label": "[max: ${!MAX}] Errors", "yAxis": "right" } ] + ], + "view": "timeSeries", + "stacked": false, + "region": "${AWS::Region}", + "title": "DDB Health", + "period": 60, + "liveData": true, + "stat": "Sum" + } + }, + { + "height": 4, + "width": 6, + "y": 4, + "x": 18, + "type": "metric", + "properties": { + "metrics": [ + [ "AWS/Lambda", "Duration", "FunctionName", "${AnalyticsFrequencyFunction}", { "label": "Average", "stat": "Average" } ], + [ "AWS/Lambda", "Duration", "FunctionName", "${AnalyticsFrequencyFunction}", { "label": "Max", "stat": "Maximum" } ] + ], + "view": "timeSeries", + "stacked": false, + "region": "${AWS::Region}", + "title": "DDB Duration", + "period": 60, + "liveData": true + } + }, + { "height": 4, "width": 6, diff --git a/spire/templates/apps/dovetail-router.yml b/spire/templates/apps/dovetail-router.yml index 814b1451..2b7c47e1 100644 --- a/spire/templates/apps/dovetail-router.yml +++ b/spire/templates/apps/dovetail-router.yml @@ -94,6 +94,8 @@ Parameters: DovetailCdnHostname: { Type: String } DovetailRouterHostname: { Type: String } DovetailCdnRedirectPrefix: { Type: AWS::SSM::Parameter::Value } + FrequencyDynamodbTableName: { Type: AWS::SSM::Parameter::Value } + FrequencyDynamodbAccessRoleArn: { Type: AWS::SSM::Parameter::Value } Conditions: IsProduction: !Equals [!Ref EnvironmentType, Production] @@ -740,6 +742,28 @@ Resources: Principal: Service: ecs-tasks.amazonaws.com Version: "2012-10-17" + Policies: + - PolicyDocument: + Statement: + - Action: + - dynamodb:BatchGetItem + - dynamodb:ConditionCheck + - dynamodb:DescribeTable + - dynamodb:DescribeTimeToLive + - dynamodb:GetItem + - dynamodb:Query + Effect: Allow + Resource: + - !Sub "arn:aws:dynamodb:*:*:table/${FrequencyDynamodbTableName}" + Version: "2012-10-17" + PolicyName: FrequencyDdbActions + - PolicyDocument: + Statement: + - Action: sts:AssumeRole + Effect: Allow + Resource: !Ref FrequencyDynamodbAccessRoleArn + Version: "2012-10-17" + PolicyName: FrequencyDdbAssumeRole Tags: - { Key: prx:meta:tagging-version, Value: "2021-04-07" } - { Key: prx:cloudformation:stack-name, Value: !Ref AWS::StackName } @@ -801,6 +825,10 @@ Resources: Value: !Ref NewRelicApiKeyPrxLite - Name: AGENTS_URL Value: https://raw.githubusercontent.com/PRX/prx-podagent/main/db/agents.lock.json + - Name: DDB_FREQUENCY_TABLE + Value: !Ref FrequencyDynamodbTableName + - Name: DDB_ROLE + Value: !Ref FrequencyDynamodbAccessRoleArn Essential: true Image: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${EcrImageTag} LogConfiguration: