diff --git a/dataeng/jobs/analytics/SnowflakeEventRetirementCleanup.groovy b/dataeng/jobs/analytics/SnowflakeEventRetirementCleanup.groovy new file mode 100644 index 000000000..36612c6f7 --- /dev/null +++ b/dataeng/jobs/analytics/SnowflakeEventRetirementCleanup.groovy @@ -0,0 +1,74 @@ +package analytics + +import static org.edx.jenkins.dsl.AnalyticsConstants.common_authorization +import static org.edx.jenkins.dsl.AnalyticsConstants.common_log_rotator +import static org.edx.jenkins.dsl.AnalyticsConstants.common_publishers +import static org.edx.jenkins.dsl.AnalyticsConstants.secure_scm_parameters +import static org.edx.jenkins.dsl.AnalyticsConstants.secure_scm + + +class SnowflakeEventRetirementCleanup { + public static def job = { dslFactory, allVars -> + dslFactory.job("snowflake-user-retirement-status-cleanup") { + description( + 'Remove any soft-deleted user-retirement-statuses from Snowflake. ' + + 'The presence of a soft-deleted row in this table indicates that the ' + + 'user-retirement-status has been successfully archived and removed from ' + + 'the LMS database.' + ) + logRotator common_log_rotator(allVars) + authorization common_authorization(allVars) + parameters secure_scm_parameters(allVars) + parameters { + stringParam( + 'ANALYTICS_TOOLS_URL', allVars.get('ANALYTICS_TOOLS_URL'), + 'URL for the analytics tools repo.' + ) + stringParam( + 'ANALYTICS_TOOLS_BRANCH', allVars.get('ANALYTICS_TOOLS_BRANCH'), + 'Branch of analytics tools repo to use.' + ) + stringParam( + 'NOTIFY', allVars.get('NOTIFY','$PAGER_NOTIFY'), + 'Space separated list of emails to send notifications to.' + ) + stringParam('PYTHON_VENV_VERSION', 'python3.7', 'Python virtual environment version to used.') + } + environmentVariables { + // Path to the key file used to authenticate to Snowflake + env('KEY_PATH', allVars.get('KEY_PATH')) + // Path to the key-phrase file used to decrypt the key file + env('PASSPHRASE_PATH', allVars.get('PASSPHRASE_PATH')) + env('USER', allVars.get('USER')) + env('ACCOUNT', allVars.get('ACCOUNT')) + } + multiscm secure_scm(allVars) << { + git { + remote { + url('$ANALYTICS_TOOLS_URL') + branch('$ANALYTICS_TOOLS_BRANCH') + credentials('1') + } + extensions { + relativeTargetDirectory('analytics-tools') + pruneBranches() + cleanAfterCheckout() + } + } + } + triggers { + // Run on the fourth day of every month. The user-retirement-archiver job + // runs on the first of every month, so this should allow enough time to + // address any issues with the archival process. + cron('H H 4 * *') + } + wrappers { + timestamps() + } + publishers common_publishers(allVars) + steps { + shell(dslFactory.readFileFromWorkspace('dataeng/resources/snowflake-events-retirement-cleanup.sh')) + } + } + } +} \ No newline at end of file diff --git a/dataeng/resources/snowflake-event-retirement-cleanup.sh b/dataeng/resources/snowflake-event-retirement-cleanup.sh new file mode 100644 index 000000000..7f773e031 --- /dev/null +++ b/dataeng/resources/snowflake-event-retirement-cleanup.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -ex + +# Creating Python virtual env +PYTHON_VENV="python_venv" +virtualenv --python=$PYTHON_VENV_VERSION --clear "${PYTHON_VENV}" +source "${PYTHON_VENV}/bin/activate" + +# Setup +cd $WORKSPACE/analytics-tools/snowflake +make requirements + +python retire_events_cleanup.py \ + --key_path $WORKSPACE/analytics-secure/$KEY_PATH \ + --passphrase_path $WORKSPACE/analytics-secure/$PASSPHRASE_PATH \ + --user $USER \ + --account $ACCOUNT \ No newline at end of file