forked from great-expectations/great_expectations
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathazure-pipelines-manual-staging-json-to-prod.yml
126 lines (111 loc) · 6.15 KB
/
azure-pipelines-manual-staging-json-to-prod.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
# This file is responsible for configuring the `manual-staging-json-to-prod` pipeline (https://dev.azure.com/great-expectations/great_expectations/_build)
#
# The pipeline is run under the following conditions:
# - Manually triggered
#
# The sole purpose of this pipeline is to copy staging JSON files to prod for the Expectation Gallery and trigger the prod algolia indices.
variables:
isManual: $[eq(variables['Build.Reason'], 'Manual')]
stages:
- stage: deploy_gallery_prod
pool:
vmImage: 'ubuntu-22.04'
jobs:
- job: copy_staging_to_prod
condition: eq(variables.isManual, true)
timeoutInMinutes: 15
variables:
python.version: '3.8'
steps:
- task: UsePythonVersion@0
inputs:
versionSpec: '$(python.version)'
displayName: 'Use Python $(python.version)'
- bash: python -m pip install --upgrade pip==21.3.1 awscli
displayName: 'Update pip and install awscli'
- bash: bash ./copy_s3_staging_to_prod.sh
workingDirectory: $(Build.SourcesDirectory)/assets/scripts/
displayName: 'Copy staging JSON files in S3 to prod'
env:
# AWS credentials
AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)
AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
AWS_DEFAULT_REGION: $(AWS_DEFAULT_REGION)
- task: NodeTool@0
inputs:
versionSpec: '16.16'
- bash: bash ./trigger_algolia.sh
workingDirectory: $(Build.SourcesDirectory)/assets/scripts/
displayName: 'Update Algolia indexes from S3'
env:
# algolia credentials
ALGOLIA_ACCOUNT: $(ALGOLIA_ACCOUNT)
ALGOLIA_EXPECTATION_INDEX: $(ALGOLIA_EXPECTATION_INDEX)
ALGOLIA_PACKAGE_EXPEC_INDEX: $(ALGOLIA_PACKAGE_EXPEC_INDEX)
ALGOLIA_PACKAGE_INDEX: $(ALGOLIA_PACKAGE_INDEX)
ALGOLIA_WRITE_KEY: $(ALGOLIA_WRITE_KEY)
# replica indices from expectations for sorting
ALGOLIA_EXPEC_REPLICA_ALPHA_ASC_INDEX: $(ALGOLIA_EXPEC_REPLICA_ALPHA_ASC_INDEX)
ALGOLIA_EXPEC_REPLICA_ALPHA_DSC_INDEX: $(ALGOLIA_EXPEC_REPLICA_ALPHA_DSC_INDEX)
ALGOLIA_EXPEC_REPLICA_COVERAGE_ASC_INDEX: $(ALGOLIA_EXPEC_REPLICA_COVERAGE_ASC_INDEX)
ALGOLIA_EXPEC_REPLICA_COVERAGE_DSC_INDEX: $(ALGOLIA_EXPEC_REPLICA_COVERAGE_DSC_INDEX)
ALGOLIA_EXPEC_REPLICA_CREATED_ASC_INDEX: $(ALGOLIA_EXPEC_REPLICA_CREATED_ASC_INDEX)
ALGOLIA_EXPEC_REPLICA_CREATED_DSC_INDEX: $(ALGOLIA_EXPEC_REPLICA_CREATED_DSC_INDEX)
ALGOLIA_EXPEC_REPLICA_UPDATED_ASC_INDEX: $(ALGOLIA_EXPEC_REPLICA_UPDATED_ASC_INDEX)
ALGOLIA_EXPEC_REPLICA_UPDATED_DSC_INDEX: $(ALGOLIA_EXPEC_REPLICA_UPDATED_DSC_INDEX)
# replica indices from package expectations for sorting
ALGOLIA_PACK_EXPEC_REPLICA_ALPHA_ASC_INDEX: $(ALGOLIA_PACK_EXPEC_REPLICA_ALPHA_ASC_INDEX)
ALGOLIA_PACK_EXPEC_REPLICA_ALPHA_DSC_INDEX: $(ALGOLIA_PACK_EXPEC_REPLICA_ALPHA_DSC_INDEX)
ALGOLIA_PACK_EXPEC_REPLICA_COVERAGE_ASC_INDEX: $(ALGOLIA_PACK_EXPEC_REPLICA_COVERAGE_ASC_INDEX)
ALGOLIA_PACK_EXPEC_REPLICA_COVERAGE_DSC_INDEX: $(ALGOLIA_PACK_EXPEC_REPLICA_COVERAGE_DSC_INDEX)
- stage: revert_gallery_prod
pool:
vmImage: 'ubuntu-22.04'
jobs:
- job: copy_old_prod_to_prod
condition: eq(variables.isManual, true)
timeoutInMinutes: 15
variables:
python.version: '3.8'
steps:
- task: UsePythonVersion@0
inputs:
versionSpec: '$(python.version)'
displayName: 'Use Python $(python.version)'
- bash: python -m pip install --upgrade pip==21.3.1 awscli
displayName: 'Update pip and install awscli'
- bash: bash ./copy_s3_old_prod_to_prod.sh
workingDirectory: $(Build.SourcesDirectory)/assets/scripts/
displayName: 'Copy old prod JSON files in S3 to prod'
env:
# AWS credentials
AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)
AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
AWS_DEFAULT_REGION: $(AWS_DEFAULT_REGION)
- task: NodeTool@0
inputs:
versionSpec: '16.16'
- bash: bash ./trigger_algolia.sh
workingDirectory: $(Build.SourcesDirectory)/assets/scripts/
displayName: 'Update Algolia indexes from S3'
env:
# algolia credentials
ALGOLIA_ACCOUNT: $(ALGOLIA_ACCOUNT)
ALGOLIA_EXPECTATION_INDEX: $(ALGOLIA_EXPECTATION_INDEX)
ALGOLIA_PACKAGE_EXPEC_INDEX: $(ALGOLIA_PACKAGE_EXPEC_INDEX)
ALGOLIA_PACKAGE_INDEX: $(ALGOLIA_PACKAGE_INDEX)
ALGOLIA_WRITE_KEY: $(ALGOLIA_WRITE_KEY)
# replica indices from expectations for sorting
ALGOLIA_EXPEC_REPLICA_ALPHA_ASC_INDEX: $(ALGOLIA_EXPEC_REPLICA_ALPHA_ASC_INDEX)
ALGOLIA_EXPEC_REPLICA_ALPHA_DSC_INDEX: $(ALGOLIA_EXPEC_REPLICA_ALPHA_DSC_INDEX)
ALGOLIA_EXPEC_REPLICA_COVERAGE_ASC_INDEX: $(ALGOLIA_EXPEC_REPLICA_COVERAGE_ASC_INDEX)
ALGOLIA_EXPEC_REPLICA_COVERAGE_DSC_INDEX: $(ALGOLIA_EXPEC_REPLICA_COVERAGE_DSC_INDEX)
ALGOLIA_EXPEC_REPLICA_CREATED_ASC_INDEX: $(ALGOLIA_EXPEC_REPLICA_CREATED_ASC_INDEX)
ALGOLIA_EXPEC_REPLICA_CREATED_DSC_INDEX: $(ALGOLIA_EXPEC_REPLICA_CREATED_DSC_INDEX)
ALGOLIA_EXPEC_REPLICA_UPDATED_ASC_INDEX: $(ALGOLIA_EXPEC_REPLICA_UPDATED_ASC_INDEX)
ALGOLIA_EXPEC_REPLICA_UPDATED_DSC_INDEX: $(ALGOLIA_EXPEC_REPLICA_UPDATED_DSC_INDEX)
# replica indices from package expectations for sorting
ALGOLIA_PACK_EXPEC_REPLICA_ALPHA_ASC_INDEX: $(ALGOLIA_PACK_EXPEC_REPLICA_ALPHA_ASC_INDEX)
ALGOLIA_PACK_EXPEC_REPLICA_ALPHA_DSC_INDEX: $(ALGOLIA_PACK_EXPEC_REPLICA_ALPHA_DSC_INDEX)
ALGOLIA_PACK_EXPEC_REPLICA_COVERAGE_ASC_INDEX: $(ALGOLIA_PACK_EXPEC_REPLICA_COVERAGE_ASC_INDEX)
ALGOLIA_PACK_EXPEC_REPLICA_COVERAGE_DSC_INDEX: $(ALGOLIA_PACK_EXPEC_REPLICA_COVERAGE_DSC_INDEX)