-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathazure-pipelines.yml
153 lines (134 loc) · 6.3 KB
/
azure-pipelines.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
# Starter pipeline
# Start with a minimal pipeline that you can customize to build and deploy your code.
# Add steps that build, run tests, deploy, and more:
# https://aka.ms/yaml
trigger:
- main
pool:
vmImage: ubuntu-latest
variables:
- name: workspace
value: mlopsdemows
- name: resourceGroup
value: demo
- name: storageAccount
value: mlopsdemodatalake
- name: storageContainer
value: mnist-dataset
- name: keyVault
value: mlopsdemows3775522951
- name: containerRegistry
value: mlopsdemosws8765456765
jobs:
- job: Train
steps:
- task: AzureKeyVault@2
displayName: Retrieve Secrets from Key Vault
inputs:
azureSubscription: access_subscription_max
KeyVaultName: $(keyVault)
SecretsFilter: storage-key
RunAsPreJob: true
- task: AzureCLI@2
displayName: Install Azure CLI ML extension
inputs:
azureSubscription: access_subscription_max
scriptLocation: inlineScript
scriptType: bash
inlineScript: |
az extension add --name ml --upgrade
az configure --defaults group=$(resourceGroup) workspace=$(workspace)
- task: PowerShell@2
inputs:
targetType: inline
script: |
Install-Module -Name Az.Storage -Scope CurrentUser -Force
displayName: Install PowerShell Modules
- bash: |
echo "##vso[task.setvariable variable=runDataPath;]$(date +%Y)/$(date +%b)/Day$(date +%d)/$(date +%s)"
displayName: Generate unique Data Folder
- task: PowerShell@2
inputs:
targetType: inline
script: |
$Context = New-AzStorageContext -StorageAccountName $(storageAccount) -StorageAccountKey $(storage-key)
Start-AzStorageBlobCopy -AbsoluteUri "https://mlopsdemodatalake.blob.core.windows.net/mnist-dataset/mnist_train.csv" -DestContainer $(storageContainer) -DestBlob "/$(runDataPath)/mnist_train.csv" -Context $context -DestContext $context
Start-AzStorageBlobCopy -AbsoluteUri "https://mlopsdemodatalake.blob.core.windows.net/mnist-dataset/mnist_test.csv" -DestContainer $(storageContainer) -DestBlob "/$(runDataPath)/mnist_test.csv" -Context $context -DestContext $context
displayName: Copy Blobs to run folder
- task: AzureCLI@2
displayName: Copy Blobs to run folder
inputs:
azureSubscription: access_subscription_max
scriptType: bash
scriptLocation: inlineScript
inlineScript: |
az storage blob copy start --account-name $(storageAccount) --account-key $(storage-key) --destination-blob $(runDataPath)/mnist_train.csv --destination-container $(storageContainer) --source-uri https://mlopsdemodatalake.blob.core.windows.net/mnist-dataset/mnist_train.csv --requires-sync true
az storage blob copy start --account-name $(storageAccount) --account-key $(storage-key) --destination-blob $(runDataPath)/mnist_test.csv --destination-container $(storageContainer) --source-uri https://mlopsdemodatalake.blob.core.windows.net/mnist-dataset/mnist_test.csv --requires-sync true
- task: AzureCLI@2
displayName: Create Compute Cluster
inputs:
azureSubscription: access_subscription_max
scriptType: bash
scriptLocation: inlineScript
inlineScript: az ml compute create --file pipeline/compute/demo-cpu-cluster.yml
- task: AzureCLI@2
displayName: Create Virtual Environment
inputs:
azureSubscription: access_subscription_max
scriptType: bash
scriptLocation: inlineScript
inlineScript: az ml environment create --file pipeline/environment/myenv.yml
- task: AzureCLI@2
displayName: Create Datastore
inputs:
azureSubscription: access_subscription_max
scriptType: bash
scriptLocation: inlineScript
inlineScript: az ml datastore create --name demo_blob_storage --file pipeline/datastore/demo-blob-storage.yml --set credentials.account_key=$(storage-key)
- task: AzureCLI@2
displayName: Create Datasets
inputs:
azureSubscription: access_subscription_max
scriptType: bash
scriptLocation: inlineScript
inlineScript: |
az ml dataset create --name demo_train_set_mnist --paths file:azureml://datastores/demo_blob_storage/paths/$(runDataPath)/mnist_train.csv
az ml dataset create --name demo_test_set_mnist --paths file:azureml://datastores/demo_blob_storage/paths/$(runDataPath)/mnist_test.csv
- bash: |
echo "##vso[task.setvariable variable=runID;]demo-sklearn-train-$(date +%s)"
displayName: Generate unique Run ID
- task: AzureCLI@2
displayName: Run Training Job
inputs:
azureSubscription: access_subscription_max
scriptType: bash
scriptLocation: scriptPath
scriptPath: pipeline/jobs/run_train.sh
arguments: $(runId) azureml://datastores/demo_blob_storage/paths/$(runDataPath)/
- task: AzureCLI@2
displayName: Create Model
inputs:
azureSubscription: access_subscription_max
scriptType: bash
scriptLocation: inlineScript
inlineScript: az ml model create --file pipeline/model/demo-sklearn-mnist.yml --path runs:/$(runID)/outputs
- task: AzureCLI@2
displayName: Link Datasets to Model
inputs:
azureSubscription: access_subscription_max
scriptType: bash
scriptLocation: inlineScript
inlineScript: |
az ml job create --name $(runId)-link --file pipeline/jobs/link.yml
az ml job stream --name $(runId)-link
- task: AzureCLI@2
displayName: Create ACR Container
inputs:
azureSubscription: access_subscription_max
scriptType: bash
scriptLocation: inlineScript
inlineScript: |
VERSION=$(az ml model list --name demo-sklearn-mnist --resource-group $(resourceGroup) --workspace-name $(workspace) --query "reverse(sort_by([].{version:version}, &version)) | [0]" --output tsv)
az ml model download --name demo-sklearn-mnist --resource-group $(resourceGroup) --workspace-name $(workspace) --version $VERSION --download-path pipeline/container
az acr login --name $(containerRegistry)
az acr build --image demo-sklearn-mnist:$VERSION --registry $(containerRegistry) --file pipeline/container/Dockerfile .