diff --git a/README.md b/README.md index 4f52ad3d..f361f27c 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ # DevOps For AI +[![Build Status](https://teamdanielletrialcicd.visualstudio.com/MLAKSDeployAML/_apis/build/status/DevopsforAI-AML?branchName=master)](https://teamdanielletrialcicd.visualstudio.com/MLAKSDeployAML/_build/latest?definitionId=9&branchName=buildpipeline) + [DevOps for AI template](https://azuredevopsdemogenerator.azurewebsites.net/?name=azure%20machine%20learning) will help you to understand how to build the Continuous Integration and Continuous Delivery pipeline for a ML/AI project. We will be using the Azure DevOps Project for build and release pipelines along with Azure ML services for ML/AI model management and operationalization. This template contains code and pipeline definition for a machine learning project demonstrating how to automate the end to end ML/AI project. The build pipelines include DevOps tasks for data sanity test, unit test, model training on different compute targets, model version management, model evaluation/model selection, model deployment as realtime web service, staged deployment to QA/prod, integration testing and functional testing. diff --git a/aml_service/00-WorkSpace.py b/aml_service/00-WorkSpace.py index 89716a4e..413a112f 100644 --- a/aml_service/00-WorkSpace.py +++ b/aml_service/00-WorkSpace.py @@ -24,8 +24,9 @@ POSSIBILITY OF SUCH DAMAGE. """ from azureml.core import Workspace -import os, json +import os, json, sys import azureml.core +from azureml.core.authentication import AzureCliAuthentication print("SDK Version:", azureml.core.VERSION) # print('current dir is ' +os.curdir) @@ -36,12 +37,15 @@ resource_group = config["resource_group"] subscription_id = config["subscription_id"] location = config["location"] -# location = 'southcentralus' + +cli_auth = AzureCliAuthentication() + try: ws = Workspace.get( name=workspace_name, subscription_id=subscription_id, resource_group=resource_group, + auth=cli_auth ) except: @@ -53,6 +57,8 @@ resource_group=resource_group, # create_resource_group=True, location=location, + auth=cli_auth + ) # print Workspace details diff --git a/aml_service/10-TrainOnLocal.py b/aml_service/10-TrainOnLocal.py index 807d1843..d303aab9 100644 --- a/aml_service/10-TrainOnLocal.py +++ b/aml_service/10-TrainOnLocal.py @@ -29,9 +29,11 @@ from azureml.core import Experiment from azureml.core import ScriptRunConfig import json +from azureml.core.authentication import AzureCliAuthentication +cli_auth = AzureCliAuthentication() # Get workspace -ws = Workspace.from_config() +ws = Workspace.from_config(auth=cli_auth) # Attach Experiment experiment_name = "devops-ai-demo" diff --git a/aml_service/15-EvaluateModel.py b/aml_service/15-EvaluateModel.py index 2dd9d99d..2aad743d 100644 --- a/aml_service/15-EvaluateModel.py +++ b/aml_service/15-EvaluateModel.py @@ -29,10 +29,11 @@ from azureml.core.model import Model import azureml.core from azureml.core import Run - +from azureml.core.authentication import AzureCliAuthentication +cli_auth = AzureCliAuthentication() # Get workspace -ws = Workspace.from_config() +ws = Workspace.from_config(auth=cli_auth) # Paramaterize the matrics on which the models should be compared diff --git a/aml_service/20-RegisterModel.py b/aml_service/20-RegisterModel.py index 47870ae0..efaac896 100644 --- a/aml_service/20-RegisterModel.py +++ b/aml_service/20-RegisterModel.py @@ -30,9 +30,11 @@ from azureml.core.model import Model from azureml.core.runconfig import RunConfiguration +from azureml.core.authentication import AzureCliAuthentication +cli_auth = AzureCliAuthentication() # Get workspace -ws = Workspace.from_config() +ws = Workspace.from_config(auth=cli_auth) # Get the latest evaluation result try: diff --git a/aml_service/30-CreateScoringImage.py b/aml_service/30-CreateScoringImage.py index c52b3de1..f0426442 100644 --- a/aml_service/30-CreateScoringImage.py +++ b/aml_service/30-CreateScoringImage.py @@ -27,9 +27,11 @@ from azureml.core import Workspace from azureml.core.image import ContainerImage, Image from azureml.core.model import Model +from azureml.core.authentication import AzureCliAuthentication +cli_auth = AzureCliAuthentication() # Get workspace -ws = Workspace.from_config() +ws = Workspace.from_config(auth=cli_auth) # Get the latest model details diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 00000000..1fdcd167 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,68 @@ +pool: + vmImage: 'Ubuntu 16.04' +#Your build pipeline references a secret variable named ‘sp_username’. Create or edit the build pipeline for this YAML file, define the variable on the Variables tab, and then select the option to make it secret. See https://go.microsoft.com/fwlink/?linkid=865972 +#Your build pipeline references a secret variable named ‘sp_password’. Create or edit the build pipeline for this YAML file, define the variable on the Variables tab, and then select the option to make it secret. See https://go.microsoft.com/fwlink/?linkid=865972 +#Your build pipeline references a secret variable named ‘sp_tenantid’. Create or edit the build pipeline for this YAML file, define the variable on the Variables tab, and then select the option to make it secret. See https://go.microsoft.com/fwlink/?linkid=865972 +#Your build pipeline references a secret variable named ‘subscription_id’. Create or edit the build pipeline for this YAML file, define the variable on the Variables tab, and then select the option to make it secret. See https://go.microsoft.com/fwlink/?linkid=865972 + +steps: +- task: CondaEnvironment@1 + displayName: 'Create Conda Environment ' + inputs: + createCustomEnvironment: true + environmentName: azuremlsdk + packageSpecs: 'python=3.6' + updateConda: false + createOptions: 'cython numpy' + +- task: Bash@3 + displayName: 'Install Requirements' + inputs: + targetType: filePath + filePath: 'environment_setup/install_requirements.sh' + workingDirectory: 'environment_setup' + +- script: | + az login --service-principal -u $(sp_username) -p $(sp_password) --tenant $(sp_tenantid) + + displayName: 'Login to Azure' + +- script: | + sed -i 's#"subscription_id": "<>"#"subscription_id": "$(subscription_id)"#g' aml_config/config.json + + displayName: 'replace subscription value' + +- script: 'python code/testing/data_test.py data/diabetes.csv && python code/testing/data_test.py data/diabetes_bad_dist.csv && python code/testing/data_test.py data/diabetes_bad_schema.csv && python code/testing/data_test.py data/diabetes_missing_values.csv' + displayName: 'Data Quality Check' + +- script: 'python aml_service/00-WorkSpace.py' + displayName: 'Get or Create workspace copy' + +- script: 'python aml_service/10-TrainOnLocal.py' + displayName: 'Train on Local' + +- script: 'python aml_service/15-EvaluateModel.py' + displayName: 'Evaluate Model' + +- script: 'python aml_service/20-RegisterModel.py' + displayName: 'Register Model' + +- script: 'python aml_service/30-CreateScoringImage.py' + displayName: 'Creating scoring image' + +- task: CopyFiles@2 + displayName: 'Copy Files to: $(Build.ArtifactStagingDirectory)' + inputs: + SourceFolder: '$(Build.SourcesDirectory)' + TargetFolder: '$(Build.ArtifactStagingDirectory)' + Contents: '**' + +- task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: devops-for-ai' + inputs: + ArtifactName: 'devops-for-ai' + publishLocation: 'container' + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + TargetPath: '$(Build.ArtifactStagingDirectory)' + +