Multi-stage pipeline + scoring observability

.pipelines/azdo-base-pipeline.yml

@@ -10,7 +10,7 @@ steps:
   enabled: 'true'
 
 - script: |
-   pip install --user -r $(Build.SourcesDirectory)/tests/requirements.txt
+   pip install --user -r $(Build.SourcesDirectory)/tests/requirements.txt && \
    pytest --junitxml=$(Build.BinariesDirectory)/unit-testresults.xml $(Build.SourcesDirectory)/tests/unit
   displayName: 'Run unit tests'
   enabled: 'true'

.pipelines/azdo-ci-build-train.yml

@@ -7,7 +7,8 @@ trigger:
     exclude:
     - docs/
     - environment_setup/
-    - ml_service/util/create_scoring_image.*
+    - ml_service/util/create_scoring_image.py
+    - ml_service/util/smoke_test_scoring_service.py
 
 variables:
 - template: azdo-variables.yml
@@ -36,8 +37,9 @@ stages:
           # Invoke the Python building and publishing a training pipeline
           python $(Build.SourcesDirectory)/ml_service/pipelines/${{ variables.BUILD_TRAIN_SCRIPT }}
       displayName: 'Publish Azure Machine Learning Pipeline'
+
 - stage: 'Trigger_AML_Pipeline'
-  displayName: 'Train, evaluate, register model via previously published AML pipeline'
+  displayName: 'Train model'
   jobs:
   - job: "Get_Pipeline_ID"
     condition: and(succeeded(), eq(coalesce(variables['auto-trigger-training'], 'true'), 'true'))
@@ -56,7 +58,8 @@ stages:
           export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
           python $(Build.SourcesDirectory)/ml_service/pipelines/run_train_pipeline.py --output_pipeline_id_file "pipeline_id.txt" --skip_train_execution
           # Set AMLPIPELINEID variable for next AML Pipeline task in next job
-          echo "##vso[task.setvariable variable=AMLPIPELINEID;isOutput=true]$(cat pipeline_id.txt)"
+          AMLPIPELINEID="$(cat pipeline_id.txt)"
+          echo "##vso[task.setvariable variable=AMLPIPELINEID;isOutput=true]$AMLPIPELINEID"
       name: 'getpipelineid'
       displayName: 'Get Pipeline ID'
     - bash: |
@@ -83,36 +86,116 @@ stages:
         PipelineId: '$(AMLPIPELINE_ID)'
         ExperimentName: '$(EXPERIMENT_NAME)'
         PipelineParameters: '"ParameterAssignments": {"model_name": "$(MODEL_NAME)", "hyperparameter_alpha": "$(ALPHA)"}'
-  - job: "Training_Run_Report"
-    dependsOn: "Run_ML_Pipeline"
-    displayName: "Determine if evaluation succeeded and new model is registered"
+
+- stage: 'Deploy_ACI'
+  displayName: 'Deploy to ACI'
+  dependsOn: Trigger_AML_Pipeline
+  condition: and(succeeded(), variables['ACI_DEPLOYMENT_NAME'])
+  jobs:
+  - job: "Deploy_ACI"
+    displayName: "Deploy to ACI"
     pool:
       vmImage: 'ubuntu-latest'
     container: mcr.microsoft.com/mlops/python:latest
     timeoutInMinutes: 0
     steps:
+    - template: azdo-template-get-model-version.yml
+    - task: ms-air-aiagility.vss-services-azureml.azureml-model-deploy-task.AMLModelDeploy@0
+      displayName: 'Azure ML Model Deploy'
+      inputs:
+        azureSubscription: $(WORKSPACE_SVC_CONNECTION)
+        modelSourceType: manualSpec
+        modelName: '$(MODEL_NAME)'
+        modelVersion: $(MODEL_VERSION)
+        inferencePath: '$(Build.SourcesDirectory)/code/scoring/inference_config.yml'
+        deploymentTarget: ACI
+        deploymentName: $(ACI_DEPLOYMENT_NAME)
+        deployConfig: '$(Build.SourcesDirectory)/code/scoring/deployment_config_aci.yml'
+        overwriteExistingDeployment: true
     - task: AzureCLI@1
+      displayName: 'Smoke test'
       inputs:
         azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
         scriptLocation: inlineScript
         inlineScript: |
           set -e # fail on error
           export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
-          python $(Build.SourcesDirectory)/ml_service/pipelines/verify_train_pipeline.py --build_id $(Build.BuildId) --model_name "$(MODEL_NAME)"
-          echo "##vso[task.setvariable variable=MODEL_VERSION;isOutput=true]$(cat model_version.txt)"
-      displayName: "Determine if evaluation succeeded and new model is registered"
-    - task: CopyFiles@2
-      displayName: 'Copy Files to: $(Build.ArtifactStagingDirectory)'
+          python ml_service/util/smoke_test_scoring_service.py --type ACI --service "$(ACI_DEPLOYMENT_NAME)"
+
+- stage: 'Deploy_AKS'
+  displayName: 'Deploy to AKS'
+  dependsOn: Deploy_ACI
+  condition: and(succeeded(), variables['AKS_DEPLOYMENT_NAME'])
+  jobs:
+  - job: "Deploy_AKS"
+    displayName: "Deploy to AKS"
+    pool:
+      vmImage: 'ubuntu-latest'
+    container: mcr.microsoft.com/mlops/python:latest
+    timeoutInMinutes: 0
+    steps:
+    - template: azdo-template-get-model-version.yml
+    - task: ms-air-aiagility.vss-services-azureml.azureml-model-deploy-task.AMLModelDeploy@0
+      displayName: 'Azure ML Model Deploy'
       inputs:
-        SourceFolder: '$(Build.SourcesDirectory)'
-        TargetFolder: '$(Build.ArtifactStagingDirectory)'
-        Contents: |
-          code/scoring/**
-          ml_service/util/**
-    - task: PublishBuildArtifacts@1
-      displayName: 'Publish Artifact'
+        azureSubscription: $(WORKSPACE_SVC_CONNECTION)
+        modelSourceType: manualSpec
+        modelName: '$(MODEL_NAME)'
+        modelVersion: $(MODEL_VERSION)
+        inferencePath: '$(Build.SourcesDirectory)/code/scoring/inference_config.yml'
+        deploymentTarget: AKS
+        aksCluster: $(AKS_COMPUTE_NAME)
+        deploymentName: $(AKS_DEPLOYMENT_NAME)
+        deployConfig: '$(Build.SourcesDirectory)/code/scoring/deployment_config_aks.yml'
+        overwriteExistingDeployment: true
+    - task: AzureCLI@1
+      displayName: 'Smoke test'
       inputs:
-        ArtifactName: 'mlops-pipelines'
-        publishLocation: 'container'
-        pathtoPublish: '$(Build.ArtifactStagingDirectory)' 
-        TargetPath: '$(Build.ArtifactStagingDirectory)'
+        azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
+        scriptLocation: inlineScript
+        inlineScript: |
+          set -e # fail on error
+          export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+          python ml_service/util/smoke_test_scoring_service.py --type AKS --service "$(AKS_DEPLOYMENT_NAME)"
+
+- stage: 'Deploy_Webapp'
+  displayName: 'Deploy to Webapp'
+  dependsOn: Trigger_AML_Pipeline
+  condition: and(succeeded(), variables['WEBAPP_DEPLOYMENT_NAME'])
+  jobs:
+  - job: "Deploy_Webapp"
+    displayName: "Deploy to Webapp"
+    pool:
+      vmImage: 'ubuntu-latest'
+    container: mcr.microsoft.com/mlops/python:latest
+    timeoutInMinutes: 0
+    steps:
+    - template: azdo-template-get-model-version.yml
+    - task: AzureCLI@1
+      displayName: 'Create scoring image and set IMAGE_LOCATION variable'
+      inputs:
+        azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
+        scriptLocation: inlineScript
+        inlineScript: |
+          set -e # fail on error
+          export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+          python ml_service/util/create_scoring_image.py --output_image_location_file image_location.txt
+          # Output image location to Azure DevOps job
+          IMAGE_LOCATION="$(cat image_location.txt)"
+          echo "##vso[task.setvariable variable=IMAGE_LOCATION]$IMAGE_LOCATION"
+    - task: AzureWebAppContainer@1
+      name: WebAppDeploy
+      displayName: 'Azure Web App on Container Deploy'
+      inputs:
+        azureSubscription: 'AzureResourceConnection'
+        appName: '$(WEBAPP_DEPLOYMENT_NAME)'
+        containers: '$(IMAGE_LOCATION)'
+    - task: AzureCLI@1
+      displayName: 'Smoke test'
+      inputs:
+        azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
+        scriptLocation: inlineScript
+        inlineScript: |
+          set -e # fail on error
+          export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+          python ml_service/util/smoke_test_scoring_service.py --type Webapp --service "$(WebAppDeploy.AppServiceApplicationUrl)/score"

.pipelines/azdo-template-get-model-version.yml

@@ -0,0 +1,12 @@
+steps:
+- task: AzureCLI@1
+  inputs:
+    azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
+    scriptLocation: inlineScript
+    inlineScript: |
+      set -e # fail on error
+      export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+      python $(Build.SourcesDirectory)/ml_service/pipelines/verify_train_pipeline.py --build_id $(Build.BuildId) --model_name "$(MODEL_NAME)" --output_model_version_file "model_version.txt"
+      echo "##vso[task.setvariable variable=MODEL_VERSION]$(cat model_version.txt)"
+  name: 'getversion'
+  displayName: "Determine if evaluation succeeded and new model is registered"

code/scoring/score.py

@@ -38,16 +38,33 @@ def init():
     model = joblib.load(model_path)
 
 
-def run(raw_data):
+def run(raw_data, request_headers):
     data = json.loads(raw_data)["data"]
     data = numpy.array(data)
     result = model.predict(data)
+
+    # Demonstrate how we can log custom data into the Application Insights
+    # traces collection.
+    # The 'X-Ms-Request-id' value is generated internally and can be used to
+    # correlate a log entry with the Application Insights requests collection.
+    # The HTTP 'traceparent' header may be set by the caller to implement
+    # distributed tracing (per the W3C Trace Context proposed specification)
+    # and can be used to correlate the request to external systems.
+    print(('{{"RequestId":"{0}", '
+           '"TraceParent":"{1}", '
+           '"NumberOfPredictions":{2}}}'
+           ).format(
+               request_headers.get("X-Ms-Request-Id", ""),
+               request_headers.get("Traceparent", ""),
+               len(result)
+           ))
+
     return {"result": result.tolist()}
 
 
 if __name__ == "__main__":
     # Test scoring
     init()
     test_row = '{"data":[[1,2,3,4,5,6,7,8,9,10],[10,9,8,7,6,5,4,3,2,1]]}'
-    prediction = run(test_row)
+    prediction = run(test_row, {})
     print("Test result: ", prediction)