Enable and fix linting (microsoft#44)

Author: eedorenko

aml_pipelines/model_train_pipeline.py

@@ -1,24 +1,18 @@
-from azureml.core.authentication import AzureCliAuthentication
-from azureml.core.compute import ComputeTarget
 from azureml.pipeline.core.graph import PipelineParameter
-from azureml.pipeline.core import PublishedPipeline
 from azureml.pipeline.steps import PythonScriptStep
-from azureml.pipeline.core import Pipeline, PipelineData, StepSequence
-from azureml.data.data_reference import DataReference
+from azureml.pipeline.core import Pipeline, PipelineData
 from azureml.core.runconfig import RunConfiguration, CondaDependencies
-from azureml.core import Workspace, Experiment, Datastore
-import argparse
+from azureml.core import Datastore
 import datetime
-import requests
-import json
 import os
 import sys
 from dotenv import load_dotenv
 sys.path.append(os.path.abspath("./aml_service"))  # NOQA: E402
 from workspace import get_workspace
 from attach_compute import get_compute
 
-def main():    
+
+def main():
     load_dotenv()
     workspace_name = os.environ.get("AML_WORKSPACE_NAME")
     resource_group = os.environ.get("RESOURCE_GROUP")
@@ -29,7 +23,7 @@ def main():
     sources_directory_train = os.environ.get("SOURCES_DIR_TRAIN")
     train_script_path = os.environ.get("TRAIN_SCRIPT_PATH")
     evaluate_script_path = os.environ.get("EVALUATE_SCRIPT_PATH")
-    register_script_path = os.environ.get("REGISTER_SCRIPT_PATH")    
+    register_script_path = os.environ.get("REGISTER_SCRIPT_PATH")
     vm_size_cpu = os.environ.get("AML_COMPUTE_CLUSTER_CPU_SKU")
     compute_name_cpu = os.environ.get("AML_COMPUTE_CLUSTER_NAME")
     experiment_name = os.environ.get("EXPERIMENT_NAME")
@@ -42,7 +36,7 @@ def main():
         tenant_id,
         app_id,
         app_secret)
-    print(aml_workspace)    
+    print(aml_workspace)
 
     # Get Azure machine learning cluster
     aml_compute_cpu = get_compute(
@@ -61,7 +55,8 @@ def main():
     )
     run_config.environment.docker.enabled = True
 
-    model_name = PipelineParameter(name="model_name", default_value="sklearn_regression_model.pkl")
+    model_name = PipelineParameter(
+        name="model_name", default_value="sklearn_regression_model.pkl")
     def_blob_store = Datastore(aml_workspace, "workspaceblobstore")
     jsonconfigs = PipelineData("jsonconfigs", datastore=def_blob_store)
     config_suffix = datetime.datetime.now().strftime("%Y%m%d%H")
@@ -89,15 +84,15 @@ def main():
         compute_target=aml_compute_cpu,
         source_directory=sources_directory_train,
         arguments=[
-                    "--config_suffix", config_suffix,
-                    "--json_config", jsonconfigs,
+            "--config_suffix", config_suffix,
+            "--json_config", jsonconfigs,
         ],
         runconfig=run_config,
         inputs=[jsonconfigs],
         # outputs=[jsonconfigs],
         allow_reuse=False,
     )
-    print("Step Evaluate created")    
+    print("Step Evaluate created")
 
     register_model_step = PythonScriptStep(
         name="Register New Trained Model",
@@ -122,8 +117,8 @@ def main():
 
     train_pipeline = Pipeline(workspace=aml_workspace, steps=steps)
     train_pipeline.validate()
-    pipeline_run = train_pipeline.submit(experiment_name=experiment_name)  
+    train_pipeline.submit(experiment_name=experiment_name)
 
 
 if __name__ == '__main__':
-    main()
+    main()

azdo_pipelines/base-pipeline.yml

@@ -5,15 +5,14 @@ parameters:
 steps:
 - script: |
    flake8 --output-file=$(Build.BinariesDirectory)/flake8_amlservice_testresults.xml aml_service
-   flake8 --output-file=$(Build.BinariesDirectory)/flake8_tests_testresults.xml tests
   workingDirectory: '$(Build.SourcesDirectory)'
   displayName: 'Run code quality tests'
   enabled: 'true'
 
-- script: |
-   pytest --junitxml=$(Build.BinariesDirectory)/unittest-results.xml $(Build.SourcesDirectory)/Code/tests/unit/${{parameters.pipelineType}}
-  displayName: 'Run unit tests'
-  enabled: 'true'
+# - script: |
+#    pytest --junitxml=$(Build.BinariesDirectory)/unittest-results.xml $(Build.SourcesDirectory)/Code/tests/unit/${{parameters.pipelineType}}
+#   displayName: 'Run unit tests'
+#   enabled: 'true'
 
 - task: PublishTestResults@2
   condition: succeededOrFailed()

azdo_pipelines/build-train.yml

@@ -16,7 +16,24 @@ variables:
 - group: devopsforai-aml-vg
 
 
-steps:    
+steps:
+- script: |
+   flake8 --output-file=$(Build.BinariesDirectory)/flake8_testresults.xml --format junit-xml  
+  workingDirectory: '$(Build.SourcesDirectory)'
+  displayName: 'Run code quality tests'
+  enabled: 'true'
+
+- task: PublishTestResults@2
+  condition: succeededOrFailed()
+  inputs:
+    testResultsFiles: '$(Build.BinariesDirectory)/*_testresults.xml'
+    testRunTitle: 'Linitnig'
+    failTaskOnFailedTests: true
+  displayName: 'Publish linting results'
+  enabled: 'true'
+  
+
+    
 - bash: |
    # Invoke the Python training pipeline
    python3 $(Build.SourcesDirectory)/aml_pipelines/model_train_pipeline.py

code/evaluate/evaluate_model.py

@@ -23,11 +23,9 @@
 ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE
 POSSIBILITY OF SUCH DAMAGE.
 """
-import os, json
-from azureml.core import Workspace
-from azureml.core import Experiment
+import os
+import json
 from azureml.core.model import Model
-import azureml.core
 from azureml.core import Run
 import argparse
 
@@ -68,30 +66,30 @@
 with open(train_output_path) as f:
     config = json.load(f)
 
-# parser = argparse.ArgumentParser()
-# parser.add_argument('--train_run_id',type=str,default='',help='Run id of the newly trained model')
-# #parser.add_argument('--model_assets_path',type=str,default='outputs',help='Location of trained model.')
-
 
 new_model_run_id = config["run_id"]  # args.train_run_id
 experiment_name = config["experiment_name"]
 # exp = Experiment(workspace=ws, name=experiment_name)
 
 
 try:
-    # Get most recently registered model, we assume that is the model in production. Download this model and compare it with the recently trained model by running test with same data set.
+    # Get most recently registered model, we assume that
+    # is the model in production.
+    # Download this model and compare it with the recently
+    # trained model by running test with same data set.
     model_list = Model.list(ws)
     production_model = next(
         filter(
-            lambda x: x.created_time == max(model.created_time for model in model_list),
+            lambda x: x.created_time == max(
+                model.created_time for model in model_list),
             model_list,
         )
     )
     production_model_run_id = production_model.tags.get("run_id")
     run_list = exp.get_runs()
-    # production_model_run = next(filter(lambda x: x.id == production_model_run_id, run_list))
 
-    # Get the run history for both production model and newly trained model and compare mse
+    # Get the run history for both production model and
+    # newly trained model and compare mse
     production_model_run = Run(exp, run_id=production_model_run_id)
     new_model_run = Run(exp, run_id=new_model_run_id)
 
@@ -107,9 +105,10 @@
     if new_model_mse < production_model_mse:
         promote_new_model = True
         print("New trained model performs better, thus it will be registered")
-except:
+except Exception:
     promote_new_model = True
-    print("This is the first model to be trained, thus nothing to evaluate for now")
+    print("This is the first model to be trained, \
+          thus nothing to evaluate for now")
 
 run_id = {}
 run_id["run_id"] = ""

code/register/register_model.py

@@ -23,14 +23,13 @@
 ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE
 POSSIBILITY OF SUCH DAMAGE.
 """
-import os, json, sys
-from azureml.core import Workspace
+import os
+import json
+import sys
 from azureml.core import Run
-from azureml.core import Experiment
 from azureml.core.model import Model
 import argparse
 
-from azureml.core.runconfig import RunConfiguration
 from azureml.core.authentication import AzureCliAuthentication
 
 cli_auth = AzureCliAuthentication()
@@ -78,10 +77,10 @@
     with open(evaluate_output_path) as f:
         config = json.load(f)
     if not config["run_id"]:
-        raise Exception("No new model to register as production model perform better")
-except:
+        raise Exception(
+            "No new model to register as production model perform better")
+except Exception:
     print("No new model to register as production model perform better")
-    # raise Exception('No new model to register as production model perform better')
     sys.exit(0)
 
 run_id = config["run_id"]

code/scoring/create_scoring_image.py

@@ -23,7 +23,9 @@
 ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE
 POSSIBILITY OF SUCH DAMAGE.
 """
-import os, json, sys
+import os
+import json
+import sys
 import argparse
 from azureml.core import Workspace
 from azureml.core.image import ContainerImage, Image
@@ -65,16 +67,16 @@
 try:
     with open(register_output_path) as f:
         config = json.load(f)
-except:
+except Exception:
     print("No new model to register thus no need to create new scoring image")
-    # raise Exception('No new model to register as production model perform better')
     sys.exit(0)
 
 model_name = config["model_name"]
 model_version = config["model_version"]
 
 model_list = Model.list(workspace=ws)
-model, = (m for m in model_list if m.version == model_version and m.name == model_name)
+model, = (m for m in model_list if m.version ==
+          model_version and m.name == model_name)
 print(
     "Model picked: {} \nModel Description: {} \nModel Version: {}".format(
         model.name, model.description, model.version
@@ -123,5 +125,3 @@
 output_path = os.path.join(args.json_config, filename)
 with open(output_path, "w") as outfile:
     json.dump(image_json, outfile)
-
-# How to fix the schema for a model, like if we have multiple models expecting different schema,

code/scoring/score.py

@@ -23,10 +23,8 @@
 ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE
 POSSIBILITY OF SUCH DAMAGE.
 """
-import pickle
 import json
 import numpy
-from sklearn.ensemble import RandomForestClassifier
 from azureml.core.model import Model
 
 
@@ -35,7 +33,8 @@ def init():
     from sklearn.externals import joblib
 
     # load the model from file into a global object
-    model_path = Model.get_model_path(model_name="sklearn_regression_model.pkl")
+    model_path = Model.get_model_path(
+        model_name="sklearn_regression_model.pkl")
     model = joblib.load(model_path)
 
 

code/training/train.py

@@ -23,8 +23,6 @@
 ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE
 POSSIBILITY OF SUCH DAMAGE.
 """
-import pickle
-from azureml.core import Workspace
 from azureml.core.run import Run
 import os
 import argparse
@@ -35,8 +33,6 @@
 from sklearn.externals import joblib
 import numpy as np
 import json
-import subprocess
-from typing import Tuple, List
 
 
 parser = argparse.ArgumentParser("train")
@@ -72,8 +68,10 @@
 
 X, y = load_diabetes(return_X_y=True)
 columns = ["age", "gender", "bmi", "bp", "s1", "s2", "s3", "s4", "s5", "s6"]
-X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)
-data = {"train": {"X": X_train, "y": y_train}, "test": {"X": X_test, "y": y_test}}
+X_train, X_test, y_train, y_test = train_test_split(
+    X, y, test_size=0.2, random_state=0)
+data = {"train": {"X": X_train, "y": y_train},
+        "test": {"X": X_test, "y": y_test}}
 
 print("Running train.py")
 
@@ -97,16 +95,13 @@
 
 # upload the model file explicitly into artifacts
 run.upload_file(name="./outputs/" + model_name, path_or_stream=model_name)
-print("Uploaded the model {} to experiment {}".format(model_name, run.experiment.name))
+print("Uploaded the model {} to experiment {}".format(
+    model_name, run.experiment.name))
 dirpath = os.getcwd()
 print(dirpath)
 print("Following files are uploaded ")
 print(run.get_file_names())
 
-# register the model
-# run.log_model(file_name = model_name)
-# print('Registered the model {} to run history {}'.format(model_name, run.history.name))
-
 run_id = {}
 run_id["run_id"] = run.id
 run_id["experiment_name"] = run.experiment.name
@@ -115,4 +110,4 @@
 with open(output_path, "w") as outfile:
     json.dump(run_id, outfile)
 
-run.complete()
+run.complete()

tests/unit/data_test.py

@@ -34,7 +34,8 @@ def get_absPath(filename):
     """Returns the path of the notebooks folder"""
     path = os.path.abspath(
         os.path.join(
-            os.path.dirname(__file__), os.path.pardir, os.path.pardir, "data", filename
+            os.path.dirname(
+                __file__), os.path.pardir, os.path.pardir, "data", filename
         )
     )
     return path
@@ -119,6 +120,8 @@ def test_check_distribution():
     mean = np.mean(dataset.values, axis=0)
     std = np.mean(dataset.values, axis=0)
     assert (
-        np.sum(abs(mean - historical_mean) > shift_tolerance * abs(historical_mean))
-        or np.sum(abs(std - historical_std) > shift_tolerance * abs(historical_std)) > 0
+        np.sum(abs(mean - historical_mean) >
+               shift_tolerance * abs(historical_mean))
+        or np.sum(abs(std - historical_std) >
+                  shift_tolerance * abs(historical_std)) > 0
     )