Split out azdev test from previous big PR #221

azdev/operations/testtool/__init__.py

@@ -21,7 +21,7 @@
     ENV_VAR_TEST_LIVE,
     COMMAND_MODULE_PREFIX, EXTENSION_PREFIX,
     make_dirs, get_azdev_config_dir,
-    get_path_table, require_virtual_env, get_name_index)
+    get_path_table, require_virtual_env, get_name_index, const)
 from .pytest_runner import get_test_runner
 from .profile_context import ProfileContext, current_profile
 from .incremental_strategy import CLIAzureDevOpsContext
@@ -31,7 +31,7 @@
 
 # pylint: disable=too-many-statements,too-many-locals
 def run_tests(tests, xml_path=None, discover=False, in_series=False,
-              run_live=False, profile=None, last_failed=False, pytest_args=None,
+              run_live=False, clean=False, profile=None, last_failed=False, pytest_args=None,
               no_exit_first=False, mark=None,
               git_source=None, git_target=None, git_repo=None,
               cli_ci=False):
@@ -100,23 +100,20 @@ def _find_test(index, name):
             logger.warning("'%s' not found. If newly added, re-run with --discover", t)
             continue
 
-    exit_code = 0
-
     # Tests have been collected. Now run them.
     if not test_paths:
         logger.warning('No tests selected to run.')
-        sys.exit(exit_code)
+        sys.exit(0)
 
-    exit_code = 0
     with ProfileContext(profile):
         runner = get_test_runner(parallel=not in_series,
                                  log_path=xml_path,
                                  last_failed=last_failed,
                                  no_exit_first=no_exit_first,
-                                 mark=mark)
-        exit_code = runner(test_paths=test_paths, pytest_args=pytest_args)
-
-    sys.exit(0 if not exit_code else 1)
+                                 mark=mark,
+                                 clean=clean)
+        failed = runner(test_paths=test_paths, pytest_args=pytest_args)
+    sys.exit(1 if failed else 0)
 
 
 def _filter_by_git_diff(tests, test_index, git_source, git_target, git_repo):

azdev/operations/testtool/pytest_runner.py

@@ -5,18 +5,15 @@
 # -----------------------------------------------------------------------------
 
 import os
+import subprocess
+from azdev.utilities import const, display
+import multiprocessing
 
-from knack.log import get_logger
 
-from azdev.utilities import call
-
-
-def get_test_runner(parallel, log_path, last_failed, no_exit_first, mark):
+def get_test_runner(parallel, log_path, last_failed, no_exit_first, mark, clean):
     """Create a pytest execution method"""
     def _run(test_paths, pytest_args):
 
-        logger = get_logger(__name__)
-
         if os.name == 'posix':
             arguments = ['-x', '-v', '--boxed', '-p no:warnings', '--log-level=WARN', '--junit-xml', log_path]
         else:
@@ -28,15 +25,37 @@ def _run(test_paths, pytest_args):
         if mark:
             arguments.append('-m "{}"'.format(mark))
 
-        arguments.extend(test_paths)
         if parallel:
             arguments += ['-n', 'auto']
         if last_failed:
             arguments.append('--lf')
         if pytest_args:
             arguments += pytest_args
-        cmd = 'python -m pytest {}'.format(' '.join(arguments))
-        logger.info('Running: %s', cmd)
-        return call(cmd)
+        tests_params = [(i, clean, arguments) for i in test_paths]
+        test_fail = False
+        with multiprocessing.Pool(multiprocessing.cpu_count()) as the_pool:
+            try:
+                the_pool.map(_run_test, tests_params)
+            except subprocess.CalledProcessError:
+                display("TEST FAILED! : " + str(subprocess.CalledProcessError.stdout))
+                test_fail = True
+        return test_fail
 
     return _run
+
+
+def _run_test(test_args):
+    cmd = ("python " + ('-B ' if test_args[1] else ' ') +
+           "-m pytest {}").format(' '.join([test_args[0]] + test_args[2]))
+    try:
+        subprocess.check_call(cmd.split(), shell=const.IS_WINDOWS)
+    except subprocess.CalledProcessError as e:
+        if test_args[1]:
+            display("Test failed, cleaning up recordings")
+            recordings = os.path.join(test_args[0], 'recordings')
+            if os.path.isdir(recordings):
+                recording_files = os.listdir(recordings)
+                for file in recording_files:
+                    if file.endswith(".yaml"):
+                        os.remove(os.path.join(recordings, file))
+        raise e