Django Test Runs with Coverage (microsoft#24927)

microsoft#24199

Co-authored-by: Danila Grobov (s4642g) <[email protected]>

build/test-requirements.txt

@@ -32,6 +32,7 @@ django-stubs
 coverage
 pytest-cov
 pytest-json
+pytest-timeout
 
 
 # for pytest-describe related tests

python_files/tests/pytestadapter/helpers.py

@@ -244,7 +244,7 @@ def runner_with_cwd_env(
     """
     process_args: List[str]
     pipe_name: str
-    if "MANAGE_PY_PATH" in env_add:
+    if "MANAGE_PY_PATH" in env_add and "COVERAGE_ENABLED" not in env_add:
         # If we are running Django, generate a unittest-specific pipe name.
         process_args = [sys.executable, *args]
         pipe_name = generate_random_pipe_name("unittest-discovery-test")

python_files/tests/unittestadapter/test_coverage.py

@@ -8,6 +8,8 @@
 import pathlib
 import sys
 
+import pytest
+
 sys.path.append(os.fspath(pathlib.Path(__file__).parent))
 
 python_files_path = pathlib.Path(__file__).parent.parent.parent
@@ -49,3 +51,41 @@ def test_basic_coverage():
     assert focal_function_coverage.get("lines_missed") is not None
     assert set(focal_function_coverage.get("lines_covered")) == {4, 5, 7, 9, 10, 11, 12, 13, 14}
     assert set(focal_function_coverage.get("lines_missed")) == {6}
+
+
+@pytest.mark.timeout(30)
+def test_basic_django_coverage():
+    """This test validates that the coverage is correctly calculated for a Django project."""
+    data_path: pathlib.Path = TEST_DATA_PATH / "simple_django"
+    manage_py_path: str = os.fsdecode(data_path / "manage.py")
+    execution_script: pathlib.Path = python_files_path / "unittestadapter" / "execution.py"
+
+    test_ids = [
+        "polls.tests.QuestionModelTests.test_was_published_recently_with_future_question",
+        "polls.tests.QuestionModelTests.test_was_published_recently_with_future_question_2",
+        "polls.tests.QuestionModelTests.test_question_creation_and_retrieval",
+    ]
+
+    script_str = os.fsdecode(execution_script)
+    actual = helpers.runner_with_cwd_env(
+        [script_str, "--udiscovery", "-p", "*test*.py", *test_ids],
+        data_path,
+        {
+            "MANAGE_PY_PATH": manage_py_path,
+            "_TEST_VAR_UNITTEST": "True",
+            "COVERAGE_ENABLED": os.fspath(data_path),
+        },
+    )
+
+    assert actual
+    coverage = actual[-1]
+    assert coverage
+    results = coverage["result"]
+    assert results
+    assert len(results) == 15
+    polls_views_coverage = results.get(str(data_path / "polls" / "views.py"))
+    assert polls_views_coverage
+    assert polls_views_coverage.get("lines_covered") is not None
+    assert polls_views_coverage.get("lines_missed") is not None
+    assert set(polls_views_coverage.get("lines_covered")) == {3, 4, 6}
+    assert set(polls_views_coverage.get("lines_missed")) == {7}

python_files/unittestadapter/django_handler.py

@@ -1,11 +1,17 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
 
+import importlib.util
 import os
 import pathlib
 import subprocess
 import sys
-from typing import List
+from contextlib import contextmanager, suppress
+from typing import TYPE_CHECKING, Generator, List
+
+if TYPE_CHECKING:
+    from importlib.machinery import ModuleSpec
+
 
 script_dir = pathlib.Path(__file__).parent
 sys.path.append(os.fspath(script_dir))
@@ -16,6 +22,17 @@
 )
 
 
+@contextmanager
+def override_argv(argv: List[str]) -> Generator:
+    """Context manager to temporarily override sys.argv with the provided arguments."""
+    original_argv = sys.argv
+    sys.argv = argv
+    try:
+        yield
+    finally:
+        sys.argv = original_argv
+
+
 def django_discovery_runner(manage_py_path: str, args: List[str]) -> None:
     # Attempt a small amount of validation on the manage.py path.
     if not pathlib.Path(manage_py_path).exists():
@@ -72,31 +89,28 @@ def django_execution_runner(manage_py_path: str, test_ids: List[str], args: List
         else:
             env["PYTHONPATH"] = os.fspath(custom_test_runner_dir)
 
-        # Build command to run 'python manage.py test'.
-        command: List[str] = [
-            sys.executable,
+        django_project_dir: pathlib.Path = pathlib.Path(manage_py_path).parent
+        sys.path.insert(0, os.fspath(django_project_dir))
+        print(f"Django project directory: {django_project_dir}")
+
+        manage_spec: ModuleSpec | None = importlib.util.spec_from_file_location(
+            "manage", manage_py_path
+        )
+        if manage_spec is None or manage_spec.loader is None:
+            raise VSCodeUnittestError("Error importing manage.py when running Django testing.")
+        manage_module = importlib.util.module_from_spec(manage_spec)
+        manage_spec.loader.exec_module(manage_module)
+
+        manage_argv: List[str] = [
             manage_py_path,
             "test",
             "--testrunner=django_test_runner.CustomExecutionTestRunner",
+            *args,
+            *test_ids,
         ]
-        # Add any additional arguments to the command provided by the user.
-        command.extend(args)
-        # Add the test_ids to the command.
-        print("Test IDs: ", test_ids)
-        print("args: ", args)
-        command.extend(test_ids)
-        print("Running Django run tests with command: ", command)
-        subprocess_execution = subprocess.run(
-            command,
-            capture_output=True,
-            text=True,
-            env=env,
-        )
-        print(subprocess_execution.stderr, file=sys.stderr)
-        print(subprocess_execution.stdout, file=sys.stdout)
-        # Zero return code indicates success, 1 indicates test failures, so both are considered successful.
-        if subprocess_execution.returncode not in (0, 1):
-            error_msg = "Django test execution process exited with non-zero error code See stderr above for more details."
-            print(error_msg, file=sys.stderr)
+        print(f"Django manage.py arguments: {manage_argv}")
+
+        with override_argv(manage_argv), suppress(SystemExit):
+            manage_module.main()
     except Exception as e:
         print(f"Error during Django test execution: {e}", file=sys.stderr)