Issue #163 add milestone tracking to benchmarks

Author: soxofaan

qa/benchmarks/tests/test_benchmarks.py

@@ -32,21 +32,24 @@ def test_run_benchmark(
     tmp_path: Path,
     track_metric,
     upload_assets_on_fail,
-    request
+    request,
 ):
     track_metric("scenario_id", scenario.id)
     # Check if a backend override has been provided via cli options.
     override_backend = request.config.getoption("--override-backend")
     backend_filter = request.config.getoption("--backend-filter")
     if backend_filter and not re.match(backend_filter, scenario.backend):
-        #TODO apply filter during scenario retrieval, but seems to be hard to retrieve cli param
-        pytest.skip(f"skipping scenario {scenario.id} because backend {scenario.backend} does not match filter {backend_filter!r}")
+        # TODO apply filter during scenario retrieval, but seems to be hard to retrieve cli param
+        pytest.skip(
+            f"skipping scenario {scenario.id} because backend {scenario.backend} does not match filter {backend_filter!r}"
+        )
     backend = scenario.backend
     if override_backend:
         _log.info(f"Overriding backend URL with {override_backend!r}")
         backend = override_backend
 
     connection: openeo.Connection = connection_factory(url=backend)
+    track_metric("milestone", "connected", update=True)
 
     # TODO #14 scenario option to use synchronous instead of batch job mode?
     job = connection.create_job(
@@ -55,26 +58,34 @@ def test_run_benchmark(
         additional=scenario.job_options,
     )
     track_metric("job_id", job.job_id)
+    track_metric("milestone", "job created", update=True)
 
     # TODO: monitor timing and progress
     # TODO: abort excessively long batch jobs? https://github.com/Open-EO/openeo-python-client/issues/589
     job.start_and_wait()
+    # TODO: "job started" milestone
+    track_metric("milestone", "job finished", update=True)
 
     collect_metrics_from_job_metadata(job, track_metric=track_metric)
+    track_metric("milestone", "job metadata obtained", update=True)
 
     results = job.get_results()
+    track_metric("milestone", "job results metadata obtained", update=True)
     collect_metrics_from_results_metadata(results, track_metric=track_metric)
 
     # Download actual results
     actual_dir = tmp_path / "actual"
     paths = results.download_files(target=actual_dir, include_stac_metadata=True)
+    track_metric("milestone", "job results downloaded", update=True)
+
     # Upload assets on failure
     upload_assets_on_fail(*paths)
 
     # Compare actual results with reference data
     reference_dir = download_reference_data(
         scenario=scenario, reference_dir=tmp_path / "reference"
     )
+    track_metric("milestone", "benchmark reference data downloaded", update=True)
 
     assert_job_results_allclose(
         actual=actual_dir,
@@ -83,3 +94,4 @@ def test_run_benchmark(
         rtol=scenario.reference_options.get("rtol", 1e-6),
         atol=scenario.reference_options.get("atol", 1e-6),
     )
+    track_metric("milestone", "compared actuals with reference data", update=True)