test: add black box test for false indeterministic checkouts

Make sure the build stops if predicted sources of deterministic
checkouts do not match. Verify the legacy behaviour too.

Author: jkloetzke

test/black-box/checkout-indeterminstic/.gitignore

@@ -0,0 +1,2 @@
+*/archive/
+*/dev/

test/black-box/checkout-indeterminstic/new-behaviour/config.yaml

@@ -0,0 +1,3 @@
+bobMinimumVersion: "0.25"
+policies:
+    failUnstableCheckouts: True

test/black-box/checkout-indeterminstic/new-behaviour/default.yaml

@@ -0,0 +1,3 @@
+archive:
+    backend: file
+    path: archive

test/black-box/checkout-indeterminstic/new-behaviour/recipes/root.yaml

@@ -0,0 +1,9 @@
+root: True
+
+checkoutDeterministic: True
+checkoutScript: |
+    date '+%s.%N' > epoch.txt
+
+buildVars: [VARIANT]
+buildScript: "true"
+packageScript: "true"

test/black-box/checkout-indeterminstic/old-behaviour/config.yaml

@@ -0,0 +1 @@
+bobMinimumVersion: "0.25"

test/black-box/checkout-indeterminstic/old-behaviour/default.yaml

@@ -0,0 +1,3 @@
+archive:
+    backend: file
+    path: archive

test/black-box/checkout-indeterminstic/old-behaviour/recipes/root.yaml

@@ -0,0 +1,9 @@
+root: True
+
+checkoutDeterministic: True
+checkoutScript: |
+    date '+%s.%N' > epoch.txt
+
+buildVars: [VARIANT]
+buildScript: "true"
+packageScript: "true"

test/black-box/checkout-indeterminstic/run.sh

@@ -0,0 +1,24 @@
+#!/bin/bash -e
+. ../../test-lib.sh 2>/dev/null || { echo "Must run in script directory!" ; exit 1 ; }
+cleanup
+rm -rf archive
+
+# Test that indeterministic checkouts are detected when downloading artifacts.
+pushd new-behaviour
+run_bob dev root -D VARIANT=one --upload
+
+# Building another variant makes sure that no artifact is found. But Bob will
+# still predict the checkout step. This prediction will be wrong and should be
+# detected.
+cleanup
+expect_fail run_bob dev root -D VARIANT=two
+
+popd
+
+# The old behaviour was to ignore unexpected indeterministic checkouts and just
+# restart the build. Verify that this is still working.
+pushd old-behaviour
+run_bob dev root -D VARIANT=one --upload
+cleanup
+run_bob dev root -D VARIANT=two
+popd