make the failing tests on e2e actually pass

e2e/failing_config.yaml

@@ -0,0 +1,6 @@
+# first failure should cancel the run if this config is used
+executionOrder:
+  continueOnFailure: true
+  flowsOrder:
+    - commands/ai_simple.yaml
+    - commands/fail_fast.yaml

e2e/passing_config.yaml

@@ -0,0 +1,6 @@
+# Used on runs where all flows should pass
+executionOrder:
+  continueOnFailure: false
+  flowsOrder:
+    - commands/ai_simple.yaml
+    - commands/fail_fast.yaml

e2e/run_tests

@@ -49,7 +49,7 @@ for workspace_dir in ./workspaces/*; do
 	    _h3 "$app_name" "passing" "$line"
 	done < pipe &
 
-	maestro --verbose --platform "$platform" test --include-tags passing --exclude-tags "$exclude_tags" "$workspace_dir" 1>pipe 2>&1 || WORKSPACE_PASS=false
+	maestro --verbose --platform "$platform" test --include-tags passing --config ./passing_config.yaml --exclude-tags "$exclude_tags" "$workspace_dir" 1>pipe 2>&1 || WORKSPACE_PASS=false
 
 	if [ "$WORKSPACE_PASS" = "false" ]; then
         _h2 "$app_name" "FAIL! Expected all pass, but at least some failed instead"
@@ -71,12 +71,12 @@ for workspace_dir in ./workspaces/*; do
 	    _h3 "$app_name" "failing" "$line"
 	done < pipe &
 
-	maestro --verbose --platform "$platform" test --include-tags failing --exclude-tags "$exclude_tags" "$workspace_dir" 1>pipe 2>&1 && WORKSPACE_PASS=false
+	maestro --verbose --platform "$platform" test --include-tags failing --config ./failing_config.yaml --exclude-tags "$exclude_tags" "$workspace_dir" 1>pipe 2>&1 || WORKSPACE_PASS=false
 
 	if [ "$WORKSPACE_PASS" = "false" ]; then
-		_h2 "$app_name" "FAIL! Expected all to fail, but at least some passed instead"
+        _h2 "$app_name" "FAIL! Expected all pass, but at least some failed instead"
 		ALL_PASS=false
-    fi
+	fi
 done
 
 if [ "$ALL_PASS" = "false" ]; then

e2e/workspaces/demo_app/fail_fast.yaml

@@ -4,6 +4,12 @@ tags:
 ---
 - launchApp:
     clearState: true
-- assertTrue:
-    condition: ${ false }
-    label: Fail the flow
+- evalScript: ${output.didFail = 1}
+- runFlow:
+    optional: true
+    commands:
+        - assertTrue:
+            condition: ${ false }
+            label: Fail the flow
+        - evalScript: ${output.didFail = 0} # This should never run
+- assertTrue: ${output.didFail == 1}

e2e/workspaces/demo_app/fail_launchApp.yaml

@@ -2,4 +2,10 @@ appId: com.nonexistent
 tags:
     - failing
 ---
-- launchApp
+- evalScript: ${output.didFail = 1}
+- runFlow:
+    optional: true
+    commands:
+        - launchApp
+        - evalScript: ${output.didFail = 0}
+- assertTrue: ${output.didFail == 1}

e2e/workspaces/demo_app/fail_launchApp_nonDefault.yaml

@@ -2,5 +2,11 @@ appId: com.example.example
 tags:
     - failing
 ---
-- launchApp:
-    appId: com.nonexistent
+- evalScript: ${output.didFail = 1}
+- runFlow:
+    optional: true
+    commands:
+        - launchApp:
+            appId: com.nonexistent
+        - evalScript: ${output.didFail = 0}
+- assertTrue: ${output.didFail == 1}

e2e/workspaces/demo_app/fail_not_found.yaml

@@ -4,5 +4,11 @@ tags:
 ---
 - launchApp:
     clearState: true
-- tapOn:
-    id: non-existent-id
+- evalScript: ${output.didFail = 1}
+- runFlow:
+    optional: true
+    commands:
+        - tapOn:
+            id: non-existent-id
+        - evalScript: ${output.didFail = 0}
+- assertTrue: ${output.didFail == 1}

e2e/workspaces/demo_app/fail_visible.yaml

@@ -4,5 +4,11 @@ tags:
 ---
 - launchApp:
     clearState: true
-- assertVisible:
-    id: non-existent-id
+- evalScript: ${output.didFail = 1}
+- runFlow:
+    optional: true
+    commands:
+        - assertVisible:
+            id: non-existent-id
+        - evalScript: ${output.didFail = 0}
+- assertTrue: ${output.didFail == 1}

e2e/workspaces/demo_app/fail_visible_extended.yaml

@@ -4,7 +4,13 @@ tags:
 ---
 - launchApp:
     clearState: true
-- extendedWaitUntil:
-    visible:
-      id: non-existent-id
-    timeout: 100
+- evalScript: ${output.didFail = 1}
+- runFlow:
+    optional: true
+    commands:
+        - extendedWaitUntil:
+            visible:
+              id: non-existent-id
+            timeout: 100
+        - evalScript: ${output.didFail = 0}
+- assertTrue: ${output.didFail == 1}

e2e/workspaces/nowinandroid/fail.yaml

@@ -6,5 +6,11 @@ tags:
 ---
 - launchApp:
     clearState: true
-- tapOn:
-    id: non-existent-id-to-fail-this-test
+- evalScript: ${output.didFail = 1}
+- runFlow:
+    optional: true
+    commands:
+      - tapOn:
+          id: non-existent-id-to-fail-this-test
+      - evalScript: ${output.didFail = 0} # This should never run
+- assertTrue: ${output.didFail == 1}