WIP catch exceptions thrown in any step expression

Also fixes a repeat bug, where we only repeated a step's "run" if the
"run" itself returned a non-zero exit code. If an expect expression
failed, the step would immediately fail instead of being
retried/repeated.

.github/workflows/push.yaml

@@ -50,7 +50,7 @@ jobs:
             -v $(pwd)/examples:/app/examples \
             dctest --results-file /app/examples/results.json examples /app/examples/00-intro.yaml \
             || true
-          jq --exit-status '[.pass == 4, .fail == 0] | all' examples/results.json
+          jq --exit-status '[.pass == 5, .fail == 0] | all' examples/results.json
 
       - name: Run Intro + Fail Examples with --continue-on-error
         run: |
@@ -61,7 +61,7 @@ jobs:
             -v $(pwd)/examples:/app/examples \
             dctest --continue-on-error --results-file /app/examples/results.json examples /app/examples/00-intro.yaml /app/examples/01-fails.yaml \
             || true
-          jq --exit-status '[.pass == 5, .fail == 3] | all' examples/results.json
+          jq --exit-status '[.pass == 6, .fail == 4] | all' examples/results.json
 
       - name: Run Dependency Examples
         run: |

examples/00-intro.yaml

@@ -15,7 +15,6 @@ tests:
           FOO: bar
         run: |
           [ "${FOO}" == "bar" ]
-        repeat: { retries: 3, interval: '1s' }
       - exec: node1
         if: success()
         run: echo "This will be run!"
@@ -69,3 +68,21 @@ tests:
         expect:
           - step.stdout != "2"
           - step.stdout == "3"
+
+  repeat:
+    name: Repeat test
+    steps:
+      - exec: node1
+        run: rm -f repeat-test-file
+      - exec: node1
+        repeat: { retries: 2, interval: '1s' }
+        run: |
+          if [ ! -f repeat-test-file ]; then
+            touch repeat-test-file
+          else
+            echo -n 'Repeated successfully'
+          fi
+        expect:
+          - step.stdout == "Repeated successfully"
+      - exec: node1
+        run: rm -f repeat-test-file

examples/01-fails.yaml

@@ -34,3 +34,11 @@ tests:
     steps:
       - exec: node2
         run: /bin/true
+
+  fail-expressions:
+    name: Test thrown exception in expressions
+    steps:
+      - exec: node1
+        env:
+          FOO: ${{ throw("Intentional Failure") }}
+        run: /bin/true

src/dctest/core.cljs

@@ -174,56 +174,98 @@ Options:
     (merge {:stdout (S/join "" @stdout)
             :stderr (S/join "" @stderr)}
            (when code {:code code})
-           (when error {:error error})
+           (when error {:errors [error]})
            (when signal {:signal signal}))))
 
+(defmacro if-p->
+  "When expr satisfies pred, threads it into the first form (via ->),
+  and when that result satisfies pred, through the next etc. If pred
+  is not satisfied at any form, do not apply the form, and return the
+  expr at that point."
+  [pred expr & forms]
+  (let [g (gensym)
+        steps (map (fn [step] `(if (~pred ~g) (-> ~g ~step) ~g))
+                   forms)]
+    `(let [~g ~expr
+           ~@(interleave (repeat g) (butlast steps))]
+       ~(if (empty? steps)
+          g
+          (last steps)))))
+
+(defn get-expect-error [context expect]
+  (when-not (try (expr/read-eval context expect)
+                 (catch js/Error _ false))
+    {:message (str "Expectation failure: " expect)
+     :debug (expr/explain-refs context expect)}))
+
+(defn execute-step* [context step]
+  (P/let [update-step-or-fail (fn [m k f & args]
+                                (try
+                                  (update-in m [:step k] #(apply f (:context m) % args))
+                                  (catch js/Error err
+                                    (-> m
+                                        (assoc :outcome :fail)
+                                        (update :errors conj
+                                                {:message (str "Error in '" (name k) "': " (.-message err))})))))]
+    (if-p-> #(= :pending (:outcome %))
+            {:context context
+             :outcome :pending
+             :errors []
+             :step step}
+
+            ;; Skip step when "if" is false
+            (update-step-or-fail :if expr/read-eval)
+            (as-> m
+              (if (get-in m [:step :if])
+                m
+                (assoc m :outcome :skip)))
+
+            ;; Evaluate env before everything else
+            (update-step-or-fail :env
+                                 (fn [context env]
+                                   (update-vals env #(expr/interpolate-text context %))))
+            (as-> m
+              (update-in m [:context :env] merge (get-in m [:step :env])))
+
+            ;; Normalize and/or evaluate keys required for command execution
+            (update-step-or-fail :exec expr/interpolate-text)
+            (update-step-or-fail :expect (fn [_ expect] (if (string? expect) [expect] expect)))
+            (update-step-or-fail :index (fn [_ index] (or index 1)))
+            (update-step-or-fail :run (fn [context command]
+                                        (if (string? command)
+                                          (expr/interpolate-text context command)
+                                          (mapv #(expr/interpolate-text context %) command))))
+
+            ;; Execute (with retries)
+            (as-> m
+              (P/let [{:keys [interval retries]} (get-in m [:step :repeat])
+                      run-attempt (fn []
+                                    (P/let [results (run-exec (:context m) (:step m))
+                                            m (assoc-in m [:context :step] results)
+                                            errors (into (:errors results)
+                                                         (keep #(get-expect-error (:context m) %)
+                                                               (get-in m [:step :expect])))]
+                                      (if (seq errors)
+                                        (-> m
+                                            (assoc :outcome :fail)
+                                            (update :errors into errors))
+                                        m)))]
+                (retry/retry-times run-attempt
+                                   retries
+                                   {:delay-ms interval
+                                    :check-fn #(not= :fail (:outcome %))})))
+
+            ;; Mark as successful
+            (update :outcome :pass))))
+
 (defn execute-step [context step]
-  (P/let [skip? (not (expr/read-eval context (:if step)))]
-
-    (if skip?
-      (let [results (merge {:outcome :skip}
-                           (select-keys step [:name]))]
-        {:context context
-         :results results})
-
-      (P/let [;; Interpolate step env before all other keys
-              step (update step :env update-vals #(expr/interpolate-text context %))
-              ;; Do not leak step env back into original context
-              step-context (update context :env merge (:env step))
-
-              ;; Interpolate rest of step using step-local context
-              interpolate #(expr/interpolate-text step-context %)
-              step (-> step
-                       (update :exec interpolate)
-                       (update :expect #(if (string? %) [%] %))
-                       (update :index #(or % 1))
-                       (update :run (fn [command]
-                                      (if (string? command)
-                                        (interpolate command)
-                                        (mapv interpolate command)))))
-
-              {:keys [interval retries]} (:repeat step)
-              results (retry/retry-times #(run-exec step-context step)
-                                         retries
-                                         {:delay-ms interval
-                                          :check-fn #(not (:error %))})
-
-              step-context (assoc step-context :step results)
-              run-expect (fn [expr]
-                           (when-not (expr/read-eval step-context expr)
-                             {:error {:message (str "Expectation failure: " expr)
-                                      :debug (expr/explain-refs step-context expr)}}))
-              expect-results (when-not (:error results)
-                               (first (keep run-expect (:expect step))))
-              results (merge expect-results results)
-
-              outcome (if (:error results) :fail :pass)
-              results (merge results
-                             {:outcome outcome}
-                             (select-keys step [:name]))
-              context (update-in context [:state :failed] #(or % (failure? results)))]
-        {:context context
-         :results results}))))
+  (P/let [{:keys [outcome errors step]} (execute-step* context step)]
+    {:context (update-in context [:state :failed] #(or % (= :fail outcome)))
+     :results (merge
+                {:outcome outcome}
+                (select-keys step [:name])
+                (when-let [error (first errors)]
+                  {:error error}))}))
 
 (defn execute-steps [context steps]
   (P/let [results (P/loop [steps steps

src/dctest/expressions.cljs

@@ -21,6 +21,9 @@
    "always"  {:arity 0 :fn (constantly true)}
    "success" {:arity 0 :fn #(not (get-in % [:state :failed]))}
    "failure" {:arity 0 :fn #(boolean (get-in % [:state :failed]))}
+
+   ;; Test status functions
+   "throw" {:arity 1 :fn #(throw (ex-info %2 {}))}
    })
 
 (def BEGIN_INTERP "${{")
@@ -284,7 +287,16 @@ ExpectedInterpolation ::= InterpolatedExpression PrintableChar*
 
                      ;; Recurse
                      (recur (zip/next loc)
-                            refs)))))]
-    (reduce #(assoc %1 %2 (read-eval context %2))
+                            refs)))))
+        safe-read-eval (fn [context expr]
+                         (let [{:keys [result error-msg]}
+                               , (try
+                                   {:result (read-eval context expr)}
+                                   (catch js/Error err
+                                     {:error-msg (.-message e)}))]
+                           (if error-msg
+                             (str "<Error: " error-msg ">")
+                             result)))]
+    (reduce #(assoc %1 %2 (safe-read-eval context %2))
             {}
             refs)))