From ee619a1a5f3d15b3dfe48f7182b67192868241dc Mon Sep 17 00:00:00 2001
From: Stefano Cunego <93382903+kukkok3@users.noreply.github.com>
Date: Thu, 4 Apr 2024 09:34:43 +0200
Subject: [PATCH] feat: Improve schemathesis logs (#374)

* feat: improve schemathesis logs

* adds issue url

* fix: restore checks
---
 catalyst-gateway/tests/Earthfile | 44 ++++++++++++++++----------------
 1 file changed, 22 insertions(+), 22 deletions(-)

diff --git a/catalyst-gateway/tests/Earthfile b/catalyst-gateway/tests/Earthfile
index b2794932ad9..5520801fdda 100644
--- a/catalyst-gateway/tests/Earthfile
+++ b/catalyst-gateway/tests/Earthfile
@@ -16,22 +16,14 @@ package-schemathesis:
     RUN mkdir /results
 
     VOLUME /results
-
-    ENTRYPOINT st run --checks all \
-                      # TODO: https://github.com/input-output-hk/catalyst-voices/issues/330
-                      --exclude-checks content_type_conformance \
-                      --exclude-checks response_schema_conformance \
-                      --data-generation-method all \
-                      --workers=$workers \
-                      --wait-for-schema=$wait_for_schema \
-                      --max-response-time=$max_response_time \
-                      --hypothesis-max-examples=$max_examples \
-                      --force-color \
-                      --skip-deprecated-operations \
-                      --show-trace \
-                      --force-schema-version=$schema_version \
-                      --junit-xml=/results/junit-report.xml \
-                      $openapi_spec
+    ENTRYPOINT st run --checks all $openapi_spec --workers=$workers --wait-for-schema=$wait_for_schema \
+                        --max-response-time=$max_response_time --hypothesis-max-examples=$max_examples \
+                            --data-generation-method=all --skip-deprecated-operations \
+                                --force-schema-version=$schema_version --show-trace --force-color  \
+                                  --junit-xml /results/junit-report.xml --cassette-path=/results/cassette.yaml \
+                                # TODO: https://github.com/input-output-hk/catalyst-voices/issues/330
+                                --exclude-checks content_type_conformance \
+                                --exclude-checks response_schema_conformance
 
     SAVE IMAGE schemathesis:$tag
 
@@ -39,27 +31,35 @@ package-schemathesis:
 test-fuzzer-api:
     FROM earthly/dind:alpine-3.19
     RUN apk update && apk add iptables-legacy # workaround for https://github.com/earthly/earthly/issues/3784
+    RUN apk add yq
+    ARG DB_URL="postgres://catalyst-event-dev:CHANGE_ME@localhost/CatalystEventDev"
+    ARG CAT_ADDRESS="127.0.0.1:3030"
+    ARG OPENAPI_SPEC="http://127.0.0.1:3030/docs/cat-gateway.json"
+
     COPY schemathesis-docker-compose.yml .
     WITH DOCKER \
         --compose schemathesis-docker-compose.yml \
-        --load schemathesis:latest=(+package-schemathesis --openapi_spec="http://127.0.0.1:3030/docs/cat-gateway.json") \
+        --load schemathesis:latest=(+package-schemathesis --openapi_spec=$OPENAPI_SPEC) \
         --load event-db:latest=(../event-db+build) \
-        --load cat-gateway:latest=(../+package-cat-gateway --address="127.0.0.1:3030" \
-                                        --db_url="postgres://catalyst-event-dev:CHANGE_ME@localhost/CatalystEventDev") \
+        --load cat-gateway:latest=(../+package-cat-gateway --address=$CAT_ADDRESS --db_url=$DB_URL) \
         --service event-db \
         --service cat-gateway \
         --allow-privileged
         RUN docker run --net=host --name=st schemathesis:latest || echo fail > fail && \
                 docker logs cat-gateway > ./cat-gateway.log && \
-                    docker cp st:/results/junit-report.xml junit-report.xml
+                    docker cp st:/results/junit-report.xml junit-report.xml && \
+                        docker cp st:/results/cassette.yaml cassette.yaml
     END
     WAIT
         SAVE ARTIFACT junit-report.xml AS LOCAL schemathesis.junit-report.xml
         SAVE ARTIFACT cat-gateway.log AS LOCAL cat-gateway.log
+        SAVE ARTIFACT cassette.yaml AS LOCAL cassette.yaml
     END
     IF [ -f fail ]
-        RUN echo "Schemathesis run failed" && \
-            echo "Printing catalyst gateway logs..." && \
+        RUN echo -e "\033[0;31mSchemathesis test run failed\n" && \
+            echo -e "\033[0;31mFailed tests:" && \
+            yq -PC '.http_interactions.[] | select(.status == "FAILURE")' cassette.yaml && \
+            echo -e "\033[0;31mCatalyst Gateway logs:" && \
             jq --color-output . cat-gateway.log cat-gateway.log && \
             exit 1
     END