test(cat-gateway): schemathesis for health endpoints (#1485)

* wip

* wip

* wip

* wip

* wip

* wip

* fix: remove scylla earthfile

* feat: runs schemathesis only on api/started endpoint

* clean up

* fix: blueprint

* fix: blueprint

* fix: blueprint

* fix: blueprint

* fix: remove hooks

* fix: remove some checks

* feat: adds seed param to chemathesis package

* fix: poetry

* fix: earthly dockerd bug

* fix: remove checks all param

* fix: remove duplicated function

* fix: regex

* adds all check

* chore: clean up

* clean up

* chore: update schemathesis version

* fix: update schemathesis option to latest version

* fixes

* fix: st target failing statment

---------

Co-authored-by: Steven Johnson <[email protected]>
Co-authored-by: Oleksandr Prokhorenko <[email protected]>

Author: 3 people

.gitignore

@@ -123,4 +123,8 @@ test-results/
 /playwright/.cache/
 extensions/
 browser_extensions/
-!catalyst_voices/packages/internal/catalyst_voices_driver/lib/src/extensions/
+!catalyst_voices/packages/internal/catalyst_voices_driver/lib/src/extensions/
+
+#Test report and coverage
+*.junit-report.xml
+*.coverage.info

Overall,

@@ -1,3 +1,3 @@
 __pycache__
-junit-report.xml
-.hypothesis/
+.hypothesis/
+cassette.yaml

catalyst-gateway/tests/schemathesis_tests/.gitignore

@@ -3,21 +3,25 @@ VERSION 0.8
 package-schemathesis:
     FROM python:3.12-alpine3.20
     # TODO: https://github.com/input-output-hk/catalyst-voices/issues/465
+    ARG openapi_spec
+    # optional argument that can be used to pass a --hypothesis-seed to replicate specific test failures
+    ARG seed
+    ARG version=3.39.5
+
     RUN apk add --no-cache gcc musl-dev
-    RUN python -m pip install schemathesis==3.27.1
+    RUN python -m pip install schemathesis==$version
     RUN mkdir /results
     COPY ./hooks/hooks.py .
     VOLUME /results
-    ARG openapi_spec
-    # optional argument that can be used to pass a --hypothesis-seed to replicate specific test failures
-    ARG seed
-    ENTRYPOINT st run --endpoint '^/api/v1/health' $openapi_spec \
+    ENTRYPOINT st run --include-path-regex '^/api/v1/health/' \
+                    --exclude-path '/api/v1/health/inspection' \ #excluding since this is a internal debug endpoint
+                    --checks=all $openapi_spec \
                     --workers=2 \
                     --wait-for-schema=120 \
                     --max-response-time=5000 \
                     --hypothesis-max-examples=1000 \
                     --data-generation-method=all \
-                    --skip-deprecated-operations \
+                    --exclude-deprecated \
                     --force-schema-version=30 \
                     --show-trace \
                     --force-color  \
@@ -31,7 +35,7 @@ package-schemathesis:
 # test-fuzzer-api - Fuzzy test cat-gateway using openapi specs.
 test-fuzzer-api:
     FROM earthly/dind:alpine-3.19-docker-25.0.5-r0
-    RUN apk update && apk add iptables-legacy # workaround for https://github.com/earthly/earthly/issues/3784
+    RUN apk update && apk add iptables-legacy curl # workaround for https://github.com/earthly/earthly/issues/3784
     COPY schemathesis-docker-compose.yml .
     LET OPENAPI_SPEC="http://0.0.0.0:3030/docs/cat-gateway.json"
     ARG seed
@@ -45,11 +49,15 @@ test-fuzzer-api:
         --service cat-gateway \
         --allow-privileged
 
-        RUN docker run --net=host --name=st schemathesis:latest && \
+        RUN docker run --net=host --name=st schemathesis:latest || echo fail > fail; \
             docker cp st:/results/junit-report.xml junit-report.xml && \
             docker cp st:/results/cassette.yaml cassette.yaml
     END
     WAIT
         SAVE ARTIFACT junit-report.xml AS LOCAL schemathesis.junit-report.xml
         SAVE ARTIFACT cassette.yaml AS LOCAL cassette.yaml
+    END
+    IF [ -f fail ]
+        RUN --no-cache echo "Schemathesis test failed. Check the logs for more details" && \
+            exit 1
     END