From 2b4824c3c96cc7836dfc0be0073c3a4744d305de Mon Sep 17 00:00:00 2001 From: kukkok3 <93382903+kukkok3@users.noreply.github.com> Date: Wed, 3 Apr 2024 15:22:59 +0200 Subject: [PATCH 1/3] feat: improve schemathesis logs --- catalyst-gateway/tests/Earthfile | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/catalyst-gateway/tests/Earthfile b/catalyst-gateway/tests/Earthfile index c68f3ac462b..d4f666e31a4 100644 --- a/catalyst-gateway/tests/Earthfile +++ b/catalyst-gateway/tests/Earthfile @@ -13,37 +13,47 @@ package-schemathesis: RUN python -m pip install schemathesis RUN mkdir /results VOLUME /results - ENTRYPOINT st run --checks all $openapi_spec --data-generation-method all --workers=$workers --wait-for-schema=$wait_for_schema \ + ENTRYPOINT st run --checks all $openapi_spec --workers=$workers --wait-for-schema=$wait_for_schema \ --max-response-time=$max_response_time --hypothesis-max-examples=$max_examples \ - --force-color --skip-deprecated-operations --show-trace \ - --force-schema-version=$schema_version --junit-xml /results/junit-report.xml + --data-generation-method=positive --skip-deprecated-operations \ + --force-schema-version=$schema_version --show-trace --force-color \ + --junit-xml /results/junit-report.xml --cassette-path=/results/cassette.yaml + SAVE IMAGE schemathesis:$tag # test-fuzzer-api - Fuzzy test cat-gateway using openapi specs test-fuzzer-api: FROM earthly/dind:alpine-3.19 RUN apk update && apk add iptables-legacy # workaround for https://github.com/earthly/earthly/issues/3784 + RUN apk add yq + ARG DB_URL="postgres://catalyst-event-dev:CHANGE_ME@localhost/CatalystEventDev" + ARG CAT_ADDRESS="127.0.0.1:3030" + ARG OPENAPI_SPEC="http://127.0.0.1:3030/docs/cat-gateway.json" + COPY schemathesis-docker-compose.yml . WITH DOCKER \ --compose schemathesis-docker-compose.yml \ - --load schemathesis:latest=(+package-schemathesis --openapi_spec="http://127.0.0.1:3030/docs/cat-gateway.json") \ + --load schemathesis:latest=(+package-schemathesis --openapi_spec=$OPENAPI_SPEC) \ --load event-db:latest=(../event-db+build) \ - --load cat-gateway:latest=(../+package-cat-gateway --address="127.0.0.1:3030" \ - --db_url="postgres://catalyst-event-dev:CHANGE_ME@localhost/CatalystEventDev") \ + --load cat-gateway:latest=(../+package-cat-gateway --address=$CAT_ADDRESS --db_url=$DB_URL) \ --service event-db \ --service cat-gateway \ --allow-privileged RUN docker run --net=host --name=st schemathesis:latest || echo fail > fail && \ docker logs cat-gateway > ./cat-gateway.log && \ - docker cp st:/results/junit-report.xml junit-report.xml + docker cp st:/results/junit-report.xml junit-report.xml && \ + docker cp st:/results/cassette.yaml cassette.yaml END WAIT SAVE ARTIFACT junit-report.xml AS LOCAL schemathesis.junit-report.xml SAVE ARTIFACT cat-gateway.log AS LOCAL cat-gateway.log + SAVE ARTIFACT cassette.yaml AS LOCAL cassette.yaml END IF [ -f fail ] - RUN echo "Schemathesis run failed" && \ - echo "Printing catalyst gateway logs..." && \ + RUN echo -e "\033[0;31mSchemathesis test run failed\n" && \ + echo -e "\033[0;31mFailed tests:" && \ + yq -PC '.http_interactions.[] | select(.status == "FAILURE")' cassette.yaml && \ + echo -e "\033[0;31mCatalyst Gateway logs:" && \ jq --color-output . cat-gateway.log cat-gateway.log && \ exit 1 END From 218f6fb39e45db782e88b55b73c95e17823e1069 Mon Sep 17 00:00:00 2001 From: kukkok3 <93382903+kukkok3@users.noreply.github.com> Date: Wed, 3 Apr 2024 15:27:51 +0200 Subject: [PATCH 2/3] adds issue url --- catalyst-gateway/tests/Earthfile | 1 + 1 file changed, 1 insertion(+) diff --git a/catalyst-gateway/tests/Earthfile b/catalyst-gateway/tests/Earthfile index 5a8eb691680..3bf861d948d 100644 --- a/catalyst-gateway/tests/Earthfile +++ b/catalyst-gateway/tests/Earthfile @@ -18,6 +18,7 @@ package-schemathesis: VOLUME /results ENTRYPOINT st run --checks all $openapi_spec --workers=$workers --wait-for-schema=$wait_for_schema \ --max-response-time=$max_response_time --hypothesis-max-examples=$max_examples \ + # TODO: https://github.com/input-output-hk/catalyst-voices/issues/330 --data-generation-method=positive --skip-deprecated-operations \ --force-schema-version=$schema_version --show-trace --force-color \ --junit-xml /results/junit-report.xml --cassette-path=/results/cassette.yaml From 86a8bd98753b4f152b5839f8332776859753b4a8 Mon Sep 17 00:00:00 2001 From: kukkok3 <93382903+kukkok3@users.noreply.github.com> Date: Wed, 3 Apr 2024 16:05:28 +0200 Subject: [PATCH 3/3] fix: restore checks --- catalyst-gateway/tests/Earthfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/catalyst-gateway/tests/Earthfile b/catalyst-gateway/tests/Earthfile index 3bf861d948d..5520801fdda 100644 --- a/catalyst-gateway/tests/Earthfile +++ b/catalyst-gateway/tests/Earthfile @@ -18,10 +18,12 @@ package-schemathesis: VOLUME /results ENTRYPOINT st run --checks all $openapi_spec --workers=$workers --wait-for-schema=$wait_for_schema \ --max-response-time=$max_response_time --hypothesis-max-examples=$max_examples \ - # TODO: https://github.com/input-output-hk/catalyst-voices/issues/330 - --data-generation-method=positive --skip-deprecated-operations \ + --data-generation-method=all --skip-deprecated-operations \ --force-schema-version=$schema_version --show-trace --force-color \ - --junit-xml /results/junit-report.xml --cassette-path=/results/cassette.yaml + --junit-xml /results/junit-report.xml --cassette-path=/results/cassette.yaml \ + # TODO: https://github.com/input-output-hk/catalyst-voices/issues/330 + --exclude-checks content_type_conformance \ + --exclude-checks response_schema_conformance SAVE IMAGE schemathesis:$tag