From a464c02e26a22ada9839633841d57b976e898848 Mon Sep 17 00:00:00 2001 From: Dee Date: Sun, 26 May 2024 01:22:11 +0700 Subject: [PATCH 1/7] feat: add hook for schemathesis tests --- .gitignore | 3 +++ catalyst-gateway/tests/Earthfile | 4 +++- catalyst-gateway/tests/api_tests/.gitignore | 3 ++- .../tests/api_tests/api_tests/hooks.py | 24 +++++++++++++++++++ 4 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 catalyst-gateway/tests/api_tests/api_tests/hooks.py diff --git a/.gitignore b/.gitignore index d6aba0034f7..ee537b0c120 100644 --- a/.gitignore +++ b/.gitignore @@ -107,3 +107,6 @@ dev-catalyst-voice-9f78f27c6bc5.json # Specifically exclude it in the directory it appears, if its required. # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html Cargo.lock + +#Hypothesis +.hypothesis diff --git a/catalyst-gateway/tests/Earthfile b/catalyst-gateway/tests/Earthfile index 2713223062f..4dfc90ae1ff 100644 --- a/catalyst-gateway/tests/Earthfile +++ b/catalyst-gateway/tests/Earthfile @@ -15,9 +15,11 @@ package-schemathesis: RUN apk add --no-cache gcc musl-dev RUN python -m pip install schemathesis==3.27.1 RUN mkdir /results + COPY ./api_tests/api_tests/hooks.py . VOLUME /results - ENTRYPOINT st run --checks all $openapi_spec \ + ENTRYPOINT export SCHEMATHESIS_HOOKS=hooks \ + && st run --checks all $openapi_spec \ --workers=$workers \ --wait-for-schema=$wait_for_schema \ --max-response-time=$max_response_time \ diff --git a/catalyst-gateway/tests/api_tests/.gitignore b/catalyst-gateway/tests/api_tests/.gitignore index 40f65ddf375..3b9825d2f9c 100644 --- a/catalyst-gateway/tests/api_tests/.gitignore +++ b/catalyst-gateway/tests/api_tests/.gitignore @@ -1,2 +1,3 @@ __pycache__ -junit-report.xml \ No newline at end of file +junit-report.xml +.hypothesis/ \ No newline at end of file diff --git a/catalyst-gateway/tests/api_tests/api_tests/hooks.py b/catalyst-gateway/tests/api_tests/api_tests/hooks.py new file mode 100644 index 00000000000..a70b1978359 --- /dev/null +++ b/catalyst-gateway/tests/api_tests/api_tests/hooks.py @@ -0,0 +1,24 @@ +import copy +import schemathesis +import schemathesis.schemas +from typing import Any, Dict + +@schemathesis.hook +def before_load_schema( + context: schemathesis.hooks.HookContext, + raw_schema: Dict[str, Any], + ) -> None: + paths = dict(raw_schema["paths"]) + for key in paths.keys(): + values = dict(raw_schema["paths"][key]) + if "get" in values: + ref_info = raw_schema["paths"][key]["get"] + new_info = copy.deepcopy(ref_info) + new_info["responses"] = {"405": {"description": "method not allowed"}} + raw_schema["paths"][key] = {"get": ref_info, "post": new_info} + + else: + ref_info = raw_schema["paths"][key]["post"] + new_info = copy.deepcopy(ref_info) + new_info["responses"] = {"405": {"description": "method not allowed"}} + raw_schema["paths"][key] = {"post": ref_info, "get": new_info} From 3171957f0e44d80e19113c302e36bff803a16609 Mon Sep 17 00:00:00 2001 From: Dee Date: Mon, 27 May 2024 17:02:46 +0700 Subject: [PATCH 2/7] feat: add put patch delete method, refactor --- .../tests/api_tests/api_tests/hooks.py | 36 ++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/catalyst-gateway/tests/api_tests/api_tests/hooks.py b/catalyst-gateway/tests/api_tests/api_tests/hooks.py index a70b1978359..3b36f2bbbee 100644 --- a/catalyst-gateway/tests/api_tests/api_tests/hooks.py +++ b/catalyst-gateway/tests/api_tests/api_tests/hooks.py @@ -9,16 +9,28 @@ def before_load_schema( raw_schema: Dict[str, Any], ) -> None: paths = dict(raw_schema["paths"]) - for key in paths.keys(): - values = dict(raw_schema["paths"][key]) - if "get" in values: - ref_info = raw_schema["paths"][key]["get"] - new_info = copy.deepcopy(ref_info) - new_info["responses"] = {"405": {"description": "method not allowed"}} - raw_schema["paths"][key] = {"get": ref_info, "post": new_info} + for endpoint in paths.keys(): + endpoint_data = dict(raw_schema["paths"][endpoint]) - else: - ref_info = raw_schema["paths"][key]["post"] - new_info = copy.deepcopy(ref_info) - new_info["responses"] = {"405": {"description": "method not allowed"}} - raw_schema["paths"][key] = {"post": ref_info, "get": new_info} + # Get first method data as reference + ref_info = next(iter(endpoint_data.values())) + + # Create new method data using reference data, replacing 'responses' field + new_info = copy.deepcopy(ref_info) + new_info["responses"] = {"405": {"description": "method not allowed"}} + + # Update endpoint if a method is not declared + if "get" not in endpoint_data: + endpoint_data.update([("get", new_info),]) + if "post" not in endpoint_data: + endpoint_data.update([("post", new_info),]) + if "put" not in endpoint_data: + endpoint_data.update([("put", new_info),]) + if "patch" not in endpoint_data: + endpoint_data.update([("patch", new_info),]) + if "delete" not in endpoint_data: + endpoint_data.update([("patch", new_info),]) + + # Update endpoint + raw_schema["paths"][endpoint] = endpoint_data + From c5afa0f3ad178802a0a6731bf3982181b39eaa6e Mon Sep 17 00:00:00 2001 From: Dee Date: Mon, 27 May 2024 17:54:45 +0700 Subject: [PATCH 3/7] fix: change patch to delete --- catalyst-gateway/tests/api_tests/api_tests/hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst-gateway/tests/api_tests/api_tests/hooks.py b/catalyst-gateway/tests/api_tests/api_tests/hooks.py index 3b36f2bbbee..1948e9d6db0 100644 --- a/catalyst-gateway/tests/api_tests/api_tests/hooks.py +++ b/catalyst-gateway/tests/api_tests/api_tests/hooks.py @@ -29,7 +29,7 @@ def before_load_schema( if "patch" not in endpoint_data: endpoint_data.update([("patch", new_info),]) if "delete" not in endpoint_data: - endpoint_data.update([("patch", new_info),]) + endpoint_data.update([("delete", new_info),]) # Update endpoint raw_schema["paths"][endpoint] = endpoint_data From 95aee14c6f5bcc98472bb11cfb7233d0edad3202 Mon Sep 17 00:00:00 2001 From: Dee Date: Tue, 28 May 2024 10:48:10 +0700 Subject: [PATCH 4/7] feat: move test-fuzzer-api to new folder --- .github/workflows/generate-allure-report.yml | 2 +- catalyst-gateway/tests/Earthfile | 61 ------------------ .../tests/schemathesis_tests/.gitignore | 3 + .../tests/schemathesis_tests/Earthfile | 62 +++++++++++++++++++ .../hooks}/hooks.py | 0 .../schemathesis-docker-compose.yml | 0 6 files changed, 66 insertions(+), 62 deletions(-) create mode 100644 catalyst-gateway/tests/schemathesis_tests/.gitignore create mode 100644 catalyst-gateway/tests/schemathesis_tests/Earthfile rename catalyst-gateway/tests/{api_tests/api_tests => schemathesis_tests/hooks}/hooks.py (100%) rename catalyst-gateway/tests/{ => schemathesis_tests}/schemathesis-docker-compose.yml (100%) diff --git a/.github/workflows/generate-allure-report.yml b/.github/workflows/generate-allure-report.yml index fec4a0ca2d9..89ec6926f55 100644 --- a/.github/workflows/generate-allure-report.yml +++ b/.github/workflows/generate-allure-report.yml @@ -53,7 +53,7 @@ jobs: if: always() continue-on-error: true with: - earthfile: ./catalyst-gateway/tests/ + earthfile: ./catalyst-gateway/tests/schemathesis_tests flags: --allow-privileged targets: test-fuzzer-api target_flags: diff --git a/catalyst-gateway/tests/Earthfile b/catalyst-gateway/tests/Earthfile index 4dfc90ae1ff..8e3c08596d4 100644 --- a/catalyst-gateway/tests/Earthfile +++ b/catalyst-gateway/tests/Earthfile @@ -1,66 +1,5 @@ VERSION 0.8 -package-schemathesis: - FROM python:3.12-alpine3.19 - - ARG tag="latest" - ARG max_examples=1000 - # TODO: https://github.com/input-output-hk/catalyst-voices/issues/465 - ARG max_response_time=1000 - ARG wait_for_schema=15 - ARG workers=2 - ARG schema_version=30 - ARG openapi_spec - - RUN apk add --no-cache gcc musl-dev - RUN python -m pip install schemathesis==3.27.1 - RUN mkdir /results - COPY ./api_tests/api_tests/hooks.py . - - VOLUME /results - ENTRYPOINT export SCHEMATHESIS_HOOKS=hooks \ - && st run --checks all $openapi_spec \ - --workers=$workers \ - --wait-for-schema=$wait_for_schema \ - --max-response-time=$max_response_time \ - --hypothesis-max-examples=$max_examples \ - --data-generation-method=all \ - --skip-deprecated-operations \ - --force-schema-version=$schema_version \ - --show-trace \ - --force-color \ - --junit-xml=/results/junit-report.xml \ - --cassette-path=/results/cassette.yaml - - SAVE IMAGE schemathesis:$tag - -# test-fuzzer-api - Fuzzy test cat-gateway using openapi specs -test-fuzzer-api: - FROM earthly/dind:alpine-3.19 - RUN apk update && apk add iptables-legacy # workaround for https://github.com/earthly/earthly/issues/3784 - RUN apk add yq zstd - ARG OPENAPI_SPEC="http://127.0.0.1:3030/docs/cat-gateway.json" - - COPY schemathesis-docker-compose.yml . - WITH DOCKER \ - --compose schemathesis-docker-compose.yml \ - --load schemathesis:latest=(+package-schemathesis --openapi_spec=$OPENAPI_SPEC) \ - --load event-db:latest=(../event-db+build) \ - --load cat-gateway:latest=(../+package-cat-gateway) \ - --service event-db \ - --service cat-gateway \ - --allow-privileged - RUN docker run --net=host --name=st schemathesis:latest || echo fail > fail && \ - docker-compose logs cat-gateway > ./cat-gateway.log && zstd -9 cat-gateway.log && \ - docker cp st:/results/junit-report.xml junit-report.xml && \ - docker cp st:/results/cassette.yaml cassette.yaml - END - WAIT - SAVE ARTIFACT junit-report.xml AS LOCAL schemathesis.junit-report.xml - SAVE ARTIFACT cat-gateway.log.zst AS LOCAL cat-gateway.log.zst - SAVE ARTIFACT cassette.yaml AS LOCAL cassette.yaml - END - # test-lint-openapi - OpenAPI linting from an artifact # testing whether the OpenAPI generated during build stage follows good practice. test-lint-openapi: diff --git a/catalyst-gateway/tests/schemathesis_tests/.gitignore b/catalyst-gateway/tests/schemathesis_tests/.gitignore new file mode 100644 index 00000000000..3b9825d2f9c --- /dev/null +++ b/catalyst-gateway/tests/schemathesis_tests/.gitignore @@ -0,0 +1,3 @@ +__pycache__ +junit-report.xml +.hypothesis/ \ No newline at end of file diff --git a/catalyst-gateway/tests/schemathesis_tests/Earthfile b/catalyst-gateway/tests/schemathesis_tests/Earthfile new file mode 100644 index 00000000000..35064e7926a --- /dev/null +++ b/catalyst-gateway/tests/schemathesis_tests/Earthfile @@ -0,0 +1,62 @@ +VERSION 0.8 + +package-schemathesis: + FROM python:3.12-alpine3.19 + + ARG tag="latest" + ARG max_examples=1000 + # TODO: https://github.com/input-output-hk/catalyst-voices/issues/465 + ARG max_response_time=1000 + ARG wait_for_schema=15 + ARG workers=2 + ARG schema_version=30 + ARG openapi_spec + + RUN apk add --no-cache gcc musl-dev + RUN python -m pip install schemathesis==3.27.1 + RUN mkdir /results + COPY ./hooks/hooks.py . + + VOLUME /results + ENTRYPOINT export SCHEMATHESIS_HOOKS=hooks \ + && st run --checks all $openapi_spec \ + --workers=$workers \ + --wait-for-schema=$wait_for_schema \ + --max-response-time=$max_response_time \ + --hypothesis-max-examples=$max_examples \ + --data-generation-method=all \ + # --skip-deprecated-operations \ + --force-schema-version=$schema_version \ + --show-trace \ + --force-color \ + --junit-xml=/results/junit-report.xml \ + --cassette-path=/results/cassette.yaml + + SAVE IMAGE schemathesis:$tag + +# test-fuzzer-api - Fuzzy test cat-gateway using openapi specs +test-fuzzer-api: + FROM earthly/dind:alpine-3.19 + RUN apk update && apk add iptables-legacy # workaround for https://github.com/earthly/earthly/issues/3784 + RUN apk add yq zstd + ARG OPENAPI_SPEC="http://127.0.0.1:3030/docs/cat-gateway.json" + + COPY schemathesis-docker-compose.yml . + WITH DOCKER \ + --compose schemathesis-docker-compose.yml \ + --load schemathesis:latest=(+package-schemathesis --openapi_spec=$OPENAPI_SPEC) \ + --load event-db:latest=(../../event-db+build) \ + --load cat-gateway:latest=(../../+package-cat-gateway) \ + --service event-db \ + --service cat-gateway \ + --allow-privileged + RUN docker run --net=host --name=st schemathesis:latest || echo fail > fail && \ + docker-compose logs cat-gateway > ./cat-gateway.log && zstd -9 cat-gateway.log && \ + docker cp st:/results/junit-report.xml junit-report.xml && \ + docker cp st:/results/cassette.yaml cassette.yaml + END + WAIT + SAVE ARTIFACT junit-report.xml AS LOCAL schemathesis.junit-report.xml + SAVE ARTIFACT cat-gateway.log.zst AS LOCAL cat-gateway.log.zst + SAVE ARTIFACT cassette.yaml AS LOCAL cassette.yaml + END diff --git a/catalyst-gateway/tests/api_tests/api_tests/hooks.py b/catalyst-gateway/tests/schemathesis_tests/hooks/hooks.py similarity index 100% rename from catalyst-gateway/tests/api_tests/api_tests/hooks.py rename to catalyst-gateway/tests/schemathesis_tests/hooks/hooks.py diff --git a/catalyst-gateway/tests/schemathesis-docker-compose.yml b/catalyst-gateway/tests/schemathesis_tests/schemathesis-docker-compose.yml similarity index 100% rename from catalyst-gateway/tests/schemathesis-docker-compose.yml rename to catalyst-gateway/tests/schemathesis_tests/schemathesis-docker-compose.yml From 60b1ef50c939592bc97d8508a2680ef1f9f0a579 Mon Sep 17 00:00:00 2001 From: Dee Date: Tue, 28 May 2024 10:48:31 +0700 Subject: [PATCH 5/7] fix: uncomment code --- catalyst-gateway/tests/schemathesis_tests/Earthfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst-gateway/tests/schemathesis_tests/Earthfile b/catalyst-gateway/tests/schemathesis_tests/Earthfile index 35064e7926a..0526cc5587b 100644 --- a/catalyst-gateway/tests/schemathesis_tests/Earthfile +++ b/catalyst-gateway/tests/schemathesis_tests/Earthfile @@ -25,7 +25,7 @@ package-schemathesis: --max-response-time=$max_response_time \ --hypothesis-max-examples=$max_examples \ --data-generation-method=all \ - # --skip-deprecated-operations \ + --skip-deprecated-operations \ --force-schema-version=$schema_version \ --show-trace \ --force-color \ From ccfb4b43602950ecb1b38af9aa04c87b8472c0ab Mon Sep 17 00:00:00 2001 From: Dee Date: Wed, 29 May 2024 10:47:52 +0700 Subject: [PATCH 6/7] feat: update max-response-time --- catalyst-gateway/tests/schemathesis_tests/Earthfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst-gateway/tests/schemathesis_tests/Earthfile b/catalyst-gateway/tests/schemathesis_tests/Earthfile index 0526cc5587b..161ee411a9a 100644 --- a/catalyst-gateway/tests/schemathesis_tests/Earthfile +++ b/catalyst-gateway/tests/schemathesis_tests/Earthfile @@ -6,7 +6,7 @@ package-schemathesis: ARG tag="latest" ARG max_examples=1000 # TODO: https://github.com/input-output-hk/catalyst-voices/issues/465 - ARG max_response_time=1000 + ARG max_response_time=3000 ARG wait_for_schema=15 ARG workers=2 ARG schema_version=30 From e22e0310012c5d976c592986e0b5f1e7a9539326 Mon Sep 17 00:00:00 2001 From: Dee Date: Wed, 29 May 2024 10:54:08 +0700 Subject: [PATCH 7/7] fix: syntax --- catalyst-gateway/tests/schemathesis_tests/Earthfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst-gateway/tests/schemathesis_tests/Earthfile b/catalyst-gateway/tests/schemathesis_tests/Earthfile index 161ee411a9a..b98d1d4f76b 100644 --- a/catalyst-gateway/tests/schemathesis_tests/Earthfile +++ b/catalyst-gateway/tests/schemathesis_tests/Earthfile @@ -18,7 +18,7 @@ package-schemathesis: COPY ./hooks/hooks.py . VOLUME /results - ENTRYPOINT export SCHEMATHESIS_HOOKS=hooks \ + ENTRYPOINT export SCHEMATHESIS_HOOKS=hooks \ && st run --checks all $openapi_spec \ --workers=$workers \ --wait-for-schema=$wait_for_schema \