From 9a1d081582a7ac1acfb569a51837f768fb37e48d Mon Sep 17 00:00:00 2001 From: minikin Date: Fri, 18 Oct 2024 16:25:20 +0200 Subject: [PATCH 01/37] ci: add project-fields-validator --- utilities/project-fields-validator/Earthfile | 32 +++ utilities/project-fields-validator/README.md | 0 utilities/project-fields-validator/main.py | 83 ++++++++ .../project-fields-validator/poetry.lock | 183 ++++++++++++++++++ .../project-fields-validator/pyproject.toml | 14 ++ 5 files changed, 312 insertions(+) create mode 100644 utilities/project-fields-validator/Earthfile create mode 100644 utilities/project-fields-validator/README.md create mode 100644 utilities/project-fields-validator/main.py create mode 100644 utilities/project-fields-validator/poetry.lock create mode 100644 utilities/project-fields-validator/pyproject.toml diff --git a/utilities/project-fields-validator/Earthfile b/utilities/project-fields-validator/Earthfile new file mode 100644 index 000000000..113bc019a --- /dev/null +++ b/utilities/project-fields-validator/Earthfile @@ -0,0 +1,32 @@ +VERSION 0.8 + +IMPORT github.com/input-output-hk/catalyst-ci/earthly/python:v3.1.7 AS python-ci + +check: + FROM python-ci+python-base + + COPY . . + + DO python-ci+CHECK + +VALIDATE_PROJECTS_FIELDS: + FUNCTION + + FROM python-ci+python-base + + COPY . . + + # Set environment variables + ENV ORG_NAME=input-output-hk + ENV PROJECT_NUMBER=102 + ARG PR_NUMBER + + + RUN python3 main.py + + # Check the validation results + RUN if [ -f ".github/project_validation_results.txt" ]; then \ + echo "Validation failed. See results:" && \ + cat .github/project_validation_results.txt && \ + exit 1; \ + fi \ No newline at end of file diff --git a/utilities/project-fields-validator/README.md b/utilities/project-fields-validator/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/utilities/project-fields-validator/main.py b/utilities/project-fields-validator/main.py new file mode 100644 index 000000000..7234c99a8 --- /dev/null +++ b/utilities/project-fields-validator/main.py @@ -0,0 +1,83 @@ +import os +import requests +import json + +def run_query(query, variables): + headers = {"Authorization": f"Bearer {os.environ['GITHUB_TOKEN']}"} + request = requests.post('https://api.github.com/graphql', json={'query': query, 'variables': variables}, headers=headers) + if request.status_code == 200: + return request.json() + else: + raise Exception(f"Query failed with status code: {request.status_code}. {request.json()}") + +def get_pr_related_items(org_name, project_number, pr_number): + query = """ + query($org: String!, $number: Int!, $prNumber: Int!) { + organization(login: $org) { + projectV2(number: $number) { + items(first: 100) { + nodes { + id + content { + ... on PullRequest { + number + } + } + fieldValues(first: 20) { + nodes { + ... on ProjectV2ItemFieldTextValue { + field { name } + text + } + ... on ProjectV2ItemFieldDateValue { + field { name } + date + } + ... on ProjectV2ItemFieldSingleSelectValue { + field { name } + name + } + } + } + } + } + } + } + } + """ + variables = { + "org": org_name, + "number": project_number, + "prNumber": pr_number + } + result = run_query(query, variables) + items = result['data']['organization']['projectV2']['items']['nodes'] + return [item for item in items if item['content'] and item['content'].get('number') == pr_number] + +def validate_item(item): + required_fields = ['Status', 'Area', 'Priority', 'Estimate', 'Iteration', 'Start', 'End'] + field_values = {fv['field']['name']: fv.get('text') or fv.get('date') or fv.get('name') + for fv in item['fieldValues']['nodes']} + empty_fields = [field for field in required_fields if not field_values.get(field)] + return empty_fields + +def main(): + org_name = os.environ['ORG_NAME'] + project_number = int(os.environ['PROJECT_NUMBER']) + pr_number = int(os.environ['PR_NUMBER']) + + items = get_pr_related_items(org_name, project_number, pr_number) + + validation_errors = [] + for item in items: + empty_fields = validate_item(item) + if empty_fields: + validation_errors.append(f"Item ID: {item['id']}, Empty fields: {', '.join(empty_fields)}") + + if validation_errors: + with open('.github/project_validation_results.txt', 'w') as f: + for error in validation_errors: + f.write(f"{error}\n") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/utilities/project-fields-validator/poetry.lock b/utilities/project-fields-validator/poetry.lock new file mode 100644 index 000000000..428a70ee1 --- /dev/null +++ b/utilities/project-fields-validator/poetry.lock @@ -0,0 +1,183 @@ +# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. + +[[package]] +name = "certifi" +version = "2024.8.30" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"}, + {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.0" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:4f9fc98dad6c2eaa32fc3af1417d95b5e3d08aff968df0cd320066def971f9a6"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0de7b687289d3c1b3e8660d0741874abe7888100efe14bd0f9fd7141bcbda92b"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ed2e36c3e9b4f21dd9422f6893dec0abf2cca553af509b10cd630f878d3eb99"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40d3ff7fc90b98c637bda91c89d51264a3dcf210cade3a2c6f838c7268d7a4ca"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1110e22af8ca26b90bd6364fe4c763329b0ebf1ee213ba32b68c73de5752323d"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:86f4e8cca779080f66ff4f191a685ced73d2f72d50216f7112185dc02b90b9b7"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f683ddc7eedd742e2889d2bfb96d69573fde1d92fcb811979cdb7165bb9c7d3"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:27623ba66c183eca01bf9ff833875b459cad267aeeb044477fedac35e19ba907"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f606a1881d2663630ea5b8ce2efe2111740df4b687bd78b34a8131baa007f79b"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:0b309d1747110feb25d7ed6b01afdec269c647d382c857ef4663bbe6ad95a912"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:136815f06a3ae311fae551c3df1f998a1ebd01ddd424aa5603a4336997629e95"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:14215b71a762336254351b00ec720a8e85cada43b987da5a042e4ce3e82bd68e"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:79983512b108e4a164b9c8d34de3992f76d48cadc9554c9e60b43f308988aabe"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-win32.whl", hash = "sha256:c94057af19bc953643a33581844649a7fdab902624d2eb739738a30e2b3e60fc"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:55f56e2ebd4e3bc50442fbc0888c9d8c94e4e06a933804e2af3e89e2f9c1c749"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0d99dd8ff461990f12d6e42c7347fd9ab2532fb70e9621ba520f9e8637161d7c"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c57516e58fd17d03ebe67e181a4e4e2ccab1168f8c2976c6a334d4f819fe5944"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6dba5d19c4dfab08e58d5b36304b3f92f3bd5d42c1a3fa37b5ba5cdf6dfcbcee"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf4475b82be41b07cc5e5ff94810e6a01f276e37c2d55571e3fe175e467a1a1c"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ce031db0408e487fd2775d745ce30a7cd2923667cf3b69d48d219f1d8f5ddeb6"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ff4e7cdfdb1ab5698e675ca622e72d58a6fa2a8aa58195de0c0061288e6e3ea"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3710a9751938947e6327ea9f3ea6332a09bf0ba0c09cae9cb1f250bd1f1549bc"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82357d85de703176b5587dbe6ade8ff67f9f69a41c0733cf2425378b49954de5"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47334db71978b23ebcf3c0f9f5ee98b8d65992b65c9c4f2d34c2eaf5bcaf0594"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8ce7fd6767a1cc5a92a639b391891bf1c268b03ec7e021c7d6d902285259685c"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1a2f519ae173b5b6a2c9d5fa3116ce16e48b3462c8b96dfdded11055e3d6365"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:63bc5c4ae26e4bc6be6469943b8253c0fd4e4186c43ad46e713ea61a0ba49129"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bcb4f8ea87d03bc51ad04add8ceaf9b0f085ac045ab4d74e73bbc2dc033f0236"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-win32.whl", hash = "sha256:9ae4ef0b3f6b41bad6366fb0ea4fc1d7ed051528e113a60fa2a65a9abb5b1d99"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:cee4373f4d3ad28f1ab6290684d8e2ebdb9e7a1b74fdc39e4c211995f77bec27"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0713f3adb9d03d49d365b70b84775d0a0d18e4ab08d12bc46baa6132ba78aaf6"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:de7376c29d95d6719048c194a9cf1a1b0393fbe8488a22008610b0361d834ecf"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4a51b48f42d9358460b78725283f04bddaf44a9358197b889657deba38f329db"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b295729485b06c1a0683af02a9e42d2caa9db04a373dc38a6a58cdd1e8abddf1"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ee803480535c44e7f5ad00788526da7d85525cfefaf8acf8ab9a310000be4b03"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3d59d125ffbd6d552765510e3f31ed75ebac2c7470c7274195b9161a32350284"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cda06946eac330cbe6598f77bb54e690b4ca93f593dee1568ad22b04f347c15"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07afec21bbbbf8a5cc3651aa96b980afe2526e7f048fdfb7f1014d84acc8b6d8"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6b40e8d38afe634559e398cc32b1472f376a4099c75fe6299ae607e404c033b2"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b8dcd239c743aa2f9c22ce674a145e0a25cb1566c495928440a181ca1ccf6719"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:84450ba661fb96e9fd67629b93d2941c871ca86fc38d835d19d4225ff946a631"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:44aeb140295a2f0659e113b31cfe92c9061622cadbc9e2a2f7b8ef6b1e29ef4b"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1db4e7fefefd0f548d73e2e2e041f9df5c59e178b4c72fbac4cc6f535cfb1565"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-win32.whl", hash = "sha256:5726cf76c982532c1863fb64d8c6dd0e4c90b6ece9feb06c9f202417a31f7dd7"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:b197e7094f232959f8f20541ead1d9862ac5ebea1d58e9849c1bf979255dfac9"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:dd4eda173a9fcccb5f2e2bd2a9f423d180194b1bf17cf59e3269899235b2a114"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e9e3c4c9e1ed40ea53acf11e2a386383c3304212c965773704e4603d589343ed"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92a7e36b000bf022ef3dbb9c46bfe2d52c047d5e3f3343f43204263c5addc250"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54b6a92d009cbe2fb11054ba694bc9e284dad30a26757b1e372a1fdddaf21920"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ffd9493de4c922f2a38c2bf62b831dcec90ac673ed1ca182fe11b4d8e9f2a64"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:35c404d74c2926d0287fbd63ed5d27eb911eb9e4a3bb2c6d294f3cfd4a9e0c23"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4796efc4faf6b53a18e3d46343535caed491776a22af773f366534056c4e1fbc"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e7fdd52961feb4c96507aa649550ec2a0d527c086d284749b2f582f2d40a2e0d"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:92db3c28b5b2a273346bebb24857fda45601aef6ae1c011c0a997106581e8a88"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ab973df98fc99ab39080bfb0eb3a925181454d7c3ac8a1e695fddfae696d9e90"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b67fdab07fdd3c10bb21edab3cbfe8cf5696f453afce75d815d9d7223fbe88b"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:aa41e526a5d4a9dfcfbab0716c7e8a1b215abd3f3df5a45cf18a12721d31cb5d"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffc519621dce0c767e96b9c53f09c5d215578e10b02c285809f76509a3931482"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-win32.whl", hash = "sha256:f19c1585933c82098c2a520f8ec1227f20e339e33aca8fa6f956f6691b784e67"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:707b82d19e65c9bd28b81dde95249b07bf9f5b90ebe1ef17d9b57473f8a64b7b"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:dbe03226baf438ac4fda9e2d0715022fd579cb641c4cf639fa40d53b2fe6f3e2"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd9a8bd8900e65504a305bf8ae6fa9fbc66de94178c420791d0293702fce2df7"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8831399554b92b72af5932cdbbd4ddc55c55f631bb13ff8fe4e6536a06c5c51"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a14969b8691f7998e74663b77b4c36c0337cb1df552da83d5c9004a93afdb574"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dcaf7c1524c0542ee2fc82cc8ec337f7a9f7edee2532421ab200d2b920fc97cf"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:425c5f215d0eecee9a56cdb703203dda90423247421bf0d67125add85d0c4455"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:d5b054862739d276e09928de37c79ddeec42a6e1bfc55863be96a36ba22926f6"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:f3e73a4255342d4eb26ef6df01e3962e73aa29baa3124a8e824c5d3364a65748"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:2f6c34da58ea9c1a9515621f4d9ac379871a8f21168ba1b5e09d74250de5ad62"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_s390x.whl", hash = "sha256:f09cb5a7bbe1ecae6e87901a2eb23e0256bb524a79ccc53eb0b7629fbe7677c4"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:0099d79bdfcf5c1f0c2c72f91516702ebf8b0b8ddd8905f97a8aecf49712c621"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-win32.whl", hash = "sha256:9c98230f5042f4945f957d006edccc2af1e03ed5e37ce7c373f00a5a4daa6149"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-win_amd64.whl", hash = "sha256:62f60aebecfc7f4b82e3f639a7d1433a20ec32824db2199a11ad4f5e146ef5ee"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:af73657b7a68211996527dbfeffbb0864e043d270580c5aef06dc4b659a4b578"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cab5d0b79d987c67f3b9e9c53f54a61360422a5a0bc075f43cab5621d530c3b6"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9289fd5dddcf57bab41d044f1756550f9e7cf0c8e373b8cdf0ce8773dc4bd417"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b493a043635eb376e50eedf7818f2f322eabbaa974e948bd8bdd29eb7ef2a51"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fa2566ca27d67c86569e8c85297aaf413ffab85a8960500f12ea34ff98e4c41"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8e538f46104c815be19c975572d74afb53f29650ea2025bbfaef359d2de2f7f"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fd30dc99682dc2c603c2b315bded2799019cea829f8bf57dc6b61efde6611c8"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2006769bd1640bdf4d5641c69a3d63b71b81445473cac5ded39740a226fa88ab"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:dc15e99b2d8a656f8e666854404f1ba54765871104e50c8e9813af8a7db07f12"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:ab2e5bef076f5a235c3774b4f4028a680432cded7cad37bba0fd90d64b187d19"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:4ec9dd88a5b71abfc74e9df5ebe7921c35cbb3b641181a531ca65cdb5e8e4dea"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:43193c5cda5d612f247172016c4bb71251c784d7a4d9314677186a838ad34858"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:aa693779a8b50cd97570e5a0f343538a8dbd3e496fa5dcb87e29406ad0299654"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-win32.whl", hash = "sha256:7706f5850360ac01d80c89bcef1640683cc12ed87f42579dab6c5d3ed6888613"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:c3e446d253bd88f6377260d07c895816ebf33ffffd56c1c792b13bff9c3e1ade"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:980b4f289d1d90ca5efcf07958d3eb38ed9c0b7676bf2831a54d4f66f9c27dfa"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f28f891ccd15c514a0981f3b9db9aa23d62fe1a99997512b0491d2ed323d229a"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8aacce6e2e1edcb6ac625fb0f8c3a9570ccc7bfba1f63419b3769ccf6a00ed0"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd7af3717683bea4c87acd8c0d3d5b44d56120b26fd3f8a692bdd2d5260c620a"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5ff2ed8194587faf56555927b3aa10e6fb69d931e33953943bc4f837dfee2242"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e91f541a85298cf35433bf66f3fab2a4a2cff05c127eeca4af174f6d497f0d4b"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:309a7de0a0ff3040acaebb35ec45d18db4b28232f21998851cfa709eeff49d62"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:285e96d9d53422efc0d7a17c60e59f37fbf3dfa942073f666db4ac71e8d726d0"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:5d447056e2ca60382d460a604b6302d8db69476fd2015c81e7c35417cfabe4cd"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:20587d20f557fe189b7947d8e7ec5afa110ccf72a3128d61a2a387c3313f46be"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:130272c698667a982a5d0e626851ceff662565379baf0ff2cc58067b81d4f11d"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:ab22fbd9765e6954bc0bcff24c25ff71dcbfdb185fcdaca49e81bac68fe724d3"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7782afc9b6b42200f7362858f9e73b1f8316afb276d316336c0ec3bd73312742"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-win32.whl", hash = "sha256:2de62e8801ddfff069cd5c504ce3bc9672b23266597d4e4f50eda28846c322f2"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:95c3c157765b031331dd4db3c775e58deaee050a3042fcad72cbc4189d7c8dca"}, + {file = "charset_normalizer-3.4.0-py3-none-any.whl", hash = "sha256:fe9f97feb71aa9896b81973a7bbada8c49501dc73e58a10fcef6663af95e5079"}, + {file = "charset_normalizer-3.4.0.tar.gz", hash = "sha256:223217c3d4f82c3ac5e29032b3f1c2eb0fb591b72161f86d93f5719079dae93e"}, +] + +[[package]] +name = "idna" +version = "3.10" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.6" +files = [ + {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, + {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, +] + +[package.extras] +all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] + +[[package]] +name = "requests" +version = "2.32.3" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.8" +files = [ + {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, + {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "urllib3" +version = "2.2.3" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.3-py3-none-any.whl", hash = "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac"}, + {file = "urllib3-2.2.3.tar.gz", hash = "sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.11" +content-hash = "8f6b6c24f619f4b3c83feba88e6f3e8b31d33569993e9e520ccb70cf6c4799c5" diff --git a/utilities/project-fields-validator/pyproject.toml b/utilities/project-fields-validator/pyproject.toml new file mode 100644 index 000000000..77de0c760 --- /dev/null +++ b/utilities/project-fields-validator/pyproject.toml @@ -0,0 +1,14 @@ +[tool.poetry] +name = "project-fields-validator" +version = "0.1.0" +description = "" +authors = ["Catalyst Team"] +readme = "README.md" + +[tool.poetry.dependencies] +python = "^3.11" +requests = "^2.32.3" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" From 1f8dbcb02c9188020ee27c92c7de857a3ec048f8 Mon Sep 17 00:00:00 2001 From: minikin Date: Mon, 21 Oct 2024 15:17:26 +0200 Subject: [PATCH 02/37] chore: update python and earthly --- utilities/project-fields-validator/Earthfile | 10 +-- utilities/project-fields-validator/main.py | 71 +++++++++++++++++--- 2 files changed, 68 insertions(+), 13 deletions(-) diff --git a/utilities/project-fields-validator/Earthfile b/utilities/project-fields-validator/Earthfile index 113bc019a..4f029279b 100644 --- a/utilities/project-fields-validator/Earthfile +++ b/utilities/project-fields-validator/Earthfile @@ -9,20 +9,20 @@ check: DO python-ci+CHECK -VALIDATE_PROJECTS_FIELDS: - FUNCTION - +validate-pr: FROM python-ci+python-base COPY . . + RUN pip install requests + # Set environment variables ENV ORG_NAME=input-output-hk ENV PROJECT_NUMBER=102 ARG PR_NUMBER + ARG GITHUB_TOKEN - - RUN python3 main.py + RUN --secret GITHUB_TOKEN python3 main.py # Check the validation results RUN if [ -f ".github/project_validation_results.txt" ]; then \ diff --git a/utilities/project-fields-validator/main.py b/utilities/project-fields-validator/main.py index 7234c99a8..a8fe04e15 100644 --- a/utilities/project-fields-validator/main.py +++ b/utilities/project-fields-validator/main.py @@ -1,4 +1,5 @@ import os +import sys import requests import json @@ -8,7 +9,9 @@ def run_query(query, variables): if request.status_code == 200: return request.json() else: - raise Exception(f"Query failed with status code: {request.status_code}. {request.json()}") + print(f"Query failed with status code: {request.status_code}") + print(f"Response: {request.text}") + raise Exception(f"Query failed with status code: {request.status_code}") def get_pr_related_items(org_name, project_number, pr_number): query = """ @@ -21,20 +24,34 @@ def get_pr_related_items(org_name, project_number, pr_number): content { ... on PullRequest { number + title + url } } fieldValues(first: 20) { nodes { ... on ProjectV2ItemFieldTextValue { - field { name } + field { + ... on ProjectV2FieldCommon { + name + } + } text } ... on ProjectV2ItemFieldDateValue { - field { name } + field { + ... on ProjectV2FieldCommon { + name + } + } date } ... on ProjectV2ItemFieldSingleSelectValue { - field { name } + field { + ... on ProjectV2FieldCommon { + name + } + } name } } @@ -43,6 +60,13 @@ def get_pr_related_items(org_name, project_number, pr_number): } } } + repository(owner: $org, name: "catalyst-voices") { + pullRequest(number: $prNumber) { + number + title + url + } + } } """ variables = { @@ -51,16 +75,37 @@ def get_pr_related_items(org_name, project_number, pr_number): "prNumber": pr_number } result = run_query(query, variables) + print(f"API Response: {json.dumps(result, indent=2)}") + if 'data' not in result: + print(f"Error in API response: {result.get('errors', 'Unknown error')}") + return [] + + pr_info = result['data']['repository']['pullRequest'] + print(f"Found PR: #{pr_info['number']} - {pr_info['title']} ({pr_info['url']})") + items = result['data']['organization']['projectV2']['items']['nodes'] - return [item for item in items if item['content'] and item['content'].get('number') == pr_number] + pr_items = [item for item in items if item['content'] and isinstance(item['content'], dict) and item['content'].get('number') == pr_number] + + if not pr_items: + print(f"PR #{pr_number} exists but is not linked to the project. Please add it to the project.") + return pr_items def validate_item(item): required_fields = ['Status', 'Area', 'Priority', 'Estimate', 'Iteration', 'Start', 'End'] - field_values = {fv['field']['name']: fv.get('text') or fv.get('date') or fv.get('name') - for fv in item['fieldValues']['nodes']} + field_values = {} + for field_value in item['fieldValues']['nodes']: + field_name = field_value['field']['name'] + if 'text' in field_value: + field_values[field_name] = field_value['text'] + elif 'date' in field_value: + field_values[field_name] = field_value['date'] + elif 'name' in field_value: + field_values[field_name] = field_value['name'] + empty_fields = [field for field in required_fields if not field_values.get(field)] return empty_fields + def main(): org_name = os.environ['ORG_NAME'] project_number = int(os.environ['PROJECT_NUMBER']) @@ -68,16 +113,26 @@ def main(): items = get_pr_related_items(org_name, project_number, pr_number) + if not items: + print(f"No project items found for PR #{pr_number}") + sys.exit(0) + validation_errors = [] for item in items: empty_fields = validate_item(item) if empty_fields: - validation_errors.append(f"Item ID: {item['id']}, Empty fields: {', '.join(empty_fields)}") + error_message = f"PR #{pr_number}, Item ID: {item['id']}, Empty fields: {', '.join(empty_fields)}" + validation_errors.append(error_message) + print(f"Validation Error: {error_message}") # Print to console if validation_errors: with open('.github/project_validation_results.txt', 'w') as f: for error in validation_errors: f.write(f"{error}\n") + print("Validation failed. See results in .github/project_validation_results.txt") + sys.exit(1) + else: + print(f"Validation passed for PR #{pr_number}") if __name__ == "__main__": main() \ No newline at end of file From 1edb413fbbd85994cd45116c35cba16c9c2d9e61 Mon Sep 17 00:00:00 2001 From: minikin Date: Tue, 22 Oct 2024 09:02:52 +0200 Subject: [PATCH 03/37] Update Earthfile --- utilities/project-fields-validator/Earthfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utilities/project-fields-validator/Earthfile b/utilities/project-fields-validator/Earthfile index 4f029279b..574cf76d4 100644 --- a/utilities/project-fields-validator/Earthfile +++ b/utilities/project-fields-validator/Earthfile @@ -2,7 +2,7 @@ VERSION 0.8 IMPORT github.com/input-output-hk/catalyst-ci/earthly/python:v3.1.7 AS python-ci -check: +test: FROM python-ci+python-base COPY . . From 1d738f33f99ae1521fb0ca56b66a6a687b7ac421 Mon Sep 17 00:00:00 2001 From: minikin Date: Tue, 22 Oct 2024 15:28:30 +0200 Subject: [PATCH 04/37] feat: update ProjectFieldsValidator --- utilities/project-fields-validator/Earthfile | 4 +- utilities/project-fields-validator/main.py | 382 +++++++++++++------ 2 files changed, 277 insertions(+), 109 deletions(-) diff --git a/utilities/project-fields-validator/Earthfile b/utilities/project-fields-validator/Earthfile index 574cf76d4..38f9ad145 100644 --- a/utilities/project-fields-validator/Earthfile +++ b/utilities/project-fields-validator/Earthfile @@ -17,10 +17,10 @@ validate-pr: RUN pip install requests # Set environment variables - ENV ORG_NAME=input-output-hk ENV PROJECT_NUMBER=102 - ARG PR_NUMBER ARG GITHUB_TOKEN + ARG GITHUB_REPOSITORY + ARG GITHUB_EVENT_NUMBER RUN --secret GITHUB_TOKEN python3 main.py diff --git a/utilities/project-fields-validator/main.py b/utilities/project-fields-validator/main.py index a8fe04e15..6e01e721f 100644 --- a/utilities/project-fields-validator/main.py +++ b/utilities/project-fields-validator/main.py @@ -2,137 +2,305 @@ import sys import requests import json +from typing import Optional, List, Dict, Any -def run_query(query, variables): - headers = {"Authorization": f"Bearer {os.environ['GITHUB_TOKEN']}"} - request = requests.post('https://api.github.com/graphql', json={'query': query, 'variables': variables}, headers=headers) - if request.status_code == 200: - return request.json() - else: - print(f"Query failed with status code: {request.status_code}") - print(f"Response: {request.text}") - raise Exception(f"Query failed with status code: {request.status_code}") - -def get_pr_related_items(org_name, project_number, pr_number): - query = """ - query($org: String!, $number: Int!, $prNumber: Int!) { - organization(login: $org) { - projectV2(number: $number) { - items(first: 100) { - nodes { +class ProjectFieldsValidator: + def __init__(self, github_token: str): + self.github_token = github_token + self.headers = { + "Authorization": f"Bearer {github_token}", + "Accept": "application/vnd.github.v3+json" + } + self.required_fields = ['Status', 'Area', 'Priority', 'Estimate', 'Iteration', 'Start', 'End'] + + def run_query(self, query: str, variables: Dict[str, Any]) -> Dict[str, Any]: + """Execute a GraphQL query against GitHub's API.""" + response = requests.post( + 'https://api.github.com/graphql', + json={'query': query, 'variables': variables}, + headers=self.headers + ) + + if response.status_code != 200: + print(f"Query failed with status code: {response.status_code}") + print(f"Response: {response.text}") + raise Exception(f"Query failed with status code: {response.status_code}") + + return response.json() + + def get_pr_details(self, org_name: str, repo_name: str, pr_number: int) -> Dict[str, Any]: + """Get PR details including assignees.""" + query = """ + query($org: String!, $repo: String!, $number: Int!) { + repository(owner: $org, name: $repo) { + pullRequest(number: $number) { id - content { - ... on PullRequest { - number - title - url + author { + login + } + assignees(first: 10) { + nodes { + login } } - fieldValues(first: 20) { + } + } + } + """ + + variables = { + "org": org_name, + "repo": repo_name, + "number": pr_number + } + + result = self.run_query(query, variables) + return result['data']['repository']['pullRequest'] + + def assign_pr(self, org_name: str, repo_name: str, pr_number: int, assignee: str): + """Assign PR to a user using REST API.""" + url = f"https://api.github.com/repos/{org_name}/{repo_name}/issues/{pr_number}/assignees" + data = {"assignees": [assignee]} + + response = requests.post(url, json=data, headers=self.headers) + + if response.status_code == 201: + print(f"✅ PR assigned to @{assignee}") + else: + print(f"❌ Failed to assign PR to @{assignee}") + print(f"Response: {response.text}") + + def get_project_items(self, org_name: str, project_number: int) -> List[Dict[str, Any]]: + """Fetch all items from the project.""" + query = """ + query($org: String!, $projectNumber: Int!, $cursor: String) { + organization(login: $org) { + projectV2(number: $projectNumber) { + items(first: 100, after: $cursor) { + pageInfo { + hasNextPage + endCursor + } nodes { - ... on ProjectV2ItemFieldTextValue { - field { - ... on ProjectV2FieldCommon { - name + id + content { + ... on PullRequest { + number + title + url + author { + login } - } - text - } - ... on ProjectV2ItemFieldDateValue { - field { - ... on ProjectV2FieldCommon { + repository { name } } - date } - ... on ProjectV2ItemFieldSingleSelectValue { - field { - ... on ProjectV2FieldCommon { + fieldValues(first: 20) { + nodes { + ... on ProjectV2ItemFieldTextValue { + field { + ... on ProjectV2FieldCommon { + name + } + } + text + } + ... on ProjectV2ItemFieldDateValue { + field { + ... on ProjectV2FieldCommon { + name + } + } + date + } + ... on ProjectV2ItemFieldSingleSelectValue { + field { + ... on ProjectV2FieldCommon { + name + } + } name } + ... on ProjectV2ItemFieldNumberValue { + field { + ... on ProjectV2FieldCommon { + name + } + } + number + } + ... on ProjectV2ItemFieldIterationValue { + field { + ... on ProjectV2FieldCommon { + name + } + } + title + startDate + duration + } } - name } } } } } } - } - repository(owner: $org, name: "catalyst-voices") { - pullRequest(number: $prNumber) { - number - title - url - } - } - } - """ - variables = { - "org": org_name, - "number": project_number, - "prNumber": pr_number - } - result = run_query(query, variables) - print(f"API Response: {json.dumps(result, indent=2)}") - if 'data' not in result: - print(f"Error in API response: {result.get('errors', 'Unknown error')}") - return [] - - pr_info = result['data']['repository']['pullRequest'] - print(f"Found PR: #{pr_info['number']} - {pr_info['title']} ({pr_info['url']})") - - items = result['data']['organization']['projectV2']['items']['nodes'] - pr_items = [item for item in items if item['content'] and isinstance(item['content'], dict) and item['content'].get('number') == pr_number] - - if not pr_items: - print(f"PR #{pr_number} exists but is not linked to the project. Please add it to the project.") - return pr_items - -def validate_item(item): - required_fields = ['Status', 'Area', 'Priority', 'Estimate', 'Iteration', 'Start', 'End'] - field_values = {} - for field_value in item['fieldValues']['nodes']: - field_name = field_value['field']['name'] - if 'text' in field_value: - field_values[field_name] = field_value['text'] - elif 'date' in field_value: - field_values[field_name] = field_value['date'] - elif 'name' in field_value: - field_values[field_name] = field_value['name'] - - empty_fields = [field for field in required_fields if not field_values.get(field)] - return empty_fields + """ + + all_items = [] + cursor = None + page = 1 + max_pages = 100 + total_items = 0 + + sys.stdout.write("\rFetching project items...") + sys.stdout.flush() + + while page <= max_pages: + variables = { + "org": org_name, + "projectNumber": project_number, + "cursor": cursor + } + try: + result = self.run_query(query, variables) + project_data = result['data']['organization']['projectV2']['items'] + page_items = project_data['nodes'] + + valid_items = [ + item for item in page_items + if item['content'] and isinstance(item['content'], dict) + ] + + all_items.extend(valid_items) + total_items += len(valid_items) + + sys.stdout.write(f"\rFetching project items... {total_items} found") + sys.stdout.flush() + + page_info = project_data['pageInfo'] + if not page_info['hasNextPage']: + break + + cursor = page_info['endCursor'] + page += 1 + + except Exception as e: + print(f"\nError fetching page {page}: {str(e)}") + break + + print("\n") + return all_items + + def validate_item(self, item: Dict[str, Any]) -> List[str]: + """Validate required fields for an item.""" + try: + field_values = {} + + for field_value in item['fieldValues']['nodes']: + if not isinstance(field_value, dict) or 'field' not in field_value: + continue + try: + field_name = field_value['field']['name'] + except (KeyError, TypeError): + continue + + if 'text' in field_value: + field_values[field_name] = field_value['text'] + elif 'date' in field_value: + field_values[field_name] = field_value['date'] + elif 'name' in field_value: + field_values[field_name] = field_value['name'] + elif 'number' in field_value: + field_values[field_name] = str(field_value['number']) + elif 'title' in field_value and field_name == 'Iteration': + field_values[field_name] = field_value['title'] + + print("\nCurrent field values:") + print("="*50) + for field in self.required_fields: + value = field_values.get(field, '❌ empty') + print(f" • {field}: {value}") + + return [field for field in self.required_fields if not field_values.get(field)] + + except Exception as e: + print(f"\nError validating fields: {str(e)}") + return self.required_fields + + def print_validation_results(self, empty_fields: List[str]) -> None: + """Print validation results in a formatted way.""" + print("\n" + "="*50) + print("Validation Results:") + print("="*50) + + if not empty_fields: + print("✅ All required fields are filled. Validation passed!") + else: + print("❌ Validation failed. The following fields need to be filled:") + for field in empty_fields: + print(f" • {field}") + print("\nPlease fill in these fields in the project board.") + + print("="*50) def main(): - org_name = os.environ['ORG_NAME'] + github_token = os.environ['GITHUB_TOKEN'] + github_repository = os.environ['GITHUB_REPOSITORY'] + pr_number = int(os.environ['GITHUB_EVENT_NUMBER']) project_number = int(os.environ['PROJECT_NUMBER']) - pr_number = int(os.environ['PR_NUMBER']) - - items = get_pr_related_items(org_name, project_number, pr_number) - - if not items: - print(f"No project items found for PR #{pr_number}") - sys.exit(0) - - validation_errors = [] - for item in items: - empty_fields = validate_item(item) - if empty_fields: - error_message = f"PR #{pr_number}, Item ID: {item['id']}, Empty fields: {', '.join(empty_fields)}" - validation_errors.append(error_message) - print(f"Validation Error: {error_message}") # Print to console - - if validation_errors: - with open('.github/project_validation_results.txt', 'w') as f: - for error in validation_errors: - f.write(f"{error}\n") - print("Validation failed. See results in .github/project_validation_results.txt") + + org_name, repo_name = github_repository.split('/') + + print(f"\nValidating PR #{pr_number} in {github_repository}") + print(f"Project number: {project_number}") + print("="*50) + + validator = ProjectFieldsValidator(github_token) + + try: + pr_details = validator.get_pr_details(org_name, repo_name, pr_number) + author = pr_details['author']['login'] + assignees = [node['login'] for node in pr_details['assignees']['nodes']] + + if not assignees: + print(f"\nAssigning PR to author @{author}") + validator.assign_pr(org_name, repo_name, pr_number, author) + + + project_items = validator.get_project_items(org_name, project_number) + + pr_items = [ + item for item in project_items + if (item['content'].get('number') == pr_number and + item['content'].get('repository', {}).get('name') == repo_name) + ] + + if not pr_items: + print(f"\nWarning: PR #{pr_number} is not linked to project #{project_number}") + print("Please add it to the project using the following steps:") + print("1. Go to the project board") + print("2. Click '+ Add items'") + print("3. Search for this PR") + print("4. Click 'Add selected items'") + sys.exit(0) + + validation_errors = set() + for item in pr_items: + empty_fields = validator.validate_item(item) + validation_errors.update(empty_fields) + + validator.print_validation_results(list(validation_errors)) + + if validation_errors: + sys.exit(1) + + except Exception as e: + print(f"Error: {str(e)}") sys.exit(1) - else: - print(f"Validation passed for PR #{pr_number}") if __name__ == "__main__": main() \ No newline at end of file From f0e8d76c4dc6287bbca77f41cab3a4378e365d7a Mon Sep 17 00:00:00 2001 From: minikin Date: Tue, 22 Oct 2024 15:43:48 +0200 Subject: [PATCH 05/37] chore: refactor python code --- utilities/project-fields-validator/Earthfile | 9 +- utilities/project-fields-validator/main.py | 241 ++++++++++--------- 2 files changed, 130 insertions(+), 120 deletions(-) diff --git a/utilities/project-fields-validator/Earthfile b/utilities/project-fields-validator/Earthfile index 38f9ad145..f065ecaf9 100644 --- a/utilities/project-fields-validator/Earthfile +++ b/utilities/project-fields-validator/Earthfile @@ -22,11 +22,4 @@ validate-pr: ARG GITHUB_REPOSITORY ARG GITHUB_EVENT_NUMBER - RUN --secret GITHUB_TOKEN python3 main.py - - # Check the validation results - RUN if [ -f ".github/project_validation_results.txt" ]; then \ - echo "Validation failed. See results:" && \ - cat .github/project_validation_results.txt && \ - exit 1; \ - fi \ No newline at end of file + RUN --secret GITHUB_TOKEN python3 main.py \ No newline at end of file diff --git a/utilities/project-fields-validator/main.py b/utilities/project-fields-validator/main.py index 6e01e721f..c8cf8e68a 100644 --- a/utilities/project-fields-validator/main.py +++ b/utilities/project-fields-validator/main.py @@ -1,33 +1,64 @@ import os import sys +from dataclasses import dataclass +from typing import Optional, List, Dict, Any, Set +from enum import Enum import requests -import json -from typing import Optional, List, Dict, Any +from requests.exceptions import RequestException + +class FieldType(Enum): + TEXT = "text" + DATE = "date" + SELECT = "name" + NUMBER = "number" + ITERATION = "title" + +@dataclass +class ProjectField: + name: str + value: Optional[str] = None + field_type: Optional[FieldType] = None + +class GitHubAPIError(Exception): + """Exception for GitHub API errors""" + pass class ProjectFieldsValidator: + BASE_URL = "https://api.github.com" + GRAPHQL_URL = f"{BASE_URL}/graphql" + def __init__(self, github_token: str): - self.github_token = github_token self.headers = { "Authorization": f"Bearer {github_token}", "Accept": "application/vnd.github.v3+json" } - self.required_fields = ['Status', 'Area', 'Priority', 'Estimate', 'Iteration', 'Start', 'End'] + self.required_fields = [ + ProjectField("Status"), + ProjectField("Area"), + ProjectField("Priority"), + ProjectField("Estimate"), + ProjectField("Iteration"), + ProjectField("Start"), + ProjectField("End") + ] + + def _make_request(self, method: str, url: str, **kwargs) -> Dict[str, Any]: + """Generic method to make HTTP requests with error handling""" + try: + response = requests.request(method, url, headers=self.headers, **kwargs) + response.raise_for_status() + return response.json() + except RequestException as e: + raise GitHubAPIError(f"GitHub API request failed: {str(e)}") from e def run_query(self, query: str, variables: Dict[str, Any]) -> Dict[str, Any]: """Execute a GraphQL query against GitHub's API.""" - response = requests.post( - 'https://api.github.com/graphql', - json={'query': query, 'variables': variables}, - headers=self.headers + return self._make_request( + "POST", + self.GRAPHQL_URL, + json={'query': query, 'variables': variables} ) - if response.status_code != 200: - print(f"Query failed with status code: {response.status_code}") - print(f"Response: {response.text}") - raise Exception(f"Query failed with status code: {response.status_code}") - - return response.json() - def get_pr_details(self, org_name: str, repo_name: str, pr_number: int) -> Dict[str, Any]: """Get PR details including assignees.""" query = """ @@ -47,31 +78,20 @@ def get_pr_details(self, org_name: str, repo_name: str, pr_number: int) -> Dict[ } } """ - - variables = { - "org": org_name, - "repo": repo_name, - "number": pr_number - } - - result = self.run_query(query, variables) + result = self.run_query(query, {"org": org_name, "repo": repo_name, "number": pr_number}) return result['data']['repository']['pullRequest'] - def assign_pr(self, org_name: str, repo_name: str, pr_number: int, assignee: str): + def assign_pr(self, org_name: str, repo_name: str, pr_number: int, assignee: str) -> None: """Assign PR to a user using REST API.""" - url = f"https://api.github.com/repos/{org_name}/{repo_name}/issues/{pr_number}/assignees" - data = {"assignees": [assignee]} - - response = requests.post(url, json=data, headers=self.headers) - - if response.status_code == 201: + url = f"{self.BASE_URL}/repos/{org_name}/{repo_name}/issues/{pr_number}/assignees" + try: + self._make_request("POST", url, json={"assignees": [assignee]}) print(f"✅ PR assigned to @{assignee}") - else: - print(f"❌ Failed to assign PR to @{assignee}") - print(f"Response: {response.text}") + except GitHubAPIError as e: + print(f"❌ Failed to assign PR to @{assignee}: {str(e)}") def get_project_items(self, org_name: str, project_number: int) -> List[Dict[str, Any]]: - """Fetch all items from the project.""" + """Fetch all items from the project with pagination.""" query = """ query($org: String!, $projectNumber: Int!, $cursor: String) { organization(login: $org) { @@ -148,90 +168,78 @@ def get_project_items(self, org_name: str, project_number: int) -> List[Dict[str } } """ + return self._paginate_items(query, org_name, project_number) + def _paginate_items(self, query: str, org_name: str, project_number: int) -> List[Dict[str, Any]]: + """Handle pagination for project items.""" all_items = [] cursor = None - page = 1 - max_pages = 100 total_items = 0 - sys.stdout.write("\rFetching project items...") - sys.stdout.flush() - - while page <= max_pages: + while True: variables = { "org": org_name, "projectNumber": project_number, "cursor": cursor } - try: - result = self.run_query(query, variables) - project_data = result['data']['organization']['projectV2']['items'] - page_items = project_data['nodes'] - - valid_items = [ - item for item in page_items - if item['content'] and isinstance(item['content'], dict) - ] + result = self.run_query(query, variables) + project_data = result['data']['organization']['projectV2']['items'] + valid_items = [ + item for item in project_data['nodes'] + if item.get('content') and isinstance(item['content'], dict) + ] - all_items.extend(valid_items) - total_items += len(valid_items) + all_items.extend(valid_items) + total_items += len(valid_items) - sys.stdout.write(f"\rFetching project items... {total_items} found") - sys.stdout.flush() + sys.stdout.write(f"\rFetching project items... {total_items} found") + sys.stdout.flush() - page_info = project_data['pageInfo'] - if not page_info['hasNextPage']: - break - - cursor = page_info['endCursor'] - page += 1 - - except Exception as e: - print(f"\nError fetching page {page}: {str(e)}") + if not project_data['pageInfo']['hasNextPage']: break + cursor = project_data['pageInfo']['endCursor'] + print("\n") return all_items - def validate_item(self, item: Dict[str, Any]) -> List[str]: + def validate_item(self, item: Dict[str, Any]) -> Set[str]: """Validate required fields for an item.""" - try: - field_values = {} - - for field_value in item['fieldValues']['nodes']: - if not isinstance(field_value, dict) or 'field' not in field_value: - continue - try: - field_name = field_value['field']['name'] - except (KeyError, TypeError): - continue - - if 'text' in field_value: - field_values[field_name] = field_value['text'] - elif 'date' in field_value: - field_values[field_name] = field_value['date'] - elif 'name' in field_value: - field_values[field_name] = field_value['name'] - elif 'number' in field_value: - field_values[field_name] = str(field_value['number']) - elif 'title' in field_value and field_name == 'Iteration': - field_values[field_name] = field_value['title'] - - print("\nCurrent field values:") - print("="*50) - for field in self.required_fields: - value = field_values.get(field, '❌ empty') - print(f" • {field}: {value}") - - return [field for field in self.required_fields if not field_values.get(field)] - - except Exception as e: - print(f"\nError validating fields: {str(e)}") - return self.required_fields - - def print_validation_results(self, empty_fields: List[str]) -> None: + field_values = self._extract_field_values(item) + + print("\nCurrent field values:") + print("="*50) + for field in self.required_fields: + value = field_values.get(field.name, '❌ empty') + print(f" • {field.name}: {value}") + + return {field.name for field in self.required_fields if field.name not in field_values} + + def _extract_field_values(self, item: Dict[str, Any]) -> Dict[str, str]: + """Extract field values from item data.""" + field_values = {} + + for field_value in item['fieldValues']['nodes']: + if not isinstance(field_value, dict) or 'field' not in field_value: + continue + + try: + field_name = field_value['field']['name'] + for field_type in FieldType: + if field_type.value in field_value: + value = field_value[field_type.value] + if isinstance(value, (int, float)): + value = str(value) + field_values[field_name] = value + break + except (KeyError, TypeError): + continue + + return field_values + + @staticmethod + def print_validation_results(empty_fields: Set[str]) -> None: """Print validation results in a formatted way.""" print("\n" + "="*50) print("Validation Results:") @@ -241,27 +249,37 @@ def print_validation_results(self, empty_fields: List[str]) -> None: print("✅ All required fields are filled. Validation passed!") else: print("❌ Validation failed. The following fields need to be filled:") - for field in empty_fields: + for field in sorted(empty_fields): print(f" • {field}") print("\nPlease fill in these fields in the project board.") print("="*50) def main(): - github_token = os.environ['GITHUB_TOKEN'] - github_repository = os.environ['GITHUB_REPOSITORY'] - pr_number = int(os.environ['GITHUB_EVENT_NUMBER']) - project_number = int(os.environ['PROJECT_NUMBER']) + try: + env_vars = { + 'GITHUB_TOKEN': os.environ.get('GITHUB_TOKEN'), + 'GITHUB_REPOSITORY': os.environ.get('GITHUB_REPOSITORY'), + 'GITHUB_EVENT_NUMBER': os.environ.get('GITHUB_EVENT_NUMBER'), + 'PROJECT_NUMBER': os.environ.get('PROJECT_NUMBER') + } - org_name, repo_name = github_repository.split('/') + # Validate environment variables + missing_vars = [k for k, v in env_vars.items() if not v] + if missing_vars: + raise ValueError(f"Missing required environment variables: {', '.join(missing_vars)}") - print(f"\nValidating PR #{pr_number} in {github_repository}") - print(f"Project number: {project_number}") - print("="*50) + github_repository = env_vars['GITHUB_REPOSITORY'] + pr_number = int(env_vars['GITHUB_EVENT_NUMBER']) + project_number = int(env_vars['PROJECT_NUMBER']) + org_name, repo_name = github_repository.split('/') - validator = ProjectFieldsValidator(github_token) + print(f"\nValidating PR #{pr_number} in {github_repository}") + print(f"Project number: {project_number}") + print("="*50) + + validator = ProjectFieldsValidator(env_vars['GITHUB_TOKEN']) - try: pr_details = validator.get_pr_details(org_name, repo_name, pr_number) author = pr_details['author']['login'] assignees = [node['login'] for node in pr_details['assignees']['nodes']] @@ -270,9 +288,8 @@ def main(): print(f"\nAssigning PR to author @{author}") validator.assign_pr(org_name, repo_name, pr_number, author) - + # Get and validate project items project_items = validator.get_project_items(org_name, project_number) - pr_items = [ item for item in project_items if (item['content'].get('number') == pr_number and @@ -293,7 +310,7 @@ def main(): empty_fields = validator.validate_item(item) validation_errors.update(empty_fields) - validator.print_validation_results(list(validation_errors)) + validator.print_validation_results(validation_errors) if validation_errors: sys.exit(1) From debf21b90756d4e063294a78cfc9704bda13ff9a Mon Sep 17 00:00:00 2001 From: minikin Date: Tue, 22 Oct 2024 15:49:21 +0200 Subject: [PATCH 06/37] Update README.md --- utilities/project-fields-validator/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/utilities/project-fields-validator/README.md b/utilities/project-fields-validator/README.md index e69de29bb..b1779ca9f 100644 --- a/utilities/project-fields-validator/README.md +++ b/utilities/project-fields-validator/README.md @@ -0,0 +1,6 @@ +# Project fields validator + +This module is used to validate the fields of a project. + +* auto assign PR creator as assignee +* verifies that all project fields are filled out and fail if any are left unfilled. \ No newline at end of file From dd17f55350dc9b2c4de84aa82cdb80b7c2751f13 Mon Sep 17 00:00:00 2001 From: minikin Date: Tue, 22 Oct 2024 15:50:05 +0200 Subject: [PATCH 07/37] Update main.py --- utilities/project-fields-validator/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utilities/project-fields-validator/main.py b/utilities/project-fields-validator/main.py index c8cf8e68a..0bbc75696 100644 --- a/utilities/project-fields-validator/main.py +++ b/utilities/project-fields-validator/main.py @@ -235,7 +235,7 @@ def _extract_field_values(self, item: Dict[str, Any]) -> Dict[str, str]: break except (KeyError, TypeError): continue - + return field_values @staticmethod From 045e5db94a9299694849441b1a6576a3cd44c3aa Mon Sep 17 00:00:00 2001 From: minikin Date: Tue, 22 Oct 2024 15:52:58 +0200 Subject: [PATCH 08/37] chore: ci lint fixes --- utilities/project-fields-validator/Earthfile | 1 - utilities/project-fields-validator/README.md | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/utilities/project-fields-validator/Earthfile b/utilities/project-fields-validator/Earthfile index f065ecaf9..32e088bde 100644 --- a/utilities/project-fields-validator/Earthfile +++ b/utilities/project-fields-validator/Earthfile @@ -16,7 +16,6 @@ validate-pr: RUN pip install requests - # Set environment variables ENV PROJECT_NUMBER=102 ARG GITHUB_TOKEN ARG GITHUB_REPOSITORY diff --git a/utilities/project-fields-validator/README.md b/utilities/project-fields-validator/README.md index b1779ca9f..9e8ec9c62 100644 --- a/utilities/project-fields-validator/README.md +++ b/utilities/project-fields-validator/README.md @@ -3,4 +3,4 @@ This module is used to validate the fields of a project. * auto assign PR creator as assignee -* verifies that all project fields are filled out and fail if any are left unfilled. \ No newline at end of file +* verifies that all project fields are filled out and fail if any are left unfilled. From a818e6fc1b1dc259108066adccc3ee17bee8c4a6 Mon Sep 17 00:00:00 2001 From: minikin Date: Tue, 22 Oct 2024 16:05:51 +0200 Subject: [PATCH 09/37] Update Earthfile --- utilities/project-fields-validator/Earthfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/utilities/project-fields-validator/Earthfile b/utilities/project-fields-validator/Earthfile index 32e088bde..9665aa512 100644 --- a/utilities/project-fields-validator/Earthfile +++ b/utilities/project-fields-validator/Earthfile @@ -21,4 +21,9 @@ validate-pr: ARG GITHUB_REPOSITORY ARG GITHUB_EVENT_NUMBER - RUN --secret GITHUB_TOKEN python3 main.py \ No newline at end of file + RUN --secret GITHUB_TOKEN python3 main.py + +VALIDATE_PROJECT_FIELDS: + FUNCTION + + FROM +validate-pr \ No newline at end of file From a09ca82783484aaa168a18e3a1d48f1e285af8f7 Mon Sep 17 00:00:00 2001 From: minikin Date: Wed, 23 Oct 2024 15:46:07 +0200 Subject: [PATCH 10/37] Update Earthfile --- utilities/project-fields-validator/Earthfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utilities/project-fields-validator/Earthfile b/utilities/project-fields-validator/Earthfile index 9665aa512..ba87ab5de 100644 --- a/utilities/project-fields-validator/Earthfile +++ b/utilities/project-fields-validator/Earthfile @@ -26,4 +26,7 @@ validate-pr: VALIDATE_PROJECT_FIELDS: FUNCTION + ARG GITHUB_REPOSITORY + ARG GITHUB_EVENT_NUMBER + FROM +validate-pr \ No newline at end of file From 46a289d7cc8d463f4756848ca5cd3b98574d8ac4 Mon Sep 17 00:00:00 2001 From: minikin Date: Thu, 24 Oct 2024 14:49:05 +0200 Subject: [PATCH 11/37] Update Earthfile --- utilities/project-fields-validator/Earthfile | 24 +++++++++----------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/utilities/project-fields-validator/Earthfile b/utilities/project-fields-validator/Earthfile index ba87ab5de..f3ba51e42 100644 --- a/utilities/project-fields-validator/Earthfile +++ b/utilities/project-fields-validator/Earthfile @@ -4,29 +4,27 @@ IMPORT github.com/input-output-hk/catalyst-ci/earthly/python:v3.1.7 AS python-ci test: FROM python-ci+python-base - COPY . . - DO python-ci+CHECK validate-pr: FROM python-ci+python-base - COPY . . - - RUN pip install requests + RUN pip install requests jq ENV PROJECT_NUMBER=102 ARG GITHUB_TOKEN - ARG GITHUB_REPOSITORY - ARG GITHUB_EVENT_NUMBER - RUN --secret GITHUB_TOKEN python3 main.py + # Extract PR info from GitHub event context + RUN --secret GITHUB_TOKEN \ + if [ -f "$GITHUB_EVENT_PATH" ]; then \ + PR_NUMBER=$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH"); \ + REPO=$(jq --raw-output .repository.full_name "$GITHUB_EVENT_PATH"); \ + export GITHUB_EVENT_NUMBER=$PR_NUMBER; \ + export GITHUB_REPOSITORY=$REPO; \ + fi && \ + python3 main.py VALIDATE_PROJECT_FIELDS: FUNCTION - - ARG GITHUB_REPOSITORY - ARG GITHUB_EVENT_NUMBER - - FROM +validate-pr \ No newline at end of file + DO +validate-pr \ No newline at end of file From 7e18b2969597147a340927726f2f92a712e5caa2 Mon Sep 17 00:00:00 2001 From: minikin Date: Thu, 24 Oct 2024 17:05:28 +0200 Subject: [PATCH 12/37] Update Earthfile --- utilities/project-fields-validator/Earthfile | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/utilities/project-fields-validator/Earthfile b/utilities/project-fields-validator/Earthfile index f3ba51e42..b4383f66a 100644 --- a/utilities/project-fields-validator/Earthfile +++ b/utilities/project-fields-validator/Earthfile @@ -10,20 +10,12 @@ test: validate-pr: FROM python-ci+python-base COPY . . - RUN pip install requests jq + RUN pip install requests ENV PROJECT_NUMBER=102 ARG GITHUB_TOKEN - # Extract PR info from GitHub event context - RUN --secret GITHUB_TOKEN \ - if [ -f "$GITHUB_EVENT_PATH" ]; then \ - PR_NUMBER=$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH"); \ - REPO=$(jq --raw-output .repository.full_name "$GITHUB_EVENT_PATH"); \ - export GITHUB_EVENT_NUMBER=$PR_NUMBER; \ - export GITHUB_REPOSITORY=$REPO; \ - fi && \ - python3 main.py + RUN --secret GITHUB_TOKEN python3 main.py VALIDATE_PROJECT_FIELDS: FUNCTION From 061098f377bf4a43b167fc01808224c02c6cc348 Mon Sep 17 00:00:00 2001 From: minikin Date: Thu, 24 Oct 2024 17:16:07 +0200 Subject: [PATCH 13/37] Update Earthfile --- utilities/project-fields-validator/Earthfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/utilities/project-fields-validator/Earthfile b/utilities/project-fields-validator/Earthfile index b4383f66a..038c7103c 100644 --- a/utilities/project-fields-validator/Earthfile +++ b/utilities/project-fields-validator/Earthfile @@ -13,10 +13,9 @@ validate-pr: RUN pip install requests ENV PROJECT_NUMBER=102 - ARG GITHUB_TOKEN RUN --secret GITHUB_TOKEN python3 main.py VALIDATE_PROJECT_FIELDS: FUNCTION - DO +validate-pr \ No newline at end of file + FROM +validate-pr \ No newline at end of file From f1aadfbb080183c318884af493900942abee8758 Mon Sep 17 00:00:00 2001 From: minikin Date: Fri, 25 Oct 2024 14:28:23 +0200 Subject: [PATCH 14/37] Update Earthfile --- utilities/project-fields-validator/Earthfile | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/utilities/project-fields-validator/Earthfile b/utilities/project-fields-validator/Earthfile index 038c7103c..af7ae0b08 100644 --- a/utilities/project-fields-validator/Earthfile +++ b/utilities/project-fields-validator/Earthfile @@ -7,15 +7,29 @@ test: COPY . . DO python-ci+CHECK -validate-pr: +validate-project-fields: FROM python-ci+python-base COPY . . + RUN pip install requests - ENV PROJECT_NUMBER=102 + ARG --required PROJECT_NUMBER + ARG --required GITHUB_REPOSITORY + ARG --required GITHUB_EVENT_NUMBER + + ENV PROJECT_NUMBER=${PROJECT_NUMBER} + ENV GITHUB_REPOSITORY=${GITHUB_REPOSITORY} + ENV GITHUB_EVENT_NUMBER=${GITHUB_EVENT_NUMBER} RUN --secret GITHUB_TOKEN python3 main.py VALIDATE_PROJECT_FIELDS: FUNCTION - FROM +validate-pr \ No newline at end of file + + ARG PROJECT_NUMBER = 102 + ARG --required GITHUB_REPOSITORY + ARG --required GITHUB_EVENT_NUMBER + + FROM +validate-project-fields --PROJECT_NUMBER=${PROJECT_NUMBER} \ + --GITHUB_REPOSITORY=${GITHUB_REPOSITORY} \ + --GITHUB_EVENT_NUMBER=${GITHUB_EVENT_NUMBER} \ No newline at end of file From cae42c4500db80ce7813c5b9c5bfb984685de7cd Mon Sep 17 00:00:00 2001 From: minikin Date: Fri, 25 Oct 2024 14:46:41 +0200 Subject: [PATCH 15/37] feat: add GitHub action --- .github/workflows/validate-project-fields.yml | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 .github/workflows/validate-project-fields.yml diff --git a/.github/workflows/validate-project-fields.yml b/.github/workflows/validate-project-fields.yml new file mode 100644 index 000000000..653fc67f0 --- /dev/null +++ b/.github/workflows/validate-project-fields.yml @@ -0,0 +1,72 @@ +name: Validate Projects Fields + +on: + workflow_call: + inputs: + types: + required: false + type: string + scopes: + required: false + type: string + requireScope: + required: false + type: boolean + default: false + pull_request: + types: + - opened + - edited + - synchronize + - reopened + - unassigned + +permissions: + contents: write + pull-requests: write + id-token: write + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +env: + AWS_REGION: eu-central-1 + AWS_ROLE_ARN: arn:aws:iam::332405224602:role/ci + EARTHLY_TARGET: docker + ECR_REGISTRY: 332405224602.dkr.ecr.eu-central-1.amazonaws.com + +jobs: + validate-project-fields: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup CI + uses: input-output-hk/catalyst-ci/actions/setup@master + with: + aws_role_arn: ${{ env.AWS_ROLE_ARN }} + aws_region: ${{ env.AWS_REGION }} + earthly_runner_secret: ${{ secrets.EARTHLY_RUNNER_SECRET }} + + - name: Run Project Fields Validation + uses: input-output-hk/catalyst-ci/actions/run@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + if: always() + continue-on-error: false + with: + earthfile: ./utilities/project-fields-validator + flags: --allow-privileged + target_flags: --GITHUB_REPOSITORY="${{ github.repository }}" --GITHUB_EVENT_NUMBER="${{ github.event.number || '0' }}" + runner_address: ${{ secrets.EARTHLY_SATELLITE_ADDRESS }} + artifact: false + types: ${{inputs.types}} + scopes: ${{inputs.scopes}} + requireScope: ${{inputs.requireScope}} + validateSingleCommit: false + ignoreLabels: | + bot + wip + draft + wip: false From 304b6f2df37257d874b109dd2c7a535d94723cc4 Mon Sep 17 00:00:00 2001 From: minikin Date: Fri, 25 Oct 2024 14:51:24 +0200 Subject: [PATCH 16/37] Update validate-project-fields.yml --- .github/workflows/validate-project-fields.yml | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/.github/workflows/validate-project-fields.yml b/.github/workflows/validate-project-fields.yml index 653fc67f0..b8fc97ffb 100644 --- a/.github/workflows/validate-project-fields.yml +++ b/.github/workflows/validate-project-fields.yml @@ -1,18 +1,6 @@ name: Validate Projects Fields on: - workflow_call: - inputs: - types: - required: false - type: string - scopes: - required: false - type: string - requireScope: - required: false - type: boolean - default: false pull_request: types: - opened @@ -61,12 +49,3 @@ jobs: target_flags: --GITHUB_REPOSITORY="${{ github.repository }}" --GITHUB_EVENT_NUMBER="${{ github.event.number || '0' }}" runner_address: ${{ secrets.EARTHLY_SATELLITE_ADDRESS }} artifact: false - types: ${{inputs.types}} - scopes: ${{inputs.scopes}} - requireScope: ${{inputs.requireScope}} - validateSingleCommit: false - ignoreLabels: | - bot - wip - draft - wip: false From 6586bc7d929fadb4cb0f726602f788401c2a2158 Mon Sep 17 00:00:00 2001 From: minikin Date: Fri, 25 Oct 2024 14:52:12 +0200 Subject: [PATCH 17/37] Update validate-project-fields.yml --- .github/workflows/validate-project-fields.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate-project-fields.yml b/.github/workflows/validate-project-fields.yml index b8fc97ffb..d727ea261 100644 --- a/.github/workflows/validate-project-fields.yml +++ b/.github/workflows/validate-project-fields.yml @@ -1,4 +1,4 @@ -name: Validate Projects Fields +name: Validate Project Fields on: pull_request: From c8bc9edcf579f92932c1e809b40a21df96381a32 Mon Sep 17 00:00:00 2001 From: minikin Date: Fri, 25 Oct 2024 14:55:11 +0200 Subject: [PATCH 18/37] wip: clean up --- .github/workflows/validate-project-fields.yml | 1 + utilities/project-fields-validator/Earthfile | 18 +++++++++--------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/validate-project-fields.yml b/.github/workflows/validate-project-fields.yml index d727ea261..950daa9c3 100644 --- a/.github/workflows/validate-project-fields.yml +++ b/.github/workflows/validate-project-fields.yml @@ -46,6 +46,7 @@ jobs: with: earthfile: ./utilities/project-fields-validator flags: --allow-privileged + targets: validate-project-fields target_flags: --GITHUB_REPOSITORY="${{ github.repository }}" --GITHUB_EVENT_NUMBER="${{ github.event.number || '0' }}" runner_address: ${{ secrets.EARTHLY_SATELLITE_ADDRESS }} artifact: false diff --git a/utilities/project-fields-validator/Earthfile b/utilities/project-fields-validator/Earthfile index af7ae0b08..1c3ae085d 100644 --- a/utilities/project-fields-validator/Earthfile +++ b/utilities/project-fields-validator/Earthfile @@ -13,7 +13,7 @@ validate-project-fields: RUN pip install requests - ARG --required PROJECT_NUMBER + ARG PROJECT_NUMBER=102 ARG --required GITHUB_REPOSITORY ARG --required GITHUB_EVENT_NUMBER @@ -23,13 +23,13 @@ validate-project-fields: RUN --secret GITHUB_TOKEN python3 main.py -VALIDATE_PROJECT_FIELDS: - FUNCTION +# VALIDATE_PROJECT_FIELDS: +# FUNCTION - ARG PROJECT_NUMBER = 102 - ARG --required GITHUB_REPOSITORY - ARG --required GITHUB_EVENT_NUMBER +# ARG PROJECT_NUMBER = 102 +# ARG --required GITHUB_REPOSITORY +# ARG --required GITHUB_EVENT_NUMBER - FROM +validate-project-fields --PROJECT_NUMBER=${PROJECT_NUMBER} \ - --GITHUB_REPOSITORY=${GITHUB_REPOSITORY} \ - --GITHUB_EVENT_NUMBER=${GITHUB_EVENT_NUMBER} \ No newline at end of file +# FROM +validate-project-fields --PROJECT_NUMBER=${PROJECT_NUMBER} \ +# --GITHUB_REPOSITORY=${GITHUB_REPOSITORY} \ +# --GITHUB_EVENT_NUMBER=${GITHUB_EVENT_NUMBER} \ No newline at end of file From 8042e831b4bcc5da90ad6d7d72f84e97c7f61818 Mon Sep 17 00:00:00 2001 From: minikin Date: Fri, 25 Oct 2024 15:00:14 +0200 Subject: [PATCH 19/37] chore: add GITHUB_TOKEN --- .github/workflows/validate-project-fields.yml | 4 +--- utilities/project-fields-validator/Earthfile | 14 ++------------ 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/.github/workflows/validate-project-fields.yml b/.github/workflows/validate-project-fields.yml index 950daa9c3..5da3e6b9a 100644 --- a/.github/workflows/validate-project-fields.yml +++ b/.github/workflows/validate-project-fields.yml @@ -39,14 +39,12 @@ jobs: - name: Run Project Fields Validation uses: input-output-hk/catalyst-ci/actions/run@master - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} if: always() continue-on-error: false with: earthfile: ./utilities/project-fields-validator flags: --allow-privileged targets: validate-project-fields - target_flags: --GITHUB_REPOSITORY="${{ github.repository }}" --GITHUB_EVENT_NUMBER="${{ github.event.number || '0' }}" + target_flags: --GITHUB_TOKEN ${{ secrets.GITHUB_TOKEN }} --GITHUB_REPOSITORY="${{ github.repository }}" --GITHUB_EVENT_NUMBER="${{ github.event.number || '0' }}" runner_address: ${{ secrets.EARTHLY_SATELLITE_ADDRESS }} artifact: false diff --git a/utilities/project-fields-validator/Earthfile b/utilities/project-fields-validator/Earthfile index 1c3ae085d..48878dd6a 100644 --- a/utilities/project-fields-validator/Earthfile +++ b/utilities/project-fields-validator/Earthfile @@ -14,6 +14,7 @@ validate-project-fields: RUN pip install requests ARG PROJECT_NUMBER=102 + ARG --required GITHUB_TOKEN ARG --required GITHUB_REPOSITORY ARG --required GITHUB_EVENT_NUMBER @@ -21,15 +22,4 @@ validate-project-fields: ENV GITHUB_REPOSITORY=${GITHUB_REPOSITORY} ENV GITHUB_EVENT_NUMBER=${GITHUB_EVENT_NUMBER} - RUN --secret GITHUB_TOKEN python3 main.py - -# VALIDATE_PROJECT_FIELDS: -# FUNCTION - -# ARG PROJECT_NUMBER = 102 -# ARG --required GITHUB_REPOSITORY -# ARG --required GITHUB_EVENT_NUMBER - -# FROM +validate-project-fields --PROJECT_NUMBER=${PROJECT_NUMBER} \ -# --GITHUB_REPOSITORY=${GITHUB_REPOSITORY} \ -# --GITHUB_EVENT_NUMBER=${GITHUB_EVENT_NUMBER} \ No newline at end of file + RUN --secret GITHUB_TOKEN python3 main.py \ No newline at end of file From a5a5f294659d6e05768048fe3c2925b4f08f1e71 Mon Sep 17 00:00:00 2001 From: minikin Date: Fri, 25 Oct 2024 15:03:05 +0200 Subject: [PATCH 20/37] Update Earthfile --- utilities/project-fields-validator/Earthfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utilities/project-fields-validator/Earthfile b/utilities/project-fields-validator/Earthfile index 48878dd6a..51b5670f7 100644 --- a/utilities/project-fields-validator/Earthfile +++ b/utilities/project-fields-validator/Earthfile @@ -22,4 +22,4 @@ validate-project-fields: ENV GITHUB_REPOSITORY=${GITHUB_REPOSITORY} ENV GITHUB_EVENT_NUMBER=${GITHUB_EVENT_NUMBER} - RUN --secret GITHUB_TOKEN python3 main.py \ No newline at end of file + RUN --secret GITHUB_TOKEN=${GITHUB_TOKEN} python3 main.py \ No newline at end of file From 33a8b4e147c332c1738b058c673bfa7d20196de8 Mon Sep 17 00:00:00 2001 From: minikin Date: Fri, 25 Oct 2024 15:05:30 +0200 Subject: [PATCH 21/37] Update validate-project-fields.yml --- .github/workflows/validate-project-fields.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/validate-project-fields.yml b/.github/workflows/validate-project-fields.yml index 5da3e6b9a..54f44dc01 100644 --- a/.github/workflows/validate-project-fields.yml +++ b/.github/workflows/validate-project-fields.yml @@ -39,6 +39,8 @@ jobs: - name: Run Project Fields Validation uses: input-output-hk/catalyst-ci/actions/run@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} if: always() continue-on-error: false with: From 6e1138230df75b12d2248b00811f6b427ff57c60 Mon Sep 17 00:00:00 2001 From: minikin Date: Fri, 25 Oct 2024 15:06:35 +0200 Subject: [PATCH 22/37] Update validate-project-fields.yml --- .github/workflows/validate-project-fields.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validate-project-fields.yml b/.github/workflows/validate-project-fields.yml index 54f44dc01..0e943a1cc 100644 --- a/.github/workflows/validate-project-fields.yml +++ b/.github/workflows/validate-project-fields.yml @@ -47,6 +47,9 @@ jobs: earthfile: ./utilities/project-fields-validator flags: --allow-privileged targets: validate-project-fields - target_flags: --GITHUB_TOKEN ${{ secrets.GITHUB_TOKEN }} --GITHUB_REPOSITORY="${{ github.repository }}" --GITHUB_EVENT_NUMBER="${{ github.event.number || '0' }}" + target_flags: | + --GITHUB_TOKEN ${{ secrets.GITHUB_TOKEN }} \ + --GITHUB_REPOSITORY="${{ github.repository }}" \ + --GITHUB_EVENT_NUMBER="${{ github.event.number || '0' }}" runner_address: ${{ secrets.EARTHLY_SATELLITE_ADDRESS }} artifact: false From 2cc8180750c8e0d9a88248e04709e4bd2ce421cb Mon Sep 17 00:00:00 2001 From: minikin Date: Fri, 25 Oct 2024 15:07:59 +0200 Subject: [PATCH 23/37] Update validate-project-fields.yml --- .github/workflows/validate-project-fields.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/validate-project-fields.yml b/.github/workflows/validate-project-fields.yml index 0e943a1cc..d5d8a5f05 100644 --- a/.github/workflows/validate-project-fields.yml +++ b/.github/workflows/validate-project-fields.yml @@ -47,8 +47,7 @@ jobs: earthfile: ./utilities/project-fields-validator flags: --allow-privileged targets: validate-project-fields - target_flags: | - --GITHUB_TOKEN ${{ secrets.GITHUB_TOKEN }} \ + target_flags: --GITHUB_TOKEN ${{ secrets.GITHUB_TOKEN }} \ --GITHUB_REPOSITORY="${{ github.repository }}" \ --GITHUB_EVENT_NUMBER="${{ github.event.number || '0' }}" runner_address: ${{ secrets.EARTHLY_SATELLITE_ADDRESS }} From fb45eb69205e4ff18a588eb482fcfda01aefe5b7 Mon Sep 17 00:00:00 2001 From: minikin Date: Fri, 25 Oct 2024 15:09:40 +0200 Subject: [PATCH 24/37] wip --- .github/workflows/validate-project-fields.yml | 4 +--- utilities/project-fields-validator/Earthfile | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/validate-project-fields.yml b/.github/workflows/validate-project-fields.yml index d5d8a5f05..950daa9c3 100644 --- a/.github/workflows/validate-project-fields.yml +++ b/.github/workflows/validate-project-fields.yml @@ -47,8 +47,6 @@ jobs: earthfile: ./utilities/project-fields-validator flags: --allow-privileged targets: validate-project-fields - target_flags: --GITHUB_TOKEN ${{ secrets.GITHUB_TOKEN }} \ - --GITHUB_REPOSITORY="${{ github.repository }}" \ - --GITHUB_EVENT_NUMBER="${{ github.event.number || '0' }}" + target_flags: --GITHUB_REPOSITORY="${{ github.repository }}" --GITHUB_EVENT_NUMBER="${{ github.event.number || '0' }}" runner_address: ${{ secrets.EARTHLY_SATELLITE_ADDRESS }} artifact: false diff --git a/utilities/project-fields-validator/Earthfile b/utilities/project-fields-validator/Earthfile index 51b5670f7..16b304473 100644 --- a/utilities/project-fields-validator/Earthfile +++ b/utilities/project-fields-validator/Earthfile @@ -14,7 +14,6 @@ validate-project-fields: RUN pip install requests ARG PROJECT_NUMBER=102 - ARG --required GITHUB_TOKEN ARG --required GITHUB_REPOSITORY ARG --required GITHUB_EVENT_NUMBER @@ -22,4 +21,4 @@ validate-project-fields: ENV GITHUB_REPOSITORY=${GITHUB_REPOSITORY} ENV GITHUB_EVENT_NUMBER=${GITHUB_EVENT_NUMBER} - RUN --secret GITHUB_TOKEN=${GITHUB_TOKEN} python3 main.py \ No newline at end of file + RUN python3 main.py \ No newline at end of file From 55ad662b699305cd758cc5f217134df03b40acbd Mon Sep 17 00:00:00 2001 From: minikin Date: Fri, 25 Oct 2024 15:13:40 +0200 Subject: [PATCH 25/37] wip --- .github/workflows/validate-project-fields.yml | 4 +--- utilities/project-fields-validator/Earthfile | 3 ++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/validate-project-fields.yml b/.github/workflows/validate-project-fields.yml index 950daa9c3..a0031aeb1 100644 --- a/.github/workflows/validate-project-fields.yml +++ b/.github/workflows/validate-project-fields.yml @@ -39,14 +39,12 @@ jobs: - name: Run Project Fields Validation uses: input-output-hk/catalyst-ci/actions/run@master - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} if: always() continue-on-error: false with: earthfile: ./utilities/project-fields-validator flags: --allow-privileged targets: validate-project-fields - target_flags: --GITHUB_REPOSITORY="${{ github.repository }}" --GITHUB_EVENT_NUMBER="${{ github.event.number || '0' }}" + target_flags: --GITHUB_TOKEN="${{ secrets.GITHUB_TOKEN }}" --GITHUB_REPOSITORY="${{ github.repository }}" --GITHUB_EVENT_NUMBER="${{ github.event.number || '0' }}" runner_address: ${{ secrets.EARTHLY_SATELLITE_ADDRESS }} artifact: false diff --git a/utilities/project-fields-validator/Earthfile b/utilities/project-fields-validator/Earthfile index 16b304473..51b5670f7 100644 --- a/utilities/project-fields-validator/Earthfile +++ b/utilities/project-fields-validator/Earthfile @@ -14,6 +14,7 @@ validate-project-fields: RUN pip install requests ARG PROJECT_NUMBER=102 + ARG --required GITHUB_TOKEN ARG --required GITHUB_REPOSITORY ARG --required GITHUB_EVENT_NUMBER @@ -21,4 +22,4 @@ validate-project-fields: ENV GITHUB_REPOSITORY=${GITHUB_REPOSITORY} ENV GITHUB_EVENT_NUMBER=${GITHUB_EVENT_NUMBER} - RUN python3 main.py \ No newline at end of file + RUN --secret GITHUB_TOKEN=${GITHUB_TOKEN} python3 main.py \ No newline at end of file From e4577bd17ebd99997e4f64f440a1f2dc8393ea68 Mon Sep 17 00:00:00 2001 From: minikin Date: Fri, 25 Oct 2024 15:16:28 +0200 Subject: [PATCH 26/37] wip --- .github/workflows/validate-project-fields.yml | 2 +- utilities/project-fields-validator/Earthfile | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/validate-project-fields.yml b/.github/workflows/validate-project-fields.yml index a0031aeb1..14f221abc 100644 --- a/.github/workflows/validate-project-fields.yml +++ b/.github/workflows/validate-project-fields.yml @@ -45,6 +45,6 @@ jobs: earthfile: ./utilities/project-fields-validator flags: --allow-privileged targets: validate-project-fields - target_flags: --GITHUB_TOKEN="${{ secrets.GITHUB_TOKEN }}" --GITHUB_REPOSITORY="${{ github.repository }}" --GITHUB_EVENT_NUMBER="${{ github.event.number || '0' }}" + target_flags: --GITHUB_REPOSITORY="${{ github.repository }}" --GITHUB_EVENT_NUMBER="${{ github.event.number || '0' }}" runner_address: ${{ secrets.EARTHLY_SATELLITE_ADDRESS }} artifact: false diff --git a/utilities/project-fields-validator/Earthfile b/utilities/project-fields-validator/Earthfile index 51b5670f7..f569f8b81 100644 --- a/utilities/project-fields-validator/Earthfile +++ b/utilities/project-fields-validator/Earthfile @@ -14,7 +14,6 @@ validate-project-fields: RUN pip install requests ARG PROJECT_NUMBER=102 - ARG --required GITHUB_TOKEN ARG --required GITHUB_REPOSITORY ARG --required GITHUB_EVENT_NUMBER @@ -22,4 +21,4 @@ validate-project-fields: ENV GITHUB_REPOSITORY=${GITHUB_REPOSITORY} ENV GITHUB_EVENT_NUMBER=${GITHUB_EVENT_NUMBER} - RUN --secret GITHUB_TOKEN=${GITHUB_TOKEN} python3 main.py \ No newline at end of file + RUN --secret GITHUB_TOKEN=GITHUB_TOKEN python3 main.py \ No newline at end of file From dcf7afafc12e45bfc9b5abaf95954b18ebfc101b Mon Sep 17 00:00:00 2001 From: minikin Date: Fri, 25 Oct 2024 15:18:14 +0200 Subject: [PATCH 27/37] Update validate-project-fields.yml --- .github/workflows/validate-project-fields.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/validate-project-fields.yml b/.github/workflows/validate-project-fields.yml index 14f221abc..950daa9c3 100644 --- a/.github/workflows/validate-project-fields.yml +++ b/.github/workflows/validate-project-fields.yml @@ -39,6 +39,8 @@ jobs: - name: Run Project Fields Validation uses: input-output-hk/catalyst-ci/actions/run@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} if: always() continue-on-error: false with: From 7b13c3fa16039f7f9e31d6f580f6a93eaf698004 Mon Sep 17 00:00:00 2001 From: minikin Date: Fri, 25 Oct 2024 15:56:28 +0200 Subject: [PATCH 28/37] wip --- .github/workflows/validate-project-fields.yml | 2 -- utilities/project-fields-validator/Earthfile | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/validate-project-fields.yml b/.github/workflows/validate-project-fields.yml index 950daa9c3..14f221abc 100644 --- a/.github/workflows/validate-project-fields.yml +++ b/.github/workflows/validate-project-fields.yml @@ -39,8 +39,6 @@ jobs: - name: Run Project Fields Validation uses: input-output-hk/catalyst-ci/actions/run@master - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} if: always() continue-on-error: false with: diff --git a/utilities/project-fields-validator/Earthfile b/utilities/project-fields-validator/Earthfile index f569f8b81..63c2fbde6 100644 --- a/utilities/project-fields-validator/Earthfile +++ b/utilities/project-fields-validator/Earthfile @@ -21,4 +21,4 @@ validate-project-fields: ENV GITHUB_REPOSITORY=${GITHUB_REPOSITORY} ENV GITHUB_EVENT_NUMBER=${GITHUB_EVENT_NUMBER} - RUN --secret GITHUB_TOKEN=GITHUB_TOKEN python3 main.py \ No newline at end of file + RUN --secret GITHUB_TOKEN python3 main.py \ No newline at end of file From 279e0ae00490bed218ec0a53bf279d2a7b2ddd1d Mon Sep 17 00:00:00 2001 From: Joshua Gilman Date: Fri, 25 Oct 2024 07:19:29 -0700 Subject: [PATCH 29/37] wip: testing --- .github/workflows/validate-project-fields.yml | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/.github/workflows/validate-project-fields.yml b/.github/workflows/validate-project-fields.yml index 14f221abc..e3f086018 100644 --- a/.github/workflows/validate-project-fields.yml +++ b/.github/workflows/validate-project-fields.yml @@ -37,14 +37,20 @@ jobs: aws_region: ${{ env.AWS_REGION }} earthly_runner_secret: ${{ secrets.EARTHLY_RUNNER_SECRET }} - - name: Run Project Fields Validation - uses: input-output-hk/catalyst-ci/actions/run@master - if: always() - continue-on-error: false - with: - earthfile: ./utilities/project-fields-validator - flags: --allow-privileged - targets: validate-project-fields - target_flags: --GITHUB_REPOSITORY="${{ github.repository }}" --GITHUB_EVENT_NUMBER="${{ github.event.number || '0' }}" - runner_address: ${{ secrets.EARTHLY_SATELLITE_ADDRESS }} - artifact: false + # - name: Run Project Fields Validation + # uses: input-output-hk/catalyst-ci/actions/run@master + # if: always() + # continue-on-error: false + # with: + # earthfile: ./utilities/project-fields-validator + # flags: --allow-privileged + # targets: validate-project-fields + # target_flags: --GITHUB_REPOSITORY="${{ github.repository }}" --GITHUB_EVENT_NUMBER="${{ github.event.number || '0' }}" + # runner_address: ${{ secrets.EARTHLY_SATELLITE_ADDRESS }} + # artifact: false + + - name: test + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + earthly --secret GITHUB_TOKEN --allow-privileged ./utilities/project-fields-validator+validate-project-fields --GITHUB_REPOSITORY="input-output-hk/catalyst-ci" --GITHUB_EVENT_NUMBER="334" From bcd275e34577bea276e3b97d2d0ad1bf1e60c7b9 Mon Sep 17 00:00:00 2001 From: Joshua Gilman Date: Fri, 25 Oct 2024 07:33:16 -0700 Subject: [PATCH 30/37] wip: testing --- .github/workflows/validate-project-fields.yml | 34 +++++++++---------- utilities/project-fields-validator/Earthfile | 2 +- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/validate-project-fields.yml b/.github/workflows/validate-project-fields.yml index e3f086018..853b05847 100644 --- a/.github/workflows/validate-project-fields.yml +++ b/.github/workflows/validate-project-fields.yml @@ -37,20 +37,20 @@ jobs: aws_region: ${{ env.AWS_REGION }} earthly_runner_secret: ${{ secrets.EARTHLY_RUNNER_SECRET }} - # - name: Run Project Fields Validation - # uses: input-output-hk/catalyst-ci/actions/run@master - # if: always() - # continue-on-error: false - # with: - # earthfile: ./utilities/project-fields-validator - # flags: --allow-privileged - # targets: validate-project-fields - # target_flags: --GITHUB_REPOSITORY="${{ github.repository }}" --GITHUB_EVENT_NUMBER="${{ github.event.number || '0' }}" - # runner_address: ${{ secrets.EARTHLY_SATELLITE_ADDRESS }} - # artifact: false - - - name: test - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - earthly --secret GITHUB_TOKEN --allow-privileged ./utilities/project-fields-validator+validate-project-fields --GITHUB_REPOSITORY="input-output-hk/catalyst-ci" --GITHUB_EVENT_NUMBER="334" + - name: Run Project Fields Validation + uses: input-output-hk/catalyst-ci/actions/run@master + if: always() + continue-on-error: false + with: + earthfile: ./utilities/project-fields-validator + flags: --allow-privileged + targets: validate-project-fields + target_flags: --GITHUB_REPOSITORY="${{ github.repository }}" --GITHUB_EVENT_NUMBER="${{ github.event.number || '0' }}" + runner_address: ${{ secrets.EARTHLY_SATELLITE_ADDRESS }} + artifact: false + + # - name: test + # env: + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # run: | + # earthly --secret GITHUB_TOKEN --allow-privileged ./utilities/project-fields-validator+validate-project-fields --GITHUB_REPOSITORY="input-output-hk/catalyst-ci" --GITHUB_EVENT_NUMBER="334" diff --git a/utilities/project-fields-validator/Earthfile b/utilities/project-fields-validator/Earthfile index 63c2fbde6..200ee3c47 100644 --- a/utilities/project-fields-validator/Earthfile +++ b/utilities/project-fields-validator/Earthfile @@ -21,4 +21,4 @@ validate-project-fields: ENV GITHUB_REPOSITORY=${GITHUB_REPOSITORY} ENV GITHUB_EVENT_NUMBER=${GITHUB_EVENT_NUMBER} - RUN --secret GITHUB_TOKEN python3 main.py \ No newline at end of file + RUN --no-cache --secret GITHUB_TOKEN python3 main.py \ No newline at end of file From e427593209cb0072c9d1a1466c4b46e2dc289e9a Mon Sep 17 00:00:00 2001 From: Joshua Gilman Date: Fri, 25 Oct 2024 07:36:09 -0700 Subject: [PATCH 31/37] wip: testing --- utilities/project-fields-validator/Earthfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utilities/project-fields-validator/Earthfile b/utilities/project-fields-validator/Earthfile index 200ee3c47..c7ee2acea 100644 --- a/utilities/project-fields-validator/Earthfile +++ b/utilities/project-fields-validator/Earthfile @@ -21,4 +21,6 @@ validate-project-fields: ENV GITHUB_REPOSITORY=${GITHUB_REPOSITORY} ENV GITHUB_EVENT_NUMBER=${GITHUB_EVENT_NUMBER} + RUN --secret GITHUB_TOKEN echo "TOKEN: $GITHUB_TOKEN" + RUN --no-cache --secret GITHUB_TOKEN python3 main.py \ No newline at end of file From d27bbcb696214b5576d2746c0466fa86f62f06a7 Mon Sep 17 00:00:00 2001 From: Joshua Gilman Date: Fri, 25 Oct 2024 07:40:50 -0700 Subject: [PATCH 32/37] wip: testing --- .github/workflows/validate-project-fields.yml | 2 +- actions/run/dist/index.js | 2 +- actions/run/src/run.ts | 2 +- utilities/project-fields-validator/Earthfile | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/validate-project-fields.yml b/.github/workflows/validate-project-fields.yml index 853b05847..082bd9b94 100644 --- a/.github/workflows/validate-project-fields.yml +++ b/.github/workflows/validate-project-fields.yml @@ -38,7 +38,7 @@ jobs: earthly_runner_secret: ${{ secrets.EARTHLY_RUNNER_SECRET }} - name: Run Project Fields Validation - uses: input-output-hk/catalyst-ci/actions/run@master + uses: input-output-hk/catalyst-ci/actions/run@feat/validate-project-fields-in-prs-and-issues if: always() continue-on-error: false with: diff --git a/actions/run/dist/index.js b/actions/run/dist/index.js index c6eb696d2..8a8d7012d 100644 --- a/actions/run/dist/index.js +++ b/actions/run/dist/index.js @@ -2881,7 +2881,7 @@ async function run() { const artifactPath = core.getInput('artifact_path'); const earthfile = core.getInput('earthfile'); const flags = core.getInput('flags'); - const githubToken = core.getInput('githubToken'); + const githubToken = core.getInput('github_token'); const platform = core.getInput('platform'); const privileged = core.getBooleanInput('privileged'); const runnerAddress = core.getInput('runner_address'); diff --git a/actions/run/src/run.ts b/actions/run/src/run.ts index f98cbbb4d..eca77eba1 100644 --- a/actions/run/src/run.ts +++ b/actions/run/src/run.ts @@ -7,7 +7,7 @@ export async function run(): Promise { const artifactPath = core.getInput('artifact_path') const earthfile = core.getInput('earthfile') const flags = core.getInput('flags') - const githubToken = core.getInput('githubToken') + const githubToken = core.getInput('github_token') const platform = core.getInput('platform') const privileged = core.getBooleanInput('privileged') const runnerAddress = core.getInput('runner_address') diff --git a/utilities/project-fields-validator/Earthfile b/utilities/project-fields-validator/Earthfile index c7ee2acea..898c56bfb 100644 --- a/utilities/project-fields-validator/Earthfile +++ b/utilities/project-fields-validator/Earthfile @@ -21,6 +21,6 @@ validate-project-fields: ENV GITHUB_REPOSITORY=${GITHUB_REPOSITORY} ENV GITHUB_EVENT_NUMBER=${GITHUB_EVENT_NUMBER} - RUN --secret GITHUB_TOKEN echo "TOKEN: $GITHUB_TOKEN" + RUN --no-cache --secret GITHUB_TOKEN echo "TOKEN: $GITHUB_TOKEN" RUN --no-cache --secret GITHUB_TOKEN python3 main.py \ No newline at end of file From 039d0b936ddd089bb26ea64fb2a8c816c75e8261 Mon Sep 17 00:00:00 2001 From: Joshua Gilman Date: Fri, 25 Oct 2024 08:10:37 -0700 Subject: [PATCH 33/37] wip: cleanup --- .github/workflows/validate-project-fields.yml | 8 +------- utilities/project-fields-validator/Earthfile | 2 -- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/validate-project-fields.yml b/.github/workflows/validate-project-fields.yml index 082bd9b94..14f221abc 100644 --- a/.github/workflows/validate-project-fields.yml +++ b/.github/workflows/validate-project-fields.yml @@ -38,7 +38,7 @@ jobs: earthly_runner_secret: ${{ secrets.EARTHLY_RUNNER_SECRET }} - name: Run Project Fields Validation - uses: input-output-hk/catalyst-ci/actions/run@feat/validate-project-fields-in-prs-and-issues + uses: input-output-hk/catalyst-ci/actions/run@master if: always() continue-on-error: false with: @@ -48,9 +48,3 @@ jobs: target_flags: --GITHUB_REPOSITORY="${{ github.repository }}" --GITHUB_EVENT_NUMBER="${{ github.event.number || '0' }}" runner_address: ${{ secrets.EARTHLY_SATELLITE_ADDRESS }} artifact: false - - # - name: test - # env: - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # run: | - # earthly --secret GITHUB_TOKEN --allow-privileged ./utilities/project-fields-validator+validate-project-fields --GITHUB_REPOSITORY="input-output-hk/catalyst-ci" --GITHUB_EVENT_NUMBER="334" diff --git a/utilities/project-fields-validator/Earthfile b/utilities/project-fields-validator/Earthfile index 898c56bfb..200ee3c47 100644 --- a/utilities/project-fields-validator/Earthfile +++ b/utilities/project-fields-validator/Earthfile @@ -21,6 +21,4 @@ validate-project-fields: ENV GITHUB_REPOSITORY=${GITHUB_REPOSITORY} ENV GITHUB_EVENT_NUMBER=${GITHUB_EVENT_NUMBER} - RUN --no-cache --secret GITHUB_TOKEN echo "TOKEN: $GITHUB_TOKEN" - RUN --no-cache --secret GITHUB_TOKEN python3 main.py \ No newline at end of file From 976f449425c85564b701d3752e789eb7618b33a0 Mon Sep 17 00:00:00 2001 From: minikin Date: Sun, 27 Oct 2024 11:54:27 +0100 Subject: [PATCH 34/37] Update main.py --- utilities/project-fields-validator/main.py | 25 ++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/utilities/project-fields-validator/main.py b/utilities/project-fields-validator/main.py index 0bbc75696..18d7ba1fe 100644 --- a/utilities/project-fields-validator/main.py +++ b/utilities/project-fields-validator/main.py @@ -255,13 +255,19 @@ def print_validation_results(empty_fields: Set[str]) -> None: print("="*50) +def clean_env_var(var: str) -> str: + """Clean environment variable by removing quotes and extra whitespace""" + if var is None: + return None + return var.strip().strip('"\'') + def main(): try: env_vars = { - 'GITHUB_TOKEN': os.environ.get('GITHUB_TOKEN'), - 'GITHUB_REPOSITORY': os.environ.get('GITHUB_REPOSITORY'), - 'GITHUB_EVENT_NUMBER': os.environ.get('GITHUB_EVENT_NUMBER'), - 'PROJECT_NUMBER': os.environ.get('PROJECT_NUMBER') + 'GITHUB_TOKEN': clean_env_var(os.environ.get('GITHUB_TOKEN')), + 'GITHUB_REPOSITORY': clean_env_var(os.environ.get('GITHUB_REPOSITORY')), + 'GITHUB_EVENT_NUMBER': clean_env_var(os.environ.get('GITHUB_EVENT_NUMBER')), + 'PROJECT_NUMBER': clean_env_var(os.environ.get('PROJECT_NUMBER')) } # Validate environment variables @@ -269,9 +275,13 @@ def main(): if missing_vars: raise ValueError(f"Missing required environment variables: {', '.join(missing_vars)}") + try: + pr_number = int(env_vars['GITHUB_EVENT_NUMBER']) + project_number = int(env_vars['PROJECT_NUMBER']) + except ValueError as e: + raise ValueError(f"Invalid numeric value in environment variables: {str(e)}") + github_repository = env_vars['GITHUB_REPOSITORY'] - pr_number = int(env_vars['GITHUB_EVENT_NUMBER']) - project_number = int(env_vars['PROJECT_NUMBER']) org_name, repo_name = github_repository.split('/') print(f"\nValidating PR #{pr_number} in {github_repository}") @@ -315,6 +325,9 @@ def main(): if validation_errors: sys.exit(1) + except ValueError as e: + print(f"Configuration error: {str(e)}") + sys.exit(1) except Exception as e: print(f"Error: {str(e)}") sys.exit(1) From a7cd613ef9ffd44ec12252a5dba8fe384493a588 Mon Sep 17 00:00:00 2001 From: minikin Date: Sun, 27 Oct 2024 12:12:55 +0100 Subject: [PATCH 35/37] Update main.py --- utilities/project-fields-validator/main.py | 161 +++++++++++++++------ 1 file changed, 113 insertions(+), 48 deletions(-) diff --git a/utilities/project-fields-validator/main.py b/utilities/project-fields-validator/main.py index 18d7ba1fe..3cea37a98 100644 --- a/utilities/project-fields-validator/main.py +++ b/utilities/project-fields-validator/main.py @@ -5,6 +5,7 @@ from enum import Enum import requests from requests.exceptions import RequestException +import json class FieldType(Enum): TEXT = "text" @@ -21,13 +22,18 @@ class ProjectField: class GitHubAPIError(Exception): """Exception for GitHub API errors""" - pass + def __init__(self, message: str, response_data: Optional[Dict] = None): + super().__init__(message) + self.response_data = response_data class ProjectFieldsValidator: BASE_URL = "https://api.github.com" GRAPHQL_URL = f"{BASE_URL}/graphql" def __init__(self, github_token: str): + if not github_token: + raise ValueError("GitHub token is required but was empty") + self.headers = { "Authorization": f"Bearer {github_token}", "Accept": "application/vnd.github.v3+json" @@ -47,9 +53,25 @@ def _make_request(self, method: str, url: str, **kwargs) -> Dict[str, Any]: try: response = requests.request(method, url, headers=self.headers, **kwargs) response.raise_for_status() - return response.json() + + print(f"\nAPI Response Status: {response.status_code}") + + try: + data = response.json() + + if 'errors' in data: + error_messages = '; '.join(error.get('message', 'Unknown error') for error in data['errors']) + raise GitHubAPIError(f"GraphQL API errors: {error_messages}", data) + + if 'data' in data and data['data'] is None: + raise GitHubAPIError("API returned null data", data) + + return data + except json.JSONDecodeError as e: + raise GitHubAPIError(f"Failed to parse API response: {str(e)}") + except RequestException as e: - raise GitHubAPIError(f"GitHub API request failed: {str(e)}") from e + raise GitHubAPIError(f"GitHub API request failed: {str(e)}") def run_query(self, query: str, variables: Dict[str, Any]) -> Dict[str, Any]: """Execute a GraphQL query against GitHub's API.""" @@ -78,7 +100,22 @@ def get_pr_details(self, org_name: str, repo_name: str, pr_number: int) -> Dict[ } } """ - result = self.run_query(query, {"org": org_name, "repo": repo_name, "number": pr_number}) + + print(f"\nFetching PR details for {org_name}/{repo_name}#{pr_number}") + + result = self.run_query(query, { + "org": org_name, + "repo": repo_name, + "number": pr_number + }) + + if not result.get('data'): + raise GitHubAPIError("No data returned from API", result) + if not result['data'].get('repository'): + raise GitHubAPIError("Repository not found", result) + if not result['data']['repository'].get('pullRequest'): + raise GitHubAPIError(f"PR #{pr_number} not found", result) + return result['data']['repository']['pullRequest'] def assign_pr(self, org_name: str, repo_name: str, pr_number: int, assignee: str) -> None: @@ -183,23 +220,34 @@ def _paginate_items(self, query: str, org_name: str, project_number: int) -> Lis "cursor": cursor } - result = self.run_query(query, variables) - project_data = result['data']['organization']['projectV2']['items'] - valid_items = [ - item for item in project_data['nodes'] - if item.get('content') and isinstance(item['content'], dict) - ] + try: + result = self.run_query(query, variables) + if not result.get('data', {}).get('organization', {}).get('projectV2'): + raise GitHubAPIError("Could not access project data", result) + + project_data = result['data']['organization']['projectV2']['items'] + valid_items = [ + item for item in project_data['nodes'] + if item.get('content') and isinstance(item['content'], dict) + ] - all_items.extend(valid_items) - total_items += len(valid_items) + all_items.extend(valid_items) + total_items += len(valid_items) - sys.stdout.write(f"\rFetching project items... {total_items} found") - sys.stdout.flush() + sys.stdout.write(f"\rFetching project items... {total_items} found") + sys.stdout.flush() - if not project_data['pageInfo']['hasNextPage']: - break + if not project_data['pageInfo']['hasNextPage']: + break - cursor = project_data['pageInfo']['endCursor'] + cursor = project_data['pageInfo']['endCursor'] + + except GitHubAPIError as e: + print(f"\nError fetching project items: {str(e)}") + if e.response_data: + print("\nAPI Response data:") + print(json.dumps(e.response_data, indent=2)) + raise print("\n") return all_items @@ -270,19 +318,27 @@ def main(): 'PROJECT_NUMBER': clean_env_var(os.environ.get('PROJECT_NUMBER')) } - # Validate environment variables + debug_vars = env_vars.copy() + debug_vars['GITHUB_TOKEN'] = '[REDACTED]' if env_vars['GITHUB_TOKEN'] else None + print("\nEnvironment variables:") + for key, value in debug_vars.items(): + print(f"{key}: {value}") + missing_vars = [k for k, v in env_vars.items() if not v] if missing_vars: raise ValueError(f"Missing required environment variables: {', '.join(missing_vars)}") try: pr_number = int(env_vars['GITHUB_EVENT_NUMBER']) - project_number = int(env_vars['PROJECT_NUMBER']) + project_number = int(env_vars.get('PROJECT_NUMBER', '102')) # Default to 102 if not set except ValueError as e: raise ValueError(f"Invalid numeric value in environment variables: {str(e)}") github_repository = env_vars['GITHUB_REPOSITORY'] - org_name, repo_name = github_repository.split('/') + try: + org_name, repo_name = github_repository.split('/') + except ValueError: + raise ValueError(f"Invalid repository format: {github_repository}. Expected format: owner/repo") print(f"\nValidating PR #{pr_number} in {github_repository}") print(f"Project number: {project_number}") @@ -290,39 +346,46 @@ def main(): validator = ProjectFieldsValidator(env_vars['GITHUB_TOKEN']) - pr_details = validator.get_pr_details(org_name, repo_name, pr_number) - author = pr_details['author']['login'] - assignees = [node['login'] for node in pr_details['assignees']['nodes']] + try: + pr_details = validator.get_pr_details(org_name, repo_name, pr_number) + author = pr_details['author']['login'] + assignees = [node['login'] for node in pr_details['assignees']['nodes']] + + if not assignees: + print(f"\nAssigning PR to author @{author}") + validator.assign_pr(org_name, repo_name, pr_number, author) + + project_items = validator.get_project_items(org_name, project_number) + pr_items = [ + item for item in project_items + if (item['content'].get('number') == pr_number and + item['content'].get('repository', {}).get('name') == repo_name) + ] - if not assignees: - print(f"\nAssigning PR to author @{author}") - validator.assign_pr(org_name, repo_name, pr_number, author) + if not pr_items: + print(f"\nWarning: PR #{pr_number} is not linked to project #{project_number}") + print("Please add it to the project using the following steps:") + print("1. Go to the project board") + print("2. Click '+ Add items'") + print("3. Search for this PR") + print("4. Click 'Add selected items'") + sys.exit(0) - # Get and validate project items - project_items = validator.get_project_items(org_name, project_number) - pr_items = [ - item for item in project_items - if (item['content'].get('number') == pr_number and - item['content'].get('repository', {}).get('name') == repo_name) - ] + validation_errors = set() + for item in pr_items: + empty_fields = validator.validate_item(item) + validation_errors.update(empty_fields) - if not pr_items: - print(f"\nWarning: PR #{pr_number} is not linked to project #{project_number}") - print("Please add it to the project using the following steps:") - print("1. Go to the project board") - print("2. Click '+ Add items'") - print("3. Search for this PR") - print("4. Click 'Add selected items'") - sys.exit(0) + validator.print_validation_results(validation_errors) - validation_errors = set() - for item in pr_items: - empty_fields = validator.validate_item(item) - validation_errors.update(empty_fields) + if validation_errors: + sys.exit(1) - validator.print_validation_results(validation_errors) - - if validation_errors: + except GitHubAPIError as e: + print(f"\nError accessing GitHub API: {str(e)}") + if e.response_data: + print("\nAPI Response data:") + print(json.dumps(e.response_data, indent=2)) sys.exit(1) except ValueError as e: @@ -330,6 +393,8 @@ def main(): sys.exit(1) except Exception as e: print(f"Error: {str(e)}") + import traceback + traceback.print_exc() sys.exit(1) if __name__ == "__main__": From 6736c796a07315785c784e41dfdde726ec43f43b Mon Sep 17 00:00:00 2001 From: minikin Date: Sun, 27 Oct 2024 12:44:52 +0100 Subject: [PATCH 36/37] Update main.py --- utilities/project-fields-validator/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utilities/project-fields-validator/main.py b/utilities/project-fields-validator/main.py index 3cea37a98..8ac93d194 100644 --- a/utilities/project-fields-validator/main.py +++ b/utilities/project-fields-validator/main.py @@ -6,6 +6,7 @@ import requests from requests.exceptions import RequestException import json +import traceback class FieldType(Enum): TEXT = "text" @@ -393,7 +394,6 @@ def main(): sys.exit(1) except Exception as e: print(f"Error: {str(e)}") - import traceback traceback.print_exc() sys.exit(1) From 0efd05413230fe34132ddd5c4931114ba702bc2c Mon Sep 17 00:00:00 2001 From: minikin Date: Sun, 27 Oct 2024 22:10:14 +0100 Subject: [PATCH 37/37] Update validate-project-fields.yml --- .github/workflows/validate-project-fields.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/validate-project-fields.yml b/.github/workflows/validate-project-fields.yml index 14f221abc..732a44479 100644 --- a/.github/workflows/validate-project-fields.yml +++ b/.github/workflows/validate-project-fields.yml @@ -13,6 +13,7 @@ permissions: contents: write pull-requests: write id-token: write + repository-projects: write concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}