From 4ab4707a81a103adfa0d9a5bdb90eba23e428b73 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 22 Dec 2024 08:35:59 +0100 Subject: [PATCH 1/2] Fix breeze output static checks failure (#45142) The new click 8.1.8 makes the dictionary of options used to calculate hash of commands different for setup command. by bumping it to minimum 8.1.8 version we make the hash the same for both CI and locally installed breeze (click 8.1.8 change in pyproject.toml of breeze will force reinstallation of breeze for everyone locally). --- dev/breeze/README.md | 2 +- dev/breeze/doc/images/output_setup.txt | 2 +- .../doc/images/output_setup_regenerate-command-images.txt | 2 +- dev/breeze/pyproject.toml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dev/breeze/README.md b/dev/breeze/README.md index fa0184a1901c7..1e9b7e97fe696 100644 --- a/dev/breeze/README.md +++ b/dev/breeze/README.md @@ -128,6 +128,6 @@ PLEASE DO NOT MODIFY THE HASH BELOW! IT IS AUTOMATICALLY UPDATED BY PRE-COMMIT. --------------------------------------------------------------------------------------------------------- -Package config hash: d9c2c903940272a1640ebf7e9db4e4df4a153bbc6c491932978ae805a5b4031b33ae621321f48c8e157a3ad5dd16203fa45eb4819fdef04fcec2cb0a7e36dfab +Package config hash: 6624da65e725e16cee3203afa884da62b719903470063a9d859bb4fae71346c9c4ee58d182b7f2a53f53eb10a72d62b4ae38d4e3d8a03cc293ae97aa62ec180e --------------------------------------------------------------------------------------------------------- diff --git a/dev/breeze/doc/images/output_setup.txt b/dev/breeze/doc/images/output_setup.txt index 274751197daaf..21f3c7e2527e0 100644 --- a/dev/breeze/doc/images/output_setup.txt +++ b/dev/breeze/doc/images/output_setup.txt @@ -1 +1 @@ -08c78d9dddd037a2ade6b751c5a22ff9 +ba00ab3fb2ed5a777684878c28b3ce65 diff --git a/dev/breeze/doc/images/output_setup_regenerate-command-images.txt b/dev/breeze/doc/images/output_setup_regenerate-command-images.txt index d26de039adee2..1baedb3358f51 100644 --- a/dev/breeze/doc/images/output_setup_regenerate-command-images.txt +++ b/dev/breeze/doc/images/output_setup_regenerate-command-images.txt @@ -1 +1 @@ -9001008210b2148e49e4fd11f11bb25a +74516122bcea4a5445279431d73c7cd0 diff --git a/dev/breeze/pyproject.toml b/dev/breeze/pyproject.toml index fe3c0b7cb3141..fcbb300ce33f0 100644 --- a/dev/breeze/pyproject.toml +++ b/dev/breeze/pyproject.toml @@ -45,7 +45,7 @@ requires-python = "~=3.9" dependencies = [ "black>=23.11.0", - "click>=8.1.7", + "click>=8.1.8", "filelock>=3.13.0", # # We pin flit in order to make sure reproducibility of provider packages is maintained From a8c4871ba0de5087b934f540129974de926e4038 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 22 Dec 2024 09:30:13 +0100 Subject: [PATCH 2/2] Prevent occasional 500 errors when installing npm dependencies (#45143) When we are installing npm dependencies, occasionally in CI we get 500 Internal Error. This change adds max 3 tries of such installations - retrying when 500 internal error is detected. --- scripts/ci/pre_commit/compile_ui_assets.py | 19 +++++++++++++++---- scripts/ci/pre_commit/compile_www_assets.py | 13 ++++++++++++- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/scripts/ci/pre_commit/compile_ui_assets.py b/scripts/ci/pre_commit/compile_ui_assets.py index b4c38e3d7cc60..158bde1468ec5 100755 --- a/scripts/ci/pre_commit/compile_ui_assets.py +++ b/scripts/ci/pre_commit/compile_ui_assets.py @@ -33,6 +33,8 @@ AIRFLOW_SOURCES_PATH = Path(__file__).parents[3].resolve() UI_HASH_FILE = AIRFLOW_SOURCES_PATH / ".build" / "ui" / "hash.txt" +INTERNAL_SERVER_ERROR = "500 Internal Server Error" + def get_directory_hash(directory: Path, skip_path_regexp: str | None = None) -> str: files = sorted(directory.rglob("*")) @@ -68,10 +70,19 @@ def get_directory_hash(directory: Path, skip_path_regexp: str | None = None) -> shutil.rmtree(dist_directory, ignore_errors=True) env = os.environ.copy() env["FORCE_COLOR"] = "true" - subprocess.check_call( - ["pnpm", "install", "--frozen-lockfile", "--config.confirmModulesPurge=false"], - cwd=os.fspath(ui_directory), - ) + for try_num in range(3): + print(f"### Trying to install yarn dependencies: attempt: {try_num} ###") + result = subprocess.run( + ["pnpm", "install", "--frozen-lockfile", "--config.confirmModulesPurge=false"], + cwd=os.fspath(ui_directory), + text=True, + check=False, + ) + if result.returncode == 0: + break + if try_num == 2 or INTERNAL_SERVER_ERROR not in result.stderr + result.stdout: + print(result.stdout + "\n" + result.stderr) + sys.exit(result.returncode) subprocess.check_call(["pnpm", "run", "build"], cwd=os.fspath(ui_directory), env=env) new_hash = get_directory_hash(ui_directory, skip_path_regexp=r".*node_modules.*") UI_HASH_FILE.write_text(new_hash) diff --git a/scripts/ci/pre_commit/compile_www_assets.py b/scripts/ci/pre_commit/compile_www_assets.py index bf2664685ed6c..bf4d9622412f4 100755 --- a/scripts/ci/pre_commit/compile_www_assets.py +++ b/scripts/ci/pre_commit/compile_www_assets.py @@ -52,6 +52,8 @@ def get_directory_hash(directory: Path, skip_path_regexp: str | None = None) -> f"To run this script, run the ./{__file__} command" ) +INTERNAL_SERVER_ERROR = "500 Internal Server Error" + if __name__ == "__main__": www_directory = AIRFLOW_SOURCES_PATH / "airflow" / "www" node_modules_directory = www_directory / "node_modules" @@ -68,7 +70,16 @@ def get_directory_hash(directory: Path, skip_path_regexp: str | None = None) -> shutil.rmtree(dist_directory, ignore_errors=True) env = os.environ.copy() env["FORCE_COLOR"] = "true" - subprocess.check_call(["yarn", "install", "--frozen-lockfile"], cwd=os.fspath(www_directory)) + for try_num in range(3): + print(f"### Trying to install yarn dependencies: attempt: {try_num} ###") + result = subprocess.run( + ["yarn", "install", "--frozen-lockfile"], cwd=os.fspath(www_directory), text=True, check=False + ) + if result.returncode == 0: + break + if try_num == 2 or INTERNAL_SERVER_ERROR not in result.stderr + result.stdout: + print(result.stdout + "\n" + result.stderr) + sys.exit(result.returncode) subprocess.check_call(["yarn", "run", "build"], cwd=os.fspath(www_directory), env=env) new_hash = get_directory_hash(www_directory, skip_path_regexp=r".*node_modules.*") WWW_HASH_FILE.write_text(new_hash)