diff --git a/docs/quickstart.md b/docs/quickstart.md index 6e75d0896..b1205813d 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -121,6 +121,11 @@ It takes the following parameters: * `-v` / `--verbose` - [optional] show detailed information about created/copied/modified/conflict files after build is complete. This option is in experimental mode. Default: `False`. +* `--pip-version` - [optional] pip version that will be used to install python libraries. Default: `latest`. +* `--pip-legacy-resolver` - [optional] Use old pip dependency resolver by adding flag '--use-deprecated=legacy-resolver' + to pip install command. Default: `False`. PLEASE NOTE: this flag is deprecated and will be removed from pip in the future. +Instead of using this flag, the correct solution would be to fix the packages your project depends on to work properly with the new resolver. +Additionally, please note that this flag is not compatible with pip version `23.2`, use `23.2.1` instead. #### Verbose mode diff --git a/splunk_add_on_ucc_framework/commands/build.py b/splunk_add_on_ucc_framework/commands/build.py index a6477856b..4ca985543 100644 --- a/splunk_add_on_ucc_framework/commands/build.py +++ b/splunk_add_on_ucc_framework/commands/build.py @@ -450,6 +450,8 @@ def generate( output_directory: Optional[str] = None, python_binary_name: str = "python3", verbose_file_summary_report: bool = False, + pip_version: str = "latest", + pip_legacy_resolver: bool = False, ) -> None: logger.info(f"ucc-gen version {__version__} is used") logger.info(f"Python binary name to use: {python_binary_name}") @@ -545,6 +547,8 @@ def generate( python_binary_name, includes_ui=True, os_libraries=global_config.os_libraries, + pip_version=pip_version, + pip_legacy_resolver=pip_legacy_resolver, ) except SplunktaucclibNotFound as e: logger.error(str(e)) @@ -609,7 +613,13 @@ def generate( "Skipped generating UI components as globalConfig file does not exist" ) ucc_lib_target = os.path.join(output_directory, ta_name, "lib") - install_python_libraries(source, ucc_lib_target, python_binary_name) + install_python_libraries( + source, + ucc_lib_target, + python_binary_name, + pip_version=pip_version, + pip_legacy_resolver=pip_legacy_resolver, + ) logger.info( f"Installed add-on requirements into {ucc_lib_target} from {source}" ) diff --git a/splunk_add_on_ucc_framework/install_python_libraries.py b/splunk_add_on_ucc_framework/install_python_libraries.py index cadcf2e29..f499031b7 100644 --- a/splunk_add_on_ucc_framework/install_python_libraries.py +++ b/splunk_add_on_ucc_framework/install_python_libraries.py @@ -97,6 +97,8 @@ def install_python_libraries( python_binary_name: str, includes_ui: bool = False, os_libraries: Optional[List[OSDependentLibraryConfig]] = None, + pip_version: str = "latest", + pip_legacy_resolver: bool = False, ) -> None: path_to_requirements_file = os.path.join(source_path, "lib", "requirements.txt") if os.path.isfile(path_to_requirements_file): @@ -107,6 +109,8 @@ def install_python_libraries( requirements_file_path=path_to_requirements_file, installation_path=ucc_lib_target, installer=python_binary_name, + pip_version=pip_version, + pip_legacy_resolver=pip_legacy_resolver, ) if includes_ui and not _pip_is_lib_installed( installer=python_binary_name, @@ -149,26 +153,40 @@ def install_libraries( requirements_file_path: str, installation_path: str, installer: str, + pip_version: str = "latest", + pip_legacy_resolver: bool = False, ) -> None: """ Upgrades `pip` version to the latest one and installs requirements to the specified path. """ - pip_version = "23.1.2" - pip_update_command = f"--upgrade pip=={pip_version}" + if pip_version == "latest": + pip_update_command = "--upgrade pip" + else: + pip_update_command = f"--upgrade pip=={pip_version.strip()}" + + if pip_version.strip() == "23.2" and pip_legacy_resolver: + logger.error( + "You cannot use the legacy resolver with pip 23.2. " + "Please remove '--pip-legacy-resolver' from your build command or " + "use a different version of pip e.g. 23.2.1" + ) + sys.exit(1) + + deps_resolver = "--use-deprecated=legacy-resolver " if pip_legacy_resolver else "" pip_install_command = ( f'-r "{requirements_file_path}" ' f"--no-compile " f"--prefer-binary " f"--ignore-installed " - f"--use-deprecated=legacy-resolver " + f"{deps_resolver}" f'--target "{installation_path}"' ) - _pip_install( installer=installer, command=pip_update_command, command_desc="pip upgrade" ) + _pip_install( installer=installer, command=pip_install_command, command_desc="pip install" ) diff --git a/splunk_add_on_ucc_framework/main.py b/splunk_add_on_ucc_framework/main.py index c38444ab4..7273a8fb6 100644 --- a/splunk_add_on_ucc_framework/main.py +++ b/splunk_add_on_ucc_framework/main.py @@ -113,6 +113,19 @@ def main(argv: Optional[Sequence[str]] = None) -> int: "created/copied/modified/conflict files after build is complete" ), ) + build_parser.add_argument( + "--pip-version", + type=str, + help="pip version that will be used to install libraries.", + default="latest", + ) + build_parser.add_argument( + "--pip-legacy-resolver", + action="store_true", + default=False, + help="Use old pip dependency resolver by adding flag '--use-deprecated=legacy-resolver' " + "to pip install command.", + ) package_parser = subparsers.add_parser("package", description="Package an add-on") package_parser.add_argument( @@ -189,6 +202,8 @@ def main(argv: Optional[Sequence[str]] = None) -> int: output_directory=args.output, python_binary_name=args.python_binary_name, verbose_file_summary_report=args.verbose, + pip_version=args.pip_version, + pip_legacy_resolver=args.pip_legacy_resolver, ) if args.command == "package": package.package(path_to_built_addon=args.path, output_directory=args.output) diff --git a/tests/unit/test_install_python_libraries.py b/tests/unit/test_install_python_libraries.py index 6fa84abe5..5defd5e73 100644 --- a/tests/unit/test_install_python_libraries.py +++ b/tests/unit/test_install_python_libraries.py @@ -81,10 +81,9 @@ def test_install_libraries(mock_subprocess_call): expected_install_command = ( 'python3 -m pip install -r "package/lib/requirements.txt"' " --no-compile --prefer-binary --ignore-installed " - '--use-deprecated=legacy-resolver --target "' - '/path/to/output/addon_name/lib"' + '--target "/path/to/output/addon_name/lib"' ) - expected_pip_update_command = "python3 -m pip install --upgrade pip==23.1.2" + expected_pip_update_command = "python3 -m pip install --upgrade pip" mock_subprocess_call.assert_has_calls( [ mock.call(expected_pip_update_command, shell=True, env=None), @@ -409,3 +408,69 @@ def test_install_libraries_version_mismatch( assert version_mismatch_log in caplog.messages assert error_description in caplog.messages mock_remove_packages.assert_not_called() + + +@mock.patch("subprocess.call", autospec=True) +def test_install_libraries_custom_pip(mock_subprocess_call): + mock_subprocess_call.return_value = 0 + + install_libraries( + "package/lib/requirements.txt", + "/path/to/output/addon_name/lib", + "python3", + pip_version="21.666.666", + ) + + expected_install_command = ( + 'python3 -m pip install -r "package/lib/requirements.txt"' + " --no-compile --prefer-binary --ignore-installed " + '--target "/path/to/output/addon_name/lib"' + ) + expected_pip_update_command = "python3 -m pip install --upgrade pip==21.666.666" + mock_subprocess_call.assert_has_calls( + [ + mock.call(expected_pip_update_command, shell=True, env=None), + mock.call(expected_install_command, shell=True, env=None), + ] + ) + + +@mock.patch("subprocess.call", autospec=True) +def test_install_libraries_legacy_resolver(mock_subprocess_call): + mock_subprocess_call.return_value = 0 + + install_libraries( + "package/lib/requirements.txt", + "/path/to/output/addon_name/lib", + "python3", + pip_legacy_resolver=True, + ) + + expected_install_command = ( + 'python3 -m pip install -r "package/lib/requirements.txt"' + " --no-compile --prefer-binary --ignore-installed " + '--use-deprecated=legacy-resolver --target "/path/to/output/addon_name/lib"' + ) + expected_pip_update_command = "python3 -m pip install --upgrade pip" + mock_subprocess_call.assert_has_calls( + [ + mock.call(expected_pip_update_command, shell=True, env=None), + mock.call(expected_install_command, shell=True, env=None), + ] + ) + + +def test_install_libraries_legacy_resolver_with_wrong_pip(caplog): + with pytest.raises(SystemExit): + install_libraries( + "package/lib/requirements.txt", + "/path/to/output/addon_name/lib", + "python3", + pip_version=" 23.2 ", + pip_legacy_resolver=True, + ) + expected_msg = ( + "You cannot use the legacy resolver with pip 23.2. " + "Please remove '--pip-legacy-resolver' from your build command or use a different version of pip e.g. 23.2.1" + ) + assert expected_msg in caplog.text diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index a238bb749..a424feeec 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -17,6 +17,8 @@ "output_directory": None, "python_binary_name": "python3", "verbose_file_summary_report": False, + "pip_version": "latest", + "pip_legacy_resolver": False, }, ), ( @@ -28,6 +30,8 @@ "output_directory": None, "python_binary_name": "python3", "verbose_file_summary_report": False, + "pip_version": "latest", + "pip_legacy_resolver": False, }, ), ( @@ -39,6 +43,8 @@ "output_directory": None, "python_binary_name": "python3", "verbose_file_summary_report": False, + "pip_version": "latest", + "pip_legacy_resolver": False, }, ), ( @@ -50,6 +56,8 @@ "output_directory": None, "python_binary_name": "python3", "verbose_file_summary_report": False, + "pip_version": "latest", + "pip_legacy_resolver": False, }, ), ( @@ -61,6 +69,8 @@ "output_directory": None, "python_binary_name": "python3", "verbose_file_summary_report": True, + "pip_version": "latest", + "pip_legacy_resolver": False, }, ), ( @@ -79,6 +89,8 @@ "output_directory": None, "python_binary_name": "python.exe", "verbose_file_summary_report": False, + "pip_version": "latest", + "pip_legacy_resolver": False, }, ), ( @@ -99,6 +111,8 @@ "output_directory": None, "python_binary_name": "python.exe", "verbose_file_summary_report": False, + "pip_version": "latest", + "pip_legacy_resolver": False, }, ), ( @@ -121,6 +135,8 @@ "output_directory": "new_output", "python_binary_name": "python.exe", "verbose_file_summary_report": False, + "pip_version": "latest", + "pip_legacy_resolver": False, }, ), ( @@ -144,6 +160,8 @@ "output_directory": "new_output", "python_binary_name": "python.exe", "verbose_file_summary_report": False, + "pip_version": "latest", + "pip_legacy_resolver": False, }, ), ( @@ -168,6 +186,65 @@ "output_directory": "new_output", "python_binary_name": "python.exe", "verbose_file_summary_report": True, + "pip_version": "latest", + "pip_legacy_resolver": False, + }, + ), + ( + [ + "build", + "-v", + "--source", + "package", + "--config", + "/path/to/globalConfig.yaml", + "--ta-version", + "2.2.0", + "--output", + "new_output", + "--python-binary-name", + "python.exe", + "--pip-version", + "21.0.0", + ], + { + "source": "package", + "config_path": "/path/to/globalConfig.yaml", + "addon_version": "2.2.0", + "output_directory": "new_output", + "python_binary_name": "python.exe", + "verbose_file_summary_report": True, + "pip_version": "21.0.0", + "pip_legacy_resolver": False, + }, + ), + ( + [ + "build", + "-v", + "--source", + "package", + "--config", + "/path/to/globalConfig.yaml", + "--ta-version", + "2.2.0", + "--output", + "new_output", + "--python-binary-name", + "python.exe", + "--pip-version", + "21.0.0", + "--pip-legacy-resolver", + ], + { + "source": "package", + "config_path": "/path/to/globalConfig.yaml", + "addon_version": "2.2.0", + "output_directory": "new_output", + "python_binary_name": "python.exe", + "verbose_file_summary_report": True, + "pip_version": "21.0.0", + "pip_legacy_resolver": True, }, ), ],