Skip to content

Commit

Permalink
feat: customizing version of pip in build, add legacy resolver as opt…
Browse files Browse the repository at this point in the history 
…ional (#1035)

* pip version is now adjusted by `--pip-version` flag in the build
command. If not provided latest pip is installed.
* `--use-deprecated=legacy-resolver` is now an optional parameter of pip
install (disabled by default) and can be added to pip install using
`--pip-legacy-resolver` flag in the build command
  • Loading branch information
@sgoral-splunk
sgoral-splunk authored Feb 1, 2024
1 parent 05171cd commit 751990c
Show file tree
Hide file tree
Showing 6 changed files with 198 additions and 8 deletions.
5 changes: 5 additions & 0 deletions docs/quickstart.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,11 @@ It takes the following parameters:
* `-v` / `--verbose` - [optional] show detailed information about
created/copied/modified/conflict files after build is complete.
This option is in experimental mode. Default: `False`.
* `--pip-version` - [optional] pip version that will be used to install python libraries. Default: `latest`.
* `--pip-legacy-resolver` - [optional] Use old pip dependency resolver by adding flag '--use-deprecated=legacy-resolver'
to pip install command. Default: `False`. PLEASE NOTE: this flag is deprecated and will be removed from pip in the future.
Instead of using this flag, the correct solution would be to fix the packages your project depends on to work properly with the new resolver.
Additionally, please note that this flag is not compatible with pip version `23.2`, use `23.2.1` instead.

#### Verbose mode

Expand Down
12 changes: 11 additions & 1 deletion splunk_add_on_ucc_framework/commands/build.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -450,6 +450,8 @@ def generate(
output_directory: Optional[str] = None,
python_binary_name: str = "python3",
verbose_file_summary_report: bool = False,
pip_version: str = "latest",
pip_legacy_resolver: bool = False,
) -> None:
logger.info(f"ucc-gen version {__version__} is used")
logger.info(f"Python binary name to use: {python_binary_name}")
Expand Down Expand Up @@ -545,6 +547,8 @@ def generate(
python_binary_name,
includes_ui=True,
os_libraries=global_config.os_libraries,
pip_version=pip_version,
pip_legacy_resolver=pip_legacy_resolver,
)
except SplunktaucclibNotFound as e:
logger.error(str(e))
Expand Down Expand Up @@ -609,7 +613,13 @@ def generate(
"Skipped generating UI components as globalConfig file does not exist"
)
ucc_lib_target = os.path.join(output_directory, ta_name, "lib")
install_python_libraries(source, ucc_lib_target, python_binary_name)
install_python_libraries(
source,
ucc_lib_target,
python_binary_name,
pip_version=pip_version,
pip_legacy_resolver=pip_legacy_resolver,
)
logger.info(
f"Installed add-on requirements into {ucc_lib_target} from {source}"
)
Expand Down
26 changes: 22 additions & 4 deletions splunk_add_on_ucc_framework/install_python_libraries.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,8 @@ def install_python_libraries(
python_binary_name: str,
includes_ui: bool = False,
os_libraries: Optional[List[OSDependentLibraryConfig]] = None,
pip_version: str = "latest",
pip_legacy_resolver: bool = False,
) -> None:
path_to_requirements_file = os.path.join(source_path, "lib", "requirements.txt")
if os.path.isfile(path_to_requirements_file):
Expand All @@ -107,6 +109,8 @@ def install_python_libraries(
requirements_file_path=path_to_requirements_file,
installation_path=ucc_lib_target,
installer=python_binary_name,
pip_version=pip_version,
pip_legacy_resolver=pip_legacy_resolver,
)
if includes_ui and not _pip_is_lib_installed(
installer=python_binary_name,
Expand Down Expand Up @@ -149,26 +153,40 @@ def install_libraries(
requirements_file_path: str,
installation_path: str,
installer: str,
pip_version: str = "latest",
pip_legacy_resolver: bool = False,
) -> None:
"""
Upgrades `pip` version to the latest one and installs requirements to the
specified path.
"""

pip_version = "23.1.2"
pip_update_command = f"--upgrade pip=={pip_version}"
if pip_version == "latest":
pip_update_command = "--upgrade pip"
else:
pip_update_command = f"--upgrade pip=={pip_version.strip()}"

if pip_version.strip() == "23.2" and pip_legacy_resolver:
logger.error(
"You cannot use the legacy resolver with pip 23.2. "
"Please remove '--pip-legacy-resolver' from your build command or "
"use a different version of pip e.g. 23.2.1"
)
sys.exit(1)

deps_resolver = "--use-deprecated=legacy-resolver " if pip_legacy_resolver else ""
pip_install_command = (
f'-r "{requirements_file_path}" '
f"--no-compile "
f"--prefer-binary "
f"--ignore-installed "
f"--use-deprecated=legacy-resolver "
f"{deps_resolver}"
f'--target "{installation_path}"'
)

_pip_install(
installer=installer, command=pip_update_command, command_desc="pip upgrade"
)

_pip_install(
installer=installer, command=pip_install_command, command_desc="pip install"
)
Expand Down
15 changes: 15 additions & 0 deletions splunk_add_on_ucc_framework/main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,19 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
"created/copied/modified/conflict files after build is complete"
),
)
build_parser.add_argument(
"--pip-version",
type=str,
help="pip version that will be used to install libraries.",
default="latest",
)
build_parser.add_argument(
"--pip-legacy-resolver",
action="store_true",
default=False,
help="Use old pip dependency resolver by adding flag '--use-deprecated=legacy-resolver' "
"to pip install command.",
)

package_parser = subparsers.add_parser("package", description="Package an add-on")
package_parser.add_argument(
Expand Down Expand Up @@ -189,6 +202,8 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
output_directory=args.output,
python_binary_name=args.python_binary_name,
verbose_file_summary_report=args.verbose,
pip_version=args.pip_version,
pip_legacy_resolver=args.pip_legacy_resolver,
)
if args.command == "package":
package.package(path_to_built_addon=args.path, output_directory=args.output)
Expand Down
71 changes: 68 additions & 3 deletions tests/unit/test_install_python_libraries.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,10 +81,9 @@ def test_install_libraries(mock_subprocess_call):
expected_install_command = (
'python3 -m pip install -r "package/lib/requirements.txt"'
" --no-compile --prefer-binary --ignore-installed "
'--use-deprecated=legacy-resolver --target "'
'/path/to/output/addon_name/lib"'
'--target "/path/to/output/addon_name/lib"'
)
expected_pip_update_command = "python3 -m pip install --upgrade pip==23.1.2"
expected_pip_update_command = "python3 -m pip install --upgrade pip"
mock_subprocess_call.assert_has_calls(
[
mock.call(expected_pip_update_command, shell=True, env=None),
Expand Down Expand Up @@ -409,3 +408,69 @@ def test_install_libraries_version_mismatch(
assert version_mismatch_log in caplog.messages
assert error_description in caplog.messages
mock_remove_packages.assert_not_called()


@mock.patch("subprocess.call", autospec=True)
def test_install_libraries_custom_pip(mock_subprocess_call):
mock_subprocess_call.return_value = 0

install_libraries(
"package/lib/requirements.txt",
"/path/to/output/addon_name/lib",
"python3",
pip_version="21.666.666",
)

expected_install_command = (
'python3 -m pip install -r "package/lib/requirements.txt"'
" --no-compile --prefer-binary --ignore-installed "
'--target "/path/to/output/addon_name/lib"'
)
expected_pip_update_command = "python3 -m pip install --upgrade pip==21.666.666"
mock_subprocess_call.assert_has_calls(
[
mock.call(expected_pip_update_command, shell=True, env=None),
mock.call(expected_install_command, shell=True, env=None),
]
)


@mock.patch("subprocess.call", autospec=True)
def test_install_libraries_legacy_resolver(mock_subprocess_call):
mock_subprocess_call.return_value = 0

install_libraries(
"package/lib/requirements.txt",
"/path/to/output/addon_name/lib",
"python3",
pip_legacy_resolver=True,
)

expected_install_command = (
'python3 -m pip install -r "package/lib/requirements.txt"'
" --no-compile --prefer-binary --ignore-installed "
'--use-deprecated=legacy-resolver --target "/path/to/output/addon_name/lib"'
)
expected_pip_update_command = "python3 -m pip install --upgrade pip"
mock_subprocess_call.assert_has_calls(
[
mock.call(expected_pip_update_command, shell=True, env=None),
mock.call(expected_install_command, shell=True, env=None),
]
)


def test_install_libraries_legacy_resolver_with_wrong_pip(caplog):
with pytest.raises(SystemExit):
install_libraries(
"package/lib/requirements.txt",
"/path/to/output/addon_name/lib",
"python3",
pip_version=" 23.2 ",
pip_legacy_resolver=True,
)
expected_msg = (
"You cannot use the legacy resolver with pip 23.2. "
"Please remove '--pip-legacy-resolver' from your build command or use a different version of pip e.g. 23.2.1"
)
assert expected_msg in caplog.text
77 changes: 77 additions & 0 deletions tests/unit/test_main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
"output_directory": None,
"python_binary_name": "python3",
"verbose_file_summary_report": False,
"pip_version": "latest",
"pip_legacy_resolver": False,
},
),
(
Expand All @@ -28,6 +30,8 @@
"output_directory": None,
"python_binary_name": "python3",
"verbose_file_summary_report": False,
"pip_version": "latest",
"pip_legacy_resolver": False,
},
),
(
Expand All @@ -39,6 +43,8 @@
"output_directory": None,
"python_binary_name": "python3",
"verbose_file_summary_report": False,
"pip_version": "latest",
"pip_legacy_resolver": False,
},
),
(
Expand All @@ -50,6 +56,8 @@
"output_directory": None,
"python_binary_name": "python3",
"verbose_file_summary_report": False,
"pip_version": "latest",
"pip_legacy_resolver": False,
},
),
(
Expand All @@ -61,6 +69,8 @@
"output_directory": None,
"python_binary_name": "python3",
"verbose_file_summary_report": True,
"pip_version": "latest",
"pip_legacy_resolver": False,
},
),
(
Expand All @@ -79,6 +89,8 @@
"output_directory": None,
"python_binary_name": "python.exe",
"verbose_file_summary_report": False,
"pip_version": "latest",
"pip_legacy_resolver": False,
},
),
(
Expand All @@ -99,6 +111,8 @@
"output_directory": None,
"python_binary_name": "python.exe",
"verbose_file_summary_report": False,
"pip_version": "latest",
"pip_legacy_resolver": False,
},
),
(
Expand All @@ -121,6 +135,8 @@
"output_directory": "new_output",
"python_binary_name": "python.exe",
"verbose_file_summary_report": False,
"pip_version": "latest",
"pip_legacy_resolver": False,
},
),
(
Expand All @@ -144,6 +160,8 @@
"output_directory": "new_output",
"python_binary_name": "python.exe",
"verbose_file_summary_report": False,
"pip_version": "latest",
"pip_legacy_resolver": False,
},
),
(
Expand All @@ -168,6 +186,65 @@
"output_directory": "new_output",
"python_binary_name": "python.exe",
"verbose_file_summary_report": True,
"pip_version": "latest",
"pip_legacy_resolver": False,
},
),
(
[
"build",
"-v",
"--source",
"package",
"--config",
"/path/to/globalConfig.yaml",
"--ta-version",
"2.2.0",
"--output",
"new_output",
"--python-binary-name",
"python.exe",
"--pip-version",
"21.0.0",
],
{
"source": "package",
"config_path": "/path/to/globalConfig.yaml",
"addon_version": "2.2.0",
"output_directory": "new_output",
"python_binary_name": "python.exe",
"verbose_file_summary_report": True,
"pip_version": "21.0.0",
"pip_legacy_resolver": False,
},
),
(
[
"build",
"-v",
"--source",
"package",
"--config",
"/path/to/globalConfig.yaml",
"--ta-version",
"2.2.0",
"--output",
"new_output",
"--python-binary-name",
"python.exe",
"--pip-version",
"21.0.0",
"--pip-legacy-resolver",
],
{
"source": "package",
"config_path": "/path/to/globalConfig.yaml",
"addon_version": "2.2.0",
"output_directory": "new_output",
"python_binary_name": "python.exe",
"verbose_file_summary_report": True,
"pip_version": "21.0.0",
"pip_legacy_resolver": True,
},
),
],
Expand Down

0 comments on commit 751990c

Please sign in to comment.