diff --git a/pip/cmdoptions.py b/pip/cmdoptions.py index 076204523e2..9eed409d331 100644 --- a/pip/cmdoptions.py +++ b/pip/cmdoptions.py @@ -434,6 +434,16 @@ def editable(): help="Don't periodically check PyPI to determine whether a new version " "of pip is available for download. Implied with --no-index.") +# Deprecated, Remove later +always_unzip = partial( + Option, + '-Z', '--always-unzip', + dest='always_unzip', + action='store_true', + help=SUPPRESS_HELP, +) + + ########## # groups # ########## diff --git a/pip/exceptions.py b/pip/exceptions.py index df6f3aeda28..90d8dfc40f4 100644 --- a/pip/exceptions.py +++ b/pip/exceptions.py @@ -19,8 +19,21 @@ class DistributionNotFound(InstallationError): class RequirementsFileParseError(PipError): - """Raised when an invalid state is encountered during requirement file - parsing.""" + """Raised when a general error occurs parsing a requirements file line.""" + + +class ReqFileOnlyOneReqPerLineError(PipError): + """Raised when more than one requirement is found on a line in a requirements + file.""" + + +class ReqFileOnleOneOptionPerLineError(PipError): + """Raised when an option is not allowed in a requirements file.""" + + +class ReqFileOptionNotAllowedWithReqError(PipError): + """Raised when an option is not allowed on a requirement line in a requirements + file.""" class BestVersionAlreadyInstalled(PipError): diff --git a/pip/req/req_file.py b/pip/req/req_file.py index e9603ad999b..9c9cca23589 100644 --- a/pip/req/req_file.py +++ b/pip/req/req_file.py @@ -1,6 +1,5 @@ - """ -Routines for parsing requirements files (i.e. requirements.txt). +Requirements file parsing """ from __future__ import absolute_import @@ -15,65 +14,39 @@ from pip.download import get_file_content from pip.req.req_install import InstallRequirement -from pip.exceptions import RequirementsFileParseError +from pip.exceptions import (RequirementsFileParseError, + ReqFileOnlyOneReqPerLineError, + ReqFileOnleOneOptionPerLineError, + ReqFileOptionNotAllowedWithReqError) from pip.utils import normalize_name from pip import cmdoptions - -# ---------------------------------------------------------------------------- -# Flags that don't take any options. -parser_flags = set([ - '--no-index', - '--allow-all-external', - '--no-use-wheel', -]) - -# Flags that take options. -parser_options = set([ - '-i', '--index-url', - '-f', '--find-links', - '--extra-index-url', - '--allow-external', - '--allow-unverified', -]) - -# Encountering any of these is a no-op. -parser_compat = set([ - '-Z', '--always-unzip', - '--use-wheel', # Default in 1.5 - '--no-allow-external', # Remove in 7.0 - '--no-allow-insecure', # Remove in 7.0 -]) - -# ---------------------------------------------------------------------------- -# Requirement lines may take options. For example: -# INITools==0.2 --install-option="--prefix=/opt" --global-option="-v" -# We use optparse to reliably parse these lines. -_req_parser = optparse.OptionParser(add_help_option=False) -_req_parser.add_option(cmdoptions.install_options()) -_req_parser.add_option(cmdoptions.global_options()) -_req_parser.disable_interspersed_args() - - -# By default optparse sys.exits on parsing errors. We want to wrap -# that in our own exception. -def parser_exit(self, msg): - raise RequirementsFileParseError(msg) -_req_parser.exit = parser_exit - -# ---------------------------------------------------------------------------- -# Pre-compiled regex. -_scheme_re = re.compile(r'^(http|https|file):', re.I) -_comment_re = re.compile(r'(^|\s)+#.*$') - -# ---------------------------------------------------------------------------- -# The types of lines understood by the requirements file parser. -REQUIREMENT = 0 -REQUIREMENT_FILE = 1 -REQUIREMENT_EDITABLE = 2 -FLAG = 3 -OPTION = 4 -IGNORE = 5 +__all__ = ['parse_requirements'] + +SCHEME_RE = re.compile(r'^(http|https|file):', re.I) +COMMENT_RE = re.compile(r'(^|\s)+#.*$') + +SUPPORTED_OPTIONS = [ + cmdoptions.editable, + cmdoptions.requirements, + cmdoptions.no_index, + cmdoptions.index_url, + cmdoptions.find_links, + cmdoptions.extra_index_url, + cmdoptions.allow_external, + cmdoptions.no_allow_external, + cmdoptions.allow_unsafe, + cmdoptions.no_allow_unsafe, + cmdoptions.use_wheel, + cmdoptions.no_use_wheel, + cmdoptions.always_unzip, +] + +# options allowed on requirement lines +SUPPORTED_OPTIONS_REQ = [ + cmdoptions.install_options, + cmdoptions.global_options, +] def parse_requirements(filename, finder=None, comes_from=None, options=None, @@ -97,166 +70,152 @@ def parse_requirements(filename, finder=None, comes_from=None, options=None, filename, comes_from=comes_from, session=session ) - parser = parse_content( - filename, content, finder, comes_from, options, session, cache_root - ) - - for item in parser: - yield item + lines = content.splitlines() + lines = ignore_comments(lines) + lines = join_lines(lines) + lines = skip_regex(lines, options) + for line_number, line in enumerate(lines, 1): + req_iter = process_line(line, filename, line_number, finder, + comes_from, options, session) + for req in req_iter: + yield req -def parse_content(filename, content, finder=None, comes_from=None, - options=None, session=None, cache_root=None): - # Split, sanitize and join lines with continuations. - content = content.splitlines() - content = ignore_comments(content) - content = join_lines(content) - - # Optionally exclude lines that match '--skip-requirements-regex'. - skip_regex = options.skip_requirements_regex if options else None - if skip_regex: - content = filterfalse(re.compile(skip_regex).search, content) - - for line_number, line in enumerate(content, 1): - # The returned value depends on the type of line that was parsed. - linetype, value = parse_line(line) - - # --------------------------------------------------------------------- - if linetype == REQUIREMENT: - req, opts = value - comes_from = '-r %s (line %s)' % (filename, line_number) - isolated = options.isolated_mode if options else False - yield InstallRequirement.from_line( - req, comes_from, isolated=isolated, options=opts, - cache_root=cache_root) - - # --------------------------------------------------------------------- - elif linetype == REQUIREMENT_EDITABLE: - comes_from = '-r %s (line %s)' % (filename, line_number) - isolated = options.isolated_mode if options else False - default_vcs = options.default_vcs if options else None - yield InstallRequirement.from_editable( - value, comes_from=comes_from, - default_vcs=default_vcs, isolated=isolated, - cache_root=cache_root) - - # --------------------------------------------------------------------- - elif linetype == REQUIREMENT_FILE: - if _scheme_re.search(filename): - # Relative to an URL. - req_url = urllib_parse.urljoin(filename, value) - elif not _scheme_re.search(value): - req_dir = os.path.dirname(filename) - req_url = os.path.join(os.path.dirname(filename), value) - # TODO: Why not use `comes_from='-r {} (line {})'` here as well? - parser = parse_requirements( - req_url, finder, comes_from, options, session, - cache_root=cache_root) - for req in parser: - yield req - - # --------------------------------------------------------------------- - elif linetype == FLAG: - if not finder: - continue - - if finder and value == '--no-use-wheel': - finder.use_wheel = False - elif value == '--no-index': - finder.index_urls = [] - elif value == '--allow-all-external': - finder.allow_all_external = True - - # --------------------------------------------------------------------- - elif linetype == OPTION: - if not finder: - continue - - opt, value = value - if opt == '-i' or opt == '--index-url': - finder.index_urls = [value] - elif opt == '--extra-index-url': - finder.index_urls.append(value) - elif opt == '--allow-external': - finder.allow_external |= set([normalize_name(value).lower()]) - elif opt == '--allow-insecure': - # Remove after 7.0 - finder.allow_unverified |= set([normalize_name(line).lower()]) - elif opt == '-f' or opt == '--find-links': - # FIXME: it would be nice to keep track of the source - # of the find_links: support a find-links local path - # relative to a requirements file. - req_dir = os.path.dirname(os.path.abspath(filename)) - relative_to_reqs_file = os.path.join(req_dir, value) - if os.path.exists(relative_to_reqs_file): - value = relative_to_reqs_file - finder.find_links.append(value) - - # --------------------------------------------------------------------- - elif linetype == IGNORE: - pass - - -def parse_line(line): - if not line.startswith('-'): - # Split the requirement from the options. - if ' --' in line: - req, opts = line.split(' --', 1) - opts = parse_requirement_options('--%s' % opts) - else: - req = line - opts = {} - - return REQUIREMENT, (req, opts) - - firstword, rest = partition_line(line) - # ------------------------------------------------------------------------- - if firstword == '-e' or firstword == '--editable': - return REQUIREMENT_EDITABLE, rest +def process_line(line, filename, line_number, finder=None, comes_from=None, + options=None, session=None, cache_root=None): + """ + Process a single requirements line; This can result in creating/yielding + requirements, or updating the finder. + """ - # ------------------------------------------------------------------------- - if firstword == '-r' or firstword == '--requirement': - return REQUIREMENT_FILE, rest + parser = build_parser() + args = shlex.split(line) + opts, args = parser.parse_args(args) + req = None + + if args: + # don't allow multiple requirements + if len(args) > 1: + msg = 'Only one requirement supported per line.' + raise ReqFileOnlyOneReqPerLineError(msg) + for key, value in opts.__dict__.items(): + # only certain options can be on req lines + if value and key not in get_options_dest(SUPPORTED_OPTIONS_REQ): + msg = 'Option not supported on a requirement line: %s' % key + raise ReqFileOptionNotAllowedWithReqError(msg) + req = args[0] + + # don't allow multiple/different options (on non-req lines) + if not args and len( + [v for v in opts.__dict__.values() if v is not None]) > 1: + msg = 'Only one option allowed per line.' + raise ReqFileOnleOneOptionPerLineError(msg) + + # yield a line requirement + if req: + comes_from = '-r %s (line %s)' % (filename, line_number) + isolated = options.isolated_mode if options else False + # trim the None items + keys = [opt for opt in opts.__dict__ if getattr(opts, opt) is None] + for key in keys: + delattr(opts, key) + yield InstallRequirement.from_line( + req, comes_from, isolated=isolated, options=opts.__dict__ + ) - # ------------------------------------------------------------------------- - if firstword in parser_flags: - if rest: - msg = 'Option %r does not accept values.' % firstword - raise RequirementsFileParseError(msg) - return FLAG, firstword + # yield an editable requirement + elif opts.editables: + comes_from = '-r %s (line %s)' % (filename, line_number) + isolated = options.isolated_mode if options else False + default_vcs = options.default_vcs if options else None + yield InstallRequirement.from_editable( + opts.editables[0], comes_from=comes_from, + default_vcs=default_vcs, isolated=isolated + ) - # ------------------------------------------------------------------------- - if firstword in parser_options: - if not rest: - msg = 'Option %r requires value.' % firstword - raise RequirementsFileParseError(msg) - return OPTION, (firstword, rest) + # parse a nested requirements file + elif opts.requirements: + req_file = opts.requirements[0] + if SCHEME_RE.search(filename): + # Relative to an URL. + req_url = urllib_parse.urljoin(filename, req_file) + elif not SCHEME_RE.search(req_file): + req_dir = os.path.dirname(filename) + req_url = os.path.join(os.path.dirname(filename), req_file) + # TODO: Why not use `comes_from='-r {} (line {})'` here as well? + parser = parse_requirements( + req_url, finder, comes_from, options, session, cache_root + ) + for req in parser: + yield req + + # set finder options + elif finder: + if opts.use_wheel is not None: + finder.use_wheel = opts.use_wheel + elif opts.no_index is not None: + finder.index_urls = [] + elif opts.allow_all_external is not None: + finder.allow_all_external = opts.allow_all_external + elif opts.index_url: + finder.index_urls = [opts.index_url] + elif opts.extra_index_urls: + finder.index_urls.append(opts.extra_index_url) + elif opts.allow_external is not None: + finder.allow_external |= set( + [normalize_name(opts.allow_external).lower()]) + elif opts.allow_unverified is not None: + # Remove after 7.0 + # TODO: the value was previously line? + finder.allow_unverified |= set( + [normalize_name(opts.allow_unverified).lower()]) + elif opts.find_links: + # FIXME: it would be nice to keep track of the source + # of the find_links: support a find-links local path + # relative to a requirements file. + value = opts.find_links[0] + req_dir = os.path.dirname(os.path.abspath(filename)) + relative_to_reqs_file = os.path.join(req_dir, value) + if os.path.exists(relative_to_reqs_file): + value = relative_to_reqs_file + finder.find_links.append(value) + + +def build_parser(): + """ + Return a parser for parsing requirement lines + """ + parser = optparse.OptionParser(add_help_option=False) - # ------------------------------------------------------------------------- - if firstword in parser_compat: - return IGNORE, line + options = SUPPORTED_OPTIONS + SUPPORTED_OPTIONS_REQ + for option in options: + option = option() + # we want no default values; defaults are handled in `pip install` + # parsing. just concerned with values that are specifically set. + option.default = None + parser.add_option(option) + # By default optparse sys.exits on parsing errors. We want to wrap + # that in our own exception. + def parser_exit(self, msg): + raise RequirementsFileParseError(msg) + parser.exit = parser_exit -def parse_requirement_options(args): - args = shlex.split(args) - opts, _ = _req_parser.parse_args(args) + return parser - # Remove None keys from result. - keys = [opt for opt in opts.__dict__ if getattr(opts, opt) is None] - for key in keys: - delattr(opts, key) - return opts.__dict__ +def get_options_dest(options): + """ + Return a list of 'dest' strings from a list of cmdoptions + """ + return [o().dest for o in options] -# ----------------------------------------------------------------------------- -# Utility functions related to requirements file parsing. def join_lines(iterator): """ Joins a line ending in '\' with the previous line. """ - lines = [] for line in iterator: if not line.endswith('\\'): @@ -277,24 +236,18 @@ def ignore_comments(iterator): """ Strips and filters empty or commented lines. """ - for line in iterator: - line = _comment_re.sub('', line) + line = COMMENT_RE.sub('', line) line = line.strip() if line: yield line -def partition_line(line): - firstword, _, rest = line.partition('=') - firstword = firstword.strip() - - if ' ' in firstword: - firstword, _, rest = line.partition(' ') - firstword = firstword.strip() - - rest = rest.strip() - return firstword, rest - - -__all__ = 'parse_requirements' +def skip_regex(lines, options): + """ + Optionally exclude lines that match '--skip-requirements-regex' + """ + skip_regex = options.skip_requirements_regex if options else None + if skip_regex: + lines = filterfalse(re.compile(skip_regex).search, lines) + return lines diff --git a/tests/unit/test_req_file.py b/tests/unit/test_req_file.py index 0f9fcd0a5cf..ce1a7780143 100644 --- a/tests/unit/test_req_file.py +++ b/tests/unit/test_req_file.py @@ -6,15 +6,15 @@ import pytest from pretend import stub -from pip.exceptions import RequirementsFileParseError +from pip.exceptions import (RequirementsFileParseError, + ReqFileOnlyOneReqPerLineError, + ReqFileOnleOneOptionPerLineError, + ReqFileOptionNotAllowedWithReqError) from pip.download import PipSession from pip.index import PackageFinder from pip.req.req_install import InstallRequirement -from pip.req.req_file import (parse_requirement_options, parse_content, - parse_requirements, parse_line, join_lines, - ignore_comments, partition_line, - REQUIREMENT_EDITABLE, REQUIREMENT, - REQUIREMENT_FILE, FLAG, OPTION, IGNORE) +from pip.req.req_file import (parse_requirements, process_line, join_lines, + ignore_comments) class TestIgnoreComments(object): @@ -31,26 +31,6 @@ def test_strip_comment(self): assert list(result) == ['req1', 'req2'] -class TestPartitionLine(object): - """tests for `partition_line`""" - - def test_split_req(self): - assert 'req', '' == partition_line('req') - - def test_split_req_with_flag(self): - assert 'req', '--flag' == partition_line('req --flag') - - def test_split_req_with_option_space(self): - assert 'req', '--option value' == partition_line('req --option value') - - def test_split_req_with_option_equal(self): - assert 'req', '--option=value' == partition_line('req --option=value') - - def test_split_req_with_option_and_flag(self): - assert 'req', '--option=value --flag' == \ - partition_line('req --option=value --flag') - - class TestJoinLines(object): """tests for `join_lines`""" @@ -74,110 +54,50 @@ def test_join_lines(self): assert expect == list(join_lines(lines)) -class TestParseRequirementOptions(object): - """tests for `parse_requirement_options`""" - - def test_install_options_no_quotes(self): - args = '--install-option --user' - assert {'install_options': ['--user']} == \ - parse_requirement_options(args) - - def test_install_options_quotes(self): - args = "--install-option '--user'" - assert {'install_options': ['--user']} == \ - parse_requirement_options(args) - - def test_install_options_equals(self): - args = "--install-option='--user'" - assert {'install_options': ['--user']} == \ - parse_requirement_options(args) - - def test_install_options_with_spaces(self): - args = "--install-option='--arg=value1 value2 value3'" - assert {'install_options': ['--arg=value1 value2 value3']} == \ - parse_requirement_options(args) - - def test_install_options_multiple(self): - args = "--install-option='--user' --install-option='--root'" - assert {'install_options': ['--user', '--root']} == \ - parse_requirement_options(args) - - def test_install_and_global_options(self): - args = "--install-option='--user' --global-option='--author'" - result = {'global_options': ['--author'], - 'install_options': ['--user']} - assert result == parse_requirement_options(args) - - -class TestParseLine(object): - """tests for `parse_line`""" - - def test_parse_line_editable(self): - assert parse_line('-e url') == (REQUIREMENT_EDITABLE, 'url') - assert parse_line('--editable url') == (REQUIREMENT_EDITABLE, 'url') - - def test_parse_line_req_file(self): - assert parse_line('-r file') == (REQUIREMENT_FILE, 'file') - assert parse_line('--requirement file') == (REQUIREMENT_FILE, 'file') - - def test_parse_line_flag(self): - assert parse_line('--no-index') == (FLAG, '--no-index') +class TestProcessLine(object): + """tests for `process_line`""" - def test_parse_line_option(self): - result = (OPTION, ('--index-url', 'url')) - assert parse_line('--index-url=url') == result - assert parse_line('--index-url = url') == result - assert parse_line('--index-url url') == result - result = (OPTION, ('-i', 'url')) - assert parse_line('-i=url') == result - assert parse_line('-i = url') == result - assert parse_line('-i url') == result + # TODO + # test setting all finder options + # override - def test_parse_line_ignore(self): - assert parse_line('--use-wheel') == (IGNORE, '--use-wheel') - - def test_parse_line_requirement(self): - assert parse_line('SomeProject') == (REQUIREMENT, ('SomeProject', {})) - - def test_parse_line_requirement_with_options(self): - assert parse_line('SomeProject --install-option --user') == ( - REQUIREMENT, - ('SomeProject', {'install_options': ['--user']}) - ) - - def test_flag_with_value_raises(self): - with pytest.raises(RequirementsFileParseError): - parse_line('--no-index url') + def setup(self): + self.options = stub(isolated_mode=False, default_vcs=None, + skip_requirements_regex=False) - def test_option_with_no_value_raises(self): + def test_parser_error(self): with pytest.raises(RequirementsFileParseError): - parse_line('--index-url') + list(process_line("--bogus", "file", 1)) + def test_only_one_req_per_line(self): + with pytest.raises(ReqFileOnlyOneReqPerLineError): + list(process_line("req1 req2", "file", 1)) -class TestParseContent(object): - """tests for `parse_content`""" + def test_only_one_option_per_line(self): + with pytest.raises(ReqFileOnleOneOptionPerLineError): + list(process_line("--index-url=url --no-use-wheel", "file", 1)) - def setup(self): - self.options = stub(isolated_mode=False, default_vcs=None, - skip_requirements_regex=False) + def test_option_not_allowed_on_req_line(self): + with pytest.raises(ReqFileOptionNotAllowedWithReqError): + list(process_line("req --index-url=url", "file", 1)) - def test_parse_content_requirement(self): - content = 'SomeProject' + def test_yield_line_requirement(self): + line = 'SomeProject' filename = 'filename' comes_from = '-r %s (line %s)' % (filename, 1) - req = InstallRequirement.from_line(content, comes_from=comes_from) - assert repr(list(parse_content(filename, content))[0]) == repr(req) + req = InstallRequirement.from_line(line, comes_from=comes_from) + assert repr(list(process_line(line, filename, 1))[0]) == repr(req) - def test_parse_content_editable(self): + def test_yield_editable_requirement(self): url = 'git+https://url#egg=SomeProject' - content = '-e %s' % url + line = '-e %s' % url filename = 'filename' comes_from = '-r %s (line %s)' % (filename, 1) req = InstallRequirement.from_editable(url, comes_from=comes_from) - assert repr(list(parse_content(filename, content))[0]) == repr(req) + assert repr(list(process_line(line, filename, 1))[0]) == repr(req) - def test_parse_content_requirements_file(self, monkeypatch): - content = '-r another_file' + def test_nested_requirements_file(self, monkeypatch): + line = '-r another_file' req = InstallRequirement.from_line('SomeProject') import pip.req.req_file @@ -187,42 +107,32 @@ def stub_parse_requirements(req_url, finder, comes_from, options, parse_requirements_stub = stub(call=stub_parse_requirements) monkeypatch.setattr(pip.req.req_file, 'parse_requirements', parse_requirements_stub.call) - assert list(parse_content('filename', content)) == [req] + assert list(process_line(line, 'filename', 1)) == [req] + + def test_options_on_a_requirement_line(self): + line = 'SomeProject --install-option=yo1 --install-option yo2 '\ + '--global-option="yo3" --global-option "yo4"' + filename = 'filename' + req = list(process_line(line, filename, 1))[0] + assert req.options == { + 'global_options': ['yo3', 'yo4'], + 'install_options': ['yo1', 'yo2']} - def test_parse_set_isolated(self): - content = 'SomeProject' + def test_set_isolated(self): + line = 'SomeProject' filename = 'filename' self.options.isolated_mode = True - result = parse_content(filename, content, options=self.options) + result = process_line(line, filename, 1, options=self.options) assert list(result)[0].isolated - def test_parse_set_default_vcs(self): + def test_set_default_vcs(self): url = 'https://url#egg=SomeProject' - content = '-e %s' % url + line = '-e %s' % url filename = 'filename' self.options.default_vcs = 'git' - result = parse_content(filename, content, options=self.options) + result = process_line(line, filename, 1, options=self.options) assert list(result)[0].link.url == 'git+' + url - def test_parse_set_finder(self): - content = '--index-url url' - filename = 'filename' - finder = stub() - list(parse_content(filename, content, finder=finder)) - assert finder.index_urls == ['url'] - - def test_parse_content_join_lines(self): - content = '--index-url \\\n url' - filename = 'filename' - finder = stub() - list(parse_content(filename, content, finder=finder)) - assert finder.index_urls == ['url'] - - def test_parse_content_ignore_comment(self): - content = '# SomeProject' - filename = 'filename' - assert list(parse_content(filename, content)) == [] - @pytest.fixture def session(): @@ -237,7 +147,10 @@ def finder(session): class TestParseRequirements(object): """tests for `parse_requirements`""" - # TODO some of these test are replaced by tests in classes above + # TODO: + # joins + # comments + # regex @pytest.mark.network def test_remote_reqs_parse(self): @@ -328,7 +241,7 @@ def test_install_requirements_with_options(self, tmpdir, finder, session): install_option = '--prefix=/opt' content = ''' - INITools == 2.0 --global-option="{global_option}" \ + INITools==2.0 --global-option="{global_option}" \ --install-option "{install_option}" '''.format(global_option=global_option, install_option=install_option)