import itertools import operator import re from collections.abc import Mapping, Set from dataclasses import dataclass, fields from functools import reduce from typing import Optional from pipenv.patched.pip._vendor.distlib import markers from pipenv.patched.pip._vendor.packaging.markers import InvalidMarker, Marker from pipenv.patched.pip._vendor.packaging.specifiers import ( LegacySpecifier, Specifier, SpecifierSet, ) MAX_VERSIONS = {1: 7, 2: 7, 3: 11, 4: 0} DEPRECATED_VERSIONS = ["3.0", "3.1", "3.2", "3.3"] class RequirementError(Exception): pass @dataclass class PipenvMarkers: os_name: Optional[str] = None sys_platform: Optional[str] = None platform_machine: Optional[str] = None platform_python_implementation: Optional[str] = None platform_release: Optional[str] = None platform_system: Optional[str] = None platform_version: Optional[str] = None python_version: Optional[str] = None python_full_version: Optional[str] = None implementation_name: Optional[str] = None implementation_version: Optional[str] = None @classmethod def make_marker(cls, marker_string): try: marker = Marker(marker_string) except InvalidMarker: raise RequirementError( f"Invalid requirement: Invalid marker {marker_string!r}" ) return marker @classmethod def from_pipfile(cls, name, pipfile): attr_fields = list(fields(cls)) found_keys = [k.name for k in attr_fields if k.name in pipfile] marker_strings = [f"{k} {pipfile[k]}" for k in found_keys] if pipfile.get("markers"): marker_strings.append(pipfile.get("markers")) if pipfile.get("sys_platform"): marker_strings.append(f"sys_platform '{pipfile['sys_platform']}'") if pipfile.get("platform_machine"): marker_strings.append(f"platform_machine '{pipfile['platform_machine']}'") markers = set() for marker in marker_strings: markers.add(marker) combined_marker = None try: combined_marker = cls.make_marker(" and ".join(sorted(markers))) except RequirementError: pass else: return combined_marker def is_instance(item, cls): # type: (Any, Type) -> bool if isinstance(item, cls) or item.__class__.__name__ == cls.__name__: return True return False def _tuplize_version(version): # type: (str) -> Union[Tuple[()], Tuple[int, ...], Tuple[int, int, str]] output = [] for idx, part in enumerate(version.split(".")): if part == "*": break if idx in (0, 1): # Only convert the major and minor identifiers into integers (if present), # the patch identifier can include strings like 'b' marking a beta: ex 3.11.0b1 part = int(part) output.append(part) return tuple(output) def _format_version(version) -> str: if not isinstance(version, str): return ".".join(str(i) for i in version) return version # Prefer [x,y) ranges. REPLACE_RANGES = {">": ">=", "<=": "<"} def _format_pyspec(specifier): # type: (Union[str, Specifier]) -> Specifier if isinstance(specifier, str): if not specifier.startswith(tuple(Specifier._operators.keys())): specifier = f"=={specifier}" specifier = Specifier(specifier) version = getattr(specifier, "version", specifier).rstrip() if version: if version.startswith("*"): # don't parse invalid identifiers return specifier if version.endswith("*"): if version.endswith(".*"): version = version[:-2] version = version.rstrip("*") specifier = Specifier(f"{specifier.operator}{version}") try: op = REPLACE_RANGES[specifier.operator] except KeyError: return specifier curr_tuple = _tuplize_version(version) try: next_tuple = (curr_tuple[0], curr_tuple[1] + 1) except IndexError: next_tuple = (curr_tuple[0], 1) if not next_tuple[1] <= MAX_VERSIONS[next_tuple[0]]: if specifier.operator == "<" and curr_tuple[1] <= MAX_VERSIONS[next_tuple[0]]: op = "<=" next_tuple = (next_tuple[0], curr_tuple[1]) else: return specifier specifier = Specifier(f"{op}{_format_version(next_tuple)}") return specifier def _get_specs(specset): if specset is None: return if is_instance(specset, Specifier) or is_instance(specset, LegacySpecifier): new_specset = SpecifierSet() specs = set() specs.add(specset) new_specset._specs = frozenset(specs) specset = new_specset if isinstance(specset, str): specset = SpecifierSet(specset) result = [] for spec in set(specset): version = spec.version op = spec.operator if op in ("in", "not in"): versions = version.split(",") op = "==" if op == "in" else "!=" result += [(op, _tuplize_version(ver.strip())) for ver in versions] else: result.append((spec.operator, _tuplize_version(spec.version))) return sorted(result, key=operator.itemgetter(1)) # TODO: Rename this to something meaningful def _group_by_op(specs): # type: (Union[Set[Specifier], SpecifierSet]) -> Iterator specs = [_get_specs(x) for x in list(specs)] flattened = [ ((op, len(version) > 2), version) for spec in specs for op, version in spec ] specs = sorted(flattened) grouping = itertools.groupby(specs, key=operator.itemgetter(0)) return grouping # TODO: rename this to something meaningful def normalize_specifier_set(specs): # type: (Union[str, SpecifierSet]) -> Optional[Set[Specifier]] """Given a specifier set, a string, or an iterable, normalize the specifiers. .. note:: This function exists largely to deal with ``pyzmq`` which handles the ``requires_python`` specifier incorrectly, using ``3.7*`` rather than the correct form of ``3.7.*``. This workaround can likely go away if we ever introduce enforcement for metadata standards on PyPI. :param Union[str, SpecifierSet] specs: Supplied specifiers to normalize :return: A new set of specifiers or specifierset :rtype: Union[Set[Specifier], :class:`~packaging.specifiers.SpecifierSet`] """ if not specs: return None if isinstance(specs, set): return specs # when we aren't dealing with a string at all, we can normalize this as usual elif not isinstance(specs, str): return {_format_pyspec(spec) for spec in specs} spec_list = [] for spec in specs.split(","): spec = spec.strip() if spec.endswith(".*"): spec = spec[:-2] spec = spec.rstrip("*") spec_list.append(spec) return normalize_specifier_set(SpecifierSet(",".join(spec_list))) # TODO: Check if this is used by anything public otherwise make it private # And rename it to something meaningful def get_sorted_version_string(version_set): # type: (Set[AnyStr]) -> AnyStr version_list = sorted(f"{_format_version(version)}" for version in version_set) version = ", ".join(version_list) return version # TODO: Rename this to something meaningful # TODO: Add a deprecation decorator and deprecate this -- i'm sure it's used # in other libraries def cleanup_pyspecs(specs, joiner="or"): specs = normalize_specifier_set(specs) # for != operator we want to group by version # if all are consecutive, join as a list results = {} translation_map = { # if we are doing an or operation, we need to use the min for >= # this way OR(>=2.6, >=2.7, >=3.6) picks >=2.6 # if we do an AND operation we need to use MAX to be more selective (">", ">="): { "or": lambda x: _format_version(min(x)), "and": lambda x: _format_version(max(x)), }, # we use inverse logic here so we will take the max value if we are # using OR but the min value if we are using AND ("<", "<="): { "or": lambda x: _format_version(max(x)), "and": lambda x: _format_version(min(x)), }, # leave these the same no matter what operator we use ("!=", "==", "~=", "==="): { "or": get_sorted_version_string, "and": get_sorted_version_string, }, } op_translations = { "!=": lambda x: "not in" if len(x) > 1 else "!=", "==": lambda x: "in" if len(x) > 1 else "==", } translation_keys = list(translation_map.keys()) for op_and_version_type, versions in _group_by_op(tuple(specs)): op = op_and_version_type[0] versions = [version[1] for version in versions] versions = sorted(dict.fromkeys(versions)) # remove duplicate entries op_key = next(iter(k for k in translation_keys if op in k), None) version_value = versions if op_key is not None: version_value = translation_map[op_key][joiner](versions) if op in op_translations: op = op_translations[op](versions) results[(op, op_and_version_type[1])] = version_value return sorted([(k[0], v) for k, v in results.items()], key=operator.itemgetter(1)) # TODO: Rename this to something meaningful def fix_version_tuple(version_tuple): # type: (Tuple[AnyStr, AnyStr]) -> Tuple[AnyStr, AnyStr] op, version = version_tuple max_major = max(MAX_VERSIONS.keys()) if version[0] > max_major: return (op, (max_major, MAX_VERSIONS[max_major])) max_allowed = MAX_VERSIONS[version[0]] if op == "<" and version[1] > max_allowed and version[1] - 1 <= max_allowed: op = "<=" version = (version[0], version[1] - 1) return (op, version) def _ensure_marker(marker): # type: (Union[str, Marker]) -> Marker if not is_instance(marker, Marker): return Marker(str(marker)) return marker def gen_marker(mkr): # type: (List[str]) -> Marker m = Marker("python_version == '1'") m._markers.pop() m._markers.append(mkr) return m def _strip_extra(elements): """Remove the "extra == ..." operands from the list.""" return _strip_marker_elem("extra", elements) def _strip_pyversion(elements): return _strip_marker_elem("python_version", elements) def _strip_marker_elem(elem_name, elements): """Remove the supplied element from the marker. This is not a comprehensive implementation, but relies on an important characteristic of metadata generation: The element's operand is always associated with an "and" operator. This means that we can simply remove the operand and the "and" operator associated with it. """ extra_indexes = [] preceding_operators = ["and"] if elem_name == "extra" else ["and", "or"] for i, element in enumerate(elements): if isinstance(element, list): cancelled = _strip_marker_elem(elem_name, element) if cancelled: extra_indexes.append(i) elif isinstance(element, tuple) and element[0].value == elem_name: extra_indexes.append(i) for i in reversed(extra_indexes): del elements[i] if i > 0 and elements[i - 1] in preceding_operators: # Remove the "and" before it. del elements[i - 1] elif elements: # This shouldn't ever happen, but is included for completeness. # If there is not an "and" before this element, try to remove the # operator after it. del elements[0] return not elements def _get_stripped_marker(marker, strip_func): """Build a new marker which is cleaned according to `strip_func`""" if not marker: return None marker = _ensure_marker(marker) elements = marker._markers strip_func(elements) if elements: return marker return None def get_without_extra(marker): """Build a new marker without the `extra == ...` part. The implementation relies very deep into packaging's internals, but I don't have a better way now (except implementing the whole thing myself). This could return `None` if the `extra == ...` part is the only one in the input marker. """ return _get_stripped_marker(marker, _strip_extra) def get_without_pyversion(marker): """Built a new marker without the `python_version` part. This could return `None` if the `python_version` section is the only section in the marker. """ return _get_stripped_marker(marker, _strip_pyversion) def _markers_collect_extras(markers, collection): # Optimization: the marker element is usually appended at the end. for el in reversed(markers): if isinstance(el, tuple) and el[0].value == "extra" and el[1].value == "==": collection.add(el[2].value) elif isinstance(el, list): _markers_collect_extras(el, collection) def _markers_collect_pyversions(markers, collection): local_collection = [] marker_format_str = "{0}" for el in reversed(markers): if isinstance(el, tuple) and el[0].value == "python_version": new_marker = str(gen_marker(el)) local_collection.append(marker_format_str.format(new_marker)) elif isinstance(el, list): _markers_collect_pyversions(el, local_collection) if local_collection: # local_collection = "{0}".format(" ".join(local_collection)) collection.extend(local_collection) def _markers_contains_extra(markers): # Optimization: the marker element is usually appended at the end. return _markers_contains_key(markers, "extra") def _markers_contains_pyversion(markers): return _markers_contains_key(markers, "python_version") def _markers_contains_key(markers, key): for element in reversed(markers): if isinstance(element, tuple) and element[0].value == key: return True elif isinstance(element, list) and _markers_contains_key(element, key): return True return False def get_contained_extras(marker): """Collect "extra == ..." operands from a marker. Returns a list of str. Each str is a specified extra in this marker. """ if not marker: return set() extras = set() marker = _ensure_marker(marker) _markers_collect_extras(marker._markers, extras) return extras def get_contained_pyversions(marker): """Collect all `python_version` operands from a marker.""" collection = [] if not marker: return set() marker = _ensure_marker(marker) # Collect the (Variable, Op, Value) tuples and string joiners from the marker _markers_collect_pyversions(marker._markers, collection) marker_str = " and ".join(sorted(collection)) if not marker_str: return set() # Use the distlib dictionary parser to create a dictionary 'trie' which is a bit # easier to reason about marker_dict = markers.parse_marker(marker_str)[0] version_set = set() pyversions, _ = parse_marker_dict(marker_dict) if isinstance(pyversions, set): version_set.update(pyversions) elif pyversions is not None: version_set.add(pyversions) # Each distinct element in the set was separated by an "and" operator in the marker # So we will need to reduce them with an intersection here rather than a union # in order to find the boundaries versions = set() if version_set: versions = reduce(lambda x, y: x & y, version_set) return versions def contains_extra(marker): """Check whether a marker contains an "extra == ..." operand.""" if not marker: return False marker = _ensure_marker(marker) return _markers_contains_extra(marker._markers) def contains_pyversion(marker): """Check whether a marker contains a python_version operand.""" if not marker: return False marker = _ensure_marker(marker) return _markers_contains_pyversion(marker._markers) def _split_specifierset_str(specset_str, prefix="=="): # type: (str, str) -> Set[Specifier] """Take a specifierset string and split it into a list to join for specifier sets. :param str specset_str: A string containing python versions, often comma separated :param str prefix: A prefix to use when generating the specifier set :return: A list of :class:`Specifier` instances generated with the provided prefix :rtype: Set[Specifier] """ specifiers = set() if "," not in specset_str and " " in specset_str: values = [v.strip() for v in specset_str.split()] else: values = [v.strip() for v in specset_str.split(",")] if prefix == "!=" and any(v in values for v in DEPRECATED_VERSIONS): values += DEPRECATED_VERSIONS[:] for value in sorted(values): specifiers.add(Specifier(f"{prefix}{value}")) return specifiers def _get_specifiers_from_markers(marker_item): """Given a marker item, get specifiers from the version marker. :param :class:`~packaging.markers.Marker` marker_sequence: A marker describing a version constraint :return: A set of specifiers corresponding to the marker constraint :rtype: Set[Specifier] """ specifiers = set() if isinstance(marker_item, tuple): variable, op, value = marker_item if variable.value != "python_version": return specifiers if op.value == "in": specifiers.update(_split_specifierset_str(value.value, prefix="==")) elif op.value == "not in": specifiers.update(_split_specifierset_str(value.value, prefix="!=")) else: specifiers.add(Specifier(f"{op.value}{value.value}")) elif isinstance(marker_item, list): parts = get_specset(marker_item) if parts: specifiers.update(parts) return specifiers def get_specset(marker_list): # type: (List) -> Optional[SpecifierSet] specset = set() _last_str = "and" for marker_parts in marker_list: if isinstance(marker_parts, str): _last_str = marker_parts # noqa else: specset.update(_get_specifiers_from_markers(marker_parts)) specifiers = SpecifierSet() specifiers._specs = frozenset(specset) return specifiers # TODO: Refactor this (reduce complexity) def parse_marker_dict(marker_dict): op = marker_dict["op"] lhs = marker_dict["lhs"] rhs = marker_dict["rhs"] # This is where the spec sets for each side land if we have an "or" operator side_spec_list = [] side_markers_list = [] finalized_marker = "" # And if we hit the end of the parse tree we use this format string to make a marker format_string = "{lhs} {op} {rhs}" specset = SpecifierSet() specs = set() # Essentially we will iterate over each side of the parsed marker if either one is # A mapping instance (i.e. a dictionary) and recursively parse and reduce the specset # Union the "and" specs, intersect the "or"s to find the most appropriate range if any(isinstance(side, Mapping) for side in (lhs, rhs)): for side in (lhs, rhs): side_specs = set() side_markers = set() if isinstance(side, Mapping): merged_side_specs, merged_side_markers = parse_marker_dict(side) side_specs.update(merged_side_specs) side_markers.update(merged_side_markers) else: marker = _ensure_marker(side) marker_parts = getattr(marker, "_markers", []) if marker_parts[0][0].value == "python_version": side_specs |= set(get_specset(marker_parts)) else: side_markers.add(str(marker)) side_spec_list.append(side_specs) side_markers_list.append(side_markers) if op == "and": # When we are "and"-ing things together, it probably makes the most sense # to reduce them here into a single PySpec instance specs = reduce(lambda x, y: set(x) | set(y), side_spec_list) markers = reduce(lambda x, y: set(x) | set(y), side_markers_list) if not specs and not markers: return specset, finalized_marker if markers and isinstance(markers, (tuple, list, Set)): finalized_marker = Marker(" and ".join([m for m in markers if m])) elif markers: finalized_marker = str(markers) specset._specs = frozenset(specs) return specset, finalized_marker # Actually when we "or" things as well we can also just turn them into a reduced # set using this logic now sides = reduce(lambda x, y: set(x) & set(y), side_spec_list) finalized_marker = " or ".join( [normalize_marker_str(m) for m in side_markers_list] ) specset._specs = frozenset(sorted(sides)) return specset, finalized_marker else: # At the tip of the tree we are dealing with strings all around and they just need # to be smashed together specs = set() if lhs == "python_version": format_string = "{lhs}{op}{rhs}" marker = Marker(format_string.format(**marker_dict)) marker_parts = getattr(marker, "_markers", []) _set = get_specset(marker_parts) if _set: specs |= set(_set) specset._specs = frozenset(specs) return specset, finalized_marker def _contains_micro_version(version_string): return re.search(r"\d+\.\d+\.\d+", version_string) is not None def merge_markers(m1, m2): # type: (Marker, Marker) -> Optional[Marker] if not all((m1, m2)): return next(iter(v for v in (m1, m2) if v), None) _markers = [str(_ensure_marker(marker)) for marker in (m1, m2)] marker_str = " and ".join([normalize_marker_str(m) for m in _markers if m]) return _ensure_marker(normalize_marker_str(marker_str)) def normalize_marker_str(marker) -> str: marker_str = "" if not marker: return None if not is_instance(marker, Marker): marker = _ensure_marker(marker) pyversion = get_contained_pyversions(marker) marker = get_without_pyversion(marker) if pyversion: parts = cleanup_pyspecs(pyversion) marker_str = " and ".join([format_pyversion(pv) for pv in parts]) if marker: if marker_str: marker_str = f"{marker_str!s} and {marker!s}" else: marker_str = f"{marker!s}" return marker_str.replace('"', "'") def marker_from_specifier(spec) -> Marker: if not any(spec.startswith(k) for k in Specifier._operators): if spec.strip().lower() in ["any", "", "*"]: return None spec = f"=={spec}" elif spec.startswith("==") and spec.count("=") > 3: spec = "=={}".format(spec.lstrip("=")) if not spec: return None marker_segments = [ format_pyversion(marker_segment) for marker_segment in cleanup_pyspecs(spec) ] marker_str = " and ".join(marker_segments).replace('"', "'") return Marker(marker_str) def format_pyversion(parts): op, val = parts version_marker = ( "python_full_version" if _contains_micro_version(val) else "python_version" ) return f"{version_marker} {op} '{val}'"