1221 lines
43 KiB
Python
1221 lines
43 KiB
Python
import ast
|
|
import configparser
|
|
import os
|
|
import re
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
import zipfile
|
|
from contextlib import contextmanager
|
|
from functools import lru_cache
|
|
from pathlib import Path
|
|
from tempfile import NamedTemporaryFile, TemporaryDirectory
|
|
from typing import Any, AnyStr, Dict, List, Mapping, Optional, Sequence, Union
|
|
from urllib.parse import urlparse, urlsplit, urlunsplit
|
|
|
|
from pipenv.patched.pip._internal.models.link import Link
|
|
from pipenv.patched.pip._internal.network.download import Downloader
|
|
from pipenv.patched.pip._internal.req.constructors import (
|
|
install_req_from_editable,
|
|
parse_req_from_line,
|
|
)
|
|
from pipenv.patched.pip._internal.req.req_install import InstallRequirement
|
|
from pipenv.patched.pip._internal.utils.misc import hide_url
|
|
from pipenv.patched.pip._internal.vcs.versioncontrol import VcsSupport
|
|
from pipenv.patched.pip._vendor import tomli
|
|
from pipenv.patched.pip._vendor.distlib.util import COMPARE_OP
|
|
from pipenv.patched.pip._vendor.packaging.markers import Marker
|
|
from pipenv.patched.pip._vendor.packaging.requirements import Requirement
|
|
from pipenv.patched.pip._vendor.packaging.utils import canonicalize_name
|
|
from pipenv.patched.pip._vendor.packaging.version import parse
|
|
from pipenv.utils import err
|
|
from pipenv.utils.fileutils import (
|
|
create_tracked_tempdir,
|
|
)
|
|
from pipenv.utils.requirementslib import (
|
|
add_ssh_scheme_to_git_uri,
|
|
get_pip_command,
|
|
prepare_pip_source_args,
|
|
unpack_url,
|
|
)
|
|
|
|
from .constants import (
|
|
INSTALLABLE_EXTENSIONS,
|
|
RELEVANT_PROJECT_FILES,
|
|
REMOTE_SCHEMES,
|
|
SCHEME_LIST,
|
|
VCS_LIST,
|
|
VCS_SCHEMES,
|
|
)
|
|
from .markers import PipenvMarkers
|
|
|
|
|
|
def get_version(pipfile_entry):
|
|
if str(pipfile_entry) == "{}" or is_star(pipfile_entry):
|
|
return ""
|
|
|
|
if hasattr(pipfile_entry, "keys") and "version" in pipfile_entry:
|
|
if is_star(pipfile_entry.get("version")):
|
|
return ""
|
|
version = pipfile_entry.get("version")
|
|
if version is None:
|
|
version = ""
|
|
return version.strip().lstrip("(").rstrip(")")
|
|
|
|
if isinstance(pipfile_entry, str):
|
|
return pipfile_entry.strip().lstrip("(").rstrip(")")
|
|
return ""
|
|
|
|
|
|
def python_version(path_to_python):
|
|
from pipenv.vendor.pythonfinder.utils import get_python_version
|
|
|
|
if not path_to_python:
|
|
return None
|
|
try:
|
|
version = get_python_version(path_to_python)
|
|
except Exception:
|
|
return None
|
|
return version
|
|
|
|
|
|
def clean_pkg_version(version):
|
|
"""Uses pip to prepare a package version string, from our internal version."""
|
|
return pep440_version(str(version).replace("==", ""))
|
|
|
|
|
|
def get_lockfile_section_using_pipfile_category(category):
|
|
if category == "dev-packages":
|
|
lockfile_section = "develop"
|
|
elif category == "packages":
|
|
lockfile_section = "default"
|
|
else:
|
|
lockfile_section = category
|
|
return lockfile_section
|
|
|
|
|
|
def get_pipfile_category_using_lockfile_section(category):
|
|
if category == "develop":
|
|
lockfile_section = "dev-packages"
|
|
elif category == "default":
|
|
lockfile_section = "packages"
|
|
else:
|
|
lockfile_section = category
|
|
return lockfile_section
|
|
|
|
|
|
class HackedPythonVersion:
|
|
"""A hack, which allows us to tell resolver which version of Python we're using."""
|
|
|
|
def __init__(self, python_path):
|
|
self.python_path = python_path
|
|
|
|
def __enter__(self):
|
|
if self.python_path:
|
|
os.environ["PIP_PYTHON_PATH"] = str(self.python_path)
|
|
|
|
def __exit__(self, *args):
|
|
pass
|
|
|
|
|
|
def get_canonical_names(packages):
|
|
"""Canonicalize a list of packages and return a set of canonical names"""
|
|
from pipenv.patched.pip._vendor.packaging.utils import canonicalize_name
|
|
|
|
if not isinstance(packages, Sequence):
|
|
if not isinstance(packages, str):
|
|
return packages
|
|
packages = [packages]
|
|
return {canonicalize_name(pkg) for pkg in packages if pkg}
|
|
|
|
|
|
def pep440_version(version):
|
|
"""Normalize version to PEP 440 standards"""
|
|
return str(parse(version))
|
|
|
|
|
|
def pep423_name(name):
|
|
"""Normalize package name to PEP 423 style standard."""
|
|
name = name.lower()
|
|
if any(i not in name for i in (VCS_LIST + SCHEME_LIST)):
|
|
return name.replace("_", "-")
|
|
|
|
else:
|
|
return name
|
|
|
|
|
|
def translate_markers(pipfile_entry):
|
|
from pipenv.patched.pip._vendor.packaging.markers import default_environment
|
|
|
|
allowed_marker_keys = ["markers"] + list(default_environment().keys())
|
|
provided_keys = list(pipfile_entry.keys()) if hasattr(pipfile_entry, "keys") else []
|
|
pipfile_markers = set(provided_keys) & set(allowed_marker_keys)
|
|
new_pipfile = dict(pipfile_entry).copy()
|
|
marker_set = set()
|
|
os_name_marker = None
|
|
if "markers" in new_pipfile:
|
|
marker_str = new_pipfile.pop("markers")
|
|
if marker_str:
|
|
marker = str(Marker(marker_str))
|
|
if "extra" not in marker:
|
|
marker_set.add(marker)
|
|
for m in pipfile_markers:
|
|
entry = f"{pipfile_entry[m]}"
|
|
if m != "markers":
|
|
if m != "os_name":
|
|
marker_set.add(str(Marker(f"{m} {entry}")))
|
|
new_pipfile.pop(m)
|
|
if marker_set:
|
|
markers_str = " and ".join(
|
|
f"{s}" if " and " in s else s for s in sorted(dict.fromkeys(marker_set))
|
|
)
|
|
if os_name_marker:
|
|
markers_str = f"({markers_str}) and {os_name_marker}"
|
|
new_pipfile["markers"] = str(Marker(markers_str)).replace('"', "'")
|
|
return new_pipfile
|
|
|
|
|
|
def unearth_hashes_for_dep(project, dep):
|
|
hashes = []
|
|
|
|
index_url = "https://pypi.org/simple/"
|
|
source = "pypi"
|
|
for source in project.sources:
|
|
if source.get("name") == dep.get("index"):
|
|
index_url = source.get("url")
|
|
break
|
|
|
|
# 1 Try to get hashes directly form index
|
|
install_req, markers, _ = install_req_from_pipfile(dep["name"], dep)
|
|
if not install_req or not install_req.req:
|
|
return []
|
|
if "https://pypi.org/simple/" in index_url:
|
|
hashes = project.get_hashes_from_pypi(install_req, source)
|
|
elif index_url:
|
|
hashes = project.get_hashes_from_remote_index_urls(install_req, source)
|
|
if hashes:
|
|
return hashes
|
|
|
|
return []
|
|
|
|
|
|
def clean_resolved_dep(project, dep, is_top_level=False, current_entry=None):
|
|
from pipenv.patched.pip._vendor.packaging.requirements import (
|
|
Requirement as PipRequirement,
|
|
)
|
|
|
|
name = dep["name"]
|
|
lockfile = {}
|
|
|
|
# Evaluate Markers
|
|
if "markers" in dep and dep.get("markers", "").strip():
|
|
if not is_top_level:
|
|
translated = translate_markers(dep).get("markers", "").strip()
|
|
if translated:
|
|
try:
|
|
lockfile["markers"] = translated
|
|
except TypeError:
|
|
pass
|
|
else:
|
|
try:
|
|
pipfile_entry = translate_markers(dep)
|
|
if pipfile_entry.get("markers"):
|
|
lockfile["markers"] = pipfile_entry.get("markers")
|
|
except TypeError:
|
|
pass
|
|
|
|
version = dep.get("version", None)
|
|
if version and not version.startswith("=="):
|
|
version = f"=={version}"
|
|
if version == "==*":
|
|
if current_entry:
|
|
version = current_entry.get("version")
|
|
dep["version"] = version
|
|
else:
|
|
version = None
|
|
|
|
is_vcs_or_file = False
|
|
for vcs_type in VCS_LIST:
|
|
if vcs_type in dep:
|
|
if "[" in dep[vcs_type] and "]" in dep[vcs_type]:
|
|
extras_section = dep[vcs_type].split("[").pop().replace("]", "")
|
|
lockfile["extras"] = sorted(
|
|
[extra.strip() for extra in extras_section.split(",")]
|
|
)
|
|
if has_name_with_extras(dep[vcs_type]):
|
|
lockfile[vcs_type] = dep[vcs_type].split("@ ", 1)[1]
|
|
else:
|
|
lockfile[vcs_type] = dep[vcs_type]
|
|
lockfile["ref"] = dep.get("ref")
|
|
if "subdirectory" in dep:
|
|
lockfile["subdirectory"] = dep["subdirectory"]
|
|
is_vcs_or_file = True
|
|
|
|
if "editable" in dep:
|
|
lockfile["editable"] = dep["editable"]
|
|
|
|
preferred_file_keys = ["path", "file"]
|
|
dependency_file_key = next(iter(k for k in preferred_file_keys if k in dep), None)
|
|
if dependency_file_key:
|
|
lockfile[dependency_file_key] = dep[dependency_file_key]
|
|
is_vcs_or_file = True
|
|
if "editable" in dep:
|
|
lockfile["editable"] = dep["editable"]
|
|
|
|
if version and not is_vcs_or_file:
|
|
if isinstance(version, PipRequirement):
|
|
if version.specifier:
|
|
lockfile["version"] = str(version.specifier)
|
|
if version.extras:
|
|
lockfile["extras"] = sorted(version.extras)
|
|
elif version:
|
|
lockfile["version"] = version
|
|
|
|
if dep.get("hashes"):
|
|
lockfile["hashes"] = dep["hashes"]
|
|
elif is_top_level:
|
|
potential_hashes = unearth_hashes_for_dep(project, dep)
|
|
if potential_hashes:
|
|
lockfile["hashes"] = potential_hashes
|
|
|
|
if dep.get("index"):
|
|
lockfile["index"] = dep["index"]
|
|
|
|
if dep.get("extras"):
|
|
lockfile["extras"] = sorted(dep["extras"])
|
|
|
|
# In case we lock a uri or a file when the user supplied a path
|
|
# remove the uri or file keys from the entry and keep the path
|
|
if dep and isinstance(dep, dict):
|
|
for k in preferred_file_keys:
|
|
if k in dep.keys():
|
|
lockfile[k] = dep[k]
|
|
break
|
|
|
|
if "markers" in dep:
|
|
markers = dep["markers"]
|
|
if markers:
|
|
markers = Marker(markers)
|
|
if not markers.evaluate() and current_entry:
|
|
current_entry.update(lockfile)
|
|
return {name: current_entry}
|
|
|
|
return {name: lockfile}
|
|
|
|
|
|
def as_pipfile(dep: InstallRequirement) -> Dict[str, Any]:
|
|
"""Create a pipfile entry for the given InstallRequirement."""
|
|
pipfile_dict = {}
|
|
name = dep.name
|
|
version = dep.req.specifier
|
|
|
|
# Construct the pipfile entry
|
|
pipfile_dict[name] = {
|
|
"version": str(version),
|
|
"editable": dep.editable,
|
|
"extras": list(dep.extras),
|
|
}
|
|
|
|
if dep.link:
|
|
# If it's a VCS link
|
|
if dep.link.is_vcs:
|
|
vcs = dep.link.scheme.split("+")[0]
|
|
pipfile_dict[name][vcs] = dep.link.url_without_fragment
|
|
# If it's a URL link
|
|
elif dep.link.scheme.startswith("http"):
|
|
pipfile_dict[name]["file"] = dep.link.url_without_fragment
|
|
# If it's a local file
|
|
elif dep.link.is_file:
|
|
pipfile_dict[name]["path"] = dep.link.file_path
|
|
|
|
# Convert any markers to their string representation
|
|
if dep.markers:
|
|
pipfile_dict[name]["markers"] = str(dep.markers)
|
|
|
|
# If a hash is available, add it to the pipfile entry
|
|
if dep.hash_options:
|
|
pipfile_dict[name]["hashes"] = dep.hash_options
|
|
|
|
return pipfile_dict
|
|
|
|
|
|
def is_star(val):
|
|
return isinstance(val, str) and val == "*"
|
|
|
|
|
|
def is_pinned(val):
|
|
if isinstance(val, Mapping):
|
|
val = val.get("version")
|
|
return isinstance(val, str) and val.startswith("==")
|
|
|
|
|
|
def is_pinned_requirement(ireq):
|
|
"""
|
|
Returns whether an InstallRequirement is a "pinned" requirement.
|
|
"""
|
|
if ireq.editable:
|
|
return False
|
|
|
|
if ireq.req is None or len(ireq.specifier) != 1:
|
|
return False
|
|
|
|
spec = next(iter(ireq.specifier))
|
|
return spec.operator in {"==", "==="} and not spec.version.endswith(".*")
|
|
|
|
|
|
def is_editable_path(path):
|
|
if os.path.isdir(path):
|
|
return True
|
|
return False
|
|
|
|
|
|
def dependency_as_pip_install_line(
|
|
dep_name: str,
|
|
dep: Union[str, Mapping],
|
|
include_hashes: bool,
|
|
include_markers: bool,
|
|
include_index: bool,
|
|
indexes: list,
|
|
constraint: bool = False,
|
|
):
|
|
if isinstance(dep, str):
|
|
if is_star(dep):
|
|
return dep_name
|
|
elif not COMPARE_OP.match(dep):
|
|
return f"{dep_name}=={dep}"
|
|
return f"{dep_name}{dep}"
|
|
line = []
|
|
is_constraint = False
|
|
vcs = next(iter([vcs for vcs in VCS_LIST if vcs in dep]), None)
|
|
if not vcs:
|
|
for k in ["file", "path"]:
|
|
if k in dep:
|
|
if is_editable_path(dep[k]):
|
|
line.append("-e")
|
|
extras = ""
|
|
if "extras" in dep:
|
|
extras = f"[{','.join(dep['extras'])}]"
|
|
location = dep["file"] if "file" in dep else dep["path"]
|
|
if location.startswith(("http:", "https:")):
|
|
line.append(f"{dep_name}{extras} @ {location}")
|
|
else:
|
|
line.append(f"{location}{extras}")
|
|
break
|
|
else:
|
|
# Normal/Named Requirements
|
|
is_constraint = True
|
|
line.append(dep_name)
|
|
if "extras" in dep:
|
|
line[-1] += f"[{','.join(dep['extras'])}]"
|
|
if "version" in dep:
|
|
version = dep["version"]
|
|
if version and not is_star(version):
|
|
if not COMPARE_OP.match(version):
|
|
version = f"=={version}"
|
|
line[-1] += version
|
|
if include_markers and dep.get("markers"):
|
|
line[-1] = f'{line[-1]}; {dep["markers"]}'
|
|
|
|
if include_hashes and dep.get("hashes"):
|
|
line.extend([f" --hash={hash}" for hash in dep["hashes"]])
|
|
|
|
if include_index:
|
|
if dep.get("index"):
|
|
indexes = [s for s in indexes if s.get("name") == dep["index"]]
|
|
else:
|
|
indexes = [indexes[0]] if indexes else []
|
|
index_list = prepare_pip_source_args(indexes)
|
|
line.extend(index_list)
|
|
elif vcs and vcs in dep: # VCS Requirements
|
|
extras = ""
|
|
ref = ""
|
|
if dep.get("ref"):
|
|
ref = f"@{dep['ref']}"
|
|
if "extras" in dep:
|
|
extras = f"[{','.join(dep['extras'])}]"
|
|
include_vcs = "" if f"{vcs}+" in dep[vcs] else f"{vcs}+"
|
|
vcs_url = dep[vcs]
|
|
# legacy format is the only format supported for editable installs https://github.com/pypa/pip/issues/9106
|
|
if is_editable_path(dep[vcs]) or "file://" in dep[vcs]:
|
|
if "#egg=" not in dep[vcs]:
|
|
git_req = f"-e {include_vcs}{dep[vcs]}{ref}#egg={dep_name}{extras}"
|
|
else:
|
|
git_req = f"-e {include_vcs}{dep[vcs]}{ref}"
|
|
if "subdirectory" in dep:
|
|
git_req += f"&subdirectory={dep['subdirectory']}"
|
|
else:
|
|
if "#egg=" in vcs_url:
|
|
vcs_url = vcs_url.split("#egg=")[0]
|
|
git_req = f"{dep_name}{extras}@ {include_vcs}{vcs_url}{ref}"
|
|
if "subdirectory" in dep:
|
|
git_req += f"#subdirectory={dep['subdirectory']}"
|
|
|
|
line.append(git_req)
|
|
|
|
if constraint and not is_constraint:
|
|
pip_line = ""
|
|
else:
|
|
pip_line = " ".join(line)
|
|
return pip_line
|
|
|
|
|
|
def convert_deps_to_pip(
|
|
deps,
|
|
indexes=None,
|
|
include_hashes=True,
|
|
include_markers=True,
|
|
include_index=False,
|
|
):
|
|
""" "Converts a Pipfile-formatted dependency to a pip-formatted one."""
|
|
dependencies = {}
|
|
if indexes is None:
|
|
indexes = []
|
|
for dep_name, dep in deps.items():
|
|
req = dependency_as_pip_install_line(
|
|
dep_name, dep, include_hashes, include_markers, include_index, indexes
|
|
)
|
|
dependencies[dep_name] = req
|
|
return dependencies
|
|
|
|
|
|
def parse_metadata_file(content: str):
|
|
"""
|
|
Parse a METADATA file to get the package name.
|
|
|
|
Parameters:
|
|
content (str): Contents of the METADATA file.
|
|
|
|
Returns:
|
|
str: Name of the package or None if not found.
|
|
"""
|
|
|
|
for line in content.splitlines():
|
|
if line.startswith("Name:"):
|
|
return line.split("Name: ")[1].strip()
|
|
|
|
return None
|
|
|
|
|
|
def parse_pkginfo_file(content: str):
|
|
"""
|
|
Parse a PKG-INFO file to get the package name.
|
|
|
|
Parameters:
|
|
content (str): Contents of the PKG-INFO file.
|
|
|
|
Returns:
|
|
str: Name of the package or None if not found.
|
|
"""
|
|
for line in content.splitlines():
|
|
if line.startswith("Name:"):
|
|
return line.split("Name: ")[1].strip()
|
|
|
|
return None
|
|
|
|
|
|
def parse_setup_file(content):
|
|
# A dictionary to store variable names and their values
|
|
variables = {}
|
|
try:
|
|
tree = ast.parse(content)
|
|
for node in ast.walk(tree):
|
|
# Extract variable assignments and store them
|
|
if isinstance(node, ast.Assign):
|
|
for target in node.targets:
|
|
if isinstance(target, ast.Name):
|
|
if isinstance(node.value, ast.Str): # for Python versions < 3.8
|
|
variables[target.id] = node.value.s
|
|
elif isinstance(node.value, ast.Constant) and isinstance(
|
|
node.value.value, str
|
|
):
|
|
variables[target.id] = node.value.value
|
|
|
|
# Check function calls to extract the 'name' attribute from the setup function
|
|
if isinstance(node, ast.Call):
|
|
if (
|
|
getattr(node.func, "id", "") == "setup"
|
|
or isinstance(node.func, ast.Attribute)
|
|
and node.func.attr == "setup"
|
|
):
|
|
for keyword in node.keywords:
|
|
if keyword.arg == "name":
|
|
# If it's a variable, retrieve its value
|
|
if (
|
|
isinstance(keyword.value, ast.Name)
|
|
and keyword.value.id in variables
|
|
):
|
|
return variables[keyword.value.id]
|
|
# Otherwise, check if it's directly provided
|
|
elif isinstance(keyword.value, ast.Str):
|
|
return keyword.value.s
|
|
elif isinstance(keyword.value, ast.Constant) and isinstance(
|
|
keyword.value.value, str
|
|
):
|
|
return keyword.value.value
|
|
# Additional handling for Python versions and specific ways of defining the name
|
|
elif sys.version_info < (3, 9) and isinstance(
|
|
keyword.value, ast.Subscript
|
|
):
|
|
if (
|
|
isinstance(keyword.value.value, ast.Name)
|
|
and keyword.value.value.id == "about"
|
|
):
|
|
if isinstance(
|
|
keyword.value.slice, ast.Index
|
|
) and isinstance(keyword.value.slice.value, ast.Str):
|
|
return keyword.value.slice.value.s
|
|
return keyword.value.s
|
|
elif sys.version_info >= (3, 9) and isinstance(
|
|
keyword.value, ast.Subscript
|
|
):
|
|
if (
|
|
isinstance(keyword.value.value, ast.Name)
|
|
and isinstance(keyword.value.slice, ast.Str)
|
|
and keyword.value.value.id == "about"
|
|
):
|
|
return keyword.value.slice.s
|
|
except ValueError:
|
|
pass # We will not exec unsafe code to determine the name pre-resolver
|
|
|
|
return None
|
|
|
|
|
|
def parse_cfg_file(content):
|
|
config = configparser.ConfigParser()
|
|
config.read_string(content)
|
|
try:
|
|
return config["metadata"]["name"]
|
|
except configparser.NoSectionError:
|
|
return None
|
|
except KeyError:
|
|
return None
|
|
|
|
|
|
def parse_toml_file(content):
|
|
toml_dict = tomli.loads(content)
|
|
if "project" in toml_dict and "name" in toml_dict["project"]:
|
|
return toml_dict["project"]["name"]
|
|
if "tool" in toml_dict and "poetry" in toml_dict["tool"]:
|
|
return toml_dict["tool"]["poetry"]["name"]
|
|
|
|
return None
|
|
|
|
|
|
def find_package_name_from_tarball(tarball_filepath):
|
|
if tarball_filepath.startswith("file://") and os.name != "nt":
|
|
tarball_filepath = tarball_filepath[7:]
|
|
with tarfile.open(tarball_filepath, "r") as tar_ref:
|
|
for filename in tar_ref.getnames():
|
|
if filename.endswith(RELEVANT_PROJECT_FILES):
|
|
with tar_ref.extractfile(filename) as file:
|
|
possible_name = find_package_name_from_filename(filename, file)
|
|
if possible_name:
|
|
return possible_name
|
|
|
|
|
|
def find_package_name_from_zipfile(zip_filepath):
|
|
if zip_filepath.startswith("file://") and os.name != "nt":
|
|
zip_filepath = zip_filepath[7:]
|
|
with zipfile.ZipFile(zip_filepath, "r") as zip_ref:
|
|
for filename in zip_ref.namelist():
|
|
if filename.endswith(RELEVANT_PROJECT_FILES):
|
|
with zip_ref.open(filename) as file:
|
|
possible_name = find_package_name_from_filename(file.name, file)
|
|
if possible_name:
|
|
return possible_name
|
|
|
|
|
|
def find_package_name_from_directory(directory):
|
|
parsed_url = urlparse(directory)
|
|
directory = (
|
|
os.path.normpath(parsed_url.path)
|
|
if parsed_url.scheme
|
|
else os.path.normpath(directory)
|
|
)
|
|
if "#egg=" in directory: # parse includes the fragment in py3.7 and py3.8
|
|
expected_name = directory.split("#egg=")[1]
|
|
return expected_name
|
|
if os.name == "nt":
|
|
if directory.startswith("\\") and (":\\" in directory or ":/" in directory):
|
|
directory = directory[1:]
|
|
if directory.startswith("\\\\"):
|
|
directory = directory[1:]
|
|
directory_contents = sorted(
|
|
os.listdir(directory),
|
|
key=lambda x: (os.path.isdir(os.path.join(directory, x)), x),
|
|
)
|
|
for filename in directory_contents:
|
|
filepath = os.path.join(directory, filename)
|
|
if os.path.isfile(filepath):
|
|
if filename.endswith(RELEVANT_PROJECT_FILES):
|
|
with open(filepath, "rb") as file:
|
|
possible_name = find_package_name_from_filename(filename, file)
|
|
if possible_name:
|
|
return possible_name
|
|
elif os.path.isdir(filepath):
|
|
possible_name = find_package_name_from_directory(filepath)
|
|
if possible_name:
|
|
return possible_name
|
|
|
|
return None
|
|
|
|
|
|
def ensure_path_is_relative(file_path):
|
|
abs_path = Path(file_path).resolve()
|
|
current_dir = Path.cwd()
|
|
|
|
# Check if the paths are on different drives
|
|
if abs_path.drive != current_dir.drive:
|
|
# If on different drives, return the absolute path
|
|
return str(abs_path)
|
|
|
|
try:
|
|
# Try to create a relative path
|
|
return str(abs_path.relative_to(current_dir))
|
|
except ValueError:
|
|
# If the direct relative_to fails, manually compute the relative path
|
|
common_parts = 0
|
|
for part_a, part_b in zip(abs_path.parts, current_dir.parts):
|
|
if part_a == part_b:
|
|
common_parts += 1
|
|
else:
|
|
break
|
|
|
|
# Number of ".." needed are the extra parts in the current directory
|
|
# beyond the common parts
|
|
up_levels = [".."] * (len(current_dir.parts) - common_parts)
|
|
# The relative path is constructed by going up as needed and then
|
|
# appending the non-common parts of the absolute path
|
|
rel_parts = up_levels + list(abs_path.parts[common_parts:])
|
|
relative_path = Path(*rel_parts)
|
|
return str(relative_path)
|
|
|
|
|
|
def determine_path_specifier(package: InstallRequirement):
|
|
if package.link:
|
|
if package.link.scheme in ["http", "https"]:
|
|
return package.link.url_without_fragment
|
|
if package.link.scheme == "file":
|
|
return ensure_path_is_relative(package.link.file_path)
|
|
|
|
|
|
def determine_vcs_specifier(package: InstallRequirement):
|
|
if package.link and package.link.scheme in VCS_SCHEMES:
|
|
vcs_specifier = package.link.url_without_fragment
|
|
return vcs_specifier
|
|
|
|
|
|
def get_vcs_backend(vcs_type):
|
|
backend = VcsSupport().get_backend(vcs_type)
|
|
return backend
|
|
|
|
|
|
def generate_temp_dir_path():
|
|
# Create a temporary directory using mkdtemp
|
|
temp_dir = tempfile.mkdtemp()
|
|
# Remove the created directory
|
|
os.rmdir(temp_dir)
|
|
return temp_dir
|
|
|
|
|
|
def determine_vcs_revision_hash(
|
|
package: InstallRequirement, vcs_type: str, revision: str
|
|
):
|
|
try: # Windows python 3.7 will sometimes raise PermissionError cleaning up
|
|
checkout_directory = generate_temp_dir_path()
|
|
repo_backend = get_vcs_backend(vcs_type)
|
|
repo_backend.obtain(checkout_directory, hide_url(package.link.url), verbosity=1)
|
|
return repo_backend.get_revision(checkout_directory)
|
|
except Exception as e:
|
|
err.print(
|
|
f"Error {e} obtaining {vcs_type} revision hash for {package}; falling back to {revision}."
|
|
)
|
|
return revision
|
|
|
|
|
|
@lru_cache(maxsize=None)
|
|
def determine_package_name(package: InstallRequirement):
|
|
req_name = None
|
|
if package.name:
|
|
req_name = package.name
|
|
elif "#egg=" in str(package):
|
|
req_name = str(package).split("#egg=")[1]
|
|
req_name = req_name.split("[")[0]
|
|
elif "@ " in str(package):
|
|
req_name = str(package).split("@ ")[0]
|
|
req_name = req_name.split("[")[0]
|
|
elif package.link and package.link.scheme in REMOTE_SCHEMES:
|
|
try: # Windows python 3.7 will sometimes raise PermissionError cleaning up
|
|
with TemporaryDirectory() as td:
|
|
cmd = get_pip_command()
|
|
options, _ = cmd.parser.parse_args([])
|
|
session = cmd._build_session(options)
|
|
local_file = unpack_url(
|
|
link=package.link,
|
|
location=td,
|
|
download=Downloader(session, "off"),
|
|
verbosity=1,
|
|
)
|
|
if local_file.path.endswith(".whl") or local_file.path.endswith(".zip"):
|
|
req_name = find_package_name_from_zipfile(local_file.path)
|
|
elif local_file.path.endswith(".tar.gz") or local_file.path.endswith(
|
|
".tar.bz2"
|
|
):
|
|
req_name = find_package_name_from_tarball(local_file.path)
|
|
else:
|
|
req_name = find_package_name_from_directory(local_file.path)
|
|
except PermissionError:
|
|
pass
|
|
elif package.link and package.link.scheme in [
|
|
"bzr+file",
|
|
"git+file",
|
|
"hg+file",
|
|
"svn+file",
|
|
]:
|
|
parsed_url = urlparse(package.link.url)
|
|
repository_path = parsed_url.path
|
|
repository_path = repository_path.rsplit("@", 1)[
|
|
0
|
|
] # extract the actual directory path
|
|
repository_path = repository_path.split("#egg=")[0]
|
|
req_name = find_package_name_from_directory(repository_path)
|
|
elif package.link and package.link.scheme == "file":
|
|
if package.link.file_path.endswith(".whl") or package.link.file_path.endswith(
|
|
".zip"
|
|
):
|
|
req_name = find_package_name_from_zipfile(package.link.file_path)
|
|
elif package.link.file_path.endswith(
|
|
".tar.gz"
|
|
) or package.link.file_path.endswith(".tar.bz2"):
|
|
req_name = find_package_name_from_tarball(package.link.file_path)
|
|
else:
|
|
req_name = find_package_name_from_directory(package.link.file_path)
|
|
if req_name:
|
|
return req_name
|
|
else:
|
|
raise ValueError(f"Could not determine package name from {package}")
|
|
|
|
|
|
def find_package_name_from_filename(filename, file):
|
|
if filename.endswith("METADATA"):
|
|
content = file.read().decode()
|
|
possible_name = parse_metadata_file(content)
|
|
if possible_name:
|
|
return possible_name
|
|
|
|
if filename.endswith("PKG-INFO"):
|
|
content = file.read().decode()
|
|
possible_name = parse_pkginfo_file(content)
|
|
if possible_name:
|
|
return possible_name
|
|
|
|
if filename.endswith("setup.py"):
|
|
content = file.read().decode()
|
|
possible_name = parse_setup_file(content)
|
|
if possible_name:
|
|
return possible_name
|
|
|
|
if filename.endswith("setup.cfg"):
|
|
content = file.read().decode()
|
|
possible_name = parse_cfg_file(content)
|
|
if possible_name:
|
|
return possible_name
|
|
|
|
if filename.endswith("pyproject.toml"):
|
|
content = file.read().decode()
|
|
possible_name = parse_toml_file(content)
|
|
if possible_name:
|
|
return possible_name
|
|
return None
|
|
|
|
|
|
def create_link(link):
|
|
# type: (AnyStr) -> Link
|
|
|
|
if not isinstance(link, str):
|
|
raise TypeError("must provide a string to instantiate a new link")
|
|
|
|
return Link(link)
|
|
|
|
|
|
def get_link_from_line(line):
|
|
"""Parse link information from given requirement line. Return a
|
|
6-tuple:
|
|
|
|
- `vcs_type` indicates the VCS to use (e.g. "git"), or None.
|
|
- `prefer` is either "file", "path" or "uri", indicating how the
|
|
information should be used in later stages.
|
|
- `relpath` is the relative path to use when recording the dependency,
|
|
instead of the absolute path/URI used to perform installation.
|
|
This can be None (to prefer the absolute path or URI).
|
|
- `path` is the absolute file path to the package. This will always use
|
|
forward slashes. Can be None if the line is a remote URI.
|
|
- `uri` is the absolute URI to the package. Can be None if the line is
|
|
not a URI.
|
|
- `link` is an instance of :class:`pipenv.patched.pip._internal.index.Link`,
|
|
representing a URI parse result based on the value of `uri`.
|
|
This function is provided to deal with edge cases concerning URIs
|
|
without a valid netloc. Those URIs are problematic to a straight
|
|
``urlsplit` call because they cannot be reliably reconstructed with
|
|
``urlunsplit`` due to a bug in the standard library:
|
|
>>> from urllib.parse import urlsplit, urlunsplit
|
|
>>> urlunsplit(urlsplit('git+file:///this/breaks'))
|
|
'git+file:/this/breaks'
|
|
>>> urlunsplit(urlsplit('file:///this/works'))
|
|
'file:///this/works'
|
|
See `https://bugs.python.org/issue23505#msg277350`.
|
|
"""
|
|
|
|
# Git allows `git@github.com...` lines that are not really URIs.
|
|
# Add "ssh://" so we can parse correctly, and restore afterward.
|
|
fixed_line = add_ssh_scheme_to_git_uri(line) # type: str
|
|
|
|
# We can assume a lot of things if this is a local filesystem path.
|
|
if "://" not in fixed_line:
|
|
p = Path(fixed_line).absolute() # type: Path
|
|
p.as_posix() # type: Optional[str]
|
|
uri = p.as_uri() # type: str
|
|
link = create_link(uri) # type: Link
|
|
return link
|
|
|
|
# This is an URI. We'll need to perform some elaborated parsing.
|
|
parsed_url = urlsplit(fixed_line) # type: SplitResult
|
|
|
|
# Split the VCS part out if needed.
|
|
original_scheme = parsed_url.scheme # type: str
|
|
if "+" in original_scheme:
|
|
vcs_type, _, scheme = original_scheme.partition("+")
|
|
parsed_url = parsed_url._replace(scheme=scheme) # type: ignore
|
|
else:
|
|
pass
|
|
|
|
# Re-attach VCS prefix to build a Link.
|
|
link = create_link(
|
|
urlunsplit(parsed_url._replace(scheme=original_scheme)) # type: ignore
|
|
)
|
|
|
|
return link
|
|
|
|
|
|
def has_name_with_extras(requirement):
|
|
pattern = r"^([a-zA-Z0-9_-]+(\[[a-zA-Z0-9_-]+\])?) @ .*"
|
|
match = re.match(pattern, requirement)
|
|
return match is not None
|
|
|
|
|
|
def expand_env_variables(line) -> AnyStr:
|
|
"""Expand the env vars in a line following pip's standard.
|
|
https://pip.pypa.io/en/stable/reference/pip_install/#id10.
|
|
|
|
Matches environment variable-style values in '${MY_VARIABLE_1}' with
|
|
the variable name consisting of only uppercase letters, digits or
|
|
the '_'
|
|
"""
|
|
|
|
def replace_with_env(match):
|
|
value = os.getenv(match.group(1))
|
|
return value if value else match.group()
|
|
|
|
return re.sub(r"\$\{([A-Z0-9_]+)\}", replace_with_env, line)
|
|
|
|
|
|
def expansive_install_req_from_line(
|
|
pip_line: str,
|
|
comes_from: Optional[Union[str, InstallRequirement]] = None,
|
|
*,
|
|
use_pep517: Optional[bool] = None,
|
|
isolated: bool = False,
|
|
global_options: Optional[List[str]] = None,
|
|
hash_options: Optional[Dict[str, List[str]]] = None,
|
|
constraint: bool = False,
|
|
line_source: Optional[str] = None,
|
|
user_supplied: bool = False,
|
|
config_settings: Optional[Dict[str, Union[str, List[str]]]] = None,
|
|
expand_env: bool = False,
|
|
) -> (InstallRequirement, str):
|
|
"""Create an InstallRequirement from a pip-style requirement line.
|
|
InstallRequirement is a pip internal construct that represents an installable requirement,
|
|
and is used as an intermediary between the pip command and the resolver.
|
|
:param pip_line: A pip-style requirement line.
|
|
:param comes_from: The path to the requirements file the line was found in.
|
|
:param use_pep517: Whether to use PEP 517/518 when installing the
|
|
requirement.
|
|
:param isolated: Whether to isolate the requirements when installing them. (likely unused)
|
|
:param global_options: Extra global options to be used when installing the install req (likely unused)
|
|
:param hash_options: Extra hash options to be used when installing the install req (likely unused)
|
|
:param constraint: Whether the requirement is a constraint.
|
|
:param line_source: The source of the line (e.g. "requirements.txt").
|
|
:param user_supplied: Whether the requirement was directly provided by the user.
|
|
:param config_settings: Configuration settings to be used when installing the install req (likely unused)
|
|
:param expand_env: Whether to expand environment variables in the line. (definitely used)
|
|
:return: A tuple of the InstallRequirement and the name of the package (if determined).
|
|
"""
|
|
name = None
|
|
pip_line = pip_line.strip("'").lstrip(" ")
|
|
for new_req_symbol in ("@ ", " @ "): # Check for new style pip lines
|
|
if new_req_symbol in pip_line:
|
|
pip_line_parts = pip_line.split(new_req_symbol, 1)
|
|
name = pip_line_parts[0]
|
|
pip_line = pip_line_parts[1]
|
|
if pip_line.startswith("-e "): # Editable requirements
|
|
pip_line = pip_line.split("-e ")[1]
|
|
return install_req_from_editable(pip_line, line_source), name
|
|
|
|
if expand_env:
|
|
pip_line = expand_env_variables(pip_line)
|
|
|
|
vcs_part = pip_line
|
|
for vcs in VCS_LIST:
|
|
if vcs_part.startswith(f"{vcs}+"):
|
|
link = get_link_from_line(vcs_part)
|
|
install_req = InstallRequirement(
|
|
None,
|
|
comes_from,
|
|
link=link,
|
|
use_pep517=use_pep517,
|
|
isolated=isolated,
|
|
global_options=global_options,
|
|
hash_options=hash_options,
|
|
constraint=constraint,
|
|
user_supplied=user_supplied,
|
|
)
|
|
return install_req, name
|
|
if urlparse(pip_line).scheme in ("http", "https", "file") or any(
|
|
pip_line.endswith(s) for s in INSTALLABLE_EXTENSIONS
|
|
):
|
|
parts = parse_req_from_line(pip_line, line_source)
|
|
else:
|
|
# It's a requirement
|
|
if "--index" in pip_line:
|
|
pip_line = pip_line.split("--index")[0]
|
|
if " -i " in pip_line:
|
|
pip_line = pip_line.split(" -i ")[0]
|
|
# handle local version identifiers (like the ones torch uses in their public index)
|
|
if "+" in pip_line:
|
|
pip_line = pip_line.split("+")[0]
|
|
parts = parse_req_from_line(pip_line, line_source)
|
|
|
|
install_req = InstallRequirement(
|
|
parts.requirement,
|
|
comes_from,
|
|
link=parts.link,
|
|
markers=parts.markers,
|
|
use_pep517=use_pep517,
|
|
isolated=isolated,
|
|
global_options=global_options,
|
|
hash_options=hash_options,
|
|
config_settings=config_settings,
|
|
constraint=constraint,
|
|
extras=parts.extras,
|
|
user_supplied=user_supplied,
|
|
)
|
|
return install_req, name
|
|
|
|
|
|
def file_path_from_pipfile(path_str, pipfile_entry):
|
|
"""Creates an installable file path from a pipfile entry.
|
|
Handles local and remote paths, files and directories;
|
|
supports extras and editable specification.
|
|
Outputs a pip installable line.
|
|
"""
|
|
if path_str.startswith(("http:", "https:", "ftp:")):
|
|
req_str = path_str
|
|
else:
|
|
req_str = ensure_path_is_relative(path_str)
|
|
|
|
if pipfile_entry.get("extras"):
|
|
req_str = f"{req_str}[{','.join(pipfile_entry['extras'])}]"
|
|
if pipfile_entry.get("editable", False):
|
|
req_str = f"-e {req_str}"
|
|
|
|
return req_str
|
|
|
|
|
|
def normalize_vcs_url(vcs_url):
|
|
"""Return vcs_url and possible vcs_ref from a given vcs_url."""
|
|
# We have to handle the fact that some vcs urls have a ref in them
|
|
# and some have a netloc with a username and password in them, and some have both
|
|
vcs_ref = ""
|
|
if "@" in vcs_url:
|
|
parsed_url = urlparse(vcs_url)
|
|
if "@" in parsed_url.path:
|
|
url_parts = vcs_url.rsplit("@", 1)
|
|
vcs_url = url_parts[0]
|
|
vcs_ref = url_parts[1]
|
|
return vcs_url, vcs_ref
|
|
|
|
|
|
def install_req_from_pipfile(name, pipfile):
|
|
"""Creates an InstallRequirement from a name and a pipfile entry.
|
|
Handles VCS, local & remote paths, and regular named requirements.
|
|
"file" and "path" entries are treated the same.
|
|
"""
|
|
_pipfile = {}
|
|
vcs = None
|
|
if hasattr(pipfile, "keys"):
|
|
_pipfile = dict(pipfile).copy()
|
|
else:
|
|
vcs = next(iter([vcs for vcs in VCS_LIST if pipfile.startswith(f"{vcs}+")]), None)
|
|
if vcs is not None:
|
|
_pipfile[vcs] = pipfile
|
|
|
|
extras = _pipfile.get("extras", [])
|
|
extras_str = ""
|
|
if extras:
|
|
extras_str = f"[{','.join(extras)}]"
|
|
if not vcs:
|
|
vcs = next(iter([vcs for vcs in VCS_LIST if vcs in _pipfile]), None)
|
|
|
|
if vcs:
|
|
vcs_url = _pipfile[vcs]
|
|
subdirectory = _pipfile.get("subdirectory", "")
|
|
if subdirectory:
|
|
subdirectory = f"#subdirectory={subdirectory}"
|
|
vcs_url, fallback_ref = normalize_vcs_url(vcs_url)
|
|
req_str = f"{vcs_url}@{_pipfile.get('ref', fallback_ref)}{extras_str}"
|
|
if not req_str.startswith(f"{vcs}+"):
|
|
req_str = f"{vcs}+{req_str}"
|
|
if f"{vcs}+file://" in req_str or _pipfile.get("editable", False):
|
|
req_str = (
|
|
f"-e {req_str}#egg={name}{extras_str}{subdirectory.replace('#', '&')}"
|
|
)
|
|
else:
|
|
req_str = f"{name}{extras_str}@ {req_str}{subdirectory}"
|
|
elif "path" in _pipfile:
|
|
req_str = file_path_from_pipfile(_pipfile["path"], _pipfile)
|
|
elif "file" in _pipfile:
|
|
req_str = file_path_from_pipfile(_pipfile["file"], _pipfile)
|
|
else:
|
|
# We ensure version contains an operator. Default to equals (==)
|
|
_pipfile["version"] = version = get_version(pipfile)
|
|
if version and not is_star(version) and COMPARE_OP.match(version) is None:
|
|
_pipfile["version"] = f"=={version}"
|
|
if is_star(version) or version == "==*":
|
|
version = ""
|
|
req_str = f"{name}{extras_str}{version}"
|
|
|
|
install_req, _ = expansive_install_req_from_line(
|
|
req_str,
|
|
comes_from=None,
|
|
use_pep517=False,
|
|
isolated=False,
|
|
hash_options={"hashes": _pipfile.get("hashes", [])},
|
|
constraint=False,
|
|
expand_env=True,
|
|
)
|
|
markers = PipenvMarkers.from_pipfile(name, _pipfile)
|
|
return install_req, markers, req_str
|
|
|
|
|
|
def from_pipfile(name, pipfile):
|
|
install_req, markers, req_str = install_req_from_pipfile(name, pipfile)
|
|
if markers:
|
|
markers = str(markers)
|
|
install_req.markers = Marker(markers)
|
|
|
|
# Construct the requirement string for your Requirement class
|
|
extras_str = ""
|
|
if install_req.req and install_req.req.extras:
|
|
extras_str = f"[{','.join(install_req.req.extras)}]"
|
|
specifier = install_req.req.specifier if install_req.req else ""
|
|
req_str = f"{install_req.name}{extras_str}{specifier}"
|
|
if install_req.markers:
|
|
req_str += f"; {install_req.markers}"
|
|
|
|
# Create the Requirement instance
|
|
cls_inst = Requirement(req_str)
|
|
|
|
return cls_inst
|
|
|
|
|
|
def get_constraints_from_deps(deps):
|
|
"""Get constraints from dictionary-formatted dependency"""
|
|
constraints = set()
|
|
for dep_name, dep_version in deps.items():
|
|
c = None
|
|
# Constraints cannot contain extras
|
|
dep_name = dep_name.split("[", 1)[0]
|
|
# Creating a constraint as a canonical name plus a version specifier
|
|
if isinstance(dep_version, str):
|
|
if dep_version and not is_star(dep_version):
|
|
if COMPARE_OP.match(dep_version) is None:
|
|
dep_version = f"=={dep_version}"
|
|
c = f"{canonicalize_name(dep_name)}{dep_version}"
|
|
else:
|
|
c = canonicalize_name(dep_name)
|
|
else:
|
|
if not any(k in dep_version for k in ["path", "file", "uri"]):
|
|
if dep_version.get("skip_resolver") is True:
|
|
continue
|
|
version = dep_version.get("version", None)
|
|
if version and not is_star(version):
|
|
if COMPARE_OP.match(version) is None:
|
|
version = f"=={dep_version}"
|
|
c = f"{canonicalize_name(dep_name)}{version}"
|
|
else:
|
|
c = canonicalize_name(dep_name)
|
|
if c:
|
|
constraints.add(c)
|
|
return constraints
|
|
|
|
|
|
def prepare_constraint_file(
|
|
constraints,
|
|
directory=None,
|
|
sources=None,
|
|
pip_args=None,
|
|
):
|
|
if not directory:
|
|
directory = create_tracked_tempdir(suffix="-requirements", prefix="pipenv-")
|
|
|
|
constraints = set(constraints)
|
|
constraints_file = NamedTemporaryFile(
|
|
mode="w",
|
|
prefix="pipenv-",
|
|
suffix="-constraints.txt",
|
|
dir=directory,
|
|
delete=False,
|
|
)
|
|
|
|
if sources and pip_args:
|
|
skip_args = ("build-isolation", "use-pep517", "cache-dir")
|
|
args_to_add = [
|
|
arg for arg in pip_args if not any(bad_arg in arg for bad_arg in skip_args)
|
|
]
|
|
requirementstxt_sources = " ".join(args_to_add) if args_to_add else ""
|
|
requirementstxt_sources = requirementstxt_sources.replace(" --", "\n--")
|
|
constraints_file.write(f"{requirementstxt_sources}\n")
|
|
|
|
if constraints:
|
|
constraints_file.write("\n".join(constraints))
|
|
constraints_file.close()
|
|
return constraints_file.name
|
|
|
|
|
|
def is_required_version(version, specified_version):
|
|
"""Check to see if there's a hard requirement for version
|
|
number provided in the Pipfile.
|
|
"""
|
|
# Certain packages may be defined with multiple values.
|
|
if isinstance(specified_version, dict):
|
|
specified_version = specified_version.get("version", "")
|
|
if specified_version.startswith("=="):
|
|
return version.strip() == specified_version.split("==")[1].strip()
|
|
|
|
return True
|
|
|
|
|
|
def is_editable(pipfile_entry):
|
|
if hasattr(pipfile_entry, "get"):
|
|
return pipfile_entry.get("editable", False)
|
|
return False
|
|
|
|
|
|
@contextmanager
|
|
def locked_repository(requirement):
|
|
if not requirement.is_vcs:
|
|
return
|
|
src_dir = create_tracked_tempdir(prefix="pipenv-", suffix="-src")
|
|
with requirement.req.locked_vcs_repo(src_dir=src_dir) as repo:
|
|
yield repo
|