legacy_resolve.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460
  1. """Dependency Resolution
  2. The dependency resolution in pip is performed as follows:
  3. for top-level requirements:
  4. a. only one spec allowed per project, regardless of conflicts or not.
  5. otherwise a "double requirement" exception is raised
  6. b. they override sub-dependency requirements.
  7. for sub-dependencies
  8. a. "first found, wins" (where the order is breadth first)
  9. """
  10. # The following comment should be removed at some point in the future.
  11. # mypy: strict-optional=False
  12. # mypy: disallow-untyped-defs=False
  13. import logging
  14. import sys
  15. from collections import defaultdict
  16. from itertools import chain
  17. from pip._vendor.packaging import specifiers
  18. from pip._internal.exceptions import (
  19. BestVersionAlreadyInstalled,
  20. DistributionNotFound,
  21. HashError,
  22. HashErrors,
  23. UnsupportedPythonVersion,
  24. )
  25. from pip._internal.utils.logging import indent_log
  26. from pip._internal.utils.misc import (
  27. dist_in_usersite,
  28. ensure_dir,
  29. normalize_version_info,
  30. )
  31. from pip._internal.utils.packaging import (
  32. check_requires_python,
  33. get_requires_python,
  34. )
  35. from pip._internal.utils.typing import MYPY_CHECK_RUNNING
  36. if MYPY_CHECK_RUNNING:
  37. from typing import Callable, DefaultDict, List, Optional, Set, Tuple
  38. from pip._vendor import pkg_resources
  39. from pip._internal.distributions import AbstractDistribution
  40. from pip._internal.network.session import PipSession
  41. from pip._internal.index import PackageFinder
  42. from pip._internal.operations.prepare import RequirementPreparer
  43. from pip._internal.req.req_install import InstallRequirement
  44. from pip._internal.req.req_set import RequirementSet
  45. InstallRequirementProvider = Callable[
  46. [str, InstallRequirement], InstallRequirement
  47. ]
  48. logger = logging.getLogger(__name__)
  49. def _check_dist_requires_python(
  50. dist, # type: pkg_resources.Distribution
  51. version_info, # type: Tuple[int, int, int]
  52. ignore_requires_python=False, # type: bool
  53. ):
  54. # type: (...) -> None
  55. """
  56. Check whether the given Python version is compatible with a distribution's
  57. "Requires-Python" value.
  58. :param version_info: A 3-tuple of ints representing the Python
  59. major-minor-micro version to check.
  60. :param ignore_requires_python: Whether to ignore the "Requires-Python"
  61. value if the given Python version isn't compatible.
  62. :raises UnsupportedPythonVersion: When the given Python version isn't
  63. compatible.
  64. """
  65. requires_python = get_requires_python(dist)
  66. try:
  67. is_compatible = check_requires_python(
  68. requires_python, version_info=version_info,
  69. )
  70. except specifiers.InvalidSpecifier as exc:
  71. logger.warning(
  72. "Package %r has an invalid Requires-Python: %s",
  73. dist.project_name, exc,
  74. )
  75. return
  76. if is_compatible:
  77. return
  78. version = '.'.join(map(str, version_info))
  79. if ignore_requires_python:
  80. logger.debug(
  81. 'Ignoring failed Requires-Python check for package %r: '
  82. '%s not in %r',
  83. dist.project_name, version, requires_python,
  84. )
  85. return
  86. raise UnsupportedPythonVersion(
  87. 'Package {!r} requires a different Python: {} not in {!r}'.format(
  88. dist.project_name, version, requires_python,
  89. ))
  90. class Resolver(object):
  91. """Resolves which packages need to be installed/uninstalled to perform \
  92. the requested operation without breaking the requirements of any package.
  93. """
  94. _allowed_strategies = {"eager", "only-if-needed", "to-satisfy-only"}
  95. def __init__(
  96. self,
  97. preparer, # type: RequirementPreparer
  98. session, # type: PipSession
  99. finder, # type: PackageFinder
  100. make_install_req, # type: InstallRequirementProvider
  101. use_user_site, # type: bool
  102. ignore_dependencies, # type: bool
  103. ignore_installed, # type: bool
  104. ignore_requires_python, # type: bool
  105. force_reinstall, # type: bool
  106. upgrade_strategy, # type: str
  107. py_version_info=None, # type: Optional[Tuple[int, ...]]
  108. ):
  109. # type: (...) -> None
  110. super(Resolver, self).__init__()
  111. assert upgrade_strategy in self._allowed_strategies
  112. if py_version_info is None:
  113. py_version_info = sys.version_info[:3]
  114. else:
  115. py_version_info = normalize_version_info(py_version_info)
  116. self._py_version_info = py_version_info
  117. self.preparer = preparer
  118. self.finder = finder
  119. self.session = session
  120. # This is set in resolve
  121. self.require_hashes = None # type: Optional[bool]
  122. self.upgrade_strategy = upgrade_strategy
  123. self.force_reinstall = force_reinstall
  124. self.ignore_dependencies = ignore_dependencies
  125. self.ignore_installed = ignore_installed
  126. self.ignore_requires_python = ignore_requires_python
  127. self.use_user_site = use_user_site
  128. self._make_install_req = make_install_req
  129. self._discovered_dependencies = \
  130. defaultdict(list) # type: DefaultDict[str, List]
  131. def resolve(self, requirement_set):
  132. # type: (RequirementSet) -> None
  133. """Resolve what operations need to be done
  134. As a side-effect of this method, the packages (and their dependencies)
  135. are downloaded, unpacked and prepared for installation. This
  136. preparation is done by ``pip.operations.prepare``.
  137. Once PyPI has static dependency metadata available, it would be
  138. possible to move the preparation to become a step separated from
  139. dependency resolution.
  140. """
  141. # make the wheelhouse
  142. if self.preparer.wheel_download_dir:
  143. ensure_dir(self.preparer.wheel_download_dir)
  144. # If any top-level requirement has a hash specified, enter
  145. # hash-checking mode, which requires hashes from all.
  146. root_reqs = (
  147. requirement_set.unnamed_requirements +
  148. list(requirement_set.requirements.values())
  149. )
  150. self.require_hashes = (
  151. requirement_set.require_hashes or
  152. any(req.has_hash_options for req in root_reqs)
  153. )
  154. # Display where finder is looking for packages
  155. search_scope = self.finder.search_scope
  156. locations = search_scope.get_formatted_locations()
  157. if locations:
  158. logger.info(locations)
  159. # Actually prepare the files, and collect any exceptions. Most hash
  160. # exceptions cannot be checked ahead of time, because
  161. # req.populate_link() needs to be called before we can make decisions
  162. # based on link type.
  163. discovered_reqs = [] # type: List[InstallRequirement]
  164. hash_errors = HashErrors()
  165. for req in chain(root_reqs, discovered_reqs):
  166. try:
  167. discovered_reqs.extend(
  168. self._resolve_one(requirement_set, req)
  169. )
  170. except HashError as exc:
  171. exc.req = req
  172. hash_errors.append(exc)
  173. if hash_errors:
  174. raise hash_errors
  175. def _is_upgrade_allowed(self, req):
  176. # type: (InstallRequirement) -> bool
  177. if self.upgrade_strategy == "to-satisfy-only":
  178. return False
  179. elif self.upgrade_strategy == "eager":
  180. return True
  181. else:
  182. assert self.upgrade_strategy == "only-if-needed"
  183. return req.is_direct
  184. def _set_req_to_reinstall(self, req):
  185. # type: (InstallRequirement) -> None
  186. """
  187. Set a requirement to be installed.
  188. """
  189. # Don't uninstall the conflict if doing a user install and the
  190. # conflict is not a user install.
  191. if not self.use_user_site or dist_in_usersite(req.satisfied_by):
  192. req.conflicts_with = req.satisfied_by
  193. req.satisfied_by = None
  194. def _check_skip_installed(self, req_to_install):
  195. # type: (InstallRequirement) -> Optional[str]
  196. """Check if req_to_install should be skipped.
  197. This will check if the req is installed, and whether we should upgrade
  198. or reinstall it, taking into account all the relevant user options.
  199. After calling this req_to_install will only have satisfied_by set to
  200. None if the req_to_install is to be upgraded/reinstalled etc. Any
  201. other value will be a dist recording the current thing installed that
  202. satisfies the requirement.
  203. Note that for vcs urls and the like we can't assess skipping in this
  204. routine - we simply identify that we need to pull the thing down,
  205. then later on it is pulled down and introspected to assess upgrade/
  206. reinstalls etc.
  207. :return: A text reason for why it was skipped, or None.
  208. """
  209. if self.ignore_installed:
  210. return None
  211. req_to_install.check_if_exists(self.use_user_site)
  212. if not req_to_install.satisfied_by:
  213. return None
  214. if self.force_reinstall:
  215. self._set_req_to_reinstall(req_to_install)
  216. return None
  217. if not self._is_upgrade_allowed(req_to_install):
  218. if self.upgrade_strategy == "only-if-needed":
  219. return 'already satisfied, skipping upgrade'
  220. return 'already satisfied'
  221. # Check for the possibility of an upgrade. For link-based
  222. # requirements we have to pull the tree down and inspect to assess
  223. # the version #, so it's handled way down.
  224. if not req_to_install.link:
  225. try:
  226. self.finder.find_requirement(req_to_install, upgrade=True)
  227. except BestVersionAlreadyInstalled:
  228. # Then the best version is installed.
  229. return 'already up-to-date'
  230. except DistributionNotFound:
  231. # No distribution found, so we squash the error. It will
  232. # be raised later when we re-try later to do the install.
  233. # Why don't we just raise here?
  234. pass
  235. self._set_req_to_reinstall(req_to_install)
  236. return None
  237. def _get_abstract_dist_for(self, req):
  238. # type: (InstallRequirement) -> AbstractDistribution
  239. """Takes a InstallRequirement and returns a single AbstractDist \
  240. representing a prepared variant of the same.
  241. """
  242. assert self.require_hashes is not None, (
  243. "require_hashes should have been set in Resolver.resolve()"
  244. )
  245. if req.editable:
  246. return self.preparer.prepare_editable_requirement(
  247. req, self.require_hashes, self.use_user_site, self.finder,
  248. )
  249. # satisfied_by is only evaluated by calling _check_skip_installed,
  250. # so it must be None here.
  251. assert req.satisfied_by is None
  252. skip_reason = self._check_skip_installed(req)
  253. if req.satisfied_by:
  254. return self.preparer.prepare_installed_requirement(
  255. req, self.require_hashes, skip_reason
  256. )
  257. upgrade_allowed = self._is_upgrade_allowed(req)
  258. # We eagerly populate the link, since that's our "legacy" behavior.
  259. req.populate_link(self.finder, upgrade_allowed, self.require_hashes)
  260. abstract_dist = self.preparer.prepare_linked_requirement(
  261. req, self.session, self.finder, self.require_hashes
  262. )
  263. # NOTE
  264. # The following portion is for determining if a certain package is
  265. # going to be re-installed/upgraded or not and reporting to the user.
  266. # This should probably get cleaned up in a future refactor.
  267. # req.req is only avail after unpack for URL
  268. # pkgs repeat check_if_exists to uninstall-on-upgrade
  269. # (#14)
  270. if not self.ignore_installed:
  271. req.check_if_exists(self.use_user_site)
  272. if req.satisfied_by:
  273. should_modify = (
  274. self.upgrade_strategy != "to-satisfy-only" or
  275. self.force_reinstall or
  276. self.ignore_installed or
  277. req.link.scheme == 'file'
  278. )
  279. if should_modify:
  280. self._set_req_to_reinstall(req)
  281. else:
  282. logger.info(
  283. 'Requirement already satisfied (use --upgrade to upgrade):'
  284. ' %s', req,
  285. )
  286. return abstract_dist
  287. def _resolve_one(
  288. self,
  289. requirement_set, # type: RequirementSet
  290. req_to_install # type: InstallRequirement
  291. ):
  292. # type: (...) -> List[InstallRequirement]
  293. """Prepare a single requirements file.
  294. :return: A list of additional InstallRequirements to also install.
  295. """
  296. # Tell user what we are doing for this requirement:
  297. # obtain (editable), skipping, processing (local url), collecting
  298. # (remote url or package name)
  299. if req_to_install.constraint or req_to_install.prepared:
  300. return []
  301. req_to_install.prepared = True
  302. # register tmp src for cleanup in case something goes wrong
  303. requirement_set.reqs_to_cleanup.append(req_to_install)
  304. abstract_dist = self._get_abstract_dist_for(req_to_install)
  305. # Parse and return dependencies
  306. dist = abstract_dist.get_pkg_resources_distribution()
  307. # This will raise UnsupportedPythonVersion if the given Python
  308. # version isn't compatible with the distribution's Requires-Python.
  309. _check_dist_requires_python(
  310. dist, version_info=self._py_version_info,
  311. ignore_requires_python=self.ignore_requires_python,
  312. )
  313. more_reqs = [] # type: List[InstallRequirement]
  314. def add_req(subreq, extras_requested):
  315. sub_install_req = self._make_install_req(
  316. str(subreq),
  317. req_to_install,
  318. )
  319. parent_req_name = req_to_install.name
  320. to_scan_again, add_to_parent = requirement_set.add_requirement(
  321. sub_install_req,
  322. parent_req_name=parent_req_name,
  323. extras_requested=extras_requested,
  324. )
  325. if parent_req_name and add_to_parent:
  326. self._discovered_dependencies[parent_req_name].append(
  327. add_to_parent
  328. )
  329. more_reqs.extend(to_scan_again)
  330. with indent_log():
  331. # We add req_to_install before its dependencies, so that we
  332. # can refer to it when adding dependencies.
  333. if not requirement_set.has_requirement(req_to_install.name):
  334. # 'unnamed' requirements will get added here
  335. req_to_install.is_direct = True
  336. requirement_set.add_requirement(
  337. req_to_install, parent_req_name=None,
  338. )
  339. if not self.ignore_dependencies:
  340. if req_to_install.extras:
  341. logger.debug(
  342. "Installing extra requirements: %r",
  343. ','.join(req_to_install.extras),
  344. )
  345. missing_requested = sorted(
  346. set(req_to_install.extras) - set(dist.extras)
  347. )
  348. for missing in missing_requested:
  349. logger.warning(
  350. '%s does not provide the extra \'%s\'',
  351. dist, missing
  352. )
  353. available_requested = sorted(
  354. set(dist.extras) & set(req_to_install.extras)
  355. )
  356. for subreq in dist.requires(available_requested):
  357. add_req(subreq, extras_requested=available_requested)
  358. if not req_to_install.editable and not req_to_install.satisfied_by:
  359. # XXX: --no-install leads this to report 'Successfully
  360. # downloaded' for only non-editable reqs, even though we took
  361. # action on them.
  362. requirement_set.successfully_downloaded.append(req_to_install)
  363. return more_reqs
  364. def get_installation_order(self, req_set):
  365. # type: (RequirementSet) -> List[InstallRequirement]
  366. """Create the installation order.
  367. The installation order is topological - requirements are installed
  368. before the requiring thing. We break cycles at an arbitrary point,
  369. and make no other guarantees.
  370. """
  371. # The current implementation, which we may change at any point
  372. # installs the user specified things in the order given, except when
  373. # dependencies must come earlier to achieve topological order.
  374. order = []
  375. ordered_reqs = set() # type: Set[InstallRequirement]
  376. def schedule(req):
  377. if req.satisfied_by or req in ordered_reqs:
  378. return
  379. if req.constraint:
  380. return
  381. ordered_reqs.add(req)
  382. for dep in self._discovered_dependencies[req.name]:
  383. schedule(dep)
  384. order.append(req)
  385. for install_req in req_set.requirements.values():
  386. schedule(install_req)
  387. return order