req_install.py 36 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966
  1. # The following comment should be removed at some point in the future.
  2. # mypy: strict-optional=False
  3. # mypy: disallow-untyped-defs=False
  4. from __future__ import absolute_import
  5. import atexit
  6. import logging
  7. import os
  8. import shutil
  9. import sys
  10. import sysconfig
  11. import zipfile
  12. from distutils.util import change_root
  13. from pip._vendor import pkg_resources, six
  14. from pip._vendor.packaging.requirements import Requirement
  15. from pip._vendor.packaging.utils import canonicalize_name
  16. from pip._vendor.packaging.version import Version
  17. from pip._vendor.packaging.version import parse as parse_version
  18. from pip._vendor.pep517.wrappers import Pep517HookCaller
  19. from pip._internal import pep425tags, wheel
  20. from pip._internal.build_env import NoOpBuildEnvironment
  21. from pip._internal.exceptions import InstallationError
  22. from pip._internal.models.link import Link
  23. from pip._internal.operations.generate_metadata import get_metadata_generator
  24. from pip._internal.pyproject import load_pyproject_toml, make_pyproject_path
  25. from pip._internal.req.req_uninstall import UninstallPathSet
  26. from pip._internal.utils.compat import native_str
  27. from pip._internal.utils.hashes import Hashes
  28. from pip._internal.utils.logging import indent_log
  29. from pip._internal.utils.marker_files import (
  30. PIP_DELETE_MARKER_FILENAME,
  31. has_delete_marker_file,
  32. )
  33. from pip._internal.utils.misc import (
  34. _make_build_dir,
  35. ask_path_exists,
  36. backup_dir,
  37. display_path,
  38. dist_in_site_packages,
  39. dist_in_usersite,
  40. ensure_dir,
  41. get_installed_version,
  42. hide_url,
  43. redact_auth_from_url,
  44. rmtree,
  45. )
  46. from pip._internal.utils.packaging import get_metadata
  47. from pip._internal.utils.setuptools_build import make_setuptools_shim_args
  48. from pip._internal.utils.subprocess import (
  49. call_subprocess,
  50. runner_with_spinner_message,
  51. )
  52. from pip._internal.utils.temp_dir import TempDirectory
  53. from pip._internal.utils.typing import MYPY_CHECK_RUNNING
  54. from pip._internal.utils.virtualenv import running_under_virtualenv
  55. from pip._internal.vcs import vcs
  56. if MYPY_CHECK_RUNNING:
  57. from typing import (
  58. Any, Dict, Iterable, List, Optional, Sequence, Union,
  59. )
  60. from pip._internal.build_env import BuildEnvironment
  61. from pip._internal.cache import WheelCache
  62. from pip._internal.index import PackageFinder
  63. from pip._vendor.pkg_resources import Distribution
  64. from pip._vendor.packaging.specifiers import SpecifierSet
  65. from pip._vendor.packaging.markers import Marker
  66. logger = logging.getLogger(__name__)
  67. class InstallRequirement(object):
  68. """
  69. Represents something that may be installed later on, may have information
  70. about where to fetch the relevant requirement and also contains logic for
  71. installing the said requirement.
  72. """
  73. def __init__(
  74. self,
  75. req, # type: Optional[Requirement]
  76. comes_from, # type: Optional[Union[str, InstallRequirement]]
  77. source_dir=None, # type: Optional[str]
  78. editable=False, # type: bool
  79. link=None, # type: Optional[Link]
  80. markers=None, # type: Optional[Marker]
  81. use_pep517=None, # type: Optional[bool]
  82. isolated=False, # type: bool
  83. options=None, # type: Optional[Dict[str, Any]]
  84. wheel_cache=None, # type: Optional[WheelCache]
  85. constraint=False, # type: bool
  86. extras=() # type: Iterable[str]
  87. ):
  88. # type: (...) -> None
  89. assert req is None or isinstance(req, Requirement), req
  90. self.req = req
  91. self.comes_from = comes_from
  92. self.constraint = constraint
  93. if source_dir is None:
  94. self.source_dir = None # type: Optional[str]
  95. else:
  96. self.source_dir = os.path.normpath(os.path.abspath(source_dir))
  97. self.editable = editable
  98. self._wheel_cache = wheel_cache
  99. if link is None and req and req.url:
  100. # PEP 508 URL requirement
  101. link = Link(req.url)
  102. self.link = self.original_link = link
  103. if extras:
  104. self.extras = extras
  105. elif req:
  106. self.extras = {
  107. pkg_resources.safe_extra(extra) for extra in req.extras
  108. }
  109. else:
  110. self.extras = set()
  111. if markers is None and req:
  112. markers = req.marker
  113. self.markers = markers
  114. # This holds the pkg_resources.Distribution object if this requirement
  115. # is already available:
  116. self.satisfied_by = None
  117. # This hold the pkg_resources.Distribution object if this requirement
  118. # conflicts with another installed distribution:
  119. self.conflicts_with = None
  120. # Temporary build location
  121. self._temp_build_dir = None # type: Optional[TempDirectory]
  122. # Used to store the global directory where the _temp_build_dir should
  123. # have been created. Cf move_to_correct_build_directory method.
  124. self._ideal_build_dir = None # type: Optional[str]
  125. # Set to True after successful installation
  126. self.install_succeeded = None # type: Optional[bool]
  127. self.options = options if options else {}
  128. # Set to True after successful preparation of this requirement
  129. self.prepared = False
  130. self.is_direct = False
  131. self.isolated = isolated
  132. self.build_env = NoOpBuildEnvironment() # type: BuildEnvironment
  133. # For PEP 517, the directory where we request the project metadata
  134. # gets stored. We need this to pass to build_wheel, so the backend
  135. # can ensure that the wheel matches the metadata (see the PEP for
  136. # details).
  137. self.metadata_directory = None # type: Optional[str]
  138. # The static build requirements (from pyproject.toml)
  139. self.pyproject_requires = None # type: Optional[List[str]]
  140. # Build requirements that we will check are available
  141. self.requirements_to_check = [] # type: List[str]
  142. # The PEP 517 backend we should use to build the project
  143. self.pep517_backend = None # type: Optional[Pep517HookCaller]
  144. # Are we using PEP 517 for this requirement?
  145. # After pyproject.toml has been loaded, the only valid values are True
  146. # and False. Before loading, None is valid (meaning "use the default").
  147. # Setting an explicit value before loading pyproject.toml is supported,
  148. # but after loading this flag should be treated as read only.
  149. self.use_pep517 = use_pep517
  150. def __str__(self):
  151. # type: () -> str
  152. if self.req:
  153. s = str(self.req)
  154. if self.link:
  155. s += ' from %s' % redact_auth_from_url(self.link.url)
  156. elif self.link:
  157. s = redact_auth_from_url(self.link.url)
  158. else:
  159. s = '<InstallRequirement>'
  160. if self.satisfied_by is not None:
  161. s += ' in %s' % display_path(self.satisfied_by.location)
  162. if self.comes_from:
  163. if isinstance(self.comes_from, six.string_types):
  164. comes_from = self.comes_from # type: Optional[str]
  165. else:
  166. comes_from = self.comes_from.from_path()
  167. if comes_from:
  168. s += ' (from %s)' % comes_from
  169. return s
  170. def __repr__(self):
  171. # type: () -> str
  172. return '<%s object: %s editable=%r>' % (
  173. self.__class__.__name__, str(self), self.editable)
  174. def format_debug(self):
  175. # type: () -> str
  176. """An un-tested helper for getting state, for debugging.
  177. """
  178. attributes = vars(self)
  179. names = sorted(attributes)
  180. state = (
  181. "{}={!r}".format(attr, attributes[attr]) for attr in sorted(names)
  182. )
  183. return '<{name} object: {{{state}}}>'.format(
  184. name=self.__class__.__name__,
  185. state=", ".join(state),
  186. )
  187. def populate_link(self, finder, upgrade, require_hashes):
  188. # type: (PackageFinder, bool, bool) -> None
  189. """Ensure that if a link can be found for this, that it is found.
  190. Note that self.link may still be None - if Upgrade is False and the
  191. requirement is already installed.
  192. If require_hashes is True, don't use the wheel cache, because cached
  193. wheels, always built locally, have different hashes than the files
  194. downloaded from the index server and thus throw false hash mismatches.
  195. Furthermore, cached wheels at present have undeterministic contents due
  196. to file modification times.
  197. """
  198. if self.link is None:
  199. self.link = finder.find_requirement(self, upgrade)
  200. if self._wheel_cache is not None and not require_hashes:
  201. old_link = self.link
  202. supported_tags = pep425tags.get_supported()
  203. self.link = self._wheel_cache.get(
  204. link=self.link,
  205. package_name=self.name,
  206. supported_tags=supported_tags,
  207. )
  208. if old_link != self.link:
  209. logger.debug('Using cached wheel link: %s', self.link)
  210. # Things that are valid for all kinds of requirements?
  211. @property
  212. def name(self):
  213. # type: () -> Optional[str]
  214. if self.req is None:
  215. return None
  216. return native_str(pkg_resources.safe_name(self.req.name))
  217. @property
  218. def specifier(self):
  219. # type: () -> SpecifierSet
  220. return self.req.specifier
  221. @property
  222. def is_pinned(self):
  223. # type: () -> bool
  224. """Return whether I am pinned to an exact version.
  225. For example, some-package==1.2 is pinned; some-package>1.2 is not.
  226. """
  227. specifiers = self.specifier
  228. return (len(specifiers) == 1 and
  229. next(iter(specifiers)).operator in {'==', '==='})
  230. @property
  231. def installed_version(self):
  232. # type: () -> Optional[str]
  233. return get_installed_version(self.name)
  234. def match_markers(self, extras_requested=None):
  235. # type: (Optional[Iterable[str]]) -> bool
  236. if not extras_requested:
  237. # Provide an extra to safely evaluate the markers
  238. # without matching any extra
  239. extras_requested = ('',)
  240. if self.markers is not None:
  241. return any(
  242. self.markers.evaluate({'extra': extra})
  243. for extra in extras_requested)
  244. else:
  245. return True
  246. @property
  247. def has_hash_options(self):
  248. # type: () -> bool
  249. """Return whether any known-good hashes are specified as options.
  250. These activate --require-hashes mode; hashes specified as part of a
  251. URL do not.
  252. """
  253. return bool(self.options.get('hashes', {}))
  254. def hashes(self, trust_internet=True):
  255. # type: (bool) -> Hashes
  256. """Return a hash-comparer that considers my option- and URL-based
  257. hashes to be known-good.
  258. Hashes in URLs--ones embedded in the requirements file, not ones
  259. downloaded from an index server--are almost peers with ones from
  260. flags. They satisfy --require-hashes (whether it was implicitly or
  261. explicitly activated) but do not activate it. md5 and sha224 are not
  262. allowed in flags, which should nudge people toward good algos. We
  263. always OR all hashes together, even ones from URLs.
  264. :param trust_internet: Whether to trust URL-based (#md5=...) hashes
  265. downloaded from the internet, as by populate_link()
  266. """
  267. good_hashes = self.options.get('hashes', {}).copy()
  268. link = self.link if trust_internet else self.original_link
  269. if link and link.hash:
  270. good_hashes.setdefault(link.hash_name, []).append(link.hash)
  271. return Hashes(good_hashes)
  272. def from_path(self):
  273. # type: () -> Optional[str]
  274. """Format a nice indicator to show where this "comes from"
  275. """
  276. if self.req is None:
  277. return None
  278. s = str(self.req)
  279. if self.comes_from:
  280. if isinstance(self.comes_from, six.string_types):
  281. comes_from = self.comes_from
  282. else:
  283. comes_from = self.comes_from.from_path()
  284. if comes_from:
  285. s += '->' + comes_from
  286. return s
  287. def ensure_build_location(self, build_dir):
  288. # type: (str) -> str
  289. assert build_dir is not None
  290. if self._temp_build_dir is not None:
  291. assert self._temp_build_dir.path
  292. return self._temp_build_dir.path
  293. if self.req is None:
  294. # for requirement via a path to a directory: the name of the
  295. # package is not available yet so we create a temp directory
  296. # Once run_egg_info will have run, we'll be able to fix it via
  297. # move_to_correct_build_directory().
  298. # Some systems have /tmp as a symlink which confuses custom
  299. # builds (such as numpy). Thus, we ensure that the real path
  300. # is returned.
  301. self._temp_build_dir = TempDirectory(kind="req-build")
  302. self._ideal_build_dir = build_dir
  303. return self._temp_build_dir.path
  304. if self.editable:
  305. name = self.name.lower()
  306. else:
  307. name = self.name
  308. # FIXME: Is there a better place to create the build_dir? (hg and bzr
  309. # need this)
  310. if not os.path.exists(build_dir):
  311. logger.debug('Creating directory %s', build_dir)
  312. _make_build_dir(build_dir)
  313. return os.path.join(build_dir, name)
  314. def move_to_correct_build_directory(self):
  315. # type: () -> None
  316. """Move self._temp_build_dir to "self._ideal_build_dir/self.req.name"
  317. For some requirements (e.g. a path to a directory), the name of the
  318. package is not available until we run egg_info, so the build_location
  319. will return a temporary directory and store the _ideal_build_dir.
  320. This is only called to "fix" the build directory after generating
  321. metadata.
  322. """
  323. if self.source_dir is not None:
  324. return
  325. assert self.req is not None
  326. assert self._temp_build_dir
  327. assert (
  328. self._ideal_build_dir is not None and
  329. self._ideal_build_dir.path # type: ignore
  330. )
  331. old_location = self._temp_build_dir
  332. self._temp_build_dir = None # checked inside ensure_build_location
  333. # Figure out the correct place to put the files.
  334. new_location = self.ensure_build_location(self._ideal_build_dir)
  335. if os.path.exists(new_location):
  336. raise InstallationError(
  337. 'A package already exists in %s; please remove it to continue'
  338. % display_path(new_location)
  339. )
  340. # Move the files to the correct location.
  341. logger.debug(
  342. 'Moving package %s from %s to new location %s',
  343. self, display_path(old_location.path), display_path(new_location),
  344. )
  345. shutil.move(old_location.path, new_location)
  346. # Update directory-tracking variables, to be in line with new_location
  347. self.source_dir = os.path.normpath(os.path.abspath(new_location))
  348. self._temp_build_dir = TempDirectory(
  349. path=new_location, kind="req-install",
  350. )
  351. # Correct the metadata directory, if it exists
  352. if self.metadata_directory:
  353. old_meta = self.metadata_directory
  354. rel = os.path.relpath(old_meta, start=old_location.path)
  355. new_meta = os.path.join(new_location, rel)
  356. new_meta = os.path.normpath(os.path.abspath(new_meta))
  357. self.metadata_directory = new_meta
  358. # Done with any "move built files" work, since have moved files to the
  359. # "ideal" build location. Setting to None allows to clearly flag that
  360. # no more moves are needed.
  361. self._ideal_build_dir = None
  362. def remove_temporary_source(self):
  363. # type: () -> None
  364. """Remove the source files from this requirement, if they are marked
  365. for deletion"""
  366. if self.source_dir and has_delete_marker_file(self.source_dir):
  367. logger.debug('Removing source in %s', self.source_dir)
  368. rmtree(self.source_dir)
  369. self.source_dir = None
  370. if self._temp_build_dir:
  371. self._temp_build_dir.cleanup()
  372. self._temp_build_dir = None
  373. self.build_env.cleanup()
  374. def check_if_exists(self, use_user_site):
  375. # type: (bool) -> bool
  376. """Find an installed distribution that satisfies or conflicts
  377. with this requirement, and set self.satisfied_by or
  378. self.conflicts_with appropriately.
  379. """
  380. if self.req is None:
  381. return False
  382. try:
  383. # get_distribution() will resolve the entire list of requirements
  384. # anyway, and we've already determined that we need the requirement
  385. # in question, so strip the marker so that we don't try to
  386. # evaluate it.
  387. no_marker = Requirement(str(self.req))
  388. no_marker.marker = None
  389. self.satisfied_by = pkg_resources.get_distribution(str(no_marker))
  390. if self.editable and self.satisfied_by:
  391. self.conflicts_with = self.satisfied_by
  392. # when installing editables, nothing pre-existing should ever
  393. # satisfy
  394. self.satisfied_by = None
  395. return True
  396. except pkg_resources.DistributionNotFound:
  397. return False
  398. except pkg_resources.VersionConflict:
  399. existing_dist = pkg_resources.get_distribution(
  400. self.req.name
  401. )
  402. if use_user_site:
  403. if dist_in_usersite(existing_dist):
  404. self.conflicts_with = existing_dist
  405. elif (running_under_virtualenv() and
  406. dist_in_site_packages(existing_dist)):
  407. raise InstallationError(
  408. "Will not install to the user site because it will "
  409. "lack sys.path precedence to %s in %s" %
  410. (existing_dist.project_name, existing_dist.location)
  411. )
  412. else:
  413. self.conflicts_with = existing_dist
  414. return True
  415. # Things valid for wheels
  416. @property
  417. def is_wheel(self):
  418. # type: () -> bool
  419. if not self.link:
  420. return False
  421. return self.link.is_wheel
  422. def move_wheel_files(
  423. self,
  424. wheeldir, # type: str
  425. root=None, # type: Optional[str]
  426. home=None, # type: Optional[str]
  427. prefix=None, # type: Optional[str]
  428. warn_script_location=True, # type: bool
  429. use_user_site=False, # type: bool
  430. pycompile=True # type: bool
  431. ):
  432. # type: (...) -> None
  433. wheel.move_wheel_files(
  434. self.name, self.req, wheeldir,
  435. user=use_user_site,
  436. home=home,
  437. root=root,
  438. prefix=prefix,
  439. pycompile=pycompile,
  440. isolated=self.isolated,
  441. warn_script_location=warn_script_location,
  442. )
  443. # Things valid for sdists
  444. @property
  445. def unpacked_source_directory(self):
  446. # type: () -> str
  447. return os.path.join(
  448. self.source_dir,
  449. self.link and self.link.subdirectory_fragment or '')
  450. @property
  451. def setup_py_path(self):
  452. # type: () -> str
  453. assert self.source_dir, "No source dir for %s" % self
  454. setup_py = os.path.join(self.unpacked_source_directory, 'setup.py')
  455. # Python2 __file__ should not be unicode
  456. if six.PY2 and isinstance(setup_py, six.text_type):
  457. setup_py = setup_py.encode(sys.getfilesystemencoding())
  458. return setup_py
  459. @property
  460. def pyproject_toml_path(self):
  461. # type: () -> str
  462. assert self.source_dir, "No source dir for %s" % self
  463. return make_pyproject_path(self.unpacked_source_directory)
  464. def load_pyproject_toml(self):
  465. # type: () -> None
  466. """Load the pyproject.toml file.
  467. After calling this routine, all of the attributes related to PEP 517
  468. processing for this requirement have been set. In particular, the
  469. use_pep517 attribute can be used to determine whether we should
  470. follow the PEP 517 or legacy (setup.py) code path.
  471. """
  472. pyproject_toml_data = load_pyproject_toml(
  473. self.use_pep517,
  474. self.pyproject_toml_path,
  475. self.setup_py_path,
  476. str(self)
  477. )
  478. if pyproject_toml_data is None:
  479. self.use_pep517 = False
  480. return
  481. self.use_pep517 = True
  482. requires, backend, check = pyproject_toml_data
  483. self.requirements_to_check = check
  484. self.pyproject_requires = requires
  485. self.pep517_backend = Pep517HookCaller(
  486. self.unpacked_source_directory, backend
  487. )
  488. def prepare_metadata(self):
  489. # type: () -> None
  490. """Ensure that project metadata is available.
  491. Under PEP 517, call the backend hook to prepare the metadata.
  492. Under legacy processing, call setup.py egg-info.
  493. """
  494. assert self.source_dir
  495. metadata_generator = get_metadata_generator(self)
  496. with indent_log():
  497. self.metadata_directory = metadata_generator(self)
  498. if not self.req:
  499. if isinstance(parse_version(self.metadata["Version"]), Version):
  500. op = "=="
  501. else:
  502. op = "==="
  503. self.req = Requirement(
  504. "".join([
  505. self.metadata["Name"],
  506. op,
  507. self.metadata["Version"],
  508. ])
  509. )
  510. self.move_to_correct_build_directory()
  511. else:
  512. metadata_name = canonicalize_name(self.metadata["Name"])
  513. if canonicalize_name(self.req.name) != metadata_name:
  514. logger.warning(
  515. 'Generating metadata for package %s '
  516. 'produced metadata for project name %s. Fix your '
  517. '#egg=%s fragments.',
  518. self.name, metadata_name, self.name
  519. )
  520. self.req = Requirement(metadata_name)
  521. def prepare_pep517_metadata(self):
  522. # type: () -> str
  523. assert self.pep517_backend is not None
  524. # NOTE: This needs to be refactored to stop using atexit
  525. metadata_tmpdir = TempDirectory(kind="modern-metadata")
  526. atexit.register(metadata_tmpdir.cleanup)
  527. metadata_dir = metadata_tmpdir.path
  528. with self.build_env:
  529. # Note that Pep517HookCaller implements a fallback for
  530. # prepare_metadata_for_build_wheel, so we don't have to
  531. # consider the possibility that this hook doesn't exist.
  532. runner = runner_with_spinner_message("Preparing wheel metadata")
  533. backend = self.pep517_backend
  534. with backend.subprocess_runner(runner):
  535. distinfo_dir = backend.prepare_metadata_for_build_wheel(
  536. metadata_dir
  537. )
  538. return os.path.join(metadata_dir, distinfo_dir)
  539. @property
  540. def metadata(self):
  541. # type: () -> Any
  542. if not hasattr(self, '_metadata'):
  543. self._metadata = get_metadata(self.get_dist())
  544. return self._metadata
  545. def get_dist(self):
  546. # type: () -> Distribution
  547. """Return a pkg_resources.Distribution for this requirement"""
  548. dist_dir = self.metadata_directory.rstrip(os.sep)
  549. # Determine the correct Distribution object type.
  550. if dist_dir.endswith(".egg-info"):
  551. dist_cls = pkg_resources.Distribution
  552. else:
  553. assert dist_dir.endswith(".dist-info")
  554. dist_cls = pkg_resources.DistInfoDistribution
  555. # Build a PathMetadata object, from path to metadata. :wink:
  556. base_dir, dist_dir_name = os.path.split(dist_dir)
  557. dist_name = os.path.splitext(dist_dir_name)[0]
  558. metadata = pkg_resources.PathMetadata(base_dir, dist_dir)
  559. return dist_cls(
  560. base_dir,
  561. project_name=dist_name,
  562. metadata=metadata,
  563. )
  564. def assert_source_matches_version(self):
  565. # type: () -> None
  566. assert self.source_dir
  567. version = self.metadata['version']
  568. if self.req.specifier and version not in self.req.specifier:
  569. logger.warning(
  570. 'Requested %s, but installing version %s',
  571. self,
  572. version,
  573. )
  574. else:
  575. logger.debug(
  576. 'Source in %s has version %s, which satisfies requirement %s',
  577. display_path(self.source_dir),
  578. version,
  579. self,
  580. )
  581. # For both source distributions and editables
  582. def ensure_has_source_dir(self, parent_dir):
  583. # type: (str) -> None
  584. """Ensure that a source_dir is set.
  585. This will create a temporary build dir if the name of the requirement
  586. isn't known yet.
  587. :param parent_dir: The ideal pip parent_dir for the source_dir.
  588. Generally src_dir for editables and build_dir for sdists.
  589. :return: self.source_dir
  590. """
  591. if self.source_dir is None:
  592. self.source_dir = self.ensure_build_location(parent_dir)
  593. # For editable installations
  594. def install_editable(
  595. self,
  596. install_options, # type: List[str]
  597. global_options=(), # type: Sequence[str]
  598. prefix=None # type: Optional[str]
  599. ):
  600. # type: (...) -> None
  601. logger.info('Running setup.py develop for %s', self.name)
  602. if prefix:
  603. prefix_param = ['--prefix={}'.format(prefix)]
  604. install_options = list(install_options) + prefix_param
  605. base_cmd = make_setuptools_shim_args(
  606. self.setup_py_path,
  607. global_options=global_options,
  608. no_user_config=self.isolated
  609. )
  610. with indent_log():
  611. with self.build_env:
  612. call_subprocess(
  613. base_cmd +
  614. ['develop', '--no-deps'] +
  615. list(install_options),
  616. cwd=self.unpacked_source_directory,
  617. )
  618. self.install_succeeded = True
  619. def update_editable(self, obtain=True):
  620. # type: (bool) -> None
  621. if not self.link:
  622. logger.debug(
  623. "Cannot update repository at %s; repository location is "
  624. "unknown",
  625. self.source_dir,
  626. )
  627. return
  628. assert self.editable
  629. assert self.source_dir
  630. if self.link.scheme == 'file':
  631. # Static paths don't get updated
  632. return
  633. assert '+' in self.link.url, "bad url: %r" % self.link.url
  634. vc_type, url = self.link.url.split('+', 1)
  635. vcs_backend = vcs.get_backend(vc_type)
  636. if vcs_backend:
  637. hidden_url = hide_url(self.link.url)
  638. if obtain:
  639. vcs_backend.obtain(self.source_dir, url=hidden_url)
  640. else:
  641. vcs_backend.export(self.source_dir, url=hidden_url)
  642. else:
  643. assert 0, (
  644. 'Unexpected version control type (in %s): %s'
  645. % (self.link, vc_type))
  646. # Top-level Actions
  647. def uninstall(self, auto_confirm=False, verbose=False,
  648. use_user_site=False):
  649. # type: (bool, bool, bool) -> Optional[UninstallPathSet]
  650. """
  651. Uninstall the distribution currently satisfying this requirement.
  652. Prompts before removing or modifying files unless
  653. ``auto_confirm`` is True.
  654. Refuses to delete or modify files outside of ``sys.prefix`` -
  655. thus uninstallation within a virtual environment can only
  656. modify that virtual environment, even if the virtualenv is
  657. linked to global site-packages.
  658. """
  659. if not self.check_if_exists(use_user_site):
  660. logger.warning("Skipping %s as it is not installed.", self.name)
  661. return None
  662. dist = self.satisfied_by or self.conflicts_with
  663. uninstalled_pathset = UninstallPathSet.from_dist(dist)
  664. uninstalled_pathset.remove(auto_confirm, verbose)
  665. return uninstalled_pathset
  666. def _clean_zip_name(self, name, prefix): # only used by archive.
  667. # type: (str, str) -> str
  668. assert name.startswith(prefix + os.path.sep), (
  669. "name %r doesn't start with prefix %r" % (name, prefix)
  670. )
  671. name = name[len(prefix) + 1:]
  672. name = name.replace(os.path.sep, '/')
  673. return name
  674. def _get_archive_name(self, path, parentdir, rootdir):
  675. # type: (str, str, str) -> str
  676. path = os.path.join(parentdir, path)
  677. name = self._clean_zip_name(path, rootdir)
  678. return self.name + '/' + name
  679. def archive(self, build_dir):
  680. # type: (str) -> None
  681. """Saves archive to provided build_dir.
  682. Used for saving downloaded VCS requirements as part of `pip download`.
  683. """
  684. assert self.source_dir
  685. create_archive = True
  686. archive_name = '%s-%s.zip' % (self.name, self.metadata["version"])
  687. archive_path = os.path.join(build_dir, archive_name)
  688. if os.path.exists(archive_path):
  689. response = ask_path_exists(
  690. 'The file %s exists. (i)gnore, (w)ipe, (b)ackup, (a)bort ' %
  691. display_path(archive_path), ('i', 'w', 'b', 'a'))
  692. if response == 'i':
  693. create_archive = False
  694. elif response == 'w':
  695. logger.warning('Deleting %s', display_path(archive_path))
  696. os.remove(archive_path)
  697. elif response == 'b':
  698. dest_file = backup_dir(archive_path)
  699. logger.warning(
  700. 'Backing up %s to %s',
  701. display_path(archive_path),
  702. display_path(dest_file),
  703. )
  704. shutil.move(archive_path, dest_file)
  705. elif response == 'a':
  706. sys.exit(-1)
  707. if not create_archive:
  708. return
  709. zip_output = zipfile.ZipFile(
  710. archive_path, 'w', zipfile.ZIP_DEFLATED, allowZip64=True,
  711. )
  712. with zip_output:
  713. dir = os.path.normcase(
  714. os.path.abspath(self.unpacked_source_directory)
  715. )
  716. for dirpath, dirnames, filenames in os.walk(dir):
  717. if 'pip-egg-info' in dirnames:
  718. dirnames.remove('pip-egg-info')
  719. for dirname in dirnames:
  720. dir_arcname = self._get_archive_name(
  721. dirname, parentdir=dirpath, rootdir=dir,
  722. )
  723. zipdir = zipfile.ZipInfo(dir_arcname + '/')
  724. zipdir.external_attr = 0x1ED << 16 # 0o755
  725. zip_output.writestr(zipdir, '')
  726. for filename in filenames:
  727. if filename == PIP_DELETE_MARKER_FILENAME:
  728. continue
  729. file_arcname = self._get_archive_name(
  730. filename, parentdir=dirpath, rootdir=dir,
  731. )
  732. filename = os.path.join(dirpath, filename)
  733. zip_output.write(filename, file_arcname)
  734. logger.info('Saved %s', display_path(archive_path))
  735. def install(
  736. self,
  737. install_options, # type: List[str]
  738. global_options=None, # type: Optional[Sequence[str]]
  739. root=None, # type: Optional[str]
  740. home=None, # type: Optional[str]
  741. prefix=None, # type: Optional[str]
  742. warn_script_location=True, # type: bool
  743. use_user_site=False, # type: bool
  744. pycompile=True # type: bool
  745. ):
  746. # type: (...) -> None
  747. global_options = global_options if global_options is not None else []
  748. if self.editable:
  749. self.install_editable(
  750. install_options, global_options, prefix=prefix,
  751. )
  752. return
  753. if self.is_wheel:
  754. version = wheel.wheel_version(self.source_dir)
  755. wheel.check_compatibility(version, self.name)
  756. self.move_wheel_files(
  757. self.source_dir, root=root, prefix=prefix, home=home,
  758. warn_script_location=warn_script_location,
  759. use_user_site=use_user_site, pycompile=pycompile,
  760. )
  761. self.install_succeeded = True
  762. return
  763. # Extend the list of global and install options passed on to
  764. # the setup.py call with the ones from the requirements file.
  765. # Options specified in requirements file override those
  766. # specified on the command line, since the last option given
  767. # to setup.py is the one that is used.
  768. global_options = list(global_options) + \
  769. self.options.get('global_options', [])
  770. install_options = list(install_options) + \
  771. self.options.get('install_options', [])
  772. with TempDirectory(kind="record") as temp_dir:
  773. record_filename = os.path.join(temp_dir.path, 'install-record.txt')
  774. install_args = self.get_install_args(
  775. global_options, record_filename, root, prefix, pycompile,
  776. )
  777. runner = runner_with_spinner_message(
  778. "Running setup.py install for {}".format(self.name)
  779. )
  780. with indent_log(), self.build_env:
  781. runner(
  782. cmd=install_args + install_options,
  783. cwd=self.unpacked_source_directory,
  784. )
  785. if not os.path.exists(record_filename):
  786. logger.debug('Record file %s not found', record_filename)
  787. return
  788. self.install_succeeded = True
  789. def prepend_root(path):
  790. # type: (str) -> str
  791. if root is None or not os.path.isabs(path):
  792. return path
  793. else:
  794. return change_root(root, path)
  795. with open(record_filename) as f:
  796. for line in f:
  797. directory = os.path.dirname(line)
  798. if directory.endswith('.egg-info'):
  799. egg_info_dir = prepend_root(directory)
  800. break
  801. else:
  802. logger.warning(
  803. 'Could not find .egg-info directory in install record'
  804. ' for %s',
  805. self,
  806. )
  807. # FIXME: put the record somewhere
  808. return
  809. new_lines = []
  810. with open(record_filename) as f:
  811. for line in f:
  812. filename = line.strip()
  813. if os.path.isdir(filename):
  814. filename += os.path.sep
  815. new_lines.append(
  816. os.path.relpath(prepend_root(filename), egg_info_dir)
  817. )
  818. new_lines.sort()
  819. ensure_dir(egg_info_dir)
  820. inst_files_path = os.path.join(egg_info_dir, 'installed-files.txt')
  821. with open(inst_files_path, 'w') as f:
  822. f.write('\n'.join(new_lines) + '\n')
  823. def get_install_args(
  824. self,
  825. global_options, # type: Sequence[str]
  826. record_filename, # type: str
  827. root, # type: Optional[str]
  828. prefix, # type: Optional[str]
  829. pycompile # type: bool
  830. ):
  831. # type: (...) -> List[str]
  832. install_args = make_setuptools_shim_args(
  833. self.setup_py_path,
  834. global_options=global_options,
  835. no_user_config=self.isolated,
  836. unbuffered_output=True
  837. )
  838. install_args += ['install', '--record', record_filename]
  839. install_args += ['--single-version-externally-managed']
  840. if root is not None:
  841. install_args += ['--root', root]
  842. if prefix is not None:
  843. install_args += ['--prefix', prefix]
  844. if pycompile:
  845. install_args += ["--compile"]
  846. else:
  847. install_args += ["--no-compile"]
  848. if running_under_virtualenv():
  849. py_ver_str = 'python' + sysconfig.get_python_version()
  850. install_args += ['--install-headers',
  851. os.path.join(sys.prefix, 'include', 'site',
  852. py_ver_str, self.name)]
  853. return install_args