wheel.py 42 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181
  1. """
  2. Support for installing and building the "wheel" binary package format.
  3. """
  4. # The following comment should be removed at some point in the future.
  5. # mypy: strict-optional=False
  6. # mypy: disallow-untyped-defs=False
  7. from __future__ import absolute_import
  8. import collections
  9. import compileall
  10. import csv
  11. import hashlib
  12. import logging
  13. import os.path
  14. import re
  15. import shutil
  16. import stat
  17. import sys
  18. import warnings
  19. from base64 import urlsafe_b64encode
  20. from email.parser import Parser
  21. from pip._vendor import pkg_resources
  22. from pip._vendor.distlib.scripts import ScriptMaker
  23. from pip._vendor.distlib.util import get_export_entry
  24. from pip._vendor.packaging.utils import canonicalize_name
  25. from pip._vendor.six import StringIO
  26. from pip._internal import pep425tags
  27. from pip._internal.exceptions import (
  28. InstallationError,
  29. InvalidWheelFilename,
  30. UnsupportedWheel,
  31. )
  32. from pip._internal.locations import distutils_scheme, get_major_minor_version
  33. from pip._internal.models.link import Link
  34. from pip._internal.utils.logging import indent_log
  35. from pip._internal.utils.marker_files import has_delete_marker_file
  36. from pip._internal.utils.misc import captured_stdout, ensure_dir, read_chunks
  37. from pip._internal.utils.setuptools_build import make_setuptools_shim_args
  38. from pip._internal.utils.subprocess import (
  39. LOG_DIVIDER,
  40. call_subprocess,
  41. format_command_args,
  42. runner_with_spinner_message,
  43. )
  44. from pip._internal.utils.temp_dir import TempDirectory
  45. from pip._internal.utils.typing import MYPY_CHECK_RUNNING
  46. from pip._internal.utils.ui import open_spinner
  47. from pip._internal.utils.unpacking import unpack_file
  48. from pip._internal.utils.urls import path_to_url
  49. if MYPY_CHECK_RUNNING:
  50. from typing import (
  51. Dict, List, Optional, Sequence, Mapping, Tuple, IO, Text, Any,
  52. Iterable, Callable, Set,
  53. )
  54. from pip._vendor.packaging.requirements import Requirement
  55. from pip._internal.req.req_install import InstallRequirement
  56. from pip._internal.operations.prepare import (
  57. RequirementPreparer
  58. )
  59. from pip._internal.cache import WheelCache
  60. from pip._internal.pep425tags import Pep425Tag
  61. InstalledCSVRow = Tuple[str, ...]
  62. BinaryAllowedPredicate = Callable[[InstallRequirement], bool]
  63. VERSION_COMPATIBLE = (1, 0)
  64. logger = logging.getLogger(__name__)
  65. def normpath(src, p):
  66. return os.path.relpath(src, p).replace(os.path.sep, '/')
  67. def hash_file(path, blocksize=1 << 20):
  68. # type: (str, int) -> Tuple[Any, int]
  69. """Return (hash, length) for path using hashlib.sha256()"""
  70. h = hashlib.sha256()
  71. length = 0
  72. with open(path, 'rb') as f:
  73. for block in read_chunks(f, size=blocksize):
  74. length += len(block)
  75. h.update(block)
  76. return (h, length) # type: ignore
  77. def rehash(path, blocksize=1 << 20):
  78. # type: (str, int) -> Tuple[str, str]
  79. """Return (encoded_digest, length) for path using hashlib.sha256()"""
  80. h, length = hash_file(path, blocksize)
  81. digest = 'sha256=' + urlsafe_b64encode(
  82. h.digest()
  83. ).decode('latin1').rstrip('=')
  84. # unicode/str python2 issues
  85. return (digest, str(length)) # type: ignore
  86. def open_for_csv(name, mode):
  87. # type: (str, Text) -> IO
  88. if sys.version_info[0] < 3:
  89. nl = {} # type: Dict[str, Any]
  90. bin = 'b'
  91. else:
  92. nl = {'newline': ''} # type: Dict[str, Any]
  93. bin = ''
  94. return open(name, mode + bin, **nl)
  95. def replace_python_tag(wheelname, new_tag):
  96. # type: (str, str) -> str
  97. """Replace the Python tag in a wheel file name with a new value.
  98. """
  99. parts = wheelname.split('-')
  100. parts[-3] = new_tag
  101. return '-'.join(parts)
  102. def fix_script(path):
  103. # type: (str) -> Optional[bool]
  104. """Replace #!python with #!/path/to/python
  105. Return True if file was changed."""
  106. # XXX RECORD hashes will need to be updated
  107. if os.path.isfile(path):
  108. with open(path, 'rb') as script:
  109. firstline = script.readline()
  110. if not firstline.startswith(b'#!python'):
  111. return False
  112. exename = sys.executable.encode(sys.getfilesystemencoding())
  113. firstline = b'#!' + exename + os.linesep.encode("ascii")
  114. rest = script.read()
  115. with open(path, 'wb') as script:
  116. script.write(firstline)
  117. script.write(rest)
  118. return True
  119. return None
  120. dist_info_re = re.compile(r"""^(?P<namever>(?P<name>.+?)(-(?P<ver>.+?))?)
  121. \.dist-info$""", re.VERBOSE)
  122. def root_is_purelib(name, wheeldir):
  123. # type: (str, str) -> bool
  124. """
  125. Return True if the extracted wheel in wheeldir should go into purelib.
  126. """
  127. name_folded = name.replace("-", "_")
  128. for item in os.listdir(wheeldir):
  129. match = dist_info_re.match(item)
  130. if match and match.group('name') == name_folded:
  131. with open(os.path.join(wheeldir, item, 'WHEEL')) as wheel:
  132. for line in wheel:
  133. line = line.lower().rstrip()
  134. if line == "root-is-purelib: true":
  135. return True
  136. return False
  137. def get_entrypoints(filename):
  138. # type: (str) -> Tuple[Dict[str, str], Dict[str, str]]
  139. if not os.path.exists(filename):
  140. return {}, {}
  141. # This is done because you can pass a string to entry_points wrappers which
  142. # means that they may or may not be valid INI files. The attempt here is to
  143. # strip leading and trailing whitespace in order to make them valid INI
  144. # files.
  145. with open(filename) as fp:
  146. data = StringIO()
  147. for line in fp:
  148. data.write(line.strip())
  149. data.write("\n")
  150. data.seek(0)
  151. # get the entry points and then the script names
  152. entry_points = pkg_resources.EntryPoint.parse_map(data)
  153. console = entry_points.get('console_scripts', {})
  154. gui = entry_points.get('gui_scripts', {})
  155. def _split_ep(s):
  156. """get the string representation of EntryPoint, remove space and split
  157. on '='"""
  158. return str(s).replace(" ", "").split("=")
  159. # convert the EntryPoint objects into strings with module:function
  160. console = dict(_split_ep(v) for v in console.values())
  161. gui = dict(_split_ep(v) for v in gui.values())
  162. return console, gui
  163. def message_about_scripts_not_on_PATH(scripts):
  164. # type: (Sequence[str]) -> Optional[str]
  165. """Determine if any scripts are not on PATH and format a warning.
  166. Returns a warning message if one or more scripts are not on PATH,
  167. otherwise None.
  168. """
  169. if not scripts:
  170. return None
  171. # Group scripts by the path they were installed in
  172. grouped_by_dir = collections.defaultdict(set) # type: Dict[str, Set[str]]
  173. for destfile in scripts:
  174. parent_dir = os.path.dirname(destfile)
  175. script_name = os.path.basename(destfile)
  176. grouped_by_dir[parent_dir].add(script_name)
  177. # We don't want to warn for directories that are on PATH.
  178. not_warn_dirs = [
  179. os.path.normcase(i).rstrip(os.sep) for i in
  180. os.environ.get("PATH", "").split(os.pathsep)
  181. ]
  182. # If an executable sits with sys.executable, we don't warn for it.
  183. # This covers the case of venv invocations without activating the venv.
  184. not_warn_dirs.append(os.path.normcase(os.path.dirname(sys.executable)))
  185. warn_for = {
  186. parent_dir: scripts for parent_dir, scripts in grouped_by_dir.items()
  187. if os.path.normcase(parent_dir) not in not_warn_dirs
  188. } # type: Dict[str, Set[str]]
  189. if not warn_for:
  190. return None
  191. # Format a message
  192. msg_lines = []
  193. for parent_dir, dir_scripts in warn_for.items():
  194. sorted_scripts = sorted(dir_scripts) # type: List[str]
  195. if len(sorted_scripts) == 1:
  196. start_text = "script {} is".format(sorted_scripts[0])
  197. else:
  198. start_text = "scripts {} are".format(
  199. ", ".join(sorted_scripts[:-1]) + " and " + sorted_scripts[-1]
  200. )
  201. msg_lines.append(
  202. "The {} installed in '{}' which is not on PATH."
  203. .format(start_text, parent_dir)
  204. )
  205. last_line_fmt = (
  206. "Consider adding {} to PATH or, if you prefer "
  207. "to suppress this warning, use --no-warn-script-location."
  208. )
  209. if len(msg_lines) == 1:
  210. msg_lines.append(last_line_fmt.format("this directory"))
  211. else:
  212. msg_lines.append(last_line_fmt.format("these directories"))
  213. # Returns the formatted multiline message
  214. return "\n".join(msg_lines)
  215. def sorted_outrows(outrows):
  216. # type: (Iterable[InstalledCSVRow]) -> List[InstalledCSVRow]
  217. """
  218. Return the given rows of a RECORD file in sorted order.
  219. Each row is a 3-tuple (path, hash, size) and corresponds to a record of
  220. a RECORD file (see PEP 376 and PEP 427 for details). For the rows
  221. passed to this function, the size can be an integer as an int or string,
  222. or the empty string.
  223. """
  224. # Normally, there should only be one row per path, in which case the
  225. # second and third elements don't come into play when sorting.
  226. # However, in cases in the wild where a path might happen to occur twice,
  227. # we don't want the sort operation to trigger an error (but still want
  228. # determinism). Since the third element can be an int or string, we
  229. # coerce each element to a string to avoid a TypeError in this case.
  230. # For additional background, see--
  231. # https://github.com/pypa/pip/issues/5868
  232. return sorted(outrows, key=lambda row: tuple(str(x) for x in row))
  233. def get_csv_rows_for_installed(
  234. old_csv_rows, # type: Iterable[List[str]]
  235. installed, # type: Dict[str, str]
  236. changed, # type: set
  237. generated, # type: List[str]
  238. lib_dir, # type: str
  239. ):
  240. # type: (...) -> List[InstalledCSVRow]
  241. """
  242. :param installed: A map from archive RECORD path to installation RECORD
  243. path.
  244. """
  245. installed_rows = [] # type: List[InstalledCSVRow]
  246. for row in old_csv_rows:
  247. if len(row) > 3:
  248. logger.warning(
  249. 'RECORD line has more than three elements: {}'.format(row)
  250. )
  251. # Make a copy because we are mutating the row.
  252. row = list(row)
  253. old_path = row[0]
  254. new_path = installed.pop(old_path, old_path)
  255. row[0] = new_path
  256. if new_path in changed:
  257. digest, length = rehash(new_path)
  258. row[1] = digest
  259. row[2] = length
  260. installed_rows.append(tuple(row))
  261. for f in generated:
  262. digest, length = rehash(f)
  263. installed_rows.append((normpath(f, lib_dir), digest, str(length)))
  264. for f in installed:
  265. installed_rows.append((installed[f], '', ''))
  266. return installed_rows
  267. class MissingCallableSuffix(Exception):
  268. pass
  269. def _raise_for_invalid_entrypoint(specification):
  270. entry = get_export_entry(specification)
  271. if entry is not None and entry.suffix is None:
  272. raise MissingCallableSuffix(str(entry))
  273. class PipScriptMaker(ScriptMaker):
  274. def make(self, specification, options=None):
  275. _raise_for_invalid_entrypoint(specification)
  276. return super(PipScriptMaker, self).make(specification, options)
  277. def move_wheel_files(
  278. name, # type: str
  279. req, # type: Requirement
  280. wheeldir, # type: str
  281. user=False, # type: bool
  282. home=None, # type: Optional[str]
  283. root=None, # type: Optional[str]
  284. pycompile=True, # type: bool
  285. scheme=None, # type: Optional[Mapping[str, str]]
  286. isolated=False, # type: bool
  287. prefix=None, # type: Optional[str]
  288. warn_script_location=True # type: bool
  289. ):
  290. # type: (...) -> None
  291. """Install a wheel"""
  292. # TODO: Investigate and break this up.
  293. # TODO: Look into moving this into a dedicated class for representing an
  294. # installation.
  295. if not scheme:
  296. scheme = distutils_scheme(
  297. name, user=user, home=home, root=root, isolated=isolated,
  298. prefix=prefix,
  299. )
  300. if root_is_purelib(name, wheeldir):
  301. lib_dir = scheme['purelib']
  302. else:
  303. lib_dir = scheme['platlib']
  304. info_dir = [] # type: List[str]
  305. data_dirs = []
  306. source = wheeldir.rstrip(os.path.sep) + os.path.sep
  307. # Record details of the files moved
  308. # installed = files copied from the wheel to the destination
  309. # changed = files changed while installing (scripts #! line typically)
  310. # generated = files newly generated during the install (script wrappers)
  311. installed = {} # type: Dict[str, str]
  312. changed = set()
  313. generated = [] # type: List[str]
  314. # Compile all of the pyc files that we're going to be installing
  315. if pycompile:
  316. with captured_stdout() as stdout:
  317. with warnings.catch_warnings():
  318. warnings.filterwarnings('ignore')
  319. compileall.compile_dir(source, force=True, quiet=True)
  320. logger.debug(stdout.getvalue())
  321. def record_installed(srcfile, destfile, modified=False):
  322. """Map archive RECORD paths to installation RECORD paths."""
  323. oldpath = normpath(srcfile, wheeldir)
  324. newpath = normpath(destfile, lib_dir)
  325. installed[oldpath] = newpath
  326. if modified:
  327. changed.add(destfile)
  328. def clobber(source, dest, is_base, fixer=None, filter=None):
  329. ensure_dir(dest) # common for the 'include' path
  330. for dir, subdirs, files in os.walk(source):
  331. basedir = dir[len(source):].lstrip(os.path.sep)
  332. destdir = os.path.join(dest, basedir)
  333. if is_base and basedir.split(os.path.sep, 1)[0].endswith('.data'):
  334. continue
  335. for s in subdirs:
  336. destsubdir = os.path.join(dest, basedir, s)
  337. if is_base and basedir == '' and destsubdir.endswith('.data'):
  338. data_dirs.append(s)
  339. continue
  340. elif (is_base and
  341. s.endswith('.dist-info') and
  342. canonicalize_name(s).startswith(
  343. canonicalize_name(req.name))):
  344. assert not info_dir, ('Multiple .dist-info directories: ' +
  345. destsubdir + ', ' +
  346. ', '.join(info_dir))
  347. info_dir.append(destsubdir)
  348. for f in files:
  349. # Skip unwanted files
  350. if filter and filter(f):
  351. continue
  352. srcfile = os.path.join(dir, f)
  353. destfile = os.path.join(dest, basedir, f)
  354. # directory creation is lazy and after the file filtering above
  355. # to ensure we don't install empty dirs; empty dirs can't be
  356. # uninstalled.
  357. ensure_dir(destdir)
  358. # copyfile (called below) truncates the destination if it
  359. # exists and then writes the new contents. This is fine in most
  360. # cases, but can cause a segfault if pip has loaded a shared
  361. # object (e.g. from pyopenssl through its vendored urllib3)
  362. # Since the shared object is mmap'd an attempt to call a
  363. # symbol in it will then cause a segfault. Unlinking the file
  364. # allows writing of new contents while allowing the process to
  365. # continue to use the old copy.
  366. if os.path.exists(destfile):
  367. os.unlink(destfile)
  368. # We use copyfile (not move, copy, or copy2) to be extra sure
  369. # that we are not moving directories over (copyfile fails for
  370. # directories) as well as to ensure that we are not copying
  371. # over any metadata because we want more control over what
  372. # metadata we actually copy over.
  373. shutil.copyfile(srcfile, destfile)
  374. # Copy over the metadata for the file, currently this only
  375. # includes the atime and mtime.
  376. st = os.stat(srcfile)
  377. if hasattr(os, "utime"):
  378. os.utime(destfile, (st.st_atime, st.st_mtime))
  379. # If our file is executable, then make our destination file
  380. # executable.
  381. if os.access(srcfile, os.X_OK):
  382. st = os.stat(srcfile)
  383. permissions = (
  384. st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
  385. )
  386. os.chmod(destfile, permissions)
  387. changed = False
  388. if fixer:
  389. changed = fixer(destfile)
  390. record_installed(srcfile, destfile, changed)
  391. clobber(source, lib_dir, True)
  392. assert info_dir, "%s .dist-info directory not found" % req
  393. # Get the defined entry points
  394. ep_file = os.path.join(info_dir[0], 'entry_points.txt')
  395. console, gui = get_entrypoints(ep_file)
  396. def is_entrypoint_wrapper(name):
  397. # EP, EP.exe and EP-script.py are scripts generated for
  398. # entry point EP by setuptools
  399. if name.lower().endswith('.exe'):
  400. matchname = name[:-4]
  401. elif name.lower().endswith('-script.py'):
  402. matchname = name[:-10]
  403. elif name.lower().endswith(".pya"):
  404. matchname = name[:-4]
  405. else:
  406. matchname = name
  407. # Ignore setuptools-generated scripts
  408. return (matchname in console or matchname in gui)
  409. for datadir in data_dirs:
  410. fixer = None
  411. filter = None
  412. for subdir in os.listdir(os.path.join(wheeldir, datadir)):
  413. fixer = None
  414. if subdir == 'scripts':
  415. fixer = fix_script
  416. filter = is_entrypoint_wrapper
  417. source = os.path.join(wheeldir, datadir, subdir)
  418. dest = scheme[subdir]
  419. clobber(source, dest, False, fixer=fixer, filter=filter)
  420. maker = PipScriptMaker(None, scheme['scripts'])
  421. # Ensure old scripts are overwritten.
  422. # See https://github.com/pypa/pip/issues/1800
  423. maker.clobber = True
  424. # Ensure we don't generate any variants for scripts because this is almost
  425. # never what somebody wants.
  426. # See https://bitbucket.org/pypa/distlib/issue/35/
  427. maker.variants = {''}
  428. # This is required because otherwise distlib creates scripts that are not
  429. # executable.
  430. # See https://bitbucket.org/pypa/distlib/issue/32/
  431. maker.set_mode = True
  432. scripts_to_generate = []
  433. # Special case pip and setuptools to generate versioned wrappers
  434. #
  435. # The issue is that some projects (specifically, pip and setuptools) use
  436. # code in setup.py to create "versioned" entry points - pip2.7 on Python
  437. # 2.7, pip3.3 on Python 3.3, etc. But these entry points are baked into
  438. # the wheel metadata at build time, and so if the wheel is installed with
  439. # a *different* version of Python the entry points will be wrong. The
  440. # correct fix for this is to enhance the metadata to be able to describe
  441. # such versioned entry points, but that won't happen till Metadata 2.0 is
  442. # available.
  443. # In the meantime, projects using versioned entry points will either have
  444. # incorrect versioned entry points, or they will not be able to distribute
  445. # "universal" wheels (i.e., they will need a wheel per Python version).
  446. #
  447. # Because setuptools and pip are bundled with _ensurepip and virtualenv,
  448. # we need to use universal wheels. So, as a stopgap until Metadata 2.0, we
  449. # override the versioned entry points in the wheel and generate the
  450. # correct ones. This code is purely a short-term measure until Metadata 2.0
  451. # is available.
  452. #
  453. # To add the level of hack in this section of code, in order to support
  454. # ensurepip this code will look for an ``ENSUREPIP_OPTIONS`` environment
  455. # variable which will control which version scripts get installed.
  456. #
  457. # ENSUREPIP_OPTIONS=altinstall
  458. # - Only pipX.Y and easy_install-X.Y will be generated and installed
  459. # ENSUREPIP_OPTIONS=install
  460. # - pipX.Y, pipX, easy_install-X.Y will be generated and installed. Note
  461. # that this option is technically if ENSUREPIP_OPTIONS is set and is
  462. # not altinstall
  463. # DEFAULT
  464. # - The default behavior is to install pip, pipX, pipX.Y, easy_install
  465. # and easy_install-X.Y.
  466. pip_script = console.pop('pip', None)
  467. if pip_script:
  468. if "ENSUREPIP_OPTIONS" not in os.environ:
  469. scripts_to_generate.append('pip = ' + pip_script)
  470. if os.environ.get("ENSUREPIP_OPTIONS", "") != "altinstall":
  471. scripts_to_generate.append(
  472. 'pip%s = %s' % (sys.version_info[0], pip_script)
  473. )
  474. scripts_to_generate.append(
  475. 'pip%s = %s' % (get_major_minor_version(), pip_script)
  476. )
  477. # Delete any other versioned pip entry points
  478. pip_ep = [k for k in console if re.match(r'pip(\d(\.\d)?)?$', k)]
  479. for k in pip_ep:
  480. del console[k]
  481. easy_install_script = console.pop('easy_install', None)
  482. if easy_install_script:
  483. if "ENSUREPIP_OPTIONS" not in os.environ:
  484. scripts_to_generate.append(
  485. 'easy_install = ' + easy_install_script
  486. )
  487. scripts_to_generate.append(
  488. 'easy_install-%s = %s' % (
  489. get_major_minor_version(), easy_install_script
  490. )
  491. )
  492. # Delete any other versioned easy_install entry points
  493. easy_install_ep = [
  494. k for k in console if re.match(r'easy_install(-\d\.\d)?$', k)
  495. ]
  496. for k in easy_install_ep:
  497. del console[k]
  498. # Generate the console and GUI entry points specified in the wheel
  499. scripts_to_generate.extend(
  500. '%s = %s' % kv for kv in console.items()
  501. )
  502. gui_scripts_to_generate = [
  503. '%s = %s' % kv for kv in gui.items()
  504. ]
  505. generated_console_scripts = [] # type: List[str]
  506. try:
  507. generated_console_scripts = maker.make_multiple(scripts_to_generate)
  508. generated.extend(generated_console_scripts)
  509. generated.extend(
  510. maker.make_multiple(gui_scripts_to_generate, {'gui': True})
  511. )
  512. except MissingCallableSuffix as e:
  513. entry = e.args[0]
  514. raise InstallationError(
  515. "Invalid script entry point: {} for req: {} - A callable "
  516. "suffix is required. Cf https://packaging.python.org/en/"
  517. "latest/distributing.html#console-scripts for more "
  518. "information.".format(entry, req)
  519. )
  520. if warn_script_location:
  521. msg = message_about_scripts_not_on_PATH(generated_console_scripts)
  522. if msg is not None:
  523. logger.warning(msg)
  524. # Record pip as the installer
  525. installer = os.path.join(info_dir[0], 'INSTALLER')
  526. temp_installer = os.path.join(info_dir[0], 'INSTALLER.pip')
  527. with open(temp_installer, 'wb') as installer_file:
  528. installer_file.write(b'pip\n')
  529. shutil.move(temp_installer, installer)
  530. generated.append(installer)
  531. # Record details of all files installed
  532. record = os.path.join(info_dir[0], 'RECORD')
  533. temp_record = os.path.join(info_dir[0], 'RECORD.pip')
  534. with open_for_csv(record, 'r') as record_in:
  535. with open_for_csv(temp_record, 'w+') as record_out:
  536. reader = csv.reader(record_in)
  537. outrows = get_csv_rows_for_installed(
  538. reader, installed=installed, changed=changed,
  539. generated=generated, lib_dir=lib_dir,
  540. )
  541. writer = csv.writer(record_out)
  542. # Sort to simplify testing.
  543. for row in sorted_outrows(outrows):
  544. writer.writerow(row)
  545. shutil.move(temp_record, record)
  546. def wheel_version(source_dir):
  547. # type: (Optional[str]) -> Optional[Tuple[int, ...]]
  548. """
  549. Return the Wheel-Version of an extracted wheel, if possible.
  550. Otherwise, return None if we couldn't parse / extract it.
  551. """
  552. try:
  553. dist = [d for d in pkg_resources.find_on_path(None, source_dir)][0]
  554. wheel_data = dist.get_metadata('WHEEL')
  555. wheel_data = Parser().parsestr(wheel_data)
  556. version = wheel_data['Wheel-Version'].strip()
  557. version = tuple(map(int, version.split('.')))
  558. return version
  559. except Exception:
  560. return None
  561. def check_compatibility(version, name):
  562. # type: (Optional[Tuple[int, ...]], str) -> None
  563. """
  564. Raises errors or warns if called with an incompatible Wheel-Version.
  565. Pip should refuse to install a Wheel-Version that's a major series
  566. ahead of what it's compatible with (e.g 2.0 > 1.1); and warn when
  567. installing a version only minor version ahead (e.g 1.2 > 1.1).
  568. version: a 2-tuple representing a Wheel-Version (Major, Minor)
  569. name: name of wheel or package to raise exception about
  570. :raises UnsupportedWheel: when an incompatible Wheel-Version is given
  571. """
  572. if not version:
  573. raise UnsupportedWheel(
  574. "%s is in an unsupported or invalid wheel" % name
  575. )
  576. if version[0] > VERSION_COMPATIBLE[0]:
  577. raise UnsupportedWheel(
  578. "%s's Wheel-Version (%s) is not compatible with this version "
  579. "of pip" % (name, '.'.join(map(str, version)))
  580. )
  581. elif version > VERSION_COMPATIBLE:
  582. logger.warning(
  583. 'Installing from a newer Wheel-Version (%s)',
  584. '.'.join(map(str, version)),
  585. )
  586. def format_tag(file_tag):
  587. # type: (Tuple[str, ...]) -> str
  588. """
  589. Format three tags in the form "<python_tag>-<abi_tag>-<platform_tag>".
  590. :param file_tag: A 3-tuple of tags (python_tag, abi_tag, platform_tag).
  591. """
  592. return '-'.join(file_tag)
  593. class Wheel(object):
  594. """A wheel file"""
  595. # TODO: Maybe move the class into the models sub-package
  596. # TODO: Maybe move the install code into this class
  597. wheel_file_re = re.compile(
  598. r"""^(?P<namever>(?P<name>.+?)-(?P<ver>.*?))
  599. ((-(?P<build>\d[^-]*?))?-(?P<pyver>.+?)-(?P<abi>.+?)-(?P<plat>.+?)
  600. \.whl|\.dist-info)$""",
  601. re.VERBOSE
  602. )
  603. def __init__(self, filename):
  604. # type: (str) -> None
  605. """
  606. :raises InvalidWheelFilename: when the filename is invalid for a wheel
  607. """
  608. wheel_info = self.wheel_file_re.match(filename)
  609. if not wheel_info:
  610. raise InvalidWheelFilename(
  611. "%s is not a valid wheel filename." % filename
  612. )
  613. self.filename = filename
  614. self.name = wheel_info.group('name').replace('_', '-')
  615. # we'll assume "_" means "-" due to wheel naming scheme
  616. # (https://github.com/pypa/pip/issues/1150)
  617. self.version = wheel_info.group('ver').replace('_', '-')
  618. self.build_tag = wheel_info.group('build')
  619. self.pyversions = wheel_info.group('pyver').split('.')
  620. self.abis = wheel_info.group('abi').split('.')
  621. self.plats = wheel_info.group('plat').split('.')
  622. # All the tag combinations from this file
  623. self.file_tags = {
  624. (x, y, z) for x in self.pyversions
  625. for y in self.abis for z in self.plats
  626. }
  627. def get_formatted_file_tags(self):
  628. # type: () -> List[str]
  629. """
  630. Return the wheel's tags as a sorted list of strings.
  631. """
  632. return sorted(format_tag(tag) for tag in self.file_tags)
  633. def support_index_min(self, tags):
  634. # type: (List[Pep425Tag]) -> int
  635. """
  636. Return the lowest index that one of the wheel's file_tag combinations
  637. achieves in the given list of supported tags.
  638. For example, if there are 8 supported tags and one of the file tags
  639. is first in the list, then return 0.
  640. :param tags: the PEP 425 tags to check the wheel against, in order
  641. with most preferred first.
  642. :raises ValueError: If none of the wheel's file tags match one of
  643. the supported tags.
  644. """
  645. return min(tags.index(tag) for tag in self.file_tags if tag in tags)
  646. def supported(self, tags):
  647. # type: (List[Pep425Tag]) -> bool
  648. """
  649. Return whether the wheel is compatible with one of the given tags.
  650. :param tags: the PEP 425 tags to check the wheel against.
  651. """
  652. return not self.file_tags.isdisjoint(tags)
  653. def _contains_egg_info(
  654. s, _egg_info_re=re.compile(r'([a-z0-9_.]+)-([a-z0-9_.!+-]+)', re.I)):
  655. """Determine whether the string looks like an egg_info.
  656. :param s: The string to parse. E.g. foo-2.1
  657. """
  658. return bool(_egg_info_re.search(s))
  659. def should_use_ephemeral_cache(
  660. req, # type: InstallRequirement
  661. should_unpack, # type: bool
  662. cache_available, # type: bool
  663. check_binary_allowed, # type: BinaryAllowedPredicate
  664. ):
  665. # type: (...) -> Optional[bool]
  666. """
  667. Return whether to build an InstallRequirement object using the
  668. ephemeral cache.
  669. :param cache_available: whether a cache directory is available for the
  670. should_unpack=True case.
  671. :return: True or False to build the requirement with ephem_cache=True
  672. or False, respectively; or None not to build the requirement.
  673. """
  674. if req.constraint:
  675. # never build requirements that are merely constraints
  676. return None
  677. if req.is_wheel:
  678. if not should_unpack:
  679. logger.info(
  680. 'Skipping %s, due to already being wheel.', req.name,
  681. )
  682. return None
  683. if not should_unpack:
  684. # i.e. pip wheel, not pip install;
  685. # return False, knowing that the caller will never cache
  686. # in this case anyway, so this return merely means "build it".
  687. # TODO improve this behavior
  688. return False
  689. if req.editable or not req.source_dir:
  690. return None
  691. if not check_binary_allowed(req):
  692. logger.info(
  693. "Skipping wheel build for %s, due to binaries "
  694. "being disabled for it.", req.name,
  695. )
  696. return None
  697. if req.link and req.link.is_vcs:
  698. # VCS checkout. Build wheel just for this run.
  699. return True
  700. link = req.link
  701. base, ext = link.splitext()
  702. if cache_available and _contains_egg_info(base):
  703. return False
  704. # Otherwise, build the wheel just for this run using the ephemeral
  705. # cache since we are either in the case of e.g. a local directory, or
  706. # no cache directory is available to use.
  707. return True
  708. def format_command_result(
  709. command_args, # type: List[str]
  710. command_output, # type: str
  711. ):
  712. # type: (...) -> str
  713. """
  714. Format command information for logging.
  715. """
  716. command_desc = format_command_args(command_args)
  717. text = 'Command arguments: {}\n'.format(command_desc)
  718. if not command_output:
  719. text += 'Command output: None'
  720. elif logger.getEffectiveLevel() > logging.DEBUG:
  721. text += 'Command output: [use --verbose to show]'
  722. else:
  723. if not command_output.endswith('\n'):
  724. command_output += '\n'
  725. text += 'Command output:\n{}{}'.format(command_output, LOG_DIVIDER)
  726. return text
  727. def get_legacy_build_wheel_path(
  728. names, # type: List[str]
  729. temp_dir, # type: str
  730. req, # type: InstallRequirement
  731. command_args, # type: List[str]
  732. command_output, # type: str
  733. ):
  734. # type: (...) -> Optional[str]
  735. """
  736. Return the path to the wheel in the temporary build directory.
  737. """
  738. # Sort for determinism.
  739. names = sorted(names)
  740. if not names:
  741. msg = (
  742. 'Legacy build of wheel for {!r} created no files.\n'
  743. ).format(req.name)
  744. msg += format_command_result(command_args, command_output)
  745. logger.warning(msg)
  746. return None
  747. if len(names) > 1:
  748. msg = (
  749. 'Legacy build of wheel for {!r} created more than one file.\n'
  750. 'Filenames (choosing first): {}\n'
  751. ).format(req.name, names)
  752. msg += format_command_result(command_args, command_output)
  753. logger.warning(msg)
  754. return os.path.join(temp_dir, names[0])
  755. def _always_true(_):
  756. return True
  757. class WheelBuilder(object):
  758. """Build wheels from a RequirementSet."""
  759. def __init__(
  760. self,
  761. preparer, # type: RequirementPreparer
  762. wheel_cache, # type: WheelCache
  763. build_options=None, # type: Optional[List[str]]
  764. global_options=None, # type: Optional[List[str]]
  765. check_binary_allowed=None, # type: Optional[BinaryAllowedPredicate]
  766. no_clean=False # type: bool
  767. ):
  768. # type: (...) -> None
  769. if check_binary_allowed is None:
  770. # Binaries allowed by default.
  771. check_binary_allowed = _always_true
  772. self.preparer = preparer
  773. self.wheel_cache = wheel_cache
  774. self._wheel_dir = preparer.wheel_download_dir
  775. self.build_options = build_options or []
  776. self.global_options = global_options or []
  777. self.check_binary_allowed = check_binary_allowed
  778. self.no_clean = no_clean
  779. def _build_one(self, req, output_dir, python_tag=None):
  780. """Build one wheel.
  781. :return: The filename of the built wheel, or None if the build failed.
  782. """
  783. # Install build deps into temporary directory (PEP 518)
  784. with req.build_env:
  785. return self._build_one_inside_env(req, output_dir,
  786. python_tag=python_tag)
  787. def _build_one_inside_env(self, req, output_dir, python_tag=None):
  788. with TempDirectory(kind="wheel") as temp_dir:
  789. if req.use_pep517:
  790. builder = self._build_one_pep517
  791. else:
  792. builder = self._build_one_legacy
  793. wheel_path = builder(req, temp_dir.path, python_tag=python_tag)
  794. if wheel_path is not None:
  795. wheel_name = os.path.basename(wheel_path)
  796. dest_path = os.path.join(output_dir, wheel_name)
  797. try:
  798. wheel_hash, length = hash_file(wheel_path)
  799. shutil.move(wheel_path, dest_path)
  800. logger.info('Created wheel for %s: '
  801. 'filename=%s size=%d sha256=%s',
  802. req.name, wheel_name, length,
  803. wheel_hash.hexdigest())
  804. logger.info('Stored in directory: %s', output_dir)
  805. return dest_path
  806. except Exception:
  807. pass
  808. # Ignore return, we can't do anything else useful.
  809. self._clean_one(req)
  810. return None
  811. def _base_setup_args(self, req):
  812. # NOTE: Eventually, we'd want to also -S to the flags here, when we're
  813. # isolating. Currently, it breaks Python in virtualenvs, because it
  814. # relies on site.py to find parts of the standard library outside the
  815. # virtualenv.
  816. return make_setuptools_shim_args(
  817. req.setup_py_path,
  818. global_options=self.global_options,
  819. unbuffered_output=True
  820. )
  821. def _build_one_pep517(self, req, tempd, python_tag=None):
  822. """Build one InstallRequirement using the PEP 517 build process.
  823. Returns path to wheel if successfully built. Otherwise, returns None.
  824. """
  825. assert req.metadata_directory is not None
  826. if self.build_options:
  827. # PEP 517 does not support --build-options
  828. logger.error('Cannot build wheel for %s using PEP 517 when '
  829. '--build-options is present' % (req.name,))
  830. return None
  831. try:
  832. logger.debug('Destination directory: %s', tempd)
  833. runner = runner_with_spinner_message(
  834. 'Building wheel for {} (PEP 517)'.format(req.name)
  835. )
  836. backend = req.pep517_backend
  837. with backend.subprocess_runner(runner):
  838. wheel_name = backend.build_wheel(
  839. tempd,
  840. metadata_directory=req.metadata_directory,
  841. )
  842. if python_tag:
  843. # General PEP 517 backends don't necessarily support
  844. # a "--python-tag" option, so we rename the wheel
  845. # file directly.
  846. new_name = replace_python_tag(wheel_name, python_tag)
  847. os.rename(
  848. os.path.join(tempd, wheel_name),
  849. os.path.join(tempd, new_name)
  850. )
  851. # Reassign to simplify the return at the end of function
  852. wheel_name = new_name
  853. except Exception:
  854. logger.error('Failed building wheel for %s', req.name)
  855. return None
  856. return os.path.join(tempd, wheel_name)
  857. def _build_one_legacy(self, req, tempd, python_tag=None):
  858. """Build one InstallRequirement using the "legacy" build process.
  859. Returns path to wheel if successfully built. Otherwise, returns None.
  860. """
  861. base_args = self._base_setup_args(req)
  862. spin_message = 'Building wheel for %s (setup.py)' % (req.name,)
  863. with open_spinner(spin_message) as spinner:
  864. logger.debug('Destination directory: %s', tempd)
  865. wheel_args = base_args + ['bdist_wheel', '-d', tempd] \
  866. + self.build_options
  867. if python_tag is not None:
  868. wheel_args += ["--python-tag", python_tag]
  869. try:
  870. output = call_subprocess(
  871. wheel_args,
  872. cwd=req.unpacked_source_directory,
  873. spinner=spinner,
  874. )
  875. except Exception:
  876. spinner.finish("error")
  877. logger.error('Failed building wheel for %s', req.name)
  878. return None
  879. names = os.listdir(tempd)
  880. wheel_path = get_legacy_build_wheel_path(
  881. names=names,
  882. temp_dir=tempd,
  883. req=req,
  884. command_args=wheel_args,
  885. command_output=output,
  886. )
  887. return wheel_path
  888. def _clean_one(self, req):
  889. base_args = self._base_setup_args(req)
  890. logger.info('Running setup.py clean for %s', req.name)
  891. clean_args = base_args + ['clean', '--all']
  892. try:
  893. call_subprocess(clean_args, cwd=req.source_dir)
  894. return True
  895. except Exception:
  896. logger.error('Failed cleaning build dir for %s', req.name)
  897. return False
  898. def build(
  899. self,
  900. requirements, # type: Iterable[InstallRequirement]
  901. should_unpack=False # type: bool
  902. ):
  903. # type: (...) -> List[InstallRequirement]
  904. """Build wheels.
  905. :param should_unpack: If True, after building the wheel, unpack it
  906. and replace the sdist with the unpacked version in preparation
  907. for installation.
  908. :return: True if all the wheels built correctly.
  909. """
  910. # pip install uses should_unpack=True.
  911. # pip install never provides a _wheel_dir.
  912. # pip wheel uses should_unpack=False.
  913. # pip wheel always provides a _wheel_dir (via the preparer).
  914. assert (
  915. (should_unpack and not self._wheel_dir) or
  916. (not should_unpack and self._wheel_dir)
  917. )
  918. buildset = []
  919. cache_available = bool(self.wheel_cache.cache_dir)
  920. for req in requirements:
  921. ephem_cache = should_use_ephemeral_cache(
  922. req,
  923. should_unpack=should_unpack,
  924. cache_available=cache_available,
  925. check_binary_allowed=self.check_binary_allowed,
  926. )
  927. if ephem_cache is None:
  928. continue
  929. # Determine where the wheel should go.
  930. if should_unpack:
  931. if ephem_cache:
  932. output_dir = self.wheel_cache.get_ephem_path_for_link(
  933. req.link
  934. )
  935. else:
  936. output_dir = self.wheel_cache.get_path_for_link(req.link)
  937. else:
  938. output_dir = self._wheel_dir
  939. buildset.append((req, output_dir))
  940. if not buildset:
  941. return []
  942. # TODO by @pradyunsg
  943. # Should break up this method into 2 separate methods.
  944. # Build the wheels.
  945. logger.info(
  946. 'Building wheels for collected packages: %s',
  947. ', '.join([req.name for (req, _) in buildset]),
  948. )
  949. python_tag = None
  950. if should_unpack:
  951. python_tag = pep425tags.implementation_tag
  952. with indent_log():
  953. build_success, build_failure = [], []
  954. for req, output_dir in buildset:
  955. try:
  956. ensure_dir(output_dir)
  957. except OSError as e:
  958. logger.warning(
  959. "Building wheel for %s failed: %s",
  960. req.name, e,
  961. )
  962. build_failure.append(req)
  963. continue
  964. wheel_file = self._build_one(
  965. req, output_dir,
  966. python_tag=python_tag,
  967. )
  968. if wheel_file:
  969. build_success.append(req)
  970. if should_unpack:
  971. # XXX: This is mildly duplicative with prepare_files,
  972. # but not close enough to pull out to a single common
  973. # method.
  974. # The code below assumes temporary source dirs -
  975. # prevent it doing bad things.
  976. if (
  977. req.source_dir and
  978. not has_delete_marker_file(req.source_dir)
  979. ):
  980. raise AssertionError(
  981. "bad source dir - missing marker")
  982. # Delete the source we built the wheel from
  983. req.remove_temporary_source()
  984. # set the build directory again - name is known from
  985. # the work prepare_files did.
  986. req.source_dir = req.ensure_build_location(
  987. self.preparer.build_dir
  988. )
  989. # Update the link for this.
  990. req.link = Link(path_to_url(wheel_file))
  991. assert req.link.is_wheel
  992. # extract the wheel into the dir
  993. unpack_file(req.link.file_path, req.source_dir)
  994. else:
  995. build_failure.append(req)
  996. # notify success/failure
  997. if build_success:
  998. logger.info(
  999. 'Successfully built %s',
  1000. ' '.join([req.name for req in build_success]),
  1001. )
  1002. if build_failure:
  1003. logger.info(
  1004. 'Failed to build %s',
  1005. ' '.join([req.name for req in build_failure]),
  1006. )
  1007. # Return a list of requirements that failed to build
  1008. return build_failure