git.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372
  1. # The following comment should be removed at some point in the future.
  2. # mypy: disallow-untyped-defs=False
  3. from __future__ import absolute_import
  4. import logging
  5. import os.path
  6. import re
  7. from pip._vendor.packaging.version import parse as parse_version
  8. from pip._vendor.six.moves.urllib import parse as urllib_parse
  9. from pip._vendor.six.moves.urllib import request as urllib_request
  10. from pip._internal.exceptions import BadCommand
  11. from pip._internal.utils.misc import display_path
  12. from pip._internal.utils.subprocess import make_command
  13. from pip._internal.utils.temp_dir import TempDirectory
  14. from pip._internal.utils.typing import MYPY_CHECK_RUNNING
  15. from pip._internal.vcs.versioncontrol import (
  16. RemoteNotFoundError,
  17. VersionControl,
  18. find_path_to_setup_from_repo_root,
  19. vcs,
  20. )
  21. if MYPY_CHECK_RUNNING:
  22. from typing import Optional, Tuple
  23. from pip._internal.utils.misc import HiddenText
  24. from pip._internal.vcs.versioncontrol import AuthInfo, RevOptions
  25. urlsplit = urllib_parse.urlsplit
  26. urlunsplit = urllib_parse.urlunsplit
  27. logger = logging.getLogger(__name__)
  28. HASH_REGEX = re.compile('^[a-fA-F0-9]{40}$')
  29. def looks_like_hash(sha):
  30. return bool(HASH_REGEX.match(sha))
  31. class Git(VersionControl):
  32. name = 'git'
  33. dirname = '.git'
  34. repo_name = 'clone'
  35. schemes = (
  36. 'git', 'git+http', 'git+https', 'git+ssh', 'git+git', 'git+file',
  37. )
  38. # Prevent the user's environment variables from interfering with pip:
  39. # https://github.com/pypa/pip/issues/1130
  40. unset_environ = ('GIT_DIR', 'GIT_WORK_TREE')
  41. default_arg_rev = 'HEAD'
  42. @staticmethod
  43. def get_base_rev_args(rev):
  44. return [rev]
  45. def get_git_version(self):
  46. VERSION_PFX = 'git version '
  47. version = self.run_command(['version'], show_stdout=False)
  48. if version.startswith(VERSION_PFX):
  49. version = version[len(VERSION_PFX):].split()[0]
  50. else:
  51. version = ''
  52. # get first 3 positions of the git version because
  53. # on windows it is x.y.z.windows.t, and this parses as
  54. # LegacyVersion which always smaller than a Version.
  55. version = '.'.join(version.split('.')[:3])
  56. return parse_version(version)
  57. @classmethod
  58. def get_current_branch(cls, location):
  59. """
  60. Return the current branch, or None if HEAD isn't at a branch
  61. (e.g. detached HEAD).
  62. """
  63. # git-symbolic-ref exits with empty stdout if "HEAD" is a detached
  64. # HEAD rather than a symbolic ref. In addition, the -q causes the
  65. # command to exit with status code 1 instead of 128 in this case
  66. # and to suppress the message to stderr.
  67. args = ['symbolic-ref', '-q', 'HEAD']
  68. output = cls.run_command(
  69. args, extra_ok_returncodes=(1, ), show_stdout=False, cwd=location,
  70. )
  71. ref = output.strip()
  72. if ref.startswith('refs/heads/'):
  73. return ref[len('refs/heads/'):]
  74. return None
  75. def export(self, location, url):
  76. # type: (str, HiddenText) -> None
  77. """Export the Git repository at the url to the destination location"""
  78. if not location.endswith('/'):
  79. location = location + '/'
  80. with TempDirectory(kind="export") as temp_dir:
  81. self.unpack(temp_dir.path, url=url)
  82. self.run_command(
  83. ['checkout-index', '-a', '-f', '--prefix', location],
  84. show_stdout=False, cwd=temp_dir.path
  85. )
  86. @classmethod
  87. def get_revision_sha(cls, dest, rev):
  88. """
  89. Return (sha_or_none, is_branch), where sha_or_none is a commit hash
  90. if the revision names a remote branch or tag, otherwise None.
  91. Args:
  92. dest: the repository directory.
  93. rev: the revision name.
  94. """
  95. # Pass rev to pre-filter the list.
  96. output = cls.run_command(['show-ref', rev], cwd=dest,
  97. show_stdout=False, on_returncode='ignore')
  98. refs = {}
  99. for line in output.strip().splitlines():
  100. try:
  101. sha, ref = line.split()
  102. except ValueError:
  103. # Include the offending line to simplify troubleshooting if
  104. # this error ever occurs.
  105. raise ValueError('unexpected show-ref line: {!r}'.format(line))
  106. refs[ref] = sha
  107. branch_ref = 'refs/remotes/origin/{}'.format(rev)
  108. tag_ref = 'refs/tags/{}'.format(rev)
  109. sha = refs.get(branch_ref)
  110. if sha is not None:
  111. return (sha, True)
  112. sha = refs.get(tag_ref)
  113. return (sha, False)
  114. @classmethod
  115. def resolve_revision(cls, dest, url, rev_options):
  116. # type: (str, HiddenText, RevOptions) -> RevOptions
  117. """
  118. Resolve a revision to a new RevOptions object with the SHA1 of the
  119. branch, tag, or ref if found.
  120. Args:
  121. rev_options: a RevOptions object.
  122. """
  123. rev = rev_options.arg_rev
  124. # The arg_rev property's implementation for Git ensures that the
  125. # rev return value is always non-None.
  126. assert rev is not None
  127. sha, is_branch = cls.get_revision_sha(dest, rev)
  128. if sha is not None:
  129. rev_options = rev_options.make_new(sha)
  130. rev_options.branch_name = rev if is_branch else None
  131. return rev_options
  132. # Do not show a warning for the common case of something that has
  133. # the form of a Git commit hash.
  134. if not looks_like_hash(rev):
  135. logger.warning(
  136. "Did not find branch or tag '%s', assuming revision or ref.",
  137. rev,
  138. )
  139. if not rev.startswith('refs/'):
  140. return rev_options
  141. # If it looks like a ref, we have to fetch it explicitly.
  142. cls.run_command(
  143. make_command('fetch', '-q', url, rev_options.to_args()),
  144. cwd=dest,
  145. )
  146. # Change the revision to the SHA of the ref we fetched
  147. sha = cls.get_revision(dest, rev='FETCH_HEAD')
  148. rev_options = rev_options.make_new(sha)
  149. return rev_options
  150. @classmethod
  151. def is_commit_id_equal(cls, dest, name):
  152. """
  153. Return whether the current commit hash equals the given name.
  154. Args:
  155. dest: the repository directory.
  156. name: a string name.
  157. """
  158. if not name:
  159. # Then avoid an unnecessary subprocess call.
  160. return False
  161. return cls.get_revision(dest) == name
  162. def fetch_new(self, dest, url, rev_options):
  163. # type: (str, HiddenText, RevOptions) -> None
  164. rev_display = rev_options.to_display()
  165. logger.info('Cloning %s%s to %s', url, rev_display, display_path(dest))
  166. self.run_command(make_command('clone', '-q', url, dest))
  167. if rev_options.rev:
  168. # Then a specific revision was requested.
  169. rev_options = self.resolve_revision(dest, url, rev_options)
  170. branch_name = getattr(rev_options, 'branch_name', None)
  171. if branch_name is None:
  172. # Only do a checkout if the current commit id doesn't match
  173. # the requested revision.
  174. if not self.is_commit_id_equal(dest, rev_options.rev):
  175. cmd_args = make_command(
  176. 'checkout', '-q', rev_options.to_args(),
  177. )
  178. self.run_command(cmd_args, cwd=dest)
  179. elif self.get_current_branch(dest) != branch_name:
  180. # Then a specific branch was requested, and that branch
  181. # is not yet checked out.
  182. track_branch = 'origin/{}'.format(branch_name)
  183. cmd_args = [
  184. 'checkout', '-b', branch_name, '--track', track_branch,
  185. ]
  186. self.run_command(cmd_args, cwd=dest)
  187. #: repo may contain submodules
  188. self.update_submodules(dest)
  189. def switch(self, dest, url, rev_options):
  190. # type: (str, HiddenText, RevOptions) -> None
  191. self.run_command(
  192. make_command('config', 'remote.origin.url', url),
  193. cwd=dest,
  194. )
  195. cmd_args = make_command('checkout', '-q', rev_options.to_args())
  196. self.run_command(cmd_args, cwd=dest)
  197. self.update_submodules(dest)
  198. def update(self, dest, url, rev_options):
  199. # type: (str, HiddenText, RevOptions) -> None
  200. # First fetch changes from the default remote
  201. if self.get_git_version() >= parse_version('1.9.0'):
  202. # fetch tags in addition to everything else
  203. self.run_command(['fetch', '-q', '--tags'], cwd=dest)
  204. else:
  205. self.run_command(['fetch', '-q'], cwd=dest)
  206. # Then reset to wanted revision (maybe even origin/master)
  207. rev_options = self.resolve_revision(dest, url, rev_options)
  208. cmd_args = make_command('reset', '--hard', '-q', rev_options.to_args())
  209. self.run_command(cmd_args, cwd=dest)
  210. #: update submodules
  211. self.update_submodules(dest)
  212. @classmethod
  213. def get_remote_url(cls, location):
  214. """
  215. Return URL of the first remote encountered.
  216. Raises RemoteNotFoundError if the repository does not have a remote
  217. url configured.
  218. """
  219. # We need to pass 1 for extra_ok_returncodes since the command
  220. # exits with return code 1 if there are no matching lines.
  221. stdout = cls.run_command(
  222. ['config', '--get-regexp', r'remote\..*\.url'],
  223. extra_ok_returncodes=(1, ), show_stdout=False, cwd=location,
  224. )
  225. remotes = stdout.splitlines()
  226. try:
  227. found_remote = remotes[0]
  228. except IndexError:
  229. raise RemoteNotFoundError
  230. for remote in remotes:
  231. if remote.startswith('remote.origin.url '):
  232. found_remote = remote
  233. break
  234. url = found_remote.split(' ')[1]
  235. return url.strip()
  236. @classmethod
  237. def get_revision(cls, location, rev=None):
  238. if rev is None:
  239. rev = 'HEAD'
  240. current_rev = cls.run_command(
  241. ['rev-parse', rev], show_stdout=False, cwd=location,
  242. )
  243. return current_rev.strip()
  244. @classmethod
  245. def get_subdirectory(cls, location):
  246. """
  247. Return the path to setup.py, relative to the repo root.
  248. Return None if setup.py is in the repo root.
  249. """
  250. # find the repo root
  251. git_dir = cls.run_command(
  252. ['rev-parse', '--git-dir'],
  253. show_stdout=False, cwd=location).strip()
  254. if not os.path.isabs(git_dir):
  255. git_dir = os.path.join(location, git_dir)
  256. repo_root = os.path.abspath(os.path.join(git_dir, '..'))
  257. return find_path_to_setup_from_repo_root(location, repo_root)
  258. @classmethod
  259. def get_url_rev_and_auth(cls, url):
  260. # type: (str) -> Tuple[str, Optional[str], AuthInfo]
  261. """
  262. Prefixes stub URLs like 'user@hostname:user/repo.git' with 'ssh://'.
  263. That's required because although they use SSH they sometimes don't
  264. work with a ssh:// scheme (e.g. GitHub). But we need a scheme for
  265. parsing. Hence we remove it again afterwards and return it as a stub.
  266. """
  267. # Works around an apparent Git bug
  268. # (see https://article.gmane.org/gmane.comp.version-control.git/146500)
  269. scheme, netloc, path, query, fragment = urlsplit(url)
  270. if scheme.endswith('file'):
  271. initial_slashes = path[:-len(path.lstrip('/'))]
  272. newpath = (
  273. initial_slashes +
  274. urllib_request.url2pathname(path)
  275. .replace('\\', '/').lstrip('/')
  276. )
  277. url = urlunsplit((scheme, netloc, newpath, query, fragment))
  278. after_plus = scheme.find('+') + 1
  279. url = scheme[:after_plus] + urlunsplit(
  280. (scheme[after_plus:], netloc, newpath, query, fragment),
  281. )
  282. if '://' not in url:
  283. assert 'file:' not in url
  284. url = url.replace('git+', 'git+ssh://')
  285. url, rev, user_pass = super(Git, cls).get_url_rev_and_auth(url)
  286. url = url.replace('ssh://', '')
  287. else:
  288. url, rev, user_pass = super(Git, cls).get_url_rev_and_auth(url)
  289. return url, rev, user_pass
  290. @classmethod
  291. def update_submodules(cls, location):
  292. if not os.path.exists(os.path.join(location, '.gitmodules')):
  293. return
  294. cls.run_command(
  295. ['submodule', 'update', '--init', '--recursive', '-q'],
  296. cwd=location,
  297. )
  298. @classmethod
  299. def controls_location(cls, location):
  300. if super(Git, cls).controls_location(location):
  301. return True
  302. try:
  303. r = cls.run_command(['rev-parse'],
  304. cwd=location,
  305. show_stdout=False,
  306. on_returncode='ignore',
  307. log_failed_cmd=False)
  308. return not r
  309. except BadCommand:
  310. logger.debug("could not determine if %s is under git control "
  311. "because git is not available", location)
  312. return False
  313. vcs.register(Git)