logging.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398
  1. # The following comment should be removed at some point in the future.
  2. # mypy: disallow-untyped-defs=False
  3. from __future__ import absolute_import
  4. import contextlib
  5. import errno
  6. import logging
  7. import logging.handlers
  8. import os
  9. import sys
  10. from logging import Filter, getLogger
  11. from pip._vendor.six import PY2
  12. from pip._internal.utils.compat import WINDOWS
  13. from pip._internal.utils.deprecation import DEPRECATION_MSG_PREFIX
  14. from pip._internal.utils.misc import ensure_dir
  15. try:
  16. import threading
  17. except ImportError:
  18. import dummy_threading as threading # type: ignore
  19. try:
  20. # Use "import as" and set colorama in the else clause to avoid mypy
  21. # errors and get the following correct revealed type for colorama:
  22. # `Union[_importlib_modulespec.ModuleType, None]`
  23. # Otherwise, we get an error like the following in the except block:
  24. # > Incompatible types in assignment (expression has type "None",
  25. # variable has type Module)
  26. # TODO: eliminate the need to use "import as" once mypy addresses some
  27. # of its issues with conditional imports. Here is an umbrella issue:
  28. # https://github.com/python/mypy/issues/1297
  29. from pip._vendor import colorama as _colorama
  30. # Lots of different errors can come from this, including SystemError and
  31. # ImportError.
  32. except Exception:
  33. colorama = None
  34. else:
  35. # Import Fore explicitly rather than accessing below as colorama.Fore
  36. # to avoid the following error running mypy:
  37. # > Module has no attribute "Fore"
  38. # TODO: eliminate the need to import Fore once mypy addresses some of its
  39. # issues with conditional imports. This particular case could be an
  40. # instance of the following issue (but also see the umbrella issue above):
  41. # https://github.com/python/mypy/issues/3500
  42. from pip._vendor.colorama import Fore
  43. colorama = _colorama
  44. _log_state = threading.local()
  45. _log_state.indentation = 0
  46. subprocess_logger = getLogger('pip.subprocessor')
  47. class BrokenStdoutLoggingError(Exception):
  48. """
  49. Raised if BrokenPipeError occurs for the stdout stream while logging.
  50. """
  51. pass
  52. # BrokenPipeError does not exist in Python 2 and, in addition, manifests
  53. # differently in Windows and non-Windows.
  54. if WINDOWS:
  55. # In Windows, a broken pipe can show up as EINVAL rather than EPIPE:
  56. # https://bugs.python.org/issue19612
  57. # https://bugs.python.org/issue30418
  58. if PY2:
  59. def _is_broken_pipe_error(exc_class, exc):
  60. """See the docstring for non-Windows Python 3 below."""
  61. return (exc_class is IOError and
  62. exc.errno in (errno.EINVAL, errno.EPIPE))
  63. else:
  64. # In Windows, a broken pipe IOError became OSError in Python 3.
  65. def _is_broken_pipe_error(exc_class, exc):
  66. """See the docstring for non-Windows Python 3 below."""
  67. return ((exc_class is BrokenPipeError) or # noqa: F821
  68. (exc_class is OSError and
  69. exc.errno in (errno.EINVAL, errno.EPIPE)))
  70. elif PY2:
  71. def _is_broken_pipe_error(exc_class, exc):
  72. """See the docstring for non-Windows Python 3 below."""
  73. return (exc_class is IOError and exc.errno == errno.EPIPE)
  74. else:
  75. # Then we are in the non-Windows Python 3 case.
  76. def _is_broken_pipe_error(exc_class, exc):
  77. """
  78. Return whether an exception is a broken pipe error.
  79. Args:
  80. exc_class: an exception class.
  81. exc: an exception instance.
  82. """
  83. return (exc_class is BrokenPipeError) # noqa: F821
  84. @contextlib.contextmanager
  85. def indent_log(num=2):
  86. """
  87. A context manager which will cause the log output to be indented for any
  88. log messages emitted inside it.
  89. """
  90. _log_state.indentation += num
  91. try:
  92. yield
  93. finally:
  94. _log_state.indentation -= num
  95. def get_indentation():
  96. return getattr(_log_state, 'indentation', 0)
  97. class IndentingFormatter(logging.Formatter):
  98. def __init__(self, *args, **kwargs):
  99. """
  100. A logging.Formatter that obeys the indent_log() context manager.
  101. :param add_timestamp: A bool indicating output lines should be prefixed
  102. with their record's timestamp.
  103. """
  104. self.add_timestamp = kwargs.pop("add_timestamp", False)
  105. super(IndentingFormatter, self).__init__(*args, **kwargs)
  106. def get_message_start(self, formatted, levelno):
  107. """
  108. Return the start of the formatted log message (not counting the
  109. prefix to add to each line).
  110. """
  111. if levelno < logging.WARNING:
  112. return ''
  113. if formatted.startswith(DEPRECATION_MSG_PREFIX):
  114. # Then the message already has a prefix. We don't want it to
  115. # look like "WARNING: DEPRECATION: ...."
  116. return ''
  117. if levelno < logging.ERROR:
  118. return 'WARNING: '
  119. return 'ERROR: '
  120. def format(self, record):
  121. """
  122. Calls the standard formatter, but will indent all of the log message
  123. lines by our current indentation level.
  124. """
  125. formatted = super(IndentingFormatter, self).format(record)
  126. message_start = self.get_message_start(formatted, record.levelno)
  127. formatted = message_start + formatted
  128. prefix = ''
  129. if self.add_timestamp:
  130. # TODO: Use Formatter.default_time_format after dropping PY2.
  131. t = self.formatTime(record, "%Y-%m-%dT%H:%M:%S")
  132. prefix = '%s,%03d ' % (t, record.msecs)
  133. prefix += " " * get_indentation()
  134. formatted = "".join([
  135. prefix + line
  136. for line in formatted.splitlines(True)
  137. ])
  138. return formatted
  139. def _color_wrap(*colors):
  140. def wrapped(inp):
  141. return "".join(list(colors) + [inp, colorama.Style.RESET_ALL])
  142. return wrapped
  143. class ColorizedStreamHandler(logging.StreamHandler):
  144. # Don't build up a list of colors if we don't have colorama
  145. if colorama:
  146. COLORS = [
  147. # This needs to be in order from highest logging level to lowest.
  148. (logging.ERROR, _color_wrap(Fore.RED)),
  149. (logging.WARNING, _color_wrap(Fore.YELLOW)),
  150. ]
  151. else:
  152. COLORS = []
  153. def __init__(self, stream=None, no_color=None):
  154. logging.StreamHandler.__init__(self, stream)
  155. self._no_color = no_color
  156. if WINDOWS and colorama:
  157. self.stream = colorama.AnsiToWin32(self.stream)
  158. def _using_stdout(self):
  159. """
  160. Return whether the handler is using sys.stdout.
  161. """
  162. if WINDOWS and colorama:
  163. # Then self.stream is an AnsiToWin32 object.
  164. return self.stream.wrapped is sys.stdout
  165. return self.stream is sys.stdout
  166. def should_color(self):
  167. # Don't colorize things if we do not have colorama or if told not to
  168. if not colorama or self._no_color:
  169. return False
  170. real_stream = (
  171. self.stream if not isinstance(self.stream, colorama.AnsiToWin32)
  172. else self.stream.wrapped
  173. )
  174. # If the stream is a tty we should color it
  175. if hasattr(real_stream, "isatty") and real_stream.isatty():
  176. return True
  177. # If we have an ANSI term we should color it
  178. if os.environ.get("TERM") == "ANSI":
  179. return True
  180. # If anything else we should not color it
  181. return False
  182. def format(self, record):
  183. msg = logging.StreamHandler.format(self, record)
  184. if self.should_color():
  185. for level, color in self.COLORS:
  186. if record.levelno >= level:
  187. msg = color(msg)
  188. break
  189. return msg
  190. # The logging module says handleError() can be customized.
  191. def handleError(self, record):
  192. exc_class, exc = sys.exc_info()[:2]
  193. # If a broken pipe occurred while calling write() or flush() on the
  194. # stdout stream in logging's Handler.emit(), then raise our special
  195. # exception so we can handle it in main() instead of logging the
  196. # broken pipe error and continuing.
  197. if (exc_class and self._using_stdout() and
  198. _is_broken_pipe_error(exc_class, exc)):
  199. raise BrokenStdoutLoggingError()
  200. return super(ColorizedStreamHandler, self).handleError(record)
  201. class BetterRotatingFileHandler(logging.handlers.RotatingFileHandler):
  202. def _open(self):
  203. ensure_dir(os.path.dirname(self.baseFilename))
  204. return logging.handlers.RotatingFileHandler._open(self)
  205. class MaxLevelFilter(Filter):
  206. def __init__(self, level):
  207. self.level = level
  208. def filter(self, record):
  209. return record.levelno < self.level
  210. class ExcludeLoggerFilter(Filter):
  211. """
  212. A logging Filter that excludes records from a logger (or its children).
  213. """
  214. def filter(self, record):
  215. # The base Filter class allows only records from a logger (or its
  216. # children).
  217. return not super(ExcludeLoggerFilter, self).filter(record)
  218. def setup_logging(verbosity, no_color, user_log_file):
  219. """Configures and sets up all of the logging
  220. Returns the requested logging level, as its integer value.
  221. """
  222. # Determine the level to be logging at.
  223. if verbosity >= 1:
  224. level = "DEBUG"
  225. elif verbosity == -1:
  226. level = "WARNING"
  227. elif verbosity == -2:
  228. level = "ERROR"
  229. elif verbosity <= -3:
  230. level = "CRITICAL"
  231. else:
  232. level = "INFO"
  233. level_number = getattr(logging, level)
  234. # The "root" logger should match the "console" level *unless* we also need
  235. # to log to a user log file.
  236. include_user_log = user_log_file is not None
  237. if include_user_log:
  238. additional_log_file = user_log_file
  239. root_level = "DEBUG"
  240. else:
  241. additional_log_file = "/dev/null"
  242. root_level = level
  243. # Disable any logging besides WARNING unless we have DEBUG level logging
  244. # enabled for vendored libraries.
  245. vendored_log_level = "WARNING" if level in ["INFO", "ERROR"] else "DEBUG"
  246. # Shorthands for clarity
  247. log_streams = {
  248. "stdout": "ext://sys.stdout",
  249. "stderr": "ext://sys.stderr",
  250. }
  251. handler_classes = {
  252. "stream": "pip._internal.utils.logging.ColorizedStreamHandler",
  253. "file": "pip._internal.utils.logging.BetterRotatingFileHandler",
  254. }
  255. handlers = ["console", "console_errors", "console_subprocess"] + (
  256. ["user_log"] if include_user_log else []
  257. )
  258. logging.config.dictConfig({
  259. "version": 1,
  260. "disable_existing_loggers": False,
  261. "filters": {
  262. "exclude_warnings": {
  263. "()": "pip._internal.utils.logging.MaxLevelFilter",
  264. "level": logging.WARNING,
  265. },
  266. "restrict_to_subprocess": {
  267. "()": "logging.Filter",
  268. "name": subprocess_logger.name,
  269. },
  270. "exclude_subprocess": {
  271. "()": "pip._internal.utils.logging.ExcludeLoggerFilter",
  272. "name": subprocess_logger.name,
  273. },
  274. },
  275. "formatters": {
  276. "indent": {
  277. "()": IndentingFormatter,
  278. "format": "%(message)s",
  279. },
  280. "indent_with_timestamp": {
  281. "()": IndentingFormatter,
  282. "format": "%(message)s",
  283. "add_timestamp": True,
  284. },
  285. },
  286. "handlers": {
  287. "console": {
  288. "level": level,
  289. "class": handler_classes["stream"],
  290. "no_color": no_color,
  291. "stream": log_streams["stdout"],
  292. "filters": ["exclude_subprocess", "exclude_warnings"],
  293. "formatter": "indent",
  294. },
  295. "console_errors": {
  296. "level": "WARNING",
  297. "class": handler_classes["stream"],
  298. "no_color": no_color,
  299. "stream": log_streams["stderr"],
  300. "filters": ["exclude_subprocess"],
  301. "formatter": "indent",
  302. },
  303. # A handler responsible for logging to the console messages
  304. # from the "subprocessor" logger.
  305. "console_subprocess": {
  306. "level": level,
  307. "class": handler_classes["stream"],
  308. "no_color": no_color,
  309. "stream": log_streams["stderr"],
  310. "filters": ["restrict_to_subprocess"],
  311. "formatter": "indent",
  312. },
  313. "user_log": {
  314. "level": "DEBUG",
  315. "class": handler_classes["file"],
  316. "filename": additional_log_file,
  317. "delay": True,
  318. "formatter": "indent_with_timestamp",
  319. },
  320. },
  321. "root": {
  322. "level": root_level,
  323. "handlers": handlers,
  324. },
  325. "loggers": {
  326. "pip._vendor": {
  327. "level": vendored_log_level
  328. }
  329. },
  330. })
  331. return level_number