_compat.py 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648
  1. import re
  2. import io
  3. import os
  4. import sys
  5. import codecs
  6. from weakref import WeakKeyDictionary
  7. PY2 = sys.version_info[0] == 2
  8. WIN = sys.platform.startswith('win')
  9. DEFAULT_COLUMNS = 80
  10. _ansi_re = re.compile('\033\[((?:\d|;)*)([a-zA-Z])')
  11. def get_filesystem_encoding():
  12. return sys.getfilesystemencoding() or sys.getdefaultencoding()
  13. def _make_text_stream(stream, encoding, errors):
  14. if encoding is None:
  15. encoding = get_best_encoding(stream)
  16. if errors is None:
  17. errors = 'replace'
  18. return _NonClosingTextIOWrapper(stream, encoding, errors,
  19. line_buffering=True)
  20. def is_ascii_encoding(encoding):
  21. """Checks if a given encoding is ascii."""
  22. try:
  23. return codecs.lookup(encoding).name == 'ascii'
  24. except LookupError:
  25. return False
  26. def get_best_encoding(stream):
  27. """Returns the default stream encoding if not found."""
  28. rv = getattr(stream, 'encoding', None) or sys.getdefaultencoding()
  29. if is_ascii_encoding(rv):
  30. return 'utf-8'
  31. return rv
  32. class _NonClosingTextIOWrapper(io.TextIOWrapper):
  33. def __init__(self, stream, encoding, errors, **extra):
  34. self._stream = stream = _FixupStream(stream)
  35. io.TextIOWrapper.__init__(self, stream, encoding, errors, **extra)
  36. # The io module is a place where the Python 3 text behavior
  37. # was forced upon Python 2, so we need to unbreak
  38. # it to look like Python 2.
  39. if PY2:
  40. def write(self, x):
  41. if isinstance(x, str) or is_bytes(x):
  42. try:
  43. self.flush()
  44. except Exception:
  45. pass
  46. return self.buffer.write(str(x))
  47. return io.TextIOWrapper.write(self, x)
  48. def writelines(self, lines):
  49. for line in lines:
  50. self.write(line)
  51. def __del__(self):
  52. try:
  53. self.detach()
  54. except Exception:
  55. pass
  56. def isatty(self):
  57. # https://bitbucket.org/pypy/pypy/issue/1803
  58. return self._stream.isatty()
  59. class _FixupStream(object):
  60. """The new io interface needs more from streams than streams
  61. traditionally implement. As such, this fix-up code is necessary in
  62. some circumstances.
  63. """
  64. def __init__(self, stream):
  65. self._stream = stream
  66. def __getattr__(self, name):
  67. return getattr(self._stream, name)
  68. def read1(self, size):
  69. f = getattr(self._stream, 'read1', None)
  70. if f is not None:
  71. return f(size)
  72. # We only dispatch to readline instead of read in Python 2 as we
  73. # do not want cause problems with the different implementation
  74. # of line buffering.
  75. if PY2:
  76. return self._stream.readline(size)
  77. return self._stream.read(size)
  78. def readable(self):
  79. x = getattr(self._stream, 'readable', None)
  80. if x is not None:
  81. return x()
  82. try:
  83. self._stream.read(0)
  84. except Exception:
  85. return False
  86. return True
  87. def writable(self):
  88. x = getattr(self._stream, 'writable', None)
  89. if x is not None:
  90. return x()
  91. try:
  92. self._stream.write('')
  93. except Exception:
  94. try:
  95. self._stream.write(b'')
  96. except Exception:
  97. return False
  98. return True
  99. def seekable(self):
  100. x = getattr(self._stream, 'seekable', None)
  101. if x is not None:
  102. return x()
  103. try:
  104. self._stream.seek(self._stream.tell())
  105. except Exception:
  106. return False
  107. return True
  108. if PY2:
  109. text_type = unicode
  110. bytes = str
  111. raw_input = raw_input
  112. string_types = (str, unicode)
  113. iteritems = lambda x: x.iteritems()
  114. range_type = xrange
  115. def is_bytes(x):
  116. return isinstance(x, (buffer, bytearray))
  117. _identifier_re = re.compile(r'^[a-zA-Z_][a-zA-Z0-9_]*$')
  118. # For Windows, we need to force stdout/stdin/stderr to binary if it's
  119. # fetched for that. This obviously is not the most correct way to do
  120. # it as it changes global state. Unfortunately, there does not seem to
  121. # be a clear better way to do it as just reopening the file in binary
  122. # mode does not change anything.
  123. #
  124. # An option would be to do what Python 3 does and to open the file as
  125. # binary only, patch it back to the system, and then use a wrapper
  126. # stream that converts newlines. It's not quite clear what's the
  127. # correct option here.
  128. #
  129. # This code also lives in _winconsole for the fallback to the console
  130. # emulation stream.
  131. #
  132. # There are also Windows environments where the `msvcrt` module is not
  133. # available (which is why we use try-catch instead of the WIN variable
  134. # here), such as the Google App Engine development server on Windows. In
  135. # those cases there is just nothing we can do.
  136. try:
  137. import msvcrt
  138. except ImportError:
  139. set_binary_mode = lambda x: x
  140. else:
  141. def set_binary_mode(f):
  142. try:
  143. fileno = f.fileno()
  144. except Exception:
  145. pass
  146. else:
  147. msvcrt.setmode(fileno, os.O_BINARY)
  148. return f
  149. def isidentifier(x):
  150. return _identifier_re.search(x) is not None
  151. def get_binary_stdin():
  152. return set_binary_mode(sys.stdin)
  153. def get_binary_stdout():
  154. return set_binary_mode(sys.stdout)
  155. def get_binary_stderr():
  156. return set_binary_mode(sys.stderr)
  157. def get_text_stdin(encoding=None, errors=None):
  158. rv = _get_windows_console_stream(sys.stdin, encoding, errors)
  159. if rv is not None:
  160. return rv
  161. return _make_text_stream(sys.stdin, encoding, errors)
  162. def get_text_stdout(encoding=None, errors=None):
  163. rv = _get_windows_console_stream(sys.stdout, encoding, errors)
  164. if rv is not None:
  165. return rv
  166. return _make_text_stream(sys.stdout, encoding, errors)
  167. def get_text_stderr(encoding=None, errors=None):
  168. rv = _get_windows_console_stream(sys.stderr, encoding, errors)
  169. if rv is not None:
  170. return rv
  171. return _make_text_stream(sys.stderr, encoding, errors)
  172. def filename_to_ui(value):
  173. if isinstance(value, bytes):
  174. value = value.decode(get_filesystem_encoding(), 'replace')
  175. return value
  176. else:
  177. import io
  178. text_type = str
  179. raw_input = input
  180. string_types = (str,)
  181. range_type = range
  182. isidentifier = lambda x: x.isidentifier()
  183. iteritems = lambda x: iter(x.items())
  184. def is_bytes(x):
  185. return isinstance(x, (bytes, memoryview, bytearray))
  186. def _is_binary_reader(stream, default=False):
  187. try:
  188. return isinstance(stream.read(0), bytes)
  189. except Exception:
  190. return default
  191. # This happens in some cases where the stream was already
  192. # closed. In this case, we assume the default.
  193. def _is_binary_writer(stream, default=False):
  194. try:
  195. stream.write(b'')
  196. except Exception:
  197. try:
  198. stream.write('')
  199. return False
  200. except Exception:
  201. pass
  202. return default
  203. return True
  204. def _find_binary_reader(stream):
  205. # We need to figure out if the given stream is already binary.
  206. # This can happen because the official docs recommend detaching
  207. # the streams to get binary streams. Some code might do this, so
  208. # we need to deal with this case explicitly.
  209. if _is_binary_reader(stream, False):
  210. return stream
  211. buf = getattr(stream, 'buffer', None)
  212. # Same situation here; this time we assume that the buffer is
  213. # actually binary in case it's closed.
  214. if buf is not None and _is_binary_reader(buf, True):
  215. return buf
  216. def _find_binary_writer(stream):
  217. # We need to figure out if the given stream is already binary.
  218. # This can happen because the official docs recommend detatching
  219. # the streams to get binary streams. Some code might do this, so
  220. # we need to deal with this case explicitly.
  221. if _is_binary_writer(stream, False):
  222. return stream
  223. buf = getattr(stream, 'buffer', None)
  224. # Same situation here; this time we assume that the buffer is
  225. # actually binary in case it's closed.
  226. if buf is not None and _is_binary_writer(buf, True):
  227. return buf
  228. def _stream_is_misconfigured(stream):
  229. """A stream is misconfigured if its encoding is ASCII."""
  230. # If the stream does not have an encoding set, we assume it's set
  231. # to ASCII. This appears to happen in certain unittest
  232. # environments. It's not quite clear what the correct behavior is
  233. # but this at least will force Click to recover somehow.
  234. return is_ascii_encoding(getattr(stream, 'encoding', None) or 'ascii')
  235. def _is_compatible_text_stream(stream, encoding, errors):
  236. stream_encoding = getattr(stream, 'encoding', None)
  237. stream_errors = getattr(stream, 'errors', None)
  238. # Perfect match.
  239. if stream_encoding == encoding and stream_errors == errors:
  240. return True
  241. # Otherwise, it's only a compatible stream if we did not ask for
  242. # an encoding.
  243. if encoding is None:
  244. return stream_encoding is not None
  245. return False
  246. def _force_correct_text_reader(text_reader, encoding, errors):
  247. if _is_binary_reader(text_reader, False):
  248. binary_reader = text_reader
  249. else:
  250. # If there is no target encoding set, we need to verify that the
  251. # reader is not actually misconfigured.
  252. if encoding is None and not _stream_is_misconfigured(text_reader):
  253. return text_reader
  254. if _is_compatible_text_stream(text_reader, encoding, errors):
  255. return text_reader
  256. # If the reader has no encoding, we try to find the underlying
  257. # binary reader for it. If that fails because the environment is
  258. # misconfigured, we silently go with the same reader because this
  259. # is too common to happen. In that case, mojibake is better than
  260. # exceptions.
  261. binary_reader = _find_binary_reader(text_reader)
  262. if binary_reader is None:
  263. return text_reader
  264. # At this point, we default the errors to replace instead of strict
  265. # because nobody handles those errors anyways and at this point
  266. # we're so fundamentally fucked that nothing can repair it.
  267. if errors is None:
  268. errors = 'replace'
  269. return _make_text_stream(binary_reader, encoding, errors)
  270. def _force_correct_text_writer(text_writer, encoding, errors):
  271. if _is_binary_writer(text_writer, False):
  272. binary_writer = text_writer
  273. else:
  274. # If there is no target encoding set, we need to verify that the
  275. # writer is not actually misconfigured.
  276. if encoding is None and not _stream_is_misconfigured(text_writer):
  277. return text_writer
  278. if _is_compatible_text_stream(text_writer, encoding, errors):
  279. return text_writer
  280. # If the writer has no encoding, we try to find the underlying
  281. # binary writer for it. If that fails because the environment is
  282. # misconfigured, we silently go with the same writer because this
  283. # is too common to happen. In that case, mojibake is better than
  284. # exceptions.
  285. binary_writer = _find_binary_writer(text_writer)
  286. if binary_writer is None:
  287. return text_writer
  288. # At this point, we default the errors to replace instead of strict
  289. # because nobody handles those errors anyways and at this point
  290. # we're so fundamentally fucked that nothing can repair it.
  291. if errors is None:
  292. errors = 'replace'
  293. return _make_text_stream(binary_writer, encoding, errors)
  294. def get_binary_stdin():
  295. reader = _find_binary_reader(sys.stdin)
  296. if reader is None:
  297. raise RuntimeError('Was not able to determine binary '
  298. 'stream for sys.stdin.')
  299. return reader
  300. def get_binary_stdout():
  301. writer = _find_binary_writer(sys.stdout)
  302. if writer is None:
  303. raise RuntimeError('Was not able to determine binary '
  304. 'stream for sys.stdout.')
  305. return writer
  306. def get_binary_stderr():
  307. writer = _find_binary_writer(sys.stderr)
  308. if writer is None:
  309. raise RuntimeError('Was not able to determine binary '
  310. 'stream for sys.stderr.')
  311. return writer
  312. def get_text_stdin(encoding=None, errors=None):
  313. rv = _get_windows_console_stream(sys.stdin, encoding, errors)
  314. if rv is not None:
  315. return rv
  316. return _force_correct_text_reader(sys.stdin, encoding, errors)
  317. def get_text_stdout(encoding=None, errors=None):
  318. rv = _get_windows_console_stream(sys.stdout, encoding, errors)
  319. if rv is not None:
  320. return rv
  321. return _force_correct_text_writer(sys.stdout, encoding, errors)
  322. def get_text_stderr(encoding=None, errors=None):
  323. rv = _get_windows_console_stream(sys.stderr, encoding, errors)
  324. if rv is not None:
  325. return rv
  326. return _force_correct_text_writer(sys.stderr, encoding, errors)
  327. def filename_to_ui(value):
  328. if isinstance(value, bytes):
  329. value = value.decode(get_filesystem_encoding(), 'replace')
  330. else:
  331. value = value.encode('utf-8', 'surrogateescape') \
  332. .decode('utf-8', 'replace')
  333. return value
  334. def get_streerror(e, default=None):
  335. if hasattr(e, 'strerror'):
  336. msg = e.strerror
  337. else:
  338. if default is not None:
  339. msg = default
  340. else:
  341. msg = str(e)
  342. if isinstance(msg, bytes):
  343. msg = msg.decode('utf-8', 'replace')
  344. return msg
  345. def open_stream(filename, mode='r', encoding=None, errors='strict',
  346. atomic=False):
  347. # Standard streams first. These are simple because they don't need
  348. # special handling for the atomic flag. It's entirely ignored.
  349. if filename == '-':
  350. if 'w' in mode:
  351. if 'b' in mode:
  352. return get_binary_stdout(), False
  353. return get_text_stdout(encoding=encoding, errors=errors), False
  354. if 'b' in mode:
  355. return get_binary_stdin(), False
  356. return get_text_stdin(encoding=encoding, errors=errors), False
  357. # Non-atomic writes directly go out through the regular open functions.
  358. if not atomic:
  359. if encoding is None:
  360. return open(filename, mode), True
  361. return io.open(filename, mode, encoding=encoding, errors=errors), True
  362. # Some usability stuff for atomic writes
  363. if 'a' in mode:
  364. raise ValueError(
  365. 'Appending to an existing file is not supported, because that '
  366. 'would involve an expensive `copy`-operation to a temporary '
  367. 'file. Open the file in normal `w`-mode and copy explicitly '
  368. 'if that\'s what you\'re after.'
  369. )
  370. if 'x' in mode:
  371. raise ValueError('Use the `overwrite`-parameter instead.')
  372. if 'w' not in mode:
  373. raise ValueError('Atomic writes only make sense with `w`-mode.')
  374. # Atomic writes are more complicated. They work by opening a file
  375. # as a proxy in the same folder and then using the fdopen
  376. # functionality to wrap it in a Python file. Then we wrap it in an
  377. # atomic file that moves the file over on close.
  378. import tempfile
  379. fd, tmp_filename = tempfile.mkstemp(dir=os.path.dirname(filename),
  380. prefix='.__atomic-write')
  381. if encoding is not None:
  382. f = io.open(fd, mode, encoding=encoding, errors=errors)
  383. else:
  384. f = os.fdopen(fd, mode)
  385. return _AtomicFile(f, tmp_filename, filename), True
  386. # Used in a destructor call, needs extra protection from interpreter cleanup.
  387. if hasattr(os, 'replace'):
  388. _replace = os.replace
  389. _can_replace = True
  390. else:
  391. _replace = os.rename
  392. _can_replace = not WIN
  393. class _AtomicFile(object):
  394. def __init__(self, f, tmp_filename, real_filename):
  395. self._f = f
  396. self._tmp_filename = tmp_filename
  397. self._real_filename = real_filename
  398. self.closed = False
  399. @property
  400. def name(self):
  401. return self._real_filename
  402. def close(self, delete=False):
  403. if self.closed:
  404. return
  405. self._f.close()
  406. if not _can_replace:
  407. try:
  408. os.remove(self._real_filename)
  409. except OSError:
  410. pass
  411. _replace(self._tmp_filename, self._real_filename)
  412. self.closed = True
  413. def __getattr__(self, name):
  414. return getattr(self._f, name)
  415. def __enter__(self):
  416. return self
  417. def __exit__(self, exc_type, exc_value, tb):
  418. self.close(delete=exc_type is not None)
  419. def __repr__(self):
  420. return repr(self._f)
  421. auto_wrap_for_ansi = None
  422. colorama = None
  423. get_winterm_size = None
  424. def strip_ansi(value):
  425. return _ansi_re.sub('', value)
  426. def should_strip_ansi(stream=None, color=None):
  427. if color is None:
  428. if stream is None:
  429. stream = sys.stdin
  430. return not isatty(stream)
  431. return not color
  432. # If we're on Windows, we provide transparent integration through
  433. # colorama. This will make ANSI colors through the echo function
  434. # work automatically.
  435. if WIN:
  436. # Windows has a smaller terminal
  437. DEFAULT_COLUMNS = 79
  438. from ._winconsole import _get_windows_console_stream
  439. def _get_argv_encoding():
  440. import locale
  441. return locale.getpreferredencoding()
  442. if PY2:
  443. def raw_input(prompt=''):
  444. sys.stderr.flush()
  445. if prompt:
  446. stdout = _default_text_stdout()
  447. stdout.write(prompt)
  448. stdin = _default_text_stdin()
  449. return stdin.readline().rstrip('\r\n')
  450. try:
  451. import colorama
  452. except ImportError:
  453. pass
  454. else:
  455. _ansi_stream_wrappers = WeakKeyDictionary()
  456. def auto_wrap_for_ansi(stream, color=None):
  457. """This function wraps a stream so that calls through colorama
  458. are issued to the win32 console API to recolor on demand. It
  459. also ensures to reset the colors if a write call is interrupted
  460. to not destroy the console afterwards.
  461. """
  462. try:
  463. cached = _ansi_stream_wrappers.get(stream)
  464. except Exception:
  465. cached = None
  466. if cached is not None:
  467. return cached
  468. strip = should_strip_ansi(stream, color)
  469. ansi_wrapper = colorama.AnsiToWin32(stream, strip=strip)
  470. rv = ansi_wrapper.stream
  471. _write = rv.write
  472. def _safe_write(s):
  473. try:
  474. return _write(s)
  475. except:
  476. ansi_wrapper.reset_all()
  477. raise
  478. rv.write = _safe_write
  479. try:
  480. _ansi_stream_wrappers[stream] = rv
  481. except Exception:
  482. pass
  483. return rv
  484. def get_winterm_size():
  485. win = colorama.win32.GetConsoleScreenBufferInfo(
  486. colorama.win32.STDOUT).srWindow
  487. return win.Right - win.Left, win.Bottom - win.Top
  488. else:
  489. def _get_argv_encoding():
  490. return getattr(sys.stdin, 'encoding', None) or get_filesystem_encoding()
  491. _get_windows_console_stream = lambda *x: None
  492. def term_len(x):
  493. return len(strip_ansi(x))
  494. def isatty(stream):
  495. try:
  496. return stream.isatty()
  497. except Exception:
  498. return False
  499. def _make_cached_stream_func(src_func, wrapper_func):
  500. cache = WeakKeyDictionary()
  501. def func():
  502. stream = src_func()
  503. try:
  504. rv = cache.get(stream)
  505. except Exception:
  506. rv = None
  507. if rv is not None:
  508. return rv
  509. rv = wrapper_func()
  510. try:
  511. cache[stream] = rv
  512. except Exception:
  513. pass
  514. return rv
  515. return func
  516. _default_text_stdin = _make_cached_stream_func(
  517. lambda: sys.stdin, get_text_stdin)
  518. _default_text_stdout = _make_cached_stream_func(
  519. lambda: sys.stdout, get_text_stdout)
  520. _default_text_stderr = _make_cached_stream_func(
  521. lambda: sys.stderr, get_text_stderr)
  522. binary_streams = {
  523. 'stdin': get_binary_stdin,
  524. 'stdout': get_binary_stdout,
  525. 'stderr': get_binary_stderr,
  526. }
  527. text_streams = {
  528. 'stdin': get_text_stdin,
  529. 'stdout': get_text_stdout,
  530. 'stderr': get_text_stderr,
  531. }