_reloader.py 8.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252
  1. import os
  2. import sys
  3. import time
  4. import subprocess
  5. import threading
  6. from itertools import chain
  7. from werkzeug._internal import _log
  8. from werkzeug._compat import PY2, iteritems, text_type
  9. def _iter_module_files():
  10. """This iterates over all relevant Python files. It goes through all
  11. loaded files from modules, all files in folders of already loaded modules
  12. as well as all files reachable through a package.
  13. """
  14. # The list call is necessary on Python 3 in case the module
  15. # dictionary modifies during iteration.
  16. for module in list(sys.modules.values()):
  17. if module is None:
  18. continue
  19. filename = getattr(module, '__file__', None)
  20. if filename:
  21. old = None
  22. while not os.path.isfile(filename):
  23. old = filename
  24. filename = os.path.dirname(filename)
  25. if filename == old:
  26. break
  27. else:
  28. if filename[-4:] in ('.pyc', '.pyo'):
  29. filename = filename[:-1]
  30. yield filename
  31. def _find_observable_paths(extra_files=None):
  32. """Finds all paths that should be observed."""
  33. rv = set(os.path.abspath(x) for x in sys.path)
  34. for filename in extra_files or ():
  35. rv.add(os.path.dirname(os.path.abspath(filename)))
  36. for module in list(sys.modules.values()):
  37. fn = getattr(module, '__file__', None)
  38. if fn is None:
  39. continue
  40. fn = os.path.abspath(fn)
  41. rv.add(os.path.dirname(fn))
  42. return _find_common_roots(rv)
  43. def _find_common_roots(paths):
  44. """Out of some paths it finds the common roots that need monitoring."""
  45. paths = [x.split(os.path.sep) for x in paths]
  46. root = {}
  47. for chunks in sorted(paths, key=len, reverse=True):
  48. node = root
  49. for chunk in chunks:
  50. node = node.setdefault(chunk, {})
  51. node.clear()
  52. rv = set()
  53. def _walk(node, path):
  54. for prefix, child in iteritems(node):
  55. _walk(child, path + (prefix,))
  56. if not node:
  57. rv.add('/'.join(path))
  58. _walk(root, ())
  59. return rv
  60. class ReloaderLoop(object):
  61. name = None
  62. # monkeypatched by testsuite. wrapping with `staticmethod` is required in
  63. # case time.sleep has been replaced by a non-c function (e.g. by
  64. # `eventlet.monkey_patch`) before we get here
  65. _sleep = staticmethod(time.sleep)
  66. def __init__(self, extra_files=None, interval=1):
  67. self.extra_files = set(os.path.abspath(x)
  68. for x in extra_files or ())
  69. self.interval = interval
  70. def run(self):
  71. pass
  72. def restart_with_reloader(self):
  73. """Spawn a new Python interpreter with the same arguments as this one,
  74. but running the reloader thread.
  75. """
  76. while 1:
  77. _log('info', ' * Restarting with %s' % self.name)
  78. args = [sys.executable] + sys.argv
  79. new_environ = os.environ.copy()
  80. new_environ['WERKZEUG_RUN_MAIN'] = 'true'
  81. # a weird bug on windows. sometimes unicode strings end up in the
  82. # environment and subprocess.call does not like this, encode them
  83. # to latin1 and continue.
  84. if os.name == 'nt' and PY2:
  85. for key, value in iteritems(new_environ):
  86. if isinstance(value, text_type):
  87. new_environ[key] = value.encode('iso-8859-1')
  88. exit_code = subprocess.call(args, env=new_environ,
  89. close_fds=False)
  90. if exit_code != 3:
  91. return exit_code
  92. def trigger_reload(self, filename):
  93. self.log_reload(filename)
  94. sys.exit(3)
  95. def log_reload(self, filename):
  96. filename = os.path.abspath(filename)
  97. _log('info', ' * Detected change in %r, reloading' % filename)
  98. class StatReloaderLoop(ReloaderLoop):
  99. name = 'stat'
  100. def run(self):
  101. mtimes = {}
  102. while 1:
  103. for filename in chain(_iter_module_files(),
  104. self.extra_files):
  105. try:
  106. mtime = os.stat(filename).st_mtime
  107. except OSError:
  108. continue
  109. old_time = mtimes.get(filename)
  110. if old_time is None:
  111. mtimes[filename] = mtime
  112. continue
  113. elif mtime > old_time:
  114. self.trigger_reload(filename)
  115. self._sleep(self.interval)
  116. class WatchdogReloaderLoop(ReloaderLoop):
  117. def __init__(self, *args, **kwargs):
  118. ReloaderLoop.__init__(self, *args, **kwargs)
  119. from watchdog.observers import Observer
  120. from watchdog.events import FileSystemEventHandler
  121. self.observable_paths = set()
  122. def _check_modification(filename):
  123. if filename in self.extra_files:
  124. self.trigger_reload(filename)
  125. dirname = os.path.dirname(filename)
  126. if dirname.startswith(tuple(self.observable_paths)):
  127. if filename.endswith(('.pyc', '.pyo')):
  128. self.trigger_reload(filename[:-1])
  129. elif filename.endswith('.py'):
  130. self.trigger_reload(filename)
  131. class _CustomHandler(FileSystemEventHandler):
  132. def on_created(self, event):
  133. _check_modification(event.src_path)
  134. def on_modified(self, event):
  135. _check_modification(event.src_path)
  136. def on_moved(self, event):
  137. _check_modification(event.src_path)
  138. _check_modification(event.dest_path)
  139. def on_deleted(self, event):
  140. _check_modification(event.src_path)
  141. reloader_name = Observer.__name__.lower()
  142. if reloader_name.endswith('observer'):
  143. reloader_name = reloader_name[:-8]
  144. reloader_name += ' reloader'
  145. self.name = reloader_name
  146. self.observer_class = Observer
  147. self.event_handler = _CustomHandler()
  148. self.should_reload = False
  149. def trigger_reload(self, filename):
  150. # This is called inside an event handler, which means throwing
  151. # SystemExit has no effect.
  152. # https://github.com/gorakhargosh/watchdog/issues/294
  153. self.should_reload = True
  154. self.log_reload(filename)
  155. def run(self):
  156. watches = {}
  157. observer = self.observer_class()
  158. observer.start()
  159. while not self.should_reload:
  160. to_delete = set(watches)
  161. paths = _find_observable_paths(self.extra_files)
  162. for path in paths:
  163. if path not in watches:
  164. try:
  165. watches[path] = observer.schedule(
  166. self.event_handler, path, recursive=True)
  167. except OSError:
  168. # Clear this path from list of watches We don't want
  169. # the same error message showing again in the next
  170. # iteration.
  171. watches[path] = None
  172. to_delete.discard(path)
  173. for path in to_delete:
  174. watch = watches.pop(path, None)
  175. if watch is not None:
  176. observer.unschedule(watch)
  177. self.observable_paths = paths
  178. self._sleep(self.interval)
  179. sys.exit(3)
  180. reloader_loops = {
  181. 'stat': StatReloaderLoop,
  182. 'watchdog': WatchdogReloaderLoop,
  183. }
  184. try:
  185. __import__('watchdog.observers')
  186. except ImportError:
  187. reloader_loops['auto'] = reloader_loops['stat']
  188. else:
  189. reloader_loops['auto'] = reloader_loops['watchdog']
  190. def run_with_reloader(main_func, extra_files=None, interval=1,
  191. reloader_type='auto'):
  192. """Run the given function in an independent python interpreter."""
  193. import signal
  194. reloader = reloader_loops[reloader_type](extra_files, interval)
  195. signal.signal(signal.SIGTERM, lambda *args: sys.exit(0))
  196. try:
  197. if os.environ.get('WERKZEUG_RUN_MAIN') == 'true':
  198. t = threading.Thread(target=main_func, args=())
  199. t.setDaemon(True)
  200. t.start()
  201. reloader.run()
  202. else:
  203. sys.exit(reloader.restart_with_reloader())
  204. except KeyboardInterrupt:
  205. pass