plugin_base.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540
  1. # plugin/plugin_base.py
  2. # Copyright (C) 2005-2017 the SQLAlchemy authors and contributors
  3. # <see AUTHORS file>
  4. #
  5. # This module is part of SQLAlchemy and is released under
  6. # the MIT License: http://www.opensource.org/licenses/mit-license.php
  7. """Testing extensions.
  8. this module is designed to work as a testing-framework-agnostic library,
  9. so that we can continue to support nose and also begin adding new
  10. functionality via py.test.
  11. NOTE: copied/adapted from SQLAlchemy master for backwards compatibility;
  12. this should be removable when Alembic targets SQLAlchemy 1.0.0
  13. """
  14. from __future__ import absolute_import
  15. try:
  16. # unitttest has a SkipTest also but pytest doesn't
  17. # honor it unless nose is imported too...
  18. from nose import SkipTest
  19. except ImportError:
  20. from pytest import skip
  21. SkipTest = skip.Exception
  22. import sys
  23. import re
  24. py3k = sys.version_info >= (3, 0)
  25. if py3k:
  26. import configparser
  27. else:
  28. import ConfigParser as configparser
  29. # late imports
  30. fixtures = None
  31. engines = None
  32. provision = None
  33. exclusions = None
  34. warnings = None
  35. assertions = None
  36. requirements = None
  37. config = None
  38. util = None
  39. file_config = None
  40. logging = None
  41. include_tags = set()
  42. exclude_tags = set()
  43. options = None
  44. def setup_options(make_option):
  45. make_option("--log-info", action="callback", type="string", callback=_log,
  46. help="turn on info logging for <LOG> (multiple OK)")
  47. make_option("--log-debug", action="callback",
  48. type="string", callback=_log,
  49. help="turn on debug logging for <LOG> (multiple OK)")
  50. make_option("--db", action="append", type="string", dest="db",
  51. help="Use prefab database uri. Multiple OK, "
  52. "first one is run by default.")
  53. make_option('--dbs', action='callback', callback=_list_dbs,
  54. help="List available prefab dbs")
  55. make_option("--dburi", action="append", type="string", dest="dburi",
  56. help="Database uri. Multiple OK, "
  57. "first one is run by default.")
  58. make_option("--dropfirst", action="store_true", dest="dropfirst",
  59. help="Drop all tables in the target database first")
  60. make_option("--backend-only", action="store_true", dest="backend_only",
  61. help="Run only tests marked with __backend__")
  62. make_option("--low-connections", action="store_true",
  63. dest="low_connections",
  64. help="Use a low number of distinct connections - "
  65. "i.e. for Oracle TNS")
  66. make_option("--write-idents", type="string", dest="write_idents",
  67. help="write out generated follower idents to <file>, "
  68. "when -n<num> is used")
  69. make_option("--reversetop", action="store_true",
  70. dest="reversetop", default=False,
  71. help="Use a random-ordering set implementation in the ORM "
  72. "(helps reveal dependency issues)")
  73. make_option("--requirements", action="callback", type="string",
  74. callback=_requirements_opt,
  75. help="requirements class for testing, overrides setup.cfg")
  76. make_option("--with-cdecimal", action="store_true",
  77. dest="cdecimal", default=False,
  78. help="Monkeypatch the cdecimal library into Python 'decimal' "
  79. "for all tests")
  80. make_option("--include-tag", action="callback", callback=_include_tag,
  81. type="string",
  82. help="Include tests with tag <tag>")
  83. make_option("--exclude-tag", action="callback", callback=_exclude_tag,
  84. type="string",
  85. help="Exclude tests with tag <tag>")
  86. make_option("--mysql-engine", action="store",
  87. dest="mysql_engine", default=None,
  88. help="Use the specified MySQL storage engine for all tables, "
  89. "default is a db-default/InnoDB combo.")
  90. def configure_follower(follower_ident):
  91. """Configure required state for a follower.
  92. This invokes in the parent process and typically includes
  93. database creation.
  94. """
  95. from alembic.testing import provision
  96. provision.FOLLOWER_IDENT = follower_ident
  97. def memoize_important_follower_config(dict_):
  98. """Store important configuration we will need to send to a follower.
  99. This invokes in the parent process after normal config is set up.
  100. This is necessary as py.test seems to not be using forking, so we
  101. start with nothing in memory, *but* it isn't running our argparse
  102. callables, so we have to just copy all of that over.
  103. """
  104. dict_['memoized_config'] = {
  105. 'include_tags': include_tags,
  106. 'exclude_tags': exclude_tags
  107. }
  108. def restore_important_follower_config(dict_):
  109. """Restore important configuration needed by a follower.
  110. This invokes in the follower process.
  111. """
  112. include_tags.update(dict_['memoized_config']['include_tags'])
  113. exclude_tags.update(dict_['memoized_config']['exclude_tags'])
  114. def read_config():
  115. global file_config
  116. file_config = configparser.ConfigParser()
  117. file_config.read(['setup.cfg', 'test.cfg'])
  118. def pre_begin(opt):
  119. """things to set up early, before coverage might be setup."""
  120. global options
  121. options = opt
  122. for fn in pre_configure:
  123. fn(options, file_config)
  124. def set_coverage_flag(value):
  125. options.has_coverage = value
  126. def post_begin():
  127. """things to set up later, once we know coverage is running."""
  128. # Lazy setup of other options (post coverage)
  129. for fn in post_configure:
  130. fn(options, file_config)
  131. # late imports, has to happen after config as well
  132. # as nose plugins like coverage
  133. global util, fixtures, engines, exclusions, \
  134. assertions, warnings, profiling,\
  135. config, testing
  136. from alembic.testing import config, warnings, exclusions # noqa
  137. from alembic.testing import engines, fixtures # noqa
  138. from sqlalchemy import util # noqa
  139. warnings.setup_filters()
  140. def _log(opt_str, value, parser):
  141. global logging
  142. if not logging:
  143. import logging
  144. logging.basicConfig()
  145. if opt_str.endswith('-info'):
  146. logging.getLogger(value).setLevel(logging.INFO)
  147. elif opt_str.endswith('-debug'):
  148. logging.getLogger(value).setLevel(logging.DEBUG)
  149. def _list_dbs(*args):
  150. print("Available --db options (use --dburi to override)")
  151. for macro in sorted(file_config.options('db')):
  152. print("%20s\t%s" % (macro, file_config.get('db', macro)))
  153. sys.exit(0)
  154. def _requirements_opt(opt_str, value, parser):
  155. _setup_requirements(value)
  156. def _exclude_tag(opt_str, value, parser):
  157. exclude_tags.add(value.replace('-', '_'))
  158. def _include_tag(opt_str, value, parser):
  159. include_tags.add(value.replace('-', '_'))
  160. pre_configure = []
  161. post_configure = []
  162. def pre(fn):
  163. pre_configure.append(fn)
  164. return fn
  165. def post(fn):
  166. post_configure.append(fn)
  167. return fn
  168. @pre
  169. def _setup_options(opt, file_config):
  170. global options
  171. options = opt
  172. @pre
  173. def _monkeypatch_cdecimal(options, file_config):
  174. if options.cdecimal:
  175. import cdecimal
  176. sys.modules['decimal'] = cdecimal
  177. @post
  178. def _engine_uri(options, file_config):
  179. from alembic.testing import config
  180. from alembic.testing import provision
  181. if options.dburi:
  182. db_urls = list(options.dburi)
  183. else:
  184. db_urls = []
  185. if options.db:
  186. for db_token in options.db:
  187. for db in re.split(r'[,\s]+', db_token):
  188. if db not in file_config.options('db'):
  189. raise RuntimeError(
  190. "Unknown URI specifier '%s'. "
  191. "Specify --dbs for known uris."
  192. % db)
  193. else:
  194. db_urls.append(file_config.get('db', db))
  195. if not db_urls:
  196. db_urls.append(file_config.get('db', 'default'))
  197. for db_url in db_urls:
  198. cfg = provision.setup_config(
  199. db_url, options, file_config, provision.FOLLOWER_IDENT)
  200. if not config._current:
  201. cfg.set_as_current(cfg)
  202. @post
  203. def _requirements(options, file_config):
  204. requirement_cls = file_config.get('sqla_testing', "requirement_cls")
  205. _setup_requirements(requirement_cls)
  206. def _setup_requirements(argument):
  207. from alembic.testing import config
  208. if config.requirements is not None:
  209. return
  210. modname, clsname = argument.split(":")
  211. # importlib.import_module() only introduced in 2.7, a little
  212. # late
  213. mod = __import__(modname)
  214. for component in modname.split(".")[1:]:
  215. mod = getattr(mod, component)
  216. req_cls = getattr(mod, clsname)
  217. config.requirements = req_cls()
  218. @post
  219. def _prep_testing_database(options, file_config):
  220. from alembic.testing import config
  221. from alembic.testing.exclusions import against
  222. from sqlalchemy import schema
  223. from alembic import util
  224. if util.sqla_08:
  225. from sqlalchemy import inspect
  226. else:
  227. from sqlalchemy.engine.reflection import Inspector
  228. inspect = Inspector.from_engine
  229. if options.dropfirst:
  230. for cfg in config.Config.all_configs():
  231. e = cfg.db
  232. inspector = inspect(e)
  233. try:
  234. view_names = inspector.get_view_names()
  235. except NotImplementedError:
  236. pass
  237. else:
  238. for vname in view_names:
  239. e.execute(schema._DropView(
  240. schema.Table(vname, schema.MetaData())
  241. ))
  242. if config.requirements.schemas.enabled_for_config(cfg):
  243. try:
  244. view_names = inspector.get_view_names(
  245. schema="test_schema")
  246. except NotImplementedError:
  247. pass
  248. else:
  249. for vname in view_names:
  250. e.execute(schema._DropView(
  251. schema.Table(vname, schema.MetaData(),
  252. schema="test_schema")
  253. ))
  254. for tname in reversed(inspector.get_table_names(
  255. order_by="foreign_key")):
  256. e.execute(schema.DropTable(
  257. schema.Table(tname, schema.MetaData())
  258. ))
  259. if config.requirements.schemas.enabled_for_config(cfg):
  260. for tname in reversed(inspector.get_table_names(
  261. order_by="foreign_key", schema="test_schema")):
  262. e.execute(schema.DropTable(
  263. schema.Table(tname, schema.MetaData(),
  264. schema="test_schema")
  265. ))
  266. if against(cfg, "postgresql") and util.sqla_100:
  267. from sqlalchemy.dialects import postgresql
  268. for enum in inspector.get_enums("*"):
  269. e.execute(postgresql.DropEnumType(
  270. postgresql.ENUM(
  271. name=enum['name'],
  272. schema=enum['schema'])))
  273. @post
  274. def _reverse_topological(options, file_config):
  275. if options.reversetop:
  276. from sqlalchemy.orm.util import randomize_unitofwork
  277. randomize_unitofwork()
  278. @post
  279. def _post_setup_options(opt, file_config):
  280. from alembic.testing import config
  281. config.options = options
  282. config.file_config = file_config
  283. def want_class(cls):
  284. if not issubclass(cls, fixtures.TestBase):
  285. return False
  286. elif cls.__name__.startswith('_'):
  287. return False
  288. elif config.options.backend_only and not getattr(cls, '__backend__',
  289. False):
  290. return False
  291. else:
  292. return True
  293. def want_method(cls, fn):
  294. if not fn.__name__.startswith("test_"):
  295. return False
  296. elif fn.__module__ is None:
  297. return False
  298. elif include_tags:
  299. return (
  300. hasattr(cls, '__tags__') and
  301. exclusions.tags(cls.__tags__).include_test(
  302. include_tags, exclude_tags)
  303. ) or (
  304. hasattr(fn, '_sa_exclusion_extend') and
  305. fn._sa_exclusion_extend.include_test(
  306. include_tags, exclude_tags)
  307. )
  308. elif exclude_tags and hasattr(cls, '__tags__'):
  309. return exclusions.tags(cls.__tags__).include_test(
  310. include_tags, exclude_tags)
  311. elif exclude_tags and hasattr(fn, '_sa_exclusion_extend'):
  312. return fn._sa_exclusion_extend.include_test(include_tags, exclude_tags)
  313. else:
  314. return True
  315. def generate_sub_tests(cls, module):
  316. if getattr(cls, '__backend__', False):
  317. for cfg in _possible_configs_for_cls(cls):
  318. name = "%s_%s_%s" % (cls.__name__, cfg.db.name, cfg.db.driver)
  319. subcls = type(
  320. name,
  321. (cls, ),
  322. {
  323. "__only_on__": ("%s+%s" % (cfg.db.name, cfg.db.driver)),
  324. }
  325. )
  326. setattr(module, name, subcls)
  327. yield subcls
  328. else:
  329. yield cls
  330. def start_test_class(cls):
  331. _do_skips(cls)
  332. _setup_engine(cls)
  333. def stop_test_class(cls):
  334. #from sqlalchemy import inspect
  335. #assert not inspect(testing.db).get_table_names()
  336. _restore_engine()
  337. def _restore_engine():
  338. config._current.reset()
  339. def _setup_engine(cls):
  340. if getattr(cls, '__engine_options__', None):
  341. eng = engines.testing_engine(options=cls.__engine_options__)
  342. config._current.push_engine(eng)
  343. def before_test(test, test_module_name, test_class, test_name):
  344. pass
  345. def after_test(test):
  346. pass
  347. def _possible_configs_for_cls(cls, reasons=None):
  348. all_configs = set(config.Config.all_configs())
  349. if cls.__unsupported_on__:
  350. spec = exclusions.db_spec(*cls.__unsupported_on__)
  351. for config_obj in list(all_configs):
  352. if spec(config_obj):
  353. all_configs.remove(config_obj)
  354. if getattr(cls, '__only_on__', None):
  355. spec = exclusions.db_spec(*util.to_list(cls.__only_on__))
  356. for config_obj in list(all_configs):
  357. if not spec(config_obj):
  358. all_configs.remove(config_obj)
  359. if hasattr(cls, '__requires__'):
  360. requirements = config.requirements
  361. for config_obj in list(all_configs):
  362. for requirement in cls.__requires__:
  363. check = getattr(requirements, requirement)
  364. skip_reasons = check.matching_config_reasons(config_obj)
  365. if skip_reasons:
  366. all_configs.remove(config_obj)
  367. if reasons is not None:
  368. reasons.extend(skip_reasons)
  369. break
  370. if hasattr(cls, '__prefer_requires__'):
  371. non_preferred = set()
  372. requirements = config.requirements
  373. for config_obj in list(all_configs):
  374. for requirement in cls.__prefer_requires__:
  375. check = getattr(requirements, requirement)
  376. if not check.enabled_for_config(config_obj):
  377. non_preferred.add(config_obj)
  378. if all_configs.difference(non_preferred):
  379. all_configs.difference_update(non_preferred)
  380. return all_configs
  381. def _do_skips(cls):
  382. reasons = []
  383. all_configs = _possible_configs_for_cls(cls, reasons)
  384. if getattr(cls, '__skip_if__', False):
  385. for c in getattr(cls, '__skip_if__'):
  386. if c():
  387. raise SkipTest("'%s' skipped by %s" % (
  388. cls.__name__, c.__name__)
  389. )
  390. if not all_configs:
  391. if getattr(cls, '__backend__', False):
  392. msg = "'%s' unsupported for implementation '%s'" % (
  393. cls.__name__, cls.__only_on__)
  394. else:
  395. msg = "'%s' unsupported on any DB implementation %s%s" % (
  396. cls.__name__,
  397. ", ".join(
  398. "'%s(%s)+%s'" % (
  399. config_obj.db.name,
  400. ".".join(
  401. str(dig) for dig in
  402. config_obj.db.dialect.server_version_info),
  403. config_obj.db.driver
  404. )
  405. for config_obj in config.Config.all_configs()
  406. ),
  407. ", ".join(reasons)
  408. )
  409. raise SkipTest(msg)
  410. elif hasattr(cls, '__prefer_backends__'):
  411. non_preferred = set()
  412. spec = exclusions.db_spec(*util.to_list(cls.__prefer_backends__))
  413. for config_obj in all_configs:
  414. if not spec(config_obj):
  415. non_preferred.add(config_obj)
  416. if all_configs.difference(non_preferred):
  417. all_configs.difference_update(non_preferred)
  418. if config._current not in all_configs:
  419. _setup_config(all_configs.pop(), cls)
  420. def _setup_config(config_obj, ctx):
  421. config._current.push(config_obj)