api.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480
  1. """Provide the 'autogenerate' feature which can produce migration operations
  2. automatically."""
  3. from ..operations import ops
  4. from . import render
  5. from . import compare
  6. from .. import util
  7. from sqlalchemy.engine.reflection import Inspector
  8. import contextlib
  9. def compare_metadata(context, metadata):
  10. """Compare a database schema to that given in a
  11. :class:`~sqlalchemy.schema.MetaData` instance.
  12. The database connection is presented in the context
  13. of a :class:`.MigrationContext` object, which
  14. provides database connectivity as well as optional
  15. comparison functions to use for datatypes and
  16. server defaults - see the "autogenerate" arguments
  17. at :meth:`.EnvironmentContext.configure`
  18. for details on these.
  19. The return format is a list of "diff" directives,
  20. each representing individual differences::
  21. from alembic.migration import MigrationContext
  22. from alembic.autogenerate import compare_metadata
  23. from sqlalchemy.schema import SchemaItem
  24. from sqlalchemy.types import TypeEngine
  25. from sqlalchemy import (create_engine, MetaData, Column,
  26. Integer, String, Table)
  27. import pprint
  28. engine = create_engine("sqlite://")
  29. engine.execute('''
  30. create table foo (
  31. id integer not null primary key,
  32. old_data varchar,
  33. x integer
  34. )''')
  35. engine.execute('''
  36. create table bar (
  37. data varchar
  38. )''')
  39. metadata = MetaData()
  40. Table('foo', metadata,
  41. Column('id', Integer, primary_key=True),
  42. Column('data', Integer),
  43. Column('x', Integer, nullable=False)
  44. )
  45. Table('bat', metadata,
  46. Column('info', String)
  47. )
  48. mc = MigrationContext.configure(engine.connect())
  49. diff = compare_metadata(mc, metadata)
  50. pprint.pprint(diff, indent=2, width=20)
  51. Output::
  52. [ ( 'add_table',
  53. Table('bat', MetaData(bind=None),
  54. Column('info', String(), table=<bat>), schema=None)),
  55. ( 'remove_table',
  56. Table(u'bar', MetaData(bind=None),
  57. Column(u'data', VARCHAR(), table=<bar>), schema=None)),
  58. ( 'add_column',
  59. None,
  60. 'foo',
  61. Column('data', Integer(), table=<foo>)),
  62. ( 'remove_column',
  63. None,
  64. 'foo',
  65. Column(u'old_data', VARCHAR(), table=None)),
  66. [ ( 'modify_nullable',
  67. None,
  68. 'foo',
  69. u'x',
  70. { 'existing_server_default': None,
  71. 'existing_type': INTEGER()},
  72. True,
  73. False)]]
  74. :param context: a :class:`.MigrationContext`
  75. instance.
  76. :param metadata: a :class:`~sqlalchemy.schema.MetaData`
  77. instance.
  78. .. seealso::
  79. :func:`.produce_migrations` - produces a :class:`.MigrationScript`
  80. structure based on metadata comparison.
  81. """
  82. migration_script = produce_migrations(context, metadata)
  83. return migration_script.upgrade_ops.as_diffs()
  84. def produce_migrations(context, metadata):
  85. """Produce a :class:`.MigrationScript` structure based on schema
  86. comparison.
  87. This function does essentially what :func:`.compare_metadata` does,
  88. but then runs the resulting list of diffs to produce the full
  89. :class:`.MigrationScript` object. For an example of what this looks like,
  90. see the example in :ref:`customizing_revision`.
  91. .. versionadded:: 0.8.0
  92. .. seealso::
  93. :func:`.compare_metadata` - returns more fundamental "diff"
  94. data from comparing a schema.
  95. """
  96. autogen_context = AutogenContext(context, metadata=metadata)
  97. migration_script = ops.MigrationScript(
  98. rev_id=None,
  99. upgrade_ops=ops.UpgradeOps([]),
  100. downgrade_ops=ops.DowngradeOps([]),
  101. )
  102. compare._populate_migration_script(autogen_context, migration_script)
  103. return migration_script
  104. def render_python_code(
  105. up_or_down_op,
  106. sqlalchemy_module_prefix='sa.',
  107. alembic_module_prefix='op.',
  108. render_as_batch=False,
  109. imports=(),
  110. render_item=None,
  111. ):
  112. """Render Python code given an :class:`.UpgradeOps` or
  113. :class:`.DowngradeOps` object.
  114. This is a convenience function that can be used to test the
  115. autogenerate output of a user-defined :class:`.MigrationScript` structure.
  116. """
  117. opts = {
  118. 'sqlalchemy_module_prefix': sqlalchemy_module_prefix,
  119. 'alembic_module_prefix': alembic_module_prefix,
  120. 'render_item': render_item,
  121. 'render_as_batch': render_as_batch,
  122. }
  123. autogen_context = AutogenContext(None, opts=opts)
  124. autogen_context.imports = set(imports)
  125. return render._indent(render._render_cmd_body(
  126. up_or_down_op, autogen_context))
  127. def _render_migration_diffs(context, template_args):
  128. """legacy, used by test_autogen_composition at the moment"""
  129. autogen_context = AutogenContext(context)
  130. upgrade_ops = ops.UpgradeOps([])
  131. compare._produce_net_changes(autogen_context, upgrade_ops)
  132. migration_script = ops.MigrationScript(
  133. rev_id=None,
  134. upgrade_ops=upgrade_ops,
  135. downgrade_ops=upgrade_ops.reverse(),
  136. )
  137. render._render_python_into_templatevars(
  138. autogen_context, migration_script, template_args
  139. )
  140. class AutogenContext(object):
  141. """Maintains configuration and state that's specific to an
  142. autogenerate operation."""
  143. metadata = None
  144. """The :class:`~sqlalchemy.schema.MetaData` object
  145. representing the destination.
  146. This object is the one that is passed within ``env.py``
  147. to the :paramref:`.EnvironmentContext.configure.target_metadata`
  148. parameter. It represents the structure of :class:`.Table` and other
  149. objects as stated in the current database model, and represents the
  150. destination structure for the database being examined.
  151. While the :class:`~sqlalchemy.schema.MetaData` object is primarily
  152. known as a collection of :class:`~sqlalchemy.schema.Table` objects,
  153. it also has an :attr:`~sqlalchemy.schema.MetaData.info` dictionary
  154. that may be used by end-user schemes to store additional schema-level
  155. objects that are to be compared in custom autogeneration schemes.
  156. """
  157. connection = None
  158. """The :class:`~sqlalchemy.engine.base.Connection` object currently
  159. connected to the database backend being compared.
  160. This is obtained from the :attr:`.MigrationContext.bind` and is
  161. utimately set up in the ``env.py`` script.
  162. """
  163. dialect = None
  164. """The :class:`~sqlalchemy.engine.Dialect` object currently in use.
  165. This is normally obtained from the
  166. :attr:`~sqlalchemy.engine.base.Connection.dialect` attribute.
  167. """
  168. imports = None
  169. """A ``set()`` which contains string Python import directives.
  170. The directives are to be rendered into the ``${imports}`` section
  171. of a script template. The set is normally empty and can be modified
  172. within hooks such as the :paramref:`.EnvironmentContext.configure.render_item`
  173. hook.
  174. .. versionadded:: 0.8.3
  175. .. seealso::
  176. :ref:`autogen_render_types`
  177. """
  178. migration_context = None
  179. """The :class:`.MigrationContext` established by the ``env.py`` script."""
  180. def __init__(
  181. self, migration_context, metadata=None,
  182. opts=None, autogenerate=True):
  183. if autogenerate and \
  184. migration_context is not None and migration_context.as_sql:
  185. raise util.CommandError(
  186. "autogenerate can't use as_sql=True as it prevents querying "
  187. "the database for schema information")
  188. if opts is None:
  189. opts = migration_context.opts
  190. self.metadata = metadata = opts.get('target_metadata', None) \
  191. if metadata is None else metadata
  192. if autogenerate and metadata is None and \
  193. migration_context is not None and \
  194. migration_context.script is not None:
  195. raise util.CommandError(
  196. "Can't proceed with --autogenerate option; environment "
  197. "script %s does not provide "
  198. "a MetaData object or sequence of objects to the context." % (
  199. migration_context.script.env_py_location
  200. ))
  201. include_symbol = opts.get('include_symbol', None)
  202. include_object = opts.get('include_object', None)
  203. object_filters = []
  204. if include_symbol:
  205. def include_symbol_filter(
  206. object, name, type_, reflected, compare_to):
  207. if type_ == "table":
  208. return include_symbol(name, object.schema)
  209. else:
  210. return True
  211. object_filters.append(include_symbol_filter)
  212. if include_object:
  213. object_filters.append(include_object)
  214. self._object_filters = object_filters
  215. self.migration_context = migration_context
  216. if self.migration_context is not None:
  217. self.connection = self.migration_context.bind
  218. self.dialect = self.migration_context.dialect
  219. self.imports = set()
  220. self.opts = opts
  221. self._has_batch = False
  222. @util.memoized_property
  223. def inspector(self):
  224. return Inspector.from_engine(self.connection)
  225. @contextlib.contextmanager
  226. def _within_batch(self):
  227. self._has_batch = True
  228. yield
  229. self._has_batch = False
  230. def run_filters(self, object_, name, type_, reflected, compare_to):
  231. """Run the context's object filters and return True if the targets
  232. should be part of the autogenerate operation.
  233. This method should be run for every kind of object encountered within
  234. an autogenerate operation, giving the environment the chance
  235. to filter what objects should be included in the comparison.
  236. The filters here are produced directly via the
  237. :paramref:`.EnvironmentContext.configure.include_object`
  238. and :paramref:`.EnvironmentContext.configure.include_symbol`
  239. functions, if present.
  240. """
  241. for fn in self._object_filters:
  242. if not fn(object_, name, type_, reflected, compare_to):
  243. return False
  244. else:
  245. return True
  246. @util.memoized_property
  247. def sorted_tables(self):
  248. """Return an aggregate of the :attr:`.MetaData.sorted_tables` collection(s).
  249. For a sequence of :class:`.MetaData` objects, this
  250. concatenates the :attr:`.MetaData.sorted_tables` collection
  251. for each individual :class:`.MetaData` in the order of the
  252. sequence. It does **not** collate the sorted tables collections.
  253. .. versionadded:: 0.9.0
  254. """
  255. result = []
  256. for m in util.to_list(self.metadata):
  257. result.extend(m.sorted_tables)
  258. return result
  259. @util.memoized_property
  260. def table_key_to_table(self):
  261. """Return an aggregate of the :attr:`.MetaData.tables` dictionaries.
  262. The :attr:`.MetaData.tables` collection is a dictionary of table key
  263. to :class:`.Table`; this method aggregates the dictionary across
  264. multiple :class:`.MetaData` objects into one dictionary.
  265. Duplicate table keys are **not** supported; if two :class:`.MetaData`
  266. objects contain the same table key, an exception is raised.
  267. .. versionadded:: 0.9.0
  268. """
  269. result = {}
  270. for m in util.to_list(self.metadata):
  271. intersect = set(result).intersection(set(m.tables))
  272. if intersect:
  273. raise ValueError(
  274. "Duplicate table keys across multiple "
  275. "MetaData objects: %s" %
  276. (", ".join('"%s"' % key for key in sorted(intersect)))
  277. )
  278. result.update(m.tables)
  279. return result
  280. class RevisionContext(object):
  281. """Maintains configuration and state that's specific to a revision
  282. file generation operation."""
  283. def __init__(self, config, script_directory, command_args,
  284. process_revision_directives=None):
  285. self.config = config
  286. self.script_directory = script_directory
  287. self.command_args = command_args
  288. self.process_revision_directives = process_revision_directives
  289. self.template_args = {
  290. 'config': config # Let templates use config for
  291. # e.g. multiple databases
  292. }
  293. self.generated_revisions = [
  294. self._default_revision()
  295. ]
  296. def _to_script(self, migration_script):
  297. template_args = {}
  298. for k, v in self.template_args.items():
  299. template_args.setdefault(k, v)
  300. if getattr(migration_script, '_needs_render', False):
  301. autogen_context = self._last_autogen_context
  302. # clear out existing imports if we are doing multiple
  303. # renders
  304. autogen_context.imports = set()
  305. if migration_script.imports:
  306. autogen_context.imports.union_update(migration_script.imports)
  307. render._render_python_into_templatevars(
  308. autogen_context, migration_script, template_args
  309. )
  310. return self.script_directory.generate_revision(
  311. migration_script.rev_id,
  312. migration_script.message,
  313. refresh=True,
  314. head=migration_script.head,
  315. splice=migration_script.splice,
  316. branch_labels=migration_script.branch_label,
  317. version_path=migration_script.version_path,
  318. depends_on=migration_script.depends_on,
  319. **template_args)
  320. def run_autogenerate(self, rev, migration_context):
  321. self._run_environment(rev, migration_context, True)
  322. def run_no_autogenerate(self, rev, migration_context):
  323. self._run_environment(rev, migration_context, False)
  324. def _run_environment(self, rev, migration_context, autogenerate):
  325. if autogenerate:
  326. if self.command_args['sql']:
  327. raise util.CommandError(
  328. "Using --sql with --autogenerate does not make any sense")
  329. if set(self.script_directory.get_revisions(rev)) != \
  330. set(self.script_directory.get_revisions("heads")):
  331. raise util.CommandError("Target database is not up to date.")
  332. upgrade_token = migration_context.opts['upgrade_token']
  333. downgrade_token = migration_context.opts['downgrade_token']
  334. migration_script = self.generated_revisions[-1]
  335. if not getattr(migration_script, '_needs_render', False):
  336. migration_script.upgrade_ops_list[-1].upgrade_token = upgrade_token
  337. migration_script.downgrade_ops_list[-1].downgrade_token = \
  338. downgrade_token
  339. migration_script._needs_render = True
  340. else:
  341. migration_script._upgrade_ops.append(
  342. ops.UpgradeOps([], upgrade_token=upgrade_token)
  343. )
  344. migration_script._downgrade_ops.append(
  345. ops.DowngradeOps([], downgrade_token=downgrade_token)
  346. )
  347. self._last_autogen_context = autogen_context = \
  348. AutogenContext(migration_context, autogenerate=autogenerate)
  349. if autogenerate:
  350. compare._populate_migration_script(
  351. autogen_context, migration_script)
  352. if self.process_revision_directives:
  353. self.process_revision_directives(
  354. migration_context, rev, self.generated_revisions)
  355. hook = migration_context.opts['process_revision_directives']
  356. if hook:
  357. hook(migration_context, rev, self.generated_revisions)
  358. for migration_script in self.generated_revisions:
  359. migration_script._needs_render = True
  360. def _default_revision(self):
  361. op = ops.MigrationScript(
  362. rev_id=self.command_args['rev_id'] or util.rev_id(),
  363. message=self.command_args['message'],
  364. upgrade_ops=ops.UpgradeOps([]),
  365. downgrade_ops=ops.DowngradeOps([]),
  366. head=self.command_args['head'],
  367. splice=self.command_args['splice'],
  368. branch_label=self.command_args['branch_label'],
  369. version_path=self.command_args['version_path'],
  370. depends_on=self.command_args['depends_on']
  371. )
  372. return op
  373. def generate_scripts(self):
  374. for generated_revision in self.generated_revisions:
  375. yield self._to_script(generated_revision)