base.py 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633
  1. # sql/base.py
  2. # Copyright (C) 2005-2017 the SQLAlchemy authors and contributors
  3. # <see AUTHORS file>
  4. #
  5. # This module is part of SQLAlchemy and is released under
  6. # the MIT License: http://www.opensource.org/licenses/mit-license.php
  7. """Foundational utilities common to many sql modules.
  8. """
  9. from .. import util, exc
  10. import itertools
  11. from .visitors import ClauseVisitor
  12. import re
  13. import collections
  14. PARSE_AUTOCOMMIT = util.symbol('PARSE_AUTOCOMMIT')
  15. NO_ARG = util.symbol('NO_ARG')
  16. class Immutable(object):
  17. """mark a ClauseElement as 'immutable' when expressions are cloned."""
  18. def unique_params(self, *optionaldict, **kwargs):
  19. raise NotImplementedError("Immutable objects do not support copying")
  20. def params(self, *optionaldict, **kwargs):
  21. raise NotImplementedError("Immutable objects do not support copying")
  22. def _clone(self):
  23. return self
  24. def _from_objects(*elements):
  25. return itertools.chain(*[element._from_objects for element in elements])
  26. @util.decorator
  27. def _generative(fn, *args, **kw):
  28. """Mark a method as generative."""
  29. self = args[0]._generate()
  30. fn(self, *args[1:], **kw)
  31. return self
  32. class _DialectArgView(collections.MutableMapping):
  33. """A dictionary view of dialect-level arguments in the form
  34. <dialectname>_<argument_name>.
  35. """
  36. def __init__(self, obj):
  37. self.obj = obj
  38. def _key(self, key):
  39. try:
  40. dialect, value_key = key.split("_", 1)
  41. except ValueError:
  42. raise KeyError(key)
  43. else:
  44. return dialect, value_key
  45. def __getitem__(self, key):
  46. dialect, value_key = self._key(key)
  47. try:
  48. opt = self.obj.dialect_options[dialect]
  49. except exc.NoSuchModuleError:
  50. raise KeyError(key)
  51. else:
  52. return opt[value_key]
  53. def __setitem__(self, key, value):
  54. try:
  55. dialect, value_key = self._key(key)
  56. except KeyError:
  57. raise exc.ArgumentError(
  58. "Keys must be of the form <dialectname>_<argname>")
  59. else:
  60. self.obj.dialect_options[dialect][value_key] = value
  61. def __delitem__(self, key):
  62. dialect, value_key = self._key(key)
  63. del self.obj.dialect_options[dialect][value_key]
  64. def __len__(self):
  65. return sum(len(args._non_defaults) for args in
  66. self.obj.dialect_options.values())
  67. def __iter__(self):
  68. return (
  69. util.safe_kwarg("%s_%s" % (dialect_name, value_name))
  70. for dialect_name in self.obj.dialect_options
  71. for value_name in
  72. self.obj.dialect_options[dialect_name]._non_defaults
  73. )
  74. class _DialectArgDict(collections.MutableMapping):
  75. """A dictionary view of dialect-level arguments for a specific
  76. dialect.
  77. Maintains a separate collection of user-specified arguments
  78. and dialect-specified default arguments.
  79. """
  80. def __init__(self):
  81. self._non_defaults = {}
  82. self._defaults = {}
  83. def __len__(self):
  84. return len(set(self._non_defaults).union(self._defaults))
  85. def __iter__(self):
  86. return iter(set(self._non_defaults).union(self._defaults))
  87. def __getitem__(self, key):
  88. if key in self._non_defaults:
  89. return self._non_defaults[key]
  90. else:
  91. return self._defaults[key]
  92. def __setitem__(self, key, value):
  93. self._non_defaults[key] = value
  94. def __delitem__(self, key):
  95. del self._non_defaults[key]
  96. class DialectKWArgs(object):
  97. """Establish the ability for a class to have dialect-specific arguments
  98. with defaults and constructor validation.
  99. The :class:`.DialectKWArgs` interacts with the
  100. :attr:`.DefaultDialect.construct_arguments` present on a dialect.
  101. .. seealso::
  102. :attr:`.DefaultDialect.construct_arguments`
  103. """
  104. @classmethod
  105. def argument_for(cls, dialect_name, argument_name, default):
  106. """Add a new kind of dialect-specific keyword argument for this class.
  107. E.g.::
  108. Index.argument_for("mydialect", "length", None)
  109. some_index = Index('a', 'b', mydialect_length=5)
  110. The :meth:`.DialectKWArgs.argument_for` method is a per-argument
  111. way adding extra arguments to the
  112. :attr:`.DefaultDialect.construct_arguments` dictionary. This
  113. dictionary provides a list of argument names accepted by various
  114. schema-level constructs on behalf of a dialect.
  115. New dialects should typically specify this dictionary all at once as a
  116. data member of the dialect class. The use case for ad-hoc addition of
  117. argument names is typically for end-user code that is also using
  118. a custom compilation scheme which consumes the additional arguments.
  119. :param dialect_name: name of a dialect. The dialect must be
  120. locatable, else a :class:`.NoSuchModuleError` is raised. The
  121. dialect must also include an existing
  122. :attr:`.DefaultDialect.construct_arguments` collection, indicating
  123. that it participates in the keyword-argument validation and default
  124. system, else :class:`.ArgumentError` is raised. If the dialect does
  125. not include this collection, then any keyword argument can be
  126. specified on behalf of this dialect already. All dialects packaged
  127. within SQLAlchemy include this collection, however for third party
  128. dialects, support may vary.
  129. :param argument_name: name of the parameter.
  130. :param default: default value of the parameter.
  131. .. versionadded:: 0.9.4
  132. """
  133. construct_arg_dictionary = DialectKWArgs._kw_registry[dialect_name]
  134. if construct_arg_dictionary is None:
  135. raise exc.ArgumentError(
  136. "Dialect '%s' does have keyword-argument "
  137. "validation and defaults enabled configured" %
  138. dialect_name)
  139. if cls not in construct_arg_dictionary:
  140. construct_arg_dictionary[cls] = {}
  141. construct_arg_dictionary[cls][argument_name] = default
  142. @util.memoized_property
  143. def dialect_kwargs(self):
  144. """A collection of keyword arguments specified as dialect-specific
  145. options to this construct.
  146. The arguments are present here in their original ``<dialect>_<kwarg>``
  147. format. Only arguments that were actually passed are included;
  148. unlike the :attr:`.DialectKWArgs.dialect_options` collection, which
  149. contains all options known by this dialect including defaults.
  150. The collection is also writable; keys are accepted of the
  151. form ``<dialect>_<kwarg>`` where the value will be assembled
  152. into the list of options.
  153. .. versionadded:: 0.9.2
  154. .. versionchanged:: 0.9.4 The :attr:`.DialectKWArgs.dialect_kwargs`
  155. collection is now writable.
  156. .. seealso::
  157. :attr:`.DialectKWArgs.dialect_options` - nested dictionary form
  158. """
  159. return _DialectArgView(self)
  160. @property
  161. def kwargs(self):
  162. """A synonym for :attr:`.DialectKWArgs.dialect_kwargs`."""
  163. return self.dialect_kwargs
  164. @util.dependencies("sqlalchemy.dialects")
  165. def _kw_reg_for_dialect(dialects, dialect_name):
  166. dialect_cls = dialects.registry.load(dialect_name)
  167. if dialect_cls.construct_arguments is None:
  168. return None
  169. return dict(dialect_cls.construct_arguments)
  170. _kw_registry = util.PopulateDict(_kw_reg_for_dialect)
  171. def _kw_reg_for_dialect_cls(self, dialect_name):
  172. construct_arg_dictionary = DialectKWArgs._kw_registry[dialect_name]
  173. d = _DialectArgDict()
  174. if construct_arg_dictionary is None:
  175. d._defaults.update({"*": None})
  176. else:
  177. for cls in reversed(self.__class__.__mro__):
  178. if cls in construct_arg_dictionary:
  179. d._defaults.update(construct_arg_dictionary[cls])
  180. return d
  181. @util.memoized_property
  182. def dialect_options(self):
  183. """A collection of keyword arguments specified as dialect-specific
  184. options to this construct.
  185. This is a two-level nested registry, keyed to ``<dialect_name>``
  186. and ``<argument_name>``. For example, the ``postgresql_where``
  187. argument would be locatable as::
  188. arg = my_object.dialect_options['postgresql']['where']
  189. .. versionadded:: 0.9.2
  190. .. seealso::
  191. :attr:`.DialectKWArgs.dialect_kwargs` - flat dictionary form
  192. """
  193. return util.PopulateDict(
  194. util.portable_instancemethod(self._kw_reg_for_dialect_cls)
  195. )
  196. def _validate_dialect_kwargs(self, kwargs):
  197. # validate remaining kwargs that they all specify DB prefixes
  198. if not kwargs:
  199. return
  200. for k in kwargs:
  201. m = re.match('^(.+?)_(.+)$', k)
  202. if not m:
  203. raise TypeError(
  204. "Additional arguments should be "
  205. "named <dialectname>_<argument>, got '%s'" % k)
  206. dialect_name, arg_name = m.group(1, 2)
  207. try:
  208. construct_arg_dictionary = self.dialect_options[dialect_name]
  209. except exc.NoSuchModuleError:
  210. util.warn(
  211. "Can't validate argument %r; can't "
  212. "locate any SQLAlchemy dialect named %r" %
  213. (k, dialect_name))
  214. self.dialect_options[dialect_name] = d = _DialectArgDict()
  215. d._defaults.update({"*": None})
  216. d._non_defaults[arg_name] = kwargs[k]
  217. else:
  218. if "*" not in construct_arg_dictionary and \
  219. arg_name not in construct_arg_dictionary:
  220. raise exc.ArgumentError(
  221. "Argument %r is not accepted by "
  222. "dialect %r on behalf of %r" % (
  223. k,
  224. dialect_name, self.__class__
  225. ))
  226. else:
  227. construct_arg_dictionary[arg_name] = kwargs[k]
  228. class Generative(object):
  229. """Allow a ClauseElement to generate itself via the
  230. @_generative decorator.
  231. """
  232. def _generate(self):
  233. s = self.__class__.__new__(self.__class__)
  234. s.__dict__ = self.__dict__.copy()
  235. return s
  236. class Executable(Generative):
  237. """Mark a ClauseElement as supporting execution.
  238. :class:`.Executable` is a superclass for all "statement" types
  239. of objects, including :func:`select`, :func:`delete`, :func:`update`,
  240. :func:`insert`, :func:`text`.
  241. """
  242. supports_execution = True
  243. _execution_options = util.immutabledict()
  244. _bind = None
  245. @_generative
  246. def execution_options(self, **kw):
  247. """ Set non-SQL options for the statement which take effect during
  248. execution.
  249. Execution options can be set on a per-statement or
  250. per :class:`.Connection` basis. Additionally, the
  251. :class:`.Engine` and ORM :class:`~.orm.query.Query` objects provide
  252. access to execution options which they in turn configure upon
  253. connections.
  254. The :meth:`execution_options` method is generative. A new
  255. instance of this statement is returned that contains the options::
  256. statement = select([table.c.x, table.c.y])
  257. statement = statement.execution_options(autocommit=True)
  258. Note that only a subset of possible execution options can be applied
  259. to a statement - these include "autocommit" and "stream_results",
  260. but not "isolation_level" or "compiled_cache".
  261. See :meth:`.Connection.execution_options` for a full list of
  262. possible options.
  263. .. seealso::
  264. :meth:`.Connection.execution_options()`
  265. :meth:`.Query.execution_options()`
  266. """
  267. if 'isolation_level' in kw:
  268. raise exc.ArgumentError(
  269. "'isolation_level' execution option may only be specified "
  270. "on Connection.execution_options(), or "
  271. "per-engine using the isolation_level "
  272. "argument to create_engine()."
  273. )
  274. if 'compiled_cache' in kw:
  275. raise exc.ArgumentError(
  276. "'compiled_cache' execution option may only be specified "
  277. "on Connection.execution_options(), not per statement."
  278. )
  279. self._execution_options = self._execution_options.union(kw)
  280. def execute(self, *multiparams, **params):
  281. """Compile and execute this :class:`.Executable`."""
  282. e = self.bind
  283. if e is None:
  284. label = getattr(self, 'description', self.__class__.__name__)
  285. msg = ('This %s is not directly bound to a Connection or Engine.'
  286. 'Use the .execute() method of a Connection or Engine '
  287. 'to execute this construct.' % label)
  288. raise exc.UnboundExecutionError(msg)
  289. return e._execute_clauseelement(self, multiparams, params)
  290. def scalar(self, *multiparams, **params):
  291. """Compile and execute this :class:`.Executable`, returning the
  292. result's scalar representation.
  293. """
  294. return self.execute(*multiparams, **params).scalar()
  295. @property
  296. def bind(self):
  297. """Returns the :class:`.Engine` or :class:`.Connection` to
  298. which this :class:`.Executable` is bound, or None if none found.
  299. This is a traversal which checks locally, then
  300. checks among the "from" clauses of associated objects
  301. until a bound engine or connection is found.
  302. """
  303. if self._bind is not None:
  304. return self._bind
  305. for f in _from_objects(self):
  306. if f is self:
  307. continue
  308. engine = f.bind
  309. if engine is not None:
  310. return engine
  311. else:
  312. return None
  313. class SchemaEventTarget(object):
  314. """Base class for elements that are the targets of :class:`.DDLEvents`
  315. events.
  316. This includes :class:`.SchemaItem` as well as :class:`.SchemaType`.
  317. """
  318. def _set_parent(self, parent):
  319. """Associate with this SchemaEvent's parent object."""
  320. def _set_parent_with_dispatch(self, parent):
  321. self.dispatch.before_parent_attach(self, parent)
  322. self._set_parent(parent)
  323. self.dispatch.after_parent_attach(self, parent)
  324. class SchemaVisitor(ClauseVisitor):
  325. """Define the visiting for ``SchemaItem`` objects."""
  326. __traverse_options__ = {'schema_visitor': True}
  327. class ColumnCollection(util.OrderedProperties):
  328. """An ordered dictionary that stores a list of ColumnElement
  329. instances.
  330. Overrides the ``__eq__()`` method to produce SQL clauses between
  331. sets of correlated columns.
  332. """
  333. __slots__ = '_all_columns'
  334. def __init__(self, *columns):
  335. super(ColumnCollection, self).__init__()
  336. object.__setattr__(self, '_all_columns', [])
  337. for c in columns:
  338. self.add(c)
  339. def __str__(self):
  340. return repr([str(c) for c in self])
  341. def replace(self, column):
  342. """add the given column to this collection, removing unaliased
  343. versions of this column as well as existing columns with the
  344. same key.
  345. e.g.::
  346. t = Table('sometable', metadata, Column('col1', Integer))
  347. t.columns.replace(Column('col1', Integer, key='columnone'))
  348. will remove the original 'col1' from the collection, and add
  349. the new column under the name 'columnname'.
  350. Used by schema.Column to override columns during table reflection.
  351. """
  352. remove_col = None
  353. if column.name in self and column.key != column.name:
  354. other = self[column.name]
  355. if other.name == other.key:
  356. remove_col = other
  357. del self._data[other.key]
  358. if column.key in self._data:
  359. remove_col = self._data[column.key]
  360. self._data[column.key] = column
  361. if remove_col is not None:
  362. self._all_columns[:] = [column if c is remove_col
  363. else c for c in self._all_columns]
  364. else:
  365. self._all_columns.append(column)
  366. def add(self, column):
  367. """Add a column to this collection.
  368. The key attribute of the column will be used as the hash key
  369. for this dictionary.
  370. """
  371. if not column.key:
  372. raise exc.ArgumentError(
  373. "Can't add unnamed column to column collection")
  374. self[column.key] = column
  375. def __delitem__(self, key):
  376. raise NotImplementedError()
  377. def __setattr__(self, key, object):
  378. raise NotImplementedError()
  379. def __setitem__(self, key, value):
  380. if key in self:
  381. # this warning is primarily to catch select() statements
  382. # which have conflicting column names in their exported
  383. # columns collection
  384. existing = self[key]
  385. if not existing.shares_lineage(value):
  386. util.warn('Column %r on table %r being replaced by '
  387. '%r, which has the same key. Consider '
  388. 'use_labels for select() statements.' %
  389. (key, getattr(existing, 'table', None), value))
  390. # pop out memoized proxy_set as this
  391. # operation may very well be occurring
  392. # in a _make_proxy operation
  393. util.memoized_property.reset(value, "proxy_set")
  394. self._all_columns.append(value)
  395. self._data[key] = value
  396. def clear(self):
  397. raise NotImplementedError()
  398. def remove(self, column):
  399. del self._data[column.key]
  400. self._all_columns[:] = [
  401. c for c in self._all_columns if c is not column]
  402. def update(self, iter):
  403. cols = list(iter)
  404. all_col_set = set(self._all_columns)
  405. self._all_columns.extend(
  406. c for label, c in cols if c not in all_col_set)
  407. self._data.update((label, c) for label, c in cols)
  408. def extend(self, iter):
  409. cols = list(iter)
  410. all_col_set = set(self._all_columns)
  411. self._all_columns.extend(c for c in cols if c not in all_col_set)
  412. self._data.update((c.key, c) for c in cols)
  413. __hash__ = None
  414. @util.dependencies("sqlalchemy.sql.elements")
  415. def __eq__(self, elements, other):
  416. l = []
  417. for c in getattr(other, "_all_columns", other):
  418. for local in self._all_columns:
  419. if c.shares_lineage(local):
  420. l.append(c == local)
  421. return elements.and_(*l)
  422. def __contains__(self, other):
  423. if not isinstance(other, util.string_types):
  424. raise exc.ArgumentError("__contains__ requires a string argument")
  425. return util.OrderedProperties.__contains__(self, other)
  426. def __getstate__(self):
  427. return {'_data': self._data,
  428. '_all_columns': self._all_columns}
  429. def __setstate__(self, state):
  430. object.__setattr__(self, '_data', state['_data'])
  431. object.__setattr__(self, '_all_columns', state['_all_columns'])
  432. def contains_column(self, col):
  433. return col in set(self._all_columns)
  434. def as_immutable(self):
  435. return ImmutableColumnCollection(self._data, self._all_columns)
  436. class ImmutableColumnCollection(util.ImmutableProperties, ColumnCollection):
  437. def __init__(self, data, all_columns):
  438. util.ImmutableProperties.__init__(self, data)
  439. object.__setattr__(self, '_all_columns', all_columns)
  440. extend = remove = util.ImmutableProperties._immutable
  441. class ColumnSet(util.ordered_column_set):
  442. def contains_column(self, col):
  443. return col in self
  444. def extend(self, cols):
  445. for col in cols:
  446. self.add(col)
  447. def __add__(self, other):
  448. return list(self) + list(other)
  449. @util.dependencies("sqlalchemy.sql.elements")
  450. def __eq__(self, elements, other):
  451. l = []
  452. for c in other:
  453. for local in self:
  454. if c.shares_lineage(local):
  455. l.append(c == local)
  456. return elements.and_(*l)
  457. def __hash__(self):
  458. return hash(tuple(x for x in self))
  459. def _bind_or_error(schemaitem, msg=None):
  460. bind = schemaitem.bind
  461. if not bind:
  462. name = schemaitem.__class__.__name__
  463. label = getattr(schemaitem, 'fullname',
  464. getattr(schemaitem, 'name', None))
  465. if label:
  466. item = '%s object %r' % (name, label)
  467. else:
  468. item = '%s object' % name
  469. if msg is None:
  470. msg = "%s is not bound to an Engine or Connection. "\
  471. "Execution can not proceed without a database to execute "\
  472. "against." % item
  473. raise exc.UnboundExecutionError(msg)
  474. return bind