strategies.py 62 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707
  1. # orm/strategies.py
  2. # Copyright (C) 2005-2017 the SQLAlchemy authors and contributors
  3. # <see AUTHORS file>
  4. #
  5. # This module is part of SQLAlchemy and is released under
  6. # the MIT License: http://www.opensource.org/licenses/mit-license.php
  7. """sqlalchemy.orm.interfaces.LoaderStrategy
  8. implementations, and related MapperOptions."""
  9. from .. import exc as sa_exc, inspect
  10. from .. import util, log, event
  11. from ..sql import util as sql_util, visitors
  12. from .. import sql
  13. from . import (
  14. attributes, interfaces, exc as orm_exc, loading,
  15. unitofwork, util as orm_util
  16. )
  17. from .state import InstanceState
  18. from .util import _none_set
  19. from . import properties
  20. from .interfaces import (
  21. LoaderStrategy, StrategizedProperty
  22. )
  23. from .base import _SET_DEFERRED_EXPIRED, _DEFER_FOR_STATE
  24. from .session import _state_session
  25. import itertools
  26. def _register_attribute(
  27. prop, mapper, useobject,
  28. compare_function=None,
  29. typecallable=None,
  30. callable_=None,
  31. proxy_property=None,
  32. active_history=False,
  33. impl_class=None,
  34. **kw
  35. ):
  36. attribute_ext = list(util.to_list(prop.extension, default=[]))
  37. listen_hooks = []
  38. uselist = useobject and prop.uselist
  39. if useobject and prop.single_parent:
  40. listen_hooks.append(single_parent_validator)
  41. if prop.key in prop.parent.validators:
  42. fn, opts = prop.parent.validators[prop.key]
  43. listen_hooks.append(
  44. lambda desc, prop: orm_util._validator_events(
  45. desc,
  46. prop.key, fn, **opts)
  47. )
  48. if useobject:
  49. listen_hooks.append(unitofwork.track_cascade_events)
  50. # need to assemble backref listeners
  51. # after the singleparentvalidator, mapper validator
  52. if useobject:
  53. backref = prop.back_populates
  54. if backref:
  55. listen_hooks.append(
  56. lambda desc, prop: attributes.backref_listeners(
  57. desc,
  58. backref,
  59. uselist
  60. )
  61. )
  62. # a single MapperProperty is shared down a class inheritance
  63. # hierarchy, so we set up attribute instrumentation and backref event
  64. # for each mapper down the hierarchy.
  65. # typically, "mapper" is the same as prop.parent, due to the way
  66. # the configure_mappers() process runs, however this is not strongly
  67. # enforced, and in the case of a second configure_mappers() run the
  68. # mapper here might not be prop.parent; also, a subclass mapper may
  69. # be called here before a superclass mapper. That is, can't depend
  70. # on mappers not already being set up so we have to check each one.
  71. for m in mapper.self_and_descendants:
  72. if prop is m._props.get(prop.key) and \
  73. not m.class_manager._attr_has_impl(prop.key):
  74. desc = attributes.register_attribute_impl(
  75. m.class_,
  76. prop.key,
  77. parent_token=prop,
  78. uselist=uselist,
  79. compare_function=compare_function,
  80. useobject=useobject,
  81. extension=attribute_ext,
  82. trackparent=useobject and (
  83. prop.single_parent or
  84. prop.direction is interfaces.ONETOMANY),
  85. typecallable=typecallable,
  86. callable_=callable_,
  87. active_history=active_history,
  88. impl_class=impl_class,
  89. send_modified_events=not useobject or not prop.viewonly,
  90. doc=prop.doc,
  91. **kw
  92. )
  93. for hook in listen_hooks:
  94. hook(desc, prop)
  95. @properties.ColumnProperty.strategy_for(instrument=False, deferred=False)
  96. class UninstrumentedColumnLoader(LoaderStrategy):
  97. """Represent a non-instrumented MapperProperty.
  98. The polymorphic_on argument of mapper() often results in this,
  99. if the argument is against the with_polymorphic selectable.
  100. """
  101. __slots__ = 'columns',
  102. def __init__(self, parent, strategy_key):
  103. super(UninstrumentedColumnLoader, self).__init__(parent, strategy_key)
  104. self.columns = self.parent_property.columns
  105. def setup_query(
  106. self, context, entity, path, loadopt, adapter,
  107. column_collection=None, **kwargs):
  108. for c in self.columns:
  109. if adapter:
  110. c = adapter.columns[c]
  111. column_collection.append(c)
  112. def create_row_processor(
  113. self, context, path, loadopt,
  114. mapper, result, adapter, populators):
  115. pass
  116. @log.class_logger
  117. @properties.ColumnProperty.strategy_for(instrument=True, deferred=False)
  118. class ColumnLoader(LoaderStrategy):
  119. """Provide loading behavior for a :class:`.ColumnProperty`."""
  120. __slots__ = 'columns', 'is_composite'
  121. def __init__(self, parent, strategy_key):
  122. super(ColumnLoader, self).__init__(parent, strategy_key)
  123. self.columns = self.parent_property.columns
  124. self.is_composite = hasattr(self.parent_property, 'composite_class')
  125. def setup_query(
  126. self, context, entity, path, loadopt,
  127. adapter, column_collection, memoized_populators, **kwargs):
  128. for c in self.columns:
  129. if adapter:
  130. c = adapter.columns[c]
  131. column_collection.append(c)
  132. fetch = self.columns[0]
  133. if adapter:
  134. fetch = adapter.columns[fetch]
  135. memoized_populators[self.parent_property] = fetch
  136. def init_class_attribute(self, mapper):
  137. self.is_class_level = True
  138. coltype = self.columns[0].type
  139. # TODO: check all columns ? check for foreign key as well?
  140. active_history = self.parent_property.active_history or \
  141. self.columns[0].primary_key or \
  142. mapper.version_id_col in set(self.columns)
  143. _register_attribute(
  144. self.parent_property, mapper, useobject=False,
  145. compare_function=coltype.compare_values,
  146. active_history=active_history
  147. )
  148. def create_row_processor(
  149. self, context, path,
  150. loadopt, mapper, result, adapter, populators):
  151. # look through list of columns represented here
  152. # to see which, if any, is present in the row.
  153. for col in self.columns:
  154. if adapter:
  155. col = adapter.columns[col]
  156. getter = result._getter(col, False)
  157. if getter:
  158. populators["quick"].append((self.key, getter))
  159. break
  160. else:
  161. populators["expire"].append((self.key, True))
  162. @log.class_logger
  163. @properties.ColumnProperty.strategy_for(deferred=True, instrument=True)
  164. class DeferredColumnLoader(LoaderStrategy):
  165. """Provide loading behavior for a deferred :class:`.ColumnProperty`."""
  166. __slots__ = 'columns', 'group'
  167. def __init__(self, parent, strategy_key):
  168. super(DeferredColumnLoader, self).__init__(parent, strategy_key)
  169. if hasattr(self.parent_property, 'composite_class'):
  170. raise NotImplementedError("Deferred loading for composite "
  171. "types not implemented yet")
  172. self.columns = self.parent_property.columns
  173. self.group = self.parent_property.group
  174. def create_row_processor(
  175. self, context, path, loadopt,
  176. mapper, result, adapter, populators):
  177. # this path currently does not check the result
  178. # for the column; this is because in most cases we are
  179. # working just with the setup_query() directive which does
  180. # not support this, and the behavior here should be consistent.
  181. if not self.is_class_level:
  182. set_deferred_for_local_state = \
  183. self.parent_property._deferred_column_loader
  184. populators["new"].append((self.key, set_deferred_for_local_state))
  185. else:
  186. populators["expire"].append((self.key, False))
  187. def init_class_attribute(self, mapper):
  188. self.is_class_level = True
  189. _register_attribute(
  190. self.parent_property, mapper, useobject=False,
  191. compare_function=self.columns[0].type.compare_values,
  192. callable_=self._load_for_state,
  193. expire_missing=False
  194. )
  195. def setup_query(
  196. self, context, entity, path, loadopt,
  197. adapter, column_collection, memoized_populators,
  198. only_load_props=None, **kw):
  199. if (
  200. (
  201. loadopt and
  202. 'undefer_pks' in loadopt.local_opts and
  203. set(self.columns).intersection(
  204. self.parent._should_undefer_in_wildcard)
  205. )
  206. or
  207. (
  208. loadopt and
  209. self.group and
  210. loadopt.local_opts.get('undefer_group_%s' % self.group, False)
  211. )
  212. or
  213. (
  214. only_load_props and self.key in only_load_props
  215. )
  216. ):
  217. self.parent_property._get_strategy(
  218. (("deferred", False), ("instrument", True))
  219. ).setup_query(
  220. context, entity,
  221. path, loadopt, adapter,
  222. column_collection, memoized_populators, **kw)
  223. elif self.is_class_level:
  224. memoized_populators[self.parent_property] = _SET_DEFERRED_EXPIRED
  225. else:
  226. memoized_populators[self.parent_property] = _DEFER_FOR_STATE
  227. def _load_for_state(self, state, passive):
  228. if not state.key:
  229. return attributes.ATTR_EMPTY
  230. if not passive & attributes.SQL_OK:
  231. return attributes.PASSIVE_NO_RESULT
  232. localparent = state.manager.mapper
  233. if self.group:
  234. toload = [
  235. p.key for p in
  236. localparent.iterate_properties
  237. if isinstance(p, StrategizedProperty) and
  238. isinstance(p.strategy, DeferredColumnLoader) and
  239. p.group == self.group
  240. ]
  241. else:
  242. toload = [self.key]
  243. # narrow the keys down to just those which have no history
  244. group = [k for k in toload if k in state.unmodified]
  245. session = _state_session(state)
  246. if session is None:
  247. raise orm_exc.DetachedInstanceError(
  248. "Parent instance %s is not bound to a Session; "
  249. "deferred load operation of attribute '%s' cannot proceed" %
  250. (orm_util.state_str(state), self.key)
  251. )
  252. query = session.query(localparent)
  253. if loading.load_on_ident(
  254. query, state.key,
  255. only_load_props=group, refresh_state=state) is None:
  256. raise orm_exc.ObjectDeletedError(state)
  257. return attributes.ATTR_WAS_SET
  258. class LoadDeferredColumns(object):
  259. """serializable loader object used by DeferredColumnLoader"""
  260. def __init__(self, key):
  261. self.key = key
  262. def __call__(self, state, passive=attributes.PASSIVE_OFF):
  263. key = self.key
  264. localparent = state.manager.mapper
  265. prop = localparent._props[key]
  266. strategy = prop._strategies[DeferredColumnLoader]
  267. return strategy._load_for_state(state, passive)
  268. class AbstractRelationshipLoader(LoaderStrategy):
  269. """LoaderStratgies which deal with related objects."""
  270. __slots__ = 'mapper', 'target', 'uselist'
  271. def __init__(self, parent, strategy_key):
  272. super(AbstractRelationshipLoader, self).__init__(parent, strategy_key)
  273. self.mapper = self.parent_property.mapper
  274. self.target = self.parent_property.target
  275. self.uselist = self.parent_property.uselist
  276. @log.class_logger
  277. @properties.RelationshipProperty.strategy_for(lazy="noload")
  278. @properties.RelationshipProperty.strategy_for(lazy=None)
  279. class NoLoader(AbstractRelationshipLoader):
  280. """Provide loading behavior for a :class:`.RelationshipProperty`
  281. with "lazy=None".
  282. """
  283. __slots__ = ()
  284. def init_class_attribute(self, mapper):
  285. self.is_class_level = True
  286. _register_attribute(
  287. self.parent_property, mapper,
  288. useobject=True,
  289. typecallable=self.parent_property.collection_class,
  290. )
  291. def create_row_processor(
  292. self, context, path, loadopt, mapper,
  293. result, adapter, populators):
  294. def invoke_no_load(state, dict_, row):
  295. if self.uselist:
  296. state.manager.get_impl(self.key).initialize(state, dict_)
  297. else:
  298. dict_[self.key] = None
  299. populators["new"].append((self.key, invoke_no_load))
  300. @log.class_logger
  301. @properties.RelationshipProperty.strategy_for(lazy=True)
  302. @properties.RelationshipProperty.strategy_for(lazy="select")
  303. @properties.RelationshipProperty.strategy_for(lazy="raise")
  304. @properties.RelationshipProperty.strategy_for(lazy="raise_on_sql")
  305. class LazyLoader(AbstractRelationshipLoader, util.MemoizedSlots):
  306. """Provide loading behavior for a :class:`.RelationshipProperty`
  307. with "lazy=True", that is loads when first accessed.
  308. """
  309. __slots__ = (
  310. '_lazywhere', '_rev_lazywhere', 'use_get', '_bind_to_col',
  311. '_equated_columns', '_rev_bind_to_col', '_rev_equated_columns',
  312. '_simple_lazy_clause', '_raise_always', '_raise_on_sql')
  313. def __init__(self, parent, strategy_key):
  314. super(LazyLoader, self).__init__(parent, strategy_key)
  315. self._raise_always = self.strategy_opts["lazy"] == "raise"
  316. self._raise_on_sql = self.strategy_opts["lazy"] == "raise_on_sql"
  317. join_condition = self.parent_property._join_condition
  318. self._lazywhere, \
  319. self._bind_to_col, \
  320. self._equated_columns = join_condition.create_lazy_clause()
  321. self._rev_lazywhere, \
  322. self._rev_bind_to_col, \
  323. self._rev_equated_columns = join_condition.create_lazy_clause(
  324. reverse_direction=True)
  325. self.logger.info("%s lazy loading clause %s", self, self._lazywhere)
  326. # determine if our "lazywhere" clause is the same as the mapper's
  327. # get() clause. then we can just use mapper.get()
  328. self.use_get = not self.uselist and \
  329. self.mapper._get_clause[0].compare(
  330. self._lazywhere,
  331. use_proxies=True,
  332. equivalents=self.mapper._equivalent_columns
  333. )
  334. if self.use_get:
  335. for col in list(self._equated_columns):
  336. if col in self.mapper._equivalent_columns:
  337. for c in self.mapper._equivalent_columns[col]:
  338. self._equated_columns[c] = self._equated_columns[col]
  339. self.logger.info("%s will use query.get() to "
  340. "optimize instance loads", self)
  341. def init_class_attribute(self, mapper):
  342. self.is_class_level = True
  343. active_history = (
  344. self.parent_property.active_history or
  345. self.parent_property.direction is not interfaces.MANYTOONE or
  346. not self.use_get
  347. )
  348. # MANYTOONE currently only needs the
  349. # "old" value for delete-orphan
  350. # cascades. the required _SingleParentValidator
  351. # will enable active_history
  352. # in that case. otherwise we don't need the
  353. # "old" value during backref operations.
  354. _register_attribute(
  355. self.parent_property,
  356. mapper,
  357. useobject=True,
  358. callable_=self._load_for_state,
  359. typecallable=self.parent_property.collection_class,
  360. active_history=active_history
  361. )
  362. def _memoized_attr__simple_lazy_clause(self):
  363. criterion, bind_to_col = (
  364. self._lazywhere,
  365. self._bind_to_col
  366. )
  367. params = []
  368. def visit_bindparam(bindparam):
  369. bindparam.unique = False
  370. if bindparam._identifying_key in bind_to_col:
  371. params.append((
  372. bindparam.key, bind_to_col[bindparam._identifying_key],
  373. None))
  374. elif bindparam.callable is None:
  375. params.append((bindparam.key, None, bindparam.value))
  376. criterion = visitors.cloned_traverse(
  377. criterion, {}, {'bindparam': visit_bindparam}
  378. )
  379. return criterion, params
  380. def _generate_lazy_clause(self, state, passive):
  381. criterion, param_keys = self._simple_lazy_clause
  382. if state is None:
  383. return sql_util.adapt_criterion_to_null(
  384. criterion, [key for key, ident, value in param_keys])
  385. mapper = self.parent_property.parent
  386. o = state.obj() # strong ref
  387. dict_ = attributes.instance_dict(o)
  388. if passive & attributes.INIT_OK:
  389. passive ^= attributes.INIT_OK
  390. params = {}
  391. for key, ident, value in param_keys:
  392. if ident is not None:
  393. if passive and passive & attributes.LOAD_AGAINST_COMMITTED:
  394. value = mapper._get_committed_state_attr_by_column(
  395. state, dict_, ident, passive)
  396. else:
  397. value = mapper._get_state_attr_by_column(
  398. state, dict_, ident, passive)
  399. params[key] = value
  400. return criterion, params
  401. def _invoke_raise_load(self, state, passive, lazy):
  402. raise sa_exc.InvalidRequestError(
  403. "'%s' is not available due to lazy='%s'" % (self, lazy)
  404. )
  405. def _load_for_state(self, state, passive):
  406. if not state.key and (
  407. (
  408. not self.parent_property.load_on_pending
  409. and not state._load_pending
  410. )
  411. or not state.session_id
  412. ):
  413. return attributes.ATTR_EMPTY
  414. pending = not state.key
  415. ident_key = None
  416. if (
  417. (not passive & attributes.SQL_OK and not self.use_get)
  418. or
  419. (not passive & attributes.NON_PERSISTENT_OK and pending)
  420. ):
  421. return attributes.PASSIVE_NO_RESULT
  422. if self._raise_always:
  423. self._invoke_raise_load(state, passive, "raise")
  424. session = _state_session(state)
  425. if not session:
  426. raise orm_exc.DetachedInstanceError(
  427. "Parent instance %s is not bound to a Session; "
  428. "lazy load operation of attribute '%s' cannot proceed" %
  429. (orm_util.state_str(state), self.key)
  430. )
  431. # if we have a simple primary key load, check the
  432. # identity map without generating a Query at all
  433. if self.use_get:
  434. ident = self._get_ident_for_use_get(
  435. session,
  436. state,
  437. passive
  438. )
  439. if attributes.PASSIVE_NO_RESULT in ident:
  440. return attributes.PASSIVE_NO_RESULT
  441. elif attributes.NEVER_SET in ident:
  442. return attributes.NEVER_SET
  443. if _none_set.issuperset(ident):
  444. return None
  445. ident_key = self.mapper.identity_key_from_primary_key(ident)
  446. instance = loading.get_from_identity(session, ident_key, passive)
  447. if instance is not None:
  448. return instance
  449. elif not passive & attributes.SQL_OK or \
  450. not passive & attributes.RELATED_OBJECT_OK:
  451. return attributes.PASSIVE_NO_RESULT
  452. return self._emit_lazyload(session, state, ident_key, passive)
  453. def _get_ident_for_use_get(self, session, state, passive):
  454. instance_mapper = state.manager.mapper
  455. if passive & attributes.LOAD_AGAINST_COMMITTED:
  456. get_attr = instance_mapper._get_committed_state_attr_by_column
  457. else:
  458. get_attr = instance_mapper._get_state_attr_by_column
  459. dict_ = state.dict
  460. return [
  461. get_attr(
  462. state,
  463. dict_,
  464. self._equated_columns[pk],
  465. passive=passive)
  466. for pk in self.mapper.primary_key
  467. ]
  468. @util.dependencies("sqlalchemy.orm.strategy_options")
  469. def _emit_lazyload(
  470. self, strategy_options, session, state, ident_key, passive):
  471. q = session.query(self.mapper)._adapt_all_clauses()
  472. if self.parent_property.secondary is not None:
  473. q = q.select_from(self.mapper, self.parent_property.secondary)
  474. q = q._with_invoke_all_eagers(False)
  475. pending = not state.key
  476. # don't autoflush on pending
  477. if pending or passive & attributes.NO_AUTOFLUSH:
  478. q = q.autoflush(False)
  479. if state.load_path:
  480. q = q._with_current_path(state.load_path[self.parent_property])
  481. if state.load_options:
  482. q = q._conditional_options(*state.load_options)
  483. if self.use_get:
  484. if self._raise_on_sql:
  485. self._invoke_raise_load(state, passive, "raise_on_sql")
  486. return loading.load_on_ident(q, ident_key)
  487. if self.parent_property.order_by:
  488. q = q.order_by(*util.to_list(self.parent_property.order_by))
  489. for rev in self.parent_property._reverse_property:
  490. # reverse props that are MANYTOONE are loading *this*
  491. # object from get(), so don't need to eager out to those.
  492. if rev.direction is interfaces.MANYTOONE and \
  493. rev._use_get and \
  494. not isinstance(rev.strategy, LazyLoader):
  495. q = q.options(
  496. strategy_options.Load.for_existing_path(
  497. q._current_path[rev.parent]
  498. ).lazyload(rev.key)
  499. )
  500. lazy_clause, params = self._generate_lazy_clause(
  501. state, passive=passive)
  502. if pending:
  503. if util.has_intersection(
  504. orm_util._none_set, params.values()):
  505. return None
  506. elif util.has_intersection(orm_util._never_set, params.values()):
  507. return None
  508. if self._raise_on_sql:
  509. self._invoke_raise_load(state, passive, "raise_on_sql")
  510. q = q.filter(lazy_clause).params(params)
  511. result = q.all()
  512. if self.uselist:
  513. return result
  514. else:
  515. l = len(result)
  516. if l:
  517. if l > 1:
  518. util.warn(
  519. "Multiple rows returned with "
  520. "uselist=False for lazily-loaded attribute '%s' "
  521. % self.parent_property)
  522. return result[0]
  523. else:
  524. return None
  525. def create_row_processor(
  526. self, context, path, loadopt,
  527. mapper, result, adapter, populators):
  528. key = self.key
  529. if not self.is_class_level:
  530. # we are not the primary manager for this attribute
  531. # on this class - set up a
  532. # per-instance lazyloader, which will override the
  533. # class-level behavior.
  534. # this currently only happens when using a
  535. # "lazyload" option on a "no load"
  536. # attribute - "eager" attributes always have a
  537. # class-level lazyloader installed.
  538. set_lazy_callable = InstanceState._instance_level_callable_processor(
  539. mapper.class_manager,
  540. LoadLazyAttribute(key, self), key)
  541. populators["new"].append((self.key, set_lazy_callable))
  542. elif context.populate_existing or mapper.always_refresh:
  543. def reset_for_lazy_callable(state, dict_, row):
  544. # we are the primary manager for this attribute on
  545. # this class - reset its
  546. # per-instance attribute state, so that the class-level
  547. # lazy loader is
  548. # executed when next referenced on this instance.
  549. # this is needed in
  550. # populate_existing() types of scenarios to reset
  551. # any existing state.
  552. state._reset(dict_, key)
  553. populators["new"].append((self.key, reset_for_lazy_callable))
  554. class LoadLazyAttribute(object):
  555. """serializable loader object used by LazyLoader"""
  556. def __init__(self, key, initiating_strategy):
  557. self.key = key
  558. self.strategy_key = initiating_strategy.strategy_key
  559. def __call__(self, state, passive=attributes.PASSIVE_OFF):
  560. key = self.key
  561. instance_mapper = state.manager.mapper
  562. prop = instance_mapper._props[key]
  563. strategy = prop._strategies[self.strategy_key]
  564. return strategy._load_for_state(state, passive)
  565. @properties.RelationshipProperty.strategy_for(lazy="immediate")
  566. class ImmediateLoader(AbstractRelationshipLoader):
  567. __slots__ = ()
  568. def init_class_attribute(self, mapper):
  569. self.parent_property.\
  570. _get_strategy((("lazy", "select"),)).\
  571. init_class_attribute(mapper)
  572. def setup_query(
  573. self, context, entity,
  574. path, loadopt, adapter, column_collection=None,
  575. parentmapper=None, **kwargs):
  576. pass
  577. def create_row_processor(
  578. self, context, path, loadopt,
  579. mapper, result, adapter, populators):
  580. def load_immediate(state, dict_, row):
  581. state.get_impl(self.key).get(state, dict_)
  582. populators["delayed"].append((self.key, load_immediate))
  583. @log.class_logger
  584. @properties.RelationshipProperty.strategy_for(lazy="subquery")
  585. class SubqueryLoader(AbstractRelationshipLoader):
  586. __slots__ = 'join_depth',
  587. def __init__(self, parent, strategy_key):
  588. super(SubqueryLoader, self).__init__(parent, strategy_key)
  589. self.join_depth = self.parent_property.join_depth
  590. def init_class_attribute(self, mapper):
  591. self.parent_property.\
  592. _get_strategy((("lazy", "select"),)).\
  593. init_class_attribute(mapper)
  594. def setup_query(
  595. self, context, entity,
  596. path, loadopt, adapter,
  597. column_collection=None,
  598. parentmapper=None, **kwargs):
  599. if not context.query._enable_eagerloads:
  600. return
  601. elif context.query._yield_per:
  602. context.query._no_yield_per("subquery")
  603. path = path[self.parent_property]
  604. # build up a path indicating the path from the leftmost
  605. # entity to the thing we're subquery loading.
  606. with_poly_info = path.get(
  607. context.attributes,
  608. "path_with_polymorphic", None)
  609. if with_poly_info is not None:
  610. effective_entity = with_poly_info.entity
  611. else:
  612. effective_entity = self.mapper
  613. subq_path = context.attributes.get(
  614. ('subquery_path', None),
  615. orm_util.PathRegistry.root)
  616. subq_path = subq_path + path
  617. # if not via query option, check for
  618. # a cycle
  619. if not path.contains(context.attributes, "loader"):
  620. if self.join_depth:
  621. if path.length / 2 > self.join_depth:
  622. return
  623. elif subq_path.contains_mapper(self.mapper):
  624. return
  625. leftmost_mapper, leftmost_attr, leftmost_relationship = \
  626. self._get_leftmost(subq_path)
  627. orig_query = context.attributes.get(
  628. ("orig_query", SubqueryLoader),
  629. context.query)
  630. # generate a new Query from the original, then
  631. # produce a subquery from it.
  632. left_alias = self._generate_from_original_query(
  633. orig_query, leftmost_mapper,
  634. leftmost_attr, leftmost_relationship,
  635. entity.entity_zero
  636. )
  637. # generate another Query that will join the
  638. # left alias to the target relationships.
  639. # basically doing a longhand
  640. # "from_self()". (from_self() itself not quite industrial
  641. # strength enough for all contingencies...but very close)
  642. q = orig_query.session.query(effective_entity)
  643. q._attributes = {
  644. ("orig_query", SubqueryLoader): orig_query,
  645. ('subquery_path', None): subq_path
  646. }
  647. q = q._set_enable_single_crit(False)
  648. to_join, local_attr, parent_alias = \
  649. self._prep_for_joins(left_alias, subq_path)
  650. q = q.order_by(*local_attr)
  651. q = q.add_columns(*local_attr)
  652. q = self._apply_joins(
  653. q, to_join, left_alias,
  654. parent_alias, effective_entity)
  655. q = self._setup_options(q, subq_path, orig_query, effective_entity)
  656. q = self._setup_outermost_orderby(q)
  657. # add new query to attributes to be picked up
  658. # by create_row_processor
  659. path.set(context.attributes, "subquery", q)
  660. def _get_leftmost(self, subq_path):
  661. subq_path = subq_path.path
  662. subq_mapper = orm_util._class_to_mapper(subq_path[0])
  663. # determine attributes of the leftmost mapper
  664. if self.parent.isa(subq_mapper) and \
  665. self.parent_property is subq_path[1]:
  666. leftmost_mapper, leftmost_prop = \
  667. self.parent, self.parent_property
  668. else:
  669. leftmost_mapper, leftmost_prop = \
  670. subq_mapper, \
  671. subq_path[1]
  672. leftmost_cols = leftmost_prop.local_columns
  673. leftmost_attr = [
  674. getattr(
  675. subq_path[0].entity,
  676. leftmost_mapper._columntoproperty[c].key)
  677. for c in leftmost_cols
  678. ]
  679. return leftmost_mapper, leftmost_attr, leftmost_prop
  680. def _generate_from_original_query(
  681. self,
  682. orig_query, leftmost_mapper,
  683. leftmost_attr, leftmost_relationship, orig_entity
  684. ):
  685. # reformat the original query
  686. # to look only for significant columns
  687. q = orig_query._clone().correlate(None)
  688. # set a real "from" if not present, as this is more
  689. # accurate than just going off of the column expression
  690. if not q._from_obj and orig_entity.is_mapper and \
  691. orig_entity.mapper.isa(leftmost_mapper):
  692. q._set_select_from([orig_entity], False)
  693. target_cols = q._adapt_col_list(leftmost_attr)
  694. # select from the identity columns of the outer. This will remove
  695. # other columns from the query that might suggest the right entity
  696. # which is why we try to _set_select_from above.
  697. q._set_entities(target_cols)
  698. distinct_target_key = leftmost_relationship.distinct_target_key
  699. if distinct_target_key is True:
  700. q._distinct = True
  701. elif distinct_target_key is None:
  702. # if target_cols refer to a non-primary key or only
  703. # part of a composite primary key, set the q as distinct
  704. for t in set(c.table for c in target_cols):
  705. if not set(target_cols).issuperset(t.primary_key):
  706. q._distinct = True
  707. break
  708. if q._order_by is False:
  709. q._order_by = leftmost_mapper.order_by
  710. # don't need ORDER BY if no limit/offset
  711. if q._limit is None and q._offset is None:
  712. q._order_by = None
  713. # the original query now becomes a subquery
  714. # which we'll join onto.
  715. embed_q = q.with_labels().subquery()
  716. left_alias = orm_util.AliasedClass(
  717. leftmost_mapper, embed_q,
  718. use_mapper_path=True)
  719. return left_alias
  720. def _prep_for_joins(self, left_alias, subq_path):
  721. # figure out what's being joined. a.k.a. the fun part
  722. to_join = []
  723. pairs = list(subq_path.pairs())
  724. for i, (mapper, prop) in enumerate(pairs):
  725. if i > 0:
  726. # look at the previous mapper in the chain -
  727. # if it is as or more specific than this prop's
  728. # mapper, use that instead.
  729. # note we have an assumption here that
  730. # the non-first element is always going to be a mapper,
  731. # not an AliasedClass
  732. prev_mapper = pairs[i - 1][1].mapper
  733. to_append = prev_mapper if prev_mapper.isa(mapper) else mapper
  734. else:
  735. to_append = mapper
  736. to_join.append((to_append, prop.key))
  737. # determine the immediate parent class we are joining from,
  738. # which needs to be aliased.
  739. if len(to_join) < 2:
  740. # in the case of a one level eager load, this is the
  741. # leftmost "left_alias".
  742. parent_alias = left_alias
  743. else:
  744. info = inspect(to_join[-1][0])
  745. if info.is_aliased_class:
  746. parent_alias = info.entity
  747. else:
  748. # alias a plain mapper as we may be
  749. # joining multiple times
  750. parent_alias = orm_util.AliasedClass(
  751. info.entity,
  752. use_mapper_path=True)
  753. local_cols = self.parent_property.local_columns
  754. local_attr = [
  755. getattr(parent_alias, self.parent._columntoproperty[c].key)
  756. for c in local_cols
  757. ]
  758. return to_join, local_attr, parent_alias
  759. def _apply_joins(
  760. self, q, to_join, left_alias, parent_alias,
  761. effective_entity):
  762. ltj = len(to_join)
  763. if ltj == 1:
  764. to_join = [
  765. getattr(left_alias, to_join[0][1]).of_type(effective_entity)
  766. ]
  767. elif ltj == 2:
  768. to_join = [
  769. getattr(left_alias, to_join[0][1]).of_type(parent_alias),
  770. getattr(parent_alias, to_join[-1][1]).of_type(effective_entity)
  771. ]
  772. elif ltj > 2:
  773. middle = [
  774. (
  775. orm_util.AliasedClass(item[0])
  776. if not inspect(item[0]).is_aliased_class
  777. else item[0].entity,
  778. item[1]
  779. ) for item in to_join[1:-1]
  780. ]
  781. inner = []
  782. while middle:
  783. item = middle.pop(0)
  784. attr = getattr(item[0], item[1])
  785. if middle:
  786. attr = attr.of_type(middle[0][0])
  787. else:
  788. attr = attr.of_type(parent_alias)
  789. inner.append(attr)
  790. to_join = [
  791. getattr(left_alias, to_join[0][1]).of_type(inner[0].parent)
  792. ] + inner + [
  793. getattr(parent_alias, to_join[-1][1]).of_type(effective_entity)
  794. ]
  795. for attr in to_join:
  796. q = q.join(attr, from_joinpoint=True)
  797. return q
  798. def _setup_options(self, q, subq_path, orig_query, effective_entity):
  799. # propagate loader options etc. to the new query.
  800. # these will fire relative to subq_path.
  801. q = q._with_current_path(subq_path)
  802. q = q._conditional_options(*orig_query._with_options)
  803. if orig_query._populate_existing:
  804. q._populate_existing = orig_query._populate_existing
  805. return q
  806. def _setup_outermost_orderby(self, q):
  807. if self.parent_property.order_by:
  808. # if there's an ORDER BY, alias it the same
  809. # way joinedloader does, but we have to pull out
  810. # the "eagerjoin" from the query.
  811. # this really only picks up the "secondary" table
  812. # right now.
  813. eagerjoin = q._from_obj[0]
  814. eager_order_by = \
  815. eagerjoin._target_adapter.\
  816. copy_and_process(
  817. util.to_list(
  818. self.parent_property.order_by
  819. )
  820. )
  821. q = q.order_by(*eager_order_by)
  822. return q
  823. class _SubqCollections(object):
  824. """Given a :class:`.Query` used to emit the "subquery load",
  825. provide a load interface that executes the query at the
  826. first moment a value is needed.
  827. """
  828. _data = None
  829. def __init__(self, subq):
  830. self.subq = subq
  831. def get(self, key, default):
  832. if self._data is None:
  833. self._load()
  834. return self._data.get(key, default)
  835. def _load(self):
  836. self._data = dict(
  837. (k, [vv[0] for vv in v])
  838. for k, v in itertools.groupby(
  839. self.subq,
  840. lambda x: x[1:]
  841. )
  842. )
  843. def loader(self, state, dict_, row):
  844. if self._data is None:
  845. self._load()
  846. def create_row_processor(
  847. self, context, path, loadopt,
  848. mapper, result, adapter, populators):
  849. if not self.parent.class_manager[self.key].impl.supports_population:
  850. raise sa_exc.InvalidRequestError(
  851. "'%s' does not support object "
  852. "population - eager loading cannot be applied." %
  853. self)
  854. path = path[self.parent_property]
  855. subq = path.get(context.attributes, 'subquery')
  856. if subq is None:
  857. return
  858. assert subq.session is context.session, (
  859. "Subquery session doesn't refer to that of "
  860. "our context. Are there broken context caching "
  861. "schemes being used?"
  862. )
  863. local_cols = self.parent_property.local_columns
  864. # cache the loaded collections in the context
  865. # so that inheriting mappers don't re-load when they
  866. # call upon create_row_processor again
  867. collections = path.get(context.attributes, "collections")
  868. if collections is None:
  869. collections = self._SubqCollections(subq)
  870. path.set(context.attributes, 'collections', collections)
  871. if adapter:
  872. local_cols = [adapter.columns[c] for c in local_cols]
  873. if self.uselist:
  874. self._create_collection_loader(
  875. context, collections, local_cols, populators)
  876. else:
  877. self._create_scalar_loader(
  878. context, collections, local_cols, populators)
  879. def _create_collection_loader(
  880. self, context, collections, local_cols, populators):
  881. def load_collection_from_subq(state, dict_, row):
  882. collection = collections.get(
  883. tuple([row[col] for col in local_cols]),
  884. ()
  885. )
  886. state.get_impl(self.key).\
  887. set_committed_value(state, dict_, collection)
  888. def load_collection_from_subq_existing_row(state, dict_, row):
  889. if self.key not in dict_:
  890. load_collection_from_subq(state, dict_, row)
  891. populators["new"].append(
  892. (self.key, load_collection_from_subq))
  893. populators["existing"].append(
  894. (self.key, load_collection_from_subq_existing_row))
  895. if context.invoke_all_eagers:
  896. populators["eager"].append((self.key, collections.loader))
  897. def _create_scalar_loader(
  898. self, context, collections, local_cols, populators):
  899. def load_scalar_from_subq(state, dict_, row):
  900. collection = collections.get(
  901. tuple([row[col] for col in local_cols]),
  902. (None,)
  903. )
  904. if len(collection) > 1:
  905. util.warn(
  906. "Multiple rows returned with "
  907. "uselist=False for eagerly-loaded attribute '%s' "
  908. % self)
  909. scalar = collection[0]
  910. state.get_impl(self.key).\
  911. set_committed_value(state, dict_, scalar)
  912. def load_scalar_from_subq_existing_row(state, dict_, row):
  913. if self.key not in dict_:
  914. load_scalar_from_subq(state, dict_, row)
  915. populators["new"].append(
  916. (self.key, load_scalar_from_subq))
  917. populators["existing"].append(
  918. (self.key, load_scalar_from_subq_existing_row))
  919. if context.invoke_all_eagers:
  920. populators["eager"].append((self.key, collections.loader))
  921. @log.class_logger
  922. @properties.RelationshipProperty.strategy_for(lazy="joined")
  923. @properties.RelationshipProperty.strategy_for(lazy=False)
  924. class JoinedLoader(AbstractRelationshipLoader):
  925. """Provide loading behavior for a :class:`.RelationshipProperty`
  926. using joined eager loading.
  927. """
  928. __slots__ = 'join_depth', '_aliased_class_pool'
  929. def __init__(self, parent, strategy_key):
  930. super(JoinedLoader, self).__init__(parent, strategy_key)
  931. self.join_depth = self.parent_property.join_depth
  932. self._aliased_class_pool = []
  933. def init_class_attribute(self, mapper):
  934. self.parent_property.\
  935. _get_strategy((("lazy", "select"),)).init_class_attribute(mapper)
  936. def setup_query(
  937. self, context, entity, path, loadopt, adapter,
  938. column_collection=None, parentmapper=None,
  939. chained_from_outerjoin=False,
  940. **kwargs):
  941. """Add a left outer join to the statement that's being constructed."""
  942. if not context.query._enable_eagerloads:
  943. return
  944. elif context.query._yield_per and self.uselist:
  945. context.query._no_yield_per("joined collection")
  946. path = path[self.parent_property]
  947. with_polymorphic = None
  948. user_defined_adapter = self._init_user_defined_eager_proc(
  949. loadopt, context) if loadopt else False
  950. if user_defined_adapter is not False:
  951. clauses, adapter, add_to_collection = \
  952. self._setup_query_on_user_defined_adapter(
  953. context, entity, path, adapter,
  954. user_defined_adapter
  955. )
  956. else:
  957. # if not via query option, check for
  958. # a cycle
  959. if not path.contains(context.attributes, "loader"):
  960. if self.join_depth:
  961. if path.length / 2 > self.join_depth:
  962. return
  963. elif path.contains_mapper(self.mapper):
  964. return
  965. clauses, adapter, add_to_collection, chained_from_outerjoin = \
  966. self._generate_row_adapter(
  967. context, entity, path, loadopt, adapter,
  968. column_collection, parentmapper, chained_from_outerjoin
  969. )
  970. with_poly_info = path.get(
  971. context.attributes,
  972. "path_with_polymorphic",
  973. None
  974. )
  975. if with_poly_info is not None:
  976. with_polymorphic = with_poly_info.with_polymorphic_mappers
  977. else:
  978. with_polymorphic = None
  979. path = path[self.mapper]
  980. loading._setup_entity_query(
  981. context, self.mapper, entity,
  982. path, clauses, add_to_collection,
  983. with_polymorphic=with_polymorphic,
  984. parentmapper=self.mapper,
  985. chained_from_outerjoin=chained_from_outerjoin)
  986. if with_poly_info is not None and \
  987. None in set(context.secondary_columns):
  988. raise sa_exc.InvalidRequestError(
  989. "Detected unaliased columns when generating joined "
  990. "load. Make sure to use aliased=True or flat=True "
  991. "when using joined loading with with_polymorphic()."
  992. )
  993. def _init_user_defined_eager_proc(self, loadopt, context):
  994. # check if the opt applies at all
  995. if "eager_from_alias" not in loadopt.local_opts:
  996. # nope
  997. return False
  998. path = loadopt.path.parent
  999. # the option applies. check if the "user_defined_eager_row_processor"
  1000. # has been built up.
  1001. adapter = path.get(
  1002. context.attributes,
  1003. "user_defined_eager_row_processor", False)
  1004. if adapter is not False:
  1005. # just return it
  1006. return adapter
  1007. # otherwise figure it out.
  1008. alias = loadopt.local_opts["eager_from_alias"]
  1009. root_mapper, prop = path[-2:]
  1010. #from .mapper import Mapper
  1011. #from .interfaces import MapperProperty
  1012. #assert isinstance(root_mapper, Mapper)
  1013. #assert isinstance(prop, MapperProperty)
  1014. if alias is not None:
  1015. if isinstance(alias, str):
  1016. alias = prop.target.alias(alias)
  1017. adapter = sql_util.ColumnAdapter(
  1018. alias,
  1019. equivalents=prop.mapper._equivalent_columns)
  1020. else:
  1021. if path.contains(context.attributes, "path_with_polymorphic"):
  1022. with_poly_info = path.get(
  1023. context.attributes,
  1024. "path_with_polymorphic")
  1025. adapter = orm_util.ORMAdapter(
  1026. with_poly_info.entity,
  1027. equivalents=prop.mapper._equivalent_columns)
  1028. else:
  1029. adapter = context.query._polymorphic_adapters.get(
  1030. prop.mapper, None)
  1031. path.set(
  1032. context.attributes,
  1033. "user_defined_eager_row_processor",
  1034. adapter)
  1035. return adapter
  1036. def _setup_query_on_user_defined_adapter(
  1037. self, context, entity,
  1038. path, adapter, user_defined_adapter):
  1039. # apply some more wrapping to the "user defined adapter"
  1040. # if we are setting up the query for SQL render.
  1041. adapter = entity._get_entity_clauses(context.query, context)
  1042. if adapter and user_defined_adapter:
  1043. user_defined_adapter = user_defined_adapter.wrap(adapter)
  1044. path.set(
  1045. context.attributes, "user_defined_eager_row_processor",
  1046. user_defined_adapter)
  1047. elif adapter:
  1048. user_defined_adapter = adapter
  1049. path.set(
  1050. context.attributes, "user_defined_eager_row_processor",
  1051. user_defined_adapter)
  1052. add_to_collection = context.primary_columns
  1053. return user_defined_adapter, adapter, add_to_collection
  1054. def _gen_pooled_aliased_class(self, context):
  1055. # keep a local pool of AliasedClass objects that get re-used.
  1056. # we need one unique AliasedClass per query per appearance of our
  1057. # entity in the query.
  1058. key = ('joinedloader_ac', self)
  1059. if key not in context.attributes:
  1060. context.attributes[key] = idx = 0
  1061. else:
  1062. context.attributes[key] = idx = context.attributes[key] + 1
  1063. if idx >= len(self._aliased_class_pool):
  1064. to_adapt = orm_util.AliasedClass(
  1065. self.mapper,
  1066. flat=True,
  1067. use_mapper_path=True)
  1068. # load up the .columns collection on the Alias() before
  1069. # the object becomes shared among threads. this prevents
  1070. # races for column identities.
  1071. inspect(to_adapt).selectable.c
  1072. self._aliased_class_pool.append(to_adapt)
  1073. return self._aliased_class_pool[idx]
  1074. def _generate_row_adapter(
  1075. self,
  1076. context, entity, path, loadopt, adapter,
  1077. column_collection, parentmapper, chained_from_outerjoin):
  1078. with_poly_info = path.get(
  1079. context.attributes,
  1080. "path_with_polymorphic",
  1081. None
  1082. )
  1083. if with_poly_info:
  1084. to_adapt = with_poly_info.entity
  1085. else:
  1086. to_adapt = self._gen_pooled_aliased_class(context)
  1087. clauses = inspect(to_adapt)._memo(
  1088. ("joinedloader_ormadapter", self),
  1089. orm_util.ORMAdapter,
  1090. to_adapt,
  1091. equivalents=self.mapper._equivalent_columns,
  1092. adapt_required=True, allow_label_resolve=False,
  1093. anonymize_labels=True
  1094. )
  1095. assert clauses.aliased_class is not None
  1096. if self.parent_property.uselist:
  1097. context.multi_row_eager_loaders = True
  1098. innerjoin = (
  1099. loadopt.local_opts.get(
  1100. 'innerjoin', self.parent_property.innerjoin)
  1101. if loadopt is not None
  1102. else self.parent_property.innerjoin
  1103. )
  1104. if not innerjoin:
  1105. # if this is an outer join, all non-nested eager joins from
  1106. # this path must also be outer joins
  1107. chained_from_outerjoin = True
  1108. context.create_eager_joins.append(
  1109. (
  1110. self._create_eager_join, context,
  1111. entity, path, adapter,
  1112. parentmapper, clauses, innerjoin, chained_from_outerjoin
  1113. )
  1114. )
  1115. add_to_collection = context.secondary_columns
  1116. path.set(context.attributes, "eager_row_processor", clauses)
  1117. return clauses, adapter, add_to_collection, chained_from_outerjoin
  1118. def _create_eager_join(
  1119. self, context, entity,
  1120. path, adapter, parentmapper,
  1121. clauses, innerjoin, chained_from_outerjoin):
  1122. if parentmapper is None:
  1123. localparent = entity.mapper
  1124. else:
  1125. localparent = parentmapper
  1126. # whether or not the Query will wrap the selectable in a subquery,
  1127. # and then attach eager load joins to that (i.e., in the case of
  1128. # LIMIT/OFFSET etc.)
  1129. should_nest_selectable = context.multi_row_eager_loaders and \
  1130. context.query._should_nest_selectable
  1131. entity_key = None
  1132. if entity not in context.eager_joins and \
  1133. not should_nest_selectable and \
  1134. context.from_clause:
  1135. index, clause = sql_util.find_join_source(
  1136. context.from_clause, entity.selectable)
  1137. if clause is not None:
  1138. # join to an existing FROM clause on the query.
  1139. # key it to its list index in the eager_joins dict.
  1140. # Query._compile_context will adapt as needed and
  1141. # append to the FROM clause of the select().
  1142. entity_key, default_towrap = index, clause
  1143. if entity_key is None:
  1144. entity_key, default_towrap = entity, entity.selectable
  1145. towrap = context.eager_joins.setdefault(entity_key, default_towrap)
  1146. if adapter:
  1147. if getattr(adapter, 'aliased_class', None):
  1148. # joining from an adapted entity. The adapted entity
  1149. # might be a "with_polymorphic", so resolve that to our
  1150. # specific mapper's entity before looking for our attribute
  1151. # name on it.
  1152. efm = inspect(adapter.aliased_class).\
  1153. _entity_for_mapper(
  1154. localparent
  1155. if localparent.isa(self.parent) else self.parent)
  1156. # look for our attribute on the adapted entity, else fall back
  1157. # to our straight property
  1158. onclause = getattr(
  1159. efm.entity, self.key,
  1160. self.parent_property)
  1161. else:
  1162. onclause = getattr(
  1163. orm_util.AliasedClass(
  1164. self.parent,
  1165. adapter.selectable,
  1166. use_mapper_path=True
  1167. ),
  1168. self.key, self.parent_property
  1169. )
  1170. else:
  1171. onclause = self.parent_property
  1172. assert clauses.aliased_class is not None
  1173. attach_on_outside = (
  1174. not chained_from_outerjoin or
  1175. not innerjoin or innerjoin == 'unnested')
  1176. if attach_on_outside:
  1177. # this is the "classic" eager join case.
  1178. eagerjoin = orm_util._ORMJoin(
  1179. towrap,
  1180. clauses.aliased_class,
  1181. onclause,
  1182. isouter=not innerjoin or (
  1183. chained_from_outerjoin and isinstance(towrap, sql.Join)
  1184. ), _left_memo=self.parent, _right_memo=self.mapper
  1185. )
  1186. else:
  1187. # all other cases are innerjoin=='nested' approach
  1188. eagerjoin = self._splice_nested_inner_join(
  1189. path, towrap, clauses, onclause)
  1190. context.eager_joins[entity_key] = eagerjoin
  1191. # send a hint to the Query as to where it may "splice" this join
  1192. eagerjoin.stop_on = entity.selectable
  1193. if not parentmapper:
  1194. # for parentclause that is the non-eager end of the join,
  1195. # ensure all the parent cols in the primaryjoin are actually
  1196. # in the
  1197. # columns clause (i.e. are not deferred), so that aliasing applied
  1198. # by the Query propagates those columns outward.
  1199. # This has the effect
  1200. # of "undefering" those columns.
  1201. for col in sql_util._find_columns(
  1202. self.parent_property.primaryjoin):
  1203. if localparent.mapped_table.c.contains_column(col):
  1204. if adapter:
  1205. col = adapter.columns[col]
  1206. context.primary_columns.append(col)
  1207. if self.parent_property.order_by:
  1208. context.eager_order_by += eagerjoin._target_adapter.\
  1209. copy_and_process(
  1210. util.to_list(
  1211. self.parent_property.order_by
  1212. )
  1213. )
  1214. def _splice_nested_inner_join(
  1215. self, path, join_obj, clauses, onclause, splicing=False):
  1216. if splicing is False:
  1217. # first call is always handed a join object
  1218. # from the outside
  1219. assert isinstance(join_obj, orm_util._ORMJoin)
  1220. elif isinstance(join_obj, sql.selectable.FromGrouping):
  1221. return self._splice_nested_inner_join(
  1222. path, join_obj.element, clauses, onclause, splicing
  1223. )
  1224. elif not isinstance(join_obj, orm_util._ORMJoin):
  1225. if path[-2] is splicing:
  1226. return orm_util._ORMJoin(
  1227. join_obj, clauses.aliased_class,
  1228. onclause, isouter=False,
  1229. _left_memo=splicing,
  1230. _right_memo=path[-1].mapper
  1231. )
  1232. else:
  1233. # only here if splicing == True
  1234. return None
  1235. target_join = self._splice_nested_inner_join(
  1236. path, join_obj.right, clauses,
  1237. onclause, join_obj._right_memo)
  1238. if target_join is None:
  1239. right_splice = False
  1240. target_join = self._splice_nested_inner_join(
  1241. path, join_obj.left, clauses,
  1242. onclause, join_obj._left_memo)
  1243. if target_join is None:
  1244. # should only return None when recursively called,
  1245. # e.g. splicing==True
  1246. assert splicing is not False, \
  1247. "assertion failed attempting to produce joined eager loads"
  1248. return None
  1249. else:
  1250. right_splice = True
  1251. if right_splice:
  1252. # for a right splice, attempt to flatten out
  1253. # a JOIN b JOIN c JOIN .. to avoid needless
  1254. # parenthesis nesting
  1255. if not join_obj.isouter and not target_join.isouter:
  1256. eagerjoin = join_obj._splice_into_center(target_join)
  1257. else:
  1258. eagerjoin = orm_util._ORMJoin(
  1259. join_obj.left, target_join,
  1260. join_obj.onclause, isouter=join_obj.isouter,
  1261. _left_memo=join_obj._left_memo)
  1262. else:
  1263. eagerjoin = orm_util._ORMJoin(
  1264. target_join, join_obj.right,
  1265. join_obj.onclause, isouter=join_obj.isouter,
  1266. _right_memo=join_obj._right_memo)
  1267. eagerjoin._target_adapter = target_join._target_adapter
  1268. return eagerjoin
  1269. def _create_eager_adapter(self, context, result, adapter, path, loadopt):
  1270. user_defined_adapter = self._init_user_defined_eager_proc(
  1271. loadopt, context) if loadopt else False
  1272. if user_defined_adapter is not False:
  1273. decorator = user_defined_adapter
  1274. # user defined eagerloads are part of the "primary"
  1275. # portion of the load.
  1276. # the adapters applied to the Query should be honored.
  1277. if context.adapter and decorator:
  1278. decorator = decorator.wrap(context.adapter)
  1279. elif context.adapter:
  1280. decorator = context.adapter
  1281. else:
  1282. decorator = path.get(context.attributes, "eager_row_processor")
  1283. if decorator is None:
  1284. return False
  1285. if self.mapper._result_has_identity_key(result, decorator):
  1286. return decorator
  1287. else:
  1288. # no identity key - don't return a row
  1289. # processor, will cause a degrade to lazy
  1290. return False
  1291. def create_row_processor(
  1292. self, context, path, loadopt, mapper,
  1293. result, adapter, populators):
  1294. if not self.parent.class_manager[self.key].impl.supports_population:
  1295. raise sa_exc.InvalidRequestError(
  1296. "'%s' does not support object "
  1297. "population - eager loading cannot be applied." %
  1298. self
  1299. )
  1300. our_path = path[self.parent_property]
  1301. eager_adapter = self._create_eager_adapter(
  1302. context,
  1303. result,
  1304. adapter, our_path, loadopt)
  1305. if eager_adapter is not False:
  1306. key = self.key
  1307. _instance = loading._instance_processor(
  1308. self.mapper,
  1309. context,
  1310. result,
  1311. our_path[self.mapper],
  1312. eager_adapter)
  1313. if not self.uselist:
  1314. self._create_scalar_loader(context, key, _instance, populators)
  1315. else:
  1316. self._create_collection_loader(
  1317. context, key, _instance, populators)
  1318. else:
  1319. self.parent_property._get_strategy((("lazy", "select"),)).\
  1320. create_row_processor(
  1321. context, path, loadopt,
  1322. mapper, result, adapter, populators)
  1323. def _create_collection_loader(self, context, key, _instance, populators):
  1324. def load_collection_from_joined_new_row(state, dict_, row):
  1325. collection = attributes.init_state_collection(
  1326. state, dict_, key)
  1327. result_list = util.UniqueAppender(collection,
  1328. 'append_without_event')
  1329. context.attributes[(state, key)] = result_list
  1330. inst = _instance(row)
  1331. if inst is not None:
  1332. result_list.append(inst)
  1333. def load_collection_from_joined_existing_row(state, dict_, row):
  1334. if (state, key) in context.attributes:
  1335. result_list = context.attributes[(state, key)]
  1336. else:
  1337. # appender_key can be absent from context.attributes
  1338. # with isnew=False when self-referential eager loading
  1339. # is used; the same instance may be present in two
  1340. # distinct sets of result columns
  1341. collection = attributes.init_state_collection(
  1342. state, dict_, key)
  1343. result_list = util.UniqueAppender(
  1344. collection,
  1345. 'append_without_event')
  1346. context.attributes[(state, key)] = result_list
  1347. inst = _instance(row)
  1348. if inst is not None:
  1349. result_list.append(inst)
  1350. def load_collection_from_joined_exec(state, dict_, row):
  1351. _instance(row)
  1352. populators["new"].append((self.key, load_collection_from_joined_new_row))
  1353. populators["existing"].append(
  1354. (self.key, load_collection_from_joined_existing_row))
  1355. if context.invoke_all_eagers:
  1356. populators["eager"].append(
  1357. (self.key, load_collection_from_joined_exec))
  1358. def _create_scalar_loader(self, context, key, _instance, populators):
  1359. def load_scalar_from_joined_new_row(state, dict_, row):
  1360. # set a scalar object instance directly on the parent
  1361. # object, bypassing InstrumentedAttribute event handlers.
  1362. dict_[key] = _instance(row)
  1363. def load_scalar_from_joined_existing_row(state, dict_, row):
  1364. # call _instance on the row, even though the object has
  1365. # been created, so that we further descend into properties
  1366. existing = _instance(row)
  1367. # conflicting value already loaded, this shouldn't happen
  1368. if key in dict_:
  1369. if existing is not dict_[key]:
  1370. util.warn(
  1371. "Multiple rows returned with "
  1372. "uselist=False for eagerly-loaded attribute '%s' "
  1373. % self)
  1374. else:
  1375. # this case is when one row has multiple loads of the
  1376. # same entity (e.g. via aliasing), one has an attribute
  1377. # that the other doesn't.
  1378. dict_[key] = existing
  1379. def load_scalar_from_joined_exec(state, dict_, row):
  1380. _instance(row)
  1381. populators["new"].append((self.key, load_scalar_from_joined_new_row))
  1382. populators["existing"].append(
  1383. (self.key, load_scalar_from_joined_existing_row))
  1384. if context.invoke_all_eagers:
  1385. populators["eager"].append((self.key, load_scalar_from_joined_exec))
  1386. def single_parent_validator(desc, prop):
  1387. def _do_check(state, value, oldvalue, initiator):
  1388. if value is not None and initiator.key == prop.key:
  1389. hasparent = initiator.hasparent(attributes.instance_state(value))
  1390. if hasparent and oldvalue is not value:
  1391. raise sa_exc.InvalidRequestError(
  1392. "Instance %s is already associated with an instance "
  1393. "of %s via its %s attribute, and is only allowed a "
  1394. "single parent." %
  1395. (orm_util.instance_str(value), state.class_, prop)
  1396. )
  1397. return value
  1398. def append(state, value, initiator):
  1399. return _do_check(state, value, None, initiator)
  1400. def set_(state, value, oldvalue, initiator):
  1401. return _do_check(state, value, oldvalue, initiator)
  1402. event.listen(
  1403. desc, 'append', append, raw=True, retval=True,
  1404. active_history=True)
  1405. event.listen(
  1406. desc, 'set', set_, raw=True, retval=True,
  1407. active_history=True)