pysqlite.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377
  1. # sqlite/pysqlite.py
  2. # Copyright (C) 2005-2017 the SQLAlchemy authors and contributors
  3. # <see AUTHORS file>
  4. #
  5. # This module is part of SQLAlchemy and is released under
  6. # the MIT License: http://www.opensource.org/licenses/mit-license.php
  7. r"""
  8. .. dialect:: sqlite+pysqlite
  9. :name: pysqlite
  10. :dbapi: sqlite3
  11. :connectstring: sqlite+pysqlite:///file_path
  12. :url: http://docs.python.org/library/sqlite3.html
  13. Note that ``pysqlite`` is the same driver as the ``sqlite3``
  14. module included with the Python distribution.
  15. Driver
  16. ------
  17. When using Python 2.5 and above, the built in ``sqlite3`` driver is
  18. already installed and no additional installation is needed. Otherwise,
  19. the ``pysqlite2`` driver needs to be present. This is the same driver as
  20. ``sqlite3``, just with a different name.
  21. The ``pysqlite2`` driver will be loaded first, and if not found, ``sqlite3``
  22. is loaded. This allows an explicitly installed pysqlite driver to take
  23. precedence over the built in one. As with all dialects, a specific
  24. DBAPI module may be provided to :func:`~sqlalchemy.create_engine()` to control
  25. this explicitly::
  26. from sqlite3 import dbapi2 as sqlite
  27. e = create_engine('sqlite+pysqlite:///file.db', module=sqlite)
  28. Connect Strings
  29. ---------------
  30. The file specification for the SQLite database is taken as the "database"
  31. portion of the URL. Note that the format of a SQLAlchemy url is::
  32. driver://user:pass@host/database
  33. This means that the actual filename to be used starts with the characters to
  34. the **right** of the third slash. So connecting to a relative filepath
  35. looks like::
  36. # relative path
  37. e = create_engine('sqlite:///path/to/database.db')
  38. An absolute path, which is denoted by starting with a slash, means you
  39. need **four** slashes::
  40. # absolute path
  41. e = create_engine('sqlite:////path/to/database.db')
  42. To use a Windows path, regular drive specifications and backslashes can be
  43. used. Double backslashes are probably needed::
  44. # absolute path on Windows
  45. e = create_engine('sqlite:///C:\\path\\to\\database.db')
  46. The sqlite ``:memory:`` identifier is the default if no filepath is
  47. present. Specify ``sqlite://`` and nothing else::
  48. # in-memory database
  49. e = create_engine('sqlite://')
  50. Compatibility with sqlite3 "native" date and datetime types
  51. -----------------------------------------------------------
  52. The pysqlite driver includes the sqlite3.PARSE_DECLTYPES and
  53. sqlite3.PARSE_COLNAMES options, which have the effect of any column
  54. or expression explicitly cast as "date" or "timestamp" will be converted
  55. to a Python date or datetime object. The date and datetime types provided
  56. with the pysqlite dialect are not currently compatible with these options,
  57. since they render the ISO date/datetime including microseconds, which
  58. pysqlite's driver does not. Additionally, SQLAlchemy does not at
  59. this time automatically render the "cast" syntax required for the
  60. freestanding functions "current_timestamp" and "current_date" to return
  61. datetime/date types natively. Unfortunately, pysqlite
  62. does not provide the standard DBAPI types in ``cursor.description``,
  63. leaving SQLAlchemy with no way to detect these types on the fly
  64. without expensive per-row type checks.
  65. Keeping in mind that pysqlite's parsing option is not recommended,
  66. nor should be necessary, for use with SQLAlchemy, usage of PARSE_DECLTYPES
  67. can be forced if one configures "native_datetime=True" on create_engine()::
  68. engine = create_engine('sqlite://',
  69. connect_args={'detect_types':
  70. sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES},
  71. native_datetime=True
  72. )
  73. With this flag enabled, the DATE and TIMESTAMP types (but note - not the
  74. DATETIME or TIME types...confused yet ?) will not perform any bind parameter
  75. or result processing. Execution of "func.current_date()" will return a string.
  76. "func.current_timestamp()" is registered as returning a DATETIME type in
  77. SQLAlchemy, so this function still receives SQLAlchemy-level result
  78. processing.
  79. .. _pysqlite_threading_pooling:
  80. Threading/Pooling Behavior
  81. ---------------------------
  82. Pysqlite's default behavior is to prohibit the usage of a single connection
  83. in more than one thread. This is originally intended to work with older
  84. versions of SQLite that did not support multithreaded operation under
  85. various circumstances. In particular, older SQLite versions
  86. did not allow a ``:memory:`` database to be used in multiple threads
  87. under any circumstances.
  88. Pysqlite does include a now-undocumented flag known as
  89. ``check_same_thread`` which will disable this check, however note that
  90. pysqlite connections are still not safe to use in concurrently in multiple
  91. threads. In particular, any statement execution calls would need to be
  92. externally mutexed, as Pysqlite does not provide for thread-safe propagation
  93. of error messages among other things. So while even ``:memory:`` databases
  94. can be shared among threads in modern SQLite, Pysqlite doesn't provide enough
  95. thread-safety to make this usage worth it.
  96. SQLAlchemy sets up pooling to work with Pysqlite's default behavior:
  97. * When a ``:memory:`` SQLite database is specified, the dialect by default
  98. will use :class:`.SingletonThreadPool`. This pool maintains a single
  99. connection per thread, so that all access to the engine within the current
  100. thread use the same ``:memory:`` database - other threads would access a
  101. different ``:memory:`` database.
  102. * When a file-based database is specified, the dialect will use
  103. :class:`.NullPool` as the source of connections. This pool closes and
  104. discards connections which are returned to the pool immediately. SQLite
  105. file-based connections have extremely low overhead, so pooling is not
  106. necessary. The scheme also prevents a connection from being used again in
  107. a different thread and works best with SQLite's coarse-grained file locking.
  108. .. versionchanged:: 0.7
  109. Default selection of :class:`.NullPool` for SQLite file-based databases.
  110. Previous versions select :class:`.SingletonThreadPool` by
  111. default for all SQLite databases.
  112. Using a Memory Database in Multiple Threads
  113. ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  114. To use a ``:memory:`` database in a multithreaded scenario, the same
  115. connection object must be shared among threads, since the database exists
  116. only within the scope of that connection. The
  117. :class:`.StaticPool` implementation will maintain a single connection
  118. globally, and the ``check_same_thread`` flag can be passed to Pysqlite
  119. as ``False``::
  120. from sqlalchemy.pool import StaticPool
  121. engine = create_engine('sqlite://',
  122. connect_args={'check_same_thread':False},
  123. poolclass=StaticPool)
  124. Note that using a ``:memory:`` database in multiple threads requires a recent
  125. version of SQLite.
  126. Using Temporary Tables with SQLite
  127. ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  128. Due to the way SQLite deals with temporary tables, if you wish to use a
  129. temporary table in a file-based SQLite database across multiple checkouts
  130. from the connection pool, such as when using an ORM :class:`.Session` where
  131. the temporary table should continue to remain after :meth:`.Session.commit` or
  132. :meth:`.Session.rollback` is called, a pool which maintains a single
  133. connection must be used. Use :class:`.SingletonThreadPool` if the scope is
  134. only needed within the current thread, or :class:`.StaticPool` is scope is
  135. needed within multiple threads for this case::
  136. # maintain the same connection per thread
  137. from sqlalchemy.pool import SingletonThreadPool
  138. engine = create_engine('sqlite:///mydb.db',
  139. poolclass=SingletonThreadPool)
  140. # maintain the same connection across all threads
  141. from sqlalchemy.pool import StaticPool
  142. engine = create_engine('sqlite:///mydb.db',
  143. poolclass=StaticPool)
  144. Note that :class:`.SingletonThreadPool` should be configured for the number
  145. of threads that are to be used; beyond that number, connections will be
  146. closed out in a non deterministic way.
  147. Unicode
  148. -------
  149. The pysqlite driver only returns Python ``unicode`` objects in result sets,
  150. never plain strings, and accommodates ``unicode`` objects within bound
  151. parameter values in all cases. Regardless of the SQLAlchemy string type in
  152. use, string-based result values will by Python ``unicode`` in Python 2.
  153. The :class:`.Unicode` type should still be used to indicate those columns that
  154. require unicode, however, so that non-``unicode`` values passed inadvertently
  155. will emit a warning. Pysqlite will emit an error if a non-``unicode`` string
  156. is passed containing non-ASCII characters.
  157. .. _pysqlite_serializable:
  158. Serializable isolation / Savepoints / Transactional DDL
  159. -------------------------------------------------------
  160. In the section :ref:`sqlite_concurrency`, we refer to the pysqlite
  161. driver's assortment of issues that prevent several features of SQLite
  162. from working correctly. The pysqlite DBAPI driver has several
  163. long-standing bugs which impact the correctness of its transactional
  164. behavior. In its default mode of operation, SQLite features such as
  165. SERIALIZABLE isolation, transactional DDL, and SAVEPOINT support are
  166. non-functional, and in order to use these features, workarounds must
  167. be taken.
  168. The issue is essentially that the driver attempts to second-guess the user's
  169. intent, failing to start transactions and sometimes ending them prematurely, in
  170. an effort to minimize the SQLite databases's file locking behavior, even
  171. though SQLite itself uses "shared" locks for read-only activities.
  172. SQLAlchemy chooses to not alter this behavior by default, as it is the
  173. long-expected behavior of the pysqlite driver; if and when the pysqlite
  174. driver attempts to repair these issues, that will be more of a driver towards
  175. defaults for SQLAlchemy.
  176. The good news is that with a few events, we can implement transactional
  177. support fully, by disabling pysqlite's feature entirely and emitting BEGIN
  178. ourselves. This is achieved using two event listeners::
  179. from sqlalchemy import create_engine, event
  180. engine = create_engine("sqlite:///myfile.db")
  181. @event.listens_for(engine, "connect")
  182. def do_connect(dbapi_connection, connection_record):
  183. # disable pysqlite's emitting of the BEGIN statement entirely.
  184. # also stops it from emitting COMMIT before any DDL.
  185. dbapi_connection.isolation_level = None
  186. @event.listens_for(engine, "begin")
  187. def do_begin(conn):
  188. # emit our own BEGIN
  189. conn.execute("BEGIN")
  190. Above, we intercept a new pysqlite connection and disable any transactional
  191. integration. Then, at the point at which SQLAlchemy knows that transaction
  192. scope is to begin, we emit ``"BEGIN"`` ourselves.
  193. When we take control of ``"BEGIN"``, we can also control directly SQLite's
  194. locking modes, introduced at `BEGIN TRANSACTION <http://sqlite.org/lang_transaction.html>`_,
  195. by adding the desired locking mode to our ``"BEGIN"``::
  196. @event.listens_for(engine, "begin")
  197. def do_begin(conn):
  198. conn.execute("BEGIN EXCLUSIVE")
  199. .. seealso::
  200. `BEGIN TRANSACTION <http://sqlite.org/lang_transaction.html>`_ - on the SQLite site
  201. `sqlite3 SELECT does not BEGIN a transaction <http://bugs.python.org/issue9924>`_ - on the Python bug tracker
  202. `sqlite3 module breaks transactions and potentially corrupts data <http://bugs.python.org/issue10740>`_ - on the Python bug tracker
  203. """
  204. from sqlalchemy.dialects.sqlite.base import SQLiteDialect, DATETIME, DATE
  205. from sqlalchemy import exc, pool
  206. from sqlalchemy import types as sqltypes
  207. from sqlalchemy import util
  208. import os
  209. class _SQLite_pysqliteTimeStamp(DATETIME):
  210. def bind_processor(self, dialect):
  211. if dialect.native_datetime:
  212. return None
  213. else:
  214. return DATETIME.bind_processor(self, dialect)
  215. def result_processor(self, dialect, coltype):
  216. if dialect.native_datetime:
  217. return None
  218. else:
  219. return DATETIME.result_processor(self, dialect, coltype)
  220. class _SQLite_pysqliteDate(DATE):
  221. def bind_processor(self, dialect):
  222. if dialect.native_datetime:
  223. return None
  224. else:
  225. return DATE.bind_processor(self, dialect)
  226. def result_processor(self, dialect, coltype):
  227. if dialect.native_datetime:
  228. return None
  229. else:
  230. return DATE.result_processor(self, dialect, coltype)
  231. class SQLiteDialect_pysqlite(SQLiteDialect):
  232. default_paramstyle = 'qmark'
  233. colspecs = util.update_copy(
  234. SQLiteDialect.colspecs,
  235. {
  236. sqltypes.Date: _SQLite_pysqliteDate,
  237. sqltypes.TIMESTAMP: _SQLite_pysqliteTimeStamp,
  238. }
  239. )
  240. if not util.py2k:
  241. description_encoding = None
  242. driver = 'pysqlite'
  243. def __init__(self, **kwargs):
  244. SQLiteDialect.__init__(self, **kwargs)
  245. if self.dbapi is not None:
  246. sqlite_ver = self.dbapi.version_info
  247. if sqlite_ver < (2, 1, 3):
  248. util.warn(
  249. ("The installed version of pysqlite2 (%s) is out-dated "
  250. "and will cause errors in some cases. Version 2.1.3 "
  251. "or greater is recommended.") %
  252. '.'.join([str(subver) for subver in sqlite_ver]))
  253. @classmethod
  254. def dbapi(cls):
  255. try:
  256. from pysqlite2 import dbapi2 as sqlite
  257. except ImportError as e:
  258. try:
  259. from sqlite3 import dbapi2 as sqlite # try 2.5+ stdlib name.
  260. except ImportError:
  261. raise e
  262. return sqlite
  263. @classmethod
  264. def get_pool_class(cls, url):
  265. if url.database and url.database != ':memory:':
  266. return pool.NullPool
  267. else:
  268. return pool.SingletonThreadPool
  269. def _get_server_version_info(self, connection):
  270. return self.dbapi.sqlite_version_info
  271. def create_connect_args(self, url):
  272. if url.username or url.password or url.host or url.port:
  273. raise exc.ArgumentError(
  274. "Invalid SQLite URL: %s\n"
  275. "Valid SQLite URL forms are:\n"
  276. " sqlite:///:memory: (or, sqlite://)\n"
  277. " sqlite:///relative/path/to/file.db\n"
  278. " sqlite:////absolute/path/to/file.db" % (url,))
  279. filename = url.database or ':memory:'
  280. if filename != ':memory:':
  281. filename = os.path.abspath(filename)
  282. opts = url.query.copy()
  283. util.coerce_kw_type(opts, 'timeout', float)
  284. util.coerce_kw_type(opts, 'isolation_level', str)
  285. util.coerce_kw_type(opts, 'detect_types', int)
  286. util.coerce_kw_type(opts, 'check_same_thread', bool)
  287. util.coerce_kw_type(opts, 'cached_statements', int)
  288. return ([filename], opts)
  289. def is_disconnect(self, e, connection, cursor):
  290. return isinstance(e, self.dbapi.ProgrammingError) and \
  291. "Cannot operate on a closed database." in str(e)
  292. dialect = SQLiteDialect_pysqlite