config.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548
  1. from __future__ import absolute_import, unicode_literals
  2. import io
  3. import os
  4. import sys
  5. from collections import defaultdict
  6. from functools import partial
  7. from distutils.errors import DistutilsOptionError, DistutilsFileError
  8. from setuptools.py26compat import import_module
  9. from six import string_types
  10. def read_configuration(
  11. filepath, find_others=False, ignore_option_errors=False):
  12. """Read given configuration file and returns options from it as a dict.
  13. :param str|unicode filepath: Path to configuration file
  14. to get options from.
  15. :param bool find_others: Whether to search for other configuration files
  16. which could be on in various places.
  17. :param bool ignore_option_errors: Whether to silently ignore
  18. options, values of which could not be resolved (e.g. due to exceptions
  19. in directives such as file:, attr:, etc.).
  20. If False exceptions are propagated as expected.
  21. :rtype: dict
  22. """
  23. from setuptools.dist import Distribution, _Distribution
  24. filepath = os.path.abspath(filepath)
  25. if not os.path.isfile(filepath):
  26. raise DistutilsFileError(
  27. 'Configuration file %s does not exist.' % filepath)
  28. current_directory = os.getcwd()
  29. os.chdir(os.path.dirname(filepath))
  30. try:
  31. dist = Distribution()
  32. filenames = dist.find_config_files() if find_others else []
  33. if filepath not in filenames:
  34. filenames.append(filepath)
  35. _Distribution.parse_config_files(dist, filenames=filenames)
  36. handlers = parse_configuration(
  37. dist, dist.command_options,
  38. ignore_option_errors=ignore_option_errors)
  39. finally:
  40. os.chdir(current_directory)
  41. return configuration_to_dict(handlers)
  42. def configuration_to_dict(handlers):
  43. """Returns configuration data gathered by given handlers as a dict.
  44. :param list[ConfigHandler] handlers: Handlers list,
  45. usually from parse_configuration()
  46. :rtype: dict
  47. """
  48. config_dict = defaultdict(dict)
  49. for handler in handlers:
  50. obj_alias = handler.section_prefix
  51. target_obj = handler.target_obj
  52. for option in handler.set_options:
  53. getter = getattr(target_obj, 'get_%s' % option, None)
  54. if getter is None:
  55. value = getattr(target_obj, option)
  56. else:
  57. value = getter()
  58. config_dict[obj_alias][option] = value
  59. return config_dict
  60. def parse_configuration(
  61. distribution, command_options, ignore_option_errors=False):
  62. """Performs additional parsing of configuration options
  63. for a distribution.
  64. Returns a list of used option handlers.
  65. :param Distribution distribution:
  66. :param dict command_options:
  67. :param bool ignore_option_errors: Whether to silently ignore
  68. options, values of which could not be resolved (e.g. due to exceptions
  69. in directives such as file:, attr:, etc.).
  70. If False exceptions are propagated as expected.
  71. :rtype: list
  72. """
  73. meta = ConfigMetadataHandler(
  74. distribution.metadata, command_options, ignore_option_errors)
  75. meta.parse()
  76. options = ConfigOptionsHandler(
  77. distribution, command_options, ignore_option_errors)
  78. options.parse()
  79. return [meta, options]
  80. class ConfigHandler(object):
  81. """Handles metadata supplied in configuration files."""
  82. section_prefix = None
  83. """Prefix for config sections handled by this handler.
  84. Must be provided by class heirs.
  85. """
  86. aliases = {}
  87. """Options aliases.
  88. For compatibility with various packages. E.g.: d2to1 and pbr.
  89. Note: `-` in keys is replaced with `_` by config parser.
  90. """
  91. def __init__(self, target_obj, options, ignore_option_errors=False):
  92. sections = {}
  93. section_prefix = self.section_prefix
  94. for section_name, section_options in options.items():
  95. if not section_name.startswith(section_prefix):
  96. continue
  97. section_name = section_name.replace(section_prefix, '').strip('.')
  98. sections[section_name] = section_options
  99. self.ignore_option_errors = ignore_option_errors
  100. self.target_obj = target_obj
  101. self.sections = sections
  102. self.set_options = []
  103. @property
  104. def parsers(self):
  105. """Metadata item name to parser function mapping."""
  106. raise NotImplementedError(
  107. '%s must provide .parsers property' % self.__class__.__name__)
  108. def __setitem__(self, option_name, value):
  109. unknown = tuple()
  110. target_obj = self.target_obj
  111. # Translate alias into real name.
  112. option_name = self.aliases.get(option_name, option_name)
  113. current_value = getattr(target_obj, option_name, unknown)
  114. if current_value is unknown:
  115. raise KeyError(option_name)
  116. if current_value:
  117. # Already inhabited. Skipping.
  118. return
  119. skip_option = False
  120. parser = self.parsers.get(option_name)
  121. if parser:
  122. try:
  123. value = parser(value)
  124. except Exception:
  125. skip_option = True
  126. if not self.ignore_option_errors:
  127. raise
  128. if skip_option:
  129. return
  130. setter = getattr(target_obj, 'set_%s' % option_name, None)
  131. if setter is None:
  132. setattr(target_obj, option_name, value)
  133. else:
  134. setter(value)
  135. self.set_options.append(option_name)
  136. @classmethod
  137. def _parse_list(cls, value, separator=','):
  138. """Represents value as a list.
  139. Value is split either by separator (defaults to comma) or by lines.
  140. :param value:
  141. :param separator: List items separator character.
  142. :rtype: list
  143. """
  144. if isinstance(value, list): # _get_parser_compound case
  145. return value
  146. if '\n' in value:
  147. value = value.splitlines()
  148. else:
  149. value = value.split(separator)
  150. return [chunk.strip() for chunk in value if chunk.strip()]
  151. @classmethod
  152. def _parse_dict(cls, value):
  153. """Represents value as a dict.
  154. :param value:
  155. :rtype: dict
  156. """
  157. separator = '='
  158. result = {}
  159. for line in cls._parse_list(value):
  160. key, sep, val = line.partition(separator)
  161. if sep != separator:
  162. raise DistutilsOptionError(
  163. 'Unable to parse option value to dict: %s' % value)
  164. result[key.strip()] = val.strip()
  165. return result
  166. @classmethod
  167. def _parse_bool(cls, value):
  168. """Represents value as boolean.
  169. :param value:
  170. :rtype: bool
  171. """
  172. value = value.lower()
  173. return value in ('1', 'true', 'yes')
  174. @classmethod
  175. def _parse_file(cls, value):
  176. """Represents value as a string, allowing including text
  177. from nearest files using `file:` directive.
  178. Directive is sandboxed and won't reach anything outside
  179. directory with setup.py.
  180. Examples:
  181. include: LICENSE
  182. include: src/file.txt
  183. :param str value:
  184. :rtype: str
  185. """
  186. if not isinstance(value, string_types):
  187. return value
  188. include_directive = 'file:'
  189. if not value.startswith(include_directive):
  190. return value
  191. current_directory = os.getcwd()
  192. filepath = value.replace(include_directive, '').strip()
  193. filepath = os.path.abspath(filepath)
  194. if not filepath.startswith(current_directory):
  195. raise DistutilsOptionError(
  196. '`file:` directive can not access %s' % filepath)
  197. if os.path.isfile(filepath):
  198. with io.open(filepath, encoding='utf-8') as f:
  199. value = f.read()
  200. return value
  201. @classmethod
  202. def _parse_attr(cls, value):
  203. """Represents value as a module attribute.
  204. Examples:
  205. attr: package.attr
  206. attr: package.module.attr
  207. :param str value:
  208. :rtype: str
  209. """
  210. attr_directive = 'attr:'
  211. if not value.startswith(attr_directive):
  212. return value
  213. attrs_path = value.replace(attr_directive, '').strip().split('.')
  214. attr_name = attrs_path.pop()
  215. module_name = '.'.join(attrs_path)
  216. module_name = module_name or '__init__'
  217. sys.path.insert(0, os.getcwd())
  218. try:
  219. module = import_module(module_name)
  220. value = getattr(module, attr_name)
  221. finally:
  222. sys.path = sys.path[1:]
  223. return value
  224. @classmethod
  225. def _get_parser_compound(cls, *parse_methods):
  226. """Returns parser function to represents value as a list.
  227. Parses a value applying given methods one after another.
  228. :param parse_methods:
  229. :rtype: callable
  230. """
  231. def parse(value):
  232. parsed = value
  233. for method in parse_methods:
  234. parsed = method(parsed)
  235. return parsed
  236. return parse
  237. @classmethod
  238. def _parse_section_to_dict(cls, section_options, values_parser=None):
  239. """Parses section options into a dictionary.
  240. Optionally applies a given parser to values.
  241. :param dict section_options:
  242. :param callable values_parser:
  243. :rtype: dict
  244. """
  245. value = {}
  246. values_parser = values_parser or (lambda val: val)
  247. for key, (_, val) in section_options.items():
  248. value[key] = values_parser(val)
  249. return value
  250. def parse_section(self, section_options):
  251. """Parses configuration file section.
  252. :param dict section_options:
  253. """
  254. for (name, (_, value)) in section_options.items():
  255. try:
  256. self[name] = value
  257. except KeyError:
  258. pass # Keep silent for a new option may appear anytime.
  259. def parse(self):
  260. """Parses configuration file items from one
  261. or more related sections.
  262. """
  263. for section_name, section_options in self.sections.items():
  264. method_postfix = ''
  265. if section_name: # [section.option] variant
  266. method_postfix = '_%s' % section_name
  267. section_parser_method = getattr(
  268. self,
  269. # Dots in section names are tranlsated into dunderscores.
  270. ('parse_section%s' % method_postfix).replace('.', '__'),
  271. None)
  272. if section_parser_method is None:
  273. raise DistutilsOptionError(
  274. 'Unsupported distribution option section: [%s.%s]' % (
  275. self.section_prefix, section_name))
  276. section_parser_method(section_options)
  277. class ConfigMetadataHandler(ConfigHandler):
  278. section_prefix = 'metadata'
  279. aliases = {
  280. 'home_page': 'url',
  281. 'summary': 'description',
  282. 'classifier': 'classifiers',
  283. 'platform': 'platforms',
  284. }
  285. strict_mode = False
  286. """We need to keep it loose, to be partially compatible with
  287. `pbr` and `d2to1` packages which also uses `metadata` section.
  288. """
  289. @property
  290. def parsers(self):
  291. """Metadata item name to parser function mapping."""
  292. parse_list = self._parse_list
  293. parse_file = self._parse_file
  294. return {
  295. 'platforms': parse_list,
  296. 'keywords': parse_list,
  297. 'provides': parse_list,
  298. 'requires': parse_list,
  299. 'obsoletes': parse_list,
  300. 'classifiers': self._get_parser_compound(parse_file, parse_list),
  301. 'license': parse_file,
  302. 'description': parse_file,
  303. 'long_description': parse_file,
  304. 'version': self._parse_version,
  305. }
  306. def _parse_version(self, value):
  307. """Parses `version` option value.
  308. :param value:
  309. :rtype: str
  310. """
  311. version = self._parse_attr(value)
  312. if callable(version):
  313. version = version()
  314. if not isinstance(version, string_types):
  315. if hasattr(version, '__iter__'):
  316. version = '.'.join(map(str, version))
  317. else:
  318. version = '%s' % version
  319. return version
  320. class ConfigOptionsHandler(ConfigHandler):
  321. section_prefix = 'options'
  322. @property
  323. def parsers(self):
  324. """Metadata item name to parser function mapping."""
  325. parse_list = self._parse_list
  326. parse_list_semicolon = partial(self._parse_list, separator=';')
  327. parse_bool = self._parse_bool
  328. parse_dict = self._parse_dict
  329. return {
  330. 'zip_safe': parse_bool,
  331. 'use_2to3': parse_bool,
  332. 'include_package_data': parse_bool,
  333. 'package_dir': parse_dict,
  334. 'use_2to3_fixers': parse_list,
  335. 'use_2to3_exclude_fixers': parse_list,
  336. 'convert_2to3_doctests': parse_list,
  337. 'scripts': parse_list,
  338. 'eager_resources': parse_list,
  339. 'dependency_links': parse_list,
  340. 'namespace_packages': parse_list,
  341. 'install_requires': parse_list_semicolon,
  342. 'setup_requires': parse_list_semicolon,
  343. 'tests_require': parse_list_semicolon,
  344. 'packages': self._parse_packages,
  345. 'entry_points': self._parse_file,
  346. 'py_modules': parse_list,
  347. }
  348. def _parse_packages(self, value):
  349. """Parses `packages` option value.
  350. :param value:
  351. :rtype: list
  352. """
  353. find_directive = 'find:'
  354. if not value.startswith(find_directive):
  355. return self._parse_list(value)
  356. # Read function arguments from a dedicated section.
  357. find_kwargs = self.parse_section_packages__find(
  358. self.sections.get('packages.find', {}))
  359. from setuptools import find_packages
  360. return find_packages(**find_kwargs)
  361. def parse_section_packages__find(self, section_options):
  362. """Parses `packages.find` configuration file section.
  363. To be used in conjunction with _parse_packages().
  364. :param dict section_options:
  365. """
  366. section_data = self._parse_section_to_dict(
  367. section_options, self._parse_list)
  368. valid_keys = ['where', 'include', 'exclude']
  369. find_kwargs = dict(
  370. [(k, v) for k, v in section_data.items() if k in valid_keys and v])
  371. where = find_kwargs.get('where')
  372. if where is not None:
  373. find_kwargs['where'] = where[0] # cast list to single val
  374. return find_kwargs
  375. def parse_section_entry_points(self, section_options):
  376. """Parses `entry_points` configuration file section.
  377. :param dict section_options:
  378. """
  379. parsed = self._parse_section_to_dict(section_options, self._parse_list)
  380. self['entry_points'] = parsed
  381. def _parse_package_data(self, section_options):
  382. parsed = self._parse_section_to_dict(section_options, self._parse_list)
  383. root = parsed.get('*')
  384. if root:
  385. parsed[''] = root
  386. del parsed['*']
  387. return parsed
  388. def parse_section_package_data(self, section_options):
  389. """Parses `package_data` configuration file section.
  390. :param dict section_options:
  391. """
  392. self['package_data'] = self._parse_package_data(section_options)
  393. def parse_section_exclude_package_data(self, section_options):
  394. """Parses `exclude_package_data` configuration file section.
  395. :param dict section_options:
  396. """
  397. self['exclude_package_data'] = self._parse_package_data(
  398. section_options)
  399. def parse_section_extras_require(self, section_options):
  400. """Parses `extras_require` configuration file section.
  401. :param dict section_options:
  402. """
  403. parse_list = partial(self._parse_list, separator=';')
  404. self['extras_require'] = self._parse_section_to_dict(
  405. section_options, parse_list)