wheel.py 32 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853
  1. """
  2. Support for installing and building the "wheel" binary package format.
  3. """
  4. from __future__ import absolute_import
  5. import compileall
  6. import csv
  7. import errno
  8. import functools
  9. import hashlib
  10. import logging
  11. import os
  12. import os.path
  13. import re
  14. import shutil
  15. import stat
  16. import sys
  17. import tempfile
  18. import warnings
  19. from base64 import urlsafe_b64encode
  20. from email.parser import Parser
  21. from pip._vendor.six import StringIO
  22. import pip
  23. from pip.compat import expanduser
  24. from pip.download import path_to_url, unpack_url
  25. from pip.exceptions import (
  26. InstallationError, InvalidWheelFilename, UnsupportedWheel)
  27. from pip.locations import distutils_scheme, PIP_DELETE_MARKER_FILENAME
  28. from pip import pep425tags
  29. from pip.utils import (
  30. call_subprocess, ensure_dir, captured_stdout, rmtree, read_chunks,
  31. )
  32. from pip.utils.ui import open_spinner
  33. from pip.utils.logging import indent_log
  34. from pip.utils.setuptools_build import SETUPTOOLS_SHIM
  35. from pip._vendor.distlib.scripts import ScriptMaker
  36. from pip._vendor import pkg_resources
  37. from pip._vendor.packaging.utils import canonicalize_name
  38. from pip._vendor.six.moves import configparser
  39. wheel_ext = '.whl'
  40. VERSION_COMPATIBLE = (1, 0)
  41. logger = logging.getLogger(__name__)
  42. class WheelCache(object):
  43. """A cache of wheels for future installs."""
  44. def __init__(self, cache_dir, format_control):
  45. """Create a wheel cache.
  46. :param cache_dir: The root of the cache.
  47. :param format_control: A pip.index.FormatControl object to limit
  48. binaries being read from the cache.
  49. """
  50. self._cache_dir = expanduser(cache_dir) if cache_dir else None
  51. self._format_control = format_control
  52. def cached_wheel(self, link, package_name):
  53. return cached_wheel(
  54. self._cache_dir, link, self._format_control, package_name)
  55. def _cache_for_link(cache_dir, link):
  56. """
  57. Return a directory to store cached wheels in for link.
  58. Because there are M wheels for any one sdist, we provide a directory
  59. to cache them in, and then consult that directory when looking up
  60. cache hits.
  61. We only insert things into the cache if they have plausible version
  62. numbers, so that we don't contaminate the cache with things that were not
  63. unique. E.g. ./package might have dozens of installs done for it and build
  64. a version of 0.0...and if we built and cached a wheel, we'd end up using
  65. the same wheel even if the source has been edited.
  66. :param cache_dir: The cache_dir being used by pip.
  67. :param link: The link of the sdist for which this will cache wheels.
  68. """
  69. # We want to generate an url to use as our cache key, we don't want to just
  70. # re-use the URL because it might have other items in the fragment and we
  71. # don't care about those.
  72. key_parts = [link.url_without_fragment]
  73. if link.hash_name is not None and link.hash is not None:
  74. key_parts.append("=".join([link.hash_name, link.hash]))
  75. key_url = "#".join(key_parts)
  76. # Encode our key url with sha224, we'll use this because it has similar
  77. # security properties to sha256, but with a shorter total output (and thus
  78. # less secure). However the differences don't make a lot of difference for
  79. # our use case here.
  80. hashed = hashlib.sha224(key_url.encode()).hexdigest()
  81. # We want to nest the directories some to prevent having a ton of top level
  82. # directories where we might run out of sub directories on some FS.
  83. parts = [hashed[:2], hashed[2:4], hashed[4:6], hashed[6:]]
  84. # Inside of the base location for cached wheels, expand our parts and join
  85. # them all together.
  86. return os.path.join(cache_dir, "wheels", *parts)
  87. def cached_wheel(cache_dir, link, format_control, package_name):
  88. if not cache_dir:
  89. return link
  90. if not link:
  91. return link
  92. if link.is_wheel:
  93. return link
  94. if not link.is_artifact:
  95. return link
  96. if not package_name:
  97. return link
  98. canonical_name = canonicalize_name(package_name)
  99. formats = pip.index.fmt_ctl_formats(format_control, canonical_name)
  100. if "binary" not in formats:
  101. return link
  102. root = _cache_for_link(cache_dir, link)
  103. try:
  104. wheel_names = os.listdir(root)
  105. except OSError as e:
  106. if e.errno in (errno.ENOENT, errno.ENOTDIR):
  107. return link
  108. raise
  109. candidates = []
  110. for wheel_name in wheel_names:
  111. try:
  112. wheel = Wheel(wheel_name)
  113. except InvalidWheelFilename:
  114. continue
  115. if not wheel.supported():
  116. # Built for a different python/arch/etc
  117. continue
  118. candidates.append((wheel.support_index_min(), wheel_name))
  119. if not candidates:
  120. return link
  121. candidates.sort()
  122. path = os.path.join(root, candidates[0][1])
  123. return pip.index.Link(path_to_url(path))
  124. def rehash(path, algo='sha256', blocksize=1 << 20):
  125. """Return (hash, length) for path using hashlib.new(algo)"""
  126. h = hashlib.new(algo)
  127. length = 0
  128. with open(path, 'rb') as f:
  129. for block in read_chunks(f, size=blocksize):
  130. length += len(block)
  131. h.update(block)
  132. digest = 'sha256=' + urlsafe_b64encode(
  133. h.digest()
  134. ).decode('latin1').rstrip('=')
  135. return (digest, length)
  136. def open_for_csv(name, mode):
  137. if sys.version_info[0] < 3:
  138. nl = {}
  139. bin = 'b'
  140. else:
  141. nl = {'newline': ''}
  142. bin = ''
  143. return open(name, mode + bin, **nl)
  144. def fix_script(path):
  145. """Replace #!python with #!/path/to/python
  146. Return True if file was changed."""
  147. # XXX RECORD hashes will need to be updated
  148. if os.path.isfile(path):
  149. with open(path, 'rb') as script:
  150. firstline = script.readline()
  151. if not firstline.startswith(b'#!python'):
  152. return False
  153. exename = sys.executable.encode(sys.getfilesystemencoding())
  154. firstline = b'#!' + exename + os.linesep.encode("ascii")
  155. rest = script.read()
  156. with open(path, 'wb') as script:
  157. script.write(firstline)
  158. script.write(rest)
  159. return True
  160. dist_info_re = re.compile(r"""^(?P<namever>(?P<name>.+?)(-(?P<ver>\d.+?))?)
  161. \.dist-info$""", re.VERBOSE)
  162. def root_is_purelib(name, wheeldir):
  163. """
  164. Return True if the extracted wheel in wheeldir should go into purelib.
  165. """
  166. name_folded = name.replace("-", "_")
  167. for item in os.listdir(wheeldir):
  168. match = dist_info_re.match(item)
  169. if match and match.group('name') == name_folded:
  170. with open(os.path.join(wheeldir, item, 'WHEEL')) as wheel:
  171. for line in wheel:
  172. line = line.lower().rstrip()
  173. if line == "root-is-purelib: true":
  174. return True
  175. return False
  176. def get_entrypoints(filename):
  177. if not os.path.exists(filename):
  178. return {}, {}
  179. # This is done because you can pass a string to entry_points wrappers which
  180. # means that they may or may not be valid INI files. The attempt here is to
  181. # strip leading and trailing whitespace in order to make them valid INI
  182. # files.
  183. with open(filename) as fp:
  184. data = StringIO()
  185. for line in fp:
  186. data.write(line.strip())
  187. data.write("\n")
  188. data.seek(0)
  189. cp = configparser.RawConfigParser()
  190. cp.optionxform = lambda option: option
  191. cp.readfp(data)
  192. console = {}
  193. gui = {}
  194. if cp.has_section('console_scripts'):
  195. console = dict(cp.items('console_scripts'))
  196. if cp.has_section('gui_scripts'):
  197. gui = dict(cp.items('gui_scripts'))
  198. return console, gui
  199. def move_wheel_files(name, req, wheeldir, user=False, home=None, root=None,
  200. pycompile=True, scheme=None, isolated=False, prefix=None):
  201. """Install a wheel"""
  202. if not scheme:
  203. scheme = distutils_scheme(
  204. name, user=user, home=home, root=root, isolated=isolated,
  205. prefix=prefix,
  206. )
  207. if root_is_purelib(name, wheeldir):
  208. lib_dir = scheme['purelib']
  209. else:
  210. lib_dir = scheme['platlib']
  211. info_dir = []
  212. data_dirs = []
  213. source = wheeldir.rstrip(os.path.sep) + os.path.sep
  214. # Record details of the files moved
  215. # installed = files copied from the wheel to the destination
  216. # changed = files changed while installing (scripts #! line typically)
  217. # generated = files newly generated during the install (script wrappers)
  218. installed = {}
  219. changed = set()
  220. generated = []
  221. # Compile all of the pyc files that we're going to be installing
  222. if pycompile:
  223. with captured_stdout() as stdout:
  224. with warnings.catch_warnings():
  225. warnings.filterwarnings('ignore')
  226. compileall.compile_dir(source, force=True, quiet=True)
  227. logger.debug(stdout.getvalue())
  228. def normpath(src, p):
  229. return os.path.relpath(src, p).replace(os.path.sep, '/')
  230. def record_installed(srcfile, destfile, modified=False):
  231. """Map archive RECORD paths to installation RECORD paths."""
  232. oldpath = normpath(srcfile, wheeldir)
  233. newpath = normpath(destfile, lib_dir)
  234. installed[oldpath] = newpath
  235. if modified:
  236. changed.add(destfile)
  237. def clobber(source, dest, is_base, fixer=None, filter=None):
  238. ensure_dir(dest) # common for the 'include' path
  239. for dir, subdirs, files in os.walk(source):
  240. basedir = dir[len(source):].lstrip(os.path.sep)
  241. destdir = os.path.join(dest, basedir)
  242. if is_base and basedir.split(os.path.sep, 1)[0].endswith('.data'):
  243. continue
  244. for s in subdirs:
  245. destsubdir = os.path.join(dest, basedir, s)
  246. if is_base and basedir == '' and destsubdir.endswith('.data'):
  247. data_dirs.append(s)
  248. continue
  249. elif (is_base and
  250. s.endswith('.dist-info') and
  251. canonicalize_name(s).startswith(
  252. canonicalize_name(req.name))):
  253. assert not info_dir, ('Multiple .dist-info directories: ' +
  254. destsubdir + ', ' +
  255. ', '.join(info_dir))
  256. info_dir.append(destsubdir)
  257. for f in files:
  258. # Skip unwanted files
  259. if filter and filter(f):
  260. continue
  261. srcfile = os.path.join(dir, f)
  262. destfile = os.path.join(dest, basedir, f)
  263. # directory creation is lazy and after the file filtering above
  264. # to ensure we don't install empty dirs; empty dirs can't be
  265. # uninstalled.
  266. ensure_dir(destdir)
  267. # We use copyfile (not move, copy, or copy2) to be extra sure
  268. # that we are not moving directories over (copyfile fails for
  269. # directories) as well as to ensure that we are not copying
  270. # over any metadata because we want more control over what
  271. # metadata we actually copy over.
  272. shutil.copyfile(srcfile, destfile)
  273. # Copy over the metadata for the file, currently this only
  274. # includes the atime and mtime.
  275. st = os.stat(srcfile)
  276. if hasattr(os, "utime"):
  277. os.utime(destfile, (st.st_atime, st.st_mtime))
  278. # If our file is executable, then make our destination file
  279. # executable.
  280. if os.access(srcfile, os.X_OK):
  281. st = os.stat(srcfile)
  282. permissions = (
  283. st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
  284. )
  285. os.chmod(destfile, permissions)
  286. changed = False
  287. if fixer:
  288. changed = fixer(destfile)
  289. record_installed(srcfile, destfile, changed)
  290. clobber(source, lib_dir, True)
  291. assert info_dir, "%s .dist-info directory not found" % req
  292. # Get the defined entry points
  293. ep_file = os.path.join(info_dir[0], 'entry_points.txt')
  294. console, gui = get_entrypoints(ep_file)
  295. def is_entrypoint_wrapper(name):
  296. # EP, EP.exe and EP-script.py are scripts generated for
  297. # entry point EP by setuptools
  298. if name.lower().endswith('.exe'):
  299. matchname = name[:-4]
  300. elif name.lower().endswith('-script.py'):
  301. matchname = name[:-10]
  302. elif name.lower().endswith(".pya"):
  303. matchname = name[:-4]
  304. else:
  305. matchname = name
  306. # Ignore setuptools-generated scripts
  307. return (matchname in console or matchname in gui)
  308. for datadir in data_dirs:
  309. fixer = None
  310. filter = None
  311. for subdir in os.listdir(os.path.join(wheeldir, datadir)):
  312. fixer = None
  313. if subdir == 'scripts':
  314. fixer = fix_script
  315. filter = is_entrypoint_wrapper
  316. source = os.path.join(wheeldir, datadir, subdir)
  317. dest = scheme[subdir]
  318. clobber(source, dest, False, fixer=fixer, filter=filter)
  319. maker = ScriptMaker(None, scheme['scripts'])
  320. # Ensure old scripts are overwritten.
  321. # See https://github.com/pypa/pip/issues/1800
  322. maker.clobber = True
  323. # Ensure we don't generate any variants for scripts because this is almost
  324. # never what somebody wants.
  325. # See https://bitbucket.org/pypa/distlib/issue/35/
  326. maker.variants = set(('', ))
  327. # This is required because otherwise distlib creates scripts that are not
  328. # executable.
  329. # See https://bitbucket.org/pypa/distlib/issue/32/
  330. maker.set_mode = True
  331. # Simplify the script and fix the fact that the default script swallows
  332. # every single stack trace.
  333. # See https://bitbucket.org/pypa/distlib/issue/34/
  334. # See https://bitbucket.org/pypa/distlib/issue/33/
  335. def _get_script_text(entry):
  336. if entry.suffix is None:
  337. raise InstallationError(
  338. "Invalid script entry point: %s for req: %s - A callable "
  339. "suffix is required. Cf https://packaging.python.org/en/"
  340. "latest/distributing.html#console-scripts for more "
  341. "information." % (entry, req)
  342. )
  343. return maker.script_template % {
  344. "module": entry.prefix,
  345. "import_name": entry.suffix.split(".")[0],
  346. "func": entry.suffix,
  347. }
  348. maker._get_script_text = _get_script_text
  349. maker.script_template = """# -*- coding: utf-8 -*-
  350. import re
  351. import sys
  352. from %(module)s import %(import_name)s
  353. if __name__ == '__main__':
  354. sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0])
  355. sys.exit(%(func)s())
  356. """
  357. # Special case pip and setuptools to generate versioned wrappers
  358. #
  359. # The issue is that some projects (specifically, pip and setuptools) use
  360. # code in setup.py to create "versioned" entry points - pip2.7 on Python
  361. # 2.7, pip3.3 on Python 3.3, etc. But these entry points are baked into
  362. # the wheel metadata at build time, and so if the wheel is installed with
  363. # a *different* version of Python the entry points will be wrong. The
  364. # correct fix for this is to enhance the metadata to be able to describe
  365. # such versioned entry points, but that won't happen till Metadata 2.0 is
  366. # available.
  367. # In the meantime, projects using versioned entry points will either have
  368. # incorrect versioned entry points, or they will not be able to distribute
  369. # "universal" wheels (i.e., they will need a wheel per Python version).
  370. #
  371. # Because setuptools and pip are bundled with _ensurepip and virtualenv,
  372. # we need to use universal wheels. So, as a stopgap until Metadata 2.0, we
  373. # override the versioned entry points in the wheel and generate the
  374. # correct ones. This code is purely a short-term measure until Metadata 2.0
  375. # is available.
  376. #
  377. # To add the level of hack in this section of code, in order to support
  378. # ensurepip this code will look for an ``ENSUREPIP_OPTIONS`` environment
  379. # variable which will control which version scripts get installed.
  380. #
  381. # ENSUREPIP_OPTIONS=altinstall
  382. # - Only pipX.Y and easy_install-X.Y will be generated and installed
  383. # ENSUREPIP_OPTIONS=install
  384. # - pipX.Y, pipX, easy_install-X.Y will be generated and installed. Note
  385. # that this option is technically if ENSUREPIP_OPTIONS is set and is
  386. # not altinstall
  387. # DEFAULT
  388. # - The default behavior is to install pip, pipX, pipX.Y, easy_install
  389. # and easy_install-X.Y.
  390. pip_script = console.pop('pip', None)
  391. if pip_script:
  392. if "ENSUREPIP_OPTIONS" not in os.environ:
  393. spec = 'pip = ' + pip_script
  394. generated.extend(maker.make(spec))
  395. if os.environ.get("ENSUREPIP_OPTIONS", "") != "altinstall":
  396. spec = 'pip%s = %s' % (sys.version[:1], pip_script)
  397. generated.extend(maker.make(spec))
  398. spec = 'pip%s = %s' % (sys.version[:3], pip_script)
  399. generated.extend(maker.make(spec))
  400. # Delete any other versioned pip entry points
  401. pip_ep = [k for k in console if re.match(r'pip(\d(\.\d)?)?$', k)]
  402. for k in pip_ep:
  403. del console[k]
  404. easy_install_script = console.pop('easy_install', None)
  405. if easy_install_script:
  406. if "ENSUREPIP_OPTIONS" not in os.environ:
  407. spec = 'easy_install = ' + easy_install_script
  408. generated.extend(maker.make(spec))
  409. spec = 'easy_install-%s = %s' % (sys.version[:3], easy_install_script)
  410. generated.extend(maker.make(spec))
  411. # Delete any other versioned easy_install entry points
  412. easy_install_ep = [
  413. k for k in console if re.match(r'easy_install(-\d\.\d)?$', k)
  414. ]
  415. for k in easy_install_ep:
  416. del console[k]
  417. # Generate the console and GUI entry points specified in the wheel
  418. if len(console) > 0:
  419. generated.extend(
  420. maker.make_multiple(['%s = %s' % kv for kv in console.items()])
  421. )
  422. if len(gui) > 0:
  423. generated.extend(
  424. maker.make_multiple(
  425. ['%s = %s' % kv for kv in gui.items()],
  426. {'gui': True}
  427. )
  428. )
  429. # Record pip as the installer
  430. installer = os.path.join(info_dir[0], 'INSTALLER')
  431. temp_installer = os.path.join(info_dir[0], 'INSTALLER.pip')
  432. with open(temp_installer, 'wb') as installer_file:
  433. installer_file.write(b'pip\n')
  434. shutil.move(temp_installer, installer)
  435. generated.append(installer)
  436. # Record details of all files installed
  437. record = os.path.join(info_dir[0], 'RECORD')
  438. temp_record = os.path.join(info_dir[0], 'RECORD.pip')
  439. with open_for_csv(record, 'r') as record_in:
  440. with open_for_csv(temp_record, 'w+') as record_out:
  441. reader = csv.reader(record_in)
  442. writer = csv.writer(record_out)
  443. for row in reader:
  444. row[0] = installed.pop(row[0], row[0])
  445. if row[0] in changed:
  446. row[1], row[2] = rehash(row[0])
  447. writer.writerow(row)
  448. for f in generated:
  449. h, l = rehash(f)
  450. writer.writerow((normpath(f, lib_dir), h, l))
  451. for f in installed:
  452. writer.writerow((installed[f], '', ''))
  453. shutil.move(temp_record, record)
  454. def _unique(fn):
  455. @functools.wraps(fn)
  456. def unique(*args, **kw):
  457. seen = set()
  458. for item in fn(*args, **kw):
  459. if item not in seen:
  460. seen.add(item)
  461. yield item
  462. return unique
  463. # TODO: this goes somewhere besides the wheel module
  464. @_unique
  465. def uninstallation_paths(dist):
  466. """
  467. Yield all the uninstallation paths for dist based on RECORD-without-.pyc
  468. Yield paths to all the files in RECORD. For each .py file in RECORD, add
  469. the .pyc in the same directory.
  470. UninstallPathSet.add() takes care of the __pycache__ .pyc.
  471. """
  472. from pip.utils import FakeFile # circular import
  473. r = csv.reader(FakeFile(dist.get_metadata_lines('RECORD')))
  474. for row in r:
  475. path = os.path.join(dist.location, row[0])
  476. yield path
  477. if path.endswith('.py'):
  478. dn, fn = os.path.split(path)
  479. base = fn[:-3]
  480. path = os.path.join(dn, base + '.pyc')
  481. yield path
  482. def wheel_version(source_dir):
  483. """
  484. Return the Wheel-Version of an extracted wheel, if possible.
  485. Otherwise, return False if we couldn't parse / extract it.
  486. """
  487. try:
  488. dist = [d for d in pkg_resources.find_on_path(None, source_dir)][0]
  489. wheel_data = dist.get_metadata('WHEEL')
  490. wheel_data = Parser().parsestr(wheel_data)
  491. version = wheel_data['Wheel-Version'].strip()
  492. version = tuple(map(int, version.split('.')))
  493. return version
  494. except:
  495. return False
  496. def check_compatibility(version, name):
  497. """
  498. Raises errors or warns if called with an incompatible Wheel-Version.
  499. Pip should refuse to install a Wheel-Version that's a major series
  500. ahead of what it's compatible with (e.g 2.0 > 1.1); and warn when
  501. installing a version only minor version ahead (e.g 1.2 > 1.1).
  502. version: a 2-tuple representing a Wheel-Version (Major, Minor)
  503. name: name of wheel or package to raise exception about
  504. :raises UnsupportedWheel: when an incompatible Wheel-Version is given
  505. """
  506. if not version:
  507. raise UnsupportedWheel(
  508. "%s is in an unsupported or invalid wheel" % name
  509. )
  510. if version[0] > VERSION_COMPATIBLE[0]:
  511. raise UnsupportedWheel(
  512. "%s's Wheel-Version (%s) is not compatible with this version "
  513. "of pip" % (name, '.'.join(map(str, version)))
  514. )
  515. elif version > VERSION_COMPATIBLE:
  516. logger.warning(
  517. 'Installing from a newer Wheel-Version (%s)',
  518. '.'.join(map(str, version)),
  519. )
  520. class Wheel(object):
  521. """A wheel file"""
  522. # TODO: maybe move the install code into this class
  523. wheel_file_re = re.compile(
  524. r"""^(?P<namever>(?P<name>.+?)-(?P<ver>\d.*?))
  525. ((-(?P<build>\d.*?))?-(?P<pyver>.+?)-(?P<abi>.+?)-(?P<plat>.+?)
  526. \.whl|\.dist-info)$""",
  527. re.VERBOSE
  528. )
  529. def __init__(self, filename):
  530. """
  531. :raises InvalidWheelFilename: when the filename is invalid for a wheel
  532. """
  533. wheel_info = self.wheel_file_re.match(filename)
  534. if not wheel_info:
  535. raise InvalidWheelFilename(
  536. "%s is not a valid wheel filename." % filename
  537. )
  538. self.filename = filename
  539. self.name = wheel_info.group('name').replace('_', '-')
  540. # we'll assume "_" means "-" due to wheel naming scheme
  541. # (https://github.com/pypa/pip/issues/1150)
  542. self.version = wheel_info.group('ver').replace('_', '-')
  543. self.pyversions = wheel_info.group('pyver').split('.')
  544. self.abis = wheel_info.group('abi').split('.')
  545. self.plats = wheel_info.group('plat').split('.')
  546. # All the tag combinations from this file
  547. self.file_tags = set(
  548. (x, y, z) for x in self.pyversions
  549. for y in self.abis for z in self.plats
  550. )
  551. def support_index_min(self, tags=None):
  552. """
  553. Return the lowest index that one of the wheel's file_tag combinations
  554. achieves in the supported_tags list e.g. if there are 8 supported tags,
  555. and one of the file tags is first in the list, then return 0. Returns
  556. None is the wheel is not supported.
  557. """
  558. if tags is None: # for mock
  559. tags = pep425tags.supported_tags
  560. indexes = [tags.index(c) for c in self.file_tags if c in tags]
  561. return min(indexes) if indexes else None
  562. def supported(self, tags=None):
  563. """Is this wheel supported on this system?"""
  564. if tags is None: # for mock
  565. tags = pep425tags.supported_tags
  566. return bool(set(tags).intersection(self.file_tags))
  567. class WheelBuilder(object):
  568. """Build wheels from a RequirementSet."""
  569. def __init__(self, requirement_set, finder, build_options=None,
  570. global_options=None):
  571. self.requirement_set = requirement_set
  572. self.finder = finder
  573. self._cache_root = requirement_set._wheel_cache._cache_dir
  574. self._wheel_dir = requirement_set.wheel_download_dir
  575. self.build_options = build_options or []
  576. self.global_options = global_options or []
  577. def _build_one(self, req, output_dir, python_tag=None):
  578. """Build one wheel.
  579. :return: The filename of the built wheel, or None if the build failed.
  580. """
  581. tempd = tempfile.mkdtemp('pip-wheel-')
  582. try:
  583. if self.__build_one(req, tempd, python_tag=python_tag):
  584. try:
  585. wheel_name = os.listdir(tempd)[0]
  586. wheel_path = os.path.join(output_dir, wheel_name)
  587. shutil.move(os.path.join(tempd, wheel_name), wheel_path)
  588. logger.info('Stored in directory: %s', output_dir)
  589. return wheel_path
  590. except:
  591. pass
  592. # Ignore return, we can't do anything else useful.
  593. self._clean_one(req)
  594. return None
  595. finally:
  596. rmtree(tempd)
  597. def _base_setup_args(self, req):
  598. return [
  599. sys.executable, "-u", '-c',
  600. SETUPTOOLS_SHIM % req.setup_py
  601. ] + list(self.global_options)
  602. def __build_one(self, req, tempd, python_tag=None):
  603. base_args = self._base_setup_args(req)
  604. spin_message = 'Running setup.py bdist_wheel for %s' % (req.name,)
  605. with open_spinner(spin_message) as spinner:
  606. logger.debug('Destination directory: %s', tempd)
  607. wheel_args = base_args + ['bdist_wheel', '-d', tempd] \
  608. + self.build_options
  609. if python_tag is not None:
  610. wheel_args += ["--python-tag", python_tag]
  611. try:
  612. call_subprocess(wheel_args, cwd=req.setup_py_dir,
  613. show_stdout=False, spinner=spinner)
  614. return True
  615. except:
  616. spinner.finish("error")
  617. logger.error('Failed building wheel for %s', req.name)
  618. return False
  619. def _clean_one(self, req):
  620. base_args = self._base_setup_args(req)
  621. logger.info('Running setup.py clean for %s', req.name)
  622. clean_args = base_args + ['clean', '--all']
  623. try:
  624. call_subprocess(clean_args, cwd=req.source_dir, show_stdout=False)
  625. return True
  626. except:
  627. logger.error('Failed cleaning build dir for %s', req.name)
  628. return False
  629. def build(self, autobuilding=False):
  630. """Build wheels.
  631. :param unpack: If True, replace the sdist we built from with the
  632. newly built wheel, in preparation for installation.
  633. :return: True if all the wheels built correctly.
  634. """
  635. assert self._wheel_dir or (autobuilding and self._cache_root)
  636. # unpack sdists and constructs req set
  637. self.requirement_set.prepare_files(self.finder)
  638. reqset = self.requirement_set.requirements.values()
  639. buildset = []
  640. for req in reqset:
  641. if req.constraint:
  642. continue
  643. if req.is_wheel:
  644. if not autobuilding:
  645. logger.info(
  646. 'Skipping %s, due to already being wheel.', req.name)
  647. elif autobuilding and req.editable:
  648. pass
  649. elif autobuilding and req.link and not req.link.is_artifact:
  650. pass
  651. elif autobuilding and not req.source_dir:
  652. pass
  653. else:
  654. if autobuilding:
  655. link = req.link
  656. base, ext = link.splitext()
  657. if pip.index.egg_info_matches(base, None, link) is None:
  658. # Doesn't look like a package - don't autobuild a wheel
  659. # because we'll have no way to lookup the result sanely
  660. continue
  661. if "binary" not in pip.index.fmt_ctl_formats(
  662. self.finder.format_control,
  663. canonicalize_name(req.name)):
  664. logger.info(
  665. "Skipping bdist_wheel for %s, due to binaries "
  666. "being disabled for it.", req.name)
  667. continue
  668. buildset.append(req)
  669. if not buildset:
  670. return True
  671. # Build the wheels.
  672. logger.info(
  673. 'Building wheels for collected packages: %s',
  674. ', '.join([req.name for req in buildset]),
  675. )
  676. with indent_log():
  677. build_success, build_failure = [], []
  678. for req in buildset:
  679. python_tag = None
  680. if autobuilding:
  681. python_tag = pep425tags.implementation_tag
  682. output_dir = _cache_for_link(self._cache_root, req.link)
  683. try:
  684. ensure_dir(output_dir)
  685. except OSError as e:
  686. logger.warning("Building wheel for %s failed: %s",
  687. req.name, e)
  688. build_failure.append(req)
  689. continue
  690. else:
  691. output_dir = self._wheel_dir
  692. wheel_file = self._build_one(
  693. req, output_dir,
  694. python_tag=python_tag,
  695. )
  696. if wheel_file:
  697. build_success.append(req)
  698. if autobuilding:
  699. # XXX: This is mildly duplicative with prepare_files,
  700. # but not close enough to pull out to a single common
  701. # method.
  702. # The code below assumes temporary source dirs -
  703. # prevent it doing bad things.
  704. if req.source_dir and not os.path.exists(os.path.join(
  705. req.source_dir, PIP_DELETE_MARKER_FILENAME)):
  706. raise AssertionError(
  707. "bad source dir - missing marker")
  708. # Delete the source we built the wheel from
  709. req.remove_temporary_source()
  710. # set the build directory again - name is known from
  711. # the work prepare_files did.
  712. req.source_dir = req.build_location(
  713. self.requirement_set.build_dir)
  714. # Update the link for this.
  715. req.link = pip.index.Link(
  716. path_to_url(wheel_file))
  717. assert req.link.is_wheel
  718. # extract the wheel into the dir
  719. unpack_url(
  720. req.link, req.source_dir, None, False,
  721. session=self.requirement_set.session)
  722. else:
  723. build_failure.append(req)
  724. # notify success/failure
  725. if build_success:
  726. logger.info(
  727. 'Successfully built %s',
  728. ' '.join([req.name for req in build_success]),
  729. )
  730. if build_failure:
  731. logger.info(
  732. 'Failed to build %s',
  733. ' '.join([req.name for req in build_failure]),
  734. )
  735. # Return True if all builds were successful
  736. return len(build_failure) == 0