nosetester.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516
  1. """
  2. Nose test running.
  3. This module implements ``test()`` and ``bench()`` functions for NumPy modules.
  4. """
  5. from __future__ import division, absolute_import, print_function
  6. import os
  7. import sys
  8. import warnings
  9. from numpy.compat import basestring
  10. import numpy as np
  11. def get_package_name(filepath):
  12. """
  13. Given a path where a package is installed, determine its name.
  14. Parameters
  15. ----------
  16. filepath : str
  17. Path to a file. If the determination fails, "numpy" is returned.
  18. Examples
  19. --------
  20. >>> np.testing.nosetester.get_package_name('nonsense')
  21. 'numpy'
  22. """
  23. fullpath = filepath[:]
  24. pkg_name = []
  25. while 'site-packages' in filepath or 'dist-packages' in filepath:
  26. filepath, p2 = os.path.split(filepath)
  27. if p2 in ('site-packages', 'dist-packages'):
  28. break
  29. pkg_name.append(p2)
  30. # if package name determination failed, just default to numpy/scipy
  31. if not pkg_name:
  32. if 'scipy' in fullpath:
  33. return 'scipy'
  34. else:
  35. return 'numpy'
  36. # otherwise, reverse to get correct order and return
  37. pkg_name.reverse()
  38. # don't include the outer egg directory
  39. if pkg_name[0].endswith('.egg'):
  40. pkg_name.pop(0)
  41. return '.'.join(pkg_name)
  42. def import_nose():
  43. """ Import nose only when needed.
  44. """
  45. fine_nose = True
  46. minimum_nose_version = (1, 0, 0)
  47. try:
  48. import nose
  49. except ImportError:
  50. fine_nose = False
  51. else:
  52. if nose.__versioninfo__ < minimum_nose_version:
  53. fine_nose = False
  54. if not fine_nose:
  55. msg = ('Need nose >= %d.%d.%d for tests - see '
  56. 'http://somethingaboutorange.com/mrl/projects/nose' %
  57. minimum_nose_version)
  58. raise ImportError(msg)
  59. return nose
  60. def run_module_suite(file_to_run=None, argv=None):
  61. """
  62. Run a test module.
  63. Equivalent to calling ``$ nosetests <argv> <file_to_run>`` from
  64. the command line
  65. Parameters
  66. ----------
  67. file_to_run : str, optional
  68. Path to test module, or None.
  69. By default, run the module from which this function is called.
  70. argv : list of strings
  71. Arguments to be passed to the nose test runner. ``argv[0]`` is
  72. ignored. All command line arguments accepted by ``nosetests``
  73. will work. If it is the default value None, sys.argv is used.
  74. .. versionadded:: 1.9.0
  75. Examples
  76. --------
  77. Adding the following::
  78. if __name__ == "__main__" :
  79. run_module_suite(argv=sys.argv)
  80. at the end of a test module will run the tests when that module is
  81. called in the python interpreter.
  82. Alternatively, calling::
  83. >>> run_module_suite(file_to_run="numpy/tests/test_matlib.py")
  84. from an interpreter will run all the test routine in 'test_matlib.py'.
  85. """
  86. if file_to_run is None:
  87. f = sys._getframe(1)
  88. file_to_run = f.f_locals.get('__file__', None)
  89. if file_to_run is None:
  90. raise AssertionError
  91. if argv is None:
  92. argv = sys.argv + [file_to_run]
  93. else:
  94. argv = argv + [file_to_run]
  95. nose = import_nose()
  96. from .noseclasses import KnownFailurePlugin
  97. nose.run(argv=argv, addplugins=[KnownFailurePlugin()])
  98. class NoseTester(object):
  99. """
  100. Nose test runner.
  101. This class is made available as numpy.testing.Tester, and a test function
  102. is typically added to a package's __init__.py like so::
  103. from numpy.testing import Tester
  104. test = Tester().test
  105. Calling this test function finds and runs all tests associated with the
  106. package and all its sub-packages.
  107. Attributes
  108. ----------
  109. package_path : str
  110. Full path to the package to test.
  111. package_name : str
  112. Name of the package to test.
  113. Parameters
  114. ----------
  115. package : module, str or None, optional
  116. The package to test. If a string, this should be the full path to
  117. the package. If None (default), `package` is set to the module from
  118. which `NoseTester` is initialized.
  119. raise_warnings : None, str or sequence of warnings, optional
  120. This specifies which warnings to configure as 'raise' instead
  121. of 'warn' during the test execution. Valid strings are:
  122. - "develop" : equals ``(DeprecationWarning, RuntimeWarning)``
  123. - "release" : equals ``()``, don't raise on any warnings.
  124. Default is "release".
  125. depth : int, optional
  126. If `package` is None, then this can be used to initialize from the
  127. module of the caller of (the caller of (...)) the code that
  128. initializes `NoseTester`. Default of 0 means the module of the
  129. immediate caller; higher values are useful for utility routines that
  130. want to initialize `NoseTester` objects on behalf of other code.
  131. """
  132. def __init__(self, package=None, raise_warnings="release", depth=0):
  133. # Back-compat: 'None' used to mean either "release" or "develop"
  134. # depending on whether this was a release or develop version of
  135. # numpy. Those semantics were fine for testing numpy, but not so
  136. # helpful for downstream projects like scipy that use
  137. # numpy.testing. (They want to set this based on whether *they* are a
  138. # release or develop version, not whether numpy is.) So we continue to
  139. # accept 'None' for back-compat, but it's now just an alias for the
  140. # default "release".
  141. if raise_warnings is None:
  142. raise_warnings = "release"
  143. package_name = None
  144. if package is None:
  145. f = sys._getframe(1 + depth)
  146. package_path = f.f_locals.get('__file__', None)
  147. if package_path is None:
  148. raise AssertionError
  149. package_path = os.path.dirname(package_path)
  150. package_name = f.f_locals.get('__name__', None)
  151. elif isinstance(package, type(os)):
  152. package_path = os.path.dirname(package.__file__)
  153. package_name = getattr(package, '__name__', None)
  154. else:
  155. package_path = str(package)
  156. self.package_path = package_path
  157. # Find the package name under test; this name is used to limit coverage
  158. # reporting (if enabled).
  159. if package_name is None:
  160. package_name = get_package_name(package_path)
  161. self.package_name = package_name
  162. # Set to "release" in constructor in maintenance branches.
  163. self.raise_warnings = raise_warnings
  164. def _test_argv(self, label, verbose, extra_argv):
  165. ''' Generate argv for nosetest command
  166. Parameters
  167. ----------
  168. label : {'fast', 'full', '', attribute identifier}, optional
  169. see ``test`` docstring
  170. verbose : int, optional
  171. Verbosity value for test outputs, in the range 1-10. Default is 1.
  172. extra_argv : list, optional
  173. List with any extra arguments to pass to nosetests.
  174. Returns
  175. -------
  176. argv : list
  177. command line arguments that will be passed to nose
  178. '''
  179. argv = [__file__, self.package_path, '-s']
  180. if label and label != 'full':
  181. if not isinstance(label, basestring):
  182. raise TypeError('Selection label should be a string')
  183. if label == 'fast':
  184. label = 'not slow'
  185. argv += ['-A', label]
  186. argv += ['--verbosity', str(verbose)]
  187. # When installing with setuptools, and also in some other cases, the
  188. # test_*.py files end up marked +x executable. Nose, by default, does
  189. # not run files marked with +x as they might be scripts. However, in
  190. # our case nose only looks for test_*.py files under the package
  191. # directory, which should be safe.
  192. argv += ['--exe']
  193. if extra_argv:
  194. argv += extra_argv
  195. return argv
  196. def _show_system_info(self):
  197. nose = import_nose()
  198. import numpy
  199. print("NumPy version %s" % numpy.__version__)
  200. relaxed_strides = numpy.ones((10, 1), order="C").flags.f_contiguous
  201. print("NumPy relaxed strides checking option:", relaxed_strides)
  202. npdir = os.path.dirname(numpy.__file__)
  203. print("NumPy is installed in %s" % npdir)
  204. if 'scipy' in self.package_name:
  205. import scipy
  206. print("SciPy version %s" % scipy.__version__)
  207. spdir = os.path.dirname(scipy.__file__)
  208. print("SciPy is installed in %s" % spdir)
  209. pyversion = sys.version.replace('\n', '')
  210. print("Python version %s" % pyversion)
  211. print("nose version %d.%d.%d" % nose.__versioninfo__)
  212. def _get_custom_doctester(self):
  213. """ Return instantiated plugin for doctests
  214. Allows subclassing of this class to override doctester
  215. A return value of None means use the nose builtin doctest plugin
  216. """
  217. from .noseclasses import NumpyDoctest
  218. return NumpyDoctest()
  219. def prepare_test_args(self, label='fast', verbose=1, extra_argv=None,
  220. doctests=False, coverage=False):
  221. """
  222. Run tests for module using nose.
  223. This method does the heavy lifting for the `test` method. It takes all
  224. the same arguments, for details see `test`.
  225. See Also
  226. --------
  227. test
  228. """
  229. # fail with nice error message if nose is not present
  230. import_nose()
  231. # compile argv
  232. argv = self._test_argv(label, verbose, extra_argv)
  233. # our way of doing coverage
  234. if coverage:
  235. argv += ['--cover-package=%s' % self.package_name, '--with-coverage',
  236. '--cover-tests', '--cover-erase']
  237. # construct list of plugins
  238. import nose.plugins.builtin
  239. from .noseclasses import KnownFailurePlugin, Unplugger
  240. plugins = [KnownFailurePlugin()]
  241. plugins += [p() for p in nose.plugins.builtin.plugins]
  242. # add doctesting if required
  243. doctest_argv = '--with-doctest' in argv
  244. if doctests == False and doctest_argv:
  245. doctests = True
  246. plug = self._get_custom_doctester()
  247. if plug is None:
  248. # use standard doctesting
  249. if doctests and not doctest_argv:
  250. argv += ['--with-doctest']
  251. else: # custom doctesting
  252. if doctest_argv: # in fact the unplugger would take care of this
  253. argv.remove('--with-doctest')
  254. plugins += [Unplugger('doctest'), plug]
  255. if doctests:
  256. argv += ['--with-' + plug.name]
  257. return argv, plugins
  258. def test(self, label='fast', verbose=1, extra_argv=None,
  259. doctests=False, coverage=False,
  260. raise_warnings=None):
  261. """
  262. Run tests for module using nose.
  263. Parameters
  264. ----------
  265. label : {'fast', 'full', '', attribute identifier}, optional
  266. Identifies the tests to run. This can be a string to pass to
  267. the nosetests executable with the '-A' option, or one of several
  268. special values. Special values are:
  269. * 'fast' - the default - which corresponds to the ``nosetests -A``
  270. option of 'not slow'.
  271. * 'full' - fast (as above) and slow tests as in the
  272. 'no -A' option to nosetests - this is the same as ''.
  273. * None or '' - run all tests.
  274. attribute_identifier - string passed directly to nosetests as '-A'.
  275. verbose : int, optional
  276. Verbosity value for test outputs, in the range 1-10. Default is 1.
  277. extra_argv : list, optional
  278. List with any extra arguments to pass to nosetests.
  279. doctests : bool, optional
  280. If True, run doctests in module. Default is False.
  281. coverage : bool, optional
  282. If True, report coverage of NumPy code. Default is False.
  283. (This requires the `coverage module:
  284. <http://nedbatchelder.com/code/modules/coverage.html>`_).
  285. raise_warnings : str or sequence of warnings, optional
  286. This specifies which warnings to configure as 'raise' instead
  287. of 'warn' during the test execution. Valid strings are:
  288. - "develop" : equals ``(DeprecationWarning, RuntimeWarning)``
  289. - "release" : equals ``()``, don't raise on any warnings.
  290. Returns
  291. -------
  292. result : object
  293. Returns the result of running the tests as a
  294. ``nose.result.TextTestResult`` object.
  295. Notes
  296. -----
  297. Each NumPy module exposes `test` in its namespace to run all tests for it.
  298. For example, to run all tests for numpy.lib:
  299. >>> np.lib.test() #doctest: +SKIP
  300. Examples
  301. --------
  302. >>> result = np.lib.test() #doctest: +SKIP
  303. Running unit tests for numpy.lib
  304. ...
  305. Ran 976 tests in 3.933s
  306. OK
  307. >>> result.errors #doctest: +SKIP
  308. []
  309. >>> result.knownfail #doctest: +SKIP
  310. []
  311. """
  312. # cap verbosity at 3 because nose becomes *very* verbose beyond that
  313. verbose = min(verbose, 3)
  314. from . import utils
  315. utils.verbose = verbose
  316. if doctests:
  317. print("Running unit tests and doctests for %s" % self.package_name)
  318. else:
  319. print("Running unit tests for %s" % self.package_name)
  320. self._show_system_info()
  321. # reset doctest state on every run
  322. import doctest
  323. doctest.master = None
  324. if raise_warnings is None:
  325. raise_warnings = self.raise_warnings
  326. _warn_opts = dict(develop=(DeprecationWarning, RuntimeWarning),
  327. release=())
  328. if isinstance(raise_warnings, basestring):
  329. raise_warnings = _warn_opts[raise_warnings]
  330. with warnings.catch_warnings():
  331. # Reset the warning filters to the default state,
  332. # so that running the tests is more repeatable.
  333. warnings.resetwarnings()
  334. # Set all warnings to 'warn', this is because the default 'once'
  335. # has the bad property of possibly shadowing later warnings.
  336. warnings.filterwarnings('always')
  337. # Force the requested warnings to raise
  338. for warningtype in raise_warnings:
  339. warnings.filterwarnings('error', category=warningtype)
  340. # Filter out annoying import messages.
  341. warnings.filterwarnings('ignore', message='Not importing directory')
  342. warnings.filterwarnings("ignore", message="numpy.dtype size changed")
  343. warnings.filterwarnings("ignore", message="numpy.ufunc size changed")
  344. warnings.filterwarnings("ignore", category=np.ModuleDeprecationWarning)
  345. warnings.filterwarnings("ignore", category=FutureWarning)
  346. # Filter out boolean '-' deprecation messages. This allows
  347. # older versions of scipy to test without a flood of messages.
  348. warnings.filterwarnings("ignore", message=".*boolean negative.*")
  349. warnings.filterwarnings("ignore", message=".*boolean subtract.*")
  350. # Filter out some deprecation warnings inside nose 1.3.7 when run
  351. # on python 3.5b2. See
  352. # https://github.com/nose-devs/nose/issues/929
  353. warnings.filterwarnings("ignore", message=".*getargspec.*",
  354. category=DeprecationWarning,
  355. module="nose\.")
  356. from .noseclasses import NumpyTestProgram
  357. argv, plugins = self.prepare_test_args(
  358. label, verbose, extra_argv, doctests, coverage)
  359. t = NumpyTestProgram(argv=argv, exit=False, plugins=plugins)
  360. return t.result
  361. def bench(self, label='fast', verbose=1, extra_argv=None):
  362. """
  363. Run benchmarks for module using nose.
  364. Parameters
  365. ----------
  366. label : {'fast', 'full', '', attribute identifier}, optional
  367. Identifies the benchmarks to run. This can be a string to pass to
  368. the nosetests executable with the '-A' option, or one of several
  369. special values. Special values are:
  370. * 'fast' - the default - which corresponds to the ``nosetests -A``
  371. option of 'not slow'.
  372. * 'full' - fast (as above) and slow benchmarks as in the
  373. 'no -A' option to nosetests - this is the same as ''.
  374. * None or '' - run all tests.
  375. attribute_identifier - string passed directly to nosetests as '-A'.
  376. verbose : int, optional
  377. Verbosity value for benchmark outputs, in the range 1-10. Default is 1.
  378. extra_argv : list, optional
  379. List with any extra arguments to pass to nosetests.
  380. Returns
  381. -------
  382. success : bool
  383. Returns True if running the benchmarks works, False if an error
  384. occurred.
  385. Notes
  386. -----
  387. Benchmarks are like tests, but have names starting with "bench" instead
  388. of "test", and can be found under the "benchmarks" sub-directory of the
  389. module.
  390. Each NumPy module exposes `bench` in its namespace to run all benchmarks
  391. for it.
  392. Examples
  393. --------
  394. >>> success = np.lib.bench() #doctest: +SKIP
  395. Running benchmarks for numpy.lib
  396. ...
  397. using 562341 items:
  398. unique:
  399. 0.11
  400. unique1d:
  401. 0.11
  402. ratio: 1.0
  403. nUnique: 56230 == 56230
  404. ...
  405. OK
  406. >>> success #doctest: +SKIP
  407. True
  408. """
  409. print("Running benchmarks for %s" % self.package_name)
  410. self._show_system_info()
  411. argv = self._test_argv(label, verbose, extra_argv)
  412. argv += ['--match', r'(?:^|[\\b_\\.%s-])[Bb]ench' % os.sep]
  413. # import nose or make informative error
  414. nose = import_nose()
  415. # get plugin to disable doctests
  416. from .noseclasses import Unplugger
  417. add_plugins = [Unplugger('doctest')]
  418. return nose.run(argv=argv, addplugins=add_plugins)
  419. def _numpy_tester():
  420. if hasattr(np, "__version__") and ".dev0" in np.__version__:
  421. mode = "develop"
  422. else:
  423. mode = "release"
  424. return NoseTester(raise_warnings=mode, depth=1)