suite.py 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609
  1. """
  2. Test Suites
  3. -----------
  4. Provides a LazySuite, which is a suite whose test list is a generator
  5. function, and ContextSuite,which can run fixtures (setup/teardown
  6. functions or methods) for the context that contains its tests.
  7. """
  8. from __future__ import generators
  9. import logging
  10. import sys
  11. import unittest
  12. from nose.case import Test
  13. from nose.config import Config
  14. from nose.proxy import ResultProxyFactory
  15. from nose.util import isclass, resolve_name, try_run
  16. if sys.platform == 'cli':
  17. if sys.version_info[:2] < (2, 6):
  18. import clr
  19. clr.AddReference("IronPython")
  20. from IronPython.Runtime.Exceptions import StringException
  21. else:
  22. class StringException(Exception):
  23. pass
  24. log = logging.getLogger(__name__)
  25. #log.setLevel(logging.DEBUG)
  26. # Singleton for default value -- see ContextSuite.__init__ below
  27. _def = object()
  28. def _strclass(cls):
  29. return "%s.%s" % (cls.__module__, cls.__name__)
  30. class MixedContextError(Exception):
  31. """Error raised when a context suite sees tests from more than
  32. one context.
  33. """
  34. pass
  35. class LazySuite(unittest.TestSuite):
  36. """A suite that may use a generator as its list of tests
  37. """
  38. def __init__(self, tests=()):
  39. """Initialize the suite. tests may be an iterable or a generator
  40. """
  41. super(LazySuite, self).__init__()
  42. self._set_tests(tests)
  43. def __iter__(self):
  44. return iter(self._tests)
  45. def __repr__(self):
  46. return "<%s tests=generator (%s)>" % (
  47. _strclass(self.__class__), id(self))
  48. def __hash__(self):
  49. return object.__hash__(self)
  50. __str__ = __repr__
  51. def addTest(self, test):
  52. self._precache.append(test)
  53. # added to bypass run changes in 2.7's unittest
  54. def run(self, result):
  55. for test in self._tests:
  56. if result.shouldStop:
  57. break
  58. test(result)
  59. return result
  60. def __nonzero__(self):
  61. log.debug("tests in %s?", id(self))
  62. if self._precache:
  63. return True
  64. if self.test_generator is None:
  65. return False
  66. try:
  67. test = self.test_generator.next()
  68. if test is not None:
  69. self._precache.append(test)
  70. return True
  71. except StopIteration:
  72. pass
  73. return False
  74. def _get_tests(self):
  75. log.debug("precache is %s", self._precache)
  76. for test in self._precache:
  77. yield test
  78. if self.test_generator is None:
  79. return
  80. for test in self.test_generator:
  81. yield test
  82. def _set_tests(self, tests):
  83. self._precache = []
  84. is_suite = isinstance(tests, unittest.TestSuite)
  85. if callable(tests) and not is_suite:
  86. self.test_generator = tests()
  87. elif is_suite:
  88. # Suites need special treatment: they must be called like
  89. # tests for their setup/teardown to run (if any)
  90. self.addTests([tests])
  91. self.test_generator = None
  92. else:
  93. self.addTests(tests)
  94. self.test_generator = None
  95. _tests = property(_get_tests, _set_tests, None,
  96. "Access the tests in this suite. Access is through a "
  97. "generator, so iteration may not be repeatable.")
  98. class ContextSuite(LazySuite):
  99. """A suite with context.
  100. A ContextSuite executes fixtures (setup and teardown functions or
  101. methods) for the context containing its tests.
  102. The context may be explicitly passed. If it is not, a context (or
  103. nested set of contexts) will be constructed by examining the tests
  104. in the suite.
  105. """
  106. failureException = unittest.TestCase.failureException
  107. was_setup = False
  108. was_torndown = False
  109. classSetup = ('setup_class', 'setup_all', 'setupClass', 'setupAll',
  110. 'setUpClass', 'setUpAll')
  111. classTeardown = ('teardown_class', 'teardown_all', 'teardownClass',
  112. 'teardownAll', 'tearDownClass', 'tearDownAll')
  113. moduleSetup = ('setup_module', 'setupModule', 'setUpModule', 'setup',
  114. 'setUp')
  115. moduleTeardown = ('teardown_module', 'teardownModule', 'tearDownModule',
  116. 'teardown', 'tearDown')
  117. packageSetup = ('setup_package', 'setupPackage', 'setUpPackage')
  118. packageTeardown = ('teardown_package', 'teardownPackage',
  119. 'tearDownPackage')
  120. def __init__(self, tests=(), context=None, factory=None,
  121. config=None, resultProxy=None, can_split=True):
  122. log.debug("Context suite for %s (%s) (%s)", tests, context, id(self))
  123. self.context = context
  124. self.factory = factory
  125. if config is None:
  126. config = Config()
  127. self.config = config
  128. self.resultProxy = resultProxy
  129. self.has_run = False
  130. self.can_split = can_split
  131. self.error_context = None
  132. super(ContextSuite, self).__init__(tests)
  133. def __repr__(self):
  134. return "<%s context=%s>" % (
  135. _strclass(self.__class__),
  136. getattr(self.context, '__name__', self.context))
  137. __str__ = __repr__
  138. def id(self):
  139. if self.error_context:
  140. return '%s:%s' % (repr(self), self.error_context)
  141. else:
  142. return repr(self)
  143. def __hash__(self):
  144. return object.__hash__(self)
  145. # 2.3 compat -- force 2.4 call sequence
  146. def __call__(self, *arg, **kw):
  147. return self.run(*arg, **kw)
  148. def exc_info(self):
  149. """Hook for replacing error tuple output
  150. """
  151. return sys.exc_info()
  152. def _exc_info(self):
  153. """Bottleneck to fix up IronPython string exceptions
  154. """
  155. e = self.exc_info()
  156. if sys.platform == 'cli':
  157. if isinstance(e[0], StringException):
  158. # IronPython throws these StringExceptions, but
  159. # traceback checks type(etype) == str. Make a real
  160. # string here.
  161. e = (str(e[0]), e[1], e[2])
  162. return e
  163. def run(self, result):
  164. """Run tests in suite inside of suite fixtures.
  165. """
  166. # proxy the result for myself
  167. log.debug("suite %s (%s) run called, tests: %s", id(self), self, self._tests)
  168. #import pdb
  169. #pdb.set_trace()
  170. if self.resultProxy:
  171. result, orig = self.resultProxy(result, self), result
  172. else:
  173. result, orig = result, result
  174. try:
  175. self.setUp()
  176. except KeyboardInterrupt:
  177. raise
  178. except:
  179. self.error_context = 'setup'
  180. result.addError(self, self._exc_info())
  181. return
  182. try:
  183. for test in self._tests:
  184. if result.shouldStop:
  185. log.debug("stopping")
  186. break
  187. # each nose.case.Test will create its own result proxy
  188. # so the cases need the original result, to avoid proxy
  189. # chains
  190. test(orig)
  191. finally:
  192. self.has_run = True
  193. try:
  194. self.tearDown()
  195. except KeyboardInterrupt:
  196. raise
  197. except:
  198. self.error_context = 'teardown'
  199. result.addError(self, self._exc_info())
  200. def hasFixtures(self, ctx_callback=None):
  201. context = self.context
  202. if context is None:
  203. return False
  204. if self.implementsAnyFixture(context, ctx_callback=ctx_callback):
  205. return True
  206. # My context doesn't have any, but its ancestors might
  207. factory = self.factory
  208. if factory:
  209. ancestors = factory.context.get(self, [])
  210. for ancestor in ancestors:
  211. if self.implementsAnyFixture(
  212. ancestor, ctx_callback=ctx_callback):
  213. return True
  214. return False
  215. def implementsAnyFixture(self, context, ctx_callback):
  216. if isclass(context):
  217. names = self.classSetup + self.classTeardown
  218. else:
  219. names = self.moduleSetup + self.moduleTeardown
  220. if hasattr(context, '__path__'):
  221. names += self.packageSetup + self.packageTeardown
  222. # If my context has any fixture attribute, I have fixtures
  223. fixt = False
  224. for m in names:
  225. if hasattr(context, m):
  226. fixt = True
  227. break
  228. if ctx_callback is None:
  229. return fixt
  230. return ctx_callback(context, fixt)
  231. def setUp(self):
  232. log.debug("suite %s setUp called, tests: %s", id(self), self._tests)
  233. if not self:
  234. # I have no tests
  235. log.debug("suite %s has no tests", id(self))
  236. return
  237. if self.was_setup:
  238. log.debug("suite %s already set up", id(self))
  239. return
  240. context = self.context
  241. if context is None:
  242. return
  243. # before running my own context's setup, I need to
  244. # ask the factory if my context's contexts' setups have been run
  245. factory = self.factory
  246. if factory:
  247. # get a copy, since we'll be destroying it as we go
  248. ancestors = factory.context.get(self, [])[:]
  249. while ancestors:
  250. ancestor = ancestors.pop()
  251. log.debug("ancestor %s may need setup", ancestor)
  252. if ancestor in factory.was_setup:
  253. continue
  254. log.debug("ancestor %s does need setup", ancestor)
  255. self.setupContext(ancestor)
  256. if not context in factory.was_setup:
  257. self.setupContext(context)
  258. else:
  259. self.setupContext(context)
  260. self.was_setup = True
  261. log.debug("completed suite setup")
  262. def setupContext(self, context):
  263. self.config.plugins.startContext(context)
  264. log.debug("%s setup context %s", self, context)
  265. if self.factory:
  266. if context in self.factory.was_setup:
  267. return
  268. # note that I ran the setup for this context, so that I'll run
  269. # the teardown in my teardown
  270. self.factory.was_setup[context] = self
  271. if isclass(context):
  272. names = self.classSetup
  273. else:
  274. names = self.moduleSetup
  275. if hasattr(context, '__path__'):
  276. names = self.packageSetup + names
  277. try_run(context, names)
  278. def shortDescription(self):
  279. if self.context is None:
  280. return "test suite"
  281. return "test suite for %s" % self.context
  282. def tearDown(self):
  283. log.debug('context teardown')
  284. if not self.was_setup or self.was_torndown:
  285. log.debug(
  286. "No reason to teardown (was_setup? %s was_torndown? %s)"
  287. % (self.was_setup, self.was_torndown))
  288. return
  289. self.was_torndown = True
  290. context = self.context
  291. if context is None:
  292. log.debug("No context to tear down")
  293. return
  294. # for each ancestor... if the ancestor was setup
  295. # and I did the setup, I can do teardown
  296. factory = self.factory
  297. if factory:
  298. ancestors = factory.context.get(self, []) + [context]
  299. for ancestor in ancestors:
  300. log.debug('ancestor %s may need teardown', ancestor)
  301. if not ancestor in factory.was_setup:
  302. log.debug('ancestor %s was not setup', ancestor)
  303. continue
  304. if ancestor in factory.was_torndown:
  305. log.debug('ancestor %s already torn down', ancestor)
  306. continue
  307. setup = factory.was_setup[ancestor]
  308. log.debug("%s setup ancestor %s", setup, ancestor)
  309. if setup is self:
  310. self.teardownContext(ancestor)
  311. else:
  312. self.teardownContext(context)
  313. def teardownContext(self, context):
  314. log.debug("%s teardown context %s", self, context)
  315. if self.factory:
  316. if context in self.factory.was_torndown:
  317. return
  318. self.factory.was_torndown[context] = self
  319. if isclass(context):
  320. names = self.classTeardown
  321. else:
  322. names = self.moduleTeardown
  323. if hasattr(context, '__path__'):
  324. names = self.packageTeardown + names
  325. try_run(context, names)
  326. self.config.plugins.stopContext(context)
  327. # FIXME the wrapping has to move to the factory?
  328. def _get_wrapped_tests(self):
  329. for test in self._get_tests():
  330. if isinstance(test, Test) or isinstance(test, unittest.TestSuite):
  331. yield test
  332. else:
  333. yield Test(test,
  334. config=self.config,
  335. resultProxy=self.resultProxy)
  336. _tests = property(_get_wrapped_tests, LazySuite._set_tests, None,
  337. "Access the tests in this suite. Tests are returned "
  338. "inside of a context wrapper.")
  339. class ContextSuiteFactory(object):
  340. """Factory for ContextSuites. Called with a collection of tests,
  341. the factory decides on a hierarchy of contexts by introspecting
  342. the collection or the tests themselves to find the objects
  343. containing the test objects. It always returns one suite, but that
  344. suite may consist of a hierarchy of nested suites.
  345. """
  346. suiteClass = ContextSuite
  347. def __init__(self, config=None, suiteClass=None, resultProxy=_def):
  348. if config is None:
  349. config = Config()
  350. self.config = config
  351. if suiteClass is not None:
  352. self.suiteClass = suiteClass
  353. # Using a singleton to represent default instead of None allows
  354. # passing resultProxy=None to turn proxying off.
  355. if resultProxy is _def:
  356. resultProxy = ResultProxyFactory(config=config)
  357. self.resultProxy = resultProxy
  358. self.suites = {}
  359. self.context = {}
  360. self.was_setup = {}
  361. self.was_torndown = {}
  362. def __call__(self, tests, **kw):
  363. """Return ``ContextSuite`` for tests. ``tests`` may either
  364. be a callable (in which case the resulting ContextSuite will
  365. have no parent context and be evaluated lazily) or an
  366. iterable. In that case the tests will wrapped in
  367. nose.case.Test, be examined and the context of each found and a
  368. suite of suites returned, organized into a stack with the
  369. outermost suites belonging to the outermost contexts.
  370. """
  371. log.debug("Create suite for %s", tests)
  372. context = kw.pop('context', getattr(tests, 'context', None))
  373. log.debug("tests %s context %s", tests, context)
  374. if context is None:
  375. tests = self.wrapTests(tests)
  376. try:
  377. context = self.findContext(tests)
  378. except MixedContextError:
  379. return self.makeSuite(self.mixedSuites(tests), None, **kw)
  380. return self.makeSuite(tests, context, **kw)
  381. def ancestry(self, context):
  382. """Return the ancestry of the context (that is, all of the
  383. packages and modules containing the context), in order of
  384. descent with the outermost ancestor last.
  385. This method is a generator.
  386. """
  387. log.debug("get ancestry %s", context)
  388. if context is None:
  389. return
  390. # Methods include reference to module they are defined in, we
  391. # don't want that, instead want the module the class is in now
  392. # (classes are re-ancestored elsewhere).
  393. if hasattr(context, 'im_class'):
  394. context = context.im_class
  395. elif hasattr(context, '__self__'):
  396. context = context.__self__.__class__
  397. if hasattr(context, '__module__'):
  398. ancestors = context.__module__.split('.')
  399. elif hasattr(context, '__name__'):
  400. ancestors = context.__name__.split('.')[:-1]
  401. else:
  402. raise TypeError("%s has no ancestors?" % context)
  403. while ancestors:
  404. log.debug(" %s ancestors %s", context, ancestors)
  405. yield resolve_name('.'.join(ancestors))
  406. ancestors.pop()
  407. def findContext(self, tests):
  408. if callable(tests) or isinstance(tests, unittest.TestSuite):
  409. return None
  410. context = None
  411. for test in tests:
  412. # Don't look at suites for contexts, only tests
  413. ctx = getattr(test, 'context', None)
  414. if ctx is None:
  415. continue
  416. if context is None:
  417. context = ctx
  418. elif context != ctx:
  419. raise MixedContextError(
  420. "Tests with different contexts in same suite! %s != %s"
  421. % (context, ctx))
  422. return context
  423. def makeSuite(self, tests, context, **kw):
  424. suite = self.suiteClass(
  425. tests, context=context, config=self.config, factory=self,
  426. resultProxy=self.resultProxy, **kw)
  427. if context is not None:
  428. self.suites.setdefault(context, []).append(suite)
  429. self.context.setdefault(suite, []).append(context)
  430. log.debug("suite %s has context %s", suite,
  431. getattr(context, '__name__', None))
  432. for ancestor in self.ancestry(context):
  433. self.suites.setdefault(ancestor, []).append(suite)
  434. self.context[suite].append(ancestor)
  435. log.debug("suite %s has ancestor %s", suite, ancestor.__name__)
  436. return suite
  437. def mixedSuites(self, tests):
  438. """The complex case where there are tests that don't all share
  439. the same context. Groups tests into suites with common ancestors,
  440. according to the following (essentially tail-recursive) procedure:
  441. Starting with the context of the first test, if it is not
  442. None, look for tests in the remaining tests that share that
  443. ancestor. If any are found, group into a suite with that
  444. ancestor as the context, and replace the current suite with
  445. that suite. Continue this process for each ancestor of the
  446. first test, until all ancestors have been processed. At this
  447. point if any tests remain, recurse with those tests as the
  448. input, returning a list of the common suite (which may be the
  449. suite or test we started with, if no common tests were found)
  450. plus the results of recursion.
  451. """
  452. if not tests:
  453. return []
  454. head = tests.pop(0)
  455. if not tests:
  456. return [head] # short circuit when none are left to combine
  457. suite = head # the common ancestry suite, so far
  458. tail = tests[:]
  459. context = getattr(head, 'context', None)
  460. if context is not None:
  461. ancestors = [context] + [a for a in self.ancestry(context)]
  462. for ancestor in ancestors:
  463. common = [suite] # tests with ancestor in common, so far
  464. remain = [] # tests that remain to be processed
  465. for test in tail:
  466. found_common = False
  467. test_ctx = getattr(test, 'context', None)
  468. if test_ctx is None:
  469. remain.append(test)
  470. continue
  471. if test_ctx is ancestor:
  472. common.append(test)
  473. continue
  474. for test_ancestor in self.ancestry(test_ctx):
  475. if test_ancestor is ancestor:
  476. common.append(test)
  477. found_common = True
  478. break
  479. if not found_common:
  480. remain.append(test)
  481. if common:
  482. suite = self.makeSuite(common, ancestor)
  483. tail = self.mixedSuites(remain)
  484. return [suite] + tail
  485. def wrapTests(self, tests):
  486. log.debug("wrap %s", tests)
  487. if callable(tests) or isinstance(tests, unittest.TestSuite):
  488. log.debug("I won't wrap")
  489. return tests
  490. wrapped = []
  491. for test in tests:
  492. log.debug("wrapping %s", test)
  493. if isinstance(test, Test) or isinstance(test, unittest.TestSuite):
  494. wrapped.append(test)
  495. elif isinstance(test, ContextList):
  496. wrapped.append(self.makeSuite(test, context=test.context))
  497. else:
  498. wrapped.append(
  499. Test(test, config=self.config, resultProxy=self.resultProxy)
  500. )
  501. return wrapped
  502. class ContextList(object):
  503. """Not quite a suite -- a group of tests in a context. This is used
  504. to hint the ContextSuiteFactory about what context the tests
  505. belong to, in cases where it may be ambiguous or missing.
  506. """
  507. def __init__(self, tests, context=None):
  508. self.tests = tests
  509. self.context = context
  510. def __iter__(self):
  511. return iter(self.tests)
  512. class FinalizingSuiteWrapper(unittest.TestSuite):
  513. """Wraps suite and calls final function after suite has
  514. executed. Used to call final functions in cases (like running in
  515. the standard test runner) where test running is not under nose's
  516. control.
  517. """
  518. def __init__(self, suite, finalize):
  519. super(FinalizingSuiteWrapper, self).__init__()
  520. self.suite = suite
  521. self.finalize = finalize
  522. def __call__(self, *arg, **kw):
  523. return self.run(*arg, **kw)
  524. # 2.7 compat
  525. def __iter__(self):
  526. return iter(self.suite)
  527. def run(self, *arg, **kw):
  528. try:
  529. return self.suite(*arg, **kw)
  530. finally:
  531. self.finalize(*arg, **kw)
  532. # backwards compat -- sort of
  533. class TestDir:
  534. def __init__(*arg, **kw):
  535. raise NotImplementedError(
  536. "TestDir is not usable with nose 0.10. The class is present "
  537. "in nose.suite for backwards compatibility purposes but it "
  538. "may not be used.")
  539. class TestModule:
  540. def __init__(*arg, **kw):
  541. raise NotImplementedError(
  542. "TestModule is not usable with nose 0.10. The class is present "
  543. "in nose.suite for backwards compatibility purposes but it "
  544. "may not be used.")