testid.py 9.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311
  1. """
  2. This plugin adds a test id (like #1) to each test name output. After
  3. you've run once to generate test ids, you can re-run individual
  4. tests by activating the plugin and passing the ids (with or
  5. without the # prefix) instead of test names.
  6. For example, if your normal test run looks like::
  7. % nosetests -v
  8. tests.test_a ... ok
  9. tests.test_b ... ok
  10. tests.test_c ... ok
  11. When adding ``--with-id`` you'll see::
  12. % nosetests -v --with-id
  13. #1 tests.test_a ... ok
  14. #2 tests.test_b ... ok
  15. #3 tests.test_c ... ok
  16. Then you can re-run individual tests by supplying just an id number::
  17. % nosetests -v --with-id 2
  18. #2 tests.test_b ... ok
  19. You can also pass multiple id numbers::
  20. % nosetests -v --with-id 2 3
  21. #2 tests.test_b ... ok
  22. #3 tests.test_c ... ok
  23. Since most shells consider '#' a special character, you can leave it out when
  24. specifying a test id.
  25. Note that when run without the -v switch, no special output is displayed, but
  26. the ids file is still written.
  27. Looping over failed tests
  28. -------------------------
  29. This plugin also adds a mode that will direct the test runner to record
  30. failed tests. Subsequent test runs will then run only the tests that failed
  31. last time. Activate this mode with the ``--failed`` switch::
  32. % nosetests -v --failed
  33. #1 test.test_a ... ok
  34. #2 test.test_b ... ERROR
  35. #3 test.test_c ... FAILED
  36. #4 test.test_d ... ok
  37. On the second run, only tests #2 and #3 will run::
  38. % nosetests -v --failed
  39. #2 test.test_b ... ERROR
  40. #3 test.test_c ... FAILED
  41. As you correct errors and tests pass, they'll drop out of subsequent runs.
  42. First::
  43. % nosetests -v --failed
  44. #2 test.test_b ... ok
  45. #3 test.test_c ... FAILED
  46. Second::
  47. % nosetests -v --failed
  48. #3 test.test_c ... FAILED
  49. When all tests pass, the full set will run on the next invocation.
  50. First::
  51. % nosetests -v --failed
  52. #3 test.test_c ... ok
  53. Second::
  54. % nosetests -v --failed
  55. #1 test.test_a ... ok
  56. #2 test.test_b ... ok
  57. #3 test.test_c ... ok
  58. #4 test.test_d ... ok
  59. .. note ::
  60. If you expect to use ``--failed`` regularly, it's a good idea to always run
  61. using the ``--with-id`` option. This will ensure that an id file is always
  62. created, allowing you to add ``--failed`` to the command line as soon as
  63. you have failing tests. Otherwise, your first run using ``--failed`` will
  64. (perhaps surprisingly) run *all* tests, because there won't be an id file
  65. containing the record of failed tests from your previous run.
  66. """
  67. __test__ = False
  68. import logging
  69. import os
  70. from nose.plugins import Plugin
  71. from nose.util import src, set
  72. try:
  73. from cPickle import dump, load
  74. except ImportError:
  75. from pickle import dump, load
  76. log = logging.getLogger(__name__)
  77. class TestId(Plugin):
  78. """
  79. Activate to add a test id (like #1) to each test name output. Activate
  80. with --failed to rerun failing tests only.
  81. """
  82. name = 'id'
  83. idfile = None
  84. collecting = True
  85. loopOnFailed = False
  86. def options(self, parser, env):
  87. """Register commandline options.
  88. """
  89. Plugin.options(self, parser, env)
  90. parser.add_option('--id-file', action='store', dest='testIdFile',
  91. default='.noseids', metavar="FILE",
  92. help="Store test ids found in test runs in this "
  93. "file. Default is the file .noseids in the "
  94. "working directory.")
  95. parser.add_option('--failed', action='store_true',
  96. dest='failed', default=False,
  97. help="Run the tests that failed in the last "
  98. "test run.")
  99. def configure(self, options, conf):
  100. """Configure plugin.
  101. """
  102. Plugin.configure(self, options, conf)
  103. if options.failed:
  104. self.enabled = True
  105. self.loopOnFailed = True
  106. log.debug("Looping on failed tests")
  107. self.idfile = os.path.expanduser(options.testIdFile)
  108. if not os.path.isabs(self.idfile):
  109. self.idfile = os.path.join(conf.workingDir, self.idfile)
  110. self.id = 1
  111. # Ids and tests are mirror images: ids are {id: test address} and
  112. # tests are {test address: id}
  113. self.ids = {}
  114. self.tests = {}
  115. self.failed = []
  116. self.source_names = []
  117. # used to track ids seen when tests is filled from
  118. # loaded ids file
  119. self._seen = {}
  120. self._write_hashes = conf.verbosity >= 2
  121. def finalize(self, result):
  122. """Save new ids file, if needed.
  123. """
  124. if result.wasSuccessful():
  125. self.failed = []
  126. if self.collecting:
  127. ids = dict(list(zip(list(self.tests.values()), list(self.tests.keys()))))
  128. else:
  129. ids = self.ids
  130. fh = open(self.idfile, 'wb')
  131. dump({'ids': ids,
  132. 'failed': self.failed,
  133. 'source_names': self.source_names}, fh)
  134. fh.close()
  135. log.debug('Saved test ids: %s, failed %s to %s',
  136. ids, self.failed, self.idfile)
  137. def loadTestsFromNames(self, names, module=None):
  138. """Translate ids in the list of requested names into their
  139. test addresses, if they are found in my dict of tests.
  140. """
  141. log.debug('ltfn %s %s', names, module)
  142. try:
  143. fh = open(self.idfile, 'rb')
  144. data = load(fh)
  145. if 'ids' in data:
  146. self.ids = data['ids']
  147. self.failed = data['failed']
  148. self.source_names = data['source_names']
  149. else:
  150. # old ids field
  151. self.ids = data
  152. self.failed = []
  153. self.source_names = names
  154. if self.ids:
  155. self.id = max(self.ids) + 1
  156. self.tests = dict(list(zip(list(self.ids.values()), list(self.ids.keys()))))
  157. else:
  158. self.id = 1
  159. log.debug(
  160. 'Loaded test ids %s tests %s failed %s sources %s from %s',
  161. self.ids, self.tests, self.failed, self.source_names,
  162. self.idfile)
  163. fh.close()
  164. except ValueError, e:
  165. # load() may throw a ValueError when reading the ids file, if it
  166. # was generated with a newer version of Python than we are currently
  167. # running.
  168. log.debug('Error loading %s : %s', self.idfile, str(e))
  169. except IOError:
  170. log.debug('IO error reading %s', self.idfile)
  171. if self.loopOnFailed and self.failed:
  172. self.collecting = False
  173. names = self.failed
  174. self.failed = []
  175. # I don't load any tests myself, only translate names like '#2'
  176. # into the associated test addresses
  177. translated = []
  178. new_source = []
  179. really_new = []
  180. for name in names:
  181. trans = self.tr(name)
  182. if trans != name:
  183. translated.append(trans)
  184. else:
  185. new_source.append(name)
  186. # names that are not ids and that are not in the current
  187. # list of source names go into the list for next time
  188. if new_source:
  189. new_set = set(new_source)
  190. old_set = set(self.source_names)
  191. log.debug("old: %s new: %s", old_set, new_set)
  192. really_new = [s for s in new_source
  193. if not s in old_set]
  194. if really_new:
  195. # remember new sources
  196. self.source_names.extend(really_new)
  197. if not translated:
  198. # new set of source names, no translations
  199. # means "run the requested tests"
  200. names = new_source
  201. else:
  202. # no new names to translate and add to id set
  203. self.collecting = False
  204. log.debug("translated: %s new sources %s names %s",
  205. translated, really_new, names)
  206. return (None, translated + really_new or names)
  207. def makeName(self, addr):
  208. log.debug("Make name %s", addr)
  209. filename, module, call = addr
  210. if filename is not None:
  211. head = src(filename)
  212. else:
  213. head = module
  214. if call is not None:
  215. return "%s:%s" % (head, call)
  216. return head
  217. def setOutputStream(self, stream):
  218. """Get handle on output stream so the plugin can print id #s
  219. """
  220. self.stream = stream
  221. def startTest(self, test):
  222. """Maybe output an id # before the test name.
  223. Example output::
  224. #1 test.test ... ok
  225. #2 test.test_two ... ok
  226. """
  227. adr = test.address()
  228. log.debug('start test %s (%s)', adr, adr in self.tests)
  229. if adr in self.tests:
  230. if adr in self._seen:
  231. self.write(' ')
  232. else:
  233. self.write('#%s ' % self.tests[adr])
  234. self._seen[adr] = 1
  235. return
  236. self.tests[adr] = self.id
  237. self.write('#%s ' % self.id)
  238. self.id += 1
  239. def afterTest(self, test):
  240. # None means test never ran, False means failed/err
  241. if test.passed is False:
  242. try:
  243. key = str(self.tests[test.address()])
  244. except KeyError:
  245. # never saw this test -- startTest didn't run
  246. pass
  247. else:
  248. if key not in self.failed:
  249. self.failed.append(key)
  250. def tr(self, name):
  251. log.debug("tr '%s'", name)
  252. try:
  253. key = int(name.replace('#', ''))
  254. except ValueError:
  255. return name
  256. log.debug("Got key %s", key)
  257. # I'm running tests mapped from the ids file,
  258. # not collecting new ones
  259. if key in self.ids:
  260. return self.makeName(self.ids[key])
  261. return name
  262. def write(self, output):
  263. if self._write_hashes:
  264. self.stream.write(output)