framework.ipp 46 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307
  1. // (C) Copyright Gennadiy Rozental 2001.
  2. // Distributed under the Boost Software License, Version 1.0.
  3. // (See accompanying file LICENSE_1_0.txt or copy at
  4. // http://www.boost.org/LICENSE_1_0.txt)
  5. // See http://www.boost.org/libs/test for the library home page.
  6. //
  7. // File : $RCSfile$
  8. //
  9. // Version : $Revision$
  10. //
  11. // Description : implements framework API - main driver for the test
  12. // ***************************************************************************
  13. #ifndef BOOST_TEST_FRAMEWORK_IPP_021005GER
  14. #define BOOST_TEST_FRAMEWORK_IPP_021005GER
  15. // Boost.Test
  16. #include <boost/test/framework.hpp>
  17. #include <boost/test/execution_monitor.hpp>
  18. #include <boost/test/debug.hpp>
  19. #include <boost/test/unit_test_parameters.hpp>
  20. #include <boost/test/unit_test_log.hpp>
  21. #include <boost/test/unit_test_monitor.hpp>
  22. #include <boost/test/results_collector.hpp>
  23. #include <boost/test/progress_monitor.hpp>
  24. #include <boost/test/results_reporter.hpp>
  25. #include <boost/test/tree/observer.hpp>
  26. #include <boost/test/tree/test_unit.hpp>
  27. #include <boost/test/tree/visitor.hpp>
  28. #include <boost/test/tree/traverse.hpp>
  29. #include <boost/test/tree/test_case_counter.hpp>
  30. #if BOOST_TEST_SUPPORT_TOKEN_ITERATOR
  31. #include <boost/test/utils/iterator/token_iterator.hpp>
  32. #endif
  33. #include <boost/test/utils/foreach.hpp>
  34. #include <boost/test/utils/basic_cstring/io.hpp>
  35. #include <boost/test/detail/global_typedef.hpp>
  36. #include <boost/test/detail/throw_exception.hpp>
  37. // Boost
  38. #include <boost/timer.hpp>
  39. #include <boost/bind.hpp>
  40. // STL
  41. #include <limits>
  42. #include <map>
  43. #include <set>
  44. #include <cstdlib>
  45. #include <ctime>
  46. #ifdef BOOST_NO_STDC_NAMESPACE
  47. namespace std { using ::time; using ::srand; }
  48. #endif
  49. #include <boost/test/detail/suppress_warnings.hpp>
  50. //____________________________________________________________________________//
  51. namespace boost {
  52. namespace unit_test {
  53. namespace framework {
  54. namespace impl {
  55. // ************************************************************************** //
  56. // ************** order detection helpers ************** //
  57. // ************************************************************************** //
  58. struct order_info {
  59. order_info() : depth(-1) {}
  60. int depth;
  61. std::vector<test_unit_id> dependant_siblings;
  62. };
  63. typedef std::set<test_unit_id> tu_id_set;
  64. typedef std::map<test_unit_id,order_info> order_info_per_tu; // !! ?? unordered map
  65. //____________________________________________________________________________//
  66. static test_unit_id
  67. get_tu_parent( test_unit_id tu_id )
  68. {
  69. return framework::get( tu_id, TUT_ANY ).p_parent_id;
  70. }
  71. //____________________________________________________________________________//
  72. static int
  73. tu_depth( test_unit_id tu_id, test_unit_id master_tu_id, order_info_per_tu& tuoi )
  74. {
  75. if( tu_id == master_tu_id )
  76. return 0;
  77. order_info& info = tuoi[tu_id];
  78. if( info.depth == -1 )
  79. info.depth = tu_depth( get_tu_parent( tu_id ), master_tu_id, tuoi ) + 1;
  80. return info.depth;
  81. }
  82. //____________________________________________________________________________//
  83. static void
  84. collect_dependant_siblings( test_unit_id from, test_unit_id to, test_unit_id master_tu_id, order_info_per_tu& tuoi )
  85. {
  86. int from_depth = tu_depth( from, master_tu_id, tuoi );
  87. int to_depth = tu_depth( to, master_tu_id, tuoi );
  88. while(from_depth > to_depth) {
  89. from = get_tu_parent( from );
  90. --from_depth;
  91. }
  92. while(from_depth < to_depth) {
  93. to = get_tu_parent( to );
  94. --to_depth;
  95. }
  96. while(true) {
  97. test_unit_id from_parent = get_tu_parent( from );
  98. test_unit_id to_parent = get_tu_parent( to );
  99. if( from_parent == to_parent )
  100. break;
  101. from = from_parent;
  102. to = to_parent;
  103. }
  104. tuoi[from].dependant_siblings.push_back( to );
  105. }
  106. //____________________________________________________________________________//
  107. static counter_t
  108. assign_sibling_rank( test_unit_id tu_id, order_info_per_tu& tuoi )
  109. {
  110. test_unit& tu = framework::get( tu_id, TUT_ANY );
  111. BOOST_TEST_SETUP_ASSERT( tu.p_sibling_rank != (std::numeric_limits<counter_t>::max)(),
  112. "Cyclic dependency detected involving test unit \"" + tu.full_name() + "\"" );
  113. if( tu.p_sibling_rank != 0 )
  114. return tu.p_sibling_rank;
  115. order_info const& info = tuoi[tu_id];
  116. // indicate in progress
  117. tu.p_sibling_rank.value = (std::numeric_limits<counter_t>::max)();
  118. counter_t new_rank = 1;
  119. BOOST_TEST_FOREACH( test_unit_id, sibling_id, info.dependant_siblings )
  120. new_rank = (std::max)(new_rank, assign_sibling_rank( sibling_id, tuoi ) + 1);
  121. return tu.p_sibling_rank.value = new_rank;
  122. }
  123. //____________________________________________________________________________//
  124. // ************************************************************************** //
  125. // ************** test_init call wrapper ************** //
  126. // ************************************************************************** //
  127. static void
  128. invoke_init_func( init_unit_test_func init_func )
  129. {
  130. #ifdef BOOST_TEST_ALTERNATIVE_INIT_API
  131. BOOST_TEST_I_ASSRT( (*init_func)(), std::runtime_error( "test module initialization failed" ) );
  132. #else
  133. test_suite* manual_test_units = (*init_func)( framework::master_test_suite().argc, framework::master_test_suite().argv );
  134. if( manual_test_units )
  135. framework::master_test_suite().add( manual_test_units );
  136. #endif
  137. }
  138. // ************************************************************************** //
  139. // ************** name_filter ************** //
  140. // ************************************************************************** //
  141. class name_filter : public test_tree_visitor {
  142. struct component {
  143. component( const_string name ) // has to be implicit
  144. {
  145. if( name == "*" )
  146. m_kind = SFK_ALL;
  147. else if( first_char( name ) == '*' && last_char( name ) == '*' ) {
  148. m_kind = SFK_SUBSTR;
  149. m_name = name.substr( 1, name.size()-1 );
  150. }
  151. else if( first_char( name ) == '*' ) {
  152. m_kind = SFK_TRAILING;
  153. m_name = name.substr( 1 );
  154. }
  155. else if( last_char( name ) == '*' ) {
  156. m_kind = SFK_LEADING;
  157. m_name = name.substr( 0, name.size()-1 );
  158. }
  159. else {
  160. m_kind = SFK_MATCH;
  161. m_name = name;
  162. }
  163. }
  164. bool pass( test_unit const& tu ) const
  165. {
  166. const_string name( tu.p_name );
  167. switch( m_kind ) {
  168. default:
  169. case SFK_ALL:
  170. return true;
  171. case SFK_LEADING:
  172. return name.substr( 0, m_name.size() ) == m_name;
  173. case SFK_TRAILING:
  174. return name.size() >= m_name.size() && name.substr( name.size() - m_name.size() ) == m_name;
  175. case SFK_SUBSTR:
  176. return name.find( m_name ) != const_string::npos;
  177. case SFK_MATCH:
  178. return m_name == tu.p_name.get();
  179. }
  180. }
  181. enum kind { SFK_ALL, SFK_LEADING, SFK_TRAILING, SFK_SUBSTR, SFK_MATCH };
  182. kind m_kind;
  183. const_string m_name;
  184. };
  185. public:
  186. // Constructor
  187. name_filter( test_unit_id_list& targ_list, const_string filter_expr ) : m_targ_list( targ_list ), m_depth( 0 )
  188. {
  189. #ifdef BOOST_TEST_SUPPORT_TOKEN_ITERATOR
  190. utils::string_token_iterator tit( filter_expr, (utils::dropped_delimeters = "/",
  191. utils::kept_delimeters = utils::dt_none) );
  192. while( tit != utils::string_token_iterator() ) {
  193. m_components.push_back(
  194. std::vector<component>( utils::string_token_iterator( *tit, (utils::dropped_delimeters = ",",
  195. utils::kept_delimeters = utils::dt_none) ),
  196. utils::string_token_iterator() ) );
  197. ++tit;
  198. }
  199. #endif
  200. }
  201. private:
  202. bool filter_unit( test_unit const& tu )
  203. {
  204. // skip master test suite
  205. if( m_depth == 0 )
  206. return true;
  207. // corresponding name filters are at level m_depth-1
  208. std::vector<component> const& filters = m_components[m_depth-1];
  209. // look for match
  210. return std::find_if( filters.begin(), filters.end(), bind( &component::pass, _1, boost::ref(tu) ) ) != filters.end();
  211. }
  212. // test_tree_visitor interface
  213. virtual void visit( test_case const& tc )
  214. {
  215. // make sure we only accept test cases if we match last component of the filter
  216. if( m_depth == m_components.size() && filter_unit( tc ) )
  217. m_targ_list.push_back( tc.p_id ); // found a test case
  218. }
  219. virtual bool test_suite_start( test_suite const& ts )
  220. {
  221. if( !filter_unit( ts ) )
  222. return false;
  223. if( m_depth < m_components.size() ) {
  224. ++m_depth;
  225. return true;
  226. }
  227. m_targ_list.push_back( ts.p_id ); // found a test suite
  228. return false;
  229. }
  230. virtual void test_suite_finish( test_suite const& /*ts*/ )
  231. {
  232. --m_depth;
  233. }
  234. // Data members
  235. typedef std::vector<std::vector<component> > components_per_level;
  236. components_per_level m_components;
  237. test_unit_id_list& m_targ_list;
  238. unsigned m_depth;
  239. };
  240. // ************************************************************************** //
  241. // ************** label_filter ************** //
  242. // ************************************************************************** //
  243. class label_filter : public test_tree_visitor {
  244. public:
  245. label_filter( test_unit_id_list& targ_list, const_string label )
  246. : m_targ_list( targ_list )
  247. , m_label( label )
  248. {}
  249. private:
  250. // test_tree_visitor interface
  251. virtual bool visit( test_unit const& tu )
  252. {
  253. if( tu.has_label( m_label ) ) {
  254. // found a test unit; add it to list of tu to enable with children and stop recursion in case of suites
  255. m_targ_list.push_back( tu.p_id );
  256. return false;
  257. }
  258. return true;
  259. }
  260. // Data members
  261. test_unit_id_list& m_targ_list;
  262. const_string m_label;
  263. };
  264. // ************************************************************************** //
  265. // ************** set_run_status ************** //
  266. // ************************************************************************** //
  267. class set_run_status : public test_tree_visitor {
  268. public:
  269. explicit set_run_status( test_unit::run_status rs, test_unit_id_list* dep_collector = 0 )
  270. : m_new_status( rs )
  271. , m_dep_collector( dep_collector )
  272. {}
  273. private:
  274. // test_tree_visitor interface
  275. virtual bool visit( test_unit const& tu )
  276. {
  277. const_cast<test_unit&>(tu).p_run_status.value = m_new_status == test_unit::RS_INVALID ? tu.p_default_status : m_new_status;
  278. if( m_dep_collector ) {
  279. BOOST_TEST_FOREACH( test_unit_id, dep_id, tu.p_dependencies.get() ) {
  280. test_unit const& dep = framework::get( dep_id, TUT_ANY );
  281. if( dep.p_run_status == tu.p_run_status )
  282. continue;
  283. BOOST_TEST_MESSAGE( "Including test " << dep.p_type_name << ' ' << dep.full_name() <<
  284. " as a dependency of test " << tu.p_type_name << ' ' << tu.full_name() );
  285. m_dep_collector->push_back( dep_id );
  286. }
  287. }
  288. return true;
  289. }
  290. // Data members
  291. test_unit::run_status m_new_status;
  292. test_unit_id_list* m_dep_collector;
  293. };
  294. // ************************************************************************** //
  295. // ************** parse_filters ************** //
  296. // ************************************************************************** //
  297. static void
  298. add_filtered_test_units( test_unit_id master_tu_id, const_string filter, test_unit_id_list& targ )
  299. {
  300. // Choose between two kinds of filters
  301. if( filter[0] == '@' ) {
  302. filter.trim_left( 1 );
  303. label_filter lf( targ, filter );
  304. traverse_test_tree( master_tu_id, lf, true );
  305. }
  306. else {
  307. name_filter nf( targ, filter );
  308. traverse_test_tree( master_tu_id, nf, true );
  309. }
  310. }
  311. //____________________________________________________________________________//
  312. static bool
  313. parse_filters( test_unit_id master_tu_id, test_unit_id_list& tu_to_enable, test_unit_id_list& tu_to_disable )
  314. {
  315. // 10. collect tu to enable and disable based on filters
  316. bool had_selector_filter = false;
  317. std::vector<std::string> const& filters = runtime_config::get<std::vector<std::string> >( runtime_config::RUN_FILTERS );
  318. BOOST_TEST_FOREACH( const_string, filter, filters ) {
  319. BOOST_TEST_SETUP_ASSERT( !filter.is_empty(), "Invalid filter specification" );
  320. enum { SELECTOR, ENABLER, DISABLER } filter_type = SELECTOR;
  321. // 11. Deduce filter type
  322. if( filter[0] == '!' || filter[0] == '+' ) {
  323. filter_type = filter[0] == '+' ? ENABLER : DISABLER;
  324. filter.trim_left( 1 );
  325. BOOST_TEST_SETUP_ASSERT( !filter.is_empty(), "Invalid filter specification" );
  326. }
  327. had_selector_filter |= filter_type == SELECTOR;
  328. // 12. Add test units to corresponding list
  329. switch( filter_type ) {
  330. case SELECTOR:
  331. case ENABLER: add_filtered_test_units( master_tu_id, filter, tu_to_enable ); break;
  332. case DISABLER: add_filtered_test_units( master_tu_id, filter, tu_to_disable ); break;
  333. }
  334. }
  335. return had_selector_filter;
  336. }
  337. //____________________________________________________________________________//
  338. } // namespace impl
  339. // ************************************************************************** //
  340. // ************** framework::state ************** //
  341. // ************************************************************************** //
  342. unsigned const TIMEOUT_EXCEEDED = static_cast<unsigned>( -1 );
  343. class state {
  344. public:
  345. state()
  346. : m_curr_test_case( INV_TEST_UNIT_ID )
  347. , m_next_test_case_id( MIN_TEST_CASE_ID )
  348. , m_next_test_suite_id( MIN_TEST_SUITE_ID )
  349. , m_test_in_progress( false )
  350. , m_context_idx( 0 )
  351. , m_log_sink( std::cout )
  352. , m_report_sink( std::cerr )
  353. {
  354. }
  355. ~state() { clear(); }
  356. void clear()
  357. {
  358. while( !m_test_units.empty() ) {
  359. test_unit_store::value_type const& tu = *m_test_units.begin();
  360. test_unit const* tu_ptr = tu.second;
  361. // the delete will erase this element from map
  362. if( ut_detail::test_id_2_unit_type( tu.second->p_id ) == TUT_SUITE )
  363. delete static_cast<test_suite const*>(tu_ptr);
  364. else
  365. delete static_cast<test_case const*>(tu_ptr);
  366. }
  367. }
  368. void set_tu_id( test_unit& tu, test_unit_id id ) { tu.p_id.value = id; }
  369. //////////////////////////////////////////////////////////////////
  370. // Validates the dependency graph and deduces the sibling dependency rank for each child
  371. void deduce_siblings_order( test_unit_id tu_id, test_unit_id master_tu_id, impl::order_info_per_tu& tuoi )
  372. {
  373. test_unit& tu = framework::get( tu_id, TUT_ANY );
  374. // collect all sibling dependancy from tu own list
  375. BOOST_TEST_FOREACH( test_unit_id, dep_id, tu.p_dependencies.get() )
  376. collect_dependant_siblings( tu_id, dep_id, master_tu_id, tuoi );
  377. if( tu.p_type != TUT_SUITE )
  378. return;
  379. test_suite& ts = static_cast<test_suite&>(tu);
  380. // recursive call to children first
  381. BOOST_TEST_FOREACH( test_unit_id, chld_id, ts.m_children )
  382. deduce_siblings_order( chld_id, master_tu_id, tuoi );
  383. ts.m_ranked_children.clear();
  384. BOOST_TEST_FOREACH( test_unit_id, chld_id, ts.m_children ) {
  385. counter_t rank = assign_sibling_rank( chld_id, tuoi );
  386. ts.m_ranked_children.insert( std::make_pair( rank, chld_id ) );
  387. }
  388. }
  389. //////////////////////////////////////////////////////////////////
  390. // Finalize default run status:
  391. // 1) inherit run status from parent where applicable
  392. // 2) if any of test units in test suite enabled enable it as well
  393. bool finalize_default_run_status( test_unit_id tu_id, test_unit::run_status parent_status )
  394. {
  395. test_unit& tu = framework::get( tu_id, TUT_ANY );
  396. if( tu.p_default_status == test_suite::RS_INHERIT )
  397. tu.p_default_status.value = parent_status;
  398. // go through list of children
  399. if( tu.p_type == TUT_SUITE ) {
  400. bool has_enabled_child = false;
  401. BOOST_TEST_FOREACH( test_unit_id, chld_id, static_cast<test_suite const&>(tu).m_children )
  402. has_enabled_child |= finalize_default_run_status( chld_id, tu.p_default_status );
  403. tu.p_default_status.value = has_enabled_child ? test_suite::RS_ENABLED : test_suite::RS_DISABLED;
  404. }
  405. return tu.p_default_status == test_suite::RS_ENABLED;
  406. }
  407. //////////////////////////////////////////////////////////////////
  408. bool finalize_run_status( test_unit_id tu_id )
  409. {
  410. test_unit& tu = framework::get( tu_id, TUT_ANY );
  411. // go through list of children
  412. if( tu.p_type == TUT_SUITE ) {
  413. bool has_enabled_child = false;
  414. BOOST_TEST_FOREACH( test_unit_id, chld_id, static_cast<test_suite const&>(tu).m_children)
  415. has_enabled_child |= finalize_run_status( chld_id );
  416. tu.p_run_status.value = has_enabled_child ? test_suite::RS_ENABLED : test_suite::RS_DISABLED;
  417. }
  418. return tu.is_enabled();
  419. }
  420. //////////////////////////////////////////////////////////////////
  421. void deduce_run_status( test_unit_id master_tu_id )
  422. {
  423. using namespace framework::impl;
  424. test_unit_id_list tu_to_enable;
  425. test_unit_id_list tu_to_disable;
  426. // 10. If there are any filters supplied, figure out lists of test units to enable/disable
  427. bool had_selector_filter = !runtime_config::get<std::vector<std::string> >( runtime_config::RUN_FILTERS ).empty() &&
  428. parse_filters( master_tu_id, tu_to_enable, tu_to_disable );
  429. // 20. Set the stage: either use default run status or disable all test units
  430. set_run_status initial_setter( had_selector_filter ? test_unit::RS_DISABLED : test_unit::RS_INVALID );
  431. traverse_test_tree( master_tu_id, initial_setter, true );
  432. // 30. Apply all selectors and enablers.
  433. while( !tu_to_enable.empty() ) {
  434. test_unit& tu = framework::get( tu_to_enable.back(), TUT_ANY );
  435. tu_to_enable.pop_back();
  436. // 35. Ignore test units which already enabled
  437. if( tu.is_enabled() )
  438. continue;
  439. // set new status and add all dependencies into tu_to_enable
  440. set_run_status enabler( test_unit::RS_ENABLED, &tu_to_enable );
  441. traverse_test_tree( tu.p_id, enabler, true );
  442. }
  443. // 40. Apply all disablers
  444. while( !tu_to_disable.empty() ) {
  445. test_unit const& tu = framework::get( tu_to_disable.back(), TUT_ANY );
  446. tu_to_disable.pop_back();
  447. // 35. Ignore test units which already disabled
  448. if( !tu.is_enabled() )
  449. continue;
  450. set_run_status disabler( test_unit::RS_DISABLED );
  451. traverse_test_tree( tu.p_id, disabler, true );
  452. }
  453. // 50. Make sure parents of enabled test units are also enabled
  454. finalize_run_status( master_tu_id );
  455. }
  456. //////////////////////////////////////////////////////////////////
  457. typedef unit_test_monitor_t::error_level execution_result;
  458. // Random generator using the std::rand function (seeded prior to the call)
  459. struct random_generator_helper {
  460. size_t operator()(size_t i) const {
  461. return std::rand() % i;
  462. }
  463. };
  464. // Executed the test tree with the root at specified test unit
  465. execution_result execute_test_tree( test_unit_id tu_id,
  466. unsigned timeout = 0,
  467. random_generator_helper const * const p_random_generator = 0)
  468. {
  469. test_unit const& tu = framework::get( tu_id, TUT_ANY );
  470. execution_result result = unit_test_monitor_t::test_ok;
  471. if( !tu.is_enabled() )
  472. return result;
  473. // 10. Check preconditions, including zero time left for execution and
  474. // successful execution of all dependencies
  475. if( timeout == TIMEOUT_EXCEEDED ) {
  476. // notify all observers about skipped test unit
  477. BOOST_TEST_FOREACH( test_observer*, to, m_observers )
  478. to->test_unit_skipped( tu, "timeout for the test unit is exceeded" );
  479. return unit_test_monitor_t::os_timeout;
  480. }
  481. else if( timeout == 0 || timeout > tu.p_timeout ) // deduce timeout for this test unit
  482. timeout = tu.p_timeout;
  483. test_tools::assertion_result const precondition_res = tu.check_preconditions();
  484. if( !precondition_res ) {
  485. // notify all observers about skipped test unit
  486. BOOST_TEST_FOREACH( test_observer*, to, m_observers )
  487. to->test_unit_skipped( tu, precondition_res.message() );
  488. return unit_test_monitor_t::precondition_failure;
  489. }
  490. // 20. Notify all observers about the start of the test unit
  491. BOOST_TEST_FOREACH( test_observer*, to, m_observers )
  492. to->test_unit_start( tu );
  493. // 30. Execute setup fixtures if any; any failure here leads to test unit abortion
  494. BOOST_TEST_FOREACH( test_unit_fixture_ptr, F, tu.p_fixtures.get() ) {
  495. result = unit_test_monitor.execute_and_translate( boost::bind( &test_unit_fixture::setup, F ) );
  496. if( result != unit_test_monitor_t::test_ok )
  497. break;
  498. }
  499. // This is the time we are going to spend executing the test unit
  500. unsigned long elapsed = 0;
  501. if( result == unit_test_monitor_t::test_ok ) {
  502. // 40. We are going to time the execution
  503. boost::timer tu_timer;
  504. if( tu.p_type == TUT_SUITE ) {
  505. test_suite const& ts = static_cast<test_suite const&>( tu );
  506. if( runtime_config::get<unsigned>( runtime_config::RANDOM_SEED ) == 0 ) {
  507. typedef std::pair<counter_t,test_unit_id> value_type;
  508. BOOST_TEST_FOREACH( value_type, chld, ts.m_ranked_children ) {
  509. unsigned chld_timeout = child_timeout( timeout, tu_timer.elapsed() );
  510. result = (std::min)( result, execute_test_tree( chld.second, chld_timeout ) );
  511. if( unit_test_monitor.is_critical_error( result ) )
  512. break;
  513. }
  514. }
  515. else {
  516. // Go through ranges of chldren with the same dependency rank and shuffle them
  517. // independently. Execute each subtree in this order
  518. test_unit_id_list children_with_the_same_rank;
  519. typedef test_suite::children_per_rank::const_iterator it_type;
  520. it_type it = ts.m_ranked_children.begin();
  521. while( it != ts.m_ranked_children.end() ) {
  522. children_with_the_same_rank.clear();
  523. std::pair<it_type,it_type> range = ts.m_ranked_children.equal_range( it->first );
  524. it = range.first;
  525. while( it != range.second ) {
  526. children_with_the_same_rank.push_back( it->second );
  527. it++;
  528. }
  529. const random_generator_helper& rand_gen = p_random_generator ? *p_random_generator : random_generator_helper();
  530. std::random_shuffle( children_with_the_same_rank.begin(), children_with_the_same_rank.end(), rand_gen );
  531. BOOST_TEST_FOREACH( test_unit_id, chld, children_with_the_same_rank ) {
  532. unsigned chld_timeout = child_timeout( timeout, tu_timer.elapsed() );
  533. result = (std::min)( result, execute_test_tree( chld, chld_timeout, &rand_gen ) );
  534. if( unit_test_monitor.is_critical_error( result ) )
  535. break;
  536. }
  537. }
  538. }
  539. elapsed = static_cast<unsigned long>( tu_timer.elapsed() * 1e6 );
  540. }
  541. else { // TUT_CASE
  542. test_case const& tc = static_cast<test_case const&>( tu );
  543. // setup contexts
  544. m_context_idx = 0;
  545. // setup current test case
  546. test_unit_id bkup = m_curr_test_case;
  547. m_curr_test_case = tc.p_id;
  548. // execute the test case body
  549. result = unit_test_monitor.execute_and_translate( tc.p_test_func, timeout );
  550. elapsed = static_cast<unsigned long>( tu_timer.elapsed() * 1e6 );
  551. // cleanup leftover context
  552. m_context.clear();
  553. // restore state and abort if necessary
  554. m_curr_test_case = bkup;
  555. }
  556. }
  557. // if run error is critical skip teardown, who knows what the state of the program at this point
  558. if( !unit_test_monitor.is_critical_error( result ) ) {
  559. // execute teardown fixtures if any in reverse order
  560. BOOST_TEST_REVERSE_FOREACH( test_unit_fixture_ptr, F, tu.p_fixtures.get() ) {
  561. result = (std::min)( result, unit_test_monitor.execute_and_translate( boost::bind( &test_unit_fixture::teardown, F ), 0 ) );
  562. if( unit_test_monitor.is_critical_error( result ) )
  563. break;
  564. }
  565. }
  566. // notify all observers about abortion
  567. if( unit_test_monitor.is_critical_error( result ) ) {
  568. BOOST_TEST_FOREACH( test_observer*, to, m_observers )
  569. to->test_aborted();
  570. }
  571. // notify all observers about completion
  572. BOOST_TEST_REVERSE_FOREACH( test_observer*, to, m_observers )
  573. to->test_unit_finish( tu, elapsed );
  574. return result;
  575. }
  576. //////////////////////////////////////////////////////////////////
  577. unsigned child_timeout( unsigned tu_timeout, double elapsed )
  578. {
  579. if( tu_timeout == 0U )
  580. return 0U;
  581. unsigned elpsed_sec = static_cast<unsigned>(elapsed); // rounding to number of whole seconds
  582. return tu_timeout > elpsed_sec ? tu_timeout - elpsed_sec : TIMEOUT_EXCEEDED;
  583. }
  584. struct priority_order {
  585. bool operator()( test_observer* lhs, test_observer* rhs ) const
  586. {
  587. return (lhs->priority() < rhs->priority()) || ((lhs->priority() == rhs->priority()) && (lhs < rhs));
  588. }
  589. };
  590. // Data members
  591. typedef std::map<test_unit_id,test_unit*> test_unit_store;
  592. typedef std::set<test_observer*,priority_order> observer_store;
  593. struct context_frame {
  594. context_frame( std::string const& d, int id, bool sticky )
  595. : descr( d )
  596. , frame_id( id )
  597. , is_sticky( sticky )
  598. {}
  599. std::string descr;
  600. int frame_id;
  601. bool is_sticky;
  602. };
  603. typedef std::vector<context_frame> context_data;
  604. master_test_suite_t* m_master_test_suite;
  605. std::vector<test_suite*> m_auto_test_suites;
  606. test_unit_id m_curr_test_case;
  607. test_unit_store m_test_units;
  608. test_unit_id m_next_test_case_id;
  609. test_unit_id m_next_test_suite_id;
  610. bool m_test_in_progress;
  611. observer_store m_observers;
  612. context_data m_context;
  613. int m_context_idx;
  614. boost::execution_monitor m_aux_em;
  615. runtime_config::stream_holder m_log_sink;
  616. runtime_config::stream_holder m_report_sink;
  617. };
  618. //____________________________________________________________________________//
  619. namespace impl {
  620. namespace {
  621. #if defined(__CYGWIN__)
  622. framework::state& s_frk_state() { static framework::state* the_inst = 0; if(!the_inst) the_inst = new framework::state; return *the_inst; }
  623. #else
  624. framework::state& s_frk_state() { static framework::state the_inst; return the_inst; }
  625. #endif
  626. } // local namespace
  627. void
  628. setup_for_execution( test_unit const& tu )
  629. {
  630. s_frk_state().deduce_run_status( tu.p_id );
  631. }
  632. //____________________________________________________________________________//
  633. } // namespace impl
  634. //____________________________________________________________________________//
  635. // ************************************************************************** //
  636. // ************** framework::init ************** //
  637. // ************************************************************************** //
  638. void
  639. init( init_unit_test_func init_func, int argc, char* argv[] )
  640. {
  641. using namespace impl;
  642. // 10. Set up runtime parameters
  643. runtime_config::init( argc, argv );
  644. // 20. Set the desired log level, format and sink
  645. unit_test_log.set_threshold_level( runtime_config::get<log_level>( runtime_config::LOG_LEVEL ) );
  646. unit_test_log.set_format( runtime_config::get<output_format>( runtime_config::LOG_FORMAT ) );
  647. s_frk_state().m_log_sink.setup( runtime_config::LOG_SINK );
  648. unit_test_log.set_stream( s_frk_state().m_log_sink.ref() );
  649. // 30. Set the desired report level, format and sink
  650. results_reporter::set_level( runtime_config::get<report_level>( runtime_config::REPORT_LEVEL ) );
  651. results_reporter::set_format( runtime_config::get<output_format>( runtime_config::REPORT_FORMAT ) );
  652. s_frk_state().m_report_sink.setup( runtime_config::REPORT_SINK );
  653. results_reporter::set_stream( s_frk_state().m_report_sink.ref() );
  654. // 40. Register default test observers
  655. register_observer( results_collector );
  656. register_observer( unit_test_log );
  657. if( runtime_config::get<bool>( runtime_config::SHOW_PROGRESS ) ) {
  658. progress_monitor.set_stream( s_frk_state().m_log_sink.ref() );
  659. register_observer( progress_monitor );
  660. }
  661. // 50. Set up memory leak detection
  662. unsigned long detect_mem_leak = runtime_config::get<unsigned long>( runtime_config::DETECT_MEM_LEAKS );
  663. if( detect_mem_leak > 0 ) {
  664. debug::detect_memory_leaks( true, runtime_config::get<std::string>( runtime_config::REPORT_MEM_LEAKS ) );
  665. debug::break_memory_alloc( (long)detect_mem_leak );
  666. }
  667. // 60. Initialize master unit test suite
  668. master_test_suite().argc = argc;
  669. master_test_suite().argv = argv;
  670. // 70. Invoke test module initialization routine
  671. BOOST_TEST_I_TRY {
  672. s_frk_state().m_aux_em.vexecute( boost::bind( &impl::invoke_init_func, init_func ) );
  673. }
  674. BOOST_TEST_I_CATCH( execution_exception, ex ) {
  675. BOOST_TEST_SETUP_ASSERT( false, ex.what() );
  676. }
  677. }
  678. //____________________________________________________________________________//
  679. void
  680. finalize_setup_phase( test_unit_id master_tu_id )
  681. {
  682. if( master_tu_id == INV_TEST_UNIT_ID )
  683. master_tu_id = master_test_suite().p_id;
  684. // 10. Apply all decorators to the auto test units
  685. class apply_decorators : public test_tree_visitor {
  686. private:
  687. // test_tree_visitor interface
  688. virtual bool visit( test_unit const& tu )
  689. {
  690. BOOST_TEST_FOREACH( decorator::base_ptr, d, tu.p_decorators.get() )
  691. d->apply( const_cast<test_unit&>(tu) );
  692. return true;
  693. }
  694. } ad;
  695. traverse_test_tree( master_tu_id, ad, true );
  696. // 20. Finalize setup phase
  697. impl::order_info_per_tu tuoi;
  698. impl::s_frk_state().deduce_siblings_order( master_tu_id, master_tu_id, tuoi );
  699. impl::s_frk_state().finalize_default_run_status( master_tu_id, test_unit::RS_INVALID );
  700. }
  701. // ************************************************************************** //
  702. // ************** test_in_progress ************** //
  703. // ************************************************************************** //
  704. bool
  705. test_in_progress()
  706. {
  707. return impl::s_frk_state().m_test_in_progress;
  708. }
  709. //____________________________________________________________________________//
  710. // ************************************************************************** //
  711. // ************** framework::shutdown ************** //
  712. // ************************************************************************** //
  713. void
  714. shutdown()
  715. {
  716. // eliminating some fake memory leak reports. See for more details:
  717. // http://connect.microsoft.com/VisualStudio/feedback/details/106937/memory-leaks-reported-by-debug-crt-inside-typeinfo-name
  718. # if BOOST_WORKAROUND(BOOST_MSVC, <= 1600 ) && !defined(_DLL) && defined(_DEBUG)
  719. # if BOOST_WORKAROUND(BOOST_MSVC, < 1600 )
  720. #define _Next next
  721. #define _MemPtr memPtr
  722. #endif
  723. __type_info_node* pNode = __type_info_root_node._Next;
  724. __type_info_node* tmpNode = &__type_info_root_node;
  725. for( ; pNode!=NULL; pNode = tmpNode ) {
  726. tmpNode = pNode->_Next;
  727. delete pNode->_MemPtr;
  728. delete pNode;
  729. }
  730. # if BOOST_WORKAROUND(BOOST_MSVC, < 1600 )
  731. #undef _Next
  732. #undef _MemPtr
  733. #endif
  734. # endif
  735. }
  736. //____________________________________________________________________________//
  737. // ************************************************************************** //
  738. // ************** register_test_unit ************** //
  739. // ************************************************************************** //
  740. void
  741. register_test_unit( test_case* tc )
  742. {
  743. BOOST_TEST_SETUP_ASSERT( tc->p_id == INV_TEST_UNIT_ID, BOOST_TEST_L( "test case already registered" ) );
  744. test_unit_id new_id = impl::s_frk_state().m_next_test_case_id;
  745. BOOST_TEST_SETUP_ASSERT( new_id != MAX_TEST_CASE_ID, BOOST_TEST_L( "too many test cases" ) );
  746. typedef state::test_unit_store::value_type map_value_type;
  747. impl::s_frk_state().m_test_units.insert( map_value_type( new_id, tc ) );
  748. impl::s_frk_state().m_next_test_case_id++;
  749. impl::s_frk_state().set_tu_id( *tc, new_id );
  750. }
  751. //____________________________________________________________________________//
  752. // ************************************************************************** //
  753. // ************** register_test_unit ************** //
  754. // ************************************************************************** //
  755. void
  756. register_test_unit( test_suite* ts )
  757. {
  758. BOOST_TEST_SETUP_ASSERT( ts->p_id == INV_TEST_UNIT_ID, BOOST_TEST_L( "test suite already registered" ) );
  759. test_unit_id new_id = impl::s_frk_state().m_next_test_suite_id;
  760. BOOST_TEST_SETUP_ASSERT( new_id != MAX_TEST_SUITE_ID, BOOST_TEST_L( "too many test suites" ) );
  761. typedef state::test_unit_store::value_type map_value_type;
  762. impl::s_frk_state().m_test_units.insert( map_value_type( new_id, ts ) );
  763. impl::s_frk_state().m_next_test_suite_id++;
  764. impl::s_frk_state().set_tu_id( *ts, new_id );
  765. }
  766. //____________________________________________________________________________//
  767. // ************************************************************************** //
  768. // ************** deregister_test_unit ************** //
  769. // ************************************************************************** //
  770. void
  771. deregister_test_unit( test_unit* tu )
  772. {
  773. impl::s_frk_state().m_test_units.erase( tu->p_id );
  774. }
  775. //____________________________________________________________________________//
  776. // ************************************************************************** //
  777. // ************** clear ************** //
  778. // ************************************************************************** //
  779. void
  780. clear()
  781. {
  782. impl::s_frk_state().clear();
  783. }
  784. //____________________________________________________________________________//
  785. // ************************************************************************** //
  786. // ************** register_observer ************** //
  787. // ************************************************************************** //
  788. void
  789. register_observer( test_observer& to )
  790. {
  791. impl::s_frk_state().m_observers.insert( &to );
  792. }
  793. //____________________________________________________________________________//
  794. // ************************************************************************** //
  795. // ************** deregister_observer ************** //
  796. // ************************************************************************** //
  797. void
  798. deregister_observer( test_observer& to )
  799. {
  800. impl::s_frk_state().m_observers.erase( &to );
  801. }
  802. //____________________________________________________________________________//
  803. // ************************************************************************** //
  804. // ************** add_context ************** //
  805. // ************************************************************************** //
  806. int
  807. add_context( ::boost::unit_test::lazy_ostream const& context_descr, bool sticky )
  808. {
  809. std::stringstream buffer;
  810. context_descr( buffer );
  811. int res_idx = impl::s_frk_state().m_context_idx++;
  812. impl::s_frk_state().m_context.push_back( state::context_frame( buffer.str(), res_idx, sticky ) );
  813. return res_idx;
  814. }
  815. //____________________________________________________________________________//
  816. // ************************************************************************** //
  817. // ************** clear_context ************** //
  818. // ************************************************************************** //
  819. struct frame_with_id {
  820. explicit frame_with_id( int id ) : m_id( id ) {}
  821. bool operator()( state::context_frame const& f )
  822. {
  823. return f.frame_id == m_id;
  824. }
  825. int m_id;
  826. };
  827. //____________________________________________________________________________//
  828. void
  829. clear_context( int frame_id )
  830. {
  831. if( frame_id == -1 ) { // clear all non sticky frames
  832. for( int i=static_cast<int>(impl::s_frk_state().m_context.size())-1; i>=0; i-- )
  833. if( !impl::s_frk_state().m_context[i].is_sticky )
  834. impl::s_frk_state().m_context.erase( impl::s_frk_state().m_context.begin()+i );
  835. }
  836. else { // clear specific frame
  837. state::context_data::iterator it =
  838. std::find_if( impl::s_frk_state().m_context.begin(), impl::s_frk_state().m_context.end(), frame_with_id( frame_id ) );
  839. if( it != impl::s_frk_state().m_context.end() ) // really an internal error if this is not true
  840. impl::s_frk_state().m_context.erase( it );
  841. }
  842. }
  843. //____________________________________________________________________________//
  844. // ************************************************************************** //
  845. // ************** get_context ************** //
  846. // ************************************************************************** //
  847. context_generator
  848. get_context()
  849. {
  850. return context_generator();
  851. }
  852. //____________________________________________________________________________//
  853. // ************************************************************************** //
  854. // ************** context_generator ************** //
  855. // ************************************************************************** //
  856. bool
  857. context_generator::is_empty() const
  858. {
  859. return impl::s_frk_state().m_context.empty();
  860. }
  861. //____________________________________________________________________________//
  862. const_string
  863. context_generator::next() const
  864. {
  865. return m_curr_frame < impl::s_frk_state().m_context.size() ? impl::s_frk_state().m_context[m_curr_frame++].descr : const_string();
  866. }
  867. //____________________________________________________________________________//
  868. // ************************************************************************** //
  869. // ************** master_test_suite ************** //
  870. // ************************************************************************** //
  871. master_test_suite_t&
  872. master_test_suite()
  873. {
  874. if( !impl::s_frk_state().m_master_test_suite )
  875. impl::s_frk_state().m_master_test_suite = new master_test_suite_t;
  876. return *impl::s_frk_state().m_master_test_suite;
  877. }
  878. //____________________________________________________________________________//
  879. // ************************************************************************** //
  880. // ************** current_auto_test_suite ************** //
  881. // ************************************************************************** //
  882. test_suite&
  883. current_auto_test_suite( test_suite* ts, bool push_or_pop )
  884. {
  885. if( impl::s_frk_state().m_auto_test_suites.empty() )
  886. impl::s_frk_state().m_auto_test_suites.push_back( &framework::master_test_suite() );
  887. if( !push_or_pop )
  888. impl::s_frk_state().m_auto_test_suites.pop_back();
  889. else if( ts )
  890. impl::s_frk_state().m_auto_test_suites.push_back( ts );
  891. return *impl::s_frk_state().m_auto_test_suites.back();
  892. }
  893. //____________________________________________________________________________//
  894. // ************************************************************************** //
  895. // ************** current_test_case ************** //
  896. // ************************************************************************** //
  897. test_case const&
  898. current_test_case()
  899. {
  900. return get<test_case>( impl::s_frk_state().m_curr_test_case );
  901. }
  902. //____________________________________________________________________________//
  903. test_unit_id
  904. current_test_case_id()
  905. {
  906. return impl::s_frk_state().m_curr_test_case;
  907. }
  908. //____________________________________________________________________________//
  909. // ************************************************************************** //
  910. // ************** framework::get ************** //
  911. // ************************************************************************** //
  912. test_unit&
  913. get( test_unit_id id, test_unit_type t )
  914. {
  915. test_unit* res = impl::s_frk_state().m_test_units[id];
  916. BOOST_TEST_I_ASSRT( (res->p_type & t) != 0, internal_error( "Invalid test unit type" ) );
  917. return *res;
  918. }
  919. //____________________________________________________________________________//
  920. // ************************************************************************** //
  921. // ************** framework::run ************** //
  922. // ************************************************************************** //
  923. void
  924. run( test_unit_id id, bool continue_test )
  925. {
  926. if( id == INV_TEST_UNIT_ID )
  927. id = master_test_suite().p_id;
  928. // Figure out run status for execution phase
  929. impl::s_frk_state().deduce_run_status( id );
  930. test_case_counter tcc;
  931. traverse_test_tree( id, tcc );
  932. BOOST_TEST_SETUP_ASSERT( tcc.p_count != 0 , runtime_config::get<std::vector<std::string> >( runtime_config::RUN_FILTERS ).empty()
  933. ? BOOST_TEST_L( "test tree is empty" )
  934. : BOOST_TEST_L( "no test cases matching filter or all test cases were disabled" ) );
  935. bool was_in_progress = framework::test_in_progress();
  936. bool call_start_finish = !continue_test || !was_in_progress;
  937. impl::s_frk_state().m_test_in_progress = true;
  938. if( call_start_finish ) {
  939. BOOST_TEST_FOREACH( test_observer*, to, impl::s_frk_state().m_observers ) {
  940. BOOST_TEST_I_TRY {
  941. impl::s_frk_state().m_aux_em.vexecute( boost::bind( &test_observer::test_start, to, tcc.p_count ) );
  942. }
  943. BOOST_TEST_I_CATCH( execution_exception, ex ) {
  944. BOOST_TEST_SETUP_ASSERT( false, ex.what() );
  945. }
  946. }
  947. }
  948. unsigned seed = runtime_config::get<unsigned>( runtime_config::RANDOM_SEED );
  949. switch( seed ) {
  950. case 0:
  951. break;
  952. case 1:
  953. seed = static_cast<unsigned>( std::rand() ^ std::time( 0 ) ); // better init using std::rand() ^ ...
  954. default:
  955. BOOST_TEST_MESSAGE( "Test cases order is shuffled using seed: " << seed );
  956. std::srand( seed );
  957. }
  958. impl::s_frk_state().execute_test_tree( id );
  959. if( call_start_finish ) {
  960. BOOST_TEST_REVERSE_FOREACH( test_observer*, to, impl::s_frk_state().m_observers )
  961. to->test_finish();
  962. }
  963. impl::s_frk_state().m_test_in_progress = was_in_progress;
  964. }
  965. //____________________________________________________________________________//
  966. void
  967. run( test_unit const* tu, bool continue_test )
  968. {
  969. run( tu->p_id, continue_test );
  970. }
  971. //____________________________________________________________________________//
  972. // ************************************************************************** //
  973. // ************** assertion_result ************** //
  974. // ************************************************************************** //
  975. void
  976. assertion_result( unit_test::assertion_result ar )
  977. {
  978. BOOST_TEST_FOREACH( test_observer*, to, impl::s_frk_state().m_observers )
  979. to->assertion_result( ar );
  980. }
  981. //____________________________________________________________________________//
  982. // ************************************************************************** //
  983. // ************** exception_caught ************** //
  984. // ************************************************************************** //
  985. void
  986. exception_caught( execution_exception const& ex )
  987. {
  988. BOOST_TEST_FOREACH( test_observer*, to, impl::s_frk_state().m_observers )
  989. to->exception_caught( ex );
  990. }
  991. //____________________________________________________________________________//
  992. // ************************************************************************** //
  993. // ************** test_unit_aborted ************** //
  994. // ************************************************************************** //
  995. void
  996. test_unit_aborted( test_unit const& tu )
  997. {
  998. BOOST_TEST_FOREACH( test_observer*, to, impl::s_frk_state().m_observers )
  999. to->test_unit_aborted( tu );
  1000. }
  1001. //____________________________________________________________________________//
  1002. } // namespace framework
  1003. } // namespace unit_test
  1004. } // namespace boost
  1005. #include <boost/test/detail/enable_warnings.hpp>
  1006. #endif // BOOST_TEST_FRAMEWORK_IPP_021005GER