cmCTestTestHandler.h 9.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318
  1. /* Distributed under the OSI-approved BSD 3-Clause License. See accompanying
  2. file Copyright.txt or https://cmake.org/licensing for details. */
  3. #ifndef cmCTestTestHandler_h
  4. #define cmCTestTestHandler_h
  5. #include "cmConfigure.h" // IWYU pragma: keep
  6. #include "cmCTestGenericHandler.h"
  7. #include "cmDuration.h"
  8. #include "cmsys/RegularExpression.hxx"
  9. #include <chrono>
  10. #include <iosfwd>
  11. #include <map>
  12. #include <set>
  13. #include <stddef.h>
  14. #include <string>
  15. #include <utility>
  16. #include <vector>
  17. class cmCTest;
  18. class cmMakefile;
  19. class cmXMLWriter;
  20. /** \class cmCTestTestHandler
  21. * \brief A class that handles ctest -S invocations
  22. *
  23. */
  24. class cmCTestTestHandler : public cmCTestGenericHandler
  25. {
  26. friend class cmCTestRunTest;
  27. friend class cmCTestMultiProcessHandler;
  28. public:
  29. typedef cmCTestGenericHandler Superclass;
  30. /**
  31. * The main entry point for this class
  32. */
  33. int ProcessHandler() override;
  34. /**
  35. * When both -R and -I are used should the resulting test list be the
  36. * intersection or the union of the lists. By default it is the
  37. * intersection.
  38. */
  39. void SetUseUnion(bool val) { this->UseUnion = val; }
  40. /**
  41. * Set whether or not CTest should only execute the tests that failed
  42. * on the previous run. By default this is false.
  43. */
  44. void SetRerunFailed(bool val) { this->RerunFailed = val; }
  45. /**
  46. * This method is called when reading CTest custom file
  47. */
  48. void PopulateCustomVectors(cmMakefile* mf) override;
  49. ///! Control the use of the regular expresisons, call these methods to turn
  50. /// them on
  51. void UseIncludeRegExp();
  52. void UseExcludeRegExp();
  53. void SetIncludeRegExp(const char*);
  54. void SetExcludeRegExp(const char*);
  55. void SetMaxIndex(int n) { this->MaxIndex = n; }
  56. int GetMaxIndex() { return this->MaxIndex; }
  57. void SetTestOutputSizePassed(int n)
  58. {
  59. this->CustomMaximumPassedTestOutputSize = n;
  60. }
  61. void SetTestOutputSizeFailed(int n)
  62. {
  63. this->CustomMaximumFailedTestOutputSize = n;
  64. }
  65. ///! pass the -I argument down
  66. void SetTestsToRunInformation(const char*);
  67. cmCTestTestHandler();
  68. /*
  69. * Add the test to the list of tests to be executed
  70. */
  71. bool AddTest(const std::vector<std::string>& args);
  72. /*
  73. * Set tests properties
  74. */
  75. bool SetTestsProperties(const std::vector<std::string>& args);
  76. /**
  77. * Set directory properties
  78. */
  79. bool SetDirectoryProperties(const std::vector<std::string>& args);
  80. void Initialize() override;
  81. // NOTE: This struct is Saved/Restored
  82. // in cmCTestTestHandler, if you add to this class
  83. // then you must add the new members to that code or
  84. // ctest -j N will break for that feature
  85. struct cmCTestTestProperties
  86. {
  87. std::string Name;
  88. std::string Directory;
  89. std::vector<std::string> Args;
  90. std::vector<std::string> RequiredFiles;
  91. std::vector<std::string> Depends;
  92. std::vector<std::string> AttachedFiles;
  93. std::vector<std::string> AttachOnFail;
  94. std::vector<std::pair<cmsys::RegularExpression, std::string>>
  95. ErrorRegularExpressions;
  96. std::vector<std::pair<cmsys::RegularExpression, std::string>>
  97. RequiredRegularExpressions;
  98. std::vector<std::pair<cmsys::RegularExpression, std::string>>
  99. TimeoutRegularExpressions;
  100. std::map<std::string, std::string> Measurements;
  101. bool IsInBasedOnREOptions;
  102. bool WillFail;
  103. bool Disabled;
  104. float Cost;
  105. int PreviousRuns;
  106. bool RunSerial;
  107. cmDuration Timeout;
  108. bool ExplicitTimeout;
  109. cmDuration AlternateTimeout;
  110. int Index;
  111. // Requested number of process slots
  112. int Processors;
  113. // return code of test which will mark test as "not run"
  114. int SkipReturnCode;
  115. std::vector<std::string> Environment;
  116. std::vector<std::string> Labels;
  117. std::set<std::string> LockedResources;
  118. std::set<std::string> FixturesSetup;
  119. std::set<std::string> FixturesCleanup;
  120. std::set<std::string> FixturesRequired;
  121. std::set<std::string> RequireSuccessDepends;
  122. };
  123. struct cmCTestTestResult
  124. {
  125. std::string Name;
  126. std::string Path;
  127. std::string Reason;
  128. std::string FullCommandLine;
  129. cmDuration ExecutionTime;
  130. int ReturnValue;
  131. int Status;
  132. std::string ExceptionStatus;
  133. bool CompressOutput;
  134. std::string CompletionStatus;
  135. std::string Output;
  136. std::string DartString;
  137. int TestCount;
  138. cmCTestTestProperties* Properties;
  139. };
  140. struct cmCTestTestResultLess
  141. {
  142. bool operator()(const cmCTestTestResult& lhs,
  143. const cmCTestTestResult& rhs) const
  144. {
  145. return lhs.TestCount < rhs.TestCount;
  146. }
  147. };
  148. // add configurations to a search path for an executable
  149. static void AddConfigurations(cmCTest* ctest,
  150. std::vector<std::string>& attempted,
  151. std::vector<std::string>& attemptedConfigs,
  152. std::string filepath, std::string& filename);
  153. // full signature static method to find an executable
  154. static std::string FindExecutable(cmCTest* ctest, const char* testCommand,
  155. std::string& resultingConfig,
  156. std::vector<std::string>& extraPaths,
  157. std::vector<std::string>& failed);
  158. typedef std::vector<cmCTestTestProperties> ListOfTests;
  159. protected:
  160. // compute a final test list
  161. virtual int PreProcessHandler();
  162. virtual int PostProcessHandler();
  163. virtual void GenerateTestCommand(std::vector<std::string>& args, int test);
  164. int ExecuteCommands(std::vector<std::string>& vec);
  165. void WriteTestResultHeader(cmXMLWriter& xml,
  166. cmCTestTestResult const& result);
  167. void WriteTestResultFooter(cmXMLWriter& xml,
  168. cmCTestTestResult const& result);
  169. // Write attached test files into the xml
  170. void AttachFiles(cmXMLWriter& xml, cmCTestTestResult& result);
  171. //! Clean test output to specified length
  172. bool CleanTestOutput(std::string& output, size_t length);
  173. cmDuration ElapsedTestingTime;
  174. typedef std::vector<cmCTestTestResult> TestResultsVector;
  175. TestResultsVector TestResults;
  176. std::vector<std::string> CustomTestsIgnore;
  177. std::string StartTest;
  178. std::string EndTest;
  179. std::chrono::system_clock::time_point StartTestTime;
  180. std::chrono::system_clock::time_point EndTestTime;
  181. bool MemCheck;
  182. int CustomMaximumPassedTestOutputSize;
  183. int CustomMaximumFailedTestOutputSize;
  184. int MaxIndex;
  185. public:
  186. enum
  187. { // Program statuses
  188. NOT_RUN = 0,
  189. TIMEOUT,
  190. SEGFAULT,
  191. ILLEGAL,
  192. INTERRUPT,
  193. NUMERICAL,
  194. OTHER_FAULT,
  195. FAILED,
  196. BAD_COMMAND,
  197. COMPLETED
  198. };
  199. private:
  200. /**
  201. * Generate the Dart compatible output
  202. */
  203. virtual void GenerateDartOutput(cmXMLWriter& xml);
  204. void PrintLabelOrSubprojectSummary(bool isSubProject);
  205. /**
  206. * Run the tests for a directory and any subdirectories
  207. */
  208. void ProcessDirectory(std::vector<std::string>& passed,
  209. std::vector<std::string>& failed);
  210. /**
  211. * Get the list of tests in directory and subdirectories.
  212. */
  213. void GetListOfTests();
  214. // compute the lists of tests that will actually run
  215. // based on union regex and -I stuff
  216. void ComputeTestList();
  217. // compute the lists of tests that will actually run
  218. // based on LastTestFailed.log
  219. void ComputeTestListForRerunFailed();
  220. // add required setup/cleanup tests not already in the
  221. // list of tests to be run and update dependencies between
  222. // tests to account for fixture setup/cleanup
  223. void UpdateForFixtures(ListOfTests& tests) const;
  224. void UpdateMaxTestNameWidth();
  225. bool GetValue(const char* tag, std::string& value, std::istream& fin);
  226. bool GetValue(const char* tag, int& value, std::istream& fin);
  227. bool GetValue(const char* tag, size_t& value, std::istream& fin);
  228. bool GetValue(const char* tag, bool& value, std::istream& fin);
  229. bool GetValue(const char* tag, double& value, std::istream& fin);
  230. /**
  231. * Find the executable for a test
  232. */
  233. std::string FindTheExecutable(const char* exe);
  234. const char* GetTestStatus(cmCTestTestResult const&);
  235. void ExpandTestsToRunInformation(size_t numPossibleTests);
  236. void ExpandTestsToRunInformationForRerunFailed();
  237. std::vector<std::string> CustomPreTest;
  238. std::vector<std::string> CustomPostTest;
  239. std::vector<int> TestsToRun;
  240. bool UseIncludeLabelRegExpFlag;
  241. bool UseExcludeLabelRegExpFlag;
  242. bool UseIncludeRegExpFlag;
  243. bool UseExcludeRegExpFlag;
  244. bool UseExcludeRegExpFirst;
  245. std::string IncludeLabelRegExp;
  246. std::string ExcludeLabelRegExp;
  247. std::string IncludeRegExp;
  248. std::string ExcludeRegExp;
  249. std::string ExcludeFixtureRegExp;
  250. std::string ExcludeFixtureSetupRegExp;
  251. std::string ExcludeFixtureCleanupRegExp;
  252. cmsys::RegularExpression IncludeLabelRegularExpression;
  253. cmsys::RegularExpression ExcludeLabelRegularExpression;
  254. cmsys::RegularExpression IncludeTestsRegularExpression;
  255. cmsys::RegularExpression ExcludeTestsRegularExpression;
  256. void GenerateRegressionImages(cmXMLWriter& xml, const std::string& dart);
  257. cmsys::RegularExpression DartStuff1;
  258. void CheckLabelFilter(cmCTestTestProperties& it);
  259. void CheckLabelFilterExclude(cmCTestTestProperties& it);
  260. void CheckLabelFilterInclude(cmCTestTestProperties& it);
  261. std::string TestsToRunString;
  262. bool UseUnion;
  263. ListOfTests TestList;
  264. size_t TotalNumberOfTests;
  265. cmsys::RegularExpression DartStuff;
  266. std::ostream* LogFile;
  267. bool RerunFailed;
  268. };
  269. #endif