A Discrete-Event Network Simulator
API
Loading...
Searching...
No Matches
test.cc
Go to the documentation of this file.
1/*
2 * Copyright (c) 2009 University of Washington
3 *
4 * SPDX-License-Identifier: GPL-2.0-only
5 */
6
7#include "test.h"
8
9#include "abort.h"
10#include "assert.h"
11#include "config.h"
12#include "des-metrics.h"
13#include "log.h"
14#include "rng-seed-manager.h"
15#include "singleton.h"
16#include "system-path.h"
17
18#include <cmath>
19#include <cstring>
20#include <list>
21#include <map>
22#include <vector>
23
24/**
25 * @file
26 * @ingroup testing
27 * @brief ns3::TestCase, ns3::TestSuite, ns3::TestRunner implementations,
28 */
29
30namespace ns3
31{
32
34
35bool
36TestDoubleIsEqual(const double x1, const double x2, const double epsilon)
37{
38 NS_LOG_FUNCTION(x1 << x2 << epsilon);
39 int exponent;
40 double delta;
41 double difference;
42
43 //
44 // Find exponent of largest absolute value
45 //
46 {
47 double max = (std::fabs(x1) > std::fabs(x2)) ? x1 : x2;
48 std::frexp(max, &exponent);
49 }
50
51 //
52 // Form a neighborhood of size 2 * delta
53 //
54 delta = std::ldexp(epsilon, exponent);
55 difference = x1 - x2;
56
57 return difference <= delta && difference >= -delta;
58}
59
60/**
61 * @ingroup testingimpl
62 * Container for details of a test failure.
63 */
65{
66 /**
67 * Constructor.
68 *
69 * @param [in] _cond The name of the condition being tested.
70 * @param [in] _actual The actual value returned by the test.
71 * @param [in] _limit The expected value.
72 * @param [in] _message The associated message.
73 * @param [in] _file The source file.
74 * @param [in] _line The source line.
75 */
76 TestCaseFailure(std::string _cond,
77 std::string _actual,
78 std::string _limit,
79 std::string _message,
80 std::string _file,
81 int32_t _line);
82 std::string cond; /**< The name of the condition being tested. */
83 std::string actual; /**< The actual value returned by the test. */
84 std::string limit; /**< The expected value. */
85 std::string message; /**< The associated message. */
86 std::string file; /**< The source file. */
87 int32_t line; /**< The source line. */
88};
89
90/**
91 * Output streamer for TestCaseFailure.
92 *
93 * @param [in,out] os The output stream.
94 * @param [in] failure The TestCaseFailure to print.
95 * @returns The stream.
96 */
97std::ostream&
98operator<<(std::ostream& os, const TestCaseFailure& failure)
99{
100 os << " test=\"" << failure.cond << "\" actual=\"" << failure.actual << "\" limit=\""
101 << failure.limit << "\" in=\"" << failure.file << ":" << failure.line << "\" "
102 << failure.message;
103
104 return os;
105}
106
107/**
108 * @ingroup testingimpl
109 * Container for results from a TestCase.
110 */
112{
113 /** Constructor. */
114 Result();
115
116 /** Test running time. */
118 /** TestCaseFailure records for each child. */
119 std::vector<TestCaseFailure> failure;
120 /** \c true if any child TestCases failed. */
122};
123
124/**
125 * @ingroup testingimpl
126 * Container for all tests.
127 * @todo Move TestRunnerImpl to separate file.
128 */
129class TestRunnerImpl : public Singleton<TestRunnerImpl>
130{
131 public:
132 /** Constructor. */
134
135 /**
136 * Add a new top-level TestSuite.
137 * @param [in] testSuite The new TestSuite.
138 */
139 void AddTestSuite(TestSuite* testSuite);
140 /** @copydoc TestCase::MustAssertOnFailure() */
141 bool MustAssertOnFailure() const;
142 /** @copydoc TestCase::MustContinueOnFailure() */
143 bool MustContinueOnFailure() const;
144 /**
145 * Check if this run should update the reference data.
146 * @return \c true if we should update the reference data.
147 */
148 bool MustUpdateData() const;
149 /**
150 * Get the path to the root of the source tree.
151 *
152 * The root directory is defined by the presence of two files:
153 * "VERSION" and "LICENSE".
154 *
155 * @returns The path to the root.
156 */
157 std::string GetTopLevelSourceDir() const;
158 /**
159 * Get the path to temporary directory.
160 * @return The temporary directory path.
161 */
162 std::string GetTempDir() const;
163 /** @copydoc TestRunner::Run() */
164 int Run(int argc, char* argv[]);
165
166 private:
167 /**
168 * Check if this is the root of the source tree.
169 * @param [in] path The path to test.
170 * @returns \c true if \pname{path} is the root.
171 */
172 bool IsTopLevelSourceDir(std::string path) const;
173 /**
174 * Clean up characters not allowed in XML.
175 *
176 * XML files have restrictions on certain characters that may be present in
177 * data. We need to replace these characters with their alternate
178 * representation on the way into the XML file.
179 *
180 * Specifically, we make these replacements:
181 * Raw Source | Replacement
182 * :--------: | :---------:
183 * '<' | "&lt;"
184 * '>' | "&gt;"
185 * '&' | "&amp;"
186 * '"' | "&39;"
187 * '\' | "&quot;"
188 *
189 * @param [in] xml The raw string.
190 * @returns The sanitized string.
191 */
192 std::string ReplaceXmlSpecialCharacters(std::string xml) const;
193 /**
194 * Print the test report.
195 *
196 * @param [in] test The TestCase to print.
197 * @param [in,out] os The output stream.
198 * @param [in] xml Generate XML output if \c true.
199 * @param [in] level Indentation level.
200 */
201 void PrintReport(TestCase* test, std::ostream* os, bool xml, int level);
202 /**
203 * Print the list of all requested test suites.
204 *
205 * @param [in] begin Iterator to the first TestCase to print.
206 * @param [in] end Iterator to the end of the list.
207 * @param [in] printTestType Prepend the test type label if \c true.
208 */
209 void PrintTestNameList(std::list<TestCase*>::const_iterator begin,
210 std::list<TestCase*>::const_iterator end,
211 bool printTestType) const;
212 /** Print the list of test types. */
213 void PrintTestTypeList() const;
214 /**
215 * Print the help text.
216 * @param [in] programName The name of the invoking program.
217 */
218 void PrintHelp(const char* programName) const;
219 /**
220 * Generate the list of tests matching the constraints.
221 *
222 * Test name and type constraints are or'ed. The duration constraint
223 * is and'ed.
224 *
225 * @param [in] testName Include a specific test by name.
226 * @param [in] testType Include all tests of give type.
227 * @param [in] filteredTestDuration Restrict tests that match these durations.
228 * @returns The list of tests matching the filter constraints.
229 */
230 std::list<TestCase*> FilterTests(std::string testName,
231 TestSuite::Type testType,
232 std::vector<TestCase::Duration> filteredTestDuration);
233
234 /** Container type for the test. */
235 typedef std::vector<TestSuite*> TestSuiteVector;
236
237 TestSuiteVector m_suites; //!< The list of tests.
238 std::string m_tempDir; //!< The temporary directory.
239 bool m_verbose; //!< Produce verbose output.
240 bool m_assertOnFailure; //!< \c true if we should assert on failure.
241 bool m_continueOnFailure; //!< \c true if we should continue on failure.
242 bool m_updateData; //!< \c true if we should update reference data.
243};
244
246 std::string _actual,
247 std::string _limit,
248 std::string _message,
249 std::string _file,
250 int32_t _line)
251 : cond(_cond),
252 actual(_actual),
253 limit(_limit),
254 message(_message),
255 file(_file),
256 line(_line)
257{
258 NS_LOG_FUNCTION(this << _cond << _actual << _limit << _message << _file << _line);
259}
260
262 : childrenFailed(false)
263{
264 NS_LOG_FUNCTION(this);
265}
266
267TestCase::TestCase(std::string name)
268 : m_parent(nullptr),
269 m_dataDir(""),
270 m_runner(nullptr),
271 m_result(nullptr),
272 m_name(name),
274{
275 NS_LOG_FUNCTION(this << name);
276}
277
279{
280 NS_LOG_FUNCTION(this);
281 NS_ASSERT(m_runner == nullptr);
282 m_parent = nullptr;
283 delete m_result;
284 for (auto i = m_children.begin(); i != m_children.end(); ++i)
285 {
286 delete *i;
287 }
288 m_children.clear();
289}
290
291void
293{
294 NS_LOG_FUNCTION(&testCase << duration);
295
296 // Test names are used to create temporary directories,
297 // so we test for illegal characters.
298 //
299 // Windows: <>:"/\|?*
300 // http://msdn.microsoft.com/en-us/library/aa365247(v=vs.85).aspx
301 // Mac: : (deprecated, was path separator in Mac OS Classic, pre X)
302 // Unix: / (and .. may give trouble?)
303 //
304 // The Windows list is too restrictive: we like to label
305 // tests with "val = v1 * v2" or "v1 < 3" or "case: foo --> bar"
306 // So we allow ':<>*"
307
308 std::string badchars = "\"/\\|?";
309 // Badchar Class Regex Count of failing test names
310 // All ":<>\"/\\|?*" 611
311 // Allow ':' "<>\"/\\|?*" 128
312 // Allow ':<>' "\"/\\|?*" 12
313 // Allow ':<>*' "\"/\\|?" 0
314
315 std::string::size_type badch = testCase->m_name.find_first_of(badchars);
316 if (badch != std::string::npos)
317 {
318 /*
319 To count the bad test names, use NS_LOG_UNCOND instead
320 of NS_FATAL_ERROR, and the command
321 $ ./ns3 run "test-runner --list" 2>&1 | grep "^Invalid" | wc
322 */
323 NS_LOG_UNCOND("Invalid test name: cannot contain any of '" << badchars
324 << "': " << testCase->m_name);
325 }
326
327 testCase->m_duration = duration;
328 testCase->m_parent = this;
329 m_children.push_back(testCase);
330}
331
332bool
334{
335 NS_LOG_FUNCTION(this);
336 return m_result->childrenFailed || !m_result->failure.empty();
337}
338
339void
341{
342 NS_LOG_FUNCTION(this << runner);
343 m_result = new Result();
344 m_runner = runner;
346 DoSetup();
347 m_result->clock.Start();
348 for (auto i = m_children.begin(); i != m_children.end(); ++i)
349 {
351 TestCase* test = *i;
352 test->Run(runner);
353 if (IsFailed())
354 {
355 goto out;
356 }
357 }
358 DoRun();
359out:
360 m_result->clock.End();
361 DoTeardown();
363 m_runner = nullptr;
364}
365
366std::string
368{
369 NS_LOG_FUNCTION(this);
370 return m_name;
371}
372
375{
376 return m_parent;
377}
378
379void
381 std::string actual,
382 std::string limit,
383 std::string message,
384 std::string file,
385 int32_t line)
386{
387 NS_LOG_FUNCTION(this << cond << actual << limit << message << file << line);
388 m_result->failure.emplace_back(cond, actual, limit, message, file, line);
389 // set childrenFailed flag on parents.
390 TestCase* current = m_parent;
391 while (current != nullptr)
392 {
393 current->m_result->childrenFailed = true;
394 current = current->m_parent;
395 }
396}
397
398bool
400{
401 NS_LOG_FUNCTION(this);
402 return m_runner->MustAssertOnFailure();
403}
404
405bool
407{
408 NS_LOG_FUNCTION(this);
409 return m_runner->MustContinueOnFailure();
410}
411
412std::string
414{
415 NS_LOG_FUNCTION(this << filename);
416 const TestCase* current = this;
417 while (current != nullptr && current->m_dataDir.empty())
418 {
419 current = current->m_parent;
420 }
421 if (current == nullptr)
422 {
423 NS_FATAL_ERROR("No one called SetDataDir prior to calling this function");
424 }
425
426 std::string a = SystemPath::Append(m_runner->GetTopLevelSourceDir(), current->m_dataDir);
427 std::string b = SystemPath::Append(a, filename);
428 return b;
429}
430
431std::string
433{
434 NS_LOG_FUNCTION(this << filename);
435 if (m_runner->MustUpdateData())
436 {
437 return CreateDataDirFilename(filename);
438 }
439 else
440 {
441 std::list<std::string> names;
442 const TestCase* current = this;
443 while (current != nullptr)
444 {
445 names.push_front(current->m_name);
446 current = current->m_parent;
447 }
448 std::string tempDir = SystemPath::Append(m_runner->GetTempDir(),
449 SystemPath::Join(names.begin(), names.end()));
450 tempDir = SystemPath::CreateValidSystemPath(tempDir);
451
453 return SystemPath::Append(tempDir, filename);
454 }
455}
456
457bool
459{
460 NS_LOG_FUNCTION(this);
461 return !IsStatusSuccess();
462}
463
464bool
466{
467 NS_LOG_FUNCTION(this);
468 return m_result->failure.empty();
469}
470
471void
472TestCase::SetDataDir(std::string directory)
473{
474 NS_LOG_FUNCTION(this << directory);
475 m_dataDir = directory;
476}
477
478void
480{
481 NS_LOG_FUNCTION(this);
482}
483
484void
489
491 : TestCase(name),
492 m_type(type)
493{
494 NS_LOG_FUNCTION(this << name << type);
496}
497
500{
501 NS_LOG_FUNCTION(this);
502 return m_type;
503}
504
505void
507{
508 NS_LOG_FUNCTION(this);
509}
510
512 : m_tempDir(""),
513 m_assertOnFailure(false),
515 m_updateData(false)
516{
517 NS_LOG_FUNCTION(this);
518}
519
520void
522{
523 NS_LOG_FUNCTION(this << testSuite);
524 m_suites.push_back(testSuite);
525}
526
527bool
533
534bool
540
541bool
543{
544 NS_LOG_FUNCTION(this);
545 return m_updateData;
546}
547
548std::string
550{
551 NS_LOG_FUNCTION(this);
552 return m_tempDir;
553}
554
555bool
557{
558 NS_LOG_FUNCTION(this << path);
559 bool haveVersion = false;
560 bool haveLicense = false;
561
562 //
563 // If there's a file named VERSION and a file named LICENSE in this
564 // directory, we assume it's our top level source directory.
565 //
566
567 std::list<std::string> files = SystemPath::ReadFiles(path);
568 for (auto i = files.begin(); i != files.end(); ++i)
569 {
570 if (*i == "VERSION")
571 {
572 haveVersion = true;
573 }
574 else if (*i == "LICENSE")
575 {
576 haveLicense = true;
577 }
578 }
579
580 return haveVersion && haveLicense;
581}
582
583std::string
585{
586 NS_LOG_FUNCTION(this);
587 std::string self = SystemPath::FindSelfDirectory();
588 std::list<std::string> elements = SystemPath::Split(self);
589 while (!elements.empty())
590 {
591 std::string path = SystemPath::Join(elements.begin(), elements.end());
592 if (IsTopLevelSourceDir(path))
593 {
594 return path;
595 }
596 elements.pop_back();
597 }
598 NS_FATAL_ERROR("Could not find source directory from self=" << self);
599 return self;
600}
601
602//
603// XML files have restrictions on certain characters that may be present in
604// data. We need to replace these characters with their alternate
605// representation on the way into the XML file.
606//
607std::string
609{
610 NS_LOG_FUNCTION(this << xml);
611 typedef std::map<char, std::string> specials_map;
612 specials_map specials;
613 specials['<'] = "&lt;";
614 specials['>'] = "&gt;";
615 specials['&'] = "&amp;";
616 specials['"'] = "&#39;";
617 specials['\''] = "&quot;";
618
619 std::string result;
620 std::size_t length = xml.length();
621
622 for (size_t i = 0; i < length; ++i)
623 {
624 char character = xml[i];
625
626 auto it = specials.find(character);
627
628 if (it == specials.end())
629 {
630 result.push_back(character);
631 }
632 else
633 {
634 result += it->second;
635 }
636 }
637 return result;
638}
639
640/** Helper to indent output a specified number of steps. */
641struct Indent
642{
643 /**
644 * Constructor.
645 * @param [in] level The number of steps. A step is " ".
646 */
647 Indent(int level);
648 /** The number of steps. */
649 int level;
650};
651
652Indent::Indent(int _level)
653 : level(_level)
654{
655 NS_LOG_FUNCTION(this << _level);
656}
657
658/**
659 * Output streamer for Indent.
660 * @param [in,out] os The output stream.
661 * @param [in] val The Indent object.
662 * @returns The stream.
663 */
664std::ostream&
665operator<<(std::ostream& os, const Indent& val)
666{
667 for (int i = 0; i < val.level; i++)
668 {
669 os << " ";
670 }
671 return os;
672}
673
674void
675TestRunnerImpl::PrintReport(TestCase* test, std::ostream* os, bool xml, int level)
676{
677 NS_LOG_FUNCTION(this << test << os << xml << level);
678 if (test->m_result == nullptr)
679 {
680 // Do not print reports for tests that were not run.
681 return;
682 }
683 // Report times in seconds, from ms timer
684 const double MS_PER_SEC = 1000.;
685 double real = test->m_result->clock.GetElapsedReal() / MS_PER_SEC;
686 double user = test->m_result->clock.GetElapsedUser() / MS_PER_SEC;
687 double system = test->m_result->clock.GetElapsedSystem() / MS_PER_SEC;
688
689 std::streamsize oldPrecision = (*os).precision(3);
690 *os << std::fixed;
691
692 std::string statusString = test->IsFailed() ? "FAIL" : "PASS";
693 if (xml)
694 {
695 *os << Indent(level) << "<Test>" << std::endl;
696 *os << Indent(level + 1) << "<Name>" << ReplaceXmlSpecialCharacters(test->m_name)
697 << "</Name>" << std::endl;
698 *os << Indent(level + 1) << "<Result>" << statusString << "</Result>" << std::endl;
699 *os << Indent(level + 1) << "<Time real=\"" << real << "\" user=\"" << user
700 << "\" system=\"" << system << "\"/>" << std::endl;
701 for (uint32_t i = 0; i < test->m_result->failure.size(); i++)
702 {
703 TestCaseFailure failure = test->m_result->failure[i];
704 *os << Indent(level + 2) << "<FailureDetails>" << std::endl
705 << Indent(level + 3) << "<Condition>" << ReplaceXmlSpecialCharacters(failure.cond)
706 << "</Condition>" << std::endl
707 << Indent(level + 3) << "<Actual>" << ReplaceXmlSpecialCharacters(failure.actual)
708 << "</Actual>" << std::endl
709 << Indent(level + 3) << "<Limit>" << ReplaceXmlSpecialCharacters(failure.limit)
710 << "</Limit>" << std::endl
711 << Indent(level + 3) << "<Message>" << ReplaceXmlSpecialCharacters(failure.message)
712 << "</Message>" << std::endl
713 << Indent(level + 3) << "<File>" << ReplaceXmlSpecialCharacters(failure.file)
714 << "</File>" << std::endl
715 << Indent(level + 3) << "<Line>" << failure.line << "</Line>" << std::endl
716 << Indent(level + 2) << "</FailureDetails>" << std::endl;
717 }
718 for (uint32_t i = 0; i < test->m_children.size(); i++)
719 {
720 TestCase* child = test->m_children[i];
721 PrintReport(child, os, xml, level + 1);
722 }
723 *os << Indent(level) << "</Test>" << std::endl;
724 }
725 else
726 {
727 *os << Indent(level) << statusString << " " << test->GetName() << " " << real << " s"
728 << std::endl;
729 if (m_verbose)
730 {
731 for (uint32_t i = 0; i < test->m_result->failure.size(); i++)
732 {
733 *os << Indent(level) << test->m_result->failure[i] << std::endl;
734 }
735 for (uint32_t i = 0; i < test->m_children.size(); i++)
736 {
737 TestCase* child = test->m_children[i];
738 PrintReport(child, os, xml, level + 1);
739 }
740 }
741 }
742
743 (*os).unsetf(std::ios_base::floatfield);
744 (*os).precision(oldPrecision);
745}
746
747void
748TestRunnerImpl::PrintHelp(const char* program_name) const
749{
750 NS_LOG_FUNCTION(this << program_name);
751 std::cout
752 << "Usage: " << program_name << " [OPTIONS]" << std::endl
753 << std::endl
754 << "Options: " << std::endl
755 << " --help : print these options" << std::endl
756 << " --print-test-name-list : print the list of names of tests available" << std::endl
757 << " --list : an alias for --print-test-name-list" << std::endl
758 << " --print-test-types : print the type of tests along with their names"
759 << std::endl
760 << " --print-test-type-list : print the list of types of tests available" << std::endl
761 << " --print-temp-dir : print name of temporary directory before running "
762 << std::endl
763 << " the tests" << std::endl
764 << " --test-type=TYPE : process only tests of type TYPE" << std::endl
765 << " --test-name=NAME : process only test whose name matches NAME" << std::endl
766 << " --suite=NAME : an alias (here for compatibility reasons only) "
767 << std::endl
768 << " for --test-name=NAME" << std::endl
769 << " --assert-on-failure : when a test fails, crash immediately (useful" << std::endl
770 << " when running under a debugger" << std::endl
771 << " --stop-on-failure : when a test fails, stop immediately" << std::endl
772 << " --fullness=FULLNESS : choose the duration of tests to run: QUICK, " << std::endl
773 << " EXTENSIVE, or TAKES_FOREVER, where EXTENSIVE " << std::endl
774 << " includes QUICK and TAKES_FOREVER includes " << std::endl
775 << " QUICK and EXTENSIVE (only QUICK tests are " << std::endl
776 << " run by default)" << std::endl
777 << " --only-fullness=FULLNESS : choose the duration of tests to run: QUICK, " << std::endl
778 << " EXTENSIVE, TAKES_FOREVER (only tests marked " << std::endl
779 << " with fullness will be executed)" << std::endl
780 << " --verbose : print details of test execution" << std::endl
781 << " --xml : format test run output as xml" << std::endl
782 << " --tempdir=DIR : set temp dir for tests to store output files" << std::endl
783 << " --datadir=DIR : set data dir for tests to read reference files"
784 << std::endl
785 << " --out=FILE : send test result to FILE instead of standard output"
786 << std::endl
787 << " --append=FILE : append test result to FILE instead of standard output"
788 << std::endl;
789}
790
791void
792TestRunnerImpl::PrintTestNameList(std::list<TestCase*>::const_iterator begin,
793 std::list<TestCase*>::const_iterator end,
794 bool printTestType) const
795{
796 NS_LOG_FUNCTION(this << &begin << &end << printTestType);
797 std::map<TestSuite::Type, std::string> label;
798
799 label[TestSuite::Type::ALL] = "all ";
800 label[TestSuite::Type::UNIT] = "unit ";
801 label[TestSuite::Type::SYSTEM] = "system ";
802 label[TestSuite::Type::EXAMPLE] = "example-as-test ";
803 label[TestSuite::Type::PERFORMANCE] = "performance ";
804
805 for (auto i = begin; i != end; ++i)
806 {
807 auto test = dynamic_cast<TestSuite*>(*i);
808 NS_ASSERT(test != nullptr);
809 if (printTestType)
810 {
811 std::cout << label[test->GetTestType()];
812 }
813 std::cout << test->GetName() << std::endl;
814 }
815}
816
817void
819{
820 NS_LOG_FUNCTION(this);
821 std::cout << " core: Run all TestSuite-based tests (exclude examples)" << std::endl
822 << " example: Examples (to see if example programs run successfully)"
823 << std::endl
824 << " example-as-test: Examples (which are tested against reference outputs)"
825 << std::endl
826 << " performance: Performance Tests (check to see if the system is as fast as "
827 "expected)"
828 << std::endl
829 << " system: System Tests (spans modules to check integration of modules)"
830 << std::endl
831 << " unit: Unit Tests (within modules to check basic functionality)"
832 << std::endl;
833}
834
835std::list<TestCase*>
836TestRunnerImpl::FilterTests(std::string testName,
837 TestSuite::Type testType,
838 std::vector<TestCase::Duration> filteredTestDuration)
839{
840 NS_LOG_FUNCTION(this << testName << testType);
841 std::list<TestCase*> tests;
842 for (uint32_t i = 0; i < m_suites.size(); ++i)
843 {
844 TestSuite* test = m_suites[i];
845 if (testType != TestSuite::Type::ALL && test->GetTestType() != testType)
846 {
847 // skip test
848 continue;
849 }
850 if (!testName.empty() && test->GetName() != testName)
851 {
852 // skip test
853 continue;
854 }
855
856 // Remove any test cases that should be skipped.
857 for (auto j = test->m_children.begin(); j != test->m_children.end();)
858 {
859 TestCase* testCase = *j;
860
861 // If this test case takes longer than the maximum test
862 // duration that should be run, then don't run it.
863 auto it = std::find(filteredTestDuration.begin(),
864 filteredTestDuration.end(),
865 testCase->m_duration);
866 if (it == filteredTestDuration.end())
867 {
868 // Free this test case's memory.
869 delete *j;
870
871 // Remove this test case from the test suite.
872 j = test->m_children.erase(j);
873 }
874 else
875 {
876 // Only advance through the vector elements if this test
877 // case wasn't deleted.
878 ++j;
879 }
880 }
881
882 // Add this test suite.
883 tests.push_back(test);
884 }
885 return tests;
886}
887
888int
889TestRunnerImpl::Run(int argc, char* argv[])
890{
891 NS_LOG_FUNCTION(this << argc << argv);
892 std::string testName = "";
893 std::string testTypeString = "";
894 std::string out = "";
895 std::string fullness = "";
896 std::string onlyFullness = "";
897 bool xml = false;
898 bool append = false;
899 bool printTempDir = false;
900 bool printTestTypeList = false;
901 bool printTestNameList = false;
902 bool printTestTypeAndName = false;
903 std::vector<TestCase::Duration> filteredTestDuration{TestCase::Duration::QUICK};
904 char* progname = argv[0];
905
906 char** argi = argv;
907 ++argi;
908
909 while (*argi != nullptr)
910 {
911 std::string arg = *argi;
912
913 if (arg == "--assert-on-failure")
914 {
915 m_assertOnFailure = true;
916 }
917 else if (arg == "--stop-on-failure")
918 {
919 m_continueOnFailure = false;
920 }
921 else if (arg == "--verbose")
922 {
923 m_verbose = true;
924 }
925 else if (arg == "--print-temp-dir")
926 {
927 printTempDir = true;
928 }
929 else if (arg == "--update-data")
930 {
931 m_updateData = true;
932 }
933 else if (arg == "--print-test-name-list" || arg == "--list")
934 {
935 printTestNameList = true;
936 }
937 else if (arg == "--print-test-types")
938 {
939 printTestTypeAndName = true;
940 }
941 else if (arg == "--print-test-type-list")
942 {
943 printTestTypeList = true;
944 }
945 else if (arg == "--append")
946 {
947 append = true;
948 }
949 else if (arg == "--xml")
950 {
951 xml = true;
952 }
953 else if (arg.find("--test-type=") != std::string::npos)
954 {
955 testTypeString = arg.substr(arg.find_first_of('=') + 1);
956 }
957 else if (arg.find("--test-name=") != std::string::npos ||
958 arg.find("--suite=") != std::string::npos)
959 {
960 testName = arg.substr(arg.find_first_of('=') + 1);
961 }
962 else if (arg.find("--tempdir=") != std::string::npos)
963 {
964 m_tempDir = arg.substr(arg.find_first_of('=') + 1);
965 }
966 else if (arg.find("--out=") != std::string::npos)
967 {
968 out = arg.substr(arg.find_first_of('=') + 1);
969 }
970 else if (arg.find("--fullness=") != std::string::npos)
971 {
972 fullness = arg.substr(arg.find_first_of('=') + 1);
973
974 // Set the maximum test length allowed.
975 if (fullness == "QUICK")
976 {
977 filteredTestDuration = {TestCase::Duration::QUICK};
978 }
979 else if (fullness == "EXTENSIVE")
980 {
982 }
983 else if (fullness == "TAKES_FOREVER")
984 {
985 filteredTestDuration = {TestCase::Duration::QUICK,
988 }
989 else
990 {
991 // Wrong fullness option
992 PrintHelp(progname);
993 return 3;
994 }
995 }
996 else if (arg.find("--only-fullness=") != std::string::npos)
997 {
998 onlyFullness = arg.substr(arg.find_first_of('=') + 1);
999
1000 // Set the maximum test length allowed.
1001 if (onlyFullness == "QUICK")
1002 {
1003 filteredTestDuration = {TestCase::Duration::QUICK};
1004 }
1005 else if (onlyFullness == "EXTENSIVE")
1006 {
1007 filteredTestDuration = {TestCase::Duration::EXTENSIVE};
1008 }
1009 else if (onlyFullness == "TAKES_FOREVER")
1010 {
1011 filteredTestDuration = {TestCase::Duration::TAKES_FOREVER};
1012 }
1013 else
1014 {
1015 // Wrong fullness option
1016 PrintHelp(progname);
1017 return 4;
1018 }
1019 }
1020 else
1021 {
1022 // Print the help if arg == "--help" or arg is an un-recognized command-line argument
1023 PrintHelp(progname);
1024 return 0;
1025 }
1026 argi++;
1027 }
1028 TestSuite::Type testType;
1029 if (testTypeString.empty() || testTypeString == "core")
1030 {
1031 testType = TestSuite::Type::ALL;
1032 }
1033 else if (testTypeString == "example")
1034 {
1035 testType = TestSuite::Type::EXAMPLE;
1036 }
1037 else if (testTypeString == "unit")
1038 {
1039 testType = TestSuite::Type::UNIT;
1040 }
1041 else if (testTypeString == "system")
1042 {
1043 testType = TestSuite::Type::SYSTEM;
1044 }
1045 else if (testTypeString == "performance")
1046 {
1048 }
1049 else
1050 {
1051 std::cout << "Invalid test type specified: " << testTypeString << std::endl;
1053 return 1;
1054 }
1055
1056 if (!fullness.empty() && !onlyFullness.empty())
1057 {
1058 std::cout << "Invalid simultaneous use of '--fullness' and '--only-fullness'. "
1059 << "Use only one of them." << std::endl;
1060 PrintHelp(progname);
1061 return 5;
1062 }
1063
1064 std::list<TestCase*> tests = FilterTests(testName, testType, filteredTestDuration);
1065
1066 if (m_tempDir.empty())
1067 {
1069 }
1070 if (printTempDir)
1071 {
1072 std::cout << m_tempDir << std::endl;
1073 }
1074 if (printTestNameList)
1075 {
1076 PrintTestNameList(tests.begin(), tests.end(), printTestTypeAndName);
1077 return 0;
1078 }
1079 if (printTestTypeList)
1080 {
1082 return 0;
1083 }
1084
1085 std::ostream* os;
1086 if (!out.empty())
1087 {
1088 std::ofstream* ofs;
1089 ofs = new std::ofstream();
1090 std::ios_base::openmode mode = std::ios_base::out;
1091 if (append)
1092 {
1093 mode |= std::ios_base::app;
1094 }
1095 else
1096 {
1097 mode |= std::ios_base::trunc;
1098 }
1099 ofs->open(out, mode);
1100 os = ofs;
1101 }
1102 else
1103 {
1104 os = &std::cout;
1105 }
1106
1107 // let's run our tests now.
1108 bool failed = false;
1109 if (tests.empty())
1110 {
1111 std::cerr << "Error: no tests match the requested string" << std::endl;
1112 return 1;
1113 }
1114 else if (tests.size() > 1)
1115 {
1116 std::cerr << "Error: tests should be launched separately (one at a time)" << std::endl;
1117 return 1;
1118 }
1119
1120 for (auto i = tests.begin(); i != tests.end(); ++i)
1121 {
1122 TestCase* test = *i;
1123
1124#ifdef ENABLE_DES_METRICS
1125 {
1126 /*
1127 Reorganize argv
1128 Since DES Metrics uses argv[0] for the trace file name,
1129 grab the test name and put it in argv[0],
1130 with test-runner as argv[1]
1131 then the rest of the original arguments.
1132 */
1133 std::string testname = test->GetName();
1134 std::string runner = "[" + SystemPath::Split(argv[0]).back() + "]";
1135
1136 std::vector<std::string> desargs;
1137 desargs.push_back(testname);
1138 desargs.push_back(runner);
1139 for (int i = 1; i < argc; ++i)
1140 {
1141 desargs.push_back(argv[i]);
1142 }
1143
1145 }
1146#endif
1147
1148 test->Run(this);
1149 PrintReport(test, os, xml, 0);
1150 if (test->IsFailed())
1151 {
1152 failed = true;
1154 {
1155 return 1;
1156 }
1157 }
1158 }
1159
1160 if (!out.empty())
1161 {
1162 delete os;
1163 }
1164
1165 return failed ? 1 : 0;
1166}
1167
1168int
1169TestRunner::Run(int argc, char* argv[])
1170{
1171 NS_LOG_FUNCTION(argc << argv);
1172 return TestRunnerImpl::Get()->Run(argc, argv);
1173}
1174
1175std::ostream&
1176operator<<(std::ostream& os, TestSuite::Type type)
1177{
1178 switch (type)
1179 {
1181 return os << "ALL";
1183 return os << "UNIT";
1185 return os << "SYSTEM";
1187 return os << "EXAMPLE";
1189 return os << "PERFORMANCE";
1190 };
1191 return os << "UNKNOWN(" << static_cast<uint32_t>(type) << ")";
1192}
1193
1194std::ostream&
1195operator<<(std::ostream& os, TestCase::Duration duration)
1196{
1197 switch (duration)
1198 {
1200 return os << "QUICK";
1202 return os << "EXTENSIVE";
1204 return os << "TAKES_FOREVER";
1205 };
1206 return os << "UNKNOWN(" << static_cast<uint32_t>(duration) << ")";
1207}
1208
1209} // namespace ns3
NS_ABORT_x macro definitions.
NS_ASSERT() and NS_ASSERT_MSG() macro definitions.
void Initialize(std::vector< std::string > args, std::string outDir="")
Open the DesMetrics trace file and print the header.
static void ResetNextStreamIndex()
Resets the global stream index counter.
Singleton(const Singleton< TestRunnerImpl > &)=delete
static TestRunnerImpl * Get()
Definition singleton.h:96
Measure elapsed wall clock time in milliseconds.
encapsulates test code
Definition test.h:1050
std::string m_name
TestCase name.
Definition test.h:1255
bool MustContinueOnFailure() const
Check if this run should continue on failure.
Definition test.cc:406
bool IsStatusFailure() const
Check if any tests failed.
Definition test.cc:458
std::string m_dataDir
My data directory.
Definition test.h:1252
static constexpr auto QUICK
Deprecated test duration simple enums.
Definition test.h:1067
std::string CreateDataDirFilename(std::string filename)
Construct the full path to a file in the data directory.
Definition test.cc:413
TestCase * m_parent
Pointer to my parent TestCase.
Definition test.h:1250
void AddTestCase(TestCase *testCase, Duration duration=Duration::QUICK)
Add an individual child TestCase to this test suite.
Definition test.cc:292
Result * m_result
Results data.
Definition test.h:1254
bool IsStatusSuccess() const
Check if all tests passed.
Definition test.cc:465
virtual void DoSetup()
Implementation to do any local setup required for this TestCase.
Definition test.cc:479
virtual ~TestCase()
Destructor.
Definition test.cc:278
Duration
How long the test takes to execute.
Definition test.h:1054
@ EXTENSIVE
Medium length test.
Definition test.h:1056
@ QUICK
Fast test.
Definition test.h:1055
@ TAKES_FOREVER
Very long running test.
Definition test.h:1057
std::string CreateTempDirFilename(std::string filename)
Construct the full path to a file in a temporary directory.
Definition test.cc:432
TestRunnerImpl * m_runner
Pointer to the TestRunner.
Definition test.h:1253
TestCase * GetParent() const
Get the parent of this TestCase.
Definition test.cc:374
bool MustAssertOnFailure() const
Check if this run should assert on failure.
Definition test.cc:399
void SetDataDir(std::string directory)
Set the data directory where reference trace files can be found.
Definition test.cc:472
virtual void DoTeardown()
Implementation to do any local setup required for this TestCase.
Definition test.cc:485
void Run(TestRunnerImpl *runner)
Executes DoSetup(), DoRun(), and DoTeardown() for the TestCase.
Definition test.cc:340
TestCase(const TestCase &)=delete
virtual void DoRun()=0
Implementation to actually run this TestCase.
std::string GetName() const
Definition test.cc:367
friend class TestRunnerImpl
Needs access to the TestCase data members.
Definition test.h:1202
Duration m_duration
TestCase duration.
Definition test.h:1256
void ReportTestFailure(std::string cond, std::string actual, std::string limit, std::string message, std::string file, int32_t line)
Log the failure of this TestCase.
Definition test.cc:380
bool IsFailed() const
Check if any tests failed.
Definition test.cc:333
std::vector< TestCase * > m_children
Vector of my children.
Definition test.h:1251
static int Run(int argc, char *argv[])
Run the requested suite of tests, according to the given command line arguments.
Definition test.cc:1169
void PrintHelp(const char *programName) const
Print the help text.
Definition test.cc:748
bool m_assertOnFailure
true if we should assert on failure.
Definition test.cc:240
std::string ReplaceXmlSpecialCharacters(std::string xml) const
Clean up characters not allowed in XML.
Definition test.cc:608
bool MustUpdateData() const
Check if this run should update the reference data.
Definition test.cc:542
bool IsTopLevelSourceDir(std::string path) const
Check if this is the root of the source tree.
Definition test.cc:556
bool m_continueOnFailure
true if we should continue on failure.
Definition test.cc:241
bool m_updateData
true if we should update reference data.
Definition test.cc:242
std::list< TestCase * > FilterTests(std::string testName, TestSuite::Type testType, std::vector< TestCase::Duration > filteredTestDuration)
Generate the list of tests matching the constraints.
Definition test.cc:836
std::string m_tempDir
The temporary directory.
Definition test.cc:238
std::string GetTempDir() const
Get the path to temporary directory.
Definition test.cc:549
void PrintReport(TestCase *test, std::ostream *os, bool xml, int level)
Print the test report.
Definition test.cc:675
std::vector< TestSuite * > TestSuiteVector
Container type for the test.
Definition test.cc:235
bool MustContinueOnFailure() const
Check if this run should continue on failure.
Definition test.cc:535
int Run(int argc, char *argv[])
Run the requested suite of tests, according to the given command line arguments.
Definition test.cc:889
bool MustAssertOnFailure() const
Check if this run should assert on failure.
Definition test.cc:528
TestRunnerImpl()
Constructor.
Definition test.cc:511
void AddTestSuite(TestSuite *testSuite)
Add a new top-level TestSuite.
Definition test.cc:521
bool m_verbose
Produce verbose output.
Definition test.cc:239
TestSuiteVector m_suites
The list of tests.
Definition test.cc:237
std::string GetTopLevelSourceDir() const
Get the path to the root of the source tree.
Definition test.cc:584
void PrintTestTypeList() const
Print the list of test types.
Definition test.cc:818
void PrintTestNameList(std::list< TestCase * >::const_iterator begin, std::list< TestCase * >::const_iterator end, bool printTestType) const
Print the list of all requested test suites.
Definition test.cc:792
A suite of tests to run.
Definition test.h:1267
Type
Type of test.
Definition test.h:1274
@ PERFORMANCE
This test suite implements a Performance Test.
Definition test.h:1279
@ ALL
Token to represent all tests.
Definition test.h:1275
@ EXAMPLE
This test suite implements an Example Test.
Definition test.h:1278
@ UNIT
This test suite implements a Unit Test.
Definition test.h:1276
@ SYSTEM
This test suite implements a System Test.
Definition test.h:1277
TestSuite(std::string name, Type type=Type::UNIT)
Construct a new test suite.
Definition test.cc:490
TestSuite::Type m_type
Type of this TestSuite.
Definition test.h:1319
void DoRun() override
Implementation to actually run this TestCase.
Definition test.cc:506
TestSuite::Type GetTestType()
get the kind of test this test suite implements
Definition test.cc:499
Declaration of the various ns3::Config functions and classes.
ns3::DesMetrics declaration.
#define NS_ASSERT(condition)
At runtime, in debugging builds, if this condition is not true, the program prints the source file,...
Definition assert.h:55
void Reset()
Reset the initial value of every attribute as well as the value of every global to what they were bef...
Definition config.cc:851
#define NS_FATAL_ERROR(msg)
Report a fatal error with a message and terminate.
#define NS_LOG_UNCOND(msg)
Output the requested message unconditionally.
#define NS_LOG_COMPONENT_DEFINE(name)
Define a Log component with a specific name.
Definition log.h:191
#define NS_LOG_FUNCTION(parameters)
If log level LOG_FUNCTION is enabled, this macro will output all input parameters separated by ",...
std::list< std::string > ReadFiles(std::string path)
Get the list of files located in a file system directory.
std::list< std::string > Split(std::string path)
Split a file system path into directories according to the local path separator.
void MakeDirectories(std::string path)
Create all the directories leading to path.
std::string MakeTemporaryDirectoryName()
Get the name of a temporary directory.
std::string Append(std::string left, std::string right)
Join two file system path elements.
std::string Join(std::list< std::string >::const_iterator begin, std::list< std::string >::const_iterator end)
Join a list of file system path directories into a single file system path.
std::string CreateValidSystemPath(const std::string path)
Replace incompatible characters in a path, to get a path compatible with different file systems.
std::string FindSelfDirectory()
Get the file system path to the current executable.
bool TestDoubleIsEqual(const double x1, const double x2, const double epsilon)
Compare two double precision floating point numbers and declare them equal if they are within some ep...
Definition test.cc:36
Debug message logging.
Namespace for test files, TestCases and TestSuites.
Every class exported by the ns3 library is enclosed in the ns3 namespace.
std::ostream & operator<<(std::ostream &os, const Angles &a)
Definition angles.cc:148
-ns3 Test suite for the ns3 wrapper script
ns3::RngSeedManager declaration.
ns3::Singleton declaration and template implementation.
Helper to indent output a specified number of steps.
Definition test.cc:642
Indent(int level)
Constructor.
Definition test.cc:652
int level
The number of steps.
Definition test.cc:649
Container for results from a TestCase.
Definition test.cc:112
Result()
Constructor.
Definition test.cc:261
std::vector< TestCaseFailure > failure
TestCaseFailure records for each child.
Definition test.cc:119
bool childrenFailed
true if any child TestCases failed.
Definition test.cc:121
SystemWallClockMs clock
Test running time.
Definition test.cc:117
Container for details of a test failure.
Definition test.cc:65
std::string actual
The actual value returned by the test.
Definition test.cc:83
std::string file
The source file.
Definition test.cc:86
std::string message
The associated message.
Definition test.cc:85
int32_t line
The source line.
Definition test.cc:87
TestCaseFailure(std::string _cond, std::string _actual, std::string _limit, std::string _message, std::string _file, int32_t _line)
Constructor.
Definition test.cc:245
std::string cond
The name of the condition being tested.
Definition test.cc:82
std::string limit
The expected value.
Definition test.cc:84
ns3::SystemPath declarations.
ns3::TestCase, ns3::TestSuite, ns3::TestRunner declarations, and NS_TEST_ASSERT macro definitions.