A Discrete-Event Network Simulator
API
test.cc
Go to the documentation of this file.
1 /* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */
2 /*
3  * Copyright (c) 2009 University of Washington
4  *
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License version 2 as
7  * published by the Free Software Foundation;
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software
16  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17  */
18 
19 #include "test.h"
20 #include "assert.h"
21 #include "abort.h"
22 #include "singleton.h"
23 #include "system-path.h"
24 #include "log.h"
25 #include "des-metrics.h"
26 #include <cmath>
27 #include <cstring>
28 #include <vector>
29 #include <list>
30 #include <map>
31 
32 
39 namespace ns3 {
40 
42 
43 bool
44 TestDoubleIsEqual (const double x1, const double x2, const double epsilon)
45 {
46  NS_LOG_FUNCTION (x1 << x2 << epsilon);
47  int exponent;
48  double delta, difference;
49 
50  //
51  // Find exponent of largest absolute value
52  //
53  {
54  double max = (std::fabs (x1) > std::fabs (x2)) ? x1 : x2;
55  (void)std::frexp (max, &exponent);
56  }
57 
58  //
59  // Form a neighborhood of size 2 * delta
60  //
61  delta = std::ldexp (epsilon, exponent);
62  difference = x1 - x2;
63 
64  if (difference > delta || difference < -delta)
65  {
66  return false;
67  }
68  return true;
69 }
70 
76 {
87  TestCaseFailure (std::string _cond, std::string _actual,
88  std::string _limit, std::string _message,
89  std::string _file, int32_t _line);
90  std::string cond;
91  std::string actual;
92  std::string limit;
93  std::string message;
94  std::string file;
95  int32_t line;
96 };
104 std::ostream & operator << (std::ostream & os, const TestCaseFailure & failure)
105 {
106  os << " test=\"" << failure.cond
107  << "\" actual=\"" << failure.actual
108  << "\" limit=\"" << failure.limit
109  << "\" in=\"" << failure.file
110  << ":" << failure.line
111  << "\" " << failure.message;
112 
113  return os;
114 }
115 
121 {
123  Result ();
124 
128  std::vector<TestCaseFailure> failure;
131 };
132 
137 class TestRunnerImpl : public Singleton<TestRunnerImpl>
138 {
139 public:
141  TestRunnerImpl ();
142 
147  void AddTestSuite (TestSuite *testSuite);
149  bool MustAssertOnFailure (void) const;
151  bool MustContinueOnFailure (void) const;
156  bool MustUpdateData (void) const;
165  std::string GetTopLevelSourceDir (void) const;
170  std::string GetTempDir (void) const;
172  int Run (int argc, char *argv[]);
173 
174 private:
175 
181  bool IsTopLevelSourceDir (std::string path) const;
201  std::string ReplaceXmlSpecialCharacters (std::string xml) const;
210  void PrintReport (TestCase *test, std::ostream *os, bool xml, int level);
218  void PrintTestNameList (std::list<TestCase *>::const_iterator begin,
219  std::list<TestCase *>::const_iterator end,
220  bool printTestType) const;
222  void PrintTestTypeList (void) const;
227  void PrintHelp (const char *programName) const;
239  std::list<TestCase *> FilterTests (std::string testName,
240  enum TestSuite::Type testType,
241  enum TestCase::TestDuration maximumTestDuration);
242 
243 
245  typedef std::vector<TestSuite *> TestSuiteVector;
246 
247  TestSuiteVector m_suites;
248  std::string m_tempDir;
249  bool m_verbose;
253 };
254 
255 
256 
257 TestCaseFailure::TestCaseFailure (std::string _cond, std::string _actual,
258  std::string _limit, std::string _message,
259  std::string _file, int32_t _line)
260  : cond (_cond), actual (_actual), limit (_limit),
261  message (_message), file (_file), line (_line)
262 {
263  NS_LOG_FUNCTION (this << _cond << _actual << _limit << _message << _file << _line);
264 }
266  : childrenFailed (false)
267 {
268  NS_LOG_FUNCTION (this);
269 }
270 
271 
272 
273 TestCase::TestCase (std::string name)
274  : m_parent (0),
275  m_dataDir (""),
276  m_runner (0),
277  m_result (0),
278  m_name (name),
280 {
281  NS_LOG_FUNCTION (this << name);
282 }
283 
285 {
286  NS_LOG_FUNCTION (this);
287  NS_ASSERT (m_runner == 0);
288  m_parent = 0;
289  delete m_result;
290  for (std::vector<TestCase *>::const_iterator i = m_children.begin (); i != m_children.end (); ++i)
291  {
292  delete *i;
293  }
294  m_children.clear ();
295 }
296 
297 void
299 {
300  NS_LOG_FUNCTION (&testCase << duration);
301 
302  // Test names are used to create temporary directories,
303  // so we test for illegal characters.
304  //
305  // Windows: <>:"/\|?*
306  // http://msdn.microsoft.com/en-us/library/aa365247(v=vs.85).aspx
307  // Mac: : (deprecated, was path separator in Mac OS Classic, pre X)
308  // Unix: / (and .. may give trouble?)
309  //
310  // The Windows list is too restrictive: we like to label
311  // tests with "val = v1 * v2" or "v1 < 3" or "case: foo --> bar"
312  // So we allow ':<>*"
313 
314  std::string badchars = "\"/\\|?";
315  // Badchar Class Regex Count of failing test names
316  // All ":<>\"/\\|?*" 611
317  // Allow ':' "<>\"/\\|?*" 128
318  // Allow ':<>' "\"/\\|?*" 12
319  // Allow ':<>*' "\"/\\|?" 0
320 
321  std::string::size_type badch = testCase->m_name.find_first_of (badchars);
322  if (badch != std::string::npos)
323  {
324  /*
325  To count the bad test names, use NS_LOG_UNCOND instead
326  of NS_FATAL_ERROR, and the command
327  $ ./waf --run "test-runner --list" 2>&1 | grep "^Invalid" | wc
328  */
329  NS_LOG_UNCOND ("Invalid test name: cannot contain any of '"
330  << badchars << "': " << testCase->m_name);
331  }
332 
333  testCase->m_duration = duration;
334  testCase->m_parent = this;
335  m_children.push_back (testCase);
336 }
337 
338 bool
339 TestCase::IsFailed (void) const
340 {
341  NS_LOG_FUNCTION (this);
342  return m_result->childrenFailed || !m_result->failure.empty ();
343 }
344 
345 void
347 {
348  NS_LOG_FUNCTION (this << runner);
349  m_result = new Result ();
350  m_runner = runner;
351  DoSetup ();
352  m_result->clock.Start ();
353  for (std::vector<TestCase *>::const_iterator i = m_children.begin (); i != m_children.end (); ++i)
354  {
355  TestCase *test = *i;
356  test->Run (runner);
357  if (IsFailed ())
358  {
359  goto out;
360  }
361  }
362  DoRun ();
363  out:
364  m_result->clock.End ();
365  DoTeardown ();
366  m_runner = 0;
367 }
368 std::string
369 TestCase::GetName (void) const
370 {
371  NS_LOG_FUNCTION (this);
372  return m_name;
373 }
374 TestCase *
376 {
377  return m_parent;
378 }
379 
380 void
381 TestCase::ReportTestFailure (std::string cond, std::string actual,
382  std::string limit, std::string message,
383  std::string file, int32_t line)
384 {
385  NS_LOG_FUNCTION (this << cond << actual << limit << message << file << line);
386  m_result->failure.push_back (TestCaseFailure (cond, actual, limit,
387  message, file, line));
388  // set childrenFailed flag on parents.
389  TestCase *current = m_parent;
390  while (current != 0)
391  {
392  current->m_result->childrenFailed = true;
393  current = current->m_parent;
394  }
395 
396 }
397 bool
399 {
400  NS_LOG_FUNCTION (this);
401  return m_runner->MustAssertOnFailure ();
402 }
403 bool
405 {
406  NS_LOG_FUNCTION (this);
407  return m_runner->MustContinueOnFailure ();
408 }
409 
410 std::string
411 TestCase::CreateDataDirFilename (std::string filename)
412 {
413  NS_LOG_FUNCTION (this << filename);
414  const TestCase *current = this;
415  while (current != 0 && current->m_dataDir == "")
416  {
417  current = current->m_parent;
418  }
419  if (current == 0)
420  {
421  NS_FATAL_ERROR ("No one called SetDataDir prior to calling this function");
422  }
423 
424  std::string a = SystemPath::Append (m_runner->GetTopLevelSourceDir (), current->m_dataDir);
425  std::string b = SystemPath::Append (a, filename);
426  return b;
427 }
428 std::string
429 TestCase::CreateTempDirFilename (std::string filename)
430 {
431  NS_LOG_FUNCTION (this << filename);
432  if (m_runner->MustUpdateData ())
433  {
434  return CreateDataDirFilename (filename);
435  }
436  else
437  {
438  std::list<std::string> names;
439  const TestCase *current = this;
440  while (current != 0)
441  {
442  names.push_front (current->m_name);
443  current = current->m_parent;
444  }
445  std::string tempDir = SystemPath::Append (m_runner->GetTempDir (), SystemPath::Join (names.begin (), names.end ()));
446  SystemPath::MakeDirectories (tempDir);
447  return SystemPath::Append (tempDir, filename);
448  }
449 }
450 bool
452 {
453  NS_LOG_FUNCTION (this);
454  return !IsStatusSuccess ();
455 }
456 bool
458 {
459  NS_LOG_FUNCTION (this);
460  return m_result->failure.empty ();
461 }
462 
463 void
464 TestCase::SetDataDir (std::string directory)
465 {
466  NS_LOG_FUNCTION (this << directory);
467  m_dataDir = directory;
468 }
469 
470 void
472 {
473  NS_LOG_FUNCTION (this);
474 }
475 void
477 {
478  NS_LOG_FUNCTION (this);
479 }
480 
481 
482 TestSuite::TestSuite (std::string name, TestSuite::Type type)
483  : TestCase (name),
484  m_type (type)
485 {
486  NS_LOG_FUNCTION (this << name << type);
488 }
489 
492 {
493  NS_LOG_FUNCTION (this);
494  return m_type;
495 }
496 
497 void
499 {
500  NS_LOG_FUNCTION (this);
501 }
502 
504  : m_tempDir (""),
505  m_assertOnFailure (false),
506  m_continueOnFailure (true),
507  m_updateData (false)
508 {
509  NS_LOG_FUNCTION (this);
510 }
511 
512 void
514 {
515  NS_LOG_FUNCTION (this << testSuite);
516  m_suites.push_back (testSuite);
517 }
518 
519 
520 bool
522 {
523  NS_LOG_FUNCTION (this);
524  return m_assertOnFailure;
525 }
526 bool
528 {
529  NS_LOG_FUNCTION (this);
530  return m_continueOnFailure;
531 }
532 
533 bool
535 {
536  NS_LOG_FUNCTION (this);
537  return m_updateData;
538 }
539 std::string
541 {
542  NS_LOG_FUNCTION (this);
543  return m_tempDir;
544 }
545 bool
546 TestRunnerImpl::IsTopLevelSourceDir (std::string path) const
547 {
548  NS_LOG_FUNCTION (this << path);
549  bool haveVersion = false;
550  bool haveLicense = false;
551 
552  //
553  // If there's a file named VERSION and a file named LICENSE in this
554  // directory, we assume it's our top level source directory.
555  //
556 
557  std::list<std::string> files = SystemPath::ReadFiles (path);
558  for (std::list<std::string>::const_iterator i = files.begin (); i != files.end (); ++i)
559  {
560  if (*i == "VERSION")
561  {
562  haveVersion = true;
563  }
564  else if (*i == "LICENSE")
565  {
566  haveLicense = true;
567  }
568  }
569 
570  return haveVersion && haveLicense;
571 }
572 
573 std::string
575 {
576  NS_LOG_FUNCTION (this);
577  std::string self = SystemPath::FindSelfDirectory ();
578  std::list<std::string> elements = SystemPath::Split (self);
579  while (!elements.empty ())
580  {
581  std::string path = SystemPath::Join (elements.begin (), elements.end ());
582  if (IsTopLevelSourceDir (path))
583  {
584  return path;
585  }
586  elements.pop_back ();
587  }
588  NS_FATAL_ERROR ("Could not find source directory from self=" << self);
589 }
590 
591 //
592 // XML files have restrictions on certain characters that may be present in
593 // data. We need to replace these characters with their alternate
594 // representation on the way into the XML file.
595 //
596 std::string
598 {
599  NS_LOG_FUNCTION (this << xml);
600  typedef std::map <char, std::string> specials_map;
601  specials_map specials;
602  specials['<'] = "&lt;";
603  specials['>'] = "&gt;";
604  specials['&'] = "&amp;";
605  specials['"'] = "&#39;";
606  specials['\''] = "&quot;";
607 
608  std::string result;
609  std::size_t length = xml.length ();
610 
611  for (size_t i = 0; i < length; ++i)
612  {
613  char character = xml[i];
614 
615  specials_map::const_iterator it = specials.find (character);
616 
617  if (it == specials.end ())
618  {
619  result.push_back (character);
620  }
621  else
622  {
623  result += it->second;
624  }
625  }
626  return result;
627 }
628 
630 struct Indent
631 {
636  Indent (int level);
638  int level;
639 };
640 Indent::Indent (int _level)
641  : level (_level)
642 {
643  NS_LOG_FUNCTION (this << _level);
644 }
651 std::ostream &operator << (std::ostream &os, const Indent &val)
652 {
653  for (int i = 0; i < val.level; i++)
654  {
655  os << " ";
656  }
657  return os;
658 }
659 
660 void
661 TestRunnerImpl::PrintReport (TestCase *test, std::ostream *os, bool xml, int level)
662 {
663  NS_LOG_FUNCTION (this << test << os << xml << level);
664  if (test->m_result == 0)
665  {
666  // Do not print reports for tests that were not run.
667  return;
668  }
669  // Report times in seconds, from ms timer
670  const double MS_PER_SEC = 1000.;
671  double real = test->m_result->clock.GetElapsedReal () / MS_PER_SEC;
672  double user = test->m_result->clock.GetElapsedUser () / MS_PER_SEC;
673  double system = test->m_result->clock.GetElapsedSystem () / MS_PER_SEC;
674 
675  std::streamsize oldPrecision = (*os).precision (3);
676  *os << std::fixed;
677 
678  std::string statusString = test->IsFailed ()?"FAIL":"PASS";
679  if (xml)
680  {
681  *os << Indent (level) << "<Test>" << std::endl;
682  *os << Indent (level+1) << "<Name>" << ReplaceXmlSpecialCharacters (test->m_name)
683  << "</Name>" << std::endl;
684  *os << Indent (level+1) << "<Result>" << statusString << "</Result>" << std::endl;
685  *os << Indent (level+1) << "<Time real=\"" << real << "\" user=\"" << user
686  << "\" system=\"" << system << "\"/>" << std::endl;
687  for (uint32_t i = 0; i < test->m_result->failure.size (); i++)
688  {
689  TestCaseFailure failure = test->m_result->failure[i];
690  *os << Indent (level+2) << "<FailureDetails>" << std::endl
691  << Indent (level+3) << "<Condition>"
692  << ReplaceXmlSpecialCharacters (failure.cond) << "</Condition>" << std::endl
693  << Indent (level+3) << "<Actual>"
694  << ReplaceXmlSpecialCharacters (failure.actual) << "</Actual>" << std::endl
695  << Indent (level+3) << "<Limit>"
696  << ReplaceXmlSpecialCharacters (failure.limit) << "</Limit>" << std::endl
697  << Indent (level+3) << "<Message>"
698  << ReplaceXmlSpecialCharacters (failure.message) << "</Message>" << std::endl
699  << Indent (level+3) << "<File>"
700  << ReplaceXmlSpecialCharacters (failure.file) << "</File>" << std::endl
701  << Indent (level+3) << "<Line>" << failure.line << "</Line>" << std::endl
702  << Indent (level+2) << "</FailureDetails>" << std::endl;
703  }
704  for (uint32_t i = 0; i < test->m_children.size (); i++)
705  {
706  TestCase *child = test->m_children[i];
707  PrintReport (child, os, xml, level + 1);
708  }
709  *os << Indent (level) << "</Test>" << std::endl;
710  }
711  else
712  {
713  *os << Indent (level) << statusString << " " << test->GetName ()
714  << " " << real << " s" << std::endl;
715  if (m_verbose)
716  {
717  for (uint32_t i = 0; i < test->m_result->failure.size (); i++)
718  {
719  *os << Indent (level) << test->m_result->failure[i] << std::endl;
720  }
721  for (uint32_t i = 0; i < test->m_children.size (); i++)
722  {
723  TestCase *child = test->m_children[i];
724  PrintReport (child, os, xml, level + 1);
725  }
726  }
727  }
728 
729  (*os).unsetf(std::ios_base::floatfield);
730  (*os).precision (oldPrecision);
731 }
732 
733 void
734 TestRunnerImpl::PrintHelp (const char *program_name) const
735 {
736  NS_LOG_FUNCTION (this << program_name);
737  std::cout << "Usage: " << program_name << " [OPTIONS]" << std::endl
738  << std::endl
739  << "Options: " << std::endl
740  << " --help : print these options" << std::endl
741  << " --print-test-name-list : print the list of names of tests available" << std::endl
742  << " --list : an alias for --print-test-name-list" << std::endl
743  << " --print-test-types : print the type of tests along with their names" << std::endl
744  << " --print-test-type-list : print the list of types of tests available" << std::endl
745  << " --print-temp-dir : print name of temporary directory before running " << std::endl
746  << " the tests" << std::endl
747  << " --test-type=TYPE : process only tests of type TYPE" << std::endl
748  << " --test-name=NAME : process only test whose name matches NAME" << std::endl
749  << " --suite=NAME : an alias (here for compatibility reasons only) " << std::endl
750  << " for --test-name=NAME" << std::endl
751  << " --assert-on-failure : when a test fails, crash immediately (useful" << std::endl
752  << " when running under a debugger" << std::endl
753  << " --stop-on-failure : when a test fails, stop immediately" << std::endl
754  << " --fullness=FULLNESS : choose the duration of tests to run: QUICK, " << std::endl
755  << " EXTENSIVE, or TAKES_FOREVER, where EXTENSIVE " << std::endl
756  << " includes QUICK and TAKES_FOREVER includes " << std::endl
757  << " QUICK and EXTENSIVE (only QUICK tests are " << std::endl
758  << " run by default)" << std::endl
759  << " --verbose : print details of test execution" << std::endl
760  << " --xml : format test run output as xml" << std::endl
761  << " --tempdir=DIR : set temp dir for tests to store output files" << std::endl
762  << " --datadir=DIR : set data dir for tests to read reference files" << std::endl
763  << " --out=FILE : send test result to FILE instead of standard "
764  << "output" << std::endl
765  << " --append=FILE : append test result to FILE instead of standard "
766  << "output" << std::endl
767  ;
768 }
769 
770 void
771 TestRunnerImpl::PrintTestNameList (std::list<TestCase *>::const_iterator begin,
772  std::list<TestCase *>::const_iterator end,
773  bool printTestType) const
774 {
775  NS_LOG_FUNCTION (this << &begin << &end << printTestType);
776  std::map<TestSuite::Type, std::string> label;
777 
778  label[TestSuite::ALL] = "all ";
779  label[TestSuite::BVT] = "bvt ";
780  label[TestSuite::UNIT] = "unit ";
781  label[TestSuite::SYSTEM] = "system ";
782  label[TestSuite::EXAMPLE] = "example ";
783  label[TestSuite::PERFORMANCE] = "performance ";
784 
785  for (std::list<TestCase *>::const_iterator i = begin; i != end; ++i)
786  {
787  TestSuite * test= dynamic_cast<TestSuite *>(*i);
788  NS_ASSERT (test != 0);
789  if (printTestType)
790  {
791  std::cout << label[test->GetTestType ()];
792  }
793  std::cout << test->GetName () << std::endl;
794  }
795 }
796 
797 void
799 {
800  NS_LOG_FUNCTION (this);
801  std::cout << " bvt: Build Verification Tests (to see if build completed successfully)" << std::endl;
802  std::cout << " core: Run all TestSuite-based tests (exclude examples)" << std::endl;
803  std::cout << " example: Examples (to see if example programs run successfully)" << std::endl;
804  std::cout << " performance: Performance Tests (check to see if the system is as fast as expected)" << std::endl;
805  std::cout << " system: System Tests (spans modules to check integration of modules)" << std::endl;
806  std::cout << " unit: Unit Tests (within modules to check basic functionality)" << std::endl;
807 }
808 
809 
810 std::list<TestCase *>
811 TestRunnerImpl::FilterTests (std::string testName,
812  enum TestSuite::Type testType,
813  enum TestCase::TestDuration maximumTestDuration)
814 {
815  NS_LOG_FUNCTION (this << testName << testType);
816  std::list<TestCase *> tests;
817  for (uint32_t i = 0; i < m_suites.size (); ++i)
818  {
819  TestSuite *test = m_suites[i];
820  if (testType != TestSuite::ALL && test->GetTestType () != testType)
821  {
822  // skip test
823  continue;
824  }
825  if (testName != "" && test->GetName () != testName)
826  {
827  // skip test
828  continue;
829  }
830 
831  // Remove any test cases that should be skipped.
832  std::vector<TestCase *>::iterator j;
833  for (j = test->m_children.begin (); j != test->m_children.end ();)
834  {
835  TestCase *testCase = *j;
836 
837  // If this test case takes longer than the maximum test
838  // duration that should be run, then don't run it.
839  if (testCase->m_duration > maximumTestDuration)
840  {
841  // Free this test case's memory.
842  delete *j;
843 
844  // Remove this test case from the test suite.
845  j = test->m_children.erase (j);
846  }
847  else
848  {
849  // Only advance through the vector elements if this test
850  // case wasn't deleted.
851  ++j;
852  }
853  }
854 
855  // Add this test suite.
856  tests.push_back (test);
857  }
858  return tests;
859 }
860 
861 
862 int
863 TestRunnerImpl::Run (int argc, char *argv[])
864 {
865  NS_LOG_FUNCTION (this << argc << argv);
866  std::string testName = "";
867  std::string testTypeString = "";
868  std::string out = "";
869  std::string fullness = "";
870  bool xml = false;
871  bool append = false;
872  bool printTempDir = false;
873  bool printTestTypeList = false;
874  bool printTestNameList = false;
875  bool printTestTypeAndName = false;
876  enum TestCase::TestDuration maximumTestDuration = TestCase::QUICK;
877  char *progname = argv[0];
878 
879  char ** argi = argv;
880  ++argi;
881 
882  while (*argi != 0)
883  {
884  char *arg = *argi;
885 
886  if (strcmp(arg, "--assert-on-failure") == 0)
887  {
888  m_assertOnFailure = true;
889  }
890  else if (strcmp (arg, "--stop-on-failure") == 0)
891  {
892  m_continueOnFailure = false;
893  }
894  else if (strcmp (arg, "--verbose") == 0)
895  {
896  m_verbose = true;
897  }
898  else if (strcmp (arg, "--print-temp-dir") == 0)
899  {
900  printTempDir = true;
901  }
902  else if (strcmp (arg, "--update-data") == 0)
903  {
904  m_updateData = true;
905  }
906  else if (strcmp (arg, "--help") == 0)
907  {
908  PrintHelp (progname);
909  return 0;
910  }
911  else if (strcmp (arg, "--print-test-name-list") == 0 ||
912  strcmp(arg, "--list") == 0)
913  {
914  printTestNameList = true;
915  }
916  else if (strcmp (arg, "--print-test-types") == 0)
917  {
918  printTestTypeAndName = true;
919  }
920  else if (strcmp (arg, "--print-test-type-list") == 0)
921  {
922  printTestTypeList = true;
923  }
924  else if (strcmp(arg, "--append") == 0)
925  {
926  append = true;
927  }
928  else if (strcmp(arg, "--xml") == 0)
929  {
930  xml = true;
931  }
932  else if (strncmp(arg, "--test-type=", strlen("--test-type=")) == 0)
933  {
934  testTypeString = arg + strlen("--test-type=");
935  }
936  else if (strncmp(arg, "--test-name=", strlen("--test-name=")) == 0)
937  {
938  testName = arg + strlen("--test-name=");
939  }
940  else if (strncmp(arg, "--suite=", strlen("--suite=")) == 0)
941  {
942  testName = arg + strlen("--suite=");
943  }
944  else if (strncmp(arg, "--tempdir=", strlen("--tempdir=")) == 0)
945  {
946  m_tempDir = arg + strlen("--tempdir=");
947  }
948  else if (strncmp(arg, "--out=", strlen("--out=")) == 0)
949  {
950  out = arg + strlen("--out=");
951  }
952  else if (strncmp(arg, "--fullness=", strlen("--fullness=")) == 0)
953  {
954  fullness = arg + strlen("--fullness=");
955 
956  // Set the maximum test length allowed.
957  if (fullness == "EXTENSIVE")
958  {
959  maximumTestDuration = TestCase::EXTENSIVE;
960  }
961  else if (fullness == "TAKES_FOREVER")
962  {
963  maximumTestDuration = TestCase::TAKES_FOREVER;
964  }
965  else
966  {
967  maximumTestDuration = TestCase::QUICK;
968  }
969  }
970  else
971  {
972  // un-recognized command-line argument
973  PrintHelp (progname);
974  return 0;
975  }
976  argi++;
977  }
978  enum TestSuite::Type testType;
979  if (testTypeString == "")
980  {
981  testType = TestSuite::ALL;
982  }
983  else if (testTypeString == "bvt")
984  {
985  testType = TestSuite::BVT;
986  }
987  else if (testTypeString == "core")
988  {
989  testType = TestSuite::ALL;
990  }
991  else if (testTypeString == "example")
992  {
993  testType = TestSuite::EXAMPLE;
994  }
995  else if (testTypeString == "unit")
996  {
997  testType = TestSuite::UNIT;
998  }
999  else if (testTypeString == "system")
1000  {
1001  testType = TestSuite::SYSTEM;
1002  }
1003  else if (testTypeString == "performance")
1004  {
1005  testType = TestSuite::PERFORMANCE;
1006  }
1007  else
1008  {
1009  std::cout << "Invalid test type specified: " << testTypeString << std::endl;
1010  PrintTestTypeList ();
1011  return 1;
1012  }
1013 
1014  std::list<TestCase *> tests = FilterTests (testName, testType, maximumTestDuration);
1015 
1016  if (m_tempDir == "")
1017  {
1019  }
1020  if (printTempDir)
1021  {
1022  std::cout << m_tempDir << std::endl;
1023  }
1024  if (printTestNameList)
1025  {
1026  PrintTestNameList (tests.begin (), tests.end (), printTestTypeAndName);
1027  return 0;
1028  }
1029  if (printTestTypeList)
1030  {
1031  PrintTestTypeList ();
1032  return 0;
1033  }
1034 
1035 
1036  std::ostream *os;
1037  if (out != "")
1038  {
1039  std::ofstream *ofs;
1040  ofs = new std::ofstream();
1041  std::ios_base::openmode mode = std::ios_base::out;
1042  if (append)
1043  {
1044  mode |= std::ios_base::app;
1045  }
1046  else
1047  {
1048  mode |= std::ios_base::trunc;
1049  }
1050  ofs->open (out.c_str (), mode);
1051  os = ofs;
1052  }
1053  else
1054  {
1055  os = &std::cout;
1056  }
1057 
1058  // let's run our tests now.
1059  bool failed = false;
1060  if (tests.size () == 0)
1061  {
1062  std::cerr << "Error: no tests match the requested string" << std::endl;
1063  return 1;
1064  }
1065  for (std::list<TestCase *>::const_iterator i = tests.begin (); i != tests.end (); ++i)
1066  {
1067  TestCase *test = *i;
1068 
1069 #ifdef ENABLE_DES_METRICS
1070  {
1071  /*
1072  Reorganize argv
1073  Since DES Metrics uses argv[0] for the trace file name,
1074  grab the test name and put it in argv[0],
1075  with test-runner as argv[1]
1076  then the rest of the original arguments.
1077  */
1078  std::string testname = test->GetName ();
1079  std::string runner = "[" + SystemPath::Split (argv[0]).back () + "]";
1080 
1081  int desargc = argc + 1;
1082  char ** desargv = new char * [desargc];
1083  desargv[0] = const_cast<char *>(testname.c_str ());
1084  desargv[1] = const_cast<char *>(runner.c_str ());
1085  for (int i = 2; i < desargc; ++i)
1086  {
1087  desargv[i] = argv[i - 1];
1088  }
1089  DesMetrics::Get ()->Initialize (desargc, desargv, m_tempDir);
1090  delete [] desargv;
1091  }
1092 #endif
1093 
1094  test->Run (this);
1095  PrintReport (test, os, xml, 0);
1096  if (test->IsFailed ())
1097  {
1098  failed = true;
1099  if (!m_continueOnFailure)
1100  {
1101  return 1;
1102  }
1103  }
1104  }
1105 
1106  if (out != "")
1107  {
1108  delete os;
1109  }
1110 
1111  return failed?1:0;
1112 }
1113 
1114 int
1115 TestRunner::Run (int argc, char *argv[])
1116 {
1117  NS_LOG_FUNCTION (argc << argv);
1118  return TestRunnerImpl::Get ()->Run (argc, argv);
1119 }
1120 
1121 } // namespace ns3
std::string message
The associated message.
Definition: test.cc:93
virtual void DoSetup(void)
Implementation to do any local setup required for this TestCase.
Definition: test.cc:471
bool MustAssertOnFailure(void) const
Check if this run should assert on failure.
Definition: test.cc:521
void PrintHelp(const char *programName) const
Print the help text.
Definition: test.cc:734
Helper to indent output a specified number of steps.
Definition: test.cc:630
virtual void DoTeardown(void)
Implementation to do any local setup required for this TestCase.
Definition: test.cc:476
void Initialize(int argc, char *argv[], std::string outDir="")
Open the DesMetrics trace file and print the header.
Definition: des-metrics.cc:42
#define NS_LOG_FUNCTION(parameters)
If log level LOG_FUNCTION is enabled, this macro will output all input parameters separated by "...
ns3::Singleton declaration and template implementation.
Container for all tests.
Definition: test.cc:137
TestCaseFailure(std::string _cond, std::string _actual, std::string _limit, std::string _message, std::string _file, int32_t _line)
Constructor.
Definition: test.cc:257
SystemWallClockMs clock
Test running time.
Definition: test.cc:126
Fast test.
Definition: test.h:1152
A suite of tests to run.
Definition: test.h:1333
Container for results from a TestCase.
Definition: test.cc:120
void AddTestSuite(TestSuite *testSuite)
Add a new top-level TestSuite.
Definition: test.cc:513
#define NS_ASSERT(condition)
At runtime, in debugging builds, if this condition is not true, the program prints the source file...
Definition: assert.h:67
Medium length test.
Definition: test.h:1153
#define NS_LOG_COMPONENT_DEFINE(name)
Define a Log component with a specific name.
Definition: log.h:201
bool IsFailed(void) const
Check if any tests failed.
Definition: test.cc:339
bool m_continueOnFailure
true if we should continue on failure.
Definition: test.cc:251
#define NS_FATAL_ERROR(msg)
Report a fatal error with a message and terminate.
Definition: fatal-error.h:162
Definition of the testing macros and declaration of the testing classes.
encapsulates test code
Definition: test.h:1147
enum TestDuration m_duration
TestCase duration.
Definition: test.h:1325
This test suite implements a Unit Test.
Definition: test.h:1343
static TestRunnerImpl * Get(void)
Get a pointer to the singleton instance.
TestSuiteVector m_suites
The list of tests.
Definition: test.cc:247
void PrintReport(TestCase *test, std::ostream *os, bool xml, int level)
Print the test report.
Definition: test.cc:661
ns3::DesMetrics declaration.
void PrintTestNameList(std::list< TestCase * >::const_iterator begin, std::list< TestCase * >::const_iterator end, bool printTestType) const
Print the list of all requested test suites.
Definition: test.cc:771
TestSuite(std::string name, Type type=UNIT)
Constuct a new test suite.
Definition: test.cc:482
std::list< std::string > Split(std::string path)
Split a file system path into directories according to the local path separator.
Definition: system-path.cc:204
TestDuration
How long the test takes to execute.
Definition: test.h:1151
This test suite implements an Example Test.
Definition: test.h:1345
virtual void DoRun(void)
Implementation to actually run this TestCase.
Definition: test.cc:498
std::list< TestCase * > FilterTests(std::string testName, enum TestSuite::Type testType, enum TestCase::TestDuration maximumTestDuration)
Generate the list of tests matching the constraints.
Definition: test.cc:811
System-independent file and directory function declarations.
std::string GetTempDir(void) const
Get the path to temporary directory.
Definition: test.cc:540
A template singleton.
Definition: singleton.h:63
TestRunnerImpl()
Constructor.
Definition: test.cc:503
Measure elapsed wall clock time in milliseconds.
std::string Join(std::list< std::string >::const_iterator begin, std::list< std::string >::const_iterator end)
Join a list of file system path directories into a single file system path.
Definition: system-path.cc:222
TestCase * m_parent
Pointer to my parent TestCase.
Definition: test.h:1317
TestRunnerImpl * m_runner
Pointer to the TestRunner.
Definition: test.h:1322
#define max(a, b)
Definition: 80211b.c:45
void AddTestCase(TestCase *testCase, enum TestDuration duration)
Add an individual child TestCase to this test suite.
Definition: test.cc:298
std::string cond
The name of the condition being tested.
Definition: test.cc:90
Definition of assertion macros NS_ASSERT() and NS_ASSERT_MSG().
std::string MakeTemporaryDirectoryName(void)
Get the name of a temporary directory.
Definition: system-path.cc:280
std::list< std::string > ReadFiles(std::string path)
Get the list of files located in a file system directory.
Definition: system-path.cc:241
std::string GetTopLevelSourceDir(void) const
Get the path to the root of the source tree.
Definition: test.cc:574
bool childrenFailed
true if any child TestCases failed.
Definition: test.cc:130
bool TestDoubleIsEqual(const double x1, const double x2, const double epsilon)
Compare two double precision floating point numbers and declare them equal if they are within some ep...
Definition: test.cc:44
std::string m_tempDir
The temporary directory.
Definition: test.cc:248
void Start(void)
Start a measure.
Type
Type of test.
Definition: test.h:1340
bool MustUpdateData(void) const
Check if this run should update the reference data.
Definition: test.cc:534
TestSuite::Type m_type
Type of this TestSuite.
Definition: test.h:1368
bool MustAssertOnFailure(void) const
Check if this run should assert on failure.
Definition: test.cc:398
TestCase(std::string name)
Constructor.
Definition: test.cc:273
std::ostream & operator<<(std::ostream &os, const Angles &a)
print a struct Angles to output
Definition: angles.cc:42
bool MustContinueOnFailure(void) const
Check if this run should continue on failure.
Definition: test.cc:527
virtual ~TestCase()
Destructor.
Definition: test.cc:284
bool m_updateData
true if we should update reference data.
Definition: test.cc:252
int32_t line
The source line.
Definition: test.cc:95
Every class exported by the ns3 library is enclosed in the ns3 namespace.
std::string CreateDataDirFilename(std::string filename)
Construct the full path to a file in the data directory.
Definition: test.cc:411
This test suite implements a Performance Test.
Definition: test.h:1346
virtual void DoRun(void)=0
Implementation to actually run this TestCase.
struct Result * m_result
Results data.
Definition: test.h:1323
bool IsStatusFailure(void) const
Check if any tests failed.
Definition: test.cc:451
TestCase * GetParent() const
Get the parent of this TestCsse.
Definition: test.cc:375
void ReportTestFailure(std::string cond, std::string actual, std::string limit, std::string message, std::string file, int32_t line)
Log the failure of this TestCase.
Definition: test.cc:381
#define NS_LOG_UNCOND(msg)
Output the requested message unconditionaly.
bool m_verbose
Produce verbose output.
Definition: test.cc:249
void Run(TestRunnerImpl *runner)
Actually run this TestCase.
Definition: test.cc:346
bool IsStatusSuccess(void) const
Check if all tests passed.
Definition: test.cc:457
void MakeDirectories(std::string path)
Create all the directories leading to path.
Definition: system-path.cc:327
int64_t GetElapsedUser(void) const
Indent(int level)
Constructor.
Definition: test.cc:640
This test suite implements a System Test.
Definition: test.h:1344
std::string actual
The actual value returned by the test.
Definition: test.cc:91
std::string FindSelfDirectory(void)
Get the file system path to the current executable.
Definition: system-path.cc:97
std::string file
The soure file.
Definition: test.cc:94
std::string CreateTempDirFilename(std::string filename)
Construct the full path to a file in a temporary directory.
Definition: test.cc:429
std::string Append(std::string left, std::string right)
Join two file system path elements.
Definition: system-path.cc:187
This test suite implements a Build Verification Test.
Definition: test.h:1342
int64_t GetElapsedReal(void) const
std::string ReplaceXmlSpecialCharacters(std::string xml) const
Clean up characters not allowed in XML.
Definition: test.cc:597
Result()
Constructor.
Definition: test.cc:265
void SetDataDir(std::string directory)
Set the data directory where reference trace files can be found.
Definition: test.cc:464
std::vector< TestSuite * > TestSuiteVector
Container type for the test.
Definition: test.cc:245
std::vector< TestCase * > m_children
Vector of my children.
Definition: test.h:1320
int64_t GetElapsedSystem(void) const
Container for details of a test failure.
Definition: test.cc:75
std::string GetName(void) const
Definition: test.cc:369
bool IsTopLevelSourceDir(std::string path) const
Check if this is the root of the source tree.
Definition: test.cc:546
int level
The number of steps.
Definition: test.cc:638
Very long running test.
Definition: test.h:1154
TestSuite::Type GetTestType(void)
get the kind of test this test suite implements
Definition: test.cc:491
void PrintTestTypeList(void) const
Print the list of test types.
Definition: test.cc:798
std::vector< TestCaseFailure > failure
TestCaseFailure records for each child.
Definition: test.cc:128
Debug message logging.
int64_t End(void)
Stop measuring the time since Start() was called.
static int Run(int argc, char *argv[])
Run the requested suite of tests, according to the given command line arguments.
Definition: test.cc:1115
int Run(int argc, char *argv[])
Run the requested suite of tests, according to the given command line arguments.
Definition: test.cc:863
std::string m_name
TestCase name.
Definition: test.h:1324
void test(void)
Example use of ns3::SystemThread.
std::string m_dataDir
My data directory.
Definition: test.h:1321
bool m_assertOnFailure
true if we should assert on failure.
Definition: test.cc:250
bool MustContinueOnFailure(void) const
Check if this run should continue on failure.
Definition: test.cc:404
NS_ABORT_x macro definitions.
std::string limit
The expected value.
Definition: test.cc:92