Changed test output to hopefully make it easier to associate failure with the test it failed in and which test suite is being run easier to find. Will add some ostream magic later to allow for a brief option where test output (printed mails, etc) won't be displayed
authorKrista Bennett <krista@pep-project.org>
Thu, 06 Sep 2018 17:08:17 +0200
changeset 29286d8e51098e7a
parent 2898 601cdaeef663
child 2929 133ea04d3ea3
Changed test output to hopefully make it easier to associate failure with the test it failed in and which test suite is being run easier to find. Will add some ostream magic later to allow for a brief option where test output (printed mails, etc) won't be displayed
test/include/EngineTestSuite.h
test/include/pEpTestOutput.h
test/include/test_util.h
test/src/EngineTestIndividualSuite.cc
test/src/EngineTestSessionSuite.cc
test/src/EngineTestSuite.cc
test/src/TestDriver.cc
test/src/engine_tests/LeastColorGroupTests.cc
test/src/engine_tests/LeastCommonDenomColorTests.cc
test/src/engine_tests/MessageApiTests.cc
test/src/pEpTestOutput.cc
     1.1 --- a/test/include/EngineTestSuite.h	Tue Aug 28 17:37:56 2018 +0200
     1.2 +++ b/test/include/EngineTestSuite.h	Thu Sep 06 17:08:17 2018 +0200
     1.3 @@ -16,7 +16,9 @@
     1.4          virtual ~EngineTestSuite();
     1.5          
     1.6          void add_test_to_suite(std::pair<std::string, void (Test::Suite::*)()> test_func);
     1.7 -        
     1.8 +        void output_test_header(const char* test_name);
     1.9 +        void output_test_footer(const char* test_name);
    1.10 +
    1.11      protected:
    1.12          PEP_SESSION session;
    1.13          string test_home;
     2.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     2.2 +++ b/test/include/pEpTestOutput.h	Thu Sep 06 17:08:17 2018 +0200
     2.3 @@ -0,0 +1,43 @@
     2.4 +
     2.5 +#ifndef PEP_TEST_OUTPUT_H
     2.6 +#define PEP_TEST_OUTPUT_H
     2.7 +
     2.8 +#include <cpptest.h>
     2.9 +#include <vector>
    2.10 +#include <string>
    2.11 +
    2.12 +namespace Test {
    2.13 +    class pEpTestOutput : public Output {
    2.14 +        public:
    2.15 +            pEpTestOutput();
    2.16 +            ~pEpTestOutput() {};
    2.17 +            void finished(int tests, const Time& time);
    2.18 +            void initialize(int tests) {};
    2.19 +            void suite_start(int tests, const std::string& name);
    2.20 +            void suite_end(int tests, const std::string& name,
    2.21 +                           const Time& time);
    2.22 +            void test_start(const std::string& name);
    2.23 +            void test_end(const std::string& name, bool ok,
    2.24 +                          const Time& time);
    2.25 +            void assertment(const Test::Source& s);
    2.26 +            
    2.27 +            void outputCorrectPercentage(int num_tests, int failures, int width);
    2.28 +            
    2.29 +        private:
    2.30 +            static constexpr const char* huge_sepline = "///////////////////////////////////////////////////////////////////////////\n";
    2.31 +            static constexpr const char* alt_sepline = "++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n";
    2.32 +            static constexpr const char* big_sepline = "********************************************************\n";
    2.33 +            static constexpr const char* med_sepline = "-----------------------------------\n";
    2.34 +            static constexpr const char* sml_sepline = "++++++++++++++++++++++++++\n";
    2.35 +            static constexpr const char* lil_sepline = "~~~~~~\n";
    2.36 +            int _total_failed;
    2.37 +            int _total_tests;
    2.38 +            int _suite_failed;
    2.39 +            int _suite_total;
    2.40 +            std::string _suite_name;
    2.41 +            std::string _test_name;
    2.42 +            std::vector<Source> _test_errors;
    2.43 +            
    2.44 +    };
    2.45 +}
    2.46 +#endif
    2.47 \ No newline at end of file
     3.1 --- a/test/include/test_util.h	Tue Aug 28 17:37:56 2018 +0200
     3.2 +++ b/test/include/test_util.h	Thu Sep 06 17:08:17 2018 +0200
     3.3 @@ -43,5 +43,6 @@
     3.4                           const struct stat *file_stat, 
     3.5                           int ftw_info, 
     3.6                           struct FTW * ftw_struct);
     3.7 +                         
     3.8          
     3.9  #endif
     4.1 --- a/test/src/EngineTestIndividualSuite.cc	Tue Aug 28 17:37:56 2018 +0200
     4.2 +++ b/test/src/EngineTestIndividualSuite.cc	Thu Sep 06 17:08:17 2018 +0200
     4.3 @@ -27,4 +27,4 @@
     4.4  void EngineTestIndividualSuite::tear_down() {
     4.5      restore_full_env();
     4.6      EngineTestSuite::tear_down();
     4.7 -}
     4.8 +}
     4.9 \ No newline at end of file
     5.1 --- a/test/src/EngineTestSessionSuite.cc	Tue Aug 28 17:37:56 2018 +0200
     5.2 +++ b/test/src/EngineTestSessionSuite.cc	Thu Sep 06 17:08:17 2018 +0200
     5.3 @@ -23,7 +23,6 @@
     5.4  }
     5.5  
     5.6  void EngineTestSessionSuite::tear_down() {
     5.7 -    cout << "Session: tear_down - OTN: " << on_test_number << " NOT: " << number_of_tests << endl; 
     5.8      if (on_test_number == number_of_tests)
     5.9          restore_full_env();
    5.10          
     6.1 --- a/test/src/EngineTestSuite.cc	Tue Aug 28 17:37:56 2018 +0200
     6.2 +++ b/test/src/EngineTestSuite.cc	Thu Sep 06 17:08:17 2018 +0200
     6.3 @@ -18,6 +18,7 @@
     6.4  
     6.5  #include "test_util.h"
     6.6  #include "EngineTestSuite.h"
     6.7 +
     6.8  using namespace std;
     6.9  
    6.10  // Constructor
    6.11 @@ -107,7 +108,7 @@
    6.12          struct stat buf;
    6.13  
    6.14          if (stat(test_home.c_str(), &buf) == 0) {
    6.15 -            cout << test_home << " exists. We'll recursively delete. We hope we're not horking your whole system..." << endl;
    6.16 +//            cout << test_home << " exists. We'll recursively delete. We hope we're not horking your whole system..." << endl;
    6.17              int success = nftw((test_home + "/.").c_str(), util_delete_filepath, 100, FTW_DEPTH);
    6.18          }
    6.19      }
    6.20 @@ -141,7 +142,6 @@
    6.21  
    6.22      
    6.23      string home = getenv("HOME");
    6.24 -    cout << "home is " << home << endl;
    6.25      assert(temp_test_home.compare(home) != 0);
    6.26      assert(temp_test_home.compare(home + "/") != 0);
    6.27      assert(temp_test_home.compare(home + "/.gnupg") != 0); // This is an EXCLUSION test, so we leave this.
    6.28 @@ -163,6 +163,7 @@
    6.29          throw std::runtime_error("SETUP: new GNUPGHOME threatens to mess up user GNUPGHOME (and delete all their keys). NO DICE.");
    6.30      
    6.31  //    cout << "Ok - checked if new test home will be safe. We'll try and make the directory, deleting it if it has already exists." << endl;
    6.32 +    cout << "Test home directory is " << temp_test_home << endl;
    6.33      
    6.34      struct stat buf;
    6.35      
    6.36 @@ -170,7 +171,7 @@
    6.37      if (success != 0)
    6.38          throw std::runtime_error("SETUP: Error when setting GNUPGHOME.");
    6.39  
    6.40 -    cout << "New GNUPGHOME is " << getenv("GNUPGHOME") << endl;
    6.41 +    cout << "New GNUPGHOME is " << getenv("GNUPGHOME") << endl << endl;
    6.42      
    6.43      success = setenv("HOME", temp_test_home.c_str(), 1);
    6.44      if (success != 0)
    6.45 @@ -217,8 +218,8 @@
    6.46      success = setenv("HOME", real_home.c_str(), 1);
    6.47      if (success != 0)
    6.48          throw std::runtime_error("RESTORE: Cannot reset home directory! Either set environment variable manually back to your home, or quit this session!");
    6.49 -    else
    6.50 -        cout << "RESTORE: HOME is now " << getenv("HOME") << endl;
    6.51 +    // else
    6.52 +    //     cout << "RESTORE: HOME is now " << getenv("HOME") << endl;
    6.53      unix_local_db(true);
    6.54      gpg_conf(true);
    6.55      gpg_agent_conf(true);
    6.56 @@ -234,4 +235,3 @@
    6.57  void EngineTestSuite::set_my_name() {
    6.58      my_name = typeid(*this).name();
    6.59  }
    6.60 -
     7.1 --- a/test/src/TestDriver.cc	Tue Aug 28 17:37:56 2018 +0200
     7.2 +++ b/test/src/TestDriver.cc	Thu Sep 06 17:08:17 2018 +0200
     7.3 @@ -1,7 +1,9 @@
     7.4  #include <stdlib.h>
     7.5  #include <cpptest.h>
     7.6  #include <cpptest-suite.h>
     7.7 -#include <cpptest-textoutput.h>
     7.8 +#include <cpptest-output.h>
     7.9 +#include "pEpTestOutput.h"
    7.10 +
    7.11  #include <string>
    7.12  #include <vector>
    7.13  #include <sys/stat.h>
    7.14 @@ -51,7 +53,7 @@
    7.15          test_runner->add(suite); 
    7.16      }
    7.17  
    7.18 -    Test::TextOutput output(Test::TextOutput::Verbose);
    7.19 -    return test_runner->run(output, false) ? 0 : 1;
    7.20 -    
    7.21 +    Test::Output* output = new Test::pEpTestOutput(); // blah
    7.22 +    return test_runner->run(*output, false) ? 0 : 1;
    7.23 +    delete(output);
    7.24  }
     8.1 --- a/test/src/engine_tests/LeastColorGroupTests.cc	Tue Aug 28 17:37:56 2018 +0200
     8.2 +++ b/test/src/engine_tests/LeastColorGroupTests.cc	Thu Sep 06 17:08:17 2018 +0200
     8.3 @@ -57,7 +57,6 @@
     8.4      pEp_identity * sender1 = new_identity("pep.color.test.V@kgrothoff.org",
     8.5                                            NULL, "TOFU_pep.color.test.V@kgrothoff.org",
     8.6                                            "Pep Color Test V (sender)");
     8.7 -    
     8.8      status = update_identity(session, sender1);
     8.9      trust_personal_key(session, sender1);
    8.10      status = update_identity(session, sender1);
     9.1 --- a/test/src/engine_tests/LeastCommonDenomColorTests.cc	Tue Aug 28 17:37:56 2018 +0200
     9.2 +++ b/test/src/engine_tests/LeastCommonDenomColorTests.cc	Thu Sep 06 17:08:17 2018 +0200
     9.3 @@ -26,7 +26,7 @@
     9.4  }
     9.5  
     9.6  void LeastCommonDenomColorTests::check_least_common_denom_color() {
     9.7 -        
     9.8 +
     9.9      const char* mailfile = "test_mails/Test_Message_JSON-21_Color_Problems.eml";
    9.10              
    9.11      // import keys
    9.12 @@ -137,5 +137,4 @@
    9.13      dest_msg = nullptr;
    9.14      keylist = nullptr;
    9.15      rating = PEP_rating_unreliable;
    9.16 -    
    9.17  }
    10.1 --- a/test/src/engine_tests/MessageApiTests.cc	Tue Aug 28 17:37:56 2018 +0200
    10.2 +++ b/test/src/engine_tests/MessageApiTests.cc	Thu Sep 06 17:08:17 2018 +0200
    10.3 @@ -26,7 +26,6 @@
    10.4  }
    10.5  
    10.6  void MessageApiTests::check_message_api() {
    10.7 -
    10.8      cout << "Importing Alice's key " << endl;
    10.9      const string alice_pub_key = slurp("test_keys/pub/pep-test-alice-0x6FF00E97_pub.asc");
   10.10      const string alice_priv_key = slurp("test_keys/priv/pep-test-alice-0x6FF00E97_priv.asc");
   10.11 @@ -221,5 +220,5 @@
   10.12      cout << "done.\n";
   10.13  
   10.14      free(enc_msg);
   10.15 -    free(dec_msg);
   10.16 +    free(dec_msg);    
   10.17  }
    11.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
    11.2 +++ b/test/src/pEpTestOutput.cc	Thu Sep 06 17:08:17 2018 +0200
    11.3 @@ -0,0 +1,98 @@
    11.4 +#include <cpptest.h>
    11.5 +#include <iostream>
    11.6 +#include <iomanip>
    11.7 +#include <vector>
    11.8 +#include <string>
    11.9 +#include <cstring>
   11.10 +
   11.11 +#include "pEpTestOutput.h"
   11.12 +using namespace std;
   11.13 +
   11.14 +namespace Test {
   11.15 +	pEpTestOutput::pEpTestOutput() : _total_failed(0), _total_tests(0) {}
   11.16 +    
   11.17 +    void pEpTestOutput::outputCorrectPercentage(int num_tests, int failures, int width) {
   11.18 +        cout << setw(width);
   11.19 +        if (num_tests) {
   11.20 +            double percentage = ((num_tests - failures) / (double)num_tests) * 100;
   11.21 +            cout << std::fixed << setprecision(1) << percentage << "\%"; 
   11.22 +        }
   11.23 +        else 
   11.24 +            cout << "N/A";
   11.25 +    }
   11.26 +    	
   11.27 +	void pEpTestOutput::finished(int tests, const Test::Time& time) {
   11.28 +        cout << huge_sepline << endl;
   11.29 +        string header = "FULL TEST RUN RESULTS:";
   11.30 +        cout << alt_sepline << left << setw(header.size()) << header;
   11.31 +        cout << right << setw(56 - header.size()) << "+" << endl;
   11.32 +        cout << right << setw(30) << "Number of tests run: " << setw(7) << tests << setw(19) << "+" << endl;
   11.33 +        cout << right << setw(30) << "Tests failed: " << setw(7) << _total_failed << setw(19) << "+" << endl;
   11.34 +        cout << right << setw(30) << "Pass percentage: ";
   11.35 +        outputCorrectPercentage(_total_tests, _total_failed, 7);
   11.36 +        cout << setw(18) << "+" << endl;
   11.37 +        cout << setw(56) << "+" << endl;
   11.38 +        string finalstr = std::to_string(tests) + " tests run in " + std::to_string(time.seconds()) + "." + std::to_string(time.microseconds()) + " seconds.";
   11.39 +        int remlen = 56 - finalstr.size();
   11.40 +        cout << left << setw(finalstr.size()) << finalstr << right << setw(remlen) << "+" << endl;
   11.41 +        cout << alt_sepline << endl;
   11.42 +	}
   11.43 +	
   11.44 +	void pEpTestOutput::suite_start(int tests, const string& name) {
   11.45 +        _suite_failed = 0;
   11.46 +        _suite_name = name;
   11.47 +        _suite_total = 0;
   11.48 +        if (tests > 0) {
   11.49 +            cout << endl << huge_sepline;
   11.50 +            cout << "BEGIN TEST SUITE: " << name << endl << endl;
   11.51 +        }
   11.52 +	}
   11.53 +	
   11.54 +	void pEpTestOutput::suite_end(int tests, const string& name, const Test::Time& time)
   11.55 +	{
   11.56 +        if (tests > 0) {
   11.57 +            cout << endl << "Suite results:" << endl; 
   11.58 +            cout << right << setw(30) << "Number of tests run: " << setw(7) << tests << endl;
   11.59 +            cout << right << setw(30) << "Tests failed: " << setw(7) << _total_failed << endl;
   11.60 +            cout << right << setw(30) << "Pass percentage: ";
   11.61 +            outputCorrectPercentage(_suite_total, _suite_failed, 7);
   11.62 +            cout << endl << endl;
   11.63 +            cout << tests << " tests run in " << time << " seconds." << endl;
   11.64 +            cout << endl;
   11.65 +            cout << "END TEST SUITE: " << name << endl;
   11.66 +        }
   11.67 +	}
   11.68 +    
   11.69 +	void pEpTestOutput::test_start(const std::string& name) {
   11.70 +        _test_name = name;
   11.71 +        cout << med_sepline;
   11.72 +        cout << "Begin test " << name << endl;
   11.73 +        cout << lil_sepline;
   11.74 +        _test_errors.clear();
   11.75 +    }
   11.76 +    void pEpTestOutput::test_end(const string& name, bool ok, const Test::Time&) {
   11.77 +	    if (!ok) {
   11.78 +            _suite_failed++;
   11.79 +            _total_failed++;
   11.80 +            cout << endl << endl << alt_sepline;
   11.81 +            cout << "*** Test " << name << " failed!" << endl;
   11.82 +            vector<Source>::iterator it;
   11.83 +            for (it = _test_errors.begin(); it != _test_errors.end(); it++) {
   11.84 +                Source src = *it;
   11.85 +                cout << lil_sepline;
   11.86 +                cout << left << setw(25) << "*** Assert location: " << src.file() << ":" << src.line() << endl;
   11.87 +                cout << left << setw(25) << "*** Message: " << src.message() << endl;                
   11.88 +            }
   11.89 +            cout << alt_sepline << endl;
   11.90 +        }
   11.91 +        _total_tests++;
   11.92 +        _suite_total++;
   11.93 +
   11.94 +        cout << "End test " << name << endl;
   11.95 +        cout << med_sepline;        
   11.96 +	}
   11.97 +	
   11.98 +	void pEpTestOutput::assertment(const Source& s) {
   11.99 +		_test_errors.push_back(s);
  11.100 +	}
  11.101 +}
  11.102 \ No newline at end of file