Added generic Regex test support.

Added tolerance support and specification in test definition file.
Cleaned up code.
Added error checks.

solvers/IncNavierStokesSolver/Tests/Channel_Flow_3modes.tst

@@ -8,14 +8,14 @@
     </files>
     <metrics>
         <metric type="L2" id="1">
-            <value variable="u">5.20394e-16</value>
-            <value variable="v">0</value>
-            <value variable="p">8.50369e-15</value>
+            <value variable="u" tolerance="1e-12">5.20394e-16</value>
+            <value variable="v" tolerance="1e-12">0</value>
+            <value variable="p" tolerance="1e-12">8.50369e-15</value>
         </metric>
         <metric type="Linf" id="2">
-            <value variable="u">4.7462e-15</value>
-            <value variable="v">3.04438e-16</value>
-            <value variable="p">6.23945e-14</value>
+            <value variable="u" tolerance="1e-12">4.7462e-15</value>
+            <value variable="v" tolerance="1e-12">3.04438e-16</value>
+            <value variable="p" tolerance="1e-12">6.23945e-14</value>
         </metric>
     </metrics>
 </test>

solvers/IncNavierStokesSolver/Tests/Channel_Flow_3modes_Parallel.tst

@@ -9,14 +9,14 @@
     </files>
     <metrics>
         <metric type="L2" id="1">
-            <value variable="u">5.20394e-16</value>
-            <value variable="v">0</value>
-            <value variable="p">8.50369e-15</value>
+            <value variable="u" tolerance="1e-12">5.20394e-16</value>
+            <value variable="v" tolerance="1e-12">0</value>
+            <value variable="p" tolerance="1e-6">8.50369e-15</value>
         </metric>
         <metric type="Linf" id="2">
-            <value variable="u">4.7462e-15</value>
-            <value variable="v">3.04438e-16</value>
-            <value variable="p">6.23945e-14</value>
+            <value variable="u" tolerance="1e-12">4.7462e-15</value>
+            <value variable="v" tolerance="1e-12">3.04438e-16</value>
+            <value variable="p" tolerance="1e-6">6.23945e-14</value>
         </metric>
     </metrics>
 </test>

tests/Metric.cpp

@@ -35,6 +35,7 @@
 
 #include <Metric.h>
 #include <loki/Singleton.h>
+#include <boost/algorithm/string.hpp>
 
 using namespace std;
 
@@ -55,9 +56,14 @@ namespace Nektar
     {
         if (!metric->Attribute("id"))
         {
-            cout << "Metric has no ID" << endl;
+            cerr << "Metric has no ID" << endl;
+        }
+        if (!metric->Attribute("type"))
+        {
+            cerr << "Metric has no type" << endl;
         }
         m_id = atoi(metric->Attribute("id"));
+        m_type = boost::to_upper_copy(string(metric->Attribute("type")));
     }
 
     /**
@@ -67,9 +73,4 @@ namespace Nektar
     {
         return v_Test(pStdout, pStderr);
     }
-    
-    bool Metric::v_Test(std::istream& pStdout, std::istream& pStderr)
-    {
-        return true;
-    }
 }

tests/Metric.h

@@ -48,19 +48,17 @@ namespace Nektar
     public:
         Metric(TiXmlElement *metric);
         
+        /// Perform the test, given the standard output and error streams
         bool Test  (std::istream& pStdout, std::istream& pStderr);
-        bool FinishTest();
         
     protected:
-        // Stores the ID of this metric.
+        /// Stores the ID of this metric.
         int m_id;
-        
-//        virtual void v_Parse     (TiXmlElement *metric) = 0;
-        virtual bool v_Test      (std::istream& pStdout, std::istream& pStderr);
-//        virtual bool v_FinishTest();
 
-//    private:
-//        void Parse     (TiXmlElement *metric);
+        /// Stores the type of this metric (uppercase).
+        std::string m_type;
+        
+        virtual bool v_Test (std::istream& pStdout, std::istream& pStderr) = 0;
     };
 
     /// A shared pointer to an EquationSystem object
@@ -68,9 +66,10 @@ namespace Nektar
     
     /// Datatype of the NekFactory used to instantiate classes derived from the
     /// Advection class.
-    typedef LibUtilities::NekFactory<std::string, Metric, TiXmlElement*> MetricFactory;
-    MetricFactory& GetMetricFactory();
+    typedef LibUtilities::NekFactory<std::string, Metric, TiXmlElement*>
+                                                                MetricFactory;
 
+    MetricFactory& GetMetricFactory();
 }
 
 

tests/MetricL2.cpp

@@ -45,27 +45,36 @@ namespace Nektar
         // Set up the regular expression. This (optionally) matches a variable
         // name if it exists: first field is variable name, second field is L2
         // error.
-        m_regex = 
-           "^L 2 error\\s*(?:\\(variable (\\w+)\\))?\\s*:\\s*([+-]?\\d.+\\d|0).*";
+        m_regex = "^L 2 error\\s*(?:\\(variable "
+                  "(\\w+)\\))?\\s*:\\s*([+-]?\\d.+\\d|0).*";
         
         // Find the L2 error to match against.
         TiXmlElement *value = metric->FirstChildElement("value");
         while (value)
         {
-            // Find name of field.
-            std::string variable = value->Attribute("variable");
-            
             // Set up a match with two fields which correspond with the
             // subexpression above. The first is the variable name, second is
             // the L2 error.
-            std::vector<std::string> tmp(2);
-            tmp[0] = variable;
-            tmp[1] = value->GetText();
+            ASSERTL0(value->Attribute("variable"),
+                     "Missing variable name in L2 metric.");
+            ASSERTL0(value->Attribute("tolerance"),
+                     "Missing tolerance in L2 metric");
+            ASSERTL0(value->GetText() || value->GetText() == "",
+                     "Missing value in L2 metric.");
+
+            MetricRegexFieldValue var;
+            var.m_value = value->Attribute("variable");
+
+            MetricRegexFieldValue val;
+            val.m_value = value->GetText();
+            val.m_useTolerance = true;
+            val.m_tolerance = atof(value->Attribute("tolerance"));
+
+            std::vector<MetricRegexFieldValue> tmp(2);
+            tmp[0] = var;
+            tmp[1] = val;
             m_matches.push_back(tmp);
             
-            // Indicate that the L2 error needs tolerance testing.
-            m_tolerance.insert(1);
-
             value = value->NextSiblingElement("value");
         }
     }

tests/MetricLInf.cpp

@@ -45,28 +45,38 @@ namespace Nektar
         // Set up the regular expression. This (optionally) matches a variable
         // name if it exists: first field is variable name, second field is L2
         // error.
-        m_regex =
-           "^L inf error\\s*(?:\\(variable (\\w+)\\))?\\s*:\\s*([+-]?\\d.+\\d|0).*";
+        m_regex = "^L inf error\\s*(?:\\(variable "
+                  "(\\w+)\\))?\\s*:\\s*([+-]?\\d.+\\d|0).*";
 
         // Find the L2 error to match against.
         TiXmlElement *value = metric->FirstChildElement("value");
         while (value)
         {
-            // Find name of field.
-            std::string variable = value->Attribute("variable");
-
             // Set up a match with two fields which correspond with the
             // subexpression above. The first is the variable name, second is
-            // the LInf error.
-            std::vector<std::string> tmp(2);
-            tmp[0] = variable;
-            tmp[1] = value->GetText();
-            m_matches.push_back(tmp);
+            // the L2 error.
+            ASSERTL0(value->Attribute("variable"),
+                     "Missing variable name in LInf metric.");
+            ASSERTL0(value->Attribute("tolerance"),
+                     "Missing tolerance in LInf metric");
+            ASSERTL0(value->GetText() || value->GetText() == "",
+                     "Missing value in LInf metric.");
+
+            MetricRegexFieldValue var;
+            var.m_value = value->Attribute("variable");
 
-            // Indicate that the L2 error needs tolerance testing.
-            m_tolerance.insert(1);
+            MetricRegexFieldValue val;
+            val.m_value = value->GetText();
+            val.m_useTolerance = true;
+            val.m_tolerance = atof(value->Attribute("tolerance"));
+
+            std::vector<MetricRegexFieldValue> tmp(2);
+            tmp[0] = var;
+            tmp[1] = val;
+            m_matches.push_back(tmp);
 
             value = value->NextSiblingElement("value");
+
         }
     }
 

tests/MetricRegex.cpp

@@ -49,20 +49,58 @@ namespace Nektar
      */
     MetricRegex::MetricRegex(TiXmlElement *metric) : Metric(metric)
     {
-        // Parse a regex <METRIC> tag. This would populate m_regex, m_matches
-        // and m_tolerance below.
+        // If we are a derived class, do nothing
+        if (m_type != "REGEX")
+        {
+            return;
+        }
+
+        // Parse Regex expression
+        TiXmlElement *regex = metric->FirstChildElement("regex");
+        ASSERTL0(regex, "No Regex defined.");
+        ASSERTL0(regex->GetText(), "Failed to get text");
+        m_regex = regex->GetText();
+
+        // Parse matching values
+        TiXmlElement *matches = metric->FirstChildElement("matches");
+        ASSERTL0(matches, "No matches defined.");
+        TiXmlElement *match = matches->FirstChildElement("match");
+        while (match)
+        {
+            std::vector<MetricRegexFieldValue> tmp;
+            TiXmlElement *field = match->FirstChildElement("field");
+            while (field)
+            {
+                MetricRegexFieldValue v;
+                v.m_value = field->GetText();
+                v.m_useTolerance = false;
+
+                const char * tol = field->Attribute("tolerance");
+                if (tol)
+                {
+                    v.m_useTolerance = true;
+                    v.m_tolerance = atof(tol);
+                }
+                tmp.push_back(v);
+                field = field->NextSiblingElement("field");
+            }
+            m_matches.push_back(tmp);
+
+            match = match->NextSiblingElement("match");
+        }
     }
 
+
+    /**
+     * @brief Test output against a regex expression and set of matches.
+     */
     bool MetricRegex::v_Test(std::istream& pStdout, std::istream& pStderr)
     {
-        // If we have matched everything, nothing to do.
-        if (m_matches.size() == 0)
-        {
-            return true;
-        }
-        
+        ASSERTL0(m_matches.size(), "No test conditions defined for Regex.");
+
+        bool success = true;
         boost::cmatch             matches;
-        std::vector<std::string> &okValues = m_matches[0];
+        std::vector<MetricRegexFieldValue> &okValues = m_matches[0];
 
         // Process output file line by line searching for regex matches
         std::string line;
@@ -83,28 +121,33 @@ namespace Nektar
                 {
                     std::string match(matches[i].first, matches[i].second);
 
-                    if (m_tolerance.count(i-1) > 0)
+                    if (okValues[i-1].m_useTolerance)
                     {
+                        double val = fabs(boost::lexical_cast<double>(
+                                                        okValues[i-1].m_value)
+                                          - boost::lexical_cast<double>(match));
                         // If the okValues are not within tolerance, failed the
                         // test.
-                        if (fabs(boost::lexical_cast<double>(okValues[i-1]) -
-                                 boost::lexical_cast<double>(match)) > 1e-6)
+                        if (val > okValues[i-1].m_tolerance)
                         {
-                            cout << "Failed tolerance match." << endl;
-                            cout << "  Expected: " << okValues[i-1] << endl;
-                            cout << "  Result:   " << match << endl;
-                            return false;
+                            cerr << "Failed tolerance match." << endl;
+                            cerr << "  Expected: " << okValues[i-1].m_value
+                                 << " +/- " << okValues[i-1].m_tolerance
+                                 << endl;
+                            cerr << "  Result:   " << match << endl;
+                            success = false;
                         }
                     }
                     else
                     {
                         // Case insensitive match.
-                        if (!boost::iequals(match, okValues[i-1]))
+                        if (!boost::iequals(match, okValues[i-1].m_value))
                         {
-                            cout << "Failed case-insensitive match." << endl;
-                            cout << "  Expected: " << okValues[i-1] << endl;
-                            cout << "  Result:   " << match << endl;
-                            return false;
+                            cerr << "Failed case-insensitive match." << endl;
+                            cerr << "  Expected: " << okValues[i-1].m_value
+                                 << endl;
+                            cerr << "  Result:   " << match << endl;
+                            success = false;
                         }
                     }
                 }
@@ -114,6 +157,6 @@ namespace Nektar
             }
         }
 
-        return true;
+        return success;
     }
 }

tests/MetricRegex.h

@@ -42,6 +42,19 @@
 
 namespace Nektar
 {
+    /**
+     * @brief Data structure for a Regex value to match.
+     */
+    struct MetricRegexFieldValue
+    {
+        MetricRegexFieldValue()
+            : m_value(""), m_useTolerance(false), m_tolerance(0.0) {}
+
+        std::string m_value;
+        bool m_useTolerance;
+        double m_tolerance;
+    };
+
     class MetricRegex : public Metric
     {
     public:
@@ -56,9 +69,7 @@ namespace Nektar
         /// Storage for the boost regex.
         boost::regex                           m_regex;
         /// Stores the multiple matches defined in each <MATCH> tag.
-        std::vector<std::vector<std::string> > m_matches;
-        /// Indicates which fields of a match need to be tested for tolerance.
-        std::set<int>                          m_tolerance;
+        std::vector<std::vector<MetricRegexFieldValue> > m_matches;
 
         MetricRegex(TiXmlElement *metric);
 

tests/Tester.cpp

@@ -72,7 +72,7 @@ int main(int argc, char *argv[])
     {
         cerr << "Error: incorrect number of arguments" << endl;
         cerr << "Usage: " << argv[0] << " [test-definition.tst]" << endl;
-        return -1;
+        return 2;
     }
 
     // Path to test definition file
@@ -82,7 +82,8 @@ int main(int argc, char *argv[])
     const fs::path specPath = specFile.parent_path();
 
     // Temporary directory to create and in which to conduct test
-    const fs::path tmpDir = fs::current_path() / fs::path("tmp_" + specFile.stem().string());
+    const fs::path tmpDir = fs::current_path()
+                            / fs::path("tmp_" + specFile.stem().string());
 
     // The current directory
     const fs::path startDir = fs::current_path();
@@ -187,10 +188,10 @@ int main(int argc, char *argv[])
         // Return status of test. 0 = PASS, 1 = FAIL
         return status;
     }
-    catch (const fs::filesystem_error e)
+    catch (const fs::filesystem_error& e)
     {
-        cout << "Filesystem operation error occurred:" << endl;
-        cout << "  " << e.what() << endl;
+        cerr << "Filesystem operation error occurred:" << endl;
+        cerr << "  " << e.what() << endl;
         if (fs::exists(tmpDir))
         {
             fs::remove_all(tmpDir);
@@ -204,6 +205,6 @@ int main(int argc, char *argv[])
         }
     }
 
-    // If a system error, return -1
-    return -1;
+    // If a system error, return 2
+    return 2;
 }