Commit 601ed819 authored by Chris Cantwell's avatar Chris Cantwell

Added generic Regex test support.

Added tolerance support and specification in test definition file.
Cleaned up code.
Added error checks.
parent 6bc8bd79
......@@ -8,14 +8,14 @@
</files>
<metrics>
<metric type="L2" id="1">
<value variable="u">5.20394e-16</value>
<value variable="v">0</value>
<value variable="p">8.50369e-15</value>
<value variable="u" tolerance="1e-12">5.20394e-16</value>
<value variable="v" tolerance="1e-12">0</value>
<value variable="p" tolerance="1e-12">8.50369e-15</value>
</metric>
<metric type="Linf" id="2">
<value variable="u">4.7462e-15</value>
<value variable="v">3.04438e-16</value>
<value variable="p">6.23945e-14</value>
<value variable="u" tolerance="1e-12">4.7462e-15</value>
<value variable="v" tolerance="1e-12">3.04438e-16</value>
<value variable="p" tolerance="1e-12">6.23945e-14</value>
</metric>
</metrics>
</test>
......
......@@ -9,14 +9,14 @@
</files>
<metrics>
<metric type="L2" id="1">
<value variable="u">5.20394e-16</value>
<value variable="v">0</value>
<value variable="p">8.50369e-15</value>
<value variable="u" tolerance="1e-12">5.20394e-16</value>
<value variable="v" tolerance="1e-12">0</value>
<value variable="p" tolerance="1e-6">8.50369e-15</value>
</metric>
<metric type="Linf" id="2">
<value variable="u">4.7462e-15</value>
<value variable="v">3.04438e-16</value>
<value variable="p">6.23945e-14</value>
<value variable="u" tolerance="1e-12">4.7462e-15</value>
<value variable="v" tolerance="1e-12">3.04438e-16</value>
<value variable="p" tolerance="1e-6">6.23945e-14</value>
</metric>
</metrics>
</test>
......
......@@ -35,6 +35,7 @@
#include <Metric.h>
#include <loki/Singleton.h>
#include <boost/algorithm/string.hpp>
using namespace std;
......@@ -55,9 +56,14 @@ namespace Nektar
{
if (!metric->Attribute("id"))
{
cout << "Metric has no ID" << endl;
cerr << "Metric has no ID" << endl;
}
if (!metric->Attribute("type"))
{
cerr << "Metric has no type" << endl;
}
m_id = atoi(metric->Attribute("id"));
m_type = boost::to_upper_copy(string(metric->Attribute("type")));
}
/**
......@@ -67,9 +73,4 @@ namespace Nektar
{
return v_Test(pStdout, pStderr);
}
bool Metric::v_Test(std::istream& pStdout, std::istream& pStderr)
{
return true;
}
}
......@@ -48,19 +48,17 @@ namespace Nektar
public:
Metric(TiXmlElement *metric);
/// Perform the test, given the standard output and error streams
bool Test (std::istream& pStdout, std::istream& pStderr);
bool FinishTest();
protected:
// Stores the ID of this metric.
/// Stores the ID of this metric.
int m_id;
// virtual void v_Parse (TiXmlElement *metric) = 0;
virtual bool v_Test (std::istream& pStdout, std::istream& pStderr);
// virtual bool v_FinishTest();
// private:
// void Parse (TiXmlElement *metric);
/// Stores the type of this metric (uppercase).
std::string m_type;
virtual bool v_Test (std::istream& pStdout, std::istream& pStderr) = 0;
};
/// A shared pointer to an EquationSystem object
......@@ -68,9 +66,10 @@ namespace Nektar
/// Datatype of the NekFactory used to instantiate classes derived from the
/// Advection class.
typedef LibUtilities::NekFactory<std::string, Metric, TiXmlElement*> MetricFactory;
MetricFactory& GetMetricFactory();
typedef LibUtilities::NekFactory<std::string, Metric, TiXmlElement*>
MetricFactory;
MetricFactory& GetMetricFactory();
}
......
......@@ -45,27 +45,36 @@ namespace Nektar
// Set up the regular expression. This (optionally) matches a variable
// name if it exists: first field is variable name, second field is L2
// error.
m_regex =
"^L 2 error\\s*(?:\\(variable (\\w+)\\))?\\s*:\\s*([+-]?\\d.+\\d|0).*";
m_regex = "^L 2 error\\s*(?:\\(variable "
"(\\w+)\\))?\\s*:\\s*([+-]?\\d.+\\d|0).*";
// Find the L2 error to match against.
TiXmlElement *value = metric->FirstChildElement("value");
while (value)
{
// Find name of field.
std::string variable = value->Attribute("variable");
// Set up a match with two fields which correspond with the
// subexpression above. The first is the variable name, second is
// the L2 error.
std::vector<std::string> tmp(2);
tmp[0] = variable;
tmp[1] = value->GetText();
ASSERTL0(value->Attribute("variable"),
"Missing variable name in L2 metric.");
ASSERTL0(value->Attribute("tolerance"),
"Missing tolerance in L2 metric");
ASSERTL0(value->GetText() || value->GetText() == "",
"Missing value in L2 metric.");
MetricRegexFieldValue var;
var.m_value = value->Attribute("variable");
MetricRegexFieldValue val;
val.m_value = value->GetText();
val.m_useTolerance = true;
val.m_tolerance = atof(value->Attribute("tolerance"));
std::vector<MetricRegexFieldValue> tmp(2);
tmp[0] = var;
tmp[1] = val;
m_matches.push_back(tmp);
// Indicate that the L2 error needs tolerance testing.
m_tolerance.insert(1);
value = value->NextSiblingElement("value");
}
}
......
......@@ -45,28 +45,38 @@ namespace Nektar
// Set up the regular expression. This (optionally) matches a variable
// name if it exists: first field is variable name, second field is L2
// error.
m_regex =
"^L inf error\\s*(?:\\(variable (\\w+)\\))?\\s*:\\s*([+-]?\\d.+\\d|0).*";
m_regex = "^L inf error\\s*(?:\\(variable "
"(\\w+)\\))?\\s*:\\s*([+-]?\\d.+\\d|0).*";
// Find the L2 error to match against.
TiXmlElement *value = metric->FirstChildElement("value");
while (value)
{
// Find name of field.
std::string variable = value->Attribute("variable");
// Set up a match with two fields which correspond with the
// subexpression above. The first is the variable name, second is
// the LInf error.
std::vector<std::string> tmp(2);
tmp[0] = variable;
tmp[1] = value->GetText();
m_matches.push_back(tmp);
// the L2 error.
ASSERTL0(value->Attribute("variable"),
"Missing variable name in LInf metric.");
ASSERTL0(value->Attribute("tolerance"),
"Missing tolerance in LInf metric");
ASSERTL0(value->GetText() || value->GetText() == "",
"Missing value in LInf metric.");
MetricRegexFieldValue var;
var.m_value = value->Attribute("variable");
// Indicate that the L2 error needs tolerance testing.
m_tolerance.insert(1);
MetricRegexFieldValue val;
val.m_value = value->GetText();
val.m_useTolerance = true;
val.m_tolerance = atof(value->Attribute("tolerance"));
std::vector<MetricRegexFieldValue> tmp(2);
tmp[0] = var;
tmp[1] = val;
m_matches.push_back(tmp);
value = value->NextSiblingElement("value");
}
}
......
......@@ -49,20 +49,58 @@ namespace Nektar
*/
MetricRegex::MetricRegex(TiXmlElement *metric) : Metric(metric)
{
// Parse a regex <METRIC> tag. This would populate m_regex, m_matches
// and m_tolerance below.
// If we are a derived class, do nothing
if (m_type != "REGEX")
{
return;
}
// Parse Regex expression
TiXmlElement *regex = metric->FirstChildElement("regex");
ASSERTL0(regex, "No Regex defined.");
ASSERTL0(regex->GetText(), "Failed to get text");
m_regex = regex->GetText();
// Parse matching values
TiXmlElement *matches = metric->FirstChildElement("matches");
ASSERTL0(matches, "No matches defined.");
TiXmlElement *match = matches->FirstChildElement("match");
while (match)
{
std::vector<MetricRegexFieldValue> tmp;
TiXmlElement *field = match->FirstChildElement("field");
while (field)
{
MetricRegexFieldValue v;
v.m_value = field->GetText();
v.m_useTolerance = false;
const char * tol = field->Attribute("tolerance");
if (tol)
{
v.m_useTolerance = true;
v.m_tolerance = atof(tol);
}
tmp.push_back(v);
field = field->NextSiblingElement("field");
}
m_matches.push_back(tmp);
match = match->NextSiblingElement("match");
}
}
/**
* @brief Test output against a regex expression and set of matches.
*/
bool MetricRegex::v_Test(std::istream& pStdout, std::istream& pStderr)
{
// If we have matched everything, nothing to do.
if (m_matches.size() == 0)
{
return true;
}
ASSERTL0(m_matches.size(), "No test conditions defined for Regex.");
bool success = true;
boost::cmatch matches;
std::vector<std::string> &okValues = m_matches[0];
std::vector<MetricRegexFieldValue> &okValues = m_matches[0];
// Process output file line by line searching for regex matches
std::string line;
......@@ -83,28 +121,33 @@ namespace Nektar
{
std::string match(matches[i].first, matches[i].second);
if (m_tolerance.count(i-1) > 0)
if (okValues[i-1].m_useTolerance)
{
double val = fabs(boost::lexical_cast<double>(
okValues[i-1].m_value)
- boost::lexical_cast<double>(match));
// If the okValues are not within tolerance, failed the
// test.
if (fabs(boost::lexical_cast<double>(okValues[i-1]) -
boost::lexical_cast<double>(match)) > 1e-6)
if (val > okValues[i-1].m_tolerance)
{
cout << "Failed tolerance match." << endl;
cout << " Expected: " << okValues[i-1] << endl;
cout << " Result: " << match << endl;
return false;
cerr << "Failed tolerance match." << endl;
cerr << " Expected: " << okValues[i-1].m_value
<< " +/- " << okValues[i-1].m_tolerance
<< endl;
cerr << " Result: " << match << endl;
success = false;
}
}
else
{
// Case insensitive match.
if (!boost::iequals(match, okValues[i-1]))
if (!boost::iequals(match, okValues[i-1].m_value))
{
cout << "Failed case-insensitive match." << endl;
cout << " Expected: " << okValues[i-1] << endl;
cout << " Result: " << match << endl;
return false;
cerr << "Failed case-insensitive match." << endl;
cerr << " Expected: " << okValues[i-1].m_value
<< endl;
cerr << " Result: " << match << endl;
success = false;
}
}
}
......@@ -114,6 +157,6 @@ namespace Nektar
}
}
return true;
return success;
}
}
......@@ -42,6 +42,19 @@
namespace Nektar
{
/**
* @brief Data structure for a Regex value to match.
*/
struct MetricRegexFieldValue
{
MetricRegexFieldValue()
: m_value(""), m_useTolerance(false), m_tolerance(0.0) {}
std::string m_value;
bool m_useTolerance;
double m_tolerance;
};
class MetricRegex : public Metric
{
public:
......@@ -56,9 +69,7 @@ namespace Nektar
/// Storage for the boost regex.
boost::regex m_regex;
/// Stores the multiple matches defined in each <MATCH> tag.
std::vector<std::vector<std::string> > m_matches;
/// Indicates which fields of a match need to be tested for tolerance.
std::set<int> m_tolerance;
std::vector<std::vector<MetricRegexFieldValue> > m_matches;
MetricRegex(TiXmlElement *metric);
......
......@@ -72,7 +72,7 @@ int main(int argc, char *argv[])
{
cerr << "Error: incorrect number of arguments" << endl;
cerr << "Usage: " << argv[0] << " [test-definition.tst]" << endl;
return -1;
return 2;
}
// Path to test definition file
......@@ -82,7 +82,8 @@ int main(int argc, char *argv[])
const fs::path specPath = specFile.parent_path();
// Temporary directory to create and in which to conduct test
const fs::path tmpDir = fs::current_path() / fs::path("tmp_" + specFile.stem().string());
const fs::path tmpDir = fs::current_path()
/ fs::path("tmp_" + specFile.stem().string());
// The current directory
const fs::path startDir = fs::current_path();
......@@ -187,10 +188,10 @@ int main(int argc, char *argv[])
// Return status of test. 0 = PASS, 1 = FAIL
return status;
}
catch (const fs::filesystem_error e)
catch (const fs::filesystem_error& e)
{
cout << "Filesystem operation error occurred:" << endl;
cout << " " << e.what() << endl;
cerr << "Filesystem operation error occurred:" << endl;
cerr << " " << e.what() << endl;
if (fs::exists(tmpDir))
{
fs::remove_all(tmpDir);
......@@ -204,6 +205,6 @@ int main(int argc, char *argv[])
}
}
// If a system error, return -1
return -1;
// If a system error, return 2
return 2;
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment