Commit bf4aff58 authored by Kilian Lackhove's avatar Kilian Lackhove
Browse files

turn PtsIO into a derived class of FieldIO. This removes most of the code duplication in PtsIO

parent f6dfcc40
......@@ -147,8 +147,9 @@ namespace Nektar
*
*/
FieldIO::FieldIO(LibUtilities::CommSharedPtr pComm,
bool sharedFilesystem)
: m_comm(pComm), m_sharedFilesystem(sharedFilesystem)
bool sharedFilesystem) :
m_comm(pComm),
m_sharedFilesystem(sharedFilesystem)
{
}
......@@ -178,7 +179,8 @@ namespace Nektar
// Prepare to write out data. In parallel, we must create directory
// and determine the full pathname to the file to write out.
// Any existing file/directory which is in the way is removed.
std::string filename = SetUpOutput(outFile, fielddefs, fieldmetadatamap);
std::string filename = SetUpOutput(outFile);
SetUpFieldMetaData(outFile, fielddefs, fieldmetadatamap);
// Create the file (partition)
TiXmlDocument doc;
......@@ -1137,9 +1139,7 @@ namespace Nektar
/**
*
*/
std::string FieldIO::SetUpOutput(const std::string outname,
const std::vector<FieldDefinitionsSharedPtr>& fielddefs,
const FieldMetaDataMap &fieldmetadatamap)
std::string FieldIO::SetUpOutput(const std::string outname)
{
ASSERTL0(!outname.empty(), "Empty path given to SetUpOutput()");
......@@ -1158,8 +1158,8 @@ namespace Nektar
else
{
// Guess at filename that might belong to this process.
boost::format pad("P%1$07d.fld");
pad % m_comm->GetRank();
boost::format pad("P%1$07d.%2$s");
pad % m_comm->GetRank() % GetFileEnding();
// Generate full path name
fs::path poutfile(pad.str());
......@@ -1212,20 +1212,6 @@ namespace Nektar
return LibUtilities::PortablePath(specPath);
}
// Compute number of elements on this process and share with other
// processes. Also construct list of elements on this process from
// available vector of field definitions.
std::vector<unsigned int> elmtnums(nprocs,0);
std::vector<unsigned int> idlist;
int i;
for (i = 0; i < fielddefs.size(); ++i)
{
elmtnums[rank] += fielddefs[i]->m_elementIDs.size();
idlist.insert(idlist.end(), fielddefs[i]->m_elementIDs.begin(),
fielddefs[i]->m_elementIDs.end());
}
m_comm->AllReduce(elmtnums,LibUtilities::ReduceMax);
// Create the destination directory
try
{
......@@ -1241,6 +1227,37 @@ namespace Nektar
m_comm->Block();
// Return the full path to the partition for this process
return LibUtilities::PortablePath(fulloutname);
}
void FieldIO::SetUpFieldMetaData(
const string outname,
const vector< FieldDefinitionsSharedPtr > &fielddefs,
const FieldMetaDataMap &fieldmetadatamap)
{
ASSERTL0(!outname.empty(), "Empty path given to SetUpFieldMetaData()");
int nprocs = m_comm->GetSize();
int rank = m_comm->GetRank();
fs::path specPath (outname);
// Compute number of elements on this process and share with other
// processes. Also construct list of elements on this process from
// available vector of field definitions.
std::vector<unsigned int> elmtnums(nprocs,0);
std::vector<unsigned int> idlist;
int i;
for (i = 0; i < fielddefs.size(); ++i)
{
elmtnums[rank] += fielddefs[i]->m_elementIDs.size();
idlist.insert(idlist.end(), fielddefs[i]->m_elementIDs.begin(),
fielddefs[i]->m_elementIDs.end());
}
m_comm->AllReduce(elmtnums,LibUtilities::ReduceMax);
// Collate per-process element lists on root process to generate
// the info file.
if (rank == 0)
......@@ -1260,8 +1277,8 @@ namespace Nektar
std::vector<std::string> filenames;
for(int i = 0; i < nprocs; ++i)
{
boost::format pad("P%1$07d.fld");
pad % i;
boost::format pad("P%1$07d.%2$s");
pad % i % GetFileEnding();
filenames.push_back(pad.str());
}
......@@ -1279,10 +1296,9 @@ namespace Nektar
m_comm->Send(0, idlist);
}
// Return the full path to the partition for this process
return LibUtilities::PortablePath(fulloutname);
}
/**
*
*/
......
......@@ -180,7 +180,7 @@ public:
std::vector<std::vector<unsigned int> > &elementList,
const FieldMetaDataMap &fieldinfomap = NullFieldMetaDataMap);
private:
protected:
/// Communicator to use when writing parallel format
LibUtilities::CommSharedPtr m_comm;
/// True if same filesystem accessible by all processes.
......@@ -192,7 +192,9 @@ private:
LIB_UTILITIES_EXPORT void GenerateSeqString(
const std::vector<unsigned int> &elmtids, std::string &idString);
LIB_UTILITIES_EXPORT std::string SetUpOutput(
LIB_UTILITIES_EXPORT std::string SetUpOutput(const std::string outname);
LIB_UTILITIES_EXPORT void SetUpFieldMetaData(
const std::string outname,
const std::vector<FieldDefinitionsSharedPtr> &fielddefs,
const FieldMetaDataMap &fieldmetadatamap);
......@@ -205,6 +207,12 @@ private:
LIB_UTILITIES_EXPORT int CheckFieldDefinition(
const FieldDefinitionsSharedPtr &fielddefs);
LIB_UTILITIES_EXPORT virtual std::string GetFileEnding() const
{
return "fld";
};
};
typedef boost::shared_ptr<FieldIO> FieldIOSharedPtr;
......
......@@ -142,15 +142,16 @@ void PtsIO::Import(const string &inFile, PtsFieldSharedPtr &ptsField, FieldMetaD
std::vector<std::vector<unsigned int> > elementIDs_OnPartitions;
ImportMultiFldFileIDs(infile,filenames, fieldmetadatamap);
ImportMultiFldFileIDs(infile,filenames, elementIDs_OnPartitions,
fieldmetadatamap);
// Load metadata
ImportFieldMetaData(infile, fieldmetadatamap);
//HACK: only load the filename matching our rank.
//TODO: This currently only loads the filename matching our rank.
filenames.clear();
boost::format pad("P%1$07d.pts");
pad % m_comm->GetRank();
boost::format pad("P%1$07d.%2$s");
pad % m_comm->GetRank() % GetFileEnding();
filenames.push_back(pad.str());
for(int i = 0; i < filenames.size(); ++i)
......@@ -201,8 +202,7 @@ void PtsIO::Write(const string &outFile,
int np = ptsField->GetNpoints();
std::string filename = SetUpOutput(outFile);
cout << "writing to " << filename << endl;
SetUpFieldMetaData(outFile);
// until tinyxml gains support for line break, write the xml manually
std::ofstream ptsFile;
......@@ -233,6 +233,8 @@ void PtsIO::Write(const string &outFile,
ptsFile.close();
// this is what the above cpart would read if tinyxml
// supported line breaks
/*
// Create the file (partition)
TiXmlDocument doc;
......@@ -269,312 +271,6 @@ void PtsIO::Write(const string &outFile,
*/
}
std::string PtsIO::SetUpOutput(const std::string outname)
{
ASSERTL0(!outname.empty(), "Empty path given to SetUpOutput()");
int nprocs = m_comm->GetSize();
int rank = m_comm->GetRank();
// Directory name if in parallel, regular filename if in serial
fs::path specPath(outname);
// Remove any existing file which is in the way
if (m_comm->RemoveExistingFiles())
{
try
{
fs::remove_all(specPath);
}
catch (fs::filesystem_error &e)
{
ASSERTL0(e.code().value() == berrc::no_such_file_or_directory,
"Filesystem error: " + string(e.what()));
}
}
m_comm->Block();
// serial processing just add ending.
if (nprocs == 1)
{
cout << "Writing: " << specPath << endl;
return LibUtilities::PortablePath(specPath);
}
// Collate per-process element lists on root process to generate
// the info file.
if (rank == 0)
{
// Create the destination directory
try
{
fs::create_directory(specPath);
}
catch (fs::filesystem_error &e)
{
ASSERTL0(false, "Filesystem error: " + string(e.what()));
}
// Set up output names
std::vector<std::string> filenames;
for (int i = 0; i < nprocs; ++i)
{
boost::format pad("P%1$07d.pts");
pad % i;
filenames.push_back(pad.str());
}
// Write the Info.xml file
string infofile = LibUtilities::PortablePath(
specPath / fs::path("Info.xml"));
cout << "Writing: " << specPath << endl;
WriteMultiFldFileIDs(infofile, filenames);
}
m_comm->Block();
// Pad rank to 8char filenames, e.g. P0000000.pts
boost::format pad("P%1$07d.pts");
pad % m_comm->GetRank();
// Generate full path name
fs::path poutfile(pad.str());
fs::path fulloutname = specPath / poutfile;
// Return the full path to the partition for this process
return LibUtilities::PortablePath(fulloutname);
}
// TODO: copied from FieldIO, unify
void PtsIO::WriteMultiFldFileIDs(const std::string &outFile,
const std::vector<std::string> fileNames,
const PtsMetaDataMap &fieldmetadatamap)
{
TiXmlDocument doc;
TiXmlDeclaration *decl = new TiXmlDeclaration("1.0", "utf-8", "");
doc.LinkEndChild(decl);
TiXmlElement *root = new TiXmlElement("NEKTAR");
doc.LinkEndChild(root);
AddInfoTag(root, fieldmetadatamap);
for (int t = 0; t < fileNames.size(); ++t)
{
TiXmlElement *elemIDs = new TiXmlElement("Partition");
root->LinkEndChild(elemIDs);
elemIDs->SetAttribute("FileName", fileNames[t]);
}
doc.SaveFile(outFile);
}
void PtsIO::ImportMultiFldFileIDs(const std::string &inFile,
std::vector<std::string> &fileNames,
FieldMetaDataMap &fieldmetadatamap)
{
TiXmlDocument doc(inFile);
bool loadOkay = doc.LoadFile();
std::stringstream errstr;
errstr << "Unable to load file: " << inFile<< std::endl;
errstr << "Reason: " << doc.ErrorDesc() << std::endl;
errstr << "Position: Line " << doc.ErrorRow() << ", Column " << doc.ErrorCol() << std::endl;
ASSERTL0(loadOkay, errstr.str());
// Handle on XML document
TiXmlHandle docHandle(&doc);
// Retrieve main NEKTAR tag - XML specification states one
// top-level element tag per file.
TiXmlElement* master = doc.FirstChildElement("NEKTAR");
ASSERTL0(master, "Unable to find NEKTAR tag in file.");
// Partition element tag name
std::string strPartition = "Partition";
// First attempt to get the first Partition element
TiXmlElement* fldfileIDs = master->FirstChildElement(strPartition.c_str());
if (!fldfileIDs)
{
// If this files try previous name
strPartition = "MultipleFldFiles";
fldfileIDs = master->FirstChildElement("MultipleFldFiles");
}
ASSERTL0(fldfileIDs,
"Unable to find 'Partition' or 'MultipleFldFiles' tag "
"within nektar tag.");
while (fldfileIDs)
{
// Read file name of partition file
const char *attr = fldfileIDs->Attribute("FileName");
ASSERTL0(attr, "'FileName' not provided as an attribute of '"
+ strPartition + "' tag.");
fileNames.push_back(std::string(attr));
// const char* elementIDs = fldfileIDs->GetText();
// ASSERTL0(elementIDs, "Element IDs not specified.");
//
// std::string elementIDsStr(elementIDs);
//
// std::vector<unsigned int> idvec;
// ParseUtils::GenerateSeqVector(elementIDsStr.c_str(),idvec);
//
// elementList.push_back(idvec);
fldfileIDs = fldfileIDs->NextSiblingElement(strPartition.c_str());
}
}
void PtsIO::ImportFieldMetaData(std::string filename,
FieldMetaDataMap &fieldmetadatamap)
{
TiXmlDocument doc(filename);
bool loadOkay = doc.LoadFile();
std::stringstream errstr;
errstr << "Unable to load file: " << filename << std::endl;
errstr << "Reason: " << doc.ErrorDesc() << std::endl;
errstr << "Position: Line " << doc.ErrorRow() << ", Column " << doc.ErrorCol() << std::endl;
ASSERTL0(loadOkay, errstr.str());
ImportFieldMetaData(doc,fieldmetadatamap);
}
void PtsIO::ImportFieldMetaData(TiXmlDocument &doc,
FieldMetaDataMap &fieldmetadatamap)
{
TiXmlHandle docHandle(&doc);
TiXmlElement* master = 0; // Master tag within which all data is contained.
TiXmlElement* metadata = 0;
master = doc.FirstChildElement("NEKTAR");
ASSERTL0(master, "Unable to find NEKTAR tag in file.");
std::string strLoop = "NEKTAR";
// Retain original metadata structure for backwards compatibility
// TODO: Remove old metadata format
metadata = master->FirstChildElement("FIELDMETADATA");
if(metadata)
{
TiXmlElement *param = metadata->FirstChildElement("P");
while (param)
{
TiXmlAttribute *paramAttr = param->FirstAttribute();
std::string attrName(paramAttr->Name());
std::string paramString;
if(attrName == "PARAM")
{
paramString.insert(0,paramAttr->Value());
}
else
{
ASSERTL0(false,"PARAM not provided as an attribute in FIELDMETADATA section");
}
// Now read body of param
std::string paramBodyStr;
TiXmlNode *paramBody = param->FirstChild();
paramBodyStr += paramBody->ToText()->Value();
fieldmetadatamap[paramString] = paramBodyStr;
param = param->NextSiblingElement("P");
}
}
// New metadata format
metadata = master->FirstChildElement("Metadata");
if(metadata)
{
TiXmlElement *param = metadata->FirstChildElement();
while (param)
{
std::string paramString = param->Value();
if (paramString != "Provenance")
{
// Now read body of param
TiXmlNode *paramBody = param->FirstChild();
std::string paramBodyStr = paramBody->ToText()->Value();
fieldmetadatamap[paramString] = paramBodyStr;
}
param = param->NextSiblingElement();
}
}
}
// TODO: copied from FieldIO, unify
/**
* \brief add information about provenance and fieldmetadata
*/
void PtsIO::AddInfoTag(TiXmlElement *root,
const PtsMetaDataMap &fieldmetadatamap)
{
PtsMetaDataMap ProvenanceMap;
// Nektar++ release version from VERSION file
ProvenanceMap["NektarVersion"] = string(NEKTAR_VERSION);
// Date/time stamp
ptime::time_facet *facet = new ptime::time_facet("%d-%b-%Y %H:%M:%S");
std::stringstream wss;
wss.imbue(locale(wss.getloc(), facet));
wss << ptime::second_clock::local_time();
ProvenanceMap["Timestamp"] = wss.str();
// Hostname
boost::system::error_code ec;
ProvenanceMap["Hostname"] = ip::host_name(ec);
// Git information
// If built from a distributed package, do not include this
if (NekConstants::kGitSha1 != "GITDIR-NOTFOUND")
{
ProvenanceMap["GitSHA1"] = NekConstants::kGitSha1;
ProvenanceMap["GitBranch"] = NekConstants::kGitBranch;
}
TiXmlElement *infoTag = new TiXmlElement("Metadata");
root->LinkEndChild(infoTag);
TiXmlElement *v;
PtsMetaDataMap::const_iterator infoit;
TiXmlElement *provTag = new TiXmlElement("Provenance");
infoTag->LinkEndChild(provTag);
for (infoit = ProvenanceMap.begin(); infoit != ProvenanceMap.end(); ++infoit)
{
v = new TiXmlElement((infoit->first).c_str());
v->LinkEndChild(new TiXmlText((infoit->second).c_str()));
provTag->LinkEndChild(v);
}
//---------------------------------------------
// write field info section
if (fieldmetadatamap != NullPtsMetaDataMap)
{
for (infoit = fieldmetadatamap.begin(); infoit != fieldmetadatamap.end(); ++infoit)
{
v = new TiXmlElement((infoit->first).c_str());
v->LinkEndChild(new TiXmlText((infoit->second).c_str()));
infoTag->LinkEndChild(v);
}
}
}
void PtsIO::ImportFieldData(TiXmlDocument docInput, PtsFieldSharedPtr &ptsField)
{
......@@ -637,7 +333,45 @@ void PtsIO::ImportFieldData(TiXmlDocument docInput, PtsFieldSharedPtr &ptsField)
}
ptsField = MemoryManager<PtsField>::AllocateSharedPtr(dim, fieldNames, pts);
}
void PtsIO::SetUpFieldMetaData(const string outname)
{
ASSERTL0(!outname.empty(), "Empty path given to SetUpFieldMetaData()");
int nprocs = m_comm->GetSize();
int rank = m_comm->GetRank();
fs::path specPath (outname);
// Collate per-process element lists on root process to generate
// the info file.
if (rank == 0)
{
// Set up output names
std::vector<std::string> filenames;
std::vector<std::vector<unsigned int> > ElementIDs;
for(int i = 0; i < nprocs; ++i)
{
boost::format pad("P%1$07d.%2$s");
pad % i % GetFileEnding();
filenames.push_back(pad.str());
std::vector<unsigned int> tmp;
tmp.push_back(0);
ElementIDs.push_back(tmp);
}
// Write the Info.xml file
string infofile = LibUtilities::PortablePath(
specPath / fs::path("Info.xml"));
cout << "Writing: " << specPath << endl;
const FieldMetaDataMap fieldmetadatamap;
WriteMultiFldFileIDs(infofile, filenames, ElementIDs, fieldmetadatamap);
}
}
......
......@@ -68,55 +68,37 @@ LIB_UTILITIES_EXPORT void Import(const string &inFile,
LIB_UTILITIES_EXPORT void Write(const string &outFile,
const PtsFieldSharedPtr &ptsField);
class PtsIO
class PtsIO : protected FieldIO
{
public:
LIB_UTILITIES_EXPORT PtsIO(LibUtilities::CommSharedPtr pComm):
m_comm(pComm)
LIB_UTILITIES_EXPORT PtsIO(LibUtilities::CommSharedPtr pComm,
bool sharedFilesystem = false):
FieldIO(pComm, sharedFilesystem)
{
};
LIB_UTILITIES_EXPORT void Import(const string &inFile, Nektar::LibUtilities::PtsFieldSharedPtr &ptsField, Nektar::LibUtilities::FieldMetaDataMap &fieldmetadatamap = NullFieldMetaDataMap);
LIB_UTILITIES_EXPORT void Import(
const string &inFile,
PtsFieldSharedPtr &ptsField,
FieldMetaDataMap &fieldmetadatamap = NullFieldMetaDataMap);
LIB_UTILITIES_EXPORT void Write(const string &outFile,
const Nektar::LibUtilities::PtsFieldSharedPtr &ptsField);
LIB_UTILITIES_EXPORT void WriteMultiFldFileIDs(
const std::string &outfile,
const std::vector<std::string> fileNames,
const PtsMetaDataMap &fieldinfomap = NullPtsMetaDataMap);
/// Imports the definition of the meta data
LIB_UTILITIES_EXPORT void ImportFieldMetaData(
std::string filename,
FieldMetaDataMap &fieldmetadatamap);
/// Imports the definition of the meta data
LIB_UTILITIES_EXPORT void ImportFieldMetaData(
TiXmlDocument &doc,
FieldMetaDataMap &fieldmetadatamap);
const PtsFieldSharedPtr &ptsField);
LIB_UTILITIES_EXPORT void ImportFieldData(
TiXmlDocument docInput, Nektar::LibUtilities::PtsFieldSharedPtr &ptsField);
private:
LibUtilities::CommSharedPtr m_comm;
LIB_UTILITIES_EXPORT std::string SetUpOutput(const string outname);