Commit 65cf8cad authored by Dave Moxey's avatar Dave Moxey
Browse files

Add fairly extensive documentation to FieldIO classes

parent 250dd23e
......@@ -62,6 +62,10 @@ namespace Nektar
{
namespace LibUtilities
{
/**
* @brief Returns the FieldIO factory.
*/
FieldIOFactory &GetFieldIOFactory()
{
typedef Loki::
......@@ -176,6 +180,12 @@ FieldIOSharedPtr MakeFieldIOForFile(
* @brief This function allows for data to be written to an FLD file when a
* session and/or communicator is not instantiated. Typically used in utilities
* which do not take XML input and operate in serial only.
*
* @param outFile Output filename
* @param fielddefs Field definitions that define the output
* @param fielddata Binary field data that stores the output corresponding
* to @p fielddefs.
* @param fieldinfomap Associated field metadata map.
*/
void Write(const std::string &outFile,
std::vector<FieldDefinitionsSharedPtr> &fielddefs,
......@@ -209,13 +219,23 @@ void Write(const std::string &outFile,
* @brief This function allows for data to be imported from an FLD file when a
* session and/or communicator is not instantiated. Typically used in utilities
* which only operate in serial.
*
* @param infilename Input filename (or directory if parallel format)
* @param fielddefs On return contains field definitions as read from the
* input.
* @param fielddata On return, contains binary field data that stores the
* input corresponding to @p fielddefs.
* @param fieldinfo On returnm, contains the associated field metadata map.
* @param ElementIDs Element IDs that lie on this processor, which can be
* optionally supplied to avoid reading the entire file on
* each processor.
*/
LIB_UTILITIES_EXPORT void Import(
const std::string &infilename,
std::vector<FieldDefinitionsSharedPtr> &fielddefs,
std::vector<std::vector<NekDouble> > &fielddata,
FieldMetaDataMap &fieldinfomap,
const Array<OneD, int> ElementiDs)
const Array<OneD, int> ElementIDs)
{
#ifdef NEKTAR_USE_MPI
int size;
......@@ -237,7 +257,7 @@ LIB_UTILITIES_EXPORT void Import(
#endif
CommSharedPtr c = GetCommFactory().CreateInstance("Serial", 0, 0);
FieldIOSharedPtr f = GetFieldIOFactory().CreateInstance("Xml", c, false);
f->Import(infilename, fielddefs, fielddata, fieldinfomap, ElementiDs);
f->Import(infilename, fielddefs, fielddata, fieldinfomap, ElementIDs);
}
/**
......@@ -248,6 +268,22 @@ FieldIO::FieldIO(LibUtilities::CommSharedPtr pComm, bool sharedFilesystem)
{
}
/**
* @brief Add provenance information to the field metadata map.
*
* This routine adds some basic provenance information to the field metadata to
* enable better tracking of version information:
*
* - Nektar++ version
* - Date and time of simulation
* - Hostname of the machine the simulation was performed on
* - git SHA1 and branch name, if Nektar++ was compiled from git and not
* e.g. a tarball.
*
* @param root Root tag, which is encapsulated using the TagWriter
* structure to enable multi-file format support.
* @param fieldmetadatamap Any existing field metadata.
*/
void FieldIO::AddInfoTag(TagWriterSharedPtr root,
const FieldMetaDataMap &fieldmetadatamap)
{
......@@ -300,7 +336,121 @@ void FieldIO::AddInfoTag(TagWriterSharedPtr root,
}
/**
* @brief Set up the filesystem ready for output.
*
* This function sets up the output given an output filename @p outname. This
* will therefore remove any file or directory with the desired output filename
* and return the absolute path to the output.
*
* If @p perRank is set, a new directory will be created to contain one file per
* process rank.
*
* @param outname Desired output filename.
* @param perRank True if one file-per-rank output is required.
*
* @return Absolute path to resulting file.
*/
std::string FieldIOXml::SetUpOutput(const std::string outname, bool perRank)
{
ASSERTL0(!outname.empty(), "Empty path given to SetUpOutput()");
int nprocs = m_comm->GetSize();
int rank = m_comm->GetRank();
// Path to output: will be directory if parallel, normal file if
// serial.
fs::path specPath(outname), fulloutname;
if (nprocs == 1)
{
fulloutname = specPath;
}
else
{
// Guess at filename that might belong to this process.
boost::format pad("P%1$07d.%2$s");
pad % m_comm->GetRank() % GetFileEnding();
// Generate full path name
fs::path poutfile(pad.str());
fulloutname = specPath / poutfile;
}
// Remove any existing file which is in the way
if (m_comm->RemoveExistingFiles())
{
if (m_sharedFilesystem)
{
// First, each process clears up its .fld file. This might or might
// not be there (we might have changed numbers of processors between
// runs, for example), but we can try anyway.
try
{
fs::remove_all(fulloutname);
}
catch (fs::filesystem_error &e)
{
ASSERTL0(e.code().value() == berrc::no_such_file_or_directory,
"Filesystem error: " + string(e.what()));
}
}
m_comm->Block();
// Now get rank 0 processor to tidy everything else up.
if (rank == 0 || !m_sharedFilesystem)
{
try
{
fs::remove_all(specPath);
}
catch (fs::filesystem_error &e)
{
ASSERTL0(e.code().value() == berrc::no_such_file_or_directory,
"Filesystem error: " + string(e.what()));
}
}
m_comm->Block();
}
// serial processing just add ending.
if (nprocs == 1 || rank == 0)
{
cout << "Writing: " << specPath << endl;
return LibUtilities::PortablePath(specPath);
}
// Create the destination directory
if (perRank)
{
try
{
if (rank == 0 || !m_sharedFilesystem)
{
fs::create_directory(specPath);
}
}
catch (fs::filesystem_error &e)
{
ASSERTL0(false, "Filesystem error: " + string(e.what()));
}
m_comm->Block();
}
else
{
fulloutname = specPath;
}
// Return the full path to the partition for this process
return LibUtilities::PortablePath(fulloutname);
}
/**
* @brief Check field definitions for correctness and return storage size.
*
* @param fielddefs Field definitions to check.
*/
int FieldIO::CheckFieldDefinition(const FieldDefinitionsSharedPtr &fielddefs)
{
......
......@@ -190,7 +190,7 @@ LIB_UTILITIES_EXPORT void Import(
std::vector<FieldDefinitionsSharedPtr> &fielddefs,
std::vector<std::vector<NekDouble> > &fielddata = NullVectorNekDoubleVector,
FieldMetaDataMap &fieldinfomap = NullFieldMetaDataMap,
const Array<OneD, int> ElementiDs = NullInt1DArray);
const Array<OneD, int> ElementIDs = NullInt1DArray);
// Forward declare
class FieldIO;
......@@ -203,7 +203,23 @@ typedef LibUtilities::NekFactory<std::string,
LIB_UTILITIES_EXPORT FieldIOFactory &GetFieldIOFactory();
/// Class for operating on FLD files
/**
* @brief Class for operating on Nektar++ input/output files.
*
* Nektar++ input/output of field data can be described as follows:
*
* - The FieldDefinitions class defines the metadata that is associated with
* one or more scalar field variables, and determines the storage length.
* - Each scalar field is stored in a single contiguous vector which is
* written/read by the functions defined in this class.
* - Optional metadata can be read/written in a simple key/value pair map
* FieldMetaDataMap. This can be used to define, e.g. the timestep of the
* simulation.
*
* This base class represents the minimum functionality that subclasses need to
* implement in order to implement the above functionality. Each subclass is
* free to determine its own file structure and parallel behaviour.
*/
class FieldIO : public boost::enable_shared_from_this<FieldIO>
{
public:
......@@ -222,7 +238,7 @@ public:
std::vector<std::vector<NekDouble> > &fielddata =
NullVectorNekDoubleVector,
FieldMetaDataMap &fieldinfomap = NullFieldMetaDataMap,
const Array<OneD, int> ElementiDs = NullInt1DArray);
const Array<OneD, int> ElementIDs = NullInt1DArray);
LIB_UTILITIES_EXPORT DataSourceSharedPtr ImportFieldMetaData(
std::string filename,
......@@ -245,31 +261,50 @@ protected:
LIB_UTILITIES_EXPORT int CheckFieldDefinition(
const FieldDefinitionsSharedPtr &fielddefs);
/**
* @brief Helper function that determines default file extension.
*/
LIB_UTILITIES_EXPORT virtual std::string GetFileEnding() const
{
return "fld";
}
LIB_UTILITIES_EXPORT std::string SetUpOutput(const std::string outname);
/// @copydoc FieldIO::Write
LIB_UTILITIES_EXPORT virtual void v_Write(
const std::string &outFile,
std::vector<FieldDefinitionsSharedPtr> &fielddefs,
std::vector<std::vector<NekDouble> > &fielddata,
const FieldMetaDataMap &fieldinfomap) = 0;
/// @copydoc FieldIO::Import
LIB_UTILITIES_EXPORT virtual void v_Import(
const std::string &infilename,
std::vector<FieldDefinitionsSharedPtr> &fielddefs,
std::vector<std::vector<NekDouble> >
&fielddata = NullVectorNekDoubleVector,
FieldMetaDataMap &fieldinfomap = NullFieldMetaDataMap,
const Array<OneD, int> ElementiDs = NullInt1DArray) = 0;
const Array<OneD, int> ElementIDs = NullInt1DArray) = 0;
/// @copydoc FieldIO::ImportFieldMetaData
LIB_UTILITIES_EXPORT virtual DataSourceSharedPtr v_ImportFieldMetaData(
std::string filename, FieldMetaDataMap &fieldmetadatamap) = 0;
};
typedef boost::shared_ptr<FieldIO> FieldIOSharedPtr;
/**
* @brief Returns an object for the default FieldIO method.
*
* This function returns a FieldIO class as determined by the hard-coded default
* (XML), which can be overridden by changing the session reader SOLVERINFO
* variable FieldIOFormat.
*
* @param session Session reader
*
* @return FieldIO object
*/
inline FieldIOSharedPtr MakeDefaultFieldIO(
const LibUtilities::SessionReaderSharedPtr session)
{
......@@ -284,11 +319,19 @@ inline FieldIOSharedPtr MakeDefaultFieldIO(
session->DefinesCmdLineArgument("shared-filesystem"));
}
// Collective on session's communicator
FieldIOSharedPtr MakeFieldIOForFile(
const LibUtilities::SessionReaderSharedPtr session,
const std::string &filename);
/**
* @brief Write out the field information to the file @p outFile.
*
* @param outFile Output filename
* @param fielddefs Field definitions that define the output
* @param fielddata Binary field data that stores the output corresponding
* to @p fielddefs.
* @param fieldinfomap Associated field metadata map.
*/
inline void FieldIO::Write(const std::string &outFile,
std::vector<FieldDefinitionsSharedPtr> &fielddefs,
std::vector<std::vector<NekDouble> > &fielddata,
......@@ -297,15 +340,35 @@ inline void FieldIO::Write(const std::string &outFile,
v_Write(outFile, fielddefs, fielddata, fieldinfomap);
}
/**
* @brief Read field information from the file @p infilename.
*
* @param infilename Input filename (or directory if parallel format)
* @param fielddefs On return contains field definitions as read from the
* input.
* @param fielddata On return, contains binary field data that stores the
* input corresponding to @p fielddefs.
* @param fieldinfo On returnm, contains the associated field metadata map.
* @param ElementIDs Element IDs that lie on this processor, which can be
* optionally supplied to avoid reading the entire file on
* each processor.
*/
inline void FieldIO::Import(const std::string &infilename,
std::vector<FieldDefinitionsSharedPtr> &fielddefs,
std::vector<std::vector<NekDouble> > &fielddata,
FieldMetaDataMap &fieldinfo,
const Array<OneD, int> ElementiDs)
const Array<OneD, int> ElementIDs)
{
v_Import(infilename, fielddefs, fielddata, fieldinfo, ElementiDs);
v_Import(infilename, fielddefs, fielddata, fieldinfo, ElementIDs);
}
/**
* @brief Import the metadata from a field file.
*
* @param filename Input filename.
* @param fieldmetadatamap On return contains the field metadata map from @p
* filename.
*/
inline DataSourceSharedPtr FieldIO::ImportFieldMetaData(
std::string filename, FieldMetaDataMap &fieldmetadatamap)
{
......
......@@ -90,7 +90,6 @@ void FieldIOHdf5::v_Write(const std::string &outFile,
}
// We make a number of assumptions in this code:
//
// 1. All element ids have the same type: unsigned int
// 2. All elements within a given field have the same number of values
// 3. All element values have the same type, NekDouble
......@@ -131,8 +130,8 @@ void FieldIOHdf5::v_Write(const std::string &outFile,
homoYIDs(nFields), homoZIDs(nFields);
std::vector<std::string> numModesPerDirs(nFields);
// calculate the total number of elements handled by this MPI process and
// the total number of bytes required to store the elements, base the name
// Calculate the total number of elements handled by this MPI process and
// the total number of bytes required to store the elements. Base the name
// of each field on the hash of the field definition.
for (int f = 0; f < nFields; ++f)
{
......@@ -704,6 +703,14 @@ void FieldIOHdf5::v_Import(const std::string &infilename,
m_comm->Block();
}
/**
* @brief Import field definitions from a HDF5 document.
*
* @param readPL Reading parameter list.
* @param root Root group containing field definitions.
* @param group Group name to process.
* @param def On output contains field definitions.
*/
void FieldIOHdf5::ImportFieldDef(
H5::PListSharedPtr readPL,
H5::GroupSharedPtr root,
......@@ -876,6 +883,18 @@ void FieldIOHdf5::ImportFieldDef(
}
}
/**
* @brief Import the field data from the HDF5 document.
*
* @param readPL Reading parameter list.
* @param data_dset Pointer to the `DATA` dataset.
* @param data_fspace Pointer to the `DATA` data space.
* @param data_i ...
* @param decomps Information from the `DECOMPOSITION` dataset.
* @param decomp ...
* @param fielddef Field definitions for this file
* @param fielddata On return contains resulting field data.
*/
void FieldIOHdf5::ImportFieldData(
H5::PListSharedPtr readPL,
H5::DataSetSharedPtr data_dset,
......@@ -903,14 +922,6 @@ void FieldIOHdf5::ImportFieldData(
"input data is not the same length as header information.");
}
DataSourceSharedPtr FieldIOHdf5::v_ImportFieldMetaData(
std::string filename, FieldMetaDataMap &fieldmetadatamap)
{
DataSourceSharedPtr ans = H5DataSource::create(filename);
v_ImportFieldMetaData(ans, fieldmetadatamap);
return ans;
}
void FieldIOHdf5::v_ImportFieldMetaData(DataSourceSharedPtr dataSource,
FieldMetaDataMap &fieldmetadatamap)
{
......
......@@ -51,55 +51,108 @@ class Group;
typedef boost::shared_ptr<Group> GroupSharedPtr;
}
/**
* @class Class encapsulating simple HDF5 data source using H5 reader utilities.
*/
class H5DataSource : public DataSource
{
H5::FileSharedPtr doc;
public:
/// Constructor based on filename.
H5DataSource(const std::string &fn)
: doc(H5::File::Open(fn, H5F_ACC_RDONLY))
{
}
/// Get H5::FileSharedPtr reference to file.
H5::FileSharedPtr Get()
{
return doc;
}
/// Get H5::FileSharedPtr reference to file.
const H5::FileSharedPtr Get() const
{
return doc;
}
/// Static constructor for this data source.
static DataSourceSharedPtr create(const std::string &fn)
{
return DataSourceSharedPtr(new H5DataSource(fn));
}
private:
/// HDF5 document.
H5::FileSharedPtr doc;
};
typedef boost::shared_ptr<H5DataSource> H5DataSourceSharedPtr;
/**
* @class Simple class for writing hierarchical data using HDF5.
*/
class H5TagWriter : public TagWriter
{
public:
/// Default constructor.
H5TagWriter(H5::GroupSharedPtr grp) : m_Group(grp) {}
/// Add a child node.
TagWriterSharedPtr AddChild(const std::string &name)
{
H5::GroupSharedPtr child = m_Group->CreateGroup(name);
return TagWriterSharedPtr(new H5TagWriter(child));
}
/// Set an attribute key/value pair on this tag.
void SetAttr(const std::string &key, const std::string &val)
{
m_Group->SetAttribute(key, val);
}
private:
/// HDF5 group for this tag.
H5::GroupSharedPtr m_Group;
};
typedef boost::shared_ptr<H5TagWriter> H5TagWriterSharedPtr;
/**
* @class Class for operating on HDF5-based FLD files.
*
* This class implements a HDF5 reader/writer based on MPI/O that is designed to
* operate on a single file across all processors of a simulation. The
* definition follows vaguely similar lines to XML output but is stored somewhat
* differently. At a basic level metadata is organised as follows:
*
* - Nektar++ data lies in the root `/NEKTAR` group.
* - The contents of a FieldDefinitions object is hashed to construct a unique
* identifier for each object, which becomes the name of a group within the
* root group. We then use the H5TagWriter to assign the field definitions
* to each group.
* - In a similar fashion, we create a `Metadata` group to contain field
* metadata that is written
*
* We then define two data sets to contain field data:
*
* - The `DATA` dataset contains the double-precision modal coefficient data.
* - The `IDS` dataset contains the element IDs of the elements that are
* written out.
*
* The ordering is defined according to:
*
* - The `INDEXES` dataset, of size NPROCS * 2 contains the following
* information per processor:
* - Offset of the start of this block's element IDs
* (FieldIOHdf5::IDS_IDX_IDX)
* - Offset of the start of this block in the data array
* (FieldIOHdf5::DATA_IDX_IDX)
* - The `DECOMPOSITION` dataset contains the following three integers of
* information per field definition per processor:
* - Number of elements in this field definition
* (FieldIOHdf5::ELEM_DCMP_IDX)
* - Number of entries in the data array for this field definition
* (FieldIOHdf5::VAL_DCMP_IDX)
* - Hash of the field definition that these entries belong inside
* (FieldIOHdf5::HASH_DCMP_IDX).
*/
class FieldIOHdf5 : public FieldIO
{
......@@ -108,9 +161,11 @@ public:
static const unsigned int VAL_DCMP_IDX;
static const unsigned int HASH_DCMP_IDX;
static const unsigned int MAX_DCMPS;
static const unsigned int ELEM_CNT_IDX;
static const unsigned int VAL_CNT_IDX;
static const unsigned int MAX_CNTS;
static const unsigned int IDS_IDX_IDX;
static const unsigned int DATA_IDX_IDX;
static const unsigned int MAX_IDXS;
......@@ -128,13 +183,13 @@ public:
FieldIOHdf5(LibUtilities::CommSharedPtr pComm, bool sharedFilesystem);
/// Get class name
inline virtual const std::string &GetClassName() const
{
return className;
}
private:
/// Write data in FLD format
LIB_UTILITIES_EXPORT virtual void v_Write(
const std::string &outFile,
std::vector<FieldDefinitionsSharedPtr> &fielddefs,
......@@ -148,18 +203,14 @@ private:
FieldMetaDataMap &fieldinfomap = NullFieldMetaDataMap,
const Array<OneD, int> ElementiDs = NullInt1DArray);
LIB_UTILITIES_EXPORT virtual DataSourceSharedPtr v_ImportFieldMetaData(
std::string filename, FieldMetaDataMap &fieldmetadatamap);
LIB_UTILITIES_EXPORT void v_ImportFieldMetaData(
DataSourceSharedPtr dataSource, FieldMetaDataMap &fieldmetadatamap);
/// Imports the field definitions.
LIB_UTILITIES_EXPORT void ImportFieldDef(H5::PListSharedPtr readPL,
H5::GroupSharedPtr root,
std::string group,
FieldDefinitionsSharedPtr def);
/// Imports the data fields.
LIB_UTILITIES_EXPORT void ImportFieldData(
H5::PListSharedPtr readPL,
H5::DataSetSharedPtr data_dset,
......
......@@ -52,14 +52,18 @@ namespace LibUtilities
std::string FieldIOXml::className = GetFieldIOFactory().RegisterCreatorFunction(
"Xml", FieldIOXml::create, "XML-based output of field data.");
/**
* @brief Default constructor.
*
* @param pComm Communicator.
* @param sharedFilesystem True if the underlying filesystem is shared by the
* compute nodes.
*/
FieldIOXml::FieldIOXml(LibUtilities::CommSharedPtr pComm, bool sharedFilesystem)
: FieldIO(pComm, sharedFilesystem)
{
}
/**
*
*/
void FieldIOXml::v_Write(const std::string &outFile,
std::vector<FieldDefinitionsSharedPtr> &fielddefs,
std::vector<std::vector<NekDouble> > &fielddata,
......@@ -91,7 +95,7 @@ void FieldIOXml::v_Write(const std::string &outFile,
// Prepare to write out data. In parallel, we must create directory and
// determine the full pathname to the file to write out. Any existing
// file/directory which is in the way is removed.
std::string filename = SetUpOutput(outFile);
std::string filename = SetUpOutput(outFile, true);
SetUpFieldMetaData(outFile, fielddefs, fieldmetadatamap);
// Create the file (partition)
......@@ -332,13 +336,22 @@ void FieldIOXml::v_Write(const std::string &outFile,
}
/**
* @brief Write out a file containing element ID to partition mapping.
*
* This function writes out an XML file (usually called `Info.xml`) that
* contains the element IDs that are contained within each partition, as well as
* the field metadata map.
*
* @param outFile Output multi-field file name.
* @param fileNames List of partition filenames.