Commit 336552bf authored by Michael Bareford's avatar Michael Bareford

Overhauled FieldIOHdf5 write/import routines.

parent 25364757
......@@ -91,6 +91,7 @@ struct Experiment
CommSharedPtr comm;
};
IOSettingsSharedPtr hdf5_settings(new IOSettings);
IOSettingsSharedPtr sionlib_settings(new IOSettings);
typedef std::vector<double> Results;
......@@ -114,6 +115,8 @@ int main(int argc, char *argv[])
exp.n = 3;
exp.comm = GetCommFactory().CreateInstance("ParallelMPI", argc, argv);
hdf5_settings->insert( IOSettings::value_type("Reformatting", "1") );
sionlib_settings->insert( IOSettings::value_type("IOBlockSize", "65536") );
sionlib_settings->insert( IOSettings::value_type("IOBlocksPerChunk", SIONlib_GetIOBlocksPerChunk(exp.comm->GetSize(), AORTIC_ARCH)) );
sionlib_settings->insert( IOSettings::value_type("IOReadMode", "br,collective,collsize=-1") );
......@@ -431,6 +434,10 @@ Results TestRead(Experiment &exp)
{
fio->InitFromBenchmarker(sionlib_settings);
}
else if (exp.hdf)
{
fio->InitFromBenchmarker(hdf5_settings);
}
Array<OneD, int> ElementIDs = ReadIDsForThisRank(exp);
......@@ -492,6 +499,10 @@ Results TestWrite(Experiment &exp)
{
fio->InitFromBenchmarker(sionlib_settings);
}
else if (exp.hdf)
{
fio->InitFromBenchmarker(hdf5_settings);
}
Array<OneD, int> ElementIDs = ReadIDsForThisRank(exp);
fio->Import(exp.dataSource, fielddefs, fielddata,
......
......@@ -66,79 +66,34 @@ const unsigned int FieldIOHdf5::FORMAT_VERSION = 1;
// The following definitions allow us to consistently refer to indexes pulled
// out of the various datasets.
/// A helper for FieldIOHdf5::v_Write and FieldIOHdf5::v_Import. Describes the
/// position of the number of elements in decomposition (i.e. field definition).
const unsigned int FieldIOHdf5::ELEM_DCMP_IDX = 0;
/// A helper for FieldIOHdf5::v_Write and FieldIOHdf5::v_Import. Describes the
/// position of the number of data points in decomposition (i.e. field
/// definition).
const unsigned int FieldIOHdf5::VAL_DCMP_IDX = 1;
/// A helper for FieldIOHdf5::v_Write and FieldIOHdf5::v_Import. Describes the
/// position of the number of elements multiplied by the dimension of the
/// element, giving number of modes when variable polynomial order is defined.
const unsigned int FieldIOHdf5::ORDER_DCMP_IDX = 2;
/// A helper for FieldIOHdf5::v_Write and FieldIOHdf5::v_Import. Describes the
/// position of the number of the number of y-planes for homogeneous
/// simulations.
const unsigned int FieldIOHdf5::HOMY_DCMP_IDX = 3;
/// A helper for FieldIOHdf5::v_Write and FieldIOHdf5::v_Import. Describes the
/// position of the number of the number of z-planes for homogeneous
/// simulations.
const unsigned int FieldIOHdf5::HOMZ_DCMP_IDX = 4;
/// A helper for FieldIOHdf5::v_Write and FieldIOHdf5::v_Import. Describes the
/// position of the number of the number of strips for homogeneous simulations.
const unsigned int FieldIOHdf5::HOMS_DCMP_IDX = 5;
/// The hash of the field definition information, which defines the name of the
/// attribute containing the field definition itself.
const unsigned int FieldIOHdf5::HASH_DCMP_IDX = 6;
/// A helper for FieldIOHdf5::v_Write. Describes the maximum number of items in
/// the decomposition per field definition.
const unsigned int FieldIOHdf5::MAX_DCMPS = FieldIOHdf5::HASH_DCMP_IDX + 1;
/// A helper for FieldIOHdf5::v_Write. Describes the position of the number of
/// elements in the cnt array.
const unsigned int FieldIOHdf5::ELEM_CNT_IDX = 0;
/// A helper for FieldIOHdf5::v_Write. Describes the position of the number of
/// data points in the cnt array.
const unsigned int FieldIOHdf5::VAL_CNT_IDX = 1;
/// A helper for FieldIOHdf5::v_Write. Describes the position of the number of
/// order points in the cnt array.
const unsigned int FieldIOHdf5::ORDER_CNT_IDX = 2;
/// A helper for FieldIOHdf5::v_Write. Describes the position of the number of
/// homogeneous y-planes in the cnt array.
const unsigned int FieldIOHdf5::HOMY_CNT_IDX = 3;
/// A helper for FieldIOHdf5::v_Write. Describes the position of the number of
/// homogeneous z-planes in the cnt array.
const unsigned int FieldIOHdf5::HOMZ_CNT_IDX = 4;
/// A helper for FieldIOHdf5::v_Write. Describes the position of the number of
/// homogeneous strips in the cnt array.
const unsigned int FieldIOHdf5::HOMS_CNT_IDX = 5;
/// A helper for FieldIOHdf5::v_Write. Describes the maximum number of items in
/// the cnt array per field definition.
const unsigned int FieldIOHdf5::MAX_CNTS = FieldIOHdf5::HOMS_CNT_IDX + 1;
/// A helper for FieldIOHdf5::v_Write. Describes the position of the element IDs
/// within the indexing set.
const unsigned int FieldIOHdf5::IDS_IDX_IDX = 0;
/// A helper for FieldIOHdf5::v_Write. Describes the position of the data size
/// within the indexing set.
const unsigned int FieldIOHdf5::DATA_IDX_IDX = 1;
/// A helper for FieldIOHdf5::v_Write. Describes the position of the element
/// order within the indexing set.
const unsigned int FieldIOHdf5::ORDER_IDX_IDX = 2;
/// A helper for FieldIOHdf5::v_Write. Describes the position of the number of
/// y-planes within the indexing set.
const unsigned int FieldIOHdf5::HOMY_IDX_IDX = 3;
/// A helper for FieldIOHdf5::v_Write. Describes the position of the number of
/// z-planes within the indexing set.
const unsigned int FieldIOHdf5::HOMZ_IDX_IDX = 4;
/// A helper for FieldIOHdf5::v_Write. Describes the position of the number of
/// homogeneous strips within the indexing set.
const unsigned int FieldIOHdf5::HOMS_IDX_IDX = 5;
/// A helper for FieldIOHdf5::v_Write. Describes the maximum number of items in
/// the indexing set.
const unsigned int FieldIOHdf5::MAX_IDXS = FieldIOHdf5::HOMS_IDX_IDX + 1;
const unsigned int FieldIOHdf5::FIELD_COUNT_IDS = 0;
const unsigned int FieldIOHdf5::FIELD_COUNT_DATA = 1;
const unsigned int FieldIOHdf5::FIELD_COUNT_HOMY = 2;
const unsigned int FieldIOHdf5::FIELD_COUNT_HOMZ = 3;
const unsigned int FieldIOHdf5::FIELD_COUNT_HOMS = 4;
const unsigned int FieldIOHdf5::FIELD_COUNT_ORDER = 5;
const unsigned int FieldIOHdf5::FIELD_COUNT_SIZE = 6;
const unsigned int FieldIOHdf5::FIELD_DECOMP_OFF = 0;
const unsigned int FieldIOHdf5::FIELD_DECOMP_CNT = 1;
const unsigned int FieldIOHdf5::FIELD_DECOMP_SIZE = 2;
const unsigned int FieldIOHdf5::DATA_DECOMP_FIELD_HASH = 0;
const unsigned int FieldIOHdf5::DATA_DECOMP_IDS_OFF = 1;
const unsigned int FieldIOHdf5::DATA_DECOMP_IDS_CNT = 2;
const unsigned int FieldIOHdf5::DATA_DECOMP_DATA_OFF = 3;
const unsigned int FieldIOHdf5::DATA_DECOMP_DATA_CNT = 4;
const unsigned int FieldIOHdf5::DATA_DECOMP_HOMY_OFF = 5;
const unsigned int FieldIOHdf5::DATA_DECOMP_HOMY_CNT = 6;
const unsigned int FieldIOHdf5::DATA_DECOMP_HOMZ_OFF = 7;
const unsigned int FieldIOHdf5::DATA_DECOMP_HOMZ_CNT = 8;
const unsigned int FieldIOHdf5::DATA_DECOMP_HOMS_OFF = 9;
const unsigned int FieldIOHdf5::DATA_DECOMP_HOMS_CNT = 10;
const unsigned int FieldIOHdf5::DATA_DECOMP_ORDER_OFF = 11;
const unsigned int FieldIOHdf5::DATA_DECOMP_ORDER_CNT = 12;
const unsigned int FieldIOHdf5::DATA_DECOMP_SIZE = 13;
/**
* @brief Construct the FieldIO object for HDF5 output.
*
......@@ -147,10 +102,29 @@ const unsigned int FieldIOHdf5::MAX_IDXS = FieldIOHdf5::HOMS_IDX_IDX + 1;
*/
FieldIOHdf5::FieldIOHdf5(LibUtilities::CommSharedPtr pComm,
bool sharedFilesystem)
: FieldIO(pComm, sharedFilesystem)
: reformatting(true), FieldIO(pComm, sharedFilesystem)
{
}
void FieldIOHdf5::v_Init(const LibUtilities::SessionReaderSharedPtr session)
{
reformatting = true;
}
void FieldIOHdf5::v_InitFromBenchmarker(const LibUtilities::IOSettingsSharedPtr iosettings)
{
LibUtilities::IOSettings::iterator it;
reformatting = true;
it = iosettings->find("Reformatting");
if (iosettings->end() != it)
{
std::istringstream(it->second) >> reformatting;
}
}
/**
* @brief Write a HDF5 file to @p outFile given the field definitions @p
* fielddefs, field data @p fielddata and metadata @p fieldmetadatamap.
......@@ -159,90 +133,70 @@ FieldIOHdf5::FieldIOHdf5(LibUtilities::CommSharedPtr pComm,
*
* - Each rank determines the amount of data needed to be written into each
* dataset.
* - Each rank communicates its decomposition information to the root process.
* - The root processor initialises the output structure, writes the
* decomposition dataset and all the field definition information.
* - Other ranks may have field definitions that do not belong to the root
* - Each rank communicates its decomposition information to the formatting process.
* - The formatting processor initialises the output structure, writes the
* decomposition datasets and all the field definition information.
* - Other ranks may have field definitions that do not belong to the formatting
* process, in which case they open the file and append this (since
* attributes cannot be written in parallel).
* - Each of the other ranks writes their data contributions to the rest of
* the set.
*
* @param outFile Output filename.
* @param fielddefs Input field definitions.
* @param fielddata Input field data.
* @param fieldmetadatamap Field metadata.
* @param outFilename Output filename.
* @param fieldDefs Input field definitions.
* @param fieldData Input field data.
* @param fieldMetaDataMap Field metadata.
* @return The number of bytes written.
*/
uint64_t FieldIOHdf5::v_Write(const std::string &outFile,
std::vector<FieldDefinitionsSharedPtr> &fielddefs,
std::vector<std::vector<NekDouble> > &fielddata,
const FieldMetaDataMap &fieldmetadatamap,
uint64_t FieldIOHdf5::v_Write(const std::string &outFilename,
std::vector<FieldDefinitionsSharedPtr> &fieldDefs,
std::vector<std::vector<NekDouble> > &fieldData,
const FieldMetaDataMap &fieldMetaDataMap,
const bool backup)
{
int nprocs = m_comm->GetSize();
int rk = m_comm->GetRank();
std::stringstream prfx;
prfx << m_comm->GetRank() << ": FieldIOHdf5::v_Write(): ";
uint64_t nWritten = 0;
double tm0 = 0.0, tm1 = 0.0;
if (m_comm->GetRank() == 0)
prfx << rk << ": FieldIOHdf5::v_Write(): ";
/*
double tm0 = 0.0;
if (0 == rk)
{
tm0 = m_comm->Wtime();
}
SetUpOutput(outFile, false, backup);
*/
if (reformatting)
{
SetUpOutput(outFilename, false, backup);
}
// We make a number of assumptions in this code:
// 1. All element ids have the same type: unsigned int
// 2. All elements within a given field have the same number of values
// 3. All element values have the same type, NekDouble
// We make a number of assumptions in this code.
// 1. All element ids have the same type: unsigned int.
// 2. All elements within a given field have the same number of values.
// 3. All element values have the same type, NekDouble.
// Determine the root MPI process, i.e., the lowest ranked process handling
// nMaxFields fields, that will be responsible for writing our file.
ASSERTL1(fielddefs.size() == fielddata.size(),
prfx.str() + "fielddefs and fielddata have incompatible lengths.");
ASSERTL1(fieldDefs.size() == fieldData.size(),
prfx.str() + "fielddefs and fielddata have incompatible lengths.");
std::size_t nFields = fielddefs.size();
uint64_t nWritten = 0;
std::size_t nFields = fieldDefs.size();
std::size_t nMaxFields = nFields;
std::size_t nMinFields = nFields;
m_comm->AllReduce(nMaxFields, LibUtilities::ReduceMax);
m_comm->AllReduce(nMinFields, LibUtilities::ReduceMin);
//cout << prfx.str() << "nFields " << nFields << endl;
int rkFormatter = -1;
std::vector<uint64_t> fieldHashes(nFields);
std::vector<uint64_t> fieldCounts(FIELD_COUNT_SIZE*nFields);
std::vector<uint64_t> totalFieldCounts(FIELD_COUNT_SIZE);
int root_rank = -1;
bool amRoot = false;
LibUtilities::CommSharedPtr max_fields_comm;
if (m_comm->GetSize() > 1)
{
max_fields_comm = m_comm->CommCreateIf((nFields == nMaxFields) ? 1 : 0);
}
else
{
max_fields_comm = m_comm;
}
if (max_fields_comm)
if (reformatting)
{
int rank = m_comm->GetRank();
root_rank = rank;
max_fields_comm->AllReduce(root_rank, LibUtilities::ReduceMin);
amRoot = (rank == root_rank);
if (!amRoot)
{
root_rank = -1;
}
m_comm->AllReduce(nMaxFields, LibUtilities::ReduceMax);
rkFormatter = GetFormattingRank(nFields, nMaxFields);
fieldHashes.resize(nMaxFields);
fieldCounts.resize(FIELD_COUNT_SIZE*nMaxFields);
}
m_comm->AllReduce(root_rank, LibUtilities::ReduceMax);
ASSERTL1(root_rank >= 0 && root_rank < m_comm->GetSize(),
prfx.str() + "invalid root rank.");
std::vector<uint64_t> decomps(nMaxFields * MAX_DCMPS, 0);
std::vector<uint64_t> all_hashes(nMaxFields * m_comm->GetSize(), 0);
std::vector<uint64_t> cnts(MAX_CNTS, 0);
std::vector<std::string> fieldNames(nFields);
std::vector<std::string> shapeStrings(nFields);
std::vector<std::vector<NekDouble> > homoLengths(nFields);
......@@ -250,77 +204,71 @@ uint64_t FieldIOHdf5::v_Write(const std::string &outFile,
homoYIDs(nFields), homoZIDs(nFields);
std::vector<std::vector<unsigned int> > numModesPerDirVar(nFields);
std::vector<std::string> numModesPerDirUni(nFields);
int homDim = -1;
int varOrder = 0;
for (std::size_t f = 0; f < nFields; ++f)
{
if (!fielddefs[f]->m_uniOrder)
if (!fieldDefs[f]->m_uniOrder)
{
varOrder = 1;
break;
}
}
m_comm->AllReduce(varOrder, LibUtilities::ReduceMax);
// Calculate the total number of elements handled by this MPI process and
// the total number of bytes required to store the elements. Base the name
// of each field on the hash of the field definition.
// of each field as the hash of the field definition.
for (std::size_t f = 0; f < nFields; ++f)
{
ASSERTL1(fielddata[f].size() > 0,
prfx.str() +
"fielddata vector must contain at least one value.");
ASSERTL1(fielddata[f].size() ==
fielddefs[f]->m_fields.size() *
CheckFieldDefinition(fielddefs[f]),
prfx.str() + "fielddata vector has invalid size.");
std::size_t nFieldElems = fielddefs[f]->m_elementIDs.size();
std::size_t nElemVals = fielddata[f].size();
decomps[f * MAX_DCMPS + ELEM_DCMP_IDX] = nFieldElems;
decomps[f * MAX_DCMPS + VAL_DCMP_IDX] = nElemVals;
cnts[ELEM_CNT_IDX] += nFieldElems;
cnts[VAL_CNT_IDX] += nElemVals;
std::size_t fco = FIELD_COUNT_SIZE*f;
ASSERTL1(fieldData[f].size() > 0,
prfx.str() + "fieldData vector must contain at least one value.");
ASSERTL1(fieldData[f].size() ==
fieldDefs[f]->m_fields.size() * CheckFieldDefinition(fieldDefs[f]),
prfx.str() + "fieldData vector has invalid size.");
std::size_t nElemIds = fieldDefs[f]->m_elementIDs.size();
fieldCounts[fco + FIELD_COUNT_IDS] = nElemIds;
fieldCounts[fco + FIELD_COUNT_DATA] = fieldData[f].size();
totalFieldCounts[FIELD_COUNT_IDS] += nElemIds;
totalFieldCounts[FIELD_COUNT_DATA] += fieldData[f].size();
// Hash the field specification
std::stringstream hashStream;
std::size_t nSubFields = fielddefs[f]->m_fields.size();
std::size_t nSubFields = fieldDefs[f]->m_fields.size();
for (std::size_t sf = 0; sf < nSubFields; ++sf)
{
hashStream << fielddefs[f]->m_fields[sf];
hashStream << fieldDefs[f]->m_fields[sf];
}
nSubFields = fielddefs[f]->m_basis.size();
nSubFields = fieldDefs[f]->m_basis.size();
for (std::size_t sf = 0; sf < nSubFields; ++sf)
{
hashStream << fielddefs[f]->m_basis[sf];
hashStream << fieldDefs[f]->m_basis[sf];
}
// Determine SHAPE attribute
std::stringstream shapeStringStream;
shapeStringStream << ShapeTypeMap[fielddefs[f]->m_shapeType];
shapeStringStream << ShapeTypeMap[fieldDefs[f]->m_shapeType];
if (fielddefs[f]->m_numHomogeneousDir > 0)
if (fieldDefs[f]->m_numHomogeneousDir > 0)
{
if (homDim == -1)
{
homDim = fielddefs[f]->m_numHomogeneousDir;
homDim = fieldDefs[f]->m_numHomogeneousDir;
}
ASSERTL1(homDim == fielddefs[f]->m_numHomogeneousDir,
ASSERTL1(homDim == fieldDefs[f]->m_numHomogeneousDir,
"HDF5 does not support variable homogeneous directions in "
"the same file.");
shapeStringStream << "-HomogenousExp"
<< fielddefs[f]->m_numHomogeneousDir << "D";
<< fieldDefs[f]->m_numHomogeneousDir << "D";
}
if (fielddefs[f]->m_homoStrips)
if (fieldDefs[f]->m_homoStrips)
{
shapeStringStream << "-Strips";
}
......@@ -329,74 +277,73 @@ uint64_t FieldIOHdf5::v_Write(const std::string &outFile,
hashStream << shapeStringStream.str();
// Determine HOMOGENEOUS attributes
if (fielddefs[f]->m_numHomogeneousDir)
if (fieldDefs[f]->m_numHomogeneousDir)
{
nSubFields = fielddefs[f]->m_homogeneousLengths.size();
nSubFields = fieldDefs[f]->m_homogeneousLengths.size();
homoLengths[f].resize(nSubFields);
for (std::size_t sf = 0; sf < nSubFields; ++sf)
{
NekDouble len = fielddefs[f]->m_homogeneousLengths[sf];
NekDouble len = fieldDefs[f]->m_homogeneousLengths[sf];
hashStream << len;
homoLengths[f][sf] = len;
}
nSubFields = fielddefs[f]->m_homogeneousYIDs.size();
nSubFields = fieldDefs[f]->m_homogeneousYIDs.size();
if (nSubFields > 0)
{
homoYIDs[f].resize(nSubFields);
decomps[f * MAX_DCMPS + HOMY_DCMP_IDX] = nSubFields;
cnts[HOMY_CNT_IDX] += nSubFields;
fieldCounts[fco + FIELD_COUNT_HOMY] = nSubFields;
totalFieldCounts[FIELD_COUNT_HOMY] += nSubFields;
for (std::size_t sf = 0; sf < nSubFields; ++sf)
{
homoYIDs[f][sf] = fielddefs[f]->m_homogeneousYIDs[sf];
homoYIDs[f][sf] = fieldDefs[f]->m_homogeneousYIDs[sf];
}
}
nSubFields = fielddefs[f]->m_homogeneousZIDs.size();
nSubFields = fieldDefs[f]->m_homogeneousZIDs.size();
if (nSubFields > 0)
{
homoZIDs[f].resize(nSubFields);
decomps[f * MAX_DCMPS + HOMZ_DCMP_IDX] = nSubFields;
cnts[HOMZ_CNT_IDX] += nSubFields;
fieldCounts[fco + FIELD_COUNT_HOMZ] = nSubFields;
totalFieldCounts[FIELD_COUNT_HOMZ] += nSubFields;
for (std::size_t sf = 0; sf < nSubFields; ++sf)
{
homoZIDs[f][sf] = fielddefs[f]->m_homogeneousZIDs[sf];
homoZIDs[f][sf] = fieldDefs[f]->m_homogeneousZIDs[sf];
}
}
nSubFields = fielddefs[f]->m_homogeneousSIDs.size();
nSubFields = fieldDefs[f]->m_homogeneousSIDs.size();
if (nSubFields > 0)
{
homoSIDs[f].resize(nSubFields);
decomps[f * MAX_DCMPS + HOMS_DCMP_IDX] = nSubFields;
cnts[HOMS_CNT_IDX] += nSubFields;
fieldCounts[fco + FIELD_COUNT_HOMS] = nSubFields;
totalFieldCounts[FIELD_COUNT_HOMS] += nSubFields;
for (std::size_t sf = 0; sf < nSubFields; ++sf)
{
homoSIDs[f][sf] = fielddefs[f]->m_homogeneousSIDs[sf];
homoSIDs[f][sf] = fieldDefs[f]->m_homogeneousSIDs[sf];
}
}
}
if (fielddefs[f]->m_uniOrder)
if (fieldDefs[f]->m_uniOrder)
{
std::vector<unsigned int> elemModes(fielddefs[f]->m_basis.size());
std::vector<unsigned int> elemModes(fieldDefs[f]->m_basis.size());
for (std::vector<int>::size_type i = 0;
i < fielddefs[f]->m_basis.size(); ++i)
i < fieldDefs[f]->m_basis.size(); ++i)
{
elemModes[i] = fielddefs[f]->m_numModes[i];
elemModes[i] = fieldDefs[f]->m_numModes[i];
}
if (varOrder)
{
for (std::vector<int>::size_type i = 0; i < nFieldElems; ++i)
for (std::vector<int>::size_type i = 0; i < nElemIds; ++i)
{
std::copy(elemModes.begin(), elemModes.end(),
std::back_inserter(numModesPerDirVar[f]));
}
decomps[f * MAX_DCMPS + ORDER_DCMP_IDX] =
nFieldElems * elemModes.size();
cnts[ORDER_CNT_IDX] += nFieldElems * elemModes.size();
fieldCounts[fco + FIELD_COUNT_ORDER] = nElemIds * elemModes.size();
totalFieldCounts[FIELD_COUNT_ORDER] += fieldCounts[fco + FIELD_COUNT_ORDER];
}
else
{
......@@ -418,395 +365,661 @@ uint64_t FieldIOHdf5::v_Write(const std::string &outFile,
}
else
{
numModesPerDirVar[f] = fielddefs[f]->m_numModes;
decomps[f * MAX_DCMPS + ORDER_DCMP_IDX] =
fielddefs[f]->m_numModes.size();
cnts[ORDER_CNT_IDX] += fielddefs[f]->m_numModes.size();
numModesPerDirVar[f] = fieldDefs[f]->m_numModes;
fieldCounts[fco + FIELD_COUNT_ORDER] = fieldDefs[f]->m_numModes.size();
totalFieldCounts[FIELD_COUNT_ORDER] += fieldCounts[fco + FIELD_COUNT_ORDER];
}
std::hash<std::string> string_hasher;
std::hash<std::string> stringHasher;
std::stringstream fieldNameStream;
uint64_t fieldDefHash = string_hasher(hashStream.str());
uint64_t fieldDefHash = stringHasher(hashStream.str());
decomps[f * MAX_DCMPS + HASH_DCMP_IDX] = fieldDefHash;
all_hashes[m_comm->GetRank() * nMaxFields + f] = fieldDefHash;
if (reformatting)
{
fieldHashes[f] = fieldDefHash;
}
fieldNameStream << fieldDefHash;
fieldNames[f] = fieldNameStream.str();
}
// Gather information from all MPI processes
std::vector<uint64_t> all_cnts = m_comm->Gather(root_rank, cnts);
std::vector<uint64_t> all_idxs(m_comm->GetSize() * MAX_IDXS, 0);
std::vector<uint64_t> all_decomps = m_comm->Gather(root_rank, decomps);
std::vector<uint64_t> all_dsetsize(MAX_CNTS, 0);
} // end of <for (std::size_t f = 0; f < nFields; ++f)> loop
// The root rank creates the file layout from scratch
if (amRoot)
std::vector<uint64_t> firstDataDecomps(DATA_DECOMP_SIZE, 0);
if (reformatting)
{
H5::FileSharedPtr outfile = H5::File::Create(outFile, H5F_ACC_TRUNC);
ASSERTL1(outfile, prfx.str() + "cannot create HDF5 file.");
H5::GroupSharedPtr root = outfile->CreateGroup("NEKTAR");
ASSERTL1(root, prfx.str() + "cannot create root group.");
TagWriterSharedPtr info_writer(new H5TagWriter(root));
AddInfoTag(info_writer, fieldmetadatamap);
// Record file format version as attribute in main group.
root->SetAttribute("FORMAT_VERSION", FORMAT_VERSION);
nWritten += sizeof(FORMAT_VERSION);
// Calculate the indexes to be used by each MPI process when reading the
// IDS and DATA datasets
std::size_t nTotElems = 0, nTotVals = 0, nTotOrder = 0;
std::size_t nTotHomY = 0, nTotHomZ = 0, nTotHomS = 0;
int nRanks = m_comm->GetSize();
for (int r = 0; r < nRanks; ++r)
std::vector<uint64_t> fieldDecomps(FIELD_DECOMP_SIZE, 0);
fieldDecomps[FIELD_DECOMP_CNT] = nFields;
std::vector<uint64_t> allFieldDecomps = m_comm->Gather(rkFormatter, fieldDecomps);
std::vector<uint64_t> allFieldCounts = m_comm->Gather(rkFormatter, fieldCounts);
std::vector<uint64_t> allFieldHashes = m_comm->Gather(rkFormatter, fieldHashes);
std::vector<uint64_t> allFirstDataDecomps;
std::size_t nTotFields = 0;
m_comm->Reduce(nFields, nTotFields, LibUtilities::ReduceSum, rkFormatter);
if (rkFormatter == rk)
{
all_idxs[r * MAX_IDXS + IDS_IDX_IDX] = nTotElems;
all_idxs[r * MAX_IDXS + DATA_IDX_IDX] = nTotVals;
all_idxs[r * MAX_IDXS + ORDER_IDX_IDX] = nTotOrder;
all_idxs[r * MAX_IDXS + HOMY_IDX_IDX] = nTotHomY;
all_idxs[r * MAX_IDXS + HOMZ_IDX_IDX] = nTotHomZ;
all_idxs[r * MAX_IDXS + HOMS_IDX_IDX] = nTotHomS;
nTotElems += all_cnts[r * MAX_CNTS + ELEM_CNT_IDX];
nTotVals += all_cnts[r * MAX_CNTS + VAL_CNT_IDX];
nTotOrder += all_cnts[r * MAX_CNTS + ORDER_CNT_IDX];
nTotHomY += all_cnts[r * MAX_CNTS + HOMY_CNT_IDX];
nTotHomZ += all_cnts[r * MAX_CNTS + HOMZ_CNT_IDX];
nTotHomS += all_cnts[r * MAX_CNTS + HOMS_CNT_IDX];
}
std::vector<uint64_t> allDataDecomps(DATA_DECOMP_SIZE*nTotFields);
allFirstDataDecomps.resize(DATA_DECOMP_SIZE*nprocs);
nWritten += CreateDataSets(outFilename, rkFormatter, nMaxFields, nTotFields,
allFieldHashes, allFieldCounts, allFieldDecomps,
allFirstDataDecomps, allDataDecomps,
fieldDefs, homoYIDs, homoZIDs, homoSIDs, numModesPerDirVar,
fieldData, fieldMetaDataMap);
nWritten += WriteDecompositionData(outFilename, rkFormatter,
allFieldDecomps, allDataDecomps);
std::set<uint64_t> hashesAssigned;
for (std::size_t f = 0; f < nMaxFields; ++f)
{
uint64_t hash = allFieldHashes[nMaxFields*rkFormatter + f];
hashesAssigned.insert(hash);
}
for (int r = 0; r < nprocs; ++r)
{
if (r == rkFormatter) continue;
for (std::size_t f = 0; f < nMaxFields; ++f)
{
// Note hash can be zero if, for process r, nFields < nMaxFields.
uint64_t hash = allFieldHashes[nMaxFields*r + f];
if (0 == hash) break;
if (hashesAssigned.find(hash) == hashesAssigned.end())
{
hashesAssigned.insert(hash);
}
else
{
// This field hash has been assigned to another process.
allFieldHashes[nMaxFields*r + f] = 0;
}
}
}
} // end of <if (rkFormatter == rk)> clause
// Scatter those field hashes that indicate which the field definitions that each process writes to file.
std::vector<uint64_t> writeFieldHashes = m_comm->Scatter(rkFormatter, allFieldHashes);
// Write field definitions to checkpoint file.
nWritten += WriteFieldAttributes(outFilename, nFields, varOrder, writeFieldHashes, fieldDefs,
fieldNames, shapeStrings, homoLengths, numModesPerDirUni);
} // end of <if (reformatting)> clause
all_dsetsize[ELEM_CNT_IDX ] = nTotElems;
all_dsetsize[VAL_CNT_IDX ] = nTotVals;
all_dsetsize[ORDER_CNT_IDX] = nTotOrder;
all_dsetsize[HOMY_CNT_IDX ] = nTotHomY;
all_dsetsize[HOMZ_CNT_IDX ] = nTotHomZ;
all_dsetsize[HOMS_CNT_IDX ] = nTotHomS;
// Create DECOMPOSITION dataset: basic field info for each MPI process
H5::DataTypeSharedPtr decomps_type =
H5::DataType::OfObject(all_decomps[0]);
H5::DataSpaceSharedPtr decomps_space =
H5::DataSpace::OneD(all_decomps.size());
H5::DataSetSharedPtr decomps_dset =
root->CreateDataSet("DECOMPOSITION", decomps_type, decomps_space);
ASSERTL1(decomps_dset,
prfx.str() + "cannot create DECOMPOSITION dataset.");
// Create IDS dataset: element ids
H5::DataTypeSharedPtr ids_type =
H5::DataType::OfObject(fielddefs[0]->m_elementIDs[0]);
H5::DataSpaceSharedPtr ids_space = H5::DataSpace::OneD(nTotElems);
H5::DataSetSharedPtr ids_dset =
root->CreateDataSet("ELEMENTIDS", ids_type, ids_space);
ASSERTL1(ids_dset, prfx.str() + "cannot create ELEMENTIDS dataset.");
// Create DATA dataset: element data
H5::DataTypeSharedPtr data_type =
H5::DataType::OfObject(fielddata[0][0]);
H5::DataSpaceSharedPtr data_space = H5::DataSpace::OneD(nTotVals);
H5::DataSetSharedPtr data_dset =
root->CreateDataSet("DATA", data_type, data_space);
ASSERTL1(data_dset, prfx.str() + "cannot create DATA dataset.");
// Create HOMOGENEOUSYIDS dataset: homogeneous y-plane IDs
if (nTotHomY > 0)
{
H5::DataTypeSharedPtr homy_type =
H5::DataType::OfObject(homoYIDs[0][0]);