Commit 4af7803c authored by Dave Moxey's avatar Dave Moxey
Browse files

Fix bug in HDF5 group writing in parallel. Also remove const correctness from...

Fix bug in HDF5 group writing in parallel. Also remove const correctness from Comm classes, since this breaks older MPI implementations.
parent f768d1c3
......@@ -154,19 +154,20 @@ void FieldIOHdf5::v_Write(const std::string &outFile,
m_comm->CommCreateIf((nFields == nMaxFields) ? 1 : 0);
if (max_fields_comm)
{
int rank = max_fields_comm->GetRank();
int rank = m_comm->GetRank();
root_rank = rank;
max_fields_comm->AllReduce(root_rank, LibUtilities::ReduceMin);
amRoot = (rank == root_rank);
if (!amRoot)
root_rank = -1;
}
max_fields_comm->AllReduce(root_rank, LibUtilities::ReduceMax);
m_comm->AllReduce(root_rank, LibUtilities::ReduceMax);
ASSERTL1(root_rank >= 0 && root_rank < m_comm->GetSize(),
prfx.str() + "invalid root rank.");
/////////////////////////////////////////////////////////////////////////////////////////////
std::vector<std::size_t> decomps(nMaxFields * MAX_DCMPS, 0);
std::vector<std::size_t> all_hashes(nMaxFields * m_comm->GetSize(), 0);
std::vector<std::size_t> cnts(MAX_CNTS, 0);
std::vector<std::string> fieldNames(nFields);
std::vector<std::string> shapeStrings(nFields);
......@@ -316,6 +317,7 @@ void FieldIOHdf5::v_Write(const std::string &outFile,
std::stringstream fieldNameStream;
std::size_t fieldDefHash = string_hasher(hashStream.str());
decomps[f * MAX_DCMPS + HASH_DCMP_IDX] = fieldDefHash;
all_hashes[m_comm->GetRank() * nMaxFields + f] = fieldDefHash;
fieldNameStream << fieldDefHash;
fieldNames[f] = fieldNameStream.str();
/////////////////////////////////////////////////////////////////////////////////////////
......@@ -399,24 +401,6 @@ void FieldIOHdf5::v_Write(const std::string &outFile,
ASSERTL1(data_dset, prfx.str() + "cannot create DATA dataset.");
/////////////////////////////////////////////////////////////////////////////////////////
// write a hdf5 group for each field handled by the root rank
for (int f = 0; f < nFields; ++f)
{
H5::GroupSharedPtr field_group = root->CreateGroup(fieldNames[f]);
ASSERTL1(field_group, prfx.str() + "cannot create field group.");
field_group->SetAttribute("FIELDS", fielddefs[f]->m_fields);
field_group->SetAttribute("BASIS", fielddefs[f]->m_basis);
field_group->SetAttribute("SHAPE", shapeStrings[f]);
if (homoLengths[f].size() > 0)
field_group->SetAttribute("HOMOGENEOUSLENGTHS", homoLengths[f]);
if (homoYIDs[f].size() > 0)
field_group->SetAttribute("HOMOGENEOUSYIDS", homoYIDs[f]);
if (homoZIDs[f].size() > 0)
field_group->SetAttribute("HOMOGENEOUSZIDS", homoZIDs[f]);
if (homoSIDs[f].size() > 0)
field_group->SetAttribute("HOMOGENEOUSSIDS", homoSIDs[f]);
field_group->SetAttribute("NUMMODESPERDIR", numModesPerDirs[f]);
}
// RAII => field group is closed automatically
} // if (amRoot)
......@@ -424,6 +408,82 @@ void FieldIOHdf5::v_Write(const std::string &outFile,
// closed automatically
////////////////////////////////////////////////////////////////////////////////////////////////
// Determine which process will write the group representing the field
// description in the HDF5 file.
std::map<size_t, int> hashToProc;
std::map<int, std::vector<size_t> > writingProcs;
// Gather all field hashes to every processor.
m_comm->AllReduce(all_hashes, LibUtilities::ReduceMax);
std::cout << "RANK " << m_comm->GetRank() << " HAS: ";
for (int i = 0; i < all_hashes.size(); ++i)
{
std::cout << all_hashes[i] << " ";
}
std::cout << std::endl;
for (int n = 0; n < m_comm->GetSize(); ++n)
{
for (int i = 0; i < nMaxFields; ++i)
{
size_t hash = all_hashes[n*nMaxFields + i];
if (hashToProc.find(hash) != hashToProc.end() || hash == 0)
{
continue;
}
hashToProc[hash] = m_comm->GetRank();
writingProcs[n].push_back(hash);
}
}
map<int, std::vector<size_t> >::iterator sIt;
for (sIt = writingProcs.begin(); sIt != writingProcs.end(); sIt++)
{
int rank = sIt->first;
if (m_comm->GetRank() == rank)
{
H5::PListSharedPtr serialProps = H5::PList::Default();
H5::PListSharedPtr writeSR = H5::PList::Default();
// reopen the file
H5::FileSharedPtr outfile =
H5::File::Open(outFile, H5F_ACC_RDWR, serialProps);
ASSERTL1(outfile, prfx.str() + "cannot open HDF5 file.");
H5::GroupSharedPtr root = outfile->OpenGroup("NEKTAR");
ASSERTL1(root, prfx.str() + "cannot open root group.");
// write a hdf5 group for each field
for (int i = 0; i < sIt->second.size(); ++i)
{
for (int f = 0; f < nFields; ++f)
{
if (sIt->second[i] != all_hashes[m_comm->GetRank() * nMaxFields + f])
{
continue;
}
std::cout << "RANK " << rank << " WRITING HASH " << sIt->second[i] << std::endl;
H5::GroupSharedPtr field_group = root->CreateGroup(fieldNames[f]);
ASSERTL1(field_group, prfx.str() + "cannot create field group.");
field_group->SetAttribute("FIELDS", fielddefs[f]->m_fields);
field_group->SetAttribute("BASIS", fielddefs[f]->m_basis);
field_group->SetAttribute("SHAPE", shapeStrings[f]);
if (homoLengths[f].size() > 0)
field_group->SetAttribute("HOMOGENEOUSLENGTHS", homoLengths[f]);
if (homoYIDs[f].size() > 0)
field_group->SetAttribute("HOMOGENEOUSYIDS", homoYIDs[f]);
if (homoZIDs[f].size() > 0)
field_group->SetAttribute("HOMOGENEOUSZIDS", homoZIDs[f]);
if (homoSIDs[f].size() > 0)
field_group->SetAttribute("HOMOGENEOUSSIDS", homoSIDs[f]);
field_group->SetAttribute("NUMMODESPERDIR", numModesPerDirs[f]);
}
}
}
m_comm->Block();
}
// write the DECOMPOSITION and INDEXES datasets
/////////////////////////////////////////////////////////////////////////////////////////
if (amRoot)
......@@ -494,32 +554,6 @@ void FieldIOHdf5::v_Write(const std::string &outFile,
H5::GroupSharedPtr root = outfile->OpenGroup("NEKTAR");
ASSERTL1(root, prfx.str() + "cannot open root group.");
// ensure that all field groups have been created
// the root rank might not handle all types of field
////////////////////////////////////////////////////////////////////////////////////////////////
for (int f = 0; f < nFields; ++f)
{
H5::GroupSharedPtr field_group = root->OpenGroup(fieldNames[f]);
if (!field_group)
{
cout << prfx.str() << "creating group " << fieldNames[f] << "..."
<< endl;
field_group = root->CreateGroup(fieldNames[f]);
ASSERTL1(field_group, prfx.str() + "cannot create field group.");
field_group->SetAttribute("FIELDS", fielddefs[f]->m_fields);
field_group->SetAttribute("BASIS", fielddefs[f]->m_basis);
field_group->SetAttribute("SHAPE", shapeStrings[f]);
if (homoLengths[f].size() > 0)
field_group->SetAttribute("HOMOGENEOUSLENGTHS", homoLengths[f]);
if (homoYIDs[f].size() > 0)
field_group->SetAttribute("HOMOGENEOUSYIDS", homoYIDs[f]);
if (homoZIDs[f].size() > 0)
field_group->SetAttribute("HOMOGENEOUSZIDS", homoZIDs[f]);
if (homoSIDs[f].size() > 0)
field_group->SetAttribute("HOMOGENEOUSSIDS", homoSIDs[f]);
field_group->SetAttribute("NUMMODESPERDIR", numModesPerDirs[f]);
}
}
// RAII => field group is closed automatically
m_comm->Block();
// all hdf5 groups have now been created
......
......@@ -92,11 +92,11 @@ namespace Nektar
LIB_UTILITIES_EXPORT inline double Wtime();
template<class T>
void Send(int pProc, const T& pData);
void Send(int pProc, T& pData);
template<class T>
void Recv(int pProc, T& pData);
template<class T>
void SendRecv(int pSendProc, const T& pSendData,
void SendRecv(int pSendProc, T& pSendData,
int pRecvProc, T& pRecvData);
template<class T>
void SendRecvReplace(int pSendProc, int pRecvProc, T& pData);
......@@ -105,25 +105,25 @@ namespace Nektar
void AllReduce(T& pData, enum ReduceOperator pOp);
template<class T>
void AlltoAll(const T& pSendData, T& pRecvData);
void AlltoAll(T& pSendData, T& pRecvData);
template<class T>
void AlltoAllv(const Array<OneD, T>& pSendData,
const Array<OneD, int>& pSendDataSizeMap,
const Array<OneD, int>& pSendDataOffsetMap,
void AlltoAllv(Array<OneD, T>& pSendData,
Array<OneD, int>& pSendDataSizeMap,
Array<OneD, int>& pSendDataOffsetMap,
Array<OneD, T>& pRecvData,
const Array<OneD, int>& pRecvDataSizeMap,
const Array<OneD, int>& pRecvDataOffsetMap);
Array<OneD, int>& pRecvDataSizeMap,
Array<OneD, int>& pRecvDataOffsetMap);
template<class T>
void Bcast(T& data, int rootProc);
template<class T>
void Exscan(const T& pData, const enum ReduceOperator pOp, T& ans);
void Exscan(T& pData, const enum ReduceOperator pOp, T& ans);
template<class T>
T Gather(const int rootProc, const T& val);
T Gather(const int rootProc, T& val);
template<class T>
T Scatter(const int rootProc, const T& pData);
T Scatter(const int rootProc, T& pData);
LIB_UTILITIES_EXPORT inline CommSharedPtr CommCreateIf(int flag);
......@@ -147,24 +147,24 @@ namespace Nektar
virtual int v_GetRank() = 0;
virtual void v_Block() = 0;
virtual double v_Wtime() = 0;
virtual void v_Send(const void* buf, int count, CommDataType dt, int dest) = 0;
virtual void v_Send(void* buf, int count, CommDataType dt, int dest) = 0;
virtual void v_Recv(void* buf, int count, CommDataType dt, int source) = 0;
virtual void v_Sendrecv(const void *sendbuf, int sendcount, CommDataType sendtype, int dest,
virtual void v_Sendrecv(void *sendbuf, int sendcount, CommDataType sendtype, int dest,
void *recvbuf, int recvcount, CommDataType recvtype, int source) = 0;
virtual void v_SendRecvReplace(void* buf, int count, CommDataType dt,
int pSendProc, int pRecvProc) = 0;
virtual void v_AllReduce(void* buf, int count, CommDataType dt, enum ReduceOperator pOp) = 0;
virtual void v_AlltoAll(const void* sendbuf, int sendcount, CommDataType sendtype,
void* recvbuf, int recvcount, CommDataType recvtype) = 0;
virtual void v_AlltoAllv(const void *sendbuf, const int sendcounts[], const int sensdispls[], CommDataType sendtype,
void *recvbuf, const int recvcounts[], const int rdispls[], CommDataType recvtype) = 0;
virtual void v_Bcast(void* buffer, int count, CommDataType dt, int root) = 0;
virtual void v_Exscan(const Array<OneD, unsigned long long>& pData, const enum ReduceOperator pOp, Array<OneD, unsigned long long>& ans) = 0;
virtual void v_Gather(const void* sendbuf, int sendcount, CommDataType sendtype,
virtual void v_SendRecvReplace(void* buf, int count, CommDataType dt,
int pSendProc, int pRecvProc) = 0;
virtual void v_AllReduce(void* buf, int count, CommDataType dt, enum ReduceOperator pOp) = 0;
virtual void v_AlltoAll(void* sendbuf, int sendcount, CommDataType sendtype,
void* recvbuf, int recvcount, CommDataType recvtype) = 0;
virtual void v_AlltoAllv(void *sendbuf, int sendcounts[], int sensdispls[], CommDataType sendtype,
void *recvbuf, int recvcounts[], int rdispls[], CommDataType recvtype) = 0;
virtual void v_Bcast(void* buffer, int count, CommDataType dt, int root) = 0;
virtual void v_Exscan(Array<OneD, unsigned long long>& pData, const enum ReduceOperator pOp, Array<OneD, unsigned long long>& ans) = 0;
virtual void v_Gather(void* sendbuf, int sendcount, CommDataType sendtype,
void *recvbuf, int recvcount, CommDataType recvtype, int root) = 0;
virtual void v_Scatter(const void *sendbuf, int sendcount, CommDataType sendtype,
virtual void v_Scatter(void *sendbuf, int sendcount, CommDataType sendtype,
void *recvbuf, int recvcount, CommDataType recvtype, int root) = 0;
virtual CommSharedPtr v_CommCreateIf(int flag) = 0;
......@@ -223,7 +223,7 @@ namespace Nektar
}
template<class T>
void Comm::Send(int pProc, const T& pData)
void Comm::Send(int pProc, T& pData)
{
v_Send(CommDataTypeTraits<T>::GetPointer(pData),
CommDataTypeTraits<T>::GetCount(pData),
......@@ -245,7 +245,7 @@ namespace Nektar
*/
template<class T>
void Comm::SendRecv(int pSendProc,
const T& pSendData,
T& pSendData,
int pRecvProc,
T& pRecvData)
{
......@@ -286,7 +286,7 @@ namespace Nektar
}
template <class T>
void Comm::AlltoAll(const T& pSendData, T& pRecvData)
void Comm::AlltoAll(T& pSendData, T& pRecvData)
{
BOOST_STATIC_ASSERT_MSG(CommDataTypeTraits<T>::IsVector,
"AlltoAll only valid with Array or vector arguments.");
......@@ -311,12 +311,12 @@ namespace Nektar
*
*/
template<class T>
void Comm::AlltoAllv(const Array<OneD, T>& pSendData,
const Array<OneD, int>& pSendDataSizeMap,
const Array<OneD, int>& pSendDataOffsetMap,
void Comm::AlltoAllv(Array<OneD, T>& pSendData,
Array<OneD, int>& pSendDataSizeMap,
Array<OneD, int>& pSendDataOffsetMap,
Array<OneD, T>& pRecvData,
const Array<OneD, int>& pRecvDataSizeMap,
const Array<OneD, int>& pRecvDataOffsetMap)
Array<OneD, int>& pRecvDataSizeMap,
Array<OneD, int>& pRecvDataOffsetMap)
{
v_AlltoAllv(pSendData.get(),
pSendDataSizeMap.get(),
......@@ -340,7 +340,7 @@ namespace Nektar
}
template<class T>
void Comm::Exscan(const T& pData, const enum ReduceOperator pOp, T& ans)
void Comm::Exscan(T& pData, const enum ReduceOperator pOp, T& ans)
{
ASSERTL0(CommDataTypeTraits<T>::GetCount(pData) == CommDataTypeTraits<T>::GetCount(ans),
"Input and output array sizes don't match");
......@@ -355,7 +355,7 @@ namespace Nektar
* Concatenate all the input arrays, in rank order, onto the process with rank == rootProc
*/
template<class T>
T Comm::Gather(const int rootProc, const T& val)
T Comm::Gather(const int rootProc, T& val)
{
BOOST_STATIC_ASSERT_MSG(CommDataTypeTraits<T>::IsVector,
"Gather only valid with Array or vector arguments.");
......@@ -375,7 +375,7 @@ namespace Nektar
* Scatter pData across ranks in chunks of len(pData)/num_ranks
*/
template<class T>
T Comm::Scatter(const int rootProc, const T& pData)
T Comm::Scatter(const int rootProc, T& pData)
{
BOOST_STATIC_ASSERT_MSG(CommDataTypeTraits<T>::IsVector,
"Scatter only valid with Array or vector arguments.");
......@@ -383,7 +383,7 @@ namespace Nektar
bool amRoot = (GetRank() == rootProc);
unsigned nEl = CommDataTypeTraits<T>::GetCount(pData) / GetSize();
const void* sendbuf = amRoot ? CommDataTypeTraits<T>::GetPointer(pData) : NULL;
void* sendbuf = amRoot ? CommDataTypeTraits<T>::GetPointer(pData) : NULL;
T ans(nEl);
v_Scatter(sendbuf, nEl, CommDataTypeTraits<T>::GetDataType(),
......
......@@ -165,7 +165,7 @@ namespace Nektar
/**
*
*/
void CommMpi::v_Send(const void* buf, int count, CommDataType dt, int dest)
void CommMpi::v_Send(void* buf, int count, CommDataType dt, int dest)
{
if (MPISYNC)
{
......@@ -191,7 +191,7 @@ namespace Nektar
/**
*
*/
void CommMpi::v_Sendrecv(const void *sendbuf, int sendcount, CommDataType sendtype, int dest,
void CommMpi::v_Sendrecv(void *sendbuf, int sendcount, CommDataType sendtype, int dest,
void *recvbuf, int recvcount, CommDataType recvtype, int source)
{
MPI_Status status;
......@@ -252,10 +252,11 @@ namespace Nektar
ASSERTL0(retval == MPI_SUCCESS,
"MPI error performing All-reduce.");
}
/**
*
*/
void CommMpi::v_AlltoAll(const void* sendbuf, int sendcount, CommDataType sendtype,
void CommMpi::v_AlltoAll(void* sendbuf, int sendcount, CommDataType sendtype,
void* recvbuf, int recvcount, CommDataType recvtype)
{
int retval = MPI_Alltoall(sendbuf, sendcount, sendtype,
......@@ -269,8 +270,8 @@ namespace Nektar
/**
*
*/
void CommMpi::v_AlltoAllv(const void *sendbuf, const int sendcounts[], const int sdispls[], CommDataType sendtype,
void *recvbuf, const int recvcounts[], const int rdispls[], CommDataType recvtype)
void CommMpi::v_AlltoAllv(void *sendbuf, int sendcounts[], int sdispls[], CommDataType sendtype,
void *recvbuf, int recvcounts[], int rdispls[], CommDataType recvtype)
{
int retval = MPI_Alltoallv(sendbuf, sendcounts, sdispls, sendtype,
recvbuf, recvcounts, rdispls, recvtype,
......@@ -287,7 +288,7 @@ namespace Nektar
"MPI error performing Bcast-v.");
}
void CommMpi::v_Exscan(const Array<OneD, unsigned long long>& pData, const enum ReduceOperator pOp, Array<OneD, unsigned long long>& ans)
void CommMpi::v_Exscan(Array<OneD, unsigned long long>& pData, const enum ReduceOperator pOp, Array<OneD, unsigned long long>& ans)
{
int n = pData.num_elements();
ASSERTL0(n == ans.num_elements(),
......@@ -311,7 +312,7 @@ namespace Nektar
"MPI error performing Exscan-v.");
}
void CommMpi::v_Gather(const void* sendbuf, int sendcount, CommDataType sendtype,
void CommMpi::v_Gather(void* sendbuf, int sendcount, CommDataType sendtype,
void *recvbuf, int recvcount, CommDataType recvtype, int root)
{
int retval = MPI_Gather(sendbuf, sendcount, sendtype,
......@@ -322,7 +323,7 @@ namespace Nektar
"MPI error performing Gather.");
}
void CommMpi::v_Scatter(const void* sendbuf, int sendcount, CommDataType sendtype,
void CommMpi::v_Scatter(void* sendbuf, int sendcount, CommDataType sendtype,
void *recvbuf, int recvcount, CommDataType recvtype, int root)
{
int retval = MPI_Scatter(sendbuf, sendcount, sendtype,
......
......@@ -81,23 +81,23 @@ namespace Nektar
virtual void v_Block();
virtual double v_Wtime();
virtual bool v_TreatAsRankZero(void);
virtual void v_Send(const void* buf, int count, CommDataType dt, int dest);
virtual void v_Send(void* buf, int count, CommDataType dt, int dest);
virtual void v_Recv(void* buf, int count, CommDataType dt, int source);
virtual void v_Sendrecv(const void *sendbuf, int sendcount, CommDataType sendtype, int dest,
virtual void v_Sendrecv(void *sendbuf, int sendcount, CommDataType sendtype, int dest,
void *recvbuf, int recvcount, CommDataType recvtype, int source);
virtual void v_SendRecvReplace(void* buf, int count, CommDataType dt,
int pSendProc, int pRecvProc);
virtual void v_AllReduce(void* buf, int count, CommDataType dt, enum ReduceOperator pOp);
virtual void v_AlltoAll(const void* sendbuf, int sendcount, CommDataType sendtype,
virtual void v_AlltoAll(void* sendbuf, int sendcount, CommDataType sendtype,
void* recvbuf, int recvcount, CommDataType recvtype);
virtual void v_AlltoAllv(const void *sendbuf, const int sendcounts[], const int sensdispls[], CommDataType sendtype,
void *recvbuf, const int recvcounts[], const int rdispls[], CommDataType recvtype);
virtual void v_AlltoAllv(void *sendbuf, int sendcounts[], int sensdispls[], CommDataType sendtype,
void *recvbuf, int recvcounts[], int rdispls[], CommDataType recvtype);
virtual void v_Bcast(void* buffer, int count, CommDataType dt, int root);
virtual void v_Exscan(const Array<OneD, unsigned long long>& pData, const enum ReduceOperator pOp, Array<OneD, unsigned long long>& ans);
virtual void v_Exscan(Array<OneD, unsigned long long>& pData, const enum ReduceOperator pOp, Array<OneD, unsigned long long>& ans);
virtual void v_Gather(const void* sendbuf, int sendcount, CommDataType sendtype,
virtual void v_Gather(void* sendbuf, int sendcount, CommDataType sendtype,
void *recvbuf, int recvcount, CommDataType recvtype, int root);
virtual void v_Scatter(const void *sendbuf, int sendcount, CommDataType sendtype,
virtual void v_Scatter(void *sendbuf, int sendcount, CommDataType sendtype,
void *recvbuf, int recvcount, CommDataType recvtype, int root);
virtual void v_SplitComm(int pRows, int pColumns);
......
......@@ -110,7 +110,7 @@ namespace Nektar
/**
*
*/
void CommSerial::v_Send(const void* buf, int count, CommDataType dt, int dest)
void CommSerial::v_Send(void* buf, int count, CommDataType dt, int dest)
{
}
......@@ -125,7 +125,7 @@ namespace Nektar
/**
*
*/
void CommSerial::v_Sendrecv(const void *sendbuf, int sendcount, CommDataType sendtype, int dest,
void CommSerial::v_Sendrecv(void *sendbuf, int sendcount, CommDataType sendtype, int dest,
void *recvbuf, int recvcount, CommDataType recvtype, int source)
{
}
......@@ -150,7 +150,7 @@ namespace Nektar
/**
*
*/
void CommSerial::v_AlltoAll(const void* sendbuf, int sendcount, CommDataType sendtype,
void CommSerial::v_AlltoAll(void* sendbuf, int sendcount, CommDataType sendtype,
void* recvbuf, int recvcount, CommDataType recvtype)
{
......@@ -159,8 +159,8 @@ namespace Nektar
/**
*
*/
void CommSerial::v_AlltoAllv(const void *sendbuf, const int sendcounts[], const int sensdispls[], CommDataType sendtype,
void *recvbuf, const int recvcounts[], const int rdispls[], CommDataType recvtype)
void CommSerial::v_AlltoAllv(void *sendbuf, int sendcounts[], int sensdispls[], CommDataType sendtype,
void *recvbuf, int recvcounts[], int rdispls[], CommDataType recvtype)
{
}
......@@ -170,18 +170,18 @@ namespace Nektar
}
void CommSerial::v_Exscan(const Array<OneD, unsigned long long>& pData, const enum ReduceOperator pOp, Array<OneD, unsigned long long>& ans)
void CommSerial::v_Exscan(Array<OneD, unsigned long long>& pData, const enum ReduceOperator pOp, Array<OneD, unsigned long long>& ans)
{
}
void CommSerial::v_Gather(const void* sendbuf, int sendcount, CommDataType sendtype,
void CommSerial::v_Gather(void* sendbuf, int sendcount, CommDataType sendtype,
void *recvbuf, int recvcount, CommDataType recvtype, int root)
{
std::memcpy(recvbuf, sendbuf, sendcount*CommDataTypeGetSize(sendtype));
}
void CommSerial::v_Scatter(const void *sendbuf, int sendcount, CommDataType sendtype,
void CommSerial::v_Scatter(void *sendbuf, int sendcount, CommDataType sendtype,
void *recvbuf, int recvcount, CommDataType recvtype, int root)
{
std::memcpy(recvbuf, sendbuf, sendcount*CommDataTypeGetSize(sendtype));
......
......@@ -74,22 +74,22 @@ namespace Nektar
LIB_UTILITIES_EXPORT virtual void v_Block();
LIB_UTILITIES_EXPORT virtual double v_Wtime();
LIB_UTILITIES_EXPORT virtual void v_Send(const void* buf, int count, CommDataType dt, int dest);
LIB_UTILITIES_EXPORT virtual void v_Send(void* buf, int count, CommDataType dt, int dest);
LIB_UTILITIES_EXPORT virtual void v_Recv(void* buf, int count, CommDataType dt, int source);
LIB_UTILITIES_EXPORT virtual void v_Sendrecv(const void *sendbuf, int sendcount, CommDataType sendtype, int dest,
LIB_UTILITIES_EXPORT virtual void v_Sendrecv(void *sendbuf, int sendcount, CommDataType sendtype, int dest,
void *recvbuf, int recvcount, CommDataType recvtype, int source);
LIB_UTILITIES_EXPORT virtual void v_SendRecvReplace(void* buf, int count, CommDataType dt,
int pSendProc, int pRecvProc);
LIB_UTILITIES_EXPORT virtual void v_AllReduce(void* buf, int count, CommDataType dt, enum ReduceOperator pOp);
LIB_UTILITIES_EXPORT virtual void v_AlltoAll(const void* sendbuf, int sendcount, CommDataType sendtype,
LIB_UTILITIES_EXPORT virtual void v_AlltoAll(void* sendbuf, int sendcount, CommDataType sendtype,
void* recvbuf, int recvcount, CommDataType recvtype);
LIB_UTILITIES_EXPORT virtual void v_AlltoAllv(const void *sendbuf, const int sendcounts[], const int sensdispls[], CommDataType sendtype,
void *recvbuf, const int recvcounts[], const int rdispls[], CommDataType recvtype);
LIB_UTILITIES_EXPORT virtual void v_AlltoAllv(void *sendbuf, int sendcounts[], int sensdispls[], CommDataType sendtype,
void *recvbuf, int recvcounts[], int rdispls[], CommDataType recvtype);
LIB_UTILITIES_EXPORT virtual void v_Bcast(void* buffer, int count, CommDataType dt, int root);
LIB_UTILITIES_EXPORT virtual void v_Exscan(const Array<OneD, unsigned long long>& pData, const enum ReduceOperator pOp, Array<OneD, unsigned long long>& ans);
LIB_UTILITIES_EXPORT virtual void v_Gather(const void* sendbuf, int sendcount, CommDataType sendtype,
LIB_UTILITIES_EXPORT virtual void v_Exscan(Array<OneD, unsigned long long>& pData, const enum ReduceOperator pOp, Array<OneD, unsigned long long>& ans);
LIB_UTILITIES_EXPORT virtual void v_Gather(void* sendbuf, int sendcount, CommDataType sendtype,
void *recvbuf, int recvcount, CommDataType recvtype, int root);
LIB_UTILITIES_EXPORT virtual void v_Scatter(const void* sendbuf, int sendcount, CommDataType sendtype,
LIB_UTILITIES_EXPORT virtual void v_Scatter(void* sendbuf, int sendcount, CommDataType sendtype,
void *recvbuf, int recvcount, CommDataType recvtype, int root);
LIB_UTILITIES_EXPORT virtual void v_SplitComm(int pRows, int pColumns);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment