Commit c5d95c2f authored by Chris Cantwell's avatar Chris Cantwell

Implementation of GS recovery. Recovers to end of transaction log.

parent 33eaba6e
......@@ -37,6 +37,7 @@
#include <vector>
#include <LibUtilities/Communication/GsLib.hpp>
#include <LibUtilities/BasicUtils/NekFactory.hpp>
#include <LibUtilities/LibUtilitiesDeclspec.h>
#include <boost/enable_shared_from_this.hpp>
......@@ -71,6 +72,9 @@ enum ReduceOperator
ReduceMin
};
typedef int GsHandleId;
struct GsHandle;
/// Base communications class
class Comm : public boost::enable_shared_from_this<Comm>
{
......@@ -128,6 +132,17 @@ public:
LIB_UTILITIES_EXPORT inline bool TreatAsRankZero(void);
LIB_UTILITIES_EXPORT inline bool RemoveExistingFiles(void);
LIB_UTILITIES_EXPORT inline GsHandle GsInit(
const Nektar::Array<OneD, long> pId,
bool verbose = true);
LIB_UTILITIES_EXPORT inline void GsUnique(
const Nektar::Array<OneD, long> pId);
LIB_UTILITIES_EXPORT inline void GsGather(
Nektar::Array<OneD, NekDouble> pU,
Gs::gs_op pOp,
GsHandleId pGsh,
Nektar::Array<OneD, NekDouble> pBuffer = NullNekDouble1DArray);
LIB_UTILITIES_EXPORT inline int EnrolSpare();
LIB_UTILITIES_EXPORT inline void BeginTransactionLog();
LIB_UTILITIES_EXPORT inline void EndTransactionLog();
......@@ -175,6 +190,16 @@ protected:
void *recvbuf, int recvcount, CommDataType recvtype,
int root) = 0;
virtual GsHandle v_GsInit(const Nektar::Array<OneD, long> pId,
bool verbose) = 0;
virtual void v_GsUnique(
const Nektar::Array<OneD, long> pId) = 0;
virtual void v_GsGather(
Nektar::Array<OneD, NekDouble> pU,
Gs::gs_op pOp,
GsHandleId pGsh,
Nektar::Array<OneD, NekDouble> pBuffer) = 0;
virtual CommSharedPtr v_CommCreateIf(int colour) = 0;
virtual void v_SplitComm(int pRows, int pColumns) = 0;
virtual bool v_TreatAsRankZero(void) = 0;
......@@ -186,6 +211,26 @@ protected:
};
struct GsHandle {
GsHandle() : comm(CommSharedPtr()), idx(0) {}
GsHandle(const GsHandle& pSrc) : comm(pSrc.comm), idx(pSrc.idx) {}
GsHandle(CommSharedPtr pComm, GsHandleId pId)
: comm(pComm), idx(pId) {}
void Gather( Nektar::Array<OneD, NekDouble> pU,
Gs::gs_op pOp,
Nektar::Array<OneD, NekDouble> pBuffer = NullNekDouble1DArray) const
{
comm->GsGather(pU, pOp, idx, pBuffer);
}
CommSharedPtr comm;
GsHandleId idx;
};
/**
*
*/
......@@ -393,6 +438,30 @@ template <class T> T Comm::Scatter(const int rootProc, T &pData)
return ans;
}
inline GsHandle Comm::GsInit(
const Nektar::Array<OneD, long> pId,
bool verbose)
{
return v_GsInit(pId, verbose);
}
inline void Comm::GsUnique(
const Nektar::Array<OneD, long> pId)
{
v_GsUnique(pId);
}
inline void Comm::GsGather(
Nektar::Array<OneD, NekDouble> pU,
Gs::gs_op pOp,
GsHandleId pGsh,
Nektar::Array<OneD, NekDouble> pBuffer)
{
v_GsGather(pU, pOp, pGsh, pBuffer);
}
/**
* @brief If the flag is non-zero create a new communicator.
*/
......
......@@ -154,6 +154,10 @@ CommMpi::CommMpi(MPI_Comm pComm) : Comm()
*/
CommMpi::~CommMpi()
{
for (unsigned int i = 0; i < m_gsHandles.size(); ++i)
{
Gs::Finalise(m_gsHandles[i]);
}
MPI_Comm_free(&m_comm);
}
......@@ -460,6 +464,128 @@ void CommMpi::v_Scatter(void *sendbuf, int sendcount, CommDataType sendtype,
ASSERTL0(retval == MPI_SUCCESS, "MPI error performing Scatter.");
}
GsHandle CommMpi::v_GsInit(const Nektar::Array<OneD, long> pId,
bool verbose)
{
CommDataType dt = CommDataTypeTraits<long>::GetDataType();
int count = pId.num_elements();
int dtsize;
MPI_Type_size(dt, &dtsize);
if (m_isRecovering)
{
GsHandle x;
x.comm = shared_from_this();
x.idx = m_gsInitHandles.front();
m_gsInitHandles.pop();
return x;
}
else
{
if (m_isLogging)
{
std::vector<char> x;
if (count > 0)
{
x.assign((char*)(&pId[0]), (char*)(&pId[0])+count*dtsize);
}
m_gsInitData.push(x);
cout << "GsInit: Appended " << dtsize << " bytes of data." << endl;
}
Gs::gs_data * handle = Gs::Init(pId, m_comm, verbose);
GsHandle x;
x.idx = m_gsHandles.size();
x.comm = shared_from_this();
m_gsHandles.push_back(handle);
return x;
}
}
void CommMpi::v_GsUnique(
Nektar::Array<OneD, long> pId)
{
CommDataType dt = CommDataTypeTraits<long>::GetDataType();
int count = pId.num_elements();
int dtsize;
MPI_Type_size(dt, &dtsize);
if (m_isRecovering)
{
// For recovery, just pull out the resulting data from the store
ASSERTL0(!m_data.empty(), "QUEUE IS EMPTY!!");
std::vector<char> x = m_data.front();
m_data.pop();
if (count > 0)
{
memcpy(&pId[0], &x[0], count*dtsize);
}
}
else
{
Gs::Unique(pId, m_comm);
// Record the result in case we need to recover
if (m_isLogging)
{
std::vector<char> x;
if (count > 0)
{
x.assign((char*)(&pId[0]), (char*)(&pId[0])+count*dtsize);
}
m_data.push(x);
cout << "GsInit: Appended " << dtsize << " bytes of data." << endl;
}
}
}
void CommMpi::v_GsGather(
Nektar::Array<OneD, NekDouble> pU,
Gs::gs_op pOp,
GsHandleId pGsh,
Nektar::Array<OneD, NekDouble> pBuffer)
{
CommDataType dt = CommDataTypeTraits<NekDouble>::GetDataType();
int count = pU.num_elements();
int dtsize;
MPI_Type_size(dt, &dtsize);
if (m_isRecovering)
{
// For recovery, just pull out the resulting data from the store
ASSERTL0(!m_data.empty(), "QUEUE IS EMPTY!!");
std::vector<char> x = m_data.front();
m_data.pop();
if (count > 0)
{
memcpy(&pU[0], &x[0], count*dtsize);
}
}
else
{
Gs::Gather(pU, pOp, m_gsHandles[pGsh], pBuffer);
if (m_isLogging)
{
std::vector<char> x;
if (count > 0)
{
x.assign((char*)(&pU[0]), (char*)(&pU[0])+count*dtsize);
}
m_data.push(x);
cout << "GsGather: Appended " << dtsize << " bytes of data." << endl;
}
}
}
/**
* Processes are considered as a grid of size pRows*pColumns. Comm
* objects are created corresponding to the rows and columns of this
......@@ -689,7 +815,7 @@ int CommMpi::v_EnrolSpare()
void CommMpi::BackupState()
{
mpi::communicator c(m_comm, mpi::comm_attach);
mpi::request reqs[4];
mpi::request reqs[6];
int rank = c.rank();
int size = c.size();
......@@ -700,13 +826,16 @@ void CommMpi::BackupState()
cout << "Backup: Sending " << m_data.size() << " items in queue." << endl;
reqs[0] = c.isend(send_rank, 0, m_data);
reqs[1] = c.isend(send_rank, 1, m_derivedCommFlag);
reqs[2] = c.irecv(recv_rank, 0, m_dataBackup);
reqs[3] = c.irecv(recv_rank, 1, m_derivedCommFlagBackup);
reqs[2] = c.isend(send_rank, 2, m_gsInitData);
reqs[3] = c.irecv(recv_rank, 0, m_dataBackup);
reqs[4] = c.irecv(recv_rank, 1, m_derivedCommFlagBackup);
reqs[5] = c.irecv(recv_rank, 2, m_gsInitDataBackup);
cout << "Backup: Sent to " << send_rank << endl;
cout << "Backup: Waiting for data from " << recv_rank << endl;
mpi::wait_all(reqs, reqs + 4);
mpi::wait_all(reqs, reqs + 6);
cout << "Backup: Received " << m_dataBackup.size() << " items." << endl;
cout << "Backup: Received " << m_derivedCommFlagBackup.size() << " derived comm flags." << endl;
cout << "Backup: Received " << m_gsInitDataBackup.size() << " gs init items." << endl;
}
else
{
......@@ -744,7 +873,7 @@ void CommMpi::RestoreState()
// a) backup of the recovering process's data for its recovery
// b) replacement copy of this processes backup data
// c) queue of flags for recovering derived communicators on recovering process
const int nReq = 4;
const int nReq = 6;
if (sendRecoveryData)
{
cout << "Restore: Sending " << m_dataBackup.size() << " backup items." << endl;
......@@ -754,6 +883,8 @@ void CommMpi::RestoreState()
reqs[1] = c.isend(send_rank, 1, m_data);
reqs[2] = c.isend(send_rank, 2, m_derivedCommFlagBackup);
reqs[3] = c.isend(send_rank, 3, m_derivedCommFlag);
reqs[4] = c.isend(send_rank, 4, m_gsInitDataBackup);
reqs[5] = c.isend(send_rank, 5, m_gsInitData);
cout << "Restore: Waiting for data to be sent to " << send_rank << endl;
mpi::wait_all(reqs, reqs + nReq);
cout << "Restore: Complete" << endl;
......@@ -769,6 +900,8 @@ void CommMpi::RestoreState()
reqs[1] = c.irecv(recv_rank, 1, m_dataBackup);
reqs[2] = c.irecv(recv_rank, 2, m_derivedCommFlag);
reqs[3] = c.irecv(recv_rank, 3, m_derivedCommFlagBackup);
reqs[4] = c.irecv(recv_rank, 4, m_gsInitData);
reqs[5] = c.irecv(recv_rank, 5, m_gsInitDataBackup);
cout << "Restore: Waiting for data from " << recv_rank << endl;
mpi::wait_all(reqs, reqs + nReq);
cout << "Restore: Complete" << endl;
......@@ -835,7 +968,35 @@ cout << "Colour is " << colour << endl;
cout << "### Finished restore of sub-communicator ###" << endl << endl;
derivedCommIt++;
}
}
// Fix GSLib handles
CommDataType dt = CommDataTypeTraits<long>::GetDataType();
int dtsize;
MPI_Type_size(dt, &dtsize);
int n = m_gsInitData.size();
StorageType vInitData = m_gsInitData;
cout << "Restoring " << n << " GS handles" << endl;
for (int i = 0; i < n; ++i)
{
std::vector<char> x = vInitData.front();
vInitData.pop();
int count = x.size() / dtsize;
Array<OneD, long> pId(count);
cout << "Data size was " << x.size() << ", Array size: " << count << endl;
if (count > 0)
{
memcpy(&pId[0], &x[0], x.size());
}
Gs::gs_data * handle = Gs::Init(pId, m_comm, false);
if (!m_isRecovering)
{
m_gsHandles.push_back(handle);
}
}
}
......@@ -875,7 +1036,6 @@ void CommMpi::v_BeginTransactionLog()
void CommMpi::v_EndTransactionLog()
{
m_isLogging = false;
m_isRecovering = false;
for (auto x : m_derivedComm)
{
......@@ -883,7 +1043,14 @@ void CommMpi::v_EndTransactionLog()
x->EndTransactionLog();
}
BackupState();
if (m_isRecovering)
{
m_isRecovering = false;
}
else
{
BackupState();
}
}
void CommMpi::ReplaceComm(MPI_Comm commptr)
......
......@@ -134,6 +134,16 @@ protected:
void *recvbuf, int recvcount, CommDataType recvtype,
int root);
virtual GsHandle v_GsInit(const Nektar::Array<OneD, long> pId,
bool verbose);
virtual void v_GsUnique(
const Nektar::Array<OneD, long> pId);
virtual void v_GsGather(
Nektar::Array<OneD, NekDouble> pU,
Gs::gs_op pOp,
GsHandleId pGsh,
Nektar::Array<OneD, NekDouble> pBuffer);
virtual void v_SplitComm(int pRows, int pColumns);
virtual CommSharedPtr v_CommCreateIf(int flag);
......@@ -153,6 +163,10 @@ private:
DerivedCommType m_derivedComm; ///< Temporary derived comm list used during restore
DerivedCommFlagType m_derivedCommFlag; ///< Log derived comm flags
DerivedCommFlagType m_derivedCommFlagBackup; ///< Backup of neighbour flags
StorageType m_gsInitData;
StorageType m_gsInitDataBackup;
std::queue<GsHandleId> m_gsInitHandles;
std::vector<Gs::gs_data*> m_gsHandles; ///< Handles to Gs library
static void HandleMpiError(MPI_Comm* pcomm, int* perr, ...);
......
......@@ -37,6 +37,7 @@
#include <string>
#include <LibUtilities/Communication/GsLib.hpp>
#include <LibUtilities/Communication/Comm.h>
#include <LibUtilities/LibUtilitiesDeclspec.h>
#include <LibUtilities/Memory/NekMemoryManager.hpp>
......@@ -112,6 +113,21 @@ protected:
CommDataType recvtype,
int root);
virtual GsHandle v_GsInit(const Nektar::Array<OneD, long> pId,
bool verbose) {
GsHandle x;
x.comm = shared_from_this();
x.idx = 0;
return x;
}
virtual void v_GsUnique(
const Nektar::Array<OneD, long> pId) {}
virtual void v_GsGather(
Nektar::Array<OneD, NekDouble> pU,
Gs::gs_op pOp,
GsHandleId pGsh,
Nektar::Array<OneD, NekDouble> pBuffer) {}
LIB_UTILITIES_EXPORT virtual void v_SplitComm(int pRows, int pColumns);
LIB_UTILITIES_EXPORT virtual CommSharedPtr v_CommCreateIf(int flag);
......
......@@ -40,9 +40,9 @@
#include <LibUtilities/BasicConst/NektarUnivTypeDefs.hpp>
#include <LibUtilities/BasicUtils/SharedArray.hpp>
#ifdef NEKTAR_USE_MPI
#include <LibUtilities/Communication/Comm.h>
#endif
//#ifdef NEKTAR_USE_MPI
//#include <LibUtilities/Communication/Comm.h>
//#endif
namespace Gs
{
......@@ -164,21 +164,22 @@ void nektar_gs_unique(const long *id, unsigned int n, const struct comm *comm);
* @return GSLib data structure containing mapping information.
*/
static inline gs_data *Init(const Nektar::Array<OneD, long> pId,
const LibUtilities::CommSharedPtr &pComm,
MPI_Comm pComm,
bool verbose = true)
{
#ifdef NEKTAR_USE_MPI
if (pComm->GetSize() == 1)
int size, rank;
MPI_Comm_size(pComm, &size);
MPI_Comm_rank(pComm, &rank);
if (size == 1)
{
return 0;
}
//LibUtilities::CommMpiSharedPtr vCommMpi =
// boost::dynamic_pointer_cast<LibUtilities::CommMpi>(pComm);
//ASSERTL1(vCommMpi, "Failed to cast MPI Comm object.");
comm vComm;
MPI_Comm_dup((MPI_Comm)(pComm->GetComm()), &vComm.c);
vComm.id = pComm->GetRank();
vComm.np = pComm->GetSize();
MPI_Comm_dup(pComm, &vComm.c);
vComm.id = rank;
vComm.np = size;
gs_data *result = nektar_gs_setup(pId.get(), pId.num_elements(), &vComm, 0,
gs_auto, (int)verbose);
MPI_Comm_free(&vComm.c);
......@@ -198,20 +199,21 @@ static inline gs_data *Init(const Nektar::Array<OneD, long> pId,
* be included only once.
*/
static inline void Unique(const Nektar::Array<OneD, long> pId,
const LibUtilities::CommSharedPtr &pComm)
MPI_Comm pComm)
{
#ifdef NEKTAR_USE_MPI
if (pComm->GetSize() == 1)
int size, rank;
MPI_Comm_size(pComm, &size);
MPI_Comm_rank(pComm, &rank);
if (size == 1)
{
return;
}
// LibUtilities::CommMpiSharedPtr vCommMpi =
// boost::dynamic_pointer_cast<LibUtilities::CommMpi>(pComm);
// ASSERTL1(vCommMpi, "Failed to cast MPI Comm object.");
comm vComm;
vComm.c = (MPI_Comm)(pComm->GetComm());
vComm.id = pComm->GetRank();
vComm.np = pComm->GetSize();
vComm.c = pComm;
vComm.id = rank;
vComm.np = size;
nektar_gs_unique(pId.get(), pId.num_elements(), &vComm);
#endif
}
......
......@@ -172,7 +172,7 @@ static inline struct crs_data *Init(
boost::dynamic_pointer_cast<LibUtilities::CommMpi>(pComm);
ASSERTL1(vCommMpi, "Failed to cast MPI Comm object.");
comm vComm;
MPI_Comm_dup(vCommMpi->GetComm(), &vComm.c);
MPI_Comm_dup((MPI_Comm)(vCommMpi->GetComm()), &vComm.c);
vComm.id = vCommMpi->GetRank();
vComm.np = vCommMpi->GetSize();
crs_data *result = nektar_crs_setup(pRank, &pId[0], nz, &pAi[0], &pAj[0],
......
......@@ -87,8 +87,8 @@ namespace Nektar
m_solnType(eNoSolnType),
m_bndSystemBandWidth(0),
m_successiveRHS(0),
m_gsh(0),
m_bndGsh(0)
m_gsh(),
m_bndGsh()
{
}
......@@ -104,8 +104,8 @@ namespace Nektar
m_numGlobalDirBndCoeffs(0),
m_bndSystemBandWidth(0),
m_successiveRHS(0),
m_gsh(0),
m_bndGsh(0)
m_gsh(),
m_bndGsh()
{
// Default value from Solver Info
m_solnType = pSession->GetSolverInfoAsEnum<GlobalSysSolnType>(
......@@ -1130,7 +1130,7 @@ namespace Nektar
{
ASSERTL1(pGlobal.num_elements() == m_numGlobalBndCoeffs,
"Wrong size.");
Gs::Gather(pGlobal, Gs::gs_add, m_bndGsh);
m_bndGsh.Gather(pGlobal, Gs::gs_add);
}
void AssemblyMap::UniversalAssembleBnd(
......
......@@ -376,8 +376,8 @@ namespace Nektar
/// sucessive RHS for iterative solver
int m_successiveRHS;
Gs::gs_data * m_gsh;
Gs::gs_data * m_bndGsh;
LibUtilities::GsHandle m_gsh;
LibUtilities::GsHandle m_bndGsh;
/// The level of recursion in the case of multi-level static
/// condensation.
......
......@@ -1119,27 +1119,27 @@ namespace Nektar
// the vertices and edges respectively to identify those
// vertices and edges which are located on partition boundary.
Array<OneD, long> vertArray(unique_verts, &procVerts[0]);
Gs::gs_data *tmp1 = Gs::Init(vertArray, vComm, verbose);
LibUtilities::GsHandle tmp1 = vComm->GsInit(vertArray, verbose);
Array<OneD, NekDouble> tmp4(unique_verts, 1.0);
Array<OneD, NekDouble> tmp5(unique_edges, 1.0);
Array<OneD, NekDouble> tmp6(unique_faces, 1.0);
Gs::Gather(tmp4, Gs::gs_add, tmp1);
Gs::Finalise(tmp1);
tmp1.Gather(tmp4, Gs::gs_add);
//Gs::Finalise(tmp1);
if (unique_edges > 0)
{
Array<OneD, long> edgeArray(unique_edges, &procEdges[0]);
Gs::gs_data *tmp2 = Gs::Init(edgeArray, vComm, verbose);
Gs::Gather(tmp5, Gs::gs_add, tmp2);
Gs::Finalise(tmp2);
LibUtilities::GsHandle tmp2 = vComm->GsInit(edgeArray, verbose);
tmp2.Gather(tmp5, Gs::gs_add);
//Gs::Finalise(tmp2);
}
if (unique_faces > 0)
{
Array<OneD, long> faceArray(unique_faces, &procFaces[0]);
Gs::gs_data *tmp3 = Gs::Init(faceArray, vComm, verbose);
Gs::Gather(tmp6, Gs::gs_add, tmp3);
Gs::Finalise(tmp3);
LibUtilities::GsHandle tmp3 = vComm->GsInit(faceArray, verbose);
tmp3.Gather(tmp6, Gs::gs_add);
//Gs::Finalise(tmp3);
}
// Finally, fill the partVerts set with all non-Dirichlet
......@@ -1438,9 +1438,9 @@ namespace Nektar
edgeId[i] = dofIt->first + 1;
edgeDof[i] = (NekDouble) dofIt->second;
}
Gs::gs_data *tmp = Gs::Init(edgeId, vComm, verbose);
Gs::Gather(edgeDof, Gs::gs_min, tmp);
Gs::Finalise(tmp);
LibUtilities::GsHandle tmp = vComm->GsInit(edgeId, verbose);
tmp.Gather(edgeDof, Gs::gs_min);
//Gs::Finalise(tmp);
for (i=0; i < dofs[1].size(); i++)
{
dofs[1][edgeId[i]-1] = (int) (edgeDof[i]+0.5);
......@@ -1469,10 +1469,10 @@ namespace Nektar
faceP[i] = (NekDouble) dofIt->second;
faceQ[i] = (NekDouble) dofIt2->second;
}
Gs::gs_data *tmp2 = Gs::Init(faceId, vComm, verbose);
Gs::Gather(faceP, Gs::gs_min, tmp2);
Gs::Gather(faceQ, Gs::gs_min, tmp2);
Gs::Finalise(tmp2);
LibUtilities::GsHandle tmp2 = vComm->GsInit(faceId, verbose);
tmp2.Gather(faceP, Gs::gs_min);
tmp2.Gather(faceQ, Gs::gs_min);
//Gs::Finalise(tmp2);
for (i=0; i < faceModes[0].size(); i++)
{
faceModes[0][faceId[i]-1] = (int) (faceP[i]+0.5);
......@@ -2048,8 +2048,8 @@ namespace Nektar
*/
AssemblyMapCG::~AssemblyMapCG()
{
Gs::Finalise(m_gsh);
Gs::Finalise(m_bndGsh);
// Gs::Finalise(m_gsh);
// Gs::Finalise(m_bndGsh);
}
/**
......@@ -2363,9 +2363,9 @@ namespace Nektar
tmp[i] = m_globalToUniversalMap[i];
}
m_gsh = Gs::Init(tmp, vCommRow, verbose);
m_bndGsh = Gs::Init(tmp2, vCommRow, verbose);
Gs::Unique(tmp, vCommRow);
m_gsh = vCommRow->GsInit(tmp, verbose);
m_bndGsh = vCommRow->GsInit(tmp2, verbose);
vCommRow->GsUnique(tmp);
for (unsigned int i = 0; i < m_numGlobalCoeffs; ++i)
{
m_globalToUniversalMapUnique[i] = (tmp[i] >= 0 ? 1 : 0);
......@@ -2491,8 +2491,8 @@ namespace Nektar
{
tmp[i] = returnval->m_globalToUniversalMap[i];
}
returnval->m_gsh = Gs::Init(tmp, vCommRow, verbose);
Gs::Unique(tmp, vCommRow);
returnval->m_gsh = vCommRow->GsInit(tmp, verbose);
vCommRow->GsUnique(tmp);
for (unsigned int i = 0; i < nglocoeffs; ++i)
{
returnval->m_globalToUniversalMapUnique[i]
......@@ -2637,7 +2637,7 @@ namespace Nektar
}
// ensure all values are unique by calling a max
Gs::Gather(global, Gs::gs_max, m_gsh);
m_gsh.Gather(global, Gs::gs_max);
}
void AssemblyMapCG::v_LocalToGlobal(
......@@ -2717,7 +2717,7 @@ namespace Nektar
void AssemblyMapCG::v_UniversalAssemble(
Array<OneD, NekDouble>& pGlobal) const
{
Gs::Gather(pGlobal, Gs::gs_add, m_gsh);
m_gsh.Gather(pGlobal, Gs::gs_add);
}
void AssemblyMapCG::v_UniversalAssemble(
......
......@@ -704,8 +704,8 @@ namespace Nektar
{
tmp[i] = m_globalToUniversalBndMap[i];
}
m_bndGsh = m_gsh = Gs::Init(tmp, m_comm);
Gs::Unique(tmp, m_comm);
m_bndGsh = m_gsh = m_comm->GsInit(tmp);
m_comm->GsUnique(tmp);
for (i = 0; i < m_globalToUniversalBndMap.num_elements(); ++i)
{
m_globalToUniversalBndMapUnique[i] = (tmp[i] >= 0 ? 1 : 0);
......@@ -829,8 +829,8 @@ namespace Nektar
{
tmp2[i] = m_traceToUniversalMap[i];
}
m_traceGsh = Gs::Init(tmp2, m_comm);
Gs::Unique(tmp2, m_comm);
m_traceGsh = m_comm->GsInit(tmp2);
m_comm->GsUnique(tmp2);
for (int i = 0; i < nTracePhys; ++i)
{
m_traceToUniversalMapUnique[i] = tmp2[i];
......@@ -920,7 +920,7 @@ namespace Nektar
void AssemblyMapDG::UniversalTraceAssemble(
Array<OneD, NekDouble> &pGlobal) const
{
Gs::Gather(pGlobal, Gs::gs_add, m_traceGsh);
m_traceGsh.Gather(pGlobal, Gs::gs_add);
}