Skip to content

GitLab

Commit 78e291d3 authored by Alessandro Bolis's avatar Alessandro Bolis
Browse files

Parallelisation INS (homogeneous and gslib), parallel output, 

regression tests, fixed bug in communication class


git-svn-id: https://gforge.sci.utah.edu/svn/nektar/trunk@3761 305cdda6-5ce1-45b3-a98d-dfc68c8b3305
parent c64aef1a
Hide whitespace changes
Inline Side-by-side
library/LibUtilities/BasicUtils/SessionReader.cpp
View file @ 78e291d3
......@@ -1110,10 +1110,20 @@ namespace Nektar
ASSERTL0(e, "Unable to find CONDITIONS tag in file.");
ReadParameters(e);
if (m_comm->GetSize() > 1 && DefinesParameter("PROC_X"))
if (m_comm->GetSize() > 1)
{
int nProcX;
LoadParameter("PROC_X", nProcX, 1);
if(DefinesParameter("PROC_X"))
{
LoadParameter("PROC_X", nProcX, 1);
}
else
{
nProcX = 1;
}
ASSERTL0(m_comm->GetSize() % nProcX == 0,
"Cannot exactly partition using PROC_X value.");
int nProcSem = m_comm->GetSize() / nProcX;
......
library/LibUtilities/Communication/Comm.h
View file @ 78e291d3
......@@ -108,6 +108,18 @@ namespace Nektar
Array<OneD, NekDouble>& pRecvData);
LIB_UTILITIES_EXPORT inline void AlltoAll(Array<OneD, int>& pSendData,
Array<OneD, int>& pRecvData);
LIB_UTILITIES_EXPORT inline void AlltoAllv(Array<OneD, NekDouble>& pSendData,
Array<OneD, int>& pSendDataSizeMap,
Array<OneD, int>& pSendDataOffsetMap,
Array<OneD, NekDouble>& pRecvData,
Array<OneD, int>& pRecvDataSizeMap,
Array<OneD, int>& pRecvDataOffsetMap);
LIB_UTILITIES_EXPORT inline void AlltoAllv(Array<OneD, int>& pSendData,
Array<OneD, int>& pSendDataSizeMap,
Array<OneD, int>& pSendDataOffsetMap,
Array<OneD, int>& pRecvData,
Array<OneD, int>& pRecvDataSizeMap,
Array<OneD, int>& pRecvDataOffsetMap);
LIB_UTILITIES_EXPORT inline void SplitComm(int pRows, int pColumns);
LIB_UTILITIES_EXPORT inline CommSharedPtr GetRowComm();
......@@ -154,6 +166,18 @@ namespace Nektar
Array<OneD, NekDouble>& pRecvData) = 0;
virtual void v_AlltoAll(Array<OneD, int>& pSendData,
Array<OneD, int>& pRecvData) = 0;
virtual void v_AlltoAllv(Array<OneD, NekDouble>& pSendData,
Array<OneD, int>& pSendDataSizeMap,
Array<OneD, int>& pSendDataOffsetMap,
Array<OneD, NekDouble>& pRecvData,
Array<OneD, int>& pRecvDataSizeMap,
Array<OneD, int>& pRecvDataOffsetMap) = 0;
virtual void v_AlltoAllv(Array<OneD, int>& pSendData,
Array<OneD, int>& pSendDataSizeMap,
Array<OneD, int>& pSendDataOffsetMap,
Array<OneD, int>& pRecvData,
Array<OneD, int>& pRecvDataSizeMap,
Array<OneD, int>& pRecvDataOffsetMap) = 0;
virtual void v_SplitComm(int pRows, int pColumns) = 0;
};
......@@ -328,6 +352,33 @@ namespace Nektar
{
v_AlltoAll(pSendData,pRecvData);
}
/**
*
*/
inline void Comm::AlltoAllv(Array<OneD, NekDouble>& pSendData,
Array<OneD, int>& pSendDataSizeMap,
Array<OneD, int>& pSendDataOffsetMap,
Array<OneD, NekDouble>& pRecvData,
Array<OneD, int>& pRecvDataSizeMap,
Array<OneD, int>& pRecvDataOffsetMap)
{
v_AlltoAllv(pSendData,pSendDataSizeMap,pSendDataOffsetMap,pRecvData,pRecvDataSizeMap,pRecvDataOffsetMap);
}
/**
*
*/
inline void Comm::AlltoAllv(Array<OneD, int>& pSendData,
Array<OneD, int>& pSendDataSizeMap,
Array<OneD, int>& pSendDataOffsetMap,
Array<OneD, int>& pRecvData,
Array<OneD, int>& pRecvDataSizeMap,
Array<OneD, int>& pRecvDataOffsetMap)
{
v_AlltoAllv(pSendData,pSendDataSizeMap,pSendDataOffsetMap,pRecvData,pRecvDataSizeMap,pRecvDataOffsetMap);
}
/**
......
library/LibUtilities/Communication/CommMpi.cpp
View file @ 78e291d3
......@@ -94,7 +94,7 @@ namespace Nektar
*/
MPI_Comm CommMpi::GetComm()
{
return m_comm;;
return m_comm;
}
......@@ -162,7 +162,7 @@ namespace Nektar
MPI_DOUBLE,
pProc,
0,
MPI_COMM_WORLD,
m_comm,
&status);
ASSERTL0(status.MPI_ERROR == MPI_SUCCESS,
......@@ -182,7 +182,7 @@ namespace Nektar
MPI_INT,
pProc,
0,
MPI_COMM_WORLD);
m_comm);
}
else
{
......@@ -191,7 +191,7 @@ namespace Nektar
MPI_INT,
pProc,
0,
MPI_COMM_WORLD);
m_comm);
}
}
......@@ -207,7 +207,7 @@ namespace Nektar
MPI_INT,
pProc,
0,
MPI_COMM_WORLD,
m_comm,
&status);
ASSERTL0(status.MPI_ERROR == MPI_SUCCESS,
......@@ -234,7 +234,7 @@ namespace Nektar
MPI_DOUBLE,
pSendProc,
0,
MPI_COMM_WORLD,
m_comm,
&status);
ASSERTL0(retval == MPI_SUCCESS,
......@@ -261,7 +261,7 @@ namespace Nektar
MPI_INT,
pSendProc,
0,
MPI_COMM_WORLD,
m_comm,
&status);
ASSERTL0(retval == MPI_SUCCESS,
......@@ -283,7 +283,7 @@ namespace Nektar
0,
pSendProc,
0,
MPI_COMM_WORLD,
m_comm,
&status);
ASSERTL0(retval == MPI_SUCCESS,
......@@ -307,7 +307,7 @@ namespace Nektar
0,
pSendProc,
0,
MPI_COMM_WORLD,
m_comm,
&status);
ASSERTL0(retval == MPI_SUCCESS,
......@@ -332,7 +332,7 @@ namespace Nektar
1,
MPI_DOUBLE,
vOp,
MPI_COMM_WORLD);
m_comm);
ASSERTL0(retval == MPI_SUCCESS,
"MPI error performing All-reduce.");
......@@ -356,7 +356,7 @@ namespace Nektar
1,
MPI_INT,
vOp,
MPI_COMM_WORLD);
m_comm);
ASSERTL0(retval == MPI_SUCCESS,
"MPI error performing All-reduce.");
......@@ -380,7 +380,7 @@ namespace Nektar
(int) pData.num_elements(),
MPI_DOUBLE,
vOp,
MPI_COMM_WORLD);
m_comm);
ASSERTL0(retval == MPI_SUCCESS,
"MPI error performing All-reduce.");
......@@ -404,7 +404,7 @@ namespace Nektar
(int) pData.num_elements(),
MPI_INT,
vOp,
MPI_COMM_WORLD);
m_comm);
ASSERTL0(retval == MPI_SUCCESS,
"MPI error performing All-reduce.");
......@@ -422,7 +422,7 @@ namespace Nektar
pRecvData.get(),
(int) pRecvData.num_elements(),
MPI_DOUBLE,
MPI_COMM_WORLD);
m_comm);
ASSERTL0(retval == MPI_SUCCESS,
"MPI error performing All-to-All.");
......@@ -440,11 +440,60 @@ namespace Nektar
pRecvData.get(),
(int) pRecvData.num_elements(),
MPI_INT,
MPI_COMM_WORLD);
m_comm);
ASSERTL0(retval == MPI_SUCCESS,
"MPI error performing All-to-All.");
}
/**
*
*/
void CommMpi::v_AlltoAllv(Array<OneD, NekDouble>& pSendData,
Array<OneD, int>& pSendDataSizeMap,
Array<OneD, int>& pSendDataOffsetMap,
Array<OneD, NekDouble>& pRecvData,
Array<OneD, int>& pRecvDataSizeMap,
Array<OneD, int>& pRecvDataOffsetMap)
{
int retval = MPI_Alltoallv(pSendData.get(),
pSendDataSizeMap.get(),
pSendDataOffsetMap.get(),
MPI_DOUBLE,
pRecvData.get(),
pRecvDataSizeMap.get(),
pRecvDataOffsetMap.get(),
MPI_DOUBLE,
m_comm);
ASSERTL0(retval == MPI_SUCCESS,
"MPI error performing All-to-All-v.");
}
/**
*
*/
void CommMpi::v_AlltoAllv(Array<OneD, int>& pSendData,
Array<OneD, int>& pSendDataSizeMap,
Array<OneD, int>& pSendDataOffsetMap,
Array<OneD, int>& pRecvData,
Array<OneD, int>& pRecvDataSizeMap,
Array<OneD, int>& pRecvDataOffsetMap)
{
int retval = MPI_Alltoallv(pSendData.get(),
pSendDataSizeMap.get(),
pSendDataOffsetMap.get(),
MPI_INT,
pRecvData.get(),
pRecvDataSizeMap.get(),
pRecvDataOffsetMap.get(),
MPI_INT,
m_comm);
ASSERTL0(retval == MPI_SUCCESS,
"MPI error performing All-to-All-v.");
}
/**
......
library/LibUtilities/Communication/CommMpi.h
View file @ 78e291d3
......@@ -109,6 +109,18 @@ namespace Nektar
Array<OneD, NekDouble>& pRecvData);
virtual void v_AlltoAll(Array<OneD, int>& pSendData,
Array<OneD, int>& pRecvData);
virtual void v_AlltoAllv(Array<OneD, NekDouble>& pSendData,
Array<OneD, int>& pSendDataSizeMap,
Array<OneD, int>& pSendDataOffsetMap,
Array<OneD, NekDouble>& pRecvData,
Array<OneD, int>& pRecvDataSizeMap,
Array<OneD, int>& pRecvDataOffsetMap);
virtual void v_AlltoAllv(Array<OneD, int>& pSendData,
Array<OneD, int>& pSendDataSizeMap,
Array<OneD, int>& pSendDataOffsetMap,
Array<OneD, int>& pRecvData,
Array<OneD, int>& pRecvDataSizeMap,
Array<OneD, int>& pRecvDataOffsetMap);
virtual void v_SplitComm(int pRows, int pColumns);
private:
......
library/LibUtilities/Communication/CommSerial.cpp
View file @ 78e291d3
......@@ -207,6 +207,34 @@ namespace Nektar
{
}
/**
*
*/
void CommSerial::v_AlltoAllv(Array<OneD, NekDouble>& pSendData,
Array<OneD, int>& pSendDataSizeMap,
Array<OneD, int>& pSendDataOffsetMap,
Array<OneD, NekDouble>& pRecvData,
Array<OneD, int>& pRecvDataSizeMap,
Array<OneD, int>& pRecvDataOffsetMap)
{
}
/**
*
*/
void CommSerial::v_AlltoAllv(Array<OneD, int>& pSendData,
Array<OneD, int>& pSendDataSizeMap,
Array<OneD, int>& pSendDataOffsetMap,
Array<OneD, int>& pRecvData,
Array<OneD, int>& pRecvDataSizeMap,
Array<OneD, int>& pRecvDataOffsetMap)
{
}
/**
......
library/LibUtilities/Communication/CommSerial.h
View file @ 78e291d3
......@@ -101,6 +101,18 @@ namespace Nektar
Array<OneD, NekDouble>& pRecvData);
LIB_UTILITIES_EXPORT virtual void v_AlltoAll(Array<OneD, int>& pSendData,
Array<OneD, int>& pRecvData);
LIB_UTILITIES_EXPORT virtual void v_AlltoAllv(Array<OneD, NekDouble>& pSendData,
Array<OneD, int>& pSendDataSizeMap,
Array<OneD, int>& pSendDataOffsetMap,
Array<OneD, NekDouble>& pRecvData,
Array<OneD, int>& pRecvDataSizeMap,
Array<OneD, int>& pRecvDataOffsetMap);
LIB_UTILITIES_EXPORT virtual void v_AlltoAllv(Array<OneD, int>& pSendData,
Array<OneD, int>& pSendDataSizeMap,
Array<OneD, int>& pSendDataOffsetMap,
Array<OneD, int>& pRecvData,
Array<OneD, int>& pRecvDataSizeMap,
Array<OneD, int>& pRecvDataOffsetMap);
LIB_UTILITIES_EXPORT virtual void v_SplitComm(int pRows, int pColumns);
};
......
library/MultiRegions/ExpListHomogeneous1D.cpp
View file @ 78e291d3
......@@ -58,7 +58,7 @@ namespace Nektar
{
ASSERTL2(HomoBasis != LibUtilities::NullBasisKey,"Homogeneous Basis is a null basis");
m_homogeneousBasis = LibUtilities::BasisManager()[HomoBasis];
SetParalleInfo();
m_planes = Array<OneD,ExpListSharedPtr>(m_num_planes_per_proc);
......@@ -73,7 +73,7 @@ namespace Nektar
ASSERTL0(m_num_processes == 1,"Remove dealiasing if you want to run in parallel");
SetPaddingBase();
}
}
}
/**
......@@ -342,7 +342,7 @@ namespace Nektar
* Inner product element by element
*/
void ExpListHomogeneous1D::v_IProductWRTBase_IterPerExp(const Array<OneD, const NekDouble> &inarray, Array<OneD, NekDouble> &outarray)
{
{
int cnt = 0, cnt1 = 0;
Array<OneD, NekDouble> tmparray;
......@@ -352,7 +352,7 @@ namespace Nektar
cnt1 += m_planes[n]->GetNcoeffs();
cnt += m_planes[n]->GetTotPoints();
}
}
}
/**
......@@ -448,12 +448,13 @@ namespace Nektar
int num_dfts_per_proc = num_points_per_plane/m_num_processes + (num_points_per_plane%m_num_processes > 0);
int copy_len = num_dfts_per_proc;
Array<OneD, Array< OneD, NekDouble> > tmp_inarray(m_num_processes);
for(i = 0; i < m_num_processes;i++)
Array<OneD, int> SizeMap(m_num_processes,0);
Array<OneD, int> OffsetMap(m_num_processes,0);
Array< OneD, NekDouble> tmp_outarray(num_dfts_per_proc*m_num_fourier_points,0.0);
for(i = 0; i < m_num_processes ; i++)
{
tmp_inarray[i] = Array<OneD, NekDouble> (num_dfts_per_proc*m_num_planes_per_proc,0.0);
if(i == m_num_processes-1)
{
copy_len = num_dfts_per_proc - ((num_dfts_per_proc*m_num_processes) - num_points_per_plane);
......@@ -463,17 +464,15 @@ namespace Nektar
{
Vmath::Vcopy(copy_len,
&(inarray[i*num_dfts_per_proc+j*num_points_per_plane]),1,
&(tmp_inarray[i][j*num_dfts_per_proc]),1);
}
}
for(i = 0; i < m_num_processes;i++)
{
if(i != m_rank_id)
{
m_comm->GetColumnComm()->SendRecvReplace(m_rank_id,i,tmp_inarray[i]);
&(outarray[i*num_dfts_per_proc*m_num_planes_per_proc+j*num_dfts_per_proc]),1);
}
SizeMap[i] = num_dfts_per_proc*m_num_planes_per_proc;
OffsetMap[i] = i*num_dfts_per_proc*m_num_planes_per_proc;
}
m_comm->GetColumnComm()->AlltoAllv(outarray,SizeMap,OffsetMap,tmp_outarray,SizeMap,OffsetMap);
// reshuffle
int packed_len;
......@@ -490,7 +489,7 @@ namespace Nektar
for(i = 0; i < packed_len; ++i)
{
Vmath::Vcopy(num_dfts_per_proc,&(tmp_inarray[0][i*num_dfts_per_proc]),1,
Vmath::Vcopy(num_dfts_per_proc,&(tmp_outarray[i*num_dfts_per_proc]),1,
&(outarray[i]),packed_len);
}
}
......@@ -507,7 +506,13 @@ namespace Nektar
int num_dofs = inarray.num_elements();
int num_points_per_plane = num_dofs/m_num_planes_per_proc;
int num_dfts_per_proc = num_points_per_plane/m_num_processes + (num_points_per_plane%m_num_processes > 0);
int copy_len;
int copy_len = num_dfts_per_proc;
Array<OneD, int> SizeMap(m_num_processes,0);
Array<OneD, int> OffsetMap(m_num_processes,0);
Array< OneD, NekDouble> tmp_inarray(num_dfts_per_proc*m_num_fourier_points,0.0);
Array< OneD, NekDouble> tmp_outarray(num_dfts_per_proc*m_num_fourier_points,0.0);
int packed_len;
......@@ -522,26 +527,22 @@ namespace Nektar
ASSERTL1(&inarray[0] != &outarray[0],"Inarray and outarray cannot be the same");
Array<OneD, Array< OneD, NekDouble> > tmp_inarray(m_num_processes);
for(i = 0; i < m_num_processes;i++)
{
tmp_inarray[i] = Array<OneD, NekDouble> (num_dfts_per_proc*m_num_planes_per_proc,0.0);
}
for(i = 0; i < packed_len; ++i)
{
Vmath::Vcopy(num_dfts_per_proc,&(inarray[i]),packed_len,
&(tmp_inarray[0][i*num_dfts_per_proc]),1);
&(tmp_inarray[i*num_dfts_per_proc]),1);
}
for(i = 0; i < m_num_processes ; i++)
{
SizeMap[i] = num_dfts_per_proc*m_num_planes_per_proc;
OffsetMap[i] = i*num_dfts_per_proc*m_num_planes_per_proc;
}
m_comm->GetColumnComm()->AlltoAllv(tmp_inarray,SizeMap,OffsetMap,tmp_outarray,SizeMap,OffsetMap);
for(i = 0; i < m_num_processes;i++)
{
if(i != m_rank_id)
{
m_comm->GetColumnComm()->SendRecvReplace(m_rank_id,i,tmp_inarray[i]);
}
{
if(i == m_num_processes-1)
{
copy_len = num_dfts_per_proc - ((num_dfts_per_proc*m_num_processes)-num_points_per_plane);
......@@ -550,7 +551,7 @@ namespace Nektar
for(j = 0; j < m_num_planes_per_proc;j++)
{
Vmath::Vcopy(copy_len,
&(tmp_inarray[i][j*num_dfts_per_proc]),1,
&(tmp_outarray[i*num_dfts_per_proc*m_num_planes_per_proc+j*num_dfts_per_proc]),1,
&(outarray[i*num_dfts_per_proc+j*num_points_per_plane]),1);
}
}
......@@ -658,7 +659,7 @@ namespace Nektar
std::vector<NekDouble> HomoLen;
HomoLen.push_back(m_lhom);
std::vector<unsigned int> PlanesIDs(m_num_planes_per_proc);
std::vector<unsigned int> PlanesIDs;
for(int i = 0; i < m_num_planes_per_proc; i++)
{
PlanesIDs.push_back(m_planes_IDs[i]);
......@@ -677,7 +678,7 @@ namespace Nektar
std::vector<NekDouble> HomoLen;
HomoLen.push_back(m_lhom);
std::vector<unsigned int> PlanesIDs(m_num_planes_per_proc);
std::vector<unsigned int> PlanesIDs;
for(int i = 0; i < m_num_planes_per_proc; i++)
{
PlanesIDs.push_back(m_planes_IDs[i]);
......
library/SpatialDomains/MeshGraph.cpp
View file @ 78e291d3
......@@ -1875,7 +1875,7 @@ namespace Nektar
if(fielddefs->m_numHomogeneousDir == 1)
{
datasize += StdRegions::StdTriData::getNumberOfCoefficients(l,m)*fielddefs->m_numModes[cnt++];
datasize += StdRegions::StdTriData::getNumberOfCoefficients(l,m)*fielddefs->m_homogeneousZIDs.size();
}
else
{
......@@ -1889,7 +1889,7 @@ namespace Nektar
{
datasize += fielddefs->m_numModes[cnt++]*
fielddefs->m_numModes[cnt++]*
fielddefs->m_numModes[cnt++];
fielddefs->m_homogeneousZIDs.size();
}
else
{
......
regressionTests/Solvers/IncNavierStokesSolver/IncNavierStokesSolverTests.cpp
View file @ 78e291d3
......@@ -147,6 +147,8 @@ int main(int argc, char* argv[])
ExecuteParallel("IncNavierStokesSolver","Test_Hex_channel_m8_par.xml","3D channel flow, Hex elements, par(2), P=8", 2);
ExecuteParallel("IncNavierStokesSolver","Test_Tet_channel_m8_par.xml","3D channel flow, Tet elements, par(2), P=8", 2);
ExecuteParallel("IncNavierStokesSolver","Test_ChanFlow_m3_par.xml","3D channel flow, 2D, par(2), P=8", 2);
ExecuteParallel("IncNavierStokesSolver","Test_ChanFlow_3DH1D_Parallel_mode1.xml","3D homogeneous 1D Channel Flow, SEM parallelisation (2 proc)", 2);
ExecuteParallel("IncNavierStokesSolver","Test_ChanFlow_3DH1D_Parallel_mode2.xml","3D homogeneous 1D Channel Flow, HOM parallelisation (2 proc)", 2);
#endif
if (tests_failed && !quiet)
{
......
regressionTests/Solvers/IncNavierStokesSolver/InputFiles/Test_ChanFlow_3DH1D_Parallel_mode1.xml 0 → 100644
View file @ 78e291d3
<?xml version="1.0" encoding="utf-8"?>
<NEKTAR>
<EXPANSIONS>
<E COMPOSITE="C[0]" NUMMODES="3" FIELDS="u,v,w,p" TYPE="MODIFIED" />
</EXPANSIONS>
<CONDITIONS>
<SOLVERINFO>
<I PROPERTY="SolverType" VALUE="VelocityCorrectionScheme"/>
<I PROPERTY="EQTYPE" VALUE="UnsteadyNavierStokes"/>
<I PROPERTY="AdvectionForm" VALUE="Convective"/>
<I PROPERTY="Projection" VALUE="Galerkin"/>
<I PROPERTY="TimeIntegrationMethod" VALUE="IMEXOrder2"/>
<I PROPERTY="HOMOGENEOUS" VALUE="1D"/>
</SOLVERINFO>
<PARAMETERS>
<P> TimeStep = 0.001 </P>
<P> NumSteps = 1000 </P>
<P> IO_CheckSteps = 1000 </P>
<P> IO_InfoSteps = 1000 </P>
<P> Kinvis = 1 </P>
<P> HomModesZ = 4 </P>
<P> LZ = 1.0 </P>
</PARAMETERS>
<VARIABLES>
<V ID="0"> u </V>
<V ID="1"> v </V>
<V ID="2"> w </V>
<V ID="3"> p </V>
</VARIABLES>
<BOUNDARYREGIONS>
<B ID="0"> C[1] </B>
<B ID="1"> C[2] </B>
<B ID="2"> C[3] </B>
</BOUNDARYREGIONS>
<BOUNDARYCONDITIONS>
<REGION REF="0">
<D VAR="u" VALUE="0" />
<D VAR="v" VALUE="0" />
<D VAR="w" VALUE="0" />
<N VAR="p" USERDEFINEDTYPE="H" VALUE="0" /> // High Order Pressure BC
</REGION>
<REGION REF="1">
<D VAR="u" VALUE="y*(1-y)" />
<D VAR="v" VALUE="0" />