Commit b8519986 authored by Ale's avatar Ale

Hybrid parallelisation bug. Mesh was unevely partioned across

the cartesian communicator
parent 93956964
......@@ -62,7 +62,7 @@ namespace Nektar
namespace LibUtilities
{
MeshPartition::MeshPartition(const LibUtilities::SessionReaderSharedPtr& pSession) :
m_comm(pSession->GetComm()->GetRowComm())
m_comm(pSession->GetComm())
{
ReadMesh(pSession);
}
......@@ -74,9 +74,9 @@ namespace Nektar
void MeshPartition::PartitionMesh()
{
ASSERTL0(m_comm->GetSize() > 1,
ASSERTL0(m_comm->GetRowComm()->GetSize() > 1,
"Partitioning only necessary in parallel case.");
ASSERTL0(m_meshElements.size() >= m_comm->GetSize(),
ASSERTL0(m_meshElements.size() >= m_comm->GetRowComm()->GetSize(),
"Too few elements for this many processes.");
CreateGraph(m_mesh);
......@@ -96,7 +96,7 @@ namespace Nektar
vNew.LinkEndChild(vElmtNektar);
std::string vFilename = pSession->GetSessionName() + "_P" + boost::lexical_cast<std::string>(m_comm->GetRank()) + ".xml";
std::string vFilename = pSession->GetSessionName() + "_P" + boost::lexical_cast<std::string>(m_comm->GetRowComm()->GetRank()) + ".xml";
vNew.SaveFile(vFilename.c_str());
}
......@@ -325,7 +325,7 @@ namespace Nektar
BoostVertexIterator vertit, vertit_end;
Array<OneD, int> part(nGraphVerts,0);
if (m_comm->GetRank() == 0)
if (m_comm->GetRowComm()->GetRank() == 0)
{
int acnt = 0;
int vcnt = 0;
......@@ -349,21 +349,36 @@ namespace Nektar
}
// Call Metis and partition graph
int npart = m_comm->GetSize();
int npart = m_comm->GetRowComm()->GetSize();
int vol = 0;
try
{
// Attempt partitioning using METIS.
Metis::PartGraphVKway(nGraphVerts, xadj, adjncy, vwgt, vsize, npart, vol, part);
// Check METIS produced a valid partition and fix if not.
CheckPartitions(part);
// Distribute partitioning to all processes.
for (i = 1; i < m_comm->GetSize(); ++i)
//////////////////////////////////////////////////////
// On a cartesian communicator do mesh partiotion just on the first column
// so there is no doubt the partitions are all the same in all the columns
if(m_comm->GetColumnComm()->GetRank() == 0)
{
// Attempt partitioning using METIS.
Metis::PartGraphVKway(nGraphVerts, xadj, adjncy, vwgt, vsize, npart, vol, part);
// Check METIS produced a valid partition and fix if not.
CheckPartitions(part);
// distribute among columns
for (i = 1; i < m_comm->GetColumnComm()->GetSize(); ++i)
{
m_comm->GetColumnComm()->Send(i, part);
}
}
else
{
m_comm->GetColumnComm()->Recv(0, part);
}
m_comm->GetColumnComm()->Block();
//////////////////////////////////
// distribute among rows
for (i = 1; i < m_comm->GetRowComm()->GetSize(); ++i)
{
m_comm->Send(i, part);
m_comm->GetRowComm()->Send(i, part);
}
}
catch (...)
......@@ -374,7 +389,7 @@ namespace Nektar
}
else
{
m_comm->Recv(0, part);
m_comm->GetRowComm()->Recv(0, part);
}
// Create boost subgraph for this process's partitions
......@@ -386,7 +401,7 @@ namespace Nektar
vertit != vertit_end;
++vertit, ++i)
{
if (part[i] == m_comm->GetRank())
if (part[i] == m_comm->GetRowComm()->GetRank())
{
pGraph[*vertit].partition = part[i];
pGraph[*vertit].partid = boost::num_vertices(pLocalPartition);
......@@ -400,7 +415,7 @@ namespace Nektar
{
unsigned int i = 0;
unsigned int cnt = 0;
const unsigned int npart = m_comm->GetSize();
const unsigned int npart = m_comm->GetRowComm()->GetSize();
bool valid = true;
// Check that every process has at least one element assigned
......
......@@ -1166,7 +1166,7 @@ namespace Nektar
vPartitioner->PartitionMesh();
vPartitioner->WriteLocalPartition(vSession);
vCommMesh->Block();
m_comm->Block();
m_filename = GetSessionNameRank() + ".xml";
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment