Hybrid parallelisation bug. Mesh was unevely partioned across

the cartesian communicator

library/LibUtilities/BasicUtils/MeshPartition.cpp

@@ -62,7 +62,7 @@ namespace Nektar
     namespace LibUtilities
     {
         MeshPartition::MeshPartition(const LibUtilities::SessionReaderSharedPtr& pSession) :
-                m_comm(pSession->GetComm()->GetRowComm())
+                m_comm(pSession->GetComm())
         {
             ReadMesh(pSession);
         }
@@ -74,9 +74,9 @@ namespace Nektar
 
         void MeshPartition::PartitionMesh()
         {
-            ASSERTL0(m_comm->GetSize() > 1,
+            ASSERTL0(m_comm->GetRowComm()->GetSize() > 1,
                      "Partitioning only necessary in parallel case.");
-            ASSERTL0(m_meshElements.size() >= m_comm->GetSize(),
+            ASSERTL0(m_meshElements.size() >= m_comm->GetRowComm()->GetSize(),
                      "Too few elements for this many processes.");
 
             CreateGraph(m_mesh);
@@ -96,7 +96,7 @@ namespace Nektar
 
             vNew.LinkEndChild(vElmtNektar);
 
-            std::string vFilename = pSession->GetSessionName() + "_P" + boost::lexical_cast<std::string>(m_comm->GetRank()) + ".xml";
+            std::string vFilename = pSession->GetSessionName() + "_P" + boost::lexical_cast<std::string>(m_comm->GetRowComm()->GetRank()) + ".xml";
             vNew.SaveFile(vFilename.c_str());
         }
 
@@ -325,7 +325,7 @@ namespace Nektar
             BoostVertexIterator    vertit, vertit_end;
             Array<OneD, int> part(nGraphVerts,0);
 
-            if (m_comm->GetRank() == 0)
+            if (m_comm->GetRowComm()->GetRank() == 0)
             {
                 int acnt = 0;
                 int vcnt = 0;
@@ -349,21 +349,36 @@ namespace Nektar
                 }
 
                 // Call Metis and partition graph
-                int npart = m_comm->GetSize();
+                int npart = m_comm->GetRowComm()->GetSize();
                 int vol = 0;
 
                 try
                 {
-                    // Attempt partitioning using METIS.
-                    Metis::PartGraphVKway(nGraphVerts, xadj, adjncy, vwgt, vsize, npart, vol, part);
-
-                    // Check METIS produced a valid partition and fix if not.
-                    CheckPartitions(part);
-
-                    // Distribute partitioning to all processes.
-                    for (i = 1; i < m_comm->GetSize(); ++i)
+					//////////////////////////////////////////////////////
+					// On a cartesian communicator do mesh partiotion just on the first column
+					// so there is no doubt the partitions are all the same in all the columns
+					if(m_comm->GetColumnComm()->GetRank() == 0)
+					{
+						// Attempt partitioning using METIS.
+						Metis::PartGraphVKway(nGraphVerts, xadj, adjncy, vwgt, vsize, npart, vol, part);
+						// Check METIS produced a valid partition and fix if not.
+						CheckPartitions(part);
+						// distribute among columns
+						for (i = 1; i < m_comm->GetColumnComm()->GetSize(); ++i)
+						{
+							m_comm->GetColumnComm()->Send(i, part);
+						}
+					}
+					else 
+					{
+						m_comm->GetColumnComm()->Recv(0, part);
+					}
+					m_comm->GetColumnComm()->Block();
+					//////////////////////////////////
+					// distribute among rows
+                    for (i = 1; i < m_comm->GetRowComm()->GetSize(); ++i)
                     {
-                        m_comm->Send(i, part);
+                        m_comm->GetRowComm()->Send(i, part);
                     }
                 }
                 catch (...)
@@ -374,7 +389,7 @@ namespace Nektar
             }
             else
             {
-                m_comm->Recv(0, part);
+                m_comm->GetRowComm()->Recv(0, part);
             }
 
             // Create boost subgraph for this process's partitions
@@ -386,7 +401,7 @@ namespace Nektar
                   vertit != vertit_end;
                   ++vertit, ++i)
             {
-                if (part[i] == m_comm->GetRank())
+                if (part[i] == m_comm->GetRowComm()->GetRank())
                 {
                     pGraph[*vertit].partition = part[i];
                     pGraph[*vertit].partid = boost::num_vertices(pLocalPartition);
@@ -400,7 +415,7 @@ namespace Nektar
         {
             unsigned int       i     = 0;
             unsigned int       cnt   = 0;
-            const unsigned int npart = m_comm->GetSize();
+            const unsigned int npart = m_comm->GetRowComm()->GetSize();
             bool               valid = true;
 
             // Check that every process has at least one element assigned

library/LibUtilities/BasicUtils/SessionReader.cpp

@@ -1166,7 +1166,7 @@ namespace Nektar
                 vPartitioner->PartitionMesh();
                 vPartitioner->WriteLocalPartition(vSession);
                 
-                vCommMesh->Block();
+                m_comm->Block();
 
                 m_filename = GetSessionNameRank() + ".xml";