Commit 7063c50e authored by Chris Cantwell's avatar Chris Cantwell
Browse files

Updated MeshPartitioner to write partition to file on local process.


git-svn-id: https://gforge.sci.utah.edu/svn/nektar/trunk@3999 305cdda6-5ce1-45b3-a98d-dfc68c8b3305
parent 3569667d
......@@ -59,7 +59,8 @@ namespace Nektar
{
namespace LibUtilities
{
MeshPartition::MeshPartition(const LibUtilities::SessionReaderSharedPtr& pSession)
MeshPartition::MeshPartition(const LibUtilities::SessionReaderSharedPtr& pSession) :
m_comm(pSession->GetComm()->GetRowComm())
{
ReadMesh(pSession);
}
......@@ -69,46 +70,32 @@ namespace Nektar
}
void MeshPartition::PartitionMesh(unsigned int pNumPartitions)
void MeshPartition::PartitionMesh()
{
ASSERTL0(pNumPartitions > 0, "Invalid number of partitions.");
CreateGraph(m_mesh);
ASSERTL0(m_meshElements.size() >= pNumPartitions,
ASSERTL0(m_comm->GetSize() > 1,
"Partitioning only necessary in parallel case.");
ASSERTL0(m_meshElements.size() >= m_comm->GetSize(),
"Too few elements for this many processes.");
m_partitions.clear();
m_partitions.resize(pNumPartitions);
if (pNumPartitions > 1)
{
PartitionGraph(m_mesh, m_partitions);
}
else
{
m_partitions[0] = m_mesh;
}
CreateGraph(m_mesh);
PartitionGraph(m_mesh, m_localPartition);
}
void MeshPartition::WritePartitions(LibUtilities::SessionReaderSharedPtr& pSession)
void MeshPartition::WriteLocalPartition(LibUtilities::SessionReaderSharedPtr& pSession)
{
for (unsigned int i = 0; i < m_partitions.size(); ++i)
{
TiXmlDocument vNew;
TiXmlDeclaration * decl = new TiXmlDeclaration("1.0", "utf-8", "");
vNew.LinkEndChild(decl);
TiXmlDocument vNew;
TiXmlDeclaration * decl = new TiXmlDeclaration("1.0", "utf-8", "");
vNew.LinkEndChild(decl);
TiXmlElement* vElmtNektar;
vElmtNektar = new TiXmlElement("NEKTAR");
TiXmlElement* vElmtNektar;
vElmtNektar = new TiXmlElement("NEKTAR");
OutputPartition(pSession, m_partitions[i], vElmtNektar);
OutputPartition(pSession, m_localPartition, vElmtNektar);
vNew.LinkEndChild(vElmtNektar);
vNew.LinkEndChild(vElmtNektar);
std::string vFilename = pSession->GetSessionName() + "_P" + boost::lexical_cast<std::string>(i) + ".xml";
vNew.SaveFile(vFilename.c_str());
}
std::string vFilename = pSession->GetSessionName() + "_P" + boost::lexical_cast<std::string>(m_comm->GetRank()) + ".xml";
vNew.SaveFile(vFilename.c_str());
}
void MeshPartition::ReadMesh(const LibUtilities::SessionReaderSharedPtr& pSession)
......@@ -314,66 +301,77 @@ namespace Nektar
}
void MeshPartition::PartitionGraph(BoostSubGraph& pGraph,
std::vector<BoostSubGraph>& pPartitions)
BoostSubGraph& pLocalPartition)
{
int i;
int nGraphVerts = boost::num_vertices(pGraph);
int nGraphEdges = boost::num_edges(pGraph);
// Convert boost graph into CSR format
int acnt = 0;
int vcnt = 0;
BoostVertexIterator vertit, vertit_end;
BoostEdgeIterator edgeit, edgeit_end;
BoostAdjacencyIterator adjvertit, adjvertit_end;
Array<OneD, int> xadj(nGraphVerts+1,0);
Array<OneD, int> adjncy(2*nGraphEdges);
Array<OneD, int> vwgt(nGraphVerts, 1);
Array<OneD, int> vsize(nGraphVerts, 1);
Array<OneD, int> part(nGraphVerts,0);
for ( boost::tie(vertit, vertit_end) = boost::vertices(pGraph);
vertit != vertit_end;
++vertit)
{
for ( boost::tie(adjvertit, adjvertit_end) = boost::adjacent_vertices(*vertit,pGraph);
adjvertit != adjvertit_end;
++adjvertit)
if (m_comm->GetRank() == 0)
{
int acnt = 0;
int vcnt = 0;
BoostAdjacencyIterator adjvertit, adjvertit_end;
Array<OneD, int> xadj(nGraphVerts+1,0);
Array<OneD, int> adjncy(2*nGraphEdges);
Array<OneD, int> vwgt(nGraphVerts, 1);
Array<OneD, int> vsize(nGraphVerts, 1);
for ( boost::tie(vertit, vertit_end) = boost::vertices(pGraph);
vertit != vertit_end;
++vertit)
{
adjncy[acnt++] = *adjvertit;
for ( boost::tie(adjvertit, adjvertit_end) = boost::adjacent_vertices(*vertit,pGraph);
adjvertit != adjvertit_end;
++adjvertit)
{
adjncy[acnt++] = *adjvertit;
}
xadj[++vcnt] = acnt;
}
xadj[++vcnt] = acnt;
}
// Call Metis and partition graph
int npart = pPartitions.size();
int vol = 0;
Array<OneD, int> part(nGraphVerts,0);
try
{
Metis::PartGraphVKway(nGraphVerts, xadj, adjncy, vwgt, vsize, npart, vol, part);
// Call Metis and partition graph
int npart = m_comm->GetSize();
int vol = 0;
try
{
Metis::PartGraphVKway(nGraphVerts, xadj, adjncy, vwgt, vsize, npart, vol, part);
for (i = 1; i < m_comm->GetSize(); ++i)
{
m_comm->Send(i, part);
}
}
catch (...)
{
NEKERROR(ErrorUtil::efatal,
"Error in calling metis to partition graph.");
}
}
catch (...)
else
{
NEKERROR(ErrorUtil::efatal,
"Error in calling metis to partition graph.");
m_comm->Recv(0, part);
}
// Create boost subgraphs for partitions
int i;
for (i = 0; i < npart; ++i)
{
pPartitions[i] = pGraph.create_subgraph();
}
// Create boost subgraph for this process's partitions
pLocalPartition = pGraph.create_subgraph();
// Populate subgraphs
// Populate subgraph
i = 0;
for ( boost::tie(vertit, vertit_end) = boost::vertices(pGraph);
vertit != vertit_end;
++vertit, ++i)
{
pGraph[*vertit].partition = part[i];
pGraph[*vertit].partid = boost::num_vertices(pPartitions[i]);
BoostVertex v = boost::add_vertex(i, pPartitions[part[i]]);
if (part[i] == m_comm->GetRank())
{
pGraph[*vertit].partition = part[i];
pGraph[*vertit].partid = boost::num_vertices(pLocalPartition);
BoostVertex v = boost::add_vertex(i, pLocalPartition);
}
}
}
......
......@@ -39,6 +39,7 @@
#include <boost/graph/subgraph.hpp>
#include <boost/graph/adjacency_list.hpp>
#include <LibUtilities/BasicUtils/SessionReader.h>
#include <LibUtilities/Communication/Comm.h>
namespace Nektar
{
......@@ -51,8 +52,8 @@ namespace Nektar
LIB_UTILITIES_EXPORT MeshPartition(const LibUtilities::SessionReaderSharedPtr& pSession);
LIB_UTILITIES_EXPORT ~MeshPartition();
LIB_UTILITIES_EXPORT void PartitionMesh(unsigned int pNumPartitions);
LIB_UTILITIES_EXPORT void WritePartitions(
LIB_UTILITIES_EXPORT void PartitionMesh();
LIB_UTILITIES_EXPORT void WriteLocalPartition(
LibUtilities::SessionReaderSharedPtr& pSession);
private:
......@@ -166,12 +167,14 @@ namespace Nektar
std::vector<unsigned int> m_domain;
BoostSubGraph m_mesh;
std::vector<BoostSubGraph> m_partitions;
BoostSubGraph m_localPartition;
CommSharedPtr m_comm;
void ReadMesh(const LibUtilities::SessionReaderSharedPtr& pSession);
void CreateGraph(BoostSubGraph& pGraph);
void PartitionGraph(BoostSubGraph& pGraph,
std::vector<BoostSubGraph>& pPartitions);
BoostSubGraph& pLocalPartition);
void OutputPartition(LibUtilities::SessionReaderSharedPtr& pSession, BoostSubGraph& pGraph, TiXmlElement* pGeometry);
};
......
......@@ -1081,16 +1081,15 @@ namespace Nektar
// Partition mesh into length of row comms
if (vCommMesh->GetSize() > 1)
{
// Only do partitioning on the rank-0 proc in MPI_COMM_WORLD
if (m_comm->GetRank() == 0)
{
SessionReaderSharedPtr vSession = GetSharedThisPtr();
MeshPartitionSharedPtr vPartitioner = MemoryManager<MeshPartition>::AllocateSharedPtr(vSession);
vPartitioner->PartitionMesh(vCommMesh->GetSize());
vPartitioner->WritePartitions(vSession);
}
m_comm->Block();
// Partitioner now operates in parallel
// Each process receives partitioning over interconnect
// and writes its own session file to the working directory.
SessionReaderSharedPtr vSession = GetSharedThisPtr();
MeshPartitionSharedPtr vPartitioner = MemoryManager<MeshPartition>::AllocateSharedPtr(vSession);
vPartitioner->PartitionMesh();
vPartitioner->WriteLocalPartition(vSession);
vCommMesh->Block();
m_filename = GetSessionNameRank() + ".xml";
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment