Commit f2c371d1 authored by Chris Cantwell's avatar Chris Cantwell

Allow mesh to be pre-partitioned.

parent 74b307ea
......@@ -79,17 +79,15 @@ namespace Nektar
}
void MeshPartition::PartitionMesh(bool shared)
void MeshPartition::PartitionMesh(int nParts, bool shared)
{
ASSERTL0(m_comm->GetRowComm()->GetSize() > 1,
"Partitioning only necessary in parallel case.");
ASSERTL0(m_meshElements.size() >= m_comm->GetRowComm()->GetSize(),
ASSERTL0(m_meshElements.size() >= nParts,
"Too few elements for this many processes.");
m_shared = shared;
if (m_weightingRequired) WeightElements();
CreateGraph(m_mesh);
PartitionGraph(m_mesh, m_localPartition);
PartitionGraph(m_mesh, nParts, m_localPartition);
}
void MeshPartition::WriteLocalPartition(LibUtilities::SessionReaderSharedPtr& pSession)
......@@ -124,7 +122,7 @@ namespace Nektar
void MeshPartition::WriteAllPartitions(LibUtilities::SessionReaderSharedPtr& pSession)
{
for (int i = 0; i < m_comm->GetRowComm()->GetSize(); ++i)
for (int i = 0; i < m_localPartition.size(); ++i)
{
TiXmlDocument vNew;
TiXmlDeclaration * decl = new TiXmlDeclaration("1.0", "utf-8", "");
......@@ -471,11 +469,12 @@ namespace Nektar
void MeshPartition::PrintPartInfo(std::ostream &out)
{
int nElmt = boost::num_vertices(m_mesh);
int nPart = m_comm->GetRowComm()->GetSize();
int nPart = m_localPartition.size();
out << "# Partition information:" << std::endl;
out << "# No. elements : " << nElmt << std::endl;
out << "# No. partitions: " << nPart << std::endl;
out << "# ID nElmt nLocDof nBndDof" << std::endl;
BoostVertexIterator vertit, vertit_end;
std::vector<int> partElmtCount(nPart, 0);
......@@ -756,6 +755,7 @@ namespace Nektar
}
void MeshPartition::PartitionGraph(BoostSubGraph& pGraph,
int nParts,
std::vector<BoostSubGraph>& pLocalPartition)
{
int i;
......@@ -800,7 +800,6 @@ namespace Nektar
}
// Call Metis and partition graph
int npart = m_comm->GetRowComm()->GetSize();
int vol = 0;
try
......@@ -812,9 +811,9 @@ namespace Nektar
{
// Attempt partitioning using METIS.
int ncon = 1;
Metis::PartGraphVKway(nGraphVerts, ncon, xadj, adjncy, vwgt, vsize, npart, vol, part);
Metis::PartGraphVKway(nGraphVerts, ncon, xadj, adjncy, vwgt, vsize, nParts, vol, part);
// Check METIS produced a valid partition and fix if not.
CheckPartitions(part);
CheckPartitions(nParts, part);
if (!m_shared)
{
// distribute among columns
......@@ -852,7 +851,7 @@ namespace Nektar
}
// Create boost subgraph for this process's partitions
int nCols = m_comm->GetRowComm()->GetSize();
int nCols = nParts;
pLocalPartition.resize(nCols);
for (i = 0; i < nCols; ++i)
{
......@@ -872,7 +871,7 @@ namespace Nektar
}
void MeshPartition::CheckPartitions(Array<OneD, int> &pPart)
void MeshPartition::CheckPartitions(int nParts, Array<OneD, int> &pPart)
{
unsigned int i = 0;
unsigned int cnt = 0;
......@@ -880,7 +879,7 @@ namespace Nektar
bool valid = true;
// Check that every process has at least one element assigned
for (i = 0; i < npart; ++i)
for (i = 0; i < nParts; ++i)
{
cnt = std::count(pPart.begin(), pPart.end(), i);
if (cnt == 0)
......@@ -898,7 +897,7 @@ namespace Nektar
{
for (i = 0; i < pPart.num_elements(); ++i)
{
pPart[i] = i % npart;
pPart[i] = i % nParts;
}
}
}
......
......@@ -58,7 +58,7 @@ namespace Nektar
LIB_UTILITIES_EXPORT MeshPartition(const SessionReaderSharedPtr& pSession);
LIB_UTILITIES_EXPORT ~MeshPartition();
LIB_UTILITIES_EXPORT void PartitionMesh(bool shared = false);
LIB_UTILITIES_EXPORT void PartitionMesh(int nParts, bool shared = false);
LIB_UTILITIES_EXPORT void WriteLocalPartition(
SessionReaderSharedPtr& pSession);
LIB_UTILITIES_EXPORT void WriteAllPartitions(
......@@ -213,9 +213,10 @@ namespace Nektar
void WeightElements();
void CreateGraph(BoostSubGraph& pGraph);
void PartitionGraph(BoostSubGraph& pGraph,
int nParts,
std::vector<BoostSubGraph>& pLocalPartition);
void OutputPartition(SessionReaderSharedPtr& pSession, BoostSubGraph& pGraph, TiXmlElement* pGeometry);
void CheckPartitions(Array<OneD, int> &pPart);
void CheckPartitions(int nParts, Array<OneD, int> &pPart);
};
typedef boost::shared_ptr<MeshPartition> MeshPartitionSharedPtr;
......
......@@ -297,6 +297,8 @@ namespace Nektar
"number of procs in Y-dir")
("npz", po::value<int>(),
"number of procs in Z-dir")
("part-only", po::value<int>(),
"only partition mesh into N partitions.")
("part-info", "Output partition information")
;
......@@ -1251,6 +1253,19 @@ namespace Nektar
"Error: File '" + pFilename + "' is corrupt.");
}
}
else if (pFilename.size() > 4 &&
pFilename.substr(pFilename.size() - 4, 4) == "_xml")
{
fs::path pdirname(pFilename);
boost::format pad("P%1$07d.xml");
pad % m_comm->GetRank();
fs::path pRankFilename(pad.str());
fs::path fullpath = pdirname / pRankFilename;
ifstream file(PortablePath(fullpath).c_str());
ASSERTL0(file.good(), "Unable to open file: " + fullpath.string());
file >> (*pDoc);
}
else
{
ifstream file(pFilename.c_str());
......@@ -1379,20 +1394,83 @@ namespace Nektar
// Get row of comm, or the whole comm if not split
CommSharedPtr vCommMesh = m_comm->GetRowComm();
const bool isRoot = (m_comm->GetRank() == 0);
// Delete any existing loaded mesh
if (m_xmlDoc)
{
delete m_xmlDoc;
}
// Load file for root process only (since this is always needed)
// and determine if the provided geometry has already been
// partitioned. This will be the case if the user provides the
// directory of mesh partitions as an input. Partitioned geometries
// have the attribute
// PARTITION=X
// where X is the number of the partition (and should match the
// process rank). The result is shared with all other processes.
int isPartitioned = 0;
if (isRoot)
{
m_xmlDoc = MergeDoc(m_filenames);
if (GetElement("Nektar/Geometry")->Attribute("PARTITION"))
{
cout << "Using pre-partitioned mesh." << endl;
isPartitioned = 1;
}
}
GetComm()->AllReduce(isPartitioned, LibUtilities::ReduceMax);
// If the mesh is already partitioned, we are done. Remaining
// processes must load their partitions.
if (isPartitioned) {
if (!isRoot)
{
m_xmlDoc = MergeDoc(m_filenames);
}
return;
}
// Partition mesh into length of row comms
if (vCommMesh->GetSize() > 1)
// Mesh has not been partitioned so do partitioning if required.
// Note in the serial case nothing is done as we have already loaded
// the mesh.
if (DefinesCmdLineArgument("part-only"))
{
// Perform partitioning of the mesh only. For this we insist
// the code is run in serial (parallel execution is pointless).
ASSERTL0(GetComm()->GetSize() == 1,
"The 'part-only' option should be used in serial.");
// Number of partitions is specified by the parameter.
int nParts = GetCmdLineArgument<int>("part-only");
SessionReaderSharedPtr vSession = GetSharedThisPtr();
MeshPartitionSharedPtr vPartitioner = MemoryManager<
MeshPartition>::AllocateSharedPtr(vSession);
vPartitioner->PartitionMesh(nParts, true);
vPartitioner->WriteAllPartitions(vSession);
vPartitioner->GetCompositeOrdering(m_compOrder);
vPartitioner->GetBndRegionOrdering(m_bndRegOrder);
if (isRoot && DefinesCmdLineArgument("part-info"))
{
vPartitioner->PrintPartInfo(std::cout);
}
Finalise();
exit(0);
}
else if (vCommMesh->GetSize() > 1)
{
if (DefinesCmdLineArgument("shared-filesystem"))
{
if (GetComm()->GetRank() == 0)
if (isRoot)
{
m_xmlDoc = MergeDoc(m_filenames);
SessionReaderSharedPtr vSession = GetSharedThisPtr();
MeshPartitionSharedPtr vPartitioner = MemoryManager<
MeshPartition>::AllocateSharedPtr(vSession);
vPartitioner->PartitionMesh(true);
vPartitioner->PartitionMesh(GetComm()->GetSize(), true);
vPartitioner->WriteAllPartitions(vSession);
vPartitioner->GetCompositeOrdering(m_compOrder);
vPartitioner->GetBndRegionOrdering(m_bndRegOrder);
......@@ -1405,7 +1483,11 @@ namespace Nektar
}
else
{
m_xmlDoc = MergeDoc(m_filenames);
// Need to load mesh on non-root processes.
if (!isRoot)
{
m_xmlDoc = MergeDoc(m_filenames);
}
// Partitioner now operates in parallel
// Each process receives partitioning over interconnect
......@@ -1413,12 +1495,12 @@ namespace Nektar
SessionReaderSharedPtr vSession = GetSharedThisPtr();
MeshPartitionSharedPtr vPartitioner = MemoryManager<
MeshPartition>::AllocateSharedPtr(vSession);
vPartitioner->PartitionMesh(false);
vPartitioner->PartitionMesh(GetComm()->GetSize(), false);
vPartitioner->WriteLocalPartition(vSession);
vPartitioner->GetCompositeOrdering(m_compOrder);
vPartitioner->GetBndRegionOrdering(m_bndRegOrder);
if (DefinesCmdLineArgument("part-info"))
if (DefinesCmdLineArgument("part-info") && isRoot)
{
vPartitioner->PrintPartInfo(std::cout);
}
......@@ -1447,14 +1529,6 @@ namespace Nektar
". Check XML standards compliance. Error on line: " +
boost::lexical_cast<std::string>(m_xmlDoc->Row()));
}
else
{
if (m_xmlDoc)
{
delete m_xmlDoc;
}
m_xmlDoc = MergeDoc(m_filenames);
}
}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment