Commit 790e7416 authored by Douglas Serson's avatar Douglas Serson

Merge branch 'feature/stats-sc-level' into 'master'

Add statistics for multi-level static condensation

This MR adds additional statistics when `--verbose` is enabled if multi-level static condensation is used. It outputs:
* the local/global dof statistics at the lowest level across all processes
* distribution of the lowest levels across each of the processors

Example output:
```
Assembly map statistics for field p:
  - Number of local/global dof             : 216888 151247
  - Number of local/global boundary dof    : 120528 54887
  - Number of local/global Dirichlet dof   : 133 114
  - dof valency (min/max/mean)             : 1 10 2
  - Local dof dist. (min/max/mean/dev)     : 22932 39319 30984 6106.94
  - Local bnd dof dist. (min/max/mean/dev) : 14742 19614 17218.3 1797.01
  - M-level sc. dist. (min/max/mean/dev)   : 13 15 13.8571 0.638877
Stats at lowest static cond. level:
  - Number of local/global dof             : 0 2746
  - Number of local/global boundary dof    : 4600 1875
  - Number of local/global Dirichlet dof   : 133 114
  - dof valency (min/max/mean)             : 1 9 2
  - Local bnd dof dist. (min/max/mean/dev) : 510 1096 657.143 188.189
```

See merge request !656
parents bd7e75cb a797c969
......@@ -8,6 +8,8 @@ v4.4.0
Galerkin discretisation (!604)
- Add support for variable polynomial order with periodic boundary conditions
(!658)
- Statistics are now printed for lowest level of multi-level static condensation
(!656)
- Sped up interpolataion from pts files and fixed parallel pts import (!584)
- Increased required boost version to 1.56.0 (!584)
......
......@@ -2,7 +2,7 @@
<test>
<description>Helmholtz 2D CG with P=7, all BCs, iterative ML, par(3)</description>
<executable>Helmholtz2D</executable>
<parameters>--use-metis -I GlobalSysSoln=IterativeMultiLevelStaticCond Helmholtz2D_P7_AllBCs.xml</parameters>
<parameters>--use-metis --verbose -I GlobalSysSoln=IterativeMultiLevelStaticCond Helmholtz2D_P7_AllBCs.xml</parameters>
<processes>3</processes>
<files>
<file description="Session File">Helmholtz2D_P7_AllBCs.xml</file>
......
......@@ -2,7 +2,7 @@
<test>
<description>Helmholtz 3D CG, hexes, mixed BCs, iterative ML</description>
<executable>Helmholtz3D</executable>
<parameters>--use-metis -I GlobalSysSoln=IterativeMultiLevelStaticCond Helmholtz3D_Hex_AllBCs_P6.xml</parameters>
<parameters>--use-metis --verbose -I GlobalSysSoln=IterativeMultiLevelStaticCond Helmholtz3D_Hex_AllBCs_P6.xml</parameters>
<processes>3</processes>
<files>
<file description="Session File">Helmholtz3D_Hex_AllBCs_P6.xml</file>
......
......@@ -2,7 +2,7 @@
<test>
<description>Helmholtz 3D CG, hexes, mixed BCs, iterative ML, Scotch</description>
<executable>Helmholtz3D</executable>
<parameters>--use-scotch -I GlobalSysSoln=IterativeMultiLevelStaticCond Helmholtz3D_Hex_AllBCs_P6.xml</parameters>
<parameters>--use-scotch --verbose -I GlobalSysSoln=IterativeMultiLevelStaticCond Helmholtz3D_Hex_AllBCs_P6.xml</parameters>
<processes>3</processes>
<files>
<file description="Session File">Helmholtz3D_Hex_AllBCs_P6.xml</file>
......
......@@ -2,7 +2,7 @@
<test>
<description>Helmholtz 3D CG, prisms, Neumann BCs, iterative ML, Par(3), Scotch</description>
<executable>Helmholtz3D</executable>
<parameters>--use-scotch Helmholtz3D_Prism.xml</parameters>
<parameters>--use-scotch --verbose Helmholtz3D_Prism.xml</parameters>
<processes>3</processes>
<files>
<file description="Session File">Helmholtz3D_Prism.xml</file>
......
......@@ -1232,7 +1232,7 @@ namespace Nektar
}
void AssemblyMap::PrintStats(
std::ostream &out, std::string variable) const
std::ostream &out, std::string variable, bool printHeader) const
{
LibUtilities::CommSharedPtr vRowComm
= m_session->GetComm()->GetRowComm();
......@@ -1262,7 +1262,9 @@ namespace Nektar
Array<OneD, NekDouble> tmpLoc (m_numLocalBndCoeffs, 1.0);
Array<OneD, NekDouble> tmpGlob(m_numGlobalBndCoeffs, 0.0);
Vmath::Assmb(m_numLocalBndCoeffs, tmpLoc.get(), m_localToGlobalBndMap.get(), tmpGlob.get());
Vmath::Assmb(
m_numLocalBndCoeffs, tmpLoc.get(),
m_localToGlobalBndMap.get(), tmpGlob.get());
UniversalAssembleBnd(tmpGlob);
int totGlobDof = globCnt;
......@@ -1307,8 +1309,12 @@ namespace Nektar
if (isRoot)
{
out << "Assembly map statistics for field " << variable << ":"
<< endl;
if (printHeader)
{
out << "Assembly map statistics for field " << variable
<< ":" << endl;
}
out << " - Number of local/global dof : "
<< totLocalDof << " " << totGlobDof << endl;
out << " - Number of local/global boundary dof : "
......@@ -1341,9 +1347,12 @@ namespace Nektar
}
}
out << " - Local dof dist. (min/max/mean/dev) : "
<< minval << " " << maxval << " " << (mean / n) << " "
<< sqrt(mean2/n - mean*mean/n/n) << endl;
if (maxval > 0.1)
{
out << " - Local dof dist. (min/max/mean/dev) : "
<< minval << " " << maxval << " " << (mean / n)
<< " " << sqrt(mean2/n - mean*mean/n/n) << endl;
}
vRowComm->Block();
......@@ -1380,6 +1389,69 @@ namespace Nektar
tmp[0] = m_numLocalBndCoeffs;
vRowComm->Send(0, tmp);
}
// Either we have no more levels in the static condensation, or we
// are not multi-level.
if (!m_nextLevelLocalToGlobalMap)
{
return;
}
int level = 2;
AssemblyMapSharedPtr tmp = m_nextLevelLocalToGlobalMap;
while (tmp->m_nextLevelLocalToGlobalMap)
{
tmp = tmp->m_nextLevelLocalToGlobalMap;
++level;
}
// Print out multi-level static condensation information.
if (n > 1)
{
if (isRoot)
{
NekDouble mean = level, mean2 = mean * mean;
int minval = level, maxval = level;
Array<OneD, NekDouble> tmpRecv(1);
for (i = 1; i < n; ++i)
{
vRowComm->Recv(i, tmpRecv);
mean += tmpRecv[0];
mean2 += tmpRecv[0]*tmpRecv[0];
if (tmpRecv[0] > maxval)
{
maxval = (int)(tmpRecv[0] + 0.5);
}
if (tmpRecv[0] < minval)
{
minval = (int)(tmpRecv[0] + 0.5);
}
}
out << " - M-level sc. dist. (min/max/mean/dev) : "
<< minval << " " << maxval << " " << (mean / n) << " "
<< sqrt(mean2/n - mean*mean/n/n) << endl;
}
else
{
Array<OneD, NekDouble> tmpSend(1);
tmpSend[0] = level;
vRowComm->Send(0, tmpSend);
}
}
else
{
out << " - Number of static cond. levels : "
<< level << endl;
}
if (isRoot)
{
out << "Stats at lowest static cond. level:" << endl;
}
tmp->PrintStats(out, variable, false);
}
} // namespace
} // namespace
......@@ -252,7 +252,7 @@ namespace Nektar
MULTI_REGIONS_EXPORT int GetNumNonDirFaces() const;
MULTI_REGIONS_EXPORT void PrintStats(std::ostream &out, std::string variable) const;
MULTI_REGIONS_EXPORT void PrintStats(std::ostream &out, std::string variable, bool printHeader = true) const;
MULTI_REGIONS_EXPORT const Array<OneD, const int>&
GetExtraDirEdges();
......
......@@ -1252,8 +1252,10 @@ namespace Nektar
// For parallel multi-level static condensation determine the lowest
// static condensation level amongst processors.
if (m_solnType == eIterativeMultiLevelStaticCond ||
m_solnType == eXxtMultiLevelStaticCond)
if ((m_solnType == eDirectMultiLevelStaticCond ||
m_solnType == ePETScMultiLevelStaticCond ||
m_solnType == eIterativeMultiLevelStaticCond ||
m_solnType == eXxtMultiLevelStaticCond) && bottomUpGraph)
{
m_lowestStaticCondLevel = bottomUpGraph->GetNlevels()-1;
vComm->AllReduce(m_lowestStaticCondLevel,
......
......@@ -149,6 +149,11 @@ namespace Nektar
m_bndCondExpansions, m_bndConditions,
m_periodicVerts, variable);
if (m_session->DefinesCmdLineArgument("verbose"))
{
m_traceMap->PrintStats(std::cout, variable);
}
// Scatter trace points to 1D elements. For each element, we find
// the trace point associated to each vertex. The element then
// retains a pointer to the trace space points, to ensure
......
......@@ -404,6 +404,11 @@ namespace Nektar
m_periodicEdges,
variable);
if (m_session->DefinesCmdLineArgument("verbose"))
{
m_traceMap->PrintStats(std::cout, variable);
}
Array<OneD, Array<OneD, LocalRegions::ExpansionSharedPtr> >
&elmtToTrace = m_traceMap->GetElmtToTrace();
......
......@@ -333,6 +333,11 @@ using namespace boost::assign;
m_session,graph3D,trace,*this,m_bndCondExpansions,
m_bndConditions, m_periodicFaces,variable);
if (m_session->DefinesCmdLineArgument("verbose"))
{
m_traceMap->PrintStats(std::cout, variable);
}
Array<OneD, Array<OneD, LocalRegions::ExpansionSharedPtr> >
&elmtToTrace = m_traceMap->GetElmtToTrace();
......
......@@ -2,7 +2,7 @@
<test>
<description>3D channel flow, Tetrahedral elements, P=4, periodic BCs, XxtMultiLevelStaticCond, par(2)</description>
<executable>IncNavierStokesSolver</executable>
<parameters>--use-metis -I GlobalSysSoln=XxtMultiLevelStaticCond Tet_channel_m4_per.xml</parameters>
<parameters>--use-metis --verbose -I GlobalSysSoln=XxtMultiLevelStaticCond Tet_channel_m4_per.xml</parameters>
<processes>2</processes>
<files>
<file description="Session File">Tet_channel_m4_per.xml</file>
......
......@@ -2,7 +2,7 @@
<test>
<description>3D channel flow, Tet elements, par(3), P=8</description>
<executable>IncNavierStokesSolver</executable>
<parameters>--use-metis -I GlobalSysSoln=IterativeMultiLevelStaticCond Tet_channel_m8_par.xml</parameters>
<parameters>--use-metis --verbose -I GlobalSysSoln=IterativeMultiLevelStaticCond Tet_channel_m8_par.xml</parameters>
<processes>3</processes>
<files>
<file description="Session File">Tet_channel_m8_par.xml</file>
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment