From d4381b44d0227b39bb1e9c51612221ce0d62ef9b Mon Sep 17 00:00:00 2001 From: Douglas Serson Date: Thu, 21 Apr 2016 16:15:05 +0100 Subject: [PATCH 01/11] Implementation of expansion from restart file in serial --- library/SpatialDomains/MeshGraph.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/library/SpatialDomains/MeshGraph.cpp b/library/SpatialDomains/MeshGraph.cpp index 8d04931a9..7aa7620e4 100644 --- a/library/SpatialDomains/MeshGraph.cpp +++ b/library/SpatialDomains/MeshGraph.cpp @@ -1052,6 +1052,20 @@ namespace Nektar cout << " Number of elements: " << fielddefs.size() << endl; SetExpansions(fielddefs); } + else if(expType == "F") + { + ASSERTL0(expansion->Attribute("FILE"), + "Attribute FILE expected for type F expansion"); + std::string filenameStr = expansion->Attribute("FILE"); + ASSERTL0(!filenameStr.empty(), + "A filename must be specified for the FILE " + "attribute of expansion"); + + std::vector fielddefs; + LibUtilities::FieldIO f(m_session->GetComm()); + f.Import(filenameStr, fielddefs); + SetExpansions(fielddefs); + } else { ASSERTL0(false,"Expansion type not defined"); -- GitLab From 9329d1ef09fe7ac5c8facdfb456a3a008507fe06 Mon Sep 17 00:00:00 2001 From: Douglas Serson Date: Thu, 21 Apr 2016 18:38:19 +0100 Subject: [PATCH 02/11] Expansion from file in parallel --- .../LibUtilities/BasicUtils/MeshPartition.cpp | 259 ++++++++++++------ .../LibUtilities/BasicUtils/MeshPartition.h | 5 +- library/SpatialDomains/MeshGraph.cpp | 44 +-- 3 files changed, 196 insertions(+), 112 deletions(-) diff --git a/library/LibUtilities/BasicUtils/MeshPartition.cpp b/library/LibUtilities/BasicUtils/MeshPartition.cpp index 564b454dd..8b0d2e243 100644 --- a/library/LibUtilities/BasicUtils/MeshPartition.cpp +++ b/library/LibUtilities/BasicUtils/MeshPartition.cpp @@ -52,6 +52,7 @@ #include #include #include +#include #include @@ -195,12 +196,6 @@ namespace Nektar TiXmlElement *expansion = expansionTypes->FirstChildElement(); std::string expType = expansion->Value(); - - if(expType != "E") - { - ASSERTL0(false,"Expansion type not defined or not supported at the moment"); - } - /// Expansiontypes will contain plenty of data, /// where relevant at this stage are composite /// ID(s) that this expansion type describes, @@ -208,81 +203,175 @@ namespace Nektar /// expansion relates to. If this does not exist /// the variable is only set to "DefaultVar". - while (expansion) + if(expType == "E") { - std::vector composite; - std::vector nummodes; - std::vector fieldName; + while (expansion) + { + std::vector composite; + std::vector nummodes; + std::vector fieldName; - const char *nModesStr = expansion->Attribute("NUMMODES"); - ASSERTL0(nModesStr,"NUMMODES was not defined in EXPANSION section of input"); - std::string numModesStr = nModesStr; - bool valid = ParseUtils::GenerateOrderedVector(numModesStr.c_str(), nummodes); - ASSERTL0(valid, "Unable to correctly parse the number of modes."); + const char *nModesStr = expansion->Attribute("NUMMODES"); + ASSERTL0(nModesStr,"NUMMODES was not defined in EXPANSION section of input"); + std::string numModesStr = nModesStr; + bool valid = ParseUtils::GenerateOrderedVector(numModesStr.c_str(), nummodes); + ASSERTL0(valid, "Unable to correctly parse the number of modes."); - if (nummodes.size() == 1) - { - for (int i = 1; i < m_dim; i++) + if (nummodes.size() == 1) { - nummodes.push_back( nummodes[0] ); + for (int i = 1; i < m_dim; i++) + { + nummodes.push_back( nummodes[0] ); + } } - } - ASSERTL0(nummodes.size() == m_dim,"Number of modes should match mesh dimension"); + ASSERTL0(nummodes.size() == m_dim,"Number of modes should match mesh dimension"); - const char *fStr = expansion->Attribute("FIELDS"); - if(fStr) - { - std::string fieldStr = fStr; - bool valid = ParseUtils::GenerateOrderedStringVector(fieldStr.c_str(),fieldName); - ASSERTL0(valid,"Unable to correctly parse the field string in ExpansionTypes."); - - for (int i = 0; i < fieldName.size(); ++i) + const char *fStr = expansion->Attribute("FIELDS"); + if(fStr) { - if (m_fieldNameToId.count(fieldName[i]) == 0) + std::string fieldStr = fStr; + bool valid = ParseUtils::GenerateOrderedStringVector(fieldStr.c_str(),fieldName); + ASSERTL0(valid,"Unable to correctly parse the field string in ExpansionTypes."); + + for (int i = 0; i < fieldName.size(); ++i) { - int k = m_fieldNameToId.size(); - m_fieldNameToId[ fieldName[i] ] = k; - m_numFields++; + if (m_fieldNameToId.count(fieldName[i]) == 0) + { + int k = m_fieldNameToId.size(); + m_fieldNameToId[ fieldName[i] ] = k; + m_numFields++; + } } } - } - else - { - fieldName.push_back("DefaultVar"); - int k = m_fieldNameToId.size(); - - if (m_fieldNameToId.count("DefaultVar") == 0) + else { - ASSERTL0(k == 0, - "Omitting field variables and explicitly listing " \ - "them in different ExpansionTypes is wrong practise"); + fieldName.push_back("DefaultVar"); + int k = m_fieldNameToId.size(); - m_fieldNameToId[ "DefaultVar" ] = k; - m_numFields++; + if (m_fieldNameToId.count("DefaultVar") == 0) + { + ASSERTL0(k == 0, + "Omitting field variables and explicitly listing " \ + "them in different ExpansionTypes is wrong practise"); + + m_fieldNameToId[ "DefaultVar" ] = k; + m_numFields++; + } } - } - std::string compositeStr = expansion->Attribute("COMPOSITE"); - ASSERTL0(compositeStr.length() > 3, "COMPOSITE must be specified in expansion definition"); - int beg = compositeStr.find_first_of("["); - int end = compositeStr.find_first_of("]"); - std::string compositeListStr = compositeStr.substr(beg+1,end-beg-1); - bool parseGood = ParseUtils::GenerateSeqVector(compositeListStr.c_str(), composite); - ASSERTL0(parseGood && !composite.empty(), - (std::string("Unable to read composite index range: ") + compositeListStr).c_str()); + std::string compositeStr = expansion->Attribute("COMPOSITE"); + ASSERTL0(compositeStr.length() > 3, "COMPOSITE must be specified in expansion definition"); + int beg = compositeStr.find_first_of("["); + int end = compositeStr.find_first_of("]"); + std::string compositeListStr = compositeStr.substr(beg+1,end-beg-1); + bool parseGood = ParseUtils::GenerateSeqVector(compositeListStr.c_str(), composite); + ASSERTL0(parseGood && !composite.empty(), + (std::string("Unable to read composite index range: ") + compositeListStr).c_str()); - // construct mapping (field name, CompositeID) -> nummodes - for (int i = 0; i < composite.size(); ++i) - { - for (int j = 0; j < fieldName.size(); j++) + // construct mapping (elmt id, field name) -> nummodes + for (int i = 0; i < composite.size(); ++i) { - m_expansions[composite[i]][fieldName[j]] = nummodes; + for (int j = 0; j < fieldName.size(); j++) + { + for (unsigned int k = 0; k < m_meshComposites[i].list.size(); ++k) + { + int elid = m_meshComposites[composite[i]].list[k]; + m_expansions[elid][fieldName[j]] = nummodes; + m_shape[elid] = m_meshComposites[composite[i]].type; + } + } } - } - expansion = expansion->NextSiblingElement("E"); + expansion = expansion->NextSiblingElement("E"); + } + } + else if(expType == "F") + { + ASSERTL0(expansion->Attribute("FILE"), + "Attribute FILE expected for type F expansion"); + std::string filenameStr = expansion->Attribute("FILE"); + ASSERTL0(!filenameStr.empty(), + "A filename must be specified for the FILE " + "attribute of expansion"); + + std::vector fielddefs; + LibUtilities::FieldIO f(pSession->GetComm()); + f.Import(filenameStr, fielddefs); + + for (int i = 0; i < fielddefs.size(); ++i) + { + int numHomoDir = fielddefs[i]->m_numHomogeneousDir; + int cnt = 0; + for (int j = 0; j < fielddefs[i]->m_elementIDs.size(); ++j) + { + int elid = fielddefs[i]->m_elementIDs[j]; + std::vector nummodes; + for (int k = 0; k < m_dim; k++) + { + nummodes.push_back(fielddefs[i]->m_numModes[cnt++]); + } + if (fielddefs[i]->m_uniOrder) + { + cnt = 0; + } + else + { + cnt += numHomoDir; + } + for (int k = 0; k < fielddefs[i]->m_fields.size(); k++) + { + std::string fieldName = fielddefs[i]->m_fields[k]; + m_expansions[elid][fieldName] = nummodes; + } + switch (fielddefs[i]->m_shapeType) + { + case eSegment: + { + m_shape[elid] = 'S'; + break; + } + case eTriangle: + { + m_shape[elid] = 'T'; + break; + } + case eQuadrilateral: + { + m_shape[elid] = 'Q'; + break; + } + case eTetrahedron: + { + m_shape[elid] = 'A'; + break; + } + case ePyramid: + { + m_shape[elid] = 'R'; + break; + } + case ePrism: + { + m_shape[elid] = 'P'; + break; + } + case eHexahedron: + { + m_shape[elid] = 'H'; + break; + } + default: + ASSERTL0 (false, "Shape not recognized."); + break; + } + } + } + } + else + { + ASSERTL0(false,"Expansion type not defined or not supported at the moment"); } } @@ -931,17 +1020,18 @@ namespace Nektar std::map elmtSizes; std::map elmtBndSizes; - for (unsigned int i = 0; i < m_domain.size(); ++i) + for (std::map::iterator expIt = + m_expansions.begin(); expIt != m_expansions.end(); ++expIt) { - int cId = m_domain[i]; - NummodesPerField npf = m_expansions[cId]; + int elid = expIt->first; + NummodesPerField npf = expIt->second; for (NummodesPerField::iterator it = npf.begin(); it != npf.end(); ++it) { ASSERTL0(it->second.size() == m_dim, " Number of directional" \ - " modes in expansion spec for composite id = " + - boost::lexical_cast(cId) + + " modes in expansion spec for element id = " + + boost::lexical_cast(elid) + " and field " + boost::lexical_cast(it->first) + " does not correspond to mesh dimension"); @@ -954,17 +1044,10 @@ namespace Nektar nc = it->second[2]; } - int weight = CalculateElementWeight( - m_meshComposites[cId].type, false, na, nb, nc); - int bndWeight = CalculateElementWeight( - m_meshComposites[cId].type, true, na, nb, nc); - - for (unsigned int j = 0; j < m_meshComposites[cId].list.size(); ++j) - { - int elid = m_meshComposites[cId].list[j]; - elmtSizes[elid] = weight; - elmtBndSizes[elid] = bndWeight; - } + elmtSizes[elid] = CalculateElementWeight( + m_shape[elid], false, na, nb, nc); + elmtBndSizes[elid] = CalculateElementWeight( + m_shape[elid], true, na, nb, nc); } } @@ -1061,17 +1144,18 @@ namespace Nektar m_vertWeights[eIt->first] = weight; } - for (unsigned int i = 0; i < m_domain.size(); ++i) + for (std::map::iterator expIt = + m_expansions.begin(); expIt != m_expansions.end(); ++expIt) { - int cId = m_domain[i]; - NummodesPerField npf = m_expansions[cId]; + int elid = expIt->first; + NummodesPerField npf = expIt->second; for (NummodesPerField::iterator it = npf.begin(); it != npf.end(); ++it) { ASSERTL0(it->second.size() == m_dim, " Number of directional" \ - " modes in expansion spec for composite id = " + - boost::lexical_cast(cId) + + " modes in expansion spec for element id = " + + boost::lexical_cast(elid) + " and field " + boost::lexical_cast(it->first) + " does not correspond to mesh dimension"); @@ -1088,14 +1172,9 @@ namespace Nektar nc = it->second[2]; } - int bndWeight = CalculateElementWeight( - m_meshComposites[cId].type, true, na, nb, nc); - - for (unsigned int j = 0; j < m_meshComposites[cId].list.size(); ++j) - { - int elmtId = m_meshComposites[cId].list[j]; - m_vertWeights[elmtId][m_fieldNameToId[it->first]] = bndWeight; - } + m_vertWeights[elid][m_fieldNameToId[it->first]] = + CalculateElementWeight(m_shape[elid], true, + na, nb, nc); } } // for i } diff --git a/library/LibUtilities/BasicUtils/MeshPartition.h b/library/LibUtilities/BasicUtils/MeshPartition.h index 45d1e819a..2a0102e13 100644 --- a/library/LibUtilities/BasicUtils/MeshPartition.h +++ b/library/LibUtilities/BasicUtils/MeshPartition.h @@ -189,10 +189,13 @@ namespace Nektar std::vector m_domain; std::map m_vertexAttributes; - // hierarchial mapping: composite id -> field name -> integer list + // hierarchial mapping: elmt id -> field name -> integer list // of directional nummodes described by expansion type clause. std::map m_expansions; + // map of each elements shape + std::map m_shape; + std::map m_fieldNameToId; std::map m_vertWeights; diff --git a/library/SpatialDomains/MeshGraph.cpp b/library/SpatialDomains/MeshGraph.cpp index 7aa7620e4..c515c11a2 100644 --- a/library/SpatialDomains/MeshGraph.cpp +++ b/library/SpatialDomains/MeshGraph.cpp @@ -2375,7 +2375,7 @@ namespace Nektar std::string field = fielddef[i]->m_fields[j]; if(m_expansionMapShPtrMap.count(field) == 0) { - expansionMap = MemoryManager::AllocateSharedPtr(); + expansionMap = SetUpExpansionMap(); m_expansionMapShPtrMap[field] = expansionMap; // check to see if DefaultVar also not set and @@ -2384,26 +2384,6 @@ namespace Nektar { m_expansionMapShPtrMap["DefaultVar"] = expansionMap; } - - // loop over all elements and set expansion - for(k = 0; k < fielddef.size(); ++k) - { - for(int h = 0; h < fielddef[k]->m_fields.size(); ++h) - { - if(fielddef[k]->m_fields[h] == field) - { - expansionMap = m_expansionMapShPtrMap.find(field)->second; - LibUtilities::BasisKeyVector def; - - for(int g = 0; g < fielddef[k]->m_elementIDs.size(); ++g) - { - ExpansionShPtr tmpexp = - MemoryManager::AllocateSharedPtr(geom, def); - (*expansionMap)[fielddef[k]->m_elementIDs[g]] = tmpexp; - } - } - } - } } } } @@ -2475,6 +2455,11 @@ namespace Nektar if(m_triGeoms.count(fielddef[i]->m_elementIDs[j]) == 0) { // skip element likely from parallel read + if(!UniOrder) + { + cnt += 2; + cnt += fielddef[i]->m_numHomogeneousDir; + } continue; } geom = m_triGeoms[fielddef[i]->m_elementIDs[j]]; @@ -2530,6 +2515,11 @@ namespace Nektar if(m_quadGeoms.count(fielddef[i]->m_elementIDs[j]) == 0) { // skip element likely from parallel read + if(!UniOrder) + { + cnt += 2; + cnt += fielddef[i]->m_numHomogeneousDir; + } continue; } @@ -2575,6 +2565,10 @@ namespace Nektar // parallel runs if(m_tetGeoms.count(k) == 0) { + if(!UniOrder) + { + cnt += 3; + } continue; } geom = m_tetGeoms[k]; @@ -2661,6 +2655,10 @@ namespace Nektar k = fielddef[i]->m_elementIDs[j]; if(m_prismGeoms.count(k) == 0) { + if(!UniOrder) + { + cnt += 3; + } continue; } geom = m_prismGeoms[k]; @@ -2784,6 +2782,10 @@ namespace Nektar k = fielddef[i]->m_elementIDs[j]; if(m_hexGeoms.count(k) == 0) { + if(!UniOrder) + { + cnt += 3; + } continue; } -- GitLab From 56112160d79fd82f8b77a4074de3414bf6fbe464 Mon Sep 17 00:00:00 2001 From: Douglas Serson Date: Thu, 21 Apr 2016 19:06:23 +0100 Subject: [PATCH 03/11] Add some information to the user guide --- docs/user-guide/solvers/incompressible-ns.tex | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/user-guide/solvers/incompressible-ns.tex b/docs/user-guide/solvers/incompressible-ns.tex index 2e2d6bb73..6a85f212c 100644 --- a/docs/user-guide/solvers/incompressible-ns.tex +++ b/docs/user-guide/solvers/incompressible-ns.tex @@ -1103,6 +1103,20 @@ element is taken as the average of the function in the quadrature points of the element. If these functions are defined, the values set for the tolerances in the \texttt{PARAMETERS} section are ignored. +\subsection{Restarting the simulation} + +The simulation can be restarted using the final distribution of +polynomial orders obtained from the adaptive procedure by setting +the expansions as +\begin{lstlisting}[style=XMLStyle] + + + +\end{lstlisting} +note that this will only affect the polynomial order. The initial condition +still needs to be set correctly, and does not need to come from the same file +used for the expansions. + \section{Examples} \subsection{Kovasznay Flow 2D} -- GitLab From 9743915ad52f99470fe9d07b80df4c8c5cb8efbf Mon Sep 17 00:00:00 2001 From: Douglas Serson Date: Fri, 22 Apr 2016 11:15:08 +0100 Subject: [PATCH 04/11] Small fixes in 1D --- library/LibUtilities/BasicUtils/MeshPartition.cpp | 6 +++++- library/SpatialDomains/MeshGraph.cpp | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/library/LibUtilities/BasicUtils/MeshPartition.cpp b/library/LibUtilities/BasicUtils/MeshPartition.cpp index 8b0d2e243..ffaa489b3 100644 --- a/library/LibUtilities/BasicUtils/MeshPartition.cpp +++ b/library/LibUtilities/BasicUtils/MeshPartition.cpp @@ -1037,8 +1037,12 @@ namespace Nektar " does not correspond to mesh dimension"); int na = it->second[0]; - int nb = it->second[1]; + int nb = 0; int nc = 0; + if (m_dim >= 2) + { + nb = it->second[1]; + } if (m_dim == 3) { nc = it->second[2]; diff --git a/library/SpatialDomains/MeshGraph.cpp b/library/SpatialDomains/MeshGraph.cpp index c515c11a2..832fbb2e6 100644 --- a/library/SpatialDomains/MeshGraph.cpp +++ b/library/SpatialDomains/MeshGraph.cpp @@ -2418,6 +2418,11 @@ namespace Nektar if(m_segGeoms.count(fielddef[i]->m_elementIDs[j]) == 0) { // skip element likely from parallel read + if(!UniOrder) + { + cnt++; + cnt += fielddef[i]->m_numHomogeneousDir; + } continue; } geom = m_segGeoms[fielddef[i]->m_elementIDs[j]]; -- GitLab From c16661dfc170fdf04acba6216b931b422abcf79d Mon Sep 17 00:00:00 2001 From: Douglas Serson Date: Fri, 22 Apr 2016 14:35:57 +0100 Subject: [PATCH 05/11] Start working on partition weighting --- .../LibUtilities/BasicUtils/MeshPartition.cpp | 74 +++++++++++++++++-- .../LibUtilities/BasicUtils/MeshPartition.h | 6 ++ .../BasicUtils/MeshPartitionMetis.cpp | 3 +- .../BasicUtils/MeshPartitionMetis.h | 1 + .../BasicUtils/MeshPartitionScotch.cpp | 1 + .../BasicUtils/MeshPartitionScotch.h | 1 + library/LibUtilities/BasicUtils/Metis.hpp | 12 ++- 7 files changed, 88 insertions(+), 10 deletions(-) diff --git a/library/LibUtilities/BasicUtils/MeshPartition.cpp b/library/LibUtilities/BasicUtils/MeshPartition.cpp index ffaa489b3..1dd60f457 100644 --- a/library/LibUtilities/BasicUtils/MeshPartition.cpp +++ b/library/LibUtilities/BasicUtils/MeshPartition.cpp @@ -275,7 +275,7 @@ namespace Nektar { for (int j = 0; j < fieldName.size(); j++) { - for (unsigned int k = 0; k < m_meshComposites[i].list.size(); ++k) + for (unsigned int k = 0; k < m_meshComposites[composite[i]].list.size(); ++k) { int elid = m_meshComposites[composite[i]].list[k]; m_expansions[elid][fieldName[j]] = nummodes; @@ -1146,6 +1146,8 @@ namespace Nektar for (eIt = m_meshElements.begin(); eIt != m_meshElements.end(); ++eIt) { m_vertWeights[eIt->first] = weight; + m_vertBndWeights[eIt->first] = weight; + m_edgeWeights[eIt->first] = weight; } for (std::map::iterator expIt = @@ -1177,8 +1179,14 @@ namespace Nektar } m_vertWeights[elid][m_fieldNameToId[it->first]] = + CalculateElementWeight(m_shape[elid], false, + na, nb, nc); + m_vertBndWeights[elid][m_fieldNameToId[it->first]] = CalculateElementWeight(m_shape[elid], true, na, nb, nc); + m_edgeWeights[elid][m_fieldNameToId[it->first]] = + CalculateEdgeWeight(m_shape[elid], + na, nb, nc); } } // for i } @@ -1200,7 +1208,9 @@ namespace Nektar if (m_weightingRequired) { - pGraph[v].weight = m_vertWeights[eIt->first]; + pGraph[v].weight = m_vertWeights[eIt->first]; + pGraph[v].bndWeight = m_vertBndWeights[eIt->first]; + pGraph[v].edgeWeight = m_edgeWeights[eIt->first]; } // Process element entries and add graph edges @@ -1258,9 +1268,10 @@ namespace Nektar { int acnt = 0; int vcnt = 0; - int nWeight = nGraphVerts; + int nWeight = 2*nGraphVerts; Array xadj(nGraphVerts+1,0); Array adjncy(2*nGraphEdges); + Array adjwgt(2*nGraphEdges, 1); Array vwgt(nWeight, 1); Array vsize(nGraphVerts, 1); @@ -1273,13 +1284,18 @@ namespace Nektar ++adjvertit) { adjncy[acnt++] = *adjvertit; + if (m_weightingRequired) + { + adjwgt[acnt-1] = pGraph[*vertit].edgeWeight[0]; + } } xadj[++vcnt] = acnt; if (m_weightingRequired) { - vwgt[vcnt-1] = pGraph[*vertit].weight[0]; + vwgt[2*(vcnt-1)] = pGraph[*vertit].weight[0]; + vwgt[2*(vcnt-1)+1] = pGraph[*vertit].bndWeight[0]; } else { @@ -1298,8 +1314,8 @@ namespace Nektar if(m_comm->GetColumnComm()->GetRank() == 0) { // Attempt partitioning using METIS. - int ncon = 1; - PartitionGraphImpl(nGraphVerts, ncon, xadj, adjncy, vwgt, vsize, nParts, vol, part); + int ncon = 2; + PartitionGraphImpl(nGraphVerts, ncon, xadj, adjncy, vwgt, vsize, adjwgt, nParts, vol, part); // Check METIS produced a valid partition and fix if not. CheckPartitions(nParts, part); @@ -2306,5 +2322,51 @@ namespace Nektar return weight; } + + /** + * Calculate the number of modes needed for communication when + * in partition boundary, to be used as weighting for edges. + * Since we do not know exactly which face this refers to, assume + * the max order and quad face (for prisms) as arbitrary choices + */ + int MeshPartition::CalculateEdgeWeight( + char elmtType, + int na, + int nb, + int nc) + { + int weight = 0; + int n = std::max ( na, std::max(nb, nc)); + switch (elmtType) + { + case 'A': + weight = + StdTriData ::getNumberOfCoefficients (n, n); + break; + case 'R': + weight = + StdQuadData ::getNumberOfCoefficients (n, n); + break; + case 'H': + weight = + StdQuadData ::getNumberOfCoefficients (n, n); + break; + case 'P': + weight = + StdQuadData ::getNumberOfCoefficients (n, n); + break; + case 'Q': + case 'T': + weight = n; + break; + case 'S': + weight = 1; + break; + default: + break; + } + + return weight; + } } } diff --git a/library/LibUtilities/BasicUtils/MeshPartition.h b/library/LibUtilities/BasicUtils/MeshPartition.h index 2a0102e13..d2850cd34 100644 --- a/library/LibUtilities/BasicUtils/MeshPartition.h +++ b/library/LibUtilities/BasicUtils/MeshPartition.h @@ -131,6 +131,8 @@ namespace Nektar int id; ///< Universal ID of the vertex int partition; ///< Index of the partition to which it belongs MultiWeight weight; ///< Weightings to this graph vertex + MultiWeight bndWeight; + MultiWeight edgeWeight; }; // Face/Edge/Vertex between two adjacent elements @@ -198,6 +200,8 @@ namespace Nektar std::map m_fieldNameToId; std::map m_vertWeights; + std::map m_vertBndWeights; + std::map m_edgeWeights; BndRegionOrdering m_bndRegOrder; @@ -226,6 +230,7 @@ namespace Nektar Nektar::Array& adjcy, Nektar::Array& vertWgt, Nektar::Array& vertSize, + Nektar::Array& edgeWgt, int& nparts, int& volume, Nektar::Array& part) = 0; @@ -233,6 +238,7 @@ namespace Nektar void OutputPartition(SessionReaderSharedPtr& pSession, BoostSubGraph& pGraph, TiXmlElement* pGeometry); void CheckPartitions(int nParts, Array &pPart); int CalculateElementWeight(char elmtType, bool bndWeight, int na, int nb, int nc); + int CalculateEdgeWeight(char elmtType, int na, int nb, int nc); }; typedef boost::shared_ptr MeshPartitionSharedPtr; diff --git a/library/LibUtilities/BasicUtils/MeshPartitionMetis.cpp b/library/LibUtilities/BasicUtils/MeshPartitionMetis.cpp index c8049547a..49d549a6d 100644 --- a/library/LibUtilities/BasicUtils/MeshPartitionMetis.cpp +++ b/library/LibUtilities/BasicUtils/MeshPartitionMetis.cpp @@ -70,11 +70,12 @@ namespace LibUtilities Nektar::Array& adjcy, Nektar::Array& vertWgt, Nektar::Array& vertSize, + Nektar::Array& edgeWgt, int& nparts, int& volume, Nektar::Array& part) { - Metis::PartGraphVKway(nVerts, nVertConds, xadj, adjcy, vertWgt, vertSize, nparts, volume, part); + Metis::PartGraphVKway(nVerts, nVertConds, xadj, adjcy, vertWgt, vertSize, edgeWgt, nparts, volume, part); } } } diff --git a/library/LibUtilities/BasicUtils/MeshPartitionMetis.h b/library/LibUtilities/BasicUtils/MeshPartitionMetis.h index 76079d88a..7a607b869 100644 --- a/library/LibUtilities/BasicUtils/MeshPartitionMetis.h +++ b/library/LibUtilities/BasicUtils/MeshPartitionMetis.h @@ -68,6 +68,7 @@ namespace LibUtilities Nektar::Array& adjcy, Nektar::Array& vertWgt, Nektar::Array& vertSize, + Nektar::Array& edgeWgt, int& nparts, int& volume, Nektar::Array& part); diff --git a/library/LibUtilities/BasicUtils/MeshPartitionScotch.cpp b/library/LibUtilities/BasicUtils/MeshPartitionScotch.cpp index 4f8894896..ff0d5c455 100644 --- a/library/LibUtilities/BasicUtils/MeshPartitionScotch.cpp +++ b/library/LibUtilities/BasicUtils/MeshPartitionScotch.cpp @@ -70,6 +70,7 @@ namespace LibUtilities Nektar::Array& adjcy, Nektar::Array& vertWgt, Nektar::Array& vertSize, + Nektar::Array& edgeWgt, int& nparts, int& volume, Nektar::Array& part) diff --git a/library/LibUtilities/BasicUtils/MeshPartitionScotch.h b/library/LibUtilities/BasicUtils/MeshPartitionScotch.h index d4106f749..4dc4ab568 100644 --- a/library/LibUtilities/BasicUtils/MeshPartitionScotch.h +++ b/library/LibUtilities/BasicUtils/MeshPartitionScotch.h @@ -72,6 +72,7 @@ namespace LibUtilities Nektar::Array& adjcy, Nektar::Array& vertWgt, Nektar::Array& vertSize, + Nektar::Array& edgeWgt, int& nparts, int& volume, Nektar::Array& part); diff --git a/library/LibUtilities/BasicUtils/Metis.hpp b/library/LibUtilities/BasicUtils/Metis.hpp index 7e019f3ad..05bc1c416 100644 --- a/library/LibUtilities/BasicUtils/Metis.hpp +++ b/library/LibUtilities/BasicUtils/Metis.hpp @@ -75,12 +75,14 @@ namespace Metis Nektar::Array& adjcy, Nektar::Array& vertWgt, Nektar::Array& vertSize, + Nektar::Array& edgeWgt, int& nparts, int& volume, Nektar::Array& part) { - int *vwgt = 0; - int *vsize = 0; + int *vwgt = 0; + int *vsize = 0; + int *adjwgt = 0; if (vertWgt.num_elements() > 0) { vwgt = &vertWgt[0]; @@ -89,12 +91,16 @@ namespace Metis { vsize = &vertSize[0]; } + if (edgeWgt.num_elements() > 0) + { + adjwgt = &edgeWgt[0]; + } // number of balancing conditions (size of vertex multi-weight) int ncon = nVertConds; int options[METIS_NOPTIONS]; METIS_SetDefaultOptions(options); METIS_PartGraphKway(&nVerts, &ncon, &xadj[0], &adjcy[0], vwgt, vsize, - 0, &nparts, 0, 0, options, &volume, &part[0]); + adjwgt, &nparts, 0, 0, options, &volume, &part[0]); } } #endif //NEKTAR_LIB_UTILITIES_BASICUTILS_METIS_HPP -- GitLab From 6081d685be5b0231c1a6cf7d78bd53a8ea965541 Mon Sep 17 00:00:00 2001 From: Douglas Serson Date: Fri, 22 Apr 2016 14:55:57 +0100 Subject: [PATCH 06/11] Add options for partitioning --- .../LibUtilities/BasicUtils/MeshPartition.cpp | 42 ++++++++++++++----- .../LibUtilities/BasicUtils/MeshPartition.h | 2 + 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/library/LibUtilities/BasicUtils/MeshPartition.cpp b/library/LibUtilities/BasicUtils/MeshPartition.cpp index 1dd60f457..aff885cdd 100644 --- a/library/LibUtilities/BasicUtils/MeshPartition.cpp +++ b/library/LibUtilities/BasicUtils/MeshPartition.cpp @@ -81,7 +81,9 @@ namespace Nektar m_numFields(0), m_fieldNameToId(), m_comm(pSession->GetComm()), - m_weightingRequired(false) + m_weightingRequired(false), + m_weightBnd(false), + m_weightDofs(false) { ReadConditions(pSession); ReadGeometry(pSession); @@ -1114,9 +1116,21 @@ namespace Nektar if (solverPropertyUpper == "WEIGHTPARTITIONS") { - if (propertyValueUpper != "UNIFORM") + if (propertyValueUpper == "DOF") { m_weightingRequired = true; + m_weightDofs = true; + } + else if (propertyValueUpper == "BOUNDARY") + { + m_weightingRequired = true; + m_weightBnd = true; + } + else if (propertyValueUpper == "BOTH") + { + m_weightingRequired = true; + m_weightDofs = true; + m_weightBnd = true; } return; } @@ -1259,6 +1273,11 @@ namespace Nektar int nGraphVerts = boost::num_vertices(pGraph); int nGraphEdges = boost::num_edges(pGraph); + int ncon = 1; + if (m_weightDofs && m_weightBnd) + { + ncon = 2; + } // Convert boost graph into CSR format BoostVertexIterator vertit, vertit_end; BoostAdjacencyIterator adjvertit, adjvertit_end; @@ -1268,7 +1287,7 @@ namespace Nektar { int acnt = 0; int vcnt = 0; - int nWeight = 2*nGraphVerts; + int nWeight = ncon*nGraphVerts; Array xadj(nGraphVerts+1,0); Array adjncy(2*nGraphEdges); Array adjwgt(2*nGraphEdges, 1); @@ -1294,12 +1313,16 @@ namespace Nektar if (m_weightingRequired) { - vwgt[2*(vcnt-1)] = pGraph[*vertit].weight[0]; - vwgt[2*(vcnt-1)+1] = pGraph[*vertit].bndWeight[0]; - } - else - { - vwgt[vcnt-1] = 1; + int ccnt = 0; + if (m_weightDofs) + { + vwgt[ncon*(vcnt-1)+ccnt] = pGraph[*vertit].weight[0]; + ccnt++; + } + if (m_weightBnd) + { + vwgt[ncon*(vcnt-1)+ccnt] = pGraph[*vertit].bndWeight[0]; + } } } @@ -1314,7 +1337,6 @@ namespace Nektar if(m_comm->GetColumnComm()->GetRank() == 0) { // Attempt partitioning using METIS. - int ncon = 2; PartitionGraphImpl(nGraphVerts, ncon, xadj, adjncy, vwgt, vsize, adjwgt, nParts, vol, part); // Check METIS produced a valid partition and fix if not. diff --git a/library/LibUtilities/BasicUtils/MeshPartition.h b/library/LibUtilities/BasicUtils/MeshPartition.h index d2850cd34..ef3fba69c 100644 --- a/library/LibUtilities/BasicUtils/MeshPartition.h +++ b/library/LibUtilities/BasicUtils/MeshPartition.h @@ -211,6 +211,8 @@ namespace Nektar CommSharedPtr m_comm; bool m_weightingRequired; + bool m_weightBnd; + bool m_weightDofs; bool m_shared; void ReadExpansions(const SessionReaderSharedPtr& pSession); -- GitLab From 89914f2fccdb8b73137da68aa91b07269fd24e44 Mon Sep 17 00:00:00 2001 From: Douglas Serson Date: Fri, 22 Apr 2016 15:46:14 +0100 Subject: [PATCH 07/11] Fix small problem in MeshGraph --- library/SpatialDomains/MeshGraph.cpp | 35 +++++++--------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/library/SpatialDomains/MeshGraph.cpp b/library/SpatialDomains/MeshGraph.cpp index 832fbb2e6..c259cb77d 100644 --- a/library/SpatialDomains/MeshGraph.cpp +++ b/library/SpatialDomains/MeshGraph.cpp @@ -2852,7 +2852,7 @@ namespace Nektar std::vector &fielddef, std::vector< std::vector > &pointstype) { - int i,j,k,g,h,cnt,id; + int i,j,k,cnt,id; GeometrySharedPtr geom; ExpansionMapShPtr expansionMap; @@ -2866,7 +2866,7 @@ namespace Nektar std::string field = fielddef[i]->m_fields[j]; if(m_expansionMapShPtrMap.count(field) == 0) { - expansionMap = MemoryManager::AllocateSharedPtr(); + expansionMap = SetUpExpansionMap(); m_expansionMapShPtrMap[field] = expansionMap; // check to see if DefaultVar also not set and @@ -2875,26 +2875,6 @@ namespace Nektar { m_expansionMapShPtrMap["DefaultVar"] = expansionMap; } - - // loop over all elements and set expansion - for(k = 0; k < fielddef.size(); ++k) - { - for(h = 0; h < fielddef[k]->m_fields.size(); ++h) - { - if(fielddef[k]->m_fields[h] == field) - { - expansionMap = m_expansionMapShPtrMap.find(field)->second; - LibUtilities::BasisKeyVector def; - - for(g = 0; g < fielddef[k]->m_elementIDs.size(); ++g) - { - ExpansionShPtr tmpexp = - MemoryManager::AllocateSharedPtr(geom, def); - (*expansionMap)[fielddef[k]->m_elementIDs[g]] = tmpexp; - } - } - } - } } } } @@ -2929,6 +2909,7 @@ namespace Nektar if(!UniOrder) { cnt++; + cnt += fielddef[i]->m_numHomogeneousDir; } bkeyvec.push_back(bkey); } @@ -2949,6 +2930,7 @@ namespace Nektar if(!UniOrder) { cnt += 2; + cnt += fielddef[i]->m_numHomogeneousDir; } } break; @@ -2969,6 +2951,7 @@ namespace Nektar if(!UniOrder) { cnt += 2; + cnt += fielddef[i]->m_numHomogeneousDir; } } break; @@ -2988,7 +2971,7 @@ namespace Nektar if(!UniOrder) { - cnt += 2; + cnt += 3; } } break; @@ -3008,7 +2991,7 @@ namespace Nektar if(!UniOrder) { - cnt += 2; + cnt += 3; } } break; @@ -3028,7 +3011,7 @@ namespace Nektar if(!UniOrder) { - cnt += 2; + cnt += 3; } } break; @@ -3048,7 +3031,7 @@ namespace Nektar if(!UniOrder) { - cnt += 2; + cnt += 3; } } break; -- GitLab From b861322334943faff867d0a55232898022402207 Mon Sep 17 00:00:00 2001 From: Douglas Serson Date: Mon, 25 Apr 2016 14:33:00 +0100 Subject: [PATCH 08/11] Fix partitioning when loading expansion from file --- library/LibUtilities/BasicUtils/MeshPartition.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/library/LibUtilities/BasicUtils/MeshPartition.cpp b/library/LibUtilities/BasicUtils/MeshPartition.cpp index aff885cdd..af85aa167 100644 --- a/library/LibUtilities/BasicUtils/MeshPartition.cpp +++ b/library/LibUtilities/BasicUtils/MeshPartition.cpp @@ -298,12 +298,26 @@ namespace Nektar "A filename must be specified for the FILE " "attribute of expansion"); + // Load field definitions from file std::vector fielddefs; LibUtilities::FieldIO f(pSession->GetComm()); f.Import(filenameStr, fielddefs); + // Parse field definitions for (int i = 0; i < fielddefs.size(); ++i) { + // Name of fields + for (int j = 0; j < fielddefs[i]->m_fields.size(); ++j) + { + std::string fieldName = fielddefs[i]->m_fields[j]; + if (m_fieldNameToId.count(fieldName) == 0) + { + int k = m_fieldNameToId.size(); + m_fieldNameToId[ fieldName ] = k; + m_numFields++; + } + } + // Number of modes and shape for each element int numHomoDir = fielddefs[i]->m_numHomogeneousDir; int cnt = 0; for (int j = 0; j < fielddefs[i]->m_elementIDs.size(); ++j) -- GitLab From a0626bc9c714f167d2ef34f9bd72ac3c981724bf Mon Sep 17 00:00:00 2001 From: Douglas Serson Date: Sat, 4 Jun 2016 14:18:41 +0100 Subject: [PATCH 09/11] Add regression test --- solvers/IncNavierStokesSolver/CMakeLists.txt | 1 + .../Tests/KovaFlow_expFromFile.xml | 86 +++++++++++++++++++ .../Tests/KovaFlow_expFromFile_par.tst | 23 +++++ 3 files changed, 110 insertions(+) create mode 100644 solvers/IncNavierStokesSolver/Tests/KovaFlow_expFromFile.xml create mode 100644 solvers/IncNavierStokesSolver/Tests/KovaFlow_expFromFile_par.tst diff --git a/solvers/IncNavierStokesSolver/CMakeLists.txt b/solvers/IncNavierStokesSolver/CMakeLists.txt index 1fc6e37dd..8f7c0ebc8 100644 --- a/solvers/IncNavierStokesSolver/CMakeLists.txt +++ b/solvers/IncNavierStokesSolver/CMakeLists.txt @@ -115,6 +115,7 @@ IF( NEKTAR_SOLVER_INCNAVIERSTOKES ) ADD_NEKTAR_TEST(ChanFlow_3DH1D_Parallel_mode1) ADD_NEKTAR_TEST(ChanFlow_3DH1D_Parallel_mode2) ADD_NEKTAR_TEST(ChanFlow_m3_par) + ADD_NEKTAR_TEST(KovaFlow_expFromFile_par) ADD_NEKTAR_TEST_LENGTHY(ChanFlow_m8_BodyForce_par) ADD_NEKTAR_TEST_LENGTHY(Hex_channel_m8_par) ADD_NEKTAR_TEST_LENGTHY(Pyr_channel_m6_par) diff --git a/solvers/IncNavierStokesSolver/Tests/KovaFlow_expFromFile.xml b/solvers/IncNavierStokesSolver/Tests/KovaFlow_expFromFile.xml new file mode 100644 index 000000000..53c65fd95 --- /dev/null +++ b/solvers/IncNavierStokesSolver/Tests/KovaFlow_expFromFile.xml @@ -0,0 +1,86 @@ + + + + + + + + + + + + + + + + + + + + +

TimeStep = 0.001

+

NumSteps = 100

+

IO_CheckSteps = 100

+

IO_InfoSteps = 100

+

Kinvis = 0.025

+
+ + + u + v + p + + + + C[1] + C[2] + C[3] + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + eJxjYEAGD/aj0gwMjFjlf9jDRJhQ5D/Yo8szY5VHmM+C1fwPcP2s2N0Hl2dDk18/MZnnf1aFLUyEHUpHO9a8Cq6u2YtuPweqfgz3caK5H2r+XpgIF5o8uvu4scoj/MeD1X5E+PGiut8WXZ4Pq36E+fxY5RHuE0CTdze//3OXWwLcf4JQ+vG+paKKtTm26OYLQemmsp+NV/7326CbL4zi/py9UPPh8QMANMBNrQAA + eJx1kskSgjAQBUFckDUIigv6/5/pgelLVyWXV/3yMjWTpCj2dQjtQkvxoBzchlZickflx9CTuAk9i6l3UT6F1uJrRhupz7di5unE1Ou1Dw/iSX1PyiX5twyPYnTOKHUX+dzjXczcD/XPP1jFvOtTTN8v+bz3W8wcH/n8g03MHF/5rJ+Y+f7dGwRp + + eJx1zskOgyAABmFsRaq0bl3t4vs/ppfh4CTl8iVDAn8I+1PhAY9/eo0RG92XnvCErd4tvcOMZ/1zwR4H7YrqI056p1Gf8aq9Sf2Gd+0t+x/4xJf2tuoLvrW3U//gV3uz+g9X7d0A9rcC/gAA + + + Q[0-11] + E[23,25,27,29] + E[3,6,9,12] + E[0,11,13,21-22,30] + + C[0] + +
diff --git a/solvers/IncNavierStokesSolver/Tests/KovaFlow_expFromFile_par.tst b/solvers/IncNavierStokesSolver/Tests/KovaFlow_expFromFile_par.tst new file mode 100644 index 000000000..1a3320d8f --- /dev/null +++ b/solvers/IncNavierStokesSolver/Tests/KovaFlow_expFromFile_par.tst @@ -0,0 +1,23 @@ + + + Kovasznay Flow, expansion from restart file, par(2) + IncNavierStokesSolver + --use-metis KovaFlow_expFromFile.xml + 2 + + KovaFlow_expFromFile.xml + KovaFlow_m8.rst + + + + 3.22767e-07 + 1.68898e-06 + 9.87849e-06 + + + 4.31288e-07 + 2.23918e-06 + 3.43342e-05 + + + -- GitLab From 67ed2eb6a6da9a32f3a5a421e10129f98820df50 Mon Sep 17 00:00:00 2001 From: Chris Cantwell Date: Thu, 18 Aug 2016 10:55:02 +0100 Subject: [PATCH 10/11] Updated CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77315a775..d40b65be8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,8 @@ v4.4.0 - Add support for HDF5 as an alternative output to XML-based output, including refactoring of FieldIO, improvements to MPI interface and added communicators to boundary conditions (!615) +- Allow expansions to be loaded directly from field file (!617) +- New options for load balancing (DOF or BOUNDARY) in mesh partitioner (!617) **ADRSolver:** - Add a projection equation system for C^0 projections (!675) -- GitLab From 312cc844ebf373cca9783ba4256bbd0b28d48e81 Mon Sep 17 00:00:00 2001 From: Douglas Serson Date: Thu, 18 Aug 2016 16:20:20 +0100 Subject: [PATCH 11/11] Adjust expansion from file to Hdf5 --- library/LibUtilities/BasicUtils/MeshPartition.cpp | 14 ++++++++++++-- library/SpatialDomains/MeshGraph.cpp | 6 ++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/library/LibUtilities/BasicUtils/MeshPartition.cpp b/library/LibUtilities/BasicUtils/MeshPartition.cpp index af85aa167..6605afe1a 100644 --- a/library/LibUtilities/BasicUtils/MeshPartition.cpp +++ b/library/LibUtilities/BasicUtils/MeshPartition.cpp @@ -298,10 +298,20 @@ namespace Nektar "A filename must be specified for the FILE " "attribute of expansion"); + // Create fieldIO object to load file + // need a serial communicator to avoid problems with + // shared file system + CommSharedPtr comm= + GetCommFactory().CreateInstance("Serial", 0, 0); + std::string iofmt = FieldIO::GetFileType( + filenameStr, comm); + FieldIOSharedPtr f = GetFieldIOFactory().CreateInstance( + iofmt, + comm, + pSession->GetSharedFilesystem()); // Load field definitions from file std::vector fielddefs; - LibUtilities::FieldIO f(pSession->GetComm()); - f.Import(filenameStr, fielddefs); + f->Import(filenameStr, fielddefs); // Parse field definitions for (int i = 0; i < fielddefs.size(); ++i) diff --git a/library/SpatialDomains/MeshGraph.cpp b/library/SpatialDomains/MeshGraph.cpp index 70ff29da0..6314bce2f 100644 --- a/library/SpatialDomains/MeshGraph.cpp +++ b/library/SpatialDomains/MeshGraph.cpp @@ -1067,8 +1067,10 @@ namespace Nektar "attribute of expansion"); std::vector fielddefs; - LibUtilities::FieldIO f(m_session->GetComm()); - f.Import(filenameStr, fielddefs); + LibUtilities::FieldIOSharedPtr f = + LibUtilities::FieldIO::CreateForFile( + m_session, filenameStr); + f->Import(filenameStr, fielddefs); SetExpansions(fielddefs); } else -- GitLab