Commit 76d4865d authored by Dave Moxey's avatar Dave Moxey

Merge branch 'master' into 'ticket/97-nekmesh-1d'

# Conflicts:
#   CHANGELOG.md
parents dc73d485 4dcc5ec3
......@@ -16,6 +16,7 @@ v5.0.0
- Add ARPACK thirdparty build capabilities (!828)
- Added native support for csv files in addititon to pts (!760, !835, !906)
- Utilize LAPACK_DIR env variable to find the native blas/lapack install (!827)
- Extend AeroForces filter to compressible flows (!815)
- Remove StdExpansion use from MultiRegion (use Expansions instead). (!831)
- Move steady state check and CFL output from solvers to SolverUtils (!832)
- Remove DG advection implementation from EquationSystem (!832)
......@@ -47,6 +48,7 @@ v5.0.0
WeightPartitions was used in parallel (!923)
- Removed instance count from beginning of Array storage to improve memory
alignment (!921)
- Fix naming issue of duplicate Unit tests (!924)
**NekMesh**:
- Add feature to read basic 2D geo files as CAD (!731)
......
......@@ -246,6 +246,7 @@ INCLUDE (ThirdPartyOCE)
INCLUDE (ThirdPartyTriangle)
INCLUDE (ThirdPartyTetGen)
INCLUDE (ThirdPartyCCM)
INCLUDE (ThirdPartyCwipi)
INCLUDE (FindCFI)
INCLUDE (Doxygen)
......
# - Try to find cwipi
# Once done this will define
# CWIPI_FOUND - System has cwipi
# CWIPI_INCLUDE_DIRS - The cwipi include directories
# CWIPI_LIBRARIES - The libraries needed to use cwipi
# CWIPI_DEFINITIONS - Compiler switches required for using cwipi
SET(CWIPI_SEARCH_PATHS
${CMAKE_SOURCE_DIR}/ThirdParty/cwipi/build/lib
${CMAKE_SOURCE_DIR}/../ThirdParty/cwipi/build/lib
${CMAKE_SOURCE_DIR}/ThirdParty/dist/build/lib
${CMAKE_SOURCE_DIR}/../ThirdParty/dist/build/lib)
FIND_PATH(CWIPI_INCLUDE_DIR NAMES cwipi.h PATHS ${CWIPI_SEARCH_PATHS})
FIND_LIBRARY(CWIPI_LIBRARY NAMES cwipi PATHS ${CWIPI_SEARCH_PATHS})
FIND_LIBRARY(CWIPI_LIBRARY_FVMC NAMES fvmc PATHS ${CWIPI_SEARCH_PATHS})
FIND_LIBRARY(CWIPI_LIBRARY_BFTC NAMES BFTC PATHS ${CWIPI_SEARCH_PATHS})
SET(CWIPI_FOUND FALSE)
IF (CWIPI_LIBRARY)
SET(CWIPI_FOUND TRUE)
INCLUDE_DIRECTORIES(${CWIPI_INCLUDE_DIR})
MARK_AS_ADVANCED(CWIPI_LIBRARY)
MARK_AS_ADVANCED(CWIPI_LIBRARY_FVMC)
MARK_AS_ADVANCED(CWIPI_LIBRARY_BFTC)
MARK_AS_ADVANCED(CWIPI_INCLUDE_DIR)
ENDIF (CWIPI_LIBRARY)
IF (CWIPI_FOUND)
IF (NOT CWIPI_FIND_QUIETLY)
MESSAGE(STATUS "Found CWIPI: ${CWIPI_INCLUDE_DIR}")
ENDIF (NOT CWIPI_FIND_QUIETLY)
ELSE(CWIPI_FOUND)
IF (CWIPI_FIND_REQUIRED)
MESSAGE(FATAL_ERROR "Could not find CWIPI")
ENDIF (CWIPI_FIND_REQUIRED)
ENDIF (CWIPI_FOUND)
########################################################################
#
# ThirdParty configuration for Nektar++
#
# CWIPI
#
########################################################################
OPTION(NEKTAR_USE_CWIPI
"Use CWIPI for Coupling." OFF)
IF ( NEKTAR_USE_CWIPI )
find_package(Cwipi)
include_directories(${CWIPI_INCLUDE_DIRS})
ADD_DEFINITIONS(-DNEKTAR_USE_CWIPI)
# Set some common CWIPI search paths.
SET(CWIPI_SEARCH_PATHS $ENV{LD_LIBRARY_PATH} $ENV{CWIPI_HOME}/lib)
FIND_LIBRARY(CWIPI_LIBRARY NAMES cwipi fvmc bftc PATHS ${CWIPI_SEARCH_PATHS})
IF (CWIPI_LIBRARY)
GET_FILENAME_COMPONENT(CWIPI_PATH ${CWIPI_LIBRARY} PATH)
SET(CWIPI_INCLUDE_DIR ${CWIPI_PATH}/../include CACHE FILEPATH "CWIPI include directory.")
SET(BUILD_CWIPI OFF)
ELSE()
SET(BUILD_CWIPI ON)
ENDIF ()
CMAKE_DEPENDENT_OPTION(THIRDPARTY_BUILD_CWIPI
"Build CWIPI from ThirdParty" ${BUILD_CWIPI}
"NEKTAR_USE_CWIPI" OFF)
IF (THIRDPARTY_BUILD_CWIPI)
INCLUDE(ExternalProject)
IF(NOT CMAKE_Fortran_COMPILER)
MESSAGE(ERROR "MPI_Fortran_COMPILER not set")
ENDIF()
IF(NOT NEKTAR_USE_MPI)
MESSAGE(ERROR "NEKTAR_USE_MPI not set")
ENDIF()
EXTERNALPROJECT_ADD(
cwipi-0.8.2
URL ${TPURL}/cwipi-0.8.2.tgz
URL_MD5 "cd28dbea20a08d71f5ff4b4770867268"
STAMP_DIR ${TPBUILD}/stamp
DOWNLOAD_DIR ${TPSRC}
SOURCE_DIR ${TPSRC}/cwipi-0.8.2
PATCH_COMMAND patch -p 0 < ${PROJECT_SOURCE_DIR}/cmake/thirdparty-patches/cwipi_fix-compile.patch
BINARY_DIR ${TPBUILD}/cwipi-0.8.2
TMP_DIR ${TPBUILD}/cwipi-0.8.2-tmp
INSTALL_DIR ${TPDIST}
CONFIGURE_COMMAND
OMPI_FC=${CMAKE_Fortran_COMPILER}
OMPI_CC=${CMAKE_C_COMPILER}
OMPI_CXX=${CMAKE_CXX_COMPILER}
CFLAGS=-std=c99
CXXFLAGS=-std=c++11
${TPSRC}/cwipi-0.8.2/configure
CC=${MPI_C_COMPILER}
CXX=${MPI_CXX_COMPILER}
FC=${MPI_Fortran_COMPILER}
--prefix=${TPDIST}
--libdir=${TPDIST}/lib
--quiet
BUILD_COMMAND make -j 1
)
THIRDPARTY_LIBRARY(CWIPI_LIBRARY SHARED cwipi
DESCRIPTION "CWIPI main library")
THIRDPARTY_LIBRARY(CWIPI_LIBRARY_FVMC SHARED fvmc
DESCRIPTION "CWIPI fvmc library")
THIRDPARTY_LIBRARY(CWIPI_LIBRARY_BFTC SHARED bftc
DESCRIPTION "CWIPI bftc library")
SET(CWIPI_INCLUDE_DIR ${TPDIST}/include CACHE FILEPATH
"CWIPI include" FORCE)
LINK_DIRECTORIES(${TPDIST}/lib)
MESSAGE(STATUS "Build CWIPI:
${CWIPI_LIBRARY}
${CWIPI_LIBRARY_FVMC}
${CWIPI_LIBRARY_BFTC}")
SET(CWIPI_CONFIG_INCLUDE_DIR ${TPINC})
ELSE ()
ADD_CUSTOM_TARGET(cwipi-0.8.2 ALL)
MESSAGE(STATUS "Found CWIPI:
${CWIPI_LIBRARY}
${CWIPI_LIBRARY_FVMC}
${CWIPI_LIBRARY_BFTC}")
SET(CWIPI_CONFIG_INCLUDE_DIR ${CWIPI_INCLUDE_DIR})
ENDIF()
ENDIF( NEKTAR_USE_CWIPI )
INCLUDE_DIRECTORIES(SYSTEM ${CWIPI_INCLUDE_DIR})
MARK_AS_ADVANCED(CWIPI_LIBRARY)
MARK_AS_ADVANCED(CWIPI_LIBRARY_FVMC)
MARK_AS_ADVANCED(CWIPI_LIBRARY_BFTC)
MARK_AS_ADVANCED(CWIPI_INCLUDE_DIR)
......@@ -39,7 +39,14 @@ IF (NEKTAR_USE_FFTW)
BINARY_DIR ${TPBUILD}/fftw-3.2.2
TMP_DIR ${TPBUILD}/fftw-3.2.2-tmp
INSTALL_DIR ${TPDIST}
CONFIGURE_COMMAND CC=${CMAKE_C_COMPILER} ${TPSRC}/fftw-3.2.2/configure --prefix=${TPDIST} --quiet --enable-shared --disable-dependency-tracking
CONFIGURE_COMMAND
CC=${CMAKE_C_COMPILER}
${TPSRC}/fftw-3.2.2/configure
--prefix=${TPDIST}
--libdir=${TPDIST}/lib
--quiet
--enable-shared
--disable-dependency-tracking
)
SET(FFTW_LIBRARY fftw3 CACHE FILEPATH
......
diff -crB cwipi-0.8.2/src/coupling.cxx cwipi-0.8.2.patched/src/coupling.cxx
*** src/coupling.cxx 2014-12-10 16:16:36.000000000 +0100
--- src/coupling.cxx 2017-06-15 12:42:35.919177812 +0200
***************
*** 1221,1227 ****
#ifdef NAN
if (receivingField != NULL && nInteriorList > 0) {
const int idx = 0;
! if (isnan(receivingField[idx]))
status = CWIPI_EXCHANGE_BAD_RECEIVING;
}
#endif
--- 1221,1227 ----
#ifdef NAN
if (receivingField != NULL && nInteriorList > 0) {
const int idx = 0;
! if (std::isnan(receivingField[idx]))
status = CWIPI_EXCHANGE_BAD_RECEIVING;
}
#endif
Subproject commit b1461b45abb08c48397fe6d046249703c4f8f160
Subproject commit 3616b5f04d9ee4f9d70d90553f3a8949654c1b2c
......@@ -505,4 +505,12 @@ year = {2016}
pages={3128--3144},
year={2006},
publisher={Elsevier}
}
\ No newline at end of file
}
@inproceedings{LaSaJa17,
author = {Lackhove, Kilian and Sadiki, Amsini and Janicka, Johannes},
booktitle = {Proceedings of ASME Turbo Expo 2007},
keywords = {GT2017-63050},
title = {{Efficient Three Dimensional Time-Domain Combustion Noise Simulation of a Premixed Flame Using Acoustic Perturbation Equations and Incompressible LES}},
year = {2017}
}
\chapter{Dummy Solver}
\section{Synopsis}
The Dummy solver does not solve any equation systems but only serves to exchange fields with other solvers and applications.
It is intended for demonstrating and testing the coupling implementations only.
......@@ -5,6 +5,7 @@
\input{ape}
\input{cardiac-ep}
\input{compressible-flow}
\input{dummy}
\input{incompressible-ns}
\input{linear-elastic}
\input{pulse-wave}
......
\section{Coupling}
Nektar++ Solvers can be run in parallel with third party applications and other Nektar++ solvers, where run-time data exchange is enabled by the coupling interface.
The interface is configured in the \inltt{COUPLING} tag as
\begin{lstlisting}[style=XMLStyle]
<COUPLING TYPE="[type]" NAME="[name]">
<I PROPERTY="SendSteps" VALUE="1" />
<I PROPERTY="SendVariables" VALUE="u0S,v0S" />
<I PROPERTY="ReceiveSteps" VALUE="1" />
<I PROPERTY="ReceiveVariables" VALUE="u0R,v0R" />
...
</COUPLING>
\end{lstlisting}
The coupling type can be any of the following:
\begin{itemize}
\item "File"
\item "Cwipi"
\end{itemize}
while the name can be chosen arbitrarily.
Inside each coupling block, the send and receive frequencies are defined by the \inltt{SendSteps} and \inltt{ReceiveSteps} parameters, respectively.
Which variables are to be sent or received is specified by the \inltt{SendVariables} and \inltt{ReceiveVariables}.
By default, the send and receive frequencies is set to zero, which disables the corresponding exchange in this coupling.
An empty \inltt{SendVariables} or \inltt{ReceiveVariables} list has the same effect.
\begin{center}
\begin{tabularx}{0.99\textwidth}{lllX}
\toprule
\textbf{Option name} & \textbf{Required} & \textbf{Default} &
\textbf{Description} \\
\midrule
\inltt{SendSteps} & \xmark & \texttt{0} &
Frequency (in steps) at which fields are sent. Sending is disabled if set to zero.\\
\inltt{SendVariables} & \xmark & \texttt{<empty>} &
Comma-separated list of sent variables. Sending is disabled if the list is empty.\\
\inltt{ReceiveSteps} & \xmark & \texttt{0} &
Frequency (in steps) at which fields are received. Receiving is disabled if set to zero.\\
\inltt{ReceiveVariables} & \xmark & \texttt{<empty>} &
Comma-separated list of received variables. Receiving is disabled if the list is empty.\\
\bottomrule
\end{tabularx}
\end{center}
\subsection{File}
This coupling type allows the user to exchange fields at run time by reading from and writing to files.
Besides the basic parameters which define the exchanged variables and the exchange frequency, the file coupling type requires the \inltt{SendFileName} and \inltt{ReceiveFunction} parameters to be set.
The Coupling name is not used for this type and can be ignored.
\begin{lstlisting}[style=XMLStyle]
<COUPLING NAME="coupling1" TYPE="File">
<I PROPERTY="SendSteps" VALUE="1" />
<I PROPERTY="SendVariables" VALUE="u0S,v0S" />
<I PROPERTY="SendFileName" VALUE="Dummy0out_%14.8E.pts" />
<I PROPERTY="ReceiveSteps" VALUE="1" />
<I PROPERTY="ReceiveVariables" VALUE="u0R,v0R" />
<I PROPERTY="ReceiveFunction" VALUE="CouplingIn" />
</COUPLING>
\end{lstlisting}
\inltt{SendFileName} specifies a file name template to write the field data to.
Currently, only \inltt{.pts} files are supported and the file is only created once fully written, avoiding race conditions between sender and receiver.
Receiving is implemented by evaluating a session function specified in the \inltt{ReceiveFunction} parameter.
The coupling waits for the file given in the receive function to appear.
\begin{center}
\begin{tabularx}{0.99\textwidth}{lllX}
\toprule
\textbf{Option name} & \textbf{Required} & \textbf{Default} &
\textbf{Description} \\
\midrule
\inltt{SendFileName} & (\cmark) & - &
File name where the sent fields should be written to. Required if sending is enabled. Time dependent file names are supported.\\
\inltt{ReceiveFunction} & (\cmark) & - &
Function to evaluate to obtain the received fields.Required if receiving is enabled.\\
\bottomrule
\end{tabularx}
\end{center}
\subsection{Cwipi}
\begin{notebox}
The Cwipi coupling is only available when Nektar++ is compiled with OpenMPI and CWIPI
\end{notebox}
The Cwipi coupling uses CWIPI\footnote{http://sites.onera.fr/cwipi/} to facilitate real time data exchange over MPI.
See \cite{LaSaJa17} for details.
All data transfers are non-blocking to minimize the computational overhead.
The interface must be enabled with the command line option \inltt{--cwipi} and a unique application name, e.g:\begin{lstlisting}[style=BashInputStyle]
DummySolver --cwipi 'Dummy1' Dummy_3DCubeCwipi_1.xml
\end{lstlisting}
CWIPI uses the names of the current application and the coupling to identify two peers in cosimulation setups.
The name of the remote application must be provided by the \inltt{RemoteName} parameter.
Unlike the File-type coupling, a linear interpolation in time is applied to the received fields if non-unity values are set for \inltt{ReceiveSteps}.
\begin{lstlisting}[style=XMLStyle]
<COUPLING NAME="coupling1" TYPE="Cwipi">
<I PROPERTY="RemoteName" VALUE="Dummy1" />
<I PROPERTY="SendSteps" VALUE="1" />
<I PROPERTY="SendVariables" VALUE="u0S,v0S" />
<I PROPERTY="SendMethod" VALUE="NearestNeighbour" />
<I PROPERTY="ReceiveSteps" VALUE="1" />
<I PROPERTY="ReceiveVariables" VALUE="u0R,v0R" />
<I PROPERTY="Oversample" VALUE="5" />
<I PROPERTY="FilterWidth" VALUE="10E-3" />
<I PROPERTY="NotLocMethod" VALUE="Extrapolate" />
</COUPLING>
\end{lstlisting}
Additional options which define the coupling include \inltt{SendMethod}, the method used to retrieve the physical values at the locations requested by the remote application.
Available options are \inltt{NearestNeighbour}, \inltt{Shepard} and \inltt{Evaluate}.
The nearest neighbor interpolation gives the best performance at the cost of accuracy.
The last option directly evaluates the expansions using a backward transform, giving superior accuracy while being the most computationally expensive.
The Shepard interpolation can be a good compromise between computational efficiency and accuracy.
When using non-conforming domains, the current application might request values outside of the computational domain of the remote application.
How to handle these not-located points is specified by the \inltt{NotLocMethod} parameter.
When set to \inltt{keep}, the point value is not altered.
With \inltt{Extrapolate}, the nearest neighbor value of the current application is used.
Note that this can be very inefficient when using many MPI ranks.
\begin{center}
\begin{tabularx}{0.99\textwidth}{lllX}
\toprule
\textbf{Option name} & \textbf{Required} & \textbf{Default} &
\textbf{Description} \\
\midrule
\inltt{RemoteName} & \cmark & - &
Name of the remote application.\\
\inltt{SendMethod} & \xmark & \inltt{NearestNeighbour} &
Specifies how to evaluate fields before sending. Available options are \inltt{NearestNeighbour}, \inltt{Shepard} and \inltt{Evaluate}.\\
\inltt{Oversample} & \xmark & 0 &
Receive fields at a higher (or lower) number of quadrature points before filtering to avoid aliasing.\\
\inltt{FilterWidth} & \xmark & 0 &
Apply a spatial filter of a given filter width to the received fields. Disabled when set to zero.\\
\inltt{NotLocMethod} & \xmark & \inltt{keep} &
Specifies how not located points in non-conformal domains are handled. Possible values are \inltt{keep} and \inltt{Extrapolate}.\\
\bottomrule
\end{tabularx}
\end{center}
......@@ -74,4 +74,6 @@ files as illustrated below:
\input{xml/xml-forcing.tex}
\input{xml/xml-coupling.tex}
\input{xml/xml-analytic-expressions.tex}
......@@ -93,7 +93,7 @@ namespace ErrorUtil
// Default rank is zero. If MPI used and initialised, populate with
// the correct rank. Messages are only printed on rank zero.
int rank = 0;
#if defined(NEKTAR_USE_MPI)
#if defined(NEKTAR_USE_MPI) && !defined(NEKTAR_USE_CWIPI)
int flag = 0;
if(DoComm)
{
......@@ -140,7 +140,8 @@ namespace ErrorUtil
<< std::endl;
}
}
#if defined(NEKTAR_USE_MPI)
#if defined(NEKTAR_USE_MPI) && !defined(NEKTAR_USE_CWIPI)
if(DoComm)
{
if (flag)
......
......@@ -663,10 +663,16 @@ DataSourceSharedPtr FieldIOXml::v_ImportFieldMetaData(
if (paramString != "Provenance")
{
// Now read body of param
TiXmlNode *paramBody = param->FirstChild();
std::string paramBodyStr = paramBody->ToText()->Value();
fieldmetadatamap[paramString] = paramBodyStr;
if (param->NoChildren())
{
fieldmetadatamap[paramString] = "";
}
else
{
TiXmlNode *paramBody = param->FirstChild();
std::string paramBodyStr = paramBody->ToText()->Value();
fieldmetadatamap[paramString] = paramBodyStr;
}
}
param = param->NextSiblingElement();
}
......
......@@ -396,6 +396,10 @@ namespace Nektar
("part-only-overlapping", po::value<int>(),
"only partition mesh into N overlapping partitions.")
("part-info", "Output partition information")
#ifdef NEKTAR_USE_CWIPI
("cwipi", po::value<std::string>(),
"set CWIPI name")
#endif
;
for (auto &cmdIt : GetCmdLineArgMap())
......@@ -1590,8 +1594,12 @@ namespace Nektar
{
vCommModule = "ParallelMPI";
}
if (m_cmdLineOptions.count("cwipi") && GetCommFactory().ModuleExists("CWIPI"))
{
vCommModule = "CWIPI";
}
m_comm = GetCommFactory().CreateInstance(vCommModule,argc,argv);
m_comm = GetCommFactory().CreateInstance(vCommModule, argc, argv);
}
}
......
......@@ -88,6 +88,13 @@ IF(NEKTAR_USE_MPI)
./Communication/CommMpi.cpp)
ENDIF(NEKTAR_USE_MPI)
IF ( NEKTAR_USE_CWIPI )
SET(CommunicationHeaders ${CommunicationHeaders}
./Communication/CommCwipi.h)
SET(CommunicationSources ${CommunicationSources}
./Communication/CommCwipi.cpp)
ENDIF ()
SET(FFTHeaders
./FFT/NektarFFT.h
)
......@@ -342,6 +349,12 @@ ADD_DEPENDENCIES(LibUtilities boost tinyxml-2.6.2 zlib-1.2.7)
### Libraries needed specifically by LibUtilities ###
# CWIPI
IF ( NEKTAR_USE_CWIPI )
TARGET_LINK_LIBRARIES(LibUtilities LINK_PRIVATE ${CWIPI_LIBRARY} ${CWIPI_LIBRARY_FVMC} ${CWIPI_LIBRARY_BFTC})
ADD_DEPENDENCIES(LibUtilities cwipi-0.8.2)
ENDIF ()
# FFTW
IF( NEKTAR_USE_FFTW )
TARGET_LINK_LIBRARIES(LibUtilities LINK_PUBLIC ${FFTW_LIBRARY})
......
......@@ -108,6 +108,17 @@ public:
Array<OneD, int> &pRecvDataSizeMap,
Array<OneD, int> &pRecvDataOffsetMap);
template <class T> void AllGather(T &pSendData, T &pRecvData);
template <class T>
void AllGatherv(T &pSendData,
T &pRecvData,
Array<OneD, int> &pRecvDataSizeMap,
Array<OneD, int> &pRecvDataOffsetMap);
template <class T>
void AllGatherv(T &pRecvData,
Array<OneD, int> &pRecvDataSizeMap,
Array<OneD, int> &pRecvDataOffsetMap);
template <class T> void Bcast(T &data, int rootProc);
template <class T>
......@@ -153,6 +164,14 @@ protected:
CommDataType sendtype, void *recvbuf,
int recvcounts[], int rdispls[],
CommDataType recvtype) = 0;
virtual void v_AllGather(void *sendbuf, int sendcount, CommDataType sendtype,
void *recvbuf, int recvcount,
CommDataType recvtype) = 0;
virtual void v_AllGatherv(void *sendbuf, int sendcount, CommDataType sendtype,
void *recvbuf, int recvcounts[], int rdispls[],
CommDataType recvtype) = 0;
virtual void v_AllGatherv(void *recvbuf, int recvcounts[], int rdispls[],
CommDataType recvtype) = 0;
virtual void v_Bcast(void *buffer, int count, CommDataType dt,
int root) = 0;
......@@ -308,6 +327,64 @@ void Comm::AlltoAllv(Array<OneD, T> &pSendData,
pRecvDataOffsetMap.get(), CommDataTypeTraits<T>::GetDataType());
}
template <class T> void Comm::AllGather(T &pSendData, T &pRecvData)
{
BOOST_STATIC_ASSERT_MSG(
CommDataTypeTraits<T>::IsVector,
"AllGather only valid with Array or vector arguments.");
int sendSize = CommDataTypeTraits<T>::GetCount(pSendData);
int recvSize = sendSize;
pRecvData = T(recvSize * GetSize());
v_AllGather(CommDataTypeTraits<T>::GetPointer(pSendData), sendSize,
CommDataTypeTraits<T>::GetDataType(),
CommDataTypeTraits<T>::GetPointer(pRecvData), recvSize,
CommDataTypeTraits<T>::GetDataType());
}
/**
*
*/
template <class T>
void Comm::AllGatherv(T &pSendData,
T &pRecvData,
Array<OneD, int> &pRecvDataSizeMap,
Array<OneD, int> &pRecvDataOffsetMap)
{
BOOST_STATIC_ASSERT_MSG(
CommDataTypeTraits<T>::IsVector,
"AllGatherv only valid with Array or vector arguments.");
int sendSize = CommDataTypeTraits<T>::GetCount(pSendData);
v_AllGatherv(CommDataTypeTraits<T>::GetPointer(pSendData), sendSize,
CommDataTypeTraits<T>::GetDataType(),
CommDataTypeTraits<T>::GetPointer(pRecvData),
pRecvDataSizeMap.get(),
pRecvDataOffsetMap.get(),
CommDataTypeTraits<T>::GetDataType());
}
/**
*
*/
template <class T>
void Comm::AllGatherv(T &pRecvData,
Array<OneD, int> &pRecvDataSizeMap,
Array<OneD, int> &pRecvDataOffsetMap)
{
BOOST_STATIC_ASSERT_MSG(
CommDataTypeTraits<T>::IsVector,
"AllGatherv only valid with Array or vector arguments.");
v_AllGatherv(CommDataTypeTraits<T>::GetPointer(pRecvData),
pRecvDataSizeMap.get(),
pRecvDataOffsetMap.get(),
CommDataTypeTraits<T>::GetDataType());
}
/**
*
*/
......
///////////////////////////////////////////////////////////////////////////////
//
// File FilterEnergy.h
// File CommCwipi.cpp
//
// For more information, please see: http://www.nektar.info
//
// The MIT License
//
// Copyright (c) 2018 Kilian Lackhove
// Copyright (c) 2006 Division of Applied Mathematics, Brown University (USA),
// Department of Aeronautics, Imperial College London (UK), and Scientific
// Computing and Imaging Institute, University of Utah (USA).
......@@ -29,49 +30,71 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
//
// Description: Outputs solution fields during time-stepping.
// Description: MPI communication implementation
//
///////////////////////////////////////////////////////////////////////////////
#ifndef NEKTAR_COMPRESSIBLEFLOWSOLVER_FILTERS_FILTERENERGY_H
#define NEKTAR_COMPRESSIBLEFLOWSOLVER_FILTERS_FILTERENERGY_H
#include <LibUtilities/BasicUtils/SharedArray.hpp>
#include <LibUtilities/Communication/CommCwipi.h>
#include <SolverUtils/Filters/FilterEnergyBase.h>
#include <cwipi.h>
namespace Nektar
{
namespace LibUtilities
{
std::string CommCwipi::className = GetCommFactory().RegisterCreatorFunction(
"CWIPI", CommCwipi::create, "Parallel communication using MPI with CWIPI.");
class FilterEnergy : public SolverUtils::FilterEnergyBase
/**
*
*/
CommCwipi::CommCwipi(int narg, char *arg[]) : CommMpi()
{
public:
friend class MemoryManager<FilterEnergy>;
int init = 0;
MPI_Initialized(&init);
ASSERTL0(!init, "MPI has already been initialised.");
/// Creates an instance of this class
static SolverUtils::FilterSharedPtr create(
const LibUtilities::SessionReaderSharedPtr &pSession,
const ParamMap &pParams) {
SolverUtils::FilterSharedPtr p = MemoryManager<FilterEnergy>
::AllocateSharedPtr(pSession, pParams);
return p;
int retval = MPI_Init(&narg, &arg);
if (retval != MPI_SUCCESS)
{
ASSERTL0(false, "Failed to initialise MPI");
}
///Name of the class
static std::string className;
std::string localName = "";
for (int i = 0; i < narg; ++i)
{
if (!std::strcmp(arg[i], "--cwipi"))
{
localName = arg[i + 1];
}
}
SOLVER_UTILS_EXPORT FilterEnergy(
const LibUtilities::SessionReaderSharedPtr &pSession,
const ParamMap &pParams);
SOLVER_UTILS_EXPORT ~FilterEnergy();
MPI_Comm localComm;
cwipi_init(MPI_COMM_WORLD, localName.c_str(), &localComm);
m_comm = localComm;
protected:
virtual void v_GetVelocity(
const Array<OneD, const MultiRegions::ExpListSharedPtr> &pFields,
const int i,
Array<OneD, NekDouble> &velocity);
virtual Array<OneD, NekDouble> v_GetDensity(
const Array<OneD, const MultiRegions::ExpListSharedPtr> &pFields);
};
MPI_Comm_size(m_comm, &m_size);
MPI_Comm_rank(m_comm, &m_rank);
m_type = "Parallel MPI with CWIPI";
}