Commit 4dcc5ec3 by Chris Cantwell

Merge branch 'feature/Coupling' into 'master'

Coupling Interface for Cosimulations

See merge request !853
parents 50675fb4 288f9cd6
......@@ -246,6 +246,7 @@ INCLUDE (ThirdPartyOCE)
INCLUDE (ThirdPartyTriangle)
INCLUDE (ThirdPartyTetGen)
INCLUDE (ThirdPartyCCM)
INCLUDE (ThirdPartyCwipi)
INCLUDE (FindCFI)
INCLUDE (Doxygen)
......
# - Try to find cwipi
# Once done this will define
# CWIPI_FOUND - System has cwipi
# CWIPI_INCLUDE_DIRS - The cwipi include directories
# CWIPI_LIBRARIES - The libraries needed to use cwipi
# CWIPI_DEFINITIONS - Compiler switches required for using cwipi
SET(CWIPI_SEARCH_PATHS
${CMAKE_SOURCE_DIR}/ThirdParty/cwipi/build/lib
${CMAKE_SOURCE_DIR}/../ThirdParty/cwipi/build/lib
${CMAKE_SOURCE_DIR}/ThirdParty/dist/build/lib
${CMAKE_SOURCE_DIR}/../ThirdParty/dist/build/lib)
FIND_PATH(CWIPI_INCLUDE_DIR NAMES cwipi.h PATHS ${CWIPI_SEARCH_PATHS})
FIND_LIBRARY(CWIPI_LIBRARY NAMES cwipi PATHS ${CWIPI_SEARCH_PATHS})
FIND_LIBRARY(CWIPI_LIBRARY_FVMC NAMES fvmc PATHS ${CWIPI_SEARCH_PATHS})
FIND_LIBRARY(CWIPI_LIBRARY_BFTC NAMES BFTC PATHS ${CWIPI_SEARCH_PATHS})
SET(CWIPI_FOUND FALSE)
IF (CWIPI_LIBRARY)
SET(CWIPI_FOUND TRUE)
INCLUDE_DIRECTORIES(${CWIPI_INCLUDE_DIR})
MARK_AS_ADVANCED(CWIPI_LIBRARY)
MARK_AS_ADVANCED(CWIPI_LIBRARY_FVMC)
MARK_AS_ADVANCED(CWIPI_LIBRARY_BFTC)
MARK_AS_ADVANCED(CWIPI_INCLUDE_DIR)
ENDIF (CWIPI_LIBRARY)
IF (CWIPI_FOUND)
IF (NOT CWIPI_FIND_QUIETLY)
MESSAGE(STATUS "Found CWIPI: ${CWIPI_INCLUDE_DIR}")
ENDIF (NOT CWIPI_FIND_QUIETLY)
ELSE(CWIPI_FOUND)
IF (CWIPI_FIND_REQUIRED)
MESSAGE(FATAL_ERROR "Could not find CWIPI")
ENDIF (CWIPI_FIND_REQUIRED)
ENDIF (CWIPI_FOUND)
########################################################################
#
# ThirdParty configuration for Nektar++
#
# CWIPI
#
########################################################################
OPTION(NEKTAR_USE_CWIPI
"Use CWIPI for Coupling." OFF)
IF ( NEKTAR_USE_CWIPI )
find_package(Cwipi)
include_directories(${CWIPI_INCLUDE_DIRS})
ADD_DEFINITIONS(-DNEKTAR_USE_CWIPI)
# Set some common CWIPI search paths.
SET(CWIPI_SEARCH_PATHS $ENV{LD_LIBRARY_PATH} $ENV{CWIPI_HOME}/lib)
FIND_LIBRARY(CWIPI_LIBRARY NAMES cwipi fvmc bftc PATHS ${CWIPI_SEARCH_PATHS})
IF (CWIPI_LIBRARY)
GET_FILENAME_COMPONENT(CWIPI_PATH ${CWIPI_LIBRARY} PATH)
SET(CWIPI_INCLUDE_DIR ${CWIPI_PATH}/../include CACHE FILEPATH "CWIPI include directory.")
SET(BUILD_CWIPI OFF)
ELSE()
SET(BUILD_CWIPI ON)
ENDIF ()
CMAKE_DEPENDENT_OPTION(THIRDPARTY_BUILD_CWIPI
"Build CWIPI from ThirdParty" ${BUILD_CWIPI}
"NEKTAR_USE_CWIPI" OFF)
IF (THIRDPARTY_BUILD_CWIPI)
INCLUDE(ExternalProject)
IF(NOT CMAKE_Fortran_COMPILER)
MESSAGE(ERROR "MPI_Fortran_COMPILER not set")
ENDIF()
IF(NOT NEKTAR_USE_MPI)
MESSAGE(ERROR "NEKTAR_USE_MPI not set")
ENDIF()
EXTERNALPROJECT_ADD(
cwipi-0.8.2
URL ${TPURL}/cwipi-0.8.2.tgz
URL_MD5 "cd28dbea20a08d71f5ff4b4770867268"
STAMP_DIR ${TPBUILD}/stamp
DOWNLOAD_DIR ${TPSRC}
SOURCE_DIR ${TPSRC}/cwipi-0.8.2
PATCH_COMMAND patch -p 0 < ${PROJECT_SOURCE_DIR}/cmake/thirdparty-patches/cwipi_fix-compile.patch
BINARY_DIR ${TPBUILD}/cwipi-0.8.2
TMP_DIR ${TPBUILD}/cwipi-0.8.2-tmp
INSTALL_DIR ${TPDIST}
CONFIGURE_COMMAND
OMPI_FC=${CMAKE_Fortran_COMPILER}
OMPI_CC=${CMAKE_C_COMPILER}
OMPI_CXX=${CMAKE_CXX_COMPILER}
CFLAGS=-std=c99
CXXFLAGS=-std=c++11
${TPSRC}/cwipi-0.8.2/configure
CC=${MPI_C_COMPILER}
CXX=${MPI_CXX_COMPILER}
FC=${MPI_Fortran_COMPILER}
--prefix=${TPDIST}
--libdir=${TPDIST}/lib
--quiet
BUILD_COMMAND make -j 1
)
THIRDPARTY_LIBRARY(CWIPI_LIBRARY SHARED cwipi
DESCRIPTION "CWIPI main library")
THIRDPARTY_LIBRARY(CWIPI_LIBRARY_FVMC SHARED fvmc
DESCRIPTION "CWIPI fvmc library")
THIRDPARTY_LIBRARY(CWIPI_LIBRARY_BFTC SHARED bftc
DESCRIPTION "CWIPI bftc library")
SET(CWIPI_INCLUDE_DIR ${TPDIST}/include CACHE FILEPATH
"CWIPI include" FORCE)
LINK_DIRECTORIES(${TPDIST}/lib)
MESSAGE(STATUS "Build CWIPI:
${CWIPI_LIBRARY}
${CWIPI_LIBRARY_FVMC}
${CWIPI_LIBRARY_BFTC}")
SET(CWIPI_CONFIG_INCLUDE_DIR ${TPINC})
ELSE ()
ADD_CUSTOM_TARGET(cwipi-0.8.2 ALL)
MESSAGE(STATUS "Found CWIPI:
${CWIPI_LIBRARY}
${CWIPI_LIBRARY_FVMC}
${CWIPI_LIBRARY_BFTC}")
SET(CWIPI_CONFIG_INCLUDE_DIR ${CWIPI_INCLUDE_DIR})
ENDIF()
ENDIF( NEKTAR_USE_CWIPI )
INCLUDE_DIRECTORIES(SYSTEM ${CWIPI_INCLUDE_DIR})
MARK_AS_ADVANCED(CWIPI_LIBRARY)
MARK_AS_ADVANCED(CWIPI_LIBRARY_FVMC)
MARK_AS_ADVANCED(CWIPI_LIBRARY_BFTC)
MARK_AS_ADVANCED(CWIPI_INCLUDE_DIR)
......@@ -39,7 +39,14 @@ IF (NEKTAR_USE_FFTW)
BINARY_DIR ${TPBUILD}/fftw-3.2.2
TMP_DIR ${TPBUILD}/fftw-3.2.2-tmp
INSTALL_DIR ${TPDIST}
CONFIGURE_COMMAND CC=${CMAKE_C_COMPILER} ${TPSRC}/fftw-3.2.2/configure --prefix=${TPDIST} --quiet --enable-shared --disable-dependency-tracking
CONFIGURE_COMMAND
CC=${CMAKE_C_COMPILER}
${TPSRC}/fftw-3.2.2/configure
--prefix=${TPDIST}
--libdir=${TPDIST}/lib
--quiet
--enable-shared
--disable-dependency-tracking
)
SET(FFTW_LIBRARY fftw3 CACHE FILEPATH
......
diff -crB cwipi-0.8.2/src/coupling.cxx cwipi-0.8.2.patched/src/coupling.cxx
*** src/coupling.cxx 2014-12-10 16:16:36.000000000 +0100
--- src/coupling.cxx 2017-06-15 12:42:35.919177812 +0200
***************
*** 1221,1227 ****
#ifdef NAN
if (receivingField != NULL && nInteriorList > 0) {
const int idx = 0;
! if (isnan(receivingField[idx]))
status = CWIPI_EXCHANGE_BAD_RECEIVING;
}
#endif
--- 1221,1227 ----
#ifdef NAN
if (receivingField != NULL && nInteriorList > 0) {
const int idx = 0;
! if (std::isnan(receivingField[idx]))
status = CWIPI_EXCHANGE_BAD_RECEIVING;
}
#endif
......@@ -505,4 +505,12 @@ year = {2016}
pages={3128--3144},
year={2006},
publisher={Elsevier}
}
\ No newline at end of file
}
@inproceedings{LaSaJa17,
author = {Lackhove, Kilian and Sadiki, Amsini and Janicka, Johannes},
booktitle = {Proceedings of ASME Turbo Expo 2007},
keywords = {GT2017-63050},
title = {{Efficient Three Dimensional Time-Domain Combustion Noise Simulation of a Premixed Flame Using Acoustic Perturbation Equations and Incompressible LES}},
year = {2017}
}
\chapter{Dummy Solver}
\section{Synopsis}
The Dummy solver does not solve any equation systems but only serves to exchange fields with other solvers and applications.
It is intended for demonstrating and testing the coupling implementations only.
......@@ -5,6 +5,7 @@
\input{ape}
\input{cardiac-ep}
\input{compressible-flow}
\input{dummy}
\input{incompressible-ns}
\input{linear-elastic}
\input{pulse-wave}
......
\section{Coupling}
Nektar++ Solvers can be run in parallel with third party applications and other Nektar++ solvers, where run-time data exchange is enabled by the coupling interface.
The interface is configured in the \inltt{COUPLING} tag as
\begin{lstlisting}[style=XMLStyle]
<COUPLING TYPE="[type]" NAME="[name]">
<I PROPERTY="SendSteps" VALUE="1" />
<I PROPERTY="SendVariables" VALUE="u0S,v0S" />
<I PROPERTY="ReceiveSteps" VALUE="1" />
<I PROPERTY="ReceiveVariables" VALUE="u0R,v0R" />
...
</COUPLING>
\end{lstlisting}
The coupling type can be any of the following:
\begin{itemize}
\item "File"
\item "Cwipi"
\end{itemize}
while the name can be chosen arbitrarily.
Inside each coupling block, the send and receive frequencies are defined by the \inltt{SendSteps} and \inltt{ReceiveSteps} parameters, respectively.
Which variables are to be sent or received is specified by the \inltt{SendVariables} and \inltt{ReceiveVariables}.
By default, the send and receive frequencies is set to zero, which disables the corresponding exchange in this coupling.
An empty \inltt{SendVariables} or \inltt{ReceiveVariables} list has the same effect.
\begin{center}
\begin{tabularx}{0.99\textwidth}{lllX}
\toprule
\textbf{Option name} & \textbf{Required} & \textbf{Default} &
\textbf{Description} \\
\midrule
\inltt{SendSteps} & \xmark & \texttt{0} &
Frequency (in steps) at which fields are sent. Sending is disabled if set to zero.\\
\inltt{SendVariables} & \xmark & \texttt{<empty>} &
Comma-separated list of sent variables. Sending is disabled if the list is empty.\\
\inltt{ReceiveSteps} & \xmark & \texttt{0} &
Frequency (in steps) at which fields are received. Receiving is disabled if set to zero.\\
\inltt{ReceiveVariables} & \xmark & \texttt{<empty>} &
Comma-separated list of received variables. Receiving is disabled if the list is empty.\\
\bottomrule
\end{tabularx}
\end{center}
\subsection{File}
This coupling type allows the user to exchange fields at run time by reading from and writing to files.
Besides the basic parameters which define the exchanged variables and the exchange frequency, the file coupling type requires the \inltt{SendFileName} and \inltt{ReceiveFunction} parameters to be set.
The Coupling name is not used for this type and can be ignored.
\begin{lstlisting}[style=XMLStyle]
<COUPLING NAME="coupling1" TYPE="File">
<I PROPERTY="SendSteps" VALUE="1" />
<I PROPERTY="SendVariables" VALUE="u0S,v0S" />
<I PROPERTY="SendFileName" VALUE="Dummy0out_%14.8E.pts" />
<I PROPERTY="ReceiveSteps" VALUE="1" />
<I PROPERTY="ReceiveVariables" VALUE="u0R,v0R" />
<I PROPERTY="ReceiveFunction" VALUE="CouplingIn" />
</COUPLING>
\end{lstlisting}
\inltt{SendFileName} specifies a file name template to write the field data to.
Currently, only \inltt{.pts} files are supported and the file is only created once fully written, avoiding race conditions between sender and receiver.
Receiving is implemented by evaluating a session function specified in the \inltt{ReceiveFunction} parameter.
The coupling waits for the file given in the receive function to appear.
\begin{center}
\begin{tabularx}{0.99\textwidth}{lllX}
\toprule
\textbf{Option name} & \textbf{Required} & \textbf{Default} &
\textbf{Description} \\
\midrule
\inltt{SendFileName} & (\cmark) & - &
File name where the sent fields should be written to. Required if sending is enabled. Time dependent file names are supported.\\
\inltt{ReceiveFunction} & (\cmark) & - &
Function to evaluate to obtain the received fields.Required if receiving is enabled.\\
\bottomrule
\end{tabularx}
\end{center}
\subsection{Cwipi}
\begin{notebox}
The Cwipi coupling is only available when Nektar++ is compiled with OpenMPI and CWIPI
\end{notebox}
The Cwipi coupling uses CWIPI\footnote{http://sites.onera.fr/cwipi/} to facilitate real time data exchange over MPI.
See \cite{LaSaJa17} for details.
All data transfers are non-blocking to minimize the computational overhead.
The interface must be enabled with the command line option \inltt{--cwipi} and a unique application name, e.g:\begin{lstlisting}[style=BashInputStyle]
DummySolver --cwipi 'Dummy1' Dummy_3DCubeCwipi_1.xml
\end{lstlisting}
CWIPI uses the names of the current application and the coupling to identify two peers in cosimulation setups.
The name of the remote application must be provided by the \inltt{RemoteName} parameter.
Unlike the File-type coupling, a linear interpolation in time is applied to the received fields if non-unity values are set for \inltt{ReceiveSteps}.
\begin{lstlisting}[style=XMLStyle]
<COUPLING NAME="coupling1" TYPE="Cwipi">
<I PROPERTY="RemoteName" VALUE="Dummy1" />
<I PROPERTY="SendSteps" VALUE="1" />
<I PROPERTY="SendVariables" VALUE="u0S,v0S" />
<I PROPERTY="SendMethod" VALUE="NearestNeighbour" />
<I PROPERTY="ReceiveSteps" VALUE="1" />
<I PROPERTY="ReceiveVariables" VALUE="u0R,v0R" />
<I PROPERTY="Oversample" VALUE="5" />
<I PROPERTY="FilterWidth" VALUE="10E-3" />
<I PROPERTY="NotLocMethod" VALUE="Extrapolate" />
</COUPLING>
\end{lstlisting}
Additional options which define the coupling include \inltt{SendMethod}, the method used to retrieve the physical values at the locations requested by the remote application.
Available options are \inltt{NearestNeighbour}, \inltt{Shepard} and \inltt{Evaluate}.
The nearest neighbor interpolation gives the best performance at the cost of accuracy.
The last option directly evaluates the expansions using a backward transform, giving superior accuracy while being the most computationally expensive.
The Shepard interpolation can be a good compromise between computational efficiency and accuracy.
When using non-conforming domains, the current application might request values outside of the computational domain of the remote application.
How to handle these not-located points is specified by the \inltt{NotLocMethod} parameter.
When set to \inltt{keep}, the point value is not altered.
With \inltt{Extrapolate}, the nearest neighbor value of the current application is used.
Note that this can be very inefficient when using many MPI ranks.
\begin{center}
\begin{tabularx}{0.99\textwidth}{lllX}
\toprule
\textbf{Option name} & \textbf{Required} & \textbf{Default} &
\textbf{Description} \\
\midrule
\inltt{RemoteName} & \cmark & - &
Name of the remote application.\\
\inltt{SendMethod} & \xmark & \inltt{NearestNeighbour} &
Specifies how to evaluate fields before sending. Available options are \inltt{NearestNeighbour}, \inltt{Shepard} and \inltt{Evaluate}.\\
\inltt{Oversample} & \xmark & 0 &
Receive fields at a higher (or lower) number of quadrature points before filtering to avoid aliasing.\\
\inltt{FilterWidth} & \xmark & 0 &
Apply a spatial filter of a given filter width to the received fields. Disabled when set to zero.\\
\inltt{NotLocMethod} & \xmark & \inltt{keep} &
Specifies how not located points in non-conformal domains are handled. Possible values are \inltt{keep} and \inltt{Extrapolate}.\\
\bottomrule
\end{tabularx}
\end{center}
......@@ -74,4 +74,6 @@ files as illustrated below:
\input{xml/xml-forcing.tex}
\input{xml/xml-coupling.tex}
\input{xml/xml-analytic-expressions.tex}
......@@ -93,7 +93,7 @@ namespace ErrorUtil
// Default rank is zero. If MPI used and initialised, populate with
// the correct rank. Messages are only printed on rank zero.
int rank = 0;
#if defined(NEKTAR_USE_MPI)
#if defined(NEKTAR_USE_MPI) && !defined(NEKTAR_USE_CWIPI)
int flag = 0;
if(DoComm)
{
......@@ -140,7 +140,8 @@ namespace ErrorUtil
<< std::endl;
}
}
#if defined(NEKTAR_USE_MPI)
#if defined(NEKTAR_USE_MPI) && !defined(NEKTAR_USE_CWIPI)
if(DoComm)
{
if (flag)
......
......@@ -663,10 +663,16 @@ DataSourceSharedPtr FieldIOXml::v_ImportFieldMetaData(
if (paramString != "Provenance")
{
// Now read body of param
TiXmlNode *paramBody = param->FirstChild();
std::string paramBodyStr = paramBody->ToText()->Value();
fieldmetadatamap[paramString] = paramBodyStr;
if (param->NoChildren())
{
fieldmetadatamap[paramString] = "";
}
else
{
TiXmlNode *paramBody = param->FirstChild();
std::string paramBodyStr = paramBody->ToText()->Value();
fieldmetadatamap[paramString] = paramBodyStr;
}
}
param = param->NextSiblingElement();
}
......
......@@ -396,6 +396,10 @@ namespace Nektar
("part-only-overlapping", po::value<int>(),
"only partition mesh into N overlapping partitions.")
("part-info", "Output partition information")
#ifdef NEKTAR_USE_CWIPI
("cwipi", po::value<std::string>(),
"set CWIPI name")
#endif
;
for (auto &cmdIt : GetCmdLineArgMap())
......@@ -1590,8 +1594,12 @@ namespace Nektar
{
vCommModule = "ParallelMPI";
}
if (m_cmdLineOptions.count("cwipi") && GetCommFactory().ModuleExists("CWIPI"))
{
vCommModule = "CWIPI";
}
m_comm = GetCommFactory().CreateInstance(vCommModule,argc,argv);
m_comm = GetCommFactory().CreateInstance(vCommModule, argc, argv);
}
}
......
......@@ -88,6 +88,13 @@ IF(NEKTAR_USE_MPI)
./Communication/CommMpi.cpp)
ENDIF(NEKTAR_USE_MPI)
IF ( NEKTAR_USE_CWIPI )
SET(CommunicationHeaders ${CommunicationHeaders}
./Communication/CommCwipi.h)
SET(CommunicationSources ${CommunicationSources}
./Communication/CommCwipi.cpp)
ENDIF ()
SET(FFTHeaders
./FFT/NektarFFT.h
)
......@@ -342,6 +349,12 @@ ADD_DEPENDENCIES(LibUtilities boost tinyxml-2.6.2 zlib-1.2.7)
### Libraries needed specifically by LibUtilities ###
# CWIPI
IF ( NEKTAR_USE_CWIPI )
TARGET_LINK_LIBRARIES(LibUtilities LINK_PRIVATE ${CWIPI_LIBRARY} ${CWIPI_LIBRARY_FVMC} ${CWIPI_LIBRARY_BFTC})
ADD_DEPENDENCIES(LibUtilities cwipi-0.8.2)
ENDIF ()
# FFTW
IF( NEKTAR_USE_FFTW )
TARGET_LINK_LIBRARIES(LibUtilities LINK_PUBLIC ${FFTW_LIBRARY})
......
......@@ -108,6 +108,17 @@ public:
Array<OneD, int> &pRecvDataSizeMap,
Array<OneD, int> &pRecvDataOffsetMap);
template <class T> void AllGather(T &pSendData, T &pRecvData);
template <class T>
void AllGatherv(T &pSendData,
T &pRecvData,
Array<OneD, int> &pRecvDataSizeMap,
Array<OneD, int> &pRecvDataOffsetMap);
template <class T>
void AllGatherv(T &pRecvData,
Array<OneD, int> &pRecvDataSizeMap,
Array<OneD, int> &pRecvDataOffsetMap);
template <class T> void Bcast(T &data, int rootProc);
template <class T>
......@@ -153,6 +164,14 @@ protected:
CommDataType sendtype, void *recvbuf,
int recvcounts[], int rdispls[],
CommDataType recvtype) = 0;
virtual void v_AllGather(void *sendbuf, int sendcount, CommDataType sendtype,
void *recvbuf, int recvcount,
CommDataType recvtype) = 0;
virtual void v_AllGatherv(void *sendbuf, int sendcount, CommDataType sendtype,
void *recvbuf, int recvcounts[], int rdispls[],
CommDataType recvtype) = 0;
virtual void v_AllGatherv(void *recvbuf, int recvcounts[], int rdispls[],
CommDataType recvtype) = 0;
virtual void v_Bcast(void *buffer, int count, CommDataType dt,
int root) = 0;
......@@ -308,6 +327,64 @@ void Comm::AlltoAllv(Array<OneD, T> &pSendData,
pRecvDataOffsetMap.get(), CommDataTypeTraits<T>::GetDataType());
}
template <class T> void Comm::AllGather(T &pSendData, T &pRecvData)
{
BOOST_STATIC_ASSERT_MSG(
CommDataTypeTraits<T>::IsVector,
"AllGather only valid with Array or vector arguments.");
int sendSize = CommDataTypeTraits<T>::GetCount(pSendData);
int recvSize = sendSize;
pRecvData = T(recvSize * GetSize());
v_AllGather(CommDataTypeTraits<T>::GetPointer(pSendData), sendSize,
CommDataTypeTraits<T>::GetDataType(),
CommDataTypeTraits<T>::GetPointer(pRecvData), recvSize,
CommDataTypeTraits<T>::GetDataType());
}
/**
*
*/
template <class T>
void Comm::AllGatherv(T &pSendData,
T &pRecvData,
Array<OneD, int> &pRecvDataSizeMap,
Array<OneD, int> &pRecvDataOffsetMap)
{
BOOST_STATIC_ASSERT_MSG(
CommDataTypeTraits<T>::IsVector,
"AllGatherv only valid with Array or vector arguments.");
int sendSize = CommDataTypeTraits<T>::GetCount(pSendData);
v_AllGatherv(CommDataTypeTraits<T>::GetPointer(pSendData), sendSize,
CommDataTypeTraits<T>::GetDataType(),
CommDataTypeTraits<T>::GetPointer(pRecvData),
pRecvDataSizeMap.get(),
pRecvDataOffsetMap.get(),
CommDataTypeTraits<T>::GetDataType());
}
/**
*
*/
template <class T>
void Comm::AllGatherv(T &pRecvData,
Array<OneD, int> &pRecvDataSizeMap,
Array<OneD, int> &pRecvDataOffsetMap)
{
BOOST_STATIC_ASSERT_MSG(
CommDataTypeTraits<T>::IsVector,
"AllGatherv only valid with Array or vector arguments.");
v_AllGatherv(CommDataTypeTraits<T>::GetPointer(pRecvData),
pRecvDataSizeMap.get(),
pRecvDataOffsetMap.get(),
CommDataTypeTraits<T>::GetDataType());
}
/**
*
*/
......
///////////////////////////////////////////////////////////////////////////////
//
// File CommCwipi.cpp
//
// For more information, please see: http://www.nektar.info
//
// The MIT License
//
// Copyright (c) 2018 Kilian Lackhove
// Copyright (c) 2006 Division of Applied Mathematics, Brown University (USA),
// Department of Aeronautics, Imperial College London (UK), and Scientific
// Computing and Imaging Institute, University of Utah (USA).
//
// License for the specific language governing rights and limitations under
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
//
// Description: MPI communication implementation
//
///////////////////////////////////////////////////////////////////////////////
#include <LibUtilities/BasicUtils/SharedArray.hpp>
#include <LibUtilities/Communication/CommCwipi.h>
#include <cwipi.h>
namespace Nektar
{
namespace LibUtilities
{
std::string CommCwipi::className = GetCommFactory().RegisterCreatorFunction(
"CWIPI", CommCwipi::create, "Parallel communication using MPI with CWIPI.");
/**
*
*/
CommCwipi::CommCwipi(int narg, char *arg[]) : CommMpi()
{
int init = 0;
MPI_Initialized(&init);
ASSERTL0(!init, "MPI has already been initialised.");
int retval = MPI_Init(&narg, &arg);
if (retval != MPI_SUCCESS)
{
ASSERTL0(false, "Failed to initialise MPI");
}
std::string localName = "";
for (int i = 0; i < narg; ++i)
{
if (!std::strcmp(arg[i], "--cwipi"))
{
localName = arg[i + 1];
}
}
MPI_Comm localComm;
cwipi_init(MPI_COMM_WORLD, localName.c_str(), &localComm);
m_comm = localComm;
MPI_Comm_size(m_comm, &m_size);
MPI_Comm_rank(m_comm, &m_rank);
m_type = "Parallel MPI with CWIPI";
}
/**
*
*/
CommCwipi::~CommCwipi()
{
}
/**
*
*/
void CommCwipi::v_Finalise()
{
cwipi_finalize();
CommMpi::v_Finalise();
}
}
}
///////////////////////////////////////////////////////////////////////////////
//
// File CommCwipi.h
//
// For more information, please see: http://www.nektar.info
//
// The MIT License
//
// Copyright (c) 2018 Kilian Lackhove
// Copyright (c) 2006 Division of Applied Mathematics, Brown University (USA),
// Department of Aeronautics, Imperial College London (UK), and Scientific
// Computing and Imaging Institute, University of Utah (USA).
//
// License for the specific language governing rights and limitations under
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
//
// Description: CommCwipi header
//
///////////////////////////////////////////////////////////////////////////////
#ifndef NEKTAR_LIB_UTILITIES_COMMCWIPI_H
#define NEKTAR_LIB_UTILITIES_COMMCWIPI_H
#include <mpi.h>
#include <string>
#include <LibUtilities/Communication/CommMpi.h>
#include <LibUtilities/Memory/NekMemoryManager.hpp>
namespace Nektar
{
namespace LibUtilities
{
// Forward declarations
class CommCwipi