Skip to content

GitLab

Nektar
Nektar
Commit ee24062e authored by Kilian Lackhove's avatar Kilian Lackhove
Browse files
Options

CWIPI: Port to pure CWIPI

parent 60bd978b
Hide whitespace changes
Inline Side-by-side
Showing with 149 additions and 294 deletions
CMakeLists.txt
View file @ ee24062e
......@@ -191,13 +191,15 @@ INCLUDE (ThirdPartyTriangle)
INCLUDE (ThirdPartyTetGen)
INCLUDE (ThirdPartyCCM)
set(OPENPALM_DIR "/home/lackhove/code/OpenPalm/PALM_MP/linux64r4openmpi_debug" CACHE PATH "openPALM base path")
find_package(OpenPALM)
include_directories(${OPENPALM_INCLUDE_DIRS})
set(CWIPI_DIR "/home/lackhove/code/OpenPalm/cwipi-0.8.1/dist" CACHE PATH "CWIPI base path")
find_package(Cwipi)
include_directories(${CWIPI_INCLUDE_DIRS})
OPTION(NEKTAR_USE_CWIPI
"Use CWIPI." OFF)
IF ( NEKTAR_USE_CWIPI )
set(CWIPI_DIR "/usr/local/cwipi" CACHE PATH "CWIPI base path")
find_package(Cwipi)
include_directories(${CWIPI_INCLUDE_DIRS})
ADD_DEFINITIONS(-DNEKTAR_USE_CWIPI)
ENDIF ( NEKTAR_USE_CWIPI )
INCLUDE (Doxygen)
......
cmake/FindCwipi.cmake
View file @ ee24062e
......@@ -12,12 +12,24 @@ find_path(CWIPI_INCLUDE_DIR cwipi.h
)
find_library(CWIPI_LIBRARY
NAMES "libcwipi.a"
NAMES "libcwipi.so"
HINTS "${CWIPI_DIR}/lib64/" "${CWIPI_DIR}/lib/"
)
find_library(CWIPI_LIBRARY_FVMC
NAMES "libfvmc.so"
HINTS "${CWIPI_DIR}/lib64/" "${CWIPI_DIR}/lib/"
)
find_library(CWIPI_LIBRARY_BFTC
NAMES "libbftc.so"
HINTS "${CWIPI_DIR}/lib64/" "${CWIPI_DIR}/lib/"
)
set(CWIPI_LIBRARIES
${CWIPI_LIBRARY}
${CWIPI_LIBRARY_FVMC}
${CWIPI_LIBRARY_BFTC}
)
set(CWIPI_INCLUDE_DIRS ${CWIPI_INCLUDE_DIR})
......@@ -26,6 +38,8 @@ include(FindPackageHandleStandardArgs)
# if all listed variables are TRUE
find_package_handle_standard_args(cwipi DEFAULT_MSG
CWIPI_LIBRARY
CWIPI_LIBRARY_FVMC
CWIPI_LIBRARY_BFTC
CWIPI_INCLUDE_DIR
)
......
cmake/FindOpenPALM.cmake deleted 100644 → 0
View file @ 60bd978b
# - Try to find openPALM
# Once done this will define
# OPENPALM_FOUND - System has openPALM
# OPENPALM_INCLUDE_DIRS - The openPALM include directories
# OPENPALM_LIBRARIES - The libraries needed to use openPALM
# OPENPALM_DEFINITIONS - Compiler switches required for using openPALM
set(OPENPALM_DEFINITIONS "")
find_path(OPENPALM_INCLUDE_DIR palmlib.h
HINTS ${OPENPALM_DIR}/include/
)
find_library(OPENPALM_LIBRARY
NAMES "libpalm.a"
HINTS "${OPENPALM_DIR}/lib64/" "${OPENPALM_DIR}/lib/"
)
set(OPENPALM_LIBRARIES
${OPENPALM_LIBRARY}
)
set(OPENPALM_INCLUDE_DIRS ${OPENPALM_INCLUDE_DIR})
include(FindPackageHandleStandardArgs)
# handle the QUIETLY and REQUIRED arguments and set OPENPALM_FOUND to TRUE
# if all listed variables are TRUE
find_package_handle_standard_args(openPALM DEFAULT_MSG
OPENPALM_LIBRARY
OPENPALM_INCLUDE_DIR
)
mark_as_advanced(OPENPALM_LIBRARY OPENPALM_INCLUDE_DIR)
library/LibUtilities/BasicUtils/SessionReader.cpp
View file @ ee24062e
......@@ -1581,9 +1581,9 @@ namespace Nektar
else
{
string vCommModule("Serial");
if (GetCommFactory().ModuleExists("ParallelMPI"))
if (GetCommFactory().ModuleExists("CWIPI"))
{
vCommModule = "ParallelMPI";
vCommModule = "CWIPI";
}
m_comm = GetCommFactory().CreateInstance(vCommModule,argc,argv);
......
library/LibUtilities/CMakeLists.txt
View file @ ee24062e
......@@ -82,9 +82,14 @@ IF(NEKTAR_USE_MPI)
SET(CommunicationHeaders ${CommunicationHeaders}
./Communication/CommMpi.h)
SET(CommunicationSources ${CommunicationSources}
./Communication/CommMpi.cpp
./Communication/CommOpenPalm.cpp)
./Communication/CommMpi.cpp)
ENDIF(NEKTAR_USE_MPI)
IF ( NEKTAR_USE_CWIPI )
SET(CommunicationHeaders ${CommunicationHeaders}
./Communication/CommCwipi.h)
SET(CommunicationSources ${CommunicationSources}
./Communication/CommCwipi.cpp)
ENDIF ()
SET(FFTHeaders
./FFT/NektarFFT.h
......
library/LibUtilities/Communication/CommOpenPalm.cpp → library/LibUtilities/Communication/CommCwipi.cpp
View file @ ee24062e
///////////////////////////////////////////////////////////////////////////////
//
// File CommOpenPalm.cpp
// File CommCwipi.cpp
//
// For more information, please see: http://www.nektar.info
//
// The MIT License
//
// Copyright (c) 2015 Kilian Lackhove
// Copyright (c) 2006 Division of Applied Mathematics, Brown University (USA),
// Department of Aeronautics, Imperial College London (UK), and Scientific
// Computing and Imaging Institute, University of Utah (USA).
......@@ -33,63 +34,60 @@
//
///////////////////////////////////////////////////////////////////////////////
#include <LibUtilities/Communication/CommOpenPalm.h>
#include <LibUtilities/Communication/CommCwipi.h>
#include <LibUtilities/BasicUtils/SharedArray.hpp>
#include <palmlibc.h>
#include <cwipi.h>
namespace Nektar
{
namespace LibUtilities
{
std::string CommOpenPalm::className
= GetCommFactory().RegisterCreatorFunction("OpenPalm", CommOpenPalm::create,
"Parallel communication using MPI with OpenPalm.");
std::string CommCwipi::className
= GetCommFactory().RegisterCreatorFunction("CWIPI", CommCwipi::create,
"Parallel communication using MPI with CWIPI.");
/**
*
*/
CommOpenPalm::CommOpenPalm(int narg, char* arg[])
CommCwipi::CommCwipi(int narg, char* arg[])
: CommMpi()
{
int init = 0;
MPI_Initialized(&init);
ASSERTL0(!init, "MPI has already been initialised.");
m_comm = PL_COMM_EXEC;
MPI_Comm_size( m_comm, &m_size );
MPI_Comm_rank( m_comm, &m_rank );
int retval = MPI_Init(&narg, &arg);
if (retval != MPI_SUCCESS)
{
ASSERTL0(false, "Failed to initialise MPI");
}
m_type = "Parallel MPI with OpenPalm";
}
MPI_Comm localComm;
cwipi_init(MPI_COMM_WORLD, "nektar", &localComm);
m_comm = localComm;
/**
*
*/
CommOpenPalm::CommOpenPalm(MPI_Comm pComm)
: CommMpi()
{
m_comm = pComm;
MPI_Comm_size( m_comm, &m_size );
MPI_Comm_rank( m_comm, &m_rank );
m_type = "Parallel MPI with OpenPalm";
m_type = "Parallel MPI with CWIPI";
}
/**
*
*/
CommOpenPalm::~CommOpenPalm()
CommCwipi::~CommCwipi()
{
}
/**
*
*/
void CommOpenPalm::v_Finalise()
void CommCwipi::v_Finalise()
{
// do nothing
cwipi_finalize();
}
......
library/LibUtilities/Communication/CommOpenPalm.h → library/LibUtilities/Communication/CommCwipi.h
View file @ ee24062e
///////////////////////////////////////////////////////////////////////////////
//
// File CommOpenPalm.h
// File CommCwipi.h
//
// For more information, please see: http://www.nektar.info
//
// The MIT License
//
// Copyright (c) 2015 Kilian Lackhove
// Copyright (c) 2006 Division of Applied Mathematics, Brown University (USA),
// Department of Aeronautics, Imperial College London (UK), and Scientific
// Computing and Imaging Institute, University of Utah (USA).
......@@ -29,11 +30,11 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
//
// Description: CommOpenPalm header
// Description: CommCwipi header
//
///////////////////////////////////////////////////////////////////////////////
#ifndef NEKTAR_LIB_UTILITIES_COMMOPENPALM_H
#define NEKTAR_LIB_UTILITIES_COMMOPENPALM_H
#ifndef NEKTAR_LIB_UTILITIES_COMMCWIPI_H
#define NEKTAR_LIB_UTILITIES_COMMCWIPI_H
#include <string>
#include <mpi.h>
......@@ -46,35 +47,32 @@ namespace Nektar
namespace LibUtilities
{
// Forward declarations
class CommOpenPalm;
class CommCwipi;
/// Pointer to a Communicator object.
typedef boost::shared_ptr<CommOpenPalm> CommOpenPalmSharedPtr;
typedef boost::shared_ptr<CommCwipi> CommCwipiSharedPtr;
/// A global linear system.
class CommOpenPalm : public CommMpi
class CommCwipi : public CommMpi
{
public:
/// Creates an instance of this class
static CommSharedPtr create(int narg, char* arg[])
{
return MemoryManager<CommOpenPalm>::AllocateSharedPtr(narg, arg);
return MemoryManager<CommCwipi>::AllocateSharedPtr(narg, arg);
}
/// Name of class
static std::string className;
CommOpenPalm(int narg, char* arg[]);
CommCwipi(int narg, char* arg[]);
virtual ~CommOpenPalm();
virtual ~CommCwipi();
protected:
virtual void v_Finalise();
private:
CommOpenPalm(MPI_Comm pComm);
};
......
library/SolverUtils/CMakeLists.txt
View file @ ee24062e
......@@ -7,6 +7,7 @@ SET(SOLVER_UTILS_SOURCES
Advection/AdvectionFR.cpp
Advection/AdvectionNonConservative.cpp
Advection/AdvectionWeakDG.cpp
CwipiExchange.cpp
Diffusion/Diffusion.cpp
Diffusion/Diffusion3DHomogeneous1D.cpp
Diffusion/DiffusionLDG.cpp
......@@ -53,6 +54,7 @@ SET(SOLVER_UTILS_HEADERS
Advection/Advection3DHomogeneous1D.h
Advection/AdvectionNonConservative.h
Advection/AdvectionWeakDG.h
CwipiExchange.h
Diffusion/Diffusion.h
Diffusion/Diffusion3DHomogeneous1D.h
Diffusion/DiffusionLDG.h
......
templates/openPalm/OpenPalmExchange.cpp → library/SolverUtils/CwipiExchange.cpp
View file @ ee24062e
////////////////////////////////////////////////////////////////////////////////
//
// File: OpenPalmExchange.cpp
// File: CwipiExchange.cpp
//
// For more information, please see: http://www.nektar.info/
//
......@@ -27,20 +27,19 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
//
// Description: openPALM Exchange class
// Description: CWIPI Exchange class
//
////////////////////////////////////////////////////////////////////////////////
#include "OpenPalmExchange.h"
#include "CwipiExchange.h"
#include <palmlibc.h>
#include <cwipi.h>
namespace Nektar
{
OpenPalmCoupling::OpenPalmCoupling(MultiRegions::ExpListSharedPtr field,
CwipiCoupling::CwipiCoupling(MultiRegions::ExpListSharedPtr field,
string name, int outputFreq, double geomTol) :
Coupling(field, name),
m_outputFormat("Ensight Gold"),
......@@ -48,23 +47,21 @@ OpenPalmCoupling::OpenPalmCoupling(MultiRegions::ExpListSharedPtr field,
m_outputFreq(outputFreq),
m_geomTol(geomTol)
{
// Init OpenPALM
int retVal = PCW_Init();
PCW_Dump_application_properties();
cwipi_dump_application_properties();
// Init Coupling
cwipi_solver_type_t solver_type = CWIPI_SOLVER_CELL_VERTEX;
retVal = PCW_Create_coupling(m_name.c_str(),
CWIPI_COUPLING_PARALLEL_WITH_PARTITIONING,
3,
m_geomTol,
CWIPI_STATIC_MESH,
solver_type,
m_outputFreq,
m_outputFormat.c_str(),
m_outputFormatOption.c_str());
ASSERTL0(retVal == 0, "ERROR: PCW_Create_coupling failed with code" + retVal);
cwipi_create_coupling(m_name.c_str(),
CWIPI_COUPLING_PARALLEL_WITH_PARTITIONING,
"precise",
3,
m_geomTol,
CWIPI_STATIC_MESH,
solver_type,
m_outputFreq,
m_outputFormat.c_str(),
m_outputFormatOption.c_str());
SpatialDomains::MeshGraphSharedPtr graph = m_field->GetGraph();
......@@ -79,13 +76,13 @@ OpenPalmCoupling::OpenPalmCoupling(MultiRegions::ExpListSharedPtr field,
m_nPoints = m_field->GetTotPoints();
// allocate openPALM arrays
m_coords = (double *) malloc(sizeof(double) * 3 * nVerts );
// allocate CWIPI arrays
m_coords = (double *) malloc(sizeof(double) * 3 * nVerts);
ASSERTL1(m_coords != NULL, "malloc failed for m_coords");
m_points = (double *) malloc(sizeof(double) * 3 * m_nPoints );
m_points = (double *) malloc(sizeof(double) * 3 * m_nPoints);
ASSERTL1(m_points != NULL, "malloc failed for m_points");
int tmp = 4*tetgeom.size() + 5*pyrgeom.size() +
6*prismgeom.size() + 8*hexgeom.size();
int tmp = 4 * tetgeom.size() + 5 * pyrgeom.size() +
6 * prismgeom.size() + 8 * hexgeom.size();
m_connec = (int *) malloc(sizeof(int) * tmp);
ASSERTL1(m_connec != NULL, "malloc failed for m_connec");
m_connecIdx = (int *) malloc(sizeof(int) * (nElts + 1));
......@@ -120,50 +117,42 @@ OpenPalmCoupling::OpenPalmCoupling(MultiRegions::ExpListSharedPtr field,
}
*/
retVal = PCW_Define_mesh(m_name.c_str(),
nVerts,
nElts,
m_coords,
m_connecIdx,
m_connec);
ASSERTL0(retVal == 0, "ERROR: PCW_Define_mesh failed with code" + retVal);
cwipi_define_mesh(m_name.c_str(),
nVerts,
nElts,
m_coords,
m_connecIdx,
m_connec);
Array<OneD,NekDouble> x0(m_nPoints);
Array<OneD,NekDouble> x1(m_nPoints);
Array<OneD,NekDouble> x2(m_nPoints);
Array<OneD, NekDouble> x0(m_nPoints);
Array<OneD, NekDouble> x1(m_nPoints);
Array<OneD, NekDouble> x2(m_nPoints);
m_field->GetCoords(x0, x1, x2);
for (int i = 0; i < m_nPoints; ++i)
{
m_points[3*i + 0] = double(x0[i]);
m_points[3*i + 1] = double(x1[i]);
m_points[3*i + 2] = double(x2[i]);
m_points[3 * i + 0] = double(x0[i]);
m_points[3 * i + 1] = double(x1[i]);
m_points[3 * i + 2] = double(x2[i]);
}
retVal = PCW_Set_points_to_locate(m_name.c_str(), m_nPoints, m_points);
ASSERTL0(retVal == 0, "ERROR: PCW_Set_points_to_locate failed with code" + retVal);
cwipi_set_points_to_locate(m_name.c_str(), m_nPoints, m_points);
}
OpenPalmCoupling::~OpenPalmCoupling()
CwipiCoupling::~CwipiCoupling()
{
free(m_coords);
free(m_points);
free(m_connec);
free(m_connecIdx);
int retVal;
retVal = PCW_Delete_coupling(m_name.c_str());
ASSERTL0(retVal == 0, "ERROR: PCW_Delete_coupling failed with code" + retVal);
retVal = PCW_Finalize();
ASSERTL0(retVal == 0, "ERROR: PCW_Finalize failed with code" + retVal);
cwipi_delete_coupling(m_name.c_str());
}
template <typename T>
void OpenPalmCoupling::AddElementsToMesh(T geom, int &coordsPos, int &connecPos,
void CwipiCoupling::AddElementsToMesh(T geom, int &coordsPos, int &connecPos,
int &conidxPos)
{
// helper variables
......@@ -173,7 +162,9 @@ void OpenPalmCoupling::AddElementsToMesh(T geom, int &coordsPos, int &connecPos,
// iterate over all elements
typename T::iterator it;
for(it = geom.begin(); it != geom.end(); it++) {
for (it = geom.begin(); it != geom.end(); it++)
{
// iterate over the elements vertices
for (int j = 0; j < it->second->kNverts; ++j)
{
......@@ -181,13 +172,13 @@ void OpenPalmCoupling::AddElementsToMesh(T geom, int &coordsPos, int &connecPos,
vertID = vert->GetVid();
// check if we already stored the vertex
if (m_vertMap.count( vertID ) == 0)
if (m_vertMap.count(vertID) == 0)
{
// store the vertex
vert->GetCoords(x[0], x[1], x[2]);
m_coords[3*coordsPos + 0] = double(x[0]);
m_coords[3*coordsPos + 1] = double(x[1]);
m_coords[3*coordsPos + 2] = double(x[2]);
m_coords[3 * coordsPos + 0] = double(x[0]);
m_coords[3 * coordsPos + 1] = double(x[1]);
m_coords[3 * coordsPos + 2] = double(x[2]);
// store the vertex position in the m_coords array
m_vertMap[vertID] = coordsPos;
......@@ -198,7 +189,7 @@ void OpenPalmCoupling::AddElementsToMesh(T geom, int &coordsPos, int &connecPos,
connecPos++;
}
m_connecIdx[conidxPos+1] = m_connecIdx[conidxPos] + it->second->kNverts;
m_connecIdx[conidxPos + 1] = m_connecIdx[conidxPos] + it->second->kNverts;
conidxPos++;
}
}
......@@ -210,7 +201,7 @@ void OpenPalmCoupling::AddElementsToMesh(T geom, int &coordsPos, int &connecPos,
OpenPalmExchange::OpenPalmExchange(SolverUtils::CouplingSharedPointer coupling,
CwipiExchange::CwipiExchange(SolverUtils::CouplingSharedPointer coupling,
string name, int nEVars) :
Exchange(coupling, name),
m_nEVars(nEVars)
......@@ -218,24 +209,24 @@ OpenPalmExchange::OpenPalmExchange(SolverUtils::CouplingSharedPointer coupling,
int nPoints = m_coupling->GetNPoints();
m_rValsInterl = (double *) malloc(sizeof(double) * nPoints * m_nEVars);
ASSERTL1(recvValues != NULL, "malloc failed for m_rValsInterl");
ASSERTL1(m_rValsInterl != NULL, "malloc failed for m_rValsInterl");
}
OpenPalmExchange::~OpenPalmExchange()
CwipiExchange::~CwipiExchange()
{
free(m_rValsInterl);
}
void OpenPalmExchange::v_SendFields(const int step, const NekDouble time,
void CwipiExchange::v_SendFields(const int step, const NekDouble time,
Array<OneD, Array<OneD, NekDouble> > &field)
{
ASSERTL0(false, "not implemented yet")
}
void OpenPalmExchange::v_ReceiveFields(const int step, const NekDouble time,
void CwipiExchange::v_ReceiveFields(const int step, const NekDouble time,
Array<OneD, Array<OneD, NekDouble> > &field)
{
static int lastUdate = -1;
......@@ -245,6 +236,7 @@ void OpenPalmExchange::v_ReceiveFields(const int step, const NekDouble time,
cout << "returning" << endl;
return;
}
lastUdate = step;
int nPoints = m_coupling->GetNPoints();
......@@ -253,22 +245,17 @@ void OpenPalmExchange::v_ReceiveFields(const int step, const NekDouble time,
cout << "receiving fields at i = " << step << ", t = " << time << endl;
int retVal;
int nNotLoc;
char varrec[PL_LNAME];
sprintf(varrec, m_recvFieldName.c_str());
char cp_name[PL_LNAME];
sprintf(cp_name, m_coupling->GetName().c_str());
retVal = PCW_Recv(cp_name,
m_name.c_str(),
m_nEVars,
step,
double(time),
varrec,
m_rValsInterl,
&nNotLoc);
ASSERTL0(retVal == 0, "ERROR: PCW_Recv failed with code" + retVal);
cwipi_exchange(m_coupling->GetName().c_str(),
m_name.c_str(),
m_nEVars,
step,
time,
"",
NULL,
m_recvFieldName.c_str(),
m_rValsInterl,
&nNotLoc);
if (nNotLoc != 0)
{
......@@ -278,10 +265,10 @@ void OpenPalmExchange::v_ReceiveFields(const int step, const NekDouble time,
notLoc = (int *) malloc(sizeof(int) * nNotLoc);
ASSERTL1(notLoc != NULL, "malloc failed for notLoc");
retVal = PCW_Get_not_located_points(cp_name, nNotLoc, notLoc);
ASSERTL0(retVal == 0, "ERROR: PCW_Get_not_located_points failed with code" + retVal);
retVal = PCW_Reorder(m_rValsInterl, nPoints, m_nEVars, 0.0, notLoc, nNotLoc);
ASSERTL0(retVal == 0, "ERROR: PCW_Reorder failed with code" + retVal);
// retVal = PCW_Get_not_located_points(cp_name, nNotLoc, notLoc);
// ASSERTL0(retVal == 0, "ERROR: PCW_Get_not_located_points failed with code" + retVal);
// retVal = PCW_Reorder(m_rValsInterl, nPoints, m_nEVars, 0.0, notLoc, nNotLoc);
// ASSERTL0(retVal == 0, "ERROR: PCW_Reorder failed with code" + retVal);
free(notLoc);
}
......
templates/openPalm/OpenPalmExchange.h → library/SolverUtils/CwipiExchange.h
View file @ ee24062e
///////////////////////////////////////////////////////////////////////////////
//
// File OpenPalmExchange.h
// File CwipiExchange.h
//
// For more information, please see: http://www.nektar.info
//
......@@ -27,12 +27,12 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
//
// Description: openPALM Exchange class
// Description: CWIPI Exchange class
//
///////////////////////////////////////////////////////////////////////////////
#ifndef NEKTAR_OPENPALMEXCHANGE
#define NEKTAR_OPENPALMEXCHANGE
#ifndef NEKTAR_CWIPIEXCHANGE
#define NEKTAR_CWIPIEXCHANGE
#include <SolverUtils/EquationSystem.h>
#include <SolverUtils/Exchange.h>
......@@ -40,19 +40,19 @@
namespace Nektar
{
class OpenPalmCoupling : public SolverUtils::Coupling
class CwipiCoupling : public SolverUtils::Coupling
{
public:
OpenPalmCoupling()
CwipiCoupling()
{
};
OpenPalmCoupling(MultiRegions::ExpListSharedPtr field,
CwipiCoupling(MultiRegions::ExpListSharedPtr field,
string name, int outputFreq, double geomTol);
~OpenPalmCoupling();
~CwipiCoupling();
protected:
......@@ -76,21 +76,21 @@ private: