Commit 60bd978b authored by Kilian Lackhove's avatar Kilian Lackhove

Coupling: First OpenPalm Implementaion

parent 979bd397
......@@ -191,6 +191,14 @@ INCLUDE (ThirdPartyTriangle)
INCLUDE (ThirdPartyTetGen)
INCLUDE (ThirdPartyCCM)
set(OPENPALM_DIR "/home/lackhove/code/OpenPalm/PALM_MP/linux64r4openmpi_debug" CACHE PATH "openPALM base path")
find_package(OpenPALM)
include_directories(${OPENPALM_INCLUDE_DIRS})
set(CWIPI_DIR "/home/lackhove/code/OpenPalm/cwipi-0.8.1/dist" CACHE PATH "CWIPI base path")
find_package(Cwipi)
include_directories(${CWIPI_INCLUDE_DIRS})
INCLUDE (Doxygen)
IF( NEKTAR_USE_MKL )
......
# - Try to find cwipi
# Once done this will define
# CWIPI_FOUND - System has cwipi
# CWIPI_INCLUDE_DIRS - The cwipi include directories
# CWIPI_LIBRARIES - The libraries needed to use cwipi
# CWIPI_DEFINITIONS - Compiler switches required for using cwipi
set(CWIPI_DEFINITIONS "")
find_path(CWIPI_INCLUDE_DIR cwipi.h
HINTS ${CWIPI_DIR}/include/
)
find_library(CWIPI_LIBRARY
NAMES "libcwipi.a"
HINTS "${CWIPI_DIR}/lib64/" "${CWIPI_DIR}/lib/"
)
set(CWIPI_LIBRARIES
${CWIPI_LIBRARY}
)
set(CWIPI_INCLUDE_DIRS ${CWIPI_INCLUDE_DIR})
include(FindPackageHandleStandardArgs)
# handle the QUIETLY and REQUIRED arguments and set CWIPI_FOUND to TRUE
# if all listed variables are TRUE
find_package_handle_standard_args(cwipi DEFAULT_MSG
CWIPI_LIBRARY
CWIPI_INCLUDE_DIR
)
mark_as_advanced(CWIPI_LIBRARY CWIPI_INCLUDE_DIR)
# - Try to find openPALM
# Once done this will define
# OPENPALM_FOUND - System has openPALM
# OPENPALM_INCLUDE_DIRS - The openPALM include directories
# OPENPALM_LIBRARIES - The libraries needed to use openPALM
# OPENPALM_DEFINITIONS - Compiler switches required for using openPALM
set(OPENPALM_DEFINITIONS "")
find_path(OPENPALM_INCLUDE_DIR palmlib.h
HINTS ${OPENPALM_DIR}/include/
)
find_library(OPENPALM_LIBRARY
NAMES "libpalm.a"
HINTS "${OPENPALM_DIR}/lib64/" "${OPENPALM_DIR}/lib/"
)
set(OPENPALM_LIBRARIES
${OPENPALM_LIBRARY}
)
set(OPENPALM_INCLUDE_DIRS ${OPENPALM_INCLUDE_DIR})
include(FindPackageHandleStandardArgs)
# handle the QUIETLY and REQUIRED arguments and set OPENPALM_FOUND to TRUE
# if all listed variables are TRUE
find_package_handle_standard_args(openPALM DEFAULT_MSG
OPENPALM_LIBRARY
OPENPALM_INCLUDE_DIR
)
mark_as_advanced(OPENPALM_LIBRARY OPENPALM_INCLUDE_DIR)
......@@ -82,7 +82,8 @@ IF(NEKTAR_USE_MPI)
SET(CommunicationHeaders ${CommunicationHeaders}
./Communication/CommMpi.h)
SET(CommunicationSources ${CommunicationSources}
./Communication/CommMpi.cpp)
./Communication/CommMpi.cpp
./Communication/CommOpenPalm.cpp)
ENDIF(NEKTAR_USE_MPI)
SET(FFTHeaders
......
......@@ -76,6 +76,16 @@ namespace Nektar
MPI_Comm GetComm();
protected:
MPI_Comm m_comm;
int m_rank;
CommMpi(MPI_Comm pComm);
CommMpi()
{
};
virtual void v_Finalise();
virtual int v_GetRank();
virtual void v_Block();
......@@ -128,11 +138,6 @@ namespace Nektar
Array<OneD, int>& pRecvDataOffsetMap);
virtual void v_SplitComm(int pRows, int pColumns);
private:
MPI_Comm m_comm;
int m_rank;
CommMpi(MPI_Comm pComm);
};
}
}
......
///////////////////////////////////////////////////////////////////////////////
//
// File CommOpenPalm.cpp
//
// For more information, please see: http://www.nektar.info
//
// The MIT License
//
// Copyright (c) 2006 Division of Applied Mathematics, Brown University (USA),
// Department of Aeronautics, Imperial College London (UK), and Scientific
// Computing and Imaging Institute, University of Utah (USA).
//
// License for the specific language governing rights and limitations under
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
//
// Description: MPI communication implementation
//
///////////////////////////////////////////////////////////////////////////////
#include <LibUtilities/Communication/CommOpenPalm.h>
#include <LibUtilities/BasicUtils/SharedArray.hpp>
#include <palmlibc.h>
namespace Nektar
{
namespace LibUtilities
{
std::string CommOpenPalm::className
= GetCommFactory().RegisterCreatorFunction("OpenPalm", CommOpenPalm::create,
"Parallel communication using MPI with OpenPalm.");
/**
*
*/
CommOpenPalm::CommOpenPalm(int narg, char* arg[])
: CommMpi()
{
m_comm = PL_COMM_EXEC;
MPI_Comm_size( m_comm, &m_size );
MPI_Comm_rank( m_comm, &m_rank );
m_type = "Parallel MPI with OpenPalm";
}
/**
*
*/
CommOpenPalm::CommOpenPalm(MPI_Comm pComm)
: CommMpi()
{
m_comm = pComm;
MPI_Comm_size( m_comm, &m_size );
MPI_Comm_rank( m_comm, &m_rank );
m_type = "Parallel MPI with OpenPalm";
}
/**
*
*/
CommOpenPalm::~CommOpenPalm()
{
}
/**
*
*/
void CommOpenPalm::v_Finalise()
{
// do nothing
}
}
}
///////////////////////////////////////////////////////////////////////////////
//
// File CommOpenPalm.h
//
// For more information, please see: http://www.nektar.info
//
// The MIT License
//
// Copyright (c) 2006 Division of Applied Mathematics, Brown University (USA),
// Department of Aeronautics, Imperial College London (UK), and Scientific
// Computing and Imaging Institute, University of Utah (USA).
//
// License for the specific language governing rights and limitations under
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
//
// Description: CommOpenPalm header
//
///////////////////////////////////////////////////////////////////////////////
#ifndef NEKTAR_LIB_UTILITIES_COMMOPENPALM_H
#define NEKTAR_LIB_UTILITIES_COMMOPENPALM_H
#include <string>
#include <mpi.h>
#include <LibUtilities/Communication/CommMpi.h>
#include <LibUtilities/Memory/NekMemoryManager.hpp>
namespace Nektar
{
namespace LibUtilities
{
// Forward declarations
class CommOpenPalm;
/// Pointer to a Communicator object.
typedef boost::shared_ptr<CommOpenPalm> CommOpenPalmSharedPtr;
/// A global linear system.
class CommOpenPalm : public CommMpi
{
public:
/// Creates an instance of this class
static CommSharedPtr create(int narg, char* arg[])
{
return MemoryManager<CommOpenPalm>::AllocateSharedPtr(narg, arg);
}
/// Name of class
static std::string className;
CommOpenPalm(int narg, char* arg[]);
virtual ~CommOpenPalm();
protected:
virtual void v_Finalise();
private:
CommOpenPalm(MPI_Comm pComm);
};
}
}
#endif
......@@ -66,6 +66,7 @@ SET(SOLVER_UTILS_HEADERS
DriverStandard.h
DriverSteadyState.h
EquationSystem.h
Exchange.h
Filters/Filter.h
Filters/FilterAeroForces.h
Filters/FilterAverageFields.h
......
......@@ -49,6 +49,7 @@
#include <SolverUtils/Interpolator.h>
#include <SolverUtils/SolverUtilsDeclspec.h>
#include <SolverUtils/Core/Misc.h>
#include <SolverUtils/Exchange.h>
namespace Nektar
{
......@@ -442,6 +443,16 @@ namespace Nektar
/// Evaluates the boundary conditions at the given time.
SOLVER_UTILS_EXPORT void SetBoundaryConditions(NekDouble time);
CouplingSharedPointer GetCoupling()
{
return m_coupling;
}
void SetCoupling(CouplingSharedPointer coupling)
{
m_coupling = coupling;
}
/// Virtual function to identify if operator is negated in DoSolve
SOLVER_UTILS_EXPORT virtual bool v_NegatedOp();
......@@ -521,6 +532,8 @@ namespace Nektar
Array<OneD, bool> m_checkIfSystemSingular;
/// Map to identify relevant solver info to dump in output fields
LibUtilities::FieldMetaDataMap m_fieldMetaDataMap;
/// External tool connector
CouplingSharedPointer m_coupling;
/// Number of Quadrature points used to work out the error
int m_NumQuadPointsError;
......
///////////////////////////////////////////////////////////////////////////////
//
// File Exchange.h
//
// For more information, please see: http://www.nektar.info
//
// The MIT License
//
// Copyright (c) 2015 Kilian Lackhove
//
// License for the specific language governing rights and limitations under
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
//
// Description: openPALM Exchange class
//
///////////////////////////////////////////////////////////////////////////////
#ifndef NEKTAR_SOLVERUTILS_EXCHANGE
#define NEKTAR_SOLVERUTILS_EXCHANGE
#include <string>
#include <MultiRegions/ExpList.h>
#include <SolverUtils/EquationSystem.h>
using namespace std;
namespace Nektar
{
namespace SolverUtils
{
class Coupling
{
public:
Coupling()
{
};
Coupling(MultiRegions::ExpListSharedPtr field, string name) :
m_field(field),
m_name(name)
{
};
~Coupling()
{
};
string GetName()
{
return m_name;
};
int GetNPoints()
{
return m_nPoints;
};
void GetPoints(double *points)
{
points = m_points;
}
void FinalizeCoupling();
protected:
MultiRegions::ExpListSharedPtr m_field;
int m_nPoints;
double *m_points = NULL;
string m_name;
};
typedef boost::shared_ptr<Coupling> CouplingSharedPointer;
class Exchange
{
public:
Exchange()
{
};
Exchange(CouplingSharedPointer coupling, string name) :
m_coupling(coupling),
m_name(name)
{
};
~Exchange()
{
};
virtual void SendFields(const int step, const NekDouble time,
Array<OneD, Array<OneD, NekDouble> > &field)
{
v_SendFields(step, time, field);
}
virtual void ReceiveFields(const int step, const NekDouble time,
Array<OneD, Array<OneD, NekDouble> > &field)
{
v_ReceiveFields(step, time, field);
}
protected:
CouplingSharedPointer m_coupling;
string m_name;
virtual void v_SendFields(const int step, const NekDouble time,
Array<OneD, Array<OneD, NekDouble> > &field)
{
ASSERTL0(false, "not valid for this class");
};
virtual void v_ReceiveFields(const int step, const NekDouble time,
Array<OneD, Array<OneD, NekDouble> > &field)
{
ASSERTL0(false, "not valid for this class");
};
};
typedef boost::shared_ptr<Exchange> ExchangeSharedPtr;
}
}
#endif
//PALM_UNIT -name APESolver\
// -functions {C APESolver}\
// -parallel mpi\
// -minproc 1\
// -maxproc 100000\
// -object_files {APESolver.o APESolver/EquationSystems/APE.o APESolver/RiemannSolvers/APESolver.o APESolver/RiemannSolvers/UpwindSolver.o APESolver/RiemannSolvers/LaxFriedrichsSolver.o OpenPalmExchange.o}\
// -comment {APESolver openPALM unit}
//
//PALM_CWIPI_COUPLING -name PRECISE-Nektar
//
//PALM_CWIPI_OBJECT -name basefields\
// -coupling PRECISE-Nektar\
// -intent IN
//
///////////////////////////////////////////////////////////////////////////////
//
// File APESolver.cpp
//
// For more information, please see: http://www.nektar.info
//
// The MIT License
//
// Copyright (c) 2006 Division of Applied Mathematics, Brown University (USA),
// Department of Aeronautics, Imperial College London (UK), and Scientific
// Computing and Imaging Institute, University of Utah (USA).
//
// License for the specific language governing rights and limitations under
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
//
// Description: APE Equations framework solver
//
///////////////////////////////////////////////////////////////////////////////
#include <stdio.h>
#include <SolverUtils/Driver.h>
#include <LibUtilities/BasicUtils/SessionReader.h>
#include <LibUtilities/Communication/Comm.h>
#include "OpenPalmExchange.h"
#include <palmlibc.h>
#include <cwipi.h>
using namespace std;
using namespace Nektar;
using namespace Nektar::SolverUtils;
extern "C" int APESolver()
{
// redirect stdout to a file
FILE *logFile;
logFile = fopen ("nektar.log","wt");
dup2(fileno(logFile), STDOUT_FILENO);
dup2(fileno(logFile), STDERR_FILENO);
// set casename
int argc = 1;
char *argv[] = {
"APESolverExt",
};
LibUtilities::SessionReaderSharedPtr session;
string vDriverModule;
DriverSharedPtr drv;
try
{
ASSERTL0(LibUtilities::GetCommFactory().ModuleExists("OpenPalm"), "OpenPalm comm module not found");
LibUtilities::CommSharedPtr comm = LibUtilities::GetCommFactory().CreateInstance("OpenPalm", argc, argv);
std::vector<std::string> filenames;
filenames.push_back("APE_3DPulseWall_FRDG_MODIFIED.xml");
// Create session reader.
session = LibUtilities::SessionReader::CreateInstance(argc, argv, filenames, comm);
// Create driver
session->LoadSolverInfo("Driver", vDriverModule, "Standard");
drv = GetDriverFactory().CreateInstance(vDriverModule, session);
// this initializes the APE system
// Execute driver
drv->Execute();
// This runs UnsteadySystem::v_DoSolve() exchange should happen in APE:v_preIntegrate()
// Finalise session
session->Finalise();
}
catch (const std::runtime_error& e)
{
return 1;
}
catch (const std::string& eStr)
{
cout << "Error: " << eStr << endl;
}
fclose(logFile);
return 0;
}
This diff is collapsed.
///////////////////////////////////////////////////////////////////////////////
//
// File OpenPalmExchange.h
//
// For more information, please see: http://www.nektar.info
//
// The MIT License
//
// Copyright (c) 2015 Kilian Lackhove
//
// License for the specific language governing rights and limitations under
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
//
// Description: openPALM Exchange class
//
///////////////////////////////////////////////////////////////////////////////
#ifndef NEKTAR_OPENPALMEXCHANGE
#define NEKTAR_OPENPALMEXCHANGE
#include <SolverUtils/EquationSystem.h>
#include <SolverUtils/Exchange.h>
namespace Nektar
{
class OpenPalmCoupling : public SolverUtils::Coupling
{
public:
OpenPalmCoupling()
{
};
OpenPalmCoupling(MultiRegions::ExpListSharedPtr field,
string name, int outputFreq, double geomTol);
~OpenPalmCoupling();
protected:
string m_outputFormat;
string m_outputFormatOption;
int m_outputFreq;
double m_geomTol;
double *m_coords = NULL;
int *m_connecIdx = NULL;
int *m_connec = NULL;
map< int, int > m_vertMap;
private:
template <typename T>
void AddElementsToMesh(T geom, int& coordsPos, int& connecPos, int& conidxPos);
};
typedef boost::shared_ptr<OpenPalmCoupling> OpenPalmCouplingSharedPointer;
class OpenPalmExchange : public SolverUtils::Exchange
{
public:
OpenPalmExchange()
{
};
OpenPalmExchange(SolverUtils::CouplingSharedPointer coupling, string name,
int nEVars);
~OpenPalmExchange();
protected:
int m_nEVars;
string m_sendFieldName;
string m_recvFieldName;
double *m_rValsInterl = NULL;
virtual void v_SendFields(const int step, const NekDouble time,