Commit 25695881 authored by Spencer Sherwin's avatar Spencer Sherwin

Merge branch 'feature/dev-guide-draft-mk' into Current-Draft

parents 85a28cd6 af9e88f2
title={Spectral/hp element methods for Computational Fluid Dynamics (Second Edition)},
author={George Em Karniadakis and Spencer J. Sherwin},
publisher={Oxford University Press}
title={p-- and hp-- Finite Element Methods: Theory and Applications in Solid and Fluid Mechanics},
author={Ch. Schwab},
publisher={Oxford University Press}
title = {Testing Computer Software},
author = {Cem Kaner and Jack Falk and Hung Quoc Nguyen},
year = {2010},
publisher={John Wiley \& Sons}
author="C. Canuto and M.Y. Hussaini and A. Quarteroni and T.A. Zang",
title="{Spectral Methods in Fluid Mechanics}",
publisher="{Springer-Verlag, New York}",
year="1987" }
author="D. Funaro",
title="{Polynomial Approximations of Differential Equations: Lecture Notes in Physics, Volume 8}",
publisher="{Springer-Verlag, New York}",
year="1992" }
author = "George Em Karniadakis and Robert M. Kirby",
title = "Parallel Scientific Computing in C++ and MPI",
publisher = "Cambridge University Press",
address = "New-York, NY, USA",
year = 2003
author = "T. J. R. Hughes",
title = "The Finite Element Method",
publisher = "Prentice-Hall, Inc.",
year = "1987",
address = "Englewood Cliffs, New Jersey"
author = "B.A. Szab\'{o} and I. Babu\v{s}ka",
title = "Finite Element Analysis",
publisher = "John Wiley \& Sons",
address = "New York",
year = 1991
author = "Lloyd N. Trefethen and David Bau, III",
title = "Numerical Linear Algebra",
publisher = "SIAM",
address = "Philadelphia, PA, USA",
year = 1997
author = "James W. Demmel",
title = "Applied Numerical Linear Algebra",
publisher = "SIAM",
address = "Philadelphia, PA, USA",
year = 1997
title={A spectral element method for fluid dynamics: laminar flow in a channel expansion},
author={Patera, Anthony T},
journal={Journal of computational Physics},
title={The p-version of the finite element method},
author={Babuska, Ivo and Szabo, Barna A and Katz, I Norman},
journal={SIAM journal on numerical analysis},
title={Formulation of a Galerkin spectral element--Fourier method for three-dimensional incompressible flows in cylindrical geometries},
author={Blackburn, Hugh M and Sherwin, SJ},
journal={Journal of Computational Physics},
title={Modular $hp$-{FEM} system {HERMES} and its application to {M}axwell’s equations},
author={Vejchodsk{\`y}, Tom{\'a}{\v{s}} and {\v{S}}ol{\'\i}n, Pavel and Z{\'\i}tka, Martin},
journal={Mathematics and Computers in Simulation},
title={A generic interface for parallel and adaptive discretization schemes: abstraction principles and the {DUNE-FEM} module},
author={Dedner, Andreas and Kl{\"o}fkorn, Robert and Nolte, Martin and Ohlberger, Mario},
title={deal.{II}--a general-purpose object-oriented finite element library},
author={Bangerth, Wolfgang and Hartmann, Ralf and Kanschat, Guido},
journal={ACM Transactions on Mathematical Software (TOMS)},
title={Nodal discontinuous Galerkin methods: algorithms, analysis, and applications},
author={Hesthaven, Jan S and Warburton, Tim},
title = {A Guide to the Implementation of Boundary Conditions in Compact High-Order Methods for Compressible Aerodynamics},
author = {Mengaldo, G. and De Grazia, D. and Peiro, J. and Farrington, A. and Witherden, F. and Vincent, P. E. and Sherwin, S. J.},
year = {2014},
booktitle = {7th AIAA Theoretical Fluid Mechanics Conference},
series = {AIAA Aviation},
publisher = {American Institute of Aeronautics and Astronautics},
title={{PyFR}: An open source framework for solving advection–diffusion type problems on streaming architectures using the flux reconstruction approach},
author={Witherden, FR and Farrington, AM and Vincent, PE},
journal={Computer Physics Communications},
%%% Software
author = "Peter E. J. Vos and Spencer J. Sherwin and Robert M. Kirby",
title = "h-to-p Efficiently: Implementing Finite and Spectral/hp Element Methods to Achieve Optimal Performance for Low- and High-Order Discretisations",
journal = "Journal of Computational Physics",
volume = "229",
issue = "13",
pages = "5161-5181",
year = "2010"
author = "C.D. Cantwell and S.J. Sherwin and R.M. Kirby and P.H.J. Kelly",
title = "From h-to-p Efficiently: Strategy Selection for Operator Evaluation on Hexahedral and Tetrahedral Elements",
journal = "Computers and Fluids",
volume = "43",
issue = "1",
pages = "23-28",
year = "2011"
author = "C.D. Cantwell and S.J. Sherwin and R.M. Kirby and P.H. Kelly",
title = "From h-to-p Efficiently: Selecting the Optimal Spectral/hp Discretisation in Three Dimensions",
journal = "Math. Model. Nat. Phenom.",
volume = "6",
number = "3",
pages = "84-96",
year = "2011"
author = "A. Bolis and C.D. Cantwell and R.M. Kirby and S.J. Sherwin",
title = "h-to-p efficiently: Optimal implementation strategies for explicit time-dependent problems using the spectral/hp element method",
journal = "International Journal for Numerical Methods in Fluids",
volume = "75",
issue = "8",
pages = "591-607",
year = "2014"
author = "C.D. Cantwell and D. Moxey and A. Comerford and A. Bolis and G. Rocco and G. Mengaldo and D. de Grazia and S. Yakovlev and J-E Lombard and D. Ekelschot and B. Jordi and H. Xu and Y. Mohamied and C. Eskilsson and B. Nelson and P. Vos and C. Biotto and R.M. Kirby and S.J. Sherwin",
title = "Nektar++: An open-source spectral/hp element framework",
journal = "Computer Physics Communications",
volume = "192",
pages = "205-219",
year = "2015"
title = {Camellia: {A} software framework for discontinuous {Petrov-Galerkin} methods},
volume = {68},
shorttitle = {Camellia},
abstract = {The discontinuous Petrov-Galerkin (DPG) methodology of Demkowicz and Gopalakrishnan minimizes the solution residual in a user-determinable energy norm and offers a built-in mechanism for evaluating error in the energy norm, among other desirable features. However, the methodology also brings with it some additional complexity for researchers who wish to experiment with DPG in their computations. In this paper, we introduce Camellia, a software framework whose central design goal is to enable developers to create efficient hp-adaptive DPG solvers with minimal effort.},
number = {11},
journal = {Computers \& Mathematics with Applications},
author = {Roberts, Nathan V.},
year = {2014},
pages = {1581--1604}
title = {Elvis: {A} system for the accurate and interactive visualization of high-order finite element solutions},
volume = {18},
shorttitle = {Elvis},
abstract = {This paper presents the Element Visualizer (ElVis), a new, open-source scientific visualization system for use with high-order finite element solutions to PDEs in three dimensions. This system is designed to minimize visualization errors of these types of fields by querying the underlying finite element basis functions (e.g., high-order polynomials) directly, leading to pixel-exact representations of solutions and geometry. The system interacts with simulation data through runtime plugins, which only require users to implement a handful of operations fundamental to finite element solvers. The data in turn can be visualized through the use of cut surfaces, contours, isosurfaces, and volume rendering. These visualization algorithms are implemented using NVIDIA's OptiX GPU-based ray-tracing engine, which provides accelerated ray traversal of the high-order geometry, and CUDA, which allows for effective parallel evaluation of the visualization algorithms. The direct interface between ElVis and the underlying data differentiates it from existing visualization tools. Current tools assume the underlying data is composed of linear primitives; high-order data must be interpolated with linear functions as a result. In this work, examples drawn from aerodynamic simulations-high-order discontinuous Galerkin finite element solutions of aerodynamic flows in particular-will demonstrate the superiority of ElVis' pixel-exact approach when compared with traditional linear-interpolation methods. Such methods can introduce a number of inaccuracies in the resulting visualization, making it unclear if visual artifacts are genuine to the solution data or if these artifacts are the result of interpolation errors. Linear methods additionally cannot properly visualize curved geometries (elements or boundaries) which can greatly inhibit developers' debugging efforts. As we will show, pixel-exact visualization exhibits none of these issues, removing the visualization scheme as a source of - ncertainty for engineers using ElVis.},
number = {12},
journal = {Visualization and Computer Graphics, IEEE Transactions on},
author = {Nelson, Blake and Liu, Eric and Kirby, Robert M. and Haimes, Robert},
year = {2012},
pages = {2325--2334}
title = {Gmsh: {A} 3-{D} finite element mesh generator with built-in pre-and post-processing facilities},
volume = {79},
shorttitle = {Gmsh},
abstract = {Gmsh is an open-source 3-D finite element grid generator with a build-in CAD
engine and post-processor. Its design goal is to provide a fast, light and user-friendly
meshing tool with parametric input and advanced visualization capabilities. This paper presents the overall philosophy, the main design choices and some of the original algorithms implemented in Gmsh. Copyright© 2009 John Wiley \& Sons, Ltd.},
number = {11},
journal = {International Journal for Numerical Methods in Engineering},
author = {Geuzaine, Christophe and Remacle, Jean-François},
year = {2009},
pages = {1309--1331},
file = {[PDF] from Support/Zotero/Profiles/7xumqoyz.default/zotero/storage/IASXUV6Z/Geuzaine and Remacle - 2009 - Gmsh A 3-D finite element mesh generator with bui.pdf:application/pdf}
title = "Nek5000 User Manual",
author = "Paul Fischer and James Lottes and Stefan Kerkemeier and Oana Marin and Katherine Heisey and Aleks Obabko and Elia Merzari and Yulia Peet",
publisher = "ANL/MCS-TM-351",
year = "2014"
title = "High-Order Methods for Incompressible Fluid Flow",
author = "M.O. Deville and P.F. Fisher and E.H. Mund",
publisher = "Cambridge University Press",
year = "2002"
author = "J.S. Hesthaven and T.C. Warburton",
title = "Nodal Discontinuous {G}alerkin Methods: Algorithms, Analysis, and Applications",
publisher = "Springer Texts in Applied Mathematics 54. Springer Verlag: New York",
year = "2008"
author = "F.X. Giraldo and J.F. Kelly and E.M. Constantinescu",
title = "Implicit explicit formulations of a three dimensional non-hydrostatic unified model of the atmosphere {(NUMA)}",
journal = "SIAM Journal of Scientific Computing",
volume = "35",
pages = "1162-1194",
year = "2013"
title = "Automated Solution of Differential Equations by the Finite Element Method",
author = "Anders Logg and Kent-Andre Mardal and Garth Wells (editors)",
publisher = "Springer Lecture Notes in Computational Science and Engineering, Volume 84",
year = "2012"
author = "Mark Ainsworth and Gaelle Andriamaro and Oleg Davydov",
title = "Bernstein-B{\'e}zier Finite Elements of Arbitrary Order and Optimal Assembly Procedures",
journal = "SIAM Journal of Scientific Computing",
volume = "33",
issue = "6",
pages = "3087-3109",
year = "2011"
author = "Mark Ainsworth",
title = "Pyramid Algorithms for Bernstein-B{\'e}zier Finite Elements of High, Nonuniform Order in Any Dimension",
journal = "SIAM Journal of Scientific Computing",
volume = "36",
issue = "2",
pages = "A543-A569",
year = "2014"
author = "T. Warburton and L.F. Pavarino and J.S. Hesthaven",
title = "A Pseudo-spectral scheme for the incompressible {N}avier-{S}tokes equations using unstructured nodal elements",
journal = "J. Comp. Phys.",
volume = "164",
pages = "1-21",
year = "2000"
author = "M. Dubiner",
title = "Spectral Methods on Triangles and Other Domains",
journal = "J. Sci. Comp.",
volume = "6",
pages = "345",
year = "1991"
author = {Hesthaven, J.S. },
title = {From electrostatics to almost optimal nodal sets for
polynomial interpolation in a simplex},
journal = {SIAM J. Numer. Anal.},
year = {1998},
volume = {35},
number = {2},
pages = {655-676},
author = {Taylor, M. and Wingate, B.A. and Vincent, R.E.},
title = {An Algorithm for computing {F}ekete points in the triangle},
journal = {SIAM J. Num. Anal.},
year = {2000},
volume = {38},
pages = {1707-1720},
author = "Duffy, M.G.",
title = "Quadrature over a pyramid or cube of integrands
with a singularity at a vertex",
journal ="SIAM J. Numer. Anal.",
volume = "19",
pages = "1260",
year = "1982"
author = "C.D. Cantwell and S. Yakovlev and R.M. Kirby and N.S. Peters and S.J. Sherwin",
title = "High-order continuous spectral/hp element discretisation for reaction-diffusion problems on a surface",
journal = "Journal of Computational Physics",
volume = "257",
issue = "A",
pages = "813-829",
year = "2014"
\title{A Programmer's Guide to Nektar++}
% Render pretty title page if not building HTML
\huge{Nektar++ Developer's Guide}
\ No newline at end of file
\ No newline at end of file
\ No newline at end of file
\ No newline at end of file
\ No newline at end of file
\ No newline at end of file
\ No newline at end of file
\ No newline at end of file
\ No newline at end of file
% Introduction
Finite element methods (FEM) are commonplace among a wide range of engineering
and biomedical disciplines for the solution of partial differential equations
(PDEs) on complex geometries. However, low-order formulations often struggle to
capture certain complex solution characteristics without the use of excessive
mesh refinement due to numerical dissipation. In contrast, spectral techniques
offer improved numerical characteristics, but are typically restricted to
relatively simple regular domains.
High-order finite element methods, such as the traditional spectral element
method \cite{Pa84}, the p-type method \cite{BaSzKa81} and the more recent \shp{}
element method \cite{KaSh05}, exhibit the convergence properties of spectral methods while retaining the geometric
flexibility of traditional linear FEM.
%and transient flow simulation,
%where the higher accuracy enables improved prediction of downstream dynamics.
% %
They potentially offer greater efficiency on modern CPU architectures
as well as more exotic platforms such as many-core general-purpose graphics
processing units (GPGPUs).
The data structures which arise from using high-order methods are more compact
and localised than their linear finite element counterparts, for a fixed number
of degrees of freedom, providing increased cache coherency and reduced memory
accesses, which is increasingly the primary bottleneck of modern computer systems.
The methods have had greatest
prominence in the structural mechanics community and subsequently the academic
fluid dynamics community. They are also showing promise
in other areas of engineering, biomedical and environmental research. The most
common concern cited with respect to using high-order finite element techniques
outside of academia is the implementational complexity, stemming from the
complex data structures, necessary to produce a computationally efficient
implementation. This is a considerable hurdle which has limited their widespread
uptake in many application domains and industries.
% Overview of Nektar++ and its goals/features
\nek{} is a cross-platform \shp{} element framework which aims to make
high-order finite element methods accessible to the broader community. This is
achieved by providing a structured hierarchy of C++ components, encapsulating
the complexities of these methods, which can be readily applied to a range of
application areas. These components are distributed in the form of
cross-platform software libraries, enabling rapid development of solvers for use
in a wide variety of computing environments. The code accommodates both
small research problems, suitable for desktop computers, and large-scale
industrial simulations, requiring modern HPC infrastructure, where
there is a need to maintain efficiency and scalability up to many thousands of
processor cores.
% % Previous software - Nektar (does not do anything more than IncNS)
A number of software packages already exist for fluid dynamics which implement
high-order finite element methods, although these packages are typically targeted at a specific
domain or provide limited high-order capabilities as an extension.
% %
The \emph{Nektar flow solver} is the predecessor to Nektar++ and
implements the \shp{} element method for solving the incompressible
and compressible Navier-Stokes equations in both 2D and 3D. While it is widely
used and the implementation is computationally efficient on small parallel problems,
achieving scaling on large HPC clusters is challenging. Semtex \cite{BlSh04}
implements the 2D spectral element method coupled with a Fourier expansion in
the third direction. The implementation is highly efficient, but can only be
parallelised through Fourier-mode decomposition.
Nek5000 \cite{Nek5000} is a 3D spectral element code, based on hexahedral
elements, which has been used for massively parallel simulations up to 300,000
cores. Hermes \cite{VeSoZi07} implements hp-FEM for two-dimensional problems and
has been used in a number of application areas. Limited high-order finite
element capabilities are also included in a number of general purpose PDE
frameworks including the DUNE project \cite{DeKlNoOh11} and deal.II
% %
A number of codes also implement high-order finite element methods on
GPGPUs including nudg++, which
implments a nodal discontinuous Galerkin scheme \cite{HeWa07}, and PyFR
\cite{WiFaVi14}, which supports a range of flux reconstruction techniques.
% % However, the present software is instead a collection of libraries which
% implement the underlying discretisation technique, providing the ability to
% solve a range of PDE problems across a broad range of application areas. The
% included incompressible Navier-Stokes solver supports all the existing
% functionality of the Nektar flow solver, including improvements in many areas.
% %
\nek{} provides a single codebase with the following key features:
\item Arbitrary-order \shp{} element discretisations in one, two and three
\item Support for variable polynomial order in space and heterogeneous
polynomial order within two- and three-dimensional elements;
\item High-order representation of the geometry;
\item Continuous Galerkin, discontinuous Galerkin and hybridised discontinuous
Galerkin projections;
\item Support for a Fourier extension of the spectral element mesh;
\item Support for a range of linear solvers and preconditioners;
\item Multiple implementation strategies for achieving linear algebra
performance on a range of platforms;
\item Efficient parallel communication using MPI showing strong scaling up to
2048-cores on Archer, the UK national HPC system;
\item A range of time integration schemes implemented using generalised linear
methods; and
\item Cross-platform support for Linux, OS X and Windows operating
In addition to the core functionality listed above, Nektar++ includes a
number of solvers covering a range of application areas. A range
of pre-processing and post-processing utilities are also included with support
for popular mesh and visualization formats, and an extensive test suite ensures
the robustness of the core functionality.
\section{The {\em Ethos} of Nektar++}
As with any research effort, one is required to decide on a set of guiding principles that will
drive the investigation. Similarly with a software development effort of this form, we early on
spent a lot of time considering what things we wanted to be distictive about Nektar++ and also
what guiding principles would we use to help set both the goals and the boundaries of what
we wanted to do. We did this for at least two reasons: (1) We acknowledged then and now that
there are various software packages and open-source efforts that deal with finite element frameworks,
and so we wanted to be able to understand and express to people those things we thought were
distinctive to us -- that is, our ``selling points". (2) We also acknowledged, from our own experience
on software projects, that if we did not set up some collection of guiding principles for our work, that
we would gravitate towards trying to be ``all things to all men", and in doing so be at odds with the
first item. Below are a list of the guiding principles, the ``ethos", of the Nektar++ software development
effort. The first three boldface items denote the three major themes of our work (i.e. respecting reason (1) above) and the
subsequent items denotes the guardrails (i.e. respecting reason (2) above) that
we put in place to help guide our efforts.
\item \textbf{Efficiently:} Nektar++ was to be a ``true" high-order code. ``True" is put in quotations because we acknowledge
that high-order means different things to different communities. Based upon a review of the literature, we
came to the conclusion that part of our {\em h-to-p} philosophy should be that we accommodate polynomial
degrees ranging from zero (finite volumes) or one (traditional linear finite elements) up to what is considered
``spectral" (pseodspectral) orders of 16 degree. As part of our early work \cite{vos}, we established that in
order to span this range of polynomial degrees and attempt to maintain some level of computational
efficiency, we would need to develop {\em order-aware} algorithms: that is, we would need to utilize
different (equivalent) algorithms appropriate for a particular order. This principle was the starting point