Add collection timing utility based on boost::timer

cmake/NektarPackageGenerator.cmake

@@ -63,7 +63,7 @@ solvers in the nektar++-solvers package.")
 		
 		SET(CPACK_RPM_PACKAGE_URL "www.nektar.info")
 		SET(CPACK_RPM_COMPONENT_INSTALL ON)
-		SET(CPACK_RPM_PACKAGE_REQUIRES "fftw3, libboost_date_time1_44_0, libboost_filesystem1_44_0, libboost_iostreams1_44_0, libboost_system1_44_0, libboost_thread1_44_0, zlib")
+		SET(CPACK_RPM_PACKAGE_REQUIRES "fftw3, libboost_date_time1_44_0, libboost_filesystem1_44_0, libboost_iostreams1_44_0, libboost_system1_44_0, libboost_thread1_44_0, libboost_timer1_44_0, zlib")
 		SET(CPACK_RPM_PACKAGE_DESCRIPTION "
 The nektar++ packages provide a spectral/hp element framework for the numerical
 solution of partial differential equations (PDEs). Demonstration codes are 
@@ -75,7 +75,7 @@ solvers in the nektar++-solvers package.")
         MESSAGE(STATUS "Generating Packaging for DEB")
         SET(CPACK_DEB_PACKAGE_URL "www.nektar.info")
         SET(CPACK_DEB_COMPONENT_INSTALL ON)
-        SET(CPACK_DEBIAN_PACKAGE_DEPENDS "libfftw3-3,libboost-date-time1.42.0,libboost-filesystem1.42.0,libboost-iostreams1.42.0,libboost-program-options1.42.0,libboost-system1.42.0,libboost-thread1.42.0,zlib1g")
+        SET(CPACK_DEBIAN_PACKAGE_DEPENDS "libfftw3-3,libboost-date-time1.42.0,libboost-filesystem1.42.0,libboost-iostreams1.42.0,libboost-program-options1.42.0,libboost-system1.42.0,libboost-thread1.42.0,libboost-timer1.42.0,zlib1g")
         SET(CPACK_DEBIAN_PACKAGE_DESCRIPTION 
             "${CPACK_PACKAGE_DESCRIPTION_SUMMARY}
         ${CPACK_PACKAGE_DESCRIPTION}")

cmake/ThirdPartyBoost.cmake

@@ -9,7 +9,7 @@
 #If the user has not set BOOST_ROOT, look in a couple common places first.
 MESSAGE(STATUS "Searching for Boost:")
 SET(NEEDED_BOOST_LIBS thread iostreams date_time filesystem system
-    program_options regex)
+    program_options regex timer)
 SET(Boost_DEBUG 0)
 SET(Boost_NO_BOOST_CMAKE ON)
 IF( BOOST_ROOT )
@@ -66,7 +66,7 @@ IF (THIRDPARTY_BUILD_BOOST)
     # Only build the libraries we need
     SET(BOOST_LIB_LIST --with-system --with-iostreams --with-filesystem
         --with-program_options --with-date_time --with-thread
-        --with-regex)
+        --with-regex --with-timer)
 
     IF (NOT WIN32)
         # We need -fPIC for 64-bit builds
@@ -180,11 +180,15 @@ IF (THIRDPARTY_BUILD_BOOST)
     SET(Boost_THREAD_LIBRARY boost_thread)
     SET(Boost_THREAD_LIBRARY_DEBUG boost_thread)
     SET(Boost_THREAD_LIBRARY_RELEASE boost_thread)
+    SET(Boost_TIMER_LIBRARY boost_timer)
+    SET(Boost_TIMER_LIBRARY_DEBUG boost_timer)
+    SET(Boost_TIMER_LIBRARY_RELEASE boost_timer)
+
     SET(Boost_INCLUDE_DIRS ${TPSRC}/dist/include)
     SET(Boost_CONFIG_INCLUDE_DIR ${TPINC})
     SET(Boost_LIBRARY_DIRS ${TPSRC}/dist/lib)
     SET(Boost_CONFIG_LIBRARY_DIR ${TPLIB})
-    SET(Boost_LIBRARIES boost_date_time boost_filesystem boost_iostreams boost_program_options boost_regex boost_system boost_thread)
+    SET(Boost_LIBRARIES boost_date_time boost_filesystem boost_iostreams boost_program_options boost_regex boost_system boost_thread boost_timer)
     LINK_DIRECTORIES(${Boost_LIBRARY_DIRS})
 
     STRING(REPLACE ";" ", " NEEDED_BOOST_LIBS_STRING "${NEEDED_BOOST_LIBS}")

library/Collections/CollectionOptimisation.cpp

@@ -55,10 +55,10 @@ CollectionOptimisation::CollectionOptimisation(
                     (pSession->DefinesCmdLineArgument("verbose")) &&
                     (pSession->GetComm()->GetRank() == 0);
 
-    m_setByXml = false;
-    m_autotune = false;
+    m_setByXml    = false;
+    m_autotune    = false;
     m_maxCollSize = 0;
-    m_defaultType = defaultType;
+    m_defaultType = defaultType == eNoImpType ? eNoCollection : defaultType;
 
     map<string, LibUtilities::ShapeType> elTypes;
     map<string, LibUtilities::ShapeType>::iterator it2;
@@ -73,7 +73,7 @@ CollectionOptimisation::CollectionOptimisation(
     // Set defaults for all element types.
     for (it2 = elTypes.begin(); it2 != elTypes.end(); ++it2)
     {
-        defaults[ElmtOrder(it2->second, -1)] = defaultType;
+        defaults[ElmtOrder(it2->second, -1)] = m_defaultType;
     }
 
     map<string, OperatorType> opTypes;
@@ -243,7 +243,6 @@ CollectionOptimisation::CollectionOptimisation(
     }
 }
 
-
 OperatorImpMap  CollectionOptimisation::GetOperatorImpMap(
         StdRegions::StdExpansionSharedPtr pExp)
 {
@@ -266,8 +265,8 @@ OperatorImpMap  CollectionOptimisation::GetOperatorImpMap(
             it2 = it->second.find(defSearch);
             if (it2 == it->second.end())
             {
-                cout << "shouldn't be here..." << endl;
-                impType = eStdMat;
+                // Shouldn't be able to reach here.
+                impType = eNoCollection;
             }
             else
             {

library/Demos/Collections/CMakeLists.txt

@@ -16,6 +16,10 @@ SET(CollectionTiming3DSource  CollectionTiming3D.cpp)
 ADD_NEKTAR_EXECUTABLE(CollectionTiming3D demos CollectionTiming3DSource)
 TARGET_LINK_LIBRARIES(CollectionTiming3D ${LinkLibraries})
 
+SET(CollectionTimingSource  CollectionTiming.cpp)
+ADD_NEKTAR_EXECUTABLE(CollectionTiming demos CollectionTimingSource)
+TARGET_LINK_LIBRARIES(CollectionTiming ${LinkLibraries})
+
 #ADD_NEKTAR_TEST(LinearAdvDiffReact2D_P7_Modes)
 
 IF (NEKTAR_USE_MPI)

library/Demos/Collections/CollectionTiming.cpp

+#include <cstdio>
+#include <cstdlib>
+#include <iomanip>
+
+#include <boost/timer/timer.hpp>
+
+#include <LibUtilities/Memory/NekMemoryManager.hpp>
+#include <LibUtilities/BasicUtils/SessionReader.h>
+#include <LibUtilities/Communication/Comm.h>
+#include <MultiRegions/ExpList3D.h>
+#include <Collections/Collection.h>
+#include <SpatialDomains/MeshGraph.h>
+
+using namespace Nektar;
+
+using boost::timer::cpu_timer;
+using boost::timer::cpu_times;
+using boost::timer::nanosecond_type;
+using boost::timer::format;
+
+MultiRegions::ExpListSharedPtr SetupExpList(
+    int                                  N,
+    LibUtilities::SessionReaderSharedPtr session,
+    SpatialDomains::MeshGraphSharedPtr   graph,
+    Collections::ImplementationType      impType)
+{
+    graph->SetExpansionsToPolyOrder(N);
+
+    MultiRegions::ExpListSharedPtr expList =
+        MemoryManager<MultiRegions::ExpList3D>::AllocateSharedPtr(
+            session, graph);
+
+    expList->CreateCollections(impType);
+
+    return expList;
+}
+
+void printOutput(int N, int Ntest, cpu_timer &timer, bool fmt)
+{
+    cpu_times times = timer.elapsed();
+    nanosecond_type total = times.user + times.system;
+
+    const double sec = 1000000000.0L;
+
+    // Normalize timings
+    double wall_sec  = times.wall / sec;
+    double total_sec = total / sec;
+
+    wall_sec /= Ntest;
+    total_sec /= Ntest;
+
+    if (fmt)
+    {
+        cout << setw(6)  << N-1
+             << setw(18) << wall_sec
+             << setw(18) << total_sec
+             << endl;
+    }
+    else
+    {
+        cout << "P = " << N-1 << ": "
+             << wall_sec << " (wall) "
+             << total_sec << " (total), "
+             << (100.0 * total_sec / wall_sec) << "% CPU"
+             << endl;
+    }
+}
+
+int main(int argc, char *argv[])
+{
+    LibUtilities::SessionReader::RegisterCmdLineFlag(
+        "data", "d", "Print in data format");
+    LibUtilities::SessionReaderSharedPtr session
+        = LibUtilities::SessionReader::CreateInstance(argc, argv);
+
+    bool fmt = session->DefinesCmdLineArgument("data");
+
+    MultiRegions::ExpListSharedPtr expList;
+
+    cpu_timer timer;
+
+    int Ntest, maxOrder;
+    session->LoadParameter("Ntest",    Ntest,    1000);
+    session->LoadParameter("maxOrder", maxOrder, 10);
+    maxOrder++;
+
+    string sl = fmt ? "# " : "";
+
+    // Read in mesh
+    SpatialDomains::MeshGraphSharedPtr graph =
+        SpatialDomains::MeshGraph::Read(session);
+
+    // BwdTrans operator
+    for (int imp = 1; imp < 5; ++imp)
+    {
+        Collections::ImplementationType impType =
+            (Collections::ImplementationType)imp;
+        cout << sl << "Using " << Collections::ImplementationTypeMap[imp]
+             << " Collection Implementation:" << endl;
+
+        cout << sl << "BwdTrans Op: Ntest = " << Ntest << endl;
+        for (int N = 2; N < maxOrder; ++N)
+        {
+            expList = SetupExpList(N, session, graph, impType);
+            Array<OneD, NekDouble> input (expList->GetNcoeffs());
+            Array<OneD, NekDouble> output(expList->GetNpoints());
+
+            timer.start();
+            for (int i = 0; i < Ntest; ++i)
+            {
+                expList->BwdTrans(input, output);
+            }
+            timer.stop();
+
+            printOutput(N, Ntest, timer, fmt);
+        }
+
+        cout << sl << "IProductWRTBase Op: Ntest = " << Ntest << endl;
+        for (int N = 2; N < maxOrder; ++N)
+        {
+            expList = SetupExpList(N, session, graph, impType);
+            Array<OneD, NekDouble> input (expList->GetNpoints());
+            Array<OneD, NekDouble> output(expList->GetNcoeffs());
+
+            timer.start();
+            for (int i = 0; i < Ntest; ++i)
+            {
+                expList->IProductWRTBase(input, output);
+            }
+            timer.stop();
+
+            printOutput(N, Ntest, timer, fmt);
+        }
+
+        cout << sl << "IProductWRTDerivBase Op: Ntest = " << Ntest << endl;
+        for (int N = 2; N < maxOrder; ++N)
+        {
+            expList = SetupExpList(N, session, graph, impType);
+            int nDim = expList->GetCoordim(0);
+            Array<OneD, Array<OneD, NekDouble> > input (nDim);
+            Array<OneD, NekDouble>               output(expList->GetNcoeffs());
+
+            for (int i = 0; i < nDim; ++i)
+            {
+                input[i] = Array<OneD, NekDouble>(expList->GetNpoints());
+            }
+
+            timer.start();
+            for (int i = 0; i < Ntest; ++i)
+            {
+                expList->IProductWRTDerivBase(input, output);
+            }
+            timer.stop();
+
+            printOutput(N, Ntest, timer, fmt);
+        }
+
+        cout << sl << "PhysDeriv Op: Ntest = " << Ntest << endl;
+        for (int N = 2; N < maxOrder; ++N)
+        {
+            expList = SetupExpList(N, session, graph, impType);
+            Array<OneD, NekDouble> input  (expList->GetNpoints());
+            Array<OneD, NekDouble> output0(expList->GetNpoints());
+            Array<OneD, NekDouble> output1(expList->GetNpoints());
+            Array<OneD, NekDouble> output2(expList->GetNpoints());
+
+            timer.start();
+            for (int i = 0; i < Ntest; ++i)
+            {
+                expList->PhysDeriv(input, output0, output1, output2);
+            }
+            timer.stop();
+
+            printOutput(N, Ntest, timer, fmt);
+        }
+    }
+}

library/LibUtilities/CMakeLists.txt

@@ -389,6 +389,7 @@ TARGET_LINK_LIBRARIES(LibUtilities LINK_PUBLIC
     ${Boost_PROGRAM_OPTIONS_LIBRARY}
     ${Boost_FILESYSTEM_LIBRARY}
     ${Boost_SYSTEM_LIBRARY}
+    ${Boost_TIMER_LIBRARY}
     debug ${ZLIB_LIBRARY_DEBUG} optimized ${ZLIB_LIBRARY_RELEASE}
 )
 

library/MultiRegions/ExpList.cpp

@@ -2684,7 +2684,6 @@ namespace Nektar
          */
         void ExpList::CreateCollections(Collections::ImplementationType ImpType)
         {
-            //return;
             map<LibUtilities::ShapeType,
                 vector<std::pair<LocalRegions::ExpansionSharedPtr,int> > > collections;
             map<LibUtilities::ShapeType,
@@ -2695,15 +2694,6 @@ namespace Nektar
             Collections::CollectionOptimisation colOpt(m_session, ImpType);
             ImpType = colOpt.GetDefaultImplementationType();
 
-            // If ImpType is not specified by default argument call
-            // then set ImpType to eStdMat for large collections or
-            // eSumFac for small
-            if(ImpType == Collections::eNoImpType)
-            {
-                ImpType = (m_exp->size() < 100 ? Collections::eSumFac
-                                               : Collections::eStdMat);
-            }
-
             bool autotuning = colOpt.IsUsingAutotuning();
             bool verbose    = (m_session->DefinesCmdLineArgument("verbose")) &&
                               (m_comm->GetRank() == 0);
@@ -2719,7 +2709,8 @@ namespace Nektar
             // Loop over expansions, and create collections for each element type
             for (int i = 0; i < m_exp->size(); ++i)
             {
-                collections[(*m_exp)[i]->DetShapeType()].push_back(std::pair<LocalRegions::ExpansionSharedPtr,int> ((*m_exp)[i],i));
+                collections[(*m_exp)[i]->DetShapeType()].push_back(
+                    std::pair<LocalRegions::ExpansionSharedPtr,int> ((*m_exp)[i],i));
             }
 
             for (it = collections.begin(); it != collections.end(); ++it)