Merge branch 'fix/fs_wait' into 'master'

wait until fs operations are propagated to all ranks

See merge request !741

CHANGELOG.md

@@ -65,6 +65,7 @@ v4.4.1
   (!821)
 - Fix performance issue with ExtractPhysToBndElmt (!796)
 - Fix available classes being listed multiple times (!817)
+- Fix overwriting and backup of chk/fld files on slow file systes (!741)
 
 **FieldConvert:**
 - Fix issue with field ordering in the interppointdatatofld module (!754)

library/LibUtilities/BasicUtils/FieldIO.cpp

@@ -424,10 +424,18 @@ std::string FieldIO::SetUpOutput(const std::string outname, bool perRank, bool b
                         "Filesystem error: " + string(e.what()));
         }
     }
-    // wait until rank 0 has backed up the old specPath
+
+    // wait until rank 0 has moved the old specPath and the changes
+    // have propagated through the filesystem
     if (backup)
     {
         m_comm->Block();
+        int exists = 1;
+        while (exists && perRank)
+        {
+            exists = fs::exists(specPath);
+            m_comm->AllReduce(exists, ReduceMax);
+        }
     }
 
     if (nprocs == 1)
@@ -446,7 +454,7 @@ std::string FieldIO::SetUpOutput(const std::string outname, bool perRank, bool b
     }
 
     // Remove any existing file which is in the way
-    if (m_comm->RemoveExistingFiles())
+    if (m_comm->RemoveExistingFiles() && !backup)
     {
         if (m_sharedFilesystem)
         {
@@ -480,7 +488,15 @@ std::string FieldIO::SetUpOutput(const std::string outname, bool perRank, bool b
             }
         }
 
+        // wait until rank 0 has removed specPath and the changes
+        // have propagated through the filesystem
         m_comm->Block();
+        int exists = 1;
+        while (exists && perRank)
+        {
+            exists = fs::exists(specPath);
+            m_comm->AllReduce(exists, ReduceMax);
+        }
     }
 
     if (root)