Fix bug with distributed meshes

include/mesh/FoamMesh.h

@@ -2,6 +2,7 @@
 
 #include "FoamRuntime.h"
 #include "Foam2MooseMeshGen.h"
+#include "libmesh/id_types.h"
 
 #include <MooseTypes.h>
 #include <MooseMesh.h>
@@ -42,7 +43,8 @@ class FoamMesh : public MooseMesh
   Foam::fvMesh & fvMesh() { return _foam_mesh; }
   // Create a MOOSE element from a Foam face, removing collinear points on edges.
   std::unique_ptr<Elem> createElement(Hippo::Foam2MooseMeshAdapter * mesh_adapter,
-                                      const Hippo::FoamFace & face);
+                                      const Hippo::FoamFace & face,
+                                      dof_id_type id);
 
   int64_t getPatchCount(int subdomain_id) const { return _patch_counts.at(subdomain_id); };
   int64_t getPatchOffset(int subdomain_id) const { return _patch_offsets.at(subdomain_id); };

src/mesh/FoamMesh.C

@@ -103,7 +103,9 @@ checkPointOnLine(const libMesh::Node * pt,
   return true;
 }
 std::unique_ptr<Elem>
-FoamMesh::createElement(Hippo::Foam2MooseMeshAdapter * mesh_adapter, const Hippo::FoamFace & face)
+FoamMesh::createElement(Hippo::Foam2MooseMeshAdapter * mesh_adapter,
+                        const Hippo::FoamFace & face,
+                        dof_id_type id)
 {
   // get all MOOSE nodes associated with points in Foam face
   std::vector<libMesh::Node *> points;
@@ -140,6 +142,8 @@ FoamMesh::createElement(Hippo::Foam2MooseMeshAdapter * mesh_adapter, const Hippo
   else
     elem = std::make_unique<libMesh::C0Polygon>(points.size());
 
+  elem->set_id(id);
+
   // set points, ranks and subdomain ids
   int count = 0;
   for (auto point : points)
@@ -170,7 +174,7 @@ FoamMesh::buildMesh()
   for (int32_t fc = 0; fc < mesh_adapter->nface(); ++fc)
   {
     auto face = mesh_adapter->face(fc);
-    _mesh->add_elem(createElement(mesh_adapter.get(), face));
+    _mesh->add_elem(createElement(mesh_adapter.get(), face, fc));
   }
 
   // patch id has the openfoam id that corresponds to the patch name

test/tests/mesh/polygonal/fluid-openfoam/system/decomposeParDict

@@ -0,0 +1,26 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     | Website:  https://openfoam.org
+    \\  /    A nd           | Version:  10
+     \\/     M anipulation  |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    format      ascii;
+    class       dictionary;
+    location    "system";
+    object      decomposeParDict;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+numberOfSubdomains 2;
+
+method          simple;
+
+simpleCoeffs
+{
+    n               (1 1 2);
+}
+
+// ************************************************************************* //

test/tests/mesh/polygonal/tests

@@ -2,18 +2,30 @@
   [poly]
     [setup]
       type = RunCommand
-      command = 'bash -c "cd fluid-openfoam && ./Allclean && fluentMeshToFoam constant/polygon.msh && polyDualMesh -overwrite 75"'
+      command = 'bash -c "cd fluid-openfoam && ./Allclean && fluentMeshToFoam constant/polygon.msh && polyDualMesh -overwrite 75 && decomposePar"'
     []
     [run]
       type = RunApp
       input = run.i
       prereq = poly/setup
     []
-
     [verify]
       type = PythonUnitTest
       input = test.py
       prereq = poly/run
     []
+    [run_distributed]
+      type = RunApp
+      input = run.i
+      prereq = poly/setup
+      min_parallel=2
+      max_parallel=2
+      cli_args='--distributed-mesh'
+    []
+    [verify_distributed]
+      type = PythonUnitTest
+      input = test.py
+      prereq = poly/run_distributed
+    []
   []
 []

test/tests/mesh/quadrilateral/fluid-openfoam/system/decomposeParDict

@@ -0,0 +1,26 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     | Website:  https://openfoam.org
+    \\  /    A nd           | Version:  10
+     \\/     M anipulation  |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    format      ascii;
+    class       dictionary;
+    location    "system";
+    object      decomposeParDict;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+numberOfSubdomains 2;
+
+method          simple;
+
+simpleCoeffs
+{
+    n               (1 1 2);
+}
+
+// ************************************************************************* //

test/tests/mesh/quadrilateral/tests

@@ -2,18 +2,30 @@
   [quad]
     [setup]
       type = RunCommand
-      command = 'bash -c "cd fluid-openfoam && ./Allclean && blockMesh"'
+      command = 'bash -c "cd fluid-openfoam && ./Allclean && blockMesh && decomposePar"'
     []
     [run]
       type = RunApp
       input = run.i
       prereq = quad/setup
     []
-
     [verify]
       type = PythonUnitTest
       input = test.py
       prereq = quad/run
     []
+    [run_distributed]
+      type = RunApp
+      input = run.i
+      prereq = quad/setup
+      min_parallel=2
+      max_parallel=2
+      cli_args='--distributed-mesh'
+    []
+    [verify_distributed]
+      type = PythonUnitTest
+      input = test.py
+      prereq = quad/run_distributed
+    []
   []
 []

test/tests/mesh/triangular/fluid-openfoam/system/decomposeParDict

@@ -0,0 +1,26 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     | Website:  https://openfoam.org
+    \\  /    A nd           | Version:  10
+     \\/     M anipulation  |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    format      ascii;
+    class       dictionary;
+    location    "system";
+    object      decomposeParDict;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+numberOfSubdomains 2;
+
+method          simple;
+
+simpleCoeffs
+{
+    n               (1 1 2);
+}
+
+// ************************************************************************* //

test/tests/mesh/triangular/tests

@@ -2,18 +2,30 @@
   [tri]
     [setup]
       type = RunCommand
-      command = 'bash -c "cd fluid-openfoam && ./Allclean && fluentMeshToFoam constant/triangular.msh"'
+      command = 'bash -c "cd fluid-openfoam && ./Allclean && fluentMeshToFoam constant/triangular.msh && decomposePar"'
     []
     [run]
       type = RunApp
       input = run.i
       prereq = tri/setup
     []
-
     [verify]
       type = PythonUnitTest
       input = test.py
       prereq = tri/run
     []
+    [run_distributed]
+      type = RunApp
+      input = run.i
+      prereq = tri/setup
+      min_parallel=2
+      max_parallel=2
+      cli_args='--distributed-mesh'
+    []
+    [verify_distributed]
+      type = PythonUnitTest
+      input = test.py
+      prereq = tri/run_distributed
+    []
   []
 []