Rethink GhostMode - "shared facet" hangs when a process does not have ghosts

[IgorBaratta]

GhostMode.none ignores ghost information and GhostMode.shared_facet requires at least one ghost per process. However the partioner interface allows for greater flexibility in partitioning and mesh distribution (currently limited by the ghost types).

The following code hangs with 2 processes (process 0 has no ghosts, and process 1 has 1 ghost). If both processes have ghosts (commented lines in partitioner) the code works.

import dolfinx
import numpy
import ufl
from mpi4py import MPI


comm = MPI.COMM_WORLD

if comm.rank == 0:
    cells = numpy.array([[0, 1, 2, 3], [2, 3, 4, 5]], dtype=numpy.int64)
    x = numpy.array([[0., 0.],
                    [0., 1.],
                    [1., 1.],
                    [1., 0.],
                    [2., 0.],
                    [2., 1.]], dtype=numpy.float64)
else:
    x = numpy.zeros((0, 3), dtype=numpy.float64)
    cells = numpy.zeros((0, 4), dtype=numpy.int64)


def partitioner(*args):
    if comm.rank == 0:
        # data = numpy.array([0, 1,  1, 0], dtype=numpy.int32)
        # ofssets = numpy.array([0, 2,  4], dtype=numpy.int32)
        data = numpy.array([0, 1,  1], dtype=numpy.int32)
        ofssets = numpy.array([0, 2,  3], dtype=numpy.int32)
        return dolfinx.cpp.graph.AdjacencyList_int32(data, ofssets)
    else:
        return dolfinx.cpp.graph.AdjacencyList_int32(numpy.array([],  dtype=numpy.int32))


domain = ufl.Mesh(ufl.VectorElement("Lagrange", "quadrilateral", 1))
ghost_mode = dolfinx.cpp.mesh.GhostMode.shared_facet

mesh = dolfinx.mesh.create_mesh(comm, cells, x, domain, ghost_mode, partitioner)

print(mesh.topology.index_map(2).num_ghosts)