Merge branch 'production' of https://github.com/illinois-ceesd/grudge …

…into tensor-product-axis-tags

.github/dependabot.yml

@@ -0,0 +1,9 @@
+version: 2
+updates:
+    # Set update schedule for GitHub Actions
+    - package-ecosystem: "github-actions"
+      directory: "/"
+      schedule:
+          interval: "weekly"
+
+# vim: sw=4

.github/workflows/autopush.yml

@@ -9,7 +9,7 @@ jobs:
         name: Automatic push to gitlab.tiker.net
         runs-on: ubuntu-latest
         steps:
-        -   uses: actions/checkout@v3
+        -   uses: actions/checkout@v4
         -   run: |
                 mkdir ~/.ssh && echo -e "Host gitlab.tiker.net\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
                 eval $(ssh-agent) && echo "$GITLAB_AUTOPUSH_KEY" | ssh-add -

.github/workflows/ci.yml

@@ -12,9 +12,9 @@ jobs:
         name: Flake8
         runs-on: ubuntu-latest
         steps:
-        -   uses: actions/checkout@v3
+        -   uses: actions/checkout@v4
         -
-            uses: actions/setup-python@v4
+            uses: actions/setup-python@v5
             with:
                 # matches compat target in setup.py
                 python-version: '3.8'
@@ -27,7 +27,7 @@ jobs:
         name: Pylint
         runs-on: ubuntu-latest
         steps:
-        -   uses: actions/checkout@v3
+        -   uses: actions/checkout@v4
         -   name: "Main Script"
             run: |
                 echo "- matplotlib" >> .test-conda-env-py3.yml
@@ -46,7 +46,7 @@ jobs:
         name: Pytest on Py3
         runs-on: ubuntu-latest
         steps:
-        -   uses: actions/checkout@v3
+        -   uses: actions/checkout@v4
         -   name: "Main Script"
             run: |
                 curl -L -O https://gitlab.tiker.net/inducer/ci-support/raw/main/build-and-test-py-project-within-miniconda.sh
@@ -56,7 +56,7 @@ jobs:
         name: Examples on Py3
         runs-on: ubuntu-latest
         steps:
-        -   uses: actions/checkout@v3
+        -   uses: actions/checkout@v4
         -   name: "Main Script"
             run: |
                 curl -L -O https://tiker.net/ci-support-v0
@@ -78,7 +78,7 @@ jobs:
         name: Documentation
         runs-on: ubuntu-latest
         steps:
-        -   uses: actions/checkout@v3
+        -   uses: actions/checkout@v4
         -   name: "Main Script"
             run: |
                 echo "- matplotlib" >> .test-conda-env-py3.yml
@@ -100,7 +100,7 @@ jobs:
         name: Tests for downstream project ${{ matrix.downstream_project }}
         runs-on: ubuntu-latest
         steps:
-        -   uses: actions/checkout@v3
+        -   uses: actions/checkout@v4
         -   name: "Main Script"
             env:
                 DOWNSTREAM_PROJECT: ${{ matrix.downstream_project }}

grudge/discretization.py

@@ -1,5 +1,4 @@
 """
-
 .. autoclass:: DiscretizationTag
 
 .. currentmodule:: grudge
@@ -38,6 +37,8 @@
 
 from typing import (
     Sequence, Mapping, Optional, Union, List, Tuple, TYPE_CHECKING, Any)
+from typing import Mapping, Optional, Union, TYPE_CHECKING, Any
+from meshmode.discretization.poly_element import ModalGroupFactory
 
 from pytools import memoize_method, single_valued
 
@@ -181,11 +182,8 @@ def _normalize_discr_tag_to_group_factory(
     assert discr_tag_to_group_factory is not None
 
     # Modal discr should always come from the base discretization
-    if DISCR_TAG_MODAL not in discr_tag_to_group_factory:
-        discr_tag_to_group_factory[DISCR_TAG_MODAL] = \
-            _generate_modal_group_factory(
-                discr_tag_to_group_factory[DISCR_TAG_BASE]
-            )
+    if DISCR_TAG_MODAL not in discr_tag_to_group_factory and order is not None:
+        discr_tag_to_group_factory[DISCR_TAG_MODAL] = ModalGroupFactory(order)
 
     return discr_tag_to_group_factory
 
@@ -347,6 +345,71 @@ def is_management_rank(self):
             return self.mpi_communicator.Get_rank() \
                     == self.get_management_rank_index()
 
+    # {{{ distributed
+
+    def _set_up_distributed_communication(
+            self, vtag, mpi_communicator, array_context):
+        from_dd = DOFDesc(VolumeDomainTag(vtag), DISCR_TAG_BASE)
+
+        boundary_connections = {}
+
+        from meshmode.distributed import get_connected_partitions
+        connected_parts = get_connected_partitions(self._volume_discrs[vtag].mesh)
+
+        if connected_parts:
+            if mpi_communicator is None:
+                raise RuntimeError("must supply an MPI communicator when using a "
+                    "distributed mesh")
+
+            grp_factory = \
+                self.group_factory_for_discretization_tag(DISCR_TAG_BASE)
+
+            local_boundary_connections = {}
+            for i_remote_part in connected_parts:
+                local_boundary_connections[i_remote_part] = self.connection_from_dds(
+                        from_dd, from_dd.trace(BTAG_PARTITION(i_remote_part)))
+
+            from meshmode.distributed import MPIBoundaryCommSetupHelper
+            with MPIBoundaryCommSetupHelper(mpi_communicator, array_context,
+                    local_boundary_connections, grp_factory) as bdry_setup_helper:
+                while True:
+                    conns = bdry_setup_helper.complete_some()
+                    if not conns:
+                        break
+                    for i_remote_part, conn in conns.items():
+                        boundary_connections[i_remote_part] = conn
+
+        return boundary_connections
+
+    def distributed_boundary_swap_connection(self, dd):
+        """Provides a mapping from the base volume discretization
+        to the exterior boundary restriction on a parallel boundary
+        partition described by *dd*. This connection is used to
+        communicate across element boundaries in different parallel
+        partitions during distributed runs.
+
+        :arg dd: a :class:`~grudge.dof_desc.DOFDesc`, or a value
+            convertible to one. The domain tag must be a subclass
+            of :class:`grudge.dof_desc.BoundaryDomainTag` with an
+            associated :class:`meshmode.mesh.BTAG_PARTITION`
+            corresponding to a particular communication rank.
+        """
+        if dd.discretization_tag is not DISCR_TAG_BASE:
+            # FIXME
+            raise NotImplementedError(
+                "Distributed communication with discretization tag "
+                f"{dd.discretization_tag} is not implemented."
+            )
+
+        assert isinstance(dd.domain_tag, BoundaryDomainTag)
+        assert isinstance(dd.domain_tag.tag, BTAG_PARTITION)
+
+        vtag = dd.domain_tag.volume_tag
+
+        return self._dist_boundary_connections[vtag][dd.domain_tag.tag.part_id]
+
+    # }}}
+
     # {{{ discr_from_dd
 
     @memoize_method
@@ -421,15 +484,15 @@ def _base_to_geoderiv_connection(self, dd: DOFDesc) -> DiscretizationConnection:
         base_group_factory = self.group_factory_for_discretization_tag(
                 dd.discretization_tag)
 
-        def geo_group_factory(megrp, index):
+        def geo_group_factory(megrp):
             from modepy.shapes import Simplex
             from meshmode.discretization.poly_element import \
                     PolynomialEquidistantSimplexElementGroup
             if megrp.is_affine and issubclass(megrp._modepy_shape_cls, Simplex):
                 return PolynomialEquidistantSimplexElementGroup(
-                        megrp, order=0, index=index)
+                        megrp, order=0)
             else:
-                return base_group_factory(megrp, index)
+                return base_group_factory(megrp)
 
         from meshmode.discretization import Discretization
         geo_deriv_discr = Discretization(
@@ -931,7 +994,6 @@ def _generate_modal_group_factory(nodal_group_factory):
 
 # }}}
 
-
 # {{{ make_discretization_collection
 
 MeshOrDiscr = Union[Mesh, Discretization]

grudge/projection.py

@@ -86,7 +86,7 @@ def project(
 
     # }}}
 
-    if isinstance(vec, Number) or src_dofdesc == tgt_dofdesc:
+    if src_dofdesc == tgt_dofdesc:
         return vec
 
     return dcoll.connection_from_dds(src_dofdesc, tgt_dofdesc)(vec)

grudge/trace_pair.py

@@ -322,9 +322,10 @@ def interior_trace_pair(dcoll: DiscretizationCollection, vec) -> TracePair:
     return local_interior_trace_pair(dcoll, vec)
 
 
-def interior_trace_pairs(dcoll: DiscretizationCollection, vec, *,
-        comm_tag: Optional[Hashable] = None, tag: Hashable = None,
-        volume_dd: Optional[DOFDesc] = None) -> List[TracePair]:
+def interior_trace_pairs(
+        dcoll: DiscretizationCollection, vec, *,
+        comm_tag: Optional[Hashable] = None, volume_dd: Optional[DOFDesc] = None
+        ) -> List[TracePair]:
     r"""Return a :class:`list` of :class:`TracePair` objects
     defined on the interior faces of *dcoll* and any faces connected to a
     parallel boundary.
@@ -342,15 +343,6 @@ def interior_trace_pairs(dcoll: DiscretizationCollection, vec, *,
     :returns: a :class:`list` of :class:`TracePair` objects.
     """
 
-    if tag is not None:
-        warn("Specifying 'tag' is deprecated and will stop working in July of 2022. "
-                "Specify 'comm_tag' instead.", DeprecationWarning, stacklevel=2)
-        if comm_tag is not None:
-            raise TypeError("may only specify one of 'tag' and 'comm_tag'")
-        else:
-            comm_tag = tag
-    del tag
-
     if volume_dd is None:
         volume_dd = DD_VOLUME_ALL
 

test/test_modal_connections.py

@@ -23,12 +23,15 @@
 
 from grudge.array_context import PytestPyOpenCLArrayContextFactory
 from arraycontext import pytest_generate_tests_for_array_contexts
+
+from grudge.discretization import make_discretization_collection
 pytest_generate_tests = pytest_generate_tests_for_array_contexts(
         [PytestPyOpenCLArrayContextFactory])
 
 from meshmode.discretization.poly_element import (
     # Simplex group factories
     InterpolatoryQuadratureSimplexGroupFactory,
+    ModalGroupFactory,
     PolynomialWarpAndBlend2DRestrictingGroupFactory,
     PolynomialEquidistantSimplexGroupFactory,
     # Tensor product group factories
@@ -39,7 +42,6 @@
 from meshmode.dof_array import flat_norm
 import meshmode.mesh.generation as mgen
 
-from grudge import DiscretizationCollection
 import grudge.dof_desc as dof_desc
 
 import pytest
@@ -65,15 +67,16 @@ def f(x):
         group_cls=nodal_group_factory.mesh_group_class
     )
 
-    dcoll = DiscretizationCollection(
+    dcoll = make_discretization_collection(
         actx, mesh,
         discr_tag_to_group_factory={
-            dof_desc.DISCR_TAG_BASE: nodal_group_factory(order)
+            dof_desc.DISCR_TAG_BASE: nodal_group_factory(order),
+            dof_desc.DISCR_TAG_MODAL: ModalGroupFactory(order),
         }
     )
 
-    dd_modal = dof_desc.DD_VOLUME_MODAL
-    dd_volume = dof_desc.DD_VOLUME
+    dd_modal = dof_desc.DD_VOLUME_ALL_MODAL
+    dd_volume = dof_desc.DD_VOLUME_ALL
 
     x_nodal = actx.thaw(dcoll.discr_from_dd(dd_volume).nodes()[0])
     nodal_f = f(x_nodal)
@@ -105,17 +108,18 @@ def f(x):
         group_cls=QuadratureSimplexGroupFactory.mesh_group_class
     )
 
-    dcoll = DiscretizationCollection(
+    dcoll = make_discretization_collection(
         actx, mesh,
         discr_tag_to_group_factory={
             dof_desc.DISCR_TAG_BASE:
             PolynomialWarpAndBlend2DRestrictingGroupFactory(order),
-            dof_desc.DISCR_TAG_QUAD: QuadratureSimplexGroupFactory(2*order)
+            dof_desc.DISCR_TAG_QUAD: QuadratureSimplexGroupFactory(2*order),
+            dof_desc.DISCR_TAG_MODAL: ModalGroupFactory(order),
         }
     )
 
     # Use dof descriptors on the quadrature grid
-    dd_modal = dof_desc.DD_VOLUME_MODAL
+    dd_modal = dof_desc.DD_VOLUME_ALL_MODAL
     dd_quad = dof_desc.DOFDesc(dof_desc.DTAG_VOLUME_ALL,
                                dof_desc.DISCR_TAG_QUAD)