Merge branch 'main' into host_map_host_data

.github/workflows/fpga-ci.yml

@@ -16,7 +16,7 @@ jobs:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-ci') }}
     runs-on: [self-hosted, linux, intel-fpga, xilinx-fpga]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
         submodules: 'recursive'
     - name: Install dependencies

.github/workflows/general-ci.yml

@@ -18,11 +18,11 @@ jobs:
         simplify: [0,1,autoopt]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
         submodules: 'recursive'
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
@@ -55,7 +55,7 @@ jobs:
         else
             export DACE_optimizer_automatic_simplification=${{ matrix.simplify }}
         fi
-        pytest -n auto --cov-report=xml --cov=dace --tb=short -m "not gpu and not verilator and not tensorflow and not mkl and not sve and not papi and not mlir and not lapack and not fpga and not mpi and not rtl_hardware and not scalapack and not datainstrument"
+        pytest -n auto --cov-report=xml --cov=dace --tb=short -m "not gpu and not verilator and not tensorflow and not mkl and not sve and not papi and not mlir and not lapack and not fpga and not mpi and not rtl_hardware and not scalapack and not datainstrument and not long"
         ./codecov
 
     - name: Test OpenBLAS LAPACK

.github/workflows/gpu-ci.yml

@@ -19,7 +19,7 @@ jobs:
     if: "!contains(github.event.pull_request.labels.*.name, 'no-ci')"
     runs-on: [self-hosted, gpu]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
         submodules: 'recursive'
     - name: Install dependencies

.github/workflows/hardware_test.yml

@@ -4,7 +4,7 @@ jobs:
   test-rtl:
     runs-on: [self-hosted, linux, xilinx-fpga]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
         submodules: 'recursive'
     - name: Install dependencies

.github/workflows/heterogeneous-ci.yml

@@ -19,7 +19,7 @@ jobs:
     if: "!contains(github.event.pull_request.labels.*.name, 'no-ci')"
     runs-on: [self-hosted, linux]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
         submodules: 'recursive'
     - name: Install dependencies

.github/workflows/pyFV3-ci.yml

@@ -21,18 +21,18 @@ jobs:
               python-version: [3.11.7]
 
       steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
               repository: 'NOAA-GFDL/PyFV3'
               ref: 'ci/DaCe'
               submodules: 'recursive'
               path: 'pyFV3'
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with: 
             path: 'dace'
             submodules: 'recursive'
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
               python-version: ${{ matrix.python-version }}
       - name: Install library dependencies
@@ -53,11 +53,11 @@ jobs:
           cd pyFV3
           mkdir -p test_data
           cd test_data
-          wget https://portal.nccs.nasa.gov/datashare/astg/smt/pace-regression-data/8.1.3_c12_6ranks_standard.D_SW.tar.gz
+          wget --retry-connrefused https://portal.nccs.nasa.gov/datashare/astg/smt/pace-regression-data/8.1.3_c12_6ranks_standard.D_SW.tar.gz
           tar -xzvf 8.1.3_c12_6ranks_standard.D_SW.tar.gz
-          wget https://portal.nccs.nasa.gov/datashare/astg/smt/pace-regression-data/8.1.3_c12_6ranks_standard.RiemSolver3.tar.gz
+          wget --retry-connrefused https://portal.nccs.nasa.gov/datashare/astg/smt/pace-regression-data/8.1.3_c12_6ranks_standard.RiemSolver3.tar.gz
           tar -xzvf 8.1.3_c12_6ranks_standard.RiemSolver3.tar.gz
-          wget https://portal.nccs.nasa.gov/datashare/astg/smt/pace-regression-data/8.1.3_c12_6ranks_standard.Remapping.tar.gz
+          wget --retry-connrefused https://portal.nccs.nasa.gov/datashare/astg/smt/pace-regression-data/8.1.3_c12_6ranks_standard.Remapping.tar.gz
           tar -xzvf 8.1.3_c12_6ranks_standard.Remapping.tar.gz
           cd ../..
       # Clean up caches between run for stale un-expanded SDFG to trip the build system (NDSL side issue)

.github/workflows/verilator_compatibility.yml

@@ -17,14 +17,14 @@ jobs:
     steps:
       - name: trigger reason
         run: echo "Trigger Reason:" ${{ github.event.inputs.reason }}
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: checkout submodules
         run: git submodule update --init --recursive
       - name: install apt packages
         run: sudo apt-get update && sudo apt-get -y install git make autoconf g++ flex bison libfl2 libfl-dev
       - name: compile verilator
         run: git clone https://github.com/verilator/verilator.git && cd verilator && git fetch origin && if [ ! "${{ matrix.verilator_version }}" == "master" ]; then  git checkout v${{ matrix.verilator_version }}; fi && autoconf && ./configure && make -j2 && sudo make install
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.8'
           architecture: 'x64'

dace/cli/dacelab.py

@@ -2,11 +2,6 @@
 # Copyright 2019-2021 ETH Zurich and the DaCe authors. All rights reserved.
 
 import argparse
-import numpy
-import pickle
-import json
-
-import dace
 from dace.frontend.octave import parse
 from dace.sdfg.nodes import AccessNode
 

dace/cli/sdfg_diff.py

@@ -7,9 +7,12 @@
 import os
 import platform
 import tempfile
-from typing import Dict, Literal, Set, Tuple, Union
+from typing import Dict, Set, Tuple, Union
+try:
+    from typing import Literal
+except ImportError:
+    from typing_extensions import Literal
 
-import jinja2
 import dace
 from dace import memlet as mlt
 from dace.sdfg import nodes as nd
@@ -179,6 +182,11 @@ def main():
     diff_sets = _sdfg_diff(sdfg_A, sdfg_B, eq_strategy)
 
     if args.graphical:
+        try:
+            import jinja2
+        except (ImportError, ModuleNotFoundError):
+            raise ImportError('Graphical SDFG diff requires jinja2, please install by running `pip install jinja2`')
+
         basepath = os.path.join(os.path.dirname(os.path.realpath(dace.__file__)), 'viewer')
         template_loader = jinja2.FileSystemLoader(searchpath=os.path.join(basepath, 'templates'))
         template_env = jinja2.Environment(loader=template_loader)

dace/cli/sdfgcc.py

@@ -48,7 +48,7 @@ def main():
         sdfg = SDFGOptimizer(sdfg).optimize()
 
     # Compile SDFG
-    sdfg.compile(outpath)
+    sdfg.compile(outpath, return_program_handle=False)
 
     # Copying header file to optional path
     if outpath is not None:

dace/cli/sdfv.py

@@ -13,7 +13,6 @@
 
 import dace
 import tempfile
-import jinja2
 
 
 def partialclass(cls, *args, **kwds):
@@ -44,10 +43,19 @@ def view(sdfg: dace.SDFG, filename: Optional[Union[str, int]] = None, verbose: b
         ):
             fd, filename = tempfile.mkstemp(suffix='.sdfg')
             sdfg.save(filename)
-            os.system(f'code {filename}')
+            if platform.system() == 'Darwin':
+                # Special case for MacOS
+                os.system(f'open {filename}')
+            else:
+                os.system(f'code {filename}')
             os.close(fd)
             return
 
+    try:
+        import jinja2
+    except (ImportError, ModuleNotFoundError):
+        raise ImportError('SDFG.view() requires jinja2, please install by running `pip install jinja2`')
+
     if type(sdfg) is dace.SDFG:
         sdfg = dace.serialize.dumps(sdfg.to_json())
 

dace/codegen/compiled_sdfg.py

@@ -518,6 +518,9 @@ def _construct_args(self, kwargs) -> Tuple[Tuple[Any], Tuple[Any]]:
                     # Otherwise, None values are passed as null pointers below
                 elif isinstance(arg, ctypes._Pointer):
                     pass
+                elif isinstance(arg, str):
+                    # Cast to bytes
+                    arglist[i] = ctypes.c_char_p(arg.encode('utf-8'))
                 else:
                     raise TypeError(f'Passing an object (type {type(arg).__name__}) to an array in argument "{a}"')
             elif is_array and not is_dtArray:
@@ -550,6 +553,8 @@ def _construct_args(self, kwargs) -> Tuple[Tuple[Any], Tuple[Any]]:
                     pass
                 elif isinstance(arg, float) and atype.dtype.type == np.float64:
                     pass
+                elif isinstance(arg, bool) and atype.dtype.type == np.bool_:
+                    pass
                 elif (isinstance(arg, str) or arg is None) and atype.dtype == dtypes.string:
                     if arg is None:
                         arglist[i] = ctypes.c_char_p(None)
@@ -575,7 +580,7 @@ def _construct_args(self, kwargs) -> Tuple[Tuple[Any], Tuple[Any]]:
         arg_ctypes = tuple(at.dtype.as_ctypes() for at in argtypes)
 
         constants = self.sdfg.constants
-        callparams = tuple((actype(arg.get()) if isinstance(arg, symbolic.symbol) else arg, actype, atype, aname)
+        callparams = tuple((arg, actype, atype, aname)
                            for arg, actype, atype, aname in zip(arglist, arg_ctypes, argtypes, argnames)
                            if not (symbolic.issymbolic(arg) and (hasattr(arg, 'name') and arg.name in constants)))
 

dace/codegen/compiler.py

@@ -13,6 +13,7 @@
 import subprocess
 import re
 from typing import Any, Callable, Dict, List, Set, Tuple, TypeVar, Union
+import warnings
 
 import dace
 from dace.config import Config
@@ -57,6 +58,18 @@ def generate_program_folder(sdfg, code_objects: List[CodeObject], out_path: str,
         code_path = os.path.join(target_folder, basename)
         clean_code = code_object.clean_code
 
+        if Config.get_bool('compiler', 'format_code'):
+            config_file = Config.get('compiler', 'format_config_file')
+            if config_file is not None and config_file != "":
+                run_arg_list = ['clang-format', f"-style=file:{config_file}"]
+            else:
+                run_arg_list = ['clang-format']
+            result = subprocess.run(run_arg_list, input=clean_code, text=True, capture_output=True)
+            if result.returncode or result.stderr:
+                warnings.warn(f'clang-format failed to run: {result.stderr}')
+            else:
+                clean_code = result.stdout
+
         # Save the file only if it changed (keeps old timestamps and saves
         # build time)
         if not identical_file_exists(code_path, clean_code):
@@ -213,7 +226,7 @@ def configure_and_compile(program_folder, program_name=None, output_stream=None)
         # Clean CMake directory and try once more
         if Config.get_bool('debugprint'):
             print('Cleaning CMake build folder and retrying...')
-        shutil.rmtree(build_folder)
+        shutil.rmtree(build_folder, ignore_errors=True)
         os.makedirs(build_folder)
         try:
             _run_liveoutput(cmake_command, shell=True, cwd=build_folder, output_stream=output_stream)
@@ -260,7 +273,7 @@ def get_environment_flags(environments) -> Tuple[List[str], Set[str]]:
     """
     Returns the CMake environment and linkage flags associated with the
     given input environments/libraries.
-    
+
     :param environments: A list of ``@dace.library.environment``-decorated
                          classes.
     :return: A 2-tuple of (environment CMake flags, linkage CMake flags)

dace/codegen/cppunparse.py

@@ -349,6 +349,8 @@ def _Assign(self, t):
                             # if the veclen is greater than one, this should be defined with a vector data type
                             self.write("{}{} ".format(dace.dtypes._OCL_VECTOR_TYPES[inferred_type.type],
                                                       inferred_type.veclen))
+                        elif self.language == dace.dtypes.Language.OpenCL:
+                            self.write(dace.dtypes._OCL_TYPES[inferred_type.type] + " ")
                         else:
                             self.write(dace.dtypes._CTYPES[inferred_type.type] + " ")
                     else:
@@ -555,7 +557,11 @@ def _write_constant(self, value):
             if result.find("b'") >= 0:
                 self.write(result)
             else:
-                self.write(result.replace('\'', '\"'))
+                towrite = result
+                if result.startswith("'"):
+                    towrite = result[1:-1].replace('"', '\\"')
+                    towrite = f'"{towrite}"'
+                self.write(towrite)
 
     def _Constant(self, t):
         value = t.value
@@ -749,6 +755,8 @@ def _Num(self, t):
         # For complex values, use ``dtype_to_typeclass``
         if isinstance(t_n, complex):
             dtype = dtypes.dtype_to_typeclass(complex)
+            repr_n = f'{dtype}({t_n.real}, {t_n.imag})'
+
 
         # Handle large integer values
         if isinstance(t_n, int):
@@ -765,10 +773,8 @@ def _Num(self, t):
             elif bits >= 64:
                 warnings.warn(f'Value wider than 64 bits encountered in expression ({t_n}), emitting as-is')
 
-        if repr_n.endswith("j"):
-            self.write("%s(0, %s)" % (dtype, repr_n.replace("inf", INFSTR)[:-1]))
-        else:
-            self.write(repr_n.replace("inf", INFSTR))
+        repr_n = repr_n.replace("inf", INFSTR)
+        self.write(repr_n)
 
     def _List(self, t):
         raise NotImplementedError('Invalid C++')
@@ -1187,6 +1193,8 @@ def py2cpp(code, expr_semicolon=True, defined_symbols=None):
         return cppunparse(ast.parse(symbolic.symstr(code, cpp_mode=True)),
                           expr_semicolon,
                           defined_symbols=defined_symbols)
+    elif isinstance(code, int):
+        return str(code)
     elif code.__class__.__name__ == 'function':
         try:
             code_str = inspect.getsource(code)