PGAS

CMakeLists.txt

@@ -139,6 +139,7 @@ add_subdirectory(vem/proxy)
 add_subdirectory(ve/openmp)
 add_subdirectory(ve/opencl)
 add_subdirectory(ve/cuda)
+add_subdirectory(ve/pgas)
 
 add_subdirectory(filter/pprint)
 add_subdirectory(filter/bccon)

bridge/cxx/include/bhxx/BhArray.hpp

@@ -345,6 +345,16 @@ class BhArray : public BhArrayUnTypedCore {
      * @return The new array
      */
     BhArray<T> newAxis(int axis) const;
+
+    /** Create a new view with a pgas-enabled base array */
+    BhArray(Shape shape_, Stride stride_, size_t offset_, BhPGAS pgas)
+            : offset(offset_),
+              shape(shape_),
+              stride(std::move(stride_)),
+              base(make_base_ptr(T(0), shape_.prod(), std::move(pgas))) {
+        assert(shape.size() == stride.size());
+        assert(shape.prod() > 0);
+    }
 };
 
 /** Pretty printing the data of an array to a stream

bridge/cxx/include/bhxx/BhBase.hpp

@@ -26,6 +26,8 @@ If not, see <http://www.gnu.org/licenses/>.
 
 namespace bhxx {
 
+using BhPGAS = BhPGAS;
+
 /** The base underlying (multiple) arrays */
 class BhBase : public bh_base {
 public:
@@ -86,6 +88,13 @@ class BhBase : public bh_base {
         assert(dummy == T(0));
     }
 
+    template<typename T>
+    BhBase(T dummy, size_t nelem, BhPGAS pgas) : bh_base(nelem, bh_type_from_template<T>(), std::move(pgas)),
+                                                 m_own_memory(true) {
+        // The dummy is a dummy argument and should always be identical zero.
+        assert(dummy == T(0));
+    }
+
     /** Destructor */
     ~BhBase() {
         // All memory here should be handed over to the Runtime

bridge/pgas4py/pgas4py/__init__.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+"""
+================================
+pgas4py: bringing PGAS to Python
+================================
+"""
+from __future__ import absolute_import
+from pkg_resources import get_distribution, DistributionNotFound
+
+# Set the package version
+try:
+    __version__ = get_distribution(__name__).version
+except DistributionNotFound:
+    __version__ = "0.0.0"  # package is not installed

bridge/pgas4py/setup.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+This file is part of Bohrium and copyright (c) 2018 the Bohrium
+<http://www.bh107.org>
+
+Bohrium is free software: you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as
+published by the Free Software Foundation, either version 3
+of the License, or (at your option) any later version.
+
+Bohrium is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the
+GNU Lesser General Public License along with Bohrium.
+
+If not, see <http://www.gnu.org/licenses/>.
+"""
+
+from setuptools import setup, find_packages
+from setuptools.extension import Extension
+from setuptools.command.build_ext import build_ext as setup_build_ext
+import numbers
+import os
+
+
+def script_path(*paths):
+    prefix = os.path.abspath(os.path.dirname(__file__))
+    assert len(prefix) > 0
+    return os.path.join(prefix, *paths)
+
+
+class BuildExt(setup_build_ext):
+    """We delay the numpy and bohrium dependency to the build command.
+    Hopefully, PIP has installed them at this point."""
+
+    def run(self):
+        if not self.dry_run:
+            import numpy
+            import bohrium_api
+            for ext in self.extensions:
+                ext.include_dirs.extend([numpy.get_include(), bohrium_api.get_include()])
+        setup_build_ext.run(self)
+
+
+class DelayedVersion(numbers.Number):
+    """In order to delay the version evaluation that depend on `bohrium_api`, we use this class"""
+
+    def __str__(self):
+        import bohrium_api
+        return bohrium_api.__version__
+
+
+setup(
+    cmdclass={'build_ext': BuildExt},
+    name='pgas4py',
+    version=DelayedVersion(),
+    description='Bohrium PGAS Frontend',
+    long_description='Bohrium for Python <www.bh107.org>',
+    url='http://bh107.org',
+    author='The Bohrium Team',
+    author_email='[email protected]',
+    maintainer='Mads R. B. Kristensen',
+    maintainer_email='[email protected]',
+    platforms=['Linux', 'OSX'],
+
+    # Choose your license
+    license='GPL',
+
+    # See https://pypi.python.org/pypi?%3Aaction=list_classifiers
+    classifiers=[
+        # How mature is this project? Common values are
+        #   3 - Alpha
+        #   4 - Beta
+        #   5 - Production/Stable
+        'Development Status :: 4 - Beta',
+
+        # Indicate who your project is intended for
+        'Intended Audience :: Developers',
+        'Intended Audience :: Science/Research',
+
+        'Topic :: Scientific/Engineering',
+        'Topic :: Software Development',
+
+        # Pick your license as you wish (should match "license" above)
+        'License :: OSI Approved :: GNU General Public License (GPL)',
+
+        # Specify the Python versions you support here. In particular, ensure
+        # that you indicate whether you support Python 2, Python 3 or both.
+        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: C',
+    ],
+
+    # What does your project relate to?
+    keywords='Bohrium, bh107, Python, C, HPC, MPI, PGAS, CUDA, OpenCL, OpenMP',
+
+    # Dependencies
+    install_requires=['numpy>=1.7', 'bohrium_api'],
+    setup_requires=['numpy>=1.7', 'bohrium_api'],
+
+    # You can just specify the packages manually here if your project is
+    # simple. Or you can use find_packages().
+    packages=find_packages(exclude=['tests']),
+
+    # ext_modules=[
+    #     Extension(
+    #         name='_bh_api',
+    #         sources=[script_path('src', '_bh_api.c')],
+    #         depends=[script_path('src', '_bh_api.h')],
+    #         extra_compile_args=["-std=c99"],
+    #     ),
+    # ]
+)

bridge/pgas4py/src/_bh_api.c

@@ -0,0 +1,63 @@
+/*
+This file is part of Bohrium and copyright (c) 2012 the Bohrium
+team <http://www.bh107.org>.
+
+Bohrium is free software: you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as
+published by the Free Software Foundation, either version 3
+of the License, or (at your option) any later version.
+
+Bohrium is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the
+GNU Lesser General Public License along with Bohrium.
+
+If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <Python.h>
+#include <bohrium_api.h>
+
+#if PY_MAJOR_VERSION >= 3
+    #define NPY_PY3K
+#endif
+
+
+// The methods (functions) of this module
+static PyMethodDef _bh_apiMethods[] = {
+    {"flush", PyFlush,  METH_NOARGS, "Evaluate all delayed array operations"},
+    {NULL, NULL, 0, NULL}        /* Sentinel */
+};
+
+#if defined(NPY_PY3K)
+static struct PyModuleDef moduledef = {
+    PyModuleDef_HEAD_INIT,
+    "_bh_api",/* name of module */
+    NULL, /* module documentation, may be NULL */
+    -1,   /* size of per-interpreter state of the module or -1 if the module keeps state in global variables. */
+    _bh_apiMethods /* the methods of this module */
+};
+#endif
+
+#if defined(NPY_PY3K)
+    #define RETVAL m
+    PyMODINIT_FUNC PyInit__bh_api(void)
+#else
+    #define RETVAL
+    PyMODINIT_FUNC init_bh_api(void)
+#endif
+{
+    PyObject *m;
+#if defined(NPY_PY3K)
+    m = PyModule_Create(&moduledef);
+#else
+    m = Py_InitModule("_bh_api", _bh_apiMethods);
+#endif
+    if (m == NULL) {
+        return RETVAL;
+    }
+    return RETVAL;
+}

config.ini.in

@@ -14,6 +14,7 @@ cuda         = bcexp_gpu, bccon, node, cuda, openmp
 proxy_openmp = bcexp_cpu, bccon, proxy, node, openmp
 proxy_opencl = bcexp_cpu, bccon, proxy, node, opencl, openmp
 proxy_cuda   = bcexp_cpu, bccon, proxy, node, cuda, openmp
+pgas         = bcexp_cpu, bccon, node, pgas, openmp
 
 ############
 # Managers #
@@ -209,3 +210,38 @@ work_group_size_3dy = 2
 work_group_size_3dz = 2
 # Optimize instructions for GPU access (column-major)
 to_col_major = false
+
+###########
+# Engines #
+###########
+[pgas]
+impl = ${CMAKE_INSTALL_PREFIX}/${LIBDIR}/libbh_ve_pgas${CMAKE_SHARED_LIBRARY_SUFFIX}
+verbose = false
+# Profiling statistics
+prof = false
+prof_filename =
+# Write a Graphviz graph for each kernel
+graph = false
+# Directory for temporary files (e.g. /tmp/). Default: `boost::filesystem::temp_directory_path()`
+tmp_dir =
+# Directory for cache files (persistent between executions). Default: the empty string, which disable the cache
+cache_dir = ${BIN_KERNEL_CACHE_DIR}
+# Maximum number of cache files to keep in the cache dir (use -1 for infinity)
+cache_file_max = 50000
+# The command to execute the compiler where {OUT} is replaced with the binary file output, {IN} with the source file,
+# and {CONF_PATH} with the path to this config file
+compiler_cmd = "${VE_PGAS_COMPILER_CMD} ${VE_PGAS_COMPILER_FLG} ${VE_PGAS_COMPILER_INC} ${VE_PGAS_COMPILER_LIB} {IN} -o {OUT}"
+# List of extension methods
+libs = ${BH_PGAS_LIBS}
+# The pre-fuser to use ('none' or 'lossy')
+pre_fuser = lossy
+# List of instruction fuser/transformers
+fuser_list = greedy, collapse_redundant_axes
+# Number of edges in the fusion graph that makes the greedy fuser use the `reshapable_first` fuser instead
+greedy_threshold = 10000
+# *_as_var specifies whether to hard-code variables or have them as variables
+index_as_var = true
+strides_as_var = true
+const_as_var = true
+# Monolithic combines all blocks into one shared library rather than a block-nest per shared library
+monolithic = false

core/bh_main_memory.cpp

@@ -107,16 +107,22 @@ void main_mem_free(void *mem, uint64_t nbytes) {
 MallocCache malloc_cache(main_mem_malloc, main_mem_free, 0);
 }
 
-void bh_data_malloc(bh_base *base) {
+void bh_data_malloc(bh_base *base, bool pgas_malloc) {
     if (base == nullptr) return;
     if (base->getDataPtr() != nullptr) return;
-    base->resetDataPtr(malloc_cache.alloc(base->nbytes()));
+    if (base->pgas.enabled() and not pgas_malloc) {
+        throw std::runtime_error("bh_data_malloc(): will not allocate PGAS-enabled arrays when `pgas_malloc=false`");
+    }
+
+    // Notice, when PGAS isn't enabled: `base->pgas.localSize() == base->nelem()`
+    base->resetDataPtr(malloc_cache.alloc(base->pgas.localSize() * bh_type_size(base->dtype())));
 }
 
 void bh_data_free(bh_base *base) {
     if (base == nullptr) return;
     if (base->getDataPtr() == nullptr) return;
-    malloc_cache.free(base->nbytes(), base->getDataPtr());
+    // Notice, when PGAS isn't enabled: `base->pgas.localSize() == base->nelem()`
+    malloc_cache.free(base->pgas.localSize() * bh_type_size(base->dtype()), base->getDataPtr());
     base->resetDataPtr();
 }
 

filter/bcexp/expand_reduce1d.cpp

@@ -72,7 +72,7 @@ int Expander::expandReduce1d(BhIR &bhir, int pc, int thread_limit) {
     in.shape = {fold, elements / fold};
     in.stride = {in.stride[0] * elements / fold, in.stride[0]};
 
-    bh_view temp = createTemp(in.base->dtype(), elements / fold);
+    bh_view temp = createTemp(in.base->dtype(), elements / fold, in.base->pgas);
     inject(bhir, ++pc, opcode, temp, in, 0, bh_type::INT64);
     inject(bhir, ++pc, opcode, out, temp, 0, bh_type::INT64);
     inject(bhir, ++pc, BH_FREE, temp);

filter/bcexp/expander.cpp

@@ -102,11 +102,11 @@ size_t Expander::gc(void)
     return collected;
 }
 
-bh_base* Expander::createBase(bh_type type, int64_t nelem)
+bh_base * Expander::createBase(bh_type type, int64_t nelem, const BhPGAS &pgas)
 {
     bh_base* base = nullptr;
     try {
-        base = new bh_base(nelem, type);
+        base = new bh_base(nelem, type, pgas);
     } catch (std::bad_alloc& ba) {
         fprintf(stderr, "Expander::createBase(...) bh_base allocation failed.\n");
         throw std::runtime_error("Expander::createBase(...) bh_base allocation failed.\n");
@@ -118,14 +118,14 @@ bh_base* Expander::createBase(bh_type type, int64_t nelem)
 bh_view Expander::createTemp(bh_view& meta, bh_type type, int64_t nelem)
 {
     bh_view view = meta;
-    view.base = createBase(type, nelem);
+    view.base = createBase(type, nelem, meta.base->pgas);
     return view;
 }
 
-bh_view Expander::createTemp(bh_type type, int64_t nelem)
+bh_view Expander::createTemp(bh_type type, int64_t nelem, const BhPGAS &pgas)
 {
     bh_view view;
-    view.base = createBase(type, nelem);
+    view.base = createBase(type, nelem, pgas);
     view.start = 0;
     view.ndim = 1;
     view.shape.push_back(nelem);

filter/bcexp/expander.hpp

@@ -49,9 +49,9 @@ class Expander
      *
      * @return Pointer to the created base.
      */
-    bh_base* createBase(bh_type type, int64_t nelem);
+    bh_base *createBase(bh_type type, int64_t nelem, const BhPGAS &pgas);
     bh_view createTemp(bh_view& meta, bh_type type, int64_t nelem);
-    bh_view createTemp(bh_type type, int64_t nelem);
+    bh_view createTemp(bh_type type, int64_t nelem, const BhPGAS &pgas);
 
     /**
      *  Inject an instruction.